diff --git a/modules/packages/rpm/metadata.go b/modules/packages/rpm/metadata.go index f4f78c2cab..d8ac7ea75f 100644 --- a/modules/packages/rpm/metadata.go +++ b/modules/packages/rpm/metadata.go @@ -46,10 +46,11 @@ type Package struct { } type VersionMetadata struct { - License string `json:"license,omitempty"` - ProjectURL string `json:"project_url,omitempty"` - Summary string `json:"summary,omitempty"` - Description string `json:"description,omitempty"` + License string `json:"license,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + Summary string `json:"summary,omitempty"` + Description string `json:"description,omitempty"` + Updates []*Update `json:"updates,omitempty"` } type FileMetadata struct { @@ -296,3 +297,43 @@ func getChangelogs(h *rpmutils.RpmHeader) []*Changelog { } return changelogs } + +type DateAttr struct { + Date string `xml:"date,attr" json:"date"` +} + +type Update struct { + From string `xml:"from,attr" json:"from"` + Status string `xml:"status,attr" json:"status"` + Type string `xml:"type,attr" json:"type"` + Version string `xml:"version,attr" json:"version"` + ID string `xml:"id" json:"id"` + Title string `xml:"title" json:"title"` + Severity string `xml:"severity" json:"severity"` + Description string `xml:"description" json:"description"` + Issued *DateAttr `xml:"issued" json:"issued"` + Updated *DateAttr `xml:"updated" json:"updated"` + References []*Reference `xml:"references>reference" json:"references"` + PkgList []*Collection `xml:"pkglist>collection" json:"pkg_list"` +} + +type Reference struct { + Href string `xml:"href,attr" json:"href"` + ID string `xml:"id,attr" json:"id"` + Title string `xml:"title,attr" json:"title"` + Type string `xml:"type,attr" json:"type"` +} + +type Collection struct { + Short string `xml:"short,attr" json:"short"` + Packages []*UpdatePackage `xml:"package" json:"packages"` +} + +type UpdatePackage struct { + Arch string `xml:"arch,attr" json:"arch"` + Name string `xml:"name,attr" json:"name"` + Release string `xml:"release,attr" json:"release"` + Src string `xml:"src,attr" json:"src"` + Version string `xml:"version,attr" json:"version"` + Filename string `xml:"filename" json:"filename"` +} diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 876d6aaa62..9283eb8a5e 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -473,6 +473,7 @@ func CommonRoutes() *web.Router { g.MatchPath("HEAD", "//repodata/", rpm.CheckRepositoryFileExistence) g.MatchPath("GET", "//repodata/", rpm.GetRepositoryFile) g.MatchPath("PUT", "//upload", reqPackageAccess(perm.AccessModeWrite), rpm.UploadPackageFile) + g.MatchPath("POST", "//package///errata", reqPackageAccess(perm.AccessModeWrite), rpm.UploadErrata) // this URL pattern is only used internally in the RPM index, it is generated by us, the filename part is not really used (can be anything) g.MatchPath("HEAD,GET", "//package////", rpm.DownloadPackageFile) g.MatchPath("HEAD,GET", "//package///", rpm.DownloadPackageFile) diff --git a/routers/api/packages/rpm/rpm.go b/routers/api/packages/rpm/rpm.go index 4447a0c3cf..51cedd2a9f 100644 --- a/routers/api/packages/rpm/rpm.go +++ b/routers/api/packages/rpm/rpm.go @@ -10,6 +10,7 @@ import ( "io" "net/http" "strings" + "time" "code.gitea.io/gitea/models/db" packages_model "code.gitea.io/gitea/models/packages" @@ -316,3 +317,146 @@ func DeletePackageFile(webctx *context.Context) { webctx.Status(http.StatusNoContent) } + +// UploadErrata handles uploading errata information for a package version +func UploadErrata(ctx *context.Context) { + name := ctx.PathParam("name") + version := ctx.PathParam("version") + group := ctx.PathParam("group") + + var updates []*rpm_module.Update + if err := json.NewDecoder(ctx.Req.Body).Decode(&updates); err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + + pv, err := packages_model.GetVersionByNameAndVersion(ctx, + ctx.Package.Owner.ID, + packages_model.TypeRpm, + name, + version, + ) + if err != nil { + if errors.Is(err, util.ErrNotExist) { + apiError(ctx, http.StatusNotFound, err) + } else { + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + var vm *rpm_module.VersionMetadata + if pv.MetadataJSON != "" { + if err := json.Unmarshal([]byte(pv.MetadataJSON), &vm); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + } else { + vm = &rpm_module.VersionMetadata{} + } + + now := time.Now().Format("2006-01-02 15:04:05") + for _, u := range updates { + if u == nil { + continue + } + + // Sanitize to remove nil elements from JSON payload + var cleanPkgList []*rpm_module.Collection + for _, coll := range u.PkgList { + if coll == nil { + continue + } + var cleanPackages []*rpm_module.UpdatePackage + for _, pkg := range coll.Packages { + if pkg == nil { + continue + } + cleanPackages = append(cleanPackages, pkg) + } + coll.Packages = cleanPackages + cleanPkgList = append(cleanPkgList, coll) + } + u.PkgList = cleanPkgList + + found := false + for i, existing := range vm.Updates { + if existing.ID == u.ID { + // Merge PkgList with deduplication + for _, newColl := range u.PkgList { + if newColl == nil { + continue + } + collFound := false + for j, existingColl := range existing.PkgList { + if existingColl.Short == newColl.Short { + // Merge packages + for _, newPkg := range newColl.Packages { + if newPkg == nil { + continue + } + pkgFound := false + for _, existingPkg := range existingColl.Packages { + if existingPkg.Name == newPkg.Name && + existingPkg.Version == newPkg.Version && + existingPkg.Release == newPkg.Release && + existingPkg.Arch == newPkg.Arch { + pkgFound = true + break + } + } + if !pkgFound { + vm.Updates[i].PkgList[j].Packages = append(vm.Updates[i].PkgList[j].Packages, newPkg) + } + } + collFound = true + break + } + } + if !collFound { + vm.Updates[i].PkgList = append(vm.Updates[i].PkgList, newColl) + } + } + vm.Updates[i].From = u.From + vm.Updates[i].Status = u.Status + vm.Updates[i].Type = u.Type + vm.Updates[i].Version = u.Version + vm.Updates[i].Title = u.Title + vm.Updates[i].Severity = u.Severity + vm.Updates[i].Description = u.Description + vm.Updates[i].References = u.References + vm.Updates[i].Updated = &rpm_module.DateAttr{Date: now} + found = true + break + } + } + if !found { + if u.Issued == nil { + u.Issued = &rpm_module.DateAttr{Date: now} + } + if u.Updated == nil { + u.Updated = &rpm_module.DateAttr{Date: now} + } + vm.Updates = append(vm.Updates, u) + } + } + + vmBytes, err := json.Marshal(vm) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + pv.MetadataJSON = string(vmBytes) + if err := packages_model.UpdateVersion(ctx, pv); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + if err := rpm_service.BuildSpecificRepositoryFiles(ctx, ctx.Package.Owner.ID, group); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.Status(http.StatusOK) +} diff --git a/services/packages/rpm/repository.go b/services/packages/rpm/repository.go index fbbf8d7dad..ddf306b698 100644 --- a/services/packages/rpm/repository.go +++ b/services/packages/rpm/repository.go @@ -13,6 +13,7 @@ import ( "errors" "fmt" "io" + "slices" "strings" "time" @@ -241,15 +242,22 @@ func BuildSpecificRepositoryFiles(ctx context.Context, ownerID int64, group stri return err } + data := []*repoData{primary, filelists, other} + + updates := collectUpdateInfoUpdates(pfs, cache) + if len(updates) > 0 { + updateInfo, err := buildUpdateInfo(ctx, pv, updates, group) + if err != nil { + return err + } + data = append(data, updateInfo) + } + return buildRepomd( ctx, pv, ownerID, - []*repoData{ - primary, - filelists, - other, - }, + data, group, ) } @@ -563,6 +571,93 @@ func buildOther(ctx context.Context, pv *packages_model.PackageVersion, pfs []*p }, group) } +func collectUpdateInfoUpdates(pfs []*packages_model.PackageFile, c packageCache) (updates []*rpm_module.Update) { + seenVersions := make(map[int64]bool) + for _, pf := range pfs { + pd := c[pf] + if pd.Version != nil && !seenVersions[pd.Version.ID] && pd.VersionMetadata.Updates != nil { + updates = append(updates, pd.VersionMetadata.Updates...) + seenVersions[pd.Version.ID] = true + } + } + return updates +} + +// buildUpdateInfo builds the updateinfo.xml file +func buildUpdateInfo(ctx context.Context, pv *packages_model.PackageVersion, updates []*rpm_module.Update, group string) (*repoData, error) { + // Group updates by ID to merge package lists + type updateKey struct { + ID string + } + updateMap := make(map[updateKey]*rpm_module.Update) + + for _, u := range updates { + key := updateKey{ID: u.ID} + if existing, ok := updateMap[key]; ok { + for _, newColl := range u.PkgList { + collFound := false + for j, existingColl := range existing.PkgList { + if existingColl.Short == newColl.Short { + for _, newPkg := range newColl.Packages { + pkgFound := false + for _, existingPkg := range existingColl.Packages { + if existingPkg.Name == newPkg.Name && + existingPkg.Version == newPkg.Version && + existingPkg.Release == newPkg.Release && + existingPkg.Arch == newPkg.Arch { + pkgFound = true + break + } + } + if !pkgFound { + existing.PkgList[j].Packages = append(existing.PkgList[j].Packages, newPkg) + } + } + collFound = true + break + } + } + if !collFound { + collCopy := *newColl + collCopy.Packages = append([]*rpm_module.UpdatePackage(nil), newColl.Packages...) + existing.PkgList = append(existing.PkgList, &collCopy) + } + } + } else { + // Create a shallow copy so we don't mutate the original cached pointer + uCopy := *u + // Deep copy PkgList and Collections to avoid mutating cache + // Note: References is shallow-copied, but safe as long as it remains immutable + uCopy.PkgList = make([]*rpm_module.Collection, len(u.PkgList)) + for i, coll := range u.PkgList { + collCopy := *coll + collCopy.Packages = append([]*rpm_module.UpdatePackage(nil), coll.Packages...) + uCopy.PkgList[i] = &collCopy + } + updateMap[key] = &uCopy + } + } + + var mergedUpdates []*rpm_module.Update + for _, u := range updateMap { + mergedUpdates = append(mergedUpdates, u) + } + slices.SortFunc(mergedUpdates, func(a, b *rpm_module.Update) int { + return strings.Compare(a.ID, b.ID) + }) + + type updateInfo struct { + XMLName xml.Name `xml:"updates"` + Xmlns string `xml:"xmlns,attr"` + Updates []*rpm_module.Update `xml:"update"` + } + + return addDataAsFileToRepo(ctx, pv, "updateinfo", &updateInfo{ + Xmlns: "http://linux.duke.edu/metadata/updateinfo", + Updates: mergedUpdates, + }, group) +} + // writtenCounter counts all written bytes type writtenCounter struct { written int64 diff --git a/tests/integration/api_packages_rpm_test.go b/tests/integration/api_packages_rpm_test.go index 61bd8ffc63..ecde21cc70 100644 --- a/tests/integration/api_packages_rpm_test.go +++ b/tests/integration/api_packages_rpm_test.go @@ -12,12 +12,14 @@ import ( "io" "net/http" "net/http/httptest" + "slices" "strings" "testing" "code.gitea.io/gitea/models/packages" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/json" rpm_module "code.gitea.io/gitea/modules/packages/rpm" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/tests" @@ -74,6 +76,15 @@ Mu0UFYgZ/bYnuvn/vz4wtCz8qMwsHUvP0PX3tbYFUctAPdrY6tiiDtcCddDECahx7SuVNP5dpmb5 content, err := io.ReadAll(zr) assert.NoError(t, err) + decodeGzipXML := func(t testing.TB, resp *httptest.ResponseRecorder, v any) { + t.Helper() + + zr, err := gzip.NewReader(resp.Body) + assert.NoError(t, err) + + assert.NoError(t, xml.NewDecoder(zr).Decode(v)) + } + rootURL := fmt.Sprintf("/api/packages/%s/rpm", user.Name) for _, group := range []string{"", "el9", "el9/stable"} { @@ -247,15 +258,6 @@ gpgkey=%sapi/packages/%s/rpm/repository.key`, assert.Contains(t, resp.Body.String(), "-----BEGIN PGP SIGNATURE-----") }) - decodeGzipXML := func(t testing.TB, resp *httptest.ResponseRecorder, v any) { - t.Helper() - - zr, err := gzip.NewReader(resp.Body) - assert.NoError(t, err) - - assert.NoError(t, xml.NewDecoder(zr).Decode(v)) - } - t.Run("primary.xml.gz", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -420,6 +422,328 @@ gpgkey=%sapi/packages/%s/rpm/repository.key`, }) }) + t.Run("Errata", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + type updateInfo struct { + XMLName xml.Name `xml:"updates"` + Xmlns string `xml:"xmlns,attr"` + Updates []*rpm_module.Update `xml:"update"` + } + + errataURL := fmt.Sprintf("%s/package/%s/%s/errata", groupURL, packageName, packageVersion) + + advisory := rpm_module.Update{ + From: "security@example.com", + Status: "stable", + Type: "security", + Version: "1.0", + ID: "CVE-2023-1234", + Title: "Test Security Update", + Severity: "Important", + Description: "This is a test security update.", + References: []*rpm_module.Reference{ + { + Href: "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-1234", + ID: "CVE-2023-1234", + Title: "CVE-2023-1234", + Type: "cve", + }, + }, + PkgList: []*rpm_module.Collection{ + { + Short: "el9", + Packages: []*rpm_module.UpdatePackage{ + { + Arch: packageArchitecture, + Name: packageName, + Release: "1", + Src: "gitea-test-1.0.2-1.src.rpm", + Version: "1.0.2", + Filename: fmt.Sprintf("%s-%s.%s.rpm", packageName, packageVersion, packageArchitecture), + }, + }, + }, + }, + } + + t.Run("Success", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + updates := []*rpm_module.Update{&advisory} + body, err := json.Marshal(updates) + assert.NoError(t, err) + + req := NewRequestWithBody(t, "POST", errataURL, bytes.NewReader(body)). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusOK) + + url := groupURL + "/repodata" + + // Check repomd.xml contains updateinfo + req = NewRequest(t, "GET", url+"/repomd.xml") + resp := MakeRequest(t, req, http.StatusOK) + + type testRepoData struct { + Type string `xml:"type,attr"` + } + var repomd struct { + Data []*testRepoData `xml:"data"` + } + err = xml.NewDecoder(resp.Body).Decode(&repomd) + require.NoError(t, err) + + found := slices.IndexFunc(repomd.Data, func(s *testRepoData) bool { + return s.Type == "updateinfo" + }) >= 0 + assert.True(t, found, "updateinfo not found in repomd.xml") + + // Now check updateinfo.xml.gz + req = NewRequest(t, "GET", url+"/updateinfo.xml.gz") + resp = MakeRequest(t, req, http.StatusOK) + + var result updateInfo + decodeGzipXML(t, resp, &result) + + assert.Equal(t, "http://linux.duke.edu/metadata/updateinfo", result.Xmlns) + assert.Len(t, result.Updates, 1) + assert.Equal(t, "CVE-2023-1234", result.Updates[0].ID) + assert.NotEmpty(t, result.Updates[0].Issued.Date) + assert.NotEmpty(t, result.Updates[0].Updated.Date) + }) + + t.Run("InvalidJSON", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequestWithBody(t, "POST", errataURL, strings.NewReader("invalid json")). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusBadRequest) + }) + + t.Run("NullElementsInJSON", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Send a payload with null inside arrays + payload := `[ + { + "id": "CVE-2023-5678", + "from": "security@example.com", + "status": "stable", + "type": "security", + "version": "1.0", + "title": "Test Null Elements", + "severity": "Important", + "description": "Test null elements", + "pkg_list": [ + null, + { + "short": "el9", + "packages": [ + null, + { + "arch": "x86_64", + "name": "gitea", + "release": "1", + "src": "gitea-1.0.0-1.src.rpm", + "version": "1.0.0", + "filename": "gitea-1.0.0-1.x86_64.rpm" + } + ] + } + ] + } + ]` + + req := NewRequestWithBody(t, "POST", errataURL, strings.NewReader(payload)). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusOK) + + // Verify it was stored correctly (skipping nulls) + url := groupURL + "/repodata" + req = NewRequest(t, "GET", url+"/updateinfo.xml.gz") + resp := MakeRequest(t, req, http.StatusOK) + + var result updateInfo + decodeGzipXML(t, resp, &result) + + // We need to find the new advisory CVE-2023-5678 + var newAdvisory *rpm_module.Update + for _, u := range result.Updates { + if u.ID == "CVE-2023-5678" { + newAdvisory = u + break + } + } + assert.NotNil(t, newAdvisory) + assert.Len(t, newAdvisory.PkgList, 1) + assert.Equal(t, "el9", newAdvisory.PkgList[0].Short) + assert.Len(t, newAdvisory.PkgList[0].Packages, 1) + assert.Equal(t, "gitea", newAdvisory.PkgList[0].Packages[0].Name) + }) + + t.Run("PackageNotFound", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + badURL := fmt.Sprintf("%s/package/%s/non-existent-version/errata", groupURL, packageName) + updates := []*rpm_module.Update{&advisory} + body, err := json.Marshal(updates) + assert.NoError(t, err) + + req := NewRequestWithBody(t, "POST", badURL, bytes.NewReader(body)). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusNotFound) + }) + + t.Run("MergeAdvisories", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Upload a second advisory with the same ID but a different package + advisory2 := advisory + advisory2.PkgList = []*rpm_module.Collection{ + { + Short: "el9", + Packages: []*rpm_module.UpdatePackage{ + { + Arch: packageArchitecture, + Name: "another-package", + Release: "1", + Src: "another-package-1.0.0-1.src.rpm", + Version: "1.0.0", + Filename: "another-package-1.0.0-1.x86_64.rpm", + }, + }, + }, + } + + updates := []*rpm_module.Update{&advisory2} + body, err := json.Marshal(updates) + assert.NoError(t, err) + + req := NewRequestWithBody(t, "POST", errataURL, bytes.NewReader(body)). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusOK) + + // Check updateinfo.xml.gz again + url := groupURL + "/repodata" + req = NewRequest(t, "GET", url+"/updateinfo.xml.gz") + resp := MakeRequest(t, req, http.StatusOK) + + var result updateInfo + decodeGzipXML(t, resp, &result) + + var targetUpdate *rpm_module.Update + for _, u := range result.Updates { + if u.ID == "CVE-2023-1234" { + targetUpdate = u + break + } + } + assert.NotNil(t, targetUpdate) + // Verify that package lists are merged into the same collection + assert.Len(t, targetUpdate.PkgList, 1) + assert.Len(t, targetUpdate.PkgList[0].Packages, 2) + assert.Equal(t, "another-package", result.Updates[0].PkgList[0].Packages[1].Name) + }) + + t.Run("NewCollection", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Upload a third advisory with the same ID but a different collection + advisory3 := advisory + advisory3.PkgList = []*rpm_module.Collection{ + { + Short: "el8", + Packages: []*rpm_module.UpdatePackage{ + { + Arch: packageArchitecture, + Name: packageName, + Release: "1", + Src: "gitea-test-1.0.2-1.src.rpm", + Version: "1.0.2", + Filename: fmt.Sprintf("%s-%s.%s.rpm", packageName, packageVersion, packageArchitecture), + }, + }, + }, + } + + updates := []*rpm_module.Update{&advisory3} + body, err := json.Marshal(updates) + assert.NoError(t, err) + + req := NewRequestWithBody(t, "POST", errataURL, bytes.NewReader(body)). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusOK) + + // Check updateinfo.xml.gz again + url := groupURL + "/repodata" + req = NewRequest(t, "GET", url+"/updateinfo.xml.gz") + resp := MakeRequest(t, req, http.StatusOK) + + var result updateInfo + decodeGzipXML(t, resp, &result) + + var targetUpdate *rpm_module.Update + for _, u := range result.Updates { + if u.ID == "CVE-2023-1234" { + targetUpdate = u + break + } + } + assert.NotNil(t, targetUpdate) + // Verify that we now have 2 collections + assert.Len(t, targetUpdate.PkgList, 2) + // We need to be careful with order, map iteration is random + // Let's check both exist + shorts := []string{targetUpdate.PkgList[0].Short, targetUpdate.PkgList[1].Short} + assert.Contains(t, shorts, "el9") + assert.Contains(t, shorts, "el8") + }) + + t.Run("Idempotency", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + updates := []*rpm_module.Update{&advisory} + body, err := json.Marshal(updates) + assert.NoError(t, err) + + // Post twice + req := NewRequestWithBody(t, "POST", errataURL, bytes.NewReader(body)). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusOK) + + req = NewRequestWithBody(t, "POST", errataURL, bytes.NewReader(body)). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusOK) + + // Check updateinfo.xml.gz + url := groupURL + "/repodata" + req = NewRequest(t, "GET", url+"/updateinfo.xml.gz") + resp := MakeRequest(t, req, http.StatusOK) + + var result updateInfo + decodeGzipXML(t, resp, &result) + + var targetUpdate *rpm_module.Update + for _, u := range result.Updates { + if u.ID == "CVE-2023-1234" { + targetUpdate = u + break + } + } + assert.NotNil(t, targetUpdate) + assert.Len(t, targetUpdate.PkgList, 2) + + var el9Coll *rpm_module.Collection + for _, coll := range targetUpdate.PkgList { + if coll.Short == "el9" { + el9Coll = coll + break + } + } + assert.NotNil(t, el9Coll) + assert.Len(t, el9Coll.Packages, 2) + }) + }) + t.Run("Delete", func(t *testing.T) { defer tests.PrintCurrentTest(t)()