From 51b93d1d2772cb360dd0149fa85e8297f5e0f87f Mon Sep 17 00:00:00 2001 From: silverwind Date: Thu, 1 Oct 2026 09:50:47 +0200 Subject: [PATCH] enhance(packages/npm): improve npm client compatibility (#39434) Aligns the npm registry with what npm, pnpm and yarn expect: 1. Raise the publish body cap from https://github.com/go-gitea/gitea/pull/37890 to 256 MiB like npmjs, larger bodies get 413 2. Pick the tarball attachment by name, `npm publish --provenance` failed at random 3. Store and serve `libc`, so mismatched glibc/musl optional binaries are skipped 4. Treat root `*.gyp` files as an install script, like npm does 5. Always serve a `latest` dist-tag, yarn and pnpm fail without it 6. Take top-level metadata from `latest` and drop the per-version readme 7. Serve tarballs at the npmjs path `//-/`, former URLs keep working 8. Add ETag revalidation for metadata, `npm ping` and `npm whoami` Tested with npm 12.1, pnpm 12.4, yarn 1.22 and yarn 4.18. --------- Co-authored-by: wxiaoguang --- modules/json/jsonv2.go | 4 + modules/packages/npm/creator.go | 74 +++------ modules/packages/npm/creator_test.go | 78 +++------ modules/packages/npm/metadata.go | 1 + routers/api/packages/api.go | 5 +- routers/api/packages/npm/api.go | 21 ++- routers/api/packages/npm/api_test.go | 17 +- routers/api/packages/npm/npm.go | 180 ++++++++++----------- tests/integration/api_packages_npm_test.go | 41 ++--- 9 files changed, 195 insertions(+), 226 deletions(-) diff --git a/modules/json/jsonv2.go b/modules/json/jsonv2.go index 01246ca0f8b..08762f7e337 100644 --- a/modules/json/jsonv2.go +++ b/modules/json/jsonv2.go @@ -45,6 +45,10 @@ func MarshalKeepOptionalEmpty(v any) ([]byte, error) { return jsonv2.Marshal(v, jsonV2.marshalKeepOptionalEmptyOptions) } +func MarshalDeterministic(v any) ([]byte, error) { + return jsonv2.Marshal(v, jsonV2.marshalOptions, jsonv2.Deterministic(true)) +} + func (j *JSONv2) Marshal(v any) ([]byte, error) { return jsonv2.Marshal(v, j.marshalOptions) } diff --git a/modules/packages/npm/creator.go b/modules/packages/npm/creator.go index c286c8d7543..6a40f876a50 100644 --- a/modules/packages/npm/creator.go +++ b/modules/packages/npm/creator.go @@ -121,6 +121,7 @@ type PackageMetadataVersion struct { Engines map[string]string `json:"engines,omitempty"` CPU []string `json:"cpu,omitempty"` OS []string `json:"os,omitempty"` + Libc []string `json:"libc,omitempty"` Directories map[string]string `json:"directories,omitempty"` Funding any `json:"funding,omitempty"` AcceptDependencies map[string]string `json:"acceptDependencies,omitempty"` @@ -129,12 +130,9 @@ type PackageMetadataVersion struct { // PackageDistribution https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version type PackageDistribution struct { - Integrity string `json:"integrity"` - Shasum string `json:"shasum"` - Tarball string `json:"tarball"` - FileCount int `json:"fileCount,omitempty"` - UnpackedSize int `json:"unpackedSize,omitempty"` - NpmSignature string `json:"npm-signature,omitempty"` + Integrity string `json:"integrity"` + Shasum string `json:"shasum"` + Tarball string `json:"tarball"` } type PackageSearch struct { @@ -226,7 +224,7 @@ func (r *Repository) UnmarshalJSON(data []byte) error { } // Bin maps command names to executable files. npm also allows a single string, -// in which case the command is named after the package (resolved in ParsePackage). +// in which case the command is named after the package (resolved in parseUploadPackage). type Bin map[string]string // UnmarshalJSON is needed because the bin field can be a string or an object. @@ -264,7 +262,7 @@ type packageUpload struct { // is non-nil on success; a body without `_attachments` is a deprecate request, // otherwise it is a "publish". func ParseUpload(r io.Reader) (*Package, *PackageDeprecation, error) { - body, err := io.ReadAll(io.LimitReader(r, 10*1024*1024)) + body, err := io.ReadAll(r) if err != nil { return nil, nil, err } @@ -280,16 +278,6 @@ func ParseUpload(r io.Reader) (*Package, *PackageDeprecation, error) { return p, nil, err } -// ParsePackage parses a npm publish PUT body. Bodies without `_attachments` -// surface as ErrInvalidAttachment once name/version validation has passed. -func ParsePackage(r io.Reader) (*Package, error) { - var upload packageUpload - if err := json.NewDecoder(r).Decode(&upload); err != nil { - return nil, err - } - return parseUploadPackage(&upload) -} - // parseUploadPackage builds a Package from a decoded publish body. func parseUploadPackage(upload *packageUpload) (*Package, error) { for _, meta := range upload.Versions { @@ -343,6 +331,7 @@ func parseUploadPackage(upload *packageUpload) (*Package, error) { Engines: meta.Engines, CPU: meta.CPU, OS: meta.OS, + Libc: meta.Libc, Directories: meta.Directories, Funding: meta.Funding, AcceptDependencies: meta.AcceptDependencies, @@ -356,12 +345,12 @@ func parseUploadPackage(upload *packageUpload) (*Package, error) { p.Filename = strings.ToLower(fmt.Sprintf("%s-%s.tgz", name, p.Version)) - attachment := func() *PackageAttachment { + attachment := upload.Attachments[meta.Name+"-"+meta.Version+".tgz"] // not the sigstore bundle of `npm publish --provenance` + if attachment == nil && len(upload.Attachments) == 1 { for _, a := range upload.Attachments { - return a + attachment = a } - return nil - }() + } if attachment == nil || len(attachment.Data) == 0 { return nil, ErrInvalidAttachment } @@ -393,8 +382,6 @@ func parseUploadPackage(upload *packageUpload) (*Package, error) { return nil, ErrInvalidIntegrity } - // Derive _hasShrinkwrap and hasInstallScript from the tarball; the - // packument can lie about either. p.Metadata.HasShrinkwrap, p.Metadata.HasInstallScript = inspectTarball(data) return p, nil @@ -410,11 +397,7 @@ const maxNpmTarballScanBytes = int64(32 * 1024 * 1024) // 32 MiB // maxNpmPackageJSONBytes caps the package.json bytes decoded from the tarball. const maxNpmPackageJSONBytes = int64(1 * 1024 * 1024) // 1 MiB -// inspectTarball reports hasShrinkwrap (presence of package/npm-shrinkwrap.json) -// and hasInstallScript (package/package.json declares any of preinstall, -// install, postinstall). Both must be derived server-side because the client -// can lie in the packument. Any read/decode error yields (false, false) so a -// malformed archive does not block publishing. +// inspectTarball trusts the tarball over the client's packument, read errors yield zero values to not block publishing func inspectTarball(data []byte) (hasShrinkwrap, hasInstallScript bool) { gr, err := gzip.NewReader(bytes.NewReader(data)) if err != nil { @@ -422,11 +405,16 @@ func inspectTarball(data []byte) (hasShrinkwrap, hasInstallScript bool) { } defer gr.Close() + var hasGypFile bool + var pkg struct { + Scripts map[string]string `json:"scripts"` + Gypfile any `json:"gypfile"` + } tr := tar.NewReader(io.LimitReader(gr, maxNpmTarballScanBytes)) for { hdr, err := tr.Next() if err != nil { - return hasShrinkwrap, hasInstallScript + break } // npm pack puts files under a single root directory (usually "package/"). name := strings.TrimPrefix(hdr.Name, "./") @@ -436,30 +424,14 @@ func inspectTarball(data []byte) (hasShrinkwrap, hasInstallScript bool) { switch { case strings.HasSuffix(name, "/npm-shrinkwrap.json"): hasShrinkwrap = true + case strings.HasSuffix(name, ".gyp"): + hasGypFile = true case strings.HasSuffix(name, "/package.json"): - hasInstallScript = tarballDeclaresInstallScript(tr) - } - if hasShrinkwrap && hasInstallScript { - return hasShrinkwrap, hasInstallScript + _ = json.NewDecoder(io.LimitReader(tr, maxNpmPackageJSONBytes)).Decode(&pkg) } } -} - -// tarballDeclaresInstallScript reports whether a package.json declares any -// of preinstall, install, postinstall. -func tarballDeclaresInstallScript(r io.Reader) bool { - var pkg struct { - Scripts map[string]string `json:"scripts"` - } - if err := json.NewDecoder(io.LimitReader(r, maxNpmPackageJSONBytes)).Decode(&pkg); err != nil { - return false - } - for _, name := range []string{"preinstall", "install", "postinstall"} { - if strings.TrimSpace(pkg.Scripts[name]) != "" { - return true - } - } - return false + // npm publish adds "install": "node-gyp rebuild" for a root *.gyp file to the manifest, but not to the tarball + return hasShrinkwrap, strings.TrimSpace(pkg.Scripts["preinstall"]+pkg.Scripts["install"]+pkg.Scripts["postinstall"]) != "" || hasGypFile && pkg.Gypfile != false } func validateName(name string) bool { diff --git a/modules/packages/npm/creator_test.go b/modules/packages/npm/creator_test.go index 154557512cc..983ad032b37 100644 --- a/modules/packages/npm/creator_test.go +++ b/modules/packages/npm/creator_test.go @@ -41,21 +41,20 @@ func TestParsePackage(t *testing.T) { integrity := "sha512-" + base64Sha512(dataBytes) t.Run("InvalidUpload", func(t *testing.T) { - p, err := ParsePackage(bytes.NewReader([]byte{0})) + p, _, err := ParseUpload(bytes.NewReader([]byte{0})) assert.Nil(t, p) assert.Error(t, err) }) t.Run("InvalidUploadNoData", func(t *testing.T) { - b, _ := json.Marshal(packageUpload{}) - p, err := ParsePackage(bytes.NewReader(b)) + p, err := parseUploadPackage(&packageUpload{}) assert.Nil(t, p) assert.ErrorIs(t, err, ErrInvalidPackage) }) t.Run("InvalidPackageName", func(t *testing.T) { test := func(t *testing.T, name string) { - b, _ := json.Marshal(packageUpload{ + p, err := parseUploadPackage(&packageUpload{ PackageMetadata: PackageMetadata{ ID: name, Name: name, @@ -66,8 +65,6 @@ func TestParsePackage(t *testing.T) { }, }, }) - - p, err := ParsePackage(bytes.NewReader(b)) assert.Nil(t, p) assert.ErrorIs(t, err, ErrInvalidPackageName) } @@ -94,7 +91,7 @@ func TestParsePackage(t *testing.T) { t.Run("ValidPackageName", func(t *testing.T) { test := func(t *testing.T, name string) { - b, _ := json.Marshal(packageUpload{ + p, err := parseUploadPackage(&packageUpload{ PackageMetadata: PackageMetadata{ ID: name, Name: name, @@ -105,8 +102,6 @@ func TestParsePackage(t *testing.T) { }, }, }) - - p, err := ParsePackage(bytes.NewReader(b)) assert.Nil(t, p) assert.ErrorIs(t, err, ErrInvalidPackageVersion) } @@ -125,7 +120,7 @@ func TestParsePackage(t *testing.T) { t.Run("InvalidPackageVersion", func(t *testing.T) { version := "first-version" - b, _ := json.Marshal(packageUpload{ + p, err := parseUploadPackage(&packageUpload{ PackageMetadata: PackageMetadata{ ID: packageFullName, Name: packageFullName, @@ -137,8 +132,6 @@ func TestParsePackage(t *testing.T) { }, }, }) - - p, err := ParsePackage(bytes.NewReader(b)) assert.Nil(t, p) assert.ErrorIs(t, err, ErrInvalidPackageVersion) }) @@ -160,7 +153,7 @@ func TestParsePackage(t *testing.T) { }, }) - p, err := ParsePackage(bytes.NewReader(b)) + p, _, err := ParseUpload(bytes.NewReader(b)) assert.Nil(t, p) assert.ErrorIs(t, err, ErrInvalidAttachment) }) @@ -185,7 +178,7 @@ func TestParsePackage(t *testing.T) { }, }) - p, err := ParsePackage(bytes.NewReader(b)) + p, _, err := ParseUpload(bytes.NewReader(b)) assert.Nil(t, p) assert.ErrorIs(t, err, ErrInvalidAttachment) }) @@ -213,7 +206,7 @@ func TestParsePackage(t *testing.T) { }, }) - p, err := ParsePackage(bytes.NewReader(b)) + p, _, err := ParseUpload(bytes.NewReader(b)) assert.Nil(t, p) assert.ErrorIs(t, err, ErrInvalidIntegrity) }) @@ -241,7 +234,7 @@ func TestParsePackage(t *testing.T) { }, }) - p, err := ParsePackage(bytes.NewReader(b)) + p, _, err := ParseUpload(bytes.NewReader(b)) assert.Nil(t, p) assert.ErrorIs(t, err, ErrInvalidIntegrity) }) @@ -281,10 +274,13 @@ func TestParsePackage(t *testing.T) { filename: { Data: data, }, + packageFullName + "-" + packageVersion + ".sigstore": { + Data: "{}", + }, }, }) - p, err := ParsePackage(bytes.NewReader(b)) + p, _, err := ParseUpload(bytes.NewReader(b)) assert.NotNil(t, p) assert.NoError(t, err) @@ -329,7 +325,7 @@ func TestParsePackage(t *testing.T) { } } }` - p, err := ParsePackage(strings.NewReader(packageJSON)) + p, _, err := ParseUpload(strings.NewReader(packageJSON)) require.NoError(t, err) require.Equal(t, "MIT", string(p.Metadata.License)) }) @@ -354,7 +350,7 @@ func TestParsePackage(t *testing.T) { } } }` - p, err := ParsePackage(strings.NewReader(packageJSON)) + p, _, err := ParseUpload(strings.NewReader(packageJSON)) require.NoError(t, err) require.Equal(t, "https://gitea.io/gitea/test.git", p.Metadata.Repository.URL) // a string bin is named after the package @@ -426,6 +422,15 @@ func TestInspectTarball(t *testing.T) { // npm pack sometimes emits "./package/..." entries. wantShrinkwrap: true, }, + { + name: "gyp file implies node-gyp install", + files: map[string]string{"package/binding.gyp": "{}"}, + wantInstaller: true, + }, + { + name: "gypfile false disables gyp install", + files: map[string]string{"package/binding.gyp": "{}", "package/package.json": `{"gypfile":false}`}, + }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -460,41 +465,6 @@ func TestParseUpload(t *testing.T) { require.NotNil(t, dep) assert.Equal(t, map[string]string{"1.0.0": "gone", "1.0.1": ""}, dep.Versions) }) - - t.Run("dispatches publish when _attachments present", func(t *testing.T) { - // Reuse a minimal tarball with a package.json. - data := buildTarball(map[string]string{"package/package.json": `{}`}) - integrity := "sha512-" + base64Sha512(data) - body := fmt.Sprintf( - `{"name":%q,"versions":{"1.0.0":{"name":%q,"version":"1.0.0","dist":{"integrity":%q}}},"_attachments":{"x.tgz":{"data":%q}}}`, - pkg, pkg, integrity, base64.StdEncoding.EncodeToString(data), - ) - p, dep, err := ParseUpload(strings.NewReader(body)) - require.NoError(t, err) - assert.Nil(t, dep) - require.NotNil(t, p) - assert.Equal(t, pkg, p.Name) - }) - - t.Run("publish whose readme mentions deprecated is not misrouted", func(t *testing.T) { - // The old fast-path used a substring check for "deprecated"; make sure - // the new dispatch keys off _attachments only. - data := buildTarball(map[string]string{"package/package.json": `{}`}) - integrity := "sha512-" + base64Sha512(data) - body := fmt.Sprintf( - `{"name":%q,"versions":{"1.0.0":{"name":%q,"version":"1.0.0","readme":"this package is deprecated!","dist":{"integrity":%q}}},"_attachments":{"x.tgz":{"data":%q}}}`, - pkg, pkg, integrity, base64.StdEncoding.EncodeToString(data), - ) - p, dep, err := ParseUpload(strings.NewReader(body)) - require.NoError(t, err) - assert.Nil(t, dep) - require.NotNil(t, p) - }) - - t.Run("invalid json errors out", func(t *testing.T) { - _, _, err := ParseUpload(strings.NewReader("not json")) - assert.Error(t, err) - }) } func base64Sha512(data []byte) string { diff --git a/modules/packages/npm/metadata.go b/modules/packages/npm/metadata.go index e4823be7b2e..f0682d8934e 100644 --- a/modules/packages/npm/metadata.go +++ b/modules/packages/npm/metadata.go @@ -29,6 +29,7 @@ type Metadata struct { Engines map[string]string `json:"engines,omitempty"` CPU []string `json:"cpu,omitempty"` OS []string `json:"os,omitempty"` + Libc []string `json:"libc,omitempty"` Directories map[string]string `json:"directories,omitempty"` Funding any `json:"funding,omitempty"` AcceptDependencies map[string]string `json:"accept_dependencies,omitempty"` diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 67e64a66471..63244a14159 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -406,12 +406,15 @@ func CommonRoutes() *web.Router { }) r.Group("/npm", func() { r.Get("/-/v1/search", npm.PackageSearch) + r.Get("/-/ping", npm.Ping) + r.Get("/-/whoami", npm.Whoami) r.PathGroup("/*", func(g *web.RouterPathGroup) { // HINT: NPM-ROUTE-PATH-PATTERN: search this keyword to see more details packageId := `/" g.UseUnescapedPath() g.MatchPath("DELETE", packageId+"/-///-rev/", reqPackageAccess(perm.AccessModeWrite), npm.DeletePackageVersion) - g.MatchPath("GET", packageId+"/-//", npm.DownloadPackageFile) + g.MatchPath("DELETE", packageId+"/-//-rev/", reqPackageAccess(perm.AccessModeWrite), npm.DeletePackageVersion) + g.MatchPath("GET", packageId+"/-//", npm.DownloadPackageFileByName) // former tarball URL, still in lockfiles g.MatchPath("GET", packageId+"/-/", npm.DownloadPackageFileByName) g.MatchPath("DELETE", packageId+"/-rev/", reqPackageAccess(perm.AccessModeWrite), npm.DeletePackage) g.MatchPath("PUT", packageId+"/-rev/", reqPackageAccess(perm.AccessModeWrite), npm.DeletePreview) diff --git a/routers/api/packages/npm/api.go b/routers/api/packages/npm/api.go index b88a2cf70f8..7640529feba 100644 --- a/routers/api/packages/npm/api.go +++ b/routers/api/packages/npm/api.go @@ -8,7 +8,7 @@ import ( "encoding/base64" "encoding/hex" "fmt" - "net/url" + "slices" "sort" "time" @@ -25,6 +25,7 @@ func createPackageMetadataResponse(registryURL string, pds []*packages_model.Pac distTags := make(map[string]string) times := make(map[string]time.Time) firstPublished, lastPublished := pds[0].Version.CreatedUnix, pds[0].Version.CreatedUnix + var latest *packages_model.PackageDescriptor for _, pd := range pds { semVer := pd.SemVer.String() versions[semVer] = createPackageMetadataVersion(registryURL, pd) @@ -35,6 +36,9 @@ func createPackageMetadataResponse(registryURL string, pds []*packages_model.Pac for _, pvp := range pd.VersionProperties { if pvp.Name == npm_module.TagProperty { distTags[pvp.Value] = pd.Version.Version + if pvp.Value == "latest" { + latest = pd + } } } } @@ -43,7 +47,16 @@ func createPackageMetadataResponse(registryURL string, pds []*packages_model.Pac times["created"] = firstPublished.AsTimeInLocation(time.UTC) times["modified"] = lastPublished.AsTimeInLocation(time.UTC) - latest := pds[len(pds)-1] + if latest == nil { // yarn and pnpm fail without it, e.g. after its version got deleted + latest = pds[len(pds)-1] + for _, pd := range slices.Backward(pds) { + if pd.SemVer.Prerelease() == "" { + latest = pd + break + } + } + distTags["latest"] = latest.Version.Version + } metadata := packages_model.DescriptorMetadata[*npm_module.Metadata](latest) @@ -86,13 +99,13 @@ func createPackageMetadataVersion(registryURL string, pd *packages_model.Package PeerDependencies: metadata.PeerDependencies, PeerDependenciesMeta: metadata.PeerDependenciesMeta, OptionalDependencies: metadata.OptionalDependencies, - Readme: metadata.Readme, Bin: metadata.Bin, HasInstallScript: metadata.HasInstallScript, HasShrinkwrap: metadata.HasShrinkwrap, Engines: metadata.Engines, CPU: metadata.CPU, OS: metadata.OS, + Libc: metadata.Libc, Directories: metadata.Directories, Funding: metadata.Funding, AcceptDependencies: metadata.AcceptDependencies, @@ -100,7 +113,7 @@ func createPackageMetadataVersion(registryURL string, pd *packages_model.Package Dist: npm_module.PackageDistribution{ Shasum: pd.Files[0].Blob.HashSHA1, Integrity: "sha512-" + base64.StdEncoding.EncodeToString(hashBytes), - Tarball: fmt.Sprintf("%s/%s/-/%s/%s", registryURL, url.PathEscape(pd.Package.Name), url.PathEscape(pd.Version.Version), url.PathEscape(pd.Files[0].File.LowerName)), + Tarball: fmt.Sprintf("%s/%s/-/%s", registryURL, pd.Package.Name, pd.Files[0].File.LowerName), // npmjs shape, which npm parses for allowScripts and yarn keeps registry-relative }, } } diff --git a/routers/api/packages/npm/api_test.go b/routers/api/packages/npm/api_test.go index 5c0215f7c9c..a6e4d176ad6 100644 --- a/routers/api/packages/npm/api_test.go +++ b/routers/api/packages/npm/api_test.go @@ -25,7 +25,7 @@ func TestCreatePackageMetadataResponse(t *testing.T) { Owner: &user_model.User{Name: "alice"}, Version: &packages_model.PackageVersion{Version: v, CreatedUnix: timeutil.TimeStamp(publishedUnix)}, SemVer: version.Must(version.NewVersion(v)), - Metadata: &npm_module.Metadata{Keywords: []string{"gitea"}, Repository: repo}, + Metadata: &npm_module.Metadata{Readme: v, Keywords: []string{"gitea"}, Repository: repo}, Files: []*packages_model.PackageFileDescriptor{{ File: &packages_model.PackageFile{LowerName: "test-" + v + ".tgz"}, Blob: &packages_model.PackageBlob{}, @@ -35,21 +35,26 @@ func TestCreatePackageMetadataResponse(t *testing.T) { result := createPackageMetadataResponse("https://gitea.dev/api/packages/alice/npm", []*packages_model.PackageDescriptor{ descriptor("1.1.0", 1000, npm_module.Repository{}), + descriptor("2.0.0-rc.1", 1500, repository), descriptor("1.0.0", 2000, repository), }) assert.Equal(t, map[string]time.Time{ - "1.0.0": time.Unix(2000, 0).UTC(), - "1.1.0": time.Unix(1000, 0).UTC(), - "created": time.Unix(1000, 0).UTC(), - "modified": time.Unix(2000, 0).UTC(), + "1.0.0": time.Unix(2000, 0).UTC(), + "1.1.0": time.Unix(1000, 0).UTC(), + "2.0.0-rc.1": time.Unix(1500, 0).UTC(), + "created": time.Unix(1000, 0).UTC(), + "modified": time.Unix(2000, 0).UTC(), }, result.Time) + assert.Equal(t, map[string]string{"latest": "1.1.0"}, result.DistTags) + assert.Equal(t, "1.1.0", result.Readme) + assert.Empty(t, result.Versions["1.1.0"].Readme) assert.Equal(t, []npm_module.User{{Name: "alice"}}, result.Maintainers) assert.Equal(t, []string{"gitea"}, result.Keywords) assert.Equal(t, []string{"gitea"}, result.Versions["1.0.0"].Keywords) assert.Equal(t, []npm_module.User{{Name: "alice"}}, result.Versions["1.0.0"].Maintainers) assert.Equal(t, - "https://gitea.dev/api/packages/alice/npm/@scope%2Ftest/-/1.0.0/test-1.0.0.tgz", + "https://gitea.dev/api/packages/alice/npm/@scope/test/-/test-1.0.0.tgz", result.Versions["1.0.0"].Dist.Tarball, ) assert.Equal(t, repository, result.Versions["1.0.0"].Repository) diff --git a/routers/api/packages/npm/npm.go b/routers/api/packages/npm/npm.go index 55489fc9301..da75fc597c1 100644 --- a/routers/api/packages/npm/npm.go +++ b/routers/api/packages/npm/npm.go @@ -6,7 +6,9 @@ package npm import ( "bytes" std_ctx "context" + "crypto/sha256" "errors" + "fmt" "io" "net/http" "net/url" @@ -55,28 +57,41 @@ func buildNpmRegistryURL(ctx std_ctx.Context, owner *user_model.User) string { return httplib.GuessCurrentAppURL(ctx) + "api/packages/" + url.PathEscape(owner.Name) + "/npm" } -// PackageMetadata returns the metadata for a single package -func PackageMetadata(ctx *context.Context) { - packageName := packageNameFromParams(ctx) - - pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeNpm, packageName) +func packageMetadata(ctx *context.Context) *npm_module.PackageMetadata { + pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeNpm, packageNameFromParams(ctx)) if err != nil { apiError(ctx, http.StatusInternalServerError, err) - return + return nil } if len(pvs) == 0 { apiError(ctx, http.StatusNotFound, err) - return + return nil } pds, err := packages_model.GetPackageDescriptors(ctx, pvs) if err != nil { apiError(ctx, http.StatusInternalServerError, err) - return + return nil } - resp := createPackageMetadataResponse(buildNpmRegistryURL(ctx, ctx.Package.Owner), pds) - ctx.JSON(http.StatusOK, resp) + return createPackageMetadataResponse(buildNpmRegistryURL(ctx, ctx.Package.Owner), pds) +} + +// PackageMetadata returns the metadata for a single package +func PackageMetadata(ctx *context.Context) { + if metadata := packageMetadata(ctx); metadata != nil { + serveMetadata(ctx, metadata) + } +} + +func serveMetadata(ctx *context.Context, obj any) { + body, err := json.MarshalDeterministic(obj) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + ctx.Resp.Header().Set("ETag", fmt.Sprintf(`W/"%x"`, sha256.Sum256(body))) + ctx.ServeContent(bytes.NewReader(body), context.ServeHeaderOptions{ContentType: "application/json;charset=utf-8"}) } // PackageVersionMetadata returns the metadata for a single version or dist-tag @@ -100,7 +115,11 @@ func PackageVersionMetadata(ctx *context.Context) { return } if len(pvs) == 0 { - apiError(ctx, http.StatusNotFound, "version not found: "+versionOrTag) + if versionOrTag != "latest" { + apiError(ctx, http.StatusNotFound, "version not found: "+versionOrTag) + } else if metadata := packageMetadata(ctx); metadata != nil { // unset, so serve the packument's fallback + serveMetadata(ctx, metadata.Versions[metadata.DistTags["latest"]]) + } return } @@ -110,25 +129,40 @@ func PackageVersionMetadata(ctx *context.Context) { return } - ctx.JSON(http.StatusOK, createPackageMetadataVersion(buildNpmRegistryURL(ctx, ctx.Package.Owner), pd)) + serveMetadata(ctx, createPackageMetadataVersion(buildNpmRegistryURL(ctx, ctx.Package.Owner), pd)) } -// DownloadPackageFile serves the content of a package -func DownloadPackageFile(ctx *context.Context) { - packageName := packageNameFromParams(ctx) - packageVersion := ctx.PathParam("version") - filename := ctx.PathParam("filename") +func packageVersionByFilename(ctx *context.Context) *packages_model.PackageVersion { + pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ + OwnerID: ctx.Package.Owner.ID, + Type: packages_model.TypeNpm, + Name: packages_model.SearchValue{ExactMatch: true, Value: packageNameFromParams(ctx)}, + HasFileWithName: ctx.PathParam("filename"), + IsInternal: optional.Some(false), + }) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return nil + } + if len(pvs) != 1 { + apiError(ctx, http.StatusNotFound, nil) + return nil + } + return pvs[0] +} - s, u, pf, err := packages_service.OpenFileForDownloadByPackageNameAndVersion( +// DownloadPackageFileByName finds the version and serves the contents of a package +func DownloadPackageFileByName(ctx *context.Context) { + pv := packageVersionByFilename(ctx) + if pv == nil { + return + } + + s, u, pf, err := packages_service.OpenFileForDownloadByPackageVersion( ctx, - &packages_service.PackageInfo{ - Owner: ctx.Package.Owner, - PackageType: packages_model.TypeNpm, - Name: packageName, - Version: packageVersion, - }, + pv, &packages_service.PackageFileInfo{ - Filename: filename, + Filename: ctx.PathParam("filename"), }, ctx.Req.Method, ) @@ -140,54 +174,14 @@ func DownloadPackageFile(ctx *context.Context) { helper.ServePackageFile(ctx, s, u, pf) } -// DownloadPackageFileByName finds the version and serves the contents of a package -func DownloadPackageFileByName(ctx *context.Context) { - filename := ctx.PathParam("filename") - - pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ - OwnerID: ctx.Package.Owner.ID, - Type: packages_model.TypeNpm, - Name: packages_model.SearchValue{ - ExactMatch: true, - Value: packageNameFromParams(ctx), - }, - HasFileWithName: filename, - IsInternal: optional.Some(false), - }) - if err != nil { - apiError(ctx, http.StatusInternalServerError, err) - return - } - if len(pvs) != 1 { - apiError(ctx, http.StatusNotFound, nil) - return - } - - s, u, pf, err := packages_service.OpenFileForDownloadByPackageVersion( - ctx, - pvs[0], - &packages_service.PackageFileInfo{ - Filename: filename, - }, - ctx.Req.Method, - ) - if err != nil { - if errors.Is(err, packages_model.ErrPackageFileNotExist) { - apiError(ctx, http.StatusNotFound, err) - return - } - apiError(ctx, http.StatusInternalServerError, err) - return - } - - helper.ServePackageFile(ctx, s, u, pf) -} - // UploadPackage creates a new package func UploadPackage(ctx *context.Context) { - npmPackage, deprecation, err := npm_module.ParseUpload(ctx.Req.Body) + // about the npmjs and GitHub Packages limit, fits base64 tarballs up to ~200 MB + npmPackage, deprecation, err := npm_module.ParseUpload(http.MaxBytesReader(ctx.Resp, ctx.Req.Body, 256*1024*1024)) if err != nil { - if errors.Is(err, util.ErrInvalidArgument) { + if _, ok := errors.AsType[*http.MaxBytesError](err); ok { + apiError(ctx, http.StatusRequestEntityTooLarge, err) + } else if errors.Is(err, util.ErrInvalidArgument) { apiError(ctx, http.StatusBadRequest, err) } else { apiError(ctx, http.StatusInternalServerError, err) @@ -340,26 +334,14 @@ func deprecatePackage(ctx *context.Context, dep *npm_module.PackageDeprecation) ctx.Status(http.StatusOK) } -// DeletePackageVersion deletes the package version +// DeletePackageVersion deletes the package version, which `npm unpublish` addresses by its tarball func DeletePackageVersion(ctx *context.Context) { - packageName := packageNameFromParams(ctx) - packageVersion := ctx.PathParam("version") + pv := packageVersionByFilename(ctx) + if pv == nil { + return + } - err := packages_service.RemovePackageVersionByNameAndVersion( - ctx, - ctx.Doer, - &packages_service.PackageInfo{ - Owner: ctx.Package.Owner, - PackageType: packages_model.TypeNpm, - Name: packageName, - Version: packageVersion, - }, - ) - if err != nil { - if errors.Is(err, packages_model.ErrPackageNotExist) { - apiError(ctx, http.StatusNotFound, err) - return - } + if err := packages_service.RemovePackageVersion(ctx, ctx.Doer, pv); err != nil { apiError(ctx, http.StatusInternalServerError, err) return } @@ -394,9 +376,7 @@ func DeletePackage(ctx *context.Context) { // ListPackageTags returns all tags for a package func ListPackageTags(ctx *context.Context) { - packageName := packageNameFromParams(ctx) - - pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeNpm, packageName) + pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeNpm, packageNameFromParams(ctx)) if err != nil { apiError(ctx, http.StatusInternalServerError, err) return @@ -414,7 +394,11 @@ func ListPackageTags(ctx *context.Context) { } } - ctx.JSON(http.StatusOK, tags) + if _, ok := tags["latest"]; ok { + ctx.JSON(http.StatusOK, tags) + } else if metadata := packageMetadata(ctx); metadata != nil { // unset, so list the packument's fallback + ctx.JSON(http.StatusOK, metadata.DistTags) + } } // AddPackageTag adds a tag to the package @@ -524,6 +508,18 @@ func setPackageTag(ctx std_ctx.Context, tag string, pv *packages_model.PackageVe }) } +func Ping(ctx *context.Context) { + ctx.JSON(http.StatusOK, map[string]any{}) +} + +func Whoami(ctx *context.Context) { + if ctx.Doer == nil { + apiError(ctx, http.StatusUnauthorized, "Unauthorized") + return + } + ctx.JSON(http.StatusOK, map[string]string{"username": ctx.Doer.Name}) +} + func PackageSearch(ctx *context.Context) { pvs, total, err := packages_model.SearchLatestVersions(ctx, &packages_model.PackageSearchOptions{ OwnerID: ctx.Package.Owner.ID, diff --git a/tests/integration/api_packages_npm_test.go b/tests/integration/api_packages_npm_test.go index a13f053fdb6..a0d5243fcb9 100644 --- a/tests/integration/api_packages_npm_test.go +++ b/tests/integration/api_packages_npm_test.go @@ -104,6 +104,7 @@ func TestPackageNpm(t *testing.T) { }, "cpu": ["x64", "arm64"], "os": ["linux", "darwin"], + "libc": ["glibc"], "directories": { "doc": "./doc", "man": "./man" @@ -218,7 +219,7 @@ func TestPackageNpm(t *testing.T) { assert.Equal(t, packageBinPath, pmv.Bin[packageBinName]) assert.Equal(t, integrity, pmv.Dist.Integrity) assert.Equal(t, sha1SumHex, pmv.Dist.Shasum) - assert.Equal(t, fmt.Sprintf("%s%s/-/%s/%s", setting.AppURL, root[1:], packageVersion, filename), pmv.Dist.Tarball) + assert.Equal(t, fmt.Sprintf("%sapi/packages/%s/npm/%s/-/%s", setting.AppURL, user.Name, packageName, filename), pmv.Dist.Tarball) assert.Equal(t, repoType, result.Repository.Type) assert.Equal(t, repoURL, result.Repository.URL) assert.Equal(t, map[string]string{"tea": "2.x", "soy-milk": "1.2"}, pmv.PeerDependencies) @@ -228,10 +229,24 @@ func TestPackageNpm(t *testing.T) { assert.Equal(t, map[string]string{"node": ">=22.7.0", "npm": ">=10.8.2"}, pmv.Engines) assert.Equal(t, []string{"x64", "arm64"}, pmv.CPU) assert.Equal(t, []string{"linux", "darwin"}, pmv.OS) + assert.Equal(t, []string{"glibc"}, pmv.Libc) assert.Equal(t, map[string]string{"doc": "./doc", "man": "./man"}, pmv.Directories) assert.Equal(t, "https://example.com/fund", pmv.Funding) assert.Equal(t, map[string]string{"left-pad": "1.x"}, pmv.AcceptDependencies) assert.Empty(t, pmv.Deprecated) + + req = NewRequest(t, "GET", root).AddTokenAuth(token).SetHeader("If-None-Match", resp.Header().Get("ETag")) + MakeRequest(t, req, http.StatusNotModified) + }) + + t.Run("PingWhoami", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + registry := fmt.Sprintf("/api/packages/%s/npm/-/", user.Name) + MakeRequest(t, NewRequest(t, "GET", registry+"ping"), http.StatusOK) + MakeRequest(t, NewRequest(t, "GET", registry+"whoami"), http.StatusUnauthorized) + resp := MakeRequest(t, NewRequest(t, "GET", registry+"whoami").AddTokenAuth(token), http.StatusOK) + assert.JSONEq(t, `{"username":"`+user.Name+`"}`, resp.Body.String()) }) t.Run("PackageVersionMetadata", func(t *testing.T) { @@ -290,22 +305,6 @@ func TestPackageNpm(t *testing.T) { assert.Equal(t, packageVersion, result[packageTag2]) }) - t.Run("PackageMetadataDistTags", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - req := NewRequest(t, "GET", root). - AddTokenAuth(token) - resp := MakeRequest(t, req, http.StatusOK) - - result := DecodeJSON(t, resp, &npm.PackageMetadata{}) - - assert.Len(t, result.DistTags, 2) - assert.Contains(t, result.DistTags, packageTag) - assert.Equal(t, packageVersion, result.DistTags[packageTag]) - assert.Contains(t, result.DistTags, packageTag2) - assert.Equal(t, packageVersion, result.DistTags[packageTag2]) - }) - t.Run("DeleteTag", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -319,6 +318,12 @@ func TestPackageNpm(t *testing.T) { test(t, http.StatusBadRequest, "1.0") test(t, http.StatusOK, "dummy") test(t, http.StatusOK, packageTag2) + test(t, http.StatusOK, packageTag) + + resp := MakeRequest(t, NewRequest(t, "GET", tagsRoot).AddTokenAuth(token), http.StatusOK) + assert.Equal(t, map[string]string{packageTag: packageVersion}, DecodeJSON(t, resp, map[string]string{})) + resp = MakeRequest(t, NewRequest(t, "GET", root+"/"+packageTag).AddTokenAuth(token), http.StatusOK) + assert.Equal(t, packageVersion, DecodeJSON(t, resp, &npm.PackageMetadataVersion{}).Version) }) t.Run("Search", func(t *testing.T) { @@ -523,7 +528,7 @@ func TestPackageNpm(t *testing.T) { req := NewRequest(t, "DELETE", fmt.Sprintf("%s/-/%s/%s/-rev/dummy", root, packageVersion, filename)) MakeRequest(t, req, http.StatusUnauthorized) - req = NewRequest(t, "DELETE", fmt.Sprintf("%s/-/%s/%s/-rev/dummy", root, packageVersion, filename)). + req = NewRequest(t, "DELETE", fmt.Sprintf("%s/-/%s/-rev/dummy", root, filename)). AddTokenAuth(token) MakeRequest(t, req, http.StatusOK)