enhance(packages/npm): improve npm client compatibility (#39434) (#39522)

Backport #39434 by @silverwind

Aligns the npm registry with what npm, pnpm and yarn expect:

1. Raise the publish body cap from
https://github.com/go-gitea/gitea/pull/37890 to 256 MiB like npmjs,
larger bodies get 413
2. Pick the tarball attachment by name, `npm publish --provenance`
failed at random
3. Store and serve `libc`, so mismatched glibc/musl optional binaries
are skipped
4. Treat root `*.gyp` files as an install script, like npm does
5. Always serve a `latest` dist-tag, yarn and pnpm fail without it
6. Take top-level metadata from `latest` and drop the per-version readme
7. Serve tarballs at the npmjs path `/<name>/-/<file>`, former URLs keep
working
8. Add ETag revalidation for metadata, `npm ping` and `npm whoami`

Tested with npm 12.1, pnpm 12.4, yarn 1.22 and yarn 4.18.

Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

routers/api/packages/api.go

@@ -406,12 +406,15 @@ func CommonRoutes() *web.Router {
 		})
 		r.Group("/npm", func() {
 			r.Get("/-/v1/search", npm.PackageSearch)
+			r.Get("/-/ping", npm.Ping)
+			r.Get("/-/whoami", npm.Whoami)
 			r.PathGroup("/*", func(g *web.RouterPathGroup) {
 				// HINT: NPM-ROUTE-PATH-PATTERN: search this keyword to see more details
 				packageId := `/<id:(@` + npm_module.RegexpNamePart + `/)?` + npm_module.RegexpNamePart + ">"
 				g.UseUnescapedPath()
 				g.MatchPath("DELETE", packageId+"/-/<version>/<filename>/-rev/<revision>", reqPackageAccess(perm.AccessModeWrite), npm.DeletePackageVersion)
-				g.MatchPath("GET", packageId+"/-/<version>/<filename>", npm.DownloadPackageFile)
+				g.MatchPath("DELETE", packageId+"/-/<filename>/-rev/<revision>", reqPackageAccess(perm.AccessModeWrite), npm.DeletePackageVersion)
+				g.MatchPath("GET", packageId+"/-/<version>/<filename>", npm.DownloadPackageFileByName) // former tarball URL, still in lockfiles
 				g.MatchPath("GET", packageId+"/-/<filename>", npm.DownloadPackageFileByName)
 				g.MatchPath("DELETE", packageId+"/-rev/<revision>", reqPackageAccess(perm.AccessModeWrite), npm.DeletePackage)
 				g.MatchPath("PUT", packageId+"/-rev/<revision>", reqPackageAccess(perm.AccessModeWrite), npm.DeletePreview)

routers/api/packages/npm/api.go

@@ -8,7 +8,7 @@ import (
 	"encoding/base64"
 	"encoding/hex"
 	"fmt"
-	"net/url"
+	"slices"
 	"sort"
 	"time"
 
@@ -25,6 +25,7 @@ func createPackageMetadataResponse(registryURL string, pds []*packages_model.Pac
 	distTags := make(map[string]string)
 	times := make(map[string]time.Time)
 	firstPublished, lastPublished := pds[0].Version.CreatedUnix, pds[0].Version.CreatedUnix
+	var latest *packages_model.PackageDescriptor
 	for _, pd := range pds {
 		semVer := pd.SemVer.String()
 		versions[semVer] = createPackageMetadataVersion(registryURL, pd)
@@ -35,6 +36,9 @@ func createPackageMetadataResponse(registryURL string, pds []*packages_model.Pac
 		for _, pvp := range pd.VersionProperties {
 			if pvp.Name == npm_module.TagProperty {
 				distTags[pvp.Value] = pd.Version.Version
+				if pvp.Value == "latest" {
+					latest = pd
+				}
 			}
 		}
 	}
@@ -43,7 +47,16 @@ func createPackageMetadataResponse(registryURL string, pds []*packages_model.Pac
 	times["created"] = firstPublished.AsTimeInLocation(time.UTC)
 	times["modified"] = lastPublished.AsTimeInLocation(time.UTC)
 
-	latest := pds[len(pds)-1]
+	if latest == nil { // yarn and pnpm fail without it, e.g. after its version got deleted
+		latest = pds[len(pds)-1]
+		for _, pd := range slices.Backward(pds) {
+			if pd.SemVer.Prerelease() == "" {
+				latest = pd
+				break
+			}
+		}
+		distTags["latest"] = latest.Version.Version
+	}
 
 	metadata := packages_model.DescriptorMetadata[*npm_module.Metadata](latest)
 
@@ -86,13 +99,13 @@ func createPackageMetadataVersion(registryURL string, pd *packages_model.Package
 		PeerDependencies:     metadata.PeerDependencies,
 		PeerDependenciesMeta: metadata.PeerDependenciesMeta,
 		OptionalDependencies: metadata.OptionalDependencies,
-		Readme:               metadata.Readme,
 		Bin:                  metadata.Bin,
 		HasInstallScript:     metadata.HasInstallScript,
 		HasShrinkwrap:        metadata.HasShrinkwrap,
 		Engines:              metadata.Engines,
 		CPU:                  metadata.CPU,
 		OS:                   metadata.OS,
+		Libc:                 metadata.Libc,
 		Directories:          metadata.Directories,
 		Funding:              metadata.Funding,
 		AcceptDependencies:   metadata.AcceptDependencies,
@@ -100,7 +113,7 @@ func createPackageMetadataVersion(registryURL string, pd *packages_model.Package
 		Dist: npm_module.PackageDistribution{
 			Shasum:    pd.Files[0].Blob.HashSHA1,
 			Integrity: "sha512-" + base64.StdEncoding.EncodeToString(hashBytes),
-			Tarball:   fmt.Sprintf("%s/%s/-/%s/%s", registryURL, url.PathEscape(pd.Package.Name), url.PathEscape(pd.Version.Version), url.PathEscape(pd.Files[0].File.LowerName)),
+			Tarball:   fmt.Sprintf("%s/%s/-/%s", registryURL, pd.Package.Name, pd.Files[0].File.LowerName), // npmjs shape, which npm parses for allowScripts and yarn keeps registry-relative
 		},
 	}
 }

routers/api/packages/npm/api_test.go

@@ -25,7 +25,7 @@ func TestCreatePackageMetadataResponse(t *testing.T) {
 			Owner:    &user_model.User{Name: "alice"},
 			Version:  &packages_model.PackageVersion{Version: v, CreatedUnix: timeutil.TimeStamp(publishedUnix)},
 			SemVer:   version.Must(version.NewVersion(v)),
-			Metadata: &npm_module.Metadata{Keywords: []string{"gitea"}, Repository: repo},
+			Metadata: &npm_module.Metadata{Readme: v, Keywords: []string{"gitea"}, Repository: repo},
 			Files: []*packages_model.PackageFileDescriptor{{
 				File: &packages_model.PackageFile{LowerName: "test-" + v + ".tgz"},
 				Blob: &packages_model.PackageBlob{},
@@ -35,21 +35,26 @@ func TestCreatePackageMetadataResponse(t *testing.T) {
 
 	result := createPackageMetadataResponse("https://gitea.dev/api/packages/alice/npm", []*packages_model.PackageDescriptor{
 		descriptor("1.1.0", 1000, npm_module.Repository{}),
+		descriptor("2.0.0-rc.1", 1500, repository),
 		descriptor("1.0.0", 2000, repository),
 	})
 
 	assert.Equal(t, map[string]time.Time{
-		"1.0.0":    time.Unix(2000, 0).UTC(),
-		"1.1.0":    time.Unix(1000, 0).UTC(),
-		"created":  time.Unix(1000, 0).UTC(),
-		"modified": time.Unix(2000, 0).UTC(),
+		"1.0.0":      time.Unix(2000, 0).UTC(),
+		"1.1.0":      time.Unix(1000, 0).UTC(),
+		"2.0.0-rc.1": time.Unix(1500, 0).UTC(),
+		"created":    time.Unix(1000, 0).UTC(),
+		"modified":   time.Unix(2000, 0).UTC(),
 	}, result.Time)
+	assert.Equal(t, map[string]string{"latest": "1.1.0"}, result.DistTags)
+	assert.Equal(t, "1.1.0", result.Readme)
+	assert.Empty(t, result.Versions["1.1.0"].Readme)
 	assert.Equal(t, []npm_module.User{{Name: "alice"}}, result.Maintainers)
 	assert.Equal(t, []string{"gitea"}, result.Keywords)
 	assert.Equal(t, []string{"gitea"}, result.Versions["1.0.0"].Keywords)
 	assert.Equal(t, []npm_module.User{{Name: "alice"}}, result.Versions["1.0.0"].Maintainers)
 	assert.Equal(t,
-		"https://gitea.dev/api/packages/alice/npm/@scope%2Ftest/-/1.0.0/test-1.0.0.tgz",
+		"https://gitea.dev/api/packages/alice/npm/@scope/test/-/test-1.0.0.tgz",
 		result.Versions["1.0.0"].Dist.Tarball,
 	)
 	assert.Equal(t, repository, result.Versions["1.0.0"].Repository)

routers/api/packages/npm/npm.go

@@ -6,7 +6,9 @@ package npm
 import (
 	"bytes"
 	std_ctx "context"
+	"crypto/sha256"
 	"errors"
+	"fmt"
 	"io"
 	"net/http"
 	"net/url"
@@ -55,28 +57,41 @@ func buildNpmRegistryURL(ctx std_ctx.Context, owner *user_model.User) string {
 	return httplib.GuessCurrentAppURL(ctx) + "api/packages/" + url.PathEscape(owner.Name) + "/npm"
 }
 
-// PackageMetadata returns the metadata for a single package
-func PackageMetadata(ctx *context.Context) {
-	packageName := packageNameFromParams(ctx)
-
-	pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeNpm, packageName)
+func packageMetadata(ctx *context.Context) *npm_module.PackageMetadata {
+	pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeNpm, packageNameFromParams(ctx))
 	if err != nil {
 		apiError(ctx, http.StatusInternalServerError, err)
-		return
+		return nil
 	}
 	if len(pvs) == 0 {
 		apiError(ctx, http.StatusNotFound, err)
-		return
+		return nil
 	}
 
 	pds, err := packages_model.GetPackageDescriptors(ctx, pvs)
 	if err != nil {
 		apiError(ctx, http.StatusInternalServerError, err)
-		return
+		return nil
 	}
 
-	resp := createPackageMetadataResponse(buildNpmRegistryURL(ctx, ctx.Package.Owner), pds)
-	ctx.JSON(http.StatusOK, resp)
+	return createPackageMetadataResponse(buildNpmRegistryURL(ctx, ctx.Package.Owner), pds)
+}
+
+// PackageMetadata returns the metadata for a single package
+func PackageMetadata(ctx *context.Context) {
+	if metadata := packageMetadata(ctx); metadata != nil {
+		serveMetadata(ctx, metadata)
+	}
+}
+
+func serveMetadata(ctx *context.Context, obj any) {
+	body, err := json.MarshalDeterministic(obj)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	ctx.Resp.Header().Set("ETag", fmt.Sprintf(`W/"%x"`, sha256.Sum256(body)))
+	ctx.ServeContent(bytes.NewReader(body), context.ServeHeaderOptions{ContentType: "application/json;charset=utf-8"})
 }
 
 // PackageVersionMetadata returns the metadata for a single version or dist-tag
@@ -100,7 +115,11 @@ func PackageVersionMetadata(ctx *context.Context) {
 		return
 	}
 	if len(pvs) == 0 {
-		apiError(ctx, http.StatusNotFound, "version not found: "+versionOrTag)
+		if versionOrTag != "latest" {
+			apiError(ctx, http.StatusNotFound, "version not found: "+versionOrTag)
+		} else if metadata := packageMetadata(ctx); metadata != nil { // unset, so serve the packument's fallback
+			serveMetadata(ctx, metadata.Versions[metadata.DistTags["latest"]])
+		}
 		return
 	}
 
@@ -110,25 +129,40 @@ func PackageVersionMetadata(ctx *context.Context) {
 		return
 	}
 
-	ctx.JSON(http.StatusOK, createPackageMetadataVersion(buildNpmRegistryURL(ctx, ctx.Package.Owner), pd))
+	serveMetadata(ctx, createPackageMetadataVersion(buildNpmRegistryURL(ctx, ctx.Package.Owner), pd))
 }
 
-// DownloadPackageFile serves the content of a package
-func DownloadPackageFile(ctx *context.Context) {
-	packageName := packageNameFromParams(ctx)
-	packageVersion := ctx.PathParam("version")
-	filename := ctx.PathParam("filename")
+func packageVersionByFilename(ctx *context.Context) *packages_model.PackageVersion {
+	pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
+		OwnerID:         ctx.Package.Owner.ID,
+		Type:            packages_model.TypeNpm,
+		Name:            packages_model.SearchValue{ExactMatch: true, Value: packageNameFromParams(ctx)},
+		HasFileWithName: ctx.PathParam("filename"),
+		IsInternal:      optional.Some(false),
+	})
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return nil
+	}
+	if len(pvs) != 1 {
+		apiError(ctx, http.StatusNotFound, nil)
+		return nil
+	}
+	return pvs[0]
+}
 
-	s, u, pf, err := packages_service.OpenFileForDownloadByPackageNameAndVersion(
+// DownloadPackageFileByName finds the version and serves the contents of a package
+func DownloadPackageFileByName(ctx *context.Context) {
+	pv := packageVersionByFilename(ctx)
+	if pv == nil {
+		return
+	}
+
+	s, u, pf, err := packages_service.OpenFileForDownloadByPackageVersion(
 		ctx,
-		&packages_service.PackageInfo{
-			Owner:       ctx.Package.Owner,
-			PackageType: packages_model.TypeNpm,
-			Name:        packageName,
-			Version:     packageVersion,
-		},
+		pv,
 		&packages_service.PackageFileInfo{
-			Filename: filename,
+			Filename: ctx.PathParam("filename"),
 		},
 		ctx.Req.Method,
 	)
@@ -140,54 +174,14 @@ func DownloadPackageFile(ctx *context.Context) {
 	helper.ServePackageFile(ctx, s, u, pf)
 }
 
-// DownloadPackageFileByName finds the version and serves the contents of a package
-func DownloadPackageFileByName(ctx *context.Context) {
-	filename := ctx.PathParam("filename")
-
-	pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
-		OwnerID: ctx.Package.Owner.ID,
-		Type:    packages_model.TypeNpm,
-		Name: packages_model.SearchValue{
-			ExactMatch: true,
-			Value:      packageNameFromParams(ctx),
-		},
-		HasFileWithName: filename,
-		IsInternal:      optional.Some(false),
-	})
-	if err != nil {
-		apiError(ctx, http.StatusInternalServerError, err)
-		return
-	}
-	if len(pvs) != 1 {
-		apiError(ctx, http.StatusNotFound, nil)
-		return
-	}
-
-	s, u, pf, err := packages_service.OpenFileForDownloadByPackageVersion(
-		ctx,
-		pvs[0],
-		&packages_service.PackageFileInfo{
-			Filename: filename,
-		},
-		ctx.Req.Method,
-	)
-	if err != nil {
-		if errors.Is(err, packages_model.ErrPackageFileNotExist) {
-			apiError(ctx, http.StatusNotFound, err)
-			return
-		}
-		apiError(ctx, http.StatusInternalServerError, err)
-		return
-	}
-
-	helper.ServePackageFile(ctx, s, u, pf)
-}
-
 // UploadPackage creates a new package
 func UploadPackage(ctx *context.Context) {
-	npmPackage, deprecation, err := npm_module.ParseUpload(ctx.Req.Body)
+	// about the npmjs and GitHub Packages limit, fits base64 tarballs up to ~200 MB
+	npmPackage, deprecation, err := npm_module.ParseUpload(http.MaxBytesReader(ctx.Resp, ctx.Req.Body, 256*1024*1024))
 	if err != nil {
-		if errors.Is(err, util.ErrInvalidArgument) {
+		if _, ok := errors.AsType[*http.MaxBytesError](err); ok {
+			apiError(ctx, http.StatusRequestEntityTooLarge, err)
+		} else if errors.Is(err, util.ErrInvalidArgument) {
 			apiError(ctx, http.StatusBadRequest, err)
 		} else {
 			apiError(ctx, http.StatusInternalServerError, err)
@@ -340,26 +334,14 @@ func deprecatePackage(ctx *context.Context, dep *npm_module.PackageDeprecation)
 	ctx.Status(http.StatusOK)
 }
 
-// DeletePackageVersion deletes the package version
+// DeletePackageVersion deletes the package version, which `npm unpublish` addresses by its tarball
 func DeletePackageVersion(ctx *context.Context) {
-	packageName := packageNameFromParams(ctx)
-	packageVersion := ctx.PathParam("version")
+	pv := packageVersionByFilename(ctx)
+	if pv == nil {
+		return
+	}
 
-	err := packages_service.RemovePackageVersionByNameAndVersion(
-		ctx,
-		ctx.Doer,
-		&packages_service.PackageInfo{
-			Owner:       ctx.Package.Owner,
-			PackageType: packages_model.TypeNpm,
-			Name:        packageName,
-			Version:     packageVersion,
-		},
-	)
-	if err != nil {
-		if errors.Is(err, packages_model.ErrPackageNotExist) {
-			apiError(ctx, http.StatusNotFound, err)
-			return
-		}
+	if err := packages_service.RemovePackageVersion(ctx, ctx.Doer, pv); err != nil {
 		apiError(ctx, http.StatusInternalServerError, err)
 		return
 	}
@@ -394,9 +376,7 @@ func DeletePackage(ctx *context.Context) {
 
 // ListPackageTags returns all tags for a package
 func ListPackageTags(ctx *context.Context) {
-	packageName := packageNameFromParams(ctx)
-
-	pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeNpm, packageName)
+	pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeNpm, packageNameFromParams(ctx))
 	if err != nil {
 		apiError(ctx, http.StatusInternalServerError, err)
 		return
@@ -414,7 +394,11 @@ func ListPackageTags(ctx *context.Context) {
 		}
 	}
 
-	ctx.JSON(http.StatusOK, tags)
+	if _, ok := tags["latest"]; ok {
+		ctx.JSON(http.StatusOK, tags)
+	} else if metadata := packageMetadata(ctx); metadata != nil { // unset, so list the packument's fallback
+		ctx.JSON(http.StatusOK, metadata.DistTags)
+	}
 }
 
 // AddPackageTag adds a tag to the package
@@ -524,6 +508,18 @@ func setPackageTag(ctx std_ctx.Context, tag string, pv *packages_model.PackageVe
 	})
 }
 
+func Ping(ctx *context.Context) {
+	ctx.JSON(http.StatusOK, map[string]any{})
+}
+
+func Whoami(ctx *context.Context) {
+	if ctx.Doer == nil {
+		apiError(ctx, http.StatusUnauthorized, "Unauthorized")
+		return
+	}
+	ctx.JSON(http.StatusOK, map[string]string{"username": ctx.Doer.Name})
+}
+
 func PackageSearch(ctx *context.Context) {
 	pvs, total, err := packages_model.SearchLatestVersions(ctx, &packages_model.PackageSearchOptions{
 		OwnerID:    ctx.Package.Owner.ID,