enhance(packages/npm): improve npm client compatibility (#39434) (#39522)

Backport #39434 by @silverwind Aligns the npm registry with what npm, pnpm and yarn expect: 1. Raise the publish body cap from https://github.com/go-gitea/gitea/pull/37890 to 256 MiB like npmjs, larger bodies get 413 2. Pick the tarball attachment by name, `npm publish --provenance` failed at random 3. Store and serve `libc`, so mismatched glibc/musl optional binaries are skipped 4. Treat root `*.gyp` files as an install script, like npm does 5. Always serve a `latest` dist-tag, yarn and pnpm fail without it 6. Take top-level metadata from `latest` and drop the per-version readme 7. Serve tarballs at the npmjs path `/<name>/-/<file>`, former URLs keep working 8. Add ETag revalidation for metadata, `npm ping` and `npm whoami` Tested with npm 12.1, pnpm 12.4, yarn 1.22 and yarn 4.18. Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
2026-10-01 20:59:45 +09:00 · 2026-10-01 01:09:21 -07:00
parent d16daed041
commit 1c4fff096f
9 changed files with 195 additions and 226 deletions
@@ -45,6 +45,10 @@ func MarshalKeepOptionalEmpty(v any) ([]byte, error) {
 	return jsonv2.Marshal(v, jsonV2.marshalKeepOptionalEmptyOptions)
 }

+func MarshalDeterministic(v any) ([]byte, error) {
+	return jsonv2.Marshal(v, jsonV2.marshalOptions, jsonv2.Deterministic(true))
+}
+
 func (j *JSONv2) Marshal(v any) ([]byte, error) {
 	return jsonv2.Marshal(v, j.marshalOptions)
 }
@@ -121,6 +121,7 @@ type PackageMetadataVersion struct {
 	Engines              map[string]string   `json:"engines,omitempty"`
 	CPU                  []string            `json:"cpu,omitempty"`
 	OS                   []string            `json:"os,omitempty"`
+	Libc                 []string            `json:"libc,omitempty"`
 	Directories          map[string]string   `json:"directories,omitempty"`
 	Funding              any                 `json:"funding,omitempty"`
 	AcceptDependencies   map[string]string   `json:"acceptDependencies,omitempty"`
@@ -129,12 +130,9 @@ type PackageMetadataVersion struct {

 // PackageDistribution https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version
 type PackageDistribution struct {
-	Integrity    string `json:"integrity"`
-	Shasum       string `json:"shasum"`
-	Tarball      string `json:"tarball"`
-	FileCount    int    `json:"fileCount,omitempty"`
-	UnpackedSize int    `json:"unpackedSize,omitempty"`
-	NpmSignature string `json:"npm-signature,omitempty"`
+	Integrity string `json:"integrity"`
+	Shasum    string `json:"shasum"`
+	Tarball   string `json:"tarball"`
 }

 type PackageSearch struct {
@@ -226,7 +224,7 @@ func (r *Repository) UnmarshalJSON(data []byte) error {
 }

 // Bin maps command names to executable files. npm also allows a single string,
-// in which case the command is named after the package (resolved in ParsePackage).
+// in which case the command is named after the package (resolved in parseUploadPackage).
 type Bin map[string]string

 // UnmarshalJSON is needed because the bin field can be a string or an object.
@@ -264,7 +262,7 @@ type packageUpload struct {
 // is non-nil on success; a body without `_attachments` is a deprecate request,
 // otherwise it is a "publish".
 func ParseUpload(r io.Reader) (*Package, *PackageDeprecation, error) {
-	body, err := io.ReadAll(io.LimitReader(r, 10*1024*1024))
+	body, err := io.ReadAll(r)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -280,16 +278,6 @@ func ParseUpload(r io.Reader) (*Package, *PackageDeprecation, error) {
 	return p, nil, err
 }

-// ParsePackage parses a npm publish PUT body. Bodies without `_attachments`
-// surface as ErrInvalidAttachment once name/version validation has passed.
-func ParsePackage(r io.Reader) (*Package, error) {
-	var upload packageUpload
-	if err := json.NewDecoder(r).Decode(&upload); err != nil {
-		return nil, err
-	}
-	return parseUploadPackage(&upload)
-}
-
 // parseUploadPackage builds a Package from a decoded publish body.
 func parseUploadPackage(upload *packageUpload) (*Package, error) {
 	for _, meta := range upload.Versions {
@@ -343,6 +331,7 @@ func parseUploadPackage(upload *packageUpload) (*Package, error) {
 				Engines:                 meta.Engines,
 				CPU:                     meta.CPU,
 				OS:                      meta.OS,
+				Libc:                    meta.Libc,
 				Directories:             meta.Directories,
 				Funding:                 meta.Funding,
 				AcceptDependencies:      meta.AcceptDependencies,
@@ -356,12 +345,12 @@ func parseUploadPackage(upload *packageUpload) (*Package, error) {

 		p.Filename = strings.ToLower(fmt.Sprintf("%s-%s.tgz", name, p.Version))

-		attachment := func() *PackageAttachment {
+		attachment := upload.Attachments[meta.Name+"-"+meta.Version+".tgz"] // not the sigstore bundle of `npm publish --provenance`
+		if attachment == nil && len(upload.Attachments) == 1 {
 			for _, a := range upload.Attachments {
-				return a
+				attachment = a
 			}
-			return nil
-		}()
+		}
 		if attachment == nil || len(attachment.Data) == 0 {
 			return nil, ErrInvalidAttachment
 		}
@@ -393,8 +382,6 @@ func parseUploadPackage(upload *packageUpload) (*Package, error) {
 			return nil, ErrInvalidIntegrity
 		}

-		// Derive _hasShrinkwrap and hasInstallScript from the tarball; the
-		// packument can lie about either.
 		p.Metadata.HasShrinkwrap, p.Metadata.HasInstallScript = inspectTarball(data)

 		return p, nil
@@ -410,11 +397,7 @@ const maxNpmTarballScanBytes = int64(32 * 1024 * 1024) // 32 MiB
 // maxNpmPackageJSONBytes caps the package.json bytes decoded from the tarball.
 const maxNpmPackageJSONBytes = int64(1 * 1024 * 1024) // 1 MiB

-// inspectTarball reports hasShrinkwrap (presence of package/npm-shrinkwrap.json)
-// and hasInstallScript (package/package.json declares any of preinstall,
-// install, postinstall). Both must be derived server-side because the client
-// can lie in the packument. Any read/decode error yields (false, false) so a
-// malformed archive does not block publishing.
+// inspectTarball trusts the tarball over the client's packument, read errors yield zero values to not block publishing
 func inspectTarball(data []byte) (hasShrinkwrap, hasInstallScript bool) {
 	gr, err := gzip.NewReader(bytes.NewReader(data))
 	if err != nil {
@@ -422,11 +405,16 @@ func inspectTarball(data []byte) (hasShrinkwrap, hasInstallScript bool) {
 	}
 	defer gr.Close()

+	var hasGypFile bool
+	var pkg struct {
+		Scripts map[string]string `json:"scripts"`
+		Gypfile any               `json:"gypfile"`
+	}
 	tr := tar.NewReader(io.LimitReader(gr, maxNpmTarballScanBytes))
 	for {
 		hdr, err := tr.Next()
 		if err != nil {
-			return hasShrinkwrap, hasInstallScript
+			break
 		}
 		// npm pack puts files under a single root directory (usually "package/").
 		name := strings.TrimPrefix(hdr.Name, "./")
@@ -436,30 +424,14 @@ func inspectTarball(data []byte) (hasShrinkwrap, hasInstallScript bool) {
 		switch {
 		case strings.HasSuffix(name, "/npm-shrinkwrap.json"):
 			hasShrinkwrap = true
+		case strings.HasSuffix(name, ".gyp"):
+			hasGypFile = true
 		case strings.HasSuffix(name, "/package.json"):
-			hasInstallScript = tarballDeclaresInstallScript(tr)
-		}
-		if hasShrinkwrap && hasInstallScript {
-			return hasShrinkwrap, hasInstallScript
+			_ = json.NewDecoder(io.LimitReader(tr, maxNpmPackageJSONBytes)).Decode(&pkg)
 		}
 	}
-}
-
-// tarballDeclaresInstallScript reports whether a package.json declares any
-// of preinstall, install, postinstall.
-func tarballDeclaresInstallScript(r io.Reader) bool {
-	var pkg struct {
-		Scripts map[string]string `json:"scripts"`
-	}
-	if err := json.NewDecoder(io.LimitReader(r, maxNpmPackageJSONBytes)).Decode(&pkg); err != nil {
-		return false
-	}
-	for _, name := range []string{"preinstall", "install", "postinstall"} {
-		if strings.TrimSpace(pkg.Scripts[name]) != "" {
-			return true
-		}
-	}
-	return false
+	// npm publish adds "install": "node-gyp rebuild" for a root *.gyp file to the manifest, but not to the tarball
+	return hasShrinkwrap, strings.TrimSpace(pkg.Scripts["preinstall"]+pkg.Scripts["install"]+pkg.Scripts["postinstall"]) != "" || hasGypFile && pkg.Gypfile != false
 }

 func validateName(name string) bool {
@@ -41,21 +41,20 @@ func TestParsePackage(t *testing.T) {
 	integrity := "sha512-" + base64Sha512(dataBytes)

 	t.Run("InvalidUpload", func(t *testing.T) {
-		p, err := ParsePackage(bytes.NewReader([]byte{0}))
+		p, _, err := ParseUpload(bytes.NewReader([]byte{0}))
 		assert.Nil(t, p)
 		assert.Error(t, err)
 	})

 	t.Run("InvalidUploadNoData", func(t *testing.T) {
-		b, _ := json.Marshal(packageUpload{})
-		p, err := ParsePackage(bytes.NewReader(b))
+		p, err := parseUploadPackage(&packageUpload{})
 		assert.Nil(t, p)
 		assert.ErrorIs(t, err, ErrInvalidPackage)
 	})

 	t.Run("InvalidPackageName", func(t *testing.T) {
 		test := func(t *testing.T, name string) {
-			b, _ := json.Marshal(packageUpload{
+			p, err := parseUploadPackage(&packageUpload{
 				PackageMetadata: PackageMetadata{
 					ID:   name,
 					Name: name,
@@ -66,8 +65,6 @@ func TestParsePackage(t *testing.T) {
 					},
 				},
 			})
-
-			p, err := ParsePackage(bytes.NewReader(b))
 			assert.Nil(t, p)
 			assert.ErrorIs(t, err, ErrInvalidPackageName)
 		}
@@ -94,7 +91,7 @@ func TestParsePackage(t *testing.T) {

 	t.Run("ValidPackageName", func(t *testing.T) {
 		test := func(t *testing.T, name string) {
-			b, _ := json.Marshal(packageUpload{
+			p, err := parseUploadPackage(&packageUpload{
 				PackageMetadata: PackageMetadata{
 					ID:   name,
 					Name: name,
@@ -105,8 +102,6 @@ func TestParsePackage(t *testing.T) {
 					},
 				},
 			})
-
-			p, err := ParsePackage(bytes.NewReader(b))
 			assert.Nil(t, p)
 			assert.ErrorIs(t, err, ErrInvalidPackageVersion)
 		}
@@ -125,7 +120,7 @@ func TestParsePackage(t *testing.T) {

 	t.Run("InvalidPackageVersion", func(t *testing.T) {
 		version := "first-version"
-		b, _ := json.Marshal(packageUpload{
+		p, err := parseUploadPackage(&packageUpload{
 			PackageMetadata: PackageMetadata{
 				ID:   packageFullName,
 				Name: packageFullName,
@@ -137,8 +132,6 @@ func TestParsePackage(t *testing.T) {
 				},
 			},
 		})
-
-		p, err := ParsePackage(bytes.NewReader(b))
 		assert.Nil(t, p)
 		assert.ErrorIs(t, err, ErrInvalidPackageVersion)
 	})
@@ -160,7 +153,7 @@ func TestParsePackage(t *testing.T) {
 			},
 		})

-		p, err := ParsePackage(bytes.NewReader(b))
+		p, _, err := ParseUpload(bytes.NewReader(b))
 		assert.Nil(t, p)
 		assert.ErrorIs(t, err, ErrInvalidAttachment)
 	})
@@ -185,7 +178,7 @@ func TestParsePackage(t *testing.T) {
 			},
 		})

-		p, err := ParsePackage(bytes.NewReader(b))
+		p, _, err := ParseUpload(bytes.NewReader(b))
 		assert.Nil(t, p)
 		assert.ErrorIs(t, err, ErrInvalidAttachment)
 	})
@@ -213,7 +206,7 @@ func TestParsePackage(t *testing.T) {
 			},
 		})

-		p, err := ParsePackage(bytes.NewReader(b))
+		p, _, err := ParseUpload(bytes.NewReader(b))
 		assert.Nil(t, p)
 		assert.ErrorIs(t, err, ErrInvalidIntegrity)
 	})
@@ -241,7 +234,7 @@ func TestParsePackage(t *testing.T) {
 			},
 		})

-		p, err := ParsePackage(bytes.NewReader(b))
+		p, _, err := ParseUpload(bytes.NewReader(b))
 		assert.Nil(t, p)
 		assert.ErrorIs(t, err, ErrInvalidIntegrity)
 	})
@@ -281,10 +274,13 @@ func TestParsePackage(t *testing.T) {
 				filename: {
 					Data: data,
 				},
+				packageFullName + "-" + packageVersion + ".sigstore": {
+					Data: "{}",
+				},
 			},
 		})

-		p, err := ParsePackage(bytes.NewReader(b))
+		p, _, err := ParseUpload(bytes.NewReader(b))
 		assert.NotNil(t, p)
 		assert.NoError(t, err)

@@ -329,7 +325,7 @@ func TestParsePackage(t *testing.T) {
 		}
 	}
 }`
-		p, err := ParsePackage(strings.NewReader(packageJSON))
+		p, _, err := ParseUpload(strings.NewReader(packageJSON))
 		require.NoError(t, err)
 		require.Equal(t, "MIT", string(p.Metadata.License))
 	})
@@ -354,7 +350,7 @@ func TestParsePackage(t *testing.T) {
 		}
 	}
 }`
-		p, err := ParsePackage(strings.NewReader(packageJSON))
+		p, _, err := ParseUpload(strings.NewReader(packageJSON))
 		require.NoError(t, err)
 		require.Equal(t, "https://gitea.io/gitea/test.git", p.Metadata.Repository.URL)
 		// a string bin is named after the package
@@ -426,6 +422,15 @@ func TestInspectTarball(t *testing.T) {
 			// npm pack sometimes emits "./package/..." entries.
 			wantShrinkwrap: true,
 		},
+		{
+			name:          "gyp file implies node-gyp install",
+			files:         map[string]string{"package/binding.gyp": "{}"},
+			wantInstaller: true,
+		},
+		{
+			name:  "gypfile false disables gyp install",
+			files: map[string]string{"package/binding.gyp": "{}", "package/package.json": `{"gypfile":false}`},
+		},
 	}
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
@@ -460,41 +465,6 @@ func TestParseUpload(t *testing.T) {
 		require.NotNil(t, dep)
 		assert.Equal(t, map[string]string{"1.0.0": "gone", "1.0.1": ""}, dep.Versions)
 	})
-
-	t.Run("dispatches publish when _attachments present", func(t *testing.T) {
-		// Reuse a minimal tarball with a package.json.
-		data := buildTarball(map[string]string{"package/package.json": `{}`})
-		integrity := "sha512-" + base64Sha512(data)
-		body := fmt.Sprintf(
-			`{"name":%q,"versions":{"1.0.0":{"name":%q,"version":"1.0.0","dist":{"integrity":%q}}},"_attachments":{"x.tgz":{"data":%q}}}`,
-			pkg, pkg, integrity, base64.StdEncoding.EncodeToString(data),
-		)
-		p, dep, err := ParseUpload(strings.NewReader(body))
-		require.NoError(t, err)
-		assert.Nil(t, dep)
-		require.NotNil(t, p)
-		assert.Equal(t, pkg, p.Name)
-	})
-
-	t.Run("publish whose readme mentions deprecated is not misrouted", func(t *testing.T) {
-		// The old fast-path used a substring check for "deprecated"; make sure
-		// the new dispatch keys off _attachments only.
-		data := buildTarball(map[string]string{"package/package.json": `{}`})
-		integrity := "sha512-" + base64Sha512(data)
-		body := fmt.Sprintf(
-			`{"name":%q,"versions":{"1.0.0":{"name":%q,"version":"1.0.0","readme":"this package is deprecated!","dist":{"integrity":%q}}},"_attachments":{"x.tgz":{"data":%q}}}`,
-			pkg, pkg, integrity, base64.StdEncoding.EncodeToString(data),
-		)
-		p, dep, err := ParseUpload(strings.NewReader(body))
-		require.NoError(t, err)
-		assert.Nil(t, dep)
-		require.NotNil(t, p)
-	})
-
-	t.Run("invalid json errors out", func(t *testing.T) {
-		_, _, err := ParseUpload(strings.NewReader("not json"))
-		assert.Error(t, err)
-	})
 }

 func base64Sha512(data []byte) string {
@@ -29,6 +29,7 @@ type Metadata struct {
 	Engines                 map[string]string `json:"engines,omitempty"`
 	CPU                     []string          `json:"cpu,omitempty"`
 	OS                      []string          `json:"os,omitempty"`
+	Libc                    []string          `json:"libc,omitempty"`
 	Directories             map[string]string `json:"directories,omitempty"`
 	Funding                 any               `json:"funding,omitempty"`
 	AcceptDependencies      map[string]string `json:"accept_dependencies,omitempty"`