tools/bump: keep the tested Terraform provider releases current

A new release arrives as a bump pull request; verify reports a stale set.

AGENTS.md

@@ -297,6 +297,13 @@ major version. One provider instance serves both node certificates and certmagic
 so they move together; a provider on another libdns major cannot be added until
 certmagic moves.
 
+The servertest Terraform tests run the newest three `tailscale/tailscale`
+provider releases, listed in `terraformProviderVersions`
+(`hscontrol/servertest/apiv2_test.go`). The list is committed, not resolved at
+test time, so a new release arrives as a bump pull request rather than failing
+unrelated ones. A test of a provider feature calls `requireProvider` with the
+first release that has it.
+
 `Dockerfile.tailscale-HEAD` and `Dockerfile.derper` compile a tailscale tree
 cloned from an unpinned branch, so their builder image is a floor, not a
 target: it has to be at least upstream's `go` directive. They set

tools/bump/plan.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"strings"
 )
 
 // cmdPlan resolves every upstream the bump reads and prints the gap against
@@ -27,6 +28,7 @@ func planLines(ctx context.Context, r *repo) []string {
 	lines = append(lines, planBuilders(ctx, r)...)
 	lines = append(lines, planLockstep(ctx, r)...)
 	lines = append(lines, planImages(ctx, r)...)
+	lines = append(lines, planProviders(ctx, r))
 
 	if version, err := oapiVersion(r); err != nil { //nolint:noinlineerr
 		lines = append(lines, "oapi-codegen: "+err.Error())
@@ -39,6 +41,20 @@ func planLines(ctx context.Context, r *repo) []string {
 	return lines
 }
 
+func planProviders(ctx context.Context, r *repo) string {
+	have, err := currentProviders(r)
+	if err != nil {
+		return "Terraform provider: " + err.Error()
+	}
+
+	want, err := newestProviders(ctx)
+	if err != nil {
+		return "Terraform provider: " + err.Error()
+	}
+
+	return gap("Terraform provider", strings.Join(have, ", "), strings.Join(want, ", "))
+}
+
 func planImages(ctx context.Context, r *repo) []string {
 	defs := imageBumps()
 	lines := make([]string, 0, len(defs))

tools/bump/tools.go

@@ -5,12 +5,23 @@ import (
 	"encoding/json"
 	"fmt"
 	"regexp"
+	"slices"
 	"strings"
 
 	"golang.org/x/mod/semver"
 )
 
-const pypiURL = "https://pypi.org/pypi/%s/json"
+const (
+	pypiURL = "https://pypi.org/pypi/%s/json"
+
+	providerRegistryURL = "https://registry.opentofu.org/v1/providers/tailscale/tailscale/versions"
+	// providerPinFile holds the provider releases the servertest Terraform
+	// tests run against.
+	providerPinFile = "hscontrol/servertest/apiv2_test.go"
+	// testedProviders is how many of the newest provider releases are tested:
+	// the latest, and the ones before it that users still run.
+	testedProviders = 3
+)
 
 var (
 	// preCommitRev matches the pinned revision of the upstream hook repository.
@@ -18,6 +29,10 @@ var (
 
 	// pyRequirement splits "mkdocs-materialx[imaging]~=10.1" into its parts.
 	pyRequirement = regexp.MustCompile(`^([A-Za-z0-9._-]+)(\[[^\]]*\])?~=(\d+)\.(\d+)$`)
+
+	// providerPin captures the body of the tested provider release list.
+	providerPin   = regexp.MustCompile(`(?m)^(var terraformProviderVersions = \[\]string\{)([^}]*)\}`)
+	quotedVersion = regexp.MustCompile(`"([^"]+)"`)
 )
 
 // applyOapiCodegen moves the generator pin in the Makefile. The generate area
@@ -210,6 +225,106 @@ func bumpRequirement(ctx context.Context, line string) (string, string, error) {
 		fmt.Sprintf("%s ~=%s -> ~=%s", name, have, want), nil
 }
 
+// newestProviders is the newest testedProviders releases of the
+// tailscale/tailscale Terraform provider, oldest first.
+func newestProviders(ctx context.Context) ([]string, error) {
+	body, err := fetch(ctx, providerRegistryURL)
+	if err != nil {
+		return nil, err
+	}
+
+	var listing struct {
+		Versions []struct {
+			Version string `json:"version"`
+		} `json:"versions"`
+	}
+
+	if err := json.Unmarshal(body, &listing); err != nil { //nolint:noinlineerr
+		return nil, fmt.Errorf("decoding provider versions: %w", err)
+	}
+
+	versions := make([]string, 0, len(listing.Versions))
+	for _, v := range listing.Versions {
+		versions = append(versions, v.Version)
+	}
+
+	newest := newestReleases(versions, testedProviders)
+	if len(newest) == 0 {
+		return nil, fmt.Errorf("%w: tailscale/tailscale provider release", errNoMatchingTag)
+	}
+
+	return newest, nil
+}
+
+// newestReleases is the newest n stable versions, oldest first. A
+// prerelease is not something users run, so it is never tested.
+func newestReleases(versions []string, n int) []string {
+	stable := slices.DeleteFunc(slices.Clone(versions), func(v string) bool {
+		return !semver.IsValid("v"+v) || semver.Prerelease("v"+v) != ""
+	})
+
+	slices.SortFunc(stable, func(a, b string) int { return semver.Compare("v"+a, "v"+b) })
+
+	return stable[max(0, len(stable)-n):]
+}
+
+// currentProviders reads the tested provider releases out of servertest.
+func currentProviders(r *repo) ([]string, error) {
+	content, err := r.readFile(providerPinFile)
+	if err != nil {
+		return nil, err
+	}
+
+	m := providerPin.FindStringSubmatch(content)
+	if m == nil {
+		return nil, fmt.Errorf("%w: terraformProviderVersions in %s", errNoMatchingTag, providerPinFile)
+	}
+
+	var versions []string
+	for _, q := range quotedVersion.FindAllStringSubmatch(m[2], -1) {
+		versions = append(versions, q[1])
+	}
+
+	return versions, nil
+}
+
+// applyProviders moves the tested Terraform provider releases to the newest
+// ones, so a provider release that breaks headscale shows up here rather than
+// on whichever pull request runs first after it.
+func applyProviders(ctx context.Context, r *repo) (change, error) {
+	have, err := currentProviders(r)
+	if err != nil {
+		return change{}, err
+	}
+
+	want, err := newestProviders(ctx)
+	if err != nil {
+		return change{}, err
+	}
+
+	if slices.Equal(have, want) {
+		return change{Empty: true}, nil
+	}
+
+	content, err := r.readFile(providerPinFile)
+	if err != nil {
+		return change{}, err
+	}
+
+	list := `"` + strings.Join(want, `", "`) + `"`
+
+	err = r.writeFile(providerPinFile, providerPin.ReplaceAllString(content, "${1}"+list+"}"))
+	if err != nil {
+		return change{}, err
+	}
+
+	return change{
+		Summary: "Terraform provider " + strings.Join(want, ", "),
+		Detail: []string{fmt.Sprintf("%s terraformProviderVersions %s -> %s",
+			providerPinFile, strings.Join(have, ", "), strings.Join(want, ", "))},
+	}, nil
+}
+
 func toolAreas() []area {
 	return []area{
 		{
@@ -228,5 +343,10 @@ func toolAreas() []area {
 			Apply:   applyDocsRequirements,
 			Message: func(c change) string { return "docs: raise " + c.Summary },
 		},
+		{
+			Name:    "tools:terraform-provider",
+			Apply:   applyProviders,
+			Message: func(c change) string { return "servertest: test " + c.Summary },
+		},
 	}
 }

tools/bump/tools_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"slices"
 	"strings"
 	"testing"
 )
@@ -76,3 +77,41 @@ func TestPreCommitRevRewrite(t *testing.T) {
 		t.Error("rewrite disturbed the local hooks block")
 	}
 }
+
+func TestNewestReleases(t *testing.T) {
+	versions := []string{"0.9.0", "0.29.3", "0.10.0", "0.30.0-rc1", "0.29.1", "latest", "0.29.2"}
+
+	got := newestReleases(versions, 3)
+	if want := []string{"0.29.1", "0.29.2", "0.29.3"}; !slices.Equal(got, want) {
+		t.Errorf("newestReleases = %v, want %v", got, want)
+	}
+
+	if got := newestReleases([]string{"0.1.0"}, 3); !slices.Equal(got, []string{"0.1.0"}) {
+		t.Errorf("fewer releases than wanted = %v, want all of them", got)
+	}
+}
+
+// TestProviderPinInTree reads the real pin, so renaming the list in servertest
+// fails here rather than in the nightly bump.
+func TestProviderPinInTree(t *testing.T) {
+	r := &repo{Root: "../.."}
+
+	have, err := currentProviders(r)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(have) != testedProviders {
+		t.Fatalf("servertest tests %d provider releases, want %d: %v", len(have), testedProviders, have)
+	}
+
+	content, err := r.readFile(providerPinFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	got := providerPin.ReplaceAllString(content, `${1}"9.9.9"}`)
+	if !strings.Contains(got, `var terraformProviderVersions = []string{"9.9.9"}`) {
+		t.Error("rewrite did not land on the list")
+	}
+}

tools/bump/verify.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"log"
+	"slices"
 	"strings"
 
 	"golang.org/x/mod/semver"
@@ -30,6 +31,7 @@ func cmdVerify(ctx context.Context) error {
 	findings = append(findings, verifyBuilders(ctx, r)...)
 	findings = append(findings, verifyToolchain(ctx, r)...)
 	findings = append(findings, verifyVendorHash(ctx, r)...)
+	findings = append(findings, verifyProviders(ctx, r)...)
 
 	if len(findings) == 0 {
 		log.Print("all version pins agree")
@@ -136,3 +138,24 @@ func verifyVendorHash(ctx context.Context, r *repo) []finding {
 
 	return nil
 }
+
+// verifyProviders reports a tested provider set that has fallen behind the
+// registry. The tests never resolve it themselves, so nothing else notices.
+func verifyProviders(ctx context.Context, r *repo) []finding {
+	have, err := currentProviders(r)
+	if err != nil {
+		return []finding{finding(err.Error())}
+	}
+
+	want, err := newestProviders(ctx)
+	if err != nil {
+		return []finding{finding("could not resolve the Terraform provider releases: " + err.Error())}
+	}
+
+	if !slices.Equal(have, want) {
+		return []finding{finding(fmt.Sprintf("%s tests Terraform provider %s, but the newest %d releases are %s",
+			providerPinFile, strings.Join(have, ", "), testedProviders, strings.Join(want, ", ")))}
+	}
+
+	return nil
+}