From 08c7ca0fa274284b1b6f8792703994c2b9e8d94b Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Fri, 4 Sep 2026 14:44:06 +0000 Subject: [PATCH] tools/bump: bump the remaining container base images Debian is resolved from the numeric tags, which exist only for released versions: forky-slim is published today and is testing. Distroless follows that same release rather than its own repository names, since gcr answers for base-debian99 as readily as for base-debian13. --- tools/bump/area.go | 32 +++-- tools/bump/images.go | 257 ++++++++++++++++++++++++++++++++++++++ tools/bump/images_test.go | 115 +++++++++++++++++ tools/bump/main.go | 2 +- tools/bump/plan.go | 26 ++++ tools/bump/registry.go | 186 +++++++++++++++++++++++++++ 6 files changed, 605 insertions(+), 13 deletions(-) create mode 100644 tools/bump/images.go create mode 100644 tools/bump/images_test.go create mode 100644 tools/bump/registry.go diff --git a/tools/bump/area.go b/tools/bump/area.go index 7702a522..02cbbcc8 100644 --- a/tools/bump/area.go +++ b/tools/bump/area.go @@ -49,36 +49,44 @@ type result struct { Commit string } -func coreAreas() []area { - return []area{ +// allAreas is the running order. Regeneration comes last because it consumes +// what everything before it settled: the toolchain from the lock, the module +// versions, and the generator pin from the Makefile. +func allAreas() []area { + areas := []area{ { Name: "flake", Apply: applyFlake, Gate: gateFlake, Message: func(c change) string { return "flake.lock: update " + c.Summary }, }, - { + } + + areas = append(areas, + area{ Name: "gomod", Needs: []string{"flake"}, Apply: applyGoMod, Gate: gateGoMod, Message: func(c change) string { return "go.mod: " + c.Summary }, }, - { + area{ Name: "docker-go", Needs: []string{"flake"}, Apply: applyDockerGo, Gate: gateDockerGo, Message: func(c change) string { return "Dockerfile: bump " + c.Summary }, }, - { - Name: "generate", - Needs: []string{"gomod"}, - Apply: applyGenerate, - Gate: gateGenerate, - Message: func(change) string { return "all: regenerate generated files" }, - }, - } + ) + areas = append(areas, imageAreas()...) + + return append(areas, area{ + Name: "generate", + Needs: []string{"gomod"}, + Apply: applyGenerate, + Gate: gateGenerate, + Message: func(change) string { return "all: regenerate generated files" }, + }) } // runAreas applies each area in order, committing the ones that hold and diff --git a/tools/bump/images.go b/tools/bump/images.go new file mode 100644 index 00000000..b1784f5f --- /dev/null +++ b/tools/bump/images.go @@ -0,0 +1,257 @@ +package main + +import ( + "context" + "fmt" + "regexp" + "strconv" + "strings" +) + +// imageBump is a base image reference the bot keeps current. Each becomes its +// own area, and therefore its own commit, so a distribution jump can be dropped +// without taking the rest of the day's bump with it. +type imageBump struct { + Name string + // Prefix is the commit subject's package field. + Prefix string + Files []string + // Ref captures the reference in two groups: everything up to the value, + // and the value itself. Only the second is rewritten. + Ref *regexp.Regexp + // Resolve returns the value that should be pinned, given the current one. + Resolve func(ctx context.Context, have string) (string, error) + // Verify proves the resolved reference is actually published, so a bump + // cannot point CI at an image that does not exist yet. + Verify func(ctx context.Context, want string) error +} + +// Every reference is captured whole, so a value that folds two things together +// (a Rust version and the Debian codename it is built on) moves as one piece. +var ( + alpineRef = regexp.MustCompile(`(?m)^(FROM\s+alpine:)(\S+)`) + debianRef = regexp.MustCompile(`(?m)^(FROM\s+debian:)(\S+)`) + nodeRef = regexp.MustCompile(`(?m)^(FROM\s+node:)(\S+)`) + rustRef = regexp.MustCompile(`(?m)^(FROM\s+rust:)(\S+)`) + distrolessRef = regexp.MustCompile(`(gcr\.io/distroless/base-debian)(\d+)`) + + alpineTag = regexp.MustCompile(`^(\d+\.\d+)$`) + nodeTag = regexp.MustCompile(`^(\d+)-alpine$`) +) + +func imageBumps() []imageBump { + return []imageBump{ + { + Name: "alpine", + Prefix: "Dockerfile", + Files: []string{"Dockerfile.derper", "Dockerfile.tailscale-HEAD"}, + Ref: alpineRef, + Resolve: func(ctx context.Context, _ string) (string, error) { + return highestTag(ctx, "alpine", "3.", alpineTag, nil) + }, + Verify: func(ctx context.Context, want string) error { + return tagExists(ctx, "alpine", want) + }, + }, + { + Name: "debian", + Prefix: "Dockerfile", + Files: []string{"Dockerfile.integration", "Dockerfile.integration-ci", "Dockerfile.tailscale-rs"}, + Ref: debianRef, + Resolve: func(ctx context.Context, _ string) (string, error) { + code, err := debianStableCodename(ctx) + if err != nil { + return "", err + } + + return code + "-slim", nil + }, + Verify: func(ctx context.Context, want string) error { + return tagExists(ctx, "debian", want) + }, + }, + { + Name: "node", + Prefix: "Dockerfile", + Files: []string{"Dockerfile.wasmclient"}, + Ref: nodeRef, + Resolve: func(ctx context.Context, _ string) (string, error) { + // Node ships even majors as long-term support; an odd major is + // a short-lived Current release, not something to pin to. + major, err := highestTag(ctx, "node", "-alpine", nodeTag, isEvenMajor) + if err != nil { + return "", err + } + + return major + "-alpine", nil + }, + Verify: func(ctx context.Context, want string) error { + return tagExists(ctx, "node", want) + }, + }, + { + Name: "rust", + Prefix: "Dockerfile", + Files: []string{"Dockerfile.tailscale-rs"}, + Ref: rustRef, + // The Rust tag carries the Debian codename it is built on, so it + // has to follow whatever the debian area settles on. + Resolve: func(ctx context.Context, _ string) (string, error) { + code, err := debianStableCodename(ctx) + if err != nil { + return "", err + } + + pattern := regexp.MustCompile(`^(\d+\.\d+)-` + code + `$`) + + version, err := highestTag(ctx, "rust", "-"+code, pattern, nil) + if err != nil { + return "", err + } + + return version + "-" + code, nil + }, + Verify: func(ctx context.Context, want string) error { + return tagExists(ctx, "rust", want) + }, + }, + { + Name: "distroless", + Prefix: "ko", + Files: []string{".goreleaser.yml", ".github/workflows/container-main.yml"}, + Ref: distrolessRef, + Resolve: resolveDistroless, + Verify: func(ctx context.Context, want string) error { + if !gcrRepositoryPublished(ctx, "distroless/base-debian"+want) { + return fmt.Errorf("%w: distroless/base-debian%s", errTagMissing, want) + } + + return nil + }, + }, + } +} + +func isEvenMajor(v string) bool { + n, err := strconv.Atoi(v) + + return err == nil && n%2 == 0 +} + +// resolveDistroless follows Debian stable, not whatever repository names +// happen to resolve. Distroless carries the release in the repository name and +// creates the next one well before it has anything in it, so "newer name +// exists" is not the same question as "newer base is usable". +func resolveDistroless(ctx context.Context, have string) (string, error) { + stable, err := debianStableMajor(ctx) + if err != nil { + return "", err + } + + // Distroless can lag a Debian release; staying put is the right answer + // until it catches up. + if !gcrRepositoryPublished(ctx, "distroless/base-debian"+stable) { + return have, nil + } + + return stable, nil +} + +// currentImageRef reads the pinned value out of the first file that has one. +func currentImageRef(r *repo, def imageBump) (string, error) { + for _, file := range def.Files { + content, err := r.readFile(file) + if err != nil { + return "", err + } + + m := def.Ref.FindStringSubmatch(content) + if m != nil { + return m[2], nil + } + } + + return "", fmt.Errorf("%w: %s in %s", errNoMatchingTag, def.Name, strings.Join(def.Files, ", ")) +} + +func applyImage(def imageBump) func(context.Context, *repo) (change, error) { + return func(ctx context.Context, r *repo) (change, error) { + have, err := currentImageRef(r, def) + if err != nil { + return change{}, err + } + + want, err := def.Resolve(ctx, have) + if err != nil { + return change{}, err + } + + if want == have { + return change{Empty: true}, nil + } + + err = def.Verify(ctx, want) + if err != nil { + return change{}, err + } + + var touched []string + + for _, file := range def.Files { + content, err := r.readFile(file) + if err != nil { + return change{}, err + } + + updated := def.Ref.ReplaceAllString(content, "${1}"+want) + if updated == content { + continue + } + + err = r.writeFile(file, updated) + if err != nil { + return change{}, err + } + + touched = append(touched, file) + } + + if len(touched) == 0 { + return change{Empty: true}, nil + } + + return change{ + Summary: fmt.Sprintf("%s %s to %s", def.Name, have, want), + Detail: []string{fmt.Sprintf("%s %s -> %s in %s", def.Name, have, want, strings.Join(touched, ", "))}, + }, nil + } +} + +// gateImage re-reads the written value and re-checks it is published, so a bad +// rewrite is caught before it is committed. +func gateImage(def imageBump) func(context.Context, *repo) error { + return func(ctx context.Context, r *repo) error { + have, err := currentImageRef(r, def) + if err != nil { + return err + } + + return def.Verify(ctx, have) + } +} + +func imageAreas() []area { + defs := imageBumps() + areas := make([]area, 0, len(defs)) + + for _, def := range defs { + areas = append(areas, area{ + Name: "image:" + def.Name, + Apply: applyImage(def), + Gate: gateImage(def), + Message: func(c change) string { return def.Prefix + ": bump " + c.Summary }, + }) + } + + return areas +} diff --git a/tools/bump/images_test.go b/tools/bump/images_test.go new file mode 100644 index 00000000..0234ffae --- /dev/null +++ b/tools/bump/images_test.go @@ -0,0 +1,115 @@ +package main + +import ( + "regexp" + "testing" +) + +func TestImageRefPatterns(t *testing.T) { + tests := []struct { + name string + content string + want string + bumped string + }{ + { + name: "alpine", + content: "FROM alpine:3.23\nRUN true\n", + want: "3.23", + bumped: "FROM alpine:3.24\nRUN true\n", + }, + { + name: "debian slim", + content: "FROM debian:trixie-slim\n", + want: "trixie-slim", + bumped: "FROM debian:3.24\n", + }, + { + name: "node alpine", + content: "FROM node:24-alpine\n", + want: "24-alpine", + bumped: "FROM node:3.24\n", + }, + { + // The Rust tag folds the version and the Debian codename together, + // so it has to be captured and replaced as one value. + name: "rust on debian", + content: "FROM rust:1.95-trixie AS builder\n", + want: "1.95-trixie", + bumped: "FROM rust:3.24 AS builder\n", + }, + } + + refs := map[string]*regexp.Regexp{ + "alpine": alpineRef, + "debian slim": debianRef, + "node alpine": nodeRef, + "rust on debian": rustRef, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + re := refs[test.name] + + m := re.FindStringSubmatch(test.content) + if m == nil { + t.Fatalf("pattern did not match %q", test.content) + } + + if m[2] != test.want { + t.Errorf("captured %q, want %q", m[2], test.want) + } + + if got := re.ReplaceAllString(test.content, "${1}3.24"); got != test.bumped { + t.Errorf("rewrite = %q, want %q", got, test.bumped) + } + }) + } +} + +// The Rust builder and the runtime stage live in the same file; matching the +// wrong one would rewrite a Debian tag with a Rust version. +func TestImageRefsDoNotCrossMatch(t *testing.T) { + const tailscaleRS = "FROM rust:1.95-trixie AS builder\nRUN true\nFROM debian:trixie-slim\n" + + if got := rustRef.FindStringSubmatch(tailscaleRS)[2]; got != "1.95-trixie" { + t.Errorf("rustRef captured %q", got) + } + + if got := debianRef.FindStringSubmatch(tailscaleRS)[2]; got != "trixie-slim" { + t.Errorf("debianRef captured %q", got) + } +} + +func TestDistrolessRef(t *testing.T) { + const goreleaser = " base_image: gcr.io/distroless/base-debian13\n" + + " base_image: gcr.io/distroless/base-debian13:debug\n" + + got := distrolessRef.ReplaceAllString(goreleaser, "${1}14") + + want := " base_image: gcr.io/distroless/base-debian14\n" + + " base_image: gcr.io/distroless/base-debian14:debug\n" + if got != want { + t.Errorf("rewrite =\n%q\nwant\n%q", got, want) + } +} + +func TestIsEvenMajor(t *testing.T) { + tests := []struct { + in string + want bool + }{ + {in: "24", want: true}, + {in: "26", want: true}, + {in: "25"}, + {in: "lts"}, + } + + for _, test := range tests { + t.Run(test.in, func(t *testing.T) { + if got := isEvenMajor(test.in); got != test.want { + t.Errorf("isEvenMajor(%q) = %v, want %v", test.in, got, test.want) + } + }) + } +} diff --git a/tools/bump/main.go b/tools/bump/main.go index 08227191..0aebe2a4 100644 --- a/tools/bump/main.go +++ b/tools/bump/main.go @@ -116,7 +116,7 @@ func cmdRun(ctx context.Context) error { return err } - results, err := runAreas(ctx, r, coreAreas(), selector(runCfg.Areas, runCfg.Skip)) + results, err := runAreas(ctx, r, allAreas(), selector(runCfg.Areas, runCfg.Skip)) if err != nil { return err } diff --git a/tools/bump/plan.go b/tools/bump/plan.go index da1b84cd..12072c7f 100644 --- a/tools/bump/plan.go +++ b/tools/bump/plan.go @@ -26,6 +26,7 @@ func planLines(ctx context.Context, r *repo) []string { lines = append(lines, planBuilders(ctx, r)...) lines = append(lines, planLockstep(ctx, r)...) + lines = append(lines, planImages(ctx, r)...) if version, err := oapiVersion(r); err != nil { //nolint:noinlineerr lines = append(lines, "oapi-codegen: "+err.Error()) @@ -38,6 +39,31 @@ func planLines(ctx context.Context, r *repo) []string { return lines } +func planImages(ctx context.Context, r *repo) []string { + defs := imageBumps() + lines := make([]string, 0, len(defs)) + + for _, def := range defs { + have, err := currentImageRef(r, def) + if err != nil { + lines = append(lines, def.Name+": "+err.Error()) + + continue + } + + want, err := def.Resolve(ctx, have) + if err != nil { + lines = append(lines, def.Name+": "+err.Error()) + + continue + } + + lines = append(lines, gap(def.Name, have, want)) + } + + return lines +} + func planBuilders(ctx context.Context, r *repo) []string { var lines []string diff --git a/tools/bump/registry.go b/tools/bump/registry.go new file mode 100644 index 00000000..3dc7541c --- /dev/null +++ b/tools/bump/registry.go @@ -0,0 +1,186 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "regexp" + + "golang.org/x/mod/semver" +) + +const ( + dockerHubTagsURL = "https://hub.docker.com/v2/repositories/library/%s/tags?page_size=100&name=%s" + dockerHubTagURL2 = "https://hub.docker.com/v2/repositories/library/%s/tags/%s" + gcrTagsURL = "https://gcr.io/v2/%s/tags/list" +) + +var errNoMatchingTag = errors.New("no matching tag published") + +type hubTags struct { + Results []struct { + Name string `json:"name"` + Digest string `json:"digest"` + } `json:"results"` +} + +// dockerHubTags lists tags of an official image whose name contains filter. +// One page is enough: the listing is ordered newest first, and every reference +// this tool tracks is a rolling tag that is rebuilt constantly. +func dockerHubTags(ctx context.Context, image, filter string) ([]string, error) { + body, err := fetch(ctx, fmt.Sprintf(dockerHubTagsURL, image, filter)) + if err != nil { + return nil, err + } + + var tags hubTags + if err := json.Unmarshal(body, &tags); err != nil { //nolint:noinlineerr + return nil, fmt.Errorf("decoding tags of %s: %w", image, err) + } + + names := make([]string, 0, len(tags.Results)) + for _, t := range tags.Results { + names = append(names, t.Name) + } + + return names, nil +} + +// highestTag returns the newest version captured by pattern across an image's +// published tags. pattern must have exactly one capture group. +func highestTag(ctx context.Context, image, filter string, pattern *regexp.Regexp, keep func(string) bool) (string, error) { + names, err := dockerHubTags(ctx, image, filter) + if err != nil { + return "", err + } + + best := "" + + for _, name := range names { + m := pattern.FindStringSubmatch(name) + if m == nil { + continue + } + + if keep != nil && !keep(m[1]) { + continue + } + + if best == "" || semver.Compare("v"+m[1], "v"+best) > 0 { + best = m[1] + } + } + + if best == "" { + return "", fmt.Errorf("%w: %s matching %s", errNoMatchingTag, image, pattern) + } + + return best, nil +} + +// dockerHubDigest is the manifest a tag points at. Two names sharing a digest +// are the same image, which is how a codename is matched to the release it +// currently stands for. +func dockerHubDigest(ctx context.Context, image, tag string) (string, error) { + body, err := fetch(ctx, fmt.Sprintf(dockerHubTagURL2, image, tag)) + if err != nil { + return "", err + } + + var info struct { + Digest string `json:"digest"` + } + + if err := json.Unmarshal(body, &info); err != nil { //nolint:noinlineerr + return "", fmt.Errorf("decoding %s:%s: %w", image, tag, err) + } + + if info.Digest == "" { + return "", fmt.Errorf("%w: %s:%s has no digest", errNoMatchingTag, image, tag) + } + + return info.Digest, nil +} + +var debianSlimMajor = regexp.MustCompile(`^(\d+)-slim$`) + +// debianStableMajor is the newest released Debian version. +// +// The numeric tags are the signal: Debian publishes 11-slim, 12-slim and +// 13-slim, but nothing numeric for the release under development. Codename tags +// cannot be used for this, because forky-slim exists today and is testing. +func debianStableMajor(ctx context.Context) (string, error) { + return highestTag(ctx, "debian", "-slim", debianSlimMajor, nil) +} + +// debianStableCodename is the codename of the newest released Debian, found by +// matching the numeric tag to the codename tag that carries the same image. +// +// stable-slim is not usable here: Docker Hub builds it separately, so it has a +// different digest from the release it aliases. +func debianStableCodename(ctx context.Context) (string, error) { + major, err := debianStableMajor(ctx) + if err != nil { + return "", err + } + + want, err := dockerHubDigest(ctx, "debian", major+"-slim") + if err != nil { + return "", err + } + + body, err := fetch(ctx, fmt.Sprintf(dockerHubTagsURL, "debian", "-slim")) + if err != nil { + return "", err + } + + var tags hubTags + if err := json.Unmarshal(body, &tags); err != nil { //nolint:noinlineerr + return "", fmt.Errorf("decoding debian tags: %w", err) + } + + codename := regexp.MustCompile(`^([a-z]+)-slim$`) + + for _, t := range tags.Results { + m := codename.FindStringSubmatch(t.Name) + // stable, testing and friends are moving aliases, not codenames. + if m == nil || t.Digest != want || isDebianAlias(m[1]) { + continue + } + + return m[1], nil + } + + return "", fmt.Errorf("%w: no debian codename matches %s-slim", errNoMatchingTag, major) +} + +func isDebianAlias(name string) bool { + switch name { + case "stable", "testing", "unstable", "oldstable", "oldoldstable", "sid", "experimental": + return true + default: + return false + } +} + +// gcrRepositoryPublished reports whether a Google Container Registry repository +// has any images. Existence alone is not enough: gcr answers 200 with an empty +// tag list for a repository that was never pushed, so base-debian99 looks just +// as real as base-debian13. +func gcrRepositoryPublished(ctx context.Context, repo string) bool { + body, err := fetch(ctx, fmt.Sprintf(gcrTagsURL, repo)) + if err != nil { + return false + } + + var listing struct { + Tags []string `json:"tags"` + } + + if err := json.Unmarshal(body, &listing); err != nil { //nolint:noinlineerr + return false + } + + return len(listing.Tags) > 0 +}