From 98e9ff4d363f2db5d5e29d40d164944679c889e0 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Wed, 13 May 2026 13:16:24 +0000 Subject: [PATCH] integration: authenticate Docker Hub pulls and retry transient errors Anonymous Hub pulls trip the 100/6h IP cap on shared CI runners, turning into singleton FAIL reports whenever the runner egress IP crosses the quota. Route every pull through Docker Hub credentials when present, and retry transient errors with backoff. tsic and hi use the same helper so both surfaces honour ~/.docker/config.json and the GHA secrets. --- .../workflows/integration-test-template.yml | 15 ++ .github/workflows/test-integration.yaml | 18 ++ cmd/hi/docker.go | 63 +++++-- cmd/hi/doctor.go | 27 +++ integration/dockertestutil/auth.go | 171 ++++++++++++++++++ integration/tsic/tsic.go | 12 ++ 6 files changed, 290 insertions(+), 16 deletions(-) create mode 100644 integration/dockertestutil/auth.go diff --git a/.github/workflows/integration-test-template.yml b/.github/workflows/integration-test-template.yml index efaaac4c..427f63e2 100644 --- a/.github/workflows/integration-test-template.yml +++ b/.github/workflows/integration-test-template.yml @@ -73,6 +73,15 @@ jobs: echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json sudo systemctl restart docker docker version + - name: Login to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }} + if: env.DOCKERHUB_USERNAME != '' + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + username: ${{ env.DOCKERHUB_USERNAME }} + password: ${{ env.DOCKERHUB_TOKEN }} - name: Load Docker images, Go cache, and prepare binary run: | gunzip -c /tmp/artifacts/headscale-image.tar.gz | docker load @@ -93,6 +102,12 @@ jobs: HEADSCALE_INTEGRATION_POSTGRES_IMAGE: ${{ inputs.postgres_flag == '--postgres=1' && format('postgres:{0}', github.sha) || '' }} HEADSCALE_INTEGRATION_GO_CACHE: /tmp/go-cache/go HEADSCALE_INTEGRATION_GO_BUILD_CACHE: /tmp/go-cache/.cache/go-build + # Mirror the docker/login-action secrets into env so the + # dockertestutil.Credentials resolver picks them up directly + # (otherwise it falls back to parsing ~/.docker/config.json, + # which works but is one step further from the source). + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }} run: /tmp/artifacts/hi run --stats --ts-memory-limit=300 --hs-memory-limit=1500 "^${{ inputs.test }}$" \ --timeout=120m \ ${{ inputs.postgres_flag }} diff --git a/.github/workflows/test-integration.yaml b/.github/workflows/test-integration.yaml index 67fe0c47..f257ae0e 100644 --- a/.github/workflows/test-integration.yaml +++ b/.github/workflows/test-integration.yaml @@ -80,6 +80,15 @@ jobs: echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json sudo systemctl restart docker docker version + - name: Login to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }} + if: env.DOCKERHUB_USERNAME != '' + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + username: ${{ env.DOCKERHUB_USERNAME }} + password: ${{ env.DOCKERHUB_TOKEN }} - name: Build headscale image if: steps.changed-files.outputs.files == 'true' run: | @@ -121,6 +130,15 @@ jobs: echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json sudo systemctl restart docker docker version + - name: Login to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }} + if: env.DOCKERHUB_USERNAME != '' + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + username: ${{ env.DOCKERHUB_USERNAME }} + password: ${{ env.DOCKERHUB_TOKEN }} - name: Pull and save postgres image run: | docker pull postgres:latest diff --git a/cmd/hi/docker.go b/cmd/hi/docker.go index 060057a9..03778bf7 100644 --- a/cmd/hi/docker.go +++ b/cmd/hi/docker.go @@ -14,6 +14,7 @@ import ( "strings" "time" + "github.com/cenkalti/backoff/v5" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/image" "github.com/docker/docker/api/types/mount" @@ -522,9 +523,9 @@ func checkImageAvailableLocally(ctx context.Context, cli *client.Client, imageNa return true, nil } -// ensureImageAvailable checks if the image is available locally first, then pulls if needed. +// ensureImageAvailable pulls imageName if missing, using Docker Hub +// credentials and retrying transient errors. func ensureImageAvailable(ctx context.Context, cli *client.Client, imageName string, verbose bool) error { - // First check if image is available locally available, err := checkImageAvailableLocally(ctx, cli, imageName) if err != nil { return fmt.Errorf("checking local image availability: %w", err) @@ -538,34 +539,64 @@ func ensureImageAvailable(ctx context.Context, cli *client.Client, imageName str return nil } - // Image not available locally, try to pull it if verbose { log.Printf("Image %s not found locally, pulling...", imageName) } - reader, err := cli.ImagePull(ctx, imageName, image.PullOptions{}) + registryAuth, err := dockertestutil.RegistryAuth() if err != nil { - return fmt.Errorf("pulling image %s: %w", imageName, err) + return fmt.Errorf("resolving registry auth: %w", err) } - defer reader.Close() - if verbose { - _, err = io.Copy(os.Stdout, reader) - if err != nil { - return fmt.Errorf("reading pull output: %w", err) - } - } else { - _, err = io.Copy(io.Discard, reader) - if err != nil { - return fmt.Errorf("reading pull output: %w", err) - } + _, err = backoff.Retry( + ctx, + func() (struct{}, error) { + reader, pullErr := cli.ImagePull(ctx, imageName, image.PullOptions{RegistryAuth: registryAuth}) + if pullErr != nil { + if isPermanentDockerPullError(pullErr) { + return struct{}{}, backoff.Permanent(pullErr) + } + return struct{}{}, fmt.Errorf("pulling image %s: %w", imageName, pullErr) + } + defer reader.Close() + + sink := io.Discard + if verbose { + sink = os.Stdout + } + + _, copyErr := io.Copy(sink, reader) + if copyErr != nil { + return struct{}{}, fmt.Errorf("reading pull output: %w", copyErr) + } + + return struct{}{}, nil + }, + backoff.WithBackOff(backoff.NewExponentialBackOff()), + backoff.WithMaxElapsedTime(60*time.Second), + ) + if err != nil { + return err + } + + if !verbose { log.Printf("Image %s pulled successfully", imageName) } return nil } +func isPermanentDockerPullError(err error) bool { + msg := strings.ToLower(err.Error()) + + return strings.Contains(msg, "manifest unknown") || + strings.Contains(msg, "manifest not found") || + strings.Contains(msg, "repository does not exist") || + strings.Contains(msg, "name unknown") || + strings.Contains(msg, "no such image") +} + // listControlFiles displays the headscale test artifacts created in the control logs directory. func listControlFiles(logsDir string) { entries, err := os.ReadDir(logsDir) diff --git a/cmd/hi/doctor.go b/cmd/hi/doctor.go index 1791d66d..e235cf09 100644 --- a/cmd/hi/doctor.go +++ b/cmd/hi/doctor.go @@ -7,6 +7,8 @@ import ( "log" "os/exec" "strings" + + "github.com/juanfont/headscale/integration/dockertestutil" ) var ErrSystemChecksFailed = errors.New("system checks failed") @@ -34,6 +36,7 @@ func runDoctorCheck(ctx context.Context) error { if dockerResult.Status == "PASS" { results = append(results, checkDockerContext(ctx)) results = append(results, checkDockerSocket(ctx)) + results = append(results, checkDockerHubCredentials()) results = append(results, checkGolangImage(ctx)) } @@ -190,6 +193,30 @@ func checkDockerSocket(ctx context.Context) DoctorResult { } } +// checkDockerHubCredentials warns when pulls would be anonymous and +// therefore rate-limited. +func checkDockerHubCredentials() DoctorResult { + _, _, source := dockertestutil.Credentials() + if source == dockertestutil.CredentialSourceAnonymous { + return DoctorResult{ + Name: "Docker Hub Credentials", + Status: "WARN", + Message: "No Docker Hub credentials found — pulls will be rate-limited (100/6h per IP)", + Suggestions: []string{ + "Run: docker login", + "Or export DOCKERHUB_USERNAME and DOCKERHUB_TOKEN", + "In CI: ensure the docker/login-action step is configured with secrets", + }, + } + } + + return DoctorResult{ + Name: "Docker Hub Credentials", + Status: "PASS", + Message: fmt.Sprintf("Credentials available (source: %s)", source), + } +} + // checkGolangImage verifies the golang Docker image is available locally or can be pulled. func checkGolangImage(ctx context.Context) DoctorResult { cli, err := createDockerClient(ctx) diff --git a/integration/dockertestutil/auth.go b/integration/dockertestutil/auth.go new file mode 100644 index 00000000..3962204d --- /dev/null +++ b/integration/dockertestutil/auth.go @@ -0,0 +1,171 @@ +package dockertestutil + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/cenkalti/backoff/v5" + "github.com/ory/dockertest/v3" + "github.com/ory/dockertest/v3/docker" +) + +const dockerHubServer = "https://index.docker.io/v1/" + +type CredentialSource string + +const ( + CredentialSourceEnv CredentialSource = "env" + CredentialSourceConfig CredentialSource = "config" + CredentialSourceAnonymous CredentialSource = "anonymous" +) + +// Credentials resolves Docker Hub credentials from +// DOCKERHUB_USERNAME/DOCKERHUB_TOKEN, then ~/.docker/config.json, then +// anonymous. The Docker Go SDKs do not read config.json on their own. +func Credentials() (string, string, CredentialSource) { + if u := os.Getenv("DOCKERHUB_USERNAME"); u != "" { + return u, os.Getenv("DOCKERHUB_TOKEN"), CredentialSourceEnv + } + + user, pass, ok := credentialsFromConfig() + if ok { + return user, pass, CredentialSourceConfig + } + + return "", "", CredentialSourceAnonymous +} + +// AuthConfiguration returns Docker Hub auth for the dockertest pool. +func AuthConfiguration() docker.AuthConfiguration { + u, p, _ := Credentials() + + return docker.AuthConfiguration{ + Username: u, + Password: p, + ServerAddress: dockerHubServer, + } +} + +// RegistryAuth returns base64-encoded credentials for the modern +// Docker SDK's image.PullOptions{RegistryAuth: ...}, or "" when none. +func RegistryAuth() (string, error) { + u, p, _ := Credentials() + if u == "" && p == "" { + return "", nil + } + + auth := struct { + Username string `json:"username"` + Password string `json:"password"` + }{Username: u, Password: p} + + b, err := json.Marshal(auth) //nolint:gosec // G117: password field holds the Docker Hub token, intentional + if err != nil { + return "", fmt.Errorf("marshalling docker auth: %w", err) + } + + return base64.URLEncoding.EncodeToString(b), nil +} + +// PullWithAuth ensures imageRef is local, pulling with auth and +// retrying transient errors when it is not. +func PullWithAuth(pool *dockertest.Pool, imageRef string) error { + if img, _ := pool.Client.InspectImage(imageRef); img != nil { + return nil + } + + repo, tag := splitImageRef(imageRef) + auth := AuthConfiguration() + + _, err := backoff.Retry( + context.Background(), + func() (struct{}, error) { + err := pool.Client.PullImage(docker.PullImageOptions{ + Repository: repo, + Tag: tag, + }, auth) + if err == nil { + return struct{}{}, nil + } + + if isPermanentPullError(err) { + return struct{}{}, backoff.Permanent(err) + } + + return struct{}{}, fmt.Errorf("pulling %s: %w", imageRef, err) + }, + backoff.WithBackOff(backoff.NewExponentialBackOff()), + backoff.WithMaxElapsedTime(60*time.Second), + ) + if err != nil { + return fmt.Errorf("pulling %s with auth (source=%s): %w", imageRef, AuthConfiguration().ServerAddress, err) + } + + return nil +} + +func splitImageRef(ref string) (string, string) { + if i := strings.LastIndex(ref, ":"); i >= 0 { + return ref[:i], ref[i+1:] + } + + return ref, "latest" +} + +func isPermanentPullError(err error) bool { + msg := strings.ToLower(err.Error()) + + return strings.Contains(msg, "manifest unknown") || + strings.Contains(msg, "manifest not found") || + strings.Contains(msg, "repository does not exist") || + strings.Contains(msg, "name unknown") || + strings.Contains(msg, "no such image") +} + +// credentialsFromConfig reads the Hub entry from ~/.docker/config.json. +// Credential helpers (osxkeychain etc.) are not supported; use env vars. +func credentialsFromConfig() (string, string, bool) { + home, err := os.UserHomeDir() + if err != nil { + return "", "", false + } + + raw, err := os.ReadFile(filepath.Join(home, ".docker", "config.json")) + if err != nil { + return "", "", false + } + + var cfg struct { + Auths map[string]struct { + Auth string `json:"auth"` + } `json:"auths"` + } + + err = json.Unmarshal(raw, &cfg) + if err != nil { + return "", "", false + } + + entry, found := cfg.Auths[dockerHubServer] + if !found || entry.Auth == "" { + return "", "", false + } + + decoded, err := base64.StdEncoding.DecodeString(entry.Auth) + if err != nil { + return "", "", false + } + + parts := strings.SplitN(string(decoded), ":", 2) + if len(parts) != 2 { + return "", "", false + } + + return parts[0], parts[1], true +} diff --git a/integration/tsic/tsic.go b/integration/tsic/tsic.go index 4a558a04..fcabb5e9 100644 --- a/integration/tsic/tsic.go +++ b/integration/tsic/tsic.go @@ -523,6 +523,12 @@ func New( tailscaleOptions.Repository = "tailscale/tailscale" tailscaleOptions.Tag = version + err = dockertestutil.PullWithAuth(pool, tailscaleOptions.Repository+":"+tailscaleOptions.Tag) + if err != nil { + //nolint:gosec // G706: tag value is from MustTestVersions, not external input + log.Printf("Pull failed for %s:%s, error: %v", tailscaleOptions.Repository, tailscaleOptions.Tag, err) + } + container, err = pool.RunWithOptions( tailscaleOptions, dockertestutil.DockerRestartPolicy, @@ -537,6 +543,12 @@ func New( tailscaleOptions.Repository = "tailscale/tailscale" tailscaleOptions.Tag = "v" + version + err = dockertestutil.PullWithAuth(pool, tailscaleOptions.Repository+":"+tailscaleOptions.Tag) + if err != nil { + //nolint:gosec // G706: tag value is from MustTestVersions, not external input + log.Printf("Pull failed for %s:%s, error: %v", tailscaleOptions.Repository, tailscaleOptions.Tag, err) + } + container, err = pool.RunWithOptions( tailscaleOptions, dockertestutil.DockerRestartPolicy,