integration: authenticate Docker Hub pulls and retry transient errors

Anonymous Hub pulls trip the 100/6h IP cap on shared CI runners, turning
into singleton FAIL reports whenever the runner egress IP crosses the
quota. Route every pull through Docker Hub credentials when present, and
retry transient errors with backoff. tsic and hi use the same helper so
both surfaces honour ~/.docker/config.json and the GHA secrets.

.github/workflows/integration-test-template.yml

@@ -73,6 +73,15 @@ jobs:
           echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json
           sudo systemctl restart docker
           docker version
+      - name: Login to Docker Hub
+        env:
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }}
+          DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }}
+        if: env.DOCKERHUB_USERNAME != ''
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
+        with:
+          username: ${{ env.DOCKERHUB_USERNAME }}
+          password: ${{ env.DOCKERHUB_TOKEN }}
       - name: Load Docker images, Go cache, and prepare binary
         run: |
           gunzip -c /tmp/artifacts/headscale-image.tar.gz | docker load
@@ -93,6 +102,12 @@ jobs:
           HEADSCALE_INTEGRATION_POSTGRES_IMAGE: ${{ inputs.postgres_flag == '--postgres=1' && format('postgres:{0}', github.sha) || '' }}
           HEADSCALE_INTEGRATION_GO_CACHE: /tmp/go-cache/go
           HEADSCALE_INTEGRATION_GO_BUILD_CACHE: /tmp/go-cache/.cache/go-build
+          # Mirror the docker/login-action secrets into env so the
+          # dockertestutil.Credentials resolver picks them up directly
+          # (otherwise it falls back to parsing ~/.docker/config.json,
+          # which works but is one step further from the source).
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }}
+          DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }}
         run: /tmp/artifacts/hi run --stats --ts-memory-limit=300 --hs-memory-limit=1500 "^${{ inputs.test }}$" \
           --timeout=120m \
           ${{ inputs.postgres_flag }}

.github/workflows/test-integration.yaml

@@ -80,6 +80,15 @@ jobs:
           echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json
           sudo systemctl restart docker
           docker version
+      - name: Login to Docker Hub
+        env:
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }}
+          DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }}
+        if: env.DOCKERHUB_USERNAME != ''
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
+        with:
+          username: ${{ env.DOCKERHUB_USERNAME }}
+          password: ${{ env.DOCKERHUB_TOKEN }}
       - name: Build headscale image
         if: steps.changed-files.outputs.files == 'true'
         run: |
@@ -121,6 +130,15 @@ jobs:
           echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json
           sudo systemctl restart docker
           docker version
+      - name: Login to Docker Hub
+        env:
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }}
+          DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }}
+        if: env.DOCKERHUB_USERNAME != ''
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
+        with:
+          username: ${{ env.DOCKERHUB_USERNAME }}
+          password: ${{ env.DOCKERHUB_TOKEN }}
       - name: Pull and save postgres image
         run: |
           docker pull postgres:latest

cmd/hi/docker.go

@@ -14,6 +14,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/cenkalti/backoff/v5"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/image"
 	"github.com/docker/docker/api/types/mount"
@@ -522,9 +523,9 @@ func checkImageAvailableLocally(ctx context.Context, cli *client.Client, imageNa
 	return true, nil
 }
 
-// ensureImageAvailable checks if the image is available locally first, then pulls if needed.
+// ensureImageAvailable pulls imageName if missing, using Docker Hub
+// credentials and retrying transient errors.
 func ensureImageAvailable(ctx context.Context, cli *client.Client, imageName string, verbose bool) error {
-	// First check if image is available locally
 	available, err := checkImageAvailableLocally(ctx, cli, imageName)
 	if err != nil {
 		return fmt.Errorf("checking local image availability: %w", err)
@@ -538,34 +539,64 @@ func ensureImageAvailable(ctx context.Context, cli *client.Client, imageName str
 		return nil
 	}
 
-	// Image not available locally, try to pull it
 	if verbose {
 		log.Printf("Image %s not found locally, pulling...", imageName)
 	}
 
-	reader, err := cli.ImagePull(ctx, imageName, image.PullOptions{})
+	registryAuth, err := dockertestutil.RegistryAuth()
 	if err != nil {
-		return fmt.Errorf("pulling image %s: %w", imageName, err)
+		return fmt.Errorf("resolving registry auth: %w", err)
 	}
-	defer reader.Close()
 
-	if verbose {
-		_, err = io.Copy(os.Stdout, reader)
-		if err != nil {
-			return fmt.Errorf("reading pull output: %w", err)
-		}
-	} else {
-		_, err = io.Copy(io.Discard, reader)
-		if err != nil {
-			return fmt.Errorf("reading pull output: %w", err)
-		}
+	_, err = backoff.Retry(
+		ctx,
+		func() (struct{}, error) {
+			reader, pullErr := cli.ImagePull(ctx, imageName, image.PullOptions{RegistryAuth: registryAuth})
+			if pullErr != nil {
+				if isPermanentDockerPullError(pullErr) {
+					return struct{}{}, backoff.Permanent(pullErr)
+				}
 
+				return struct{}{}, fmt.Errorf("pulling image %s: %w", imageName, pullErr)
+			}
+			defer reader.Close()
+
+			sink := io.Discard
+			if verbose {
+				sink = os.Stdout
+			}
+
+			_, copyErr := io.Copy(sink, reader)
+			if copyErr != nil {
+				return struct{}{}, fmt.Errorf("reading pull output: %w", copyErr)
+			}
+
+			return struct{}{}, nil
+		},
+		backoff.WithBackOff(backoff.NewExponentialBackOff()),
+		backoff.WithMaxElapsedTime(60*time.Second),
+	)
+	if err != nil {
+		return err
+	}
+
+	if !verbose {
 		log.Printf("Image %s pulled successfully", imageName)
 	}
 
 	return nil
 }
 
+func isPermanentDockerPullError(err error) bool {
+	msg := strings.ToLower(err.Error())
+
+	return strings.Contains(msg, "manifest unknown") ||
+		strings.Contains(msg, "manifest not found") ||
+		strings.Contains(msg, "repository does not exist") ||
+		strings.Contains(msg, "name unknown") ||
+		strings.Contains(msg, "no such image")
+}
+
 // listControlFiles displays the headscale test artifacts created in the control logs directory.
 func listControlFiles(logsDir string) {
 	entries, err := os.ReadDir(logsDir)

cmd/hi/doctor.go

@@ -7,6 +7,8 @@ import (
 	"log"
 	"os/exec"
 	"strings"
+
+	"github.com/juanfont/headscale/integration/dockertestutil"
 )
 
 var ErrSystemChecksFailed = errors.New("system checks failed")
@@ -34,6 +36,7 @@ func runDoctorCheck(ctx context.Context) error {
 	if dockerResult.Status == "PASS" {
 		results = append(results, checkDockerContext(ctx))
 		results = append(results, checkDockerSocket(ctx))
+		results = append(results, checkDockerHubCredentials())
 		results = append(results, checkGolangImage(ctx))
 	}
 
@@ -190,6 +193,30 @@ func checkDockerSocket(ctx context.Context) DoctorResult {
 	}
 }
 
+// checkDockerHubCredentials warns when pulls would be anonymous and
+// therefore rate-limited.
+func checkDockerHubCredentials() DoctorResult {
+	_, _, source := dockertestutil.Credentials()
+	if source == dockertestutil.CredentialSourceAnonymous {
+		return DoctorResult{
+			Name:    "Docker Hub Credentials",
+			Status:  "WARN",
+			Message: "No Docker Hub credentials found — pulls will be rate-limited (100/6h per IP)",
+			Suggestions: []string{
+				"Run: docker login",
+				"Or export DOCKERHUB_USERNAME and DOCKERHUB_TOKEN",
+				"In CI: ensure the docker/login-action step is configured with secrets",
+			},
+		}
+	}
+
+	return DoctorResult{
+		Name:    "Docker Hub Credentials",
+		Status:  "PASS",
+		Message: fmt.Sprintf("Credentials available (source: %s)", source),
+	}
+}
+
 // checkGolangImage verifies the golang Docker image is available locally or can be pulled.
 func checkGolangImage(ctx context.Context) DoctorResult {
 	cli, err := createDockerClient(ctx)

integration/dockertestutil/auth.go

@@ -0,0 +1,171 @@
+package dockertestutil
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/cenkalti/backoff/v5"
+	"github.com/ory/dockertest/v3"
+	"github.com/ory/dockertest/v3/docker"
+)
+
+const dockerHubServer = "https://index.docker.io/v1/"
+
+type CredentialSource string
+
+const (
+	CredentialSourceEnv       CredentialSource = "env"
+	CredentialSourceConfig    CredentialSource = "config"
+	CredentialSourceAnonymous CredentialSource = "anonymous"
+)
+
+// Credentials resolves Docker Hub credentials from
+// DOCKERHUB_USERNAME/DOCKERHUB_TOKEN, then ~/.docker/config.json, then
+// anonymous. The Docker Go SDKs do not read config.json on their own.
+func Credentials() (string, string, CredentialSource) {
+	if u := os.Getenv("DOCKERHUB_USERNAME"); u != "" {
+		return u, os.Getenv("DOCKERHUB_TOKEN"), CredentialSourceEnv
+	}
+
+	user, pass, ok := credentialsFromConfig()
+	if ok {
+		return user, pass, CredentialSourceConfig
+	}
+
+	return "", "", CredentialSourceAnonymous
+}
+
+// AuthConfiguration returns Docker Hub auth for the dockertest pool.
+func AuthConfiguration() docker.AuthConfiguration {
+	u, p, _ := Credentials()
+
+	return docker.AuthConfiguration{
+		Username:      u,
+		Password:      p,
+		ServerAddress: dockerHubServer,
+	}
+}
+
+// RegistryAuth returns base64-encoded credentials for the modern
+// Docker SDK's image.PullOptions{RegistryAuth: ...}, or "" when none.
+func RegistryAuth() (string, error) {
+	u, p, _ := Credentials()
+	if u == "" && p == "" {
+		return "", nil
+	}
+
+	auth := struct {
+		Username string `json:"username"`
+		Password string `json:"password"`
+	}{Username: u, Password: p}
+
+	b, err := json.Marshal(auth) //nolint:gosec // G117: password field holds the Docker Hub token, intentional
+	if err != nil {
+		return "", fmt.Errorf("marshalling docker auth: %w", err)
+	}
+
+	return base64.URLEncoding.EncodeToString(b), nil
+}
+
+// PullWithAuth ensures imageRef is local, pulling with auth and
+// retrying transient errors when it is not.
+func PullWithAuth(pool *dockertest.Pool, imageRef string) error {
+	if img, _ := pool.Client.InspectImage(imageRef); img != nil {
+		return nil
+	}
+
+	repo, tag := splitImageRef(imageRef)
+	auth := AuthConfiguration()
+
+	_, err := backoff.Retry(
+		context.Background(),
+		func() (struct{}, error) {
+			err := pool.Client.PullImage(docker.PullImageOptions{
+				Repository: repo,
+				Tag:        tag,
+			}, auth)
+			if err == nil {
+				return struct{}{}, nil
+			}
+
+			if isPermanentPullError(err) {
+				return struct{}{}, backoff.Permanent(err)
+			}
+
+			return struct{}{}, fmt.Errorf("pulling %s: %w", imageRef, err)
+		},
+		backoff.WithBackOff(backoff.NewExponentialBackOff()),
+		backoff.WithMaxElapsedTime(60*time.Second),
+	)
+	if err != nil {
+		return fmt.Errorf("pulling %s with auth (source=%s): %w", imageRef, AuthConfiguration().ServerAddress, err)
+	}
+
+	return nil
+}
+
+func splitImageRef(ref string) (string, string) {
+	if i := strings.LastIndex(ref, ":"); i >= 0 {
+		return ref[:i], ref[i+1:]
+	}
+
+	return ref, "latest"
+}
+
+func isPermanentPullError(err error) bool {
+	msg := strings.ToLower(err.Error())
+
+	return strings.Contains(msg, "manifest unknown") ||
+		strings.Contains(msg, "manifest not found") ||
+		strings.Contains(msg, "repository does not exist") ||
+		strings.Contains(msg, "name unknown") ||
+		strings.Contains(msg, "no such image")
+}
+
+// credentialsFromConfig reads the Hub entry from ~/.docker/config.json.
+// Credential helpers (osxkeychain etc.) are not supported; use env vars.
+func credentialsFromConfig() (string, string, bool) {
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return "", "", false
+	}
+
+	raw, err := os.ReadFile(filepath.Join(home, ".docker", "config.json"))
+	if err != nil {
+		return "", "", false
+	}
+
+	var cfg struct {
+		Auths map[string]struct {
+			Auth string `json:"auth"`
+		} `json:"auths"`
+	}
+
+	err = json.Unmarshal(raw, &cfg)
+	if err != nil {
+		return "", "", false
+	}
+
+	entry, found := cfg.Auths[dockerHubServer]
+	if !found || entry.Auth == "" {
+		return "", "", false
+	}
+
+	decoded, err := base64.StdEncoding.DecodeString(entry.Auth)
+	if err != nil {
+		return "", "", false
+	}
+
+	parts := strings.SplitN(string(decoded), ":", 2)
+	if len(parts) != 2 {
+		return "", "", false
+	}
+
+	return parts[0], parts[1], true
+}

integration/tsic/tsic.go

@@ -523,6 +523,12 @@ func New(
 		tailscaleOptions.Repository = "tailscale/tailscale"
 		tailscaleOptions.Tag = version
 
+		err = dockertestutil.PullWithAuth(pool, tailscaleOptions.Repository+":"+tailscaleOptions.Tag)
+		if err != nil {
+			//nolint:gosec // G706: tag value is from MustTestVersions, not external input
+			log.Printf("Pull failed for %s:%s, error: %v", tailscaleOptions.Repository, tailscaleOptions.Tag, err)
+		}
+
 		container, err = pool.RunWithOptions(
 			tailscaleOptions,
 			dockertestutil.DockerRestartPolicy,
@@ -537,6 +543,12 @@ func New(
 		tailscaleOptions.Repository = "tailscale/tailscale"
 		tailscaleOptions.Tag = "v" + version
 
+		err = dockertestutil.PullWithAuth(pool, tailscaleOptions.Repository+":"+tailscaleOptions.Tag)
+		if err != nil {
+			//nolint:gosec // G706: tag value is from MustTestVersions, not external input
+			log.Printf("Pull failed for %s:%s, error: %v", tailscaleOptions.Repository, tailscaleOptions.Tag, err)
+		}
+
 		container, err = pool.RunWithOptions(
 			tailscaleOptions,
 			dockertestutil.DockerRestartPolicy,