diff --git a/.github/workflows/integration-test-template.yml b/.github/workflows/integration-test-template.yml index efaaac4c..427f63e2 100644 --- a/.github/workflows/integration-test-template.yml +++ b/.github/workflows/integration-test-template.yml @@ -73,6 +73,15 @@ jobs: echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json sudo systemctl restart docker docker version + - name: Login to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }} + if: env.DOCKERHUB_USERNAME != '' + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + username: ${{ env.DOCKERHUB_USERNAME }} + password: ${{ env.DOCKERHUB_TOKEN }} - name: Load Docker images, Go cache, and prepare binary run: | gunzip -c /tmp/artifacts/headscale-image.tar.gz | docker load @@ -93,6 +102,12 @@ jobs: HEADSCALE_INTEGRATION_POSTGRES_IMAGE: ${{ inputs.postgres_flag == '--postgres=1' && format('postgres:{0}', github.sha) || '' }} HEADSCALE_INTEGRATION_GO_CACHE: /tmp/go-cache/go HEADSCALE_INTEGRATION_GO_BUILD_CACHE: /tmp/go-cache/.cache/go-build + # Mirror the docker/login-action secrets into env so the + # dockertestutil.Credentials resolver picks them up directly + # (otherwise it falls back to parsing ~/.docker/config.json, + # which works but is one step further from the source). + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }} run: /tmp/artifacts/hi run --stats --ts-memory-limit=300 --hs-memory-limit=1500 "^${{ inputs.test }}$" \ --timeout=120m \ ${{ inputs.postgres_flag }} diff --git a/.github/workflows/test-integration.yaml b/.github/workflows/test-integration.yaml index 67fe0c47..f257ae0e 100644 --- a/.github/workflows/test-integration.yaml +++ b/.github/workflows/test-integration.yaml @@ -80,6 +80,15 @@ jobs: echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json sudo systemctl restart docker docker version + - name: Login to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }} + if: env.DOCKERHUB_USERNAME != '' + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + username: ${{ env.DOCKERHUB_USERNAME }} + password: ${{ env.DOCKERHUB_TOKEN }} - name: Build headscale image if: steps.changed-files.outputs.files == 'true' run: | @@ -121,6 +130,15 @@ jobs: echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json sudo systemctl restart docker docker version + - name: Login to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }} + if: env.DOCKERHUB_USERNAME != '' + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + username: ${{ env.DOCKERHUB_USERNAME }} + password: ${{ env.DOCKERHUB_TOKEN }} - name: Pull and save postgres image run: | docker pull postgres:latest diff --git a/cmd/hi/docker.go b/cmd/hi/docker.go index 060057a9..03778bf7 100644 --- a/cmd/hi/docker.go +++ b/cmd/hi/docker.go @@ -14,6 +14,7 @@ import ( "strings" "time" + "github.com/cenkalti/backoff/v5" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/image" "github.com/docker/docker/api/types/mount" @@ -522,9 +523,9 @@ func checkImageAvailableLocally(ctx context.Context, cli *client.Client, imageNa return true, nil } -// ensureImageAvailable checks if the image is available locally first, then pulls if needed. +// ensureImageAvailable pulls imageName if missing, using Docker Hub +// credentials and retrying transient errors. func ensureImageAvailable(ctx context.Context, cli *client.Client, imageName string, verbose bool) error { - // First check if image is available locally available, err := checkImageAvailableLocally(ctx, cli, imageName) if err != nil { return fmt.Errorf("checking local image availability: %w", err) @@ -538,34 +539,64 @@ func ensureImageAvailable(ctx context.Context, cli *client.Client, imageName str return nil } - // Image not available locally, try to pull it if verbose { log.Printf("Image %s not found locally, pulling...", imageName) } - reader, err := cli.ImagePull(ctx, imageName, image.PullOptions{}) + registryAuth, err := dockertestutil.RegistryAuth() if err != nil { - return fmt.Errorf("pulling image %s: %w", imageName, err) + return fmt.Errorf("resolving registry auth: %w", err) } - defer reader.Close() - if verbose { - _, err = io.Copy(os.Stdout, reader) - if err != nil { - return fmt.Errorf("reading pull output: %w", err) - } - } else { - _, err = io.Copy(io.Discard, reader) - if err != nil { - return fmt.Errorf("reading pull output: %w", err) - } + _, err = backoff.Retry( + ctx, + func() (struct{}, error) { + reader, pullErr := cli.ImagePull(ctx, imageName, image.PullOptions{RegistryAuth: registryAuth}) + if pullErr != nil { + if isPermanentDockerPullError(pullErr) { + return struct{}{}, backoff.Permanent(pullErr) + } + return struct{}{}, fmt.Errorf("pulling image %s: %w", imageName, pullErr) + } + defer reader.Close() + + sink := io.Discard + if verbose { + sink = os.Stdout + } + + _, copyErr := io.Copy(sink, reader) + if copyErr != nil { + return struct{}{}, fmt.Errorf("reading pull output: %w", copyErr) + } + + return struct{}{}, nil + }, + backoff.WithBackOff(backoff.NewExponentialBackOff()), + backoff.WithMaxElapsedTime(60*time.Second), + ) + if err != nil { + return err + } + + if !verbose { log.Printf("Image %s pulled successfully", imageName) } return nil } +func isPermanentDockerPullError(err error) bool { + msg := strings.ToLower(err.Error()) + + return strings.Contains(msg, "manifest unknown") || + strings.Contains(msg, "manifest not found") || + strings.Contains(msg, "repository does not exist") || + strings.Contains(msg, "name unknown") || + strings.Contains(msg, "no such image") +} + // listControlFiles displays the headscale test artifacts created in the control logs directory. func listControlFiles(logsDir string) { entries, err := os.ReadDir(logsDir) diff --git a/cmd/hi/doctor.go b/cmd/hi/doctor.go index 1791d66d..e235cf09 100644 --- a/cmd/hi/doctor.go +++ b/cmd/hi/doctor.go @@ -7,6 +7,8 @@ import ( "log" "os/exec" "strings" + + "github.com/juanfont/headscale/integration/dockertestutil" ) var ErrSystemChecksFailed = errors.New("system checks failed") @@ -34,6 +36,7 @@ func runDoctorCheck(ctx context.Context) error { if dockerResult.Status == "PASS" { results = append(results, checkDockerContext(ctx)) results = append(results, checkDockerSocket(ctx)) + results = append(results, checkDockerHubCredentials()) results = append(results, checkGolangImage(ctx)) } @@ -190,6 +193,30 @@ func checkDockerSocket(ctx context.Context) DoctorResult { } } +// checkDockerHubCredentials warns when pulls would be anonymous and +// therefore rate-limited. +func checkDockerHubCredentials() DoctorResult { + _, _, source := dockertestutil.Credentials() + if source == dockertestutil.CredentialSourceAnonymous { + return DoctorResult{ + Name: "Docker Hub Credentials", + Status: "WARN", + Message: "No Docker Hub credentials found — pulls will be rate-limited (100/6h per IP)", + Suggestions: []string{ + "Run: docker login", + "Or export DOCKERHUB_USERNAME and DOCKERHUB_TOKEN", + "In CI: ensure the docker/login-action step is configured with secrets", + }, + } + } + + return DoctorResult{ + Name: "Docker Hub Credentials", + Status: "PASS", + Message: fmt.Sprintf("Credentials available (source: %s)", source), + } +} + // checkGolangImage verifies the golang Docker image is available locally or can be pulled. func checkGolangImage(ctx context.Context) DoctorResult { cli, err := createDockerClient(ctx) diff --git a/integration/dockertestutil/auth.go b/integration/dockertestutil/auth.go new file mode 100644 index 00000000..3962204d --- /dev/null +++ b/integration/dockertestutil/auth.go @@ -0,0 +1,171 @@ +package dockertestutil + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/cenkalti/backoff/v5" + "github.com/ory/dockertest/v3" + "github.com/ory/dockertest/v3/docker" +) + +const dockerHubServer = "https://index.docker.io/v1/" + +type CredentialSource string + +const ( + CredentialSourceEnv CredentialSource = "env" + CredentialSourceConfig CredentialSource = "config" + CredentialSourceAnonymous CredentialSource = "anonymous" +) + +// Credentials resolves Docker Hub credentials from +// DOCKERHUB_USERNAME/DOCKERHUB_TOKEN, then ~/.docker/config.json, then +// anonymous. The Docker Go SDKs do not read config.json on their own. +func Credentials() (string, string, CredentialSource) { + if u := os.Getenv("DOCKERHUB_USERNAME"); u != "" { + return u, os.Getenv("DOCKERHUB_TOKEN"), CredentialSourceEnv + } + + user, pass, ok := credentialsFromConfig() + if ok { + return user, pass, CredentialSourceConfig + } + + return "", "", CredentialSourceAnonymous +} + +// AuthConfiguration returns Docker Hub auth for the dockertest pool. +func AuthConfiguration() docker.AuthConfiguration { + u, p, _ := Credentials() + + return docker.AuthConfiguration{ + Username: u, + Password: p, + ServerAddress: dockerHubServer, + } +} + +// RegistryAuth returns base64-encoded credentials for the modern +// Docker SDK's image.PullOptions{RegistryAuth: ...}, or "" when none. +func RegistryAuth() (string, error) { + u, p, _ := Credentials() + if u == "" && p == "" { + return "", nil + } + + auth := struct { + Username string `json:"username"` + Password string `json:"password"` + }{Username: u, Password: p} + + b, err := json.Marshal(auth) //nolint:gosec // G117: password field holds the Docker Hub token, intentional + if err != nil { + return "", fmt.Errorf("marshalling docker auth: %w", err) + } + + return base64.URLEncoding.EncodeToString(b), nil +} + +// PullWithAuth ensures imageRef is local, pulling with auth and +// retrying transient errors when it is not. +func PullWithAuth(pool *dockertest.Pool, imageRef string) error { + if img, _ := pool.Client.InspectImage(imageRef); img != nil { + return nil + } + + repo, tag := splitImageRef(imageRef) + auth := AuthConfiguration() + + _, err := backoff.Retry( + context.Background(), + func() (struct{}, error) { + err := pool.Client.PullImage(docker.PullImageOptions{ + Repository: repo, + Tag: tag, + }, auth) + if err == nil { + return struct{}{}, nil + } + + if isPermanentPullError(err) { + return struct{}{}, backoff.Permanent(err) + } + + return struct{}{}, fmt.Errorf("pulling %s: %w", imageRef, err) + }, + backoff.WithBackOff(backoff.NewExponentialBackOff()), + backoff.WithMaxElapsedTime(60*time.Second), + ) + if err != nil { + return fmt.Errorf("pulling %s with auth (source=%s): %w", imageRef, AuthConfiguration().ServerAddress, err) + } + + return nil +} + +func splitImageRef(ref string) (string, string) { + if i := strings.LastIndex(ref, ":"); i >= 0 { + return ref[:i], ref[i+1:] + } + + return ref, "latest" +} + +func isPermanentPullError(err error) bool { + msg := strings.ToLower(err.Error()) + + return strings.Contains(msg, "manifest unknown") || + strings.Contains(msg, "manifest not found") || + strings.Contains(msg, "repository does not exist") || + strings.Contains(msg, "name unknown") || + strings.Contains(msg, "no such image") +} + +// credentialsFromConfig reads the Hub entry from ~/.docker/config.json. +// Credential helpers (osxkeychain etc.) are not supported; use env vars. +func credentialsFromConfig() (string, string, bool) { + home, err := os.UserHomeDir() + if err != nil { + return "", "", false + } + + raw, err := os.ReadFile(filepath.Join(home, ".docker", "config.json")) + if err != nil { + return "", "", false + } + + var cfg struct { + Auths map[string]struct { + Auth string `json:"auth"` + } `json:"auths"` + } + + err = json.Unmarshal(raw, &cfg) + if err != nil { + return "", "", false + } + + entry, found := cfg.Auths[dockerHubServer] + if !found || entry.Auth == "" { + return "", "", false + } + + decoded, err := base64.StdEncoding.DecodeString(entry.Auth) + if err != nil { + return "", "", false + } + + parts := strings.SplitN(string(decoded), ":", 2) + if len(parts) != 2 { + return "", "", false + } + + return parts[0], parts[1], true +} diff --git a/integration/tsic/tsic.go b/integration/tsic/tsic.go index 4a558a04..fcabb5e9 100644 --- a/integration/tsic/tsic.go +++ b/integration/tsic/tsic.go @@ -523,6 +523,12 @@ func New( tailscaleOptions.Repository = "tailscale/tailscale" tailscaleOptions.Tag = version + err = dockertestutil.PullWithAuth(pool, tailscaleOptions.Repository+":"+tailscaleOptions.Tag) + if err != nil { + //nolint:gosec // G706: tag value is from MustTestVersions, not external input + log.Printf("Pull failed for %s:%s, error: %v", tailscaleOptions.Repository, tailscaleOptions.Tag, err) + } + container, err = pool.RunWithOptions( tailscaleOptions, dockertestutil.DockerRestartPolicy, @@ -537,6 +543,12 @@ func New( tailscaleOptions.Repository = "tailscale/tailscale" tailscaleOptions.Tag = "v" + version + err = dockertestutil.PullWithAuth(pool, tailscaleOptions.Repository+":"+tailscaleOptions.Tag) + if err != nil { + //nolint:gosec // G706: tag value is from MustTestVersions, not external input + log.Printf("Pull failed for %s:%s, error: %v", tailscaleOptions.Repository, tailscaleOptions.Tag, err) + } + container, err = pool.RunWithOptions( tailscaleOptions, dockertestutil.DockerRestartPolicy,