integration: avoid toolchain work and login waits in CI

cmd/hi/README.md

@@ -56,6 +56,7 @@ changing.
 | `--postgres`        | `false`        | Use PostgreSQL instead of SQLite                                            |
 | `--failfast`        | `true`         | Stop on first test failure                                                  |
 | `--go-version`      | auto           | Detected from `go.mod` (currently 1.27.0)                                   |
+| `--test-binary`     | unset          | Execute a precompiled integration test binary instead of running `go test`  |
 | `--clean-before`    | `true`         | Clean stale (stopped/exited) containers before starting                     |
 | `--clean-after`     | `true`         | Clean this run's containers after completion                                |
 | `--keep-on-failure` | `false`        | Preserve containers for manual inspection on failure                        |
@@ -65,6 +66,29 @@ changing.
 | `--hs-memory-limit` | `0`            | Fail if any headscale container exceeds N MB (0 = disabled)                 |
 | `--ts-memory-limit` | `0`            | Fail if any tailscale container exceeds N MB                                |
 
+### Precompiled tests in CI
+
+CI builds the Headscale server, `hi`, and the integration test binary once
+for the worker architecture. Each test job loads the prebuilt images and
+runs its own fresh test container:
+
+```bash
+CGO_ENABLED=0 GOOS=linux go test -c ./integration -o integration.test
+go run ./cmd/hi run --test-binary=./integration.test "^TestHeadscale$"
+```
+
+With `--test-binary`, the runner mounts the executable read-only and uses
+`HEADSCALE_INTEGRATION_HEADSCALE_IMAGE` as its runtime image when set.
+Otherwise it uses the usual Go image. Build the binary with `CGO_ENABLED=0`
+for the runner architecture so it needs no compiler or shared Go libraries.
+The checkout, test fixtures, Docker access, per-run logs, resource checks,
+and cleanup work the same as in source-based runs. CI does not distribute
+Go caches to these workers.
+
+Automatic preflight checks Docker and the checkout. Images are prepared by
+their consumers, so ordinary tests do not pull Kubernetes images. The
+standalone `doctor` command retains the comprehensive development checks.
+
 ### Timeout guidance
 
 The default `120m` is generous for a single test. If you must tune it,

cmd/hi/docker.go

@@ -73,17 +73,17 @@ func runTestContainer(ctx context.Context, config *RunConfig) error {
 		}
 	}
 
-	goTestCmd := buildGoTestCommand(config)
+	testCmd := buildTestCommand(config)
 	if config.Verbose {
-		log.Printf("Command: %s", strings.Join(goTestCmd, " "))
+		log.Printf("Command: %s", strings.Join(testCmd, " "))
 	}
 
-	imageName := "golang:" + config.GoVersion
+	imageName := testRunnerImage(config)
 	if err := ensureImageAvailable(ctx, cli, imageName, config.Verbose); err != nil { //nolint:noinlineerr
 		return fmt.Errorf("ensuring image availability: %w", err)
 	}
 
-	resp, err := createGoTestContainer(ctx, cli, config, containerName, absLogsDir, goTestCmd)
+	resp, err := createTestContainer(ctx, cli, config, containerName, absLogsDir, testCmd)
 	if err != nil {
 		return fmt.Errorf("creating container: %w", err)
 	}
@@ -202,26 +202,44 @@ func runTestContainer(ctx context.Context, config *RunConfig) error {
 	return nil
 }
 
-// buildGoTestCommand constructs the go test command arguments.
-func buildGoTestCommand(config *RunConfig) []string {
+// testRunnerImage reuses the CI server image for the statically built test binary.
+// Source-based runs and local runs without a server image use the Go image.
+func testRunnerImage(config *RunConfig) string {
+	if config.TestBinary != "" {
+		if image := os.Getenv("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE"); image != "" {
+			return image
+		}
+	}
+
+	return "golang:" + config.GoVersion
+}
+
+// buildTestCommand supports both go test and its compiled test binary flags.
+func buildTestCommand(config *RunConfig) []string {
 	cmd := []string{"go", "test", "./..."}
+	flagPrefix := "-"
+
+	if config.TestBinary != "" {
+		cmd = []string{"/integration.test"}
+		flagPrefix = "-test."
+	}
 
 	if config.TestPattern != "" {
-		cmd = append(cmd, "-run", config.TestPattern)
+		cmd = append(cmd, flagPrefix+"run", config.TestPattern)
 	}
 
 	if config.FailFast {
-		cmd = append(cmd, "-failfast")
+		cmd = append(cmd, flagPrefix+"failfast")
 	}
 
-	cmd = append(cmd, "-timeout", config.Timeout.String())
-	cmd = append(cmd, "-v")
+	cmd = append(cmd, flagPrefix+"timeout", config.Timeout.String())
+	cmd = append(cmd, flagPrefix+"v")
 
 	return cmd
 }
 
-// createGoTestContainer creates a Docker container configured for running integration tests.
-func createGoTestContainer(ctx context.Context, cli *client.Client, config *RunConfig, containerName, logsDir string, goTestCmd []string) (container.CreateResponse, error) {
+// createTestContainer creates a Docker container configured for running integration tests.
+func createTestContainer(ctx context.Context, cli *client.Client, config *RunConfig, containerName, logsDir string, testCmd []string) (container.CreateResponse, error) {
 	pwd, err := os.Getwd()
 	if err != nil {
 		return container.CreateResponse{}, fmt.Errorf("getting working directory: %w", err)
@@ -254,12 +272,14 @@ func createGoTestContainer(ctx context.Context, cli *client.Client, config *RunC
 		}
 	}
 
-	// Set GOCACHE to a known location (used by both bind mount and volume cases)
-	env = append(env, "GOCACHE=/cache/go-build")
+	if config.TestBinary == "" {
+		// Used by both bind mount and volume cases for source-based runs.
+		env = append(env, "GOCACHE=/cache/go-build")
+	}
 
 	containerConfig := &container.Config{
-		Image:      "golang:" + config.GoVersion,
-		Cmd:        goTestCmd,
+		Image:      testRunnerImage(config),
+		Cmd:        testCmd,
 		Env:        env,
 		WorkingDir: projectRoot + "/integration",
 		Tty:        true,
@@ -286,27 +306,36 @@ func createGoTestContainer(ctx context.Context, cli *client.Client, config *RunC
 	// otherwise fall back to Docker volumes for local development
 	var mounts []mount.Mount
 
-	goCache := os.Getenv("HEADSCALE_INTEGRATION_GO_CACHE")
-	goBuildCache := os.Getenv("HEADSCALE_INTEGRATION_GO_BUILD_CACHE")
-
-	if goCache != "" {
-		binds = append(binds, goCache+":/go")
-	} else {
+	if config.TestBinary != "" {
 		mounts = append(mounts, mount.Mount{
-			Type:   mount.TypeVolume,
-			Source: "hs-integration-go-cache",
-			Target: "/go",
+			Type:     mount.TypeBind,
+			Source:   config.TestBinary,
+			Target:   "/integration.test",
+			ReadOnly: true,
 		})
-	}
-
-	if goBuildCache != "" {
-		binds = append(binds, goBuildCache+":/cache/go-build")
 	} else {
-		mounts = append(mounts, mount.Mount{
-			Type:   mount.TypeVolume,
-			Source: "hs-integration-go-build-cache",
-			Target: "/cache/go-build",
-		})
+		goCache := os.Getenv("HEADSCALE_INTEGRATION_GO_CACHE")
+		goBuildCache := os.Getenv("HEADSCALE_INTEGRATION_GO_BUILD_CACHE")
+
+		if goCache != "" {
+			binds = append(binds, goCache+":/go")
+		} else {
+			mounts = append(mounts, mount.Mount{
+				Type:   mount.TypeVolume,
+				Source: "hs-integration-go-cache",
+				Target: "/go",
+			})
+		}
+
+		if goBuildCache != "" {
+			binds = append(binds, goBuildCache+":/cache/go-build")
+		} else {
+			mounts = append(mounts, mount.Mount{
+				Type:   mount.TypeVolume,
+				Source: "hs-integration-go-build-cache",
+				Target: "/cache/go-build",
+			})
+		}
 	}
 
 	hostConfig := &container.HostConfig{
@@ -330,8 +359,12 @@ func streamAndWait(ctx context.Context, cli *client.Client, containerID string)
 	}
 	defer out.Close()
 
+	logsDone := make(chan struct{})
+
 	go func() {
 		_, _ = io.Copy(os.Stdout, out)
+
+		close(logsDone)
 	}()
 
 	statusCh, errCh := cli.ContainerWait(ctx, containerID, container.WaitConditionNotRunning)
@@ -341,6 +374,14 @@ func streamAndWait(ctx context.Context, cli *client.Client, containerID string)
 			return -1, fmt.Errorf("waiting for container: %w", err)
 		}
 	case status := <-statusCh:
+		// ContainerWait can finish before the log stream delivers its tail.
+		// Drain through EOF so fast test exits retain their final summary.
+		select {
+		case <-logsDone:
+		case <-ctx.Done():
+			return int(status.StatusCode), fmt.Errorf("waiting for container logs: %w", ctx.Err())
+		}
+
 		return int(status.StatusCode), nil
 	}
 

cmd/hi/doctor.go

@@ -53,6 +53,12 @@ func fail(name, message string, suggestions ...string) DoctorResult {
 
 // runDoctorCheck performs comprehensive pre-flight checks for integration testing.
 func runDoctorCheck(ctx context.Context) error {
+	return runPreflightChecks(ctx, true)
+}
+
+// runPreflightChecks leaves image preparation to the tests that need it.
+// The standalone doctor additionally checks development tools and images.
+func runPreflightChecks(ctx context.Context, full bool) error {
 	results := []DoctorResult{}
 
 	// Check 1: Docker binary availability
@@ -67,12 +73,17 @@ func runDoctorCheck(ctx context.Context) error {
 		results = append(results, checkDockerContext(ctx))
 		results = append(results, checkDockerSocket(ctx))
 		results = append(results, checkDockerHubCredentials())
-		results = append(results, checkGolangImage(ctx))
-		results = append(results, checkK3sImage(ctx))
+
+		if full {
+			results = append(results, checkGolangImage(ctx))
+			results = append(results, checkK3sImage(ctx))
+		}
 	}
 
 	// Check 3: Go installation
-	results = append(results, checkGoInstallation(ctx))
+	if full {
+		results = append(results, checkGoInstallation(ctx))
+	}
 
 	// Check 4: Git repository
 	results = append(results, checkGitRepository(ctx))

cmd/hi/run.go

@@ -12,10 +12,15 @@ import (
 	"github.com/creachadair/command"
 )
 
-var ErrTestPatternRequired = errors.New("test pattern is required as first argument or use --test flag")
+var (
+	ErrTestPatternRequired     = errors.New("test pattern is required as first argument or use --test flag")
+	ErrUnexpectedTestArguments = errors.New("expected a single test pattern; check flag spelling and shell quoting")
+	ErrInvalidTestBinary       = errors.New("test binary must be an executable regular file")
+)
 
 type RunConfig struct {
 	TestPattern   string        `flag:"test,Test pattern to run"`
+	TestBinary    string        `flag:"test-binary,Path to a precompiled integration test binary"`
 	Timeout       time.Duration `flag:"timeout,default=120m,Test timeout"`
 	FailFast      bool          `flag:"failfast,default=true,Stop on first test failure"`
 	UsePostgres   bool          `flag:"postgres,default=false,Use PostgreSQL instead of SQLite"`
@@ -30,17 +35,47 @@ type RunConfig struct {
 	TSMemoryLimit float64       `flag:"ts-memory-limit,default=0,Fail test if any Tailscale container exceeds this memory limit in MB (0 = disabled)"`
 }
 
-// runIntegrationTest executes the integration test workflow.
-func runIntegrationTest(env *command.Env) error {
-	args := env.Args
-	if len(args) > 0 && runConfig.TestPattern == "" {
-		runConfig.TestPattern = args[0]
+func (c *RunConfig) validate(args []string) error {
+	if len(args) > 1 || (len(args) != 0 && c.TestPattern != "") {
+		return ErrUnexpectedTestArguments
 	}
 
-	if runConfig.TestPattern == "" {
+	if len(args) == 1 {
+		c.TestPattern = args[0]
+	}
+
+	if c.TestPattern == "" {
 		return ErrTestPatternRequired
 	}
 
+	if c.TestBinary != "" {
+		binary, err := filepath.Abs(c.TestBinary)
+		if err != nil {
+			return fmt.Errorf("resolving test binary: %w", err)
+		}
+
+		info, err := os.Stat(binary)
+		if err != nil {
+			return fmt.Errorf("reading test binary: %w", err)
+		}
+
+		if !info.Mode().IsRegular() || info.Mode().Perm()&0o111 == 0 {
+			return fmt.Errorf("%s: %w", binary, ErrInvalidTestBinary)
+		}
+
+		c.TestBinary = binary
+	}
+
+	return nil
+}
+
+// runIntegrationTest executes the integration test workflow.
+func runIntegrationTest(env *command.Env) error {
+	err := runConfig.validate(env.Args)
+	if err != nil {
+		return err
+	}
+
 	if runConfig.GoVersion == "" {
 		runConfig.GoVersion = detectGoVersion()
 	}
@@ -50,7 +85,7 @@ func runIntegrationTest(env *command.Env) error {
 		log.Printf("Running pre-flight system checks...")
 	}
 
-	err := runDoctorCheck(env.Context())
+	err = runPreflightChecks(env.Context(), false)
 	if err != nil {
 		return fmt.Errorf("pre-flight checks failed: %w", err)
 	}
@@ -62,6 +97,13 @@ func runIntegrationTest(env *command.Env) error {
 		log.Printf("Use PostgreSQL: %t", runConfig.UsePostgres)
 	}
 
+	backend := "sqlite"
+	if runConfig.UsePostgres {
+		backend = "postgres"
+	}
+
+	log.Printf("Database backend: %s", backend)
+
 	return runTestContainer(env.Context(), &runConfig)
 }
 

cmd/hi/run_test.go

@@ -0,0 +1,247 @@
+package main
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/creachadair/command"
+	"github.com/creachadair/flax"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/mount"
+	"github.com/docker/docker/client"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Exercise the real flag parser: folded YAML shell continuations used to turn
+// --postgres into a positional argument, silently selecting SQLite.
+func TestRunArguments(t *testing.T) {
+	tests := []struct {
+		name     string
+		args     []string
+		postgres bool
+		wantErr  error
+	}{
+		{
+			name:     "postgres before selector",
+			args:     []string{"--postgres=1", "--timeout=15m", "^TestHeadscale$"},
+			postgres: true,
+		},
+		{
+			name:     "postgres after selector",
+			args:     []string{"^TestHeadscale$", "--postgres", "--timeout=15m"},
+			postgres: true,
+		},
+		{
+			name: "sqlite",
+			args: []string{"--postgres=0", "--timeout=15m", "--test=^TestHeadscale$"},
+		},
+		{
+			name:    "folded shell continuations",
+			args:    []string{"^TestHeadscale$", " --timeout=15m", " --postgres=1"},
+			wantErr: ErrUnexpectedTestArguments,
+		},
+		{
+			name:    "extra selector",
+			args:    []string{"^TestHeadscale$", "TestNodeCommand"},
+			wantErr: ErrUnexpectedTestArguments,
+		},
+		{
+			name:    "positional selector with test flag",
+			args:    []string{"--test=^TestHeadscale$", "TestNodeCommand"},
+			wantErr: ErrUnexpectedTestArguments,
+		},
+		{
+			name:    "missing selector",
+			wantErr: ErrTestPatternRequired,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var config RunConfig
+
+			cmd := command.C{
+				Name:     "run",
+				SetFlags: command.Flags(flax.MustBind, &config),
+				Run:      func(env *command.Env) error { return config.validate(env.Args) },
+			}
+
+			err := command.Run(cmd.NewEnv(nil).MergeFlags(true), tt.args)
+			if tt.wantErr != nil {
+				require.ErrorIs(t, err, tt.wantErr)
+				return
+			}
+
+			require.NoError(t, err)
+			assert.Equal(t, tt.postgres, config.UsePostgres)
+			assert.Equal(t, "^TestHeadscale$", config.TestPattern)
+			assert.Equal(t, 15*time.Minute, config.Timeout)
+		})
+	}
+}
+
+func TestCompiledTestContainer(t *testing.T) {
+	t.Setenv("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE", "headscale:ci-test")
+	// Neither inherited backend settings nor Go cache mounts should override
+	// the requested backend or leak into a compiled test execution.
+	t.Setenv("HEADSCALE_INTEGRATION_POSTGRES", "0")
+	t.Setenv("HEADSCALE_INTEGRATION_GO_CACHE", "/unused/go")
+	t.Setenv("HEADSCALE_INTEGRATION_GO_BUILD_CACHE", "/unused/build")
+
+	binary := filepath.Join(t.TempDir(), "integration.test")
+	require.NoError(t, os.WriteFile(binary, []byte("test executable"), 0o700)) //nolint:gosec // The fixture must be executable to validate --test-binary.
+	config := RunConfig{
+		TestPattern: "^TestAutoApproveMultiNetwork/webauth-user.*$",
+		TestBinary:  binary,
+		UsePostgres: true,
+		Timeout:     15 * time.Minute,
+		FailFast:    true,
+	}
+	require.NoError(t, config.validate(nil))
+
+	type createRequest struct {
+		container.Config
+
+		HostConfig container.HostConfig
+	}
+
+	requests := make(chan createRequest, 1)
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var request createRequest
+
+		err := json.NewDecoder(r.Body).Decode(&request)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		requests <- request
+
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusCreated)
+		_, _ = w.Write([]byte(`{"Id":"test-runner","Warnings":[]}`))
+	}))
+	t.Cleanup(server.Close)
+	cli, err := client.NewClientWithOpts(client.WithHost(server.URL), client.WithVersion("1.44"))
+	require.NoError(t, err)
+	t.Cleanup(func() { require.NoError(t, cli.Close()) })
+
+	const runID = "20261007-120000-abcdef"
+
+	logs := t.TempDir()
+	_, err = createTestContainer(t.Context(), cli, &config,
+		"headscale-test-suite-"+runID, logs, buildTestCommand(&config))
+	require.NoError(t, err)
+
+	request := <-requests
+	assert.Equal(t, "headscale:ci-test", request.Image)
+	assert.Equal(t, []string{
+		"/integration.test", "-test.run", config.TestPattern,
+		"-test.failfast", "-test.timeout", "15m0s", "-test.v",
+	}, []string(request.Cmd))
+	assert.Contains(t, request.Env, "HEADSCALE_INTEGRATION_POSTGRES=1")
+	assert.NotContains(t, request.Env, "HEADSCALE_INTEGRATION_POSTGRES=0")
+	assert.Contains(t, request.Env, "HEADSCALE_INTEGRATION_RUN_ID="+runID)
+	assert.NotContains(t, request.Env, "GOCACHE=/cache/go-build")
+	assert.Equal(t, runID, request.Labels["hi.run-id"])
+	assert.Equal(t, "integration", filepath.Base(request.WorkingDir))
+	assert.Contains(t, request.HostConfig.Binds, logs+":/tmp/control")
+	assert.NotContains(t, request.HostConfig.Binds, "/unused/go:/go")
+	assert.Equal(t, []mount.Mount{{
+		Type: mount.TypeBind, Source: binary, Target: "/integration.test", ReadOnly: true,
+	}}, request.HostConfig.Mounts)
+}
+
+func TestSourceTestCommand(t *testing.T) {
+	t.Setenv("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE", "headscale:ci-test")
+
+	config := RunConfig{
+		TestPattern: "^TestHeadscale$", GoVersion: "1.27.0", Timeout: time.Minute,
+	}
+	assert.Equal(t, []string{"go", "test", "./...", "-run", "^TestHeadscale$", "-timeout", "1m0s", "-v"}, buildTestCommand(&config))
+	assert.Equal(t, "golang:1.27.0", testRunnerImage(&config))
+
+	config.TestBinary = "/integration.test"
+
+	t.Setenv("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE", "")
+	assert.Equal(t, "golang:1.27.0", testRunnerImage(&config))
+}
+
+func TestStreamAndWaitDrainsFinalOutput(t *testing.T) {
+	output, err := os.CreateTemp(t.TempDir(), "test-output")
+	require.NoError(t, err)
+	t.Cleanup(func() { require.NoError(t, output.Close()) })
+
+	stdout := os.Stdout
+	os.Stdout = output
+
+	t.Cleanup(func() { os.Stdout = stdout })
+
+	waitSent := make(chan struct{})
+	releaseLogs := make(chan struct{})
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case strings.HasSuffix(r.URL.Path, "/logs"):
+			w.Header().Set("Content-Type", "application/vnd.docker.raw-stream")
+			w.WriteHeader(http.StatusOK)
+			_ = http.NewResponseController(w).Flush()
+
+			select {
+			case <-releaseLogs:
+				_, _ = w.Write([]byte("--- PASS: TestExample (0.01s)\nPASS\n"))
+			case <-r.Context().Done():
+			}
+		case strings.HasSuffix(r.URL.Path, "/wait"):
+			w.Header().Set("Content-Type", "application/json")
+			_, _ = w.Write([]byte(`{"StatusCode":0}`))
+			_ = http.NewResponseController(w).Flush()
+
+			close(waitSent)
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	t.Cleanup(server.Close)
+	cli, err := client.NewClientWithOpts(client.WithHost(server.URL), client.WithVersion("1.44"))
+	require.NoError(t, err)
+	t.Cleanup(func() { require.NoError(t, cli.Close()) })
+
+	type result struct {
+		exitCode int
+		err      error
+	}
+
+	done := make(chan result, 1)
+
+	go func() {
+		code, err := streamAndWait(t.Context(), cli, "test-runner")
+		done <- result{code, err}
+	}()
+
+	<-waitSent
+
+	// Hold back the last log packet after Docker has reported the exit code.
+	select {
+	case <-done:
+		close(releaseLogs)
+		t.Fatal("returned before the final test output was delivered")
+	case <-time.After(50 * time.Millisecond):
+	}
+
+	close(releaseLogs)
+
+	got := <-done
+	require.NoError(t, got.err)
+	assert.Zero(t, got.exitCode)
+
+	data, err := os.ReadFile(output.Name())
+	require.NoError(t, err)
+	assert.Equal(t, "--- PASS: TestExample (0.01s)\nPASS\n", string(data))
+}