From 52e01807dbbe750f629ebefdab510f609e5e56bd Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Fri, 9 Oct 2026 12:35:00 +0000 Subject: [PATCH] ci: reuse compiled integration tests and prebuilt images Updates #3542 (cherry picked from commit c9852c43da33eb6df41c16ce447a0c5d72ad5e3a) --- .../workflows/integration-test-template.yml | 25 +- .github/workflows/test-integration.yaml | 17 +- cmd/hi/README.md | 26 +- cmd/hi/docker.go | 115 ++-- cmd/hi/doctor.go | 15 +- cmd/hi/run.go | 58 +- cmd/hi/run_test.go | 249 +++++++++ integration/auth_key_test.go | 529 +++++++++--------- integration/auth_oidc_test.go | 3 + integration/cli_policy_test.go | 142 +++-- integration/scenario.go | 79 ++- integration/tsic/login_test.go | 171 ++++++ integration/tsic/tsic.go | 142 ++++- 13 files changed, 1125 insertions(+), 446 deletions(-) create mode 100644 cmd/hi/run_test.go create mode 100644 integration/tsic/login_test.go diff --git a/.github/workflows/integration-test-template.yml b/.github/workflows/integration-test-template.yml index 68dbfd366..5a06c819e 100644 --- a/.github/workflows/integration-test-template.yml +++ b/.github/workflows/integration-test-template.yml @@ -61,10 +61,10 @@ jobs: with: name: hi-binary path: /tmp/artifacts - - name: Download Go cache + - name: Download integration test binary uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: - name: go-cache + name: integration-test-binary path: /tmp/artifacts - name: Download postgres image if: ${{ inputs.postgres_flag == '--postgres=1' }} @@ -87,7 +87,7 @@ jobs: with: username: ${{ env.DOCKERHUB_USERNAME }} password: ${{ env.DOCKERHUB_TOKEN }} - - name: Load Docker images, Go cache, and prepare binary + - name: Load Docker images and prepare binaries run: | gunzip -c /tmp/artifacts/headscale-image.tar.gz | docker load gunzip -c /tmp/artifacts/tailscale-head-image.tar.gz | docker load @@ -95,33 +95,30 @@ jobs: if [ -f /tmp/artifacts/postgres-image.tar.gz ]; then gunzip -c /tmp/artifacts/postgres-image.tar.gz | docker load fi - chmod +x /tmp/artifacts/hi + chmod +x /tmp/artifacts/hi /tmp/artifacts/integration.test docker images - # Extract Go cache to host directories for bind mounting - mkdir -p /tmp/go-cache - tar -xzf /tmp/artifacts/go-cache.tar.gz -C /tmp/go-cache - ls -la /tmp/go-cache/ /tmp/go-cache/.cache/ - name: Run Integration Test env: HEADSCALE_INTEGRATION_HEADSCALE_IMAGE: headscale:${{ github.sha }} HEADSCALE_INTEGRATION_TAILSCALE_IMAGE: tailscale-head:${{ github.sha }} HEADSCALE_INTEGRATION_POSTGRES_IMAGE: ${{ inputs.postgres_flag == '--postgres=1' && format('postgres:{0}', github.sha) || '' }} - HEADSCALE_INTEGRATION_GO_CACHE: /tmp/go-cache/go - HEADSCALE_INTEGRATION_GO_BUILD_CACHE: /tmp/go-cache/.cache/go-build # Mirror the docker/login-action secrets into env so the # dockertestutil.Credentials resolver picks them up directly # (otherwise it falls back to parsing ~/.docker/config.json, # which works but is one step further from the source). DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }} - run: /tmp/artifacts/hi run --stats --ts-memory-limit=300 --hs-memory-limit=1500 "^${{ inputs.test }}$" \ - --timeout=120m \ - ${{ inputs.postgres_flag }} + run: | + /tmp/artifacts/hi run --stats --ts-memory-limit=300 --hs-memory-limit=1500 \ + --test-binary=/tmp/artifacts/integration.test \ + --timeout=120m \ + ${{ inputs.postgres_flag }} \ + "^${{ inputs.test }}$" # Sanitize test name for artifact upload (replace invalid characters: " : < > | * ? \ / with -) - name: Sanitize test name for artifacts if: always() id: sanitize - run: echo "name=${TEST_NAME//[\":<>|*?\\\/]/-}" >> $GITHUB_OUTPUT + run: echo "name=${TEST_NAME//[\":<>|*?\\\/]/-}" >> "$GITHUB_OUTPUT" env: TEST_NAME: ${{ inputs.test }} - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 diff --git a/.github/workflows/test-integration.yaml b/.github/workflows/test-integration.yaml index 15d13163d..1bb93618b 100644 --- a/.github/workflows/test-integration.yaml +++ b/.github/workflows/test-integration.yaml @@ -43,15 +43,13 @@ jobs: with: primary-key: nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix', '**/flake.lock') }} restore-prefixes-first-match: nix-${{ runner.os }}-${{ runner.arch }} - - name: Build binaries and warm Go cache + - name: Build binaries if: steps.changed-files.outputs.files == 'true' run: | - # Build all Go binaries in one nix shell to maximize cache reuse nix develop --command -- bash -c ' go build -o hi ./cmd/hi CGO_ENABLED=0 GOOS=linux go build -o headscale ./cmd/headscale - # Build integration test binary to warm the cache with all dependencies - go test -c ./integration -o /dev/null 2>/dev/null || true + CGO_ENABLED=0 GOOS=linux go test -c ./integration -o integration.test ' - name: Upload hi binary if: steps.changed-files.outputs.files == 'true' @@ -60,17 +58,12 @@ jobs: name: hi-binary path: hi retention-days: 10 - - name: Package Go cache - if: steps.changed-files.outputs.files == 'true' - run: | - # Package Go module cache and build cache - tar -czf go-cache.tar.gz -C ~ go .cache/go-build - - name: Upload Go cache + - name: Upload integration test binary if: steps.changed-files.outputs.files == 'true' uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: - name: go-cache - path: go-cache.tar.gz + name: integration-test-binary + path: integration.test retention-days: 10 - name: Force overlay2 storage driver if: steps.changed-files.outputs.files == 'true' diff --git a/cmd/hi/README.md b/cmd/hi/README.md index 7678111de..97d758edb 100644 --- a/cmd/hi/README.md +++ b/cmd/hi/README.md @@ -55,7 +55,8 @@ changing. | `--timeout` | `120m` | Total test timeout. Use the built-in flag — never wrap with bash `timeout`. | | `--postgres` | `false` | Use PostgreSQL instead of SQLite | | `--failfast` | `true` | Stop on first test failure | -| `--go-version` | auto | Detected from `go.mod` (currently 1.26.1) | +| `--go-version` | auto | Detected from `go.mod` (currently 1.26.5) | +| `--test-binary` | unset | Execute a precompiled integration test binary instead of running `go test` | | `--clean-before` | `true` | Clean stale (stopped/exited) containers before starting | | `--clean-after` | `true` | Clean this run's containers after completion | | `--keep-on-failure` | `false` | Preserve containers for manual inspection on failure | @@ -65,6 +66,29 @@ changing. | `--hs-memory-limit` | `0` | Fail if any headscale container exceeds N MB (0 = disabled) | | `--ts-memory-limit` | `0` | Fail if any tailscale container exceeds N MB | +### Precompiled tests in CI + +CI builds the Headscale server, `hi`, and the integration test binary once +for the worker architecture. Each test job loads the prebuilt images and +runs its own fresh test container: + +```bash +CGO_ENABLED=0 GOOS=linux go test -c ./integration -o integration.test +go run ./cmd/hi run --test-binary=./integration.test "^TestHeadscale$" +``` + +With `--test-binary`, the runner mounts the executable read-only and uses +`HEADSCALE_INTEGRATION_HEADSCALE_IMAGE` as its runtime image when set. +Otherwise it uses the usual Go image. Build the binary with `CGO_ENABLED=0` +for the runner architecture so it needs no compiler or shared Go libraries. +The checkout, test fixtures, Docker access, per-run logs, resource checks, +and cleanup work the same as in source-based runs. CI does not distribute +Go caches to these workers. + +Automatic preflight checks Docker and the checkout. Images are prepared by +their consumers, so ordinary tests do not pull Kubernetes images. The +standalone `doctor` command retains the comprehensive development checks. + ### Timeout guidance The default `120m` is generous for a single test. If you must tune it, diff --git a/cmd/hi/docker.go b/cmd/hi/docker.go index 1ed8709ce..46b80ad39 100644 --- a/cmd/hi/docker.go +++ b/cmd/hi/docker.go @@ -73,17 +73,17 @@ func runTestContainer(ctx context.Context, config *RunConfig) error { } } - goTestCmd := buildGoTestCommand(config) + testCmd := buildTestCommand(config) if config.Verbose { - log.Printf("Command: %s", strings.Join(goTestCmd, " ")) + log.Printf("Command: %s", strings.Join(testCmd, " ")) } - imageName := "golang:" + config.GoVersion + imageName := testRunnerImage(config) if err := ensureImageAvailable(ctx, cli, imageName, config.Verbose); err != nil { //nolint:noinlineerr return fmt.Errorf("ensuring image availability: %w", err) } - resp, err := createGoTestContainer(ctx, cli, config, containerName, absLogsDir, goTestCmd) + resp, err := createTestContainer(ctx, cli, config, containerName, absLogsDir, testCmd) if err != nil { return fmt.Errorf("creating container: %w", err) } @@ -202,26 +202,44 @@ func runTestContainer(ctx context.Context, config *RunConfig) error { return nil } -// buildGoTestCommand constructs the go test command arguments. -func buildGoTestCommand(config *RunConfig) []string { +// testRunnerImage reuses the CI server image for the statically built test binary. +// Source-based runs and local runs without a server image use the Go image. +func testRunnerImage(config *RunConfig) string { + if config.TestBinary != "" { + if image := os.Getenv("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE"); image != "" { + return image + } + } + + return "golang:" + config.GoVersion +} + +// buildTestCommand supports both go test and its compiled test binary flags. +func buildTestCommand(config *RunConfig) []string { cmd := []string{"go", "test", "./..."} + flagPrefix := "-" + + if config.TestBinary != "" { + cmd = []string{"/integration.test"} + flagPrefix = "-test." + } if config.TestPattern != "" { - cmd = append(cmd, "-run", config.TestPattern) + cmd = append(cmd, flagPrefix+"run", config.TestPattern) } if config.FailFast { - cmd = append(cmd, "-failfast") + cmd = append(cmd, flagPrefix+"failfast") } - cmd = append(cmd, "-timeout", config.Timeout.String()) - cmd = append(cmd, "-v") + cmd = append(cmd, flagPrefix+"timeout", config.Timeout.String()) + cmd = append(cmd, flagPrefix+"v") return cmd } -// createGoTestContainer creates a Docker container configured for running integration tests. -func createGoTestContainer(ctx context.Context, cli *client.Client, config *RunConfig, containerName, logsDir string, goTestCmd []string) (container.CreateResponse, error) { +// createTestContainer creates a Docker container configured for running integration tests. +func createTestContainer(ctx context.Context, cli *client.Client, config *RunConfig, containerName, logsDir string, testCmd []string) (container.CreateResponse, error) { pwd, err := os.Getwd() if err != nil { return container.CreateResponse{}, fmt.Errorf("getting working directory: %w", err) @@ -254,12 +272,14 @@ func createGoTestContainer(ctx context.Context, cli *client.Client, config *RunC } } - // Set GOCACHE to a known location (used by both bind mount and volume cases) - env = append(env, "GOCACHE=/cache/go-build") + if config.TestBinary == "" { + // Used by both bind mount and volume cases for source-based runs. + env = append(env, "GOCACHE=/cache/go-build") + } containerConfig := &container.Config{ - Image: "golang:" + config.GoVersion, - Cmd: goTestCmd, + Image: testRunnerImage(config), + Cmd: testCmd, Env: env, WorkingDir: projectRoot + "/integration", Tty: true, @@ -269,6 +289,12 @@ func createGoTestContainer(ctx context.Context, cli *client.Client, config *RunC }, } + if config.TestBinary != "" { + // A custom server image may have its own entrypoint; run the tests directly. + containerConfig.Entrypoint = testCmd[:1] + containerConfig.Cmd = testCmd[1:] + } + // Get the correct Docker socket path from the current context dockerSocketPath := getDockerSocketPath() @@ -286,27 +312,36 @@ func createGoTestContainer(ctx context.Context, cli *client.Client, config *RunC // otherwise fall back to Docker volumes for local development var mounts []mount.Mount - goCache := os.Getenv("HEADSCALE_INTEGRATION_GO_CACHE") - goBuildCache := os.Getenv("HEADSCALE_INTEGRATION_GO_BUILD_CACHE") - - if goCache != "" { - binds = append(binds, goCache+":/go") - } else { + if config.TestBinary != "" { mounts = append(mounts, mount.Mount{ - Type: mount.TypeVolume, - Source: "hs-integration-go-cache", - Target: "/go", + Type: mount.TypeBind, + Source: config.TestBinary, + Target: "/integration.test", + ReadOnly: true, }) - } - - if goBuildCache != "" { - binds = append(binds, goBuildCache+":/cache/go-build") } else { - mounts = append(mounts, mount.Mount{ - Type: mount.TypeVolume, - Source: "hs-integration-go-build-cache", - Target: "/cache/go-build", - }) + goCache := os.Getenv("HEADSCALE_INTEGRATION_GO_CACHE") + goBuildCache := os.Getenv("HEADSCALE_INTEGRATION_GO_BUILD_CACHE") + + if goCache != "" { + binds = append(binds, goCache+":/go") + } else { + mounts = append(mounts, mount.Mount{ + Type: mount.TypeVolume, + Source: "hs-integration-go-cache", + Target: "/go", + }) + } + + if goBuildCache != "" { + binds = append(binds, goBuildCache+":/cache/go-build") + } else { + mounts = append(mounts, mount.Mount{ + Type: mount.TypeVolume, + Source: "hs-integration-go-build-cache", + Target: "/cache/go-build", + }) + } } hostConfig := &container.HostConfig{ @@ -330,8 +365,12 @@ func streamAndWait(ctx context.Context, cli *client.Client, containerID string) } defer out.Close() + logsDone := make(chan struct{}) + go func() { _, _ = io.Copy(os.Stdout, out) + + close(logsDone) }() statusCh, errCh := cli.ContainerWait(ctx, containerID, container.WaitConditionNotRunning) @@ -341,6 +380,14 @@ func streamAndWait(ctx context.Context, cli *client.Client, containerID string) return -1, fmt.Errorf("waiting for container: %w", err) } case status := <-statusCh: + // ContainerWait can finish before the log stream delivers its tail. + // Drain through EOF so fast test exits retain their final summary. + select { + case <-logsDone: + case <-ctx.Done(): + return int(status.StatusCode), fmt.Errorf("waiting for container logs: %w", ctx.Err()) + } + return int(status.StatusCode), nil } diff --git a/cmd/hi/doctor.go b/cmd/hi/doctor.go index 9291674d8..f05bd93be 100644 --- a/cmd/hi/doctor.go +++ b/cmd/hi/doctor.go @@ -35,6 +35,12 @@ type DoctorResult struct { // runDoctorCheck performs comprehensive pre-flight checks for integration testing. func runDoctorCheck(ctx context.Context) error { + return runPreflightChecks(ctx, true) +} + +// runPreflightChecks leaves image preparation to the tests that need it. +// The standalone doctor additionally checks development tools and images. +func runPreflightChecks(ctx context.Context, full bool) error { results := []DoctorResult{} // Check 1: Docker binary availability @@ -49,11 +55,16 @@ func runDoctorCheck(ctx context.Context) error { results = append(results, checkDockerContext(ctx)) results = append(results, checkDockerSocket(ctx)) results = append(results, checkDockerHubCredentials()) - results = append(results, checkGolangImage(ctx)) + + if full { + results = append(results, checkGolangImage(ctx)) + } } // Check 3: Go installation - results = append(results, checkGoInstallation(ctx)) + if full { + results = append(results, checkGoInstallation(ctx)) + } // Check 4: Git repository results = append(results, checkGitRepository(ctx)) diff --git a/cmd/hi/run.go b/cmd/hi/run.go index 300022088..28b24bf58 100644 --- a/cmd/hi/run.go +++ b/cmd/hi/run.go @@ -11,10 +11,15 @@ import ( "github.com/creachadair/command" ) -var ErrTestPatternRequired = errors.New("test pattern is required as first argument or use --test flag") +var ( + ErrTestPatternRequired = errors.New("test pattern is required as first argument or use --test flag") + ErrUnexpectedTestArguments = errors.New("expected a single test pattern; check flag spelling and shell quoting") + ErrInvalidTestBinary = errors.New("test binary must be an executable regular file") +) type RunConfig struct { TestPattern string `flag:"test,Test pattern to run"` + TestBinary string `flag:"test-binary,Path to a precompiled integration test binary"` Timeout time.Duration `flag:"timeout,default=120m,Test timeout"` FailFast bool `flag:"failfast,default=true,Stop on first test failure"` UsePostgres bool `flag:"postgres,default=false,Use PostgreSQL instead of SQLite"` @@ -29,17 +34,47 @@ type RunConfig struct { TSMemoryLimit float64 `flag:"ts-memory-limit,default=0,Fail test if any Tailscale container exceeds this memory limit in MB (0 = disabled)"` } -// runIntegrationTest executes the integration test workflow. -func runIntegrationTest(env *command.Env) error { - args := env.Args - if len(args) > 0 && runConfig.TestPattern == "" { - runConfig.TestPattern = args[0] +func (c *RunConfig) validate(args []string) error { + if len(args) > 1 || (len(args) != 0 && c.TestPattern != "") { + return ErrUnexpectedTestArguments } - if runConfig.TestPattern == "" { + if len(args) == 1 { + c.TestPattern = args[0] + } + + if c.TestPattern == "" { return ErrTestPatternRequired } + if c.TestBinary != "" { + binary, err := filepath.Abs(c.TestBinary) + if err != nil { + return fmt.Errorf("resolving test binary: %w", err) + } + + info, err := os.Stat(binary) + if err != nil { + return fmt.Errorf("reading test binary: %w", err) + } + + if !info.Mode().IsRegular() || info.Mode().Perm()&0o111 == 0 { + return fmt.Errorf("%s: %w", binary, ErrInvalidTestBinary) + } + + c.TestBinary = binary + } + + return nil +} + +// runIntegrationTest executes the integration test workflow. +func runIntegrationTest(env *command.Env) error { + err := runConfig.validate(env.Args) + if err != nil { + return err + } + if runConfig.GoVersion == "" { runConfig.GoVersion = detectGoVersion() } @@ -49,7 +84,7 @@ func runIntegrationTest(env *command.Env) error { log.Printf("Running pre-flight system checks...") } - err := runDoctorCheck(env.Context()) + err = runPreflightChecks(env.Context(), false) if err != nil { return fmt.Errorf("pre-flight checks failed: %w", err) } @@ -61,6 +96,13 @@ func runIntegrationTest(env *command.Env) error { log.Printf("Use PostgreSQL: %t", runConfig.UsePostgres) } + backend := "sqlite" + if runConfig.UsePostgres { + backend = "postgres" + } + + log.Printf("Database backend: %s", backend) + return runTestContainer(env.Context(), &runConfig) } diff --git a/cmd/hi/run_test.go b/cmd/hi/run_test.go new file mode 100644 index 000000000..48df6e83f --- /dev/null +++ b/cmd/hi/run_test.go @@ -0,0 +1,249 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/creachadair/command" + "github.com/creachadair/flax" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/mount" + "github.com/docker/docker/client" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Exercise the real flag parser: folded YAML shell continuations used to turn +// --postgres into a positional argument, silently selecting SQLite. +func TestRunArguments(t *testing.T) { + tests := []struct { + name string + args []string + postgres bool + wantErr error + }{ + { + name: "postgres before selector", + args: []string{"--postgres=1", "--timeout=15m", "^TestHeadscale$"}, + postgres: true, + }, + { + name: "postgres after selector", + args: []string{"^TestHeadscale$", "--postgres", "--timeout=15m"}, + postgres: true, + }, + { + name: "sqlite", + args: []string{"--postgres=0", "--timeout=15m", "--test=^TestHeadscale$"}, + }, + { + name: "folded shell continuations", + args: []string{"^TestHeadscale$", " --timeout=15m", " --postgres=1"}, + wantErr: ErrUnexpectedTestArguments, + }, + { + name: "extra selector", + args: []string{"^TestHeadscale$", "TestNodeCommand"}, + wantErr: ErrUnexpectedTestArguments, + }, + { + name: "positional selector with test flag", + args: []string{"--test=^TestHeadscale$", "TestNodeCommand"}, + wantErr: ErrUnexpectedTestArguments, + }, + { + name: "missing selector", + wantErr: ErrTestPatternRequired, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var config RunConfig + + cmd := command.C{ + Name: "run", + SetFlags: command.Flags(flax.MustBind, &config), + Run: func(env *command.Env) error { return config.validate(env.Args) }, + } + + err := command.Run(cmd.NewEnv(nil).MergeFlags(true), tt.args) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + return + } + + require.NoError(t, err) + assert.Equal(t, tt.postgres, config.UsePostgres) + assert.Equal(t, "^TestHeadscale$", config.TestPattern) + assert.Equal(t, 15*time.Minute, config.Timeout) + }) + } +} + +func TestCompiledTestContainer(t *testing.T) { + t.Setenv("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE", "headscale:ci-test") + // Neither inherited backend settings nor Go cache mounts should override + // the requested backend or leak into a compiled test execution. + t.Setenv("HEADSCALE_INTEGRATION_POSTGRES", "0") + t.Setenv("HEADSCALE_INTEGRATION_GO_CACHE", "/unused/go") + t.Setenv("HEADSCALE_INTEGRATION_GO_BUILD_CACHE", "/unused/build") + + binary := filepath.Join(t.TempDir(), "integration.test") + require.NoError(t, os.WriteFile(binary, []byte("test executable"), 0o700)) //nolint:gosec // The fixture must be executable to validate --test-binary. + config := RunConfig{ + TestPattern: "^TestAutoApproveMultiNetwork/webauth-user.*$", + TestBinary: binary, + UsePostgres: true, + Timeout: 15 * time.Minute, + FailFast: true, + } + require.NoError(t, config.validate(nil)) + + type createRequest struct { + container.Config + + HostConfig container.HostConfig + } + + requests := make(chan createRequest, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var request createRequest + + err := json.NewDecoder(r.Body).Decode(&request) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + requests <- request + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"Id":"test-runner","Warnings":[]}`)) + })) + t.Cleanup(server.Close) + cli, err := client.NewClientWithOpts(client.WithHost(server.URL), client.WithVersion("1.44")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, cli.Close()) }) + + const runID = "20261007-120000-abcdef" + + logs := t.TempDir() + _, err = createTestContainer(t.Context(), cli, &config, + "headscale-test-suite-"+runID, logs, buildTestCommand(&config)) + require.NoError(t, err) + + request := <-requests + assert.Equal(t, "headscale:ci-test", request.Image) + // A custom server image may already have a headscale entrypoint. + assert.Equal(t, []string{"/integration.test"}, []string(request.Entrypoint)) + assert.Equal(t, []string{ + "-test.run", config.TestPattern, + "-test.failfast", "-test.timeout", "15m0s", "-test.v", + }, []string(request.Cmd)) + assert.Contains(t, request.Env, "HEADSCALE_INTEGRATION_POSTGRES=1") + assert.NotContains(t, request.Env, "HEADSCALE_INTEGRATION_POSTGRES=0") + assert.Contains(t, request.Env, "HEADSCALE_INTEGRATION_RUN_ID="+runID) + assert.NotContains(t, request.Env, "GOCACHE=/cache/go-build") + assert.Equal(t, runID, request.Labels["hi.run-id"]) + assert.Equal(t, "integration", filepath.Base(request.WorkingDir)) + assert.Contains(t, request.HostConfig.Binds, logs+":/tmp/control") + assert.NotContains(t, request.HostConfig.Binds, "/unused/go:/go") + assert.Equal(t, []mount.Mount{{ + Type: mount.TypeBind, Source: binary, Target: "/integration.test", ReadOnly: true, + }}, request.HostConfig.Mounts) +} + +func TestSourceTestCommand(t *testing.T) { + t.Setenv("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE", "headscale:ci-test") + + config := RunConfig{ + TestPattern: "^TestHeadscale$", GoVersion: "1.27.0", Timeout: time.Minute, + } + assert.Equal(t, []string{"go", "test", "./...", "-run", "^TestHeadscale$", "-timeout", "1m0s", "-v"}, buildTestCommand(&config)) + assert.Equal(t, "golang:1.27.0", testRunnerImage(&config)) + + config.TestBinary = "/integration.test" + + t.Setenv("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE", "") + assert.Equal(t, "golang:1.27.0", testRunnerImage(&config)) +} + +func TestStreamAndWaitDrainsFinalOutput(t *testing.T) { + output, err := os.CreateTemp(t.TempDir(), "test-output") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, output.Close()) }) + + stdout := os.Stdout + os.Stdout = output + + t.Cleanup(func() { os.Stdout = stdout }) + + waitSent := make(chan struct{}) + releaseLogs := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/logs"): + w.Header().Set("Content-Type", "application/vnd.docker.raw-stream") + w.WriteHeader(http.StatusOK) + _ = http.NewResponseController(w).Flush() + + select { + case <-releaseLogs: + _, _ = w.Write([]byte("--- PASS: TestExample (0.01s)\nPASS\n")) + case <-r.Context().Done(): + } + case strings.HasSuffix(r.URL.Path, "/wait"): + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"StatusCode":0}`)) + _ = http.NewResponseController(w).Flush() + + close(waitSent) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(server.Close) + cli, err := client.NewClientWithOpts(client.WithHost(server.URL), client.WithVersion("1.44")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, cli.Close()) }) + + type result struct { + exitCode int + err error + } + + done := make(chan result, 1) + + go func() { + code, err := streamAndWait(t.Context(), cli, "test-runner") + done <- result{code, err} + }() + + <-waitSent + + // Hold back the last log packet after Docker has reported the exit code. + select { + case <-done: + close(releaseLogs) + t.Fatal("returned before the final test output was delivered") + case <-time.After(50 * time.Millisecond): + } + + close(releaseLogs) + + got := <-done + require.NoError(t, got.err) + assert.Zero(t, got.exitCode) + + data, err := os.ReadFile(output.Name()) + require.NoError(t, err) + assert.Equal(t, "--- PASS: TestExample (0.01s)\nPASS\n", string(data)) +} diff --git a/integration/auth_key_test.go b/integration/auth_key_test.go index 7aa928c59..d51202958 100644 --- a/integration/auth_key_test.go +++ b/integration/auth_key_test.go @@ -1,7 +1,6 @@ package integration import ( - "fmt" "net/netip" "slices" "strconv" @@ -23,196 +22,182 @@ import ( func TestAuthKeyLogoutAndReloginSameUser(t *testing.T) { IntegrationSkip(t) - for _, https := range []bool{true, false} { - t.Run(fmt.Sprintf("with-https-%t", https), func(t *testing.T) { - spec := ScenarioSpec{ - NodesPerUser: len(MustTestVersions), - Users: []string{"user1", "user2"}, - } - - scenario, err := NewScenario(spec) - - require.NoError(t, err) - defer scenario.ShutdownAssertNoPanics(t) - - opts := []hsic.Option{ - hsic.WithTestName("authkey-relogsame"), - } - - err = scenario.CreateHeadscaleEnv([]tsic.Option{}, opts...) - requireNoErrHeadscaleEnv(t, err) - - allClients, err := scenario.ListTailscaleClients() - requireNoErrListClients(t, err) - - allIps, err := scenario.ListTailscaleClientsIPs() - requireNoErrListClientIPs(t, err) - - err = scenario.WaitForTailscaleSync() - requireNoErrSync(t, err) - - headscale, err := scenario.Headscale() - requireNoErrGetHeadscale(t, err) - - expectedNodes := collectExpectedNodeIDs(t, allClients) - requireAllClientsOnline(t, headscale, expectedNodes, true, "all clients should be connected", integrationutil.ScaledTimeout(120*time.Second)) - - // Validate that all nodes have [tailcfg.NetInfo] and DERP servers before logout - requireAllClientsNetInfoAndDERP(t, headscale, expectedNodes, "all clients should have NetInfo and DERP before logout", 3*time.Minute) - - // assertClientsState(t, allClients) - - clientIPs := make(map[TailscaleClient][]netip.Addr) - - for _, client := range allClients { - ips, err := client.IPs() - if err != nil { - t.Fatalf("failed to get IPs for client %s: %s", client.Hostname(), err) - } - - clientIPs[client] = ips - } - - var ( - listNodes []*v1.Node - nodeCountBeforeLogout int - ) - - assert.EventuallyWithT(t, func(c *assert.CollectT) { - var err error - - listNodes, err = headscale.ListNodes() - assert.NoError(c, err) - assert.Len(c, listNodes, len(allClients)) - - for _, node := range listNodes { - assertLastSeenSetWithCollect(c, node) - } - }, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "Waiting for expected node list before logout") - - nodeCountBeforeLogout = len(listNodes) - t.Logf("node count before logout: %d", nodeCountBeforeLogout) - - for _, client := range allClients { - err := client.Logout() - if err != nil { - t.Fatalf("failed to logout client %s: %s", client.Hostname(), err) - } - } - - err = scenario.WaitForTailscaleLogout() - requireNoErrLogout(t, err) - - // After taking down all nodes, verify all systems show nodes offline - requireAllClientsOnline(t, headscale, expectedNodes, false, "all nodes should have logged out", integrationutil.ScaledTimeout(120*time.Second)) - - t.Logf("all clients logged out") - - t.Logf("Validating node persistence after logout at %s", time.Now().Format(TimestampFormat)) - assert.EventuallyWithT(t, func(ct *assert.CollectT) { - var err error - - listNodes, err = headscale.ListNodes() - assert.NoError(ct, err, "Failed to list nodes after logout") - assert.Len(ct, listNodes, nodeCountBeforeLogout, "Node count should match before logout count - expected %d nodes, got %d", nodeCountBeforeLogout, len(listNodes)) - }, integrationutil.StatusReadyTimeout, 2*time.Second, "validating node persistence after logout (nodes should remain in database)") - - for _, node := range listNodes { - assertLastSeenSet(t, node) - } - - // if the server is not running with HTTPS, we have to wait a bit before - // reconnection as the newest Tailscale client has a measure that will only - // reconnect over HTTPS if they saw a noise connection previously. - // https://github.com/tailscale/tailscale/commit/1eaad7d3deb0815e8932e913ca1a862afa34db38 - // https://github.com/juanfont/headscale/issues/2164 - if !https { - //nolint:forbidigo // Intentional delay: Tailscale client requires 5 min wait before reconnecting over non-HTTPS - time.Sleep(5 * time.Minute) - } - - userMap, err := headscale.MapUsers() - require.NoError(t, err) - - for _, userName := range spec.Users { - key, err := scenario.CreatePreAuthKey(userMap[userName].GetId(), true, false) - if err != nil { - t.Fatalf("failed to create pre-auth key for user %s: %s", userName, err) - } - - err = scenario.RunTailscaleUp(userName, headscale.GetEndpoint(), key.GetKey()) - if err != nil { - t.Fatalf("failed to run tailscale up for user %s: %s", userName, err) - } - } - - t.Logf("Validating node persistence after relogin at %s", time.Now().Format(TimestampFormat)) - assert.EventuallyWithT(t, func(ct *assert.CollectT) { - var err error - - listNodes, err = headscale.ListNodes() - assert.NoError(ct, err, "Failed to list nodes after relogin") - assert.Len(ct, listNodes, nodeCountBeforeLogout, "Node count should remain unchanged after relogin - expected %d nodes, got %d", nodeCountBeforeLogout, len(listNodes)) - }, integrationutil.HAConvergeTimeout, 2*time.Second, "validating node count stability after same-user auth key relogin") - - for _, node := range listNodes { - assertLastSeenSet(t, node) - } - - requireAllClientsOnline(t, headscale, expectedNodes, true, "all clients should be connected to batcher", integrationutil.ScaledTimeout(120*time.Second)) - - // Wait for Tailscale sync before validating [tailcfg.NetInfo] to ensure proper state propagation - err = scenario.WaitForTailscaleSync() - requireNoErrSync(t, err) - - // Validate that all nodes have [tailcfg.NetInfo] and DERP servers after reconnection - requireAllClientsNetInfoAndDERP(t, headscale, expectedNodes, "all clients should have NetInfo and DERP after reconnection", 3*time.Minute) - - err = scenario.WaitForTailscaleSync() - requireNoErrSync(t, err) - - allAddrs := lo.Map(allIps, func(x netip.Addr, index int) string { - return x.String() - }) - - assertPingAll(t, allClients, allAddrs) - - for _, client := range allClients { - ips, err := client.IPs() - if err != nil { - t.Fatalf("failed to get IPs for client %s: %s", client.Hostname(), err) - } - - // lets check if the IPs are the same - if len(ips) != len(clientIPs[client]) { - t.Fatalf("IPs changed for client %s", client.Hostname()) - } - - for _, ip := range ips { - if !slices.Contains(clientIPs[client], ip) { - t.Fatalf( - "IPs changed for client %s. Used to be %v now %v", - client.Hostname(), - clientIPs[client], - ips, - ) - } - } - } - - assert.EventuallyWithT(t, func(c *assert.CollectT) { - var err error - - listNodes, err = headscale.ListNodes() - assert.NoError(c, err) - assert.Len(c, listNodes, nodeCountBeforeLogout) - - for _, node := range listNodes { - assertLastSeenSetWithCollect(c, node) - } - }, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "Waiting for node list after relogin") - }) + spec := ScenarioSpec{ + NodesPerUser: len(MustTestVersions), + Users: []string{"user1", "user2"}, } + + scenario, err := NewScenario(spec) + + require.NoError(t, err) + defer scenario.ShutdownAssertNoPanics(t) + + opts := []hsic.Option{ + hsic.WithTestName("authkey-relogsame"), + } + + err = scenario.CreateHeadscaleEnv([]tsic.Option{}, opts...) + requireNoErrHeadscaleEnv(t, err) + + allClients, err := scenario.ListTailscaleClients() + requireNoErrListClients(t, err) + + allIps, err := scenario.ListTailscaleClientsIPs() + requireNoErrListClientIPs(t, err) + + err = scenario.WaitForTailscaleSync() + requireNoErrSync(t, err) + + headscale, err := scenario.Headscale() + requireNoErrGetHeadscale(t, err) + + expectedNodes := collectExpectedNodeIDs(t, allClients) + requireAllClientsOnline(t, headscale, expectedNodes, true, "all clients should be connected", integrationutil.ScaledTimeout(120*time.Second)) + + // Validate that all nodes have [tailcfg.NetInfo] and DERP servers before logout + requireAllClientsNetInfoAndDERP(t, headscale, expectedNodes, "all clients should have NetInfo and DERP before logout", 3*time.Minute) + + // assertClientsState(t, allClients) + + clientIPs := make(map[TailscaleClient][]netip.Addr) + + for _, client := range allClients { + ips, err := client.IPs() + if err != nil { + t.Fatalf("failed to get IPs for client %s: %s", client.Hostname(), err) + } + + clientIPs[client] = ips + } + + var ( + listNodes []*v1.Node + nodeCountBeforeLogout int + ) + + assert.EventuallyWithT(t, func(c *assert.CollectT) { + var err error + + listNodes, err = headscale.ListNodes() + assert.NoError(c, err) + assert.Len(c, listNodes, len(allClients)) + + for _, node := range listNodes { + assertLastSeenSetWithCollect(c, node) + } + }, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "Waiting for expected node list before logout") + + nodeCountBeforeLogout = len(listNodes) + t.Logf("node count before logout: %d", nodeCountBeforeLogout) + + for _, client := range allClients { + err := client.Logout() + if err != nil { + t.Fatalf("failed to logout client %s: %s", client.Hostname(), err) + } + } + + err = scenario.WaitForTailscaleLogout() + requireNoErrLogout(t, err) + + // After taking down all nodes, verify all systems show nodes offline + requireAllClientsOnline(t, headscale, expectedNodes, false, "all nodes should have logged out", integrationutil.ScaledTimeout(120*time.Second)) + + t.Logf("all clients logged out") + + t.Logf("Validating node persistence after logout at %s", time.Now().Format(TimestampFormat)) + assert.EventuallyWithT(t, func(ct *assert.CollectT) { + var err error + + listNodes, err = headscale.ListNodes() + assert.NoError(ct, err, "Failed to list nodes after logout") + assert.Len(ct, listNodes, nodeCountBeforeLogout, "Node count should match before logout count - expected %d nodes, got %d", nodeCountBeforeLogout, len(listNodes)) + }, integrationutil.StatusReadyTimeout, 2*time.Second, "validating node persistence after logout (nodes should remain in database)") + + for _, node := range listNodes { + assertLastSeenSet(t, node) + } + + userMap, err := headscale.MapUsers() + require.NoError(t, err) + + for _, userName := range spec.Users { + key, err := scenario.CreatePreAuthKey(userMap[userName].GetId(), true, false) + if err != nil { + t.Fatalf("failed to create pre-auth key for user %s: %s", userName, err) + } + + err = scenario.RunTailscaleUp(userName, headscale.GetEndpoint(), key.GetKey()) + if err != nil { + t.Fatalf("failed to run tailscale up for user %s: %s", userName, err) + } + } + + t.Logf("Validating node persistence after relogin at %s", time.Now().Format(TimestampFormat)) + assert.EventuallyWithT(t, func(ct *assert.CollectT) { + var err error + + listNodes, err = headscale.ListNodes() + assert.NoError(ct, err, "Failed to list nodes after relogin") + assert.Len(ct, listNodes, nodeCountBeforeLogout, "Node count should remain unchanged after relogin - expected %d nodes, got %d", nodeCountBeforeLogout, len(listNodes)) + }, integrationutil.HAConvergeTimeout, 2*time.Second, "validating node count stability after same-user auth key relogin") + + for _, node := range listNodes { + assertLastSeenSet(t, node) + } + + requireAllClientsOnline(t, headscale, expectedNodes, true, "all clients should be connected to batcher", integrationutil.ScaledTimeout(120*time.Second)) + + // Wait for Tailscale sync before validating [tailcfg.NetInfo] to ensure proper state propagation + err = scenario.WaitForTailscaleSync() + requireNoErrSync(t, err) + + // Validate that all nodes have [tailcfg.NetInfo] and DERP servers after reconnection + requireAllClientsNetInfoAndDERP(t, headscale, expectedNodes, "all clients should have NetInfo and DERP after reconnection", 3*time.Minute) + + err = scenario.WaitForTailscaleSync() + requireNoErrSync(t, err) + + allAddrs := lo.Map(allIps, func(x netip.Addr, index int) string { + return x.String() + }) + + assertPingAll(t, allClients, allAddrs) + + for _, client := range allClients { + ips, err := client.IPs() + if err != nil { + t.Fatalf("failed to get IPs for client %s: %s", client.Hostname(), err) + } + + // lets check if the IPs are the same + if len(ips) != len(clientIPs[client]) { + t.Fatalf("IPs changed for client %s", client.Hostname()) + } + + for _, ip := range ips { + if !slices.Contains(clientIPs[client], ip) { + t.Fatalf( + "IPs changed for client %s. Used to be %v now %v", + client.Hostname(), + clientIPs[client], + ips, + ) + } + } + } + + assert.EventuallyWithT(t, func(c *assert.CollectT) { + var err error + + listNodes, err = headscale.ListNodes() + assert.NoError(c, err) + assert.Len(c, listNodes, nodeCountBeforeLogout) + + for _, node := range listNodes { + assertLastSeenSetWithCollect(c, node) + } + }, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "Waiting for node list after relogin") } // This test will first log in two sets of nodes to two sets of users, then @@ -353,120 +338,106 @@ func TestAuthKeyLogoutAndReloginNewUser(t *testing.T) { func TestAuthKeyLogoutAndReloginSameUserExpiredKey(t *testing.T) { IntegrationSkip(t) - for _, https := range []bool{true, false} { - t.Run(fmt.Sprintf("with-https-%t", https), func(t *testing.T) { - spec := ScenarioSpec{ - NodesPerUser: len(MustTestVersions), - Users: []string{"user1", "user2"}, - } + spec := ScenarioSpec{ + NodesPerUser: len(MustTestVersions), + Users: []string{"user1", "user2"}, + } - scenario, err := NewScenario(spec) + scenario, err := NewScenario(spec) - require.NoError(t, err) - defer scenario.ShutdownAssertNoPanics(t) + require.NoError(t, err) + defer scenario.ShutdownAssertNoPanics(t) - opts := []hsic.Option{ - hsic.WithTestName("authkey-rlogexpired"), - } + opts := []hsic.Option{ + hsic.WithTestName("authkey-rlogexpired"), + } - err = scenario.CreateHeadscaleEnv([]tsic.Option{}, opts...) - requireNoErrHeadscaleEnv(t, err) + err = scenario.CreateHeadscaleEnv([]tsic.Option{}, opts...) + requireNoErrHeadscaleEnv(t, err) - allClients, err := scenario.ListTailscaleClients() - requireNoErrListClients(t, err) + allClients, err := scenario.ListTailscaleClients() + requireNoErrListClients(t, err) - err = scenario.WaitForTailscaleSync() - requireNoErrSync(t, err) + err = scenario.WaitForTailscaleSync() + requireNoErrSync(t, err) - // assertClientsState(t, allClients) + // assertClientsState(t, allClients) - clientIPs := make(map[TailscaleClient][]netip.Addr) + clientIPs := make(map[TailscaleClient][]netip.Addr) - for _, client := range allClients { - ips, err := client.IPs() - if err != nil { - t.Fatalf("failed to get IPs for client %s: %s", client.Hostname(), err) - } + for _, client := range allClients { + ips, err := client.IPs() + if err != nil { + t.Fatalf("failed to get IPs for client %s: %s", client.Hostname(), err) + } - clientIPs[client] = ips - } + clientIPs[client] = ips + } - headscale, err := scenario.Headscale() - requireNoErrGetHeadscale(t, err) + headscale, err := scenario.Headscale() + requireNoErrGetHeadscale(t, err) - // Collect expected node IDs for validation - expectedNodes := collectExpectedNodeIDs(t, allClients) + // Collect expected node IDs for validation + expectedNodes := collectExpectedNodeIDs(t, allClients) - // Validate initial connection state - requireAllClientsOnline(t, headscale, expectedNodes, true, "all clients should be connected after initial login", integrationutil.ScaledTimeout(120*time.Second)) - requireAllClientsNetInfoAndDERP(t, headscale, expectedNodes, "all clients should have NetInfo and DERP after initial login", 3*time.Minute) + // Validate initial connection state + requireAllClientsOnline(t, headscale, expectedNodes, true, "all clients should be connected after initial login", integrationutil.ScaledTimeout(120*time.Second)) + requireAllClientsNetInfoAndDERP(t, headscale, expectedNodes, "all clients should have NetInfo and DERP after initial login", 3*time.Minute) - var ( - listNodes []*v1.Node - nodeCountBeforeLogout int - ) + var ( + listNodes []*v1.Node + nodeCountBeforeLogout int + ) - assert.EventuallyWithT(t, func(c *assert.CollectT) { - var err error + assert.EventuallyWithT(t, func(c *assert.CollectT) { + var err error - listNodes, err = headscale.ListNodes() - assert.NoError(c, err) - assert.Len(c, listNodes, len(allClients)) - }, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "Waiting for expected node list before logout") + listNodes, err = headscale.ListNodes() + assert.NoError(c, err) + assert.Len(c, listNodes, len(allClients)) + }, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "Waiting for expected node list before logout") - nodeCountBeforeLogout = len(listNodes) - t.Logf("node count before logout: %d", nodeCountBeforeLogout) + nodeCountBeforeLogout = len(listNodes) + t.Logf("node count before logout: %d", nodeCountBeforeLogout) - for _, client := range allClients { - err := client.Logout() - if err != nil { - t.Fatalf("failed to logout client %s: %s", client.Hostname(), err) - } - } + for _, client := range allClients { + err := client.Logout() + if err != nil { + t.Fatalf("failed to logout client %s: %s", client.Hostname(), err) + } + } - err = scenario.WaitForTailscaleLogout() - requireNoErrLogout(t, err) + err = scenario.WaitForTailscaleLogout() + requireNoErrLogout(t, err) - // Validate that all nodes are offline after logout - requireAllClientsOnline(t, headscale, expectedNodes, false, "all nodes should be offline after logout", integrationutil.ScaledTimeout(120*time.Second)) + // Validate that all nodes are offline after logout + requireAllClientsOnline(t, headscale, expectedNodes, false, "all nodes should be offline after logout", integrationutil.ScaledTimeout(120*time.Second)) - t.Logf("all clients logged out") + t.Logf("all clients logged out") - // if the server is not running with HTTPS, we have to wait a bit before - // reconnection as the newest Tailscale client has a measure that will only - // reconnect over HTTPS if they saw a noise connection previously. - // https://github.com/tailscale/tailscale/commit/1eaad7d3deb0815e8932e913ca1a862afa34db38 - // https://github.com/juanfont/headscale/issues/2164 - if !https { - //nolint:forbidigo // Intentional delay: Tailscale client requires 5 min wait before reconnecting over non-HTTPS - time.Sleep(5 * time.Minute) - } + userMap, err := headscale.MapUsers() + require.NoError(t, err) - userMap, err := headscale.MapUsers() - require.NoError(t, err) + for _, userName := range spec.Users { + key, err := scenario.CreatePreAuthKey(userMap[userName].GetId(), true, false) + if err != nil { + t.Fatalf("failed to create pre-auth key for user %s: %s", userName, err) + } - for _, userName := range spec.Users { - key, err := scenario.CreatePreAuthKey(userMap[userName].GetId(), true, false) - if err != nil { - t.Fatalf("failed to create pre-auth key for user %s: %s", userName, err) - } + // Expire the key so it can't be used + _, err = headscale.Execute( + []string{ + "headscale", + "preauthkeys", + "expire", + "--id", + strconv.FormatUint(key.GetId(), 10), + }) + require.NoError(t, err) + require.NoError(t, err) - // Expire the key so it can't be used - _, err = headscale.Execute( - []string{ - "headscale", - "preauthkeys", - "expire", - "--id", - strconv.FormatUint(key.GetId(), 10), - }) - require.NoError(t, err) - require.NoError(t, err) - - err = scenario.RunTailscaleUp(userName, headscale.GetEndpoint(), key.GetKey()) - assert.ErrorContains(t, err, "authkey expired") - } - }) + err = scenario.RunTailscaleUp(userName, headscale.GetEndpoint(), key.GetKey()) + assert.ErrorContains(t, err, "authkey expired") } } diff --git a/integration/auth_oidc_test.go b/integration/auth_oidc_test.go index 96bd980e5..e513863d1 100644 --- a/integration/auth_oidc_test.go +++ b/integration/auth_oidc_test.go @@ -938,6 +938,9 @@ func TestOIDCFollowUpUrl(t *testing.T) { _, err = doLoginURL(ts.Hostname(), newUrl) require.NoError(t, err) + err = ts.WaitForRunning(integrationutil.PeerSyncTimeout()) + require.NoError(t, err) + listUsers, err = headscale.ListUsers() require.NoError(t, err) assert.Len(t, listUsers, 1) diff --git a/integration/cli_policy_test.go b/integration/cli_policy_test.go index 9773b895a..d8d5ffb7e 100644 --- a/integration/cli_policy_test.go +++ b/integration/cli_policy_test.go @@ -7,7 +7,9 @@ import ( policyv2 "github.com/juanfont/headscale/hscontrol/policy/v2" "github.com/juanfont/headscale/hscontrol/types" "github.com/juanfont/headscale/integration/hsic" + "github.com/juanfont/headscale/integration/integrationutil" "github.com/juanfont/headscale/integration/tsic" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "tailscale.com/tailcfg" ) @@ -22,9 +24,10 @@ import ( // - bypass: no-bypass talks to the server over gRPC; bypass opens the // database directly. // -// Each row spins up its own scenario because policy_mode is fixed at boot -// via `HEADSCALE_POLICY_MODE`. The two users + two nodes give the tests -// block real `user@` aliases to resolve against. +// Each policy mode shares one scenario across its sequential checks because +// policy_mode is fixed at boot. The two users + two nodes give the tests +// block real `user@` aliases to resolve against. Check inputs are separate +// from the live policy, which must remain unchanged after every check. func TestPolicyCheckCommand(t *testing.T) { IntegrationSkip(t) @@ -77,44 +80,56 @@ func TestPolicyCheckCommand(t *testing.T) { type row struct { name string - policyMode string fixture fixture bypass bool wantErr string wantStdout string } - modes := []string{"file", "database"} //nolint:goconst // axis labels match HEADSCALE_POLICY_MODE values + modes := []types.PolicyMode{types.PolicyModeFile, types.PolicyModeDB} bypasses := []bool{false, true} - rows := make([]row, 0, len(modes)*len(fixtures)*len(bypasses)) + rows := make([]row, 0, len(fixtures)*len(bypasses)) - for _, mode := range modes { - for _, f := range fixtures { - for _, bypass := range bypasses { - suffix := "no-bypass" - if bypass { - suffix = "bypass" - } - - r := row{ - name: mode + "-" + f.name + "-" + suffix, - policyMode: mode, - fixture: f, - bypass: bypass, - wantStdout: "Policy is valid", - } - if f.name == "acl-plus-failing-tests" { - r.wantErr = "test(s) failed" - r.wantStdout = "" - } - - rows = append(rows, r) + for _, f := range fixtures { + for _, bypass := range bypasses { + suffix := "no-bypass" + if bypass { + suffix = "bypass" } + + r := row{ + name: f.name + "-" + suffix, + fixture: f, + bypass: bypass, + wantStdout: "Policy is valid", + } + if f.name == "acl-plus-failing-tests" { + r.wantErr = "test(s) failed" + r.wantStdout = "" + } + + rows = append(rows, r) } } - for _, tt := range rows { - t.Run(tt.name, func(t *testing.T) { + // Use a live policy distinct from every check fixture so an accidental + // policy update is observable, including for the ACL-only fixture. + livePolicy := policyv2.Policy{ + ACLs: []policyv2.ACL{ + { + Action: policyv2.ActionAccept, + Sources: []policyv2.Alias{wildcard()}, + Destinations: []policyv2.AliasWithPorts{ + aliasWithPorts(wildcard(), tailcfg.PortRangeAny), + }, + }, + }, + } + livePolicyBytes, err := json.Marshal(livePolicy) + require.NoError(t, err) + + for _, mode := range modes { + t.Run(string(mode), func(t *testing.T) { spec := ScenarioSpec{ NodesPerUser: 1, Users: []string{"user1", "user2"}, //nolint:goconst // matches usernamep("user1@")/("user2@") above @@ -123,45 +138,62 @@ func TestPolicyCheckCommand(t *testing.T) { scenario, err := NewScenario(spec) require.NoError(t, err) - defer scenario.ShutdownAssertNoPanics(t) + t.Cleanup(func() { scenario.ShutdownAssertNoPanics(t) }) err = scenario.CreateHeadscaleEnv( []tsic.Option{}, hsic.WithTestName("cli-policycheck"), - hsic.WithConfigEnv(map[string]string{ - "HEADSCALE_POLICY_MODE": tt.policyMode, //nolint:goconst // env var name from hscontrol/types/config.go - }), + hsic.WithPolicyMode(mode), + hsic.WithACLPolicy(&livePolicy), ) require.NoError(t, err) headscale, err := scenario.Headscale() require.NoError(t, err) - pBytes, err := json.Marshal(tt.fixture.policy) - require.NoError(t, err) + require.EventuallyWithT(t, func(c *assert.CollectT) { + stdout, err := headscale.Execute([]string{"headscale", "policy", "get"}) + assert.NoError(c, err) + assert.JSONEq(c, string(livePolicyBytes), stdout) + }, integrationutil.StatusReadyTimeout, integrationutil.FastPoll, "live policy should be loaded before checks") - policyFilePath := "/etc/headscale/policy.json" //nolint:goconst // standard headscale policy path - err = headscale.WriteFile(policyFilePath, pBytes) - require.NoError(t, err) + for _, tt := range rows { + t.Run(tt.name, func(t *testing.T) { + pBytes, err := json.Marshal(tt.fixture.policy) + require.NoError(t, err) - cmd := []string{"headscale", "policy", "check", "-f", policyFilePath} //nolint:goconst // CLI invocation - if tt.bypass { - // --force suppresses the "is the server running?" - // confirmation prompt so the command can run - // non-interactively under the test harness. - cmd = append(cmd, "--bypass-grpc-and-access-database-directly", "--force") + policyFilePath := "/etc/headscale/policy-check-" + tt.name + ".json" + err = headscale.WriteFile(policyFilePath, pBytes) + require.NoError(t, err) + + t.Cleanup(func() { + assert.EventuallyWithT(t, func(c *assert.CollectT) { + stdout, err := headscale.Execute([]string{"headscale", "policy", "get"}) + assert.NoError(c, err) + assert.JSONEq(c, string(livePolicyBytes), stdout) + }, integrationutil.StatusReadyTimeout, integrationutil.FastPoll, "policy check must not alter the live policy") + }) + + cmd := []string{"headscale", "policy", "check", "-f", policyFilePath} //nolint:goconst // CLI invocation + if tt.bypass { + // --force suppresses the "is the server running?" + // confirmation prompt so the command can run + // non-interactively under the test harness. + cmd = append(cmd, "--bypass-grpc-and-access-database-directly", "--force") + } + + stdout, err := headscale.Execute(cmd) + + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + + return + } + + require.NoError(t, err) + require.Contains(t, stdout, tt.wantStdout) + }) } - - stdout, err := headscale.Execute(cmd) - - if tt.wantErr != "" { - require.ErrorContains(t, err, tt.wantErr) - - return - } - - require.NoError(t, err) - require.Contains(t, stdout, tt.wantStdout) }) } } diff --git a/integration/scenario.go b/integration/scenario.go index ba6712ee9..ec7f2a750 100644 --- a/integration/scenario.go +++ b/integration/scenario.go @@ -51,9 +51,12 @@ const ( var usePostgresForTest = envknob.Bool("HEADSCALE_INTEGRATION_POSTGRES") var ( - errNoHeadscaleAvailable = errors.New("no headscale available") - errNoUserAvailable = errors.New("no user available") - errNoClientFound = errors.New("client not found") + errNoHeadscaleAvailable = errors.New("no headscale available") + errNoUserAvailable = errors.New("no user available") + errNoClientFound = errors.New("client not found") + errInvalidHeadscaleImageFormat = errors.New("invalid HEADSCALE_INTEGRATION_HEADSCALE_IMAGE format, expected repository:tag") + errInvalidMockOIDCImage = errors.New("invalid HEADSCALE_INTEGRATION_HEADSCALE_IMAGE format, expected repository:tag") + errMockOIDCImageRequiredInCI = errors.New("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE must be set for mock OIDC in CI") // AllVersions represents a list of Tailscale versions the suite // uses to test compatibility with the [ControlServer]. @@ -684,7 +687,8 @@ func (s *Scenario) CreateTailscaleNodesInUser( s.mu.Lock() - opts = append(opts, + clientOpts := append( + slices.Clone(opts), tsic.WithCACert(cert), tsic.WithHeadscaleName(hostname), tsic.WithExtraHosts(extraHosts), @@ -697,7 +701,7 @@ func (s *Scenario) CreateTailscaleNodesInUser( tsClient, err := tsic.New( s.pool, version, - opts..., + clientOpts..., ) s.mu.Unlock() @@ -1660,15 +1664,38 @@ func (s *Scenario) runMockOIDC(accessTTL time.Duration, users []mockoidc.MockUse // Add integration test labels if running under hi tool dockertestutil.DockerAddIntegrationLabels(mockOidcOptions, "oidc") - if pmockoidc, err := s.pool.BuildAndRunWithBuildOptions( //nolint:noinlineerr - headscaleBuildOptions, - mockOidcOptions, - dockertestutil.DockerRestartPolicy); err == nil { - s.mockOIDC.r = pmockoidc + var container *dockertest.Resource + + // CI already built and loaded the Headscale image, including its mockoidc command. + // Reuse it instead of downloading dependencies and rebuilding for every OIDC test. + if prebuiltImage := os.Getenv("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE"); prebuiltImage != "" { + repo, tag, ok := strings.Cut(prebuiltImage, ":") + if !ok || repo == "" || tag == "" { + return errInvalidMockOIDCImage + } + + mockOidcOptions.Repository = repo + mockOidcOptions.Tag = tag + container, err = s.pool.RunWithOptions( + mockOidcOptions, + dockertestutil.DockerRestartPolicy, + ) + } else if util.IsCI() { + return errMockOIDCImageRequiredInCI } else { - return err + container, err = s.pool.BuildAndRunWithBuildOptions( + headscaleBuildOptions, + mockOidcOptions, + dockertestutil.DockerRestartPolicy, + ) } + if err != nil { + return fmt.Errorf("starting mock OIDC container: %w", err) + } + + s.mockOIDC.r = container + // headscale needs to set up the provider with a specific // IP addr to ensure we get the correct config from the well-known // endpoint. @@ -1753,10 +1780,32 @@ func Webservice(s *Scenario, networkName string) (*dockertest.Resource, error) { ContextDir: dockerContextPath, } - web, err := s.pool.BuildAndRunWithBuildOptions( - webBOpts, - webOpts, - dockertestutil.DockerRestartPolicy) + var ( + web *dockertest.Resource + err error + ) + + if prebuiltImage := os.Getenv("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE"); prebuiltImage != "" { + repo, tag, ok := strings.Cut(prebuiltImage, ":") + if !ok { + return nil, errInvalidHeadscaleImageFormat + } + + webOpts.Repository = repo + webOpts.Tag = tag + + web, err = s.pool.RunWithOptions( + webOpts, + dockertestutil.DockerRestartPolicy, + ) + } else { + web, err = s.pool.BuildAndRunWithBuildOptions( + webBOpts, + webOpts, + dockertestutil.DockerRestartPolicy, + ) + } + if err != nil { return nil, err } diff --git a/integration/tsic/login_test.go b/integration/tsic/login_test.go new file mode 100644 index 000000000..552a1bf94 --- /dev/null +++ b/integration/tsic/login_test.go @@ -0,0 +1,171 @@ +package tsic + +import ( + "context" + "fmt" + "io" + "testing" + "time" + + "github.com/juanfont/headscale/hscontrol/util" + "github.com/juanfont/headscale/integration/dockertestutil" + "github.com/ory/dockertest/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Simulate Docker's streamed output without requiring Docker or a shell. +func TestLoginReturnsURLBeforeCLIExit(t *testing.T) { + const wantURL = "https://headscale.test/register/complete-token" + + for _, stream := range []string{"stdout", "stderr"} { + t.Run(stream, func(t *testing.T) { + partialWritten := make(chan struct{}) + completeURL := make(chan struct{}) + authenticate := make(chan struct{}) + + t.Cleanup(func() { close(authenticate) }) + + login := startLogin([]string{"tailscale", "up"}, func(_ []string, opts dockertest.ExecOptions) (int, error) { + output := opts.StdErr + if stream == "stdout" { + output = opts.StdOut + } + + _, _ = io.WriteString(output, "To authenticate, visit:\n\thttps://headscale.test/reg") + + close(partialWritten) + <-completeURL + + _, _ = io.WriteString(output, "ister/complete-token\n\n") + + <-authenticate + // A later URL must not block output draining or replace the first. + _, _ = io.WriteString(output, "https://headscale.test/register/another-token\nSuccess.\n") + + return 0, nil + }) + + <-partialWritten + + select { + case u := <-login.url: + t.Fatalf("returned incomplete URL: %s", u) + default: + } + + close(completeURL) + + u, err := login.waitForURL(time.Second) + require.NoError(t, err) + require.Equal(t, wantURL, u.String()) + + select { + case <-login.done: + t.Fatal("CLI exited before authentication") + default: + } + + // Permit natural completion and verify its exit status is retained. + authenticate <- struct{}{} + + select { + case <-login.done: + require.NoError(t, login.err) + assert.Contains(t, login.output.String(), "Success.") + case <-time.After(time.Second): + t.Fatal("CLI did not complete after authentication") + } + }) + } +} + +func TestLoginCommandFailure(t *testing.T) { + execErr := io.ErrClosedPipe + + for _, tt := range []struct { + name string + output string + exitCode int + err error + wantErr error + }{ + {name: "exec error", err: execErr, wantErr: execErr}, + {name: "CLI failure", output: "invalid login option", exitCode: 7, wantErr: dockertestutil.ErrDockertestCommandFailed}, + {name: "no URL", output: "Success.\n", wantErr: util.ErrNoURLFound}, + {name: "incomplete URL", output: "https://headscale.test/register/incomplete", wantErr: util.ErrNoURLFound}, + } { + t.Run(tt.name, func(t *testing.T) { + login := startLogin(nil, func(_ []string, opts dockertest.ExecOptions) (int, error) { + _, _ = io.WriteString(opts.StdErr, tt.output) + return tt.exitCode, tt.err + }) + _, err := login.waitForURL(time.Second) + require.ErrorIs(t, err, tt.wantErr) + assert.Contains(t, err.Error(), tt.output) + }) + } +} + +func TestLoginFailureAfterURL(t *testing.T) { + finish := make(chan struct{}) + + t.Cleanup(func() { close(finish) }) + + login := startLogin(nil, func(_ []string, opts dockertest.ExecOptions) (int, error) { + _, _ = io.WriteString(opts.StdErr, "https://headscale.test/register/token\n") + + <-finish + + _, _ = io.WriteString(opts.StdErr, "authentication rejected\n") + + return 1, nil + }) + + _, err := login.waitForURL(time.Second) + require.NoError(t, err) + + client := &TailscaleInContainer{hostname: "client", login: login} + // A URL alone is not a completed login. The caller's wait stays bounded. + require.ErrorIs(t, client.WaitForRunning(0), context.DeadlineExceeded) + + finish <- struct{}{} + + select { + case <-login.done: + for range 2 { + // All observers must see the failure, not just the first channel reader. + err = client.WaitForRunning(time.Second) + require.ErrorIs(t, err, dockertestutil.ErrDockertestCommandFailed) + assert.Contains(t, err.Error(), "authentication rejected") + } + case <-time.After(time.Second): + t.Fatal("CLI did not finish") + } +} + +func TestLoginURLDeadlineDoesNotStopCLI(t *testing.T) { + finish := make(chan struct{}) + + t.Cleanup(func() { close(finish) }) + + login := startLogin(nil, func(_ []string, opts dockertest.ExecOptions) (int, error) { + <-finish + + _, _ = fmt.Fprintln(opts.StdErr, "https://headscale.test/register/token") + + return 0, nil + }) + + _, err := login.waitForURL(0) + require.ErrorIs(t, err, dockertestutil.ErrDockertestCommandTimeout) + + finish <- struct{}{} + + select { + case <-login.done: + require.NoError(t, login.err) + case <-time.After(time.Second): + t.Fatal("CLI did not finish after URL deadline") + } +} diff --git a/integration/tsic/tsic.go b/integration/tsic/tsic.go index 9893f5d5e..f2a6aeac0 100644 --- a/integration/tsic/tsic.go +++ b/integration/tsic/tsic.go @@ -17,6 +17,7 @@ import ( "slices" "strconv" "strings" + "sync" "time" "github.com/cenkalti/backoff/v5" @@ -43,7 +44,9 @@ const ( dockerContextPath = "../." caCertRoot = "/usr/local/share/ca-certificates" dockerExecuteTimeout = 60 * time.Second - tailscaleBin = "tailscale" + // Some OIDC tests deliberately delay authentication for two minutes. + loginTimeout = 5 * time.Minute + tailscaleBin = "tailscale" ) // defaultPingTimeoutVal returns the per-attempt timeout for tailscale ping. @@ -58,17 +61,16 @@ func defaultPingTimeoutVal() time.Duration { } var ( - errTailscalePingFailed = errors.New("ping failed") - errTailscalePingNotDERP = errors.New("ping not via DERP") - errTailscaleNotLoggedIn = errors.New("tailscale not logged in") - errTailscaleWrongPeerCount = errors.New("wrong peer count") - errTailscaleCannotUpWithoutAuthkey = errors.New("cannot up without authkey") - errInvalidClientConfig = errors.New("verifiably invalid client config requested") - errInvalidTailscaleImageFormat = errors.New("invalid HEADSCALE_INTEGRATION_TAILSCALE_IMAGE format, expected repository:tag") - errTailscaleImageRequiredInCI = errors.New("HEADSCALE_INTEGRATION_TAILSCALE_IMAGE must be set in CI for HEAD version") - errContainerNotInitialized = errors.New("container not initialized") - errFQDNNotYetAvailable = errors.New("FQDN not yet available") - errCurlEmptyResponseBody = errors.New("curl returned empty response body") + errTailscalePingFailed = errors.New("ping failed") + errTailscalePingNotDERP = errors.New("ping not via DERP") + errTailscaleNotLoggedIn = errors.New("tailscale not logged in") + errTailscaleWrongPeerCount = errors.New("wrong peer count") + errInvalidClientConfig = errors.New("verifiably invalid client config requested") + errInvalidTailscaleImageFormat = errors.New("invalid HEADSCALE_INTEGRATION_TAILSCALE_IMAGE format, expected repository:tag") + errTailscaleImageRequiredInCI = errors.New("HEADSCALE_INTEGRATION_TAILSCALE_IMAGE must be set in CI for HEAD version") + errContainerNotInitialized = errors.New("container not initialized") + errFQDNNotYetAvailable = errors.New("FQDN not yet available") + errCurlEmptyResponseBody = errors.New("curl returned empty response body") ) const ( @@ -93,6 +95,10 @@ type TailscaleInContainer struct { ips []netip.Addr fqdn string + // Only the current attempt is checked: earlier logins may be deliberately + // abandoned or rejected before the test starts another one. + login *loginAttempt + // optional config caCerts [][]byte headscaleHostname string @@ -698,6 +704,7 @@ func (t *TailscaleInContainer) buildLoginCommand( func (t *TailscaleInContainer) Login( loginServer, authKey string, ) error { + t.login = nil command := t.buildLoginCommand(loginServer, authKey) if _, _, err := t.Execute(command, dockertestutil.ExecuteCommandTimeout(dockerExecuteTimeout)); err != nil { //nolint:noinlineerr @@ -712,34 +719,106 @@ func (t *TailscaleInContainer) Login( return nil } +// loginAttempt streams the first complete URL line while retaining all output. +// done closes only after Docker exec returns; err can then be read repeatedly. +type loginAttempt struct { + mu sync.Mutex + output bytes.Buffer + lineStart int + foundURL bool + url chan *url.URL + done chan struct{} + err error +} + +func (l *loginAttempt) Write(p []byte) (int, error) { + l.mu.Lock() + defer l.mu.Unlock() + + n, _ := l.output.Write(p) + + for !l.foundURL { + line, _, complete := bytes.Cut(l.output.Bytes()[l.lineStart:], []byte{'\n'}) + if !complete { + break + } + + l.lineStart += len(line) + 1 + + u, err := util.ParseLoginURLFromCLILogin(string(line)) + if err == nil { + l.foundURL = true + l.url <- u + } + } + + return n, nil +} + +func startLogin( + command []string, + exec func([]string, dockertest.ExecOptions) (int, error), +) *loginAttempt { + l := &loginAttempt{url: make(chan *url.URL, 1), done: make(chan struct{})} + + go func() { + defer close(l.done) + + exitCode, err := exec(command, dockertest.ExecOptions{StdOut: l, StdErr: l}) + log.Printf("%v finished (exit %d): %s", command, exitCode, l.output.String()) + + switch { + case err != nil: + l.err = fmt.Errorf("tailscale up: %s: %w", l.output.String(), err) + case exitCode != 0: + l.err = fmt.Errorf("tailscale up exited %d: %s: %w", exitCode, l.output.String(), dockertestutil.ErrDockertestCommandFailed) + case !l.foundURL: + l.err = fmt.Errorf("tailscale up: %s: %w", l.output.String(), util.ErrNoURLFound) + } + }() + + return l +} + +func (l *loginAttempt) waitForURL(timeout time.Duration) (*url.URL, error) { + select { + case u := <-l.url: + return u, nil + case <-l.done: + if l.err != nil { + return nil, l.err + } + + return <-l.url, nil + case <-time.After(timeout): + return nil, dockertestutil.ErrDockertestCommandTimeout + } +} + // LoginWithURL runs the login routine on the given Tailscale instance. // This login mechanism uses web + command line flow for authentication. func (t *TailscaleInContainer) LoginWithURL( loginServer string, ) (*url.URL, error) { command := t.buildLoginCommand(loginServer, "") + command = append(command, "--timeout="+loginTimeout.String()) - stdout, stderr, err := t.Execute(command) - if errors.Is(err, errTailscaleNotLoggedIn) { - return nil, errTailscaleCannotUpWithoutAuthkey - } + // Return the URL promptly, but let the CLI finish authentication normally. + // Its own timeout bounds abandoned logins; WaitForRunning checks its exit. + t.login = startLogin(command, t.container.Exec) - defer func() { - if err != nil { - log.Printf("join command: %q", strings.Join(command, " ")) - } - }() - - loginURL, err := util.ParseLoginURLFromCLILogin(stdout + stderr) + u, err := t.login.waitForURL(dockerExecuteTimeout) if err != nil { - return nil, err + return nil, fmt.Errorf("%s fetching login URL: %w", t.hostname, err) } - return loginURL, nil + return u, nil } // Logout runs the logout routine on the given Tailscale instance. func (t *TailscaleInContainer) Logout() error { + t.login = nil + _, _, err := t.Execute([]string{tailscaleBin, "logout"}) if err != nil { return err @@ -1240,7 +1319,7 @@ func (t *TailscaleInContainer) WaitForNeedsLogin(timeout time.Duration) error { } // WaitForRunning blocks until the Tailscale (tailscaled) instance is logged in -// and ready to be used. +// and ready to be used, and the current web login command has exited successfully. func (t *TailscaleInContainer) WaitForRunning(timeout time.Duration) error { return t.waitForBackendState("Running", timeout) } @@ -1252,6 +1331,17 @@ func (t *TailscaleInContainer) waitForBackendState(state string, timeout time.Du ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() + if state == "Running" && t.login != nil { + select { + case <-t.login.done: + if t.login.err != nil { + return fmt.Errorf("%s login failed: %w", t.hostname, t.login.err) + } + case <-ctx.Done(): + return fmt.Errorf("%s waiting for login command: %w", t.hostname, ctx.Err()) + } + } + for { select { case <-ctx.Done():