From 7918187e7ac3eb2ae6b37bb1895c28ecdb531904 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Wed, 10 Jun 2026 12:21:16 +0000 Subject: [PATCH] servertest: add logout/relogin storm repro with poll churn Mirrors the flaky integration relogin tests with production batcher tuning. The churn variant restarts map polls around login like newer tailscaled does; pre-fix it stranded nodes online after logout 2/2. --- hscontrol/servertest/client.go | 102 +++++++++++ hscontrol/servertest/lifecycle_test.go | 243 +++++++++++++++++++++++++ hscontrol/servertest/server.go | 7 + 3 files changed, 352 insertions(+) diff --git a/hscontrol/servertest/client.go b/hscontrol/servertest/client.go index d88e4d23..5a9ff5a4 100644 --- a/hscontrol/servertest/client.go +++ b/hscontrol/servertest/client.go @@ -299,6 +299,108 @@ drained: c.startPoll(tb) } +// LogoutAndDisconnect sends a logout [tailcfg.RegisterRequest] (expiry in +// the past) and tears down the long-poll session, mirroring what +// tailscaled does on `tailscale logout`. The server marks the node +// expired; the poll teardown then triggers the server's disconnect +// grace period, after which the node goes offline. +// +// Safe to call from non-test goroutines: errors are returned, not +// fataled, so many clients can log out concurrently. +func (c *TestClient) LogoutAndDisconnect(ctx context.Context) error { + err := c.direct.TryLogout(ctx) + if err != nil { + return fmt.Errorf("servertest: TryLogout(%s): %w", c.Name, err) + } + + if c.pollCancel != nil { + c.pollCancel() + + select { + case <-c.pollDone: + case <-ctx.Done(): + return fmt.Errorf("servertest: LogoutAndDisconnect(%s): poll did not exit: %w", c.Name, ctx.Err()) + } + } + + return nil +} + +// ReloginAndPoll logs the client back in after [TestClient.LogoutAndDisconnect] +// and starts a fresh long-poll session. [controlclient.Direct.TryLogout] cleared +// the persisted node key, so this generates a new NodeKey and re-registers with +// the same pre-auth key and machine key — the same shape as a real client +// running `tailscale up --authkey=...` after a logout. +// +// Safe to call from non-test goroutines. +func (c *TestClient) ReloginAndPoll(ctx context.Context) error { + url, err := c.direct.TryLogin(ctx, controlclient.LoginDefault) + if err != nil { + return fmt.Errorf("servertest: TryLogin(%s): %w", c.Name, err) + } + + if url != "" { + return fmt.Errorf("servertest: TryLogin(%s): unexpected auth URL %q (expected auto-auth with preauth key)", c.Name, url) //nolint:err113 + } + + // Clear stale netmap state from the previous session so that + // convergence waits observe only the new session's maps. + c.mu.Lock() + c.netmap = nil + c.mu.Unlock() + + for { + select { + case <-c.updates: + continue + default: + } + + break + } + + c.pollCtx, c.pollCancel = context.WithCancel(context.Background()) + c.pollDone = make(chan struct{}) + + go func() { + defer close(c.pollDone) + + _ = c.direct.PollNetMap(c.pollCtx, c) + }() + + return nil +} + +// RestartPoll tears down the current long-poll session and immediately +// starts a new one without re-registering, the way tailscaled restarts +// its map poll on state-machine transitions (pause/unpause around +// login). The node key is unchanged; the server sees a rapid +// disconnect/reconnect. +// +// Safe to call from non-test goroutines. +func (c *TestClient) RestartPoll(ctx context.Context) error { + if c.pollCancel != nil { + c.pollCancel() + + select { + case <-c.pollDone: + case <-ctx.Done(): + return fmt.Errorf("servertest: RestartPoll(%s): old poll did not exit: %w", c.Name, ctx.Err()) + } + } + + c.pollCtx, c.pollCancel = context.WithCancel(context.Background()) + c.pollDone = make(chan struct{}) + + go func() { + defer close(c.pollDone) + + _ = c.direct.PollNetMap(c.pollCtx, c) + }() + + return nil +} + // ReconnectAfter disconnects, waits for d, then reconnects. // The timer works correctly with testing/synctest for // time-controlled tests. diff --git a/hscontrol/servertest/lifecycle_test.go b/hscontrol/servertest/lifecycle_test.go index c7565f39..c1531550 100644 --- a/hscontrol/servertest/lifecycle_test.go +++ b/hscontrol/servertest/lifecycle_test.go @@ -1,12 +1,17 @@ package servertest_test import ( + "context" "fmt" + "math/rand/v2" + "strings" "testing" "time" "github.com/juanfont/headscale/hscontrol/servertest" + "github.com/juanfont/headscale/hscontrol/types" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "tailscale.com/types/netmap" ) @@ -116,3 +121,241 @@ func TestConnectionLifecycle(t *testing.T) { } }) } + +// TestLogoutReloginAllClientsConverge is an in-process reproduction of the +// flaky integration tests TestAuthKeyLogoutAndReloginSameUser, +// TestAuthWebFlowLogoutAndReloginSameUser and +// TestAuthWebFlowLogoutAndReloginNewUser: a full mesh of clients logs out, +// the server marks every node expired and offline, then all clients log +// back in near-simultaneously with fresh NodeKeys. In the flake, a subset +// of clients never converges — their netmaps stay empty through the whole +// retry window even though the server believes everything is connected. +// +// Each client here is a real [controlclient.Direct], so the client-side +// netmap assembly semantics (full peer list vs. delta, patch handling for +// unknown peers) match the real Tailscale client. +func TestLogoutReloginAllClientsConverge(t *testing.T) { + if testing.Short() { + t.Skip("relogin convergence test includes 10s+ disconnect grace per iteration") + } + + const ( + numClients = 12 + iterations = 4 + // Maximum random delay between the relogins of different + // clients, so registrations and fresh map streams interleave + // the way concurrent `tailscale up` invocations do. + reloginStagger = 500 * time.Millisecond + ) + + // Production tuning: the integration flake happens with the default + // 800ms batch delay (large coalescing windows) and a multi-worker + // batcher, so reproduce with the same knobs. + h := servertest.NewHarness(t, numClients, + servertest.WithServerOptions( + servertest.WithBatchDelay(800*time.Millisecond), + servertest.WithBatcherWorkers(types.DefaultBatcherWorkers()), + ), + servertest.WithConvergenceTimeout(60*time.Second), + ) + + for iteration := range iterations { + t.Logf("iteration %d: logging out all clients", iteration) + logoutAllAndWaitOffline(t, h) + + t.Logf("iteration %d: relogging in all clients", iteration) + + clients := h.Clients() + errs := make(chan error, len(clients)) + + for _, c := range clients { + go func() { + time.Sleep(rand.N(reloginStagger)) //nolint:forbidigo,gosec // intentional jitter so relogins interleave; weak random is fine + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + errs <- c.ReloginAndPoll(ctx) + }() + } + + for range clients { + require.NoError(t, <-errs) + } + + // Every client must converge to the full mesh. A stuck client — + // the flake — sits at zero peers and fails here. + deadline := time.Now().Add(30 * time.Second) + for _, c := range clients { + waitForMeshOrDump(t, clients, c, numClients-1, time.Until(deadline)) + } + } +} + +// TestLogoutReloginWithPollChurn is the same logout/relogin storm as +// [TestLogoutReloginAllClientsConverge], but each client also restarts its +// map poll once or twice shortly after logging back in — without +// re-registering — the way newer tailscaled versions cycle their map +// session around login state transitions. The integration flake hits the +// head and unstable clients, which churn their sessions far more than +// older releases, so the rapid session replacement is the prime suspect. +func TestLogoutReloginWithPollChurn(t *testing.T) { + if testing.Short() { + t.Skip("relogin convergence test includes 10s+ disconnect grace per iteration") + } + + const ( + numClients = 12 + iterations = 4 + reloginStagger = 500 * time.Millisecond + ) + + h := servertest.NewHarness(t, numClients, + servertest.WithServerOptions( + servertest.WithBatchDelay(800*time.Millisecond), + servertest.WithBatcherWorkers(types.DefaultBatcherWorkers()), + ), + servertest.WithConvergenceTimeout(60*time.Second), + ) + + for iteration := range iterations { + t.Logf("iteration %d: logging out all clients", iteration) + logoutAllAndWaitOffline(t, h) + + t.Logf("iteration %d: relogging in all clients with poll churn", iteration) + + clients := h.Clients() + errs := make(chan error, len(clients)) + + for _, c := range clients { + go func() { + time.Sleep(rand.N(reloginStagger)) //nolint:forbidigo,gosec // intentional jitter so relogins interleave; weak random is fine + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + err := c.ReloginAndPoll(ctx) + if err != nil { + errs <- err + return + } + + // Churn the map session like a freshly logged-in + // tailscaled: restart the poll once or twice with + // small random gaps. + for range 1 + rand.IntN(2) { //nolint:gosec // weak random is fine for test jitter + time.Sleep(rand.N(400 * time.Millisecond)) //nolint:forbidigo,gosec // intentional jitter between poll restarts; weak random is fine + + err = c.RestartPoll(ctx) + if err != nil { + errs <- err + return + } + } + + errs <- nil + }() + } + + for range clients { + require.NoError(t, <-errs) + } + + deadline := time.Now().Add(30 * time.Second) + for _, c := range clients { + waitForMeshOrDump(t, clients, c, numClients-1, time.Until(deadline)) + } + } +} + +// logoutAllAndWaitOffline logs every client out concurrently, then blocks +// until the server reports each node expired and offline — the integration +// tests' logout barrier, including the ~10s disconnect grace period. +func logoutAllAndWaitOffline(t *testing.T, h *servertest.TestHarness) { + t.Helper() + + clients := h.Clients() + errs := make(chan error, len(clients)) + + for _, c := range clients { + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + errs <- c.LogoutAndDisconnect(ctx) + }() + } + + for range clients { + require.NoError(t, <-errs) + } + + st := h.Server.State() + + require.EventuallyWithT(t, func(c *assert.CollectT) { + for _, node := range st.ListNodes().All() { + assert.True(c, node.IsExpired(), "node %d should be expired after logout", node.ID()) + online := node.IsOnline() + assert.True(c, online.Valid() && !online.Get(), "node %d should be offline after logout", node.ID()) + } + }, 30*time.Second, 100*time.Millisecond, "all nodes expired and offline after logout") +} + +// waitForMeshOrDump waits until client c reports at least wantPeers peers. +// On timeout it dumps every client's view of the mesh before failing, so a +// reproduced flake shows exactly which clients are stuck and what they see. +func waitForMeshOrDump(t *testing.T, all []*servertest.TestClient, c *servertest.TestClient, wantPeers int, timeout time.Duration) { + t.Helper() + + deadline := time.After(timeout) + ticker := time.NewTicker(100 * time.Millisecond) + + defer ticker.Stop() + + for { + if nm := c.Netmap(); nm != nil && len(nm.Peers) >= wantPeers { + return + } + + select { + case <-ticker.C: + case <-deadline: + for _, other := range all { + t.Logf("client %s netmap: %s", other.Name, describeNetmap(other)) + } + + nm := c.Netmap() + + got := 0 + if nm != nil { + got = len(nm.Peers) + } + + t.Fatalf("client %s did not converge: want %d peers, got %d", c.Name, wantPeers, got) + } + } +} + +// describeNetmap renders a client's current netmap as a compact string for +// failure dumps: peer names with their expiry/online flags. +func describeNetmap(c *servertest.TestClient) string { + nm := c.Netmap() + if nm == nil { + return "" + } + + var out strings.Builder + + fmt.Fprintf(&out, "%d peers:", len(nm.Peers)) + + for _, p := range nm.Peers { + hostname := "" + if hi := p.Hostinfo(); hi.Valid() { + hostname = hi.Hostname() + } + + fmt.Fprintf(&out, " %s(id=%d expired=%t online=%v)", hostname, p.ID(), p.KeyExpiry().Before(time.Now()), p.Online()) + } + + return out.String() +} diff --git a/hscontrol/servertest/server.go b/hscontrol/servertest/server.go index 7de2c401..fd20c519 100644 --- a/hscontrol/servertest/server.go +++ b/hscontrol/servertest/server.go @@ -65,6 +65,13 @@ func WithBufferedChanSize(n int) ServerOption { return func(c *serverConfig) { c.bufferedChanSize = n } } +// WithBatcherWorkers sets the number of batcher worker goroutines. +// Defaults to 1 for deterministic tests; pass +// [types.DefaultBatcherWorkers] to match production concurrency. +func WithBatcherWorkers(n int) ServerOption { + return func(c *serverConfig) { c.batcherWorkers = n } +} + // WithEphemeralTimeout sets the ephemeral node inactivity timeout. func WithEphemeralTimeout(d time.Duration) ServerOption { return func(c *serverConfig) { c.ephemeralTimeout = d }