poll: do not cancel ephemeral GC until Connect succeeds

With node.ephemeral.inactivity_timeout set, ephemeral nodes are
usually deleted after they go offline, but under reconnect churn some
departed nodes stayed in the node list as disconnected indefinitely
until removed manually or until Headscale restarted.

Ephemeral cleanup is timer-based via EphemeralGarbageCollector, not a
periodic LastSeen scan. serveLongPoll cancelled any pending GC timer
at the very start of a long-poll attempt and only rescheduled on a
clean disconnect after Connect. If a reconnect cancelled the timer and
then failed before Connect (for example an UpdateNodeFromMapRequest
error), the deferred cleanup saw connectGen == 0 and returned without
Schedule. The node remained offline with no deletion timer and no
reconciler to recover it.

Cancel the ephemeral GC timer only after a successful Connect, so a
failed reconnect leaves an already-armed inactivity timer intact.
Successful reconnects still cancel GC once the node is online, and a
later disconnect reschedules as before.

Add TestFailedReconnectDoesNotCancelEphemeralGC to lock in the
ordering, plus IsScheduled and DeleteNodeFromStoreForTest helpers for
the test.

Fixes #3382

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Igor Serganov
2026-07-24 14:15:54 -07:00
committed by Kristoffer Dalby
parent dc3c0bc587
commit cfd845cb53
4 changed files with 86 additions and 8 deletions
+62
View File
@@ -15,6 +15,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"tailscale.com/tailcfg"
"tailscale.com/types/key"
)
type delayedSuccessResponseWriter struct {
@@ -216,6 +217,67 @@ func TestServeLongPollWritesErrorWhenInitialMapFails(t *testing.T) {
"serveLongPoll must write an HTTP error response when the initial map cannot be built, not an empty 200")
}
// TestFailedReconnectDoesNotCancelEphemeralGC proves that a
// long-poll reconnect attempt which fails before [state.State.Connect] must
// not cancel a previously armed ephemeral GC timer. Cancelling at the start of
// [mapSession.serveLongPoll] left departed ephemeral nodes stuck offline with
// no deletion scheduled (https://github.com/juanfont/headscale/issues/3382).
func TestFailedReconnectDoesNotCancelEphemeralGC(t *testing.T) {
t.Parallel()
app := createTestApp(t)
app.StartEphemeralGCForTest(t)
user := app.state.CreateUserForTest("eph-gc-cancel-user")
pak, err := app.state.CreatePreAuthKey(user.TypedID(), false, true, nil, nil)
require.NoError(t, err)
machineKey := key.NewMachine()
nodeKey := key.NewNode()
_, err = app.handleRegister(context.Background(), tailcfg.RegisterRequest{
Auth: &tailcfg.RegisterResponseAuth{
AuthKey: pak.Key,
},
NodeKey: nodeKey.Public(),
Hostinfo: &tailcfg.Hostinfo{
Hostname: "eph-gc-cancel-node",
},
Expiry: time.Now().Add(24 * time.Hour),
}, machineKey.Public())
require.NoError(t, err)
nodeView, ok := app.state.GetNodeByNodeKey(nodeKey.Public())
require.True(t, ok)
require.True(t, nodeView.IsEphemeral(), "node must be ephemeral so Cancel would arm on long-poll")
node := nodeView.AsStruct()
// Arm a long-lived deletion timer — the state after a normal disconnect
// has called afterServeLongPoll. A long expiry avoids racing the
// fail-before-Connect path below.
app.ephemeralGC.Schedule(node.ID, time.Hour)
require.True(t, app.ephemeralGC.IsScheduled(node.ID), "test sanity: GC timer must be armed")
// Drop the node from the NodeStore so UpdateNodeFromMapRequest fails before
// Connect, while the session still carries an ephemeral AuthKey (so the
// old Cancel-on-entry path would clear the timer).
app.state.DeleteNodeFromStoreForTest(node.ID)
writer := &recordingResponseWriter{}
session := app.newMapSession(context.Background(), tailcfg.MapRequest{
Stream: true,
Version: tailcfg.CapabilityVersion(100),
}, writer, node)
session.serveLongPoll()
assert.GreaterOrEqual(t, writer.statusCode(), http.StatusInternalServerError,
"failed reconnect must write an HTTP error before Connect")
assert.True(t, app.ephemeralGC.IsScheduled(node.ID),
"failed reconnect must not cancel the ephemeral GC timer (issue #3382)")
}
// TestGitHubIssue3129_TransientlyBlockedWriteDoesNotLeaveLiveStaleSession
// tests the scenario reported in
// https://github.com/juanfont/headscale/issues/3129.