From ca512f2c57ff4e073ae7035252b2d28c3b41c1e6 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Tue, 10 Nov 2026 14:00:57 +0000 Subject: [PATCH] noise: answer a deleted node's map request with 404 As Tailscale does. A netmap resets the client's map backoff, so ending the stream after one made it re-poll every few milliseconds. Fixes #3688 --- hscontrol/noise.go | 48 ++++------------ hscontrol/noise_test.go | 77 +++++--------------------- hscontrol/poll.go | 3 - hscontrol/servertest/client.go | 4 +- hscontrol/servertest/lifecycle_test.go | 28 +--------- 5 files changed, 29 insertions(+), 131 deletions(-) diff --git a/hscontrol/noise.go b/hscontrol/noise.go index 0bb897f5e..b28ee0db3 100644 --- a/hscontrol/noise.go +++ b/hscontrol/noise.go @@ -344,7 +344,7 @@ func (ns *noiseServer) C2NResponseHandler( nv, ok := ns.headscale.state.GetNodeByID(nodeID) if !ok { - httpError(writer, NewHTTPError(http.StatusNotFound, "node not found", errNodeNotInStore)) + httpError(writer, NewHTTPError(http.StatusNotFound, "node not found", nil)) return } @@ -818,35 +818,12 @@ func (ns *noiseServer) PollNetMapHandler( nv, err := ns.getAndValidateNode(mapRequest) if err != nil { - // The node is gone, but the client does not know that. Tailscale - // clients treat every non-200 on the map path the same way and retry - // forever with loggedIn still set; only a self node whose KeyExpiry is - // in the past drives them to NeedsLogin. There is no MapResponse field - // that says "deleted", so reuse the expiry signal headscale already - // sends for expired nodes. - // See: https://github.com/juanfont/headscale/issues/3410 - if errors.Is(err, errNodeNotInStore) && mapRequest.Stream { - expired := &tailcfg.MapResponse{ - Node: &tailcfg.Node{ - Key: mapRequest.NodeKey, - // Zero means that the key does not expire. Use a fixed, - // ancient non-zero value so this remains expired even when - // the client's clock is substantially behind the server. - KeyExpiry: time.Unix(1, 0).UTC(), - Expired: true, - }, - // This frame opens the stream in place of the initial map. - Debug: ns.headscale.cfg.TailcfgDebug(), - } - - err = writeMapResponse(writer, mapRequest.Compress, true, expired) - if err != nil { - log.Error().Caller().Err(err).Msg("noise map handler: failed to write expired response for deleted node") - } - - return - } - + // A deleted node gets a bare 404, as Tailscale's control server sends; + // clients since 1.106 read it as removal and go to NeedsLogin. Do not + // answer with a netmap: clients reset their map backoff on every netmap, + // so one followed by the end of the stream is retried within + // milliseconds, forever. + // See: https://github.com/juanfont/headscale/issues/3688 httpError(writer, err) return @@ -932,7 +909,7 @@ func (ns *noiseServer) SetDNSHandler(writer http.ResponseWriter, req *http.Reque nv, ok := ns.headscale.state.GetNodeByNodeKey(setDNS.NodeKey) if !ok { - httpError(writer, NewHTTPError(http.StatusNotFound, "node not found", errNodeNotInStore)) + httpError(writer, NewHTTPError(http.StatusNotFound, "node not found", nil)) return } @@ -1117,17 +1094,14 @@ func (ns *noiseServer) RegistrationHandler( } } -// errNodeNotInStore distinguishes an unknown NodeKey from a NodeKey presented -// by the wrong machine key. Both are answered with 404, but only the former -// means the node is gone and its client should re-authenticate. -var errNodeNotInStore = errors.New("node not found") - // getAndValidateNode retrieves the node from the in-memory NodeStore using the // NodeKey and validates that it matches the MachineKey from the Noise session. +// Its 404s end a client's session: clients read a map 404 as removal from the +// tailnet, so a node that still exists must never be answered with one. func (ns *noiseServer) getAndValidateNode(mapRequest tailcfg.MapRequest) (types.NodeView, error) { nv, ok := ns.headscale.state.GetNodeByNodeKey(mapRequest.NodeKey) if !ok { - return types.NodeView{}, NewHTTPError(http.StatusNotFound, "node not found", errNodeNotInStore) + return types.NodeView{}, NewHTTPError(http.StatusNotFound, "node not found", nil) } // Validate that the MachineKey in the Noise session matches the one associated with the NodeKey. diff --git a/hscontrol/noise_test.go b/hscontrol/noise_test.go index 10c03eb1b..804eae6e4 100644 --- a/hscontrol/noise_test.go +++ b/hscontrol/noise_test.go @@ -5,7 +5,6 @@ import ( "context" "crypto/sha256" "encoding/base64" - "encoding/binary" "encoding/json" "errors" "fmt" @@ -29,7 +28,6 @@ import ( "github.com/juanfont/headscale/hscontrol/dns" "github.com/juanfont/headscale/hscontrol/dns/dnstest" "github.com/juanfont/headscale/hscontrol/types" - "github.com/juanfont/headscale/hscontrol/util" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -39,7 +37,6 @@ import ( "tailscale.com/net/tsdial" "tailscale.com/tailcfg" "tailscale.com/types/key" - "tailscale.com/util/zstdframe" ) var privateDiagnosticPaths = []string{ @@ -1541,57 +1538,22 @@ func newMapRequest(t *testing.T, req tailcfg.MapRequest) *http.Request { return httptest.NewRequestWithContext(context.Background(), http.MethodPost, "/machine/map", bytes.NewReader(body)) } -// decodeMapResponse reads a map response frame the way a Tailscale client -// does: a little-endian length prefix followed by a body that is zstd-framed -// whenever the request asked for compression. -func decodeMapResponse(t *testing.T, compress string, body []byte) tailcfg.MapResponse { - t.Helper() - - require.GreaterOrEqual(t, len(body), reservedResponseHeaderSize, "response too short to carry a length prefix") - - size := binary.LittleEndian.Uint32(body[:reservedResponseHeaderSize]) - payload := body[reservedResponseHeaderSize:] - require.Len(t, payload, int(size), "length prefix must match the body it precedes") - - if compress == util.ZstdCompression { - decoded, err := zstdframe.AppendDecode(nil, payload) - require.NoError(t, err, "client decodes every frame as zstd when it asked for zstd") - - payload = decoded - } - - var resp tailcfg.MapResponse - require.NoError(t, json.Unmarshal(payload, &resp)) - - return resp -} - -// TestPollNetMapHandler_DeletedNodeGetsExpiredSelf verifies that a streaming -// map request for a node that no longer exists is answered with an expired -// self node instead of a bare 404. A Tailscale client treats every non-200 on -// the map path identically and retries forever with loggedIn still set; only a -// self node whose KeyExpiry is in the past moves it to NeedsLogin. +// TestPollNetMapHandler_DeletedNodeGetsNotFound pins parity with Tailscale's +// control server: a deleted node's map request gets a bare 404. Clients read a +// map 404 as "removed from the tailnet" ([controlclient.ErrNodeRemoved]) and +// retry with a growing backoff. A 200 carrying a netmap instead resets that +// backoff, so when the stream then ends the client re-polls within +// milliseconds, forever. // -// See: https://github.com/juanfont/headscale/issues/3410 -func TestPollNetMapHandler_DeletedNodeGetsExpiredSelf(t *testing.T) { +// See: https://github.com/juanfont/headscale/issues/3688 +func TestPollNetMapHandler_DeletedNodeGetsNotFound(t *testing.T) { t.Parallel() - for _, tc := range []struct { - compress string - logTail bool - wantDebug *tailcfg.Debug - }{ - {"", false, &tailcfg.Debug{DisableLogTail: true}}, - {util.ZstdCompression, false, &tailcfg.Debug{DisableLogTail: true}}, - {"", true, nil}, - {util.ZstdCompression, true, nil}, - } { - compress := tc.compress - t.Run(fmt.Sprintf("compress=%s/logtail=%t", compress, tc.logTail), func(t *testing.T) { + for _, stream := range []bool{true, false} { + t.Run(fmt.Sprintf("stream=%t", stream), func(t *testing.T) { t.Parallel() app := createTestApp(t) - app.cfg.LogTail.Enabled = tc.logTail user := app.state.CreateUserForTest("deleted-node-user") node := putTestNodeInStore(t, app, user, "deleted-node") @@ -1605,23 +1567,12 @@ func TestPollNetMapHandler_DeletedNodeGetsExpiredSelf(t *testing.T) { rec := httptest.NewRecorder() ns.PollNetMapHandler(rec, newMapRequest(t, tailcfg.MapRequest{ - Version: tailcfg.CurrentCapabilityVersion, - NodeKey: node.NodeKey, - Stream: true, - Compress: compress, + Version: tailcfg.CurrentCapabilityVersion, + NodeKey: node.NodeKey, + Stream: stream, })) - require.Equal(t, http.StatusOK, rec.Code, "body=%q", rec.Body.String()) - - resp := decodeMapResponse(t, compress, rec.Body.Bytes()) - require.NotNil(t, resp.Node, "clients reject an initial map response without a node") - assert.Equal(t, node.NodeKey, resp.Node.Key) - assert.Equal(t, time.Unix(1, 0).UTC(), resp.Node.KeyExpiry, - "a fixed ancient KeyExpiry must remain expired despite client clock skew") - assert.True(t, resp.Node.Expired) - // This frame starts the stream, so it must carry the logtail - // instruction the initial map would have. - assert.Equal(t, tc.wantDebug, resp.Debug) + assert.Equal(t, http.StatusNotFound, rec.Code, "body=%q", rec.Body.String()) }) } } diff --git a/hscontrol/poll.go b/hscontrol/poll.go index 6ed325a5c..2612ddbd3 100644 --- a/hscontrol/poll.go +++ b/hscontrol/poll.go @@ -422,9 +422,6 @@ func (m *mapSession) writeMap(msg *tailcfg.MapResponse) error { // zstd-framed when the client asked for compression, behind a little-endian // length prefix. Tailscale clients request zstd unconditionally and decode // every frame with it, so the compression step is not optional. -// -// It is shared with the deleted-node path in [noiseServer.PollNetMapHandler], -// which has no [mapSession] to write through. func writeMapResponse(w http.ResponseWriter, compress string, flush bool, msg *tailcfg.MapResponse) error { jsonBody, err := json.Marshal(msg) if err != nil { diff --git a/hscontrol/servertest/client.go b/hscontrol/servertest/client.go index 538cfa996..3cbb21a03 100644 --- a/hscontrol/servertest/client.go +++ b/hscontrol/servertest/client.go @@ -496,8 +496,8 @@ func (c *TestClient) UpdateFullNetmap(nm *netmap.NetworkMap) { c.history = append(c.history, nm) } - // Removals sent ahead of their batch and a deleted node's expiry carry - // no ControlTime, so a rebuild with none before it is a response too. + // Removals sent ahead of their batch carry no ControlTime, so a rebuild + // with none before it is a response too. // shortcut: one right after a response a delta client applied in place // goes uncounted; count at the decoder if headscale sends one alone. if !c.stamped { diff --git a/hscontrol/servertest/lifecycle_test.go b/hscontrol/servertest/lifecycle_test.go index 1e214a11b..a9ee23eda 100644 --- a/hscontrol/servertest/lifecycle_test.go +++ b/hscontrol/servertest/lifecycle_test.go @@ -552,8 +552,8 @@ func TestPathsDiscoAndExpiryHelpers(t *testing.T) { // TestFramesCountResponsesWithoutControlTime checks that Frames counts the // responses headscale sends without a ControlTime: a removal ahead of its -// batch, and the expiry a deleted node's new stream opens with. A client -// that rebuilds its netmap on every response counts one frame per rebuild. +// batch. A client that rebuilds its netmap on every response counts one frame +// per rebuild. func TestFramesCountResponsesWithoutControlTime(t *testing.T) { t.Parallel() @@ -591,30 +591,6 @@ func TestFramesCountResponsesWithoutControlTime(t *testing.T) { viewer.WaitForPeerCount(t, 0, 15*time.Second) requireFramePerRebuild(t, viewer) }) - - t.Run("deleted-node-expiry", func(t *testing.T) { - t.Parallel() - - srv := servertest.NewServer(t) - cl := servertest.NewClient(t, srv, "frames-deleted") - cl.WaitForCondition(t, "initial netmap", 15*time.Second, - func(nm *netmap.NetworkMap) bool { return nm != nil }) - - node, ok := srv.State().GetNodeByID(findNodeID(t, srv, "frames-deleted")) - require.True(t, ok) - - changes, err := srv.State().DeleteNode(node) - require.NoError(t, err) - srv.App.Change(changes...) - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - require.NoError(t, cl.RestartPoll(ctx)) - cl.WaitForCondition(t, "expired self", 15*time.Second, - func(nm *netmap.NetworkMap) bool { return nm != nil && nm.SelfNode.Expired() }) - requireFramePerRebuild(t, cl) - }) } func TestNodeExpiryRouteFailover(t *testing.T) {