noise: answer a deleted node's map request with 404

As Tailscale does. A netmap resets the client's map backoff, so ending
the stream after one made it re-poll every few milliseconds.

Fixes #3688
This commit is contained in:
Kristoffer Dalby authored and Kristoffer Dalby committed 2026-11-11 16:45:24 +01:00
1 parent 934a1f12b8
commit ca512f2c57
5 files changed
+29 -131

No files matched your search

+11 -37
View File
@@ -344,7 +344,7 @@ func (ns *noiseServer) C2NResponseHandler(
nv, ok := ns.headscale.state.GetNodeByID(nodeID)
if !ok {
httpError(writer, NewHTTPError(http.StatusNotFound, "node not found", errNodeNotInStore))
httpError(writer, NewHTTPError(http.StatusNotFound, "node not found", nil))
return
}
@@ -818,35 +818,12 @@ func (ns *noiseServer) PollNetMapHandler(
nv, err := ns.getAndValidateNode(mapRequest)
if err != nil {
// The node is gone, but the client does not know that. Tailscale
// clients treat every non-200 on the map path the same way and retry
// forever with loggedIn still set; only a self node whose KeyExpiry is
// in the past drives them to NeedsLogin. There is no MapResponse field
// that says "deleted", so reuse the expiry signal headscale already
// sends for expired nodes.
// See: https://github.com/juanfont/headscale/issues/3410
if errors.Is(err, errNodeNotInStore) && mapRequest.Stream {
expired := &tailcfg.MapResponse{
Node: &tailcfg.Node{
Key: mapRequest.NodeKey,
// Zero means that the key does not expire. Use a fixed,
// ancient non-zero value so this remains expired even when
// the client's clock is substantially behind the server.
KeyExpiry: time.Unix(1, 0).UTC(),
Expired: true,
},
// This frame opens the stream in place of the initial map.
Debug: ns.headscale.cfg.TailcfgDebug(),
}
err = writeMapResponse(writer, mapRequest.Compress, true, expired)
if err != nil {
log.Error().Caller().Err(err).Msg("noise map handler: failed to write expired response for deleted node")
}
return
}
// A deleted node gets a bare 404, as Tailscale's control server sends;
// clients since 1.106 read it as removal and go to NeedsLogin. Do not
// answer with a netmap: clients reset their map backoff on every netmap,
// so one followed by the end of the stream is retried within
// milliseconds, forever.
// See: https://github.com/juanfont/headscale/issues/3688
httpError(writer, err)
return
@@ -932,7 +909,7 @@ func (ns *noiseServer) SetDNSHandler(writer http.ResponseWriter, req *http.Reque
nv, ok := ns.headscale.state.GetNodeByNodeKey(setDNS.NodeKey)
if !ok {
httpError(writer, NewHTTPError(http.StatusNotFound, "node not found", errNodeNotInStore))
httpError(writer, NewHTTPError(http.StatusNotFound, "node not found", nil))
return
}
@@ -1117,17 +1094,14 @@ func (ns *noiseServer) RegistrationHandler(
}
}
// errNodeNotInStore distinguishes an unknown NodeKey from a NodeKey presented
// by the wrong machine key. Both are answered with 404, but only the former
// means the node is gone and its client should re-authenticate.
var errNodeNotInStore = errors.New("node not found")
// getAndValidateNode retrieves the node from the in-memory NodeStore using the
// NodeKey and validates that it matches the MachineKey from the Noise session.
// Its 404s end a client's session: clients read a map 404 as removal from the
// tailnet, so a node that still exists must never be answered with one.
func (ns *noiseServer) getAndValidateNode(mapRequest tailcfg.MapRequest) (types.NodeView, error) {
nv, ok := ns.headscale.state.GetNodeByNodeKey(mapRequest.NodeKey)
if !ok {
return types.NodeView{}, NewHTTPError(http.StatusNotFound, "node not found", errNodeNotInStore)
return types.NodeView{}, NewHTTPError(http.StatusNotFound, "node not found", nil)
}
// Validate that the MachineKey in the Noise session matches the one associated with the NodeKey.
+14 -63
View File
@@ -5,7 +5,6 @@ import (
"context"
"crypto/sha256"
"encoding/base64"
"encoding/binary"
"encoding/json"
"errors"
"fmt"
@@ -29,7 +28,6 @@ import (
"github.com/juanfont/headscale/hscontrol/dns"
"github.com/juanfont/headscale/hscontrol/dns/dnstest"
"github.com/juanfont/headscale/hscontrol/types"
"github.com/juanfont/headscale/hscontrol/util"
"github.com/rs/zerolog"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -39,7 +37,6 @@ import (
"tailscale.com/net/tsdial"
"tailscale.com/tailcfg"
"tailscale.com/types/key"
"tailscale.com/util/zstdframe"
)
var privateDiagnosticPaths = []string{
@@ -1541,57 +1538,22 @@ func newMapRequest(t *testing.T, req tailcfg.MapRequest) *http.Request {
return httptest.NewRequestWithContext(context.Background(), http.MethodPost, "/machine/map", bytes.NewReader(body))
}
// decodeMapResponse reads a map response frame the way a Tailscale client
// does: a little-endian length prefix followed by a body that is zstd-framed
// whenever the request asked for compression.
func decodeMapResponse(t *testing.T, compress string, body []byte) tailcfg.MapResponse {
t.Helper()
require.GreaterOrEqual(t, len(body), reservedResponseHeaderSize, "response too short to carry a length prefix")
size := binary.LittleEndian.Uint32(body[:reservedResponseHeaderSize])
payload := body[reservedResponseHeaderSize:]
require.Len(t, payload, int(size), "length prefix must match the body it precedes")
if compress == util.ZstdCompression {
decoded, err := zstdframe.AppendDecode(nil, payload)
require.NoError(t, err, "client decodes every frame as zstd when it asked for zstd")
payload = decoded
}
var resp tailcfg.MapResponse
require.NoError(t, json.Unmarshal(payload, &resp))
return resp
}
// TestPollNetMapHandler_DeletedNodeGetsExpiredSelf verifies that a streaming
// map request for a node that no longer exists is answered with an expired
// self node instead of a bare 404. A Tailscale client treats every non-200 on
// the map path identically and retries forever with loggedIn still set; only a
// self node whose KeyExpiry is in the past moves it to NeedsLogin.
// TestPollNetMapHandler_DeletedNodeGetsNotFound pins parity with Tailscale's
// control server: a deleted node's map request gets a bare 404. Clients read a
// map 404 as "removed from the tailnet" ([controlclient.ErrNodeRemoved]) and
// retry with a growing backoff. A 200 carrying a netmap instead resets that
// backoff, so when the stream then ends the client re-polls within
// milliseconds, forever.
//
// See: https://github.com/juanfont/headscale/issues/3410
func TestPollNetMapHandler_DeletedNodeGetsExpiredSelf(t *testing.T) {
// See: https://github.com/juanfont/headscale/issues/3688
func TestPollNetMapHandler_DeletedNodeGetsNotFound(t *testing.T) {
t.Parallel()
for _, tc := range []struct {
compress string
logTail bool
wantDebug *tailcfg.Debug
}{
{"", false, &tailcfg.Debug{DisableLogTail: true}},
{util.ZstdCompression, false, &tailcfg.Debug{DisableLogTail: true}},
{"", true, nil},
{util.ZstdCompression, true, nil},
} {
compress := tc.compress
t.Run(fmt.Sprintf("compress=%s/logtail=%t", compress, tc.logTail), func(t *testing.T) {
for _, stream := range []bool{true, false} {
t.Run(fmt.Sprintf("stream=%t", stream), func(t *testing.T) {
t.Parallel()
app := createTestApp(t)
app.cfg.LogTail.Enabled = tc.logTail
user := app.state.CreateUserForTest("deleted-node-user")
node := putTestNodeInStore(t, app, user, "deleted-node")
@@ -1605,23 +1567,12 @@ func TestPollNetMapHandler_DeletedNodeGetsExpiredSelf(t *testing.T) {
rec := httptest.NewRecorder()
ns.PollNetMapHandler(rec, newMapRequest(t, tailcfg.MapRequest{
Version: tailcfg.CurrentCapabilityVersion,
NodeKey: node.NodeKey,
Stream: true,
Compress: compress,
Version: tailcfg.CurrentCapabilityVersion,
NodeKey: node.NodeKey,
Stream: stream,
}))
require.Equal(t, http.StatusOK, rec.Code, "body=%q", rec.Body.String())
resp := decodeMapResponse(t, compress, rec.Body.Bytes())
require.NotNil(t, resp.Node, "clients reject an initial map response without a node")
assert.Equal(t, node.NodeKey, resp.Node.Key)
assert.Equal(t, time.Unix(1, 0).UTC(), resp.Node.KeyExpiry,
"a fixed ancient KeyExpiry must remain expired despite client clock skew")
assert.True(t, resp.Node.Expired)
// This frame starts the stream, so it must carry the logtail
// instruction the initial map would have.
assert.Equal(t, tc.wantDebug, resp.Debug)
assert.Equal(t, http.StatusNotFound, rec.Code, "body=%q", rec.Body.String())
})
}
}
-3
View File
@@ -422,9 +422,6 @@ func (m *mapSession) writeMap(msg *tailcfg.MapResponse) error {
// zstd-framed when the client asked for compression, behind a little-endian
// length prefix. Tailscale clients request zstd unconditionally and decode
// every frame with it, so the compression step is not optional.
//
// It is shared with the deleted-node path in [noiseServer.PollNetMapHandler],
// which has no [mapSession] to write through.
func writeMapResponse(w http.ResponseWriter, compress string, flush bool, msg *tailcfg.MapResponse) error {
jsonBody, err := json.Marshal(msg)
if err != nil {
+2 -2
View File
@@ -496,8 +496,8 @@ func (c *TestClient) UpdateFullNetmap(nm *netmap.NetworkMap) {
c.history = append(c.history, nm)
}
// Removals sent ahead of their batch and a deleted node's expiry carry
// no ControlTime, so a rebuild with none before it is a response too.
// Removals sent ahead of their batch carry no ControlTime, so a rebuild
// with none before it is a response too.
// shortcut: one right after a response a delta client applied in place
// goes uncounted; count at the decoder if headscale sends one alone.
if !c.stamped {
+2 -26
View File
@@ -552,8 +552,8 @@ func TestPathsDiscoAndExpiryHelpers(t *testing.T) {
// TestFramesCountResponsesWithoutControlTime checks that Frames counts the
// responses headscale sends without a ControlTime: a removal ahead of its
// batch, and the expiry a deleted node's new stream opens with. A client
// that rebuilds its netmap on every response counts one frame per rebuild.
// batch. A client that rebuilds its netmap on every response counts one frame
// per rebuild.
func TestFramesCountResponsesWithoutControlTime(t *testing.T) {
t.Parallel()
@@ -591,30 +591,6 @@ func TestFramesCountResponsesWithoutControlTime(t *testing.T) {
viewer.WaitForPeerCount(t, 0, 15*time.Second)
requireFramePerRebuild(t, viewer)
})
t.Run("deleted-node-expiry", func(t *testing.T) {
t.Parallel()
srv := servertest.NewServer(t)
cl := servertest.NewClient(t, srv, "frames-deleted")
cl.WaitForCondition(t, "initial netmap", 15*time.Second,
func(nm *netmap.NetworkMap) bool { return nm != nil })
node, ok := srv.State().GetNodeByID(findNodeID(t, srv, "frames-deleted"))
require.True(t, ok)
changes, err := srv.State().DeleteNode(node)
require.NoError(t, err)
srv.App.Change(changes...)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
require.NoError(t, cl.RestartPoll(ctx))
cl.WaitForCondition(t, "expired self", 15*time.Second,
func(nm *netmap.NetworkMap) bool { return nm != nil && nm.SelfNode.Expired() })
requireFramePerRebuild(t, cl)
})
}
func TestNodeExpiryRouteFailover(t *testing.T) {