mirror of
https://github.com/juanfont/headscale.git
synced 2026-11-19 11:57:07 +09:00
noise: answer a deleted node's map request with 404
As Tailscale does. A netmap resets the client's map backoff, so ending the stream after one made it re-poll every few milliseconds. Fixes #3688
This commit is contained in:
1 parent
934a1f12b8
commit
ca512f2c57
5 files changed
+29
-131
No files matched your search
+11
-37
@@ -344,7 +344,7 @@ func (ns *noiseServer) C2NResponseHandler(
|
||||
|
||||
nv, ok := ns.headscale.state.GetNodeByID(nodeID)
|
||||
if !ok {
|
||||
httpError(writer, NewHTTPError(http.StatusNotFound, "node not found", errNodeNotInStore))
|
||||
httpError(writer, NewHTTPError(http.StatusNotFound, "node not found", nil))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -818,35 +818,12 @@ func (ns *noiseServer) PollNetMapHandler(
|
||||
|
||||
nv, err := ns.getAndValidateNode(mapRequest)
|
||||
if err != nil {
|
||||
// The node is gone, but the client does not know that. Tailscale
|
||||
// clients treat every non-200 on the map path the same way and retry
|
||||
// forever with loggedIn still set; only a self node whose KeyExpiry is
|
||||
// in the past drives them to NeedsLogin. There is no MapResponse field
|
||||
// that says "deleted", so reuse the expiry signal headscale already
|
||||
// sends for expired nodes.
|
||||
// See: https://github.com/juanfont/headscale/issues/3410
|
||||
if errors.Is(err, errNodeNotInStore) && mapRequest.Stream {
|
||||
expired := &tailcfg.MapResponse{
|
||||
Node: &tailcfg.Node{
|
||||
Key: mapRequest.NodeKey,
|
||||
// Zero means that the key does not expire. Use a fixed,
|
||||
// ancient non-zero value so this remains expired even when
|
||||
// the client's clock is substantially behind the server.
|
||||
KeyExpiry: time.Unix(1, 0).UTC(),
|
||||
Expired: true,
|
||||
},
|
||||
// This frame opens the stream in place of the initial map.
|
||||
Debug: ns.headscale.cfg.TailcfgDebug(),
|
||||
}
|
||||
|
||||
err = writeMapResponse(writer, mapRequest.Compress, true, expired)
|
||||
if err != nil {
|
||||
log.Error().Caller().Err(err).Msg("noise map handler: failed to write expired response for deleted node")
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// A deleted node gets a bare 404, as Tailscale's control server sends;
|
||||
// clients since 1.106 read it as removal and go to NeedsLogin. Do not
|
||||
// answer with a netmap: clients reset their map backoff on every netmap,
|
||||
// so one followed by the end of the stream is retried within
|
||||
// milliseconds, forever.
|
||||
// See: https://github.com/juanfont/headscale/issues/3688
|
||||
httpError(writer, err)
|
||||
|
||||
return
|
||||
@@ -932,7 +909,7 @@ func (ns *noiseServer) SetDNSHandler(writer http.ResponseWriter, req *http.Reque
|
||||
|
||||
nv, ok := ns.headscale.state.GetNodeByNodeKey(setDNS.NodeKey)
|
||||
if !ok {
|
||||
httpError(writer, NewHTTPError(http.StatusNotFound, "node not found", errNodeNotInStore))
|
||||
httpError(writer, NewHTTPError(http.StatusNotFound, "node not found", nil))
|
||||
|
||||
return
|
||||
}
|
||||
@@ -1117,17 +1094,14 @@ func (ns *noiseServer) RegistrationHandler(
|
||||
}
|
||||
}
|
||||
|
||||
// errNodeNotInStore distinguishes an unknown NodeKey from a NodeKey presented
|
||||
// by the wrong machine key. Both are answered with 404, but only the former
|
||||
// means the node is gone and its client should re-authenticate.
|
||||
var errNodeNotInStore = errors.New("node not found")
|
||||
|
||||
// getAndValidateNode retrieves the node from the in-memory NodeStore using the
|
||||
// NodeKey and validates that it matches the MachineKey from the Noise session.
|
||||
// Its 404s end a client's session: clients read a map 404 as removal from the
|
||||
// tailnet, so a node that still exists must never be answered with one.
|
||||
func (ns *noiseServer) getAndValidateNode(mapRequest tailcfg.MapRequest) (types.NodeView, error) {
|
||||
nv, ok := ns.headscale.state.GetNodeByNodeKey(mapRequest.NodeKey)
|
||||
if !ok {
|
||||
return types.NodeView{}, NewHTTPError(http.StatusNotFound, "node not found", errNodeNotInStore)
|
||||
return types.NodeView{}, NewHTTPError(http.StatusNotFound, "node not found", nil)
|
||||
}
|
||||
|
||||
// Validate that the MachineKey in the Noise session matches the one associated with the NodeKey.
|
||||
|
||||
+14
-63
@@ -5,7 +5,6 @@ import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -29,7 +28,6 @@ import (
|
||||
"github.com/juanfont/headscale/hscontrol/dns"
|
||||
"github.com/juanfont/headscale/hscontrol/dns/dnstest"
|
||||
"github.com/juanfont/headscale/hscontrol/types"
|
||||
"github.com/juanfont/headscale/hscontrol/util"
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -39,7 +37,6 @@ import (
|
||||
"tailscale.com/net/tsdial"
|
||||
"tailscale.com/tailcfg"
|
||||
"tailscale.com/types/key"
|
||||
"tailscale.com/util/zstdframe"
|
||||
)
|
||||
|
||||
var privateDiagnosticPaths = []string{
|
||||
@@ -1541,57 +1538,22 @@ func newMapRequest(t *testing.T, req tailcfg.MapRequest) *http.Request {
|
||||
return httptest.NewRequestWithContext(context.Background(), http.MethodPost, "/machine/map", bytes.NewReader(body))
|
||||
}
|
||||
|
||||
// decodeMapResponse reads a map response frame the way a Tailscale client
|
||||
// does: a little-endian length prefix followed by a body that is zstd-framed
|
||||
// whenever the request asked for compression.
|
||||
func decodeMapResponse(t *testing.T, compress string, body []byte) tailcfg.MapResponse {
|
||||
t.Helper()
|
||||
|
||||
require.GreaterOrEqual(t, len(body), reservedResponseHeaderSize, "response too short to carry a length prefix")
|
||||
|
||||
size := binary.LittleEndian.Uint32(body[:reservedResponseHeaderSize])
|
||||
payload := body[reservedResponseHeaderSize:]
|
||||
require.Len(t, payload, int(size), "length prefix must match the body it precedes")
|
||||
|
||||
if compress == util.ZstdCompression {
|
||||
decoded, err := zstdframe.AppendDecode(nil, payload)
|
||||
require.NoError(t, err, "client decodes every frame as zstd when it asked for zstd")
|
||||
|
||||
payload = decoded
|
||||
}
|
||||
|
||||
var resp tailcfg.MapResponse
|
||||
require.NoError(t, json.Unmarshal(payload, &resp))
|
||||
|
||||
return resp
|
||||
}
|
||||
|
||||
// TestPollNetMapHandler_DeletedNodeGetsExpiredSelf verifies that a streaming
|
||||
// map request for a node that no longer exists is answered with an expired
|
||||
// self node instead of a bare 404. A Tailscale client treats every non-200 on
|
||||
// the map path identically and retries forever with loggedIn still set; only a
|
||||
// self node whose KeyExpiry is in the past moves it to NeedsLogin.
|
||||
// TestPollNetMapHandler_DeletedNodeGetsNotFound pins parity with Tailscale's
|
||||
// control server: a deleted node's map request gets a bare 404. Clients read a
|
||||
// map 404 as "removed from the tailnet" ([controlclient.ErrNodeRemoved]) and
|
||||
// retry with a growing backoff. A 200 carrying a netmap instead resets that
|
||||
// backoff, so when the stream then ends the client re-polls within
|
||||
// milliseconds, forever.
|
||||
//
|
||||
// See: https://github.com/juanfont/headscale/issues/3410
|
||||
func TestPollNetMapHandler_DeletedNodeGetsExpiredSelf(t *testing.T) {
|
||||
// See: https://github.com/juanfont/headscale/issues/3688
|
||||
func TestPollNetMapHandler_DeletedNodeGetsNotFound(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, tc := range []struct {
|
||||
compress string
|
||||
logTail bool
|
||||
wantDebug *tailcfg.Debug
|
||||
}{
|
||||
{"", false, &tailcfg.Debug{DisableLogTail: true}},
|
||||
{util.ZstdCompression, false, &tailcfg.Debug{DisableLogTail: true}},
|
||||
{"", true, nil},
|
||||
{util.ZstdCompression, true, nil},
|
||||
} {
|
||||
compress := tc.compress
|
||||
t.Run(fmt.Sprintf("compress=%s/logtail=%t", compress, tc.logTail), func(t *testing.T) {
|
||||
for _, stream := range []bool{true, false} {
|
||||
t.Run(fmt.Sprintf("stream=%t", stream), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
app := createTestApp(t)
|
||||
app.cfg.LogTail.Enabled = tc.logTail
|
||||
user := app.state.CreateUserForTest("deleted-node-user")
|
||||
node := putTestNodeInStore(t, app, user, "deleted-node")
|
||||
|
||||
@@ -1605,23 +1567,12 @@ func TestPollNetMapHandler_DeletedNodeGetsExpiredSelf(t *testing.T) {
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
ns.PollNetMapHandler(rec, newMapRequest(t, tailcfg.MapRequest{
|
||||
Version: tailcfg.CurrentCapabilityVersion,
|
||||
NodeKey: node.NodeKey,
|
||||
Stream: true,
|
||||
Compress: compress,
|
||||
Version: tailcfg.CurrentCapabilityVersion,
|
||||
NodeKey: node.NodeKey,
|
||||
Stream: stream,
|
||||
}))
|
||||
|
||||
require.Equal(t, http.StatusOK, rec.Code, "body=%q", rec.Body.String())
|
||||
|
||||
resp := decodeMapResponse(t, compress, rec.Body.Bytes())
|
||||
require.NotNil(t, resp.Node, "clients reject an initial map response without a node")
|
||||
assert.Equal(t, node.NodeKey, resp.Node.Key)
|
||||
assert.Equal(t, time.Unix(1, 0).UTC(), resp.Node.KeyExpiry,
|
||||
"a fixed ancient KeyExpiry must remain expired despite client clock skew")
|
||||
assert.True(t, resp.Node.Expired)
|
||||
// This frame starts the stream, so it must carry the logtail
|
||||
// instruction the initial map would have.
|
||||
assert.Equal(t, tc.wantDebug, resp.Debug)
|
||||
assert.Equal(t, http.StatusNotFound, rec.Code, "body=%q", rec.Body.String())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -422,9 +422,6 @@ func (m *mapSession) writeMap(msg *tailcfg.MapResponse) error {
|
||||
// zstd-framed when the client asked for compression, behind a little-endian
|
||||
// length prefix. Tailscale clients request zstd unconditionally and decode
|
||||
// every frame with it, so the compression step is not optional.
|
||||
//
|
||||
// It is shared with the deleted-node path in [noiseServer.PollNetMapHandler],
|
||||
// which has no [mapSession] to write through.
|
||||
func writeMapResponse(w http.ResponseWriter, compress string, flush bool, msg *tailcfg.MapResponse) error {
|
||||
jsonBody, err := json.Marshal(msg)
|
||||
if err != nil {
|
||||
|
||||
@@ -496,8 +496,8 @@ func (c *TestClient) UpdateFullNetmap(nm *netmap.NetworkMap) {
|
||||
c.history = append(c.history, nm)
|
||||
}
|
||||
|
||||
// Removals sent ahead of their batch and a deleted node's expiry carry
|
||||
// no ControlTime, so a rebuild with none before it is a response too.
|
||||
// Removals sent ahead of their batch carry no ControlTime, so a rebuild
|
||||
// with none before it is a response too.
|
||||
// shortcut: one right after a response a delta client applied in place
|
||||
// goes uncounted; count at the decoder if headscale sends one alone.
|
||||
if !c.stamped {
|
||||
|
||||
@@ -552,8 +552,8 @@ func TestPathsDiscoAndExpiryHelpers(t *testing.T) {
|
||||
|
||||
// TestFramesCountResponsesWithoutControlTime checks that Frames counts the
|
||||
// responses headscale sends without a ControlTime: a removal ahead of its
|
||||
// batch, and the expiry a deleted node's new stream opens with. A client
|
||||
// that rebuilds its netmap on every response counts one frame per rebuild.
|
||||
// batch. A client that rebuilds its netmap on every response counts one frame
|
||||
// per rebuild.
|
||||
func TestFramesCountResponsesWithoutControlTime(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -591,30 +591,6 @@ func TestFramesCountResponsesWithoutControlTime(t *testing.T) {
|
||||
viewer.WaitForPeerCount(t, 0, 15*time.Second)
|
||||
requireFramePerRebuild(t, viewer)
|
||||
})
|
||||
|
||||
t.Run("deleted-node-expiry", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
srv := servertest.NewServer(t)
|
||||
cl := servertest.NewClient(t, srv, "frames-deleted")
|
||||
cl.WaitForCondition(t, "initial netmap", 15*time.Second,
|
||||
func(nm *netmap.NetworkMap) bool { return nm != nil })
|
||||
|
||||
node, ok := srv.State().GetNodeByID(findNodeID(t, srv, "frames-deleted"))
|
||||
require.True(t, ok)
|
||||
|
||||
changes, err := srv.State().DeleteNode(node)
|
||||
require.NoError(t, err)
|
||||
srv.App.Change(changes...)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
require.NoError(t, cl.RestartPoll(ctx))
|
||||
cl.WaitForCondition(t, "expired self", 15*time.Second,
|
||||
func(nm *netmap.NetworkMap) bool { return nm != nil && nm.SelfNode.Expired() })
|
||||
requireFramePerRebuild(t, cl)
|
||||
})
|
||||
}
|
||||
|
||||
func TestNodeExpiryRouteFailover(t *testing.T) {
|
||||
|
||||
Reference in new issue
Block a user