From 8230c58a0c500f800ff6a49002114359ee77dab0 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Wed, 4 Nov 2026 13:43:47 +0000 Subject: [PATCH] api/v2: answer update-client as Tailscale does No body, 200 {} once started, 404 "not found" for a device that disallows updates, has no map stream, or refuses. c2n wait 10s. Updates #3629 --- hscontrol/api/v2/README.md | 9 +- hscontrol/api/v2/api.go | 4 + hscontrol/api/v2/c2n.go | 12 +- hscontrol/api/v2/devices.go | 65 +++------ hscontrol/apiv2_oauth_matrix_test.go | 2 +- hscontrol/app.go | 20 ++- .../apiv2_device_update_client_test.go | 127 +++++++++--------- 7 files changed, 113 insertions(+), 126 deletions(-) diff --git a/hscontrol/api/v2/README.md b/hscontrol/api/v2/README.md index d6730c3b2..4b558e507 100644 --- a/hscontrol/api/v2/README.md +++ b/hscontrol/api/v2/README.md @@ -18,11 +18,10 @@ headscale's own conventions. The headscale-native admin API stays at `/api/v1` ## Conventions - Operations derived from Tailscale carry the `Tailscale compat` tag. -- Operations on undocumented Tailscale paths do not carry it, since their wire - format cannot be checked. `POST /device/{id}/update-client` - (`updateDeviceClient`) is one: the Tailscale admin console calls it to start - a client update, which reaches the node as c2n `POST /update`. Its request - and response shapes are headscale's own. +- `POST /device/{id}/update-client` is undocumented by Tailscale. Its shape + comes from SaaS captures (`servertest/testdata/steps/update-client*.hujson`): + no body, `200 {}` once the node started the update, `404 "not found"` for + every device it will not update. It reaches the node as c2n `POST /update`. - The `{tailnet}` path segment must be `-` (the single Headscale tailnet); anything else is `404`. See `requireDefaultTailnet`. - Errors use **Tailscale's** body (`{"message","data","status"}`), installed as diff --git a/hscontrol/api/v2/api.go b/hscontrol/api/v2/api.go index ecc3ea4f0..615b50afd 100644 --- a/hscontrol/api/v2/api.go +++ b/hscontrol/api/v2/api.go @@ -38,6 +38,10 @@ type Backend struct { State *state.State Change func(...change.Change) Cfg *types.Config + // Connected reports whether a node has a live map stream. A node stays + // online through the disconnect grace period, but only a stream can + // carry a c2n request. + Connected func(types.NodeID) bool } // security is the requirement applied to authenticated operations: an API key diff --git a/hscontrol/api/v2/c2n.go b/hscontrol/api/v2/c2n.go index 3ad852961..425ed4745 100644 --- a/hscontrol/api/v2/c2n.go +++ b/hscontrol/api/v2/c2n.go @@ -11,12 +11,15 @@ import ( "github.com/danielgtaylor/huma/v2" "github.com/juanfont/headscale/hscontrol/types" "github.com/juanfont/headscale/hscontrol/types/change" + "github.com/juanfont/headscale/hscontrol/util/zlog/zf" + "github.com/rs/zerolog/log" "tailscale.com/tailcfg" ) // c2nTimeout bounds how long a c2n request waits for the node to answer. // For an update, the node replies once the update has started, not finished. -const c2nTimeout = 30 * time.Second +// It stays well under the server's write timeout, so the caller gets the 504. +const c2nTimeout = 10 * time.Second // sendC2N sends a POST to path on the node's c2n handler, carried in a // [tailcfg.PingRequest], and waits for the node's HTTP response. @@ -71,13 +74,16 @@ func sendC2N(ctx context.Context, b Backend, nodeID types.NodeID, path string) ( if res.StatusCode != http.StatusOK { res.Body.Close() - return nil, huma.Error502BadGateway("device answered c2n request with " + res.Status) + // The status text is the node's; it is logged, not echoed. + log.Info().Uint64(zf.NodeID, nodeID.Uint64()).Str("status", res.Status).Msg("node answered c2n request with an error") + + return nil, huma.Error502BadGateway("device answered c2n request with an error") } return res, nil case <-timer.C: return nil, huma.Error504GatewayTimeout("device did not answer within " + c2nTimeout.String()) case <-ctx.Done(): - return nil, ctx.Err() + return nil, huma.Error504GatewayTimeout("request ended before the device answered", ctx.Err()) } } diff --git a/hscontrol/api/v2/devices.go b/hscontrol/api/v2/devices.go index d1918a5e7..61c96d1f0 100644 --- a/hscontrol/api/v2/devices.go +++ b/hscontrol/api/v2/devices.go @@ -14,6 +14,7 @@ import ( "github.com/juanfont/headscale/hscontrol/scope" "github.com/juanfont/headscale/hscontrol/types" "github.com/juanfont/headscale/hscontrol/util" + "github.com/rs/zerolog/log" "tailscale.com/net/tsaddr" "tailscale.com/tailcfg" ) @@ -107,16 +108,6 @@ type DevicePostureAttributes struct { Expiries map[string]time.Time `json:"expiries,omitempty"` } -// DeviceClientUpdate is the POST /device/{id}/update-client response. It mirrors -// [tailcfg.C2NUpdateResponse], the device's own answer: Error carries its -// reason when Started is false. -type DeviceClientUpdate struct { - Enabled bool `json:"enabled"` - Supported bool `json:"supported"` - Started bool `json:"started"` - Error string `json:"error"` -} - // Request bodies match the Tailscale SDK wire shapes and, like Tailscale's, // ignore properties they do not define. type ( @@ -148,11 +139,6 @@ type ( Routes []string `json:"routes" nullable:"false"` } - // updateDeviceClientRequest is headscale's own shape: Tailscale's - // update-client endpoint is undocumented, so its wire format is unknown. - updateDeviceClientRequest struct { - Force bool `json:"force,omitempty"` - } ) type ( @@ -187,9 +173,8 @@ type ( DeviceID string `path:"id"` Body setSubnetRoutesRequest } - updateDeviceClientInput struct { - DeviceID string `path:"id"` - Body *updateDeviceClientRequest `required:"false"` + deviceIDInput struct { + DeviceID string `path:"id"` } deviceOutput struct{ Body Device } @@ -201,8 +186,6 @@ type ( } } emptyOutput struct{ Body struct{} } - - deviceClientUpdateOutput struct{ Body DeviceClientUpdate } ) func registerDevices(api huma.API, b Backend) { @@ -519,28 +502,27 @@ func registerDevices(api huma.API, b Backend) { Method: http.MethodPost, Path: "/api/v2/device/{id}/update-client", Summary: "Update a device's Tailscale client", - Description: "Asks a connected device over c2n to update its Tailscale client. The device only starts the update if it has auto-updates enabled and its platform supports them.", - Tags: []string{"Devices"}, + Description: "Asks a connected device over c2n to update its Tailscale client. A device that does not allow remote updates, is not connected, or refuses is not found.", + Tags: deviceTags, Security: security, DefaultStatus: http.StatusOK, - MaxBodyBytes: maxBodyBytes, Errors: []int{ http.StatusUnauthorized, http.StatusForbidden, http.StatusNotFound, - http.StatusConflict, http.StatusBadGateway, http.StatusGatewayTimeout, + http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout, }, - }, scope.DevicesCore), func(ctx context.Context, in *updateDeviceClientInput) (*deviceClientUpdateOutput, error) { + }, scope.DevicesCore), func(ctx context.Context, in *deviceIDInput) (*emptyOutput, error) { node, err := lookupNode(b, in.DeviceID) if err != nil { return nil, err } - if !node.IsOnline().Valid() || !node.IsOnline().Get() { - return nil, huma.Error409Conflict("device is not connected") + // Tailscale answers every device it will not update with the same 404. + hi := node.Hostinfo() + if !hi.Valid() || !hi.AllowsUpdate() || !b.Connected(node.ID()) { + return nil, huma.Error404NotFound("not found") } - force := in.Body != nil && in.Body.Force - - res, err := sendC2N(ctx, b, node.ID(), c2nUpdatePath(force)) + res, err := sendC2N(ctx, b, node.ID(), "/update") if err != nil { return nil, err } @@ -553,25 +535,16 @@ func registerDevices(api huma.API, b Backend) { return nil, huma.Error502BadGateway("decoding client update response", err) } - return &deviceClientUpdateOutput{Body: DeviceClientUpdate{ - Enabled: update.Enabled, - Supported: update.Supported, - Started: update.Started, - Error: update.Err, - }}, nil + if !update.Started { + log.Info().EmbedObject(node).Str("reason", update.Err).Msg("node refused client update") + + return nil, huma.Error404NotFound("not found") + } + + return &emptyOutput{}, nil }) } -// c2nUpdatePath is the c2n endpoint that starts a client update. force -// overrides the client's refusal to update with active SSH sessions. -func c2nUpdatePath(force bool) string { - if force { - return "/update?force=true" - } - - return "/update" -} - // lookupNode resolves a device id to its NodeView, mapping a malformed or // unknown id to 404 (the Tailscale SDK keys IsNotFound off the status code). func lookupNode(b Backend, rawID string) (types.NodeView, error) { diff --git a/hscontrol/apiv2_oauth_matrix_test.go b/hscontrol/apiv2_oauth_matrix_test.go index c4324f5ac..1ef5bda3b 100644 --- a/hscontrol/apiv2_oauth_matrix_test.go +++ b/hscontrol/apiv2_oauth_matrix_test.go @@ -122,7 +122,7 @@ func TestAPIv2OAuthMatrix_Enforcement(t *testing.T) { {"setDeviceName", http.MethodPost, "/api/v2/device/1/name", scope.DevicesCore, map[string]any{}, false}, {"setDeviceTags", http.MethodPost, "/api/v2/device/1/tags", scope.DevicesCore, map[string]any{}, false}, {"setDeviceKey", http.MethodPost, "/api/v2/device/1/key", scope.DevicesCore, map[string]any{}, false}, - {"updateDeviceClient", http.MethodPost, "/api/v2/device/1/update-client", scope.DevicesCore, map[string]any{}, false}, + {"updateDeviceClient", http.MethodPost, "/api/v2/device/1/update-client", scope.DevicesCore, nil, false}, {"setDeviceRoutes", http.MethodPost, "/api/v2/device/1/routes", scope.DevicesRoutes, map[string]any{}, false}, {"getDeviceRoutes", http.MethodGet, "/api/v2/device/1/routes", scope.DevicesRoutesRead, nil, false}, {"getACL", http.MethodGet, "/api/v2/tailnet/-/acl", scope.PolicyFileRead, nil, false}, diff --git a/hscontrol/app.go b/hscontrol/app.go index 1f0d11bf7..c1b1bf092 100644 --- a/hscontrol/app.go +++ b/hscontrol/app.go @@ -717,9 +717,10 @@ func (h *Headscale) Serve() error { // manage OAuth clients through the same v2 keys handler the Tailscale // ecosystem uses. humaV2Mux, _ := apiv2.Handler(apiv2.Backend{ - State: h.state, - Change: h.Change, - Cfg: h.cfg, + State: h.state, + Change: h.Change, + Cfg: h.cfg, + Connected: h.isConnected, }) // Serve both Huma APIs over the unix socket without TLS or auth: socket @@ -1380,9 +1381,10 @@ func (h *Headscale) HTTPHandler() http.Handler { }) humaV2Mux, _ := apiv2.Handler(apiv2.Backend{ - State: h.state, - Change: h.Change, - Cfg: h.cfg, + State: h.state, + Change: h.Change, + Cfg: h.cfg, + Connected: h.isConnected, }) return h.createRouter(humaMux, humaV2Mux) @@ -1428,6 +1430,12 @@ func (h *Headscale) MapBatcher() *mapper.Batcher { return h.mapBatcher } +// isConnected reads mapBatcher on each call: tests build the HTTP handler +// before they start the batcher. +func (h *Headscale) isConnected(id types.NodeID) bool { + return h.mapBatcher.IsConnected(id) +} + // StartEphemeralGCForTest starts the ephemeral node garbage collector. // It registers a cleanup function on tb to stop the collector. // It panics when called outside of tests. diff --git a/hscontrol/servertest/apiv2_device_update_client_test.go b/hscontrol/servertest/apiv2_device_update_client_test.go index 0b15e141a..dcca4c213 100644 --- a/hscontrol/servertest/apiv2_device_update_client_test.go +++ b/hscontrol/servertest/apiv2_device_update_client_test.go @@ -23,13 +23,12 @@ import ( ) // fakeUpdater answers c2n "POST /update" the way tailscaled does, with a -// fixed [tailcfg.C2NUpdateResponse], and records the force query value -// of every request. +// fixed [tailcfg.C2NUpdateResponse], and counts the requests. type fakeUpdater struct { res tailcfg.C2NUpdateResponse - mu sync.Mutex - forces []string + mu sync.Mutex + calls int } func (f *fakeUpdater) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -39,7 +38,7 @@ func (f *fakeUpdater) ServeHTTP(w http.ResponseWriter, r *http.Request) { } f.mu.Lock() - f.forces = append(f.forces, r.FormValue("force")) + f.calls++ f.mu.Unlock() w.Header().Set("Content-Type", "application/json") @@ -50,38 +49,37 @@ func (f *fakeUpdater) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } -func (f *fakeUpdater) seenForces() []string { +func (f *fakeUpdater) seenCalls() int { f.mu.Lock() defer f.mu.Unlock() - return append([]string(nil), f.forces...) + return f.calls } -type deviceClientUpdate struct { - Enabled bool `json:"enabled"` - Supported bool `json:"supported"` - Started bool `json:"started"` - Error string `json:"error"` -} - -// updateDeviceClient calls POST /api/v2/device/{id}/update-client and -// returns the status code and decoded body. -func updateDeviceClient(t *testing.T, srv *servertest.TestServer, nodeID types.NodeID, force bool) (int, deviceClientUpdate) { +// updatableHarness runs one client that allows remote updates, as +// "tailscale set --auto-update" makes it, and answers c2n with updater. +func updatableHarness(t *testing.T, updater *fakeUpdater) *servertest.TestHarness { t.Helper() - body, err := json.Marshal(map[string]bool{"force": force}) - require.NoError(t, err) + hi := &tailcfg.Hostinfo{BackendLogID: "servertest-updatable", Hostname: "updatable", AllowsUpdate: true} + + return servertest.NewHarness(t, 1, servertest.WithDefaultClientOptions( + servertest.WithHostinfo(hi), servertest.WithC2NHandler(updater))) +} + +// updateDeviceClient calls POST /api/v2/device/{id}/update-client, with no +// body as Tailscale's console sends it, and returns the status and body. +func updateDeviceClient(t *testing.T, srv *servertest.TestServer, nodeID types.NodeID) (int, string) { + t.Helper() ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodPost, - srv.URL+"/api/v2/device/"+strconv.FormatUint(nodeID.Uint64(), 10)+"/update-client", - bytes.NewReader(body)) + srv.URL+"/api/v2/device/"+strconv.FormatUint(nodeID.Uint64(), 10)+"/update-client", nil) require.NoError(t, err) req.Header.Set("Authorization", "Bearer "+srv.CreateAPIKey(t, nil)) - req.Header.Set("Content-Type", "application/json") client := &http.Client{Transport: &http.Transport{DialContext: srv.MemNet().Dial}} @@ -90,18 +88,16 @@ func updateDeviceClient(t *testing.T, srv *servertest.TestServer, nodeID types.N defer resp.Body.Close() - var out deviceClientUpdate - if resp.StatusCode == http.StatusOK { - require.NoError(t, json.NewDecoder(resp.Body).Decode(&out)) - } + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) - return resp.StatusCode, out + return resp.StatusCode, string(body) } // TestAPIv2DeviceUpdateClient verifies the full c2n round-trip: the API sends a // c2n [tailcfg.PingRequest] carrying "POST /update", the real // [controlclient.Direct] runs it against its c2n handler and posts the -// response back over Noise, and the API returns the node's answer. +// response back over Noise, and the API answers 200 {} once it started. func TestAPIv2DeviceUpdateClient(t *testing.T) { t.Parallel() @@ -111,57 +107,57 @@ func TestAPIv2DeviceUpdateClient(t *testing.T) { Started: true, }} - h := servertest.NewHarness(t, 1, - servertest.WithDefaultClientOptions(servertest.WithC2NHandler(updater))) + h := updatableHarness(t, updater) + nodeID := types.NodeID(h.Client(0).Netmap().SelfNode.ID()) //nolint:gosec - nm := h.Client(0).Netmap() - require.NotNil(t, nm) - - nodeID := types.NodeID(nm.SelfNode.ID()) //nolint:gosec - - code, res := updateDeviceClient(t, h.Server, nodeID, false) - require.Equal(t, http.StatusOK, code) - assert.True(t, res.Enabled) - assert.True(t, res.Supported) - assert.True(t, res.Started) - assert.Empty(t, res.Error) - - code, _ = updateDeviceClient(t, h.Server, nodeID, true) - require.Equal(t, http.StatusOK, code) - - assert.Equal(t, []string{"", "true"}, updater.seenForces(), - "force must reach the client as ?force=true only when requested") + code, body := updateDeviceClient(t, h.Server, nodeID) + require.Equal(t, http.StatusOK, code, body) + assert.JSONEq(t, `{}`, body) + assert.Equal(t, 1, updater.seenCalls()) } // TestAPIv2DeviceUpdateClientRefused verifies that a client refusing the update -// (auto-updates off, unsupported platform, active SSH) is reported back -// as a successful call with started=false and the client's reason. +// (platform cannot update, active SSH) is answered 404, as SaaS answers every +// device it will not update. func TestAPIv2DeviceUpdateClientRefused(t *testing.T) { t.Parallel() updater := &fakeUpdater{res: tailcfg.C2NUpdateResponse{ - Supported: true, - Err: "not enabled", + Enabled: true, + Err: "not supported", }} + h := updatableHarness(t, updater) + nodeID := types.NodeID(h.Client(0).Netmap().SelfNode.ID()) //nolint:gosec + + code, body := updateDeviceClient(t, h.Server, nodeID) + assert.Equal(t, http.StatusNotFound, code) + assert.Equal(t, "not found", errorMessage([]byte(body))) + assert.Equal(t, 1, updater.seenCalls(), "the refusal should come from the node") +} + +// TestAPIv2DeviceUpdateClientNotAllowed verifies that a node that does not +// allow remote updates is answered 404 without being asked. +func TestAPIv2DeviceUpdateClientNotAllowed(t *testing.T) { + t.Parallel() + + updater := &fakeUpdater{res: tailcfg.C2NUpdateResponse{Enabled: true, Supported: true, Started: true}} h := servertest.NewHarness(t, 1, servertest.WithDefaultClientOptions(servertest.WithC2NHandler(updater))) nodeID := types.NodeID(h.Client(0).Netmap().SelfNode.ID()) //nolint:gosec - code, res := updateDeviceClient(t, h.Server, nodeID, false) - require.Equal(t, http.StatusOK, code) - assert.False(t, res.Enabled) - assert.False(t, res.Started) - assert.Equal(t, "not enabled", res.Error) + code, _ := updateDeviceClient(t, h.Server, nodeID) + assert.Equal(t, http.StatusNotFound, code) + assert.Zero(t, updater.seenCalls()) } // TestAPIv2DeviceUpdateClientNotConnected verifies that a disconnected device is -// rejected with 409 instead of waiting for an answer that cannot come. +// answered 404 instead of waiting for an answer that cannot come. func TestAPIv2DeviceUpdateClientNotConnected(t *testing.T) { t.Parallel() - h := servertest.NewHarness(t, 1) + h := updatableHarness(t, &fakeUpdater{}) nodeID := types.NodeID(h.Client(0).Netmap().SelfNode.ID()) //nolint:gosec @@ -176,8 +172,8 @@ func TestAPIv2DeviceUpdateClientNotConnected(t *testing.T) { assert.True(c, known && !isOnline, "node should be offline") }, 20*time.Second, 200*time.Millisecond) - code, _ := updateDeviceClient(t, h.Server, nodeID, false) - assert.Equal(t, http.StatusConflict, code) + code, _ := updateDeviceClient(t, h.Server, nodeID) + assert.Equal(t, http.StatusNotFound, code) } // TestAPIv2DeviceUpdateClientDuringDisconnectGrace verifies that a node whose @@ -186,7 +182,7 @@ func TestAPIv2DeviceUpdateClientNotConnected(t *testing.T) { func TestAPIv2DeviceUpdateClientDuringDisconnectGrace(t *testing.T) { t.Parallel() - h := servertest.NewHarness(t, 1) + h := updatableHarness(t, &fakeUpdater{}) nodeID := types.NodeID(h.Client(0).Netmap().SelfNode.ID()) //nolint:gosec @@ -202,7 +198,7 @@ func TestAPIv2DeviceUpdateClientDuringDisconnectGrace(t *testing.T) { // SaaS answers a device it cannot reach 404 (testdata/steps/update-client.hujson). start := time.Now() - code, _ := updateDeviceClient(t, h.Server, nodeID, false) + code, _ := updateDeviceClient(t, h.Server, nodeID) assert.Equal(t, http.StatusNotFound, code) assert.Less(t, time.Since(start), 5*time.Second, "refusal should not wait for the c2n timeout") } @@ -213,7 +209,7 @@ func TestAPIv2DeviceUpdateClientUnknownDevice(t *testing.T) { h := servertest.NewHarness(t, 0) - code, _ := updateDeviceClient(t, h.Server, types.NodeID(9999), false) + code, _ := updateDeviceClient(t, h.Server, types.NodeID(9999)) assert.Equal(t, http.StatusNotFound, code) } @@ -226,6 +222,7 @@ func TestAPIv2DeviceUpdateClientUnknownDevice(t *testing.T) { // Test data source: testdata/steps/update-client.hujson. Rows by // devices:core:read and dns:read tokens are not replayed: SaaS answers them // 404 before any scope check, which shows its order, not this endpoint's shape. +// Nor are GETs: headscale answers a wrong method 405 on every route. func TestAPIv2DeviceUpdateClientCompat(t *testing.T) { t.Parallel() @@ -274,7 +271,7 @@ func TestAPIv2DeviceUpdateClientCompat(t *testing.T) { } for _, a := range step.Actions { - if a.Kind != "api" || !strings.Contains(a.Path, "/update-client") || skipped[a.Principal] { + if a.Kind != "api" || !strings.Contains(a.Path, "/update-client") || a.Method != http.MethodPost || skipped[a.Principal] { continue } @@ -295,7 +292,7 @@ func TestAPIv2DeviceUpdateClientCompat(t *testing.T) { } } - require.Equal(t, 19, ran, "update-client rows replayed") + require.Equal(t, 17, ran, "update-client rows replayed") } // TestAPIv2DeviceUpdateClientConsoleCompat replays the admin console's Start