From 7ea7208ec029ac03a1ae2d4e7d5d387f3ccb0489 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Tue, 3 Nov 2026 15:08:26 +0000 Subject: [PATCH] servertest: replay update-client captures against headscale Fails until update-client matches SaaS; grace test now expects SaaS's 404. Updates #3629 --- .../apiv2_device_update_client_test.go | 333 ++++++++++++++++++ 1 file changed, 333 insertions(+) diff --git a/hscontrol/servertest/apiv2_device_update_client_test.go b/hscontrol/servertest/apiv2_device_update_client_test.go index bcb80d90f..0b15e141a 100644 --- a/hscontrol/servertest/apiv2_device_update_client_test.go +++ b/hscontrol/servertest/apiv2_device_update_client_test.go @@ -4,17 +4,22 @@ import ( "bytes" "context" "encoding/json" + "io" "net/http" + "net/url" "strconv" + "strings" "sync" "testing" "time" "github.com/juanfont/headscale/hscontrol/servertest" "github.com/juanfont/headscale/hscontrol/types" + "github.com/juanfont/headscale/hscontrol/types/testcapture" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "tailscale.com/tailcfg" + "tailscale.com/types/netmap" ) // fakeUpdater answers c2n "POST /update" the way tailscaled does, with a @@ -175,6 +180,33 @@ func TestAPIv2DeviceUpdateClientNotConnected(t *testing.T) { assert.Equal(t, http.StatusConflict, code) } +// TestAPIv2DeviceUpdateClientDuringDisconnectGrace verifies that a node whose +// map stream is gone is refused at once, even while the disconnect grace +// period still reports it online: no stream can carry the c2n request. +func TestAPIv2DeviceUpdateClientDuringDisconnectGrace(t *testing.T) { + t.Parallel() + + h := servertest.NewHarness(t, 1) + + nodeID := types.NodeID(h.Client(0).Netmap().SelfNode.ID()) //nolint:gosec + + h.Client(0).Disconnect(t) + + require.EventuallyWithT(t, func(c *assert.CollectT) { + assert.False(c, h.Server.App.MapBatcher().IsConnected(nodeID)) + }, 5*time.Second, 50*time.Millisecond) + + nv, ok := h.Server.State().GetNodeByID(nodeID) + require.True(t, ok) + require.True(t, nv.IsOnline().Get(), "the grace period should still report the node online") + + // SaaS answers a device it cannot reach 404 (testdata/steps/update-client.hujson). + start := time.Now() + code, _ := updateDeviceClient(t, h.Server, nodeID, false) + assert.Equal(t, http.StatusNotFound, code) + assert.Less(t, time.Since(start), 5*time.Second, "refusal should not wait for the c2n timeout") +} + // TestAPIv2DeviceUpdateClientUnknownDevice verifies that an unknown device ID is a 404. func TestAPIv2DeviceUpdateClientUnknownDevice(t *testing.T) { t.Parallel() @@ -184,3 +216,304 @@ func TestAPIv2DeviceUpdateClientUnknownDevice(t *testing.T) { code, _ := updateDeviceClient(t, h.Server, types.NodeID(9999), false) assert.Equal(t, http.StatusNotFound, code) } + +// TestAPIv2DeviceUpdateClientCompat replays SaaS's update-client answers for +// devices it would not update: plain has auto-updates off; remote allows +// remote updates on a platform that cannot update itself, then goes offline. +// SaaS answered every such row 404 "not found", whatever the body, method or +// node state, without asking the node. +// +// Test data source: testdata/steps/update-client.hujson. Rows by +// devices:core:read and dns:read tokens are not replayed: SaaS answers them +// 404 before any scope check, which shows its order, not this endpoint's shape. +func TestAPIv2DeviceUpdateClientCompat(t *testing.T) { + t.Parallel() + + c := readStepCapture(t, "update-client") + r := newStepReplay(t, c) + r.setPolicy(t, "setup", c.Input.FullPolicy) + + // The probe's nodes ran on NixOS; each answers c2n POST /update as its + // tailscaled would have, had SaaS asked. + answers := map[string]tailcfg.C2NUpdateResponse{ + "plain": {Err: "not enabled"}, + "remote": {Enabled: true, Err: "not supported"}, + } + + ids := map[string]string{} + clients := map[string]*servertest.TestClient{} + + for alias, res := range answers { + hi := &tailcfg.Hostinfo{ + BackendLogID: "servertest-" + alias, + Hostname: alias, + AllowsUpdate: captureHostinfo(t, c, alias).AllowsUpdate, + } + cl := servertest.NewClient(t, r.srv, alias, servertest.WithUser(r.admin), + servertest.WithHostinfo(hi), servertest.WithC2NHandler(&fakeUpdater{res: res})) + cl.WaitForCondition(t, alias+" self", 10*time.Second, func(nm *netmap.NetworkMap) bool { return nm.SelfNode.Valid() }) + + clients[alias] = cl + ids["{device:"+alias+"}"] = types.NodeID(cl.Netmap().SelfNode.ID()).String() //nolint:gosec + } + + token := captureToken(t, r, c, "C-core") + skipped := map[string]bool{"token:C-coreread": true, "token:C-dnsread": true} + ran := 0 + + for _, step := range c.Steps { + if step.Name == "setup-offline" { + nodeID := types.NodeID(clients["remote"].Netmap().SelfNode.ID()) //nolint:gosec + clients["remote"].Disconnect(t) + + require.EventuallyWithT(t, func(c *assert.CollectT) { + nv, ok := r.srv.State().GetNodeByID(nodeID) + assert.True(c, ok) + assert.False(c, nv.IsOnline().Get(), "remote should be offline") + }, 20*time.Second, 200*time.Millisecond) + } + + for _, a := range step.Actions { + if a.Kind != "api" || !strings.Contains(a.Path, "/update-client") || skipped[a.Principal] { + continue + } + + ran++ + + bearer := r.apiKey + if a.Principal != "admin" { + bearer = token + } + + status, body := sendCaptureAction(t, r, a, ids, bearer) + assert.Equalf(t, a.Status, status, "%s: %s %s: %s", step.Name, a.Method, a.Path, body) + + // unknown-id's message comes from the lookup every device route shares. + if step.Name != "unknown-id" { + assert.Equalf(t, errorMessage(a.Response), errorMessage(body), "%s: %s %s", step.Name, a.Method, a.Path) + } + } + } + + require.Equal(t, 19, ran, "update-client rows replayed") +} + +// TestAPIv2DeviceUpdateClientConsoleCompat replays the admin console's Start +// update on a device that can update itself: POST update-client answers at +// once, then update-status says updating until the device runs a new version. +// The console's /admin/api/public routes are the v2 API inside +// {"status","data"}, so headscale must answer the data. +// +// Test data source: testdata/steps/update-client-console.hujson, imported +// from a browser HAR. Its 304s are the browser's cache and are not replayed; +// its machine reads give the device's versions and auto-update state. +func TestAPIv2DeviceUpdateClientConsoleCompat(t *testing.T) { + t.Parallel() + + c := readStepCapture(t, "update-client-console") + before, after := consoleMachines(t, c) + require.NotEqual(t, before.IPNVersion, after.IPNVersion, "the device should end on a new version") + + srv := servertest.NewServer(t) + hi := &tailcfg.Hostinfo{ + BackendLogID: "servertest-updater", + Hostname: "updater", + IPNVersion: before.IPNVersion, + AllowsUpdate: before.AutoUpdatesEnabled, + } + cl := servertest.NewClient(t, srv, "updater", servertest.WithHostinfo(hi.Clone()), + servertest.WithC2NHandler(&fakeUpdater{res: tailcfg.C2NUpdateResponse{Enabled: true, Supported: true, Started: true}})) + cl.WaitForCondition(t, "updater self", 10*time.Second, func(nm *netmap.NetworkMap) bool { return nm.SelfNode.Valid() }) + + r := &stepReplay{srv: srv, apiKey: srv.CreateAPIKey(t, nil), http: &http.Client{Transport: &http.Transport{DialContext: srv.MemNet().Dial}}} + ids := map[string]string{"{device:updater}": types.NodeID(cl.Netmap().SelfNode.ID()).String()} //nolint:gosec + restarted := false + ran := 0 + + for _, step := range c.Steps { + for _, a := range step.Actions { + if !strings.HasPrefix(a.Path, "/admin/api/public/") || a.Status == http.StatusNotModified { + continue + } + + ran++ + + var want struct { + Data json.RawMessage `json:"data"` + } + require.NoError(t, json.Unmarshal(a.Response, &want)) + + a.Path = strings.Replace(a.Path, "/admin/api/public", "", 1) + + // The device restarts on its new version between the last + // "updating" and the first "not-updating". + if strings.Contains(string(want.Data), `"not-updating"`) && !restarted { + restarted = true + hi.IPNVersion = after.IPNVersion + require.NoError(t, cl.Restart(t.Context(), hi.Clone())) + + require.EventuallyWithT(t, func(ct *assert.CollectT) { + status, body := sendCaptureAction(t, r, a, ids, r.apiKey) + assert.Equal(ct, a.Status, status) + assert.True(ct, sameData(want.Data, body), "%s %s: %s, want %s", a.Method, a.Path, body, want.Data) + }, 20*time.Second, 2*time.Second) + + continue + } + + status, body := sendCaptureAction(t, r, a, ids, r.apiKey) + assert.Equalf(t, a.Status, status, "%s: %s %s: %s", step.Name, a.Method, a.Path, body) + assert.Truef(t, sameData(want.Data, body), "%s: %s %s: %s, want %s", step.Name, a.Method, a.Path, body, want.Data) + } + } + + require.Equal(t, 3, ran, "update-client and update-status answers replayed") +} + +// captureHostinfo returns the Hostinfo of alias's first map request. +func captureHostinfo(t *testing.T, c *testcapture.Capture, alias string) *tailcfg.Hostinfo { + t.Helper() + + for _, s := range c.Steps { + for _, f := range s.Frames[alias] { + if f.Request != nil && f.Request.Hostinfo != nil { + return f.Request.Hostinfo + } + } + } + + t.Fatalf("%s: no map request from %s", c.TestID, alias) + + return nil +} + +// captureToken makes the capture's OAuth client with the admin's body, as +// the probe did, and returns an access token for it. +func captureToken(t *testing.T, r *stepReplay, c *testcapture.Capture, client string) string { + t.Helper() + + for _, s := range c.Steps { + for _, a := range s.Actions { + if a.Kind != "api" || a.Client != client { + continue + } + + status, body := sendCaptureAction(t, r, a, nil, r.apiKey) + require.Equal(t, http.StatusOK, status, "%s: %s", client, body) + + var created struct { + Key string `json:"key"` + } + require.NoError(t, json.Unmarshal(body, &created)) + + form := url.Values{"grant_type": {"client_credentials"}, "client_secret": {created.Key}} + req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, r.srv.URL+"/api/v2/oauth/token", strings.NewReader(form.Encode())) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + resp, err := r.http.Do(req) + require.NoError(t, err) + + defer resp.Body.Close() + + var tok struct { + AccessToken string `json:"access_token"` + } + require.NoError(t, json.NewDecoder(resp.Body).Decode(&tok)) + require.NotEmpty(t, tok.AccessToken, "%s: token", client) + + return tok.AccessToken + } + } + + t.Fatalf("%s: no OAuth client %s", c.TestID, client) + + return "" +} + +// sendCaptureAction sends a recorded API action to headscale's v2 API, its +// labels replaced by ids, with the body and headers SaaS received. +func sendCaptureAction(t *testing.T, r *stepReplay, a testcapture.Action, ids map[string]string, bearer string) (int, []byte) { + t.Helper() + + path := a.Path + for label, id := range ids { + path = strings.ReplaceAll(path, label, id) + } + + // A body that is not JSON is stored as a JSON string; "" is an empty one. + body := []byte(a.Body) + + var s string + if json.Unmarshal(a.Body, &s) == nil { + body = []byte(s) + } + + req, err := http.NewRequestWithContext(t.Context(), a.Method, r.srv.URL+"/api/v2"+path, bytes.NewReader(body)) + require.NoError(t, err) + + for k, v := range a.Header { + req.Header.Set(k, v) + } + + req.Header.Set("Authorization", "Bearer "+bearer) + + resp, err := r.http.Do(req) + require.NoError(t, err) + + defer resp.Body.Close() + + out, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + return resp.StatusCode, out +} + +// sameData reports whether a v2 answer is the console's data. The console +// sent no data for a POST that succeeded: its v2 answer is empty, null or {}. +func sameData(want json.RawMessage, got []byte) bool { + if len(want) == 0 { + g := strings.TrimSpace(string(got)) + return g == "" || g == "null" || g == "{}" + } + + var w, g any + + return json.Unmarshal(want, &w) == nil && json.Unmarshal(got, &g) == nil && assert.ObjectsAreEqual(w, g) +} + +type consoleMachine struct { + IPNVersion string `json:"ipnVersion"` + AutoUpdatesEnabled bool `json:"autoUpdatesEnabled"` +} + +// consoleMachines returns the device as the console's first and last machine +// reads show it. +func consoleMachines(t *testing.T, c *testcapture.Capture) (consoleMachine, consoleMachine) { + t.Helper() + + var seen []consoleMachine + + for _, s := range c.Steps { + for _, a := range s.Actions { + if !strings.HasPrefix(a.Path, "/admin/api/machines") { + continue + } + + var env struct { + Data struct { + consoleMachine + + Machines []consoleMachine `json:"machines"` + } `json:"data"` + } + require.NoError(t, json.Unmarshal(a.Response, &env)) + + seen = append(seen, append(env.Data.Machines, env.Data.consoleMachine)[0]) + } + } + + require.NotEmpty(t, seen, "%s: no machine reads", c.TestID) + + return seen[0], seen[len(seen)-1] +}