servertest: add route approval lifecycle golden data

Owner changes, re-auth, revokes, policy pushes, withdraw and logout
replayed step by step against SaaS approvals and served routes.

.gitignore

@@ -30,6 +30,7 @@ config*.yaml
 derp.yaml
 *.hujson
 !hscontrol/policy/v2/testdata/**/*.hujson
+!hscontrol/servertest/testdata/*/*.hujson
 *.key
 /db.sqlite
 *.sqlite3

hscontrol/servertest/lifecycle_compat_test.go

@@ -0,0 +1,543 @@
+// This file replays multi-step lifecycle captures from Tailscale SaaS and
+// compares each step's outcome with what SaaS showed.
+//
+// Test data source: testdata/steps/{route-approval,owner-routes}.hujson,
+// loaded by name. owner-routes is consumed by TestReauthCompat.
+// Source format:    github.com/juanfont/headscale/hscontrol/types/testcapture
+
+package servertest_test
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"slices"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/juanfont/headscale/hscontrol/types"
+	"github.com/juanfont/headscale/hscontrol/types/testcapture"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/key"
+	"tailscale.com/types/netmap"
+)
+
+// lifecycleSkipReasons lists checks where headscale differs from SaaS, keyed
+// <test_id>/<step>/<check>. A listed check that matches fails, so a fix must
+// delete its entry.
+var lifecycleSkipReasons = map[string]string{
+	"route-approval/b6/enabled":     approvalOnHostinfo,
+	"route-approval/b6/served":      approvalOnHostinfo,
+	"route-approval/b6/self-routes": approvalOnHostinfo,
+
+	"route-approval/b7/enabled":     approvalOnPolicy,
+	"route-approval/b7/served":      approvalOnPolicy,
+	"route-approval/b7/self-routes": approvalOnPolicy,
+	"route-approval/b8/enabled":     approvalOnPolicy,
+	"route-approval/b8/served":      approvalOnPolicy,
+	"route-approval/b8/self-routes": approvalOnPolicy,
+
+	"route-approval/b9/enabled":     approvalOnSameKeyReauth,
+	"route-approval/b9/served":      approvalOnSameKeyReauth,
+	"route-approval/b9/self-routes": approvalOnSameKeyReauth,
+
+	"route-approval/r1/served": loggedOutRouterUnserved,
+}
+
+// SaaS runs auto-approval when a router's announced routes change and on a
+// re-auth that rotates its node key, over every announced route.
+const (
+	approvalOnHostinfo      = "APPROVAL_ON_HOSTINFO: headscale auto-approves when a router with no approved routes sends a Hostinfo change that keeps its routes; SaaS does not"
+	approvalOnPolicy        = "APPROVAL_ON_POLICY: headscale auto-approves announced routes when a policy is set; SaaS waits for the router's next route change or key rotation"
+	approvalOnSameKeyReauth = "APPROVAL_ON_SAME_KEY_REAUTH: headscale auto-approves on a re-auth that keeps the node key; SaaS does not"
+	loggedOutRouterUnserved = "LOGGED_OUT_ROUTER_UNSERVED: headscale stops serving a logged-out router's routes; SaaS keeps them on the expired, offline peer"
+)
+
+// Used by reauthSkipReasons.
+const (
+	expiredRouterUnserved = "EXPIRED_ROUTER_UNSERVED: headscale stops serving an expired router's routes; SaaS keeps them in the peer's AllowedIPs and PrimaryRoutes"
+	expiredPeerOffline    = "EXPIRED_PEER_OFFLINE: headscale shows an expired peer offline while it is connected; SaaS shows it online"
+)
+
+// TestRouteApprovalCompat replays a user-owned router through admin
+// approvals and revokes, same-key and forced re-auth, a hostname change,
+// policy pushes, a withdraw and re-advertise, then logout and a login with
+// a new key. Each step compares the routes API, the routes a tagged
+// observer is served, and the router's own view with SaaS.
+func TestRouteApprovalCompat(t *testing.T) {
+	t.Parallel()
+
+	t.Run("route-approval", func(t *testing.T) {
+		t.Parallel()
+
+		runRouteLifecycleCompat(t, "route-approval", "t-client", lifecycleSkipReasons)
+	})
+}
+
+// stepCheck is one rendering of a step outcome: SaaS's, and how to read
+// headscale's.
+type stepCheck struct {
+	name string
+	want string
+	got  func() string
+}
+
+// runRouteLifecycleCompat replays capture id step by step and, after each
+// step that read a router's routes, checks against SaaS:
+//   - advertised, enabled: the v2 routes API;
+//   - owner, device, node-key: who owns the router, whether its machine
+//     still holds one node with the join-time ID and addresses, and whether
+//     the step rotated its node key;
+//   - served, expired, online: the router as the observer's peer;
+//   - self-routes: the router's own routes, while SaaS showed it running;
+//   - removed: whether the observer lost the router during the step, when
+//     SaaS recorded the observer's map stream.
+func runRouteLifecycleCompat(t *testing.T, id, observer string, skips map[string]string) {
+	t.Helper()
+
+	c := readStepCapture(t, id)
+	r := newStepReplay(t, c)
+
+	type joined struct {
+		saasID, saasAddrs string
+		hsID              types.NodeID
+		hsAddrs           string
+	}
+
+	first := map[string]joined{}
+	saasKey := map[string]string{}
+	hsKey := map[string]key.NodePublic{}
+	evaluated := map[string]bool{}
+
+	var prevObserved *netmap.NetworkMap
+
+	for _, step := range c.Steps {
+		require.Equal(t, "observed", step.Result, "%s: the replay follows every step", step.Name)
+
+		// From the last netmap before the step, so a peer present then and
+		// missing from the step's first netmap counts as removed.
+		historyStart := 0
+		if obs, ok := r.clients[observer]; ok {
+			historyStart = max(len(obs.History())-1, 0)
+		}
+
+		r.apply(t, step)
+
+		prevSaaS := prevObserved
+		prevObserved = step.Captures[observer].Netmap
+
+		router, saasRoutes, ok := lastRoutesRead(t, step)
+		if !ok {
+			continue
+		}
+
+		dev := saasDeviceOf(t, step, router)
+		saasRouterID := c.NodeIDs[router]
+
+		t.Run(step.Name, func(t *testing.T) {
+			if _, ok := first[router]; !ok {
+				nv := r.node(t, router)
+				first[router] = joined{
+					saasID: dev.ID, saasAddrs: sortedStrings(dev.Addresses),
+					hsID: nv.ID(), hsAddrs: sortedStrings(nv.IPsAsString()),
+				}
+			}
+
+			join := first[router]
+
+			hsRoutes := func() (deviceRoutes, bool) {
+				nodes := r.machineNodes(router)
+				if len(nodes) != 1 {
+					return deviceRoutes{}, false
+				}
+
+				status, body := r.call(t, http.MethodGet, "/api/v2/device/"+nodes[0].StringID()+"/routes", nil)
+				require.Equal(t, http.StatusOK, status, "%s", body)
+
+				var out deviceRoutes
+				require.NoError(t, json.Unmarshal(body, &out))
+
+				return out, true
+			}
+
+			// hsPeer is the router as the observer's peer in headscale.
+			hsPeer := func() (tailcfg.NodeView, string) {
+				nodes := r.machineNodes(router)
+				if len(nodes) != 1 {
+					return tailcfg.NodeView{}, fmt.Sprintf("devices %d", len(nodes))
+				}
+
+				nm := r.clients[observer].Netmap()
+				if nm == nil {
+					return tailcfg.NodeView{}, "no netmap"
+				}
+
+				return peerByID(nm, nodes[0].ID().NodeID())
+			}
+
+			saasPeer, saasPeerState := peerByID(step.Captures[observer].Netmap, saasRouterID)
+
+			checks := []stepCheck{
+				{"advertised", sortedStrings(saasRoutes.Advertised), func() string {
+					got, ok := hsRoutes()
+					if !ok {
+						return "no single device"
+					}
+
+					return sortedStrings(got.Advertised)
+				}},
+				{"enabled", sortedStrings(saasRoutes.Enabled), func() string {
+					got, ok := hsRoutes()
+					if !ok {
+						return "no single device"
+					}
+
+					return sortedStrings(got.Enabled)
+				}},
+				{"owner", saasOwner(dev), func() string {
+					nodes := r.machineNodes(router)
+					if len(nodes) != 1 {
+						return fmt.Sprintf("devices %d", len(nodes))
+					}
+
+					if nodes[0].IsTagged() {
+						return "tags " + sortedStrings(nodes[0].Tags().AsSlice())
+					}
+
+					return "user " + nodes[0].User().Name()
+				}},
+				{"device", deviceState(
+					countDevicesOnMachine(t, step, dev.MachineKey),
+					dev.ID == join.saasID, sortedStrings(dev.Addresses) == join.saasAddrs,
+				), func() string {
+					nodes := r.machineNodes(router)
+					if len(nodes) != 1 {
+						return deviceState(len(nodes), false, false)
+					}
+
+					return deviceState(1, nodes[0].ID() == join.hsID,
+						sortedStrings(nodes[0].IPsAsString()) == join.hsAddrs)
+				}},
+				{"served", servedRoutes(saasPeer, saasPeerState), func() string {
+					return servedRoutes(hsPeer())
+				}},
+				{"expired", peerExpired(saasPeer, saasPeerState), func() string {
+					return peerExpired(hsPeer())
+				}},
+				{"online", peerOnline(saasPeer, saasPeerState), func() string {
+					return peerOnline(hsPeer())
+				}},
+			}
+
+			if prev, ok := saasKey[router]; ok {
+				prevHS := hsKey[router]
+
+				checks = append(checks, stepCheck{"node-key", keyChange(prev == dev.NodeKey), func() string {
+					nodes := r.machineNodes(router)
+					if len(nodes) != 1 {
+						return fmt.Sprintf("devices %d", len(nodes))
+					}
+
+					return keyChange(nodes[0].NodeKey() == prevHS)
+				}})
+			}
+
+			if self := step.Captures[router]; self.BackendState == "Running" && self.Netmap != nil {
+				checks = append(checks, stepCheck{"self-routes", selfRoutes(self.Netmap), func() string {
+					nm := r.clients[router].Netmap()
+					if nm == nil {
+						return "no netmap"
+					}
+
+					return selfRoutes(nm)
+				}})
+			}
+
+			if frames := step.Frames[observer]; len(frames) > 0 {
+				checks = append(checks, stepCheck{"removed", saasRemoved(prevSaaS, frames, saasRouterID), func() string {
+					nodes := r.machineNodes(router)
+					if len(nodes) != 1 {
+						return fmt.Sprintf("devices %d", len(nodes))
+					}
+
+					return hsRemoved(r.clients[observer].History()[historyStart:], nodes[0].ID().NodeID())
+				}})
+			}
+
+			holdChecks(t, skips, id+"/"+step.Name+"/", checks, evaluated)
+		})
+
+		saasKey[router] = dev.NodeKey
+		if nodes := r.machineNodes(router); len(nodes) == 1 {
+			hsKey[router] = nodes[0].NodeKey()
+		}
+	}
+
+	for k := range skips {
+		if strings.HasPrefix(k, id+"/") && !evaluated[k] {
+			t.Errorf("skip entry %q names no check", k)
+		}
+	}
+}
+
+// holdChecks polls the checks until, for several ticks in a row, each one
+// listed in skips differs from SaaS and every other one matches, then
+// reports each as a subtest. A listed check that matches fails.
+func holdChecks(
+	t *testing.T,
+	skips map[string]string,
+	prefix string,
+	checks []stepCheck,
+	evaluated map[string]bool,
+) {
+	t.Helper()
+
+	// Long enough for a batch still in flight to land.
+	const holdTicks = 5
+
+	got := make([]string, len(checks))
+	held := 0
+
+	tick := time.NewTicker(100 * time.Millisecond)
+	defer tick.Stop()
+
+	for deadline := time.Now().Add(15 * time.Second); held < holdTicks && time.Now().Before(deadline); <-tick.C {
+		settled := true
+
+		for i, c := range checks {
+			got[i] = c.got()
+			if _, listed := skips[prefix+c.name]; (got[i] == c.want) == listed {
+				settled = false
+			}
+		}
+
+		if settled {
+			held++
+		} else {
+			held = 0
+		}
+	}
+
+	for i, c := range checks {
+		key := prefix + c.name
+		evaluated[key] = true
+
+		t.Run(c.name, func(t *testing.T) {
+			reason, listed := skips[key]
+
+			switch {
+			case listed && got[i] == c.want:
+				t.Errorf("matches SaaS; delete the skip entry %q (%s)", key, reason)
+			case listed:
+				t.Skip(reason)
+			default:
+				assert.Equal(t, c.want, got[i])
+			}
+		})
+	}
+}
+
+// deviceRoutes is the routes API body, on SaaS and on headscale's v2 API.
+type deviceRoutes struct {
+	Advertised []string `json:"advertisedRoutes"`
+	Enabled    []string `json:"enabledRoutes"`
+}
+
+// saasDevice is the part of a SaaS device-list entry the checks read.
+// Keys and IDs are the capture's labels.
+type saasDevice struct {
+	ID         string   `json:"id"`
+	NodeKey    string   `json:"nodeKey"`
+	MachineKey string   `json:"machineKey"`
+	Addresses  []string `json:"addresses"`
+	Tags       []string `json:"tags"`
+	User       string   `json:"user"`
+}
+
+// lastRoutesRead returns the step's last routes read: the device it named
+// and SaaS's answer.
+func lastRoutesRead(t *testing.T, step testcapture.Step) (string, deviceRoutes, bool) {
+	t.Helper()
+
+	for _, a := range slices.Backward(step.Actions) {
+		if a.Kind != "api" || a.Method != http.MethodGet || !strings.HasSuffix(a.Path, "/routes") {
+			continue
+		}
+
+		require.Equal(t, http.StatusOK, a.Status, "%s: %s", step.Name, a.Path)
+
+		var out deviceRoutes
+		require.NoError(t, json.Unmarshal(a.Response, &out), "%s: %s", step.Name, a.Path)
+
+		return deviceAlias(t, a.Path), out, true
+	}
+
+	return "", deviceRoutes{}, false
+}
+
+// stepDevices decodes the step's last device-list read.
+func stepDevices(t *testing.T, step testcapture.Step) []saasDevice {
+	t.Helper()
+
+	for _, a := range slices.Backward(step.Actions) {
+		if a.Kind != "api" || a.Method != http.MethodGet || !strings.HasPrefix(a.Path, "/tailnet/-/devices") {
+			continue
+		}
+
+		var out struct {
+			Devices []saasDevice `json:"devices"`
+		}
+		require.NoError(t, json.Unmarshal(a.Response, &out), "%s: %s", step.Name, a.Path)
+
+		return out.Devices
+	}
+
+	t.Fatalf("%s: no device-list read", step.Name)
+
+	return nil
+}
+
+// saasDeviceOf finds alias's device by its machine, so a device SaaS
+// re-created would still be found.
+func saasDeviceOf(t *testing.T, step testcapture.Step, alias string) saasDevice {
+	t.Helper()
+
+	for _, d := range stepDevices(t, step) {
+		if d.MachineKey == "{machine:"+alias+"}" {
+			return d
+		}
+	}
+
+	t.Fatalf("%s: no device on %s's machine", step.Name, alias)
+
+	return saasDevice{}
+}
+
+func countDevicesOnMachine(t *testing.T, step testcapture.Step, machine string) int {
+	t.Helper()
+
+	n := 0
+
+	for _, d := range stepDevices(t, step) {
+		if d.MachineKey == machine {
+			n++
+		}
+	}
+
+	return n
+}
+
+func saasOwner(d saasDevice) string {
+	if len(d.Tags) > 0 {
+		return "tags " + sortedStrings(d.Tags)
+	}
+
+	return "user " + captureUserNames[d.User]
+}
+
+func deviceState(devices int, sameID, sameAddrs bool) string {
+	return fmt.Sprintf("devices %d, id kept %t, addresses kept %t", devices, sameID, sameAddrs)
+}
+
+func keyChange(kept bool) string {
+	if kept {
+		return "kept"
+	}
+
+	return "rotated"
+}
+
+// peerByID finds a peer, or says why there is none.
+func peerByID(nm *netmap.NetworkMap, id tailcfg.NodeID) (tailcfg.NodeView, string) {
+	if nm == nil {
+		return tailcfg.NodeView{}, "no netmap"
+	}
+
+	for _, p := range nm.Peers {
+		if p.ID() == id {
+			return p, ""
+		}
+	}
+
+	return tailcfg.NodeView{}, "absent"
+}
+
+func servedRoutes(p tailcfg.NodeView, missing string) string {
+	if missing != "" {
+		return missing
+	}
+
+	return "allowed " + selfAllowed(p) + " primary " + prefixList(p.PrimaryRoutes())
+}
+
+func peerExpired(p tailcfg.NodeView, missing string) string {
+	if missing != "" {
+		return missing
+	}
+
+	return strconv.FormatBool(p.Expired())
+}
+
+func peerOnline(p tailcfg.NodeView, missing string) string {
+	if missing != "" {
+		return missing
+	}
+
+	if !p.Online().Valid() {
+		return "unknown"
+	}
+
+	return strconv.FormatBool(p.Online().Get())
+}
+
+// saasRemoved reports whether the peer, present before the step or once a
+// frame named it, was then removed or left out of a full peer list.
+func saasRemoved(before *netmap.NetworkMap, frames []testcapture.Frame, id tailcfg.NodeID) string {
+	_, missing := peerByID(before, id)
+	seen := missing == ""
+
+	for _, f := range frames {
+		resp := f.Response
+		if resp == nil {
+			continue
+		}
+
+		if slices.Contains(resp.PeersRemoved, id) {
+			return "removed"
+		}
+
+		listed := slices.ContainsFunc(resp.Peers, func(n *tailcfg.Node) bool { return n.ID == id })
+
+		switch {
+		case listed || slices.ContainsFunc(resp.PeersChanged, func(n *tailcfg.Node) bool { return n.ID == id }):
+			seen = true
+		case resp.Peers != nil && seen:
+			return "removed"
+		}
+	}
+
+	return "kept"
+}
+
+// hsRemoved reports whether the peer, once present, went missing from a
+// later netmap.
+func hsRemoved(history []*netmap.NetworkMap, id tailcfg.NodeID) string {
+	seen := false
+
+	for _, nm := range history {
+		_, missing := peerByID(nm, id)
+
+		switch {
+		case missing == "":
+			seen = true
+		case seen:
+			return "removed"
+		}
+	}
+
+	return "kept"
+}

hscontrol/servertest/reauth_compat_test.go

@@ -0,0 +1,47 @@
+// This file replays re-authentication captures from Tailscale SaaS.
+//
+// Test data source: testdata/steps/owner-routes.hujson, loaded by name; the
+// replay and checks are runRouteLifecycleCompat in lifecycle_compat_test.go.
+// Source format:    github.com/juanfont/headscale/hscontrol/types/testcapture
+
+package servertest_test
+
+import "testing"
+
+// reauthSkipReasons lists checks where headscale differs from SaaS, keyed
+// <test_id>/<step>/<check>. A listed check that matches fails, so a fix must
+// delete its entry.
+var reauthSkipReasons = map[string]string{
+	"owner-routes/r-expired-retag/expire/served": expiredRouterUnserved,
+	"owner-routes/r-expired-retag/expire/online": expiredPeerOffline,
+
+	"owner-routes/r-expired-retag/owner/node-key": expiredKeyAccepted,
+
+	"owner-routes/r-admin/owner/enabled":     adminRetagNoApproval,
+	"owner-routes/r-admin/owner/served":      adminRetagNoApproval,
+	"owner-routes/r-admin/owner/self-routes": adminRetagNoApproval,
+
+	"owner-routes/r-retag/withdraw/enabled":         withdrawKeepsAutoApproval,
+	"owner-routes/r-expired-retag/withdraw/enabled": withdrawKeepsAutoApproval,
+	"owner-routes/u-convert/withdraw/enabled":       withdrawKeepsAutoApproval,
+}
+
+const (
+	expiredKeyAccepted        = "EXPIRED_KEY_ACCEPTED: headscale re-registers an expired node under its old node key; SaaS answers NodeKeyExpired, so the client rotates it"
+	adminRetagNoApproval      = "ADMIN_RETAG_NO_APPROVAL: an admin tag change does not auto-approve the routes the new tags may approve; SaaS approves them at once"
+	withdrawKeepsAutoApproval = "WITHDRAW_KEEPS_AUTO_APPROVAL: headscale keeps an auto-approved route once it is withdrawn; SaaS drops it and keeps only admin-set approvals"
+)
+
+// TestReauthCompat replays owner changes on routers with approved routes:
+// re-auth with another tag's key, the same after expiry and key deletion,
+// same-tag rotation, admin retag, a user router converted by a tagged key
+// and same-user re-auth, each followed by a withdraw and re-advertise.
+func TestReauthCompat(t *testing.T) {
+	t.Parallel()
+
+	t.Run("owner-routes", func(t *testing.T) {
+		t.Parallel()
+
+		runRouteLifecycleCompat(t, "owner-routes", "obs", reauthSkipReasons)
+	})
+}

hscontrol/servertest/via_ha_compat_test.go

@@ -10,7 +10,12 @@
 package servertest_test
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
+	"io"
+	"maps"
+	"net/http"
 	"net/netip"
 	"path/filepath"
 	"slices"
@@ -410,15 +415,23 @@ func createCaptureUsers(t *testing.T, srv *servertest.TestServer) map[string]*ty
 	return users
 }
 
-// convertCapturePolicy converts a [testcapture.Capture]'s policy for headscale,
-// replacing each SaaS email with the "<name>@" form that resolves to the
-// users [createCaptureUsers] creates. Fails the test if none of the known
-// SaaS emails are present: that would mean the capture was regenerated
-// with a new identity and captureUserNames needs updating.
+// convertCapturePolicy converts a [testcapture.Capture]'s policy for headscale;
+// see [convertPolicyEmails].
 func convertCapturePolicy(t *testing.T, c *testcapture.Capture) []byte {
 	t.Helper()
 
-	s := c.Input.FullPolicy
+	return convertPolicyEmails(t, c.TestID, c.Input.FullPolicy)
+}
+
+// convertPolicyEmails replaces each SaaS email in policy with the "<name>@"
+// form that resolves to the users [createCaptureUsers] creates. Fails the
+// test if none of the known SaaS emails are present: that would mean the
+// capture was regenerated with a new identity and captureUserNames needs
+// updating.
+func convertPolicyEmails(t *testing.T, testID, policy string) []byte {
+	t.Helper()
+
+	s := policy
 
 	substituted := false
 
@@ -433,8 +446,275 @@ func convertCapturePolicy(t *testing.T, c *testcapture.Capture) []byte {
 		t,
 		substituted,
 		"%s: no known SaaS email found in policy; update captureUserNames",
-		c.TestID,
+		testID,
 	)
 
 	return []byte(s)
 }
+
+// readStepCapture loads a multi-step capture by name from testdata/steps.
+// Step files are read by name, never globbed, so a consumer cannot pick up
+// another family's file by accident.
+func readStepCapture(t *testing.T, id string) *testcapture.Capture {
+	t.Helper()
+
+	c, err := testcapture.Read(filepath.Join("testdata", "steps", id+".hujson"))
+	require.NoError(t, err)
+	require.NotEmpty(t, c.Steps, "%s: not a step capture", id)
+
+	return c
+}
+
+// stepReplay drives a step capture's actions against headscale: policy
+// pushes, key mints, joins, relogins, route announcements, logouts and
+// admin calls, the last through the v2 API with the bodies SaaS received.
+// Nothing is seeded from the capture's end state.
+type stepReplay struct {
+	c        *testcapture.Capture
+	srv      *servertest.TestServer
+	admin    *types.User
+	apiKey   string
+	http     *http.Client
+	keys     map[string]string
+	clients  map[string]*servertest.TestClient
+	hostinfo map[string]*tailcfg.Hostinfo
+}
+
+func newStepReplay(t *testing.T, c *testcapture.Capture) *stepReplay {
+	t.Helper()
+
+	srv := servertest.NewServer(t)
+	users := createCaptureUsers(t, srv)
+
+	// SaaS keys and admin calls act as the tailnet's admin.
+	admin, ok := users[captureUserNames[c.Tailnet]]
+	require.True(t, ok, "%s: tailnet %q is not a capture user", c.TestID, c.Tailnet)
+
+	return &stepReplay{
+		c:        c,
+		srv:      srv,
+		admin:    admin,
+		apiKey:   srv.CreateAPIKey(t, admin),
+		http:     &http.Client{Transport: &http.Transport{DialContext: srv.MemNet().Dial}},
+		keys:     map[string]string{},
+		clients:  map[string]*servertest.TestClient{},
+		hostinfo: map[string]*tailcfg.Hostinfo{},
+	}
+}
+
+// apply runs the step's actions in order. Reads are skipped: they are what
+// consumers compare.
+func (r *stepReplay) apply(t *testing.T, step testcapture.Step) {
+	t.Helper()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	for _, a := range step.Actions {
+		switch a.Kind {
+		case "policy":
+			require.Equal(t, http.StatusOK, a.Status, "%s: replay of a refused policy", step.Name)
+
+			// HuJSON bodies are stored as a JSON string.
+			policy := string(a.Body)
+			if strings.HasPrefix(policy, `"`) {
+				require.NoError(t, json.Unmarshal(a.Body, &policy))
+			}
+
+			changed, err := r.srv.State().SetPolicy(convertPolicyEmails(t, r.c.TestID, policy))
+			require.NoError(t, err, "%s: policy", step.Name)
+
+			if changed {
+				changes, err := r.srv.State().ReloadPolicy()
+				require.NoError(t, err)
+				r.srv.App.Change(changes...)
+			}
+
+		case "api":
+			r.api(t, step.Name, a)
+
+		case "join":
+			require.Contains(t, r.keys, a.Key, "%s: join with unknown key", step.Name)
+
+			r.hostinfo[a.Node] = &tailcfg.Hostinfo{BackendLogID: "servertest-" + a.Node, Hostname: a.Hostname}
+			r.clients[a.Node] = servertest.NewClient(t, r.srv, a.Node,
+				servertest.WithUser(r.admin),
+				servertest.WithAuthKey(r.keys[a.Key]),
+				servertest.WithHostname(a.Hostname),
+			)
+
+			if len(a.Routes) > 0 {
+				r.announce(ctx, t, a.Node, a.Routes)
+			}
+
+		case "relogin":
+			require.Contains(t, r.keys, a.Key, "%s: relogin with unknown key", step.Name)
+
+			// The probe restates hostname and routes on every login.
+			hi := r.hostinfo[a.Node].Clone()
+			hi.Hostname = a.Hostname
+			hi.RoutableIPs = prefixes(t, a.Routes)
+			r.hostinfo[a.Node] = hi
+
+			require.NoError(t, r.clients[a.Node].Relogin(ctx, r.keys[a.Key], a.Force, hi.Clone()),
+				"%s: relogin", step.Name)
+
+		case "advertise":
+			r.announce(ctx, t, a.Node, a.Routes)
+
+		case "cli":
+			hostname, ok := strings.CutPrefix(strings.Join(a.Args, " "), "set --hostname=")
+			require.True(t, ok, "%s: no replay for cli %q", step.Name, a.Args)
+
+			r.hostinfo[a.Node].Hostname = hostname
+			r.sendHostinfo(ctx, t, a.Node)
+
+		case "logout":
+			require.NoError(t, r.clients[a.Node].LogoutAndDisconnect(ctx), "%s: logout", step.Name)
+
+		case "wait", "retire":
+			// Waits are SaaS settle time; retire only frees SaaS devices.
+
+		default:
+			t.Fatalf("%s: no replay for action %q", step.Name, a.Kind)
+		}
+	}
+}
+
+// api replays one admin exchange. Device paths go to the v2 API; key
+// mints and expiry, which the v2 API lacks or would answer with secrets,
+// go to State.
+func (r *stepReplay) api(t *testing.T, stepName string, a testcapture.Action) {
+	t.Helper()
+
+	switch {
+	case a.Method == http.MethodGet:
+		return
+
+	case a.Method == http.MethodPost && a.Path == "/tailnet/-/keys":
+		var body struct {
+			Capabilities struct {
+				Devices struct {
+					Create struct {
+						Reusable  bool     `json:"reusable"`
+						Ephemeral bool     `json:"ephemeral"`
+						Tags      []string `json:"tags"`
+					} `json:"create"`
+				} `json:"devices"`
+			} `json:"capabilities"`
+			ExpirySeconds int `json:"expirySeconds"`
+		}
+		require.NoError(t, json.Unmarshal(a.Body, &body), "%s: key body", stepName)
+
+		create := body.Capabilities.Devices.Create
+		expiry := time.Now().Add(time.Duration(body.ExpirySeconds) * time.Second)
+		uid := types.UserID(r.admin.ID)
+
+		pak, err := r.srv.State().CreatePreAuthKey(&uid, create.Reusable, create.Ephemeral, &expiry, create.Tags)
+		require.NoError(t, err, "%s: mint %s", stepName, a.Key)
+
+		r.keys[a.Key] = pak.Key
+
+	case a.Method == http.MethodDelete && strings.HasPrefix(a.Path, "/tailnet/-/keys/"):
+		pak, err := r.srv.State().GetPreAuthKey(r.keys[a.Key])
+		require.NoError(t, err, "%s: key %s", stepName, a.Key)
+		require.NoError(t, r.srv.State().DeletePreAuthKey(pak.ID), "%s: delete %s", stepName, a.Key)
+
+	case a.Method == http.MethodPost && strings.HasSuffix(a.Path, "/expire"):
+		now := time.Now()
+
+		_, c, err := r.srv.State().SetNodeExpiry(r.node(t, deviceAlias(t, a.Path)).ID(), &now)
+		require.NoError(t, err, "%s: expire", stepName)
+		r.srv.App.Change(c)
+
+	case strings.HasPrefix(a.Path, "/device/"):
+		alias := deviceAlias(t, a.Path)
+		path := "/api/v2/device/" + r.node(t, alias).StringID() +
+			strings.TrimPrefix(a.Path, "/device/{device:"+alias+"}")
+
+		status, resp := r.call(t, a.Method, path, a.Body)
+		require.Equal(t, a.Status, status, "%s: %s %s: %s", stepName, a.Method, a.Path, resp)
+
+	default:
+		t.Fatalf("%s: no replay for %s %s", stepName, a.Method, a.Path)
+	}
+}
+
+// call sends one v2 API request as the admin.
+func (r *stepReplay) call(t *testing.T, method, path string, body []byte) (int, []byte) {
+	t.Helper()
+
+	req, err := http.NewRequestWithContext(context.Background(), method, r.srv.URL+path, bytes.NewReader(body))
+	require.NoError(t, err)
+	req.Header.Set("Authorization", "Bearer "+r.apiKey)
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := r.http.Do(req)
+	require.NoError(t, err)
+
+	defer resp.Body.Close()
+
+	out, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+
+	return resp.StatusCode, out
+}
+
+// machineNodes lists the nodes alias's machine holds now.
+func (r *stepReplay) machineNodes(alias string) []types.NodeView {
+	cl, ok := r.clients[alias]
+	if !ok {
+		return nil
+	}
+
+	nodes := r.srv.State().GetNodesByMachineKeyAllUsers(cl.MachineKey())
+
+	return slices.Collect(maps.Values(nodes))
+}
+
+// node is the one node alias's machine holds now.
+func (r *stepReplay) node(t *testing.T, alias string) types.NodeView {
+	t.Helper()
+
+	nodes := r.machineNodes(alias)
+	require.Len(t, nodes, 1, "%s: nodes on its machine", alias)
+
+	return nodes[0]
+}
+
+func (r *stepReplay) announce(ctx context.Context, t *testing.T, alias string, routes []string) {
+	t.Helper()
+
+	r.hostinfo[alias].RoutableIPs = prefixes(t, routes)
+	r.sendHostinfo(ctx, t, alias)
+}
+
+func (r *stepReplay) sendHostinfo(ctx context.Context, t *testing.T, alias string) {
+	t.Helper()
+
+	cl := r.clients[alias]
+	cl.Direct().SetHostinfo(r.hostinfo[alias].Clone())
+	require.NoError(t, cl.Direct().SendUpdate(ctx), "%s: hostinfo update", alias)
+}
+
+// deviceAlias extracts x from a "/device/{device:x}/..." path.
+func deviceAlias(t *testing.T, path string) string {
+	t.Helper()
+
+	rest, ok := strings.CutPrefix(path, "/device/{device:")
+	alias, _, found := strings.Cut(rest, "}")
+	require.True(t, ok && found, "no device label in %q", path)
+
+	return alias
+}
+
+func prefixes(t *testing.T, routes []string) []netip.Prefix {
+	t.Helper()
+
+	out := make([]netip.Prefix, 0, len(routes))
+	for _, r := range routes {
+		out = append(out, netip.MustParsePrefix(r))
+	}
+
+	return out
+}