servertest, hscontrol: add ephemeral, expiry, pending login golden data

Ephemeral devices through logout, deletes, re-auth and collection;
expired routers and nodes, a client's own expiry and logout; a login
waiting across restarts, replayed exchange by exchange.

hscontrol/auth_compat_test.go

@@ -0,0 +1,632 @@
+// This file replays pending interactive logins captured from Tailscale SaaS
+// through the register handler, one client exchange at a time.
+//
+// Test data source: servertest/testdata/steps/pending-login.hujson, loaded
+// by name.
+// Source format:    github.com/juanfont/headscale/hscontrol/types/testcapture
+
+package hscontrol
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"maps"
+	"net/http"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/juanfont/headscale/hscontrol/types"
+	"github.com/juanfont/headscale/hscontrol/types/testcapture"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/key"
+)
+
+// pendingLoginSkipReasons lists checks where headscale differs from SaaS,
+// keyed <test_id>/<step>/<node>/<check>. A listed check that matches fails,
+// so a fix must delete its entry.
+var pendingLoginSkipReasons = map[string]string{
+	"pending-login/PL-E2-expire-restart/pl-laptop/exchange-0": expiredRestartUnauthorized,
+	"pending-login/PL-E3-restart/pl-laptop/exchange-1":        expiredRestartUnauthorized,
+}
+
+const expiredRestartUnauthorized = "EXPIRED_RESTART_UNAUTHORIZED: headscale answers an expired node's restart with MachineAuthorized false; SaaS answers true, and both say the node key expired"
+
+// pendingLoginNotCompared lists, by key prefix, checks left out: SaaS's
+// answer there is in doubt, so it is no reference.
+var pendingLoginNotCompared = []string{
+	"pending-login/PL-E4-restore/pl-laptop/exchange-0",
+	"pending-login/PL-E4-restore/pl-laptop/devices",
+}
+
+// TestPendingLoginCompat replays a fresh machine's interactive login across
+// a restart and a stop, and an expired device restarted twice while its
+// login waits, then restored. Each exchange the client logged is sent as the
+// client sent it: same machine and node keys, follow-ups to the URL the
+// server gave. It compares each answer's flags, whether an auth URL is new or
+// repeats an earlier one, and the devices on each machine with their node
+// key.
+func TestPendingLoginCompat(t *testing.T) {
+	t.Parallel()
+
+	runPendingLoginCompat(t, "pending-login")
+}
+
+func runPendingLoginCompat(t *testing.T, id string) {
+	t.Helper()
+
+	c, err := testcapture.Read(filepath.Join("servertest", "testdata", "steps", id+".hujson"))
+	require.NoError(t, err)
+	require.NotEmpty(t, c.Steps, "%s: not a step capture", id)
+
+	r := newPendingReplay(t, c)
+
+	for _, step := range c.Steps {
+		require.Equal(t, "observed", step.Result, "%s: the replay follows every step", step.Name)
+
+		r.step(t, step)
+	}
+
+	for k := range pendingLoginSkipReasons {
+		if strings.HasPrefix(k, id+"/") && !r.evaluated[k] {
+			t.Errorf("skip entry %q names no check", k)
+		}
+	}
+}
+
+// pendingReplay drives the register handler as the capture's clients did.
+// SaaS labels for node keys and auth URLs map to the keys the replay makes
+// and the auth IDs headscale returns.
+type pendingReplay struct {
+	c     *testcapture.Capture
+	app   *Headscale
+	users map[string]*types.User
+	owner *types.User
+
+	machines  map[string]key.MachinePrivate
+	nodeKeys  map[string]key.NodePrivate
+	keyLabels map[key.NodePublic]string
+	authKeys  map[string]string
+
+	// urls holds each auth URL in the order it was given: SaaS's label
+	// and headscale's auth ID.
+	urls []pendingURL
+
+	// held is the auth key each client still holds in memory, lastURL the
+	// auth URL label it was last given, waiting its follow-up in flight.
+	held     map[string]string
+	hostname map[string]string
+	lastURL  map[string]types.AuthID
+	waiting  map[string]*pendingFollowup
+
+	evaluated map[string]bool
+}
+
+type pendingURL struct {
+	label  string
+	authID types.AuthID
+}
+
+// pendingFollowup is a follow-up parked in the register handler.
+type pendingFollowup struct {
+	nodeKey string
+	cancel  context.CancelFunc
+	done    chan struct{}
+	answer  string
+}
+
+func newPendingReplay(t *testing.T, c *testcapture.Capture) *pendingReplay {
+	t.Helper()
+
+	r := &pendingReplay{
+		c:         c,
+		app:       createTestApp(t),
+		users:     map[string]*types.User{},
+		machines:  map[string]key.MachinePrivate{},
+		nodeKeys:  map[string]key.NodePrivate{},
+		keyLabels: map[key.NodePublic]string{},
+		authKeys:  map[string]string{},
+		held:      map[string]string{},
+		hostname:  map[string]string{},
+		lastURL:   map[string]types.AuthID{},
+		waiting:   map[string]*pendingFollowup{},
+		evaluated: map[string]bool{},
+	}
+
+	for _, u := range c.Topology.Users {
+		r.users[u.Name] = r.app.state.CreateUserForTest(u.Name)
+		if u.Email == c.Tailnet {
+			r.owner = r.users[u.Name]
+		}
+	}
+
+	require.NotNil(t, r.owner, "%s: tailnet %q is not a capture user", c.TestID, c.Tailnet)
+
+	t.Cleanup(func() {
+		for _, f := range r.waiting {
+			f.cancel()
+			<-f.done
+		}
+	})
+
+	return r
+}
+
+// pendingCheck is one rendering of an outcome: SaaS's, and how to read
+// headscale's.
+type pendingCheck struct {
+	name string
+	want string
+	got  func() string
+}
+
+// step applies the step's actions, then sends its register exchanges, then
+// compares the outcome.
+func (r *pendingReplay) step(t *testing.T, step testcapture.Step) {
+	t.Helper()
+
+	r.apply(t, step)
+
+	var checks []pendingCheck
+
+	for _, node := range slices.Sorted(maps.Keys(step.Register)) {
+		for i, a := range step.Register[node] {
+			name := fmt.Sprintf("%s/exchange-%d", node, i)
+
+			switch {
+			case strings.Contains(a.Err, "context canceled"):
+				// The client gave up on a request when it restarted.
+				continue
+
+			case a.Err != "":
+				t.Fatalf("%s: %s: no replay for a refusal: %s", step.Name, name, a.Err)
+
+			case a.Followup && a.OffsetMS < 0:
+				// Sent in an earlier step and answered in this one.
+				f := r.waiting[node]
+				require.NotNil(t, f, "%s: %s answers no follow-up", step.Name, name)
+				require.Equal(t, a.NodeKey, f.nodeKey, "%s: %s", step.Name, name)
+
+				checks = append(checks, pendingCheck{name, registerFlags(a), f.state})
+
+			case a.Followup:
+				f := r.followup(t, step, node, a)
+				checks = append(checks, pendingCheck{name, registerFlags(a), f.state})
+
+			default:
+				got := r.register(t, step, node, a)
+				checks = append(checks, pendingCheck{name, registerFlags(a), func() string { return got }})
+
+				if a.AuthURL != "" {
+					checks = append(checks, r.authURLCheck(name))
+				}
+			}
+		}
+	}
+
+	checks = append(checks, r.deviceChecks(t, step)...)
+
+	r.hold(t, r.c.TestID+"/"+step.Name+"/", checks)
+}
+
+// apply runs the step's actions.
+func (r *pendingReplay) apply(t *testing.T, step testcapture.Step) {
+	t.Helper()
+
+	for _, a := range step.Actions {
+		switch a.Kind {
+		case "join", "relogin":
+			r.held[a.Node] = a.Key
+			r.hostname[a.Node] = a.Hostname
+
+		case "restart", "kill":
+			// A new process: the follow-up dies and the key held in memory
+			// is gone.
+			r.stopWaiting(a.Node)
+			r.held[a.Node] = ""
+
+		case "retire":
+			r.stopWaiting(a.Node)
+
+			for _, nv := range r.nodes(a.Node) {
+				changes, err := r.app.state.DeleteNode(nv)
+				require.NoError(t, err, "%s: retire %s", step.Name, a.Node)
+				r.app.Change(changes...)
+			}
+
+		case "wait":
+
+		case "api":
+			r.api(t, step, a)
+
+		default:
+			t.Fatalf("%s: no replay for action %q", step.Name, a.Kind)
+		}
+	}
+}
+
+// api replays one admin exchange: key mints, expiry and the key expiry
+// setting. Reads are what the checks compare.
+func (r *pendingReplay) api(t *testing.T, step testcapture.Step, a testcapture.Action) {
+	t.Helper()
+
+	switch {
+	case a.Method == http.MethodGet:
+
+	case a.Method == http.MethodPost && a.Path == "/tailnet/-/keys":
+		var body struct {
+			Capabilities struct {
+				Devices struct {
+					Create struct {
+						Reusable  bool     `json:"reusable"`
+						Ephemeral bool     `json:"ephemeral"`
+						Tags      []string `json:"tags"`
+					} `json:"create"`
+				} `json:"devices"`
+			} `json:"capabilities"`
+			ExpirySeconds int `json:"expirySeconds"`
+		}
+		require.NoError(t, json.Unmarshal(a.Body, &body), "%s: key body", step.Name)
+
+		create := body.Capabilities.Devices.Create
+		expiry := time.Now().Add(time.Duration(body.ExpirySeconds) * time.Second)
+		uid := types.UserID(r.owner.ID)
+
+		pak, err := r.app.state.CreatePreAuthKey(&uid, create.Reusable, create.Ephemeral, &expiry, create.Tags)
+		require.NoError(t, err, "%s: mint %s", step.Name, a.Key)
+
+		r.authKeys[a.Key] = pak.Key
+
+	case a.Method == http.MethodPost && strings.HasSuffix(a.Path, "/expire"):
+		now := time.Now()
+		r.setExpiry(t, step, a, &now)
+
+	case a.Method == http.MethodPost && strings.HasSuffix(a.Path, "/key"):
+		var body struct {
+			KeyExpiryDisabled bool `json:"keyExpiryDisabled"`
+		}
+		require.NoError(t, json.Unmarshal(a.Body, &body), "%s: key body", step.Name)
+
+		// As the v2 API: only turning expiry off maps onto headscale.
+		if body.KeyExpiryDisabled {
+			r.setExpiry(t, step, a, nil)
+		}
+
+	default:
+		t.Fatalf("%s: no replay for %s %s", step.Name, a.Method, a.Path)
+	}
+}
+
+func (r *pendingReplay) setExpiry(t *testing.T, step testcapture.Step, a testcapture.Action, expiry *time.Time) {
+	t.Helper()
+
+	alias, _, _ := strings.Cut(strings.TrimPrefix(a.Path, "/device/{device:"), "}")
+	nodes := r.nodes(alias)
+	require.Len(t, nodes, 1, "%s: %s", step.Name, a.Path)
+
+	_, changes, err := r.app.state.SetNodeExpiry(nodes[0].ID(), expiry)
+	require.NoError(t, err, "%s: %s", step.Name, a.Path)
+	r.app.Change(changes)
+}
+
+// request builds the register request a client sent: its node keys, the
+// key it holds, and its hostname.
+func (r *pendingReplay) request(node string, a testcapture.RegisterAttempt) tailcfg.RegisterRequest {
+	req := tailcfg.RegisterRequest{
+		NodeKey:  r.nodeKey(a.NodeKey).Public(),
+		Hostinfo: &tailcfg.Hostinfo{Hostname: r.hostname[node]},
+	}
+
+	if a.OldNodeKey != "" {
+		req.OldNodeKey = r.nodeKey(a.OldNodeKey).Public()
+	}
+
+	if label := r.held[node]; label != "" {
+		req.Auth = &tailcfg.RegisterResponseAuth{AuthKey: r.authKeys[label]}
+	}
+
+	return req
+}
+
+// register sends one exchange and returns its answer's flags. An auth URL
+// is mapped to SaaS's label, so a follow-up can go to it.
+func (r *pendingReplay) register(t *testing.T, step testcapture.Step, node string, a testcapture.RegisterAttempt) string {
+	t.Helper()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	resp, err := r.app.handleRegister(ctx, r.request(node, a), r.machine(node).Public())
+
+	got := responseFlags(resp, err)
+
+	if a.AuthURL != "" && resp != nil && resp.AuthURL != "" {
+		authID, err := extractRegistrationIDFromAuthURL(resp.AuthURL)
+		require.NoError(t, err, "%s: %s's auth URL", step.Name, node)
+
+		r.urls = append(r.urls, pendingURL{label: a.AuthURL, authID: authID})
+		r.lastURL[node] = authID
+	}
+
+	return got
+}
+
+// followup parks a follow-up to the auth URL node was last given, as
+// tailscaled does while the login waits.
+func (r *pendingReplay) followup(t *testing.T, step testcapture.Step, node string, a testcapture.RegisterAttempt) *pendingFollowup {
+	t.Helper()
+
+	authID, ok := r.lastURL[node]
+	require.True(t, ok, "%s: %s follows up on no auth URL", step.Name, node)
+
+	r.stopWaiting(node)
+
+	req := r.request(node, a)
+	req.Followup = r.app.authProvider.RegisterURL(authID)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	f := &pendingFollowup{nodeKey: a.NodeKey, cancel: cancel, done: make(chan struct{})}
+	machine := r.machine(node).Public()
+
+	go func() {
+		defer close(f.done)
+
+		resp, err := r.app.handleRegister(ctx, req, machine)
+		if ctx.Err() == nil {
+			f.answer = responseFlags(resp, err)
+		}
+	}()
+
+	r.waiting[node] = f
+
+	return f
+}
+
+// state is the follow-up's answer, or "unanswered" while it waits.
+func (f *pendingFollowup) state() string {
+	select {
+	case <-f.done:
+		if f.answer == "" {
+			return "cancelled"
+		}
+
+		return f.answer
+	default:
+		return "unanswered"
+	}
+}
+
+func (r *pendingReplay) stopWaiting(node string) {
+	if f, ok := r.waiting[node]; ok {
+		f.cancel()
+		<-f.done
+		delete(r.waiting, node)
+	}
+}
+
+// authURLCheck says whether the last auth URL given is new or repeats an
+// earlier one, on SaaS by label and on headscale by auth ID.
+func (r *pendingReplay) authURLCheck(name string) pendingCheck {
+	last := r.urls[len(r.urls)-1]
+	want, got := "new", "new"
+
+	for _, earlier := range r.urls[:len(r.urls)-1] {
+		if earlier.label == last.label {
+			want = "same as " + earlier.label
+		}
+
+		if earlier.authID == last.authID {
+			got = "same as " + earlier.label
+		}
+	}
+
+	return pendingCheck{name + "-auth-url", want, func() string { return got }}
+}
+
+// deviceChecks compares the devices on each machine after the step and
+// their node keys, when SaaS read them.
+func (r *pendingReplay) deviceChecks(t *testing.T, step testcapture.Step) []pendingCheck {
+	t.Helper()
+
+	type device struct {
+		MachineKey string `json:"machineKey"`
+		NodeKey    string `json:"nodeKey"`
+	}
+
+	var (
+		devices []device
+		read    bool
+		single  = map[string]*device{}
+	)
+
+	for _, a := range step.Actions {
+		if a.Kind != "api" || a.Method != http.MethodGet {
+			continue
+		}
+
+		switch {
+		case strings.HasPrefix(a.Path, "/tailnet/-/devices"):
+			var out struct {
+				Devices []device `json:"devices"`
+			}
+			require.NoError(t, json.Unmarshal(a.Response, &out), "%s: %s", step.Name, a.Path)
+
+			devices, read = out.Devices, true
+
+		case strings.HasPrefix(a.Path, "/device/{device:"):
+			// The last read of each device.
+			alias, _, _ := strings.Cut(strings.TrimPrefix(a.Path, "/device/{device:"), "}")
+			single[alias] = nil
+
+			if a.Status == http.StatusOK {
+				var d device
+				require.NoError(t, json.Unmarshal(a.Response, &d), "%s: %s", step.Name, a.Path)
+
+				single[alias] = &d
+			}
+		}
+	}
+
+	if !read {
+		for _, d := range single {
+			if d != nil {
+				devices = append(devices, *d)
+			}
+		}
+	}
+
+	var aliases []string
+
+	for alias := range r.machines {
+		if _, ok := single[alias]; read || ok {
+			aliases = append(aliases, alias)
+		}
+	}
+
+	slices.Sort(aliases)
+
+	checks := make([]pendingCheck, 0, len(aliases))
+
+	for _, alias := range aliases {
+		var keys []string
+
+		for _, d := range devices {
+			if d.MachineKey == "{machine:"+alias+"}" {
+				keys = append(keys, d.NodeKey)
+			}
+		}
+
+		checks = append(checks, pendingCheck{alias + "/devices", renderNodeKeys(keys), func() string {
+			var got []string
+
+			for _, nv := range r.nodes(alias) {
+				label, ok := r.keyLabels[nv.NodeKey()]
+				if !ok {
+					label = "unknown node key"
+				}
+
+				got = append(got, label)
+			}
+
+			return renderNodeKeys(got)
+		}})
+	}
+
+	return checks
+}
+
+func renderNodeKeys(keys []string) string {
+	slices.Sort(keys)
+
+	return fmt.Sprintf("devices %d %v", len(keys), keys)
+}
+
+// hold polls the checks until, for several ticks in a row, each one listed
+// in pendingLoginSkipReasons differs from SaaS and every other one matches,
+// then reports each as a subtest. A listed check that matches fails.
+func (r *pendingReplay) hold(t *testing.T, prefix string, checks []pendingCheck) {
+	t.Helper()
+
+	checks = slices.DeleteFunc(checks, func(c pendingCheck) bool {
+		return notCompared(pendingLoginNotCompared, prefix+c.name)
+	})
+
+	const holdTicks = 5
+
+	got := make([]string, len(checks))
+	held := 0
+
+	tick := time.NewTicker(100 * time.Millisecond)
+	defer tick.Stop()
+
+	for deadline := time.Now().Add(10 * time.Second); held < holdTicks && time.Now().Before(deadline); <-tick.C {
+		settled := true
+
+		for i, c := range checks {
+			got[i] = c.got()
+			if _, listed := pendingLoginSkipReasons[prefix+c.name]; (got[i] == c.want) == listed {
+				settled = false
+			}
+		}
+
+		if settled {
+			held++
+		} else {
+			held = 0
+		}
+	}
+
+	for i, c := range checks {
+		k := prefix + c.name
+		r.evaluated[k] = true
+
+		t.Run(strings.TrimPrefix(k, r.c.TestID+"/"), func(t *testing.T) {
+			reason, listed := pendingLoginSkipReasons[k]
+
+			switch {
+			case listed && got[i] == c.want:
+				t.Errorf("matches SaaS; delete the skip entry %q (%s)", k, reason)
+			case listed:
+				t.Skip(reason)
+			default:
+				assert.Equal(t, c.want, got[i])
+			}
+		})
+	}
+}
+
+func (r *pendingReplay) machine(node string) key.MachinePrivate {
+	mk, ok := r.machines[node]
+	if !ok {
+		mk = key.NewMachine()
+		r.machines[node] = mk
+	}
+
+	return mk
+}
+
+func (r *pendingReplay) nodeKey(label string) key.NodePrivate {
+	nk, ok := r.nodeKeys[label]
+	if !ok {
+		nk = key.NewNode()
+		r.nodeKeys[label] = nk
+		r.keyLabels[nk.Public()] = label
+	}
+
+	return nk
+}
+
+func (r *pendingReplay) nodes(alias string) []types.NodeView {
+	mk, ok := r.machines[alias]
+	if !ok {
+		return nil
+	}
+
+	return slices.Collect(maps.Values(r.app.state.GetNodesByMachineKeyAllUsers(mk.Public())))
+}
+
+// registerFlags renders an answer as the client logged it: its
+// NodeKeyExpired and MachineAuthorized flags and whether it carried an auth
+// URL, or "unanswered" for a request still waiting.
+func registerFlags(a testcapture.RegisterAttempt) string {
+	if a.NodeKeyExpired == nil || a.MachineAuthorized == nil {
+		return "unanswered"
+	}
+
+	return fmt.Sprintf("expired=%t authorized=%t url=%t", *a.NodeKeyExpired, *a.MachineAuthorized, a.AuthURL != "")
+}
+
+// responseFlags renders headscale's answer the way [registerFlags] renders
+// SaaS's; a refusal is "error".
+func responseFlags(resp *tailcfg.RegisterResponse, err error) string {
+	if err != nil || resp == nil || resp.Error != "" {
+		return "error"
+	}
+
+	return fmt.Sprintf("expired=%t authorized=%t url=%t", resp.NodeKeyExpired, resp.MachineAuthorized, resp.AuthURL != "")
+}

hscontrol/servertest/expiry_compat_test.go

@@ -0,0 +1,690 @@
+// This file replays key expiry captures from Tailscale SaaS: a router whose
+// key expires beside a standby, a client's own expiry request and logout,
+// and an expired node that keeps its control session while its peers,
+// routes, DNS, policy and SSH change.
+//
+// Test data source: testdata/steps/expiry.hujson, loaded by name.
+// Source format:    github.com/juanfont/headscale/hscontrol/types/testcapture
+
+package servertest_test
+
+import (
+	"encoding/json"
+	"fmt"
+	"maps"
+	"net/http"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/juanfont/headscale/hscontrol/servertest"
+	"github.com/juanfont/headscale/hscontrol/types/testcapture"
+	"github.com/stretchr/testify/require"
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/netmap"
+)
+
+// expirySkipReasons lists checks where headscale differs from SaaS, keyed
+// <test_id>/<step>/<node>/<check>. A listed check that matches fails, so a
+// fix must delete its entry.
+var expirySkipReasons = map[string]string{
+	"expiry/C2-expire-r1/r1/online-observer": expiredPeerOffline,
+	"expiry/C2-expire-r1/r1/online-r2":       expiredPeerOffline,
+	"expiry/C2-expire-r1/r1/paths-observer":  expiredPeerPaths,
+	"expiry/C2-expire-r1/r1/paths-r2":        expiredPeerPaths,
+	"expiry/C2-expire-r1/r1/self":            selfExpiredFlags,
+
+	"expiry/C3-restore-r1/r1/routes-observer": restoredRouterTakesRoute,
+	"expiry/C3-restore-r1/r1/routes-r2":       restoredRouterTakesRoute,
+	"expiry/C3-restore-r1/r2/routes-observer": restoredRouterTakesRoute,
+	"expiry/C3-restore-r1/r2/routes-r1":       restoredRouterTakesRoute,
+
+	"expiry/C4-expire-r2/r2/online-observer": expiredPeerOffline,
+	"expiry/C4-expire-r2/r2/online-r1":       expiredPeerOffline,
+	"expiry/C4-expire-r2/r2/paths-observer":  expiredPeerPaths,
+	"expiry/C4-expire-r2/r2/paths-r1":        expiredPeerPaths,
+	"expiry/C4-expire-r2/r2/self":            selfExpiredFlags,
+
+	"expiry/D2-set-expire/subject/key-expiry": clientExpiryRefused,
+	"expiry/D2-set-expire/subject/outcome":    clientExpiryRefused,
+
+	"expiry/D3-past-expiry/subject/expired-observer": clientExpiryRefused,
+	"expiry/D3-past-expiry/subject/key-expiry":       clientExpiryRefused,
+	"expiry/D3-past-expiry/subject/paths-observer":   clientExpiryRefused,
+	"expiry/D3-past-expiry/subject/self":             clientExpiryRefused,
+
+	"expiry/D5-logout-tagged/tagged/key-expiry": taggedLogoutNotExpired,
+	"expiry/D5-logout-tagged/tagged/outcome":    taggedLogoutNotExpired,
+
+	"expiry/A2-expire/laptop/online-observer": expiredPeerOffline,
+	"expiry/A2-expire/laptop/online-peer1":    expiredPeerOffline,
+	"expiry/A2-expire/laptop/paths-observer":  expiredPeerPaths,
+	"expiry/A2-expire/laptop/paths-peer1":     expiredPeerPaths,
+	"expiry/A2-expire/laptop/self":            selfExpiredFlags,
+
+	"expiry/A3-peer-change/laptop/online-observer": expiredPeerOffline,
+	"expiry/A3-peer-change/laptop/online-peer1":    expiredPeerOffline,
+	"expiry/A3-peer-change/laptop/online-peer3":    expiredPeerOffline,
+	"expiry/A3-peer-change/laptop/paths-observer":  expiredPeerPaths,
+	"expiry/A3-peer-change/laptop/paths-peer1":     expiredPeerPaths,
+	"expiry/A3-peer-change/laptop/paths-peer3":     expiredPeerPaths,
+	"expiry/A3-peer-change/laptop/self":            selfExpiredFlags,
+
+	"expiry/A4-rename/laptop/online-observer": expiredPeerOffline,
+	"expiry/A4-rename/laptop/paths-observer":  expiredPeerPaths,
+	"expiry/A4-rename/laptop/self":            selfExpiredFlags,
+
+	"expiry/A5-advertise/laptop/online-observer": expiredPeerOffline,
+	"expiry/A5-advertise/laptop/paths-observer":  expiredPeerPaths,
+	"expiry/A5-advertise/laptop/routes-observer": expiredRouterUnserved,
+	"expiry/A5-advertise/laptop/self":            selfExpiredFlags,
+
+	"expiry/A7-dns/laptop/dns":             nameserversFromConfig,
+	"expiry/A7-dns/laptop/online-observer": expiredPeerOffline,
+	"expiry/A7-dns/laptop/paths-observer":  expiredPeerPaths,
+	"expiry/A7-dns/laptop/routes-observer": expiredRouterUnserved,
+	"expiry/A7-dns/laptop/self":            selfExpiredFlags,
+	"expiry/A7-dns/observer/dns":           nameserversFromConfig,
+
+	"expiry/A8-policy/laptop/online-observer": expiredPeerOffline,
+	"expiry/A8-policy/laptop/paths-observer":  expiredPeerPaths,
+	"expiry/A8-policy/laptop/routes-observer": expiredRouterUnserved,
+	"expiry/A8-policy/laptop/self":            selfExpiredFlags,
+
+	"expiry/A9-ssh/laptop/online-observer": expiredPeerOffline,
+	"expiry/A9-ssh/laptop/paths-observer":  expiredPeerPaths,
+	"expiry/A9-ssh/laptop/routes-observer": expiredRouterUnserved,
+	"expiry/A9-ssh/laptop/self":            selfExpiredFlags,
+
+	"expiry/A10-reconnect/laptop/online-observer": expiredPeerOffline,
+	"expiry/A10-reconnect/laptop/paths-observer":  expiredPeerPaths,
+	"expiry/A10-reconnect/laptop/routes-observer": expiredRouterUnserved,
+	"expiry/A10-reconnect/laptop/self":            selfExpiredFlags,
+
+	"expiry/A6a-expire-again/laptop/online-observer": expiredPeerOffline,
+	"expiry/A6a-expire-again/laptop/online-peer1":    expiredPeerOffline,
+	"expiry/A6a-expire-again/laptop/paths-observer":  expiredPeerPaths,
+	"expiry/A6a-expire-again/laptop/paths-peer1":     expiredPeerPaths,
+	"expiry/A6a-expire-again/laptop/routes-observer": expiredRouterUnserved,
+	"expiry/A6a-expire-again/laptop/routes-peer1":    expiredRouterUnserved,
+	"expiry/A6a-expire-again/laptop/self":            selfExpiredFlags,
+
+	"expiry/A6b-rotate-disco/laptop/online-observer": expiredPeerOffline,
+	"expiry/A6b-rotate-disco/laptop/online-peer1":    expiredPeerOffline,
+	"expiry/A6b-rotate-disco/laptop/online-peer3":    expiredPeerOffline,
+	"expiry/A6b-rotate-disco/laptop/paths-observer":  expiredPeerPaths,
+	"expiry/A6b-rotate-disco/laptop/paths-peer1":     expiredPeerPaths,
+	"expiry/A6b-rotate-disco/laptop/paths-peer3":     expiredPeerPaths,
+	"expiry/A6b-rotate-disco/laptop/routes-observer": expiredRouterUnserved,
+	"expiry/A6b-rotate-disco/laptop/routes-peer1":    expiredRouterUnserved,
+	"expiry/A6b-rotate-disco/laptop/routes-peer3":    expiredRouterUnserved,
+	"expiry/A6b-rotate-disco/laptop/self":            selfExpiredFlags,
+}
+
+const (
+	selfExpiredFlags         = "SELF_EXPIRED_FLAGS: headscale sends an expired node its own entry with MachineAuthorized false and Expired set; SaaS keeps MachineAuthorized true and leaves Expired out"
+	expiredPeerPaths         = "EXPIRED_PEER_PATHS: headscale sends peers an expired node's endpoints and home DERP; SaaS leaves both out"
+	clientExpiryRefused      = "CLIENT_EXPIRY_REFUSED: headscale refuses a client's request for an earlier key expiry; SaaS stores it, and the node expires then"
+	nameserversFromConfig    = "NAMESERVERS_FROM_CONFIG: headscale reads nameservers from its config and cannot change them while running; SaaS pushes a change to every node, an expired one too"
+	restoredRouterTakesRoute = "RESTORED_ROUTER_TAKES_ROUTE: SaaS hands a route back to a router whose key expiry is lifted, taking it from the standby that served it meanwhile; headscale keeps the standby"
+)
+
+// TestNodeExpiryCompat replays key expiry against SaaS: an expired router
+// beside a standby, restored, then the standby expired; a client asking for
+// an earlier expiry, waiting past it and logging out, and a tagged node
+// logging out; an expired node whose peers rename and join, which renames
+// itself, advertises a route, and keeps its stream while DNS, the policy and
+// SSH change, reconnects, is restored, expired again and rotates its disco
+// key. Each step compares every captured node's own view, every captured
+// node's view of the others, its register answers, the routes API and the
+// device's key expiry.
+func TestNodeExpiryCompat(t *testing.T) {
+	t.Parallel()
+
+	t.Run("expiry", func(t *testing.T) {
+		t.Parallel()
+
+		runNodeExpiryCompat(t, "expiry", "observer")
+	})
+}
+
+func runNodeExpiryCompat(t *testing.T, id, observer string) {
+	t.Helper()
+
+	c := readStepCapture(t, id)
+
+	base, ok := c.Captures[observer]
+	require.True(t, ok && base.Netmap != nil, "%s: no baseline for %s", id, observer)
+
+	opts := []servertest.ServerOption{
+		servertest.WithNodeExpiry(captureKeyDuration(t, c)),
+		servertest.WithDNSResolvers(c.Input.Tailnet.DNS.Nameservers...),
+	}
+	if c.Input.Tailnet.DNS.MagicDNS && len(base.Netmap.DNS.Domains) > 0 {
+		opts = append(opts, servertest.WithMagicDNS(base.Netmap.DNS.Domains[0]))
+	}
+
+	r := newStepReplay(t, c, opts...)
+	r.paths = true
+
+	// A change is judged against the step before only, so both sides
+	// compare the same span.
+	prevCaps := c.Captures
+	saasPrev, hsPrev := map[string]deviceView{}, map[string]deviceView{}
+	evaluated := map[string]bool{}
+
+	for _, step := range c.Steps {
+		require.Equal(t, "observed", step.Result, "%s: the replay follows every step", step.Name)
+
+		start := map[string]*netmap.NetworkMap{}
+		historyStart, attempts := map[string]int{}, map[string]int{}
+
+		for alias, cl := range r.clients {
+			start[alias] = cl.Netmap()
+			historyStart[alias] = max(len(cl.History())-1, 0)
+			attempts[alias] = len(cl.RegisterAttempts())
+
+			if v := r.hsViews(alias); len(v) == 1 {
+				hsPrev[alias] = v[0]
+			}
+		}
+
+		r.apply(t, step)
+
+		e := &expiryStep{
+			r: r, c: c, step: step, at: stepCaptureTime(step),
+			prevCaps: prevCaps, start: start, historyStart: historyStart,
+		}
+
+		var checks []stepCheck
+
+		captured := slices.Sorted(maps.Keys(step.Captures))
+		for _, x := range captured {
+			if step.Captures[x].Netmap == nil || r.clients[x] == nil {
+				continue
+			}
+
+			checks = append(checks, e.selfChecks(x)...)
+
+			for _, v := range captured {
+				if v != x && step.Captures[v].Netmap != nil && r.clients[v] != nil {
+					checks = append(checks, e.viewChecks(x, v)...)
+				}
+			}
+		}
+
+		for _, node := range slices.Sorted(maps.Keys(step.Register)) {
+			if cl, ok := r.clients[node]; ok {
+				checks = append(checks, stepCheck{node + "/outcome", registerFlags(step.Register[node]), func() string {
+					return registerFlags(cl.RegisterAttempts()[attempts[node]:])
+				}})
+			}
+		}
+
+		checks = append(checks, e.apiChecks(t, saasPrev, hsPrev)...)
+
+		prefix := id + "/" + step.Name + "/"
+		for _, ch := range checks {
+			evaluated[prefix+ch.name] = true
+		}
+
+		if len(checks) > 0 {
+			t.Run(step.Name, func(t *testing.T) {
+				holdChecks(t, expirySkipReasons, prefix, checks, evaluated)
+			})
+		}
+
+		prevCaps = step.Captures
+	}
+
+	for k := range expirySkipReasons {
+		if strings.HasPrefix(k, id+"/") && !evaluated[k] {
+			t.Errorf("skip entry %q names no check", k)
+		}
+	}
+}
+
+// expiryStep is one replayed step and what both sides showed before it.
+type expiryStep struct {
+	r    *stepReplay
+	c    *testcapture.Capture
+	step testcapture.Step
+
+	// at is when SaaS's captures were taken.
+	at time.Time
+
+	// prevCaps holds SaaS's captures of the step before; start and
+	// historyStart headscale's netmaps when the step began.
+	prevCaps     map[string]testcapture.Node
+	start        map[string]*netmap.NetworkMap
+	historyStart map[string]int
+}
+
+// selfChecks compares x's own view: its key expiry flags and its peers among
+// the capture's nodes and, when the step before captured x too, whether it
+// lost any during the step and whether its DNS, packet filter and SSH policy
+// changed.
+func (e *expiryStep) selfChecks(x string) []stepCheck {
+	saas := e.step.Captures[x].Netmap
+	cl := e.r.clients[x]
+
+	checks := []stepCheck{
+		{x + "/self", renderSelfExpiry(saas, e.at), func() string { return renderSelfExpiry(cl.Netmap(), time.Now()) }},
+		{x + "/peers", e.saasPeers(saas), func() string { return e.hsPeers(cl.Netmap()) }},
+	}
+
+	prev := e.prevCaps[x].Netmap
+	if prev == nil || e.start[x] == nil {
+		return checks
+	}
+
+	if frames := e.step.Frames[x]; len(frames) > 0 {
+		checks = append(checks, stepCheck{x + "/removed", e.saasRemovedPeers(prev, frames), func() string {
+			return e.hsRemovedPeers(x)
+		}})
+	}
+
+	for _, part := range []struct {
+		name   string
+		render func(*netmap.NetworkMap) string
+	}{
+		{"dns", renderResolvers},
+		{"filter", renderFilterRules},
+		{"ssh", renderSSHPolicy},
+	} {
+		checks = append(checks, stepCheck{x + "/" + part.name, changed(part.render(prev), part.render(saas)), func() string {
+			return changed(part.render(e.start[x]), part.render(cl.Netmap()))
+		}})
+	}
+
+	return checks
+}
+
+// viewChecks compares how v sees x: expired, online, whether it has paths
+// to it, the routes it is served through it, its name and, when the step
+// before captured v too, whether its disco key changed.
+func (e *expiryStep) viewChecks(x, v string) []stepCheck {
+	saasView := e.step.Captures[v].Netmap
+	saasID := e.c.NodeIDs[x]
+	saasPeer, saasMissing := peerByID(saasView, saasID)
+
+	hsPeer := func() (tailcfg.NodeView, string) {
+		nodes := e.r.machineNodes(x)
+		if len(nodes) != 1 {
+			return tailcfg.NodeView{}, fmt.Sprintf("devices %d", len(nodes))
+		}
+
+		return peerByID(e.r.clients[v].Netmap(), nodes[0].ID().NodeID())
+	}
+
+	checks := make([]stepCheck, 0, 6)
+	checks = append(checks,
+		stepCheck{x + "/expired-" + v, peerExpired(saasPeer, saasMissing), func() string { return peerExpired(hsPeer()) }},
+		stepCheck{x + "/online-" + v, peerOnline(saasPeer, saasMissing), func() string { return peerOnline(hsPeer()) }},
+		stepCheck{x + "/paths-" + v, peerPaths(saasPeer, saasMissing), func() string { return peerPaths(hsPeer()) }},
+		stepCheck{x + "/routes-" + v, servedRoutes(saasPeer, saasMissing), func() string { return servedRoutes(hsPeer()) }},
+		stepCheck{x + "/name-" + v, peerName(saasPeer, saasMissing), func() string { return peerName(hsPeer()) }},
+	)
+
+	prev := e.prevCaps[v].Netmap
+	nodes := e.r.machineNodes(x)
+
+	if prev == nil || e.start[v] == nil || len(nodes) != 1 {
+		return checks
+	}
+
+	hsBefore := peerDisco(e.start[v], nodes[0].ID().NodeID())
+
+	return append(checks, stepCheck{x + "/disco-" + v, discoState(peerDisco(prev, saasID), peerDisco(saasView, saasID)), func() string {
+		p, missing := hsPeer()
+		if missing != "" {
+			return discoState(hsBefore, missing)
+		}
+
+		return discoState(hsBefore, p.DiscoKey().String())
+	}})
+}
+
+// apiChecks compares the step's last routes read and last device read of
+// each device: advertised and enabled routes, and the key expiry, which for
+// a future expiry also says whether the step moved it.
+func (e *expiryStep) apiChecks(t *testing.T, saasPrev, hsPrev map[string]deviceView) []stepCheck {
+	t.Helper()
+
+	routes := map[string]testcapture.Action{}
+	device := map[string]testcapture.Action{}
+
+	for _, a := range e.step.Actions {
+		if a.Kind != "api" || a.Method != http.MethodGet || !strings.HasPrefix(a.Path, "/device/{device:") {
+			continue
+		}
+
+		alias := deviceAlias(t, a.Path)
+
+		switch {
+		case strings.HasSuffix(a.Path, "/routes"):
+			routes[alias] = a
+		case strings.HasSuffix(a.Path, "?fields=all"):
+			device[alias] = a
+		}
+	}
+
+	var checks []stepCheck
+
+	for _, alias := range slices.Sorted(maps.Keys(routes)) {
+		a := routes[alias]
+		require.Equal(t, http.StatusOK, a.Status, "%s: %s", e.step.Name, a.Path)
+
+		var want deviceRoutes
+		require.NoError(t, json.Unmarshal(a.Response, &want), "%s: %s", e.step.Name, a.Path)
+
+		got := func() (deviceRoutes, bool) {
+			path, ok := e.r.v2Path(a.Path)
+			if !ok {
+				return deviceRoutes{}, false
+			}
+
+			status, body := e.r.call(t, http.MethodGet, path, nil)
+
+			var out deviceRoutes
+
+			return out, status == http.StatusOK && json.Unmarshal(body, &out) == nil
+		}
+
+		checks = append(checks,
+			stepCheck{alias + "/advertised", sortedStrings(want.Advertised), func() string {
+				out, ok := got()
+				if !ok {
+					return "no device"
+				}
+
+				return sortedStrings(out.Advertised)
+			}},
+			stepCheck{alias + "/enabled", sortedStrings(want.Enabled), func() string {
+				out, ok := got()
+				if !ok {
+					return "no device"
+				}
+
+				return sortedStrings(out.Enabled)
+			}},
+		)
+	}
+
+	for _, alias := range slices.Sorted(maps.Keys(device)) {
+		a := device[alias]
+		require.Equal(t, http.StatusOK, a.Status, "%s: %s", e.step.Name, a.Path)
+
+		var d saasDevice
+		require.NoError(t, json.Unmarshal(a.Response, &d), "%s: %s", e.step.Name, a.Path)
+
+		want := saasDeviceView(e.c, d, e.step.At.Add(time.Duration(a.OffsetMS)*time.Millisecond))
+		prevWant, hadWant := saasPrev[alias]
+		prevGot, hadGot := hsPrev[alias]
+		saasPrev[alias] = want
+
+		checks = append(checks, stepCheck{alias + "/key-expiry", renderKeyExpiry(want, prevWant, hadWant), func() string {
+			views := e.r.hsViews(alias)
+			if len(views) != 1 {
+				return fmt.Sprintf("devices %d", len(views))
+			}
+
+			return renderKeyExpiry(views[0], prevGot, hadGot)
+		}})
+	}
+
+	return checks
+}
+
+// saasPeers names the capture's nodes among SaaS's peers in nm.
+func (e *expiryStep) saasPeers(nm *netmap.NetworkMap) string {
+	names := map[tailcfg.NodeID]string{}
+	for alias, id := range e.c.NodeIDs {
+		names[id] = alias
+	}
+
+	var out []string
+
+	for _, p := range nm.Peers {
+		if alias, ok := names[p.ID()]; ok {
+			out = append(out, alias)
+		}
+	}
+
+	return sortedStrings(out)
+}
+
+// hsPeers names the replay's nodes among headscale's peers in nm.
+func (e *expiryStep) hsPeers(nm *netmap.NetworkMap) string {
+	if nm == nil {
+		return "no netmap"
+	}
+
+	var out []string
+
+	for _, p := range nm.Peers {
+		if alias, ok := e.hsAlias(p.ID()); ok {
+			out = append(out, alias)
+		}
+	}
+
+	return sortedStrings(out)
+}
+
+// hsAlias is the alias whose machine holds node id.
+func (e *expiryStep) hsAlias(id tailcfg.NodeID) (string, bool) {
+	for alias := range e.r.clients {
+		for _, nv := range e.r.machineNodes(alias) {
+			if nv.ID().NodeID() == id {
+				return alias, true
+			}
+		}
+	}
+
+	return "", false
+}
+
+// saasRemovedPeers lists the capture's nodes x had as peers before the step
+// and lost during it, or "none".
+func (e *expiryStep) saasRemovedPeers(prev *netmap.NetworkMap, frames []testcapture.Frame) string {
+	var out []string
+
+	for alias, id := range e.c.NodeIDs {
+		if _, missing := peerByID(prev, id); missing == "" && saasRemoved(prev, frames, id) == "removed" {
+			out = append(out, alias)
+		}
+	}
+
+	return removedList(out)
+}
+
+func (e *expiryStep) hsRemovedPeers(x string) string {
+	cl := e.r.clients[x]
+	history := cl.History()[e.historyStart[x]:]
+
+	var out []string
+
+	if before := e.start[x]; before != nil {
+		for _, p := range before.Peers {
+			if alias, ok := e.hsAlias(p.ID()); ok && hsRemoved(history, p.ID()) == "removed" {
+				out = append(out, alias)
+			}
+		}
+	}
+
+	return removedList(out)
+}
+
+func removedList(aliases []string) string {
+	if len(aliases) == 0 {
+		return "none"
+	}
+
+	return sortedStrings(aliases)
+}
+
+// stepCaptureTime is when the probe took the step's captures: after its last
+// action, a wait counted to its end.
+func stepCaptureTime(step testcapture.Step) time.Time {
+	end := step.At
+
+	for _, a := range step.Actions {
+		at := step.At.Add(time.Duration(a.OffsetMS) * time.Millisecond)
+
+		if a.Kind == "wait" && len(a.Args) == 1 {
+			d, err := time.ParseDuration(a.Args[0])
+			if err == nil {
+				at = at.Add(d)
+			}
+		}
+
+		if at.After(end) {
+			end = at
+		}
+	}
+
+	return end
+}
+
+// captureKeyDuration is the tailnet's key lifetime, read off the first
+// device that holds its key expiry.
+func captureKeyDuration(t *testing.T, c *testcapture.Capture) time.Duration {
+	t.Helper()
+
+	for _, step := range c.Steps {
+		for _, a := range step.Actions {
+			if a.Kind != "api" || a.Method != http.MethodGet || !strings.HasSuffix(a.Path, "?fields=all") || a.Status != http.StatusOK {
+				continue
+			}
+
+			var d struct {
+				Created           time.Time `json:"created"`
+				Expires           time.Time `json:"expires"`
+				KeyExpiryDisabled bool      `json:"keyExpiryDisabled"`
+				Tags              []string  `json:"tags"`
+			}
+			if json.Unmarshal(a.Response, &d) == nil && !d.KeyExpiryDisabled && len(d.Tags) == 0 {
+				return d.Expires.Sub(d.Created)
+			}
+		}
+	}
+
+	t.Fatalf("%s: no device shows the key lifetime", c.TestID)
+
+	return 0
+}
+
+// renderSelfExpiry gives a node's own entry: its key expiry relative to now,
+// and its MachineAuthorized and Expired flags.
+func renderSelfExpiry(nm *netmap.NetworkMap, now time.Time) string {
+	if nm == nil || !nm.SelfNode.Valid() {
+		return "no netmap"
+	}
+
+	self := nm.SelfNode
+
+	expiry := "zero"
+	if ke := self.KeyExpiry(); !ke.IsZero() {
+		expiry = "future"
+		if ke.Before(now) {
+			expiry = "past"
+		}
+	}
+
+	return fmt.Sprintf("key expiry %s, authorized %t, expired %t", expiry, self.MachineAuthorized(), self.Expired())
+}
+
+// renderKeyExpiry gives a device's key expiry class; a future expiry also
+// says whether it moved since prev.
+func renderKeyExpiry(v, prev deviceView, hadPrev bool) string {
+	switch {
+	case v.expiry != "future" || !hadPrev || prev.expiry != "future":
+		return v.expiry
+	case prev.expires == v.expires:
+		return "future, unchanged"
+	default:
+		return "future, moved"
+	}
+}
+
+func peerPaths(p tailcfg.NodeView, missing string) string {
+	if missing != "" {
+		return missing
+	}
+
+	return fmt.Sprintf("endpoints %t, derp %t", p.Endpoints().Len() > 0, p.HomeDERP() != 0)
+}
+
+// peerName is the first label of the peer's MagicDNS name.
+func peerName(p tailcfg.NodeView, missing string) string {
+	if missing != "" {
+		return missing
+	}
+
+	label, _, _ := strings.Cut(p.Name(), ".")
+
+	return label
+}
+
+// discoState says whether a peer's disco key was kept or rotated, or that
+// the peer appeared or is missing.
+func discoState(before, after string) string {
+	switch {
+	case !strings.HasPrefix(after, "discokey:"):
+		return after
+	case !strings.HasPrefix(before, "discokey:"):
+		return "appeared"
+	case before == after:
+		return "kept"
+	default:
+		return "rotated"
+	}
+}
+
+func changed(before, after string) string {
+	if before == after {
+		return "unchanged"
+	}
+
+	return "changed"
+}
+
+func renderResolvers(nm *netmap.NetworkMap) string {
+	if nm == nil {
+		return "no netmap"
+	}
+
+	out := make([]string, 0, len(nm.DNS.Resolvers))
+	for _, r := range nm.DNS.Resolvers {
+		out = append(out, r.Addr)
+	}
+
+	return sortedStrings(out)
+}
+
+func renderFilterRules(nm *netmap.NetworkMap) string {
+	if nm == nil {
+		return "no netmap"
+	}
+
+	b, err := json.Marshal(nm.PacketFilterRules)
+	if err != nil {
+		return err.Error()
+	}
+
+	return string(b)
+}
+
+func renderSSHPolicy(nm *netmap.NetworkMap) string {
+	if nm == nil {
+		return "no netmap"
+	}
+
+	b, err := json.Marshal(nm.SSHPolicy)
+	if err != nil {
+		return err.Error()
+	}
+
+	return string(b)
+}

hscontrol/servertest/lifecycle_compat_test.go

@@ -1,8 +1,10 @@
 // This file replays multi-step lifecycle captures from Tailscale SaaS and
 // compares each step's outcome with what SaaS showed.
 //
-// Test data source: testdata/steps/{route-approval,owner-routes}.hujson,
-// loaded by name. owner-routes is consumed by TestReauthCompat.
+// Test data source: testdata/steps/{route-approval,owner-routes,
+// ephemeral-lifecycle}.hujson, loaded by name. owner-routes is consumed by
+// TestReauthCompat; the refused logins of ephemeral-lifecycle are rows of
+// TestAuthKeyRefusalCompat.
 // Source format:    github.com/juanfont/headscale/hscontrol/types/testcapture
 
 package servertest_test
@@ -10,6 +12,7 @@ package servertest_test
 import (
 	"encoding/json"
 	"fmt"
+	"maps"
 	"net/http"
 	"slices"
 	"strconv"
@@ -17,6 +20,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/juanfont/headscale/hscontrol/servertest"
 	"github.com/juanfont/headscale/hscontrol/types"
 	"github.com/juanfont/headscale/hscontrol/types/testcapture"
 	"github.com/stretchr/testify/assert"
@@ -27,8 +31,8 @@ import (
 )
 
 // lifecycleSkipReasons lists checks where headscale differs from SaaS, keyed
-// <test_id>/<step>/<check>. A listed check that matches fails, so a fix must
-// delete its entry.
+// <test_id>/<step>[/<node or action index>]/<check>. A listed check that
+// matches fails, so a fix must delete its entry.
 var lifecycleSkipReasons = map[string]string{
 	"route-approval/b6/enabled":     approvalOnHostinfo,
 	"route-approval/b6/served":      approvalOnHostinfo,
@@ -46,6 +50,22 @@ var lifecycleSkipReasons = map[string]string{
 	"route-approval/b9/self-routes": approvalOnSameKeyReauth,
 
 	"route-approval/r1/served": loggedOutRouterUnserved,
+
+	"ephemeral-lifecycle/E1-logout/eph-logout/outcome":      logoutAnswerUnauthorized,
+	"ephemeral-lifecycle/E1-logout/eph-logout-user/outcome": logoutAnswerUnauthorized,
+
+	"ephemeral-lifecycle/E3-key-delete/2/keys":   deletedKeyListed,
+	"ephemeral-lifecycle/E3-key-delete/3/keys":   deletedKeyListed,
+	"ephemeral-lifecycle/E3-key-delete/4/status": deletedKeyDeleteNotFound,
+
+	"ephemeral-lifecycle/E4-reauth-other-key/eph-to-perm/ephemeral": ephemeralFollowsKey,
+	"ephemeral-lifecycle/E4-reauth-other-key/perm-to-eph/ephemeral": ephemeralFollowsKey,
+	"ephemeral-lifecycle/E6-poll-14/eph-to-perm/left":               ephemeralFollowsKey,
+	"ephemeral-lifecycle/E6-poll-14/perm-to-eph/left":               ephemeralFollowsKey,
+
+	"ephemeral-lifecycle/E5-mem-shutdown/mem/expiry":        taggedLogoutNotExpired,
+	"ephemeral-lifecycle/E5-mem-shutdown/mem/outcome":       taggedLogoutNotExpired,
+	"ephemeral-lifecycle/E5-mem-shutdown/mem/peer-observer": taggedLogoutNotExpired,
 }
 
 // SaaS runs auto-approval when a router's announced routes change and on a
@@ -57,6 +77,13 @@ const (
 	loggedOutRouterUnserved = "LOGGED_OUT_ROUTER_UNSERVED: headscale stops serving a logged-out router's routes; SaaS keeps them on the expired, offline peer"
 )
 
+const (
+	logoutAnswerUnauthorized = "LOGOUT_ANSWER_UNAUTHORIZED: headscale answers the logout that deletes an ephemeral node with MachineAuthorized false; SaaS answers true"
+	deletedKeyListed         = "DELETED_KEY_LISTED: headscale lists a deleted auth key; SaaS leaves it out of both key lists, while a read by ID still answers"
+	deletedKeyDeleteNotFound = "DELETED_KEY_DELETE_NOT_FOUND: headscale answers 404 to a second delete of an auth key; SaaS answers 200"
+	ephemeralFollowsKey      = "EPHEMERAL_FOLLOWS_KEY: headscale takes a node's ephemeral flag from the key it last logged in with; SaaS keeps the flag the device was created with, so it is collected, or kept, as created"
+)
+
 // Used by reauthSkipReasons.
 const (
 	expiredRouterUnserved = "EXPIRED_ROUTER_UNSERVED: headscale stops serving an expired router's routes; SaaS keeps them in the peer's AllowedIPs and PrimaryRoutes"
@@ -544,3 +571,425 @@ func hsRemoved(history []*netmap.NetworkMap, id tailcfg.NodeID) string {
 
 	return "kept"
 }
+
+// TestEphemeralLifecycleCompat replays ephemeral devices through logout, a
+// rejoin, a device deleted twice, a deleted key and a restart after it,
+// re-auth with a key of the other kind, a shutdown with in-memory state and
+// a kill of several devices. Each step compares with SaaS: the status of
+// every admin exchange, key validity and listing, register answers, the
+// devices on each machine, their ephemeral flag, node key, address and
+// expiry, the observer's view and removals, and at the last read which
+// devices are left once the killed ones could be collected.
+func TestEphemeralLifecycleCompat(t *testing.T) {
+	t.Parallel()
+
+	t.Run("ephemeral-lifecycle", func(t *testing.T) {
+		t.Parallel()
+
+		runEphemeralLifecycleCompat(t, "ephemeral-lifecycle", "observer")
+	})
+}
+
+func runEphemeralLifecycleCompat(t *testing.T, id, observer string) {
+	t.Helper()
+
+	c := readStepCapture(t, id)
+
+	// Short, so the killed devices can be collected while the last read is
+	// checked; collection timing is not compared.
+	r := newStepReplay(t, c, servertest.WithEphemeralTimeout(2*time.Second))
+	r.readAPI = true
+
+	lastRead := -1
+
+	for i, step := range c.Steps {
+		if _, _, ok := stepDeviceRead(step); ok {
+			lastRead = i
+		}
+	}
+
+	saasFirst, hsFirst := map[string]deviceView{}, map[string]deviceView{}
+	saasPrev := map[string][]deviceView{}
+	down := map[string]bool{}
+	evaluated := map[string]bool{}
+	prevObs := c.Captures[observer].Netmap
+
+	for i, step := range c.Steps {
+		require.Contains(t, []string{"observed", "expected-refusal"}, step.Result,
+			"%s: the replay follows every step", step.Name)
+
+		cells := lifecycleCells(step)
+
+		hsPrev := map[string][]deviceView{}
+		attempts := map[string]int{}
+
+		for _, cell := range cells {
+			hsPrev[cell] = r.hsViews(cell)
+			if cl, ok := r.clients[cell]; ok {
+				attempts[cell] = len(cl.RegisterAttempts())
+			}
+		}
+
+		historyStart := 0
+		if obs, ok := r.clients[observer]; ok {
+			historyStart = max(len(obs.History())-1, 0)
+		}
+
+		r.apply(t, step)
+		markDown(step, down)
+
+		checks := r.answerChecks(step)
+		checks = append(checks, r.keyChecks(step)...)
+		checks = append(checks, r.keyListChecks(t, step)...)
+
+		devices, readAt, listed := stepDeviceRead(step)
+
+		r.firstDevices(c, step, saasFirst, hsFirst)
+
+		for _, cell := range cells {
+			// A device killed in the step is judged at the last read only:
+			// when it is collected is timing.
+			want, ok := saasCellDevices(c, step, cell)
+			if down[cell] || !ok {
+				continue
+			}
+
+			checks = append(checks, ephemeralCellChecks(r, step, cell, observer, cellState{
+				want: want, saasPrev: saasPrev[cell], hsPrev: hsPrev[cell],
+				saasFirst: saasFirst[cell], hsFirst: hsFirst[cell],
+				attempts: attempts[cell], prevObs: prevObs, historyStart: historyStart,
+				saasAll: devices, listed: listed,
+			})...)
+		}
+
+		if i == lastRead {
+			checks = append(checks, r.leftChecks(c, step)...)
+		}
+
+		prefix := id + "/" + step.Name + "/"
+		for _, ch := range checks {
+			evaluated[prefix+ch.name] = true
+		}
+
+		if len(checks) > 0 {
+			t.Run(step.Name, func(t *testing.T) {
+				holdChecks(t, lifecycleSkipReasons, prefix, checks, evaluated)
+			})
+		}
+
+		// A device list shows every machine; a device read only its own.
+		if listed {
+			for alias := range r.clients {
+				saasPrev[alias] = saasDeviceViews(c, devices, alias, readAt)
+			}
+		}
+
+		for _, cell := range cells {
+			if want, ok := saasCellDevices(c, step, cell); ok {
+				saasPrev[cell] = want
+			}
+		}
+
+		if obs, ok := step.Captures[observer]; ok && obs.Netmap != nil {
+			prevObs = obs.Netmap
+		}
+	}
+
+	for k := range lifecycleSkipReasons {
+		if strings.HasPrefix(k, id+"/") && !evaluated[k] {
+			t.Errorf("skip entry %q names no check", k)
+		}
+	}
+}
+
+// markDown records the nodes a step killed, and those it brought back.
+func markDown(step testcapture.Step, down map[string]bool) {
+	for _, a := range step.Actions {
+		switch a.Kind {
+		case "kill":
+			down[a.Node] = true
+		case "start", "join", "relogin", "restart":
+			down[a.Node] = false
+		}
+	}
+}
+
+// firstDevices records each machine's device as it first joined: on SaaS
+// from the first device list that shows it.
+func (r *stepReplay) firstDevices(c *testcapture.Capture, step testcapture.Step, saasFirst, hsFirst map[string]deviceView) {
+	devices, at, listed := stepDeviceRead(step)
+
+	for alias := range r.clients {
+		if _, seen := saasFirst[alias]; !seen && listed {
+			if views := saasDeviceViews(c, devices, alias, at); len(views) > 0 {
+				saasFirst[alias] = views[0]
+			}
+		}
+
+		if _, seen := hsFirst[alias]; !seen {
+			if views := r.hsViews(alias); len(views) > 0 {
+				hsFirst[alias] = views[0]
+			}
+		}
+	}
+}
+
+// leftChecks compares how many devices each machine holds at the last read.
+// A device the polls still saw when they gave up is not observed, not kept.
+func (r *stepReplay) leftChecks(c *testcapture.Capture, step testcapture.Step) []stepCheck {
+	devices, at, _ := stepDeviceRead(step)
+
+	_, unseen, _ := strings.Cut(step.Note, "not observed: ")
+	notObserved := strings.Split(unseen, ", ")
+
+	checks := make([]stepCheck, 0, len(r.clients))
+
+	for _, alias := range slices.Sorted(maps.Keys(r.clients)) {
+		if slices.Contains(notObserved, alias) {
+			continue
+		}
+
+		checks = append(checks, stepCheck{alias + "/left", strconv.Itoa(len(saasDeviceViews(c, devices, alias, at))), func() string {
+			return strconv.Itoa(len(r.machineNodes(alias)))
+		}})
+	}
+
+	return checks
+}
+
+// cellState is what a cell's checks compare against: SaaS's devices for the
+// cell in the step and both sides' devices before it.
+type cellState struct {
+	want               []deviceView
+	saasPrev, hsPrev   []deviceView
+	saasFirst, hsFirst deviceView
+	attempts           int
+	prevObs            *netmap.NetworkMap
+	historyStart       int
+	saasAll            []saasDevice
+	listed             bool
+}
+
+// ephemeralCellChecks compares one cell after a step: its devices, their
+// ephemeral flag, node key, address and expiry, its register answers, and the
+// observer's view of it and whether the observer lost it.
+func ephemeralCellChecks(r *stepReplay, step testcapture.Step, cell, observer string, st cellState) []stepCheck {
+	got := func() []deviceView { return r.hsViews(cell) }
+	byID := func(views []deviceView) map[string]deviceView {
+		out := make(map[string]deviceView, len(views))
+		for _, v := range views {
+			out[v.id] = v
+		}
+
+		return out
+	}
+
+	checks := []stepCheck{
+		{cell + "/devices", renderDevices(st.want, st.saasFirst), func() string { return renderDevices(got(), st.hsFirst) }},
+		{cell + "/ephemeral", renderEphemeral(st.want), func() string { return renderEphemeral(got()) }},
+		{cell + "/expiry", renderExpiry(st.want, byID(st.saasPrev)), func() string { return renderExpiry(got(), byID(st.hsPrev)) }},
+	}
+
+	if len(st.saasPrev) > 0 {
+		checks = append(checks, stepCheck{cell + "/node-key", renderNodeKeys(st.want, byID(st.saasPrev)), func() string {
+			return renderNodeKeys(got(), byID(st.hsPrev))
+		}})
+	}
+
+	if st.listed {
+		saasAddrs := map[string]int{}
+		for _, d := range st.saasAll {
+			saasAddrs[sortedStrings(d.Addresses)]++
+		}
+
+		checks = append(checks, stepCheck{cell + "/address", renderAddresses(st.want, st.saasFirst, saasAddrs), func() string {
+			hsAddrs := map[string]int{}
+			for _, nv := range r.srv.State().ListNodes().All() {
+				hsAddrs[sortedStrings(nv.IPsAsString())]++
+			}
+
+			return renderAddresses(got(), st.hsFirst, hsAddrs)
+		}})
+	}
+
+	if saas := step.Register[cell]; len(saas) > 0 {
+		checks = append(checks, stepCheck{cell + "/outcome", registerFlags(saas), func() string {
+			return registerFlags(r.clients[cell].RegisterAttempts()[st.attempts:])
+		}})
+	}
+
+	if view, ok := step.Captures[observer]; ok && cell != observer {
+		checks = append(checks, stepCheck{cell + "/peer-" + observer, renderPeers(st.want, view.Netmap), func() string {
+			return renderPeers(got(), r.clients[observer].Netmap())
+		}})
+	}
+
+	if frames := step.Frames[observer]; len(frames) > 0 && cell != observer {
+		saas := make([]string, 0, len(st.saasPrev))
+		for _, v := range st.saasPrev {
+			saas = append(saas, saasRemoved(st.prevObs, frames, v.nodeID))
+		}
+
+		checks = append(checks, stepCheck{cell + "/removed-" + observer, strings.Join(saas, "; "), func() string {
+			history := r.clients[observer].History()[st.historyStart:]
+
+			hs := make([]string, 0, len(st.hsPrev))
+			for _, v := range st.hsPrev {
+				hs = append(hs, hsRemoved(history, v.nodeID))
+			}
+
+			return strings.Join(hs, "; ")
+		}})
+	}
+
+	return checks
+}
+
+// lifecycleCells returns the nodes a step's actions name, in order: the
+// node of a client action and the device of an API path.
+func lifecycleCells(step testcapture.Step) []string {
+	var cells []string
+
+	for _, a := range step.Actions {
+		node := a.Node
+		if rest, ok := strings.CutPrefix(a.Path, "/device/{device:"); ok {
+			node, _, _ = strings.Cut(rest, "}")
+		}
+
+		if node != "" && !slices.Contains(cells, node) {
+			cells = append(cells, node)
+		}
+	}
+
+	return cells
+}
+
+// saasCellDevices returns SaaS's devices on cell's machine after the step:
+// from its last device list, else from its last read of cell's device. It
+// reports false when the step read neither.
+func saasCellDevices(c *testcapture.Capture, step testcapture.Step, cell string) ([]deviceView, bool) {
+	if devices, at, ok := stepDeviceRead(step); ok {
+		return saasDeviceViews(c, devices, cell, at), true
+	}
+
+	for _, a := range slices.Backward(step.Actions) {
+		if a.Kind != "api" || a.Method != http.MethodGet || !strings.HasPrefix(a.Path, "/device/{device:"+cell+"}?") {
+			continue
+		}
+
+		if a.Status != http.StatusOK {
+			return nil, true
+		}
+
+		var d saasDevice
+		if json.Unmarshal(a.Response, &d) != nil {
+			return nil, false
+		}
+
+		return []deviceView{saasDeviceView(c, d, step.At.Add(time.Duration(a.OffsetMS)*time.Millisecond))}, true
+	}
+
+	return nil, false
+}
+
+// renderAddresses says, per device, whether it holds the first device's
+// addresses and whether no other device holds them.
+func renderAddresses(views []deviceView, first deviceView, holders map[string]int) string {
+	return renderEach(views, func(v deviceView) string {
+		return fmt.Sprintf("first's %t, unique %t", v.addrs == first.addrs, holders[v.addrs] == 1)
+	})
+}
+
+// registerFlags renders each register answer the client logged: its
+// NodeKeyExpired and MachineAuthorized flags and whether it carried an auth
+// URL, or "error" for a refusal and "unanswered" for a request still waiting.
+// Requests the client cancelled are left out.
+func registerFlags(attempts []testcapture.RegisterAttempt) string {
+	out := make([]string, 0, len(attempts))
+
+	for _, a := range attempts {
+		switch {
+		case strings.Contains(a.Err, "context canceled"):
+		case a.Err != "":
+			out = append(out, "error")
+		case a.MachineAuthorized == nil || a.NodeKeyExpired == nil:
+			out = append(out, "unanswered")
+		default:
+			out = append(out, fmt.Sprintf("expired=%t authorized=%t url=%t",
+				*a.NodeKeyExpired, *a.MachineAuthorized, a.AuthURL != ""))
+		}
+	}
+
+	return strings.Join(out, "; ")
+}
+
+// answerChecks compares the status of each admin exchange headscale
+// answered in the step, keyed by the action's index.
+func (r *stepReplay) answerChecks(step testcapture.Step) []stepCheck {
+	checks := make([]stepCheck, 0, len(r.answers))
+
+	for _, i := range slices.Sorted(maps.Keys(r.answers)) {
+		got := strconv.Itoa(r.answers[i].status)
+		checks = append(checks, stepCheck{strconv.Itoa(i) + "/status", strconv.Itoa(step.Actions[i].Status), func() string {
+			return got
+		}})
+	}
+
+	return checks
+}
+
+// keyListChecks compares which of the replay's keys each key listing in the
+// step names.
+func (r *stepReplay) keyListChecks(t *testing.T, step testcapture.Step) []stepCheck {
+	t.Helper()
+
+	labels := map[string]string{}
+	for label, id := range r.keyIDs {
+		labels[strconv.FormatUint(id, 10)] = label
+	}
+
+	listedIn := func(body []byte, label func(string) (string, bool)) string {
+		var out struct {
+			Keys []struct {
+				ID string `json:"id"`
+			} `json:"keys"`
+		}
+		require.NoError(t, json.Unmarshal(body, &out), "%s: key list", step.Name)
+
+		var got []string
+
+		for _, k := range out.Keys {
+			if l, ok := label(k.ID); ok {
+				got = append(got, l)
+			}
+		}
+
+		return sortedStrings(got)
+	}
+
+	var checks []stepCheck
+
+	for i, a := range step.Actions {
+		ans, ok := r.answers[i]
+		if !ok || a.Method != http.MethodGet || !strings.HasPrefix(a.Path, "/tailnet/-/keys") || strings.Contains(a.Path, "{key:") {
+			continue
+		}
+
+		want := listedIn(a.Response, func(id string) (string, bool) {
+			l, ok := strings.CutPrefix(id, "{key:")
+			l = strings.TrimSuffix(l, "}")
+			_, minted := r.keyIDs[l]
+
+			return l, ok && minted
+		})
+
+		got := ""
+		if ans.status == http.StatusOK {
+			got = listedIn(ans.body, func(id string) (string, bool) { l, ok := labels[id]; return l, ok })
+		}
+
+		checks = append(checks, stepCheck{strconv.Itoa(i) + "/keys", want, func() string { return got }})
+	}
+
+	return checks
+}

hscontrol/servertest/reauth_compat_test.go

@@ -3,7 +3,8 @@
 // Test data source: testdata/steps/{owner-routes,reauth,reauth-foreign,
 // reauth-owner,identity,reauth-user}.hujson, loaded by name. owner-routes is
 // checked by runRouteLifecycleCompat in lifecycle_compat_test.go. The tag
-// owner verdicts of tag-owner-chain.hujson are checked in ../policy/v2.
+// owner verdicts of tag-owner-chain.hujson are checked in ../policy/v2. The
+// refused logins of ephemeral-lifecycle.hujson are rows here too.
 // Source format:    github.com/juanfont/headscale/hscontrol/types/testcapture
 
 package servertest_test
@@ -81,6 +82,11 @@ var reauthSkipReasons = map[string]string{
 
 	"reauth/late-pak-spent-key/late-pak/text":        refusalText,
 	"reauth/late-pak-spent-key/late-pak/legacy-text": refusalText,
+
+	"ephemeral-lifecycle/late-revoked/late/text":            refusalText,
+	"ephemeral-lifecycle/late-revoked/late/legacy-text":     refusalText,
+	"ephemeral-lifecycle/late-expired/late-exp/text":        refusalText,
+	"ephemeral-lifecycle/late-expired/late-exp/legacy-text": refusalText,
 }
 
 const (
@@ -928,12 +934,15 @@ func (r *stepReplay) addrNames() map[netip.Addr]string {
 }
 
 // authKeyRefusalRows are fresh machines logging in with a key whose state
-// the capture shows: spent, and a valid control row.
+// the capture shows: spent, revoked by a delete, expired, and a valid
+// control row.
 var authKeyRefusalRows = []struct {
 	id, step, node string
 }{
 	{"reauth", "late-pak-spent-key", "late-pak"},
 	{"reauth", "t-valid-join", "t-valid"},
+	{"ephemeral-lifecycle", "late-revoked", "late"},
+	{"ephemeral-lifecycle", "late-expired", "late-exp"},
 }
 
 // refusalCheck reports whether a check name belongs to
@@ -991,6 +1000,9 @@ func runAuthKeyRefusalCompat(t *testing.T, id, stepName, node string, legacy boo
 
 	c := readStepCapture(t, id)
 	r := newStepReplay(t, c)
+	// Statuses are compared by the capture's own test; here a difference
+	// must not stop the replay.
+	r.readAPI = true
 
 	i := slices.IndexFunc(c.Steps, func(s testcapture.Step) bool { return s.Name == stepName })
 	require.GreaterOrEqual(t, i, 0, "%s: no step %s", id, stepName)

hscontrol/servertest/via_ha_compat_test.go

@@ -478,9 +478,9 @@ func readStepCapture(t *testing.T, id string) *testcapture.Capture {
 
 // stepReplay drives a step capture's actions against headscale: policy
 // pushes, key mints and revokes, joins, relogins, profile switches,
-// restarts, route announcements, logouts, retires and admin calls, the
-// last through the v2 API with the bodies SaaS received. Nothing is seeded
-// from the capture's end state.
+// restarts, kills, shutdowns, route announcements, logouts, retires and admin
+// calls, the last through the v2 API with the bodies SaaS received. Nothing
+// is seeded from the capture's end state.
 type stepReplay struct {
 	c        *testcapture.Capture
 	srv      *servertest.TestServer
@@ -500,6 +500,27 @@ type stepReplay struct {
 	// profiles holds the login a node switched away from.
 	profiles map[string]persist.Persist
 
+	// keyExpires holds each minted key's SaaS expiry until the replay has
+	// expired it; see [stepReplay.expireDueKeys].
+	keyExpires map[string]time.Time
+
+	// lastIDs holds each machine's last single node, so a deleted device can
+	// still be addressed.
+	lastIDs map[string]string
+
+	// memState lists the nodes whose tailscaled ran with --state=mem:.
+	memState map[string]bool
+
+	// paths, when set, gives each joining client a home DERP and an
+	// endpoint, as a running tailscaled reports them.
+	paths bool
+
+	// readAPI, when set, replays reads too and keeps headscale's answer to
+	// each admin exchange in answers, by action index, instead of failing on
+	// a status that differs from SaaS's.
+	readAPI bool
+	answers map[int]apiAnswer
+
 	// interactive, when set, picks the joins and logins replayed as a login
 	// approved by the key's user instead of with the key.
 	interactive func(step testcapture.Step, a testcapture.Action) bool
@@ -529,11 +550,28 @@ func newStepReplay(t *testing.T, c *testcapture.Capture, opts ...servertest.Serv
 		keyTags:  map[string][]string{},
 		keyUsers: map[string]*types.User{},
 		profiles: map[string]persist.Persist{},
+
+		keyExpires: map[string]time.Time{},
+		lastIDs:    map[string]string{},
+		memState:   memStateNodes[c.TestID],
 	}
 }
 
+// memStateNodes lists, per capture, the nodes the probe ran with
+// --state=mem:, which a join action does not record.
+var memStateNodes = map[string]map[string]bool{
+	"ephemeral-lifecycle": {"mem": true, "mem-kill": true},
+}
+
+// apiAnswer is headscale's answer to one replayed admin exchange.
+type apiAnswer struct {
+	status int
+	body   []byte
+}
+
 // apply runs the step's actions in order and returns the logins headscale
-// refused, by node. Reads are skipped: they are what consumers compare.
+// refused, by node. Reads are what consumers compare, so they are skipped
+// unless readAPI is set.
 func (r *stepReplay) apply(t *testing.T, step testcapture.Step) map[string]error {
 	t.Helper()
 
@@ -542,7 +580,13 @@ func (r *stepReplay) apply(t *testing.T, step testcapture.Step) map[string]error
 
 	refused := map[string]error{}
 
-	for _, a := range step.Actions {
+	r.expireDueKeys(t, step.At)
+
+	if r.readAPI {
+		r.answers = map[int]apiAnswer{}
+	}
+
+	for i, a := range step.Actions {
 		switch a.Kind {
 		case "policy":
 			require.Equal(t, http.StatusOK, a.Status, "%s: replay of a refused policy", step.Name)
@@ -556,7 +600,7 @@ func (r *stepReplay) apply(t *testing.T, step testcapture.Step) map[string]error
 			r.setPolicy(t, step.Name, policy)
 
 		case "api":
-			r.api(t, step.Name, a)
+			r.api(t, step.Name, i, a)
 
 		case "join":
 			hi := &tailcfg.Hostinfo{
@@ -566,7 +610,22 @@ func (r *stepReplay) apply(t *testing.T, step testcapture.Step) map[string]error
 			}
 			r.hostinfo[a.Node] = hi
 
-			opts := []servertest.ClientOption{servertest.WithUser(r.admin), servertest.WithHostinfo(hi.Clone())}
+			opts := []servertest.ClientOption{servertest.WithUser(r.admin)}
+			if r.memState[a.Node] {
+				opts = append(opts, servertest.WithLoginFlags(controlclient.LoginEphemeral))
+			}
+
+			if r.paths {
+				hi.NetInfo = &tailcfg.NetInfo{PreferredDERP: 1}
+
+				opts = append(opts, servertest.WithEndpoints(tailcfg.Endpoint{
+					Addr: netip.AddrPortFrom(netip.MustParseAddr("192.0.2.1"), uint16(41641+len(r.clients))), //nolint:gosec // a handful of clients
+					Type: tailcfg.EndpointExplicitConf,
+				}))
+			}
+
+			opts = append(opts, servertest.WithHostinfo(hi.Clone()))
+
 			if r.interactive != nil && r.interactive(step, a) {
 				opts = append(opts, servertest.WithInteractiveLogin(r.approver(ctx, t, a.Key)))
 			} else {
@@ -587,6 +646,8 @@ func (r *stepReplay) apply(t *testing.T, step testcapture.Step) map[string]error
 				r.announce(ctx, t, a.Node, a.Routes)
 			}
 
+			r.remember(a.Node)
+
 		case "relogin":
 			// The probe restates hostname and routes on every login.
 			hi := r.hostinfo[a.Node].Clone()
@@ -609,6 +670,8 @@ func (r *stepReplay) apply(t *testing.T, step testcapture.Step) map[string]error
 				refused[a.Node] = err
 			}
 
+			r.remember(a.Node)
+
 		case "switch_profile":
 			cl := r.clients[a.Node]
 
@@ -644,28 +707,9 @@ func (r *stepReplay) apply(t *testing.T, step testcapture.Step) map[string]error
 				refused[a.Node] = err
 			}
 
-		case "restart":
-			require.NoError(t, r.clients[a.Node].Restart(ctx, r.hostinfo[a.Node].Clone()), "%s: restart", step.Name)
-
-		case "advertise":
-			r.announce(ctx, t, a.Node, a.Routes)
-
-		case "cli":
-			args := strings.Join(a.Args, " ")
-			if args == "logout" {
-				require.NoError(t, r.clients[a.Node].LogoutAndDisconnect(ctx), "%s: logout", step.Name)
-
-				continue
-			}
-
-			hostname, ok := strings.CutPrefix(args, "set --hostname=")
-			require.True(t, ok, "%s: no replay for cli %q", step.Name, a.Args)
-
-			r.hostinfo[a.Node].Hostname = hostname
-			r.sendHostinfo(ctx, t, a.Node)
-
-		case "logout":
-			require.NoError(t, r.clients[a.Node].LogoutAndDisconnect(ctx), "%s: logout", step.Name)
+		case "restart", "start", "kill", "shutdown", "logout", "advertise", "set_hostname",
+			"break_tcp_conns", "rotate_disco_key", "cli":
+			r.daemon(ctx, t, step.Name, a)
 
 		case "retire":
 			// SaaS stops the daemon and deletes its devices.
@@ -688,6 +732,110 @@ func (r *stepReplay) apply(t *testing.T, step testcapture.Step) map[string]error
 	return refused
 }
 
+// daemon replays what a node's tailscaled or its CLI did.
+func (r *stepReplay) daemon(ctx context.Context, t *testing.T, stepName string, a testcapture.Action) {
+	t.Helper()
+
+	cl := r.clients[a.Node]
+	args := strings.Join(a.Args, " ")
+
+	switch {
+	case a.Kind == "restart" || a.Kind == "start":
+		// start follows a kill: the daemon comes back on its state.
+		require.NoError(t, cl.Restart(ctx, r.hostinfo[a.Node].Clone()), "%s: %s", stepName, a.Kind)
+
+	case a.Kind == "kill", a.Kind == "shutdown" && !r.memState[a.Node]:
+		cl.Disconnect(t)
+
+	case a.Kind == "shutdown", a.Kind == "logout", a.Kind == "cli" && args == "logout":
+		// tailscaled logs out on shutdown only with in-memory state.
+		require.NoError(t, cl.LogoutAndDisconnect(ctx), "%s: %s", stepName, a.Kind)
+
+	case a.Kind == "advertise":
+		r.announce(ctx, t, a.Node, a.Routes)
+
+	case a.Kind == "set_hostname":
+		r.hostinfo[a.Node].Hostname = a.Hostname
+		r.sendHostinfo(ctx, t, a.Node)
+
+	case a.Kind == "break_tcp_conns":
+		// The client polls again under the same key.
+		require.NoError(t, cl.RestartPoll(ctx), "%s: reconnect", stepName)
+
+	case a.Kind == "rotate_disco_key":
+		require.NoError(t, cl.RotateDisco(ctx), "%s: disco rotation", stepName)
+
+	case strings.HasPrefix(args, "debug set-expire --in="):
+		d, err := time.ParseDuration(strings.TrimPrefix(args, "debug set-expire --in="))
+		require.NoError(t, err, "%s: set-expire duration", stepName)
+
+		// shortcut: the replay asks for at most setExpireLifetime, so the
+		// wait that follows need not be real; a check of the exact expiry
+		// would need the full duration.
+		// A refusal is in the client's register attempts.
+		_ = cl.SetExpirySooner(ctx, time.Now().Add(min(d, setExpireLifetime)))
+
+	case strings.HasPrefix(args, "set --hostname="):
+		r.hostinfo[a.Node].Hostname = strings.TrimPrefix(args, "set --hostname=")
+		r.sendHostinfo(ctx, t, a.Node)
+
+	default:
+		t.Fatalf("%s: no replay for %s %q", stepName, a.Kind, a.Args)
+	}
+}
+
+// setExpireLifetime bounds the key lifetime a replayed
+// `tailscale debug set-expire` asks for.
+const setExpireLifetime = 5 * time.Second
+
+// expireDueKeys expires each minted key whose SaaS expiry passed before at,
+// so a step taken after a key's lifetime finds it expired, as SaaS did,
+// without the replay waiting.
+func (r *stepReplay) expireDueKeys(t *testing.T, at time.Time) {
+	t.Helper()
+
+	for label, exp := range r.keyExpires {
+		if exp.Before(at) {
+			require.NoError(t, r.srv.State().ExpirePreAuthKey(r.keyIDs[label]), "expire key %s", label)
+			delete(r.keyExpires, label)
+		}
+	}
+}
+
+// remember records alias's node while its machine holds exactly one.
+func (r *stepReplay) remember(alias string) {
+	if nodes := r.machineNodes(alias); len(nodes) == 1 {
+		r.lastIDs[alias] = nodes[0].StringID()
+	}
+}
+
+// v2Path maps a SaaS API path onto headscale's v2 API, and its device and
+// key labels onto the replay's IDs. A device is its machine's one node, or
+// the last one it held. It reports false for a label the replay never
+// created and for a machine holding several nodes.
+func (r *stepReplay) v2Path(path string) (string, bool) {
+	if rest, ok := strings.CutPrefix(path, "/device/{device:"); ok {
+		alias, tail, _ := strings.Cut(rest, "}")
+		if len(r.machineNodes(alias)) > 1 {
+			return "", false
+		}
+
+		r.remember(alias)
+		id, ok := r.lastIDs[alias]
+
+		return "/api/v2/device/" + id + tail, ok
+	}
+
+	if rest, ok := strings.CutPrefix(path, "/tailnet/-/keys/{key:"); ok {
+		label, tail, _ := strings.Cut(rest, "}")
+		id, ok := r.keyIDs[label]
+
+		return "/api/v2/tailnet/-/keys/" + strconv.FormatUint(id, 10) + tail, ok
+	}
+
+	return "/api/v2" + path, true
+}
+
 // requestTags is the Hostinfo.RequestTags a login sends: the action's when
 // it set them, else the ones the node sent last.
 func requestTags(a testcapture.Action, last []string) []string {
@@ -724,15 +872,26 @@ func (r *stepReplay) setPolicy(t *testing.T, stepName, policy string) {
 	}
 }
 
-// api replays one admin exchange. Device and key deletes go to the v2 API;
-// key mints and expiry, which the v2 API lacks or would answer with
-// secrets, go to State.
-func (r *stepReplay) api(t *testing.T, stepName string, a testcapture.Action) {
+// api replays one admin exchange, the i-th action of its step. Device and
+// key deletes go to the v2 API; key mints and expiry, which the v2 API lacks
+// or would answer with secrets, go to State. Reads are replayed only with
+// readAPI.
+func (r *stepReplay) api(t *testing.T, stepName string, i int, a testcapture.Action) {
 	t.Helper()
 
 	switch {
 	case a.Method == http.MethodGet:
-		return
+		if !r.readAPI {
+			return
+		}
+
+		if path, ok := r.v2Path(a.Path); ok {
+			r.exchange(t, stepName, i, a, path)
+		}
+
+	case a.Method == http.MethodPost && strings.HasPrefix(a.Path, "/tailnet/-/dns/"):
+		// Headscale reads DNS from its config, which a consumer sets from
+		// the capture's input; it has no API to change it.
 
 	case a.Method == http.MethodPost && a.Path == "/tailnet/-/acl":
 		require.Equal(t, http.StatusOK, a.Status, "%s: replay of a refused policy", stepName)
@@ -775,13 +934,18 @@ func (r *stepReplay) api(t *testing.T, stepName string, a testcapture.Action) {
 		r.keyTags[a.Key] = create.Tags
 		r.keyUsers[a.Key] = user
 
-	case a.Method == http.MethodDelete && strings.HasPrefix(a.Path, "/tailnet/-/keys/{key:"):
-		label := strings.TrimSuffix(strings.TrimPrefix(a.Path, "/tailnet/-/keys/{key:"), "}")
-		id, ok := r.keyIDs[label]
-		require.True(t, ok, "%s: delete of unknown key %q", stepName, label)
+		var minted struct {
+			Expires time.Time `json:"expires"`
+		}
+		if json.Unmarshal(a.Response, &minted) == nil && !minted.Expires.IsZero() {
+			r.keyExpires[a.Key] = minted.Expires
+		}
 
-		status, resp := r.call(t, a.Method, "/api/v2/tailnet/-/keys/"+strconv.FormatUint(id, 10), nil)
-		require.Equal(t, a.Status, status, "%s: %s %s: %s", stepName, a.Method, a.Path, resp)
+	case a.Method == http.MethodDelete && strings.HasPrefix(a.Path, "/tailnet/-/keys/{key:"):
+		path, ok := r.v2Path(a.Path)
+		require.True(t, ok, "%s: delete of unknown key %q", stepName, a.Path)
+
+		r.exchange(t, stepName, i, a, path)
 
 	case a.Method == http.MethodPost && strings.HasSuffix(a.Path, "/expire"):
 		now := time.Now()
@@ -791,18 +955,31 @@ func (r *stepReplay) api(t *testing.T, stepName string, a testcapture.Action) {
 		r.srv.App.Change(c)
 
 	case strings.HasPrefix(a.Path, "/device/"):
-		alias := deviceAlias(t, a.Path)
-		path := "/api/v2/device/" + r.node(t, alias).StringID() +
-			strings.TrimPrefix(a.Path, "/device/{device:"+alias+"}")
+		path, ok := r.v2Path(a.Path)
+		require.True(t, ok, "%s: %s names no single device", stepName, a.Path)
 
-		status, resp := r.call(t, a.Method, path, a.Body)
-		require.Equal(t, a.Status, status, "%s: %s %s: %s", stepName, a.Method, a.Path, resp)
+		r.exchange(t, stepName, i, a, path)
 
 	default:
 		t.Fatalf("%s: no replay for %s %s", stepName, a.Method, a.Path)
 	}
 }
 
+// exchange sends a's request to headscale at path. With readAPI the answer
+// is kept for the consumer; otherwise its status must be SaaS's.
+func (r *stepReplay) exchange(t *testing.T, stepName string, i int, a testcapture.Action, path string) {
+	t.Helper()
+
+	status, resp := r.call(t, a.Method, path, a.Body)
+	if r.readAPI {
+		r.answers[i] = apiAnswer{status: status, body: resp}
+
+		return
+	}
+
+	require.Equal(t, a.Status, status, "%s: %s %s: %s", stepName, a.Method, a.Path, resp)
+}
+
 // call sends one v2 API request as the admin.
 func (r *stepReplay) call(t *testing.T, method, path string, body []byte) (int, []byte) {
 	t.Helper()