servertest, state: add map delivery golden data

A retag through the API, a running peer's DERP, disco, hostname and
port, down and up gaps around policy pushes; peers, filters and frames
per node.

hscontrol/servertest/client.go

@@ -66,6 +66,9 @@ type TestClient struct {
 	updatesReceived int
 	noHistory       bool
 
+	// streams and streamMaps: see [TestClient.Stream].
+	streams, streamMaps int
+
 	// deltaUpdates and deltaRemoved: see [WithDeltaUpdates].
 	deltaUpdates bool
 	deltaRemoved []tailcfg.NodeID
@@ -399,6 +402,11 @@ func (c *TestClient) startPollLoop() {
 	c.pollCtx, c.pollCancel = context.WithCancel(context.Background())
 	c.pollDone = make(chan struct{})
 
+	c.mu.Lock()
+	c.streams++
+	c.streamMaps = 0
+	c.mu.Unlock()
+
 	go func() {
 		defer close(c.pollDone)
 
@@ -434,6 +442,7 @@ func (c *TestClient) UpdateFullNetmap(nm *netmap.NetworkMap) {
 	c.mu.Lock()
 	c.netmap = nm
 	c.updatesReceived++
+	c.streamMaps++
 
 	if !c.noHistory {
 		c.history = append(c.history, nm)
@@ -910,6 +919,16 @@ func (c *TestClient) UpdateCount() int {
 	return c.updatesReceived
 }
 
+// Stream returns how many map streams the client has opened, then how many
+// netmaps the current one delivered. [TestClient.Netmap] outlives a stream,
+// so only this tells a new stream that was answered from one that was not.
+func (c *TestClient) Stream() (int, int) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	return c.streams, c.streamMaps
+}
+
 // History returns the retained [netmap.NetworkMap] snapshots in order.
 // It is empty when [WithoutNetmapHistory] is set.
 func (c *TestClient) History() []*netmap.NetworkMap {

hscontrol/servertest/mapstream_compat_test.go

@@ -0,0 +1,812 @@
+// This file replays map stream captures from Tailscale SaaS: what each node
+// is sent when a device is retagged, when a running peer changes its DERP
+// home, disco key, hostname or port, when a node goes down and up around
+// policy pushes, and which Hostinfo fields peers and the node itself get.
+//
+// Test data source: testdata/steps/map-delivery.hujson, loaded by name. Its
+// disco rotation also pins a row of TestBuildMapRequestChangeResponse in
+// ../state.
+// Source format:    github.com/juanfont/headscale/hscontrol/types/testcapture
+
+package servertest_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"maps"
+	"net/netip"
+	"slices"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/juanfont/headscale/hscontrol/types/testcapture"
+	"github.com/stretchr/testify/require"
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/netmap"
+)
+
+// mapStreamSkipReasons lists checks where headscale differs from SaaS, keyed
+// <test_id>/<step>/<node>/<check>. A listed check that matches fails, so a
+// fix must delete its entry.
+var mapStreamSkipReasons = map[string]string{
+	"map-delivery/S1.1/appc/frames":   policyResponseToEveryNode,
+	"map-delivery/S1.1/router/frames": policyResponseToEveryNode,
+	"map-delivery/S1.1/sshd/frames":   policyResponseToEveryNode,
+	"map-delivery/S1.2/appc/frames":   policyResponseToEveryNode,
+	"map-delivery/S1.2/router/frames": policyResponseToEveryNode,
+	"map-delivery/S1.2/sshd/frames":   policyResponseToEveryNode,
+	"map-delivery/S1.3/appc/frames":   policyResponseToEveryNode,
+	"map-delivery/S1.3/router/frames": policyResponseToEveryNode,
+	"map-delivery/S1.3/sshd/frames":   policyResponseToEveryNode,
+	"map-delivery/S1.4/appc/frames":   policyResponseToEveryNode,
+	"map-delivery/S1.4/router/frames": policyResponseToEveryNode,
+	"map-delivery/S1.4/sshd/frames":   policyResponseToEveryNode,
+
+	"map-delivery/S3.5-remove/db1/frames":   policyResponseToEveryNode,
+	"map-delivery/S3.5-restore/appc/frames": policyResponseToEveryNode,
+	"map-delivery/S3.5-restore/db1/frames":  policyResponseToEveryNode,
+	"map-delivery/S3.5-restore/sshd/frames": policyResponseToEveryNode,
+
+	"map-delivery/S2.1/laptop/frames":         ownReportEchoed,
+	"map-delivery/S2.3/laptop/frames":         ownReportEchoed,
+	"map-delivery/S2.5/laptop/self-endpoints": ownReportEchoed,
+
+	"map-delivery/S2.5/laptop/endpoints-gap-bystander": restartEndpointsKept,
+	"map-delivery/S2.5/laptop/endpoints-gap-svc":       restartEndpointsKept,
+
+	"map-delivery/S3.1/appc/frames":      reconnectAnnounced,
+	"map-delivery/S3.1/bystander/frames": reconnectAnnounced,
+	"map-delivery/S3.1/router/frames":    reconnectAnnounced,
+	"map-delivery/S3.1/sshd/frames":      reconnectAnnounced,
+	"map-delivery/S3.1/svc/frames":       reconnectAnnounced,
+
+	"map-delivery/S3.4/appc/frames":      routerReconnectRecompute,
+	"map-delivery/S3.4/bystander/frames": routerReconnectRecompute,
+	"map-delivery/S3.4/db1/frames":       routerReconnectRecompute,
+	"map-delivery/S3.4/laptop/frames":    routerReconnectRecompute,
+	"map-delivery/S3.4/sshd/frames":      routerReconnectRecompute,
+	"map-delivery/S3.4/svc/frames":       routerReconnectRecompute,
+
+	"map-delivery/S3.6/bystander/peers": sameUserPeers,
+	"map-delivery/S3.6/laptop/peers":    sameUserPeers,
+
+	"map-delivery/S2.0/appc/self-hostinfo-keys":      selfHostinfoWhole,
+	"map-delivery/S2.0/bystander/self-hostinfo-keys": selfHostinfoWhole,
+	"map-delivery/S2.0/sshd/self-hostinfo-keys":      selfHostinfoWhole,
+
+	"map-delivery/S2.0/appc/peer-hostinfo-keys":      peerHostinfoWhole,
+	"map-delivery/S2.0/bystander/peer-hostinfo-keys": peerHostinfoWhole,
+	"map-delivery/S2.0/sshd/peer-hostinfo-keys":      peerHostinfoWhole,
+
+	"map-delivery/S2.0/appc/peer-hostinfo":      peerServicesWhole,
+	"map-delivery/S2.0/bystander/peer-hostinfo": peerServicesWhole,
+	"map-delivery/S2.0/sshd/peer-hostinfo":      peerServicesWhole,
+}
+
+const (
+	policyResponseToEveryNode = "POLICY_RESPONSE_TO_EVERY_NODE: headscale sends every connected node a policy response when tags or the policy change; SaaS sends only the nodes whose peers or filter change"
+	ownReportEchoed           = "OWN_REPORT_ECHOED: headscale sends a node its own endpoint or DERP report back; SaaS sends it nothing, so its own entry keeps the endpoints of its last full map"
+	restartEndpointsKept      = "RESTART_ENDPOINTS_KEPT: headscale keeps a restarted node's last endpoints at its peers until it reports new ones; SaaS sends peers the node without endpoints until then"
+	reconnectAnnounced        = "RECONNECT_ANNOUNCED: headscale tells peers a node is online after a stream gap of a few seconds; SaaS sends them nothing"
+	routerReconnectRecompute  = "ROUTER_RECONNECT_RECOMPUTE: headscale sends every node a policy response when a subnet router's stream comes back after a few seconds; SaaS sends nothing"
+	selfHostinfoWhole         = "SELF_HOSTINFO_WHOLE: headscale sends a node its whole reported Hostinfo; SaaS sends only Hostname, RequestTags and Services"
+	peerHostinfoWhole         = "PEER_HOSTINFO_WHOLE: headscale sends peers a node's whole reported Hostinfo; SaaS sends only Hostname, OS, Services, the SSH host keys and AppConnector"
+	peerServicesWhole         = "PEER_SERVICES_WHOLE: headscale sends peers every service a node reports; SaaS leaves out peerapi-dns-proxy"
+)
+
+// TestMapDeliveryCompat replays a device retagged through the API, a running
+// peer that picks new DERP homes, rotates its disco key, renames itself and
+// restarts on another port, down and up gaps of a node and a router, a rule
+// removed and restored around a reconnect, and a policy with no rules. After
+// each scenario step it compares, per captured node, its peers, packet
+// filter, tags and own endpoints; per viewer, how it sees the step's subject;
+// and, per node, whether it was sent anything at all.
+func TestMapDeliveryCompat(t *testing.T) {
+	t.Parallel()
+
+	t.Run("map-delivery", func(t *testing.T) {
+		t.Parallel()
+
+		replayMapDelivery(t, "map-delivery", "", deliveryChecks, func(key string) bool {
+			return !strings.Contains(key, "hostinfo")
+		})
+	})
+}
+
+// TestPeerHostinfoCompat replays the same capture up to its quiet step, with
+// sshd running Tailscale SSH and appc advertising an app connector, and
+// compares the Hostinfo fields and values each captured node gets for its
+// peers and for itself.
+func TestPeerHostinfoCompat(t *testing.T) {
+	t.Parallel()
+
+	t.Run("map-delivery", func(t *testing.T) {
+		t.Parallel()
+
+		replayMapDelivery(t, "map-delivery", hostinfoStep, hostinfoChecks, func(key string) bool {
+			return strings.Contains(key, "hostinfo")
+		})
+	})
+}
+
+// hostinfoStep is the quiet step whose captures hold the Hostinfo snapshots.
+const hostinfoStep = "S2.0"
+
+// settledGap is how far apart two actions must be sent for SaaS to have
+// handled them one after the other rather than in one batch.
+const settledGap = time.Second
+
+// replayMapDelivery replays capture id step by step, through the step named
+// last (every step when empty), and holds the checks of each scenario step
+// against [mapStreamSkipReasons]. Setup steps (policy, keys, joins, the steady
+// wait) are replayed, not compared: their join bursts and SaaS's own stream
+// restarts are not what the scenarios ask about. owns picks the skip entries
+// this caller's checks evaluate.
+func replayMapDelivery(
+	t *testing.T,
+	id, last string,
+	checks func(d *deliveryStep) []stepCheck,
+	owns func(key string) bool,
+) {
+	t.Helper()
+
+	c := readStepCapture(t, id)
+	r := newStepReplay(t, c)
+	evaluated := map[string]bool{}
+
+	saasLast := map[string]*netmap.NetworkMap{}
+	for alias, n := range c.Captures {
+		saasLast[alias] = n.Netmap
+	}
+
+	for _, step := range c.Steps {
+		require.Equal(t, "observed", step.Result, "%s: the replay follows every step", step.Name)
+
+		r.settle(t)
+
+		d := &deliveryStep{
+			r: r, step: step, caps: step.Captures, saasBefore: maps.Clone(saasLast),
+			hsBefore: map[string]*netmap.NetworkMap{}, updates: map[string]int{},
+		}
+		// The start check has no captures of its own: it is the baseline.
+		if len(d.caps) == 0 {
+			d.caps = c.Captures
+		}
+
+		for alias, cl := range r.clients {
+			d.hsBefore[alias] = cl.Netmap()
+			d.updates[alias] = cl.UpdateCount()
+		}
+
+		r.replayDelivery(t, step)
+
+		if strings.HasPrefix(step.Name, "S") {
+			cs := checks(d)
+			prefix := id + "/" + step.Name + "/"
+
+			if len(cs) > 0 {
+				t.Run(step.Name, func(t *testing.T) {
+					holdChecks(t, mapStreamSkipReasons, prefix, cs, evaluated)
+				})
+			}
+		}
+
+		for alias, n := range step.Captures {
+			saasLast[alias] = n.Netmap
+		}
+
+		if step.Name == last {
+			break
+		}
+	}
+
+	for k := range mapStreamSkipReasons {
+		if strings.HasPrefix(k, id+"/") && owns(k) && !evaluated[k] {
+			t.Errorf("skip entry %q names no check", k)
+		}
+	}
+}
+
+// replayDelivery applies the step's actions one at a time. What a node
+// reported is replayed from its captured map requests, so the CLI lever SaaS
+// was driven with does not decide what headscale sees. Headscale settles
+// where SaaS waited or sent the next action a while later; actions sent
+// together stay together.
+func (r *stepReplay) replayDelivery(t *testing.T, step testcapture.Step) {
+	t.Helper()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	reports := stepReports(t, step)
+
+	for i, a := range step.Actions {
+		switch a.Kind {
+		case "wait":
+			r.settle(t)
+		case "cli", "restart":
+			r.deliveryDaemon(ctx, t, step, a)
+		default:
+			one := step
+			one.Actions = []testcapture.Action{a}
+			require.Empty(t, r.apply(t, one), "%s: every login succeeds", step.Name)
+		}
+
+		if reqs, ok := reports[i]; ok {
+			r.report(ctx, t, a.Node, reqs)
+		}
+
+		// A wait on either side settles anyway.
+		if i+1 < len(step.Actions) {
+			next := step.Actions[i+1]
+			if a.Kind != "wait" && next.Kind != "wait" &&
+				time.Duration(next.OffsetMS-a.OffsetMS)*time.Millisecond >= settledGap {
+				r.settle(t)
+			}
+		}
+	}
+}
+
+// deliveryDaemon replays what a node's tailscaled did around its reports: a
+// restart, a disco key rotation, or its stream going down and coming back.
+func (r *stepReplay) deliveryDaemon(ctx context.Context, t *testing.T, step testcapture.Step, a testcapture.Action) {
+	t.Helper()
+
+	cl := r.clients[a.Node]
+
+	switch args := strings.Join(a.Args, " "); {
+	case a.Kind == "restart":
+		// The daemon starts again on the Hostinfo of its first request, with
+		// no endpoints or DERP home until it reports them.
+		var hi *tailcfg.Hostinfo
+
+		for _, f := range step.Frames[a.Node] {
+			if f.Request != nil && f.Request.Hostinfo != nil {
+				hi = f.Request.Hostinfo.Clone()
+
+				break
+			}
+		}
+
+		require.NotNil(t, hi, "%s: no request from restarted %s", step.Name, a.Node)
+		require.NoError(t, cl.Restart(ctx, hi), "%s: restart", step.Name)
+
+	case args == "down":
+		cl.Disconnect(t)
+
+	case args == "up":
+		// tailscaled registers again on its node key, then polls.
+		require.NoError(t, cl.ReloginAndPoll(ctx), "%s: up", step.Name)
+
+	case args == "debug rotate-disco-key":
+		require.NoError(t, cl.RotateDisco(ctx), "%s: disco rotation", step.Name)
+
+		// SaaS got the node's next report after the new stream's first map.
+		// Sent sooner, it can reach the server ahead of the stream request,
+		// whose older Hostinfo then overwrites it.
+		require.Eventually(t, func() bool {
+			_, maps := cl.Stream()
+
+			return maps > 0
+		}, 10*time.Second, 10*time.Millisecond, "%s: new stream answered", step.Name)
+
+	case args == "debug pick-new-derp", args == "set --ssh", args == "set --advertise-connector",
+		strings.HasPrefix(args, "set --hostname="):
+		// Only what the node reports changes.
+
+	default:
+		t.Fatalf("%s: no replay for %s %q", step.Name, a.Kind, a.Args)
+	}
+}
+
+// stepReports hands each node's captured map requests, other than stream
+// starts, to the actions after which it reported, by action index: a node's
+// only reporting action takes them all, and several take one each.
+func stepReports(t *testing.T, step testcapture.Step) map[int][]*tailcfg.MapRequest {
+	t.Helper()
+
+	actions := map[string][]int{}
+
+	for i, a := range step.Actions {
+		switch {
+		case a.Kind == "join", a.Kind == "restart", a.Kind == "cli" && strings.Join(a.Args, " ") != "down":
+			actions[a.Node] = append(actions[a.Node], i)
+		}
+	}
+
+	out := map[int][]*tailcfg.MapRequest{}
+
+	for node, idx := range actions {
+		var reqs []*tailcfg.MapRequest
+
+		for _, f := range step.Frames[node] {
+			if f.Request != nil && !f.Request.Stream {
+				reqs = append(reqs, f.Request)
+			}
+		}
+
+		switch {
+		case len(idx) == 1:
+			out[idx[0]] = reqs
+		case len(idx) == len(reqs):
+			for k, i := range idx {
+				out[i] = reqs[k : k+1]
+			}
+		default:
+			t.Fatalf("%s: %d reporting actions on %s, %d requests", step.Name, len(idx), node, len(reqs))
+		}
+	}
+
+	return out
+}
+
+// report sends what the node told SaaS, request by request: its Hostinfo, and
+// its DERP home and endpoints where the request carried them. A request that
+// changes nothing is not sent.
+func (r *stepReplay) report(ctx context.Context, t *testing.T, alias string, reqs []*tailcfg.MapRequest) {
+	t.Helper()
+
+	d := r.clients[alias].Direct()
+
+	for _, req := range reqs {
+		changed := false
+
+		if hi := req.Hostinfo; hi != nil {
+			changed = d.SetHostinfo(hi)
+			if hi.NetInfo != nil {
+				changed = d.SetNetInfo(hi.NetInfo) || changed
+			}
+		}
+
+		if len(req.Endpoints) > 0 {
+			eps := make([]tailcfg.Endpoint, len(req.Endpoints))
+			for i, ap := range req.Endpoints {
+				eps[i].Addr = ap
+				if i < len(req.EndpointTypes) {
+					eps[i].Type = req.EndpointTypes[i]
+				}
+			}
+
+			changed = d.SetEndpoints(eps) || changed
+		}
+
+		if changed {
+			require.NoError(t, d.SendUpdate(ctx), "%s: report", alias)
+		}
+	}
+}
+
+// settle waits until no client has been sent a map response for a while, so
+// a batch in flight lands before the next action.
+func (r *stepReplay) settle(t *testing.T) {
+	t.Helper()
+
+	const quiet = 300 * time.Millisecond
+
+	counts := func() map[string]int {
+		out := make(map[string]int, len(r.clients))
+		for alias, cl := range r.clients {
+			out[alias] = cl.UpdateCount()
+		}
+
+		return out
+	}
+
+	last, since := counts(), time.Now()
+
+	tick := time.NewTicker(25 * time.Millisecond)
+	defer tick.Stop()
+
+	for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); <-tick.C {
+		now := counts()
+		if !maps.Equal(now, last) {
+			last, since = now, time.Now()
+
+			continue
+		}
+
+		if time.Since(since) >= quiet {
+			return
+		}
+	}
+
+	t.Fatalf("map streams did not settle: %v", last)
+}
+
+// deliveryStep is one replayed step, what SaaS showed after it and what both
+// sides showed before it.
+type deliveryStep struct {
+	r    *stepReplay
+	step testcapture.Step
+
+	// caps is SaaS's end state; saasBefore each node's last capture before
+	// the step.
+	caps       map[string]testcapture.Node
+	saasBefore map[string]*netmap.NetworkMap
+
+	// hsBefore and updates are each client's netmap and update count, which
+	// is its history's length, when the step began.
+	hsBefore map[string]*netmap.NetworkMap
+	updates  map[string]int
+}
+
+// deliveryChecks compares, per captured node, its peers, filter, tags and
+// own endpoints; per captured viewer, how it sees each node the step acted
+// on; and, per node, whether it was sent anything.
+func deliveryChecks(d *deliveryStep) []stepCheck {
+	r := d.r
+	saasNames := map[netip.Addr]string{}
+
+	for alias, n := range r.c.Topology.Nodes {
+		for _, s := range []string{n.IPv4, n.IPv6} {
+			addr, err := netip.ParseAddr(s)
+			if err == nil {
+				saasNames[addr] = alias
+			}
+		}
+	}
+
+	var checks []stepCheck
+
+	for _, x := range slices.Sorted(maps.Keys(r.c.NodeIDs)) {
+		cl, ok := r.clients[x]
+		if !ok {
+			continue
+		}
+
+		// A client without delta handling rebuilds its netmap, so its update
+		// count moves, on every map response but a keep-alive.
+		checks = append(checks, stepCheck{x + "/frames", saasDelivered(d.step.Frames[x]), func() string {
+			if cl.UpdateCount() > d.updates[x] {
+				return "frame"
+			}
+
+			return "silent"
+		}})
+	}
+
+	captured := slices.Sorted(maps.Keys(d.caps))
+
+	for _, x := range captured {
+		saas, cl := d.caps[x].Netmap, r.clients[x]
+		if saas == nil || cl == nil {
+			continue
+		}
+
+		checks = append(checks,
+			stepCheck{x + "/peers", peerAliases(saas, r.saasAlias), func() string { return peerAliases(cl.Netmap(), r.hsAlias) }},
+			stepCheck{x + "/filter", renderMatches(saas, saasNames), func() string { return renderMatches(cl.Netmap(), r.addrNames()) }},
+			stepCheck{x + "/tags", selfTags(saas), func() string { return selfTags(cl.Netmap()) }},
+			stepCheck{x + "/self-endpoints", selfEndpoints(saas), func() string { return selfEndpoints(cl.Netmap()) }},
+		)
+	}
+
+	for _, x := range stepSubjects(d.step) {
+		for _, v := range captured {
+			if v != x && d.caps[v].Netmap != nil && r.clients[v] != nil {
+				checks = append(checks, d.subjectChecks(x, v)...)
+			}
+		}
+	}
+
+	return checks
+}
+
+// subjectChecks compares how v sees x: online, its DERP home, endpoints,
+// name, routes, whether its disco key changed during the step and whether
+// its endpoints were cleared on the way.
+func (d *deliveryStep) subjectChecks(x, v string) []stepCheck {
+	r := d.r
+	saasID := r.c.NodeIDs[x]
+	saasPeer, saasMissing := peerByID(d.caps[v].Netmap, saasID)
+
+	hsID := func() (tailcfg.NodeID, string) {
+		nodes := r.machineNodes(x)
+		if len(nodes) != 1 {
+			return 0, fmt.Sprintf("devices %d", len(nodes))
+		}
+
+		return nodes[0].ID().NodeID(), ""
+	}
+	hsPeer := func() (tailcfg.NodeView, string) {
+		id, missing := hsID()
+		if missing != "" {
+			return tailcfg.NodeView{}, missing
+		}
+
+		return peerByID(r.clients[v].Netmap(), id)
+	}
+
+	hsBefore := ""
+	if id, missing := hsID(); missing == "" {
+		hsBefore = peerDisco(d.hsBefore[v], id)
+	}
+
+	return []stepCheck{
+		{x + "/online-" + v, peerOnline(saasPeer, saasMissing), func() string { return peerOnline(hsPeer()) }},
+		{x + "/derp-" + v, peerDERP(saasPeer, saasMissing), func() string { return peerDERP(hsPeer()) }},
+		{x + "/endpoints-" + v, peerEndpoints(saasPeer, saasMissing), func() string { return peerEndpoints(hsPeer()) }},
+		{x + "/name-" + v, peerLabel(saasPeer, saasMissing), func() string { return peerLabel(hsPeer()) }},
+		{x + "/routes-" + v, servedRoutes(saasPeer, saasMissing), func() string { return servedRoutes(hsPeer()) }},
+		{
+			x + "/disco-" + v,
+			discoState(peerDisco(d.saasBefore[v], saasID), peerDisco(d.caps[v].Netmap, saasID)),
+			func() string {
+				p, missing := hsPeer()
+				if missing != "" {
+					return discoState(hsBefore, missing)
+				}
+
+				return discoState(hsBefore, p.DiscoKey().String())
+			},
+		},
+		{x + "/endpoints-gap-" + v, saasEndpointsCleared(d.step.Frames[v], saasID), func() string {
+			id, missing := hsID()
+			if missing != "" {
+				return missing
+			}
+
+			return hsEndpointsCleared(r.clients[v].History()[d.updates[v]:], id)
+		}},
+	}
+}
+
+// stepSubjects names the nodes the step acted on, by action or device path.
+func stepSubjects(step testcapture.Step) []string {
+	set := map[string]bool{}
+
+	for _, a := range step.Actions {
+		switch {
+		case a.Node != "":
+			set[a.Node] = true
+		case strings.HasPrefix(a.Path, "/device/{device:"):
+			alias, _, _ := strings.Cut(strings.TrimPrefix(a.Path, "/device/{device:"), "}")
+			set[alias] = true
+		}
+	}
+
+	return slices.Sorted(maps.Keys(set))
+}
+
+// saasDelivered says whether a node's stream carried any map response but a
+// keep-alive during the step.
+func saasDelivered(frames []testcapture.Frame) string {
+	for _, f := range frames {
+		if f.Response != nil && !f.Response.KeepAlive {
+			return "frame"
+		}
+	}
+
+	return "silent"
+}
+
+// saasEndpointsCleared says whether a frame left the viewer holding peer id
+// without endpoints: a whole node with none, or a patch that empties them.
+func saasEndpointsCleared(frames []testcapture.Frame, id tailcfg.NodeID) string {
+	for _, f := range frames {
+		resp := f.Response
+		if resp == nil {
+			continue
+		}
+
+		for _, n := range slices.Concat(resp.Peers, resp.PeersChanged) {
+			if n.ID == id && len(n.Endpoints) == 0 {
+				return "cleared"
+			}
+		}
+
+		for i, p := range resp.PeersChangedPatch {
+			if p.NodeID == id && f.Empty[fmt.Sprintf("PeersChangedPatch[%d].Endpoints", i)] != "" {
+				return "cleared"
+			}
+		}
+	}
+
+	return "kept"
+}
+
+// hsEndpointsCleared says whether any of the netmaps held peer id without
+// endpoints.
+func hsEndpointsCleared(history []*netmap.NetworkMap, id tailcfg.NodeID) string {
+	for _, nm := range history {
+		if p, missing := peerByID(nm, id); missing == "" && p.Endpoints().Len() == 0 {
+			return "cleared"
+		}
+	}
+
+	return "kept"
+}
+
+func peerDERP(p tailcfg.NodeView, missing string) string {
+	if missing != "" {
+		return missing
+	}
+
+	return strconv.FormatInt(int64(p.HomeDERP()), 10)
+}
+
+func peerEndpoints(p tailcfg.NodeView, missing string) string {
+	if missing != "" {
+		return missing
+	}
+
+	return addrPorts(p)
+}
+
+// peerLabel is the first label of the peer's MagicDNS name and its hostname.
+func peerLabel(p tailcfg.NodeView, missing string) string {
+	if missing != "" {
+		return missing
+	}
+
+	return peerName(p, "") + " " + p.Hostinfo().Hostname()
+}
+
+func selfTags(nm *netmap.NetworkMap) string {
+	if nm == nil || !nm.SelfNode.Valid() {
+		return "no netmap"
+	}
+
+	return sortedStrings(nm.SelfNode.Tags().AsSlice())
+}
+
+func selfEndpoints(nm *netmap.NetworkMap) string {
+	if nm == nil || !nm.SelfNode.Valid() {
+		return "no netmap"
+	}
+
+	return addrPorts(nm.SelfNode)
+}
+
+func addrPorts(n tailcfg.NodeView) string {
+	out := make([]string, 0, n.Endpoints().Len())
+	for _, ap := range n.Endpoints().All() {
+		out = append(out, ap.String())
+	}
+
+	return sortedStrings(out)
+}
+
+// renderMatches lists nm's packet filter as "sources => ports to
+// destination" lines, devices named by names.
+func renderMatches(nm *netmap.NetworkMap, names map[netip.Addr]string) string {
+	if nm == nil {
+		return "no netmap"
+	}
+
+	var out []string
+
+	for _, m := range nm.PacketFilter {
+		for _, d := range m.Dsts {
+			dst := d.Net.String()
+			if name, ok := names[d.Net.Addr()]; ok && d.Net.IsSingleIP() {
+				dst = name
+			}
+
+			out = append(out, filterLine(m.Srcs, names, d.Ports.First, d.Ports.Last)+" to "+dst)
+		}
+	}
+
+	return filterLines(out)
+}
+
+// hostinfoChecks compares, at the quiet step, each captured node's Hostinfo
+// for itself and for each peer: which fields it holds, and the values of the
+// fields SaaS sent.
+func hostinfoChecks(d *deliveryStep) []stepCheck {
+	if d.step.Name != hostinfoStep {
+		return nil
+	}
+
+	r := d.r
+
+	var checks []stepCheck
+
+	for _, v := range slices.Sorted(maps.Keys(d.caps)) {
+		saas, cl := d.caps[v].Netmap, r.clients[v]
+		if saas == nil || cl == nil {
+			continue
+		}
+
+		saasSelf := hostinfoFields(saas.SelfNode.Hostinfo())
+		saasPeers := peerHostinfos(saas, r.saasAlias)
+
+		checks = append(checks,
+			stepCheck{v + "/self-hostinfo-keys", fieldNames(saasSelf), func() string {
+				nm := cl.Netmap()
+				if nm == nil {
+					return "no netmap"
+				}
+
+				return fieldNames(hostinfoFields(nm.SelfNode.Hostinfo()))
+			}},
+			stepCheck{v + "/self-hostinfo", fieldValues(saasSelf, saasSelf), func() string {
+				nm := cl.Netmap()
+				if nm == nil {
+					return "no netmap"
+				}
+
+				return fieldValues(hostinfoFields(nm.SelfNode.Hostinfo()), saasSelf)
+			}},
+			stepCheck{v + "/peer-hostinfo-keys", renderPeerFields(saasPeers, nil), func() string {
+				return renderPeerFields(peerHostinfos(cl.Netmap(), r.hsAlias), nil)
+			}},
+			stepCheck{v + "/peer-hostinfo", renderPeerFields(saasPeers, saasPeers), func() string {
+				return renderPeerFields(peerHostinfos(cl.Netmap(), r.hsAlias), saasPeers)
+			}},
+		)
+	}
+
+	return checks
+}
+
+// hostinfoFields is hi's JSON object, field by field.
+func hostinfoFields(hi tailcfg.HostinfoView) map[string]json.RawMessage {
+	out := map[string]json.RawMessage{}
+	if !hi.Valid() {
+		return out
+	}
+
+	b, err := json.Marshal(hi)
+	if err == nil {
+		_ = json.Unmarshal(b, &out)
+	}
+
+	return out
+}
+
+// peerHostinfos holds the Hostinfo fields of each peer alias names.
+func peerHostinfos(nm *netmap.NetworkMap, alias func(tailcfg.NodeID) (string, bool)) map[string]map[string]json.RawMessage {
+	out := map[string]map[string]json.RawMessage{}
+	if nm == nil {
+		return out
+	}
+
+	for _, p := range nm.Peers {
+		if name, ok := alias(p.ID()); ok {
+			out[name] = hostinfoFields(p.Hostinfo())
+		}
+	}
+
+	return out
+}
+
+func fieldNames(fields map[string]json.RawMessage) string {
+	return strings.Join(slices.Sorted(maps.Keys(fields)), ",")
+}
+
+// fieldValues renders the fields named in want, as fields holds them.
+func fieldValues(fields, want map[string]json.RawMessage) string {
+	out := make([]string, 0, len(want))
+
+	for _, k := range slices.Sorted(maps.Keys(want)) {
+		v, ok := fields[k]
+		if !ok {
+			v = json.RawMessage("absent")
+		}
+
+		out = append(out, k+"="+string(v))
+	}
+
+	return strings.Join(out, " ")
+}
+
+// renderPeerFields lists each peer's field names or, with want, the values
+// of the fields want holds for it.
+func renderPeerFields(peers, want map[string]map[string]json.RawMessage) string {
+	out := make([]string, 0, len(peers))
+
+	for _, name := range slices.Sorted(maps.Keys(peers)) {
+		line := fieldNames(peers[name])
+		if want != nil {
+			line = fieldValues(peers[name], want[name])
+		}
+
+		out = append(out, name+": "+line)
+	}
+
+	return strings.Join(out, "\n")
+}

hscontrol/state/maprequest_test.go

@@ -3,6 +3,7 @@ package state
 import (
 	"fmt"
 	"net/netip"
+	"path/filepath"
 	"reflect"
 	"strings"
 	"sync"
@@ -13,6 +14,7 @@ import (
 	"github.com/juanfont/headscale/hscontrol/db"
 	"github.com/juanfont/headscale/hscontrol/types"
 	"github.com/juanfont/headscale/hscontrol/types/change"
+	"github.com/juanfont/headscale/hscontrol/types/testcapture"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"gorm.io/gorm"
@@ -986,6 +988,8 @@ func TestHostinfoDERP(t *testing.T) {
 }
 
 func TestBuildMapRequestChangeResponse(t *testing.T) {
+	discoKeys, discoEndpoints := saasDiscoRotation(t)
+
 	node := types.Node{
 		ID:        1,
 		NodeKey:   key.NewNode().Public(),
@@ -1032,8 +1036,8 @@ func TestBuildMapRequestChangeResponse(t *testing.T) {
 			name:  "disco key only",
 			delta: mapRequestDelta{discoKeyChanged: true},
 
-			wantKeys:      true,
-			wantEndpoints: true,
+			wantKeys:      discoKeys,
+			wantEndpoints: discoEndpoints,
 		},
 		{
 			name:     "disco key and derp move",
@@ -1097,6 +1101,65 @@ func TestBuildMapRequestChangeResponse(t *testing.T) {
 	}
 }
 
+// saasDiscoRotation reads how Tailscale SaaS told a peer about a running
+// node's disco key rotation: whether the peer's entry carried the node key
+// and the endpoints. Test data source:
+// ../servertest/testdata/steps/map-delivery.hujson, step S2.2, where laptop
+// rotates and bystander watches; the servertest step captures live there.
+func saasDiscoRotation(t *testing.T) (bool, bool) {
+	t.Helper()
+
+	c, err := testcapture.Read(filepath.Join("..", "servertest", "testdata", "steps", "map-delivery.hujson"))
+	require.NoError(t, err)
+
+	id := c.NodeIDs["laptop"]
+
+	var last *tailcfg.MapRequest
+
+	for _, step := range c.Steps {
+		if step.Name != "S2.2" {
+			for _, f := range step.Frames["laptop"] {
+				if f.Request != nil {
+					last = f.Request
+				}
+			}
+
+			continue
+		}
+
+		// The rotation's first report changes the disco key and nothing
+		// else this row looks at.
+		first := step.Frames["laptop"][0].Request
+		require.NotNil(t, first, "S2.2 starts with laptop's report")
+		require.NotNil(t, last, "laptop reported before S2.2")
+		require.NotEqual(t, last.DiscoKey, first.DiscoKey, "disco key rotated")
+		require.Equal(t, last.Endpoints, first.Endpoints, "endpoints kept")
+		require.Equal(t, last.Hostinfo.NetInfo.PreferredDERP, first.Hostinfo.NetInfo.PreferredDERP, "DERP home kept")
+
+		for _, f := range step.Frames["bystander"] {
+			if f.Response == nil {
+				continue
+			}
+
+			for _, n := range f.Response.PeersChanged {
+				if n.ID == id {
+					return !n.Key.IsZero() && n.DiscoKey == first.DiscoKey, len(n.Endpoints) > 0
+				}
+			}
+
+			for _, p := range f.Response.PeersChangedPatch {
+				if p.NodeID == id {
+					return p.Key != nil && p.DiscoKey != nil, p.Endpoints != nil
+				}
+			}
+		}
+	}
+
+	t.Fatal("no disco rotation reached bystander")
+
+	return false, false
+}
+
 // benchMapRequestSetup pre-creates a sqlite database with n registered
 // nodes spread across 10 users (~10% tagged tag:srv, half of those also
 // carrying an approved and announced 10.x.0.0/24 route), then constructs a State