policy/v2, servertest: add policy generation golden data

One policy over joins, a via router swap, an admin tag, a delete and a
group edit; filter, SSH, peers and routes per step, and the manager
against a fresh one.

hscontrol/policy/v2/tailscale_sequence_compat_test.go

@@ -0,0 +1,422 @@
+// Compatibility tests for one policy compiled again and again as the nodes
+// under it change: members join, a second via router joins and the first
+// withdraws its routes, the admin tags one member's device and deletes
+// another, then the group the policy names everywhere is edited. Each step
+// is compared with what Tailscale SaaS sent every live node.
+//
+// Test data source: ../../servertest/testdata/steps/policy-generation.hujson,
+// loaded by name; TestPolicySequenceMapCompat in ../../servertest replays it
+// through real clients.
+// Source format:    github.com/juanfont/headscale/hscontrol/types/testcapture
+
+package v2
+
+import (
+	"encoding/json"
+	"fmt"
+	"maps"
+	"net/http"
+	"net/netip"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/juanfont/headscale/hscontrol/types"
+	"github.com/juanfont/headscale/hscontrol/types/testcapture"
+	"github.com/stretchr/testify/require"
+	"tailscale.com/tailcfg"
+)
+
+// policySequenceSkipReasons lists checks where headscale differs from SaaS,
+// keyed <test_id>/<step>/<node>/<check>. A listed check that matches fails,
+// so a fix must delete its entry.
+var policySequenceSkipReasons = map[string]string{
+	"policy-generation/G0/srv/filter": aclDstDedupe,
+	"policy-generation/G1/srv/filter": aclDstDedupe,
+	"policy-generation/G2/srv/filter": aclDstDedupe,
+	"policy-generation/G3/srv/filter": aclDstDedupe,
+	"policy-generation/G4/srv/filter": aclDstDedupe,
+	"policy-generation/G5/srv/filter": aclDstDedupe,
+
+	"policy-generation/G0/rt/filter":   viaRuleMerged,
+	"policy-generation/G1/rt/filter":   viaRuleMerged,
+	"policy-generation/G2/rt/filter":   viaRuleMerged,
+	"policy-generation/G3/rt/filter":   viaRuleMerged,
+	"policy-generation/G3/rt-b/filter": viaRuleMerged,
+	"policy-generation/G4/rt-b/filter": viaRuleMerged,
+	"policy-generation/G5/rt-b/filter": viaRuleMerged,
+
+	"policy-generation/G0/odin-1/uid-zero":  sshUIDZero,
+	"policy-generation/G0/srv/uid-zero":     sshUIDZero,
+	"policy-generation/G0/thor-1/uid-zero":  sshUIDZero,
+	"policy-generation/G1/freya-1/uid-zero": sshUIDZero,
+	"policy-generation/G1/odin-1/uid-zero":  sshUIDZero,
+	"policy-generation/G1/srv/uid-zero":     sshUIDZero,
+	"policy-generation/G1/thor-1/uid-zero":  sshUIDZero,
+	"policy-generation/G2/freya-1/uid-zero": sshUIDZero,
+	"policy-generation/G2/odin-1/uid-zero":  sshUIDZero,
+	"policy-generation/G2/srv/uid-zero":     sshUIDZero,
+	"policy-generation/G2/thor-1/uid-zero":  sshUIDZero,
+	"policy-generation/G2/thor-2/uid-zero":  sshUIDZero,
+	"policy-generation/G3/freya-1/uid-zero": sshUIDZero,
+	"policy-generation/G3/odin-1/uid-zero":  sshUIDZero,
+	"policy-generation/G3/srv/uid-zero":     sshUIDZero,
+	"policy-generation/G3/thor-1/uid-zero":  sshUIDZero,
+	"policy-generation/G3/thor-2/uid-zero":  sshUIDZero,
+	"policy-generation/G4/freya-1/uid-zero": sshUIDZero,
+	"policy-generation/G4/odin-1/uid-zero":  sshUIDZero,
+	"policy-generation/G4/srv/uid-zero":     sshUIDZero,
+	"policy-generation/G4/thor-1/uid-zero":  sshUIDZero,
+	"policy-generation/G4/thor-2/uid-zero":  sshUIDZero,
+	"policy-generation/G5/freya-1/uid-zero": sshUIDZero,
+	"policy-generation/G5/odin-1/uid-zero":  sshUIDZero,
+	"policy-generation/G5/srv/uid-zero":     sshUIDZero,
+	"policy-generation/G5/thor-2/uid-zero":  sshUIDZero,
+	"policy-generation/G7/freya-1/uid-zero": sshUIDZero,
+	"policy-generation/G7/odin-1/uid-zero":  sshUIDZero,
+	"policy-generation/G7/srv/uid-zero":     sshUIDZero,
+	"policy-generation/G8/freya-1/uid-zero": sshUIDZero,
+	"policy-generation/G8/odin-1/uid-zero":  sshUIDZero,
+}
+
+const viaRuleMerged = "VIA_RULE_MERGED: on a via router headscale merges the via grant's filter rule with an ACL rule of the same sources; SaaS sends the two apart"
+
+// policySequenceNotCompared lists, by key prefix, checks left out: SaaS's
+// answer there is in doubt, so it is no reference.
+var policySequenceNotCompared = []string{
+	"policy-generation/G7/rt-plain/filter",
+	"policy-generation/G7/srv/ssh-rules",
+	"policy-generation/G7/srv/filter",
+	"policy-generation/G7/rt-b/filter",
+}
+
+// TestPolicySequenceCompat follows the capture with one PolicyManager, as
+// headscale does: SetNodes after each node change, SetPolicy for the edit,
+// with nodes rebuilt from the step's device list and route reads. After each
+// step it compares, per node SaaS captured, the filter and SSH rules, and
+// the auto-approval of routes the step advertised. At every step the manager
+// must also answer as one built from that step's inputs alone, so a cache
+// that outlives its inputs shows without SaaS.
+func TestPolicySequenceCompat(t *testing.T) {
+	t.Parallel()
+
+	t.Run("policy-generation", func(t *testing.T) {
+		t.Parallel()
+
+		runPolicySequenceCompat(t, "policy-generation")
+	})
+}
+
+// runPolicySequenceCompat replays capture id step by step. Steps run in
+// order, since the one manager carries each into the next.
+func runPolicySequenceCompat(t *testing.T, id string) {
+	t.Helper()
+
+	c, err := testcapture.Read(filepath.Join("..", "..", "servertest", "testdata", "steps", id+".hujson"))
+	require.NoError(t, err)
+	require.NotEmpty(t, c.Steps, "%s: not a step capture", id)
+
+	users := setupGrantsCompatUsers()
+
+	// Node IDs follow join order, as headscale would assign them.
+	ids := map[string]types.NodeID{}
+
+	var (
+		pm     *PolicyManager
+		policy []byte
+	)
+
+	evaluated := map[string]bool{}
+
+	for _, step := range c.Steps {
+		require.Equal(t, "observed", step.Result, "%s: the replay follows every step", step.Name)
+
+		for _, a := range step.Actions {
+			if _, ok := ids[a.Node]; a.Kind == "join" && !ok {
+				ids[a.Node] = types.NodeID(len(ids) + 1) //nolint:gosec // a handful of nodes
+			}
+		}
+
+		nodes, approved := sequenceNodes(t, users, step, ids)
+		pushed := stepPolicy(t, step)
+
+		if pm == nil {
+			require.NotNil(t, pushed, "%s: the first step sets the policy", step.Name)
+
+			policy = pushed
+			pm, err = NewPolicyManager(policy, users, nodes.ViewSlice())
+			require.NoError(t, err, "%s: policy", step.Name)
+		} else {
+			_, err = pm.SetNodes(nodes.ViewSlice())
+			require.NoError(t, err, "%s: nodes", step.Name)
+
+			if pushed != nil {
+				policy = pushed
+				_, err = pm.SetPolicy(policy)
+				require.NoError(t, err, "%s: policy", step.Name)
+			}
+		}
+
+		pol, err := unmarshalPolicy(policy)
+		require.NoError(t, err, "%s: policy", step.Name)
+
+		fresh, err := NewPolicyManager(policy, users, nodes.ViewSlice())
+		require.NoError(t, err, "%s: policy", step.Name)
+
+		t.Run(step.Name, func(t *testing.T) {
+			t.Run("fresh", func(t *testing.T) {
+				if diff := freshMismatch(pm, fresh, nodes); diff != "" {
+					t.Error(diff)
+				}
+			})
+
+			for _, alias := range slices.Sorted(maps.Keys(step.Captures)) {
+				capture := step.Captures[alias]
+				node := findGrantsNode(nodes, alias)
+				require.NotNil(t, node, "%s: %s is not in the device list", step.Name, alias)
+
+				key := id + "/" + step.Name + "/" + alias + "/"
+
+				t.Run(alias, func(t *testing.T) {
+					check := func(name string, mismatch func() string) {
+						if notCompared(policySequenceNotCompared, key+name) {
+							return
+						}
+
+						evaluated[key+name] = true
+						compatCheck(t, policySequenceSkipReasons, key+name, mismatch)
+					}
+
+					check("filter", func() string {
+						got, err := pm.FilterForNode(node.View())
+						if err != nil {
+							return err.Error()
+						}
+
+						return cmp.Diff(capture.PacketFilterRules, got, append(cmpOptions(), cmpopts.EquateEmpty())...)
+					})
+
+					check("ssh-rules", func() string {
+						return sshRulesMismatch(pol, pm, users, nodes, node, capture, withoutUIDZero)
+					})
+
+					check("uid-zero", func() string {
+						return uidZeroMismatch(pm, node, capture)
+					})
+
+					// Approval is judged only where the step advertised:
+					// approvals that outlive eligibility are not asked here.
+					if advertised := joinRoutes(step, alias); len(advertised) > 0 {
+						check("auto-approval", func() string {
+							var want, got []string
+
+							for _, p := range advertised {
+								want = append(want, fmt.Sprintf("%s %t", p, slices.Contains(approved[alias], p)))
+								got = append(got, fmt.Sprintf("%s %t", p, pm.NodeCanApproveRoute(node.View(), p)))
+							}
+
+							return cmp.Diff(want, got)
+						})
+					}
+				})
+			}
+		})
+	}
+
+	// The map serves this one capture, so a key naming another fails too.
+	for k := range policySequenceSkipReasons {
+		if !evaluated[k] {
+			t.Errorf("skip entry %q names no check", k)
+		}
+	}
+}
+
+// freshMismatch compares the manager that followed every step with one built
+// from the step's inputs alone: per node its filter, SSH policy and via
+// routes towards every peer, and the peer map; "" is a match.
+func freshMismatch(pm, fresh *PolicyManager, nodes types.Nodes) string {
+	opts := cmpopts.EquateComparable(netip.Prefix{}, netip.Addr{})
+
+	var diffs []string
+
+	add := func(what string, want, got any, errs ...error) {
+		for _, err := range errs {
+			if err != nil {
+				diffs = append(diffs, what+": "+err.Error())
+
+				return
+			}
+		}
+
+		if diff := cmp.Diff(want, got, opts); diff != "" {
+			diffs = append(diffs, fmt.Sprintf("%s (-fresh +incremental):\n%s", what, diff))
+		}
+	}
+
+	for _, n := range nodes {
+		nv := n.View()
+
+		wantFilter, errW := fresh.FilterForNode(nv)
+		gotFilter, errG := pm.FilterForNode(nv)
+		add(n.GivenName+" filter", wantFilter, gotFilter, errW, errG)
+
+		wantSSH, errW := fresh.SSHPolicy("https://unused", nv)
+		gotSSH, errG := pm.SSHPolicy("https://unused", nv)
+		add(n.GivenName+" SSH policy", wantSSH, gotSSH, errW, errG)
+
+		for _, p := range nodes {
+			if p.ID != n.ID {
+				add(n.GivenName+" via routes of "+p.GivenName,
+					fresh.ViaRoutesForPeer(nv, p.View()), pm.ViaRoutesForPeer(nv, p.View()))
+			}
+		}
+	}
+
+	add("peer map", fresh.BuildPeerMap(nodes.ViewSlice()), pm.BuildPeerMap(nodes.ViewSlice()))
+
+	return strings.Join(diffs, "\n")
+}
+
+// sequenceDevice is the part of a SaaS device-list entry a node is built
+// from. The ID is the capture's "{device:<alias>}" label.
+type sequenceDevice struct {
+	ID        string   `json:"id"`
+	Addresses []string `json:"addresses"`
+	Tags      []string `json:"tags"`
+	User      string   `json:"user"`
+}
+
+// sequenceNodes builds the nodes SaaS listed after step: addresses, tags and,
+// for untagged devices, the owner from the device list; advertised and
+// approved routes from the step's route reads. It also returns each node's
+// approved routes. A tagged node has no user, as headscale stores it.
+func sequenceNodes(
+	t *testing.T,
+	users types.Users,
+	step testcapture.Step,
+	ids map[string]types.NodeID,
+) (types.Nodes, map[string][]netip.Prefix) {
+	t.Helper()
+
+	var (
+		devices    []sequenceDevice
+		listed     bool
+		advertised = map[string][]netip.Prefix{}
+		approved   = map[string][]netip.Prefix{}
+	)
+
+	for _, a := range step.Actions {
+		if a.Kind != "api" || a.Method != http.MethodGet {
+			continue
+		}
+
+		switch {
+		case a.Path == "/tailnet/-/devices":
+			var out struct {
+				Devices []sequenceDevice `json:"devices"`
+			}
+			require.NoError(t, json.Unmarshal(a.Response, &out), "%s: device list", step.Name)
+
+			devices, listed = out.Devices, true
+
+		case strings.HasPrefix(a.Path, "/device/{device:") && strings.HasSuffix(a.Path, "}/routes"):
+			alias := strings.TrimSuffix(strings.TrimPrefix(a.Path, "/device/{device:"), "}/routes")
+
+			var out struct {
+				Advertised []netip.Prefix `json:"advertisedRoutes"`
+				Enabled    []netip.Prefix `json:"enabledRoutes"`
+			}
+			require.NoError(t, json.Unmarshal(a.Response, &out), "%s: routes of %s", step.Name, alias)
+
+			advertised[alias], approved[alias] = out.Advertised, out.Enabled
+		}
+	}
+
+	require.True(t, listed, "%s: no device list", step.Name)
+
+	nodes := make(types.Nodes, 0, len(devices))
+
+	for _, d := range devices {
+		alias := strings.TrimSuffix(strings.TrimPrefix(d.ID, "{device:"), "}")
+
+		id, ok := ids[alias]
+		require.True(t, ok, "%s: %s never joined", step.Name, alias)
+
+		_, read := approved[alias]
+		require.True(t, read, "%s: no route read for %s", step.Name, alias)
+
+		node := &types.Node{
+			ID:             id,
+			Hostname:       alias,
+			GivenName:      alias,
+			Tags:           d.Tags,
+			Hostinfo:       &tailcfg.Hostinfo{RoutableIPs: advertised[alias]},
+			ApprovedRoutes: approved[alias],
+		}
+
+		for _, s := range d.Addresses {
+			addr := netip.MustParseAddr(s)
+			if addr.Is4() {
+				node.IPv4 = &addr
+			} else {
+				node.IPv6 = &addr
+			}
+		}
+
+		if len(d.Tags) == 0 {
+			i := slices.IndexFunc(users, func(u types.User) bool { return u.Email == d.User })
+			require.GreaterOrEqual(t, i, 0, "%s: %s owned by unknown %q", step.Name, alias, d.User)
+
+			node.User = &users[i]
+			node.UserID = &users[i].ID
+		}
+
+		nodes = append(nodes, node)
+	}
+
+	return nodes, approved
+}
+
+// stepPolicy returns the policy the step pushed, or nil.
+func stepPolicy(t *testing.T, step testcapture.Step) []byte {
+	t.Helper()
+
+	var out []byte
+
+	for _, a := range step.Actions {
+		if a.Kind != "policy" {
+			continue
+		}
+
+		require.Equal(t, http.StatusOK, a.Status, "%s: refused policy", step.Name)
+
+		// HuJSON bodies are stored as a JSON string.
+		out = a.Body
+		if strings.HasPrefix(string(a.Body), `"`) {
+			var s string
+			require.NoError(t, json.Unmarshal(a.Body, &s), "%s: policy body", step.Name)
+
+			out = []byte(s)
+		}
+	}
+
+	return out
+}
+
+// joinRoutes returns the routes alias advertised when it joined in step.
+func joinRoutes(step testcapture.Step, alias string) []netip.Prefix {
+	var out []netip.Prefix
+
+	for _, a := range step.Actions {
+		if a.Kind == "join" && a.Node == alias {
+			for _, r := range a.Routes {
+				out = append(out, netip.MustParsePrefix(r))
+			}
+		}
+	}
+
+	return out
+}

hscontrol/servertest/mapstream_compat_test.go

@@ -2,12 +2,15 @@
 // is sent when a device is retagged, when a running peer changes its DERP
 // home, disco key, hostname or port, when a node goes down and up around
 // policy pushes, and which Hostinfo fields peers and the node itself get;
-// what a node hears of nodes it cannot see; and what a router's peers hear
-// of routes it announces but has no approval for.
+// what a node hears of nodes it cannot see; what a router's peers hear of
+// routes it announces but has no approval for; and what every node is sent
+// as one policy is compiled again for node changes and a group edit.
 //
-// Test data source: testdata/steps/{map-delivery,hidden-peers,route-announce}.hujson,
-// loaded by name. The disco rotation of map-delivery also pins a row of
-// TestBuildMapRequestChangeResponse in ../state.
+// Test data source: testdata/steps/{map-delivery,hidden-peers,route-announce,
+// policy-generation}.hujson, loaded by name. The disco rotation of
+// map-delivery also pins a row of TestBuildMapRequestChangeResponse in
+// ../state; policy-generation is also read by TestPolicySequenceCompat in
+// ../policy/v2.
 // Source format:    github.com/juanfont/headscale/hscontrol/types/testcapture
 
 package servertest_test
@@ -17,6 +20,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"maps"
+	"net/http"
 	"net/netip"
 	"slices"
 	"strconv"
@@ -123,6 +127,33 @@ var mapStreamSkipReasons = map[string]string{
 	"route-announce/RA9/rtr/self-routes":  selfRoutesReduced,
 	"route-announce/RA10/rtr/self-routes": selfRoutesReduced,
 	"route-announce/RA11/rtr/self-routes": selfRoutesReduced,
+
+	"policy-generation/G2/thor-1/peers":       sameUserPeers,
+	"policy-generation/G2/thor-2/peers":       sameUserPeers,
+	"policy-generation/G3/thor-1/peers":       sameUserPeers,
+	"policy-generation/G3/thor-2/peers":       sameUserPeers,
+	"policy-generation/G4/thor-1/peers":       sameUserPeers,
+	"policy-generation/G4/thor-2/peers":       sameUserPeers,
+	"policy-generation/G2/thor-1/peer-owners": sameUserPeers,
+	"policy-generation/G2/thor-2/peer-owners": sameUserPeers,
+	"policy-generation/G3/thor-1/peer-owners": sameUserPeers,
+	"policy-generation/G3/thor-2/peer-owners": sameUserPeers,
+	"policy-generation/G4/thor-1/peer-owners": sameUserPeers,
+	"policy-generation/G4/thor-2/peer-owners": sameUserPeers,
+
+	"policy-generation/G2/freya-1/filter": filterDstMerged,
+	"policy-generation/G2/thor-2/filter":  filterDstMerged,
+	"policy-generation/G3/freya-1/filter": filterDstMerged,
+	"policy-generation/G3/thor-2/filter":  filterDstMerged,
+	"policy-generation/G4/freya-1/filter": filterDstMerged,
+	"policy-generation/G4/thor-2/filter":  filterDstMerged,
+	"policy-generation/G5/freya-1/filter": filterDstMerged,
+	"policy-generation/G5/thor-2/filter":  filterDstMerged,
+
+	"policy-generation/G4/rt/enabled": withdrawnRouteEnabled,
+	"policy-generation/G5/rt/enabled": withdrawnRouteEnabled,
+	"policy-generation/G7/rt/enabled": withdrawnRouteEnabled,
+	"policy-generation/G8/rt/enabled": withdrawnRouteEnabled,
 }
 
 const (
@@ -136,6 +167,7 @@ const (
 	peerServicesWhole         = "PEER_SERVICES_WHOLE: headscale sends peers every service a node reports; SaaS leaves out peerapi-dns-proxy"
 	removalSentToAll          = "REMOVAL_SENT_TO_ALL: headscale tells every connected node that a deleted node is gone; SaaS tells only the nodes that had it as a peer"
 	announcePolicyResend      = "ANNOUNCE_POLICY_RESEND: under an autogroup:self or via policy, headscale sends every connected node a policy response when a router changes only routes nobody approved; SaaS sends nothing to nodes that cannot see the router"
+	withdrawnRouteEnabled     = "WITHDRAWN_ROUTE_ENABLED: headscale's routes API keeps a route the device stopped advertising in enabledRoutes; SaaS lists only advertised routes as enabled"
 )
 
 // mapStreamNotCompared lists, by key prefix, checks left out: SaaS's answer
@@ -159,6 +191,11 @@ var mapStreamNotCompared = []string{
 	"route-announce/RA11/observer/presence",
 	"route-announce/RA11/srv/presence",
 	"route-announce/RA11/witness/presence",
+
+	"policy-generation/G7/rt-b/filter",
+	"policy-generation/G7/rt-plain/filter",
+	"policy-generation/G7/srv/filter",
+	"policy-generation/G7/srv/ssh",
 }
 
 // compared drops the checks mapStreamNotCompared leaves out.
@@ -502,16 +539,7 @@ type deliveryStep struct {
 // on; and, per node, whether it was sent anything.
 func deliveryChecks(d *deliveryStep) []stepCheck {
 	r := d.r
-	saasNames := map[netip.Addr]string{}
-
-	for alias, n := range r.c.Topology.Nodes {
-		for _, s := range []string{n.IPv4, n.IPv6} {
-			addr, err := netip.ParseAddr(s)
-			if err == nil {
-				saasNames[addr] = alias
-			}
-		}
-	}
+	saasNames := topologyAddrNames(r.c)
 
 	var checks []stepCheck
 
@@ -618,6 +646,23 @@ func (d *deliveryStep) subjectChecks(x, v string) []stepCheck {
 	}
 }
 
+// topologyAddrNames names each SaaS address in c's topology by alias, so a
+// rule naming a device that has since left still reads by name.
+func topologyAddrNames(c *testcapture.Capture) map[netip.Addr]string {
+	out := map[netip.Addr]string{}
+
+	for alias, n := range c.Topology.Nodes {
+		for _, s := range []string{n.IPv4, n.IPv6} {
+			addr, err := netip.ParseAddr(s)
+			if err == nil {
+				out[addr] = alias
+			}
+		}
+	}
+
+	return out
+}
+
 // stepSubjects names the nodes the step acted on, by action or device path.
 func stepSubjects(step testcapture.Step) []string {
 	set := map[string]bool{}
@@ -910,7 +955,7 @@ func TestRouteAnnouncementCompat(t *testing.T) {
 func TestMapStreamSkipEntriesNameCaptures(t *testing.T) {
 	t.Parallel()
 
-	requireSkipCaptures(t, mapStreamSkipReasons, "map-delivery", "hidden-peers", "route-announce")
+	requireSkipCaptures(t, mapStreamSkipReasons, "map-delivery", "hidden-peers", "route-announce", "policy-generation")
 }
 
 // visibilityMark is what a client had been sent when a step began.
@@ -1143,3 +1188,158 @@ func (r *stepReplay) hsRemovedPeers(cl *servertest.TestClient, mark visibilityMa
 
 	return sortedStrings(slices.Collect(maps.Keys(set)))
 }
+
+// TestPolicySequenceMapCompat replays one policy while members join, a
+// second via router joins and the first withdraws its routes, the admin tags
+// one member's device and deletes another, then the group the policy names
+// in ACLs, a via grant, SSH and autoApprovers is edited. After each step it
+// compares, per captured node, its peers, whom it sees each owned by, which
+// peers serve each route, and its filter and SSH rules by device; and per
+// device, its owner and routes. TestPolicySequenceCompat in ../policy/v2
+// compares the same steps at the policy layer.
+func TestPolicySequenceMapCompat(t *testing.T) {
+	t.Parallel()
+
+	t.Run("policy-generation", func(t *testing.T) {
+		t.Parallel()
+
+		replayPolicySequence(t, "policy-generation")
+	})
+}
+
+func replayPolicySequence(t *testing.T, id string) {
+	t.Helper()
+
+	c := readStepCapture(t, id)
+	r := newStepReplay(t, c)
+	saasNames := topologyAddrNames(c)
+	evaluated := map[string]bool{}
+
+	// Kept past a deletion, as saasNames are, so a deleted device's
+	// addresses still read as that device in rules that keep them.
+	hsNames := map[netip.Addr]string{}
+	names := func() map[netip.Addr]string {
+		maps.Copy(hsNames, r.addrNames())
+
+		return hsNames
+	}
+
+	for _, step := range c.Steps {
+		require.Equal(t, "observed", step.Result, "%s: the replay follows every step", step.Name)
+
+		// The probe let each action land before the next, and which router
+		// serves a route depends on join order.
+		for _, a := range step.Actions {
+			one := step
+			one.Actions = []testcapture.Action{a}
+			require.Empty(t, r.apply(t, one), "%s: every login succeeds", step.Name)
+			r.waitSession(t, step.Name, a)
+		}
+
+		t.Run(step.Name, func(t *testing.T) {
+			prefix := id + "/" + step.Name + "/"
+			holdChecks(t, mapStreamSkipReasons, prefix, compared(prefix, sequenceChecks(t, r, step, saasNames, names)), evaluated)
+		})
+	}
+
+	for k := range mapStreamSkipReasons {
+		if strings.HasPrefix(k, id+"/") && !evaluated[k] {
+			t.Errorf("skip entry %q names no check", k)
+		}
+	}
+}
+
+// sequenceChecks compares, per node SaaS captured in the step, its peers,
+// their owners and routes, and its filter and SSH rules with devices named
+// by saasNames and hsNames; per device in the step's device list, its owner;
+// and per device whose routes the step read, the routes API.
+func sequenceChecks(
+	t *testing.T,
+	r *stepReplay,
+	step testcapture.Step,
+	saasNames map[netip.Addr]string,
+	hsNames func() map[netip.Addr]string,
+) []stepCheck {
+	t.Helper()
+
+	var checks []stepCheck
+
+	for _, x := range slices.Sorted(maps.Keys(step.Captures)) {
+		saas, cl := step.Captures[x].Netmap, r.clients[x]
+		require.NotNil(t, saas, "%s: no netmap of %s", step.Name, x)
+		require.NotNil(t, cl, "%s: %s never joined", step.Name, x)
+
+		checks = append(checks,
+			stepCheck{x + "/peers", peerAliases(saas, r.saasAlias), func() string { return peerAliases(cl.Netmap(), r.hsAlias) }},
+			stepCheck{x + "/peer-owners", peerOwners(saas, r.saasAlias), func() string { return peerOwners(cl.Netmap(), r.hsAlias) }},
+			stepCheck{x + "/route-owner", routeOwners(saas, r.saasAlias), func() string { return routeOwners(cl.Netmap(), r.hsAlias) }},
+			stepCheck{x + "/filter", renderMatches(saas, saasNames), func() string { return renderMatches(cl.Netmap(), hsNames()) }},
+			stepCheck{x + "/ssh", renderSSHRules(saas, saasNames), func() string { return renderSSHRules(cl.Netmap(), hsNames()) }},
+		)
+	}
+
+	for _, d := range stepDevices(t, step) {
+		x := deviceLabelAlias(d.ID)
+		checks = append(checks, stepCheck{x + "/owner", saasOwner(d), func() string { return renderOwners(r.hsViews(x)) }})
+	}
+
+	for _, a := range step.Actions {
+		if a.Kind != "api" || a.Method != http.MethodGet || !strings.HasSuffix(a.Path, "/routes") {
+			continue
+		}
+
+		x := deviceAlias(t, a.Path)
+
+		var want deviceRoutes
+		require.NoError(t, json.Unmarshal(a.Response, &want), "%s: %s", step.Name, a.Path)
+
+		hs := func(pick func(deviceRoutes) []string) func() string {
+			return func() string {
+				got, ok := r.hsDeviceRoutes(t, x)
+				if !ok {
+					return "no single device"
+				}
+
+				return sortedStrings(pick(got))
+			}
+		}
+
+		checks = append(checks,
+			stepCheck{x + "/advertised", sortedStrings(want.Advertised), hs(func(d deviceRoutes) []string { return d.Advertised })},
+			stepCheck{x + "/enabled", sortedStrings(want.Enabled), hs(func(d deviceRoutes) []string { return d.Enabled })},
+		)
+	}
+
+	return checks
+}
+
+// peerOwners gives, per peer in nm that alias names, its tags and the login
+// of the user it is shown under, SaaS emails mapped to user names.
+func peerOwners(nm *netmap.NetworkMap, alias func(tailcfg.NodeID) (string, bool)) string {
+	if nm == nil {
+		return "no netmap"
+	}
+
+	var out []string
+
+	for _, p := range nm.Peers {
+		name, ok := alias(p.ID())
+		if !ok {
+			continue
+		}
+
+		login := "no profile"
+		if prof, ok := nm.UserProfiles[p.User()]; ok {
+			login = prof.LoginName()
+			if user, ok := captureUserNames[login]; ok {
+				login = user
+			}
+		}
+
+		out = append(out, fmt.Sprintf("%s %s as %s", name, sortedStrings(p.Tags().AsSlice()), login))
+	}
+
+	slices.Sort(out)
+
+	return strings.Join(out, "; ")
+}

hscontrol/servertest/via_compat_test.go

@@ -114,10 +114,8 @@ var viaCompatSkipReasons = map[string]string{
 	"via-grant-v67/rename/blastoise/packet-filter": viaSourceLossNotSent,
 	"via-grant-v67/rename/ivysaur/peers":           viaSourceLossNotSent,
 
-	// Adjacent member addresses merge into one prefix, and the reduced
-	// filter keeps all of it; SaaS lists only the node's own address.
-	"routes-f2-subnet-as-acl-source/ivysaur/packet-filter":  "FILTER_DST_MERGED: own filter names a neighbour's address",
-	"routes-f2-subnet-as-acl-source/venusaur/packet-filter": "FILTER_DST_MERGED: own filter names a neighbour's address",
+	"routes-f2-subnet-as-acl-source/ivysaur/packet-filter":  filterDstMerged,
+	"routes-f2-subnet-as-acl-source/venusaur/packet-filter": filterDstMerged,
 
 	// SaaS sends a router every approved route in its own AllowedIPs and
 	// every route it is primary for in PrimaryRoutes.
@@ -269,6 +267,10 @@ const viaSourceLossNotSent = "VIA_SOURCE_LOSS_NOT_SENT: a user change that leave
 
 const exitRoutesOnOverlap = "EXIT_ROUTES_ON_OVERLAP: headscale offers a peer's exit routes to a viewer whose rules reach only nodes or subnets; SaaS leaves them out"
 
+// Adjacent member addresses merge into one prefix, and the reduced filter
+// keeps all of it; SaaS lists only the node's own address.
+const filterDstMerged = "FILTER_DST_MERGED: own filter names a neighbour's address"
+
 const sameUserPeers = "SAME_USER_PEERS: SaaS lists a user's own devices as each other's peers with no rule linking them; headscale peers only what the policy links"
 
 const selfRoutesReduced = "SELF_ROUTES_REDUCED: headscale sends a router only the primary routes its policy lets it reach"

hscontrol/servertest/via_ha_compat_test.go

@@ -1078,6 +1078,10 @@ func (r *stepReplay) api(t *testing.T, stepName string, i int, a testcapture.Act
 		// Headscale reads DNS from its config, which a consumer sets from
 		// the capture's input; it has no API to change it.
 
+	case a.Method == http.MethodPost && a.Path == "/tailnet/-/acl/validate":
+		// A validation changes nothing, and headscale's v2 API has none;
+		// TestAPIv2ACLCompat in .. lists that difference.
+
 	case a.Method == http.MethodPost && a.Path == "/tailnet/-/acl":
 		require.Equal(t, http.StatusOK, a.Status, "%s: replay of a refused policy", stepName)
 		r.setPolicy(t, stepName, string(a.Body))