// This file replays map stream captures from Tailscale SaaS: what each node // is sent when a device is retagged, when a running peer changes its DERP // home, disco key, hostname or port, when a node goes down and up around // policy pushes, and which Hostinfo fields peers and the node itself get. // // Test data source: testdata/steps/map-delivery.hujson, loaded by name. Its // disco rotation also pins a row of TestBuildMapRequestChangeResponse in // ../state. // Source format: github.com/juanfont/headscale/hscontrol/types/testcapture package servertest_test import ( "context" "encoding/json" "fmt" "maps" "net/netip" "slices" "strconv" "strings" "testing" "time" "github.com/juanfont/headscale/hscontrol/types/testcapture" "github.com/stretchr/testify/require" "tailscale.com/tailcfg" "tailscale.com/types/netmap" ) // mapStreamSkipReasons lists checks where headscale differs from SaaS, keyed // ///. A listed check that matches fails, so a // fix must delete its entry. var mapStreamSkipReasons = map[string]string{ "map-delivery/S1.1/appc/frames": policyResponseToEveryNode, "map-delivery/S1.1/router/frames": policyResponseToEveryNode, "map-delivery/S1.1/sshd/frames": policyResponseToEveryNode, "map-delivery/S1.2/appc/frames": policyResponseToEveryNode, "map-delivery/S1.2/router/frames": policyResponseToEveryNode, "map-delivery/S1.2/sshd/frames": policyResponseToEveryNode, "map-delivery/S1.3/appc/frames": policyResponseToEveryNode, "map-delivery/S1.3/router/frames": policyResponseToEveryNode, "map-delivery/S1.3/sshd/frames": policyResponseToEveryNode, "map-delivery/S1.4/appc/frames": policyResponseToEveryNode, "map-delivery/S1.4/router/frames": policyResponseToEveryNode, "map-delivery/S1.4/sshd/frames": policyResponseToEveryNode, "map-delivery/S3.5-remove/db1/frames": policyResponseToEveryNode, "map-delivery/S3.5-restore/appc/frames": policyResponseToEveryNode, "map-delivery/S3.5-restore/db1/frames": policyResponseToEveryNode, "map-delivery/S3.5-restore/sshd/frames": policyResponseToEveryNode, "map-delivery/S2.1/laptop/frames": ownReportEchoed, "map-delivery/S2.3/laptop/frames": ownReportEchoed, "map-delivery/S2.5/laptop/self-endpoints": ownReportEchoed, "map-delivery/S2.5/laptop/endpoints-gap-bystander": restartEndpointsKept, "map-delivery/S2.5/laptop/endpoints-gap-svc": restartEndpointsKept, "map-delivery/S3.1/appc/frames": reconnectAnnounced, "map-delivery/S3.1/bystander/frames": reconnectAnnounced, "map-delivery/S3.1/router/frames": reconnectAnnounced, "map-delivery/S3.1/sshd/frames": reconnectAnnounced, "map-delivery/S3.1/svc/frames": reconnectAnnounced, "map-delivery/S3.4/appc/frames": routerReconnectRecompute, "map-delivery/S3.4/bystander/frames": routerReconnectRecompute, "map-delivery/S3.4/db1/frames": routerReconnectRecompute, "map-delivery/S3.4/laptop/frames": routerReconnectRecompute, "map-delivery/S3.4/sshd/frames": routerReconnectRecompute, "map-delivery/S3.4/svc/frames": routerReconnectRecompute, "map-delivery/S3.6/bystander/peers": sameUserPeers, "map-delivery/S3.6/laptop/peers": sameUserPeers, "map-delivery/S2.0/appc/self-hostinfo-keys": selfHostinfoWhole, "map-delivery/S2.0/bystander/self-hostinfo-keys": selfHostinfoWhole, "map-delivery/S2.0/sshd/self-hostinfo-keys": selfHostinfoWhole, "map-delivery/S2.0/appc/peer-hostinfo-keys": peerHostinfoWhole, "map-delivery/S2.0/bystander/peer-hostinfo-keys": peerHostinfoWhole, "map-delivery/S2.0/sshd/peer-hostinfo-keys": peerHostinfoWhole, "map-delivery/S2.0/appc/peer-hostinfo": peerServicesWhole, "map-delivery/S2.0/bystander/peer-hostinfo": peerServicesWhole, "map-delivery/S2.0/sshd/peer-hostinfo": peerServicesWhole, } const ( policyResponseToEveryNode = "POLICY_RESPONSE_TO_EVERY_NODE: headscale sends every connected node a policy response when tags or the policy change; SaaS sends only the nodes whose peers or filter change" ownReportEchoed = "OWN_REPORT_ECHOED: headscale sends a node its own endpoint or DERP report back; SaaS sends it nothing, so its own entry keeps the endpoints of its last full map" restartEndpointsKept = "RESTART_ENDPOINTS_KEPT: headscale keeps a restarted node's last endpoints at its peers until it reports new ones; SaaS sends peers the node without endpoints until then" reconnectAnnounced = "RECONNECT_ANNOUNCED: headscale tells peers a node is online after a stream gap of a few seconds; SaaS sends them nothing" routerReconnectRecompute = "ROUTER_RECONNECT_RECOMPUTE: headscale sends every node a policy response when a subnet router's stream comes back after a few seconds; SaaS sends nothing" selfHostinfoWhole = "SELF_HOSTINFO_WHOLE: headscale sends a node its whole reported Hostinfo; SaaS sends only Hostname, RequestTags and Services" peerHostinfoWhole = "PEER_HOSTINFO_WHOLE: headscale sends peers a node's whole reported Hostinfo; SaaS sends only Hostname, OS, Services, the SSH host keys and AppConnector" peerServicesWhole = "PEER_SERVICES_WHOLE: headscale sends peers every service a node reports; SaaS leaves out peerapi-dns-proxy" ) // TestMapDeliveryCompat replays a device retagged through the API, a running // peer that picks new DERP homes, rotates its disco key, renames itself and // restarts on another port, down and up gaps of a node and a router, a rule // removed and restored around a reconnect, and a policy with no rules. After // each scenario step it compares, per captured node, its peers, packet // filter, tags and own endpoints; per viewer, how it sees the step's subject; // and, per node, whether it was sent anything at all. func TestMapDeliveryCompat(t *testing.T) { t.Parallel() t.Run("map-delivery", func(t *testing.T) { t.Parallel() replayMapDelivery(t, "map-delivery", "", deliveryChecks, func(key string) bool { return !strings.Contains(key, "hostinfo") }) }) } // TestPeerHostinfoCompat replays the same capture up to its quiet step, with // sshd running Tailscale SSH and appc advertising an app connector, and // compares the Hostinfo fields and values each captured node gets for its // peers and for itself. func TestPeerHostinfoCompat(t *testing.T) { t.Parallel() t.Run("map-delivery", func(t *testing.T) { t.Parallel() replayMapDelivery(t, "map-delivery", hostinfoStep, hostinfoChecks, func(key string) bool { return strings.Contains(key, "hostinfo") }) }) } // hostinfoStep is the quiet step whose captures hold the Hostinfo snapshots. const hostinfoStep = "S2.0" // settledGap is how far apart two actions must be sent for SaaS to have // handled them one after the other rather than in one batch. const settledGap = time.Second // replayMapDelivery replays capture id step by step, through the step named // last (every step when empty), and holds the checks of each scenario step // against [mapStreamSkipReasons]. Setup steps (policy, keys, joins, the steady // wait) are replayed, not compared: their join bursts and SaaS's own stream // restarts are not what the scenarios ask about. owns picks the skip entries // this caller's checks evaluate. func replayMapDelivery( t *testing.T, id, last string, checks func(d *deliveryStep) []stepCheck, owns func(key string) bool, ) { t.Helper() c := readStepCapture(t, id) r := newStepReplay(t, c) evaluated := map[string]bool{} saasLast := map[string]*netmap.NetworkMap{} for alias, n := range c.Captures { saasLast[alias] = n.Netmap } for _, step := range c.Steps { require.Equal(t, "observed", step.Result, "%s: the replay follows every step", step.Name) r.settle(t) d := &deliveryStep{ r: r, step: step, caps: step.Captures, saasBefore: maps.Clone(saasLast), hsBefore: map[string]*netmap.NetworkMap{}, updates: map[string]int{}, } // The start check has no captures of its own: it is the baseline. if len(d.caps) == 0 { d.caps = c.Captures } for alias, cl := range r.clients { d.hsBefore[alias] = cl.Netmap() d.updates[alias] = cl.UpdateCount() } r.replayDelivery(t, step) if strings.HasPrefix(step.Name, "S") { cs := checks(d) prefix := id + "/" + step.Name + "/" if len(cs) > 0 { t.Run(step.Name, func(t *testing.T) { holdChecks(t, mapStreamSkipReasons, prefix, cs, evaluated) }) } } for alias, n := range step.Captures { saasLast[alias] = n.Netmap } if step.Name == last { break } } for k := range mapStreamSkipReasons { if strings.HasPrefix(k, id+"/") && owns(k) && !evaluated[k] { t.Errorf("skip entry %q names no check", k) } } } // replayDelivery applies the step's actions one at a time. What a node // reported is replayed from its captured map requests, so the CLI lever SaaS // was driven with does not decide what headscale sees. Headscale settles // where SaaS waited or sent the next action a while later; actions sent // together stay together. func (r *stepReplay) replayDelivery(t *testing.T, step testcapture.Step) { t.Helper() ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() reports := stepReports(t, step) for i, a := range step.Actions { switch a.Kind { case "wait": r.settle(t) case "cli", "restart": r.deliveryDaemon(ctx, t, step, a) default: one := step one.Actions = []testcapture.Action{a} require.Empty(t, r.apply(t, one), "%s: every login succeeds", step.Name) } if reqs, ok := reports[i]; ok { r.report(ctx, t, a.Node, reqs) } // A wait on either side settles anyway. if i+1 < len(step.Actions) { next := step.Actions[i+1] if a.Kind != "wait" && next.Kind != "wait" && time.Duration(next.OffsetMS-a.OffsetMS)*time.Millisecond >= settledGap { r.settle(t) } } } } // deliveryDaemon replays what a node's tailscaled did around its reports: a // restart, a disco key rotation, or its stream going down and coming back. func (r *stepReplay) deliveryDaemon(ctx context.Context, t *testing.T, step testcapture.Step, a testcapture.Action) { t.Helper() cl := r.clients[a.Node] switch args := strings.Join(a.Args, " "); { case a.Kind == "restart": // The daemon starts again on the Hostinfo of its first request, with // no endpoints or DERP home until it reports them. var hi *tailcfg.Hostinfo for _, f := range step.Frames[a.Node] { if f.Request != nil && f.Request.Hostinfo != nil { hi = f.Request.Hostinfo.Clone() break } } require.NotNil(t, hi, "%s: no request from restarted %s", step.Name, a.Node) require.NoError(t, cl.Restart(ctx, hi), "%s: restart", step.Name) case args == "down": cl.Disconnect(t) case args == "up": // tailscaled registers again on its node key, then polls. require.NoError(t, cl.ReloginAndPoll(ctx), "%s: up", step.Name) case args == "debug rotate-disco-key": require.NoError(t, cl.RotateDisco(ctx), "%s: disco rotation", step.Name) // SaaS got the node's next report after the new stream's first map. // Sent sooner, it can reach the server ahead of the stream request, // whose older Hostinfo then overwrites it. require.Eventually(t, func() bool { _, maps := cl.Stream() return maps > 0 }, 10*time.Second, 10*time.Millisecond, "%s: new stream answered", step.Name) case args == "debug pick-new-derp", args == "set --ssh", args == "set --advertise-connector", strings.HasPrefix(args, "set --hostname="): // Only what the node reports changes. default: t.Fatalf("%s: no replay for %s %q", step.Name, a.Kind, a.Args) } } // stepReports hands each node's captured map requests, other than stream // starts, to the actions after which it reported, by action index: a node's // only reporting action takes them all, and several take one each. func stepReports(t *testing.T, step testcapture.Step) map[int][]*tailcfg.MapRequest { t.Helper() actions := map[string][]int{} for i, a := range step.Actions { switch { case a.Kind == "join", a.Kind == "restart", a.Kind == "cli" && strings.Join(a.Args, " ") != "down": actions[a.Node] = append(actions[a.Node], i) } } out := map[int][]*tailcfg.MapRequest{} for node, idx := range actions { var reqs []*tailcfg.MapRequest for _, f := range step.Frames[node] { if f.Request != nil && !f.Request.Stream { reqs = append(reqs, f.Request) } } switch { case len(idx) == 1: out[idx[0]] = reqs case len(idx) == len(reqs): for k, i := range idx { out[i] = reqs[k : k+1] } default: t.Fatalf("%s: %d reporting actions on %s, %d requests", step.Name, len(idx), node, len(reqs)) } } return out } // report sends what the node told SaaS, request by request: its Hostinfo, and // its DERP home and endpoints where the request carried them. A request that // changes nothing is not sent. func (r *stepReplay) report(ctx context.Context, t *testing.T, alias string, reqs []*tailcfg.MapRequest) { t.Helper() d := r.clients[alias].Direct() for _, req := range reqs { changed := false if hi := req.Hostinfo; hi != nil { changed = d.SetHostinfo(hi) if hi.NetInfo != nil { changed = d.SetNetInfo(hi.NetInfo) || changed } } if len(req.Endpoints) > 0 { eps := make([]tailcfg.Endpoint, len(req.Endpoints)) for i, ap := range req.Endpoints { eps[i].Addr = ap if i < len(req.EndpointTypes) { eps[i].Type = req.EndpointTypes[i] } } changed = d.SetEndpoints(eps) || changed } if changed { require.NoError(t, d.SendUpdate(ctx), "%s: report", alias) } } } // settle waits until no client has been sent a map response for a while, so // a batch in flight lands before the next action. func (r *stepReplay) settle(t *testing.T) { t.Helper() const quiet = 300 * time.Millisecond counts := func() map[string]int { out := make(map[string]int, len(r.clients)) for alias, cl := range r.clients { out[alias] = cl.UpdateCount() } return out } last, since := counts(), time.Now() tick := time.NewTicker(25 * time.Millisecond) defer tick.Stop() for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); <-tick.C { now := counts() if !maps.Equal(now, last) { last, since = now, time.Now() continue } if time.Since(since) >= quiet { return } } t.Fatalf("map streams did not settle: %v", last) } // deliveryStep is one replayed step, what SaaS showed after it and what both // sides showed before it. type deliveryStep struct { r *stepReplay step testcapture.Step // caps is SaaS's end state; saasBefore each node's last capture before // the step. caps map[string]testcapture.Node saasBefore map[string]*netmap.NetworkMap // hsBefore and updates are each client's netmap and update count, which // is its history's length, when the step began. hsBefore map[string]*netmap.NetworkMap updates map[string]int } // deliveryChecks compares, per captured node, its peers, filter, tags and // own endpoints; per captured viewer, how it sees each node the step acted // on; and, per node, whether it was sent anything. func deliveryChecks(d *deliveryStep) []stepCheck { r := d.r saasNames := map[netip.Addr]string{} for alias, n := range r.c.Topology.Nodes { for _, s := range []string{n.IPv4, n.IPv6} { addr, err := netip.ParseAddr(s) if err == nil { saasNames[addr] = alias } } } var checks []stepCheck for _, x := range slices.Sorted(maps.Keys(r.c.NodeIDs)) { cl, ok := r.clients[x] if !ok { continue } // A client without delta handling rebuilds its netmap, so its update // count moves, on every map response but a keep-alive. checks = append(checks, stepCheck{x + "/frames", saasDelivered(d.step.Frames[x]), func() string { if cl.UpdateCount() > d.updates[x] { return "frame" } return "silent" }}) } captured := slices.Sorted(maps.Keys(d.caps)) for _, x := range captured { saas, cl := d.caps[x].Netmap, r.clients[x] if saas == nil || cl == nil { continue } checks = append(checks, stepCheck{x + "/peers", peerAliases(saas, r.saasAlias), func() string { return peerAliases(cl.Netmap(), r.hsAlias) }}, stepCheck{x + "/filter", renderMatches(saas, saasNames), func() string { return renderMatches(cl.Netmap(), r.addrNames()) }}, stepCheck{x + "/tags", selfTags(saas), func() string { return selfTags(cl.Netmap()) }}, stepCheck{x + "/self-endpoints", selfEndpoints(saas), func() string { return selfEndpoints(cl.Netmap()) }}, ) } for _, x := range stepSubjects(d.step) { for _, v := range captured { if v != x && d.caps[v].Netmap != nil && r.clients[v] != nil { checks = append(checks, d.subjectChecks(x, v)...) } } } return checks } // subjectChecks compares how v sees x: online, its DERP home, endpoints, // name, routes, whether its disco key changed during the step and whether // its endpoints were cleared on the way. func (d *deliveryStep) subjectChecks(x, v string) []stepCheck { r := d.r saasID := r.c.NodeIDs[x] saasPeer, saasMissing := peerByID(d.caps[v].Netmap, saasID) hsID := func() (tailcfg.NodeID, string) { nodes := r.machineNodes(x) if len(nodes) != 1 { return 0, fmt.Sprintf("devices %d", len(nodes)) } return nodes[0].ID().NodeID(), "" } hsPeer := func() (tailcfg.NodeView, string) { id, missing := hsID() if missing != "" { return tailcfg.NodeView{}, missing } return peerByID(r.clients[v].Netmap(), id) } hsBefore := "" if id, missing := hsID(); missing == "" { hsBefore = peerDisco(d.hsBefore[v], id) } return []stepCheck{ {x + "/online-" + v, peerOnline(saasPeer, saasMissing), func() string { return peerOnline(hsPeer()) }}, {x + "/derp-" + v, peerDERP(saasPeer, saasMissing), func() string { return peerDERP(hsPeer()) }}, {x + "/endpoints-" + v, peerEndpoints(saasPeer, saasMissing), func() string { return peerEndpoints(hsPeer()) }}, {x + "/name-" + v, peerLabel(saasPeer, saasMissing), func() string { return peerLabel(hsPeer()) }}, {x + "/routes-" + v, servedRoutes(saasPeer, saasMissing), func() string { return servedRoutes(hsPeer()) }}, { x + "/disco-" + v, discoState(peerDisco(d.saasBefore[v], saasID), peerDisco(d.caps[v].Netmap, saasID)), func() string { p, missing := hsPeer() if missing != "" { return discoState(hsBefore, missing) } return discoState(hsBefore, p.DiscoKey().String()) }, }, {x + "/endpoints-gap-" + v, saasEndpointsCleared(d.step.Frames[v], saasID), func() string { id, missing := hsID() if missing != "" { return missing } return hsEndpointsCleared(r.clients[v].History()[d.updates[v]:], id) }}, } } // stepSubjects names the nodes the step acted on, by action or device path. func stepSubjects(step testcapture.Step) []string { set := map[string]bool{} for _, a := range step.Actions { switch { case a.Node != "": set[a.Node] = true case strings.HasPrefix(a.Path, "/device/{device:"): alias, _, _ := strings.Cut(strings.TrimPrefix(a.Path, "/device/{device:"), "}") set[alias] = true } } return slices.Sorted(maps.Keys(set)) } // saasDelivered says whether a node's stream carried any map response but a // keep-alive during the step. func saasDelivered(frames []testcapture.Frame) string { for _, f := range frames { if f.Response != nil && !f.Response.KeepAlive { return "frame" } } return "silent" } // saasEndpointsCleared says whether a frame left the viewer holding peer id // without endpoints: a whole node with none, or a patch that empties them. func saasEndpointsCleared(frames []testcapture.Frame, id tailcfg.NodeID) string { for _, f := range frames { resp := f.Response if resp == nil { continue } for _, n := range slices.Concat(resp.Peers, resp.PeersChanged) { if n.ID == id && len(n.Endpoints) == 0 { return "cleared" } } for i, p := range resp.PeersChangedPatch { if p.NodeID == id && f.Empty[fmt.Sprintf("PeersChangedPatch[%d].Endpoints", i)] != "" { return "cleared" } } } return "kept" } // hsEndpointsCleared says whether any of the netmaps held peer id without // endpoints. func hsEndpointsCleared(history []*netmap.NetworkMap, id tailcfg.NodeID) string { for _, nm := range history { if p, missing := peerByID(nm, id); missing == "" && p.Endpoints().Len() == 0 { return "cleared" } } return "kept" } func peerDERP(p tailcfg.NodeView, missing string) string { if missing != "" { return missing } return strconv.FormatInt(int64(p.HomeDERP()), 10) } func peerEndpoints(p tailcfg.NodeView, missing string) string { if missing != "" { return missing } return addrPorts(p) } // peerLabel is the first label of the peer's MagicDNS name and its hostname. func peerLabel(p tailcfg.NodeView, missing string) string { if missing != "" { return missing } return peerName(p, "") + " " + p.Hostinfo().Hostname() } func selfTags(nm *netmap.NetworkMap) string { if nm == nil || !nm.SelfNode.Valid() { return "no netmap" } return sortedStrings(nm.SelfNode.Tags().AsSlice()) } func selfEndpoints(nm *netmap.NetworkMap) string { if nm == nil || !nm.SelfNode.Valid() { return "no netmap" } return addrPorts(nm.SelfNode) } func addrPorts(n tailcfg.NodeView) string { out := make([]string, 0, n.Endpoints().Len()) for _, ap := range n.Endpoints().All() { out = append(out, ap.String()) } return sortedStrings(out) } // renderMatches lists nm's packet filter as "sources => ports to // destination" lines, devices named by names. func renderMatches(nm *netmap.NetworkMap, names map[netip.Addr]string) string { if nm == nil { return "no netmap" } var out []string for _, m := range nm.PacketFilter { for _, d := range m.Dsts { dst := d.Net.String() if name, ok := names[d.Net.Addr()]; ok && d.Net.IsSingleIP() { dst = name } out = append(out, filterLine(m.Srcs, names, d.Ports.First, d.Ports.Last)+" to "+dst) } } return filterLines(out) } // hostinfoChecks compares, at the quiet step, each captured node's Hostinfo // for itself and for each peer: which fields it holds, and the values of the // fields SaaS sent. func hostinfoChecks(d *deliveryStep) []stepCheck { if d.step.Name != hostinfoStep { return nil } r := d.r var checks []stepCheck for _, v := range slices.Sorted(maps.Keys(d.caps)) { saas, cl := d.caps[v].Netmap, r.clients[v] if saas == nil || cl == nil { continue } saasSelf := hostinfoFields(saas.SelfNode.Hostinfo()) saasPeers := peerHostinfos(saas, r.saasAlias) checks = append(checks, stepCheck{v + "/self-hostinfo-keys", fieldNames(saasSelf), func() string { nm := cl.Netmap() if nm == nil { return "no netmap" } return fieldNames(hostinfoFields(nm.SelfNode.Hostinfo())) }}, stepCheck{v + "/self-hostinfo", fieldValues(saasSelf, saasSelf), func() string { nm := cl.Netmap() if nm == nil { return "no netmap" } return fieldValues(hostinfoFields(nm.SelfNode.Hostinfo()), saasSelf) }}, stepCheck{v + "/peer-hostinfo-keys", renderPeerFields(saasPeers, nil), func() string { return renderPeerFields(peerHostinfos(cl.Netmap(), r.hsAlias), nil) }}, stepCheck{v + "/peer-hostinfo", renderPeerFields(saasPeers, saasPeers), func() string { return renderPeerFields(peerHostinfos(cl.Netmap(), r.hsAlias), saasPeers) }}, ) } return checks } // hostinfoFields is hi's JSON object, field by field. func hostinfoFields(hi tailcfg.HostinfoView) map[string]json.RawMessage { out := map[string]json.RawMessage{} if !hi.Valid() { return out } b, err := json.Marshal(hi) if err == nil { _ = json.Unmarshal(b, &out) } return out } // peerHostinfos holds the Hostinfo fields of each peer alias names. func peerHostinfos(nm *netmap.NetworkMap, alias func(tailcfg.NodeID) (string, bool)) map[string]map[string]json.RawMessage { out := map[string]map[string]json.RawMessage{} if nm == nil { return out } for _, p := range nm.Peers { if name, ok := alias(p.ID()); ok { out[name] = hostinfoFields(p.Hostinfo()) } } return out } func fieldNames(fields map[string]json.RawMessage) string { return strings.Join(slices.Sorted(maps.Keys(fields)), ",") } // fieldValues renders the fields named in want, as fields holds them. func fieldValues(fields, want map[string]json.RawMessage) string { out := make([]string, 0, len(want)) for _, k := range slices.Sorted(maps.Keys(want)) { v, ok := fields[k] if !ok { v = json.RawMessage("absent") } out = append(out, k+"="+string(v)) } return strings.Join(out, " ") } // renderPeerFields lists each peer's field names or, with want, the values // of the fields want holds for it. func renderPeerFields(peers, want map[string]map[string]json.RawMessage) string { out := make([]string, 0, len(peers)) for _, name := range slices.Sorted(maps.Keys(peers)) { line := fieldNames(peers[name]) if want != nil { line = fieldValues(peers[name], want[name]) } out = append(out, name+": "+line) } return strings.Join(out, "\n") }