state: fold primary route election into NodeStore snapshot

primaryRoutes lived as a separate write-then-read store guarded by an
external mutex. Each Connect, Disconnect, SetApprovedRoutes, and HA
prober write touched both NodeStore and primaryRoutes in sequence,
opening a TOCTOU window in which a stale Disconnect could overwrite a
fresh Connect's route assignment (issue #3203).

The election algorithm is a pure function of the snapshot: every node
contributes its IsOnline + AllApprovedRoutes + a runtime Unhealthy
bit. Move the computation into snapshotFromNodes so it runs in the
NodeStore writer goroutine and recomputes from immutable input on
every batch — no separate primaryRoutes mutation is required.

Caller-side changes are local. GetNodePrimaryRoutes, RoutesForPeer,
the HA prober and the debug endpoint all read PrimaryRoutesForNode /
HANodes / DebugRoutes off NodeStore. Health writes flow through
State.SetNodeUnhealthy, which captures the snapshot's primaries
before/after and signals back whether failover happened. The
SetApprovedRoutes and UpdateNodeFromMapRequest paths use the same
prevPrimaries / maps.Equal pattern to detect when announced/approved
changes shifted a primary.

The per-node lock and connectGen counter that closed the original
TOCTOU window are gone too: Connect bumps a SessionEpoch field on
the node inside its UpdateNode closure, Disconnect re-checks it in
its closure and no-ops on mismatch. The NodeStore writer goroutine
serialises both batches so the check + mutation are atomic by
construction.

Updates #3203
This commit is contained in:
Kristoffer Dalby
2026-04-28 13:33:31 +00:00
parent c68e763e62
commit 5dd622a31b
8 changed files with 224 additions and 208 deletions
+59 -8
View File
@@ -2,11 +2,12 @@ package state
import (
"fmt"
"net/netip"
"slices"
"strings"
"time"
hsdb "github.com/juanfont/headscale/hscontrol/db"
"github.com/juanfont/headscale/hscontrol/routes"
"github.com/juanfont/headscale/hscontrol/types"
"tailscale.com/tailcfg"
)
@@ -132,8 +133,10 @@ func (s *State) DebugOverview() string {
sb.WriteString("\n")
// Route information
routeCount := len(strings.Split(strings.TrimSpace(s.primaryRoutes.String()), "\n"))
if s.primaryRoutes.String() == "" {
primaryStr := s.PrimaryRoutesString()
routeCount := len(strings.Split(strings.TrimSpace(primaryStr), "\n"))
if primaryStr == "" {
routeCount = 0
}
@@ -253,9 +256,55 @@ func (s *State) DebugFilter() ([]tailcfg.FilterRule, error) {
return filter, nil
}
// DebugRoutes returns the current primary routes information as a structured object.
func (s *State) DebugRoutes() routes.DebugRoutes {
return s.primaryRoutes.DebugJSON()
// DebugRoutes returns the current primary routes information as a
// structured object built from the NodeStore snapshot.
func (s *State) DebugRoutes() types.DebugRoutes {
debug := types.DebugRoutes{
AvailableRoutes: make(map[types.NodeID][]netip.Prefix),
PrimaryRoutes: make(map[string]types.NodeID),
}
for _, nv := range s.nodeStore.ListNodes().All() {
if !nv.Valid() {
continue
}
online, known := nv.IsOnline().GetOk()
if !known || !online {
continue
}
approved := nv.AllApprovedRoutes()
if len(approved) == 0 {
continue
}
slices.SortFunc(approved, netip.Prefix.Compare)
debug.AvailableRoutes[nv.ID()] = approved
}
for prefix, id := range s.nodeStore.PrimaryRoutes() {
debug.PrimaryRoutes[prefix.String()] = id
}
var unhealthy []types.NodeID
for _, nv := range s.nodeStore.ListNodes().All() {
if !nv.Valid() {
continue
}
if !s.nodeStore.IsNodeHealthy(nv.ID()) {
unhealthy = append(unhealthy, nv.ID())
}
}
if len(unhealthy) > 0 {
slices.Sort(unhealthy)
debug.UnhealthyNodes = unhealthy
}
return debug
}
// DebugRoutesString returns the current primary routes information as a string.
@@ -322,8 +371,10 @@ func (s *State) DebugOverviewJSON() DebugOverviewInfo {
}
// Route information
routeCount := len(strings.Split(strings.TrimSpace(s.primaryRoutes.String()), "\n"))
if s.primaryRoutes.String() == "" {
primaryStr := s.PrimaryRoutesString()
routeCount := len(strings.Split(strings.TrimSpace(primaryStr), "\n"))
if primaryStr == "" {
routeCount = 0
}