state, policy/v2: elect available routers and retain unavailable routes

Recompute via elections on health changes and check online state and expiry.
With no available router, retain the previous primary or lowest-ID approver;
exclude routers the prober marked unreachable.
This commit is contained in:
Kristoffer Dalby committed 2026-11-02 19:15:40 +01:00
1 parent 25a01070e9
commit e0717bc116
16 files changed
+604 -245

No files matched your search

+7
View File
@@ -63,12 +63,19 @@ type PolicyManager interface {
// avoid a tailnet-wide recompute on every ordinary reconnect.
NodeNeedsPeerRecompute(node types.NodeView) bool
// NodeIsViaTarget reports whether a grant steers traffic through the node.
NodeIsViaTarget(node types.NodeView) bool
// ViaRoutesForPeer computes via grant effects for a viewer-peer pair.
// It returns which routes should be included (peer is via-designated for viewer)
// and excluded (steered to a different peer). When no via grants apply,
// both fields are empty and the caller falls back to existing behavior.
ViaRoutesForPeer(viewer, peer types.NodeView) types.ViaRouteResult
// SetRouterAvailable sets the via election's runtime availability check.
// The callback runs under the policy lock and must remain lock-free.
SetRouterAvailable(available func(types.NodeID) bool)
// NodeCapMap returns compiled policy capabilities for the given node,
// or nil when none target it. The map is a defensive clone; its values
// are shared and read-only. It describes the node's own capabilities,
+49 -4
View File
@@ -118,6 +118,8 @@ type PolicyManager struct {
// nodeAttrsPending mirrors len(nodeAttrsChanged) > 0 so the drain,
// called on every dispatched change, skips pm.mu when idle.
nodeAttrsPending atomic.Bool
routerAvailable func(types.NodeID) bool
}
// filterAndPolicy combines the compiled filter rules with policy content for hashing.
@@ -468,6 +470,22 @@ func (pm *PolicyManager) NodeNeedsPeerRecompute(node types.NodeView) bool {
return true
}
return pm.isViaTargetLocked(node)
}
// NodeIsViaTarget reports whether a grant steers traffic through node.
func (pm *PolicyManager) NodeIsViaTarget(node types.NodeView) bool {
if pm == nil || !node.Valid() {
return false
}
pm.mu.RLock()
defer pm.mu.RUnlock()
return pm.isViaTargetLocked(node)
}
func (pm *PolicyManager) isViaTargetLocked(node types.NodeView) bool {
for tag := range pm.viaTargetTags {
if node.HasTag(string(tag)) {
return true
@@ -1434,6 +1452,20 @@ func (pm *PolicyManager) NodeCanApproveRoute(node types.NodeView, route netip.Pr
return false
}
// SetRouterAvailable installs the availability check used by the via election.
// Runtime status changes bypass SetNodes; nil treats every router as available.
// The callback runs with pm.mu held and must remain a lock-free snapshot read.
func (pm *PolicyManager) SetRouterAvailable(available func(types.NodeID) bool) {
if pm == nil {
return
}
pm.mu.Lock()
defer pm.mu.Unlock()
pm.routerAvailable = available
}
// ViaRoutesForPeer computes via grant effects for a viewer-peer pair.
// For each via grant where the viewer matches the source, it checks whether the
// peer advertises any of the grant's destination prefixes. If the peer has the
@@ -1607,9 +1639,9 @@ func (pm *PolicyManager) ViaRoutesForPeer(viewer, peer types.NodeView) types.Via
if len(result.Include) > 0 || len(result.Exclude) > 0 {
// Multi-router via election: when a via grant's tag matches
// multiple peers advertising the same prefix, only the
// lowest-ID peer (the via-group primary) keeps the prefix in
// Include. The others move to Exclude. This mirrors HA
// primary election scoped to the via tag group.
// lowest-ID available peer keeps the prefix in Include. The others
// move to Exclude. With no available router, the lowest ID keeps the
// prefix so the viewer still gets exactly one route.
//
// Unlike the global [tailcfg.Node.PrimaryRoutes] election
// (routes/primary.go), which picks one primary across ALL
@@ -1639,12 +1671,21 @@ func (pm *PolicyManager) ViaRoutesForPeer(viewer, peer types.NodeView) types.Via
continue
}
var viaPrimaryID types.NodeID
var viaPrimaryID, viaFallbackID types.NodeID
for _, viaTag := range grant.Via {
for _, node := range pm.nodes.All() {
if node.HasTag(string(viaTag)) &&
slices.Contains(node.SubnetRoutes(), included) {
if viaFallbackID == 0 || node.ID() < viaFallbackID {
viaFallbackID = node.ID()
}
if pm.routerAvailable != nil &&
!pm.routerAvailable(node.ID()) {
continue
}
if viaPrimaryID == 0 || node.ID() < viaPrimaryID {
viaPrimaryID = node.ID()
}
@@ -1652,6 +1693,10 @@ func (pm *PolicyManager) ViaRoutesForPeer(viewer, peer types.NodeView) types.Via
}
}
if viaPrimaryID == 0 {
viaPrimaryID = viaFallbackID
}
if viaPrimaryID != 0 && peer.ID() != viaPrimaryID {
result.Include = slices.DeleteFunc(
result.Include,
+74
View File
@@ -2754,6 +2754,80 @@ func TestViaRoutesForPeer(t *testing.T) {
require.Empty(t, result.Include)
require.Equal(t, []netip.Prefix{mp("10.0.0.0/24")}, result.Exclude)
})
t.Run("via_election_skips_unavailable_routers", func(t *testing.T) {
t.Parallel()
route := mp("10.0.0.0/24")
nodes := types.Nodes{
{
ID: 1,
Hostname: "viewer",
IPv4: ap("100.64.0.1"),
User: new(users[0]),
UserID: new(users[0].ID),
Tags: []string{"tag:group-a"},
Hostinfo: &tailcfg.Hostinfo{},
},
}
for id := types.NodeID(2); id <= 4; id++ {
nodes = append(nodes, &types.Node{
ID: id,
Hostname: fmt.Sprintf("router-%d", id),
IPv4: ap(fmt.Sprintf("100.64.0.%d", id)),
User: new(users[0]),
UserID: new(users[0].ID),
Tags: []string{"tag:router-a"},
Hostinfo: &tailcfg.Hostinfo{RoutableIPs: []netip.Prefix{route}},
ApprovedRoutes: []netip.Prefix{route},
})
}
pol := `{
"tagOwners": {
"tag:router-a": ["user1@"],
"tag:group-a": ["user1@"]
},
"grants": [{
"src": ["tag:group-a"],
"dst": ["10.0.0.0/24"],
"ip": ["*"],
"via": ["tag:router-a"]
}]
}`
pm, err := NewPolicyManager([]byte(pol), users, nodes.ViewSlice())
require.NoError(t, err)
for _, tt := range []struct {
name string
available []types.NodeID
want types.NodeID
}{
{name: "all available", available: []types.NodeID{2, 3, 4}, want: 2},
{name: "lowest ID down", available: []types.NodeID{3, 4}, want: 3},
{name: "only highest ID left", available: []types.NodeID{4}, want: 4},
{name: "whole group down", want: 2},
{name: "lowest ID recovers", available: []types.NodeID{2, 3, 4}, want: 2},
} {
t.Run(tt.name, func(t *testing.T) {
pm.SetRouterAvailable(func(id types.NodeID) bool {
return slices.Contains(tt.available, id)
})
for _, router := range nodes[1:] {
result := pm.ViaRoutesForPeer(nodes[0].View(), router.View())
if router.ID == tt.want {
require.Equal(t, []netip.Prefix{route}, result.Include)
require.Empty(t, result.Exclude)
} else {
require.Empty(t, result.Include)
require.Equal(t, []netip.Prefix{route}, result.Exclude)
}
}
})
}
})
}
// TestBuildPeerMap_AutogroupInternetMakesExitNodeVisible reproduces
@@ -78,37 +78,30 @@ var expirySkipReasons = map[string]string{
"expiry/A5-advertise/laptop/online-observer": expiredPeerOffline,
"expiry/A5-advertise/laptop/paths-observer": expiredPeerPaths,
"expiry/A5-advertise/laptop/routes-observer": expiredRouterUnserved,
"expiry/A5-advertise/laptop/self": selfExpiredFlags,
"expiry/A7-dns/laptop/dns": nameserversFromConfig,
"expiry/A7-dns/laptop/online-observer": expiredPeerOffline,
"expiry/A7-dns/laptop/paths-observer": expiredPeerPaths,
"expiry/A7-dns/laptop/routes-observer": expiredRouterUnserved,
"expiry/A7-dns/laptop/self": selfExpiredFlags,
"expiry/A7-dns/observer/dns": nameserversFromConfig,
"expiry/A8-policy/laptop/online-observer": expiredPeerOffline,
"expiry/A8-policy/laptop/paths-observer": expiredPeerPaths,
"expiry/A8-policy/laptop/routes-observer": expiredRouterUnserved,
"expiry/A8-policy/laptop/self": selfExpiredFlags,
"expiry/A9-ssh/laptop/online-observer": expiredPeerOffline,
"expiry/A9-ssh/laptop/paths-observer": expiredPeerPaths,
"expiry/A9-ssh/laptop/routes-observer": expiredRouterUnserved,
"expiry/A9-ssh/laptop/self": selfExpiredFlags,
"expiry/A10-reconnect/laptop/online-observer": expiredPeerOffline,
"expiry/A10-reconnect/laptop/paths-observer": expiredPeerPaths,
"expiry/A10-reconnect/laptop/routes-observer": expiredRouterUnserved,
"expiry/A10-reconnect/laptop/self": selfExpiredFlags,
"expiry/A6a-expire-again/laptop/online-observer": expiredPeerOffline,
"expiry/A6a-expire-again/laptop/online-peer1": expiredPeerOffline,
"expiry/A6a-expire-again/laptop/paths-observer": expiredPeerPaths,
"expiry/A6a-expire-again/laptop/paths-peer1": expiredPeerPaths,
"expiry/A6a-expire-again/laptop/routes-observer": expiredRouterUnserved,
"expiry/A6a-expire-again/laptop/routes-peer1": expiredRouterUnserved,
"expiry/A6a-expire-again/laptop/self": selfExpiredFlags,
"expiry/A6b-rotate-disco/laptop/online-observer": expiredPeerOffline,
@@ -117,9 +110,6 @@ var expirySkipReasons = map[string]string{
"expiry/A6b-rotate-disco/laptop/paths-observer": expiredPeerPaths,
"expiry/A6b-rotate-disco/laptop/paths-peer1": expiredPeerPaths,
"expiry/A6b-rotate-disco/laptop/paths-peer3": expiredPeerPaths,
"expiry/A6b-rotate-disco/laptop/routes-observer": expiredRouterUnserved,
"expiry/A6b-rotate-disco/laptop/routes-peer1": expiredRouterUnserved,
"expiry/A6b-rotate-disco/laptop/routes-peer3": expiredRouterUnserved,
"expiry/A6b-rotate-disco/laptop/self": selfExpiredFlags,
}
+136
View File
@@ -3,10 +3,13 @@ package servertest_test
import (
"context"
"net/netip"
"slices"
"testing"
"time"
"github.com/juanfont/headscale/hscontrol/servertest"
"github.com/juanfont/headscale/hscontrol/types"
"github.com/juanfont/headscale/hscontrol/types/change"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"tailscale.com/tailcfg"
@@ -1113,3 +1116,136 @@ func hasDstRules(matches []filtertype.Match) bool {
return false
}
func TestGrantViaSubnetFailover(t *testing.T) {
t.Parallel()
route := netip.MustParsePrefix("10.0.0.0/24")
setup := func(
t *testing.T,
) (*servertest.TestServer, map[string]*servertest.TestClient) {
t.Helper()
srv := servertest.NewServer(t)
user := srv.CreateUser(t, "via-ha-user")
changed, err := srv.State().SetPolicy([]byte(`{
"tagOwners": {
"tag:other": ["via-ha-user@"],
"tag:router": ["via-ha-user@"],
"tag:client": ["via-ha-user@"]
},
"grants": [{
"src": ["tag:client"],
"dst": ["10.0.0.0/24"],
"ip": ["*"],
"via": ["tag:router"]
}],
"autoApprovers": {
"routes": {
"10.0.0.0/24": ["tag:other", "tag:router"]
}
}
}`))
require.NoError(t, err)
if changed {
changes, err := srv.State().ReloadPolicy()
require.NoError(t, err)
srv.App.Change(changes...)
}
clients := map[string]*servertest.TestClient{}
// Register the router outside the via group first to hold global primary.
for _, c := range []struct{ name, tag string }{
{"other", "tag:other"},
{"router-1", "tag:router"},
{"router-2", "tag:router"},
{"client", "tag:client"},
} {
clients[c.name] = servertest.NewClient(t, srv, c.name,
servertest.WithUser(user), servertest.WithTags(c.tag))
}
for _, name := range []string{"other", "router-1", "router-2"} {
clients[name].Direct().SetHostinfo(&tailcfg.Hostinfo{
BackendLogID: "servertest-" + name,
Hostname: name,
RoutableIPs: []netip.Prefix{route},
})
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
_ = clients[name].Direct().SendUpdate(ctx)
cancel()
}
return srv, clients
}
onlyHolder := func(want string) func(*netmap.NetworkMap) bool {
return func(nm *netmap.NetworkMap) bool {
var holders []string
for _, p := range nm.Peers {
if slices.Contains(p.AllowedIPs().AsSlice(), route) {
holders = append(holders, p.Hostinfo().Hostname())
}
}
return slices.Equal(holders, []string{want})
}
}
setExpiry := func(t *testing.T, srv *servertest.TestServer, id types.NodeID, expiry *time.Time) {
t.Helper()
_, c, err := srv.State().SetNodeExpiry(id, expiry)
require.NoError(t, err)
srv.App.Change(c)
}
for _, mode := range []string{"offline", "unhealthy", "expired"} {
t.Run(mode, func(t *testing.T) {
t.Parallel()
srv, clients := setup(t)
clients["client"].WaitForCondition(t, "route on router-1",
15*time.Second, onlyHolder("router-1"))
otherID := findNodeID(t, srv, "other")
router1ID := findNodeID(t, srv, "router-1")
require.Contains(t, srv.State().GetNodePrimaryRoutes(otherID), route)
switch mode {
case "offline":
clients["router-1"].Disconnect(t)
case "unhealthy":
changed := srv.State().SetNodeHealth(router1ID, false)
require.True(t, changed, "a via health change must trigger a recompute")
srv.App.Change(change.PolicyChange())
case "expired":
setExpiry(t, srv, router1ID, new(time.Now().Add(-time.Minute)))
}
clients["client"].WaitForCondition(t, "route moves to router-2",
30*time.Second, onlyHolder("router-2"))
assert.Contains(t, srv.State().GetNodePrimaryRoutes(otherID), route)
switch mode {
case "offline":
clients["router-1"].Reconnect(t)
case "unhealthy":
changed := srv.State().SetNodeHealth(router1ID, true)
require.True(t, changed, "a via recovery must trigger a recompute")
srv.App.Change(change.PolicyChange())
case "expired":
setExpiry(t, srv, router1ID, nil)
}
clients["client"].WaitForCondition(t, "route returns to router-1",
30*time.Second, onlyHolder("router-1"))
assert.Contains(t, srv.State().GetNodePrimaryRoutes(otherID), route)
})
}
}
+57 -70
View File
@@ -153,8 +153,8 @@ func checkHAInvariants(
st := srv.State()
primaries := readPrimaries(rt, srv, clients)
// Collect every prefix advertised by the fleet so the all-down
// honesty check can iterate them, even prefixes with zero current
// Collect every prefix advertised by the fleet so the no-router
// check can iterate them, even prefixes with zero current
// advertisers.
prefixSet := make(map[netip.Prefix]struct{})
for _, c := range clients {
@@ -167,7 +167,10 @@ func checkHAInvariants(
// prober's mapBatcher.IsConnected view is a separate gate the
// prober consults before dispatching a probe, not part of the
// election input.
// approversByPrefix also holds the offline routers, which keep a
// prefix while no candidate can serve it.
advertisersByPrefix := make(map[netip.Prefix][]types.NodeID)
approversByPrefix := make(map[netip.Prefix][]types.NodeID)
for _, c := range clients {
nv, ok := st.GetNodeByID(c.id)
@@ -176,12 +179,15 @@ func checkHAInvariants(
}
online, known := nv.IsOnline().GetOk()
if !known || !online {
continue
}
for _, p := range nv.AllApprovedRoutes() {
if p == c.route {
if p != c.route {
continue
}
approversByPrefix[p] = append(approversByPrefix[p], c.id)
if known && online {
advertisersByPrefix[p] = append(advertisersByPrefix[p], c.id)
}
}
@@ -207,20 +213,24 @@ func checkHAInvariants(
)
}
// Invariant 1: primary IsOnline in the NodeStore view.
// state.Disconnect's 10s grace can leave a recently-disconnected
// node with IsOnline=true; the structural rule still requires
// the bit before the snapshot may keep it as primary.
// Invariant 1: primary IsOnline in the NodeStore view while any
// candidate is healthy. state.Disconnect's 10s grace can leave a
// recently-disconnected node with IsOnline=true; the structural
// rule still requires the bit before the snapshot may keep it as
// primary over a healthy candidate.
healthyCandidate := slices.ContainsFunc(advertisersByPrefix[prefix], st.IsNodeHealthy)
online, known := nv.IsOnline().GetOk()
if !known || !online {
if (!known || !online) && healthyCandidate {
rt.Fatalf(
"prefix %s primary %d is not online (IsOnline=%v known=%v)",
prefix, owner, online, known,
"prefix %s primary %d is not online (IsOnline=%v known=%v) "+
"but %v has a healthy candidate",
prefix, owner, online, known, advertisersByPrefix[prefix],
)
}
// Invariant 2: primary advertises the prefix and is among the
// snapshot's candidate set.
// Invariant 2: primary advertises the prefix and, while any
// candidate is healthy, is among the snapshot's candidate set.
approved := nv.AllApprovedRoutes()
if !slices.Contains(approved, prefix) {
rt.Fatalf(
@@ -229,7 +239,7 @@ func checkHAInvariants(
)
}
if !slices.Contains(advertisersByPrefix[prefix], owner) {
if healthyCandidate && !slices.Contains(advertisersByPrefix[prefix], owner) {
rt.Fatalf(
"prefix %s primary %d not in advertiser set %v",
prefix, owner, advertisersByPrefix[prefix],
@@ -265,72 +275,49 @@ func checkHAInvariants(
}
}
// Invariant 6 (all-down honesty): if every candidate of a prefix is
// either offline OR unhealthy, the snapshot must either drop the
// primary OR keep one that is still a candidate from the same set
// (the all-unhealthy preserve-prev rule). A primary pointed at a
// non-candidate after every candidate has gone dark would point
// peers at a node the prober has already declared unreachable.
// Invariant 6 (no router available): with no healthy online
// candidate the prefix stays mapped. The previous primary keeps it
// while it still approves the prefix; otherwise the primary must be
// a router the prober has not declared unreachable, or there is none.
for prefix := range prefixSet {
owner, hasPrimary := primaries[prefix]
if !hasPrimary {
continue
}
candidates := advertisersByPrefix[prefix]
// Empty candidates with a primary present trips invariant 2
// above; reaching this branch with len(candidates) == 0 means
// the snapshot is internally inconsistent.
if len(candidates) == 0 {
rt.Fatalf(
"prefix %s has primary %d but no advertiser in the snapshot",
prefix, owner,
)
}
anyHealthy := slices.ContainsFunc(candidates, st.IsNodeHealthy)
// Healthy preference: a primary should not be unhealthy when a
// healthy candidate exists. Asserting this through the real
// prober → State → NodeStore seam catches any divergence the
// unit-level model would miss.
if anyHealthy && !st.IsNodeHealthy(owner) {
if hasPrimary && anyHealthy && !st.IsNodeHealthy(owner) {
rt.Fatalf(
"prefix %s primary %d unhealthy but %v has a healthy candidate",
prefix, owner, candidates,
)
}
// All-unhealthy guards: with every candidate unhealthy, the
// election has exactly two correct choices:
// - preserve prev primary when it is still a candidate
// - leave the prefix unmapped (no candidate fallback)
//
// Any other outcome points peers at a node already declared
// unreachable.
if !anyHealthy {
prevOwner, hadPrev := prevPrimaries[prefix]
prevStillCandidate := hadPrev &&
slices.Contains(candidates, prevOwner)
if anyHealthy {
continue
}
if prevStillCandidate && owner != prevOwner {
rt.Fatalf(
"prefix %s all-unhealthy election picked %d, "+
"but prev primary %d is still a candidate — "+
"preserve-prev rule violated",
prefix, owner, prevOwner,
)
}
prevOwner, hadPrev := prevPrimaries[prefix]
prevStillApproves := hadPrev &&
slices.Contains(approversByPrefix[prefix], prevOwner)
if !prevStillCandidate && (!hadPrev || owner != prevOwner) {
rt.Fatalf(
"prefix %s all-unhealthy fallback elected %d "+
"but prev primary %v is gone; election "+
"must leave the prefix unmapped",
prefix, owner, prevOwner,
)
}
if prevStillApproves && owner != prevOwner {
rt.Fatalf(
"prefix %s with no router available elected %d, "+
"but prev primary %d still approves it",
prefix, owner, prevOwner,
)
}
if !prevStillApproves && hasPrimary && owner != prevOwner &&
!st.IsNodeHealthy(owner) {
rt.Fatalf(
"prefix %s with no router available moved to %d, "+
"which the prober declared unreachable",
prefix, owner,
)
}
}
}
@@ -367,14 +354,14 @@ func snapshotPrimaries(
// so a reconnect mid-cycle does not install a stale Unhealthy.
//
// - The batched BatchSetNodeHealth is observable through the
// all-unhealthy preserve-prev invariant: per-call writes would
// no-router preserve-prev invariant: per-call writes would
// publish an intermediate "one unhealthy, one healthy" snapshot,
// leaving primary on a non-prev node after both flips land.
//
// - The all-unhealthy fallback that leaves prefixes unmapped is
// covered by the honesty invariant: with every candidate
// unhealthy AND prev gone from the candidate set, the prefix
// must stay unmapped instead of pinned to any candidate.
// - The fallback with no router available is covered by the
// no-router invariant: the previous primary keeps the prefix
// while it still approves it, and the fallback never moves it
// to a router the prober declared unreachable.
//
// Caveat: TestClient is a real controlclient.Direct that responds to
// PingRequest over Noise, so the timeout path of the prober rarely
@@ -51,8 +51,6 @@ var lifecycleSkipReasons = map[string]string{
"route-approval/b9/served": approvalOnSameKeyReauth,
"route-approval/b9/self-routes": approvalOnSameKeyReauth,
"route-approval/r1/served": loggedOutRouterUnserved,
"ephemeral-lifecycle/E1-logout/eph-logout/outcome": logoutAnswerUnauthorized,
"ephemeral-lifecycle/E1-logout/eph-logout-user/outcome": logoutAnswerUnauthorized,
@@ -81,7 +79,6 @@ const (
approvalOnHostinfo = "APPROVAL_ON_HOSTINFO: headscale auto-approves when a router with no approved routes sends a Hostinfo change that keeps its routes; SaaS does not"
approvalOnPolicy = "APPROVAL_ON_POLICY: headscale auto-approves announced routes when a policy is set; SaaS waits for the router's next route change or key rotation"
approvalOnSameKeyReauth = "APPROVAL_ON_SAME_KEY_REAUTH: headscale auto-approves on a re-auth that keeps the node key; SaaS does not"
loggedOutRouterUnserved = "LOGGED_OUT_ROUTER_UNSERVED: headscale stops serving a logged-out router's routes; SaaS keeps them on the expired, offline peer"
)
const (
@@ -93,8 +90,7 @@ const (
// Used by reauthSkipReasons.
const (
expiredRouterUnserved = "EXPIRED_ROUTER_UNSERVED: headscale stops serving an expired router's routes; SaaS keeps them in the peer's AllowedIPs and PrimaryRoutes"
expiredPeerOffline = "EXPIRED_PEER_OFFLINE: headscale shows an expired peer offline while it is connected; SaaS shows it online"
expiredPeerOffline = "EXPIRED_PEER_OFFLINE: headscale shows an expired peer offline while it is connected; SaaS shows it online"
)
// TestRouteApprovalCompat replays a user-owned router through admin
@@ -37,7 +37,6 @@ import (
// <test_id>/<step>[/<node>]/<check>. A listed check that matches fails, so a
// fix must delete its entry.
var reauthSkipReasons = map[string]string{
"owner-routes/r-expired-retag/expire/served": expiredRouterUnserved,
"owner-routes/r-expired-retag/expire/online": expiredPeerOffline,
"owner-routes/r-expired-retag/owner/node-key": expiredKeyAccepted,
+7 -4
View File
@@ -307,11 +307,14 @@ func TestRoutes(t *testing.T) {
20*time.Second, 200*time.Millisecond,
"r2 should take over as primary after r1 offline")
// 2. Take r2 offline. With both routers gone, no primary should remain.
// 2. Take r2 offline. With both routers gone, r2 keeps the route.
r2.Disconnect(t)
assert.Eventually(t, func() bool { return !hasRoute(r1ID) && !hasRoute(r2ID) },
20*time.Second, 200*time.Millisecond,
"no primary should be assigned while both routers are offline")
assert.Eventually(t, func() bool {
nv, ok := srv.State().GetNodeByID(r2ID)
return ok && !nv.IsOnline().Get() && hasRoute(r2ID) && !hasRoute(r1ID)
}, 20*time.Second, 200*time.Millisecond,
"the last primary keeps the route while both routers are offline")
// 3. Reconnect r2 (cable plugged back in).
r2.Reconnect(t)
@@ -222,22 +222,14 @@ func runViaHACompat(t *testing.T, c *testcapture.Capture) {
// keyed <test_id>/<step>/<viewer>/<check>. A listed check that matches
// fails, so a fix must delete its entry.
var viaHAFailoverSkipReasons = map[string]string{
"via-ha-failover/V1/client-a/route-owner": viaOfflineRouterKept,
"via-ha-failover/V9/client-a/route-owner": viaOfflineRouterKept,
"via-ha-failover/V2/client-a/route-owner": viaLowestIDAfterRestart,
"via-ha-failover/V10/client-a/route-owner": viaLowestIDAfterRestart,
"via-ha-failover/V5/client-a/route-owner": viaExpiredRouterKept,
"via-ha-failover/V5b/client-a/route-owner": viaExpiredRouterKept,
"via-ha-failover/V12/client-a/route-owner": readvertisedRouterTakesRoute,
"via-ha-failover/V12/client-b/route-owner": readvertisedRouterTakesRoute,
}
const (
viaOfflineRouterKept = "VIA_OFFLINE_ROUTER_KEPT: headscale keeps a via route on an offline router while another router of the via group is online; SaaS moves it to the online one"
viaExpiredRouterKept = "VIA_EXPIRED_ROUTER_KEPT: headscale keeps a via route on a router whose key expired while another router of the via group is live; SaaS moves it to the live one"
viaLowestIDAfterRestart = "VIA_LOWEST_ID_AFTER_RESTART: headscale serves a via route from the group's lowest-ID router once it reconnects; SaaS keeps the router that served while it was down"
readvertisedRouterTakesRoute = "READVERTISED_ROUTER_TAKES_ROUTE: SaaS hands a route back to an HA router that withdrew and re-advertised it, taking it from the router that served it meanwhile; headscale keeps that router"
)
+67 -49
View File
@@ -206,14 +206,18 @@ type work struct {
// updateChanges reports whether an in-place update moved a peer-visibility
// input (which also re-elects routes) or only a route-election input.
// Peer visibility depends on what the policy reads; election additionally
// depends on online and health state, treating unknown online as offline.
// depends on online, expiry and health state, treating unknown online as
// offline. Fields are compared, not [servesRoutes]: an expiry that passed
// before the write reads the same on both sides.
func updateChanges(pre, post *types.Node) (bool, bool) {
preView, postView := pre.View(), post.View()
if postView.HasPolicyChange(preView) || postView.HasNetworkChanges(preView) {
return true, true
}
return false, pre.Online() != post.Online() || pre.Unhealthy != post.Unhealthy
return false, pre.Online() != post.Online() ||
!preView.Expiry().GetOr(time.Time{}).Equal(postView.Expiry().GetOr(time.Time{})) ||
pre.Unhealthy != post.Unhealthy
}
// PutNode adds or updates a node in the store.
@@ -778,18 +782,25 @@ func snapshotFromNodes(
return newSnap
}
// onlineAdvertisers maps each non-exit prefix to the IDs of online nodes
// that approve it. Nodes are visited in the order of ids, so the per-prefix
// servesRoutes reports whether n can carry its routes. Expiry is tested
// apart from Online because an expired node keeps its control session.
func servesRoutes(n *types.Node) bool {
return n.Online() && !n.IsExpired()
}
// advertisers maps each non-exit prefix to the IDs of nodes that approve it
// and pass keep. Nodes are visited in the order of ids, so the per-prefix
// slices preserve that order.
func onlineAdvertisers(
func advertisers(
nodes map[types.NodeID]types.Node,
ids []types.NodeID,
keep func(*types.Node) bool,
) map[netip.Prefix][]types.NodeID {
advertisers := make(map[netip.Prefix][]types.NodeID)
out := make(map[netip.Prefix][]types.NodeID)
for _, id := range ids {
n := nodes[id]
if n.IsOnline == nil || !*n.IsOnline {
if !keep(&n) {
continue
}
@@ -798,65 +809,63 @@ func onlineAdvertisers(
continue
}
advertisers[p] = append(advertisers[p], id)
out[p] = append(out[p], id)
}
}
return advertisers
return out
}
// electPrimaryRoutes picks the primary advertiser for each non-exit
// prefix. Inputs are restricted to online nodes that advertise the
// prefix. The previous primary is preserved when it is still online
// and healthy (anti-flap); otherwise the lowest-NodeID healthy
// advertiser wins. When every advertiser is unhealthy the previous
// primary is preserved only if still a candidate — falling back to
// any other candidate would point peers at a node the prober has
// already declared unreachable, so leaving the prefix unmapped is
// preferred until a probe cycle finds one that responds.
// electPrimaryRoutes picks the primary router for each non-exit prefix any
// node approves. A router is available when it is online, unexpired and
// healthy. The previous primary is preserved while it is available
// (anti-flap); otherwise the lowest-NodeID available router wins.
//
// With no router available the prefix stays mapped, so peers keep the route
// and use it once its router returns: the previous primary keeps it while it
// still approves the prefix, else the lowest-NodeID approving router does.
// That fallback never picks a router the prober declared unreachable; with
// only such routers left the prefix is unmapped until a probe cycle finds one
// that responds.
func electPrimaryRoutes(
nodes map[types.NodeID]types.Node,
prev map[netip.Prefix]types.NodeID,
) (map[netip.Prefix]types.NodeID, map[types.NodeID]bool) {
advertisers := onlineAdvertisers(nodes, slices.Sorted(maps.Keys(nodes)))
approvers := advertisers(nodes, slices.Sorted(maps.Keys(nodes)),
func(*types.Node) bool { return true })
routes := make(map[netip.Prefix]types.NodeID, len(advertisers))
for prefix, candidates := range advertisers {
if cur, ok := prev[prefix]; ok &&
slices.Contains(candidates, cur) &&
!nodes[cur].Unhealthy {
available := func(id types.NodeID) bool {
n := nodes[id]
return servesRoutes(&n) && !n.Unhealthy
}
healthy := func(id types.NodeID) bool { return !nodes[id].Unhealthy }
routes := make(map[netip.Prefix]types.NodeID, len(approvers))
for prefix, ids := range approvers {
cur, hasPrev := prev[prefix]
hasPrev = hasPrev && slices.Contains(ids, cur)
if hasPrev && available(cur) {
routes[prefix] = cur
continue
}
var (
selected types.NodeID
found bool
)
if i := slices.IndexFunc(ids, available); i >= 0 {
routes[prefix] = ids[i]
for _, c := range candidates {
if !nodes[c].Unhealthy {
selected = c
found = true
break
}
continue
}
// All-unhealthy fallback: preserve the previous primary only
// when it is still a candidate. Falling back to any candidate
// would point peers at a node the prober has already declared
// unreachable; leaving the prefix unmapped is honest until a
// probe cycle picks one that responds.
if !found {
if cur, ok := prev[prefix]; ok && slices.Contains(candidates, cur) {
selected = cur
found = true
}
if hasPrev {
routes[prefix] = cur
continue
}
if found {
routes[prefix] = selected
if i := slices.IndexFunc(ids, healthy); i >= 0 {
routes[prefix] = ids[i]
}
}
@@ -1051,9 +1060,10 @@ func (s *NodeStore) PrimaryRoutesForNode(id types.NodeID) []netip.Prefix {
func (s *NodeStore) HANodes() map[netip.Prefix][]types.NodeID {
snap := s.data.Load()
advertisers := onlineAdvertisers(
online := advertisers(
snap.nodesByID,
slices.Sorted(maps.Keys(snap.nodesByID)),
servesRoutes,
)
unhealthy := func(id types.NodeID) bool {
@@ -1062,7 +1072,7 @@ func (s *NodeStore) HANodes() map[netip.Prefix][]types.NodeID {
out := make(map[netip.Prefix][]types.NodeID)
for p, ids := range advertisers {
for p, ids := range online {
if len(ids) < 2 && !slices.ContainsFunc(ids, unhealthy) {
continue
}
@@ -1085,6 +1095,14 @@ func (s *NodeStore) IsNodeHealthy(id types.NodeID) bool {
return !n.Unhealthy
}
// IsRouterAvailable reports whether id may win a primary election: online,
// unexpired and healthy.
func (s *NodeStore) IsRouterAvailable(id types.NodeID) bool {
n, ok := s.data.Load().nodesByID[id]
return ok && servesRoutes(&n) && !n.Unhealthy
}
// PrimaryRoutes returns the snapshot's prefix→primary map. The map is
// owned by the snapshot and must not be mutated; it is safe to read
// concurrently because snapshots are immutable once published.
+2 -1
View File
@@ -1910,7 +1910,7 @@ var nodeFieldImpact = map[string]string{
"MachineKey": "payload", "NodeKey": "payload", "DiscoKey": "payload",
"Endpoints": "payload", "Hostname": "payload", "GivenName": "payload",
"RegisterMethod": "payload", "AuthKeyID": "payload", "AuthKey": "payload",
"Expiry": "payload", "LastSeen": "payload", "CreatedAt": "payload",
"Expiry": "election", "LastSeen": "payload", "CreatedAt": "payload",
"UpdatedAt": "payload", "DeletedAt": "payload",
"ActiveSessions": "payload", "SessionEpoch": "payload",
}
@@ -1966,6 +1966,7 @@ func TestUpdateChangesReportsRelationFields(t *testing.T) {
},
{name: "online", mutate: func(n *types.Node) { n.IsOnline = new(true) }, wantElection: true},
{name: "unhealthy", mutate: func(n *types.Node) { n.Unhealthy = true }, wantElection: true},
{name: "expiry", mutate: func(n *types.Node) { n.Expiry = new(time.Now().Add(-time.Minute)) }, wantElection: true},
{name: "endpoint", mutate: func(n *types.Node) { n.Endpoints = []netip.AddrPort{netip.MustParseAddrPort("192.0.2.1:1")} }},
{name: "lastseen", mutate: func(n *types.Node) { n.LastSeen = new(time.Now()) }},
}
+67 -78
View File
@@ -91,11 +91,23 @@ func (m *primariesModel) recomputeEffective(id types.NodeID) {
func (m *primariesModel) advertisersByPrefix() map[netip.Prefix][]types.NodeID {
out := map[netip.Prefix][]types.NodeID{}
for n, prefs := range m.prefixes {
if !m.connected[n] {
continue
for p, nodes := range m.approversByPrefix() {
for _, n := range nodes {
if m.connected[n] {
out[p] = append(out[p], n)
}
}
}
return out
}
// approversByPrefix returns every node that approves each prefix, connected
// or not, sorted by NodeID.
func (m *primariesModel) approversByPrefix() map[netip.Prefix][]types.NodeID {
out := map[netip.Prefix][]types.NodeID{}
for n, prefs := range m.prefixes {
for _, p := range prefs {
out[p] = append(out[p], n)
}
@@ -108,78 +120,57 @@ func (m *primariesModel) advertisersByPrefix() map[netip.Prefix][]types.NodeID {
return out
}
// available reports whether n may win an election: connected and healthy.
func (m *primariesModel) available(n types.NodeID) bool {
return m.connected[n] && !m.unhealthy[n]
}
// updatePrimaries reapplies the algorithm to recompute the primary
// for each prefix. Called after every operation.
func (m *primariesModel) updatePrimaries() {
advertisers := m.advertisersByPrefix()
approvers := m.approversByPrefix()
// Drop primaries for prefixes that no longer have any advertiser.
// Drop primaries for prefixes no node approves any more.
for p := range m.primary {
if _, ok := advertisers[p]; !ok {
if _, ok := approvers[p]; !ok {
delete(m.primary, p)
}
}
for p, nodes := range advertisers {
if cur, ok := m.primary[p]; ok {
if slices.Contains(nodes, cur) && !m.unhealthy[cur] {
continue
}
for p, nodes := range approvers {
cur, hasPrev := m.primary[p]
hasPrev = hasPrev && slices.Contains(nodes, cur)
if hasPrev && m.available(cur) {
continue
}
var (
selected types.NodeID
found bool
)
if i := slices.IndexFunc(nodes, m.available); i >= 0 {
m.primary[p] = nodes[i]
for _, n := range nodes {
if !m.unhealthy[n] {
selected = n
found = true
break
}
continue
}
// All-unhealthy fallback: preserve the previous primary if it
// is still a candidate, otherwise leave the prefix unmapped.
// Choosing any candidate would point peers at a node already
// declared unreachable; the model mirrors that policy.
if !found && len(nodes) >= 1 {
if cur, ok := m.primary[p]; ok && slices.Contains(nodes, cur) {
selected = cur
found = true
}
// No router available: the previous primary keeps the prefix,
// else the lowest-ID approver the prober has not declared
// unreachable; with neither the prefix is unmapped.
if hasPrev {
continue
}
if found {
m.primary[p] = selected
if i := slices.IndexFunc(nodes, func(n types.NodeID) bool {
return !m.unhealthy[n]
}); i >= 0 {
m.primary[p] = nodes[i]
} else {
delete(m.primary, p)
}
}
}
// allPrefixes returns every prefix mentioned by any connected node.
// allPrefixes returns every prefix any node approves.
func (m *primariesModel) allPrefixes() []netip.Prefix {
seen := map[netip.Prefix]bool{}
for n, prefs := range m.prefixes {
if !m.connected[n] {
continue
}
for _, p := range prefs {
seen[p] = true
}
}
out := make([]netip.Prefix, 0, len(seen))
for p := range seen {
out = append(out, p)
}
return out
return slices.Collect(maps.Keys(m.approversByPrefix()))
}
func samePrefixSet(a, b []netip.Prefix) bool {
@@ -233,10 +224,9 @@ func checkPrimariesProperties(
}
}
// Every prefix that has at least one connected advertiser must
// have a primary in the snapshot: a prefix silently losing its
// primary after a disconnect/reconnect cycle would leave peers
// without a route.
// Every prefix the model maps must have the same primary in the
// snapshot: a prefix silently losing its primary after a
// disconnect/reconnect cycle would leave peers without a route.
for _, p := range m.allPrefixes() {
want, expectExists := m.primary[p]
if !expectExists {
@@ -246,7 +236,7 @@ func checkPrimariesProperties(
got, ok := ns.PrimaryRouteFor(p)
if !ok {
rt.Fatalf(
"prefix %s has at least one advertiser in the model but no primary in NodeStore",
"prefix %s has a primary in the model but none in NodeStore",
p,
)
}
@@ -261,7 +251,7 @@ func checkPrimariesProperties(
// Structural invariants on the live snapshot, independent of the
// model. These catch shapes the model alone cannot — e.g. an owner
// that is offline but still has the prefix attributed to it.
// that no longer approves the prefix attributed to it.
snapshotPrimaries := ns.PrimaryRoutes()
// A primary that owns ≥1 prefix in `routes` must also light up
@@ -283,9 +273,11 @@ func checkPrimariesProperties(
}
}
// Per-owner structural checks: every primary must currently be
// online and must currently advertise the prefix it owns.
// Per-owner structural checks: every primary must currently
// advertise the prefix it owns, and be available whenever any of
// its approvers is.
advertisersByPrefix := m.advertisersByPrefix()
approversByPrefix := m.approversByPrefix()
for prefix, owner := range snapshotPrimaries {
nv, ok := ns.GetNode(owner)
@@ -296,13 +288,14 @@ func checkPrimariesProperties(
)
}
// Primary is online: an offline node cannot move packets, so
// election must never leave one as the snapshot's primary.
// An offline primary is only allowed while no approver could
// move packets for the prefix.
online, known := nv.IsOnline().GetOk()
if !known || !online {
if (!known || !online) &&
slices.ContainsFunc(approversByPrefix[prefix], m.available) {
rt.Fatalf(
"prefix %s primary %d is not online",
prefix, owner,
"prefix %s primary %d is not online but %v had an available router",
prefix, owner, approversByPrefix[prefix],
)
}
@@ -317,21 +310,17 @@ func checkPrimariesProperties(
)
}
// Healthy preference: if any candidate for this prefix is
// healthy, the elected primary must also be healthy. Leaving
// the prefix unmapped is fine; pointing at an unhealthy
// candidate while a healthy one was available is not.
// Healthy preference: if any approver for this prefix is
// healthy, the elected primary must also be healthy, unless it
// is the previous primary kept while no router is available.
// Leaving the prefix unmapped is fine; moving it to an
// unhealthy router while a healthy one was there is not.
candidates := advertisersByPrefix[prefix]
anyHealthy := false
anyHealthy := slices.ContainsFunc(approversByPrefix[prefix], func(n types.NodeID) bool {
return !m.unhealthy[n]
})
for _, c := range candidates {
if !m.unhealthy[c] {
anyHealthy = true
break
}
}
if anyHealthy && m.unhealthy[owner] {
if anyHealthy && m.unhealthy[owner] && owner != prevSnapshotPrimaries[prefix] {
rt.Fatalf(
"prefix %s primary %d is unhealthy but %v had a healthy candidate",
prefix, owner, candidates,
+113 -9
View File
@@ -4,6 +4,7 @@ import (
"net/netip"
"slices"
"testing"
"time"
"github.com/juanfont/headscale/hscontrol/types"
"github.com/puzpuzpuz/xsync/v4"
@@ -171,13 +172,98 @@ func TestPrimaries_ClearRoutesDropsPrimary(t *testing.T) {
f.requireNoPrimary(mp("192.168.1.0/24"))
}
func TestPrimaries_DisconnectDropsLastAdvertiserPrimary(t *testing.T) {
func TestPrimaries_DisconnectKeepsLastRouterPrimary(t *testing.T) {
f := newPrimariesFixture(t, 1)
f.advertise(1, mp("192.168.1.0/24"))
f.requirePrimary(mp("192.168.1.0/24"), 1)
f.disconnect(1)
f.requireNoPrimary(mp("192.168.1.0/24"))
f.requirePrimary(mp("192.168.1.0/24"), 1)
}
func TestPrimaries_NoRouterAvailable(t *testing.T) {
prefix := mp("10.0.0.0/24")
expire := func(f *primariesFixture, id types.NodeID) {
f.ns.UpdateNode(id, func(n *types.Node) {
n.Expiry = new(time.Now().Add(-time.Minute))
})
}
unapprove := func(f *primariesFixture, id types.NodeID) {
f.ns.UpdateNode(id, func(n *types.Node) { n.ApprovedRoutes = nil })
}
tests := []struct {
name string
setup func(*primariesFixture)
want types.NodeID
}{
{
name: "expired router keeps its route",
setup: func(f *primariesFixture) {
f.advertise(1, prefix)
expire(f, 1)
},
want: 1,
},
{
name: "previous primary kept over a lower ID",
setup: func(f *primariesFixture) {
f.advertise(1, prefix)
f.advertise(2, prefix)
f.disconnect(1)
f.disconnect(2)
},
want: 2,
},
{
name: "lowest ID approver once the previous primary is gone",
setup: func(f *primariesFixture) {
f.advertise(1, prefix)
f.advertise(2, prefix)
f.advertise(3, prefix)
f.disconnect(2)
f.disconnect(3)
f.disconnect(1)
unapprove(f, 1)
},
want: 2,
},
{
name: "unhealthy router passed over",
setup: func(f *primariesFixture) {
f.advertise(1, prefix)
f.advertise(2, prefix)
f.advertise(3, prefix)
f.disconnect(2)
f.unhealthy(1)
f.disconnect(3)
unapprove(f, 3)
},
want: 2,
},
{
name: "only unhealthy routers left",
setup: func(f *primariesFixture) {
f.advertise(1, prefix)
f.advertise(3, prefix)
f.unhealthy(1)
unapprove(f, 3)
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
f := newPrimariesFixture(t, 1, 2, 3)
tt.setup(f)
if tt.want == 0 {
f.requireNoPrimary(prefix)
} else {
f.requirePrimary(prefix, tt.want)
}
})
}
}
func TestPrimaries_UnhealthyTriggersFailover(t *testing.T) {
@@ -238,6 +324,25 @@ func TestPrimaries_AllUnhealthyPreservesPrevious(t *testing.T) {
f.requirePrimary(prefix, 2)
}
func TestPrimaries_ExpiredRouterNotElected(t *testing.T) {
// Expiry alone takes a router out of the election, even while its
// cached online flag still says connected.
prefix := mp("10.0.0.0/24")
f := newPrimariesFixture(t, 1, 2)
f.advertise(1, prefix)
f.advertise(2, prefix)
f.requirePrimary(prefix, 1)
require.True(t, f.ns.IsRouterAvailable(1))
f.ns.UpdateNode(1, func(n *types.Node) {
n.Expiry = new(time.Now().Add(-time.Minute))
})
f.requirePrimary(prefix, 2)
require.False(t, f.ns.IsRouterAvailable(1))
require.True(t, f.ns.IsRouterAvailable(2))
}
func TestPrimaries_ExitRouteNotElected(t *testing.T) {
// Exit routes (0.0.0.0/0, ::/0) are not subject to HA primary
// election — every approved exit-route advertiser keeps it.
@@ -249,10 +354,9 @@ func TestPrimaries_ExitRouteNotElected(t *testing.T) {
}
func TestPrimaries_BothOfflineThenOneReturns(t *testing.T) {
// With two HA advertisers, dropping both then bringing one back
// used to leave the prefix without any primary. The snapshot
// recomputes primaries on every NodeStore write, so the
// returning advertiser must be elected.
// With two HA advertisers both offline, the last primary keeps the
// prefix. The snapshot recomputes primaries on every NodeStore
// write, so the first advertiser to return must be elected.
prefix := mp("10.0.0.0/24")
f := newPrimariesFixture(t, 1, 2)
f.advertise(1, prefix)
@@ -263,10 +367,10 @@ func TestPrimaries_BothOfflineThenOneReturns(t *testing.T) {
f.requirePrimary(prefix, 2)
f.disconnect(2)
f.requireNoPrimary(prefix)
f.advertise(2, prefix)
f.requirePrimary(prefix, 2)
f.advertise(1, prefix)
f.requirePrimary(prefix, 1)
}
func TestPrimaries_HANodes(t *testing.T) {
+15 -3
View File
@@ -372,6 +372,7 @@ func NewState(cfg *types.Config) (*State, error) {
batchTimeout := cmp.Or(cfg.Tuning.NodeStoreBatchTimeout, defaultNodeStoreBatchTimeout)
nodeStore := NewNodeStore(nodes, policyPeersFunc(polMan), batchSize, batchTimeout)
polMan.SetRouterAvailable(nodeStore.IsRouterAvailable)
nodeStore.Start()
s := &State{
@@ -1692,7 +1693,7 @@ func (s *State) IsNodeHealthy(id types.NodeID) bool {
}
// SetNodeHealth flips the runtime health bit for one node and reports
// whether the resulting primary-route assignment changed, so the HA
// whether peers need to recompute global or via routes, so the HA
// prober can decide whether to fan out a PolicyChange. true means
// healthy; false means unhealthy. An unhealthy mark is dropped when
// the node is no longer an HA candidate (offline or no approved
@@ -1708,7 +1709,7 @@ func (s *State) SetNodeHealth(id types.NodeID, healthy bool) bool {
// BatchSetNodeHealth applies a set of health updates atomically: the
// election runs once after every flag has been flipped, so observers
// never see an intermediate snapshot. Returns true when the
// primary-route assignment differs from before the batch so callers
// global primary routes or a via target's health changed, so callers
// can gate a single PolicyChange dispatch. Map value true = healthy;
// false = unhealthy (gated as in [State.SetNodeHealth]).
//
@@ -1744,7 +1745,18 @@ func (s *State) BatchSetNodeHealth(updates map[types.NodeID]bool) bool {
s.nodeStore.UpdateNodes(fns)
return !maps.Equal(prevRoutes, s.nodeStore.PrimaryRoutes())
if !maps.Equal(prevRoutes, s.nodeStore.PrimaryRoutes()) {
return true
}
for id := range fns {
nv, ok := s.nodeStore.GetNode(id)
if ok && nv.Unhealthy() == !updates[id] && s.polMan.NodeIsViaTarget(nv) {
return true
}
}
return false
}
// healthSetter returns an UpdateNodeFunc that flips n.Unhealthy to
+9 -3
View File
@@ -3995,15 +3995,21 @@ func TestHASubnetRouterFailoverBothOffline(t *testing.T) {
}, pr.PrimaryRoutes, "r2 should be primary after r1 offline")
}, propagationTime, 500*time.Millisecond, "waiting for failover to r2")
t.Log("=== Step 2: r2 also goes offline. No primary should remain. ===")
t.Log("=== Step 2: r2 also goes offline. r2 keeps the route. ===")
require.NoError(t, subRouter2.Down())
assert.EventuallyWithT(t, func(c *assert.CollectT) {
current, err := headscale.ListNodes()
assert.NoError(c, err)
assert.False(c, MustFindNode(subRouter2.Hostname(), current).Online, "r2 should be offline")
pr, err := headscale.PrimaryRoutes()
assert.NoError(c, err)
assert.Empty(c, pr.PrimaryRoutes,
"no primary should be assigned while both routers are offline")
assert.Equal(c, map[string]types.NodeID{
pref.String(): nodeID2,
}, pr.PrimaryRoutes,
"the last primary keeps the route while both routers are offline")
}, propagationTime, 500*time.Millisecond, "waiting for both routers to be offline")
t.Log("=== Step 3: r2 returns. ===")