mirror of
https://github.com/juanfont/headscale.git
synced 2026-11-04 04:34:07 +09:00
state, policy/v2: elect available routers and retain unavailable routes
Recompute via elections on health changes and check online state and expiry. With no available router, retain the previous primary or lowest-ID approver; exclude routers the prober marked unreachable.
This commit is contained in:
1 parent
1c94588bf9
commit
de8a0cc7eb
16 files changed
+604
-245
No files matched your search
@@ -63,12 +63,19 @@ type PolicyManager interface {
|
||||
// avoid a tailnet-wide recompute on every ordinary reconnect.
|
||||
NodeNeedsPeerRecompute(node types.NodeView) bool
|
||||
|
||||
// NodeIsViaTarget reports whether a grant steers traffic through the node.
|
||||
NodeIsViaTarget(node types.NodeView) bool
|
||||
|
||||
// ViaRoutesForPeer computes via grant effects for a viewer-peer pair.
|
||||
// It returns which routes should be included (peer is via-designated for viewer)
|
||||
// and excluded (steered to a different peer). When no via grants apply,
|
||||
// both fields are empty and the caller falls back to existing behavior.
|
||||
ViaRoutesForPeer(viewer, peer types.NodeView) types.ViaRouteResult
|
||||
|
||||
// SetRouterAvailable sets the via election's runtime availability check.
|
||||
// The callback runs under the policy lock and must remain lock-free.
|
||||
SetRouterAvailable(available func(types.NodeID) bool)
|
||||
|
||||
// NodeCapMap returns compiled policy capabilities for the given node,
|
||||
// or nil when none target it. The map is a defensive clone; its values
|
||||
// are shared and read-only. It describes the node's own capabilities,
|
||||
|
||||
@@ -118,6 +118,8 @@ type PolicyManager struct {
|
||||
// nodeAttrsPending mirrors len(nodeAttrsChanged) > 0 so the drain,
|
||||
// called on every dispatched change, skips pm.mu when idle.
|
||||
nodeAttrsPending atomic.Bool
|
||||
|
||||
routerAvailable func(types.NodeID) bool
|
||||
}
|
||||
|
||||
// filterAndPolicy combines the compiled filter rules with policy content for hashing.
|
||||
@@ -468,6 +470,22 @@ func (pm *PolicyManager) NodeNeedsPeerRecompute(node types.NodeView) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
return pm.isViaTargetLocked(node)
|
||||
}
|
||||
|
||||
// NodeIsViaTarget reports whether a grant steers traffic through node.
|
||||
func (pm *PolicyManager) NodeIsViaTarget(node types.NodeView) bool {
|
||||
if pm == nil || !node.Valid() {
|
||||
return false
|
||||
}
|
||||
|
||||
pm.mu.RLock()
|
||||
defer pm.mu.RUnlock()
|
||||
|
||||
return pm.isViaTargetLocked(node)
|
||||
}
|
||||
|
||||
func (pm *PolicyManager) isViaTargetLocked(node types.NodeView) bool {
|
||||
for tag := range pm.viaTargetTags {
|
||||
if node.HasTag(string(tag)) {
|
||||
return true
|
||||
@@ -1434,6 +1452,20 @@ func (pm *PolicyManager) NodeCanApproveRoute(node types.NodeView, route netip.Pr
|
||||
return false
|
||||
}
|
||||
|
||||
// SetRouterAvailable installs the availability check used by the via election.
|
||||
// Runtime status changes bypass SetNodes; nil treats every router as available.
|
||||
// The callback runs with pm.mu held and must remain a lock-free snapshot read.
|
||||
func (pm *PolicyManager) SetRouterAvailable(available func(types.NodeID) bool) {
|
||||
if pm == nil {
|
||||
return
|
||||
}
|
||||
|
||||
pm.mu.Lock()
|
||||
defer pm.mu.Unlock()
|
||||
|
||||
pm.routerAvailable = available
|
||||
}
|
||||
|
||||
// ViaRoutesForPeer computes via grant effects for a viewer-peer pair.
|
||||
// For each via grant where the viewer matches the source, it checks whether the
|
||||
// peer advertises any of the grant's destination prefixes. If the peer has the
|
||||
@@ -1607,9 +1639,9 @@ func (pm *PolicyManager) ViaRoutesForPeer(viewer, peer types.NodeView) types.Via
|
||||
if len(result.Include) > 0 || len(result.Exclude) > 0 {
|
||||
// Multi-router via election: when a via grant's tag matches
|
||||
// multiple peers advertising the same prefix, only the
|
||||
// lowest-ID peer (the via-group primary) keeps the prefix in
|
||||
// Include. The others move to Exclude. This mirrors HA
|
||||
// primary election scoped to the via tag group.
|
||||
// lowest-ID available peer keeps the prefix in Include. The others
|
||||
// move to Exclude. With no available router, the lowest ID keeps the
|
||||
// prefix so the viewer still gets exactly one route.
|
||||
//
|
||||
// Unlike the global [tailcfg.Node.PrimaryRoutes] election
|
||||
// (routes/primary.go), which picks one primary across ALL
|
||||
@@ -1639,12 +1671,21 @@ func (pm *PolicyManager) ViaRoutesForPeer(viewer, peer types.NodeView) types.Via
|
||||
continue
|
||||
}
|
||||
|
||||
var viaPrimaryID types.NodeID
|
||||
var viaPrimaryID, viaFallbackID types.NodeID
|
||||
|
||||
for _, viaTag := range grant.Via {
|
||||
for _, node := range pm.nodes.All() {
|
||||
if node.HasTag(string(viaTag)) &&
|
||||
slices.Contains(node.SubnetRoutes(), included) {
|
||||
if viaFallbackID == 0 || node.ID() < viaFallbackID {
|
||||
viaFallbackID = node.ID()
|
||||
}
|
||||
|
||||
if pm.routerAvailable != nil &&
|
||||
!pm.routerAvailable(node.ID()) {
|
||||
continue
|
||||
}
|
||||
|
||||
if viaPrimaryID == 0 || node.ID() < viaPrimaryID {
|
||||
viaPrimaryID = node.ID()
|
||||
}
|
||||
@@ -1652,6 +1693,10 @@ func (pm *PolicyManager) ViaRoutesForPeer(viewer, peer types.NodeView) types.Via
|
||||
}
|
||||
}
|
||||
|
||||
if viaPrimaryID == 0 {
|
||||
viaPrimaryID = viaFallbackID
|
||||
}
|
||||
|
||||
if viaPrimaryID != 0 && peer.ID() != viaPrimaryID {
|
||||
result.Include = slices.DeleteFunc(
|
||||
result.Include,
|
||||
|
||||
@@ -2754,6 +2754,80 @@ func TestViaRoutesForPeer(t *testing.T) {
|
||||
require.Empty(t, result.Include)
|
||||
require.Equal(t, []netip.Prefix{mp("10.0.0.0/24")}, result.Exclude)
|
||||
})
|
||||
|
||||
t.Run("via_election_skips_unavailable_routers", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
route := mp("10.0.0.0/24")
|
||||
nodes := types.Nodes{
|
||||
{
|
||||
ID: 1,
|
||||
Hostname: "viewer",
|
||||
IPv4: ap("100.64.0.1"),
|
||||
User: new(users[0]),
|
||||
UserID: new(users[0].ID),
|
||||
Tags: []string{"tag:group-a"},
|
||||
Hostinfo: &tailcfg.Hostinfo{},
|
||||
},
|
||||
}
|
||||
|
||||
for id := types.NodeID(2); id <= 4; id++ {
|
||||
nodes = append(nodes, &types.Node{
|
||||
ID: id,
|
||||
Hostname: fmt.Sprintf("router-%d", id),
|
||||
IPv4: ap(fmt.Sprintf("100.64.0.%d", id)),
|
||||
User: new(users[0]),
|
||||
UserID: new(users[0].ID),
|
||||
Tags: []string{"tag:router-a"},
|
||||
Hostinfo: &tailcfg.Hostinfo{RoutableIPs: []netip.Prefix{route}},
|
||||
ApprovedRoutes: []netip.Prefix{route},
|
||||
})
|
||||
}
|
||||
|
||||
pol := `{
|
||||
"tagOwners": {
|
||||
"tag:router-a": ["user1@"],
|
||||
"tag:group-a": ["user1@"]
|
||||
},
|
||||
"grants": [{
|
||||
"src": ["tag:group-a"],
|
||||
"dst": ["10.0.0.0/24"],
|
||||
"ip": ["*"],
|
||||
"via": ["tag:router-a"]
|
||||
}]
|
||||
}`
|
||||
pm, err := NewPolicyManager([]byte(pol), users, nodes.ViewSlice())
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
available []types.NodeID
|
||||
want types.NodeID
|
||||
}{
|
||||
{name: "all available", available: []types.NodeID{2, 3, 4}, want: 2},
|
||||
{name: "lowest ID down", available: []types.NodeID{3, 4}, want: 3},
|
||||
{name: "only highest ID left", available: []types.NodeID{4}, want: 4},
|
||||
{name: "whole group down", want: 2},
|
||||
{name: "lowest ID recovers", available: []types.NodeID{2, 3, 4}, want: 2},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
pm.SetRouterAvailable(func(id types.NodeID) bool {
|
||||
return slices.Contains(tt.available, id)
|
||||
})
|
||||
|
||||
for _, router := range nodes[1:] {
|
||||
result := pm.ViaRoutesForPeer(nodes[0].View(), router.View())
|
||||
if router.ID == tt.want {
|
||||
require.Equal(t, []netip.Prefix{route}, result.Include)
|
||||
require.Empty(t, result.Exclude)
|
||||
} else {
|
||||
require.Empty(t, result.Include)
|
||||
require.Equal(t, []netip.Prefix{route}, result.Exclude)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestBuildPeerMap_AutogroupInternetMakesExitNodeVisible reproduces
|
||||
|
||||
@@ -78,37 +78,30 @@ var expirySkipReasons = map[string]string{
|
||||
|
||||
"expiry/A5-advertise/laptop/online-observer": expiredPeerOffline,
|
||||
"expiry/A5-advertise/laptop/paths-observer": expiredPeerPaths,
|
||||
"expiry/A5-advertise/laptop/routes-observer": expiredRouterUnserved,
|
||||
"expiry/A5-advertise/laptop/self": selfExpiredFlags,
|
||||
|
||||
"expiry/A7-dns/laptop/dns": nameserversFromConfig,
|
||||
"expiry/A7-dns/laptop/online-observer": expiredPeerOffline,
|
||||
"expiry/A7-dns/laptop/paths-observer": expiredPeerPaths,
|
||||
"expiry/A7-dns/laptop/routes-observer": expiredRouterUnserved,
|
||||
"expiry/A7-dns/laptop/self": selfExpiredFlags,
|
||||
"expiry/A7-dns/observer/dns": nameserversFromConfig,
|
||||
|
||||
"expiry/A8-policy/laptop/online-observer": expiredPeerOffline,
|
||||
"expiry/A8-policy/laptop/paths-observer": expiredPeerPaths,
|
||||
"expiry/A8-policy/laptop/routes-observer": expiredRouterUnserved,
|
||||
"expiry/A8-policy/laptop/self": selfExpiredFlags,
|
||||
|
||||
"expiry/A9-ssh/laptop/online-observer": expiredPeerOffline,
|
||||
"expiry/A9-ssh/laptop/paths-observer": expiredPeerPaths,
|
||||
"expiry/A9-ssh/laptop/routes-observer": expiredRouterUnserved,
|
||||
"expiry/A9-ssh/laptop/self": selfExpiredFlags,
|
||||
|
||||
"expiry/A10-reconnect/laptop/online-observer": expiredPeerOffline,
|
||||
"expiry/A10-reconnect/laptop/paths-observer": expiredPeerPaths,
|
||||
"expiry/A10-reconnect/laptop/routes-observer": expiredRouterUnserved,
|
||||
"expiry/A10-reconnect/laptop/self": selfExpiredFlags,
|
||||
|
||||
"expiry/A6a-expire-again/laptop/online-observer": expiredPeerOffline,
|
||||
"expiry/A6a-expire-again/laptop/online-peer1": expiredPeerOffline,
|
||||
"expiry/A6a-expire-again/laptop/paths-observer": expiredPeerPaths,
|
||||
"expiry/A6a-expire-again/laptop/paths-peer1": expiredPeerPaths,
|
||||
"expiry/A6a-expire-again/laptop/routes-observer": expiredRouterUnserved,
|
||||
"expiry/A6a-expire-again/laptop/routes-peer1": expiredRouterUnserved,
|
||||
"expiry/A6a-expire-again/laptop/self": selfExpiredFlags,
|
||||
|
||||
"expiry/A6b-rotate-disco/laptop/online-observer": expiredPeerOffline,
|
||||
@@ -117,9 +110,6 @@ var expirySkipReasons = map[string]string{
|
||||
"expiry/A6b-rotate-disco/laptop/paths-observer": expiredPeerPaths,
|
||||
"expiry/A6b-rotate-disco/laptop/paths-peer1": expiredPeerPaths,
|
||||
"expiry/A6b-rotate-disco/laptop/paths-peer3": expiredPeerPaths,
|
||||
"expiry/A6b-rotate-disco/laptop/routes-observer": expiredRouterUnserved,
|
||||
"expiry/A6b-rotate-disco/laptop/routes-peer1": expiredRouterUnserved,
|
||||
"expiry/A6b-rotate-disco/laptop/routes-peer3": expiredRouterUnserved,
|
||||
"expiry/A6b-rotate-disco/laptop/self": selfExpiredFlags,
|
||||
}
|
||||
|
||||
|
||||
@@ -3,10 +3,13 @@ package servertest_test
|
||||
import (
|
||||
"context"
|
||||
"net/netip"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/juanfont/headscale/hscontrol/servertest"
|
||||
"github.com/juanfont/headscale/hscontrol/types"
|
||||
"github.com/juanfont/headscale/hscontrol/types/change"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"tailscale.com/tailcfg"
|
||||
@@ -1113,3 +1116,136 @@ func hasDstRules(matches []filtertype.Match) bool {
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func TestGrantViaSubnetFailover(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
route := netip.MustParsePrefix("10.0.0.0/24")
|
||||
setup := func(
|
||||
t *testing.T,
|
||||
) (*servertest.TestServer, map[string]*servertest.TestClient) {
|
||||
t.Helper()
|
||||
|
||||
srv := servertest.NewServer(t)
|
||||
user := srv.CreateUser(t, "via-ha-user")
|
||||
|
||||
changed, err := srv.State().SetPolicy([]byte(`{
|
||||
"tagOwners": {
|
||||
"tag:other": ["via-ha-user@"],
|
||||
"tag:router": ["via-ha-user@"],
|
||||
"tag:client": ["via-ha-user@"]
|
||||
},
|
||||
"grants": [{
|
||||
"src": ["tag:client"],
|
||||
"dst": ["10.0.0.0/24"],
|
||||
"ip": ["*"],
|
||||
"via": ["tag:router"]
|
||||
}],
|
||||
"autoApprovers": {
|
||||
"routes": {
|
||||
"10.0.0.0/24": ["tag:other", "tag:router"]
|
||||
}
|
||||
}
|
||||
}`))
|
||||
require.NoError(t, err)
|
||||
|
||||
if changed {
|
||||
changes, err := srv.State().ReloadPolicy()
|
||||
require.NoError(t, err)
|
||||
srv.App.Change(changes...)
|
||||
}
|
||||
|
||||
clients := map[string]*servertest.TestClient{}
|
||||
|
||||
// Register the router outside the via group first to hold global primary.
|
||||
for _, c := range []struct{ name, tag string }{
|
||||
{"other", "tag:other"},
|
||||
{"router-1", "tag:router"},
|
||||
{"router-2", "tag:router"},
|
||||
{"client", "tag:client"},
|
||||
} {
|
||||
clients[c.name] = servertest.NewClient(t, srv, c.name,
|
||||
servertest.WithUser(user), servertest.WithTags(c.tag))
|
||||
}
|
||||
|
||||
for _, name := range []string{"other", "router-1", "router-2"} {
|
||||
clients[name].Direct().SetHostinfo(&tailcfg.Hostinfo{
|
||||
BackendLogID: "servertest-" + name,
|
||||
Hostname: name,
|
||||
RoutableIPs: []netip.Prefix{route},
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
|
||||
_ = clients[name].Direct().SendUpdate(ctx)
|
||||
|
||||
cancel()
|
||||
}
|
||||
|
||||
return srv, clients
|
||||
}
|
||||
|
||||
onlyHolder := func(want string) func(*netmap.NetworkMap) bool {
|
||||
return func(nm *netmap.NetworkMap) bool {
|
||||
var holders []string
|
||||
|
||||
for _, p := range nm.Peers {
|
||||
if slices.Contains(p.AllowedIPs().AsSlice(), route) {
|
||||
holders = append(holders, p.Hostinfo().Hostname())
|
||||
}
|
||||
}
|
||||
|
||||
return slices.Equal(holders, []string{want})
|
||||
}
|
||||
}
|
||||
|
||||
setExpiry := func(t *testing.T, srv *servertest.TestServer, id types.NodeID, expiry *time.Time) {
|
||||
t.Helper()
|
||||
|
||||
_, c, err := srv.State().SetNodeExpiry(id, expiry)
|
||||
require.NoError(t, err)
|
||||
srv.App.Change(c)
|
||||
}
|
||||
|
||||
for _, mode := range []string{"offline", "unhealthy", "expired"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
srv, clients := setup(t)
|
||||
clients["client"].WaitForCondition(t, "route on router-1",
|
||||
15*time.Second, onlyHolder("router-1"))
|
||||
otherID := findNodeID(t, srv, "other")
|
||||
router1ID := findNodeID(t, srv, "router-1")
|
||||
require.Contains(t, srv.State().GetNodePrimaryRoutes(otherID), route)
|
||||
|
||||
switch mode {
|
||||
case "offline":
|
||||
clients["router-1"].Disconnect(t)
|
||||
case "unhealthy":
|
||||
changed := srv.State().SetNodeHealth(router1ID, false)
|
||||
require.True(t, changed, "a via health change must trigger a recompute")
|
||||
srv.App.Change(change.PolicyChange())
|
||||
case "expired":
|
||||
setExpiry(t, srv, router1ID, new(time.Now().Add(-time.Minute)))
|
||||
}
|
||||
|
||||
clients["client"].WaitForCondition(t, "route moves to router-2",
|
||||
30*time.Second, onlyHolder("router-2"))
|
||||
assert.Contains(t, srv.State().GetNodePrimaryRoutes(otherID), route)
|
||||
|
||||
switch mode {
|
||||
case "offline":
|
||||
clients["router-1"].Reconnect(t)
|
||||
case "unhealthy":
|
||||
changed := srv.State().SetNodeHealth(router1ID, true)
|
||||
require.True(t, changed, "a via recovery must trigger a recompute")
|
||||
srv.App.Change(change.PolicyChange())
|
||||
case "expired":
|
||||
setExpiry(t, srv, router1ID, nil)
|
||||
}
|
||||
|
||||
clients["client"].WaitForCondition(t, "route returns to router-1",
|
||||
30*time.Second, onlyHolder("router-1"))
|
||||
assert.Contains(t, srv.State().GetNodePrimaryRoutes(otherID), route)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -153,8 +153,8 @@ func checkHAInvariants(
|
||||
st := srv.State()
|
||||
primaries := readPrimaries(rt, srv, clients)
|
||||
|
||||
// Collect every prefix advertised by the fleet so the all-down
|
||||
// honesty check can iterate them, even prefixes with zero current
|
||||
// Collect every prefix advertised by the fleet so the no-router
|
||||
// check can iterate them, even prefixes with zero current
|
||||
// advertisers.
|
||||
prefixSet := make(map[netip.Prefix]struct{})
|
||||
for _, c := range clients {
|
||||
@@ -167,7 +167,10 @@ func checkHAInvariants(
|
||||
// prober's mapBatcher.IsConnected view is a separate gate the
|
||||
// prober consults before dispatching a probe, not part of the
|
||||
// election input.
|
||||
// approversByPrefix also holds the offline routers, which keep a
|
||||
// prefix while no candidate can serve it.
|
||||
advertisersByPrefix := make(map[netip.Prefix][]types.NodeID)
|
||||
approversByPrefix := make(map[netip.Prefix][]types.NodeID)
|
||||
|
||||
for _, c := range clients {
|
||||
nv, ok := st.GetNodeByID(c.id)
|
||||
@@ -176,12 +179,15 @@ func checkHAInvariants(
|
||||
}
|
||||
|
||||
online, known := nv.IsOnline().GetOk()
|
||||
if !known || !online {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, p := range nv.AllApprovedRoutes() {
|
||||
if p == c.route {
|
||||
if p != c.route {
|
||||
continue
|
||||
}
|
||||
|
||||
approversByPrefix[p] = append(approversByPrefix[p], c.id)
|
||||
|
||||
if known && online {
|
||||
advertisersByPrefix[p] = append(advertisersByPrefix[p], c.id)
|
||||
}
|
||||
}
|
||||
@@ -207,20 +213,24 @@ func checkHAInvariants(
|
||||
)
|
||||
}
|
||||
|
||||
// Invariant 1: primary IsOnline in the NodeStore view.
|
||||
// state.Disconnect's 10s grace can leave a recently-disconnected
|
||||
// node with IsOnline=true; the structural rule still requires
|
||||
// the bit before the snapshot may keep it as primary.
|
||||
// Invariant 1: primary IsOnline in the NodeStore view while any
|
||||
// candidate is healthy. state.Disconnect's 10s grace can leave a
|
||||
// recently-disconnected node with IsOnline=true; the structural
|
||||
// rule still requires the bit before the snapshot may keep it as
|
||||
// primary over a healthy candidate.
|
||||
healthyCandidate := slices.ContainsFunc(advertisersByPrefix[prefix], st.IsNodeHealthy)
|
||||
|
||||
online, known := nv.IsOnline().GetOk()
|
||||
if !known || !online {
|
||||
if (!known || !online) && healthyCandidate {
|
||||
rt.Fatalf(
|
||||
"prefix %s primary %d is not online (IsOnline=%v known=%v)",
|
||||
prefix, owner, online, known,
|
||||
"prefix %s primary %d is not online (IsOnline=%v known=%v) "+
|
||||
"but %v has a healthy candidate",
|
||||
prefix, owner, online, known, advertisersByPrefix[prefix],
|
||||
)
|
||||
}
|
||||
|
||||
// Invariant 2: primary advertises the prefix and is among the
|
||||
// snapshot's candidate set.
|
||||
// Invariant 2: primary advertises the prefix and, while any
|
||||
// candidate is healthy, is among the snapshot's candidate set.
|
||||
approved := nv.AllApprovedRoutes()
|
||||
if !slices.Contains(approved, prefix) {
|
||||
rt.Fatalf(
|
||||
@@ -229,7 +239,7 @@ func checkHAInvariants(
|
||||
)
|
||||
}
|
||||
|
||||
if !slices.Contains(advertisersByPrefix[prefix], owner) {
|
||||
if healthyCandidate && !slices.Contains(advertisersByPrefix[prefix], owner) {
|
||||
rt.Fatalf(
|
||||
"prefix %s primary %d not in advertiser set %v",
|
||||
prefix, owner, advertisersByPrefix[prefix],
|
||||
@@ -265,72 +275,49 @@ func checkHAInvariants(
|
||||
}
|
||||
}
|
||||
|
||||
// Invariant 6 (all-down honesty): if every candidate of a prefix is
|
||||
// either offline OR unhealthy, the snapshot must either drop the
|
||||
// primary OR keep one that is still a candidate from the same set
|
||||
// (the all-unhealthy preserve-prev rule). A primary pointed at a
|
||||
// non-candidate after every candidate has gone dark would point
|
||||
// peers at a node the prober has already declared unreachable.
|
||||
// Invariant 6 (no router available): with no healthy online
|
||||
// candidate the prefix stays mapped. The previous primary keeps it
|
||||
// while it still approves the prefix; otherwise the primary must be
|
||||
// a router the prober has not declared unreachable, or there is none.
|
||||
for prefix := range prefixSet {
|
||||
owner, hasPrimary := primaries[prefix]
|
||||
if !hasPrimary {
|
||||
continue
|
||||
}
|
||||
|
||||
candidates := advertisersByPrefix[prefix]
|
||||
|
||||
// Empty candidates with a primary present trips invariant 2
|
||||
// above; reaching this branch with len(candidates) == 0 means
|
||||
// the snapshot is internally inconsistent.
|
||||
if len(candidates) == 0 {
|
||||
rt.Fatalf(
|
||||
"prefix %s has primary %d but no advertiser in the snapshot",
|
||||
prefix, owner,
|
||||
)
|
||||
}
|
||||
|
||||
anyHealthy := slices.ContainsFunc(candidates, st.IsNodeHealthy)
|
||||
|
||||
// Healthy preference: a primary should not be unhealthy when a
|
||||
// healthy candidate exists. Asserting this through the real
|
||||
// prober → State → NodeStore seam catches any divergence the
|
||||
// unit-level model would miss.
|
||||
if anyHealthy && !st.IsNodeHealthy(owner) {
|
||||
if hasPrimary && anyHealthy && !st.IsNodeHealthy(owner) {
|
||||
rt.Fatalf(
|
||||
"prefix %s primary %d unhealthy but %v has a healthy candidate",
|
||||
prefix, owner, candidates,
|
||||
)
|
||||
}
|
||||
|
||||
// All-unhealthy guards: with every candidate unhealthy, the
|
||||
// election has exactly two correct choices:
|
||||
// - preserve prev primary when it is still a candidate
|
||||
// - leave the prefix unmapped (no candidate fallback)
|
||||
//
|
||||
// Any other outcome points peers at a node already declared
|
||||
// unreachable.
|
||||
if !anyHealthy {
|
||||
prevOwner, hadPrev := prevPrimaries[prefix]
|
||||
prevStillCandidate := hadPrev &&
|
||||
slices.Contains(candidates, prevOwner)
|
||||
if anyHealthy {
|
||||
continue
|
||||
}
|
||||
|
||||
if prevStillCandidate && owner != prevOwner {
|
||||
rt.Fatalf(
|
||||
"prefix %s all-unhealthy election picked %d, "+
|
||||
"but prev primary %d is still a candidate — "+
|
||||
"preserve-prev rule violated",
|
||||
prefix, owner, prevOwner,
|
||||
)
|
||||
}
|
||||
prevOwner, hadPrev := prevPrimaries[prefix]
|
||||
prevStillApproves := hadPrev &&
|
||||
slices.Contains(approversByPrefix[prefix], prevOwner)
|
||||
|
||||
if !prevStillCandidate && (!hadPrev || owner != prevOwner) {
|
||||
rt.Fatalf(
|
||||
"prefix %s all-unhealthy fallback elected %d "+
|
||||
"but prev primary %v is gone; election "+
|
||||
"must leave the prefix unmapped",
|
||||
prefix, owner, prevOwner,
|
||||
)
|
||||
}
|
||||
if prevStillApproves && owner != prevOwner {
|
||||
rt.Fatalf(
|
||||
"prefix %s with no router available elected %d, "+
|
||||
"but prev primary %d still approves it",
|
||||
prefix, owner, prevOwner,
|
||||
)
|
||||
}
|
||||
|
||||
if !prevStillApproves && hasPrimary && owner != prevOwner &&
|
||||
!st.IsNodeHealthy(owner) {
|
||||
rt.Fatalf(
|
||||
"prefix %s with no router available moved to %d, "+
|
||||
"which the prober declared unreachable",
|
||||
prefix, owner,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -367,14 +354,14 @@ func snapshotPrimaries(
|
||||
// so a reconnect mid-cycle does not install a stale Unhealthy.
|
||||
//
|
||||
// - The batched BatchSetNodeHealth is observable through the
|
||||
// all-unhealthy preserve-prev invariant: per-call writes would
|
||||
// no-router preserve-prev invariant: per-call writes would
|
||||
// publish an intermediate "one unhealthy, one healthy" snapshot,
|
||||
// leaving primary on a non-prev node after both flips land.
|
||||
//
|
||||
// - The all-unhealthy fallback that leaves prefixes unmapped is
|
||||
// covered by the honesty invariant: with every candidate
|
||||
// unhealthy AND prev gone from the candidate set, the prefix
|
||||
// must stay unmapped instead of pinned to any candidate.
|
||||
// - The fallback with no router available is covered by the
|
||||
// no-router invariant: the previous primary keeps the prefix
|
||||
// while it still approves it, and the fallback never moves it
|
||||
// to a router the prober declared unreachable.
|
||||
//
|
||||
// Caveat: TestClient is a real controlclient.Direct that responds to
|
||||
// PingRequest over Noise, so the timeout path of the prober rarely
|
||||
|
||||
@@ -51,8 +51,6 @@ var lifecycleSkipReasons = map[string]string{
|
||||
"route-approval/b9/served": approvalOnSameKeyReauth,
|
||||
"route-approval/b9/self-routes": approvalOnSameKeyReauth,
|
||||
|
||||
"route-approval/r1/served": loggedOutRouterUnserved,
|
||||
|
||||
"ephemeral-lifecycle/E1-logout/eph-logout/outcome": logoutAnswerUnauthorized,
|
||||
"ephemeral-lifecycle/E1-logout/eph-logout-user/outcome": logoutAnswerUnauthorized,
|
||||
|
||||
@@ -81,7 +79,6 @@ const (
|
||||
approvalOnHostinfo = "APPROVAL_ON_HOSTINFO: headscale auto-approves when a router with no approved routes sends a Hostinfo change that keeps its routes; SaaS does not"
|
||||
approvalOnPolicy = "APPROVAL_ON_POLICY: headscale auto-approves announced routes when a policy is set; SaaS waits for the router's next route change or key rotation"
|
||||
approvalOnSameKeyReauth = "APPROVAL_ON_SAME_KEY_REAUTH: headscale auto-approves on a re-auth that keeps the node key; SaaS does not"
|
||||
loggedOutRouterUnserved = "LOGGED_OUT_ROUTER_UNSERVED: headscale stops serving a logged-out router's routes; SaaS keeps them on the expired, offline peer"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -93,8 +90,7 @@ const (
|
||||
|
||||
// Used by reauthSkipReasons.
|
||||
const (
|
||||
expiredRouterUnserved = "EXPIRED_ROUTER_UNSERVED: headscale stops serving an expired router's routes; SaaS keeps them in the peer's AllowedIPs and PrimaryRoutes"
|
||||
expiredPeerOffline = "EXPIRED_PEER_OFFLINE: headscale shows an expired peer offline while it is connected; SaaS shows it online"
|
||||
expiredPeerOffline = "EXPIRED_PEER_OFFLINE: headscale shows an expired peer offline while it is connected; SaaS shows it online"
|
||||
)
|
||||
|
||||
// TestRouteApprovalCompat replays a user-owned router through admin
|
||||
|
||||
@@ -37,7 +37,6 @@ import (
|
||||
// <test_id>/<step>[/<node>]/<check>. A listed check that matches fails, so a
|
||||
// fix must delete its entry.
|
||||
var reauthSkipReasons = map[string]string{
|
||||
"owner-routes/r-expired-retag/expire/served": expiredRouterUnserved,
|
||||
"owner-routes/r-expired-retag/expire/online": expiredPeerOffline,
|
||||
|
||||
"owner-routes/r-expired-retag/owner/node-key": expiredKeyAccepted,
|
||||
|
||||
@@ -307,11 +307,14 @@ func TestRoutes(t *testing.T) {
|
||||
20*time.Second, 200*time.Millisecond,
|
||||
"r2 should take over as primary after r1 offline")
|
||||
|
||||
// 2. Take r2 offline. With both routers gone, no primary should remain.
|
||||
// 2. Take r2 offline. With both routers gone, r2 keeps the route.
|
||||
r2.Disconnect(t)
|
||||
assert.Eventually(t, func() bool { return !hasRoute(r1ID) && !hasRoute(r2ID) },
|
||||
20*time.Second, 200*time.Millisecond,
|
||||
"no primary should be assigned while both routers are offline")
|
||||
assert.Eventually(t, func() bool {
|
||||
nv, ok := srv.State().GetNodeByID(r2ID)
|
||||
|
||||
return ok && !nv.IsOnline().Get() && hasRoute(r2ID) && !hasRoute(r1ID)
|
||||
}, 20*time.Second, 200*time.Millisecond,
|
||||
"the last primary keeps the route while both routers are offline")
|
||||
|
||||
// 3. Reconnect r2 (cable plugged back in).
|
||||
r2.Reconnect(t)
|
||||
|
||||
@@ -222,22 +222,14 @@ func runViaHACompat(t *testing.T, c *testcapture.Capture) {
|
||||
// keyed <test_id>/<step>/<viewer>/<check>. A listed check that matches
|
||||
// fails, so a fix must delete its entry.
|
||||
var viaHAFailoverSkipReasons = map[string]string{
|
||||
"via-ha-failover/V1/client-a/route-owner": viaOfflineRouterKept,
|
||||
"via-ha-failover/V9/client-a/route-owner": viaOfflineRouterKept,
|
||||
|
||||
"via-ha-failover/V2/client-a/route-owner": viaLowestIDAfterRestart,
|
||||
"via-ha-failover/V10/client-a/route-owner": viaLowestIDAfterRestart,
|
||||
|
||||
"via-ha-failover/V5/client-a/route-owner": viaExpiredRouterKept,
|
||||
"via-ha-failover/V5b/client-a/route-owner": viaExpiredRouterKept,
|
||||
|
||||
"via-ha-failover/V12/client-a/route-owner": readvertisedRouterTakesRoute,
|
||||
"via-ha-failover/V12/client-b/route-owner": readvertisedRouterTakesRoute,
|
||||
}
|
||||
|
||||
const (
|
||||
viaOfflineRouterKept = "VIA_OFFLINE_ROUTER_KEPT: headscale keeps a via route on an offline router while another router of the via group is online; SaaS moves it to the online one"
|
||||
viaExpiredRouterKept = "VIA_EXPIRED_ROUTER_KEPT: headscale keeps a via route on a router whose key expired while another router of the via group is live; SaaS moves it to the live one"
|
||||
viaLowestIDAfterRestart = "VIA_LOWEST_ID_AFTER_RESTART: headscale serves a via route from the group's lowest-ID router once it reconnects; SaaS keeps the router that served while it was down"
|
||||
readvertisedRouterTakesRoute = "READVERTISED_ROUTER_TAKES_ROUTE: SaaS hands a route back to an HA router that withdrew and re-advertised it, taking it from the router that served it meanwhile; headscale keeps that router"
|
||||
)
|
||||
|
||||
@@ -206,14 +206,18 @@ type work struct {
|
||||
// updateChanges reports whether an in-place update moved a peer-visibility
|
||||
// input (which also re-elects routes) or only a route-election input.
|
||||
// Peer visibility depends on what the policy reads; election additionally
|
||||
// depends on online and health state, treating unknown online as offline.
|
||||
// depends on online, expiry and health state, treating unknown online as
|
||||
// offline. Fields are compared, not [servesRoutes]: an expiry that passed
|
||||
// before the write reads the same on both sides.
|
||||
func updateChanges(pre, post *types.Node) (bool, bool) {
|
||||
preView, postView := pre.View(), post.View()
|
||||
if postView.HasPolicyChange(preView) || postView.HasNetworkChanges(preView) {
|
||||
return true, true
|
||||
}
|
||||
|
||||
return false, pre.Online() != post.Online() || pre.Unhealthy != post.Unhealthy
|
||||
return false, pre.Online() != post.Online() ||
|
||||
!preView.Expiry().GetOr(time.Time{}).Equal(postView.Expiry().GetOr(time.Time{})) ||
|
||||
pre.Unhealthy != post.Unhealthy
|
||||
}
|
||||
|
||||
// PutNode adds or updates a node in the store.
|
||||
@@ -778,18 +782,25 @@ func snapshotFromNodes(
|
||||
return newSnap
|
||||
}
|
||||
|
||||
// onlineAdvertisers maps each non-exit prefix to the IDs of online nodes
|
||||
// that approve it. Nodes are visited in the order of ids, so the per-prefix
|
||||
// servesRoutes reports whether n can carry its routes. Expiry is tested
|
||||
// apart from Online because an expired node keeps its control session.
|
||||
func servesRoutes(n *types.Node) bool {
|
||||
return n.Online() && !n.IsExpired()
|
||||
}
|
||||
|
||||
// advertisers maps each non-exit prefix to the IDs of nodes that approve it
|
||||
// and pass keep. Nodes are visited in the order of ids, so the per-prefix
|
||||
// slices preserve that order.
|
||||
func onlineAdvertisers(
|
||||
func advertisers(
|
||||
nodes map[types.NodeID]types.Node,
|
||||
ids []types.NodeID,
|
||||
keep func(*types.Node) bool,
|
||||
) map[netip.Prefix][]types.NodeID {
|
||||
advertisers := make(map[netip.Prefix][]types.NodeID)
|
||||
out := make(map[netip.Prefix][]types.NodeID)
|
||||
|
||||
for _, id := range ids {
|
||||
n := nodes[id]
|
||||
if n.IsOnline == nil || !*n.IsOnline {
|
||||
if !keep(&n) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -798,65 +809,63 @@ func onlineAdvertisers(
|
||||
continue
|
||||
}
|
||||
|
||||
advertisers[p] = append(advertisers[p], id)
|
||||
out[p] = append(out[p], id)
|
||||
}
|
||||
}
|
||||
|
||||
return advertisers
|
||||
return out
|
||||
}
|
||||
|
||||
// electPrimaryRoutes picks the primary advertiser for each non-exit
|
||||
// prefix. Inputs are restricted to online nodes that advertise the
|
||||
// prefix. The previous primary is preserved when it is still online
|
||||
// and healthy (anti-flap); otherwise the lowest-NodeID healthy
|
||||
// advertiser wins. When every advertiser is unhealthy the previous
|
||||
// primary is preserved only if still a candidate — falling back to
|
||||
// any other candidate would point peers at a node the prober has
|
||||
// already declared unreachable, so leaving the prefix unmapped is
|
||||
// preferred until a probe cycle finds one that responds.
|
||||
// electPrimaryRoutes picks the primary router for each non-exit prefix any
|
||||
// node approves. A router is available when it is online, unexpired and
|
||||
// healthy. The previous primary is preserved while it is available
|
||||
// (anti-flap); otherwise the lowest-NodeID available router wins.
|
||||
//
|
||||
// With no router available the prefix stays mapped, so peers keep the route
|
||||
// and use it once its router returns: the previous primary keeps it while it
|
||||
// still approves the prefix, else the lowest-NodeID approving router does.
|
||||
// That fallback never picks a router the prober declared unreachable; with
|
||||
// only such routers left the prefix is unmapped until a probe cycle finds one
|
||||
// that responds.
|
||||
func electPrimaryRoutes(
|
||||
nodes map[types.NodeID]types.Node,
|
||||
prev map[netip.Prefix]types.NodeID,
|
||||
) (map[netip.Prefix]types.NodeID, map[types.NodeID]bool) {
|
||||
advertisers := onlineAdvertisers(nodes, slices.Sorted(maps.Keys(nodes)))
|
||||
approvers := advertisers(nodes, slices.Sorted(maps.Keys(nodes)),
|
||||
func(*types.Node) bool { return true })
|
||||
|
||||
routes := make(map[netip.Prefix]types.NodeID, len(advertisers))
|
||||
for prefix, candidates := range advertisers {
|
||||
if cur, ok := prev[prefix]; ok &&
|
||||
slices.Contains(candidates, cur) &&
|
||||
!nodes[cur].Unhealthy {
|
||||
available := func(id types.NodeID) bool {
|
||||
n := nodes[id]
|
||||
|
||||
return servesRoutes(&n) && !n.Unhealthy
|
||||
}
|
||||
healthy := func(id types.NodeID) bool { return !nodes[id].Unhealthy }
|
||||
|
||||
routes := make(map[netip.Prefix]types.NodeID, len(approvers))
|
||||
for prefix, ids := range approvers {
|
||||
cur, hasPrev := prev[prefix]
|
||||
hasPrev = hasPrev && slices.Contains(ids, cur)
|
||||
|
||||
if hasPrev && available(cur) {
|
||||
routes[prefix] = cur
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
var (
|
||||
selected types.NodeID
|
||||
found bool
|
||||
)
|
||||
if i := slices.IndexFunc(ids, available); i >= 0 {
|
||||
routes[prefix] = ids[i]
|
||||
|
||||
for _, c := range candidates {
|
||||
if !nodes[c].Unhealthy {
|
||||
selected = c
|
||||
found = true
|
||||
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// All-unhealthy fallback: preserve the previous primary only
|
||||
// when it is still a candidate. Falling back to any candidate
|
||||
// would point peers at a node the prober has already declared
|
||||
// unreachable; leaving the prefix unmapped is honest until a
|
||||
// probe cycle picks one that responds.
|
||||
if !found {
|
||||
if cur, ok := prev[prefix]; ok && slices.Contains(candidates, cur) {
|
||||
selected = cur
|
||||
found = true
|
||||
}
|
||||
if hasPrev {
|
||||
routes[prefix] = cur
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if found {
|
||||
routes[prefix] = selected
|
||||
if i := slices.IndexFunc(ids, healthy); i >= 0 {
|
||||
routes[prefix] = ids[i]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1051,9 +1060,10 @@ func (s *NodeStore) PrimaryRoutesForNode(id types.NodeID) []netip.Prefix {
|
||||
func (s *NodeStore) HANodes() map[netip.Prefix][]types.NodeID {
|
||||
snap := s.data.Load()
|
||||
|
||||
advertisers := onlineAdvertisers(
|
||||
online := advertisers(
|
||||
snap.nodesByID,
|
||||
slices.Sorted(maps.Keys(snap.nodesByID)),
|
||||
servesRoutes,
|
||||
)
|
||||
|
||||
unhealthy := func(id types.NodeID) bool {
|
||||
@@ -1062,7 +1072,7 @@ func (s *NodeStore) HANodes() map[netip.Prefix][]types.NodeID {
|
||||
|
||||
out := make(map[netip.Prefix][]types.NodeID)
|
||||
|
||||
for p, ids := range advertisers {
|
||||
for p, ids := range online {
|
||||
if len(ids) < 2 && !slices.ContainsFunc(ids, unhealthy) {
|
||||
continue
|
||||
}
|
||||
@@ -1085,6 +1095,14 @@ func (s *NodeStore) IsNodeHealthy(id types.NodeID) bool {
|
||||
return !n.Unhealthy
|
||||
}
|
||||
|
||||
// IsRouterAvailable reports whether id may win a primary election: online,
|
||||
// unexpired and healthy.
|
||||
func (s *NodeStore) IsRouterAvailable(id types.NodeID) bool {
|
||||
n, ok := s.data.Load().nodesByID[id]
|
||||
|
||||
return ok && servesRoutes(&n) && !n.Unhealthy
|
||||
}
|
||||
|
||||
// PrimaryRoutes returns the snapshot's prefix→primary map. The map is
|
||||
// owned by the snapshot and must not be mutated; it is safe to read
|
||||
// concurrently because snapshots are immutable once published.
|
||||
|
||||
@@ -1910,7 +1910,7 @@ var nodeFieldImpact = map[string]string{
|
||||
"MachineKey": "payload", "NodeKey": "payload", "DiscoKey": "payload",
|
||||
"Endpoints": "payload", "Hostname": "payload", "GivenName": "payload",
|
||||
"RegisterMethod": "payload", "AuthKeyID": "payload", "AuthKey": "payload",
|
||||
"Expiry": "payload", "LastSeen": "payload", "CreatedAt": "payload",
|
||||
"Expiry": "election", "LastSeen": "payload", "CreatedAt": "payload",
|
||||
"UpdatedAt": "payload", "DeletedAt": "payload",
|
||||
"ActiveSessions": "payload", "SessionEpoch": "payload",
|
||||
}
|
||||
@@ -1966,6 +1966,7 @@ func TestUpdateChangesReportsRelationFields(t *testing.T) {
|
||||
},
|
||||
{name: "online", mutate: func(n *types.Node) { n.IsOnline = new(true) }, wantElection: true},
|
||||
{name: "unhealthy", mutate: func(n *types.Node) { n.Unhealthy = true }, wantElection: true},
|
||||
{name: "expiry", mutate: func(n *types.Node) { n.Expiry = new(time.Now().Add(-time.Minute)) }, wantElection: true},
|
||||
{name: "endpoint", mutate: func(n *types.Node) { n.Endpoints = []netip.AddrPort{netip.MustParseAddrPort("192.0.2.1:1")} }},
|
||||
{name: "lastseen", mutate: func(n *types.Node) { n.LastSeen = new(time.Now()) }},
|
||||
}
|
||||
|
||||
@@ -91,11 +91,23 @@ func (m *primariesModel) recomputeEffective(id types.NodeID) {
|
||||
func (m *primariesModel) advertisersByPrefix() map[netip.Prefix][]types.NodeID {
|
||||
out := map[netip.Prefix][]types.NodeID{}
|
||||
|
||||
for n, prefs := range m.prefixes {
|
||||
if !m.connected[n] {
|
||||
continue
|
||||
for p, nodes := range m.approversByPrefix() {
|
||||
for _, n := range nodes {
|
||||
if m.connected[n] {
|
||||
out[p] = append(out[p], n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// approversByPrefix returns every node that approves each prefix, connected
|
||||
// or not, sorted by NodeID.
|
||||
func (m *primariesModel) approversByPrefix() map[netip.Prefix][]types.NodeID {
|
||||
out := map[netip.Prefix][]types.NodeID{}
|
||||
|
||||
for n, prefs := range m.prefixes {
|
||||
for _, p := range prefs {
|
||||
out[p] = append(out[p], n)
|
||||
}
|
||||
@@ -108,78 +120,57 @@ func (m *primariesModel) advertisersByPrefix() map[netip.Prefix][]types.NodeID {
|
||||
return out
|
||||
}
|
||||
|
||||
// available reports whether n may win an election: connected and healthy.
|
||||
func (m *primariesModel) available(n types.NodeID) bool {
|
||||
return m.connected[n] && !m.unhealthy[n]
|
||||
}
|
||||
|
||||
// updatePrimaries reapplies the algorithm to recompute the primary
|
||||
// for each prefix. Called after every operation.
|
||||
func (m *primariesModel) updatePrimaries() {
|
||||
advertisers := m.advertisersByPrefix()
|
||||
approvers := m.approversByPrefix()
|
||||
|
||||
// Drop primaries for prefixes that no longer have any advertiser.
|
||||
// Drop primaries for prefixes no node approves any more.
|
||||
for p := range m.primary {
|
||||
if _, ok := advertisers[p]; !ok {
|
||||
if _, ok := approvers[p]; !ok {
|
||||
delete(m.primary, p)
|
||||
}
|
||||
}
|
||||
|
||||
for p, nodes := range advertisers {
|
||||
if cur, ok := m.primary[p]; ok {
|
||||
if slices.Contains(nodes, cur) && !m.unhealthy[cur] {
|
||||
continue
|
||||
}
|
||||
for p, nodes := range approvers {
|
||||
cur, hasPrev := m.primary[p]
|
||||
hasPrev = hasPrev && slices.Contains(nodes, cur)
|
||||
|
||||
if hasPrev && m.available(cur) {
|
||||
continue
|
||||
}
|
||||
|
||||
var (
|
||||
selected types.NodeID
|
||||
found bool
|
||||
)
|
||||
if i := slices.IndexFunc(nodes, m.available); i >= 0 {
|
||||
m.primary[p] = nodes[i]
|
||||
|
||||
for _, n := range nodes {
|
||||
if !m.unhealthy[n] {
|
||||
selected = n
|
||||
found = true
|
||||
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// All-unhealthy fallback: preserve the previous primary if it
|
||||
// is still a candidate, otherwise leave the prefix unmapped.
|
||||
// Choosing any candidate would point peers at a node already
|
||||
// declared unreachable; the model mirrors that policy.
|
||||
if !found && len(nodes) >= 1 {
|
||||
if cur, ok := m.primary[p]; ok && slices.Contains(nodes, cur) {
|
||||
selected = cur
|
||||
found = true
|
||||
}
|
||||
// No router available: the previous primary keeps the prefix,
|
||||
// else the lowest-ID approver the prober has not declared
|
||||
// unreachable; with neither the prefix is unmapped.
|
||||
if hasPrev {
|
||||
continue
|
||||
}
|
||||
|
||||
if found {
|
||||
m.primary[p] = selected
|
||||
if i := slices.IndexFunc(nodes, func(n types.NodeID) bool {
|
||||
return !m.unhealthy[n]
|
||||
}); i >= 0 {
|
||||
m.primary[p] = nodes[i]
|
||||
} else {
|
||||
delete(m.primary, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// allPrefixes returns every prefix mentioned by any connected node.
|
||||
// allPrefixes returns every prefix any node approves.
|
||||
func (m *primariesModel) allPrefixes() []netip.Prefix {
|
||||
seen := map[netip.Prefix]bool{}
|
||||
|
||||
for n, prefs := range m.prefixes {
|
||||
if !m.connected[n] {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, p := range prefs {
|
||||
seen[p] = true
|
||||
}
|
||||
}
|
||||
|
||||
out := make([]netip.Prefix, 0, len(seen))
|
||||
for p := range seen {
|
||||
out = append(out, p)
|
||||
}
|
||||
|
||||
return out
|
||||
return slices.Collect(maps.Keys(m.approversByPrefix()))
|
||||
}
|
||||
|
||||
func samePrefixSet(a, b []netip.Prefix) bool {
|
||||
@@ -233,10 +224,9 @@ func checkPrimariesProperties(
|
||||
}
|
||||
}
|
||||
|
||||
// Every prefix that has at least one connected advertiser must
|
||||
// have a primary in the snapshot: a prefix silently losing its
|
||||
// primary after a disconnect/reconnect cycle would leave peers
|
||||
// without a route.
|
||||
// Every prefix the model maps must have the same primary in the
|
||||
// snapshot: a prefix silently losing its primary after a
|
||||
// disconnect/reconnect cycle would leave peers without a route.
|
||||
for _, p := range m.allPrefixes() {
|
||||
want, expectExists := m.primary[p]
|
||||
if !expectExists {
|
||||
@@ -246,7 +236,7 @@ func checkPrimariesProperties(
|
||||
got, ok := ns.PrimaryRouteFor(p)
|
||||
if !ok {
|
||||
rt.Fatalf(
|
||||
"prefix %s has at least one advertiser in the model but no primary in NodeStore",
|
||||
"prefix %s has a primary in the model but none in NodeStore",
|
||||
p,
|
||||
)
|
||||
}
|
||||
@@ -261,7 +251,7 @@ func checkPrimariesProperties(
|
||||
|
||||
// Structural invariants on the live snapshot, independent of the
|
||||
// model. These catch shapes the model alone cannot — e.g. an owner
|
||||
// that is offline but still has the prefix attributed to it.
|
||||
// that no longer approves the prefix attributed to it.
|
||||
snapshotPrimaries := ns.PrimaryRoutes()
|
||||
|
||||
// A primary that owns ≥1 prefix in `routes` must also light up
|
||||
@@ -283,9 +273,11 @@ func checkPrimariesProperties(
|
||||
}
|
||||
}
|
||||
|
||||
// Per-owner structural checks: every primary must currently be
|
||||
// online and must currently advertise the prefix it owns.
|
||||
// Per-owner structural checks: every primary must currently
|
||||
// advertise the prefix it owns, and be available whenever any of
|
||||
// its approvers is.
|
||||
advertisersByPrefix := m.advertisersByPrefix()
|
||||
approversByPrefix := m.approversByPrefix()
|
||||
|
||||
for prefix, owner := range snapshotPrimaries {
|
||||
nv, ok := ns.GetNode(owner)
|
||||
@@ -296,13 +288,14 @@ func checkPrimariesProperties(
|
||||
)
|
||||
}
|
||||
|
||||
// Primary is online: an offline node cannot move packets, so
|
||||
// election must never leave one as the snapshot's primary.
|
||||
// An offline primary is only allowed while no approver could
|
||||
// move packets for the prefix.
|
||||
online, known := nv.IsOnline().GetOk()
|
||||
if !known || !online {
|
||||
if (!known || !online) &&
|
||||
slices.ContainsFunc(approversByPrefix[prefix], m.available) {
|
||||
rt.Fatalf(
|
||||
"prefix %s primary %d is not online",
|
||||
prefix, owner,
|
||||
"prefix %s primary %d is not online but %v had an available router",
|
||||
prefix, owner, approversByPrefix[prefix],
|
||||
)
|
||||
}
|
||||
|
||||
@@ -317,21 +310,17 @@ func checkPrimariesProperties(
|
||||
)
|
||||
}
|
||||
|
||||
// Healthy preference: if any candidate for this prefix is
|
||||
// healthy, the elected primary must also be healthy. Leaving
|
||||
// the prefix unmapped is fine; pointing at an unhealthy
|
||||
// candidate while a healthy one was available is not.
|
||||
// Healthy preference: if any approver for this prefix is
|
||||
// healthy, the elected primary must also be healthy, unless it
|
||||
// is the previous primary kept while no router is available.
|
||||
// Leaving the prefix unmapped is fine; moving it to an
|
||||
// unhealthy router while a healthy one was there is not.
|
||||
candidates := advertisersByPrefix[prefix]
|
||||
anyHealthy := false
|
||||
anyHealthy := slices.ContainsFunc(approversByPrefix[prefix], func(n types.NodeID) bool {
|
||||
return !m.unhealthy[n]
|
||||
})
|
||||
|
||||
for _, c := range candidates {
|
||||
if !m.unhealthy[c] {
|
||||
anyHealthy = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if anyHealthy && m.unhealthy[owner] {
|
||||
if anyHealthy && m.unhealthy[owner] && owner != prevSnapshotPrimaries[prefix] {
|
||||
rt.Fatalf(
|
||||
"prefix %s primary %d is unhealthy but %v had a healthy candidate",
|
||||
prefix, owner, candidates,
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"net/netip"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/juanfont/headscale/hscontrol/types"
|
||||
"github.com/puzpuzpuz/xsync/v4"
|
||||
@@ -171,13 +172,98 @@ func TestPrimaries_ClearRoutesDropsPrimary(t *testing.T) {
|
||||
f.requireNoPrimary(mp("192.168.1.0/24"))
|
||||
}
|
||||
|
||||
func TestPrimaries_DisconnectDropsLastAdvertiserPrimary(t *testing.T) {
|
||||
func TestPrimaries_DisconnectKeepsLastRouterPrimary(t *testing.T) {
|
||||
f := newPrimariesFixture(t, 1)
|
||||
f.advertise(1, mp("192.168.1.0/24"))
|
||||
f.requirePrimary(mp("192.168.1.0/24"), 1)
|
||||
|
||||
f.disconnect(1)
|
||||
f.requireNoPrimary(mp("192.168.1.0/24"))
|
||||
f.requirePrimary(mp("192.168.1.0/24"), 1)
|
||||
}
|
||||
|
||||
func TestPrimaries_NoRouterAvailable(t *testing.T) {
|
||||
prefix := mp("10.0.0.0/24")
|
||||
expire := func(f *primariesFixture, id types.NodeID) {
|
||||
f.ns.UpdateNode(id, func(n *types.Node) {
|
||||
n.Expiry = new(time.Now().Add(-time.Minute))
|
||||
})
|
||||
}
|
||||
unapprove := func(f *primariesFixture, id types.NodeID) {
|
||||
f.ns.UpdateNode(id, func(n *types.Node) { n.ApprovedRoutes = nil })
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
setup func(*primariesFixture)
|
||||
want types.NodeID
|
||||
}{
|
||||
{
|
||||
name: "expired router keeps its route",
|
||||
setup: func(f *primariesFixture) {
|
||||
f.advertise(1, prefix)
|
||||
expire(f, 1)
|
||||
},
|
||||
want: 1,
|
||||
},
|
||||
{
|
||||
name: "previous primary kept over a lower ID",
|
||||
setup: func(f *primariesFixture) {
|
||||
f.advertise(1, prefix)
|
||||
f.advertise(2, prefix)
|
||||
f.disconnect(1)
|
||||
f.disconnect(2)
|
||||
},
|
||||
want: 2,
|
||||
},
|
||||
{
|
||||
name: "lowest ID approver once the previous primary is gone",
|
||||
setup: func(f *primariesFixture) {
|
||||
f.advertise(1, prefix)
|
||||
f.advertise(2, prefix)
|
||||
f.advertise(3, prefix)
|
||||
f.disconnect(2)
|
||||
f.disconnect(3)
|
||||
f.disconnect(1)
|
||||
unapprove(f, 1)
|
||||
},
|
||||
want: 2,
|
||||
},
|
||||
{
|
||||
name: "unhealthy router passed over",
|
||||
setup: func(f *primariesFixture) {
|
||||
f.advertise(1, prefix)
|
||||
f.advertise(2, prefix)
|
||||
f.advertise(3, prefix)
|
||||
f.disconnect(2)
|
||||
f.unhealthy(1)
|
||||
f.disconnect(3)
|
||||
unapprove(f, 3)
|
||||
},
|
||||
want: 2,
|
||||
},
|
||||
{
|
||||
name: "only unhealthy routers left",
|
||||
setup: func(f *primariesFixture) {
|
||||
f.advertise(1, prefix)
|
||||
f.advertise(3, prefix)
|
||||
f.unhealthy(1)
|
||||
unapprove(f, 3)
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
f := newPrimariesFixture(t, 1, 2, 3)
|
||||
tt.setup(f)
|
||||
|
||||
if tt.want == 0 {
|
||||
f.requireNoPrimary(prefix)
|
||||
} else {
|
||||
f.requirePrimary(prefix, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrimaries_UnhealthyTriggersFailover(t *testing.T) {
|
||||
@@ -238,6 +324,25 @@ func TestPrimaries_AllUnhealthyPreservesPrevious(t *testing.T) {
|
||||
f.requirePrimary(prefix, 2)
|
||||
}
|
||||
|
||||
func TestPrimaries_ExpiredRouterNotElected(t *testing.T) {
|
||||
// Expiry alone takes a router out of the election, even while its
|
||||
// cached online flag still says connected.
|
||||
prefix := mp("10.0.0.0/24")
|
||||
f := newPrimariesFixture(t, 1, 2)
|
||||
f.advertise(1, prefix)
|
||||
f.advertise(2, prefix)
|
||||
f.requirePrimary(prefix, 1)
|
||||
require.True(t, f.ns.IsRouterAvailable(1))
|
||||
|
||||
f.ns.UpdateNode(1, func(n *types.Node) {
|
||||
n.Expiry = new(time.Now().Add(-time.Minute))
|
||||
})
|
||||
|
||||
f.requirePrimary(prefix, 2)
|
||||
require.False(t, f.ns.IsRouterAvailable(1))
|
||||
require.True(t, f.ns.IsRouterAvailable(2))
|
||||
}
|
||||
|
||||
func TestPrimaries_ExitRouteNotElected(t *testing.T) {
|
||||
// Exit routes (0.0.0.0/0, ::/0) are not subject to HA primary
|
||||
// election — every approved exit-route advertiser keeps it.
|
||||
@@ -249,10 +354,9 @@ func TestPrimaries_ExitRouteNotElected(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPrimaries_BothOfflineThenOneReturns(t *testing.T) {
|
||||
// With two HA advertisers, dropping both then bringing one back
|
||||
// used to leave the prefix without any primary. The snapshot
|
||||
// recomputes primaries on every NodeStore write, so the
|
||||
// returning advertiser must be elected.
|
||||
// With two HA advertisers both offline, the last primary keeps the
|
||||
// prefix. The snapshot recomputes primaries on every NodeStore
|
||||
// write, so the first advertiser to return must be elected.
|
||||
prefix := mp("10.0.0.0/24")
|
||||
f := newPrimariesFixture(t, 1, 2)
|
||||
f.advertise(1, prefix)
|
||||
@@ -263,10 +367,10 @@ func TestPrimaries_BothOfflineThenOneReturns(t *testing.T) {
|
||||
f.requirePrimary(prefix, 2)
|
||||
|
||||
f.disconnect(2)
|
||||
f.requireNoPrimary(prefix)
|
||||
|
||||
f.advertise(2, prefix)
|
||||
f.requirePrimary(prefix, 2)
|
||||
|
||||
f.advertise(1, prefix)
|
||||
f.requirePrimary(prefix, 1)
|
||||
}
|
||||
|
||||
func TestPrimaries_HANodes(t *testing.T) {
|
||||
|
||||
@@ -372,6 +372,7 @@ func NewState(cfg *types.Config) (*State, error) {
|
||||
batchTimeout := cmp.Or(cfg.Tuning.NodeStoreBatchTimeout, defaultNodeStoreBatchTimeout)
|
||||
|
||||
nodeStore := NewNodeStore(nodes, policyPeersFunc(polMan), batchSize, batchTimeout)
|
||||
polMan.SetRouterAvailable(nodeStore.IsRouterAvailable)
|
||||
nodeStore.Start()
|
||||
|
||||
s := &State{
|
||||
@@ -1692,7 +1693,7 @@ func (s *State) IsNodeHealthy(id types.NodeID) bool {
|
||||
}
|
||||
|
||||
// SetNodeHealth flips the runtime health bit for one node and reports
|
||||
// whether the resulting primary-route assignment changed, so the HA
|
||||
// whether peers need to recompute global or via routes, so the HA
|
||||
// prober can decide whether to fan out a PolicyChange. true means
|
||||
// healthy; false means unhealthy. An unhealthy mark is dropped when
|
||||
// the node is no longer an HA candidate (offline or no approved
|
||||
@@ -1708,7 +1709,7 @@ func (s *State) SetNodeHealth(id types.NodeID, healthy bool) bool {
|
||||
// BatchSetNodeHealth applies a set of health updates atomically: the
|
||||
// election runs once after every flag has been flipped, so observers
|
||||
// never see an intermediate snapshot. Returns true when the
|
||||
// primary-route assignment differs from before the batch so callers
|
||||
// global primary routes or a via target's health changed, so callers
|
||||
// can gate a single PolicyChange dispatch. Map value true = healthy;
|
||||
// false = unhealthy (gated as in [State.SetNodeHealth]).
|
||||
//
|
||||
@@ -1744,7 +1745,18 @@ func (s *State) BatchSetNodeHealth(updates map[types.NodeID]bool) bool {
|
||||
|
||||
s.nodeStore.UpdateNodes(fns)
|
||||
|
||||
return !maps.Equal(prevRoutes, s.nodeStore.PrimaryRoutes())
|
||||
if !maps.Equal(prevRoutes, s.nodeStore.PrimaryRoutes()) {
|
||||
return true
|
||||
}
|
||||
|
||||
for id := range fns {
|
||||
nv, ok := s.nodeStore.GetNode(id)
|
||||
if ok && nv.Unhealthy() == !updates[id] && s.polMan.NodeIsViaTarget(nv) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// healthSetter returns an UpdateNodeFunc that flips n.Unhealthy to
|
||||
|
||||
@@ -3995,15 +3995,21 @@ func TestHASubnetRouterFailoverBothOffline(t *testing.T) {
|
||||
}, pr.PrimaryRoutes, "r2 should be primary after r1 offline")
|
||||
}, propagationTime, 500*time.Millisecond, "waiting for failover to r2")
|
||||
|
||||
t.Log("=== Step 2: r2 also goes offline. No primary should remain. ===")
|
||||
t.Log("=== Step 2: r2 also goes offline. r2 keeps the route. ===")
|
||||
|
||||
require.NoError(t, subRouter2.Down())
|
||||
|
||||
assert.EventuallyWithT(t, func(c *assert.CollectT) {
|
||||
current, err := headscale.ListNodes()
|
||||
assert.NoError(c, err)
|
||||
assert.False(c, MustFindNode(subRouter2.Hostname(), current).Online, "r2 should be offline")
|
||||
|
||||
pr, err := headscale.PrimaryRoutes()
|
||||
assert.NoError(c, err)
|
||||
assert.Empty(c, pr.PrimaryRoutes,
|
||||
"no primary should be assigned while both routers are offline")
|
||||
assert.Equal(c, map[string]types.NodeID{
|
||||
pref.String(): nodeID2,
|
||||
}, pr.PrimaryRoutes,
|
||||
"the last primary keeps the route while both routers are offline")
|
||||
}, propagationTime, 500*time.Millisecond, "waiting for both routers to be offline")
|
||||
|
||||
t.Log("=== Step 3: r2 returns. ===")
|
||||
|
||||
Reference in new issue
Block a user