servertest, integration: regression tests for HA both-offline recovery

Cover the user-reported sequence in #3203: two subnet routers
advertise the same prefix, both go offline, one returns. Three
variants:

  - servertest TestRoutes/ha_secondary_recovers_after_all_offline:
    in-process Disconnect/Reconnect.
  - integration TestHASubnetRouterFailoverBothOffline: tailscale
    down/up between two routers, asserts both server and client
    state restore.
  - integration TestHASubnetRouterFailoverBothOfflineCablePull:
    iptables -j DROP between router and headscale, mimicking a
    real cable pull that breaks the ESTABLISHED long-poll.

All three pass on main — the production failure narrows to a
direct State.Connect/State.Disconnect race that the cable-pull
harness does not reliably hit, addressed in a follow-up commit.
The tests stay as regression coverage for the user's scenario.

Updates #3203

hscontrol/servertest/routes_test.go

@@ -3,10 +3,12 @@ package servertest_test
 import (
 	"context"
 	"net/netip"
+	"slices"
 	"testing"
 	"time"
 
 	"github.com/juanfont/headscale/hscontrol/servertest"
+	"github.com/juanfont/headscale/hscontrol/types"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"tailscale.com/tailcfg"
@@ -211,6 +213,134 @@ func TestRoutes(t *testing.T) {
 			}
 		}
 	})
+
+	// Reproduces https://github.com/juanfont/headscale/issues/3203:
+	// HA tracking loses the secondary subnet router after all routers serving
+	// the route have been offline simultaneously and one of them returns.
+	//
+	// Two assertions split the failure surface:
+	//   R1 — server-side primary route state restores after reconnect.
+	//   R2 — observer's netmap shows the reconnected router online with
+	//        the route in its primary set.
+	// If R1 fails the bug is in state.Connect / primaryRoutes; if R1 passes
+	// and R2 fails the bug is in change broadcast / mapBatcher.
+	//
+	// Caveat: servertest's Reconnect re-registers via TryLogin in addition
+	// to starting a new poll session. Production reconnects after a brief
+	// network outage may bypass re-registration. If this test passes on
+	// main, fall back to the integration variant noted in the plan
+	// (TestHASubnetRouterFailover with all routers offline simultaneously).
+	t.Run("ha_secondary_recovers_after_all_offline", func(t *testing.T) {
+		t.Parallel()
+
+		srv := servertest.NewServer(t)
+		user := srv.CreateUser(t, "ha3203-user")
+
+		route := netip.MustParsePrefix("10.0.0.0/24")
+
+		r1 := servertest.NewClient(t, srv, "ha3203-router1",
+			servertest.WithUser(user))
+		r2 := servertest.NewClient(t, srv, "ha3203-router2",
+			servertest.WithUser(user))
+		obs := servertest.NewClient(t, srv, "ha3203-observer",
+			servertest.WithUser(user))
+
+		obs.WaitForPeers(t, 2, 10*time.Second)
+
+		// Both routers advertise the same route via their hostinfo.
+		advertise := func(c *servertest.TestClient, name string) {
+			t.Helper()
+			c.Direct().SetHostinfo(&tailcfg.Hostinfo{
+				BackendLogID: "servertest-" + name,
+				Hostname:     name,
+				RoutableIPs:  []netip.Prefix{route},
+			})
+
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			defer cancel()
+
+			_ = c.Direct().SendUpdate(ctx)
+		}
+		advertise(r1, "ha3203-router1")
+		advertise(r2, "ha3203-router2")
+
+		// Approve the route on both routers explicitly. Auto-approvers
+		// would also work but introduce a policy dependency the harness
+		// does not currently set up here.
+		approve := func(name string) {
+			t.Helper()
+			id := findNodeID(t, srv, name)
+
+			_, ch, err := srv.State().SetApprovedRoutes(id, []netip.Prefix{route})
+			require.NoError(t, err)
+			srv.App.Change(ch)
+		}
+		approve("ha3203-router1")
+		approve("ha3203-router2")
+
+		// Sanity: r1 starts as primary (lower NodeID by registration order).
+		r1ID := findNodeID(t, srv, "ha3203-router1")
+		r2ID := findNodeID(t, srv, "ha3203-router2")
+
+		hasRoute := func(id types.NodeID) bool {
+			return slices.Contains(srv.State().GetNodePrimaryRoutes(id), route)
+		}
+
+		assert.Eventually(t, func() bool { return hasRoute(r1ID) },
+			10*time.Second, 100*time.Millisecond,
+			"r1 should be primary initially")
+
+		// 1. Take r1 offline. After the 10s grace period, r2 should take over.
+		r1.Disconnect(t)
+		assert.Eventually(t, func() bool { return hasRoute(r2ID) && !hasRoute(r1ID) },
+			20*time.Second, 200*time.Millisecond,
+			"r2 should take over as primary after r1 offline")
+
+		// 2. Take r2 offline. With both routers gone, no primary should remain.
+		r2.Disconnect(t)
+		assert.Eventually(t, func() bool { return !hasRoute(r1ID) && !hasRoute(r2ID) },
+			20*time.Second, 200*time.Millisecond,
+			"no primary should be assigned while both routers are offline")
+
+		// 3. Reconnect r2 (cable plugged back in).
+		r2.Reconnect(t)
+
+		// Hostinfo is part of the controlclient.Direct state; the Reconnect
+		// helper re-registers via TryLogin which carries the same Hostinfo
+		// that was set above. Push it again to be sure the announced route
+		// is registered in the new session.
+		advertise(r2, "ha3203-router2")
+
+		// R1: server-side state must restore r2 as primary.
+		assert.Eventually(t, func() bool { return hasRoute(r2ID) },
+			15*time.Second, 200*time.Millisecond,
+			"R1: r2 should be re-registered as primary after reconnect — issue #3203")
+
+		// R2: observer must see r2 online with the route in its primary set.
+		obs.WaitForCondition(t, "R2: observer sees r2 online with primary route",
+			15*time.Second,
+			func(nm *netmap.NetworkMap) bool {
+				for _, p := range nm.Peers {
+					hi := p.Hostinfo()
+					if !hi.Valid() || hi.Hostname() != "ha3203-router2" {
+						continue
+					}
+
+					online, known := p.Online().GetOk()
+					if !known || !online {
+						return false
+					}
+
+					for i := range p.PrimaryRoutes().Len() {
+						if p.PrimaryRoutes().At(i) == route {
+							return true
+						}
+					}
+				}
+
+				return false
+			})
+	})
 }
 
 // findNodeID is defined in issues_test.go.