Files
headscale/hscontrol/servertest/issues_test.go
T
Kristoffer Dalby 14baf55421 state, oidc: persist user changes and refresh nodes and clients
Write cleared user columns, discard OIDC email the provider no longer
vouches for, refresh NodeStore users, and notify clients from the callback.

Fixes #3471
2026-11-02 19:15:40 +01:00

1273 lines
37 KiB
Go

package servertest_test
import (
"context"
"fmt"
"net/netip"
"slices"
"testing"
"time"
"github.com/juanfont/headscale/hscontrol/servertest"
"github.com/juanfont/headscale/hscontrol/types"
"github.com/juanfont/headscale/hscontrol/types/change"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"tailscale.com/net/tsaddr"
"tailscale.com/tailcfg"
"tailscale.com/types/netmap"
)
// These tests are intentionally strict about expected behavior.
// Failures surface real issues in the control plane.
// TestIssuesMapContent tests issues with [tailcfg.MapResponse] content correctness.
func TestIssuesMapContent(t *testing.T) {
t.Parallel()
// After mesh formation, all peers should have a known Online status.
// The Online field is set when [state.State.Connect] sends a NodeOnline [tailcfg.PeerChange]
// patch. The initial [tailcfg.MapResponse] (from auth handler) may have Online=nil
// because [state.State.Connect] hasn't run yet, so we wait for the status to propagate.
t.Run("initial_map_should_include_peer_online_status", func(t *testing.T) {
t.Parallel()
h := servertest.NewHarness(t, 3)
for _, c := range h.Clients() {
c.WaitForCondition(t, "all peers have known Online status",
10*time.Second,
func(nm *netmap.NetworkMap) bool {
if len(nm.Peers) < 2 {
return false
}
for _, peer := range nm.Peers {
if _, known := peer.Online().GetOk(); !known {
return false
}
}
return true
})
}
})
// DiscoPublicKey set by the client should be visible to peers.
t.Run("disco_key_should_propagate_to_peers", func(t *testing.T) {
t.Parallel()
h := servertest.NewHarness(t, 2)
// The DiscoKey is sent in the first [tailcfg.MapRequest] (not the [tailcfg.RegisterRequest]),
// so it may take an extra map update to propagate to peers. Wait for
// the condition rather than checking the initial netmap.
h.Client(0).WaitForCondition(t, "peer has non-zero DiscoKey",
10*time.Second,
func(nm *netmap.NetworkMap) bool {
if len(nm.Peers) < 1 {
return false
}
return !nm.Peers[0].DiscoKey().IsZero()
})
})
// All peers should reference a valid DERP region.
t.Run("peers_have_valid_derp_region", func(t *testing.T) {
t.Parallel()
h := servertest.NewHarness(t, 3)
for _, c := range h.Clients() {
nm := c.Netmap()
require.NotNil(t, nm)
require.NotNil(t, nm.DERPMap)
for _, peer := range nm.Peers {
derpRegion := peer.HomeDERP()
if derpRegion != 0 {
_, regionExists := nm.DERPMap.Regions[derpRegion]
assert.True(t, regionExists,
"client %s: peer %d has HomeDERP=%d which is not in DERPMap",
c.Name, peer.ID(), derpRegion)
}
}
}
})
// Each peer should have a valid user profile in the [netmap.NetworkMap].
t.Run("all_peers_have_user_profiles", func(t *testing.T) {
t.Parallel()
srv := servertest.NewServer(t)
user1 := srv.CreateUser(t, "profile-user1")
user2 := srv.CreateUser(t, "profile-user2")
c1 := servertest.NewClient(t, srv, "profile-node1",
servertest.WithUser(user1))
c2 := servertest.NewClient(t, srv, "profile-node2",
servertest.WithUser(user2))
c1.WaitForPeers(t, 1, 10*time.Second)
c2.WaitForPeers(t, 1, 10*time.Second)
nm := c1.Netmap()
require.NotNil(t, nm)
selfUserID := nm.SelfNode.User()
selfProfile, hasSelf := nm.UserProfiles[selfUserID]
assert.True(t, hasSelf, "should have self user profile")
if hasSelf {
assert.NotEmpty(t, selfProfile.DisplayName(),
"self user profile should have a display name")
}
require.Len(t, nm.Peers, 1)
peerUserID := nm.Peers[0].User()
peerProfile, hasPeer := nm.UserProfiles[peerUserID]
assert.True(t, hasPeer,
"should have peer's user profile (user %d)", peerUserID)
if hasPeer {
assert.NotEmpty(t, peerProfile.DisplayName(),
"peer user profile should have a display name")
}
})
// When a new cross-user peer first becomes visible via a policy change,
// MapResponse should carry that peer's UserProfile alongside it.
// Otherwise the netmap has a peer whose owner is unknown and code
// reading nm.UserProfiles[peer.User()] sees a zero UserProfileView.
t.Run("policy_change_carries_user_profile_for_new_peer", func(t *testing.T) {
t.Parallel()
srv := servertest.NewServer(t)
user1 := srv.CreateUser(t, "pc-user1")
c1 := servertest.NewClient(t, srv, "pc-node1", servertest.WithUser(user1))
c1.WaitForCondition(t, "c1 has its initial netmap",
5*time.Second,
func(nm *netmap.NetworkMap) bool { return nm.SelfNode.Valid() })
// Pre-register node2 directly so c1's first view of it arrives
// via the policy-change broadcast, not via a per-node NodeAdded.
user2 := srv.CreateUser(t, "pc-user2")
node2 := srv.State().CreateRegisteredNodeForTest(user2, "pc-node2")
node2.User = user2
srv.State().PutNodeInStoreForTest(*node2)
srv.App.Change(change.PolicyChange())
c1.WaitForPeers(t, 1, 5*time.Second)
nm := c1.Netmap()
peer := nm.Peers[0]
profile, ok := nm.UserProfiles[peer.User()]
require.True(t, ok,
"peer is visible but nm.UserProfiles[%d] is missing", peer.User())
assert.Equal(t, "pc-user2", profile.LoginName())
})
}
// TestIssuesRoutes tests issues with route propagation.
func TestIssuesRoutes(t *testing.T) {
t.Parallel()
// Approving a route via API without the node announcing it must NOT
// make the route visible in AllowedIPs. Tailscale uses a strict
// advertise-then-approve model: routes are only distributed when the
// node advertises them ([tailcfg.Hostinfo.RoutableIPs]) AND they are approved.
// An approval without advertisement is a dormant pre-approval that
// activates once the node starts advertising.
t.Run("approved_route_without_announcement_not_distributed", func(t *testing.T) {
t.Parallel()
srv := servertest.NewServer(t)
user := srv.CreateUser(t, "noannounce-user")
c1 := servertest.NewClient(t, srv, "noannounce-node1",
servertest.WithUser(user))
c2 := servertest.NewClient(t, srv, "noannounce-node2",
servertest.WithUser(user))
c1.WaitForPeers(t, 1, 10*time.Second)
nodeID := findNodeID(t, srv, "noannounce-node1")
route := netip.MustParsePrefix("10.0.0.0/24")
// The API should accept the approval without error — the route
// is stored but dormant because the node is not advertising it.
_, routeChange, err := srv.State().SetApprovedRoutes(
nodeID, []netip.Prefix{route})
require.NoError(t, err)
srv.App.Change(routeChange)
// Wait for any updates triggered by the route change to propagate,
// then verify the route does NOT appear in AllowedIPs.
timer := time.NewTimer(3 * time.Second)
defer timer.Stop()
<-timer.C
nm := c2.Netmap()
require.NotNil(t, nm)
for _, p := range nm.Peers {
hi := p.Hostinfo()
if hi.Valid() && hi.Hostname() == "noannounce-node1" {
for i := range p.AllowedIPs().Len() {
assert.NotEqual(t, route, p.AllowedIPs().At(i),
"approved-but-not-announced route should not appear in AllowedIPs")
}
}
}
})
// The client derives ExitNodeOption from its own SelfNode.AllowedIPs,
// and SelfNode only changes when a MapResponse carries Node. SaaS
// captures show approved exit routes on the exit node's own SelfNode
// (routes-ea*, routes-b17/b18). Peers are the control: they must see
// the routes, proving the change was dispatched.
// https://github.com/juanfont/headscale/issues/3502
t.Run("exit_route_approval_reaches_self", func(t *testing.T) {
t.Parallel()
srv := servertest.NewServer(t)
user := srv.CreateUser(t, "selfexit-user")
exit := servertest.NewClient(t, srv, "selfexit-node",
servertest.WithUser(user))
obs := servertest.NewClient(t, srv, "selfexit-obs",
servertest.WithUser(user))
exit.WaitForPeers(t, 1, 10*time.Second)
advertiseRoutes(t, exit, tsaddr.ExitRoutes())
_, c, err := srv.State().SetApprovedRoutes(
findNodeID(t, srv, "selfexit-node"), tsaddr.ExitRoutes())
require.NoError(t, err)
srv.App.Change(c)
obs.WaitForCondition(t, "peer offers exit node", 10*time.Second,
peerOffersExit("selfexit-node"))
exit.WaitForCondition(t, "self offers exit node", 10*time.Second,
selfOffersExit)
})
// Advertising an already-approved exit route goes through
// UpdateNodeFromMapRequest instead of SetApprovedRoutes. This is the
// re-advertise workaround from the issue that 0.29.4 broke.
t.Run("exit_route_advertised_after_approval_reaches_self", func(t *testing.T) {
t.Parallel()
srv := servertest.NewServer(t)
user := srv.CreateUser(t, "preexit-user")
exit := servertest.NewClient(t, srv, "preexit-node",
servertest.WithUser(user))
obs := servertest.NewClient(t, srv, "preexit-obs",
servertest.WithUser(user))
exit.WaitForPeers(t, 1, 10*time.Second)
_, c, err := srv.State().SetApprovedRoutes(
findNodeID(t, srv, "preexit-node"), tsaddr.ExitRoutes())
require.NoError(t, err)
srv.App.Change(c)
advertiseRoutes(t, exit, tsaddr.ExitRoutes())
obs.WaitForCondition(t, "peer offers exit node", 10*time.Second,
peerOffersExit("preexit-node"))
exit.WaitForCondition(t, "self offers exit node", 10*time.Second,
selfOffersExit)
})
// An auto-approver approves the route inside the advertising map
// request, a third producer of the same change.
t.Run("exit_route_auto_approved_on_advertise_reaches_self", func(t *testing.T) {
t.Parallel()
srv := servertest.NewServer(t)
user := srv.CreateUser(t, "advexit-user")
_, err := srv.State().SetPolicyInDB(`{
"acls": [{"action": "accept", "src": ["*"], "dst": ["*:*"]}],
"autoApprovers": {"exitNode": ["advexit-user@"]}
}`)
require.NoError(t, err)
changes, err := srv.State().ReloadPolicy()
require.NoError(t, err)
srv.App.Change(changes...)
exit := servertest.NewClient(t, srv, "advexit-node",
servertest.WithUser(user))
obs := servertest.NewClient(t, srv, "advexit-obs",
servertest.WithUser(user))
exit.WaitForPeers(t, 1, 10*time.Second)
advertiseRoutes(t, exit, tsaddr.ExitRoutes())
obs.WaitForCondition(t, "peer offers exit node", 10*time.Second,
peerOffersExit("advexit-node"))
exit.WaitForCondition(t, "self offers exit node", 10*time.Second,
selfOffersExit)
})
// [tailcfg.Hostinfo] route advertisement should be stored on server.
t.Run("hostinfo_route_advertisement_stored_on_server", func(t *testing.T) {
t.Parallel()
srv := servertest.NewServer(t)
user := srv.CreateUser(t, "histore-user")
c1 := servertest.NewClient(t, srv, "histore-node1",
servertest.WithUser(user))
servertest.NewClient(t, srv, "histore-node2",
servertest.WithUser(user))
c1.WaitForPeers(t, 1, 10*time.Second)
route := netip.MustParsePrefix("10.99.0.0/24")
c1.Direct().SetHostinfo(&tailcfg.Hostinfo{
BackendLogID: "servertest-histore-node1",
Hostname: "histore-node1",
RoutableIPs: []netip.Prefix{route},
})
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
require.NoError(t, c1.Direct().SendUpdate(ctx))
nodeID := findNodeID(t, srv, "histore-node1")
nv, ok := srv.State().GetNodeByID(nodeID)
require.True(t, ok, "node should exist in server state")
announced := nv.AnnouncedRoutes()
assert.Contains(t, announced, route,
"server should store the advertised route as announced")
})
// A grant steering autogroup:internet via a tagged exit node must
// offer the exit node to viewers whose only matching rule is the via
// grant. The via rule is compiled onto the via-tagged node, so only
// the exit node's matchers authorise the pair.
t.Run("via_exit_node_offered_to_members", func(t *testing.T) {
t.Parallel()
srv := servertest.NewServer(t)
adminUser := srv.CreateUser(t, "via3408-admin")
memberUser := srv.CreateUser(t, "via3408-member")
changed, err := srv.State().SetPolicy([]byte(`{
"tagOwners": {"tag:exit": ["via3408-admin@"]},
"autoApprovers": {"exitNode": ["tag:exit"]},
"groups": {"group:admins": ["via3408-admin@"]},
"grants": [
{
"src": ["group:admins"],
"dst": ["*"],
"ip": ["*"]
},
{
"src": ["autogroup:member"],
"dst": ["autogroup:internet"],
"via": ["tag:exit"],
"ip": ["*"]
}
]
}`))
require.NoError(t, err)
if changed {
changes, err := srv.State().ReloadPolicy()
require.NoError(t, err)
srv.App.Change(changes...)
}
adminNode := servertest.NewClient(t, srv, "via3408-admin-node",
servertest.WithUser(adminUser))
memberNode := servertest.NewClient(t, srv, "via3408-member-node",
servertest.WithUser(memberUser))
exitNode := servertest.NewClient(t, srv, "via3408-exit",
servertest.WithUser(adminUser),
servertest.WithTags("tag:exit"))
exitNode.Direct().SetHostinfo(&tailcfg.Hostinfo{
BackendLogID: "servertest-via3408-exit",
Hostname: "via3408-exit",
RoutableIPs: tsaddr.ExitRoutes(),
})
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
require.NoError(t, exitNode.Direct().SendUpdate(ctx))
exitID := findNodeID(t, srv, "via3408-exit")
_, routeChange, err := srv.State().SetApprovedRoutes(
exitID, tsaddr.ExitRoutes())
require.NoError(t, err)
srv.App.Change(routeChange)
seesExitNode := func(nm *netmap.NetworkMap) bool {
for _, p := range nm.Peers {
hi := p.Hostinfo()
if !hi.Valid() || hi.Hostname() != "via3408-exit" {
continue
}
var v4, v6 bool
for i := range p.AllowedIPs().Len() {
switch p.AllowedIPs().At(i) {
case netip.MustParsePrefix("0.0.0.0/0"):
v4 = true
case netip.MustParsePrefix("::/0"):
v6 = true
}
}
return v4 && v6
}
return false
}
memberNode.WaitForCondition(t,
"member sees the tag:exit exit node with exit routes in AllowedIPs",
15*time.Second, seesExitNode)
adminNode.WaitForCondition(t,
"admin sees the tag:exit exit node with exit routes in AllowedIPs",
15*time.Second, seesExitNode)
})
}
// TestIssuesIPAllocation tests IP address allocation correctness.
func TestIssuesIPAllocation(t *testing.T) {
t.Parallel()
// Every node should get unique IPs.
t.Run("ip_addresses_are_unique_across_nodes", func(t *testing.T) {
t.Parallel()
srv := servertest.NewServer(t)
user := srv.CreateUser(t, "ipuniq-user")
const n = 10
clients := make([]*servertest.TestClient, n)
for i := range n {
clients[i] = servertest.NewClient(t, srv,
fmt.Sprintf("ipuniq-%d", i),
servertest.WithUser(user))
}
for _, c := range clients {
c.WaitForUpdate(t, 15*time.Second)
}
seen := make(map[netip.Prefix]string)
for _, c := range clients {
nm := c.Netmap()
require.NotNil(t, nm)
require.True(t, nm.SelfNode.Valid())
for i := range nm.SelfNode.Addresses().Len() {
addr := nm.SelfNode.Addresses().At(i)
if other, exists := seen[addr]; exists {
t.Errorf("IP collision: %v assigned to both %s and %s",
addr, other, c.Name)
}
seen[addr] = c.Name
}
}
})
// After reconnect, IP addresses should be stable.
t.Run("reconnect_preserves_ip_addresses", func(t *testing.T) {
t.Parallel()
h := servertest.NewHarness(t, 2)
nm := h.Client(0).Netmap()
require.NotNil(t, nm)
require.True(t, nm.SelfNode.Valid())
addrsBefore := make([]netip.Prefix, 0, nm.SelfNode.Addresses().Len())
for i := range nm.SelfNode.Addresses().Len() {
addrsBefore = append(addrsBefore, nm.SelfNode.Addresses().At(i))
}
require.NotEmpty(t, addrsBefore)
h.Client(0).Disconnect(t)
h.Client(0).Reconnect(t)
h.Client(0).WaitForPeers(t, 1, 15*time.Second)
nmAfter := h.Client(0).Netmap()
require.NotNil(t, nmAfter)
require.True(t, nmAfter.SelfNode.Valid())
addrsAfter := make([]netip.Prefix, 0, nmAfter.SelfNode.Addresses().Len())
for i := range nmAfter.SelfNode.Addresses().Len() {
addrsAfter = append(addrsAfter, nmAfter.SelfNode.Addresses().At(i))
}
assert.Equal(t, addrsBefore, addrsAfter,
"IP addresses should be stable across reconnect")
})
// New peers should have addresses immediately.
t.Run("new_peer_has_addresses_immediately", func(t *testing.T) {
t.Parallel()
srv := servertest.NewServer(t)
user := srv.CreateUser(t, "newaddr-user")
c1 := servertest.NewClient(t, srv, "newaddr-node1",
servertest.WithUser(user))
c1.WaitForUpdate(t, 10*time.Second)
servertest.NewClient(t, srv, "newaddr-node2",
servertest.WithUser(user))
c1.WaitForPeers(t, 1, 10*time.Second)
nm := c1.Netmap()
require.NotNil(t, nm)
require.Len(t, nm.Peers, 1)
assert.Positive(t, nm.Peers[0].Addresses().Len(),
"new peer should have addresses in the first update that includes it")
})
}
// TestIssuesServerMutations tests that server-side mutations propagate correctly.
func TestIssuesServerMutations(t *testing.T) {
t.Parallel()
// Renaming a node via API should propagate to peers.
t.Run("node_rename_propagates_to_peers", func(t *testing.T) {
t.Parallel()
srv := servertest.NewServer(t)
user := srv.CreateUser(t, "rename-user")
c1 := servertest.NewClient(t, srv, "rename-node1",
servertest.WithUser(user))
c2 := servertest.NewClient(t, srv, "rename-node2",
servertest.WithUser(user))
c1.WaitForPeers(t, 1, 10*time.Second)
nodeID := findNodeID(t, srv, "rename-node1")
_, renameChange, err := srv.State().RenameNode(nodeID, "renamed-node1")
require.NoError(t, err)
srv.App.Change(renameChange)
c2.WaitForCondition(t, "renamed peer visible", 10*time.Second,
func(nm *netmap.NetworkMap) bool {
for _, p := range nm.Peers {
if p.Name() == "renamed-node1" {
return true
}
}
return false
})
})
// Deleting a node via API should remove it from all peers.
t.Run("node_delete_removes_from_all_peers", func(t *testing.T) {
t.Parallel()
srv := servertest.NewServer(t)
user := srv.CreateUser(t, "del-user")
c1 := servertest.NewClient(t, srv, "del-node1",
servertest.WithUser(user))
servertest.NewClient(t, srv, "del-node2",
servertest.WithUser(user))
c3 := servertest.NewClient(t, srv, "del-node3",
servertest.WithUser(user))
c1.WaitForPeers(t, 2, 15*time.Second)
nodeID2 := findNodeID(t, srv, "del-node2")
node2View, ok := srv.State().GetNodeByID(nodeID2)
require.True(t, ok)
deleteChange, err := srv.State().DeleteNode(node2View)
require.NoError(t, err)
srv.App.Change(deleteChange...)
c1.WaitForCondition(t, "deleted peer gone", 10*time.Second,
func(nm *netmap.NetworkMap) bool {
for _, p := range nm.Peers {
hi := p.Hostinfo()
if hi.Valid() && hi.Hostname() == "del-node2" {
return false
}
}
return true
})
c3.WaitForCondition(t, "deleted peer gone from c3", 10*time.Second,
func(nm *netmap.NetworkMap) bool {
for _, p := range nm.Peers {
hi := p.Hostinfo()
if hi.Valid() && hi.Hostname() == "del-node2" {
return false
}
}
return true
})
assert.Len(t, c1.Peers(), 1)
assert.Len(t, c3.Peers(), 1)
})
// [tailcfg.Hostinfo] changes should propagate to peers.
t.Run("hostinfo_changes_propagate_to_peers", func(t *testing.T) {
t.Parallel()
srv := servertest.NewServer(t)
user := srv.CreateUser(t, "hichange-user")
c1 := servertest.NewClient(t, srv, "hichange-node1",
servertest.WithUser(user))
c2 := servertest.NewClient(t, srv, "hichange-node2",
servertest.WithUser(user))
c1.WaitForPeers(t, 1, 10*time.Second)
c1.Direct().SetHostinfo(&tailcfg.Hostinfo{
BackendLogID: "servertest-hichange-node1",
Hostname: "hichange-node1",
OS: "TestOS",
})
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = c1.Direct().SendUpdate(ctx)
c2.WaitForCondition(t, "OS change visible", 10*time.Second,
func(nm *netmap.NetworkMap) bool {
for _, p := range nm.Peers {
hi := p.Hostinfo()
if hi.Valid() && hi.Hostname() == "hichange-node1" {
return hi.OS() == "TestOS"
}
}
return false
})
})
}
// TestIssuesNodeStoreConsistency tests [state.NodeStore] + DB consistency.
func TestIssuesNodeStoreConsistency(t *testing.T) {
t.Parallel()
// [state.NodeStore] and DB should agree after mutations.
t.Run("nodestore_db_consistency_after_operations", func(t *testing.T) {
t.Parallel()
srv := servertest.NewServer(t)
user := srv.CreateUser(t, "consist-user")
c1 := servertest.NewClient(t, srv, "consist-node1",
servertest.WithUser(user))
servertest.NewClient(t, srv, "consist-node2",
servertest.WithUser(user))
c1.WaitForPeers(t, 1, 10*time.Second)
nodeID1 := findNodeID(t, srv, "consist-node1")
route := netip.MustParsePrefix("10.50.0.0/24")
_, routeChange, err := srv.State().SetApprovedRoutes(
nodeID1, []netip.Prefix{route})
require.NoError(t, err)
srv.App.Change(routeChange)
nsView, ok := srv.State().GetNodeByID(nodeID1)
require.True(t, ok, "node should be in NodeStore")
dbNode, err := srv.State().DB().GetNodeByID(nodeID1)
require.NoError(t, err, "node should be in database")
nsRoutes := nsView.ApprovedRoutes().AsSlice()
dbRoutes := dbNode.ApprovedRoutes.List()
assert.Equal(t, nsRoutes, dbRoutes,
"NodeStore and DB should agree on approved routes")
})
// After rapid reconnect, [state.NodeStore] should reflect correct state.
t.Run("nodestore_correct_after_rapid_reconnect", func(t *testing.T) {
t.Parallel()
srv := servertest.NewServer(t)
user := srv.CreateUser(t, "nsrecon-user")
c1 := servertest.NewClient(t, srv, "nsrecon-node1",
servertest.WithUser(user))
servertest.NewClient(t, srv, "nsrecon-node2",
servertest.WithUser(user))
c1.WaitForPeers(t, 1, 10*time.Second)
nodeID1 := findNodeID(t, srv, "nsrecon-node1")
for range 5 {
c1.Disconnect(t)
c1.Reconnect(t)
}
c1.WaitForPeers(t, 1, 15*time.Second)
nv, ok := srv.State().GetNodeByID(nodeID1)
require.True(t, ok)
isOnline, known := nv.IsOnline().GetOk()
assert.True(t, known, "NodeStore should know online status after reconnect")
assert.True(t, isOnline, "NodeStore should show node as online after reconnect")
})
}
// TestIssuesGracePeriod tests the disconnect grace period behavior.
func TestIssuesGracePeriod(t *testing.T) {
t.Parallel()
// Offline status should arrive promptly after grace period.
t.Run("offline_status_arrives_within_grace_period_plus_margin", func(t *testing.T) {
t.Parallel()
h := servertest.NewHarness(t, 2)
peerName := h.Client(1).Name
h.Client(0).WaitForCondition(t, "peer online", 15*time.Second,
func(nm *netmap.NetworkMap) bool {
for _, p := range nm.Peers {
hi := p.Hostinfo()
if hi.Valid() && hi.Hostname() == peerName {
isOnline, known := p.Online().GetOk()
return known && isOnline
}
}
return false
})
disconnectTime := time.Now()
h.Client(1).Disconnect(t)
h.Client(0).WaitForCondition(t, "peer offline", 20*time.Second,
func(nm *netmap.NetworkMap) bool {
for _, p := range nm.Peers {
hi := p.Hostinfo()
if hi.Valid() && hi.Hostname() == peerName {
isOnline, known := p.Online().GetOk()
return known && !isOnline
}
}
return false
})
elapsed := time.Since(disconnectTime)
t.Logf("offline status arrived after %v", elapsed)
assert.Greater(t, elapsed, 8*time.Second,
"offline status arrived too quickly -- grace period may not be working")
assert.Less(t, elapsed, 20*time.Second,
"offline status took too long -- propagation delay issue")
})
// Ephemeral nodes should be fully deleted.
t.Run("ephemeral_node_deleted_not_just_offline", func(t *testing.T) {
t.Parallel()
srv := servertest.NewServer(t,
servertest.WithEphemeralTimeout(3*time.Second))
user := srv.CreateUser(t, "eph-del-user")
regular := servertest.NewClient(t, srv, "eph-del-regular",
servertest.WithUser(user))
ephemeral := servertest.NewClient(t, srv, "eph-del-ephemeral",
servertest.WithUser(user), servertest.WithEphemeral())
regular.WaitForPeers(t, 1, 10*time.Second)
_, found := regular.PeerByName("eph-del-ephemeral")
require.True(t, found)
// Ensure the ephemeral node's long-poll session is fully
// established on the server before disconnecting. Without
// this, the [TestClient.Disconnect] may cancel a [controlclient.Direct.PollNetMap] that hasn't
// yet reached serveLongPoll, so no grace period would ever
// start.
ephemeral.WaitForPeers(t, 1, 10*time.Second)
ephemeral.Disconnect(t)
// Grace period (10s) + ephemeral GC timeout (3s) + propagation.
// Use a generous timeout for CI environments under load.
regular.WaitForCondition(t, "ephemeral peer removed", 60*time.Second,
func(nm *netmap.NetworkMap) bool {
for _, p := range nm.Peers {
hi := p.Hostinfo()
if hi.Valid() && hi.Hostname() == "eph-del-ephemeral" {
return false
}
}
return true
})
nodes := srv.State().ListNodes()
for i := range nodes.Len() {
n := nodes.At(i)
assert.NotEqual(t, "eph-del-ephemeral", n.Hostname(),
"ephemeral node should be deleted from server state")
}
})
}
// TestIssuesScale tests behavior under scale and rapid changes.
func TestIssuesScale(t *testing.T) {
t.Parallel()
t.Run("simultaneous_connect_all_see_all", func(t *testing.T) {
t.Parallel()
srv := servertest.NewServer(t)
user := srv.CreateUser(t, "simul-user")
const n = 10
clients := make([]*servertest.TestClient, n)
for i := range n {
clients[i] = servertest.NewClient(t, srv,
fmt.Sprintf("simul-node-%d", i),
servertest.WithUser(user))
}
for _, c := range clients {
c.WaitForPeers(t, n-1, 30*time.Second)
}
servertest.AssertMeshComplete(t, clients)
servertest.AssertSymmetricVisibility(t, clients)
})
// Many rapid additions should all be delivered.
t.Run("rapid_sequential_additions", func(t *testing.T) {
t.Parallel()
srv := servertest.NewServer(t)
user := srv.CreateUser(t, "rapid-user")
c1 := servertest.NewClient(t, srv, "rapid-node1",
servertest.WithUser(user))
c1.WaitForUpdate(t, 10*time.Second)
for i := range 5 {
servertest.NewClient(t, srv,
fmt.Sprintf("rapid-node-%d", i+2),
servertest.WithUser(user))
}
c1.WaitForPeers(t, 5, 30*time.Second)
assert.Len(t, c1.Peers(), 5)
})
// Reconnect should give a complete map.
t.Run("reconnect_gets_complete_map", func(t *testing.T) {
t.Parallel()
h := servertest.NewHarness(t, 3)
h.Client(0).Disconnect(t)
h.Client(0).Reconnect(t)
h.Client(0).WaitForPeers(t, 2, 15*time.Second)
nm := h.Client(0).Netmap()
require.NotNil(t, nm)
assert.Len(t, nm.Peers, 2)
assert.True(t, nm.SelfNode.Valid())
assert.Positive(t, nm.SelfNode.Addresses().Len())
})
}
// TestIssuesIdentity tests node identity and naming behavior.
func TestIssuesIdentity(t *testing.T) {
t.Parallel()
// Cross-user visibility with default policy.
t.Run("cross_user_visibility_default_policy", func(t *testing.T) {
t.Parallel()
srv := servertest.NewServer(t)
user1 := srv.CreateUser(t, "xuser1")
user2 := srv.CreateUser(t, "xuser2")
c1 := servertest.NewClient(t, srv, "xuser-node1",
servertest.WithUser(user1))
c2 := servertest.NewClient(t, srv, "xuser-node2",
servertest.WithUser(user2))
c1.WaitForPeers(t, 1, 10*time.Second)
c2.WaitForPeers(t, 1, 10*time.Second)
_, found := c1.PeerByName("xuser-node2")
assert.True(t, found, "user1's node should see user2's node")
_, found = c2.PeerByName("xuser-node1")
assert.True(t, found, "user2's node should see user1's node")
})
// Multiple nodes same user should be distinct.
t.Run("multiple_nodes_same_user_distinct", func(t *testing.T) {
t.Parallel()
srv := servertest.NewServer(t)
user := srv.CreateUser(t, "sameuser")
c1 := servertest.NewClient(t, srv, "sameuser-node1",
servertest.WithUser(user))
c2 := servertest.NewClient(t, srv, "sameuser-node2",
servertest.WithUser(user))
c3 := servertest.NewClient(t, srv, "sameuser-node3",
servertest.WithUser(user))
c1.WaitForPeers(t, 2, 15*time.Second)
c2.WaitForPeers(t, 2, 15*time.Second)
c3.WaitForPeers(t, 2, 15*time.Second)
nm1 := c1.Netmap()
nm2 := c2.Netmap()
nm3 := c3.Netmap()
require.NotNil(t, nm1)
require.NotNil(t, nm2)
require.NotNil(t, nm3)
ids := map[tailcfg.NodeID]string{
nm1.SelfNode.ID(): c1.Name,
nm2.SelfNode.ID(): c2.Name,
nm3.SelfNode.ID(): c3.Name,
}
assert.Len(t, ids, 3,
"three nodes with same user should have distinct node IDs")
})
// Same hostname should get unique GivenNames.
t.Run("same_hostname_gets_unique_given_names", func(t *testing.T) {
t.Parallel()
srv := servertest.NewServer(t)
user := srv.CreateUser(t, "samename-user")
c1 := servertest.NewClient(t, srv, "samename",
servertest.WithUser(user))
c2 := servertest.NewClient(t, srv, "samename",
servertest.WithUser(user))
c1.WaitForPeers(t, 1, 10*time.Second)
c2.WaitForPeers(t, 1, 10*time.Second)
nm1 := c1.Netmap()
nm2 := c2.Netmap()
require.NotNil(t, nm1)
require.NotNil(t, nm2)
require.True(t, nm1.SelfNode.Valid())
require.True(t, nm2.SelfNode.Valid())
name1 := nm1.SelfNode.Name()
name2 := nm2.SelfNode.Name()
assert.NotEqual(t, name1, name2,
"nodes with same hostname should get distinct Name (GivenName): %q vs %q",
name1, name2)
})
// Policy change during connect should still converge.
t.Run("policy_change_during_connect", func(t *testing.T) {
t.Parallel()
srv := servertest.NewServer(t)
user := srv.CreateUser(t, "polcon-user")
c1 := servertest.NewClient(t, srv, "polcon-node1",
servertest.WithUser(user))
c1.WaitForUpdate(t, 10*time.Second)
changed, err := srv.State().SetPolicy([]byte(`{
"acls": [
{"action": "accept", "src": ["*"], "dst": ["*:*"]}
]
}`))
require.NoError(t, err)
if changed {
changes, err := srv.State().ReloadPolicy()
require.NoError(t, err)
srv.App.Change(changes...)
}
c2 := servertest.NewClient(t, srv, "polcon-node2",
servertest.WithUser(user))
c1.WaitForPeers(t, 1, 15*time.Second)
c2.WaitForPeers(t, 1, 15*time.Second)
for _, c := range []*servertest.TestClient{c1, c2} {
nm := c.Netmap()
require.NotNil(t, nm)
assert.NotNil(t, nm.PacketFilter,
"client %s should have packet filter after policy change", c.Name)
}
})
}
// TestPeerRemovedAsDelta checks a peer leaving a client's view reaches it
// in a response it can apply as a delta. Bundled with DNSConfig or
// SSHPolicy, the removal forces a full netmap rebuild, which never tells IPN
// bus watchers opted out of full netmaps (the Android app since 1.100) that
// the peer is gone, so they keep showing it.
//
// TODO(kradalby): with the tailscale/tailscale#15660 compat gone, removals
// ride full rebuilds; assert they reach the netmap instead.
func TestPeerRemovedAsDelta(t *testing.T) {
t.Parallel()
// MagicDNS puts DNSConfig in policy responses, making them full rebuilds.
setup := func(t *testing.T) (*servertest.TestServer, *servertest.TestClient, types.NodeID) {
t.Helper()
srv := servertest.NewServer(t, servertest.WithMagicDNS("delta.example.com"))
user1 := srv.CreateUser(t, "delta-user1")
user2 := srv.CreateUser(t, "delta-user2")
c1 := servertest.NewClient(t, srv, "delta-node1",
servertest.WithUser(user1), servertest.WithDeltaUpdates())
servertest.NewClient(t, srv, "delta-node2", servertest.WithUser(user2))
c1.WaitForPeers(t, 1, 15*time.Second)
return srv, c1, findNodeID(t, srv, "delta-node2")
}
assertRemovedAsDelta := func(t *testing.T, c1 *servertest.TestClient, id types.NodeID) {
t.Helper()
assert.EventuallyWithT(t, func(c *assert.CollectT) {
assert.Contains(c, c1.DeltaRemovedPeers(), id.NodeID())
}, 10*time.Second, 50*time.Millisecond, "peer removal never applied as a delta")
}
t.Run("node_deleted", func(t *testing.T) {
t.Parallel()
srv, c1, nodeID2 := setup(t)
node2, ok := srv.State().GetNodeByID(nodeID2)
require.True(t, ok)
changes, err := srv.State().DeleteNode(node2)
require.NoError(t, err)
srv.App.Change(changes...)
assertRemovedAsDelta(t, c1, nodeID2)
})
// The new stream's initial map is a full rebuild, so a removal missed
// while disconnected must still follow as a delta.
t.Run("deleted_while_disconnected", func(t *testing.T) {
t.Parallel()
srv, c1, nodeID2 := setup(t)
node2, ok := srv.State().GetNodeByID(nodeID2)
require.True(t, ok)
c1.Disconnect(t)
changes, err := srv.State().DeleteNode(node2)
require.NoError(t, err)
srv.App.Change(changes...)
c1.Reconnect(t)
assertRemovedAsDelta(t, c1, nodeID2)
})
t.Run("hidden_by_policy", func(t *testing.T) {
t.Parallel()
srv, c1, nodeID2 := setup(t)
changed, err := srv.State().SetPolicy([]byte(`{
"acls": [
{"action": "accept", "src": ["delta-user1@"], "dst": ["delta-user1@:*"]},
{"action": "accept", "src": ["delta-user2@"], "dst": ["delta-user2@:*"]}
]
}`))
require.NoError(t, err)
require.True(t, changed)
changes, err := srv.State().ReloadPolicy()
require.NoError(t, err)
srv.App.Change(changes...)
assertRemovedAsDelta(t, c1, nodeID2)
})
}
// TestDeletedNodeNotAnnouncedToHiddenPeers deletes a node another user's node
// cannot see under the policy: only the deleted node's peers are told it is
// gone.
func TestDeletedNodeNotAnnouncedToHiddenPeers(t *testing.T) {
t.Parallel()
srv := servertest.NewServer(t)
_, err := srv.State().SetPolicy([]byte(`{
"acls": [
{"action": "accept", "src": ["alice@"], "dst": ["alice@:*"]},
{"action": "accept", "src": ["bob@"], "dst": ["bob@:*"]}
]
}`))
require.NoError(t, err)
alice := srv.CreateUser(t, "alice")
bob := srv.CreateUser(t, "bob")
laptop := servertest.NewClient(t, srv, "alice-laptop",
servertest.WithUser(alice), servertest.WithDeltaUpdates())
servertest.NewClient(t, srv, "alice-db", servertest.WithUser(alice))
other := servertest.NewClient(t, srv, "bob-node",
servertest.WithUser(bob), servertest.WithDeltaUpdates())
laptop.WaitForPeers(t, 1, 10*time.Second)
other.WaitForPeers(t, 0, 10*time.Second)
db := findNodeID(t, srv, "alice-db")
dbView, ok := srv.State().GetNodeByID(db)
require.True(t, ok)
changes, err := srv.State().DeleteNode(dbView)
require.NoError(t, err)
srv.App.Change(changes...)
assert.EventuallyWithT(t, func(c *assert.CollectT) {
assert.Contains(c, laptop.DeltaRemovedPeers(), db.NodeID())
}, 10*time.Second, 50*time.Millisecond, "a peer of the deleted node must be told it is gone")
// bob-node's stream got the same batch; give it time to deliver.
assert.Never(t, func() bool {
return slices.Contains(other.DeltaRemovedPeers(), db.NodeID())
}, 2*time.Second, 50*time.Millisecond,
"a node the policy hid the deleted node from must not be told of it")
}
func findNodeID(tb testing.TB, srv *servertest.TestServer, hostname string) types.NodeID {
tb.Helper()
nodes := srv.State().ListNodes()
for i := range nodes.Len() {
n := nodes.At(i)
if n.Hostname() == hostname {
return n.ID()
}
}
tb.Fatalf("node %q not found in server state", hostname)
return 0
}
// advertiseRoutes pushes routes in c's Hostinfo. The server stores the
// announcement before it answers.
func advertiseRoutes(tb testing.TB, c *servertest.TestClient, routes []netip.Prefix) {
tb.Helper()
c.Direct().SetHostinfo(&tailcfg.Hostinfo{
BackendLogID: "servertest-" + c.Name,
Hostname: c.Name,
RoutableIPs: routes,
})
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
require.NoError(tb, c.Direct().SendUpdate(ctx))
}
// selfOffersExit mirrors how tailscaled computes Self.ExitNodeOption.
func selfOffersExit(nm *netmap.NetworkMap) bool {
return nm.SelfNode.Valid() && tsaddr.ContainsExitRoutes(nm.SelfNode.AllowedIPs())
}
func peerOffersExit(hostname string) func(*netmap.NetworkMap) bool {
return func(nm *netmap.NetworkMap) bool {
p, ok := peerByHostname(nm, hostname)
return ok && tsaddr.ContainsExitRoutes(p.AllowedIPs())
}
}
func peerByHostname(nm *netmap.NetworkMap, hostname string) (tailcfg.NodeView, bool) {
for _, p := range nm.Peers {
if hi := p.Hostinfo(); hi.Valid() && hi.Hostname() == hostname {
return p, true
}
}
return tailcfg.NodeView{}, false
}
func TestUserProfileFollowsRename(t *testing.T) {
srv := servertest.NewServer(t)
owner := srv.CreateUser(t, "profile-original")
profileID := owner.TailscaleUserProfile().ID
peer := srv.CreateUser(t, "profile-peer")
ownerClient := servertest.NewClient(t, srv, "owner", servertest.WithUser(owner))
peerClient := servertest.NewClient(t, srv, "peer", servertest.WithUser(peer))
ownerClient.WaitForPeers(t, 1, 5*time.Second)
peerClient.WaitForPeers(t, 1, 5*time.Second)
_, c, err := srv.State().RenameUser(types.UserID(owner.ID), "profile-renamed")
require.NoError(t, err)
srv.App.Change(c)
for _, client := range []*servertest.TestClient{ownerClient, peerClient} {
client.WaitForCondition(t, "renamed user profile", 5*time.Second,
func(nm *netmap.NetworkMap) bool {
profile, ok := nm.UserProfiles[profileID]
return ok && profile.LoginName() == "profile-renamed"
})
}
late := servertest.NewClient(t, srv, "late", servertest.WithUser(peer))
late.WaitForCondition(t, "renamed user profile on join", 5*time.Second,
func(nm *netmap.NetworkMap) bool {
profile, ok := nm.UserProfiles[profileID]
return ok && profile.LoginName() == "profile-renamed"
})
}