integration: Use Eventually around external calls (#2685)

flake.nix

@@ -143,7 +143,6 @@
           yq-go
           ripgrep
           postgresql
-          traceroute
 
           # 'dot' is needed for pprof graphs
           # go tool pprof -http=: <source>
@@ -160,7 +159,8 @@
 
           # Add hi to make it even easier to use ci runner.
           hi
-        ];
+        ]
+        ++ lib.optional pkgs.stdenv.isLinux [traceroute];
 
       # Add entry to build a docker image with headscale
       # caveat: only works on Linux

integration/auth_key_test.go

@@ -84,8 +84,12 @@ func TestAuthKeyLogoutAndReloginSameUser(t *testing.T) {
 
 			t.Logf("all clients logged out")
 
+			assert.EventuallyWithT(t, func(ct *assert.CollectT) {
+				var err error
 				listNodes, err = headscale.ListNodes()
-			require.Equal(t, nodeCountBeforeLogout, len(listNodes))
+				assert.NoError(ct, err)
+				assert.Equal(ct, nodeCountBeforeLogout, len(listNodes), "Node count should match before logout count")
+			}, 20*time.Second, 1*time.Second)
 
 			for _, node := range listNodes {
 				assertLastSeenSet(t, node)
@@ -115,8 +119,12 @@ func TestAuthKeyLogoutAndReloginSameUser(t *testing.T) {
 				}
 			}
 
+			assert.EventuallyWithT(t, func(ct *assert.CollectT) {
+				var err error
 				listNodes, err = headscale.ListNodes()
-			require.Equal(t, nodeCountBeforeLogout, len(listNodes))
+				assert.NoError(ct, err)
+				assert.Equal(ct, nodeCountBeforeLogout, len(listNodes), "Node count should match after HTTPS reconnection")
+			}, 30*time.Second, 2*time.Second)
 
 			for _, node := range listNodes {
 				assertLastSeenSet(t, node)
@@ -234,22 +242,29 @@ func TestAuthKeyLogoutAndReloginNewUser(t *testing.T) {
 		}
 	}
 
-	user1Nodes, err := headscale.ListNodes("user1")
+	var user1Nodes []*v1.Node
-	assertNoErr(t, err)
+	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
-	assert.Len(t, user1Nodes, len(allClients))
+		var err error
+		user1Nodes, err = headscale.ListNodes("user1")
+		assert.NoError(ct, err)
+		assert.Len(ct, user1Nodes, len(allClients), "User1 should have all clients after re-login")
+	}, 20*time.Second, 1*time.Second)
 
 	// Validate that all the old nodes are still present with user2
-	user2Nodes, err := headscale.ListNodes("user2")
+	var user2Nodes []*v1.Node
-	assertNoErr(t, err)
+	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
-	assert.Len(t, user2Nodes, len(allClients)/2)
+		var err error
+		user2Nodes, err = headscale.ListNodes("user2")
+		assert.NoError(ct, err)
+		assert.Len(ct, user2Nodes, len(allClients)/2, "User2 should have half the clients")
+	}, 20*time.Second, 1*time.Second)
 
 	for _, client := range allClients {
+		assert.EventuallyWithT(t, func(ct *assert.CollectT) {
 			status, err := client.Status()
-		if err != nil {
+			assert.NoError(ct, err, "Failed to get status for client %s", client.Hostname())
-			t.Fatalf("failed to get status for client %s: %s", client.Hostname(), err)
+			assert.Equal(ct, "user1@test.no", status.User[status.Self.UserID].LoginName, "Client %s should be logged in as user1", client.Hostname())
-		}
+		}, 30*time.Second, 2*time.Second)
-
-		assert.Equal(t, "user1@test.no", status.User[status.Self.UserID].LoginName)
 	}
 }
 

integration/auth_web_flow_test.go

@@ -4,11 +4,12 @@ import (
 	"net/netip"
 	"slices"
 	"testing"
+	"time"
 
+	v1 "github.com/juanfont/headscale/gen/go/headscale/v1"
 	"github.com/juanfont/headscale/integration/hsic"
 	"github.com/samber/lo"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 func TestAuthWebFlowAuthenticationPingAll(t *testing.T) {
@@ -92,8 +93,13 @@ func TestAuthWebFlowLogoutAndRelogin(t *testing.T) {
 	headscale, err := scenario.Headscale()
 	assertNoErrGetHeadscale(t, err)
 
-	listNodes, err := headscale.ListNodes()
+	var listNodes []*v1.Node
-	assert.Len(t, allClients, len(listNodes))
+	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
+		var err error
+		listNodes, err = headscale.ListNodes()
+		assert.NoError(ct, err)
+		assert.Len(ct, listNodes, len(allClients), "Node count should match client count after login")
+	}, 20*time.Second, 1*time.Second)
 	nodeCountBeforeLogout := len(listNodes)
 	t.Logf("node count before logout: %d", nodeCountBeforeLogout)
 
@@ -137,8 +143,12 @@ func TestAuthWebFlowLogoutAndRelogin(t *testing.T) {
 	success = pingAllHelper(t, allClients, allAddrs)
 	t.Logf("%d successful pings out of %d", success, len(allClients)*len(allIps))
 
+	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
+		var err error
 		listNodes, err = headscale.ListNodes()
-	require.Len(t, listNodes, nodeCountBeforeLogout)
+		assert.NoError(ct, err)
+		assert.Len(ct, listNodes, nodeCountBeforeLogout, "Node count should match before logout count after re-login")
+	}, 20*time.Second, 1*time.Second)
 	t.Logf("node count first login: %d, after relogin: %d", nodeCountBeforeLogout, len(listNodes))
 
 	for _, client := range allClients {

integration/cli_test.go

@@ -64,7 +64,9 @@ func TestUserCommand(t *testing.T) {
 	assertNoErr(t, err)
 
 	var listUsers []*v1.User
-	err = executeAndUnmarshal(headscale,
+	var result []string
+	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
+		err := executeAndUnmarshal(headscale,
 			[]string{
 				"headscale",
 				"users",
@@ -74,16 +76,18 @@ func TestUserCommand(t *testing.T) {
 			},
 			&listUsers,
 		)
-	assertNoErr(t, err)
+		assert.NoError(ct, err)
 
 		slices.SortFunc(listUsers, sortWithID)
-	result := []string{listUsers[0].GetName(), listUsers[1].GetName()}
+		result = []string{listUsers[0].GetName(), listUsers[1].GetName()}
 
 		assert.Equal(
-		t,
+			ct,
 			[]string{"user1", "user2"},
 			result,
+			"Should have user1 and user2 in users list",
 		)
+	}, 20*time.Second, 1*time.Second)
 
 	_, err = headscale.Execute(
 		[]string{
@@ -98,7 +102,8 @@ func TestUserCommand(t *testing.T) {
 	assertNoErr(t, err)
 
 	var listAfterRenameUsers []*v1.User
-	err = executeAndUnmarshal(headscale,
+	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
+		err := executeAndUnmarshal(headscale,
 			[]string{
 				"headscale",
 				"users",
@@ -108,16 +113,18 @@ func TestUserCommand(t *testing.T) {
 			},
 			&listAfterRenameUsers,
 		)
-	assertNoErr(t, err)
+		assert.NoError(ct, err)
 
-	slices.SortFunc(listUsers, sortWithID)
+		slices.SortFunc(listAfterRenameUsers, sortWithID)
 		result = []string{listAfterRenameUsers[0].GetName(), listAfterRenameUsers[1].GetName()}
 
 		assert.Equal(
-		t,
+			ct,
 			[]string{"user1", "newname"},
 			result,
+			"Should have user1 and newname after rename operation",
 		)
+	}, 20*time.Second, 1*time.Second)
 
 	var listByUsername []*v1.User
 	err = executeAndUnmarshal(headscale,
@@ -187,7 +194,8 @@ func TestUserCommand(t *testing.T) {
 	assert.Contains(t, deleteResult, "User destroyed")
 
 	var listAfterIDDelete []*v1.User
-	err = executeAndUnmarshal(headscale,
+	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
+		err := executeAndUnmarshal(headscale,
 			[]string{
 				"headscale",
 				"users",
@@ -197,10 +205,10 @@ func TestUserCommand(t *testing.T) {
 			},
 			&listAfterIDDelete,
 		)
-	assertNoErr(t, err)
+		assert.NoError(ct, err)
 
 		slices.SortFunc(listAfterIDDelete, sortWithID)
-	want = []*v1.User{
+		want := []*v1.User{
 			{
 				Id:    2,
 				Name:  "newname",
@@ -209,8 +217,9 @@ func TestUserCommand(t *testing.T) {
 		}
 
 		if diff := tcmp.Diff(want, listAfterIDDelete, cmpopts.IgnoreUnexported(v1.User{}), cmpopts.IgnoreFields(v1.User{}, "CreatedAt")); diff != "" {
-		t.Errorf("unexpected users (-want +got):\n%s", diff)
+			assert.Fail(ct, "unexpected users", "diff (-want +got):\n%s", diff)
 		}
+	}, 20*time.Second, 1*time.Second)
 
 	deleteResult, err = headscale.Execute(
 		[]string{
@@ -569,10 +578,14 @@ func TestPreAuthKeyCorrectUserLoggedInCommand(t *testing.T) {
 	)
 	assertNoErr(t, err)
 
-	listNodes, err := headscale.ListNodes()
+	var listNodes []*v1.Node
-	require.NoError(t, err)
+	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
-	require.Len(t, listNodes, 1)
+		var err error
-	assert.Equal(t, user1, listNodes[0].GetUser().GetName())
+		listNodes, err = headscale.ListNodes()
+		assert.NoError(ct, err)
+		assert.Len(ct, listNodes, 1, "Should have exactly 1 node for user1")
+		assert.Equal(ct, user1, listNodes[0].GetUser().GetName(), "Node should belong to user1")
+	}, 15*time.Second, 1*time.Second)
 
 	allClients, err := scenario.ListTailscaleClients()
 	assertNoErrListClients(t, err)
@@ -588,30 +601,31 @@ func TestPreAuthKeyCorrectUserLoggedInCommand(t *testing.T) {
 	err = scenario.WaitForTailscaleLogout()
 	assertNoErr(t, err)
 
+	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
 		status, err := client.Status()
-	assertNoErr(t, err)
+		assert.NoError(ct, err)
-	if status.BackendState == "Starting" || status.BackendState == "Running" {
+		assert.NotContains(ct, []string{"Starting", "Running"}, status.BackendState, 
-		t.Fatalf("expected node to be logged out, backend state: %s", status.BackendState)
+			"Expected node to be logged out, backend state: %s", status.BackendState)
-	}
+	}, 30*time.Second, 2*time.Second)
 
 	err = client.Login(headscale.GetEndpoint(), user2Key.GetKey())
 	assertNoErr(t, err)
 
-	status, err = client.Status()
+	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
-	assertNoErr(t, err)
+		status, err := client.Status()
-	if status.BackendState != "Running" {
+		assert.NoError(ct, err)
-		t.Fatalf("expected node to be logged in, backend state: %s", status.BackendState)
+		assert.Equal(ct, "Running", status.BackendState, "Expected node to be logged in, backend state: %s", status.BackendState)
-	}
+		assert.Equal(ct, "userid:2", status.Self.UserID.String(), "Expected node to be logged in as userid:2")
-
+	}, 30*time.Second, 2*time.Second)
-	if status.Self.UserID.String() != "userid:2" {
-		t.Fatalf("expected node to be logged in as userid:2, got: %s", status.Self.UserID.String())
-	}
 
+	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
+		var err error
 		listNodes, err = headscale.ListNodes()
-	require.NoError(t, err)
+		assert.NoError(ct, err)
-	require.Len(t, listNodes, 2)
+		assert.Len(ct, listNodes, 2, "Should have 2 nodes after re-login")
-	assert.Equal(t, user1, listNodes[0].GetUser().GetName())
+		assert.Equal(ct, user1, listNodes[0].GetUser().GetName(), "First node should belong to user1")
-	assert.Equal(t, user2, listNodes[1].GetUser().GetName())
+		assert.Equal(ct, user2, listNodes[1].GetUser().GetName(), "Second node should belong to user2")
+	}, 20*time.Second, 1*time.Second)
 }
 
 func TestApiKeyCommand(t *testing.T) {
@@ -844,7 +858,9 @@ func TestNodeTagCommand(t *testing.T) {
 
 		nodes[index] = &node
 	}
-	assert.Len(t, nodes, len(regIDs))
+	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
+		assert.Len(ct, nodes, len(regIDs), "Should have correct number of nodes after CLI operations")
+	}, 15*time.Second, 1*time.Second)
 
 	var node v1.Node
 	err = executeAndUnmarshal(
@@ -1096,11 +1112,14 @@ func TestNodeCommand(t *testing.T) {
 		nodes[index] = &node
 	}
 
-	assert.Len(t, nodes, len(regIDs))
+	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
+		assert.Len(ct, nodes, len(regIDs), "Should have correct number of nodes after CLI operations")
+	}, 15*time.Second, 1*time.Second)
 
 	// Test list all nodes after added seconds
 	var listAll []v1.Node
-	err = executeAndUnmarshal(
+	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
+		err := executeAndUnmarshal(
 			headscale,
 			[]string{
 				"headscale",
@@ -1111,9 +1130,9 @@ func TestNodeCommand(t *testing.T) {
 			},
 			&listAll,
 		)
-	assert.NoError(t, err)
+		assert.NoError(ct, err)
-
+		assert.Len(ct, listAll, len(regIDs), "Should list all nodes after CLI operations")
-	assert.Len(t, listAll, 5)
+	}, 20*time.Second, 1*time.Second)
 
 	assert.Equal(t, uint64(1), listAll[0].GetId())
 	assert.Equal(t, uint64(2), listAll[1].GetId())
@@ -1173,7 +1192,9 @@ func TestNodeCommand(t *testing.T) {
 		otherUserMachines[index] = &node
 	}
 
-	assert.Len(t, otherUserMachines, len(otherUserRegIDs))
+	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
+		assert.Len(ct, otherUserMachines, len(otherUserRegIDs), "Should have correct number of otherUser machines after CLI operations")
+	}, 15*time.Second, 1*time.Second)
 
 	// Test list all nodes after added otherUser
 	var listAllWithotherUser []v1.Node
@@ -1250,7 +1271,8 @@ func TestNodeCommand(t *testing.T) {
 
 	// Test: list main user after node is deleted
 	var listOnlyMachineUserAfterDelete []v1.Node
-	err = executeAndUnmarshal(
+	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
+		err := executeAndUnmarshal(
 			headscale,
 			[]string{
 				"headscale",
@@ -1263,9 +1285,9 @@ func TestNodeCommand(t *testing.T) {
 			},
 			&listOnlyMachineUserAfterDelete,
 		)
-	assert.NoError(t, err)
+		assert.NoError(ct, err)
-
+		assert.Len(ct, listOnlyMachineUserAfterDelete, 4, "Should have 4 nodes for node-user after deletion")
-	assert.Len(t, listOnlyMachineUserAfterDelete, 4)
+	}, 20*time.Second, 1*time.Second)
 }
 
 func TestNodeExpireCommand(t *testing.T) {

integration/dns_test.go

@@ -50,34 +50,21 @@ func TestResolveMagicDNS(t *testing.T) {
 
 			assert.Equal(t, peer.Hostname()+".headscale.net.", peerFQDN)
 
+			assert.EventuallyWithT(t, func(ct *assert.CollectT) {
 				command := []string{
 					"tailscale",
 					"ip", peerFQDN,
 				}
 				result, _, err := client.Execute(command)
-			if err != nil {
+				assert.NoError(ct, err, "Failed to execute resolve/ip command %s from %s", peerFQDN, client.Hostname())
-				t.Fatalf(
-					"failed to execute resolve/ip command %s from %s: %s",
-					peerFQDN,
-					client.Hostname(),
-					err,
-				)
-			}
 
 				ips, err := peer.IPs()
-			if err != nil {
+				assert.NoError(ct, err, "Failed to get IPs for %s", peer.Hostname())
-				t.Fatalf(
-					"failed to get ips for %s: %s",
-					peer.Hostname(),
-					err,
-				)
-			}
 
 				for _, ip := range ips {
-				if !strings.Contains(result, ip.String()) {
+					assert.Contains(ct, result, ip.String(), "IP %s should be found in DNS resolution result from %s to %s", ip.String(), client.Hostname(), peer.Hostname())
-					t.Fatalf("ip %s is not found in \n%s\n", ip.String(), result)
-				}
 				}
+			}, 30*time.Second, 2*time.Second)
 		}
 	}
 }

integration/embedded_derp_test.go

@@ -1,12 +1,12 @@
 package integration
 
 import (
-	"strings"
 	"testing"
 	"time"
 
 	"github.com/juanfont/headscale/integration/hsic"
 	"github.com/juanfont/headscale/integration/tsic"
+	"github.com/stretchr/testify/assert"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
 )
@@ -140,17 +140,17 @@ func derpServerScenario(
 	assertNoErrListFQDN(t, err)
 
 	for _, client := range allClients {
+		assert.EventuallyWithT(t, func(ct *assert.CollectT) {
 			status, err := client.Status()
-		assertNoErr(t, err)
+			assert.NoError(ct, err, "Failed to get status for client %s", client.Hostname())
 
 			for _, health := range status.Health {
-			if strings.Contains(health, "could not connect to any relay server") {
+				assert.NotContains(ct, health, "could not connect to any relay server", 
-				t.Errorf("expected to be connected to derp, found: %s", health)
+					"Client %s should be connected to DERP relay", client.Hostname())
-			}
+				assert.NotContains(ct, health, "could not connect to the 'Headscale Embedded DERP' relay server.", 
-			if strings.Contains(health, "could not connect to the 'Headscale Embedded DERP' relay server.") {
+					"Client %s should be connected to Headscale Embedded DERP", client.Hostname())
-				t.Errorf("expected to be connected to derp, found: %s", health)
-			}
 			}
+		}, 30*time.Second, 2*time.Second)
 	}
 
 	success := pingDerpAllHelper(t, allClients, allHostnames)
@@ -161,17 +161,17 @@ func derpServerScenario(
 	}
 
 	for _, client := range allClients {
+		assert.EventuallyWithT(t, func(ct *assert.CollectT) {
 			status, err := client.Status()
-		assertNoErr(t, err)
+			assert.NoError(ct, err, "Failed to get status for client %s", client.Hostname())
 
 			for _, health := range status.Health {
-			if strings.Contains(health, "could not connect to any relay server") {
+				assert.NotContains(ct, health, "could not connect to any relay server", 
-				t.Errorf("expected to be connected to derp, found: %s", health)
+					"Client %s should be connected to DERP relay after first run", client.Hostname())
-			}
+				assert.NotContains(ct, health, "could not connect to the 'Headscale Embedded DERP' relay server.", 
-			if strings.Contains(health, "could not connect to the 'Headscale Embedded DERP' relay server.") {
+					"Client %s should be connected to Headscale Embedded DERP after first run", client.Hostname())
-				t.Errorf("expected to be connected to derp, found: %s", health)
-			}
 			}
+		}, 30*time.Second, 2*time.Second)
 	}
 
 	t.Logf("Run 1: %d successful pings out of %d", success, len(allClients)*len(allHostnames))
@@ -186,17 +186,17 @@ func derpServerScenario(
 	}
 
 	for _, client := range allClients {
+		assert.EventuallyWithT(t, func(ct *assert.CollectT) {
 			status, err := client.Status()
-		assertNoErr(t, err)
+			assert.NoError(ct, err, "Failed to get status for client %s", client.Hostname())
 
 			for _, health := range status.Health {
-			if strings.Contains(health, "could not connect to any relay server") {
+				assert.NotContains(ct, health, "could not connect to any relay server", 
-				t.Errorf("expected to be connected to derp, found: %s", health)
+					"Client %s should be connected to DERP relay after second run", client.Hostname())
-			}
+				assert.NotContains(ct, health, "could not connect to the 'Headscale Embedded DERP' relay server.", 
-			if strings.Contains(health, "could not connect to the 'Headscale Embedded DERP' relay server.") {
+					"Client %s should be connected to Headscale Embedded DERP after second run", client.Hostname())
-				t.Errorf("expected to be connected to derp, found: %s", health)
-			}
 			}
+		}, 30*time.Second, 2*time.Second)
 	}
 
 	t.Logf("Run2: %d successful pings out of %d", success, len(allClients)*len(allHostnames))

integration/general_test.go

@@ -179,9 +179,11 @@ func testEphemeralWithOptions(t *testing.T, opts ...hsic.Option) {
 
 	t.Logf("all clients logged out")
 
+	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
 		nodes, err := headscale.ListNodes()
-	assertNoErr(t, err)
+		assert.NoError(ct, err)
-	require.Len(t, nodes, 0)
+		assert.Len(ct, nodes, 0, "All ephemeral nodes should be cleaned up after logout")
+	}, 30*time.Second, 2*time.Second)
 }
 
 // TestEphemeral2006DeletedTooQuickly verifies that ephemeral nodes are not
@@ -534,7 +536,8 @@ func TestUpdateHostnameFromClient(t *testing.T) {
 	assertNoErrSync(t, err)
 
 	var nodes []*v1.Node
-	err = executeAndUnmarshal(
+	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
+		err := executeAndUnmarshal(
 			headscale,
 			[]string{
 				"headscale",
@@ -545,15 +548,15 @@ func TestUpdateHostnameFromClient(t *testing.T) {
 			},
 			&nodes,
 		)
-
+		assert.NoError(ct, err)
-	assertNoErr(t, err)
+		assert.Len(ct, nodes, 3, "Should have 3 nodes after hostname updates")
-	assert.Len(t, nodes, 3)
 
 		for _, node := range nodes {
 			hostname := hostnames[strconv.FormatUint(node.GetId(), 10)]
-		assert.Equal(t, hostname, node.GetName())
+			assert.Equal(ct, hostname, node.GetName(), "Node name should match hostname")
-		assert.Equal(t, util.ConvertWithFQDNRules(hostname), node.GetGivenName())
+			assert.Equal(ct, util.ConvertWithFQDNRules(hostname), node.GetGivenName(), "Given name should match FQDN rules")
 		}
+	}, 20*time.Second, 1*time.Second)
 
 	// Rename givenName in nodes
 	for _, node := range nodes {
@@ -684,11 +687,13 @@ func TestExpireNode(t *testing.T) {
 	t.Logf("before expire: %d successful pings out of %d", success, len(allClients)*len(allIps))
 
 	for _, client := range allClients {
+		assert.EventuallyWithT(t, func(ct *assert.CollectT) {
 			status, err := client.Status()
-		assertNoErr(t, err)
+			assert.NoError(ct, err)
 
 			// Assert that we have the original count - self
-		assert.Len(t, status.Peers(), spec.NodesPerUser-1)
+			assert.Len(ct, status.Peers(), spec.NodesPerUser-1, "Client %s should see correct number of peers", client.Hostname())
+		}, 30*time.Second, 1*time.Second)
 	}
 
 	headscale, err := scenario.Headscale()
@@ -850,31 +855,34 @@ func TestNodeOnlineStatus(t *testing.T) {
 			return
 		}
 
+		var nodes []*v1.Node
+		assert.EventuallyWithT(t, func(ct *assert.CollectT) {
 			result, err := headscale.Execute([]string{
 				"headscale", "nodes", "list", "--output", "json",
 			})
-		assertNoErr(t, err)
+			assert.NoError(ct, err)
 
-		var nodes []*v1.Node
 			err = json.Unmarshal([]byte(result), &nodes)
-		assertNoErr(t, err)
+			assert.NoError(ct, err)
 
 			// Verify that headscale reports the nodes as online
 			for _, node := range nodes {
 				// All nodes should be online
 				assert.Truef(
-				t,
+					ct,
 					node.GetOnline(),
 					"expected %s to have online status in Headscale, marked as offline %s after start",
 					node.GetName(),
 					time.Since(start),
 				)
 			}
+		}, 15*time.Second, 1*time.Second)
 
 		// Verify that all nodes report all nodes to be online
 		for _, client := range allClients {
+			assert.EventuallyWithT(t, func(ct *assert.CollectT) {
 				status, err := client.Status()
-			assertNoErr(t, err)
+				assert.NoError(ct, err)
 
 				for _, peerKey := range status.Peers() {
 					peerStatus := status.Peer[peerKey]
@@ -889,7 +897,7 @@ func TestNodeOnlineStatus(t *testing.T) {
 					// All peers of this nodes are reporting to be
 					// connected to the control server
 					assert.Truef(
-					t,
+						ct,
 						peerStatus.Online,
 						"expected node %s to be marked as online in %s peer list, marked as offline %s after start",
 						peerStatus.HostName,
@@ -897,6 +905,7 @@ func TestNodeOnlineStatus(t *testing.T) {
 						time.Since(start),
 					)
 				}
+			}, 15*time.Second, 1*time.Second)
 		}
 
 		// Check maximum once per second

integration/utils.go

@@ -21,7 +21,12 @@ import (
 )
 
 const (
+	// derpPingTimeout defines the timeout for individual DERP ping operations
+	// Used in DERP connectivity tests to verify relay server communication
 	derpPingTimeout = 2 * time.Second
+	
+	// derpPingCount defines the number of ping attempts for DERP connectivity tests
+	// Higher count provides better reliability assessment of DERP connectivity
 	derpPingCount = 10
 )
 
@@ -105,6 +110,9 @@ func didClientUseWebsocketForDERP(t *testing.T, client TailscaleClient) bool {
 	return count > 0
 }
 
+// pingAllHelper performs ping tests between all clients and addresses, returning success count.
+// This is used to validate network connectivity in integration tests.
+// Returns the total number of successful ping operations.
 func pingAllHelper(t *testing.T, clients []TailscaleClient, addrs []string, opts ...tsic.PingOption) int {
 	t.Helper()
 	success := 0
@@ -123,6 +131,9 @@ func pingAllHelper(t *testing.T, clients []TailscaleClient, addrs []string, opts
 	return success
 }
 
+// pingDerpAllHelper performs DERP-based ping tests between all clients and addresses.
+// This specifically tests connectivity through DERP relay servers, which is important
+// for validating NAT traversal and relay functionality. Returns success count.
 func pingDerpAllHelper(t *testing.T, clients []TailscaleClient, addrs []string) int {
 	t.Helper()
 	success := 0
@@ -304,9 +315,13 @@ func assertValidNetcheck(t *testing.T, client TailscaleClient) {
 	assert.NotEqualf(t, 0, report.PreferredDERP, "%q does not have a DERP relay", client.Hostname())
 }
 
-// assertCommandOutputContains executes a command for a set time and asserts that the output
+// assertCommandOutputContains executes a command with exponential backoff retry until the output
-// reaches a desired state.
+// contains the expected string or timeout is reached (10 seconds).
-// It should be used instead of sleeping before executing.
+// This implements eventual consistency patterns and should be used instead of time.Sleep 
+// before executing commands that depend on network state propagation.
+//
+// Timeout: 10 seconds with exponential backoff
+// Use cases: DNS resolution, route propagation, policy updates
 func assertCommandOutputContains(t *testing.T, c TailscaleClient, command []string, contains string) {
 	t.Helper()