stability and race conditions in auth and node store (#2781)

This PR addresses some consistency issues that was introduced or discovered with the nodestore.

nodestore:
Now returns the node that is being put or updated when it is finished. This closes a race condition where when we read it back, we do not necessarily get the node with the given change and it ensures we get all the other updates from that batch write.

auth:
Authentication paths have been unified and simplified. It removes a lot of bad branches and ensures we only do the minimal work.
A comprehensive auth test set has been created so we do not have to run integration tests to validate auth and it has allowed us to generate test cases for all the branches we currently know of.

integration:
added a lot more tooling and checks to validate that nodes reach the expected state when they come up and down. Standardised between the different auth models. A lot of this is to support or detect issues in the changes to nodestore (races) and auth (inconsistencies after login and reaching correct state)

This PR was assisted, particularly tests, by claude code.

integration/route_test.go

@@ -53,16 +53,16 @@ func TestEnablingRoutes(t *testing.T) {
 	err = scenario.CreateHeadscaleEnv(
 		[]tsic.Option{tsic.WithAcceptRoutes()},
 		hsic.WithTestName("clienableroute"))
-	assertNoErrHeadscaleEnv(t, err)
+	requireNoErrHeadscaleEnv(t, err)
 
 	allClients, err := scenario.ListTailscaleClients()
-	assertNoErrListClients(t, err)
+	requireNoErrListClients(t, err)
 
 	err = scenario.WaitForTailscaleSync()
-	assertNoErrSync(t, err)
+	requireNoErrSync(t, err)
 
 	headscale, err := scenario.Headscale()
-	assertNoErrGetHeadscale(t, err)
+	requireNoErrGetHeadscale(t, err)
 
 	expectedRoutes := map[string]string{
 		"1": "10.0.0.0/24",
@@ -83,7 +83,7 @@ func TestEnablingRoutes(t *testing.T) {
 	}
 
 	err = scenario.WaitForTailscaleSync()
-	assertNoErrSync(t, err)
+	requireNoErrSync(t, err)
 
 	var nodes []*v1.Node
 	// Wait for route advertisements to propagate to NodeStore
@@ -256,16 +256,16 @@ func TestHASubnetRouterFailover(t *testing.T) {
 		hsic.WithEmbeddedDERPServerOnly(),
 		hsic.WithTLS(),
 	)
-	assertNoErrHeadscaleEnv(t, err)
+	requireNoErrHeadscaleEnv(t, err)
 
 	allClients, err := scenario.ListTailscaleClients()
-	assertNoErrListClients(t, err)
+	requireNoErrListClients(t, err)
 
 	err = scenario.WaitForTailscaleSync()
-	assertNoErrSync(t, err)
+	requireNoErrSync(t, err)
 
 	headscale, err := scenario.Headscale()
-	assertNoErrGetHeadscale(t, err)
+	requireNoErrGetHeadscale(t, err)
 
 	prefp, err := scenario.SubnetOfNetwork("usernet1")
 	require.NoError(t, err)
@@ -319,7 +319,7 @@ func TestHASubnetRouterFailover(t *testing.T) {
 	}
 
 	err = scenario.WaitForTailscaleSync()
-	assertNoErrSync(t, err)
+	requireNoErrSync(t, err)
 
 	// Wait for route configuration changes after advertising routes
 	var nodes []*v1.Node
@@ -1341,16 +1341,16 @@ func TestSubnetRouteACL(t *testing.T) {
 			},
 		},
 	))
-	assertNoErrHeadscaleEnv(t, err)
+	requireNoErrHeadscaleEnv(t, err)
 
 	allClients, err := scenario.ListTailscaleClients()
-	assertNoErrListClients(t, err)
+	requireNoErrListClients(t, err)
 
 	err = scenario.WaitForTailscaleSync()
-	assertNoErrSync(t, err)
+	requireNoErrSync(t, err)
 
 	headscale, err := scenario.Headscale()
-	assertNoErrGetHeadscale(t, err)
+	requireNoErrGetHeadscale(t, err)
 
 	expectedRoutes := map[string]string{
 		"1": "10.33.0.0/16",
@@ -1393,7 +1393,7 @@ func TestSubnetRouteACL(t *testing.T) {
 	}
 
 	err = scenario.WaitForTailscaleSync()
-	assertNoErrSync(t, err)
+	requireNoErrSync(t, err)
 
 	// Wait for route advertisements to propagate to the server
 	var nodes []*v1.Node
@@ -1572,25 +1572,25 @@ func TestEnablingExitRoutes(t *testing.T) {
 	}
 
 	scenario, err := NewScenario(spec)
-	assertNoErrf(t, "failed to create scenario: %s", err)
+	require.NoErrorf(t, err, "failed to create scenario")
 	defer scenario.ShutdownAssertNoPanics(t)
 
 	err = scenario.CreateHeadscaleEnv([]tsic.Option{
 		tsic.WithExtraLoginArgs([]string{"--advertise-exit-node"}),
 	}, hsic.WithTestName("clienableroute"))
-	assertNoErrHeadscaleEnv(t, err)
+	requireNoErrHeadscaleEnv(t, err)
 
 	allClients, err := scenario.ListTailscaleClients()
-	assertNoErrListClients(t, err)
+	requireNoErrListClients(t, err)
 
 	err = scenario.WaitForTailscaleSync()
-	assertNoErrSync(t, err)
+	requireNoErrSync(t, err)
 
 	headscale, err := scenario.Headscale()
-	assertNoErrGetHeadscale(t, err)
+	requireNoErrGetHeadscale(t, err)
 
 	err = scenario.WaitForTailscaleSync()
-	assertNoErrSync(t, err)
+	requireNoErrSync(t, err)
 
 	nodes, err := headscale.ListNodes()
 	require.NoError(t, err)
@@ -1686,16 +1686,16 @@ func TestSubnetRouterMultiNetwork(t *testing.T) {
 		hsic.WithEmbeddedDERPServerOnly(),
 		hsic.WithTLS(),
 	)
-	assertNoErrHeadscaleEnv(t, err)
+	requireNoErrHeadscaleEnv(t, err)
 
 	allClients, err := scenario.ListTailscaleClients()
-	assertNoErrListClients(t, err)
+	requireNoErrListClients(t, err)
 
 	err = scenario.WaitForTailscaleSync()
-	assertNoErrSync(t, err)
+	requireNoErrSync(t, err)
 
 	headscale, err := scenario.Headscale()
-	assertNoErrGetHeadscale(t, err)
+	requireNoErrGetHeadscale(t, err)
 	assert.NotNil(t, headscale)
 
 	pref, err := scenario.SubnetOfNetwork("usernet1")
@@ -1833,16 +1833,16 @@ func TestSubnetRouterMultiNetworkExitNode(t *testing.T) {
 		hsic.WithEmbeddedDERPServerOnly(),
 		hsic.WithTLS(),
 	)
-	assertNoErrHeadscaleEnv(t, err)
+	requireNoErrHeadscaleEnv(t, err)
 
 	allClients, err := scenario.ListTailscaleClients()
-	assertNoErrListClients(t, err)
+	requireNoErrListClients(t, err)
 
 	err = scenario.WaitForTailscaleSync()
-	assertNoErrSync(t, err)
+	requireNoErrSync(t, err)
 
 	headscale, err := scenario.Headscale()
-	assertNoErrGetHeadscale(t, err)
+	requireNoErrGetHeadscale(t, err)
 	assert.NotNil(t, headscale)
 
 	var user1c, user2c TailscaleClient
@@ -2247,13 +2247,13 @@ func TestAutoApproveMultiNetwork(t *testing.T) {
 					err = scenario.createHeadscaleEnv(tt.withURL, tsOpts,
 						opts...,
 					)
-					assertNoErrHeadscaleEnv(t, err)
+					requireNoErrHeadscaleEnv(t, err)
 
 					allClients, err := scenario.ListTailscaleClients()
-					assertNoErrListClients(t, err)
+					requireNoErrListClients(t, err)
 
 					err = scenario.WaitForTailscaleSync()
-					assertNoErrSync(t, err)
+					requireNoErrSync(t, err)
 
 					services, err := scenario.Services("usernet1")
 					require.NoError(t, err)
@@ -2263,7 +2263,7 @@ func TestAutoApproveMultiNetwork(t *testing.T) {
 					require.NoError(t, err)
 
 					headscale, err := scenario.Headscale()
-					assertNoErrGetHeadscale(t, err)
+					requireNoErrGetHeadscale(t, err)
 					assert.NotNil(t, headscale)
 
 					// Add the Docker network route to the auto-approvers
@@ -2304,21 +2304,21 @@ func TestAutoApproveMultiNetwork(t *testing.T) {
 
 					if tt.withURL {
 						u, err := routerUsernet1.LoginWithURL(headscale.GetEndpoint())
-						assertNoErr(t, err)
+						require.NoError(t, err)
 
 						body, err := doLoginURL(routerUsernet1.Hostname(), u)
-						assertNoErr(t, err)
+						require.NoError(t, err)
 
 						scenario.runHeadscaleRegister("user1", body)
 					} else {
 						userMap, err := headscale.MapUsers()
-						assertNoErr(t, err)
+						require.NoError(t, err)
 
 						pak, err := scenario.CreatePreAuthKey(userMap["user1"].GetId(), false, false)
-						assertNoErr(t, err)
+						require.NoError(t, err)
 
 						err = routerUsernet1.Login(headscale.GetEndpoint(), pak.GetKey())
-						assertNoErr(t, err)
+						require.NoError(t, err)
 					}
 					// extra creation end.
 
@@ -2893,13 +2893,13 @@ func TestSubnetRouteACLFiltering(t *testing.T) {
 		hsic.WithACLPolicy(aclPolicy),
 		hsic.WithPolicyMode(types.PolicyModeDB),
 	)
-	assertNoErrHeadscaleEnv(t, err)
+	requireNoErrHeadscaleEnv(t, err)
 
 	err = scenario.WaitForTailscaleSync()
-	assertNoErrSync(t, err)
+	requireNoErrSync(t, err)
 
 	headscale, err := scenario.Headscale()
-	assertNoErrGetHeadscale(t, err)
+	requireNoErrGetHeadscale(t, err)
 
 	// Get the router and node clients by user
 	routerClients, err := scenario.ListTailscaleClients(routerUser)
@@ -2944,7 +2944,7 @@ func TestSubnetRouteACLFiltering(t *testing.T) {
 	require.NoErrorf(t, err, "failed to advertise routes: %s", err)
 
 	err = scenario.WaitForTailscaleSync()
-	assertNoErrSync(t, err)
+	requireNoErrSync(t, err)
 
 	var routerNode, nodeNode *v1.Node
 	// Wait for route advertisements to propagate to NodeStore