ci: reuse compiled integration tests and prebuilt images

Updates #3542

(cherry picked from commit c9852c43da)
This commit is contained in:
Kristoffer Dalby committed 2026-10-13 14:35:52 +02:00
1 parent 8106636c7f
commit 52e01807db
13 files changed
+1125 -446

No files matched your search

+250 -279
View File
@@ -1,7 +1,6 @@
package integration
import (
"fmt"
"net/netip"
"slices"
"strconv"
@@ -23,196 +22,182 @@ import (
func TestAuthKeyLogoutAndReloginSameUser(t *testing.T) {
IntegrationSkip(t)
for _, https := range []bool{true, false} {
t.Run(fmt.Sprintf("with-https-%t", https), func(t *testing.T) {
spec := ScenarioSpec{
NodesPerUser: len(MustTestVersions),
Users: []string{"user1", "user2"},
}
scenario, err := NewScenario(spec)
require.NoError(t, err)
defer scenario.ShutdownAssertNoPanics(t)
opts := []hsic.Option{
hsic.WithTestName("authkey-relogsame"),
}
err = scenario.CreateHeadscaleEnv([]tsic.Option{}, opts...)
requireNoErrHeadscaleEnv(t, err)
allClients, err := scenario.ListTailscaleClients()
requireNoErrListClients(t, err)
allIps, err := scenario.ListTailscaleClientsIPs()
requireNoErrListClientIPs(t, err)
err = scenario.WaitForTailscaleSync()
requireNoErrSync(t, err)
headscale, err := scenario.Headscale()
requireNoErrGetHeadscale(t, err)
expectedNodes := collectExpectedNodeIDs(t, allClients)
requireAllClientsOnline(t, headscale, expectedNodes, true, "all clients should be connected", integrationutil.ScaledTimeout(120*time.Second))
// Validate that all nodes have [tailcfg.NetInfo] and DERP servers before logout
requireAllClientsNetInfoAndDERP(t, headscale, expectedNodes, "all clients should have NetInfo and DERP before logout", 3*time.Minute)
// assertClientsState(t, allClients)
clientIPs := make(map[TailscaleClient][]netip.Addr)
for _, client := range allClients {
ips, err := client.IPs()
if err != nil {
t.Fatalf("failed to get IPs for client %s: %s", client.Hostname(), err)
}
clientIPs[client] = ips
}
var (
listNodes []*v1.Node
nodeCountBeforeLogout int
)
assert.EventuallyWithT(t, func(c *assert.CollectT) {
var err error
listNodes, err = headscale.ListNodes()
assert.NoError(c, err)
assert.Len(c, listNodes, len(allClients))
for _, node := range listNodes {
assertLastSeenSetWithCollect(c, node)
}
}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "Waiting for expected node list before logout")
nodeCountBeforeLogout = len(listNodes)
t.Logf("node count before logout: %d", nodeCountBeforeLogout)
for _, client := range allClients {
err := client.Logout()
if err != nil {
t.Fatalf("failed to logout client %s: %s", client.Hostname(), err)
}
}
err = scenario.WaitForTailscaleLogout()
requireNoErrLogout(t, err)
// After taking down all nodes, verify all systems show nodes offline
requireAllClientsOnline(t, headscale, expectedNodes, false, "all nodes should have logged out", integrationutil.ScaledTimeout(120*time.Second))
t.Logf("all clients logged out")
t.Logf("Validating node persistence after logout at %s", time.Now().Format(TimestampFormat))
assert.EventuallyWithT(t, func(ct *assert.CollectT) {
var err error
listNodes, err = headscale.ListNodes()
assert.NoError(ct, err, "Failed to list nodes after logout")
assert.Len(ct, listNodes, nodeCountBeforeLogout, "Node count should match before logout count - expected %d nodes, got %d", nodeCountBeforeLogout, len(listNodes))
}, integrationutil.StatusReadyTimeout, 2*time.Second, "validating node persistence after logout (nodes should remain in database)")
for _, node := range listNodes {
assertLastSeenSet(t, node)
}
// if the server is not running with HTTPS, we have to wait a bit before
// reconnection as the newest Tailscale client has a measure that will only
// reconnect over HTTPS if they saw a noise connection previously.
// https://github.com/tailscale/tailscale/commit/1eaad7d3deb0815e8932e913ca1a862afa34db38
// https://github.com/juanfont/headscale/issues/2164
if !https {
//nolint:forbidigo // Intentional delay: Tailscale client requires 5 min wait before reconnecting over non-HTTPS
time.Sleep(5 * time.Minute)
}
userMap, err := headscale.MapUsers()
require.NoError(t, err)
for _, userName := range spec.Users {
key, err := scenario.CreatePreAuthKey(userMap[userName].GetId(), true, false)
if err != nil {
t.Fatalf("failed to create pre-auth key for user %s: %s", userName, err)
}
err = scenario.RunTailscaleUp(userName, headscale.GetEndpoint(), key.GetKey())
if err != nil {
t.Fatalf("failed to run tailscale up for user %s: %s", userName, err)
}
}
t.Logf("Validating node persistence after relogin at %s", time.Now().Format(TimestampFormat))
assert.EventuallyWithT(t, func(ct *assert.CollectT) {
var err error
listNodes, err = headscale.ListNodes()
assert.NoError(ct, err, "Failed to list nodes after relogin")
assert.Len(ct, listNodes, nodeCountBeforeLogout, "Node count should remain unchanged after relogin - expected %d nodes, got %d", nodeCountBeforeLogout, len(listNodes))
}, integrationutil.HAConvergeTimeout, 2*time.Second, "validating node count stability after same-user auth key relogin")
for _, node := range listNodes {
assertLastSeenSet(t, node)
}
requireAllClientsOnline(t, headscale, expectedNodes, true, "all clients should be connected to batcher", integrationutil.ScaledTimeout(120*time.Second))
// Wait for Tailscale sync before validating [tailcfg.NetInfo] to ensure proper state propagation
err = scenario.WaitForTailscaleSync()
requireNoErrSync(t, err)
// Validate that all nodes have [tailcfg.NetInfo] and DERP servers after reconnection
requireAllClientsNetInfoAndDERP(t, headscale, expectedNodes, "all clients should have NetInfo and DERP after reconnection", 3*time.Minute)
err = scenario.WaitForTailscaleSync()
requireNoErrSync(t, err)
allAddrs := lo.Map(allIps, func(x netip.Addr, index int) string {
return x.String()
})
assertPingAll(t, allClients, allAddrs)
for _, client := range allClients {
ips, err := client.IPs()
if err != nil {
t.Fatalf("failed to get IPs for client %s: %s", client.Hostname(), err)
}
// lets check if the IPs are the same
if len(ips) != len(clientIPs[client]) {
t.Fatalf("IPs changed for client %s", client.Hostname())
}
for _, ip := range ips {
if !slices.Contains(clientIPs[client], ip) {
t.Fatalf(
"IPs changed for client %s. Used to be %v now %v",
client.Hostname(),
clientIPs[client],
ips,
)
}
}
}
assert.EventuallyWithT(t, func(c *assert.CollectT) {
var err error
listNodes, err = headscale.ListNodes()
assert.NoError(c, err)
assert.Len(c, listNodes, nodeCountBeforeLogout)
for _, node := range listNodes {
assertLastSeenSetWithCollect(c, node)
}
}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "Waiting for node list after relogin")
})
spec := ScenarioSpec{
NodesPerUser: len(MustTestVersions),
Users: []string{"user1", "user2"},
}
scenario, err := NewScenario(spec)
require.NoError(t, err)
defer scenario.ShutdownAssertNoPanics(t)
opts := []hsic.Option{
hsic.WithTestName("authkey-relogsame"),
}
err = scenario.CreateHeadscaleEnv([]tsic.Option{}, opts...)
requireNoErrHeadscaleEnv(t, err)
allClients, err := scenario.ListTailscaleClients()
requireNoErrListClients(t, err)
allIps, err := scenario.ListTailscaleClientsIPs()
requireNoErrListClientIPs(t, err)
err = scenario.WaitForTailscaleSync()
requireNoErrSync(t, err)
headscale, err := scenario.Headscale()
requireNoErrGetHeadscale(t, err)
expectedNodes := collectExpectedNodeIDs(t, allClients)
requireAllClientsOnline(t, headscale, expectedNodes, true, "all clients should be connected", integrationutil.ScaledTimeout(120*time.Second))
// Validate that all nodes have [tailcfg.NetInfo] and DERP servers before logout
requireAllClientsNetInfoAndDERP(t, headscale, expectedNodes, "all clients should have NetInfo and DERP before logout", 3*time.Minute)
// assertClientsState(t, allClients)
clientIPs := make(map[TailscaleClient][]netip.Addr)
for _, client := range allClients {
ips, err := client.IPs()
if err != nil {
t.Fatalf("failed to get IPs for client %s: %s", client.Hostname(), err)
}
clientIPs[client] = ips
}
var (
listNodes []*v1.Node
nodeCountBeforeLogout int
)
assert.EventuallyWithT(t, func(c *assert.CollectT) {
var err error
listNodes, err = headscale.ListNodes()
assert.NoError(c, err)
assert.Len(c, listNodes, len(allClients))
for _, node := range listNodes {
assertLastSeenSetWithCollect(c, node)
}
}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "Waiting for expected node list before logout")
nodeCountBeforeLogout = len(listNodes)
t.Logf("node count before logout: %d", nodeCountBeforeLogout)
for _, client := range allClients {
err := client.Logout()
if err != nil {
t.Fatalf("failed to logout client %s: %s", client.Hostname(), err)
}
}
err = scenario.WaitForTailscaleLogout()
requireNoErrLogout(t, err)
// After taking down all nodes, verify all systems show nodes offline
requireAllClientsOnline(t, headscale, expectedNodes, false, "all nodes should have logged out", integrationutil.ScaledTimeout(120*time.Second))
t.Logf("all clients logged out")
t.Logf("Validating node persistence after logout at %s", time.Now().Format(TimestampFormat))
assert.EventuallyWithT(t, func(ct *assert.CollectT) {
var err error
listNodes, err = headscale.ListNodes()
assert.NoError(ct, err, "Failed to list nodes after logout")
assert.Len(ct, listNodes, nodeCountBeforeLogout, "Node count should match before logout count - expected %d nodes, got %d", nodeCountBeforeLogout, len(listNodes))
}, integrationutil.StatusReadyTimeout, 2*time.Second, "validating node persistence after logout (nodes should remain in database)")
for _, node := range listNodes {
assertLastSeenSet(t, node)
}
userMap, err := headscale.MapUsers()
require.NoError(t, err)
for _, userName := range spec.Users {
key, err := scenario.CreatePreAuthKey(userMap[userName].GetId(), true, false)
if err != nil {
t.Fatalf("failed to create pre-auth key for user %s: %s", userName, err)
}
err = scenario.RunTailscaleUp(userName, headscale.GetEndpoint(), key.GetKey())
if err != nil {
t.Fatalf("failed to run tailscale up for user %s: %s", userName, err)
}
}
t.Logf("Validating node persistence after relogin at %s", time.Now().Format(TimestampFormat))
assert.EventuallyWithT(t, func(ct *assert.CollectT) {
var err error
listNodes, err = headscale.ListNodes()
assert.NoError(ct, err, "Failed to list nodes after relogin")
assert.Len(ct, listNodes, nodeCountBeforeLogout, "Node count should remain unchanged after relogin - expected %d nodes, got %d", nodeCountBeforeLogout, len(listNodes))
}, integrationutil.HAConvergeTimeout, 2*time.Second, "validating node count stability after same-user auth key relogin")
for _, node := range listNodes {
assertLastSeenSet(t, node)
}
requireAllClientsOnline(t, headscale, expectedNodes, true, "all clients should be connected to batcher", integrationutil.ScaledTimeout(120*time.Second))
// Wait for Tailscale sync before validating [tailcfg.NetInfo] to ensure proper state propagation
err = scenario.WaitForTailscaleSync()
requireNoErrSync(t, err)
// Validate that all nodes have [tailcfg.NetInfo] and DERP servers after reconnection
requireAllClientsNetInfoAndDERP(t, headscale, expectedNodes, "all clients should have NetInfo and DERP after reconnection", 3*time.Minute)
err = scenario.WaitForTailscaleSync()
requireNoErrSync(t, err)
allAddrs := lo.Map(allIps, func(x netip.Addr, index int) string {
return x.String()
})
assertPingAll(t, allClients, allAddrs)
for _, client := range allClients {
ips, err := client.IPs()
if err != nil {
t.Fatalf("failed to get IPs for client %s: %s", client.Hostname(), err)
}
// lets check if the IPs are the same
if len(ips) != len(clientIPs[client]) {
t.Fatalf("IPs changed for client %s", client.Hostname())
}
for _, ip := range ips {
if !slices.Contains(clientIPs[client], ip) {
t.Fatalf(
"IPs changed for client %s. Used to be %v now %v",
client.Hostname(),
clientIPs[client],
ips,
)
}
}
}
assert.EventuallyWithT(t, func(c *assert.CollectT) {
var err error
listNodes, err = headscale.ListNodes()
assert.NoError(c, err)
assert.Len(c, listNodes, nodeCountBeforeLogout)
for _, node := range listNodes {
assertLastSeenSetWithCollect(c, node)
}
}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "Waiting for node list after relogin")
}
// This test will first log in two sets of nodes to two sets of users, then
@@ -353,120 +338,106 @@ func TestAuthKeyLogoutAndReloginNewUser(t *testing.T) {
func TestAuthKeyLogoutAndReloginSameUserExpiredKey(t *testing.T) {
IntegrationSkip(t)
for _, https := range []bool{true, false} {
t.Run(fmt.Sprintf("with-https-%t", https), func(t *testing.T) {
spec := ScenarioSpec{
NodesPerUser: len(MustTestVersions),
Users: []string{"user1", "user2"},
}
spec := ScenarioSpec{
NodesPerUser: len(MustTestVersions),
Users: []string{"user1", "user2"},
}
scenario, err := NewScenario(spec)
scenario, err := NewScenario(spec)
require.NoError(t, err)
defer scenario.ShutdownAssertNoPanics(t)
require.NoError(t, err)
defer scenario.ShutdownAssertNoPanics(t)
opts := []hsic.Option{
hsic.WithTestName("authkey-rlogexpired"),
}
opts := []hsic.Option{
hsic.WithTestName("authkey-rlogexpired"),
}
err = scenario.CreateHeadscaleEnv([]tsic.Option{}, opts...)
requireNoErrHeadscaleEnv(t, err)
err = scenario.CreateHeadscaleEnv([]tsic.Option{}, opts...)
requireNoErrHeadscaleEnv(t, err)
allClients, err := scenario.ListTailscaleClients()
requireNoErrListClients(t, err)
allClients, err := scenario.ListTailscaleClients()
requireNoErrListClients(t, err)
err = scenario.WaitForTailscaleSync()
requireNoErrSync(t, err)
err = scenario.WaitForTailscaleSync()
requireNoErrSync(t, err)
// assertClientsState(t, allClients)
// assertClientsState(t, allClients)
clientIPs := make(map[TailscaleClient][]netip.Addr)
clientIPs := make(map[TailscaleClient][]netip.Addr)
for _, client := range allClients {
ips, err := client.IPs()
if err != nil {
t.Fatalf("failed to get IPs for client %s: %s", client.Hostname(), err)
}
for _, client := range allClients {
ips, err := client.IPs()
if err != nil {
t.Fatalf("failed to get IPs for client %s: %s", client.Hostname(), err)
}
clientIPs[client] = ips
}
clientIPs[client] = ips
}
headscale, err := scenario.Headscale()
requireNoErrGetHeadscale(t, err)
headscale, err := scenario.Headscale()
requireNoErrGetHeadscale(t, err)
// Collect expected node IDs for validation
expectedNodes := collectExpectedNodeIDs(t, allClients)
// Collect expected node IDs for validation
expectedNodes := collectExpectedNodeIDs(t, allClients)
// Validate initial connection state
requireAllClientsOnline(t, headscale, expectedNodes, true, "all clients should be connected after initial login", integrationutil.ScaledTimeout(120*time.Second))
requireAllClientsNetInfoAndDERP(t, headscale, expectedNodes, "all clients should have NetInfo and DERP after initial login", 3*time.Minute)
// Validate initial connection state
requireAllClientsOnline(t, headscale, expectedNodes, true, "all clients should be connected after initial login", integrationutil.ScaledTimeout(120*time.Second))
requireAllClientsNetInfoAndDERP(t, headscale, expectedNodes, "all clients should have NetInfo and DERP after initial login", 3*time.Minute)
var (
listNodes []*v1.Node
nodeCountBeforeLogout int
)
var (
listNodes []*v1.Node
nodeCountBeforeLogout int
)
assert.EventuallyWithT(t, func(c *assert.CollectT) {
var err error
assert.EventuallyWithT(t, func(c *assert.CollectT) {
var err error
listNodes, err = headscale.ListNodes()
assert.NoError(c, err)
assert.Len(c, listNodes, len(allClients))
}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "Waiting for expected node list before logout")
listNodes, err = headscale.ListNodes()
assert.NoError(c, err)
assert.Len(c, listNodes, len(allClients))
}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "Waiting for expected node list before logout")
nodeCountBeforeLogout = len(listNodes)
t.Logf("node count before logout: %d", nodeCountBeforeLogout)
nodeCountBeforeLogout = len(listNodes)
t.Logf("node count before logout: %d", nodeCountBeforeLogout)
for _, client := range allClients {
err := client.Logout()
if err != nil {
t.Fatalf("failed to logout client %s: %s", client.Hostname(), err)
}
}
for _, client := range allClients {
err := client.Logout()
if err != nil {
t.Fatalf("failed to logout client %s: %s", client.Hostname(), err)
}
}
err = scenario.WaitForTailscaleLogout()
requireNoErrLogout(t, err)
err = scenario.WaitForTailscaleLogout()
requireNoErrLogout(t, err)
// Validate that all nodes are offline after logout
requireAllClientsOnline(t, headscale, expectedNodes, false, "all nodes should be offline after logout", integrationutil.ScaledTimeout(120*time.Second))
// Validate that all nodes are offline after logout
requireAllClientsOnline(t, headscale, expectedNodes, false, "all nodes should be offline after logout", integrationutil.ScaledTimeout(120*time.Second))
t.Logf("all clients logged out")
t.Logf("all clients logged out")
// if the server is not running with HTTPS, we have to wait a bit before
// reconnection as the newest Tailscale client has a measure that will only
// reconnect over HTTPS if they saw a noise connection previously.
// https://github.com/tailscale/tailscale/commit/1eaad7d3deb0815e8932e913ca1a862afa34db38
// https://github.com/juanfont/headscale/issues/2164
if !https {
//nolint:forbidigo // Intentional delay: Tailscale client requires 5 min wait before reconnecting over non-HTTPS
time.Sleep(5 * time.Minute)
}
userMap, err := headscale.MapUsers()
require.NoError(t, err)
userMap, err := headscale.MapUsers()
require.NoError(t, err)
for _, userName := range spec.Users {
key, err := scenario.CreatePreAuthKey(userMap[userName].GetId(), true, false)
if err != nil {
t.Fatalf("failed to create pre-auth key for user %s: %s", userName, err)
}
for _, userName := range spec.Users {
key, err := scenario.CreatePreAuthKey(userMap[userName].GetId(), true, false)
if err != nil {
t.Fatalf("failed to create pre-auth key for user %s: %s", userName, err)
}
// Expire the key so it can't be used
_, err = headscale.Execute(
[]string{
"headscale",
"preauthkeys",
"expire",
"--id",
strconv.FormatUint(key.GetId(), 10),
})
require.NoError(t, err)
require.NoError(t, err)
// Expire the key so it can't be used
_, err = headscale.Execute(
[]string{
"headscale",
"preauthkeys",
"expire",
"--id",
strconv.FormatUint(key.GetId(), 10),
})
require.NoError(t, err)
require.NoError(t, err)
err = scenario.RunTailscaleUp(userName, headscale.GetEndpoint(), key.GetKey())
assert.ErrorContains(t, err, "authkey expired")
}
})
err = scenario.RunTailscaleUp(userName, headscale.GetEndpoint(), key.GetKey())
assert.ErrorContains(t, err, "authkey expired")
}
}
+3
View File
@@ -938,6 +938,9 @@ func TestOIDCFollowUpUrl(t *testing.T) {
_, err = doLoginURL(ts.Hostname(), newUrl)
require.NoError(t, err)
err = ts.WaitForRunning(integrationutil.PeerSyncTimeout())
require.NoError(t, err)
listUsers, err = headscale.ListUsers()
require.NoError(t, err)
assert.Len(t, listUsers, 1)
+87 -55
View File
@@ -7,7 +7,9 @@ import (
policyv2 "github.com/juanfont/headscale/hscontrol/policy/v2"
"github.com/juanfont/headscale/hscontrol/types"
"github.com/juanfont/headscale/integration/hsic"
"github.com/juanfont/headscale/integration/integrationutil"
"github.com/juanfont/headscale/integration/tsic"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"tailscale.com/tailcfg"
)
@@ -22,9 +24,10 @@ import (
// - bypass: no-bypass talks to the server over gRPC; bypass opens the
// database directly.
//
// Each row spins up its own scenario because policy_mode is fixed at boot
// via `HEADSCALE_POLICY_MODE`. The two users + two nodes give the tests
// block real `user@` aliases to resolve against.
// Each policy mode shares one scenario across its sequential checks because
// policy_mode is fixed at boot. The two users + two nodes give the tests
// block real `user@` aliases to resolve against. Check inputs are separate
// from the live policy, which must remain unchanged after every check.
func TestPolicyCheckCommand(t *testing.T) {
IntegrationSkip(t)
@@ -77,44 +80,56 @@ func TestPolicyCheckCommand(t *testing.T) {
type row struct {
name string
policyMode string
fixture fixture
bypass bool
wantErr string
wantStdout string
}
modes := []string{"file", "database"} //nolint:goconst // axis labels match HEADSCALE_POLICY_MODE values
modes := []types.PolicyMode{types.PolicyModeFile, types.PolicyModeDB}
bypasses := []bool{false, true}
rows := make([]row, 0, len(modes)*len(fixtures)*len(bypasses))
rows := make([]row, 0, len(fixtures)*len(bypasses))
for _, mode := range modes {
for _, f := range fixtures {
for _, bypass := range bypasses {
suffix := "no-bypass"
if bypass {
suffix = "bypass"
}
r := row{
name: mode + "-" + f.name + "-" + suffix,
policyMode: mode,
fixture: f,
bypass: bypass,
wantStdout: "Policy is valid",
}
if f.name == "acl-plus-failing-tests" {
r.wantErr = "test(s) failed"
r.wantStdout = ""
}
rows = append(rows, r)
for _, f := range fixtures {
for _, bypass := range bypasses {
suffix := "no-bypass"
if bypass {
suffix = "bypass"
}
r := row{
name: f.name + "-" + suffix,
fixture: f,
bypass: bypass,
wantStdout: "Policy is valid",
}
if f.name == "acl-plus-failing-tests" {
r.wantErr = "test(s) failed"
r.wantStdout = ""
}
rows = append(rows, r)
}
}
for _, tt := range rows {
t.Run(tt.name, func(t *testing.T) {
// Use a live policy distinct from every check fixture so an accidental
// policy update is observable, including for the ACL-only fixture.
livePolicy := policyv2.Policy{
ACLs: []policyv2.ACL{
{
Action: policyv2.ActionAccept,
Sources: []policyv2.Alias{wildcard()},
Destinations: []policyv2.AliasWithPorts{
aliasWithPorts(wildcard(), tailcfg.PortRangeAny),
},
},
},
}
livePolicyBytes, err := json.Marshal(livePolicy)
require.NoError(t, err)
for _, mode := range modes {
t.Run(string(mode), func(t *testing.T) {
spec := ScenarioSpec{
NodesPerUser: 1,
Users: []string{"user1", "user2"}, //nolint:goconst // matches usernamep("user1@")/("user2@") above
@@ -123,45 +138,62 @@ func TestPolicyCheckCommand(t *testing.T) {
scenario, err := NewScenario(spec)
require.NoError(t, err)
defer scenario.ShutdownAssertNoPanics(t)
t.Cleanup(func() { scenario.ShutdownAssertNoPanics(t) })
err = scenario.CreateHeadscaleEnv(
[]tsic.Option{},
hsic.WithTestName("cli-policycheck"),
hsic.WithConfigEnv(map[string]string{
"HEADSCALE_POLICY_MODE": tt.policyMode, //nolint:goconst // env var name from hscontrol/types/config.go
}),
hsic.WithPolicyMode(mode),
hsic.WithACLPolicy(&livePolicy),
)
require.NoError(t, err)
headscale, err := scenario.Headscale()
require.NoError(t, err)
pBytes, err := json.Marshal(tt.fixture.policy)
require.NoError(t, err)
require.EventuallyWithT(t, func(c *assert.CollectT) {
stdout, err := headscale.Execute([]string{"headscale", "policy", "get"})
assert.NoError(c, err)
assert.JSONEq(c, string(livePolicyBytes), stdout)
}, integrationutil.StatusReadyTimeout, integrationutil.FastPoll, "live policy should be loaded before checks")
policyFilePath := "/etc/headscale/policy.json" //nolint:goconst // standard headscale policy path
err = headscale.WriteFile(policyFilePath, pBytes)
require.NoError(t, err)
for _, tt := range rows {
t.Run(tt.name, func(t *testing.T) {
pBytes, err := json.Marshal(tt.fixture.policy)
require.NoError(t, err)
cmd := []string{"headscale", "policy", "check", "-f", policyFilePath} //nolint:goconst // CLI invocation
if tt.bypass {
// --force suppresses the "is the server running?"
// confirmation prompt so the command can run
// non-interactively under the test harness.
cmd = append(cmd, "--bypass-grpc-and-access-database-directly", "--force")
policyFilePath := "/etc/headscale/policy-check-" + tt.name + ".json"
err = headscale.WriteFile(policyFilePath, pBytes)
require.NoError(t, err)
t.Cleanup(func() {
assert.EventuallyWithT(t, func(c *assert.CollectT) {
stdout, err := headscale.Execute([]string{"headscale", "policy", "get"})
assert.NoError(c, err)
assert.JSONEq(c, string(livePolicyBytes), stdout)
}, integrationutil.StatusReadyTimeout, integrationutil.FastPoll, "policy check must not alter the live policy")
})
cmd := []string{"headscale", "policy", "check", "-f", policyFilePath} //nolint:goconst // CLI invocation
if tt.bypass {
// --force suppresses the "is the server running?"
// confirmation prompt so the command can run
// non-interactively under the test harness.
cmd = append(cmd, "--bypass-grpc-and-access-database-directly", "--force")
}
stdout, err := headscale.Execute(cmd)
if tt.wantErr != "" {
require.ErrorContains(t, err, tt.wantErr)
return
}
require.NoError(t, err)
require.Contains(t, stdout, tt.wantStdout)
})
}
stdout, err := headscale.Execute(cmd)
if tt.wantErr != "" {
require.ErrorContains(t, err, tt.wantErr)
return
}
require.NoError(t, err)
require.Contains(t, stdout, tt.wantStdout)
})
}
}
+64 -15
View File
@@ -51,9 +51,12 @@ const (
var usePostgresForTest = envknob.Bool("HEADSCALE_INTEGRATION_POSTGRES")
var (
errNoHeadscaleAvailable = errors.New("no headscale available")
errNoUserAvailable = errors.New("no user available")
errNoClientFound = errors.New("client not found")
errNoHeadscaleAvailable = errors.New("no headscale available")
errNoUserAvailable = errors.New("no user available")
errNoClientFound = errors.New("client not found")
errInvalidHeadscaleImageFormat = errors.New("invalid HEADSCALE_INTEGRATION_HEADSCALE_IMAGE format, expected repository:tag")
errInvalidMockOIDCImage = errors.New("invalid HEADSCALE_INTEGRATION_HEADSCALE_IMAGE format, expected repository:tag")
errMockOIDCImageRequiredInCI = errors.New("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE must be set for mock OIDC in CI")
// AllVersions represents a list of Tailscale versions the suite
// uses to test compatibility with the [ControlServer].
@@ -684,7 +687,8 @@ func (s *Scenario) CreateTailscaleNodesInUser(
s.mu.Lock()
opts = append(opts,
clientOpts := append(
slices.Clone(opts),
tsic.WithCACert(cert),
tsic.WithHeadscaleName(hostname),
tsic.WithExtraHosts(extraHosts),
@@ -697,7 +701,7 @@ func (s *Scenario) CreateTailscaleNodesInUser(
tsClient, err := tsic.New(
s.pool,
version,
opts...,
clientOpts...,
)
s.mu.Unlock()
@@ -1660,15 +1664,38 @@ func (s *Scenario) runMockOIDC(accessTTL time.Duration, users []mockoidc.MockUse
// Add integration test labels if running under hi tool
dockertestutil.DockerAddIntegrationLabels(mockOidcOptions, "oidc")
if pmockoidc, err := s.pool.BuildAndRunWithBuildOptions( //nolint:noinlineerr
headscaleBuildOptions,
mockOidcOptions,
dockertestutil.DockerRestartPolicy); err == nil {
s.mockOIDC.r = pmockoidc
var container *dockertest.Resource
// CI already built and loaded the Headscale image, including its mockoidc command.
// Reuse it instead of downloading dependencies and rebuilding for every OIDC test.
if prebuiltImage := os.Getenv("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE"); prebuiltImage != "" {
repo, tag, ok := strings.Cut(prebuiltImage, ":")
if !ok || repo == "" || tag == "" {
return errInvalidMockOIDCImage
}
mockOidcOptions.Repository = repo
mockOidcOptions.Tag = tag
container, err = s.pool.RunWithOptions(
mockOidcOptions,
dockertestutil.DockerRestartPolicy,
)
} else if util.IsCI() {
return errMockOIDCImageRequiredInCI
} else {
return err
container, err = s.pool.BuildAndRunWithBuildOptions(
headscaleBuildOptions,
mockOidcOptions,
dockertestutil.DockerRestartPolicy,
)
}
if err != nil {
return fmt.Errorf("starting mock OIDC container: %w", err)
}
s.mockOIDC.r = container
// headscale needs to set up the provider with a specific
// IP addr to ensure we get the correct config from the well-known
// endpoint.
@@ -1753,10 +1780,32 @@ func Webservice(s *Scenario, networkName string) (*dockertest.Resource, error) {
ContextDir: dockerContextPath,
}
web, err := s.pool.BuildAndRunWithBuildOptions(
webBOpts,
webOpts,
dockertestutil.DockerRestartPolicy)
var (
web *dockertest.Resource
err error
)
if prebuiltImage := os.Getenv("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE"); prebuiltImage != "" {
repo, tag, ok := strings.Cut(prebuiltImage, ":")
if !ok {
return nil, errInvalidHeadscaleImageFormat
}
webOpts.Repository = repo
webOpts.Tag = tag
web, err = s.pool.RunWithOptions(
webOpts,
dockertestutil.DockerRestartPolicy,
)
} else {
web, err = s.pool.BuildAndRunWithBuildOptions(
webBOpts,
webOpts,
dockertestutil.DockerRestartPolicy,
)
}
if err != nil {
return nil, err
}
+171
View File
@@ -0,0 +1,171 @@
package tsic
import (
"context"
"fmt"
"io"
"testing"
"time"
"github.com/juanfont/headscale/hscontrol/util"
"github.com/juanfont/headscale/integration/dockertestutil"
"github.com/ory/dockertest/v3"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Simulate Docker's streamed output without requiring Docker or a shell.
func TestLoginReturnsURLBeforeCLIExit(t *testing.T) {
const wantURL = "https://headscale.test/register/complete-token"
for _, stream := range []string{"stdout", "stderr"} {
t.Run(stream, func(t *testing.T) {
partialWritten := make(chan struct{})
completeURL := make(chan struct{})
authenticate := make(chan struct{})
t.Cleanup(func() { close(authenticate) })
login := startLogin([]string{"tailscale", "up"}, func(_ []string, opts dockertest.ExecOptions) (int, error) {
output := opts.StdErr
if stream == "stdout" {
output = opts.StdOut
}
_, _ = io.WriteString(output, "To authenticate, visit:\n\thttps://headscale.test/reg")
close(partialWritten)
<-completeURL
_, _ = io.WriteString(output, "ister/complete-token\n\n")
<-authenticate
// A later URL must not block output draining or replace the first.
_, _ = io.WriteString(output, "https://headscale.test/register/another-token\nSuccess.\n")
return 0, nil
})
<-partialWritten
select {
case u := <-login.url:
t.Fatalf("returned incomplete URL: %s", u)
default:
}
close(completeURL)
u, err := login.waitForURL(time.Second)
require.NoError(t, err)
require.Equal(t, wantURL, u.String())
select {
case <-login.done:
t.Fatal("CLI exited before authentication")
default:
}
// Permit natural completion and verify its exit status is retained.
authenticate <- struct{}{}
select {
case <-login.done:
require.NoError(t, login.err)
assert.Contains(t, login.output.String(), "Success.")
case <-time.After(time.Second):
t.Fatal("CLI did not complete after authentication")
}
})
}
}
func TestLoginCommandFailure(t *testing.T) {
execErr := io.ErrClosedPipe
for _, tt := range []struct {
name string
output string
exitCode int
err error
wantErr error
}{
{name: "exec error", err: execErr, wantErr: execErr},
{name: "CLI failure", output: "invalid login option", exitCode: 7, wantErr: dockertestutil.ErrDockertestCommandFailed},
{name: "no URL", output: "Success.\n", wantErr: util.ErrNoURLFound},
{name: "incomplete URL", output: "https://headscale.test/register/incomplete", wantErr: util.ErrNoURLFound},
} {
t.Run(tt.name, func(t *testing.T) {
login := startLogin(nil, func(_ []string, opts dockertest.ExecOptions) (int, error) {
_, _ = io.WriteString(opts.StdErr, tt.output)
return tt.exitCode, tt.err
})
_, err := login.waitForURL(time.Second)
require.ErrorIs(t, err, tt.wantErr)
assert.Contains(t, err.Error(), tt.output)
})
}
}
func TestLoginFailureAfterURL(t *testing.T) {
finish := make(chan struct{})
t.Cleanup(func() { close(finish) })
login := startLogin(nil, func(_ []string, opts dockertest.ExecOptions) (int, error) {
_, _ = io.WriteString(opts.StdErr, "https://headscale.test/register/token\n")
<-finish
_, _ = io.WriteString(opts.StdErr, "authentication rejected\n")
return 1, nil
})
_, err := login.waitForURL(time.Second)
require.NoError(t, err)
client := &TailscaleInContainer{hostname: "client", login: login}
// A URL alone is not a completed login. The caller's wait stays bounded.
require.ErrorIs(t, client.WaitForRunning(0), context.DeadlineExceeded)
finish <- struct{}{}
select {
case <-login.done:
for range 2 {
// All observers must see the failure, not just the first channel reader.
err = client.WaitForRunning(time.Second)
require.ErrorIs(t, err, dockertestutil.ErrDockertestCommandFailed)
assert.Contains(t, err.Error(), "authentication rejected")
}
case <-time.After(time.Second):
t.Fatal("CLI did not finish")
}
}
func TestLoginURLDeadlineDoesNotStopCLI(t *testing.T) {
finish := make(chan struct{})
t.Cleanup(func() { close(finish) })
login := startLogin(nil, func(_ []string, opts dockertest.ExecOptions) (int, error) {
<-finish
_, _ = fmt.Fprintln(opts.StdErr, "https://headscale.test/register/token")
return 0, nil
})
_, err := login.waitForURL(0)
require.ErrorIs(t, err, dockertestutil.ErrDockertestCommandTimeout)
finish <- struct{}{}
select {
case <-login.done:
require.NoError(t, login.err)
case <-time.After(time.Second):
t.Fatal("CLI did not finish after URL deadline")
}
}
+116 -26
View File
@@ -17,6 +17,7 @@ import (
"slices"
"strconv"
"strings"
"sync"
"time"
"github.com/cenkalti/backoff/v5"
@@ -43,7 +44,9 @@ const (
dockerContextPath = "../."
caCertRoot = "/usr/local/share/ca-certificates"
dockerExecuteTimeout = 60 * time.Second
tailscaleBin = "tailscale"
// Some OIDC tests deliberately delay authentication for two minutes.
loginTimeout = 5 * time.Minute
tailscaleBin = "tailscale"
)
// defaultPingTimeoutVal returns the per-attempt timeout for tailscale ping.
@@ -58,17 +61,16 @@ func defaultPingTimeoutVal() time.Duration {
}
var (
errTailscalePingFailed = errors.New("ping failed")
errTailscalePingNotDERP = errors.New("ping not via DERP")
errTailscaleNotLoggedIn = errors.New("tailscale not logged in")
errTailscaleWrongPeerCount = errors.New("wrong peer count")
errTailscaleCannotUpWithoutAuthkey = errors.New("cannot up without authkey")
errInvalidClientConfig = errors.New("verifiably invalid client config requested")
errInvalidTailscaleImageFormat = errors.New("invalid HEADSCALE_INTEGRATION_TAILSCALE_IMAGE format, expected repository:tag")
errTailscaleImageRequiredInCI = errors.New("HEADSCALE_INTEGRATION_TAILSCALE_IMAGE must be set in CI for HEAD version")
errContainerNotInitialized = errors.New("container not initialized")
errFQDNNotYetAvailable = errors.New("FQDN not yet available")
errCurlEmptyResponseBody = errors.New("curl returned empty response body")
errTailscalePingFailed = errors.New("ping failed")
errTailscalePingNotDERP = errors.New("ping not via DERP")
errTailscaleNotLoggedIn = errors.New("tailscale not logged in")
errTailscaleWrongPeerCount = errors.New("wrong peer count")
errInvalidClientConfig = errors.New("verifiably invalid client config requested")
errInvalidTailscaleImageFormat = errors.New("invalid HEADSCALE_INTEGRATION_TAILSCALE_IMAGE format, expected repository:tag")
errTailscaleImageRequiredInCI = errors.New("HEADSCALE_INTEGRATION_TAILSCALE_IMAGE must be set in CI for HEAD version")
errContainerNotInitialized = errors.New("container not initialized")
errFQDNNotYetAvailable = errors.New("FQDN not yet available")
errCurlEmptyResponseBody = errors.New("curl returned empty response body")
)
const (
@@ -93,6 +95,10 @@ type TailscaleInContainer struct {
ips []netip.Addr
fqdn string
// Only the current attempt is checked: earlier logins may be deliberately
// abandoned or rejected before the test starts another one.
login *loginAttempt
// optional config
caCerts [][]byte
headscaleHostname string
@@ -698,6 +704,7 @@ func (t *TailscaleInContainer) buildLoginCommand(
func (t *TailscaleInContainer) Login(
loginServer, authKey string,
) error {
t.login = nil
command := t.buildLoginCommand(loginServer, authKey)
if _, _, err := t.Execute(command, dockertestutil.ExecuteCommandTimeout(dockerExecuteTimeout)); err != nil { //nolint:noinlineerr
@@ -712,34 +719,106 @@ func (t *TailscaleInContainer) Login(
return nil
}
// loginAttempt streams the first complete URL line while retaining all output.
// done closes only after Docker exec returns; err can then be read repeatedly.
type loginAttempt struct {
mu sync.Mutex
output bytes.Buffer
lineStart int
foundURL bool
url chan *url.URL
done chan struct{}
err error
}
func (l *loginAttempt) Write(p []byte) (int, error) {
l.mu.Lock()
defer l.mu.Unlock()
n, _ := l.output.Write(p)
for !l.foundURL {
line, _, complete := bytes.Cut(l.output.Bytes()[l.lineStart:], []byte{'\n'})
if !complete {
break
}
l.lineStart += len(line) + 1
u, err := util.ParseLoginURLFromCLILogin(string(line))
if err == nil {
l.foundURL = true
l.url <- u
}
}
return n, nil
}
func startLogin(
command []string,
exec func([]string, dockertest.ExecOptions) (int, error),
) *loginAttempt {
l := &loginAttempt{url: make(chan *url.URL, 1), done: make(chan struct{})}
go func() {
defer close(l.done)
exitCode, err := exec(command, dockertest.ExecOptions{StdOut: l, StdErr: l})
log.Printf("%v finished (exit %d): %s", command, exitCode, l.output.String())
switch {
case err != nil:
l.err = fmt.Errorf("tailscale up: %s: %w", l.output.String(), err)
case exitCode != 0:
l.err = fmt.Errorf("tailscale up exited %d: %s: %w", exitCode, l.output.String(), dockertestutil.ErrDockertestCommandFailed)
case !l.foundURL:
l.err = fmt.Errorf("tailscale up: %s: %w", l.output.String(), util.ErrNoURLFound)
}
}()
return l
}
func (l *loginAttempt) waitForURL(timeout time.Duration) (*url.URL, error) {
select {
case u := <-l.url:
return u, nil
case <-l.done:
if l.err != nil {
return nil, l.err
}
return <-l.url, nil
case <-time.After(timeout):
return nil, dockertestutil.ErrDockertestCommandTimeout
}
}
// LoginWithURL runs the login routine on the given Tailscale instance.
// This login mechanism uses web + command line flow for authentication.
func (t *TailscaleInContainer) LoginWithURL(
loginServer string,
) (*url.URL, error) {
command := t.buildLoginCommand(loginServer, "")
command = append(command, "--timeout="+loginTimeout.String())
stdout, stderr, err := t.Execute(command)
if errors.Is(err, errTailscaleNotLoggedIn) {
return nil, errTailscaleCannotUpWithoutAuthkey
}
// Return the URL promptly, but let the CLI finish authentication normally.
// Its own timeout bounds abandoned logins; WaitForRunning checks its exit.
t.login = startLogin(command, t.container.Exec)
defer func() {
if err != nil {
log.Printf("join command: %q", strings.Join(command, " "))
}
}()
loginURL, err := util.ParseLoginURLFromCLILogin(stdout + stderr)
u, err := t.login.waitForURL(dockerExecuteTimeout)
if err != nil {
return nil, err
return nil, fmt.Errorf("%s fetching login URL: %w", t.hostname, err)
}
return loginURL, nil
return u, nil
}
// Logout runs the logout routine on the given Tailscale instance.
func (t *TailscaleInContainer) Logout() error {
t.login = nil
_, _, err := t.Execute([]string{tailscaleBin, "logout"})
if err != nil {
return err
@@ -1240,7 +1319,7 @@ func (t *TailscaleInContainer) WaitForNeedsLogin(timeout time.Duration) error {
}
// WaitForRunning blocks until the Tailscale (tailscaled) instance is logged in
// and ready to be used.
// and ready to be used, and the current web login command has exited successfully.
func (t *TailscaleInContainer) WaitForRunning(timeout time.Duration) error {
return t.waitForBackendState("Running", timeout)
}
@@ -1252,6 +1331,17 @@ func (t *TailscaleInContainer) waitForBackendState(state string, timeout time.Du
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
if state == "Running" && t.login != nil {
select {
case <-t.login.done:
if t.login.err != nil {
return fmt.Errorf("%s login failed: %w", t.hostname, t.login.err)
}
case <-ctx.Done():
return fmt.Errorf("%s waiting for login command: %w", t.hostname, ctx.Err())
}
}
for {
select {
case <-ctx.Done():