integration, hsic, tsic, dockertestutil: consolidate container helpers

This commit is contained in:
Kristoffer Dalby
2026-06-16 07:06:21 +00:00
parent 75b09cb991
commit 722a2f1e18
6 changed files with 160 additions and 220 deletions
+28 -34
View File
@@ -30,12 +30,7 @@ func retryDockerOp(ctx context.Context, op func() error) error {
bo.MaxInterval = DockerOpMaxInterval
_, err := backoff.Retry(ctx, func() (struct{}, error) {
err := op()
if err != nil {
return struct{}{}, err
}
return struct{}{}, nil
return struct{}{}, op()
}, backoff.WithBackOff(bo), backoff.WithMaxElapsedTime(DockerOpMaxElapsedTime))
return err
@@ -68,18 +63,17 @@ func GetFirstOrCreateNetworkWithSubnet(pool *dockertest.Pool, name, subnet strin
})
}
if _, err := pool.CreateNetwork(name, opts...); err == nil { //nolint:noinlineerr // intentional inline check
// Create does not give us an updated version of the resource, so we need to
// get it again.
networks, err := pool.NetworksByName(name)
if err != nil {
return nil, err
}
return &networks[0], nil
} else {
_, err = pool.CreateNetwork(name, opts...)
if err != nil {
return nil, fmt.Errorf("creating network: %w", err)
}
// Create does not give us an updated version of the resource, so we need to
// get it again.
networks, err = pool.NetworksByName(name)
if err != nil {
return nil, fmt.Errorf("looking up network names: %w", err)
}
}
return &networks[0], nil
@@ -222,11 +216,13 @@ func DisconnectAndReconnect(
return nil
}
func waitNetworkContainerAbsent(
func waitNetworkContainer(
pool *dockertest.Pool,
network *dockertest.Network,
testContainer string,
timeout time.Duration,
want bool,
match func(docker.Endpoint) bool,
) error {
return pollUntil(timeout, func() (bool, error) {
net, err := pool.Client.NetworkInfo(network.Network.ID)
@@ -234,36 +230,34 @@ func waitNetworkContainerAbsent(
return false, fmt.Errorf("inspecting network %s: %w", network.Network.Name, err)
}
found := false
for _, c := range net.Containers {
if c.Name == testContainer || c.Name == "/"+testContainer {
return false, nil
if (c.Name == testContainer || c.Name == "/"+testContainer) && match(c) {
found = true
break
}
}
return true, nil
return found == want, nil
})
}
func waitNetworkContainerAbsent(
pool *dockertest.Pool,
network *dockertest.Network,
testContainer string,
timeout time.Duration,
) error {
return waitNetworkContainer(pool, network, testContainer, timeout, false, func(docker.Endpoint) bool { return true })
}
func waitNetworkContainerPresent(
pool *dockertest.Pool,
network *dockertest.Network,
testContainer string,
timeout time.Duration,
) error {
return pollUntil(timeout, func() (bool, error) {
net, err := pool.Client.NetworkInfo(network.Network.ID)
if err != nil {
return false, fmt.Errorf("inspecting network %s: %w", network.Network.Name, err)
}
for _, c := range net.Containers {
if (c.Name == testContainer || c.Name == "/"+testContainer) && c.IPv4Address != "" {
return true, nil
}
}
return false, nil
})
return waitNetworkContainer(pool, network, testContainer, timeout, true, func(c docker.Endpoint) bool { return c.IPv4Address != "" })
}
// waitContainerRouteAbsent polls the container's routing table until no
+17 -19
View File
@@ -55,6 +55,15 @@ const (
stateOffline = "offline"
)
// onlineLabel returns the log string for the given online state.
func onlineLabel(online bool) string {
if online {
return stateOnline
}
return stateOffline
}
var errNoNewClientFound = errors.New("no new client found")
// NodeSystemStatus represents the status of a node across different systems.
@@ -166,10 +175,7 @@ func requireAllClientsOnline(t *testing.T, headscale ControlServer, expectedNode
startTime := time.Now()
stateStr := stateOffline
if expectedOnline {
stateStr = stateOnline
}
stateStr := onlineLabel(expectedOnline)
t.Logf("requireAllSystemsOnline: Starting %s validation for %d nodes at %s - %s", stateStr, len(expectedNodes), startTime.Format(TimestampFormat), message)
@@ -193,6 +199,8 @@ func requireAllClientsOnlineWithSingleTimeout(t *testing.T, headscale ControlSer
var prevReport string
stateStr := onlineLabel(expectedOnline)
require.EventuallyWithT(t, func(c *assert.CollectT) {
// Get batcher state
debugInfo, err := headscale.DebugBatcher()
@@ -239,7 +247,7 @@ func requireAllClientsOnlineWithSingleTimeout(t *testing.T, headscale ControlSer
// Check batcher state for expected nodes
for _, nodeID := range expectedNodes {
nodeIDStr := fmt.Sprintf("%d", nodeID)
nodeIDStr := nodeID.String()
if nodeInfo, exists := debugInfo.ConnectedNodes[nodeIDStr]; exists {
if status, exists := nodeStatus[nodeID]; exists {
status.Batcher = nodeInfo.Connected
@@ -326,8 +334,7 @@ func requireAllClientsOnlineWithSingleTimeout(t *testing.T, headscale ControlSer
var failureReport strings.Builder
ids := types.NodeIDs(slices.AppendSeq(make([]types.NodeID, 0, len(nodeStatus)), maps.Keys(nodeStatus)))
slices.Sort(ids)
ids := types.NodeIDs(slices.Sorted(maps.Keys(nodeStatus)))
for _, nodeID := range ids {
status := nodeStatus[nodeID]
@@ -338,11 +345,6 @@ func requireAllClientsOnlineWithSingleTimeout(t *testing.T, headscale ControlSer
if !systemsMatch {
allMatch = false
stateStr := stateOffline
if expectedOnline {
stateStr = stateOnline
}
fmt.Fprintf(&failureReport, "node:%d is not fully %s (timestamp: %s):\n", nodeID, stateStr, time.Now().Format(TimestampFormat))
fmt.Fprintf(&failureReport, " - batcher: %t (expected: %t)\n", status.Batcher, expectedOnline)
fmt.Fprintf(&failureReport, " - conn count: %d\n", status.BatcherConnCount)
@@ -367,11 +369,6 @@ func requireAllClientsOnlineWithSingleTimeout(t *testing.T, headscale ControlSer
assert.Fail(c, failureReport.String())
}
stateStr := stateOffline
if expectedOnline {
stateStr = stateOnline
}
assert.True(c, allMatch, "Not all %d nodes are %s across all systems (batcher, mapresponses, nodestore)", len(expectedNodes), stateStr)
}, timeout, 2*time.Second, message)
}
@@ -393,7 +390,7 @@ func requireAllClientsOfflineStaged(t *testing.T, headscale ControlServer, expec
allBatcherOffline := true
for _, nodeID := range expectedNodes {
nodeIDStr := fmt.Sprintf("%d", nodeID)
nodeIDStr := nodeID.String()
if nodeInfo, exists := debugInfo.ConnectedNodes[nodeIDStr]; exists && nodeInfo.Connected {
allBatcherOffline = false
@@ -650,7 +647,8 @@ func assertPingAll(t *testing.T, clients []TailscaleClient, addrs []string, opts
perPingBudget := 2 * time.Second
timeout := max(
// Floor at 30s for small matrices.
integrationutil.ScaledTimeout(time.Duration(pingCount)*perPingBudget*2), integrationutil.ScaledTimeout(30*time.Second))
integrationutil.ScaledTimeout(time.Duration(pingCount)*perPingBudget*2), integrationutil.ScaledTimeout(30*time.Second),
)
assert.EventuallyWithT(t, func(c *assert.CollectT) {
assertPingAllWithCollect(c, clients, addrs, opts...)
+23 -78
View File
@@ -490,7 +490,8 @@ func New(
for _, hostPort := range hostPorts {
runOptions.PortBindings[docker.Port(port)] = append(
runOptions.PortBindings[docker.Port(port)],
docker.PortBinding{HostPort: hostPort})
docker.PortBinding{HostPort: hostPort},
)
}
}
}
@@ -1151,7 +1152,8 @@ func (t *HeadscaleInContainer) CreateAuthKeyWithOptions(opts AuthKeyOptions) (*v
command = append(command, "--user", strconv.FormatUint(*opts.User, 10))
}
command = append(command,
command = append(
command,
"preauthkeys",
"create",
"--expiration",
@@ -1330,13 +1332,10 @@ func (t *HeadscaleInContainer) NodesByUser() (map[string][]*v1.Node, error) {
return nil, err
}
var userMap map[string][]*v1.Node
userMap := make(map[string][]*v1.Node)
for _, node := range nodes {
if _, ok := userMap[node.GetUser().GetName()]; !ok {
mak.Set(&userMap, node.GetUser().GetName(), []*v1.Node{node})
} else {
userMap[node.GetUser().GetName()] = append(userMap[node.GetUser().GetName()], node)
}
name := node.GetUser().GetName()
userMap[name] = append(userMap[name], node)
}
return userMap, nil
@@ -1636,102 +1635,48 @@ func (t *HeadscaleInContainer) SendInterrupt() error {
}
func (t *HeadscaleInContainer) GetAllMapReponses() (map[types.NodeID][]tailcfg.MapResponse, error) {
// Execute curl inside the container to access the debug endpoint locally
command := []string{
"curl", "-s", "-H", acceptJSON, "http://localhost:9090/debug/mapresponses",
}
result, err := t.Execute(command)
if err != nil {
return nil, fmt.Errorf("fetching mapresponses from debug endpoint: %w", err)
}
var res map[types.NodeID][]tailcfg.MapResponse
if err := json.Unmarshal([]byte(result), &res); err != nil { //nolint:noinlineerr
return nil, fmt.Errorf("decoding routes response: %w", err)
}
return res, nil
return debugJSON[map[types.NodeID][]tailcfg.MapResponse](t, "mapresponses")
}
// PrimaryRoutes fetches the primary routes from the debug endpoint.
func (t *HeadscaleInContainer) PrimaryRoutes() (*types.DebugRoutes, error) {
// Execute curl inside the container to access the debug endpoint locally
command := []string{
"curl", "-s", "-H", acceptJSON, "http://localhost:9090/debug/routes",
}
result, err := t.Execute(command)
if err != nil {
return nil, fmt.Errorf("fetching routes from debug endpoint: %w", err)
}
var debugRoutes types.DebugRoutes
if err := json.Unmarshal([]byte(result), &debugRoutes); err != nil { //nolint:noinlineerr
return nil, fmt.Errorf("decoding routes response: %w", err)
}
return &debugRoutes, nil
return debugJSON[*types.DebugRoutes](t, "routes")
}
// DebugBatcher fetches the batcher debug information from the debug endpoint.
func (t *HeadscaleInContainer) DebugBatcher() (*hscontrol.DebugBatcherInfo, error) {
// Execute curl inside the container to access the debug endpoint locally
command := []string{
"curl", "-s", "-H", acceptJSON, "http://localhost:9090/debug/batcher",
}
result, err := t.Execute(command)
if err != nil {
return nil, fmt.Errorf("fetching batcher debug info: %w", err)
}
var debugInfo hscontrol.DebugBatcherInfo
if err := json.Unmarshal([]byte(result), &debugInfo); err != nil { //nolint:noinlineerr
return nil, fmt.Errorf("decoding batcher debug response: %w", err)
}
return &debugInfo, nil
return debugJSON[*hscontrol.DebugBatcherInfo](t, "batcher")
}
// DebugNodeStore fetches the [state.NodeStore] data from the debug endpoint.
func (t *HeadscaleInContainer) DebugNodeStore() (map[types.NodeID]types.Node, error) {
// Execute curl inside the container to access the debug endpoint locally
command := []string{
"curl", "-s", "-H", acceptJSON, "http://localhost:9090/debug/nodestore",
}
result, err := t.Execute(command)
if err != nil {
return nil, fmt.Errorf("fetching nodestore debug info: %w", err)
}
var nodeStore map[types.NodeID]types.Node
if err := json.Unmarshal([]byte(result), &nodeStore); err != nil { //nolint:noinlineerr
return nil, fmt.Errorf("decoding nodestore debug response: %w", err)
}
return nodeStore, nil
return debugJSON[map[types.NodeID]types.Node](t, "nodestore")
}
// DebugFilter fetches the current filter rules from the debug endpoint.
func (t *HeadscaleInContainer) DebugFilter() ([]tailcfg.FilterRule, error) {
return debugJSON[[]tailcfg.FilterRule](t, "filter")
}
// debugJSON fetches and decodes a JSON-returning debug endpoint by name.
func debugJSON[T any](t *HeadscaleInContainer, endpoint string) (T, error) {
var res T
// Execute curl inside the container to access the debug endpoint locally
command := []string{
"curl", "-s", "-H", acceptJSON, "http://localhost:9090/debug/filter",
"curl", "-s", "-H", acceptJSON, "http://localhost:9090/debug/" + endpoint,
}
result, err := t.Execute(command)
if err != nil {
return nil, fmt.Errorf("fetching filter from debug endpoint: %w", err)
return res, fmt.Errorf("fetching %s from debug endpoint: %w", endpoint, err)
}
var filterRules []tailcfg.FilterRule
if err := json.Unmarshal([]byte(result), &filterRules); err != nil { //nolint:noinlineerr
return nil, fmt.Errorf("decoding filter response: %w", err)
if err := json.Unmarshal([]byte(result), &res); err != nil { //nolint:noinlineerr
return res, fmt.Errorf("decoding %s response: %w", endpoint, err)
}
return filterRules, nil
return res, nil
}
// DebugPolicy fetches the current policy from the debug endpoint.
+9 -9
View File
@@ -241,23 +241,23 @@ func BuildExpectedOnlineMap(all map[types.NodeID][]tailcfg.MapResponse) map[type
for nid, mrs := range all {
res[nid] = make(map[types.NodeID]bool)
set := func(id tailcfg.NodeID, online *bool) {
if online != nil {
res[nid][types.NodeID(id)] = *online //nolint:gosec // safe conversion for peer ID
}
}
for _, mr := range mrs {
for _, peer := range mr.Peers {
if peer.Online != nil {
res[nid][types.NodeID(peer.ID)] = *peer.Online //nolint:gosec // safe conversion for peer ID
}
set(peer.ID, peer.Online)
}
for _, peer := range mr.PeersChanged {
if peer.Online != nil {
res[nid][types.NodeID(peer.ID)] = *peer.Online //nolint:gosec // safe conversion for peer ID
}
set(peer.ID, peer.Online)
}
for _, peer := range mr.PeersChangedPatch {
if peer.Online != nil {
res[nid][types.NodeID(peer.NodeID)] = *peer.Online //nolint:gosec // safe conversion for peer ID
}
set(peer.NodeID, peer.Online)
}
}
}
+20 -32
View File
@@ -211,7 +211,7 @@ func NewScenario(spec ScenarioSpec) (*Scenario, error) {
var userToNetwork map[string]*dockertest.Network
if spec.Networks != nil || len(spec.Networks) != 0 {
if spec.Networks != nil {
for name, netSpec := range s.spec.Networks {
networkName := testHashPrefix + "-" + name
@@ -588,7 +588,8 @@ func (s *Scenario) CreateTailscaleNode(
s.mu.Lock()
defer s.mu.Unlock()
opts = append(opts,
opts = append(
opts,
tsic.WithCACert(cert),
tsic.WithHeadscaleName(hostname),
)
@@ -662,7 +663,8 @@ func (s *Scenario) CreateTailscaleNodesInUser(
s.mu.Lock()
opts = append(opts,
opts = append(
opts,
tsic.WithCACert(cert),
tsic.WithHeadscaleName(hostname),
tsic.WithExtraHosts(extraHosts),
@@ -818,45 +820,29 @@ func (s *Scenario) WaitForTailscaleSyncPerUser(timeout, retryInterval time.Durat
}
}
var allErrors []error
for _, user := range s.users {
// Calculate expected peer count: number of nodes in this user minus 1 (self)
expectedPeers := len(user.Clients) - 1
for _, client := range user.Clients {
c := client
expectedCount := expectedPeers
user.syncWaitGroup.Go(func() error {
return c.WaitForPeers(expectedCount, timeout, retryInterval)
})
}
err := user.syncWaitGroup.Wait()
if err != nil {
allErrors = append(allErrors, err)
}
}
if len(allErrors) > 0 {
return multierr.New(allErrors...)
}
return nil
// Calculate expected peer count: number of nodes in this user minus 1 (self)
return s.waitPeers(func(u *User) int { return len(u.Clients) - 1 }, timeout, retryInterval)
}
// WaitForTailscaleSyncWithPeerCount blocks execution until all the [TailscaleClient] reports
// to have all other [TailscaleClient]s present in their [netmap.NetworkMap].
func (s *Scenario) WaitForTailscaleSyncWithPeerCount(peerCount int, timeout, retryInterval time.Duration) error {
return s.waitPeers(func(*User) int { return peerCount }, timeout, retryInterval)
}
// waitPeers blocks until every [TailscaleClient] reports the expected peer
// count returned by perUser for its owning user, fanning out per user.
func (s *Scenario) waitPeers(perUser func(*User) int, timeout, retryInterval time.Duration) error {
var allErrors []error
for _, user := range s.users {
expectedCount := perUser(user)
for _, client := range user.Clients {
c := client
user.syncWaitGroup.Go(func() error {
return c.WaitForPeers(peerCount, timeout, retryInterval)
return c.WaitForPeers(expectedCount, timeout, retryInterval)
})
}
@@ -1636,7 +1622,8 @@ func (s *Scenario) runMockOIDC(accessTTL time.Duration, users []mockoidc.MockUse
if pmockoidc, err := s.pool.BuildAndRunWithBuildOptions( //nolint:noinlineerr
headscaleBuildOptions,
mockOidcOptions,
dockertestutil.DockerRestartPolicy); err == nil {
dockertestutil.DockerRestartPolicy,
); err == nil {
s.mockOIDC.r = pmockoidc
} else {
return err
@@ -1729,7 +1716,8 @@ func Webservice(s *Scenario, networkName string) (*dockertest.Resource, error) {
web, err := s.pool.BuildAndRunWithBuildOptions(
webBOpts,
webOpts,
dockertestutil.DockerRestartPolicy)
dockertestutil.DockerRestartPolicy,
)
if err != nil {
return nil, err
}
+63 -48
View File
@@ -685,7 +685,8 @@ func (t *TailscaleInContainer) buildLoginCommand(
}
if len(t.withTags) > 0 {
command = append(command,
command = append(
command,
"--advertise-tags="+strings.Join(t.withTags, ","),
)
}
@@ -930,6 +931,31 @@ func (t *TailscaleInContainer) MustIPv6() netip.Addr {
panic("no ipv6 found")
}
// execJSON runs command on the Tailscale instance, unmarshals the stdout into a
// fresh T, and returns the value alongside the raw JSON for callers that persist
// it. stderr is printed when the command fails. execErr and unmarshalErr provide
// the error context used in test diagnostics.
func execJSON[T any](
t *TailscaleInContainer,
command []string,
execErr, unmarshalErr string,
) (*T, string, error) {
result, stderr, err := t.Execute(command)
if err != nil {
fmt.Printf("stderr: %s\n", stderr)
return nil, "", fmt.Errorf("%s: %w", execErr, err)
}
var v T
err = json.Unmarshal([]byte(result), &v)
if err != nil {
return nil, "", fmt.Errorf("%s: %w", unmarshalErr, err)
}
return &v, result, nil
}
// Status returns the [ipnstate.Status] of the Tailscale instance.
func (t *TailscaleInContainer) Status(save ...bool) (*ipnstate.Status, error) {
command := []string{
@@ -938,24 +964,22 @@ func (t *TailscaleInContainer) Status(save ...bool) (*ipnstate.Status, error) {
"--json",
}
result, _, err := t.Execute(command)
status, raw, err := execJSON[ipnstate.Status](
t,
command,
"executing tailscale status command",
"unmarshalling tailscale status",
)
if err != nil {
return nil, fmt.Errorf("executing tailscale status command: %w", err)
return nil, err
}
var status ipnstate.Status
err = json.Unmarshal([]byte(result), &status)
if err != nil {
return nil, fmt.Errorf("unmarshalling tailscale status: %w", err)
}
err = os.WriteFile(fmt.Sprintf("/tmp/control/%s_status.json", t.hostname), []byte(result), 0o755) //nolint:gosec // test infrastructure log files
err = os.WriteFile(fmt.Sprintf("/tmp/control/%s_status.json", t.hostname), []byte(raw), 0o755) //nolint:gosec // test infrastructure log files
if err != nil {
return nil, fmt.Errorf("status netmap to /tmp/control: %w", err)
}
return &status, err
return status, nil
}
// MustStatus returns the [ipnstate.Status] of the Tailscale instance.
@@ -997,25 +1021,22 @@ func (t *TailscaleInContainer) Netmap() (*netmap.NetworkMap, error) {
"netmap",
}
result, stderr, err := t.Execute(command)
nm, raw, err := execJSON[netmap.NetworkMap](
t,
command,
"executing tailscale debug netmap command",
"unmarshalling tailscale netmap",
)
if err != nil {
fmt.Printf("stderr: %s\n", stderr)
return nil, fmt.Errorf("executing tailscale debug netmap command: %w", err)
return nil, err
}
var nm netmap.NetworkMap
err = json.Unmarshal([]byte(result), &nm)
if err != nil {
return nil, fmt.Errorf("unmarshalling tailscale netmap: %w", err)
}
err = os.WriteFile(fmt.Sprintf("/tmp/control/%s_netmap.json", t.hostname), []byte(result), 0o755) //nolint:gosec // test infrastructure log files
err = os.WriteFile(fmt.Sprintf("/tmp/control/%s_netmap.json", t.hostname), []byte(raw), 0o755) //nolint:gosec // test infrastructure log files
if err != nil {
return nil, fmt.Errorf("saving netmap to /tmp/control: %w", err)
}
return &nm, err
return nm, nil
}
// Netmap returns the current Netmap ([netmap.NetworkMap]) of the Tailscale instance.
@@ -1125,21 +1146,17 @@ func (t *TailscaleInContainer) DebugDERPRegion(region string) (*ipnstate.DebugDE
region,
}
result, stderr, err := t.Execute(command)
report, _, err := execJSON[ipnstate.DebugDERPRegionReport](
t,
command,
"executing tailscale debug derp command",
"unmarshalling tailscale derp region report",
)
if err != nil {
fmt.Printf("stderr: %s\n", stderr) // nolint
return nil, fmt.Errorf("executing tailscale debug derp command: %w", err)
return nil, err
}
var report ipnstate.DebugDERPRegionReport
err = json.Unmarshal([]byte(result), &report)
if err != nil {
return nil, fmt.Errorf("unmarshalling tailscale derp region report: %w", err)
}
return &report, err
return report, nil
}
// Netcheck returns the current Netcheck Report ([netcheck.Report]) of the Tailscale instance.
@@ -1150,20 +1167,17 @@ func (t *TailscaleInContainer) Netcheck() (*netcheck.Report, error) {
"--format=json",
}
result, stderr, err := t.Execute(command)
nm, _, err := execJSON[netcheck.Report](
t,
command,
"executing tailscale debug netcheck command",
"unmarshalling tailscale netcheck",
)
if err != nil {
fmt.Printf("stderr: %s\n", stderr)
return nil, fmt.Errorf("executing tailscale debug netcheck command: %w", err)
return nil, err
}
var nm netcheck.Report
err = json.Unmarshal([]byte(result), &nm)
if err != nil {
return nil, fmt.Errorf("unmarshalling tailscale netcheck: %w", err)
}
return &nm, err
return nm, nil
}
// FQDN returns the FQDN as a string of the Tailscale instance.
@@ -1400,7 +1414,8 @@ func (t *TailscaleInContainer) Ping(hostnameOrIP string, opts ...PingOption) err
}
command := make([]string, 0, 6)
command = append(command,
command = append(
command,
tailscaleBin, "ping",
fmt.Sprintf("--timeout=%s", args.timeout),
fmt.Sprintf("--c=%d", args.count),