Compare commits

..

1 Commits

Author SHA1 Message Date
dependabot[bot] c64b7977c1 build(deps): bump github.com/opencontainers/runc from 1.3.2 to 1.3.6
Bumps [github.com/opencontainers/runc](https://github.com/opencontainers/runc) from 1.3.2 to 1.3.6.
- [Release notes](https://github.com/opencontainers/runc/releases)
- [Changelog](https://github.com/opencontainers/runc/blob/main/CHANGELOG.md)
- [Commits](https://github.com/opencontainers/runc/compare/v1.3.2...v1.3.6)

---
updated-dependencies:
- dependency-name: github.com/opencontainers/runc
  dependency-version: 1.3.6
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-06-26 07:19:47 +00:00
39 changed files with 92 additions and 2801 deletions
@@ -78,13 +78,6 @@ jobs:
echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
docker version
- name: Load br_netfilter for in-cluster service routing
if: inputs.test == 'TestK8sOperator'
# TestK8sOperator runs k3s in a container; without br_netfilter on the
# host, bridged pod-to-pod traffic skips kube-proxy's ClusterIP DNAT and
# in-cluster DNS (kube-dns) is unreachable. The module cannot be loaded
# from inside the unprivileged-module rancher/k3s image, so load it here.
run: sudo modprobe br_netfilter
- name: Login to Docker Hub
env:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }}
-3
View File
@@ -311,7 +311,6 @@ jobs:
- Test2118DeletingOnlineNodePanics
- TestGrantCapRelay
- TestGrantCapDrive
- TestK8sOperator
- TestEnablingRoutes
- TestHASubnetRouterFailover
- TestSubnetRouteACL
@@ -379,8 +378,6 @@ jobs:
- TestTagsAuthKeyWithoutUserInheritsTags
- TestTagsAuthKeyWithoutUserRejectsAdvertisedTags
- TestTagsAuthKeyConvertToUserViaCLIRegister
- TestTS2021WebSocketGET
- TestTS2021WASMClientUnderNode
- TestTailscaleRustAxum
uses: ./.github/workflows/integration-test-template.yml
secrets: inherit
-10
View File
@@ -45,16 +45,6 @@ keys remain all-access.
- Expiring or deleting a non-existent pre-auth key now returns an error instead of silently succeeding [#3324](https://github.com/juanfont/headscale/pull/3324)
- Improve systemd service file hardening [#3341](https://github.com/juanfont/headscale/pull/3341)
## 0.29.2 (2026-07-01)
**Minimum supported Tailscale client version: v1.80.0**
### Changes
- Fix map generation serializing on the policy lock, so a mass reconnect on `autogroup:self`, via or relay policies no longer stalls clients into `unexpected EOF` retry loops [#3358](https://github.com/juanfont/headscale/pull/3358)
- Fix `/ts2021` rejecting the WebSocket `GET` upgrade with 405, which prevented Tailscale JS/WASM control clients from connecting [#3359](https://github.com/juanfont/headscale/pull/3359)
- Gracefully handle nodes with an invalid FQDN (empty or too long) instead of failing map delivery; offending names are logged at startup with the fix command [#3349](https://github.com/juanfont/headscale/pull/3349)
## 0.29.1 (2026-06-18)
**Minimum supported Tailscale client version: v1.80.0**
-30
View File
@@ -1,30 +0,0 @@
# For integration testing only.
#
# Builds the Tailscale control client (integration/wasmic/wasmclient) for
# GOOS=js/GOARCH=wasm and packages it with Go's wasm_exec Node runner. The
# container idles; the integration test execs
# node /app/wasm_exec_node.js /app/client.wasm <control-url>
# to drive a real browser-style WebSocket GET against headscale's /ts2021,
# guarding the regression in issue #3357.
FROM golang:1.26.4-alpine AS build
WORKDIR /src
# Only the module metadata and the wasm client package are needed to build the
# js/wasm binary; its imports (tailscale.com/control/controlhttp, ...) resolve
# from the module proxy.
COPY go.mod go.sum ./
COPY integration/wasmic/wasmclient ./integration/wasmic/wasmclient
RUN GOOS=js GOARCH=wasm go build -o /out/client.wasm ./integration/wasmic/wasmclient \
&& cp "$(go env GOROOT)/lib/wasm/wasm_exec.js" /out/wasm_exec.js \
&& cp "$(go env GOROOT)/lib/wasm/wasm_exec_node.js" /out/wasm_exec_node.js
FROM node:24-alpine
WORKDIR /app
COPY --from=build /out/ /app/
# Idle; the test execs the client on demand with the headscale control URL.
ENTRYPOINT ["tail", "-f", "/dev/null"]
+3 -18
View File
@@ -187,29 +187,14 @@ func removeContainerWithRetry(ctx context.Context, cli *client.Client, container
return err == nil
}
// testContainerNamePrefixes are the name prefixes used by containers that the
// integration test harness creates (headscale, tailscale, DERP, and k3s).
var testContainerNamePrefixes = []string{"hs-", "ts-", "derp-", "k3s-"}
// matchesTestContainerPrefix reports whether name belongs to an integration
// test container, ignoring any leading "/" that Docker prefixes names with.
func matchesTestContainerPrefix(name string) bool {
name = strings.TrimPrefix(name, "/")
for _, prefix := range testContainerNamePrefixes {
if strings.HasPrefix(name, prefix) {
return true
}
}
return false
}
// isTestContainerName reports whether any of the container names belong to an
// integration test container.
func isTestContainerName(names []string) bool {
for _, name := range names {
if strings.Contains(name, "headscale-test-suite") ||
matchesTestContainerPrefix(name) {
strings.Contains(name, "hs-") ||
strings.Contains(name, "ts-") ||
strings.Contains(name, "derp-") {
return true
}
}
+1 -1
View File
@@ -735,7 +735,7 @@ func getCurrentTestContainers(containers []container.Summary, testContainerID st
for _, cont := range containers {
for _, name := range cont.Names {
containerName := strings.TrimPrefix(name, "/")
if matchesTestContainerPrefix(containerName) {
if strings.HasPrefix(containerName, "hs-") || strings.HasPrefix(containerName, "ts-") {
// Check if container has matching run ID label
if cont.Labels != nil && cont.Labels["hi.run-id"] == runID {
testRunContainers = append(testRunContainers, testContainer{
-41
View File
@@ -10,7 +10,6 @@ import (
"strings"
"github.com/juanfont/headscale/integration/dockertestutil"
"github.com/juanfont/headscale/integration/k3sic"
)
const (
@@ -22,7 +21,6 @@ const (
nameDockerContext = "Docker Context"
nameDockerSocket = "Docker Socket"
nameGolangImage = "Golang Image"
nameK3sImage = "K3s Image"
nameGoInstall = "Go Installation"
)
@@ -68,7 +66,6 @@ func runDoctorCheck(ctx context.Context) error {
results = append(results, checkDockerSocket(ctx))
results = append(results, checkDockerHubCredentials())
results = append(results, checkGolangImage(ctx))
results = append(results, checkK3sImage(ctx))
}
// Check 3: Go installation
@@ -245,44 +242,6 @@ func checkGolangImage(ctx context.Context) DoctorResult {
return pass(nameGolangImage, fmt.Sprintf("Golang image %s is now available", imageName))
}
// checkK3sImage verifies the ghcr k3s image used by TestK8sOperator is available
// locally or can be pulled. The image is pinned (see [k3sic.K3sImage]).
func checkK3sImage(ctx context.Context) DoctorResult {
cli, err := createDockerClient(ctx)
if err != nil {
return fail(nameK3sImage, "Cannot create Docker client for image check")
}
defer cli.Close()
imageName := k3sic.K3sImage
available, err := checkImageAvailableLocally(ctx, cli, imageName)
if err != nil {
return fail(
nameK3sImage,
fmt.Sprintf("Cannot check k3s image %s: %v", imageName, err),
"Check Docker daemon status",
"Try: docker images | grep k3s",
)
}
if available {
return pass(nameK3sImage, fmt.Sprintf("K3s image %s is available locally", imageName))
}
err = ensureImageAvailable(ctx, cli, imageName, false)
if err != nil {
return warn(
nameK3sImage,
fmt.Sprintf("K3s image %s not available locally and could not pull: %v", imageName, err),
"Only TestK8sOperator needs this image; other tests are unaffected",
"Try: docker pull "+imageName,
)
}
return pass(nameK3sImage, fmt.Sprintf("K3s image %s is now available", imageName))
}
// checkGoInstallation verifies Go is installed and working.
func checkGoInstallation(ctx context.Context) DoctorResult {
_, err := exec.LookPath("go")
+1 -1
View File
@@ -196,7 +196,7 @@ require (
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.1 // indirect
github.com/opencontainers/runc v1.3.2 // indirect
github.com/opencontainers/runc v1.3.6 // indirect
github.com/pelletier/go-toml/v2 v2.3.1 // indirect
github.com/peterbourgon/ff/v3 v3.4.0 // indirect
github.com/petermattis/goid v0.0.0-20260330135022-df67b199bc81 // indirect
+2 -2
View File
@@ -368,8 +368,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
github.com/opencontainers/runc v1.3.2 h1:GUwgo0Fx9M/pl2utaSYlJfdBcXAB/CZXDxe322lvJ3Y=
github.com/opencontainers/runc v1.3.2/go.mod h1:F7UQQEsxcjUNnFpT1qPLHZBKYP7yWwk6hq8suLy9cl0=
github.com/opencontainers/runc v1.3.6 h1:SLGIymCtsk80iNPWgbc8dtjI30r+5mTVV+4dN8/17Sk=
github.com/opencontainers/runc v1.3.6/go.mod h1:o1wyv76EDlTkcf0KTFgN8bMWLPvgF/HfX709lDv+rr4=
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
github.com/ory/dockertest/v3 v3.12.0 h1:3oV9d0sDzlSQfHtIaB5k6ghUCVMVLpAY8hwrqoCyRCw=
github.com/ory/dockertest/v3 v3.12.0/go.mod h1:aKNDTva3cp8dwOWwb9cWuX84aH5akkxXRvO7KCwWVjE=
-5
View File
@@ -460,11 +460,6 @@ func (h *Headscale) createRouter(apiV1Mux, apiV2Mux http.Handler) *chi.Mux {
r.Use(middleware.Recoverer)
r.Use(securityHeaders)
// TS2021 accepts both the native client's HTTP POST upgrade and the
// browser/WASM client's WebSocket GET upgrade; NoiseUpgradeHandler
// dispatches on the Upgrade header, not the method. Registering GET as
// well keeps the router from rejecting the WebSocket handshake with 405.
r.Get(ts2021UpgradePath, h.NoiseUpgradeHandler)
r.Post(ts2021UpgradePath, h.NoiseUpgradeHandler)
r.Get("/robots.txt", h.RobotsHandler)
+2 -4
View File
@@ -5,10 +5,10 @@ package capver
import (
"maps"
"slices"
"sort"
"strings"
"tailscale.com/tailcfg"
"tailscale.com/util/cmpver"
"tailscale.com/util/set"
)
@@ -74,9 +74,7 @@ func TailscaleLatestMajorMinor(n int, stripV bool) []string {
}
majorSl := majors.Slice()
// cmpver orders versions numerically, so v1.100 sorts after v1.98 rather than
// lexically before it.
slices.SortFunc(majorSl, cmpver.Compare)
sort.Strings(majorSl)
if n > len(majorSl) {
return majorSl
+2 -24
View File
@@ -9,8 +9,6 @@ import (
"github.com/juanfont/headscale/hscontrol/policy"
policyv2 "github.com/juanfont/headscale/hscontrol/policy/v2"
"github.com/juanfont/headscale/hscontrol/types"
"github.com/juanfont/headscale/hscontrol/util/zlog/zf"
"github.com/rs/zerolog/log"
"tailscale.com/tailcfg"
"tailscale.com/types/views"
"tailscale.com/util/multierr"
@@ -154,15 +152,7 @@ func (b *MapResponseBuilder) WithSSHPolicy() *MapResponseBuilder {
sshPolicy, err := b.mapper.state.SSHPolicy(node)
if err != nil {
// SSH policy is optional for a node to function. Rather than fail the
// whole map (leaving the node unable to connect), log and continue
// without it; the node still receives a usable netmap.
log.Warn().Caller().
Err(err).
Uint64(zf.NodeID, node.ID().Uint64()).
Str(zf.NodeHostname, node.Hostname()).
Msg("building map response: skipping SSH policy for node; node will receive a map without SSH rules")
b.addError(err)
return b
}
@@ -294,19 +284,7 @@ func (b *MapResponseBuilder) buildTailPeers(peers views.Slice[types.NodeView]) (
return b.mapper.state.RoutesForPeer(node, peer, matchers)
}, b.mapper.cfg, allCapMaps[peer.ID()])
if err != nil {
// One peer with invalid data (e.g. an empty or over-long
// GivenName that fails GetFQDN) must not blank out the map for
// every node that can see it. Drop the offending peer, log it
// with the identity an operator needs to fix it, and keep
// building from the remaining valid peers.
log.Warn().Caller().
Err(err).
Uint64(zf.NodeID, peer.ID().Uint64()).
Str(zf.NodeHostname, peer.Hostname()).
Uint64("map.viewer.node.id", b.nodeID.Uint64()).
Msgf("dropping peer %d from map response: invalid node data; fix with `headscale nodes rename %d <name>`", peer.ID(), peer.ID())
continue
return nil, err
}
// [tailcfg.Node.CapMap] on a peer carries the small set of
-189
View File
@@ -1,189 +0,0 @@
//go:build !race
// This is a timing-sensitive performance regression test; the race detector's
// ~10x slowdown makes its wall-clock assertion meaningless, so it is excluded
// from -race builds. The concurrency correctness of the policy lock change it
// guards is covered under -race by TestPolicyManagerConcurrentReads in
// hscontrol/policy/v2.
package mapper
import (
"net/netip"
"runtime"
"slices"
"sync"
"testing"
"time"
"github.com/juanfont/headscale/hscontrol/db"
"github.com/juanfont/headscale/hscontrol/derp"
"github.com/juanfont/headscale/hscontrol/state"
"github.com/juanfont/headscale/hscontrol/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"tailscale.com/tailcfg"
)
// setupStormBatcher builds a real state+batcher with production-default
// NodeStore batching so the reconnect-storm contention is realistic. It mirrors
// setupBatcherWithTestData but lets the test control BatcherWorkers and the
// policy.
func setupStormBatcher(tb testing.TB, nodeCount, workers int, policy string) (*TestData, func()) {
tb.Helper()
tmpDir := tb.TempDir()
prefixV4 := netip.MustParsePrefix("100.64.0.0/10")
prefixV6 := netip.MustParsePrefix("fd7a:115c:a1e0::/48")
cfg := &types.Config{
Database: types.DatabaseConfig{
Type: types.DatabaseSqlite,
Sqlite: types.SqliteConfig{Path: tmpDir + "/headscale_test.db"},
},
PrefixV4: &prefixV4,
PrefixV6: &prefixV6,
IPAllocation: types.IPAllocationStrategySequential,
BaseDomain: "headscale.test",
Policy: types.PolicyConfig{Mode: types.PolicyModeDB},
DERP: types.DERPConfig{
ServerEnabled: false,
DERPMap: &tailcfg.DERPMap{
Regions: map[int]*tailcfg.DERPRegion{999: {RegionID: 999}},
},
},
Tuning: types.Tuning{
BatchChangeDelay: 10 * time.Millisecond,
BatcherWorkers: workers,
// Production defaults: coalesce writes so the storm is not
// exaggerated by an unrealistically small NodeStore batch.
NodeStoreBatchSize: 100,
NodeStoreBatchTimeout: 500 * time.Millisecond,
},
}
database, err := db.NewHeadscaleDatabase(cfg)
require.NoError(tb, err)
users := database.CreateUsersForTest(1, "testuser")
dbNodes := database.CreateRegisteredNodesForTest(users[0], nodeCount, "node")
allNodes := make([]node, 0, nodeCount)
for i := range dbNodes {
allNodes = append(allNodes, node{
n: dbNodes[i],
ch: make(chan *tailcfg.MapResponse, normalBufferSize),
})
}
st, err := state.NewState(cfg)
require.NoError(tb, err)
derpMap, err := derp.GetDERPMap(cfg.DERP)
require.NoError(tb, err)
st.SetDERPMap(derpMap)
_, err = st.SetPolicy([]byte(policy))
require.NoError(tb, err)
batcher := wrapBatcherForTest(NewBatcherAndMapper(cfg, st), st)
batcher.Start()
td := &TestData{
Database: database,
Users: users,
Nodes: allNodes,
State: st,
Config: cfg,
Batcher: batcher,
}
return td, func() {
batcher.Close()
st.Close()
database.Close()
}
}
// TestInitialMapNotStarvedByReconnectStorm reproduces juanfont/headscale#3346.
//
// When every node redials at once (e.g. after a server upgrade restart), each
// connection writes the NodeStore (UpdateNodeFromMapRequest + Connect) and the
// batcher generates its initial map. All of that reads the policy through the
// PolicyManager. Before the fix the PolicyManager guarded every read with a
// single exclusive mutex, so the NodeStore writer's O(n^2) BuildPeerMap and
// every node's FilterForNode serialised against each other. On a per-node
// filter policy (autogroup:self, via, relay grants) each hold is expensive, so
// under the storm time-to-initial-map grew without bound.
//
// On the production server in #3346 this drove the batcher's per-node
// total.duration from ~4s to ~76s; tailscale clients aborted the map POST
// first and reported
//
// PollNetMap: Post ".../machine/map": unexpected EOF
//
// then redialled, feeding the storm so it never converged. An allow-all policy
// does NOT reproduce this — BuildPeerMap is cheap there; the per-node filter
// path is what makes it expensive, matching a real deployment's ACLs.
//
// The fix makes PolicyManager reads take a shared RLock so map generation runs
// concurrently. AddNode blocks until the initial map is generated and handed to
// the node channel, so its wall-clock duration is the time-to-initial-map the
// client experiences. Without the fix this test's slowest node takes ~10s+ at
// this scale (lock-bound, and more workers do not help); with it, generation
// parallelises across workers and stays well within a client's patience.
func TestInitialMapNotStarvedByReconnectStorm(t *testing.T) {
if testing.Short() {
t.Skip("timing-sensitive storm regression; skipped in -short")
}
const (
nodeCount = 300
// A per-node-filter policy: forces BuildPeerMap and FilterForNode onto
// the slow path that recompiles filter rules per node, the same shape
// as a real ACL using autogroup:self / via / relay grants.
perNodeFilterPolicy = `{"acls":[{"action":"accept","src":["autogroup:member"],"dst":["autogroup:self:*"]}]}`
// Deliberately roomy so it passes on CI's few-core runners, where the
// single-writer BuildPeerMap sets the floor (~10s) whatever the reads
// do. It still trips on a hang or a return to the ~76s serialised
// behaviour; the fine-grained concurrency is verified separately by
// TestPolicyManagerConcurrentReads under -race.
maxAcceptableLatency = 30 * time.Second
)
// Use the real available parallelism, as production does.
workers := runtime.NumCPU()
td, cleanup := setupStormBatcher(t, nodeCount, workers, perNodeFilterPolicy)
defer cleanup()
latencies := make([]time.Duration, nodeCount)
var wg sync.WaitGroup
for i := range td.Nodes {
wg.Go(func() {
n := &td.Nodes[i]
start := time.Now()
err := td.Batcher.AddNode(n.n.ID, n.ch, tailcfg.CapabilityVersion(100), nil)
latencies[i] = time.Since(start)
assert.NoError(t, err) //nolint:testifylint // assert (not require) is correct off the test goroutine
})
}
wg.Wait()
slices.Sort(latencies)
p50 := latencies[len(latencies)/2]
p95 := latencies[len(latencies)*95/100]
maxLatency := latencies[len(latencies)-1]
t.Logf("initial-map latency over %d nodes (workers=%d): p50=%s p95=%s max=%s",
nodeCount, workers, p50, p95, maxLatency)
require.Less(t, maxLatency, maxAcceptableLatency,
"slowest initial map took %s: policy reads are serialising instead of running concurrently (issue #3346)", maxLatency)
}
-89
View File
@@ -3,7 +3,6 @@ package mapper
import (
"fmt"
"net/netip"
"strings"
"testing"
"github.com/google/go-cmp/cmp"
@@ -537,94 +536,6 @@ func TestBuildFromChangeVisibilityMatchesFullMap(t *testing.T) {
}
}
// TestFullMapResponseSurvivesPeerWithInvalidName proves a single node with an
// FQDN-invalid GivenName must not break map generation for its peers.
//
// A node whose stored GivenName is empty (ErrNodeHasNoGivenName) or yields an
// FQDN longer than MaxHostnameLength (ErrHostnameTooLong) makes GetFQDN, and
// therefore TailNode, return an error. buildTailPeers used to abort the entire
// peer list on the first such error, so MapResponseBuilder.Build() failed for
// every node that could see the bad peer; on the initial-connection path that
// surfaced as "PollNetMap: ... unexpected EOF" and the "Unable to connect to
// the Tailscale coordination server" health warning. A legacy DB row loads
// verbatim (NewNodeStore reads db.ListNodes() without re-sanitising names), so
// the bad peer persists across restart. The build for an unaffected viewer
// must succeed: the bad peer is dropped, valid peers and self survive.
func TestFullMapResponseSurvivesPeerWithInvalidName(t *testing.T) {
for _, tt := range []struct {
name string
badName string
}{
{"empty given name", ""},
{"over-long fqdn", strings.Repeat("a", types.MaxHostnameLength+1)},
} {
t.Run(tt.name, func(t *testing.T) {
tmp := t.TempDir()
p4 := netip.MustParsePrefix("100.64.0.0/10")
p6 := netip.MustParsePrefix("fd7a:115c:a1e0::/48")
cfg := &types.Config{
Database: types.DatabaseConfig{
Type: types.DatabaseSqlite,
Sqlite: types.SqliteConfig{Path: tmp + "/h.db"},
},
PrefixV4: &p4,
PrefixV6: &p6,
IPAllocation: types.IPAllocationStrategySequential,
BaseDomain: "headscale.test",
Policy: types.PolicyConfig{Mode: types.PolicyModeDB},
DERP: types.DERPConfig{
DERPMap: &tailcfg.DERPMap{
Regions: map[int]*tailcfg.DERPRegion{999: {RegionID: 999}},
},
},
Tuning: types.Tuning{
NodeStoreBatchSize: state.TestBatchSize,
NodeStoreBatchTimeout: state.TestBatchTimeout,
},
}
database, err := db.NewHeadscaleDatabase(cfg)
require.NoError(t, err)
user := database.CreateUserForTest("u1")
n1 := database.CreateRegisteredNodeForTest(user, "n1") // viewer, valid
bad := database.CreateRegisteredNodeForTest(user, "bad") // peer, name corrupted below
good := database.CreateRegisteredNodeForTest(user, "good") // peer, valid control
// Simulate a legacy/corrupt row that v29 loads verbatim.
require.NoError(t, database.DB.
Model(&types.Node{}).
Where("id = ?", bad.ID).
Update("given_name", tt.badName).Error)
require.NoError(t, database.Close())
s, err := state.NewState(cfg)
require.NoError(t, err)
t.Cleanup(func() { _ = s.Close() })
// Allow-all so n1 sees both peers; the bad one must still be dropped.
_, err = s.SetPolicy([]byte(`{"acls":[{"action":"accept","src":["*"],"dst":["*:*"]}]}`))
require.NoError(t, err)
m := &mapper{state: s, cfg: cfg}
capVer := tailcfg.CurrentCapabilityVersion
resp, err := m.fullMapResponse(n1.ID, capVer)
require.NoError(t, err, "n1's map must build despite a peer with an invalid name")
require.NotNil(t, resp)
require.NotNil(t, resp.Node, "n1 must receive its own self node")
peers := map[tailcfg.NodeID]bool{}
for _, p := range resp.Peers {
peers[p.ID] = true
}
assert.False(t, peers[bad.ID.NodeID()], "the peer with an invalid name must be dropped")
assert.True(t, peers[good.ID.NodeID()], "valid peers must remain in the map")
})
}
}
// TestGenerateDNSConfigNilHostinfoNoPanic proves generateDNSConfig does not
// panic when a node's Hostinfo is nil (e.g. a legacy DB row with a NULL
// host_info column). addNextDNSMetadata dereferenced node.Hostinfo().OS()
-65
View File
@@ -459,71 +459,6 @@ func TestSSHActionHandler_RejectsMissingSessionWithoutCheck(t *testing.T) {
"a bogus auth_id with no active check must be rejected, body=%s", rec.Body.String())
}
// TestTS2021Route_AcceptsGETAndPOST reproduces a regression where the
// browser/WASM control client could not connect. Tailscale's JS/WASM control
// client opens /ts2021 as a WebSocket, which is an HTTP GET upgrade; the native
// Go client uses an HTTP POST upgrade. The gorilla->chi router migration
// registered /ts2021 for POST only, so the GET WebSocket handshake was rejected
// with 405 Method Not Allowed by the router before it could reach
// NoiseUpgradeHandler. Both methods must route to the handler.
//
// NoiseUpgradeHandler dispatches on the Upgrade header, not the HTTP method, so
// once the route is reachable the handler handles both upgrade styles. The
// httptest recorder is not an http.Hijacker, so the upgrade itself fails past
// the router (501 for the WebSocket path, 400 for the native path) — the point
// is only that neither is 405, i.e. the router no longer rejects GET early.
func TestTS2021Route_AcceptsGETAndPOST(t *testing.T) {
t.Parallel()
handler := createTestApp(t).HTTPHandler()
tests := []struct {
name string
method string
headers map[string]string
}{
{
name: "websocket_get_from_wasm_client",
method: http.MethodGet,
headers: map[string]string{
"Connection": "Upgrade",
"Upgrade": "websocket",
"Sec-WebSocket-Version": "13",
"Sec-WebSocket-Key": "dGhlIHNhbXBsZSBub25jZQ==",
"Sec-WebSocket-Protocol": "tailscale-control-protocol",
},
},
{
name: "native_post_upgrade",
method: http.MethodPost,
headers: map[string]string{
"Connection": "upgrade",
"Upgrade": "tailscale-control-protocol",
"X-Tailscale-Handshake": "AAAA",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
req := httptest.NewRequestWithContext(context.Background(), tt.method,
"/ts2021?X-Tailscale-Handshake=AAAA", nil)
for k, v := range tt.headers {
req.Header.Set(k, v)
}
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
assert.NotEqual(t, http.StatusMethodNotAllowed, rec.Code,
"%s /ts2021 must reach NoiseUpgradeHandler, not be rejected by the router with 405",
tt.method)
})
}
}
// newSSHActionFollowUpRequest is like newSSHActionRequest but carries the
// auth_id query parameter that marks a follow-up poll.
func newSSHActionFollowUpRequest(t *testing.T, src, dst types.NodeID, authID types.AuthID) *http.Request {
+66 -76
View File
@@ -15,7 +15,6 @@ import (
"github.com/juanfont/headscale/hscontrol/policy/matcher"
"github.com/juanfont/headscale/hscontrol/policy/policyutil"
"github.com/juanfont/headscale/hscontrol/types"
"github.com/puzpuzpuz/xsync/v4"
"github.com/rs/zerolog/log"
"go4.org/netipx"
"tailscale.com/net/tsaddr"
@@ -29,10 +28,7 @@ import (
var ErrInvalidTagOwner = errors.New("tag owner is not an Alias")
type PolicyManager struct {
// RWMutex, not Mutex, so concurrent map generation does not serialise on
// reads. The per-node caches are xsync.Maps so a read can fill them without
// taking the write lock.
mu sync.RWMutex
mu sync.Mutex
pol *Policy
users []types.User
nodes views.Slice[types.NodeView]
@@ -59,7 +55,7 @@ type PolicyManager struct {
viaTargetTags map[Tag]struct{}
// Lazy map of SSH policies
sshPolicyMap *xsync.Map[types.NodeID, *tailcfg.SSHPolicy]
sshPolicyMap map[types.NodeID]*tailcfg.SSHPolicy
// compiledGrants are the grants with sources pre-resolved.
// The single source of truth for filter compilation. Both
@@ -68,12 +64,12 @@ type PolicyManager struct {
userNodeIdx userNodeIndex
// Lazy map of per-node filter rules (reduced, for packet filters)
filterRulesMap *xsync.Map[types.NodeID, []tailcfg.FilterRule]
filterRulesMap map[types.NodeID][]tailcfg.FilterRule
// Lazy map of per-node matchers derived from UNREDUCED filter
// rules. Only populated on the slow path when needsPerNodeFilter
// is true; the fast path returns pm.matchers directly.
matchersForNodeMap *xsync.Map[types.NodeID, []matcher.Match]
matchersForNodeMap map[types.NodeID][]matcher.Match
// needsPerNodeFilter is true when any compiled grant requires
// per-node work (autogroup:self or via grants).
@@ -201,9 +197,9 @@ func NewPolicyManager(b []byte, users []types.User, nodes views.Slice[types.Node
pol: policy,
users: users,
nodes: nodes,
sshPolicyMap: xsync.NewMap[types.NodeID, *tailcfg.SSHPolicy](),
filterRulesMap: xsync.NewMap[types.NodeID, []tailcfg.FilterRule](),
matchersForNodeMap: xsync.NewMap[types.NodeID, []matcher.Match](),
sshPolicyMap: make(map[types.NodeID]*tailcfg.SSHPolicy, nodes.Len()),
filterRulesMap: make(map[types.NodeID][]tailcfg.FilterRule, nodes.Len()),
matchersForNodeMap: make(map[types.NodeID][]matcher.Match, nodes.Len()),
}
_, err = pm.updateLocked()
@@ -358,9 +354,9 @@ func (pm *PolicyManager) updateLocked() (bool, error) {
// TODO(kradalby): This could potentially be optimized by only clearing the
// policies for nodes that have changed. Particularly if the only difference is
// that nodes has been added or removed.
pm.sshPolicyMap.Clear()
pm.filterRulesMap.Clear()
pm.matchersForNodeMap.Clear()
clear(pm.sshPolicyMap)
clear(pm.filterRulesMap)
clear(pm.matchersForNodeMap)
}
// If nothing changed, no need to update nodes
@@ -404,8 +400,8 @@ func (pm *PolicyManager) NodeNeedsPeerRecompute(node types.NodeView) bool {
return true
}
pm.mu.RLock()
defer pm.mu.RUnlock()
pm.mu.Lock()
defer pm.mu.Unlock()
if pm.relayTargetIPs != nil && node.InIPSet(pm.relayTargetIPs) {
return true
@@ -426,10 +422,10 @@ func (pm *PolicyManager) NodeNeedsPeerRecompute(node types.NodeView) bool {
// /machine/ssh/action/{src}/to/{dst}?local_user={local_user} per the
// SaaS wire format. Cache is invalidated on policy reload.
func (pm *PolicyManager) SSHPolicy(baseURL string, node types.NodeView) (*tailcfg.SSHPolicy, error) {
pm.mu.RLock()
defer pm.mu.RUnlock()
pm.mu.Lock()
defer pm.mu.Unlock()
if sshPol, ok := pm.sshPolicyMap.Load(node.ID()); ok {
if sshPol, ok := pm.sshPolicyMap[node.ID()]; ok {
return sshPol, nil
}
@@ -438,7 +434,7 @@ func (pm *PolicyManager) SSHPolicy(baseURL string, node types.NodeView) (*tailcf
return nil, fmt.Errorf("compiling SSH policy: %w", err)
}
pm.sshPolicyMap.Store(node.ID(), sshPol)
pm.sshPolicyMap[node.ID()] = sshPol
return sshPol, nil
}
@@ -454,8 +450,8 @@ func (pm *PolicyManager) SSHPolicy(baseURL string, node types.NodeView) (*tailcf
func (pm *PolicyManager) SSHCheckParams(
srcNodeID, dstNodeID types.NodeID,
) (time.Duration, bool) {
pm.mu.RLock()
defer pm.mu.RUnlock()
pm.mu.Lock()
defer pm.mu.Unlock()
if pm.pol == nil || len(pm.pol.SSHs) == 0 {
return 0, false
@@ -588,8 +584,8 @@ func (pm *PolicyManager) Filter() ([]tailcfg.FilterRule, []matcher.Match) {
return nil, nil
}
pm.mu.RLock()
defer pm.mu.RUnlock()
pm.mu.Lock()
defer pm.mu.Unlock()
return pm.filter, pm.matchers
}
@@ -608,8 +604,8 @@ func (pm *PolicyManager) BuildPeerMap(nodes views.Slice[types.NodeView]) map[typ
return nil
}
pm.mu.RLock()
defer pm.mu.RUnlock()
pm.mu.Lock()
defer pm.mu.Unlock()
// Precompute each node's subnet routes and exit-node status once; the
// O(n^2) pair scans below would otherwise recompute them for every pair.
@@ -730,7 +726,7 @@ func (pm *PolicyManager) filterForNodeLocked(
return nil
}
if rules, ok := pm.filterRulesMap.Load(node.ID()); ok {
if rules, ok := pm.filterRulesMap[node.ID()]; ok {
return rules
}
@@ -742,7 +738,7 @@ func (pm *PolicyManager) filterForNodeLocked(
}
reduced := policyutil.ReduceFilterRules(node, unreduced)
pm.filterRulesMap.Store(node.ID(), reduced)
pm.filterRulesMap[node.ID()] = reduced
return reduced
}
@@ -759,8 +755,8 @@ func (pm *PolicyManager) FilterForNode(node types.NodeView) ([]tailcfg.FilterRul
return nil, nil
}
pm.mu.RLock()
defer pm.mu.RUnlock()
pm.mu.Lock()
defer pm.mu.Unlock()
return pm.filterForNodeLocked(node), nil
}
@@ -780,8 +776,8 @@ func (pm *PolicyManager) MatchersForNode(node types.NodeView) ([]matcher.Match,
return nil, nil
}
pm.mu.RLock()
defer pm.mu.RUnlock()
pm.mu.Lock()
defer pm.mu.Unlock()
// For global policies, return the shared global matchers.
// Via grants require per-node matchers because the global matchers
@@ -790,7 +786,7 @@ func (pm *PolicyManager) MatchersForNode(node types.NodeView) ([]matcher.Match,
return pm.matchers, nil
}
if cached, ok := pm.matchersForNodeMap.Load(node.ID()); ok {
if cached, ok := pm.matchersForNodeMap[node.ID()]; ok {
return cached, nil
}
@@ -798,7 +794,7 @@ func (pm *PolicyManager) MatchersForNode(node types.NodeView) ([]matcher.Match,
// the stored compiled grants for this specific node.
unreduced := pm.filterRulesForNodeLocked(node)
matchers := matcher.MatchesFromFilterRules(unreduced)
pm.matchersForNodeMap.Store(node.ID(), matchers)
pm.matchersForNodeMap[node.ID()] = matchers
return matchers, nil
}
@@ -817,7 +813,7 @@ func (pm *PolicyManager) SetUsers(users []types.User) (bool, error) {
// Clear SSH policy map when users change to force SSH policy recomputation
// This ensures that if SSH policy compilation previously failed due to missing users,
// it will be retried with the new user list
pm.sshPolicyMap.Clear()
clear(pm.sshPolicyMap)
changed, err := pm.updateLocked()
if err != nil {
@@ -870,9 +866,9 @@ func (pm *PolicyManager) SetNodes(nodes views.Slice[types.NodeView]) (bool, erro
if !needsUpdate {
// This ensures fresh filter rules are generated for all nodes
pm.sshPolicyMap.Clear()
pm.filterRulesMap.Clear()
pm.matchersForNodeMap.Clear()
clear(pm.sshPolicyMap)
clear(pm.filterRulesMap)
clear(pm.matchersForNodeMap)
}
// Always return true when nodes changed, even if filter hash didn't change
// (can happen with autogroup:self or when nodes are added but don't affect rules)
@@ -933,8 +929,8 @@ func (pm *PolicyManager) NodeCanHaveTag(node types.NodeView, tag string) bool {
return false
}
pm.mu.RLock()
defer pm.mu.RUnlock()
pm.mu.Lock()
defer pm.mu.Unlock()
// pm.pol is written by SetPolicy under pm.mu; reading it before the
// lock races with concurrent policy reloads.
@@ -995,8 +991,8 @@ func (pm *PolicyManager) TagOwnedByTags(tag string, ownerTags []string) bool {
return true
}
pm.mu.RLock()
defer pm.mu.RUnlock()
pm.mu.Lock()
defer pm.mu.Unlock()
// Owned-by delegation requires the policy's tagOwners.
if pm.pol == nil {
@@ -1082,8 +1078,8 @@ func (pm *PolicyManager) TagExists(tag string) bool {
return false
}
pm.mu.RLock()
defer pm.mu.RUnlock()
pm.mu.Lock()
defer pm.mu.Unlock()
// pm.pol is written by SetPolicy under pm.mu; reading it before the
// lock races with concurrent policy reloads.
@@ -1101,8 +1097,8 @@ func (pm *PolicyManager) NodeCanApproveRoute(node types.NodeView, route netip.Pr
return false
}
pm.mu.RLock()
defer pm.mu.RUnlock()
pm.mu.Lock()
defer pm.mu.Unlock()
// If the route to-be-approved is an exit route, then we need to check
// if the node is in allowed to approve it. This is treated differently
@@ -1164,8 +1160,8 @@ func (pm *PolicyManager) ViaRoutesForPeer(viewer, peer types.NodeView) types.Via
return result
}
pm.mu.RLock()
defer pm.mu.RUnlock()
pm.mu.Lock()
defer pm.mu.Unlock()
// pm.pol is written by SetPolicy under pm.mu; reading it before the
// lock races with concurrent policy reloads.
@@ -1383,8 +1379,8 @@ func (pm *PolicyManager) DebugString() string {
// pm.pol, filter, matchers, and the derived maps are all written
// under pm.mu by SetPolicy/SetUsers/SetNodes.
pm.mu.RLock()
defer pm.mu.RUnlock()
pm.mu.Lock()
defer pm.mu.Unlock()
var sb strings.Builder
@@ -1529,7 +1525,7 @@ func (pm *PolicyManager) invalidateAutogroupSelfCache(oldNodes, newNodes views.S
// Clear cache entries for affected users only.
// For autogroup:self, we need to clear all nodes belonging to affected users
// because autogroup:self rules depend on the entire user's device set.
pm.filterRulesMap.Range(func(nodeID types.NodeID, _ []tailcfg.FilterRule) bool {
for nodeID := range pm.filterRulesMap {
// Find the user for this cached node using the already-built indexes.
node, ok := newNodeMap[nodeID]
if !ok {
@@ -1538,10 +1534,10 @@ func (pm *PolicyManager) invalidateAutogroupSelfCache(oldNodes, newNodes views.S
// Node not found in either old or new list, clear it.
if !ok {
pm.filterRulesMap.Delete(nodeID)
pm.matchersForNodeMap.Delete(nodeID)
delete(pm.filterRulesMap, nodeID)
delete(pm.matchersForNodeMap, nodeID)
return true
continue
}
// Tagged nodes don't participate in autogroup:self, so their cache
@@ -1553,17 +1549,15 @@ func (pm *PolicyManager) invalidateAutogroupSelfCache(oldNodes, newNodes views.S
// If the owning user is affected, clear this cache entry.
if _, affected := affectedUsers[nodeUserID]; affected {
pm.filterRulesMap.Delete(nodeID)
pm.matchersForNodeMap.Delete(nodeID)
delete(pm.filterRulesMap, nodeID)
delete(pm.matchersForNodeMap, nodeID)
}
return true
})
}
if len(affectedUsers) > 0 {
log.Debug().
Int("affected_users", len(affectedUsers)).
Int("remaining_cache_entries", pm.filterRulesMap.Size()).
Int("remaining_cache_entries", len(pm.filterRulesMap)).
Msg("Selectively cleared autogroup:self cache for affected users")
}
}
@@ -1597,27 +1591,23 @@ func (pm *PolicyManager) invalidateGlobalPolicyCache(newNodes views.Slice[types.
}
if newNode.HasNetworkChanges(oldNode) {
pm.filterRulesMap.Delete(nodeID)
pm.matchersForNodeMap.Delete(nodeID)
delete(pm.filterRulesMap, nodeID)
delete(pm.matchersForNodeMap, nodeID)
}
}
// Remove deleted nodes from cache
pm.filterRulesMap.Range(func(nodeID types.NodeID, _ []tailcfg.FilterRule) bool {
for nodeID := range pm.filterRulesMap {
if _, exists := newNodeMap[nodeID]; !exists {
pm.filterRulesMap.Delete(nodeID)
delete(pm.filterRulesMap, nodeID)
}
}
return true
})
pm.matchersForNodeMap.Range(func(nodeID types.NodeID, _ []matcher.Match) bool {
for nodeID := range pm.matchersForNodeMap {
if _, exists := newNodeMap[nodeID]; !exists {
pm.matchersForNodeMap.Delete(nodeID)
delete(pm.matchersForNodeMap, nodeID)
}
return true
})
}
}
// flattenTags resolves nested tag-owner references. Cycles
@@ -1799,8 +1789,8 @@ func (pm *PolicyManager) NodeCapMap(id types.NodeID) tailcfg.NodeCapMap {
return nil
}
pm.mu.RLock()
defer pm.mu.RUnlock()
pm.mu.Lock()
defer pm.mu.Unlock()
src := pm.nodeAttrsMap[id]
if len(src) == 0 {
@@ -1823,8 +1813,8 @@ func (pm *PolicyManager) NodeCapMaps() map[types.NodeID]tailcfg.NodeCapMap {
return nil
}
pm.mu.RLock()
defer pm.mu.RUnlock()
pm.mu.Lock()
defer pm.mu.Unlock()
out := make(map[types.NodeID]tailcfg.NodeCapMap, len(pm.nodeAttrsMap))
maps.Copy(out, pm.nodeAttrsMap)
@@ -1,103 +0,0 @@
package v2
import (
"fmt"
"sync"
"testing"
"github.com/juanfont/headscale/hscontrol/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/gorm"
)
// TestPolicyManagerConcurrentReads is the correctness guard for the #3346 fix:
// PolicyManager read methods take a shared RLock and populate their per-node
// caches (filterRulesMap, matchersForNodeMap) concurrently. This test hammers
// those reads from many goroutines while a writer mutates the node set, so the
// race detector catches any unsafe access to the shared caches or policy state.
//
// It uses an autogroup:self policy so reads take the per-node filter slow path
// — the same path that made #3346's reconnect storm expensive — which is where
// the lazy caches are written.
func TestPolicyManagerConcurrentReads(t *testing.T) {
users := types.Users{
{Model: gorm.Model{ID: 1}, Name: "user1", Email: "user1@headscale.net"},
{Model: gorm.Model{ID: 2}, Name: "user2", Email: "user2@headscale.net"},
{Model: gorm.Model{ID: 3}, Name: "user3", Email: "user3@headscale.net"},
}
policy := `{
"acls": [
{
"action": "accept",
"src": ["autogroup:member"],
"dst": ["autogroup:self:*"]
}
]
}`
const nodeCount = 60
nodes := make(types.Nodes, 0, nodeCount)
for i := range nodeCount {
n := node(
fmt.Sprintf("node%d", i),
fmt.Sprintf("100.64.0.%d", i+1),
fmt.Sprintf("fd7a:115c:a1e0::%d", i+1),
users[i%len(users)],
)
n.ID = types.NodeID(i + 1) //nolint:gosec // safe in test
nodes = append(nodes, n)
}
pm, err := NewPolicyManager([]byte(policy), users, nodes.ViewSlice())
require.NoError(t, err)
const (
readers = 16
iterations = 60
mutatorReloads = 30
)
var wg sync.WaitGroup
// Concurrent readers exercise every converted RLock read path, including
// the two lazily populated per-node caches. Assertions inside the
// goroutines use assert (not require) so a failure does not call
// t.FailNow from a non-test goroutine.
for r := range readers {
wg.Go(func() {
for i := range iterations {
nv := nodes[(r+i)%len(nodes)].View()
rules, err := pm.FilterForNode(nv)
assert.NoError(t, err) //nolint:testifylint // assert (not require) is correct off the test goroutine
assert.NotNil(t, rules)
_, err = pm.MatchersForNode(nv)
assert.NoError(t, err) //nolint:testifylint // assert (not require) is correct off the test goroutine
pm.Filter()
pm.NodeCapMap(nv.ID())
// BuildPeerMap is the O(n^2) writer-side read; exercise it
// under RLock too, but not every iteration.
if i%8 == 0 {
assert.NotNil(t, pm.BuildPeerMap(nodes.ViewSlice()))
}
}
})
}
// A writer repeatedly re-sets the node set, invalidating and racing the
// caches the readers are populating.
wg.Go(func() {
for range mutatorReloads {
_, err := pm.SetNodes(nodes.ViewSlice())
assert.NoError(t, err) //nolint:testifylint // assert (not require) is correct off the test goroutine
}
})
wg.Wait()
}
+7 -13
View File
@@ -8,7 +8,6 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/juanfont/headscale/hscontrol/policy/matcher"
"github.com/juanfont/headscale/hscontrol/types"
"github.com/puzpuzpuz/xsync/v4"
"github.com/stretchr/testify/require"
"gorm.io/gorm"
"tailscale.com/net/tsaddr"
@@ -123,7 +122,7 @@ func TestInvalidateAutogroupSelfCache(t *testing.T) {
require.NoError(t, err)
}
require.Equal(t, len(initialNodes), pm.filterRulesMap.Size())
require.Len(t, pm.filterRulesMap, len(initialNodes))
tests := []struct {
name string
@@ -208,20 +207,19 @@ func TestInvalidateAutogroupSelfCache(t *testing.T) {
}
}
pm.filterRulesMap.Clear()
pm.filterRulesMap = make(map[types.NodeID][]tailcfg.FilterRule)
for _, n := range initialNodes {
_, err := pm.FilterForNode(n.View())
require.NoError(t, err)
}
initialCacheSize := pm.filterRulesMap.Size()
initialCacheSize := len(pm.filterRulesMap)
require.Equal(t, len(initialNodes), initialCacheSize)
pm.invalidateAutogroupSelfCache(initialNodes.ViewSlice(), tt.newNodes.ViewSlice())
// Verify the expected number of cache entries were cleared
finalCacheSize := pm.filterRulesMap.Size()
finalCacheSize := len(pm.filterRulesMap)
clearedEntries := initialCacheSize - finalCacheSize
require.Equal(t, tt.expectedCleared, clearedEntries, tt.description)
})
@@ -500,19 +498,15 @@ func TestInvalidateGlobalPolicyCache(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
pm := &PolicyManager{
nodes: tt.oldNodes.ViewSlice(),
filterRulesMap: xsync.NewMap[types.NodeID, []tailcfg.FilterRule](),
matchersForNodeMap: xsync.NewMap[types.NodeID, []matcher.Match](),
}
for id, rules := range tt.initialCache {
pm.filterRulesMap.Store(id, rules)
nodes: tt.oldNodes.ViewSlice(),
filterRulesMap: tt.initialCache,
}
pm.invalidateGlobalPolicyCache(tt.newNodes.ViewSlice())
// Verify cache state
for nodeID, shouldExist := range tt.expectedCacheAfter {
_, exists := pm.filterRulesMap.Load(nodeID)
_, exists := pm.filterRulesMap[nodeID]
require.Equal(t, shouldExist, exists, "node %d cache existence mismatch", nodeID)
}
})
+2 -2
View File
@@ -122,8 +122,8 @@ func (pm *PolicyManager) RunSSHTests() error {
return nil
}
pm.mu.RLock()
defer pm.mu.RUnlock()
pm.mu.Lock()
defer pm.mu.Unlock()
cache := make(map[types.NodeID]*tailcfg.SSHPolicy)
results := runSSHPolicyTests(pm.pol, pm.users, pm.nodes, cache)
+2 -2
View File
@@ -215,8 +215,8 @@ func (pm *PolicyManager) RunTests() error {
return nil
}
pm.mu.RLock()
defer pm.mu.RUnlock()
pm.mu.Lock()
defer pm.mu.Unlock()
results := runPolicyTests(pm.pol, pm.filter, pm.users, pm.nodes)
+1 -1
View File
@@ -8,7 +8,7 @@ import (
"tailscale.com/tailcfg"
)
// TestSplitDestinationAndPort tests the splitDestinationAndPort function using table-driven tests.
// TestParseDestinationAndPort tests the splitDestinationAndPort function using table-driven tests.
func TestSplitDestinationAndPort(t *testing.T) {
testCases := []struct {
input string
-10
View File
@@ -230,11 +230,6 @@ func (m *mapSession) serveLongPoll() {
mapReqChange, err := m.h.state.UpdateNodeFromMapRequest(m.node.ID, m.req)
if err != nil {
m.log.Error().Caller().Err(err).Msg("failed to update node from initial MapRequest")
// Write an explicit error rather than returning silently: a bare
// return leaves net/http to send an empty 200, which the client
// reads as "unexpected EOF" and retries forever (issue #3346).
httpError(m.w, err)
return
}
@@ -257,11 +252,6 @@ func (m *mapSession) serveLongPoll() {
// time between the node connecting and the batcher being ready.
if err := m.h.mapBatcher.AddNode(m.node.ID, m.ch, m.capVer, m.stopFromBatcher); err != nil { //nolint:noinlineerr
m.log.Error().Caller().Err(err).Msg("failed to add node to batcher")
// Write an explicit error rather than returning silently: a bare
// return leaves net/http to send an empty 200, which the client
// reads as "unexpected EOF" and retries forever (issue #3346).
httpError(m.w, err)
return
}
-127
View File
@@ -7,10 +7,8 @@ import (
"testing"
"time"
"github.com/juanfont/headscale/hscontrol/db"
"github.com/juanfont/headscale/hscontrol/mapper"
"github.com/juanfont/headscale/hscontrol/state"
"github.com/juanfont/headscale/hscontrol/types"
"github.com/juanfont/headscale/hscontrol/types/change"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -91,131 +89,6 @@ func (w *delayedSuccessResponseWriter) WriteCount() int {
return w.writeCount
}
// recordingResponseWriter records the status code and whether anything was
// written, so a test can tell an explicit error response apart from a handler
// that returned without writing (which net/http turns into an empty 200 the
// client reads as "unexpected EOF").
type recordingResponseWriter struct {
mu sync.Mutex
header http.Header
status int
writes int
}
func (w *recordingResponseWriter) Header() http.Header {
if w.header == nil {
w.header = make(http.Header)
}
return w.header
}
func (w *recordingResponseWriter) WriteHeader(code int) {
w.mu.Lock()
defer w.mu.Unlock()
if w.status == 0 {
w.status = code
}
}
func (w *recordingResponseWriter) Write(data []byte) (int, error) {
w.mu.Lock()
defer w.mu.Unlock()
if w.status == 0 {
w.status = http.StatusOK
}
w.writes++
return len(data), nil
}
func (w *recordingResponseWriter) Flush() {}
func (w *recordingResponseWriter) statusCode() int {
w.mu.Lock()
defer w.mu.Unlock()
return w.status
}
// TestServeLongPollWritesErrorWhenInitialMapFails proves that when the initial
// map cannot be generated (here: the node's own GivenName is invalid, so
// WithSelfNode fails and AddNode errors), serveLongPoll writes an explicit HTTP
// error instead of returning with no body. Returning empty leaves net/http to
// send an empty 200, which the Tailscale client reports as
// "PollNetMap: ... unexpected EOF" and retries forever (issue #3346).
func TestServeLongPollWritesErrorWhenInitialMapFails(t *testing.T) {
app := createTestApp(t)
user := app.state.CreateUserForTest("self-bad-name-user")
createdNode := app.state.CreateRegisteredNodeForTest(user, "self-bad-name-node")
// Corrupt the node's stored name to empty so GetFQDN fails for itself,
// then reload state so the bad row enters the NodeStore verbatim.
app.mapBatcher.Close()
require.NoError(t, app.state.Close())
database, err := db.NewHeadscaleDatabase(app.cfg)
require.NoError(t, err)
require.NoError(t, database.DB.
Model(&types.Node{}).
Where("id = ?", createdNode.ID).
Update("given_name", "").Error)
require.NoError(t, database.Close())
app.state, err = state.NewState(app.cfg)
require.NoError(t, err)
app.mapBatcher = mapper.NewBatcherAndMapper(app.cfg, app.state)
app.mapBatcher.Start()
t.Cleanup(func() {
app.mapBatcher.Close()
require.NoError(t, app.state.Close())
})
nodeView, ok := app.state.GetNodeByID(createdNode.ID)
require.True(t, ok)
node := nodeView.AsStruct()
ctx, cancel := context.WithCancel(context.Background())
writer := &recordingResponseWriter{}
session := app.newMapSession(ctx, tailcfg.MapRequest{
Stream: true,
Version: tailcfg.CapabilityVersion(100),
}, writer, node)
serveDone := make(chan struct{})
go func() {
session.serveLongPoll()
close(serveDone)
}()
t.Cleanup(func() {
// Break the post-disconnect reconnect wait so the goroutine exits.
dummyCh := make(chan *tailcfg.MapResponse, 1)
_ = app.mapBatcher.AddNode(node.ID, dummyCh, tailcfg.CapabilityVersion(100), nil)
cancel()
select {
case <-serveDone:
case <-time.After(2 * time.Second):
}
_ = app.mapBatcher.RemoveNode(node.ID, dummyCh)
})
assert.Eventually(t, func() bool {
return writer.statusCode() >= http.StatusInternalServerError
}, 2*time.Second, 10*time.Millisecond,
"serveLongPoll must write an HTTP error response when the initial map cannot be built, not an empty 200")
}
// TestGitHubIssue3129_TransientlyBlockedWriteDoesNotLeaveLiveStaleSession
// tests the scenario reported in
// https://github.com/juanfont/headscale/issues/3129.
-92
View File
@@ -1,92 +0,0 @@
package state
import (
"fmt"
"github.com/juanfont/headscale/hscontrol/types"
"github.com/juanfont/headscale/hscontrol/util/zlog/zf"
"github.com/rs/zerolog/log"
)
// nodeHealthCheck names a class of stored-node-data defect that breaks normal
// operation and explains how to fix it. ok == true means the node passes the
// check. This is the extension point for node-data validation: add a check
// here as new corrupt-data classes surface (nil hostinfo, invalid IPs,
// tags-XOR-user violations, ...) and both the boot scan and any future caller
// run the whole set.
type nodeHealthCheck struct {
name string
check func(nv types.NodeView, cfg *types.Config) (problem, fixHint string, ok bool)
}
// nodeHealthChecks is the registry of node-data health checks. Today it carries
// the one issue #3346 needs; append to it rather than reshaping callers.
var nodeHealthChecks = []nodeHealthCheck{givenNameMapsToValidFQDN}
// givenNameMapsToValidFQDN flags a node whose stored GivenName cannot produce a
// valid FQDN (empty, or longer than MaxHostnameLength once base_domain is
// applied). Such a node cannot be rendered into a netmap — neither its own nor
// any peer's — so it must be renamed to recover.
var givenNameMapsToValidFQDN = nodeHealthCheck{
name: "given-name-maps-to-valid-fqdn",
check: func(nv types.NodeView, cfg *types.Config) (string, string, bool) {
err := types.ValidateGivenName(nv.GivenName(), cfg.BaseDomain)
if err != nil {
return err.Error(), fmt.Sprintf("headscale nodes rename %d <name>", nv.ID()), false
}
return "", "", true
},
}
// nodeHealthFinding is a single failed check for a single node.
type nodeHealthFinding struct {
nodeID types.NodeID
hostname string
check string
problem string
fixHint string
}
// scanNodeHealth runs every registered check against every node in the store
// and returns one finding per failure. It only reports — it never mutates a
// node — so an operator can repair the underlying data without the server
// silently rewriting a user-visible name.
func (s *State) scanNodeHealth() []nodeHealthFinding {
var findings []nodeHealthFinding
for _, nv := range s.nodeStore.ListNodes().All() {
for _, c := range nodeHealthChecks {
problem, fixHint, ok := c.check(nv, s.cfg)
if ok {
continue
}
findings = append(findings, nodeHealthFinding{
nodeID: nv.ID(),
hostname: nv.Hostname(),
check: c.name,
problem: problem,
fixHint: fixHint,
})
}
}
return findings
}
// logNodeHealth scans the store once and logs an actionable warning per
// finding. Called at startup so an operator learns — by node id and fix
// command — about stored data that will break map generation, without the
// server changing anything itself.
func (s *State) logNodeHealth() {
for _, f := range s.scanNodeHealth() {
log.Warn().
Uint64(zf.NodeID, f.nodeID.Uint64()).
Str(zf.NodeHostname, f.hostname).
Str("check", f.check).
Str("problem", f.problem).
Str("fix", f.fixHint).
Msg("node has invalid data that breaks map generation; rename it to restore connectivity")
}
}
-67
View File
@@ -1,67 +0,0 @@
package state
import (
"testing"
"github.com/juanfont/headscale/hscontrol/db"
"github.com/juanfont/headscale/hscontrol/types"
"github.com/stretchr/testify/require"
)
func TestGivenNameMapsToValidFQDNCheck(t *testing.T) {
cfg := &types.Config{BaseDomain: "example.com"}
_, _, ok := givenNameMapsToValidFQDN.check((&types.Node{ID: 1, GivenName: "valid"}).View(), cfg)
require.True(t, ok, "a valid given name must pass the check")
problem, fixHint, ok := givenNameMapsToValidFQDN.check((&types.Node{ID: 7, GivenName: ""}).View(), cfg)
require.False(t, ok, "an empty given name must fail the check")
require.NotEmpty(t, problem)
require.Contains(t, fixHint, "rename 7", "fix hint must name the offending node")
}
// TestScanNodeHealthReportsInvalidNameWithoutMutating proves the boot scan
// reports a node whose stored name would break map generation (issue #3346)
// with an actionable fix, and that it never rewrites the stored name — the
// maintainer's decision is log-only, no silent mutation.
func TestScanNodeHealthReportsInvalidNameWithoutMutating(t *testing.T) {
dbPath := t.TempDir() + "/headscale.db"
cfg := persistTestConfig(dbPath)
database, err := db.NewHeadscaleDatabase(cfg)
require.NoError(t, err)
user := database.CreateUserForTest("scan-user")
bad := database.CreateRegisteredNodeForTest(user, "scan-bad")
good := database.CreateRegisteredNodeForTest(user, "scan-good")
require.NoError(t, database.DB.
Model(&types.Node{}).
Where("id = ?", bad.ID).
Update("given_name", "").Error)
require.NoError(t, database.Close())
s, err := NewState(cfg)
require.NoError(t, err)
t.Cleanup(func() { _ = s.Close() })
findings := s.scanNodeHealth()
var badFinding *nodeHealthFinding
for i := range findings {
require.NotEqual(t, good.ID, findings[i].nodeID, "a valid node must not be reported")
if findings[i].nodeID == bad.ID {
badFinding = &findings[i]
}
}
require.NotNil(t, badFinding, "a node with an invalid name must be reported")
require.Contains(t, badFinding.fixHint, "rename", "finding must carry an actionable fix")
// Log-only: neither the scan nor boot may rewrite the stored name.
nv, ok := s.GetNodeByID(bad.ID)
require.True(t, ok)
require.Empty(t, nv.GivenName(), "boot scan must not mutate the stored name")
}
-40
View File
@@ -1,40 +0,0 @@
package state
import (
"strings"
"testing"
"github.com/juanfont/headscale/hscontrol/db"
"github.com/stretchr/testify/require"
)
// TestRenameNodeRejectsNameExceedingFQDNLimit proves RenameNode rejects a name
// that is a valid DNS label but whose FQDN, under the configured base_domain,
// exceeds MaxHostnameLength. Without the FQDN-length gate such a name persists
// and then breaks map generation for the node and its peers (issue #3346):
// admin-facing writes must not be able to introduce an unmappable name.
func TestRenameNodeRejectsNameExceedingFQDNLimit(t *testing.T) {
dbPath := t.TempDir() + "/headscale.db"
cfg := persistTestConfig(dbPath)
// A long base domain so a 63-char label overflows the 255-char FQDN bound.
cfg.BaseDomain = strings.Repeat("b", 200) + ".example.com"
database, err := db.NewHeadscaleDatabase(cfg)
require.NoError(t, err)
user := database.CreateUserForTest("rename-user")
node := database.CreateRegisteredNodeForTest(user, "rename-node")
require.NoError(t, database.Close())
s, err := NewState(cfg)
require.NoError(t, err)
t.Cleanup(func() { _ = s.Close() })
// Valid 63-char DNS label, but the resulting FQDN exceeds 255 chars.
_, _, err = s.RenameNode(node.ID, strings.Repeat("a", 63))
require.Error(t, err, "rename to a name whose FQDN exceeds the limit must be rejected")
// A short, valid name is still accepted.
_, _, err = s.RenameNode(node.ID, "short")
require.NoError(t, err)
}
+3 -13
View File
@@ -277,7 +277,7 @@ func NewState(cfg *types.Config) (*State, error) {
)
nodeStore.Start()
s := &State{
return &State{
cfg: cfg,
db: db,
@@ -289,14 +289,7 @@ func NewState(cfg *types.Config) (*State, error) {
sshCheckAuth: make(map[sshCheckPair]time.Time),
registerLocks: xsync.NewMap[key.MachinePublic, *sync.Mutex](),
}
// Surface nodes whose stored data would break map generation (e.g. an
// invalid given name from a legacy row) so an operator can fix them. This
// only logs; it never mutates a node's stored name at boot.
s.logNodeHealth()
return s, nil
}, nil
}
// Close gracefully shuts down the [State] instance and releases all resources.
@@ -1039,10 +1032,7 @@ func (s *State) SetApprovedRoutes(nodeID types.NodeID, routes []netip.Prefix) (t
// auto-sanitisation) and collisions error out rather than silently
// bumping a user-facing label. See HOSTNAME.md for the CLI contract.
func (s *State) RenameNode(nodeID types.NodeID, newName string) (types.NodeView, change.Change, error) {
// Validate the label AND that the resulting FQDN fits MaxHostnameLength:
// a valid 63-char label can still overflow under a long base_domain, and
// an unmappable name would break this node and its peers (issue #3346).
err := types.ValidateGivenName(newName, s.cfg.BaseDomain)
err := dnsname.ValidLabel(newName)
if err != nil {
return types.NodeView{}, change.Change{}, fmt.Errorf("%w: %w", ErrGivenNameInvalid, err)
}
-23
View File
@@ -19,7 +19,6 @@ import (
"tailscale.com/tailcfg"
"tailscale.com/types/key"
"tailscale.com/types/views"
"tailscale.com/util/dnsname"
)
var (
@@ -518,28 +517,6 @@ func (node *Node) GetFQDN(baseDomain string) (string, error) {
return hostname, nil
}
// ValidateGivenName reports whether givenName is usable as a node's DNS label:
// a valid DNS label that, combined with baseDomain, yields an FQDN within
// MaxHostnameLength. Admin-facing write paths (e.g. node rename) reject names
// that fail this, since the mapper cannot build a map for a node — or any of
// its peers — whose GetFQDN fails. Derived paths sanitise/coerce instead.
func ValidateGivenName(givenName, baseDomain string) error {
err := dnsname.ValidLabel(givenName)
if err != nil {
return fmt.Errorf("%q is not a valid DNS label: %w", givenName, err)
}
// Reuse GetFQDN so the length bound stays identical to what the mapper
// enforces; a valid 63-char label can still overflow under a long
// base_domain.
_, err = (&Node{GivenName: givenName}).GetFQDN(baseDomain)
if err != nil {
return err
}
return nil
}
// AnnouncedRoutes returns the list of routes the node announces, as
// reported by the client in [tailcfg.Hostinfo.RoutableIPs]. Announcement alone
// does not grant visibility — see [Node.SubnetRoutes] for approval-gated
-27
View File
@@ -417,33 +417,6 @@ func TestNodeFQDN(t *testing.T) {
}
}
func TestValidateGivenName(t *testing.T) {
tests := []struct {
name string
givenName string
baseDomain string
wantErr bool
}{
{"valid", "test", "example.com", false},
{"empty", "", "example.com", true},
{"invalid label chars", "not valid", "example.com", true},
{"label too long", strings.Repeat("a", 64), "example.com", true},
// A valid 63-char label whose FQDN overflows only because the base
// domain is long: ValidLabel passes, the FQDN-length bound rejects it.
{"fqdn too long under long base domain", strings.Repeat("a", 63), strings.Repeat("b", 200) + ".example.com", true},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
err := ValidateGivenName(tc.givenName, tc.baseDomain)
if (err != nil) != tc.wantErr {
t.Errorf("ValidateGivenName(%q, %q) error = %v, wantErr %v",
tc.givenName, tc.baseDomain, err, tc.wantErr)
}
})
}
}
func TestPeerChangeFromMapRequest(t *testing.T) {
nKeys := []key.NodePublic{
key.NewNode().Public(),
-3
View File
@@ -1,7 +1,6 @@
package integration
import (
"context"
"net/netip"
clientv1 "github.com/juanfont/headscale/gen/client/v1"
@@ -23,8 +22,6 @@ type ControlServer interface {
ConnectToNetwork(network *dockertest.Network) error
GetHealthEndpoint() string
GetEndpoint() string
GetIPEndpoint() string
CreateOAuthClient(ctx context.Context, scopes, tags []string) (string, string, error)
WaitForRunning() error
Restart() error
CreateUser(user string) (*clientv1.User, error)
-86
View File
@@ -6,7 +6,6 @@ import (
"cmp"
"context"
"crypto/tls"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
@@ -26,7 +25,6 @@ import (
"github.com/davecgh/go-spew/spew"
clientv1 "github.com/juanfont/headscale/gen/client/v1"
clientv2 "github.com/juanfont/headscale/gen/client/v2"
"github.com/juanfont/headscale/hscontrol"
policyv2 "github.com/juanfont/headscale/hscontrol/policy/v2"
"github.com/juanfont/headscale/hscontrol/types"
@@ -1025,90 +1023,6 @@ func (t *HeadscaleInContainer) GetEndpoint() string {
return t.getEndpoint(false)
}
var errOAuthSecretMissing = errors.New(`OAuth client response missing secret in "key" field`)
// CreateOAuthClient mints an admin API key and uses it to create an OAuth client
// via the v2 keys HTTP API (POST /api/v2/tailnet/-/keys, keyType=client),
// returning the client id and secret. The secret is only returned once, in the
// "key" field. It is a reusable building block for tests that need OAuth client
// credentials (such as the Kubernetes operator).
func (t *HeadscaleInContainer) CreateOAuthClient(
ctx context.Context,
scopes, tags []string,
) (string, string, error) {
apiKey, err := t.Execute([]string{"headscale", "apikeys", "create", "--expiration", "24h"})
if err != nil {
return "", "", fmt.Errorf("creating admin api key: %w", err)
}
apiKey = strings.TrimSpace(apiKey)
client, err := clientv2.NewClientWithResponses(
t.GetEndpoint(),
clientv2.WithHTTPClient(t.httpClient()),
clientv2.WithRequestEditorFn(func(_ context.Context, req *http.Request) error {
req.Header.Set("Authorization", "Bearer "+apiKey)
return nil
}),
)
if err != nil {
return "", "", fmt.Errorf("building v2 API client: %w", err)
}
keyType := "client"
resp, err := client.CreateKeyWithResponse(ctx, "-", clientv2.CreateKeyRequest{
KeyType: &keyType,
Scopes: &scopes,
Tags: &tags,
})
if err != nil {
return "", "", fmt.Errorf("creating OAuth client: %w", err)
}
if resp.JSON200 == nil {
return "", "", fmt.Errorf( //nolint:err113
"creating OAuth client: status %s: %s", resp.Status(), strings.TrimSpace(string(resp.Body)))
}
if resp.JSON200.Key == nil || *resp.JSON200.Key == "" {
return "", "", errOAuthSecretMissing
}
// The operator expects clientId and clientSecret as separate values. When the
// server returns a single opaque credential, the client-credentials grant
// splits it on "-" (id-secret), matching the Tailscale SaaS shape. Fall back
// to the whole key as the secret when no id is given.
clientID, clientSecret := resp.JSON200.Id, *resp.JSON200.Key
if clientID == "" {
if id, secret, ok := strings.Cut(*resp.JSON200.Key, "-"); ok {
clientID, clientSecret = id, secret
}
}
return clientID, clientSecret, nil
}
// httpClient returns an HTTP client that trusts this Headscale's TLS CA when TLS
// is enabled, or a default client when it serves plain HTTP.
func (t *HeadscaleInContainer) httpClient() *http.Client {
if !t.hasTLS() {
return &http.Client{Timeout: 30 * time.Second}
}
pool := x509.NewCertPool()
pool.AppendCertsFromPEM(t.tlsCACert)
return &http.Client{
Timeout: 30 * time.Second,
Transport: &http.Transport{
TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12},
},
}
}
// GetIPEndpoint returns the Headscale endpoint using IP address instead of hostname.
func (t *HeadscaleInContainer) GetIPEndpoint() string {
return t.getEndpoint(true)
-603
View File
@@ -1,603 +0,0 @@
// Package k3sic wraps a single-container k3s cluster (server + agent) as a
// privileged sibling container on the host docker daemon, like the
// DERP-in-container wrapper in the dsic package.
//
// It exists so that integration tests can install the real Tailscale
// Kubernetes operator (via its Helm chart) into a real Kubernetes cluster and
// point it at an in-test Headscale. k3s bundles kubectl and we run helm inside
// the container through Execute, so the host dev shell needs no kube tooling.
//
// The operator is pointed at Headscale over plain HTTP (see
// hsic.WithoutTLS), so the operator and proxy pods need no CA: there is no
// image baking and no CoreDNS hostname mapping. To run instead against a TLS
// Headscale with a private CA, see tls-ca-baking.md.
//
// The harness runs as sibling containers on the host docker daemon (the
// test-suite container has the host docker socket bind-mounted); it is NOT
// docker-in-docker. We therefore run the purpose-built single-container k3s image
// as one more privileged sibling joined to the scenario networks, rather than using
// k3d/kind which shell out to the docker daemon and fight the sibling model.
// Privileged is the harness-wide norm here, not a k3s-specific escalation: the
// tsic (client) and dsic (DERP) containers run privileged too (see
// dockertestutil.DockerAllowNetworkAdministration).
package k3sic
import (
"archive/tar"
"compress/gzip"
"context"
"errors"
"fmt"
"io"
"log"
"net/http"
"runtime"
"strings"
"time"
"github.com/juanfont/headscale/hscontrol/capver"
"github.com/juanfont/headscale/integration/dockertestutil"
"github.com/juanfont/headscale/integration/integrationutil"
"github.com/ory/dockertest/v3"
"github.com/ory/dockertest/v3/docker"
"tailscale.com/util/rands"
)
const (
k3sicHashLength = 6
// K3sImage is the single-container k3s server+agent image, pinned on ghcr (no
// anonymous Docker Hub rate limit). Pinned rather than resolved from the k3s
// stable channel because that channel floats the k8s minor unpredictably and
// would outrun the cgroup-v2 and br_netfilter workarounds below. Bump by hand.
K3sImage = "ghcr.io/k3s-io/k3s:v1.35.5-k3s1"
// operatorImageRepo and proxyImageRepo are the ghcr-hosted operator and
// proxy images. ghcr is used instead of Docker Hub to avoid anonymous pull
// rate limits; the pods pull these directly.
operatorImageRepo = "ghcr.io/tailscale/k8s-operator"
proxyImageRepo = "ghcr.io/tailscale/tailscale"
dockerExecuteTimeout = 300 * time.Second
// helmVersionFallback is used when the latest helm release cannot be
// resolved at runtime (see resolveHelmVersion). The image ships no helm and
// cannot fetch it itself, so we inject a binary that matches the container
// arch.
helmVersionFallback = "v3.19.1"
// kubeconfigPath is where k3s writes the kubeconfig (see RunOptions.Env);
// helm needs it pointed explicitly, kubectl finds it by default.
kubeconfigPath = "/etc/rancher/k3s/k3s.yaml"
// kubectlBin is the in-container kubectl the k3s image ships on PATH.
kubectlBin = "kubectl"
// shellBin is the in-container shell used for compound commands.
shellBin = "/bin/sh"
// tailscaleNamespace is where the operator and its proxies are installed.
tailscaleNamespace = "tailscale"
// kubeSystemNamespace holds CoreDNS and the rest of the k3s system addons.
kubeSystemNamespace = "kube-system"
)
var (
errHelmDownload = errors.New("helm download failed")
errHelmNotInTarball = errors.New("helm binary not found in release tarball")
errNoKubeDNSEndpoints = errors.New("kube-dns Service has no ready endpoints yet")
)
// OperatorImageTag is the image tag the operator and proxy images use, derived
// from the Tailscale minor Headscale tracks via capver (e.g. "v1.98"). The
// operator shares the Tailscale release train, so this keeps the images and the
// Helm chart in lockstep with the client versions Headscale is tested against,
// without a hand-pinned constant. The tag is a rolling tag within the minor.
func OperatorImageTag() string {
return capver.TailscaleLatestMajorMinor(1, false)[0]
}
// OperatorImage and ProxyImage are the ghcr operator/proxy image references the
// test wires into the Helm chart.
func OperatorImage() string { return operatorImageRepo + ":" + OperatorImageTag() }
func ProxyImage() string { return proxyImageRepo + ":" + OperatorImageTag() }
// OperatorChartVersion is the Helm chart version constraint matching the derived
// minor; helm resolves the latest patch in that line. The top-level loginServer
// value the operator needs to target Headscale instead of the Tailscale SaaS
// first shipped in chart 1.98.4.
func OperatorChartVersion() string {
return strings.TrimPrefix(OperatorImageTag(), "v") + ".*"
}
// K3sInContainer represents a k3s cluster running in a single privileged
// container (K3sInContainer, hence k3sic).
type K3sInContainer struct {
hostname string
pool *dockertest.Pool
container *dockertest.Resource
networks []*dockertest.Network
}
// New starts a new [K3sInContainer] joined to the given networks.
func New(
pool *dockertest.Pool,
networks []*dockertest.Network,
) (*K3sInContainer, error) {
hash := rands.HexString(k3sicHashLength)
// Include the run ID in the hostname for easier identification of which
// test run owns this container, matching the dsic/tsic convention.
runID := dockertestutil.GetIntegrationRunID()
var hostname string
if runID != "" {
runIDShort := runID[len(runID)-6:]
hostname = fmt.Sprintf("k3s-%s-%s", runIDShort, hash)
} else {
hostname = "k3s-" + hash
}
k := &K3sInContainer{
hostname: hostname,
pool: pool,
networks: networks,
}
// Pull the k3s image (ghcr, not built) via PullWithAuth, as hsic does for
// prebuilt/pulled images.
err := dockertestutil.PullWithAuth(pool, K3sImage)
if err != nil {
return nil, fmt.Errorf("pulling %s: %w", K3sImage, err)
}
repo, tag, ok := strings.Cut(K3sImage, ":")
if !ok {
return nil, fmt.Errorf("invalid k3s image reference %q", K3sImage) //nolint:err113
}
runOptions := &dockertest.RunOptions{
Name: hostname,
Repository: repo,
Tag: tag,
Networks: networks,
// "server" runs both the control plane and a built-in agent in one
// container. --disable traefik/servicelb/metrics-server keeps the
// cluster lean: the test only needs the API server and the ability to
// schedule the operator pods. --tls-san pins the hostname into the
// apiserver cert (not strictly needed since we exec kubectl in-container,
// but harmless and future-proof).
Cmd: []string{
"server",
"--disable", "traefik",
"--disable", "servicelb",
"--disable", "metrics-server",
"--disable-network-policy",
"--snapshotter", "native",
"--tls-san", hostname,
},
Env: []string{
"K3S_KUBECONFIG_OUTPUT=" + kubeconfigPath,
"K3S_KUBECONFIG_MODE=0644",
},
}
// Stamp the run-id label or the reaper leaks the container.
dockertestutil.DockerAddIntegrationLabels(runOptions, "k3s")
// dockertest does not handle pre-existing containers well; make sure a
// stale one with this name is gone first.
err = pool.RemoveContainerByName(hostname)
if err != nil {
return nil, err
}
container, err := pool.RunWithOptions(
runOptions,
dockertestutil.DockerRestartPolicy,
// Privileged + NET_ADMIN: k3s manages iptables/ipvs, mounts cgroups and
// runs containerd. This is the same knob dsic uses for the DERP server.
dockertestutil.DockerAllowNetworkAdministration,
withK3sHostConfig,
)
if err != nil {
return nil, fmt.Errorf("%s starting k3s container: %w", hostname, err)
}
log.Printf("Created %s container\n", hostname)
k.container = container
// Make ClusterIP DNAT work before k3s programs kube-proxy rules; without it
// in-cluster DNS times out on hosts where br_netfilter is not preloaded.
k.ensureBridgeNetfilter()
return k, nil
}
// withK3sHostConfig sets the HostConfig knobs k3s needs beyond
// privileged/NET_ADMIN: a tmpfs on /run and /var/run, which the k3s image
// expects.
//
// It deliberately does NOT bind-mount the host /sys/fs/cgroup. On a cgroup-v2
// host, bind-mounting the host cgroup tree into a container that keeps its own
// (private) cgroup namespace makes the cgroup root visible to the container
// disagree with the namespace runc places workload pods under. The kubelet can
// then start (the apiserver and node go Ready), but every workload pod fails to
// create its sandbox with "failed to apply cgroup configuration: ...
// cgroup.procs: no such file or directory", so nothing the operator schedules
// ever runs. Leaving the bind-mount off lets the privileged k3s entrypoint set
// up cgroup-v2 delegation within its own namespace, which it is
// designed to do, and workload pods schedule normally.
func withK3sHostConfig(config *docker.HostConfig) {
config.Tmpfs = map[string]string{
"/run": "",
"/var/run": "",
}
// Bind the host kernel modules read-only so the container can load
// br_netfilter (see ensureBridgeNetfilter). Without bridge netfilter,
// kube-proxy's ClusterIP DNAT rules do not apply to bridged pod-to-pod
// traffic, so kube-dns (and every other Service) is unreachable from pods —
// the in-cluster DNS timeout seen on the arm64 CI runner. The container
// shares the host kernel, so the modules match.
config.Binds = append(config.Binds, "/lib/modules:/lib/modules:ro")
}
// ensureBridgeNetfilter loads br_netfilter and enables the sysctls that make
// kube-proxy's ClusterIP DNAT apply to bridged pod-to-pod traffic. On a host
// where the module is already loaded (e.g. the amd64 dev box, where Docker
// loads it for bridge networks) these are no-ops; on the arm64 CI runner the
// module is absent and pods cannot reach any Service IP — kube-dns included —
// so in-cluster DNS times out. Best-effort: k3s also loads the module, and a
// genuinely missing module surfaces in DumpDiagnostics rather than here.
func (k *K3sInContainer) ensureBridgeNetfilter() {
for _, cmd := range []string{
"modprobe br_netfilter || true",
"sysctl -w net.bridge.bridge-nf-call-iptables=1 || true",
"sysctl -w net.bridge.bridge-nf-call-ip6tables=1 || true",
"sysctl -w net.ipv4.ip_forward=1 || true",
} {
out, stderr, err := k.Execute([]string{shellBin, "-c", cmd})
if err != nil {
log.Printf("[k3s] %q failed: %v (stdout: %s, stderr: %s)", cmd, err, out, stderr)
}
}
}
// Hostname returns the hostname of the [K3sInContainer].
func (k *K3sInContainer) Hostname() string {
return k.hostname
}
// ID returns the docker container ID of the [K3sInContainer].
func (k *K3sInContainer) ID() string {
return k.container.Container.ID
}
// ConnectToNetwork connects the cluster container to an additional network.
func (k *K3sInContainer) ConnectToNetwork(network *dockertest.Network) error {
return k.container.ConnectToNetwork(network)
}
// Execute runs a command inside the k3s container and returns its stdout.
// kubectl and the k3s-bundled tools (and helm, once installed via
// [K3sInContainer.InstallHelm]) are on PATH. KUBECONFIG is exported so helm,
// which (unlike the image's kubectl) does not default to the k3s config, can
// reach the cluster.
func (k *K3sInContainer) Execute(command []string) (string, string, error) {
return dockertestutil.ExecuteCommand(
k.container,
command,
[]string{"KUBECONFIG=" + kubeconfigPath},
dockertestutil.ExecuteCommandTimeout(dockerExecuteTimeout),
)
}
// WriteFile saves a file inside the container.
func (k *K3sInContainer) WriteFile(path string, data []byte) error {
return integrationutil.WriteFileToContainer(k.pool, k.container, path, data)
}
// WaitForRunning blocks until the cluster is ready to schedule DNS-dependent
// workloads: the kube-apiserver is serving, the single node reports Ready, and
// in-cluster DNS is servable (CoreDNS rolled out with a backed kube-dns
// Service). Gating on DNS here pins a missing-DNS failure at its source rather
// than letting the operator crashloop on an opaque lookup timeout.
func (k *K3sInContainer) WaitForRunning() error {
log.Printf("waiting for k3s API server in %s to be ready", k.hostname)
err := k.pool.Retry(func() error {
// `kubectl get --raw=/readyz` returns "ok" once the apiserver is up.
out, _, err := k.Execute([]string{
kubectlBin, "get", "--raw=/readyz",
})
if err != nil {
return fmt.Errorf("k3s apiserver not ready: %w", err)
}
if !strings.Contains(out, "ok") {
return fmt.Errorf("k3s apiserver readyz returned %q", strings.TrimSpace(out)) //nolint:err113
}
// Wait for the node object to exist and be Ready before returning so
// pods can actually be scheduled.
nodeOut, _, err := k.Execute([]string{
kubectlBin, "get", "nodes", "--no-headers",
})
if err != nil {
return fmt.Errorf("k3s node not ready: %w", err)
}
if !strings.Contains(nodeOut, " Ready") {
return fmt.Errorf("k3s node not Ready yet: %q", strings.TrimSpace(nodeOut)) //nolint:err113
}
return nil
})
if err != nil {
return err
}
return k.waitForClusterDNS()
}
// InstallHelm installs the helm binary into the container so the operator can be
// installed. The k3s image ships kubectl but not helm, has no curl, and its
// busybox wget cannot do HTTPS, so we download helm in the test process (which
// has network egress) and inject the binary. helm's own HTTPS client then
// fetches the operator chart from inside the container.
func (k *K3sInContainer) InstallHelm() error {
bin, err := fetchHelmBinary(resolveHelmVersion(), runtime.GOARCH)
if err != nil {
return fmt.Errorf("fetching helm: %w", err)
}
err = k.WriteFile("/usr/local/bin/helm", bin)
if err != nil {
return fmt.Errorf("writing helm binary: %w", err)
}
// WriteFile uploads with mode 0; make it readable+executable.
_, stderr, err := k.Execute([]string{"chmod", "0755", "/usr/local/bin/helm"})
if err != nil {
return fmt.Errorf("chmod helm (stderr: %s): %w", stderr, err)
}
return nil
}
// waitForClusterDNS blocks until CoreDNS is rolled out and the kube-dns Service
// has at least one ready endpoint — i.e. in-cluster name resolution is actually
// servable, which every workload (starting with the operator) depends on. k3s
// deploys CoreDNS via its addon manager shortly after the node reports Ready, so
// the deployment may not exist yet; retry until it does before checking rollout.
func (k *K3sInContainer) waitForClusterDNS() error {
err := k.pool.Retry(func() error {
_, stderr, err := k.Execute([]string{
kubectlBin, "-n", kubeSystemNamespace, "get", "deployment", "coredns",
})
if err != nil {
return fmt.Errorf("coredns deployment not present yet (stderr: %s): %w", stderr, err)
}
return nil
})
if err != nil {
return fmt.Errorf("waiting for coredns deployment to appear: %w", err)
}
_, stderr, err := k.Execute([]string{
kubectlBin, "-n", kubeSystemNamespace, "rollout", "status",
"deployment/coredns", "--timeout=150s",
})
if err != nil {
return fmt.Errorf("coredns did not become available (stderr: %s): %w", stderr, err)
}
return k.pool.Retry(func() error {
// A populated endpoint set means a CoreDNS pod is serving :53 and
// kube-proxy has a backend to DNAT the kube-dns ClusterIP to; empty means
// in-cluster lookups will time out no matter how long a client waits.
out, stderr, err := k.Execute([]string{
kubectlBin, "-n", kubeSystemNamespace, "get", "endpoints", "kube-dns",
"-o", "jsonpath={.subsets[*].addresses[*].ip}",
})
if err != nil {
return fmt.Errorf("reading kube-dns endpoints (stderr: %s): %w", stderr, err)
}
if strings.TrimSpace(out) == "" {
return errNoKubeDNSEndpoints
}
return nil
})
}
// ConfigureCoreDNSHost makes in-cluster pods resolve hostname to ip via CoreDNS.
// The operator targets Headscale's control plane by IP, but the embedded DERP map
// references Headscale by hostname; without this, the proxy pods cannot resolve
// the DERP server, never connect to it, and — since they only advertise
// unreachable pod-network endpoints — get no data path to nodes outside the
// cluster. It installs a coredns-custom ConfigMap (a k3s-native extension point:
// keys ending in .server become additional server blocks), which CoreDNS's reload
// plugin picks up without a restart.
func (k *K3sInContainer) ConfigureCoreDNSHost(hostname, ip string) error {
manifest := fmt.Sprintf(`apiVersion: v1
kind: ConfigMap
metadata:
name: coredns-custom
namespace: kube-system
data:
headscale.server: |
%s {
hosts {
%s %s
fallthrough
}
}
`, hostname, ip, hostname)
return k.ApplyManifest("coredns-custom", manifest)
}
// DumpDiagnostics logs cluster state useful for debugging a failed operator
// install: pod status across namespaces and the operator's own logs and events.
// Best-effort — every command's failure is logged, not returned.
func (k *K3sInContainer) DumpDiagnostics() {
for _, c := range [][]string{
{kubectlBin, "get", "pods", "-A", "-o", "wide"},
{kubectlBin, "-n", tailscaleNamespace, "get", "events", "--sort-by=.lastTimestamp"},
{kubectlBin, "-n", tailscaleNamespace, "describe", "pods"},
{kubectlBin, "-n", tailscaleNamespace, "logs", "deployment/operator", "--tail=200"},
// A crashlooping operator's fatal error is in the previous container.
{kubectlBin, "-n", tailscaleNamespace, "logs", "deployment/operator", "--previous", "--tail=200"},
{kubectlBin, "-n", tailscaleNamespace, "get", "statefulsets,pods", "-o", "wide"},
// Proxy pods' tailscaled logs: DERP-connection and registration failures
// (the data-path culprits) surface here, not in the operator log.
{
shellBin, "-c",
"for p in $(kubectl -n " + tailscaleNamespace + " get pods -o name | grep /ts-); do " +
"echo \"== $p ==\"; kubectl -n " + tailscaleNamespace +
" logs $p -c tailscale --tail=80 2>&1; done",
},
// In-cluster DNS: the operator's first dependency. A "lookup
// kubernetes.default.svc ... i/o timeout" crash means CoreDNS is not
// serving, so capture its pod state, logs (Corefile parse errors land
// here), and the Service endpoints.
{kubectlBin, "-n", kubeSystemNamespace, "get", "pods", "-l", "k8s-app=kube-dns", "-o", "wide"},
{kubectlBin, "-n", kubeSystemNamespace, "logs", "-l", "k8s-app=kube-dns", "--tail=100"},
{kubectlBin, "-n", kubeSystemNamespace, "get", "endpoints", "kube-dns", "-o", "wide"},
// Host network state behind a ClusterIP-unreachable DNS timeout: whether
// br_netfilter is loaded and the call-iptables/forward sysctls are on, and
// whether kube-proxy actually programmed the kube-dns DNAT rule. If CoreDNS
// is healthy (above) but these are missing, the fault is the Service DNAT
// path, not DNS.
{shellBin, "-c", "lsmod | grep -E 'br_netfilter|nf_conntrack' || echo 'br_netfilter NOT loaded'"},
{shellBin, "-c", "sysctl net.bridge.bridge-nf-call-iptables net.ipv4.ip_forward 2>&1 || true"},
{shellBin, "-c", "iptables-save -t nat 2>/dev/null | grep -iE 'KUBE-SERVICES|kube-dns|10.43.0.10' | head -40 || echo 'no kube-dns nat rules'"},
} {
out, stderr, err := k.Execute(c)
label := strings.Join(c, " ")
if err != nil {
log.Printf("[k3s diag] %s failed: %v (stderr: %s)", label, err, stderr)
continue
}
log.Printf("[k3s diag] %s:\n%s", label, out)
}
}
// resolveHelmVersion returns the latest published helm release tag (e.g.
// "v3.19.1"), falling back to [helmVersionFallback] if it cannot be resolved.
// get.helm.sh is not Docker Hub and has no anonymous rate limit, so a "rolling"
// latest is cheap; the fallback keeps a broken release from breaking CI.
func resolveHelmVersion() string {
req, err := http.NewRequestWithContext(
context.Background(), http.MethodGet, "https://get.helm.sh/helm-latest-version", nil)
if err != nil {
return helmVersionFallback
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return helmVersionFallback
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return helmVersionFallback
}
body, err := io.ReadAll(io.LimitReader(resp.Body, 32))
if err != nil {
return helmVersionFallback
}
version := strings.TrimSpace(string(body))
if !strings.HasPrefix(version, "v") {
return helmVersionFallback
}
return version
}
// fetchHelmBinary downloads the helm release tarball for version and goarch and
// returns the helm binary bytes.
func fetchHelmBinary(version, goarch string) ([]byte, error) {
url := fmt.Sprintf("https://get.helm.sh/helm-%s-linux-%s.tar.gz", version, goarch)
req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil)
if err != nil {
return nil, err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("%w: %s returned status %d", errHelmDownload, url, resp.StatusCode)
}
gz, err := gzip.NewReader(resp.Body)
if err != nil {
return nil, err
}
defer gz.Close()
want := "linux-" + goarch + "/helm"
tr := tar.NewReader(gz)
for {
hdr, err := tr.Next()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return nil, err
}
if hdr.Name == want {
return io.ReadAll(tr)
}
}
return nil, fmt.Errorf("%w: %s", errHelmNotInTarball, want)
}
// Shutdown saves the container log and then runs k3s-killall in-container so
// k3s's own child processes/containers (containerd-shims, pods) do not leak,
// before purging the container itself.
func (k *K3sInContainer) Shutdown() error {
err := k.SaveLog("/tmp/control")
if err != nil {
log.Printf("saving log from %s: %s", k.hostname, err)
}
// k3s spawns containerd and a tree of child processes inside this
// container; the bundled k3s-killall.sh tears them down. Best-effort: the
// Purge below removes the container regardless.
_, _, err = k.Execute([]string{shellBin, "-c", "k3s-killall.sh || true"})
if err != nil {
log.Printf("running k3s-killall in %s: %s", k.hostname, err)
}
return k.pool.Purge(k.container)
}
// SaveLog saves the container stdout/stderr logs to a path on the host.
func (k *K3sInContainer) SaveLog(path string) error {
_, _, err := dockertestutil.SaveLog(k.pool, k.container, path)
return err
}
-243
View File
@@ -1,243 +0,0 @@
package k3sic
import (
"fmt"
"strings"
)
// This file holds the reusable building blocks for driving the Tailscale
// Kubernetes operator inside the cluster: installing it and applying the CRs and
// workloads a test needs. Tests compose these methods rather than embedding
// kubectl/helm invocations, so adding a new operator test is a few method calls.
// ApplyManifest writes manifest into the container as /tmp/<name>.yaml and
// kubectl-applies it. It is the building block the helpers below use, and is
// exported so tests can apply ad-hoc manifests without a bespoke method.
func (k *K3sInContainer) ApplyManifest(name, manifest string) error {
path := "/tmp/" + name + ".yaml"
err := k.WriteFile(path, []byte(manifest))
if err != nil {
return fmt.Errorf("writing manifest %s: %w", name, err)
}
_, stderr, err := k.Execute([]string{kubectlBin, "apply", "-f", path})
if err != nil {
return fmt.Errorf("applying manifest %s (stderr: %s): %w", name, stderr, err)
}
return nil
}
// InstallOperator installs the Tailscale Kubernetes operator via Helm into the
// tailscale namespace, pointed at loginServer with the given OAuth client
// credentials. loginServer is used by the operator for both the control plane
// and the management API; for an in-test Headscale pass its HTTP endpoint by IP
// (hsic.HeadscaleInContainer.GetIPEndpoint) so the pods need no DNS or CA. The
// operator and proxy images come from ghcr at the capver-derived tag. Blocks
// (helm --wait) until the operator deployment is available.
func (k *K3sInContainer) InstallOperator(loginServer, clientID, clientSecret string) error {
repoAdd := "helm repo add tailscale https://pkgs.tailscale.com/helmcharts && helm repo update"
_, stderr, err := k.Execute([]string{shellBin, "-c", repoAdd})
if err != nil {
return fmt.Errorf("helm repo add/update (stderr: %s): %w", stderr, err)
}
_, stderr, err = k.Execute([]string{
kubectlBin, "create", "namespace", tailscaleNamespace,
})
if err != nil {
return fmt.Errorf("creating %s namespace (stderr: %s): %w", tailscaleNamespace, stderr, err)
}
// Precreate the operator-oauth Secret with --from-literal instead of the
// chart's oauth.clientId/clientSecret: the chart interpolates those unquoted,
// so an all-digit credential renders as a YAML number and the apiserver
// rejects it. --from-literal always stores strings. The chart uses a Secret
// named operator-oauth when oauth.clientId is unset.
_, stderr, err = k.Execute([]string{
kubectlBin, "-n", tailscaleNamespace, "create", "secret", "generic", "operator-oauth",
"--from-literal=client_id=" + clientID,
"--from-literal=client_secret=" + clientSecret,
})
if err != nil {
return fmt.Errorf("creating operator-oauth secret (stderr: %s): %w", stderr, err)
}
opRepo, opTag, _ := strings.Cut(OperatorImage(), ":")
proxyRepo, proxyTag, _ := strings.Cut(ProxyImage(), ":")
const set = "--set-string"
install := []string{
"helm", "upgrade", "--install", "tailscale-operator",
"tailscale/tailscale-operator",
"--version", OperatorChartVersion(),
"--namespace", tailscaleNamespace,
set, "loginServer=" + loginServer,
set, "operatorConfig.image.repository=" + opRepo,
set, "operatorConfig.image.tag=" + opTag,
set, "proxyConfig.image.repository=" + proxyRepo,
set, "proxyConfig.image.tag=" + proxyTag,
"--wait", "--timeout", "5m",
}
_, stderr, err = k.Execute(install)
if err != nil {
k.DumpDiagnostics()
return fmt.Errorf("helm install operator (stderr: %s): %w", stderr, err)
}
// hsic serves the embedded DERP without TLS, but a proxy dials DERP over HTTPS
// and so cannot relay through it. Pods on the k3s pod network can only reach
// off-cluster nodes via DERP (their only endpoint is an unreachable pod IP),
// so without this the ingress/egress proxies get no data path. The ProxyClass
// injects TS_DEBUG_DERP_WS_CLIENT + TS_DEBUG_USE_DERP_HTTP, switching proxies to
// plain-HTTP websocket DERP. The proxy-creating helpers below reference it.
return k.applyDERPWebsocketProxyClass()
}
// DERPWebsocketProxyClass is the ProxyClass [InstallOperator] creates to make
// operator proxies reach the embedded (non-TLS) DERP over websocket. Proxy
// resources reference it via spec.proxyClass / the tailscale.com/proxy-class
// annotation.
const DERPWebsocketProxyClass = "headscale-derp-ws" //nolint:gosec // G101 false positive: a ProxyClass name, not a credential
func (k *K3sInContainer) applyDERPWebsocketProxyClass() error {
manifest := fmt.Sprintf(`apiVersion: tailscale.com/v1alpha1
kind: ProxyClass
metadata:
name: %s
spec:
statefulSet:
pod:
tailscaleContainer:
env:
- name: TS_DEBUG_DERP_WS_CLIENT
value: "true"
- name: TS_DEBUG_USE_DERP_HTTP
value: "true"
`, DERPWebsocketProxyClass)
return k.ApplyManifest("proxyclass-"+DERPWebsocketProxyClass, manifest)
}
// DeployConnector applies a Connector CR advertising an egress subnet router for
// advertiseRoutes, tagged with tags. The operator provisions a proxy and
// registers it as a node in Headscale.
func (k *K3sInContainer) DeployConnector(name string, tags, advertiseRoutes []string) error {
manifest := fmt.Sprintf(`apiVersion: tailscale.com/v1alpha1
kind: Connector
metadata:
name: %s
spec:
proxyClass: %s
tags:
%s
subnetRouter:
advertiseRoutes:
%s
`, name, DERPWebsocketProxyClass, yamlList(tags, 4), yamlList(advertiseRoutes, 6))
return k.ApplyManifest("connector-"+name, manifest)
}
// DeployProxyGroup applies a ProxyGroup CR of the given type ("ingress" or
// "egress") with replicas proxies tagged with tags. ProxyGroups are the current
// way to run a pool of operator proxies for HA ingress/egress.
func (k *K3sInContainer) DeployProxyGroup(name, proxyType string, replicas int, tags []string) error {
manifest := fmt.Sprintf(`apiVersion: tailscale.com/v1alpha1
kind: ProxyGroup
metadata:
name: %s
spec:
type: %s
replicas: %d
proxyClass: %s
tags:
%s
`, name, proxyType, replicas, DERPWebsocketProxyClass, yamlList(tags, 4))
return k.ApplyManifest("proxygroup-"+name, manifest)
}
// DeployEchoServer deploys a minimal HTTP server (agnhost, served from
// registry.k8s.io to avoid Docker Hub rate limits) labelled app=<name> with a
// ClusterIP Service of the same name on port 80. Use it as the in-cluster target
// for connectivity tests; expose it to the tailnet with [ExposeServiceToTailnet].
func (k *K3sInContainer) DeployEchoServer(name string) error {
manifest := fmt.Sprintf(`apiVersion: apps/v1
kind: Deployment
metadata:
name: %s
spec:
replicas: 1
selector:
matchLabels:
app: %s
template:
metadata:
labels:
app: %s
spec:
containers:
- name: echo
image: registry.k8s.io/e2e-test-images/agnhost:2.47
args: ["netexec", "--http-port=80"]
ports:
- containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
name: %s
spec:
selector:
app: %s
ports:
- port: 80
targetPort: 80
`, name, name, name, name, name)
return k.ApplyManifest("echo-"+name, manifest)
}
// ExposeServiceToTailnet creates a tailscale LoadBalancer Service named
// "<name>-ts" that exposes the pods labelled app=<name> to the tailnet, tagged
// with tags. The operator provisions an ingress proxy and registers a node, so a
// node outside the cluster (a regular tsic client) can reach the service over
// the tailnet.
func (k *K3sInContainer) ExposeServiceToTailnet(name string, tags []string) error {
manifest := fmt.Sprintf(`apiVersion: v1
kind: Service
metadata:
name: %s-ts
annotations:
tailscale.com/tags: "%s"
tailscale.com/proxy-class: %s
spec:
type: LoadBalancer
loadBalancerClass: tailscale
selector:
app: %s
ports:
- port: 80
targetPort: 80
`, name, strings.Join(tags, ","), DERPWebsocketProxyClass, name)
return k.ApplyManifest("expose-"+name, manifest)
}
// yamlList renders items as a YAML block sequence indented by indent spaces,
// e.g. " - tag:k8s". Returns "" for an empty list.
func yamlList(items []string, indent int) string {
var b strings.Builder
pad := strings.Repeat(" ", indent)
for _, item := range items {
fmt.Fprintf(&b, "%s- %s\n", pad, item)
}
return strings.TrimRight(b.String(), "\n")
}
-70
View File
@@ -1,70 +0,0 @@
# Running the operator against a private-CA TLS Headscale
`TestK8sOperator` points the Tailscale Kubernetes operator at an **HTTP**
Headscale (`hsic.WithoutTLS`, `loginServer = http://<ip>:<port>`). That keeps
the harness small: the operator and proxy pods need no CA, so there is no image
baking, no containerd import, and no CoreDNS hostname mapping.
This note records how to run the same test against a **TLS** Headscale serving a
private CA, in case a future test needs to exercise realistic TLS. It is not
wired up; reconstruct it from here.
## Why it is involved
tailscaled/tsnet verify the control connection against Go's
`x509.SystemCertPool` (on the Alpine images: `/etc/ssl/certs/ca-certificates.crt`).
A private CA must therefore be in the pod's **system trust store**.
As of the tailscale operator chart on `main`, there is no supported way to hand
a CA _file_ to a proxy pod:
- `ProxyClass.spec.statefulSet.pod` has no `volumes`.
- `ProxyClass...tailscaleContainer` has no `volumeMounts`, and its `env` is a
reduced schema (`name`/`value` only — no `valueFrom`/`envFrom`).
- `operatorConfig` exposes `extraEnv` but no `extraVolumes`.
`SSL_CERT_FILE`/`SSL_CERT_DIR` _are_ honoured by tailscaled, but they need a
file that cannot be projected in. So the CA has to be baked into the images.
## The recipe
1. Serve Headscale with TLS (the `hsic` default) and grab `headscale.GetCert()`.
Pass it to the cluster so in-container `helm`/`kubectl` trust it.
2. Bake the CA into derived operator and proxy images. For each of
`tailscale/k8s-operator:<tag>` and `tailscale/tailscale:<tag>`, build:
```Dockerfile
FROM <base>
COPY headscale-ca.crt /usr/local/share/ca-certificates/headscale-ca.crt
RUN update-ca-certificates
```
Build on the host docker daemon (it has egress to pull the base images),
`ExportImage` the result to a docker-format tarball, stream it into the k3s
container, and import it into the kubelet's containerd namespace:
```
ctr --namespace k8s.io images import <tarball>
```
Tag the derived images `headscale.local/...:<tag>-ca` and run them with
`imagePullPolicy: Never` (the `headscale.local/` prefix is never resolved by
a registry).
3. Wire the derived images into the chart via `operatorConfig.image` /
`proxyConfig.image`. The operator's proxy StatefulSet template hard-codes
`imagePullPolicy: Always`, so a `ProxyClass` must override the proxy image
**and** set `imagePullPolicy: Never`; Connectors must reference that
ProxyClass explicitly (`defaultProxyClass` does not apply to them).
4. Pods resolve Headscale through CoreDNS, not the container's `/etc/hosts`, and
the cert's only SAN is the Headscale hostname (dialing by IP fails TLS
verification). Install a `coredns-custom` ConfigMap mapping the hostname to
the Headscale IP and gate on the `kube-dns` Service having ready endpoints
before starting the operator.
The full implementation lived in `integration/k3sic/k3sic.go` and
`integration/k8s_operator_test.go` before the switch to HTTP; recover it from
git history (`PrepareTailscaleImages`, `bakeAndImportImage`, `caBuildContext`,
`ConfigureCoreDNSHost`) if needed.
-266
View File
@@ -1,266 +0,0 @@
package integration
import (
"fmt"
"net/netip"
"slices"
"strings"
"testing"
"time"
clientv1 "github.com/juanfont/headscale/gen/client/v1"
policyv2 "github.com/juanfont/headscale/hscontrol/policy/v2"
"github.com/juanfont/headscale/integration/hsic"
"github.com/juanfont/headscale/integration/integrationutil"
"github.com/juanfont/headscale/integration/k3sic"
"github.com/juanfont/headscale/integration/tsic"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"tailscale.com/tailcfg"
)
const (
tagK8sOperator = "tag:k8s-operator"
tagK8s = "tag:k8s"
)
// k8sOperatorPolicy is the tagOwners policy the Tailscale Kubernetes operator
// requires: tag:k8s-operator is self/admin-owned, and the operator
// (tag:k8s-operator) owns tag:k8s so it can mint auth keys for the proxy nodes
// it spins up. The wildcard ACL lets the in-cluster proxies and the out-of-cluster
// tsic client reach each other for the connectivity checks.
func k8sOperatorPolicy() *policyv2.Policy {
return &policyv2.Policy{
TagOwners: policyv2.TagOwners{
tagK8sOperator: policyv2.Owners{},
tagK8s: policyv2.Owners{new(policyv2.Tag(tagK8sOperator))},
},
ACLs: []policyv2.ACL{
{
Action: "accept",
Sources: []policyv2.Alias{policyv2.Wildcard},
Destinations: []policyv2.AliasWithPorts{
{Alias: policyv2.Wildcard, Ports: []tailcfg.PortRange{tailcfg.PortRangeAny}},
},
},
},
}
}
// TestK8sOperator verifies that the real Tailscale Kubernetes operator, installed
// into a real k3s cluster via its Helm chart and pointed at an in-test Headscale,
// can authenticate with OAuth client credentials, mint auth keys, and register
// nodes that then interoperate with a regular tailnet node.
//
// The operator targets Headscale over plain HTTP by IP (hsic.WithoutTLS +
// GetIPEndpoint), so the operator and proxy pods need no CA and no DNS entry for
// Headscale. See integration/k3sic/tls-ca-baking.md for the TLS variant.
//
// The cluster-side steps are reusable building blocks on k3sic.K3sInContainer
// (InstallOperator, DeployConnector, DeployEchoServer, ExposeServiceToTailnet,
// DeployProxyGroup), so further operator scenarios are a few method calls.
//
// Run it with `go run ./cmd/hi run "TestK8sOperator"`.
func TestK8sOperator(t *testing.T) {
IntegrationSkip(t)
// One regular user+node provides the out-of-cluster tailnet peer used by the
// connectivity subtest; the operator registers its own nodes on top.
spec := ScenarioSpec{
Users: []string{"k8s-user"},
NodesPerUser: 1,
}
scenario, err := NewScenario(spec)
require.NoError(t, err)
defer scenario.ShutdownAssertNoPanics(t)
err = scenario.CreateHeadscaleEnv(
// The tsic client reaches the in-cluster proxy only via DERP (no direct
// path to the k3s pod network), and hsic's embedded DERP is non-TLS, so the
// client must reach DERP over plain-HTTP websockets — as the proxy pods do.
[]tsic.Option{tsic.WithDERPOverHTTP()},
hsic.WithTestName("k8soperator"),
hsic.WithoutTLS(),
hsic.WithACLPolicy(k8sOperatorPolicy()),
)
require.NoError(t, err)
headscale, err := scenario.Headscale()
require.NoError(t, err)
// Mint an OAuth client for the operator: devices:core + auth_keys scopes,
// tagged tag:k8s-operator. CreateOAuthClient mints the admin API key and calls
// the v2 keys API itself.
clientID, clientSecret, err := headscale.CreateOAuthClient(
t.Context(),
[]string{"devices:core", "auth_keys"},
[]string{tagK8sOperator},
)
require.NoError(t, err, "creating OAuth client (server-side OAuth must be implemented)")
require.NotEmpty(t, clientID)
require.NotEmpty(t, clientSecret)
// Bring up the k3s cluster on the scenario networks.
k3s, err := k3sic.New(scenario.Pool(), scenario.Networks())
require.NoError(t, err)
defer func() {
shutdownErr := k3s.Shutdown()
if shutdownErr != nil {
t.Logf("shutting down k3s: %s", shutdownErr)
}
}()
// Registered after Shutdown so it runs first (defers are LIFO): dump cluster
// state while it is still up if anything below fails.
defer func() {
if t.Failed() {
k3s.DumpDiagnostics()
}
}()
require.NoError(t, k3s.WaitForRunning())
require.NoError(t, k3s.InstallHelm())
// The operator reaches the control plane by IP, but the embedded DERP map
// references Headscale by hostname; teach CoreDNS to resolve it so the proxy
// pods can connect to DERP and get a data path to nodes outside the cluster.
hsIP := headscale.GetIPInNetwork(scenario.Networks()[0])
require.NoError(t, k3s.ConfigureCoreDNSHost(headscale.GetHostname(), hsIP))
// loginServer is the in-cluster-reachable HTTP endpoint by IP; the operator
// uses it for both the control plane and the management API.
loginServer := headscale.GetIPEndpoint()
require.NoError(t, k3s.InstallOperator(loginServer, clientID, clientSecret))
t.Run("operator-registers", func(t *testing.T) {
assert.EventuallyWithT(t, func(c *assert.CollectT) {
nodes, err := headscale.ListNodes()
assert.NoError(c, err)
assert.True(c, hasNodeWithTag(nodes, tagK8sOperator),
"expected a node tagged %s registered by the operator, got %s",
tagK8sOperator, describeNodes(nodes))
}, integrationutil.ScaledTimeout(180*time.Second), 2*time.Second,
"operator node should register and be tagged "+tagK8sOperator)
})
t.Run("egress-connector", func(t *testing.T) {
require.NoError(t, k3s.DeployConnector("k8s-egress", []string{tagK8s}, []string{"10.40.0.0/14"}))
assert.EventuallyWithT(t, func(c *assert.CollectT) {
nodes, err := headscale.ListNodes()
assert.NoError(c, err)
assert.True(c, hasNodeWithTag(nodes, tagK8s),
"expected a proxy node tagged %s registered by the operator, got %s",
tagK8s, describeNodes(nodes))
}, integrationutil.ScaledTimeout(180*time.Second), 2*time.Second,
"egress proxy node should register and be tagged "+tagK8s)
})
t.Run("ingress-service-reachable-from-tailnet", func(t *testing.T) {
require.NoError(t, k3s.DeployEchoServer("echo"))
require.NoError(t, k3s.ExposeServiceToTailnet("echo", []string{tagK8s}))
clients, err := scenario.ListTailscaleClients("k8s-user")
require.NoError(t, err)
require.NotEmpty(t, clients)
// The operator registers the ingress proxy as a tailnet node named after
// the exposed Service (<namespace>-<service>, here default-echo-ts). Read
// its IP from Headscale rather than the Service's LoadBalancer status,
// which the operator does not populate against Headscale.
var svcIP string
assert.EventuallyWithT(t, func(c *assert.CollectT) {
nodes, err := headscale.ListNodes()
assert.NoError(c, err)
ip, ok := nodeIPv4ByName(nodes, "echo")
assert.True(c, ok, "ingress proxy node for echo should register, got %s", describeNodes(nodes))
svcIP = ip
}, integrationutil.ScaledTimeout(180*time.Second), 2*time.Second,
"operator should register an ingress proxy node for the exposed service")
// The out-of-cluster node reaches the in-cluster service through the proxy
// over the tailnet. /hostname is an agnhost endpoint that returns the
// backend pod's hostname, proving the request reached the service.
assert.EventuallyWithT(t, func(c *assert.CollectT) {
body, err := clients[0].Curl("http://" + svcIP + "/hostname")
assert.NoError(c, err)
assert.NotEmpty(c, body, "expected a response from the exposed service")
}, integrationutil.ScaledTimeout(120*time.Second), 2*time.Second,
"tsic node should reach the k8s-exposed service over the tailnet")
})
t.Run("proxy-group", func(t *testing.T) {
nodes, err := headscale.ListNodes()
require.NoError(t, err)
before := countNodesWithTag(nodes, tagK8s)
const replicas = 2
require.NoError(t, k3s.DeployProxyGroup("ts-ingress", "ingress", replicas, []string{tagK8s}))
// A ProxyGroup runs a pool of proxies; each replica registers its own node.
assert.EventuallyWithT(t, func(c *assert.CollectT) {
nodes, err := headscale.ListNodes()
assert.NoError(c, err)
assert.GreaterOrEqual(c, countNodesWithTag(nodes, tagK8s), before+replicas,
"expected %d more %s nodes from the ProxyGroup, got %s",
replicas, tagK8s, describeNodes(nodes))
}, integrationutil.ScaledTimeout(180*time.Second), 2*time.Second,
"ProxyGroup replicas should each register a node tagged "+tagK8s)
})
}
// hasNodeWithTag reports whether any node carries the given tag.
func hasNodeWithTag(nodes []*clientv1.Node, tag string) bool {
return countNodesWithTag(nodes, tag) > 0
}
// countNodesWithTag counts the nodes carrying the given tag.
func countNodesWithTag(nodes []*clientv1.Node, tag string) int {
count := 0
for _, node := range nodes {
if slices.Contains(node.Tags, tag) {
count++
}
}
return count
}
// nodeIPv4ByName returns the IPv4 of the first node whose given name contains
// substr, identifying an operator-registered proxy by the Service/Connector it
// fronts (e.g. "echo" matches the default-echo-ts ingress proxy).
func nodeIPv4ByName(nodes []*clientv1.Node, substr string) (string, bool) {
for _, node := range nodes {
if !strings.Contains(node.GivenName, substr) {
continue
}
for _, ip := range node.IpAddresses {
addr, err := netip.ParseAddr(ip)
if err == nil && addr.Is4() {
return ip, true
}
}
}
return "", false
}
// describeNodes renders a compact name->tags summary for failure messages.
func describeNodes(nodes []*clientv1.Node) string {
parts := make([]string, 0, len(nodes))
for _, node := range nodes {
parts = append(parts, fmt.Sprintf("%s%v", node.Name, node.Tags))
}
return "[" + strings.Join(parts, " ") + "]"
}
-315
View File
@@ -1,315 +0,0 @@
package integration
import (
"context"
"crypto/tls"
"crypto/x509"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/netip"
"net/url"
"testing"
"time"
"github.com/coder/websocket"
"github.com/juanfont/headscale/integration/dockertestutil"
"github.com/juanfont/headscale/integration/hsic"
"github.com/juanfont/headscale/integration/tsic"
"github.com/ory/dockertest/v3"
"github.com/samber/lo"
"github.com/stretchr/testify/require"
"tailscale.com/control/controlbase"
"tailscale.com/control/controlhttp/controlhttpcommon"
"tailscale.com/net/wsconn"
"tailscale.com/tailcfg"
"tailscale.com/types/key"
"tailscale.com/util/rands"
)
// Tailscale's JS/WASM control client opens /ts2021 as a browser WebSocket — an
// HTTP GET upgrade — rather than the native client's HTTP POST upgrade. A router
// that registers /ts2021 for POST only rejects that GET with 405 before the
// Noise handshake starts, which breaks every WASM client (issue #3357).
//
// These two tests guard that path against real headscale:
//
// - TestTS2021WebSocketGET dials the WebSocket GET directly from the test
// process using the same coder/websocket + controlbase primitives the WASM
// client uses. It is fast and always on.
// - TestTS2021WASMClientUnderNode runs the *actual* tailscale.com js/wasm
// control dial (integration/wasmic/wasmclient, built for GOOS=js) inside a
// Node container, alongside normal Tailscale clients, and asserts it
// completes the Noise handshake with headscale over the WebSocket.
//
// The server cannot tell the two apart: both send GET /ts2021 with
// Sec-WebSocket-Protocol: tailscale-control-protocol. Before the fix both fail
// with 405; after it, both complete the handshake.
// TestTS2021WebSocketGET connects to /ts2021 over a WebSocket GET from the test
// process and completes the Noise handshake, exactly as a browser/WASM client
// would.
func TestTS2021WebSocketGET(t *testing.T) {
IntegrationSkip(t)
t.Parallel()
spec := ScenarioSpec{
NodesPerUser: 1,
Users: []string{"user1"},
}
scenario, err := NewScenario(spec)
require.NoErrorf(t, err, "failed to create scenario: %s", err)
defer scenario.ShutdownAssertNoPanics(t)
err = scenario.CreateHeadscaleEnv([]tsic.Option{}, hsic.WithTestName("ts2021ws"))
requireNoErrHeadscaleEnv(t, err)
headscale, err := scenario.Headscale()
requireNoErrGetHeadscale(t, err)
conn, err := dialTS2021WebSocket(t, headscale.GetEndpoint(), headscale.GetCert())
require.NoError(t, err,
"WebSocket GET to /ts2021 must reach NoiseUpgradeHandler, not be rejected by the router with 405")
require.NotNil(t, conn)
t.Cleanup(func() { _ = conn.Close() })
t.Logf("noise established over websocket, protocol version %d", conn.ProtocolVersion())
}
// TestTS2021WASMClientUnderNode runs the real tailscale.com js/wasm control dial
// inside a Node container against real headscale, next to normal Tailscale
// clients, and asserts the WASM client completes the /ts2021 handshake.
func TestTS2021WASMClientUnderNode(t *testing.T) {
IntegrationSkip(t)
t.Parallel()
spec := ScenarioSpec{
NodesPerUser: 2,
Users: []string{"user1"},
Networks: map[string]NetworkSpec{
"wasmnet": {Users: []string{"user1"}},
},
ExtraService: map[string][]extraServiceFunc{
"wasmnet": {wasmClientService},
},
// The wasm client image builds from this module; pair it with the
// head Tailscale clients so the whole environment is current.
Versions: []string{"head"},
}
scenario, err := NewScenario(spec)
require.NoErrorf(t, err, "failed to create scenario: %s", err)
defer scenario.ShutdownAssertNoPanics(t)
// The Tailscale JS/WASM client dials the control server as a WebSocket.
// client_js.go only honours a custom port for ws:// (plain HTTP); over
// wss:// it always targets :443, so it cannot reach a TLS control server on
// :8080. Run headscale without TLS, matching the http:// setup in the issue.
err = scenario.CreateHeadscaleEnv(
[]tsic.Option{},
hsic.WithTestName("ts2021wasm"),
hsic.WithoutTLS(),
)
requireNoErrHeadscaleEnv(t, err)
allClients, err := scenario.ListTailscaleClients()
requireNoErrListClients(t, err)
allIPs, err := scenario.ListTailscaleClientsIPs()
requireNoErrListClientIPs(t, err)
// Normal Tailscale clients come up and form a working tailnet alongside the
// WASM control client.
err = scenario.WaitForTailscaleSync()
requireNoErrSync(t, err)
headscale, err := scenario.Headscale()
requireNoErrGetHeadscale(t, err)
// Sanity-check the tailnet the WASM client is joining: the normal clients
// must be able to reach each other.
allAddrs := lo.Map(allIPs, func(x netip.Addr, _ int) string { return x.String() })
assertPingAll(t, allClients, allAddrs)
services, err := scenario.Services("wasmnet")
require.NoError(t, err)
require.Len(t, services, 1, "expected the wasm client container")
wasm := services[0]
controlURL := headscale.GetEndpoint()
// Fetch the server's Noise key here and pass it to the WASM client: Go's
// net/http DNS resolver is unavailable under GOOS=js, so the client can only
// use the JS WebSocket transport, not an HTTP GET to /key.
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
controlKey, err := fetchServerNoiseKey(ctx, &http.Client{Timeout: 15 * time.Second}, controlURL)
require.NoError(t, err)
controlKeyText, err := controlKey.MarshalText()
require.NoError(t, err)
// Run the real js/wasm control client under Node: it dials /ts2021 as a
// WebSocket GET; success means the Noise handshake completed.
stdout, stderr, err := dockertestutil.ExecuteCommand(
wasm,
[]string{"node", "/app/wasm_exec_node.js", "/app/client.wasm", controlURL, string(controlKeyText)},
[]string{},
dockertestutil.ExecuteCommandTimeout(60*time.Second),
)
t.Logf("wasm client stdout:\n%s", stdout)
t.Logf("wasm client stderr:\n%s", stderr)
require.NoError(t, err,
"wasm control client must connect to /ts2021 over websocket (405 means the router rejected the GET)")
require.Contains(t, stdout, "WASM_TS2021_OK",
"wasm control client should report a completed Noise handshake")
}
// wasmClientService builds and starts the Node + js/wasm control-client
// container (Dockerfile.wasmclient) on the given network so it can reach
// headscale by hostname. It idles; the test execs the client on demand.
func wasmClientService(s *Scenario, networkName string) (*dockertest.Resource, error) {
hash := rands.HexString(hsicOIDCMockHashLength)
hostname := "hs-wasmclient-" + hash
network, ok := s.networks[s.prefixedNetworkName(networkName)]
if !ok {
return nil, fmt.Errorf("network does not exist: %s", networkName) //nolint:err113
}
runOpts := &dockertest.RunOptions{
Name: hostname,
Networks: []*dockertest.Network{network},
Env: []string{},
}
dockertestutil.DockerAddIntegrationLabels(runOpts, "wasmclient")
buildOpts := &dockertest.BuildOptions{
Dockerfile: "Dockerfile.wasmclient",
ContextDir: dockerContextPath,
}
resource, err := s.pool.BuildAndRunWithBuildOptions(
buildOpts,
runOpts,
dockertestutil.DockerRestartPolicy,
)
if err != nil {
return nil, fmt.Errorf("building wasm client container: %w", err)
}
return resource, nil
}
// dialTS2021WebSocket opens /ts2021 as a WebSocket GET (subprotocol
// tailscale-control-protocol) and completes the Noise handshake, mirroring what
// tailscale.com/control/controlhttp/client_js.go does in a browser. It returns
// the established Noise connection, or an error (a router that only allows POST
// returns 405 here).
func dialTS2021WebSocket(t *testing.T, endpoint string, caCert []byte) (*controlbase.Conn, error) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
t.Cleanup(cancel)
u, err := url.Parse(endpoint)
require.NoError(t, err)
httpClient := &http.Client{Timeout: 15 * time.Second}
if u.Scheme == "https" {
pool := x509.NewCertPool()
pool.AppendCertsFromPEM(caCert)
httpClient.Transport = &http.Transport{
TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12},
}
}
controlKey, err := fetchServerNoiseKey(ctx, httpClient, endpoint)
require.NoError(t, err)
init, cont, err := controlbase.ClientDeferred(
key.NewMachine(),
controlKey,
uint16(tailcfg.CurrentCapabilityVersion),
)
require.NoError(t, err)
wsScheme := "ws"
if u.Scheme == "https" {
wsScheme = "wss"
}
wsURL := &url.URL{
Scheme: wsScheme,
Host: u.Host,
Path: "/ts2021",
RawQuery: url.Values{
controlhttpcommon.HandshakeHeaderName: []string{base64.StdEncoding.EncodeToString(init)},
}.Encode(),
}
wsConn, resp, err := websocket.Dial(ctx, wsURL.String(), &websocket.DialOptions{
Subprotocols: []string{controlhttpcommon.UpgradeHeaderValue},
HTTPClient: httpClient,
})
if resp != nil && resp.Body != nil {
_ = resp.Body.Close()
}
if err != nil {
return nil, err
}
netConn := wsconn.NetConn(ctx, wsConn, websocket.MessageBinary, wsURL.String())
cbConn, err := cont(ctx, netConn)
if err != nil {
_ = netConn.Close()
return nil, fmt.Errorf("noise handshake over websocket: %w", err)
}
return cbConn, nil
}
// fetchServerNoiseKey retrieves headscale's Noise public key from /key, the same
// endpoint a real client consults before dialing /ts2021.
func fetchServerNoiseKey(
ctx context.Context,
client *http.Client,
endpoint string,
) (key.MachinePublic, error) {
var zero key.MachinePublic
keyURL := fmt.Sprintf("%s/key?v=%d", endpoint, tailcfg.CurrentCapabilityVersion)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, keyURL, nil)
if err != nil {
return zero, err
}
resp, err := client.Do(req)
if err != nil {
return zero, err
}
defer resp.Body.Close()
var k tailcfg.OverTLSPublicKeyResponse
err = json.NewDecoder(resp.Body).Decode(&k)
if err != nil {
return zero, fmt.Errorf("decoding /key response: %w", err)
}
if k.PublicKey.IsZero() {
return zero, errors.New("server returned zero Noise public key") //nolint:err113
}
return k.PublicKey, nil
}
-19
View File
@@ -98,7 +98,6 @@ type TailscaleInContainer struct {
caCerts [][]byte
headscaleHostname string
withWebsocketDERP bool
withDERPOverHTTP bool
withSSH bool
withTags []string
withEntrypoint []string
@@ -161,18 +160,6 @@ func WithWebsocketDERP(enabled bool) Option {
}
}
// WithDERPOverHTTP makes the client reach the DERP server over plain-HTTP
// websockets (TS_DEBUG_DERP_WS_CLIENT + TS_DEBUG_USE_DERP_HTTP). It is the
// counterpart to [hsic.WithoutTLS]: a Headscale serving its embedded DERP without
// TLS is otherwise unreachable, because the client defaults to dialing DERP over
// HTTPS.
func WithDERPOverHTTP() Option {
return func(tsic *TailscaleInContainer) {
tsic.withWebsocketDERP = true
tsic.withDERPOverHTTP = true
}
}
// WithSSH enables SSH for the Tailscale instance.
func WithSSH() Option {
return func(tsic *TailscaleInContainer) {
@@ -385,12 +372,6 @@ func New(
tailscaleOptions.Env,
fmt.Sprintf("TS_DEBUG_DERP_WS_CLIENT=%t", tsic.withWebsocketDERP),
)
// Plain-HTTP DERP additionally needs the client to dial http:// instead of
// the default https://; see [WithDERPOverHTTP].
if tsic.withDERPOverHTTP {
tailscaleOptions.Env = append(tailscaleOptions.Env, "TS_DEBUG_USE_DERP_HTTP=true")
}
}
tailscaleOptions.ExtraHosts = append(tailscaleOptions.ExtraHosts,
-103
View File
@@ -1,103 +0,0 @@
//go:build js
// Command wasmclient is a minimal Tailscale control client compiled to
// GOOS=js/GOARCH=wasm and run under Node. It exercises the real
// tailscale.com/control/controlhttp js/wasm dial path
// (control/controlhttp/client_js.go), which opens /ts2021 as a browser-style
// WebSocket GET — the exact transport a Tailscale JS/WASM client uses.
//
// It is the container-side half of the integration test guarding issue #3357:
// headscale must register /ts2021 for GET, not POST only, or the WebSocket
// upgrade is rejected with 405 before the Noise handshake can start.
//
// It is intentionally not the full tsconnect IPN — the regression is entirely
// in the control-connection upgrade, and this drives the real upgrade code with
// the smallest possible harness. On success it prints wasmSuccessMarker and
// exits 0; on any failure it prints wasmFailureMarker and exits non-zero.
package main
import (
"context"
"fmt"
"net/url"
"os"
"time"
"tailscale.com/control/controlhttp"
"tailscale.com/tailcfg"
"tailscale.com/types/key"
)
// These markers are matched by the integration test on the client's stdout.
const (
wasmSuccessMarker = "WASM_TS2021_OK"
wasmFailureMarker = "WASM_TS2021_FAIL"
)
func main() {
if len(os.Args) < 3 {
fmt.Printf("%s: usage: wasmclient <control-url> <noise-key>\n", wasmFailureMarker)
os.Exit(2)
}
if err := run(os.Args[1], os.Args[2]); err != nil {
fmt.Printf("%s: %v\n", wasmFailureMarker, err)
os.Exit(1)
}
}
// run dials /ts2021 exactly as tailscale.com/control/controlhttp/client_js.go
// does in a browser: a WebSocket GET via the JS/undici WebSocket. The server's
// Noise key is passed in (the test fetches /key) rather than fetched here,
// because Go's net/http DNS resolver is unavailable under GOOS=js — only the
// WebSocket transport, which runs through the JS host, works.
func run(controlURL, noiseKeyText string) error {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
u, err := url.Parse(controlURL)
if err != nil {
return fmt.Errorf("parse control url %q: %w", controlURL, err)
}
var controlKey key.MachinePublic
if err := controlKey.UnmarshalText([]byte(noiseKeyText)); err != nil {
return fmt.Errorf("parse noise key %q: %w", noiseKeyText, err)
}
port := u.Port()
if port == "" {
if u.Scheme == "https" {
port = "443"
} else {
port = "80"
}
}
// client_js.go selects ws:// (and appends the port) only when HTTPPort is a
// custom non-80 port and HTTPS is 443 or disabled; otherwise it dials wss://
// on the default port. Set the fields to match the server's actual scheme.
d := &controlhttp.Dialer{
Hostname: u.Hostname(),
MachineKey: key.NewMachine(),
ControlKey: controlKey,
ProtocolVersion: uint16(tailcfg.CurrentCapabilityVersion),
}
if u.Scheme == "https" {
d.HTTPSPort = port
} else {
d.HTTPPort = port
d.HTTPSPort = controlhttp.NoPort
}
conn, err := d.Dial(ctx)
if err != nil {
return fmt.Errorf("ts2021 websocket dial: %w", err)
}
defer conn.Close()
fmt.Printf("%s: noise established over websocket, protocol version %d\n",
wasmSuccessMarker, conn.ProtocolVersion())
return nil
}
-8
View File
@@ -1,8 +0,0 @@
//go:build !js
// This package only does something when built for GOOS=js (see main.go). The
// stub exists so `go build ./...` and `go vet ./...` on the host don't fail with
// "build constraints exclude all Go files" for this directory.
package main
func main() {}