diff --git a/AGENTS.md b/AGENTS.md index ecbd799f..6d8700cd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -331,3 +331,9 @@ a broken one. - **Do not edit `gen/`** — it is regenerated from `proto/` by `make generate`. - **Proto changes + code changes should be two commits**, not one. +- **The NixOS test kit is a public contract** consumed by other projects' + NixOS tests: the "Contract" in `nix/README.md`, covering + `nix/testkit.nix`, `nix/testkit-peer.nix`, the embedded DERP region + and `HEADSCALE_DEBUG_INSECURE_TLS_LISTEN_ADDR`, and the CLI commands + `hs-authkey` calls. Breaking it needs a "NixOS test kit" BREAKING entry + in `CHANGELOG.md`. diff --git a/CHANGELOG.md b/CHANGELOG.md index ada2953e..757c023b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,24 @@ removed on this schedule: [#3352](https://github.com/juanfont/headscale/pull/3352) +### NixOS test kit + +Projects built on Tailscale, such as tsnet services, tailscaled integrations or +Tailscale client implementations, can now test against a real control server +in their NixOS VM tests. Import `nixosModules.testkit` on a node named +`headscale`. Clients join `http://headscale` with no certificates or other +setup, and `hs-authkey USER` on that node mints their auth keys. +`nixosModules.testkit-peer` adds a tailscaled peer that joins with `hs-join KEY`: + +```nix +nodes.headscale.imports = [ inputs.headscale.nixosModules.testkit ]; +nodes.peer.imports = [ inputs.headscale.nixosModules.testkit-peer ]; +# testScript: peer.succeed(f"hs-join {headscale.succeed('hs-authkey alice').strip()}") +``` + +See `nix/README.md` for the full contract, recipes for tsnet and non-Go +clients, and how to run the same setup without Nix. + ### BREAKING #### Database diff --git a/flake.nix b/flake.nix index c7008e81..f048cc84 100644 --- a/flake.nix +++ b/flake.nix @@ -25,8 +25,13 @@ { # NixOS module nixosModules = rec { - headscale = import ./nix/module.nix; + # A path, so importing it twice (directly and via testkit) dedupes. + headscale = ./nix/module.nix; default = headscale; + # Control node for NixOS VM tests of Tailscale clients, and a peer + # that joins it; nix/README.md. + testkit = import ./nix/testkit.nix self; + testkit-peer = ./nix/testkit-peer.nix; }; overlays.default = diff --git a/nix/README.md b/nix/README.md index 533e4b5e..08ce55c1 100644 --- a/nix/README.md +++ b/nix/README.md @@ -14,6 +14,9 @@ aim to be upstreamed to nixpkgs. - **[`module.nix`](./module.nix)** - The NixOS module implementation - **[`example-configuration.nix`](./example-configuration.nix)** - Example configuration demonstrating all major features +- **[`testkit.nix`](./testkit.nix)**, + **[`testkit-peer.nix`](./testkit-peer.nix)** - Test kit: a control node for + NixOS VM tests of Tailscale clients, and a peer that joins it (see below) - **[`tests/`](./tests/)** - NixOS integration tests ## Usage @@ -33,6 +36,163 @@ imports = [ inputs.headscale.nixosModules.default ]; See [`example-configuration.nix`](./example-configuration.nix) for configuration options. +## Test kit + +`nixosModules.testkit` turns a node of a NixOS VM test into a headscale control +server. Projects that implement or embed Tailscale (tailscaled, tsnet, +tailscale-rs, …) use it to test against a real control server without writing +one. Go clients join without a CA, a hosts entry or an env var. +`nixosModules.testkit-peer` is an optional tailscaled peer that joins it in one +line. + +```nix +pkgs.testers.runNixOSTest { + name = "my-app"; + nodes.headscale.imports = [ inputs.headscale.nixosModules.testkit ]; + nodes.peer.imports = [ inputs.headscale.nixosModules.testkit-peer ]; + testScript = '' + start_all() + key = headscale.succeed("hs-authkey alice").strip() + peer.succeed(f"hs-join {key}") + ''; +} +``` + +It works with `nixosTest`, `runNixOSTest` and `runTest`. The headscale package +must come from the same release or newer: the kit relies on its +`preauthkeys create --user NAME` and `HEADSCALE_DEBUG_INSECURE_TLS_LISTEN_ADDR`. + +### Contract + +Breaking changes to anything below get a "NixOS test kit" BREAKING entry in the +CHANGELOG. + +- Control: `http://`, so `http://headscale` for a node named + `headscale`. Use it for `tailscale up --login-server`, + `tsnet.Server.ControlURL` and tailscale-rs `TS_CONTROL_URL`. +- The same router is also served on TLS :443 with a throwaway self-signed + certificate. DERP is region 999 (`headscale`) on :443, marked + `InsecureForTests`, with STUN on UDP 3478. +- `hs-authkey USER [headscale preauthkeys create flags]`, on the control node: + - waits for headscale for up to `HEADSCALE_CLI_TIMEOUT` (default `60s`); + - creates `USER` if missing; + - prints one reusable 24h key. + + Extra flags pass through, e.g. `--ephemeral` or `--tags tag:ci`. In a policy, + refer to these users as `USER@`, because they have no email. + +- `hs-join KEY [tailscale up flags]`, on a `testkit-peer` node, waits for + tailscaled, then joins the control node named `headscale` with a 60s + `tailscale up --timeout`, so a broken control plane fails the test instead of + hanging it. A trailing `--login-server` overrides the URL. +- Defaults you can override: + - `policy.mode = "database"`: nothing stored allows all, and + `headscale policy set -f FILE` applies live. For a fixed policy, set + `settings.policy = { mode = "file"; path = pkgs.writeText "policy.json" (builtins.toJSON { … }); }`. + A policy may name users before `hs-authkey` creates them. + - MagicDNS with `dns.base_domain = "tailnet"`. + - `dns.override_local_dns = false`. +- Everything else is plain `services.headscale.*` on that node. + - Override the package with a plain assignment, e.g. to carry a patch: + `services.headscale.package = inputs.headscale.packages.${system}.headscale.overrideAttrs (o: { patches = (o.patches or [ ]) ++ [ ./fix.patch ]; });` + - Don't bind :80 or :443 on that node yourself. + +### Clients + +- **tailscaled**: import `testkit-peer` and use `hs-join`, or run + `tailscale up --login-server http://headscale --auth-key KEY` yourself. Its + log warns that it "could not establish an encrypted connection": that is the + untrusted certificate on :443, and it is expected. +- **tsnet**: set `ControlURL` to `http://headscale` and hand it the key, + e.g. `TS_AUTHKEY` in an `EnvironmentFile` that the test writes before it + starts the unit. +- **tailscale-rs**: build with its `ts_control/insecure-keyfetch` and + `ts_control/insecure-derp` features, which allow plain-HTTP `/key` and + honour `InsecureForTests`. See its examples for the control URL, auth key and + hostname flags. +- **Proving relay**: set `TS_DEBUG_ALWAYS_USE_DERP=1` on a Go peer. + - Between Go peers, `tailscale ping --until-direct=false PEER` reports + `via DERP(headscale)`. + - A non-Go peer may not answer pings over DERP. Check its home DERP from a Go + peer with + `tailscale status --json | jq -e '.Peer[] | select(.HostName == "NAME") | .Relay == "headscale"'` + and send application traffic. +- **Logs**: `testkit-peer` turns off tailscaled's log uploads. Inside the Nix + build sandbox other clients' uploads just fail; set `TS_NO_LOGS_NO_SUPPORT=1` + for tsnet if interactive runs should not upload either. +- **Unprivileged tsnet**: without `CAP_NET_ADMIN`, tsnet binds its sockets to + the default-route interface. In a test VM that is the user-net NIC, not the + VLAN, so control is unreachable. The same thing happens on multi-homed + production hosts, so fix it in the service: grant `CAP_NET_ADMIN`, or call + `netns.SetEnabled(false)`. +- **Versions**: headscale supports the Tailscale client versions named at the + top of each CHANGELOG release, with no upper bound. Pin a headscale release + that accepts your client. + +### Without Nix + +A harness that runs the headscale binary itself, such as an Android emulator +test, can use the same shape: + +- Serve plain HTTP. +- Set `HEADSCALE_DEBUG_INSECURE_TLS_LISTEN_ADDR` to serve the router and DERP + over a throwaway TLS certificate. +- Mint keys: + + ```sh + headscale -c CONFIG users create NAME + headscale -c CONFIG preauthkeys create --user NAME --reusable --expiration 24h + ``` + +A minimal config: + +```yaml +# Clients dial this host for DERP and STUN too, so all of them must reach it. +server_url: http://127.0.0.1:8080 +listen_addr: 127.0.0.1:8080 +noise: { private_key_path: /tmp/hs/noise.key } +database: { type: sqlite, sqlite: { path: /tmp/hs/db.sqlite } } +prefixes: { v4: 100.64.0.0/10, v6: "fd7a:115c:a1e0::/48" } +dns: { magic_dns: false, override_local_dns: false } +unix_socket: /tmp/hs/headscale.sock +disable_check_updates: true +derp: + urls: [] + server: + enabled: true + region_id: 999 + region_code: headscale + stun_listen_addr: 127.0.0.1:3478 + private_key_path: /tmp/hs/derp.key +``` + +- Wait for `HEADSCALE_CLI_TIMEOUT=60s headscale -c CONFIG health`. +- If `server_url` is a hostname rather than an IP, Go clients redial noise only + on :443 after a recent dial. Put the TLS listener on :443 then. +- If clients reach headscale at different addresses, e.g. an emulator behind + NAT, set `derp.server.enabled: false`. Instead, list a DERP map of your own + in `derp.paths`, as `.yaml`, `.json` or `.hujson`. + +### Migrating a hand-rolled control node + +Delete these and import the kit: + +- the self-signed cert, `security.pki.certificateFiles` and the nginx TLS proxy; +- the embedded DERP block (`region_id = 999`, `urls = [ ]`) and its firewall + ports; +- `ip_prefixes`, which headscale no longer reads; +- `/etc/hosts` pins for the control node; +- jq lookups of the user ID before `preauthkeys create`. + +### Cost + +- headscale builds from source with its own pinned nixpkgs and Go. +- Setting `inputs.headscale.inputs.nixpkgs.follows` works only if your nixpkgs + has the Go version `go.mod` requires. Compare + `nix eval --raw nixpkgs#go_latest.version` against it. +- The input also brings headscale's development inputs into your lock file. If + you already depend on them, `follows` them. + ## Upstream - [nixpkgs module](https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/networking/headscale.nix) diff --git a/nix/testkit-peer.nix b/nix/testkit-peer.nix new file mode 100644 index 00000000..5ef5ccdb --- /dev/null +++ b/nix/testkit-peer.nix @@ -0,0 +1,29 @@ +# A tailscaled peer for nix/testkit.nix: `hs-join KEY` joins the kit's control +# node, which is named headscale. Contract: nix/README.md. +{ + config, + lib, + pkgs, + ... +}: +let + hs-join = pkgs.writeShellApplication { + name = "hs-join"; + runtimeInputs = [ config.services.tailscale.package ]; + text = '' + key=''${1:?usage: hs-join KEY [tailscale up flags]} + shift + # Returns once tailscaled is up, so joining straight after boot can't race it. + systemctl start tailscaled.service + # Broken control fails here instead of hanging the test to its global timeout. + exec tailscale up --timeout=60s --login-server http://headscale --auth-key "$key" "$@" + ''; + }; +in +{ + key = "headscale-testkit-peer"; + services.tailscale.enable = true; + # Interactive driver runs have a route out; keep their logs off Tailscale's servers. + services.tailscale.disableUpstreamLogging = lib.mkDefault true; + environment.systemPackages = [ hs-join ]; +} diff --git a/nix/testkit.nix b/nix/testkit.nix new file mode 100644 index 00000000..9db36d99 --- /dev/null +++ b/nix/testkit.nix @@ -0,0 +1,78 @@ +# Makes a NixOS test node a headscale control server that any Tailscale client +# joins without trust setup: plain-HTTP control, plus the same router on a +# self-signed TLS :443 whose DERP region is InsecureForTests. That is the one +# shape tailscaled, tsnet, webpki-only tailscale-rs and the Android app all +# accept. Contract: nix/README.md. +self: +{ + config, + lib, + pkgs, + ... +}: +let + hs-authkey = pkgs.writeShellApplication { + name = "hs-authkey"; + runtimeInputs = [ + config.services.headscale.package + pkgs.jq + ]; + text = '' + user=''${1:?usage: hs-authkey USER [headscale preauthkeys create flags]} + shift + # Safe straight after start_all(): the CLI retries the socket until this. + HEADSCALE_CLI_TIMEOUT=''${HEADSCALE_CLI_TIMEOUT:-60s} headscale health >/dev/null + headscale users list --name "$user" -o json | jq -e 'length > 0' >/dev/null || + headscale users create "$user" >/dev/null + headscale preauthkeys create --user "$user" --reusable --expiration 24h "$@" + ''; + }; +in +{ + key = "headscale-testkit"; + _file = ./testkit.nix; + imports = [ self.nixosModules.headscale ]; + + services.headscale = { + enable = true; + package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.headscale; + address = "[::]"; + port = 80; + settings = { + server_url = "http://${config.networking.hostName}"; + dns.base_domain = lib.mkDefault "tailnet"; + dns.override_local_dns = lib.mkDefault false; + # Nothing stored allows all, and `headscale policy set` works live. + policy.mode = lib.mkDefault "database"; + derp = { + urls = [ ]; + server = { + enabled = true; + region_id = 999; + region_code = "headscale"; + region_name = "headscale test kit"; + stun_listen_addr = "[::]:3478"; + }; + }; + }; + }; + + # DERP rides this listener, and so does noise: Go clients redial only :443 + # for a hostname URL after a recent dial. + systemd.services.headscale = { + environment.HEADSCALE_DEBUG_INSECURE_TLS_LISTEN_ADDR = "[::]:443"; + # The module grants this only for a privileged main port. + serviceConfig.AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ]; + serviceConfig.CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ]; + }; + + networking.firewall = { + allowedTCPPorts = [ + 80 + 443 + ]; + allowedUDPPorts = [ 3478 ]; + }; + + environment.systemPackages = [ hs-authkey ]; +}