From d2dc13696a1d888d1892f6f05715fba8b63a0203 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Tue, 10 Nov 2026 09:07:12 +0000 Subject: [PATCH] config, cli, packaging, nix: stop restarting headscale on a configuration error serve exits 78 (EX_CONFIG) when the configuration file it loaded is wrong: format, syntax, keys or values. No file, an unreadable one, the database and ports still exit 1, so systemd keeps retrying what can heal. The deb unit and the NixOS module set RestartPreventExitStatus=78. A parse error gives a position and fixed words, never the file's text. Updates #1926 --- cmd/headscale/cli/root.go | 13 +++++- cmd/headscale/cli/serve.go | 18 ++++++++ cmd/headscale/cli/serve_test.go | 70 +++++++++++++++++++++++++++++ hscontrol/config/keys.go | 44 ++++++++++++++++++ hscontrol/config/load.go | 9 +++- hscontrol/config/load_test.go | 35 ++++++++++++++- nix/module.nix | 3 ++ nix/tests/checks.nix | 15 ++++++- packaging/deb/postinst | 2 +- packaging/systemd/headscale.service | 3 ++ 10 files changed, 205 insertions(+), 7 deletions(-) diff --git a/cmd/headscale/cli/root.go b/cmd/headscale/cli/root.go index 0e2858820..a84062fd9 100644 --- a/cmd/headscale/cli/root.go +++ b/cmd/headscale/cli/root.go @@ -112,7 +112,10 @@ func initConfig() { if err != nil { // No log.format applies: main's console logger, on stderr. logWarnings(l.Warnings) - log.Fatal().Caller().Err(err).Msg("error loading config") + log.WithLevel(zerolog.FatalLevel).Caller().Err(err).Msg("error loading config") + + cmd, _, _ := rootCmd.Find(os.Args[1:]) + os.Exit(exitCode(cmd, l.File, err)) } loaded = l @@ -201,6 +204,12 @@ func Execute() { if err != nil { outputFormat, _ := cmd.Flags().GetString("output") printError(err, outputFormat) - os.Exit(1) + + var file string + if loaded != nil { + file = loaded.File + } + + os.Exit(exitCode(cmd, file, err)) } } diff --git a/cmd/headscale/cli/serve.go b/cmd/headscale/cli/serve.go index 2698e3dea..8eacda4a7 100644 --- a/cmd/headscale/cli/serve.go +++ b/cmd/headscale/cli/serve.go @@ -42,6 +42,24 @@ var serveCmd = &cobra.Command{ }, } +// exitConfig is EX_CONFIG from sysexits.h. serve exits with it when its +// configuration is the cause, which a restart with the same inputs cannot +// fix, so the systemd units set RestartPreventExitStatus to it. +const exitConfig = 78 + +// exitCode is how cmd exits on err, file being the configuration file it +// loaded: exitConfig when serve stops on what that configuration says; 1 +// otherwise. Without a file, the one expected may be on a mount not yet up, +// and a database, a port or a file the configuration names may yet become +// available, so systemd keeps retrying those. +func exitCode(cmd *cobra.Command, file string, err error) int { + if cmd == serveCmd && file != "" && errors.Is(err, types.ErrConfig) { + return exitConfig + } + + return 1 +} + // logNoConfigFile says when the server runs without a file. The loader does // not warn: client commands need no file unless they use an API key. func logNoConfigFile(l *config.Loaded) { diff --git a/cmd/headscale/cli/serve_test.go b/cmd/headscale/cli/serve_test.go index 23fc5a9b4..00dc86891 100644 --- a/cmd/headscale/cli/serve_test.go +++ b/cmd/headscale/cli/serve_test.go @@ -6,6 +6,9 @@ import ( "errors" "fmt" "net" + "os" + "os/exec" + "path/filepath" "strings" "syscall" "testing" @@ -172,3 +175,70 @@ func TestLogNoConfigFile(t *testing.T) { }) } } + +// TestServeExitCode: serve exits with exitConfig only when the configuration +// file it loaded is the cause, so systemd stops restarting it; a failure that +// can heal, no file at all, and every other command exit with 1. +func TestServeExitCode(t *testing.T) { + dir := t.TempDir() + notADir := writeFile(t, dir, "file", "") + + const valid = "server_url: https://headscale.example.com\ndisable_check_updates: true\n" + + tests := []struct { + name string + file string // the config file's name, config.yaml when "" + config string // written to file; "" passes a path that does not exist + search bool // no -c: the file is searched for, in ~/.headscale + args []string + want int + stderr string + }{ + {name: "rejected value", config: "server_url: ftp://headscale.example.com\ndisable_check_updates: true\n", args: []string{"serve"}, want: exitConfig, stderr: "server_url"}, + {name: "unknown key", config: valid + "bogus_key: 1\n", args: []string{"serve"}, want: exitConfig, stderr: "unknown configuration key bogus_key"}, + {name: "syntax error", config: valid + "derp: [\n", args: []string{"serve"}, want: exitConfig, stderr: "yaml: line 3"}, + {name: "missing file", args: []string{"serve"}, want: 1, stderr: "error loading config"}, + {name: "no file found", search: true, args: []string{"serve"}, want: 1, stderr: "server_url is required"}, + {name: "old format found", file: "config.toml", config: "server_url = \"https://headscale.example.com\"\n", search: true, args: []string{"serve"}, want: exitConfig, stderr: "unsupported config file format"}, + {name: "unwritable noise key", config: valid + "noise:\n private_key_path: " + notADir + "/noise.key\n", args: []string{"serve"}, want: 1, stderr: "Noise"}, + {name: "other command", config: valid + "bogus_key: 1\n", args: []string{"users", "list"}, want: 1, stderr: "unknown configuration key bogus_key"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + home, dir, name := t.TempDir(), t.TempDir(), tt.file + if name == "" { + name = "config.yaml" + } + + if tt.search { + dir = filepath.Join(home, ".headscale") + require.NoError(t, os.Mkdir(dir, 0o700)) + } + + path := filepath.Join(dir, "absent", name) + if tt.config != "" { + path = writeFile(t, dir, name, tt.config) + } + + args := append([]string{"-c", path}, tt.args...) + if tt.search { + // The search always looks in /etc/headscale; a host config there + // would be found, or serve would start on it. + if found, _ := filepath.Glob("/etc/headscale/config.*"); len(found) > 0 { + t.Skipf("the search would find %v", found) + } + + args = tt.args + } + + env := []string{"HOME=" + home, "XDG_CONFIG_DIRS=" + t.TempDir(), "HEADSCALE_DISABLE_CHECK_UPDATES=true"} + _, stderr, err := runHeadscale(t, env, args...) + + exitErr, ok := errors.AsType[*exec.ExitError](err) + require.True(t, ok, "want exit %d, got %v; stderr:\n%s", tt.want, err, stderr) + assert.Equal(t, tt.want, exitErr.ExitCode(), stderr) + assert.Contains(t, stderr, tt.stderr) + }) + } +} diff --git a/hscontrol/config/keys.go b/hscontrol/config/keys.go index 81131a17b..75e351204 100644 --- a/hscontrol/config/keys.go +++ b/hscontrol/config/keys.go @@ -1,6 +1,11 @@ package config import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "regexp" "slices" "strings" @@ -98,6 +103,45 @@ func dottedPosition(n *yaml.Node, prefix string, path []string) (int, int, bool) return 0, 0, false } +// yamlSyntaxError is the shape of a YAML syntax error, whose text after the +// line is one of libyaml's fixed problem descriptions. +var yamlSyntaxError = regexp.MustCompile(`^yaml: line [0-9]+: `) + +// parseDetail is where and why err, from parsing b, a configuration file in +// format ext, found it invalid, in words that quote nothing from b: parser +// messages may quote a value, and a configuration holds secrets. +func parseDetail(ext string, b []byte, err error) string { + var ( + jsonSyntax *json.SyntaxError + jsonType *json.UnmarshalTypeError + yamlType *yaml.TypeError + line, col int + ) + + switch ext = strings.ToLower(ext); { + case errors.As(err, &jsonSyntax): + // Offset counts the byte at fault. + n := min(max(int(jsonSyntax.Offset)-1, 0), len(b)) + line = 1 + bytes.Count(b[:n], []byte("\n")) + col = n - bytes.LastIndexByte(b[:n], '\n') + case errors.As(err, &jsonType), errors.As(err, &yamlType): + // Also a YAML key given twice; both messages quote the file. + return "the file is not a map of unique keys" + case (ext == ".yaml" || ext == ".yml") && yamlSyntaxError.MatchString(err.Error()): + return err.Error() + case ext == ".hujson": + // hujson has no error type, only this prefix before its message. + _, _ = fmt.Sscanf(err.Error(), "hujson: line %d, column %d:", &line, &col) + } + + what := map[string]string{".json": "JSON", ".hujson": "HuJSON", ".yaml": "YAML", ".yml": "YAML"}[ext] + if line == 0 { + return "not valid " + what + } + + return fmt.Sprintf("line %d, column %d: not valid %s", line, col, what) +} + // removedKey is a key that once worked and is now rejected. In files Old // matches the key and everything beneath it; for environment variables only // the exact name, so the old dns_config.* leaves are listed too. New holds diff --git a/hscontrol/config/load.go b/hscontrol/config/load.go index 660f0a244..0d4aedc93 100644 --- a/hscontrol/config/load.go +++ b/hscontrol/config/load.go @@ -478,10 +478,12 @@ func newConfigSource(path string, isFile bool, environ []string) (*configSource, return s, cmp.Or(fileErr, envErr) } +// loadFile reads and parses s.file. What the file says fails as a +// ConfigError, which serve does not retry; only reading it fails otherwise. func (s *configSource) loadFile() error { parser, err := configParser(s.file) if err != nil { - return fmt.Errorf("%w (%s)", err, s.file) + return &types.ConfigError{Reason: fmt.Sprintf("%s (%s)", err, s.file), Cause: err} } raw, err := os.ReadFile(s.file) @@ -508,7 +510,10 @@ func (s *configSource) loadFile() error { } if err != nil { - return fmt.Errorf("fatal error reading config file %s: %w", s.file, err) + return &types.ConfigError{ + Reason: "cannot parse config file " + s.file, + Detail: parseDetail(filepath.Ext(s.file), s.fileBytes, err), + } } s.findNulls(m, "") diff --git a/hscontrol/config/load_test.go b/hscontrol/config/load_test.go index 863cbd8e6..4b6bf10c1 100644 --- a/hscontrol/config/load_test.go +++ b/hscontrol/config/load_test.go @@ -204,7 +204,7 @@ func TestConfigSource(t *testing.T) { name: "hujson syntax error", fileName: "config.hujson", file: "{\n // c\n \"server_url\": x,\n}", - errs: []string{"config.hujson", "hujson: line 3, column 16"}, + errs: []string{"config.hujson", "line 3, column 16: not valid HuJSON"}, }, { name: "toml is found and rejected", @@ -345,6 +345,39 @@ func TestConfigSource(t *testing.T) { } } +// TestFileContentErrors: a file that cannot be parsed or has an unsupported +// format is a ConfigError, so serve does not retry it, and the error quotes +// nothing from the file, which may hold secrets, whatever the parser says. +func TestFileContentErrors(t *testing.T) { + for _, tt := range []struct{ name, file, content, want string }{ + {name: "yaml syntax", file: "config.yaml", content: "oidc:\n client_secret: marker-secret\nderp: [\n", want: "yaml: line 3: did not find expected node content"}, + {name: "yaml scalar document", file: "config.yaml", content: "marker-secret\n", want: "why: the file is not a map of unique keys"}, + {name: "yaml duplicate key", file: "config.yaml", content: "marker-secret: 1\nmarker-secret: 2\n", want: "why: the file is not a map of unique keys"}, + {name: "yaml unknown anchor", file: "config.yaml", content: "server_url: *marker-secret\n", want: "why: not valid YAML"}, + {name: "hujson literal", file: "config.hujson", content: "{\"server_url\": marker-secret}\n", want: "why: line 1, column 16: not valid HuJSON"}, + {name: "json literal", file: "config.json", content: "{\"server_url\": marker-secret}\n", want: "why: line 1, column 16: not valid JSON"}, + {name: "json scalar document", file: "config.json", content: "\"marker-secret\"\n", want: "why: the file is not a map of unique keys"}, + {name: "unsupported format", file: "config.ini", content: "a=marker-secret\n", want: "unsupported config file format"}, + } { + t.Run(tt.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), tt.file) + require.NoError(t, os.WriteFile(path, []byte(tt.content), 0o600)) + + _, err := Load(path, true, nil) + require.ErrorIs(t, err, types.ErrConfig) + require.ErrorContains(t, err, tt.want) + // Not even a truncated value: YAML cuts a quoted value to 7 characters. + assert.NotContains(t, err.Error(), "marker") + }) + } + + t.Run("unreadable", func(t *testing.T) { + _, err := Load(filepath.Join(t.TempDir(), "config.yaml"), true, nil) + require.ErrorIs(t, err, fs.ErrNotExist) + assert.NotErrorIs(t, err, types.ErrConfig) + }) +} + const yamlSample = "server_url: https://x\nnode:\n expiry: 1h\ndns:\n magic_dns: false\n nameservers:\n global: [1.1.1.1, 8.8.8.8]\noidc:\n extra_params:\n domain_hint: example.com\n" func checkSameValues(t *testing.T, cfg *types.Config) { diff --git a/nix/module.nix b/nix/module.nix index dd964e74e..f6157d4fe 100644 --- a/nix/module.nix +++ b/nix/module.nix @@ -305,6 +305,9 @@ in ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; Restart = "always"; RestartSec = "5s"; + # headscale serve exits 78 (EX_CONFIG) on a configuration it rejects, + # which a restart cannot fix. + RestartPreventExitStatus = 78; Type = "simple"; # Read here, so setting either read-only option fails evaluation. User = cfg.user; diff --git a/nix/tests/checks.nix b/nix/tests/checks.nix index 46b45715e..ac5d4b1cf 100644 --- a/nix/tests/checks.nix +++ b/nix/tests/checks.nix @@ -119,7 +119,8 @@ in settings.policy.path = lib.mkForce mutablePolicy; check.config = false; }; - # Builds only because check.config = false also drops the build check. + # Builds only because check.config = false also drops the build check; + # headscale itself refuses it. unchecked = { settings.server_url = badServerUrl; check = { @@ -240,6 +241,18 @@ in want = headscale.succeed("${jq} -S . ${testsPass}") assert served_policy() == want, "new policy not served" + with subtest("a configuration headscale rejects is not restarted"): + # The switch may report the unit failed; the unit is what counts. + headscale.execute(stc("unchecked", "test")) + headscale.wait_until_succeeds("systemctl is-failed headscale.service", timeout=timedelta(seconds=60)) + status = headscale.succeed("systemctl show -P ExecMainStatus headscale.service").strip() + assert status == "78", f"exit status {status}, want 78" + restarts = headscale.succeed("systemctl show -P NRestarts headscale.service") + # Past RestartSec. + headscale.sleep(10) + headscale.succeed("systemctl is-failed headscale.service") + assert headscale.succeed("systemctl show -P NRestarts headscale.service") == restarts, "headscale was restarted" + # Last: it takes headscale's account away. with subtest("an account activation has yet to create skips the checks"): headscale.succeed("systemctl stop headscale.service") diff --git a/packaging/deb/postinst b/packaging/deb/postinst index 9ba44c181..a6aa64515 100644 --- a/packaging/deb/postinst +++ b/packaging/deb/postinst @@ -53,7 +53,7 @@ case "$1" in else if [ -n "$2" ]; then echo 'headscale was not restarted after the upgrade: its configuration does not pass "headscale check config" (errors above).' >&2 - echo 'A running headscale keeps the previous version only until it restarts: needrestart after apt, a reboot or a crash starts this version, which refuses the configuration.' >&2 + echo 'A running headscale keeps the previous version only until it restarts: needrestart after apt, a reboot or a crash starts this version, which refuses the configuration and stays stopped until it is fixed.' >&2 else echo 'headscale was not started: its configuration does not pass "headscale check config" (errors above).' >&2 fi diff --git a/packaging/systemd/headscale.service b/packaging/systemd/headscale.service index 9d3b69712..4a0f3e094 100644 --- a/packaging/systemd/headscale.service +++ b/packaging/systemd/headscale.service @@ -10,6 +10,9 @@ ExecStart=/usr/bin/headscale serve ExecReload=/usr/bin/kill -HUP $MAINPID Restart=always RestartSec=5 +# headscale serve exits 78 (EX_CONFIG) on a configuration it rejects, which +# a restart cannot fix. +RestartPreventExitStatus=78 WorkingDirectory=/var/lib/headscale ReadWritePaths=/var/lib/headscale