diff --git a/cmd/headscale/cli/root.go b/cmd/headscale/cli/root.go index 0e2858820..a84062fd9 100644 --- a/cmd/headscale/cli/root.go +++ b/cmd/headscale/cli/root.go @@ -112,7 +112,10 @@ func initConfig() { if err != nil { // No log.format applies: main's console logger, on stderr. logWarnings(l.Warnings) - log.Fatal().Caller().Err(err).Msg("error loading config") + log.WithLevel(zerolog.FatalLevel).Caller().Err(err).Msg("error loading config") + + cmd, _, _ := rootCmd.Find(os.Args[1:]) + os.Exit(exitCode(cmd, l.File, err)) } loaded = l @@ -201,6 +204,12 @@ func Execute() { if err != nil { outputFormat, _ := cmd.Flags().GetString("output") printError(err, outputFormat) - os.Exit(1) + + var file string + if loaded != nil { + file = loaded.File + } + + os.Exit(exitCode(cmd, file, err)) } } diff --git a/cmd/headscale/cli/serve.go b/cmd/headscale/cli/serve.go index 2698e3dea..8eacda4a7 100644 --- a/cmd/headscale/cli/serve.go +++ b/cmd/headscale/cli/serve.go @@ -42,6 +42,24 @@ var serveCmd = &cobra.Command{ }, } +// exitConfig is EX_CONFIG from sysexits.h. serve exits with it when its +// configuration is the cause, which a restart with the same inputs cannot +// fix, so the systemd units set RestartPreventExitStatus to it. +const exitConfig = 78 + +// exitCode is how cmd exits on err, file being the configuration file it +// loaded: exitConfig when serve stops on what that configuration says; 1 +// otherwise. Without a file, the one expected may be on a mount not yet up, +// and a database, a port or a file the configuration names may yet become +// available, so systemd keeps retrying those. +func exitCode(cmd *cobra.Command, file string, err error) int { + if cmd == serveCmd && file != "" && errors.Is(err, types.ErrConfig) { + return exitConfig + } + + return 1 +} + // logNoConfigFile says when the server runs without a file. The loader does // not warn: client commands need no file unless they use an API key. func logNoConfigFile(l *config.Loaded) { diff --git a/cmd/headscale/cli/serve_test.go b/cmd/headscale/cli/serve_test.go index 23fc5a9b4..00dc86891 100644 --- a/cmd/headscale/cli/serve_test.go +++ b/cmd/headscale/cli/serve_test.go @@ -6,6 +6,9 @@ import ( "errors" "fmt" "net" + "os" + "os/exec" + "path/filepath" "strings" "syscall" "testing" @@ -172,3 +175,70 @@ func TestLogNoConfigFile(t *testing.T) { }) } } + +// TestServeExitCode: serve exits with exitConfig only when the configuration +// file it loaded is the cause, so systemd stops restarting it; a failure that +// can heal, no file at all, and every other command exit with 1. +func TestServeExitCode(t *testing.T) { + dir := t.TempDir() + notADir := writeFile(t, dir, "file", "") + + const valid = "server_url: https://headscale.example.com\ndisable_check_updates: true\n" + + tests := []struct { + name string + file string // the config file's name, config.yaml when "" + config string // written to file; "" passes a path that does not exist + search bool // no -c: the file is searched for, in ~/.headscale + args []string + want int + stderr string + }{ + {name: "rejected value", config: "server_url: ftp://headscale.example.com\ndisable_check_updates: true\n", args: []string{"serve"}, want: exitConfig, stderr: "server_url"}, + {name: "unknown key", config: valid + "bogus_key: 1\n", args: []string{"serve"}, want: exitConfig, stderr: "unknown configuration key bogus_key"}, + {name: "syntax error", config: valid + "derp: [\n", args: []string{"serve"}, want: exitConfig, stderr: "yaml: line 3"}, + {name: "missing file", args: []string{"serve"}, want: 1, stderr: "error loading config"}, + {name: "no file found", search: true, args: []string{"serve"}, want: 1, stderr: "server_url is required"}, + {name: "old format found", file: "config.toml", config: "server_url = \"https://headscale.example.com\"\n", search: true, args: []string{"serve"}, want: exitConfig, stderr: "unsupported config file format"}, + {name: "unwritable noise key", config: valid + "noise:\n private_key_path: " + notADir + "/noise.key\n", args: []string{"serve"}, want: 1, stderr: "Noise"}, + {name: "other command", config: valid + "bogus_key: 1\n", args: []string{"users", "list"}, want: 1, stderr: "unknown configuration key bogus_key"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + home, dir, name := t.TempDir(), t.TempDir(), tt.file + if name == "" { + name = "config.yaml" + } + + if tt.search { + dir = filepath.Join(home, ".headscale") + require.NoError(t, os.Mkdir(dir, 0o700)) + } + + path := filepath.Join(dir, "absent", name) + if tt.config != "" { + path = writeFile(t, dir, name, tt.config) + } + + args := append([]string{"-c", path}, tt.args...) + if tt.search { + // The search always looks in /etc/headscale; a host config there + // would be found, or serve would start on it. + if found, _ := filepath.Glob("/etc/headscale/config.*"); len(found) > 0 { + t.Skipf("the search would find %v", found) + } + + args = tt.args + } + + env := []string{"HOME=" + home, "XDG_CONFIG_DIRS=" + t.TempDir(), "HEADSCALE_DISABLE_CHECK_UPDATES=true"} + _, stderr, err := runHeadscale(t, env, args...) + + exitErr, ok := errors.AsType[*exec.ExitError](err) + require.True(t, ok, "want exit %d, got %v; stderr:\n%s", tt.want, err, stderr) + assert.Equal(t, tt.want, exitErr.ExitCode(), stderr) + assert.Contains(t, stderr, tt.stderr) + }) + } +} diff --git a/hscontrol/config/keys.go b/hscontrol/config/keys.go index 81131a17b..75e351204 100644 --- a/hscontrol/config/keys.go +++ b/hscontrol/config/keys.go @@ -1,6 +1,11 @@ package config import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "regexp" "slices" "strings" @@ -98,6 +103,45 @@ func dottedPosition(n *yaml.Node, prefix string, path []string) (int, int, bool) return 0, 0, false } +// yamlSyntaxError is the shape of a YAML syntax error, whose text after the +// line is one of libyaml's fixed problem descriptions. +var yamlSyntaxError = regexp.MustCompile(`^yaml: line [0-9]+: `) + +// parseDetail is where and why err, from parsing b, a configuration file in +// format ext, found it invalid, in words that quote nothing from b: parser +// messages may quote a value, and a configuration holds secrets. +func parseDetail(ext string, b []byte, err error) string { + var ( + jsonSyntax *json.SyntaxError + jsonType *json.UnmarshalTypeError + yamlType *yaml.TypeError + line, col int + ) + + switch ext = strings.ToLower(ext); { + case errors.As(err, &jsonSyntax): + // Offset counts the byte at fault. + n := min(max(int(jsonSyntax.Offset)-1, 0), len(b)) + line = 1 + bytes.Count(b[:n], []byte("\n")) + col = n - bytes.LastIndexByte(b[:n], '\n') + case errors.As(err, &jsonType), errors.As(err, &yamlType): + // Also a YAML key given twice; both messages quote the file. + return "the file is not a map of unique keys" + case (ext == ".yaml" || ext == ".yml") && yamlSyntaxError.MatchString(err.Error()): + return err.Error() + case ext == ".hujson": + // hujson has no error type, only this prefix before its message. + _, _ = fmt.Sscanf(err.Error(), "hujson: line %d, column %d:", &line, &col) + } + + what := map[string]string{".json": "JSON", ".hujson": "HuJSON", ".yaml": "YAML", ".yml": "YAML"}[ext] + if line == 0 { + return "not valid " + what + } + + return fmt.Sprintf("line %d, column %d: not valid %s", line, col, what) +} + // removedKey is a key that once worked and is now rejected. In files Old // matches the key and everything beneath it; for environment variables only // the exact name, so the old dns_config.* leaves are listed too. New holds diff --git a/hscontrol/config/load.go b/hscontrol/config/load.go index 660f0a244..0d4aedc93 100644 --- a/hscontrol/config/load.go +++ b/hscontrol/config/load.go @@ -478,10 +478,12 @@ func newConfigSource(path string, isFile bool, environ []string) (*configSource, return s, cmp.Or(fileErr, envErr) } +// loadFile reads and parses s.file. What the file says fails as a +// ConfigError, which serve does not retry; only reading it fails otherwise. func (s *configSource) loadFile() error { parser, err := configParser(s.file) if err != nil { - return fmt.Errorf("%w (%s)", err, s.file) + return &types.ConfigError{Reason: fmt.Sprintf("%s (%s)", err, s.file), Cause: err} } raw, err := os.ReadFile(s.file) @@ -508,7 +510,10 @@ func (s *configSource) loadFile() error { } if err != nil { - return fmt.Errorf("fatal error reading config file %s: %w", s.file, err) + return &types.ConfigError{ + Reason: "cannot parse config file " + s.file, + Detail: parseDetail(filepath.Ext(s.file), s.fileBytes, err), + } } s.findNulls(m, "") diff --git a/hscontrol/config/load_test.go b/hscontrol/config/load_test.go index 863cbd8e6..4b6bf10c1 100644 --- a/hscontrol/config/load_test.go +++ b/hscontrol/config/load_test.go @@ -204,7 +204,7 @@ func TestConfigSource(t *testing.T) { name: "hujson syntax error", fileName: "config.hujson", file: "{\n // c\n \"server_url\": x,\n}", - errs: []string{"config.hujson", "hujson: line 3, column 16"}, + errs: []string{"config.hujson", "line 3, column 16: not valid HuJSON"}, }, { name: "toml is found and rejected", @@ -345,6 +345,39 @@ func TestConfigSource(t *testing.T) { } } +// TestFileContentErrors: a file that cannot be parsed or has an unsupported +// format is a ConfigError, so serve does not retry it, and the error quotes +// nothing from the file, which may hold secrets, whatever the parser says. +func TestFileContentErrors(t *testing.T) { + for _, tt := range []struct{ name, file, content, want string }{ + {name: "yaml syntax", file: "config.yaml", content: "oidc:\n client_secret: marker-secret\nderp: [\n", want: "yaml: line 3: did not find expected node content"}, + {name: "yaml scalar document", file: "config.yaml", content: "marker-secret\n", want: "why: the file is not a map of unique keys"}, + {name: "yaml duplicate key", file: "config.yaml", content: "marker-secret: 1\nmarker-secret: 2\n", want: "why: the file is not a map of unique keys"}, + {name: "yaml unknown anchor", file: "config.yaml", content: "server_url: *marker-secret\n", want: "why: not valid YAML"}, + {name: "hujson literal", file: "config.hujson", content: "{\"server_url\": marker-secret}\n", want: "why: line 1, column 16: not valid HuJSON"}, + {name: "json literal", file: "config.json", content: "{\"server_url\": marker-secret}\n", want: "why: line 1, column 16: not valid JSON"}, + {name: "json scalar document", file: "config.json", content: "\"marker-secret\"\n", want: "why: the file is not a map of unique keys"}, + {name: "unsupported format", file: "config.ini", content: "a=marker-secret\n", want: "unsupported config file format"}, + } { + t.Run(tt.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), tt.file) + require.NoError(t, os.WriteFile(path, []byte(tt.content), 0o600)) + + _, err := Load(path, true, nil) + require.ErrorIs(t, err, types.ErrConfig) + require.ErrorContains(t, err, tt.want) + // Not even a truncated value: YAML cuts a quoted value to 7 characters. + assert.NotContains(t, err.Error(), "marker") + }) + } + + t.Run("unreadable", func(t *testing.T) { + _, err := Load(filepath.Join(t.TempDir(), "config.yaml"), true, nil) + require.ErrorIs(t, err, fs.ErrNotExist) + assert.NotErrorIs(t, err, types.ErrConfig) + }) +} + const yamlSample = "server_url: https://x\nnode:\n expiry: 1h\ndns:\n magic_dns: false\n nameservers:\n global: [1.1.1.1, 8.8.8.8]\noidc:\n extra_params:\n domain_hint: example.com\n" func checkSameValues(t *testing.T, cfg *types.Config) { diff --git a/nix/module.nix b/nix/module.nix index dd964e74e..f6157d4fe 100644 --- a/nix/module.nix +++ b/nix/module.nix @@ -305,6 +305,9 @@ in ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; Restart = "always"; RestartSec = "5s"; + # headscale serve exits 78 (EX_CONFIG) on a configuration it rejects, + # which a restart cannot fix. + RestartPreventExitStatus = 78; Type = "simple"; # Read here, so setting either read-only option fails evaluation. User = cfg.user; diff --git a/nix/tests/checks.nix b/nix/tests/checks.nix index 46b45715e..ac5d4b1cf 100644 --- a/nix/tests/checks.nix +++ b/nix/tests/checks.nix @@ -119,7 +119,8 @@ in settings.policy.path = lib.mkForce mutablePolicy; check.config = false; }; - # Builds only because check.config = false also drops the build check. + # Builds only because check.config = false also drops the build check; + # headscale itself refuses it. unchecked = { settings.server_url = badServerUrl; check = { @@ -240,6 +241,18 @@ in want = headscale.succeed("${jq} -S . ${testsPass}") assert served_policy() == want, "new policy not served" + with subtest("a configuration headscale rejects is not restarted"): + # The switch may report the unit failed; the unit is what counts. + headscale.execute(stc("unchecked", "test")) + headscale.wait_until_succeeds("systemctl is-failed headscale.service", timeout=timedelta(seconds=60)) + status = headscale.succeed("systemctl show -P ExecMainStatus headscale.service").strip() + assert status == "78", f"exit status {status}, want 78" + restarts = headscale.succeed("systemctl show -P NRestarts headscale.service") + # Past RestartSec. + headscale.sleep(10) + headscale.succeed("systemctl is-failed headscale.service") + assert headscale.succeed("systemctl show -P NRestarts headscale.service") == restarts, "headscale was restarted" + # Last: it takes headscale's account away. with subtest("an account activation has yet to create skips the checks"): headscale.succeed("systemctl stop headscale.service") diff --git a/packaging/deb/postinst b/packaging/deb/postinst index 9ba44c181..a6aa64515 100644 --- a/packaging/deb/postinst +++ b/packaging/deb/postinst @@ -53,7 +53,7 @@ case "$1" in else if [ -n "$2" ]; then echo 'headscale was not restarted after the upgrade: its configuration does not pass "headscale check config" (errors above).' >&2 - echo 'A running headscale keeps the previous version only until it restarts: needrestart after apt, a reboot or a crash starts this version, which refuses the configuration.' >&2 + echo 'A running headscale keeps the previous version only until it restarts: needrestart after apt, a reboot or a crash starts this version, which refuses the configuration and stays stopped until it is fixed.' >&2 else echo 'headscale was not started: its configuration does not pass "headscale check config" (errors above).' >&2 fi diff --git a/packaging/systemd/headscale.service b/packaging/systemd/headscale.service index 9d3b69712..4a0f3e094 100644 --- a/packaging/systemd/headscale.service +++ b/packaging/systemd/headscale.service @@ -10,6 +10,9 @@ ExecStart=/usr/bin/headscale serve ExecReload=/usr/bin/kill -HUP $MAINPID Restart=always RestartSec=5 +# headscale serve exits 78 (EX_CONFIG) on a configuration it rejects, which +# a restart cannot fix. +RestartPreventExitStatus=78 WorkingDirectory=/var/lib/headscale ReadWritePaths=/var/lib/headscale