config, cli, packaging, nix: stop restarting headscale on a configuration error

serve exits 78 (EX_CONFIG) when the configuration file it loaded is
wrong: format, syntax, keys or values. No file, an unreadable one, the
database and ports still exit 1, so systemd keeps retrying what can
heal. The deb unit and the NixOS module set RestartPreventExitStatus=78.
A parse error gives a position and fixed words, never the file's text.

Updates #1926
This commit is contained in:
Kristoffer Dalby committed 2026-11-11 15:02:16 +01:00
1 parent a23ea307df
commit d2dc13696a
10 files changed
+205 -7

No files matched your search

+11 -2
View File
@@ -112,7 +112,10 @@ func initConfig() {
if err != nil {
// No log.format applies: main's console logger, on stderr.
logWarnings(l.Warnings)
log.Fatal().Caller().Err(err).Msg("error loading config")
log.WithLevel(zerolog.FatalLevel).Caller().Err(err).Msg("error loading config")
cmd, _, _ := rootCmd.Find(os.Args[1:])
os.Exit(exitCode(cmd, l.File, err))
}
loaded = l
@@ -201,6 +204,12 @@ func Execute() {
if err != nil {
outputFormat, _ := cmd.Flags().GetString("output")
printError(err, outputFormat)
os.Exit(1)
var file string
if loaded != nil {
file = loaded.File
}
os.Exit(exitCode(cmd, file, err))
}
}
+18
View File
@@ -42,6 +42,24 @@ var serveCmd = &cobra.Command{
},
}
// exitConfig is EX_CONFIG from sysexits.h. serve exits with it when its
// configuration is the cause, which a restart with the same inputs cannot
// fix, so the systemd units set RestartPreventExitStatus to it.
const exitConfig = 78
// exitCode is how cmd exits on err, file being the configuration file it
// loaded: exitConfig when serve stops on what that configuration says; 1
// otherwise. Without a file, the one expected may be on a mount not yet up,
// and a database, a port or a file the configuration names may yet become
// available, so systemd keeps retrying those.
func exitCode(cmd *cobra.Command, file string, err error) int {
if cmd == serveCmd && file != "" && errors.Is(err, types.ErrConfig) {
return exitConfig
}
return 1
}
// logNoConfigFile says when the server runs without a file. The loader does
// not warn: client commands need no file unless they use an API key.
func logNoConfigFile(l *config.Loaded) {
+70
View File
@@ -6,6 +6,9 @@ import (
"errors"
"fmt"
"net"
"os"
"os/exec"
"path/filepath"
"strings"
"syscall"
"testing"
@@ -172,3 +175,70 @@ func TestLogNoConfigFile(t *testing.T) {
})
}
}
// TestServeExitCode: serve exits with exitConfig only when the configuration
// file it loaded is the cause, so systemd stops restarting it; a failure that
// can heal, no file at all, and every other command exit with 1.
func TestServeExitCode(t *testing.T) {
dir := t.TempDir()
notADir := writeFile(t, dir, "file", "")
const valid = "server_url: https://headscale.example.com\ndisable_check_updates: true\n"
tests := []struct {
name string
file string // the config file's name, config.yaml when ""
config string // written to file; "" passes a path that does not exist
search bool // no -c: the file is searched for, in ~/.headscale
args []string
want int
stderr string
}{
{name: "rejected value", config: "server_url: ftp://headscale.example.com\ndisable_check_updates: true\n", args: []string{"serve"}, want: exitConfig, stderr: "server_url"},
{name: "unknown key", config: valid + "bogus_key: 1\n", args: []string{"serve"}, want: exitConfig, stderr: "unknown configuration key bogus_key"},
{name: "syntax error", config: valid + "derp: [\n", args: []string{"serve"}, want: exitConfig, stderr: "yaml: line 3"},
{name: "missing file", args: []string{"serve"}, want: 1, stderr: "error loading config"},
{name: "no file found", search: true, args: []string{"serve"}, want: 1, stderr: "server_url is required"},
{name: "old format found", file: "config.toml", config: "server_url = \"https://headscale.example.com\"\n", search: true, args: []string{"serve"}, want: exitConfig, stderr: "unsupported config file format"},
{name: "unwritable noise key", config: valid + "noise:\n private_key_path: " + notADir + "/noise.key\n", args: []string{"serve"}, want: 1, stderr: "Noise"},
{name: "other command", config: valid + "bogus_key: 1\n", args: []string{"users", "list"}, want: 1, stderr: "unknown configuration key bogus_key"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
home, dir, name := t.TempDir(), t.TempDir(), tt.file
if name == "" {
name = "config.yaml"
}
if tt.search {
dir = filepath.Join(home, ".headscale")
require.NoError(t, os.Mkdir(dir, 0o700))
}
path := filepath.Join(dir, "absent", name)
if tt.config != "" {
path = writeFile(t, dir, name, tt.config)
}
args := append([]string{"-c", path}, tt.args...)
if tt.search {
// The search always looks in /etc/headscale; a host config there
// would be found, or serve would start on it.
if found, _ := filepath.Glob("/etc/headscale/config.*"); len(found) > 0 {
t.Skipf("the search would find %v", found)
}
args = tt.args
}
env := []string{"HOME=" + home, "XDG_CONFIG_DIRS=" + t.TempDir(), "HEADSCALE_DISABLE_CHECK_UPDATES=true"}
_, stderr, err := runHeadscale(t, env, args...)
exitErr, ok := errors.AsType[*exec.ExitError](err)
require.True(t, ok, "want exit %d, got %v; stderr:\n%s", tt.want, err, stderr)
assert.Equal(t, tt.want, exitErr.ExitCode(), stderr)
assert.Contains(t, stderr, tt.stderr)
})
}
}
+44
View File
@@ -1,6 +1,11 @@
package config
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"regexp"
"slices"
"strings"
@@ -98,6 +103,45 @@ func dottedPosition(n *yaml.Node, prefix string, path []string) (int, int, bool)
return 0, 0, false
}
// yamlSyntaxError is the shape of a YAML syntax error, whose text after the
// line is one of libyaml's fixed problem descriptions.
var yamlSyntaxError = regexp.MustCompile(`^yaml: line [0-9]+: `)
// parseDetail is where and why err, from parsing b, a configuration file in
// format ext, found it invalid, in words that quote nothing from b: parser
// messages may quote a value, and a configuration holds secrets.
func parseDetail(ext string, b []byte, err error) string {
var (
jsonSyntax *json.SyntaxError
jsonType *json.UnmarshalTypeError
yamlType *yaml.TypeError
line, col int
)
switch ext = strings.ToLower(ext); {
case errors.As(err, &jsonSyntax):
// Offset counts the byte at fault.
n := min(max(int(jsonSyntax.Offset)-1, 0), len(b))
line = 1 + bytes.Count(b[:n], []byte("\n"))
col = n - bytes.LastIndexByte(b[:n], '\n')
case errors.As(err, &jsonType), errors.As(err, &yamlType):
// Also a YAML key given twice; both messages quote the file.
return "the file is not a map of unique keys"
case (ext == ".yaml" || ext == ".yml") && yamlSyntaxError.MatchString(err.Error()):
return err.Error()
case ext == ".hujson":
// hujson has no error type, only this prefix before its message.
_, _ = fmt.Sscanf(err.Error(), "hujson: line %d, column %d:", &line, &col)
}
what := map[string]string{".json": "JSON", ".hujson": "HuJSON", ".yaml": "YAML", ".yml": "YAML"}[ext]
if line == 0 {
return "not valid " + what
}
return fmt.Sprintf("line %d, column %d: not valid %s", line, col, what)
}
// removedKey is a key that once worked and is now rejected. In files Old
// matches the key and everything beneath it; for environment variables only
// the exact name, so the old dns_config.* leaves are listed too. New holds
+7 -2
View File
@@ -478,10 +478,12 @@ func newConfigSource(path string, isFile bool, environ []string) (*configSource,
return s, cmp.Or(fileErr, envErr)
}
// loadFile reads and parses s.file. What the file says fails as a
// ConfigError, which serve does not retry; only reading it fails otherwise.
func (s *configSource) loadFile() error {
parser, err := configParser(s.file)
if err != nil {
return fmt.Errorf("%w (%s)", err, s.file)
return &types.ConfigError{Reason: fmt.Sprintf("%s (%s)", err, s.file), Cause: err}
}
raw, err := os.ReadFile(s.file)
@@ -508,7 +510,10 @@ func (s *configSource) loadFile() error {
}
if err != nil {
return fmt.Errorf("fatal error reading config file %s: %w", s.file, err)
return &types.ConfigError{
Reason: "cannot parse config file " + s.file,
Detail: parseDetail(filepath.Ext(s.file), s.fileBytes, err),
}
}
s.findNulls(m, "")
+34 -1
View File
@@ -204,7 +204,7 @@ func TestConfigSource(t *testing.T) {
name: "hujson syntax error",
fileName: "config.hujson",
file: "{\n // c\n \"server_url\": x,\n}",
errs: []string{"config.hujson", "hujson: line 3, column 16"},
errs: []string{"config.hujson", "line 3, column 16: not valid HuJSON"},
},
{
name: "toml is found and rejected",
@@ -345,6 +345,39 @@ func TestConfigSource(t *testing.T) {
}
}
// TestFileContentErrors: a file that cannot be parsed or has an unsupported
// format is a ConfigError, so serve does not retry it, and the error quotes
// nothing from the file, which may hold secrets, whatever the parser says.
func TestFileContentErrors(t *testing.T) {
for _, tt := range []struct{ name, file, content, want string }{
{name: "yaml syntax", file: "config.yaml", content: "oidc:\n client_secret: marker-secret\nderp: [\n", want: "yaml: line 3: did not find expected node content"},
{name: "yaml scalar document", file: "config.yaml", content: "marker-secret\n", want: "why: the file is not a map of unique keys"},
{name: "yaml duplicate key", file: "config.yaml", content: "marker-secret: 1\nmarker-secret: 2\n", want: "why: the file is not a map of unique keys"},
{name: "yaml unknown anchor", file: "config.yaml", content: "server_url: *marker-secret\n", want: "why: not valid YAML"},
{name: "hujson literal", file: "config.hujson", content: "{\"server_url\": marker-secret}\n", want: "why: line 1, column 16: not valid HuJSON"},
{name: "json literal", file: "config.json", content: "{\"server_url\": marker-secret}\n", want: "why: line 1, column 16: not valid JSON"},
{name: "json scalar document", file: "config.json", content: "\"marker-secret\"\n", want: "why: the file is not a map of unique keys"},
{name: "unsupported format", file: "config.ini", content: "a=marker-secret\n", want: "unsupported config file format"},
} {
t.Run(tt.name, func(t *testing.T) {
path := filepath.Join(t.TempDir(), tt.file)
require.NoError(t, os.WriteFile(path, []byte(tt.content), 0o600))
_, err := Load(path, true, nil)
require.ErrorIs(t, err, types.ErrConfig)
require.ErrorContains(t, err, tt.want)
// Not even a truncated value: YAML cuts a quoted value to 7 characters.
assert.NotContains(t, err.Error(), "marker")
})
}
t.Run("unreadable", func(t *testing.T) {
_, err := Load(filepath.Join(t.TempDir(), "config.yaml"), true, nil)
require.ErrorIs(t, err, fs.ErrNotExist)
assert.NotErrorIs(t, err, types.ErrConfig)
})
}
const yamlSample = "server_url: https://x\nnode:\n expiry: 1h\ndns:\n magic_dns: false\n nameservers:\n global: [1.1.1.1, 8.8.8.8]\noidc:\n extra_params:\n domain_hint: example.com\n"
func checkSameValues(t *testing.T, cfg *types.Config) {
+3
View File
@@ -305,6 +305,9 @@ in
ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
Restart = "always";
RestartSec = "5s";
# headscale serve exits 78 (EX_CONFIG) on a configuration it rejects,
# which a restart cannot fix.
RestartPreventExitStatus = 78;
Type = "simple";
# Read here, so setting either read-only option fails evaluation.
User = cfg.user;
+14 -1
View File
@@ -119,7 +119,8 @@ in
settings.policy.path = lib.mkForce mutablePolicy;
check.config = false;
};
# Builds only because check.config = false also drops the build check.
# Builds only because check.config = false also drops the build check;
# headscale itself refuses it.
unchecked = {
settings.server_url = badServerUrl;
check = {
@@ -240,6 +241,18 @@ in
want = headscale.succeed("${jq} -S . ${testsPass}")
assert served_policy() == want, "new policy not served"
with subtest("a configuration headscale rejects is not restarted"):
# The switch may report the unit failed; the unit is what counts.
headscale.execute(stc("unchecked", "test"))
headscale.wait_until_succeeds("systemctl is-failed headscale.service", timeout=timedelta(seconds=60))
status = headscale.succeed("systemctl show -P ExecMainStatus headscale.service").strip()
assert status == "78", f"exit status {status}, want 78"
restarts = headscale.succeed("systemctl show -P NRestarts headscale.service")
# Past RestartSec.
headscale.sleep(10)
headscale.succeed("systemctl is-failed headscale.service")
assert headscale.succeed("systemctl show -P NRestarts headscale.service") == restarts, "headscale was restarted"
# Last: it takes headscale's account away.
with subtest("an account activation has yet to create skips the checks"):
headscale.succeed("systemctl stop headscale.service")
+1 -1
View File
@@ -53,7 +53,7 @@ case "$1" in
else
if [ -n "$2" ]; then
echo 'headscale was not restarted after the upgrade: its configuration does not pass "headscale check config" (errors above).' >&2
echo 'A running headscale keeps the previous version only until it restarts: needrestart after apt, a reboot or a crash starts this version, which refuses the configuration.' >&2
echo 'A running headscale keeps the previous version only until it restarts: needrestart after apt, a reboot or a crash starts this version, which refuses the configuration and stays stopped until it is fixed.' >&2
else
echo 'headscale was not started: its configuration does not pass "headscale check config" (errors above).' >&2
fi
+3
View File
@@ -10,6 +10,9 @@ ExecStart=/usr/bin/headscale serve
ExecReload=/usr/bin/kill -HUP $MAINPID
Restart=always
RestartSec=5
# headscale serve exits 78 (EX_CONFIG) on a configuration it rejects, which
# a restart cannot fix.
RestartPreventExitStatus=78
WorkingDirectory=/var/lib/headscale
ReadWritePaths=/var/lib/headscale