mirror of
https://github.com/juanfont/headscale.git
synced 2026-11-20 04:14:32 +09:00
config, cli, packaging, nix: stop restarting headscale on a configuration error
serve exits 78 (EX_CONFIG) when the configuration file it loaded is wrong: format, syntax, keys or values. No file, an unreadable one, the database and ports still exit 1, so systemd keeps retrying what can heal. The deb unit and the NixOS module set RestartPreventExitStatus=78. A parse error gives a position and fixed words, never the file's text. Updates #1926
This commit is contained in:
1 parent
a23ea307df
commit
d2dc13696a
10 files changed
+205
-7
No files matched your search
@@ -112,7 +112,10 @@ func initConfig() {
|
||||
if err != nil {
|
||||
// No log.format applies: main's console logger, on stderr.
|
||||
logWarnings(l.Warnings)
|
||||
log.Fatal().Caller().Err(err).Msg("error loading config")
|
||||
log.WithLevel(zerolog.FatalLevel).Caller().Err(err).Msg("error loading config")
|
||||
|
||||
cmd, _, _ := rootCmd.Find(os.Args[1:])
|
||||
os.Exit(exitCode(cmd, l.File, err))
|
||||
}
|
||||
|
||||
loaded = l
|
||||
@@ -201,6 +204,12 @@ func Execute() {
|
||||
if err != nil {
|
||||
outputFormat, _ := cmd.Flags().GetString("output")
|
||||
printError(err, outputFormat)
|
||||
os.Exit(1)
|
||||
|
||||
var file string
|
||||
if loaded != nil {
|
||||
file = loaded.File
|
||||
}
|
||||
|
||||
os.Exit(exitCode(cmd, file, err))
|
||||
}
|
||||
}
|
||||
@@ -42,6 +42,24 @@ var serveCmd = &cobra.Command{
|
||||
},
|
||||
}
|
||||
|
||||
// exitConfig is EX_CONFIG from sysexits.h. serve exits with it when its
|
||||
// configuration is the cause, which a restart with the same inputs cannot
|
||||
// fix, so the systemd units set RestartPreventExitStatus to it.
|
||||
const exitConfig = 78
|
||||
|
||||
// exitCode is how cmd exits on err, file being the configuration file it
|
||||
// loaded: exitConfig when serve stops on what that configuration says; 1
|
||||
// otherwise. Without a file, the one expected may be on a mount not yet up,
|
||||
// and a database, a port or a file the configuration names may yet become
|
||||
// available, so systemd keeps retrying those.
|
||||
func exitCode(cmd *cobra.Command, file string, err error) int {
|
||||
if cmd == serveCmd && file != "" && errors.Is(err, types.ErrConfig) {
|
||||
return exitConfig
|
||||
}
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
// logNoConfigFile says when the server runs without a file. The loader does
|
||||
// not warn: client commands need no file unless they use an API key.
|
||||
func logNoConfigFile(l *config.Loaded) {
|
||||
|
||||
@@ -6,6 +6,9 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
@@ -172,3 +175,70 @@ func TestLogNoConfigFile(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestServeExitCode: serve exits with exitConfig only when the configuration
|
||||
// file it loaded is the cause, so systemd stops restarting it; a failure that
|
||||
// can heal, no file at all, and every other command exit with 1.
|
||||
func TestServeExitCode(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
notADir := writeFile(t, dir, "file", "")
|
||||
|
||||
const valid = "server_url: https://headscale.example.com\ndisable_check_updates: true\n"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
file string // the config file's name, config.yaml when ""
|
||||
config string // written to file; "" passes a path that does not exist
|
||||
search bool // no -c: the file is searched for, in ~/.headscale
|
||||
args []string
|
||||
want int
|
||||
stderr string
|
||||
}{
|
||||
{name: "rejected value", config: "server_url: ftp://headscale.example.com\ndisable_check_updates: true\n", args: []string{"serve"}, want: exitConfig, stderr: "server_url"},
|
||||
{name: "unknown key", config: valid + "bogus_key: 1\n", args: []string{"serve"}, want: exitConfig, stderr: "unknown configuration key bogus_key"},
|
||||
{name: "syntax error", config: valid + "derp: [\n", args: []string{"serve"}, want: exitConfig, stderr: "yaml: line 3"},
|
||||
{name: "missing file", args: []string{"serve"}, want: 1, stderr: "error loading config"},
|
||||
{name: "no file found", search: true, args: []string{"serve"}, want: 1, stderr: "server_url is required"},
|
||||
{name: "old format found", file: "config.toml", config: "server_url = \"https://headscale.example.com\"\n", search: true, args: []string{"serve"}, want: exitConfig, stderr: "unsupported config file format"},
|
||||
{name: "unwritable noise key", config: valid + "noise:\n private_key_path: " + notADir + "/noise.key\n", args: []string{"serve"}, want: 1, stderr: "Noise"},
|
||||
{name: "other command", config: valid + "bogus_key: 1\n", args: []string{"users", "list"}, want: 1, stderr: "unknown configuration key bogus_key"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
home, dir, name := t.TempDir(), t.TempDir(), tt.file
|
||||
if name == "" {
|
||||
name = "config.yaml"
|
||||
}
|
||||
|
||||
if tt.search {
|
||||
dir = filepath.Join(home, ".headscale")
|
||||
require.NoError(t, os.Mkdir(dir, 0o700))
|
||||
}
|
||||
|
||||
path := filepath.Join(dir, "absent", name)
|
||||
if tt.config != "" {
|
||||
path = writeFile(t, dir, name, tt.config)
|
||||
}
|
||||
|
||||
args := append([]string{"-c", path}, tt.args...)
|
||||
if tt.search {
|
||||
// The search always looks in /etc/headscale; a host config there
|
||||
// would be found, or serve would start on it.
|
||||
if found, _ := filepath.Glob("/etc/headscale/config.*"); len(found) > 0 {
|
||||
t.Skipf("the search would find %v", found)
|
||||
}
|
||||
|
||||
args = tt.args
|
||||
}
|
||||
|
||||
env := []string{"HOME=" + home, "XDG_CONFIG_DIRS=" + t.TempDir(), "HEADSCALE_DISABLE_CHECK_UPDATES=true"}
|
||||
_, stderr, err := runHeadscale(t, env, args...)
|
||||
|
||||
exitErr, ok := errors.AsType[*exec.ExitError](err)
|
||||
require.True(t, ok, "want exit %d, got %v; stderr:\n%s", tt.want, err, stderr)
|
||||
assert.Equal(t, tt.want, exitErr.ExitCode(), stderr)
|
||||
assert.Contains(t, stderr, tt.stderr)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,11 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
@@ -98,6 +103,45 @@ func dottedPosition(n *yaml.Node, prefix string, path []string) (int, int, bool)
|
||||
return 0, 0, false
|
||||
}
|
||||
|
||||
// yamlSyntaxError is the shape of a YAML syntax error, whose text after the
|
||||
// line is one of libyaml's fixed problem descriptions.
|
||||
var yamlSyntaxError = regexp.MustCompile(`^yaml: line [0-9]+: `)
|
||||
|
||||
// parseDetail is where and why err, from parsing b, a configuration file in
|
||||
// format ext, found it invalid, in words that quote nothing from b: parser
|
||||
// messages may quote a value, and a configuration holds secrets.
|
||||
func parseDetail(ext string, b []byte, err error) string {
|
||||
var (
|
||||
jsonSyntax *json.SyntaxError
|
||||
jsonType *json.UnmarshalTypeError
|
||||
yamlType *yaml.TypeError
|
||||
line, col int
|
||||
)
|
||||
|
||||
switch ext = strings.ToLower(ext); {
|
||||
case errors.As(err, &jsonSyntax):
|
||||
// Offset counts the byte at fault.
|
||||
n := min(max(int(jsonSyntax.Offset)-1, 0), len(b))
|
||||
line = 1 + bytes.Count(b[:n], []byte("\n"))
|
||||
col = n - bytes.LastIndexByte(b[:n], '\n')
|
||||
case errors.As(err, &jsonType), errors.As(err, &yamlType):
|
||||
// Also a YAML key given twice; both messages quote the file.
|
||||
return "the file is not a map of unique keys"
|
||||
case (ext == ".yaml" || ext == ".yml") && yamlSyntaxError.MatchString(err.Error()):
|
||||
return err.Error()
|
||||
case ext == ".hujson":
|
||||
// hujson has no error type, only this prefix before its message.
|
||||
_, _ = fmt.Sscanf(err.Error(), "hujson: line %d, column %d:", &line, &col)
|
||||
}
|
||||
|
||||
what := map[string]string{".json": "JSON", ".hujson": "HuJSON", ".yaml": "YAML", ".yml": "YAML"}[ext]
|
||||
if line == 0 {
|
||||
return "not valid " + what
|
||||
}
|
||||
|
||||
return fmt.Sprintf("line %d, column %d: not valid %s", line, col, what)
|
||||
}
|
||||
|
||||
// removedKey is a key that once worked and is now rejected. In files Old
|
||||
// matches the key and everything beneath it; for environment variables only
|
||||
// the exact name, so the old dns_config.* leaves are listed too. New holds
|
||||
|
||||
@@ -478,10 +478,12 @@ func newConfigSource(path string, isFile bool, environ []string) (*configSource,
|
||||
return s, cmp.Or(fileErr, envErr)
|
||||
}
|
||||
|
||||
// loadFile reads and parses s.file. What the file says fails as a
|
||||
// ConfigError, which serve does not retry; only reading it fails otherwise.
|
||||
func (s *configSource) loadFile() error {
|
||||
parser, err := configParser(s.file)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w (%s)", err, s.file)
|
||||
return &types.ConfigError{Reason: fmt.Sprintf("%s (%s)", err, s.file), Cause: err}
|
||||
}
|
||||
|
||||
raw, err := os.ReadFile(s.file)
|
||||
@@ -508,7 +510,10 @@ func (s *configSource) loadFile() error {
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("fatal error reading config file %s: %w", s.file, err)
|
||||
return &types.ConfigError{
|
||||
Reason: "cannot parse config file " + s.file,
|
||||
Detail: parseDetail(filepath.Ext(s.file), s.fileBytes, err),
|
||||
}
|
||||
}
|
||||
|
||||
s.findNulls(m, "")
|
||||
|
||||
@@ -204,7 +204,7 @@ func TestConfigSource(t *testing.T) {
|
||||
name: "hujson syntax error",
|
||||
fileName: "config.hujson",
|
||||
file: "{\n // c\n \"server_url\": x,\n}",
|
||||
errs: []string{"config.hujson", "hujson: line 3, column 16"},
|
||||
errs: []string{"config.hujson", "line 3, column 16: not valid HuJSON"},
|
||||
},
|
||||
{
|
||||
name: "toml is found and rejected",
|
||||
@@ -345,6 +345,39 @@ func TestConfigSource(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestFileContentErrors: a file that cannot be parsed or has an unsupported
|
||||
// format is a ConfigError, so serve does not retry it, and the error quotes
|
||||
// nothing from the file, which may hold secrets, whatever the parser says.
|
||||
func TestFileContentErrors(t *testing.T) {
|
||||
for _, tt := range []struct{ name, file, content, want string }{
|
||||
{name: "yaml syntax", file: "config.yaml", content: "oidc:\n client_secret: marker-secret\nderp: [\n", want: "yaml: line 3: did not find expected node content"},
|
||||
{name: "yaml scalar document", file: "config.yaml", content: "marker-secret\n", want: "why: the file is not a map of unique keys"},
|
||||
{name: "yaml duplicate key", file: "config.yaml", content: "marker-secret: 1\nmarker-secret: 2\n", want: "why: the file is not a map of unique keys"},
|
||||
{name: "yaml unknown anchor", file: "config.yaml", content: "server_url: *marker-secret\n", want: "why: not valid YAML"},
|
||||
{name: "hujson literal", file: "config.hujson", content: "{\"server_url\": marker-secret}\n", want: "why: line 1, column 16: not valid HuJSON"},
|
||||
{name: "json literal", file: "config.json", content: "{\"server_url\": marker-secret}\n", want: "why: line 1, column 16: not valid JSON"},
|
||||
{name: "json scalar document", file: "config.json", content: "\"marker-secret\"\n", want: "why: the file is not a map of unique keys"},
|
||||
{name: "unsupported format", file: "config.ini", content: "a=marker-secret\n", want: "unsupported config file format"},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), tt.file)
|
||||
require.NoError(t, os.WriteFile(path, []byte(tt.content), 0o600))
|
||||
|
||||
_, err := Load(path, true, nil)
|
||||
require.ErrorIs(t, err, types.ErrConfig)
|
||||
require.ErrorContains(t, err, tt.want)
|
||||
// Not even a truncated value: YAML cuts a quoted value to 7 characters.
|
||||
assert.NotContains(t, err.Error(), "marker")
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("unreadable", func(t *testing.T) {
|
||||
_, err := Load(filepath.Join(t.TempDir(), "config.yaml"), true, nil)
|
||||
require.ErrorIs(t, err, fs.ErrNotExist)
|
||||
assert.NotErrorIs(t, err, types.ErrConfig)
|
||||
})
|
||||
}
|
||||
|
||||
const yamlSample = "server_url: https://x\nnode:\n expiry: 1h\ndns:\n magic_dns: false\n nameservers:\n global: [1.1.1.1, 8.8.8.8]\noidc:\n extra_params:\n domain_hint: example.com\n"
|
||||
|
||||
func checkSameValues(t *testing.T, cfg *types.Config) {
|
||||
|
||||
@@ -305,6 +305,9 @@ in
|
||||
ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
|
||||
Restart = "always";
|
||||
RestartSec = "5s";
|
||||
# headscale serve exits 78 (EX_CONFIG) on a configuration it rejects,
|
||||
# which a restart cannot fix.
|
||||
RestartPreventExitStatus = 78;
|
||||
Type = "simple";
|
||||
# Read here, so setting either read-only option fails evaluation.
|
||||
User = cfg.user;
|
||||
|
||||
+14
-1
@@ -119,7 +119,8 @@ in
|
||||
settings.policy.path = lib.mkForce mutablePolicy;
|
||||
check.config = false;
|
||||
};
|
||||
# Builds only because check.config = false also drops the build check.
|
||||
# Builds only because check.config = false also drops the build check;
|
||||
# headscale itself refuses it.
|
||||
unchecked = {
|
||||
settings.server_url = badServerUrl;
|
||||
check = {
|
||||
@@ -240,6 +241,18 @@ in
|
||||
want = headscale.succeed("${jq} -S . ${testsPass}")
|
||||
assert served_policy() == want, "new policy not served"
|
||||
|
||||
with subtest("a configuration headscale rejects is not restarted"):
|
||||
# The switch may report the unit failed; the unit is what counts.
|
||||
headscale.execute(stc("unchecked", "test"))
|
||||
headscale.wait_until_succeeds("systemctl is-failed headscale.service", timeout=timedelta(seconds=60))
|
||||
status = headscale.succeed("systemctl show -P ExecMainStatus headscale.service").strip()
|
||||
assert status == "78", f"exit status {status}, want 78"
|
||||
restarts = headscale.succeed("systemctl show -P NRestarts headscale.service")
|
||||
# Past RestartSec.
|
||||
headscale.sleep(10)
|
||||
headscale.succeed("systemctl is-failed headscale.service")
|
||||
assert headscale.succeed("systemctl show -P NRestarts headscale.service") == restarts, "headscale was restarted"
|
||||
|
||||
# Last: it takes headscale's account away.
|
||||
with subtest("an account activation has yet to create skips the checks"):
|
||||
headscale.succeed("systemctl stop headscale.service")
|
||||
|
||||
@@ -53,7 +53,7 @@ case "$1" in
|
||||
else
|
||||
if [ -n "$2" ]; then
|
||||
echo 'headscale was not restarted after the upgrade: its configuration does not pass "headscale check config" (errors above).' >&2
|
||||
echo 'A running headscale keeps the previous version only until it restarts: needrestart after apt, a reboot or a crash starts this version, which refuses the configuration.' >&2
|
||||
echo 'A running headscale keeps the previous version only until it restarts: needrestart after apt, a reboot or a crash starts this version, which refuses the configuration and stays stopped until it is fixed.' >&2
|
||||
else
|
||||
echo 'headscale was not started: its configuration does not pass "headscale check config" (errors above).' >&2
|
||||
fi
|
||||
|
||||
@@ -10,6 +10,9 @@ ExecStart=/usr/bin/headscale serve
|
||||
ExecReload=/usr/bin/kill -HUP $MAINPID
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
# headscale serve exits 78 (EX_CONFIG) on a configuration it rejects, which
|
||||
# a restart cannot fix.
|
||||
RestartPreventExitStatus=78
|
||||
|
||||
WorkingDirectory=/var/lib/headscale
|
||||
ReadWritePaths=/var/lib/headscale
|
||||
|
||||
Reference in new issue
Block a user