From b3ffec4269ce7a6c91c1463b5cc2ec4621236261 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Fri, 30 Oct 2026 15:51:20 +0000 Subject: [PATCH] hscontrol: reload manual TLS certificate on SIGHUP Files that fail to load or are not valid now keep the previous certificate. Fixes #3027 Claude-Session: https://claude.ai/code/session_01RHEZM1qJ2hgndiXio7P56h --- CHANGELOG.md | 1 + docs/ref/tls.md | 6 ++ hscontrol/app.go | 73 ++++++++++++++++--- hscontrol/app_tls_test.go | 150 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 221 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index decaf7dc0..958338889 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -169,6 +169,7 @@ the ACME DNS-01 challenge through a built-in DNS provider: `acmedns`, ### Changes +- A SIGHUP now reloads the certificate from `tls.cert_path` and `tls.key_path` without a restart; a certificate that cannot be loaded or is not valid now is logged as an error and the previous one keeps serving [#3027](https://github.com/juanfont/headscale/issues/3027) - PostgreSQL connection settings are assembled into a `postgres://` URL with proper escaping, so a password (or host, user, database name) containing spaces, quotes or other reserved characters no longer truncates the DSN [#3477](https://github.com/juanfont/headscale/pull/3477) - Expiring or deleting a non-existent pre-auth key now returns an error instead of silently succeeding [#3324](https://github.com/juanfont/headscale/pull/3324) - Improve systemd service file hardening [#3341](https://github.com/juanfont/headscale/pull/3341) diff --git a/docs/ref/tls.md b/docs/ref/tls.md index 7643476e2..33f23bec5 100644 --- a/docs/ref/tls.md +++ b/docs/ref/tls.md @@ -12,6 +12,12 @@ tls: The certificate should contain the full chain, else some clients, like the Tailscale Android client, will reject it. +### Reloading the certificate + +Headscale reads the certificate and key again when it receives a SIGHUP signal, so a renewed certificate takes effect without a restart. Either reload Headscale via its systemd service (`sudo systemctl reload headscale`) or send the signal directly (`sudo kill -HUP $(pidof headscale)`). The same signal also [reloads the policy](policy.md). New connections get the new certificate; open connections stay as they are. + +If the files cannot be loaded, or the certificate is expired or not yet valid, Headscale logs an error and keeps serving the previous certificate. `systemctl reload` reports success either way, because it only delivers the signal. Look for `TLS certificate reloaded` in the log, or [check the served certificate](#checking-certificate-expiry). + ## Let's Encrypt / ACME To get a certificate automatically via [Let's Encrypt](https://letsencrypt.org/), set `tls.acme.hostname` to the desired certificate hostname. This name must resolve to the IP address(es) headscale is reachable on (i.e., it must correspond to the `server_url` configuration parameter). The certificate and Let's Encrypt account credentials will be stored in the directory configured in `tls.acme.cache_dir`. If the path is relative, it will be interpreted as relative to the directory the configuration file was read from. diff --git a/hscontrol/app.go b/hscontrol/app.go index 2ae50c9a3..c37c7a47c 100644 --- a/hscontrol/app.go +++ b/hscontrol/app.go @@ -20,6 +20,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "syscall" "testing" "time" @@ -57,6 +58,7 @@ import ( var ( errSTUNAddressNotSet = errors.New("STUN address not set") + errTLSCertificateNotValidNow = errors.New("TLS certificate is not valid now") errUnsupportedLetsEncryptChallengeType = errors.New( "unknown value for Lets Encrypt challenge type", ) @@ -901,12 +903,21 @@ func (h *Headscale) Serve() error { case syscall.SIGHUP: log.Info(). Str("signal", sig.String()). - Msg("Received SIGHUP, reloading ACL policy") + Msg("Received SIGHUP") + + if tlsBundle.Reload != nil { + err := tlsBundle.Reload() + if err != nil { + log.Error().Err(err).Msg("reloading TLS certificate failed, still serving the previous one") + } + } if h.cfg.Policy.IsEmpty() { continue } + log.Info().Msg("Reloading ACL policy") + changes, err := h.state.ReloadPolicy() h.Change(changes...) @@ -1056,6 +1067,10 @@ type tlsBundle struct { Manage func(context.Context) error // Stop ends certificate maintenance. Nil without ACME. Stop func() + // Reload swaps in the certificate now on disk for new connections. On + // error the previous certificate keeps serving. Nil unless the + // certificate comes from tls.cert_path and tls.key_path. + Reload func() error } // acmeTrustedRoots verifies the ACME directory; nil means the system roots. @@ -1199,19 +1214,59 @@ func (h *Headscale) getTLSSettings(ctx context.Context) (*tlsBundle, error) { return &tlsBundle{}, nil } - cert, err := tls.LoadX509KeyPair(h.cfg.TLS.CertPath, h.cfg.TLS.KeyPath) - if err != nil { - return nil, fmt.Errorf("loading TLS keypair (tls.cert_path=%q, tls.key_path=%q): %w", - h.cfg.TLS.CertPath, h.cfg.TLS.KeyPath, err) + load := func() (*tls.Certificate, error) { + cert, err := tls.LoadX509KeyPair(h.cfg.TLS.CertPath, h.cfg.TLS.KeyPath) + if err != nil { + return nil, fmt.Errorf("loading TLS keypair (tls.cert_path=%q, tls.key_path=%q): %w", + h.cfg.TLS.CertPath, h.cfg.TLS.KeyPath, err) + } + + return &cert, nil } + cert, err := load() + if err != nil { + return nil, err + } + + var current atomic.Pointer[tls.Certificate] + + current.Store(cert) + tlsConfig := &tls.Config{ - NextProtos: []string{"http/1.1"}, - Certificates: []tls.Certificate{cert}, - MinVersion: tls.VersionTLS12, + NextProtos: []string{"http/1.1"}, + // Not Certificates: crypto/tls prefers those when a client sends no + // SNI, so that client would never see a reload. + GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return current.Load(), nil }, + MinVersion: tls.VersionTLS12, } - return &tlsBundle{Config: tlsConfig}, nil + reload := func() error { + cert, err := load() + if err != nil { + return err + } + + // Loading does not check validity. Swapping in a certificate no client + // accepts would turn a bad deploy into an outage. + now, leaf := time.Now(), cert.Leaf + if now.Before(leaf.NotBefore) || now.After(leaf.NotAfter) { + return fmt.Errorf("%w (tls.cert_path=%q): valid from %s until %s", + errTLSCertificateNotValidNow, h.cfg.TLS.CertPath, + leaf.NotBefore.Format(time.RFC3339), leaf.NotAfter.Format(time.RFC3339)) + } + + current.Store(cert) + + log.Info(). + Str("cert_path", h.cfg.TLS.CertPath). + Str("not_after", leaf.NotAfter.Format(time.RFC3339)). + Msg("TLS certificate reloaded") + + return nil + } + + return &tlsBundle{Config: tlsConfig, Reload: reload}, nil } // selfSignedTLSConfig returns a TLS config with a fresh in-memory certificate. diff --git a/hscontrol/app_tls_test.go b/hscontrol/app_tls_test.go index b62d446b6..dc5da52ad 100644 --- a/hscontrol/app_tls_test.go +++ b/hscontrol/app_tls_test.go @@ -2,14 +2,21 @@ package hscontrol import ( "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" "crypto/tls" "crypto/x509" + "encoding/pem" "log" + "math/big" "net" "net/http/httptest" "os" + "path/filepath" "strconv" "strings" + "sync" "testing" "time" @@ -219,6 +226,7 @@ func TestACMETLSSettings(t *testing.T) { bundle, err := h.getTLSSettings(t.Context()) require.NoError(t, err) t.Cleanup(bundle.Stop) + assert.Nil(t, bundle.Reload, "certmagic renews the certificate itself") ln, err := tls.Listen("tcp", h.cfg.Addr, bundle.Config) require.NoError(t, err) @@ -267,3 +275,145 @@ func TestACMETLSSettings(t *testing.T) { }) } } + +// testKeypair returns a new certificate as DER, and as PEM with its key. +func testKeypair(t *testing.T, notBefore, notAfter time.Time) ([]byte, []byte, []byte) { + t.Helper() + + priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + + template := &x509.Certificate{SerialNumber: big.NewInt(1), NotBefore: notBefore, NotAfter: notAfter} + + der, err := x509.CreateCertificate(rand.Reader, template, template, &priv.PublicKey, priv) + require.NoError(t, err) + + keyDER, err := x509.MarshalPKCS8PrivateKey(priv) + require.NoError(t, err) + + return der, + pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), + pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}) +} + +// servedCertificate returns the certificate a new connection gets. +func servedCertificate(t *testing.T, cfg *tls.Config, serverName string) []byte { + t.Helper() + + clientEnd, serverEnd := net.Pipe() + + defer clientEnd.Close() + defer serverEnd.Close() + + go func() { _ = tls.Server(serverEnd, cfg).HandshakeContext(t.Context()) }() + + client := tls.Client(clientEnd, &tls.Config{ServerName: serverName, InsecureSkipVerify: true}) //nolint:gosec // reads the served certificate, trusts nothing + require.NoError(t, client.HandshakeContext(t.Context())) + + return client.ConnectionState().PeerCertificates[0].Raw +} + +func TestStaticTLSReload(t *testing.T) { + dir := t.TempDir() + certPath, keyPath := filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem") + + write := func(t *testing.T, certPEM, keyPEM []byte) { + t.Helper() + require.NoError(t, os.WriteFile(certPath, certPEM, 0o600)) + require.NoError(t, os.WriteFile(keyPath, keyPEM, 0o600)) + } + + now := time.Now() + derA, certA, keyA := testKeypair(t, now.Add(-time.Hour), now.Add(time.Hour)) + derB, certB, keyB := testKeypair(t, now.Add(-time.Hour), now.Add(time.Hour)) + _, certExpired, keyExpired := testKeypair(t, now.Add(-2*time.Hour), now.Add(-time.Hour)) + _, certFuture, keyFuture := testKeypair(t, now.Add(time.Hour), now.Add(2*time.Hour)) + + write(t, certA, keyA) + + h := &Headscale{cfg: &types.Config{TLS: types.TLSConfig{CertPath: certPath, KeyPath: keyPath}}} + + bundle, err := h.getTLSSettings(t.Context()) + require.NoError(t, err) + require.NotNil(t, bundle.Reload) + + // A client without SNI is the one case where crypto/tls prefers a static + // Certificates entry over GetCertificate. + assertServed := func(t *testing.T, want []byte) { + t.Helper() + + for _, serverName := range []string{"", "headscale.example.com"} { + assert.Equal(t, want, servedCertificate(t, bundle.Config, serverName), "SNI %q", serverName) + } + } + + assertServed(t, derA) + + write(t, certB, keyB) + require.NoError(t, bundle.Reload()) + assertServed(t, derB) + + for name, pair := range map[string][2][]byte{ + "garbage": {[]byte("garbage"), []byte("garbage")}, + "mismatched": {certA, keyB}, + "expired": {certExpired, keyExpired}, + "not yet valid": {certFuture, keyFuture}, + } { + t.Run(name, func(t *testing.T) { + write(t, pair[0], pair[1]) + require.ErrorContains(t, bundle.Reload(), certPath, "the operator must see which file failed") + assertServed(t, derB) + }) + } + + // Meaningful under -race, which the gorace check runs on this package. + write(t, certA, keyA) + + var wg sync.WaitGroup + + wg.Go(func() { + for range 20 { + assert.NoError(t, bundle.Reload()) + } + }) + + for range 20 { + servedCertificate(t, bundle.Config, "") + } + + wg.Wait() +} + +// Expiry only blocks a reload: refusing to start would also take down the +// CLI socket, the tool an operator needs to recover. +func TestStaticTLSExpiredAtStartup(t *testing.T) { + dir := t.TempDir() + certPath, keyPath := filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem") + + now := time.Now() + der, certPEM, keyPEM := testKeypair(t, now.Add(-2*time.Hour), now.Add(-time.Hour)) + require.NoError(t, os.WriteFile(certPath, certPEM, 0o600)) + require.NoError(t, os.WriteFile(keyPath, keyPEM, 0o600)) + + h := &Headscale{cfg: &types.Config{TLS: types.TLSConfig{CertPath: certPath, KeyPath: keyPath}}} + + bundle, err := h.getTLSSettings(t.Context()) + require.NoError(t, err) + assert.Equal(t, der, servedCertificate(t, bundle.Config, "")) +} + +func TestStaticTLSLoadErrorNamesConfigKeys(t *testing.T) { + h := &Headscale{cfg: &types.Config{TLS: types.TLSConfig{CertPath: "/nonexistent/cert", KeyPath: "/nonexistent/key"}}} + + _, err := h.getTLSSettings(t.Context()) + require.ErrorContains(t, err, "tls.cert_path") + require.ErrorContains(t, err, "tls.key_path") +} + +func TestPlainHTTPHasNoTLSReload(t *testing.T) { + h := &Headscale{cfg: &types.Config{ServerURL: "http://headscale.example.com"}} + + bundle, err := h.getTLSSettings(t.Context()) + require.NoError(t, err) + assert.Nil(t, bundle.Reload) +}