hscontrol: reload manual TLS certificate on SIGHUP

Files that fail to load or are not valid now keep the previous certificate.

Fixes #3027

Claude-Session: https://claude.ai/code/session_01RHEZM1qJ2hgndiXio7P56h

CHANGELOG.md

@@ -169,6 +169,7 @@ the ACME DNS-01 challenge through a built-in DNS provider: `acmedns`,
 
 ### Changes
 
+- A SIGHUP now reloads the certificate from `tls.cert_path` and `tls.key_path` without a restart; a certificate that cannot be loaded or is not valid now is logged as an error and the previous one keeps serving [#3027](https://github.com/juanfont/headscale/issues/3027)
 - PostgreSQL connection settings are assembled into a `postgres://` URL with proper escaping, so a password (or host, user, database name) containing spaces, quotes or other reserved characters no longer truncates the DSN [#3477](https://github.com/juanfont/headscale/pull/3477)
 - Expiring or deleting a non-existent pre-auth key now returns an error instead of silently succeeding [#3324](https://github.com/juanfont/headscale/pull/3324)
 - Improve systemd service file hardening [#3341](https://github.com/juanfont/headscale/pull/3341)

docs/ref/tls.md

@@ -12,6 +12,12 @@ tls:
 
 The certificate should contain the full chain, else some clients, like the Tailscale Android client, will reject it.
 
+### Reloading the certificate
+
+Headscale reads the certificate and key again when it receives a SIGHUP signal, so a renewed certificate takes effect without a restart. Either reload Headscale via its systemd service (`sudo systemctl reload headscale`) or send the signal directly (`sudo kill -HUP $(pidof headscale)`). The same signal also [reloads the policy](policy.md). New connections get the new certificate; open connections stay as they are.
+
+If the files cannot be loaded, or the certificate is expired or not yet valid, Headscale logs an error and keeps serving the previous certificate. `systemctl reload` reports success either way, because it only delivers the signal. Look for `TLS certificate reloaded` in the log, or [check the served certificate](#checking-certificate-expiry).
+
 ## Let's Encrypt / ACME
 
 To get a certificate automatically via [Let's Encrypt](https://letsencrypt.org/), set `tls.acme.hostname` to the desired certificate hostname. This name must resolve to the IP address(es) headscale is reachable on (i.e., it must correspond to the `server_url` configuration parameter). The certificate and Let's Encrypt account credentials will be stored in the directory configured in `tls.acme.cache_dir`. If the path is relative, it will be interpreted as relative to the directory the configuration file was read from.

hscontrol/app.go

@@ -20,6 +20,7 @@ import (
 	"slices"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"syscall"
 	"testing"
 	"time"
@@ -57,6 +58,7 @@ import (
 
 var (
 	errSTUNAddressNotSet                   = errors.New("STUN address not set")
+	errTLSCertificateNotValidNow           = errors.New("TLS certificate is not valid now")
 	errUnsupportedLetsEncryptChallengeType = errors.New(
 		"unknown value for Lets Encrypt challenge type",
 	)
@@ -901,12 +903,21 @@ func (h *Headscale) Serve() error {
 			case syscall.SIGHUP:
 				log.Info().
 					Str("signal", sig.String()).
-					Msg("Received SIGHUP, reloading ACL policy")
+					Msg("Received SIGHUP")
+
+				if tlsBundle.Reload != nil {
+					err := tlsBundle.Reload()
+					if err != nil {
+						log.Error().Err(err).Msg("reloading TLS certificate failed, still serving the previous one")
+					}
+				}
 
 				if h.cfg.Policy.IsEmpty() {
 					continue
 				}
 
+				log.Info().Msg("Reloading ACL policy")
+
 				changes, err := h.state.ReloadPolicy()
 				h.Change(changes...)
 
@@ -1056,6 +1067,10 @@ type tlsBundle struct {
 	Manage func(context.Context) error
 	// Stop ends certificate maintenance. Nil without ACME.
 	Stop func()
+	// Reload swaps in the certificate now on disk for new connections. On
+	// error the previous certificate keeps serving. Nil unless the
+	// certificate comes from tls.cert_path and tls.key_path.
+	Reload func() error
 }
 
 // acmeTrustedRoots verifies the ACME directory; nil means the system roots.
@@ -1199,19 +1214,59 @@ func (h *Headscale) getTLSSettings(ctx context.Context) (*tlsBundle, error) {
 		return &tlsBundle{}, nil
 	}
 
-	cert, err := tls.LoadX509KeyPair(h.cfg.TLS.CertPath, h.cfg.TLS.KeyPath)
-	if err != nil {
-		return nil, fmt.Errorf("loading TLS keypair (tls.cert_path=%q, tls.key_path=%q): %w",
-			h.cfg.TLS.CertPath, h.cfg.TLS.KeyPath, err)
+	load := func() (*tls.Certificate, error) {
+		cert, err := tls.LoadX509KeyPair(h.cfg.TLS.CertPath, h.cfg.TLS.KeyPath)
+		if err != nil {
+			return nil, fmt.Errorf("loading TLS keypair (tls.cert_path=%q, tls.key_path=%q): %w",
+				h.cfg.TLS.CertPath, h.cfg.TLS.KeyPath, err)
+		}
+
+		return &cert, nil
 	}
 
+	cert, err := load()
+	if err != nil {
+		return nil, err
+	}
+
+	var current atomic.Pointer[tls.Certificate]
+
+	current.Store(cert)
+
 	tlsConfig := &tls.Config{
-		NextProtos:   []string{"http/1.1"},
-		Certificates: []tls.Certificate{cert},
-		MinVersion:   tls.VersionTLS12,
+		NextProtos: []string{"http/1.1"},
+		// Not Certificates: crypto/tls prefers those when a client sends no
+		// SNI, so that client would never see a reload.
+		GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return current.Load(), nil },
+		MinVersion:     tls.VersionTLS12,
 	}
 
-	return &tlsBundle{Config: tlsConfig}, nil
+	reload := func() error {
+		cert, err := load()
+		if err != nil {
+			return err
+		}
+
+		// Loading does not check validity. Swapping in a certificate no client
+		// accepts would turn a bad deploy into an outage.
+		now, leaf := time.Now(), cert.Leaf
+		if now.Before(leaf.NotBefore) || now.After(leaf.NotAfter) {
+			return fmt.Errorf("%w (tls.cert_path=%q): valid from %s until %s",
+				errTLSCertificateNotValidNow, h.cfg.TLS.CertPath,
+				leaf.NotBefore.Format(time.RFC3339), leaf.NotAfter.Format(time.RFC3339))
+		}
+
+		current.Store(cert)
+
+		log.Info().
+			Str("cert_path", h.cfg.TLS.CertPath).
+			Str("not_after", leaf.NotAfter.Format(time.RFC3339)).
+			Msg("TLS certificate reloaded")
+
+		return nil
+	}
+
+	return &tlsBundle{Config: tlsConfig, Reload: reload}, nil
 }
 
 // selfSignedTLSConfig returns a TLS config with a fresh in-memory certificate.

hscontrol/app_tls_test.go

@@ -2,14 +2,21 @@ package hscontrol
 
 import (
 	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
 	"crypto/tls"
 	"crypto/x509"
+	"encoding/pem"
 	"log"
+	"math/big"
 	"net"
 	"net/http/httptest"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -219,6 +226,7 @@ func TestACMETLSSettings(t *testing.T) {
 			bundle, err := h.getTLSSettings(t.Context())
 			require.NoError(t, err)
 			t.Cleanup(bundle.Stop)
+			assert.Nil(t, bundle.Reload, "certmagic renews the certificate itself")
 
 			ln, err := tls.Listen("tcp", h.cfg.Addr, bundle.Config)
 			require.NoError(t, err)
@@ -267,3 +275,145 @@ func TestACMETLSSettings(t *testing.T) {
 		})
 	}
 }
+
+// testKeypair returns a new certificate as DER, and as PEM with its key.
+func testKeypair(t *testing.T, notBefore, notAfter time.Time) ([]byte, []byte, []byte) {
+	t.Helper()
+
+	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+
+	template := &x509.Certificate{SerialNumber: big.NewInt(1), NotBefore: notBefore, NotAfter: notAfter}
+
+	der, err := x509.CreateCertificate(rand.Reader, template, template, &priv.PublicKey, priv)
+	require.NoError(t, err)
+
+	keyDER, err := x509.MarshalPKCS8PrivateKey(priv)
+	require.NoError(t, err)
+
+	return der,
+		pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}),
+		pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER})
+}
+
+// servedCertificate returns the certificate a new connection gets.
+func servedCertificate(t *testing.T, cfg *tls.Config, serverName string) []byte {
+	t.Helper()
+
+	clientEnd, serverEnd := net.Pipe()
+
+	defer clientEnd.Close()
+	defer serverEnd.Close()
+
+	go func() { _ = tls.Server(serverEnd, cfg).HandshakeContext(t.Context()) }()
+
+	client := tls.Client(clientEnd, &tls.Config{ServerName: serverName, InsecureSkipVerify: true}) //nolint:gosec // reads the served certificate, trusts nothing
+	require.NoError(t, client.HandshakeContext(t.Context()))
+
+	return client.ConnectionState().PeerCertificates[0].Raw
+}
+
+func TestStaticTLSReload(t *testing.T) {
+	dir := t.TempDir()
+	certPath, keyPath := filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem")
+
+	write := func(t *testing.T, certPEM, keyPEM []byte) {
+		t.Helper()
+		require.NoError(t, os.WriteFile(certPath, certPEM, 0o600))
+		require.NoError(t, os.WriteFile(keyPath, keyPEM, 0o600))
+	}
+
+	now := time.Now()
+	derA, certA, keyA := testKeypair(t, now.Add(-time.Hour), now.Add(time.Hour))
+	derB, certB, keyB := testKeypair(t, now.Add(-time.Hour), now.Add(time.Hour))
+	_, certExpired, keyExpired := testKeypair(t, now.Add(-2*time.Hour), now.Add(-time.Hour))
+	_, certFuture, keyFuture := testKeypair(t, now.Add(time.Hour), now.Add(2*time.Hour))
+
+	write(t, certA, keyA)
+
+	h := &Headscale{cfg: &types.Config{TLS: types.TLSConfig{CertPath: certPath, KeyPath: keyPath}}}
+
+	bundle, err := h.getTLSSettings(t.Context())
+	require.NoError(t, err)
+	require.NotNil(t, bundle.Reload)
+
+	// A client without SNI is the one case where crypto/tls prefers a static
+	// Certificates entry over GetCertificate.
+	assertServed := func(t *testing.T, want []byte) {
+		t.Helper()
+
+		for _, serverName := range []string{"", "headscale.example.com"} {
+			assert.Equal(t, want, servedCertificate(t, bundle.Config, serverName), "SNI %q", serverName)
+		}
+	}
+
+	assertServed(t, derA)
+
+	write(t, certB, keyB)
+	require.NoError(t, bundle.Reload())
+	assertServed(t, derB)
+
+	for name, pair := range map[string][2][]byte{
+		"garbage":       {[]byte("garbage"), []byte("garbage")},
+		"mismatched":    {certA, keyB},
+		"expired":       {certExpired, keyExpired},
+		"not yet valid": {certFuture, keyFuture},
+	} {
+		t.Run(name, func(t *testing.T) {
+			write(t, pair[0], pair[1])
+			require.ErrorContains(t, bundle.Reload(), certPath, "the operator must see which file failed")
+			assertServed(t, derB)
+		})
+	}
+
+	// Meaningful under -race, which the gorace check runs on this package.
+	write(t, certA, keyA)
+
+	var wg sync.WaitGroup
+
+	wg.Go(func() {
+		for range 20 {
+			assert.NoError(t, bundle.Reload())
+		}
+	})
+
+	for range 20 {
+		servedCertificate(t, bundle.Config, "")
+	}
+
+	wg.Wait()
+}
+
+// Expiry only blocks a reload: refusing to start would also take down the
+// CLI socket, the tool an operator needs to recover.
+func TestStaticTLSExpiredAtStartup(t *testing.T) {
+	dir := t.TempDir()
+	certPath, keyPath := filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem")
+
+	now := time.Now()
+	der, certPEM, keyPEM := testKeypair(t, now.Add(-2*time.Hour), now.Add(-time.Hour))
+	require.NoError(t, os.WriteFile(certPath, certPEM, 0o600))
+	require.NoError(t, os.WriteFile(keyPath, keyPEM, 0o600))
+
+	h := &Headscale{cfg: &types.Config{TLS: types.TLSConfig{CertPath: certPath, KeyPath: keyPath}}}
+
+	bundle, err := h.getTLSSettings(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, der, servedCertificate(t, bundle.Config, ""))
+}
+
+func TestStaticTLSLoadErrorNamesConfigKeys(t *testing.T) {
+	h := &Headscale{cfg: &types.Config{TLS: types.TLSConfig{CertPath: "/nonexistent/cert", KeyPath: "/nonexistent/key"}}}
+
+	_, err := h.getTLSSettings(t.Context())
+	require.ErrorContains(t, err, "tls.cert_path")
+	require.ErrorContains(t, err, "tls.key_path")
+}
+
+func TestPlainHTTPHasNoTLSReload(t *testing.T) {
+	h := &Headscale{cfg: &types.Config{ServerURL: "http://headscale.example.com"}}
+
+	bundle, err := h.getTLSSettings(t.Context())
+	require.NoError(t, err)
+	assert.Nil(t, bundle.Reload)
+}