hscontrol: stop Serve through its context

Serve only stopped on signals; a failed listener left it half running.

Updates #3499
This commit is contained in:
Kristoffer Dalby committed 2026-11-06 13:05:40 +00:00
1 parent 475861cdd1
commit 9358c026d4
3 files changed
+146 -65

No files matched your search

+8 -5
View File
@@ -3,7 +3,7 @@ package cli
import (
"errors"
"fmt"
"net/http"
"os/signal"
"syscall"
"github.com/juanfont/headscale/hscontrol/types"
@@ -29,12 +29,15 @@ var serveCmd = &cobra.Command{
return fmt.Errorf("initializing: %w", err)
}
err = app.Serve()
if err == nil || errors.Is(err, http.ErrServerClosed) {
return nil
ctx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
defer stop()
err = app.Serve(ctx)
if err != nil {
return classifyServeError(err)
}
return classifyServeError(err)
return nil
},
}
+58 -60
View File
@@ -67,6 +67,8 @@ var (
errEmptyInitialDERPMap = errors.New(
"initial DERPMap is empty, Headscale requires at least one entry",
)
errTailSQLDatabase = errors.New("tailsql only supports sqlite")
errTailSQLAuthKey = errors.New("tailsql requires TS_AUTHKEY to be set")
)
var (
@@ -570,19 +572,31 @@ func (h *Headscale) createRouter(apiV1Mux, apiV2Mux http.Handler) *chi.Mux {
return r
}
// Serve launches the HTTP servers that run Headscale and its API.
// Serve launches the HTTP servers that run Headscale and its API. It shuts
// down gracefully when ctx is cancelled or any of its components fails, and
// returns that component's error.
//
//nolint:gocyclo // complex server startup function
func (h *Headscale) Serve() error {
func (h *Headscale) Serve(ctx context.Context) error {
var err error
capver.CanOldCodeBeCleanedUp()
if tailsqlEnabled {
if h.cfg.Database.Type != types.DatabaseSqlite {
return fmt.Errorf("%w, got %q", errTailSQLDatabase, h.cfg.Database.Type)
}
if tailsqlTSKey == "" {
return errTailSQLAuthKey
}
}
if profilingEnabled {
if profilingPath != "" {
err = os.MkdirAll(profilingPath, os.ModePerm)
if err != nil {
log.Fatal().Err(err).Msg("failed to create profiling directory")
return fmt.Errorf("creating profiling directory: %w", err)
}
defer profile.Start(profile.ProfilePath(profilingPath)).Stop()
@@ -601,6 +615,12 @@ func (h *Headscale) Serve() error {
Str("minimum_version", capver.TailscaleVersion(capver.MinSupportedCapabilityVersion)).
Msg("Clients with a lower minimum version will be rejected")
// The cancel stops what already started when Serve returns early.
ctx, cancel := context.WithCancel(ctx)
defer cancel()
errorGroup, ctx := errgroup.WithContext(ctx)
h.mapBatcher = mapper.NewBatcherAndMapper(h.cfg, h.state)
h.mapBatcher.Start()
@@ -617,13 +637,13 @@ func (h *Headscale) Serve() error {
// Start on the sources that loaded; only a map with no region at all,
// after the embedded region and the policy's derpMap, is fatal.
derpMap, derpErr := derp.GetDERPMap(h.cfg.DERP)
derpMap, derpErr := derp.GetDERPMap(h.cfg.DERP) //nolint:contextcheck // startup fetch, bounded by its HTTP client
if derpErr != nil {
log.Warn().Err(derpErr).Msg("skipping DERP map sources that failed to load")
}
if h.cfg.DERP.ServerEnabled && h.cfg.DERP.AutomaticallyAddEmbeddedDerpRegion {
region, err := h.DERPServer.GenerateRegion()
region, err := h.DERPServer.GenerateRegion() //nolint:contextcheck // only a debug DNS lookup uses a context
if err != nil {
return fmt.Errorf("generating embedded DERP region: %w", err)
}
@@ -656,7 +676,7 @@ func (h *Headscale) Serve() error {
h.cfg.SetExtraRecords(h.extraRecordMan.Records())
go h.extraRecordMan.Run()
go h.extraRecordMan.Run() //nolint:contextcheck // stopped by Close
defer h.extraRecordMan.Close()
}
@@ -665,18 +685,7 @@ func (h *Headscale) Serve() error {
// Start all scheduled tasks, e.g. expiring nodes, derp updates and
// records updates
scheduleCtx, scheduleCancel := context.WithCancel(context.Background())
defer scheduleCancel()
go h.scheduledTasks(scheduleCtx)
// Prepare group for running listeners
errorGroup := new(errgroup.Group)
ctx := context.Background()
ctx, cancel := context.WithCancel(ctx)
defer cancel()
go h.scheduledTasks(ctx)
//
//
@@ -695,7 +704,7 @@ func (h *Headscale) Serve() error {
return fmt.Errorf("setting up unix socket: %w", err)
}
socketListener, err := new(net.ListenConfig).Listen(context.Background(), "unix", h.cfg.UnixSocket)
socketListener, err := new(net.ListenConfig).Listen(ctx, "unix", h.cfg.UnixSocket)
if err != nil {
return fmt.Errorf("setting up socket: %w", err)
}
@@ -738,7 +747,7 @@ func (h *Headscale) Serve() error {
ReadTimeout: types.HTTPTimeout,
}
errorGroup.Go(func() error { return socketServer.Serve(socketListener) })
errorGroup.Go(serveHTTP(socketServer, socketListener))
//
//
@@ -776,7 +785,7 @@ func (h *Headscale) Serve() error {
httpServer.TLSConfig = tlsConfig
httpListener, err = tls.Listen("tcp", h.cfg.Addr, tlsConfig)
} else {
httpListener, err = new(net.ListenConfig).Listen(context.Background(), "tcp", h.cfg.Addr)
httpListener, err = new(net.ListenConfig).Listen(ctx, "tcp", h.cfg.Addr)
}
if err != nil {
@@ -788,7 +797,7 @@ func (h *Headscale) Serve() error {
}
}
errorGroup.Go(func() error { return httpServer.Serve(httpListener) })
errorGroup.Go(serveHTTP(httpServer, httpListener))
log.Info().
Msgf("listening and serving HTTP on: %s", h.cfg.Addr)
@@ -820,7 +829,7 @@ func (h *Headscale) Serve() error {
WriteTimeout: types.HTTPTimeout,
}
errorGroup.Go(func() error { return insecureTLSServer.Serve(insecureTLSListener) })
errorGroup.Go(serveHTTP(insecureTLSServer, insecureTLSListener))
log.Warn().
Str("addr", addr).
@@ -866,7 +875,7 @@ func (h *Headscale) Serve() error {
debugHTTPServer = h.debugHTTPServer()
errorGroup.Go(func() error { return debugHTTPServer.Serve(debugHTTPListener) })
errorGroup.Go(serveHTTP(debugHTTPServer, debugHTTPListener))
log.Info().
Msgf("listening and serving debug and metrics on: %s", h.cfg.MetricsAddr)
@@ -877,18 +886,6 @@ func (h *Headscale) Serve() error {
var tailsqlCancel context.CancelFunc
if tailsqlEnabled {
if h.cfg.Database.Type != types.DatabaseSqlite {
//nolint:gocritic // exitAfterDefer: Fatal exits during initialization before servers start
log.Fatal().
Str("type", h.cfg.Database.Type).
Msgf("tailsql only support %q", types.DatabaseSqlite)
}
if tailsqlTSKey == "" {
//nolint:gocritic // exitAfterDefer: Fatal exits during initialization before servers start
log.Fatal().Msg("tailsql requires TS_AUTHKEY to be set")
}
var tailsqlCtx context.Context
tailsqlCtx, tailsqlCancel = context.WithCancel(ctx)
@@ -898,21 +895,16 @@ func (h *Headscale) Serve() error {
})
}
// Handle common process-killing signals so we can gracefully shut down:
sigc := make(chan os.Signal, 1)
signal.Notify(sigc,
syscall.SIGHUP,
syscall.SIGINT,
syscall.SIGTERM,
syscall.SIGQUIT,
syscall.SIGHUP)
// Stop signals belong to the caller, which cancels ctx; SIGHUP reloads.
sighup := make(chan os.Signal, 1)
sigFunc := func(c chan os.Signal) {
// Wait for a SIGINT or SIGKILL:
signal.Notify(sighup, syscall.SIGHUP)
defer signal.Stop(sighup)
errorGroup.Go(func() error {
for {
sig := <-c
switch sig {
case syscall.SIGHUP:
select {
case sig := <-sighup:
log.Info().
Str("signal", sig.String()).
Msg("Received SIGHUP")
@@ -938,14 +930,13 @@ func (h *Headscale) Serve() error {
continue
}
default:
case <-ctx.Done():
info := func(msg string) { log.Info().Msg(msg) }
log.Info().
Str("signal", sig.String()).
Msg("Received signal to stop, shutting down gracefully")
Str("reason", context.Cause(ctx).Error()).
Msg("shutting down gracefully")
scheduleCancel()
h.ephemeralGC.Close()
// Gracefully shut down servers
@@ -1050,20 +1041,27 @@ func (h *Headscale) Serve() error {
log.Info().
Msg("Headscale stopped")
return
return nil
}
}
}
errorGroup.Go(func() error {
sigFunc(sigc)
return nil
})
return errorGroup.Wait()
}
// serveHTTP treats a graceful Shutdown as success so only a real failure
// cancels the group.
func serveHTTP(srv *http.Server, l net.Listener) func() error {
return func() error {
err := srv.Serve(l)
if errors.Is(err, http.ErrServerClosed) {
return nil
}
return err
}
}
// tlsBundle carries the TLS settings produced by getTLSSettings. When
// HTTP-01 ACME is configured, ACMEServer and ACMEListener are populated
// so the caller can register the challenge listener with the errgroup
+80
View File
@@ -2,11 +2,16 @@ package hscontrol
import (
"context"
"net"
"net/http"
"net/http/httptest"
"path/filepath"
"testing"
"time"
"github.com/juanfont/headscale/hscontrol/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSecurityHeaders(t *testing.T) {
@@ -79,3 +84,78 @@ func TestDebugHostGuard(t *testing.T) {
})
}
}
// serveTestConfig returns a config for running Serve on loopback with the
// embedded DERP server enabled on stunAddr.
func serveTestConfig(t *testing.T, stunAddr string) *types.Config {
t.Helper()
dir := t.TempDir()
return &types.Config{
ServerURL: "http://127.0.0.1:8080",
Addr: "127.0.0.1:0",
UnixSocket: filepath.Join(dir, "hs.sock"),
UnixSocketPermission: 0o770,
NoisePrivateKeyPath: filepath.Join(dir, "noise.key"),
Database: types.DatabaseConfig{
Type: types.DatabaseSqlite,
Sqlite: types.SqliteConfig{Path: filepath.Join(dir, "hs.db")},
},
Policy: types.PolicyConfig{Mode: types.PolicyModeDB},
DERP: types.DERPConfig{
ServerEnabled: true,
AutomaticallyAddEmbeddedDerpRegion: true,
ServerPrivateKeyPath: filepath.Join(dir, "derp.key"),
ServerRegionID: 999,
ServerRegionCode: "test",
STUNAddr: stunAddr,
},
Tuning: types.Tuning{BatchChangeDelay: 100 * time.Millisecond, BatcherWorkers: 1},
}
}
// Cancelling the context must run the graceful shutdown and return nil, so
// an embedder can stop Serve without sending the process a signal.
func TestServeStopsOnContextCancel(t *testing.T) {
cfg := serveTestConfig(t, "127.0.0.1:0")
app, err := NewHeadscale(cfg)
require.NoError(t, err)
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
served := make(chan error, 1)
go func() { served <- app.Serve(ctx) }()
client := &http.Client{Transport: &http.Transport{
DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
return new(net.Dialer).DialContext(ctx, "unix", cfg.UnixSocket)
},
}}
assert.EventuallyWithT(t, func(c *assert.CollectT) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://unix/api/v1/health", nil)
require.NoError(c, err)
res, err := client.Do(req)
require.NoError(c, err)
defer res.Body.Close()
assert.Equal(c, http.StatusOK, res.StatusCode)
}, 10*time.Second, 50*time.Millisecond)
cancel()
select {
case err := <-served:
require.NoError(t, err)
case <-time.After(10 * time.Second):
t.Fatal("Serve did not return after its context was cancelled")
}
assert.NoFileExists(t, cfg.UnixSocket)
}