diff --git a/cmd/headscale/cli/serve.go b/cmd/headscale/cli/serve.go index f5b84378d..a2e879f9a 100644 --- a/cmd/headscale/cli/serve.go +++ b/cmd/headscale/cli/serve.go @@ -3,7 +3,7 @@ package cli import ( "errors" "fmt" - "net/http" + "os/signal" "syscall" "github.com/juanfont/headscale/hscontrol/types" @@ -29,12 +29,15 @@ var serveCmd = &cobra.Command{ return fmt.Errorf("initializing: %w", err) } - err = app.Serve() - if err == nil || errors.Is(err, http.ErrServerClosed) { - return nil + ctx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + defer stop() + + err = app.Serve(ctx) + if err != nil { + return classifyServeError(err) } - return classifyServeError(err) + return nil }, } diff --git a/hscontrol/app.go b/hscontrol/app.go index af627dbd2..560598c7a 100644 --- a/hscontrol/app.go +++ b/hscontrol/app.go @@ -67,6 +67,8 @@ var ( errEmptyInitialDERPMap = errors.New( "initial DERPMap is empty, Headscale requires at least one entry", ) + errTailSQLDatabase = errors.New("tailsql only supports sqlite") + errTailSQLAuthKey = errors.New("tailsql requires TS_AUTHKEY to be set") ) var ( @@ -570,19 +572,31 @@ func (h *Headscale) createRouter(apiV1Mux, apiV2Mux http.Handler) *chi.Mux { return r } -// Serve launches the HTTP servers that run Headscale and its API. +// Serve launches the HTTP servers that run Headscale and its API. It shuts +// down gracefully when ctx is cancelled or any of its components fails, and +// returns that component's error. // //nolint:gocyclo // complex server startup function -func (h *Headscale) Serve() error { +func (h *Headscale) Serve(ctx context.Context) error { var err error capver.CanOldCodeBeCleanedUp() + if tailsqlEnabled { + if h.cfg.Database.Type != types.DatabaseSqlite { + return fmt.Errorf("%w, got %q", errTailSQLDatabase, h.cfg.Database.Type) + } + + if tailsqlTSKey == "" { + return errTailSQLAuthKey + } + } + if profilingEnabled { if profilingPath != "" { err = os.MkdirAll(profilingPath, os.ModePerm) if err != nil { - log.Fatal().Err(err).Msg("failed to create profiling directory") + return fmt.Errorf("creating profiling directory: %w", err) } defer profile.Start(profile.ProfilePath(profilingPath)).Stop() @@ -601,6 +615,12 @@ func (h *Headscale) Serve() error { Str("minimum_version", capver.TailscaleVersion(capver.MinSupportedCapabilityVersion)). Msg("Clients with a lower minimum version will be rejected") + // The cancel stops what already started when Serve returns early. + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + errorGroup, ctx := errgroup.WithContext(ctx) + h.mapBatcher = mapper.NewBatcherAndMapper(h.cfg, h.state) h.mapBatcher.Start() @@ -617,13 +637,13 @@ func (h *Headscale) Serve() error { // Start on the sources that loaded; only a map with no region at all, // after the embedded region and the policy's derpMap, is fatal. - derpMap, derpErr := derp.GetDERPMap(h.cfg.DERP) + derpMap, derpErr := derp.GetDERPMap(h.cfg.DERP) //nolint:contextcheck // startup fetch, bounded by its HTTP client if derpErr != nil { log.Warn().Err(derpErr).Msg("skipping DERP map sources that failed to load") } if h.cfg.DERP.ServerEnabled && h.cfg.DERP.AutomaticallyAddEmbeddedDerpRegion { - region, err := h.DERPServer.GenerateRegion() + region, err := h.DERPServer.GenerateRegion() //nolint:contextcheck // only a debug DNS lookup uses a context if err != nil { return fmt.Errorf("generating embedded DERP region: %w", err) } @@ -656,7 +676,7 @@ func (h *Headscale) Serve() error { h.cfg.SetExtraRecords(h.extraRecordMan.Records()) - go h.extraRecordMan.Run() + go h.extraRecordMan.Run() //nolint:contextcheck // stopped by Close defer h.extraRecordMan.Close() } @@ -665,18 +685,7 @@ func (h *Headscale) Serve() error { // Start all scheduled tasks, e.g. expiring nodes, derp updates and // records updates - scheduleCtx, scheduleCancel := context.WithCancel(context.Background()) - defer scheduleCancel() - - go h.scheduledTasks(scheduleCtx) - - // Prepare group for running listeners - errorGroup := new(errgroup.Group) - - ctx := context.Background() - - ctx, cancel := context.WithCancel(ctx) - defer cancel() + go h.scheduledTasks(ctx) // // @@ -695,7 +704,7 @@ func (h *Headscale) Serve() error { return fmt.Errorf("setting up unix socket: %w", err) } - socketListener, err := new(net.ListenConfig).Listen(context.Background(), "unix", h.cfg.UnixSocket) + socketListener, err := new(net.ListenConfig).Listen(ctx, "unix", h.cfg.UnixSocket) if err != nil { return fmt.Errorf("setting up socket: %w", err) } @@ -738,7 +747,7 @@ func (h *Headscale) Serve() error { ReadTimeout: types.HTTPTimeout, } - errorGroup.Go(func() error { return socketServer.Serve(socketListener) }) + errorGroup.Go(serveHTTP(socketServer, socketListener)) // // @@ -776,7 +785,7 @@ func (h *Headscale) Serve() error { httpServer.TLSConfig = tlsConfig httpListener, err = tls.Listen("tcp", h.cfg.Addr, tlsConfig) } else { - httpListener, err = new(net.ListenConfig).Listen(context.Background(), "tcp", h.cfg.Addr) + httpListener, err = new(net.ListenConfig).Listen(ctx, "tcp", h.cfg.Addr) } if err != nil { @@ -788,7 +797,7 @@ func (h *Headscale) Serve() error { } } - errorGroup.Go(func() error { return httpServer.Serve(httpListener) }) + errorGroup.Go(serveHTTP(httpServer, httpListener)) log.Info(). Msgf("listening and serving HTTP on: %s", h.cfg.Addr) @@ -820,7 +829,7 @@ func (h *Headscale) Serve() error { WriteTimeout: types.HTTPTimeout, } - errorGroup.Go(func() error { return insecureTLSServer.Serve(insecureTLSListener) }) + errorGroup.Go(serveHTTP(insecureTLSServer, insecureTLSListener)) log.Warn(). Str("addr", addr). @@ -866,7 +875,7 @@ func (h *Headscale) Serve() error { debugHTTPServer = h.debugHTTPServer() - errorGroup.Go(func() error { return debugHTTPServer.Serve(debugHTTPListener) }) + errorGroup.Go(serveHTTP(debugHTTPServer, debugHTTPListener)) log.Info(). Msgf("listening and serving debug and metrics on: %s", h.cfg.MetricsAddr) @@ -877,18 +886,6 @@ func (h *Headscale) Serve() error { var tailsqlCancel context.CancelFunc if tailsqlEnabled { - if h.cfg.Database.Type != types.DatabaseSqlite { - //nolint:gocritic // exitAfterDefer: Fatal exits during initialization before servers start - log.Fatal(). - Str("type", h.cfg.Database.Type). - Msgf("tailsql only support %q", types.DatabaseSqlite) - } - - if tailsqlTSKey == "" { - //nolint:gocritic // exitAfterDefer: Fatal exits during initialization before servers start - log.Fatal().Msg("tailsql requires TS_AUTHKEY to be set") - } - var tailsqlCtx context.Context tailsqlCtx, tailsqlCancel = context.WithCancel(ctx) @@ -898,21 +895,16 @@ func (h *Headscale) Serve() error { }) } - // Handle common process-killing signals so we can gracefully shut down: - sigc := make(chan os.Signal, 1) - signal.Notify(sigc, - syscall.SIGHUP, - syscall.SIGINT, - syscall.SIGTERM, - syscall.SIGQUIT, - syscall.SIGHUP) + // Stop signals belong to the caller, which cancels ctx; SIGHUP reloads. + sighup := make(chan os.Signal, 1) - sigFunc := func(c chan os.Signal) { - // Wait for a SIGINT or SIGKILL: + signal.Notify(sighup, syscall.SIGHUP) + defer signal.Stop(sighup) + + errorGroup.Go(func() error { for { - sig := <-c - switch sig { - case syscall.SIGHUP: + select { + case sig := <-sighup: log.Info(). Str("signal", sig.String()). Msg("Received SIGHUP") @@ -938,14 +930,13 @@ func (h *Headscale) Serve() error { continue } - default: + case <-ctx.Done(): info := func(msg string) { log.Info().Msg(msg) } log.Info(). - Str("signal", sig.String()). - Msg("Received signal to stop, shutting down gracefully") + Str("reason", context.Cause(ctx).Error()). + Msg("shutting down gracefully") - scheduleCancel() h.ephemeralGC.Close() // Gracefully shut down servers @@ -1050,20 +1041,27 @@ func (h *Headscale) Serve() error { log.Info(). Msg("Headscale stopped") - return + return nil } } - } - - errorGroup.Go(func() error { - sigFunc(sigc) - - return nil }) return errorGroup.Wait() } +// serveHTTP treats a graceful Shutdown as success so only a real failure +// cancels the group. +func serveHTTP(srv *http.Server, l net.Listener) func() error { + return func() error { + err := srv.Serve(l) + if errors.Is(err, http.ErrServerClosed) { + return nil + } + + return err + } +} + // tlsBundle carries the TLS settings produced by getTLSSettings. When // HTTP-01 ACME is configured, ACMEServer and ACMEListener are populated // so the caller can register the challenge listener with the errgroup diff --git a/hscontrol/app_test.go b/hscontrol/app_test.go index 6b0379884..855163848 100644 --- a/hscontrol/app_test.go +++ b/hscontrol/app_test.go @@ -2,11 +2,16 @@ package hscontrol import ( "context" + "net" "net/http" "net/http/httptest" + "path/filepath" "testing" + "time" + "github.com/juanfont/headscale/hscontrol/types" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestSecurityHeaders(t *testing.T) { @@ -79,3 +84,78 @@ func TestDebugHostGuard(t *testing.T) { }) } } + +// serveTestConfig returns a config for running Serve on loopback with the +// embedded DERP server enabled on stunAddr. +func serveTestConfig(t *testing.T, stunAddr string) *types.Config { + t.Helper() + + dir := t.TempDir() + + return &types.Config{ + ServerURL: "http://127.0.0.1:8080", + Addr: "127.0.0.1:0", + UnixSocket: filepath.Join(dir, "hs.sock"), + UnixSocketPermission: 0o770, + NoisePrivateKeyPath: filepath.Join(dir, "noise.key"), + Database: types.DatabaseConfig{ + Type: types.DatabaseSqlite, + Sqlite: types.SqliteConfig{Path: filepath.Join(dir, "hs.db")}, + }, + Policy: types.PolicyConfig{Mode: types.PolicyModeDB}, + DERP: types.DERPConfig{ + ServerEnabled: true, + AutomaticallyAddEmbeddedDerpRegion: true, + ServerPrivateKeyPath: filepath.Join(dir, "derp.key"), + ServerRegionID: 999, + ServerRegionCode: "test", + STUNAddr: stunAddr, + }, + Tuning: types.Tuning{BatchChangeDelay: 100 * time.Millisecond, BatcherWorkers: 1}, + } +} + +// Cancelling the context must run the graceful shutdown and return nil, so +// an embedder can stop Serve without sending the process a signal. +func TestServeStopsOnContextCancel(t *testing.T) { + cfg := serveTestConfig(t, "127.0.0.1:0") + + app, err := NewHeadscale(cfg) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + served := make(chan error, 1) + + go func() { served <- app.Serve(ctx) }() + + client := &http.Client{Transport: &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + return new(net.Dialer).DialContext(ctx, "unix", cfg.UnixSocket) + }, + }} + + assert.EventuallyWithT(t, func(c *assert.CollectT) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://unix/api/v1/health", nil) + require.NoError(c, err) + + res, err := client.Do(req) + require.NoError(c, err) + + defer res.Body.Close() + + assert.Equal(c, http.StatusOK, res.StatusCode) + }, 10*time.Second, 50*time.Millisecond) + + cancel() + + select { + case err := <-served: + require.NoError(t, err) + case <-time.After(10 * time.Second): + t.Fatal("Serve did not return after its context was cancelled") + } + + assert.NoFileExists(t, cfg.UnixSocket) +}