diff --git a/cmd/headscale/cli/serve.go b/cmd/headscale/cli/serve.go index 8eacda4a7..9e21d406c 100644 --- a/cmd/headscale/cli/serve.go +++ b/cmd/headscale/cli/serve.go @@ -3,7 +3,7 @@ package cli import ( "errors" "fmt" - "net/http" + "os/signal" "syscall" "github.com/juanfont/headscale/hscontrol/config" @@ -33,12 +33,15 @@ var serveCmd = &cobra.Command{ return fmt.Errorf("initializing: %w", err) } - err = app.Serve() - if err == nil || errors.Is(err, http.ErrServerClosed) { - return nil + ctx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + defer stop() + + err = app.Serve(ctx) + if err != nil { + return classifyServeError(err) } - return classifyServeError(err) + return nil }, } diff --git a/hscontrol/app.go b/hscontrol/app.go index 49b2f29ac..a673f3bbe 100644 --- a/hscontrol/app.go +++ b/hscontrol/app.go @@ -65,6 +65,8 @@ var ( errEmptyInitialDERPMap = errors.New( "initial DERPMap is empty, Headscale requires at least one entry", ) + errTailSQLDatabase = errors.New("tailsql only supports sqlite") + errTailSQLAuthKey = errors.New("tailsql requires TS_AUTHKEY to be set") ) var ( @@ -576,19 +578,31 @@ func (h *Headscale) createRouter(apiV1Mux, apiV2Mux http.Handler) *chi.Mux { return r } -// Serve launches the HTTP servers that run Headscale and its API. +// Serve launches the HTTP servers that run Headscale and its API. It shuts +// down gracefully when ctx is cancelled or any of its components fails, and +// returns that component's error. // //nolint:gocyclo // complex server startup function -func (h *Headscale) Serve() error { +func (h *Headscale) Serve(ctx context.Context) error { var err error capver.CanOldCodeBeCleanedUp() + if tailsqlEnabled { + if !h.cfg.Database.IsSQLite() { + return fmt.Errorf("%w, got %q", errTailSQLDatabase, h.cfg.Database.Type) + } + + if tailsqlTSKey == "" { + return errTailSQLAuthKey + } + } + if profilingEnabled { if profilingPath != "" { err = os.MkdirAll(profilingPath, os.ModePerm) if err != nil { - log.Fatal().Err(err).Msg("failed to create profiling directory") + return fmt.Errorf("creating profiling directory: %w", err) } defer profile.Start(profile.ProfilePath(profilingPath)).Stop() @@ -612,6 +626,12 @@ func (h *Headscale) Serve() error { Str("minimum_version", capver.TailscaleVersion(capver.MinSupportedCapabilityVersion)). Msg("Clients with a lower minimum version will be rejected") + // The cancel stops what already started when Serve returns early. + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + errorGroup, ctx := errgroup.WithContext(ctx) + h.mapBatcher = mapper.NewBatcherAndMapper(h.cfg, h.state) h.mapBatcher.Start() @@ -628,13 +648,13 @@ func (h *Headscale) Serve() error { // Start on the sources that loaded; only a map with no region at all, // after the embedded region and the policy's derpMap, is fatal. - derpMap, derpErr := derp.GetDERPMap(h.cfg.DERP, h.cfg.DERPShuffleSeed()) + derpMap, derpErr := derp.GetDERPMap(h.cfg.DERP, h.cfg.DERPShuffleSeed()) //nolint:contextcheck // startup fetch, bounded by its HTTP client if derpErr != nil { log.Warn().Err(derpErr).Msg("skipping DERP map sources that failed to load") } if h.cfg.DERP.Server.Enabled && h.cfg.DERP.Server.AutomaticallyAddEmbeddedDerpRegion { - region, err := h.DERPServer.GenerateRegion() + region, err := h.DERPServer.GenerateRegion() //nolint:contextcheck // only a debug DNS lookup uses a context if err != nil { return fmt.Errorf("generating embedded DERP region: %w", err) } @@ -667,7 +687,7 @@ func (h *Headscale) Serve() error { h.cfg.SetExtraRecords(h.extraRecordMan.Records()) - go h.extraRecordMan.Run() + go h.extraRecordMan.Run() //nolint:contextcheck // stopped by Close defer h.extraRecordMan.Close() } @@ -676,18 +696,7 @@ func (h *Headscale) Serve() error { // Start all scheduled tasks, e.g. expiring nodes, derp updates and // records updates - scheduleCtx, scheduleCancel := context.WithCancel(context.Background()) - defer scheduleCancel() - - go h.scheduledTasks(scheduleCtx) - - // Prepare group for running listeners - errorGroup := new(errgroup.Group) - - ctx := context.Background() - - ctx, cancel := context.WithCancel(ctx) - defer cancel() + go h.scheduledTasks(ctx) // // @@ -706,7 +715,7 @@ func (h *Headscale) Serve() error { return fmt.Errorf("setting up unix socket: %w", err) } - socketListener, err := new(net.ListenConfig).Listen(context.Background(), "unix", h.cfg.UnixSocket) + socketListener, err := new(net.ListenConfig).Listen(ctx, "unix", h.cfg.UnixSocket) if err != nil { return fmt.Errorf("setting up socket: %w", err) } @@ -749,7 +758,7 @@ func (h *Headscale) Serve() error { ReadTimeout: types.HTTPTimeout, } - errorGroup.Go(func() error { return socketServer.Serve(socketListener) }) + errorGroup.Go(serveHTTP(socketServer, socketListener)) // // @@ -787,7 +796,7 @@ func (h *Headscale) Serve() error { httpServer.TLSConfig = tlsConfig httpListener, err = tls.Listen("tcp", string(h.cfg.Addr), tlsConfig) } else { - httpListener, err = new(net.ListenConfig).Listen(context.Background(), "tcp", string(h.cfg.Addr)) + httpListener, err = new(net.ListenConfig).Listen(ctx, "tcp", string(h.cfg.Addr)) } if err != nil { @@ -799,7 +808,7 @@ func (h *Headscale) Serve() error { } } - errorGroup.Go(func() error { return httpServer.Serve(httpListener) }) + errorGroup.Go(serveHTTP(httpServer, httpListener)) log.Info(). Msgf("listening and serving HTTP on: %s", h.cfg.Addr) @@ -831,7 +840,7 @@ func (h *Headscale) Serve() error { WriteTimeout: types.HTTPTimeout, } - errorGroup.Go(func() error { return insecureTLSServer.Serve(insecureTLSListener) }) + errorGroup.Go(serveHTTP(insecureTLSServer, insecureTLSListener)) log.Warn(). Str("addr", addr). @@ -877,7 +886,7 @@ func (h *Headscale) Serve() error { debugHTTPServer = h.debugHTTPServer() - errorGroup.Go(func() error { return debugHTTPServer.Serve(debugHTTPListener) }) + errorGroup.Go(serveHTTP(debugHTTPServer, debugHTTPListener)) log.Info(). Msgf("listening and serving debug and metrics on: %s", h.cfg.MetricsAddr) @@ -888,18 +897,6 @@ func (h *Headscale) Serve() error { var tailsqlCancel context.CancelFunc if tailsqlEnabled { - if !h.cfg.Database.IsSQLite() { - //nolint:gocritic // exitAfterDefer: Fatal exits during initialization before servers start - log.Fatal(). - Str("type", string(h.cfg.Database.Type)). - Msgf("tailsql only support %q", types.DatabaseSqlite) - } - - if tailsqlTSKey == "" { - //nolint:gocritic // exitAfterDefer: Fatal exits during initialization before servers start - log.Fatal().Msg("tailsql requires TS_AUTHKEY to be set") - } - var tailsqlCtx context.Context tailsqlCtx, tailsqlCancel = context.WithCancel(ctx) @@ -909,21 +906,16 @@ func (h *Headscale) Serve() error { }) } - // Handle common process-killing signals so we can gracefully shut down: - sigc := make(chan os.Signal, 1) - signal.Notify(sigc, - syscall.SIGHUP, - syscall.SIGINT, - syscall.SIGTERM, - syscall.SIGQUIT, - syscall.SIGHUP) + // Stop signals belong to the caller, which cancels ctx; SIGHUP reloads. + sighup := make(chan os.Signal, 1) - sigFunc := func(c chan os.Signal) { - // Wait for a SIGINT or SIGKILL: + signal.Notify(sighup, syscall.SIGHUP) + defer signal.Stop(sighup) + + errorGroup.Go(func() error { for { - sig := <-c - switch sig { - case syscall.SIGHUP: + select { + case sig := <-sighup: log.Info(). Str("signal", sig.String()). Msg("Received SIGHUP") @@ -949,14 +941,13 @@ func (h *Headscale) Serve() error { continue } - default: + case <-ctx.Done(): info := func(msg string) { log.Info().Msg(msg) } log.Info(). - Str("signal", sig.String()). - Msg("Received signal to stop, shutting down gracefully") + Str("reason", context.Cause(ctx).Error()). + Msg("shutting down gracefully") - scheduleCancel() h.ephemeralGC.Close() // Gracefully shut down servers @@ -1061,20 +1052,27 @@ func (h *Headscale) Serve() error { log.Info(). Msg("Headscale stopped") - return + return nil } } - } - - errorGroup.Go(func() error { - sigFunc(sigc) - - return nil }) return errorGroup.Wait() } +// serveHTTP treats a graceful Shutdown as success so only a real failure +// cancels the group. +func serveHTTP(srv *http.Server, l net.Listener) func() error { + return func() error { + err := srv.Serve(l) + if errors.Is(err, http.ErrServerClosed) { + return nil + } + + return err + } +} + // tlsBundle carries the TLS settings produced by getTLSSettings. When // HTTP-01 ACME is configured, ACMEServer and ACMEListener are populated // so the caller can register the challenge listener with the errgroup diff --git a/hscontrol/app_test.go b/hscontrol/app_test.go index 68356e9af..fb2da6014 100644 --- a/hscontrol/app_test.go +++ b/hscontrol/app_test.go @@ -2,6 +2,7 @@ package hscontrol import ( "context" + "net" "net/http" "net/http/httptest" "net/netip" @@ -483,3 +484,70 @@ func TestDERPRefreshDoesNotBlockScheduledTasks(t *testing.T) { assert.Eventually(t, func() bool { return app.cfg.HasExtraRecord("a.example.com") }, 10*time.Second, 50*time.Millisecond, "extra records not reloaded while the DERP refresh is stuck") } + +// serveTestConfig is validConfig serving on loopback and a unix socket, with +// the embedded DERP server and its STUN listener on stunAddr. +func serveTestConfig(t *testing.T, stunAddr string) *types.Config { + t.Helper() + + cfg := validConfig(t) + dir := filepath.Dir(cfg.Noise.PrivateKeyPath) + cfg.Addr = "127.0.0.1:0" + cfg.UnixSocket = filepath.Join(dir, "hs.sock") + cfg.DERP.URLs = nil + cfg.DERP.Server = types.DERPServerConfig{ + Enabled: true, + AutomaticallyAddEmbeddedDerpRegion: true, + PrivateKeyPath: filepath.Join(dir, "derp.key"), + RegionID: 999, + RegionCode: "test", + STUNAddr: types.ListenAddr(stunAddr), + } + + return &cfg +} + +// Cancelling the context must run the graceful shutdown and return nil, so +// an embedder can stop Serve without sending the process a signal. +func TestServeStopsOnContextCancel(t *testing.T) { + cfg := serveTestConfig(t, "127.0.0.1:0") + + app, err := NewHeadscale(cfg) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + served := make(chan error, 1) + + go func() { served <- app.Serve(ctx) }() + + client := &http.Client{Transport: &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + return new(net.Dialer).DialContext(ctx, "unix", cfg.UnixSocket) + }, + }} + + assert.EventuallyWithT(t, func(c *assert.CollectT) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://unix/api/v1/health", nil) + require.NoError(c, err) + + res, err := client.Do(req) + require.NoError(c, err) + + defer res.Body.Close() + + assert.Equal(c, http.StatusOK, res.StatusCode) + }, 10*time.Second, 50*time.Millisecond) + + cancel() + + select { + case err := <-served: + require.NoError(t, err) + case <-time.After(10 * time.Second): + t.Fatal("Serve did not return after its context was cancelled") + } + + assert.NoFileExists(t, cfg.UnixSocket) +}