mirror of
https://github.com/juanfont/headscale.git
synced 2026-11-20 04:14:32 +09:00
hscontrol: stop Serve through its context
Serve only stopped on signals; a failed listener left it half running. Updates #3499
This commit is contained in:
1 parent
475861cdd1
commit
9358c026d4
3 files changed
+146
-65
No files matched your search
@@ -3,7 +3,7 @@ package cli
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/juanfont/headscale/hscontrol/types"
|
||||
@@ -29,12 +29,15 @@ var serveCmd = &cobra.Command{
|
||||
return fmt.Errorf("initializing: %w", err)
|
||||
}
|
||||
|
||||
err = app.Serve()
|
||||
if err == nil || errors.Is(err, http.ErrServerClosed) {
|
||||
return nil
|
||||
ctx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
|
||||
defer stop()
|
||||
|
||||
err = app.Serve(ctx)
|
||||
if err != nil {
|
||||
return classifyServeError(err)
|
||||
}
|
||||
|
||||
return classifyServeError(err)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
+58
-60
@@ -67,6 +67,8 @@ var (
|
||||
errEmptyInitialDERPMap = errors.New(
|
||||
"initial DERPMap is empty, Headscale requires at least one entry",
|
||||
)
|
||||
errTailSQLDatabase = errors.New("tailsql only supports sqlite")
|
||||
errTailSQLAuthKey = errors.New("tailsql requires TS_AUTHKEY to be set")
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -570,19 +572,31 @@ func (h *Headscale) createRouter(apiV1Mux, apiV2Mux http.Handler) *chi.Mux {
|
||||
return r
|
||||
}
|
||||
|
||||
// Serve launches the HTTP servers that run Headscale and its API.
|
||||
// Serve launches the HTTP servers that run Headscale and its API. It shuts
|
||||
// down gracefully when ctx is cancelled or any of its components fails, and
|
||||
// returns that component's error.
|
||||
//
|
||||
//nolint:gocyclo // complex server startup function
|
||||
func (h *Headscale) Serve() error {
|
||||
func (h *Headscale) Serve(ctx context.Context) error {
|
||||
var err error
|
||||
|
||||
capver.CanOldCodeBeCleanedUp()
|
||||
|
||||
if tailsqlEnabled {
|
||||
if h.cfg.Database.Type != types.DatabaseSqlite {
|
||||
return fmt.Errorf("%w, got %q", errTailSQLDatabase, h.cfg.Database.Type)
|
||||
}
|
||||
|
||||
if tailsqlTSKey == "" {
|
||||
return errTailSQLAuthKey
|
||||
}
|
||||
}
|
||||
|
||||
if profilingEnabled {
|
||||
if profilingPath != "" {
|
||||
err = os.MkdirAll(profilingPath, os.ModePerm)
|
||||
if err != nil {
|
||||
log.Fatal().Err(err).Msg("failed to create profiling directory")
|
||||
return fmt.Errorf("creating profiling directory: %w", err)
|
||||
}
|
||||
|
||||
defer profile.Start(profile.ProfilePath(profilingPath)).Stop()
|
||||
@@ -601,6 +615,12 @@ func (h *Headscale) Serve() error {
|
||||
Str("minimum_version", capver.TailscaleVersion(capver.MinSupportedCapabilityVersion)).
|
||||
Msg("Clients with a lower minimum version will be rejected")
|
||||
|
||||
// The cancel stops what already started when Serve returns early.
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
errorGroup, ctx := errgroup.WithContext(ctx)
|
||||
|
||||
h.mapBatcher = mapper.NewBatcherAndMapper(h.cfg, h.state)
|
||||
|
||||
h.mapBatcher.Start()
|
||||
@@ -617,13 +637,13 @@ func (h *Headscale) Serve() error {
|
||||
|
||||
// Start on the sources that loaded; only a map with no region at all,
|
||||
// after the embedded region and the policy's derpMap, is fatal.
|
||||
derpMap, derpErr := derp.GetDERPMap(h.cfg.DERP)
|
||||
derpMap, derpErr := derp.GetDERPMap(h.cfg.DERP) //nolint:contextcheck // startup fetch, bounded by its HTTP client
|
||||
if derpErr != nil {
|
||||
log.Warn().Err(derpErr).Msg("skipping DERP map sources that failed to load")
|
||||
}
|
||||
|
||||
if h.cfg.DERP.ServerEnabled && h.cfg.DERP.AutomaticallyAddEmbeddedDerpRegion {
|
||||
region, err := h.DERPServer.GenerateRegion()
|
||||
region, err := h.DERPServer.GenerateRegion() //nolint:contextcheck // only a debug DNS lookup uses a context
|
||||
if err != nil {
|
||||
return fmt.Errorf("generating embedded DERP region: %w", err)
|
||||
}
|
||||
@@ -656,7 +676,7 @@ func (h *Headscale) Serve() error {
|
||||
|
||||
h.cfg.SetExtraRecords(h.extraRecordMan.Records())
|
||||
|
||||
go h.extraRecordMan.Run()
|
||||
go h.extraRecordMan.Run() //nolint:contextcheck // stopped by Close
|
||||
defer h.extraRecordMan.Close()
|
||||
}
|
||||
|
||||
@@ -665,18 +685,7 @@ func (h *Headscale) Serve() error {
|
||||
|
||||
// Start all scheduled tasks, e.g. expiring nodes, derp updates and
|
||||
// records updates
|
||||
scheduleCtx, scheduleCancel := context.WithCancel(context.Background())
|
||||
defer scheduleCancel()
|
||||
|
||||
go h.scheduledTasks(scheduleCtx)
|
||||
|
||||
// Prepare group for running listeners
|
||||
errorGroup := new(errgroup.Group)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
go h.scheduledTasks(ctx)
|
||||
|
||||
//
|
||||
//
|
||||
@@ -695,7 +704,7 @@ func (h *Headscale) Serve() error {
|
||||
return fmt.Errorf("setting up unix socket: %w", err)
|
||||
}
|
||||
|
||||
socketListener, err := new(net.ListenConfig).Listen(context.Background(), "unix", h.cfg.UnixSocket)
|
||||
socketListener, err := new(net.ListenConfig).Listen(ctx, "unix", h.cfg.UnixSocket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("setting up socket: %w", err)
|
||||
}
|
||||
@@ -738,7 +747,7 @@ func (h *Headscale) Serve() error {
|
||||
ReadTimeout: types.HTTPTimeout,
|
||||
}
|
||||
|
||||
errorGroup.Go(func() error { return socketServer.Serve(socketListener) })
|
||||
errorGroup.Go(serveHTTP(socketServer, socketListener))
|
||||
|
||||
//
|
||||
//
|
||||
@@ -776,7 +785,7 @@ func (h *Headscale) Serve() error {
|
||||
httpServer.TLSConfig = tlsConfig
|
||||
httpListener, err = tls.Listen("tcp", h.cfg.Addr, tlsConfig)
|
||||
} else {
|
||||
httpListener, err = new(net.ListenConfig).Listen(context.Background(), "tcp", h.cfg.Addr)
|
||||
httpListener, err = new(net.ListenConfig).Listen(ctx, "tcp", h.cfg.Addr)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
@@ -788,7 +797,7 @@ func (h *Headscale) Serve() error {
|
||||
}
|
||||
}
|
||||
|
||||
errorGroup.Go(func() error { return httpServer.Serve(httpListener) })
|
||||
errorGroup.Go(serveHTTP(httpServer, httpListener))
|
||||
|
||||
log.Info().
|
||||
Msgf("listening and serving HTTP on: %s", h.cfg.Addr)
|
||||
@@ -820,7 +829,7 @@ func (h *Headscale) Serve() error {
|
||||
WriteTimeout: types.HTTPTimeout,
|
||||
}
|
||||
|
||||
errorGroup.Go(func() error { return insecureTLSServer.Serve(insecureTLSListener) })
|
||||
errorGroup.Go(serveHTTP(insecureTLSServer, insecureTLSListener))
|
||||
|
||||
log.Warn().
|
||||
Str("addr", addr).
|
||||
@@ -866,7 +875,7 @@ func (h *Headscale) Serve() error {
|
||||
|
||||
debugHTTPServer = h.debugHTTPServer()
|
||||
|
||||
errorGroup.Go(func() error { return debugHTTPServer.Serve(debugHTTPListener) })
|
||||
errorGroup.Go(serveHTTP(debugHTTPServer, debugHTTPListener))
|
||||
|
||||
log.Info().
|
||||
Msgf("listening and serving debug and metrics on: %s", h.cfg.MetricsAddr)
|
||||
@@ -877,18 +886,6 @@ func (h *Headscale) Serve() error {
|
||||
var tailsqlCancel context.CancelFunc
|
||||
|
||||
if tailsqlEnabled {
|
||||
if h.cfg.Database.Type != types.DatabaseSqlite {
|
||||
//nolint:gocritic // exitAfterDefer: Fatal exits during initialization before servers start
|
||||
log.Fatal().
|
||||
Str("type", h.cfg.Database.Type).
|
||||
Msgf("tailsql only support %q", types.DatabaseSqlite)
|
||||
}
|
||||
|
||||
if tailsqlTSKey == "" {
|
||||
//nolint:gocritic // exitAfterDefer: Fatal exits during initialization before servers start
|
||||
log.Fatal().Msg("tailsql requires TS_AUTHKEY to be set")
|
||||
}
|
||||
|
||||
var tailsqlCtx context.Context
|
||||
|
||||
tailsqlCtx, tailsqlCancel = context.WithCancel(ctx)
|
||||
@@ -898,21 +895,16 @@ func (h *Headscale) Serve() error {
|
||||
})
|
||||
}
|
||||
|
||||
// Handle common process-killing signals so we can gracefully shut down:
|
||||
sigc := make(chan os.Signal, 1)
|
||||
signal.Notify(sigc,
|
||||
syscall.SIGHUP,
|
||||
syscall.SIGINT,
|
||||
syscall.SIGTERM,
|
||||
syscall.SIGQUIT,
|
||||
syscall.SIGHUP)
|
||||
// Stop signals belong to the caller, which cancels ctx; SIGHUP reloads.
|
||||
sighup := make(chan os.Signal, 1)
|
||||
|
||||
sigFunc := func(c chan os.Signal) {
|
||||
// Wait for a SIGINT or SIGKILL:
|
||||
signal.Notify(sighup, syscall.SIGHUP)
|
||||
defer signal.Stop(sighup)
|
||||
|
||||
errorGroup.Go(func() error {
|
||||
for {
|
||||
sig := <-c
|
||||
switch sig {
|
||||
case syscall.SIGHUP:
|
||||
select {
|
||||
case sig := <-sighup:
|
||||
log.Info().
|
||||
Str("signal", sig.String()).
|
||||
Msg("Received SIGHUP")
|
||||
@@ -938,14 +930,13 @@ func (h *Headscale) Serve() error {
|
||||
continue
|
||||
}
|
||||
|
||||
default:
|
||||
case <-ctx.Done():
|
||||
info := func(msg string) { log.Info().Msg(msg) }
|
||||
|
||||
log.Info().
|
||||
Str("signal", sig.String()).
|
||||
Msg("Received signal to stop, shutting down gracefully")
|
||||
Str("reason", context.Cause(ctx).Error()).
|
||||
Msg("shutting down gracefully")
|
||||
|
||||
scheduleCancel()
|
||||
h.ephemeralGC.Close()
|
||||
|
||||
// Gracefully shut down servers
|
||||
@@ -1050,20 +1041,27 @@ func (h *Headscale) Serve() error {
|
||||
log.Info().
|
||||
Msg("Headscale stopped")
|
||||
|
||||
return
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
errorGroup.Go(func() error {
|
||||
sigFunc(sigc)
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
return errorGroup.Wait()
|
||||
}
|
||||
|
||||
// serveHTTP treats a graceful Shutdown as success so only a real failure
|
||||
// cancels the group.
|
||||
func serveHTTP(srv *http.Server, l net.Listener) func() error {
|
||||
return func() error {
|
||||
err := srv.Serve(l)
|
||||
if errors.Is(err, http.ErrServerClosed) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// tlsBundle carries the TLS settings produced by getTLSSettings. When
|
||||
// HTTP-01 ACME is configured, ACMEServer and ACMEListener are populated
|
||||
// so the caller can register the challenge listener with the errgroup
|
||||
|
||||
@@ -2,11 +2,16 @@ package hscontrol
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/juanfont/headscale/hscontrol/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSecurityHeaders(t *testing.T) {
|
||||
@@ -79,3 +84,78 @@ func TestDebugHostGuard(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// serveTestConfig returns a config for running Serve on loopback with the
|
||||
// embedded DERP server enabled on stunAddr.
|
||||
func serveTestConfig(t *testing.T, stunAddr string) *types.Config {
|
||||
t.Helper()
|
||||
|
||||
dir := t.TempDir()
|
||||
|
||||
return &types.Config{
|
||||
ServerURL: "http://127.0.0.1:8080",
|
||||
Addr: "127.0.0.1:0",
|
||||
UnixSocket: filepath.Join(dir, "hs.sock"),
|
||||
UnixSocketPermission: 0o770,
|
||||
NoisePrivateKeyPath: filepath.Join(dir, "noise.key"),
|
||||
Database: types.DatabaseConfig{
|
||||
Type: types.DatabaseSqlite,
|
||||
Sqlite: types.SqliteConfig{Path: filepath.Join(dir, "hs.db")},
|
||||
},
|
||||
Policy: types.PolicyConfig{Mode: types.PolicyModeDB},
|
||||
DERP: types.DERPConfig{
|
||||
ServerEnabled: true,
|
||||
AutomaticallyAddEmbeddedDerpRegion: true,
|
||||
ServerPrivateKeyPath: filepath.Join(dir, "derp.key"),
|
||||
ServerRegionID: 999,
|
||||
ServerRegionCode: "test",
|
||||
STUNAddr: stunAddr,
|
||||
},
|
||||
Tuning: types.Tuning{BatchChangeDelay: 100 * time.Millisecond, BatcherWorkers: 1},
|
||||
}
|
||||
}
|
||||
|
||||
// Cancelling the context must run the graceful shutdown and return nil, so
|
||||
// an embedder can stop Serve without sending the process a signal.
|
||||
func TestServeStopsOnContextCancel(t *testing.T) {
|
||||
cfg := serveTestConfig(t, "127.0.0.1:0")
|
||||
|
||||
app, err := NewHeadscale(cfg)
|
||||
require.NoError(t, err)
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
|
||||
served := make(chan error, 1)
|
||||
|
||||
go func() { served <- app.Serve(ctx) }()
|
||||
|
||||
client := &http.Client{Transport: &http.Transport{
|
||||
DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
|
||||
return new(net.Dialer).DialContext(ctx, "unix", cfg.UnixSocket)
|
||||
},
|
||||
}}
|
||||
|
||||
assert.EventuallyWithT(t, func(c *assert.CollectT) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://unix/api/v1/health", nil)
|
||||
require.NoError(c, err)
|
||||
|
||||
res, err := client.Do(req)
|
||||
require.NoError(c, err)
|
||||
|
||||
defer res.Body.Close()
|
||||
|
||||
assert.Equal(c, http.StatusOK, res.StatusCode)
|
||||
}, 10*time.Second, 50*time.Millisecond)
|
||||
|
||||
cancel()
|
||||
|
||||
select {
|
||||
case err := <-served:
|
||||
require.NoError(t, err)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("Serve did not return after its context was cancelled")
|
||||
}
|
||||
|
||||
assert.NoFileExists(t, cfg.UnixSocket)
|
||||
}
|
||||
Reference in new issue
Block a user