From 5af7d585bd74eb2185986e751121d3a8f1ae2069 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Mon, 9 Nov 2026 05:39:46 +0000 Subject: [PATCH] hscontrol: refresh the DERP map without blocking other scheduled tasks Updates #1926 --- hscontrol/app.go | 81 ++++++++++++++++++++++++------------------- hscontrol/app_test.go | 54 +++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 35 deletions(-) diff --git a/hscontrol/app.go b/hscontrol/app.go index 057353336..aae699a0b 100644 --- a/hscontrol/app.go +++ b/hscontrol/app.go @@ -322,13 +322,10 @@ func (h *Headscale) scheduledTasks(ctx context.Context) { lastExpiryCheck := time.Unix(0, 0) - var derpTickerChan <-chan time.Time - if h.cfg.DERP.AutoUpdate && h.cfg.DERP.UpdateFrequency != 0 { - derpTicker := time.NewTicker(h.cfg.DERP.UpdateFrequency) - defer derpTicker.Stop() - - derpTickerChan = derpTicker.C + // A refresh retries for minutes when a source is unreachable; the + // tasks below must not wait for it. + go h.refreshDERPMap(ctx) } var extraRecordsUpdate <-chan []tailcfg.DNSRecord @@ -414,35 +411,6 @@ func (h *Headscale) scheduledTasks(ctx context.Context) { } } - case <-derpTickerChan: - log.Info().Msg("fetching DERPMap updates") - - derpMap, err := backoff.Retry(ctx, func() (*tailcfg.DERPMap, error) { //nolint:contextcheck - derpMap, err := derp.GetDERPMap(h.cfg.DERP, h.cfg.DNSConfig.BaseDomain) - if err != nil { - return nil, err - } - - if h.cfg.DERP.Server.Enabled && h.cfg.DERP.Server.AutomaticallyAddEmbeddedDerpRegion { - region, err := h.DERPServer.GenerateRegion() - if err != nil { - return nil, fmt.Errorf("generating embedded DERP region: %w", err) - } - - derpMap.Regions[region.RegionID] = ®ion - } - - return derpMap, nil - }, backoff.WithBackOff(backoff.NewExponentialBackOff())) - if err != nil { - log.Error().Err(err).Msg("failed to build new DERPMap, retrying later") - continue - } - - h.state.SetDERPMap(derpMap) - - h.Change(change.DERPMap()) - case records, ok := <-extraRecordsUpdate: if !ok { continue @@ -459,6 +427,49 @@ func (h *Headscale) scheduledTasks(ctx context.Context) { } } +// refreshDERPMap reloads the DERP map every derp.update_frequency until +// ctx ends. +func (h *Headscale) refreshDERPMap(ctx context.Context) { + ticker := time.NewTicker(h.cfg.DERP.UpdateFrequency) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + + log.Info().Msg("fetching DERPMap updates") + + derpMap, err := backoff.Retry(ctx, func() (*tailcfg.DERPMap, error) { //nolint:contextcheck + derpMap, err := derp.GetDERPMap(h.cfg.DERP, h.cfg.DNSConfig.BaseDomain) + if err != nil { + return nil, err + } + + if h.cfg.DERP.Server.Enabled && h.cfg.DERP.Server.AutomaticallyAddEmbeddedDerpRegion { + region, err := h.DERPServer.GenerateRegion() + if err != nil { + return nil, fmt.Errorf("generating embedded DERP region: %w", err) + } + + derpMap.Regions[region.RegionID] = ®ion + } + + return derpMap, nil + }, backoff.WithBackOff(backoff.NewExponentialBackOff())) + if err != nil { + log.Error().Err(err).Msg("failed to build new DERPMap, retrying later") + continue + } + + h.state.SetDERPMap(derpMap) + + h.Change(change.DERPMap()) + } +} + // ensureUnixSocketIsAbsent will check if the given path for headscales unix socket is clear // and will remove it if it is not. func (h *Headscale) ensureUnixSocketIsAbsent() error { diff --git a/hscontrol/app_test.go b/hscontrol/app_test.go index 51b0d3a61..d432633da 100644 --- a/hscontrol/app_test.go +++ b/hscontrol/app_test.go @@ -427,3 +427,57 @@ func TestNewHeadscaleGoOnlyProvider(t *testing.T) { assert.Same(t, provider, cfg.DNSProvider()) } + +// TestDERPRefreshDoesNotBlockScheduledTasks: a DERP refresh stuck on an +// unreachable source leaves the other scheduled tasks running. +func TestDERPRefreshDoesNotBlockScheduledTasks(t *testing.T) { + requested := make(chan struct{}, 1) + release := make(chan struct{}) + stuck := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + select { + case requested <- struct{}{}: + default: + } + + select { + case <-release: + case <-r.Context().Done(): + } + })) + t.Cleanup(stuck.Close) + t.Cleanup(func() { close(release) }) + + app := createTestApp(t, func(c *types.Config) { + c.DERP.URLs = []types.HTTPURL{types.HTTPURL(stuck.URL)} + c.DERP.AutoUpdate = true + c.DERP.UpdateFrequency = time.Millisecond + }) + + records := filepath.Join(t.TempDir(), "records.json") + require.NoError(t, os.WriteFile(records, []byte("[]"), 0o600)) + + var err error + + app.extraRecordMan, err = dns.NewExtraRecordsManager(records) + require.NoError(t, err) + + go app.extraRecordMan.Run() + + t.Cleanup(app.extraRecordMan.Close) + + ctx, cancel := context.WithCancel(t.Context()) + t.Cleanup(cancel) + + go app.scheduledTasks(ctx) + + select { + case <-requested: + case <-time.After(10 * time.Second): + t.Fatal("no DERP refresh started") + } + + require.NoError(t, os.WriteFile(records, []byte(`[{"name":"a.example.com","type":"A","value":"100.64.0.1"}]`), 0o600)) + + assert.Eventually(t, func() bool { return app.cfg.HasExtraRecord("a.example.com") }, 10*time.Second, 50*time.Millisecond, + "extra records not reloaded while the DERP refresh is stuck") +}