derp: skip DERP map sources that fail to load

Warn and start on what loaded; abort only if the served map, policy
derpMap included, has no region. Refresh still keeps the old map.

Fixes #3142

CHANGELOG.md

@@ -163,9 +163,9 @@ is not connected, or refuses is "not found". See [the client docs](https://heads
 
 - `derp.paths` files must end in `.yaml`, `.yml`, `.json` or `.hujson`; the extension picks the format
 - `dns.extra_records_path` must end in `.json`, `.hujson`, `.yaml` or `.yml`; the extension picks the format
-- A `derp.paths` file that decodes to neither regions nor `homeparams` now stops headscale from starting instead of being silently ignored
-- A `derp.paths` file with a key `tailcfg.DERPMap` has no field for (e.g. a misspelt `regionscore`) stops headscale from starting instead of the key being ignored, as Tailscale SaaS rejects such a policy `derpMap` [#2932](https://github.com/juanfont/headscale/pull/2932)
-- A DERP map from `derp.paths` or `derp.urls` must meet the region rules Tailscale SaaS applies to a policy `derpMap`: each region's `regionid` matches its key, each node's `regionid` matches its region, and a region has at least one node. A `derp.paths` file that breaks one stops headscale from starting; a `derp.urls` map that breaks one fails to load. Clients count latency by `regionid`, so a mismatch made them pick a home region the map did not have [#2932](https://github.com/juanfont/headscale/pull/2932)
+- A `derp.paths` file that decodes to neither regions nor `homeparams` now fails to load instead of being silently ignored
+- A `derp.paths` file with a key `tailcfg.DERPMap` has no field for (e.g. a misspelt `regionscore`) fails to load instead of the key being ignored, as Tailscale SaaS rejects such a policy `derpMap` [#2932](https://github.com/juanfont/headscale/pull/2932)
+- A DERP map from `derp.paths` or `derp.urls` must meet the region rules Tailscale SaaS applies to a policy `derpMap`: each region's `regionid` matches its key, each node's `regionid` matches its region, and a region has at least one node. A map that breaks one fails to load. Clients count latency by `regionid`, so a mismatch made them pick a home region the map did not have [#2932](https://github.com/juanfont/headscale/pull/2932)
 - `unix_socket_permission` must be a quoted octal string no higher than `"0777"`, e.g. `"0770"`; an unquoted number or an invalid value now stops headscale from starting instead of silently using `0700` [#3540](https://github.com/juanfont/headscale/pull/3540)
 - `tls_cert_path`, `tls_key_path`, `tls_letsencrypt_*`, `acme_url` and `acme_email` are deprecated in favour of `tls.cert_path`, `tls.key_path` and `tls.acme.*` [#3554](https://github.com/juanfont/headscale/pull/3554)
   - The old keys continue to work, with a warning, but will be removed in a future release
@@ -190,6 +190,7 @@ is not connected, or refuses is "not found". See [the client docs](https://heads
 - A SIGHUP now reloads the certificate from `tls.cert_path` and `tls.key_path` without a restart; a certificate that cannot be loaded or is not valid now is logged as an error and the previous one keeps serving [#3027](https://github.com/juanfont/headscale/issues/3027)
 - The NixOS module's `headscale` service now supports `systemctl reload`
 - DERP map refreshes keep the node order. Clients keep their home relay. [#3649](https://github.com/juanfont/headscale/pull/3649)
+- A `derp.urls` or `derp.paths` source that fails to load (unreachable, an HTTP status other than 200, or an invalid map) is logged as a warning and skipped instead of stopping headscale from starting. Headscale still refuses to start when no DERP region remains; a refresh with a failing source keeps the previous map [#3142](https://github.com/juanfont/headscale/issues/3142)
 - Nodes with `disable-ipv4` get their peers over IPv6 only. [#3649](https://github.com/juanfont/headscale/pull/3649)
 - A node name follows a new hostname at login, unless an administrator set the name. [#3649](https://github.com/juanfont/headscale/pull/3649)
 - `AllowedIPs` list the node's own addresses first. `tailscale whois` reads them in this order. [#3649](https://github.com/juanfont/headscale/pull/3649)

config-example.yaml

@@ -110,7 +110,9 @@ derp:
     ipv4: 198.51.100.1
     ipv6: 2001:db8::1
 
-  # List of externally available DERP maps encoded in JSON
+  # List of externally available DERP maps encoded in JSON.
+  # A URL or path that fails to load is logged and skipped; headscale only
+  # refuses to start when no DERP region remains.
   urls:
     - https://controlplane.tailscale.com/derpmap/default
 

docs/ref/derp.md

@@ -57,10 +57,15 @@ The DERP map offered to clients can be customized with a [dedicated YAML-configu
 file](https://github.com/juanfont/headscale/blob/main/derp-example.yaml). The file extension picks the format: `.yaml`
 or `.yml`, or the JSON format Tailscale serves DERP maps in as `.json` or `.hujson`. Both JSON extensions are read as
 [HuJSON](https://github.com/tailscale/hujson), so comments and trailing commas are allowed. This allows to modify previously
-loaded DERP maps fetched via URL or to offer your own, custom DERP servers to nodes. A key that has no field in
-[DERPMap](https://pkg.go.dev/tailscale.com/tailcfg#DERPMap) stops headscale from starting. So does a region whose
-`regionid` differs from its key, a node whose `regionid` differs from its region, or a region without nodes: clients
-count latency by `regionid`.
+loaded DERP maps fetched via URL or to offer your own, custom DERP servers to nodes. A file with a key that has no field
+in [DERPMap](https://pkg.go.dev/tailscale.com/tailcfg#DERPMap) fails to load. So does a region whose `regionid` differs
+from its key, a node whose `regionid` differs from its region, or a region without nodes: clients count latency by
+`regionid`.
+
+A `derp.urls` or `derp.paths` source that fails to load, for example a URL that is down or a file with an error, is
+logged as a warning and skipped. Headscale starts with the DERP regions from the other sources, the embedded DERP server
+and the policy's `derpMap`, and refuses to start only when no region remains. A refresh with a failing source keeps the
+previous DERP map. A skipped file also skips what it removes: a region it sets to `null` comes back.
 
 === "Remove specific DERP regions"
 

hscontrol/app.go

@@ -615,9 +615,11 @@ func (h *Headscale) Serve() error {
 		go h.DERPServer.ServeSTUN()
 	}
 
-	derpMap, err := derp.GetDERPMap(h.cfg.DERP)
-	if err != nil {
-		return fmt.Errorf("getting DERPMap: %w", err)
+	// Start on the sources that loaded; only a map with no region at all,
+	// after the embedded region and the policy's derpMap, is fatal.
+	derpMap, derpErr := derp.GetDERPMap(h.cfg.DERP)
+	if derpErr != nil {
+		log.Warn().Err(derpErr).Msg("skipping DERP map sources that failed to load")
 	}
 
 	if h.cfg.DERP.ServerEnabled && h.cfg.DERP.AutomaticallyAddEmbeddedDerpRegion {
@@ -629,12 +631,12 @@ func (h *Headscale) Serve() error {
 		derpMap.Regions[region.RegionID] = &region
 	}
 
-	if len(derpMap.Regions) == 0 {
-		return errEmptyInitialDERPMap
-	}
-
 	h.state.SetDERPMap(derpMap)
 
+	if h.state.DERPMap().Regions().Len() == 0 {
+		return errors.Join(errEmptyInitialDERPMap, derpErr)
+	}
+
 	// Start ephemeral node garbage collector and schedule all nodes
 	// that are already in the database and ephemeral. If they are still
 	// around between restarts, they will reconnect and the GC will

hscontrol/derp/derp.go

@@ -235,8 +235,16 @@ func ApplyPolicy(base, pol *tailcfg.DERPMap) *tailcfg.DERPMap {
 	return out
 }
 
+// GetDERPMap merges every configured DERP map source. A source that fails to
+// load is skipped rather than failing the rest, as a derp.urls server is often
+// a third party's: the map always holds what did load, and the error joins
+// every failure for the caller to judge.
 func GetDERPMap(cfg types.DERPConfig) (*tailcfg.DERPMap, error) {
-	var derpMaps []*tailcfg.DERPMap
+	var (
+		derpMaps []*tailcfg.DERPMap
+		errs     []error
+	)
+
 	if cfg.DERPMap != nil {
 		derpMaps = append(derpMaps, cfg.DERPMap)
 	}
@@ -244,7 +252,8 @@ func GetDERPMap(cfg types.DERPConfig) (*tailcfg.DERPMap, error) {
 	for _, addr := range cfg.URLs {
 		derpMap, err := loadDERPMapFromURL(addr)
 		if err != nil {
-			return nil, err
+			errs = append(errs, err)
+			continue
 		}
 
 		derpMaps = append(derpMaps, derpMap)
@@ -253,7 +262,8 @@ func GetDERPMap(cfg types.DERPConfig) (*tailcfg.DERPMap, error) {
 	for _, path := range cfg.Paths {
 		derpMap, err := loadDERPMapFromPath(path)
 		if err != nil {
-			return nil, err
+			errs = append(errs, err)
+			continue
 		}
 
 		derpMaps = append(derpMaps, derpMap)
@@ -262,7 +272,7 @@ func GetDERPMap(cfg types.DERPConfig) (*tailcfg.DERPMap, error) {
 	derpMap := mergeDERPMaps(derpMaps)
 	shuffleDERPMap(derpMap)
 
-	return derpMap, nil
+	return derpMap, errors.Join(errs...)
 }
 
 func shuffleDERPMap(dm *tailcfg.DERPMap) {

hscontrol/derp/derp_test.go

@@ -500,6 +500,49 @@ func TestLoadDERPMapFromURLValidates(t *testing.T) {
 	require.ErrorIs(t, err, ErrRegionIDKey)
 }
 
+// TestGetDERPMapSkipsFailedSources: a derp.urls server is often run by a
+// third party, so one failing source must not cost the regions of the others
+// (https://github.com/juanfont/headscale/issues/3142).
+func TestGetDERPMapSkipsFailedSources(t *testing.T) {
+	good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte(`{"Regions": {"1": {"RegionID": 1, "Nodes": [{"Name": "1a", "RegionID": 1}]}}}`))
+	}))
+	defer good.Close()
+
+	down := httptest.NewServer(http.NotFoundHandler())
+	defer down.Close()
+
+	goodURL, err := url.Parse(good.URL)
+	require.NoError(t, err)
+
+	downURL, err := url.Parse(down.URL)
+	require.NoError(t, err)
+
+	goodPath := filepath.Join(t.TempDir(), "good.yaml")
+	require.NoError(t, os.WriteFile(goodPath, []byte(`
+regions:
+  2:
+    regionid: 2
+    nodes:
+      - name: 2a
+        regionid: 2
+`), 0o600))
+
+	brokenPath := filepath.Join(t.TempDir(), "broken.yaml")
+	require.NoError(t, os.WriteFile(brokenPath, []byte("regions: [\n"), 0o600))
+
+	dm, err := GetDERPMap(types.DERPConfig{
+		URLs:  []url.URL{*goodURL, *downURL},
+		Paths: []string{goodPath, brokenPath, filepath.Join(t.TempDir(), "missing.yaml")},
+	})
+	require.ErrorIs(t, err, errDERPMapStatus)
+	require.ErrorContains(t, err, down.URL)
+	require.ErrorContains(t, err, brokenPath)
+	require.ErrorContains(t, err, "missing.yaml")
+	require.Contains(t, dm.Regions, tailcfg.DERPRegionID(1))
+	require.Contains(t, dm.Regions, tailcfg.DERPRegionID(2))
+}
+
 // TestLoadDERPMapFromURLRejectsHTTPError: an error page that happens to be
 // JSON must not pass as an empty map, or a refresh drops that URL's regions.
 func TestLoadDERPMapFromURLRejectsHTTPError(t *testing.T) {