diff --git a/CHANGELOG.md b/CHANGELOG.md index c8262ac7d..6f0345a1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -163,9 +163,9 @@ is not connected, or refuses is "not found". See [the client docs](https://heads - `derp.paths` files must end in `.yaml`, `.yml`, `.json` or `.hujson`; the extension picks the format - `dns.extra_records_path` must end in `.json`, `.hujson`, `.yaml` or `.yml`; the extension picks the format -- A `derp.paths` file that decodes to neither regions nor `homeparams` now stops headscale from starting instead of being silently ignored -- A `derp.paths` file with a key `tailcfg.DERPMap` has no field for (e.g. a misspelt `regionscore`) stops headscale from starting instead of the key being ignored, as Tailscale SaaS rejects such a policy `derpMap` [#2932](https://github.com/juanfont/headscale/pull/2932) -- A DERP map from `derp.paths` or `derp.urls` must meet the region rules Tailscale SaaS applies to a policy `derpMap`: each region's `regionid` matches its key, each node's `regionid` matches its region, and a region has at least one node. A `derp.paths` file that breaks one stops headscale from starting; a `derp.urls` map that breaks one fails to load. Clients count latency by `regionid`, so a mismatch made them pick a home region the map did not have [#2932](https://github.com/juanfont/headscale/pull/2932) +- A `derp.paths` file that decodes to neither regions nor `homeparams` now fails to load instead of being silently ignored +- A `derp.paths` file with a key `tailcfg.DERPMap` has no field for (e.g. a misspelt `regionscore`) fails to load instead of the key being ignored, as Tailscale SaaS rejects such a policy `derpMap` [#2932](https://github.com/juanfont/headscale/pull/2932) +- A DERP map from `derp.paths` or `derp.urls` must meet the region rules Tailscale SaaS applies to a policy `derpMap`: each region's `regionid` matches its key, each node's `regionid` matches its region, and a region has at least one node. A map that breaks one fails to load. Clients count latency by `regionid`, so a mismatch made them pick a home region the map did not have [#2932](https://github.com/juanfont/headscale/pull/2932) - `unix_socket_permission` must be a quoted octal string no higher than `"0777"`, e.g. `"0770"`; an unquoted number or an invalid value now stops headscale from starting instead of silently using `0700` [#3540](https://github.com/juanfont/headscale/pull/3540) - `tls_cert_path`, `tls_key_path`, `tls_letsencrypt_*`, `acme_url` and `acme_email` are deprecated in favour of `tls.cert_path`, `tls.key_path` and `tls.acme.*` [#3554](https://github.com/juanfont/headscale/pull/3554) - The old keys continue to work, with a warning, but will be removed in a future release @@ -190,6 +190,7 @@ is not connected, or refuses is "not found". See [the client docs](https://heads - A SIGHUP now reloads the certificate from `tls.cert_path` and `tls.key_path` without a restart; a certificate that cannot be loaded or is not valid now is logged as an error and the previous one keeps serving [#3027](https://github.com/juanfont/headscale/issues/3027) - The NixOS module's `headscale` service now supports `systemctl reload` - DERP map refreshes keep the node order. Clients keep their home relay. [#3649](https://github.com/juanfont/headscale/pull/3649) +- A `derp.urls` or `derp.paths` source that fails to load (unreachable, an HTTP status other than 200, or an invalid map) is logged as a warning and skipped instead of stopping headscale from starting. Headscale still refuses to start when no DERP region remains; a refresh with a failing source keeps the previous map [#3142](https://github.com/juanfont/headscale/issues/3142) - Nodes with `disable-ipv4` get their peers over IPv6 only. [#3649](https://github.com/juanfont/headscale/pull/3649) - A node name follows a new hostname at login, unless an administrator set the name. [#3649](https://github.com/juanfont/headscale/pull/3649) - `AllowedIPs` list the node's own addresses first. `tailscale whois` reads them in this order. [#3649](https://github.com/juanfont/headscale/pull/3649) diff --git a/config-example.yaml b/config-example.yaml index 53dad8de1..1e78d0b17 100644 --- a/config-example.yaml +++ b/config-example.yaml @@ -110,7 +110,9 @@ derp: ipv4: 198.51.100.1 ipv6: 2001:db8::1 - # List of externally available DERP maps encoded in JSON + # List of externally available DERP maps encoded in JSON. + # A URL or path that fails to load is logged and skipped; headscale only + # refuses to start when no DERP region remains. urls: - https://controlplane.tailscale.com/derpmap/default diff --git a/docs/ref/derp.md b/docs/ref/derp.md index 9c47ec592..390220c7a 100644 --- a/docs/ref/derp.md +++ b/docs/ref/derp.md @@ -57,10 +57,15 @@ The DERP map offered to clients can be customized with a [dedicated YAML-configu file](https://github.com/juanfont/headscale/blob/main/derp-example.yaml). The file extension picks the format: `.yaml` or `.yml`, or the JSON format Tailscale serves DERP maps in as `.json` or `.hujson`. Both JSON extensions are read as [HuJSON](https://github.com/tailscale/hujson), so comments and trailing commas are allowed. This allows to modify previously -loaded DERP maps fetched via URL or to offer your own, custom DERP servers to nodes. A key that has no field in -[DERPMap](https://pkg.go.dev/tailscale.com/tailcfg#DERPMap) stops headscale from starting. So does a region whose -`regionid` differs from its key, a node whose `regionid` differs from its region, or a region without nodes: clients -count latency by `regionid`. +loaded DERP maps fetched via URL or to offer your own, custom DERP servers to nodes. A file with a key that has no field +in [DERPMap](https://pkg.go.dev/tailscale.com/tailcfg#DERPMap) fails to load. So does a region whose `regionid` differs +from its key, a node whose `regionid` differs from its region, or a region without nodes: clients count latency by +`regionid`. + +A `derp.urls` or `derp.paths` source that fails to load, for example a URL that is down or a file with an error, is +logged as a warning and skipped. Headscale starts with the DERP regions from the other sources, the embedded DERP server +and the policy's `derpMap`, and refuses to start only when no region remains. A refresh with a failing source keeps the +previous DERP map. A skipped file also skips what it removes: a region it sets to `null` comes back. === "Remove specific DERP regions" diff --git a/hscontrol/app.go b/hscontrol/app.go index 8f4950097..af627dbd2 100644 --- a/hscontrol/app.go +++ b/hscontrol/app.go @@ -615,9 +615,11 @@ func (h *Headscale) Serve() error { go h.DERPServer.ServeSTUN() } - derpMap, err := derp.GetDERPMap(h.cfg.DERP) - if err != nil { - return fmt.Errorf("getting DERPMap: %w", err) + // Start on the sources that loaded; only a map with no region at all, + // after the embedded region and the policy's derpMap, is fatal. + derpMap, derpErr := derp.GetDERPMap(h.cfg.DERP) + if derpErr != nil { + log.Warn().Err(derpErr).Msg("skipping DERP map sources that failed to load") } if h.cfg.DERP.ServerEnabled && h.cfg.DERP.AutomaticallyAddEmbeddedDerpRegion { @@ -629,12 +631,12 @@ func (h *Headscale) Serve() error { derpMap.Regions[region.RegionID] = ®ion } - if len(derpMap.Regions) == 0 { - return errEmptyInitialDERPMap - } - h.state.SetDERPMap(derpMap) + if h.state.DERPMap().Regions().Len() == 0 { + return errors.Join(errEmptyInitialDERPMap, derpErr) + } + // Start ephemeral node garbage collector and schedule all nodes // that are already in the database and ephemeral. If they are still // around between restarts, they will reconnect and the GC will diff --git a/hscontrol/derp/derp.go b/hscontrol/derp/derp.go index 7ba9e2482..c6bea7bf0 100644 --- a/hscontrol/derp/derp.go +++ b/hscontrol/derp/derp.go @@ -235,8 +235,16 @@ func ApplyPolicy(base, pol *tailcfg.DERPMap) *tailcfg.DERPMap { return out } +// GetDERPMap merges every configured DERP map source. A source that fails to +// load is skipped rather than failing the rest, as a derp.urls server is often +// a third party's: the map always holds what did load, and the error joins +// every failure for the caller to judge. func GetDERPMap(cfg types.DERPConfig) (*tailcfg.DERPMap, error) { - var derpMaps []*tailcfg.DERPMap + var ( + derpMaps []*tailcfg.DERPMap + errs []error + ) + if cfg.DERPMap != nil { derpMaps = append(derpMaps, cfg.DERPMap) } @@ -244,7 +252,8 @@ func GetDERPMap(cfg types.DERPConfig) (*tailcfg.DERPMap, error) { for _, addr := range cfg.URLs { derpMap, err := loadDERPMapFromURL(addr) if err != nil { - return nil, err + errs = append(errs, err) + continue } derpMaps = append(derpMaps, derpMap) @@ -253,7 +262,8 @@ func GetDERPMap(cfg types.DERPConfig) (*tailcfg.DERPMap, error) { for _, path := range cfg.Paths { derpMap, err := loadDERPMapFromPath(path) if err != nil { - return nil, err + errs = append(errs, err) + continue } derpMaps = append(derpMaps, derpMap) @@ -262,7 +272,7 @@ func GetDERPMap(cfg types.DERPConfig) (*tailcfg.DERPMap, error) { derpMap := mergeDERPMaps(derpMaps) shuffleDERPMap(derpMap) - return derpMap, nil + return derpMap, errors.Join(errs...) } func shuffleDERPMap(dm *tailcfg.DERPMap) { diff --git a/hscontrol/derp/derp_test.go b/hscontrol/derp/derp_test.go index e19ce9b86..7915461a1 100644 --- a/hscontrol/derp/derp_test.go +++ b/hscontrol/derp/derp_test.go @@ -500,6 +500,49 @@ func TestLoadDERPMapFromURLValidates(t *testing.T) { require.ErrorIs(t, err, ErrRegionIDKey) } +// TestGetDERPMapSkipsFailedSources: a derp.urls server is often run by a +// third party, so one failing source must not cost the regions of the others +// (https://github.com/juanfont/headscale/issues/3142). +func TestGetDERPMapSkipsFailedSources(t *testing.T) { + good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"Regions": {"1": {"RegionID": 1, "Nodes": [{"Name": "1a", "RegionID": 1}]}}}`)) + })) + defer good.Close() + + down := httptest.NewServer(http.NotFoundHandler()) + defer down.Close() + + goodURL, err := url.Parse(good.URL) + require.NoError(t, err) + + downURL, err := url.Parse(down.URL) + require.NoError(t, err) + + goodPath := filepath.Join(t.TempDir(), "good.yaml") + require.NoError(t, os.WriteFile(goodPath, []byte(` +regions: + 2: + regionid: 2 + nodes: + - name: 2a + regionid: 2 +`), 0o600)) + + brokenPath := filepath.Join(t.TempDir(), "broken.yaml") + require.NoError(t, os.WriteFile(brokenPath, []byte("regions: [\n"), 0o600)) + + dm, err := GetDERPMap(types.DERPConfig{ + URLs: []url.URL{*goodURL, *downURL}, + Paths: []string{goodPath, brokenPath, filepath.Join(t.TempDir(), "missing.yaml")}, + }) + require.ErrorIs(t, err, errDERPMapStatus) + require.ErrorContains(t, err, down.URL) + require.ErrorContains(t, err, brokenPath) + require.ErrorContains(t, err, "missing.yaml") + require.Contains(t, dm.Regions, tailcfg.DERPRegionID(1)) + require.Contains(t, dm.Regions, tailcfg.DERPRegionID(2)) +} + // TestLoadDERPMapFromURLRejectsHTTPError: an error page that happens to be // JSON must not pass as an empty map, or a refresh drops that URL's regions. func TestLoadDERPMapFromURLRejectsHTTPError(t *testing.T) {