diff --git a/.claude/agents/headscale-integration-tester.md b/.claude/agents/headscale-integration-tester.md index 2b25977d..54474ce9 100644 --- a/.claude/agents/headscale-integration-tester.md +++ b/.claude/agents/headscale-integration-tester.md @@ -52,7 +52,7 @@ go test ./integration -timeout 45m **Timeout Guidelines by Test Type**: - **Basic functionality tests**: `--timeout=900s` (15 minutes minimum) - **Route/ACL tests**: `--timeout=1200s` (20 minutes) -- **HA/failover tests**: `--timeout=1800s` (30 minutes) +- **HA/failover tests**: `--timeout=1800s` (30 minutes) - **Long-running tests**: `--timeout=2100s` (35 minutes) - **Full test suite**: `-timeout 45m` (45 minutes) @@ -433,7 +433,7 @@ When you understand a test's purpose through debugging, always add comprehensive // // The test verifies: // - Route announcements are received and tracked -// - ACL policies control route approval correctly +// - ACL policies control route approval correctly // - Only approved routes appear in peer network maps // - Route state persists correctly in the database func TestSubnetRoutes(t *testing.T) { @@ -535,7 +535,7 @@ var nodeKey key.NodePublic assert.EventuallyWithT(t, func(c *assert.CollectT) { nodes, err := headscale.ListNodes() assert.NoError(c, err) - + for _, node := range nodes { if node.GetName() == "router" { routeNode = node @@ -550,7 +550,7 @@ assert.EventuallyWithT(t, func(c *assert.CollectT) { assert.EventuallyWithT(t, func(c *assert.CollectT) { status, err := client.Status() assert.NoError(c, err) - + peerStatus, ok := status.Peer[nodeKey] assert.True(c, ok, "peer should exist in status") requirePeerSubnetRoutesWithCollect(c, peerStatus, expectedPrefixes) @@ -566,7 +566,7 @@ assert.EventuallyWithT(t, func(c *assert.CollectT) { nodes, err := headscale.ListNodes() assert.NoError(c, err) assert.Len(c, nodes, 2) - + // Second unrelated external call - WRONG! status, err := client.Status() assert.NoError(c, err) @@ -577,7 +577,7 @@ assert.EventuallyWithT(t, func(c *assert.CollectT) { assert.EventuallyWithT(t, func(c *assert.CollectT) { nodes, err := headscale.ListNodes() assert.NoError(c, err) - + // NEVER do this! assert.EventuallyWithT(t, func(c2 *assert.CollectT) { status, _ := client.Status() @@ -666,11 +666,11 @@ When working within EventuallyWithT blocks where you need to prevent panics: assert.EventuallyWithT(t, func(c *assert.CollectT) { nodes, err := headscale.ListNodes() assert.NoError(c, err) - + // For array bounds - use require with t to prevent panic assert.Len(c, nodes, 6) // Test expectation require.GreaterOrEqual(t, len(nodes), 3, "need at least 3 nodes to avoid panic") - + // For nil pointer access - use require with t before dereferencing assert.NotNil(c, srs1PeerStatus.PrimaryRoutes) // Test expectation require.NotNil(t, srs1PeerStatus.PrimaryRoutes, "primary routes must be set to avoid panic") @@ -681,7 +681,7 @@ assert.EventuallyWithT(t, func(c *assert.CollectT) { }, 5*time.Second, 200*time.Millisecond, "checking route state") ``` -**Key Principle**: +**Key Principle**: - Use `assert` with `c` (*assert.CollectT) for test expectations that can be retried - Use `require` with `t` (*testing.T) for MUST conditions that prevent panics - Within EventuallyWithT, both are available - choose based on whether failure would cause a panic @@ -704,7 +704,7 @@ assert.EventuallyWithT(t, func(c *assert.CollectT) { assert.EventuallyWithT(t, func(c *assert.CollectT) { status, err := client.Status() assert.NoError(c, err) - + // Check all peers have expected routes for _, peerKey := range status.Peers() { peerStatus := status.Peer[peerKey] diff --git a/.golangci-lint-hook.sh b/.golangci-lint-hook.sh new file mode 100755 index 00000000..ba62e432 --- /dev/null +++ b/.golangci-lint-hook.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Wrapper script for golangci-lint pre-commit hook +# Finds where the current branch diverged from the main branch + +set -euo pipefail + +# Try to find the main branch reference in order of preference: +# 1. upstream/main (common in forks) +# 2. origin/main (common in direct clones) +# 3. main (local branch) +for ref in upstream/main origin/main main; do + if git rev-parse --verify "$ref" >/dev/null 2>&1; then + MAIN_REF="$ref" + break + fi +done + +# If we couldn't find any main branch, just check the last commit +if [ -z "${MAIN_REF:-}" ]; then + MAIN_REF="HEAD~1" +fi + +# Find where current branch diverged from main +MERGE_BASE=$(git merge-base HEAD "$MAIN_REF" 2>/dev/null || echo "HEAD~1") + +# Run golangci-lint only on changes since branch point +exec golangci-lint run --new-from-rev="$MERGE_BASE" --timeout=5m --fix diff --git a/.mcp.json b/.mcp.json index 1303afda..71554002 100644 --- a/.mcp.json +++ b/.mcp.json @@ -3,45 +3,31 @@ "claude-code-mcp": { "type": "stdio", "command": "npx", - "args": [ - "-y", - "@steipete/claude-code-mcp@latest" - ], + "args": ["-y", "@steipete/claude-code-mcp@latest"], "env": {} }, "sequential-thinking": { "type": "stdio", "command": "npx", - "args": [ - "-y", - "@modelcontextprotocol/server-sequential-thinking" - ], + "args": ["-y", "@modelcontextprotocol/server-sequential-thinking"], "env": {} }, "nixos": { "type": "stdio", "command": "uvx", - "args": [ - "mcp-nixos" - ], + "args": ["mcp-nixos"], "env": {} }, "context7": { "type": "stdio", "command": "npx", - "args": [ - "-y", - "@upstash/context7-mcp" - ], + "args": ["-y", "@upstash/context7-mcp"], "env": {} }, "git": { "type": "stdio", "command": "npx", - "args": [ - "-y", - "@cyanheads/git-mcp-server" - ], + "args": ["-y", "@cyanheads/git-mcp-server"], "env": {} } } diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..4d98d4d3 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,75 @@ +# prek/pre-commit configuration for headscale +# See: https://prek.j178.dev/quickstart/ +# See: https://prek.j178.dev/builtin/ + +# Global exclusions - ignore docs and generated code +exclude: ^(docs/|gen/) + +repos: + # Built-in hooks from pre-commit/pre-commit-hooks + # prek will use fast-path optimized versions automatically + # See: https://prek.j178.dev/builtin/ + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-executables-have-shebangs + - id: check-json + - id: check-merge-conflict + - id: check-symlinks + - id: check-toml + - id: check-xml + - id: check-yaml + - id: detect-private-key + - id: end-of-file-fixer + - id: fix-byte-order-marker + - id: mixed-line-ending + - id: trailing-whitespace + + # Local hooks for project-specific tooling + - repo: local + hooks: + # nixpkgs-fmt for Nix files + - id: nixpkgs-fmt + name: nixpkgs-fmt + entry: nixpkgs-fmt + language: system + files: \.nix$ + + # Prettier for formatting + - id: prettier + name: prettier + entry: prettier --write --list-different + language: system + types_or: + [ + javascript, + jsx, + ts, + tsx, + yaml, + json, + toml, + html, + css, + scss, + sass, + markdown, + ] + exclude: ^CHANGELOG\.md$ + + # Prettier for CHANGELOG.md with special formatting + - id: prettier-changelog + name: prettier-changelog + entry: prettier --write --print-width 80 --prose-wrap always + language: system + files: ^CHANGELOG\.md$ + + # golangci-lint for Go code quality + - id: golangci-lint + name: golangci-lint + entry: .golangci-lint-hook.sh + language: system + types: [go] + pass_filenames: false diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..e5dd1b01 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,699 @@ +# AGENTS.md + +This file provides guidance to AI agents when working with code in this repository. + +## Overview + +Headscale is an open-source implementation of the Tailscale control server written in Go. It provides self-hosted coordination for Tailscale networks (tailnets), managing node registration, IP allocation, policy enforcement, and DERP routing. + +## Development Commands + +### Quick Setup + +```bash +# Recommended: Use Nix for dependency management +nix develop + +# Full development workflow +make dev # runs fmt + lint + test + build +``` + +### Essential Commands + +```bash +# Build headscale binary +make build + +# Run tests +make test +go test ./... # All unit tests +go test -race ./... # With race detection + +# Run specific integration test +go run ./cmd/hi run "TestName" --postgres + +# Code formatting and linting +make fmt # Format all code (Go, docs, proto) +make lint # Lint all code (Go, proto) +make fmt-go # Format Go code only +make lint-go # Lint Go code only + +# Protocol buffer generation (after modifying proto/) +make generate + +# Clean build artifacts +make clean +``` + +### Integration Testing + +```bash +# Use the hi (Headscale Integration) test runner +go run ./cmd/hi doctor # Check system requirements +go run ./cmd/hi run "TestPattern" # Run specific test +go run ./cmd/hi run "TestPattern" --postgres # With PostgreSQL backend + +# Test artifacts are saved to control_logs/ with logs and debug data +``` + +## Pre-Commit Quality Checks + +### **MANDATORY: Automated Pre-Commit Hooks with prek** + +**CRITICAL REQUIREMENT**: This repository uses [prek](https://prek.j178.dev/) for automated pre-commit hooks. All commits are automatically validated for code quality, formatting, and common issues. + +### Initial Setup + +When you first clone the repository or enter the nix shell, install the git hooks: + +```bash +# Enter nix development environment +nix develop + +# Install prek git hooks (one-time setup) +prek install +``` + +This installs the pre-commit hook at `.git/hooks/pre-commit` which automatically runs all configured checks before each commit. + +### Configured Hooks + +The repository uses `.pre-commit-config.yaml` with the following hooks: + +**Built-in Checks** (optimized fast-path execution): + +- `check-added-large-files` - Prevents accidentally committing large files +- `check-case-conflict` - Checks for files that would conflict in case-insensitive filesystems +- `check-executables-have-shebangs` - Ensures executables have proper shebangs +- `check-json` - Validates JSON syntax +- `check-merge-conflict` - Prevents committing files with merge conflict markers +- `check-symlinks` - Checks for broken symlinks +- `check-toml` - Validates TOML syntax +- `check-xml` - Validates XML syntax +- `check-yaml` - Validates YAML syntax +- `detect-private-key` - Detects accidentally committed private keys +- `end-of-file-fixer` - Ensures files end with a newline +- `fix-byte-order-marker` - Removes UTF-8 byte order markers +- `mixed-line-ending` - Prevents mixed line endings +- `trailing-whitespace` - Removes trailing whitespace + +**Project-Specific Hooks**: + +- `nixpkgs-fmt` - Formats Nix files +- `prettier` - Formats markdown, YAML, JSON, and TOML files +- `golangci-lint` - Runs Go linter with auto-fix on changed files only + +### Manual Hook Execution + +Run hooks manually without making a commit: + +```bash +# Run hooks on staged files only +prek run + +# Run hooks on all files in the repository +prek run --all-files + +# Run a specific hook +prek run golangci-lint + +# Run hooks on specific files +prek run --files path/to/file1.go path/to/file2.go +``` + +### Workflow Pattern + +With prek installed, your normal workflow becomes: + +```bash +# 1. Make your code changes +vim hscontrol/state/state.go + +# 2. Stage your changes +git add . + +# 3. Commit - hooks run automatically +git commit -m "feat: add new feature" + +# If hooks fail, they will show which checks failed +# Fix the issues and try committing again +``` + +### Manual golangci-lint (Optional) + +While golangci-lint runs automatically via prek, you can also run it manually: + +```bash +# Use the same logic as the pre-commit hook (recommended) +./.golangci-lint-hook.sh + +# Or manually specify a base reference +golangci-lint run --new-from-rev=upstream/main --timeout=5m --fix +``` + +The `.golangci-lint-hook.sh` script automatically finds where your branch diverged from the main branch by checking `upstream/main`, `origin/main`, or `main` in that order. + +### Skipping Hooks (Not Recommended) + +In rare cases where you need to skip hooks (e.g., work-in-progress commits), use: + +```bash +git commit --no-verify -m "WIP: work in progress" +``` + +**WARNING**: Only use `--no-verify` for temporary WIP commits on feature branches. All commits to main must pass all hooks. + +### Troubleshooting + +**Hook installation issues**: + +```bash +# Check if hooks are installed +ls -la .git/hooks/pre-commit + +# Reinstall hooks +prek install +``` + +**Hooks running slow**: + +```bash +# prek uses optimized fast-path for built-in hooks +# If running slow, check which hook is taking time with verbose output +prek run -v +``` + +**Update hook configuration**: + +```bash +# After modifying .pre-commit-config.yaml, hooks will automatically use new config +# No reinstallation needed +``` + +## Project Structure & Architecture + +### Top-Level Organization + +``` +headscale/ +├── cmd/ # Command-line applications +│ ├── headscale/ # Main headscale server binary +│ └── hi/ # Headscale Integration test runner +├── hscontrol/ # Core control plane logic +├── integration/ # End-to-end Docker-based tests +├── proto/ # Protocol buffer definitions +├── gen/ # Generated code (protobuf) +├── docs/ # Documentation +└── packaging/ # Distribution packaging +``` + +### Core Packages (`hscontrol/`) + +**Main Server (`hscontrol/`)** + +- `app.go`: Application setup, dependency injection, server lifecycle +- `handlers.go`: HTTP/gRPC API endpoints for management operations +- `grpcv1.go`: gRPC service implementation for headscale API +- `poll.go`: **Critical** - Handles Tailscale MapRequest/MapResponse protocol +- `noise.go`: Noise protocol implementation for secure client communication +- `auth.go`: Authentication flows (web, OIDC, command-line) +- `oidc.go`: OpenID Connect integration for user authentication + +**State Management (`hscontrol/state/`)** + +- `state.go`: Central coordinator for all subsystems (database, policy, IP allocation, DERP) +- `node_store.go`: **Performance-critical** - In-memory cache with copy-on-write semantics +- Thread-safe operations with deadlock detection +- Coordinates between database persistence and real-time operations + +**Database Layer (`hscontrol/db/`)** + +- `db.go`: Database abstraction, GORM setup, migration management +- `node.go`: Node lifecycle, registration, expiration, IP assignment +- `users.go`: User management, namespace isolation +- `api_key.go`: API authentication tokens +- `preauth_keys.go`: Pre-authentication keys for automated node registration +- `ip.go`: IP address allocation and management +- `policy.go`: Policy storage and retrieval +- Schema migrations in `schema.sql` with extensive test data coverage + +**Policy Engine (`hscontrol/policy/`)** + +- `policy.go`: Core ACL evaluation logic, HuJSON parsing +- `v2/`: Next-generation policy system with improved filtering +- `matcher/`: ACL rule matching and evaluation engine +- Determines peer visibility, route approval, and network access rules +- Supports both file-based and database-stored policies + +**Network Management (`hscontrol/`)** + +- `derp/`: DERP (Designated Encrypted Relay for Packets) server implementation + - NAT traversal when direct connections fail + - Fallback relay for firewall-restricted environments +- `mapper/`: Converts internal Headscale state to Tailscale's wire protocol format + - `tail.go`: Tailscale-specific data structure generation +- `routes/`: Subnet route management and primary route selection +- `dns/`: DNS record management and MagicDNS implementation + +**Utilities & Support (`hscontrol/`)** + +- `types/`: Core data structures, configuration, validation +- `util/`: Helper functions for networking, DNS, key management +- `templates/`: Client configuration templates (Apple, Windows, etc.) +- `notifier/`: Event notification system for real-time updates +- `metrics.go`: Prometheus metrics collection +- `capver/`: Tailscale capability version management + +### Key Subsystem Interactions + +**Node Registration Flow** + +1. **Client Connection**: `noise.go` handles secure protocol handshake +2. **Authentication**: `auth.go` validates credentials (web/OIDC/preauth) +3. **State Creation**: `state.go` coordinates IP allocation via `db/ip.go` +4. **Storage**: `db/node.go` persists node, `NodeStore` caches in memory +5. **Network Setup**: `mapper/` generates initial Tailscale network map + +**Ongoing Operations** + +1. **Poll Requests**: `poll.go` receives periodic client updates +2. **State Updates**: `NodeStore` maintains real-time node information +3. **Policy Application**: `policy/` evaluates ACL rules for peer relationships +4. **Map Distribution**: `mapper/` sends network topology to all affected clients + +**Route Management** + +1. **Advertisement**: Clients announce routes via `poll.go` Hostinfo updates +2. **Storage**: `db/` persists routes, `NodeStore` caches for performance +3. **Approval**: `policy/` auto-approves routes based on ACL rules +4. **Distribution**: `routes/` selects primary routes, `mapper/` distributes to peers + +### Command-Line Tools (`cmd/`) + +**Main Server (`cmd/headscale/`)** + +- `headscale.go`: CLI parsing, configuration loading, server startup +- Supports daemon mode, CLI operations (user/node management), database operations + +**Integration Test Runner (`cmd/hi/`)** + +- `main.go`: Test execution framework with Docker orchestration +- `run.go`: Individual test execution with artifact collection +- `doctor.go`: System requirements validation +- `docker.go`: Container lifecycle management +- Essential for validating changes against real Tailscale clients + +### Generated & External Code + +**Protocol Buffers (`proto/` → `gen/`)** + +- Defines gRPC API for headscale management operations +- Client libraries can generate from these definitions +- Run `make generate` after modifying `.proto` files + +**Integration Testing (`integration/`)** + +- `scenario.go`: Docker test environment setup +- `tailscale.go`: Tailscale client container management +- Individual test files for specific functionality areas +- Real end-to-end validation with network isolation + +### Critical Performance Paths + +**High-Frequency Operations** + +1. **MapRequest Processing** (`poll.go`): Every 15-60 seconds per client +2. **NodeStore Reads** (`node_store.go`): Every operation requiring node data +3. **Policy Evaluation** (`policy/`): On every peer relationship calculation +4. **Route Lookups** (`routes/`): During network map generation + +**Database Write Patterns** + +- **Frequent**: Node heartbeats, endpoint updates, route changes +- **Moderate**: User operations, policy updates, API key management +- **Rare**: Schema migrations, bulk operations + +### Configuration & Deployment + +**Configuration** (`hscontrol/types/config.go`)\*\* + +- Database connection settings (SQLite/PostgreSQL) +- Network configuration (IP ranges, DNS settings) +- Policy mode (file vs database) +- DERP relay configuration +- OIDC provider settings + +**Key Dependencies** + +- **GORM**: Database ORM with migration support +- **Tailscale Libraries**: Core networking and protocol code +- **Zerolog**: Structured logging throughout the application +- **Buf**: Protocol buffer toolchain for code generation + +### Development Workflow Integration + +The architecture supports incremental development: + +- **Unit Tests**: Focus on individual packages (`*_test.go` files) +- **Integration Tests**: Validate cross-component interactions +- **Database Tests**: Extensive migration and data integrity validation +- **Policy Tests**: ACL rule evaluation and edge cases +- **Performance Tests**: NodeStore and high-frequency operation validation + +## Integration Testing System + +### Overview + +Headscale uses Docker-based integration tests with real Tailscale clients to validate end-to-end functionality. The integration test system is complex and requires specialized knowledge for effective execution and debugging. + +### **MANDATORY: Use the headscale-integration-tester Agent** + +**CRITICAL REQUIREMENT**: For ANY integration test execution, analysis, troubleshooting, or validation, you MUST use the `headscale-integration-tester` agent. This agent contains specialized knowledge about: + +- Test execution strategies and timing requirements +- Infrastructure vs code issue distinction (99% vs 1% failure patterns) +- Security-critical debugging rules and forbidden practices +- Comprehensive artifact analysis workflows +- Real-world failure patterns from HA debugging experiences + +### Quick Reference Commands + +```bash +# Check system requirements (always run first) +go run ./cmd/hi doctor + +# Run single test (recommended for development) +go run ./cmd/hi run "TestName" + +# Use PostgreSQL for database-heavy tests +go run ./cmd/hi run "TestName" --postgres + +# Pattern matching for related tests +go run ./cmd/hi run "TestPattern*" +``` + +**Critical Notes**: + +- Only ONE test can run at a time (Docker port conflicts) +- Tests generate ~100MB of logs per run in `control_logs/` +- Clean environment before each test: `rm -rf control_logs/202507* && docker system prune -f` + +### Test Artifacts Location + +All test runs save comprehensive debugging artifacts to `control_logs/TIMESTAMP-ID/` including server logs, client logs, database dumps, MapResponse protocol data, and Prometheus metrics. + +**For all integration test work, use the headscale-integration-tester agent - it contains the complete knowledge needed for effective testing and debugging.** + +## NodeStore Implementation Details + +**Key Insight from Recent Work**: The NodeStore is a critical performance optimization that caches node data in memory while ensuring consistency with the database. When working with route advertisements or node state changes: + +1. **Timing Considerations**: Route advertisements need time to propagate from clients to server. Use `require.EventuallyWithT()` patterns in tests instead of immediate assertions. + +2. **Synchronization Points**: NodeStore updates happen at specific points like `poll.go:420` after Hostinfo changes. Ensure these are maintained when modifying the polling logic. + +3. **Peer Visibility**: The NodeStore's `peersFunc` determines which nodes are visible to each other. Policy-based filtering is separate from monitoring visibility - expired nodes should remain visible for debugging but marked as expired. + +## Testing Guidelines + +### Integration Test Patterns + +#### **CRITICAL: EventuallyWithT Pattern for External Calls** + +**All external calls in integration tests MUST be wrapped in EventuallyWithT blocks** to handle eventual consistency in distributed systems. External calls include: + +- `client.Status()` - Getting Tailscale client status +- `client.Curl()` - Making HTTP requests through clients +- `client.Traceroute()` - Running network diagnostics +- `headscale.ListNodes()` - Querying headscale server state +- Any other calls that interact with external systems or network operations + +**Key Rules**: + +1. **Never use bare `require.NoError(t, err)` with external calls** - Always wrap in EventuallyWithT +2. **Keep related assertions together** - If multiple assertions depend on the same external call, keep them in the same EventuallyWithT block +3. **Split unrelated external calls** - Different external calls should be in separate EventuallyWithT blocks +4. **Never nest EventuallyWithT calls** - Each EventuallyWithT should be at the same level +5. **Declare shared variables at function scope** - Variables used across multiple EventuallyWithT blocks must be declared before first use + +**Examples**: + +```go +// CORRECT: External call wrapped in EventuallyWithT +assert.EventuallyWithT(t, func(c *assert.CollectT) { + status, err := client.Status() + assert.NoError(c, err) + + // Related assertions using the same status call + for _, peerKey := range status.Peers() { + peerStatus := status.Peer[peerKey] + assert.NotNil(c, peerStatus.PrimaryRoutes) + requirePeerSubnetRoutesWithCollect(c, peerStatus, expectedRoutes) + } +}, 5*time.Second, 200*time.Millisecond, "Verifying client status and routes") + +// INCORRECT: Bare external call without EventuallyWithT +status, err := client.Status() // ❌ Will fail intermittently +require.NoError(t, err) + +// CORRECT: Separate EventuallyWithT for different external calls +// First external call - headscale.ListNodes() +assert.EventuallyWithT(t, func(c *assert.CollectT) { + nodes, err := headscale.ListNodes() + assert.NoError(c, err) + assert.Len(c, nodes, 2) + requireNodeRouteCountWithCollect(c, nodes[0], 2, 2, 2) +}, 10*time.Second, 500*time.Millisecond, "route state changes should propagate to nodes") + +// Second external call - client.Status() +assert.EventuallyWithT(t, func(c *assert.CollectT) { + status, err := client.Status() + assert.NoError(c, err) + + for _, peerKey := range status.Peers() { + peerStatus := status.Peer[peerKey] + requirePeerSubnetRoutesWithCollect(c, peerStatus, []netip.Prefix{tsaddr.AllIPv4(), tsaddr.AllIPv6()}) + } +}, 10*time.Second, 500*time.Millisecond, "routes should be visible to client") + +// INCORRECT: Multiple unrelated external calls in same EventuallyWithT +assert.EventuallyWithT(t, func(c *assert.CollectT) { + nodes, err := headscale.ListNodes() // ❌ First external call + assert.NoError(c, err) + + status, err := client.Status() // ❌ Different external call - should be separate + assert.NoError(c, err) +}, 10*time.Second, 500*time.Millisecond, "mixed calls") + +// CORRECT: Variable scoping for shared data +var ( + srs1, srs2, srs3 *ipnstate.Status + clientStatus *ipnstate.Status + srs1PeerStatus *ipnstate.PeerStatus +) + +assert.EventuallyWithT(t, func(c *assert.CollectT) { + srs1 = subRouter1.MustStatus() // = not := + srs2 = subRouter2.MustStatus() + clientStatus = client.MustStatus() + + srs1PeerStatus = clientStatus.Peer[srs1.Self.PublicKey] + // assertions... +}, 5*time.Second, 200*time.Millisecond, "checking router status") + +// CORRECT: Wrapping client operations +assert.EventuallyWithT(t, func(c *assert.CollectT) { + result, err := client.Curl(weburl) + assert.NoError(c, err) + assert.Len(c, result, 13) +}, 5*time.Second, 200*time.Millisecond, "Verifying HTTP connectivity") + +assert.EventuallyWithT(t, func(c *assert.CollectT) { + tr, err := client.Traceroute(webip) + assert.NoError(c, err) + assertTracerouteViaIPWithCollect(c, tr, expectedRouter.MustIPv4()) +}, 5*time.Second, 200*time.Millisecond, "Verifying network path") +``` + +**Helper Functions**: + +- Use `requirePeerSubnetRoutesWithCollect` instead of `requirePeerSubnetRoutes` inside EventuallyWithT +- Use `requireNodeRouteCountWithCollect` instead of `requireNodeRouteCount` inside EventuallyWithT +- Use `assertTracerouteViaIPWithCollect` instead of `assertTracerouteViaIP` inside EventuallyWithT + +```go +// Node route checking by actual node properties, not array position +var routeNode *v1.Node +for _, node := range nodes { + if nodeIDStr := fmt.Sprintf("%d", node.GetId()); expectedRoutes[nodeIDStr] != "" { + routeNode = node + break + } +} +``` + +### Running Problematic Tests + +- Some tests require significant time (e.g., `TestNodeOnlineStatus` runs for 12 minutes) +- Infrastructure issues like disk space can cause test failures unrelated to code changes +- Use `--postgres` flag when testing database-heavy scenarios + +## Quality Assurance and Testing Requirements + +### **MANDATORY: Always Use Specialized Testing Agents** + +**CRITICAL REQUIREMENT**: For ANY task involving testing, quality assurance, review, or validation, you MUST use the appropriate specialized agent at the END of your task list. This ensures comprehensive quality validation and prevents regressions. + +**Required Agents for Different Task Types**: + +1. **Integration Testing**: Use `headscale-integration-tester` agent for: + - Running integration tests with `cmd/hi` + - Analyzing test failures and artifacts + - Troubleshooting Docker-based test infrastructure + - Validating end-to-end functionality changes + +2. **Quality Control**: Use `quality-control-enforcer` agent for: + - Code review and validation + - Ensuring best practices compliance + - Preventing common pitfalls and anti-patterns + - Validating architectural decisions + +**Agent Usage Pattern**: Always add the appropriate agent as the FINAL step in any task list to ensure quality validation occurs after all work is complete. + +### Integration Test Debugging Reference + +Test artifacts are preserved in `control_logs/TIMESTAMP-ID/` including: + +- Headscale server logs (stderr/stdout) +- Tailscale client logs and status +- Database dumps and network captures +- MapResponse JSON files for protocol debugging + +**For integration test issues, ALWAYS use the headscale-integration-tester agent - do not attempt manual debugging.** + +## EventuallyWithT Pattern for Integration Tests + +### Overview + +EventuallyWithT is a testing pattern used to handle eventual consistency in distributed systems. In Headscale integration tests, many operations are asynchronous - clients advertise routes, the server processes them, updates propagate through the network. EventuallyWithT allows tests to wait for these operations to complete while making assertions. + +### External Calls That Must Be Wrapped + +The following operations are **external calls** that interact with the headscale server or tailscale clients and MUST be wrapped in EventuallyWithT: + +- `headscale.ListNodes()` - Queries server state +- `client.Status()` - Gets client network status +- `client.Curl()` - Makes HTTP requests through the network +- `client.Traceroute()` - Performs network diagnostics +- `client.Execute()` when running commands that query state +- Any operation that reads from the headscale server or tailscale client + +### Operations That Must NOT Be Wrapped + +The following are **blocking operations** that modify state and should NOT be wrapped in EventuallyWithT: + +- `tailscale set` commands (e.g., `--advertise-routes`, `--exit-node`) +- Any command that changes configuration or state +- Use `client.MustStatus()` instead of `client.Status()` when you just need the ID for a blocking operation + +### Five Key Rules for EventuallyWithT + +1. **One External Call Per EventuallyWithT Block** + - Each EventuallyWithT should make ONE external call (e.g., ListNodes OR Status) + - Related assertions based on that single call can be grouped together + - Unrelated external calls must be in separate EventuallyWithT blocks + +2. **Variable Scoping** + - Declare variables that need to be shared across EventuallyWithT blocks at function scope + - Use `=` for assignment inside EventuallyWithT, not `:=` (unless the variable is only used within that block) + - Variables declared with `:=` inside EventuallyWithT are not accessible outside + +3. **No Nested EventuallyWithT** + - NEVER put an EventuallyWithT inside another EventuallyWithT + - This is a critical anti-pattern that must be avoided + +4. **Use CollectT for Assertions** + - Inside EventuallyWithT, use `assert` methods with the CollectT parameter + - Helper functions called within EventuallyWithT must accept `*assert.CollectT` + +5. **Descriptive Messages** + - Always provide a descriptive message as the last parameter + - Message should explain what condition is being waited for + +### Correct Pattern Examples + +```go +// CORRECT: Blocking operation NOT wrapped +for _, client := range allClients { + status := client.MustStatus() + command := []string{ + "tailscale", + "set", + "--advertise-routes=" + expectedRoutes[string(status.Self.ID)], + } + _, _, err = client.Execute(command) + require.NoErrorf(t, err, "failed to advertise route: %s", err) +} + +// CORRECT: Single external call with related assertions +var nodes []*v1.Node +assert.EventuallyWithT(t, func(c *assert.CollectT) { + nodes, err = headscale.ListNodes() + assert.NoError(c, err) + assert.Len(c, nodes, 2) + requireNodeRouteCountWithCollect(c, nodes[0], 2, 2, 2) +}, 10*time.Second, 500*time.Millisecond, "nodes should have expected route counts") + +// CORRECT: Separate EventuallyWithT for different external call +assert.EventuallyWithT(t, func(c *assert.CollectT) { + status, err := client.Status() + assert.NoError(c, err) + for _, peerKey := range status.Peers() { + peerStatus := status.Peer[peerKey] + requirePeerSubnetRoutesWithCollect(c, peerStatus, expectedPrefixes) + } +}, 10*time.Second, 500*time.Millisecond, "client should see expected routes") +``` + +### Incorrect Patterns to Avoid + +```go +// INCORRECT: Blocking operation wrapped in EventuallyWithT +assert.EventuallyWithT(t, func(c *assert.CollectT) { + status, err := client.Status() + assert.NoError(c, err) + + // This is a blocking operation - should NOT be in EventuallyWithT! + command := []string{ + "tailscale", + "set", + "--advertise-routes=" + expectedRoutes[string(status.Self.ID)], + } + _, _, err = client.Execute(command) + assert.NoError(c, err) +}, 5*time.Second, 200*time.Millisecond, "wrong pattern") + +// INCORRECT: Multiple unrelated external calls in same EventuallyWithT +assert.EventuallyWithT(t, func(c *assert.CollectT) { + // First external call + nodes, err := headscale.ListNodes() + assert.NoError(c, err) + assert.Len(c, nodes, 2) + + // Second unrelated external call - WRONG! + status, err := client.Status() + assert.NoError(c, err) + assert.NotNil(c, status) +}, 10*time.Second, 500*time.Millisecond, "mixed operations") +``` + +## Important Notes + +- **Dependencies**: Use `nix develop` for consistent toolchain (Go, buf, protobuf tools, linting) +- **Protocol Buffers**: Changes to `proto/` require `make generate` and should be committed separately +- **Code Style**: Enforced via golangci-lint with golines (width 88) and gofumpt formatting +- **Linting**: ALL code must pass `golangci-lint run --new-from-rev=upstream/main --timeout=5m --fix` before commit +- **Database**: Supports both SQLite (development) and PostgreSQL (production/testing) +- **Integration Tests**: Require Docker and can consume significant disk space - use headscale-integration-tester agent +- **Performance**: NodeStore optimizations are critical for scale - be careful with changes to state management +- **Quality Assurance**: Always use appropriate specialized agents for testing and validation tasks diff --git a/CLAUDE.md b/CLAUDE.md index d4034367..43c994c2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,531 +1 @@ -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -## Overview - -Headscale is an open-source implementation of the Tailscale control server written in Go. It provides self-hosted coordination for Tailscale networks (tailnets), managing node registration, IP allocation, policy enforcement, and DERP routing. - -## Development Commands - -### Quick Setup -```bash -# Recommended: Use Nix for dependency management -nix develop - -# Full development workflow -make dev # runs fmt + lint + test + build -``` - -### Essential Commands -```bash -# Build headscale binary -make build - -# Run tests -make test -go test ./... # All unit tests -go test -race ./... # With race detection - -# Run specific integration test -go run ./cmd/hi run "TestName" --postgres - -# Code formatting and linting -make fmt # Format all code (Go, docs, proto) -make lint # Lint all code (Go, proto) -make fmt-go # Format Go code only -make lint-go # Lint Go code only - -# Protocol buffer generation (after modifying proto/) -make generate - -# Clean build artifacts -make clean -``` - -### Integration Testing -```bash -# Use the hi (Headscale Integration) test runner -go run ./cmd/hi doctor # Check system requirements -go run ./cmd/hi run "TestPattern" # Run specific test -go run ./cmd/hi run "TestPattern" --postgres # With PostgreSQL backend - -# Test artifacts are saved to control_logs/ with logs and debug data -``` - -## Project Structure & Architecture - -### Top-Level Organization - -``` -headscale/ -├── cmd/ # Command-line applications -│ ├── headscale/ # Main headscale server binary -│ └── hi/ # Headscale Integration test runner -├── hscontrol/ # Core control plane logic -├── integration/ # End-to-end Docker-based tests -├── proto/ # Protocol buffer definitions -├── gen/ # Generated code (protobuf) -├── docs/ # Documentation -└── packaging/ # Distribution packaging -``` - -### Core Packages (`hscontrol/`) - -**Main Server (`hscontrol/`)** -- `app.go`: Application setup, dependency injection, server lifecycle -- `handlers.go`: HTTP/gRPC API endpoints for management operations -- `grpcv1.go`: gRPC service implementation for headscale API -- `poll.go`: **Critical** - Handles Tailscale MapRequest/MapResponse protocol -- `noise.go`: Noise protocol implementation for secure client communication -- `auth.go`: Authentication flows (web, OIDC, command-line) -- `oidc.go`: OpenID Connect integration for user authentication - -**State Management (`hscontrol/state/`)** -- `state.go`: Central coordinator for all subsystems (database, policy, IP allocation, DERP) -- `node_store.go`: **Performance-critical** - In-memory cache with copy-on-write semantics -- Thread-safe operations with deadlock detection -- Coordinates between database persistence and real-time operations - -**Database Layer (`hscontrol/db/`)** -- `db.go`: Database abstraction, GORM setup, migration management -- `node.go`: Node lifecycle, registration, expiration, IP assignment -- `users.go`: User management, namespace isolation -- `api_key.go`: API authentication tokens -- `preauth_keys.go`: Pre-authentication keys for automated node registration -- `ip.go`: IP address allocation and management -- `policy.go`: Policy storage and retrieval -- Schema migrations in `schema.sql` with extensive test data coverage - -**Policy Engine (`hscontrol/policy/`)** -- `policy.go`: Core ACL evaluation logic, HuJSON parsing -- `v2/`: Next-generation policy system with improved filtering -- `matcher/`: ACL rule matching and evaluation engine -- Determines peer visibility, route approval, and network access rules -- Supports both file-based and database-stored policies - -**Network Management (`hscontrol/`)** -- `derp/`: DERP (Designated Encrypted Relay for Packets) server implementation - - NAT traversal when direct connections fail - - Fallback relay for firewall-restricted environments -- `mapper/`: Converts internal Headscale state to Tailscale's wire protocol format - - `tail.go`: Tailscale-specific data structure generation -- `routes/`: Subnet route management and primary route selection -- `dns/`: DNS record management and MagicDNS implementation - -**Utilities & Support (`hscontrol/`)** -- `types/`: Core data structures, configuration, validation -- `util/`: Helper functions for networking, DNS, key management -- `templates/`: Client configuration templates (Apple, Windows, etc.) -- `notifier/`: Event notification system for real-time updates -- `metrics.go`: Prometheus metrics collection -- `capver/`: Tailscale capability version management - -### Key Subsystem Interactions - -**Node Registration Flow** -1. **Client Connection**: `noise.go` handles secure protocol handshake -2. **Authentication**: `auth.go` validates credentials (web/OIDC/preauth) -3. **State Creation**: `state.go` coordinates IP allocation via `db/ip.go` -4. **Storage**: `db/node.go` persists node, `NodeStore` caches in memory -5. **Network Setup**: `mapper/` generates initial Tailscale network map - -**Ongoing Operations** -1. **Poll Requests**: `poll.go` receives periodic client updates -2. **State Updates**: `NodeStore` maintains real-time node information -3. **Policy Application**: `policy/` evaluates ACL rules for peer relationships -4. **Map Distribution**: `mapper/` sends network topology to all affected clients - -**Route Management** -1. **Advertisement**: Clients announce routes via `poll.go` Hostinfo updates -2. **Storage**: `db/` persists routes, `NodeStore` caches for performance -3. **Approval**: `policy/` auto-approves routes based on ACL rules -4. **Distribution**: `routes/` selects primary routes, `mapper/` distributes to peers - -### Command-Line Tools (`cmd/`) - -**Main Server (`cmd/headscale/`)** -- `headscale.go`: CLI parsing, configuration loading, server startup -- Supports daemon mode, CLI operations (user/node management), database operations - -**Integration Test Runner (`cmd/hi/`)** -- `main.go`: Test execution framework with Docker orchestration -- `run.go`: Individual test execution with artifact collection -- `doctor.go`: System requirements validation -- `docker.go`: Container lifecycle management -- Essential for validating changes against real Tailscale clients - -### Generated & External Code - -**Protocol Buffers (`proto/` → `gen/`)** -- Defines gRPC API for headscale management operations -- Client libraries can generate from these definitions -- Run `make generate` after modifying `.proto` files - -**Integration Testing (`integration/`)** -- `scenario.go`: Docker test environment setup -- `tailscale.go`: Tailscale client container management -- Individual test files for specific functionality areas -- Real end-to-end validation with network isolation - -### Critical Performance Paths - -**High-Frequency Operations** -1. **MapRequest Processing** (`poll.go`): Every 15-60 seconds per client -2. **NodeStore Reads** (`node_store.go`): Every operation requiring node data -3. **Policy Evaluation** (`policy/`): On every peer relationship calculation -4. **Route Lookups** (`routes/`): During network map generation - -**Database Write Patterns** -- **Frequent**: Node heartbeats, endpoint updates, route changes -- **Moderate**: User operations, policy updates, API key management -- **Rare**: Schema migrations, bulk operations - -### Configuration & Deployment - -**Configuration** (`hscontrol/types/config.go`)** -- Database connection settings (SQLite/PostgreSQL) -- Network configuration (IP ranges, DNS settings) -- Policy mode (file vs database) -- DERP relay configuration -- OIDC provider settings - -**Key Dependencies** -- **GORM**: Database ORM with migration support -- **Tailscale Libraries**: Core networking and protocol code -- **Zerolog**: Structured logging throughout the application -- **Buf**: Protocol buffer toolchain for code generation - -### Development Workflow Integration - -The architecture supports incremental development: -- **Unit Tests**: Focus on individual packages (`*_test.go` files) -- **Integration Tests**: Validate cross-component interactions -- **Database Tests**: Extensive migration and data integrity validation -- **Policy Tests**: ACL rule evaluation and edge cases -- **Performance Tests**: NodeStore and high-frequency operation validation - -## Integration Testing System - -### Overview -Headscale uses Docker-based integration tests with real Tailscale clients to validate end-to-end functionality. The integration test system is complex and requires specialized knowledge for effective execution and debugging. - -### **MANDATORY: Use the headscale-integration-tester Agent** - -**CRITICAL REQUIREMENT**: For ANY integration test execution, analysis, troubleshooting, or validation, you MUST use the `headscale-integration-tester` agent. This agent contains specialized knowledge about: - -- Test execution strategies and timing requirements -- Infrastructure vs code issue distinction (99% vs 1% failure patterns) -- Security-critical debugging rules and forbidden practices -- Comprehensive artifact analysis workflows -- Real-world failure patterns from HA debugging experiences - -### Quick Reference Commands - -```bash -# Check system requirements (always run first) -go run ./cmd/hi doctor - -# Run single test (recommended for development) -go run ./cmd/hi run "TestName" - -# Use PostgreSQL for database-heavy tests -go run ./cmd/hi run "TestName" --postgres - -# Pattern matching for related tests -go run ./cmd/hi run "TestPattern*" -``` - -**Critical Notes**: -- Only ONE test can run at a time (Docker port conflicts) -- Tests generate ~100MB of logs per run in `control_logs/` -- Clean environment before each test: `rm -rf control_logs/202507* && docker system prune -f` - -### Test Artifacts Location -All test runs save comprehensive debugging artifacts to `control_logs/TIMESTAMP-ID/` including server logs, client logs, database dumps, MapResponse protocol data, and Prometheus metrics. - -**For all integration test work, use the headscale-integration-tester agent - it contains the complete knowledge needed for effective testing and debugging.** - -## NodeStore Implementation Details - -**Key Insight from Recent Work**: The NodeStore is a critical performance optimization that caches node data in memory while ensuring consistency with the database. When working with route advertisements or node state changes: - -1. **Timing Considerations**: Route advertisements need time to propagate from clients to server. Use `require.EventuallyWithT()` patterns in tests instead of immediate assertions. - -2. **Synchronization Points**: NodeStore updates happen at specific points like `poll.go:420` after Hostinfo changes. Ensure these are maintained when modifying the polling logic. - -3. **Peer Visibility**: The NodeStore's `peersFunc` determines which nodes are visible to each other. Policy-based filtering is separate from monitoring visibility - expired nodes should remain visible for debugging but marked as expired. - -## Testing Guidelines - -### Integration Test Patterns - -#### **CRITICAL: EventuallyWithT Pattern for External Calls** - -**All external calls in integration tests MUST be wrapped in EventuallyWithT blocks** to handle eventual consistency in distributed systems. External calls include: -- `client.Status()` - Getting Tailscale client status -- `client.Curl()` - Making HTTP requests through clients -- `client.Traceroute()` - Running network diagnostics -- `headscale.ListNodes()` - Querying headscale server state -- Any other calls that interact with external systems or network operations - -**Key Rules**: -1. **Never use bare `require.NoError(t, err)` with external calls** - Always wrap in EventuallyWithT -2. **Keep related assertions together** - If multiple assertions depend on the same external call, keep them in the same EventuallyWithT block -3. **Split unrelated external calls** - Different external calls should be in separate EventuallyWithT blocks -4. **Never nest EventuallyWithT calls** - Each EventuallyWithT should be at the same level -5. **Declare shared variables at function scope** - Variables used across multiple EventuallyWithT blocks must be declared before first use - -**Examples**: - -```go -// CORRECT: External call wrapped in EventuallyWithT -assert.EventuallyWithT(t, func(c *assert.CollectT) { - status, err := client.Status() - assert.NoError(c, err) - - // Related assertions using the same status call - for _, peerKey := range status.Peers() { - peerStatus := status.Peer[peerKey] - assert.NotNil(c, peerStatus.PrimaryRoutes) - requirePeerSubnetRoutesWithCollect(c, peerStatus, expectedRoutes) - } -}, 5*time.Second, 200*time.Millisecond, "Verifying client status and routes") - -// INCORRECT: Bare external call without EventuallyWithT -status, err := client.Status() // ❌ Will fail intermittently -require.NoError(t, err) - -// CORRECT: Separate EventuallyWithT for different external calls -// First external call - headscale.ListNodes() -assert.EventuallyWithT(t, func(c *assert.CollectT) { - nodes, err := headscale.ListNodes() - assert.NoError(c, err) - assert.Len(c, nodes, 2) - requireNodeRouteCountWithCollect(c, nodes[0], 2, 2, 2) -}, 10*time.Second, 500*time.Millisecond, "route state changes should propagate to nodes") - -// Second external call - client.Status() -assert.EventuallyWithT(t, func(c *assert.CollectT) { - status, err := client.Status() - assert.NoError(c, err) - - for _, peerKey := range status.Peers() { - peerStatus := status.Peer[peerKey] - requirePeerSubnetRoutesWithCollect(c, peerStatus, []netip.Prefix{tsaddr.AllIPv4(), tsaddr.AllIPv6()}) - } -}, 10*time.Second, 500*time.Millisecond, "routes should be visible to client") - -// INCORRECT: Multiple unrelated external calls in same EventuallyWithT -assert.EventuallyWithT(t, func(c *assert.CollectT) { - nodes, err := headscale.ListNodes() // ❌ First external call - assert.NoError(c, err) - - status, err := client.Status() // ❌ Different external call - should be separate - assert.NoError(c, err) -}, 10*time.Second, 500*time.Millisecond, "mixed calls") - -// CORRECT: Variable scoping for shared data -var ( - srs1, srs2, srs3 *ipnstate.Status - clientStatus *ipnstate.Status - srs1PeerStatus *ipnstate.PeerStatus -) - -assert.EventuallyWithT(t, func(c *assert.CollectT) { - srs1 = subRouter1.MustStatus() // = not := - srs2 = subRouter2.MustStatus() - clientStatus = client.MustStatus() - - srs1PeerStatus = clientStatus.Peer[srs1.Self.PublicKey] - // assertions... -}, 5*time.Second, 200*time.Millisecond, "checking router status") - -// CORRECT: Wrapping client operations -assert.EventuallyWithT(t, func(c *assert.CollectT) { - result, err := client.Curl(weburl) - assert.NoError(c, err) - assert.Len(c, result, 13) -}, 5*time.Second, 200*time.Millisecond, "Verifying HTTP connectivity") - -assert.EventuallyWithT(t, func(c *assert.CollectT) { - tr, err := client.Traceroute(webip) - assert.NoError(c, err) - assertTracerouteViaIPWithCollect(c, tr, expectedRouter.MustIPv4()) -}, 5*time.Second, 200*time.Millisecond, "Verifying network path") -``` - -**Helper Functions**: -- Use `requirePeerSubnetRoutesWithCollect` instead of `requirePeerSubnetRoutes` inside EventuallyWithT -- Use `requireNodeRouteCountWithCollect` instead of `requireNodeRouteCount` inside EventuallyWithT -- Use `assertTracerouteViaIPWithCollect` instead of `assertTracerouteViaIP` inside EventuallyWithT - -```go -// Node route checking by actual node properties, not array position -var routeNode *v1.Node -for _, node := range nodes { - if nodeIDStr := fmt.Sprintf("%d", node.GetId()); expectedRoutes[nodeIDStr] != "" { - routeNode = node - break - } -} -``` - -### Running Problematic Tests -- Some tests require significant time (e.g., `TestNodeOnlineStatus` runs for 12 minutes) -- Infrastructure issues like disk space can cause test failures unrelated to code changes -- Use `--postgres` flag when testing database-heavy scenarios - -## Quality Assurance and Testing Requirements - -### **MANDATORY: Always Use Specialized Testing Agents** - -**CRITICAL REQUIREMENT**: For ANY task involving testing, quality assurance, review, or validation, you MUST use the appropriate specialized agent at the END of your task list. This ensures comprehensive quality validation and prevents regressions. - -**Required Agents for Different Task Types**: - -1. **Integration Testing**: Use `headscale-integration-tester` agent for: - - Running integration tests with `cmd/hi` - - Analyzing test failures and artifacts - - Troubleshooting Docker-based test infrastructure - - Validating end-to-end functionality changes - -2. **Quality Control**: Use `quality-control-enforcer` agent for: - - Code review and validation - - Ensuring best practices compliance - - Preventing common pitfalls and anti-patterns - - Validating architectural decisions - -**Agent Usage Pattern**: Always add the appropriate agent as the FINAL step in any task list to ensure quality validation occurs after all work is complete. - -### Integration Test Debugging Reference - -Test artifacts are preserved in `control_logs/TIMESTAMP-ID/` including: -- Headscale server logs (stderr/stdout) -- Tailscale client logs and status -- Database dumps and network captures -- MapResponse JSON files for protocol debugging - -**For integration test issues, ALWAYS use the headscale-integration-tester agent - do not attempt manual debugging.** - -## EventuallyWithT Pattern for Integration Tests - -### Overview -EventuallyWithT is a testing pattern used to handle eventual consistency in distributed systems. In Headscale integration tests, many operations are asynchronous - clients advertise routes, the server processes them, updates propagate through the network. EventuallyWithT allows tests to wait for these operations to complete while making assertions. - -### External Calls That Must Be Wrapped -The following operations are **external calls** that interact with the headscale server or tailscale clients and MUST be wrapped in EventuallyWithT: -- `headscale.ListNodes()` - Queries server state -- `client.Status()` - Gets client network status -- `client.Curl()` - Makes HTTP requests through the network -- `client.Traceroute()` - Performs network diagnostics -- `client.Execute()` when running commands that query state -- Any operation that reads from the headscale server or tailscale client - -### Operations That Must NOT Be Wrapped -The following are **blocking operations** that modify state and should NOT be wrapped in EventuallyWithT: -- `tailscale set` commands (e.g., `--advertise-routes`, `--exit-node`) -- Any command that changes configuration or state -- Use `client.MustStatus()` instead of `client.Status()` when you just need the ID for a blocking operation - -### Five Key Rules for EventuallyWithT - -1. **One External Call Per EventuallyWithT Block** - - Each EventuallyWithT should make ONE external call (e.g., ListNodes OR Status) - - Related assertions based on that single call can be grouped together - - Unrelated external calls must be in separate EventuallyWithT blocks - -2. **Variable Scoping** - - Declare variables that need to be shared across EventuallyWithT blocks at function scope - - Use `=` for assignment inside EventuallyWithT, not `:=` (unless the variable is only used within that block) - - Variables declared with `:=` inside EventuallyWithT are not accessible outside - -3. **No Nested EventuallyWithT** - - NEVER put an EventuallyWithT inside another EventuallyWithT - - This is a critical anti-pattern that must be avoided - -4. **Use CollectT for Assertions** - - Inside EventuallyWithT, use `assert` methods with the CollectT parameter - - Helper functions called within EventuallyWithT must accept `*assert.CollectT` - -5. **Descriptive Messages** - - Always provide a descriptive message as the last parameter - - Message should explain what condition is being waited for - -### Correct Pattern Examples - -```go -// CORRECT: Blocking operation NOT wrapped -for _, client := range allClients { - status := client.MustStatus() - command := []string{ - "tailscale", - "set", - "--advertise-routes=" + expectedRoutes[string(status.Self.ID)], - } - _, _, err = client.Execute(command) - require.NoErrorf(t, err, "failed to advertise route: %s", err) -} - -// CORRECT: Single external call with related assertions -var nodes []*v1.Node -assert.EventuallyWithT(t, func(c *assert.CollectT) { - nodes, err = headscale.ListNodes() - assert.NoError(c, err) - assert.Len(c, nodes, 2) - requireNodeRouteCountWithCollect(c, nodes[0], 2, 2, 2) -}, 10*time.Second, 500*time.Millisecond, "nodes should have expected route counts") - -// CORRECT: Separate EventuallyWithT for different external call -assert.EventuallyWithT(t, func(c *assert.CollectT) { - status, err := client.Status() - assert.NoError(c, err) - for _, peerKey := range status.Peers() { - peerStatus := status.Peer[peerKey] - requirePeerSubnetRoutesWithCollect(c, peerStatus, expectedPrefixes) - } -}, 10*time.Second, 500*time.Millisecond, "client should see expected routes") -``` - -### Incorrect Patterns to Avoid - -```go -// INCORRECT: Blocking operation wrapped in EventuallyWithT -assert.EventuallyWithT(t, func(c *assert.CollectT) { - status, err := client.Status() - assert.NoError(c, err) - - // This is a blocking operation - should NOT be in EventuallyWithT! - command := []string{ - "tailscale", - "set", - "--advertise-routes=" + expectedRoutes[string(status.Self.ID)], - } - _, _, err = client.Execute(command) - assert.NoError(c, err) -}, 5*time.Second, 200*time.Millisecond, "wrong pattern") - -// INCORRECT: Multiple unrelated external calls in same EventuallyWithT -assert.EventuallyWithT(t, func(c *assert.CollectT) { - // First external call - nodes, err := headscale.ListNodes() - assert.NoError(c, err) - assert.Len(c, nodes, 2) - - // Second unrelated external call - WRONG! - status, err := client.Status() - assert.NoError(c, err) - assert.NotNil(c, status) -}, 10*time.Second, 500*time.Millisecond, "mixed operations") -``` - -## Important Notes - -- **Dependencies**: Use `nix develop` for consistent toolchain (Go, buf, protobuf tools, linting) -- **Protocol Buffers**: Changes to `proto/` require `make generate` and should be committed separately -- **Code Style**: Enforced via golangci-lint with golines (width 88) and gofumpt formatting -- **Database**: Supports both SQLite (development) and PostgreSQL (production/testing) -- **Integration Tests**: Require Docker and can consume significant disk space - use headscale-integration-tester agent -- **Performance**: NodeStore optimizations are critical for scale - be careful with changes to state management -- **Quality Assurance**: Always use appropriate specialized agents for testing and validation tasks -- **NEVER create gists in the user's name**: Do not use the `create_gist` tool - present information directly in the response instead +@AGENTS.md diff --git a/Makefile b/Makefile index d9b2c76b..9a5b8dfa 100644 --- a/Makefile +++ b/Makefile @@ -117,7 +117,7 @@ help: @echo "" @echo "Specific targets:" @echo " fmt-go - Format Go code only" - @echo " fmt-prettier - Format documentation only" + @echo " fmt-prettier - Format documentation only" @echo " fmt-proto - Format Protocol Buffer files only" @echo " lint-go - Lint Go code only" @echo " lint-proto - Lint Protocol Buffer files only" @@ -126,4 +126,4 @@ help: @echo " check-deps - Verify required tools are available" @echo "" @echo "Note: If not running in a nix shell, ensure dependencies are available:" - @echo " nix develop" \ No newline at end of file + @echo " nix develop" diff --git a/README.md b/README.md index 61a2c92c..dbde74d9 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,7 @@ make build We recommend using Nix for dependency management to ensure you have all required tools. If you prefer to manage dependencies yourself, you can use Make directly: **With Nix (recommended):** + ```shell nix develop make test @@ -154,6 +155,7 @@ make build ``` **With your own dependencies:** + ```shell make test make build diff --git a/derp-example.yaml b/derp-example.yaml index 532475ef..ea93427c 100644 --- a/derp-example.yaml +++ b/derp-example.yaml @@ -1,6 +1,6 @@ # If you plan to somehow use headscale, please deploy your own DERP infra: https://tailscale.com/kb/1118/custom-derp-servers/ regions: - 1: null # Disable DERP region with ID 1 + 1: null # Disable DERP region with ID 1 900: regionid: 900 regioncode: custom diff --git a/docs/logo/headscale3-dots.svg b/docs/logo/headscale3-dots.svg index 6a20973c..f7120395 100644 --- a/docs/logo/headscale3-dots.svg +++ b/docs/logo/headscale3-dots.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/docs/logo/headscale3_header_stacked_left.svg b/docs/logo/headscale3_header_stacked_left.svg index d00af00e..0c3702c6 100644 --- a/docs/logo/headscale3_header_stacked_left.svg +++ b/docs/logo/headscale3_header_stacked_left.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/flake.nix b/flake.nix index f8eb6dd1..86f8b005 100644 --- a/flake.nix +++ b/flake.nix @@ -6,239 +6,246 @@ flake-utils.url = "github:numtide/flake-utils"; }; - outputs = { - self, - nixpkgs, - flake-utils, - ... - }: let - headscaleVersion = self.shortRev or self.dirtyShortRev; - commitHash = self.rev or self.dirtyRev; - in + outputs = + { self + , nixpkgs + , flake-utils + , ... + }: + let + headscaleVersion = self.shortRev or self.dirtyShortRev; + commitHash = self.rev or self.dirtyRev; + in { - overlay = _: prev: let - pkgs = nixpkgs.legacyPackages.${prev.system}; - buildGo = pkgs.buildGo125Module; - vendorHash = "sha256-VOi4PGZ8I+2MiwtzxpKc/4smsL5KcH/pHVkjJfAFPJ0="; - in { - headscale = buildGo { - pname = "headscale"; - version = headscaleVersion; - src = pkgs.lib.cleanSource self; + overlay = _: prev: + let + pkgs = nixpkgs.legacyPackages.${prev.system}; + buildGo = pkgs.buildGo125Module; + vendorHash = "sha256-VOi4PGZ8I+2MiwtzxpKc/4smsL5KcH/pHVkjJfAFPJ0="; + in + { + headscale = buildGo { + pname = "headscale"; + version = headscaleVersion; + src = pkgs.lib.cleanSource self; - # Only run unit tests when testing a build - checkFlags = ["-short"]; + # Only run unit tests when testing a build + checkFlags = [ "-short" ]; - # When updating go.mod or go.sum, a new sha will need to be calculated, - # update this if you have a mismatch after doing a change to those files. - inherit vendorHash; + # When updating go.mod or go.sum, a new sha will need to be calculated, + # update this if you have a mismatch after doing a change to those files. + inherit vendorHash; - subPackages = ["cmd/headscale"]; + subPackages = [ "cmd/headscale" ]; - ldflags = [ - "-s" - "-w" - "-X github.com/juanfont/headscale/hscontrol/types.Version=${headscaleVersion}" - "-X github.com/juanfont/headscale/hscontrol/types.GitCommitHash=${commitHash}" - ]; - }; - - hi = buildGo { - pname = "hi"; - version = headscaleVersion; - src = pkgs.lib.cleanSource self; - - checkFlags = ["-short"]; - inherit vendorHash; - - subPackages = ["cmd/hi"]; - }; - - protoc-gen-grpc-gateway = buildGo rec { - pname = "grpc-gateway"; - version = "2.24.0"; - - src = pkgs.fetchFromGitHub { - owner = "grpc-ecosystem"; - repo = "grpc-gateway"; - rev = "v${version}"; - sha256 = "sha256-lUEoqXJF1k4/il9bdDTinkUV5L869njZNYqObG/mHyA="; + ldflags = [ + "-s" + "-w" + "-X github.com/juanfont/headscale/hscontrol/types.Version=${headscaleVersion}" + "-X github.com/juanfont/headscale/hscontrol/types.GitCommitHash=${commitHash}" + ]; }; - vendorHash = "sha256-Ttt7bPKU+TMKRg5550BS6fsPwYp0QJqcZ7NLrhttSdw="; + hi = buildGo { + pname = "hi"; + version = headscaleVersion; + src = pkgs.lib.cleanSource self; - nativeBuildInputs = [pkgs.installShellFiles]; + checkFlags = [ "-short" ]; + inherit vendorHash; - subPackages = ["protoc-gen-grpc-gateway" "protoc-gen-openapiv2"]; - }; - - protobuf-language-server = buildGo rec { - pname = "protobuf-language-server"; - version = "2546944"; - - src = pkgs.fetchFromGitHub { - owner = "lasorda"; - repo = "protobuf-language-server"; - rev = "${version}"; - sha256 = "sha256-Cbr3ktT86RnwUntOiDKRpNTClhdyrKLTQG2ZEd6fKDc="; + subPackages = [ "cmd/hi" ]; }; - vendorHash = "sha256-PfT90dhfzJZabzLTb1D69JCO+kOh2khrlpF5mCDeypk="; + protoc-gen-grpc-gateway = buildGo rec { + pname = "grpc-gateway"; + version = "2.24.0"; - subPackages = ["."]; + src = pkgs.fetchFromGitHub { + owner = "grpc-ecosystem"; + repo = "grpc-gateway"; + rev = "v${version}"; + sha256 = "sha256-lUEoqXJF1k4/il9bdDTinkUV5L869njZNYqObG/mHyA="; + }; + + vendorHash = "sha256-Ttt7bPKU+TMKRg5550BS6fsPwYp0QJqcZ7NLrhttSdw="; + + nativeBuildInputs = [ pkgs.installShellFiles ]; + + subPackages = [ "protoc-gen-grpc-gateway" "protoc-gen-openapiv2" ]; + }; + + protobuf-language-server = buildGo rec { + pname = "protobuf-language-server"; + version = "2546944"; + + src = pkgs.fetchFromGitHub { + owner = "lasorda"; + repo = "protobuf-language-server"; + rev = "${version}"; + sha256 = "sha256-Cbr3ktT86RnwUntOiDKRpNTClhdyrKLTQG2ZEd6fKDc="; + }; + + vendorHash = "sha256-PfT90dhfzJZabzLTb1D69JCO+kOh2khrlpF5mCDeypk="; + + subPackages = [ "." ]; + }; + + # Upstream does not override buildGoModule properly, + # importing a specific module, so comment out for now. + # golangci-lint = prev.golangci-lint.override { + # buildGoModule = buildGo; + # }; + # golangci-lint-langserver = prev.golangci-lint.override { + # buildGoModule = buildGo; + # }; + + # The package uses buildGo125Module, not the convention. + # goreleaser = prev.goreleaser.override { + # buildGoModule = buildGo; + # }; + + gotestsum = prev.gotestsum.override { + buildGoModule = buildGo; + }; + + gotests = prev.gotests.override { + buildGoModule = buildGo; + }; + + gofumpt = prev.gofumpt.override { + buildGoModule = buildGo; + }; + + # gopls = prev.gopls.override { + # buildGoModule = buildGo; + # }; }; - - # Upstream does not override buildGoModule properly, - # importing a specific module, so comment out for now. - # golangci-lint = prev.golangci-lint.override { - # buildGoModule = buildGo; - # }; - # golangci-lint-langserver = prev.golangci-lint.override { - # buildGoModule = buildGo; - # }; - - # The package uses buildGo125Module, not the convention. - # goreleaser = prev.goreleaser.override { - # buildGoModule = buildGo; - # }; - - gotestsum = prev.gotestsum.override { - buildGoModule = buildGo; - }; - - gotests = prev.gotests.override { - buildGoModule = buildGo; - }; - - gofumpt = prev.gofumpt.override { - buildGoModule = buildGo; - }; - - # gopls = prev.gopls.override { - # buildGoModule = buildGo; - # }; - }; } // flake-utils.lib.eachDefaultSystem - (system: let - pkgs = import nixpkgs { - overlays = [self.overlay]; - inherit system; - }; - buildDeps = with pkgs; [git go_1_25 gnumake]; - devDeps = with pkgs; - buildDeps - ++ [ - golangci-lint - golangci-lint-langserver - golines - nodePackages.prettier - goreleaser - nfpm - gotestsum - gotests - gofumpt - gopls - ksh - ko - yq-go - ripgrep - postgresql - - # 'dot' is needed for pprof graphs - # go tool pprof -http=: - graphviz - - # Protobuf dependencies - protobuf - protoc-gen-go - protoc-gen-go-grpc - protoc-gen-grpc-gateway - buf - clang-tools # clang-format - protobuf-language-server - - # Add hi to make it even easier to use ci runner. - hi - ] - ++ lib.optional pkgs.stdenv.isLinux [traceroute]; - - # Add entry to build a docker image with headscale - # caveat: only works on Linux - # - # Usage: - # nix build .#headscale-docker - # docker load < result - headscale-docker = pkgs.dockerTools.buildLayeredImage { - name = "headscale"; - tag = headscaleVersion; - contents = [pkgs.headscale]; - config.Entrypoint = [(pkgs.headscale + "/bin/headscale")]; - }; - in rec { - # `nix develop` - devShell = pkgs.mkShell { - buildInputs = - devDeps + (system: + let + pkgs = import nixpkgs { + overlays = [ self.overlay ]; + inherit system; + }; + buildDeps = with pkgs; [ git go_1_25 gnumake ]; + devDeps = with pkgs; + buildDeps ++ [ - (pkgs.writeShellScriptBin - "nix-vendor-sri" - '' - set -eu + golangci-lint + golangci-lint-langserver + golines + nodePackages.prettier + nixpkgs-fmt + goreleaser + nfpm + gotestsum + gotests + gofumpt + gopls + ksh + ko + yq-go + ripgrep + postgresql + prek - OUT=$(mktemp -d -t nar-hash-XXXXXX) - rm -rf "$OUT" + # 'dot' is needed for pprof graphs + # go tool pprof -http=: + graphviz - go mod vendor -o "$OUT" - go run tailscale.com/cmd/nardump --sri "$OUT" - rm -rf "$OUT" - '') + # Protobuf dependencies + protobuf + protoc-gen-go + protoc-gen-go-grpc + protoc-gen-grpc-gateway + buf + clang-tools # clang-format + protobuf-language-server - (pkgs.writeShellScriptBin - "go-mod-update-all" - '' - cat go.mod | ${pkgs.silver-searcher}/bin/ag "\t" | ${pkgs.silver-searcher}/bin/ag -v indirect | ${pkgs.gawk}/bin/awk '{print $1}' | ${pkgs.findutils}/bin/xargs go get -u - go mod tidy - '') - ]; + # Add hi to make it even easier to use ci runner. + hi + ] + ++ lib.optional pkgs.stdenv.isLinux [ traceroute ]; - shellHook = '' - export PATH="$PWD/result/bin:$PATH" - ''; - }; + # Add entry to build a docker image with headscale + # caveat: only works on Linux + # + # Usage: + # nix build .#headscale-docker + # docker load < result + headscale-docker = pkgs.dockerTools.buildLayeredImage { + name = "headscale"; + tag = headscaleVersion; + contents = [ pkgs.headscale ]; + config.Entrypoint = [ (pkgs.headscale + "/bin/headscale") ]; + }; + in + rec { + # `nix develop` + devShell = pkgs.mkShell { + buildInputs = + devDeps + ++ [ + (pkgs.writeShellScriptBin + "nix-vendor-sri" + '' + set -eu - # `nix build` - packages = with pkgs; { - inherit headscale; - inherit headscale-docker; - }; - defaultPackage = pkgs.headscale; + OUT=$(mktemp -d -t nar-hash-XXXXXX) + rm -rf "$OUT" - # `nix run` - apps.headscale = flake-utils.lib.mkApp { - drv = packages.headscale; - }; - apps.default = apps.headscale; + go mod vendor -o "$OUT" + go run tailscale.com/cmd/nardump --sri "$OUT" + rm -rf "$OUT" + '') - checks = { - format = - pkgs.runCommand "check-format" - { - buildInputs = with pkgs; [ - gnumake - nixpkgs-fmt - golangci-lint - nodePackages.prettier - golines - clang-tools + (pkgs.writeShellScriptBin + "go-mod-update-all" + '' + cat go.mod | ${pkgs.silver-searcher}/bin/ag "\t" | ${pkgs.silver-searcher}/bin/ag -v indirect | ${pkgs.gawk}/bin/awk '{print $1}' | ${pkgs.findutils}/bin/xargs go get -u + go mod tidy + '') ]; - } '' - ${pkgs.nixpkgs-fmt}/bin/nixpkgs-fmt ${./.} - ${pkgs.golangci-lint}/bin/golangci-lint run --fix --timeout 10m - ${pkgs.nodePackages.prettier}/bin/prettier --write '**/**.{ts,js,md,yaml,yml,sass,css,scss,html}' - ${pkgs.golines}/bin/golines --max-len=88 --base-formatter=gofumpt -w ${./.} - ${pkgs.clang-tools}/bin/clang-format -i ${./.} + + shellHook = '' + export PATH="$PWD/result/bin:$PATH" ''; - }; - }); + }; + + # `nix build` + packages = with pkgs; { + inherit headscale; + inherit headscale-docker; + }; + defaultPackage = pkgs.headscale; + + # `nix run` + apps.headscale = flake-utils.lib.mkApp { + drv = packages.headscale; + }; + apps.default = apps.headscale; + + checks = { + format = + pkgs.runCommand "check-format" + { + buildInputs = with pkgs; [ + gnumake + nixpkgs-fmt + golangci-lint + nodePackages.prettier + golines + clang-tools + ]; + } '' + ${pkgs.nixpkgs-fmt}/bin/nixpkgs-fmt ${./.} + ${pkgs.golangci-lint}/bin/golangci-lint run --fix --timeout 10m + ${pkgs.nodePackages.prettier}/bin/prettier --write '**/**.{ts,js,md,yaml,yml,sass,css,scss,html}' + ${pkgs.golines}/bin/golines --max-len=88 --base-formatter=gofumpt -w ${./.} + ${pkgs.clang-tools}/bin/clang-format -i ${./.} + ''; + }; + }); } diff --git a/integration/auth_key_test.go b/integration/auth_key_test.go index 75106dc5..12a5bf67 100644 --- a/integration/auth_key_test.go +++ b/integration/auth_key_test.go @@ -455,4 +455,3 @@ func TestAuthKeyLogoutAndReloginSameUserExpiredKey(t *testing.T) { }) } } -