From 93b8fdcd684bba1b956767bf8ce7e85bc828a196 Mon Sep 17 00:00:00 2001 From: silverwind Date: Thu, 21 May 2026 06:58:41 +0200 Subject: [PATCH] ci: shard tests and reduce redundant work (#37618) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Critical path ~25:42 → ~19:56 (−22%), ~0% CI minutes. - `test-pgsql` shards 2-way. Branch protection: replace `test-pgsql` with `test-pgsql-shards (1)` + `test-pgsql-shards (2)`; `test-unit`, sqlite/mysql/mssql unchanged — pgsql dominates the critical path. - `test-unit` runs `bindata` then `bindata gogit` sequentially. cache-seeder pre-warms the race-instrumented test compile cache and the integration test binary so PR jobs warm-start. - Cache writes restricted to cache-seeder; PR jobs use `actions/cache/restore`. Defends against PR cache poisoning and frees the 10 GB cap from PR churn. - `go-cache` action: dropped the `cache-name` input. One gobuild cache, one golangci-lint cache. Seeder lint job restores but doesn't save gobuild, so only one writer populates it. - `tools/test-integration.sh` shards the integration binary via `-test.list`; `TestMain` short-circuits DB init in list mode. `TestAPILFSNotStarted` / `TestAPILFSLocksNotStarted` switched to `test.MockVariableValue` — latent `setting.LFS.StartServer` global-state leak uncovered by sharding. --- This PR was written with the help of Claude Opus 4.7 --------- Signed-off-by: silverwind Co-authored-by: Claude (Opus 4.7) Co-authored-by: wxiaoguang Co-authored-by: Nicolas --- .github/actions/go-cache/action.yml | 54 ++++++++++++++------------- .github/workflows/cache-seeder.yml | 39 +++++++++++-------- .github/workflows/pull-compliance.yml | 6 --- .github/workflows/pull-db-tests.yml | 34 ++++++++--------- .github/workflows/pull-e2e-tests.yml | 3 -- Makefile | 6 ++- tools/test-integration.sh | 31 +++++++++++++++ 7 files changed, 103 insertions(+), 70 deletions(-) create mode 100755 tools/test-integration.sh diff --git a/.github/actions/go-cache/action.yml b/.github/actions/go-cache/action.yml index 04d4bac3673..7096fa3952c 100644 --- a/.github/actions/go-cache/action.yml +++ b/.github/actions/go-cache/action.yml @@ -1,47 +1,51 @@ name: go-caches -description: Restore and save go module, build, and golangci-lint caches +description: Restore the go module, build, and golangci-lint caches. Save only on the cache-seeder workflow. + +# Only the cache-seeder workflow saves; rename requires updating cache-seeder.yml. +# The lint job restores but does not save the gobuild cache, so only one writer +# (the gobuild job) populates it and there is no contention on the cache key. inputs: - cache-name: - description: Short identifier used in the per-caller build cache key - required: true - build-cache: - description: Whether to include ~/.cache/go-build - default: "true" - build-cache-rotate: - description: Whether to rotate the build cache key per run so Go's test result cache can accumulate across runs - default: "false" lint-cache: - description: Whether to include ~/.cache/golangci-lint + description: Restore (and save in cache-seeder) ~/.cache/golangci-lint default: "false" runs: using: composite steps: - - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + - if: ${{ github.workflow == 'cache-seeder' }} + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ~/go/pkg/mod key: gomod-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('go.sum') }} restore-keys: gomod-${{ runner.os }}-${{ runner.arch }} - - if: ${{ inputs.build-cache == 'true' && inputs.build-cache-rotate == 'true' }} - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + - if: ${{ github.workflow != 'cache-seeder' }} + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: - path: ~/.cache/go-build - key: gobuild-${{ runner.os }}-${{ runner.arch }}-${{ inputs.cache-name }}-${{ hashFiles('go.sum') }}-${{ github.run_id }} - restore-keys: | - gobuild-${{ runner.os }}-${{ runner.arch }}-${{ inputs.cache-name }}-${{ hashFiles('go.sum') }} - gobuild-${{ runner.os }}-${{ runner.arch }}-${{ inputs.cache-name }} - gobuild-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('go.sum') }} - gobuild-${{ runner.os }}-${{ runner.arch }} - - if: ${{ inputs.build-cache == 'true' && inputs.build-cache-rotate != 'true' }} + path: ~/go/pkg/mod + key: gomod-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('go.sum') }} + restore-keys: gomod-${{ runner.os }}-${{ runner.arch }} + - if: ${{ github.workflow == 'cache-seeder' && inputs.lint-cache != 'true' }} uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ~/.cache/go-build key: gobuild-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('go.sum') }} restore-keys: gobuild-${{ runner.os }}-${{ runner.arch }} - - if: ${{ inputs.lint-cache == 'true' }} + - if: ${{ github.workflow != 'cache-seeder' || inputs.lint-cache == 'true' }} + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ~/.cache/go-build + key: gobuild-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('go.sum') }} + restore-keys: gobuild-${{ runner.os }}-${{ runner.arch }} + - if: ${{ inputs.lint-cache == 'true' && github.workflow == 'cache-seeder' }} uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ~/.cache/golangci-lint - key: golangci-${{ runner.os }}-${{ runner.arch }}-${{ inputs.cache-name }}-${{ hashFiles('go.sum', '.golangci.yml') }} - restore-keys: golangci-${{ runner.os }}-${{ runner.arch }}-${{ inputs.cache-name }} + key: golint-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('go.sum', '.golangci.yml') }} + restore-keys: golint-${{ runner.os }}-${{ runner.arch }} + - if: ${{ inputs.lint-cache == 'true' && github.workflow != 'cache-seeder' }} + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ~/.cache/golangci-lint + key: golint-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('go.sum', '.golangci.yml') }} + restore-keys: golint-${{ runner.os }}-${{ runner.arch }} diff --git a/.github/workflows/cache-seeder.yml b/.github/workflows/cache-seeder.yml index 317ddf1510a..91109b1c06d 100644 --- a/.github/workflows/cache-seeder.yml +++ b/.github/workflows/cache-seeder.yml @@ -1,15 +1,9 @@ -# Populates the go module, build, and golangci-lint caches under the default -# branch's cache scope so that PR runs have a warm fallback to restore from. +# Populates main's cache scope so PR runs warm-start from it. Saves the go +# module, go build (incl. test compile), and golangci-lint caches. # -# GitHub Actions caches are scoped per ref: a PR run can only write to its own -# branch's scope, but can read from the base branch's scope as a fallback. -# PRs therefore cannot seed main's scope themselves. Running the same cache -# steps on push-to-main is the only opportunity to populate that fallback -# scope so fresh PR branches start with a useful cache on first run. - -# A PR job's exact key lives in its own PR-scope (empty on first run, filled -# by later runs of the same PR); on miss, actions/cache's restore-keys fall -# back to prefix matches against entries this seeder saves in main's scope. +# Caches are ref-scoped: PR runs read their own scope then fall back to the +# base branch. Per .github/actions/go-cache/action.yml, PRs are restore-only, +# so push-to-main is the only opportunity to populate the fallback scope. name: cache-seeder @@ -41,11 +35,25 @@ jobs: check-latest: true cache: false - uses: ./.github/actions/go-cache - with: - cache-name: seed - - run: make deps-backend + - run: make deps-backend deps-tools - run: TAGS="bindata" make backend - run: TAGS="bindata gogit" GOEXPERIMENT="" make backend + - name: warm test compile cache (bindata) + env: + TAGS: bindata + GOTEST_FLAGS: -race -list=^$$ -count=1 + run: make test-backend + - name: warm test compile cache (bindata gogit) + env: + TAGS: bindata gogit + GOEXPERIMENT: + GOTEST_FLAGS: -race -list=^$$ -count=1 + run: make test-backend + - name: warm integration compile cache + run: | + TAGS="bindata" make test-integration-compile + TAGS="bindata gogit" GOEXPERIMENT="" make test-integration-compile + TAGS="bindata gogit" GOTEST_FLAGS="-race" make test-integration-compile lint: runs-on: ubuntu-latest @@ -53,7 +61,7 @@ jobs: fail-fast: false matrix: include: - - { job: lint-backend, tags: "bindata", target: "lint-backend" } + - { tags: "bindata", target: "lint-backend" } steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 @@ -63,7 +71,6 @@ jobs: cache: false - uses: ./.github/actions/go-cache with: - cache-name: ${{ matrix.job }} lint-cache: "true" - run: make deps-backend deps-tools - run: make generate-go diff --git a/.github/workflows/pull-compliance.yml b/.github/workflows/pull-compliance.yml index 7f24d4c5845..ada537a188c 100644 --- a/.github/workflows/pull-compliance.yml +++ b/.github/workflows/pull-compliance.yml @@ -27,7 +27,6 @@ jobs: cache: false - uses: ./.github/actions/go-cache with: - cache-name: lint-backend lint-cache: "true" - run: make deps-backend deps-tools - run: TAGS="bindata" make generate-go # lint-go also lints with "bindata" tags which requires "_bindata.go" @@ -75,9 +74,6 @@ jobs: check-latest: true cache: false - uses: ./.github/actions/go-cache - with: - cache-name: checks-backend - build-cache: "false" - run: make deps-backend deps-tools - run: make --always-make checks-backend # ensure the "go-licenses" make target runs @@ -111,8 +107,6 @@ jobs: check-latest: true cache: false - uses: ./.github/actions/go-cache - with: - cache-name: compliance-backend - run: make deps-backend generate-go # no frontend build here as backend should be able to build, even without any frontend files # CGO is not used when cross-compile, so these steps also test if the code is compatible with CGO disabled diff --git a/.github/workflows/pull-db-tests.yml b/.github/workflows/pull-db-tests.yml index 6e91eafe3ec..bc73d6391ca 100644 --- a/.github/workflows/pull-db-tests.yml +++ b/.github/workflows/pull-db-tests.yml @@ -14,10 +14,14 @@ jobs: files-changed: uses: ./.github/workflows/files-changed.yml - test-pgsql: + test-pgsql-shards: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: [1, 2] services: pgsql: image: postgres:14 @@ -48,8 +52,6 @@ jobs: check-latest: true cache: false - uses: ./.github/actions/go-cache - with: - cache-name: pgsql - name: Add hosts to /etc/hosts run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 pgsql ldap minio" | sudo tee -a /etc/hosts' - run: make deps-backend @@ -57,6 +59,7 @@ jobs: env: TAGS: bindata - name: run migration tests + if: matrix.shard == 1 run: GITEA_TEST_DATABASE=pgsql make test-migration - name: run tests run: GITEA_TEST_DATABASE=pgsql make test-integration @@ -66,6 +69,8 @@ jobs: GOTEST_FLAGS: -race -timeout=40m TAGS: bindata gogit TEST_LDAP: 1 + TEST_SHARD: ${{ matrix.shard }} + TEST_TOTAL_SHARDS: ${{ strategy.job-total }} test-sqlite: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' @@ -79,15 +84,12 @@ jobs: check-latest: true cache: false - uses: ./.github/actions/go-cache - with: - cache-name: sqlite - run: make deps-backend - run: make backend env: TAGS: bindata gogit GOEXPERIMENT: - - name: run migration tests - run: GITEA_TEST_DATABASE=sqlite make test-migration + - run: GITEA_TEST_DATABASE=sqlite make test-migration env: TAGS: bindata gogit - name: run tests @@ -145,28 +147,27 @@ jobs: check-latest: true cache: false - uses: ./.github/actions/go-cache - with: - cache-name: unit - build-cache-rotate: "true" - name: Add hosts to /etc/hosts run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 minio devstoreaccount1.azurite.local mysql elasticsearch meilisearch smtpimap" | sudo tee -a /etc/hosts' - run: make deps-backend - - run: make backend + - run: make generate-go env: TAGS: bindata - name: unit-tests - run: make test-backend test-check + run: make test-backend env: GOTEST_FLAGS: -race -timeout=20m TAGS: bindata GITHUB_READ_TOKEN: ${{ secrets.GITHUB_READ_TOKEN }} - name: unit-tests-gogit - run: make test-backend test-check + run: make test-backend env: GOTEST_FLAGS: -race -timeout=20m TAGS: bindata gogit GOEXPERIMENT: GITHUB_READ_TOKEN: ${{ secrets.GITHUB_READ_TOKEN }} + GITEA_TEST_CI_SKIP_EXTERNAL: true + - run: make test-check test-mysql: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' @@ -205,16 +206,13 @@ jobs: check-latest: true cache: false - uses: ./.github/actions/go-cache - with: - cache-name: mysql - name: Add hosts to /etc/hosts run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 mysql elasticsearch smtpimap" | sudo tee -a /etc/hosts' - run: make deps-backend - run: make backend env: TAGS: bindata - - name: run migration tests - run: GITEA_TEST_DATABASE=mysql make test-migration + - run: GITEA_TEST_DATABASE=mysql make test-migration - name: run tests run: GITEA_TEST_DATABASE=mysql make test-integration env: @@ -246,8 +244,6 @@ jobs: check-latest: true cache: false - uses: ./.github/actions/go-cache - with: - cache-name: mssql - name: Add hosts to /etc/hosts run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 mssql devstoreaccount1.azurite.local" | sudo tee -a /etc/hosts' - run: make deps-backend diff --git a/.github/workflows/pull-e2e-tests.yml b/.github/workflows/pull-e2e-tests.yml index adadc0d8076..7bf73d4a0f9 100644 --- a/.github/workflows/pull-e2e-tests.yml +++ b/.github/workflows/pull-e2e-tests.yml @@ -26,9 +26,6 @@ jobs: check-latest: true cache: false - uses: ./.github/actions/go-cache - with: - cache-name: e2e - build-cache: "false" - uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: diff --git a/Makefile b/Makefile index 782e31ccfdd..94f07a44125 100644 --- a/Makefile +++ b/Makefile @@ -445,7 +445,11 @@ test-integration: @# would flood output per passing test. testcache can't help these tests anyway — @# they mutate the work directory, so cache inputs change between runs. $(GO) test $(GOTEST_FLAGS) -tags '$(TAGS)' -c code.gitea.io/gitea/tests/integration -o ./test-integration-$(GITEA_TEST_DATABASE).test - ./test-integration-$(GITEA_TEST_DATABASE).test + ./tools/test-integration.sh ./test-integration-$(GITEA_TEST_DATABASE).test + +.PHONY: test-integration-compile +test-integration-compile: + $(GO) test $(GOTEST_FLAGS) -tags '$(TAGS)' -c -o /dev/null code.gitea.io/gitea/tests/integration .PHONY: test-integration\#% test-integration\#%: diff --git a/tools/test-integration.sh b/tools/test-integration.sh new file mode 100755 index 00000000000..8e9cac51b1b --- /dev/null +++ b/tools/test-integration.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -euo pipefail + +# Run a compiled *.test binary. When TEST_SHARD is set, enumerate top-level +# tests via -test.list and run only the shard's slice; TestMain skips +# environment setup in -test.list mode. Without TEST_SHARD, runs the binary +# directly. + +BINARY=${1:?usage: $0 BINARY} + +if [ -z "${TEST_SHARD:-}" ]; then + exec "$BINARY" +fi + +if ! [[ "${TEST_TOTAL_SHARDS:-}" =~ ^[1-9][0-9]*$ ]]; then + echo "TEST_TOTAL_SHARDS must be a positive integer, got: ${TEST_TOTAL_SHARDS:-}" >&2 + exit 2 +fi +if ! [[ "$TEST_SHARD" =~ ^[1-9][0-9]*$ ]] || [ "$TEST_SHARD" -gt "$TEST_TOTAL_SHARDS" ]; then + echo "TEST_SHARD must be in [1, $TEST_TOTAL_SHARDS], got: $TEST_SHARD" >&2 + exit 2 +fi + +NAMES=$("$BINARY" -test.list='^Test' | LC_ALL=C sort -u | awk -v r=$((TEST_SHARD - 1)) -v t="$TEST_TOTAL_SHARDS" '(NR - 1) % t == r') +if [ -z "$NAMES" ]; then + echo "shard $TEST_SHARD/$TEST_TOTAL_SHARDS has no tests assigned" >&2 + exit 1 +fi +PATTERN=$(echo "$NAMES" | paste -sd '|' -) +echo "Running shard $TEST_SHARD/$TEST_TOTAL_SHARDS ($(echo "$NAMES" | wc -l | tr -d ' ') tests)" +exec "$BINARY" -test.run "^($PATTERN)\$"