From f95c210abe7333808fc7b140d91000b0b33efaf4 Mon Sep 17 00:00:00 2001 From: silverwind Date: Fri, 22 May 2026 21:07:35 +0200 Subject: [PATCH] ci: split pgsql shards into plain jobs, dedupe setup actions (#37802) 1. Split the psql matrix jobs into composite actions. Matrix jobs that can skip do not work with required checks on GitHub because skipped and unskipped emit different job names (GitHub bug https://github.com/orgs/community/discussions/9141). 2. Dedupe node and go setup steps into composite actions Currently test-psql branch protection is disabled, will re-enable when merging this. --------- Signed-off-by: silverwind Co-authored-by: Claude (Opus 4.7) Co-authored-by: Nicolas --- .github/actions/go-setup/action.yml | 23 +++++++ .github/actions/node-setup/action.yml | 22 ++++++ .github/actions/pgsql-shard/action.yml | 40 +++++++++++ .github/workflows/cache-seeder.yml | 15 +--- .github/workflows/files-changed.yml | 1 + .github/workflows/pull-compliance.yml | 39 +++-------- .github/workflows/pull-db-tests.yml | 95 ++++++++++++-------------- .github/workflows/pull-e2e-tests.yml | 14 +--- 8 files changed, 143 insertions(+), 106 deletions(-) create mode 100644 .github/actions/go-setup/action.yml create mode 100644 .github/actions/node-setup/action.yml create mode 100644 .github/actions/pgsql-shard/action.yml diff --git a/.github/actions/go-setup/action.yml b/.github/actions/go-setup/action.yml new file mode 100644 index 00000000000..d6f88bbc6da --- /dev/null +++ b/.github/actions/go-setup/action.yml @@ -0,0 +1,23 @@ +name: go-setup +description: Set up go and restore caches + +inputs: + cache: + description: Restore go caches + default: "true" + lint-cache: + description: Also restore the golangci-lint cache + default: "false" + +runs: + using: composite + steps: + - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + go-version-file: go.mod + check-latest: true + cache: false + - if: ${{ inputs.cache == 'true' }} + uses: ./.github/actions/go-cache + with: + lint-cache: ${{ inputs.lint-cache }} diff --git a/.github/actions/node-setup/action.yml b/.github/actions/node-setup/action.yml new file mode 100644 index 00000000000..c9ab484d57e --- /dev/null +++ b/.github/actions/node-setup/action.yml @@ -0,0 +1,22 @@ +name: node-setup +description: Set up pnpm and node and restore caches + +inputs: + cache: + description: Cache pnpm downloads + default: "true" + +runs: + using: composite + steps: + - uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 + - if: ${{ inputs.cache == 'true' }} + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 24 + cache: pnpm + cache-dependency-path: pnpm-lock.yaml + - if: ${{ inputs.cache != 'true' }} + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 24 diff --git a/.github/actions/pgsql-shard/action.yml b/.github/actions/pgsql-shard/action.yml new file mode 100644 index 00000000000..9a5a03ae0de --- /dev/null +++ b/.github/actions/pgsql-shard/action.yml @@ -0,0 +1,40 @@ +name: pgsql-shard +description: Run one pgsql integration test shard + +inputs: + shard: + description: Shard index + required: true + total-shards: + description: Total shard count + required: true + run-migration: + description: Also run migration tests + default: "false" + +runs: + using: composite + steps: + - name: Add hosts to /etc/hosts + shell: bash + run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 pgsql ldap minio" | sudo tee -a /etc/hosts' + - shell: bash + run: make deps-backend + - shell: bash + run: make backend + env: + TAGS: bindata + - name: run migration tests + if: ${{ inputs.run-migration == 'true' }} + shell: bash + run: GITEA_TEST_DATABASE=pgsql make test-migration + - name: run tests + shell: bash + run: GITEA_TEST_DATABASE=pgsql make test-integration + env: + # pgsql is chosen to be the unlucky one to run with the slow "race detector", it is about 60% slower. + GOTEST_FLAGS: -race -timeout=40m + TAGS: bindata gogit + TEST_LDAP: 1 + TEST_SHARD: ${{ inputs.shard }} + TEST_TOTAL_SHARDS: ${{ inputs.total-shards }} diff --git a/.github/workflows/cache-seeder.yml b/.github/workflows/cache-seeder.yml index 91109b1c06d..8ec7adee07d 100644 --- a/.github/workflows/cache-seeder.yml +++ b/.github/workflows/cache-seeder.yml @@ -15,6 +15,7 @@ on: - "go.sum" - ".golangci.yml" - ".github/actions/go-cache/action.yml" + - ".github/actions/go-setup/action.yml" - ".github/workflows/cache-seeder.yml" concurrency: @@ -29,12 +30,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 - with: - go-version-file: go.mod - check-latest: true - cache: false - - uses: ./.github/actions/go-cache + - uses: ./.github/actions/go-setup - run: make deps-backend deps-tools - run: TAGS="bindata" make backend - run: TAGS="bindata gogit" GOEXPERIMENT="" make backend @@ -64,12 +60,7 @@ jobs: - { tags: "bindata", target: "lint-backend" } steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 - with: - go-version-file: go.mod - check-latest: true - cache: false - - uses: ./.github/actions/go-cache + - uses: ./.github/actions/go-setup with: lint-cache: "true" - run: make deps-backend deps-tools diff --git a/.github/workflows/files-changed.yml b/.github/workflows/files-changed.yml index ac2bd1a5ba5..2975a0cfcbf 100644 --- a/.github/workflows/files-changed.yml +++ b/.github/workflows/files-changed.yml @@ -81,6 +81,7 @@ jobs: actions: - ".github/workflows/*" + - ".github/actions/**" - "Makefile" templates: diff --git a/.github/workflows/pull-compliance.yml b/.github/workflows/pull-compliance.yml index ada537a188c..9ccfc76315e 100644 --- a/.github/workflows/pull-compliance.yml +++ b/.github/workflows/pull-compliance.yml @@ -20,12 +20,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 - with: - go-version-file: go.mod - check-latest: true - cache: false - - uses: ./.github/actions/go-cache + - uses: ./.github/actions/go-setup with: lint-cache: "true" - run: make deps-backend deps-tools @@ -37,15 +32,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + - uses: ./.github/actions/go-setup with: - go-version-file: go.mod - check-latest: true - cache: false - - uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + cache: "false" + - uses: ./.github/actions/node-setup with: - node-version: 24 + cache: "false" - run: make lint-spell @@ -68,12 +60,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 - with: - go-version-file: go.mod - check-latest: true - cache: false - - uses: ./.github/actions/go-cache + - uses: ./.github/actions/go-setup - run: make deps-backend deps-tools - run: make --always-make checks-backend # ensure the "go-licenses" make target runs @@ -83,12 +70,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: 24 - cache: pnpm - cache-dependency-path: pnpm-lock.yaml + - uses: ./.github/actions/node-setup - run: make deps-frontend - run: make lint-frontend - run: make checks-frontend @@ -101,12 +83,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 - with: - go-version-file: go.mod - check-latest: true - cache: false - - uses: ./.github/actions/go-cache + - uses: ./.github/actions/go-setup - run: make deps-backend generate-go # no frontend build here as backend should be able to build, even without any frontend files # CGO is not used when cross-compile, so these steps also test if the code is compatible with CGO disabled diff --git a/.github/workflows/pull-db-tests.yml b/.github/workflows/pull-db-tests.yml index bc73d6391ca..e4de01e711a 100644 --- a/.github/workflows/pull-db-tests.yml +++ b/.github/workflows/pull-db-tests.yml @@ -14,14 +14,11 @@ jobs: files-changed: uses: ./.github/workflows/files-changed.yml - test-pgsql-shards: + test-pgsql-shard-1: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - shard: [1, 2] + timeout-minutes: 50 services: pgsql: image: postgres:14 @@ -46,31 +43,47 @@ jobs: - "9000:9000" steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + - uses: ./.github/actions/go-setup + - uses: ./.github/actions/pgsql-shard with: - go-version-file: go.mod - check-latest: true - cache: false - - uses: ./.github/actions/go-cache - - name: Add hosts to /etc/hosts - run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 pgsql ldap minio" | sudo tee -a /etc/hosts' - - run: make deps-backend - - run: make backend + shard: 1 + total-shards: 2 + run-migration: "true" + + test-pgsql-shard-2: + if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' + needs: files-changed + runs-on: ubuntu-latest + timeout-minutes: 50 + services: + pgsql: + image: postgres:14 env: - TAGS: bindata - - name: run migration tests - if: matrix.shard == 1 - run: GITEA_TEST_DATABASE=pgsql make test-migration - - name: run tests - run: GITEA_TEST_DATABASE=pgsql make test-integration - timeout-minutes: 50 + POSTGRES_DB: test + POSTGRES_PASSWORD: postgres + ports: + - "5432:5432" + ldap: + image: gitea/test-openldap:latest@sha256:4ac633b01d684e6b2a458cc0c8530c92f9b3702f6e040ce5f365607df34fbda0 + ports: + - "389:389" + - "636:636" + minio: + # as github actions doesn't support "entrypoint", we need to use a non-official image + # that has a custom entrypoint set to "minio server /data" + image: bitnamilegacy/minio:2025.7.23 env: - # pgsql is chosen to be the unlucky one to run with the slow "race detector", it is about 60% slower. - GOTEST_FLAGS: -race -timeout=40m - TAGS: bindata gogit - TEST_LDAP: 1 - TEST_SHARD: ${{ matrix.shard }} - TEST_TOTAL_SHARDS: ${{ strategy.job-total }} + MINIO_ROOT_USER: 123456 + MINIO_ROOT_PASSWORD: 12345678 + ports: + - "9000:9000" + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: ./.github/actions/go-setup + - uses: ./.github/actions/pgsql-shard + with: + shard: 2 + total-shards: 2 test-sqlite: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' @@ -78,12 +91,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 - with: - go-version-file: go.mod - check-latest: true - cache: false - - uses: ./.github/actions/go-cache + - uses: ./.github/actions/go-setup - run: make deps-backend - run: make backend env: @@ -141,12 +149,7 @@ jobs: - 10000:10000 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 - with: - go-version-file: go.mod - check-latest: true - cache: false - - uses: ./.github/actions/go-cache + - uses: ./.github/actions/go-setup - name: Add hosts to /etc/hosts run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 minio devstoreaccount1.azurite.local mysql elasticsearch meilisearch smtpimap" | sudo tee -a /etc/hosts' - run: make deps-backend @@ -200,12 +203,7 @@ jobs: - "993:993" steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 - with: - go-version-file: go.mod - check-latest: true - cache: false - - uses: ./.github/actions/go-cache + - uses: ./.github/actions/go-setup - name: Add hosts to /etc/hosts run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 mysql elasticsearch smtpimap" | sudo tee -a /etc/hosts' - run: make deps-backend @@ -238,12 +236,7 @@ jobs: - 10000:10000 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 - with: - go-version-file: go.mod - check-latest: true - cache: false - - uses: ./.github/actions/go-cache + - uses: ./.github/actions/go-setup - name: Add hosts to /etc/hosts run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 mssql devstoreaccount1.azurite.local" | sudo tee -a /etc/hosts' - run: make deps-backend diff --git a/.github/workflows/pull-e2e-tests.yml b/.github/workflows/pull-e2e-tests.yml index 7bf73d4a0f9..f0283f40227 100644 --- a/.github/workflows/pull-e2e-tests.yml +++ b/.github/workflows/pull-e2e-tests.yml @@ -20,18 +20,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 - with: - go-version-file: go.mod - check-latest: true - cache: false - - uses: ./.github/actions/go-cache - - uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: 24 - cache: pnpm - cache-dependency-path: pnpm-lock.yaml + - uses: ./.github/actions/go-setup + - uses: ./.github/actions/node-setup - run: make deps-frontend - run: make frontend - run: make deps-backend