public/gitea
1
0
Fork 0
mirror of https://github.com/go-gitea/gitea.git synced 2026-10-09 04:30:16 +09:00
Code Issues Packages Projects Releases Wiki Activity

fix(models): chunk commit status queries to avoid SQLite expression depth limit (#39611) (#39638)

Browse Source 
Backport #39611 by @Kshot3000

* Fixes #39606
* Fixes #39329

`GetLatestCommitStatusForRepoCommitIDs` built one `OR` condition per
commit SHA. Rendering a commit list of more than ~1000 commits (for
example a large pull request, via `processGitCommits`) therefore failed
on SQLite with `SQL logic error: Expression tree is too large (maximum
depth 1000)`.

The max-index lookup now queries SHAs in chunks of 500 with `IN`, and
the follow-up status fetch runs in chunks of 100 `(index, sha)` pairs,
so the expression depth stays bounded regardless of the commit count.
Results are unchanged.

Verified with a new regression test that calls the function with 2001
SHAs: it fails with the reported error on `main` and passes with this
change. `go test ./models/git/` passes.

—
Fix offered freely; tips welcome: PayPal kyleblake0659@gmail.com · BTC
3GnR7TWBXAB3pPztBWpNF4LMNEX5yX8vZK

Co-authored-by: KShot <kshot9000@gmail.com>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
This commit is contained in:
Giteabot
2026-10-06 07:13:54 -07:00
committed by GitHub
parent 69a2bfcfbd
commit d6945fb0b5
2 changed files with 95 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
models/git/commit_status.go
+33 -30
View File
@@ -9,6 +9,7 @@ import (
"errors"
"fmt"
"net/url"
"slices"
"strconv"
"strings"
"time"
@@ -35,11 +36,11 @@ import (
// CommitStatus holds a single Status of a single Commit
type CommitStatus struct {
ID int64 `xorm:"pk autoincr"`
RepoID int64 `xorm:"UNIQUE(repo_sha_index)"`
SHA string `xorm:"VARCHAR(64) NOT NULL INDEX UNIQUE(repo_sha_index)"`
Index int64 `xorm:"INDEX UNIQUE(repo_sha_index)"`
RepoID int64 `xorm:"INDEX UNIQUE(repo_sha_index)"`
Repo *repo_model.Repository `xorm:"-"`
State commitstatus.CommitStatusState `xorm:"VARCHAR(7) NOT NULL"`
SHA string `xorm:"VARCHAR(64) NOT NULL INDEX UNIQUE(repo_sha_index)"`
// TargetURL points to the commit status page reported by a CI system
// If Gitea Actions is used, it is a relative link like "{RepoLink}/actions/runs/{RunID}/jobs{JobID}"
@@ -420,46 +421,48 @@ func GetLatestCommitStatusForPairs(ctx context.Context, repoSHAs []RepoSHA) (map
return repoStatuses, nil
}
// GetLatestCommitStatusForRepoCommitIDs returns all statuses with a unique context for a given list of repo-sha pairs
func GetLatestCommitStatusForRepoCommitIDs(ctx context.Context, repoID int64, commitIDs []string) (map[string][]*CommitStatus, error) {
type result struct {
// GetLatestCommitStatusForRepoCommitIDs returns the commit statuses with a unique context for a given list of repo-sha pairs
// If the provided commit IDs are too many, only the first part and the last part of the commit IDs will be queried.
func GetLatestCommitStatusForRepoCommitIDs(ctx context.Context, repoID int64, allCommitIDs []string) (map[string][]*CommitStatus, error) {
const maxCommitIDs = 500
const maxBatchSize = 200
queryCommitIDs := allCommitIDs
if len(allCommitIDs) > maxCommitIDs {
// The commit IDs are usually from "commits list" or "compare" page (create a PR or compare commits), nobody can read so many commits at once.
// The commit IDs are usually sorted by time, so we can take the first half and the last half of the commit IDs to get the latest statuses.
log.Warn("GetLatestCommitStatusForRepoCommitIDs: too many commit IDs (%d) for repo %d, truncating to %d", len(allCommitIDs), repoID, maxCommitIDs)
queryCommitIDs = allCommitIDs[:maxCommitIDs/2]
queryCommitIDs = append(queryCommitIDs, allCommitIDs[len(allCommitIDs)-maxCommitIDs/2:]...)
}
baseSql := func() db.Session {
return db.GetEngine(ctx).Table(&CommitStatus{}).Where("repo_id = ?", repoID)
}
type shaMaxIndexResult struct {
Index int64
SHA string
}
getBase := func() db.Session {
return db.GetEngine(ctx).Table(&CommitStatus{}).Where("repo_id = ?", repoID)
}
results := make([]result, 0, len(commitIDs))
conds := make([]builder.Cond, 0, len(commitIDs))
for _, sha := range commitIDs {
conds = append(conds, builder.Eq{"sha": sha})
}
sess := getBase().And(builder.Or(conds...)).
Select("max( `index` ) as `index`, sha").
GroupBy("context_hash, sha").OrderBy("max( `index` ) desc")
err := sess.Find(&results)
shaMaxIndexResults := make([]*shaMaxIndexResult, 0, len(allCommitIDs))
err := baseSql().And(builder.In("sha", queryCommitIDs)).
Select("max(`index`) as `index`, sha").
GroupBy("context_hash, sha").
Find(&shaMaxIndexResults)
if err != nil {
return nil, err
}
repoStatuses := make(map[string][]*CommitStatus)
if len(results) > 0 {
statuses := make([]*CommitStatus, 0, len(results))
conds = make([]builder.Cond, 0, len(results))
for _, result := range results {
conds = append(conds, builder.Eq{"`index`": result.Index, "sha": result.SHA})
for chunk := range slices.Chunk(shaMaxIndexResults, maxBatchSize) {
statuses := make([]*CommitStatus, 0, len(chunk))
condIndexSha := make([]builder.Cond, 0, len(chunk))
for _, res := range chunk {
condIndexSha = append(condIndexSha, builder.Eq{"`index`": res.Index, "sha": res.SHA})
}
err = getBase().And(builder.Or(conds...)).Find(&statuses)
err = baseSql().And(builder.Or(condIndexSha...)).Find(&statuses)
if err != nil {
return nil, err
}
// Group the statuses by commit
for _, status := range statuses {
repoStatuses[status.SHA] = append(repoStatuses[status.SHA], status)
}
models/git/commit_status_test.go
+62 -46
View File
@@ -4,6 +4,7 @@
package git_test
import (
"fmt"
"testing"
"time"
@@ -22,33 +23,71 @@ func TestGetCommitStatuses(t *testing.T) {
assert.NoError(t, unittest.PrepareTestDatabase())
repo1 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1})
sha1 := "1234123412341234123412341234123412341234" // the mocked commit ID in test fixtures
statuses, maxResults, err := db.FindAndCount[git_model.CommitStatus](t.Context(), &git_model.CommitStatusOptions{
ListOptions: db.ListOptions{Page: 1, PageSize: 50},
RepoID: repo1.ID,
SHA: sha1,
})
assert.NoError(t, err)
assert.Equal(t, 5, int(maxResults))
var indexes []int64
for _, status := range statuses {
indexes = append(indexes, status.Index)
}
assert.Equal(t, []int64{5, 4, 3, 2, 1}, indexes)
assert.Equal(t, "deploy/awesomeness", statuses[0].Context)
assert.Equal(t, commitstatus.CommitStatusError, statuses[0].State)
assert.Equal(t, "https://try.gitea.io/api/v1/repos/user2/repo1/statuses/1234123412341234123412341234123412341234", statuses[0].APIURL(t.Context()))
t.Run("CommitStatusOptions", func(t *testing.T) {
statuses, maxResults, err := db.FindAndCount[git_model.CommitStatus](t.Context(), &git_model.CommitStatusOptions{
ListOptions: db.ListOptions{Page: 1, PageSize: 50},
RepoID: repo1.ID,
SHA: sha1,
})
assert.NoError(t, err)
assert.Equal(t, 5, int(maxResults))
var indexes []int64
for _, status := range statuses {
indexes = append(indexes, status.Index)
}
assert.Equal(t, []int64{5, 4, 3, 2, 1}, indexes)
assert.Equal(t, "deploy/awesomeness", statuses[0].Context)
assert.Equal(t, commitstatus.CommitStatusError, statuses[0].State)
assert.Equal(t, "https://try.gitea.io/api/v1/repos/user2/repo1/statuses/1234123412341234123412341234123412341234", statuses[0].APIURL(t.Context()))
statuses, maxResults, err = db.FindAndCount[git_model.CommitStatus](t.Context(), &git_model.CommitStatusOptions{
ListOptions: db.ListOptions{Page: 2, PageSize: 50},
RepoID: repo1.ID,
SHA: sha1,
statuses, maxResults, err = db.FindAndCount[git_model.CommitStatus](t.Context(), &git_model.CommitStatusOptions{
ListOptions: db.ListOptions{Page: 2, PageSize: 50},
RepoID: repo1.ID,
SHA: sha1,
})
assert.NoError(t, err)
assert.Equal(t, 5, int(maxResults))
assert.Empty(t, statuses)
})
t.Run("GetCountLatestCommitStatus", func(t *testing.T) {
commitStatuses, err := git_model.GetLatestCommitStatus(t.Context(), repo1.ID, sha1, db.ListOptions{
Page: 1,
PageSize: 2,
})
assert.NoError(t, err)
assert.Len(t, commitStatuses, 2)
assert.Equal(t, commitstatus.CommitStatusFailure, commitStatuses[0].State)
assert.Equal(t, "ci/awesomeness", commitStatuses[0].Context)
assert.Equal(t, commitstatus.CommitStatusError, commitStatuses[1].State)
assert.Equal(t, "deploy/awesomeness", commitStatuses[1].Context)
count, err := git_model.CountLatestCommitStatus(t.Context(), repo1.ID, sha1)
assert.NoError(t, err)
assert.EqualValues(t, 3, count)
})
t.Run("GetLatestCommitStatusForRepoCommitIDs", func(t *testing.T) {
commitIDs := []string{sha1}
// SQLite has a limit of 1000 for WHERE expression variables per query, use a larger slice to test
for i := range 2000 {
commitIDs = append(commitIDs, fmt.Sprintf("%040x", i+1))
}
statuses, err := git_model.GetLatestCommitStatusForRepoCommitIDs(t.Context(), repo1.ID, commitIDs)
assert.NoError(t, err)
assert.Len(t, statuses[sha1], 3)
latestIndexes := make(map[string]int64)
for _, status := range statuses[sha1] {
latestIndexes[status.Context] = status.Index
}
assert.Equal(t, map[string]int64{
"ci/awesomeness": 4,
"cov/awesomeness": 3,
"deploy/awesomeness": 5,
}, latestIndexes)
})
assert.NoError(t, err)
assert.Equal(t, 5, int(maxResults))
assert.Empty(t, statuses)
}
func Test_CalcCommitStatus(t *testing.T) {
@@ -247,26 +286,3 @@ func TestCommitStatusesApplyDoerPermission(t *testing.T) {
assert.Empty(t, statuses[1].TargetURL)
assert.Equal(t, "https://mycicd.org/1", statuses[2].TargetURL)
}
func TestGetCountLatestCommitStatus(t *testing.T) {
assert.NoError(t, unittest.PrepareTestDatabase())
repo1 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1})
sha1 := "1234123412341234123412341234123412341234" // the mocked commit ID in test fixtures
commitStatuses, err := git_model.GetLatestCommitStatus(t.Context(), repo1.ID, sha1, db.ListOptions{
Page: 1,
PageSize: 2,
})
assert.NoError(t, err)
assert.Len(t, commitStatuses, 2)
assert.Equal(t, commitstatus.CommitStatusFailure, commitStatuses[0].State)
assert.Equal(t, "ci/awesomeness", commitStatuses[0].Context)
assert.Equal(t, commitstatus.CommitStatusError, commitStatuses[1].State)
assert.Equal(t, "deploy/awesomeness", commitStatuses[1].Context)
count, err := git_model.CountLatestCommitStatus(t.Context(), repo1.ID, sha1)
assert.NoError(t, err)
assert.EqualValues(t, 3, count)
}