test: run integration tests on random ports and speed up slow tests (#39842)

Integration test processes now get their own work directory, config file
and random ports, so several runs can execute in parallel on one
machine. The slowest tests also lose fixed sleeps and needlessly
expensive setup, with their assertions kept.

- Batch PostgreSQL sequence resets, the main per-test fixture cost there
- Cheap hashing params in hash and OAuth2 fixture tests, full-cost
vectors stay
- Single fetch for negative Actions runner assertions instead of 1s
polls
- Queue flushes and backdated timestamps instead of sleeps
- In-memory SQLite test journal, tmpfs for PostgreSQL in CI
- E2E instance writes SSH keys into its work directory instead of
`~/.ssh/authorized_keys`

tests/integration/actions_concurrency_test.go

@@ -1555,9 +1555,11 @@ jobs:
 		// wf1-job2 is waiting but no runner will run it
 		assert.Equal(t, actions_model.StatusWaiting, w1j2Job.Status)
 
-		time.Sleep(time.Second)
 		now := time.Now()
-		time.Sleep(time.Second)
+		_, err := db.GetEngine(t.Context()).NoAutoTime().ID(w1j2Job.ID).Cols("updated").Update(&actions_model.ActionRunJob{
+			Updated: timeutil.TimeStamp(now.Unix()).Add(-1),
+		})
+		require.NoError(t, err)
 
 		// push workflow2
 		opts2 := getWorkflowCreateFileOptions(user2, repo.DefaultBranch, "create %s"+wf2TreePath, wf2FileContent)

tests/integration/actions_dynamic_matrix_test.go

@@ -119,7 +119,7 @@ jobs:
 				}
 				runner.execTask(t, task, outcome)
 			}
-			runner.fetchNoTask(t, 300*time.Millisecond)
+			runner.fetchNoTask(t)
 			return names
 		}
 

tests/integration/actions_job_test.go

@@ -370,7 +370,7 @@ func TestRunnerDisableEnable(t *testing.T) {
 			MakeRequest(t, req, http.StatusOK)
 
 			testData.runner.execTask(t, task1, &mockTaskOutcome{result: runnerv1.Result_RESULT_SUCCESS})
-			testData.runner.fetchNoTask(t, 2*time.Second)
+			testData.runner.fetchNoTask(t)
 
 			req = newRunnerUpdateRequest(t, fmt.Sprintf("/api/v1/repos/%s/%s/actions/runners/%d", user2.Name, testData.repo.Name, testData.runnerID), false).AddTokenAuth(token)
 			MakeRequest(t, req, http.StatusOK)
@@ -398,7 +398,6 @@ func TestRunnerDisableEnable(t *testing.T) {
 
 			// Trigger a second run so there is a pending job after we re-enable the runner
 			triggerRunnerDisableEnableRun(t, user2, token, testData.repo, "second-push.txt")
-			time.Sleep(500 * time.Millisecond) // allow workflow run to be created
 
 			req := newRunnerUpdateRequest(t, fmt.Sprintf("/api/v1/repos/%s/%s/actions/runners/%d", user2.Name, testData.repo.Name, testData.runnerID), true).AddTokenAuth(token)
 			MakeRequest(t, req, http.StatusOK)

tests/integration/actions_runner_test.go

@@ -98,8 +98,8 @@ func (r *mockRunner) fetchTask(t *testing.T, timeout ...time.Duration) *runnerv1
 	return task
 }
 
-func (r *mockRunner) fetchNoTask(t *testing.T, timeout ...time.Duration) {
-	task := r.tryFetchTask(t, timeout...)
+func (r *mockRunner) fetchNoTask(t *testing.T) {
+	task, _ := r.fetchTaskOnce(t, 0)
 	require.Nil(t, task, "a task is fetched")
 }
 

tests/integration/api_repo_languages_test.go

@@ -9,9 +9,11 @@ import (
 	"testing"
 	"time"
 
+	"gitea.dev/modules/queue"
 	"gitea.dev/modules/test"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestRepoLanguages(t *testing.T) {
@@ -36,8 +38,7 @@ func TestRepoLanguages(t *testing.T) {
 		resp = session.MakeRequest(t, req, http.StatusOK)
 		assert.NotEmpty(t, test.RedirectURL(resp))
 
-		// let gitea calculate language stats
-		time.Sleep(time.Second)
+		require.NoError(t, queue.GetManager().FlushAll(t.Context(), 10*time.Second))
 
 		// Save new file to master branch
 		req = NewRequest(t, "GET", "/api/v1/repos/user2/repo1/languages")

tests/integration/api_repo_license_test.go

@@ -11,10 +11,12 @@ import (
 	"time"
 
 	auth_model "gitea.dev/models/auth"
+	"gitea.dev/modules/queue"
 	api "gitea.dev/modules/structs"
 	"gitea.dev/modules/test"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 var testLicenseContent = `
@@ -51,8 +53,7 @@ func TestAPIRepoLicense(t *testing.T) {
 		resp = session.MakeRequest(t, req, http.StatusOK)
 		assert.NotEmpty(t, test.RedirectURL(resp))
 
-		// let gitea update repo license
-		time.Sleep(time.Second)
+		require.NoError(t, queue.GetManager().FlushAll(t.Context(), 10*time.Second))
 		checkRepoLicense(t, "user2", "repo1", []string{"BSD-2-Clause"})
 
 		// Change default branch
@@ -63,8 +64,7 @@ func TestAPIRepoLicense(t *testing.T) {
 		}).AddTokenAuth(token)
 		session.MakeRequest(t, req, http.StatusOK)
 
-		// let gitea update repo license
-		time.Sleep(time.Second)
+		require.NoError(t, queue.GetManager().FlushAll(t.Context(), 10*time.Second))
 		checkRepoLicense(t, "user2", "repo1", []string{"MIT"})
 	})
 }

tests/integration/git_general_test.go

@@ -33,6 +33,7 @@ import (
 	"gitea.dev/modules/setting"
 	api "gitea.dev/modules/structs"
 	"gitea.dev/modules/test"
+	"gitea.dev/services/automergequeue"
 	"gitea.dev/tests"
 
 	"github.com/kballard/go-shellquote"
@@ -817,8 +818,7 @@ func doAutoPRMerge(baseCtx *APITestContext, dstPath string) func(t *testing.T) {
 
 		// Call API to add Success status for commit
 		t.Run("CreateStatus", doAPICreateCommitStatusTest(ctx, commitID, commitstatus.CommitStatusSuccess, "testci"))
-		// wait to let gitea merge stuff
-		time.Sleep(time.Second)
+		require.NoError(t, automergequeue.AutoMergeQueue.FlushWithContext(t.Context(), 5*time.Second))
 
 		// test pr status
 		pr, err = doAPIGetPullRequest(ctx, baseCtx.Username, baseCtx.Reponame, pr.Index)(t)

tests/integration/git_push_test.go

@@ -31,7 +31,7 @@ func TestGitPush(t *testing.T) {
 func testGitPush(t *testing.T, u *url.URL) {
 	t.Run("Push branches at once", func(t *testing.T) {
 		runTestGitPush(t, u, func(t *testing.T, gitPath string) (pushed, deleted []string) {
-			for i := range 10 {
+			for i := range 4 {
 				branchName := fmt.Sprintf("branch-%d", i)
 				pushed = append(pushed, branchName)
 				doGitCreateBranch(gitPath, branchName)(t)
@@ -44,21 +44,19 @@ func testGitPush(t *testing.T, u *url.URL) {
 
 	t.Run("Push branches exists", func(t *testing.T) {
 		runTestGitPush(t, u, func(t *testing.T, gitPath string) (pushed, deleted []string) {
-			for i := range 10 {
+			for i := range 4 {
 				branchName := fmt.Sprintf("branch-%d", i)
-				if i < 5 {
+				if i < 2 {
 					pushed = append(pushed, branchName)
 				}
 				doGitCreateBranch(gitPath, branchName)(t)
 			}
-			// only push master and the first 5 branches
 			pushed = append(pushed, "master")
 			args := append([]string{"origin"}, pushed...)
 			doGitPushTestRepository(gitPath, args...)(t)
 
 			pushed = pushed[:0]
-			// do some changes for the first 5 branches created above
-			for i := range 5 {
+			for i := range 2 {
 				branchName := fmt.Sprintf("branch-%d", i)
 				pushed = append(pushed, branchName)
 				doGitCheckoutWriteFileCommit(localGitAddCommitOptions{
@@ -69,7 +67,7 @@ func testGitPush(t *testing.T, u *url.URL) {
 				})(t)
 			}
 
-			for i := 5; i < 10; i++ {
+			for i := 2; i < 4; i++ {
 				pushed = append(pushed, fmt.Sprintf("branch-%d", i))
 			}
 			pushed = append(pushed, "master")
@@ -83,7 +81,7 @@ func testGitPush(t *testing.T, u *url.URL) {
 
 	t.Run("Push branches one by one", func(t *testing.T) {
 		runTestGitPush(t, u, func(t *testing.T, gitPath string) (pushed, deleted []string) {
-			for i := range 10 {
+			for i := range 4 {
 				branchName := fmt.Sprintf("branch-%d", i)
 				doGitCreateBranch(gitPath, branchName)(t)
 				doGitPushTestRepository(gitPath, "origin", branchName)(t)
@@ -109,14 +107,14 @@ func testGitPush(t *testing.T, u *url.URL) {
 			doGitPushTestRepository(gitPath, "origin", "master")(t) // make sure master is the default branch instead of a branch we are going to delete
 			pushed = append(pushed, "master")
 
-			for i := range 10 {
+			for i := range 4 {
 				branchName := fmt.Sprintf("branch-%d", i)
 				pushed = append(pushed, branchName)
 				doGitCreateBranch(gitPath, branchName)(t)
 			}
 			doGitPushTestRepository(gitPath, "origin", "--all")(t)
 
-			for i := range 5 {
+			for i := range 2 {
 				branchName := fmt.Sprintf("branch-%d", i)
 				doGitPushTestRepository(gitPath, "origin", "--delete", branchName)(t)
 				deleted = append(deleted, branchName)

tests/integration/integration_test.go

@@ -48,7 +48,7 @@ func testMain(m *testing.M) int {
 	graceful.InitManager(managerCtx)
 	defer cancel()
 
-	err := tests.InitIntegrationTest()
+	workPath, err := tests.InitIntegrationTest()
 	if err != nil {
 		return testlogger.MainErrorf("InitTest error: %v", err)
 	}
@@ -67,11 +67,12 @@ func testMain(m *testing.M) int {
 	// Instead, "No tests were found",  last nonsense log is "According to the configuration, subsequent logs will not be printed to the console"
 	exitCode := m.Run()
 
-	if err = os.RemoveAll(setting.Indexer.IssuePath); err != nil {
-		log.Error("Failed to remove indexer path: %v", err)
-	}
-	if err = os.RemoveAll(setting.Indexer.RepoPath); err != nil {
-		log.Error("Failed to remove indexer path: %v", err)
+	if exitCode == 0 {
+		graceful.GetManager().DoGracefulShutdown() // let background tasks finish writing
+		<-graceful.GetManager().Done()
+		if err = os.RemoveAll(workPath); err != nil {
+			log.Error("Failed to remove work path: %v", err)
+		}
 	}
 	return exitCode
 }

tests/integration/issue_test.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"slices"
 	"strconv"
 	"strings"
 	"testing"
@@ -29,6 +30,7 @@ import (
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func getIssuesSelection(t testing.TB, htmlDoc *HTMLDoc) *goquery.Selection {
@@ -131,8 +133,11 @@ func TestViewIssuesKeyword(t *testing.T) {
 		Index:  1,
 	})
 	issues.UpdateIssueIndexer(t.Context(), issue.ID)
-	time.Sleep(time.Second * 1)
 	const keyword = "first"
+	require.Eventually(t, func() bool {
+		issueIDs, _, err := issues.SearchIssues(t.Context(), &issues.SearchOptions{Keyword: keyword, RepoIDs: []int64{repo.ID}})
+		return err == nil && slices.Contains(issueIDs, issue.ID)
+	}, 10*time.Second, 10*time.Millisecond)
 	req := NewRequestf(t, "GET", "%s/issues?q=%s", repo.Link(), keyword)
 	resp := MakeRequest(t, req, http.StatusOK)
 

tests/integration/migrate_test.go

@@ -67,7 +67,7 @@ func TestMigrateGiteaForm(t *testing.T) {
 		repoName := "repo1"
 		repoOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: ownerName})
 		session := loginUser(t, ownerName)
-		token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeWriteRepository, auth_model.AccessTokenScopeReadMisc)
+		token := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeWriteRepository, auth_model.AccessTokenScopeReadMisc, auth_model.AccessTokenScopeReadIssue)
 
 		// Step 0: verify the repo is available
 		req := NewRequestf(t, "GET", "/%s/%s", ownerName, repoName)
@@ -99,7 +99,7 @@ func TestMigrateGiteaForm(t *testing.T) {
 		loc := *test.ParseJSONRedirect(resp.Body.Bytes()).Redirect
 		assert.Equal(t, fmt.Sprintf("/%s/%s", ownerName, migratedRepoName), loc)
 		// Step 6: check the repo was created
-		unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{Name: migratedRepoName})
+		assert.Equal(t, repo_model.RepositoryReady, unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{Name: migratedRepoName}).Status)
 	})
 }
 

tests/integration/migration-test/migration_test.go

@@ -34,7 +34,13 @@ var currentEngine db.EngineMigration
 
 func initMigrationTest(t *testing.T) func() {
 	testlogger.Init()
-	require.NoError(t, setting.PrepareIntegrationTestConfig())
+	workPath, err := setting.PrepareIntegrationTestConfig()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		if !t.Failed() {
+			_ = os.RemoveAll(workPath)
+		}
+	})
 	setting.SetupGiteaTestEnv()
 
 	assert.NotEmpty(t, setting.RepoRootPath)

tests/integration/pull_merge_test.go

@@ -11,6 +11,7 @@ import (
 	"net/url"
 	"os"
 	"path"
+	"slices"
 	"strconv"
 	"strings"
 	"testing"
@@ -29,8 +30,9 @@ import (
 	"gitea.dev/modules/commitstatus"
 	"gitea.dev/modules/git"
 	"gitea.dev/modules/git/gitcmd"
+	issue_indexer "gitea.dev/modules/indexer/issues"
 	"gitea.dev/modules/json"
-	"gitea.dev/modules/queue"
+	"gitea.dev/modules/optional"
 	"gitea.dev/modules/setting"
 	api "gitea.dev/modules/structs"
 	"gitea.dev/modules/test"
@@ -746,9 +748,6 @@ func TestPullMergeIndexerNotifier(t *testing.T) {
 		testEditFile(t, session, "user1", "repo1", "master", "README.md", "Hello, World (Edited)\n")
 		createPullResp := testPullCreate(t, session, "user1", "repo1", false, "master", "master", "Indexer notifier test pull")
 
-		assert.NoError(t, queue.GetManager().FlushAll(t.Context(), 0))
-		time.Sleep(time.Second)
-
 		repo1 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{
 			OwnerName: "user2",
 			Name:      "repo1",
@@ -759,6 +758,13 @@ func TestPullMergeIndexerNotifier(t *testing.T) {
 			IsPull:   true,
 			IsClosed: false,
 		})
+		waitIndexed := func(isClosed bool) { // elasticsearch makes documents searchable asynchronously
+			require.Eventually(t, func() bool {
+				ids, _, err := issue_indexer.SearchIssues(t.Context(), &issue_indexer.SearchOptions{Keyword: "notifier", RepoIDs: []int64{repo1.ID}, IsClosed: optional.Some(isClosed)})
+				return err == nil && slices.Contains(ids, issue.ID)
+			}, 10*time.Second, 10*time.Millisecond)
+		}
+		waitIndexed(false)
 
 		// build the request for searching issues
 		link, _ := url.Parse("/api/v1/repos/issues/search")
@@ -786,9 +792,7 @@ func TestPullMergeIndexerNotifier(t *testing.T) {
 			ID: issue.ID,
 		})
 		assert.True(t, issue.IsClosed)
-
-		assert.NoError(t, queue.GetManager().FlushAll(t.Context(), 0))
-		time.Sleep(time.Second)
+		waitIndexed(true)
 
 		// search issues again
 		searchIssuesResp = session.MakeRequest(t, NewRequest(t, "GET", link.String()), http.StatusOK)
@@ -937,7 +941,7 @@ func TestPullAutoMergeAfterCommitStatusSucceedAndApproval(t *testing.T) {
 		})
 		assert.NoError(t, err)
 
-		time.Sleep(2 * time.Second)
+		require.NoError(t, automergequeue.AutoMergeQueue.FlushWithContext(t.Context(), 5*time.Second))
 
 		// reload pr again
 		pr = unittest.AssertExistsAndLoadBean(t, &issues_model.PullRequest{ID: pr.ID})

tests/integration/repo_webhook_test.go

@@ -1600,8 +1600,9 @@ jobs:
 		}
 	}
 
-	// Add this sleep to ensure the func can find the tasks by timestamp.
-	time.Sleep(time.Second)
+	_, err = db_model.GetEngine(ctx).Where("repo_id = ?", testRepo.ID).NoAutoTime().
+		SetExpr("updated", "updated - 2").Update(new(actions_model.ActionRunJob))
+	require.NoError(t, err)
 
 	err = actions.CancelAbandonedJobs(ctx)
 	assert.NoError(t, err)
@@ -1685,7 +1686,9 @@ jobs:
 
 	initialRunEventsLen := len(webhookData.payloads)
 
-	time.Sleep(2 * time.Second)
+	_, err := db_model.GetEngine(t.Context()).In("id", task1.Id, task2.Id).
+		SetExpr("started", "started - 3").Update(new(actions_model.ActionTask))
+	require.NoError(t, err)
 
 	require.NoError(t, actions.StopEndlessTasks(t.Context()))
 

tests/mssql.ini.tmpl

@@ -33,12 +33,12 @@ SIGNING_KEY = none
 
 [server]
 SSH_DOMAIN       = localhost
-HTTP_PORT        = 3003
-ROOT_URL         = http://localhost:3003/
-LOCAL_ROOT_URL   = http://127.0.0.1:3003/
+HTTP_PORT        = {{TEST_HTTP_PORT}}
+ROOT_URL         = http://localhost:{{TEST_HTTP_PORT}}/
+LOCAL_ROOT_URL   = http://127.0.0.1:{{TEST_HTTP_PORT}}/
 DISABLE_SSH      = false
 SSH_LISTEN_HOST  = localhost
-SSH_PORT         = 2201
+SSH_PORT         = {{TEST_SSH_PORT}}
 START_SSH_SERVER = true
 LFS_START_SERVER = true
 LFS_JWT_SECRET   = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w

tests/mysql.ini.tmpl

@@ -35,12 +35,12 @@ SIGNING_KEY = none
 
 [server]
 SSH_DOMAIN       = localhost
-HTTP_PORT        = 3001
-ROOT_URL         = http://localhost:3001/
-LOCAL_ROOT_URL   = http://127.0.0.1:3001/
+HTTP_PORT        = {{TEST_HTTP_PORT}}
+ROOT_URL         = http://localhost:{{TEST_HTTP_PORT}}/
+LOCAL_ROOT_URL   = http://127.0.0.1:{{TEST_HTTP_PORT}}/
 DISABLE_SSH      = false
 SSH_LISTEN_HOST  = localhost
-SSH_PORT         = 2201
+SSH_PORT         = {{TEST_SSH_PORT}}
 BUILTIN_SSH_SERVER_USER = git
 START_SSH_SERVER = true
 

tests/pgsql.ini.tmpl

@@ -34,12 +34,12 @@ SIGNING_KEY = none
 
 [server]
 SSH_DOMAIN       = localhost
-HTTP_PORT        = 3002
-ROOT_URL         = http://localhost:3002/
-LOCAL_ROOT_URL   = http://127.0.0.1:3002/
+HTTP_PORT        = {{TEST_HTTP_PORT}}
+ROOT_URL         = http://localhost:{{TEST_HTTP_PORT}}/
+LOCAL_ROOT_URL   = http://127.0.0.1:{{TEST_HTTP_PORT}}/
 DISABLE_SSH      = false
 SSH_LISTEN_HOST  = localhost
-SSH_PORT         = 2202
+SSH_PORT         = {{TEST_SSH_PORT}}
 START_SSH_SERVER = true
 LFS_START_SERVER = true
 LFS_JWT_SECRET   = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w

tests/sqlite.ini.tmpl

@@ -4,7 +4,8 @@ RUN_MODE = prod
 
 [database]
 DB_TYPE = sqlite3
-PATH    = gitea-test.db
+PATH    = {{TEST_WORK_PATH}}/gitea-test.db
+SQLITE_JOURNAL_MODE = MEMORY
 
 [indexer]
 REPO_INDEXER_ENABLED = true
@@ -29,12 +30,12 @@ SIGNING_KEY = none
 
 [server]
 SSH_DOMAIN       = localhost
-HTTP_PORT        = 3003
-ROOT_URL         = http://localhost:3003/
-LOCAL_ROOT_URL   = http://127.0.0.1:3003/
+HTTP_PORT        = {{TEST_HTTP_PORT}}
+ROOT_URL         = http://localhost:{{TEST_HTTP_PORT}}/
+LOCAL_ROOT_URL   = http://127.0.0.1:{{TEST_HTTP_PORT}}/
 DISABLE_SSH      = false
 SSH_LISTEN_HOST  = localhost
-SSH_PORT         = 2203
+SSH_PORT         = {{TEST_SSH_PORT}}
 START_SSH_SERVER = true
 LFS_START_SERVER = true
 LFS_JWT_SECRET   = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w

tests/test_utils.go

@@ -21,35 +21,35 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func InitIntegrationTest() error {
+func InitIntegrationTest() (workPath string, err error) {
 	testlogger.Init()
 
-	err := setting.PrepareIntegrationTestConfig()
+	workPath, err = setting.PrepareIntegrationTestConfig()
 	if err != nil {
-		return err
+		return "", err
 	}
 
 	setting.SetupGiteaTestEnv()
 	setting.Repository.DefaultBranch = "master" // many test code still assume that default branch is called "master"
 
 	if err := git.InitFull(); err != nil {
-		return err
+		return "", err
 	}
 
 	setting.Database.SlowQueryThreshold = 0
 	setting.LoadDBSetting()
 	cleanupDb, err := unittest.ResetTestDatabase()
 	if err != nil {
-		return err
+		return "", err
 	}
 	_ = cleanupDb // no clean up yet (not really needed at the moment)
 
 	if err := storage.Init(); err != nil {
-		return err
+		return "", err
 	}
 
 	routers.InitWebInstalled(graceful.GetManager().HammerContext())
-	return nil
+	return workPath, nil
 }
 
 func PrepareAttachmentsStorage(t testing.TB) {