Use merge tree to detect conflicts when possible (#36400)

In Git 2.38, the `merge-tree` command introduced the `--write-tree`
option, which works directly on bare repositories. In Git 2.40, a new parameter `--merge-base` introduced so we require Git 2.40 to use the merge tree feature.

This option produces the merged tree object ID, allowing us to perform
diffs between commits without creating a temporary repository. By
avoiding the overhead of setting up and tearing down temporary repos,
this approach delivers a notable performance improvement.

It also fixes a possible situation that conflict files might be empty
but it's a conflict status according to
https://git-scm.com/docs/git-merge-tree#_mistakes_to_avoid

Replace #35542

---------

Signed-off-by: Lunny Xiao <xiaolunwen@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

modules/gitrepo/main_test.go

@@ -4,29 +4,21 @@
 package gitrepo
 
 import (
-	"os"
 	"path/filepath"
 	"testing"
 
-	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/setting"
-	"code.gitea.io/gitea/modules/tempdir"
+	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/test"
 )
 
 func TestMain(m *testing.M) {
-	gitHomePath, cleanup, err := tempdir.OsTempDir("gitea-test").MkdirTempRandom("git-home")
-	if err != nil {
-		log.Fatal("Unable to create temp dir: %v", err)
-	}
-	defer cleanup()
-
 	// resolve repository path relative to the test directory
 	testRootDir := test.SetupGiteaRoot()
 	repoPath = func(repo Repository) string {
-		return filepath.Join(testRootDir, "/modules/git/tests/repos", repo.RelativePath())
+		if filepath.IsAbs(repo.RelativePath()) {
+			return repo.RelativePath() // for testing purpose only
+		}
+		return filepath.Join(testRootDir, "modules/git/tests/repos", repo.RelativePath())
 	}
-
-	setting.Git.HomePath = gitHomePath
-	os.Exit(m.Run())
+	git.RunGitTests(m)
 }

modules/gitrepo/merge_tree.go

@@ -0,0 +1,59 @@
+// Copyright 2026 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package gitrepo
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+
+	"code.gitea.io/gitea/modules/git/gitcmd"
+	"code.gitea.io/gitea/modules/util"
+)
+
+const MaxConflictedDetectFiles = 10
+
+// MergeTree performs a merge between two commits (baseRef and headRef) with an optional merge base.
+// It returns the resulting tree hash, a list of conflicted files (if any), and an error if the operation fails.
+// If there are no conflicts, the list of conflicted files will be nil.
+func MergeTree(ctx context.Context, repo Repository, baseRef, headRef, mergeBase string) (treeID string, isErrHasConflicts bool, conflictFiles []string, _ error) {
+	cmd := gitcmd.NewCommand("merge-tree", "--write-tree", "-z", "--name-only", "--no-messages").
+		AddOptionFormat("--merge-base=%s", mergeBase).
+		AddDynamicArguments(baseRef, headRef)
+
+	stdout, stdoutClose := cmd.MakeStdoutPipe()
+	defer stdoutClose()
+	cmd.WithPipelineFunc(func(ctx gitcmd.Context) error {
+		// https://git-scm.com/docs/git-merge-tree/2.38.0#OUTPUT
+		// For a conflicted merge, the output is:
+		// <OID of toplevel tree>NUL
+		// <Conflicted file name 1>NUL
+		// <Conflicted file name 2>NUL
+		// ...
+		scanner := bufio.NewScanner(stdout)
+		scanner.Split(util.BufioScannerSplit(0))
+		for scanner.Scan() {
+			line := scanner.Text()
+			if treeID == "" { // first line is tree ID
+				treeID = line
+				continue
+			}
+			conflictFiles = append(conflictFiles, line)
+			if len(conflictFiles) >= MaxConflictedDetectFiles {
+				break
+			}
+		}
+		return scanner.Err()
+	})
+
+	err := RunCmdWithStderr(ctx, repo, cmd)
+	// For a successful, non-conflicted merge, the exit status is 0. When the merge has conflicts, the exit status is 1.
+	// A merge can have conflicts without having individual files conflict
+	// https://git-scm.com/docs/git-merge-tree/2.38.0#_mistakes_to_avoid
+	isErrHasConflicts = gitcmd.IsErrorExitCode(err, 1)
+	if err == nil || isErrHasConflicts {
+		return treeID, isErrHasConflicts, conflictFiles, nil
+	}
+	return "", false, nil, fmt.Errorf("run merge-tree failed: %w", err)
+}

modules/gitrepo/merge_tree_test.go

@@ -0,0 +1,82 @@
+// Copyright 2026 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package gitrepo
+
+import (
+	"path/filepath"
+	"testing"
+
+	"code.gitea.io/gitea/modules/git/gitcmd"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func prepareRepoDirRenameConflict(t *testing.T) string {
+	repoDir := filepath.Join(t.TempDir(), "repo-dir-rename-conflict.git")
+	require.NoError(t, gitcmd.NewCommand("init", "--bare").AddDynamicArguments(repoDir).Run(t.Context()))
+	stdin := `blob
+mark :1
+data 2
+b
+
+blob
+mark :2
+data 2
+c
+
+reset refs/heads/master
+commit refs/heads/master
+mark :3
+author test <test@example.com> 1769202331 -0800
+committer test <test@example.com> 1769202331 -0800
+data 2
+O
+M 100644 :1 z/b
+M 100644 :2 z/c
+
+commit refs/heads/split
+mark :4
+author test <test@example.com> 1769202336 -0800
+committer test <test@example.com> 1769202336 -0800
+data 2
+A
+from :3
+M 100644 :2 w/c
+M 100644 :1 y/b
+D z/b
+D z/c
+
+blob
+mark :5
+data 2
+d
+
+commit refs/heads/add
+mark :6
+author test <test@example.com> 1769202342 -0800
+committer test <test@example.com> 1769202342 -0800
+data 2
+B
+from :3
+M 100644 :5 z/d
+`
+	require.NoError(t, gitcmd.NewCommand("fast-import").WithDir(repoDir).WithStdinBytes([]byte(stdin)).Run(t.Context()))
+	return repoDir
+}
+
+func TestMergeTreeDirectoryRenameConflictWithoutFiles(t *testing.T) {
+	repoDir := prepareRepoDirRenameConflict(t)
+	require.DirExists(t, repoDir)
+	repo := &mockRepository{path: repoDir}
+
+	mergeBase, err := MergeBase(t.Context(), repo, "add", "split")
+	require.NoError(t, err)
+
+	treeID, conflicted, conflictedFiles, err := MergeTree(t.Context(), repo, "add", "split", mergeBase)
+	require.NoError(t, err)
+	assert.True(t, conflicted)
+	assert.Empty(t, conflictedFiles)
+	assert.Equal(t, "5e3dd4cfc5b11e278a35b2daa83b7274175e3ab1", treeID)
+}