Files
gitea/modules/indexer/code/git.go
T
Rafail Giavrimisandsilverwind 38389f00be feat: unify code search, add search API (#39403)
Adds GitHub-style code search endpoints and makes code search behave the
same in the web UI and the API, whether it runs on git grep, bleve or
Elasticsearch.

- `GET /repos/{owner}/{repo}/code/search` searches any branch, tag or
commit with git grep, so it works without the code indexer.
- `GET /search/code` searches the default branches of all accessible
repositories with the code indexer.
- One query syntax everywhere: every term must match a file's content or
path, case-insensitively. It supports `"phrases"`, `/regexp/` (git grep
only) and the `path:`, `language:`, `repo:`, `user:` and `org:`
qualifiers. Invalid queries return an error.
- All engines apply the same include, exclude, vendored and size rules,
and return every match. The web UI shows them as multiple snippets per
file with the matched text highlighted.

<img width="359" height="295" alt="image"
src="https://github.com/user-attachments/assets/59b7aa66-b1dc-4938-9ef9-66668ae08d7c"
/>

Fixes: https://github.com/go-gitea/gitea/issues/10706
Fixed: https://github.com/go-gitea/gitea/issues/15841
Fixes: https://github.com/go-gitea/gitea/issues/31375
Related: https://gitea.com/gitea/gitea-mcp/issues/280
Docs: https://gitea.com/gitea/docs/pulls/615

AI usage disclosure: Yes, I used AI to write code, review changes and
verify the factuality of the overview I provided. I take full
responsibility for the code in this PR.

---------

Co-authored-by: silverwind <me@silverwind.io>
2026-11-13 07:24:37 +00:00

184 lines
5.7 KiB
Go

// Copyright 2019 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package code
import (
"context"
"strconv"
"strings"
repo_model "gitea.dev/models/repo"
"gitea.dev/modules/git"
"gitea.dev/modules/git/gitcmd"
"gitea.dev/modules/indexer/code/internal"
"gitea.dev/modules/log"
)
func getDefaultBranchSha(ctx context.Context, repo *repo_model.Repository) (string, error) {
stdout, _, err := gitcmd.NewCommand("show-ref", "-s").AddDynamicArguments(git.BranchPrefix + repo.DefaultBranch).WithRepo(repo).RunStdString(ctx)
if err != nil {
return "", err
}
return strings.TrimSpace(stdout), nil
}
// getRepoChanges returns changes to repo since last indexer update
func getRepoChanges(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, revision string) (*internal.RepoChanges, error) {
status, err := repo_model.GetIndexerStatus(ctx, repo, repo_model.RepoIndexerTypeCode)
if err != nil {
return nil, err
}
needGenesis := len(status.CommitSha) == 0
if !needGenesis {
hasAncestorCmd := gitcmd.NewCommand("merge-base").AddDynamicArguments(status.CommitSha, revision)
stdout, _, _ := hasAncestorCmd.WithRepo(repo).RunStdString(ctx) // FIXME: error is not handled
needGenesis = len(stdout) == 0
}
if needGenesis {
return genesisChanges(ctx, gitRepo, revision)
}
return nonGenesisChanges(ctx, repo, gitRepo, revision)
}
// parseGitLsTreeOutput parses the output of a `git ls-tree -r --full-name` command
func parseGitLsTreeOutput(ctx context.Context, gitRepo *git.Repository, stdout []byte) ([]internal.FileUpdate, error) {
entries, err := git.ParseTreeEntries(stdout)
if err != nil {
return nil, err
}
idxCount := 0
updates := make([]internal.FileUpdate, len(entries))
for _, entry := range entries {
if (entry.IsRegular() || entry.IsExecutable()) && internal.IsSearchable(entry.Name(), 0) {
updates[idxCount] = internal.FileUpdate{
Filename: entry.Name(),
BlobSha: entry.ID.String(),
Size: entry.GetSize(ctx, gitRepo),
}
idxCount++
}
}
return updates[:idxCount], nil
}
func ListFiles(ctx context.Context, gitRepo *git.Repository, revision string, paths ...string) ([]internal.FileUpdate, error) {
stdout, _, err := gitcmd.NewCommand("ls-tree", "--full-tree", "-l", "-r").AddDynamicArguments(revision).AddDashesAndList(paths...).WithRepo(gitRepo).RunStdBytes(ctx)
if err != nil {
return nil, err
}
return parseGitLsTreeOutput(ctx, gitRepo, stdout)
}
// genesisChanges get changes to add repo to the indexer for the first time
func genesisChanges(ctx context.Context, gitRepo *git.Repository, revision string) (*internal.RepoChanges, error) {
var changes internal.RepoChanges
var err error
changes.Updates, err = ListFiles(ctx, gitRepo, revision)
return &changes, err
}
// nonGenesisChanges get changes since the previous indexer update
func nonGenesisChanges(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, revision string) (*internal.RepoChanges, error) {
diffCmd := gitcmd.NewCommand("diff", "--name-status").AddDynamicArguments(repo.CodeIndexerStatus.CommitSha, revision)
stdout, _, runErr := diffCmd.WithRepo(repo).RunStdString(ctx)
if runErr != nil {
// previous commit sha may have been removed by a force push, so
// try rebuilding from scratch
log.Warn("git diff: %v", runErr)
if err := (*globalIndexer.Load()).Delete(ctx, repo.ID); err != nil {
return nil, err
}
return genesisChanges(ctx, gitRepo, revision)
}
var changes internal.RepoChanges
var err error
updatedFilenames := make([]string, 0, 10)
updateChanges := func() error {
cmd := gitcmd.NewCommand("ls-tree", "--full-tree", "-l").AddDynamicArguments(revision).
AddDashesAndList(updatedFilenames...)
lsTreeStdout, _, err := cmd.WithRepo(repo).RunStdBytes(ctx)
if err != nil {
return err
}
updates, err1 := parseGitLsTreeOutput(ctx, gitRepo, lsTreeStdout)
if err1 != nil {
return err1
}
changes.Updates = append(changes.Updates, updates...)
return nil
}
lines := strings.SplitSeq(stdout, "\n")
for line := range lines {
line = strings.TrimSpace(line)
if len(line) == 0 {
continue
}
fields := strings.Split(line, "\t")
if len(fields) < 2 {
log.Warn("Unparseable output for diff --name-status: `%s`)", line)
continue
}
filename := fields[1]
if len(filename) == 0 {
continue
} else if filename[0] == '"' {
filename, err = strconv.Unquote(filename)
if err != nil {
return nil, err
}
}
switch status := fields[0][0]; status {
case 'M', 'A':
updatedFilenames = append(updatedFilenames, filename)
case 'D':
changes.RemovedFilenames = append(changes.RemovedFilenames, filename)
case 'R', 'C':
if len(fields) < 3 {
log.Warn("Unparseable output for diff --name-status: `%s`)", line)
continue
}
dest := fields[2]
if len(dest) == 0 {
log.Warn("Unparseable output for diff --name-status: `%s`)", line)
continue
}
if dest[0] == '"' {
dest, err = strconv.Unquote(dest)
if err != nil {
return nil, err
}
}
if status == 'R' {
changes.RemovedFilenames = append(changes.RemovedFilenames, filename)
}
updatedFilenames = append(updatedFilenames, dest)
default:
log.Warn("Unrecognized status: %c (line=%s)", status, line)
}
// According to https://learn.microsoft.com/en-us/troubleshoot/windows-client/shell-experience/command-line-string-limitation#more-information
// the command line length should less than 8191 characters, assume filepath is 256, then 8191/256 = 31, so we use 30
if len(updatedFilenames) >= 30 {
if err := updateChanges(); err != nil {
return nil, err
}
updatedFilenames = updatedFilenames[0:0]
}
}
if len(updatedFilenames) > 0 {
if err := updateChanges(); err != nil {
return nil, err
}
}
return &changes, err
}