Files
gitea/modules/indexer/code/search.go
T
Rafail Giavrimisandsilverwind 38389f00be feat: unify code search, add search API (#39403)
Adds GitHub-style code search endpoints and makes code search behave the
same in the web UI and the API, whether it runs on git grep, bleve or
Elasticsearch.

- `GET /repos/{owner}/{repo}/code/search` searches any branch, tag or
commit with git grep, so it works without the code indexer.
- `GET /search/code` searches the default branches of all accessible
repositories with the code indexer.
- One query syntax everywhere: every term must match a file's content or
path, case-insensitively. It supports `"phrases"`, `/regexp/` (git grep
only) and the `path:`, `language:`, `repo:`, `user:` and `org:`
qualifiers. Invalid queries return an error.
- All engines apply the same include, exclude, vendored and size rules,
and return every match. The web UI shows them as multiple snippets per
file with the matched text highlighted.

<img width="359" height="295" alt="image"
src="https://github.com/user-attachments/assets/59b7aa66-b1dc-4938-9ef9-66668ae08d7c"
/>

Fixes: https://github.com/go-gitea/gitea/issues/10706
Fixed: https://github.com/go-gitea/gitea/issues/15841
Fixes: https://github.com/go-gitea/gitea/issues/31375
Related: https://gitea.com/gitea/gitea-mcp/issues/280
Docs: https://gitea.com/gitea/docs/pulls/615

AI usage disclosure: Yes, I used AI to write code, review changes and
verify the factuality of the overview I provided. I take full
responsibility for the code in this PR.

---------

Co-authored-by: silverwind <me@silverwind.io>
2026-11-13 07:24:37 +00:00

167 lines
5.0 KiB
Go

// Copyright 2017 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package code
import (
"context"
"html"
"html/template"
"strings"
"unicode/utf8"
"gitea.dev/modules/highlight"
"gitea.dev/modules/indexer/code/internal"
"gitea.dev/modules/util"
)
type Result = internal.SearchResult
const (
MaxResultFragments = 10
MaxResultFragmentSize = 4 * 1024
)
type ResultFragment struct {
StartLine int
Content string // UTF-8 clipped at a rune boundary
Matches []MatchRange // byte ranges into Content
}
type ResultLine struct {
Num int
FormattedContent template.HTML
}
type SearchResultLanguages = internal.SearchResultLanguages
type MatchRange = internal.MatchRange
type SearchOptions = internal.SearchOptions
func HighlightSearchResultCode(filename, language string, lineNums []int, code string, matches ...MatchRange) []*ResultLine {
// we should highlight the whole code block first, otherwise it doesn't work well with multiple line highlighting
lexer := highlight.DetectChromaLexerByFileName(filename, language)
hl := markMatches(code, string(highlight.RenderCodeByLexer(lexer, code)), matches)
highlightedLines := highlight.UnsafeSplitHighlightedLines(template.HTML(hl))
// The lineNums outputted by render might not match the original lineNums, because "highlight" removes the last `\n`
lines := make([]*ResultLine, min(len(highlightedLines), len(lineNums)))
for i := range lines {
lines[i] = &ResultLine{
Num: lineNums[i],
FormattedContent: template.HTML(highlightedLines[i]),
}
}
return lines
}
// markMatches wraps the code byte ranges in <mark>, split at tags, skipping code bytes the lexer dropped
func markMatches(code, highlighted string, matches []MatchRange) string {
if len(matches) == 0 {
return highlighted
}
var marked strings.Builder
marked.Grow(len(highlighted))
offset, open := 0, false
for highlighted != "" {
var token string
switch highlighted[0] {
case '<':
token = highlighted[:strings.IndexByte(highlighted, '>')+1]
highlighted = highlighted[len(token):]
if open {
marked.WriteString("</mark>")
open = false
}
marked.WriteString(token)
continue
case '&':
token = highlighted[:strings.IndexByte(highlighted, ';')+1]
default:
token = highlighted[:1]
}
highlighted = highlighted[len(token):]
text := html.UnescapeString(token)
for offset < len(code) && !strings.HasPrefix(code[offset:], text) {
offset++
}
for len(matches) > 0 && matches[0].End <= offset {
matches = matches[1:]
}
if inMatch := len(matches) > 0 && matches[0].Start <= offset; inMatch != open {
marked.WriteString(util.Iif(inMatch, "<mark>", "</mark>"))
open = inMatch
}
marked.WriteString(token)
offset += len(text)
}
if open {
marked.WriteString("</mark>")
}
return marked.String()
}
// PerformSearch perform a search on a repository
func PerformSearch(ctx context.Context, opts *SearchOptions) (int64, []*Result, []*SearchResultLanguages, error) {
if opts == nil || len(opts.Terms) == 0 {
return 0, nil, nil, nil
}
return (*globalIndexer.Load()).Search(ctx, opts)
}
func BuildResultFragments(content string, matches []MatchRange) []*ResultFragment {
if len(matches) == 0 {
return nil
}
var blocks []MatchRange
for _, match := range matches {
lineStart := strings.LastIndexByte(content[:match.Start], '\n') + 1
start := strings.LastIndexByte(content[:max(0, lineStart-1)], '\n') + 1
_, matchRuneSize := utf8.DecodeRuneInString(content[match.Start:match.End])
if match.Start+matchRuneSize-start > MaxResultFragmentSize {
start = lineStart
if match.Start+matchRuneSize-start > MaxResultFragmentSize {
start = match.Start
}
}
end := match.End - 1
limit := min(len(content), end+MaxResultFragmentSize+utf8.UTFMax) // fragments are truncated to MaxResultFragmentSize anyway
for range 2 {
newline := strings.IndexByte(content[end:limit], '\n')
if newline < 0 {
end = limit
break
}
end += newline + 1
}
if len(blocks) > 0 && start <= blocks[len(blocks)-1].End {
previous := &blocks[len(blocks)-1]
mergedEnd := previous.Start + len(util.TruncateStringBytes(content[previous.Start:max(previous.End, end)], MaxResultFragmentSize))
if match.Start+matchRuneSize <= mergedEnd {
previous.End = mergedEnd
continue
}
start = previous.End
}
if len(blocks) == MaxResultFragments {
break
}
blocks = append(blocks, MatchRange{Start: start, End: start + len(util.TruncateStringBytes(content[start:end], MaxResultFragmentSize))})
}
var fragments []*ResultFragment
for _, block := range blocks {
fragment := &ResultFragment{StartLine: 1 + strings.Count(content[:block.Start], "\n"), Content: content[block.Start:block.End]}
for _, match := range matches {
if match.Start < block.End && match.End > block.Start {
fragment.Matches = append(fragment.Matches, MatchRange{
Start: max(match.Start, block.Start) - block.Start,
End: min(match.End, block.End) - block.Start,
})
}
}
fragments = append(fragments, fragment)
}
return fragments
}