Files
gitea/modules/indexer/code/internal/util.go
T
Rafail Giavrimisandsilverwind 38389f00be feat: unify code search, add search API (#39403)
Adds GitHub-style code search endpoints and makes code search behave the
same in the web UI and the API, whether it runs on git grep, bleve or
Elasticsearch.

- `GET /repos/{owner}/{repo}/code/search` searches any branch, tag or
commit with git grep, so it works without the code indexer.
- `GET /search/code` searches the default branches of all accessible
repositories with the code indexer.
- One query syntax everywhere: every term must match a file's content or
path, case-insensitively. It supports `"phrases"`, `/regexp/` (git grep
only) and the `path:`, `language:`, `repo:`, `user:` and `org:`
qualifiers. Invalid queries return an error.
- All engines apply the same include, exclude, vendored and size rules,
and return every match. The web UI shows them as multiple snippets per
file with the matched text highlighted.

<img width="359" height="295" alt="image"
src="https://github.com/user-attachments/assets/59b7aa66-b1dc-4938-9ef9-66668ae08d7c"
/>

Fixes: https://github.com/go-gitea/gitea/issues/10706
Fixed: https://github.com/go-gitea/gitea/issues/15841
Fixes: https://github.com/go-gitea/gitea/issues/31375
Related: https://gitea.com/gitea/gitea-mcp/issues/280
Docs: https://gitea.com/gitea/docs/pulls/615

AI usage disclosure: Yes, I used AI to write code, review changes and
verify the factuality of the overview I provided. I take full
responsibility for the code in this PR.

---------

Co-authored-by: silverwind <me@silverwind.io>
2026-11-13 07:24:37 +00:00

187 lines
5.0 KiB
Go

// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package internal
import (
"cmp"
"io"
"regexp"
"slices"
"strings"
"unicode"
"unicode/utf8"
"gitea.dev/modules/analyze"
"gitea.dev/modules/charset"
"gitea.dev/modules/git"
"gitea.dev/modules/indexer"
"gitea.dev/modules/indexer/internal"
"gitea.dev/modules/log"
"gitea.dev/modules/setting"
"gitea.dev/modules/typesniffer"
"github.com/go-enry/go-enry/v2"
"golang.org/x/text/unicode/norm"
)
func FilenameIndexerID(repoID int64, filename string) string {
return internal.Base36(repoID) + "_" + filename
}
func ParseIndexerID(indexerID string) (int64, string) {
before, after, ok := strings.Cut(indexerID, "_")
if !ok {
log.Error("Unexpected ID in repo indexer: %s", indexerID)
}
repoID, _ := internal.ParseBase36(before)
return repoID, after
}
func MergeMatchRanges(ranges []MatchRange) []MatchRange {
slices.SortFunc(ranges, func(a, b MatchRange) int { return cmp.Compare(a.Start, b.Start) })
merged := ranges[:0]
for _, r := range ranges {
if n := len(merged); n > 0 && r.Start < merged[n-1].End {
merged[n-1].End = max(merged[n-1].End, r.End)
continue
}
merged = append(merged, r)
}
return merged
}
type FileMatcher []struct {
text string
pattern *regexp.Regexp
}
func NewFileMatcher(terms []indexer.SearchTerm) (FileMatcher, error) {
matcher := make(FileMatcher, len(terms))
for i, term := range terms {
matcher[i].text = strings.ToLower(term.Text)
if term.Kind == indexer.SearchTermRegexp {
pattern, err := regexp.Compile("(?i)" + term.Text)
if err != nil {
return nil, &indexer.SearchQueryError{Message: err.Error()}
}
matcher[i].pattern = pattern
}
}
return matcher, nil
}
func LiteralMatcher(terms []string) FileMatcher {
matcher := make(FileMatcher, len(terms))
for i, term := range terms {
matcher[i].text = strings.ToLower(term)
}
return matcher
}
func (m FileMatcher) MatchFile(path, content string) (ranges []MatchRange, ok bool) {
var lowerPath, lowerContent string
lowered := false
ok = len(m) > 0
var lowerRanges []MatchRange
for _, term := range m {
found := false
if pattern := term.pattern; pattern != nil {
offset, count := 0, 0
for line := range strings.Lines(content) {
line = strings.TrimSuffix(line, "\n")
for _, location := range pattern.FindAllStringIndex(line, 1000-count) {
found = true
count++
if location[0] < location[1] {
ranges = append(ranges, MatchRange{Start: offset + location[0], End: offset + location[1]})
}
}
offset += len(line) + 1
if count == 1000 {
break
}
}
found = found || pattern.MatchString(norm.NFC.String(path))
} else if term.text != "" {
if !lowered {
lowerPath, lowerContent = norm.NFC.String(strings.ToLower(path)), strings.ToLower(content)
lowered = true
}
for offset, count := 0, 0; count < 1000; count++ {
position := strings.Index(lowerContent[offset:], term.text)
if position < 0 {
break
}
found = true
offset += position + len(term.text)
lowerRanges = append(lowerRanges, MatchRange{Start: offset - len(term.text), End: offset})
}
found = found || strings.Contains(lowerPath, norm.NFC.String(term.text))
}
if !found {
return nil, false
}
}
lowerRanges = MergeMatchRanges(lowerRanges)
lowerOffset, index, originalStart := 0, 0, 0
for offset := 0; index < len(lowerRanges); {
if lowerOffset == lowerRanges[index].Start {
originalStart = offset
}
if lowerOffset == lowerRanges[index].End {
ranges = append(ranges, MatchRange{Start: originalStart, End: offset})
index++
continue
}
character, size := utf8.DecodeRuneInString(content[offset:])
lowerOffset += utf8.RuneLen(unicode.ToLower(character))
offset += size
}
return MergeMatchRanges(ranges), ok
}
func IsSearchable(path string, size int64) bool {
if size > setting.Indexer.MaxIndexerFileSize || setting.Indexer.ExcludeVendored && analyze.IsVendor(path) {
return false
}
name := strings.ToLower(path)
for _, pattern := range setting.Indexer.ExcludePatterns {
if pattern.Match(name) {
return false
}
}
for _, pattern := range setting.Indexer.IncludePatterns {
if pattern.Match(name) {
return true
}
}
return len(setting.Indexer.IncludePatterns) == 0
}
func ReadFile(batch git.CatFileBatch, update FileUpdate) ([]byte, bool, error) {
if !IsSearchable(update.Filename, update.Size) {
return nil, false, nil
}
info, reader, err := batch.QueryContent(update.BlobSha)
if err != nil {
return nil, false, err
}
content, err := io.ReadAll(io.LimitReader(reader, info.Size))
if err != nil {
return nil, false, err
}
if !typesniffer.DetectContentType(content).IsText() {
content = nil
}
_, err = reader.Discard(1)
return charset.ToUTF8DropErrors(content), true, err
}
func (m FileMatcher) MatchResult(result *SearchResult) bool {
var ok bool
result.ContentMatches, ok = m.MatchFile(result.Filename, result.Content)
result.Color = enry.GetColor(result.Language)
return ok
}