mirror of
https://github.com/go-gitea/gitea.git
synced 2026-11-19 15:14:40 +09:00
Adds GitHub-style code search endpoints and makes code search behave the
same in the web UI and the API, whether it runs on git grep, bleve or
Elasticsearch.
- `GET /repos/{owner}/{repo}/code/search` searches any branch, tag or
commit with git grep, so it works without the code indexer.
- `GET /search/code` searches the default branches of all accessible
repositories with the code indexer.
- One query syntax everywhere: every term must match a file's content or
path, case-insensitively. It supports `"phrases"`, `/regexp/` (git grep
only) and the `path:`, `language:`, `repo:`, `user:` and `org:`
qualifiers. Invalid queries return an error.
- All engines apply the same include, exclude, vendored and size rules,
and return every match. The web UI shows them as multiple snippets per
file with the matched text highlighted.
<img width="359" height="295" alt="image"
src="https://github.com/user-attachments/assets/59b7aa66-b1dc-4938-9ef9-66668ae08d7c"
/>
Fixes: https://github.com/go-gitea/gitea/issues/10706
Fixed: https://github.com/go-gitea/gitea/issues/15841
Fixes: https://github.com/go-gitea/gitea/issues/31375
Related: https://gitea.com/gitea/gitea-mcp/issues/280
Docs: https://gitea.com/gitea/docs/pulls/615
AI usage disclosure: Yes, I used AI to write code, review changes and
verify the factuality of the overview I provided. I take full
responsibility for the code in this PR.
---------
Co-authored-by: silverwind <me@silverwind.io>
187 lines
5.0 KiB
Go
187 lines
5.0 KiB
Go
// Copyright 2023 The Gitea Authors. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package internal
|
|
|
|
import (
|
|
"cmp"
|
|
"io"
|
|
"regexp"
|
|
"slices"
|
|
"strings"
|
|
"unicode"
|
|
"unicode/utf8"
|
|
|
|
"gitea.dev/modules/analyze"
|
|
"gitea.dev/modules/charset"
|
|
"gitea.dev/modules/git"
|
|
"gitea.dev/modules/indexer"
|
|
"gitea.dev/modules/indexer/internal"
|
|
"gitea.dev/modules/log"
|
|
"gitea.dev/modules/setting"
|
|
"gitea.dev/modules/typesniffer"
|
|
|
|
"github.com/go-enry/go-enry/v2"
|
|
"golang.org/x/text/unicode/norm"
|
|
)
|
|
|
|
func FilenameIndexerID(repoID int64, filename string) string {
|
|
return internal.Base36(repoID) + "_" + filename
|
|
}
|
|
|
|
func ParseIndexerID(indexerID string) (int64, string) {
|
|
before, after, ok := strings.Cut(indexerID, "_")
|
|
if !ok {
|
|
log.Error("Unexpected ID in repo indexer: %s", indexerID)
|
|
}
|
|
repoID, _ := internal.ParseBase36(before)
|
|
return repoID, after
|
|
}
|
|
|
|
func MergeMatchRanges(ranges []MatchRange) []MatchRange {
|
|
slices.SortFunc(ranges, func(a, b MatchRange) int { return cmp.Compare(a.Start, b.Start) })
|
|
merged := ranges[:0]
|
|
for _, r := range ranges {
|
|
if n := len(merged); n > 0 && r.Start < merged[n-1].End {
|
|
merged[n-1].End = max(merged[n-1].End, r.End)
|
|
continue
|
|
}
|
|
merged = append(merged, r)
|
|
}
|
|
return merged
|
|
}
|
|
|
|
type FileMatcher []struct {
|
|
text string
|
|
pattern *regexp.Regexp
|
|
}
|
|
|
|
func NewFileMatcher(terms []indexer.SearchTerm) (FileMatcher, error) {
|
|
matcher := make(FileMatcher, len(terms))
|
|
for i, term := range terms {
|
|
matcher[i].text = strings.ToLower(term.Text)
|
|
if term.Kind == indexer.SearchTermRegexp {
|
|
pattern, err := regexp.Compile("(?i)" + term.Text)
|
|
if err != nil {
|
|
return nil, &indexer.SearchQueryError{Message: err.Error()}
|
|
}
|
|
matcher[i].pattern = pattern
|
|
}
|
|
}
|
|
return matcher, nil
|
|
}
|
|
|
|
func LiteralMatcher(terms []string) FileMatcher {
|
|
matcher := make(FileMatcher, len(terms))
|
|
for i, term := range terms {
|
|
matcher[i].text = strings.ToLower(term)
|
|
}
|
|
return matcher
|
|
}
|
|
|
|
func (m FileMatcher) MatchFile(path, content string) (ranges []MatchRange, ok bool) {
|
|
var lowerPath, lowerContent string
|
|
lowered := false
|
|
ok = len(m) > 0
|
|
var lowerRanges []MatchRange
|
|
for _, term := range m {
|
|
found := false
|
|
if pattern := term.pattern; pattern != nil {
|
|
offset, count := 0, 0
|
|
for line := range strings.Lines(content) {
|
|
line = strings.TrimSuffix(line, "\n")
|
|
for _, location := range pattern.FindAllStringIndex(line, 1000-count) {
|
|
found = true
|
|
count++
|
|
if location[0] < location[1] {
|
|
ranges = append(ranges, MatchRange{Start: offset + location[0], End: offset + location[1]})
|
|
}
|
|
}
|
|
offset += len(line) + 1
|
|
if count == 1000 {
|
|
break
|
|
}
|
|
}
|
|
found = found || pattern.MatchString(norm.NFC.String(path))
|
|
} else if term.text != "" {
|
|
if !lowered {
|
|
lowerPath, lowerContent = norm.NFC.String(strings.ToLower(path)), strings.ToLower(content)
|
|
lowered = true
|
|
}
|
|
for offset, count := 0, 0; count < 1000; count++ {
|
|
position := strings.Index(lowerContent[offset:], term.text)
|
|
if position < 0 {
|
|
break
|
|
}
|
|
found = true
|
|
offset += position + len(term.text)
|
|
lowerRanges = append(lowerRanges, MatchRange{Start: offset - len(term.text), End: offset})
|
|
}
|
|
found = found || strings.Contains(lowerPath, norm.NFC.String(term.text))
|
|
}
|
|
if !found {
|
|
return nil, false
|
|
}
|
|
}
|
|
lowerRanges = MergeMatchRanges(lowerRanges)
|
|
lowerOffset, index, originalStart := 0, 0, 0
|
|
for offset := 0; index < len(lowerRanges); {
|
|
if lowerOffset == lowerRanges[index].Start {
|
|
originalStart = offset
|
|
}
|
|
if lowerOffset == lowerRanges[index].End {
|
|
ranges = append(ranges, MatchRange{Start: originalStart, End: offset})
|
|
index++
|
|
continue
|
|
}
|
|
character, size := utf8.DecodeRuneInString(content[offset:])
|
|
lowerOffset += utf8.RuneLen(unicode.ToLower(character))
|
|
offset += size
|
|
}
|
|
return MergeMatchRanges(ranges), ok
|
|
}
|
|
|
|
func IsSearchable(path string, size int64) bool {
|
|
if size > setting.Indexer.MaxIndexerFileSize || setting.Indexer.ExcludeVendored && analyze.IsVendor(path) {
|
|
return false
|
|
}
|
|
name := strings.ToLower(path)
|
|
for _, pattern := range setting.Indexer.ExcludePatterns {
|
|
if pattern.Match(name) {
|
|
return false
|
|
}
|
|
}
|
|
for _, pattern := range setting.Indexer.IncludePatterns {
|
|
if pattern.Match(name) {
|
|
return true
|
|
}
|
|
}
|
|
return len(setting.Indexer.IncludePatterns) == 0
|
|
}
|
|
|
|
func ReadFile(batch git.CatFileBatch, update FileUpdate) ([]byte, bool, error) {
|
|
if !IsSearchable(update.Filename, update.Size) {
|
|
return nil, false, nil
|
|
}
|
|
info, reader, err := batch.QueryContent(update.BlobSha)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
content, err := io.ReadAll(io.LimitReader(reader, info.Size))
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
if !typesniffer.DetectContentType(content).IsText() {
|
|
content = nil
|
|
}
|
|
_, err = reader.Discard(1)
|
|
return charset.ToUTF8DropErrors(content), true, err
|
|
}
|
|
|
|
func (m FileMatcher) MatchResult(result *SearchResult) bool {
|
|
var ok bool
|
|
result.ContentMatches, ok = m.MatchFile(result.Filename, result.Content)
|
|
result.Color = enry.GetColor(result.Language)
|
|
return ok
|
|
}
|