fix(indexer): search code as case-insensitive literal substrings (#40028)

Code search now matches case-insensitive literal substrings like GitHub,
in every script. Word tokenizers can't do this: the ES pattern dropped
every non-ASCII word, and the `standard` tokenizer keeps `console.log`
as one token.

- Elasticsearch indexes positional trigrams, plus 1-2 grams for short
terms
- Bleve indexes distinct 1-3 grams as a candidate filter and verifies
candidates against the content before paging, because phrase queries
over trigrams are quadratic in repeated grams
- Snippet offsets come from the original content, so `<em>` in sources
and width-changing lowercase no longer misplace them
- `search_mode=fuzzy` requests behave like Words, the UI never offered
fuzzy for code search
- The ES code indexer test now runs in CI, it was skipped before

There are no UI changes. Both code indexes rebuild after upgrade.

Fixes: https://github.com/go-gitea/gitea/issues/40027
Fixes: https://github.com/go-gitea/gitea/issues/16158
Fixes: https://github.com/go-gitea/gitea/issues/21359

---------

Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

modules/indexer/code/bleve/bleve.go

@@ -7,9 +7,11 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
+	"unicode/utf8"
 
 	repo_model "gitea.dev/models/repo"
 	"gitea.dev/modules/analyze"
@@ -68,7 +70,7 @@ const (
 	repoIndexerAnalyzer      = "repoIndexerAnalyzer"
 	filenameIndexerAnalyzer  = "filenameIndexerAnalyzer"
 	repoIndexerDocType       = "repoIndexerDocType"
-	repoIndexerLatestVersion = 11
+	repoIndexerLatestVersion = 12
 )
 
 // generateBleveIndexMapping generates a bleve index mapping for the repo indexer
@@ -80,6 +82,8 @@ func generateBleveIndexMapping() (mapping.IndexMapping, error) {
 
 	textFieldMapping := bleve.NewTextFieldMapping()
 	textFieldMapping.IncludeInAll = false
+	textFieldMapping.DocValues = false
+	textFieldMapping.IncludeTermVectors = false
 	docMapping.AddFieldMappingsAt("Content", textFieldMapping)
 
 	fileNamedMapping := bleve.NewTextFieldMapping()
@@ -105,7 +109,7 @@ func generateBleveIndexMapping() (mapping.IndexMapping, error) {
 		"type":          analyzer_custom.Name,
 		"char_filters":  []string{},
 		"tokenizer":     codeTokenizerName,
-		"token_filters": []string{unicodeNormalizeName, codeTokenFilterName, lowercase.Name},
+		"token_filters": []string{},
 	}); err != nil {
 		return nil, err
 	}
@@ -141,7 +145,6 @@ func (b *Indexer) SupportedSearchModes() []indexer.SearchMode {
 func init() {
 	// due to bleve's design problem, the "Register" must be done in the main goroutine, otherwise data-race
 	util.MustNoError(registry.RegisterTokenizer(codeTokenizerName, codeTokenizerConstructor))
-	util.MustNoError(registry.RegisterTokenFilter(codeTokenFilterName, codeTokenFilterConstructor))
 	util.MustNoError(registry.RegisterTokenizer(pathTokenizerName, pathTokenizerConstructor))
 }
 
@@ -266,25 +269,15 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int
 	pathQuery.FieldVal = "Filename"
 	pathQuery.SetBoost(10)
 
-	searchMode := util.IfZero(opts.SearchMode, b.SupportedSearchModes()[0].ModeValue)
-	if searchMode == indexer.SearchModeExact {
-		// 1.21 used NewPrefixQuery, but it seems not working well, and later releases changed to NewMatchPhraseQuery
-		q := bleve.NewMatchPhraseQuery(opts.Keyword)
-		q.Analyzer = repoIndexerAnalyzer
-		q.FieldVal = "Content"
-		contentQuery = q
-	} else /* words */ {
-		q := bleve.NewMatchQuery(opts.Keyword)
-		q.FieldVal = "Content"
-		q.Analyzer = repoIndexerAnalyzer
-		if searchMode == indexer.SearchModeFuzzy {
-			// this logic doesn't seem right, it is only used to pass the test-case `Keyword:    "dESCRIPTION"`, which doesn't seem to be a real-life use-case.
-			q.Fuzziness = inner_bleve.GuessFuzzinessByKeyword(opts.Keyword)
-		} else {
-			q.Operator = query.MatchQueryOperatorAnd
-		}
-		contentQuery = q
+	terms := internal.SearchTerms(opts)
+	if len(terms) == 0 {
+		return 0, nil, nil, nil
 	}
+	contentQueries := make([]query.Query, 0, len(terms))
+	for _, term := range terms {
+		contentQueries = append(contentQueries, inner_bleve.MatchAndQuery(term, "Content", repoIndexerAnalyzer, 0))
+	}
+	contentQuery = bleve.NewConjunctionQuery(contentQueries...)
 
 	keywordQuery = bleve.NewDisjunctionQuery(contentQuery, pathQuery)
 
@@ -316,67 +309,79 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int
 	}
 
 	from, pageSize := opts.GetSkipTake()
-	searchRequest := bleve.NewSearchRequestOptions(indexerQuery, pageSize, from, false)
-	searchRequest.Fields = []string{"Content", "Filename", "RepoID", "Language", "CommitID", "UpdatedAt"}
-	searchRequest.IncludeLocations = true
-
-	if len(opts.Language) == 0 {
-		searchRequest.AddFacet("languages", bleve.NewFacetRequest("Language", 10))
+	// grams prove terms of up to 3 runes, longer ones are verified against the content, so fetch candidates until the page is full
+	matched, scanned := 0, 0
+	if !slices.ContainsFunc(terms, func(term string) bool { return utf8.RuneCountInString(term) > 3 }) {
+		matched, scanned = from, from
 	}
+	var result *bleve.SearchResult
+	var total int64
+	searchResults := make([]*internal.SearchResult, 0, pageSize)
+	for {
+		size := from + pageSize - matched
+		searchRequest := bleve.NewSearchRequestOptions(indexerQuery, size, scanned, false)
+		searchRequest.Fields = []string{"Content", "Filename", "RepoID", "Language", "CommitID", "UpdatedAt"}
+		searchRequest.IncludeLocations = true
+		if result == nil && len(opts.Language) == 0 {
+			searchRequest.AddFacet("languages", bleve.NewFacetRequest("Language", 10))
+		}
+		searchRequest.SortBy([]string{"-_score", "UpdatedAt"})
 
-	searchRequest.SortBy([]string{"-_score", "UpdatedAt"})
+		batch, err := b.inner.Indexer.SearchInContext(ctx, searchRequest)
+		if err != nil {
+			return 0, nil, nil, err
+		}
+		if result == nil {
+			result = batch
+		}
 
-	result, err := b.inner.Indexer.SearchInContext(ctx, searchRequest)
-	if err != nil {
-		return 0, nil, nil, err
+		for _, hit := range batch.Hits {
+			scanned++
+			content, okContent := hit.Fields["Content"].(string)
+			language, okLanguage := hit.Fields["Language"].(string)
+			commitID, okCommitID := hit.Fields["CommitID"].(string)
+			updatedAt, okUpdatedAt := hit.Fields["UpdatedAt"].(string)
+			repoID, okRepoID := hit.Fields["RepoID"].(float64)
+			if !okContent || !okLanguage || !okCommitID || !okUpdatedAt || !okRepoID {
+				hitFieldsJson, _ := json.Marshal(hit.Fields)
+				setting.PanicInDevOrTesting("unexpected field types in search hit %q: %s", hit.ID, string(hitFieldsJson))
+				return 0, nil, nil, fmt.Errorf("unexpected field types in search hit %q", hit.ID)
+			}
+
+			var startIndex, endIndex int
+			if len(hit.Locations["Filename"]) > 0 {
+				startIndex, endIndex = internal.FilenameMatchIndexPos(content)
+			} else if startIndex, endIndex = internal.ContentMatchIndexPos(content, terms); startIndex < 0 {
+				continue
+			}
+			matched++
+			if matched <= from {
+				continue
+			}
+
+			var updatedUnix timeutil.TimeStamp
+			if t, err := time.Parse(time.RFC3339, updatedAt); err == nil {
+				updatedUnix = timeutil.TimeStamp(t.Unix())
+			}
+			searchResults = append(searchResults, &internal.SearchResult{
+				RepoID:      int64(repoID),
+				StartIndex:  startIndex,
+				EndIndex:    endIndex,
+				Filename:    internal.FilenameOfIndexerID(hit.ID),
+				Content:     content,
+				CommitID:    commitID,
+				UpdatedUnix: updatedUnix,
+				Language:    language,
+				Color:       enry.GetColor(language),
+			})
+		}
+		if matched == from+pageSize || len(batch.Hits) < size {
+			total = max(int64(matched), int64(matched)+int64(batch.Total)-int64(scanned))
+			break
+		}
 	}
-
-	total := int64(result.Total)
-
-	searchResults := make([]*internal.SearchResult, len(result.Hits))
-	for i, hit := range result.Hits {
-		content, okContent := hit.Fields["Content"].(string)
-		language, okLanguage := hit.Fields["Language"].(string)
-		commitID, okCommitID := hit.Fields["CommitID"].(string)
-		updatedAt, okUpdatedAt := hit.Fields["UpdatedAt"].(string)
-		repoID, okRepoID := hit.Fields["RepoID"].(float64)
-		if !okContent || !okLanguage || !okCommitID || !okUpdatedAt || !okRepoID {
-			hitFieldsJson, _ := json.Marshal(hit.Fields)
-			setting.PanicInDevOrTesting("unexpected field types in search hit %q: %s", hit.ID, string(hitFieldsJson))
-			return 0, nil, nil, fmt.Errorf("unexpected field types in search hit %q", hit.ID)
-		}
-
-		startIndex, endIndex := -1, -1
-		for _, locations := range hit.Locations["Content"] {
-			location := locations[0]
-			locationStart := int(location.Start)
-			locationEnd := int(location.End)
-			if startIndex < 0 || locationStart < startIndex {
-				startIndex = locationStart
-			}
-			if endIndex < 0 || locationEnd > endIndex {
-				endIndex = locationEnd
-			}
-		}
-		if len(hit.Locations["Filename"]) > 0 {
-			startIndex, endIndex = internal.FilenameMatchIndexPos(content)
-		}
-
-		var updatedUnix timeutil.TimeStamp
-		if t, err := time.Parse(time.RFC3339, updatedAt); err == nil {
-			updatedUnix = timeutil.TimeStamp(t.Unix())
-		}
-		searchResults[i] = &internal.SearchResult{
-			RepoID:      int64(repoID),
-			StartIndex:  startIndex,
-			EndIndex:    endIndex,
-			Filename:    internal.FilenameOfIndexerID(hit.ID),
-			Content:     content,
-			CommitID:    commitID,
-			UpdatedUnix: updatedUnix,
-			Language:    language,
-			Color:       enry.GetColor(language),
-		}
+	if total == 0 {
+		return 0, nil, nil, nil
 	}
 
 	searchResultLanguages := make([]*internal.SearchResultLanguages, 0, 10)
@@ -384,9 +389,9 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int
 		// Use separate query to go get all language counts
 		facetRequest := bleve.NewSearchRequestOptions(facetQuery, 1, 0, false)
 		facetRequest.Fields = []string{"Content", "RepoID", "Language", "CommitID", "UpdatedAt"}
-		facetRequest.IncludeLocations = true
 		facetRequest.AddFacet("languages", bleve.NewFacetRequest("Language", 10))
 
+		var err error
 		if result, err = b.inner.Indexer.Search(facetRequest); err != nil {
 			return 0, nil, nil, err
 		}

modules/indexer/code/bleve/bleve_test.go

@@ -4,12 +4,15 @@
 package bleve
 
 import (
+	"strconv"
 	"testing"
 	"time"
 
 	"gitea.dev/models/db"
+	indexer_module "gitea.dev/modules/indexer"
 	"gitea.dev/modules/indexer/code/internal"
 	inner_bleve "gitea.dev/modules/indexer/internal/bleve"
+	"gitea.dev/modules/util"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -32,7 +35,7 @@ func TestBleveIndexerTokenFilter(t *testing.T) {
 		expectedIDs []int64
 	}{
 		{keyword: "12345", expectedIDs: []int64{2}},
-		{keyword: "DNS", expectedIDs: []int64{}},
+		{keyword: "DNS", expectedIDs: []int64{2}},
 		{keyword: "mdns", expectedIDs: []int64{2}},
 		{keyword: "port", expectedIDs: []int64{2}},
 		{keyword: "port2", expectedIDs: []int64{2}},
@@ -54,6 +57,38 @@ func TestBleveIndexerTokenFilter(t *testing.T) {
 	}
 }
 
+func TestBleveIndexerPagesVerifiedCandidates(t *testing.T) {
+	indexer := NewIndexer(t.TempDir())
+	defer indexer.Close()
+	_, err := indexer.Init(t.Context())
+	require.NoError(t, err)
+
+	batch := inner_bleve.NewFlushingBatch(indexer.inner.Indexer, maxBatchSize)
+	for i, content := range []string{"abc bcd cde def", "abcdef", "abc bcd cde def", "abcdef", "abc bcd cde def"} {
+		filename := strconv.Itoa(i)
+		require.NoError(t, batch.Index(internal.FilenameIndexerID(1, filename), &RepoIndexerData{RepoID: 1, Filename: filename, Content: content, UpdatedAt: time.Now()}))
+	}
+	require.NoError(t, batch.Flush())
+
+	var filenames []string
+	for page := 1; page <= 3; page++ {
+		total, results, _, err := indexer.Search(t.Context(), &internal.SearchOptions{
+			Paginator:  &db.ListOptions{Page: page, PageSize: 1},
+			Keyword:    "abcdef",
+			SearchMode: indexer_module.SearchModeExact,
+		})
+		require.NoError(t, err)
+		require.Len(t, results, util.Iif(page < 3, 1, 0))
+		if page == 3 {
+			assert.EqualValues(t, 2, total)
+		}
+		for _, result := range results {
+			filenames = append(filenames, result.Filename)
+		}
+	}
+	assert.ElementsMatch(t, []string{"1", "3"}, filenames)
+}
+
 func searchResultIDs(result []*internal.SearchResult) []int64 {
 	ids := make([]int64, 0, len(result))
 	for _, hit := range result {

modules/indexer/code/bleve/token_code.go

@@ -4,55 +4,38 @@
 package bleve
 
 import (
-	"regexp"
-	"unicode"
+	"bytes"
+	"unicode/utf8"
 
 	"github.com/blevesearch/bleve/v2/analysis"
-	"github.com/blevesearch/bleve/v2/analysis/tokenizer/character"
 	"github.com/blevesearch/bleve/v2/registry"
 )
 
 const codeTokenizerName = "codeTokenizer"
 
-func codeTokenizerConstructor(_ map[string]any, _ *registry.Cache) (analysis.Tokenizer, error) {
-	// Old code used "letter" tokenizer which doesn't support CJK.
-	// Here it still doesn't support CJK, since there is no usable CJK tokenizer at the moment.
-	return character.NewCharacterTokenizer(func(r rune) bool {
-		return unicode.IsLetter(r) || unicode.IsNumber(r)
-	}), nil
-}
+// codeTokenizer emits every distinct lowercase gram of 1 to 3 runes once, search verifies longer terms against the content
+type codeTokenizer struct{}
 
-const codeTokenFilterName = "codeTokenFilter"
-
-type codeTokenFilter struct {
-	re *regexp.Regexp
-}
-
-func (c codeTokenFilter) Filter(stream analysis.TokenStream) (ret analysis.TokenStream) {
-	// split one token to "letter" parts and "number" parts (to keep the old behavior).
-	// e.g.: input token="port123", then the output tokens are "port123", "port", "123"
-	for _, token := range stream {
-		ret = append(ret, token)
-		m := c.re.FindAllIndex(token.Term, -1)
-		if len(m) > 1 {
-			for _, it := range m {
-				p1, p2 := it[0], it[1]
-				t := &analysis.Token{
-					Start:    token.Start + p1,
-					End:      token.Start + p2,
-					Term:     token.Term[p1:p2],
-					Position: token.Position,
-					Type:     analysis.AlphaNumeric,
-				}
-				ret = append(ret, t)
+func (codeTokenizer) Tokenize(input []byte) analysis.TokenStream {
+	lower := bytes.ToLower(input)
+	seen := make(map[string]bool)
+	var tokens analysis.TokenStream
+	var starts [3]int
+	for end, count := 0, 0; end < len(lower); {
+		starts[0], starts[1], starts[2] = starts[1], starts[2], end
+		count = min(count+1, len(starts))
+		_, size := utf8.DecodeRune(lower[end:])
+		end += size
+		for _, start := range starts[len(starts)-count:] {
+			if gram := lower[start:end]; !seen[string(gram)] {
+				seen[string(gram)] = true
+				tokens = append(tokens, &analysis.Token{Start: start, End: end, Term: gram, Position: len(tokens) + 1, Type: analysis.AlphaNumeric})
 			}
 		}
 	}
-	return ret
+	return tokens
 }
 
-func codeTokenFilterConstructor(_ map[string]any, _ *registry.Cache) (analysis.TokenFilter, error) {
-	return &codeTokenFilter{
-		re: regexp.MustCompile("[a-zA-Z]+|[0-9]+"),
-	}, nil
+func codeTokenizerConstructor(_config map[string]any, _cache *registry.Cache) (analysis.Tokenizer, error) {
+	return codeTokenizer{}, nil
 }