fix(indexer): search code as case-insensitive literal substrings (#40028)

Code search now matches case-insensitive literal substrings like GitHub,
in every script. Word tokenizers can't do this: the ES pattern dropped
every non-ASCII word, and the `standard` tokenizer keeps `console.log`
as one token.

- Elasticsearch indexes positional trigrams, plus 1-2 grams for short
terms
- Bleve indexes distinct 1-3 grams as a candidate filter and verifies
candidates against the content before paging, because phrase queries
over trigrams are quadratic in repeated grams
- Snippet offsets come from the original content, so `<em>` in sources
and width-changing lowercase no longer misplace them
- `search_mode=fuzzy` requests behave like Words, the UI never offered
fuzzy for code search
- The ES code indexer test now runs in CI, it was skipped before

There are no UI changes. Both code indexes rebuild after upgrade.

Fixes: https://github.com/go-gitea/gitea/issues/40027
Fixes: https://github.com/go-gitea/gitea/issues/16158
Fixes: https://github.com/go-gitea/gitea/issues/21359

---------

Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

modules/indexer/code/bleve/bleve.go

@@ -7,9 +7,11 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
+	"unicode/utf8"
 
 	repo_model "gitea.dev/models/repo"
 	"gitea.dev/modules/analyze"
@@ -68,7 +70,7 @@ const (
 	repoIndexerAnalyzer      = "repoIndexerAnalyzer"
 	filenameIndexerAnalyzer  = "filenameIndexerAnalyzer"
 	repoIndexerDocType       = "repoIndexerDocType"
-	repoIndexerLatestVersion = 11
+	repoIndexerLatestVersion = 12
 )
 
 // generateBleveIndexMapping generates a bleve index mapping for the repo indexer
@@ -80,6 +82,8 @@ func generateBleveIndexMapping() (mapping.IndexMapping, error) {
 
 	textFieldMapping := bleve.NewTextFieldMapping()
 	textFieldMapping.IncludeInAll = false
+	textFieldMapping.DocValues = false
+	textFieldMapping.IncludeTermVectors = false
 	docMapping.AddFieldMappingsAt("Content", textFieldMapping)
 
 	fileNamedMapping := bleve.NewTextFieldMapping()
@@ -105,7 +109,7 @@ func generateBleveIndexMapping() (mapping.IndexMapping, error) {
 		"type":          analyzer_custom.Name,
 		"char_filters":  []string{},
 		"tokenizer":     codeTokenizerName,
-		"token_filters": []string{unicodeNormalizeName, codeTokenFilterName, lowercase.Name},
+		"token_filters": []string{},
 	}); err != nil {
 		return nil, err
 	}
@@ -141,7 +145,6 @@ func (b *Indexer) SupportedSearchModes() []indexer.SearchMode {
 func init() {
 	// due to bleve's design problem, the "Register" must be done in the main goroutine, otherwise data-race
 	util.MustNoError(registry.RegisterTokenizer(codeTokenizerName, codeTokenizerConstructor))
-	util.MustNoError(registry.RegisterTokenFilter(codeTokenFilterName, codeTokenFilterConstructor))
 	util.MustNoError(registry.RegisterTokenizer(pathTokenizerName, pathTokenizerConstructor))
 }
 
@@ -266,25 +269,15 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int
 	pathQuery.FieldVal = "Filename"
 	pathQuery.SetBoost(10)
 
-	searchMode := util.IfZero(opts.SearchMode, b.SupportedSearchModes()[0].ModeValue)
-	if searchMode == indexer.SearchModeExact {
-		// 1.21 used NewPrefixQuery, but it seems not working well, and later releases changed to NewMatchPhraseQuery
-		q := bleve.NewMatchPhraseQuery(opts.Keyword)
-		q.Analyzer = repoIndexerAnalyzer
-		q.FieldVal = "Content"
-		contentQuery = q
-	} else /* words */ {
-		q := bleve.NewMatchQuery(opts.Keyword)
-		q.FieldVal = "Content"
-		q.Analyzer = repoIndexerAnalyzer
-		if searchMode == indexer.SearchModeFuzzy {
-			// this logic doesn't seem right, it is only used to pass the test-case `Keyword:    "dESCRIPTION"`, which doesn't seem to be a real-life use-case.
-			q.Fuzziness = inner_bleve.GuessFuzzinessByKeyword(opts.Keyword)
-		} else {
-			q.Operator = query.MatchQueryOperatorAnd
-		}
-		contentQuery = q
+	terms := internal.SearchTerms(opts)
+	if len(terms) == 0 {
+		return 0, nil, nil, nil
 	}
+	contentQueries := make([]query.Query, 0, len(terms))
+	for _, term := range terms {
+		contentQueries = append(contentQueries, inner_bleve.MatchAndQuery(term, "Content", repoIndexerAnalyzer, 0))
+	}
+	contentQuery = bleve.NewConjunctionQuery(contentQueries...)
 
 	keywordQuery = bleve.NewDisjunctionQuery(contentQuery, pathQuery)
 
@@ -316,67 +309,79 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int
 	}
 
 	from, pageSize := opts.GetSkipTake()
-	searchRequest := bleve.NewSearchRequestOptions(indexerQuery, pageSize, from, false)
-	searchRequest.Fields = []string{"Content", "Filename", "RepoID", "Language", "CommitID", "UpdatedAt"}
-	searchRequest.IncludeLocations = true
-
-	if len(opts.Language) == 0 {
-		searchRequest.AddFacet("languages", bleve.NewFacetRequest("Language", 10))
+	// grams prove terms of up to 3 runes, longer ones are verified against the content, so fetch candidates until the page is full
+	matched, scanned := 0, 0
+	if !slices.ContainsFunc(terms, func(term string) bool { return utf8.RuneCountInString(term) > 3 }) {
+		matched, scanned = from, from
 	}
+	var result *bleve.SearchResult
+	var total int64
+	searchResults := make([]*internal.SearchResult, 0, pageSize)
+	for {
+		size := from + pageSize - matched
+		searchRequest := bleve.NewSearchRequestOptions(indexerQuery, size, scanned, false)
+		searchRequest.Fields = []string{"Content", "Filename", "RepoID", "Language", "CommitID", "UpdatedAt"}
+		searchRequest.IncludeLocations = true
+		if result == nil && len(opts.Language) == 0 {
+			searchRequest.AddFacet("languages", bleve.NewFacetRequest("Language", 10))
+		}
+		searchRequest.SortBy([]string{"-_score", "UpdatedAt"})
 
-	searchRequest.SortBy([]string{"-_score", "UpdatedAt"})
+		batch, err := b.inner.Indexer.SearchInContext(ctx, searchRequest)
+		if err != nil {
+			return 0, nil, nil, err
+		}
+		if result == nil {
+			result = batch
+		}
 
-	result, err := b.inner.Indexer.SearchInContext(ctx, searchRequest)
-	if err != nil {
-		return 0, nil, nil, err
+		for _, hit := range batch.Hits {
+			scanned++
+			content, okContent := hit.Fields["Content"].(string)
+			language, okLanguage := hit.Fields["Language"].(string)
+			commitID, okCommitID := hit.Fields["CommitID"].(string)
+			updatedAt, okUpdatedAt := hit.Fields["UpdatedAt"].(string)
+			repoID, okRepoID := hit.Fields["RepoID"].(float64)
+			if !okContent || !okLanguage || !okCommitID || !okUpdatedAt || !okRepoID {
+				hitFieldsJson, _ := json.Marshal(hit.Fields)
+				setting.PanicInDevOrTesting("unexpected field types in search hit %q: %s", hit.ID, string(hitFieldsJson))
+				return 0, nil, nil, fmt.Errorf("unexpected field types in search hit %q", hit.ID)
+			}
+
+			var startIndex, endIndex int
+			if len(hit.Locations["Filename"]) > 0 {
+				startIndex, endIndex = internal.FilenameMatchIndexPos(content)
+			} else if startIndex, endIndex = internal.ContentMatchIndexPos(content, terms); startIndex < 0 {
+				continue
+			}
+			matched++
+			if matched <= from {
+				continue
+			}
+
+			var updatedUnix timeutil.TimeStamp
+			if t, err := time.Parse(time.RFC3339, updatedAt); err == nil {
+				updatedUnix = timeutil.TimeStamp(t.Unix())
+			}
+			searchResults = append(searchResults, &internal.SearchResult{
+				RepoID:      int64(repoID),
+				StartIndex:  startIndex,
+				EndIndex:    endIndex,
+				Filename:    internal.FilenameOfIndexerID(hit.ID),
+				Content:     content,
+				CommitID:    commitID,
+				UpdatedUnix: updatedUnix,
+				Language:    language,
+				Color:       enry.GetColor(language),
+			})
+		}
+		if matched == from+pageSize || len(batch.Hits) < size {
+			total = max(int64(matched), int64(matched)+int64(batch.Total)-int64(scanned))
+			break
+		}
 	}
-
-	total := int64(result.Total)
-
-	searchResults := make([]*internal.SearchResult, len(result.Hits))
-	for i, hit := range result.Hits {
-		content, okContent := hit.Fields["Content"].(string)
-		language, okLanguage := hit.Fields["Language"].(string)
-		commitID, okCommitID := hit.Fields["CommitID"].(string)
-		updatedAt, okUpdatedAt := hit.Fields["UpdatedAt"].(string)
-		repoID, okRepoID := hit.Fields["RepoID"].(float64)
-		if !okContent || !okLanguage || !okCommitID || !okUpdatedAt || !okRepoID {
-			hitFieldsJson, _ := json.Marshal(hit.Fields)
-			setting.PanicInDevOrTesting("unexpected field types in search hit %q: %s", hit.ID, string(hitFieldsJson))
-			return 0, nil, nil, fmt.Errorf("unexpected field types in search hit %q", hit.ID)
-		}
-
-		startIndex, endIndex := -1, -1
-		for _, locations := range hit.Locations["Content"] {
-			location := locations[0]
-			locationStart := int(location.Start)
-			locationEnd := int(location.End)
-			if startIndex < 0 || locationStart < startIndex {
-				startIndex = locationStart
-			}
-			if endIndex < 0 || locationEnd > endIndex {
-				endIndex = locationEnd
-			}
-		}
-		if len(hit.Locations["Filename"]) > 0 {
-			startIndex, endIndex = internal.FilenameMatchIndexPos(content)
-		}
-
-		var updatedUnix timeutil.TimeStamp
-		if t, err := time.Parse(time.RFC3339, updatedAt); err == nil {
-			updatedUnix = timeutil.TimeStamp(t.Unix())
-		}
-		searchResults[i] = &internal.SearchResult{
-			RepoID:      int64(repoID),
-			StartIndex:  startIndex,
-			EndIndex:    endIndex,
-			Filename:    internal.FilenameOfIndexerID(hit.ID),
-			Content:     content,
-			CommitID:    commitID,
-			UpdatedUnix: updatedUnix,
-			Language:    language,
-			Color:       enry.GetColor(language),
-		}
+	if total == 0 {
+		return 0, nil, nil, nil
 	}
 
 	searchResultLanguages := make([]*internal.SearchResultLanguages, 0, 10)
@@ -384,9 +389,9 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int
 		// Use separate query to go get all language counts
 		facetRequest := bleve.NewSearchRequestOptions(facetQuery, 1, 0, false)
 		facetRequest.Fields = []string{"Content", "RepoID", "Language", "CommitID", "UpdatedAt"}
-		facetRequest.IncludeLocations = true
 		facetRequest.AddFacet("languages", bleve.NewFacetRequest("Language", 10))
 
+		var err error
 		if result, err = b.inner.Indexer.Search(facetRequest); err != nil {
 			return 0, nil, nil, err
 		}

modules/indexer/code/bleve/bleve_test.go

@@ -4,12 +4,15 @@
 package bleve
 
 import (
+	"strconv"
 	"testing"
 	"time"
 
 	"gitea.dev/models/db"
+	indexer_module "gitea.dev/modules/indexer"
 	"gitea.dev/modules/indexer/code/internal"
 	inner_bleve "gitea.dev/modules/indexer/internal/bleve"
+	"gitea.dev/modules/util"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -32,7 +35,7 @@ func TestBleveIndexerTokenFilter(t *testing.T) {
 		expectedIDs []int64
 	}{
 		{keyword: "12345", expectedIDs: []int64{2}},
-		{keyword: "DNS", expectedIDs: []int64{}},
+		{keyword: "DNS", expectedIDs: []int64{2}},
 		{keyword: "mdns", expectedIDs: []int64{2}},
 		{keyword: "port", expectedIDs: []int64{2}},
 		{keyword: "port2", expectedIDs: []int64{2}},
@@ -54,6 +57,38 @@ func TestBleveIndexerTokenFilter(t *testing.T) {
 	}
 }
 
+func TestBleveIndexerPagesVerifiedCandidates(t *testing.T) {
+	indexer := NewIndexer(t.TempDir())
+	defer indexer.Close()
+	_, err := indexer.Init(t.Context())
+	require.NoError(t, err)
+
+	batch := inner_bleve.NewFlushingBatch(indexer.inner.Indexer, maxBatchSize)
+	for i, content := range []string{"abc bcd cde def", "abcdef", "abc bcd cde def", "abcdef", "abc bcd cde def"} {
+		filename := strconv.Itoa(i)
+		require.NoError(t, batch.Index(internal.FilenameIndexerID(1, filename), &RepoIndexerData{RepoID: 1, Filename: filename, Content: content, UpdatedAt: time.Now()}))
+	}
+	require.NoError(t, batch.Flush())
+
+	var filenames []string
+	for page := 1; page <= 3; page++ {
+		total, results, _, err := indexer.Search(t.Context(), &internal.SearchOptions{
+			Paginator:  &db.ListOptions{Page: page, PageSize: 1},
+			Keyword:    "abcdef",
+			SearchMode: indexer_module.SearchModeExact,
+		})
+		require.NoError(t, err)
+		require.Len(t, results, util.Iif(page < 3, 1, 0))
+		if page == 3 {
+			assert.EqualValues(t, 2, total)
+		}
+		for _, result := range results {
+			filenames = append(filenames, result.Filename)
+		}
+	}
+	assert.ElementsMatch(t, []string{"1", "3"}, filenames)
+}
+
 func searchResultIDs(result []*internal.SearchResult) []int64 {
 	ids := make([]int64, 0, len(result))
 	for _, hit := range result {

modules/indexer/code/bleve/token_code.go

@@ -4,55 +4,38 @@
 package bleve
 
 import (
-	"regexp"
-	"unicode"
+	"bytes"
+	"unicode/utf8"
 
 	"github.com/blevesearch/bleve/v2/analysis"
-	"github.com/blevesearch/bleve/v2/analysis/tokenizer/character"
 	"github.com/blevesearch/bleve/v2/registry"
 )
 
 const codeTokenizerName = "codeTokenizer"
 
-func codeTokenizerConstructor(_ map[string]any, _ *registry.Cache) (analysis.Tokenizer, error) {
-	// Old code used "letter" tokenizer which doesn't support CJK.
-	// Here it still doesn't support CJK, since there is no usable CJK tokenizer at the moment.
-	return character.NewCharacterTokenizer(func(r rune) bool {
-		return unicode.IsLetter(r) || unicode.IsNumber(r)
-	}), nil
-}
+// codeTokenizer emits every distinct lowercase gram of 1 to 3 runes once, search verifies longer terms against the content
+type codeTokenizer struct{}
 
-const codeTokenFilterName = "codeTokenFilter"
-
-type codeTokenFilter struct {
-	re *regexp.Regexp
-}
-
-func (c codeTokenFilter) Filter(stream analysis.TokenStream) (ret analysis.TokenStream) {
-	// split one token to "letter" parts and "number" parts (to keep the old behavior).
-	// e.g.: input token="port123", then the output tokens are "port123", "port", "123"
-	for _, token := range stream {
-		ret = append(ret, token)
-		m := c.re.FindAllIndex(token.Term, -1)
-		if len(m) > 1 {
-			for _, it := range m {
-				p1, p2 := it[0], it[1]
-				t := &analysis.Token{
-					Start:    token.Start + p1,
-					End:      token.Start + p2,
-					Term:     token.Term[p1:p2],
-					Position: token.Position,
-					Type:     analysis.AlphaNumeric,
-				}
-				ret = append(ret, t)
+func (codeTokenizer) Tokenize(input []byte) analysis.TokenStream {
+	lower := bytes.ToLower(input)
+	seen := make(map[string]bool)
+	var tokens analysis.TokenStream
+	var starts [3]int
+	for end, count := 0, 0; end < len(lower); {
+		starts[0], starts[1], starts[2] = starts[1], starts[2], end
+		count = min(count+1, len(starts))
+		_, size := utf8.DecodeRune(lower[end:])
+		end += size
+		for _, start := range starts[len(starts)-count:] {
+			if gram := lower[start:end]; !seen[string(gram)] {
+				seen[string(gram)] = true
+				tokens = append(tokens, &analysis.Token{Start: start, End: end, Term: gram, Position: len(tokens) + 1, Type: analysis.AlphaNumeric})
 			}
 		}
 	}
-	return ret
+	return tokens
 }
 
-func codeTokenFilterConstructor(_ map[string]any, _ *registry.Cache) (analysis.TokenFilter, error) {
-	return &codeTokenFilter{
-		re: regexp.MustCompile("[a-zA-Z]+|[0-9]+"),
-	}, nil
+func codeTokenizerConstructor(_config map[string]any, _cache *registry.Cache) (analysis.Tokenizer, error) {
+	return codeTokenizer{}, nil
 }

modules/indexer/code/elasticsearch/elasticsearch.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"strconv"
 	"strings"
+	"unicode/utf8"
 
 	repo_model "gitea.dev/models/repo"
 	"gitea.dev/modules/analyze"
@@ -23,12 +24,11 @@ import (
 	"gitea.dev/modules/setting"
 	"gitea.dev/modules/timeutil"
 	"gitea.dev/modules/typesniffer"
-	"gitea.dev/modules/util"
 
 	"github.com/go-enry/go-enry/v2"
 )
 
-const esRepoIndexerLatestVersion = 3
+const esRepoIndexerLatestVersion = 4
 
 var _ internal.Indexer = &Indexer{}
 
@@ -55,6 +55,10 @@ const (
 						"tokenizer": "content_tokenizer",
 						"filter" : ["lowercase"]
 					},
+					"content_short_analyzer": {
+						"tokenizer": "content_short_tokenizer",
+						"filter" : ["lowercase"]
+					},
         			"filename_path_analyzer": {
           				"tokenizer": "path_tokenizer"
         			},
@@ -64,8 +68,14 @@ const (
       			},
 				"tokenizer": {
 					"content_tokenizer": {
-						"type": "simple_pattern_split",
-						"pattern": "[^a-zA-Z0-9]"
+						"type": "ngram",
+						"min_gram": 3,
+						"max_gram": 3
+					},
+					"content_short_tokenizer": {
+						"type": "ngram",
+						"min_gram": 1,
+						"max_gram": 2
 					},
 					"path_tokenizer": {
 						"type": "path_hierarchy",
@@ -87,7 +97,6 @@ const (
 				},
 				"filename": {
 					"type": "text",
-					"term_vector": "with_positions_offsets",
 					"index": true,
 					"fields": {
          		  		"path": {
@@ -102,9 +111,16 @@ const (
 				},
 				"content": {
 					"type": "text",
-					"term_vector": "with_positions_offsets",
 					"index": true,
-					"analyzer": "content_analyzer"
+					"analyzer": "content_analyzer",
+					"fields": {
+						"short": {
+							"type": "text",
+							"analyzer": "content_short_analyzer",
+							"index_options": "docs",
+							"norms": false
+						}
+					}
 				},
 				"commit_id": {
 					"type": "keyword",
@@ -236,23 +252,7 @@ func (b *Indexer) doDelete(ctx context.Context, repoID int64) error {
 	return b.DeleteByQuery(ctx, es.TermsQuery("repo_id", repoID))
 }
 
-// contentMatchIndexPos find words positions for start and the following end on content. It will
-// return the beginning position of the first start and the ending position of the
-// first end following the start string.
-// If not found any of the positions, it will return -1, -1.
-func contentMatchIndexPos(content, start, end string) (int, int) {
-	startIdx := strings.Index(content, start)
-	if startIdx < 0 {
-		return -1, -1
-	}
-	endIdx := strings.Index(content[startIdx+len(start):], end)
-	if endIdx < 0 {
-		return -1, -1
-	}
-	return startIdx, (startIdx + len(start) + endIdx + len(end)) - 9 // remove the length <em></em> since we give Content the original data
-}
-
-func convertResult(searchResult *es.SearchResponse, kw string, pageSize int) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
+func convertResult(searchResult *es.SearchResponse, terms []string, pageSize int) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
 	hits := make([]*internal.SearchResult, 0, pageSize)
 	for _, hit := range searchResult.Hits {
 		repoID, fileName := internal.ParseIndexerID(hit.ID)
@@ -270,22 +270,9 @@ func convertResult(searchResult *es.SearchResponse, kw string, pageSize int) (in
 			return 0, nil, nil, fmt.Errorf("unexpected field types in search hit %q", hit.ID)
 		}
 
-		// FIXME: There is no way to get the position the keyword on the content currently on the same request.
-		// So we get it from content, this may made the query slower. See
-		// https://discuss.elastic.co/t/fetching-position-of-keyword-in-matched-document/94291
-		var startIndex, endIndex int
-		if c, ok := hit.Highlight["filename"]; ok && len(c) > 0 {
+		startIndex, endIndex := internal.ContentMatchIndexPos(content, terms)
+		if startIndex < 0 {
 			startIndex, endIndex = internal.FilenameMatchIndexPos(content)
-		} else if c, ok := hit.Highlight["content"]; ok && len(c) > 0 {
-			// FIXME: Since the highlighting content will include <em> and </em> for the keywords,
-			// now we should find the positions. But how to avoid html content which contains the
-			// <em> and </em> tags? If elastic search has handled that?
-			startIndex, endIndex = contentMatchIndexPos(c[0], "<em>", "</em>")
-			if startIndex == -1 {
-				panic(fmt.Sprintf("1===%s,,,%#v,,,%s", kw, hit.Highlight, c[0]))
-			}
-		} else {
-			panic(fmt.Sprintf("2===%#v", hit.Highlight))
 		}
 
 		hits = append(hits, &internal.SearchResult{
@@ -328,11 +315,19 @@ func extractAggs(searchResult *es.SearchResponse) []*internal.SearchResultLangua
 
 // Search searches for codes and language stats by given conditions.
 func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
-	searchMode := util.IfZero(opts.SearchMode, b.SupportedSearchModes()[0].ModeValue)
-	contentQuery := es.Query(es.NewMultiMatchQuery(opts.Keyword, "content").Type(es.MultiMatchTypeBestFields).Operator("and"))
-	if searchMode == indexer.SearchModeExact {
-		contentQuery = es.MatchPhraseQuery("content", opts.Keyword)
+	terms := internal.SearchTerms(opts)
+	if len(terms) == 0 {
+		return 0, nil, nil, nil
 	}
+	contentQuery := es.NewBoolQuery()
+	for _, term := range terms {
+		if utf8.RuneCountInString(term) < 3 {
+			contentQuery.Must(es.NewMultiMatchQuery(term, "content.short").Operator("and"))
+		} else {
+			contentQuery.Must(es.MatchPhraseQuery("content", term))
+		}
+	}
+
 	kwQuery := es.NewBoolQuery().Should(
 		contentQuery,
 		es.NewMultiMatchQuery(opts.Keyword, "filename^10").Type(es.MultiMatchTypePhrasePrefix),
@@ -343,7 +338,6 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int
 	}
 
 	start, pageSize := opts.GetSkipTake()
-	kw := "<em>" + opts.Keyword + "</em>"
 	languageAggs := map[string]any{
 		"language": map[string]any{
 			"terms": map[string]any{
@@ -353,15 +347,6 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int
 			},
 		},
 	}
-	// number_of_fragments=0 returns the full highlighted content (no fragmentation).
-	highlight := map[string]any{
-		"fields": map[string]any{
-			"content":  map[string]any{},
-			"filename": map[string]any{},
-		},
-		"number_of_fragments": 0,
-		"type":                "fvh",
-	}
 	sort := []es.SortField{
 		{Field: "_score", Desc: true},
 		{Field: "updated_at", Desc: false},
@@ -375,12 +360,11 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int
 			Size:         pageSize,
 			TrackTotal:   true,
 			Aggregations: languageAggs,
-			Highlight:    highlight,
 		})
 		if err != nil {
 			return 0, nil, nil, err
 		}
-		return convertResult(resp, kw, pageSize)
+		return convertResult(resp, terms, pageSize)
 	}
 
 	countResp, err := b.Indexer.Search(ctx, es.SearchRequest{
@@ -400,12 +384,11 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int
 		From:       start,
 		Size:       pageSize,
 		TrackTotal: true,
-		Highlight:  highlight,
 	})
 	if err != nil {
 		return 0, nil, nil, err
 	}
 
-	total, hits, _, err := convertResult(resp, kw, pageSize)
+	total, hits, _, err := convertResult(resp, terms, pageSize)
 	return total, hits, extractAggs(countResp), err
 }

modules/indexer/code/elasticsearch/elasticsearch_test.go

@@ -1,16 +0,0 @@
-// Copyright 2020 The Gitea Authors. All rights reserved.
-// SPDX-License-Identifier: MIT
-
-package elasticsearch
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestIndexPos(t *testing.T) {
-	startIdx, endIdx := contentMatchIndexPos("test index start and end", "start", "end")
-	assert.Equal(t, 11, startIdx)
-	assert.Equal(t, 15, endIdx)
-}

modules/indexer/code/indexer_test.go

@@ -5,18 +5,19 @@ package code
 
 import (
 	"context"
-	"os"
+	"fmt"
 	"slices"
 	"testing"
 	"time"
 
 	"gitea.dev/models/db"
+	repo_model "gitea.dev/models/repo"
 	"gitea.dev/models/unittest"
+	"gitea.dev/modules/git"
 	indexer_module "gitea.dev/modules/indexer"
 	"gitea.dev/modules/indexer/code/bleve"
 	"gitea.dev/modules/indexer/code/elasticsearch"
 	"gitea.dev/modules/indexer/code/internal"
-	"gitea.dev/modules/setting"
 	"gitea.dev/modules/test"
 	"gitea.dev/modules/util"
 
@@ -37,8 +38,29 @@ func TestMain(m *testing.M) {
 	unittest.MainTest(m)
 }
 
+const unicodeSearchContent = `Поставщик
+日本語のテキスト
+हिन्दी
+ภาษาไทย
+İSTANBUL
+<em>literal</em>
+abc bcd cde def
+`
+
 func testIndexer(name string, t *testing.T, indexer internal.Indexer) {
 	t.Run(name, func(t *testing.T) {
+		repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 62})
+		require.NoError(t, git.ForceFastImport(t.Context(), repo, []git.FastImportCommit{{
+			Ref: "refs/heads/master",
+			Files: []git.FastImportFile{
+				{Path: "avocado.md", Content: "# repo1\n\npineaple pie of cucumber juice"},
+				{Path: "cucumber.md", Content: "Salad is good for your health"},
+				{Path: "example-file.js", Content: "console.log(\"Hello, World!\")"},
+				{Path: "ham.md", Content: "This is also not cheese"},
+				{Path: "potato/ham.md", Content: "This is not cheese"},
+				{Path: "unicode.txt", Content: unicodeSearchContent},
+			},
+		}}))
 		assert.NoError(t, setupRepositoryIndexes(t.Context(), indexer))
 		// Wait for the index to catch up: ES/OpenSearch make writes visible
 		// only after a refresh (default interval: 1s). Bleve is synchronous
@@ -57,6 +79,7 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) {
 			Langs      int
 			SearchMode indexer_module.SearchModeType
 			Results    []codeSearchResult
+			Match      string
 		}{
 			// Search for an exact match on the contents of a file
 			// This scenario yields a single result (the file README.md on the repo '1')
@@ -89,6 +112,10 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) {
 						Filename: "README.md",
 						Content:  "# repo1\n\nDescription for repo1",
 					},
+					{
+						Filename: "avocado.md",
+						Content:  "# repo1\n\npineaple pie of cucumber juice",
+					},
 				},
 			},
 			// Search for an exact match on the contents of a file within the repo '2'.
@@ -191,7 +218,7 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) {
 				},
 			},
 			// Search for matches on the contents of files within the repo '62'.
-			// This scenario yields two results (both are based on contents, the first one is an exact match where as the second is a 'fuzzy' one)
+			// This scenario yields two results (both are based on contents, the first one is an exact match where as the second is a words match)
 			{
 				RepoIDs: []int64{62},
 				Keyword: "This is not cheese",
@@ -209,10 +236,9 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) {
 			},
 			// Search for matches on the contents of files regardless of case.
 			{
-				RepoIDs:    nil,
-				Keyword:    "dESCRIPTION",
-				Langs:      1,
-				SearchMode: indexer_module.SearchModeFuzzy,
+				RepoIDs: nil,
+				Keyword: "dESCRIPTION",
+				Langs:   1,
 				Results: []codeSearchResult{
 					{
 						Filename: "README.md",
@@ -257,6 +283,13 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) {
 					},
 				},
 			},
+			{RepoIDs: []int64{62}, Keyword: "ставщ", Langs: 1, SearchMode: indexer_module.SearchModeExact, Results: []codeSearchResult{{Filename: "unicode.txt", Content: unicodeSearchContent}}},
+			{RepoIDs: []int64{62}, Keyword: "テキスト", Langs: 1, Results: []codeSearchResult{{Filename: "unicode.txt", Content: unicodeSearchContent}}},
+			{RepoIDs: []int64{62}, Keyword: "हिन्दी", Langs: 1, SearchMode: indexer_module.SearchModeExact, Results: []codeSearchResult{{Filename: "unicode.txt", Content: unicodeSearchContent}}},
+			{RepoIDs: []int64{62}, Keyword: "iSTANBUL", Langs: 1, SearchMode: indexer_module.SearchModeExact, Match: "İSTANBUL", Results: []codeSearchResult{{Filename: "unicode.txt", Content: unicodeSearchContent}}},
+			{RepoIDs: []int64{62}, Keyword: "literal", Langs: 1, Match: "literal", Results: []codeSearchResult{{Filename: "unicode.txt", Content: unicodeSearchContent}}},
+			{RepoIDs: []int64{62}, Keyword: "日本 ภาษา", Langs: 1, Match: "日本", Results: []codeSearchResult{{Filename: "unicode.txt", Content: unicodeSearchContent}}},
+			{RepoIDs: []int64{62}, Keyword: "abcdef", SearchMode: indexer_module.SearchModeExact},
 		}
 
 		for _, kw := range keywords {
@@ -272,14 +305,15 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) {
 				})
 				require.NoError(t, err)
 				require.Len(t, langs, kw.Langs)
+				require.EqualValues(t, len(kw.Results), total)
+				require.Len(t, res, len(kw.Results))
 
 				hits := make([]codeSearchResult, 0, len(res))
 
-				if total > 0 {
-					assert.NotEmpty(t, kw.Results, "The given scenario does not provide any expected results")
-				}
-
 				for _, hit := range res {
+					if kw.Match != "" {
+						require.Equal(t, kw.Match, hit.Content[hit.StartIndex:hit.EndIndex])
+					}
 					hits = append(hits, codeSearchResult{
 						Filename: hit.Filename,
 						Content:  hit.Content,
@@ -307,7 +341,6 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) {
 
 func TestBleveIndexAndSearch(t *testing.T) {
 	unittest.PrepareTestEnv(t)
-	defer test.MockVariableValue(&setting.Indexer.TypeBleveMaxFuzzniess, 2)()
 	dir := t.TempDir()
 
 	idx := bleve.NewIndexer(dir)
@@ -322,13 +355,9 @@ func TestBleveIndexAndSearch(t *testing.T) {
 func TestESIndexAndSearch(t *testing.T) {
 	unittest.PrepareTestEnv(t)
 
-	u := os.Getenv("TEST_INDEXER_CODE_ES_URL")
-	if u == "" {
-		t.SkipNow()
-		return
-	}
+	u := test.ExternalServiceHTTP(t, "TEST_ELASTICSEARCH_URL", "http://elasticsearch:9200")
 
-	indexer := elasticsearch.NewIndexer(u, "gitea_codes")
+	indexer := elasticsearch.NewIndexer(u, fmt.Sprintf("test_code_indexer_%d", time.Now().UnixNano()))
 	if _, err := indexer.Init(t.Context()); err != nil {
 		if indexer != nil {
 			indexer.Close()

modules/indexer/code/internal/util.go

@@ -5,7 +5,10 @@ package internal
 
 import (
 	"strings"
+	"unicode"
+	"unicode/utf8"
 
+	"gitea.dev/modules/indexer"
 	"gitea.dev/modules/indexer/internal"
 	"gitea.dev/modules/log"
 )
@@ -46,3 +49,43 @@ func FilenameMatchIndexPos(content string) (int, int) {
 	}
 	return 0, len(content)
 }
+
+func SearchTerms(opts *SearchOptions) []string {
+	if opts.Keyword == "" {
+		return nil
+	}
+	if opts.SearchMode == "" || opts.SearchMode == indexer.SearchModeExact {
+		return []string{opts.Keyword}
+	} else {
+		return strings.Fields(opts.Keyword)
+	}
+}
+
+// ContentMatchIndexPos returns the byte range of the earliest term in content, or -1, -1 unless all terms occur
+func ContentMatchIndexPos(content string, terms []string) (int, int) {
+	lowerContent := strings.ToLower(content)
+	start, end := -1, -1
+	for _, term := range terms {
+		lowerTerm := strings.ToLower(term)
+		position := strings.Index(lowerContent, lowerTerm)
+		if position < 0 {
+			return -1, -1
+		}
+		if start < 0 || position < start {
+			start, end = position, position+len(lowerTerm)
+		}
+	}
+	lowerOffset, originalStart := 0, -1
+	for offset := 0; offset < len(content); {
+		character, size := utf8.DecodeRuneInString(content[offset:])
+		if lowerOffset == start {
+			originalStart = offset
+		}
+		lowerOffset += utf8.RuneLen(unicode.ToLower(character))
+		offset += size
+		if lowerOffset == end {
+			return originalStart, offset
+		}
+	}
+	return -1, -1
+}