fix(indexer): search code as case-insensitive literal substrings (#40028)

Code search now matches case-insensitive literal substrings like GitHub,
in every script. Word tokenizers can't do this: the ES pattern dropped
every non-ASCII word, and the `standard` tokenizer keeps `console.log`
as one token.

- Elasticsearch indexes positional trigrams, plus 1-2 grams for short
terms
- Bleve indexes distinct 1-3 grams as a candidate filter and verifies
candidates against the content before paging, because phrase queries
over trigrams are quadratic in repeated grams
- Snippet offsets come from the original content, so `<em>` in sources
and width-changing lowercase no longer misplace them
- `search_mode=fuzzy` requests behave like Words, the UI never offered
fuzzy for code search
- The ES code indexer test now runs in CI, it was skipped before

There are no UI changes. Both code indexes rebuild after upgrade.

Fixes: https://github.com/go-gitea/gitea/issues/40027
Fixes: https://github.com/go-gitea/gitea/issues/16158
Fixes: https://github.com/go-gitea/gitea/issues/21359

---------

Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

modules/indexer/code/internal/util.go

@@ -5,7 +5,10 @@ package internal
 
 import (
 	"strings"
+	"unicode"
+	"unicode/utf8"
 
+	"gitea.dev/modules/indexer"
 	"gitea.dev/modules/indexer/internal"
 	"gitea.dev/modules/log"
 )
@@ -46,3 +49,43 @@ func FilenameMatchIndexPos(content string) (int, int) {
 	}
 	return 0, len(content)
 }
+
+func SearchTerms(opts *SearchOptions) []string {
+	if opts.Keyword == "" {
+		return nil
+	}
+	if opts.SearchMode == "" || opts.SearchMode == indexer.SearchModeExact {
+		return []string{opts.Keyword}
+	} else {
+		return strings.Fields(opts.Keyword)
+	}
+}
+
+// ContentMatchIndexPos returns the byte range of the earliest term in content, or -1, -1 unless all terms occur
+func ContentMatchIndexPos(content string, terms []string) (int, int) {
+	lowerContent := strings.ToLower(content)
+	start, end := -1, -1
+	for _, term := range terms {
+		lowerTerm := strings.ToLower(term)
+		position := strings.Index(lowerContent, lowerTerm)
+		if position < 0 {
+			return -1, -1
+		}
+		if start < 0 || position < start {
+			start, end = position, position+len(lowerTerm)
+		}
+	}
+	lowerOffset, originalStart := 0, -1
+	for offset := 0; offset < len(content); {
+		character, size := utf8.DecodeRuneInString(content[offset:])
+		if lowerOffset == start {
+			originalStart = offset
+		}
+		lowerOffset += utf8.RuneLen(unicode.ToLower(character))
+		offset += size
+		if lowerOffset == end {
+			return originalStart, offset
+		}
+	}
+	return -1, -1
+}