// Copyright 2020 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT package code import ( "context" "fmt" "slices" "testing" "time" "gitea.dev/models/db" repo_model "gitea.dev/models/repo" "gitea.dev/models/unittest" "gitea.dev/modules/git" indexer_module "gitea.dev/modules/indexer" "gitea.dev/modules/indexer/code/bleve" "gitea.dev/modules/indexer/code/elasticsearch" "gitea.dev/modules/indexer/code/internal" "gitea.dev/modules/test" "gitea.dev/modules/util" _ "gitea.dev/models" _ "gitea.dev/models/actions" _ "gitea.dev/models/activities" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) type codeSearchResult struct { Filename string Content string } func TestMain(m *testing.M) { unittest.MainTest(m) } const unicodeSearchContent = `Поставщик 日本語のテキスト हिन्दी ภาษาไทย İSTANBUL literal abc bcd cde def ` func testIndexer(name string, t *testing.T, indexer internal.Indexer) { t.Run(name, func(t *testing.T) { repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 62}) require.NoError(t, git.ForceFastImport(t.Context(), repo, []git.FastImportCommit{{ Ref: "refs/heads/master", Files: []git.FastImportFile{ {Path: "avocado.md", Content: "# repo1\n\npineaple pie of cucumber juice"}, {Path: "cucumber.md", Content: "Salad is good for your health"}, {Path: "example-file.js", Content: "console.log(\"Hello, World!\")"}, {Path: "ham.md", Content: "This is also not cheese"}, {Path: "potato/ham.md", Content: "This is not cheese"}, {Path: "unicode.txt", Content: unicodeSearchContent}, }, }})) assert.NoError(t, setupRepositoryIndexes(t.Context(), indexer)) // Wait for the index to catch up: ES/OpenSearch make writes visible // only after a refresh (default interval: 1s). Bleve is synchronous // and passes on the first iteration. require.Eventually(t, func() bool { total, _, _, err := indexer.Search(t.Context(), &internal.SearchOptions{ Keyword: "Description", Paginator: &db.ListOptions{Page: 1, PageSize: 1}, }) return err == nil && total > 0 }, 10*time.Second, 100*time.Millisecond, "index did not become searchable") keywords := []struct { RepoIDs []int64 Keyword string Langs int SearchMode indexer_module.SearchModeType Results []codeSearchResult Match string }{ // Search for an exact match on the contents of a file // This scenario yields a single result (the file README.md on the repo '1') { RepoIDs: nil, Keyword: "Description", Langs: 1, Results: []codeSearchResult{ { Filename: "README.md", Content: "# repo1\n\nDescription for repo1", }, }, }, // Search for an exact match on the contents of a file within the repo '2'. // This scenario yields no results { RepoIDs: []int64{2}, Keyword: "Description", Langs: 0, }, // Search for an exact match on the contents of a file // This scenario yields a single result (the file README.md on the repo '1') { RepoIDs: nil, Keyword: "repo1", Langs: 1, Results: []codeSearchResult{ { Filename: "README.md", Content: "# repo1\n\nDescription for repo1", }, { Filename: "avocado.md", Content: "# repo1\n\npineaple pie of cucumber juice", }, }, }, // Search for an exact match on the contents of a file within the repo '2'. // This scenario yields no results { RepoIDs: []int64{2}, Keyword: "repo1", Langs: 0, }, // Search for a non-existing term. // This scenario yields no results { RepoIDs: nil, Keyword: "non-exist", Langs: 0, }, // Search for an exact match on the contents of a file within the repo '62'. // This scenario yields a single result (the file avocado.md on the repo '62') { RepoIDs: []int64{62}, Keyword: "pineaple", Langs: 1, Results: []codeSearchResult{ { Filename: "avocado.md", Content: "# repo1\n\npineaple pie of cucumber juice", }, }, }, // Search for an exact match on the filename within the repo '62'. // This scenario yields a single result (the file avocado.md on the repo '62') { RepoIDs: []int64{62}, Keyword: "avocado.md", Langs: 1, Results: []codeSearchResult{ { Filename: "avocado.md", Content: "# repo1\n\npineaple pie of cucumber juice", }, }, }, // Search for an partial match on the filename within the repo '62'. // This scenario yields a single result (the file avocado.md on the repo '62') { RepoIDs: []int64{62}, Keyword: "avo", Langs: 1, Results: []codeSearchResult{ { Filename: "avocado.md", Content: "# repo1\n\npineaple pie of cucumber juice", }, }, }, // Search for matches on both the contents and the filenames within the repo '62'. // This scenario yields two results: the first result is based on the file (cucumber.md) while the second is based on the contents { RepoIDs: []int64{62}, Keyword: "cucumber", Langs: 1, Results: []codeSearchResult{ { Filename: "cucumber.md", Content: "Salad is good for your health", }, { Filename: "avocado.md", Content: "# repo1\n\npineaple pie of cucumber juice", }, }, }, // Search for matches on the filenames within the repo '62'. // This scenario yields two results (both are based on filename, the first one is an exact match) { RepoIDs: []int64{62}, Keyword: "ham", Langs: 1, Results: []codeSearchResult{ { Filename: "ham.md", Content: "This is also not cheese", }, { Filename: "potato/ham.md", Content: "This is not cheese", }, }, }, // Search for a match on the full path of a file within the repo '62'. { RepoIDs: []int64{62}, Keyword: "potato/ham", Langs: 1, Results: []codeSearchResult{ { Filename: "potato/ham.md", Content: "This is not cheese", }, }, }, // Search for matches on the contents of files within the repo '62'. // This scenario yields two results (both are based on contents, the first one is an exact match where as the second is a words match) { RepoIDs: []int64{62}, Keyword: "This is not cheese", Langs: 1, Results: []codeSearchResult{ { Filename: "potato/ham.md", Content: "This is not cheese", }, { Filename: "ham.md", Content: "This is also not cheese", }, }, }, // Search for matches on the contents of files regardless of case. { RepoIDs: nil, Keyword: "dESCRIPTION", Langs: 1, Results: []codeSearchResult{ { Filename: "README.md", Content: "# repo1\n\nDescription for repo1", }, }, }, // Search for an exact match on the filename within the repo '62' (case-insensitive). // This scenario yields a single result (the file avocado.md on the repo '62') { RepoIDs: []int64{62}, Keyword: "AVOCADO.MD", Langs: 1, Results: []codeSearchResult{ { Filename: "avocado.md", Content: "# repo1\n\npineaple pie of cucumber juice", }, }, }, // Search for matches on the contents of files when the criteria are an expression. { RepoIDs: []int64{62}, Keyword: "console.log", Langs: 1, Results: []codeSearchResult{ { Filename: "example-file.js", Content: "console.log(\"Hello, World!\")", }, }, }, // Search for matches on the contents of files when the criteria are parts of an expression. { RepoIDs: []int64{62}, Keyword: "log", Langs: 1, Results: []codeSearchResult{ { Filename: "example-file.js", Content: "console.log(\"Hello, World!\")", }, }, }, {RepoIDs: []int64{62}, Keyword: "ставщ", Langs: 1, SearchMode: indexer_module.SearchModeExact, Results: []codeSearchResult{{Filename: "unicode.txt", Content: unicodeSearchContent}}}, {RepoIDs: []int64{62}, Keyword: "テキスト", Langs: 1, Results: []codeSearchResult{{Filename: "unicode.txt", Content: unicodeSearchContent}}}, {RepoIDs: []int64{62}, Keyword: "हिन्दी", Langs: 1, SearchMode: indexer_module.SearchModeExact, Results: []codeSearchResult{{Filename: "unicode.txt", Content: unicodeSearchContent}}}, {RepoIDs: []int64{62}, Keyword: "iSTANBUL", Langs: 1, SearchMode: indexer_module.SearchModeExact, Match: "İSTANBUL", Results: []codeSearchResult{{Filename: "unicode.txt", Content: unicodeSearchContent}}}, {RepoIDs: []int64{62}, Keyword: "literal", Langs: 1, Match: "literal", Results: []codeSearchResult{{Filename: "unicode.txt", Content: unicodeSearchContent}}}, {RepoIDs: []int64{62}, Keyword: "日本 ภาษา", Langs: 1, Match: "日本", Results: []codeSearchResult{{Filename: "unicode.txt", Content: unicodeSearchContent}}}, {RepoIDs: []int64{62}, Keyword: "abcdef", SearchMode: indexer_module.SearchModeExact}, } for _, kw := range keywords { t.Run(kw.Keyword, func(t *testing.T) { total, res, langs, err := indexer.Search(t.Context(), &internal.SearchOptions{ RepoIDs: kw.RepoIDs, Keyword: kw.Keyword, SearchMode: util.IfZero(kw.SearchMode, indexer_module.SearchModeWords), Paginator: &db.ListOptions{ Page: 1, PageSize: 10, }, }) require.NoError(t, err) require.Len(t, langs, kw.Langs) require.EqualValues(t, len(kw.Results), total) require.Len(t, res, len(kw.Results)) hits := make([]codeSearchResult, 0, len(res)) for _, hit := range res { if kw.Match != "" { require.Equal(t, kw.Match, hit.Content[hit.StartIndex:hit.EndIndex]) } hits = append(hits, codeSearchResult{ Filename: hit.Filename, Content: hit.Content, }) } lastIndex := -1 for _, expected := range kw.Results { index := slices.Index(hits, expected) if index == -1 { assert.Failf(t, "Result not found", "Expected %v in %v", expected, hits) } else if lastIndex > index { assert.Failf(t, "Result is out of order", "The order of %v within %v is wrong", expected, hits) } else { lastIndex = index } } }) } assert.NoError(t, tearDownRepositoryIndexes(t.Context(), indexer)) }) } func TestBleveIndexAndSearch(t *testing.T) { unittest.PrepareTestEnv(t) dir := t.TempDir() idx := bleve.NewIndexer(dir) defer idx.Close() _, err := idx.Init(t.Context()) require.NoError(t, err) testIndexer("bleve", t, idx) } func TestESIndexAndSearch(t *testing.T) { unittest.PrepareTestEnv(t) u := test.ExternalServiceHTTP(t, "TEST_ELASTICSEARCH_URL", "http://elasticsearch:9200") indexer := elasticsearch.NewIndexer(u, fmt.Sprintf("test_code_indexer_%d", time.Now().UnixNano())) if _, err := indexer.Init(t.Context()); err != nil { if indexer != nil { indexer.Close() } require.NoError(t, err, "Unable to init ES indexer") } defer indexer.Close() testIndexer("elastic_search", t, indexer) } func setupRepositoryIndexes(ctx context.Context, indexer internal.Indexer) error { for _, repoID := range repositoriesToSearch() { if err := index(ctx, indexer, repoID); err != nil { return err } } return nil } func tearDownRepositoryIndexes(ctx context.Context, indexer internal.Indexer) error { for _, repoID := range repositoriesToSearch() { if err := indexer.Delete(ctx, repoID); err != nil { return err } } return nil } func repositoriesToSearch() []int64 { return []int64{1, 62} }