From c10503afeccd5172ace7613094dd5fe1e0770c55 Mon Sep 17 00:00:00 2001 From: Jui-Nan Lin Date: Wed, 27 Jan 2021 18:00:35 +0800 Subject: [PATCH] [Feature] add precise search type for Elastic Search (#12869) * feat: add type query parameters for specifying precise search * feat: add select dropdown in search box Co-authored-by: Lauris BH Co-authored-by: techknowlogick --- modules/context/pagination.go | 1 + modules/indexer/code/bleve.go | 25 ++++++++++++++++++------- modules/indexer/code/elastic_search.go | 13 +++++++++++-- modules/indexer/code/indexer.go | 2 +- modules/indexer/code/indexer_test.go | 2 +- modules/indexer/code/search.go | 4 ++-- modules/indexer/code/wrapped.go | 4 ++-- options/locale/locale_en-US.ini | 4 ++++ routers/home.go | 8 ++++++-- routers/repo/search.go | 6 +++++- templates/explore/code.tmpl | 13 +++++++++++-- templates/repo/search.tmpl | 20 +++++++++++++++----- 12 files changed, 77 insertions(+), 25 deletions(-) diff --git a/modules/context/pagination.go b/modules/context/pagination.go index a6638f408..b678fccd1 100644 --- a/modules/context/pagination.go +++ b/modules/context/pagination.go @@ -53,4 +53,5 @@ func (p *Pagination) SetDefaultParams(ctx *Context) { p.AddParam(ctx, "sort", "SortType") p.AddParam(ctx, "q", "Keyword") p.AddParam(ctx, "tab", "TabName") + p.AddParam(ctx, "t", "queryType") } diff --git a/modules/indexer/code/bleve.go b/modules/indexer/code/bleve.go index b0822ad22..826efde4c 100644 --- a/modules/indexer/code/bleve.go +++ b/modules/indexer/code/bleve.go @@ -280,12 +280,23 @@ func (b *BleveIndexer) Delete(repoID int64) error { // Search searches for files in the specified repo. // Returns the matching file-paths -func (b *BleveIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int) (int64, []*SearchResult, []*SearchResultLanguages, error) { - phraseQuery := bleve.NewMatchPhraseQuery(keyword) - phraseQuery.FieldVal = "Content" - phraseQuery.Analyzer = repoIndexerAnalyzer +func (b *BleveIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) { + var ( + indexerQuery query.Query + keywordQuery query.Query + ) + + if isMatch { + prefixQuery := bleve.NewPrefixQuery(keyword) + prefixQuery.FieldVal = "Content" + keywordQuery = prefixQuery + } else { + phraseQuery := bleve.NewMatchPhraseQuery(keyword) + phraseQuery.FieldVal = "Content" + phraseQuery.Analyzer = repoIndexerAnalyzer + keywordQuery = phraseQuery + } - var indexerQuery query.Query if len(repoIDs) > 0 { var repoQueries = make([]query.Query, 0, len(repoIDs)) for _, repoID := range repoIDs { @@ -294,10 +305,10 @@ func (b *BleveIndexer) Search(repoIDs []int64, language, keyword string, page, p indexerQuery = bleve.NewConjunctionQuery( bleve.NewDisjunctionQuery(repoQueries...), - phraseQuery, + keywordQuery, ) } else { - indexerQuery = phraseQuery + indexerQuery = keywordQuery } // Save for reuse without language filter diff --git a/modules/indexer/code/elastic_search.go b/modules/indexer/code/elastic_search.go index 0f61c4e59..f81dbb34d 100644 --- a/modules/indexer/code/elastic_search.go +++ b/modules/indexer/code/elastic_search.go @@ -27,6 +27,10 @@ import ( const ( esRepoIndexerLatestVersion = 1 + // multi-match-types, currently only 2 types are used + // Reference: https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-multi-match-query.html#multi-match-types + esMultiMatchTypeBestFields = "best_fields" + esMultiMatchTypePhrasePrefix = "phrase_prefix" ) var ( @@ -330,8 +334,13 @@ func extractAggs(searchResult *elastic.SearchResult) []*SearchResultLanguages { } // Search searches for codes and language stats by given conditions. -func (b *ElasticSearchIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int) (int64, []*SearchResult, []*SearchResultLanguages, error) { - kwQuery := elastic.NewMultiMatchQuery(keyword, "content") +func (b *ElasticSearchIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) { + searchType := esMultiMatchTypeBestFields + if isMatch { + searchType = esMultiMatchTypePhrasePrefix + } + + kwQuery := elastic.NewMultiMatchQuery(keyword, "content").Type(searchType) query := elastic.NewBoolQuery() query = query.Must(kwQuery) if len(repoIDs) > 0 { diff --git a/modules/indexer/code/indexer.go b/modules/indexer/code/indexer.go index 35c298a54..a7d78e9fd 100644 --- a/modules/indexer/code/indexer.go +++ b/modules/indexer/code/indexer.go @@ -43,7 +43,7 @@ type SearchResultLanguages struct { type Indexer interface { Index(repo *models.Repository, sha string, changes *repoChanges) error Delete(repoID int64) error - Search(repoIDs []int64, language, keyword string, page, pageSize int) (int64, []*SearchResult, []*SearchResultLanguages, error) + Search(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) Close() } diff --git a/modules/indexer/code/indexer_test.go b/modules/indexer/code/indexer_test.go index 0b4851a48..8fcb7a0e8 100644 --- a/modules/indexer/code/indexer_test.go +++ b/modules/indexer/code/indexer_test.go @@ -64,7 +64,7 @@ func testIndexer(name string, t *testing.T, indexer Indexer) { for _, kw := range keywords { t.Run(kw.Keyword, func(t *testing.T) { - total, res, langs, err := indexer.Search(kw.RepoIDs, "", kw.Keyword, 1, 10) + total, res, langs, err := indexer.Search(kw.RepoIDs, "", kw.Keyword, 1, 10, false) assert.NoError(t, err) assert.EqualValues(t, len(kw.IDs), total) assert.EqualValues(t, kw.Langs, len(langs)) diff --git a/modules/indexer/code/search.go b/modules/indexer/code/search.go index 29ed41654..51b7c9427 100644 --- a/modules/indexer/code/search.go +++ b/modules/indexer/code/search.go @@ -106,12 +106,12 @@ func searchResult(result *SearchResult, startIndex, endIndex int) (*Result, erro } // PerformSearch perform a search on a repository -func PerformSearch(repoIDs []int64, language, keyword string, page, pageSize int) (int, []*Result, []*SearchResultLanguages, error) { +func PerformSearch(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int, []*Result, []*SearchResultLanguages, error) { if len(keyword) == 0 { return 0, nil, nil, nil } - total, results, resultLanguages, err := indexer.Search(repoIDs, language, keyword, page, pageSize) + total, results, resultLanguages, err := indexer.Search(repoIDs, language, keyword, page, pageSize, isMatch) if err != nil { return 0, nil, nil, err } diff --git a/modules/indexer/code/wrapped.go b/modules/indexer/code/wrapped.go index d83954487..5b19f9c62 100644 --- a/modules/indexer/code/wrapped.go +++ b/modules/indexer/code/wrapped.go @@ -73,12 +73,12 @@ func (w *wrappedIndexer) Delete(repoID int64) error { return indexer.Delete(repoID) } -func (w *wrappedIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int) (int64, []*SearchResult, []*SearchResultLanguages, error) { +func (w *wrappedIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) { indexer, err := w.get() if err != nil { return 0, nil, nil, err } - return indexer.Search(repoIDs, language, keyword, page, pageSize) + return indexer.Search(repoIDs, language, keyword, page, pageSize, isMatch) } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 1fe43ce29..4083f7114 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -237,6 +237,8 @@ users = Users organizations = Organizations search = Search code = Code +search.fuzzy = Fuzzy +search.match = Match repo_no_results = No matching repositories found. user_no_results = No matching users found. org_no_results = No matching organizations found. @@ -1462,6 +1464,8 @@ activity.git_stats_deletion_n = %d deletions search = Search search.search_repo = Search repository +search.fuzzy = Fuzzy +search.match = Match search.results = Search results for "%s" in %s settings = Settings diff --git a/routers/home.go b/routers/home.go index d37bf8e31..f82ee9808 100644 --- a/routers/home.go +++ b/routers/home.go @@ -299,6 +299,9 @@ func ExploreCode(ctx *context.Context) { page = 1 } + queryType := strings.TrimSpace(ctx.Query("t")) + isMatch := queryType == "match" + var ( repoIDs []int64 err error @@ -342,14 +345,14 @@ func ExploreCode(ctx *context.Context) { ctx.Data["RepoMaps"] = rightRepoMap - total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum) + total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch) if err != nil { ctx.ServerError("SearchResults", err) return } // if non-login user or isAdmin, no need to check UnitTypeCode } else if (ctx.User == nil && len(repoIDs) > 0) || isAdmin { - total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum) + total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch) if err != nil { ctx.ServerError("SearchResults", err) return @@ -380,6 +383,7 @@ func ExploreCode(ctx *context.Context) { ctx.Data["Keyword"] = keyword ctx.Data["Language"] = language + ctx.Data["queryType"] = queryType ctx.Data["SearchResults"] = searchResults ctx.Data["SearchResultLanguages"] = searchResultLanguages ctx.Data["RequireHighlightJS"] = true diff --git a/routers/repo/search.go b/routers/repo/search.go index e110ae2a7..42fe3d758 100644 --- a/routers/repo/search.go +++ b/routers/repo/search.go @@ -28,14 +28,18 @@ func Search(ctx *context.Context) { if page <= 0 { page = 1 } + queryType := strings.TrimSpace(ctx.Query("t")) + isMatch := queryType == "match" + total, searchResults, searchResultLanguages, err := code_indexer.PerformSearch([]int64{ctx.Repo.Repository.ID}, - language, keyword, page, setting.UI.RepoSearchPagingNum) + language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch) if err != nil { ctx.ServerError("SearchResults", err) return } ctx.Data["Keyword"] = keyword ctx.Data["Language"] = language + ctx.Data["queryType"] = queryType ctx.Data["SourcePath"] = setting.AppSubURL + "/" + path.Join(ctx.Repo.Repository.Owner.Name, ctx.Repo.Repository.Name) ctx.Data["SearchResults"] = searchResults diff --git a/templates/explore/code.tmpl b/templates/explore/code.tmpl index 1c41dd843..2465663a6 100644 --- a/templates/explore/code.tmpl +++ b/templates/explore/code.tmpl @@ -5,9 +5,19 @@
+
+
+
+ +
+
+
@@ -18,7 +28,7 @@
{{range $term := .SearchResultLanguages}} - + {{$term.Language}}
{{$term.Count}}
@@ -62,4 +72,3 @@
{{template "base/footer" .}} - diff --git a/templates/repo/search.tmpl b/templates/repo/search.tmpl index b66391a5a..ab9e9be2d 100644 --- a/templates/repo/search.tmpl +++ b/templates/repo/search.tmpl @@ -5,10 +5,20 @@
- - +
+ +
+
+ +
+
+ +
@@ -18,7 +28,7 @@
{{range $term := .SearchResultLanguages}} - + {{$term.Language}}
{{$term.Count}}