From 9e5d0a09eb7370daedb1cf572d25ffa150eacf17 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 16 Mar 2018 22:04:33 +0800 Subject: [PATCH] Global code search support (#3664) * add global code search on explore * fix bug when no anyone public repos * change the icon * fix typo and add UnitTypeCode check for login non-admin user * fix ui description when no match --- models/repo.go | 6 ++ models/repo_list.go | 25 +++++++ modules/indexer/repo.go | 27 ++++++-- modules/search/search.go | 6 +- options/locale/locale_en-US.ini | 3 + routers/home.go | 116 ++++++++++++++++++++++++++++++++ routers/repo/search.go | 3 +- routers/routes/routes.go | 1 + templates/explore/code.tmpl | 55 +++++++++++++++ templates/explore/navbar.tmpl | 5 ++ 10 files changed, 238 insertions(+), 9 deletions(-) create mode 100644 templates/explore/code.tmpl diff --git a/models/repo.go b/models/repo.go index cddd57dc3..e9b3307d9 100644 --- a/models/repo.go +++ b/models/repo.go @@ -1945,6 +1945,12 @@ func GetRepositoryByID(id int64) (*Repository, error) { return getRepositoryByID(x, id) } +// GetRepositoriesMapByIDs returns the repositories by given id slice. +func GetRepositoriesMapByIDs(ids []int64) (map[int64]*Repository, error) { + var repos = make(map[int64]*Repository, len(ids)) + return repos, x.In("id", ids).Find(&repos) +} + // GetUserRepositories returns a list of repositories of given user. func GetUserRepositories(userID int64, private bool, page, pageSize int, orderBy string) ([]*Repository, error) { if len(orderBy) == 0 { diff --git a/models/repo_list.go b/models/repo_list.go index bc9b831d3..df6b81cb8 100644 --- a/models/repo_list.go +++ b/models/repo_list.go @@ -249,3 +249,28 @@ func SearchRepositoryByName(opts *SearchRepoOptions) (RepositoryList, int64, err return repos, count, nil } + +// FindUserAccessibleRepoIDs find all accessible repositories' ID by user's id +func FindUserAccessibleRepoIDs(userID int64) ([]int64, error) { + var accessCond builder.Cond = builder.Eq{"is_private": false} + + if userID > 0 { + accessCond = accessCond.Or( + builder.Eq{"owner_id": userID}, + builder.And( + builder.Expr("id IN (SELECT repo_id FROM `access` WHERE access.user_id = ?)", userID), + builder.Neq{"owner_id": userID}, + ), + ) + } + + repoIDs := make([]int64, 0, 10) + if err := x. + Table("repository"). + Cols("id"). + Where(accessCond). + Find(&repoIDs); err != nil { + return nil, fmt.Errorf("FindUserAccesibleRepoIDs: %v", err) + } + return repoIDs, nil +} diff --git a/modules/indexer/repo.go b/modules/indexer/repo.go index ffb1dc1e6..4d1e79215 100644 --- a/modules/indexer/repo.go +++ b/modules/indexer/repo.go @@ -16,6 +16,7 @@ import ( "github.com/blevesearch/bleve/analysis/token/lowercase" "github.com/blevesearch/bleve/analysis/token/unique" "github.com/blevesearch/bleve/analysis/tokenizer/unicode" + "github.com/blevesearch/bleve/search/query" "github.com/ethantkoenig/rupture" ) @@ -158,6 +159,7 @@ func DeleteRepoFromIndexer(repoID int64) error { // RepoSearchResult result of performing a search in a repo type RepoSearchResult struct { + RepoID int64 StartIndex int EndIndex int Filename string @@ -166,17 +168,29 @@ type RepoSearchResult struct { // SearchRepoByKeyword searches for files in the specified repo. // Returns the matching file-paths -func SearchRepoByKeyword(repoID int64, keyword string, page, pageSize int) (int64, []*RepoSearchResult, error) { +func SearchRepoByKeyword(repoIDs []int64, keyword string, page, pageSize int) (int64, []*RepoSearchResult, error) { phraseQuery := bleve.NewMatchPhraseQuery(keyword) phraseQuery.FieldVal = "Content" phraseQuery.Analyzer = repoIndexerAnalyzer - indexerQuery := bleve.NewConjunctionQuery( - numericEqualityQuery(repoID, "RepoID"), - phraseQuery, - ) + + var indexerQuery query.Query + if len(repoIDs) > 0 { + var repoQueries = make([]query.Query, 0, len(repoIDs)) + for _, repoID := range repoIDs { + repoQueries = append(repoQueries, numericEqualityQuery(repoID, "RepoID")) + } + + indexerQuery = bleve.NewConjunctionQuery( + bleve.NewDisjunctionQuery(repoQueries...), + phraseQuery, + ) + } else { + indexerQuery = phraseQuery + } + from := (page - 1) * pageSize searchRequest := bleve.NewSearchRequestOptions(indexerQuery, pageSize, from, false) - searchRequest.Fields = []string{"Content"} + searchRequest.Fields = []string{"Content", "RepoID"} searchRequest.IncludeLocations = true result, err := repoIndexer.Search(searchRequest) @@ -199,6 +213,7 @@ func SearchRepoByKeyword(repoID int64, keyword string, page, pageSize int) (int6 } } searchResults[i] = &RepoSearchResult{ + RepoID: int64(hit.Fields["RepoID"].(float64)), StartIndex: startIndex, EndIndex: endIndex, Filename: filenameOfIndexerID(hit.ID), diff --git a/modules/search/search.go b/modules/search/search.go index db0c0a616..9b93fe58f 100644 --- a/modules/search/search.go +++ b/modules/search/search.go @@ -17,6 +17,7 @@ import ( // Result a search result to display type Result struct { + RepoID int64 Filename string HighlightClass string LineNumbers []int @@ -98,6 +99,7 @@ func searchResult(result *indexer.RepoSearchResult, startIndex, endIndex int) (* index += len(line) } return &Result{ + RepoID: result.RepoID, Filename: result.Filename, HighlightClass: highlight.FileNameToHighlightClass(result.Filename), LineNumbers: lineNumbers, @@ -106,12 +108,12 @@ func searchResult(result *indexer.RepoSearchResult, startIndex, endIndex int) (* } // PerformSearch perform a search on a repository -func PerformSearch(repoID int64, keyword string, page, pageSize int) (int, []*Result, error) { +func PerformSearch(repoIDs []int64, keyword string, page, pageSize int) (int, []*Result, error) { if len(keyword) == 0 { return 0, nil, nil } - total, results, err := indexer.SearchRepoByKeyword(repoID, keyword, page, pageSize) + total, results, err := indexer.SearchRepoByKeyword(repoIDs, keyword, page, pageSize) if err != nil { return 0, nil, err } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 8ff353e41..2e51f68c2 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -169,9 +169,12 @@ repos = Repositories users = Users organizations = Organizations search = Search +code = Code repo_no_results = No matching repositories have been found. user_no_results = No matching users have been found. org_no_results = No matching organizations have been found. +code_no_results = No matching codes have been found. +code_search_results = Search results for "%s" [auth] create_new_account = Create Account diff --git a/routers/home.go b/routers/home.go index 74125aa2d..4810eb4e6 100644 --- a/routers/home.go +++ b/routers/home.go @@ -11,6 +11,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/search" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/routers/user" @@ -27,6 +28,8 @@ const ( tplExploreUsers base.TplName = "explore/users" // tplExploreOrganizations explore organizations page template tplExploreOrganizations base.TplName = "explore/organizations" + // tplExploreCode explore code page template + tplExploreCode base.TplName = "explore/code" ) // Home render home page @@ -49,6 +52,7 @@ func Home(ctx *context.Context) { } ctx.Data["PageIsHome"] = true + ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled ctx.HTML(200, tplHome) } @@ -124,6 +128,7 @@ func RenderRepoSearch(ctx *context.Context, opts *RepoSearchOptions) { ctx.Data["Total"] = count ctx.Data["Page"] = paginater.New(int(count), opts.PageSize, page, 5) ctx.Data["Repos"] = repos + ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled ctx.HTML(200, opts.TplName) } @@ -133,6 +138,7 @@ func ExploreRepos(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("explore") ctx.Data["PageIsExplore"] = true ctx.Data["PageIsExploreRepositories"] = true + ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled var ownerID int64 if ctx.User != nil && !ctx.User.IsAdmin { @@ -194,6 +200,7 @@ func RenderUserSearch(ctx *context.Context, opts *models.SearchUserOptions, tplN ctx.Data["Page"] = paginater.New(int(count), opts.PageSize, opts.Page, 5) ctx.Data["Users"] = users ctx.Data["ShowUserEmail"] = setting.UI.ShowUserEmail + ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled ctx.HTML(200, tplName) } @@ -203,6 +210,7 @@ func ExploreUsers(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("explore") ctx.Data["PageIsExplore"] = true ctx.Data["PageIsExploreUsers"] = true + ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled RenderUserSearch(ctx, &models.SearchUserOptions{ Type: models.UserTypeIndividual, @@ -216,6 +224,7 @@ func ExploreOrganizations(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("explore") ctx.Data["PageIsExplore"] = true ctx.Data["PageIsExploreOrganizations"] = true + ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled RenderUserSearch(ctx, &models.SearchUserOptions{ Type: models.UserTypeOrganization, @@ -223,6 +232,113 @@ func ExploreOrganizations(ctx *context.Context) { }, tplExploreOrganizations) } +// ExploreCode render explore code page +func ExploreCode(ctx *context.Context) { + if !setting.Indexer.RepoIndexerEnabled { + ctx.Redirect("/explore", 302) + return + } + + ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled + ctx.Data["Title"] = ctx.Tr("explore") + ctx.Data["PageIsExplore"] = true + ctx.Data["PageIsExploreCode"] = true + + keyword := strings.TrimSpace(ctx.Query("q")) + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + + var ( + repoIDs []int64 + err error + isAdmin bool + userID int64 + ) + if ctx.User != nil { + userID = ctx.User.ID + isAdmin = ctx.User.IsAdmin + } + + // guest user or non-admin user + if ctx.User == nil || !isAdmin { + repoIDs, err = models.FindUserAccessibleRepoIDs(userID) + if err != nil { + ctx.ServerError("SearchResults", err) + return + } + } + + var ( + total int + searchResults []*search.Result + ) + + // if non-admin login user, we need check UnitTypeCode at first + if ctx.User != nil && len(repoIDs) > 0 { + repoMaps, err := models.GetRepositoriesMapByIDs(repoIDs) + if err != nil { + ctx.ServerError("SearchResults", err) + return + } + + var rightRepoMap = make(map[int64]*models.Repository, len(repoMaps)) + repoIDs = make([]int64, 0, len(repoMaps)) + for id, repo := range repoMaps { + if repo.CheckUnitUser(userID, isAdmin, models.UnitTypeCode) { + rightRepoMap[id] = repo + repoIDs = append(repoIDs, id) + } + } + + ctx.Data["RepoMaps"] = rightRepoMap + + total, searchResults, err = search.PerformSearch(repoIDs, keyword, page, setting.UI.RepoSearchPagingNum) + if err != nil { + ctx.ServerError("SearchResults", err) + return + } + // if non-login user or isAdmin, no need to check UnitTypeCode + } else if (ctx.User == nil && len(repoIDs) > 0) || isAdmin { + total, searchResults, err = search.PerformSearch(repoIDs, keyword, page, setting.UI.RepoSearchPagingNum) + if err != nil { + ctx.ServerError("SearchResults", err) + return + } + + var loadRepoIDs = make([]int64, 0, len(searchResults)) + for _, result := range searchResults { + var find bool + for _, id := range loadRepoIDs { + if id == result.RepoID { + find = true + break + } + } + if !find { + loadRepoIDs = append(loadRepoIDs, result.RepoID) + } + } + + repoMaps, err := models.GetRepositoriesMapByIDs(loadRepoIDs) + if err != nil { + ctx.ServerError("SearchResults", err) + return + } + + ctx.Data["RepoMaps"] = repoMaps + } + + ctx.Data["Keyword"] = keyword + pager := paginater.New(total, setting.UI.RepoSearchPagingNum, page, 5) + ctx.Data["Page"] = pager + ctx.Data["SearchResults"] = searchResults + ctx.Data["RequireHighlightJS"] = true + ctx.Data["PageIsViewCode"] = true + ctx.HTML(200, tplExploreCode) +} + // NotFound render 404 page func NotFound(ctx *context.Context) { ctx.Data["Title"] = "Page Not Found" diff --git a/routers/repo/search.go b/routers/repo/search.go index ed209f2dd..95715c30c 100644 --- a/routers/repo/search.go +++ b/routers/repo/search.go @@ -29,7 +29,8 @@ func Search(ctx *context.Context) { if page <= 0 { page = 1 } - total, searchResults, err := search.PerformSearch(ctx.Repo.Repository.ID, keyword, page, setting.UI.RepoSearchPagingNum) + total, searchResults, err := search.PerformSearch([]int64{ctx.Repo.Repository.ID}, + keyword, page, setting.UI.RepoSearchPagingNum) if err != nil { ctx.ServerError("SearchResults", err) return diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 1d95bb4c7..da7157080 100644 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -170,6 +170,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/repos", routers.ExploreRepos) m.Get("/users", routers.ExploreUsers) m.Get("/organizations", routers.ExploreOrganizations) + m.Get("/code", routers.ExploreCode) }, ignSignIn) m.Combo("/install", routers.InstallInit).Get(routers.Install). Post(bindIgnErr(auth.InstallForm{}), routers.InstallPost) diff --git a/templates/explore/code.tmpl b/templates/explore/code.tmpl new file mode 100644 index 000000000..600348621 --- /dev/null +++ b/templates/explore/code.tmpl @@ -0,0 +1,55 @@ +{{template "base/head" .}} +
+ {{template "explore/navbar" .}} +
+
+
+ + + +
+
+
+ +
+ {{if .SearchResults}} +

+ {{.i18n.Tr "explore.code_search_results" (.Keyword|Escape) | Str2html }} +

+
+ {{range $result := .SearchResults}} + {{$repo := (index $.RepoMaps .RepoID)}} +
+

+ {{$repo.FullName}} - {{.Filename}} + {{$.i18n.Tr "repo.diff.view_file"}} +

+
+
+ + + + + + + +
+ {{range .LineNumbers}} + {{.}} + {{end}} + 
    {{.FormattedLines}}
+
+
+
+ {{end}} +
+ {{else}} +
{{$.i18n.Tr "explore.code_no_results"}}
+ {{end}} +
+ + {{template "base/paginate" .}} +
+
+{{template "base/footer" .}} + diff --git a/templates/explore/navbar.tmpl b/templates/explore/navbar.tmpl index aa4a21d59..3bd52645e 100644 --- a/templates/explore/navbar.tmpl +++ b/templates/explore/navbar.tmpl @@ -8,4 +8,9 @@ {{.i18n.Tr "explore.organizations"}} + {{if .IsRepoIndexerEnabled}} + + {{.i18n.Tr "explore.code"}} + + {{end}}