diff --git a/integrations/mssql.ini.tmpl b/integrations/mssql.ini.tmpl index cfb359412..054f848bb 100644 --- a/integrations/mssql.ini.tmpl +++ b/integrations/mssql.ini.tmpl @@ -17,6 +17,9 @@ REPO_INDEXER_PATH = integrations/indexers-mssql/repos.bleve [queue.code_indexer] TYPE = immediate +[queue.push_update] +TYPE = immediate + [repository] ROOT = {{REPO_TEST_DIR}}integrations/gitea-integration-mssql/gitea-repositories @@ -89,4 +92,4 @@ LEVEL = Debug [security] INSTALL_LOCK = true SECRET_KEY = 9pCviYTWSb -INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.hhSVGOANkaKk3vfCd2jDOIww4pUk0xtg9JRde5UogyQ +INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.hhSVGOANkaKk3vfCd2jDOIww4pUk0xtg9JRde5UogyQ \ No newline at end of file diff --git a/integrations/mysql.ini.tmpl b/integrations/mysql.ini.tmpl index 5211a4693..1bdd95834 100644 --- a/integrations/mysql.ini.tmpl +++ b/integrations/mysql.ini.tmpl @@ -19,6 +19,9 @@ REPO_INDEXER_PATH = integrations/indexers-mysql/repos.bleve [queue.code_indexer] TYPE = immediate +[queue.push_update] +TYPE = immediate + [repository] ROOT = {{REPO_TEST_DIR}}integrations/gitea-integration-mysql/gitea-repositories @@ -109,4 +112,4 @@ LEVEL = Debug [security] INSTALL_LOCK = true SECRET_KEY = 9pCviYTWSb -INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.hhSVGOANkaKk3vfCd2jDOIww4pUk0xtg9JRde5UogyQ +INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.hhSVGOANkaKk3vfCd2jDOIww4pUk0xtg9JRde5UogyQ \ No newline at end of file diff --git a/integrations/mysql8.ini.tmpl b/integrations/mysql8.ini.tmpl index ca77babf4..e87b9c794 100644 --- a/integrations/mysql8.ini.tmpl +++ b/integrations/mysql8.ini.tmpl @@ -17,6 +17,9 @@ REPO_INDEXER_PATH = integrations/indexers-mysql8/repos.bleve [queue.code_indexer] TYPE = immediate +[queue.push_update] +TYPE = immediate + [repository] ROOT = {{REPO_TEST_DIR}}integrations/gitea-integration-mysql8/gitea-repositories @@ -83,3 +86,4 @@ LEVEL = Debug INSTALL_LOCK = true SECRET_KEY = 9pCviYTWSb INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.hhSVGOANkaKk3vfCd2jDOIww4pUk0xtg9JRde5UogyQ + diff --git a/integrations/pgsql.ini.tmpl b/integrations/pgsql.ini.tmpl index 802296cf6..5a2a2d6a5 100644 --- a/integrations/pgsql.ini.tmpl +++ b/integrations/pgsql.ini.tmpl @@ -18,6 +18,9 @@ REPO_INDEXER_PATH = integrations/indexers-pgsql/repos.bleve [queue.code_indexer] TYPE = immediate +[queue.push_update] +TYPE = immediate + [repository] ROOT = {{REPO_TEST_DIR}}integrations/gitea-integration-pgsql/gitea-repositories @@ -90,4 +93,4 @@ LEVEL = Debug [security] INSTALL_LOCK = true SECRET_KEY = 9pCviYTWSb -INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.hhSVGOANkaKk3vfCd2jDOIww4pUk0xtg9JRde5UogyQ +INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.hhSVGOANkaKk3vfCd2jDOIww4pUk0xtg9JRde5UogyQ \ No newline at end of file diff --git a/integrations/sqlite.ini.tmpl b/integrations/sqlite.ini.tmpl index 5d54c5f9f..4c0d40cc4 100644 --- a/integrations/sqlite.ini.tmpl +++ b/integrations/sqlite.ini.tmpl @@ -13,6 +13,9 @@ REPO_INDEXER_PATH = integrations/indexers-sqlite/repos.bleve [queue.code_indexer] TYPE = immediate +[queue.push_update] +TYPE = immediate + [repository] ROOT = {{REPO_TEST_DIR}}integrations/gitea-integration-sqlite/gitea-repositories diff --git a/modules/repofiles/action.go b/modules/repofiles/action.go index 05e9fc958..52cc89dba 100644 --- a/modules/repofiles/action.go +++ b/modules/repofiles/action.go @@ -5,7 +5,6 @@ package repofiles import ( - "encoding/json" "fmt" "html" "regexp" @@ -14,12 +13,9 @@ import ( "time" "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/references" "code.gitea.io/gitea/modules/repository" - "code.gitea.io/gitea/modules/setting" ) const ( @@ -220,140 +216,3 @@ func UpdateIssuesCommit(doer *models.User, repo *models.Repository, commits []*r } return nil } - -// CommitRepoActionOptions represent options of a new commit action. -type CommitRepoActionOptions struct { - PushUpdateOptions - - RepoOwnerID int64 - Commits *repository.PushCommits -} - -// CommitRepoAction adds new commit action to the repository, and prepare -// corresponding webhooks. -func CommitRepoAction(optsList ...*CommitRepoActionOptions) error { - var pusher *models.User - var repo *models.Repository - actions := make([]*models.Action, len(optsList)) - - for i, opts := range optsList { - if pusher == nil || pusher.Name != opts.PusherName { - var err error - pusher, err = models.GetUserByName(opts.PusherName) - if err != nil { - return fmt.Errorf("GetUserByName [%s]: %v", opts.PusherName, err) - } - } - - if repo == nil || repo.OwnerID != opts.RepoOwnerID || repo.Name != opts.RepoName { - var err error - if repo != nil { - // Change repository empty status and update last updated time. - if err := models.UpdateRepository(repo, false); err != nil { - return fmt.Errorf("UpdateRepository: %v", err) - } - } - repo, err = models.GetRepositoryByName(opts.RepoOwnerID, opts.RepoName) - if err != nil { - return fmt.Errorf("GetRepositoryByName [owner_id: %d, name: %s]: %v", opts.RepoOwnerID, opts.RepoName, err) - } - } - refName := git.RefEndName(opts.RefFullName) - - // Change default branch and empty status only if pushed ref is non-empty branch. - if repo.IsEmpty && opts.IsBranch() && !opts.IsDelRef() { - repo.DefaultBranch = refName - repo.IsEmpty = false - if refName != "master" { - gitRepo, err := git.OpenRepository(repo.RepoPath()) - if err != nil { - return err - } - if err := gitRepo.SetDefaultBranch(repo.DefaultBranch); err != nil { - if !git.IsErrUnsupportedVersion(err) { - gitRepo.Close() - return err - } - } - gitRepo.Close() - } - } - - opType := models.ActionCommitRepo - - // Check it's tag push or branch. - if opts.IsTag() { - opType = models.ActionPushTag - if opts.IsDelRef() { - opType = models.ActionDeleteTag - } - opts.Commits = &repository.PushCommits{} - } else if opts.IsDelRef() { - opType = models.ActionDeleteBranch - opts.Commits = &repository.PushCommits{} - } else { - // if not the first commit, set the compare URL. - if !opts.IsNewRef() { - opts.Commits.CompareURL = repo.ComposeCompareURL(opts.OldCommitID, opts.NewCommitID) - } - - if err := UpdateIssuesCommit(pusher, repo, opts.Commits.Commits, refName); err != nil { - log.Error("updateIssuesCommit: %v", err) - } - } - - if len(opts.Commits.Commits) > setting.UI.FeedMaxCommitNum { - opts.Commits.Commits = opts.Commits.Commits[:setting.UI.FeedMaxCommitNum] - } - - data, err := json.Marshal(opts.Commits) - if err != nil { - return fmt.Errorf("Marshal: %v", err) - } - - actions[i] = &models.Action{ - ActUserID: pusher.ID, - ActUser: pusher, - OpType: opType, - Content: string(data), - RepoID: repo.ID, - Repo: repo, - RefName: refName, - IsPrivate: repo.IsPrivate, - } - - var isHookEventPush = true - switch opType { - case models.ActionCommitRepo: // Push - if opts.IsNewBranch() { - notification.NotifyCreateRef(pusher, repo, "branch", opts.RefFullName) - } - case models.ActionDeleteBranch: // Delete Branch - notification.NotifyDeleteRef(pusher, repo, "branch", opts.RefFullName) - - case models.ActionPushTag: // Create - notification.NotifyCreateRef(pusher, repo, "tag", opts.RefFullName) - - case models.ActionDeleteTag: // Delete Tag - notification.NotifyDeleteRef(pusher, repo, "tag", opts.RefFullName) - default: - isHookEventPush = false - } - - if isHookEventPush { - notification.NotifyPushCommits(pusher, repo, opts.RefFullName, opts.OldCommitID, opts.NewCommitID, opts.Commits) - } - } - - if repo != nil { - // Change repository empty status and update last updated time. - if err := models.UpdateRepository(repo, false); err != nil { - return fmt.Errorf("UpdateRepository: %v", err) - } - } - - if err := models.NotifyWatchers(actions...); err != nil { - return fmt.Errorf("NotifyWatchers: %v", err) - } - return nil -} diff --git a/modules/repofiles/action_test.go b/modules/repofiles/action_test.go index 8ed3ba7b3..290844de0 100644 --- a/modules/repofiles/action_test.go +++ b/modules/repofiles/action_test.go @@ -8,133 +8,12 @@ import ( "testing" "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "github.com/stretchr/testify/assert" ) -func testCorrectRepoAction(t *testing.T, opts *CommitRepoActionOptions, actionBean *models.Action) { - models.AssertNotExistsBean(t, actionBean) - assert.NoError(t, CommitRepoAction(opts)) - models.AssertExistsAndLoadBean(t, actionBean) - models.CheckConsistencyFor(t, &models.Action{}) -} - -func TestCommitRepoAction(t *testing.T) { - samples := []struct { - userID int64 - repositoryID int64 - commitRepoActionOptions CommitRepoActionOptions - action models.Action - }{ - { - userID: 2, - repositoryID: 16, - commitRepoActionOptions: CommitRepoActionOptions{ - PushUpdateOptions: PushUpdateOptions{ - RefFullName: "refName", - OldCommitID: "oldCommitID", - NewCommitID: "newCommitID", - }, - Commits: &repository.PushCommits{ - Commits: []*repository.PushCommit{ - { - Sha1: "69554a6", - CommitterEmail: "user2@example.com", - CommitterName: "User2", - AuthorEmail: "user2@example.com", - AuthorName: "User2", - Message: "not signed commit", - }, - { - Sha1: "27566bd", - CommitterEmail: "user2@example.com", - CommitterName: "User2", - AuthorEmail: "user2@example.com", - AuthorName: "User2", - Message: "good signed commit (with not yet validated email)", - }, - }, - Len: 2, - }, - }, - action: models.Action{ - OpType: models.ActionCommitRepo, - RefName: "refName", - }, - }, - { - userID: 2, - repositoryID: 1, - commitRepoActionOptions: CommitRepoActionOptions{ - PushUpdateOptions: PushUpdateOptions{ - RefFullName: git.TagPrefix + "v1.1", - OldCommitID: git.EmptySHA, - NewCommitID: "newCommitID", - }, - Commits: &repository.PushCommits{}, - }, - action: models.Action{ - OpType: models.ActionPushTag, - RefName: "v1.1", - }, - }, - { - userID: 2, - repositoryID: 1, - commitRepoActionOptions: CommitRepoActionOptions{ - PushUpdateOptions: PushUpdateOptions{ - RefFullName: git.TagPrefix + "v1.1", - OldCommitID: "oldCommitID", - NewCommitID: git.EmptySHA, - }, - Commits: &repository.PushCommits{}, - }, - action: models.Action{ - OpType: models.ActionDeleteTag, - RefName: "v1.1", - }, - }, - { - userID: 2, - repositoryID: 1, - commitRepoActionOptions: CommitRepoActionOptions{ - PushUpdateOptions: PushUpdateOptions{ - RefFullName: git.BranchPrefix + "feature/1", - OldCommitID: "oldCommitID", - NewCommitID: git.EmptySHA, - }, - Commits: &repository.PushCommits{}, - }, - action: models.Action{ - OpType: models.ActionDeleteBranch, - RefName: "feature/1", - }, - }, - } - - for _, s := range samples { - models.PrepareTestEnv(t) - - user := models.AssertExistsAndLoadBean(t, &models.User{ID: s.userID}).(*models.User) - repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: s.repositoryID, OwnerID: user.ID}).(*models.Repository) - repo.Owner = user - - s.commitRepoActionOptions.PusherName = user.Name - s.commitRepoActionOptions.RepoOwnerID = user.ID - s.commitRepoActionOptions.RepoName = repo.Name - - s.action.ActUserID = user.ID - s.action.RepoID = repo.ID - s.action.Repo = repo - s.action.IsPrivate = repo.IsPrivate - - testCorrectRepoAction(t, &s.commitRepoActionOptions, &s.action) - } -} - func TestUpdateIssuesCommit(t *testing.T) { assert.NoError(t, models.PrepareTestDatabase()) pushCommits := []*repository.PushCommit{ diff --git a/modules/repofiles/update.go b/modules/repofiles/update.go index 84a3bcb64..dcb87ec92 100644 --- a/modules/repofiles/update.go +++ b/modules/repofiles/update.go @@ -6,14 +6,12 @@ package repofiles import ( "bytes" - "container/list" "fmt" "path" "strings" "time" "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/lfs" @@ -22,7 +20,6 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/structs" - pull_service "code.gitea.io/gitea/services/pull" stdcharset "golang.org/x/net/html/charset" "golang.org/x/text/transform" @@ -466,298 +463,3 @@ func CreateOrUpdateRepoFile(repo *models.Repository, doer *models.User, opts *Up } return file, nil } - -// PushUpdateOptions defines the push update options -type PushUpdateOptions struct { - PusherID int64 - PusherName string - RepoUserName string - RepoName string - RefFullName string - OldCommitID string - NewCommitID string -} - -// IsNewRef return true if it's a first-time push to a branch, tag or etc. -func (opts PushUpdateOptions) IsNewRef() bool { - return opts.OldCommitID == git.EmptySHA -} - -// IsDelRef return true if it's a deletion to a branch or tag -func (opts PushUpdateOptions) IsDelRef() bool { - return opts.NewCommitID == git.EmptySHA -} - -// IsUpdateRef return true if it's an update operation -func (opts PushUpdateOptions) IsUpdateRef() bool { - return !opts.IsNewRef() && !opts.IsDelRef() -} - -// IsTag return true if it's an operation to a tag -func (opts PushUpdateOptions) IsTag() bool { - return strings.HasPrefix(opts.RefFullName, git.TagPrefix) -} - -// IsNewTag return true if it's a creation to a tag -func (opts PushUpdateOptions) IsNewTag() bool { - return opts.IsTag() && opts.IsNewRef() -} - -// IsDelTag return true if it's a deletion to a tag -func (opts PushUpdateOptions) IsDelTag() bool { - return opts.IsTag() && opts.IsDelRef() -} - -// IsBranch return true if it's a push to branch -func (opts PushUpdateOptions) IsBranch() bool { - return strings.HasPrefix(opts.RefFullName, git.BranchPrefix) -} - -// IsNewBranch return true if it's the first-time push to a branch -func (opts PushUpdateOptions) IsNewBranch() bool { - return opts.IsBranch() && opts.IsNewRef() -} - -// IsUpdateBranch return true if it's not the first push to a branch -func (opts PushUpdateOptions) IsUpdateBranch() bool { - return opts.IsBranch() && opts.IsUpdateRef() -} - -// IsDelBranch return true if it's a deletion to a branch -func (opts PushUpdateOptions) IsDelBranch() bool { - return opts.IsBranch() && opts.IsDelRef() -} - -// TagName returns simple tag name if it's an operation to a tag -func (opts PushUpdateOptions) TagName() string { - return opts.RefFullName[len(git.TagPrefix):] -} - -// BranchName returns simple branch name if it's an operation to branch -func (opts PushUpdateOptions) BranchName() string { - return opts.RefFullName[len(git.BranchPrefix):] -} - -// RepoFullName returns repo full name -func (opts PushUpdateOptions) RepoFullName() string { - return opts.RepoUserName + "/" + opts.RepoName -} - -// PushUpdate must be called for any push actions in order to -// generates necessary push action history feeds and other operations -func PushUpdate(repo *models.Repository, branch string, opts PushUpdateOptions) error { - if opts.IsNewRef() && opts.IsDelRef() { - return fmt.Errorf("Old and new revisions are both %s", git.EmptySHA) - } - - repoPath := models.RepoPath(opts.RepoUserName, opts.RepoName) - - _, err := git.NewCommand("update-server-info").RunInDir(repoPath) - if err != nil { - return fmt.Errorf("Failed to call 'git update-server-info': %v", err) - } - - gitRepo, err := git.OpenRepository(repoPath) - if err != nil { - return fmt.Errorf("OpenRepository: %v", err) - } - defer gitRepo.Close() - - if err = repo.UpdateSize(models.DefaultDBContext()); err != nil { - log.Error("Failed to update size for repository: %v", err) - } - - var commits = &repo_module.PushCommits{} - - if opts.IsTag() { // If is tag reference - tagName := opts.TagName() - if opts.IsDelRef() { - if err := models.PushUpdateDeleteTag(repo, tagName); err != nil { - return fmt.Errorf("PushUpdateDeleteTag: %v", err) - } - } else { - // Clear cache for tag commit count - cache.Remove(repo.GetCommitsCountCacheKey(tagName, true)) - if err := repo_module.PushUpdateAddTag(repo, gitRepo, tagName); err != nil { - return fmt.Errorf("PushUpdateAddTag: %v", err) - } - } - } else if opts.IsBranch() { // If is branch reference - pusher, err := models.GetUserByID(opts.PusherID) - if err != nil { - return err - } - - if !opts.IsDelRef() { - // Clear cache for branch commit count - cache.Remove(repo.GetCommitsCountCacheKey(opts.BranchName(), true)) - - newCommit, err := gitRepo.GetCommit(opts.NewCommitID) - if err != nil { - return fmt.Errorf("gitRepo.GetCommit: %v", err) - } - - // Push new branch. - var l *list.List - if opts.IsNewRef() { - l, err = newCommit.CommitsBeforeLimit(10) - if err != nil { - return fmt.Errorf("newCommit.CommitsBeforeLimit: %v", err) - } - } else { - l, err = newCommit.CommitsBeforeUntil(opts.OldCommitID) - if err != nil { - return fmt.Errorf("newCommit.CommitsBeforeUntil: %v", err) - } - } - - commits = repo_module.ListToPushCommits(l) - - if err = models.RemoveDeletedBranch(repo.ID, opts.BranchName()); err != nil { - log.Error("models.RemoveDeletedBranch %s/%s failed: %v", repo.ID, opts.BranchName(), err) - } - - if err = models.WatchIfAuto(opts.PusherID, repo.ID, true); err != nil { - log.Warn("Fail to perform auto watch on user %v for repo %v: %v", opts.PusherID, repo.ID, err) - } - - log.Trace("TriggerTask '%s/%s' by %s", repo.Name, branch, pusher.Name) - - go pull_service.AddTestPullRequestTask(pusher, repo.ID, branch, true, opts.OldCommitID, opts.NewCommitID) - } else if err = pull_service.CloseBranchPulls(pusher, repo.ID, branch); err != nil { - // close all related pulls - log.Error("close related pull request failed: %v", err) - } - } - - if err := CommitRepoAction(&CommitRepoActionOptions{ - PushUpdateOptions: opts, - RepoOwnerID: repo.OwnerID, - Commits: commits, - }); err != nil { - return fmt.Errorf("CommitRepoAction: %v", err) - } - - return nil -} - -// PushUpdates generates push action history feeds for push updating multiple refs -func PushUpdates(repo *models.Repository, optsList []*PushUpdateOptions) error { - repoPath := repo.RepoPath() - _, err := git.NewCommand("update-server-info").RunInDir(repoPath) - if err != nil { - return fmt.Errorf("Failed to call 'git update-server-info': %v", err) - } - gitRepo, err := git.OpenRepository(repoPath) - if err != nil { - return fmt.Errorf("OpenRepository: %v", err) - } - defer gitRepo.Close() - - if err = repo.UpdateSize(models.DefaultDBContext()); err != nil { - log.Error("Failed to update size for repository: %v", err) - } - - actions, err := createCommitRepoActions(repo, gitRepo, optsList) - if err != nil { - return err - } - if err := CommitRepoAction(actions...); err != nil { - return fmt.Errorf("CommitRepoAction: %v", err) - } - - var pusher *models.User - - for _, opts := range optsList { - if !opts.IsBranch() { - continue - } - - branch := opts.BranchName() - - if pusher == nil || pusher.ID != opts.PusherID { - var err error - pusher, err = models.GetUserByID(opts.PusherID) - if err != nil { - return err - } - } - - if !opts.IsDelRef() { - if err = models.RemoveDeletedBranch(repo.ID, branch); err != nil { - log.Error("models.RemoveDeletedBranch %s/%s failed: %v", repo.ID, branch, err) - } - - if err = models.WatchIfAuto(opts.PusherID, repo.ID, true); err != nil { - log.Warn("Fail to perform auto watch on user %v for repo %v: %v", opts.PusherID, repo.ID, err) - } - - log.Trace("TriggerTask '%s/%s' by %s", repo.Name, branch, pusher.Name) - - go pull_service.AddTestPullRequestTask(pusher, repo.ID, branch, true, opts.OldCommitID, opts.NewCommitID) - // close all related pulls - } else if err = pull_service.CloseBranchPulls(pusher, repo.ID, branch); err != nil { - log.Error("close related pull request failed: %v", err) - } - } - - return nil -} - -func createCommitRepoActions(repo *models.Repository, gitRepo *git.Repository, optsList []*PushUpdateOptions) ([]*CommitRepoActionOptions, error) { - addTags := make([]string, 0, len(optsList)) - delTags := make([]string, 0, len(optsList)) - actions := make([]*CommitRepoActionOptions, 0, len(optsList)) - - for _, opts := range optsList { - if opts.IsNewRef() && opts.IsDelRef() { - return nil, fmt.Errorf("Old and new revisions are both %s", git.EmptySHA) - } - var commits = &repo_module.PushCommits{} - if opts.IsTag() { - // If is tag reference - tagName := opts.TagName() - if opts.IsDelRef() { - delTags = append(delTags, tagName) - } else { - cache.Remove(repo.GetCommitsCountCacheKey(tagName, true)) - addTags = append(addTags, tagName) - } - } else if !opts.IsDelRef() { - // If is branch reference - - // Clear cache for branch commit count - cache.Remove(repo.GetCommitsCountCacheKey(opts.BranchName(), true)) - - newCommit, err := gitRepo.GetCommit(opts.NewCommitID) - if err != nil { - return nil, fmt.Errorf("gitRepo.GetCommit: %v", err) - } - - // Push new branch. - var l *list.List - if opts.IsNewRef() { - l, err = newCommit.CommitsBeforeLimit(10) - if err != nil { - return nil, fmt.Errorf("newCommit.CommitsBeforeLimit: %v", err) - } - } else { - l, err = newCommit.CommitsBeforeUntil(opts.OldCommitID) - if err != nil { - return nil, fmt.Errorf("newCommit.CommitsBeforeUntil: %v", err) - } - } - - commits = repo_module.ListToPushCommits(l) - } - actions = append(actions, &CommitRepoActionOptions{ - PushUpdateOptions: *opts, - RepoOwnerID: repo.OwnerID, - Commits: commits, - }) - } - if err := repo_module.PushUpdateAddDeleteTags(repo, gitRepo, addTags, delTags); err != nil { - return nil, fmt.Errorf("PushUpdateAddDeleteTags: %v", err) - } - return actions, nil -} diff --git a/routers/api/v1/repo/branch.go b/routers/api/v1/repo/branch.go index 90db597ef..6c63bde94 100644 --- a/routers/api/v1/repo/branch.go +++ b/routers/api/v1/repo/branch.go @@ -14,9 +14,9 @@ import ( "code.gitea.io/gitea/modules/convert" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/repofiles" repo_module "code.gitea.io/gitea/modules/repository" api "code.gitea.io/gitea/modules/structs" + repo_service "code.gitea.io/gitea/services/repository" ) // GetBranch get a branch of a repository @@ -160,10 +160,8 @@ func DeleteBranch(ctx *context.APIContext) { } // Don't return error below this - if err := repofiles.PushUpdate( - ctx.Repo.Repository, - ctx.Repo.BranchName, - repofiles.PushUpdateOptions{ + if err := repo_service.PushUpdate( + &repo_service.PushUpdateOptions{ RefFullName: git.BranchPrefix + ctx.Repo.BranchName, OldCommitID: c.ID.String(), NewCommitID: git.EmptySHA, diff --git a/routers/init.go b/routers/init.go index 2f12058ac..793033f4a 100644 --- a/routers/init.go +++ b/routers/init.go @@ -35,6 +35,7 @@ import ( "code.gitea.io/gitea/services/mailer" mirror_service "code.gitea.io/gitea/services/mirror" pull_service "code.gitea.io/gitea/services/pull" + "code.gitea.io/gitea/services/repository" "gitea.com/macaron/i18n" "gitea.com/macaron/macaron" @@ -58,6 +59,9 @@ func NewServices() { if err := storage.Init(); err != nil { log.Fatal("storage init failed: %v", err) } + if err := repository.NewContext(); err != nil { + log.Fatal("repository init failed: %v", err) + } mailer.NewContext() _ = cache.NewContext() notification.NewContext() diff --git a/routers/private/hook.go b/routers/private/hook.go index 2bccca3e3..05f0b124c 100644 --- a/routers/private/hook.go +++ b/routers/private/hook.go @@ -18,10 +18,10 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/private" - "code.gitea.io/gitea/modules/repofiles" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" pull_service "code.gitea.io/gitea/services/pull" + repo_service "code.gitea.io/gitea/services/repository" "gitea.com/macaron/macaron" "github.com/go-git/go-git/v5/plumbing" @@ -376,7 +376,7 @@ func HookPostReceive(ctx *macaron.Context, opts private.HookOptions) { repoName := ctx.Params(":repo") var repo *models.Repository - updates := make([]*repofiles.PushUpdateOptions, 0, len(opts.OldCommitIDs)) + updates := make([]*repo_service.PushUpdateOptions, 0, len(opts.OldCommitIDs)) wasEmpty := false for i := range opts.OldCommitIDs { @@ -403,7 +403,7 @@ func HookPostReceive(ctx *macaron.Context, opts private.HookOptions) { wasEmpty = repo.IsEmpty } - option := repofiles.PushUpdateOptions{ + option := repo_service.PushUpdateOptions{ RefFullName: refFullName, OldCommitID: opts.OldCommitIDs[i], NewCommitID: opts.NewCommitIDs[i], @@ -422,7 +422,7 @@ func HookPostReceive(ctx *macaron.Context, opts private.HookOptions) { } if repo != nil && len(updates) > 0 { - if err := repofiles.PushUpdates(repo, updates); err != nil { + if err := repo_service.PushUpdates(updates); err != nil { log.Error("Failed to Update: %s/%s Total Updates: %d", ownerName, repoName, len(updates)) for i, update := range updates { log.Error("Failed to Update: %s/%s Update: %d/%d: Branch: %s", ownerName, repoName, i, len(updates), update.BranchName()) diff --git a/routers/repo/branch.go b/routers/repo/branch.go index 4d8b9158f..0ca77cbf6 100644 --- a/routers/repo/branch.go +++ b/routers/repo/branch.go @@ -19,6 +19,7 @@ import ( repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/routers/utils" + repo_service "code.gitea.io/gitea/services/repository" ) const ( @@ -118,10 +119,8 @@ func RestoreBranchPost(ctx *context.Context) { } // Don't return error below this - if err := repofiles.PushUpdate( - ctx.Repo.Repository, - deletedBranch.Name, - repofiles.PushUpdateOptions{ + if err := repo_service.PushUpdate( + &repo_service.PushUpdateOptions{ RefFullName: git.BranchPrefix + deletedBranch.Name, OldCommitID: git.EmptySHA, NewCommitID: deletedBranch.Commit, @@ -157,10 +156,8 @@ func deleteBranch(ctx *context.Context, branchName string) error { } // Don't return error below this - if err := repofiles.PushUpdate( - ctx.Repo.Repository, - branchName, - repofiles.PushUpdateOptions{ + if err := repo_service.PushUpdate( + &repo_service.PushUpdateOptions{ RefFullName: git.BranchPrefix + branchName, OldCommitID: commit.ID.String(), NewCommitID: git.EmptySHA, diff --git a/routers/repo/pull.go b/routers/repo/pull.go index a19dbb5cb..a6f7a7074 100644 --- a/routers/repo/pull.go +++ b/routers/repo/pull.go @@ -22,7 +22,6 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/notification" - "code.gitea.io/gitea/modules/repofiles" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" @@ -1124,10 +1123,8 @@ func CleanUpPullRequest(ctx *context.Context) { return } - if err := repofiles.PushUpdate( - pr.HeadRepo, - pr.HeadBranch, - repofiles.PushUpdateOptions{ + if err := repo_service.PushUpdate( + &repo_service.PushUpdateOptions{ RefFullName: git.BranchPrefix + pr.HeadBranch, OldCommitID: branchCommitID, NewCommitID: git.EmptySHA, diff --git a/services/repository/push.go b/services/repository/push.go new file mode 100644 index 000000000..05871dea5 --- /dev/null +++ b/services/repository/push.go @@ -0,0 +1,371 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repository + +import ( + "container/list" + "encoding/json" + "fmt" + "strings" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/notification" + "code.gitea.io/gitea/modules/queue" + "code.gitea.io/gitea/modules/repofiles" + repo_module "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/setting" + pull_service "code.gitea.io/gitea/services/pull" +) + +// PushUpdateOptions defines the push update options +type PushUpdateOptions struct { + PusherID int64 + PusherName string + RepoUserName string + RepoName string + RefFullName string + OldCommitID string + NewCommitID string +} + +// IsNewRef return true if it's a first-time push to a branch, tag or etc. +func (opts PushUpdateOptions) IsNewRef() bool { + return opts.OldCommitID == git.EmptySHA +} + +// IsDelRef return true if it's a deletion to a branch or tag +func (opts PushUpdateOptions) IsDelRef() bool { + return opts.NewCommitID == git.EmptySHA +} + +// IsUpdateRef return true if it's an update operation +func (opts PushUpdateOptions) IsUpdateRef() bool { + return !opts.IsNewRef() && !opts.IsDelRef() +} + +// IsTag return true if it's an operation to a tag +func (opts PushUpdateOptions) IsTag() bool { + return strings.HasPrefix(opts.RefFullName, git.TagPrefix) +} + +// IsNewTag return true if it's a creation to a tag +func (opts PushUpdateOptions) IsNewTag() bool { + return opts.IsTag() && opts.IsNewRef() +} + +// IsDelTag return true if it's a deletion to a tag +func (opts PushUpdateOptions) IsDelTag() bool { + return opts.IsTag() && opts.IsDelRef() +} + +// IsBranch return true if it's a push to branch +func (opts PushUpdateOptions) IsBranch() bool { + return strings.HasPrefix(opts.RefFullName, git.BranchPrefix) +} + +// IsNewBranch return true if it's the first-time push to a branch +func (opts PushUpdateOptions) IsNewBranch() bool { + return opts.IsBranch() && opts.IsNewRef() +} + +// IsUpdateBranch return true if it's not the first push to a branch +func (opts PushUpdateOptions) IsUpdateBranch() bool { + return opts.IsBranch() && opts.IsUpdateRef() +} + +// IsDelBranch return true if it's a deletion to a branch +func (opts PushUpdateOptions) IsDelBranch() bool { + return opts.IsBranch() && opts.IsDelRef() +} + +// TagName returns simple tag name if it's an operation to a tag +func (opts PushUpdateOptions) TagName() string { + return opts.RefFullName[len(git.TagPrefix):] +} + +// BranchName returns simple branch name if it's an operation to branch +func (opts PushUpdateOptions) BranchName() string { + return opts.RefFullName[len(git.BranchPrefix):] +} + +// RepoFullName returns repo full name +func (opts PushUpdateOptions) RepoFullName() string { + return opts.RepoUserName + "/" + opts.RepoName +} + +// pushQueue represents a queue to handle update pull request tests +var pushQueue queue.Queue + +// handle passed PR IDs and test the PRs +func handle(data ...queue.Data) { + for _, datum := range data { + opts := datum.([]*PushUpdateOptions) + if err := pushUpdates(opts); err != nil { + log.Error("pushUpdate failed: %v", err) + } + } +} + +func initPushQueue() error { + pushQueue = queue.CreateQueue("push_update", handle, []*PushUpdateOptions{}).(queue.Queue) + if pushQueue == nil { + return fmt.Errorf("Unable to create push_update Queue") + } + + go graceful.GetManager().RunWithShutdownFns(pushQueue.Run) + return nil +} + +// PushUpdate is an alias of PushUpdates for single push update options +func PushUpdate(opts *PushUpdateOptions) error { + return PushUpdates([]*PushUpdateOptions{opts}) +} + +// PushUpdates adds a push update to push queue +func PushUpdates(opts []*PushUpdateOptions) error { + if len(opts) == 0 { + return nil + } + + for _, opt := range opts { + if opt.IsNewRef() && opt.IsDelRef() { + return fmt.Errorf("Old and new revisions are both %s", git.EmptySHA) + } + } + + return pushQueue.Push(opts) +} + +// pushUpdates generates push action history feeds for push updating multiple refs +func pushUpdates(optsList []*PushUpdateOptions) error { + if len(optsList) == 0 { + return nil + } + + repo, err := models.GetRepositoryByOwnerAndName(optsList[0].RepoUserName, optsList[0].RepoName) + if err != nil { + return fmt.Errorf("GetRepositoryByOwnerAndName failed: %v", err) + } + + repoPath := repo.RepoPath() + _, err = git.NewCommand("update-server-info").RunInDir(repoPath) + if err != nil { + return fmt.Errorf("Failed to call 'git update-server-info': %v", err) + } + gitRepo, err := git.OpenRepository(repoPath) + if err != nil { + return fmt.Errorf("OpenRepository: %v", err) + } + defer gitRepo.Close() + + if err = repo.UpdateSize(models.DefaultDBContext()); err != nil { + log.Error("Failed to update size for repository: %v", err) + } + + addTags := make([]string, 0, len(optsList)) + delTags := make([]string, 0, len(optsList)) + actions := make([]*commitRepoActionOptions, 0, len(optsList)) + var pusher *models.User + + for _, opts := range optsList { + if opts.IsNewRef() && opts.IsDelRef() { + return fmt.Errorf("Old and new revisions are both %s", git.EmptySHA) + } + var commits = &repo_module.PushCommits{} + if opts.IsTag() { // If is tag reference { + tagName := opts.TagName() + if opts.IsDelRef() { + delTags = append(delTags, tagName) + } else { // is new tag + cache.Remove(repo.GetCommitsCountCacheKey(tagName, true)) + addTags = append(addTags, tagName) + } + } else if opts.IsBranch() { // If is branch reference + if pusher == nil || pusher.ID != opts.PusherID { + var err error + if pusher, err = models.GetUserByID(opts.PusherID); err != nil { + return err + } + } + + branch := opts.BranchName() + if !opts.IsDelRef() { + // Clear cache for branch commit count + cache.Remove(repo.GetCommitsCountCacheKey(opts.BranchName(), true)) + + newCommit, err := gitRepo.GetCommit(opts.NewCommitID) + if err != nil { + return fmt.Errorf("gitRepo.GetCommit: %v", err) + } + + // Push new branch. + var l *list.List + if opts.IsNewRef() { + l, err = newCommit.CommitsBeforeLimit(10) + if err != nil { + return fmt.Errorf("newCommit.CommitsBeforeLimit: %v", err) + } + } else { + l, err = newCommit.CommitsBeforeUntil(opts.OldCommitID) + if err != nil { + return fmt.Errorf("newCommit.CommitsBeforeUntil: %v", err) + } + } + + commits = repo_module.ListToPushCommits(l) + + if err = models.RemoveDeletedBranch(repo.ID, branch); err != nil { + log.Error("models.RemoveDeletedBranch %s/%s failed: %v", repo.ID, branch, err) + } + + log.Trace("TriggerTask '%s/%s' by %s", repo.Name, branch, pusher.Name) + + go pull_service.AddTestPullRequestTask(pusher, repo.ID, branch, true, opts.OldCommitID, opts.NewCommitID) + } else if err = pull_service.CloseBranchPulls(pusher, repo.ID, branch); err != nil { + // close all related pulls + log.Error("close related pull request failed: %v", err) + } + + // Even if user delete a branch on a repository which he didn't watch, he will be watch that. + if err = models.WatchIfAuto(opts.PusherID, repo.ID, true); err != nil { + log.Warn("Fail to perform auto watch on user %v for repo %v: %v", opts.PusherID, repo.ID, err) + } + } + actions = append(actions, &commitRepoActionOptions{ + PushUpdateOptions: *opts, + Pusher: pusher, + RepoOwnerID: repo.OwnerID, + Commits: commits, + }) + } + if err := repo_module.PushUpdateAddDeleteTags(repo, gitRepo, addTags, delTags); err != nil { + return fmt.Errorf("PushUpdateAddDeleteTags: %v", err) + } + + if err := commitRepoAction(repo, gitRepo, actions...); err != nil { + return fmt.Errorf("commitRepoAction: %v", err) + } + + return nil +} + +// commitRepoActionOptions represent options of a new commit action. +type commitRepoActionOptions struct { + PushUpdateOptions + + Pusher *models.User + RepoOwnerID int64 + Commits *repo_module.PushCommits +} + +// commitRepoAction adds new commit action to the repository, and prepare +// corresponding webhooks. +func commitRepoAction(repo *models.Repository, gitRepo *git.Repository, optsList ...*commitRepoActionOptions) error { + actions := make([]*models.Action, len(optsList)) + + for i, opts := range optsList { + if opts.Pusher == nil || opts.Pusher.Name != opts.PusherName { + var err error + opts.Pusher, err = models.GetUserByName(opts.PusherName) + if err != nil { + return fmt.Errorf("GetUserByName [%s]: %v", opts.PusherName, err) + } + } + + refName := git.RefEndName(opts.RefFullName) + + // Change default branch and empty status only if pushed ref is non-empty branch. + if repo.IsEmpty && opts.IsBranch() && !opts.IsDelRef() { + repo.DefaultBranch = refName + repo.IsEmpty = false + if refName != "master" { + if err := gitRepo.SetDefaultBranch(repo.DefaultBranch); err != nil { + if !git.IsErrUnsupportedVersion(err) { + return err + } + } + } + } + + opType := models.ActionCommitRepo + + // Check it's tag push or branch. + if opts.IsTag() { + opType = models.ActionPushTag + if opts.IsDelRef() { + opType = models.ActionDeleteTag + } + opts.Commits = &repo_module.PushCommits{} + } else if opts.IsDelRef() { + opType = models.ActionDeleteBranch + opts.Commits = &repo_module.PushCommits{} + } else { + // if not the first commit, set the compare URL. + if !opts.IsNewRef() { + opts.Commits.CompareURL = repo.ComposeCompareURL(opts.OldCommitID, opts.NewCommitID) + } + + if err := repofiles.UpdateIssuesCommit(opts.Pusher, repo, opts.Commits.Commits, refName); err != nil { + log.Error("updateIssuesCommit: %v", err) + } + } + + if len(opts.Commits.Commits) > setting.UI.FeedMaxCommitNum { + opts.Commits.Commits = opts.Commits.Commits[:setting.UI.FeedMaxCommitNum] + } + + data, err := json.Marshal(opts.Commits) + if err != nil { + return fmt.Errorf("Marshal: %v", err) + } + + actions[i] = &models.Action{ + ActUserID: opts.Pusher.ID, + ActUser: opts.Pusher, + OpType: opType, + Content: string(data), + RepoID: repo.ID, + Repo: repo, + RefName: refName, + IsPrivate: repo.IsPrivate, + } + + var isHookEventPush = true + switch opType { + case models.ActionCommitRepo: // Push + if opts.IsNewBranch() { + notification.NotifyCreateRef(opts.Pusher, repo, "branch", opts.RefFullName) + } + case models.ActionDeleteBranch: // Delete Branch + notification.NotifyDeleteRef(opts.Pusher, repo, "branch", opts.RefFullName) + + case models.ActionPushTag: // Create + notification.NotifyCreateRef(opts.Pusher, repo, "tag", opts.RefFullName) + + case models.ActionDeleteTag: // Delete Tag + notification.NotifyDeleteRef(opts.Pusher, repo, "tag", opts.RefFullName) + default: + isHookEventPush = false + } + + if isHookEventPush { + notification.NotifyPushCommits(opts.Pusher, repo, opts.RefFullName, opts.OldCommitID, opts.NewCommitID, opts.Commits) + } + } + + // Change repository empty status and update last updated time. + if err := models.UpdateRepository(repo, false); err != nil { + return fmt.Errorf("UpdateRepository: %v", err) + } + + if err := models.NotifyWatchers(actions...); err != nil { + return fmt.Errorf("NotifyWatchers: %v", err) + } + return nil +} diff --git a/services/repository/push_test.go b/services/repository/push_test.go new file mode 100644 index 000000000..19ffab45e --- /dev/null +++ b/services/repository/push_test.go @@ -0,0 +1,139 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package repository + +import ( + "testing" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" + repo_module "code.gitea.io/gitea/modules/repository" + + "github.com/stretchr/testify/assert" +) + +func testCorrectRepoAction(t *testing.T, repo *models.Repository, gitRepo *git.Repository, opts *commitRepoActionOptions, actionBean *models.Action) { + models.AssertNotExistsBean(t, actionBean) + assert.NoError(t, commitRepoAction(repo, gitRepo, opts)) + models.AssertExistsAndLoadBean(t, actionBean) + models.CheckConsistencyFor(t, &models.Action{}) +} + +func TestCommitRepoAction(t *testing.T) { + samples := []struct { + userID int64 + repositoryID int64 + commitRepoActionOptions commitRepoActionOptions + action models.Action + }{ + { + userID: 2, + repositoryID: 16, + commitRepoActionOptions: commitRepoActionOptions{ + PushUpdateOptions: PushUpdateOptions{ + RefFullName: "refName", + OldCommitID: "oldCommitID", + NewCommitID: "newCommitID", + }, + Commits: &repo_module.PushCommits{ + Commits: []*repo_module.PushCommit{ + { + Sha1: "69554a6", + CommitterEmail: "user2@example.com", + CommitterName: "User2", + AuthorEmail: "user2@example.com", + AuthorName: "User2", + Message: "not signed commit", + }, + { + Sha1: "27566bd", + CommitterEmail: "user2@example.com", + CommitterName: "User2", + AuthorEmail: "user2@example.com", + AuthorName: "User2", + Message: "good signed commit (with not yet validated email)", + }, + }, + Len: 2, + }, + }, + action: models.Action{ + OpType: models.ActionCommitRepo, + RefName: "refName", + }, + }, + { + userID: 2, + repositoryID: 1, + commitRepoActionOptions: commitRepoActionOptions{ + PushUpdateOptions: PushUpdateOptions{ + RefFullName: git.TagPrefix + "v1.1", + OldCommitID: git.EmptySHA, + NewCommitID: "newCommitID", + }, + Commits: &repo_module.PushCommits{}, + }, + action: models.Action{ + OpType: models.ActionPushTag, + RefName: "v1.1", + }, + }, + { + userID: 2, + repositoryID: 1, + commitRepoActionOptions: commitRepoActionOptions{ + PushUpdateOptions: PushUpdateOptions{ + RefFullName: git.TagPrefix + "v1.1", + OldCommitID: "oldCommitID", + NewCommitID: git.EmptySHA, + }, + Commits: &repo_module.PushCommits{}, + }, + action: models.Action{ + OpType: models.ActionDeleteTag, + RefName: "v1.1", + }, + }, + { + userID: 2, + repositoryID: 1, + commitRepoActionOptions: commitRepoActionOptions{ + PushUpdateOptions: PushUpdateOptions{ + RefFullName: git.BranchPrefix + "feature/1", + OldCommitID: "oldCommitID", + NewCommitID: git.EmptySHA, + }, + Commits: &repo_module.PushCommits{}, + }, + action: models.Action{ + OpType: models.ActionDeleteBranch, + RefName: "feature/1", + }, + }, + } + + for _, s := range samples { + models.PrepareTestEnv(t) + + user := models.AssertExistsAndLoadBean(t, &models.User{ID: s.userID}).(*models.User) + repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: s.repositoryID, OwnerID: user.ID}).(*models.Repository) + repo.Owner = user + + gitRepo, err := git.OpenRepository(repo.RepoPath()) + assert.NoError(t, err) + + s.commitRepoActionOptions.PusherName = user.Name + s.commitRepoActionOptions.RepoOwnerID = user.ID + s.commitRepoActionOptions.RepoName = repo.Name + + s.action.ActUserID = user.ID + s.action.RepoID = repo.ID + s.action.Repo = repo + s.action.IsPrivate = repo.IsPrivate + + testCorrectRepoAction(t, repo, gitRepo, &s.commitRepoActionOptions, &s.action) + gitRepo.Close() + } +} diff --git a/services/repository/repository.go b/services/repository/repository.go index f50b98b64..77c8728d9 100644 --- a/services/repository/repository.go +++ b/services/repository/repository.go @@ -87,3 +87,8 @@ func PushCreateRepo(authUser, owner *models.User, repoName string) (*models.Repo return repo, nil } + +// NewContext start repository service +func NewContext() error { + return initPushQueue() +}