Browse Source
Move all push update operations to a queue (#10133)
Move all push update operations to a queue (#10133)
* Fix test * Add no queue for test only * improve code * Auto watch whatever branch operation * Fix lint * Rename noqueue to immediate * Remove old PushUpdate function * Fix tests Co-authored-by: zeripath <art27@cantab.net> Co-authored-by: techknowlogick <techknowlogick@gitea.io>mj-v1.14.3
Lunny Xiao
2 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 552 additions and 585 deletions
-
5integrations/mssql.ini.tmpl
-
5integrations/mysql.ini.tmpl
-
4integrations/mysql8.ini.tmpl
-
5integrations/pgsql.ini.tmpl
-
3integrations/sqlite.ini.tmpl
-
141modules/repofiles/action.go
-
121modules/repofiles/action_test.go
-
298modules/repofiles/update.go
-
8routers/api/v1/repo/branch.go
-
4routers/init.go
-
8routers/private/hook.go
-
13routers/repo/branch.go
-
7routers/repo/pull.go
-
371services/repository/push.go
-
139services/repository/push_test.go
-
5services/repository/repository.go
@ -0,0 +1,371 @@ |
|||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
|||
// Use of this source code is governed by a MIT-style
|
|||
// license that can be found in the LICENSE file.
|
|||
|
|||
package repository |
|||
|
|||
import ( |
|||
"container/list" |
|||
"encoding/json" |
|||
"fmt" |
|||
"strings" |
|||
|
|||
"code.gitea.io/gitea/models" |
|||
"code.gitea.io/gitea/modules/cache" |
|||
"code.gitea.io/gitea/modules/git" |
|||
"code.gitea.io/gitea/modules/graceful" |
|||
"code.gitea.io/gitea/modules/log" |
|||
"code.gitea.io/gitea/modules/notification" |
|||
"code.gitea.io/gitea/modules/queue" |
|||
"code.gitea.io/gitea/modules/repofiles" |
|||
repo_module "code.gitea.io/gitea/modules/repository" |
|||
"code.gitea.io/gitea/modules/setting" |
|||
pull_service "code.gitea.io/gitea/services/pull" |
|||
) |
|||
|
|||
// PushUpdateOptions defines the push update options
|
|||
type PushUpdateOptions struct { |
|||
PusherID int64 |
|||
PusherName string |
|||
RepoUserName string |
|||
RepoName string |
|||
RefFullName string |
|||
OldCommitID string |
|||
NewCommitID string |
|||
} |
|||
|
|||
// IsNewRef return true if it's a first-time push to a branch, tag or etc.
|
|||
func (opts PushUpdateOptions) IsNewRef() bool { |
|||
return opts.OldCommitID == git.EmptySHA |
|||
} |
|||
|
|||
// IsDelRef return true if it's a deletion to a branch or tag
|
|||
func (opts PushUpdateOptions) IsDelRef() bool { |
|||
return opts.NewCommitID == git.EmptySHA |
|||
} |
|||
|
|||
// IsUpdateRef return true if it's an update operation
|
|||
func (opts PushUpdateOptions) IsUpdateRef() bool { |
|||
return !opts.IsNewRef() && !opts.IsDelRef() |
|||
} |
|||
|
|||
// IsTag return true if it's an operation to a tag
|
|||
func (opts PushUpdateOptions) IsTag() bool { |
|||
return strings.HasPrefix(opts.RefFullName, git.TagPrefix) |
|||
} |
|||
|
|||
// IsNewTag return true if it's a creation to a tag
|
|||
func (opts PushUpdateOptions) IsNewTag() bool { |
|||
return opts.IsTag() && opts.IsNewRef() |
|||
} |
|||
|
|||
// IsDelTag return true if it's a deletion to a tag
|
|||
func (opts PushUpdateOptions) IsDelTag() bool { |
|||
return opts.IsTag() && opts.IsDelRef() |
|||
} |
|||
|
|||
// IsBranch return true if it's a push to branch
|
|||
func (opts PushUpdateOptions) IsBranch() bool { |
|||
return strings.HasPrefix(opts.RefFullName, git.BranchPrefix) |
|||
} |
|||
|
|||
// IsNewBranch return true if it's the first-time push to a branch
|
|||
func (opts PushUpdateOptions) IsNewBranch() bool { |
|||
return opts.IsBranch() && opts.IsNewRef() |
|||
} |
|||
|
|||
// IsUpdateBranch return true if it's not the first push to a branch
|
|||
func (opts PushUpdateOptions) IsUpdateBranch() bool { |
|||
return opts.IsBranch() && opts.IsUpdateRef() |
|||
} |
|||
|
|||
// IsDelBranch return true if it's a deletion to a branch
|
|||
func (opts PushUpdateOptions) IsDelBranch() bool { |
|||
return opts.IsBranch() && opts.IsDelRef() |
|||
} |
|||
|
|||
// TagName returns simple tag name if it's an operation to a tag
|
|||
func (opts PushUpdateOptions) TagName() string { |
|||
return opts.RefFullName[len(git.TagPrefix):] |
|||
} |
|||
|
|||
// BranchName returns simple branch name if it's an operation to branch
|
|||
func (opts PushUpdateOptions) BranchName() string { |
|||
return opts.RefFullName[len(git.BranchPrefix):] |
|||
} |
|||
|
|||
// RepoFullName returns repo full name
|
|||
func (opts PushUpdateOptions) RepoFullName() string { |
|||
return opts.RepoUserName + "/" + opts.RepoName |
|||
} |
|||
|
|||
// pushQueue represents a queue to handle update pull request tests
|
|||
var pushQueue queue.Queue |
|||
|
|||
// handle passed PR IDs and test the PRs
|
|||
func handle(data ...queue.Data) { |
|||
for _, datum := range data { |
|||
opts := datum.([]*PushUpdateOptions) |
|||
if err := pushUpdates(opts); err != nil { |
|||
log.Error("pushUpdate failed: %v", err) |
|||
} |
|||
} |
|||
} |
|||
|
|||
func initPushQueue() error { |
|||
pushQueue = queue.CreateQueue("push_update", handle, []*PushUpdateOptions{}).(queue.Queue) |
|||
if pushQueue == nil { |
|||
return fmt.Errorf("Unable to create push_update Queue") |
|||
} |
|||
|
|||
go graceful.GetManager().RunWithShutdownFns(pushQueue.Run) |
|||
return nil |
|||
} |
|||
|
|||
// PushUpdate is an alias of PushUpdates for single push update options
|
|||
func PushUpdate(opts *PushUpdateOptions) error { |
|||
return PushUpdates([]*PushUpdateOptions{opts}) |
|||
} |
|||
|
|||
// PushUpdates adds a push update to push queue
|
|||
func PushUpdates(opts []*PushUpdateOptions) error { |
|||
if len(opts) == 0 { |
|||
return nil |
|||
} |
|||
|
|||
for _, opt := range opts { |
|||
if opt.IsNewRef() && opt.IsDelRef() { |
|||
return fmt.Errorf("Old and new revisions are both %s", git.EmptySHA) |
|||
} |
|||
} |
|||
|
|||
return pushQueue.Push(opts) |
|||
} |
|||
|
|||
// pushUpdates generates push action history feeds for push updating multiple refs
|
|||
func pushUpdates(optsList []*PushUpdateOptions) error { |
|||
if len(optsList) == 0 { |
|||
return nil |
|||
} |
|||
|
|||
repo, err := models.GetRepositoryByOwnerAndName(optsList[0].RepoUserName, optsList[0].RepoName) |
|||
if err != nil { |
|||
return fmt.Errorf("GetRepositoryByOwnerAndName failed: %v", err) |
|||
} |
|||
|
|||
repoPath := repo.RepoPath() |
|||
_, err = git.NewCommand("update-server-info").RunInDir(repoPath) |
|||
if err != nil { |
|||
return fmt.Errorf("Failed to call 'git update-server-info': %v", err) |
|||
} |
|||
gitRepo, err := git.OpenRepository(repoPath) |
|||
if err != nil { |
|||
return fmt.Errorf("OpenRepository: %v", err) |
|||
} |
|||
defer gitRepo.Close() |
|||
|
|||
if err = repo.UpdateSize(models.DefaultDBContext()); err != nil { |
|||
log.Error("Failed to update size for repository: %v", err) |
|||
} |
|||
|
|||
addTags := make([]string, 0, len(optsList)) |
|||
delTags := make([]string, 0, len(optsList)) |
|||
actions := make([]*commitRepoActionOptions, 0, len(optsList)) |
|||
var pusher *models.User |
|||
|
|||
for _, opts := range optsList { |
|||
if opts.IsNewRef() && opts.IsDelRef() { |
|||
return fmt.Errorf("Old and new revisions are both %s", git.EmptySHA) |
|||
} |
|||
var commits = &repo_module.PushCommits{} |
|||
if opts.IsTag() { // If is tag reference {
|
|||
tagName := opts.TagName() |
|||
if opts.IsDelRef() { |
|||
delTags = append(delTags, tagName) |
|||
} else { // is new tag
|
|||
cache.Remove(repo.GetCommitsCountCacheKey(tagName, true)) |
|||
addTags = append(addTags, tagName) |
|||
} |
|||
} else if opts.IsBranch() { // If is branch reference
|
|||
if pusher == nil || pusher.ID != opts.PusherID { |
|||
var err error |
|||
if pusher, err = models.GetUserByID(opts.PusherID); err != nil { |
|||
return err |
|||
} |
|||
} |
|||
|
|||
branch := opts.BranchName() |
|||
if !opts.IsDelRef() { |
|||
// Clear cache for branch commit count
|
|||
cache.Remove(repo.GetCommitsCountCacheKey(opts.BranchName(), true)) |
|||
|
|||
newCommit, err := gitRepo.GetCommit(opts.NewCommitID) |
|||
if err != nil { |
|||
return fmt.Errorf("gitRepo.GetCommit: %v", err) |
|||
} |
|||
|
|||
// Push new branch.
|
|||
var l *list.List |
|||
if opts.IsNewRef() { |
|||
l, err = newCommit.CommitsBeforeLimit(10) |
|||
if err != nil { |
|||
return fmt.Errorf("newCommit.CommitsBeforeLimit: %v", err) |
|||
} |
|||
} else { |
|||
l, err = newCommit.CommitsBeforeUntil(opts.OldCommitID) |
|||
if err != nil { |
|||
return fmt.Errorf("newCommit.CommitsBeforeUntil: %v", err) |
|||
} |
|||
} |
|||
|
|||
commits = repo_module.ListToPushCommits(l) |
|||
|
|||
if err = models.RemoveDeletedBranch(repo.ID, branch); err != nil { |
|||
log.Error("models.RemoveDeletedBranch %s/%s failed: %v", repo.ID, branch, err) |
|||
} |
|||
|
|||
log.Trace("TriggerTask '%s/%s' by %s", repo.Name, branch, pusher.Name) |
|||
|
|||
go pull_service.AddTestPullRequestTask(pusher, repo.ID, branch, true, opts.OldCommitID, opts.NewCommitID) |
|||
} else if err = pull_service.CloseBranchPulls(pusher, repo.ID, branch); err != nil { |
|||
// close all related pulls
|
|||
log.Error("close related pull request failed: %v", err) |
|||
} |
|||
|
|||
// Even if user delete a branch on a repository which he didn't watch, he will be watch that.
|
|||
if err = models.WatchIfAuto(opts.PusherID, repo.ID, true); err != nil { |
|||
log.Warn("Fail to perform auto watch on user %v for repo %v: %v", opts.PusherID, repo.ID, err) |
|||
} |
|||
} |
|||
actions = append(actions, &commitRepoActionOptions{ |
|||
PushUpdateOptions: *opts, |
|||
Pusher: pusher, |
|||
RepoOwnerID: repo.OwnerID, |
|||
Commits: commits, |
|||
}) |
|||
} |
|||
if err := repo_module.PushUpdateAddDeleteTags(repo, gitRepo, addTags, delTags); err != nil { |
|||
return fmt.Errorf("PushUpdateAddDeleteTags: %v", err) |
|||
} |
|||
|
|||
if err := commitRepoAction(repo, gitRepo, actions...); err != nil { |
|||
return fmt.Errorf("commitRepoAction: %v", err) |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
// commitRepoActionOptions represent options of a new commit action.
|
|||
type commitRepoActionOptions struct { |
|||
PushUpdateOptions |
|||
|
|||
Pusher *models.User |
|||
RepoOwnerID int64 |
|||
Commits *repo_module.PushCommits |
|||
} |
|||
|
|||
// commitRepoAction adds new commit action to the repository, and prepare
|
|||
// corresponding webhooks.
|
|||
func commitRepoAction(repo *models.Repository, gitRepo *git.Repository, optsList ...*commitRepoActionOptions) error { |
|||
actions := make([]*models.Action, len(optsList)) |
|||
|
|||
for i, opts := range optsList { |
|||
if opts.Pusher == nil || opts.Pusher.Name != opts.PusherName { |
|||
var err error |
|||
opts.Pusher, err = models.GetUserByName(opts.PusherName) |
|||
if err != nil { |
|||
return fmt.Errorf("GetUserByName [%s]: %v", opts.PusherName, err) |
|||
} |
|||
} |
|||
|
|||
refName := git.RefEndName(opts.RefFullName) |
|||
|
|||
// Change default branch and empty status only if pushed ref is non-empty branch.
|
|||
if repo.IsEmpty && opts.IsBranch() && !opts.IsDelRef() { |
|||
repo.DefaultBranch = refName |
|||
repo.IsEmpty = false |
|||
if refName != "master" { |
|||
if err := gitRepo.SetDefaultBranch(repo.DefaultBranch); err != nil { |
|||
if !git.IsErrUnsupportedVersion(err) { |
|||
return err |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
opType := models.ActionCommitRepo |
|||
|
|||
// Check it's tag push or branch.
|
|||
if opts.IsTag() { |
|||
opType = models.ActionPushTag |
|||
if opts.IsDelRef() { |
|||
opType = models.ActionDeleteTag |
|||
} |
|||
opts.Commits = &repo_module.PushCommits{} |
|||
} else if opts.IsDelRef() { |
|||
opType = models.ActionDeleteBranch |
|||
opts.Commits = &repo_module.PushCommits{} |
|||
} else { |
|||
// if not the first commit, set the compare URL.
|
|||
if !opts.IsNewRef() { |
|||
opts.Commits.CompareURL = repo.ComposeCompareURL(opts.OldCommitID, opts.NewCommitID) |
|||
} |
|||
|
|||
if err := repofiles.UpdateIssuesCommit(opts.Pusher, repo, opts.Commits.Commits, refName); err != nil { |
|||
log.Error("updateIssuesCommit: %v", err) |
|||
} |
|||
} |
|||
|
|||
if len(opts.Commits.Commits) > setting.UI.FeedMaxCommitNum { |
|||
opts.Commits.Commits = opts.Commits.Commits[:setting.UI.FeedMaxCommitNum] |
|||
} |
|||
|
|||
data, err := json.Marshal(opts.Commits) |
|||
if err != nil { |
|||
return fmt.Errorf("Marshal: %v", err) |
|||
} |
|||
|
|||
actions[i] = &models.Action{ |
|||
ActUserID: opts.Pusher.ID, |
|||
ActUser: opts.Pusher, |
|||
OpType: opType, |
|||
Content: string(data), |
|||
RepoID: repo.ID, |
|||
Repo: repo, |
|||
RefName: refName, |
|||
IsPrivate: repo.IsPrivate, |
|||
} |
|||
|
|||
var isHookEventPush = true |
|||
switch opType { |
|||
case models.ActionCommitRepo: // Push
|
|||
if opts.IsNewBranch() { |
|||
notification.NotifyCreateRef(opts.Pusher, repo, "branch", opts.RefFullName) |
|||
} |
|||
case models.ActionDeleteBranch: // Delete Branch
|
|||
notification.NotifyDeleteRef(opts.Pusher, repo, "branch", opts.RefFullName) |
|||
|
|||
case models.ActionPushTag: // Create
|
|||
notification.NotifyCreateRef(opts.Pusher, repo, "tag", opts.RefFullName) |
|||
|
|||
case models.ActionDeleteTag: // Delete Tag
|
|||
notification.NotifyDeleteRef(opts.Pusher, repo, "tag", opts.RefFullName) |
|||
default: |
|||
isHookEventPush = false |
|||
} |
|||
|
|||
if isHookEventPush { |
|||
notification.NotifyPushCommits(opts.Pusher, repo, opts.RefFullName, opts.OldCommitID, opts.NewCommitID, opts.Commits) |
|||
} |
|||
} |
|||
|
|||
// Change repository empty status and update last updated time.
|
|||
if err := models.UpdateRepository(repo, false); err != nil { |
|||
return fmt.Errorf("UpdateRepository: %v", err) |
|||
} |
|||
|
|||
if err := models.NotifyWatchers(actions...); err != nil { |
|||
return fmt.Errorf("NotifyWatchers: %v", err) |
|||
} |
|||
return nil |
|||
} |
|||
@ -0,0 +1,139 @@ |
|||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
|||
// Use of this source code is governed by a MIT-style
|
|||
// license that can be found in the LICENSE file.
|
|||
|
|||
package repository |
|||
|
|||
import ( |
|||
"testing" |
|||
|
|||
"code.gitea.io/gitea/models" |
|||
"code.gitea.io/gitea/modules/git" |
|||
repo_module "code.gitea.io/gitea/modules/repository" |
|||
|
|||
"github.com/stretchr/testify/assert" |
|||
) |
|||
|
|||
func testCorrectRepoAction(t *testing.T, repo *models.Repository, gitRepo *git.Repository, opts *commitRepoActionOptions, actionBean *models.Action) { |
|||
models.AssertNotExistsBean(t, actionBean) |
|||
assert.NoError(t, commitRepoAction(repo, gitRepo, opts)) |
|||
models.AssertExistsAndLoadBean(t, actionBean) |
|||
models.CheckConsistencyFor(t, &models.Action{}) |
|||
} |
|||
|
|||
func TestCommitRepoAction(t *testing.T) { |
|||
samples := []struct { |
|||
userID int64 |
|||
repositoryID int64 |
|||
commitRepoActionOptions commitRepoActionOptions |
|||
action models.Action |
|||
}{ |
|||
{ |
|||
userID: 2, |
|||
repositoryID: 16, |
|||
commitRepoActionOptions: commitRepoActionOptions{ |
|||
PushUpdateOptions: PushUpdateOptions{ |
|||
RefFullName: "refName", |
|||
OldCommitID: "oldCommitID", |
|||
NewCommitID: "newCommitID", |
|||
}, |
|||
Commits: &repo_module.PushCommits{ |
|||
Commits: []*repo_module.PushCommit{ |
|||
{ |
|||
Sha1: "69554a6", |
|||
CommitterEmail: "user2@example.com", |
|||
CommitterName: "User2", |
|||
AuthorEmail: "user2@example.com", |
|||
AuthorName: "User2", |
|||
Message: "not signed commit", |
|||
}, |
|||
{ |
|||
Sha1: "27566bd", |
|||
CommitterEmail: "user2@example.com", |
|||
CommitterName: "User2", |
|||
AuthorEmail: "user2@example.com", |
|||
AuthorName: "User2", |
|||
Message: "good signed commit (with not yet validated email)", |
|||
}, |
|||
}, |
|||
Len: 2, |
|||
}, |
|||
}, |
|||
action: models.Action{ |
|||
OpType: models.ActionCommitRepo, |
|||
RefName: "refName", |
|||
}, |
|||
}, |
|||
{ |
|||
userID: 2, |
|||
repositoryID: 1, |
|||
commitRepoActionOptions: commitRepoActionOptions{ |
|||
PushUpdateOptions: PushUpdateOptions{ |
|||
RefFullName: git.TagPrefix + "v1.1", |
|||
OldCommitID: git.EmptySHA, |
|||
NewCommitID: "newCommitID", |
|||
}, |
|||
Commits: &repo_module.PushCommits{}, |
|||
}, |
|||
action: models.Action{ |
|||
OpType: models.ActionPushTag, |
|||
RefName: "v1.1", |
|||
}, |
|||
}, |
|||
{ |
|||
userID: 2, |
|||
repositoryID: 1, |
|||
commitRepoActionOptions: commitRepoActionOptions{ |
|||
PushUpdateOptions: PushUpdateOptions{ |
|||
RefFullName: git.TagPrefix + "v1.1", |
|||
OldCommitID: "oldCommitID", |
|||
NewCommitID: git.EmptySHA, |
|||
}, |
|||
Commits: &repo_module.PushCommits{}, |
|||
}, |
|||
action: models.Action{ |
|||
OpType: models.ActionDeleteTag, |
|||
RefName: "v1.1", |
|||
}, |
|||
}, |
|||
{ |
|||
userID: 2, |
|||
repositoryID: 1, |
|||
commitRepoActionOptions: commitRepoActionOptions{ |
|||
PushUpdateOptions: PushUpdateOptions{ |
|||
RefFullName: git.BranchPrefix + "feature/1", |
|||
OldCommitID: "oldCommitID", |
|||
NewCommitID: git.EmptySHA, |
|||
}, |
|||
Commits: &repo_module.PushCommits{}, |
|||
}, |
|||
action: models.Action{ |
|||
OpType: models.ActionDeleteBranch, |
|||
RefName: "feature/1", |
|||
}, |
|||
}, |
|||
} |
|||
|
|||
for _, s := range samples { |
|||
models.PrepareTestEnv(t) |
|||
|
|||
user := models.AssertExistsAndLoadBean(t, &models.User{ID: s.userID}).(*models.User) |
|||
repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: s.repositoryID, OwnerID: user.ID}).(*models.Repository) |
|||
repo.Owner = user |
|||
|
|||
gitRepo, err := git.OpenRepository(repo.RepoPath()) |
|||
assert.NoError(t, err) |
|||
|
|||
s.commitRepoActionOptions.PusherName = user.Name |
|||
s.commitRepoActionOptions.RepoOwnerID = user.ID |
|||
s.commitRepoActionOptions.RepoName = repo.Name |
|||
|
|||
s.action.ActUserID = user.ID |
|||
s.action.RepoID = repo.ID |
|||
s.action.Repo = repo |
|||
s.action.IsPrivate = repo.IsPrivate |
|||
|
|||
testCorrectRepoAction(t, repo, gitRepo, &s.commitRepoActionOptions, &s.action) |
|||
gitRepo.Close() |
|||
} |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue