diff --git a/docs/content/doc/advanced/repo-mirror.en-us.md b/docs/content/doc/advanced/repo-mirror.en-us.md new file mode 100644 index 000000000..bda5b0fa5 --- /dev/null +++ b/docs/content/doc/advanced/repo-mirror.en-us.md @@ -0,0 +1,88 @@ +--- +date: "2021-05-13T00:00:00-00:00" +title: "Repository Mirror" +slug: "repo-mirror" +weight: 45 +toc: false +draft: false +menu: + sidebar: + parent: "advanced" + name: "Repository Mirror" + weight: 45 + identifier: "repo-mirror" +--- + +# Repository Mirror + +Repository mirroring allows for the mirroring of repositories to and from external sources. You can use it to mirror branches, tags, and commits between repositories. + +**Table of Contents** + +{{< toc >}} + +## Use cases + +The following are some possible use cases for repository mirroring: + +- You migrated to Gitea but still need to keep your project in another source. In that case, you can simply set it up to mirror to Gitea (pull) and all the essential history of commits, tags, and branches are available in your Gitea instance. +- You have old projects in another source that you don’t use actively anymore, but don’t want to remove for archiving purposes. In that case, you can create a push mirror so that your active Gitea repository can push its changes to the old location. + +## Pulling from a remote repository + +For an existing remote repository, you can set up pull mirroring as follows: + +1. Select **New Migration** in the **Create...** menu on the top right. +2. Select the remote repository service. +3. Enter a repository URL. +4. If the repository needs authentication fill in your authentication information. +5. Check the box **This repository will be a mirror**. +5. Select **Migrate repository** to save the configuration. + +The repository now gets mirrored periodically from the remote repository. You can force a sync by selecting **Synchronize Now** in the repository settings. + +## Pushing to a remote repository + +For an existing repository, you can set up push mirroring as follows: + +1. In your repository, go to **Settings** > **Repository**, and then the **Mirror Settings** section. +2. Enter a repository URL. +3. If the repository needs authentication expand the **Authorization** section and fill in your authentication information. +4. Select **Add Push Mirror** to save the configuration. + +The repository now gets mirrored periodically to the remote repository. You can force a sync by selecting **Synchronize Now**. In case of an error a message displayed to help you resolve it. + +:exclamation::exclamation: **NOTE:** This will force push to the remote repository. This will overwrite any changes in the remote repository! :exclamation::exclamation: + +### Setting up a push mirror from Gitea to GitHub + +To set up a mirror from Gitea to GitHub, you need to follow these steps: + +1. Create a [GitHub personal access token](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token) with the *public_repo* box checked. +2. Fill in the **Git Remote Repository URL**: `https://github.com//.git`. +3. Fill in the **Authorization** fields with your GitHub username and the personal access token. +4. Select **Add Push Mirror** to save the configuration. + +The repository pushes shortly thereafter. To force a push, select the **Synchronize Now** button. + +### Setting up a push mirror from Gitea to GitLab + +To set up a mirror from Gitea to GitLab, you need to follow these steps: + +1. Create a [GitLab personal access token](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html) with *write_repository* scope. +2. Fill in the **Git Remote Repository URL**: `https:////.git`. +3. Fill in the **Authorization** fields with `oauth2` as **Username** and your GitLab personal access token as **Password**. +4. Select **Add Push Mirror** to save the configuration. + +The repository pushes shortly thereafter. To force a push, select the **Synchronize Now** button. + +### Setting up a push mirror from Gitea to Bitbucket + +To set up a mirror from Gitea to Bitbucket, you need to follow these steps: + +1. Create a [Bitbucket app password](https://support.atlassian.com/bitbucket-cloud/docs/app-passwords/) with the *Repository Write* box checked. +2. Fill in the **Git Remote Repository URL**: `https://bitbucket.org//.git`. +3. Fill in the **Authorization** fields with your Bitbucket username and the app password as **Password**. +4. Select **Add Push Mirror** to save the configuration. + +The repository pushes shortly thereafter. To force a push, select the **Synchronize Now** button. diff --git a/services/mirror/mirror_test.go b/integrations/mirror_pull_test.go similarity index 89% rename from services/mirror/mirror_test.go rename to integrations/mirror_pull_test.go index 20492c784..0e4da74fc 100644 --- a/services/mirror/mirror_test.go +++ b/integrations/mirror_pull_test.go @@ -2,28 +2,24 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package mirror +package integrations import ( "context" - "path/filepath" "testing" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git" migration "code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/repository" + mirror_service "code.gitea.io/gitea/services/mirror" release_service "code.gitea.io/gitea/services/release" "github.com/stretchr/testify/assert" ) -func TestMain(m *testing.M) { - models.MainTest(m, filepath.Join("..", "..")) -} - -func TestRelease_MirrorDelete(t *testing.T) { - assert.NoError(t, models.PrepareTestDatabase()) +func TestMirrorPull(t *testing.T) { + defer prepareTestEnv(t)() user := models.AssertExistsAndLoadBean(t, &models.User{ID: 2}).(*models.User) repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository) @@ -76,7 +72,7 @@ func TestRelease_MirrorDelete(t *testing.T) { err = mirror.GetMirror() assert.NoError(t, err) - _, ok := runSync(ctx, mirror.Mirror) + ok := mirror_service.SyncPullMirror(ctx, mirror.ID) assert.True(t, ok) count, err := models.GetReleaseCountByRepoID(mirror.ID, findOptions) @@ -87,7 +83,7 @@ func TestRelease_MirrorDelete(t *testing.T) { assert.NoError(t, err) assert.NoError(t, release_service.DeleteReleaseByID(release.ID, user, true)) - _, ok = runSync(ctx, mirror.Mirror) + ok = mirror_service.SyncPullMirror(ctx, mirror.ID) assert.True(t, ok) count, err = models.GetReleaseCountByRepoID(mirror.ID, findOptions) diff --git a/integrations/mirror_push_test.go b/integrations/mirror_push_test.go new file mode 100644 index 000000000..3191ef770 --- /dev/null +++ b/integrations/mirror_push_test.go @@ -0,0 +1,86 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package integrations + +import ( + "context" + "fmt" + "net/http" + "net/url" + "testing" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/setting" + mirror_service "code.gitea.io/gitea/services/mirror" + + "github.com/stretchr/testify/assert" +) + +func TestMirrorPush(t *testing.T) { + onGiteaRun(t, testMirrorPush) +} + +func testMirrorPush(t *testing.T, u *url.URL) { + defer prepareTestEnv(t)() + + setting.Migrations.AllowLocalNetworks = true + + user := models.AssertExistsAndLoadBean(t, &models.User{ID: 2}).(*models.User) + srcRepo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository) + + mirrorRepo, err := repository.CreateRepository(user, user, models.CreateRepoOptions{ + Name: "test-push-mirror", + }) + assert.NoError(t, err) + + ctx := NewAPITestContext(t, user.LowerName, srcRepo.Name) + + doCreatePushMirror(ctx, fmt.Sprintf("%s%s/%s", u.String(), url.PathEscape(ctx.Username), url.PathEscape(mirrorRepo.Name)), user.LowerName, userPassword)(t) + + mirrors, err := models.GetPushMirrorsByRepoID(srcRepo.ID) + assert.NoError(t, err) + assert.Len(t, mirrors, 1) + + ok := mirror_service.SyncPushMirror(context.Background(), mirrors[0].ID) + assert.True(t, ok) + + srcGitRepo, err := git.OpenRepository(srcRepo.RepoPath()) + assert.NoError(t, err) + defer srcGitRepo.Close() + + srcCommit, err := srcGitRepo.GetBranchCommit("master") + assert.NoError(t, err) + + mirrorGitRepo, err := git.OpenRepository(mirrorRepo.RepoPath()) + assert.NoError(t, err) + defer mirrorGitRepo.Close() + + mirrorCommit, err := mirrorGitRepo.GetBranchCommit("master") + assert.NoError(t, err) + + assert.Equal(t, srcCommit.ID, mirrorCommit.ID) +} + +func doCreatePushMirror(ctx APITestContext, address, username, password string) func(t *testing.T) { + return func(t *testing.T) { + csrf := GetCSRF(t, ctx.Session, fmt.Sprintf("/%s/%s/settings", url.PathEscape(ctx.Username), url.PathEscape(ctx.Reponame))) + + req := NewRequestWithValues(t, "POST", fmt.Sprintf("/%s/%s/settings", url.PathEscape(ctx.Username), url.PathEscape(ctx.Reponame)), map[string]string{ + "_csrf": csrf, + "action": "push-mirror-add", + "push_mirror_address": address, + "push_mirror_username": username, + "push_mirror_password": password, + "push_mirror_interval": "0", + }) + ctx.Session.MakeRequest(t, req, http.StatusFound) + + flashCookie := ctx.Session.GetCookie("macaron_flash") + assert.NotNil(t, flashCookie) + assert.Contains(t, flashCookie.Value, "success") + } +} diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 4c07db0a0..8e4f30177 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -315,6 +315,8 @@ var migrations = []Migration{ NewMigration("Always save primary email on email address table", addPrimaryEmail2EmailAddress), // v182 -> v183 NewMigration("Add issue resource index table", addIssueResourceIndexTable), + // v183 -> v184 + NewMigration("Create PushMirror table", createPushMirrorTable), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v180.go b/models/migrations/v180.go index c2a3ff961..a0471e151 100644 --- a/models/migrations/v180.go +++ b/models/migrations/v180.go @@ -64,7 +64,7 @@ func removeCredentials(payload string) (string, error) { opts.AuthPassword = "" opts.AuthToken = "" - opts.CloneAddr = util.SanitizeURLCredentials(opts.CloneAddr, true) + opts.CloneAddr = util.NewStringURLSanitizer(opts.CloneAddr, true).Replace(opts.CloneAddr) confBytes, err := json.Marshal(opts) if err != nil { diff --git a/models/migrations/v183.go b/models/migrations/v183.go new file mode 100644 index 000000000..cc752bf82 --- /dev/null +++ b/models/migrations/v183.go @@ -0,0 +1,39 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "fmt" + "time" + + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/xorm" +) + +func createPushMirrorTable(x *xorm.Engine) error { + type PushMirror struct { + ID int64 `xorm:"pk autoincr"` + RepoID int64 `xorm:"INDEX"` + RemoteName string + + Interval time.Duration + CreatedUnix timeutil.TimeStamp `xorm:"created"` + LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"` + LastError string `xorm:"text"` + } + + sess := x.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return err + } + + if err := sess.Sync2(new(PushMirror)); err != nil { + return fmt.Errorf("Sync2: %v", err) + } + + return sess.Commit() +} diff --git a/models/models.go b/models/models.go index 2b3203ecc..c325fd381 100644 --- a/models/models.go +++ b/models/models.go @@ -135,6 +135,7 @@ func init() { new(Session), new(RepoTransfer), new(IssueIndex), + new(PushMirror), ) gonicNames := []string{"SSL", "UID"} diff --git a/models/repo.go b/models/repo.go index 532b7ae1f..dc4e03a28 100644 --- a/models/repo.go +++ b/models/repo.go @@ -216,12 +216,13 @@ type Repository struct { NumClosedProjects int `xorm:"NOT NULL DEFAULT 0"` NumOpenProjects int `xorm:"-"` - IsPrivate bool `xorm:"INDEX"` - IsEmpty bool `xorm:"INDEX"` - IsArchived bool `xorm:"INDEX"` - IsMirror bool `xorm:"INDEX"` - *Mirror `xorm:"-"` - Status RepositoryStatus `xorm:"NOT NULL DEFAULT 0"` + IsPrivate bool `xorm:"INDEX"` + IsEmpty bool `xorm:"INDEX"` + IsArchived bool `xorm:"INDEX"` + IsMirror bool `xorm:"INDEX"` + *Mirror `xorm:"-"` + PushMirrors []*PushMirror `xorm:"-"` + Status RepositoryStatus `xorm:"NOT NULL DEFAULT 0"` RenderingMetas map[string]string `xorm:"-"` DocumentRenderingMetas map[string]string `xorm:"-"` @@ -255,7 +256,12 @@ func (repo *Repository) SanitizedOriginalURL() string { if repo.OriginalURL == "" { return "" } - return util.SanitizeURLCredentials(repo.OriginalURL, false) + u, err := url.Parse(repo.OriginalURL) + if err != nil { + return "" + } + u.User = nil + return u.String() } // ColorFormat returns a colored string to represent this repo @@ -657,6 +663,12 @@ func (repo *Repository) GetMirror() (err error) { return err } +// LoadPushMirrors populates the repository push mirrors. +func (repo *Repository) LoadPushMirrors() (err error) { + repo.PushMirrors, err = GetPushMirrorsByRepoID(repo.ID) + return err +} + // GetBaseRepo populates repo.BaseRepo for a fork repository and // returns an error on failure (NOTE: no error is returned for // non-fork repositories, and BaseRepo will be left untouched) @@ -1487,6 +1499,7 @@ func DeleteRepository(doer *User, uid, repoID int64) error { &Notification{RepoID: repoID}, &ProtectedBranch{RepoID: repoID}, &PullRequest{BaseRepoID: repoID}, + &PushMirror{RepoID: repoID}, &Release{RepoID: repoID}, &RepoIndexerStatus{RepoID: repoID}, &RepoRedirect{RedirectRepoID: repoID}, diff --git a/models/repo_mirror.go b/models/repo_mirror.go index 2c37b54aa..cd1f74cb2 100644 --- a/models/repo_mirror.go +++ b/models/repo_mirror.go @@ -14,6 +14,12 @@ import ( "xorm.io/xorm" ) +// RemoteMirrorer defines base methods for pull/push mirrors. +type RemoteMirrorer interface { + GetRepository() *Repository + GetRemoteName() string +} + // Mirror represents mirror information of a repository. type Mirror struct { ID int64 `xorm:"pk autoincr"` @@ -52,6 +58,16 @@ func (m *Mirror) AfterLoad(session *xorm.Session) { } } +// GetRepository returns the repository. +func (m *Mirror) GetRepository() *Repository { + return m.Repo +} + +// GetRemoteName returns the name of the remote. +func (m *Mirror) GetRemoteName() string { + return "origin" +} + // ScheduleNextUpdate calculates and sets next update time. func (m *Mirror) ScheduleNextUpdate() { if m.Interval != 0 { diff --git a/models/repo_pushmirror.go b/models/repo_pushmirror.go new file mode 100644 index 000000000..bdd4198f9 --- /dev/null +++ b/models/repo_pushmirror.go @@ -0,0 +1,106 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package models + +import ( + "errors" + "time" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/xorm" +) + +var ( + // ErrPushMirrorNotExist mirror does not exist error + ErrPushMirrorNotExist = errors.New("PushMirror does not exist") +) + +// PushMirror represents mirror information of a repository. +type PushMirror struct { + ID int64 `xorm:"pk autoincr"` + RepoID int64 `xorm:"INDEX"` + Repo *Repository `xorm:"-"` + RemoteName string + + Interval time.Duration + CreatedUnix timeutil.TimeStamp `xorm:"created"` + LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"` + LastError string `xorm:"text"` +} + +// AfterLoad is invoked from XORM after setting the values of all fields of this object. +func (m *PushMirror) AfterLoad(session *xorm.Session) { + if m == nil { + return + } + + var err error + m.Repo, err = getRepositoryByID(session, m.RepoID) + if err != nil { + log.Error("getRepositoryByID[%d]: %v", m.ID, err) + } +} + +// GetRepository returns the path of the repository. +func (m *PushMirror) GetRepository() *Repository { + return m.Repo +} + +// GetRemoteName returns the name of the remote. +func (m *PushMirror) GetRemoteName() string { + return m.RemoteName +} + +// InsertPushMirror inserts a push-mirror to database +func InsertPushMirror(m *PushMirror) error { + _, err := x.Insert(m) + return err +} + +// UpdatePushMirror updates the push-mirror +func UpdatePushMirror(m *PushMirror) error { + _, err := x.ID(m.ID).AllCols().Update(m) + return err +} + +// DeletePushMirrorByID deletes a push-mirrors by ID +func DeletePushMirrorByID(ID int64) error { + _, err := x.ID(ID).Delete(&PushMirror{}) + return err +} + +// DeletePushMirrorsByRepoID deletes all push-mirrors by repoID +func DeletePushMirrorsByRepoID(repoID int64) error { + _, err := x.Delete(&PushMirror{RepoID: repoID}) + return err +} + +// GetPushMirrorByID returns push-mirror information. +func GetPushMirrorByID(ID int64) (*PushMirror, error) { + m := &PushMirror{} + has, err := x.ID(ID).Get(m) + if err != nil { + return nil, err + } else if !has { + return nil, ErrPushMirrorNotExist + } + return m, nil +} + +// GetPushMirrorsByRepoID returns push-mirror informations of a repository. +func GetPushMirrorsByRepoID(repoID int64) ([]*PushMirror, error) { + mirrors := make([]*PushMirror, 0, 10) + return mirrors, x.Where("repo_id=?", repoID).Find(&mirrors) +} + +// PushMirrorsIterate iterates all push-mirror repositories. +func PushMirrorsIterate(f func(idx int, bean interface{}) error) error { + return x. + Where("last_update + (`interval` / ?) <= ?", time.Second, time.Now().Unix()). + And("`interval` != 0"). + Iterate(new(PushMirror), f) +} diff --git a/models/repo_pushmirror_test.go b/models/repo_pushmirror_test.go new file mode 100644 index 000000000..66c499b1c --- /dev/null +++ b/models/repo_pushmirror_test.go @@ -0,0 +1,49 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package models + +import ( + "testing" + "time" + + "code.gitea.io/gitea/modules/timeutil" + + "github.com/stretchr/testify/assert" +) + +func TestPushMirrorsIterate(t *testing.T) { + assert.NoError(t, PrepareTestDatabase()) + + now := timeutil.TimeStampNow() + + InsertPushMirror(&PushMirror{ + RemoteName: "test-1", + LastUpdateUnix: now, + Interval: 1, + }) + + long, _ := time.ParseDuration("24h") + InsertPushMirror(&PushMirror{ + RemoteName: "test-2", + LastUpdateUnix: now, + Interval: long, + }) + + InsertPushMirror(&PushMirror{ + RemoteName: "test-3", + LastUpdateUnix: now, + Interval: 0, + }) + + time.Sleep(1 * time.Millisecond) + + PushMirrorsIterate(func(idx int, bean interface{}) error { + m, ok := bean.(*PushMirror) + assert.True(t, ok) + assert.Equal(t, "test-1", m.RemoteName) + assert.Equal(t, m.RemoteName, m.GetRemoteName()) + return nil + }) +} diff --git a/models/task.go b/models/task.go index a4ab65b5e..2743d91f6 100644 --- a/models/task.go +++ b/models/task.go @@ -234,7 +234,7 @@ func FinishMigrateTask(task *Task) error { } conf.AuthPassword = "" conf.AuthToken = "" - conf.CloneAddr = util.SanitizeURLCredentials(conf.CloneAddr, true) + conf.CloneAddr = util.NewStringURLSanitizer(conf.CloneAddr, true).Replace(conf.CloneAddr) conf.AuthPasswordEncrypted = "" conf.AuthTokenEncrypted = "" conf.CloneAddrEncrypted = "" diff --git a/modules/context/repo.go b/modules/context/repo.go index 3e48b34b3..72d1cf4c8 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -360,13 +360,17 @@ func repoAssignment(ctx *Context, repo *models.Repository) { var err error ctx.Repo.Mirror, err = models.GetMirrorByRepoID(repo.ID) if err != nil { - ctx.ServerError("GetMirror", err) + ctx.ServerError("GetMirrorByRepoID", err) return } ctx.Data["MirrorEnablePrune"] = ctx.Repo.Mirror.EnablePrune ctx.Data["MirrorInterval"] = ctx.Repo.Mirror.Interval ctx.Data["Mirror"] = ctx.Repo.Mirror } + if err = repo.LoadPushMirrors(); err != nil { + ctx.ServerError("LoadPushMirrors", err) + return + } ctx.Repo.Repository = repo ctx.Data["RepoName"] = ctx.Repo.Repository.Name diff --git a/modules/git/remote.go b/modules/git/remote.go new file mode 100644 index 000000000..7ba2b35a5 --- /dev/null +++ b/modules/git/remote.go @@ -0,0 +1,31 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package git + +import "net/url" + +// GetRemoteAddress returns the url of a specific remote of the repository. +func GetRemoteAddress(repoPath, remoteName string) (*url.URL, error) { + err := LoadGitVersion() + if err != nil { + return nil, err + } + var cmd *Command + if CheckGitVersionAtLeast("2.7") == nil { + cmd = NewCommand("remote", "get-url", remoteName) + } else { + cmd = NewCommand("config", "--get", "remote."+remoteName+".url") + } + + result, err := cmd.RunInDir(repoPath) + if err != nil { + return nil, err + } + + if len(result) > 0 { + result = result[:len(result)-1] + } + return url.Parse(result) +} diff --git a/modules/git/repo.go b/modules/git/repo.go index 515899ab0..e06cd4393 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -182,10 +182,12 @@ func Pull(repoPath string, opts PullRemoteOptions) error { // PushOptions options when push to remote type PushOptions struct { - Remote string - Branch string - Force bool - Env []string + Remote string + Branch string + Force bool + Mirror bool + Env []string + Timeout time.Duration } // Push pushs local commits to given remote branch. @@ -194,10 +196,20 @@ func Push(repoPath string, opts PushOptions) error { if opts.Force { cmd.AddArguments("-f") } - cmd.AddArguments("--", opts.Remote, opts.Branch) + if opts.Mirror { + cmd.AddArguments("--mirror") + } + cmd.AddArguments("--", opts.Remote) + if len(opts.Branch) > 0 { + cmd.AddArguments(opts.Branch) + } var outbuf, errbuf strings.Builder - err := cmd.RunInDirTimeoutEnvPipeline(opts.Env, -1, repoPath, &outbuf, &errbuf) + if opts.Timeout == 0 { + opts.Timeout = -1 + } + + err := cmd.RunInDirTimeoutEnvPipeline(opts.Env, opts.Timeout, repoPath, &outbuf, &errbuf) if err != nil { if strings.Contains(errbuf.String(), "non-fast-forward") { return &ErrPushOutOfDate{ diff --git a/modules/lfs/client.go b/modules/lfs/client.go index ae35919d7..0a21440f7 100644 --- a/modules/lfs/client.go +++ b/modules/lfs/client.go @@ -10,9 +10,17 @@ import ( "net/url" ) +// DownloadCallback gets called for every requested LFS object to process its content +type DownloadCallback func(p Pointer, content io.ReadCloser, objectError error) error + +// UploadCallback gets called for every requested LFS object to provide its content +type UploadCallback func(p Pointer, objectError error) (io.ReadCloser, error) + // Client is used to communicate with a LFS source type Client interface { - Download(ctx context.Context, oid string, size int64) (io.ReadCloser, error) + BatchSize() int + Download(ctx context.Context, objects []Pointer, callback DownloadCallback) error + Upload(ctx context.Context, objects []Pointer, callback UploadCallback) error } // NewClient creates a LFS client diff --git a/modules/lfs/client_test.go b/modules/lfs/client_test.go index d4eb00546..1040b3992 100644 --- a/modules/lfs/client_test.go +++ b/modules/lfs/client_test.go @@ -6,7 +6,6 @@ package lfs import ( "net/url" - "testing" "github.com/stretchr/testify/assert" diff --git a/modules/lfs/filesystem_client.go b/modules/lfs/filesystem_client.go index 3a51564a8..dc72981a9 100644 --- a/modules/lfs/filesystem_client.go +++ b/modules/lfs/filesystem_client.go @@ -19,6 +19,11 @@ type FilesystemClient struct { lfsdir string } +// BatchSize returns the preferred size of batchs to process +func (c *FilesystemClient) BatchSize() int { + return 1 +} + func newFilesystemClient(endpoint *url.URL) *FilesystemClient { path, _ := util.FileURLToPath(endpoint) @@ -33,18 +38,56 @@ func (c *FilesystemClient) objectPath(oid string) string { return filepath.Join(c.lfsdir, oid[0:2], oid[2:4], oid) } -// Download reads the specific LFS object from the target repository -func (c *FilesystemClient) Download(ctx context.Context, oid string, size int64) (io.ReadCloser, error) { - objectPath := c.objectPath(oid) +// Download reads the specific LFS object from the target path +func (c *FilesystemClient) Download(ctx context.Context, objects []Pointer, callback DownloadCallback) error { + for _, object := range objects { + p := Pointer{object.Oid, object.Size} - if _, err := os.Stat(objectPath); os.IsNotExist(err) { - return nil, err - } + objectPath := c.objectPath(p.Oid) + + f, err := os.Open(objectPath) + if err != nil { + return err + } - file, err := os.Open(objectPath) - if err != nil { - return nil, err + if err := callback(p, f, nil); err != nil { + return err + } } + return nil +} + +// Upload writes the specific LFS object to the target path +func (c *FilesystemClient) Upload(ctx context.Context, objects []Pointer, callback UploadCallback) error { + for _, object := range objects { + p := Pointer{object.Oid, object.Size} + + objectPath := c.objectPath(p.Oid) - return file, nil + if err := os.MkdirAll(filepath.Dir(objectPath), os.ModePerm); err != nil { + return err + } + + content, err := callback(p, nil) + if err != nil { + return err + } + + err = func() error { + defer content.Close() + + f, err := os.Create(objectPath) + if err != nil { + return err + } + + _, err = io.Copy(f, content) + + return err + }() + if err != nil { + return err + } + } + return nil } diff --git a/modules/lfs/http_client.go b/modules/lfs/http_client.go index fb45defda..e799b8083 100644 --- a/modules/lfs/http_client.go +++ b/modules/lfs/http_client.go @@ -7,17 +7,19 @@ package lfs import ( "bytes" "context" - "encoding/json" "errors" "fmt" - "io" "net/http" "net/url" "strings" "code.gitea.io/gitea/modules/log" + + jsoniter "github.com/json-iterator/go" ) +const batchSize = 20 + // HTTPClient is used to communicate with the LFS server // https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md type HTTPClient struct { @@ -26,6 +28,11 @@ type HTTPClient struct { transfers map[string]TransferAdapter } +// BatchSize returns the preferred size of batchs to process +func (c *HTTPClient) BatchSize() int { + return batchSize +} + func newHTTPClient(endpoint *url.URL) *HTTPClient { hc := &http.Client{} @@ -55,21 +62,25 @@ func (c *HTTPClient) transferNames() []string { } func (c *HTTPClient) batch(ctx context.Context, operation string, objects []Pointer) (*BatchResponse, error) { + log.Trace("BATCH operation with objects: %v", objects) + url := fmt.Sprintf("%s/objects/batch", c.endpoint) request := &BatchRequest{operation, c.transferNames(), nil, objects} payload := new(bytes.Buffer) - err := json.NewEncoder(payload).Encode(request) + err := jsoniter.NewEncoder(payload).Encode(request) if err != nil { - return nil, fmt.Errorf("lfs.HTTPClient.batch json.Encode: %w", err) + log.Error("Error encoding json: %v", err) + return nil, err } - log.Trace("lfs.HTTPClient.batch NewRequestWithContext: %s", url) + log.Trace("Calling: %s", url) req, err := http.NewRequestWithContext(ctx, "POST", url, payload) if err != nil { - return nil, fmt.Errorf("lfs.HTTPClient.batch http.NewRequestWithContext: %w", err) + log.Error("Error creating request: %v", err) + return nil, err } req.Header.Set("Content-type", MediaType) req.Header.Set("Accept", MediaType) @@ -81,18 +92,20 @@ func (c *HTTPClient) batch(ctx context.Context, operation string, objects []Poin return nil, ctx.Err() default: } - return nil, fmt.Errorf("lfs.HTTPClient.batch http.Do: %w", err) + log.Error("Error while processing request: %v", err) + return nil, err } defer res.Body.Close() if res.StatusCode != http.StatusOK { - return nil, fmt.Errorf("lfs.HTTPClient.batch: Unexpected servers response: %s", res.Status) + return nil, fmt.Errorf("Unexpected server response: %s", res.Status) } var response BatchResponse - err = json.NewDecoder(res.Body).Decode(&response) + err = jsoniter.NewDecoder(res.Body).Decode(&response) if err != nil { - return nil, fmt.Errorf("lfs.HTTPClient.batch json.Decode: %w", err) + log.Error("Error decoding json: %v", err) + return nil, err } if len(response.Transfer) == 0 { @@ -103,27 +116,99 @@ func (c *HTTPClient) batch(ctx context.Context, operation string, objects []Poin } // Download reads the specific LFS object from the LFS server -func (c *HTTPClient) Download(ctx context.Context, oid string, size int64) (io.ReadCloser, error) { - var objects []Pointer - objects = append(objects, Pointer{oid, size}) +func (c *HTTPClient) Download(ctx context.Context, objects []Pointer, callback DownloadCallback) error { + return c.performOperation(ctx, objects, callback, nil) +} + +// Upload sends the specific LFS object to the LFS server +func (c *HTTPClient) Upload(ctx context.Context, objects []Pointer, callback UploadCallback) error { + return c.performOperation(ctx, objects, nil, callback) +} - result, err := c.batch(ctx, "download", objects) +func (c *HTTPClient) performOperation(ctx context.Context, objects []Pointer, dc DownloadCallback, uc UploadCallback) error { + if len(objects) == 0 { + return nil + } + + operation := "download" + if uc != nil { + operation = "upload" + } + + result, err := c.batch(ctx, operation, objects) if err != nil { - return nil, err + return err } transferAdapter, ok := c.transfers[result.Transfer] if !ok { - return nil, fmt.Errorf("lfs.HTTPClient.Download Transferadapter not found: %s", result.Transfer) + return fmt.Errorf("TransferAdapter not found: %s", result.Transfer) } - if len(result.Objects) == 0 { - return nil, errors.New("lfs.HTTPClient.Download: No objects in result") - } + for _, object := range result.Objects { + if object.Error != nil { + objectError := errors.New(object.Error.Message) + log.Trace("Error on object %v: %v", object.Pointer, objectError) + if uc != nil { + if _, err := uc(object.Pointer, objectError); err != nil { + return err + } + } else { + if err := dc(object.Pointer, nil, objectError); err != nil { + return err + } + } + continue + } - content, err := transferAdapter.Download(ctx, result.Objects[0]) - if err != nil { - return nil, err + if uc != nil { + if len(object.Actions) == 0 { + log.Trace("%v already present on server", object.Pointer) + continue + } + + link, ok := object.Actions["upload"] + if !ok { + log.Debug("%+v", object) + return errors.New("Missing action 'upload'") + } + + content, err := uc(object.Pointer, nil) + if err != nil { + return err + } + + err = transferAdapter.Upload(ctx, link, object.Pointer, content) + + content.Close() + + if err != nil { + return err + } + + link, ok = object.Actions["verify"] + if ok { + if err := transferAdapter.Verify(ctx, link, object.Pointer); err != nil { + return err + } + } + } else { + link, ok := object.Actions["download"] + if !ok { + log.Debug("%+v", object) + return errors.New("Missing action 'download'") + } + + content, err := transferAdapter.Download(ctx, link) + if err != nil { + return err + } + + if err := dc(object.Pointer, content, nil); err != nil { + return err + } + } } - return content, nil + + return nil } diff --git a/modules/lfs/http_client_test.go b/modules/lfs/http_client_test.go index 68ec947aa..0f633ede5 100644 --- a/modules/lfs/http_client_test.go +++ b/modules/lfs/http_client_test.go @@ -7,13 +7,13 @@ package lfs import ( "bytes" "context" - "encoding/json" "io" "io/ioutil" "net/http" "strings" "testing" + jsoniter "github.com/json-iterator/go" "github.com/stretchr/testify/assert" ) @@ -30,69 +30,253 @@ func (a *DummyTransferAdapter) Name() string { return "dummy" } -func (a *DummyTransferAdapter) Download(ctx context.Context, r *ObjectResponse) (io.ReadCloser, error) { +func (a *DummyTransferAdapter) Download(ctx context.Context, l *Link) (io.ReadCloser, error) { return ioutil.NopCloser(bytes.NewBufferString("dummy")), nil } -func TestHTTPClientDownload(t *testing.T) { - oid := "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041" - size := int64(6) +func (a *DummyTransferAdapter) Upload(ctx context.Context, l *Link, p Pointer, r io.Reader) error { + return nil +} + +func (a *DummyTransferAdapter) Verify(ctx context.Context, l *Link, p Pointer) error { + return nil +} + +func lfsTestRoundtripHandler(req *http.Request) *http.Response { + var batchResponse *BatchResponse + url := req.URL.String() - roundTripHandler := func(req *http.Request) *http.Response { - url := req.URL.String() - if strings.Contains(url, "status-not-ok") { - return &http.Response{StatusCode: http.StatusBadRequest} + if strings.Contains(url, "status-not-ok") { + return &http.Response{StatusCode: http.StatusBadRequest} + } else if strings.Contains(url, "invalid-json-response") { + return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(bytes.NewBufferString("invalid json"))} + } else if strings.Contains(url, "valid-batch-request-download") { + batchResponse = &BatchResponse{ + Transfer: "dummy", + Objects: []*ObjectResponse{ + { + Actions: map[string]*Link{ + "download": {}, + }, + }, + }, + } + } else if strings.Contains(url, "valid-batch-request-upload") { + batchResponse = &BatchResponse{ + Transfer: "dummy", + Objects: []*ObjectResponse{ + { + Actions: map[string]*Link{ + "upload": {}, + }, + }, + }, } - if strings.Contains(url, "invalid-json-response") { - return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(bytes.NewBufferString("invalid json"))} + } else if strings.Contains(url, "response-no-objects") { + batchResponse = &BatchResponse{Transfer: "dummy"} + } else if strings.Contains(url, "unknown-transfer-adapter") { + batchResponse = &BatchResponse{Transfer: "unknown_adapter"} + } else if strings.Contains(url, "error-in-response-objects") { + batchResponse = &BatchResponse{ + Transfer: "dummy", + Objects: []*ObjectResponse{ + { + Error: &ObjectError{ + Code: 404, + Message: "Object not found", + }, + }, + }, } - if strings.Contains(url, "valid-batch-request-download") { - assert.Equal(t, "POST", req.Method) - assert.Equal(t, MediaType, req.Header.Get("Content-type"), "case %s: error should match", url) - assert.Equal(t, MediaType, req.Header.Get("Accept"), "case %s: error should match", url) + } else if strings.Contains(url, "empty-actions-map") { + batchResponse = &BatchResponse{ + Transfer: "dummy", + Objects: []*ObjectResponse{ + { + Actions: map[string]*Link{}, + }, + }, + } + } else if strings.Contains(url, "download-actions-map") { + batchResponse = &BatchResponse{ + Transfer: "dummy", + Objects: []*ObjectResponse{ + { + Actions: map[string]*Link{ + "download": {}, + }, + }, + }, + } + } else if strings.Contains(url, "upload-actions-map") { + batchResponse = &BatchResponse{ + Transfer: "dummy", + Objects: []*ObjectResponse{ + { + Actions: map[string]*Link{ + "upload": {}, + }, + }, + }, + } + } else if strings.Contains(url, "verify-actions-map") { + batchResponse = &BatchResponse{ + Transfer: "dummy", + Objects: []*ObjectResponse{ + { + Actions: map[string]*Link{ + "verify": {}, + }, + }, + }, + } + } else if strings.Contains(url, "unknown-actions-map") { + batchResponse = &BatchResponse{ + Transfer: "dummy", + Objects: []*ObjectResponse{ + { + Actions: map[string]*Link{ + "unknown": {}, + }, + }, + }, + } + } else { + return nil + } - var batchRequest BatchRequest - err := json.NewDecoder(req.Body).Decode(&batchRequest) - assert.NoError(t, err) + payload := new(bytes.Buffer) + jsoniter.NewEncoder(payload).Encode(batchResponse) - assert.Equal(t, "download", batchRequest.Operation) - assert.Len(t, batchRequest.Objects, 1) - assert.Equal(t, oid, batchRequest.Objects[0].Oid) - assert.Equal(t, size, batchRequest.Objects[0].Size) + return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)} +} - batchResponse := &BatchResponse{ - Transfer: "dummy", - Objects: make([]*ObjectResponse, 1), - } +func TestHTTPClientDownload(t *testing.T) { + p := Pointer{Oid: "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041", Size: 6} - payload := new(bytes.Buffer) - json.NewEncoder(payload).Encode(batchResponse) + hc := &http.Client{Transport: RoundTripFunc(func(req *http.Request) *http.Response { + assert.Equal(t, "POST", req.Method) + assert.Equal(t, MediaType, req.Header.Get("Content-type")) + assert.Equal(t, MediaType, req.Header.Get("Accept")) - return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)} - } - if strings.Contains(url, "invalid-response-no-objects") { - batchResponse := &BatchResponse{Transfer: "dummy"} + var batchRequest BatchRequest + err := jsoniter.NewDecoder(req.Body).Decode(&batchRequest) + assert.NoError(t, err) - payload := new(bytes.Buffer) - json.NewEncoder(payload).Encode(batchResponse) + assert.Equal(t, "download", batchRequest.Operation) + assert.Equal(t, 1, len(batchRequest.Objects)) + assert.Equal(t, p.Oid, batchRequest.Objects[0].Oid) + assert.Equal(t, p.Size, batchRequest.Objects[0].Size) - return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)} - } - if strings.Contains(url, "unknown-transfer-adapter") { - batchResponse := &BatchResponse{Transfer: "unknown_adapter"} + return lfsTestRoundtripHandler(req) + })} + dummy := &DummyTransferAdapter{} - payload := new(bytes.Buffer) - json.NewEncoder(payload).Encode(batchResponse) + var cases = []struct { + endpoint string + expectederror string + }{ + // case 0 + { + endpoint: "https://status-not-ok.io", + expectederror: "Unexpected server response: ", + }, + // case 1 + { + endpoint: "https://invalid-json-response.io", + expectederror: "invalid json", + }, + // case 2 + { + endpoint: "https://valid-batch-request-download.io", + expectederror: "", + }, + // case 3 + { + endpoint: "https://response-no-objects.io", + expectederror: "", + }, + // case 4 + { + endpoint: "https://unknown-transfer-adapter.io", + expectederror: "TransferAdapter not found: ", + }, + // case 5 + { + endpoint: "https://error-in-response-objects.io", + expectederror: "Object not found", + }, + // case 6 + { + endpoint: "https://empty-actions-map.io", + expectederror: "Missing action 'download'", + }, + // case 7 + { + endpoint: "https://download-actions-map.io", + expectederror: "", + }, + // case 8 + { + endpoint: "https://upload-actions-map.io", + expectederror: "Missing action 'download'", + }, + // case 9 + { + endpoint: "https://verify-actions-map.io", + expectederror: "Missing action 'download'", + }, + // case 10 + { + endpoint: "https://unknown-actions-map.io", + expectederror: "Missing action 'download'", + }, + } - return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)} + for n, c := range cases { + client := &HTTPClient{ + client: hc, + endpoint: c.endpoint, + transfers: make(map[string]TransferAdapter), } + client.transfers["dummy"] = dummy - t.Errorf("Unknown test case: %s", url) - - return nil + err := client.Download(context.Background(), []Pointer{p}, func(p Pointer, content io.ReadCloser, objectError error) error { + if objectError != nil { + return objectError + } + b, err := io.ReadAll(content) + assert.NoError(t, err) + assert.Equal(t, []byte("dummy"), b) + return nil + }) + if len(c.expectederror) > 0 { + assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) + } else { + assert.NoError(t, err, "case %d", n) + } } +} + +func TestHTTPClientUpload(t *testing.T) { + p := Pointer{Oid: "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041", Size: 6} + + hc := &http.Client{Transport: RoundTripFunc(func(req *http.Request) *http.Response { + assert.Equal(t, "POST", req.Method) + assert.Equal(t, MediaType, req.Header.Get("Content-type")) + assert.Equal(t, MediaType, req.Header.Get("Accept")) + + var batchRequest BatchRequest + err := jsoniter.NewDecoder(req.Body).Decode(&batchRequest) + assert.NoError(t, err) - hc := &http.Client{Transport: RoundTripFunc(roundTripHandler)} + assert.Equal(t, "upload", batchRequest.Operation) + assert.Equal(t, 1, len(batchRequest.Objects)) + assert.Equal(t, p.Oid, batchRequest.Objects[0].Oid) + assert.Equal(t, p.Size, batchRequest.Objects[0].Size) + + return lfsTestRoundtripHandler(req) + })} dummy := &DummyTransferAdapter{} var cases = []struct { @@ -102,27 +286,57 @@ func TestHTTPClientDownload(t *testing.T) { // case 0 { endpoint: "https://status-not-ok.io", - expectederror: "Unexpected servers response: ", + expectederror: "Unexpected server response: ", }, // case 1 { endpoint: "https://invalid-json-response.io", - expectederror: "json.Decode: ", + expectederror: "invalid json", }, // case 2 { - endpoint: "https://valid-batch-request-download.io", + endpoint: "https://valid-batch-request-upload.io", expectederror: "", }, // case 3 { - endpoint: "https://invalid-response-no-objects.io", - expectederror: "No objects in result", + endpoint: "https://response-no-objects.io", + expectederror: "", }, // case 4 { endpoint: "https://unknown-transfer-adapter.io", - expectederror: "Transferadapter not found: ", + expectederror: "TransferAdapter not found: ", + }, + // case 5 + { + endpoint: "https://error-in-response-objects.io", + expectederror: "Object not found", + }, + // case 6 + { + endpoint: "https://empty-actions-map.io", + expectederror: "", + }, + // case 7 + { + endpoint: "https://download-actions-map.io", + expectederror: "Missing action 'upload'", + }, + // case 8 + { + endpoint: "https://upload-actions-map.io", + expectederror: "", + }, + // case 9 + { + endpoint: "https://verify-actions-map.io", + expectederror: "Missing action 'upload'", + }, + // case 10 + { + endpoint: "https://unknown-actions-map.io", + expectederror: "Missing action 'upload'", }, } @@ -134,7 +348,9 @@ func TestHTTPClientDownload(t *testing.T) { } client.transfers["dummy"] = dummy - _, err := client.Download(context.Background(), oid, size) + err := client.Upload(context.Background(), []Pointer{p}, func(p Pointer, objectError error) (io.ReadCloser, error) { + return ioutil.NopCloser(new(bytes.Buffer)), objectError + }) if len(c.expectederror) > 0 { assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) } else { diff --git a/modules/lfs/shared.go b/modules/lfs/shared.go index 9abbf85fb..8343d12e1 100644 --- a/modules/lfs/shared.go +++ b/modules/lfs/shared.go @@ -49,14 +49,14 @@ type ObjectResponse struct { Error *ObjectError `json:"error,omitempty"` } -// Link provides a structure used to build a hypermedia representation of an HTTP link. +// Link provides a structure with informations about how to access a object. type Link struct { Href string `json:"href"` Header map[string]string `json:"header,omitempty"` ExpiresAt *time.Time `json:"expires_at,omitempty"` } -// ObjectError defines the JSON structure returned to the client in case of an error +// ObjectError defines the JSON structure returned to the client in case of an error. type ObjectError struct { Code int `json:"code"` Message string `json:"message"` diff --git a/modules/lfs/transferadapter.go b/modules/lfs/transferadapter.go index ea3aff000..8c40ab8c0 100644 --- a/modules/lfs/transferadapter.go +++ b/modules/lfs/transferadapter.go @@ -5,18 +5,24 @@ package lfs import ( + "bytes" "context" "errors" "fmt" "io" "net/http" + + "code.gitea.io/gitea/modules/log" + + jsoniter "github.com/json-iterator/go" ) // TransferAdapter represents an adapter for downloading/uploading LFS objects type TransferAdapter interface { Name() string - Download(ctx context.Context, r *ObjectResponse) (io.ReadCloser, error) - //Upload(ctx context.Context, reader io.Reader) error + Download(ctx context.Context, l *Link) (io.ReadCloser, error) + Upload(ctx context.Context, l *Link, p Pointer, r io.Reader) error + Verify(ctx context.Context, l *Link, p Pointer) error } // BasicTransferAdapter implements the "basic" adapter @@ -30,29 +36,101 @@ func (a *BasicTransferAdapter) Name() string { } // Download reads the download location and downloads the data -func (a *BasicTransferAdapter) Download(ctx context.Context, r *ObjectResponse) (io.ReadCloser, error) { - download, ok := r.Actions["download"] - if !ok { - return nil, errors.New("lfs.BasicTransferAdapter.Download: Action 'download' not found") +func (a *BasicTransferAdapter) Download(ctx context.Context, l *Link) (io.ReadCloser, error) { + resp, err := a.performRequest(ctx, "GET", l, nil, nil) + if err != nil { + return nil, err } + return resp.Body, nil +} - req, err := http.NewRequestWithContext(ctx, "GET", download.Href, nil) +// Upload sends the content to the LFS server +func (a *BasicTransferAdapter) Upload(ctx context.Context, l *Link, p Pointer, r io.Reader) error { + _, err := a.performRequest(ctx, "PUT", l, r, func(req *http.Request) { + if len(req.Header.Get("Content-Type")) == 0 { + req.Header.Set("Content-Type", "application/octet-stream") + } + + if req.Header.Get("Transfer-Encoding") == "chunked" { + req.TransferEncoding = []string{"chunked"} + } + + req.ContentLength = p.Size + }) if err != nil { - return nil, fmt.Errorf("lfs.BasicTransferAdapter.Download http.NewRequestWithContext: %w", err) + return err } - for key, value := range download.Header { + return nil +} + +// Verify calls the verify handler on the LFS server +func (a *BasicTransferAdapter) Verify(ctx context.Context, l *Link, p Pointer) error { + b, err := jsoniter.Marshal(p) + if err != nil { + log.Error("Error encoding json: %v", err) + return err + } + + _, err = a.performRequest(ctx, "POST", l, bytes.NewReader(b), func(req *http.Request) { + req.Header.Set("Content-Type", MediaType) + }) + if err != nil { + return err + } + return nil +} + +func (a *BasicTransferAdapter) performRequest(ctx context.Context, method string, l *Link, body io.Reader, callback func(*http.Request)) (*http.Response, error) { + log.Trace("Calling: %s %s", method, l.Href) + + req, err := http.NewRequestWithContext(ctx, method, l.Href, body) + if err != nil { + log.Error("Error creating request: %v", err) + return nil, err + } + for key, value := range l.Header { req.Header.Set(key, value) } + req.Header.Set("Accept", MediaType) + + if callback != nil { + callback(req) + } res, err := a.client.Do(req) if err != nil { select { case <-ctx.Done(): - return nil, ctx.Err() + return res, ctx.Err() default: } - return nil, fmt.Errorf("lfs.BasicTransferAdapter.Download http.Do: %w", err) + log.Error("Error while processing request: %v", err) + return res, err + } + + if res.StatusCode != http.StatusOK { + return res, handleErrorResponse(res) + } + + return res, nil +} + +func handleErrorResponse(resp *http.Response) error { + defer resp.Body.Close() + + er, err := decodeReponseError(resp.Body) + if err != nil { + return fmt.Errorf("Request failed with status %s", resp.Status) } + log.Trace("ErrorRespone: %v", er) + return errors.New(er.Message) +} - return res.Body, nil +func decodeReponseError(r io.Reader) (ErrorResponse, error) { + var er ErrorResponse + err := jsoniter.NewDecoder(r).Decode(&er) + if err != nil { + log.Error("Error decoding json: %v", err) + } + return er, err } diff --git a/modules/lfs/transferadapter_test.go b/modules/lfs/transferadapter_test.go index 0eabd3fae..7dfdad417 100644 --- a/modules/lfs/transferadapter_test.go +++ b/modules/lfs/transferadapter_test.go @@ -7,11 +7,13 @@ package lfs import ( "bytes" "context" + "io" "io/ioutil" "net/http" "strings" "testing" + jsoniter "github.com/json-iterator/go" "github.com/stretchr/testify/assert" ) @@ -21,58 +23,151 @@ func TestBasicTransferAdapterName(t *testing.T) { assert.Equal(t, "basic", a.Name()) } -func TestBasicTransferAdapterDownload(t *testing.T) { +func TestBasicTransferAdapter(t *testing.T) { + p := Pointer{Oid: "b5a2c96250612366ea272ffac6d9744aaf4b45aacd96aa7cfcb931ee3b558259", Size: 5} + roundTripHandler := func(req *http.Request) *http.Response { + assert.Equal(t, MediaType, req.Header.Get("Accept")) + assert.Equal(t, "test-value", req.Header.Get("test-header")) + url := req.URL.String() - if strings.Contains(url, "valid-download-request") { + if strings.Contains(url, "download-request") { assert.Equal(t, "GET", req.Method) - assert.Equal(t, "test-value", req.Header.Get("test-header")) return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(bytes.NewBufferString("dummy"))} - } + } else if strings.Contains(url, "upload-request") { + assert.Equal(t, "PUT", req.Method) + assert.Equal(t, "application/octet-stream", req.Header.Get("Content-Type")) + + b, err := io.ReadAll(req.Body) + assert.NoError(t, err) + assert.Equal(t, "dummy", string(b)) - t.Errorf("Unknown test case: %s", url) + return &http.Response{StatusCode: http.StatusOK} + } else if strings.Contains(url, "verify-request") { + assert.Equal(t, "POST", req.Method) + assert.Equal(t, MediaType, req.Header.Get("Content-Type")) - return nil + var vp Pointer + err := jsoniter.NewDecoder(req.Body).Decode(&vp) + assert.NoError(t, err) + assert.Equal(t, p.Oid, vp.Oid) + assert.Equal(t, p.Size, vp.Size) + + return &http.Response{StatusCode: http.StatusOK} + } else if strings.Contains(url, "error-response") { + er := &ErrorResponse{ + Message: "Object not found", + } + payload := new(bytes.Buffer) + jsoniter.NewEncoder(payload).Encode(er) + + return &http.Response{StatusCode: http.StatusNotFound, Body: ioutil.NopCloser(payload)} + } else { + t.Errorf("Unknown test case: %s", url) + return nil + } } hc := &http.Client{Transport: RoundTripFunc(roundTripHandler)} a := &BasicTransferAdapter{hc} - var cases = []struct { - response *ObjectResponse - expectederror string - }{ - // case 0 - { - response: &ObjectResponse{}, - expectederror: "Action 'download' not found", - }, - // case 1 - { - response: &ObjectResponse{ - Actions: map[string]*Link{"upload": nil}, + t.Run("Download", func(t *testing.T) { + cases := []struct { + link *Link + expectederror string + }{ + // case 0 + { + link: &Link{ + Href: "https://download-request.io", + Header: map[string]string{"test-header": "test-value"}, + }, + expectederror: "", }, - expectederror: "Action 'download' not found", - }, - // case 2 - { - response: &ObjectResponse{ - Actions: map[string]*Link{"download": { - Href: "https://valid-download-request.io", + // case 1 + { + link: &Link{ + Href: "https://error-response.io", Header: map[string]string{"test-header": "test-value"}, - }}, + }, + expectederror: "Object not found", }, - expectederror: "", - }, - } + } - for n, c := range cases { - _, err := a.Download(context.Background(), c.response) - if len(c.expectederror) > 0 { - assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) - } else { - assert.NoError(t, err, "case %d", n) + for n, c := range cases { + _, err := a.Download(context.Background(), c.link) + if len(c.expectederror) > 0 { + assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) + } else { + assert.NoError(t, err, "case %d", n) + } } - } + }) + + t.Run("Upload", func(t *testing.T) { + cases := []struct { + link *Link + expectederror string + }{ + // case 0 + { + link: &Link{ + Href: "https://upload-request.io", + Header: map[string]string{"test-header": "test-value"}, + }, + expectederror: "", + }, + // case 1 + { + link: &Link{ + Href: "https://error-response.io", + Header: map[string]string{"test-header": "test-value"}, + }, + expectederror: "Object not found", + }, + } + + for n, c := range cases { + err := a.Upload(context.Background(), c.link, p, bytes.NewBufferString("dummy")) + if len(c.expectederror) > 0 { + assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) + } else { + assert.NoError(t, err, "case %d", n) + } + } + }) + + t.Run("Verify", func(t *testing.T) { + cases := []struct { + link *Link + expectederror string + }{ + // case 0 + { + link: &Link{ + Href: "https://verify-request.io", + Header: map[string]string{"test-header": "test-value"}, + }, + expectederror: "", + }, + // case 1 + { + link: &Link{ + Href: "https://error-response.io", + Header: map[string]string{"test-header": "test-value"}, + }, + expectederror: "Object not found", + }, + } + + for n, c := range cases { + err := a.Verify(context.Background(), c.link, p) + if len(c.expectederror) > 0 { + assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror) + } else { + assert.NoError(t, err, "case %d", n) + } + } + }) } diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 50eb185da..08531c04e 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -7,6 +7,7 @@ package repository import ( "context" "fmt" + "io" "net/url" "path" "strings" @@ -323,64 +324,90 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Reposi errChan := make(chan error, 1) go lfs.SearchPointerBlobs(ctx, gitRepo, pointerChan, errChan) - err := func() error { - for pointerBlob := range pointerChan { - meta, err := models.NewLFSMetaObject(&models.LFSMetaObject{Pointer: pointerBlob.Pointer, RepositoryID: repo.ID}) - if err != nil { - return fmt.Errorf("StoreMissingLfsObjectsInRepository models.NewLFSMetaObject: %w", err) - } - if meta.Existing { - continue + downloadObjects := func(pointers []lfs.Pointer) error { + err := client.Download(ctx, pointers, func(p lfs.Pointer, content io.ReadCloser, objectError error) error { + if objectError != nil { + return objectError } - log.Trace("StoreMissingLfsObjectsInRepository: LFS OID[%s] not present in repository %s", pointerBlob.Oid, repo.FullName()) + defer content.Close() - err = func() error { - exist, err := contentStore.Exists(pointerBlob.Pointer) - if err != nil { - return fmt.Errorf("StoreMissingLfsObjectsInRepository contentStore.Exists: %w", err) - } - if !exist { - if setting.LFS.MaxFileSize > 0 && pointerBlob.Size > setting.LFS.MaxFileSize { - log.Info("LFS OID[%s] download denied because of LFS_MAX_FILE_SIZE=%d < size %d", pointerBlob.Oid, setting.LFS.MaxFileSize, pointerBlob.Size) - return nil - } - - stream, err := client.Download(ctx, pointerBlob.Oid, pointerBlob.Size) - if err != nil { - return fmt.Errorf("StoreMissingLfsObjectsInRepository: LFS OID[%s] failed to download: %w", pointerBlob.Oid, err) - } - defer stream.Close() - - if err := contentStore.Put(pointerBlob.Pointer, stream); err != nil { - return fmt.Errorf("StoreMissingLfsObjectsInRepository LFS OID[%s] contentStore.Put: %w", pointerBlob.Oid, err) - } - } else { - log.Trace("StoreMissingLfsObjectsInRepository: LFS OID[%s] already present in content store", pointerBlob.Oid) - } - return nil - }() + _, err := models.NewLFSMetaObject(&models.LFSMetaObject{Pointer: p, RepositoryID: repo.ID}) if err != nil { - if _, err2 := repo.RemoveLFSMetaObjectByOid(meta.Oid); err2 != nil { - log.Error("StoreMissingLfsObjectsInRepository RemoveLFSMetaObjectByOid[Oid: %s]: %w", meta.Oid, err2) - } + log.Error("Error creating LFS meta object %v: %v", p, err) + return err + } - select { - case <-ctx.Done(): - return nil - default: + if err := contentStore.Put(p, content); err != nil { + log.Error("Error storing content for LFS meta object %v: %v", p, err) + if _, err2 := repo.RemoveLFSMetaObjectByOid(p.Oid); err2 != nil { + log.Error("Error removing LFS meta object %v: %v", p, err2) } return err } + return nil + }) + if err != nil { + select { + case <-ctx.Done(): + return nil + default: + } } - return nil - }() - if err != nil { return err } + var batch []lfs.Pointer + for pointerBlob := range pointerChan { + meta, err := repo.GetLFSMetaObjectByOid(pointerBlob.Oid) + if err != nil && err != models.ErrLFSObjectNotExist { + log.Error("Error querying LFS meta object %v: %v", pointerBlob.Pointer, err) + return err + } + if meta != nil { + log.Trace("Skipping unknown LFS meta object %v", pointerBlob.Pointer) + continue + } + + log.Trace("LFS object %v not present in repository %s", pointerBlob.Pointer, repo.FullName()) + + exist, err := contentStore.Exists(pointerBlob.Pointer) + if err != nil { + log.Error("Error checking if LFS object %v exists: %v", pointerBlob.Pointer, err) + return err + } + + if exist { + log.Trace("LFS object %v already present; creating meta object", pointerBlob.Pointer) + _, err := models.NewLFSMetaObject(&models.LFSMetaObject{Pointer: pointerBlob.Pointer, RepositoryID: repo.ID}) + if err != nil { + log.Error("Error creating LFS meta object %v: %v", pointerBlob.Pointer, err) + return err + } + } else { + if setting.LFS.MaxFileSize > 0 && pointerBlob.Size > setting.LFS.MaxFileSize { + log.Info("LFS object %v download denied because of LFS_MAX_FILE_SIZE=%d < size %d", pointerBlob.Pointer, setting.LFS.MaxFileSize, pointerBlob.Size) + continue + } + + batch = append(batch, pointerBlob.Pointer) + if len(batch) >= client.BatchSize() { + if err := downloadObjects(batch); err != nil { + return err + } + batch = nil + } + } + } + if len(batch) > 0 { + if err := downloadObjects(batch); err != nil { + return err + } + } + err, has := <-errChan if has { + log.Error("Error enumerating LFS objects for repository: %v", err) return err } diff --git a/modules/task/migrate.go b/modules/task/migrate.go index 57424abac..fe9b984d4 100644 --- a/modules/task/migrate.go +++ b/modules/task/migrate.go @@ -118,7 +118,7 @@ func runMigrateTask(t *models.Task) (err error) { } // remoteAddr may contain credentials, so we sanitize it - err = util.URLSanitizedError(err, opts.CloneAddr) + err = util.NewStringURLSanitizedError(err, opts.CloneAddr, true) if strings.Contains(err.Error(), "Authentication failed") || strings.Contains(err.Error(), "could not read Username") { return fmt.Errorf("Authentication failed: %v", err.Error()) diff --git a/modules/task/task.go b/modules/task/task.go index 0685aa23d..1c0a87e1f 100644 --- a/modules/task/task.go +++ b/modules/task/task.go @@ -74,7 +74,7 @@ func CreateMigrateTask(doer, u *models.User, opts base.MigrateOptions) (*models. if err != nil { return nil, err } - opts.CloneAddr = util.SanitizeURLCredentials(opts.CloneAddr, true) + opts.CloneAddr = util.NewStringURLSanitizer(opts.CloneAddr, true).Replace(opts.CloneAddr) opts.AuthPasswordEncrypted, err = secret.EncryptSecret(setting.SecretKey, opts.AuthPassword) if err != nil { return nil, err diff --git a/modules/templates/helper.go b/modules/templates/helper.go index 9922cfb22..83359a6ef 100644 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -27,6 +27,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/emoji" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/repository" @@ -35,7 +36,6 @@ import ( "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/gitdiff" - mirror_service "code.gitea.io/gitea/services/mirror" "github.com/editorconfig/editorconfig-core-go/v2" jsoniter "github.com/json-iterator/go" @@ -294,11 +294,8 @@ func NewFuncMap() []template.FuncMap { } return float32(n) * 100 / float32(sum) }, - "CommentMustAsDiff": gitdiff.CommentMustAsDiff, - "MirrorAddress": mirror_service.Address, - "MirrorFullAddress": mirror_service.AddressNoCredentials, - "MirrorUserName": mirror_service.Username, - "MirrorPassword": mirror_service.Password, + "CommentMustAsDiff": gitdiff.CommentMustAsDiff, + "MirrorRemoteAddress": mirrorRemoteAddress, "CommitType": func(commit interface{}) string { switch commit.(type) { case models.SignCommitWithStatuses: @@ -963,3 +960,28 @@ func buildSubjectBodyTemplate(stpl *texttmpl.Template, btpl *template.Template, log.Warn("Failed to parse template [%s/body]: %v", name, err) } } + +type remoteAddress struct { + Address string + Username string + Password string +} + +func mirrorRemoteAddress(m models.RemoteMirrorer) remoteAddress { + a := remoteAddress{} + + u, err := git.GetRemoteAddress(m.GetRepository().RepoPath(), m.GetRemoteName()) + if err != nil { + log.Error("GetRemoteAddress %v", err) + return a + } + + if u.User != nil { + a.Username = u.User.Username() + a.Password, _ = u.User.Password() + } + u.User = nil + a.Address = u.String() + + return a +} diff --git a/modules/util/sanitize.go b/modules/util/sanitize.go index a4f5479df..de59ffaa2 100644 --- a/modules/util/sanitize.go +++ b/modules/util/sanitize.go @@ -1,4 +1,4 @@ -// Copyright 2017 The Gitea Authors. All rights reserved. +// Copyright 2021 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. @@ -9,40 +9,53 @@ import ( "strings" ) -// urlSafeError wraps an error whose message may contain a sensitive URL -type urlSafeError struct { - err error - unsanitizedURL string +const userPlaceholder = "sanitized-credential" +const unparsableURL = "(unparsable url)" + +type sanitizedError struct { + err error + replacer *strings.Replacer } -func (err urlSafeError) Error() string { - return SanitizeMessage(err.err.Error(), err.unsanitizedURL) +func (err sanitizedError) Error() string { + return err.replacer.Replace(err.err.Error()) } -// URLSanitizedError returns the sanitized version an error whose message may -// contain a sensitive URL -func URLSanitizedError(err error, unsanitizedURL string) error { - return urlSafeError{err: err, unsanitizedURL: unsanitizedURL} +// NewSanitizedError wraps an error and replaces all old, new string pairs in the message text. +func NewSanitizedError(err error, oldnew ...string) error { + return sanitizedError{err: err, replacer: strings.NewReplacer(oldnew...)} } -// SanitizeMessage sanitizes a message which may contains a sensitive URL -func SanitizeMessage(message, unsanitizedURL string) string { - sanitizedURL := SanitizeURLCredentials(unsanitizedURL, true) - return strings.ReplaceAll(message, unsanitizedURL, sanitizedURL) +// NewURLSanitizedError wraps an error and replaces the url credential or removes them. +func NewURLSanitizedError(err error, u *url.URL, usePlaceholder bool) error { + return sanitizedError{err: err, replacer: NewURLSanitizer(u, usePlaceholder)} } -// SanitizeURLCredentials sanitizes a url, either removing user credentials -// or replacing them with a placeholder. -func SanitizeURLCredentials(unsanitizedURL string, usePlaceholder bool) string { - u, err := url.Parse(unsanitizedURL) - if err != nil { - // don't log the error, since it might contain unsanitized URL. - return "(unparsable url)" - } +// NewStringURLSanitizedError wraps an error and replaces the url credential or removes them. +// If the url can't get parsed it gets replaced with a placeholder string. +func NewStringURLSanitizedError(err error, unsanitizedURL string, usePlaceholder bool) error { + return sanitizedError{err: err, replacer: NewStringURLSanitizer(unsanitizedURL, usePlaceholder)} +} + +// NewURLSanitizer creates a replacer for the url with the credential sanitized or removed. +func NewURLSanitizer(u *url.URL, usePlaceholder bool) *strings.Replacer { + old := u.String() + if u.User != nil && usePlaceholder { - u.User = url.User("") + u.User = url.User(userPlaceholder) } else { u.User = nil } - return u.String() + return strings.NewReplacer(old, u.String()) +} + +// NewStringURLSanitizer creates a replacer for the url with the credential sanitized or removed. +// If the url can't get parsed it gets replaced with a placeholder string +func NewStringURLSanitizer(unsanitizedURL string, usePlaceholder bool) *strings.Replacer { + u, err := url.Parse(unsanitizedURL) + if err != nil { + // don't log the error, since it might contain unsanitized URL. + return strings.NewReplacer(unsanitizedURL, unparsableURL) + } + return NewURLSanitizer(u, usePlaceholder) } diff --git a/modules/util/sanitize_test.go b/modules/util/sanitize_test.go index 4f0710067..578f75f51 100644 --- a/modules/util/sanitize_test.go +++ b/modules/util/sanitize_test.go @@ -1,25 +1,164 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. +// Copyright 2021 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. package util import ( + "errors" "testing" "github.com/stretchr/testify/assert" ) -func TestSanitizeURLCredentials(t *testing.T) { - var kases = map[string]string{ - "https://github.com/go-gitea/test_repo.git": "https://github.com/go-gitea/test_repo.git", - "https://mytoken@github.com/go-gitea/test_repo.git": "https://github.com/go-gitea/test_repo.git", - "http://github.com/go-gitea/test_repo.git": "http://github.com/go-gitea/test_repo.git", - "/test/repos/repo1": "/test/repos/repo1", - "git@github.com:go-gitea/test_repo.git": "(unparsable url)", +func TestNewSanitizedError(t *testing.T) { + err := errors.New("error while secret on test") + err2 := NewSanitizedError(err) + assert.Equal(t, err.Error(), err2.Error()) + + var cases = []struct { + input error + oldnew []string + expected string + }{ + // case 0 + { + errors.New("error while secret on test"), + []string{"secret", "replaced"}, + "error while replaced on test", + }, + // case 1 + { + errors.New("error while sec-ret on test"), + []string{"secret", "replaced"}, + "error while sec-ret on test", + }, } - for source, value := range kases { - assert.EqualValues(t, value, SanitizeURLCredentials(source, false)) + for n, c := range cases { + err := NewSanitizedError(c.input, c.oldnew...) + + assert.Equal(t, c.expected, err.Error(), "case %d: error should match", n) + } +} + +func TestNewStringURLSanitizer(t *testing.T) { + var cases = []struct { + input string + placeholder bool + expected string + }{ + // case 0 + { + "https://github.com/go-gitea/test_repo.git", + true, + "https://github.com/go-gitea/test_repo.git", + }, + // case 1 + { + "https://github.com/go-gitea/test_repo.git", + false, + "https://github.com/go-gitea/test_repo.git", + }, + // case 2 + { + "https://mytoken@github.com/go-gitea/test_repo.git", + true, + "https://" + userPlaceholder + "@github.com/go-gitea/test_repo.git", + }, + // case 3 + { + "https://mytoken@github.com/go-gitea/test_repo.git", + false, + "https://github.com/go-gitea/test_repo.git", + }, + // case 4 + { + "https://user:password@github.com/go-gitea/test_repo.git", + true, + "https://" + userPlaceholder + "@github.com/go-gitea/test_repo.git", + }, + // case 5 + { + "https://user:password@github.com/go-gitea/test_repo.git", + false, + "https://github.com/go-gitea/test_repo.git", + }, + // case 6 + { + "https://gi\nthub.com/go-gitea/test_repo.git", + false, + unparsableURL, + }, + } + + for n, c := range cases { + // uses NewURLSanitizer internally + result := NewStringURLSanitizer(c.input, c.placeholder).Replace(c.input) + + assert.Equal(t, c.expected, result, "case %d: error should match", n) + } +} + +func TestNewStringURLSanitizedError(t *testing.T) { + var cases = []struct { + input string + placeholder bool + expected string + }{ + // case 0 + { + "https://github.com/go-gitea/test_repo.git", + true, + "https://github.com/go-gitea/test_repo.git", + }, + // case 1 + { + "https://github.com/go-gitea/test_repo.git", + false, + "https://github.com/go-gitea/test_repo.git", + }, + // case 2 + { + "https://mytoken@github.com/go-gitea/test_repo.git", + true, + "https://" + userPlaceholder + "@github.com/go-gitea/test_repo.git", + }, + // case 3 + { + "https://mytoken@github.com/go-gitea/test_repo.git", + false, + "https://github.com/go-gitea/test_repo.git", + }, + // case 4 + { + "https://user:password@github.com/go-gitea/test_repo.git", + true, + "https://" + userPlaceholder + "@github.com/go-gitea/test_repo.git", + }, + // case 5 + { + "https://user:password@github.com/go-gitea/test_repo.git", + false, + "https://github.com/go-gitea/test_repo.git", + }, + // case 6 + { + "https://gi\nthub.com/go-gitea/test_repo.git", + false, + unparsableURL, + }, + } + + encloseText := func(input string) string { + return "test " + input + " test" + } + + for n, c := range cases { + err := errors.New(encloseText(c.input)) + + result := NewStringURLSanitizedError(err, c.input, c.placeholder) + + assert.Equal(t, encloseText(c.expected), result.Error(), "case %d: error should match", n) } } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 4df9965bc..c6d8d1f61 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -91,8 +91,11 @@ loading = Loading… step1 = Step 1: step2 = Step 2: +error = Error error404 = The page you are trying to reach either does not exist or you are not authorized to view it. +never = Never + [error] occurred = An error has occurred report_message = If you are sure this is a Gitea bug, please search for issue on GitHub and open new issue if necessary. @@ -724,7 +727,7 @@ mirror_prune_desc = Remove obsolete remote-tracking references mirror_interval = Mirror Interval (valid time units are 'h', 'm', 's'). 0 to disable automatic sync. mirror_interval_invalid = The mirror interval is not valid. mirror_address = Clone From URL -mirror_address_desc = Put any required credentials in the Clone Authorization section. +mirror_address_desc = Put any required credentials in the Authorization section. mirror_address_url_invalid = The provided url is invalid. You must escape all components of the url correctly. mirror_address_protocol_invalid = The provided url is invalid. Only http(s):// or git:// locations can be mirrored from. mirror_lfs = Large File Storage (LFS) @@ -787,7 +790,7 @@ form.reach_limit_of_creation_n = You have already reached your limit of %d repos form.name_reserved = The repository name '%s' is reserved. form.name_pattern_not_allowed = The pattern '%s' is not allowed in a repository name. -need_auth = Clone Authorization +need_auth = Authorization migrate_options = Migration Options migrate_service = Migration Service migrate_options_mirror_helper = This repository will be a mirror @@ -1548,6 +1551,15 @@ settings.hooks = Webhooks settings.githooks = Git Hooks settings.basic_settings = Basic Settings settings.mirror_settings = Mirror Settings +settings.mirror_settings.docs = Set up your project to automatically push and/or pull changes to/from another repository. Branches, tags, and commits will be synced automatically. How do I mirror repositories? +settings.mirror_settings.mirrored_repository = Mirrored repository +settings.mirror_settings.direction = Direction +settings.mirror_settings.direction.pull = Pull +settings.mirror_settings.direction.push = Push +settings.mirror_settings.last_update = Last update +settings.mirror_settings.push_mirror.none = No push mirrors configured +settings.mirror_settings.push_mirror.remote_url = Git Remote Repository URL +settings.mirror_settings.push_mirror.add = Add Push Mirror settings.sync_mirror = Synchronize Now settings.mirror_sync_in_progress = Mirror synchronization is in progress. Check back in a minute. settings.email_notifications.enable = Enable Email Notifications diff --git a/routers/api/v1/repo/migrate.go b/routers/api/v1/repo/migrate.go index edae35833..5307fdc7d 100644 --- a/routers/api/v1/repo/migrate.go +++ b/routers/api/v1/repo/migrate.go @@ -231,7 +231,7 @@ func handleMigrateError(ctx *context.APIContext, repoOwner *models.User, remoteA case base.IsErrNotSupported(err): ctx.Error(http.StatusUnprocessableEntity, "", err) default: - err = util.URLSanitizedError(err, remoteAddr) + err = util.NewStringURLSanitizedError(err, remoteAddr, true) if strings.Contains(err.Error(), "Authentication failed") || strings.Contains(err.Error(), "Bad credentials") || strings.Contains(err.Error(), "could not read Username") { diff --git a/routers/web/repo/migrate.go b/routers/web/repo/migrate.go index 24d4ef409..521a856da 100644 --- a/routers/web/repo/migrate.go +++ b/routers/web/repo/migrate.go @@ -101,7 +101,7 @@ func handleMigrateError(ctx *context.Context, owner *models.User, err error, nam ctx.RenderWithErr(ctx.Tr("repo.form.name_pattern_not_allowed", err.(models.ErrNamePatternNotAllowed).Pattern), tpl, form) default: remoteAddr, _ := forms.ParseRemoteAddr(form.CloneAddr, form.AuthUsername, form.AuthPassword) - err = util.URLSanitizedError(err, remoteAddr) + err = util.NewStringURLSanitizedError(err, remoteAddr, true) if strings.Contains(err.Error(), "Authentication failed") || strings.Contains(err.Error(), "Bad credentials") || strings.Contains(err.Error(), "could not read Username") { diff --git a/routers/web/repo/setting.go b/routers/web/repo/setting.go index 21a82491f..c48b19b63 100644 --- a/routers/web/repo/setting.go +++ b/routers/web/repo/setting.go @@ -10,6 +10,7 @@ import ( "fmt" "io/ioutil" "net/http" + "strconv" "strings" "time" @@ -25,6 +26,7 @@ import ( "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/typesniffer" + "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/validation" "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/routers/utils" @@ -49,6 +51,8 @@ func Settings(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("repo.settings") ctx.Data["PageIsSettingsOptions"] = true ctx.Data["ForcePrivate"] = setting.Repository.ForcePrivate + ctx.Data["DisabledMirrors"] = setting.Repository.DisableMirrors + ctx.Data["DefaultMirrorInterval"] = setting.Mirror.DefaultInterval signing, _ := models.SigningKey(ctx.Repo.Repository.RepoPath()) ctx.Data["SigningKeyAvailable"] = len(signing) > 0 @@ -167,10 +171,9 @@ func SettingsPost(ctx *context.Context) { } } - oldUsername := mirror_service.Username(ctx.Repo.Mirror) - oldPassword := mirror_service.Password(ctx.Repo.Mirror) - if form.MirrorPassword == "" && form.MirrorUsername == oldUsername { - form.MirrorPassword = oldPassword + u, _ := git.GetRemoteAddress(ctx.Repo.Repository.RepoPath(), ctx.Repo.Mirror.GetRemoteName()) + if u.User != nil && form.MirrorPassword == "" && form.MirrorUsername == u.User.Username() { + form.MirrorPassword, _ = u.User.Password() } address, err := forms.ParseRemoteAddr(form.MirrorAddress, form.MirrorUsername, form.MirrorPassword) @@ -226,6 +229,92 @@ func SettingsPost(ctx *context.Context) { ctx.Flash.Info(ctx.Tr("repo.settings.mirror_sync_in_progress")) ctx.Redirect(repo.Link() + "/settings") + case "push-mirror-sync": + m, err := selectPushMirrorByForm(form, repo) + if err != nil { + ctx.NotFound("", nil) + return + } + + mirror_service.AddPushMirrorToQueue(m.ID) + + ctx.Flash.Info(ctx.Tr("repo.settings.mirror_sync_in_progress")) + ctx.Redirect(repo.Link() + "/settings") + + case "push-mirror-remove": + // This section doesn't require repo_name/RepoName to be set in the form, don't show it + // as an error on the UI for this action + ctx.Data["Err_RepoName"] = nil + + m, err := selectPushMirrorByForm(form, repo) + if err != nil { + ctx.NotFound("", nil) + return + } + + if err = mirror_service.RemovePushMirrorRemote(m); err != nil { + ctx.ServerError("RemovePushMirrorRemote", err) + return + } + + if err = models.DeletePushMirrorByID(m.ID); err != nil { + ctx.ServerError("DeletePushMirrorByID", err) + return + } + + ctx.Flash.Success(ctx.Tr("repo.settings.update_settings_success")) + ctx.Redirect(repo.Link() + "/settings") + + case "push-mirror-add": + // This section doesn't require repo_name/RepoName to be set in the form, don't show it + // as an error on the UI for this action + ctx.Data["Err_RepoName"] = nil + + interval, err := time.ParseDuration(form.PushMirrorInterval) + if err != nil || (interval != 0 && interval < setting.Mirror.MinInterval) { + ctx.Data["Err_PushMirrorInterval"] = true + ctx.RenderWithErr(ctx.Tr("repo.mirror_interval_invalid"), tplSettingsOptions, &form) + return + } + + address, err := forms.ParseRemoteAddr(form.PushMirrorAddress, form.PushMirrorUsername, form.PushMirrorPassword) + if err == nil { + err = migrations.IsMigrateURLAllowed(address, ctx.User) + } + if err != nil { + ctx.Data["Err_PushMirrorAddress"] = true + handleSettingRemoteAddrError(ctx, err, form) + return + } + + remoteSuffix, err := util.RandomString(10) + if err != nil { + ctx.ServerError("RandomString", err) + return + } + + m := &models.PushMirror{ + RepoID: repo.ID, + Repo: repo, + RemoteName: fmt.Sprintf("remote_mirror_%s", remoteSuffix), + Interval: interval, + } + if err := models.InsertPushMirror(m); err != nil { + ctx.ServerError("InsertPushMirror", err) + return + } + + if err := mirror_service.AddPushMirrorRemote(m, address); err != nil { + if err := models.DeletePushMirrorByID(m.ID); err != nil { + log.Error("DeletePushMirrorByID %v", err) + } + ctx.ServerError("AddPushMirrorRemote", err) + return + } + + ctx.Flash.Success(ctx.Tr("repo.settings.update_settings_success")) + ctx.Redirect(repo.Link() + "/settings") + case "advanced": var repoChanged bool var units []models.RepoUnit @@ -1051,3 +1140,22 @@ func SettingsDeleteAvatar(ctx *context.Context) { } ctx.Redirect(ctx.Repo.RepoLink + "/settings") } + +func selectPushMirrorByForm(form *forms.RepoSettingForm, repo *models.Repository) (*models.PushMirror, error) { + id, err := strconv.ParseInt(form.PushMirrorID, 10, 64) + if err != nil { + return nil, err + } + + if err = repo.LoadPushMirrors(); err != nil { + return nil, err + } + + for _, m := range repo.PushMirrors { + if m.ID == id { + return m, nil + } + } + + return nil, fmt.Errorf("PushMirror[%v] not associated to repository %v", id, repo) +} diff --git a/services/forms/repo_form.go b/services/forms/repo_form.go index 55d1f6e3b..a40b0be9a 100644 --- a/services/forms/repo_form.go +++ b/services/forms/repo_form.go @@ -113,18 +113,23 @@ func ParseRemoteAddr(remoteAddr, authUsername, authPassword string) (string, err // RepoSettingForm form for changing repository settings type RepoSettingForm struct { - RepoName string `binding:"Required;AlphaDashDot;MaxSize(100)"` - Description string `binding:"MaxSize(255)"` - Website string `binding:"ValidUrl;MaxSize(255)"` - Interval string - MirrorAddress string - MirrorUsername string - MirrorPassword string - LFS bool `form:"mirror_lfs"` - LFSEndpoint string `form:"mirror_lfs_endpoint"` - Private bool - Template bool - EnablePrune bool + RepoName string `binding:"Required;AlphaDashDot;MaxSize(100)"` + Description string `binding:"MaxSize(255)"` + Website string `binding:"ValidUrl;MaxSize(255)"` + Interval string + MirrorAddress string + MirrorUsername string + MirrorPassword string + LFS bool `form:"mirror_lfs"` + LFSEndpoint string `form:"mirror_lfs_endpoint"` + PushMirrorID string + PushMirrorAddress string + PushMirrorUsername string + PushMirrorPassword string + PushMirrorInterval string + Private bool + Template bool + EnablePrune bool // Advanced settings EnableWiki bool diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index 839d692f9..1e30c919e 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -7,585 +7,97 @@ package mirror import ( "context" "fmt" - "net/url" "strconv" "strings" - "time" "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/cache" - "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/graceful" - "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/notification" - repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/sync" - "code.gitea.io/gitea/modules/timeutil" - "code.gitea.io/gitea/modules/util" ) // mirrorQueue holds an UniqueQueue object of the mirror var mirrorQueue = sync.NewUniqueQueue(setting.Repository.MirrorQueueLength) -func readAddress(m *models.Mirror) { - if len(m.Address) > 0 { - return - } - var err error - m.Address, err = remoteAddress(m.Repo.RepoPath()) - if err != nil { - log.Error("remoteAddress: %v", err) - } -} - -func remoteAddress(repoPath string) (string, error) { - var cmd *git.Command - err := git.LoadGitVersion() - if err != nil { - return "", err - } - if git.CheckGitVersionAtLeast("2.7") == nil { - cmd = git.NewCommand("remote", "get-url", "origin") - } else { - cmd = git.NewCommand("config", "--get", "remote.origin.url") - } - - result, err := cmd.RunInDir(repoPath) - if err != nil { - if strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { - return "", nil - } - return "", err - } - if len(result) > 0 { - return result[:len(result)-1], nil - } - return "", nil -} - -// sanitizeOutput sanitizes output of a command, replacing occurrences of the -// repository's remote address with a sanitized version. -func sanitizeOutput(output, repoPath string) (string, error) { - remoteAddr, err := remoteAddress(repoPath) - if err != nil { - // if we're unable to load the remote address, then we're unable to - // sanitize. - return "", err - } - return util.SanitizeMessage(output, remoteAddr), nil -} - -// AddressNoCredentials returns mirror address from Git repository config without credentials. -func AddressNoCredentials(m *models.Mirror) string { - readAddress(m) - u, err := url.Parse(m.Address) - if err != nil { - // this shouldn't happen but just return it unsanitised - return m.Address - } - u.User = nil - return u.String() -} - -// UpdateAddress writes new address to Git repository and database -func UpdateAddress(m *models.Mirror, addr string) error { - repoPath := m.Repo.RepoPath() - // Remove old origin - _, err := git.NewCommand("remote", "rm", "origin").RunInDir(repoPath) - if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { - return err - } - - _, err = git.NewCommand("remote", "add", "origin", "--mirror=fetch", addr).RunInDir(repoPath) - if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { - return err - } - - if m.Repo.HasWiki() { - wikiPath := m.Repo.WikiPath() - wikiRemotePath := repo_module.WikiRemoteURL(addr) - // Remove old origin of wiki - _, err := git.NewCommand("remote", "rm", "origin").RunInDir(wikiPath) - if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { - return err - } - - _, err = git.NewCommand("remote", "add", "origin", "--mirror=fetch", wikiRemotePath).RunInDir(wikiPath) - if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { - return err - } - } - - m.Repo.OriginalURL = addr - return models.UpdateRepositoryCols(m.Repo, "original_url") -} - -// gitShortEmptySha Git short empty SHA -const gitShortEmptySha = "0000000" - -// mirrorSyncResult contains information of a updated reference. -// If the oldCommitID is "0000000", it means a new reference, the value of newCommitID is empty. -// If the newCommitID is "0000000", it means the reference is deleted, the value of oldCommitID is empty. -type mirrorSyncResult struct { - refName string - oldCommitID string - newCommitID string -} - -// parseRemoteUpdateOutput detects create, update and delete operations of references from upstream. -func parseRemoteUpdateOutput(output string) []*mirrorSyncResult { - results := make([]*mirrorSyncResult, 0, 3) - lines := strings.Split(output, "\n") - for i := range lines { - // Make sure reference name is presented before continue - idx := strings.Index(lines[i], "-> ") - if idx == -1 { - continue - } - - refName := lines[i][idx+3:] - - switch { - case strings.HasPrefix(lines[i], " * "): // New reference - if strings.HasPrefix(lines[i], " * [new tag]") { - refName = git.TagPrefix + refName - } else if strings.HasPrefix(lines[i], " * [new branch]") { - refName = git.BranchPrefix + refName - } - results = append(results, &mirrorSyncResult{ - refName: refName, - oldCommitID: gitShortEmptySha, - }) - case strings.HasPrefix(lines[i], " - "): // Delete reference - results = append(results, &mirrorSyncResult{ - refName: refName, - newCommitID: gitShortEmptySha, - }) - case strings.HasPrefix(lines[i], " + "): // Force update - if idx := strings.Index(refName, " "); idx > -1 { - refName = refName[:idx] - } - delimIdx := strings.Index(lines[i][3:], " ") - if delimIdx == -1 { - log.Error("SHA delimiter not found: %q", lines[i]) - continue - } - shas := strings.Split(lines[i][3:delimIdx+3], "...") - if len(shas) != 2 { - log.Error("Expect two SHAs but not what found: %q", lines[i]) - continue - } - results = append(results, &mirrorSyncResult{ - refName: refName, - oldCommitID: shas[0], - newCommitID: shas[1], - }) - case strings.HasPrefix(lines[i], " "): // New commits of a reference - delimIdx := strings.Index(lines[i][3:], " ") - if delimIdx == -1 { - log.Error("SHA delimiter not found: %q", lines[i]) - continue - } - shas := strings.Split(lines[i][3:delimIdx+3], "..") - if len(shas) != 2 { - log.Error("Expect two SHAs but not what found: %q", lines[i]) - continue - } - results = append(results, &mirrorSyncResult{ - refName: refName, - oldCommitID: shas[0], - newCommitID: shas[1], - }) - - default: - log.Warn("parseRemoteUpdateOutput: unexpected update line %q", lines[i]) - } - } - return results -} - -// runSync returns true if sync finished without error. -func runSync(ctx context.Context, m *models.Mirror) ([]*mirrorSyncResult, bool) { - repoPath := m.Repo.RepoPath() - wikiPath := m.Repo.WikiPath() - timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second - - log.Trace("SyncMirrors [repo: %-v]: running git remote update...", m.Repo) - gitArgs := []string{"remote", "update"} - if m.EnablePrune { - gitArgs = append(gitArgs, "--prune") - } - - stdoutBuilder := strings.Builder{} - stderrBuilder := strings.Builder{} - if err := git.NewCommand(gitArgs...). - SetDescription(fmt.Sprintf("Mirror.runSync: %s", m.Repo.FullName())). - RunInDirTimeoutPipeline(timeout, repoPath, &stdoutBuilder, &stderrBuilder); err != nil { - stdout := stdoutBuilder.String() - stderr := stderrBuilder.String() - // sanitize the output, since it may contain the remote address, which may - // contain a password - stderrMessage, sanitizeErr := sanitizeOutput(stderr, repoPath) - if sanitizeErr != nil { - log.Error("sanitizeOutput failed on stderr: %v", sanitizeErr) - } - stdoutMessage, sanitizeErr := sanitizeOutput(stdout, repoPath) - if sanitizeErr != nil { - log.Error("sanitizeOutput failed: %v", sanitizeErr) - } - - log.Error("Failed to update mirror repository %v:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdoutMessage, stderrMessage, err) - desc := fmt.Sprintf("Failed to update mirror repository '%s': %s", m.Repo.FullName(), stderrMessage) - if err = models.CreateRepositoryNotice(desc); err != nil { - log.Error("CreateRepositoryNotice: %v", err) - } - return nil, false - } - output := stderrBuilder.String() - - gitRepo, err := git.OpenRepository(repoPath) - if err != nil { - log.Error("OpenRepository: %v", err) - return nil, false - } - defer gitRepo.Close() - - log.Trace("SyncMirrors [repo: %-v]: syncing releases with tags...", m.Repo) - if err = repo_module.SyncReleasesWithTags(m.Repo, gitRepo); err != nil { - log.Error("Failed to synchronize tags to releases for repository: %v", err) - } - - if m.LFS && setting.LFS.StartServer { - log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo) - readAddress(m) - ep := lfs.DetermineEndpoint(m.Address, m.LFSEndpoint) - if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, ep); err != nil { - log.Error("Failed to synchronize LFS objects for repository: %v", err) - } - } - - log.Trace("SyncMirrors [repo: %-v]: updating size of repository", m.Repo) - if err := m.Repo.UpdateSize(models.DefaultDBContext()); err != nil { - log.Error("Failed to update size for mirror repository: %v", err) - } - - if m.Repo.HasWiki() { - log.Trace("SyncMirrors [repo: %-v Wiki]: running git remote update...", m.Repo) - stderrBuilder.Reset() - stdoutBuilder.Reset() - if err := git.NewCommand("remote", "update", "--prune"). - SetDescription(fmt.Sprintf("Mirror.runSync Wiki: %s ", m.Repo.FullName())). - RunInDirTimeoutPipeline(timeout, wikiPath, &stdoutBuilder, &stderrBuilder); err != nil { - stdout := stdoutBuilder.String() - stderr := stderrBuilder.String() - // sanitize the output, since it may contain the remote address, which may - // contain a password - stderrMessage, sanitizeErr := sanitizeOutput(stderr, wikiPath) - if sanitizeErr != nil { - log.Error("sanitizeOutput failed on stderr: %v", sanitizeErr) - } - stdoutMessage, sanitizeErr := sanitizeOutput(stdout, wikiPath) - if sanitizeErr != nil { - log.Error("sanitizeOutput failed: %v", sanitizeErr) - } - - log.Error("Failed to update mirror repository wiki %v:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdoutMessage, stderrMessage, err) - desc := fmt.Sprintf("Failed to update mirror repository wiki '%s': %s", m.Repo.FullName(), stderrMessage) - if err = models.CreateRepositoryNotice(desc); err != nil { - log.Error("CreateRepositoryNotice: %v", err) - } - return nil, false - } - log.Trace("SyncMirrors [repo: %-v Wiki]: git remote update complete", m.Repo) - } - - log.Trace("SyncMirrors [repo: %-v]: invalidating mirror branch caches...", m.Repo) - branches, _, err := repo_module.GetBranches(m.Repo, 0, 0) - if err != nil { - log.Error("GetBranches: %v", err) - return nil, false - } - - for _, branch := range branches { - cache.Remove(m.Repo.GetCommitsCountCacheKey(branch.Name, true)) - } - - m.UpdatedUnix = timeutil.TimeStampNow() - return parseRemoteUpdateOutput(output), true -} - -// Address returns mirror address from Git repository config without credentials. -func Address(m *models.Mirror) string { - readAddress(m) - return util.SanitizeURLCredentials(m.Address, false) -} - -// Username returns the mirror address username -func Username(m *models.Mirror) string { - readAddress(m) - u, err := url.Parse(m.Address) - if err != nil { - // this shouldn't happen but if it does return "" - return "" - } - return u.User.Username() -} - -// Password returns the mirror address password -func Password(m *models.Mirror) string { - readAddress(m) - u, err := url.Parse(m.Address) - if err != nil { - // this shouldn't happen but if it does return "" - return "" - } - password, _ := u.User.Password() - return password -} - // Update checks and updates mirror repositories. func Update(ctx context.Context) error { log.Trace("Doing: Update") - if err := models.MirrorsIterate(func(idx int, bean interface{}) error { - m := bean.(*models.Mirror) - if m.Repo == nil { - log.Error("Disconnected mirror repository found: %d", m.ID) + + handler := func(idx int, bean interface{}) error { + var item string + if m, ok := bean.(*models.Mirror); ok { + if m.Repo == nil { + log.Error("Disconnected mirror found: %d", m.ID) + return nil + } + item = fmt.Sprintf("pull %d", m.RepoID) + } else if m, ok := bean.(*models.PushMirror); ok { + if m.Repo == nil { + log.Error("Disconnected push-mirror found: %d", m.ID) + return nil + } + item = fmt.Sprintf("push %d", m.ID) + } else { + log.Error("Unknown bean: %v", bean) return nil } + select { case <-ctx.Done(): return fmt.Errorf("Aborted") default: - mirrorQueue.Add(m.RepoID) + mirrorQueue.Add(item) return nil } - }); err != nil { - log.Trace("Update: %v", err) + } + + if err := models.MirrorsIterate(handler); err != nil { + log.Error("MirrorsIterate: %v", err) + return err + } + if err := models.PushMirrorsIterate(handler); err != nil { + log.Error("PushMirrorsIterate: %v", err) return err } log.Trace("Finished: Update") return nil } -// SyncMirrors checks and syncs mirrors. +// syncMirrors checks and syncs mirrors. // FIXME: graceful: this should be a persistable queue -func SyncMirrors(ctx context.Context) { +func syncMirrors(ctx context.Context) { // Start listening on new sync requests. for { select { case <-ctx.Done(): mirrorQueue.Close() return - case repoID := <-mirrorQueue.Queue(): - syncMirror(ctx, repoID) - } - } -} - -func syncMirror(ctx context.Context, repoID string) { - log.Trace("SyncMirrors [repo_id: %v]", repoID) - defer func() { - err := recover() - if err == nil { - return - } - // There was a panic whilst syncMirrors... - log.Error("PANIC whilst syncMirrors[%s] Panic: %v\nStacktrace: %s", repoID, err, log.Stack(2)) - }() - mirrorQueue.Remove(repoID) - - id, _ := strconv.ParseInt(repoID, 10, 64) - m, err := models.GetMirrorByRepoID(id) - if err != nil { - log.Error("GetMirrorByRepoID [%s]: %v", repoID, err) - return - } - - log.Trace("SyncMirrors [repo: %-v]: Running Sync", m.Repo) - results, ok := runSync(ctx, m) - if !ok { - return - } - - log.Trace("SyncMirrors [repo: %-v]: Scheduling next update", m.Repo) - m.ScheduleNextUpdate() - if err = models.UpdateMirror(m); err != nil { - log.Error("UpdateMirror [%s]: %v", repoID, err) - return - } - - var gitRepo *git.Repository - if len(results) == 0 { - log.Trace("SyncMirrors [repo: %-v]: no branches updated", m.Repo) - } else { - log.Trace("SyncMirrors [repo: %-v]: %d branches updated", m.Repo, len(results)) - gitRepo, err = git.OpenRepository(m.Repo.RepoPath()) - if err != nil { - log.Error("OpenRepository [%d]: %v", m.RepoID, err) - return - } - defer gitRepo.Close() - - if ok := checkAndUpdateEmptyRepository(m, gitRepo, results); !ok { - return - } - } - - for _, result := range results { - // Discard GitHub pull requests, i.e. refs/pull/* - if strings.HasPrefix(result.refName, "refs/pull/") { - continue - } - - tp, _ := git.SplitRefName(result.refName) - - // Create reference - if result.oldCommitID == gitShortEmptySha { - if tp == git.TagPrefix { - tp = "tag" - } else if tp == git.BranchPrefix { - tp = "branch" + case item := <-mirrorQueue.Queue(): + id, _ := strconv.ParseInt(item[5:], 10, 64) + if strings.HasPrefix(item, "pull") { + _ = SyncPullMirror(ctx, id) + } else if strings.HasPrefix(item, "push") { + _ = SyncPushMirror(ctx, id) + } else { + log.Error("Unknown item in queue: %v", item) } - commitID, err := gitRepo.GetRefCommitID(result.refName) - if err != nil { - log.Error("gitRepo.GetRefCommitID [repo_id: %s, ref_name: %s]: %v", m.RepoID, result.refName, err) - continue - } - notification.NotifySyncPushCommits(m.Repo.MustOwner(), m.Repo, &repo_module.PushUpdateOptions{ - RefFullName: result.refName, - OldCommitID: git.EmptySHA, - NewCommitID: commitID, - }, repo_module.NewPushCommits()) - notification.NotifySyncCreateRef(m.Repo.MustOwner(), m.Repo, tp, result.refName) - continue - } - - // Delete reference - if result.newCommitID == gitShortEmptySha { - notification.NotifySyncDeleteRef(m.Repo.MustOwner(), m.Repo, tp, result.refName) - continue - } - - // Push commits - oldCommitID, err := git.GetFullCommitID(gitRepo.Path, result.oldCommitID) - if err != nil { - log.Error("GetFullCommitID [%d]: %v", m.RepoID, err) - continue - } - newCommitID, err := git.GetFullCommitID(gitRepo.Path, result.newCommitID) - if err != nil { - log.Error("GetFullCommitID [%d]: %v", m.RepoID, err) - continue - } - commits, err := gitRepo.CommitsBetweenIDs(newCommitID, oldCommitID) - if err != nil { - log.Error("CommitsBetweenIDs [repo_id: %d, new_commit_id: %s, old_commit_id: %s]: %v", m.RepoID, newCommitID, oldCommitID, err) - continue + mirrorQueue.Remove(item) } - - theCommits := repo_module.ListToPushCommits(commits) - if len(theCommits.Commits) > setting.UI.FeedMaxCommitNum { - theCommits.Commits = theCommits.Commits[:setting.UI.FeedMaxCommitNum] - } - - theCommits.CompareURL = m.Repo.ComposeCompareURL(oldCommitID, newCommitID) - - notification.NotifySyncPushCommits(m.Repo.MustOwner(), m.Repo, &repo_module.PushUpdateOptions{ - RefFullName: result.refName, - OldCommitID: oldCommitID, - NewCommitID: newCommitID, - }, theCommits) - } - log.Trace("SyncMirrors [repo: %-v]: done notifying updated branches/tags - now updating last commit time", m.Repo) - - // Get latest commit date and update to current repository updated time - commitDate, err := git.GetLatestCommitTime(m.Repo.RepoPath()) - if err != nil { - log.Error("GetLatestCommitDate [%d]: %v", m.RepoID, err) - return - } - - if err = models.UpdateRepositoryUpdatedTime(m.RepoID, commitDate); err != nil { - log.Error("Update repository 'updated_unix' [%d]: %v", m.RepoID, err) - return } - - log.Trace("SyncMirrors [repo: %-v]: Successfully updated", m.Repo) -} - -func checkAndUpdateEmptyRepository(m *models.Mirror, gitRepo *git.Repository, results []*mirrorSyncResult) bool { - if !m.Repo.IsEmpty { - return true - } - - hasDefault := false - hasMaster := false - hasMain := false - defaultBranchName := m.Repo.DefaultBranch - if len(defaultBranchName) == 0 { - defaultBranchName = setting.Repository.DefaultBranch - } - firstName := "" - for _, result := range results { - if strings.HasPrefix(result.refName, "refs/pull/") { - continue - } - tp, name := git.SplitRefName(result.refName) - if len(tp) > 0 && tp != git.BranchPrefix { - continue - } - if len(firstName) == 0 { - firstName = name - } - - hasDefault = hasDefault || name == defaultBranchName - hasMaster = hasMaster || name == "master" - hasMain = hasMain || name == "main" - } - - if len(firstName) > 0 { - if hasDefault { - m.Repo.DefaultBranch = defaultBranchName - } else if hasMaster { - m.Repo.DefaultBranch = "master" - } else if hasMain { - m.Repo.DefaultBranch = "main" - } else { - m.Repo.DefaultBranch = firstName - } - // Update the git repository default branch - if err := gitRepo.SetDefaultBranch(m.Repo.DefaultBranch); err != nil { - if !git.IsErrUnsupportedVersion(err) { - log.Error("Failed to update default branch of underlying git repository %-v. Error: %v", m.Repo, err) - desc := fmt.Sprintf("Failed to uupdate default branch of underlying git repository '%s': %v", m.Repo.RepoPath(), err) - if err = models.CreateRepositoryNotice(desc); err != nil { - log.Error("CreateRepositoryNotice: %v", err) - } - return false - } - } - m.Repo.IsEmpty = false - // Update the is empty and default_branch columns - if err := models.UpdateRepositoryCols(m.Repo, "default_branch", "is_empty"); err != nil { - log.Error("Failed to update default branch of repository %-v. Error: %v", m.Repo, err) - desc := fmt.Sprintf("Failed to uupdate default branch of repository '%s': %v", m.Repo.RepoPath(), err) - if err = models.CreateRepositoryNotice(desc); err != nil { - log.Error("CreateRepositoryNotice: %v", err) - } - return false - } - } - return true } // InitSyncMirrors initializes a go routine to sync the mirrors func InitSyncMirrors() { - go graceful.GetManager().RunWithShutdownContext(SyncMirrors) + go graceful.GetManager().RunWithShutdownContext(syncMirrors) } // StartToMirror adds repoID to mirror queue func StartToMirror(repoID int64) { - go mirrorQueue.Add(repoID) + go mirrorQueue.Add(fmt.Sprintf("pull %d", repoID)) +} + +// AddPushMirrorToQueue adds the push mirror to the queue +func AddPushMirrorToQueue(mirrorID int64) { + go mirrorQueue.Add(fmt.Sprintf("push %d", mirrorID)) } diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go new file mode 100644 index 000000000..a16724b36 --- /dev/null +++ b/services/mirror/mirror_pull.go @@ -0,0 +1,452 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package mirror + +import ( + "context" + "fmt" + "strings" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/notification" + repo_module "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" +) + +// gitShortEmptySha Git short empty SHA +const gitShortEmptySha = "0000000" + +// UpdateAddress writes new address to Git repository and database +func UpdateAddress(m *models.Mirror, addr string) error { + remoteName := m.GetRemoteName() + repoPath := m.Repo.RepoPath() + // Remove old remote + _, err := git.NewCommand("remote", "rm", remoteName).RunInDir(repoPath) + if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { + return err + } + + _, err = git.NewCommand("remote", "add", remoteName, "--mirror=fetch", addr).RunInDir(repoPath) + if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { + return err + } + + if m.Repo.HasWiki() { + wikiPath := m.Repo.WikiPath() + wikiRemotePath := repo_module.WikiRemoteURL(addr) + // Remove old remote of wiki + _, err := git.NewCommand("remote", "rm", remoteName).RunInDir(wikiPath) + if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { + return err + } + + _, err = git.NewCommand("remote", "add", remoteName, "--mirror=fetch", wikiRemotePath).RunInDir(wikiPath) + if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { + return err + } + } + + m.Repo.OriginalURL = addr + return models.UpdateRepositoryCols(m.Repo, "original_url") +} + +// mirrorSyncResult contains information of a updated reference. +// If the oldCommitID is "0000000", it means a new reference, the value of newCommitID is empty. +// If the newCommitID is "0000000", it means the reference is deleted, the value of oldCommitID is empty. +type mirrorSyncResult struct { + refName string + oldCommitID string + newCommitID string +} + +// parseRemoteUpdateOutput detects create, update and delete operations of references from upstream. +func parseRemoteUpdateOutput(output string) []*mirrorSyncResult { + results := make([]*mirrorSyncResult, 0, 3) + lines := strings.Split(output, "\n") + for i := range lines { + // Make sure reference name is presented before continue + idx := strings.Index(lines[i], "-> ") + if idx == -1 { + continue + } + + refName := lines[i][idx+3:] + + switch { + case strings.HasPrefix(lines[i], " * "): // New reference + if strings.HasPrefix(lines[i], " * [new tag]") { + refName = git.TagPrefix + refName + } else if strings.HasPrefix(lines[i], " * [new branch]") { + refName = git.BranchPrefix + refName + } + results = append(results, &mirrorSyncResult{ + refName: refName, + oldCommitID: gitShortEmptySha, + }) + case strings.HasPrefix(lines[i], " - "): // Delete reference + results = append(results, &mirrorSyncResult{ + refName: refName, + newCommitID: gitShortEmptySha, + }) + case strings.HasPrefix(lines[i], " + "): // Force update + if idx := strings.Index(refName, " "); idx > -1 { + refName = refName[:idx] + } + delimIdx := strings.Index(lines[i][3:], " ") + if delimIdx == -1 { + log.Error("SHA delimiter not found: %q", lines[i]) + continue + } + shas := strings.Split(lines[i][3:delimIdx+3], "...") + if len(shas) != 2 { + log.Error("Expect two SHAs but not what found: %q", lines[i]) + continue + } + results = append(results, &mirrorSyncResult{ + refName: refName, + oldCommitID: shas[0], + newCommitID: shas[1], + }) + case strings.HasPrefix(lines[i], " "): // New commits of a reference + delimIdx := strings.Index(lines[i][3:], " ") + if delimIdx == -1 { + log.Error("SHA delimiter not found: %q", lines[i]) + continue + } + shas := strings.Split(lines[i][3:delimIdx+3], "..") + if len(shas) != 2 { + log.Error("Expect two SHAs but not what found: %q", lines[i]) + continue + } + results = append(results, &mirrorSyncResult{ + refName: refName, + oldCommitID: shas[0], + newCommitID: shas[1], + }) + + default: + log.Warn("parseRemoteUpdateOutput: unexpected update line %q", lines[i]) + } + } + return results +} + +// runSync returns true if sync finished without error. +func runSync(ctx context.Context, m *models.Mirror) ([]*mirrorSyncResult, bool) { + repoPath := m.Repo.RepoPath() + wikiPath := m.Repo.WikiPath() + timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second + + log.Trace("SyncMirrors [repo: %-v]: running git remote update...", m.Repo) + gitArgs := []string{"remote", "update"} + if m.EnablePrune { + gitArgs = append(gitArgs, "--prune") + } + gitArgs = append(gitArgs, m.GetRemoteName()) + + remoteAddr, remoteErr := git.GetRemoteAddress(repoPath, m.GetRemoteName()) + if remoteErr != nil { + log.Error("GetRemoteAddress Error %v", remoteErr) + } + + stdoutBuilder := strings.Builder{} + stderrBuilder := strings.Builder{} + if err := git.NewCommand(gitArgs...). + SetDescription(fmt.Sprintf("Mirror.runSync: %s", m.Repo.FullName())). + RunInDirTimeoutPipeline(timeout, repoPath, &stdoutBuilder, &stderrBuilder); err != nil { + stdout := stdoutBuilder.String() + stderr := stderrBuilder.String() + + // sanitize the output, since it may contain the remote address, which may + // contain a password + + sanitizer := util.NewURLSanitizer(remoteAddr, true) + stderrMessage := sanitizer.Replace(stderr) + stdoutMessage := sanitizer.Replace(stdout) + + log.Error("Failed to update mirror repository %v:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdoutMessage, stderrMessage, err) + desc := fmt.Sprintf("Failed to update mirror repository '%s': %s", repoPath, stderrMessage) + if err = models.CreateRepositoryNotice(desc); err != nil { + log.Error("CreateRepositoryNotice: %v", err) + } + return nil, false + } + output := stderrBuilder.String() + + gitRepo, err := git.OpenRepository(repoPath) + if err != nil { + log.Error("OpenRepository: %v", err) + return nil, false + } + + log.Trace("SyncMirrors [repo: %-v]: syncing releases with tags...", m.Repo) + if err = repo_module.SyncReleasesWithTags(m.Repo, gitRepo); err != nil { + log.Error("Failed to synchronize tags to releases for repository: %v", err) + } + + if m.LFS && setting.LFS.StartServer { + log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo) + ep := lfs.DetermineEndpoint(remoteAddr.String(), m.LFSEndpoint) + if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, ep); err != nil { + log.Error("Failed to synchronize LFS objects for repository: %v", err) + } + } + gitRepo.Close() + + log.Trace("SyncMirrors [repo: %-v]: updating size of repository", m.Repo) + if err := m.Repo.UpdateSize(models.DefaultDBContext()); err != nil { + log.Error("Failed to update size for mirror repository: %v", err) + } + + if m.Repo.HasWiki() { + log.Trace("SyncMirrors [repo: %-v Wiki]: running git remote update...", m.Repo) + stderrBuilder.Reset() + stdoutBuilder.Reset() + if err := git.NewCommand("remote", "update", "--prune", m.GetRemoteName()). + SetDescription(fmt.Sprintf("Mirror.runSync Wiki: %s ", m.Repo.FullName())). + RunInDirTimeoutPipeline(timeout, wikiPath, &stdoutBuilder, &stderrBuilder); err != nil { + stdout := stdoutBuilder.String() + stderr := stderrBuilder.String() + + // sanitize the output, since it may contain the remote address, which may + // contain a password + + remoteAddr, remoteErr := git.GetRemoteAddress(wikiPath, m.GetRemoteName()) + if remoteErr != nil { + log.Error("GetRemoteAddress Error %v", remoteErr) + } + + sanitizer := util.NewURLSanitizer(remoteAddr, true) + stderrMessage := sanitizer.Replace(stderr) + stdoutMessage := sanitizer.Replace(stdout) + + log.Error("Failed to update mirror repository wiki %v:\nStdout: %s\nStderr: %s\nErr: %v", m.Repo, stdoutMessage, stderrMessage, err) + desc := fmt.Sprintf("Failed to update mirror repository wiki '%s': %s", wikiPath, stderrMessage) + if err = models.CreateRepositoryNotice(desc); err != nil { + log.Error("CreateRepositoryNotice: %v", err) + } + return nil, false + } + log.Trace("SyncMirrors [repo: %-v Wiki]: git remote update complete", m.Repo) + } + + log.Trace("SyncMirrors [repo: %-v]: invalidating mirror branch caches...", m.Repo) + branches, _, err := repo_module.GetBranches(m.Repo, 0, 0) + if err != nil { + log.Error("GetBranches: %v", err) + return nil, false + } + + for _, branch := range branches { + cache.Remove(m.Repo.GetCommitsCountCacheKey(branch.Name, true)) + } + + m.UpdatedUnix = timeutil.TimeStampNow() + return parseRemoteUpdateOutput(output), true +} + +// SyncPullMirror starts the sync of the pull mirror and schedules the next run. +func SyncPullMirror(ctx context.Context, repoID int64) bool { + log.Trace("SyncMirrors [repo_id: %v]", repoID) + defer func() { + err := recover() + if err == nil { + return + } + // There was a panic whilst syncMirrors... + log.Error("PANIC whilst syncMirrors[%d] Panic: %v\nStacktrace: %s", repoID, err, log.Stack(2)) + }() + + m, err := models.GetMirrorByRepoID(repoID) + if err != nil { + log.Error("GetMirrorByRepoID [%d]: %v", repoID, err) + return false + } + + log.Trace("SyncMirrors [repo: %-v]: Running Sync", m.Repo) + results, ok := runSync(ctx, m) + if !ok { + return false + } + + log.Trace("SyncMirrors [repo: %-v]: Scheduling next update", m.Repo) + m.ScheduleNextUpdate() + if err = models.UpdateMirror(m); err != nil { + log.Error("UpdateMirror [%d]: %v", m.RepoID, err) + return false + } + + var gitRepo *git.Repository + if len(results) == 0 { + log.Trace("SyncMirrors [repo: %-v]: no branches updated", m.Repo) + } else { + log.Trace("SyncMirrors [repo: %-v]: %d branches updated", m.Repo, len(results)) + gitRepo, err = git.OpenRepository(m.Repo.RepoPath()) + if err != nil { + log.Error("OpenRepository [%d]: %v", m.RepoID, err) + return false + } + defer gitRepo.Close() + + if ok := checkAndUpdateEmptyRepository(m, gitRepo, results); !ok { + return false + } + } + + for _, result := range results { + // Discard GitHub pull requests, i.e. refs/pull/* + if strings.HasPrefix(result.refName, "refs/pull/") { + continue + } + + tp, _ := git.SplitRefName(result.refName) + + // Create reference + if result.oldCommitID == gitShortEmptySha { + if tp == git.TagPrefix { + tp = "tag" + } else if tp == git.BranchPrefix { + tp = "branch" + } + commitID, err := gitRepo.GetRefCommitID(result.refName) + if err != nil { + log.Error("gitRepo.GetRefCommitID [repo_id: %d, ref_name: %s]: %v", m.RepoID, result.refName, err) + continue + } + notification.NotifySyncPushCommits(m.Repo.MustOwner(), m.Repo, &repo_module.PushUpdateOptions{ + RefFullName: result.refName, + OldCommitID: git.EmptySHA, + NewCommitID: commitID, + }, repo_module.NewPushCommits()) + notification.NotifySyncCreateRef(m.Repo.MustOwner(), m.Repo, tp, result.refName) + continue + } + + // Delete reference + if result.newCommitID == gitShortEmptySha { + notification.NotifySyncDeleteRef(m.Repo.MustOwner(), m.Repo, tp, result.refName) + continue + } + + // Push commits + oldCommitID, err := git.GetFullCommitID(gitRepo.Path, result.oldCommitID) + if err != nil { + log.Error("GetFullCommitID [%d]: %v", m.RepoID, err) + continue + } + newCommitID, err := git.GetFullCommitID(gitRepo.Path, result.newCommitID) + if err != nil { + log.Error("GetFullCommitID [%d]: %v", m.RepoID, err) + continue + } + commits, err := gitRepo.CommitsBetweenIDs(newCommitID, oldCommitID) + if err != nil { + log.Error("CommitsBetweenIDs [repo_id: %d, new_commit_id: %s, old_commit_id: %s]: %v", m.RepoID, newCommitID, oldCommitID, err) + continue + } + + theCommits := repo_module.ListToPushCommits(commits) + if len(theCommits.Commits) > setting.UI.FeedMaxCommitNum { + theCommits.Commits = theCommits.Commits[:setting.UI.FeedMaxCommitNum] + } + + theCommits.CompareURL = m.Repo.ComposeCompareURL(oldCommitID, newCommitID) + + notification.NotifySyncPushCommits(m.Repo.MustOwner(), m.Repo, &repo_module.PushUpdateOptions{ + RefFullName: result.refName, + OldCommitID: oldCommitID, + NewCommitID: newCommitID, + }, theCommits) + } + log.Trace("SyncMirrors [repo: %-v]: done notifying updated branches/tags - now updating last commit time", m.Repo) + + // Get latest commit date and update to current repository updated time + commitDate, err := git.GetLatestCommitTime(m.Repo.RepoPath()) + if err != nil { + log.Error("GetLatestCommitDate [%d]: %v", m.RepoID, err) + return false + } + + if err = models.UpdateRepositoryUpdatedTime(m.RepoID, commitDate); err != nil { + log.Error("Update repository 'updated_unix' [%d]: %v", m.RepoID, err) + return false + } + + log.Trace("SyncMirrors [repo: %-v]: Successfully updated", m.Repo) + + return true +} + +func checkAndUpdateEmptyRepository(m *models.Mirror, gitRepo *git.Repository, results []*mirrorSyncResult) bool { + if !m.Repo.IsEmpty { + return true + } + + hasDefault := false + hasMaster := false + hasMain := false + defaultBranchName := m.Repo.DefaultBranch + if len(defaultBranchName) == 0 { + defaultBranchName = setting.Repository.DefaultBranch + } + firstName := "" + for _, result := range results { + if strings.HasPrefix(result.refName, "refs/pull/") { + continue + } + tp, name := git.SplitRefName(result.refName) + if len(tp) > 0 && tp != git.BranchPrefix { + continue + } + if len(firstName) == 0 { + firstName = name + } + + hasDefault = hasDefault || name == defaultBranchName + hasMaster = hasMaster || name == "master" + hasMain = hasMain || name == "main" + } + + if len(firstName) > 0 { + if hasDefault { + m.Repo.DefaultBranch = defaultBranchName + } else if hasMaster { + m.Repo.DefaultBranch = "master" + } else if hasMain { + m.Repo.DefaultBranch = "main" + } else { + m.Repo.DefaultBranch = firstName + } + // Update the git repository default branch + if err := gitRepo.SetDefaultBranch(m.Repo.DefaultBranch); err != nil { + if !git.IsErrUnsupportedVersion(err) { + log.Error("Failed to update default branch of underlying git repository %-v. Error: %v", m.Repo, err) + desc := fmt.Sprintf("Failed to uupdate default branch of underlying git repository '%s': %v", m.Repo.RepoPath(), err) + if err = models.CreateRepositoryNotice(desc); err != nil { + log.Error("CreateRepositoryNotice: %v", err) + } + return false + } + } + m.Repo.IsEmpty = false + // Update the is empty and default_branch columns + if err := models.UpdateRepositoryCols(m.Repo, "default_branch", "is_empty"); err != nil { + log.Error("Failed to update default branch of repository %-v. Error: %v", m.Repo, err) + desc := fmt.Sprintf("Failed to uupdate default branch of repository '%s': %v", m.Repo.RepoPath(), err) + if err = models.CreateRepositoryNotice(desc); err != nil { + log.Error("CreateRepositoryNotice: %v", err) + } + return false + } + } + return true +} diff --git a/services/mirror/mirror_push.go b/services/mirror/mirror_push.go new file mode 100644 index 000000000..de8130368 --- /dev/null +++ b/services/mirror/mirror_push.go @@ -0,0 +1,242 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package mirror + +import ( + "context" + "errors" + "io" + "net/url" + "regexp" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" +) + +var stripExitStatus = regexp.MustCompile(`exit status \d+ - `) + +// AddPushMirrorRemote registers the push mirror remote. +func AddPushMirrorRemote(m *models.PushMirror, addr string) error { + addRemoteAndConfig := func(addr, path string) error { + if _, err := git.NewCommand("remote", "add", "--mirror=push", m.RemoteName, addr).RunInDir(path); err != nil { + return err + } + if _, err := git.NewCommand("config", "--add", "remote."+m.RemoteName+".push", "+refs/heads/*:refs/heads/*").RunInDir(path); err != nil { + return err + } + if _, err := git.NewCommand("config", "--add", "remote."+m.RemoteName+".push", "+refs/tags/*:refs/tags/*").RunInDir(path); err != nil { + return err + } + return nil + } + + if err := addRemoteAndConfig(addr, m.Repo.RepoPath()); err != nil { + return err + } + + if m.Repo.HasWiki() { + wikiRemoteURL := repository.WikiRemoteURL(addr) + if len(wikiRemoteURL) > 0 { + if err := addRemoteAndConfig(wikiRemoteURL, m.Repo.WikiPath()); err != nil { + return err + } + } + } + + return nil +} + +// RemovePushMirrorRemote removes the push mirror remote. +func RemovePushMirrorRemote(m *models.PushMirror) error { + cmd := git.NewCommand("remote", "rm", m.RemoteName) + + if _, err := cmd.RunInDir(m.Repo.RepoPath()); err != nil { + return err + } + + if m.Repo.HasWiki() { + if _, err := cmd.RunInDir(m.Repo.WikiPath()); err != nil { + // The wiki remote may not exist + log.Warn("Wiki Remote[%d] could not be removed: %v", m.ID, err) + } + } + + return nil +} + +// SyncPushMirror starts the sync of the push mirror and schedules the next run. +func SyncPushMirror(ctx context.Context, mirrorID int64) bool { + log.Trace("SyncPushMirror [mirror: %d]", mirrorID) + defer func() { + err := recover() + if err == nil { + return + } + // There was a panic whilst syncPushMirror... + log.Error("PANIC whilst syncPushMirror[%d] Panic: %v\nStacktrace: %s", mirrorID, err, log.Stack(2)) + }() + + m, err := models.GetPushMirrorByID(mirrorID) + if err != nil { + log.Error("GetPushMirrorByID [%d]: %v", mirrorID, err) + return false + } + + m.LastError = "" + + log.Trace("SyncPushMirror [mirror: %d][repo: %-v]: Running Sync", m.ID, m.Repo) + err = runPushSync(ctx, m) + if err != nil { + log.Error("SyncPushMirror [mirror: %d][repo: %-v]: %v", m.ID, m.Repo, err) + m.LastError = stripExitStatus.ReplaceAllLiteralString(err.Error(), "") + } + + m.LastUpdateUnix = timeutil.TimeStampNow() + + if err := models.UpdatePushMirror(m); err != nil { + log.Error("UpdatePushMirror [%d]: %v", m.ID, err) + + return false + } + + log.Trace("SyncPushMirror [mirror: %d][repo: %-v]: Finished", m.ID, m.Repo) + + return err == nil +} + +func runPushSync(ctx context.Context, m *models.PushMirror) error { + timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second + + performPush := func(path string) error { + remoteAddr, err := git.GetRemoteAddress(path, m.RemoteName) + if err != nil { + log.Error("GetRemoteAddress(%s) Error %v", path, err) + return errors.New("Unexpected error") + } + + if setting.LFS.StartServer { + log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo) + + gitRepo, err := git.OpenRepository(path) + if err != nil { + log.Error("OpenRepository: %v", err) + return errors.New("Unexpected error") + } + defer gitRepo.Close() + + ep := lfs.DetermineEndpoint(remoteAddr.String(), "") + if err := pushAllLFSObjects(ctx, gitRepo, ep); err != nil { + return util.NewURLSanitizedError(err, remoteAddr, true) + } + } + + log.Trace("Pushing %s mirror[%d] remote %s", path, m.ID, m.RemoteName) + + if err := git.Push(path, git.PushOptions{ + Remote: m.RemoteName, + Force: true, + Mirror: true, + Timeout: timeout, + }); err != nil { + log.Error("Error pushing %s mirror[%d] remote %s: %v", path, m.ID, m.RemoteName, err) + + return util.NewURLSanitizedError(err, remoteAddr, true) + } + + return nil + } + + err := performPush(m.Repo.RepoPath()) + if err != nil { + return err + } + + if m.Repo.HasWiki() { + wikiPath := m.Repo.WikiPath() + _, err := git.GetRemoteAddress(wikiPath, m.RemoteName) + if err == nil { + err := performPush(wikiPath) + if err != nil { + return err + } + } else { + log.Trace("Skipping wiki: No remote configured") + } + } + + return nil +} + +func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, endpoint *url.URL) error { + client := lfs.NewClient(endpoint) + contentStore := lfs.NewContentStore() + + pointerChan := make(chan lfs.PointerBlob) + errChan := make(chan error, 1) + go lfs.SearchPointerBlobs(ctx, gitRepo, pointerChan, errChan) + + uploadObjects := func(pointers []lfs.Pointer) error { + err := client.Upload(ctx, pointers, func(p lfs.Pointer, objectError error) (io.ReadCloser, error) { + if objectError != nil { + return nil, objectError + } + + content, err := contentStore.Get(p) + if err != nil { + log.Error("Error reading LFS object %v: %v", p, err) + } + return content, err + }) + if err != nil { + select { + case <-ctx.Done(): + return nil + default: + } + } + return err + } + + var batch []lfs.Pointer + for pointerBlob := range pointerChan { + exists, err := contentStore.Exists(pointerBlob.Pointer) + if err != nil { + log.Error("Error checking if LFS object %v exists: %v", pointerBlob.Pointer, err) + return err + } + if !exists { + log.Trace("Skipping missing LFS object %v", pointerBlob.Pointer) + continue + } + + batch = append(batch, pointerBlob.Pointer) + if len(batch) >= client.BatchSize() { + if err := uploadObjects(batch); err != nil { + return err + } + batch = nil + } + } + if len(batch) > 0 { + if err := uploadObjects(batch); err != nil { + return err + } + } + + err, has := <-errChan + if has { + log.Error("Error enumerating LFS objects for repository: %v", err) + return err + } + + return nil +} diff --git a/templates/repo/header.tmpl b/templates/repo/header.tmpl index ebd0333e8..49a651e6c 100644 --- a/templates/repo/header.tmpl +++ b/templates/repo/header.tmpl @@ -36,7 +36,7 @@ {{end}} - {{if .IsMirror}}
{{$.i18n.Tr "repo.mirror_from"}} {{if .SanitizedOriginalURL}}{{.SanitizedOriginalURL}}{{else}}{{MirrorAddress $.Mirror}}{{end}}
{{end}} + {{if .IsMirror}}
{{$.i18n.Tr "repo.mirror_from"}} {{if .SanitizedOriginalURL}}{{.SanitizedOriginalURL}}{{else}}{{(MirrorRemoteAddress $.Mirror).Address}}{{end}}
{{end}} {{if .IsFork}}
{{$.i18n.Tr "repo.forked_from"}} {{SubStr .BaseRepo.RelLink 1 -1}}
{{end}} {{if .IsGenerated}}
{{$.i18n.Tr "repo.generated_from"}} {{SubStr .TemplateRepo.RelLink 1 -1}}
{{end}} diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index ece439f3d..163a1a66d 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -70,81 +70,165 @@ - {{if .Repository.IsMirror}} + {{if or .Repository.IsMirror (not .DisabledMirrors)}}

{{.i18n.Tr "repo.settings.mirror_settings"}}

-
- {{.CsrfTokenHtml}} - -
- -
- - -
-
-
- - -
-
- - -

{{.i18n.Tr "repo.mirror_address_desc"}}

-
-
- - {{.i18n.Tr "repo.need_auth"}} - -
-
- - -
- -
- - -
-

{{.i18n.Tr "repo.mirror_password_help"}}

-
-
- - {{if .LFSStartServer}} -
- -
- - -
-
-
- - -

{{.i18n.Tr "repo.mirror_lfs_endpoint_desc" "https://github.com/git-lfs/git-lfs/blob/main/docs/api/server-discovery.md#server-discovery" | Str2html}}

-
+ {{$.i18n.Tr "repo.settings.mirror_settings.docs" | Safe}} + + {{if or .Repository.IsMirror .Repository.PushMirrors}} + + + + + + + + {{end}} - -
- -
- - -
- - - {{.CsrfTokenHtml}} - -
- - {{.Mirror.UpdatedUnix.AsTime}} -
-
- -
- + {{if .Repository.IsMirror}} + + + + + + + + + + + + + {{end}} + + {{range .Repository.PushMirrors}} + + {{$address := MirrorRemoteAddress .}} + + + + + + {{else}} + + + + {{end}} + + + + +
{{$.i18n.Tr "repo.settings.mirror_settings.mirrored_repository"}}{{$.i18n.Tr "repo.settings.mirror_settings.direction"}}{{$.i18n.Tr "repo.settings.mirror_settings.last_update"}}
{{(MirrorRemoteAddress .Mirror).Address}}{{$.i18n.Tr "repo.settings.mirror_settings.direction.pull"}}{{.Mirror.UpdatedUnix.AsTime}} +
+ {{.CsrfTokenHtml}} + + +
+
+
+ {{.CsrfTokenHtml}} + +
+ +
+ + +
+
+
+ + +
+ {{$address := MirrorRemoteAddress .Mirror}} +
+ + +

{{.i18n.Tr "repo.mirror_address_desc"}}

+
+
+ + {{.i18n.Tr "repo.need_auth"}} + +
+
+ + +
+ +
+ + +
+

{{.i18n.Tr "repo.mirror_password_help"}}

+
+
+ + {{if .LFSStartServer}} +
+ +
+ + +
+
+
+ + +

{{.i18n.Tr "repo.mirror_lfs_endpoint_desc" "https://github.com/git-lfs/git-lfs/blob/main/docs/api/server-discovery.md#server-discovery" | Str2html}}

+
+ {{end}} +
+ +
+
+
{{$address.Address}}{{$.i18n.Tr "repo.settings.mirror_settings.direction.push"}}{{if .LastUpdateUnix}}{{.LastUpdateUnix.AsTime}}{{else}}{{$.i18n.Tr "never"}}{{end}} {{if .LastError}}
{{$.i18n.Tr "error"}}
{{end}}		 +
+ {{$.CsrfTokenHtml}} + + + +
+
+ {{$.CsrfTokenHtml}} + + + +
+
{{$.i18n.Tr "repo.settings.mirror_settings.push_mirror.none"}}
+
+ {{.CsrfTokenHtml}} + +
+ + +

{{.i18n.Tr "repo.mirror_address_desc"}}

+
+
+ + {{.i18n.Tr "repo.need_auth"}} + +
+
+ + +
+ +
+ + +
+
+
+
+ + +
+
+ +
+
+
{{end}}