improve github downloader on migrations (#7049)

* improve github downloader on migrations

* fix tests

* fix  uppercase function parameters

modules/migrations/base/downloader.go

@@ -11,9 +11,9 @@ type Downloader interface {
 	GetMilestones() ([]*Milestone, error)
 	GetReleases() ([]*Release, error)
 	GetLabels() ([]*Label, error)
-	GetIssues(start, limit int) ([]*Issue, error)
+	GetIssues(page, perPage int) ([]*Issue, bool, error)
 	GetComments(issueNumber int64) ([]*Comment, error)
-	GetPullRequests(start, limit int) ([]*PullRequest, error)
+	GetPullRequests(page, perPage int) ([]*PullRequest, error)
 }
 
 // DownloaderFactory defines an interface to match a downloader implementation and create a downloader

modules/migrations/git.go

@@ -53,9 +53,9 @@ func (g *PlainGitDownloader) GetReleases() ([]*base.Release, error) {
 	return nil, ErrNotSupported
 }
 
-// GetIssues returns issues according start and limit
-func (g *PlainGitDownloader) GetIssues(start, limit int) ([]*base.Issue, error) {
-	return nil, ErrNotSupported
+// GetIssues returns issues according page and perPage
+func (g *PlainGitDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
+	return nil, false, ErrNotSupported
 }
 
 // GetComments returns comments according issueNumber
@@ -63,7 +63,7 @@ func (g *PlainGitDownloader) GetComments(issueNumber int64) ([]*base.Comment, er
 	return nil, ErrNotSupported
 }
 
-// GetPullRequests returns pull requests according start and limit
+// GetPullRequests returns pull requests according page and perPage
 func (g *PlainGitDownloader) GetPullRequests(start, limit int) ([]*base.PullRequest, error) {
 	return nil, ErrNotSupported
 }

modules/migrations/github.go

@@ -272,21 +272,22 @@ func convertGithubReactions(reactions *github.Reactions) *base.Reactions {
 }
 
 // GetIssues returns issues according start and limit
-func (g *GithubDownloaderV3) GetIssues(start, limit int) ([]*base.Issue, error) {
-	var perPage = 100
+func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
 	opt := &github.IssueListByRepoOptions{
 		Sort:      "created",
 		Direction: "asc",
 		State:     "all",
 		ListOptions: github.ListOptions{
 			PerPage: perPage,
+			Page:    page,
 		},
 	}
-	var allIssues = make([]*base.Issue, 0, limit)
-	for {
-		issues, resp, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
+
+	var allIssues = make([]*base.Issue, 0, perPage)
+
+	issues, _, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
 	if err != nil {
-			return nil, fmt.Errorf("error while listing repos: %v", err)
+		return nil, false, fmt.Errorf("error while listing repos: %v", err)
 	}
 	for _, issue := range issues {
 		if issue.IsPullRequest() {
@@ -327,16 +328,9 @@ func (g *GithubDownloaderV3) GetIssues(start, limit int) ([]*base.Issue, error)
 			Closed:      issue.ClosedAt,
 			IsLocked:    *issue.Locked,
 		})
-			if len(allIssues) >= limit {
-				return allIssues, nil
-			}
 	}
-		if resp.NextPage == 0 {
-			break
-		}
-		opt.Page = resp.NextPage
-	}
-	return allIssues, nil
+
+	return allIssues, len(issues) < perPage, nil
 }
 
 // GetComments returns comments according issueNumber
@@ -379,19 +373,20 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er
 	return allComments, nil
 }
 
-// GetPullRequests returns pull requests according start and limit
-func (g *GithubDownloaderV3) GetPullRequests(start, limit int) ([]*base.PullRequest, error) {
+// GetPullRequests returns pull requests according page and perPage
+func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullRequest, error) {
 	opt := &github.PullRequestListOptions{
 		Sort:      "created",
 		Direction: "asc",
 		State:     "all",
 		ListOptions: github.ListOptions{
-			PerPage: 100,
+			PerPage: perPage,
+			Page:    page,
 		},
 	}
-	var allPRs = make([]*base.PullRequest, 0, 100)
-	for {
-		prs, resp, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
+	var allPRs = make([]*base.PullRequest, 0, perPage)
+
+	prs, _, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
 	if err != nil {
 		return nil, fmt.Errorf("error while listing repos: %v", err)
 	}
@@ -462,14 +457,7 @@ func (g *GithubDownloaderV3) GetPullRequests(start, limit int) ([]*base.PullRequ
 			},
 			PatchURL: *pr.PatchURL,
 		})
-			if len(allPRs) >= limit {
-				return allPRs, nil
-			}
-		}
-		if resp.NextPage == 0 {
-			break
-		}
-		opt.Page = resp.NextPage
 	}
+
 	return allPRs, nil
 }

modules/migrations/github_test.go

@@ -166,9 +166,11 @@ func TestGitHubDownloadRepo(t *testing.T) {
 	}, releases[len(releases)-1:])
 
 	// downloader.GetIssues()
-	issues, err := downloader.GetIssues(0, 3)
+	issues, isEnd, err := downloader.GetIssues(1, 8)
 	assert.NoError(t, err)
 	assert.EqualValues(t, 3, len(issues))
+	assert.False(t, isEnd)
+
 	var (
 		closed1 = time.Date(2018, 10, 23, 02, 57, 43, 0, time.UTC)
 	)
@@ -319,7 +321,7 @@ something like in the latest 15days could be enough don't you think ?
 	}, comments[:3])
 
 	// downloader.GetPullRequests()
-	prs, err := downloader.GetPullRequests(0, 3)
+	prs, err := downloader.GetPullRequests(1, 3)
 	assert.NoError(t, err)
 	assert.EqualValues(t, 3, len(prs))
 

modules/migrations/migrate.go

@@ -128,8 +128,8 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 
 	if opts.Issues {
 		log.Trace("migrating issues and comments")
-		for {
-			issues, err := downloader.GetIssues(0, 100)
+		for i := 1; ; i++ {
+			issues, isEnd, err := downloader.GetIssues(i, 100)
 			if err != nil {
 				return err
 			}
@@ -160,7 +160,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 				}
 			}
 
-			if len(issues) < 100 {
+			if isEnd {
 				break
 			}
 		}
@@ -168,8 +168,8 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 
 	if opts.PullRequests {
 		log.Trace("migrating pull requests and comments")
-		for {
-			prs, err := downloader.GetPullRequests(0, 100)
+		for i := 1; ; i++ {
+			prs, err := downloader.GetPullRequests(i, 100)
 			if err != nil {
 				return err
 			}