From bdfd751af88c5bdb70dbdfb4f7f607f6fbf77896 Mon Sep 17 00:00:00 2001
From: Lunny Xiao <xiaolunwen@gmail.com>
Date: Fri, 15 Oct 2021 13:47:15 +0800
Subject: [PATCH] Multiple tokens support for migrating from github (#17134)
* multiple tokens support for migrating from github
* improve code and token description
* Fix bug
* Add comment for get client
---
modules/migrations/github.go | 212 ++++++++++++++++++++++---------------
options/locale/locale_en-US.ini | 1 +
templates/repo/migrate/github.tmpl | 5 +-
3 files changed, 134 insertions(+), 84 deletions(-)
diff --git a/modules/migrations/github.go b/modules/migrations/github.go
index 97e1b672ac..1a228c84a4 100644
--- a/modules/migrations/github.go
+++ b/modules/migrations/github.go
@@ -68,14 +68,15 @@ func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType {
// from github via APIv3
type GithubDownloaderV3 struct {
base.NullDownloader
- ctx context.Context
- client *github.Client
- repoOwner string
- repoName string
- userName string
- password string
- rate *github.Rate
- maxPerPage int
+ ctx context.Context
+ clients []*github.Client
+ repoOwner string
+ repoName string
+ userName string
+ password string
+ rates []*github.Rate
+ curClientIdx int
+ maxPerPage int
}
// NewGithubDownloaderV3 creates a github Downloader via github v3 API
@@ -89,25 +90,49 @@ func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, tok
maxPerPage: 100,
}
- client := &http.Client{
- Transport: &http.Transport{
- Proxy: func(req *http.Request) (*url.URL, error) {
- req.SetBasicAuth(userName, password)
- return proxy.Proxy()(req)
- },
- },
- }
if token != "" {
- ts := oauth2.StaticTokenSource(
- &oauth2.Token{AccessToken: token},
- )
- client = oauth2.NewClient(downloader.ctx, ts)
+ tokens := strings.Split(token, ",")
+ for _, token := range tokens {
+ token = strings.TrimSpace(token)
+ ts := oauth2.StaticTokenSource(
+ &oauth2.Token{AccessToken: token},
+ )
+ var client = &http.Client{
+ Transport: &oauth2.Transport{
+ Base: &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
+ Proxy: func(req *http.Request) (*url.URL, error) {
+ return proxy.Proxy()(req)
+ },
+ },
+ Source: oauth2.ReuseTokenSource(nil, ts),
+ },
+ }
+
+ downloader.addClient(client, baseURL)
+ }
+ } else {
+ var client = &http.Client{
+ Transport: &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
+ Proxy: func(req *http.Request) (*url.URL, error) {
+ req.SetBasicAuth(userName, password)
+ return proxy.Proxy()(req)
+ },
+ },
+ }
+ downloader.addClient(client, baseURL)
}
- downloader.client = github.NewClient(client)
+ return &downloader
+}
+
+func (g *GithubDownloaderV3) addClient(client *http.Client, baseURL string) {
+ githubClient := github.NewClient(client)
if baseURL != "https://github.com" {
- downloader.client, _ = github.NewEnterpriseClient(baseURL, baseURL, client)
+ githubClient, _ = github.NewEnterpriseClient(baseURL, baseURL, client)
}
- return &downloader
+ g.clients = append(g.clients, githubClient)
+ g.rates = append(g.rates, nil)
}
// SetContext set context
@@ -115,9 +140,19 @@ func (g *GithubDownloaderV3) SetContext(ctx context.Context) {
g.ctx = ctx
}
-func (g *GithubDownloaderV3) sleep() {
- for g.rate != nil && g.rate.Remaining <= GithubLimitRateRemaining {
- timer := time.NewTimer(time.Until(g.rate.Reset.Time))
+func (g *GithubDownloaderV3) waitAndPickClient() {
+ var recentIdx int
+ var maxRemaining int
+ for i := 0; i < len(g.clients); i++ {
+ if g.rates[i] != nil && g.rates[i].Remaining > maxRemaining {
+ maxRemaining = g.rates[i].Remaining
+ recentIdx = i
+ }
+ }
+ g.curClientIdx = recentIdx // if no max remain, it will always pick the first client.
+
+ for g.rates[g.curClientIdx] != nil && g.rates[g.curClientIdx].Remaining <= GithubLimitRateRemaining {
+ timer := time.NewTimer(time.Until(g.rates[g.curClientIdx].Reset.Time))
select {
case <-g.ctx.Done():
util.StopTimer(timer)
@@ -127,35 +162,43 @@ func (g *GithubDownloaderV3) sleep() {
err := g.RefreshRate()
if err != nil {
- log.Error("g.client.RateLimits: %s", err)
+ log.Error("g.getClient().RateLimits: %s", err)
}
}
}
// RefreshRate update the current rate (doesn't count in rate limit)
func (g *GithubDownloaderV3) RefreshRate() error {
- rates, _, err := g.client.RateLimits(g.ctx)
+ rates, _, err := g.getClient().RateLimits(g.ctx)
if err != nil {
// if rate limit is not enabled, ignore it
if strings.Contains(err.Error(), "404") {
- g.rate = nil
+ g.setRate(nil)
return nil
}
return err
}
- g.rate = rates.GetCore()
+ g.setRate(rates.GetCore())
return nil
}
+func (g *GithubDownloaderV3) getClient() *github.Client {
+ return g.clients[g.curClientIdx]
+}
+
+func (g *GithubDownloaderV3) setRate(rate *github.Rate) {
+ g.rates[g.curClientIdx] = rate
+}
+
// GetRepoInfo returns a repository information
func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
- g.sleep()
- gr, resp, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName)
+ g.waitAndPickClient()
+ gr, resp, err := g.getClient().Repositories.Get(g.ctx, g.repoOwner, g.repoName)
if err != nil {
return nil, err
}
- g.rate = &resp.Rate
+ g.setRate(&resp.Rate)
// convert github repo to stand Repo
return &base.Repository{
@@ -171,12 +214,12 @@ func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
// GetTopics return github topics
func (g *GithubDownloaderV3) GetTopics() ([]string, error) {
- g.sleep()
- r, resp, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName)
+ g.waitAndPickClient()
+ r, resp, err := g.getClient().Repositories.Get(g.ctx, g.repoOwner, g.repoName)
if err != nil {
return nil, err
}
- g.rate = &resp.Rate
+ g.setRate(&resp.Rate)
return r.Topics, nil
}
@@ -185,8 +228,8 @@ func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {
var perPage = g.maxPerPage
var milestones = make([]*base.Milestone, 0, perPage)
for i := 1; ; i++ {
- g.sleep()
- ms, resp, err := g.client.Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName,
+ g.waitAndPickClient()
+ ms, resp, err := g.getClient().Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName,
&github.MilestoneListOptions{
State: "all",
ListOptions: github.ListOptions{
@@ -196,7 +239,7 @@ func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {
if err != nil {
return nil, err
}
- g.rate = &resp.Rate
+ g.setRate(&resp.Rate)
for _, m := range ms {
var state = "open"
@@ -233,8 +276,8 @@ func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
var perPage = g.maxPerPage
var labels = make([]*base.Label, 0, perPage)
for i := 1; ; i++ {
- g.sleep()
- ls, resp, err := g.client.Issues.ListLabels(g.ctx, g.repoOwner, g.repoName,
+ g.waitAndPickClient()
+ ls, resp, err := g.getClient().Issues.ListLabels(g.ctx, g.repoOwner, g.repoName,
&github.ListOptions{
Page: i,
PerPage: perPage,
@@ -242,7 +285,7 @@ func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
if err != nil {
return nil, err
}
- g.rate = &resp.Rate
+ g.setRate(&resp.Rate)
for _, label := range ls {
labels = append(labels, convertGithubLabel(label))
@@ -290,17 +333,17 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
Created: asset.CreatedAt.Time,
Updated: asset.UpdatedAt.Time,
DownloadFunc: func() (io.ReadCloser, error) {
- g.sleep()
- asset, redirectURL, err := g.client.Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, assetID, nil)
+ g.waitAndPickClient()
+ asset, redirectURL, err := g.getClient().Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, assetID, nil)
if err != nil {
return nil, err
}
if err := g.RefreshRate(); err != nil {
- log.Error("g.client.RateLimits: %s", err)
+ log.Error("g.getClient().RateLimits: %s", err)
}
if asset == nil {
if redirectURL != "" {
- g.sleep()
+ g.waitAndPickClient()
req, err := http.NewRequestWithContext(g.ctx, "GET", redirectURL, nil)
if err != nil {
return nil, err
@@ -308,7 +351,7 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
resp, err := httpClient.Do(req)
err1 := g.RefreshRate()
if err1 != nil {
- log.Error("g.client.RateLimits: %s", err1)
+ log.Error("g.getClient().RateLimits: %s", err1)
}
if err != nil {
return nil, err
@@ -329,8 +372,8 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
var perPage = g.maxPerPage
var releases = make([]*base.Release, 0, perPage)
for i := 1; ; i++ {
- g.sleep()
- ls, resp, err := g.client.Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName,
+ g.waitAndPickClient()
+ ls, resp, err := g.getClient().Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName,
&github.ListOptions{
Page: i,
PerPage: perPage,
@@ -338,7 +381,7 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
if err != nil {
return nil, err
}
- g.rate = &resp.Rate
+ g.setRate(&resp.Rate)
for _, release := range ls {
releases = append(releases, g.convertGithubRelease(release))
@@ -366,13 +409,13 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool,
}
var allIssues = make([]*base.Issue, 0, perPage)
- g.sleep()
- issues, resp, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
+ g.waitAndPickClient()
+ issues, resp, err := g.getClient().Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
if err != nil {
return nil, false, fmt.Errorf("error while listing repos: %v", err)
}
log.Trace("Request get issues %d/%d, but in fact get %d", perPage, page, len(issues))
- g.rate = &resp.Rate
+ g.setRate(&resp.Rate)
for _, issue := range issues {
if issue.IsPullRequest() {
continue
@@ -386,15 +429,15 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool,
// get reactions
var reactions []*base.Reaction
for i := 1; ; i++ {
- g.sleep()
- res, resp, err := g.client.Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListOptions{
+ g.waitAndPickClient()
+ res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListOptions{
Page: i,
PerPage: perPage,
})
if err != nil {
return nil, false, err
}
- g.rate = &resp.Rate
+ g.setRate(&resp.Rate)
if len(res) == 0 {
break
}
@@ -464,25 +507,25 @@ func (g *GithubDownloaderV3) getComments(issueContext base.IssueContext) ([]*bas
},
}
for {
- g.sleep()
- comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueContext.ForeignID()), opt)
+ g.waitAndPickClient()
+ comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueContext.ForeignID()), opt)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err)
}
- g.rate = &resp.Rate
+ g.setRate(&resp.Rate)
for _, comment := range comments {
// get reactions
var reactions []*base.Reaction
for i := 1; ; i++ {
- g.sleep()
- res, resp, err := g.client.Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{
+ g.waitAndPickClient()
+ res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{
Page: i,
PerPage: g.maxPerPage,
})
if err != nil {
return nil, err
}
- g.rate = &resp.Rate
+ g.setRate(&resp.Rate)
if len(res) == 0 {
break
}
@@ -533,28 +576,28 @@ func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment,
},
}
- g.sleep()
- comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, 0, opt)
+ g.waitAndPickClient()
+ comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, 0, opt)
if err != nil {
return nil, false, fmt.Errorf("error while listing repos: %v", err)
}
var isEnd = resp.NextPage == 0
log.Trace("Request get comments %d/%d, but in fact get %d, next page is %d", perPage, page, len(comments), resp.NextPage)
- g.rate = &resp.Rate
+ g.setRate(&resp.Rate)
for _, comment := range comments {
// get reactions
var reactions []*base.Reaction
for i := 1; ; i++ {
- g.sleep()
- res, resp, err := g.client.Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{
+ g.waitAndPickClient()
+ res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{
Page: i,
PerPage: g.maxPerPage,
})
if err != nil {
return nil, false, err
}
- g.rate = &resp.Rate
+ g.setRate(&resp.Rate)
if len(res) == 0 {
break
}
@@ -598,13 +641,13 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
},
}
var allPRs = make([]*base.PullRequest, 0, perPage)
- g.sleep()
- prs, resp, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
+ g.waitAndPickClient()
+ prs, resp, err := g.getClient().PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
if err != nil {
return nil, false, fmt.Errorf("error while listing repos: %v", err)
}
log.Trace("Request get pull requests %d/%d, but in fact get %d", perPage, page, len(prs))
- g.rate = &resp.Rate
+ g.setRate(&resp.Rate)
for _, pr := range prs {
var labels = make([]*base.Label, 0, len(pr.Labels))
for _, l := range pr.Labels {
@@ -614,15 +657,15 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
// get reactions
var reactions []*base.Reaction
for i := 1; ; i++ {
- g.sleep()
- res, resp, err := g.client.Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, pr.GetNumber(), &github.ListOptions{
+ g.waitAndPickClient()
+ res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, pr.GetNumber(), &github.ListOptions{
Page: i,
PerPage: perPage,
})
if err != nil {
return nil, false, err
}
- g.rate = &resp.Rate
+ g.setRate(&resp.Rate)
if len(res) == 0 {
break
}
@@ -635,6 +678,9 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
}
}
+ // download patch and saved as tmp file
+ g.waitAndPickClient()
+
allPRs = append(allPRs, &base.PullRequest{
Title: pr.GetTitle(),
Number: int64(pr.GetNumber()),
@@ -692,15 +738,15 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques
// get reactions
var reactions []*base.Reaction
for i := 1; ; i++ {
- g.sleep()
- res, resp, err := g.client.Reactions.ListPullRequestCommentReactions(g.ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListOptions{
+ g.waitAndPickClient()
+ res, resp, err := g.getClient().Reactions.ListPullRequestCommentReactions(g.ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListOptions{
Page: i,
PerPage: g.maxPerPage,
})
if err != nil {
return nil, err
}
- g.rate = &resp.Rate
+ g.setRate(&resp.Rate)
if len(res) == 0 {
break
}
@@ -737,12 +783,12 @@ func (g *GithubDownloaderV3) GetReviews(context base.IssueContext) ([]*base.Revi
PerPage: g.maxPerPage,
}
for {
- g.sleep()
- reviews, resp, err := g.client.PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), opt)
+ g.waitAndPickClient()
+ reviews, resp, err := g.getClient().PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), opt)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err)
}
- g.rate = &resp.Rate
+ g.setRate(&resp.Rate)
for _, review := range reviews {
r := convertGithubReview(review)
r.IssueIndex = context.LocalID()
@@ -751,12 +797,12 @@ func (g *GithubDownloaderV3) GetReviews(context base.IssueContext) ([]*base.Revi
PerPage: g.maxPerPage,
}
for {
- g.sleep()
- reviewComments, resp, err := g.client.PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), review.GetID(), opt2)
+ g.waitAndPickClient()
+ reviewComments, resp, err := g.getClient().PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), review.GetID(), opt2)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err)
}
- g.rate = &resp.Rate
+ g.setRate(&resp.Rate)
cs, err := g.convertGithubReviewComments(reviewComments)
if err != nil {
diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini
index 1324303fc0..67ca8ff558 100644
--- a/options/locale/locale_en-US.ini
+++ b/options/locale/locale_en-US.ini
@@ -887,6 +887,7 @@ migrate_items_releases = Releases
migrate_repo = Migrate Repository
migrate.clone_address = Migrate / Clone From URL
migrate.clone_address_desc = The HTTP(S) or Git 'clone' URL of an existing repository
+migrate.github_token_desc = You can put one or more tokens with comma separated here to make migrating faster because of Github API rate limit. WARN: Abusing this feature may violate the service provider's policy and lead to account blocking.
migrate.clone_local_path = or a local server path
migrate.permission_denied = You are not allowed to import local repositories.
migrate.permission_denied_blocked = You are not allowed to import from blocked hosts.
diff --git a/templates/repo/migrate/github.tmpl b/templates/repo/migrate/github.tmpl
index 156f8896fc..9bd7228a43 100644
--- a/templates/repo/migrate/github.tmpl
+++ b/templates/repo/migrate/github.tmpl
@@ -14,7 +14,7 @@
<label for="clone_addr">{{.i18n.Tr "repo.migrate.clone_address"}}</label>
<input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required>
<span class="help">
- {{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}}
+ {{.i18n.Tr "repo.migrate.clone_address_desc"}}
</span>
</div>
@@ -22,6 +22,9 @@
<label for="auth_token">{{.i18n.Tr "access_token"}}</label>
<input id="auth_token" name="auth_token" value="{{.auth_token}}" {{if not .auth_token}}data-need-clear="true"{{end}}>
<a target="_blank" href="https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token">{{svg "octicon-question"}}</a>
+ <span class="help">
+ {{.i18n.Tr "repo.migrate.github_token_desc"}}
+ </span>
</div>
{{template "repo/migrate/options" .}}