mirror of
https://codeberg.org/forgejo/forgejo.git
synced 2026-02-03 20:51:07 -05:00
01fbb1499f
For the previous code with the Page attribute present in ListCursorOptions for page 1, github would not return an "After" cursor, such that the request for page 2 would request what effectively is the content of page 1 a second time. This would lead to an attempt to insert the same issues twice. Note that this is not the only reason why this can happen with the current code base. We fix this particular issue by not using the Page attribute so github does return an "After" cursor. Fixes #10794 Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/10798 Reviewed-by: Gusted <gusted@noreply.codeberg.org> Co-authored-by: Nils Goroll <nils.goroll@uplex.de> Co-committed-by: Nils Goroll <nils.goroll@uplex.de>
958 lines
27 KiB
Go
958 lines
27 KiB
Go
// Copyright 2019 The Gitea Authors. All rights reserved.
|
|
// Copyright 2018 Jonas Franz. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package migrations
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"forgejo.org/modules/git"
|
|
"forgejo.org/modules/log"
|
|
base "forgejo.org/modules/migration"
|
|
"forgejo.org/modules/proxy"
|
|
"forgejo.org/modules/structs"
|
|
|
|
"github.com/google/go-github/v81/github"
|
|
"golang.org/x/oauth2"
|
|
)
|
|
|
|
var (
|
|
_ base.Downloader = &GithubDownloaderV3{}
|
|
_ base.DownloaderFactory = &GithubDownloaderV3Factory{}
|
|
// GithubLimitRateRemaining limit to wait for new rate to apply
|
|
GithubLimitRateRemaining = 0
|
|
)
|
|
|
|
func init() {
|
|
RegisterDownloaderFactory(&GithubDownloaderV3Factory{})
|
|
}
|
|
|
|
// GithubDownloaderV3Factory defines a github downloader v3 factory
|
|
type GithubDownloaderV3Factory struct{}
|
|
|
|
// New returns a Downloader related to this factory according MigrateOptions
|
|
func (f *GithubDownloaderV3Factory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
|
|
u, err := url.Parse(opts.CloneAddr)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// some users are using the github redirect url for migration
|
|
if u.Host == "www.github.com" {
|
|
u.Host = "github.com"
|
|
}
|
|
|
|
baseURL := u.Scheme + "://" + u.Host
|
|
fields := strings.Split(u.Path, "/")
|
|
oldOwner := fields[1]
|
|
oldName := strings.TrimSuffix(fields[2], ".git")
|
|
|
|
log.Trace("Create github downloader BaseURL: %s %s/%s", baseURL, oldOwner, oldName)
|
|
|
|
return NewGithubDownloaderV3(ctx, baseURL, opts.PullRequests, opts.Issues, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName), nil
|
|
}
|
|
|
|
// GitServiceType returns the type of git service
|
|
func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType {
|
|
return structs.GithubService
|
|
}
|
|
|
|
// GithubDownloaderV3 implements a Downloader interface to get repository information
|
|
// from github via APIv3
|
|
type GithubDownloaderV3 struct {
|
|
base.NullDownloader
|
|
ctx context.Context
|
|
clients []*github.Client
|
|
baseURL string
|
|
repoOwner string
|
|
repoName string
|
|
userName string
|
|
password string
|
|
getPullRequests bool
|
|
getIssues bool
|
|
rates []*github.Rate
|
|
curClientIdx int
|
|
maxPerPage int
|
|
SkipReactions bool
|
|
SkipReviews bool
|
|
githubPagingInfo cursorPagingInfo
|
|
}
|
|
|
|
type cursorPagingInfo struct {
|
|
After string
|
|
Before string
|
|
Cursor string
|
|
}
|
|
|
|
// NewGithubDownloaderV3 creates a github Downloader via github v3 API
|
|
func NewGithubDownloaderV3(ctx context.Context, baseURL string, getPullRequests, getIssues bool, userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 {
|
|
downloader := GithubDownloaderV3{
|
|
userName: userName,
|
|
baseURL: baseURL,
|
|
password: password,
|
|
repoOwner: repoOwner,
|
|
repoName: repoName,
|
|
maxPerPage: 100,
|
|
getPullRequests: getPullRequests,
|
|
getIssues: getIssues,
|
|
}
|
|
|
|
downloader.SetContext(ctx)
|
|
|
|
if token != "" {
|
|
tokens := strings.Split(token, ",")
|
|
for _, token := range tokens {
|
|
token = strings.TrimSpace(token)
|
|
ts := oauth2.StaticTokenSource(
|
|
&oauth2.Token{AccessToken: token},
|
|
)
|
|
client := &http.Client{
|
|
Transport: &oauth2.Transport{
|
|
Base: NewMigrationHTTPTransport(),
|
|
Source: oauth2.ReuseTokenSource(nil, ts),
|
|
},
|
|
}
|
|
|
|
downloader.addClient(client, baseURL)
|
|
}
|
|
} else {
|
|
transport := NewMigrationHTTPTransport()
|
|
transport.Proxy = func(req *http.Request) (*url.URL, error) {
|
|
req.SetBasicAuth(userName, password)
|
|
return proxy.Proxy()(req)
|
|
}
|
|
client := &http.Client{
|
|
Transport: transport,
|
|
}
|
|
downloader.addClient(client, baseURL)
|
|
}
|
|
return &downloader
|
|
}
|
|
|
|
// String implements Stringer
|
|
func (g *GithubDownloaderV3) String() string {
|
|
return fmt.Sprintf("migration from github server %s %s/%s", g.baseURL, g.repoOwner, g.repoName)
|
|
}
|
|
|
|
func (g *GithubDownloaderV3) LogString() string {
|
|
if g == nil {
|
|
return "<GithubDownloaderV3 nil>"
|
|
}
|
|
return fmt.Sprintf("<GithubDownloaderV3 %s %s/%s>", g.baseURL, g.repoOwner, g.repoName)
|
|
}
|
|
|
|
func (g *GithubDownloaderV3) addClient(client *http.Client, baseURL string) {
|
|
githubClient := github.NewClient(client)
|
|
if baseURL != "https://github.com" {
|
|
githubClient, _ = githubClient.WithEnterpriseURLs(baseURL, baseURL)
|
|
}
|
|
g.clients = append(g.clients, githubClient)
|
|
g.rates = append(g.rates, nil)
|
|
}
|
|
|
|
// SetContext set context
|
|
func (g *GithubDownloaderV3) SetContext(ctx context.Context) {
|
|
ctx = context.WithValue(ctx, github.SleepUntilPrimaryRateLimitResetWhenRateLimited, true)
|
|
g.ctx = ctx
|
|
}
|
|
|
|
func (g *GithubDownloaderV3) waitAndPickClient() {
|
|
var recentIdx int
|
|
var maxRemaining int
|
|
for i := 0; i < len(g.clients); i++ {
|
|
if g.rates[i] != nil && g.rates[i].Remaining > maxRemaining {
|
|
maxRemaining = g.rates[i].Remaining
|
|
recentIdx = i
|
|
}
|
|
}
|
|
g.curClientIdx = recentIdx // if no max remain, it will always pick the first client.
|
|
|
|
for g.rates[g.curClientIdx] != nil && g.rates[g.curClientIdx].Remaining <= GithubLimitRateRemaining {
|
|
timer := time.NewTimer(time.Until(g.rates[g.curClientIdx].Reset.Time))
|
|
select {
|
|
case <-g.ctx.Done():
|
|
timer.Stop()
|
|
return
|
|
case <-timer.C:
|
|
}
|
|
|
|
err := g.RefreshRate()
|
|
if err != nil {
|
|
log.Error("g.getClient().RateLimit.Get: %s", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// RefreshRate update the current rate (doesn't count in rate limit)
|
|
func (g *GithubDownloaderV3) RefreshRate() error {
|
|
rates, _, err := g.getClient().RateLimit.Get(g.ctx)
|
|
if err != nil {
|
|
// if rate limit is not enabled, ignore it
|
|
if strings.Contains(err.Error(), "404") {
|
|
g.setRate(nil)
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
|
|
g.setRate(rates.GetCore())
|
|
return nil
|
|
}
|
|
|
|
func (g *GithubDownloaderV3) getClient() *github.Client {
|
|
return g.clients[g.curClientIdx]
|
|
}
|
|
|
|
func (g *GithubDownloaderV3) setRate(rate *github.Rate) {
|
|
g.rates[g.curClientIdx] = rate
|
|
}
|
|
|
|
// GetRepoInfo returns a repository information
|
|
func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
|
|
g.waitAndPickClient()
|
|
gr, resp, err := g.getClient().Repositories.Get(g.ctx, g.repoOwner, g.repoName)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
g.setRate(&resp.Rate)
|
|
|
|
// convert github repo to stand Repo
|
|
return &base.Repository{
|
|
Owner: g.repoOwner,
|
|
Name: gr.GetName(),
|
|
IsPrivate: gr.GetPrivate(),
|
|
Description: gr.GetDescription(),
|
|
OriginalURL: gr.GetHTMLURL(),
|
|
CloneURL: gr.GetCloneURL(),
|
|
DefaultBranch: gr.GetDefaultBranch(),
|
|
Website: gr.GetHomepage(),
|
|
}, nil
|
|
}
|
|
|
|
// GetTopics return github topics
|
|
func (g *GithubDownloaderV3) GetTopics() ([]string, error) {
|
|
g.waitAndPickClient()
|
|
r, resp, err := g.getClient().Repositories.Get(g.ctx, g.repoOwner, g.repoName)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
g.setRate(&resp.Rate)
|
|
return r.Topics, nil
|
|
}
|
|
|
|
// GetMilestones returns milestones
|
|
func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {
|
|
perPage := g.maxPerPage
|
|
milestones := make([]*base.Milestone, 0, perPage)
|
|
for i := 1; ; i++ {
|
|
g.waitAndPickClient()
|
|
ms, resp, err := g.getClient().Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName,
|
|
&github.MilestoneListOptions{
|
|
State: "all",
|
|
ListOptions: github.ListOptions{
|
|
Page: i,
|
|
PerPage: perPage,
|
|
},
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
g.setRate(&resp.Rate)
|
|
|
|
for _, m := range ms {
|
|
state := "open"
|
|
if m.State != nil {
|
|
state = *m.State
|
|
}
|
|
milestones = append(milestones, &base.Milestone{
|
|
Title: m.GetTitle(),
|
|
Description: m.GetDescription(),
|
|
Deadline: m.DueOn.GetTime(),
|
|
State: state,
|
|
Created: m.GetCreatedAt().Time,
|
|
Updated: m.UpdatedAt.GetTime(),
|
|
Closed: m.ClosedAt.GetTime(),
|
|
})
|
|
}
|
|
if len(ms) < perPage {
|
|
break
|
|
}
|
|
}
|
|
return milestones, nil
|
|
}
|
|
|
|
func convertGithubLabel(label *github.Label) *base.Label {
|
|
return &base.Label{
|
|
Name: label.GetName(),
|
|
Color: label.GetColor(),
|
|
Description: label.GetDescription(),
|
|
}
|
|
}
|
|
|
|
// GetLabels returns labels
|
|
func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
|
|
perPage := g.maxPerPage
|
|
labels := make([]*base.Label, 0, perPage)
|
|
for i := 1; ; i++ {
|
|
g.waitAndPickClient()
|
|
ls, resp, err := g.getClient().Issues.ListLabels(g.ctx, g.repoOwner, g.repoName,
|
|
&github.ListOptions{
|
|
Page: i,
|
|
PerPage: perPage,
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
g.setRate(&resp.Rate)
|
|
|
|
for _, label := range ls {
|
|
labels = append(labels, convertGithubLabel(label))
|
|
}
|
|
if len(ls) < perPage {
|
|
break
|
|
}
|
|
}
|
|
return labels, nil
|
|
}
|
|
|
|
func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) *base.Release {
|
|
// GitHub allows committish to be a reference.
|
|
// In this case, we need to remove the prefix, i.e. convert "refs/heads/main" to "main".
|
|
targetCommitish := strings.TrimPrefix(rel.GetTargetCommitish(), git.BranchPrefix)
|
|
|
|
r := &base.Release{
|
|
Name: rel.GetName(),
|
|
TagName: rel.GetTagName(),
|
|
TargetCommitish: targetCommitish,
|
|
Draft: rel.GetDraft(),
|
|
Prerelease: rel.GetPrerelease(),
|
|
Created: rel.GetCreatedAt().Time,
|
|
PublisherID: rel.GetAuthor().GetID(),
|
|
PublisherName: rel.GetAuthor().GetLogin(),
|
|
PublisherEmail: rel.GetAuthor().GetEmail(),
|
|
Body: rel.GetBody(),
|
|
}
|
|
|
|
if rel.PublishedAt != nil {
|
|
r.Published = rel.PublishedAt.Time
|
|
}
|
|
|
|
httpClient := NewMigrationHTTPClient()
|
|
|
|
for _, asset := range rel.Assets {
|
|
assetID := *asset.ID // Don't optimize this, for closure we need a local variable
|
|
r.Assets = append(r.Assets, &base.ReleaseAsset{
|
|
ID: asset.GetID(),
|
|
Name: asset.GetName(),
|
|
ContentType: asset.ContentType,
|
|
Size: asset.Size,
|
|
DownloadCount: asset.DownloadCount,
|
|
Created: asset.CreatedAt.Time,
|
|
Updated: asset.UpdatedAt.Time,
|
|
DownloadFunc: func() (io.ReadCloser, error) {
|
|
g.waitAndPickClient()
|
|
readCloser, redirectURL, err := g.getClient().Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, assetID, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := g.RefreshRate(); err != nil {
|
|
log.Error("g.getClient().RateLimits: %s", err)
|
|
}
|
|
|
|
if readCloser != nil {
|
|
return readCloser, nil
|
|
}
|
|
|
|
if redirectURL == "" {
|
|
return nil, fmt.Errorf("no release asset found for %d", assetID)
|
|
}
|
|
|
|
// Prevent open redirect
|
|
if !hasBaseURL(redirectURL, g.baseURL) &&
|
|
!hasBaseURL(redirectURL, "https://objects.githubusercontent.com/") &&
|
|
!hasBaseURL(redirectURL, "https://release-assets.githubusercontent.com/") {
|
|
WarnAndNotice("Unexpected AssetURL for assetID[%d] in %s: %s", asset.GetID(), g, redirectURL)
|
|
|
|
return io.NopCloser(strings.NewReader(redirectURL)), nil
|
|
}
|
|
|
|
g.waitAndPickClient()
|
|
req, err := http.NewRequestWithContext(g.ctx, "GET", redirectURL, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
resp, err := httpClient.Do(req)
|
|
err1 := g.RefreshRate()
|
|
if err1 != nil {
|
|
log.Error("g.RefreshRate(): %s", err1)
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return resp.Body, nil
|
|
},
|
|
})
|
|
}
|
|
return r
|
|
}
|
|
|
|
// GetReleases returns releases
|
|
func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
|
|
perPage := g.maxPerPage
|
|
releases := make([]*base.Release, 0, perPage)
|
|
for i := 1; ; i++ {
|
|
g.waitAndPickClient()
|
|
ls, resp, err := g.getClient().Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName,
|
|
&github.ListOptions{
|
|
Page: i,
|
|
PerPage: perPage,
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
g.setRate(&resp.Rate)
|
|
|
|
for _, release := range ls {
|
|
releases = append(releases, g.convertGithubRelease(release))
|
|
}
|
|
if len(ls) < perPage {
|
|
break
|
|
}
|
|
}
|
|
return releases, nil
|
|
}
|
|
|
|
// GetIssues returns issues according start and limit
|
|
func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
|
|
if perPage > g.maxPerPage {
|
|
perPage = g.maxPerPage
|
|
}
|
|
|
|
allIssues := make([]*base.Issue, 0, perPage)
|
|
g.waitAndPickClient()
|
|
|
|
issues, resp, err := g.getClient().Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, &github.IssueListByRepoOptions{
|
|
Sort: "created",
|
|
Direction: "asc",
|
|
State: "all",
|
|
ListCursorOptions: github.ListCursorOptions{
|
|
PerPage: perPage,
|
|
After: g.githubPagingInfo.After,
|
|
},
|
|
})
|
|
|
|
g.githubPagingInfo.After = resp.After
|
|
|
|
if err != nil {
|
|
return nil, false, fmt.Errorf("error while listing repos: %w", err)
|
|
}
|
|
log.Trace("Request get issues %d/%d, but in fact get %d", perPage, page, len(issues))
|
|
g.setRate(&resp.Rate)
|
|
for _, issue := range issues {
|
|
if issue.IsPullRequest() {
|
|
continue
|
|
}
|
|
|
|
labels := make([]*base.Label, 0, len(issue.Labels))
|
|
for _, l := range issue.Labels {
|
|
labels = append(labels, convertGithubLabel(l))
|
|
}
|
|
|
|
// get reactions
|
|
var reactions []*base.Reaction
|
|
if !g.SkipReactions {
|
|
for i := 1; ; i++ {
|
|
g.waitAndPickClient()
|
|
reactionOpts := &github.ListReactionOptions{
|
|
ListOptions: github.ListOptions{
|
|
Page: i,
|
|
PerPage: perPage,
|
|
},
|
|
}
|
|
res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, issue.GetNumber(), reactionOpts)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
g.setRate(&resp.Rate)
|
|
if len(res) == 0 {
|
|
break
|
|
}
|
|
for _, reaction := range res {
|
|
reactions = append(reactions, &base.Reaction{
|
|
UserID: reaction.User.GetID(),
|
|
UserName: reaction.User.GetLogin(),
|
|
Content: reaction.GetContent(),
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
var assignees []string
|
|
for i := range issue.Assignees {
|
|
assignees = append(assignees, issue.Assignees[i].GetLogin())
|
|
}
|
|
|
|
allIssues = append(allIssues, &base.Issue{
|
|
Title: *issue.Title,
|
|
Number: int64(*issue.Number),
|
|
PosterID: issue.GetUser().GetID(),
|
|
PosterName: issue.GetUser().GetLogin(),
|
|
PosterEmail: issue.GetUser().GetEmail(),
|
|
Content: issue.GetBody(),
|
|
Milestone: issue.GetMilestone().GetTitle(),
|
|
State: issue.GetState(),
|
|
Created: issue.GetCreatedAt().Time,
|
|
Updated: issue.GetUpdatedAt().Time,
|
|
Labels: labels,
|
|
Reactions: reactions,
|
|
Closed: issue.ClosedAt.GetTime(),
|
|
IsLocked: issue.GetLocked(),
|
|
Assignees: assignees,
|
|
ForeignIndex: int64(*issue.Number),
|
|
})
|
|
}
|
|
|
|
return allIssues, len(issues) < perPage, nil
|
|
}
|
|
|
|
// SupportGetRepoComments return true if it supports get repo comments
|
|
func (g *GithubDownloaderV3) SupportGetRepoComments() bool {
|
|
return true
|
|
}
|
|
|
|
// GetComments returns comments according issueNumber
|
|
func (g *GithubDownloaderV3) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) {
|
|
comments, err := g.getComments(commentable)
|
|
return comments, false, err
|
|
}
|
|
|
|
func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base.Comment, error) {
|
|
var (
|
|
allComments = make([]*base.Comment, 0, g.maxPerPage)
|
|
created = "created"
|
|
asc = "asc"
|
|
)
|
|
opt := &github.IssueListCommentsOptions{
|
|
Sort: &created,
|
|
Direction: &asc,
|
|
ListOptions: github.ListOptions{
|
|
PerPage: g.maxPerPage,
|
|
},
|
|
}
|
|
for {
|
|
g.waitAndPickClient()
|
|
comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(commentable.GetForeignIndex()), opt)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error while listing repos: %w", err)
|
|
}
|
|
g.setRate(&resp.Rate)
|
|
for _, comment := range comments {
|
|
// get reactions
|
|
var reactions []*base.Reaction
|
|
if !g.SkipReactions {
|
|
for i := 1; ; i++ {
|
|
g.waitAndPickClient()
|
|
reactionOpts := &github.ListReactionOptions{
|
|
ListOptions: github.ListOptions{
|
|
Page: i,
|
|
PerPage: g.maxPerPage,
|
|
},
|
|
}
|
|
res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), reactionOpts)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
g.setRate(&resp.Rate)
|
|
if len(res) == 0 {
|
|
break
|
|
}
|
|
for _, reaction := range res {
|
|
reactions = append(reactions, &base.Reaction{
|
|
UserID: reaction.User.GetID(),
|
|
UserName: reaction.User.GetLogin(),
|
|
Content: reaction.GetContent(),
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
allComments = append(allComments, &base.Comment{
|
|
IssueIndex: commentable.GetLocalIndex(),
|
|
Index: comment.GetID(),
|
|
PosterID: comment.GetUser().GetID(),
|
|
PosterName: comment.GetUser().GetLogin(),
|
|
PosterEmail: comment.GetUser().GetEmail(),
|
|
Content: comment.GetBody(),
|
|
Created: comment.GetCreatedAt().Time,
|
|
Updated: comment.GetUpdatedAt().Time,
|
|
Reactions: reactions,
|
|
})
|
|
}
|
|
if resp.NextPage == 0 {
|
|
break
|
|
}
|
|
opt.Page = resp.NextPage
|
|
}
|
|
return allComments, nil
|
|
}
|
|
|
|
func (g *GithubDownloaderV3) filterByHTMLURL(comments []*github.IssueComment, filterBy string) []*github.IssueComment {
|
|
var result []*github.IssueComment
|
|
for _, val := range comments {
|
|
if !strings.Contains(*val.HTMLURL, filterBy) {
|
|
result = append(result, val)
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
func (g *GithubDownloaderV3) filterPRComments(comments []*github.IssueComment) []*github.IssueComment {
|
|
return g.filterByHTMLURL(comments, "/pull/")
|
|
}
|
|
|
|
func (g *GithubDownloaderV3) filterIssueComments(comments []*github.IssueComment) []*github.IssueComment {
|
|
return g.filterByHTMLURL(comments, "/issues/")
|
|
}
|
|
|
|
// GetAllComments returns repository comments according page and perPageSize
|
|
func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment, bool, error) {
|
|
var (
|
|
allComments = make([]*base.Comment, 0, perPage)
|
|
created = "created"
|
|
asc = "asc"
|
|
)
|
|
if perPage > g.maxPerPage {
|
|
perPage = g.maxPerPage
|
|
}
|
|
opt := &github.IssueListCommentsOptions{
|
|
Sort: &created,
|
|
Direction: &asc,
|
|
ListOptions: github.ListOptions{
|
|
Page: page,
|
|
PerPage: perPage,
|
|
},
|
|
}
|
|
|
|
g.waitAndPickClient()
|
|
comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, 0, opt)
|
|
if err != nil {
|
|
return nil, false, fmt.Errorf("error while listing repos: %w", err)
|
|
}
|
|
isEnd := resp.NextPage == 0
|
|
|
|
if g.getIssues && !g.getPullRequests {
|
|
comments = g.filterPRComments(comments)
|
|
} else if !g.getIssues && g.getPullRequests {
|
|
comments = g.filterIssueComments(comments)
|
|
}
|
|
|
|
log.Trace("Request get comments %d/%d, but in fact get %d, next page is %d", perPage, page, len(comments), resp.NextPage)
|
|
g.setRate(&resp.Rate)
|
|
for _, comment := range comments {
|
|
// get reactions
|
|
var reactions []*base.Reaction
|
|
if !g.SkipReactions {
|
|
for i := 1; ; i++ {
|
|
g.waitAndPickClient()
|
|
reactionOpts := &github.ListReactionOptions{
|
|
ListOptions: github.ListOptions{
|
|
Page: i,
|
|
PerPage: g.maxPerPage,
|
|
},
|
|
}
|
|
res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), reactionOpts)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
g.setRate(&resp.Rate)
|
|
if len(res) == 0 {
|
|
break
|
|
}
|
|
for _, reaction := range res {
|
|
reactions = append(reactions, &base.Reaction{
|
|
UserID: reaction.User.GetID(),
|
|
UserName: reaction.User.GetLogin(),
|
|
Content: reaction.GetContent(),
|
|
})
|
|
}
|
|
}
|
|
}
|
|
idx := strings.LastIndex(*comment.IssueURL, "/")
|
|
issueIndex, _ := strconv.ParseInt((*comment.IssueURL)[idx+1:], 10, 64)
|
|
allComments = append(allComments, &base.Comment{
|
|
IssueIndex: issueIndex,
|
|
Index: comment.GetID(),
|
|
PosterID: comment.GetUser().GetID(),
|
|
PosterName: comment.GetUser().GetLogin(),
|
|
PosterEmail: comment.GetUser().GetEmail(),
|
|
Content: comment.GetBody(),
|
|
Created: comment.GetCreatedAt().Time,
|
|
Updated: comment.GetUpdatedAt().Time,
|
|
Reactions: reactions,
|
|
})
|
|
}
|
|
|
|
return allComments, isEnd, nil
|
|
}
|
|
|
|
// GetPullRequests returns pull requests according page and perPage
|
|
func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
|
|
if perPage > g.maxPerPage {
|
|
perPage = g.maxPerPage
|
|
}
|
|
opt := &github.PullRequestListOptions{
|
|
Sort: "created",
|
|
Direction: "asc",
|
|
State: "all",
|
|
ListOptions: github.ListOptions{
|
|
PerPage: perPage,
|
|
Page: page,
|
|
},
|
|
}
|
|
allPRs := make([]*base.PullRequest, 0, perPage)
|
|
g.waitAndPickClient()
|
|
prs, resp, err := g.getClient().PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
|
|
if err != nil {
|
|
return nil, false, fmt.Errorf("error while listing repos: %w", err)
|
|
}
|
|
log.Trace("Request get pull requests %d/%d, but in fact get %d", perPage, page, len(prs))
|
|
g.setRate(&resp.Rate)
|
|
for _, pr := range prs {
|
|
labels := make([]*base.Label, 0, len(pr.Labels))
|
|
for _, l := range pr.Labels {
|
|
labels = append(labels, convertGithubLabel(l))
|
|
}
|
|
|
|
// get reactions
|
|
var reactions []*base.Reaction
|
|
if !g.SkipReactions {
|
|
for i := 1; ; i++ {
|
|
g.waitAndPickClient()
|
|
reactionOpts := &github.ListReactionOptions{
|
|
ListOptions: github.ListOptions{
|
|
Page: i,
|
|
PerPage: perPage,
|
|
},
|
|
}
|
|
res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, pr.GetNumber(), reactionOpts)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
g.setRate(&resp.Rate)
|
|
if len(res) == 0 {
|
|
break
|
|
}
|
|
for _, reaction := range res {
|
|
reactions = append(reactions, &base.Reaction{
|
|
UserID: reaction.User.GetID(),
|
|
UserName: reaction.User.GetLogin(),
|
|
Content: reaction.GetContent(),
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
// download patch and saved as tmp file
|
|
g.waitAndPickClient()
|
|
|
|
allPRs = append(allPRs, &base.PullRequest{
|
|
Title: pr.GetTitle(),
|
|
Number: int64(pr.GetNumber()),
|
|
PosterID: pr.GetUser().GetID(),
|
|
PosterName: pr.GetUser().GetLogin(),
|
|
PosterEmail: pr.GetUser().GetEmail(),
|
|
Content: pr.GetBody(),
|
|
Milestone: pr.GetMilestone().GetTitle(),
|
|
State: pr.GetState(),
|
|
Created: pr.GetCreatedAt().Time,
|
|
Updated: pr.GetUpdatedAt().Time,
|
|
Closed: pr.ClosedAt.GetTime(),
|
|
Labels: labels,
|
|
Merged: pr.MergedAt != nil,
|
|
MergeCommitSHA: pr.GetMergeCommitSHA(),
|
|
MergedTime: pr.MergedAt.GetTime(),
|
|
IsLocked: pr.ActiveLockReason != nil,
|
|
Head: base.PullRequestBranch{
|
|
Ref: pr.GetHead().GetRef(),
|
|
SHA: pr.GetHead().GetSHA(),
|
|
OwnerName: pr.GetHead().GetUser().GetLogin(),
|
|
RepoName: pr.GetHead().GetRepo().GetName(),
|
|
CloneURL: pr.GetHead().GetRepo().GetCloneURL(), // see below for SECURITY related issues here
|
|
},
|
|
Base: base.PullRequestBranch{
|
|
Ref: pr.GetBase().GetRef(),
|
|
SHA: pr.GetBase().GetSHA(),
|
|
RepoName: pr.GetBase().GetRepo().GetName(),
|
|
OwnerName: pr.GetBase().GetUser().GetLogin(),
|
|
},
|
|
PatchURL: pr.GetPatchURL(), // see below for SECURITY related issues here
|
|
Reactions: reactions,
|
|
ForeignIndex: int64(*pr.Number),
|
|
IsDraft: pr.GetDraft(),
|
|
})
|
|
|
|
// SECURITY: Ensure that the PR is safe
|
|
_ = CheckAndEnsureSafePR(allPRs[len(allPRs)-1], g.baseURL, g)
|
|
}
|
|
|
|
return allPRs, len(prs) < perPage, nil
|
|
}
|
|
|
|
func convertGithubReview(r *github.PullRequestReview) *base.Review {
|
|
return &base.Review{
|
|
ID: r.GetID(),
|
|
ReviewerID: r.GetUser().GetID(),
|
|
ReviewerName: r.GetUser().GetLogin(),
|
|
CommitID: r.GetCommitID(),
|
|
Content: r.GetBody(),
|
|
CreatedAt: r.GetSubmittedAt().Time,
|
|
State: r.GetState(),
|
|
}
|
|
}
|
|
|
|
func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullRequestComment) ([]*base.ReviewComment, error) {
|
|
rcs := make([]*base.ReviewComment, 0, len(cs))
|
|
for _, c := range cs {
|
|
// get reactions
|
|
var reactions []*base.Reaction
|
|
if !g.SkipReactions {
|
|
for i := 1; ; i++ {
|
|
g.waitAndPickClient()
|
|
reactionOpts := &github.ListReactionOptions{
|
|
ListOptions: github.ListOptions{
|
|
Page: i,
|
|
PerPage: g.maxPerPage,
|
|
},
|
|
}
|
|
res, resp, err := g.getClient().Reactions.ListPullRequestCommentReactions(g.ctx, g.repoOwner, g.repoName, c.GetID(), reactionOpts)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
g.setRate(&resp.Rate)
|
|
if len(res) == 0 {
|
|
break
|
|
}
|
|
for _, reaction := range res {
|
|
reactions = append(reactions, &base.Reaction{
|
|
UserID: reaction.User.GetID(),
|
|
UserName: reaction.User.GetLogin(),
|
|
Content: reaction.GetContent(),
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
rcs = append(rcs, &base.ReviewComment{
|
|
ID: c.GetID(),
|
|
InReplyTo: c.GetInReplyTo(),
|
|
Content: c.GetBody(),
|
|
TreePath: c.GetPath(),
|
|
DiffHunk: c.GetDiffHunk(),
|
|
Position: c.GetPosition(),
|
|
CommitID: c.GetCommitID(),
|
|
PosterID: c.GetUser().GetID(),
|
|
Reactions: reactions,
|
|
CreatedAt: c.GetCreatedAt().Time,
|
|
UpdatedAt: c.GetUpdatedAt().Time,
|
|
})
|
|
}
|
|
return rcs, nil
|
|
}
|
|
|
|
// GetReviews returns pull requests review
|
|
func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) {
|
|
allReviews := make([]*base.Review, 0, g.maxPerPage)
|
|
if g.SkipReviews {
|
|
return allReviews, nil
|
|
}
|
|
opt := &github.ListOptions{
|
|
PerPage: g.maxPerPage,
|
|
}
|
|
// Get approve/request change reviews
|
|
for {
|
|
g.waitAndPickClient()
|
|
reviews, resp, err := g.getClient().PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error while listing repos: %w", err)
|
|
}
|
|
g.setRate(&resp.Rate)
|
|
for _, review := range reviews {
|
|
r := convertGithubReview(review)
|
|
r.IssueIndex = reviewable.GetLocalIndex()
|
|
// retrieve all review comments
|
|
opt2 := &github.ListOptions{
|
|
PerPage: g.maxPerPage,
|
|
}
|
|
for {
|
|
g.waitAndPickClient()
|
|
reviewComments, resp, err := g.getClient().PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), review.GetID(), opt2)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error while listing repos: %w", err)
|
|
}
|
|
g.setRate(&resp.Rate)
|
|
|
|
cs, err := g.convertGithubReviewComments(reviewComments)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
r.Comments = append(r.Comments, cs...)
|
|
if resp.NextPage == 0 {
|
|
break
|
|
}
|
|
opt2.Page = resp.NextPage
|
|
}
|
|
allReviews = append(allReviews, r)
|
|
}
|
|
if resp.NextPage == 0 {
|
|
break
|
|
}
|
|
opt.Page = resp.NextPage
|
|
}
|
|
// Get requested reviews
|
|
for {
|
|
g.waitAndPickClient()
|
|
reviewers, resp, err := g.getClient().PullRequests.ListReviewers(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error while listing repos: %w", err)
|
|
}
|
|
g.setRate(&resp.Rate)
|
|
for _, user := range reviewers.Users {
|
|
r := &base.Review{
|
|
ReviewerID: user.GetID(),
|
|
ReviewerName: user.GetLogin(),
|
|
State: base.ReviewStateRequestReview,
|
|
IssueIndex: reviewable.GetLocalIndex(),
|
|
}
|
|
allReviews = append(allReviews, r)
|
|
}
|
|
// TODO: Handle Team requests
|
|
if resp.NextPage == 0 {
|
|
break
|
|
}
|
|
opt.Page = resp.NextPage
|
|
}
|
|
return allReviews, nil
|
|
}
|
|
|
|
// FormatCloneURL add authentication into remote URLs
|
|
func (g *GithubDownloaderV3) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) {
|
|
u, err := url.Parse(remoteAddr)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if len(opts.AuthToken) > 0 {
|
|
// "multiple tokens" are used to benefit more "API rate limit quota"
|
|
// git clone doesn't count for rate limits, so only use the first token.
|
|
// source: https://github.com/orgs/community/discussions/44515
|
|
u.User = url.UserPassword("oauth2", strings.Split(opts.AuthToken, ",")[0])
|
|
}
|
|
return u.String(), nil
|
|
}
|