mirror of
https://github.com/go-gitea/gitea.git
synced 2026-10-26 06:16:56 +09:00
silverwindandbircni
fc49368845
Supports migrating from Gitea-compatible forges like Forgejo and fixes many migration bugs. 1. Pre-release and `+gitea-` suffixed server versions no longer drop reviews, reactions and pagination. 2. Review states, dismissals, scoped and archived labels and merge bases are migrated. 3. Disabled units, external trackers, unavailable release assets, and GitLab confidential issues and internal notes are skipped. 4. Credentials are only sent to the source host, and `Retry-After` rate limits are waited out. 5. GitHub migrations of only issues or pull requests, basic auth clones and dismissed reviews work again. 6. The API returns 404 for releases of a disabled unit and shows migrated authors as mannequins, replacing the never populated `original_author` and `original_author_id` fields. Fixes: https://github.com/go-gitea/gitea/issues/19533 Fixes: https://github.com/go-gitea/gitea/issues/35885 Fixes: https://github.com/go-gitea/gitea/issues/18275 Fixes: https://github.com/go-gitea/gitea/issues/35069 Fixes: https://github.com/go-gitea/gitea/issues/17763 Fixes: https://github.com/go-gitea/gitea/issues/39548 Co-authored-by: bircni <bircni@icloud.com>
894 lines
26 KiB
Go
894 lines
26 KiB
Go
// Copyright 2019 The Gitea Authors. All rights reserved.
|
|
// Copyright 2018 Jonas Franz. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package migrations
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.dev/modules/git"
|
|
"gitea.dev/modules/log"
|
|
base "gitea.dev/modules/migration"
|
|
"gitea.dev/modules/structs"
|
|
"gitea.dev/modules/util"
|
|
|
|
"github.com/google/go-github/v92/github"
|
|
)
|
|
|
|
var (
|
|
_ base.Downloader = &GithubDownloaderV3{}
|
|
_ base.DownloaderFactory = &GithubDownloaderV3Factory{}
|
|
// GithubLimitRateRemaining limit to wait for new rate to apply
|
|
GithubLimitRateRemaining = 0
|
|
)
|
|
|
|
func init() {
|
|
RegisterDownloaderFactory(&GithubDownloaderV3Factory{})
|
|
}
|
|
|
|
// GithubDownloaderV3Factory defines a github downloader v3 factory
|
|
type GithubDownloaderV3Factory struct{}
|
|
|
|
// New returns a Downloader related to this factory according MigrateOptions
|
|
func (f *GithubDownloaderV3Factory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
|
|
u, err := url.Parse(opts.CloneAddr)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
baseURL := u.Scheme + "://" + u.Host
|
|
fields := strings.Split(u.Path, "/")
|
|
oldOwner := fields[1]
|
|
oldName := strings.TrimSuffix(fields[2], ".git")
|
|
|
|
log.Trace("Create github downloader BaseURL: %s %s/%s", baseURL, oldOwner, oldName)
|
|
|
|
downloader, err := NewGithubDownloaderV3(ctx, baseURL, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// GitHub rejects anonymous GraphQL requests, and comments only ride along with the issue and PR sweeps
|
|
if (opts.AuthToken != "" || opts.AuthPassword != "") && (opts.Issues || opts.PullRequests) {
|
|
return newGithubGraphQLDownloader(downloader, !opts.Comments), nil
|
|
}
|
|
return downloader, nil
|
|
}
|
|
|
|
// GitServiceType returns the type of git service
|
|
func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType {
|
|
return structs.GithubService
|
|
}
|
|
|
|
// GithubDownloaderV3 implements a Downloader interface to get repository information
|
|
// from github via APIv3
|
|
type GithubDownloaderV3 struct {
|
|
base.NullDownloader
|
|
clients []*github.Client
|
|
baseURL string
|
|
repoOwner string
|
|
repoName string
|
|
userName string
|
|
password string
|
|
rates []*github.Rate
|
|
curClientIdx int
|
|
maxPerPage int
|
|
SkipReactions bool
|
|
SkipReviews bool
|
|
issuePage int
|
|
issueCursor string
|
|
nextIssueCursor string
|
|
countPaginated bool // GitBucket sends no Link header
|
|
}
|
|
|
|
// NewGithubDownloaderV3 creates a github Downloader via github v3 API
|
|
func NewGithubDownloaderV3(_ context.Context, baseURL, userName, password, token, repoOwner, repoName string) (*GithubDownloaderV3, error) {
|
|
downloader := GithubDownloaderV3{
|
|
userName: userName,
|
|
baseURL: baseURL,
|
|
password: password,
|
|
repoOwner: repoOwner,
|
|
repoName: repoName,
|
|
maxPerPage: 100,
|
|
}
|
|
|
|
apiURL := util.Iif(baseURL == "https://github.com", "https://api.github.com", baseURL)
|
|
if token != "" {
|
|
for token := range strings.SplitSeq(token, ",") {
|
|
if err := downloader.addClient(newMigrationHTTPClient(apiURL, "Bearer "+strings.TrimSpace(token)), baseURL); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
} else {
|
|
if err := downloader.addClient(newMigrationHTTPClient(apiURL, basicAuthorization(userName, password)), baseURL); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return &downloader, nil
|
|
}
|
|
|
|
// String implements Stringer
|
|
func (g *GithubDownloaderV3) String() string {
|
|
return fmt.Sprintf("migration from github server %s %s/%s", g.baseURL, g.repoOwner, g.repoName)
|
|
}
|
|
|
|
func (g *GithubDownloaderV3) LogString() string {
|
|
if g == nil {
|
|
return "<GithubDownloaderV3 nil>"
|
|
}
|
|
return fmt.Sprintf("<GithubDownloaderV3 %s %s/%s>", g.baseURL, g.repoOwner, g.repoName)
|
|
}
|
|
|
|
func (g *GithubDownloaderV3) addClient(client *http.Client, baseURL string) error {
|
|
opts := []github.ClientOptionsFunc{github.WithHTTPClient(client)}
|
|
if baseURL != "https://github.com" {
|
|
opts = append(opts, github.WithEnterpriseURLs(baseURL, baseURL))
|
|
}
|
|
githubClient, err := github.NewClient(opts...)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
g.clients = append(g.clients, githubClient)
|
|
g.rates = append(g.rates, nil)
|
|
return nil
|
|
}
|
|
|
|
func (g *GithubDownloaderV3) waitAndPickClient(ctx context.Context) {
|
|
var recentIdx int
|
|
var maxRemaining int
|
|
for i := 0; i < len(g.clients); i++ {
|
|
if g.rates[i] == nil { // probe unknown clients once, else their rate never gets learned
|
|
g.curClientIdx = i
|
|
return
|
|
}
|
|
if g.rates[i].Remaining > maxRemaining {
|
|
maxRemaining = g.rates[i].Remaining
|
|
recentIdx = i
|
|
}
|
|
}
|
|
g.curClientIdx = recentIdx
|
|
|
|
for g.rates[g.curClientIdx] != nil && g.rates[g.curClientIdx].Remaining <= GithubLimitRateRemaining {
|
|
timer := time.NewTimer(time.Until(g.rates[g.curClientIdx].Reset.Time))
|
|
select {
|
|
case <-ctx.Done():
|
|
timer.Stop()
|
|
return
|
|
case <-timer.C:
|
|
}
|
|
|
|
err := g.RefreshRate(ctx)
|
|
if err != nil {
|
|
log.Error("g.getClient().RateLimit.Get: %s", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// RefreshRate update the current rate (doesn't count in rate limit)
|
|
func (g *GithubDownloaderV3) RefreshRate(ctx context.Context) error {
|
|
rates, _, err := g.getClient().RateLimit.Get(ctx)
|
|
if err != nil {
|
|
// if rate limit is not enabled, ignore it
|
|
if strings.Contains(err.Error(), "404") {
|
|
g.setRate(nil)
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
|
|
g.setRate(rates.GetCore())
|
|
return nil
|
|
}
|
|
|
|
func (g *GithubDownloaderV3) getClient() *github.Client {
|
|
return g.clients[g.curClientIdx]
|
|
}
|
|
|
|
func (g *GithubDownloaderV3) setRate(rate *github.Rate) {
|
|
g.rates[g.curClientIdx] = rate
|
|
}
|
|
|
|
// GetRepoInfo returns a repository information
|
|
func (g *GithubDownloaderV3) GetRepoInfo(ctx context.Context) (*base.Repository, error) {
|
|
g.waitAndPickClient(ctx)
|
|
gr, resp, err := g.getClient().Repositories.Get(ctx, g.repoOwner, g.repoName)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
g.setRate(&resp.Rate)
|
|
|
|
// convert github repo to stand Repo
|
|
return &base.Repository{
|
|
Owner: g.repoOwner,
|
|
Name: gr.GetName(),
|
|
IsPrivate: gr.GetPrivate(),
|
|
Description: gr.GetDescription(),
|
|
Website: gr.GetHomepage(),
|
|
OriginalURL: gr.GetHTMLURL(),
|
|
CloneURL: gr.GetCloneURL(),
|
|
DefaultBranch: gr.GetDefaultBranch(),
|
|
}, nil
|
|
}
|
|
|
|
// GetTopics return github topics
|
|
func (g *GithubDownloaderV3) GetTopics(ctx context.Context) ([]string, error) {
|
|
g.waitAndPickClient(ctx)
|
|
r, resp, err := g.getClient().Repositories.Get(ctx, g.repoOwner, g.repoName)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
g.setRate(&resp.Rate)
|
|
return r.Topics, nil
|
|
}
|
|
|
|
// GetMilestones returns milestones
|
|
func (g *GithubDownloaderV3) GetMilestones(ctx context.Context) ([]*base.Milestone, error) {
|
|
perPage := g.maxPerPage
|
|
milestones := make([]*base.Milestone, 0, perPage)
|
|
for i := 1; ; i++ {
|
|
g.waitAndPickClient(ctx)
|
|
ms, resp, err := g.getClient().Issues.ListMilestones(ctx, g.repoOwner, g.repoName,
|
|
&github.MilestoneListOptions{
|
|
State: "all",
|
|
ListOptions: github.ListOptions{
|
|
Page: i,
|
|
PerPage: perPage,
|
|
},
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
g.setRate(&resp.Rate)
|
|
|
|
for _, m := range ms {
|
|
state := "open"
|
|
if m.State != nil {
|
|
state = *m.State
|
|
}
|
|
milestones = append(milestones, &base.Milestone{
|
|
Title: m.GetTitle(),
|
|
Description: m.GetDescription(),
|
|
Deadline: m.DueOn.GetTime(),
|
|
State: state,
|
|
Created: m.GetCreatedAt().Time,
|
|
Updated: m.UpdatedAt.GetTime(),
|
|
Closed: m.ClosedAt.GetTime(),
|
|
})
|
|
}
|
|
if len(ms) < perPage {
|
|
break
|
|
}
|
|
}
|
|
return milestones, nil
|
|
}
|
|
|
|
func convertGithubLabel(label *github.Label) *base.Label {
|
|
return &base.Label{
|
|
Name: label.GetName(),
|
|
Color: label.GetColor(),
|
|
Description: label.GetDescription(),
|
|
}
|
|
}
|
|
|
|
// GetLabels returns labels
|
|
func (g *GithubDownloaderV3) GetLabels(ctx context.Context) ([]*base.Label, error) {
|
|
perPage := g.maxPerPage
|
|
labels := make([]*base.Label, 0, perPage)
|
|
for i := 1; ; i++ {
|
|
g.waitAndPickClient(ctx)
|
|
ls, resp, err := g.getClient().Issues.ListLabels(ctx, g.repoOwner, g.repoName,
|
|
&github.ListOptions{
|
|
Page: i,
|
|
PerPage: perPage,
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
g.setRate(&resp.Rate)
|
|
|
|
for _, label := range ls {
|
|
labels = append(labels, convertGithubLabel(label))
|
|
}
|
|
if len(ls) < perPage {
|
|
break
|
|
}
|
|
}
|
|
return labels, nil
|
|
}
|
|
|
|
func (g *GithubDownloaderV3) convertGithubRelease(ctx context.Context, rel *github.RepositoryRelease) *base.Release {
|
|
// GitHub allows commitish to be a reference.
|
|
// In this case, we need to remove the prefix, i.e. convert "refs/heads/main" to "main".
|
|
targetCommitish := strings.TrimPrefix(rel.GetTargetCommitish(), git.BranchPrefix)
|
|
|
|
r := &base.Release{
|
|
Name: rel.GetName(),
|
|
TagName: rel.GetTagName(),
|
|
TargetCommitish: targetCommitish,
|
|
Draft: rel.GetDraft(),
|
|
Prerelease: rel.GetPrerelease(),
|
|
Created: rel.GetCreatedAt().Time,
|
|
PublisherID: rel.GetAuthor().GetID(),
|
|
PublisherName: rel.GetAuthor().GetLogin(),
|
|
PublisherEmail: rel.GetAuthor().GetEmail(),
|
|
Body: rel.GetBody(),
|
|
}
|
|
|
|
if rel.PublishedAt != nil {
|
|
r.Published = rel.PublishedAt.Time
|
|
}
|
|
|
|
for _, asset := range rel.Assets {
|
|
assetID := asset.GetID() // Don't optimize this, for closure we need a local variable TODO: no need to do so in new Golang
|
|
if assetID == 0 {
|
|
continue
|
|
}
|
|
r.Assets = append(r.Assets, &base.ReleaseAsset{
|
|
ID: asset.GetID(),
|
|
Name: asset.GetName(),
|
|
Size: asset.Size,
|
|
DownloadCount: asset.DownloadCount,
|
|
Created: asset.CreatedAt.Time,
|
|
Updated: asset.UpdatedAt.Time,
|
|
DownloadFunc: func() (io.ReadCloser, error) {
|
|
g.waitAndPickClient(ctx)
|
|
readCloser, redirectURL, err := g.getClient().Repositories.DownloadReleaseAsset(ctx, g.repoOwner, g.repoName, assetID, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := g.RefreshRate(ctx); err != nil {
|
|
log.Error("g.getClient().RateLimits: %s", err)
|
|
}
|
|
|
|
if readCloser != nil {
|
|
return readCloser, nil
|
|
}
|
|
|
|
if redirectURL == "" {
|
|
return nil, fmt.Errorf("no release asset found for %d", assetID)
|
|
}
|
|
|
|
// Prevent open redirect
|
|
if !hasBaseURL(redirectURL, g.baseURL) &&
|
|
!hasBaseURL(redirectURL, "https://objects.githubusercontent.com/") &&
|
|
!hasBaseURL(redirectURL, "https://release-assets.githubusercontent.com/") {
|
|
WarnAndNotice("Unexpected AssetURL for assetID[%d] in %s: %s", asset.GetID(), g, redirectURL)
|
|
|
|
return io.NopCloser(strings.NewReader(redirectURL)), nil
|
|
}
|
|
|
|
g.waitAndPickClient(ctx)
|
|
rc, err := downloadAsset(ctx, getMigrationHTTPClient(), redirectURL)
|
|
if err1 := g.RefreshRate(ctx); err1 != nil {
|
|
log.Error("g.RefreshRate(): %s", err1)
|
|
}
|
|
return rc, err
|
|
},
|
|
})
|
|
}
|
|
return r
|
|
}
|
|
|
|
// GetReleases returns releases
|
|
func (g *GithubDownloaderV3) GetReleases(ctx context.Context) ([]*base.Release, error) {
|
|
perPage := g.maxPerPage
|
|
releases := make([]*base.Release, 0, perPage)
|
|
for i := 1; ; i++ {
|
|
g.waitAndPickClient(ctx)
|
|
ls, resp, err := g.getClient().Repositories.ListReleases(ctx, g.repoOwner, g.repoName,
|
|
&github.ListOptions{
|
|
Page: i,
|
|
PerPage: perPage,
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
g.setRate(&resp.Rate)
|
|
|
|
for _, release := range ls {
|
|
releases = append(releases, g.convertGithubRelease(ctx, release))
|
|
}
|
|
if len(ls) < perPage {
|
|
break
|
|
}
|
|
}
|
|
return releases, nil
|
|
}
|
|
|
|
// GetIssues returns issues according start and limit
|
|
func (g *GithubDownloaderV3) GetIssues(ctx context.Context, page, perPage int) ([]*base.Issue, bool, error) {
|
|
if perPage > g.maxPerPage {
|
|
perPage = g.maxPerPage
|
|
}
|
|
opt := &github.IssueListByRepoOptions{
|
|
Sort: "created",
|
|
Direction: "asc",
|
|
State: "all",
|
|
ListOptions: github.ListOptions{
|
|
PerPage: perPage,
|
|
Page: page,
|
|
},
|
|
}
|
|
switch page {
|
|
case g.issuePage:
|
|
opt.After = g.issueCursor
|
|
case g.issuePage + 1:
|
|
opt.After = g.nextIssueCursor
|
|
}
|
|
if opt.After != "" {
|
|
opt.ListOptions.Page = 0
|
|
}
|
|
|
|
allIssues := make([]*base.Issue, 0, perPage)
|
|
g.waitAndPickClient(ctx)
|
|
issues, resp, err := g.getClient().Issues.ListByRepo(ctx, g.repoOwner, g.repoName, opt)
|
|
if err != nil {
|
|
return nil, false, fmt.Errorf("GetIssues: %w", err)
|
|
}
|
|
log.Trace("Request get issues %d/%d, but in fact get %d", perPage, page, len(issues))
|
|
g.setRate(&resp.Rate)
|
|
for _, issue := range issues {
|
|
if issue.IsPullRequest() {
|
|
continue
|
|
}
|
|
|
|
labels := make([]*base.Label, 0, len(issue.Labels))
|
|
for _, l := range issue.Labels {
|
|
labels = append(labels, convertGithubLabel(l))
|
|
}
|
|
|
|
reactions, err := g.getIssueReactions(ctx, issue.GetNumber(), perPage)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
|
|
var assignees []string
|
|
for i := range issue.Assignees {
|
|
assignees = append(assignees, issue.Assignees[i].GetLogin())
|
|
}
|
|
|
|
allIssues = append(allIssues, &base.Issue{
|
|
Title: *issue.Title,
|
|
Number: int64(*issue.Number),
|
|
PosterID: issue.GetUser().GetID(),
|
|
PosterName: issue.GetUser().GetLogin(),
|
|
PosterEmail: issue.GetUser().GetEmail(),
|
|
Content: issue.GetBody(),
|
|
Milestone: issue.GetMilestone().GetTitle(),
|
|
State: issue.GetState(),
|
|
Created: issue.GetCreatedAt().Time,
|
|
Updated: issue.GetUpdatedAt().Time,
|
|
Labels: labels,
|
|
Reactions: reactions,
|
|
Closed: issue.ClosedAt.GetTime(),
|
|
IsLocked: issue.GetLocked(),
|
|
Assignees: assignees,
|
|
ForeignIndex: int64(*issue.Number),
|
|
})
|
|
}
|
|
|
|
g.issuePage, g.issueCursor, g.nextIssueCursor = page, opt.After, resp.After
|
|
return allIssues, g.isLastPage(resp, len(issues), perPage), nil
|
|
}
|
|
|
|
func (g *GithubDownloaderV3) isLastPage(resp *github.Response, n, perPage int) bool {
|
|
if g.countPaginated {
|
|
return n < perPage
|
|
}
|
|
return resp.NextPage == 0 && resp.After == ""
|
|
}
|
|
|
|
// SupportGetRepoComments return true if it supports get repo comments
|
|
func (g *GithubDownloaderV3) SupportGetRepoComments() bool {
|
|
return true
|
|
}
|
|
|
|
// GetComments returns comments according issueNumber
|
|
func (g *GithubDownloaderV3) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) {
|
|
comments, err := g.getComments(ctx, commentable)
|
|
return comments, false, err
|
|
}
|
|
|
|
func (g *GithubDownloaderV3) getComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, error) {
|
|
var (
|
|
allComments = make([]*base.Comment, 0, g.maxPerPage)
|
|
created = "created"
|
|
asc = "asc"
|
|
)
|
|
opt := &github.IssueListCommentsOptions{
|
|
Sort: &created,
|
|
Direction: &asc,
|
|
ListOptions: github.ListOptions{
|
|
PerPage: g.maxPerPage,
|
|
},
|
|
}
|
|
for {
|
|
g.waitAndPickClient(ctx)
|
|
comments, resp, err := g.getClient().Issues.ListComments(ctx, g.repoOwner, g.repoName, int(commentable.GetForeignIndex()), opt)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error while listing repos: %w", err)
|
|
}
|
|
g.setRate(&resp.Rate)
|
|
for _, comment := range comments {
|
|
// get reactions
|
|
var reactions []*base.Reaction
|
|
if !g.SkipReactions {
|
|
for i := 1; ; i++ {
|
|
g.waitAndPickClient(ctx)
|
|
res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListReactionOptions{
|
|
ListOptions: github.ListOptions{
|
|
Page: i,
|
|
PerPage: g.maxPerPage,
|
|
},
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
g.setRate(&resp.Rate)
|
|
if len(res) == 0 {
|
|
break
|
|
}
|
|
for _, reaction := range res {
|
|
reactions = append(reactions, &base.Reaction{
|
|
UserID: reaction.User.GetID(),
|
|
UserName: reaction.User.GetLogin(),
|
|
Content: reaction.GetContent(),
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
allComments = append(allComments, &base.Comment{
|
|
IssueIndex: commentable.GetLocalIndex(),
|
|
Index: comment.GetID(),
|
|
PosterID: comment.GetUser().GetID(),
|
|
PosterName: comment.GetUser().GetLogin(),
|
|
PosterEmail: comment.GetUser().GetEmail(),
|
|
Content: comment.GetBody(),
|
|
Created: comment.GetCreatedAt().Time,
|
|
Updated: comment.GetUpdatedAt().Time,
|
|
Reactions: reactions,
|
|
})
|
|
}
|
|
if resp.NextPage == 0 {
|
|
break
|
|
}
|
|
opt.Page = resp.NextPage
|
|
}
|
|
return allComments, nil
|
|
}
|
|
|
|
// GetAllComments returns repository comments according page and perPageSize
|
|
func (g *GithubDownloaderV3) GetAllComments(ctx context.Context, page, perPage int) ([]*base.Comment, bool, error) {
|
|
var (
|
|
allComments = make([]*base.Comment, 0, perPage)
|
|
created = "created"
|
|
asc = "asc"
|
|
)
|
|
if perPage > g.maxPerPage {
|
|
perPage = g.maxPerPage
|
|
}
|
|
opt := &github.IssueListCommentsOptions{
|
|
Sort: &created,
|
|
Direction: &asc,
|
|
ListOptions: github.ListOptions{
|
|
Page: page,
|
|
PerPage: perPage,
|
|
},
|
|
}
|
|
|
|
g.waitAndPickClient(ctx)
|
|
comments, resp, err := g.getClient().Issues.ListComments(ctx, g.repoOwner, g.repoName, 0, opt)
|
|
if err != nil {
|
|
return nil, false, fmt.Errorf("error while listing repos: %w", err)
|
|
}
|
|
isEnd := resp.NextPage == 0
|
|
|
|
log.Trace("Request get comments %d/%d, but in fact get %d, next page is %d", perPage, page, len(comments), resp.NextPage)
|
|
g.setRate(&resp.Rate)
|
|
for _, comment := range comments {
|
|
// get reactions
|
|
var reactions []*base.Reaction
|
|
if !g.SkipReactions {
|
|
for i := 1; ; i++ {
|
|
g.waitAndPickClient(ctx)
|
|
res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListReactionOptions{
|
|
ListOptions: github.ListOptions{
|
|
Page: i,
|
|
PerPage: g.maxPerPage,
|
|
},
|
|
})
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
g.setRate(&resp.Rate)
|
|
if len(res) == 0 {
|
|
break
|
|
}
|
|
for _, reaction := range res {
|
|
reactions = append(reactions, &base.Reaction{
|
|
UserID: reaction.User.GetID(),
|
|
UserName: reaction.User.GetLogin(),
|
|
Content: reaction.GetContent(),
|
|
})
|
|
}
|
|
}
|
|
}
|
|
idx := strings.LastIndex(*comment.IssueURL, "/")
|
|
issueIndex, _ := strconv.ParseInt((*comment.IssueURL)[idx+1:], 10, 64)
|
|
allComments = append(allComments, &base.Comment{
|
|
IssueIndex: issueIndex,
|
|
Index: comment.GetID(),
|
|
PosterID: comment.GetUser().GetID(),
|
|
PosterName: comment.GetUser().GetLogin(),
|
|
PosterEmail: comment.GetUser().GetEmail(),
|
|
Content: comment.GetBody(),
|
|
Created: comment.GetCreatedAt().Time,
|
|
Updated: comment.GetUpdatedAt().Time,
|
|
Reactions: reactions,
|
|
})
|
|
}
|
|
|
|
return allComments, isEnd, nil
|
|
}
|
|
|
|
// GetPullRequests returns pull requests according page and perPage
|
|
func (g *GithubDownloaderV3) GetPullRequests(ctx context.Context, page, perPage int) ([]*base.PullRequest, bool, error) {
|
|
if perPage > g.maxPerPage {
|
|
perPage = g.maxPerPage
|
|
}
|
|
opt := &github.PullRequestListOptions{
|
|
Sort: "created",
|
|
Direction: "asc",
|
|
State: "all",
|
|
ListOptions: github.ListOptions{
|
|
PerPage: perPage,
|
|
Page: page,
|
|
},
|
|
}
|
|
allPRs := make([]*base.PullRequest, 0, perPage)
|
|
g.waitAndPickClient(ctx)
|
|
prs, resp, err := g.getClient().PullRequests.List(ctx, g.repoOwner, g.repoName, opt)
|
|
if err != nil {
|
|
return nil, false, fmt.Errorf("error while listing repos: %w", err)
|
|
}
|
|
log.Trace("Request get pull requests %d/%d, but in fact get %d", perPage, page, len(prs))
|
|
g.setRate(&resp.Rate)
|
|
for _, pr := range prs {
|
|
basePR, err := g.convertGithubPullRequest(ctx, pr, perPage)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
allPRs = append(allPRs, basePR)
|
|
|
|
// SECURITY: Ensure that the PR is safe
|
|
_ = CheckAndEnsureSafePR(basePR, g.baseURL, g)
|
|
}
|
|
|
|
return allPRs, g.isLastPage(resp, len(prs), perPage), nil
|
|
}
|
|
|
|
func (g *GithubDownloaderV3) convertGithubPullRequest(ctx context.Context, pr *github.PullRequest, perPage int) (*base.PullRequest, error) {
|
|
labels := make([]*base.Label, 0, len(pr.Labels))
|
|
for _, l := range pr.Labels {
|
|
labels = append(labels, convertGithubLabel(l))
|
|
}
|
|
|
|
reactions, err := g.getIssueReactions(ctx, pr.GetNumber(), perPage)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
g.waitAndPickClient(ctx)
|
|
|
|
return &base.PullRequest{
|
|
Title: pr.GetTitle(),
|
|
Number: int64(pr.GetNumber()),
|
|
PosterID: pr.GetUser().GetID(),
|
|
PosterName: pr.GetUser().GetLogin(),
|
|
PosterEmail: pr.GetUser().GetEmail(),
|
|
Content: pr.GetBody(),
|
|
Milestone: pr.GetMilestone().GetTitle(),
|
|
State: pr.GetState(),
|
|
Created: pr.GetCreatedAt().Time,
|
|
Updated: pr.GetUpdatedAt().Time,
|
|
Closed: pr.ClosedAt.GetTime(),
|
|
Labels: labels,
|
|
Merged: pr.MergedAt != nil,
|
|
MergeCommitSHA: pr.GetMergeCommitSHA(),
|
|
MergedTime: pr.MergedAt.GetTime(),
|
|
IsLocked: pr.GetLocked(),
|
|
Head: base.PullRequestBranch{
|
|
Ref: pr.GetHead().GetRef(),
|
|
SHA: pr.GetHead().GetSHA(),
|
|
OwnerName: pr.GetHead().GetUser().GetLogin(),
|
|
RepoName: pr.GetHead().GetRepo().GetName(),
|
|
CloneURL: pr.GetHead().GetRepo().GetCloneURL(), // see below for SECURITY related issues here
|
|
},
|
|
Base: base.PullRequestBranch{
|
|
Ref: pr.GetBase().GetRef(),
|
|
SHA: pr.GetBase().GetSHA(),
|
|
RepoName: pr.GetBase().GetRepo().GetName(),
|
|
OwnerName: pr.GetBase().GetUser().GetLogin(),
|
|
},
|
|
PatchURL: pr.GetPatchURL(), // see below for SECURITY related issues here
|
|
Reactions: reactions,
|
|
ForeignIndex: int64(*pr.Number),
|
|
IsDraft: pr.GetDraft(),
|
|
}, nil
|
|
}
|
|
|
|
func (g *GithubDownloaderV3) getIssueReactions(ctx context.Context, number, perPage int) ([]*base.Reaction, error) {
|
|
if g.SkipReactions {
|
|
return nil, nil
|
|
}
|
|
var reactions []*base.Reaction
|
|
for i := 1; ; i++ {
|
|
g.waitAndPickClient(ctx)
|
|
res, resp, err := g.getClient().Reactions.ListIssueReactions(ctx, g.repoOwner, g.repoName, number, &github.ListReactionOptions{
|
|
ListOptions: github.ListOptions{
|
|
Page: i,
|
|
PerPage: perPage,
|
|
},
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
g.setRate(&resp.Rate)
|
|
if len(res) == 0 {
|
|
return reactions, nil
|
|
}
|
|
for _, reaction := range res {
|
|
reactions = append(reactions, &base.Reaction{
|
|
UserID: reaction.User.GetID(),
|
|
UserName: reaction.User.GetLogin(),
|
|
Content: reaction.GetContent(),
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
func convertGithubReview(r *github.PullRequestReview) *base.Review {
|
|
dismissed := r.GetState() == "DISMISSED"
|
|
return &base.Review{
|
|
ID: r.GetID(),
|
|
ReviewerID: r.GetUser().GetID(),
|
|
ReviewerName: r.GetUser().GetLogin(),
|
|
CommitID: r.GetCommitID(),
|
|
Content: r.GetBody(),
|
|
CreatedAt: r.GetSubmittedAt().Time,
|
|
State: util.Iif(dismissed, base.ReviewStateCommented, r.GetState()), // GitHub drops the state a review had before its dismissal
|
|
Dismissed: dismissed,
|
|
}
|
|
}
|
|
|
|
func (g *GithubDownloaderV3) convertGithubReviewComments(ctx context.Context, cs []*github.PullRequestComment) ([]*base.ReviewComment, error) {
|
|
rcs := make([]*base.ReviewComment, 0, len(cs))
|
|
for _, c := range cs {
|
|
// get reactions
|
|
var reactions []*base.Reaction
|
|
if !g.SkipReactions {
|
|
for i := 1; ; i++ {
|
|
g.waitAndPickClient(ctx)
|
|
res, resp, err := g.getClient().Reactions.ListPullRequestCommentReactions(ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListReactionOptions{
|
|
ListOptions: github.ListOptions{
|
|
Page: i,
|
|
PerPage: g.maxPerPage,
|
|
},
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
g.setRate(&resp.Rate)
|
|
if len(res) == 0 {
|
|
break
|
|
}
|
|
for _, reaction := range res {
|
|
reactions = append(reactions, &base.Reaction{
|
|
UserID: reaction.User.GetID(),
|
|
UserName: reaction.User.GetLogin(),
|
|
Content: reaction.GetContent(),
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
rcs = append(rcs, &base.ReviewComment{
|
|
ID: c.GetID(),
|
|
InReplyTo: c.GetInReplyTo(),
|
|
Content: c.GetBody(),
|
|
TreePath: c.GetPath(),
|
|
DiffHunk: c.GetDiffHunk(),
|
|
Position: c.GetPosition(),
|
|
CommitID: c.GetCommitID(),
|
|
PosterID: c.GetUser().GetID(),
|
|
Reactions: reactions,
|
|
CreatedAt: c.GetCreatedAt().Time,
|
|
UpdatedAt: c.GetUpdatedAt().Time,
|
|
})
|
|
}
|
|
return rcs, nil
|
|
}
|
|
|
|
// GetReviews returns pull requests review
|
|
func (g *GithubDownloaderV3) GetReviews(ctx context.Context, reviewable base.Reviewable) ([]*base.Review, error) {
|
|
allReviews := make([]*base.Review, 0, g.maxPerPage)
|
|
if g.SkipReviews {
|
|
return allReviews, nil
|
|
}
|
|
opt := &github.ListOptions{
|
|
PerPage: g.maxPerPage,
|
|
}
|
|
// Get approve/request change reviews
|
|
for {
|
|
g.waitAndPickClient(ctx)
|
|
reviews, resp, err := g.getClient().PullRequests.ListReviews(ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error while listing repos: %w", err)
|
|
}
|
|
g.setRate(&resp.Rate)
|
|
for _, review := range reviews {
|
|
r := convertGithubReview(review)
|
|
r.IssueIndex = reviewable.GetLocalIndex()
|
|
// retrieve all review comments
|
|
opt2 := &github.ListOptions{
|
|
PerPage: g.maxPerPage,
|
|
}
|
|
for {
|
|
g.waitAndPickClient(ctx)
|
|
reviewComments, resp, err := g.getClient().PullRequests.ListReviewComments(ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), review.GetID(), opt2)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error while listing repos: %w", err)
|
|
}
|
|
g.setRate(&resp.Rate)
|
|
|
|
cs, err := g.convertGithubReviewComments(ctx, reviewComments)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
r.Comments = append(r.Comments, cs...)
|
|
if resp.NextPage == 0 {
|
|
break
|
|
}
|
|
opt2.Page = resp.NextPage
|
|
}
|
|
allReviews = append(allReviews, r)
|
|
}
|
|
if resp.NextPage == 0 {
|
|
break
|
|
}
|
|
opt.Page = resp.NextPage
|
|
}
|
|
// Get requested reviews
|
|
g.waitAndPickClient(ctx)
|
|
reviewers, resp, err := g.getClient().PullRequests.ListReviewers(ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error while listing repos: %w", err)
|
|
}
|
|
g.setRate(&resp.Rate)
|
|
for _, user := range reviewers.Users {
|
|
r := &base.Review{
|
|
ReviewerID: user.GetID(),
|
|
ReviewerName: user.GetLogin(),
|
|
State: base.ReviewStateRequestReview,
|
|
IssueIndex: reviewable.GetLocalIndex(),
|
|
}
|
|
allReviews = append(allReviews, r)
|
|
}
|
|
// TODO: Handle Team requests
|
|
return allReviews, nil
|
|
}
|
|
|
|
// FormatCloneURL add authentication into remote URLs
|
|
func (g *GithubDownloaderV3) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) {
|
|
opts.AuthToken, _, _ = strings.Cut(opts.AuthToken, ",") // the extra tokens only raise the API rate limit
|
|
return g.NullDownloader.FormatCloneURL(opts, remoteAddr)
|
|
}
|