feat(migrations): GraphQL fast path for GitHub migration + rate-limit resilience (#38694)

Adds a batched GraphQL path for GitHub migrations that fetches each issue or pull request together with its comments, reviews, reactions and timeline events instead of the REST N+1 pattern, sharply reducing rate-limit pressure on large repositories. - GraphQL is used for authenticated migrations with issues or pull requests and falls back to REST automatically when it fails - timeline events (close/reopen, lock, rename, label, milestone, merge, branch deletion, pin, auto-merge) are imported as typed comments showing their GitHub author - GitBucket issue migrations no longer stop after the first page, GitBucket sends no Link header Fixes: https://github.com/go-gitea/gitea/issues/13243 Fixes: https://github.com/go-gitea/gitea/issues/31856 Signed-off-by: David E. Lovas <david@rexovas.com> Co-authored-by: bircni <bircni@icloud.com> Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: Giteabot <teabot@gitea.io>
2026-11-02 13:44:16 +09:00 · 2026-10-23 02:23:28 +00:00
1 parent c45af635e4
commit 6624d96689
@@ -51,7 +51,15 @@ func (f *GithubDownloaderV3Factory) New(ctx context.Context, opts base.MigrateOp

 	log.Trace("Create github downloader BaseURL: %s %s/%s", baseURL, oldOwner, oldName)

-	return NewGithubDownloaderV3(ctx, baseURL, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName)
+	downloader, err := NewGithubDownloaderV3(ctx, baseURL, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName)
+	if err != nil {
+		return nil, err
+	}
+	// GitHub rejects anonymous GraphQL requests, and comments only ride along with the issue and PR sweeps
+	if (opts.AuthToken != "" || opts.AuthPassword != "") && (opts.Issues || opts.PullRequests) {
+		return newGithubGraphQLDownloader(downloader, !opts.Comments), nil
+	}
+	return downloader, nil
 }

 // GitServiceType returns the type of git service
@@ -63,21 +71,21 @@ func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType {
 // from github via APIv3
 type GithubDownloaderV3 struct {
 	base.NullDownloader
-	clients       []*github.Client
-	baseURL       string
-	repoOwner     string
-	repoName      string
-	userName      string
-	password      string
-	rates         []*github.Rate
-	curClientIdx  int
-	maxPerPage    int
-	SkipReactions bool
-	SkipReviews   bool
-
+	clients         []*github.Client
+	baseURL         string
+	repoOwner       string
+	repoName        string
+	userName        string
+	password        string
+	rates           []*github.Rate
+	curClientIdx    int
+	maxPerPage      int
+	SkipReactions   bool
+	SkipReviews     bool
 	issuePage       int
 	issueCursor     string
 	nextIssueCursor string
+	countPaginated  bool // GitBucket sends no Link header
 }

 // NewGithubDownloaderV3 creates a github Downloader via github v3 API
@@ -460,32 +468,9 @@ func (g *GithubDownloaderV3) GetIssues(ctx context.Context, page, perPage int) (
 			labels = append(labels, convertGithubLabel(l))
 		}

-		// get reactions
-		var reactions []*base.Reaction
-		if !g.SkipReactions {
-			for i := 1; ; i++ {
-				g.waitAndPickClient(ctx)
-				res, resp, err := g.getClient().Reactions.ListIssueReactions(ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListReactionOptions{
-					ListOptions: github.ListOptions{
-						Page:    i,
-						PerPage: perPage,
-					},
-				})
-				if err != nil {
-					return nil, false, err
-				}
-				g.setRate(&resp.Rate)
-				if len(res) == 0 {
-					break
-				}
-				for _, reaction := range res {
-					reactions = append(reactions, &base.Reaction{
-						UserID:   reaction.User.GetID(),
-						UserName: reaction.User.GetLogin(),
-						Content:  reaction.GetContent(),
-					})
-				}
-			}
+		reactions, err := g.getIssueReactions(ctx, issue.GetNumber(), perPage)
+		if err != nil {
+			return nil, false, err
 		}

 		var assignees []string
@@ -514,7 +499,14 @@ func (g *GithubDownloaderV3) GetIssues(ctx context.Context, page, perPage int) (
 	}

 	g.issuePage, g.issueCursor, g.nextIssueCursor = page, opt.After, resp.After
-	return allIssues, resp.NextPage == 0 && resp.After == "", nil
+	return allIssues, g.isLastPage(resp, len(issues), perPage), nil
+}
+
+func (g *GithubDownloaderV3) isLastPage(resp *github.Response, n, perPage int) bool {
+	if g.countPaginated {
+		return n < perPage
+	}
+	return resp.NextPage == 0 && resp.After == ""
 }

 // SupportGetRepoComments return true if it supports get repo comments
@@ -694,83 +686,97 @@ func (g *GithubDownloaderV3) GetPullRequests(ctx context.Context, page, perPage
 	log.Trace("Request get pull requests %d/%d, but in fact get %d", perPage, page, len(prs))
 	g.setRate(&resp.Rate)
 	for _, pr := range prs {
-		labels := make([]*base.Label, 0, len(pr.Labels))
-		for _, l := range pr.Labels {
-			labels = append(labels, convertGithubLabel(l))
+		basePR, err := g.convertGithubPullRequest(ctx, pr, perPage)
+		if err != nil {
+			return nil, false, err
 		}
-
-		// get reactions
-		var reactions []*base.Reaction
-		if !g.SkipReactions {
-			for i := 1; ; i++ {
-				g.waitAndPickClient(ctx)
-				res, resp, err := g.getClient().Reactions.ListIssueReactions(ctx, g.repoOwner, g.repoName, pr.GetNumber(), &github.ListReactionOptions{
-					ListOptions: github.ListOptions{
-						Page:    i,
-						PerPage: perPage,
-					},
-				})
-				if err != nil {
-					return nil, false, err
-				}
-				g.setRate(&resp.Rate)
-				if len(res) == 0 {
-					break
-				}
-				for _, reaction := range res {
-					reactions = append(reactions, &base.Reaction{
-						UserID:   reaction.User.GetID(),
-						UserName: reaction.User.GetLogin(),
-						Content:  reaction.GetContent(),
-					})
-				}
-			}
-		}
-
-		// download patch and saved as tmp file
-		g.waitAndPickClient(ctx)
-
-		allPRs = append(allPRs, &base.PullRequest{
-			Title:          pr.GetTitle(),
-			Number:         int64(pr.GetNumber()),
-			PosterID:       pr.GetUser().GetID(),
-			PosterName:     pr.GetUser().GetLogin(),
-			PosterEmail:    pr.GetUser().GetEmail(),
-			Content:        pr.GetBody(),
-			Milestone:      pr.GetMilestone().GetTitle(),
-			State:          pr.GetState(),
-			Created:        pr.GetCreatedAt().Time,
-			Updated:        pr.GetUpdatedAt().Time,
-			Closed:         pr.ClosedAt.GetTime(),
-			Labels:         labels,
-			Merged:         pr.MergedAt != nil,
-			MergeCommitSHA: pr.GetMergeCommitSHA(),
-			MergedTime:     pr.MergedAt.GetTime(),
-			IsLocked:       pr.ActiveLockReason != nil,
-			Head: base.PullRequestBranch{
-				Ref:       pr.GetHead().GetRef(),
-				SHA:       pr.GetHead().GetSHA(),
-				OwnerName: pr.GetHead().GetUser().GetLogin(),
-				RepoName:  pr.GetHead().GetRepo().GetName(),
-				CloneURL:  pr.GetHead().GetRepo().GetCloneURL(), // see below for SECURITY related issues here
-			},
-			Base: base.PullRequestBranch{
-				Ref:       pr.GetBase().GetRef(),
-				SHA:       pr.GetBase().GetSHA(),
-				RepoName:  pr.GetBase().GetRepo().GetName(),
-				OwnerName: pr.GetBase().GetUser().GetLogin(),
-			},
-			PatchURL:     pr.GetPatchURL(), // see below for SECURITY related issues here
-			Reactions:    reactions,
-			ForeignIndex: int64(*pr.Number),
-			IsDraft:      pr.GetDraft(),
-		})
+		allPRs = append(allPRs, basePR)

 		// SECURITY: Ensure that the PR is safe
-		_ = CheckAndEnsureSafePR(allPRs[len(allPRs)-1], g.baseURL, g)
+		_ = CheckAndEnsureSafePR(basePR, g.baseURL, g)
 	}

-	return allPRs, len(prs) < perPage, nil
+	return allPRs, g.isLastPage(resp, len(prs), perPage), nil
+}
+
+func (g *GithubDownloaderV3) convertGithubPullRequest(ctx context.Context, pr *github.PullRequest, perPage int) (*base.PullRequest, error) {
+	labels := make([]*base.Label, 0, len(pr.Labels))
+	for _, l := range pr.Labels {
+		labels = append(labels, convertGithubLabel(l))
+	}
+
+	reactions, err := g.getIssueReactions(ctx, pr.GetNumber(), perPage)
+	if err != nil {
+		return nil, err
+	}
+
+	g.waitAndPickClient(ctx)
+
+	return &base.PullRequest{
+		Title:          pr.GetTitle(),
+		Number:         int64(pr.GetNumber()),
+		PosterID:       pr.GetUser().GetID(),
+		PosterName:     pr.GetUser().GetLogin(),
+		PosterEmail:    pr.GetUser().GetEmail(),
+		Content:        pr.GetBody(),
+		Milestone:      pr.GetMilestone().GetTitle(),
+		State:          pr.GetState(),
+		Created:        pr.GetCreatedAt().Time,
+		Updated:        pr.GetUpdatedAt().Time,
+		Closed:         pr.ClosedAt.GetTime(),
+		Labels:         labels,
+		Merged:         pr.MergedAt != nil,
+		MergeCommitSHA: pr.GetMergeCommitSHA(),
+		MergedTime:     pr.MergedAt.GetTime(),
+		IsLocked:       pr.GetLocked(),
+		Head: base.PullRequestBranch{
+			Ref:       pr.GetHead().GetRef(),
+			SHA:       pr.GetHead().GetSHA(),
+			OwnerName: pr.GetHead().GetUser().GetLogin(),
+			RepoName:  pr.GetHead().GetRepo().GetName(),
+			CloneURL:  pr.GetHead().GetRepo().GetCloneURL(), // see below for SECURITY related issues here
+		},
+		Base: base.PullRequestBranch{
+			Ref:       pr.GetBase().GetRef(),
+			SHA:       pr.GetBase().GetSHA(),
+			RepoName:  pr.GetBase().GetRepo().GetName(),
+			OwnerName: pr.GetBase().GetUser().GetLogin(),
+		},
+		PatchURL:     pr.GetPatchURL(), // see below for SECURITY related issues here
+		Reactions:    reactions,
+		ForeignIndex: int64(*pr.Number),
+		IsDraft:      pr.GetDraft(),
+	}, nil
+}
+
+func (g *GithubDownloaderV3) getIssueReactions(ctx context.Context, number, perPage int) ([]*base.Reaction, error) {
+	if g.SkipReactions {
+		return nil, nil
+	}
+	var reactions []*base.Reaction
+	for i := 1; ; i++ {
+		g.waitAndPickClient(ctx)
+		res, resp, err := g.getClient().Reactions.ListIssueReactions(ctx, g.repoOwner, g.repoName, number, &github.ListReactionOptions{
+			ListOptions: github.ListOptions{
+				Page:    i,
+				PerPage: perPage,
+			},
+		})
+		if err != nil {
+			return nil, err
+		}
+		g.setRate(&resp.Rate)
+		if len(res) == 0 {
+			return reactions, nil
+		}
+		for _, reaction := range res {
+			reactions = append(reactions, &base.Reaction{
+				UserID:   reaction.User.GetID(),
+				UserName: reaction.User.GetLogin(),
+				Content:  reaction.GetContent(),
+			})
+		}
+	}
 }

 func convertGithubReview(r *github.PullRequestReview) *base.Review {