Files
6624d96689 feat(migrations): GraphQL fast path for GitHub migration + rate-limit resilience (#38694)
Adds a batched GraphQL path for GitHub migrations that fetches each
issue or pull request together with its comments, reviews, reactions and
timeline events instead of the REST N+1 pattern, sharply reducing
rate-limit pressure on large repositories.

- GraphQL is used for authenticated migrations with issues or pull
requests and falls back to REST automatically when it fails
- timeline events (close/reopen, lock, rename, label, milestone, merge,
branch deletion, pin, auto-merge) are imported as typed comments showing
their GitHub author
- GitBucket issue migrations no longer stop after the first page,
GitBucket sends no Link header

Fixes: https://github.com/go-gitea/gitea/issues/13243
Fixes: https://github.com/go-gitea/gitea/issues/31856
Signed-off-by: David E. Lovas <david@rexovas.com>
Co-authored-by: bircni <bircni@icloud.com>
Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: Giteabot <teabot@gitea.io>
2026-10-23 02:23:28 +00:00

231 lines
8.3 KiB
Go

// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package migrations
import (
"context"
"fmt"
"slices"
"time"
base "gitea.dev/modules/migration"
)
const graphQLPRFields = "mergedAt isDraft mergeCommit{oid} headRefName headRefOid baseRefName baseRefOid headRepository{name url} headRepositoryOwner{login} baseRepository{name owner{login}}"
var graphQLPRDiscussion = fmt.Sprintf(`
reviews(first:50){%[1]s nodes{
databaseId state body createdAt submittedAt author{%[2]s} commit{oid}
comments(first:50){%[1]s nodes{databaseId body path diffHunk position commit{oid} author{%[2]s} createdAt updatedAt replyTo{databaseId}}}
}}
reviewRequests(first:%[3]d){%[1]s nodes{requestedReviewer{%[4]s}}}`,
gqlPageInfoFields, gqlActorFields, graphQLReviewRequestPageSize, gqlReviewerFields)
var graphQLReviewRequestsSweepQuery = nodeConnQuery("PullRequest", "reviewRequests", "requestedReviewer{"+gqlReviewerFields+"}")
func (g *GithubGraphQLDownloader) graphQLPullRequestsQuery() string {
return g.graphQLContentQuery("pullRequests", graphQLPRFields, graphQLPRDiscussion)
}
type gqlRepoRef struct {
Name string `json:"name"`
URL string `json:"url"`
Owner gqlActor `json:"owner"`
}
type gqlReviewComment struct {
DatabaseID int64 `json:"databaseId"`
Body string `json:"body"`
Path string `json:"path"`
DiffHunk string `json:"diffHunk"`
Position int `json:"position"`
Commit *gqlCommit `json:"commit"`
Author gqlActor `json:"author"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
ReplyTo *struct {
DatabaseID int64 `json:"databaseId"`
} `json:"replyTo"`
}
type gqlReview struct {
DatabaseID int64 `json:"databaseId"`
State string `json:"state"`
Body string `json:"body"`
CreatedAt time.Time `json:"createdAt"`
SubmittedAt *time.Time `json:"submittedAt"`
Author gqlActor `json:"author"`
Commit *gqlCommit `json:"commit"`
Comments gqlConn[gqlReviewComment] `json:"comments"`
}
type gqlReviewRequest struct {
RequestedReviewer *gqlActor `json:"requestedReviewer"`
}
type gqlPullRequest struct {
gqlIssue
MergedAt *time.Time `json:"mergedAt"`
IsDraft bool `json:"isDraft"`
MergeCommit *gqlCommit `json:"mergeCommit"`
HeadRefName string `json:"headRefName"`
HeadRefOID string `json:"headRefOid"`
BaseRefName string `json:"baseRefName"`
BaseRefOID string `json:"baseRefOid"`
HeadRepo *gqlRepoRef `json:"headRepository"`
HeadRepoOwner *gqlActor `json:"headRepositoryOwner"`
BaseRepo *gqlRepoRef `json:"baseRepository"`
Reviews gqlConn[gqlReview] `json:"reviews"`
ReviewRequests gqlConn[gqlReviewRequest] `json:"reviewRequests"`
}
type gqlPullRequestsResponse struct {
Repository struct {
PullRequests gqlConn[gqlPullRequest] `json:"pullRequests"`
} `json:"repository"`
}
func (g *GithubGraphQLDownloader) getPullRequestsGraphQL(ctx context.Context, page, perPage int) ([]*base.PullRequest, bool, error) {
if page <= 1 {
g.prCursor = ""
}
var resp gqlPullRequestsResponse
if err := g.doGraphQLPage(ctx, g.graphQLPullRequestsQuery(), g.prCursor, graphQLPageSize(perPage, githubGraphQLPRPageSize), &resp); err != nil {
return nil, false, err
}
conn := resp.Repository.PullRequests
prefetched := map[string]*gqlPrefetched{}
prs := make([]*base.PullRequest, 0, len(conn.Nodes))
for i := range conn.Nodes {
pr, err := g.convertGraphQLPullRequest(ctx, &conn.Nodes[i], prefetched)
if err != nil {
return nil, false, err
}
_ = CheckAndEnsureSafePR(pr, g.baseURL, g) // SECURITY: like the REST path
prs = append(prs, pr)
}
if err := g.enrichGraphQLPage(ctx, prefetched); err != nil {
return nil, false, err
}
g.prCursor = conn.PageInfo.EndCursor // only now, a failed page is retried from the same cursor
return prs, !conn.PageInfo.HasNextPage, nil
}
func (g *GithubGraphQLDownloader) convertGraphQLPullRequest(ctx context.Context, node *gqlPullRequest, prefetchedByNodeID map[string]*gqlPrefetched) (*base.PullRequest, error) {
issue, err := g.convertGraphQLIssue(ctx, &node.gqlIssue, prefetchedByNodeID)
if err != nil {
return nil, err
}
if prefetched, ok := issue.Context.(*gqlPrefetched); ok {
if prefetched.reviews, err = g.prefetchReviews(ctx, node); err != nil {
return nil, err
}
}
pr := &base.PullRequest{
Number: issue.Number,
Title: issue.Title,
PosterID: issue.PosterID,
PosterName: issue.PosterName,
Content: issue.Content,
Milestone: issue.Milestone,
State: "closed", // REST reports a merged PR as closed
Created: issue.Created,
Updated: issue.Updated,
Closed: issue.Closed,
Labels: issue.Labels,
Assignees: issue.Assignees,
IsLocked: issue.IsLocked,
Reactions: issue.Reactions,
ForeignIndex: issue.ForeignIndex,
Context: issue.Context,
PatchURL: fmt.Sprintf("%s/%s/%s/pull/%d.patch", g.baseURL, g.repoOwner, g.repoName, node.Number),
Merged: node.MergedAt != nil || node.State == "MERGED",
MergedTime: node.MergedAt,
IsDraft: node.IsDraft,
Head: base.PullRequestBranch{Ref: node.HeadRefName, SHA: node.HeadRefOID},
Base: base.PullRequestBranch{Ref: node.BaseRefName, SHA: node.BaseRefOID},
}
if node.State == "OPEN" {
pr.State = "open"
}
if node.MergeCommit != nil {
pr.MergeCommitSHA = node.MergeCommit.OID
}
if node.HeadRepoOwner != nil {
pr.Head.OwnerName = node.HeadRepoOwner.Login
}
if node.HeadRepo != nil {
pr.Head.RepoName = node.HeadRepo.Name
pr.Head.CloneURL = node.HeadRepo.URL + ".git"
}
if node.BaseRepo != nil {
pr.Base.RepoName = node.BaseRepo.Name
pr.Base.OwnerName = node.BaseRepo.Owner.Login
}
return pr, nil
}
func (g *GithubGraphQLDownloader) prefetchReviews(ctx context.Context, node *gqlPullRequest) ([]*base.Review, error) {
if node.Reviews.PageInfo.HasNextPage || slices.ContainsFunc(node.Reviews.Nodes, func(r gqlReview) bool { return r.Comments.PageInfo.HasNextPage }) {
return g.GetReviews(ctx, &base.PullRequest{Number: node.Number, ForeignIndex: node.Number})
}
requests, err := allNodes(ctx, g, graphQLReviewRequestsSweepQuery, node.ID, node.ReviewRequests)
if err != nil {
return nil, err
}
return convertGraphQLReviews(node, requests), nil
}
func convertGraphQLReviews(node *gqlPullRequest, requests []gqlReviewRequest) []*base.Review {
reviews := make([]*base.Review, 0, len(node.Reviews.Nodes)+len(requests))
for _, sourceReview := range node.Reviews.Nodes {
review := &base.Review{
ID: sourceReview.DatabaseID,
IssueIndex: node.Number,
ReviewerID: sourceReview.Author.DatabaseID,
ReviewerName: sourceReview.Author.Login,
Content: sourceReview.Body,
CreatedAt: sourceReview.CreatedAt,
State: sourceReview.State,
}
if sourceReview.SubmittedAt != nil {
review.CreatedAt = *sourceReview.SubmittedAt
}
if sourceReview.Commit != nil {
review.CommitID = sourceReview.Commit.OID
}
for _, sourceComment := range sourceReview.Comments.Nodes {
comment := &base.ReviewComment{
ID: sourceComment.DatabaseID,
Content: sourceComment.Body,
TreePath: sourceComment.Path,
DiffHunk: sourceComment.DiffHunk,
Position: sourceComment.Position,
PosterID: sourceComment.Author.DatabaseID,
CreatedAt: sourceComment.CreatedAt,
UpdatedAt: sourceComment.UpdatedAt,
}
if sourceComment.ReplyTo != nil {
comment.InReplyTo = sourceComment.ReplyTo.DatabaseID
}
if sourceComment.Commit != nil {
comment.CommitID = sourceComment.Commit.OID
}
review.Comments = append(review.Comments, comment)
}
reviews = append(reviews, review)
}
for _, request := range requests {
if reviewer := request.RequestedReviewer; reviewer != nil && reviewer.Login != "" {
reviews = append(reviews, &base.Review{
IssueIndex: node.Number,
ReviewerID: reviewer.DatabaseID,
ReviewerName: reviewer.Login,
State: base.ReviewStateRequestReview,
})
}
}
return reviews
}