feat(migrations): GraphQL fast path for GitHub migration + rate-limit resilience (#38694)

Adds a batched GraphQL path for GitHub migrations that fetches each
issue or pull request together with its comments, reviews, reactions and
timeline events instead of the REST N+1 pattern, sharply reducing
rate-limit pressure on large repositories.

- GraphQL is used for authenticated migrations with issues or pull
requests and falls back to REST automatically when it fails
- timeline events (close/reopen, lock, rename, label, milestone, merge,
branch deletion, pin, auto-merge) are imported as typed comments showing
their GitHub author
- GitBucket issue migrations no longer stop after the first page,
GitBucket sends no Link header

Fixes: https://github.com/go-gitea/gitea/issues/13243
Fixes: https://github.com/go-gitea/gitea/issues/31856
Signed-off-by: David E. Lovas <david@rexovas.com>
Co-authored-by: bircni <bircni@icloud.com>
Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: Giteabot <teabot@gitea.io>
This commit is contained in:
authored and GitHub committed 2026-10-23 02:23:28 +00:00
1 parent c45af635e4
commit 6624d96689
19 files changed
+2115 -141

No files matched your search

-1
View File
@@ -9,7 +9,6 @@ import "time"
// Commentable can be commented upon // Commentable can be commented upon
type Commentable interface { type Commentable interface {
Reviewable Reviewable
GetContext() DownloaderContext
} }
// Comment is a standard comment information // Comment is a standard comment information
+1
View File
@@ -8,6 +8,7 @@ import "time"
// Reviewable can be reviewed // Reviewable can be reviewed
type Reviewable interface { type Reviewable interface {
GetLocalIndex() int64 GetLocalIndex() int64
GetContext() DownloaderContext
// GetForeignIndex presents the foreign index, which could be misused: // GetForeignIndex presents the foreign index, which could be misused:
// For example, if there are 2 Gitea sites: site-A exports a dataset, then site-B imports it: // For example, if there are 2 Gitea sites: site-A exports a dataset, then site-B imports it:
+13
View File
@@ -218,6 +218,19 @@ func TestCombineLabelComments(t *testing.T) {
}, },
}, },
}, },
{
name: "migrated events of different original authors",
beforeCombined: []*issues_model.Comment{
{Type: issues_model.CommentTypeComment, PosterID: 1},
{Type: issues_model.CommentTypeLabel, PosterID: 1, OriginalAuthor: "alice", Content: "1", Label: &issues_model.Label{Name: "kind/bug"}},
{Type: issues_model.CommentTypeLabel, PosterID: 1, OriginalAuthor: "bob", Content: "1", Label: &issues_model.Label{Name: "kind/ui"}},
},
afterCombined: []*issues_model.Comment{
{Type: issues_model.CommentTypeComment, PosterID: 1},
{Type: issues_model.CommentTypeLabel, PosterID: 1, OriginalAuthor: "alice", Content: "1", Label: &issues_model.Label{Name: "kind/bug"}, AddedLabels: []*issues_model.Label{{Name: "kind/bug"}}},
{Type: issues_model.CommentTypeLabel, PosterID: 1, OriginalAuthor: "bob", Content: "1", Label: &issues_model.Label{Name: "kind/ui"}, AddedLabels: []*issues_model.Label{{Name: "kind/ui"}}},
},
},
{ {
name: "kase 4", name: "kase 4",
beforeCombined: []*issues_model.Comment{ beforeCombined: []*issues_model.Comment{
+1 -1
View File
@@ -231,7 +231,7 @@ func combineLabelComments(issue *issues_model.Issue) {
prev = issue.Comments[i-1] prev = issue.Comments[i-1]
} }
if i == 0 || cur.Type != issues_model.CommentTypeLabel || if i == 0 || cur.Type != issues_model.CommentTypeLabel ||
(prev != nil && prev.PosterID != cur.PosterID) || (prev != nil && (prev.PosterID != cur.PosterID || prev.OriginalAuthor != cur.OriginalAuthor)) ||
(prev != nil && cur.CreatedUnix-prev.CreatedUnix >= 60) || (prev != nil && cur.CreatedUnix-prev.CreatedUnix >= 60) ||
(prev != nil && !sameCommentMetaData(prev.CommentMetaData, cur.CommentMetaData)) { (prev != nil && !sameCommentMetaData(prev.CommentMetaData, cur.CommentMetaData)) {
if cur.Type == issues_model.CommentTypeLabel && cur.Label != nil { if cur.Type == issues_model.CommentTypeLabel && cur.Label != nil {
@@ -0,0 +1,9 @@
Content-Type: application/json; charset=utf-8
X-Ratelimit-Limit: 5000
X-Ratelimit-Remaining: 4931
X-Ratelimit-Reset: 1773089427
X-Ratelimit-Resource: core
X-Ratelimit-Used: 69
Link: <https://api.github.com/repositories/220672974/issues?direction=asc&page=2&per_page=3&sort=created&state=all>; rel="next", <https://api.github.com/repositories/220672974/issues?direction=asc&page=2&per_page=3&sort=created&state=all>; rel="last"
[{"number":1,"title":"one"},{"number":2,"title":"two"},{"number":3,"title":"three"}]
@@ -0,0 +1,8 @@
Content-Type: application/json; charset=utf-8
X-Ratelimit-Limit: 5000
X-Ratelimit-Remaining: 4931
X-Ratelimit-Reset: 1773089427
X-Ratelimit-Resource: core
X-Ratelimit-Used: 69
[]
+1
View File
@@ -82,6 +82,7 @@ func NewGitBucketDownloader(ctx context.Context, baseURL, userName, password, to
githubDownloader.maxPerPage = 10 githubDownloader.maxPerPage = 10
githubDownloader.SkipReactions = true githubDownloader.SkipReactions = true
githubDownloader.SkipReviews = true githubDownloader.SkipReviews = true
githubDownloader.countPaginated = true
return &GitBucketDownloader{ return &GitBucketDownloader{
githubDownloader, githubDownloader,
}, nil }, nil
+23
View File
@@ -517,6 +517,29 @@ func (g *GiteaLocalUploader) CreateComments(ctx context.Context, comments ...*ba
cm.NewRef = fmt.Sprint(comment.Meta["NewRef"]) cm.NewRef = fmt.Sprint(comment.Meta["NewRef"])
cm.Content = "" cm.Content = ""
} }
case issues_model.CommentTypeLabel:
if name, ok := comment.Meta["LabelName"].(string); ok {
if label, ok := g.labels[name]; ok {
cm.LabelID = label.ID
}
}
if cm.LabelID == 0 {
continue // label_id 0 renders nothing and gitea doctor deletes it
}
case issues_model.CommentTypeMilestone:
if title, ok := comment.Meta["MilestoneTitle"].(string); ok {
if removed, _ := comment.Meta["Removed"].(bool); removed {
cm.OldMilestoneID = g.milestones[title]
} else {
cm.MilestoneID = g.milestones[title]
}
}
if cm.MilestoneID == 0 && cm.OldMilestoneID == 0 {
continue
}
case issues_model.CommentTypeDeleteBranch:
cm.OldRef = comment.Content
cm.Content = ""
case issues_model.CommentTypeMergePull: case issues_model.CommentTypeMergePull:
cm.Content = "" cm.Content = ""
case issues_model.CommentTypePRScheduledToAutoMerge, issues_model.CommentTypePRUnScheduledToAutoMerge: case issues_model.CommentTypePRScheduledToAutoMerge, issues_model.CommentTypePRUnScheduledToAutoMerge:
@@ -22,6 +22,7 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"xorm.io/builder"
) )
func TestGiteaUploadRepo(t *testing.T) { func TestGiteaUploadRepo(t *testing.T) {
@@ -222,3 +223,35 @@ func TestGiteaUploadRemapExternalUser(t *testing.T) {
assert.NoError(t, err) assert.NoError(t, err)
assert.Equal(t, linkedUser.ID, target.GetUserID()) assert.Equal(t, linkedUser.ID, target.GetUserID())
} }
func TestGiteaUploadTimelineLabelAndMilestoneComments(t *testing.T) {
unittest.PrepareTestEnv(t)
doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
issue := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: 1})
label := unittest.AssertExistsAndLoadBean(t, &issues_model.Label{ID: 1})
milestone := unittest.AssertExistsAndLoadBean(t, &issues_model.Milestone{ID: 1})
ctx := t.Context()
uploader := NewGiteaLocalUploader(ctx, doer, doer.Name, "migrated")
uploader.issues[issue.Index] = issue
uploader.labels[label.Name] = label
uploader.milestones[milestone.Name] = milestone.ID
require.NoError(t, uploader.CreateComments(ctx,
&base.Comment{IssueIndex: issue.Index, CommentType: "label", Content: "1", Meta: map[string]any{"LabelName": label.Name}},
&base.Comment{IssueIndex: issue.Index, CommentType: "label", Meta: map[string]any{"LabelName": "deleted-upstream"}},
&base.Comment{IssueIndex: issue.Index, CommentType: "milestone", Meta: map[string]any{"MilestoneTitle": milestone.Name}},
&base.Comment{IssueIndex: issue.Index, CommentType: "milestone", Meta: map[string]any{"MilestoneTitle": milestone.Name, "Removed": true}},
&base.Comment{IssueIndex: issue.Index, CommentType: "delete_branch", Content: "feature"},
))
unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{IssueID: issue.ID, Type: issues_model.CommentTypeLabel, LabelID: label.ID, Content: "1"})
unittest.AssertCountByCond(t, "comment", builder.Eq{
"issue_id": issue.ID,
"type": issues_model.CommentTypeLabel,
"label_id": 0,
}, 0)
unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{IssueID: issue.ID, Type: issues_model.CommentTypeMilestone, MilestoneID: milestone.ID})
unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{IssueID: issue.ID, Type: issues_model.CommentTypeMilestone, OldMilestoneID: milestone.ID})
assert.Empty(t, unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{IssueID: issue.ID, Type: issues_model.CommentTypeDeleteBranch, OldRef: "feature"}).Content)
}
+118 -112
View File
@@ -51,7 +51,15 @@ func (f *GithubDownloaderV3Factory) New(ctx context.Context, opts base.MigrateOp
log.Trace("Create github downloader BaseURL: %s %s/%s", baseURL, oldOwner, oldName) log.Trace("Create github downloader BaseURL: %s %s/%s", baseURL, oldOwner, oldName)
return NewGithubDownloaderV3(ctx, baseURL, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName) downloader, err := NewGithubDownloaderV3(ctx, baseURL, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName)
if err != nil {
return nil, err
}
// GitHub rejects anonymous GraphQL requests, and comments only ride along with the issue and PR sweeps
if (opts.AuthToken != "" || opts.AuthPassword != "") && (opts.Issues || opts.PullRequests) {
return newGithubGraphQLDownloader(downloader, !opts.Comments), nil
}
return downloader, nil
} }
// GitServiceType returns the type of git service // GitServiceType returns the type of git service
@@ -63,21 +71,21 @@ func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType {
// from github via APIv3 // from github via APIv3
type GithubDownloaderV3 struct { type GithubDownloaderV3 struct {
base.NullDownloader base.NullDownloader
clients []*github.Client clients []*github.Client
baseURL string baseURL string
repoOwner string repoOwner string
repoName string repoName string
userName string userName string
password string password string
rates []*github.Rate rates []*github.Rate
curClientIdx int curClientIdx int
maxPerPage int maxPerPage int
SkipReactions bool SkipReactions bool
SkipReviews bool SkipReviews bool
issuePage int issuePage int
issueCursor string issueCursor string
nextIssueCursor string nextIssueCursor string
countPaginated bool // GitBucket sends no Link header
} }
// NewGithubDownloaderV3 creates a github Downloader via github v3 API // NewGithubDownloaderV3 creates a github Downloader via github v3 API
@@ -460,32 +468,9 @@ func (g *GithubDownloaderV3) GetIssues(ctx context.Context, page, perPage int) (
labels = append(labels, convertGithubLabel(l)) labels = append(labels, convertGithubLabel(l))
} }
// get reactions reactions, err := g.getIssueReactions(ctx, issue.GetNumber(), perPage)
var reactions []*base.Reaction if err != nil {
if !g.SkipReactions { return nil, false, err
for i := 1; ; i++ {
g.waitAndPickClient(ctx)
res, resp, err := g.getClient().Reactions.ListIssueReactions(ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListReactionOptions{
ListOptions: github.ListOptions{
Page: i,
PerPage: perPage,
},
})
if err != nil {
return nil, false, err
}
g.setRate(&resp.Rate)
if len(res) == 0 {
break
}
for _, reaction := range res {
reactions = append(reactions, &base.Reaction{
UserID: reaction.User.GetID(),
UserName: reaction.User.GetLogin(),
Content: reaction.GetContent(),
})
}
}
} }
var assignees []string var assignees []string
@@ -514,7 +499,14 @@ func (g *GithubDownloaderV3) GetIssues(ctx context.Context, page, perPage int) (
} }
g.issuePage, g.issueCursor, g.nextIssueCursor = page, opt.After, resp.After g.issuePage, g.issueCursor, g.nextIssueCursor = page, opt.After, resp.After
return allIssues, resp.NextPage == 0 && resp.After == "", nil return allIssues, g.isLastPage(resp, len(issues), perPage), nil
}
func (g *GithubDownloaderV3) isLastPage(resp *github.Response, n, perPage int) bool {
if g.countPaginated {
return n < perPage
}
return resp.NextPage == 0 && resp.After == ""
} }
// SupportGetRepoComments return true if it supports get repo comments // SupportGetRepoComments return true if it supports get repo comments
@@ -694,83 +686,97 @@ func (g *GithubDownloaderV3) GetPullRequests(ctx context.Context, page, perPage
log.Trace("Request get pull requests %d/%d, but in fact get %d", perPage, page, len(prs)) log.Trace("Request get pull requests %d/%d, but in fact get %d", perPage, page, len(prs))
g.setRate(&resp.Rate) g.setRate(&resp.Rate)
for _, pr := range prs { for _, pr := range prs {
labels := make([]*base.Label, 0, len(pr.Labels)) basePR, err := g.convertGithubPullRequest(ctx, pr, perPage)
for _, l := range pr.Labels { if err != nil {
labels = append(labels, convertGithubLabel(l)) return nil, false, err
} }
allPRs = append(allPRs, basePR)
// get reactions
var reactions []*base.Reaction
if !g.SkipReactions {
for i := 1; ; i++ {
g.waitAndPickClient(ctx)
res, resp, err := g.getClient().Reactions.ListIssueReactions(ctx, g.repoOwner, g.repoName, pr.GetNumber(), &github.ListReactionOptions{
ListOptions: github.ListOptions{
Page: i,
PerPage: perPage,
},
})
if err != nil {
return nil, false, err
}
g.setRate(&resp.Rate)
if len(res) == 0 {
break
}
for _, reaction := range res {
reactions = append(reactions, &base.Reaction{
UserID: reaction.User.GetID(),
UserName: reaction.User.GetLogin(),
Content: reaction.GetContent(),
})
}
}
}
// download patch and saved as tmp file
g.waitAndPickClient(ctx)
allPRs = append(allPRs, &base.PullRequest{
Title: pr.GetTitle(),
Number: int64(pr.GetNumber()),
PosterID: pr.GetUser().GetID(),
PosterName: pr.GetUser().GetLogin(),
PosterEmail: pr.GetUser().GetEmail(),
Content: pr.GetBody(),
Milestone: pr.GetMilestone().GetTitle(),
State: pr.GetState(),
Created: pr.GetCreatedAt().Time,
Updated: pr.GetUpdatedAt().Time,
Closed: pr.ClosedAt.GetTime(),
Labels: labels,
Merged: pr.MergedAt != nil,
MergeCommitSHA: pr.GetMergeCommitSHA(),
MergedTime: pr.MergedAt.GetTime(),
IsLocked: pr.ActiveLockReason != nil,
Head: base.PullRequestBranch{
Ref: pr.GetHead().GetRef(),
SHA: pr.GetHead().GetSHA(),
OwnerName: pr.GetHead().GetUser().GetLogin(),
RepoName: pr.GetHead().GetRepo().GetName(),
CloneURL: pr.GetHead().GetRepo().GetCloneURL(), // see below for SECURITY related issues here
},
Base: base.PullRequestBranch{
Ref: pr.GetBase().GetRef(),
SHA: pr.GetBase().GetSHA(),
RepoName: pr.GetBase().GetRepo().GetName(),
OwnerName: pr.GetBase().GetUser().GetLogin(),
},
PatchURL: pr.GetPatchURL(), // see below for SECURITY related issues here
Reactions: reactions,
ForeignIndex: int64(*pr.Number),
IsDraft: pr.GetDraft(),
})
// SECURITY: Ensure that the PR is safe // SECURITY: Ensure that the PR is safe
_ = CheckAndEnsureSafePR(allPRs[len(allPRs)-1], g.baseURL, g) _ = CheckAndEnsureSafePR(basePR, g.baseURL, g)
} }
return allPRs, len(prs) < perPage, nil return allPRs, g.isLastPage(resp, len(prs), perPage), nil
}
func (g *GithubDownloaderV3) convertGithubPullRequest(ctx context.Context, pr *github.PullRequest, perPage int) (*base.PullRequest, error) {
labels := make([]*base.Label, 0, len(pr.Labels))
for _, l := range pr.Labels {
labels = append(labels, convertGithubLabel(l))
}
reactions, err := g.getIssueReactions(ctx, pr.GetNumber(), perPage)
if err != nil {
return nil, err
}
g.waitAndPickClient(ctx)
return &base.PullRequest{
Title: pr.GetTitle(),
Number: int64(pr.GetNumber()),
PosterID: pr.GetUser().GetID(),
PosterName: pr.GetUser().GetLogin(),
PosterEmail: pr.GetUser().GetEmail(),
Content: pr.GetBody(),
Milestone: pr.GetMilestone().GetTitle(),
State: pr.GetState(),
Created: pr.GetCreatedAt().Time,
Updated: pr.GetUpdatedAt().Time,
Closed: pr.ClosedAt.GetTime(),
Labels: labels,
Merged: pr.MergedAt != nil,
MergeCommitSHA: pr.GetMergeCommitSHA(),
MergedTime: pr.MergedAt.GetTime(),
IsLocked: pr.GetLocked(),
Head: base.PullRequestBranch{
Ref: pr.GetHead().GetRef(),
SHA: pr.GetHead().GetSHA(),
OwnerName: pr.GetHead().GetUser().GetLogin(),
RepoName: pr.GetHead().GetRepo().GetName(),
CloneURL: pr.GetHead().GetRepo().GetCloneURL(), // see below for SECURITY related issues here
},
Base: base.PullRequestBranch{
Ref: pr.GetBase().GetRef(),
SHA: pr.GetBase().GetSHA(),
RepoName: pr.GetBase().GetRepo().GetName(),
OwnerName: pr.GetBase().GetUser().GetLogin(),
},
PatchURL: pr.GetPatchURL(), // see below for SECURITY related issues here
Reactions: reactions,
ForeignIndex: int64(*pr.Number),
IsDraft: pr.GetDraft(),
}, nil
}
func (g *GithubDownloaderV3) getIssueReactions(ctx context.Context, number, perPage int) ([]*base.Reaction, error) {
if g.SkipReactions {
return nil, nil
}
var reactions []*base.Reaction
for i := 1; ; i++ {
g.waitAndPickClient(ctx)
res, resp, err := g.getClient().Reactions.ListIssueReactions(ctx, g.repoOwner, g.repoName, number, &github.ListReactionOptions{
ListOptions: github.ListOptions{
Page: i,
PerPage: perPage,
},
})
if err != nil {
return nil, err
}
g.setRate(&resp.Rate)
if len(res) == 0 {
return reactions, nil
}
for _, reaction := range res {
reactions = append(reactions, &base.Reaction{
UserID: reaction.User.GetID(),
UserName: reaction.User.GetLogin(),
Content: reaction.GetContent(),
})
}
}
} }
func convertGithubReview(r *github.PullRequestReview) *base.Review { func convertGithubReview(r *github.PullRequestReview) *base.Review {
+716
View File
@@ -0,0 +1,716 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package migrations
import (
"bytes"
"context"
"crypto/tls"
"errors"
"fmt"
"io"
"maps"
"net"
"net/http"
"slices"
"strconv"
"strings"
"time"
"gitea.dev/modules/egress/policy"
"gitea.dev/modules/json"
"gitea.dev/modules/log"
base "gitea.dev/modules/migration"
"github.com/google/go-github/v92/github"
)
const (
githubGraphQLPageSize = 100
githubGraphQLPRPageSize = 20
githubGraphQLPointsFloor = 10
maxGraphQLRateRetries = 5
maxGraphQLTransientRetries = 5
graphQLRateResetCap = time.Hour
graphQLNodeBatchSize = 100
graphQLLabelPageSize = 30
graphQLAssigneePageSize = 30
graphQLReviewRequestPageSize = 50
)
const (
gqlPageInfoFields = "pageInfo{hasNextPage endCursor}"
// databaseId is not on the Actor interface, every implementation needs its own fragment
gqlActorFields = "__typename login ... on User{databaseId} ... on Bot{databaseId} ... on Organization{databaseId} ... on Mannequin{databaseId}"
// teams in the RequestedReviewer union have no login
gqlReviewerFields = "__typename ... on User{login databaseId} ... on Bot{login databaseId} ... on Mannequin{login databaseId}"
gqlReactionFields = "content user{login databaseId}"
)
var graphQLTransientRetryBase = 2 * time.Second
var graphQLRateResetFallback = time.Minute // GitHub requires a minute between secondary limit retries.
// a deterministic 502 for a too-heavy page never succeeds at the same size, so callers shrink it
var errGraphQLTransientExhausted = errors.New("github graphql: transient retries exhausted")
var _ base.Downloader = &GithubGraphQLDownloader{}
// GithubGraphQLDownloader sweeps issues and pull requests over GraphQL, the embedded REST downloader serves the rest and takes over on failure
type GithubGraphQLDownloader struct {
*GithubDownloaderV3
gqlRates []*github.Rate // GraphQL has a separate rate budget
skipComments bool
issuesCursor string
prCursor string
restFallback bool
// REST restarts from its first page after a fallback, the migration skips what it already imported
restIssuePageOffset int
restPRPageOffset int
}
func newGithubGraphQLDownloader(rest *GithubDownloaderV3, skipComments bool) *GithubGraphQLDownloader {
return &GithubGraphQLDownloader{GithubDownloaderV3: rest, gqlRates: make([]*github.Rate, len(rest.clients)), skipComments: skipComments}
}
func (g *GithubGraphQLDownloader) GetIssues(ctx context.Context, page, perPage int) ([]*base.Issue, bool, error) {
if !g.restFallback {
issues, isEnd, err := g.getIssuesGraphQL(ctx, page, perPage)
if !g.fallBackToREST(ctx, err) {
return issues, isEnd, err
}
g.restIssuePageOffset = page - 1
}
return g.GithubDownloaderV3.GetIssues(ctx, page-g.restIssuePageOffset, perPage)
}
func (g *GithubGraphQLDownloader) GetPullRequests(ctx context.Context, page, perPage int) ([]*base.PullRequest, bool, error) {
if !g.restFallback {
prs, isEnd, err := g.getPullRequestsGraphQL(ctx, page, perPage)
if !g.fallBackToREST(ctx, err) {
return prs, isEnd, err
}
g.restPRPageOffset = page - 1
}
return g.GithubDownloaderV3.GetPullRequests(ctx, page-g.restPRPageOffset, perPage)
}
// fallBackToREST switches to REST for good when GraphQL fails, e.g. on a server or token without GraphQL support
func (g *GithubGraphQLDownloader) fallBackToREST(ctx context.Context, err error) bool {
if err == nil || ctx.Err() != nil {
return false
}
log.Warn("GitHub GraphQL failed, falling back to REST: %v", err)
g.restFallback = true
return true
}
// SupportGetRepoComments is false so comments are served per issue from what each sweep page prefetched
func (g *GithubGraphQLDownloader) SupportGetRepoComments() bool {
return false
}
func (g *GithubGraphQLDownloader) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) {
if prefetched, ok := commentable.GetContext().(*gqlPrefetched); ok {
return prefetched.comments, false, nil
}
return g.GithubDownloaderV3.GetComments(ctx, commentable)
}
func (g *GithubGraphQLDownloader) GetReviews(ctx context.Context, reviewable base.Reviewable) ([]*base.Review, error) {
if prefetched, ok := reviewable.GetContext().(*gqlPrefetched); ok {
return prefetched.reviews, nil
}
return g.GithubDownloaderV3.GetReviews(ctx, reviewable)
}
type graphQLRequest struct {
Query string `json:"query"`
Variables map[string]any `json:"variables"`
}
type graphQLError struct {
Type string `json:"type"`
Message string `json:"message"`
Path []any `json:"path"`
}
func (e graphQLError) Error() string {
if e.Type != "" {
return e.Type + ": " + e.Message
}
return e.Message
}
func (g *GithubGraphQLDownloader) graphQLEndpoint() string {
restURL := g.clients[0].BaseURL()
if host, ok := strings.CutSuffix(restURL, "/api/v3/"); ok {
return host + "/api/graphql"
}
return restURL + "graphql"
}
func (g *GithubGraphQLDownloader) doGraphQL(ctx context.Context, query string, vars map[string]any, out any) error {
payload, err := json.Marshal(graphQLRequest{Query: query, Variables: vars})
if err != nil {
return err
}
rateWaits, transientRetries := 0, 0
for {
if err := g.pickGraphQLClient(ctx); err != nil {
return err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, g.graphQLEndpoint(), bytes.NewReader(payload))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json")
resp, err := g.getClient().Client().Do(req)
if err != nil {
if ctx.Err() != nil || !isTransientTransportError(err) {
return err
}
if err := g.waitGraphQLTransient(ctx, &transientRetries, fmt.Sprintf("request failed: %v", err)); err != nil {
return err
}
continue
}
g.recordGraphQLRate(resp.Header)
if resp.StatusCode != http.StatusOK {
detail := readGraphQLErrorBody(resp.Body)
resp.Body.Close()
switch {
case resp.StatusCode >= http.StatusInternalServerError:
if err := g.waitGraphQLTransient(ctx, &transientRetries, "status "+resp.Status+detail); err != nil {
return err
}
continue
case isGraphQLThrottled(resp, detail):
if rateWaits < maxGraphQLRateRetries*len(g.clients) {
rateWaits++
g.restGraphQLClient(resp.Header)
continue
}
return fmt.Errorf("doGraphQL: rate limited, status %s%s", resp.Status, detail)
default:
return fmt.Errorf("doGraphQL: unexpected status %s%s", resp.Status, detail)
}
}
envelope := struct {
Data any `json:"data"`
Errors []graphQLError `json:"errors"`
}{Data: out}
decodeErr := json.NewDecoder(resp.Body).Decode(&envelope)
resp.Body.Close()
if decodeErr != nil {
if err := g.waitGraphQLTransient(ctx, &transientRetries, fmt.Sprintf("truncated response: %v", decodeErr)); err != nil {
return err
}
continue
}
if !onlyUnresolvedNodes(envelope.Errors) {
if isGraphQLRateLimited(envelope.Errors) && rateWaits < maxGraphQLRateRetries*len(g.clients) {
rateWaits++
g.restGraphQLClient(resp.Header)
continue
}
return fmt.Errorf("doGraphQL: %w", envelope.Errors[0])
}
return nil
}
}
const maxGraphQLErrorBody = 512
func readGraphQLErrorBody(body io.Reader) string {
buf, err := io.ReadAll(io.LimitReader(body, maxGraphQLErrorBody))
if err != nil || len(buf) == 0 {
return ""
}
return ": " + strings.Join(strings.Fields(string(buf)), " ")
}
func isGraphQLThrottled(resp *http.Response, body string) bool {
switch resp.StatusCode {
case http.StatusTooManyRequests:
return true
case http.StatusForbidden:
return resp.Header.Get("Retry-After") != "" ||
resp.Header.Get(github.HeaderRateRemaining) == "0" ||
strings.Contains(strings.ToLower(body), "secondary rate limit")
}
return false
}
func isTransientTransportError(err error) bool {
if errors.Is(err, policy.ErrDenied) {
return false
}
if _, ok := errors.AsType[*tls.CertificateVerificationError](err); ok {
return false
}
if dnsErr, ok := errors.AsType[*net.DNSError](err); ok {
return !dnsErr.IsNotFound
}
return true
}
func (g *GithubGraphQLDownloader) waitGraphQLTransient(ctx context.Context, attempt *int, reason string) error {
if *attempt >= maxGraphQLTransientRetries {
return fmt.Errorf("%w: %s", errGraphQLTransientExhausted, reason)
}
wait := graphQLTransientRetryBase << *attempt
*attempt++
log.Warn("github graphql: transient failure (%s), retrying in %s (attempt %d/%d)",
reason, wait, *attempt, maxGraphQLTransientRetries)
timer := time.NewTimer(wait)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return nil
}
}
func onlyUnresolvedNodes(errs []graphQLError) bool {
return !slices.ContainsFunc(errs, func(e graphQLError) bool {
return e.Type != "NOT_FOUND" || len(e.Path) == 0 || e.Path[0] != "nodes"
})
}
func isGraphQLRateLimited(errs []graphQLError) bool {
return slices.ContainsFunc(errs, func(e graphQLError) bool { return e.Type == "RATE_LIMITED" })
}
func isGraphQLTooHeavy(err error) bool {
var ge graphQLError
return errors.Is(err, errGraphQLTransientExhausted) || errors.As(err, &ge) && ge.Type == "RESOURCE_LIMITS_EXCEEDED"
}
func (g *GithubGraphQLDownloader) pickGraphQLClient(ctx context.Context) error {
for {
best, soonest := -1, -1
for i, rate := range g.gqlRates {
if rate == nil || !time.Now().Before(rate.Reset.Time) {
g.curClientIdx = i
return nil
}
if rate.Remaining > githubGraphQLPointsFloor && (best < 0 || rate.Remaining > g.gqlRates[best].Remaining) {
best = i
}
if soonest < 0 || rate.Reset.Before(g.gqlRates[soonest].Reset.Time) {
soonest = i
}
}
if best >= 0 {
g.curClientIdx = best
return nil
}
wait := min(time.Until(g.gqlRates[soonest].Reset.Time), graphQLRateResetCap)
log.Info("github graphql: every token's points budget is spent, sleeping %s until the earliest reset", wait.Round(time.Second))
timer := time.NewTimer(wait)
select {
case <-ctx.Done():
timer.Stop()
return ctx.Err()
case <-timer.C:
g.gqlRates[soonest] = nil // the reset may lie beyond the capped sleep
}
}
}
func (g *GithubGraphQLDownloader) recordGraphQLRate(header http.Header) {
remaining, err := strconv.Atoi(header.Get(github.HeaderRateRemaining))
if err != nil {
return
}
rate := &github.Rate{Remaining: remaining}
if epoch, err := strconv.ParseInt(header.Get(github.HeaderRateReset), 10, 64); err == nil {
rate.Reset = github.Timestamp{Time: time.Unix(epoch, 0)}
}
g.gqlRates[g.curClientIdx] = rate
}
func (g *GithubGraphQLDownloader) restGraphQLClient(header http.Header) {
wait := graphQLRateResetWait(header)
log.Info("github graphql: token %d throttled, resting it for %s", g.curClientIdx, wait.Round(time.Second))
g.gqlRates[g.curClientIdx] = &github.Rate{Reset: github.Timestamp{Time: time.Now().Add(wait)}}
}
func isPrimaryBudgetSpent(header http.Header) bool {
remaining := header.Get(github.HeaderRateRemaining)
return remaining == "" || remaining == "0"
}
func graphQLRateResetWait(header http.Header) time.Duration {
wait := graphQLRateResetFallback
if ra := header.Get("Retry-After"); ra != "" {
if secs, err := strconv.Atoi(strings.TrimSpace(ra)); err == nil && secs > 0 {
wait = time.Duration(secs) * time.Second
}
} else if reset := header.Get(github.HeaderRateReset); reset != "" && isPrimaryBudgetSpent(header) {
if epoch, err := strconv.ParseInt(reset, 10, 64); err == nil {
if d := time.Until(time.Unix(epoch, 0)); d > 0 {
wait = d
}
}
}
return min(wait, graphQLRateResetCap)
}
// graphQLContentQuery orders by creation since a mutable sort key skips rows, nested comment reactions exceed GitHub's node limit
func (g *GithubGraphQLDownloader) graphQLContentQuery(connection, prFields, prDiscussion string) string {
fields := prFields
if !g.SkipReactions {
fields += " reactions(first:100){" + gqlPageInfoFields + " nodes{" + gqlReactionFields + "}}"
}
if !g.skipComments {
fields += " comments(first:100){" + gqlPageInfoFields + " nodes{id databaseId body createdAt updatedAt author{" + gqlActorFields + "}}}" + prDiscussion
}
return fmt.Sprintf(`
query($owner:String!,$name:String!,$cursor:String,$first:Int!){
repository(owner:$owner,name:$name){
%[1]s(first:$first,after:$cursor,orderBy:{field:CREATED_AT,direction:ASC}){
%[2]s
nodes{
id number title body state createdAt updatedAt closedAt locked
author{%[3]s}
milestone{title}
labels(first:%[4]d){%[2]s nodes{name color description}}
assignees(first:%[5]d){%[2]s nodes{login}}
%[6]s
}
}
}
}`, connection, gqlPageInfoFields, gqlActorFields, graphQLLabelPageSize, graphQLAssigneePageSize, fields)
}
func (g *GithubGraphQLDownloader) graphQLIssuesQuery() string {
return g.graphQLContentQuery("issues", "", "")
}
func nodeConnQuery(onType, connection, nodeFields string) string {
return `query($id:ID!,$cursor:String){node(id:$id){... on ` + onType + `{conn:` + connection +
`(first:100,after:$cursor){` + gqlPageInfoFields + ` nodes{` + nodeFields + `}}}}}`
}
var (
graphQLLabelsSweepQuery = nodeConnQuery("Labelable", "labels", "name color description")
graphQLAssigneesSweepQuery = nodeConnQuery("Assignable", "assignees", "login")
graphQLReactionsSweepQuery = nodeConnQuery("Reactable", "reactions", gqlReactionFields)
)
const graphQLNodeReactionsQuery = `query($ids:[ID!]!){nodes(ids:$ids){id ... on Reactable{reactions(first:100){` +
gqlPageInfoFields + ` nodes{` + gqlReactionFields + `}}}}}`
type gqlPageInfo struct {
HasNextPage bool `json:"hasNextPage"`
EndCursor string `json:"endCursor"`
}
type gqlConn[T any] struct {
PageInfo gqlPageInfo `json:"pageInfo"`
Nodes []T `json:"nodes"`
}
type gqlActor struct {
Typename string `json:"__typename"`
Login string `json:"login"`
DatabaseID int64 `json:"databaseId"`
}
func (a *gqlActor) UnmarshalJSON(data []byte) error {
type plain gqlActor
if err := json.Unmarshal(data, (*plain)(a)); err != nil {
return err
}
switch {
case a.Typename == "" && a.Login == "":
a.Login = "ghost" // Match REST's ghost login with an unset ID for GHES.
case a.Typename == "Bot":
a.Login += "[bot]"
}
return nil
}
type gqlCommit struct {
OID string `json:"oid"`
}
type gqlReaction struct {
Content string `json:"content"`
User gqlActor `json:"user"`
}
type gqlComment struct {
ID string `json:"id"`
DatabaseID int64 `json:"databaseId"`
Body string `json:"body"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
Author gqlActor `json:"author"`
}
type gqlLabel struct {
Name string `json:"name"`
Color string `json:"color"`
Description string `json:"description"`
}
type gqlIssue struct {
ID string `json:"id"`
Number int64 `json:"number"`
Title string `json:"title"`
Body string `json:"body"`
State string `json:"state"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
ClosedAt *time.Time `json:"closedAt"`
Locked bool `json:"locked"`
Author gqlActor `json:"author"`
Milestone *struct {
Title string `json:"title"`
} `json:"milestone"`
Labels gqlConn[gqlLabel] `json:"labels"`
Assignees gqlConn[gqlActor] `json:"assignees"`
Reactions gqlConn[gqlReaction] `json:"reactions"`
Comments gqlConn[gqlComment] `json:"comments"`
}
type gqlIssuesResponse struct {
Repository struct {
Issues gqlConn[gqlIssue] `json:"issues"`
} `json:"repository"`
}
type gqlPrefetched struct {
index int64
comments []*base.Comment
commentIDs []string
reviews []*base.Review
}
func (g *GithubGraphQLDownloader) getIssuesGraphQL(ctx context.Context, page, perPage int) ([]*base.Issue, bool, error) {
if page <= 1 {
g.issuesCursor = ""
}
var resp gqlIssuesResponse
if err := g.doGraphQLPage(ctx, g.graphQLIssuesQuery(), g.issuesCursor, graphQLPageSize(perPage, githubGraphQLPageSize), &resp); err != nil {
return nil, false, err
}
conn := resp.Repository.Issues
prefetched := map[string]*gqlPrefetched{}
issues := make([]*base.Issue, 0, len(conn.Nodes))
for i := range conn.Nodes {
issue, err := g.convertGraphQLIssue(ctx, &conn.Nodes[i], prefetched)
if err != nil {
return nil, false, err
}
issues = append(issues, issue)
}
if err := g.enrichGraphQLPage(ctx, prefetched); err != nil {
return nil, false, err
}
g.issuesCursor = conn.PageInfo.EndCursor // only now, a failed page is retried from the same cursor
return issues, !conn.PageInfo.HasNextPage, nil
}
func (g *GithubGraphQLDownloader) convertGraphQLIssue(ctx context.Context, node *gqlIssue, prefetchedByNodeID map[string]*gqlPrefetched) (*base.Issue, error) {
labels, err := allNodes(ctx, g, graphQLLabelsSweepQuery, node.ID, node.Labels)
if err != nil {
return nil, err
}
assignees, err := allNodes(ctx, g, graphQLAssigneesSweepQuery, node.ID, node.Assignees)
if err != nil {
return nil, err
}
reactions, err := g.allReactions(ctx, node.ID, node.Reactions)
if err != nil {
return nil, err
}
issue := &base.Issue{
Number: node.Number,
Title: node.Title,
Content: node.Body,
PosterID: node.Author.DatabaseID,
PosterName: node.Author.Login,
State: strings.ToLower(node.State),
Created: node.CreatedAt,
Updated: node.UpdatedAt,
Closed: node.ClosedAt,
IsLocked: node.Locked,
Labels: make([]*base.Label, 0, len(labels)),
Reactions: reactions,
ForeignIndex: node.Number,
}
if node.Milestone != nil {
issue.Milestone = node.Milestone.Title
}
for _, label := range labels {
issue.Labels = append(issue.Labels, &base.Label{Name: label.Name, Color: label.Color, Description: label.Description})
}
for _, assignee := range assignees {
issue.Assignees = append(issue.Assignees, assignee.Login)
}
if g.skipComments {
return issue, nil
}
prefetched := &gqlPrefetched{index: node.Number}
issue.Context = prefetched
prefetchedByNodeID[node.ID] = prefetched
if node.Comments.PageInfo.HasNextPage {
prefetched.comments, err = g.getComments(ctx, issue)
return issue, err
}
for _, comment := range node.Comments.Nodes {
prefetched.commentIDs = append(prefetched.commentIDs, comment.ID)
prefetched.comments = append(prefetched.comments, &base.Comment{
IssueIndex: node.Number,
Index: comment.DatabaseID,
PosterID: comment.Author.DatabaseID,
PosterName: comment.Author.Login,
Content: comment.Body,
Created: comment.CreatedAt,
Updated: comment.UpdatedAt,
})
}
return issue, nil
}
func (g *GithubGraphQLDownloader) enrichGraphQLPage(ctx context.Context, prefetchedByNodeID map[string]*gqlPrefetched) error {
if !g.SkipReactions {
comments := map[string]*base.Comment{}
for _, prefetched := range prefetchedByNodeID {
for i, id := range prefetched.commentIDs {
comments[id] = prefetched.comments[i]
}
}
for batch := range slices.Chunk(slices.Sorted(maps.Keys(comments)), graphQLNodeBatchSize) {
reactions, err := g.fetchNodeReactions(ctx, batch)
if err != nil {
return err
}
for id, reaction := range reactions {
comments[id].Reactions = reaction
}
}
}
return g.attachTimelineEvents(ctx, prefetchedByNodeID)
}
func graphQLPageSize(perPage, maxSize int) int {
if perPage <= 0 {
return maxSize
}
return min(perPage, maxSize) // Keep pages within the caller's database batch size.
}
func (g *GithubGraphQLDownloader) doGraphQLPage(ctx context.Context, query, cursor string, pageSize int, out any) error {
vars := map[string]any{"owner": g.repoOwner, "name": g.repoName}
if cursor != "" {
vars["cursor"] = cursor
}
for {
vars["first"] = pageSize
err := g.doGraphQL(ctx, query, vars, out)
if err == nil || !isGraphQLTooHeavy(err) || pageSize <= 1 {
return err
}
pageSize /= 2
log.Warn("github graphql: page too heavy (%v), retrying with %d", err, pageSize)
}
}
func allNodes[T any](ctx context.Context, g *GithubGraphQLDownloader, sweepQuery, nodeID string, conn gqlConn[T]) ([]T, error) {
nodes := conn.Nodes
for page := conn.PageInfo; page.HasNextPage; {
var resp struct {
Node struct {
Conn gqlConn[T] `json:"conn"`
} `json:"node"`
}
if err := g.doGraphQL(ctx, sweepQuery, map[string]any{"id": nodeID, "cursor": page.EndCursor}, &resp); err != nil {
return nil, err
}
nodes = append(nodes, resp.Node.Conn.Nodes...)
page = resp.Node.Conn.PageInfo
}
return nodes, nil
}
func (g *GithubGraphQLDownloader) allReactions(ctx context.Context, nodeID string, conn gqlConn[gqlReaction]) ([]*base.Reaction, error) {
nodes, err := allNodes(ctx, g, graphQLReactionsSweepQuery, nodeID, conn)
if err != nil || len(nodes) == 0 {
return nil, err
}
reactions := make([]*base.Reaction, 0, len(nodes))
for _, reaction := range nodes {
reactions = append(reactions, &base.Reaction{
UserID: reaction.User.DatabaseID,
UserName: reaction.User.Login,
Content: graphQLReactionContent(reaction.Content),
})
}
return reactions, nil
}
func (g *GithubGraphQLDownloader) fetchNodeReactions(ctx context.Context, ids []string) (map[string][]*base.Reaction, error) {
var resp struct {
Nodes []struct {
ID string `json:"id"`
Reactions gqlConn[gqlReaction] `json:"reactions"`
} `json:"nodes"`
}
if err := g.doGraphQL(ctx, graphQLNodeReactionsQuery, map[string]any{"ids": ids}, &resp); err != nil {
if isGraphQLTooHeavy(err) && len(ids) > 1 {
return fetchHalves(ctx, ids, g.fetchNodeReactions)
}
return nil, err
}
out := make(map[string][]*base.Reaction, len(resp.Nodes))
for _, node := range resp.Nodes {
if node.ID == "" {
continue
}
reactions, err := g.allReactions(ctx, node.ID, node.Reactions)
if err != nil {
return nil, err
}
out[node.ID] = reactions
}
return out, nil
}
func fetchHalves[V any](ctx context.Context, ids []string, fetch func(context.Context, []string) (map[string]V, error)) (map[string]V, error) {
left, err := fetch(ctx, ids[:len(ids)/2])
if err != nil {
return nil, err
}
right, err := fetch(ctx, ids[len(ids)/2:])
if err != nil {
return nil, err
}
maps.Copy(left, right)
return left, nil
}
func graphQLReactionContent(enum string) string {
switch enum {
case "THUMBS_UP":
return "+1"
case "THUMBS_DOWN":
return "-1"
default:
return strings.ToLower(enum)
}
}
+230
View File
@@ -0,0 +1,230 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package migrations
import (
"context"
"fmt"
"slices"
"time"
base "gitea.dev/modules/migration"
)
const graphQLPRFields = "mergedAt isDraft mergeCommit{oid} headRefName headRefOid baseRefName baseRefOid headRepository{name url} headRepositoryOwner{login} baseRepository{name owner{login}}"
var graphQLPRDiscussion = fmt.Sprintf(`
reviews(first:50){%[1]s nodes{
databaseId state body createdAt submittedAt author{%[2]s} commit{oid}
comments(first:50){%[1]s nodes{databaseId body path diffHunk position commit{oid} author{%[2]s} createdAt updatedAt replyTo{databaseId}}}
}}
reviewRequests(first:%[3]d){%[1]s nodes{requestedReviewer{%[4]s}}}`,
gqlPageInfoFields, gqlActorFields, graphQLReviewRequestPageSize, gqlReviewerFields)
var graphQLReviewRequestsSweepQuery = nodeConnQuery("PullRequest", "reviewRequests", "requestedReviewer{"+gqlReviewerFields+"}")
func (g *GithubGraphQLDownloader) graphQLPullRequestsQuery() string {
return g.graphQLContentQuery("pullRequests", graphQLPRFields, graphQLPRDiscussion)
}
type gqlRepoRef struct {
Name string `json:"name"`
URL string `json:"url"`
Owner gqlActor `json:"owner"`
}
type gqlReviewComment struct {
DatabaseID int64 `json:"databaseId"`
Body string `json:"body"`
Path string `json:"path"`
DiffHunk string `json:"diffHunk"`
Position int `json:"position"`
Commit *gqlCommit `json:"commit"`
Author gqlActor `json:"author"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
ReplyTo *struct {
DatabaseID int64 `json:"databaseId"`
} `json:"replyTo"`
}
type gqlReview struct {
DatabaseID int64 `json:"databaseId"`
State string `json:"state"`
Body string `json:"body"`
CreatedAt time.Time `json:"createdAt"`
SubmittedAt *time.Time `json:"submittedAt"`
Author gqlActor `json:"author"`
Commit *gqlCommit `json:"commit"`
Comments gqlConn[gqlReviewComment] `json:"comments"`
}
type gqlReviewRequest struct {
RequestedReviewer *gqlActor `json:"requestedReviewer"`
}
type gqlPullRequest struct {
gqlIssue
MergedAt *time.Time `json:"mergedAt"`
IsDraft bool `json:"isDraft"`
MergeCommit *gqlCommit `json:"mergeCommit"`
HeadRefName string `json:"headRefName"`
HeadRefOID string `json:"headRefOid"`
BaseRefName string `json:"baseRefName"`
BaseRefOID string `json:"baseRefOid"`
HeadRepo *gqlRepoRef `json:"headRepository"`
HeadRepoOwner *gqlActor `json:"headRepositoryOwner"`
BaseRepo *gqlRepoRef `json:"baseRepository"`
Reviews gqlConn[gqlReview] `json:"reviews"`
ReviewRequests gqlConn[gqlReviewRequest] `json:"reviewRequests"`
}
type gqlPullRequestsResponse struct {
Repository struct {
PullRequests gqlConn[gqlPullRequest] `json:"pullRequests"`
} `json:"repository"`
}
func (g *GithubGraphQLDownloader) getPullRequestsGraphQL(ctx context.Context, page, perPage int) ([]*base.PullRequest, bool, error) {
if page <= 1 {
g.prCursor = ""
}
var resp gqlPullRequestsResponse
if err := g.doGraphQLPage(ctx, g.graphQLPullRequestsQuery(), g.prCursor, graphQLPageSize(perPage, githubGraphQLPRPageSize), &resp); err != nil {
return nil, false, err
}
conn := resp.Repository.PullRequests
prefetched := map[string]*gqlPrefetched{}
prs := make([]*base.PullRequest, 0, len(conn.Nodes))
for i := range conn.Nodes {
pr, err := g.convertGraphQLPullRequest(ctx, &conn.Nodes[i], prefetched)
if err != nil {
return nil, false, err
}
_ = CheckAndEnsureSafePR(pr, g.baseURL, g) // SECURITY: like the REST path
prs = append(prs, pr)
}
if err := g.enrichGraphQLPage(ctx, prefetched); err != nil {
return nil, false, err
}
g.prCursor = conn.PageInfo.EndCursor // only now, a failed page is retried from the same cursor
return prs, !conn.PageInfo.HasNextPage, nil
}
func (g *GithubGraphQLDownloader) convertGraphQLPullRequest(ctx context.Context, node *gqlPullRequest, prefetchedByNodeID map[string]*gqlPrefetched) (*base.PullRequest, error) {
issue, err := g.convertGraphQLIssue(ctx, &node.gqlIssue, prefetchedByNodeID)
if err != nil {
return nil, err
}
if prefetched, ok := issue.Context.(*gqlPrefetched); ok {
if prefetched.reviews, err = g.prefetchReviews(ctx, node); err != nil {
return nil, err
}
}
pr := &base.PullRequest{
Number: issue.Number,
Title: issue.Title,
PosterID: issue.PosterID,
PosterName: issue.PosterName,
Content: issue.Content,
Milestone: issue.Milestone,
State: "closed", // REST reports a merged PR as closed
Created: issue.Created,
Updated: issue.Updated,
Closed: issue.Closed,
Labels: issue.Labels,
Assignees: issue.Assignees,
IsLocked: issue.IsLocked,
Reactions: issue.Reactions,
ForeignIndex: issue.ForeignIndex,
Context: issue.Context,
PatchURL: fmt.Sprintf("%s/%s/%s/pull/%d.patch", g.baseURL, g.repoOwner, g.repoName, node.Number),
Merged: node.MergedAt != nil || node.State == "MERGED",
MergedTime: node.MergedAt,
IsDraft: node.IsDraft,
Head: base.PullRequestBranch{Ref: node.HeadRefName, SHA: node.HeadRefOID},
Base: base.PullRequestBranch{Ref: node.BaseRefName, SHA: node.BaseRefOID},
}
if node.State == "OPEN" {
pr.State = "open"
}
if node.MergeCommit != nil {
pr.MergeCommitSHA = node.MergeCommit.OID
}
if node.HeadRepoOwner != nil {
pr.Head.OwnerName = node.HeadRepoOwner.Login
}
if node.HeadRepo != nil {
pr.Head.RepoName = node.HeadRepo.Name
pr.Head.CloneURL = node.HeadRepo.URL + ".git"
}
if node.BaseRepo != nil {
pr.Base.RepoName = node.BaseRepo.Name
pr.Base.OwnerName = node.BaseRepo.Owner.Login
}
return pr, nil
}
func (g *GithubGraphQLDownloader) prefetchReviews(ctx context.Context, node *gqlPullRequest) ([]*base.Review, error) {
if node.Reviews.PageInfo.HasNextPage || slices.ContainsFunc(node.Reviews.Nodes, func(r gqlReview) bool { return r.Comments.PageInfo.HasNextPage }) {
return g.GetReviews(ctx, &base.PullRequest{Number: node.Number, ForeignIndex: node.Number})
}
requests, err := allNodes(ctx, g, graphQLReviewRequestsSweepQuery, node.ID, node.ReviewRequests)
if err != nil {
return nil, err
}
return convertGraphQLReviews(node, requests), nil
}
func convertGraphQLReviews(node *gqlPullRequest, requests []gqlReviewRequest) []*base.Review {
reviews := make([]*base.Review, 0, len(node.Reviews.Nodes)+len(requests))
for _, sourceReview := range node.Reviews.Nodes {
review := &base.Review{
ID: sourceReview.DatabaseID,
IssueIndex: node.Number,
ReviewerID: sourceReview.Author.DatabaseID,
ReviewerName: sourceReview.Author.Login,
Content: sourceReview.Body,
CreatedAt: sourceReview.CreatedAt,
State: sourceReview.State,
}
if sourceReview.SubmittedAt != nil {
review.CreatedAt = *sourceReview.SubmittedAt
}
if sourceReview.Commit != nil {
review.CommitID = sourceReview.Commit.OID
}
for _, sourceComment := range sourceReview.Comments.Nodes {
comment := &base.ReviewComment{
ID: sourceComment.DatabaseID,
Content: sourceComment.Body,
TreePath: sourceComment.Path,
DiffHunk: sourceComment.DiffHunk,
Position: sourceComment.Position,
PosterID: sourceComment.Author.DatabaseID,
CreatedAt: sourceComment.CreatedAt,
UpdatedAt: sourceComment.UpdatedAt,
}
if sourceComment.ReplyTo != nil {
comment.InReplyTo = sourceComment.ReplyTo.DatabaseID
}
if sourceComment.Commit != nil {
comment.CommitID = sourceComment.Commit.OID
}
review.Comments = append(review.Comments, comment)
}
reviews = append(reviews, review)
}
for _, request := range requests {
if reviewer := request.RequestedReviewer; reviewer != nil && reviewer.Login != "" {
reviews = append(reviews, &base.Review{
IssueIndex: node.Number,
ReviewerID: reviewer.DatabaseID,
ReviewerName: reviewer.Login,
State: base.ReviewStateRequestReview,
})
}
}
return reviews
}
@@ -0,0 +1,104 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package migrations
import (
"io"
"net/http"
"strings"
"testing"
"time"
"gitea.dev/modules/json"
base "gitea.dev/modules/migration"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGraphQLPullRequestParsing(t *testing.T) {
var response gqlPullRequestsResponse
require.NoError(t, json.Unmarshal([]byte(`{"repository":{"pullRequests":{"nodes":[{
"number":7,"title":"add feature","body":"does the thing","state":"MERGED",
"createdAt":"2023-05-01T00:00:00Z","updatedAt":"2023-06-01T00:00:00Z",
"closedAt":"2023-06-01T00:00:00Z","mergedAt":"2023-06-01T00:00:00Z",
"author":{"login":"contributor","databaseId":4242},"isDraft":true,
"mergeCommit":{"oid":"abc123"},"headRefName":"feature-x","headRefOid":"deadbeef",
"baseRefName":"main","baseRefOid":"cafebabe",
"headRepository":{"name":"gitea","url":"https://github.com/contributor/gitea"},
"headRepositoryOwner":{"login":"contributor"},
"baseRepository":{"name":"gitea","owner":{"login":"go-gitea"}},
"reviews":{"nodes":[{
"databaseId":900,"state":"CHANGES_REQUESTED","body":"please fix",
"createdAt":"2023-05-03T00:00:00Z","submittedAt":"2023-05-03T01:00:00Z",
"author":{"login":"reviewer","databaseId":77},"commit":{"oid":"sha1sha"},
"comments":{"nodes":[{"databaseId":950,"body":"nit here","path":"main.go","diffHunk":"@@ -1 +1 @@",
"position":3,"commit":{"oid":"sha1sha"},"author":{"databaseId":77},"replyTo":{"databaseId":949}}]}
}]},
"reviewRequests":{"nodes":[{"requestedReviewer":{"__typename":"Bot","login":"pending-rev","databaseId":88}},
{"requestedReviewer":{"__typename":"Team"}},{"requestedReviewer":null}]}
}]}}}`), &response))
downloader := &GithubGraphQLDownloader{GithubDownloaderV3: &GithubDownloaderV3{baseURL: "https://github.com", repoOwner: "go-gitea", repoName: "gitea"}}
pr, err := downloader.convertGraphQLPullRequest(t.Context(), &response.Repository.PullRequests.Nodes[0], map[string]*gqlPrefetched{})
require.NoError(t, err)
merged := time.Date(2023, 6, 1, 0, 0, 0, 0, time.UTC)
assert.Equal(t, &base.PullRequest{
Number: 7, Title: "add feature", Content: "does the thing", State: "closed", PosterID: 4242, PosterName: "contributor",
Created: time.Date(2023, 5, 1, 0, 0, 0, 0, time.UTC), Updated: merged, Closed: &merged,
Merged: true, MergedTime: &merged, IsDraft: true, MergeCommitSHA: "abc123", ForeignIndex: 7,
Head: base.PullRequestBranch{Ref: "feature-x", SHA: "deadbeef", OwnerName: "contributor", RepoName: "gitea", CloneURL: "https://github.com/contributor/gitea.git"},
Base: base.PullRequestBranch{Ref: "main", SHA: "cafebabe", OwnerName: "go-gitea", RepoName: "gitea"},
PatchURL: "https://github.com/go-gitea/gitea/pull/7.patch", Labels: []*base.Label{}, Context: pr.Context,
}, pr)
reviews, err := downloader.GetReviews(t.Context(), pr)
require.NoError(t, err)
assert.Equal(t, []*base.Review{
{
ID: 900, IssueIndex: 7, ReviewerID: 77, ReviewerName: "reviewer", Content: "please fix", State: "CHANGES_REQUESTED", CommitID: "sha1sha",
CreatedAt: time.Date(2023, 5, 3, 1, 0, 0, 0, time.UTC),
Comments: []*base.ReviewComment{{ID: 950, InReplyTo: 949, Content: "nit here", TreePath: "main.go", DiffHunk: "@@ -1 +1 @@", Position: 3, CommitID: "sha1sha", PosterID: 77}},
},
{IssueIndex: 7, ReviewerID: 88, ReviewerName: "pending-rev[bot]", State: base.ReviewStateRequestReview},
}, reviews)
}
func TestGraphQLDiscussionOverflowFallsBackToREST(t *testing.T) {
for _, connection := range []string{"comments", "reviews", "review comments"} {
t.Run(connection, func(t *testing.T) {
downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, request *http.Request) {
switch {
case strings.HasSuffix(request.URL.Path, "/issues/7/comments"):
_, _ = io.WriteString(writer, `[{"id":500,"body":"REST comment"}]`)
case strings.HasSuffix(request.URL.Path, "/pulls/7/reviews"):
_, _ = io.WriteString(writer, `[{"id":900,"body":"REST review","state":"APPROVED"}]`)
case strings.HasSuffix(request.URL.Path, "/requested_reviewers"):
_, _ = io.WriteString(writer, `{}`)
default:
_, _ = io.WriteString(writer, `[]`)
}
})
downloader.SkipReactions = true
node := &gqlPullRequest{gqlIssue: gqlIssue{Number: 7}}
switch connection {
case "comments":
node.Comments.PageInfo.HasNextPage = true
case "reviews":
node.Reviews.PageInfo.HasNextPage = true
case "review comments":
node.Reviews.Nodes = []gqlReview{{Comments: gqlConn[gqlReviewComment]{PageInfo: gqlPageInfo{HasNextPage: true}}}}
}
pr, err := downloader.convertGraphQLPullRequest(t.Context(), node, map[string]*gqlPrefetched{})
require.NoError(t, err)
if connection == "comments" {
comments, _, err := downloader.GetComments(t.Context(), pr)
require.NoError(t, err)
assert.Equal(t, []*base.Comment{{IssueIndex: 7, Index: 500, Content: "REST comment"}}, comments)
} else {
reviews, err := downloader.GetReviews(t.Context(), pr)
require.NoError(t, err)
assert.Equal(t, []*base.Review{{IssueIndex: 7, ID: 900, Content: "REST review", State: "APPROVED"}}, reviews)
}
})
}
}
+428
View File
@@ -0,0 +1,428 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package migrations
import (
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"net/url"
"strconv"
"strings"
"sync/atomic"
"testing"
"testing/synctest"
"time"
"gitea.dev/modules/egress/policy"
"gitea.dev/modules/json"
base "gitea.dev/modules/migration"
"gitea.dev/modules/test"
"github.com/google/go-github/v92/github"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func newGithubTestDownloader(t *testing.T, handler http.HandlerFunc, spareTokens ...string) *GithubGraphQLDownloader {
t.Helper()
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
writer.Header().Set("Content-Type", "application/json")
writer.Header().Set("X-RateLimit-Remaining", "4000")
handler(writer, request)
}))
t.Cleanup(server.Close)
downloader, err := NewGithubDownloaderV3(t.Context(), server.URL, "", "", strings.Join(append([]string{"tok"}, spareTokens...), ","), "o", "r")
require.NoError(t, err)
return newGithubGraphQLDownloader(downloader, false)
}
func TestGraphQLEndpoint(t *testing.T) {
for baseURL, endpoint := range map[string]string{
"https://github.com": "https://api.github.com/graphql",
"https://api.github.com": "https://api.github.com/graphql",
"https://ghe.example.com": "https://ghe.example.com/api/graphql",
"https://ghe.example.com/": "https://ghe.example.com/api/graphql",
} {
downloader, err := NewGithubDownloaderV3(t.Context(), baseURL, "", "", "tok", "o", "r")
require.NoError(t, err)
assert.Equal(t, endpoint, newGithubGraphQLDownloader(downloader, false).graphQLEndpoint(), baseURL)
}
}
func TestGraphQLRateResetWait(t *testing.T) {
for _, testcase := range []struct {
name string
headers map[string]string
want time.Duration
}{
{"retry after takes precedence", map[string]string{"Retry-After": "30", "X-RateLimit-Reset": "9999999999"}, 30 * time.Second},
{"no headers", nil, graphQLRateResetFallback},
{"past primary reset", map[string]string{"X-RateLimit-Reset": "1"}, graphQLRateResetFallback},
{"secondary reset is not primary reset", map[string]string{"X-RateLimit-Remaining": "4321", "X-RateLimit-Reset": "9999999999"}, graphQLRateResetFallback},
{"primary reset is capped", map[string]string{"X-RateLimit-Remaining": "0", "X-RateLimit-Reset": "9999999999"}, graphQLRateResetCap},
{"retry after is capped", map[string]string{"Retry-After": "999999"}, graphQLRateResetCap},
} {
t.Run(testcase.name, func(t *testing.T) {
header := http.Header{}
for name, value := range testcase.headers {
header.Set(name, value)
}
assert.Equal(t, testcase.want, graphQLRateResetWait(header))
})
}
}
func TestDoGraphQLRetriesTransientFailures(t *testing.T) {
defer test.MockVariableValue(&graphQLTransientRetryBase, time.Duration(0))()
for _, testcase := range []struct {
name string
failure string
failures int32
}{
{"bad gateway then success", "502", 1},
{"truncated body then success", "body", 1},
{"connection closed then success", "connection", 1},
{"persistent bad gateway", "502", maxGraphQLTransientRetries + 1},
} {
t.Run(testcase.name, func(t *testing.T) {
var calls atomic.Int32
downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, _request *http.Request) {
if calls.Add(1) > testcase.failures {
_, _ = io.WriteString(writer, `{"data":{"ok":true}}`)
return
}
switch testcase.failure {
case "502":
writer.WriteHeader(http.StatusBadGateway)
case "body":
_, _ = io.WriteString(writer, `{"data":{"ok":`)
case "connection":
connection, _, err := http.NewResponseController(writer).Hijack()
if assert.NoError(t, err) {
_ = connection.Close()
}
}
})
var response struct {
OK bool `json:"ok"`
}
err := downloader.doGraphQL(t.Context(), "query{}", nil, &response)
if testcase.failures > maxGraphQLTransientRetries {
require.ErrorIs(t, err, errGraphQLTransientExhausted)
assert.Equal(t, int32(maxGraphQLTransientRetries+1), calls.Load())
} else {
require.NoError(t, err)
assert.True(t, response.OK)
assert.Equal(t, testcase.failures+1, calls.Load())
}
})
}
}
func TestDoGraphQLWaitsOutSecondaryRateLimit(t *testing.T) {
defer test.MockVariableValue(&graphQLRateResetFallback, time.Millisecond)()
for _, testcase := range []struct {
name string
status int
body string
}{
{"GraphQL rate limit", http.StatusOK, `{"errors":[{"type":"RATE_LIMITED"}]}`},
{"429 without retry after", http.StatusTooManyRequests, ""},
{"403 spent primary quota", http.StatusForbidden, ""},
{"403 secondary limit with quota left", http.StatusForbidden, `{"message":"You have exceeded a secondary rate limit."}`},
} {
t.Run(testcase.name, func(t *testing.T) {
tokens := make(chan string, 3)
downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, request *http.Request) {
tokens <- request.Header.Get("Authorization")
if request.Header.Get("Authorization") == "Bearer tok" {
writer.Header().Set("X-RateLimit-Reset", strconv.FormatInt(time.Now().Add(time.Hour).Unix(), 10))
if testcase.body == "" {
writer.Header().Set("X-RateLimit-Remaining", "0")
}
writer.WriteHeader(testcase.status)
_, _ = io.WriteString(writer, testcase.body)
return
}
_, _ = io.WriteString(writer, `{"data":{"ok":true}}`)
}, "spare")
var response struct {
OK bool `json:"ok"`
}
require.NoError(t, downloader.doGraphQL(t.Context(), "query{}", nil, &response))
assert.True(t, response.OK)
assert.Equal(t, []string{"Bearer tok", "Bearer spare"}, []string{<-tokens, <-tokens})
assert.Zero(t, downloader.gqlRates[0].Remaining)
})
}
}
func TestDoGraphQLRateRetryBudgetPerToken(t *testing.T) {
defer test.MockVariableValue(&graphQLRateResetFallback, time.Duration(0))()
var calls atomic.Int32
downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, _request *http.Request) {
calls.Add(1)
_, _ = io.WriteString(writer, `{"errors":[{"type":"RATE_LIMITED"}]}`)
}, "spare")
require.Error(t, downloader.doGraphQL(t.Context(), "query{}", nil, &struct{}{}))
assert.Equal(t, int32(maxGraphQLRateRetries*len(downloader.clients)+1), calls.Load())
}
func TestDoGraphQLFatalStatusQuotesBody(t *testing.T) {
var calls atomic.Int32
downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, _request *http.Request) {
calls.Add(1)
writer.WriteHeader(http.StatusForbidden)
_, _ = io.WriteString(writer, `{"message":"Resource not accessible by integration"}`)
})
assert.ErrorContains(t, downloader.doGraphQL(t.Context(), "query{}", nil, &struct{}{}), "Resource not accessible by integration")
assert.EqualValues(t, 1, calls.Load())
}
func TestPickGraphQLClient(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
reset := github.Timestamp{Time: time.Now().Add(2 * graphQLRateResetCap)}
downloader := &GithubGraphQLDownloader{GithubDownloaderV3: &GithubDownloaderV3{}, gqlRates: []*github.Rate{
{Remaining: 1000, Reset: reset},
{Remaining: 4000, Reset: reset},
{Remaining: 5, Reset: reset},
}}
require.NoError(t, downloader.pickGraphQLClient(t.Context()))
assert.Equal(t, 1, downloader.curClientIdx)
downloader.gqlRates[2] = nil
require.NoError(t, downloader.pickGraphQLClient(t.Context()))
assert.Equal(t, 2, downloader.curClientIdx)
downloader.gqlRates = []*github.Rate{{Reset: github.Timestamp{Time: reset.Add(time.Hour)}}, {Reset: reset}}
started := time.Now()
require.NoError(t, downloader.pickGraphQLClient(t.Context()))
assert.Equal(t, graphQLRateResetCap, time.Since(started))
assert.Equal(t, 1, downloader.curClientIdx)
assert.Nil(t, downloader.gqlRates[1])
})
}
func TestIsTransientTransportError(t *testing.T) {
wrapped := func(err error) error {
return &url.Error{Op: "Post", URL: "https://example.com", Err: &net.OpError{Op: "dial", Err: err}}
}
for _, testcase := range []struct {
name string
err error
transient bool
}{
{"blocked by policy", fmt.Errorf("blocked: %w", policy.ErrDenied), false},
{"host does not resolve", &net.DNSError{IsNotFound: true}, false},
{"TLS verification failed", &tls.CertificateVerificationError{Err: x509.UnknownAuthorityError{}}, false},
{"DNS server failure", &net.DNSError{IsTemporary: true}, true},
{"connection reset", errors.New("connection reset by peer"), true},
} {
t.Run(testcase.name, func(t *testing.T) {
assert.Equal(t, testcase.transient, isTransientTransportError(wrapped(testcase.err)))
})
}
}
func TestGraphQLQueryGating(t *testing.T) {
for _, skipComments := range []bool{false, true} {
for _, skipReactions := range []bool{false, true} {
downloader := &GithubGraphQLDownloader{GithubDownloaderV3: &GithubDownloaderV3{SkipReactions: skipReactions}, skipComments: skipComments}
for name, query := range map[string]string{"issues": downloader.graphQLIssuesQuery(), "pullRequests": downloader.graphQLPullRequestsQuery()} {
t.Run(fmt.Sprintf("%s/comments=%t/reactions=%t", name, !skipComments, !skipReactions), func(t *testing.T) {
assert.Equal(t, !skipComments, strings.Contains(query, "comments("))
assert.Equal(t, !skipComments && name == "pullRequests", strings.Contains(query, "reviews("))
assert.Equal(t, !skipComments && name == "pullRequests", strings.Contains(query, "reviewRequests("))
assert.Equal(t, !skipReactions, strings.Contains(query, "reactions("))
assert.LessOrEqual(t, strings.Count(query, "reactions("), 1)
assert.Contains(t, query, "id number title")
assert.Contains(t, query, "orderBy:{field:CREATED_AT,direction:ASC}")
assert.Contains(t, query, "labels(first:30){"+gqlPageInfoFields)
assert.Contains(t, query, "... on Bot{databaseId}")
})
}
}
}
}
func TestGraphQLPageSize(t *testing.T) {
assert.Equal(t, 41, graphQLPageSize(41, githubGraphQLPageSize))
assert.Equal(t, githubGraphQLPageSize, graphQLPageSize(500, githubGraphQLPageSize))
assert.Equal(t, githubGraphQLPageSize, graphQLPageSize(0, githubGraphQLPageSize))
assert.Equal(t, githubGraphQLPRPageSize, graphQLPageSize(41, githubGraphQLPRPageSize))
}
func TestDoGraphQLPageShrink(t *testing.T) {
defer test.MockVariableValue(&graphQLTransientRetryBase, time.Duration(0))()
for _, failure := range []string{"502", "RESOURCE_LIMITS_EXCEEDED"} {
t.Run(failure, func(t *testing.T) {
sizes := make(chan int, 16)
downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, request *http.Request) {
var payload struct {
Variables struct {
First int `json:"first"`
Cursor string `json:"cursor"`
} `json:"variables"`
}
assert.NoError(t, json.NewDecoder(request.Body).Decode(&payload))
sizes <- payload.Variables.First
assert.Equal(t, "CURSOR", payload.Variables.Cursor)
if payload.Variables.First > 25 {
if failure == "502" {
writer.WriteHeader(http.StatusBadGateway)
} else {
_, _ = io.WriteString(writer, `{"errors":[{"type":"RESOURCE_LIMITS_EXCEEDED"}]}`)
}
return
}
_, _ = io.WriteString(writer, `{"data":{}}`)
})
require.NoError(t, downloader.doGraphQLPage(t.Context(), downloader.graphQLIssuesQuery(), "CURSOR", 100, &gqlIssuesResponse{}))
close(sizes)
var got, want []int
for size := range sizes {
got = append(got, size)
}
for _, size := range []int{100, 50, 25} {
repeats := 1
if failure == "502" && size > 25 {
repeats += maxGraphQLTransientRetries
}
for range repeats {
want = append(want, size)
}
}
assert.Equal(t, want, got)
})
}
}
func TestFetchNodeReactionsBisectsAndSkipsDeletedNodes(t *testing.T) {
defer test.MockVariableValue(&graphQLTransientRetryBase, time.Duration(0))()
for _, failure := range []string{"502", "RESOURCE_LIMITS_EXCEEDED"} {
t.Run(failure, func(t *testing.T) {
var calls atomic.Int32
downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, request *http.Request) {
calls.Add(1)
var payload struct {
Variables struct {
IDs []string `json:"ids"`
} `json:"variables"`
}
assert.NoError(t, json.NewDecoder(request.Body).Decode(&payload))
if len(payload.Variables.IDs) > 1 {
if failure == "502" {
writer.WriteHeader(http.StatusBadGateway)
} else {
_, _ = io.WriteString(writer, `{"errors":[{"type":"RESOURCE_LIMITS_EXCEEDED"}]}`)
}
return
}
if payload.Variables.IDs[0] == "deleted" {
_, _ = io.WriteString(writer, `{"data":{"nodes":[null]},"errors":[{"type":"NOT_FOUND","path":["nodes",0]}]}`)
} else {
_, _ = io.WriteString(writer, `{"data":{"nodes":[{"id":"present","reactions":{"nodes":[{"content":"HEART","user":null}]}}]}}`)
}
})
got, err := downloader.fetchNodeReactions(t.Context(), []string{"deleted", "present"})
require.NoError(t, err)
assert.Equal(t, map[string][]*base.Reaction{"present": {{UserName: "ghost", Content: "heart"}}}, got)
if failure == "502" {
assert.EqualValues(t, maxGraphQLTransientRetries+3, calls.Load())
} else {
assert.EqualValues(t, 3, calls.Load())
}
})
}
}
func TestLabelsSweepContinuesFromEndCursor(t *testing.T) {
downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, request *http.Request) {
var payload graphQLRequest
assert.NoError(t, json.NewDecoder(request.Body).Decode(&payload))
assert.Equal(t, "I_node", payload.Variables["id"])
assert.Equal(t, "LABEL1", payload.Variables["cursor"])
_, _ = io.WriteString(writer, `{"data":{"node":{"conn":{"nodes":[{"name":"second"}]}}}}`)
})
labels, err := allNodes(t.Context(), downloader, graphQLLabelsSweepQuery, "I_node", gqlConn[gqlLabel]{
PageInfo: gqlPageInfo{HasNextPage: true, EndCursor: "LABEL1"},
Nodes: []gqlLabel{{Name: "first"}},
})
require.NoError(t, err)
assert.Equal(t, []gqlLabel{{Name: "first"}, {Name: "second"}}, labels)
}
func TestGraphQLIssuesResponseParsing(t *testing.T) {
var response gqlIssuesResponse
require.NoError(t, json.Unmarshal([]byte(`{"repository":{"issues":{"nodes":[{
"id":"I_42","number":42,"title":"a bug","body":"it broke","state":"CLOSED","locked":true,
"createdAt":"2024-01-02T03:04:05Z","updatedAt":"2024-02-03T04:05:06Z","closedAt":"2024-02-03T04:05:06Z",
"author":{"__typename":"Bot","login":"reporter","databaseId":11},"milestone":{"title":"v1.0"},
"labels":{"nodes":[{"name":"bug","color":"d73a4a","description":"defect"}]},
"assignees":{"nodes":[{"login":"maintainer"}]},
"reactions":{"nodes":[{"content":"THUMBS_UP","user":{"login":"alice","databaseId":22}}]},
"comments":{"nodes":[{"id":"IC_1","databaseId":900,"body":"same here","author":null}]}
}]}}}`), &response))
downloader := &GithubGraphQLDownloader{GithubDownloaderV3: &GithubDownloaderV3{}}
issue, err := downloader.convertGraphQLIssue(t.Context(), &response.Repository.Issues.Nodes[0], map[string]*gqlPrefetched{})
require.NoError(t, err)
updated := time.Date(2024, 2, 3, 4, 5, 6, 0, time.UTC)
assert.Equal(t, &base.Issue{
Number: 42, Title: "a bug", Content: "it broke", State: "closed", IsLocked: true,
PosterName: "reporter[bot]", PosterID: 11, Milestone: "v1.0", ForeignIndex: 42,
Created: time.Date(2024, 1, 2, 3, 4, 5, 0, time.UTC), Updated: updated, Closed: &updated,
Labels: []*base.Label{{Name: "bug", Color: "d73a4a", Description: "defect"}}, Assignees: []string{"maintainer"},
Reactions: []*base.Reaction{{Content: "+1", UserName: "alice", UserID: 22}}, Context: issue.Context,
}, issue)
comments, _, err := downloader.GetComments(t.Context(), issue)
require.NoError(t, err)
assert.Equal(t, []*base.Comment{{IssueIndex: 42, Index: 900, Content: "same here", PosterName: "ghost"}}, comments)
assert.False(t, downloader.SupportGetRepoComments())
}
func TestGraphQLKeepsCursorOnPageError(t *testing.T) {
for _, connection := range []string{"issues", "pullRequests"} {
for _, phase := range []string{"timelineItems", "Reactable"} {
t.Run(connection+"/"+phase, func(t *testing.T) {
var failPage atomic.Bool
failPage.Store(true)
downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, request *http.Request) {
var payload graphQLRequest
assert.NoError(t, json.NewDecoder(request.Body).Decode(&payload))
if strings.Contains(payload.Query, "repository(owner:") {
assert.Equal(t, "PAGE1", payload.Variables["cursor"])
_, _ = fmt.Fprintf(writer, `{"data":{"repository":{"%s":{"pageInfo":{"endCursor":"PAGE2"},"nodes":[{"id":"I_1","number":1,"comments":{"nodes":[{"id":"IC_1"}]}}]}}}}`, connection)
} else if failPage.Load() && strings.Contains(payload.Query, phase) {
writer.WriteHeader(http.StatusBadRequest)
} else {
_, _ = io.WriteString(writer, `{"data":{"nodes":[]}}`)
}
})
cursor := &downloader.issuesCursor
if connection == "pullRequests" {
cursor = &downloader.prCursor
}
*cursor = "PAGE1"
fetch := func() error {
if connection == "issues" {
_, _, err := downloader.getIssuesGraphQL(t.Context(), 2, 10)
return err
} else {
_, _, err := downloader.getPullRequestsGraphQL(t.Context(), 2, 10)
return err
}
}
require.Error(t, fetch())
assert.Equal(t, "PAGE1", *cursor)
failPage.Store(false)
require.NoError(t, fetch())
assert.Equal(t, "PAGE2", *cursor)
})
}
}
}
@@ -0,0 +1,213 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package migrations
import (
"context"
"maps"
"slices"
"strings"
"time"
issues_model "gitea.dev/models/issues"
"gitea.dev/modules/log"
base "gitea.dev/modules/migration"
)
// PR-only types in the Issue fragment fail validation of the whole query, assignee events would need local user ids
const (
issueTimelineItemTypes = "CLOSED_EVENT,REOPENED_EVENT,LOCKED_EVENT,UNLOCKED_EVENT,RENAMED_TITLE_EVENT,LABELED_EVENT,UNLABELED_EVENT,MILESTONED_EVENT,DEMILESTONED_EVENT,PINNED_EVENT,UNPINNED_EVENT"
prTimelineItemTypes = issueTimelineItemTypes + ",MERGED_EVENT,HEAD_REF_DELETED_EVENT,AUTO_MERGE_ENABLED_EVENT,AUTO_MERGE_DISABLED_EVENT"
)
const issueTimelineFragments = `
... on ClosedEvent{id createdAt actor{` + gqlActorFields + `}}
... on ReopenedEvent{id createdAt actor{` + gqlActorFields + `}}
... on LockedEvent{id createdAt lockReason actor{` + gqlActorFields + `}}
... on UnlockedEvent{id createdAt actor{` + gqlActorFields + `}}
... on RenamedTitleEvent{id createdAt previousTitle currentTitle actor{` + gqlActorFields + `}}
... on LabeledEvent{id createdAt label{name} actor{` + gqlActorFields + `}}
... on UnlabeledEvent{id createdAt label{name} actor{` + gqlActorFields + `}}
... on MilestonedEvent{id createdAt milestoneTitle actor{` + gqlActorFields + `}}
... on DemilestonedEvent{id createdAt milestoneTitle actor{` + gqlActorFields + `}}
... on PinnedEvent{id createdAt actor{` + gqlActorFields + `}}
... on UnpinnedEvent{id createdAt actor{` + gqlActorFields + `}}`
const prTimelineFragments = issueTimelineFragments + `
... on MergedEvent{id createdAt actor{` + gqlActorFields + `}}
... on HeadRefDeletedEvent{id createdAt headRefName actor{` + gqlActorFields + `}}
... on AutoMergeEnabledEvent{id createdAt actor{` + gqlActorFields + `}}
... on AutoMergeDisabledEvent{id createdAt actor{` + gqlActorFields + `}}`
const (
issueTimelineConn = `{
pageInfo{hasNextPage endCursor}
nodes{
__typename` + issueTimelineFragments + `
}
}`
prTimelineConn = `{
pageInfo{hasNextPage endCursor}
nodes{
__typename` + prTimelineFragments + `
}
}`
)
const graphQLTimelineBatchQuery = `
query($ids:[ID!]!){
nodes(ids:$ids){
id
... on Issue{timelineItems(first:100,itemTypes:[` + issueTimelineItemTypes + `])` + issueTimelineConn + `}
... on PullRequest{timelineItems(first:100,itemTypes:[` + prTimelineItemTypes + `])` + prTimelineConn + `}
}
}`
const graphQLTimelineSweepQuery = `
query($id:ID!,$cursor:String){
node(id:$id){
... on Issue{conn:timelineItems(first:100,after:$cursor,itemTypes:[` + issueTimelineItemTypes + `])` + issueTimelineConn + `}
... on PullRequest{conn:timelineItems(first:100,after:$cursor,itemTypes:[` + prTimelineItemTypes + `])` + prTimelineConn + `}
}
}`
type gqlTimelineItem struct {
Typename string `json:"__typename"`
ID string `json:"id"`
CreatedAt time.Time `json:"createdAt"`
Actor gqlActor `json:"actor"`
PreviousTitle string `json:"previousTitle"`
CurrentTitle string `json:"currentTitle"`
Label *struct {
Name string `json:"name"`
} `json:"label"`
MilestoneTitle string `json:"milestoneTitle"`
HeadRefName string `json:"headRefName"`
LockReason string `json:"lockReason"`
}
func convertTimelineItem(it *gqlTimelineItem) *base.Comment {
comment := &base.Comment{
PosterID: it.Actor.DatabaseID,
PosterName: it.Actor.Login,
Created: it.CreatedAt,
Updated: it.CreatedAt,
}
switch it.Typename {
case "ClosedEvent":
comment.CommentType = issues_model.CommentTypeClose.String()
case "ReopenedEvent":
comment.CommentType = issues_model.CommentTypeReopen.String()
case "LockedEvent":
comment.CommentType = issues_model.CommentTypeLock.String()
comment.Content = strings.ToLower(strings.ReplaceAll(it.LockReason, "_", " "))
case "UnlockedEvent":
comment.CommentType = issues_model.CommentTypeUnlock.String()
case "RenamedTitleEvent":
comment.CommentType = issues_model.CommentTypeChangeTitle.String()
comment.Meta = map[string]any{"OldTitle": it.PreviousTitle, "NewTitle": it.CurrentTitle}
case "LabeledEvent", "UnlabeledEvent":
if it.Label == nil {
return nil
}
comment.CommentType = issues_model.CommentTypeLabel.String()
comment.Meta = map[string]any{"LabelName": it.Label.Name}
if it.Typename == "LabeledEvent" {
comment.Content = "1"
}
case "MilestonedEvent":
comment.CommentType = issues_model.CommentTypeMilestone.String()
comment.Meta = map[string]any{"MilestoneTitle": it.MilestoneTitle}
case "DemilestonedEvent":
comment.CommentType = issues_model.CommentTypeMilestone.String()
comment.Meta = map[string]any{"MilestoneTitle": it.MilestoneTitle, "Removed": true}
case "MergedEvent":
comment.CommentType = issues_model.CommentTypeMergePull.String()
case "HeadRefDeletedEvent":
comment.CommentType = issues_model.CommentTypeDeleteBranch.String()
comment.Content = it.HeadRefName
case "PinnedEvent":
comment.CommentType = issues_model.CommentTypePin.String()
case "UnpinnedEvent":
comment.CommentType = issues_model.CommentTypeUnpin.String()
case "AutoMergeEnabledEvent":
comment.CommentType = issues_model.CommentTypePRScheduledToAutoMerge.String()
case "AutoMergeDisabledEvent":
comment.CommentType = issues_model.CommentTypePRUnScheduledToAutoMerge.String()
default:
log.Warn("github graphql: dropping unhandled timeline event type %q (node %s)", it.Typename, it.ID)
return nil
}
return comment
}
func convertTimelineItems(items []gqlTimelineItem) []*base.Comment {
comments := make([]*base.Comment, 0, len(items))
for i := range items {
if comment := convertTimelineItem(&items[i]); comment != nil {
comments = append(comments, comment)
}
}
return comments
}
func (g *GithubGraphQLDownloader) attachTimelineEvents(ctx context.Context, prefetchedByNodeID map[string]*gqlPrefetched) error {
for batch := range slices.Chunk(slices.Sorted(maps.Keys(prefetchedByNodeID)), graphQLNodeBatchSize) {
eventsByID, err := g.fetchTimelineEvents(ctx, batch)
if err != nil {
return err
}
for id, events := range eventsByID {
prefetched := prefetchedByNodeID[id]
for _, event := range events {
event.IssueIndex = prefetched.index
}
prefetched.comments = append(prefetched.comments, events...)
}
}
return nil
}
func (g *GithubGraphQLDownloader) fetchTimelineEvents(ctx context.Context, ids []string) (map[string][]*base.Comment, error) {
var resp struct {
Nodes []struct {
ID string `json:"id"`
TimelineItems gqlConn[gqlTimelineItem] `json:"timelineItems"`
} `json:"nodes"`
}
if err := g.doGraphQL(ctx, graphQLTimelineBatchQuery, map[string]any{"ids": ids}, &resp); err != nil {
if isGraphQLTooHeavy(err) && len(ids) > 1 {
return fetchHalves(ctx, ids, g.fetchTimelineEvents)
}
return nil, err
}
out := make(map[string][]*base.Comment, len(resp.Nodes))
for _, node := range resp.Nodes {
if node.ID == "" {
continue
}
items, err := allNodes(ctx, g, graphQLTimelineSweepQuery, node.ID, node.TimelineItems)
if err != nil {
return nil, err
}
out[node.ID] = dropCloseOnMerge(convertTimelineItems(items))
}
return out, nil
}
// dropCloseOnMerge drops the close GitHub records with a merge, the last one since a merged PR cannot be reopened
func dropCloseOnMerge(events []*base.Comment) []*base.Comment {
if !slices.ContainsFunc(events, func(event *base.Comment) bool { return event.CommentType == issues_model.CommentTypeMergePull.String() }) {
return events
}
for i, event := range slices.Backward(events) {
switch event.CommentType {
case issues_model.CommentTypeReopen.String():
return events
case issues_model.CommentTypeClose.String():
return slices.Delete(events, i, i+1)
}
}
return events
}
@@ -0,0 +1,80 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package migrations
import (
"testing"
"time"
"gitea.dev/modules/json"
base "gitea.dev/modules/migration"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestConvertTimelineItems(t *testing.T) {
for _, testcase := range []struct {
name string
event string
want *base.Comment
}{
{"close by user", `{"__typename":"ClosedEvent","actor":{"login":"closer","databaseId":42}}`, &base.Comment{CommentType: "close", PosterName: "closer", PosterID: 42}},
{"reopen", `{"__typename":"ReopenedEvent"}`, &base.Comment{CommentType: "reopen"}},
{"label added", `{"__typename":"LabeledEvent","label":{"name":"bug"}}`, &base.Comment{CommentType: "label", Content: "1", Meta: map[string]any{"LabelName": "bug"}}},
{"label removed", `{"__typename":"UnlabeledEvent","label":{"name":"bug"}}`, &base.Comment{CommentType: "label", Meta: map[string]any{"LabelName": "bug"}}},
{"deleted label added is skipped", `{"__typename":"LabeledEvent","label":null}`, nil},
{"deleted label removed is skipped", `{"__typename":"UnlabeledEvent","label":null}`, nil},
{"lock reason in content", `{"__typename":"LockedEvent","lockReason":"TOO_HEATED"}`, &base.Comment{CommentType: "lock", Content: "too heated"}},
{"unlock", `{"__typename":"UnlockedEvent"}`, &base.Comment{CommentType: "unlock"}},
{"title changed", `{"__typename":"RenamedTitleEvent","previousTitle":"old","currentTitle":"new"}`, &base.Comment{CommentType: "change_title", Meta: map[string]any{"OldTitle": "old", "NewTitle": "new"}}},
{"milestone added", `{"__typename":"MilestonedEvent","milestoneTitle":"v1"}`, &base.Comment{CommentType: "milestone", Meta: map[string]any{"MilestoneTitle": "v1"}}},
{"milestone removed", `{"__typename":"DemilestonedEvent","milestoneTitle":"v1"}`, &base.Comment{CommentType: "milestone", Meta: map[string]any{"MilestoneTitle": "v1", "Removed": true}}},
{"merge by bot", `{"__typename":"MergedEvent","actor":{"__typename":"Bot","login":"merger","databaseId":88}}`, &base.Comment{CommentType: "merge_pull", PosterName: "merger[bot]", PosterID: 88}},
{"branch deleted", `{"__typename":"HeadRefDeletedEvent","headRefName":"feature"}`, &base.Comment{CommentType: "delete_branch", Content: "feature"}},
{"pin", `{"__typename":"PinnedEvent"}`, &base.Comment{CommentType: "pin"}},
{"null actor becomes ghost", `{"__typename":"UnpinnedEvent","actor":null}`, &base.Comment{CommentType: "unpin", PosterName: "ghost"}},
{"auto merge enabled", `{"__typename":"AutoMergeEnabledEvent"}`, &base.Comment{CommentType: "pull_scheduled_merge"}},
{"auto merge disabled", `{"__typename":"AutoMergeDisabledEvent"}`, &base.Comment{CommentType: "pull_cancel_scheduled_merge"}},
{"unknown event is skipped", `{"__typename":"MentionedEvent"}`, nil},
} {
t.Run(testcase.name, func(t *testing.T) {
var item gqlTimelineItem
require.NoError(t, json.Unmarshal([]byte(testcase.event), &item))
item.CreatedAt = time.Date(2020, 11, 17, 15, 54, 33, 0, time.UTC)
want := []*base.Comment{}
if testcase.want != nil {
testcase.want.Created, testcase.want.Updated = item.CreatedAt, item.CreatedAt
want = append(want, testcase.want)
}
assert.Equal(t, want, convertTimelineItems([]gqlTimelineItem{item}))
})
}
}
func TestDropCloseOnMerge(t *testing.T) {
for _, testcase := range []struct {
name string
events []string
want []string
}{
{"unmerged close stays", []string{"close"}, []string{"close"}},
{"last close after merge drops", []string{"close", "reopen", "merge_pull", "close"}, []string{"close", "reopen", "merge_pull"}},
{"last close before merge drops", []string{"close", "merge_pull"}, []string{"merge_pull"}},
{"close followed by reopen stays", []string{"close", "merge_pull", "reopen"}, []string{"close", "merge_pull", "reopen"}},
{"merge without close stays", []string{"merge_pull"}, []string{"merge_pull"}},
} {
t.Run(testcase.name, func(t *testing.T) {
var events []*base.Comment
for _, event := range testcase.events {
events = append(events, &base.Comment{CommentType: event})
}
var got []string
for _, event := range dropCloseOnMerge(events) {
got = append(got, event.CommentType)
}
assert.Equal(t, testcase.want, got)
})
}
}
+118 -1
View File
@@ -12,6 +12,8 @@ import (
"path/filepath" "path/filepath"
"runtime" "runtime"
"slices" "slices"
"strconv"
"strings"
"testing" "testing"
"time" "time"
@@ -48,7 +50,7 @@ func TestGithubIssuePagination(t *testing.T) {
w.Header().Set("X-RateLimit-Remaining", "100") w.Header().Set("X-RateLimit-Remaining", "100")
if r.URL.Path == "/api/v3/repos/o/r/issues/4/reactions" && failReaction { if r.URL.Path == "/api/v3/repos/o/r/issues/4/reactions" && failReaction {
failReaction = false failReaction = false
w.WriteHeader(http.StatusInternalServerError) w.WriteHeader(http.StatusBadRequest)
_, _ = w.Write([]byte(`{"message":"temporary failure"}`)) _, _ = w.Write([]byte(`{"message":"temporary failure"}`))
return return
} }
@@ -595,3 +597,118 @@ func TestGithubMultiTokenClientSelection(t *testing.T) {
downloader.waitAndPickClient(t.Context()) downloader.waitAndPickClient(t.Context())
assert.Equal(t, 1, downloader.curClientIdx) assert.Equal(t, 1, downloader.curClientIdx)
} }
func TestGitHubIssuesPageBasedLinkFallback(t *testing.T) {
_, callerFile, _, _ := runtime.Caller(0)
mockServer := unittest.NewMockWebServer(t, "https://api.github.com", filepath.Join(filepath.Dir(callerFile), "_mock_data/TestGitHubDownloadRepo"), false, unittest.MockServerOptions{
StripPrefix: "/api/v3",
})
downloader, err := NewGithubDownloaderV3(t.Context(), mockServer.URL, "", "", "", "go-gitea", "test_repo")
require.NoError(t, err)
downloader.SkipReactions = true
for page, want := range [][]int64{{1, 2, 3}, {}} {
issues, isEnd, err := downloader.GetIssues(t.Context(), page+1, 3)
require.NoError(t, err)
numbers := make([]int64, 0, len(issues))
for _, issue := range issues {
numbers = append(numbers, issue.Number)
}
assert.Equal(t, want, numbers)
assert.Equal(t, page == 1, isEnd)
}
}
func TestGithubFactoryUsesGraphQLOnlyWhereItWorks(t *testing.T) {
for _, testcase := range []struct {
name string
opts base.MigrateOptions
want bool
}{
{name: "token, issues", opts: base.MigrateOptions{AuthToken: "tok", Issues: true}, want: true},
{name: "basic auth, pull requests", opts: base.MigrateOptions{AuthUsername: "u", AuthPassword: "pat", PullRequests: true}, want: true},
{name: "anonymous repository", opts: base.MigrateOptions{Issues: true, PullRequests: true}},
{name: "comments only", opts: base.MigrateOptions{AuthToken: "tok", Comments: true}},
{name: "issues with comments", opts: base.MigrateOptions{AuthToken: "tok", Issues: true, Comments: true}, want: true},
} {
t.Run(testcase.name, func(t *testing.T) {
testcase.opts.CloneAddr = "https://github.com/go-gitea/test_repo"
candidate, err := (&GithubDownloaderV3Factory{}).New(t.Context(), testcase.opts)
require.NoError(t, err)
downloader, isGraphQL := candidate.(*GithubGraphQLDownloader)
assert.Equal(t, testcase.want, isGraphQL)
if isGraphQL {
assert.Equal(t, !testcase.opts.Comments, downloader.skipComments)
}
})
}
}
func TestGithubGraphQLFallsBackToREST(t *testing.T) {
for _, connection := range []string{"issues", "pull requests"} {
t.Run(connection, func(t *testing.T) {
var restPage string
downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, request *http.Request) {
if request.URL.Path == "/api/graphql" {
http.NotFound(writer, request)
return
}
restPage = request.URL.Query().Get("page")
_, _ = writer.Write([]byte(`[{"number":1,"title":"t","state":"open"}]`))
})
downloader.SkipReactions = true
if connection == "issues" {
issues, _, err := downloader.GetIssues(t.Context(), 3, 10)
require.NoError(t, err)
require.Len(t, issues, 1)
assert.EqualValues(t, 1, issues[0].Number)
} else {
prs, _, err := downloader.GetPullRequests(t.Context(), 3, 10)
require.NoError(t, err)
require.Len(t, prs, 1)
assert.EqualValues(t, 1, prs[0].Number)
}
assert.True(t, downloader.restFallback)
assert.Equal(t, "1", restPage)
})
}
}
func TestGitBucketPaginatesWithoutLinkHeader(t *testing.T) {
const total = 26
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
page, _ := strconv.Atoi(request.URL.Query().Get("page"))
var items []string
for number := (page-1)*25 + 1; number <= min(page*25, total); number++ {
items = append(items, fmt.Sprintf(`{"number":%d,"title":"t"}`, number))
}
writer.Header().Set("X-RateLimit-Remaining", "4000")
_, _ = writer.Write([]byte("[" + strings.Join(items, ",") + "]"))
}))
t.Cleanup(server.Close)
downloader, err := NewGitBucketDownloader(t.Context(), server.URL, "", "", "", "o", "r")
require.NoError(t, err)
for _, connection := range []string{"issues", "pull requests"} {
t.Run(connection, func(t *testing.T) {
var count int
for page := 1; ; page++ {
require.LessOrEqual(t, page, 3)
var isEnd bool
if connection == "issues" {
issues, done, err := downloader.GetIssues(t.Context(), page, 10)
require.NoError(t, err)
count += len(issues)
isEnd = done
} else {
prs, done, err := downloader.GetPullRequests(t.Context(), page, 10)
require.NoError(t, err)
count += len(prs)
isEnd = done
}
if isEnd {
break
}
}
assert.Equal(t, total, count)
})
}
}
+18 -26
View File
@@ -80,9 +80,7 @@
{{else if eq .Type 1}} {{else if eq .Type 1}}
<div class="timeline-item event" id="{{.HashTag}}"> <div class="timeline-item event" id="{{.HashTag}}">
<span class="badge tw-bg-green tw-text-white">{{svg "octicon-dot-fill"}}</span> <span class="badge tw-bg-green tw-text-white">{{svg "octicon-dot-fill"}}</span>
{{if not .OriginalAuthor}} {{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}}
{{template "shared/user/avatarlink" dict "user" .Poster}}
{{end}}
<span class="comment-text-line"> <span class="comment-text-line">
{{template "repo/issue/view_content/comments_authorlink" dict "comment" .}} {{template "repo/issue/view_content/comments_authorlink" dict "comment" .}}
{{if .Issue.IsPull}} {{if .Issue.IsPull}}
@@ -95,9 +93,7 @@
{{else if eq .Type 2}} {{else if eq .Type 2}}
<div class="timeline-item event" id="{{.HashTag}}"> <div class="timeline-item event" id="{{.HashTag}}">
<span class="badge tw-bg-red tw-text-white">{{svg "octicon-issue-closed"}}</span> <span class="badge tw-bg-red tw-text-white">{{svg "octicon-issue-closed"}}</span>
{{if not .OriginalAuthor}} {{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}}
{{template "shared/user/avatarlink" dict "user" .Poster}}
{{end}}
<span class="comment-text-line"> <span class="comment-text-line">
{{template "repo/issue/view_content/comments_authorlink" dict "comment" .}} {{template "repo/issue/view_content/comments_authorlink" dict "comment" .}}
{{if .Issue.IsPull}} {{if .Issue.IsPull}}
@@ -110,9 +106,7 @@
{{else if eq .Type 28}} {{else if eq .Type 28}}
<div class="timeline-item event" id="{{.HashTag}}"> <div class="timeline-item event" id="{{.HashTag}}">
<span class="badge tw-bg-purple tw-text-white">{{svg "octicon-git-merge"}}</span> <span class="badge tw-bg-purple tw-text-white">{{svg "octicon-git-merge"}}</span>
{{if not .OriginalAuthor}} {{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}}
{{template "shared/user/avatarlink" dict "user" .Poster}}
{{end}}
<span class="comment-text-line"> <span class="comment-text-line">
{{template "repo/issue/view_content/comments_authorlink" dict "comment" .}} {{template "repo/issue/view_content/comments_authorlink" dict "comment" .}}
{{$link := printf "%s/commit/%s" $.Repository.Link ($.Issue.PullRequest.MergedCommitID|PathEscape)}} {{$link := printf "%s/commit/%s" $.Repository.Link ($.Issue.PullRequest.MergedCommitID|PathEscape)}}
@@ -169,7 +163,7 @@
{{if or .AddedLabels .RemovedLabels}} {{if or .AddedLabels .RemovedLabels}}
<div class="timeline-item event" id="{{.HashTag}}"> <div class="timeline-item event" id="{{.HashTag}}">
<span class="badge">{{svg "octicon-tag"}}</span> <span class="badge">{{svg "octicon-tag"}}</span>
{{template "shared/user/avatarlink" dict "user" .Poster}} {{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}}
<span class="comment-text-line"> <span class="comment-text-line">
{{template "repo/issue/view_content/comments_authorlink" dict "comment" .}} {{template "repo/issue/view_content/comments_authorlink" dict "comment" .}}
{{if and .AddedLabels (not .RemovedLabels)}} {{if and .AddedLabels (not .RemovedLabels)}}
@@ -185,9 +179,9 @@
{{else if eq .Type 8}} {{else if eq .Type 8}}
<div class="timeline-item event" id="{{.HashTag}}"> <div class="timeline-item event" id="{{.HashTag}}">
<span class="badge">{{svg "octicon-milestone"}}</span> <span class="badge">{{svg "octicon-milestone"}}</span>
{{template "shared/user/avatarlink" dict "user" .Poster}} {{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}}
<span class="comment-text-line"> <span class="comment-text-line">
{{template "shared/user/authorlink" .Poster}} {{template "repo/issue/view_content/comments_authorlink" dict "comment" .}}
{{if gt .OldMilestoneID 0}}{{if gt .MilestoneID 0}}{{ctx.Locale.Tr "repo.issues.change_milestone_at" .OldMilestone.Name .Milestone.Name $createdStr}}{{else}}{{ctx.Locale.Tr "repo.issues.remove_milestone_at" .OldMilestone.Name $createdStr}}{{end}}{{else if gt .MilestoneID 0}}{{ctx.Locale.Tr "repo.issues.add_milestone_at" .Milestone.Name $createdStr}}{{end}} {{if gt .OldMilestoneID 0}}{{if gt .MilestoneID 0}}{{ctx.Locale.Tr "repo.issues.change_milestone_at" .OldMilestone.Name .Milestone.Name $createdStr}}{{else}}{{ctx.Locale.Tr "repo.issues.remove_milestone_at" .OldMilestone.Name $createdStr}}{{end}}{{else if gt .MilestoneID 0}}{{ctx.Locale.Tr "repo.issues.add_milestone_at" .Milestone.Name $createdStr}}{{end}}
</span> </span>
</div> </div>
@@ -219,18 +213,18 @@
{{else if eq .Type 10}} {{else if eq .Type 10}}
<div class="timeline-item event" id="{{.HashTag}}"> <div class="timeline-item event" id="{{.HashTag}}">
<span class="badge">{{ctx.RenderUtils.RenderTimelineEventBadge $comment}}</span> <span class="badge">{{ctx.RenderUtils.RenderTimelineEventBadge $comment}}</span>
{{template "shared/user/avatarlink" dict "user" .Poster}} {{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}}
<span class="comment-text-line"> <span class="comment-text-line">
{{template "shared/user/authorlink" .Poster}} {{template "repo/issue/view_content/comments_authorlink" dict "comment" .}}
{{ctx.RenderUtils.RenderTimelineEventComment $comment $createdStr}} {{ctx.RenderUtils.RenderTimelineEventComment $comment $createdStr}}
</span> </span>
</div> </div>
{{else if eq .Type 11}} {{else if eq .Type 11}}
<div class="timeline-item event" id="{{.HashTag}}"> <div class="timeline-item event" id="{{.HashTag}}">
<span class="badge">{{svg "octicon-git-branch"}}</span> <span class="badge">{{svg "octicon-git-branch"}}</span>
{{template "shared/user/avatarlink" dict "user" .Poster}} {{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}}
<span class="comment-text-line"> <span class="comment-text-line">
{{template "shared/user/authorlink" .Poster}} {{template "repo/issue/view_content/comments_authorlink" dict "comment" .}}
{{$oldRef := HTMLFormat `<span class="tw-line-through">%s</span>` .OldRef}} {{$oldRef := HTMLFormat `<span class="tw-line-through">%s</span>` .OldRef}}
{{ctx.Locale.Tr "repo.issues.delete_branch_at" $oldRef $createdStr}} {{ctx.Locale.Tr "repo.issues.delete_branch_at" $oldRef $createdStr}}
</span> </span>
@@ -461,15 +455,15 @@
{{else if eq .Type 23}} {{else if eq .Type 23}}
<div class="timeline-item event" id="{{.HashTag}}"> <div class="timeline-item event" id="{{.HashTag}}">
<span class="badge">{{svg "octicon-lock"}}</span> <span class="badge">{{svg "octicon-lock"}}</span>
{{template "shared/user/avatarlink" dict "user" .Poster}} {{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}}
{{if .Content}} {{if .Content}}
<span class="comment-text-line"> <span class="comment-text-line">
{{template "shared/user/authorlink" .Poster}} {{template "repo/issue/view_content/comments_authorlink" dict "comment" .}}
{{ctx.Locale.Tr "repo.issues.lock_with_reason" .Content $createdStr}} {{ctx.Locale.Tr "repo.issues.lock_with_reason" .Content $createdStr}}
</span> </span>
{{else}} {{else}}
<span class="comment-text-line"> <span class="comment-text-line">
{{template "shared/user/authorlink" .Poster}} {{template "repo/issue/view_content/comments_authorlink" dict "comment" .}}
{{ctx.Locale.Tr "repo.issues.lock_no_reason" $createdStr}} {{ctx.Locale.Tr "repo.issues.lock_no_reason" $createdStr}}
</span> </span>
{{end}} {{end}}
@@ -477,18 +471,16 @@
{{else if eq .Type 24}} {{else if eq .Type 24}}
<div class="timeline-item event" id="{{.HashTag}}"> <div class="timeline-item event" id="{{.HashTag}}">
<span class="badge">{{svg "octicon-key"}}</span> <span class="badge">{{svg "octicon-key"}}</span>
{{template "shared/user/avatarlink" dict "user" .Poster}} {{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}}
<span class="comment-text-line"> <span class="comment-text-line">
{{template "shared/user/authorlink" .Poster}} {{template "repo/issue/view_content/comments_authorlink" dict "comment" .}}
{{ctx.Locale.Tr "repo.issues.unlock_comment" $createdStr}} {{ctx.Locale.Tr "repo.issues.unlock_comment" $createdStr}}
</span> </span>
</div> </div>
{{else if eq .Type 25}} {{else if eq .Type 25}}
<div class="timeline-item event"> <div class="timeline-item event">
<span class="badge">{{svg "octicon-git-branch"}}</span> <span class="badge">{{svg "octicon-git-branch"}}</span>
{{if not .OriginalAuthor}} {{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}}
{{template "shared/user/avatarlink" dict "user" .Poster}}
{{end}}
<span class="comment-text-line"> <span class="comment-text-line">
{{template "repo/issue/view_content/comments_authorlink" dict "comment" .}} {{template "repo/issue/view_content/comments_authorlink" dict "comment" .}}
{{ctx.Locale.Tr "repo.pulls.change_target_branch_at" .OldRef .NewRef $createdStr}} {{ctx.Locale.Tr "repo.pulls.change_target_branch_at" .OldRef .NewRef $createdStr}}
@@ -673,9 +665,9 @@
{{else if or (eq .Type 36) (eq .Type 37)}} {{else if or (eq .Type 36) (eq .Type 37)}}
<div class="timeline-item event" id="{{.HashTag}}"> <div class="timeline-item event" id="{{.HashTag}}">
<span class="badge">{{svg "octicon-pin" 16}}</span> <span class="badge">{{svg "octicon-pin" 16}}</span>
{{template "shared/user/avatarlink" dict "user" .Poster}} {{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}}
<span class="comment-text-line"> <span class="comment-text-line">
{{template "shared/user/authorlink" .Poster}} {{template "repo/issue/view_content/comments_authorlink" dict "comment" .}}
{{if eq .Type 36}}{{ctx.Locale.Tr "repo.issues.pin_comment" $createdStr}} {{if eq .Type 36}}{{ctx.Locale.Tr "repo.issues.pin_comment" $createdStr}}
{{else}}{{ctx.Locale.Tr "repo.issues.unpin_comment" $createdStr}}{{end}} {{else}}{{ctx.Locale.Tr "repo.issues.unpin_comment" $createdStr}}{{end}}
</span> </span>
@@ -0,0 +1 @@
{{if not .comment.OriginalAuthor}}{{template "shared/user/avatarlink" dict "user" .comment.Poster}}{{end}}