mirror of
https://github.com/go-gitea/gitea.git
synced 2026-10-23 19:47:07 +09:00
feat(migrations): GraphQL fast path for GitHub migration + rate-limit resilience (#38694)
Adds a batched GraphQL path for GitHub migrations that fetches each issue or pull request together with its comments, reviews, reactions and timeline events instead of the REST N+1 pattern, sharply reducing rate-limit pressure on large repositories. - GraphQL is used for authenticated migrations with issues or pull requests and falls back to REST automatically when it fails - timeline events (close/reopen, lock, rename, label, milestone, merge, branch deletion, pin, auto-merge) are imported as typed comments showing their GitHub author - GitBucket issue migrations no longer stop after the first page, GitBucket sends no Link header Fixes: https://github.com/go-gitea/gitea/issues/13243 Fixes: https://github.com/go-gitea/gitea/issues/31856 Signed-off-by: David E. Lovas <david@rexovas.com> Co-authored-by: bircni <bircni@icloud.com> Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: Giteabot <teabot@gitea.io>
This commit is contained in:
19 files changed
+2115
-141
No files matched your search
@@ -9,7 +9,6 @@ import "time"
|
||||
// Commentable can be commented upon
|
||||
type Commentable interface {
|
||||
Reviewable
|
||||
GetContext() DownloaderContext
|
||||
}
|
||||
|
||||
// Comment is a standard comment information
|
||||
|
||||
@@ -8,6 +8,7 @@ import "time"
|
||||
// Reviewable can be reviewed
|
||||
type Reviewable interface {
|
||||
GetLocalIndex() int64
|
||||
GetContext() DownloaderContext
|
||||
|
||||
// GetForeignIndex presents the foreign index, which could be misused:
|
||||
// For example, if there are 2 Gitea sites: site-A exports a dataset, then site-B imports it:
|
||||
|
||||
@@ -218,6 +218,19 @@ func TestCombineLabelComments(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "migrated events of different original authors",
|
||||
beforeCombined: []*issues_model.Comment{
|
||||
{Type: issues_model.CommentTypeComment, PosterID: 1},
|
||||
{Type: issues_model.CommentTypeLabel, PosterID: 1, OriginalAuthor: "alice", Content: "1", Label: &issues_model.Label{Name: "kind/bug"}},
|
||||
{Type: issues_model.CommentTypeLabel, PosterID: 1, OriginalAuthor: "bob", Content: "1", Label: &issues_model.Label{Name: "kind/ui"}},
|
||||
},
|
||||
afterCombined: []*issues_model.Comment{
|
||||
{Type: issues_model.CommentTypeComment, PosterID: 1},
|
||||
{Type: issues_model.CommentTypeLabel, PosterID: 1, OriginalAuthor: "alice", Content: "1", Label: &issues_model.Label{Name: "kind/bug"}, AddedLabels: []*issues_model.Label{{Name: "kind/bug"}}},
|
||||
{Type: issues_model.CommentTypeLabel, PosterID: 1, OriginalAuthor: "bob", Content: "1", Label: &issues_model.Label{Name: "kind/ui"}, AddedLabels: []*issues_model.Label{{Name: "kind/ui"}}},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "kase 4",
|
||||
beforeCombined: []*issues_model.Comment{
|
||||
|
||||
@@ -231,7 +231,7 @@ func combineLabelComments(issue *issues_model.Issue) {
|
||||
prev = issue.Comments[i-1]
|
||||
}
|
||||
if i == 0 || cur.Type != issues_model.CommentTypeLabel ||
|
||||
(prev != nil && prev.PosterID != cur.PosterID) ||
|
||||
(prev != nil && (prev.PosterID != cur.PosterID || prev.OriginalAuthor != cur.OriginalAuthor)) ||
|
||||
(prev != nil && cur.CreatedUnix-prev.CreatedUnix >= 60) ||
|
||||
(prev != nil && !sameCommentMetaData(prev.CommentMetaData, cur.CommentMetaData)) {
|
||||
if cur.Type == issues_model.CommentTypeLabel && cur.Label != nil {
|
||||
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
Content-Type: application/json; charset=utf-8
|
||||
X-Ratelimit-Limit: 5000
|
||||
X-Ratelimit-Remaining: 4931
|
||||
X-Ratelimit-Reset: 1773089427
|
||||
X-Ratelimit-Resource: core
|
||||
X-Ratelimit-Used: 69
|
||||
Link: <https://api.github.com/repositories/220672974/issues?direction=asc&page=2&per_page=3&sort=created&state=all>; rel="next", <https://api.github.com/repositories/220672974/issues?direction=asc&page=2&per_page=3&sort=created&state=all>; rel="last"
|
||||
|
||||
[{"number":1,"title":"one"},{"number":2,"title":"two"},{"number":3,"title":"three"}]
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
Content-Type: application/json; charset=utf-8
|
||||
X-Ratelimit-Limit: 5000
|
||||
X-Ratelimit-Remaining: 4931
|
||||
X-Ratelimit-Reset: 1773089427
|
||||
X-Ratelimit-Resource: core
|
||||
X-Ratelimit-Used: 69
|
||||
|
||||
[]
|
||||
@@ -82,6 +82,7 @@ func NewGitBucketDownloader(ctx context.Context, baseURL, userName, password, to
|
||||
githubDownloader.maxPerPage = 10
|
||||
githubDownloader.SkipReactions = true
|
||||
githubDownloader.SkipReviews = true
|
||||
githubDownloader.countPaginated = true
|
||||
return &GitBucketDownloader{
|
||||
githubDownloader,
|
||||
}, nil
|
||||
|
||||
@@ -517,6 +517,29 @@ func (g *GiteaLocalUploader) CreateComments(ctx context.Context, comments ...*ba
|
||||
cm.NewRef = fmt.Sprint(comment.Meta["NewRef"])
|
||||
cm.Content = ""
|
||||
}
|
||||
case issues_model.CommentTypeLabel:
|
||||
if name, ok := comment.Meta["LabelName"].(string); ok {
|
||||
if label, ok := g.labels[name]; ok {
|
||||
cm.LabelID = label.ID
|
||||
}
|
||||
}
|
||||
if cm.LabelID == 0 {
|
||||
continue // label_id 0 renders nothing and gitea doctor deletes it
|
||||
}
|
||||
case issues_model.CommentTypeMilestone:
|
||||
if title, ok := comment.Meta["MilestoneTitle"].(string); ok {
|
||||
if removed, _ := comment.Meta["Removed"].(bool); removed {
|
||||
cm.OldMilestoneID = g.milestones[title]
|
||||
} else {
|
||||
cm.MilestoneID = g.milestones[title]
|
||||
}
|
||||
}
|
||||
if cm.MilestoneID == 0 && cm.OldMilestoneID == 0 {
|
||||
continue
|
||||
}
|
||||
case issues_model.CommentTypeDeleteBranch:
|
||||
cm.OldRef = comment.Content
|
||||
cm.Content = ""
|
||||
case issues_model.CommentTypeMergePull:
|
||||
cm.Content = ""
|
||||
case issues_model.CommentTypePRScheduledToAutoMerge, issues_model.CommentTypePRUnScheduledToAutoMerge:
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"xorm.io/builder"
|
||||
)
|
||||
|
||||
func TestGiteaUploadRepo(t *testing.T) {
|
||||
@@ -222,3 +223,35 @@ func TestGiteaUploadRemapExternalUser(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, linkedUser.ID, target.GetUserID())
|
||||
}
|
||||
|
||||
func TestGiteaUploadTimelineLabelAndMilestoneComments(t *testing.T) {
|
||||
unittest.PrepareTestEnv(t)
|
||||
doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
|
||||
issue := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: 1})
|
||||
label := unittest.AssertExistsAndLoadBean(t, &issues_model.Label{ID: 1})
|
||||
milestone := unittest.AssertExistsAndLoadBean(t, &issues_model.Milestone{ID: 1})
|
||||
|
||||
ctx := t.Context()
|
||||
uploader := NewGiteaLocalUploader(ctx, doer, doer.Name, "migrated")
|
||||
uploader.issues[issue.Index] = issue
|
||||
uploader.labels[label.Name] = label
|
||||
uploader.milestones[milestone.Name] = milestone.ID
|
||||
|
||||
require.NoError(t, uploader.CreateComments(ctx,
|
||||
&base.Comment{IssueIndex: issue.Index, CommentType: "label", Content: "1", Meta: map[string]any{"LabelName": label.Name}},
|
||||
&base.Comment{IssueIndex: issue.Index, CommentType: "label", Meta: map[string]any{"LabelName": "deleted-upstream"}},
|
||||
&base.Comment{IssueIndex: issue.Index, CommentType: "milestone", Meta: map[string]any{"MilestoneTitle": milestone.Name}},
|
||||
&base.Comment{IssueIndex: issue.Index, CommentType: "milestone", Meta: map[string]any{"MilestoneTitle": milestone.Name, "Removed": true}},
|
||||
&base.Comment{IssueIndex: issue.Index, CommentType: "delete_branch", Content: "feature"},
|
||||
))
|
||||
|
||||
unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{IssueID: issue.ID, Type: issues_model.CommentTypeLabel, LabelID: label.ID, Content: "1"})
|
||||
unittest.AssertCountByCond(t, "comment", builder.Eq{
|
||||
"issue_id": issue.ID,
|
||||
"type": issues_model.CommentTypeLabel,
|
||||
"label_id": 0,
|
||||
}, 0)
|
||||
unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{IssueID: issue.ID, Type: issues_model.CommentTypeMilestone, MilestoneID: milestone.ID})
|
||||
unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{IssueID: issue.ID, Type: issues_model.CommentTypeMilestone, OldMilestoneID: milestone.ID})
|
||||
assert.Empty(t, unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{IssueID: issue.ID, Type: issues_model.CommentTypeDeleteBranch, OldRef: "feature"}).Content)
|
||||
}
|
||||
+118
-112
@@ -51,7 +51,15 @@ func (f *GithubDownloaderV3Factory) New(ctx context.Context, opts base.MigrateOp
|
||||
|
||||
log.Trace("Create github downloader BaseURL: %s %s/%s", baseURL, oldOwner, oldName)
|
||||
|
||||
return NewGithubDownloaderV3(ctx, baseURL, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName)
|
||||
downloader, err := NewGithubDownloaderV3(ctx, baseURL, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// GitHub rejects anonymous GraphQL requests, and comments only ride along with the issue and PR sweeps
|
||||
if (opts.AuthToken != "" || opts.AuthPassword != "") && (opts.Issues || opts.PullRequests) {
|
||||
return newGithubGraphQLDownloader(downloader, !opts.Comments), nil
|
||||
}
|
||||
return downloader, nil
|
||||
}
|
||||
|
||||
// GitServiceType returns the type of git service
|
||||
@@ -63,21 +71,21 @@ func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType {
|
||||
// from github via APIv3
|
||||
type GithubDownloaderV3 struct {
|
||||
base.NullDownloader
|
||||
clients []*github.Client
|
||||
baseURL string
|
||||
repoOwner string
|
||||
repoName string
|
||||
userName string
|
||||
password string
|
||||
rates []*github.Rate
|
||||
curClientIdx int
|
||||
maxPerPage int
|
||||
SkipReactions bool
|
||||
SkipReviews bool
|
||||
|
||||
clients []*github.Client
|
||||
baseURL string
|
||||
repoOwner string
|
||||
repoName string
|
||||
userName string
|
||||
password string
|
||||
rates []*github.Rate
|
||||
curClientIdx int
|
||||
maxPerPage int
|
||||
SkipReactions bool
|
||||
SkipReviews bool
|
||||
issuePage int
|
||||
issueCursor string
|
||||
nextIssueCursor string
|
||||
countPaginated bool // GitBucket sends no Link header
|
||||
}
|
||||
|
||||
// NewGithubDownloaderV3 creates a github Downloader via github v3 API
|
||||
@@ -460,32 +468,9 @@ func (g *GithubDownloaderV3) GetIssues(ctx context.Context, page, perPage int) (
|
||||
labels = append(labels, convertGithubLabel(l))
|
||||
}
|
||||
|
||||
// get reactions
|
||||
var reactions []*base.Reaction
|
||||
if !g.SkipReactions {
|
||||
for i := 1; ; i++ {
|
||||
g.waitAndPickClient(ctx)
|
||||
res, resp, err := g.getClient().Reactions.ListIssueReactions(ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListReactionOptions{
|
||||
ListOptions: github.ListOptions{
|
||||
Page: i,
|
||||
PerPage: perPage,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
g.setRate(&resp.Rate)
|
||||
if len(res) == 0 {
|
||||
break
|
||||
}
|
||||
for _, reaction := range res {
|
||||
reactions = append(reactions, &base.Reaction{
|
||||
UserID: reaction.User.GetID(),
|
||||
UserName: reaction.User.GetLogin(),
|
||||
Content: reaction.GetContent(),
|
||||
})
|
||||
}
|
||||
}
|
||||
reactions, err := g.getIssueReactions(ctx, issue.GetNumber(), perPage)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
var assignees []string
|
||||
@@ -514,7 +499,14 @@ func (g *GithubDownloaderV3) GetIssues(ctx context.Context, page, perPage int) (
|
||||
}
|
||||
|
||||
g.issuePage, g.issueCursor, g.nextIssueCursor = page, opt.After, resp.After
|
||||
return allIssues, resp.NextPage == 0 && resp.After == "", nil
|
||||
return allIssues, g.isLastPage(resp, len(issues), perPage), nil
|
||||
}
|
||||
|
||||
func (g *GithubDownloaderV3) isLastPage(resp *github.Response, n, perPage int) bool {
|
||||
if g.countPaginated {
|
||||
return n < perPage
|
||||
}
|
||||
return resp.NextPage == 0 && resp.After == ""
|
||||
}
|
||||
|
||||
// SupportGetRepoComments return true if it supports get repo comments
|
||||
@@ -694,83 +686,97 @@ func (g *GithubDownloaderV3) GetPullRequests(ctx context.Context, page, perPage
|
||||
log.Trace("Request get pull requests %d/%d, but in fact get %d", perPage, page, len(prs))
|
||||
g.setRate(&resp.Rate)
|
||||
for _, pr := range prs {
|
||||
labels := make([]*base.Label, 0, len(pr.Labels))
|
||||
for _, l := range pr.Labels {
|
||||
labels = append(labels, convertGithubLabel(l))
|
||||
basePR, err := g.convertGithubPullRequest(ctx, pr, perPage)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
// get reactions
|
||||
var reactions []*base.Reaction
|
||||
if !g.SkipReactions {
|
||||
for i := 1; ; i++ {
|
||||
g.waitAndPickClient(ctx)
|
||||
res, resp, err := g.getClient().Reactions.ListIssueReactions(ctx, g.repoOwner, g.repoName, pr.GetNumber(), &github.ListReactionOptions{
|
||||
ListOptions: github.ListOptions{
|
||||
Page: i,
|
||||
PerPage: perPage,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
g.setRate(&resp.Rate)
|
||||
if len(res) == 0 {
|
||||
break
|
||||
}
|
||||
for _, reaction := range res {
|
||||
reactions = append(reactions, &base.Reaction{
|
||||
UserID: reaction.User.GetID(),
|
||||
UserName: reaction.User.GetLogin(),
|
||||
Content: reaction.GetContent(),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// download patch and saved as tmp file
|
||||
g.waitAndPickClient(ctx)
|
||||
|
||||
allPRs = append(allPRs, &base.PullRequest{
|
||||
Title: pr.GetTitle(),
|
||||
Number: int64(pr.GetNumber()),
|
||||
PosterID: pr.GetUser().GetID(),
|
||||
PosterName: pr.GetUser().GetLogin(),
|
||||
PosterEmail: pr.GetUser().GetEmail(),
|
||||
Content: pr.GetBody(),
|
||||
Milestone: pr.GetMilestone().GetTitle(),
|
||||
State: pr.GetState(),
|
||||
Created: pr.GetCreatedAt().Time,
|
||||
Updated: pr.GetUpdatedAt().Time,
|
||||
Closed: pr.ClosedAt.GetTime(),
|
||||
Labels: labels,
|
||||
Merged: pr.MergedAt != nil,
|
||||
MergeCommitSHA: pr.GetMergeCommitSHA(),
|
||||
MergedTime: pr.MergedAt.GetTime(),
|
||||
IsLocked: pr.ActiveLockReason != nil,
|
||||
Head: base.PullRequestBranch{
|
||||
Ref: pr.GetHead().GetRef(),
|
||||
SHA: pr.GetHead().GetSHA(),
|
||||
OwnerName: pr.GetHead().GetUser().GetLogin(),
|
||||
RepoName: pr.GetHead().GetRepo().GetName(),
|
||||
CloneURL: pr.GetHead().GetRepo().GetCloneURL(), // see below for SECURITY related issues here
|
||||
},
|
||||
Base: base.PullRequestBranch{
|
||||
Ref: pr.GetBase().GetRef(),
|
||||
SHA: pr.GetBase().GetSHA(),
|
||||
RepoName: pr.GetBase().GetRepo().GetName(),
|
||||
OwnerName: pr.GetBase().GetUser().GetLogin(),
|
||||
},
|
||||
PatchURL: pr.GetPatchURL(), // see below for SECURITY related issues here
|
||||
Reactions: reactions,
|
||||
ForeignIndex: int64(*pr.Number),
|
||||
IsDraft: pr.GetDraft(),
|
||||
})
|
||||
allPRs = append(allPRs, basePR)
|
||||
|
||||
// SECURITY: Ensure that the PR is safe
|
||||
_ = CheckAndEnsureSafePR(allPRs[len(allPRs)-1], g.baseURL, g)
|
||||
_ = CheckAndEnsureSafePR(basePR, g.baseURL, g)
|
||||
}
|
||||
|
||||
return allPRs, len(prs) < perPage, nil
|
||||
return allPRs, g.isLastPage(resp, len(prs), perPage), nil
|
||||
}
|
||||
|
||||
func (g *GithubDownloaderV3) convertGithubPullRequest(ctx context.Context, pr *github.PullRequest, perPage int) (*base.PullRequest, error) {
|
||||
labels := make([]*base.Label, 0, len(pr.Labels))
|
||||
for _, l := range pr.Labels {
|
||||
labels = append(labels, convertGithubLabel(l))
|
||||
}
|
||||
|
||||
reactions, err := g.getIssueReactions(ctx, pr.GetNumber(), perPage)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
g.waitAndPickClient(ctx)
|
||||
|
||||
return &base.PullRequest{
|
||||
Title: pr.GetTitle(),
|
||||
Number: int64(pr.GetNumber()),
|
||||
PosterID: pr.GetUser().GetID(),
|
||||
PosterName: pr.GetUser().GetLogin(),
|
||||
PosterEmail: pr.GetUser().GetEmail(),
|
||||
Content: pr.GetBody(),
|
||||
Milestone: pr.GetMilestone().GetTitle(),
|
||||
State: pr.GetState(),
|
||||
Created: pr.GetCreatedAt().Time,
|
||||
Updated: pr.GetUpdatedAt().Time,
|
||||
Closed: pr.ClosedAt.GetTime(),
|
||||
Labels: labels,
|
||||
Merged: pr.MergedAt != nil,
|
||||
MergeCommitSHA: pr.GetMergeCommitSHA(),
|
||||
MergedTime: pr.MergedAt.GetTime(),
|
||||
IsLocked: pr.GetLocked(),
|
||||
Head: base.PullRequestBranch{
|
||||
Ref: pr.GetHead().GetRef(),
|
||||
SHA: pr.GetHead().GetSHA(),
|
||||
OwnerName: pr.GetHead().GetUser().GetLogin(),
|
||||
RepoName: pr.GetHead().GetRepo().GetName(),
|
||||
CloneURL: pr.GetHead().GetRepo().GetCloneURL(), // see below for SECURITY related issues here
|
||||
},
|
||||
Base: base.PullRequestBranch{
|
||||
Ref: pr.GetBase().GetRef(),
|
||||
SHA: pr.GetBase().GetSHA(),
|
||||
RepoName: pr.GetBase().GetRepo().GetName(),
|
||||
OwnerName: pr.GetBase().GetUser().GetLogin(),
|
||||
},
|
||||
PatchURL: pr.GetPatchURL(), // see below for SECURITY related issues here
|
||||
Reactions: reactions,
|
||||
ForeignIndex: int64(*pr.Number),
|
||||
IsDraft: pr.GetDraft(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (g *GithubDownloaderV3) getIssueReactions(ctx context.Context, number, perPage int) ([]*base.Reaction, error) {
|
||||
if g.SkipReactions {
|
||||
return nil, nil
|
||||
}
|
||||
var reactions []*base.Reaction
|
||||
for i := 1; ; i++ {
|
||||
g.waitAndPickClient(ctx)
|
||||
res, resp, err := g.getClient().Reactions.ListIssueReactions(ctx, g.repoOwner, g.repoName, number, &github.ListReactionOptions{
|
||||
ListOptions: github.ListOptions{
|
||||
Page: i,
|
||||
PerPage: perPage,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
g.setRate(&resp.Rate)
|
||||
if len(res) == 0 {
|
||||
return reactions, nil
|
||||
}
|
||||
for _, reaction := range res {
|
||||
reactions = append(reactions, &base.Reaction{
|
||||
UserID: reaction.User.GetID(),
|
||||
UserName: reaction.User.GetLogin(),
|
||||
Content: reaction.GetContent(),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func convertGithubReview(r *github.PullRequestReview) *base.Review {
|
||||
|
||||
@@ -0,0 +1,716 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.dev/modules/egress/policy"
|
||||
"gitea.dev/modules/json"
|
||||
"gitea.dev/modules/log"
|
||||
base "gitea.dev/modules/migration"
|
||||
|
||||
"github.com/google/go-github/v92/github"
|
||||
)
|
||||
|
||||
const (
|
||||
githubGraphQLPageSize = 100
|
||||
githubGraphQLPRPageSize = 20
|
||||
githubGraphQLPointsFloor = 10
|
||||
maxGraphQLRateRetries = 5
|
||||
maxGraphQLTransientRetries = 5
|
||||
graphQLRateResetCap = time.Hour
|
||||
graphQLNodeBatchSize = 100
|
||||
graphQLLabelPageSize = 30
|
||||
graphQLAssigneePageSize = 30
|
||||
graphQLReviewRequestPageSize = 50
|
||||
)
|
||||
|
||||
const (
|
||||
gqlPageInfoFields = "pageInfo{hasNextPage endCursor}"
|
||||
// databaseId is not on the Actor interface, every implementation needs its own fragment
|
||||
gqlActorFields = "__typename login ... on User{databaseId} ... on Bot{databaseId} ... on Organization{databaseId} ... on Mannequin{databaseId}"
|
||||
// teams in the RequestedReviewer union have no login
|
||||
gqlReviewerFields = "__typename ... on User{login databaseId} ... on Bot{login databaseId} ... on Mannequin{login databaseId}"
|
||||
gqlReactionFields = "content user{login databaseId}"
|
||||
)
|
||||
|
||||
var graphQLTransientRetryBase = 2 * time.Second
|
||||
|
||||
var graphQLRateResetFallback = time.Minute // GitHub requires a minute between secondary limit retries.
|
||||
|
||||
// a deterministic 502 for a too-heavy page never succeeds at the same size, so callers shrink it
|
||||
var errGraphQLTransientExhausted = errors.New("github graphql: transient retries exhausted")
|
||||
|
||||
var _ base.Downloader = &GithubGraphQLDownloader{}
|
||||
|
||||
// GithubGraphQLDownloader sweeps issues and pull requests over GraphQL, the embedded REST downloader serves the rest and takes over on failure
|
||||
type GithubGraphQLDownloader struct {
|
||||
*GithubDownloaderV3
|
||||
gqlRates []*github.Rate // GraphQL has a separate rate budget
|
||||
skipComments bool
|
||||
issuesCursor string
|
||||
prCursor string
|
||||
restFallback bool
|
||||
// REST restarts from its first page after a fallback, the migration skips what it already imported
|
||||
restIssuePageOffset int
|
||||
restPRPageOffset int
|
||||
}
|
||||
|
||||
func newGithubGraphQLDownloader(rest *GithubDownloaderV3, skipComments bool) *GithubGraphQLDownloader {
|
||||
return &GithubGraphQLDownloader{GithubDownloaderV3: rest, gqlRates: make([]*github.Rate, len(rest.clients)), skipComments: skipComments}
|
||||
}
|
||||
|
||||
func (g *GithubGraphQLDownloader) GetIssues(ctx context.Context, page, perPage int) ([]*base.Issue, bool, error) {
|
||||
if !g.restFallback {
|
||||
issues, isEnd, err := g.getIssuesGraphQL(ctx, page, perPage)
|
||||
if !g.fallBackToREST(ctx, err) {
|
||||
return issues, isEnd, err
|
||||
}
|
||||
g.restIssuePageOffset = page - 1
|
||||
}
|
||||
return g.GithubDownloaderV3.GetIssues(ctx, page-g.restIssuePageOffset, perPage)
|
||||
}
|
||||
|
||||
func (g *GithubGraphQLDownloader) GetPullRequests(ctx context.Context, page, perPage int) ([]*base.PullRequest, bool, error) {
|
||||
if !g.restFallback {
|
||||
prs, isEnd, err := g.getPullRequestsGraphQL(ctx, page, perPage)
|
||||
if !g.fallBackToREST(ctx, err) {
|
||||
return prs, isEnd, err
|
||||
}
|
||||
g.restPRPageOffset = page - 1
|
||||
}
|
||||
return g.GithubDownloaderV3.GetPullRequests(ctx, page-g.restPRPageOffset, perPage)
|
||||
}
|
||||
|
||||
// fallBackToREST switches to REST for good when GraphQL fails, e.g. on a server or token without GraphQL support
|
||||
func (g *GithubGraphQLDownloader) fallBackToREST(ctx context.Context, err error) bool {
|
||||
if err == nil || ctx.Err() != nil {
|
||||
return false
|
||||
}
|
||||
log.Warn("GitHub GraphQL failed, falling back to REST: %v", err)
|
||||
g.restFallback = true
|
||||
return true
|
||||
}
|
||||
|
||||
// SupportGetRepoComments is false so comments are served per issue from what each sweep page prefetched
|
||||
func (g *GithubGraphQLDownloader) SupportGetRepoComments() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (g *GithubGraphQLDownloader) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) {
|
||||
if prefetched, ok := commentable.GetContext().(*gqlPrefetched); ok {
|
||||
return prefetched.comments, false, nil
|
||||
}
|
||||
return g.GithubDownloaderV3.GetComments(ctx, commentable)
|
||||
}
|
||||
|
||||
func (g *GithubGraphQLDownloader) GetReviews(ctx context.Context, reviewable base.Reviewable) ([]*base.Review, error) {
|
||||
if prefetched, ok := reviewable.GetContext().(*gqlPrefetched); ok {
|
||||
return prefetched.reviews, nil
|
||||
}
|
||||
return g.GithubDownloaderV3.GetReviews(ctx, reviewable)
|
||||
}
|
||||
|
||||
type graphQLRequest struct {
|
||||
Query string `json:"query"`
|
||||
Variables map[string]any `json:"variables"`
|
||||
}
|
||||
|
||||
type graphQLError struct {
|
||||
Type string `json:"type"`
|
||||
Message string `json:"message"`
|
||||
Path []any `json:"path"`
|
||||
}
|
||||
|
||||
func (e graphQLError) Error() string {
|
||||
if e.Type != "" {
|
||||
return e.Type + ": " + e.Message
|
||||
}
|
||||
return e.Message
|
||||
}
|
||||
|
||||
func (g *GithubGraphQLDownloader) graphQLEndpoint() string {
|
||||
restURL := g.clients[0].BaseURL()
|
||||
if host, ok := strings.CutSuffix(restURL, "/api/v3/"); ok {
|
||||
return host + "/api/graphql"
|
||||
}
|
||||
return restURL + "graphql"
|
||||
}
|
||||
|
||||
func (g *GithubGraphQLDownloader) doGraphQL(ctx context.Context, query string, vars map[string]any, out any) error {
|
||||
payload, err := json.Marshal(graphQLRequest{Query: query, Variables: vars})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rateWaits, transientRetries := 0, 0
|
||||
for {
|
||||
if err := g.pickGraphQLClient(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, g.graphQLEndpoint(), bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
resp, err := g.getClient().Client().Do(req)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil || !isTransientTransportError(err) {
|
||||
return err
|
||||
}
|
||||
if err := g.waitGraphQLTransient(ctx, &transientRetries, fmt.Sprintf("request failed: %v", err)); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
g.recordGraphQLRate(resp.Header)
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
detail := readGraphQLErrorBody(resp.Body)
|
||||
resp.Body.Close()
|
||||
switch {
|
||||
case resp.StatusCode >= http.StatusInternalServerError:
|
||||
if err := g.waitGraphQLTransient(ctx, &transientRetries, "status "+resp.Status+detail); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
case isGraphQLThrottled(resp, detail):
|
||||
if rateWaits < maxGraphQLRateRetries*len(g.clients) {
|
||||
rateWaits++
|
||||
g.restGraphQLClient(resp.Header)
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("doGraphQL: rate limited, status %s%s", resp.Status, detail)
|
||||
default:
|
||||
return fmt.Errorf("doGraphQL: unexpected status %s%s", resp.Status, detail)
|
||||
}
|
||||
}
|
||||
|
||||
envelope := struct {
|
||||
Data any `json:"data"`
|
||||
Errors []graphQLError `json:"errors"`
|
||||
}{Data: out}
|
||||
decodeErr := json.NewDecoder(resp.Body).Decode(&envelope)
|
||||
resp.Body.Close()
|
||||
if decodeErr != nil {
|
||||
if err := g.waitGraphQLTransient(ctx, &transientRetries, fmt.Sprintf("truncated response: %v", decodeErr)); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if !onlyUnresolvedNodes(envelope.Errors) {
|
||||
if isGraphQLRateLimited(envelope.Errors) && rateWaits < maxGraphQLRateRetries*len(g.clients) {
|
||||
rateWaits++
|
||||
g.restGraphQLClient(resp.Header)
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("doGraphQL: %w", envelope.Errors[0])
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
const maxGraphQLErrorBody = 512
|
||||
|
||||
func readGraphQLErrorBody(body io.Reader) string {
|
||||
buf, err := io.ReadAll(io.LimitReader(body, maxGraphQLErrorBody))
|
||||
if err != nil || len(buf) == 0 {
|
||||
return ""
|
||||
}
|
||||
return ": " + strings.Join(strings.Fields(string(buf)), " ")
|
||||
}
|
||||
|
||||
func isGraphQLThrottled(resp *http.Response, body string) bool {
|
||||
switch resp.StatusCode {
|
||||
case http.StatusTooManyRequests:
|
||||
return true
|
||||
case http.StatusForbidden:
|
||||
return resp.Header.Get("Retry-After") != "" ||
|
||||
resp.Header.Get(github.HeaderRateRemaining) == "0" ||
|
||||
strings.Contains(strings.ToLower(body), "secondary rate limit")
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isTransientTransportError(err error) bool {
|
||||
if errors.Is(err, policy.ErrDenied) {
|
||||
return false
|
||||
}
|
||||
if _, ok := errors.AsType[*tls.CertificateVerificationError](err); ok {
|
||||
return false
|
||||
}
|
||||
if dnsErr, ok := errors.AsType[*net.DNSError](err); ok {
|
||||
return !dnsErr.IsNotFound
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (g *GithubGraphQLDownloader) waitGraphQLTransient(ctx context.Context, attempt *int, reason string) error {
|
||||
if *attempt >= maxGraphQLTransientRetries {
|
||||
return fmt.Errorf("%w: %s", errGraphQLTransientExhausted, reason)
|
||||
}
|
||||
wait := graphQLTransientRetryBase << *attempt
|
||||
*attempt++
|
||||
log.Warn("github graphql: transient failure (%s), retrying in %s (attempt %d/%d)",
|
||||
reason, wait, *attempt, maxGraphQLTransientRetries)
|
||||
timer := time.NewTimer(wait)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func onlyUnresolvedNodes(errs []graphQLError) bool {
|
||||
return !slices.ContainsFunc(errs, func(e graphQLError) bool {
|
||||
return e.Type != "NOT_FOUND" || len(e.Path) == 0 || e.Path[0] != "nodes"
|
||||
})
|
||||
}
|
||||
|
||||
func isGraphQLRateLimited(errs []graphQLError) bool {
|
||||
return slices.ContainsFunc(errs, func(e graphQLError) bool { return e.Type == "RATE_LIMITED" })
|
||||
}
|
||||
|
||||
func isGraphQLTooHeavy(err error) bool {
|
||||
var ge graphQLError
|
||||
return errors.Is(err, errGraphQLTransientExhausted) || errors.As(err, &ge) && ge.Type == "RESOURCE_LIMITS_EXCEEDED"
|
||||
}
|
||||
|
||||
func (g *GithubGraphQLDownloader) pickGraphQLClient(ctx context.Context) error {
|
||||
for {
|
||||
best, soonest := -1, -1
|
||||
for i, rate := range g.gqlRates {
|
||||
if rate == nil || !time.Now().Before(rate.Reset.Time) {
|
||||
g.curClientIdx = i
|
||||
return nil
|
||||
}
|
||||
if rate.Remaining > githubGraphQLPointsFloor && (best < 0 || rate.Remaining > g.gqlRates[best].Remaining) {
|
||||
best = i
|
||||
}
|
||||
if soonest < 0 || rate.Reset.Before(g.gqlRates[soonest].Reset.Time) {
|
||||
soonest = i
|
||||
}
|
||||
}
|
||||
if best >= 0 {
|
||||
g.curClientIdx = best
|
||||
return nil
|
||||
}
|
||||
wait := min(time.Until(g.gqlRates[soonest].Reset.Time), graphQLRateResetCap)
|
||||
log.Info("github graphql: every token's points budget is spent, sleeping %s until the earliest reset", wait.Round(time.Second))
|
||||
timer := time.NewTimer(wait)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
g.gqlRates[soonest] = nil // the reset may lie beyond the capped sleep
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (g *GithubGraphQLDownloader) recordGraphQLRate(header http.Header) {
|
||||
remaining, err := strconv.Atoi(header.Get(github.HeaderRateRemaining))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
rate := &github.Rate{Remaining: remaining}
|
||||
if epoch, err := strconv.ParseInt(header.Get(github.HeaderRateReset), 10, 64); err == nil {
|
||||
rate.Reset = github.Timestamp{Time: time.Unix(epoch, 0)}
|
||||
}
|
||||
g.gqlRates[g.curClientIdx] = rate
|
||||
}
|
||||
|
||||
func (g *GithubGraphQLDownloader) restGraphQLClient(header http.Header) {
|
||||
wait := graphQLRateResetWait(header)
|
||||
log.Info("github graphql: token %d throttled, resting it for %s", g.curClientIdx, wait.Round(time.Second))
|
||||
g.gqlRates[g.curClientIdx] = &github.Rate{Reset: github.Timestamp{Time: time.Now().Add(wait)}}
|
||||
}
|
||||
|
||||
func isPrimaryBudgetSpent(header http.Header) bool {
|
||||
remaining := header.Get(github.HeaderRateRemaining)
|
||||
return remaining == "" || remaining == "0"
|
||||
}
|
||||
|
||||
func graphQLRateResetWait(header http.Header) time.Duration {
|
||||
wait := graphQLRateResetFallback
|
||||
if ra := header.Get("Retry-After"); ra != "" {
|
||||
if secs, err := strconv.Atoi(strings.TrimSpace(ra)); err == nil && secs > 0 {
|
||||
wait = time.Duration(secs) * time.Second
|
||||
}
|
||||
} else if reset := header.Get(github.HeaderRateReset); reset != "" && isPrimaryBudgetSpent(header) {
|
||||
if epoch, err := strconv.ParseInt(reset, 10, 64); err == nil {
|
||||
if d := time.Until(time.Unix(epoch, 0)); d > 0 {
|
||||
wait = d
|
||||
}
|
||||
}
|
||||
}
|
||||
return min(wait, graphQLRateResetCap)
|
||||
}
|
||||
|
||||
// graphQLContentQuery orders by creation since a mutable sort key skips rows, nested comment reactions exceed GitHub's node limit
|
||||
func (g *GithubGraphQLDownloader) graphQLContentQuery(connection, prFields, prDiscussion string) string {
|
||||
fields := prFields
|
||||
if !g.SkipReactions {
|
||||
fields += " reactions(first:100){" + gqlPageInfoFields + " nodes{" + gqlReactionFields + "}}"
|
||||
}
|
||||
if !g.skipComments {
|
||||
fields += " comments(first:100){" + gqlPageInfoFields + " nodes{id databaseId body createdAt updatedAt author{" + gqlActorFields + "}}}" + prDiscussion
|
||||
}
|
||||
return fmt.Sprintf(`
|
||||
query($owner:String!,$name:String!,$cursor:String,$first:Int!){
|
||||
repository(owner:$owner,name:$name){
|
||||
%[1]s(first:$first,after:$cursor,orderBy:{field:CREATED_AT,direction:ASC}){
|
||||
%[2]s
|
||||
nodes{
|
||||
id number title body state createdAt updatedAt closedAt locked
|
||||
author{%[3]s}
|
||||
milestone{title}
|
||||
labels(first:%[4]d){%[2]s nodes{name color description}}
|
||||
assignees(first:%[5]d){%[2]s nodes{login}}
|
||||
%[6]s
|
||||
}
|
||||
}
|
||||
}
|
||||
}`, connection, gqlPageInfoFields, gqlActorFields, graphQLLabelPageSize, graphQLAssigneePageSize, fields)
|
||||
}
|
||||
|
||||
func (g *GithubGraphQLDownloader) graphQLIssuesQuery() string {
|
||||
return g.graphQLContentQuery("issues", "", "")
|
||||
}
|
||||
|
||||
func nodeConnQuery(onType, connection, nodeFields string) string {
|
||||
return `query($id:ID!,$cursor:String){node(id:$id){... on ` + onType + `{conn:` + connection +
|
||||
`(first:100,after:$cursor){` + gqlPageInfoFields + ` nodes{` + nodeFields + `}}}}}`
|
||||
}
|
||||
|
||||
var (
|
||||
graphQLLabelsSweepQuery = nodeConnQuery("Labelable", "labels", "name color description")
|
||||
graphQLAssigneesSweepQuery = nodeConnQuery("Assignable", "assignees", "login")
|
||||
graphQLReactionsSweepQuery = nodeConnQuery("Reactable", "reactions", gqlReactionFields)
|
||||
)
|
||||
|
||||
const graphQLNodeReactionsQuery = `query($ids:[ID!]!){nodes(ids:$ids){id ... on Reactable{reactions(first:100){` +
|
||||
gqlPageInfoFields + ` nodes{` + gqlReactionFields + `}}}}}`
|
||||
|
||||
type gqlPageInfo struct {
|
||||
HasNextPage bool `json:"hasNextPage"`
|
||||
EndCursor string `json:"endCursor"`
|
||||
}
|
||||
|
||||
type gqlConn[T any] struct {
|
||||
PageInfo gqlPageInfo `json:"pageInfo"`
|
||||
Nodes []T `json:"nodes"`
|
||||
}
|
||||
|
||||
type gqlActor struct {
|
||||
Typename string `json:"__typename"`
|
||||
Login string `json:"login"`
|
||||
DatabaseID int64 `json:"databaseId"`
|
||||
}
|
||||
|
||||
func (a *gqlActor) UnmarshalJSON(data []byte) error {
|
||||
type plain gqlActor
|
||||
if err := json.Unmarshal(data, (*plain)(a)); err != nil {
|
||||
return err
|
||||
}
|
||||
switch {
|
||||
case a.Typename == "" && a.Login == "":
|
||||
a.Login = "ghost" // Match REST's ghost login with an unset ID for GHES.
|
||||
case a.Typename == "Bot":
|
||||
a.Login += "[bot]"
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type gqlCommit struct {
|
||||
OID string `json:"oid"`
|
||||
}
|
||||
|
||||
type gqlReaction struct {
|
||||
Content string `json:"content"`
|
||||
User gqlActor `json:"user"`
|
||||
}
|
||||
|
||||
type gqlComment struct {
|
||||
ID string `json:"id"`
|
||||
DatabaseID int64 `json:"databaseId"`
|
||||
Body string `json:"body"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
Author gqlActor `json:"author"`
|
||||
}
|
||||
|
||||
type gqlLabel struct {
|
||||
Name string `json:"name"`
|
||||
Color string `json:"color"`
|
||||
Description string `json:"description"`
|
||||
}
|
||||
|
||||
type gqlIssue struct {
|
||||
ID string `json:"id"`
|
||||
Number int64 `json:"number"`
|
||||
Title string `json:"title"`
|
||||
Body string `json:"body"`
|
||||
State string `json:"state"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
ClosedAt *time.Time `json:"closedAt"`
|
||||
Locked bool `json:"locked"`
|
||||
Author gqlActor `json:"author"`
|
||||
Milestone *struct {
|
||||
Title string `json:"title"`
|
||||
} `json:"milestone"`
|
||||
Labels gqlConn[gqlLabel] `json:"labels"`
|
||||
Assignees gqlConn[gqlActor] `json:"assignees"`
|
||||
Reactions gqlConn[gqlReaction] `json:"reactions"`
|
||||
Comments gqlConn[gqlComment] `json:"comments"`
|
||||
}
|
||||
|
||||
type gqlIssuesResponse struct {
|
||||
Repository struct {
|
||||
Issues gqlConn[gqlIssue] `json:"issues"`
|
||||
} `json:"repository"`
|
||||
}
|
||||
|
||||
type gqlPrefetched struct {
|
||||
index int64
|
||||
comments []*base.Comment
|
||||
commentIDs []string
|
||||
reviews []*base.Review
|
||||
}
|
||||
|
||||
func (g *GithubGraphQLDownloader) getIssuesGraphQL(ctx context.Context, page, perPage int) ([]*base.Issue, bool, error) {
|
||||
if page <= 1 {
|
||||
g.issuesCursor = ""
|
||||
}
|
||||
var resp gqlIssuesResponse
|
||||
if err := g.doGraphQLPage(ctx, g.graphQLIssuesQuery(), g.issuesCursor, graphQLPageSize(perPage, githubGraphQLPageSize), &resp); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
conn := resp.Repository.Issues
|
||||
prefetched := map[string]*gqlPrefetched{}
|
||||
issues := make([]*base.Issue, 0, len(conn.Nodes))
|
||||
for i := range conn.Nodes {
|
||||
issue, err := g.convertGraphQLIssue(ctx, &conn.Nodes[i], prefetched)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
issues = append(issues, issue)
|
||||
}
|
||||
if err := g.enrichGraphQLPage(ctx, prefetched); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
g.issuesCursor = conn.PageInfo.EndCursor // only now, a failed page is retried from the same cursor
|
||||
return issues, !conn.PageInfo.HasNextPage, nil
|
||||
}
|
||||
|
||||
func (g *GithubGraphQLDownloader) convertGraphQLIssue(ctx context.Context, node *gqlIssue, prefetchedByNodeID map[string]*gqlPrefetched) (*base.Issue, error) {
|
||||
labels, err := allNodes(ctx, g, graphQLLabelsSweepQuery, node.ID, node.Labels)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
assignees, err := allNodes(ctx, g, graphQLAssigneesSweepQuery, node.ID, node.Assignees)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reactions, err := g.allReactions(ctx, node.ID, node.Reactions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
issue := &base.Issue{
|
||||
Number: node.Number,
|
||||
Title: node.Title,
|
||||
Content: node.Body,
|
||||
PosterID: node.Author.DatabaseID,
|
||||
PosterName: node.Author.Login,
|
||||
State: strings.ToLower(node.State),
|
||||
Created: node.CreatedAt,
|
||||
Updated: node.UpdatedAt,
|
||||
Closed: node.ClosedAt,
|
||||
IsLocked: node.Locked,
|
||||
Labels: make([]*base.Label, 0, len(labels)),
|
||||
Reactions: reactions,
|
||||
ForeignIndex: node.Number,
|
||||
}
|
||||
if node.Milestone != nil {
|
||||
issue.Milestone = node.Milestone.Title
|
||||
}
|
||||
for _, label := range labels {
|
||||
issue.Labels = append(issue.Labels, &base.Label{Name: label.Name, Color: label.Color, Description: label.Description})
|
||||
}
|
||||
for _, assignee := range assignees {
|
||||
issue.Assignees = append(issue.Assignees, assignee.Login)
|
||||
}
|
||||
if g.skipComments {
|
||||
return issue, nil
|
||||
}
|
||||
|
||||
prefetched := &gqlPrefetched{index: node.Number}
|
||||
issue.Context = prefetched
|
||||
prefetchedByNodeID[node.ID] = prefetched
|
||||
if node.Comments.PageInfo.HasNextPage {
|
||||
prefetched.comments, err = g.getComments(ctx, issue)
|
||||
return issue, err
|
||||
}
|
||||
for _, comment := range node.Comments.Nodes {
|
||||
prefetched.commentIDs = append(prefetched.commentIDs, comment.ID)
|
||||
prefetched.comments = append(prefetched.comments, &base.Comment{
|
||||
IssueIndex: node.Number,
|
||||
Index: comment.DatabaseID,
|
||||
PosterID: comment.Author.DatabaseID,
|
||||
PosterName: comment.Author.Login,
|
||||
Content: comment.Body,
|
||||
Created: comment.CreatedAt,
|
||||
Updated: comment.UpdatedAt,
|
||||
})
|
||||
}
|
||||
return issue, nil
|
||||
}
|
||||
|
||||
func (g *GithubGraphQLDownloader) enrichGraphQLPage(ctx context.Context, prefetchedByNodeID map[string]*gqlPrefetched) error {
|
||||
if !g.SkipReactions {
|
||||
comments := map[string]*base.Comment{}
|
||||
for _, prefetched := range prefetchedByNodeID {
|
||||
for i, id := range prefetched.commentIDs {
|
||||
comments[id] = prefetched.comments[i]
|
||||
}
|
||||
}
|
||||
for batch := range slices.Chunk(slices.Sorted(maps.Keys(comments)), graphQLNodeBatchSize) {
|
||||
reactions, err := g.fetchNodeReactions(ctx, batch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for id, reaction := range reactions {
|
||||
comments[id].Reactions = reaction
|
||||
}
|
||||
}
|
||||
}
|
||||
return g.attachTimelineEvents(ctx, prefetchedByNodeID)
|
||||
}
|
||||
|
||||
func graphQLPageSize(perPage, maxSize int) int {
|
||||
if perPage <= 0 {
|
||||
return maxSize
|
||||
}
|
||||
return min(perPage, maxSize) // Keep pages within the caller's database batch size.
|
||||
}
|
||||
|
||||
func (g *GithubGraphQLDownloader) doGraphQLPage(ctx context.Context, query, cursor string, pageSize int, out any) error {
|
||||
vars := map[string]any{"owner": g.repoOwner, "name": g.repoName}
|
||||
if cursor != "" {
|
||||
vars["cursor"] = cursor
|
||||
}
|
||||
for {
|
||||
vars["first"] = pageSize
|
||||
err := g.doGraphQL(ctx, query, vars, out)
|
||||
if err == nil || !isGraphQLTooHeavy(err) || pageSize <= 1 {
|
||||
return err
|
||||
}
|
||||
pageSize /= 2
|
||||
log.Warn("github graphql: page too heavy (%v), retrying with %d", err, pageSize)
|
||||
}
|
||||
}
|
||||
|
||||
func allNodes[T any](ctx context.Context, g *GithubGraphQLDownloader, sweepQuery, nodeID string, conn gqlConn[T]) ([]T, error) {
|
||||
nodes := conn.Nodes
|
||||
for page := conn.PageInfo; page.HasNextPage; {
|
||||
var resp struct {
|
||||
Node struct {
|
||||
Conn gqlConn[T] `json:"conn"`
|
||||
} `json:"node"`
|
||||
}
|
||||
if err := g.doGraphQL(ctx, sweepQuery, map[string]any{"id": nodeID, "cursor": page.EndCursor}, &resp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodes = append(nodes, resp.Node.Conn.Nodes...)
|
||||
page = resp.Node.Conn.PageInfo
|
||||
}
|
||||
return nodes, nil
|
||||
}
|
||||
|
||||
func (g *GithubGraphQLDownloader) allReactions(ctx context.Context, nodeID string, conn gqlConn[gqlReaction]) ([]*base.Reaction, error) {
|
||||
nodes, err := allNodes(ctx, g, graphQLReactionsSweepQuery, nodeID, conn)
|
||||
if err != nil || len(nodes) == 0 {
|
||||
return nil, err
|
||||
}
|
||||
reactions := make([]*base.Reaction, 0, len(nodes))
|
||||
for _, reaction := range nodes {
|
||||
reactions = append(reactions, &base.Reaction{
|
||||
UserID: reaction.User.DatabaseID,
|
||||
UserName: reaction.User.Login,
|
||||
Content: graphQLReactionContent(reaction.Content),
|
||||
})
|
||||
}
|
||||
return reactions, nil
|
||||
}
|
||||
|
||||
func (g *GithubGraphQLDownloader) fetchNodeReactions(ctx context.Context, ids []string) (map[string][]*base.Reaction, error) {
|
||||
var resp struct {
|
||||
Nodes []struct {
|
||||
ID string `json:"id"`
|
||||
Reactions gqlConn[gqlReaction] `json:"reactions"`
|
||||
} `json:"nodes"`
|
||||
}
|
||||
if err := g.doGraphQL(ctx, graphQLNodeReactionsQuery, map[string]any{"ids": ids}, &resp); err != nil {
|
||||
if isGraphQLTooHeavy(err) && len(ids) > 1 {
|
||||
return fetchHalves(ctx, ids, g.fetchNodeReactions)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[string][]*base.Reaction, len(resp.Nodes))
|
||||
for _, node := range resp.Nodes {
|
||||
if node.ID == "" {
|
||||
continue
|
||||
}
|
||||
reactions, err := g.allReactions(ctx, node.ID, node.Reactions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[node.ID] = reactions
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func fetchHalves[V any](ctx context.Context, ids []string, fetch func(context.Context, []string) (map[string]V, error)) (map[string]V, error) {
|
||||
left, err := fetch(ctx, ids[:len(ids)/2])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
right, err := fetch(ctx, ids[len(ids)/2:])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
maps.Copy(left, right)
|
||||
return left, nil
|
||||
}
|
||||
|
||||
func graphQLReactionContent(enum string) string {
|
||||
switch enum {
|
||||
case "THUMBS_UP":
|
||||
return "+1"
|
||||
case "THUMBS_DOWN":
|
||||
return "-1"
|
||||
default:
|
||||
return strings.ToLower(enum)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
base "gitea.dev/modules/migration"
|
||||
)
|
||||
|
||||
const graphQLPRFields = "mergedAt isDraft mergeCommit{oid} headRefName headRefOid baseRefName baseRefOid headRepository{name url} headRepositoryOwner{login} baseRepository{name owner{login}}"
|
||||
|
||||
var graphQLPRDiscussion = fmt.Sprintf(`
|
||||
reviews(first:50){%[1]s nodes{
|
||||
databaseId state body createdAt submittedAt author{%[2]s} commit{oid}
|
||||
comments(first:50){%[1]s nodes{databaseId body path diffHunk position commit{oid} author{%[2]s} createdAt updatedAt replyTo{databaseId}}}
|
||||
}}
|
||||
reviewRequests(first:%[3]d){%[1]s nodes{requestedReviewer{%[4]s}}}`,
|
||||
gqlPageInfoFields, gqlActorFields, graphQLReviewRequestPageSize, gqlReviewerFields)
|
||||
|
||||
var graphQLReviewRequestsSweepQuery = nodeConnQuery("PullRequest", "reviewRequests", "requestedReviewer{"+gqlReviewerFields+"}")
|
||||
|
||||
func (g *GithubGraphQLDownloader) graphQLPullRequestsQuery() string {
|
||||
return g.graphQLContentQuery("pullRequests", graphQLPRFields, graphQLPRDiscussion)
|
||||
}
|
||||
|
||||
type gqlRepoRef struct {
|
||||
Name string `json:"name"`
|
||||
URL string `json:"url"`
|
||||
Owner gqlActor `json:"owner"`
|
||||
}
|
||||
|
||||
type gqlReviewComment struct {
|
||||
DatabaseID int64 `json:"databaseId"`
|
||||
Body string `json:"body"`
|
||||
Path string `json:"path"`
|
||||
DiffHunk string `json:"diffHunk"`
|
||||
Position int `json:"position"`
|
||||
Commit *gqlCommit `json:"commit"`
|
||||
Author gqlActor `json:"author"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
ReplyTo *struct {
|
||||
DatabaseID int64 `json:"databaseId"`
|
||||
} `json:"replyTo"`
|
||||
}
|
||||
|
||||
type gqlReview struct {
|
||||
DatabaseID int64 `json:"databaseId"`
|
||||
State string `json:"state"`
|
||||
Body string `json:"body"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
SubmittedAt *time.Time `json:"submittedAt"`
|
||||
Author gqlActor `json:"author"`
|
||||
Commit *gqlCommit `json:"commit"`
|
||||
Comments gqlConn[gqlReviewComment] `json:"comments"`
|
||||
}
|
||||
|
||||
type gqlReviewRequest struct {
|
||||
RequestedReviewer *gqlActor `json:"requestedReviewer"`
|
||||
}
|
||||
|
||||
type gqlPullRequest struct {
|
||||
gqlIssue
|
||||
MergedAt *time.Time `json:"mergedAt"`
|
||||
IsDraft bool `json:"isDraft"`
|
||||
MergeCommit *gqlCommit `json:"mergeCommit"`
|
||||
HeadRefName string `json:"headRefName"`
|
||||
HeadRefOID string `json:"headRefOid"`
|
||||
BaseRefName string `json:"baseRefName"`
|
||||
BaseRefOID string `json:"baseRefOid"`
|
||||
HeadRepo *gqlRepoRef `json:"headRepository"`
|
||||
HeadRepoOwner *gqlActor `json:"headRepositoryOwner"`
|
||||
BaseRepo *gqlRepoRef `json:"baseRepository"`
|
||||
Reviews gqlConn[gqlReview] `json:"reviews"`
|
||||
ReviewRequests gqlConn[gqlReviewRequest] `json:"reviewRequests"`
|
||||
}
|
||||
|
||||
type gqlPullRequestsResponse struct {
|
||||
Repository struct {
|
||||
PullRequests gqlConn[gqlPullRequest] `json:"pullRequests"`
|
||||
} `json:"repository"`
|
||||
}
|
||||
|
||||
func (g *GithubGraphQLDownloader) getPullRequestsGraphQL(ctx context.Context, page, perPage int) ([]*base.PullRequest, bool, error) {
|
||||
if page <= 1 {
|
||||
g.prCursor = ""
|
||||
}
|
||||
var resp gqlPullRequestsResponse
|
||||
if err := g.doGraphQLPage(ctx, g.graphQLPullRequestsQuery(), g.prCursor, graphQLPageSize(perPage, githubGraphQLPRPageSize), &resp); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
conn := resp.Repository.PullRequests
|
||||
prefetched := map[string]*gqlPrefetched{}
|
||||
prs := make([]*base.PullRequest, 0, len(conn.Nodes))
|
||||
for i := range conn.Nodes {
|
||||
pr, err := g.convertGraphQLPullRequest(ctx, &conn.Nodes[i], prefetched)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
_ = CheckAndEnsureSafePR(pr, g.baseURL, g) // SECURITY: like the REST path
|
||||
prs = append(prs, pr)
|
||||
}
|
||||
if err := g.enrichGraphQLPage(ctx, prefetched); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
g.prCursor = conn.PageInfo.EndCursor // only now, a failed page is retried from the same cursor
|
||||
return prs, !conn.PageInfo.HasNextPage, nil
|
||||
}
|
||||
|
||||
func (g *GithubGraphQLDownloader) convertGraphQLPullRequest(ctx context.Context, node *gqlPullRequest, prefetchedByNodeID map[string]*gqlPrefetched) (*base.PullRequest, error) {
|
||||
issue, err := g.convertGraphQLIssue(ctx, &node.gqlIssue, prefetchedByNodeID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if prefetched, ok := issue.Context.(*gqlPrefetched); ok {
|
||||
if prefetched.reviews, err = g.prefetchReviews(ctx, node); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
pr := &base.PullRequest{
|
||||
Number: issue.Number,
|
||||
Title: issue.Title,
|
||||
PosterID: issue.PosterID,
|
||||
PosterName: issue.PosterName,
|
||||
Content: issue.Content,
|
||||
Milestone: issue.Milestone,
|
||||
State: "closed", // REST reports a merged PR as closed
|
||||
Created: issue.Created,
|
||||
Updated: issue.Updated,
|
||||
Closed: issue.Closed,
|
||||
Labels: issue.Labels,
|
||||
Assignees: issue.Assignees,
|
||||
IsLocked: issue.IsLocked,
|
||||
Reactions: issue.Reactions,
|
||||
ForeignIndex: issue.ForeignIndex,
|
||||
Context: issue.Context,
|
||||
PatchURL: fmt.Sprintf("%s/%s/%s/pull/%d.patch", g.baseURL, g.repoOwner, g.repoName, node.Number),
|
||||
Merged: node.MergedAt != nil || node.State == "MERGED",
|
||||
MergedTime: node.MergedAt,
|
||||
IsDraft: node.IsDraft,
|
||||
Head: base.PullRequestBranch{Ref: node.HeadRefName, SHA: node.HeadRefOID},
|
||||
Base: base.PullRequestBranch{Ref: node.BaseRefName, SHA: node.BaseRefOID},
|
||||
}
|
||||
if node.State == "OPEN" {
|
||||
pr.State = "open"
|
||||
}
|
||||
if node.MergeCommit != nil {
|
||||
pr.MergeCommitSHA = node.MergeCommit.OID
|
||||
}
|
||||
if node.HeadRepoOwner != nil {
|
||||
pr.Head.OwnerName = node.HeadRepoOwner.Login
|
||||
}
|
||||
if node.HeadRepo != nil {
|
||||
pr.Head.RepoName = node.HeadRepo.Name
|
||||
pr.Head.CloneURL = node.HeadRepo.URL + ".git"
|
||||
}
|
||||
if node.BaseRepo != nil {
|
||||
pr.Base.RepoName = node.BaseRepo.Name
|
||||
pr.Base.OwnerName = node.BaseRepo.Owner.Login
|
||||
}
|
||||
return pr, nil
|
||||
}
|
||||
|
||||
func (g *GithubGraphQLDownloader) prefetchReviews(ctx context.Context, node *gqlPullRequest) ([]*base.Review, error) {
|
||||
if node.Reviews.PageInfo.HasNextPage || slices.ContainsFunc(node.Reviews.Nodes, func(r gqlReview) bool { return r.Comments.PageInfo.HasNextPage }) {
|
||||
return g.GetReviews(ctx, &base.PullRequest{Number: node.Number, ForeignIndex: node.Number})
|
||||
}
|
||||
requests, err := allNodes(ctx, g, graphQLReviewRequestsSweepQuery, node.ID, node.ReviewRequests)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return convertGraphQLReviews(node, requests), nil
|
||||
}
|
||||
|
||||
func convertGraphQLReviews(node *gqlPullRequest, requests []gqlReviewRequest) []*base.Review {
|
||||
reviews := make([]*base.Review, 0, len(node.Reviews.Nodes)+len(requests))
|
||||
for _, sourceReview := range node.Reviews.Nodes {
|
||||
review := &base.Review{
|
||||
ID: sourceReview.DatabaseID,
|
||||
IssueIndex: node.Number,
|
||||
ReviewerID: sourceReview.Author.DatabaseID,
|
||||
ReviewerName: sourceReview.Author.Login,
|
||||
Content: sourceReview.Body,
|
||||
CreatedAt: sourceReview.CreatedAt,
|
||||
State: sourceReview.State,
|
||||
}
|
||||
if sourceReview.SubmittedAt != nil {
|
||||
review.CreatedAt = *sourceReview.SubmittedAt
|
||||
}
|
||||
if sourceReview.Commit != nil {
|
||||
review.CommitID = sourceReview.Commit.OID
|
||||
}
|
||||
for _, sourceComment := range sourceReview.Comments.Nodes {
|
||||
comment := &base.ReviewComment{
|
||||
ID: sourceComment.DatabaseID,
|
||||
Content: sourceComment.Body,
|
||||
TreePath: sourceComment.Path,
|
||||
DiffHunk: sourceComment.DiffHunk,
|
||||
Position: sourceComment.Position,
|
||||
PosterID: sourceComment.Author.DatabaseID,
|
||||
CreatedAt: sourceComment.CreatedAt,
|
||||
UpdatedAt: sourceComment.UpdatedAt,
|
||||
}
|
||||
if sourceComment.ReplyTo != nil {
|
||||
comment.InReplyTo = sourceComment.ReplyTo.DatabaseID
|
||||
}
|
||||
if sourceComment.Commit != nil {
|
||||
comment.CommitID = sourceComment.Commit.OID
|
||||
}
|
||||
review.Comments = append(review.Comments, comment)
|
||||
}
|
||||
reviews = append(reviews, review)
|
||||
}
|
||||
for _, request := range requests {
|
||||
if reviewer := request.RequestedReviewer; reviewer != nil && reviewer.Login != "" {
|
||||
reviews = append(reviews, &base.Review{
|
||||
IssueIndex: node.Number,
|
||||
ReviewerID: reviewer.DatabaseID,
|
||||
ReviewerName: reviewer.Login,
|
||||
State: base.ReviewStateRequestReview,
|
||||
})
|
||||
}
|
||||
}
|
||||
return reviews
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dev/modules/json"
|
||||
base "gitea.dev/modules/migration"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestGraphQLPullRequestParsing(t *testing.T) {
|
||||
var response gqlPullRequestsResponse
|
||||
require.NoError(t, json.Unmarshal([]byte(`{"repository":{"pullRequests":{"nodes":[{
|
||||
"number":7,"title":"add feature","body":"does the thing","state":"MERGED",
|
||||
"createdAt":"2023-05-01T00:00:00Z","updatedAt":"2023-06-01T00:00:00Z",
|
||||
"closedAt":"2023-06-01T00:00:00Z","mergedAt":"2023-06-01T00:00:00Z",
|
||||
"author":{"login":"contributor","databaseId":4242},"isDraft":true,
|
||||
"mergeCommit":{"oid":"abc123"},"headRefName":"feature-x","headRefOid":"deadbeef",
|
||||
"baseRefName":"main","baseRefOid":"cafebabe",
|
||||
"headRepository":{"name":"gitea","url":"https://github.com/contributor/gitea"},
|
||||
"headRepositoryOwner":{"login":"contributor"},
|
||||
"baseRepository":{"name":"gitea","owner":{"login":"go-gitea"}},
|
||||
"reviews":{"nodes":[{
|
||||
"databaseId":900,"state":"CHANGES_REQUESTED","body":"please fix",
|
||||
"createdAt":"2023-05-03T00:00:00Z","submittedAt":"2023-05-03T01:00:00Z",
|
||||
"author":{"login":"reviewer","databaseId":77},"commit":{"oid":"sha1sha"},
|
||||
"comments":{"nodes":[{"databaseId":950,"body":"nit here","path":"main.go","diffHunk":"@@ -1 +1 @@",
|
||||
"position":3,"commit":{"oid":"sha1sha"},"author":{"databaseId":77},"replyTo":{"databaseId":949}}]}
|
||||
}]},
|
||||
"reviewRequests":{"nodes":[{"requestedReviewer":{"__typename":"Bot","login":"pending-rev","databaseId":88}},
|
||||
{"requestedReviewer":{"__typename":"Team"}},{"requestedReviewer":null}]}
|
||||
}]}}}`), &response))
|
||||
downloader := &GithubGraphQLDownloader{GithubDownloaderV3: &GithubDownloaderV3{baseURL: "https://github.com", repoOwner: "go-gitea", repoName: "gitea"}}
|
||||
pr, err := downloader.convertGraphQLPullRequest(t.Context(), &response.Repository.PullRequests.Nodes[0], map[string]*gqlPrefetched{})
|
||||
require.NoError(t, err)
|
||||
merged := time.Date(2023, 6, 1, 0, 0, 0, 0, time.UTC)
|
||||
assert.Equal(t, &base.PullRequest{
|
||||
Number: 7, Title: "add feature", Content: "does the thing", State: "closed", PosterID: 4242, PosterName: "contributor",
|
||||
Created: time.Date(2023, 5, 1, 0, 0, 0, 0, time.UTC), Updated: merged, Closed: &merged,
|
||||
Merged: true, MergedTime: &merged, IsDraft: true, MergeCommitSHA: "abc123", ForeignIndex: 7,
|
||||
Head: base.PullRequestBranch{Ref: "feature-x", SHA: "deadbeef", OwnerName: "contributor", RepoName: "gitea", CloneURL: "https://github.com/contributor/gitea.git"},
|
||||
Base: base.PullRequestBranch{Ref: "main", SHA: "cafebabe", OwnerName: "go-gitea", RepoName: "gitea"},
|
||||
PatchURL: "https://github.com/go-gitea/gitea/pull/7.patch", Labels: []*base.Label{}, Context: pr.Context,
|
||||
}, pr)
|
||||
reviews, err := downloader.GetReviews(t.Context(), pr)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []*base.Review{
|
||||
{
|
||||
ID: 900, IssueIndex: 7, ReviewerID: 77, ReviewerName: "reviewer", Content: "please fix", State: "CHANGES_REQUESTED", CommitID: "sha1sha",
|
||||
CreatedAt: time.Date(2023, 5, 3, 1, 0, 0, 0, time.UTC),
|
||||
Comments: []*base.ReviewComment{{ID: 950, InReplyTo: 949, Content: "nit here", TreePath: "main.go", DiffHunk: "@@ -1 +1 @@", Position: 3, CommitID: "sha1sha", PosterID: 77}},
|
||||
},
|
||||
{IssueIndex: 7, ReviewerID: 88, ReviewerName: "pending-rev[bot]", State: base.ReviewStateRequestReview},
|
||||
}, reviews)
|
||||
}
|
||||
|
||||
func TestGraphQLDiscussionOverflowFallsBackToREST(t *testing.T) {
|
||||
for _, connection := range []string{"comments", "reviews", "review comments"} {
|
||||
t.Run(connection, func(t *testing.T) {
|
||||
downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, request *http.Request) {
|
||||
switch {
|
||||
case strings.HasSuffix(request.URL.Path, "/issues/7/comments"):
|
||||
_, _ = io.WriteString(writer, `[{"id":500,"body":"REST comment"}]`)
|
||||
case strings.HasSuffix(request.URL.Path, "/pulls/7/reviews"):
|
||||
_, _ = io.WriteString(writer, `[{"id":900,"body":"REST review","state":"APPROVED"}]`)
|
||||
case strings.HasSuffix(request.URL.Path, "/requested_reviewers"):
|
||||
_, _ = io.WriteString(writer, `{}`)
|
||||
default:
|
||||
_, _ = io.WriteString(writer, `[]`)
|
||||
}
|
||||
})
|
||||
downloader.SkipReactions = true
|
||||
node := &gqlPullRequest{gqlIssue: gqlIssue{Number: 7}}
|
||||
switch connection {
|
||||
case "comments":
|
||||
node.Comments.PageInfo.HasNextPage = true
|
||||
case "reviews":
|
||||
node.Reviews.PageInfo.HasNextPage = true
|
||||
case "review comments":
|
||||
node.Reviews.Nodes = []gqlReview{{Comments: gqlConn[gqlReviewComment]{PageInfo: gqlPageInfo{HasNextPage: true}}}}
|
||||
}
|
||||
pr, err := downloader.convertGraphQLPullRequest(t.Context(), node, map[string]*gqlPrefetched{})
|
||||
require.NoError(t, err)
|
||||
if connection == "comments" {
|
||||
comments, _, err := downloader.GetComments(t.Context(), pr)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []*base.Comment{{IssueIndex: 7, Index: 500, Content: "REST comment"}}, comments)
|
||||
} else {
|
||||
reviews, err := downloader.GetReviews(t.Context(), pr)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []*base.Review{{IssueIndex: 7, ID: 900, Content: "REST review", State: "APPROVED"}}, reviews)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,428 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"gitea.dev/modules/egress/policy"
|
||||
"gitea.dev/modules/json"
|
||||
base "gitea.dev/modules/migration"
|
||||
"gitea.dev/modules/test"
|
||||
|
||||
"github.com/google/go-github/v92/github"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func newGithubTestDownloader(t *testing.T, handler http.HandlerFunc, spareTokens ...string) *GithubGraphQLDownloader {
|
||||
t.Helper()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
writer.Header().Set("X-RateLimit-Remaining", "4000")
|
||||
handler(writer, request)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
downloader, err := NewGithubDownloaderV3(t.Context(), server.URL, "", "", strings.Join(append([]string{"tok"}, spareTokens...), ","), "o", "r")
|
||||
require.NoError(t, err)
|
||||
return newGithubGraphQLDownloader(downloader, false)
|
||||
}
|
||||
|
||||
func TestGraphQLEndpoint(t *testing.T) {
|
||||
for baseURL, endpoint := range map[string]string{
|
||||
"https://github.com": "https://api.github.com/graphql",
|
||||
"https://api.github.com": "https://api.github.com/graphql",
|
||||
"https://ghe.example.com": "https://ghe.example.com/api/graphql",
|
||||
"https://ghe.example.com/": "https://ghe.example.com/api/graphql",
|
||||
} {
|
||||
downloader, err := NewGithubDownloaderV3(t.Context(), baseURL, "", "", "tok", "o", "r")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, endpoint, newGithubGraphQLDownloader(downloader, false).graphQLEndpoint(), baseURL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphQLRateResetWait(t *testing.T) {
|
||||
for _, testcase := range []struct {
|
||||
name string
|
||||
headers map[string]string
|
||||
want time.Duration
|
||||
}{
|
||||
{"retry after takes precedence", map[string]string{"Retry-After": "30", "X-RateLimit-Reset": "9999999999"}, 30 * time.Second},
|
||||
{"no headers", nil, graphQLRateResetFallback},
|
||||
{"past primary reset", map[string]string{"X-RateLimit-Reset": "1"}, graphQLRateResetFallback},
|
||||
{"secondary reset is not primary reset", map[string]string{"X-RateLimit-Remaining": "4321", "X-RateLimit-Reset": "9999999999"}, graphQLRateResetFallback},
|
||||
{"primary reset is capped", map[string]string{"X-RateLimit-Remaining": "0", "X-RateLimit-Reset": "9999999999"}, graphQLRateResetCap},
|
||||
{"retry after is capped", map[string]string{"Retry-After": "999999"}, graphQLRateResetCap},
|
||||
} {
|
||||
t.Run(testcase.name, func(t *testing.T) {
|
||||
header := http.Header{}
|
||||
for name, value := range testcase.headers {
|
||||
header.Set(name, value)
|
||||
}
|
||||
assert.Equal(t, testcase.want, graphQLRateResetWait(header))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDoGraphQLRetriesTransientFailures(t *testing.T) {
|
||||
defer test.MockVariableValue(&graphQLTransientRetryBase, time.Duration(0))()
|
||||
for _, testcase := range []struct {
|
||||
name string
|
||||
failure string
|
||||
failures int32
|
||||
}{
|
||||
{"bad gateway then success", "502", 1},
|
||||
{"truncated body then success", "body", 1},
|
||||
{"connection closed then success", "connection", 1},
|
||||
{"persistent bad gateway", "502", maxGraphQLTransientRetries + 1},
|
||||
} {
|
||||
t.Run(testcase.name, func(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, _request *http.Request) {
|
||||
if calls.Add(1) > testcase.failures {
|
||||
_, _ = io.WriteString(writer, `{"data":{"ok":true}}`)
|
||||
return
|
||||
}
|
||||
switch testcase.failure {
|
||||
case "502":
|
||||
writer.WriteHeader(http.StatusBadGateway)
|
||||
case "body":
|
||||
_, _ = io.WriteString(writer, `{"data":{"ok":`)
|
||||
case "connection":
|
||||
connection, _, err := http.NewResponseController(writer).Hijack()
|
||||
if assert.NoError(t, err) {
|
||||
_ = connection.Close()
|
||||
}
|
||||
}
|
||||
})
|
||||
var response struct {
|
||||
OK bool `json:"ok"`
|
||||
}
|
||||
err := downloader.doGraphQL(t.Context(), "query{}", nil, &response)
|
||||
if testcase.failures > maxGraphQLTransientRetries {
|
||||
require.ErrorIs(t, err, errGraphQLTransientExhausted)
|
||||
assert.Equal(t, int32(maxGraphQLTransientRetries+1), calls.Load())
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
assert.True(t, response.OK)
|
||||
assert.Equal(t, testcase.failures+1, calls.Load())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDoGraphQLWaitsOutSecondaryRateLimit(t *testing.T) {
|
||||
defer test.MockVariableValue(&graphQLRateResetFallback, time.Millisecond)()
|
||||
for _, testcase := range []struct {
|
||||
name string
|
||||
status int
|
||||
body string
|
||||
}{
|
||||
{"GraphQL rate limit", http.StatusOK, `{"errors":[{"type":"RATE_LIMITED"}]}`},
|
||||
{"429 without retry after", http.StatusTooManyRequests, ""},
|
||||
{"403 spent primary quota", http.StatusForbidden, ""},
|
||||
{"403 secondary limit with quota left", http.StatusForbidden, `{"message":"You have exceeded a secondary rate limit."}`},
|
||||
} {
|
||||
t.Run(testcase.name, func(t *testing.T) {
|
||||
tokens := make(chan string, 3)
|
||||
downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, request *http.Request) {
|
||||
tokens <- request.Header.Get("Authorization")
|
||||
if request.Header.Get("Authorization") == "Bearer tok" {
|
||||
writer.Header().Set("X-RateLimit-Reset", strconv.FormatInt(time.Now().Add(time.Hour).Unix(), 10))
|
||||
if testcase.body == "" {
|
||||
writer.Header().Set("X-RateLimit-Remaining", "0")
|
||||
}
|
||||
writer.WriteHeader(testcase.status)
|
||||
_, _ = io.WriteString(writer, testcase.body)
|
||||
return
|
||||
}
|
||||
_, _ = io.WriteString(writer, `{"data":{"ok":true}}`)
|
||||
}, "spare")
|
||||
var response struct {
|
||||
OK bool `json:"ok"`
|
||||
}
|
||||
require.NoError(t, downloader.doGraphQL(t.Context(), "query{}", nil, &response))
|
||||
assert.True(t, response.OK)
|
||||
assert.Equal(t, []string{"Bearer tok", "Bearer spare"}, []string{<-tokens, <-tokens})
|
||||
assert.Zero(t, downloader.gqlRates[0].Remaining)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDoGraphQLRateRetryBudgetPerToken(t *testing.T) {
|
||||
defer test.MockVariableValue(&graphQLRateResetFallback, time.Duration(0))()
|
||||
var calls atomic.Int32
|
||||
downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, _request *http.Request) {
|
||||
calls.Add(1)
|
||||
_, _ = io.WriteString(writer, `{"errors":[{"type":"RATE_LIMITED"}]}`)
|
||||
}, "spare")
|
||||
require.Error(t, downloader.doGraphQL(t.Context(), "query{}", nil, &struct{}{}))
|
||||
assert.Equal(t, int32(maxGraphQLRateRetries*len(downloader.clients)+1), calls.Load())
|
||||
}
|
||||
|
||||
func TestDoGraphQLFatalStatusQuotesBody(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, _request *http.Request) {
|
||||
calls.Add(1)
|
||||
writer.WriteHeader(http.StatusForbidden)
|
||||
_, _ = io.WriteString(writer, `{"message":"Resource not accessible by integration"}`)
|
||||
})
|
||||
assert.ErrorContains(t, downloader.doGraphQL(t.Context(), "query{}", nil, &struct{}{}), "Resource not accessible by integration")
|
||||
assert.EqualValues(t, 1, calls.Load())
|
||||
}
|
||||
|
||||
func TestPickGraphQLClient(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
reset := github.Timestamp{Time: time.Now().Add(2 * graphQLRateResetCap)}
|
||||
downloader := &GithubGraphQLDownloader{GithubDownloaderV3: &GithubDownloaderV3{}, gqlRates: []*github.Rate{
|
||||
{Remaining: 1000, Reset: reset},
|
||||
{Remaining: 4000, Reset: reset},
|
||||
{Remaining: 5, Reset: reset},
|
||||
}}
|
||||
require.NoError(t, downloader.pickGraphQLClient(t.Context()))
|
||||
assert.Equal(t, 1, downloader.curClientIdx)
|
||||
downloader.gqlRates[2] = nil
|
||||
require.NoError(t, downloader.pickGraphQLClient(t.Context()))
|
||||
assert.Equal(t, 2, downloader.curClientIdx)
|
||||
downloader.gqlRates = []*github.Rate{{Reset: github.Timestamp{Time: reset.Add(time.Hour)}}, {Reset: reset}}
|
||||
started := time.Now()
|
||||
require.NoError(t, downloader.pickGraphQLClient(t.Context()))
|
||||
assert.Equal(t, graphQLRateResetCap, time.Since(started))
|
||||
assert.Equal(t, 1, downloader.curClientIdx)
|
||||
assert.Nil(t, downloader.gqlRates[1])
|
||||
})
|
||||
}
|
||||
|
||||
func TestIsTransientTransportError(t *testing.T) {
|
||||
wrapped := func(err error) error {
|
||||
return &url.Error{Op: "Post", URL: "https://example.com", Err: &net.OpError{Op: "dial", Err: err}}
|
||||
}
|
||||
for _, testcase := range []struct {
|
||||
name string
|
||||
err error
|
||||
transient bool
|
||||
}{
|
||||
{"blocked by policy", fmt.Errorf("blocked: %w", policy.ErrDenied), false},
|
||||
{"host does not resolve", &net.DNSError{IsNotFound: true}, false},
|
||||
{"TLS verification failed", &tls.CertificateVerificationError{Err: x509.UnknownAuthorityError{}}, false},
|
||||
{"DNS server failure", &net.DNSError{IsTemporary: true}, true},
|
||||
{"connection reset", errors.New("connection reset by peer"), true},
|
||||
} {
|
||||
t.Run(testcase.name, func(t *testing.T) {
|
||||
assert.Equal(t, testcase.transient, isTransientTransportError(wrapped(testcase.err)))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphQLQueryGating(t *testing.T) {
|
||||
for _, skipComments := range []bool{false, true} {
|
||||
for _, skipReactions := range []bool{false, true} {
|
||||
downloader := &GithubGraphQLDownloader{GithubDownloaderV3: &GithubDownloaderV3{SkipReactions: skipReactions}, skipComments: skipComments}
|
||||
for name, query := range map[string]string{"issues": downloader.graphQLIssuesQuery(), "pullRequests": downloader.graphQLPullRequestsQuery()} {
|
||||
t.Run(fmt.Sprintf("%s/comments=%t/reactions=%t", name, !skipComments, !skipReactions), func(t *testing.T) {
|
||||
assert.Equal(t, !skipComments, strings.Contains(query, "comments("))
|
||||
assert.Equal(t, !skipComments && name == "pullRequests", strings.Contains(query, "reviews("))
|
||||
assert.Equal(t, !skipComments && name == "pullRequests", strings.Contains(query, "reviewRequests("))
|
||||
assert.Equal(t, !skipReactions, strings.Contains(query, "reactions("))
|
||||
assert.LessOrEqual(t, strings.Count(query, "reactions("), 1)
|
||||
assert.Contains(t, query, "id number title")
|
||||
assert.Contains(t, query, "orderBy:{field:CREATED_AT,direction:ASC}")
|
||||
assert.Contains(t, query, "labels(first:30){"+gqlPageInfoFields)
|
||||
assert.Contains(t, query, "... on Bot{databaseId}")
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphQLPageSize(t *testing.T) {
|
||||
assert.Equal(t, 41, graphQLPageSize(41, githubGraphQLPageSize))
|
||||
assert.Equal(t, githubGraphQLPageSize, graphQLPageSize(500, githubGraphQLPageSize))
|
||||
assert.Equal(t, githubGraphQLPageSize, graphQLPageSize(0, githubGraphQLPageSize))
|
||||
assert.Equal(t, githubGraphQLPRPageSize, graphQLPageSize(41, githubGraphQLPRPageSize))
|
||||
}
|
||||
|
||||
func TestDoGraphQLPageShrink(t *testing.T) {
|
||||
defer test.MockVariableValue(&graphQLTransientRetryBase, time.Duration(0))()
|
||||
for _, failure := range []string{"502", "RESOURCE_LIMITS_EXCEEDED"} {
|
||||
t.Run(failure, func(t *testing.T) {
|
||||
sizes := make(chan int, 16)
|
||||
downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, request *http.Request) {
|
||||
var payload struct {
|
||||
Variables struct {
|
||||
First int `json:"first"`
|
||||
Cursor string `json:"cursor"`
|
||||
} `json:"variables"`
|
||||
}
|
||||
assert.NoError(t, json.NewDecoder(request.Body).Decode(&payload))
|
||||
sizes <- payload.Variables.First
|
||||
assert.Equal(t, "CURSOR", payload.Variables.Cursor)
|
||||
if payload.Variables.First > 25 {
|
||||
if failure == "502" {
|
||||
writer.WriteHeader(http.StatusBadGateway)
|
||||
} else {
|
||||
_, _ = io.WriteString(writer, `{"errors":[{"type":"RESOURCE_LIMITS_EXCEEDED"}]}`)
|
||||
}
|
||||
return
|
||||
}
|
||||
_, _ = io.WriteString(writer, `{"data":{}}`)
|
||||
})
|
||||
require.NoError(t, downloader.doGraphQLPage(t.Context(), downloader.graphQLIssuesQuery(), "CURSOR", 100, &gqlIssuesResponse{}))
|
||||
close(sizes)
|
||||
var got, want []int
|
||||
for size := range sizes {
|
||||
got = append(got, size)
|
||||
}
|
||||
for _, size := range []int{100, 50, 25} {
|
||||
repeats := 1
|
||||
if failure == "502" && size > 25 {
|
||||
repeats += maxGraphQLTransientRetries
|
||||
}
|
||||
for range repeats {
|
||||
want = append(want, size)
|
||||
}
|
||||
}
|
||||
assert.Equal(t, want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchNodeReactionsBisectsAndSkipsDeletedNodes(t *testing.T) {
|
||||
defer test.MockVariableValue(&graphQLTransientRetryBase, time.Duration(0))()
|
||||
for _, failure := range []string{"502", "RESOURCE_LIMITS_EXCEEDED"} {
|
||||
t.Run(failure, func(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, request *http.Request) {
|
||||
calls.Add(1)
|
||||
var payload struct {
|
||||
Variables struct {
|
||||
IDs []string `json:"ids"`
|
||||
} `json:"variables"`
|
||||
}
|
||||
assert.NoError(t, json.NewDecoder(request.Body).Decode(&payload))
|
||||
if len(payload.Variables.IDs) > 1 {
|
||||
if failure == "502" {
|
||||
writer.WriteHeader(http.StatusBadGateway)
|
||||
} else {
|
||||
_, _ = io.WriteString(writer, `{"errors":[{"type":"RESOURCE_LIMITS_EXCEEDED"}]}`)
|
||||
}
|
||||
return
|
||||
}
|
||||
if payload.Variables.IDs[0] == "deleted" {
|
||||
_, _ = io.WriteString(writer, `{"data":{"nodes":[null]},"errors":[{"type":"NOT_FOUND","path":["nodes",0]}]}`)
|
||||
} else {
|
||||
_, _ = io.WriteString(writer, `{"data":{"nodes":[{"id":"present","reactions":{"nodes":[{"content":"HEART","user":null}]}}]}}`)
|
||||
}
|
||||
})
|
||||
got, err := downloader.fetchNodeReactions(t.Context(), []string{"deleted", "present"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string][]*base.Reaction{"present": {{UserName: "ghost", Content: "heart"}}}, got)
|
||||
if failure == "502" {
|
||||
assert.EqualValues(t, maxGraphQLTransientRetries+3, calls.Load())
|
||||
} else {
|
||||
assert.EqualValues(t, 3, calls.Load())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLabelsSweepContinuesFromEndCursor(t *testing.T) {
|
||||
downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, request *http.Request) {
|
||||
var payload graphQLRequest
|
||||
assert.NoError(t, json.NewDecoder(request.Body).Decode(&payload))
|
||||
assert.Equal(t, "I_node", payload.Variables["id"])
|
||||
assert.Equal(t, "LABEL1", payload.Variables["cursor"])
|
||||
_, _ = io.WriteString(writer, `{"data":{"node":{"conn":{"nodes":[{"name":"second"}]}}}}`)
|
||||
})
|
||||
labels, err := allNodes(t.Context(), downloader, graphQLLabelsSweepQuery, "I_node", gqlConn[gqlLabel]{
|
||||
PageInfo: gqlPageInfo{HasNextPage: true, EndCursor: "LABEL1"},
|
||||
Nodes: []gqlLabel{{Name: "first"}},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []gqlLabel{{Name: "first"}, {Name: "second"}}, labels)
|
||||
}
|
||||
|
||||
func TestGraphQLIssuesResponseParsing(t *testing.T) {
|
||||
var response gqlIssuesResponse
|
||||
require.NoError(t, json.Unmarshal([]byte(`{"repository":{"issues":{"nodes":[{
|
||||
"id":"I_42","number":42,"title":"a bug","body":"it broke","state":"CLOSED","locked":true,
|
||||
"createdAt":"2024-01-02T03:04:05Z","updatedAt":"2024-02-03T04:05:06Z","closedAt":"2024-02-03T04:05:06Z",
|
||||
"author":{"__typename":"Bot","login":"reporter","databaseId":11},"milestone":{"title":"v1.0"},
|
||||
"labels":{"nodes":[{"name":"bug","color":"d73a4a","description":"defect"}]},
|
||||
"assignees":{"nodes":[{"login":"maintainer"}]},
|
||||
"reactions":{"nodes":[{"content":"THUMBS_UP","user":{"login":"alice","databaseId":22}}]},
|
||||
"comments":{"nodes":[{"id":"IC_1","databaseId":900,"body":"same here","author":null}]}
|
||||
}]}}}`), &response))
|
||||
downloader := &GithubGraphQLDownloader{GithubDownloaderV3: &GithubDownloaderV3{}}
|
||||
issue, err := downloader.convertGraphQLIssue(t.Context(), &response.Repository.Issues.Nodes[0], map[string]*gqlPrefetched{})
|
||||
require.NoError(t, err)
|
||||
updated := time.Date(2024, 2, 3, 4, 5, 6, 0, time.UTC)
|
||||
assert.Equal(t, &base.Issue{
|
||||
Number: 42, Title: "a bug", Content: "it broke", State: "closed", IsLocked: true,
|
||||
PosterName: "reporter[bot]", PosterID: 11, Milestone: "v1.0", ForeignIndex: 42,
|
||||
Created: time.Date(2024, 1, 2, 3, 4, 5, 0, time.UTC), Updated: updated, Closed: &updated,
|
||||
Labels: []*base.Label{{Name: "bug", Color: "d73a4a", Description: "defect"}}, Assignees: []string{"maintainer"},
|
||||
Reactions: []*base.Reaction{{Content: "+1", UserName: "alice", UserID: 22}}, Context: issue.Context,
|
||||
}, issue)
|
||||
comments, _, err := downloader.GetComments(t.Context(), issue)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []*base.Comment{{IssueIndex: 42, Index: 900, Content: "same here", PosterName: "ghost"}}, comments)
|
||||
assert.False(t, downloader.SupportGetRepoComments())
|
||||
}
|
||||
|
||||
func TestGraphQLKeepsCursorOnPageError(t *testing.T) {
|
||||
for _, connection := range []string{"issues", "pullRequests"} {
|
||||
for _, phase := range []string{"timelineItems", "Reactable"} {
|
||||
t.Run(connection+"/"+phase, func(t *testing.T) {
|
||||
var failPage atomic.Bool
|
||||
failPage.Store(true)
|
||||
downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, request *http.Request) {
|
||||
var payload graphQLRequest
|
||||
assert.NoError(t, json.NewDecoder(request.Body).Decode(&payload))
|
||||
if strings.Contains(payload.Query, "repository(owner:") {
|
||||
assert.Equal(t, "PAGE1", payload.Variables["cursor"])
|
||||
_, _ = fmt.Fprintf(writer, `{"data":{"repository":{"%s":{"pageInfo":{"endCursor":"PAGE2"},"nodes":[{"id":"I_1","number":1,"comments":{"nodes":[{"id":"IC_1"}]}}]}}}}`, connection)
|
||||
} else if failPage.Load() && strings.Contains(payload.Query, phase) {
|
||||
writer.WriteHeader(http.StatusBadRequest)
|
||||
} else {
|
||||
_, _ = io.WriteString(writer, `{"data":{"nodes":[]}}`)
|
||||
}
|
||||
})
|
||||
cursor := &downloader.issuesCursor
|
||||
if connection == "pullRequests" {
|
||||
cursor = &downloader.prCursor
|
||||
}
|
||||
*cursor = "PAGE1"
|
||||
fetch := func() error {
|
||||
if connection == "issues" {
|
||||
_, _, err := downloader.getIssuesGraphQL(t.Context(), 2, 10)
|
||||
return err
|
||||
} else {
|
||||
_, _, err := downloader.getPullRequestsGraphQL(t.Context(), 2, 10)
|
||||
return err
|
||||
}
|
||||
}
|
||||
require.Error(t, fetch())
|
||||
assert.Equal(t, "PAGE1", *cursor)
|
||||
failPage.Store(false)
|
||||
require.NoError(t, fetch())
|
||||
assert.Equal(t, "PAGE2", *cursor)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
issues_model "gitea.dev/models/issues"
|
||||
"gitea.dev/modules/log"
|
||||
base "gitea.dev/modules/migration"
|
||||
)
|
||||
|
||||
// PR-only types in the Issue fragment fail validation of the whole query, assignee events would need local user ids
|
||||
const (
|
||||
issueTimelineItemTypes = "CLOSED_EVENT,REOPENED_EVENT,LOCKED_EVENT,UNLOCKED_EVENT,RENAMED_TITLE_EVENT,LABELED_EVENT,UNLABELED_EVENT,MILESTONED_EVENT,DEMILESTONED_EVENT,PINNED_EVENT,UNPINNED_EVENT"
|
||||
prTimelineItemTypes = issueTimelineItemTypes + ",MERGED_EVENT,HEAD_REF_DELETED_EVENT,AUTO_MERGE_ENABLED_EVENT,AUTO_MERGE_DISABLED_EVENT"
|
||||
)
|
||||
|
||||
const issueTimelineFragments = `
|
||||
... on ClosedEvent{id createdAt actor{` + gqlActorFields + `}}
|
||||
... on ReopenedEvent{id createdAt actor{` + gqlActorFields + `}}
|
||||
... on LockedEvent{id createdAt lockReason actor{` + gqlActorFields + `}}
|
||||
... on UnlockedEvent{id createdAt actor{` + gqlActorFields + `}}
|
||||
... on RenamedTitleEvent{id createdAt previousTitle currentTitle actor{` + gqlActorFields + `}}
|
||||
... on LabeledEvent{id createdAt label{name} actor{` + gqlActorFields + `}}
|
||||
... on UnlabeledEvent{id createdAt label{name} actor{` + gqlActorFields + `}}
|
||||
... on MilestonedEvent{id createdAt milestoneTitle actor{` + gqlActorFields + `}}
|
||||
... on DemilestonedEvent{id createdAt milestoneTitle actor{` + gqlActorFields + `}}
|
||||
... on PinnedEvent{id createdAt actor{` + gqlActorFields + `}}
|
||||
... on UnpinnedEvent{id createdAt actor{` + gqlActorFields + `}}`
|
||||
|
||||
const prTimelineFragments = issueTimelineFragments + `
|
||||
... on MergedEvent{id createdAt actor{` + gqlActorFields + `}}
|
||||
... on HeadRefDeletedEvent{id createdAt headRefName actor{` + gqlActorFields + `}}
|
||||
... on AutoMergeEnabledEvent{id createdAt actor{` + gqlActorFields + `}}
|
||||
... on AutoMergeDisabledEvent{id createdAt actor{` + gqlActorFields + `}}`
|
||||
|
||||
const (
|
||||
issueTimelineConn = `{
|
||||
pageInfo{hasNextPage endCursor}
|
||||
nodes{
|
||||
__typename` + issueTimelineFragments + `
|
||||
}
|
||||
}`
|
||||
prTimelineConn = `{
|
||||
pageInfo{hasNextPage endCursor}
|
||||
nodes{
|
||||
__typename` + prTimelineFragments + `
|
||||
}
|
||||
}`
|
||||
)
|
||||
|
||||
const graphQLTimelineBatchQuery = `
|
||||
query($ids:[ID!]!){
|
||||
nodes(ids:$ids){
|
||||
id
|
||||
... on Issue{timelineItems(first:100,itemTypes:[` + issueTimelineItemTypes + `])` + issueTimelineConn + `}
|
||||
... on PullRequest{timelineItems(first:100,itemTypes:[` + prTimelineItemTypes + `])` + prTimelineConn + `}
|
||||
}
|
||||
}`
|
||||
|
||||
const graphQLTimelineSweepQuery = `
|
||||
query($id:ID!,$cursor:String){
|
||||
node(id:$id){
|
||||
... on Issue{conn:timelineItems(first:100,after:$cursor,itemTypes:[` + issueTimelineItemTypes + `])` + issueTimelineConn + `}
|
||||
... on PullRequest{conn:timelineItems(first:100,after:$cursor,itemTypes:[` + prTimelineItemTypes + `])` + prTimelineConn + `}
|
||||
}
|
||||
}`
|
||||
|
||||
type gqlTimelineItem struct {
|
||||
Typename string `json:"__typename"`
|
||||
ID string `json:"id"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
Actor gqlActor `json:"actor"`
|
||||
PreviousTitle string `json:"previousTitle"`
|
||||
CurrentTitle string `json:"currentTitle"`
|
||||
Label *struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"label"`
|
||||
MilestoneTitle string `json:"milestoneTitle"`
|
||||
HeadRefName string `json:"headRefName"`
|
||||
LockReason string `json:"lockReason"`
|
||||
}
|
||||
|
||||
func convertTimelineItem(it *gqlTimelineItem) *base.Comment {
|
||||
comment := &base.Comment{
|
||||
PosterID: it.Actor.DatabaseID,
|
||||
PosterName: it.Actor.Login,
|
||||
Created: it.CreatedAt,
|
||||
Updated: it.CreatedAt,
|
||||
}
|
||||
switch it.Typename {
|
||||
case "ClosedEvent":
|
||||
comment.CommentType = issues_model.CommentTypeClose.String()
|
||||
case "ReopenedEvent":
|
||||
comment.CommentType = issues_model.CommentTypeReopen.String()
|
||||
case "LockedEvent":
|
||||
comment.CommentType = issues_model.CommentTypeLock.String()
|
||||
comment.Content = strings.ToLower(strings.ReplaceAll(it.LockReason, "_", " "))
|
||||
case "UnlockedEvent":
|
||||
comment.CommentType = issues_model.CommentTypeUnlock.String()
|
||||
case "RenamedTitleEvent":
|
||||
comment.CommentType = issues_model.CommentTypeChangeTitle.String()
|
||||
comment.Meta = map[string]any{"OldTitle": it.PreviousTitle, "NewTitle": it.CurrentTitle}
|
||||
case "LabeledEvent", "UnlabeledEvent":
|
||||
if it.Label == nil {
|
||||
return nil
|
||||
}
|
||||
comment.CommentType = issues_model.CommentTypeLabel.String()
|
||||
comment.Meta = map[string]any{"LabelName": it.Label.Name}
|
||||
if it.Typename == "LabeledEvent" {
|
||||
comment.Content = "1"
|
||||
}
|
||||
case "MilestonedEvent":
|
||||
comment.CommentType = issues_model.CommentTypeMilestone.String()
|
||||
comment.Meta = map[string]any{"MilestoneTitle": it.MilestoneTitle}
|
||||
case "DemilestonedEvent":
|
||||
comment.CommentType = issues_model.CommentTypeMilestone.String()
|
||||
comment.Meta = map[string]any{"MilestoneTitle": it.MilestoneTitle, "Removed": true}
|
||||
case "MergedEvent":
|
||||
comment.CommentType = issues_model.CommentTypeMergePull.String()
|
||||
case "HeadRefDeletedEvent":
|
||||
comment.CommentType = issues_model.CommentTypeDeleteBranch.String()
|
||||
comment.Content = it.HeadRefName
|
||||
case "PinnedEvent":
|
||||
comment.CommentType = issues_model.CommentTypePin.String()
|
||||
case "UnpinnedEvent":
|
||||
comment.CommentType = issues_model.CommentTypeUnpin.String()
|
||||
case "AutoMergeEnabledEvent":
|
||||
comment.CommentType = issues_model.CommentTypePRScheduledToAutoMerge.String()
|
||||
case "AutoMergeDisabledEvent":
|
||||
comment.CommentType = issues_model.CommentTypePRUnScheduledToAutoMerge.String()
|
||||
default:
|
||||
log.Warn("github graphql: dropping unhandled timeline event type %q (node %s)", it.Typename, it.ID)
|
||||
return nil
|
||||
}
|
||||
return comment
|
||||
}
|
||||
|
||||
func convertTimelineItems(items []gqlTimelineItem) []*base.Comment {
|
||||
comments := make([]*base.Comment, 0, len(items))
|
||||
for i := range items {
|
||||
if comment := convertTimelineItem(&items[i]); comment != nil {
|
||||
comments = append(comments, comment)
|
||||
}
|
||||
}
|
||||
return comments
|
||||
}
|
||||
|
||||
func (g *GithubGraphQLDownloader) attachTimelineEvents(ctx context.Context, prefetchedByNodeID map[string]*gqlPrefetched) error {
|
||||
for batch := range slices.Chunk(slices.Sorted(maps.Keys(prefetchedByNodeID)), graphQLNodeBatchSize) {
|
||||
eventsByID, err := g.fetchTimelineEvents(ctx, batch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for id, events := range eventsByID {
|
||||
prefetched := prefetchedByNodeID[id]
|
||||
for _, event := range events {
|
||||
event.IssueIndex = prefetched.index
|
||||
}
|
||||
prefetched.comments = append(prefetched.comments, events...)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *GithubGraphQLDownloader) fetchTimelineEvents(ctx context.Context, ids []string) (map[string][]*base.Comment, error) {
|
||||
var resp struct {
|
||||
Nodes []struct {
|
||||
ID string `json:"id"`
|
||||
TimelineItems gqlConn[gqlTimelineItem] `json:"timelineItems"`
|
||||
} `json:"nodes"`
|
||||
}
|
||||
if err := g.doGraphQL(ctx, graphQLTimelineBatchQuery, map[string]any{"ids": ids}, &resp); err != nil {
|
||||
if isGraphQLTooHeavy(err) && len(ids) > 1 {
|
||||
return fetchHalves(ctx, ids, g.fetchTimelineEvents)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[string][]*base.Comment, len(resp.Nodes))
|
||||
for _, node := range resp.Nodes {
|
||||
if node.ID == "" {
|
||||
continue
|
||||
}
|
||||
items, err := allNodes(ctx, g, graphQLTimelineSweepQuery, node.ID, node.TimelineItems)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[node.ID] = dropCloseOnMerge(convertTimelineItems(items))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// dropCloseOnMerge drops the close GitHub records with a merge, the last one since a merged PR cannot be reopened
|
||||
func dropCloseOnMerge(events []*base.Comment) []*base.Comment {
|
||||
if !slices.ContainsFunc(events, func(event *base.Comment) bool { return event.CommentType == issues_model.CommentTypeMergePull.String() }) {
|
||||
return events
|
||||
}
|
||||
for i, event := range slices.Backward(events) {
|
||||
switch event.CommentType {
|
||||
case issues_model.CommentTypeReopen.String():
|
||||
return events
|
||||
case issues_model.CommentTypeClose.String():
|
||||
return slices.Delete(events, i, i+1)
|
||||
}
|
||||
}
|
||||
return events
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dev/modules/json"
|
||||
base "gitea.dev/modules/migration"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestConvertTimelineItems(t *testing.T) {
|
||||
for _, testcase := range []struct {
|
||||
name string
|
||||
event string
|
||||
want *base.Comment
|
||||
}{
|
||||
{"close by user", `{"__typename":"ClosedEvent","actor":{"login":"closer","databaseId":42}}`, &base.Comment{CommentType: "close", PosterName: "closer", PosterID: 42}},
|
||||
{"reopen", `{"__typename":"ReopenedEvent"}`, &base.Comment{CommentType: "reopen"}},
|
||||
{"label added", `{"__typename":"LabeledEvent","label":{"name":"bug"}}`, &base.Comment{CommentType: "label", Content: "1", Meta: map[string]any{"LabelName": "bug"}}},
|
||||
{"label removed", `{"__typename":"UnlabeledEvent","label":{"name":"bug"}}`, &base.Comment{CommentType: "label", Meta: map[string]any{"LabelName": "bug"}}},
|
||||
{"deleted label added is skipped", `{"__typename":"LabeledEvent","label":null}`, nil},
|
||||
{"deleted label removed is skipped", `{"__typename":"UnlabeledEvent","label":null}`, nil},
|
||||
{"lock reason in content", `{"__typename":"LockedEvent","lockReason":"TOO_HEATED"}`, &base.Comment{CommentType: "lock", Content: "too heated"}},
|
||||
{"unlock", `{"__typename":"UnlockedEvent"}`, &base.Comment{CommentType: "unlock"}},
|
||||
{"title changed", `{"__typename":"RenamedTitleEvent","previousTitle":"old","currentTitle":"new"}`, &base.Comment{CommentType: "change_title", Meta: map[string]any{"OldTitle": "old", "NewTitle": "new"}}},
|
||||
{"milestone added", `{"__typename":"MilestonedEvent","milestoneTitle":"v1"}`, &base.Comment{CommentType: "milestone", Meta: map[string]any{"MilestoneTitle": "v1"}}},
|
||||
{"milestone removed", `{"__typename":"DemilestonedEvent","milestoneTitle":"v1"}`, &base.Comment{CommentType: "milestone", Meta: map[string]any{"MilestoneTitle": "v1", "Removed": true}}},
|
||||
{"merge by bot", `{"__typename":"MergedEvent","actor":{"__typename":"Bot","login":"merger","databaseId":88}}`, &base.Comment{CommentType: "merge_pull", PosterName: "merger[bot]", PosterID: 88}},
|
||||
{"branch deleted", `{"__typename":"HeadRefDeletedEvent","headRefName":"feature"}`, &base.Comment{CommentType: "delete_branch", Content: "feature"}},
|
||||
{"pin", `{"__typename":"PinnedEvent"}`, &base.Comment{CommentType: "pin"}},
|
||||
{"null actor becomes ghost", `{"__typename":"UnpinnedEvent","actor":null}`, &base.Comment{CommentType: "unpin", PosterName: "ghost"}},
|
||||
{"auto merge enabled", `{"__typename":"AutoMergeEnabledEvent"}`, &base.Comment{CommentType: "pull_scheduled_merge"}},
|
||||
{"auto merge disabled", `{"__typename":"AutoMergeDisabledEvent"}`, &base.Comment{CommentType: "pull_cancel_scheduled_merge"}},
|
||||
{"unknown event is skipped", `{"__typename":"MentionedEvent"}`, nil},
|
||||
} {
|
||||
t.Run(testcase.name, func(t *testing.T) {
|
||||
var item gqlTimelineItem
|
||||
require.NoError(t, json.Unmarshal([]byte(testcase.event), &item))
|
||||
item.CreatedAt = time.Date(2020, 11, 17, 15, 54, 33, 0, time.UTC)
|
||||
want := []*base.Comment{}
|
||||
if testcase.want != nil {
|
||||
testcase.want.Created, testcase.want.Updated = item.CreatedAt, item.CreatedAt
|
||||
want = append(want, testcase.want)
|
||||
}
|
||||
assert.Equal(t, want, convertTimelineItems([]gqlTimelineItem{item}))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDropCloseOnMerge(t *testing.T) {
|
||||
for _, testcase := range []struct {
|
||||
name string
|
||||
events []string
|
||||
want []string
|
||||
}{
|
||||
{"unmerged close stays", []string{"close"}, []string{"close"}},
|
||||
{"last close after merge drops", []string{"close", "reopen", "merge_pull", "close"}, []string{"close", "reopen", "merge_pull"}},
|
||||
{"last close before merge drops", []string{"close", "merge_pull"}, []string{"merge_pull"}},
|
||||
{"close followed by reopen stays", []string{"close", "merge_pull", "reopen"}, []string{"close", "merge_pull", "reopen"}},
|
||||
{"merge without close stays", []string{"merge_pull"}, []string{"merge_pull"}},
|
||||
} {
|
||||
t.Run(testcase.name, func(t *testing.T) {
|
||||
var events []*base.Comment
|
||||
for _, event := range testcase.events {
|
||||
events = append(events, &base.Comment{CommentType: event})
|
||||
}
|
||||
var got []string
|
||||
for _, event := range dropCloseOnMerge(events) {
|
||||
got = append(got, event.CommentType)
|
||||
}
|
||||
assert.Equal(t, testcase.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,8 @@ import (
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -48,7 +50,7 @@ func TestGithubIssuePagination(t *testing.T) {
|
||||
w.Header().Set("X-RateLimit-Remaining", "100")
|
||||
if r.URL.Path == "/api/v3/repos/o/r/issues/4/reactions" && failReaction {
|
||||
failReaction = false
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = w.Write([]byte(`{"message":"temporary failure"}`))
|
||||
return
|
||||
}
|
||||
@@ -595,3 +597,118 @@ func TestGithubMultiTokenClientSelection(t *testing.T) {
|
||||
downloader.waitAndPickClient(t.Context())
|
||||
assert.Equal(t, 1, downloader.curClientIdx)
|
||||
}
|
||||
|
||||
func TestGitHubIssuesPageBasedLinkFallback(t *testing.T) {
|
||||
_, callerFile, _, _ := runtime.Caller(0)
|
||||
mockServer := unittest.NewMockWebServer(t, "https://api.github.com", filepath.Join(filepath.Dir(callerFile), "_mock_data/TestGitHubDownloadRepo"), false, unittest.MockServerOptions{
|
||||
StripPrefix: "/api/v3",
|
||||
})
|
||||
downloader, err := NewGithubDownloaderV3(t.Context(), mockServer.URL, "", "", "", "go-gitea", "test_repo")
|
||||
require.NoError(t, err)
|
||||
downloader.SkipReactions = true
|
||||
for page, want := range [][]int64{{1, 2, 3}, {}} {
|
||||
issues, isEnd, err := downloader.GetIssues(t.Context(), page+1, 3)
|
||||
require.NoError(t, err)
|
||||
numbers := make([]int64, 0, len(issues))
|
||||
for _, issue := range issues {
|
||||
numbers = append(numbers, issue.Number)
|
||||
}
|
||||
assert.Equal(t, want, numbers)
|
||||
assert.Equal(t, page == 1, isEnd)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGithubFactoryUsesGraphQLOnlyWhereItWorks(t *testing.T) {
|
||||
for _, testcase := range []struct {
|
||||
name string
|
||||
opts base.MigrateOptions
|
||||
want bool
|
||||
}{
|
||||
{name: "token, issues", opts: base.MigrateOptions{AuthToken: "tok", Issues: true}, want: true},
|
||||
{name: "basic auth, pull requests", opts: base.MigrateOptions{AuthUsername: "u", AuthPassword: "pat", PullRequests: true}, want: true},
|
||||
{name: "anonymous repository", opts: base.MigrateOptions{Issues: true, PullRequests: true}},
|
||||
{name: "comments only", opts: base.MigrateOptions{AuthToken: "tok", Comments: true}},
|
||||
{name: "issues with comments", opts: base.MigrateOptions{AuthToken: "tok", Issues: true, Comments: true}, want: true},
|
||||
} {
|
||||
t.Run(testcase.name, func(t *testing.T) {
|
||||
testcase.opts.CloneAddr = "https://github.com/go-gitea/test_repo"
|
||||
candidate, err := (&GithubDownloaderV3Factory{}).New(t.Context(), testcase.opts)
|
||||
require.NoError(t, err)
|
||||
downloader, isGraphQL := candidate.(*GithubGraphQLDownloader)
|
||||
assert.Equal(t, testcase.want, isGraphQL)
|
||||
if isGraphQL {
|
||||
assert.Equal(t, !testcase.opts.Comments, downloader.skipComments)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGithubGraphQLFallsBackToREST(t *testing.T) {
|
||||
for _, connection := range []string{"issues", "pull requests"} {
|
||||
t.Run(connection, func(t *testing.T) {
|
||||
var restPage string
|
||||
downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, request *http.Request) {
|
||||
if request.URL.Path == "/api/graphql" {
|
||||
http.NotFound(writer, request)
|
||||
return
|
||||
}
|
||||
restPage = request.URL.Query().Get("page")
|
||||
_, _ = writer.Write([]byte(`[{"number":1,"title":"t","state":"open"}]`))
|
||||
})
|
||||
downloader.SkipReactions = true
|
||||
if connection == "issues" {
|
||||
issues, _, err := downloader.GetIssues(t.Context(), 3, 10)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, issues, 1)
|
||||
assert.EqualValues(t, 1, issues[0].Number)
|
||||
} else {
|
||||
prs, _, err := downloader.GetPullRequests(t.Context(), 3, 10)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, prs, 1)
|
||||
assert.EqualValues(t, 1, prs[0].Number)
|
||||
}
|
||||
assert.True(t, downloader.restFallback)
|
||||
assert.Equal(t, "1", restPage)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitBucketPaginatesWithoutLinkHeader(t *testing.T) {
|
||||
const total = 26
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
page, _ := strconv.Atoi(request.URL.Query().Get("page"))
|
||||
var items []string
|
||||
for number := (page-1)*25 + 1; number <= min(page*25, total); number++ {
|
||||
items = append(items, fmt.Sprintf(`{"number":%d,"title":"t"}`, number))
|
||||
}
|
||||
writer.Header().Set("X-RateLimit-Remaining", "4000")
|
||||
_, _ = writer.Write([]byte("[" + strings.Join(items, ",") + "]"))
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
downloader, err := NewGitBucketDownloader(t.Context(), server.URL, "", "", "", "o", "r")
|
||||
require.NoError(t, err)
|
||||
for _, connection := range []string{"issues", "pull requests"} {
|
||||
t.Run(connection, func(t *testing.T) {
|
||||
var count int
|
||||
for page := 1; ; page++ {
|
||||
require.LessOrEqual(t, page, 3)
|
||||
var isEnd bool
|
||||
if connection == "issues" {
|
||||
issues, done, err := downloader.GetIssues(t.Context(), page, 10)
|
||||
require.NoError(t, err)
|
||||
count += len(issues)
|
||||
isEnd = done
|
||||
} else {
|
||||
prs, done, err := downloader.GetPullRequests(t.Context(), page, 10)
|
||||
require.NoError(t, err)
|
||||
count += len(prs)
|
||||
isEnd = done
|
||||
}
|
||||
if isEnd {
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.Equal(t, total, count)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -80,9 +80,7 @@
|
||||
{{else if eq .Type 1}}
|
||||
<div class="timeline-item event" id="{{.HashTag}}">
|
||||
<span class="badge tw-bg-green tw-text-white">{{svg "octicon-dot-fill"}}</span>
|
||||
{{if not .OriginalAuthor}}
|
||||
{{template "shared/user/avatarlink" dict "user" .Poster}}
|
||||
{{end}}
|
||||
{{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}}
|
||||
<span class="comment-text-line">
|
||||
{{template "repo/issue/view_content/comments_authorlink" dict "comment" .}}
|
||||
{{if .Issue.IsPull}}
|
||||
@@ -95,9 +93,7 @@
|
||||
{{else if eq .Type 2}}
|
||||
<div class="timeline-item event" id="{{.HashTag}}">
|
||||
<span class="badge tw-bg-red tw-text-white">{{svg "octicon-issue-closed"}}</span>
|
||||
{{if not .OriginalAuthor}}
|
||||
{{template "shared/user/avatarlink" dict "user" .Poster}}
|
||||
{{end}}
|
||||
{{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}}
|
||||
<span class="comment-text-line">
|
||||
{{template "repo/issue/view_content/comments_authorlink" dict "comment" .}}
|
||||
{{if .Issue.IsPull}}
|
||||
@@ -110,9 +106,7 @@
|
||||
{{else if eq .Type 28}}
|
||||
<div class="timeline-item event" id="{{.HashTag}}">
|
||||
<span class="badge tw-bg-purple tw-text-white">{{svg "octicon-git-merge"}}</span>
|
||||
{{if not .OriginalAuthor}}
|
||||
{{template "shared/user/avatarlink" dict "user" .Poster}}
|
||||
{{end}}
|
||||
{{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}}
|
||||
<span class="comment-text-line">
|
||||
{{template "repo/issue/view_content/comments_authorlink" dict "comment" .}}
|
||||
{{$link := printf "%s/commit/%s" $.Repository.Link ($.Issue.PullRequest.MergedCommitID|PathEscape)}}
|
||||
@@ -169,7 +163,7 @@
|
||||
{{if or .AddedLabels .RemovedLabels}}
|
||||
<div class="timeline-item event" id="{{.HashTag}}">
|
||||
<span class="badge">{{svg "octicon-tag"}}</span>
|
||||
{{template "shared/user/avatarlink" dict "user" .Poster}}
|
||||
{{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}}
|
||||
<span class="comment-text-line">
|
||||
{{template "repo/issue/view_content/comments_authorlink" dict "comment" .}}
|
||||
{{if and .AddedLabels (not .RemovedLabels)}}
|
||||
@@ -185,9 +179,9 @@
|
||||
{{else if eq .Type 8}}
|
||||
<div class="timeline-item event" id="{{.HashTag}}">
|
||||
<span class="badge">{{svg "octicon-milestone"}}</span>
|
||||
{{template "shared/user/avatarlink" dict "user" .Poster}}
|
||||
{{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}}
|
||||
<span class="comment-text-line">
|
||||
{{template "shared/user/authorlink" .Poster}}
|
||||
{{template "repo/issue/view_content/comments_authorlink" dict "comment" .}}
|
||||
{{if gt .OldMilestoneID 0}}{{if gt .MilestoneID 0}}{{ctx.Locale.Tr "repo.issues.change_milestone_at" .OldMilestone.Name .Milestone.Name $createdStr}}{{else}}{{ctx.Locale.Tr "repo.issues.remove_milestone_at" .OldMilestone.Name $createdStr}}{{end}}{{else if gt .MilestoneID 0}}{{ctx.Locale.Tr "repo.issues.add_milestone_at" .Milestone.Name $createdStr}}{{end}}
|
||||
</span>
|
||||
</div>
|
||||
@@ -219,18 +213,18 @@
|
||||
{{else if eq .Type 10}}
|
||||
<div class="timeline-item event" id="{{.HashTag}}">
|
||||
<span class="badge">{{ctx.RenderUtils.RenderTimelineEventBadge $comment}}</span>
|
||||
{{template "shared/user/avatarlink" dict "user" .Poster}}
|
||||
{{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}}
|
||||
<span class="comment-text-line">
|
||||
{{template "shared/user/authorlink" .Poster}}
|
||||
{{template "repo/issue/view_content/comments_authorlink" dict "comment" .}}
|
||||
{{ctx.RenderUtils.RenderTimelineEventComment $comment $createdStr}}
|
||||
</span>
|
||||
</div>
|
||||
{{else if eq .Type 11}}
|
||||
<div class="timeline-item event" id="{{.HashTag}}">
|
||||
<span class="badge">{{svg "octicon-git-branch"}}</span>
|
||||
{{template "shared/user/avatarlink" dict "user" .Poster}}
|
||||
{{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}}
|
||||
<span class="comment-text-line">
|
||||
{{template "shared/user/authorlink" .Poster}}
|
||||
{{template "repo/issue/view_content/comments_authorlink" dict "comment" .}}
|
||||
{{$oldRef := HTMLFormat `<span class="tw-line-through">%s</span>` .OldRef}}
|
||||
{{ctx.Locale.Tr "repo.issues.delete_branch_at" $oldRef $createdStr}}
|
||||
</span>
|
||||
@@ -461,15 +455,15 @@
|
||||
{{else if eq .Type 23}}
|
||||
<div class="timeline-item event" id="{{.HashTag}}">
|
||||
<span class="badge">{{svg "octicon-lock"}}</span>
|
||||
{{template "shared/user/avatarlink" dict "user" .Poster}}
|
||||
{{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}}
|
||||
{{if .Content}}
|
||||
<span class="comment-text-line">
|
||||
{{template "shared/user/authorlink" .Poster}}
|
||||
{{template "repo/issue/view_content/comments_authorlink" dict "comment" .}}
|
||||
{{ctx.Locale.Tr "repo.issues.lock_with_reason" .Content $createdStr}}
|
||||
</span>
|
||||
{{else}}
|
||||
<span class="comment-text-line">
|
||||
{{template "shared/user/authorlink" .Poster}}
|
||||
{{template "repo/issue/view_content/comments_authorlink" dict "comment" .}}
|
||||
{{ctx.Locale.Tr "repo.issues.lock_no_reason" $createdStr}}
|
||||
</span>
|
||||
{{end}}
|
||||
@@ -477,18 +471,16 @@
|
||||
{{else if eq .Type 24}}
|
||||
<div class="timeline-item event" id="{{.HashTag}}">
|
||||
<span class="badge">{{svg "octicon-key"}}</span>
|
||||
{{template "shared/user/avatarlink" dict "user" .Poster}}
|
||||
{{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}}
|
||||
<span class="comment-text-line">
|
||||
{{template "shared/user/authorlink" .Poster}}
|
||||
{{template "repo/issue/view_content/comments_authorlink" dict "comment" .}}
|
||||
{{ctx.Locale.Tr "repo.issues.unlock_comment" $createdStr}}
|
||||
</span>
|
||||
</div>
|
||||
{{else if eq .Type 25}}
|
||||
<div class="timeline-item event">
|
||||
<span class="badge">{{svg "octicon-git-branch"}}</span>
|
||||
{{if not .OriginalAuthor}}
|
||||
{{template "shared/user/avatarlink" dict "user" .Poster}}
|
||||
{{end}}
|
||||
{{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}}
|
||||
<span class="comment-text-line">
|
||||
{{template "repo/issue/view_content/comments_authorlink" dict "comment" .}}
|
||||
{{ctx.Locale.Tr "repo.pulls.change_target_branch_at" .OldRef .NewRef $createdStr}}
|
||||
@@ -673,9 +665,9 @@
|
||||
{{else if or (eq .Type 36) (eq .Type 37)}}
|
||||
<div class="timeline-item event" id="{{.HashTag}}">
|
||||
<span class="badge">{{svg "octicon-pin" 16}}</span>
|
||||
{{template "shared/user/avatarlink" dict "user" .Poster}}
|
||||
{{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}}
|
||||
<span class="comment-text-line">
|
||||
{{template "shared/user/authorlink" .Poster}}
|
||||
{{template "repo/issue/view_content/comments_authorlink" dict "comment" .}}
|
||||
{{if eq .Type 36}}{{ctx.Locale.Tr "repo.issues.pin_comment" $createdStr}}
|
||||
{{else}}{{ctx.Locale.Tr "repo.issues.unpin_comment" $createdStr}}{{end}}
|
||||
</span>
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
{{if not .comment.OriginalAuthor}}{{template "shared/user/avatarlink" dict "user" .comment.Poster}}{{end}}
|
||||
Reference in new issue
Block a user