diff --git a/modules/migration/comment.go b/modules/migration/comment.go index e041758467d..7b86b10edcd 100644 --- a/modules/migration/comment.go +++ b/modules/migration/comment.go @@ -9,7 +9,6 @@ import "time" // Commentable can be commented upon type Commentable interface { Reviewable - GetContext() DownloaderContext } // Comment is a standard comment information diff --git a/modules/migration/review.go b/modules/migration/review.go index 79e821b2e10..88b25aa8ab3 100644 --- a/modules/migration/review.go +++ b/modules/migration/review.go @@ -8,6 +8,7 @@ import "time" // Reviewable can be reviewed type Reviewable interface { GetLocalIndex() int64 + GetContext() DownloaderContext // GetForeignIndex presents the foreign index, which could be misused: // For example, if there are 2 Gitea sites: site-A exports a dataset, then site-B imports it: diff --git a/routers/web/repo/issue_test.go b/routers/web/repo/issue_test.go index ad351079f96..05a23fdbfc3 100644 --- a/routers/web/repo/issue_test.go +++ b/routers/web/repo/issue_test.go @@ -218,6 +218,19 @@ func TestCombineLabelComments(t *testing.T) { }, }, }, + { + name: "migrated events of different original authors", + beforeCombined: []*issues_model.Comment{ + {Type: issues_model.CommentTypeComment, PosterID: 1}, + {Type: issues_model.CommentTypeLabel, PosterID: 1, OriginalAuthor: "alice", Content: "1", Label: &issues_model.Label{Name: "kind/bug"}}, + {Type: issues_model.CommentTypeLabel, PosterID: 1, OriginalAuthor: "bob", Content: "1", Label: &issues_model.Label{Name: "kind/ui"}}, + }, + afterCombined: []*issues_model.Comment{ + {Type: issues_model.CommentTypeComment, PosterID: 1}, + {Type: issues_model.CommentTypeLabel, PosterID: 1, OriginalAuthor: "alice", Content: "1", Label: &issues_model.Label{Name: "kind/bug"}, AddedLabels: []*issues_model.Label{{Name: "kind/bug"}}}, + {Type: issues_model.CommentTypeLabel, PosterID: 1, OriginalAuthor: "bob", Content: "1", Label: &issues_model.Label{Name: "kind/ui"}, AddedLabels: []*issues_model.Label{{Name: "kind/ui"}}}, + }, + }, { name: "kase 4", beforeCombined: []*issues_model.Comment{ diff --git a/routers/web/repo/issue_view.go b/routers/web/repo/issue_view.go index a195d91c951..967f84645a9 100644 --- a/routers/web/repo/issue_view.go +++ b/routers/web/repo/issue_view.go @@ -231,7 +231,7 @@ func combineLabelComments(issue *issues_model.Issue) { prev = issue.Comments[i-1] } if i == 0 || cur.Type != issues_model.CommentTypeLabel || - (prev != nil && prev.PosterID != cur.PosterID) || + (prev != nil && (prev.PosterID != cur.PosterID || prev.OriginalAuthor != cur.OriginalAuthor)) || (prev != nil && cur.CreatedUnix-prev.CreatedUnix >= 60) || (prev != nil && !sameCommentMetaData(prev.CommentMetaData, cur.CommentMetaData)) { if cur.Type == issues_model.CommentTypeLabel && cur.Label != nil { diff --git a/services/migrations/_mock_data/TestGitHubDownloadRepo/GET_%2Fapi%2Fv3%2Frepos%2Fgo-gitea%2Ftest_repo%2Fissues%3Fdirection%3Dasc%26page%3D1%26per_page%3D3%26sort%3Dcreated%26state%3Dall b/services/migrations/_mock_data/TestGitHubDownloadRepo/GET_%2Fapi%2Fv3%2Frepos%2Fgo-gitea%2Ftest_repo%2Fissues%3Fdirection%3Dasc%26page%3D1%26per_page%3D3%26sort%3Dcreated%26state%3Dall new file mode 100644 index 00000000000..aa85185ca1b --- /dev/null +++ b/services/migrations/_mock_data/TestGitHubDownloadRepo/GET_%2Fapi%2Fv3%2Frepos%2Fgo-gitea%2Ftest_repo%2Fissues%3Fdirection%3Dasc%26page%3D1%26per_page%3D3%26sort%3Dcreated%26state%3Dall @@ -0,0 +1,9 @@ +Content-Type: application/json; charset=utf-8 +X-Ratelimit-Limit: 5000 +X-Ratelimit-Remaining: 4931 +X-Ratelimit-Reset: 1773089427 +X-Ratelimit-Resource: core +X-Ratelimit-Used: 69 +Link: ; rel="next", ; rel="last" + +[{"number":1,"title":"one"},{"number":2,"title":"two"},{"number":3,"title":"three"}] diff --git a/services/migrations/_mock_data/TestGitHubDownloadRepo/GET_%2Fapi%2Fv3%2Frepos%2Fgo-gitea%2Ftest_repo%2Fissues%3Fdirection%3Dasc%26page%3D2%26per_page%3D3%26sort%3Dcreated%26state%3Dall b/services/migrations/_mock_data/TestGitHubDownloadRepo/GET_%2Fapi%2Fv3%2Frepos%2Fgo-gitea%2Ftest_repo%2Fissues%3Fdirection%3Dasc%26page%3D2%26per_page%3D3%26sort%3Dcreated%26state%3Dall new file mode 100644 index 00000000000..afbc061c584 --- /dev/null +++ b/services/migrations/_mock_data/TestGitHubDownloadRepo/GET_%2Fapi%2Fv3%2Frepos%2Fgo-gitea%2Ftest_repo%2Fissues%3Fdirection%3Dasc%26page%3D2%26per_page%3D3%26sort%3Dcreated%26state%3Dall @@ -0,0 +1,8 @@ +Content-Type: application/json; charset=utf-8 +X-Ratelimit-Limit: 5000 +X-Ratelimit-Remaining: 4931 +X-Ratelimit-Reset: 1773089427 +X-Ratelimit-Resource: core +X-Ratelimit-Used: 69 + +[] diff --git a/services/migrations/gitbucket.go b/services/migrations/gitbucket.go index 026fa3c12b8..516a9d923c9 100644 --- a/services/migrations/gitbucket.go +++ b/services/migrations/gitbucket.go @@ -82,6 +82,7 @@ func NewGitBucketDownloader(ctx context.Context, baseURL, userName, password, to githubDownloader.maxPerPage = 10 githubDownloader.SkipReactions = true githubDownloader.SkipReviews = true + githubDownloader.countPaginated = true return &GitBucketDownloader{ githubDownloader, }, nil diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 5134aa0a0a1..e3c03c8f186 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -517,6 +517,29 @@ func (g *GiteaLocalUploader) CreateComments(ctx context.Context, comments ...*ba cm.NewRef = fmt.Sprint(comment.Meta["NewRef"]) cm.Content = "" } + case issues_model.CommentTypeLabel: + if name, ok := comment.Meta["LabelName"].(string); ok { + if label, ok := g.labels[name]; ok { + cm.LabelID = label.ID + } + } + if cm.LabelID == 0 { + continue // label_id 0 renders nothing and gitea doctor deletes it + } + case issues_model.CommentTypeMilestone: + if title, ok := comment.Meta["MilestoneTitle"].(string); ok { + if removed, _ := comment.Meta["Removed"].(bool); removed { + cm.OldMilestoneID = g.milestones[title] + } else { + cm.MilestoneID = g.milestones[title] + } + } + if cm.MilestoneID == 0 && cm.OldMilestoneID == 0 { + continue + } + case issues_model.CommentTypeDeleteBranch: + cm.OldRef = comment.Content + cm.Content = "" case issues_model.CommentTypeMergePull: cm.Content = "" case issues_model.CommentTypePRScheduledToAutoMerge, issues_model.CommentTypePRUnScheduledToAutoMerge: diff --git a/services/migrations/gitea_uploader_test.go b/services/migrations/gitea_uploader_test.go index ce6b177a261..3b6d9de2727 100644 --- a/services/migrations/gitea_uploader_test.go +++ b/services/migrations/gitea_uploader_test.go @@ -22,6 +22,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "xorm.io/builder" ) func TestGiteaUploadRepo(t *testing.T) { @@ -222,3 +223,35 @@ func TestGiteaUploadRemapExternalUser(t *testing.T) { assert.NoError(t, err) assert.Equal(t, linkedUser.ID, target.GetUserID()) } + +func TestGiteaUploadTimelineLabelAndMilestoneComments(t *testing.T) { + unittest.PrepareTestEnv(t) + doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) + issue := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: 1}) + label := unittest.AssertExistsAndLoadBean(t, &issues_model.Label{ID: 1}) + milestone := unittest.AssertExistsAndLoadBean(t, &issues_model.Milestone{ID: 1}) + + ctx := t.Context() + uploader := NewGiteaLocalUploader(ctx, doer, doer.Name, "migrated") + uploader.issues[issue.Index] = issue + uploader.labels[label.Name] = label + uploader.milestones[milestone.Name] = milestone.ID + + require.NoError(t, uploader.CreateComments(ctx, + &base.Comment{IssueIndex: issue.Index, CommentType: "label", Content: "1", Meta: map[string]any{"LabelName": label.Name}}, + &base.Comment{IssueIndex: issue.Index, CommentType: "label", Meta: map[string]any{"LabelName": "deleted-upstream"}}, + &base.Comment{IssueIndex: issue.Index, CommentType: "milestone", Meta: map[string]any{"MilestoneTitle": milestone.Name}}, + &base.Comment{IssueIndex: issue.Index, CommentType: "milestone", Meta: map[string]any{"MilestoneTitle": milestone.Name, "Removed": true}}, + &base.Comment{IssueIndex: issue.Index, CommentType: "delete_branch", Content: "feature"}, + )) + + unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{IssueID: issue.ID, Type: issues_model.CommentTypeLabel, LabelID: label.ID, Content: "1"}) + unittest.AssertCountByCond(t, "comment", builder.Eq{ + "issue_id": issue.ID, + "type": issues_model.CommentTypeLabel, + "label_id": 0, + }, 0) + unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{IssueID: issue.ID, Type: issues_model.CommentTypeMilestone, MilestoneID: milestone.ID}) + unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{IssueID: issue.ID, Type: issues_model.CommentTypeMilestone, OldMilestoneID: milestone.ID}) + assert.Empty(t, unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{IssueID: issue.ID, Type: issues_model.CommentTypeDeleteBranch, OldRef: "feature"}).Content) +} diff --git a/services/migrations/github.go b/services/migrations/github.go index 40f2d3c3705..9a42c0ddac1 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -51,7 +51,15 @@ func (f *GithubDownloaderV3Factory) New(ctx context.Context, opts base.MigrateOp log.Trace("Create github downloader BaseURL: %s %s/%s", baseURL, oldOwner, oldName) - return NewGithubDownloaderV3(ctx, baseURL, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName) + downloader, err := NewGithubDownloaderV3(ctx, baseURL, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName) + if err != nil { + return nil, err + } + // GitHub rejects anonymous GraphQL requests, and comments only ride along with the issue and PR sweeps + if (opts.AuthToken != "" || opts.AuthPassword != "") && (opts.Issues || opts.PullRequests) { + return newGithubGraphQLDownloader(downloader, !opts.Comments), nil + } + return downloader, nil } // GitServiceType returns the type of git service @@ -63,21 +71,21 @@ func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType { // from github via APIv3 type GithubDownloaderV3 struct { base.NullDownloader - clients []*github.Client - baseURL string - repoOwner string - repoName string - userName string - password string - rates []*github.Rate - curClientIdx int - maxPerPage int - SkipReactions bool - SkipReviews bool - + clients []*github.Client + baseURL string + repoOwner string + repoName string + userName string + password string + rates []*github.Rate + curClientIdx int + maxPerPage int + SkipReactions bool + SkipReviews bool issuePage int issueCursor string nextIssueCursor string + countPaginated bool // GitBucket sends no Link header } // NewGithubDownloaderV3 creates a github Downloader via github v3 API @@ -460,32 +468,9 @@ func (g *GithubDownloaderV3) GetIssues(ctx context.Context, page, perPage int) ( labels = append(labels, convertGithubLabel(l)) } - // get reactions - var reactions []*base.Reaction - if !g.SkipReactions { - for i := 1; ; i++ { - g.waitAndPickClient(ctx) - res, resp, err := g.getClient().Reactions.ListIssueReactions(ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListReactionOptions{ - ListOptions: github.ListOptions{ - Page: i, - PerPage: perPage, - }, - }) - if err != nil { - return nil, false, err - } - g.setRate(&resp.Rate) - if len(res) == 0 { - break - } - for _, reaction := range res { - reactions = append(reactions, &base.Reaction{ - UserID: reaction.User.GetID(), - UserName: reaction.User.GetLogin(), - Content: reaction.GetContent(), - }) - } - } + reactions, err := g.getIssueReactions(ctx, issue.GetNumber(), perPage) + if err != nil { + return nil, false, err } var assignees []string @@ -514,7 +499,14 @@ func (g *GithubDownloaderV3) GetIssues(ctx context.Context, page, perPage int) ( } g.issuePage, g.issueCursor, g.nextIssueCursor = page, opt.After, resp.After - return allIssues, resp.NextPage == 0 && resp.After == "", nil + return allIssues, g.isLastPage(resp, len(issues), perPage), nil +} + +func (g *GithubDownloaderV3) isLastPage(resp *github.Response, n, perPage int) bool { + if g.countPaginated { + return n < perPage + } + return resp.NextPage == 0 && resp.After == "" } // SupportGetRepoComments return true if it supports get repo comments @@ -694,83 +686,97 @@ func (g *GithubDownloaderV3) GetPullRequests(ctx context.Context, page, perPage log.Trace("Request get pull requests %d/%d, but in fact get %d", perPage, page, len(prs)) g.setRate(&resp.Rate) for _, pr := range prs { - labels := make([]*base.Label, 0, len(pr.Labels)) - for _, l := range pr.Labels { - labels = append(labels, convertGithubLabel(l)) + basePR, err := g.convertGithubPullRequest(ctx, pr, perPage) + if err != nil { + return nil, false, err } - - // get reactions - var reactions []*base.Reaction - if !g.SkipReactions { - for i := 1; ; i++ { - g.waitAndPickClient(ctx) - res, resp, err := g.getClient().Reactions.ListIssueReactions(ctx, g.repoOwner, g.repoName, pr.GetNumber(), &github.ListReactionOptions{ - ListOptions: github.ListOptions{ - Page: i, - PerPage: perPage, - }, - }) - if err != nil { - return nil, false, err - } - g.setRate(&resp.Rate) - if len(res) == 0 { - break - } - for _, reaction := range res { - reactions = append(reactions, &base.Reaction{ - UserID: reaction.User.GetID(), - UserName: reaction.User.GetLogin(), - Content: reaction.GetContent(), - }) - } - } - } - - // download patch and saved as tmp file - g.waitAndPickClient(ctx) - - allPRs = append(allPRs, &base.PullRequest{ - Title: pr.GetTitle(), - Number: int64(pr.GetNumber()), - PosterID: pr.GetUser().GetID(), - PosterName: pr.GetUser().GetLogin(), - PosterEmail: pr.GetUser().GetEmail(), - Content: pr.GetBody(), - Milestone: pr.GetMilestone().GetTitle(), - State: pr.GetState(), - Created: pr.GetCreatedAt().Time, - Updated: pr.GetUpdatedAt().Time, - Closed: pr.ClosedAt.GetTime(), - Labels: labels, - Merged: pr.MergedAt != nil, - MergeCommitSHA: pr.GetMergeCommitSHA(), - MergedTime: pr.MergedAt.GetTime(), - IsLocked: pr.ActiveLockReason != nil, - Head: base.PullRequestBranch{ - Ref: pr.GetHead().GetRef(), - SHA: pr.GetHead().GetSHA(), - OwnerName: pr.GetHead().GetUser().GetLogin(), - RepoName: pr.GetHead().GetRepo().GetName(), - CloneURL: pr.GetHead().GetRepo().GetCloneURL(), // see below for SECURITY related issues here - }, - Base: base.PullRequestBranch{ - Ref: pr.GetBase().GetRef(), - SHA: pr.GetBase().GetSHA(), - RepoName: pr.GetBase().GetRepo().GetName(), - OwnerName: pr.GetBase().GetUser().GetLogin(), - }, - PatchURL: pr.GetPatchURL(), // see below for SECURITY related issues here - Reactions: reactions, - ForeignIndex: int64(*pr.Number), - IsDraft: pr.GetDraft(), - }) + allPRs = append(allPRs, basePR) // SECURITY: Ensure that the PR is safe - _ = CheckAndEnsureSafePR(allPRs[len(allPRs)-1], g.baseURL, g) + _ = CheckAndEnsureSafePR(basePR, g.baseURL, g) } - return allPRs, len(prs) < perPage, nil + return allPRs, g.isLastPage(resp, len(prs), perPage), nil +} + +func (g *GithubDownloaderV3) convertGithubPullRequest(ctx context.Context, pr *github.PullRequest, perPage int) (*base.PullRequest, error) { + labels := make([]*base.Label, 0, len(pr.Labels)) + for _, l := range pr.Labels { + labels = append(labels, convertGithubLabel(l)) + } + + reactions, err := g.getIssueReactions(ctx, pr.GetNumber(), perPage) + if err != nil { + return nil, err + } + + g.waitAndPickClient(ctx) + + return &base.PullRequest{ + Title: pr.GetTitle(), + Number: int64(pr.GetNumber()), + PosterID: pr.GetUser().GetID(), + PosterName: pr.GetUser().GetLogin(), + PosterEmail: pr.GetUser().GetEmail(), + Content: pr.GetBody(), + Milestone: pr.GetMilestone().GetTitle(), + State: pr.GetState(), + Created: pr.GetCreatedAt().Time, + Updated: pr.GetUpdatedAt().Time, + Closed: pr.ClosedAt.GetTime(), + Labels: labels, + Merged: pr.MergedAt != nil, + MergeCommitSHA: pr.GetMergeCommitSHA(), + MergedTime: pr.MergedAt.GetTime(), + IsLocked: pr.GetLocked(), + Head: base.PullRequestBranch{ + Ref: pr.GetHead().GetRef(), + SHA: pr.GetHead().GetSHA(), + OwnerName: pr.GetHead().GetUser().GetLogin(), + RepoName: pr.GetHead().GetRepo().GetName(), + CloneURL: pr.GetHead().GetRepo().GetCloneURL(), // see below for SECURITY related issues here + }, + Base: base.PullRequestBranch{ + Ref: pr.GetBase().GetRef(), + SHA: pr.GetBase().GetSHA(), + RepoName: pr.GetBase().GetRepo().GetName(), + OwnerName: pr.GetBase().GetUser().GetLogin(), + }, + PatchURL: pr.GetPatchURL(), // see below for SECURITY related issues here + Reactions: reactions, + ForeignIndex: int64(*pr.Number), + IsDraft: pr.GetDraft(), + }, nil +} + +func (g *GithubDownloaderV3) getIssueReactions(ctx context.Context, number, perPage int) ([]*base.Reaction, error) { + if g.SkipReactions { + return nil, nil + } + var reactions []*base.Reaction + for i := 1; ; i++ { + g.waitAndPickClient(ctx) + res, resp, err := g.getClient().Reactions.ListIssueReactions(ctx, g.repoOwner, g.repoName, number, &github.ListReactionOptions{ + ListOptions: github.ListOptions{ + Page: i, + PerPage: perPage, + }, + }) + if err != nil { + return nil, err + } + g.setRate(&resp.Rate) + if len(res) == 0 { + return reactions, nil + } + for _, reaction := range res { + reactions = append(reactions, &base.Reaction{ + UserID: reaction.User.GetID(), + UserName: reaction.User.GetLogin(), + Content: reaction.GetContent(), + }) + } + } } func convertGithubReview(r *github.PullRequestReview) *base.Review { diff --git a/services/migrations/github_graphql.go b/services/migrations/github_graphql.go new file mode 100644 index 00000000000..88c05e888d8 --- /dev/null +++ b/services/migrations/github_graphql.go @@ -0,0 +1,716 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package migrations + +import ( + "bytes" + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "maps" + "net" + "net/http" + "slices" + "strconv" + "strings" + "time" + + "gitea.dev/modules/egress/policy" + "gitea.dev/modules/json" + "gitea.dev/modules/log" + base "gitea.dev/modules/migration" + + "github.com/google/go-github/v92/github" +) + +const ( + githubGraphQLPageSize = 100 + githubGraphQLPRPageSize = 20 + githubGraphQLPointsFloor = 10 + maxGraphQLRateRetries = 5 + maxGraphQLTransientRetries = 5 + graphQLRateResetCap = time.Hour + graphQLNodeBatchSize = 100 + graphQLLabelPageSize = 30 + graphQLAssigneePageSize = 30 + graphQLReviewRequestPageSize = 50 +) + +const ( + gqlPageInfoFields = "pageInfo{hasNextPage endCursor}" + // databaseId is not on the Actor interface, every implementation needs its own fragment + gqlActorFields = "__typename login ... on User{databaseId} ... on Bot{databaseId} ... on Organization{databaseId} ... on Mannequin{databaseId}" + // teams in the RequestedReviewer union have no login + gqlReviewerFields = "__typename ... on User{login databaseId} ... on Bot{login databaseId} ... on Mannequin{login databaseId}" + gqlReactionFields = "content user{login databaseId}" +) + +var graphQLTransientRetryBase = 2 * time.Second + +var graphQLRateResetFallback = time.Minute // GitHub requires a minute between secondary limit retries. + +// a deterministic 502 for a too-heavy page never succeeds at the same size, so callers shrink it +var errGraphQLTransientExhausted = errors.New("github graphql: transient retries exhausted") + +var _ base.Downloader = &GithubGraphQLDownloader{} + +// GithubGraphQLDownloader sweeps issues and pull requests over GraphQL, the embedded REST downloader serves the rest and takes over on failure +type GithubGraphQLDownloader struct { + *GithubDownloaderV3 + gqlRates []*github.Rate // GraphQL has a separate rate budget + skipComments bool + issuesCursor string + prCursor string + restFallback bool + // REST restarts from its first page after a fallback, the migration skips what it already imported + restIssuePageOffset int + restPRPageOffset int +} + +func newGithubGraphQLDownloader(rest *GithubDownloaderV3, skipComments bool) *GithubGraphQLDownloader { + return &GithubGraphQLDownloader{GithubDownloaderV3: rest, gqlRates: make([]*github.Rate, len(rest.clients)), skipComments: skipComments} +} + +func (g *GithubGraphQLDownloader) GetIssues(ctx context.Context, page, perPage int) ([]*base.Issue, bool, error) { + if !g.restFallback { + issues, isEnd, err := g.getIssuesGraphQL(ctx, page, perPage) + if !g.fallBackToREST(ctx, err) { + return issues, isEnd, err + } + g.restIssuePageOffset = page - 1 + } + return g.GithubDownloaderV3.GetIssues(ctx, page-g.restIssuePageOffset, perPage) +} + +func (g *GithubGraphQLDownloader) GetPullRequests(ctx context.Context, page, perPage int) ([]*base.PullRequest, bool, error) { + if !g.restFallback { + prs, isEnd, err := g.getPullRequestsGraphQL(ctx, page, perPage) + if !g.fallBackToREST(ctx, err) { + return prs, isEnd, err + } + g.restPRPageOffset = page - 1 + } + return g.GithubDownloaderV3.GetPullRequests(ctx, page-g.restPRPageOffset, perPage) +} + +// fallBackToREST switches to REST for good when GraphQL fails, e.g. on a server or token without GraphQL support +func (g *GithubGraphQLDownloader) fallBackToREST(ctx context.Context, err error) bool { + if err == nil || ctx.Err() != nil { + return false + } + log.Warn("GitHub GraphQL failed, falling back to REST: %v", err) + g.restFallback = true + return true +} + +// SupportGetRepoComments is false so comments are served per issue from what each sweep page prefetched +func (g *GithubGraphQLDownloader) SupportGetRepoComments() bool { + return false +} + +func (g *GithubGraphQLDownloader) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) { + if prefetched, ok := commentable.GetContext().(*gqlPrefetched); ok { + return prefetched.comments, false, nil + } + return g.GithubDownloaderV3.GetComments(ctx, commentable) +} + +func (g *GithubGraphQLDownloader) GetReviews(ctx context.Context, reviewable base.Reviewable) ([]*base.Review, error) { + if prefetched, ok := reviewable.GetContext().(*gqlPrefetched); ok { + return prefetched.reviews, nil + } + return g.GithubDownloaderV3.GetReviews(ctx, reviewable) +} + +type graphQLRequest struct { + Query string `json:"query"` + Variables map[string]any `json:"variables"` +} + +type graphQLError struct { + Type string `json:"type"` + Message string `json:"message"` + Path []any `json:"path"` +} + +func (e graphQLError) Error() string { + if e.Type != "" { + return e.Type + ": " + e.Message + } + return e.Message +} + +func (g *GithubGraphQLDownloader) graphQLEndpoint() string { + restURL := g.clients[0].BaseURL() + if host, ok := strings.CutSuffix(restURL, "/api/v3/"); ok { + return host + "/api/graphql" + } + return restURL + "graphql" +} + +func (g *GithubGraphQLDownloader) doGraphQL(ctx context.Context, query string, vars map[string]any, out any) error { + payload, err := json.Marshal(graphQLRequest{Query: query, Variables: vars}) + if err != nil { + return err + } + + rateWaits, transientRetries := 0, 0 + for { + if err := g.pickGraphQLClient(ctx); err != nil { + return err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, g.graphQLEndpoint(), bytes.NewReader(payload)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + + resp, err := g.getClient().Client().Do(req) + if err != nil { + if ctx.Err() != nil || !isTransientTransportError(err) { + return err + } + if err := g.waitGraphQLTransient(ctx, &transientRetries, fmt.Sprintf("request failed: %v", err)); err != nil { + return err + } + continue + } + g.recordGraphQLRate(resp.Header) + + if resp.StatusCode != http.StatusOK { + detail := readGraphQLErrorBody(resp.Body) + resp.Body.Close() + switch { + case resp.StatusCode >= http.StatusInternalServerError: + if err := g.waitGraphQLTransient(ctx, &transientRetries, "status "+resp.Status+detail); err != nil { + return err + } + continue + case isGraphQLThrottled(resp, detail): + if rateWaits < maxGraphQLRateRetries*len(g.clients) { + rateWaits++ + g.restGraphQLClient(resp.Header) + continue + } + return fmt.Errorf("doGraphQL: rate limited, status %s%s", resp.Status, detail) + default: + return fmt.Errorf("doGraphQL: unexpected status %s%s", resp.Status, detail) + } + } + + envelope := struct { + Data any `json:"data"` + Errors []graphQLError `json:"errors"` + }{Data: out} + decodeErr := json.NewDecoder(resp.Body).Decode(&envelope) + resp.Body.Close() + if decodeErr != nil { + if err := g.waitGraphQLTransient(ctx, &transientRetries, fmt.Sprintf("truncated response: %v", decodeErr)); err != nil { + return err + } + continue + } + + if !onlyUnresolvedNodes(envelope.Errors) { + if isGraphQLRateLimited(envelope.Errors) && rateWaits < maxGraphQLRateRetries*len(g.clients) { + rateWaits++ + g.restGraphQLClient(resp.Header) + continue + } + return fmt.Errorf("doGraphQL: %w", envelope.Errors[0]) + } + return nil + } +} + +const maxGraphQLErrorBody = 512 + +func readGraphQLErrorBody(body io.Reader) string { + buf, err := io.ReadAll(io.LimitReader(body, maxGraphQLErrorBody)) + if err != nil || len(buf) == 0 { + return "" + } + return ": " + strings.Join(strings.Fields(string(buf)), " ") +} + +func isGraphQLThrottled(resp *http.Response, body string) bool { + switch resp.StatusCode { + case http.StatusTooManyRequests: + return true + case http.StatusForbidden: + return resp.Header.Get("Retry-After") != "" || + resp.Header.Get(github.HeaderRateRemaining) == "0" || + strings.Contains(strings.ToLower(body), "secondary rate limit") + } + return false +} + +func isTransientTransportError(err error) bool { + if errors.Is(err, policy.ErrDenied) { + return false + } + if _, ok := errors.AsType[*tls.CertificateVerificationError](err); ok { + return false + } + if dnsErr, ok := errors.AsType[*net.DNSError](err); ok { + return !dnsErr.IsNotFound + } + return true +} + +func (g *GithubGraphQLDownloader) waitGraphQLTransient(ctx context.Context, attempt *int, reason string) error { + if *attempt >= maxGraphQLTransientRetries { + return fmt.Errorf("%w: %s", errGraphQLTransientExhausted, reason) + } + wait := graphQLTransientRetryBase << *attempt + *attempt++ + log.Warn("github graphql: transient failure (%s), retrying in %s (attempt %d/%d)", + reason, wait, *attempt, maxGraphQLTransientRetries) + timer := time.NewTimer(wait) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func onlyUnresolvedNodes(errs []graphQLError) bool { + return !slices.ContainsFunc(errs, func(e graphQLError) bool { + return e.Type != "NOT_FOUND" || len(e.Path) == 0 || e.Path[0] != "nodes" + }) +} + +func isGraphQLRateLimited(errs []graphQLError) bool { + return slices.ContainsFunc(errs, func(e graphQLError) bool { return e.Type == "RATE_LIMITED" }) +} + +func isGraphQLTooHeavy(err error) bool { + var ge graphQLError + return errors.Is(err, errGraphQLTransientExhausted) || errors.As(err, &ge) && ge.Type == "RESOURCE_LIMITS_EXCEEDED" +} + +func (g *GithubGraphQLDownloader) pickGraphQLClient(ctx context.Context) error { + for { + best, soonest := -1, -1 + for i, rate := range g.gqlRates { + if rate == nil || !time.Now().Before(rate.Reset.Time) { + g.curClientIdx = i + return nil + } + if rate.Remaining > githubGraphQLPointsFloor && (best < 0 || rate.Remaining > g.gqlRates[best].Remaining) { + best = i + } + if soonest < 0 || rate.Reset.Before(g.gqlRates[soonest].Reset.Time) { + soonest = i + } + } + if best >= 0 { + g.curClientIdx = best + return nil + } + wait := min(time.Until(g.gqlRates[soonest].Reset.Time), graphQLRateResetCap) + log.Info("github graphql: every token's points budget is spent, sleeping %s until the earliest reset", wait.Round(time.Second)) + timer := time.NewTimer(wait) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + g.gqlRates[soonest] = nil // the reset may lie beyond the capped sleep + } + } +} + +func (g *GithubGraphQLDownloader) recordGraphQLRate(header http.Header) { + remaining, err := strconv.Atoi(header.Get(github.HeaderRateRemaining)) + if err != nil { + return + } + rate := &github.Rate{Remaining: remaining} + if epoch, err := strconv.ParseInt(header.Get(github.HeaderRateReset), 10, 64); err == nil { + rate.Reset = github.Timestamp{Time: time.Unix(epoch, 0)} + } + g.gqlRates[g.curClientIdx] = rate +} + +func (g *GithubGraphQLDownloader) restGraphQLClient(header http.Header) { + wait := graphQLRateResetWait(header) + log.Info("github graphql: token %d throttled, resting it for %s", g.curClientIdx, wait.Round(time.Second)) + g.gqlRates[g.curClientIdx] = &github.Rate{Reset: github.Timestamp{Time: time.Now().Add(wait)}} +} + +func isPrimaryBudgetSpent(header http.Header) bool { + remaining := header.Get(github.HeaderRateRemaining) + return remaining == "" || remaining == "0" +} + +func graphQLRateResetWait(header http.Header) time.Duration { + wait := graphQLRateResetFallback + if ra := header.Get("Retry-After"); ra != "" { + if secs, err := strconv.Atoi(strings.TrimSpace(ra)); err == nil && secs > 0 { + wait = time.Duration(secs) * time.Second + } + } else if reset := header.Get(github.HeaderRateReset); reset != "" && isPrimaryBudgetSpent(header) { + if epoch, err := strconv.ParseInt(reset, 10, 64); err == nil { + if d := time.Until(time.Unix(epoch, 0)); d > 0 { + wait = d + } + } + } + return min(wait, graphQLRateResetCap) +} + +// graphQLContentQuery orders by creation since a mutable sort key skips rows, nested comment reactions exceed GitHub's node limit +func (g *GithubGraphQLDownloader) graphQLContentQuery(connection, prFields, prDiscussion string) string { + fields := prFields + if !g.SkipReactions { + fields += " reactions(first:100){" + gqlPageInfoFields + " nodes{" + gqlReactionFields + "}}" + } + if !g.skipComments { + fields += " comments(first:100){" + gqlPageInfoFields + " nodes{id databaseId body createdAt updatedAt author{" + gqlActorFields + "}}}" + prDiscussion + } + return fmt.Sprintf(` +query($owner:String!,$name:String!,$cursor:String,$first:Int!){ + repository(owner:$owner,name:$name){ + %[1]s(first:$first,after:$cursor,orderBy:{field:CREATED_AT,direction:ASC}){ + %[2]s + nodes{ + id number title body state createdAt updatedAt closedAt locked + author{%[3]s} + milestone{title} + labels(first:%[4]d){%[2]s nodes{name color description}} + assignees(first:%[5]d){%[2]s nodes{login}} + %[6]s + } + } + } +}`, connection, gqlPageInfoFields, gqlActorFields, graphQLLabelPageSize, graphQLAssigneePageSize, fields) +} + +func (g *GithubGraphQLDownloader) graphQLIssuesQuery() string { + return g.graphQLContentQuery("issues", "", "") +} + +func nodeConnQuery(onType, connection, nodeFields string) string { + return `query($id:ID!,$cursor:String){node(id:$id){... on ` + onType + `{conn:` + connection + + `(first:100,after:$cursor){` + gqlPageInfoFields + ` nodes{` + nodeFields + `}}}}}` +} + +var ( + graphQLLabelsSweepQuery = nodeConnQuery("Labelable", "labels", "name color description") + graphQLAssigneesSweepQuery = nodeConnQuery("Assignable", "assignees", "login") + graphQLReactionsSweepQuery = nodeConnQuery("Reactable", "reactions", gqlReactionFields) +) + +const graphQLNodeReactionsQuery = `query($ids:[ID!]!){nodes(ids:$ids){id ... on Reactable{reactions(first:100){` + + gqlPageInfoFields + ` nodes{` + gqlReactionFields + `}}}}}` + +type gqlPageInfo struct { + HasNextPage bool `json:"hasNextPage"` + EndCursor string `json:"endCursor"` +} + +type gqlConn[T any] struct { + PageInfo gqlPageInfo `json:"pageInfo"` + Nodes []T `json:"nodes"` +} + +type gqlActor struct { + Typename string `json:"__typename"` + Login string `json:"login"` + DatabaseID int64 `json:"databaseId"` +} + +func (a *gqlActor) UnmarshalJSON(data []byte) error { + type plain gqlActor + if err := json.Unmarshal(data, (*plain)(a)); err != nil { + return err + } + switch { + case a.Typename == "" && a.Login == "": + a.Login = "ghost" // Match REST's ghost login with an unset ID for GHES. + case a.Typename == "Bot": + a.Login += "[bot]" + } + return nil +} + +type gqlCommit struct { + OID string `json:"oid"` +} + +type gqlReaction struct { + Content string `json:"content"` + User gqlActor `json:"user"` +} + +type gqlComment struct { + ID string `json:"id"` + DatabaseID int64 `json:"databaseId"` + Body string `json:"body"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` + Author gqlActor `json:"author"` +} + +type gqlLabel struct { + Name string `json:"name"` + Color string `json:"color"` + Description string `json:"description"` +} + +type gqlIssue struct { + ID string `json:"id"` + Number int64 `json:"number"` + Title string `json:"title"` + Body string `json:"body"` + State string `json:"state"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` + ClosedAt *time.Time `json:"closedAt"` + Locked bool `json:"locked"` + Author gqlActor `json:"author"` + Milestone *struct { + Title string `json:"title"` + } `json:"milestone"` + Labels gqlConn[gqlLabel] `json:"labels"` + Assignees gqlConn[gqlActor] `json:"assignees"` + Reactions gqlConn[gqlReaction] `json:"reactions"` + Comments gqlConn[gqlComment] `json:"comments"` +} + +type gqlIssuesResponse struct { + Repository struct { + Issues gqlConn[gqlIssue] `json:"issues"` + } `json:"repository"` +} + +type gqlPrefetched struct { + index int64 + comments []*base.Comment + commentIDs []string + reviews []*base.Review +} + +func (g *GithubGraphQLDownloader) getIssuesGraphQL(ctx context.Context, page, perPage int) ([]*base.Issue, bool, error) { + if page <= 1 { + g.issuesCursor = "" + } + var resp gqlIssuesResponse + if err := g.doGraphQLPage(ctx, g.graphQLIssuesQuery(), g.issuesCursor, graphQLPageSize(perPage, githubGraphQLPageSize), &resp); err != nil { + return nil, false, err + } + conn := resp.Repository.Issues + prefetched := map[string]*gqlPrefetched{} + issues := make([]*base.Issue, 0, len(conn.Nodes)) + for i := range conn.Nodes { + issue, err := g.convertGraphQLIssue(ctx, &conn.Nodes[i], prefetched) + if err != nil { + return nil, false, err + } + issues = append(issues, issue) + } + if err := g.enrichGraphQLPage(ctx, prefetched); err != nil { + return nil, false, err + } + g.issuesCursor = conn.PageInfo.EndCursor // only now, a failed page is retried from the same cursor + return issues, !conn.PageInfo.HasNextPage, nil +} + +func (g *GithubGraphQLDownloader) convertGraphQLIssue(ctx context.Context, node *gqlIssue, prefetchedByNodeID map[string]*gqlPrefetched) (*base.Issue, error) { + labels, err := allNodes(ctx, g, graphQLLabelsSweepQuery, node.ID, node.Labels) + if err != nil { + return nil, err + } + assignees, err := allNodes(ctx, g, graphQLAssigneesSweepQuery, node.ID, node.Assignees) + if err != nil { + return nil, err + } + reactions, err := g.allReactions(ctx, node.ID, node.Reactions) + if err != nil { + return nil, err + } + issue := &base.Issue{ + Number: node.Number, + Title: node.Title, + Content: node.Body, + PosterID: node.Author.DatabaseID, + PosterName: node.Author.Login, + State: strings.ToLower(node.State), + Created: node.CreatedAt, + Updated: node.UpdatedAt, + Closed: node.ClosedAt, + IsLocked: node.Locked, + Labels: make([]*base.Label, 0, len(labels)), + Reactions: reactions, + ForeignIndex: node.Number, + } + if node.Milestone != nil { + issue.Milestone = node.Milestone.Title + } + for _, label := range labels { + issue.Labels = append(issue.Labels, &base.Label{Name: label.Name, Color: label.Color, Description: label.Description}) + } + for _, assignee := range assignees { + issue.Assignees = append(issue.Assignees, assignee.Login) + } + if g.skipComments { + return issue, nil + } + + prefetched := &gqlPrefetched{index: node.Number} + issue.Context = prefetched + prefetchedByNodeID[node.ID] = prefetched + if node.Comments.PageInfo.HasNextPage { + prefetched.comments, err = g.getComments(ctx, issue) + return issue, err + } + for _, comment := range node.Comments.Nodes { + prefetched.commentIDs = append(prefetched.commentIDs, comment.ID) + prefetched.comments = append(prefetched.comments, &base.Comment{ + IssueIndex: node.Number, + Index: comment.DatabaseID, + PosterID: comment.Author.DatabaseID, + PosterName: comment.Author.Login, + Content: comment.Body, + Created: comment.CreatedAt, + Updated: comment.UpdatedAt, + }) + } + return issue, nil +} + +func (g *GithubGraphQLDownloader) enrichGraphQLPage(ctx context.Context, prefetchedByNodeID map[string]*gqlPrefetched) error { + if !g.SkipReactions { + comments := map[string]*base.Comment{} + for _, prefetched := range prefetchedByNodeID { + for i, id := range prefetched.commentIDs { + comments[id] = prefetched.comments[i] + } + } + for batch := range slices.Chunk(slices.Sorted(maps.Keys(comments)), graphQLNodeBatchSize) { + reactions, err := g.fetchNodeReactions(ctx, batch) + if err != nil { + return err + } + for id, reaction := range reactions { + comments[id].Reactions = reaction + } + } + } + return g.attachTimelineEvents(ctx, prefetchedByNodeID) +} + +func graphQLPageSize(perPage, maxSize int) int { + if perPage <= 0 { + return maxSize + } + return min(perPage, maxSize) // Keep pages within the caller's database batch size. +} + +func (g *GithubGraphQLDownloader) doGraphQLPage(ctx context.Context, query, cursor string, pageSize int, out any) error { + vars := map[string]any{"owner": g.repoOwner, "name": g.repoName} + if cursor != "" { + vars["cursor"] = cursor + } + for { + vars["first"] = pageSize + err := g.doGraphQL(ctx, query, vars, out) + if err == nil || !isGraphQLTooHeavy(err) || pageSize <= 1 { + return err + } + pageSize /= 2 + log.Warn("github graphql: page too heavy (%v), retrying with %d", err, pageSize) + } +} + +func allNodes[T any](ctx context.Context, g *GithubGraphQLDownloader, sweepQuery, nodeID string, conn gqlConn[T]) ([]T, error) { + nodes := conn.Nodes + for page := conn.PageInfo; page.HasNextPage; { + var resp struct { + Node struct { + Conn gqlConn[T] `json:"conn"` + } `json:"node"` + } + if err := g.doGraphQL(ctx, sweepQuery, map[string]any{"id": nodeID, "cursor": page.EndCursor}, &resp); err != nil { + return nil, err + } + nodes = append(nodes, resp.Node.Conn.Nodes...) + page = resp.Node.Conn.PageInfo + } + return nodes, nil +} + +func (g *GithubGraphQLDownloader) allReactions(ctx context.Context, nodeID string, conn gqlConn[gqlReaction]) ([]*base.Reaction, error) { + nodes, err := allNodes(ctx, g, graphQLReactionsSweepQuery, nodeID, conn) + if err != nil || len(nodes) == 0 { + return nil, err + } + reactions := make([]*base.Reaction, 0, len(nodes)) + for _, reaction := range nodes { + reactions = append(reactions, &base.Reaction{ + UserID: reaction.User.DatabaseID, + UserName: reaction.User.Login, + Content: graphQLReactionContent(reaction.Content), + }) + } + return reactions, nil +} + +func (g *GithubGraphQLDownloader) fetchNodeReactions(ctx context.Context, ids []string) (map[string][]*base.Reaction, error) { + var resp struct { + Nodes []struct { + ID string `json:"id"` + Reactions gqlConn[gqlReaction] `json:"reactions"` + } `json:"nodes"` + } + if err := g.doGraphQL(ctx, graphQLNodeReactionsQuery, map[string]any{"ids": ids}, &resp); err != nil { + if isGraphQLTooHeavy(err) && len(ids) > 1 { + return fetchHalves(ctx, ids, g.fetchNodeReactions) + } + return nil, err + } + out := make(map[string][]*base.Reaction, len(resp.Nodes)) + for _, node := range resp.Nodes { + if node.ID == "" { + continue + } + reactions, err := g.allReactions(ctx, node.ID, node.Reactions) + if err != nil { + return nil, err + } + out[node.ID] = reactions + } + return out, nil +} + +func fetchHalves[V any](ctx context.Context, ids []string, fetch func(context.Context, []string) (map[string]V, error)) (map[string]V, error) { + left, err := fetch(ctx, ids[:len(ids)/2]) + if err != nil { + return nil, err + } + right, err := fetch(ctx, ids[len(ids)/2:]) + if err != nil { + return nil, err + } + maps.Copy(left, right) + return left, nil +} + +func graphQLReactionContent(enum string) string { + switch enum { + case "THUMBS_UP": + return "+1" + case "THUMBS_DOWN": + return "-1" + default: + return strings.ToLower(enum) + } +} diff --git a/services/migrations/github_graphql_pr.go b/services/migrations/github_graphql_pr.go new file mode 100644 index 00000000000..29f12ab5418 --- /dev/null +++ b/services/migrations/github_graphql_pr.go @@ -0,0 +1,230 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package migrations + +import ( + "context" + "fmt" + "slices" + "time" + + base "gitea.dev/modules/migration" +) + +const graphQLPRFields = "mergedAt isDraft mergeCommit{oid} headRefName headRefOid baseRefName baseRefOid headRepository{name url} headRepositoryOwner{login} baseRepository{name owner{login}}" + +var graphQLPRDiscussion = fmt.Sprintf(` + reviews(first:50){%[1]s nodes{ + databaseId state body createdAt submittedAt author{%[2]s} commit{oid} + comments(first:50){%[1]s nodes{databaseId body path diffHunk position commit{oid} author{%[2]s} createdAt updatedAt replyTo{databaseId}}} + }} + reviewRequests(first:%[3]d){%[1]s nodes{requestedReviewer{%[4]s}}}`, + gqlPageInfoFields, gqlActorFields, graphQLReviewRequestPageSize, gqlReviewerFields) + +var graphQLReviewRequestsSweepQuery = nodeConnQuery("PullRequest", "reviewRequests", "requestedReviewer{"+gqlReviewerFields+"}") + +func (g *GithubGraphQLDownloader) graphQLPullRequestsQuery() string { + return g.graphQLContentQuery("pullRequests", graphQLPRFields, graphQLPRDiscussion) +} + +type gqlRepoRef struct { + Name string `json:"name"` + URL string `json:"url"` + Owner gqlActor `json:"owner"` +} + +type gqlReviewComment struct { + DatabaseID int64 `json:"databaseId"` + Body string `json:"body"` + Path string `json:"path"` + DiffHunk string `json:"diffHunk"` + Position int `json:"position"` + Commit *gqlCommit `json:"commit"` + Author gqlActor `json:"author"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` + ReplyTo *struct { + DatabaseID int64 `json:"databaseId"` + } `json:"replyTo"` +} + +type gqlReview struct { + DatabaseID int64 `json:"databaseId"` + State string `json:"state"` + Body string `json:"body"` + CreatedAt time.Time `json:"createdAt"` + SubmittedAt *time.Time `json:"submittedAt"` + Author gqlActor `json:"author"` + Commit *gqlCommit `json:"commit"` + Comments gqlConn[gqlReviewComment] `json:"comments"` +} + +type gqlReviewRequest struct { + RequestedReviewer *gqlActor `json:"requestedReviewer"` +} + +type gqlPullRequest struct { + gqlIssue + MergedAt *time.Time `json:"mergedAt"` + IsDraft bool `json:"isDraft"` + MergeCommit *gqlCommit `json:"mergeCommit"` + HeadRefName string `json:"headRefName"` + HeadRefOID string `json:"headRefOid"` + BaseRefName string `json:"baseRefName"` + BaseRefOID string `json:"baseRefOid"` + HeadRepo *gqlRepoRef `json:"headRepository"` + HeadRepoOwner *gqlActor `json:"headRepositoryOwner"` + BaseRepo *gqlRepoRef `json:"baseRepository"` + Reviews gqlConn[gqlReview] `json:"reviews"` + ReviewRequests gqlConn[gqlReviewRequest] `json:"reviewRequests"` +} + +type gqlPullRequestsResponse struct { + Repository struct { + PullRequests gqlConn[gqlPullRequest] `json:"pullRequests"` + } `json:"repository"` +} + +func (g *GithubGraphQLDownloader) getPullRequestsGraphQL(ctx context.Context, page, perPage int) ([]*base.PullRequest, bool, error) { + if page <= 1 { + g.prCursor = "" + } + var resp gqlPullRequestsResponse + if err := g.doGraphQLPage(ctx, g.graphQLPullRequestsQuery(), g.prCursor, graphQLPageSize(perPage, githubGraphQLPRPageSize), &resp); err != nil { + return nil, false, err + } + conn := resp.Repository.PullRequests + prefetched := map[string]*gqlPrefetched{} + prs := make([]*base.PullRequest, 0, len(conn.Nodes)) + for i := range conn.Nodes { + pr, err := g.convertGraphQLPullRequest(ctx, &conn.Nodes[i], prefetched) + if err != nil { + return nil, false, err + } + _ = CheckAndEnsureSafePR(pr, g.baseURL, g) // SECURITY: like the REST path + prs = append(prs, pr) + } + if err := g.enrichGraphQLPage(ctx, prefetched); err != nil { + return nil, false, err + } + g.prCursor = conn.PageInfo.EndCursor // only now, a failed page is retried from the same cursor + return prs, !conn.PageInfo.HasNextPage, nil +} + +func (g *GithubGraphQLDownloader) convertGraphQLPullRequest(ctx context.Context, node *gqlPullRequest, prefetchedByNodeID map[string]*gqlPrefetched) (*base.PullRequest, error) { + issue, err := g.convertGraphQLIssue(ctx, &node.gqlIssue, prefetchedByNodeID) + if err != nil { + return nil, err + } + if prefetched, ok := issue.Context.(*gqlPrefetched); ok { + if prefetched.reviews, err = g.prefetchReviews(ctx, node); err != nil { + return nil, err + } + } + pr := &base.PullRequest{ + Number: issue.Number, + Title: issue.Title, + PosterID: issue.PosterID, + PosterName: issue.PosterName, + Content: issue.Content, + Milestone: issue.Milestone, + State: "closed", // REST reports a merged PR as closed + Created: issue.Created, + Updated: issue.Updated, + Closed: issue.Closed, + Labels: issue.Labels, + Assignees: issue.Assignees, + IsLocked: issue.IsLocked, + Reactions: issue.Reactions, + ForeignIndex: issue.ForeignIndex, + Context: issue.Context, + PatchURL: fmt.Sprintf("%s/%s/%s/pull/%d.patch", g.baseURL, g.repoOwner, g.repoName, node.Number), + Merged: node.MergedAt != nil || node.State == "MERGED", + MergedTime: node.MergedAt, + IsDraft: node.IsDraft, + Head: base.PullRequestBranch{Ref: node.HeadRefName, SHA: node.HeadRefOID}, + Base: base.PullRequestBranch{Ref: node.BaseRefName, SHA: node.BaseRefOID}, + } + if node.State == "OPEN" { + pr.State = "open" + } + if node.MergeCommit != nil { + pr.MergeCommitSHA = node.MergeCommit.OID + } + if node.HeadRepoOwner != nil { + pr.Head.OwnerName = node.HeadRepoOwner.Login + } + if node.HeadRepo != nil { + pr.Head.RepoName = node.HeadRepo.Name + pr.Head.CloneURL = node.HeadRepo.URL + ".git" + } + if node.BaseRepo != nil { + pr.Base.RepoName = node.BaseRepo.Name + pr.Base.OwnerName = node.BaseRepo.Owner.Login + } + return pr, nil +} + +func (g *GithubGraphQLDownloader) prefetchReviews(ctx context.Context, node *gqlPullRequest) ([]*base.Review, error) { + if node.Reviews.PageInfo.HasNextPage || slices.ContainsFunc(node.Reviews.Nodes, func(r gqlReview) bool { return r.Comments.PageInfo.HasNextPage }) { + return g.GetReviews(ctx, &base.PullRequest{Number: node.Number, ForeignIndex: node.Number}) + } + requests, err := allNodes(ctx, g, graphQLReviewRequestsSweepQuery, node.ID, node.ReviewRequests) + if err != nil { + return nil, err + } + return convertGraphQLReviews(node, requests), nil +} + +func convertGraphQLReviews(node *gqlPullRequest, requests []gqlReviewRequest) []*base.Review { + reviews := make([]*base.Review, 0, len(node.Reviews.Nodes)+len(requests)) + for _, sourceReview := range node.Reviews.Nodes { + review := &base.Review{ + ID: sourceReview.DatabaseID, + IssueIndex: node.Number, + ReviewerID: sourceReview.Author.DatabaseID, + ReviewerName: sourceReview.Author.Login, + Content: sourceReview.Body, + CreatedAt: sourceReview.CreatedAt, + State: sourceReview.State, + } + if sourceReview.SubmittedAt != nil { + review.CreatedAt = *sourceReview.SubmittedAt + } + if sourceReview.Commit != nil { + review.CommitID = sourceReview.Commit.OID + } + for _, sourceComment := range sourceReview.Comments.Nodes { + comment := &base.ReviewComment{ + ID: sourceComment.DatabaseID, + Content: sourceComment.Body, + TreePath: sourceComment.Path, + DiffHunk: sourceComment.DiffHunk, + Position: sourceComment.Position, + PosterID: sourceComment.Author.DatabaseID, + CreatedAt: sourceComment.CreatedAt, + UpdatedAt: sourceComment.UpdatedAt, + } + if sourceComment.ReplyTo != nil { + comment.InReplyTo = sourceComment.ReplyTo.DatabaseID + } + if sourceComment.Commit != nil { + comment.CommitID = sourceComment.Commit.OID + } + review.Comments = append(review.Comments, comment) + } + reviews = append(reviews, review) + } + for _, request := range requests { + if reviewer := request.RequestedReviewer; reviewer != nil && reviewer.Login != "" { + reviews = append(reviews, &base.Review{ + IssueIndex: node.Number, + ReviewerID: reviewer.DatabaseID, + ReviewerName: reviewer.Login, + State: base.ReviewStateRequestReview, + }) + } + } + return reviews +} diff --git a/services/migrations/github_graphql_pr_test.go b/services/migrations/github_graphql_pr_test.go new file mode 100644 index 00000000000..d62d4f7a35b --- /dev/null +++ b/services/migrations/github_graphql_pr_test.go @@ -0,0 +1,104 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package migrations + +import ( + "io" + "net/http" + "strings" + "testing" + "time" + + "gitea.dev/modules/json" + base "gitea.dev/modules/migration" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGraphQLPullRequestParsing(t *testing.T) { + var response gqlPullRequestsResponse + require.NoError(t, json.Unmarshal([]byte(`{"repository":{"pullRequests":{"nodes":[{ + "number":7,"title":"add feature","body":"does the thing","state":"MERGED", + "createdAt":"2023-05-01T00:00:00Z","updatedAt":"2023-06-01T00:00:00Z", + "closedAt":"2023-06-01T00:00:00Z","mergedAt":"2023-06-01T00:00:00Z", + "author":{"login":"contributor","databaseId":4242},"isDraft":true, + "mergeCommit":{"oid":"abc123"},"headRefName":"feature-x","headRefOid":"deadbeef", + "baseRefName":"main","baseRefOid":"cafebabe", + "headRepository":{"name":"gitea","url":"https://github.com/contributor/gitea"}, + "headRepositoryOwner":{"login":"contributor"}, + "baseRepository":{"name":"gitea","owner":{"login":"go-gitea"}}, + "reviews":{"nodes":[{ + "databaseId":900,"state":"CHANGES_REQUESTED","body":"please fix", + "createdAt":"2023-05-03T00:00:00Z","submittedAt":"2023-05-03T01:00:00Z", + "author":{"login":"reviewer","databaseId":77},"commit":{"oid":"sha1sha"}, + "comments":{"nodes":[{"databaseId":950,"body":"nit here","path":"main.go","diffHunk":"@@ -1 +1 @@", + "position":3,"commit":{"oid":"sha1sha"},"author":{"databaseId":77},"replyTo":{"databaseId":949}}]} + }]}, + "reviewRequests":{"nodes":[{"requestedReviewer":{"__typename":"Bot","login":"pending-rev","databaseId":88}}, + {"requestedReviewer":{"__typename":"Team"}},{"requestedReviewer":null}]} + }]}}}`), &response)) + downloader := &GithubGraphQLDownloader{GithubDownloaderV3: &GithubDownloaderV3{baseURL: "https://github.com", repoOwner: "go-gitea", repoName: "gitea"}} + pr, err := downloader.convertGraphQLPullRequest(t.Context(), &response.Repository.PullRequests.Nodes[0], map[string]*gqlPrefetched{}) + require.NoError(t, err) + merged := time.Date(2023, 6, 1, 0, 0, 0, 0, time.UTC) + assert.Equal(t, &base.PullRequest{ + Number: 7, Title: "add feature", Content: "does the thing", State: "closed", PosterID: 4242, PosterName: "contributor", + Created: time.Date(2023, 5, 1, 0, 0, 0, 0, time.UTC), Updated: merged, Closed: &merged, + Merged: true, MergedTime: &merged, IsDraft: true, MergeCommitSHA: "abc123", ForeignIndex: 7, + Head: base.PullRequestBranch{Ref: "feature-x", SHA: "deadbeef", OwnerName: "contributor", RepoName: "gitea", CloneURL: "https://github.com/contributor/gitea.git"}, + Base: base.PullRequestBranch{Ref: "main", SHA: "cafebabe", OwnerName: "go-gitea", RepoName: "gitea"}, + PatchURL: "https://github.com/go-gitea/gitea/pull/7.patch", Labels: []*base.Label{}, Context: pr.Context, + }, pr) + reviews, err := downloader.GetReviews(t.Context(), pr) + require.NoError(t, err) + assert.Equal(t, []*base.Review{ + { + ID: 900, IssueIndex: 7, ReviewerID: 77, ReviewerName: "reviewer", Content: "please fix", State: "CHANGES_REQUESTED", CommitID: "sha1sha", + CreatedAt: time.Date(2023, 5, 3, 1, 0, 0, 0, time.UTC), + Comments: []*base.ReviewComment{{ID: 950, InReplyTo: 949, Content: "nit here", TreePath: "main.go", DiffHunk: "@@ -1 +1 @@", Position: 3, CommitID: "sha1sha", PosterID: 77}}, + }, + {IssueIndex: 7, ReviewerID: 88, ReviewerName: "pending-rev[bot]", State: base.ReviewStateRequestReview}, + }, reviews) +} + +func TestGraphQLDiscussionOverflowFallsBackToREST(t *testing.T) { + for _, connection := range []string{"comments", "reviews", "review comments"} { + t.Run(connection, func(t *testing.T) { + downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, request *http.Request) { + switch { + case strings.HasSuffix(request.URL.Path, "/issues/7/comments"): + _, _ = io.WriteString(writer, `[{"id":500,"body":"REST comment"}]`) + case strings.HasSuffix(request.URL.Path, "/pulls/7/reviews"): + _, _ = io.WriteString(writer, `[{"id":900,"body":"REST review","state":"APPROVED"}]`) + case strings.HasSuffix(request.URL.Path, "/requested_reviewers"): + _, _ = io.WriteString(writer, `{}`) + default: + _, _ = io.WriteString(writer, `[]`) + } + }) + downloader.SkipReactions = true + node := &gqlPullRequest{gqlIssue: gqlIssue{Number: 7}} + switch connection { + case "comments": + node.Comments.PageInfo.HasNextPage = true + case "reviews": + node.Reviews.PageInfo.HasNextPage = true + case "review comments": + node.Reviews.Nodes = []gqlReview{{Comments: gqlConn[gqlReviewComment]{PageInfo: gqlPageInfo{HasNextPage: true}}}} + } + pr, err := downloader.convertGraphQLPullRequest(t.Context(), node, map[string]*gqlPrefetched{}) + require.NoError(t, err) + if connection == "comments" { + comments, _, err := downloader.GetComments(t.Context(), pr) + require.NoError(t, err) + assert.Equal(t, []*base.Comment{{IssueIndex: 7, Index: 500, Content: "REST comment"}}, comments) + } else { + reviews, err := downloader.GetReviews(t.Context(), pr) + require.NoError(t, err) + assert.Equal(t, []*base.Review{{IssueIndex: 7, ID: 900, Content: "REST review", State: "APPROVED"}}, reviews) + } + }) + } +} diff --git a/services/migrations/github_graphql_test.go b/services/migrations/github_graphql_test.go new file mode 100644 index 00000000000..7efc8db3154 --- /dev/null +++ b/services/migrations/github_graphql_test.go @@ -0,0 +1,428 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package migrations + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "strings" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "gitea.dev/modules/egress/policy" + "gitea.dev/modules/json" + base "gitea.dev/modules/migration" + "gitea.dev/modules/test" + + "github.com/google/go-github/v92/github" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newGithubTestDownloader(t *testing.T, handler http.HandlerFunc, spareTokens ...string) *GithubGraphQLDownloader { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + writer.Header().Set("Content-Type", "application/json") + writer.Header().Set("X-RateLimit-Remaining", "4000") + handler(writer, request) + })) + t.Cleanup(server.Close) + downloader, err := NewGithubDownloaderV3(t.Context(), server.URL, "", "", strings.Join(append([]string{"tok"}, spareTokens...), ","), "o", "r") + require.NoError(t, err) + return newGithubGraphQLDownloader(downloader, false) +} + +func TestGraphQLEndpoint(t *testing.T) { + for baseURL, endpoint := range map[string]string{ + "https://github.com": "https://api.github.com/graphql", + "https://api.github.com": "https://api.github.com/graphql", + "https://ghe.example.com": "https://ghe.example.com/api/graphql", + "https://ghe.example.com/": "https://ghe.example.com/api/graphql", + } { + downloader, err := NewGithubDownloaderV3(t.Context(), baseURL, "", "", "tok", "o", "r") + require.NoError(t, err) + assert.Equal(t, endpoint, newGithubGraphQLDownloader(downloader, false).graphQLEndpoint(), baseURL) + } +} + +func TestGraphQLRateResetWait(t *testing.T) { + for _, testcase := range []struct { + name string + headers map[string]string + want time.Duration + }{ + {"retry after takes precedence", map[string]string{"Retry-After": "30", "X-RateLimit-Reset": "9999999999"}, 30 * time.Second}, + {"no headers", nil, graphQLRateResetFallback}, + {"past primary reset", map[string]string{"X-RateLimit-Reset": "1"}, graphQLRateResetFallback}, + {"secondary reset is not primary reset", map[string]string{"X-RateLimit-Remaining": "4321", "X-RateLimit-Reset": "9999999999"}, graphQLRateResetFallback}, + {"primary reset is capped", map[string]string{"X-RateLimit-Remaining": "0", "X-RateLimit-Reset": "9999999999"}, graphQLRateResetCap}, + {"retry after is capped", map[string]string{"Retry-After": "999999"}, graphQLRateResetCap}, + } { + t.Run(testcase.name, func(t *testing.T) { + header := http.Header{} + for name, value := range testcase.headers { + header.Set(name, value) + } + assert.Equal(t, testcase.want, graphQLRateResetWait(header)) + }) + } +} + +func TestDoGraphQLRetriesTransientFailures(t *testing.T) { + defer test.MockVariableValue(&graphQLTransientRetryBase, time.Duration(0))() + for _, testcase := range []struct { + name string + failure string + failures int32 + }{ + {"bad gateway then success", "502", 1}, + {"truncated body then success", "body", 1}, + {"connection closed then success", "connection", 1}, + {"persistent bad gateway", "502", maxGraphQLTransientRetries + 1}, + } { + t.Run(testcase.name, func(t *testing.T) { + var calls atomic.Int32 + downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, _request *http.Request) { + if calls.Add(1) > testcase.failures { + _, _ = io.WriteString(writer, `{"data":{"ok":true}}`) + return + } + switch testcase.failure { + case "502": + writer.WriteHeader(http.StatusBadGateway) + case "body": + _, _ = io.WriteString(writer, `{"data":{"ok":`) + case "connection": + connection, _, err := http.NewResponseController(writer).Hijack() + if assert.NoError(t, err) { + _ = connection.Close() + } + } + }) + var response struct { + OK bool `json:"ok"` + } + err := downloader.doGraphQL(t.Context(), "query{}", nil, &response) + if testcase.failures > maxGraphQLTransientRetries { + require.ErrorIs(t, err, errGraphQLTransientExhausted) + assert.Equal(t, int32(maxGraphQLTransientRetries+1), calls.Load()) + } else { + require.NoError(t, err) + assert.True(t, response.OK) + assert.Equal(t, testcase.failures+1, calls.Load()) + } + }) + } +} + +func TestDoGraphQLWaitsOutSecondaryRateLimit(t *testing.T) { + defer test.MockVariableValue(&graphQLRateResetFallback, time.Millisecond)() + for _, testcase := range []struct { + name string + status int + body string + }{ + {"GraphQL rate limit", http.StatusOK, `{"errors":[{"type":"RATE_LIMITED"}]}`}, + {"429 without retry after", http.StatusTooManyRequests, ""}, + {"403 spent primary quota", http.StatusForbidden, ""}, + {"403 secondary limit with quota left", http.StatusForbidden, `{"message":"You have exceeded a secondary rate limit."}`}, + } { + t.Run(testcase.name, func(t *testing.T) { + tokens := make(chan string, 3) + downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, request *http.Request) { + tokens <- request.Header.Get("Authorization") + if request.Header.Get("Authorization") == "Bearer tok" { + writer.Header().Set("X-RateLimit-Reset", strconv.FormatInt(time.Now().Add(time.Hour).Unix(), 10)) + if testcase.body == "" { + writer.Header().Set("X-RateLimit-Remaining", "0") + } + writer.WriteHeader(testcase.status) + _, _ = io.WriteString(writer, testcase.body) + return + } + _, _ = io.WriteString(writer, `{"data":{"ok":true}}`) + }, "spare") + var response struct { + OK bool `json:"ok"` + } + require.NoError(t, downloader.doGraphQL(t.Context(), "query{}", nil, &response)) + assert.True(t, response.OK) + assert.Equal(t, []string{"Bearer tok", "Bearer spare"}, []string{<-tokens, <-tokens}) + assert.Zero(t, downloader.gqlRates[0].Remaining) + }) + } +} + +func TestDoGraphQLRateRetryBudgetPerToken(t *testing.T) { + defer test.MockVariableValue(&graphQLRateResetFallback, time.Duration(0))() + var calls atomic.Int32 + downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, _request *http.Request) { + calls.Add(1) + _, _ = io.WriteString(writer, `{"errors":[{"type":"RATE_LIMITED"}]}`) + }, "spare") + require.Error(t, downloader.doGraphQL(t.Context(), "query{}", nil, &struct{}{})) + assert.Equal(t, int32(maxGraphQLRateRetries*len(downloader.clients)+1), calls.Load()) +} + +func TestDoGraphQLFatalStatusQuotesBody(t *testing.T) { + var calls atomic.Int32 + downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, _request *http.Request) { + calls.Add(1) + writer.WriteHeader(http.StatusForbidden) + _, _ = io.WriteString(writer, `{"message":"Resource not accessible by integration"}`) + }) + assert.ErrorContains(t, downloader.doGraphQL(t.Context(), "query{}", nil, &struct{}{}), "Resource not accessible by integration") + assert.EqualValues(t, 1, calls.Load()) +} + +func TestPickGraphQLClient(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + reset := github.Timestamp{Time: time.Now().Add(2 * graphQLRateResetCap)} + downloader := &GithubGraphQLDownloader{GithubDownloaderV3: &GithubDownloaderV3{}, gqlRates: []*github.Rate{ + {Remaining: 1000, Reset: reset}, + {Remaining: 4000, Reset: reset}, + {Remaining: 5, Reset: reset}, + }} + require.NoError(t, downloader.pickGraphQLClient(t.Context())) + assert.Equal(t, 1, downloader.curClientIdx) + downloader.gqlRates[2] = nil + require.NoError(t, downloader.pickGraphQLClient(t.Context())) + assert.Equal(t, 2, downloader.curClientIdx) + downloader.gqlRates = []*github.Rate{{Reset: github.Timestamp{Time: reset.Add(time.Hour)}}, {Reset: reset}} + started := time.Now() + require.NoError(t, downloader.pickGraphQLClient(t.Context())) + assert.Equal(t, graphQLRateResetCap, time.Since(started)) + assert.Equal(t, 1, downloader.curClientIdx) + assert.Nil(t, downloader.gqlRates[1]) + }) +} + +func TestIsTransientTransportError(t *testing.T) { + wrapped := func(err error) error { + return &url.Error{Op: "Post", URL: "https://example.com", Err: &net.OpError{Op: "dial", Err: err}} + } + for _, testcase := range []struct { + name string + err error + transient bool + }{ + {"blocked by policy", fmt.Errorf("blocked: %w", policy.ErrDenied), false}, + {"host does not resolve", &net.DNSError{IsNotFound: true}, false}, + {"TLS verification failed", &tls.CertificateVerificationError{Err: x509.UnknownAuthorityError{}}, false}, + {"DNS server failure", &net.DNSError{IsTemporary: true}, true}, + {"connection reset", errors.New("connection reset by peer"), true}, + } { + t.Run(testcase.name, func(t *testing.T) { + assert.Equal(t, testcase.transient, isTransientTransportError(wrapped(testcase.err))) + }) + } +} + +func TestGraphQLQueryGating(t *testing.T) { + for _, skipComments := range []bool{false, true} { + for _, skipReactions := range []bool{false, true} { + downloader := &GithubGraphQLDownloader{GithubDownloaderV3: &GithubDownloaderV3{SkipReactions: skipReactions}, skipComments: skipComments} + for name, query := range map[string]string{"issues": downloader.graphQLIssuesQuery(), "pullRequests": downloader.graphQLPullRequestsQuery()} { + t.Run(fmt.Sprintf("%s/comments=%t/reactions=%t", name, !skipComments, !skipReactions), func(t *testing.T) { + assert.Equal(t, !skipComments, strings.Contains(query, "comments(")) + assert.Equal(t, !skipComments && name == "pullRequests", strings.Contains(query, "reviews(")) + assert.Equal(t, !skipComments && name == "pullRequests", strings.Contains(query, "reviewRequests(")) + assert.Equal(t, !skipReactions, strings.Contains(query, "reactions(")) + assert.LessOrEqual(t, strings.Count(query, "reactions("), 1) + assert.Contains(t, query, "id number title") + assert.Contains(t, query, "orderBy:{field:CREATED_AT,direction:ASC}") + assert.Contains(t, query, "labels(first:30){"+gqlPageInfoFields) + assert.Contains(t, query, "... on Bot{databaseId}") + }) + } + } + } +} + +func TestGraphQLPageSize(t *testing.T) { + assert.Equal(t, 41, graphQLPageSize(41, githubGraphQLPageSize)) + assert.Equal(t, githubGraphQLPageSize, graphQLPageSize(500, githubGraphQLPageSize)) + assert.Equal(t, githubGraphQLPageSize, graphQLPageSize(0, githubGraphQLPageSize)) + assert.Equal(t, githubGraphQLPRPageSize, graphQLPageSize(41, githubGraphQLPRPageSize)) +} + +func TestDoGraphQLPageShrink(t *testing.T) { + defer test.MockVariableValue(&graphQLTransientRetryBase, time.Duration(0))() + for _, failure := range []string{"502", "RESOURCE_LIMITS_EXCEEDED"} { + t.Run(failure, func(t *testing.T) { + sizes := make(chan int, 16) + downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, request *http.Request) { + var payload struct { + Variables struct { + First int `json:"first"` + Cursor string `json:"cursor"` + } `json:"variables"` + } + assert.NoError(t, json.NewDecoder(request.Body).Decode(&payload)) + sizes <- payload.Variables.First + assert.Equal(t, "CURSOR", payload.Variables.Cursor) + if payload.Variables.First > 25 { + if failure == "502" { + writer.WriteHeader(http.StatusBadGateway) + } else { + _, _ = io.WriteString(writer, `{"errors":[{"type":"RESOURCE_LIMITS_EXCEEDED"}]}`) + } + return + } + _, _ = io.WriteString(writer, `{"data":{}}`) + }) + require.NoError(t, downloader.doGraphQLPage(t.Context(), downloader.graphQLIssuesQuery(), "CURSOR", 100, &gqlIssuesResponse{})) + close(sizes) + var got, want []int + for size := range sizes { + got = append(got, size) + } + for _, size := range []int{100, 50, 25} { + repeats := 1 + if failure == "502" && size > 25 { + repeats += maxGraphQLTransientRetries + } + for range repeats { + want = append(want, size) + } + } + assert.Equal(t, want, got) + }) + } +} + +func TestFetchNodeReactionsBisectsAndSkipsDeletedNodes(t *testing.T) { + defer test.MockVariableValue(&graphQLTransientRetryBase, time.Duration(0))() + for _, failure := range []string{"502", "RESOURCE_LIMITS_EXCEEDED"} { + t.Run(failure, func(t *testing.T) { + var calls atomic.Int32 + downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, request *http.Request) { + calls.Add(1) + var payload struct { + Variables struct { + IDs []string `json:"ids"` + } `json:"variables"` + } + assert.NoError(t, json.NewDecoder(request.Body).Decode(&payload)) + if len(payload.Variables.IDs) > 1 { + if failure == "502" { + writer.WriteHeader(http.StatusBadGateway) + } else { + _, _ = io.WriteString(writer, `{"errors":[{"type":"RESOURCE_LIMITS_EXCEEDED"}]}`) + } + return + } + if payload.Variables.IDs[0] == "deleted" { + _, _ = io.WriteString(writer, `{"data":{"nodes":[null]},"errors":[{"type":"NOT_FOUND","path":["nodes",0]}]}`) + } else { + _, _ = io.WriteString(writer, `{"data":{"nodes":[{"id":"present","reactions":{"nodes":[{"content":"HEART","user":null}]}}]}}`) + } + }) + got, err := downloader.fetchNodeReactions(t.Context(), []string{"deleted", "present"}) + require.NoError(t, err) + assert.Equal(t, map[string][]*base.Reaction{"present": {{UserName: "ghost", Content: "heart"}}}, got) + if failure == "502" { + assert.EqualValues(t, maxGraphQLTransientRetries+3, calls.Load()) + } else { + assert.EqualValues(t, 3, calls.Load()) + } + }) + } +} + +func TestLabelsSweepContinuesFromEndCursor(t *testing.T) { + downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, request *http.Request) { + var payload graphQLRequest + assert.NoError(t, json.NewDecoder(request.Body).Decode(&payload)) + assert.Equal(t, "I_node", payload.Variables["id"]) + assert.Equal(t, "LABEL1", payload.Variables["cursor"]) + _, _ = io.WriteString(writer, `{"data":{"node":{"conn":{"nodes":[{"name":"second"}]}}}}`) + }) + labels, err := allNodes(t.Context(), downloader, graphQLLabelsSweepQuery, "I_node", gqlConn[gqlLabel]{ + PageInfo: gqlPageInfo{HasNextPage: true, EndCursor: "LABEL1"}, + Nodes: []gqlLabel{{Name: "first"}}, + }) + require.NoError(t, err) + assert.Equal(t, []gqlLabel{{Name: "first"}, {Name: "second"}}, labels) +} + +func TestGraphQLIssuesResponseParsing(t *testing.T) { + var response gqlIssuesResponse + require.NoError(t, json.Unmarshal([]byte(`{"repository":{"issues":{"nodes":[{ + "id":"I_42","number":42,"title":"a bug","body":"it broke","state":"CLOSED","locked":true, + "createdAt":"2024-01-02T03:04:05Z","updatedAt":"2024-02-03T04:05:06Z","closedAt":"2024-02-03T04:05:06Z", + "author":{"__typename":"Bot","login":"reporter","databaseId":11},"milestone":{"title":"v1.0"}, + "labels":{"nodes":[{"name":"bug","color":"d73a4a","description":"defect"}]}, + "assignees":{"nodes":[{"login":"maintainer"}]}, + "reactions":{"nodes":[{"content":"THUMBS_UP","user":{"login":"alice","databaseId":22}}]}, + "comments":{"nodes":[{"id":"IC_1","databaseId":900,"body":"same here","author":null}]} + }]}}}`), &response)) + downloader := &GithubGraphQLDownloader{GithubDownloaderV3: &GithubDownloaderV3{}} + issue, err := downloader.convertGraphQLIssue(t.Context(), &response.Repository.Issues.Nodes[0], map[string]*gqlPrefetched{}) + require.NoError(t, err) + updated := time.Date(2024, 2, 3, 4, 5, 6, 0, time.UTC) + assert.Equal(t, &base.Issue{ + Number: 42, Title: "a bug", Content: "it broke", State: "closed", IsLocked: true, + PosterName: "reporter[bot]", PosterID: 11, Milestone: "v1.0", ForeignIndex: 42, + Created: time.Date(2024, 1, 2, 3, 4, 5, 0, time.UTC), Updated: updated, Closed: &updated, + Labels: []*base.Label{{Name: "bug", Color: "d73a4a", Description: "defect"}}, Assignees: []string{"maintainer"}, + Reactions: []*base.Reaction{{Content: "+1", UserName: "alice", UserID: 22}}, Context: issue.Context, + }, issue) + comments, _, err := downloader.GetComments(t.Context(), issue) + require.NoError(t, err) + assert.Equal(t, []*base.Comment{{IssueIndex: 42, Index: 900, Content: "same here", PosterName: "ghost"}}, comments) + assert.False(t, downloader.SupportGetRepoComments()) +} + +func TestGraphQLKeepsCursorOnPageError(t *testing.T) { + for _, connection := range []string{"issues", "pullRequests"} { + for _, phase := range []string{"timelineItems", "Reactable"} { + t.Run(connection+"/"+phase, func(t *testing.T) { + var failPage atomic.Bool + failPage.Store(true) + downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, request *http.Request) { + var payload graphQLRequest + assert.NoError(t, json.NewDecoder(request.Body).Decode(&payload)) + if strings.Contains(payload.Query, "repository(owner:") { + assert.Equal(t, "PAGE1", payload.Variables["cursor"]) + _, _ = fmt.Fprintf(writer, `{"data":{"repository":{"%s":{"pageInfo":{"endCursor":"PAGE2"},"nodes":[{"id":"I_1","number":1,"comments":{"nodes":[{"id":"IC_1"}]}}]}}}}`, connection) + } else if failPage.Load() && strings.Contains(payload.Query, phase) { + writer.WriteHeader(http.StatusBadRequest) + } else { + _, _ = io.WriteString(writer, `{"data":{"nodes":[]}}`) + } + }) + cursor := &downloader.issuesCursor + if connection == "pullRequests" { + cursor = &downloader.prCursor + } + *cursor = "PAGE1" + fetch := func() error { + if connection == "issues" { + _, _, err := downloader.getIssuesGraphQL(t.Context(), 2, 10) + return err + } else { + _, _, err := downloader.getPullRequestsGraphQL(t.Context(), 2, 10) + return err + } + } + require.Error(t, fetch()) + assert.Equal(t, "PAGE1", *cursor) + failPage.Store(false) + require.NoError(t, fetch()) + assert.Equal(t, "PAGE2", *cursor) + }) + } + } +} diff --git a/services/migrations/github_graphql_timeline.go b/services/migrations/github_graphql_timeline.go new file mode 100644 index 00000000000..063549a5e2e --- /dev/null +++ b/services/migrations/github_graphql_timeline.go @@ -0,0 +1,213 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package migrations + +import ( + "context" + "maps" + "slices" + "strings" + "time" + + issues_model "gitea.dev/models/issues" + "gitea.dev/modules/log" + base "gitea.dev/modules/migration" +) + +// PR-only types in the Issue fragment fail validation of the whole query, assignee events would need local user ids +const ( + issueTimelineItemTypes = "CLOSED_EVENT,REOPENED_EVENT,LOCKED_EVENT,UNLOCKED_EVENT,RENAMED_TITLE_EVENT,LABELED_EVENT,UNLABELED_EVENT,MILESTONED_EVENT,DEMILESTONED_EVENT,PINNED_EVENT,UNPINNED_EVENT" + prTimelineItemTypes = issueTimelineItemTypes + ",MERGED_EVENT,HEAD_REF_DELETED_EVENT,AUTO_MERGE_ENABLED_EVENT,AUTO_MERGE_DISABLED_EVENT" +) + +const issueTimelineFragments = ` + ... on ClosedEvent{id createdAt actor{` + gqlActorFields + `}} + ... on ReopenedEvent{id createdAt actor{` + gqlActorFields + `}} + ... on LockedEvent{id createdAt lockReason actor{` + gqlActorFields + `}} + ... on UnlockedEvent{id createdAt actor{` + gqlActorFields + `}} + ... on RenamedTitleEvent{id createdAt previousTitle currentTitle actor{` + gqlActorFields + `}} + ... on LabeledEvent{id createdAt label{name} actor{` + gqlActorFields + `}} + ... on UnlabeledEvent{id createdAt label{name} actor{` + gqlActorFields + `}} + ... on MilestonedEvent{id createdAt milestoneTitle actor{` + gqlActorFields + `}} + ... on DemilestonedEvent{id createdAt milestoneTitle actor{` + gqlActorFields + `}} + ... on PinnedEvent{id createdAt actor{` + gqlActorFields + `}} + ... on UnpinnedEvent{id createdAt actor{` + gqlActorFields + `}}` + +const prTimelineFragments = issueTimelineFragments + ` + ... on MergedEvent{id createdAt actor{` + gqlActorFields + `}} + ... on HeadRefDeletedEvent{id createdAt headRefName actor{` + gqlActorFields + `}} + ... on AutoMergeEnabledEvent{id createdAt actor{` + gqlActorFields + `}} + ... on AutoMergeDisabledEvent{id createdAt actor{` + gqlActorFields + `}}` + +const ( + issueTimelineConn = `{ + pageInfo{hasNextPage endCursor} + nodes{ + __typename` + issueTimelineFragments + ` + } + }` + prTimelineConn = `{ + pageInfo{hasNextPage endCursor} + nodes{ + __typename` + prTimelineFragments + ` + } + }` +) + +const graphQLTimelineBatchQuery = ` +query($ids:[ID!]!){ + nodes(ids:$ids){ + id + ... on Issue{timelineItems(first:100,itemTypes:[` + issueTimelineItemTypes + `])` + issueTimelineConn + `} + ... on PullRequest{timelineItems(first:100,itemTypes:[` + prTimelineItemTypes + `])` + prTimelineConn + `} + } +}` + +const graphQLTimelineSweepQuery = ` +query($id:ID!,$cursor:String){ + node(id:$id){ + ... on Issue{conn:timelineItems(first:100,after:$cursor,itemTypes:[` + issueTimelineItemTypes + `])` + issueTimelineConn + `} + ... on PullRequest{conn:timelineItems(first:100,after:$cursor,itemTypes:[` + prTimelineItemTypes + `])` + prTimelineConn + `} + } +}` + +type gqlTimelineItem struct { + Typename string `json:"__typename"` + ID string `json:"id"` + CreatedAt time.Time `json:"createdAt"` + Actor gqlActor `json:"actor"` + PreviousTitle string `json:"previousTitle"` + CurrentTitle string `json:"currentTitle"` + Label *struct { + Name string `json:"name"` + } `json:"label"` + MilestoneTitle string `json:"milestoneTitle"` + HeadRefName string `json:"headRefName"` + LockReason string `json:"lockReason"` +} + +func convertTimelineItem(it *gqlTimelineItem) *base.Comment { + comment := &base.Comment{ + PosterID: it.Actor.DatabaseID, + PosterName: it.Actor.Login, + Created: it.CreatedAt, + Updated: it.CreatedAt, + } + switch it.Typename { + case "ClosedEvent": + comment.CommentType = issues_model.CommentTypeClose.String() + case "ReopenedEvent": + comment.CommentType = issues_model.CommentTypeReopen.String() + case "LockedEvent": + comment.CommentType = issues_model.CommentTypeLock.String() + comment.Content = strings.ToLower(strings.ReplaceAll(it.LockReason, "_", " ")) + case "UnlockedEvent": + comment.CommentType = issues_model.CommentTypeUnlock.String() + case "RenamedTitleEvent": + comment.CommentType = issues_model.CommentTypeChangeTitle.String() + comment.Meta = map[string]any{"OldTitle": it.PreviousTitle, "NewTitle": it.CurrentTitle} + case "LabeledEvent", "UnlabeledEvent": + if it.Label == nil { + return nil + } + comment.CommentType = issues_model.CommentTypeLabel.String() + comment.Meta = map[string]any{"LabelName": it.Label.Name} + if it.Typename == "LabeledEvent" { + comment.Content = "1" + } + case "MilestonedEvent": + comment.CommentType = issues_model.CommentTypeMilestone.String() + comment.Meta = map[string]any{"MilestoneTitle": it.MilestoneTitle} + case "DemilestonedEvent": + comment.CommentType = issues_model.CommentTypeMilestone.String() + comment.Meta = map[string]any{"MilestoneTitle": it.MilestoneTitle, "Removed": true} + case "MergedEvent": + comment.CommentType = issues_model.CommentTypeMergePull.String() + case "HeadRefDeletedEvent": + comment.CommentType = issues_model.CommentTypeDeleteBranch.String() + comment.Content = it.HeadRefName + case "PinnedEvent": + comment.CommentType = issues_model.CommentTypePin.String() + case "UnpinnedEvent": + comment.CommentType = issues_model.CommentTypeUnpin.String() + case "AutoMergeEnabledEvent": + comment.CommentType = issues_model.CommentTypePRScheduledToAutoMerge.String() + case "AutoMergeDisabledEvent": + comment.CommentType = issues_model.CommentTypePRUnScheduledToAutoMerge.String() + default: + log.Warn("github graphql: dropping unhandled timeline event type %q (node %s)", it.Typename, it.ID) + return nil + } + return comment +} + +func convertTimelineItems(items []gqlTimelineItem) []*base.Comment { + comments := make([]*base.Comment, 0, len(items)) + for i := range items { + if comment := convertTimelineItem(&items[i]); comment != nil { + comments = append(comments, comment) + } + } + return comments +} + +func (g *GithubGraphQLDownloader) attachTimelineEvents(ctx context.Context, prefetchedByNodeID map[string]*gqlPrefetched) error { + for batch := range slices.Chunk(slices.Sorted(maps.Keys(prefetchedByNodeID)), graphQLNodeBatchSize) { + eventsByID, err := g.fetchTimelineEvents(ctx, batch) + if err != nil { + return err + } + for id, events := range eventsByID { + prefetched := prefetchedByNodeID[id] + for _, event := range events { + event.IssueIndex = prefetched.index + } + prefetched.comments = append(prefetched.comments, events...) + } + } + return nil +} + +func (g *GithubGraphQLDownloader) fetchTimelineEvents(ctx context.Context, ids []string) (map[string][]*base.Comment, error) { + var resp struct { + Nodes []struct { + ID string `json:"id"` + TimelineItems gqlConn[gqlTimelineItem] `json:"timelineItems"` + } `json:"nodes"` + } + if err := g.doGraphQL(ctx, graphQLTimelineBatchQuery, map[string]any{"ids": ids}, &resp); err != nil { + if isGraphQLTooHeavy(err) && len(ids) > 1 { + return fetchHalves(ctx, ids, g.fetchTimelineEvents) + } + return nil, err + } + out := make(map[string][]*base.Comment, len(resp.Nodes)) + for _, node := range resp.Nodes { + if node.ID == "" { + continue + } + items, err := allNodes(ctx, g, graphQLTimelineSweepQuery, node.ID, node.TimelineItems) + if err != nil { + return nil, err + } + out[node.ID] = dropCloseOnMerge(convertTimelineItems(items)) + } + return out, nil +} + +// dropCloseOnMerge drops the close GitHub records with a merge, the last one since a merged PR cannot be reopened +func dropCloseOnMerge(events []*base.Comment) []*base.Comment { + if !slices.ContainsFunc(events, func(event *base.Comment) bool { return event.CommentType == issues_model.CommentTypeMergePull.String() }) { + return events + } + for i, event := range slices.Backward(events) { + switch event.CommentType { + case issues_model.CommentTypeReopen.String(): + return events + case issues_model.CommentTypeClose.String(): + return slices.Delete(events, i, i+1) + } + } + return events +} diff --git a/services/migrations/github_graphql_timeline_test.go b/services/migrations/github_graphql_timeline_test.go new file mode 100644 index 00000000000..884f2a0cf38 --- /dev/null +++ b/services/migrations/github_graphql_timeline_test.go @@ -0,0 +1,80 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package migrations + +import ( + "testing" + "time" + + "gitea.dev/modules/json" + base "gitea.dev/modules/migration" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConvertTimelineItems(t *testing.T) { + for _, testcase := range []struct { + name string + event string + want *base.Comment + }{ + {"close by user", `{"__typename":"ClosedEvent","actor":{"login":"closer","databaseId":42}}`, &base.Comment{CommentType: "close", PosterName: "closer", PosterID: 42}}, + {"reopen", `{"__typename":"ReopenedEvent"}`, &base.Comment{CommentType: "reopen"}}, + {"label added", `{"__typename":"LabeledEvent","label":{"name":"bug"}}`, &base.Comment{CommentType: "label", Content: "1", Meta: map[string]any{"LabelName": "bug"}}}, + {"label removed", `{"__typename":"UnlabeledEvent","label":{"name":"bug"}}`, &base.Comment{CommentType: "label", Meta: map[string]any{"LabelName": "bug"}}}, + {"deleted label added is skipped", `{"__typename":"LabeledEvent","label":null}`, nil}, + {"deleted label removed is skipped", `{"__typename":"UnlabeledEvent","label":null}`, nil}, + {"lock reason in content", `{"__typename":"LockedEvent","lockReason":"TOO_HEATED"}`, &base.Comment{CommentType: "lock", Content: "too heated"}}, + {"unlock", `{"__typename":"UnlockedEvent"}`, &base.Comment{CommentType: "unlock"}}, + {"title changed", `{"__typename":"RenamedTitleEvent","previousTitle":"old","currentTitle":"new"}`, &base.Comment{CommentType: "change_title", Meta: map[string]any{"OldTitle": "old", "NewTitle": "new"}}}, + {"milestone added", `{"__typename":"MilestonedEvent","milestoneTitle":"v1"}`, &base.Comment{CommentType: "milestone", Meta: map[string]any{"MilestoneTitle": "v1"}}}, + {"milestone removed", `{"__typename":"DemilestonedEvent","milestoneTitle":"v1"}`, &base.Comment{CommentType: "milestone", Meta: map[string]any{"MilestoneTitle": "v1", "Removed": true}}}, + {"merge by bot", `{"__typename":"MergedEvent","actor":{"__typename":"Bot","login":"merger","databaseId":88}}`, &base.Comment{CommentType: "merge_pull", PosterName: "merger[bot]", PosterID: 88}}, + {"branch deleted", `{"__typename":"HeadRefDeletedEvent","headRefName":"feature"}`, &base.Comment{CommentType: "delete_branch", Content: "feature"}}, + {"pin", `{"__typename":"PinnedEvent"}`, &base.Comment{CommentType: "pin"}}, + {"null actor becomes ghost", `{"__typename":"UnpinnedEvent","actor":null}`, &base.Comment{CommentType: "unpin", PosterName: "ghost"}}, + {"auto merge enabled", `{"__typename":"AutoMergeEnabledEvent"}`, &base.Comment{CommentType: "pull_scheduled_merge"}}, + {"auto merge disabled", `{"__typename":"AutoMergeDisabledEvent"}`, &base.Comment{CommentType: "pull_cancel_scheduled_merge"}}, + {"unknown event is skipped", `{"__typename":"MentionedEvent"}`, nil}, + } { + t.Run(testcase.name, func(t *testing.T) { + var item gqlTimelineItem + require.NoError(t, json.Unmarshal([]byte(testcase.event), &item)) + item.CreatedAt = time.Date(2020, 11, 17, 15, 54, 33, 0, time.UTC) + want := []*base.Comment{} + if testcase.want != nil { + testcase.want.Created, testcase.want.Updated = item.CreatedAt, item.CreatedAt + want = append(want, testcase.want) + } + assert.Equal(t, want, convertTimelineItems([]gqlTimelineItem{item})) + }) + } +} + +func TestDropCloseOnMerge(t *testing.T) { + for _, testcase := range []struct { + name string + events []string + want []string + }{ + {"unmerged close stays", []string{"close"}, []string{"close"}}, + {"last close after merge drops", []string{"close", "reopen", "merge_pull", "close"}, []string{"close", "reopen", "merge_pull"}}, + {"last close before merge drops", []string{"close", "merge_pull"}, []string{"merge_pull"}}, + {"close followed by reopen stays", []string{"close", "merge_pull", "reopen"}, []string{"close", "merge_pull", "reopen"}}, + {"merge without close stays", []string{"merge_pull"}, []string{"merge_pull"}}, + } { + t.Run(testcase.name, func(t *testing.T) { + var events []*base.Comment + for _, event := range testcase.events { + events = append(events, &base.Comment{CommentType: event}) + } + var got []string + for _, event := range dropCloseOnMerge(events) { + got = append(got, event.CommentType) + } + assert.Equal(t, testcase.want, got) + }) + } +} diff --git a/services/migrations/github_test.go b/services/migrations/github_test.go index 9fba225ab38..ee283b8c788 100644 --- a/services/migrations/github_test.go +++ b/services/migrations/github_test.go @@ -12,6 +12,8 @@ import ( "path/filepath" "runtime" "slices" + "strconv" + "strings" "testing" "time" @@ -48,7 +50,7 @@ func TestGithubIssuePagination(t *testing.T) { w.Header().Set("X-RateLimit-Remaining", "100") if r.URL.Path == "/api/v3/repos/o/r/issues/4/reactions" && failReaction { failReaction = false - w.WriteHeader(http.StatusInternalServerError) + w.WriteHeader(http.StatusBadRequest) _, _ = w.Write([]byte(`{"message":"temporary failure"}`)) return } @@ -595,3 +597,118 @@ func TestGithubMultiTokenClientSelection(t *testing.T) { downloader.waitAndPickClient(t.Context()) assert.Equal(t, 1, downloader.curClientIdx) } + +func TestGitHubIssuesPageBasedLinkFallback(t *testing.T) { + _, callerFile, _, _ := runtime.Caller(0) + mockServer := unittest.NewMockWebServer(t, "https://api.github.com", filepath.Join(filepath.Dir(callerFile), "_mock_data/TestGitHubDownloadRepo"), false, unittest.MockServerOptions{ + StripPrefix: "/api/v3", + }) + downloader, err := NewGithubDownloaderV3(t.Context(), mockServer.URL, "", "", "", "go-gitea", "test_repo") + require.NoError(t, err) + downloader.SkipReactions = true + for page, want := range [][]int64{{1, 2, 3}, {}} { + issues, isEnd, err := downloader.GetIssues(t.Context(), page+1, 3) + require.NoError(t, err) + numbers := make([]int64, 0, len(issues)) + for _, issue := range issues { + numbers = append(numbers, issue.Number) + } + assert.Equal(t, want, numbers) + assert.Equal(t, page == 1, isEnd) + } +} + +func TestGithubFactoryUsesGraphQLOnlyWhereItWorks(t *testing.T) { + for _, testcase := range []struct { + name string + opts base.MigrateOptions + want bool + }{ + {name: "token, issues", opts: base.MigrateOptions{AuthToken: "tok", Issues: true}, want: true}, + {name: "basic auth, pull requests", opts: base.MigrateOptions{AuthUsername: "u", AuthPassword: "pat", PullRequests: true}, want: true}, + {name: "anonymous repository", opts: base.MigrateOptions{Issues: true, PullRequests: true}}, + {name: "comments only", opts: base.MigrateOptions{AuthToken: "tok", Comments: true}}, + {name: "issues with comments", opts: base.MigrateOptions{AuthToken: "tok", Issues: true, Comments: true}, want: true}, + } { + t.Run(testcase.name, func(t *testing.T) { + testcase.opts.CloneAddr = "https://github.com/go-gitea/test_repo" + candidate, err := (&GithubDownloaderV3Factory{}).New(t.Context(), testcase.opts) + require.NoError(t, err) + downloader, isGraphQL := candidate.(*GithubGraphQLDownloader) + assert.Equal(t, testcase.want, isGraphQL) + if isGraphQL { + assert.Equal(t, !testcase.opts.Comments, downloader.skipComments) + } + }) + } +} + +func TestGithubGraphQLFallsBackToREST(t *testing.T) { + for _, connection := range []string{"issues", "pull requests"} { + t.Run(connection, func(t *testing.T) { + var restPage string + downloader := newGithubTestDownloader(t, func(writer http.ResponseWriter, request *http.Request) { + if request.URL.Path == "/api/graphql" { + http.NotFound(writer, request) + return + } + restPage = request.URL.Query().Get("page") + _, _ = writer.Write([]byte(`[{"number":1,"title":"t","state":"open"}]`)) + }) + downloader.SkipReactions = true + if connection == "issues" { + issues, _, err := downloader.GetIssues(t.Context(), 3, 10) + require.NoError(t, err) + require.Len(t, issues, 1) + assert.EqualValues(t, 1, issues[0].Number) + } else { + prs, _, err := downloader.GetPullRequests(t.Context(), 3, 10) + require.NoError(t, err) + require.Len(t, prs, 1) + assert.EqualValues(t, 1, prs[0].Number) + } + assert.True(t, downloader.restFallback) + assert.Equal(t, "1", restPage) + }) + } +} + +func TestGitBucketPaginatesWithoutLinkHeader(t *testing.T) { + const total = 26 + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + page, _ := strconv.Atoi(request.URL.Query().Get("page")) + var items []string + for number := (page-1)*25 + 1; number <= min(page*25, total); number++ { + items = append(items, fmt.Sprintf(`{"number":%d,"title":"t"}`, number)) + } + writer.Header().Set("X-RateLimit-Remaining", "4000") + _, _ = writer.Write([]byte("[" + strings.Join(items, ",") + "]")) + })) + t.Cleanup(server.Close) + downloader, err := NewGitBucketDownloader(t.Context(), server.URL, "", "", "", "o", "r") + require.NoError(t, err) + for _, connection := range []string{"issues", "pull requests"} { + t.Run(connection, func(t *testing.T) { + var count int + for page := 1; ; page++ { + require.LessOrEqual(t, page, 3) + var isEnd bool + if connection == "issues" { + issues, done, err := downloader.GetIssues(t.Context(), page, 10) + require.NoError(t, err) + count += len(issues) + isEnd = done + } else { + prs, done, err := downloader.GetPullRequests(t.Context(), page, 10) + require.NoError(t, err) + count += len(prs) + isEnd = done + } + if isEnd { + break + } + } + assert.Equal(t, total, count) + }) + } +} diff --git a/templates/repo/issue/view_content/comments.tmpl b/templates/repo/issue/view_content/comments.tmpl index 0d184e07135..4a77088e706 100644 --- a/templates/repo/issue/view_content/comments.tmpl +++ b/templates/repo/issue/view_content/comments.tmpl @@ -80,9 +80,7 @@ {{else if eq .Type 1}}
{{svg "octicon-dot-fill"}} - {{if not .OriginalAuthor}} - {{template "shared/user/avatarlink" dict "user" .Poster}} - {{end}} + {{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}} {{template "repo/issue/view_content/comments_authorlink" dict "comment" .}} {{if .Issue.IsPull}} @@ -95,9 +93,7 @@ {{else if eq .Type 2}}
{{svg "octicon-issue-closed"}} - {{if not .OriginalAuthor}} - {{template "shared/user/avatarlink" dict "user" .Poster}} - {{end}} + {{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}} {{template "repo/issue/view_content/comments_authorlink" dict "comment" .}} {{if .Issue.IsPull}} @@ -110,9 +106,7 @@ {{else if eq .Type 28}}
{{svg "octicon-git-merge"}} - {{if not .OriginalAuthor}} - {{template "shared/user/avatarlink" dict "user" .Poster}} - {{end}} + {{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}} {{template "repo/issue/view_content/comments_authorlink" dict "comment" .}} {{$link := printf "%s/commit/%s" $.Repository.Link ($.Issue.PullRequest.MergedCommitID|PathEscape)}} @@ -169,7 +163,7 @@ {{if or .AddedLabels .RemovedLabels}}
{{svg "octicon-tag"}} - {{template "shared/user/avatarlink" dict "user" .Poster}} + {{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}} {{template "repo/issue/view_content/comments_authorlink" dict "comment" .}} {{if and .AddedLabels (not .RemovedLabels)}} @@ -185,9 +179,9 @@ {{else if eq .Type 8}}
{{svg "octicon-milestone"}} - {{template "shared/user/avatarlink" dict "user" .Poster}} + {{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}} - {{template "shared/user/authorlink" .Poster}} + {{template "repo/issue/view_content/comments_authorlink" dict "comment" .}} {{if gt .OldMilestoneID 0}}{{if gt .MilestoneID 0}}{{ctx.Locale.Tr "repo.issues.change_milestone_at" .OldMilestone.Name .Milestone.Name $createdStr}}{{else}}{{ctx.Locale.Tr "repo.issues.remove_milestone_at" .OldMilestone.Name $createdStr}}{{end}}{{else if gt .MilestoneID 0}}{{ctx.Locale.Tr "repo.issues.add_milestone_at" .Milestone.Name $createdStr}}{{end}}
@@ -219,18 +213,18 @@ {{else if eq .Type 10}}
{{ctx.RenderUtils.RenderTimelineEventBadge $comment}} - {{template "shared/user/avatarlink" dict "user" .Poster}} + {{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}} - {{template "shared/user/authorlink" .Poster}} + {{template "repo/issue/view_content/comments_authorlink" dict "comment" .}} {{ctx.RenderUtils.RenderTimelineEventComment $comment $createdStr}}
{{else if eq .Type 11}}
{{svg "octicon-git-branch"}} - {{template "shared/user/avatarlink" dict "user" .Poster}} + {{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}} - {{template "shared/user/authorlink" .Poster}} + {{template "repo/issue/view_content/comments_authorlink" dict "comment" .}} {{$oldRef := HTMLFormat `%s` .OldRef}} {{ctx.Locale.Tr "repo.issues.delete_branch_at" $oldRef $createdStr}} @@ -461,15 +455,15 @@ {{else if eq .Type 23}}
{{svg "octicon-lock"}} - {{template "shared/user/avatarlink" dict "user" .Poster}} + {{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}} {{if .Content}} - {{template "shared/user/authorlink" .Poster}} + {{template "repo/issue/view_content/comments_authorlink" dict "comment" .}} {{ctx.Locale.Tr "repo.issues.lock_with_reason" .Content $createdStr}} {{else}} - {{template "shared/user/authorlink" .Poster}} + {{template "repo/issue/view_content/comments_authorlink" dict "comment" .}} {{ctx.Locale.Tr "repo.issues.lock_no_reason" $createdStr}} {{end}} @@ -477,18 +471,16 @@ {{else if eq .Type 24}}
{{svg "octicon-key"}} - {{template "shared/user/avatarlink" dict "user" .Poster}} + {{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}} - {{template "shared/user/authorlink" .Poster}} + {{template "repo/issue/view_content/comments_authorlink" dict "comment" .}} {{ctx.Locale.Tr "repo.issues.unlock_comment" $createdStr}}
{{else if eq .Type 25}}
{{svg "octicon-git-branch"}} - {{if not .OriginalAuthor}} - {{template "shared/user/avatarlink" dict "user" .Poster}} - {{end}} + {{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}} {{template "repo/issue/view_content/comments_authorlink" dict "comment" .}} {{ctx.Locale.Tr "repo.pulls.change_target_branch_at" .OldRef .NewRef $createdStr}} @@ -673,9 +665,9 @@ {{else if or (eq .Type 36) (eq .Type 37)}}
{{svg "octicon-pin" 16}} - {{template "shared/user/avatarlink" dict "user" .Poster}} + {{template "repo/issue/view_content/comments_avatarlink" dict "comment" .}} - {{template "shared/user/authorlink" .Poster}} + {{template "repo/issue/view_content/comments_authorlink" dict "comment" .}} {{if eq .Type 36}}{{ctx.Locale.Tr "repo.issues.pin_comment" $createdStr}} {{else}}{{ctx.Locale.Tr "repo.issues.unpin_comment" $createdStr}}{{end}} diff --git a/templates/repo/issue/view_content/comments_avatarlink.tmpl b/templates/repo/issue/view_content/comments_avatarlink.tmpl new file mode 100644 index 00000000000..68429520711 --- /dev/null +++ b/templates/repo/issue/view_content/comments_avatarlink.tmpl @@ -0,0 +1 @@ +{{if not .comment.OriginalAuthor}}{{template "shared/user/avatarlink" dict "user" .comment.Poster}}{{end}}