feat: Improve Gitea-compatible forge migrations, fix many migration bugs (#39424)

Supports migrating from Gitea-compatible forges like Forgejo and fixes
many migration bugs.

1. Pre-release and `+gitea-` suffixed server versions no longer drop
reviews, reactions and pagination.
2. Review states, dismissals, scoped and archived labels and merge bases
are migrated.
3. Disabled units, external trackers, unavailable release assets, and
GitLab confidential issues and internal notes are skipped.
4. Credentials are only sent to the source host, and `Retry-After` rate
limits are waited out.
5. GitHub migrations of only issues or pull requests, basic auth clones
and dismissed reviews work again.
6. The API returns 404 for releases of a disabled unit and shows
migrated authors as mannequins, replacing the never populated
`original_author` and `original_author_id` fields.

Fixes: https://github.com/go-gitea/gitea/issues/19533
Fixes: https://github.com/go-gitea/gitea/issues/35885
Fixes: https://github.com/go-gitea/gitea/issues/18275
Fixes: https://github.com/go-gitea/gitea/issues/35069
Fixes: https://github.com/go-gitea/gitea/issues/17763
Fixes: https://github.com/go-gitea/gitea/issues/39548
Co-authored-by: bircni <bircni@icloud.com>
This commit is contained in:
silverwindandbircni authored and GitHub committed 2026-10-24 01:15:31 +00:00
1 parent 7c91252be3
commit fc49368845
94 files changed
+912 -417

No files matched your search

+1 -1
View File
@@ -50,7 +50,7 @@ func toIssue(ctx context.Context, doer *user_model.User, issue *issues_model.Iss
apiIssue := &api.Issue{
ID: issue.ID,
Index: issue.Index,
Poster: ToUser(ctx, issue.Poster, doer),
Poster: ToAuthor(ctx, issue.Poster, issue.OriginalAuthor, doer),
Title: issue.Title,
Body: issue.Content,
Attachments: toAttachments(ctx, issue.Repo, issue.Attachments, getDownloadURL),
+2 -2
View File
@@ -18,7 +18,7 @@ import (
func ToAPIComment(ctx context.Context, repo *repo_model.Repository, c *issues_model.Comment) *api.Comment {
return &api.Comment{
ID: c.ID,
Poster: ToUser(ctx, c.Poster, nil),
Poster: ToAuthor(ctx, c.Poster, c.OriginalAuthor, nil),
HTMLURL: c.HTMLURL(ctx),
IssueURL: c.IssueURL(ctx),
PRURL: c.PRURL(ctx),
@@ -86,7 +86,7 @@ func ToTimelineComment(ctx context.Context, repo *repo_model.Repository, c *issu
comment := &api.TimelineComment{
ID: c.ID,
Type: c.Type.String(),
Poster: ToUser(ctx, c.Poster, nil),
Poster: ToAuthor(ctx, c.Poster, c.OriginalAuthor, nil),
HTMLURL: c.HTMLURL(ctx),
IssueURL: c.IssueURL(ctx),
PRURL: c.PRURL(ctx),
+2 -2
View File
@@ -23,7 +23,7 @@ func ToPullReview(ctx context.Context, r *issues_model.Review, doer *user_model.
result := &api.PullReview{
ID: r.ID,
Reviewer: ToUser(ctx, r.Reviewer, doer),
Reviewer: ToAuthor(ctx, r.Reviewer, r.OriginalAuthor, doer),
State: api.ReviewStateUnknown,
Body: r.Content,
CommitID: r.CommitID,
@@ -104,7 +104,7 @@ func ToPullReviewComment(ctx context.Context, comment *issues_model.Comment, doe
apiComment := &api.PullReviewComment{
ID: comment.ID,
Body: comment.Content,
Poster: ToUser(ctx, comment.Poster, doer),
Poster: ToAuthor(ctx, comment.Poster, comment.OriginalAuthor, doer),
Resolver: ToUser(ctx, comment.ResolveDoer, doer),
ReviewID: comment.ReviewID,
Created: comment.CreatedUnix.AsTime(),
+1 -1
View File
@@ -28,7 +28,7 @@ func ToAPIRelease(ctx context.Context, repo *repo_model.Repository, r *repo_mode
IsPrerelease: r.IsPrerelease,
CreatedAt: r.CreatedUnix.AsTime(),
PublishedAt: util.Iif(r.IsDraft, nil, r.PublishedUnix.AsTimePtr()),
Publisher: ToUser(ctx, r.Publisher, nil),
Publisher: ToAuthor(ctx, r.Publisher, r.OriginalAuthor, nil),
Attachments: ToAPIAttachments(ctx, repo, r.Attachments),
}
}
+7
View File
@@ -50,6 +50,13 @@ func ToUser(ctx context.Context, user, doer *user_model.User) *api.User {
return toUser(ctx, user, signed, authed)
}
func ToAuthor(ctx context.Context, author *user_model.User, originalAuthor string, doer *user_model.User) *api.User {
if originalAuthor == "" {
return ToUser(ctx, author, doer)
}
return &api.User{ID: user_model.GhostUserID, UserName: originalAuthor, Type: api.UserTypeStringUser, Visibility: api.VisibilityStringPublic, AvatarURL: user_model.NewGhostUser().AvatarLink(ctx)}
}
// ToUsers convert list of user_model.User to list of api.User
func ToUsers(ctx context.Context, doer *user_model.User, users []*user_model.User) []*api.User {
result := make([]*api.User, len(users))
+2
View File
@@ -42,6 +42,8 @@ func ToGitServiceType(value string) structs.GitServiceType {
return structs.CodeCommitService
case "bitbucketserver":
return structs.BitbucketServerService
case "forgejo":
return structs.ForgejoService
default:
return structs.PlainGitService
}
+2
View File
@@ -40,6 +40,8 @@ func TestToGitServiceType(t *testing.T) {
typ: "codecommit", enum: 9,
}, {
typ: "bitbucketserver", enum: 10,
}, {
typ: "forgejo", enum: 11,
}}
for _, test := range tc {
assert.EqualValues(t, test.enum, ToGitServiceType(test.typ))
@@ -0,0 +1,6 @@
Cache-Control: max-age=0, private, must-revalidate
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
{"id":1,"owner":{"id":1,"login":"alice","login_name":"","source_id":0,"full_name":"","email":"alice@example.com","avatar_url":"https://forgejo.example/avatars/c160f8cc69a4f0bf2b0362752353d060","html_url":"https://forgejo.example/alice","language":"","is_admin":false,"last_login":"0001-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":false,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"alice"},"name":"test_repo","full_name":"alice/test_repo","description":"Test repository for testing migration from forgejo to gitea","empty":false,"private":true,"fork":false,"template":false,"parent":null,"mirror":false,"size":3,"language":"","languages_url":"https://forgejo.example/api/v1/repos/alice/test_repo/languages","html_url":"https://forgejo.example/alice/test_repo","url":"https://forgejo.example/api/v1/repos/alice/test_repo","link":"","ssh_url":"ssh://git@forgejo.example/alice/test_repo.git","clone_url":"https://forgejo.example/alice/test_repo.git","original_url":"","website":"https://forgejo.example/website","stars_count":0,"forks_count":0,"watchers_count":1,"open_issues_count":1,"open_pr_counter":2,"release_counter":2,"default_branch":"main","archived":false,"created_at":"2026-09-25T01:13:52Z","updated_at":"2026-09-25T01:14:07Z","archived_at":"1970-01-01T00:00:00Z","permissions":{"admin":true,"push":true,"pull":true},"has_issues":true,"internal_tracker":{"enable_time_tracker":true,"allow_only_contributors_to_track_time":true,"enable_issue_dependencies":true},"has_wiki":true,"has_wiki_contents":false,"wiki_branch":"main","wiki_ssh_url":"ssh://git@forgejo.example/alice/test_repo.wiki.git","wiki_clone_url":"https://forgejo.example/alice/test_repo.wiki.git","globally_editable_wiki":false,"has_pull_requests":true,"has_projects":true,"has_releases":true,"has_packages":true,"has_actions":true,"ignore_whitespace_conflicts":false,"allow_merge_commits":true,"allow_rebase":true,"allow_rebase_explicit":true,"allow_squash_merge":true,"allow_fast_forward_only_merge":true,"allow_rebase_update":true,"default_delete_branch_after_merge":false,"default_merge_style":"merge","default_allow_maintainer_edit":false,"default_update_style":"merge","avatar_url":"","internal":false,"mirror_interval":"","object_format_name":"sha1","mirror_updated":"0001-01-01T00:00:00Z","repo_transfer":null,"topics":["forgejo","migration"]}
@@ -0,0 +1,9 @@
Access-Control-Expose-Headers: X-Total-Count
Cache-Control: max-age=0, private, must-revalidate
Content-Length: 872
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
X-Total-Count: 1
[{"id":3,"html_url":"https://forgejo.example/alice/test_repo/issues/1#issuecomment-3","pull_request_url":"","issue_url":"https://forgejo.example/alice/test_repo/issues/1","user":{"id":2,"login":"bob","login_name":"","source_id":0,"full_name":"","email":"bob@noreply.forgejo.example","avatar_url":"https://forgejo.example/avatars/4b9bb80620f03eb3719e0a061c14283d","html_url":"https://forgejo.example/bob","language":"","is_admin":false,"last_login":"0001-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":false,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"bob"},"original_author":"","original_author_id":0,"body":"a comment","assets":[],"created_at":"2026-09-25T01:13:56Z","updated_at":"2026-09-25T01:13:56Z"}]
@@ -0,0 +1,9 @@
Access-Control-Expose-Headers: X-Total-Count
Cache-Control: max-age=0, private, must-revalidate
Content-Length: 1203
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
X-Total-Count: 2
[{"user":{"id":1,"login":"alice","login_name":"","source_id":0,"full_name":"","email":"alice@example.com","avatar_url":"https://forgejo.example/avatars/c160f8cc69a4f0bf2b0362752353d060","html_url":"https://forgejo.example/alice","language":"","is_admin":true,"last_login":"1970-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":true,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"alice"},"content":"heart","created_at":"2026-09-25T01:13:56Z"},{"user":{"id":1,"login":"alice","login_name":"","source_id":0,"full_name":"","email":"alice@example.com","avatar_url":"https://forgejo.example/avatars/c160f8cc69a4f0bf2b0362752353d060","html_url":"https://forgejo.example/alice","language":"","is_admin":true,"last_login":"1970-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":true,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"alice"},"content":"+1","created_at":"2026-09-25T01:13:56Z"}]
@@ -0,0 +1,9 @@
Access-Control-Expose-Headers: X-Total-Count
Cache-Control: max-age=0, private, must-revalidate
Content-Length: 5
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
X-Total-Count: 0
null
@@ -0,0 +1,9 @@
Access-Control-Expose-Headers: X-Total-Count
Cache-Control: max-age=0, private, must-revalidate
Content-Length: 5
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
X-Total-Count: 0
null
@@ -0,0 +1,9 @@
Access-Control-Expose-Headers: X-Total-Count
Cache-Control: max-age=0, private, must-revalidate
Content-Length: 5
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
X-Total-Count: 0
null
@@ -0,0 +1,7 @@
Cache-Control: max-age=0, private, must-revalidate
Content-Length: 604
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
[{"user":{"id":1,"login":"alice","login_name":"","source_id":0,"full_name":"","email":"alice@example.com","avatar_url":"https://forgejo.example/avatars/c160f8cc69a4f0bf2b0362752353d060","html_url":"https://forgejo.example/alice","language":"","is_admin":true,"last_login":"1970-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":true,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"alice"},"content":"laugh","created_at":"2026-09-25T01:13:56Z"}]
@@ -0,0 +1,9 @@
Access-Control-Expose-Headers: X-Total-Count
Cache-Control: max-age=0, private, must-revalidate
Content-Length: 1518
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
X-Total-Count: 1
[{"id":1,"url":"https://forgejo.example/api/v1/repos/alice/test_repo/issues/1","html_url":"https://forgejo.example/alice/test_repo/issues/1","number":1,"user":{"id":1,"login":"alice","login_name":"","source_id":0,"full_name":"","email":"alice@example.com","avatar_url":"https://forgejo.example/avatars/c160f8cc69a4f0bf2b0362752353d060","html_url":"https://forgejo.example/alice","language":"","is_admin":true,"last_login":"1970-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":true,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"alice"},"original_author":"","original_author_id":0,"title":"first issue","body":"issue body","ref":"","assets":[],"labels":[{"id":1,"name":"kind/bug","exclusive":true,"is_archived":false,"color":"ee0701","description":"Something is broken","url":"https://forgejo.example/api/v1/repos/alice/test_repo/labels/1"}],"milestone":{"id":1,"title":"v1","description":"first","state":"open","open_issues":1,"closed_issues":0,"created_at":"2026-09-25T01:13:54Z","updated_at":"2026-09-25T01:13:56Z","closed_at":null,"due_on":null},"assignee":null,"assignees":null,"state":"open","is_locked":false,"comments":1,"created_at":"2026-09-25T01:13:56Z","updated_at":"2026-09-25T01:13:56Z","closed_at":null,"due_date":null,"pull_request":null,"repository":{"id":1,"name":"test_repo","owner":"alice","full_name":"alice/test_repo"},"pin_order":0}]
@@ -0,0 +1,9 @@
Access-Control-Expose-Headers: X-Total-Count
Cache-Control: max-age=0, private, must-revalidate
Content-Length: 529
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
X-Total-Count: 3
[{"id":1,"name":"kind/bug","exclusive":true,"is_archived":false,"color":"ee0701","description":"Something is broken","url":"https://forgejo.example/api/v1/repos/alice/test_repo/labels/1"},{"id":2,"name":"kind/feature","exclusive":true,"is_archived":true,"color":"0e8a16","description":"","url":"https://forgejo.example/api/v1/repos/alice/test_repo/labels/2"},{"id":3,"name":"question","exclusive":false,"is_archived":false,"color":"cc317c","description":"","url":"https://forgejo.example/api/v1/repos/alice/test_repo/labels/3"}]
@@ -0,0 +1,9 @@
Access-Control-Expose-Headers: X-Total-Count
Cache-Control: max-age=0, private, must-revalidate
Content-Length: 3
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
X-Total-Count: 3
[]
@@ -0,0 +1,7 @@
Cache-Control: max-age=0, private, must-revalidate
Content-Length: 1023
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
[{"id":10,"body":"fix this","user":{"id":2,"login":"bob","login_name":"","source_id":0,"full_name":"","email":"bob@example.com","avatar_url":"https://forgejo.example/avatars/4b9bb80620f03eb3719e0a061c14283d","html_url":"https://forgejo.example/bob","language":"","is_admin":false,"last_login":"1970-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":true,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"bob"},"resolver":null,"pull_request_review_id":2,"created_at":"2026-09-25T01:14:04Z","updated_at":"2026-09-25T01:14:04Z","path":"open.txt","commit_id":"0310ab0fc7512cab1652c77eadb401e0b15fb974","original_commit_id":"","diff_hunk":"@@ -0,0 +1 @@\n+open.txt","position":1,"original_position":0,"extra_lines_count":0,"html_url":"https://forgejo.example/alice/test_repo/pulls/3#issuecomment-10","pull_request_url":"https://forgejo.example/alice/test_repo/pulls/3"}]
@@ -0,0 +1,9 @@
Access-Control-Expose-Headers: X-Total-Count
Cache-Control: max-age=0, private, must-revalidate
Content-Length: 1894
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
X-Total-Count: 2
[{"id":2,"user":{"id":2,"login":"bob","login_name":"","source_id":0,"full_name":"","email":"bob@example.com","avatar_url":"https://forgejo.example/avatars/4b9bb80620f03eb3719e0a061c14283d","html_url":"https://forgejo.example/bob","language":"","is_admin":false,"last_login":"1970-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":true,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"bob"},"team":null,"state":"REQUEST_CHANGES","body":"needs work","commit_id":"0310ab0fc7512cab1652c77eadb401e0b15fb974","stale":false,"official":true,"dismissed":true,"comments_count":1,"submitted_at":"2026-09-25T01:14:04Z","updated_at":"2026-09-25T01:14:05Z","html_url":"https://forgejo.example/alice/test_repo/pulls/3#issuecomment-11","pull_request_url":"https://forgejo.example/alice/test_repo/pulls/3"},{"id":3,"user":{"id":2,"login":"bob","login_name":"","source_id":0,"full_name":"","email":"bob@example.com","avatar_url":"https://forgejo.example/avatars/4b9bb80620f03eb3719e0a061c14283d","html_url":"https://forgejo.example/bob","language":"","is_admin":false,"last_login":"1970-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":true,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"bob"},"team":null,"state":"COMMENT","body":"just a comment","commit_id":"0310ab0fc7512cab1652c77eadb401e0b15fb974","stale":false,"official":false,"dismissed":false,"comments_count":0,"submitted_at":"2026-09-25T01:14:05Z","updated_at":"2026-09-25T01:14:05Z","html_url":"https://forgejo.example/alice/test_repo/pulls/3#issuecomment-13","pull_request_url":"https://forgejo.example/alice/test_repo/pulls/3"}]
@@ -0,0 +1,8 @@
Access-Control-Expose-Headers: X-Total-Count
Cache-Control: max-age=0, private, must-revalidate
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
X-Total-Count: 3
[{"id":3,"url":"https://forgejo.example/api/v1/repos/alice/test_repo/pulls/4","number":4,"user":{"id":2,"login":"bob","login_name":"","source_id":0,"full_name":"","email":"bob@example.com","avatar_url":"https://forgejo.example/avatars/4b9bb80620f03eb3719e0a061c14283d","html_url":"https://forgejo.example/bob","language":"","is_admin":false,"last_login":"1970-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":true,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"bob"},"title":"agit pull","body":"agit body","labels":[],"milestone":null,"assignee":null,"assignees":null,"requested_reviewers":[],"requested_reviewers_teams":[],"state":"open","draft":false,"is_locked":false,"comments":0,"review_comments":0,"additions":0,"deletions":0,"changed_files":0,"html_url":"https://forgejo.example/alice/test_repo/pulls/4","diff_url":"https://forgejo.example/alice/test_repo/pulls/4.diff","patch_url":"https://forgejo.example/alice/test_repo/pulls/4.patch","mergeable":true,"merged":false,"merged_at":null,"merge_commit_sha":null,"merged_by":null,"allow_maintainer_edit":false,"base":{"label":"main","ref":"main","sha":"588ba39bdea707266f2df950dcee1bb5a1764fed","repo_id":1,"repo":{"id":1,"owner":{"id":1,"login":"alice","login_name":"","source_id":0,"full_name":"","email":"alice@example.com","avatar_url":"https://forgejo.example/avatars/c160f8cc69a4f0bf2b0362752353d060","html_url":"https://forgejo.example/alice","language":"","is_admin":false,"last_login":"0001-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":false,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"alice"},"name":"test_repo","full_name":"alice/test_repo","description":"Test repository for testing migration from forgejo to gitea","empty":false,"private":true,"fork":false,"template":false,"parent":null,"mirror":false,"size":3,"language":"","languages_url":"https://forgejo.example/api/v1/repos/alice/test_repo/languages","html_url":"https://forgejo.example/alice/test_repo","url":"https://forgejo.example/api/v1/repos/alice/test_repo","link":"","ssh_url":"ssh://git@forgejo.example/alice/test_repo.git","clone_url":"https://forgejo.example/alice/test_repo.git","original_url":"","website":"https://forgejo.example/website","stars_count":0,"forks_count":0,"watchers_count":1,"open_issues_count":1,"open_pr_counter":2,"release_counter":2,"default_branch":"main","archived":false,"created_at":"2026-09-25T01:13:52Z","updated_at":"2026-09-25T01:14:07Z","archived_at":"1970-01-01T00:00:00Z","permissions":{"admin":true,"push":true,"pull":true},"has_issues":true,"internal_tracker":{"enable_time_tracker":true,"allow_only_contributors_to_track_time":true,"enable_issue_dependencies":true},"has_wiki":true,"has_wiki_contents":false,"wiki_branch":"main","wiki_ssh_url":"ssh://git@forgejo.example/alice/test_repo.wiki.git","wiki_clone_url":"https://forgejo.example/alice/test_repo.wiki.git","globally_editable_wiki":false,"has_pull_requests":true,"has_projects":true,"has_releases":true,"has_packages":true,"has_actions":true,"ignore_whitespace_conflicts":false,"allow_merge_commits":true,"allow_rebase":true,"allow_rebase_explicit":true,"allow_squash_merge":true,"allow_fast_forward_only_merge":true,"allow_rebase_update":true,"default_delete_branch_after_merge":false,"default_merge_style":"merge","default_allow_maintainer_edit":false,"default_update_style":"merge","avatar_url":"","internal":false,"mirror_interval":"","object_format_name":"sha1","mirror_updated":"0001-01-01T00:00:00Z","repo_transfer":null,"topics":["forgejo","migration"]}},"head":{"label":"bob/agit","ref":"refs/pull/4/head","sha":"171139b0df47db58a099ff2643ac1088776ff13c","repo_id":1,"repo":{"id":1,"owner":{"id":1,"login":"alice","login_name":"","source_id":0,"full_name":"","email":"alice@example.com","avatar_url":"https://forgejo.example/avatars/c160f8cc69a4f0bf2b0362752353d060","html_url":"https://forgejo.example/alice","language":"","is_admin":false,"last_login":"0001-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":false,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"alice"},"name":"test_repo","full_name":"alice/test_repo","description":"Test repository for testing migration from forgejo to gitea","empty":false,"private":true,"fork":false,"template":false,"parent":null,"mirror":false,"size":3,"language":"","languages_url":"https://forgejo.example/api/v1/repos/alice/test_repo/languages","html_url":"https://forgejo.example/alice/test_repo","url":"https://forgejo.example/api/v1/repos/alice/test_repo","link":"","ssh_url":"ssh://git@forgejo.example/alice/test_repo.git","cLine truncated
@@ -0,0 +1,8 @@
Access-Control-Expose-Headers: X-Total-Count
Cache-Control: max-age=0, private, must-revalidate
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
X-Total-Count: 2
[{"id":2,"tag_name":"v0.9.0","target_commitish":"main","name":"v0.9.0","body":"","url":"https://forgejo.example/api/v1/repos/alice/test_repo/releases/2","html_url":"https://forgejo.example/alice/test_repo/releases/tag/v0.9.0","tarball_url":"https://forgejo.example/alice/test_repo/archive/v0.9.0.tar.gz","zipball_url":"https://forgejo.example/alice/test_repo/archive/v0.9.0.zip","hide_archive_links":false,"upload_url":"https://forgejo.example/api/v1/repos/alice/test_repo/releases/2/assets","draft":false,"prerelease":true,"created_at":"2026-09-25T01:13:55Z","published_at":"2026-09-25T01:13:55Z","author":{"id":1,"login":"alice","login_name":"","source_id":0,"full_name":"","email":"alice@noreply.forgejo.example","avatar_url":"https://forgejo.example/avatars/c160f8cc69a4f0bf2b0362752353d060","html_url":"https://forgejo.example/alice","language":"","is_admin":false,"last_login":"0001-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":false,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"alice"},"assets":[],"archive_download_count":{"zip":0,"tar_gz":0}},{"id":1,"tag_name":"v1.0.0","target_commitish":"main","name":"v1.0.0","body":"first release","url":"https://forgejo.example/api/v1/repos/alice/test_repo/releases/1","html_url":"https://forgejo.example/alice/test_repo/releases/tag/v1.0.0","tarball_url":"https://forgejo.example/alice/test_repo/archive/v1.0.0.tar.gz","zipball_url":"https://forgejo.example/alice/test_repo/archive/v1.0.0.zip","hide_archive_links":false,"upload_url":"https://forgejo.example/api/v1/repos/alice/test_repo/releases/1/assets","draft":false,"prerelease":false,"created_at":"2026-09-25T01:13:55Z","published_at":"2026-09-25T01:13:55Z","author":{"id":1,"login":"alice","login_name":"","source_id":0,"full_name":"","email":"alice@noreply.forgejo.example","avatar_url":"https://forgejo.example/avatars/c160f8cc69a4f0bf2b0362752353d060","html_url":"https://forgejo.example/alice","language":"","is_admin":false,"last_login":"0001-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":false,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"alice"},"assets":[{"id":1,"name":"asset.txt","size":14,"download_count":0,"created_at":"2026-09-25T01:13:55Z","uuid":"4e5c9b2d-b858-48af-9ba7-8260f1b53a04","browser_download_url":"https://forgejo.example/alice/test_repo/releases/download/v1.0.0/asset.txt","type":"attachment"},{"id":2,"name":"download.zip","size":0,"download_count":0,"created_at":"2026-09-25T01:13:55Z","uuid":"5e8b7425-31bb-4103-808b-1907aacaddcb","browser_download_url":"https://example.com/download.zip","type":"external"}],"archive_download_count":{"zip":0,"tar_gz":0}}]
@@ -0,0 +1,7 @@
Cache-Control: max-age=0, private, must-revalidate
Content-Length: 115
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
{"max_response_items":3,"default_paging_num":3,"default_git_trees_per_page":1000,"default_max_blob_size":10485760}
@@ -0,0 +1,7 @@
Cache-Control: max-age=0, private, must-revalidate
Content-Length: 53
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
{"version":"17.0.0-dev-576-994e539b55+gitea-1.22.0"}
@@ -0,0 +1,12 @@
Accept-Ranges: bytes
Access-Control-Expose-Headers: Content-Disposition
Cache-Control: private, max-age=300
Content-Disposition: inline; filename="asset.txt"; filename*=UTF-8''asset.txt
Content-Length: 14
Content-Type: text/plain; charset=utf-8
Etag: "4e5c9b2d-b858-48af-9ba7-8260f1b53a04"
Last-Modified: Fri, 25 Sep 2026 01:13:55 GMT
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
forgejo asset
@@ -1,8 +0,0 @@
Alt-Svc: h3=":443"; ma=2592000
Cache-Control: max-age=0, private, must-revalidate, no-transform
Content-Length: 2
Content-Type: application/json;charset=utf-8
Vary: Origin
X-Content-Type-Options: nosniff
[]
@@ -1,8 +0,0 @@
Alt-Svc: h3=":443"; ma=2592000
Cache-Control: max-age=0, private, must-revalidate, no-transform
Content-Length: 2
Content-Type: application/json;charset=utf-8
Vary: Origin
X-Content-Type-Options: nosniff
[]
+1 -1
View File
@@ -113,7 +113,7 @@ type BitbucketServerDownloader struct {
// NewBitbucketServerDownloader creates a Bitbucket Server downloader
func NewBitbucketServerDownloader(_ context.Context, baseURL, username, password, token, projectKey, repoSlug string) *BitbucketServerDownloader {
return &BitbucketServerDownloader{
client: getMigrationHTTPClient(),
client: &http.Client{Transport: getMigrationTransport()}, // not retrying, callAPI retries rate limits itself
sleep: bitbucketServerSleep,
baseURL: strings.TrimSuffix(baseURL, "/"),
projectKey: projectKey,
+2 -10
View File
@@ -78,22 +78,14 @@ type CodebaseDownloader struct {
// NewCodebaseDownloader creates a new downloader
func NewCodebaseDownloader(_ context.Context, projectURL *url.URL, project, repoName, username, password string) *CodebaseDownloader {
baseURL, _ := url.Parse("https://api3.codebasehq.com")
transport := NewMigrationHTTPTransport()
downloader := &CodebaseDownloader{
baseURL: baseURL,
projectURL: projectURL,
project: project,
repoName: repoName,
client: &http.Client{
Transport: roundTripperFunc(func(req *http.Request) (*http.Response, error) {
if username != "" && password != "" {
req = req.Clone(req.Context())
req.SetBasicAuth(username, password)
}
return transport.RoundTrip(req)
}),
},
client: newMigrationHTTPClient(baseURL.String(), basicAuthorization(username, password)),
userMap: make(map[int64]*codebaseUser),
commitMap: make(map[string]string),
}
+19
View File
@@ -4,7 +4,10 @@
package migrations
import (
"context"
"fmt"
"io"
"net/http"
"strings"
system_model "gitea.dev/models/system"
@@ -21,6 +24,22 @@ func WarnAndNotice(fmtStr string, args ...any) {
}
}
func downloadAsset(ctx context.Context, client *http.Client, assetURL string) (io.ReadCloser, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, assetURL, nil)
if err != nil {
return nil, err
}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
return nil, fmt.Errorf("unexpected status %s", resp.Status)
}
return resp.Body, nil
}
func hasBaseURL(toCheck, baseURL string) bool {
if baseURL == "" {
return false
+22 -47
View File
@@ -8,7 +8,6 @@ import (
"errors"
"fmt"
"io"
"net/url"
"os"
"path/filepath"
"strconv"
@@ -87,22 +86,6 @@ func (g *RepositoryDumper) reviewDir() string {
return filepath.Join(g.baseDir, "reviews")
}
func (g *RepositoryDumper) setURLToken(remoteAddr string) (string, error) {
if len(g.opts.AuthToken) > 0 || len(g.opts.AuthUsername) > 0 {
u, err := url.Parse(remoteAddr)
if err != nil {
return "", err
}
u.User = url.UserPassword(g.opts.AuthUsername, g.opts.AuthPassword)
if len(g.opts.AuthToken) > 0 {
u.User = url.UserPassword("oauth2", g.opts.AuthToken)
}
remoteAddr = u.String()
}
return remoteAddr, nil
}
// CreateRepo creates a repository
func (g *RepositoryDumper) CreateRepo(ctx context.Context, repo *base.Repository, opts base.MigrateOptions) error {
f, err := os.Create(filepath.Join(g.baseDir, "repo.yml"))
@@ -147,7 +130,7 @@ func (g *RepositoryDumper) CreateRepo(ctx context.Context, repo *base.Repository
migrateTimeout := 2 * time.Hour
remoteAddr, err := g.setURLToken(repo.CloneURL)
remoteAddr, err := base.NullDownloader{}.FormatCloneURL(g.opts, repo.CloneURL)
if err != nil {
return err
}
@@ -294,41 +277,31 @@ func (g *RepositoryDumper) CreateLabels(_ context.Context, labels ...*base.Label
}
// CreateReleases creates releases
func (g *RepositoryDumper) CreateReleases(_ context.Context, releases ...*base.Release) error {
func (g *RepositoryDumper) CreateReleases(ctx context.Context, releases ...*base.Release) error {
if g.opts.ReleaseAssets {
for _, release := range releases {
attachDir := filepath.Join("release_assets", uuid.New().String())
if err := os.MkdirAll(filepath.Join(g.baseDir, attachDir), os.ModePerm); err != nil {
return err
}
assets := release.Assets[:0]
for _, asset := range release.Assets {
// SECURITY: We cannot check the DownloadFunc is safe here
// ... we must assume that it is safe and simply download the attachment
rc, err := asset.DownloadFunc()
if err != nil {
if ctx.Err() != nil {
return ctx.Err()
}
WarnAndNotice("Skipped asset %s of release %s: %v", asset.Name, release.TagName, err)
continue
}
// we cannot use asset.Name because it might contains special characters.
attachLocalPath := filepath.Join(attachDir, uuid.New().String())
// SECURITY: We cannot check the DownloadURL and DownloadFunc are safe here
// ... we must assume that they are safe and simply download the attachment
// download attachment
err := func(attachPath string) error {
var rc io.ReadCloser
var err error
if asset.DownloadURL == nil {
rc, err = asset.DownloadFunc()
if err != nil {
return err
}
defer rc.Close()
} else {
// use the migration client so the fetch (including any redirect) is
// validated against the migration host allow/block list
resp, err := getMigrationHTTPClient().Get(*asset.DownloadURL)
if err != nil {
return err
}
defer resp.Body.Close()
rc = resp.Body
}
fw, err := os.Create(attachPath)
err = func() error {
defer rc.Close()
fw, err := os.Create(filepath.Join(g.baseDir, attachLocalPath))
if err != nil {
return fmt.Errorf("create: %w", err)
}
@@ -336,12 +309,14 @@ func (g *RepositoryDumper) CreateReleases(_ context.Context, releases ...*base.R
_, err = io.Copy(fw, rc)
return err
}(filepath.Join(g.baseDir, attachLocalPath))
}()
if err != nil {
return err
return fmt.Errorf("release %s asset %s: %w", release.TagName, asset.Name, err)
}
asset.DownloadURL = &attachLocalPath // to save the filepath on the yml file, change the source
assets = append(assets, asset)
}
release.Assets = assets
}
}
@@ -454,7 +429,7 @@ func (g *RepositoryDumper) handlePullRequest(ctx context.Context, pr *base.PullR
// SECURITY: We will assume that the pr.PatchURL has been checked
// pr.PatchURL maybe a local file - but note EnsureSafe should be asserting that this safe
u, err := g.setURLToken(pr.PatchURL)
u, err := base.NullDownloader{}.FormatCloneURL(g.opts, pr.PatchURL)
if err != nil {
return err
}
+19
View File
@@ -0,0 +1,19 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package migrations
import "gitea.dev/modules/structs"
func init() {
RegisterDownloaderFactory(&ForgejoDownloaderFactory{})
}
// ForgejoDownloaderFactory uses the Gitea downloader, as Forgejo keeps a Gitea compatible API
type ForgejoDownloaderFactory struct {
GiteaDownloaderFactory
}
func (f *ForgejoDownloaderFactory) GitServiceType() structs.GitServiceType {
return structs.ForgejoService
}
+69 -42
View File
@@ -4,6 +4,7 @@
package migrations
import (
"cmp"
"context"
"errors"
"fmt"
@@ -17,7 +18,10 @@ import (
"gitea.dev/modules/log"
base "gitea.dev/modules/migration"
"gitea.dev/modules/structs"
"gitea.dev/modules/util"
gitea_sdk "gitea.dev/sdk"
"github.com/hashicorp/go-version"
)
var (
@@ -72,6 +76,7 @@ type GiteaDownloader struct {
baseURL string
repoOwner string
repoName string
httpClient *http.Client
pagination bool
maxPerPage int
}
@@ -81,16 +86,15 @@ type GiteaDownloader struct {
// Use either a username/password or personal token. token is preferred
// Note: Public access only allows very basic access
func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, password, token string) (*GiteaDownloader, error) {
giteaClient, err := gitea_sdk.NewClient(
baseURL,
gitea_sdk.SetToken(token),
gitea_sdk.SetBasicAuth(username, password),
gitea_sdk.SetHTTPClient(newMigrationHTTPClient()),
)
httpClient := newMigrationHTTPClient(baseURL, util.Iif(token != "", "token "+token, basicAuthorization(username, password)))
giteaClient, err := gitea_sdk.NewClient(baseURL, gitea_sdk.SetHTTPClient(httpClient))
if err != nil {
log.Error(fmt.Sprintf("Failed to create NewGiteaDownloader for: %s. Error: %v", baseURL, err))
return nil, err
}
if err = pinGiteaVersion(ctx, giteaClient); err != nil {
return nil, err
}
path := strings.Split(repoPath, "/")
@@ -117,11 +121,36 @@ func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, passwo
baseURL: baseURL,
repoOwner: path[0],
repoName: path[1],
httpClient: httpClient,
pagination: paginationSupport,
maxPerPage: maxPerPage,
}, nil
}
func pinGiteaVersion(ctx context.Context, client *gitea_sdk.Client) error { // the SDK's checks reject pre-releases
rawVersion, _, err := client.Meta.ServerVersion(ctx)
if err != nil {
return err
}
if serverVersion, err := version.NewVersion(rawVersion); err == nil {
if giteaVersion, ok := strings.CutPrefix(serverVersion.Metadata(), "gitea-"); ok { // Forgejo
if serverVersion, err = version.NewVersion(giteaVersion); err != nil {
return err
}
}
return gitea_sdk.SetGiteaVersion(serverVersion.Core().String())(client)
}
return nil
}
func (g *GiteaDownloader) unitDisabled(resp *gitea_sdk.Response, hasUnit func(*gitea_sdk.Repository) bool) bool { // Gitea answers a disabled unit and missing access alike with 403 or 404, Forgejo with 404
if resp == nil || (resp.StatusCode != http.StatusNotFound && resp.StatusCode != http.StatusForbidden) {
return false
}
repo, _, err := g.client.Repositories.GetRepo(g.ctx, g.repoOwner, g.repoName)
return err == nil && !hasUnit(repo)
}
// String implements Stringer
func (g *GiteaDownloader) String() string {
return fmt.Sprintf("migration from gitea server %s %s/%s", g.baseURL, g.repoOwner, g.repoName)
@@ -225,6 +254,8 @@ func (g *GiteaDownloader) convertGiteaLabel(label *gitea_sdk.Label) *base.Label
Name: label.Name,
Color: label.Color,
Description: label.Description,
Exclusive: label.Exclusive,
Archived: label.IsArchived,
}
}
@@ -273,11 +304,7 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele
Created: rel.CreatedAt,
}
httpClient := newMigrationHTTPClient()
for _, asset := range rel.Attachments {
assetID := asset.ID // Don't optimize this, for closure we need a local variable
assetDownloadURL := asset.DownloadURL
size := int(asset.Size)
dlCount := int(asset.DownloadCount)
r.Assets = append(r.Assets, &base.ReleaseAsset{
@@ -286,30 +313,8 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele
Size: &size,
DownloadCount: &dlCount,
Created: asset.Created,
DownloadURL: &asset.DownloadURL,
DownloadFunc: func() (io.ReadCloser, error) {
asset, _, err := g.client.Releases.GetReleaseAttachment(g.ctx, g.repoOwner, g.repoName, rel.ID, assetID)
if err != nil {
return nil, err
}
if !hasBaseURL(assetDownloadURL, g.baseURL) {
WarnAndNotice("Unexpected AssetURL for assetID[%d] in %s: %s", assetID, g, assetDownloadURL)
return io.NopCloser(strings.NewReader(asset.DownloadURL)), nil
}
// FIXME: for a private download?
req, err := http.NewRequest(http.MethodGet, assetDownloadURL, nil)
if err != nil {
return nil, err
}
resp, err := httpClient.Do(req)
if err != nil {
return nil, err
}
// resp.Body is closed by the uploader
return resp.Body, nil
return downloadAsset(g.ctx, g.httpClient, g.baseURL+"/attachments/"+url.PathEscape(asset.UUID))
},
})
}
@@ -328,10 +333,13 @@ func (g *GiteaDownloader) GetReleases(ctx context.Context) ([]*base.Release, err
default:
}
rl, _, err := g.client.Releases.ListReleases(g.ctx, g.repoOwner, g.repoName, gitea_sdk.ListReleasesOptions{ListOptions: gitea_sdk.ListOptions{
rl, resp, err := g.client.Releases.ListReleases(g.ctx, g.repoOwner, g.repoName, gitea_sdk.ListReleasesOptions{ListOptions: gitea_sdk.ListOptions{
PageSize: g.maxPerPage,
Page: i,
}})
if g.unitDisabled(resp, func(repo *gitea_sdk.Repository) bool { return repo.HasReleases }) {
return releases, nil
}
if err != nil {
return nil, err
}
@@ -413,11 +421,14 @@ func (g *GiteaDownloader) GetIssues(ctx context.Context, page, perPage int) ([]*
}
allIssues := make([]*base.Issue, 0, perPage)
issues, _, err := g.client.Issues.ListRepoIssues(g.ctx, g.repoOwner, g.repoName, gitea_sdk.ListIssueOption{
issues, resp, err := g.client.Issues.ListRepoIssues(g.ctx, g.repoOwner, g.repoName, gitea_sdk.ListIssueOption{
ListOptions: gitea_sdk.ListOptions{Page: page, PageSize: perPage},
State: gitea_sdk.StateAll,
Type: gitea_sdk.IssueTypeIssue,
})
if g.unitDisabled(resp, func(repo *gitea_sdk.Repository) bool { return repo.HasIssues && repo.ExternalTracker == nil }) {
return nil, true, nil
}
if err != nil {
return nil, false, fmt.Errorf("error while listing issues: %w", err)
}
@@ -459,6 +470,7 @@ func (g *GiteaDownloader) GetIssues(ctx context.Context, page, perPage int) ([]*
Assignees: assignees,
IsLocked: issue.IsLocked,
ForeignIndex: issue.Index,
Context: issue.Comments,
})
}
@@ -471,6 +483,9 @@ func (g *GiteaDownloader) GetIssues(ctx context.Context, page, perPage int) ([]*
// GetComments returns comments according issueNumber
func (g *GiteaDownloader) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) {
if commentable.GetContext() == 0 {
return nil, true, nil
}
allComments := make([]*base.Comment, 0, g.maxPerPage)
seenIDs := container.Set[int64]{}
@@ -527,13 +542,16 @@ func (g *GiteaDownloader) GetPullRequests(ctx context.Context, page, perPage int
}
allPRs := make([]*base.PullRequest, 0, perPage)
prs, _, err := g.client.PullRequests.ListRepoPullRequests(g.ctx, g.repoOwner, g.repoName, gitea_sdk.ListPullRequestsOptions{
prs, resp, err := g.client.PullRequests.ListRepoPullRequests(g.ctx, g.repoOwner, g.repoName, gitea_sdk.ListPullRequestsOptions{
ListOptions: gitea_sdk.ListOptions{
Page: page,
PageSize: perPage,
},
State: gitea_sdk.StateAll,
})
if g.unitDisabled(resp, func(repo *gitea_sdk.Repository) bool { return repo.HasPullRequests }) {
return nil, true, nil
}
if err != nil {
return nil, false, fmt.Errorf("error while listing pull requests (page: %d, pagesize: %d). Error: %w", page, perPage, err)
}
@@ -585,7 +603,7 @@ func (g *GiteaDownloader) GetPullRequests(ctx context.Context, page, perPage int
createdAt = *pr.Created
}
updatedAt := time.Time{}
if pr.Created != nil {
if pr.Updated != nil {
updatedAt = *pr.Updated
}
@@ -613,7 +631,6 @@ func (g *GiteaDownloader) GetPullRequests(ctx context.Context, page, perPage int
MergedTime: pr.Merged,
MergeCommitSHA: mergeCommitSHA,
IsLocked: pr.IsLocked,
PatchURL: pr.PatchURL,
Head: base.PullRequestBranch{
Ref: headRef,
SHA: headSHA,
@@ -623,11 +640,12 @@ func (g *GiteaDownloader) GetPullRequests(ctx context.Context, page, perPage int
},
Base: base.PullRequestBranch{
Ref: pr.Base.Ref,
SHA: pr.Base.Sha,
SHA: cmp.Or(pr.MergeBase, pr.Base.Sha), // base.sha is the base branch head, which contains the changes once merged
RepoName: g.repoName,
OwnerName: g.repoOwner,
},
ForeignIndex: pr.Index,
Context: pr.Comments,
})
// SECURITY: Ensure that the PR is safe
_ = CheckAndEnsureSafePR(allPRs[len(allPRs)-1], g.baseURL, g)
@@ -672,9 +690,11 @@ func (g *GiteaDownloader) GetReviews(ctx context.Context, reviewable base.Review
continue
}
rcl, _, err := g.client.PullRequests.ListPullReviewComments(g.ctx, g.repoOwner, g.repoName, reviewable.GetForeignIndex(), pr.ID)
if err != nil {
return nil, err
var rcl []*gitea_sdk.PullReviewComment
if pr.CodeCommentsCount > 0 {
if rcl, _, err = g.client.PullRequests.ListPullReviewComments(g.ctx, g.repoOwner, g.repoName, reviewable.GetForeignIndex(), pr.ID); err != nil {
return nil, err
}
}
var reviewComments []*base.ReviewComment
for i := range rcl {
@@ -706,8 +726,15 @@ func (g *GiteaDownloader) GetReviews(ctx context.Context, reviewable base.Review
Content: pr.Body,
CreatedAt: pr.Submitted,
State: string(pr.State),
Dismissed: pr.Dismissed,
Comments: reviewComments,
}
switch pr.State {
case gitea_sdk.ReviewStateComment:
review.State = base.ReviewStateCommented
case gitea_sdk.ReviewStateRequestChanges:
review.State = base.ReviewStateChangesRequested
}
allReviews = append(allReviews, review)
}
@@ -0,0 +1,173 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package migrations
import (
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"runtime"
"testing"
"gitea.dev/models/unittest"
base "gitea.dev/modules/migration"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestForgejoDownloadRepo(t *testing.T) {
token := os.Getenv("FORGEJO_TEST_TOKEN")
_, callerFile, _, _ := runtime.Caller(0)
mockServer := unittest.NewMockWebServer(t, "http://localhost:3000", filepath.Join(filepath.Dir(callerFile), "_mock_data/TestForgejoDownloadRepo"), token != "")
ctx := t.Context()
downloader, err := NewGiteaDownloader(ctx, mockServer.URL, "alice/test_repo", "", "", token)
require.NoError(t, err)
assert.True(t, downloader.pagination)
repo, err := downloader.GetRepoInfo(ctx)
require.NoError(t, err)
assertRepositoryEqual(t, &base.Repository{
Name: "test_repo",
Owner: "alice",
IsPrivate: true,
Description: "Test repository for testing migration from forgejo to gitea",
Website: "https://forgejo.example/website",
CloneURL: "https://forgejo.example/alice/test_repo.git",
OriginalURL: "https://forgejo.example/alice/test_repo",
DefaultBranch: "main",
}, repo)
labels, err := downloader.GetLabels(ctx)
require.NoError(t, err)
assertLabelsEqual(t, []*base.Label{
{Name: "kind/bug", Color: "ee0701", Description: "Something is broken", Exclusive: true},
{Name: "kind/feature", Color: "0e8a16", Exclusive: true, Archived: true},
{Name: "question", Color: "cc317c"},
}, labels)
releases, err := downloader.GetReleases(ctx)
require.NoError(t, err)
require.Len(t, releases, 2)
assert.Equal(t, "v0.9.0", releases[0].TagName)
assert.True(t, releases[0].Prerelease)
assert.Equal(t, "v1.0.0", releases[1].TagName)
require.Len(t, releases[1].Assets, 2)
rc, err := releases[1].Assets[0].DownloadFunc()
require.NoError(t, err)
content, err := io.ReadAll(rc)
require.NoError(t, err)
require.NoError(t, rc.Close())
assert.Equal(t, "forgejo asset\n", string(content))
issues, isEnd, err := downloader.GetIssues(ctx, 1, 50)
require.NoError(t, err)
assert.True(t, isEnd)
require.Len(t, issues, 1)
assert.Equal(t, "first issue", issues[0].Title)
assert.Equal(t, "v1", issues[0].Milestone)
assert.Equal(t, []*base.Reaction{{UserID: 1, UserName: "alice", Content: "heart"}, {UserID: 1, UserName: "alice", Content: "+1"}}, issues[0].Reactions)
comments, _, err := downloader.GetComments(ctx, issues[0])
require.NoError(t, err)
require.Len(t, comments, 1)
assert.Equal(t, "a comment", comments[0].Content)
assert.Equal(t, []*base.Reaction{{UserID: 1, UserName: "alice", Content: "laugh"}}, comments[0].Reactions)
prs, _, err := downloader.GetPullRequests(ctx, 1, 50)
require.NoError(t, err)
require.Len(t, prs, 3)
agit, open, merged := prs[0], prs[1], prs[2]
assert.Equal(t, "refs/pull/4/head", agit.Head.Ref)
assert.Equal(t, "feature-open", open.Head.Ref)
assert.True(t, merged.Merged)
assert.Equal(t, "refs/pull/2/head", merged.Head.Ref)
assert.Equal(t, "23181c5c8c25740ab7f080faf27cbcae7fd13519", merged.Base.SHA)
assert.Equal(t, "588ba39bdea707266f2df950dcee1bb5a1764fed", merged.MergeCommitSHA)
comments, _, err = downloader.GetComments(ctx, agit)
require.NoError(t, err)
assert.Empty(t, comments)
reviews, err := downloader.GetReviews(ctx, open)
require.NoError(t, err)
require.Len(t, reviews, 2)
assert.Equal(t, base.ReviewStateChangesRequested, reviews[0].State)
assert.True(t, reviews[0].Dismissed)
require.Len(t, reviews[0].Comments, 1)
assert.Equal(t, "fix this", reviews[0].Comments[0].Content)
assert.Equal(t, base.ReviewStateCommented, reviews[1].State)
assert.False(t, reviews[1].Dismissed)
}
func TestGiteaDownloaderServerVersion(t *testing.T) {
for rawVersion, pagination := range map[string]bool{
"1.11.0": false,
"1.12.0-rc1": true,
"11.0.0+gitea-1.11.0": false,
"development": false,
} {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/v1/version" {
http.NotFound(w, r)
return
}
_, _ = fmt.Fprintf(w, `{"version":%q}`, rawVersion)
}))
downloader, err := NewGiteaDownloader(t.Context(), server.URL, "owner/repo", "", "", "")
server.Close()
require.NoError(t, err)
assert.Equal(t, pagination, downloader.pagination, rawVersion)
}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
http.Error(w, "unauthorized", http.StatusUnauthorized)
}))
defer server.Close()
_, err := NewGiteaDownloader(t.Context(), server.URL, "owner/repo", "", "", "bad-token")
assert.Error(t, err)
}
func TestGiteaDownloaderDisabledUnits(t *testing.T) {
for _, testCase := range []struct {
unitsEnabled bool
externalTracker string
}{
{false, "null"},
{true, "null"},
{true, "{}"},
} {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api/v1/version":
_, _ = w.Write([]byte(`{"version":"1.27.0"}`))
case "/api/v1/repos/owner/repo":
_, _ = fmt.Fprintf(w, `{"has_issues":%[1]t,"has_pull_requests":%[1]t,"has_releases":%[1]t,"external_tracker":%[2]s}`, testCase.unitsEnabled, testCase.externalTracker)
case "/api/v1/repos/owner/repo/releases":
http.Error(w, "forbidden", http.StatusForbidden)
default:
http.NotFound(w, r)
}
}))
ctx := t.Context()
downloader, err := NewGiteaDownloader(ctx, server.URL, "owner/repo", "", "", "")
require.NoError(t, err)
_, issuesEnd, issuesErr := downloader.GetIssues(ctx, 1, 10)
_, prsEnd, prsErr := downloader.GetPullRequests(ctx, 1, 10)
_, releasesErr := downloader.GetReleases(ctx)
server.Close()
assert.Equal(t, testCase.unitsEnabled && testCase.externalTracker == "null", issuesErr != nil)
assert.Equal(t, testCase.unitsEnabled, prsErr != nil)
assert.Equal(t, testCase.unitsEnabled, releasesErr != nil)
assert.Equal(t, !testCase.unitsEnabled || testCase.externalTracker != "null", issuesEnd)
assert.Equal(t, !testCase.unitsEnabled, prsEnd)
}
}
+6 -4
View File
@@ -247,7 +247,7 @@ func TestGiteaDownloadRepo(t *testing.T) {
Base: base.PullRequestBranch{
CloneURL: "",
Ref: "master",
SHA: "827aa28a907853e5ddfa40c8f9bc52471a2685fd",
SHA: "d9e165e4c7ab6b701f0205d0ffb637e5d2856297",
RepoName: "test_repo",
OwnerName: "gitea",
},
@@ -261,7 +261,6 @@ func TestGiteaDownloadRepo(t *testing.T) {
Merged: true,
MergedTime: new(time.Unix(1598982934, 0)),
MergeCommitSHA: "827aa28a907853e5ddfa40c8f9bc52471a2685fd",
PatchURL: mockServer.URL + "/gitea/test_repo/pulls/12.patch",
}, prs[1])
reviews, err := downloader.GetReviews(ctx, &base.Issue{Number: 7, ForeignIndex: 7})
@@ -274,7 +273,8 @@ func TestGiteaDownloadRepo(t *testing.T) {
ReviewerName: "6543",
CommitID: "187ece0cb6631e2858a6872e5733433bb3ca3b03",
CreatedAt: time.Date(2020, 9, 1, 16, 12, 58, 0, time.UTC),
State: "COMMENT", // TODO
State: base.ReviewStateCommented,
Dismissed: true,
Comments: []*base.ReviewComment{
{
ID: 116561,
@@ -299,7 +299,8 @@ func TestGiteaDownloadRepo(t *testing.T) {
ReviewerName: "techknowlogick",
CommitID: "187ece0cb6631e2858a6872e5733433bb3ca3b03",
CreatedAt: time.Date(2020, 9, 1, 17, 6, 47, 0, time.UTC),
State: "REQUEST_CHANGES", // TODO
State: base.ReviewStateChangesRequested,
Dismissed: true,
Content: "I think this needs some changes",
},
{
@@ -310,6 +311,7 @@ func TestGiteaDownloadRepo(t *testing.T) {
CommitID: "187ece0cb6631e2858a6872e5733433bb3ca3b03",
CreatedAt: time.Date(2020, 9, 1, 17, 19, 51, 0, time.UTC),
State: base.ReviewStateApproved,
Dismissed: true,
Official: true,
Content: "looks good",
},
+27 -27
View File
@@ -131,7 +131,7 @@ func (g *GiteaLocalUploader) CreateRepo(ctx context.Context, repo *base.Reposito
Wiki: opts.Wiki,
Releases: opts.Releases, // if didn't get releases, then sync them from tags
MirrorInterval: opts.MirrorInterval,
}, NewMigrationHTTPTransport())
}, getMigrationHTTPClient().Transport)
g.sameApp = strings.HasPrefix(repo.OriginalURL, setting.AppURL)
g.repo = r
@@ -237,13 +237,15 @@ func (g *GiteaLocalUploader) CreateLabels(ctx context.Context, labels ...*base.L
l.Color = color
}
lbs = append(lbs, &issues_model.Label{
lb := &issues_model.Label{
RepoID: g.repo.ID,
Name: l.Name,
Exclusive: l.Exclusive,
Description: l.Description,
Color: l.Color,
})
}
lb.SetArchived(l.Archived)
lbs = append(lbs, lb)
}
err := issues_model.NewLabels(ctx, lbs...)
@@ -332,32 +334,29 @@ func (g *GiteaLocalUploader) CreateReleases(ctx context.Context, releases ...*ba
// SECURITY: We cannot check the DownloadURL and DownloadFunc are safe here
// ... we must assume that they are safe and simply download the attachment
err := func() error {
var rc io.ReadCloser
var err error
if asset.DownloadFunc != nil {
rc, err = asset.DownloadFunc()
} else if asset.DownloadURL != nil {
// asset.DownloadURL maybe a local file
var rc io.ReadCloser
var err error
if asset.DownloadFunc != nil {
rc, err = asset.DownloadFunc()
if err != nil {
return err
}
} else if asset.DownloadURL != nil {
// use the migration client so the fetch (including any redirect) is
// validated against the migration host allow/block list
rc, err = uri.OpenWithClient(*asset.DownloadURL, getMigrationHTTPClient())
if err != nil {
return err
}
}
if rc == nil {
return nil
}
_, err = storage.Attachments.Save(attach.RelativePath(), rc, int64(*asset.Size))
rc.Close()
return err
}()
// use the migration client so the fetch (including any redirect) is
// validated against the migration host allow/block list
rc, err = uri.OpenWithClient(*asset.DownloadURL, getMigrationHTTPClient())
}
if err != nil {
return err
if ctx.Err() != nil {
return ctx.Err()
}
WarnAndNotice("Skipped asset %s of release %s: %v", asset.Name, release.TagName, err)
continue
}
if rc != nil {
attach.Size, err = storage.Attachments.Save(attach.RelativePath(), rc, util.Iif(*asset.Size > 0, int64(*asset.Size), -1)) // GitLab links and Forgejo external links report no size
rc.Close()
if err != nil {
return fmt.Errorf("release %s asset %s: %w", release.TagName, asset.Name, err)
}
}
rel.Attachments = append(rel.Attachments, &attach)
@@ -890,6 +889,7 @@ func (g *GiteaLocalUploader) CreateReviews(ctx context.Context, reviews ...*base
IssueID: issue.ID,
Content: review.Content,
Official: review.Official,
Dismissed: review.Dismissed,
CreatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()),
UpdatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()),
}
@@ -5,6 +5,8 @@
package migrations
import (
"errors"
"io"
"strconv"
"strings"
"testing"
@@ -125,6 +127,28 @@ func TestGiteaUploadRepo(t *testing.T) {
assert.Len(t, pulls[0].Issue.Comments, 2)
}
func TestCreateReleasesSkipsUnavailableAssets(t *testing.T) {
unittest.PrepareTestEnv(t)
sourceRelease := &base.Release{Name: "assets", Assets: []*base.ReleaseAsset{
{Name: "gone", Size: new(0), DownloadCount: new(0), DownloadFunc: func() (io.ReadCloser, error) { return nil, errors.New("gone") }},
{Name: "kept", Size: new(0), DownloadCount: new(0), DownloadFunc: func() (io.ReadCloser, error) { return io.NopCloser(strings.NewReader("data")), nil }},
}}
doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
uploader := NewGiteaLocalUploader(t.Context(), doer, doer.Name, "repo1")
uploader.repo = unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1})
require.NoError(t, uploader.CreateReleases(t.Context(), sourceRelease))
release := unittest.AssertExistsAndLoadBean(t, &repo_model.Release{RepoID: 1, Title: "assets"})
unittest.AssertExistsAndLoadBean(t, &repo_model.Attachment{ReleaseID: release.ID, Name: "kept", Size: 4})
unittest.AssertNotExistsBean(t, &repo_model.Attachment{ReleaseID: release.ID, Name: "gone"})
dumper := &RepositoryDumper{baseDir: t.TempDir(), opts: base.MigrateOptions{ReleaseAssets: true}}
defer dumper.Close()
require.NoError(t, dumper.CreateReleases(t.Context(), sourceRelease))
require.Len(t, sourceRelease.Assets, 1)
assert.Equal(t, "kept", sourceRelease.Assets[0].Name)
}
func TestGiteaUploadRemapLocalUser(t *testing.T) {
unittest.PrepareTestEnv(t)
doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
+13 -44
View File
@@ -18,9 +18,9 @@ import (
"gitea.dev/modules/log"
base "gitea.dev/modules/migration"
"gitea.dev/modules/structs"
"gitea.dev/modules/util"
"github.com/google/go-github/v92/github"
"golang.org/x/oauth2"
)
var (
@@ -99,29 +99,15 @@ func NewGithubDownloaderV3(_ context.Context, baseURL, userName, password, token
maxPerPage: 100,
}
apiURL := util.Iif(baseURL == "https://github.com", "https://api.github.com", baseURL)
if token != "" {
tokens := strings.SplitSeq(token, ",")
for token := range tokens {
token = strings.TrimSpace(token)
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: token},
)
client := &http.Client{
Transport: &oauth2.Transport{
Base: NewMigrationHTTPTransport(),
Source: oauth2.ReuseTokenSource(nil, ts),
},
}
if err := downloader.addClient(client, baseURL); err != nil {
for token := range strings.SplitSeq(token, ",") {
if err := downloader.addClient(newMigrationHTTPClient(apiURL, "Bearer "+strings.TrimSpace(token)), baseURL); err != nil {
return nil, err
}
}
} else {
client := &http.Client{
Transport: &github.BasicAuthTransport{Transport: NewMigrationHTTPTransport(), Username: userName, Password: password},
}
if err := downloader.addClient(client, baseURL); err != nil {
if err := downloader.addClient(newMigrationHTTPClient(apiURL, basicAuthorization(userName, password)), baseURL); err != nil {
return nil, err
}
}
@@ -339,8 +325,6 @@ func (g *GithubDownloaderV3) convertGithubRelease(ctx context.Context, rel *gith
r.Published = rel.PublishedAt.Time
}
httpClient := newMigrationHTTPClient()
for _, asset := range rel.Assets {
assetID := asset.GetID() // Don't optimize this, for closure we need a local variable TODO: no need to do so in new Golang
if assetID == 0 {
@@ -381,19 +365,11 @@ func (g *GithubDownloaderV3) convertGithubRelease(ctx context.Context, rel *gith
}
g.waitAndPickClient(ctx)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, redirectURL, nil)
if err != nil {
return nil, err
}
resp, err := httpClient.Do(req)
err1 := g.RefreshRate(ctx)
if err1 != nil {
rc, err := downloadAsset(ctx, getMigrationHTTPClient(), redirectURL)
if err1 := g.RefreshRate(ctx); err1 != nil {
log.Error("g.RefreshRate(): %s", err1)
}
if err != nil {
return nil, err
}
return resp.Body, nil
return rc, err
},
})
}
@@ -780,6 +756,7 @@ func (g *GithubDownloaderV3) getIssueReactions(ctx context.Context, number, perP
}
func convertGithubReview(r *github.PullRequestReview) *base.Review {
dismissed := r.GetState() == "DISMISSED"
return &base.Review{
ID: r.GetID(),
ReviewerID: r.GetUser().GetID(),
@@ -787,7 +764,8 @@ func convertGithubReview(r *github.PullRequestReview) *base.Review {
CommitID: r.GetCommitID(),
Content: r.GetBody(),
CreatedAt: r.GetSubmittedAt().Time,
State: r.GetState(),
State: util.Iif(dismissed, base.ReviewStateCommented, r.GetState()), // GitHub drops the state a review had before its dismissal
Dismissed: dismissed,
}
}
@@ -910,15 +888,6 @@ func (g *GithubDownloaderV3) GetReviews(ctx context.Context, reviewable base.Rev
// FormatCloneURL add authentication into remote URLs
func (g *GithubDownloaderV3) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) {
u, err := url.Parse(remoteAddr)
if err != nil {
return "", err
}
if len(opts.AuthToken) > 0 {
// "multiple tokens" are used to benefit more "API rate limit quota"
// git clone doesn't count for rate limits, so only use the first token.
// source: https://github.com/orgs/community/discussions/44515
u.User = url.UserPassword("oauth2", strings.Split(opts.AuthToken, ",")[0])
}
return u.String(), nil
opts.AuthToken, _, _ = strings.Cut(opts.AuthToken, ",") // the extra tokens only raise the API rate limit
return g.NullDownloader.FormatCloneURL(opts, remoteAddr)
}
+14 -2
View File
@@ -543,10 +543,17 @@ func TestGitHubDownloadRepo(t *testing.T) {
}, reviews)
}
func TestGithubMultiToken(t *testing.T) {
func TestConvertGithubDismissedReview(t *testing.T) {
review := convertGithubReview(&github.PullRequestReview{State: new("DISMISSED")})
assert.Equal(t, base.ReviewStateCommented, review.State)
assert.True(t, review.Dismissed)
}
func TestGithubFormatCloneURL(t *testing.T) {
testCases := []struct {
desc string
token string
username string
expectedCloneURL string
}{
{
@@ -559,12 +566,17 @@ func TestGithubMultiToken(t *testing.T) {
token: "token1,token2",
expectedCloneURL: "https://oauth2:token1@github.com",
},
{
desc: "Username And Password",
username: "user",
expectedCloneURL: "https://user:password@github.com",
},
}
factory := GithubDownloaderV3Factory{}
for _, tC := range testCases {
t.Run(tC.desc, func(t *testing.T) {
opts := base.MigrateOptions{CloneAddr: "https://github.com/go-gitea/gitea", AuthToken: tC.token}
opts := base.MigrateOptions{CloneAddr: "https://github.com/go-gitea/gitea", AuthToken: tC.token, AuthUsername: tC.username, AuthPassword: "password"}
client, err := factory.New(t.Context(), opts)
require.NoError(t, err)
+16 -19
View File
@@ -21,6 +21,7 @@ import (
"gitea.dev/modules/log"
base "gitea.dev/modules/migration"
"gitea.dev/modules/structs"
"gitea.dev/modules/util"
gitlab "gitlab.com/gitlab-org/api/client-go/v3"
)
@@ -94,7 +95,8 @@ type GitlabDownloader struct {
// Use either a username/password, personal token entered into the username field, or anonymous/public access
// Note: Public access only allows very basic access
func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, token string) (*GitlabDownloader, error) {
gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(newMigrationHTTPClient()))
transport := authTransport(getMigrationTransport(), baseURL, util.Iif(token != "", "Bearer "+token, "")) // not retrying, the SDK retries rate limits itself
gitlabClient, err := gitlab.NewAuthSourceClient(gitlab.Unauthenticated{}, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(&http.Client{Transport: transport}))
if err != nil {
log.Trace("Error logging into gitlab: %v", err)
return nil, err
@@ -314,8 +316,6 @@ func (g *GitlabDownloader) convertGitlabRelease(ctx context.Context, rel *gitlab
PublisherName: rel.Author.Username,
}
httpClient := newMigrationHTTPClient()
for _, asset := range rel.Assets.Links {
assetID := asset.ID // Don't optimize this, for closure we need a local variable
r.Assets = append(r.Assets, &base.ReleaseAsset{
@@ -334,18 +334,7 @@ func (g *GitlabDownloader) convertGitlabRelease(ctx context.Context, rel *gitlab
return io.NopCloser(strings.NewReader(link.URL)), nil
}
req, err := http.NewRequest(http.MethodGet, link.URL, nil)
if err != nil {
return nil, err
}
req = req.WithContext(ctx)
resp, err := httpClient.Do(req)
if err != nil {
return nil, err
}
// resp.Body is closed by the uploader
return resp.Body, nil
return downloadAsset(ctx, g.client.HTTPClient(), link.URL)
},
})
}
@@ -408,6 +397,12 @@ func (g *GitlabDownloader) GetIssues(ctx context.Context, page, perPage int) ([]
return nil, false, fmt.Errorf("error while listing issues: %w", err)
}
for _, issue := range issues {
// record the issue IID, to be used in GetPullRequests()
g.iidResolver.recordIssueIID(issue.IID)
if issue.Confidential { // Gitea can't restrict an issue to project members
continue
}
labels := make([]*base.Label, 0, len(issue.Labels))
for _, l := range issue.Labels {
labels = append(labels, &base.Label{
@@ -454,9 +449,6 @@ func (g *GitlabDownloader) GetIssues(ctx context.Context, page, perPage int) ([]
ForeignIndex: issue.IID,
Context: gitlabIssueContext{IsMergeRequest: false},
})
// record the issue IID, to be used in GetPullRequests()
g.iidResolver.recordIssueIID(issue.IID)
}
return allIssues, len(issues) < perPage, nil
@@ -499,7 +491,9 @@ func (g *GitlabDownloader) GetComments(ctx context.Context, commentable base.Com
}
for _, comment := range comments {
for _, note := range comment.Notes {
allComments = append(allComments, g.convertNoteToComment(commentable.GetLocalIndex(), note))
if !note.Internal { // Gitea can't restrict a comment to project members
allComments = append(allComments, g.convertNoteToComment(commentable.GetLocalIndex(), note))
}
}
}
if resp.NextPage == 0 {
@@ -749,6 +743,9 @@ func (g *GitlabDownloader) GetReviews(ctx context.Context, reviewable base.Revie
reviews := make([]*base.Review, 0, len(approvals.ApprovedBy))
for _, user := range approvals.ApprovedBy {
if user.User == nil {
continue
}
reviews = append(reviews, &base.Review{
IssueIndex: reviewable.GetLocalIndex(),
ReviewerID: user.User.ID,
+34
View File
@@ -20,6 +20,7 @@ import (
base "gitea.dev/modules/migration"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
gitlab "gitlab.com/gitlab-org/api/client-go/v3"
)
@@ -513,6 +514,39 @@ func TestGitlabGetReviews(t *testing.T) {
assert.NoError(t, err)
assertReviewsEqual(t, []*base.Review{&review}, rvs)
}
mux.HandleFunc(fmt.Sprintf("/api/v4/projects/%d/merge_requests/4/approvals", repoID), func(w http.ResponseWriter, _ *http.Request) {
fmt.Fprint(w, `{"approved_by": [{"user": null}]}`)
})
rvs, err := downloader.GetReviews(ctx, &base.Issue{Number: 4, ForeignIndex: 4})
require.NoError(t, err)
assert.Empty(t, rvs)
}
func TestGitlabSkipsConfidentialIssuesAndInternalNotes(t *testing.T) {
mux, server, client := gitlabClientMockSetup(t)
defer gitlabClientMockTeardown(server)
downloader := &GitlabDownloader{client: client, repoID: 1, maxPerPage: 100}
const timestamps = `"created_at": "2020-01-01T00:00:00Z", "updated_at": "2020-01-01T00:00:00Z"`
empty := func(w http.ResponseWriter, _ *http.Request) { fmt.Fprint(w, `[]`) }
mux.HandleFunc("/api/v4/projects/1/issues", func(w http.ResponseWriter, _ *http.Request) {
fmt.Fprint(w, `[{"iid": 1, "author": {}, `+timestamps+`}, {"iid": 2, "confidential": true}]`)
})
mux.HandleFunc("/api/v4/projects/1/issues/1/award_emoji", empty)
mux.HandleFunc("/api/v4/projects/1/issues/1/discussions", func(w http.ResponseWriter, _ *http.Request) {
fmt.Fprint(w, `[{"notes": [{"id": 1, `+timestamps+`}, {"internal": true}]}]`)
})
mux.HandleFunc("/api/v4/projects/1/issues/1/resource_state_events", empty)
issues, _, err := downloader.GetIssues(t.Context(), 1, 100)
require.NoError(t, err)
require.Len(t, issues, 1)
assert.EqualValues(t, 3, downloader.iidResolver.generatePullRequestNumber(1))
comments, _, err := downloader.GetComments(t.Context(), issues[0])
require.NoError(t, err)
require.Len(t, comments, 1)
assert.EqualValues(t, 1, comments[0].Index)
}
func TestAwardsToReactions(t *testing.T) {
+6 -22
View File
@@ -14,6 +14,7 @@ import (
"gitea.dev/modules/log"
base "gitea.dev/modules/migration"
"gitea.dev/modules/structs"
"gitea.dev/modules/util"
"github.com/gogs/go-gogs-client"
)
@@ -62,9 +63,7 @@ type GogsDownloader struct {
baseURL string
repoOwner string
repoName string
userName string
password string
token string
transport http.RoundTripper
openIssuesFinished bool
openIssuesPages int
}
@@ -83,36 +82,21 @@ func (g *GogsDownloader) LogString() string {
// NewGogsDownloader creates a gogs Downloader via gogs API
func NewGogsDownloader(_ context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GogsDownloader {
downloader := GogsDownloader{
return &GogsDownloader{
baseURL: baseURL,
userName: userName,
password: password,
token: token,
transport: newMigrationHTTPClient(baseURL, util.Iif(token != "", "token "+token, basicAuthorization(userName, password))).Transport,
repoOwner: repoOwner,
repoName: repoName,
}
return &downloader
}
type roundTripperFunc func(req *http.Request) (*http.Response, error)
func (rt roundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) {
return rt(r)
}
func (g *GogsDownloader) client(ctx context.Context) *gogs.Client {
// Gogs client lacks the context support, so we use a custom transport
// Then each request uses a dedicated client with its own context
httpTransport := NewMigrationHTTPTransport()
gogsClient := gogs.NewClient(g.baseURL, g.token)
gogsClient := gogs.NewClient(g.baseURL, "")
gogsClient.SetHTTPClient(&http.Client{
Transport: roundTripperFunc(func(req *http.Request) (*http.Response, error) {
req = req.Clone(ctx)
if g.password != "" {
// Gogs client lacks the support for basic auth, this is the only way to set it
req.SetBasicAuth(g.userName, g.password)
}
return httpTransport.RoundTrip(req)
return g.transport.RoundTrip(req.WithContext(ctx))
}),
})
return gogsClient
+91 -5
View File
@@ -5,24 +5,110 @@ package migrations
import (
"crypto/tls"
"encoding/base64"
"net/http"
"net/url"
"strconv"
"sync"
"time"
"gitea.dev/modules/egress"
"gitea.dev/modules/log"
"gitea.dev/modules/setting"
)
// getMigrationHTTPClient returns the shared migration client, so downloads from one host reuse its connections
var getMigrationHTTPClient = sync.OnceValue(newMigrationHTTPClient)
var getMigrationHTTPClient = sync.OnceValue(func() *http.Client {
return &http.Client{Transport: retryAfterTransport{getMigrationTransport()}}
})
// newMigrationHTTPClient returns a HTTP client for migration
func newMigrationHTTPClient() *http.Client {
return &http.Client{Transport: NewMigrationHTTPTransport()}
var getMigrationTransport = sync.OnceValue(newMigrationTransport)
func newMigrationHTTPClient(baseURL, authorization string) *http.Client {
return &http.Client{Transport: authTransport(getMigrationHTTPClient().Transport, baseURL, authorization)}
}
// NewMigrationHTTPTransport returns a HTTP transport for migration, enforcing the migration policy on its direct dials.
func NewMigrationHTTPTransport() *http.Transport {
func NewMigrationHTTPTransport() http.RoundTripper {
return retryAfterTransport{newMigrationTransport()}
}
func newMigrationTransport() *http.Transport {
t := egress.NewMigrationPolicy().NewHTTPTransport()
t.TLSClientConfig = &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}
return t
}
type retryAfterTransport struct {
http.RoundTripper
}
func (t retryAfterTransport) RoundTrip(req *http.Request) (*http.Response, error) {
var waited time.Duration
for retries := 0; ; retries++ {
resp, err := t.RoundTripper.RoundTrip(req)
if err != nil {
return nil, err
}
if (resp.StatusCode != http.StatusTooManyRequests && resp.StatusCode != http.StatusForbidden) || retries == 5 || (req.Body != nil && req.GetBody == nil) { // GitHub's secondary rate limits answer 403
return resp, nil
}
delay, ok := parseRetryAfter(resp.Header.Get("Retry-After"))
if !ok || delay > time.Hour-waited {
return resp, nil
}
waited += delay
resp.Body.Close()
log.Info("Rate limited by %s, retrying in %s", req.URL.Host, delay)
select {
case <-req.Context().Done():
return nil, req.Context().Err()
case <-time.After(delay):
}
if req.GetBody != nil {
body, err := req.GetBody()
if err != nil {
return nil, err
}
req = req.Clone(req.Context())
req.Body = body
}
}
}
func parseRetryAfter(value string) (time.Duration, bool) {
if seconds, err := strconv.ParseUint(value, 10, 32); err == nil { // 32 bits can't overflow the Duration
return time.Duration(seconds) * time.Second, true
}
if retryAt, err := http.ParseTime(value); err == nil {
return max(time.Until(retryAt), 0), true
}
return 0, false
}
type roundTripperFunc func(req *http.Request) (*http.Response, error)
func (rt roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) {
return rt(req)
}
func authTransport(transport http.RoundTripper, baseURL, authorization string) http.RoundTripper {
base, err := url.Parse(baseURL)
if authorization == "" || err != nil {
return transport
}
return roundTripperFunc(func(req *http.Request) (*http.Response, error) {
if req.URL.Host == base.Host && (req.URL.Scheme == "https" || base.Scheme == "http") { // the source host only, never downgraded to http
req = req.Clone(req.Context())
req.Header.Set("Authorization", authorization)
}
return transport.RoundTrip(req)
})
}
func basicAuthorization(username, password string) string {
if username == "" {
return ""
}
return "Basic " + base64.StdEncoding.EncodeToString([]byte(username+":"+password))
}
+63
View File
@@ -0,0 +1,63 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package migrations
import (
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMigrationHTTPTransportRetryAfter(t *testing.T) {
var requests atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch requests.Add(1) {
case 1:
w.Header().Set("Retry-After", "0")
w.WriteHeader(http.StatusTooManyRequests)
case 2:
w.Header().Set("Retry-After", time.Now().UTC().Format(http.TimeFormat))
w.WriteHeader(http.StatusForbidden)
default:
_, _ = io.Copy(w, r.Body)
}
}))
defer server.Close()
resp, err := getMigrationHTTPClient().Post(server.URL, "text/plain", strings.NewReader("payload"))
require.NoError(t, err)
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
assert.Equal(t, http.StatusOK, resp.StatusCode)
assert.Equal(t, "payload", string(body))
assert.EqualValues(t, 3, requests.Load())
}
func TestAuthTransportSendsCredentialsToSameHostOnly(t *testing.T) {
for _, testCase := range []struct{ baseURL, target, expected string }{
{"http://source.example/sub", "https://source.example/api", "token secret"},
{"https://source.example", "https://source.example/api", "token secret"},
{"https://source.example", "http://source.example/api", ""},
{"https://source.example", "https://other.example/api", ""},
{"https://source.example:3000", "https://source.example:8443/api", ""},
} {
req, err := http.NewRequest(http.MethodGet, testCase.target, nil)
require.NoError(t, err)
resp, err := authTransport(roundTripperFunc(func(req *http.Request) (*http.Response, error) {
assert.Equal(t, testCase.expected, req.Header.Get("Authorization"), testCase.target)
return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody}, nil
}), testCase.baseURL, "token secret").RoundTrip(req)
require.NoError(t, err)
resp.Body.Close()
assert.Empty(t, req.Header.Get("Authorization"))
}
}
+2
View File
@@ -54,6 +54,7 @@ func assertCommentsEqual(t *testing.T, expected, actual []*base.Comment) {
func assertLabelEqual(t *testing.T, expected, actual *base.Label) {
assert.Equal(t, expected.Name, actual.Name)
assert.Equal(t, expected.Exclusive, actual.Exclusive)
assert.Equal(t, expected.Archived, actual.Archived)
assert.Equal(t, expected.Color, actual.Color)
assert.Equal(t, expected.Description, actual.Description)
}
@@ -228,6 +229,7 @@ func assertReviewEqual(t *testing.T, expected, actual *base.Review) {
assert.Equal(t, expected.Content, actual.Content, "Content")
assert.WithinDuration(t, expected.CreatedAt, actual.CreatedAt, 10*time.Second)
assert.Equal(t, expected.State, actual.State, "State")
assert.Equal(t, expected.Dismissed, actual.Dismissed, "Dismissed")
assertReviewCommentsEqual(t, expected.Comments, actual.Comments)
}
+7 -4
View File
@@ -10,6 +10,7 @@ import (
"fmt"
"net/url"
"path/filepath"
"slices"
"strings"
repo_model "gitea.dev/models/repo"
@@ -293,14 +294,14 @@ func migrateRepository(ctx context.Context, doer *user_model.User, downloader ba
supportAllComments := downloader.SupportGetRepoComments()
mapInsertedIssueIndexes := container.Set[int64]{} // deletions on the source during the migration shift items into the next page
mapInsertedPRIndexes := container.Set[int64]{}
if opts.Issues {
log.Trace("migrating issues and comments")
messenger("repo.migrate.migrating_issues")
issueBatchSize := uploader.MaxBatchInsertSize("issue")
// because when the migrating is running, some issues maybe removed, so after the next page
// some of issue maybe duplicated, so we need to record the inserted issue indexes
mapInsertedIssueIndexes := container.Set[int64]{}
for i := 1; ; i++ {
issues, isEnd, err := downloader.GetIssues(ctx, i, issueBatchSize)
if err != nil {
@@ -363,7 +364,6 @@ func migrateRepository(ctx context.Context, doer *user_model.User, downloader ba
log.Trace("migrating pull requests and comments")
messenger("repo.migrate.migrating_pulls")
prBatchSize := uploader.MaxBatchInsertSize("pullrequest")
mapInsertedPRIndexes := container.Set[int64]{}
for i := 1; ; i++ {
prs, isEnd, err := downloader.GetPullRequests(ctx, i, prBatchSize)
if err != nil {
@@ -466,6 +466,9 @@ func migrateRepository(ctx context.Context, doer *user_model.User, downloader ba
if err != nil {
return err
}
comments = slices.DeleteFunc(comments, func(comment *base.Comment) bool {
return !mapInsertedIssueIndexes.Contains(comment.IssueIndex) && !mapInsertedPRIndexes.Contains(comment.IssueIndex)
})
if err := uploader.CreateComments(ctx, comments...); err != nil {
return err
+4 -15
View File
@@ -80,26 +80,15 @@ type OneDevDownloader struct {
}
// NewOneDevDownloader creates a new downloader
func NewOneDevDownloader(ctx context.Context, baseURL *url.URL, username, password, repoPath string) *OneDevDownloader {
httpTransport := NewMigrationHTTPTransport()
downloader := &OneDevDownloader{
func NewOneDevDownloader(_ context.Context, baseURL *url.URL, username, password, repoPath string) *OneDevDownloader {
return &OneDevDownloader{
baseURL: baseURL,
repoPath: repoPath,
client: &http.Client{
Transport: roundTripperFunc(
func(req *http.Request) (*http.Response, error) {
req = req.Clone(ctx)
if username != "" && password != "" {
req.SetBasicAuth(username, password)
}
return httpTransport.RoundTrip(req)
}),
},
client: newMigrationHTTPClient(baseURL.String(), basicAuthorization(username, password)),
userMap: make(map[int64]*onedevUser),
milestoneMap: make(map[int64]string),
}
return downloader
}
// String implements Stringer
+1 -1
View File
@@ -71,7 +71,7 @@ func cloneWiki(ctx context.Context, repo *repo_model.Repository, opts migration.
// MigrateRepositoryGitData starts migrating git related data after created migrating repository
func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
repo *repo_model.Repository, opts migration.MigrateOptions,
httpTransport *http.Transport,
httpTransport http.RoundTripper,
) (*repo_model.Repository, error) {
if u.IsOrganization() {
t, err := organization.OrgFromUser(u).GetOwnerTeam(ctx)