mirror of
https://github.com/go-gitea/gitea.git
synced 2026-10-26 06:16:56 +09:00
feat: Improve Gitea-compatible forge migrations, fix many migration bugs (#39424)
Supports migrating from Gitea-compatible forges like Forgejo and fixes many migration bugs. 1. Pre-release and `+gitea-` suffixed server versions no longer drop reviews, reactions and pagination. 2. Review states, dismissals, scoped and archived labels and merge bases are migrated. 3. Disabled units, external trackers, unavailable release assets, and GitLab confidential issues and internal notes are skipped. 4. Credentials are only sent to the source host, and `Retry-After` rate limits are waited out. 5. GitHub migrations of only issues or pull requests, basic auth clones and dismissed reviews work again. 6. The API returns 404 for releases of a disabled unit and shows migrated authors as mannequins, replacing the never populated `original_author` and `original_author_id` fields. Fixes: https://github.com/go-gitea/gitea/issues/19533 Fixes: https://github.com/go-gitea/gitea/issues/35885 Fixes: https://github.com/go-gitea/gitea/issues/18275 Fixes: https://github.com/go-gitea/gitea/issues/35069 Fixes: https://github.com/go-gitea/gitea/issues/17763 Fixes: https://github.com/go-gitea/gitea/issues/39548 Co-authored-by: bircni <bircni@icloud.com>
This commit is contained in:
1 parent
7c91252be3
commit
fc49368845
94 files changed
+912
-417
No files matched your search
@@ -50,7 +50,7 @@ func toIssue(ctx context.Context, doer *user_model.User, issue *issues_model.Iss
|
||||
apiIssue := &api.Issue{
|
||||
ID: issue.ID,
|
||||
Index: issue.Index,
|
||||
Poster: ToUser(ctx, issue.Poster, doer),
|
||||
Poster: ToAuthor(ctx, issue.Poster, issue.OriginalAuthor, doer),
|
||||
Title: issue.Title,
|
||||
Body: issue.Content,
|
||||
Attachments: toAttachments(ctx, issue.Repo, issue.Attachments, getDownloadURL),
|
||||
|
||||
@@ -18,7 +18,7 @@ import (
|
||||
func ToAPIComment(ctx context.Context, repo *repo_model.Repository, c *issues_model.Comment) *api.Comment {
|
||||
return &api.Comment{
|
||||
ID: c.ID,
|
||||
Poster: ToUser(ctx, c.Poster, nil),
|
||||
Poster: ToAuthor(ctx, c.Poster, c.OriginalAuthor, nil),
|
||||
HTMLURL: c.HTMLURL(ctx),
|
||||
IssueURL: c.IssueURL(ctx),
|
||||
PRURL: c.PRURL(ctx),
|
||||
@@ -86,7 +86,7 @@ func ToTimelineComment(ctx context.Context, repo *repo_model.Repository, c *issu
|
||||
comment := &api.TimelineComment{
|
||||
ID: c.ID,
|
||||
Type: c.Type.String(),
|
||||
Poster: ToUser(ctx, c.Poster, nil),
|
||||
Poster: ToAuthor(ctx, c.Poster, c.OriginalAuthor, nil),
|
||||
HTMLURL: c.HTMLURL(ctx),
|
||||
IssueURL: c.IssueURL(ctx),
|
||||
PRURL: c.PRURL(ctx),
|
||||
|
||||
@@ -23,7 +23,7 @@ func ToPullReview(ctx context.Context, r *issues_model.Review, doer *user_model.
|
||||
|
||||
result := &api.PullReview{
|
||||
ID: r.ID,
|
||||
Reviewer: ToUser(ctx, r.Reviewer, doer),
|
||||
Reviewer: ToAuthor(ctx, r.Reviewer, r.OriginalAuthor, doer),
|
||||
State: api.ReviewStateUnknown,
|
||||
Body: r.Content,
|
||||
CommitID: r.CommitID,
|
||||
@@ -104,7 +104,7 @@ func ToPullReviewComment(ctx context.Context, comment *issues_model.Comment, doe
|
||||
apiComment := &api.PullReviewComment{
|
||||
ID: comment.ID,
|
||||
Body: comment.Content,
|
||||
Poster: ToUser(ctx, comment.Poster, doer),
|
||||
Poster: ToAuthor(ctx, comment.Poster, comment.OriginalAuthor, doer),
|
||||
Resolver: ToUser(ctx, comment.ResolveDoer, doer),
|
||||
ReviewID: comment.ReviewID,
|
||||
Created: comment.CreatedUnix.AsTime(),
|
||||
|
||||
@@ -28,7 +28,7 @@ func ToAPIRelease(ctx context.Context, repo *repo_model.Repository, r *repo_mode
|
||||
IsPrerelease: r.IsPrerelease,
|
||||
CreatedAt: r.CreatedUnix.AsTime(),
|
||||
PublishedAt: util.Iif(r.IsDraft, nil, r.PublishedUnix.AsTimePtr()),
|
||||
Publisher: ToUser(ctx, r.Publisher, nil),
|
||||
Publisher: ToAuthor(ctx, r.Publisher, r.OriginalAuthor, nil),
|
||||
Attachments: ToAPIAttachments(ctx, repo, r.Attachments),
|
||||
}
|
||||
}
|
||||
@@ -50,6 +50,13 @@ func ToUser(ctx context.Context, user, doer *user_model.User) *api.User {
|
||||
return toUser(ctx, user, signed, authed)
|
||||
}
|
||||
|
||||
func ToAuthor(ctx context.Context, author *user_model.User, originalAuthor string, doer *user_model.User) *api.User {
|
||||
if originalAuthor == "" {
|
||||
return ToUser(ctx, author, doer)
|
||||
}
|
||||
return &api.User{ID: user_model.GhostUserID, UserName: originalAuthor, Type: api.UserTypeStringUser, Visibility: api.VisibilityStringPublic, AvatarURL: user_model.NewGhostUser().AvatarLink(ctx)}
|
||||
}
|
||||
|
||||
// ToUsers convert list of user_model.User to list of api.User
|
||||
func ToUsers(ctx context.Context, doer *user_model.User, users []*user_model.User) []*api.User {
|
||||
result := make([]*api.User, len(users))
|
||||
|
||||
@@ -42,6 +42,8 @@ func ToGitServiceType(value string) structs.GitServiceType {
|
||||
return structs.CodeCommitService
|
||||
case "bitbucketserver":
|
||||
return structs.BitbucketServerService
|
||||
case "forgejo":
|
||||
return structs.ForgejoService
|
||||
default:
|
||||
return structs.PlainGitService
|
||||
}
|
||||
|
||||
@@ -40,6 +40,8 @@ func TestToGitServiceType(t *testing.T) {
|
||||
typ: "codecommit", enum: 9,
|
||||
}, {
|
||||
typ: "bitbucketserver", enum: 10,
|
||||
}, {
|
||||
typ: "forgejo", enum: 11,
|
||||
}}
|
||||
for _, test := range tc {
|
||||
assert.EqualValues(t, test.enum, ToGitServiceType(test.typ))
|
||||
|
||||
Generated
+6
@@ -0,0 +1,6 @@
|
||||
Cache-Control: max-age=0, private, must-revalidate
|
||||
Content-Type: application/json;charset=utf-8
|
||||
X-Content-Type-Options: nosniff
|
||||
X-Frame-Options: SAMEORIGIN
|
||||
|
||||
{"id":1,"owner":{"id":1,"login":"alice","login_name":"","source_id":0,"full_name":"","email":"alice@example.com","avatar_url":"https://forgejo.example/avatars/c160f8cc69a4f0bf2b0362752353d060","html_url":"https://forgejo.example/alice","language":"","is_admin":false,"last_login":"0001-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":false,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"alice"},"name":"test_repo","full_name":"alice/test_repo","description":"Test repository for testing migration from forgejo to gitea","empty":false,"private":true,"fork":false,"template":false,"parent":null,"mirror":false,"size":3,"language":"","languages_url":"https://forgejo.example/api/v1/repos/alice/test_repo/languages","html_url":"https://forgejo.example/alice/test_repo","url":"https://forgejo.example/api/v1/repos/alice/test_repo","link":"","ssh_url":"ssh://git@forgejo.example/alice/test_repo.git","clone_url":"https://forgejo.example/alice/test_repo.git","original_url":"","website":"https://forgejo.example/website","stars_count":0,"forks_count":0,"watchers_count":1,"open_issues_count":1,"open_pr_counter":2,"release_counter":2,"default_branch":"main","archived":false,"created_at":"2026-09-25T01:13:52Z","updated_at":"2026-09-25T01:14:07Z","archived_at":"1970-01-01T00:00:00Z","permissions":{"admin":true,"push":true,"pull":true},"has_issues":true,"internal_tracker":{"enable_time_tracker":true,"allow_only_contributors_to_track_time":true,"enable_issue_dependencies":true},"has_wiki":true,"has_wiki_contents":false,"wiki_branch":"main","wiki_ssh_url":"ssh://git@forgejo.example/alice/test_repo.wiki.git","wiki_clone_url":"https://forgejo.example/alice/test_repo.wiki.git","globally_editable_wiki":false,"has_pull_requests":true,"has_projects":true,"has_releases":true,"has_packages":true,"has_actions":true,"ignore_whitespace_conflicts":false,"allow_merge_commits":true,"allow_rebase":true,"allow_rebase_explicit":true,"allow_squash_merge":true,"allow_fast_forward_only_merge":true,"allow_rebase_update":true,"default_delete_branch_after_merge":false,"default_merge_style":"merge","default_allow_maintainer_edit":false,"default_update_style":"merge","avatar_url":"","internal":false,"mirror_interval":"","object_format_name":"sha1","mirror_updated":"0001-01-01T00:00:00Z","repo_transfer":null,"topics":["forgejo","migration"]}
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
Access-Control-Expose-Headers: X-Total-Count
|
||||
Cache-Control: max-age=0, private, must-revalidate
|
||||
Content-Length: 872
|
||||
Content-Type: application/json;charset=utf-8
|
||||
X-Content-Type-Options: nosniff
|
||||
X-Frame-Options: SAMEORIGIN
|
||||
X-Total-Count: 1
|
||||
|
||||
[{"id":3,"html_url":"https://forgejo.example/alice/test_repo/issues/1#issuecomment-3","pull_request_url":"","issue_url":"https://forgejo.example/alice/test_repo/issues/1","user":{"id":2,"login":"bob","login_name":"","source_id":0,"full_name":"","email":"bob@noreply.forgejo.example","avatar_url":"https://forgejo.example/avatars/4b9bb80620f03eb3719e0a061c14283d","html_url":"https://forgejo.example/bob","language":"","is_admin":false,"last_login":"0001-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":false,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"bob"},"original_author":"","original_author_id":0,"body":"a comment","assets":[],"created_at":"2026-09-25T01:13:56Z","updated_at":"2026-09-25T01:13:56Z"}]
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
Access-Control-Expose-Headers: X-Total-Count
|
||||
Cache-Control: max-age=0, private, must-revalidate
|
||||
Content-Length: 1203
|
||||
Content-Type: application/json;charset=utf-8
|
||||
X-Content-Type-Options: nosniff
|
||||
X-Frame-Options: SAMEORIGIN
|
||||
X-Total-Count: 2
|
||||
|
||||
[{"user":{"id":1,"login":"alice","login_name":"","source_id":0,"full_name":"","email":"alice@example.com","avatar_url":"https://forgejo.example/avatars/c160f8cc69a4f0bf2b0362752353d060","html_url":"https://forgejo.example/alice","language":"","is_admin":true,"last_login":"1970-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":true,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"alice"},"content":"heart","created_at":"2026-09-25T01:13:56Z"},{"user":{"id":1,"login":"alice","login_name":"","source_id":0,"full_name":"","email":"alice@example.com","avatar_url":"https://forgejo.example/avatars/c160f8cc69a4f0bf2b0362752353d060","html_url":"https://forgejo.example/alice","language":"","is_admin":true,"last_login":"1970-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":true,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"alice"},"content":"+1","created_at":"2026-09-25T01:13:56Z"}]
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
Access-Control-Expose-Headers: X-Total-Count
|
||||
Cache-Control: max-age=0, private, must-revalidate
|
||||
Content-Length: 5
|
||||
Content-Type: application/json;charset=utf-8
|
||||
X-Content-Type-Options: nosniff
|
||||
X-Frame-Options: SAMEORIGIN
|
||||
X-Total-Count: 0
|
||||
|
||||
null
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
Access-Control-Expose-Headers: X-Total-Count
|
||||
Cache-Control: max-age=0, private, must-revalidate
|
||||
Content-Length: 5
|
||||
Content-Type: application/json;charset=utf-8
|
||||
X-Content-Type-Options: nosniff
|
||||
X-Frame-Options: SAMEORIGIN
|
||||
X-Total-Count: 0
|
||||
|
||||
null
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
Access-Control-Expose-Headers: X-Total-Count
|
||||
Cache-Control: max-age=0, private, must-revalidate
|
||||
Content-Length: 5
|
||||
Content-Type: application/json;charset=utf-8
|
||||
X-Content-Type-Options: nosniff
|
||||
X-Frame-Options: SAMEORIGIN
|
||||
X-Total-Count: 0
|
||||
|
||||
null
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
Cache-Control: max-age=0, private, must-revalidate
|
||||
Content-Length: 604
|
||||
Content-Type: application/json;charset=utf-8
|
||||
X-Content-Type-Options: nosniff
|
||||
X-Frame-Options: SAMEORIGIN
|
||||
|
||||
[{"user":{"id":1,"login":"alice","login_name":"","source_id":0,"full_name":"","email":"alice@example.com","avatar_url":"https://forgejo.example/avatars/c160f8cc69a4f0bf2b0362752353d060","html_url":"https://forgejo.example/alice","language":"","is_admin":true,"last_login":"1970-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":true,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"alice"},"content":"laugh","created_at":"2026-09-25T01:13:56Z"}]
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
Access-Control-Expose-Headers: X-Total-Count
|
||||
Cache-Control: max-age=0, private, must-revalidate
|
||||
Content-Length: 1518
|
||||
Content-Type: application/json;charset=utf-8
|
||||
X-Content-Type-Options: nosniff
|
||||
X-Frame-Options: SAMEORIGIN
|
||||
X-Total-Count: 1
|
||||
|
||||
[{"id":1,"url":"https://forgejo.example/api/v1/repos/alice/test_repo/issues/1","html_url":"https://forgejo.example/alice/test_repo/issues/1","number":1,"user":{"id":1,"login":"alice","login_name":"","source_id":0,"full_name":"","email":"alice@example.com","avatar_url":"https://forgejo.example/avatars/c160f8cc69a4f0bf2b0362752353d060","html_url":"https://forgejo.example/alice","language":"","is_admin":true,"last_login":"1970-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":true,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"alice"},"original_author":"","original_author_id":0,"title":"first issue","body":"issue body","ref":"","assets":[],"labels":[{"id":1,"name":"kind/bug","exclusive":true,"is_archived":false,"color":"ee0701","description":"Something is broken","url":"https://forgejo.example/api/v1/repos/alice/test_repo/labels/1"}],"milestone":{"id":1,"title":"v1","description":"first","state":"open","open_issues":1,"closed_issues":0,"created_at":"2026-09-25T01:13:54Z","updated_at":"2026-09-25T01:13:56Z","closed_at":null,"due_on":null},"assignee":null,"assignees":null,"state":"open","is_locked":false,"comments":1,"created_at":"2026-09-25T01:13:56Z","updated_at":"2026-09-25T01:13:56Z","closed_at":null,"due_date":null,"pull_request":null,"repository":{"id":1,"name":"test_repo","owner":"alice","full_name":"alice/test_repo"},"pin_order":0}]
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
Access-Control-Expose-Headers: X-Total-Count
|
||||
Cache-Control: max-age=0, private, must-revalidate
|
||||
Content-Length: 529
|
||||
Content-Type: application/json;charset=utf-8
|
||||
X-Content-Type-Options: nosniff
|
||||
X-Frame-Options: SAMEORIGIN
|
||||
X-Total-Count: 3
|
||||
|
||||
[{"id":1,"name":"kind/bug","exclusive":true,"is_archived":false,"color":"ee0701","description":"Something is broken","url":"https://forgejo.example/api/v1/repos/alice/test_repo/labels/1"},{"id":2,"name":"kind/feature","exclusive":true,"is_archived":true,"color":"0e8a16","description":"","url":"https://forgejo.example/api/v1/repos/alice/test_repo/labels/2"},{"id":3,"name":"question","exclusive":false,"is_archived":false,"color":"cc317c","description":"","url":"https://forgejo.example/api/v1/repos/alice/test_repo/labels/3"}]
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
Access-Control-Expose-Headers: X-Total-Count
|
||||
Cache-Control: max-age=0, private, must-revalidate
|
||||
Content-Length: 3
|
||||
Content-Type: application/json;charset=utf-8
|
||||
X-Content-Type-Options: nosniff
|
||||
X-Frame-Options: SAMEORIGIN
|
||||
X-Total-Count: 3
|
||||
|
||||
[]
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
Cache-Control: max-age=0, private, must-revalidate
|
||||
Content-Length: 1023
|
||||
Content-Type: application/json;charset=utf-8
|
||||
X-Content-Type-Options: nosniff
|
||||
X-Frame-Options: SAMEORIGIN
|
||||
|
||||
[{"id":10,"body":"fix this","user":{"id":2,"login":"bob","login_name":"","source_id":0,"full_name":"","email":"bob@example.com","avatar_url":"https://forgejo.example/avatars/4b9bb80620f03eb3719e0a061c14283d","html_url":"https://forgejo.example/bob","language":"","is_admin":false,"last_login":"1970-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":true,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"bob"},"resolver":null,"pull_request_review_id":2,"created_at":"2026-09-25T01:14:04Z","updated_at":"2026-09-25T01:14:04Z","path":"open.txt","commit_id":"0310ab0fc7512cab1652c77eadb401e0b15fb974","original_commit_id":"","diff_hunk":"@@ -0,0 +1 @@\n+open.txt","position":1,"original_position":0,"extra_lines_count":0,"html_url":"https://forgejo.example/alice/test_repo/pulls/3#issuecomment-10","pull_request_url":"https://forgejo.example/alice/test_repo/pulls/3"}]
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
Access-Control-Expose-Headers: X-Total-Count
|
||||
Cache-Control: max-age=0, private, must-revalidate
|
||||
Content-Length: 1894
|
||||
Content-Type: application/json;charset=utf-8
|
||||
X-Content-Type-Options: nosniff
|
||||
X-Frame-Options: SAMEORIGIN
|
||||
X-Total-Count: 2
|
||||
|
||||
[{"id":2,"user":{"id":2,"login":"bob","login_name":"","source_id":0,"full_name":"","email":"bob@example.com","avatar_url":"https://forgejo.example/avatars/4b9bb80620f03eb3719e0a061c14283d","html_url":"https://forgejo.example/bob","language":"","is_admin":false,"last_login":"1970-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":true,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"bob"},"team":null,"state":"REQUEST_CHANGES","body":"needs work","commit_id":"0310ab0fc7512cab1652c77eadb401e0b15fb974","stale":false,"official":true,"dismissed":true,"comments_count":1,"submitted_at":"2026-09-25T01:14:04Z","updated_at":"2026-09-25T01:14:05Z","html_url":"https://forgejo.example/alice/test_repo/pulls/3#issuecomment-11","pull_request_url":"https://forgejo.example/alice/test_repo/pulls/3"},{"id":3,"user":{"id":2,"login":"bob","login_name":"","source_id":0,"full_name":"","email":"bob@example.com","avatar_url":"https://forgejo.example/avatars/4b9bb80620f03eb3719e0a061c14283d","html_url":"https://forgejo.example/bob","language":"","is_admin":false,"last_login":"1970-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":true,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"bob"},"team":null,"state":"COMMENT","body":"just a comment","commit_id":"0310ab0fc7512cab1652c77eadb401e0b15fb974","stale":false,"official":false,"dismissed":false,"comments_count":0,"submitted_at":"2026-09-25T01:14:05Z","updated_at":"2026-09-25T01:14:05Z","html_url":"https://forgejo.example/alice/test_repo/pulls/3#issuecomment-13","pull_request_url":"https://forgejo.example/alice/test_repo/pulls/3"}]
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
Access-Control-Expose-Headers: X-Total-Count
|
||||
Cache-Control: max-age=0, private, must-revalidate
|
||||
Content-Type: application/json;charset=utf-8
|
||||
X-Content-Type-Options: nosniff
|
||||
X-Frame-Options: SAMEORIGIN
|
||||
X-Total-Count: 3
|
||||
|
||||
[{"id":3,"url":"https://forgejo.example/api/v1/repos/alice/test_repo/pulls/4","number":4,"user":{"id":2,"login":"bob","login_name":"","source_id":0,"full_name":"","email":"bob@example.com","avatar_url":"https://forgejo.example/avatars/4b9bb80620f03eb3719e0a061c14283d","html_url":"https://forgejo.example/bob","language":"","is_admin":false,"last_login":"1970-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":true,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"bob"},"title":"agit pull","body":"agit body","labels":[],"milestone":null,"assignee":null,"assignees":null,"requested_reviewers":[],"requested_reviewers_teams":[],"state":"open","draft":false,"is_locked":false,"comments":0,"review_comments":0,"additions":0,"deletions":0,"changed_files":0,"html_url":"https://forgejo.example/alice/test_repo/pulls/4","diff_url":"https://forgejo.example/alice/test_repo/pulls/4.diff","patch_url":"https://forgejo.example/alice/test_repo/pulls/4.patch","mergeable":true,"merged":false,"merged_at":null,"merge_commit_sha":null,"merged_by":null,"allow_maintainer_edit":false,"base":{"label":"main","ref":"main","sha":"588ba39bdea707266f2df950dcee1bb5a1764fed","repo_id":1,"repo":{"id":1,"owner":{"id":1,"login":"alice","login_name":"","source_id":0,"full_name":"","email":"alice@example.com","avatar_url":"https://forgejo.example/avatars/c160f8cc69a4f0bf2b0362752353d060","html_url":"https://forgejo.example/alice","language":"","is_admin":false,"last_login":"0001-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":false,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"alice"},"name":"test_repo","full_name":"alice/test_repo","description":"Test repository for testing migration from forgejo to gitea","empty":false,"private":true,"fork":false,"template":false,"parent":null,"mirror":false,"size":3,"language":"","languages_url":"https://forgejo.example/api/v1/repos/alice/test_repo/languages","html_url":"https://forgejo.example/alice/test_repo","url":"https://forgejo.example/api/v1/repos/alice/test_repo","link":"","ssh_url":"ssh://git@forgejo.example/alice/test_repo.git","clone_url":"https://forgejo.example/alice/test_repo.git","original_url":"","website":"https://forgejo.example/website","stars_count":0,"forks_count":0,"watchers_count":1,"open_issues_count":1,"open_pr_counter":2,"release_counter":2,"default_branch":"main","archived":false,"created_at":"2026-09-25T01:13:52Z","updated_at":"2026-09-25T01:14:07Z","archived_at":"1970-01-01T00:00:00Z","permissions":{"admin":true,"push":true,"pull":true},"has_issues":true,"internal_tracker":{"enable_time_tracker":true,"allow_only_contributors_to_track_time":true,"enable_issue_dependencies":true},"has_wiki":true,"has_wiki_contents":false,"wiki_branch":"main","wiki_ssh_url":"ssh://git@forgejo.example/alice/test_repo.wiki.git","wiki_clone_url":"https://forgejo.example/alice/test_repo.wiki.git","globally_editable_wiki":false,"has_pull_requests":true,"has_projects":true,"has_releases":true,"has_packages":true,"has_actions":true,"ignore_whitespace_conflicts":false,"allow_merge_commits":true,"allow_rebase":true,"allow_rebase_explicit":true,"allow_squash_merge":true,"allow_fast_forward_only_merge":true,"allow_rebase_update":true,"default_delete_branch_after_merge":false,"default_merge_style":"merge","default_allow_maintainer_edit":false,"default_update_style":"merge","avatar_url":"","internal":false,"mirror_interval":"","object_format_name":"sha1","mirror_updated":"0001-01-01T00:00:00Z","repo_transfer":null,"topics":["forgejo","migration"]}},"head":{"label":"bob/agit","ref":"refs/pull/4/head","sha":"171139b0df47db58a099ff2643ac1088776ff13c","repo_id":1,"repo":{"id":1,"owner":{"id":1,"login":"alice","login_name":"","source_id":0,"full_name":"","email":"alice@example.com","avatar_url":"https://forgejo.example/avatars/c160f8cc69a4f0bf2b0362752353d060","html_url":"https://forgejo.example/alice","language":"","is_admin":false,"last_login":"0001-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":false,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"alice"},"name":"test_repo","full_name":"alice/test_repo","description":"Test repository for testing migration from forgejo to gitea","empty":false,"private":true,"fork":false,"template":false,"parent":null,"mirror":false,"size":3,"language":"","languages_url":"https://forgejo.example/api/v1/repos/alice/test_repo/languages","html_url":"https://forgejo.example/alice/test_repo","url":"https://forgejo.example/api/v1/repos/alice/test_repo","link":"","ssh_url":"ssh://git@forgejo.example/alice/test_repo.git","cLine truncated
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
Access-Control-Expose-Headers: X-Total-Count
|
||||
Cache-Control: max-age=0, private, must-revalidate
|
||||
Content-Type: application/json;charset=utf-8
|
||||
X-Content-Type-Options: nosniff
|
||||
X-Frame-Options: SAMEORIGIN
|
||||
X-Total-Count: 2
|
||||
|
||||
[{"id":2,"tag_name":"v0.9.0","target_commitish":"main","name":"v0.9.0","body":"","url":"https://forgejo.example/api/v1/repos/alice/test_repo/releases/2","html_url":"https://forgejo.example/alice/test_repo/releases/tag/v0.9.0","tarball_url":"https://forgejo.example/alice/test_repo/archive/v0.9.0.tar.gz","zipball_url":"https://forgejo.example/alice/test_repo/archive/v0.9.0.zip","hide_archive_links":false,"upload_url":"https://forgejo.example/api/v1/repos/alice/test_repo/releases/2/assets","draft":false,"prerelease":true,"created_at":"2026-09-25T01:13:55Z","published_at":"2026-09-25T01:13:55Z","author":{"id":1,"login":"alice","login_name":"","source_id":0,"full_name":"","email":"alice@noreply.forgejo.example","avatar_url":"https://forgejo.example/avatars/c160f8cc69a4f0bf2b0362752353d060","html_url":"https://forgejo.example/alice","language":"","is_admin":false,"last_login":"0001-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":false,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"alice"},"assets":[],"archive_download_count":{"zip":0,"tar_gz":0}},{"id":1,"tag_name":"v1.0.0","target_commitish":"main","name":"v1.0.0","body":"first release","url":"https://forgejo.example/api/v1/repos/alice/test_repo/releases/1","html_url":"https://forgejo.example/alice/test_repo/releases/tag/v1.0.0","tarball_url":"https://forgejo.example/alice/test_repo/archive/v1.0.0.tar.gz","zipball_url":"https://forgejo.example/alice/test_repo/archive/v1.0.0.zip","hide_archive_links":false,"upload_url":"https://forgejo.example/api/v1/repos/alice/test_repo/releases/1/assets","draft":false,"prerelease":false,"created_at":"2026-09-25T01:13:55Z","published_at":"2026-09-25T01:13:55Z","author":{"id":1,"login":"alice","login_name":"","source_id":0,"full_name":"","email":"alice@noreply.forgejo.example","avatar_url":"https://forgejo.example/avatars/c160f8cc69a4f0bf2b0362752353d060","html_url":"https://forgejo.example/alice","language":"","is_admin":false,"last_login":"0001-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":false,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"alice"},"assets":[{"id":1,"name":"asset.txt","size":14,"download_count":0,"created_at":"2026-09-25T01:13:55Z","uuid":"4e5c9b2d-b858-48af-9ba7-8260f1b53a04","browser_download_url":"https://forgejo.example/alice/test_repo/releases/download/v1.0.0/asset.txt","type":"attachment"},{"id":2,"name":"download.zip","size":0,"download_count":0,"created_at":"2026-09-25T01:13:55Z","uuid":"5e8b7425-31bb-4103-808b-1907aacaddcb","browser_download_url":"https://example.com/download.zip","type":"external"}],"archive_download_count":{"zip":0,"tar_gz":0}}]
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
Cache-Control: max-age=0, private, must-revalidate
|
||||
Content-Length: 115
|
||||
Content-Type: application/json;charset=utf-8
|
||||
X-Content-Type-Options: nosniff
|
||||
X-Frame-Options: SAMEORIGIN
|
||||
|
||||
{"max_response_items":3,"default_paging_num":3,"default_git_trees_per_page":1000,"default_max_blob_size":10485760}
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
Cache-Control: max-age=0, private, must-revalidate
|
||||
Content-Length: 53
|
||||
Content-Type: application/json;charset=utf-8
|
||||
X-Content-Type-Options: nosniff
|
||||
X-Frame-Options: SAMEORIGIN
|
||||
|
||||
{"version":"17.0.0-dev-576-994e539b55+gitea-1.22.0"}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
Accept-Ranges: bytes
|
||||
Access-Control-Expose-Headers: Content-Disposition
|
||||
Cache-Control: private, max-age=300
|
||||
Content-Disposition: inline; filename="asset.txt"; filename*=UTF-8''asset.txt
|
||||
Content-Length: 14
|
||||
Content-Type: text/plain; charset=utf-8
|
||||
Etag: "4e5c9b2d-b858-48af-9ba7-8260f1b53a04"
|
||||
Last-Modified: Fri, 25 Sep 2026 01:13:55 GMT
|
||||
X-Content-Type-Options: nosniff
|
||||
X-Frame-Options: SAMEORIGIN
|
||||
|
||||
forgejo asset
|
||||
-8
@@ -1,8 +0,0 @@
|
||||
Alt-Svc: h3=":443"; ma=2592000
|
||||
Cache-Control: max-age=0, private, must-revalidate, no-transform
|
||||
Content-Length: 2
|
||||
Content-Type: application/json;charset=utf-8
|
||||
Vary: Origin
|
||||
X-Content-Type-Options: nosniff
|
||||
|
||||
[]
|
||||
-8
@@ -1,8 +0,0 @@
|
||||
Alt-Svc: h3=":443"; ma=2592000
|
||||
Cache-Control: max-age=0, private, must-revalidate, no-transform
|
||||
Content-Length: 2
|
||||
Content-Type: application/json;charset=utf-8
|
||||
Vary: Origin
|
||||
X-Content-Type-Options: nosniff
|
||||
|
||||
[]
|
||||
@@ -113,7 +113,7 @@ type BitbucketServerDownloader struct {
|
||||
// NewBitbucketServerDownloader creates a Bitbucket Server downloader
|
||||
func NewBitbucketServerDownloader(_ context.Context, baseURL, username, password, token, projectKey, repoSlug string) *BitbucketServerDownloader {
|
||||
return &BitbucketServerDownloader{
|
||||
client: getMigrationHTTPClient(),
|
||||
client: &http.Client{Transport: getMigrationTransport()}, // not retrying, callAPI retries rate limits itself
|
||||
sleep: bitbucketServerSleep,
|
||||
baseURL: strings.TrimSuffix(baseURL, "/"),
|
||||
projectKey: projectKey,
|
||||
|
||||
@@ -78,22 +78,14 @@ type CodebaseDownloader struct {
|
||||
// NewCodebaseDownloader creates a new downloader
|
||||
func NewCodebaseDownloader(_ context.Context, projectURL *url.URL, project, repoName, username, password string) *CodebaseDownloader {
|
||||
baseURL, _ := url.Parse("https://api3.codebasehq.com")
|
||||
transport := NewMigrationHTTPTransport()
|
||||
|
||||
downloader := &CodebaseDownloader{
|
||||
baseURL: baseURL,
|
||||
projectURL: projectURL,
|
||||
project: project,
|
||||
repoName: repoName,
|
||||
client: &http.Client{
|
||||
Transport: roundTripperFunc(func(req *http.Request) (*http.Response, error) {
|
||||
if username != "" && password != "" {
|
||||
req = req.Clone(req.Context())
|
||||
req.SetBasicAuth(username, password)
|
||||
}
|
||||
return transport.RoundTrip(req)
|
||||
}),
|
||||
},
|
||||
|
||||
client: newMigrationHTTPClient(baseURL.String(), basicAuthorization(username, password)),
|
||||
userMap: make(map[int64]*codebaseUser),
|
||||
commitMap: make(map[string]string),
|
||||
}
|
||||
|
||||
@@ -4,7 +4,10 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
system_model "gitea.dev/models/system"
|
||||
@@ -21,6 +24,22 @@ func WarnAndNotice(fmtStr string, args ...any) {
|
||||
}
|
||||
}
|
||||
|
||||
func downloadAsset(ctx context.Context, client *http.Client, assetURL string) (io.ReadCloser, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, assetURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("unexpected status %s", resp.Status)
|
||||
}
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
func hasBaseURL(toCheck, baseURL string) bool {
|
||||
if baseURL == "" {
|
||||
return false
|
||||
|
||||
+22
-47
@@ -8,7 +8,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
@@ -87,22 +86,6 @@ func (g *RepositoryDumper) reviewDir() string {
|
||||
return filepath.Join(g.baseDir, "reviews")
|
||||
}
|
||||
|
||||
func (g *RepositoryDumper) setURLToken(remoteAddr string) (string, error) {
|
||||
if len(g.opts.AuthToken) > 0 || len(g.opts.AuthUsername) > 0 {
|
||||
u, err := url.Parse(remoteAddr)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
u.User = url.UserPassword(g.opts.AuthUsername, g.opts.AuthPassword)
|
||||
if len(g.opts.AuthToken) > 0 {
|
||||
u.User = url.UserPassword("oauth2", g.opts.AuthToken)
|
||||
}
|
||||
remoteAddr = u.String()
|
||||
}
|
||||
|
||||
return remoteAddr, nil
|
||||
}
|
||||
|
||||
// CreateRepo creates a repository
|
||||
func (g *RepositoryDumper) CreateRepo(ctx context.Context, repo *base.Repository, opts base.MigrateOptions) error {
|
||||
f, err := os.Create(filepath.Join(g.baseDir, "repo.yml"))
|
||||
@@ -147,7 +130,7 @@ func (g *RepositoryDumper) CreateRepo(ctx context.Context, repo *base.Repository
|
||||
|
||||
migrateTimeout := 2 * time.Hour
|
||||
|
||||
remoteAddr, err := g.setURLToken(repo.CloneURL)
|
||||
remoteAddr, err := base.NullDownloader{}.FormatCloneURL(g.opts, repo.CloneURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -294,41 +277,31 @@ func (g *RepositoryDumper) CreateLabels(_ context.Context, labels ...*base.Label
|
||||
}
|
||||
|
||||
// CreateReleases creates releases
|
||||
func (g *RepositoryDumper) CreateReleases(_ context.Context, releases ...*base.Release) error {
|
||||
func (g *RepositoryDumper) CreateReleases(ctx context.Context, releases ...*base.Release) error {
|
||||
if g.opts.ReleaseAssets {
|
||||
for _, release := range releases {
|
||||
attachDir := filepath.Join("release_assets", uuid.New().String())
|
||||
if err := os.MkdirAll(filepath.Join(g.baseDir, attachDir), os.ModePerm); err != nil {
|
||||
return err
|
||||
}
|
||||
assets := release.Assets[:0]
|
||||
for _, asset := range release.Assets {
|
||||
// SECURITY: We cannot check the DownloadFunc is safe here
|
||||
// ... we must assume that it is safe and simply download the attachment
|
||||
rc, err := asset.DownloadFunc()
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
WarnAndNotice("Skipped asset %s of release %s: %v", asset.Name, release.TagName, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// we cannot use asset.Name because it might contains special characters.
|
||||
attachLocalPath := filepath.Join(attachDir, uuid.New().String())
|
||||
|
||||
// SECURITY: We cannot check the DownloadURL and DownloadFunc are safe here
|
||||
// ... we must assume that they are safe and simply download the attachment
|
||||
// download attachment
|
||||
err := func(attachPath string) error {
|
||||
var rc io.ReadCloser
|
||||
var err error
|
||||
if asset.DownloadURL == nil {
|
||||
rc, err = asset.DownloadFunc()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rc.Close()
|
||||
} else {
|
||||
// use the migration client so the fetch (including any redirect) is
|
||||
// validated against the migration host allow/block list
|
||||
resp, err := getMigrationHTTPClient().Get(*asset.DownloadURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
rc = resp.Body
|
||||
}
|
||||
|
||||
fw, err := os.Create(attachPath)
|
||||
err = func() error {
|
||||
defer rc.Close()
|
||||
fw, err := os.Create(filepath.Join(g.baseDir, attachLocalPath))
|
||||
if err != nil {
|
||||
return fmt.Errorf("create: %w", err)
|
||||
}
|
||||
@@ -336,12 +309,14 @@ func (g *RepositoryDumper) CreateReleases(_ context.Context, releases ...*base.R
|
||||
|
||||
_, err = io.Copy(fw, rc)
|
||||
return err
|
||||
}(filepath.Join(g.baseDir, attachLocalPath))
|
||||
}()
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("release %s asset %s: %w", release.TagName, asset.Name, err)
|
||||
}
|
||||
asset.DownloadURL = &attachLocalPath // to save the filepath on the yml file, change the source
|
||||
assets = append(assets, asset)
|
||||
}
|
||||
release.Assets = assets
|
||||
}
|
||||
}
|
||||
|
||||
@@ -454,7 +429,7 @@ func (g *RepositoryDumper) handlePullRequest(ctx context.Context, pr *base.PullR
|
||||
|
||||
// SECURITY: We will assume that the pr.PatchURL has been checked
|
||||
// pr.PatchURL maybe a local file - but note EnsureSafe should be asserting that this safe
|
||||
u, err := g.setURLToken(pr.PatchURL)
|
||||
u, err := base.NullDownloader{}.FormatCloneURL(g.opts, pr.PatchURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package migrations
|
||||
|
||||
import "gitea.dev/modules/structs"
|
||||
|
||||
func init() {
|
||||
RegisterDownloaderFactory(&ForgejoDownloaderFactory{})
|
||||
}
|
||||
|
||||
// ForgejoDownloaderFactory uses the Gitea downloader, as Forgejo keeps a Gitea compatible API
|
||||
type ForgejoDownloaderFactory struct {
|
||||
GiteaDownloaderFactory
|
||||
}
|
||||
|
||||
func (f *ForgejoDownloaderFactory) GitServiceType() structs.GitServiceType {
|
||||
return structs.ForgejoService
|
||||
}
|
||||
@@ -4,6 +4,7 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -17,7 +18,10 @@ import (
|
||||
"gitea.dev/modules/log"
|
||||
base "gitea.dev/modules/migration"
|
||||
"gitea.dev/modules/structs"
|
||||
"gitea.dev/modules/util"
|
||||
gitea_sdk "gitea.dev/sdk"
|
||||
|
||||
"github.com/hashicorp/go-version"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -72,6 +76,7 @@ type GiteaDownloader struct {
|
||||
baseURL string
|
||||
repoOwner string
|
||||
repoName string
|
||||
httpClient *http.Client
|
||||
pagination bool
|
||||
maxPerPage int
|
||||
}
|
||||
@@ -81,16 +86,15 @@ type GiteaDownloader struct {
|
||||
// Use either a username/password or personal token. token is preferred
|
||||
// Note: Public access only allows very basic access
|
||||
func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, password, token string) (*GiteaDownloader, error) {
|
||||
giteaClient, err := gitea_sdk.NewClient(
|
||||
baseURL,
|
||||
gitea_sdk.SetToken(token),
|
||||
gitea_sdk.SetBasicAuth(username, password),
|
||||
gitea_sdk.SetHTTPClient(newMigrationHTTPClient()),
|
||||
)
|
||||
httpClient := newMigrationHTTPClient(baseURL, util.Iif(token != "", "token "+token, basicAuthorization(username, password)))
|
||||
giteaClient, err := gitea_sdk.NewClient(baseURL, gitea_sdk.SetHTTPClient(httpClient))
|
||||
if err != nil {
|
||||
log.Error(fmt.Sprintf("Failed to create NewGiteaDownloader for: %s. Error: %v", baseURL, err))
|
||||
return nil, err
|
||||
}
|
||||
if err = pinGiteaVersion(ctx, giteaClient); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
path := strings.Split(repoPath, "/")
|
||||
|
||||
@@ -117,11 +121,36 @@ func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, passwo
|
||||
baseURL: baseURL,
|
||||
repoOwner: path[0],
|
||||
repoName: path[1],
|
||||
httpClient: httpClient,
|
||||
pagination: paginationSupport,
|
||||
maxPerPage: maxPerPage,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func pinGiteaVersion(ctx context.Context, client *gitea_sdk.Client) error { // the SDK's checks reject pre-releases
|
||||
rawVersion, _, err := client.Meta.ServerVersion(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if serverVersion, err := version.NewVersion(rawVersion); err == nil {
|
||||
if giteaVersion, ok := strings.CutPrefix(serverVersion.Metadata(), "gitea-"); ok { // Forgejo
|
||||
if serverVersion, err = version.NewVersion(giteaVersion); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return gitea_sdk.SetGiteaVersion(serverVersion.Core().String())(client)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *GiteaDownloader) unitDisabled(resp *gitea_sdk.Response, hasUnit func(*gitea_sdk.Repository) bool) bool { // Gitea answers a disabled unit and missing access alike with 403 or 404, Forgejo with 404
|
||||
if resp == nil || (resp.StatusCode != http.StatusNotFound && resp.StatusCode != http.StatusForbidden) {
|
||||
return false
|
||||
}
|
||||
repo, _, err := g.client.Repositories.GetRepo(g.ctx, g.repoOwner, g.repoName)
|
||||
return err == nil && !hasUnit(repo)
|
||||
}
|
||||
|
||||
// String implements Stringer
|
||||
func (g *GiteaDownloader) String() string {
|
||||
return fmt.Sprintf("migration from gitea server %s %s/%s", g.baseURL, g.repoOwner, g.repoName)
|
||||
@@ -225,6 +254,8 @@ func (g *GiteaDownloader) convertGiteaLabel(label *gitea_sdk.Label) *base.Label
|
||||
Name: label.Name,
|
||||
Color: label.Color,
|
||||
Description: label.Description,
|
||||
Exclusive: label.Exclusive,
|
||||
Archived: label.IsArchived,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -273,11 +304,7 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele
|
||||
Created: rel.CreatedAt,
|
||||
}
|
||||
|
||||
httpClient := newMigrationHTTPClient()
|
||||
|
||||
for _, asset := range rel.Attachments {
|
||||
assetID := asset.ID // Don't optimize this, for closure we need a local variable
|
||||
assetDownloadURL := asset.DownloadURL
|
||||
size := int(asset.Size)
|
||||
dlCount := int(asset.DownloadCount)
|
||||
r.Assets = append(r.Assets, &base.ReleaseAsset{
|
||||
@@ -286,30 +313,8 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele
|
||||
Size: &size,
|
||||
DownloadCount: &dlCount,
|
||||
Created: asset.Created,
|
||||
DownloadURL: &asset.DownloadURL,
|
||||
DownloadFunc: func() (io.ReadCloser, error) {
|
||||
asset, _, err := g.client.Releases.GetReleaseAttachment(g.ctx, g.repoOwner, g.repoName, rel.ID, assetID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !hasBaseURL(assetDownloadURL, g.baseURL) {
|
||||
WarnAndNotice("Unexpected AssetURL for assetID[%d] in %s: %s", assetID, g, assetDownloadURL)
|
||||
return io.NopCloser(strings.NewReader(asset.DownloadURL)), nil
|
||||
}
|
||||
|
||||
// FIXME: for a private download?
|
||||
req, err := http.NewRequest(http.MethodGet, assetDownloadURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// resp.Body is closed by the uploader
|
||||
return resp.Body, nil
|
||||
return downloadAsset(g.ctx, g.httpClient, g.baseURL+"/attachments/"+url.PathEscape(asset.UUID))
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -328,10 +333,13 @@ func (g *GiteaDownloader) GetReleases(ctx context.Context) ([]*base.Release, err
|
||||
default:
|
||||
}
|
||||
|
||||
rl, _, err := g.client.Releases.ListReleases(g.ctx, g.repoOwner, g.repoName, gitea_sdk.ListReleasesOptions{ListOptions: gitea_sdk.ListOptions{
|
||||
rl, resp, err := g.client.Releases.ListReleases(g.ctx, g.repoOwner, g.repoName, gitea_sdk.ListReleasesOptions{ListOptions: gitea_sdk.ListOptions{
|
||||
PageSize: g.maxPerPage,
|
||||
Page: i,
|
||||
}})
|
||||
if g.unitDisabled(resp, func(repo *gitea_sdk.Repository) bool { return repo.HasReleases }) {
|
||||
return releases, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -413,11 +421,14 @@ func (g *GiteaDownloader) GetIssues(ctx context.Context, page, perPage int) ([]*
|
||||
}
|
||||
allIssues := make([]*base.Issue, 0, perPage)
|
||||
|
||||
issues, _, err := g.client.Issues.ListRepoIssues(g.ctx, g.repoOwner, g.repoName, gitea_sdk.ListIssueOption{
|
||||
issues, resp, err := g.client.Issues.ListRepoIssues(g.ctx, g.repoOwner, g.repoName, gitea_sdk.ListIssueOption{
|
||||
ListOptions: gitea_sdk.ListOptions{Page: page, PageSize: perPage},
|
||||
State: gitea_sdk.StateAll,
|
||||
Type: gitea_sdk.IssueTypeIssue,
|
||||
})
|
||||
if g.unitDisabled(resp, func(repo *gitea_sdk.Repository) bool { return repo.HasIssues && repo.ExternalTracker == nil }) {
|
||||
return nil, true, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("error while listing issues: %w", err)
|
||||
}
|
||||
@@ -459,6 +470,7 @@ func (g *GiteaDownloader) GetIssues(ctx context.Context, page, perPage int) ([]*
|
||||
Assignees: assignees,
|
||||
IsLocked: issue.IsLocked,
|
||||
ForeignIndex: issue.Index,
|
||||
Context: issue.Comments,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -471,6 +483,9 @@ func (g *GiteaDownloader) GetIssues(ctx context.Context, page, perPage int) ([]*
|
||||
|
||||
// GetComments returns comments according issueNumber
|
||||
func (g *GiteaDownloader) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) {
|
||||
if commentable.GetContext() == 0 {
|
||||
return nil, true, nil
|
||||
}
|
||||
allComments := make([]*base.Comment, 0, g.maxPerPage)
|
||||
seenIDs := container.Set[int64]{}
|
||||
|
||||
@@ -527,13 +542,16 @@ func (g *GiteaDownloader) GetPullRequests(ctx context.Context, page, perPage int
|
||||
}
|
||||
allPRs := make([]*base.PullRequest, 0, perPage)
|
||||
|
||||
prs, _, err := g.client.PullRequests.ListRepoPullRequests(g.ctx, g.repoOwner, g.repoName, gitea_sdk.ListPullRequestsOptions{
|
||||
prs, resp, err := g.client.PullRequests.ListRepoPullRequests(g.ctx, g.repoOwner, g.repoName, gitea_sdk.ListPullRequestsOptions{
|
||||
ListOptions: gitea_sdk.ListOptions{
|
||||
Page: page,
|
||||
PageSize: perPage,
|
||||
},
|
||||
State: gitea_sdk.StateAll,
|
||||
})
|
||||
if g.unitDisabled(resp, func(repo *gitea_sdk.Repository) bool { return repo.HasPullRequests }) {
|
||||
return nil, true, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("error while listing pull requests (page: %d, pagesize: %d). Error: %w", page, perPage, err)
|
||||
}
|
||||
@@ -585,7 +603,7 @@ func (g *GiteaDownloader) GetPullRequests(ctx context.Context, page, perPage int
|
||||
createdAt = *pr.Created
|
||||
}
|
||||
updatedAt := time.Time{}
|
||||
if pr.Created != nil {
|
||||
if pr.Updated != nil {
|
||||
updatedAt = *pr.Updated
|
||||
}
|
||||
|
||||
@@ -613,7 +631,6 @@ func (g *GiteaDownloader) GetPullRequests(ctx context.Context, page, perPage int
|
||||
MergedTime: pr.Merged,
|
||||
MergeCommitSHA: mergeCommitSHA,
|
||||
IsLocked: pr.IsLocked,
|
||||
PatchURL: pr.PatchURL,
|
||||
Head: base.PullRequestBranch{
|
||||
Ref: headRef,
|
||||
SHA: headSHA,
|
||||
@@ -623,11 +640,12 @@ func (g *GiteaDownloader) GetPullRequests(ctx context.Context, page, perPage int
|
||||
},
|
||||
Base: base.PullRequestBranch{
|
||||
Ref: pr.Base.Ref,
|
||||
SHA: pr.Base.Sha,
|
||||
SHA: cmp.Or(pr.MergeBase, pr.Base.Sha), // base.sha is the base branch head, which contains the changes once merged
|
||||
RepoName: g.repoName,
|
||||
OwnerName: g.repoOwner,
|
||||
},
|
||||
ForeignIndex: pr.Index,
|
||||
Context: pr.Comments,
|
||||
})
|
||||
// SECURITY: Ensure that the PR is safe
|
||||
_ = CheckAndEnsureSafePR(allPRs[len(allPRs)-1], g.baseURL, g)
|
||||
@@ -672,9 +690,11 @@ func (g *GiteaDownloader) GetReviews(ctx context.Context, reviewable base.Review
|
||||
continue
|
||||
}
|
||||
|
||||
rcl, _, err := g.client.PullRequests.ListPullReviewComments(g.ctx, g.repoOwner, g.repoName, reviewable.GetForeignIndex(), pr.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
var rcl []*gitea_sdk.PullReviewComment
|
||||
if pr.CodeCommentsCount > 0 {
|
||||
if rcl, _, err = g.client.PullRequests.ListPullReviewComments(g.ctx, g.repoOwner, g.repoName, reviewable.GetForeignIndex(), pr.ID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var reviewComments []*base.ReviewComment
|
||||
for i := range rcl {
|
||||
@@ -706,8 +726,15 @@ func (g *GiteaDownloader) GetReviews(ctx context.Context, reviewable base.Review
|
||||
Content: pr.Body,
|
||||
CreatedAt: pr.Submitted,
|
||||
State: string(pr.State),
|
||||
Dismissed: pr.Dismissed,
|
||||
Comments: reviewComments,
|
||||
}
|
||||
switch pr.State {
|
||||
case gitea_sdk.ReviewStateComment:
|
||||
review.State = base.ReviewStateCommented
|
||||
case gitea_sdk.ReviewStateRequestChanges:
|
||||
review.State = base.ReviewStateChangesRequested
|
||||
}
|
||||
|
||||
allReviews = append(allReviews, review)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"gitea.dev/models/unittest"
|
||||
base "gitea.dev/modules/migration"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestForgejoDownloadRepo(t *testing.T) {
|
||||
token := os.Getenv("FORGEJO_TEST_TOKEN")
|
||||
|
||||
_, callerFile, _, _ := runtime.Caller(0)
|
||||
mockServer := unittest.NewMockWebServer(t, "http://localhost:3000", filepath.Join(filepath.Dir(callerFile), "_mock_data/TestForgejoDownloadRepo"), token != "")
|
||||
|
||||
ctx := t.Context()
|
||||
downloader, err := NewGiteaDownloader(ctx, mockServer.URL, "alice/test_repo", "", "", token)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, downloader.pagination)
|
||||
|
||||
repo, err := downloader.GetRepoInfo(ctx)
|
||||
require.NoError(t, err)
|
||||
assertRepositoryEqual(t, &base.Repository{
|
||||
Name: "test_repo",
|
||||
Owner: "alice",
|
||||
IsPrivate: true,
|
||||
Description: "Test repository for testing migration from forgejo to gitea",
|
||||
Website: "https://forgejo.example/website",
|
||||
CloneURL: "https://forgejo.example/alice/test_repo.git",
|
||||
OriginalURL: "https://forgejo.example/alice/test_repo",
|
||||
DefaultBranch: "main",
|
||||
}, repo)
|
||||
|
||||
labels, err := downloader.GetLabels(ctx)
|
||||
require.NoError(t, err)
|
||||
assertLabelsEqual(t, []*base.Label{
|
||||
{Name: "kind/bug", Color: "ee0701", Description: "Something is broken", Exclusive: true},
|
||||
{Name: "kind/feature", Color: "0e8a16", Exclusive: true, Archived: true},
|
||||
{Name: "question", Color: "cc317c"},
|
||||
}, labels)
|
||||
|
||||
releases, err := downloader.GetReleases(ctx)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, releases, 2)
|
||||
assert.Equal(t, "v0.9.0", releases[0].TagName)
|
||||
assert.True(t, releases[0].Prerelease)
|
||||
assert.Equal(t, "v1.0.0", releases[1].TagName)
|
||||
require.Len(t, releases[1].Assets, 2)
|
||||
rc, err := releases[1].Assets[0].DownloadFunc()
|
||||
require.NoError(t, err)
|
||||
content, err := io.ReadAll(rc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, rc.Close())
|
||||
assert.Equal(t, "forgejo asset\n", string(content))
|
||||
|
||||
issues, isEnd, err := downloader.GetIssues(ctx, 1, 50)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, isEnd)
|
||||
require.Len(t, issues, 1)
|
||||
assert.Equal(t, "first issue", issues[0].Title)
|
||||
assert.Equal(t, "v1", issues[0].Milestone)
|
||||
assert.Equal(t, []*base.Reaction{{UserID: 1, UserName: "alice", Content: "heart"}, {UserID: 1, UserName: "alice", Content: "+1"}}, issues[0].Reactions)
|
||||
|
||||
comments, _, err := downloader.GetComments(ctx, issues[0])
|
||||
require.NoError(t, err)
|
||||
require.Len(t, comments, 1)
|
||||
assert.Equal(t, "a comment", comments[0].Content)
|
||||
assert.Equal(t, []*base.Reaction{{UserID: 1, UserName: "alice", Content: "laugh"}}, comments[0].Reactions)
|
||||
|
||||
prs, _, err := downloader.GetPullRequests(ctx, 1, 50)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, prs, 3)
|
||||
agit, open, merged := prs[0], prs[1], prs[2]
|
||||
assert.Equal(t, "refs/pull/4/head", agit.Head.Ref)
|
||||
assert.Equal(t, "feature-open", open.Head.Ref)
|
||||
assert.True(t, merged.Merged)
|
||||
assert.Equal(t, "refs/pull/2/head", merged.Head.Ref)
|
||||
assert.Equal(t, "23181c5c8c25740ab7f080faf27cbcae7fd13519", merged.Base.SHA)
|
||||
assert.Equal(t, "588ba39bdea707266f2df950dcee1bb5a1764fed", merged.MergeCommitSHA)
|
||||
|
||||
comments, _, err = downloader.GetComments(ctx, agit)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, comments)
|
||||
|
||||
reviews, err := downloader.GetReviews(ctx, open)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, reviews, 2)
|
||||
assert.Equal(t, base.ReviewStateChangesRequested, reviews[0].State)
|
||||
assert.True(t, reviews[0].Dismissed)
|
||||
require.Len(t, reviews[0].Comments, 1)
|
||||
assert.Equal(t, "fix this", reviews[0].Comments[0].Content)
|
||||
assert.Equal(t, base.ReviewStateCommented, reviews[1].State)
|
||||
assert.False(t, reviews[1].Dismissed)
|
||||
}
|
||||
|
||||
func TestGiteaDownloaderServerVersion(t *testing.T) {
|
||||
for rawVersion, pagination := range map[string]bool{
|
||||
"1.11.0": false,
|
||||
"1.12.0-rc1": true,
|
||||
"11.0.0+gitea-1.11.0": false,
|
||||
"development": false,
|
||||
} {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/api/v1/version" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
_, _ = fmt.Fprintf(w, `{"version":%q}`, rawVersion)
|
||||
}))
|
||||
downloader, err := NewGiteaDownloader(t.Context(), server.URL, "owner/repo", "", "", "")
|
||||
server.Close()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, pagination, downloader.pagination, rawVersion)
|
||||
}
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
}))
|
||||
defer server.Close()
|
||||
_, err := NewGiteaDownloader(t.Context(), server.URL, "owner/repo", "", "", "bad-token")
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestGiteaDownloaderDisabledUnits(t *testing.T) {
|
||||
for _, testCase := range []struct {
|
||||
unitsEnabled bool
|
||||
externalTracker string
|
||||
}{
|
||||
{false, "null"},
|
||||
{true, "null"},
|
||||
{true, "{}"},
|
||||
} {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/api/v1/version":
|
||||
_, _ = w.Write([]byte(`{"version":"1.27.0"}`))
|
||||
case "/api/v1/repos/owner/repo":
|
||||
_, _ = fmt.Fprintf(w, `{"has_issues":%[1]t,"has_pull_requests":%[1]t,"has_releases":%[1]t,"external_tracker":%[2]s}`, testCase.unitsEnabled, testCase.externalTracker)
|
||||
case "/api/v1/repos/owner/repo/releases":
|
||||
http.Error(w, "forbidden", http.StatusForbidden)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
|
||||
ctx := t.Context()
|
||||
downloader, err := NewGiteaDownloader(ctx, server.URL, "owner/repo", "", "", "")
|
||||
require.NoError(t, err)
|
||||
_, issuesEnd, issuesErr := downloader.GetIssues(ctx, 1, 10)
|
||||
_, prsEnd, prsErr := downloader.GetPullRequests(ctx, 1, 10)
|
||||
_, releasesErr := downloader.GetReleases(ctx)
|
||||
server.Close()
|
||||
|
||||
assert.Equal(t, testCase.unitsEnabled && testCase.externalTracker == "null", issuesErr != nil)
|
||||
assert.Equal(t, testCase.unitsEnabled, prsErr != nil)
|
||||
assert.Equal(t, testCase.unitsEnabled, releasesErr != nil)
|
||||
assert.Equal(t, !testCase.unitsEnabled || testCase.externalTracker != "null", issuesEnd)
|
||||
assert.Equal(t, !testCase.unitsEnabled, prsEnd)
|
||||
}
|
||||
}
|
||||
@@ -247,7 +247,7 @@ func TestGiteaDownloadRepo(t *testing.T) {
|
||||
Base: base.PullRequestBranch{
|
||||
CloneURL: "",
|
||||
Ref: "master",
|
||||
SHA: "827aa28a907853e5ddfa40c8f9bc52471a2685fd",
|
||||
SHA: "d9e165e4c7ab6b701f0205d0ffb637e5d2856297",
|
||||
RepoName: "test_repo",
|
||||
OwnerName: "gitea",
|
||||
},
|
||||
@@ -261,7 +261,6 @@ func TestGiteaDownloadRepo(t *testing.T) {
|
||||
Merged: true,
|
||||
MergedTime: new(time.Unix(1598982934, 0)),
|
||||
MergeCommitSHA: "827aa28a907853e5ddfa40c8f9bc52471a2685fd",
|
||||
PatchURL: mockServer.URL + "/gitea/test_repo/pulls/12.patch",
|
||||
}, prs[1])
|
||||
|
||||
reviews, err := downloader.GetReviews(ctx, &base.Issue{Number: 7, ForeignIndex: 7})
|
||||
@@ -274,7 +273,8 @@ func TestGiteaDownloadRepo(t *testing.T) {
|
||||
ReviewerName: "6543",
|
||||
CommitID: "187ece0cb6631e2858a6872e5733433bb3ca3b03",
|
||||
CreatedAt: time.Date(2020, 9, 1, 16, 12, 58, 0, time.UTC),
|
||||
State: "COMMENT", // TODO
|
||||
State: base.ReviewStateCommented,
|
||||
Dismissed: true,
|
||||
Comments: []*base.ReviewComment{
|
||||
{
|
||||
ID: 116561,
|
||||
@@ -299,7 +299,8 @@ func TestGiteaDownloadRepo(t *testing.T) {
|
||||
ReviewerName: "techknowlogick",
|
||||
CommitID: "187ece0cb6631e2858a6872e5733433bb3ca3b03",
|
||||
CreatedAt: time.Date(2020, 9, 1, 17, 6, 47, 0, time.UTC),
|
||||
State: "REQUEST_CHANGES", // TODO
|
||||
State: base.ReviewStateChangesRequested,
|
||||
Dismissed: true,
|
||||
Content: "I think this needs some changes",
|
||||
},
|
||||
{
|
||||
@@ -310,6 +311,7 @@ func TestGiteaDownloadRepo(t *testing.T) {
|
||||
CommitID: "187ece0cb6631e2858a6872e5733433bb3ca3b03",
|
||||
CreatedAt: time.Date(2020, 9, 1, 17, 19, 51, 0, time.UTC),
|
||||
State: base.ReviewStateApproved,
|
||||
Dismissed: true,
|
||||
Official: true,
|
||||
Content: "looks good",
|
||||
},
|
||||
|
||||
@@ -131,7 +131,7 @@ func (g *GiteaLocalUploader) CreateRepo(ctx context.Context, repo *base.Reposito
|
||||
Wiki: opts.Wiki,
|
||||
Releases: opts.Releases, // if didn't get releases, then sync them from tags
|
||||
MirrorInterval: opts.MirrorInterval,
|
||||
}, NewMigrationHTTPTransport())
|
||||
}, getMigrationHTTPClient().Transport)
|
||||
|
||||
g.sameApp = strings.HasPrefix(repo.OriginalURL, setting.AppURL)
|
||||
g.repo = r
|
||||
@@ -237,13 +237,15 @@ func (g *GiteaLocalUploader) CreateLabels(ctx context.Context, labels ...*base.L
|
||||
l.Color = color
|
||||
}
|
||||
|
||||
lbs = append(lbs, &issues_model.Label{
|
||||
lb := &issues_model.Label{
|
||||
RepoID: g.repo.ID,
|
||||
Name: l.Name,
|
||||
Exclusive: l.Exclusive,
|
||||
Description: l.Description,
|
||||
Color: l.Color,
|
||||
})
|
||||
}
|
||||
lb.SetArchived(l.Archived)
|
||||
lbs = append(lbs, lb)
|
||||
}
|
||||
|
||||
err := issues_model.NewLabels(ctx, lbs...)
|
||||
@@ -332,32 +334,29 @@ func (g *GiteaLocalUploader) CreateReleases(ctx context.Context, releases ...*ba
|
||||
|
||||
// SECURITY: We cannot check the DownloadURL and DownloadFunc are safe here
|
||||
// ... we must assume that they are safe and simply download the attachment
|
||||
err := func() error {
|
||||
var rc io.ReadCloser
|
||||
var err error
|
||||
if asset.DownloadFunc != nil {
|
||||
rc, err = asset.DownloadFunc()
|
||||
} else if asset.DownloadURL != nil {
|
||||
// asset.DownloadURL maybe a local file
|
||||
var rc io.ReadCloser
|
||||
var err error
|
||||
if asset.DownloadFunc != nil {
|
||||
rc, err = asset.DownloadFunc()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else if asset.DownloadURL != nil {
|
||||
// use the migration client so the fetch (including any redirect) is
|
||||
// validated against the migration host allow/block list
|
||||
rc, err = uri.OpenWithClient(*asset.DownloadURL, getMigrationHTTPClient())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if rc == nil {
|
||||
return nil
|
||||
}
|
||||
_, err = storage.Attachments.Save(attach.RelativePath(), rc, int64(*asset.Size))
|
||||
rc.Close()
|
||||
return err
|
||||
}()
|
||||
// use the migration client so the fetch (including any redirect) is
|
||||
// validated against the migration host allow/block list
|
||||
rc, err = uri.OpenWithClient(*asset.DownloadURL, getMigrationHTTPClient())
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
WarnAndNotice("Skipped asset %s of release %s: %v", asset.Name, release.TagName, err)
|
||||
continue
|
||||
}
|
||||
if rc != nil {
|
||||
attach.Size, err = storage.Attachments.Save(attach.RelativePath(), rc, util.Iif(*asset.Size > 0, int64(*asset.Size), -1)) // GitLab links and Forgejo external links report no size
|
||||
rc.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("release %s asset %s: %w", release.TagName, asset.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
rel.Attachments = append(rel.Attachments, &attach)
|
||||
@@ -890,6 +889,7 @@ func (g *GiteaLocalUploader) CreateReviews(ctx context.Context, reviews ...*base
|
||||
IssueID: issue.ID,
|
||||
Content: review.Content,
|
||||
Official: review.Official,
|
||||
Dismissed: review.Dismissed,
|
||||
CreatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()),
|
||||
UpdatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()),
|
||||
}
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -125,6 +127,28 @@ func TestGiteaUploadRepo(t *testing.T) {
|
||||
assert.Len(t, pulls[0].Issue.Comments, 2)
|
||||
}
|
||||
|
||||
func TestCreateReleasesSkipsUnavailableAssets(t *testing.T) {
|
||||
unittest.PrepareTestEnv(t)
|
||||
sourceRelease := &base.Release{Name: "assets", Assets: []*base.ReleaseAsset{
|
||||
{Name: "gone", Size: new(0), DownloadCount: new(0), DownloadFunc: func() (io.ReadCloser, error) { return nil, errors.New("gone") }},
|
||||
{Name: "kept", Size: new(0), DownloadCount: new(0), DownloadFunc: func() (io.ReadCloser, error) { return io.NopCloser(strings.NewReader("data")), nil }},
|
||||
}}
|
||||
|
||||
doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
|
||||
uploader := NewGiteaLocalUploader(t.Context(), doer, doer.Name, "repo1")
|
||||
uploader.repo = unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1})
|
||||
require.NoError(t, uploader.CreateReleases(t.Context(), sourceRelease))
|
||||
release := unittest.AssertExistsAndLoadBean(t, &repo_model.Release{RepoID: 1, Title: "assets"})
|
||||
unittest.AssertExistsAndLoadBean(t, &repo_model.Attachment{ReleaseID: release.ID, Name: "kept", Size: 4})
|
||||
unittest.AssertNotExistsBean(t, &repo_model.Attachment{ReleaseID: release.ID, Name: "gone"})
|
||||
|
||||
dumper := &RepositoryDumper{baseDir: t.TempDir(), opts: base.MigrateOptions{ReleaseAssets: true}}
|
||||
defer dumper.Close()
|
||||
require.NoError(t, dumper.CreateReleases(t.Context(), sourceRelease))
|
||||
require.Len(t, sourceRelease.Assets, 1)
|
||||
assert.Equal(t, "kept", sourceRelease.Assets[0].Name)
|
||||
}
|
||||
|
||||
func TestGiteaUploadRemapLocalUser(t *testing.T) {
|
||||
unittest.PrepareTestEnv(t)
|
||||
doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
|
||||
|
||||
@@ -18,9 +18,9 @@ import (
|
||||
"gitea.dev/modules/log"
|
||||
base "gitea.dev/modules/migration"
|
||||
"gitea.dev/modules/structs"
|
||||
"gitea.dev/modules/util"
|
||||
|
||||
"github.com/google/go-github/v92/github"
|
||||
"golang.org/x/oauth2"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -99,29 +99,15 @@ func NewGithubDownloaderV3(_ context.Context, baseURL, userName, password, token
|
||||
maxPerPage: 100,
|
||||
}
|
||||
|
||||
apiURL := util.Iif(baseURL == "https://github.com", "https://api.github.com", baseURL)
|
||||
if token != "" {
|
||||
tokens := strings.SplitSeq(token, ",")
|
||||
for token := range tokens {
|
||||
token = strings.TrimSpace(token)
|
||||
ts := oauth2.StaticTokenSource(
|
||||
&oauth2.Token{AccessToken: token},
|
||||
)
|
||||
client := &http.Client{
|
||||
Transport: &oauth2.Transport{
|
||||
Base: NewMigrationHTTPTransport(),
|
||||
Source: oauth2.ReuseTokenSource(nil, ts),
|
||||
},
|
||||
}
|
||||
|
||||
if err := downloader.addClient(client, baseURL); err != nil {
|
||||
for token := range strings.SplitSeq(token, ",") {
|
||||
if err := downloader.addClient(newMigrationHTTPClient(apiURL, "Bearer "+strings.TrimSpace(token)), baseURL); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
client := &http.Client{
|
||||
Transport: &github.BasicAuthTransport{Transport: NewMigrationHTTPTransport(), Username: userName, Password: password},
|
||||
}
|
||||
if err := downloader.addClient(client, baseURL); err != nil {
|
||||
if err := downloader.addClient(newMigrationHTTPClient(apiURL, basicAuthorization(userName, password)), baseURL); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
@@ -339,8 +325,6 @@ func (g *GithubDownloaderV3) convertGithubRelease(ctx context.Context, rel *gith
|
||||
r.Published = rel.PublishedAt.Time
|
||||
}
|
||||
|
||||
httpClient := newMigrationHTTPClient()
|
||||
|
||||
for _, asset := range rel.Assets {
|
||||
assetID := asset.GetID() // Don't optimize this, for closure we need a local variable TODO: no need to do so in new Golang
|
||||
if assetID == 0 {
|
||||
@@ -381,19 +365,11 @@ func (g *GithubDownloaderV3) convertGithubRelease(ctx context.Context, rel *gith
|
||||
}
|
||||
|
||||
g.waitAndPickClient(ctx)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, redirectURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := httpClient.Do(req)
|
||||
err1 := g.RefreshRate(ctx)
|
||||
if err1 != nil {
|
||||
rc, err := downloadAsset(ctx, getMigrationHTTPClient(), redirectURL)
|
||||
if err1 := g.RefreshRate(ctx); err1 != nil {
|
||||
log.Error("g.RefreshRate(): %s", err1)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return resp.Body, nil
|
||||
return rc, err
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -780,6 +756,7 @@ func (g *GithubDownloaderV3) getIssueReactions(ctx context.Context, number, perP
|
||||
}
|
||||
|
||||
func convertGithubReview(r *github.PullRequestReview) *base.Review {
|
||||
dismissed := r.GetState() == "DISMISSED"
|
||||
return &base.Review{
|
||||
ID: r.GetID(),
|
||||
ReviewerID: r.GetUser().GetID(),
|
||||
@@ -787,7 +764,8 @@ func convertGithubReview(r *github.PullRequestReview) *base.Review {
|
||||
CommitID: r.GetCommitID(),
|
||||
Content: r.GetBody(),
|
||||
CreatedAt: r.GetSubmittedAt().Time,
|
||||
State: r.GetState(),
|
||||
State: util.Iif(dismissed, base.ReviewStateCommented, r.GetState()), // GitHub drops the state a review had before its dismissal
|
||||
Dismissed: dismissed,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -910,15 +888,6 @@ func (g *GithubDownloaderV3) GetReviews(ctx context.Context, reviewable base.Rev
|
||||
|
||||
// FormatCloneURL add authentication into remote URLs
|
||||
func (g *GithubDownloaderV3) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) {
|
||||
u, err := url.Parse(remoteAddr)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(opts.AuthToken) > 0 {
|
||||
// "multiple tokens" are used to benefit more "API rate limit quota"
|
||||
// git clone doesn't count for rate limits, so only use the first token.
|
||||
// source: https://github.com/orgs/community/discussions/44515
|
||||
u.User = url.UserPassword("oauth2", strings.Split(opts.AuthToken, ",")[0])
|
||||
}
|
||||
return u.String(), nil
|
||||
opts.AuthToken, _, _ = strings.Cut(opts.AuthToken, ",") // the extra tokens only raise the API rate limit
|
||||
return g.NullDownloader.FormatCloneURL(opts, remoteAddr)
|
||||
}
|
||||
@@ -543,10 +543,17 @@ func TestGitHubDownloadRepo(t *testing.T) {
|
||||
}, reviews)
|
||||
}
|
||||
|
||||
func TestGithubMultiToken(t *testing.T) {
|
||||
func TestConvertGithubDismissedReview(t *testing.T) {
|
||||
review := convertGithubReview(&github.PullRequestReview{State: new("DISMISSED")})
|
||||
assert.Equal(t, base.ReviewStateCommented, review.State)
|
||||
assert.True(t, review.Dismissed)
|
||||
}
|
||||
|
||||
func TestGithubFormatCloneURL(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
token string
|
||||
username string
|
||||
expectedCloneURL string
|
||||
}{
|
||||
{
|
||||
@@ -559,12 +566,17 @@ func TestGithubMultiToken(t *testing.T) {
|
||||
token: "token1,token2",
|
||||
expectedCloneURL: "https://oauth2:token1@github.com",
|
||||
},
|
||||
{
|
||||
desc: "Username And Password",
|
||||
username: "user",
|
||||
expectedCloneURL: "https://user:password@github.com",
|
||||
},
|
||||
}
|
||||
factory := GithubDownloaderV3Factory{}
|
||||
|
||||
for _, tC := range testCases {
|
||||
t.Run(tC.desc, func(t *testing.T) {
|
||||
opts := base.MigrateOptions{CloneAddr: "https://github.com/go-gitea/gitea", AuthToken: tC.token}
|
||||
opts := base.MigrateOptions{CloneAddr: "https://github.com/go-gitea/gitea", AuthToken: tC.token, AuthUsername: tC.username, AuthPassword: "password"}
|
||||
client, err := factory.New(t.Context(), opts)
|
||||
require.NoError(t, err)
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"gitea.dev/modules/log"
|
||||
base "gitea.dev/modules/migration"
|
||||
"gitea.dev/modules/structs"
|
||||
"gitea.dev/modules/util"
|
||||
|
||||
gitlab "gitlab.com/gitlab-org/api/client-go/v3"
|
||||
)
|
||||
@@ -94,7 +95,8 @@ type GitlabDownloader struct {
|
||||
// Use either a username/password, personal token entered into the username field, or anonymous/public access
|
||||
// Note: Public access only allows very basic access
|
||||
func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, token string) (*GitlabDownloader, error) {
|
||||
gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(newMigrationHTTPClient()))
|
||||
transport := authTransport(getMigrationTransport(), baseURL, util.Iif(token != "", "Bearer "+token, "")) // not retrying, the SDK retries rate limits itself
|
||||
gitlabClient, err := gitlab.NewAuthSourceClient(gitlab.Unauthenticated{}, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(&http.Client{Transport: transport}))
|
||||
if err != nil {
|
||||
log.Trace("Error logging into gitlab: %v", err)
|
||||
return nil, err
|
||||
@@ -314,8 +316,6 @@ func (g *GitlabDownloader) convertGitlabRelease(ctx context.Context, rel *gitlab
|
||||
PublisherName: rel.Author.Username,
|
||||
}
|
||||
|
||||
httpClient := newMigrationHTTPClient()
|
||||
|
||||
for _, asset := range rel.Assets.Links {
|
||||
assetID := asset.ID // Don't optimize this, for closure we need a local variable
|
||||
r.Assets = append(r.Assets, &base.ReleaseAsset{
|
||||
@@ -334,18 +334,7 @@ func (g *GitlabDownloader) convertGitlabRelease(ctx context.Context, rel *gitlab
|
||||
return io.NopCloser(strings.NewReader(link.URL)), nil
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, link.URL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req = req.WithContext(ctx)
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// resp.Body is closed by the uploader
|
||||
return resp.Body, nil
|
||||
return downloadAsset(ctx, g.client.HTTPClient(), link.URL)
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -408,6 +397,12 @@ func (g *GitlabDownloader) GetIssues(ctx context.Context, page, perPage int) ([]
|
||||
return nil, false, fmt.Errorf("error while listing issues: %w", err)
|
||||
}
|
||||
for _, issue := range issues {
|
||||
// record the issue IID, to be used in GetPullRequests()
|
||||
g.iidResolver.recordIssueIID(issue.IID)
|
||||
if issue.Confidential { // Gitea can't restrict an issue to project members
|
||||
continue
|
||||
}
|
||||
|
||||
labels := make([]*base.Label, 0, len(issue.Labels))
|
||||
for _, l := range issue.Labels {
|
||||
labels = append(labels, &base.Label{
|
||||
@@ -454,9 +449,6 @@ func (g *GitlabDownloader) GetIssues(ctx context.Context, page, perPage int) ([]
|
||||
ForeignIndex: issue.IID,
|
||||
Context: gitlabIssueContext{IsMergeRequest: false},
|
||||
})
|
||||
|
||||
// record the issue IID, to be used in GetPullRequests()
|
||||
g.iidResolver.recordIssueIID(issue.IID)
|
||||
}
|
||||
|
||||
return allIssues, len(issues) < perPage, nil
|
||||
@@ -499,7 +491,9 @@ func (g *GitlabDownloader) GetComments(ctx context.Context, commentable base.Com
|
||||
}
|
||||
for _, comment := range comments {
|
||||
for _, note := range comment.Notes {
|
||||
allComments = append(allComments, g.convertNoteToComment(commentable.GetLocalIndex(), note))
|
||||
if !note.Internal { // Gitea can't restrict a comment to project members
|
||||
allComments = append(allComments, g.convertNoteToComment(commentable.GetLocalIndex(), note))
|
||||
}
|
||||
}
|
||||
}
|
||||
if resp.NextPage == 0 {
|
||||
@@ -749,6 +743,9 @@ func (g *GitlabDownloader) GetReviews(ctx context.Context, reviewable base.Revie
|
||||
|
||||
reviews := make([]*base.Review, 0, len(approvals.ApprovedBy))
|
||||
for _, user := range approvals.ApprovedBy {
|
||||
if user.User == nil {
|
||||
continue
|
||||
}
|
||||
reviews = append(reviews, &base.Review{
|
||||
IssueIndex: reviewable.GetLocalIndex(),
|
||||
ReviewerID: user.User.ID,
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
base "gitea.dev/modules/migration"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
gitlab "gitlab.com/gitlab-org/api/client-go/v3"
|
||||
)
|
||||
|
||||
@@ -513,6 +514,39 @@ func TestGitlabGetReviews(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
assertReviewsEqual(t, []*base.Review{&review}, rvs)
|
||||
}
|
||||
|
||||
mux.HandleFunc(fmt.Sprintf("/api/v4/projects/%d/merge_requests/4/approvals", repoID), func(w http.ResponseWriter, _ *http.Request) {
|
||||
fmt.Fprint(w, `{"approved_by": [{"user": null}]}`)
|
||||
})
|
||||
rvs, err := downloader.GetReviews(ctx, &base.Issue{Number: 4, ForeignIndex: 4})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, rvs)
|
||||
}
|
||||
|
||||
func TestGitlabSkipsConfidentialIssuesAndInternalNotes(t *testing.T) {
|
||||
mux, server, client := gitlabClientMockSetup(t)
|
||||
defer gitlabClientMockTeardown(server)
|
||||
downloader := &GitlabDownloader{client: client, repoID: 1, maxPerPage: 100}
|
||||
const timestamps = `"created_at": "2020-01-01T00:00:00Z", "updated_at": "2020-01-01T00:00:00Z"`
|
||||
empty := func(w http.ResponseWriter, _ *http.Request) { fmt.Fprint(w, `[]`) }
|
||||
mux.HandleFunc("/api/v4/projects/1/issues", func(w http.ResponseWriter, _ *http.Request) {
|
||||
fmt.Fprint(w, `[{"iid": 1, "author": {}, `+timestamps+`}, {"iid": 2, "confidential": true}]`)
|
||||
})
|
||||
mux.HandleFunc("/api/v4/projects/1/issues/1/award_emoji", empty)
|
||||
mux.HandleFunc("/api/v4/projects/1/issues/1/discussions", func(w http.ResponseWriter, _ *http.Request) {
|
||||
fmt.Fprint(w, `[{"notes": [{"id": 1, `+timestamps+`}, {"internal": true}]}]`)
|
||||
})
|
||||
mux.HandleFunc("/api/v4/projects/1/issues/1/resource_state_events", empty)
|
||||
|
||||
issues, _, err := downloader.GetIssues(t.Context(), 1, 100)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, issues, 1)
|
||||
assert.EqualValues(t, 3, downloader.iidResolver.generatePullRequestNumber(1))
|
||||
|
||||
comments, _, err := downloader.GetComments(t.Context(), issues[0])
|
||||
require.NoError(t, err)
|
||||
require.Len(t, comments, 1)
|
||||
assert.EqualValues(t, 1, comments[0].Index)
|
||||
}
|
||||
|
||||
func TestAwardsToReactions(t *testing.T) {
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"gitea.dev/modules/log"
|
||||
base "gitea.dev/modules/migration"
|
||||
"gitea.dev/modules/structs"
|
||||
"gitea.dev/modules/util"
|
||||
|
||||
"github.com/gogs/go-gogs-client"
|
||||
)
|
||||
@@ -62,9 +63,7 @@ type GogsDownloader struct {
|
||||
baseURL string
|
||||
repoOwner string
|
||||
repoName string
|
||||
userName string
|
||||
password string
|
||||
token string
|
||||
transport http.RoundTripper
|
||||
openIssuesFinished bool
|
||||
openIssuesPages int
|
||||
}
|
||||
@@ -83,36 +82,21 @@ func (g *GogsDownloader) LogString() string {
|
||||
|
||||
// NewGogsDownloader creates a gogs Downloader via gogs API
|
||||
func NewGogsDownloader(_ context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GogsDownloader {
|
||||
downloader := GogsDownloader{
|
||||
return &GogsDownloader{
|
||||
baseURL: baseURL,
|
||||
userName: userName,
|
||||
password: password,
|
||||
token: token,
|
||||
transport: newMigrationHTTPClient(baseURL, util.Iif(token != "", "token "+token, basicAuthorization(userName, password))).Transport,
|
||||
repoOwner: repoOwner,
|
||||
repoName: repoName,
|
||||
}
|
||||
return &downloader
|
||||
}
|
||||
|
||||
type roundTripperFunc func(req *http.Request) (*http.Response, error)
|
||||
|
||||
func (rt roundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) {
|
||||
return rt(r)
|
||||
}
|
||||
|
||||
func (g *GogsDownloader) client(ctx context.Context) *gogs.Client {
|
||||
// Gogs client lacks the context support, so we use a custom transport
|
||||
// Then each request uses a dedicated client with its own context
|
||||
httpTransport := NewMigrationHTTPTransport()
|
||||
gogsClient := gogs.NewClient(g.baseURL, g.token)
|
||||
gogsClient := gogs.NewClient(g.baseURL, "")
|
||||
gogsClient.SetHTTPClient(&http.Client{
|
||||
Transport: roundTripperFunc(func(req *http.Request) (*http.Response, error) {
|
||||
req = req.Clone(ctx)
|
||||
if g.password != "" {
|
||||
// Gogs client lacks the support for basic auth, this is the only way to set it
|
||||
req.SetBasicAuth(g.userName, g.password)
|
||||
}
|
||||
return httpTransport.RoundTrip(req)
|
||||
return g.transport.RoundTrip(req.WithContext(ctx))
|
||||
}),
|
||||
})
|
||||
return gogsClient
|
||||
|
||||
@@ -5,24 +5,110 @@ package migrations
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.dev/modules/egress"
|
||||
"gitea.dev/modules/log"
|
||||
"gitea.dev/modules/setting"
|
||||
)
|
||||
|
||||
// getMigrationHTTPClient returns the shared migration client, so downloads from one host reuse its connections
|
||||
var getMigrationHTTPClient = sync.OnceValue(newMigrationHTTPClient)
|
||||
var getMigrationHTTPClient = sync.OnceValue(func() *http.Client {
|
||||
return &http.Client{Transport: retryAfterTransport{getMigrationTransport()}}
|
||||
})
|
||||
|
||||
// newMigrationHTTPClient returns a HTTP client for migration
|
||||
func newMigrationHTTPClient() *http.Client {
|
||||
return &http.Client{Transport: NewMigrationHTTPTransport()}
|
||||
var getMigrationTransport = sync.OnceValue(newMigrationTransport)
|
||||
|
||||
func newMigrationHTTPClient(baseURL, authorization string) *http.Client {
|
||||
return &http.Client{Transport: authTransport(getMigrationHTTPClient().Transport, baseURL, authorization)}
|
||||
}
|
||||
|
||||
// NewMigrationHTTPTransport returns a HTTP transport for migration, enforcing the migration policy on its direct dials.
|
||||
func NewMigrationHTTPTransport() *http.Transport {
|
||||
func NewMigrationHTTPTransport() http.RoundTripper {
|
||||
return retryAfterTransport{newMigrationTransport()}
|
||||
}
|
||||
|
||||
func newMigrationTransport() *http.Transport {
|
||||
t := egress.NewMigrationPolicy().NewHTTPTransport()
|
||||
t.TLSClientConfig = &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}
|
||||
return t
|
||||
}
|
||||
|
||||
type retryAfterTransport struct {
|
||||
http.RoundTripper
|
||||
}
|
||||
|
||||
func (t retryAfterTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
var waited time.Duration
|
||||
for retries := 0; ; retries++ {
|
||||
resp, err := t.RoundTripper.RoundTrip(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if (resp.StatusCode != http.StatusTooManyRequests && resp.StatusCode != http.StatusForbidden) || retries == 5 || (req.Body != nil && req.GetBody == nil) { // GitHub's secondary rate limits answer 403
|
||||
return resp, nil
|
||||
}
|
||||
delay, ok := parseRetryAfter(resp.Header.Get("Retry-After"))
|
||||
if !ok || delay > time.Hour-waited {
|
||||
return resp, nil
|
||||
}
|
||||
waited += delay
|
||||
resp.Body.Close()
|
||||
log.Info("Rate limited by %s, retrying in %s", req.URL.Host, delay)
|
||||
select {
|
||||
case <-req.Context().Done():
|
||||
return nil, req.Context().Err()
|
||||
case <-time.After(delay):
|
||||
}
|
||||
if req.GetBody != nil {
|
||||
body, err := req.GetBody()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req = req.Clone(req.Context())
|
||||
req.Body = body
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func parseRetryAfter(value string) (time.Duration, bool) {
|
||||
if seconds, err := strconv.ParseUint(value, 10, 32); err == nil { // 32 bits can't overflow the Duration
|
||||
return time.Duration(seconds) * time.Second, true
|
||||
}
|
||||
if retryAt, err := http.ParseTime(value); err == nil {
|
||||
return max(time.Until(retryAt), 0), true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
type roundTripperFunc func(req *http.Request) (*http.Response, error)
|
||||
|
||||
func (rt roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
return rt(req)
|
||||
}
|
||||
|
||||
func authTransport(transport http.RoundTripper, baseURL, authorization string) http.RoundTripper {
|
||||
base, err := url.Parse(baseURL)
|
||||
if authorization == "" || err != nil {
|
||||
return transport
|
||||
}
|
||||
return roundTripperFunc(func(req *http.Request) (*http.Response, error) {
|
||||
if req.URL.Host == base.Host && (req.URL.Scheme == "https" || base.Scheme == "http") { // the source host only, never downgraded to http
|
||||
req = req.Clone(req.Context())
|
||||
req.Header.Set("Authorization", authorization)
|
||||
}
|
||||
return transport.RoundTrip(req)
|
||||
})
|
||||
}
|
||||
|
||||
func basicAuthorization(username, password string) string {
|
||||
if username == "" {
|
||||
return ""
|
||||
}
|
||||
return "Basic " + base64.StdEncoding.EncodeToString([]byte(username+":"+password))
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestMigrationHTTPTransportRetryAfter(t *testing.T) {
|
||||
var requests atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch requests.Add(1) {
|
||||
case 1:
|
||||
w.Header().Set("Retry-After", "0")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
case 2:
|
||||
w.Header().Set("Retry-After", time.Now().UTC().Format(http.TimeFormat))
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
default:
|
||||
_, _ = io.Copy(w, r.Body)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
resp, err := getMigrationHTTPClient().Post(server.URL, "text/plain", strings.NewReader("payload"))
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
assert.Equal(t, "payload", string(body))
|
||||
assert.EqualValues(t, 3, requests.Load())
|
||||
}
|
||||
|
||||
func TestAuthTransportSendsCredentialsToSameHostOnly(t *testing.T) {
|
||||
for _, testCase := range []struct{ baseURL, target, expected string }{
|
||||
{"http://source.example/sub", "https://source.example/api", "token secret"},
|
||||
{"https://source.example", "https://source.example/api", "token secret"},
|
||||
{"https://source.example", "http://source.example/api", ""},
|
||||
{"https://source.example", "https://other.example/api", ""},
|
||||
{"https://source.example:3000", "https://source.example:8443/api", ""},
|
||||
} {
|
||||
req, err := http.NewRequest(http.MethodGet, testCase.target, nil)
|
||||
require.NoError(t, err)
|
||||
resp, err := authTransport(roundTripperFunc(func(req *http.Request) (*http.Response, error) {
|
||||
assert.Equal(t, testCase.expected, req.Header.Get("Authorization"), testCase.target)
|
||||
return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody}, nil
|
||||
}), testCase.baseURL, "token secret").RoundTrip(req)
|
||||
require.NoError(t, err)
|
||||
resp.Body.Close()
|
||||
assert.Empty(t, req.Header.Get("Authorization"))
|
||||
}
|
||||
}
|
||||
@@ -54,6 +54,7 @@ func assertCommentsEqual(t *testing.T, expected, actual []*base.Comment) {
|
||||
func assertLabelEqual(t *testing.T, expected, actual *base.Label) {
|
||||
assert.Equal(t, expected.Name, actual.Name)
|
||||
assert.Equal(t, expected.Exclusive, actual.Exclusive)
|
||||
assert.Equal(t, expected.Archived, actual.Archived)
|
||||
assert.Equal(t, expected.Color, actual.Color)
|
||||
assert.Equal(t, expected.Description, actual.Description)
|
||||
}
|
||||
@@ -228,6 +229,7 @@ func assertReviewEqual(t *testing.T, expected, actual *base.Review) {
|
||||
assert.Equal(t, expected.Content, actual.Content, "Content")
|
||||
assert.WithinDuration(t, expected.CreatedAt, actual.CreatedAt, 10*time.Second)
|
||||
assert.Equal(t, expected.State, actual.State, "State")
|
||||
assert.Equal(t, expected.Dismissed, actual.Dismissed, "Dismissed")
|
||||
assertReviewCommentsEqual(t, expected.Comments, actual.Comments)
|
||||
}
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
repo_model "gitea.dev/models/repo"
|
||||
@@ -293,14 +294,14 @@ func migrateRepository(ctx context.Context, doer *user_model.User, downloader ba
|
||||
|
||||
supportAllComments := downloader.SupportGetRepoComments()
|
||||
|
||||
mapInsertedIssueIndexes := container.Set[int64]{} // deletions on the source during the migration shift items into the next page
|
||||
mapInsertedPRIndexes := container.Set[int64]{}
|
||||
|
||||
if opts.Issues {
|
||||
log.Trace("migrating issues and comments")
|
||||
messenger("repo.migrate.migrating_issues")
|
||||
issueBatchSize := uploader.MaxBatchInsertSize("issue")
|
||||
|
||||
// because when the migrating is running, some issues maybe removed, so after the next page
|
||||
// some of issue maybe duplicated, so we need to record the inserted issue indexes
|
||||
mapInsertedIssueIndexes := container.Set[int64]{}
|
||||
for i := 1; ; i++ {
|
||||
issues, isEnd, err := downloader.GetIssues(ctx, i, issueBatchSize)
|
||||
if err != nil {
|
||||
@@ -363,7 +364,6 @@ func migrateRepository(ctx context.Context, doer *user_model.User, downloader ba
|
||||
log.Trace("migrating pull requests and comments")
|
||||
messenger("repo.migrate.migrating_pulls")
|
||||
prBatchSize := uploader.MaxBatchInsertSize("pullrequest")
|
||||
mapInsertedPRIndexes := container.Set[int64]{}
|
||||
for i := 1; ; i++ {
|
||||
prs, isEnd, err := downloader.GetPullRequests(ctx, i, prBatchSize)
|
||||
if err != nil {
|
||||
@@ -466,6 +466,9 @@ func migrateRepository(ctx context.Context, doer *user_model.User, downloader ba
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
comments = slices.DeleteFunc(comments, func(comment *base.Comment) bool {
|
||||
return !mapInsertedIssueIndexes.Contains(comment.IssueIndex) && !mapInsertedPRIndexes.Contains(comment.IssueIndex)
|
||||
})
|
||||
|
||||
if err := uploader.CreateComments(ctx, comments...); err != nil {
|
||||
return err
|
||||
|
||||
@@ -80,26 +80,15 @@ type OneDevDownloader struct {
|
||||
}
|
||||
|
||||
// NewOneDevDownloader creates a new downloader
|
||||
func NewOneDevDownloader(ctx context.Context, baseURL *url.URL, username, password, repoPath string) *OneDevDownloader {
|
||||
httpTransport := NewMigrationHTTPTransport()
|
||||
downloader := &OneDevDownloader{
|
||||
func NewOneDevDownloader(_ context.Context, baseURL *url.URL, username, password, repoPath string) *OneDevDownloader {
|
||||
return &OneDevDownloader{
|
||||
baseURL: baseURL,
|
||||
repoPath: repoPath,
|
||||
client: &http.Client{
|
||||
Transport: roundTripperFunc(
|
||||
func(req *http.Request) (*http.Response, error) {
|
||||
req = req.Clone(ctx)
|
||||
if username != "" && password != "" {
|
||||
req.SetBasicAuth(username, password)
|
||||
}
|
||||
return httpTransport.RoundTrip(req)
|
||||
}),
|
||||
},
|
||||
|
||||
client: newMigrationHTTPClient(baseURL.String(), basicAuthorization(username, password)),
|
||||
userMap: make(map[int64]*onedevUser),
|
||||
milestoneMap: make(map[int64]string),
|
||||
}
|
||||
|
||||
return downloader
|
||||
}
|
||||
|
||||
// String implements Stringer
|
||||
|
||||
@@ -71,7 +71,7 @@ func cloneWiki(ctx context.Context, repo *repo_model.Repository, opts migration.
|
||||
// MigrateRepositoryGitData starts migrating git related data after created migrating repository
|
||||
func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
|
||||
repo *repo_model.Repository, opts migration.MigrateOptions,
|
||||
httpTransport *http.Transport,
|
||||
httpTransport http.RoundTripper,
|
||||
) (*repo_model.Repository, error) {
|
||||
if u.IsOrganization() {
|
||||
t, err := organization.OrgFromUser(u).GetOwnerTeam(ctx)
|
||||
|
||||
Reference in new issue
Block a user