feat: Improve Gitea-compatible forge migrations, fix many migration bugs (#39424)

Supports migrating from Gitea-compatible forges like Forgejo and fixes
many migration bugs.

1. Pre-release and `+gitea-` suffixed server versions no longer drop
reviews, reactions and pagination.
2. Review states, dismissals, scoped and archived labels and merge bases
are migrated.
3. Disabled units, external trackers, unavailable release assets, and
GitLab confidential issues and internal notes are skipped.
4. Credentials are only sent to the source host, and `Retry-After` rate
limits are waited out.
5. GitHub migrations of only issues or pull requests, basic auth clones
and dismissed reviews work again.
6. The API returns 404 for releases of a disabled unit and shows
migrated authors as mannequins, replacing the never populated
`original_author` and `original_author_id` fields.

Fixes: https://github.com/go-gitea/gitea/issues/19533
Fixes: https://github.com/go-gitea/gitea/issues/35885
Fixes: https://github.com/go-gitea/gitea/issues/18275
Fixes: https://github.com/go-gitea/gitea/issues/35069
Fixes: https://github.com/go-gitea/gitea/issues/17763
Fixes: https://github.com/go-gitea/gitea/issues/39548
Co-authored-by: bircni <bircni@icloud.com>
This commit is contained in:
silverwindandbircni authored and GitHub committed 2026-10-24 01:15:31 +00:00
1 parent 7c91252be3
commit fc49368845
94 files changed
+912 -417

No files matched your search

+2
View File
@@ -6,6 +6,8 @@
/public/assets/img/svg/*.svg linguist-generated
/templates/swagger/*.generated.json linguist-generated
/options/fileicon/** linguist-generated
/services/migrations/_mock_data/** linguist-generated
/tests/integration/_mock_data/** linguist-generated
/vendor/** -text -eol linguist-vendored
Dockerfile.* linguist-language=Dockerfile
Makefile.* linguist-language=Makefile
+3 -1
View File
@@ -33,7 +33,7 @@ func newDumpRepositoryCommand() *cli.Command {
&cli.StringFlag{
Name: "git_service",
Value: "",
Usage: "Git service, git, github, gitea, gitlab. If clone_addr could be recognized, this could be ignored.",
Usage: "Git service, git, github, gitea, forgejo, gitlab. If clone_addr could be recognized, this could be ignored.",
},
&cli.StringFlag{
Name: "repo_dir",
@@ -117,6 +117,8 @@ func runDumpRepository(ctx context.Context, cmd *cli.Command) error {
serviceStr = "gitlab"
} else if strings.HasPrefix(strings.ToLower(cloneAddr), "https://gitea.com/") {
serviceStr = "gitea"
} else if strings.HasPrefix(strings.ToLower(cloneAddr), "https://codeberg.org/") {
serviceStr = "forgejo"
}
if serviceStr == "" {
return errors.New("git_service missed or clone_addr cannot be recognized")
+12 -4
View File
@@ -170,14 +170,22 @@ func GetReviewsByIssueID(ctx context.Context, issueID int64) (latestReviews, mig
// filter them in memory to get the latest review of each reviewer
// Since the reviews should not be too many for one issue, less than 100 commonly, it's acceptable to do this in memory
// And since there are too less indexes in review table, it will be very slow to filter in the database
reviewersMap := make(map[int64][]*Review) // key is reviewer id
originalReviewersMap := make(map[int64][]*Review) // key is original author id
reviewTeamsMap := make(map[int64][]*Review) // key is reviewer team id
type originalReviewer struct {
id int64
name string
}
reviewersMap := make(map[int64][]*Review) // key is reviewer id
originalReviewersMap := make(map[originalReviewer][]*Review) // key is original author
reviewTeamsMap := make(map[int64][]*Review) // key is reviewer team id
countedReviewTypes := []ReviewType{ReviewTypeApprove, ReviewTypeReject, ReviewTypeRequest, ReviewTypeComment}
for _, review := range reviews {
if review.ReviewerTeamID == 0 && slices.Contains(countedReviewTypes, review.Type) && !review.Dismissed {
if review.OriginalAuthorID != 0 {
originalReviewersMap[review.OriginalAuthorID] = append(originalReviewersMap[review.OriginalAuthorID], review)
key := originalReviewer{id: review.OriginalAuthorID}
if review.OriginalAuthorID == user_model.GhostUserID {
key.name = review.OriginalAuthor
}
originalReviewersMap[key] = append(originalReviewersMap[key], review)
} else {
reviewersMap[review.ReviewerID] = append(reviewersMap[review.ReviewerID], review)
}
+13
View File
@@ -194,6 +194,19 @@ func TestGetReviewersByIssueID(t *testing.T) {
assert.Equal(t, expectedReviews[i].UpdatedUnix, review.UpdatedUnix)
}
}
for _, review := range []*issues_model.Review{
{OriginalAuthor: "octocat", OriginalAuthorID: user_model.GhostUserID},
{OriginalAuthor: "hubot", OriginalAuthorID: user_model.GhostUserID},
{OriginalAuthor: "octocat", OriginalAuthorID: 583231},
{OriginalAuthor: "renamed-octocat", OriginalAuthorID: 583231},
} {
review.IssueID, review.Type = issue.ID, issues_model.ReviewTypeApprove
assert.NoError(t, db.Insert(t.Context(), review))
}
_, migratedReviews, err = issues_model.GetReviewsByIssueID(t.Context(), issue.ID)
assert.NoError(t, err)
assert.Len(t, migratedReviews, 3)
}
func TestDismissReview(t *testing.T) {
+2 -2
View File
@@ -27,7 +27,7 @@ type Client interface {
}
// newClient creates a LFS client
func newClient(endpoint *url.URL, httpTransport *http.Transport) Client {
func newClient(endpoint *url.URL, httpTransport http.RoundTripper) Client {
if endpoint.Scheme == "file" {
return newFilesystemClient(endpoint)
}
@@ -35,7 +35,7 @@ func newClient(endpoint *url.URL, httpTransport *http.Transport) Client {
}
// NewClientFromEndpoint creates a LFS client after resolving its endpoint.
func NewClientFromEndpoint(cloneurl, lfsurl string, httpTransport *http.Transport) (Client, error) {
func NewClientFromEndpoint(cloneurl, lfsurl string, httpTransport http.RoundTripper) (Client, error) {
endpoint := DetermineEndpoint(cloneurl, lfsurl)
if endpoint == nil {
source := cloneurl
+1 -1
View File
@@ -34,7 +34,7 @@ func (c *HTTPClient) BatchSize() int {
return setting.LFSClient.BatchSize
}
func newHTTPClient(endpoint *url.URL, httpTransport *http.Transport) *HTTPClient {
func newHTTPClient(endpoint *url.URL, httpTransport http.RoundTripper) *HTTPClient {
if httpTransport == nil {
httpTransport = &http.Transport{
Proxy: proxy.Proxy(),
@@ -8,3 +8,7 @@
created: 2021-05-27T15:24:13+02:00
updated: 2021-11-11T10:52:45+01:00
closed: 2021-11-11T10:52:45+01:00
labels:
- name: scope/label_a
exclusive: true
archived: true
+1
View File
@@ -10,4 +10,5 @@ type Label struct {
Color string `json:"color"`
Description string `json:"description"`
Exclusive bool `json:"exclusive"`
Archived bool `json:"archived"`
}
+2 -1
View File
@@ -41,7 +41,8 @@ type Review struct {
CommitID string `yaml:"commit_id"`
Content string
CreatedAt time.Time `yaml:"created_at"`
State string // PENDING, APPROVED, REQUEST_CHANGES, or COMMENT
State string
Dismissed bool
Comments []*ReviewComment
}
+8
View File
@@ -16,6 +16,14 @@
"description": {
"description": "Long, multiline, description.",
"type": "string"
},
"exclusive": {
"description": "Whether the label's scope allows only one label of it per issue.",
"type": "boolean"
},
"archived": {
"description": "Whether the label is archived.",
"type": "boolean"
}
},
"required": [
+12 -14
View File
@@ -49,20 +49,18 @@ type RepositoryMeta struct {
// Issue represents an issue in a repository
// swagger:model
type Issue struct {
ID int64 `json:"id"`
URL string `json:"url"`
HTMLURL string `json:"html_url"`
Index int64 `json:"number"`
Poster *User `json:"user"`
OriginalAuthor string `json:"original_author"`
OriginalAuthorID int64 `json:"original_author_id"`
Title string `json:"title"`
Body string `json:"body"`
Ref string `json:"ref"`
Attachments []*Attachment `json:"assets"`
Labels []*Label `json:"labels"`
Milestone *Milestone `json:"milestone"`
Projects []*Project `json:"projects"`
ID int64 `json:"id"`
URL string `json:"url"`
HTMLURL string `json:"html_url"`
Index int64 `json:"number"`
Poster *User `json:"user"`
Title string `json:"title"`
Body string `json:"body"`
Ref string `json:"ref"`
Attachments []*Attachment `json:"assets"`
Labels []*Label `json:"labels"`
Milestone *Milestone `json:"milestone"`
Projects []*Project `json:"projects"`
// deprecated
Assignee *User `json:"assignee"`
Assignees []*User `json:"assignees"`
-4
View File
@@ -19,10 +19,6 @@ type Comment struct {
IssueURL string `json:"issue_url"`
// Poster is the user who posted the comment
Poster *User `json:"user"`
// OriginalAuthor is the original author name (for imported comments)
OriginalAuthor string `json:"original_author"`
// OriginalAuthorID is the original author ID (for imported comments)
OriginalAuthorID int64 `json:"original_author_id"`
// Body contains the comment text content
Body string `json:"body"`
// Attachments contains files attached to the comment
+6 -2
View File
@@ -358,6 +358,7 @@ const (
CodebaseService // 8 codebase service
CodeCommitService // 9 codecommit service
BitbucketServerService // 10 bitbucket server (data center) service
ForgejoService // 11 forgejo service
)
// Name represents the service type's name
@@ -387,6 +388,8 @@ func (gt GitServiceType) Title() string {
return "CodeCommit"
case BitbucketServerService:
return "Bitbucket Server"
case ForgejoService:
return "Forgejo"
case PlainGitService:
return "Git"
}
@@ -423,7 +426,7 @@ type MigrateRepoOptions struct {
// required: true
RepoName string `json:"repo_name" binding:"Required;AlphaDashDot;MaxSize(100)"`
// enum: ["git","github","gitea","gitlab","gogs","onedev","gitbucket","codebase","codecommit","bitbucketserver"]
// enum: ["git","github","gitea","gitlab","gogs","onedev","gitbucket","codebase","codecommit","bitbucketserver","forgejo"]
Service string `json:"service"`
AuthUsername string `json:"auth_username"`
AuthPassword string `json:"auth_password"`
@@ -449,7 +452,7 @@ type MigrateRepoOptions struct {
// TokenAuth represents whether a service type supports token-based auth
func (gt GitServiceType) TokenAuth() bool {
switch gt {
case GithubService, GiteaService, GitlabService:
case GithubService, GiteaService, GitlabService, ForgejoService:
return true
}
return false
@@ -461,6 +464,7 @@ var SupportedFullGitService = []GitServiceType{
GithubService,
GitlabService,
GiteaService,
ForgejoService,
GogsService,
OneDevService,
GitBucketService,
+3
View File
@@ -1099,7 +1099,9 @@
"repo.migrate_repo": "Migrate Repository",
"repo.migrate.clone_address": "Migrate / Clone From URL",
"repo.migrate.clone_address_desc": "The HTTP(S) or Git 'clone' URL of an existing repository",
"repo.migrate.gitea_token_desc": "Create an access token in the Applications section of your account settings on the source instance.",
"repo.migrate.github_token_desc": "You can put one or more tokens here, separated by commas, to make migrating faster by circumventing the GitHub API rate limit. WARNING: Abusing this feature may violate the service provider's policy and may lead to getting your account(s) blocked.",
"repo.migrate.gitlab_token_desc": "Create a personal access token with the read_api and read_repository scopes.",
"repo.migrate.clone_local_path": "or a local server path",
"repo.migrate.permission_denied": "You are not allowed to import local repositories.",
"repo.migrate.permission_denied_blocked": "You cannot import from disallowed hosts. Please ask the admin to check the ALLOWED_HOST_LIST/BLOCKED_HOST_LIST settings in the [migrations] section.",
@@ -1118,6 +1120,7 @@
"repo.migrate.git.description": "Migrate a repository only from any Git service.",
"repo.migrate.gitlab.description": "Migrate data from gitlab.com or other GitLab instances.",
"repo.migrate.gitea.description": "Migrate data from gitea.com or other Gitea instances.",
"repo.migrate.forgejo.description": "Migrate data from codeberg.org or other Forgejo instances.",
"repo.migrate.gogs.description": "Migrate data from notabug.org or other Gogs instances.",
"repo.migrate.onedev.description": "Migrate data from code.onedev.io or other OneDev instances.",
"repo.migrate.codebase.description": "Migrate data from codebasehq.com.",
+1
View File
@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 212 212" class="svg gitea-forgejo" width="16" height="16" aria-hidden="true"><g fill="none" transform="translate(6 6)"><path stroke="#f60" stroke-width="25" d="M58 168V70a50 50 0 0 1 50-50h20"/><path stroke="#d40000" stroke-width="25" d="M58 168v-30a50 50 0 0 1 50-50h20"/><circle cx="142" cy="20" r="18" stroke="#f60" stroke-width="15"/><circle cx="142" cy="88" r="18" stroke="#d40000" stroke-width="15"/><circle cx="58" cy="180" r="18" stroke="#d40000" stroke-width="15"/></g></svg>

After

Width:  |  Height:  |  Size: 537 B

+7 -6
View File
@@ -743,10 +743,11 @@ func mustEnableIssuesOrPulls(ctx *context.APIContext) {
}
}
func mustEnableWiki(ctx *context.APIContext) {
if !ctx.Repo.Permission.CanRead(unit.TypeWiki) {
ctx.APIErrorNotFound()
return
func mustEnableUnit(unitType unit.Type) func(ctx *context.APIContext) {
return func(ctx *context.APIContext) {
if !ctx.Repo.Permission.CanRead(unitType) {
ctx.APIErrorNotFound()
}
}
}
@@ -1393,7 +1394,7 @@ func Routes() *web.Router {
m.Get("/revisions/{pageName}", repo.ListPageRevisions)
m.Post("/new", reqToken(), mustNotBeArchived, reqRepoWriter(unit.TypeWiki), bind(api.CreateWikiPageOptions{}), repo.NewWikiPage)
m.Get("/pages", repo.ListWikiPages)
}, mustEnableWiki)
}, mustEnableUnit(unit.TypeWiki))
m.Post("/markup", reqToken(), bind(api.MarkupOption{}), misc.Markup)
m.Post("/markdown", reqToken(), bind(api.MarkdownOption{}), misc.Markdown)
m.Post("/markdown/raw", reqToken(), misc.MarkdownRaw)
@@ -1425,7 +1426,7 @@ func Routes() *web.Router {
Get(repo.GetReleaseByTag).
Delete(reqToken(), reqRepoWriter(unit.TypeReleases), repo.DeleteReleaseByTag)
})
}, reqRepoReader(unit.TypeReleases))
}, mustEnableUnit(unit.TypeReleases))
m.Post("/mirror-sync", reqToken(), reqRepoWriter(unit.TypeCode), mustNotBeArchived, repo.MirrorSync)
m.Post("/push_mirrors-sync", reqAdmin(), reqToken(), mustNotBeArchived, repo.PushMirrorSync)
m.Group("/push_mirrors", func() {
+2 -2
View File
@@ -87,7 +87,7 @@ func GetIssueCommentReactions(ctx *context.APIContext) {
var result []api.Reaction
for _, r := range reactions {
result = append(result, api.Reaction{
User: convert.ToUser(ctx, r.User, ctx.Doer),
User: convert.ToAuthor(ctx, r.User, r.OriginalAuthor, ctx.Doer),
Reaction: r.Type,
Created: r.CreatedUnix.AsTime(),
})
@@ -300,7 +300,7 @@ func GetIssueReactions(ctx *context.APIContext) {
var result []api.Reaction
for _, r := range reactions {
result = append(result, api.Reaction{
User: convert.ToUser(ctx, r.User, ctx.Doer),
User: convert.ToAuthor(ctx, r.User, r.OriginalAuthor, ctx.Doer),
Reaction: r.Type,
Created: r.CreatedUnix.AsTime(),
})
+1 -1
View File
@@ -50,7 +50,7 @@ func toIssue(ctx context.Context, doer *user_model.User, issue *issues_model.Iss
apiIssue := &api.Issue{
ID: issue.ID,
Index: issue.Index,
Poster: ToUser(ctx, issue.Poster, doer),
Poster: ToAuthor(ctx, issue.Poster, issue.OriginalAuthor, doer),
Title: issue.Title,
Body: issue.Content,
Attachments: toAttachments(ctx, issue.Repo, issue.Attachments, getDownloadURL),
+2 -2
View File
@@ -18,7 +18,7 @@ import (
func ToAPIComment(ctx context.Context, repo *repo_model.Repository, c *issues_model.Comment) *api.Comment {
return &api.Comment{
ID: c.ID,
Poster: ToUser(ctx, c.Poster, nil),
Poster: ToAuthor(ctx, c.Poster, c.OriginalAuthor, nil),
HTMLURL: c.HTMLURL(ctx),
IssueURL: c.IssueURL(ctx),
PRURL: c.PRURL(ctx),
@@ -86,7 +86,7 @@ func ToTimelineComment(ctx context.Context, repo *repo_model.Repository, c *issu
comment := &api.TimelineComment{
ID: c.ID,
Type: c.Type.String(),
Poster: ToUser(ctx, c.Poster, nil),
Poster: ToAuthor(ctx, c.Poster, c.OriginalAuthor, nil),
HTMLURL: c.HTMLURL(ctx),
IssueURL: c.IssueURL(ctx),
PRURL: c.PRURL(ctx),
+2 -2
View File
@@ -23,7 +23,7 @@ func ToPullReview(ctx context.Context, r *issues_model.Review, doer *user_model.
result := &api.PullReview{
ID: r.ID,
Reviewer: ToUser(ctx, r.Reviewer, doer),
Reviewer: ToAuthor(ctx, r.Reviewer, r.OriginalAuthor, doer),
State: api.ReviewStateUnknown,
Body: r.Content,
CommitID: r.CommitID,
@@ -104,7 +104,7 @@ func ToPullReviewComment(ctx context.Context, comment *issues_model.Comment, doe
apiComment := &api.PullReviewComment{
ID: comment.ID,
Body: comment.Content,
Poster: ToUser(ctx, comment.Poster, doer),
Poster: ToAuthor(ctx, comment.Poster, comment.OriginalAuthor, doer),
Resolver: ToUser(ctx, comment.ResolveDoer, doer),
ReviewID: comment.ReviewID,
Created: comment.CreatedUnix.AsTime(),
+1 -1
View File
@@ -28,7 +28,7 @@ func ToAPIRelease(ctx context.Context, repo *repo_model.Repository, r *repo_mode
IsPrerelease: r.IsPrerelease,
CreatedAt: r.CreatedUnix.AsTime(),
PublishedAt: util.Iif(r.IsDraft, nil, r.PublishedUnix.AsTimePtr()),
Publisher: ToUser(ctx, r.Publisher, nil),
Publisher: ToAuthor(ctx, r.Publisher, r.OriginalAuthor, nil),
Attachments: ToAPIAttachments(ctx, repo, r.Attachments),
}
}
+7
View File
@@ -50,6 +50,13 @@ func ToUser(ctx context.Context, user, doer *user_model.User) *api.User {
return toUser(ctx, user, signed, authed)
}
func ToAuthor(ctx context.Context, author *user_model.User, originalAuthor string, doer *user_model.User) *api.User {
if originalAuthor == "" {
return ToUser(ctx, author, doer)
}
return &api.User{ID: user_model.GhostUserID, UserName: originalAuthor, Type: api.UserTypeStringUser, Visibility: api.VisibilityStringPublic, AvatarURL: user_model.NewGhostUser().AvatarLink(ctx)}
}
// ToUsers convert list of user_model.User to list of api.User
func ToUsers(ctx context.Context, doer *user_model.User, users []*user_model.User) []*api.User {
result := make([]*api.User, len(users))
+2
View File
@@ -42,6 +42,8 @@ func ToGitServiceType(value string) structs.GitServiceType {
return structs.CodeCommitService
case "bitbucketserver":
return structs.BitbucketServerService
case "forgejo":
return structs.ForgejoService
default:
return structs.PlainGitService
}
+2
View File
@@ -40,6 +40,8 @@ func TestToGitServiceType(t *testing.T) {
typ: "codecommit", enum: 9,
}, {
typ: "bitbucketserver", enum: 10,
}, {
typ: "forgejo", enum: 11,
}}
for _, test := range tc {
assert.EqualValues(t, test.enum, ToGitServiceType(test.typ))
@@ -0,0 +1,6 @@
Cache-Control: max-age=0, private, must-revalidate
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
{"id":1,"owner":{"id":1,"login":"alice","login_name":"","source_id":0,"full_name":"","email":"alice@example.com","avatar_url":"https://forgejo.example/avatars/c160f8cc69a4f0bf2b0362752353d060","html_url":"https://forgejo.example/alice","language":"","is_admin":false,"last_login":"0001-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":false,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"alice"},"name":"test_repo","full_name":"alice/test_repo","description":"Test repository for testing migration from forgejo to gitea","empty":false,"private":true,"fork":false,"template":false,"parent":null,"mirror":false,"size":3,"language":"","languages_url":"https://forgejo.example/api/v1/repos/alice/test_repo/languages","html_url":"https://forgejo.example/alice/test_repo","url":"https://forgejo.example/api/v1/repos/alice/test_repo","link":"","ssh_url":"ssh://git@forgejo.example/alice/test_repo.git","clone_url":"https://forgejo.example/alice/test_repo.git","original_url":"","website":"https://forgejo.example/website","stars_count":0,"forks_count":0,"watchers_count":1,"open_issues_count":1,"open_pr_counter":2,"release_counter":2,"default_branch":"main","archived":false,"created_at":"2026-09-25T01:13:52Z","updated_at":"2026-09-25T01:14:07Z","archived_at":"1970-01-01T00:00:00Z","permissions":{"admin":true,"push":true,"pull":true},"has_issues":true,"internal_tracker":{"enable_time_tracker":true,"allow_only_contributors_to_track_time":true,"enable_issue_dependencies":true},"has_wiki":true,"has_wiki_contents":false,"wiki_branch":"main","wiki_ssh_url":"ssh://git@forgejo.example/alice/test_repo.wiki.git","wiki_clone_url":"https://forgejo.example/alice/test_repo.wiki.git","globally_editable_wiki":false,"has_pull_requests":true,"has_projects":true,"has_releases":true,"has_packages":true,"has_actions":true,"ignore_whitespace_conflicts":false,"allow_merge_commits":true,"allow_rebase":true,"allow_rebase_explicit":true,"allow_squash_merge":true,"allow_fast_forward_only_merge":true,"allow_rebase_update":true,"default_delete_branch_after_merge":false,"default_merge_style":"merge","default_allow_maintainer_edit":false,"default_update_style":"merge","avatar_url":"","internal":false,"mirror_interval":"","object_format_name":"sha1","mirror_updated":"0001-01-01T00:00:00Z","repo_transfer":null,"topics":["forgejo","migration"]}
@@ -0,0 +1,9 @@
Access-Control-Expose-Headers: X-Total-Count
Cache-Control: max-age=0, private, must-revalidate
Content-Length: 872
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
X-Total-Count: 1
[{"id":3,"html_url":"https://forgejo.example/alice/test_repo/issues/1#issuecomment-3","pull_request_url":"","issue_url":"https://forgejo.example/alice/test_repo/issues/1","user":{"id":2,"login":"bob","login_name":"","source_id":0,"full_name":"","email":"bob@noreply.forgejo.example","avatar_url":"https://forgejo.example/avatars/4b9bb80620f03eb3719e0a061c14283d","html_url":"https://forgejo.example/bob","language":"","is_admin":false,"last_login":"0001-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":false,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"bob"},"original_author":"","original_author_id":0,"body":"a comment","assets":[],"created_at":"2026-09-25T01:13:56Z","updated_at":"2026-09-25T01:13:56Z"}]
@@ -0,0 +1,9 @@
Access-Control-Expose-Headers: X-Total-Count
Cache-Control: max-age=0, private, must-revalidate
Content-Length: 1203
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
X-Total-Count: 2
[{"user":{"id":1,"login":"alice","login_name":"","source_id":0,"full_name":"","email":"alice@example.com","avatar_url":"https://forgejo.example/avatars/c160f8cc69a4f0bf2b0362752353d060","html_url":"https://forgejo.example/alice","language":"","is_admin":true,"last_login":"1970-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":true,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"alice"},"content":"heart","created_at":"2026-09-25T01:13:56Z"},{"user":{"id":1,"login":"alice","login_name":"","source_id":0,"full_name":"","email":"alice@example.com","avatar_url":"https://forgejo.example/avatars/c160f8cc69a4f0bf2b0362752353d060","html_url":"https://forgejo.example/alice","language":"","is_admin":true,"last_login":"1970-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":true,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"alice"},"content":"+1","created_at":"2026-09-25T01:13:56Z"}]
@@ -0,0 +1,9 @@
Access-Control-Expose-Headers: X-Total-Count
Cache-Control: max-age=0, private, must-revalidate
Content-Length: 5
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
X-Total-Count: 0
null
@@ -0,0 +1,9 @@
Access-Control-Expose-Headers: X-Total-Count
Cache-Control: max-age=0, private, must-revalidate
Content-Length: 5
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
X-Total-Count: 0
null
@@ -0,0 +1,9 @@
Access-Control-Expose-Headers: X-Total-Count
Cache-Control: max-age=0, private, must-revalidate
Content-Length: 5
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
X-Total-Count: 0
null
@@ -0,0 +1,7 @@
Cache-Control: max-age=0, private, must-revalidate
Content-Length: 604
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
[{"user":{"id":1,"login":"alice","login_name":"","source_id":0,"full_name":"","email":"alice@example.com","avatar_url":"https://forgejo.example/avatars/c160f8cc69a4f0bf2b0362752353d060","html_url":"https://forgejo.example/alice","language":"","is_admin":true,"last_login":"1970-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":true,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"alice"},"content":"laugh","created_at":"2026-09-25T01:13:56Z"}]
@@ -0,0 +1,9 @@
Access-Control-Expose-Headers: X-Total-Count
Cache-Control: max-age=0, private, must-revalidate
Content-Length: 1518
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
X-Total-Count: 1
[{"id":1,"url":"https://forgejo.example/api/v1/repos/alice/test_repo/issues/1","html_url":"https://forgejo.example/alice/test_repo/issues/1","number":1,"user":{"id":1,"login":"alice","login_name":"","source_id":0,"full_name":"","email":"alice@example.com","avatar_url":"https://forgejo.example/avatars/c160f8cc69a4f0bf2b0362752353d060","html_url":"https://forgejo.example/alice","language":"","is_admin":true,"last_login":"1970-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":true,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"alice"},"original_author":"","original_author_id":0,"title":"first issue","body":"issue body","ref":"","assets":[],"labels":[{"id":1,"name":"kind/bug","exclusive":true,"is_archived":false,"color":"ee0701","description":"Something is broken","url":"https://forgejo.example/api/v1/repos/alice/test_repo/labels/1"}],"milestone":{"id":1,"title":"v1","description":"first","state":"open","open_issues":1,"closed_issues":0,"created_at":"2026-09-25T01:13:54Z","updated_at":"2026-09-25T01:13:56Z","closed_at":null,"due_on":null},"assignee":null,"assignees":null,"state":"open","is_locked":false,"comments":1,"created_at":"2026-09-25T01:13:56Z","updated_at":"2026-09-25T01:13:56Z","closed_at":null,"due_date":null,"pull_request":null,"repository":{"id":1,"name":"test_repo","owner":"alice","full_name":"alice/test_repo"},"pin_order":0}]
@@ -0,0 +1,9 @@
Access-Control-Expose-Headers: X-Total-Count
Cache-Control: max-age=0, private, must-revalidate
Content-Length: 529
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
X-Total-Count: 3
[{"id":1,"name":"kind/bug","exclusive":true,"is_archived":false,"color":"ee0701","description":"Something is broken","url":"https://forgejo.example/api/v1/repos/alice/test_repo/labels/1"},{"id":2,"name":"kind/feature","exclusive":true,"is_archived":true,"color":"0e8a16","description":"","url":"https://forgejo.example/api/v1/repos/alice/test_repo/labels/2"},{"id":3,"name":"question","exclusive":false,"is_archived":false,"color":"cc317c","description":"","url":"https://forgejo.example/api/v1/repos/alice/test_repo/labels/3"}]
@@ -0,0 +1,9 @@
Access-Control-Expose-Headers: X-Total-Count
Cache-Control: max-age=0, private, must-revalidate
Content-Length: 3
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
X-Total-Count: 3
[]
@@ -0,0 +1,7 @@
Cache-Control: max-age=0, private, must-revalidate
Content-Length: 1023
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
[{"id":10,"body":"fix this","user":{"id":2,"login":"bob","login_name":"","source_id":0,"full_name":"","email":"bob@example.com","avatar_url":"https://forgejo.example/avatars/4b9bb80620f03eb3719e0a061c14283d","html_url":"https://forgejo.example/bob","language":"","is_admin":false,"last_login":"1970-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":true,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"bob"},"resolver":null,"pull_request_review_id":2,"created_at":"2026-09-25T01:14:04Z","updated_at":"2026-09-25T01:14:04Z","path":"open.txt","commit_id":"0310ab0fc7512cab1652c77eadb401e0b15fb974","original_commit_id":"","diff_hunk":"@@ -0,0 +1 @@\n+open.txt","position":1,"original_position":0,"extra_lines_count":0,"html_url":"https://forgejo.example/alice/test_repo/pulls/3#issuecomment-10","pull_request_url":"https://forgejo.example/alice/test_repo/pulls/3"}]
@@ -0,0 +1,9 @@
Access-Control-Expose-Headers: X-Total-Count
Cache-Control: max-age=0, private, must-revalidate
Content-Length: 1894
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
X-Total-Count: 2
[{"id":2,"user":{"id":2,"login":"bob","login_name":"","source_id":0,"full_name":"","email":"bob@example.com","avatar_url":"https://forgejo.example/avatars/4b9bb80620f03eb3719e0a061c14283d","html_url":"https://forgejo.example/bob","language":"","is_admin":false,"last_login":"1970-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":true,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"bob"},"team":null,"state":"REQUEST_CHANGES","body":"needs work","commit_id":"0310ab0fc7512cab1652c77eadb401e0b15fb974","stale":false,"official":true,"dismissed":true,"comments_count":1,"submitted_at":"2026-09-25T01:14:04Z","updated_at":"2026-09-25T01:14:05Z","html_url":"https://forgejo.example/alice/test_repo/pulls/3#issuecomment-11","pull_request_url":"https://forgejo.example/alice/test_repo/pulls/3"},{"id":3,"user":{"id":2,"login":"bob","login_name":"","source_id":0,"full_name":"","email":"bob@example.com","avatar_url":"https://forgejo.example/avatars/4b9bb80620f03eb3719e0a061c14283d","html_url":"https://forgejo.example/bob","language":"","is_admin":false,"last_login":"1970-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":true,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"bob"},"team":null,"state":"COMMENT","body":"just a comment","commit_id":"0310ab0fc7512cab1652c77eadb401e0b15fb974","stale":false,"official":false,"dismissed":false,"comments_count":0,"submitted_at":"2026-09-25T01:14:05Z","updated_at":"2026-09-25T01:14:05Z","html_url":"https://forgejo.example/alice/test_repo/pulls/3#issuecomment-13","pull_request_url":"https://forgejo.example/alice/test_repo/pulls/3"}]
@@ -0,0 +1,8 @@
Access-Control-Expose-Headers: X-Total-Count
Cache-Control: max-age=0, private, must-revalidate
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
X-Total-Count: 3
[{"id":3,"url":"https://forgejo.example/api/v1/repos/alice/test_repo/pulls/4","number":4,"user":{"id":2,"login":"bob","login_name":"","source_id":0,"full_name":"","email":"bob@example.com","avatar_url":"https://forgejo.example/avatars/4b9bb80620f03eb3719e0a061c14283d","html_url":"https://forgejo.example/bob","language":"","is_admin":false,"last_login":"1970-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":true,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"bob"},"title":"agit pull","body":"agit body","labels":[],"milestone":null,"assignee":null,"assignees":null,"requested_reviewers":[],"requested_reviewers_teams":[],"state":"open","draft":false,"is_locked":false,"comments":0,"review_comments":0,"additions":0,"deletions":0,"changed_files":0,"html_url":"https://forgejo.example/alice/test_repo/pulls/4","diff_url":"https://forgejo.example/alice/test_repo/pulls/4.diff","patch_url":"https://forgejo.example/alice/test_repo/pulls/4.patch","mergeable":true,"merged":false,"merged_at":null,"merge_commit_sha":null,"merged_by":null,"allow_maintainer_edit":false,"base":{"label":"main","ref":"main","sha":"588ba39bdea707266f2df950dcee1bb5a1764fed","repo_id":1,"repo":{"id":1,"owner":{"id":1,"login":"alice","login_name":"","source_id":0,"full_name":"","email":"alice@example.com","avatar_url":"https://forgejo.example/avatars/c160f8cc69a4f0bf2b0362752353d060","html_url":"https://forgejo.example/alice","language":"","is_admin":false,"last_login":"0001-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":false,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"alice"},"name":"test_repo","full_name":"alice/test_repo","description":"Test repository for testing migration from forgejo to gitea","empty":false,"private":true,"fork":false,"template":false,"parent":null,"mirror":false,"size":3,"language":"","languages_url":"https://forgejo.example/api/v1/repos/alice/test_repo/languages","html_url":"https://forgejo.example/alice/test_repo","url":"https://forgejo.example/api/v1/repos/alice/test_repo","link":"","ssh_url":"ssh://git@forgejo.example/alice/test_repo.git","clone_url":"https://forgejo.example/alice/test_repo.git","original_url":"","website":"https://forgejo.example/website","stars_count":0,"forks_count":0,"watchers_count":1,"open_issues_count":1,"open_pr_counter":2,"release_counter":2,"default_branch":"main","archived":false,"created_at":"2026-09-25T01:13:52Z","updated_at":"2026-09-25T01:14:07Z","archived_at":"1970-01-01T00:00:00Z","permissions":{"admin":true,"push":true,"pull":true},"has_issues":true,"internal_tracker":{"enable_time_tracker":true,"allow_only_contributors_to_track_time":true,"enable_issue_dependencies":true},"has_wiki":true,"has_wiki_contents":false,"wiki_branch":"main","wiki_ssh_url":"ssh://git@forgejo.example/alice/test_repo.wiki.git","wiki_clone_url":"https://forgejo.example/alice/test_repo.wiki.git","globally_editable_wiki":false,"has_pull_requests":true,"has_projects":true,"has_releases":true,"has_packages":true,"has_actions":true,"ignore_whitespace_conflicts":false,"allow_merge_commits":true,"allow_rebase":true,"allow_rebase_explicit":true,"allow_squash_merge":true,"allow_fast_forward_only_merge":true,"allow_rebase_update":true,"default_delete_branch_after_merge":false,"default_merge_style":"merge","default_allow_maintainer_edit":false,"default_update_style":"merge","avatar_url":"","internal":false,"mirror_interval":"","object_format_name":"sha1","mirror_updated":"0001-01-01T00:00:00Z","repo_transfer":null,"topics":["forgejo","migration"]}},"head":{"label":"bob/agit","ref":"refs/pull/4/head","sha":"171139b0df47db58a099ff2643ac1088776ff13c","repo_id":1,"repo":{"id":1,"owner":{"id":1,"login":"alice","login_name":"","source_id":0,"full_name":"","email":"alice@example.com","avatar_url":"https://forgejo.example/avatars/c160f8cc69a4f0bf2b0362752353d060","html_url":"https://forgejo.example/alice","language":"","is_admin":false,"last_login":"0001-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":false,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"alice"},"name":"test_repo","full_name":"alice/test_repo","description":"Test repository for testing migration from forgejo to gitea","empty":false,"private":true,"fork":false,"template":false,"parent":null,"mirror":false,"size":3,"language":"","languages_url":"https://forgejo.example/api/v1/repos/alice/test_repo/languages","html_url":"https://forgejo.example/alice/test_repo","url":"https://forgejo.example/api/v1/repos/alice/test_repo","link":"","ssh_url":"ssh://git@forgejo.example/alice/test_repo.git","cLine truncated
@@ -0,0 +1,8 @@
Access-Control-Expose-Headers: X-Total-Count
Cache-Control: max-age=0, private, must-revalidate
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
X-Total-Count: 2
[{"id":2,"tag_name":"v0.9.0","target_commitish":"main","name":"v0.9.0","body":"","url":"https://forgejo.example/api/v1/repos/alice/test_repo/releases/2","html_url":"https://forgejo.example/alice/test_repo/releases/tag/v0.9.0","tarball_url":"https://forgejo.example/alice/test_repo/archive/v0.9.0.tar.gz","zipball_url":"https://forgejo.example/alice/test_repo/archive/v0.9.0.zip","hide_archive_links":false,"upload_url":"https://forgejo.example/api/v1/repos/alice/test_repo/releases/2/assets","draft":false,"prerelease":true,"created_at":"2026-09-25T01:13:55Z","published_at":"2026-09-25T01:13:55Z","author":{"id":1,"login":"alice","login_name":"","source_id":0,"full_name":"","email":"alice@noreply.forgejo.example","avatar_url":"https://forgejo.example/avatars/c160f8cc69a4f0bf2b0362752353d060","html_url":"https://forgejo.example/alice","language":"","is_admin":false,"last_login":"0001-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":false,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"alice"},"assets":[],"archive_download_count":{"zip":0,"tar_gz":0}},{"id":1,"tag_name":"v1.0.0","target_commitish":"main","name":"v1.0.0","body":"first release","url":"https://forgejo.example/api/v1/repos/alice/test_repo/releases/1","html_url":"https://forgejo.example/alice/test_repo/releases/tag/v1.0.0","tarball_url":"https://forgejo.example/alice/test_repo/archive/v1.0.0.tar.gz","zipball_url":"https://forgejo.example/alice/test_repo/archive/v1.0.0.zip","hide_archive_links":false,"upload_url":"https://forgejo.example/api/v1/repos/alice/test_repo/releases/1/assets","draft":false,"prerelease":false,"created_at":"2026-09-25T01:13:55Z","published_at":"2026-09-25T01:13:55Z","author":{"id":1,"login":"alice","login_name":"","source_id":0,"full_name":"","email":"alice@noreply.forgejo.example","avatar_url":"https://forgejo.example/avatars/c160f8cc69a4f0bf2b0362752353d060","html_url":"https://forgejo.example/alice","language":"","is_admin":false,"last_login":"0001-01-01T00:00:00Z","created":"2026-09-25T01:13:51Z","restricted":false,"active":false,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":0,"following_count":0,"starred_repos_count":0,"username":"alice"},"assets":[{"id":1,"name":"asset.txt","size":14,"download_count":0,"created_at":"2026-09-25T01:13:55Z","uuid":"4e5c9b2d-b858-48af-9ba7-8260f1b53a04","browser_download_url":"https://forgejo.example/alice/test_repo/releases/download/v1.0.0/asset.txt","type":"attachment"},{"id":2,"name":"download.zip","size":0,"download_count":0,"created_at":"2026-09-25T01:13:55Z","uuid":"5e8b7425-31bb-4103-808b-1907aacaddcb","browser_download_url":"https://example.com/download.zip","type":"external"}],"archive_download_count":{"zip":0,"tar_gz":0}}]
@@ -0,0 +1,7 @@
Cache-Control: max-age=0, private, must-revalidate
Content-Length: 115
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
{"max_response_items":3,"default_paging_num":3,"default_git_trees_per_page":1000,"default_max_blob_size":10485760}
@@ -0,0 +1,7 @@
Cache-Control: max-age=0, private, must-revalidate
Content-Length: 53
Content-Type: application/json;charset=utf-8
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
{"version":"17.0.0-dev-576-994e539b55+gitea-1.22.0"}
@@ -0,0 +1,12 @@
Accept-Ranges: bytes
Access-Control-Expose-Headers: Content-Disposition
Cache-Control: private, max-age=300
Content-Disposition: inline; filename="asset.txt"; filename*=UTF-8''asset.txt
Content-Length: 14
Content-Type: text/plain; charset=utf-8
Etag: "4e5c9b2d-b858-48af-9ba7-8260f1b53a04"
Last-Modified: Fri, 25 Sep 2026 01:13:55 GMT
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
forgejo asset
@@ -1,8 +0,0 @@
Alt-Svc: h3=":443"; ma=2592000
Cache-Control: max-age=0, private, must-revalidate, no-transform
Content-Length: 2
Content-Type: application/json;charset=utf-8
Vary: Origin
X-Content-Type-Options: nosniff
[]
@@ -1,8 +0,0 @@
Alt-Svc: h3=":443"; ma=2592000
Cache-Control: max-age=0, private, must-revalidate, no-transform
Content-Length: 2
Content-Type: application/json;charset=utf-8
Vary: Origin
X-Content-Type-Options: nosniff
[]
+1 -1
View File
@@ -113,7 +113,7 @@ type BitbucketServerDownloader struct {
// NewBitbucketServerDownloader creates a Bitbucket Server downloader
func NewBitbucketServerDownloader(_ context.Context, baseURL, username, password, token, projectKey, repoSlug string) *BitbucketServerDownloader {
return &BitbucketServerDownloader{
client: getMigrationHTTPClient(),
client: &http.Client{Transport: getMigrationTransport()}, // not retrying, callAPI retries rate limits itself
sleep: bitbucketServerSleep,
baseURL: strings.TrimSuffix(baseURL, "/"),
projectKey: projectKey,
+2 -10
View File
@@ -78,22 +78,14 @@ type CodebaseDownloader struct {
// NewCodebaseDownloader creates a new downloader
func NewCodebaseDownloader(_ context.Context, projectURL *url.URL, project, repoName, username, password string) *CodebaseDownloader {
baseURL, _ := url.Parse("https://api3.codebasehq.com")
transport := NewMigrationHTTPTransport()
downloader := &CodebaseDownloader{
baseURL: baseURL,
projectURL: projectURL,
project: project,
repoName: repoName,
client: &http.Client{
Transport: roundTripperFunc(func(req *http.Request) (*http.Response, error) {
if username != "" && password != "" {
req = req.Clone(req.Context())
req.SetBasicAuth(username, password)
}
return transport.RoundTrip(req)
}),
},
client: newMigrationHTTPClient(baseURL.String(), basicAuthorization(username, password)),
userMap: make(map[int64]*codebaseUser),
commitMap: make(map[string]string),
}
+19
View File
@@ -4,7 +4,10 @@
package migrations
import (
"context"
"fmt"
"io"
"net/http"
"strings"
system_model "gitea.dev/models/system"
@@ -21,6 +24,22 @@ func WarnAndNotice(fmtStr string, args ...any) {
}
}
func downloadAsset(ctx context.Context, client *http.Client, assetURL string) (io.ReadCloser, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, assetURL, nil)
if err != nil {
return nil, err
}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
return nil, fmt.Errorf("unexpected status %s", resp.Status)
}
return resp.Body, nil
}
func hasBaseURL(toCheck, baseURL string) bool {
if baseURL == "" {
return false
+22 -47
View File
@@ -8,7 +8,6 @@ import (
"errors"
"fmt"
"io"
"net/url"
"os"
"path/filepath"
"strconv"
@@ -87,22 +86,6 @@ func (g *RepositoryDumper) reviewDir() string {
return filepath.Join(g.baseDir, "reviews")
}
func (g *RepositoryDumper) setURLToken(remoteAddr string) (string, error) {
if len(g.opts.AuthToken) > 0 || len(g.opts.AuthUsername) > 0 {
u, err := url.Parse(remoteAddr)
if err != nil {
return "", err
}
u.User = url.UserPassword(g.opts.AuthUsername, g.opts.AuthPassword)
if len(g.opts.AuthToken) > 0 {
u.User = url.UserPassword("oauth2", g.opts.AuthToken)
}
remoteAddr = u.String()
}
return remoteAddr, nil
}
// CreateRepo creates a repository
func (g *RepositoryDumper) CreateRepo(ctx context.Context, repo *base.Repository, opts base.MigrateOptions) error {
f, err := os.Create(filepath.Join(g.baseDir, "repo.yml"))
@@ -147,7 +130,7 @@ func (g *RepositoryDumper) CreateRepo(ctx context.Context, repo *base.Repository
migrateTimeout := 2 * time.Hour
remoteAddr, err := g.setURLToken(repo.CloneURL)
remoteAddr, err := base.NullDownloader{}.FormatCloneURL(g.opts, repo.CloneURL)
if err != nil {
return err
}
@@ -294,41 +277,31 @@ func (g *RepositoryDumper) CreateLabels(_ context.Context, labels ...*base.Label
}
// CreateReleases creates releases
func (g *RepositoryDumper) CreateReleases(_ context.Context, releases ...*base.Release) error {
func (g *RepositoryDumper) CreateReleases(ctx context.Context, releases ...*base.Release) error {
if g.opts.ReleaseAssets {
for _, release := range releases {
attachDir := filepath.Join("release_assets", uuid.New().String())
if err := os.MkdirAll(filepath.Join(g.baseDir, attachDir), os.ModePerm); err != nil {
return err
}
assets := release.Assets[:0]
for _, asset := range release.Assets {
// SECURITY: We cannot check the DownloadFunc is safe here
// ... we must assume that it is safe and simply download the attachment
rc, err := asset.DownloadFunc()
if err != nil {
if ctx.Err() != nil {
return ctx.Err()
}
WarnAndNotice("Skipped asset %s of release %s: %v", asset.Name, release.TagName, err)
continue
}
// we cannot use asset.Name because it might contains special characters.
attachLocalPath := filepath.Join(attachDir, uuid.New().String())
// SECURITY: We cannot check the DownloadURL and DownloadFunc are safe here
// ... we must assume that they are safe and simply download the attachment
// download attachment
err := func(attachPath string) error {
var rc io.ReadCloser
var err error
if asset.DownloadURL == nil {
rc, err = asset.DownloadFunc()
if err != nil {
return err
}
defer rc.Close()
} else {
// use the migration client so the fetch (including any redirect) is
// validated against the migration host allow/block list
resp, err := getMigrationHTTPClient().Get(*asset.DownloadURL)
if err != nil {
return err
}
defer resp.Body.Close()
rc = resp.Body
}
fw, err := os.Create(attachPath)
err = func() error {
defer rc.Close()
fw, err := os.Create(filepath.Join(g.baseDir, attachLocalPath))
if err != nil {
return fmt.Errorf("create: %w", err)
}
@@ -336,12 +309,14 @@ func (g *RepositoryDumper) CreateReleases(_ context.Context, releases ...*base.R
_, err = io.Copy(fw, rc)
return err
}(filepath.Join(g.baseDir, attachLocalPath))
}()
if err != nil {
return err
return fmt.Errorf("release %s asset %s: %w", release.TagName, asset.Name, err)
}
asset.DownloadURL = &attachLocalPath // to save the filepath on the yml file, change the source
assets = append(assets, asset)
}
release.Assets = assets
}
}
@@ -454,7 +429,7 @@ func (g *RepositoryDumper) handlePullRequest(ctx context.Context, pr *base.PullR
// SECURITY: We will assume that the pr.PatchURL has been checked
// pr.PatchURL maybe a local file - but note EnsureSafe should be asserting that this safe
u, err := g.setURLToken(pr.PatchURL)
u, err := base.NullDownloader{}.FormatCloneURL(g.opts, pr.PatchURL)
if err != nil {
return err
}
+19
View File
@@ -0,0 +1,19 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package migrations
import "gitea.dev/modules/structs"
func init() {
RegisterDownloaderFactory(&ForgejoDownloaderFactory{})
}
// ForgejoDownloaderFactory uses the Gitea downloader, as Forgejo keeps a Gitea compatible API
type ForgejoDownloaderFactory struct {
GiteaDownloaderFactory
}
func (f *ForgejoDownloaderFactory) GitServiceType() structs.GitServiceType {
return structs.ForgejoService
}
+69 -42
View File
@@ -4,6 +4,7 @@
package migrations
import (
"cmp"
"context"
"errors"
"fmt"
@@ -17,7 +18,10 @@ import (
"gitea.dev/modules/log"
base "gitea.dev/modules/migration"
"gitea.dev/modules/structs"
"gitea.dev/modules/util"
gitea_sdk "gitea.dev/sdk"
"github.com/hashicorp/go-version"
)
var (
@@ -72,6 +76,7 @@ type GiteaDownloader struct {
baseURL string
repoOwner string
repoName string
httpClient *http.Client
pagination bool
maxPerPage int
}
@@ -81,16 +86,15 @@ type GiteaDownloader struct {
// Use either a username/password or personal token. token is preferred
// Note: Public access only allows very basic access
func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, password, token string) (*GiteaDownloader, error) {
giteaClient, err := gitea_sdk.NewClient(
baseURL,
gitea_sdk.SetToken(token),
gitea_sdk.SetBasicAuth(username, password),
gitea_sdk.SetHTTPClient(newMigrationHTTPClient()),
)
httpClient := newMigrationHTTPClient(baseURL, util.Iif(token != "", "token "+token, basicAuthorization(username, password)))
giteaClient, err := gitea_sdk.NewClient(baseURL, gitea_sdk.SetHTTPClient(httpClient))
if err != nil {
log.Error(fmt.Sprintf("Failed to create NewGiteaDownloader for: %s. Error: %v", baseURL, err))
return nil, err
}
if err = pinGiteaVersion(ctx, giteaClient); err != nil {
return nil, err
}
path := strings.Split(repoPath, "/")
@@ -117,11 +121,36 @@ func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, passwo
baseURL: baseURL,
repoOwner: path[0],
repoName: path[1],
httpClient: httpClient,
pagination: paginationSupport,
maxPerPage: maxPerPage,
}, nil
}
func pinGiteaVersion(ctx context.Context, client *gitea_sdk.Client) error { // the SDK's checks reject pre-releases
rawVersion, _, err := client.Meta.ServerVersion(ctx)
if err != nil {
return err
}
if serverVersion, err := version.NewVersion(rawVersion); err == nil {
if giteaVersion, ok := strings.CutPrefix(serverVersion.Metadata(), "gitea-"); ok { // Forgejo
if serverVersion, err = version.NewVersion(giteaVersion); err != nil {
return err
}
}
return gitea_sdk.SetGiteaVersion(serverVersion.Core().String())(client)
}
return nil
}
func (g *GiteaDownloader) unitDisabled(resp *gitea_sdk.Response, hasUnit func(*gitea_sdk.Repository) bool) bool { // Gitea answers a disabled unit and missing access alike with 403 or 404, Forgejo with 404
if resp == nil || (resp.StatusCode != http.StatusNotFound && resp.StatusCode != http.StatusForbidden) {
return false
}
repo, _, err := g.client.Repositories.GetRepo(g.ctx, g.repoOwner, g.repoName)
return err == nil && !hasUnit(repo)
}
// String implements Stringer
func (g *GiteaDownloader) String() string {
return fmt.Sprintf("migration from gitea server %s %s/%s", g.baseURL, g.repoOwner, g.repoName)
@@ -225,6 +254,8 @@ func (g *GiteaDownloader) convertGiteaLabel(label *gitea_sdk.Label) *base.Label
Name: label.Name,
Color: label.Color,
Description: label.Description,
Exclusive: label.Exclusive,
Archived: label.IsArchived,
}
}
@@ -273,11 +304,7 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele
Created: rel.CreatedAt,
}
httpClient := newMigrationHTTPClient()
for _, asset := range rel.Attachments {
assetID := asset.ID // Don't optimize this, for closure we need a local variable
assetDownloadURL := asset.DownloadURL
size := int(asset.Size)
dlCount := int(asset.DownloadCount)
r.Assets = append(r.Assets, &base.ReleaseAsset{
@@ -286,30 +313,8 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele
Size: &size,
DownloadCount: &dlCount,
Created: asset.Created,
DownloadURL: &asset.DownloadURL,
DownloadFunc: func() (io.ReadCloser, error) {
asset, _, err := g.client.Releases.GetReleaseAttachment(g.ctx, g.repoOwner, g.repoName, rel.ID, assetID)
if err != nil {
return nil, err
}
if !hasBaseURL(assetDownloadURL, g.baseURL) {
WarnAndNotice("Unexpected AssetURL for assetID[%d] in %s: %s", assetID, g, assetDownloadURL)
return io.NopCloser(strings.NewReader(asset.DownloadURL)), nil
}
// FIXME: for a private download?
req, err := http.NewRequest(http.MethodGet, assetDownloadURL, nil)
if err != nil {
return nil, err
}
resp, err := httpClient.Do(req)
if err != nil {
return nil, err
}
// resp.Body is closed by the uploader
return resp.Body, nil
return downloadAsset(g.ctx, g.httpClient, g.baseURL+"/attachments/"+url.PathEscape(asset.UUID))
},
})
}
@@ -328,10 +333,13 @@ func (g *GiteaDownloader) GetReleases(ctx context.Context) ([]*base.Release, err
default:
}
rl, _, err := g.client.Releases.ListReleases(g.ctx, g.repoOwner, g.repoName, gitea_sdk.ListReleasesOptions{ListOptions: gitea_sdk.ListOptions{
rl, resp, err := g.client.Releases.ListReleases(g.ctx, g.repoOwner, g.repoName, gitea_sdk.ListReleasesOptions{ListOptions: gitea_sdk.ListOptions{
PageSize: g.maxPerPage,
Page: i,
}})
if g.unitDisabled(resp, func(repo *gitea_sdk.Repository) bool { return repo.HasReleases }) {
return releases, nil
}
if err != nil {
return nil, err
}
@@ -413,11 +421,14 @@ func (g *GiteaDownloader) GetIssues(ctx context.Context, page, perPage int) ([]*
}
allIssues := make([]*base.Issue, 0, perPage)
issues, _, err := g.client.Issues.ListRepoIssues(g.ctx, g.repoOwner, g.repoName, gitea_sdk.ListIssueOption{
issues, resp, err := g.client.Issues.ListRepoIssues(g.ctx, g.repoOwner, g.repoName, gitea_sdk.ListIssueOption{
ListOptions: gitea_sdk.ListOptions{Page: page, PageSize: perPage},
State: gitea_sdk.StateAll,
Type: gitea_sdk.IssueTypeIssue,
})
if g.unitDisabled(resp, func(repo *gitea_sdk.Repository) bool { return repo.HasIssues && repo.ExternalTracker == nil }) {
return nil, true, nil
}
if err != nil {
return nil, false, fmt.Errorf("error while listing issues: %w", err)
}
@@ -459,6 +470,7 @@ func (g *GiteaDownloader) GetIssues(ctx context.Context, page, perPage int) ([]*
Assignees: assignees,
IsLocked: issue.IsLocked,
ForeignIndex: issue.Index,
Context: issue.Comments,
})
}
@@ -471,6 +483,9 @@ func (g *GiteaDownloader) GetIssues(ctx context.Context, page, perPage int) ([]*
// GetComments returns comments according issueNumber
func (g *GiteaDownloader) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) {
if commentable.GetContext() == 0 {
return nil, true, nil
}
allComments := make([]*base.Comment, 0, g.maxPerPage)
seenIDs := container.Set[int64]{}
@@ -527,13 +542,16 @@ func (g *GiteaDownloader) GetPullRequests(ctx context.Context, page, perPage int
}
allPRs := make([]*base.PullRequest, 0, perPage)
prs, _, err := g.client.PullRequests.ListRepoPullRequests(g.ctx, g.repoOwner, g.repoName, gitea_sdk.ListPullRequestsOptions{
prs, resp, err := g.client.PullRequests.ListRepoPullRequests(g.ctx, g.repoOwner, g.repoName, gitea_sdk.ListPullRequestsOptions{
ListOptions: gitea_sdk.ListOptions{
Page: page,
PageSize: perPage,
},
State: gitea_sdk.StateAll,
})
if g.unitDisabled(resp, func(repo *gitea_sdk.Repository) bool { return repo.HasPullRequests }) {
return nil, true, nil
}
if err != nil {
return nil, false, fmt.Errorf("error while listing pull requests (page: %d, pagesize: %d). Error: %w", page, perPage, err)
}
@@ -585,7 +603,7 @@ func (g *GiteaDownloader) GetPullRequests(ctx context.Context, page, perPage int
createdAt = *pr.Created
}
updatedAt := time.Time{}
if pr.Created != nil {
if pr.Updated != nil {
updatedAt = *pr.Updated
}
@@ -613,7 +631,6 @@ func (g *GiteaDownloader) GetPullRequests(ctx context.Context, page, perPage int
MergedTime: pr.Merged,
MergeCommitSHA: mergeCommitSHA,
IsLocked: pr.IsLocked,
PatchURL: pr.PatchURL,
Head: base.PullRequestBranch{
Ref: headRef,
SHA: headSHA,
@@ -623,11 +640,12 @@ func (g *GiteaDownloader) GetPullRequests(ctx context.Context, page, perPage int
},
Base: base.PullRequestBranch{
Ref: pr.Base.Ref,
SHA: pr.Base.Sha,
SHA: cmp.Or(pr.MergeBase, pr.Base.Sha), // base.sha is the base branch head, which contains the changes once merged
RepoName: g.repoName,
OwnerName: g.repoOwner,
},
ForeignIndex: pr.Index,
Context: pr.Comments,
})
// SECURITY: Ensure that the PR is safe
_ = CheckAndEnsureSafePR(allPRs[len(allPRs)-1], g.baseURL, g)
@@ -672,9 +690,11 @@ func (g *GiteaDownloader) GetReviews(ctx context.Context, reviewable base.Review
continue
}
rcl, _, err := g.client.PullRequests.ListPullReviewComments(g.ctx, g.repoOwner, g.repoName, reviewable.GetForeignIndex(), pr.ID)
if err != nil {
return nil, err
var rcl []*gitea_sdk.PullReviewComment
if pr.CodeCommentsCount > 0 {
if rcl, _, err = g.client.PullRequests.ListPullReviewComments(g.ctx, g.repoOwner, g.repoName, reviewable.GetForeignIndex(), pr.ID); err != nil {
return nil, err
}
}
var reviewComments []*base.ReviewComment
for i := range rcl {
@@ -706,8 +726,15 @@ func (g *GiteaDownloader) GetReviews(ctx context.Context, reviewable base.Review
Content: pr.Body,
CreatedAt: pr.Submitted,
State: string(pr.State),
Dismissed: pr.Dismissed,
Comments: reviewComments,
}
switch pr.State {
case gitea_sdk.ReviewStateComment:
review.State = base.ReviewStateCommented
case gitea_sdk.ReviewStateRequestChanges:
review.State = base.ReviewStateChangesRequested
}
allReviews = append(allReviews, review)
}
@@ -0,0 +1,173 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package migrations
import (
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"runtime"
"testing"
"gitea.dev/models/unittest"
base "gitea.dev/modules/migration"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestForgejoDownloadRepo(t *testing.T) {
token := os.Getenv("FORGEJO_TEST_TOKEN")
_, callerFile, _, _ := runtime.Caller(0)
mockServer := unittest.NewMockWebServer(t, "http://localhost:3000", filepath.Join(filepath.Dir(callerFile), "_mock_data/TestForgejoDownloadRepo"), token != "")
ctx := t.Context()
downloader, err := NewGiteaDownloader(ctx, mockServer.URL, "alice/test_repo", "", "", token)
require.NoError(t, err)
assert.True(t, downloader.pagination)
repo, err := downloader.GetRepoInfo(ctx)
require.NoError(t, err)
assertRepositoryEqual(t, &base.Repository{
Name: "test_repo",
Owner: "alice",
IsPrivate: true,
Description: "Test repository for testing migration from forgejo to gitea",
Website: "https://forgejo.example/website",
CloneURL: "https://forgejo.example/alice/test_repo.git",
OriginalURL: "https://forgejo.example/alice/test_repo",
DefaultBranch: "main",
}, repo)
labels, err := downloader.GetLabels(ctx)
require.NoError(t, err)
assertLabelsEqual(t, []*base.Label{
{Name: "kind/bug", Color: "ee0701", Description: "Something is broken", Exclusive: true},
{Name: "kind/feature", Color: "0e8a16", Exclusive: true, Archived: true},
{Name: "question", Color: "cc317c"},
}, labels)
releases, err := downloader.GetReleases(ctx)
require.NoError(t, err)
require.Len(t, releases, 2)
assert.Equal(t, "v0.9.0", releases[0].TagName)
assert.True(t, releases[0].Prerelease)
assert.Equal(t, "v1.0.0", releases[1].TagName)
require.Len(t, releases[1].Assets, 2)
rc, err := releases[1].Assets[0].DownloadFunc()
require.NoError(t, err)
content, err := io.ReadAll(rc)
require.NoError(t, err)
require.NoError(t, rc.Close())
assert.Equal(t, "forgejo asset\n", string(content))
issues, isEnd, err := downloader.GetIssues(ctx, 1, 50)
require.NoError(t, err)
assert.True(t, isEnd)
require.Len(t, issues, 1)
assert.Equal(t, "first issue", issues[0].Title)
assert.Equal(t, "v1", issues[0].Milestone)
assert.Equal(t, []*base.Reaction{{UserID: 1, UserName: "alice", Content: "heart"}, {UserID: 1, UserName: "alice", Content: "+1"}}, issues[0].Reactions)
comments, _, err := downloader.GetComments(ctx, issues[0])
require.NoError(t, err)
require.Len(t, comments, 1)
assert.Equal(t, "a comment", comments[0].Content)
assert.Equal(t, []*base.Reaction{{UserID: 1, UserName: "alice", Content: "laugh"}}, comments[0].Reactions)
prs, _, err := downloader.GetPullRequests(ctx, 1, 50)
require.NoError(t, err)
require.Len(t, prs, 3)
agit, open, merged := prs[0], prs[1], prs[2]
assert.Equal(t, "refs/pull/4/head", agit.Head.Ref)
assert.Equal(t, "feature-open", open.Head.Ref)
assert.True(t, merged.Merged)
assert.Equal(t, "refs/pull/2/head", merged.Head.Ref)
assert.Equal(t, "23181c5c8c25740ab7f080faf27cbcae7fd13519", merged.Base.SHA)
assert.Equal(t, "588ba39bdea707266f2df950dcee1bb5a1764fed", merged.MergeCommitSHA)
comments, _, err = downloader.GetComments(ctx, agit)
require.NoError(t, err)
assert.Empty(t, comments)
reviews, err := downloader.GetReviews(ctx, open)
require.NoError(t, err)
require.Len(t, reviews, 2)
assert.Equal(t, base.ReviewStateChangesRequested, reviews[0].State)
assert.True(t, reviews[0].Dismissed)
require.Len(t, reviews[0].Comments, 1)
assert.Equal(t, "fix this", reviews[0].Comments[0].Content)
assert.Equal(t, base.ReviewStateCommented, reviews[1].State)
assert.False(t, reviews[1].Dismissed)
}
func TestGiteaDownloaderServerVersion(t *testing.T) {
for rawVersion, pagination := range map[string]bool{
"1.11.0": false,
"1.12.0-rc1": true,
"11.0.0+gitea-1.11.0": false,
"development": false,
} {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/v1/version" {
http.NotFound(w, r)
return
}
_, _ = fmt.Fprintf(w, `{"version":%q}`, rawVersion)
}))
downloader, err := NewGiteaDownloader(t.Context(), server.URL, "owner/repo", "", "", "")
server.Close()
require.NoError(t, err)
assert.Equal(t, pagination, downloader.pagination, rawVersion)
}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
http.Error(w, "unauthorized", http.StatusUnauthorized)
}))
defer server.Close()
_, err := NewGiteaDownloader(t.Context(), server.URL, "owner/repo", "", "", "bad-token")
assert.Error(t, err)
}
func TestGiteaDownloaderDisabledUnits(t *testing.T) {
for _, testCase := range []struct {
unitsEnabled bool
externalTracker string
}{
{false, "null"},
{true, "null"},
{true, "{}"},
} {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api/v1/version":
_, _ = w.Write([]byte(`{"version":"1.27.0"}`))
case "/api/v1/repos/owner/repo":
_, _ = fmt.Fprintf(w, `{"has_issues":%[1]t,"has_pull_requests":%[1]t,"has_releases":%[1]t,"external_tracker":%[2]s}`, testCase.unitsEnabled, testCase.externalTracker)
case "/api/v1/repos/owner/repo/releases":
http.Error(w, "forbidden", http.StatusForbidden)
default:
http.NotFound(w, r)
}
}))
ctx := t.Context()
downloader, err := NewGiteaDownloader(ctx, server.URL, "owner/repo", "", "", "")
require.NoError(t, err)
_, issuesEnd, issuesErr := downloader.GetIssues(ctx, 1, 10)
_, prsEnd, prsErr := downloader.GetPullRequests(ctx, 1, 10)
_, releasesErr := downloader.GetReleases(ctx)
server.Close()
assert.Equal(t, testCase.unitsEnabled && testCase.externalTracker == "null", issuesErr != nil)
assert.Equal(t, testCase.unitsEnabled, prsErr != nil)
assert.Equal(t, testCase.unitsEnabled, releasesErr != nil)
assert.Equal(t, !testCase.unitsEnabled || testCase.externalTracker != "null", issuesEnd)
assert.Equal(t, !testCase.unitsEnabled, prsEnd)
}
}
+6 -4
View File
@@ -247,7 +247,7 @@ func TestGiteaDownloadRepo(t *testing.T) {
Base: base.PullRequestBranch{
CloneURL: "",
Ref: "master",
SHA: "827aa28a907853e5ddfa40c8f9bc52471a2685fd",
SHA: "d9e165e4c7ab6b701f0205d0ffb637e5d2856297",
RepoName: "test_repo",
OwnerName: "gitea",
},
@@ -261,7 +261,6 @@ func TestGiteaDownloadRepo(t *testing.T) {
Merged: true,
MergedTime: new(time.Unix(1598982934, 0)),
MergeCommitSHA: "827aa28a907853e5ddfa40c8f9bc52471a2685fd",
PatchURL: mockServer.URL + "/gitea/test_repo/pulls/12.patch",
}, prs[1])
reviews, err := downloader.GetReviews(ctx, &base.Issue{Number: 7, ForeignIndex: 7})
@@ -274,7 +273,8 @@ func TestGiteaDownloadRepo(t *testing.T) {
ReviewerName: "6543",
CommitID: "187ece0cb6631e2858a6872e5733433bb3ca3b03",
CreatedAt: time.Date(2020, 9, 1, 16, 12, 58, 0, time.UTC),
State: "COMMENT", // TODO
State: base.ReviewStateCommented,
Dismissed: true,
Comments: []*base.ReviewComment{
{
ID: 116561,
@@ -299,7 +299,8 @@ func TestGiteaDownloadRepo(t *testing.T) {
ReviewerName: "techknowlogick",
CommitID: "187ece0cb6631e2858a6872e5733433bb3ca3b03",
CreatedAt: time.Date(2020, 9, 1, 17, 6, 47, 0, time.UTC),
State: "REQUEST_CHANGES", // TODO
State: base.ReviewStateChangesRequested,
Dismissed: true,
Content: "I think this needs some changes",
},
{
@@ -310,6 +311,7 @@ func TestGiteaDownloadRepo(t *testing.T) {
CommitID: "187ece0cb6631e2858a6872e5733433bb3ca3b03",
CreatedAt: time.Date(2020, 9, 1, 17, 19, 51, 0, time.UTC),
State: base.ReviewStateApproved,
Dismissed: true,
Official: true,
Content: "looks good",
},
+27 -27
View File
@@ -131,7 +131,7 @@ func (g *GiteaLocalUploader) CreateRepo(ctx context.Context, repo *base.Reposito
Wiki: opts.Wiki,
Releases: opts.Releases, // if didn't get releases, then sync them from tags
MirrorInterval: opts.MirrorInterval,
}, NewMigrationHTTPTransport())
}, getMigrationHTTPClient().Transport)
g.sameApp = strings.HasPrefix(repo.OriginalURL, setting.AppURL)
g.repo = r
@@ -237,13 +237,15 @@ func (g *GiteaLocalUploader) CreateLabels(ctx context.Context, labels ...*base.L
l.Color = color
}
lbs = append(lbs, &issues_model.Label{
lb := &issues_model.Label{
RepoID: g.repo.ID,
Name: l.Name,
Exclusive: l.Exclusive,
Description: l.Description,
Color: l.Color,
})
}
lb.SetArchived(l.Archived)
lbs = append(lbs, lb)
}
err := issues_model.NewLabels(ctx, lbs...)
@@ -332,32 +334,29 @@ func (g *GiteaLocalUploader) CreateReleases(ctx context.Context, releases ...*ba
// SECURITY: We cannot check the DownloadURL and DownloadFunc are safe here
// ... we must assume that they are safe and simply download the attachment
err := func() error {
var rc io.ReadCloser
var err error
if asset.DownloadFunc != nil {
rc, err = asset.DownloadFunc()
} else if asset.DownloadURL != nil {
// asset.DownloadURL maybe a local file
var rc io.ReadCloser
var err error
if asset.DownloadFunc != nil {
rc, err = asset.DownloadFunc()
if err != nil {
return err
}
} else if asset.DownloadURL != nil {
// use the migration client so the fetch (including any redirect) is
// validated against the migration host allow/block list
rc, err = uri.OpenWithClient(*asset.DownloadURL, getMigrationHTTPClient())
if err != nil {
return err
}
}
if rc == nil {
return nil
}
_, err = storage.Attachments.Save(attach.RelativePath(), rc, int64(*asset.Size))
rc.Close()
return err
}()
// use the migration client so the fetch (including any redirect) is
// validated against the migration host allow/block list
rc, err = uri.OpenWithClient(*asset.DownloadURL, getMigrationHTTPClient())
}
if err != nil {
return err
if ctx.Err() != nil {
return ctx.Err()
}
WarnAndNotice("Skipped asset %s of release %s: %v", asset.Name, release.TagName, err)
continue
}
if rc != nil {
attach.Size, err = storage.Attachments.Save(attach.RelativePath(), rc, util.Iif(*asset.Size > 0, int64(*asset.Size), -1)) // GitLab links and Forgejo external links report no size
rc.Close()
if err != nil {
return fmt.Errorf("release %s asset %s: %w", release.TagName, asset.Name, err)
}
}
rel.Attachments = append(rel.Attachments, &attach)
@@ -890,6 +889,7 @@ func (g *GiteaLocalUploader) CreateReviews(ctx context.Context, reviews ...*base
IssueID: issue.ID,
Content: review.Content,
Official: review.Official,
Dismissed: review.Dismissed,
CreatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()),
UpdatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()),
}
@@ -5,6 +5,8 @@
package migrations
import (
"errors"
"io"
"strconv"
"strings"
"testing"
@@ -125,6 +127,28 @@ func TestGiteaUploadRepo(t *testing.T) {
assert.Len(t, pulls[0].Issue.Comments, 2)
}
func TestCreateReleasesSkipsUnavailableAssets(t *testing.T) {
unittest.PrepareTestEnv(t)
sourceRelease := &base.Release{Name: "assets", Assets: []*base.ReleaseAsset{
{Name: "gone", Size: new(0), DownloadCount: new(0), DownloadFunc: func() (io.ReadCloser, error) { return nil, errors.New("gone") }},
{Name: "kept", Size: new(0), DownloadCount: new(0), DownloadFunc: func() (io.ReadCloser, error) { return io.NopCloser(strings.NewReader("data")), nil }},
}}
doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
uploader := NewGiteaLocalUploader(t.Context(), doer, doer.Name, "repo1")
uploader.repo = unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1})
require.NoError(t, uploader.CreateReleases(t.Context(), sourceRelease))
release := unittest.AssertExistsAndLoadBean(t, &repo_model.Release{RepoID: 1, Title: "assets"})
unittest.AssertExistsAndLoadBean(t, &repo_model.Attachment{ReleaseID: release.ID, Name: "kept", Size: 4})
unittest.AssertNotExistsBean(t, &repo_model.Attachment{ReleaseID: release.ID, Name: "gone"})
dumper := &RepositoryDumper{baseDir: t.TempDir(), opts: base.MigrateOptions{ReleaseAssets: true}}
defer dumper.Close()
require.NoError(t, dumper.CreateReleases(t.Context(), sourceRelease))
require.Len(t, sourceRelease.Assets, 1)
assert.Equal(t, "kept", sourceRelease.Assets[0].Name)
}
func TestGiteaUploadRemapLocalUser(t *testing.T) {
unittest.PrepareTestEnv(t)
doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
+13 -44
View File
@@ -18,9 +18,9 @@ import (
"gitea.dev/modules/log"
base "gitea.dev/modules/migration"
"gitea.dev/modules/structs"
"gitea.dev/modules/util"
"github.com/google/go-github/v92/github"
"golang.org/x/oauth2"
)
var (
@@ -99,29 +99,15 @@ func NewGithubDownloaderV3(_ context.Context, baseURL, userName, password, token
maxPerPage: 100,
}
apiURL := util.Iif(baseURL == "https://github.com", "https://api.github.com", baseURL)
if token != "" {
tokens := strings.SplitSeq(token, ",")
for token := range tokens {
token = strings.TrimSpace(token)
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: token},
)
client := &http.Client{
Transport: &oauth2.Transport{
Base: NewMigrationHTTPTransport(),
Source: oauth2.ReuseTokenSource(nil, ts),
},
}
if err := downloader.addClient(client, baseURL); err != nil {
for token := range strings.SplitSeq(token, ",") {
if err := downloader.addClient(newMigrationHTTPClient(apiURL, "Bearer "+strings.TrimSpace(token)), baseURL); err != nil {
return nil, err
}
}
} else {
client := &http.Client{
Transport: &github.BasicAuthTransport{Transport: NewMigrationHTTPTransport(), Username: userName, Password: password},
}
if err := downloader.addClient(client, baseURL); err != nil {
if err := downloader.addClient(newMigrationHTTPClient(apiURL, basicAuthorization(userName, password)), baseURL); err != nil {
return nil, err
}
}
@@ -339,8 +325,6 @@ func (g *GithubDownloaderV3) convertGithubRelease(ctx context.Context, rel *gith
r.Published = rel.PublishedAt.Time
}
httpClient := newMigrationHTTPClient()
for _, asset := range rel.Assets {
assetID := asset.GetID() // Don't optimize this, for closure we need a local variable TODO: no need to do so in new Golang
if assetID == 0 {
@@ -381,19 +365,11 @@ func (g *GithubDownloaderV3) convertGithubRelease(ctx context.Context, rel *gith
}
g.waitAndPickClient(ctx)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, redirectURL, nil)
if err != nil {
return nil, err
}
resp, err := httpClient.Do(req)
err1 := g.RefreshRate(ctx)
if err1 != nil {
rc, err := downloadAsset(ctx, getMigrationHTTPClient(), redirectURL)
if err1 := g.RefreshRate(ctx); err1 != nil {
log.Error("g.RefreshRate(): %s", err1)
}
if err != nil {
return nil, err
}
return resp.Body, nil
return rc, err
},
})
}
@@ -780,6 +756,7 @@ func (g *GithubDownloaderV3) getIssueReactions(ctx context.Context, number, perP
}
func convertGithubReview(r *github.PullRequestReview) *base.Review {
dismissed := r.GetState() == "DISMISSED"
return &base.Review{
ID: r.GetID(),
ReviewerID: r.GetUser().GetID(),
@@ -787,7 +764,8 @@ func convertGithubReview(r *github.PullRequestReview) *base.Review {
CommitID: r.GetCommitID(),
Content: r.GetBody(),
CreatedAt: r.GetSubmittedAt().Time,
State: r.GetState(),
State: util.Iif(dismissed, base.ReviewStateCommented, r.GetState()), // GitHub drops the state a review had before its dismissal
Dismissed: dismissed,
}
}
@@ -910,15 +888,6 @@ func (g *GithubDownloaderV3) GetReviews(ctx context.Context, reviewable base.Rev
// FormatCloneURL add authentication into remote URLs
func (g *GithubDownloaderV3) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) {
u, err := url.Parse(remoteAddr)
if err != nil {
return "", err
}
if len(opts.AuthToken) > 0 {
// "multiple tokens" are used to benefit more "API rate limit quota"
// git clone doesn't count for rate limits, so only use the first token.
// source: https://github.com/orgs/community/discussions/44515
u.User = url.UserPassword("oauth2", strings.Split(opts.AuthToken, ",")[0])
}
return u.String(), nil
opts.AuthToken, _, _ = strings.Cut(opts.AuthToken, ",") // the extra tokens only raise the API rate limit
return g.NullDownloader.FormatCloneURL(opts, remoteAddr)
}
+14 -2
View File
@@ -543,10 +543,17 @@ func TestGitHubDownloadRepo(t *testing.T) {
}, reviews)
}
func TestGithubMultiToken(t *testing.T) {
func TestConvertGithubDismissedReview(t *testing.T) {
review := convertGithubReview(&github.PullRequestReview{State: new("DISMISSED")})
assert.Equal(t, base.ReviewStateCommented, review.State)
assert.True(t, review.Dismissed)
}
func TestGithubFormatCloneURL(t *testing.T) {
testCases := []struct {
desc string
token string
username string
expectedCloneURL string
}{
{
@@ -559,12 +566,17 @@ func TestGithubMultiToken(t *testing.T) {
token: "token1,token2",
expectedCloneURL: "https://oauth2:token1@github.com",
},
{
desc: "Username And Password",
username: "user",
expectedCloneURL: "https://user:password@github.com",
},
}
factory := GithubDownloaderV3Factory{}
for _, tC := range testCases {
t.Run(tC.desc, func(t *testing.T) {
opts := base.MigrateOptions{CloneAddr: "https://github.com/go-gitea/gitea", AuthToken: tC.token}
opts := base.MigrateOptions{CloneAddr: "https://github.com/go-gitea/gitea", AuthToken: tC.token, AuthUsername: tC.username, AuthPassword: "password"}
client, err := factory.New(t.Context(), opts)
require.NoError(t, err)
+16 -19
View File
@@ -21,6 +21,7 @@ import (
"gitea.dev/modules/log"
base "gitea.dev/modules/migration"
"gitea.dev/modules/structs"
"gitea.dev/modules/util"
gitlab "gitlab.com/gitlab-org/api/client-go/v3"
)
@@ -94,7 +95,8 @@ type GitlabDownloader struct {
// Use either a username/password, personal token entered into the username field, or anonymous/public access
// Note: Public access only allows very basic access
func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, token string) (*GitlabDownloader, error) {
gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(newMigrationHTTPClient()))
transport := authTransport(getMigrationTransport(), baseURL, util.Iif(token != "", "Bearer "+token, "")) // not retrying, the SDK retries rate limits itself
gitlabClient, err := gitlab.NewAuthSourceClient(gitlab.Unauthenticated{}, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(&http.Client{Transport: transport}))
if err != nil {
log.Trace("Error logging into gitlab: %v", err)
return nil, err
@@ -314,8 +316,6 @@ func (g *GitlabDownloader) convertGitlabRelease(ctx context.Context, rel *gitlab
PublisherName: rel.Author.Username,
}
httpClient := newMigrationHTTPClient()
for _, asset := range rel.Assets.Links {
assetID := asset.ID // Don't optimize this, for closure we need a local variable
r.Assets = append(r.Assets, &base.ReleaseAsset{
@@ -334,18 +334,7 @@ func (g *GitlabDownloader) convertGitlabRelease(ctx context.Context, rel *gitlab
return io.NopCloser(strings.NewReader(link.URL)), nil
}
req, err := http.NewRequest(http.MethodGet, link.URL, nil)
if err != nil {
return nil, err
}
req = req.WithContext(ctx)
resp, err := httpClient.Do(req)
if err != nil {
return nil, err
}
// resp.Body is closed by the uploader
return resp.Body, nil
return downloadAsset(ctx, g.client.HTTPClient(), link.URL)
},
})
}
@@ -408,6 +397,12 @@ func (g *GitlabDownloader) GetIssues(ctx context.Context, page, perPage int) ([]
return nil, false, fmt.Errorf("error while listing issues: %w", err)
}
for _, issue := range issues {
// record the issue IID, to be used in GetPullRequests()
g.iidResolver.recordIssueIID(issue.IID)
if issue.Confidential { // Gitea can't restrict an issue to project members
continue
}
labels := make([]*base.Label, 0, len(issue.Labels))
for _, l := range issue.Labels {
labels = append(labels, &base.Label{
@@ -454,9 +449,6 @@ func (g *GitlabDownloader) GetIssues(ctx context.Context, page, perPage int) ([]
ForeignIndex: issue.IID,
Context: gitlabIssueContext{IsMergeRequest: false},
})
// record the issue IID, to be used in GetPullRequests()
g.iidResolver.recordIssueIID(issue.IID)
}
return allIssues, len(issues) < perPage, nil
@@ -499,7 +491,9 @@ func (g *GitlabDownloader) GetComments(ctx context.Context, commentable base.Com
}
for _, comment := range comments {
for _, note := range comment.Notes {
allComments = append(allComments, g.convertNoteToComment(commentable.GetLocalIndex(), note))
if !note.Internal { // Gitea can't restrict a comment to project members
allComments = append(allComments, g.convertNoteToComment(commentable.GetLocalIndex(), note))
}
}
}
if resp.NextPage == 0 {
@@ -749,6 +743,9 @@ func (g *GitlabDownloader) GetReviews(ctx context.Context, reviewable base.Revie
reviews := make([]*base.Review, 0, len(approvals.ApprovedBy))
for _, user := range approvals.ApprovedBy {
if user.User == nil {
continue
}
reviews = append(reviews, &base.Review{
IssueIndex: reviewable.GetLocalIndex(),
ReviewerID: user.User.ID,
+34
View File
@@ -20,6 +20,7 @@ import (
base "gitea.dev/modules/migration"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
gitlab "gitlab.com/gitlab-org/api/client-go/v3"
)
@@ -513,6 +514,39 @@ func TestGitlabGetReviews(t *testing.T) {
assert.NoError(t, err)
assertReviewsEqual(t, []*base.Review{&review}, rvs)
}
mux.HandleFunc(fmt.Sprintf("/api/v4/projects/%d/merge_requests/4/approvals", repoID), func(w http.ResponseWriter, _ *http.Request) {
fmt.Fprint(w, `{"approved_by": [{"user": null}]}`)
})
rvs, err := downloader.GetReviews(ctx, &base.Issue{Number: 4, ForeignIndex: 4})
require.NoError(t, err)
assert.Empty(t, rvs)
}
func TestGitlabSkipsConfidentialIssuesAndInternalNotes(t *testing.T) {
mux, server, client := gitlabClientMockSetup(t)
defer gitlabClientMockTeardown(server)
downloader := &GitlabDownloader{client: client, repoID: 1, maxPerPage: 100}
const timestamps = `"created_at": "2020-01-01T00:00:00Z", "updated_at": "2020-01-01T00:00:00Z"`
empty := func(w http.ResponseWriter, _ *http.Request) { fmt.Fprint(w, `[]`) }
mux.HandleFunc("/api/v4/projects/1/issues", func(w http.ResponseWriter, _ *http.Request) {
fmt.Fprint(w, `[{"iid": 1, "author": {}, `+timestamps+`}, {"iid": 2, "confidential": true}]`)
})
mux.HandleFunc("/api/v4/projects/1/issues/1/award_emoji", empty)
mux.HandleFunc("/api/v4/projects/1/issues/1/discussions", func(w http.ResponseWriter, _ *http.Request) {
fmt.Fprint(w, `[{"notes": [{"id": 1, `+timestamps+`}, {"internal": true}]}]`)
})
mux.HandleFunc("/api/v4/projects/1/issues/1/resource_state_events", empty)
issues, _, err := downloader.GetIssues(t.Context(), 1, 100)
require.NoError(t, err)
require.Len(t, issues, 1)
assert.EqualValues(t, 3, downloader.iidResolver.generatePullRequestNumber(1))
comments, _, err := downloader.GetComments(t.Context(), issues[0])
require.NoError(t, err)
require.Len(t, comments, 1)
assert.EqualValues(t, 1, comments[0].Index)
}
func TestAwardsToReactions(t *testing.T) {
+6 -22
View File
@@ -14,6 +14,7 @@ import (
"gitea.dev/modules/log"
base "gitea.dev/modules/migration"
"gitea.dev/modules/structs"
"gitea.dev/modules/util"
"github.com/gogs/go-gogs-client"
)
@@ -62,9 +63,7 @@ type GogsDownloader struct {
baseURL string
repoOwner string
repoName string
userName string
password string
token string
transport http.RoundTripper
openIssuesFinished bool
openIssuesPages int
}
@@ -83,36 +82,21 @@ func (g *GogsDownloader) LogString() string {
// NewGogsDownloader creates a gogs Downloader via gogs API
func NewGogsDownloader(_ context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GogsDownloader {
downloader := GogsDownloader{
return &GogsDownloader{
baseURL: baseURL,
userName: userName,
password: password,
token: token,
transport: newMigrationHTTPClient(baseURL, util.Iif(token != "", "token "+token, basicAuthorization(userName, password))).Transport,
repoOwner: repoOwner,
repoName: repoName,
}
return &downloader
}
type roundTripperFunc func(req *http.Request) (*http.Response, error)
func (rt roundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) {
return rt(r)
}
func (g *GogsDownloader) client(ctx context.Context) *gogs.Client {
// Gogs client lacks the context support, so we use a custom transport
// Then each request uses a dedicated client with its own context
httpTransport := NewMigrationHTTPTransport()
gogsClient := gogs.NewClient(g.baseURL, g.token)
gogsClient := gogs.NewClient(g.baseURL, "")
gogsClient.SetHTTPClient(&http.Client{
Transport: roundTripperFunc(func(req *http.Request) (*http.Response, error) {
req = req.Clone(ctx)
if g.password != "" {
// Gogs client lacks the support for basic auth, this is the only way to set it
req.SetBasicAuth(g.userName, g.password)
}
return httpTransport.RoundTrip(req)
return g.transport.RoundTrip(req.WithContext(ctx))
}),
})
return gogsClient
+91 -5
View File
@@ -5,24 +5,110 @@ package migrations
import (
"crypto/tls"
"encoding/base64"
"net/http"
"net/url"
"strconv"
"sync"
"time"
"gitea.dev/modules/egress"
"gitea.dev/modules/log"
"gitea.dev/modules/setting"
)
// getMigrationHTTPClient returns the shared migration client, so downloads from one host reuse its connections
var getMigrationHTTPClient = sync.OnceValue(newMigrationHTTPClient)
var getMigrationHTTPClient = sync.OnceValue(func() *http.Client {
return &http.Client{Transport: retryAfterTransport{getMigrationTransport()}}
})
// newMigrationHTTPClient returns a HTTP client for migration
func newMigrationHTTPClient() *http.Client {
return &http.Client{Transport: NewMigrationHTTPTransport()}
var getMigrationTransport = sync.OnceValue(newMigrationTransport)
func newMigrationHTTPClient(baseURL, authorization string) *http.Client {
return &http.Client{Transport: authTransport(getMigrationHTTPClient().Transport, baseURL, authorization)}
}
// NewMigrationHTTPTransport returns a HTTP transport for migration, enforcing the migration policy on its direct dials.
func NewMigrationHTTPTransport() *http.Transport {
func NewMigrationHTTPTransport() http.RoundTripper {
return retryAfterTransport{newMigrationTransport()}
}
func newMigrationTransport() *http.Transport {
t := egress.NewMigrationPolicy().NewHTTPTransport()
t.TLSClientConfig = &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}
return t
}
type retryAfterTransport struct {
http.RoundTripper
}
func (t retryAfterTransport) RoundTrip(req *http.Request) (*http.Response, error) {
var waited time.Duration
for retries := 0; ; retries++ {
resp, err := t.RoundTripper.RoundTrip(req)
if err != nil {
return nil, err
}
if (resp.StatusCode != http.StatusTooManyRequests && resp.StatusCode != http.StatusForbidden) || retries == 5 || (req.Body != nil && req.GetBody == nil) { // GitHub's secondary rate limits answer 403
return resp, nil
}
delay, ok := parseRetryAfter(resp.Header.Get("Retry-After"))
if !ok || delay > time.Hour-waited {
return resp, nil
}
waited += delay
resp.Body.Close()
log.Info("Rate limited by %s, retrying in %s", req.URL.Host, delay)
select {
case <-req.Context().Done():
return nil, req.Context().Err()
case <-time.After(delay):
}
if req.GetBody != nil {
body, err := req.GetBody()
if err != nil {
return nil, err
}
req = req.Clone(req.Context())
req.Body = body
}
}
}
func parseRetryAfter(value string) (time.Duration, bool) {
if seconds, err := strconv.ParseUint(value, 10, 32); err == nil { // 32 bits can't overflow the Duration
return time.Duration(seconds) * time.Second, true
}
if retryAt, err := http.ParseTime(value); err == nil {
return max(time.Until(retryAt), 0), true
}
return 0, false
}
type roundTripperFunc func(req *http.Request) (*http.Response, error)
func (rt roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) {
return rt(req)
}
func authTransport(transport http.RoundTripper, baseURL, authorization string) http.RoundTripper {
base, err := url.Parse(baseURL)
if authorization == "" || err != nil {
return transport
}
return roundTripperFunc(func(req *http.Request) (*http.Response, error) {
if req.URL.Host == base.Host && (req.URL.Scheme == "https" || base.Scheme == "http") { // the source host only, never downgraded to http
req = req.Clone(req.Context())
req.Header.Set("Authorization", authorization)
}
return transport.RoundTrip(req)
})
}
func basicAuthorization(username, password string) string {
if username == "" {
return ""
}
return "Basic " + base64.StdEncoding.EncodeToString([]byte(username+":"+password))
}
+63
View File
@@ -0,0 +1,63 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package migrations
import (
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMigrationHTTPTransportRetryAfter(t *testing.T) {
var requests atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch requests.Add(1) {
case 1:
w.Header().Set("Retry-After", "0")
w.WriteHeader(http.StatusTooManyRequests)
case 2:
w.Header().Set("Retry-After", time.Now().UTC().Format(http.TimeFormat))
w.WriteHeader(http.StatusForbidden)
default:
_, _ = io.Copy(w, r.Body)
}
}))
defer server.Close()
resp, err := getMigrationHTTPClient().Post(server.URL, "text/plain", strings.NewReader("payload"))
require.NoError(t, err)
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
assert.Equal(t, http.StatusOK, resp.StatusCode)
assert.Equal(t, "payload", string(body))
assert.EqualValues(t, 3, requests.Load())
}
func TestAuthTransportSendsCredentialsToSameHostOnly(t *testing.T) {
for _, testCase := range []struct{ baseURL, target, expected string }{
{"http://source.example/sub", "https://source.example/api", "token secret"},
{"https://source.example", "https://source.example/api", "token secret"},
{"https://source.example", "http://source.example/api", ""},
{"https://source.example", "https://other.example/api", ""},
{"https://source.example:3000", "https://source.example:8443/api", ""},
} {
req, err := http.NewRequest(http.MethodGet, testCase.target, nil)
require.NoError(t, err)
resp, err := authTransport(roundTripperFunc(func(req *http.Request) (*http.Response, error) {
assert.Equal(t, testCase.expected, req.Header.Get("Authorization"), testCase.target)
return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody}, nil
}), testCase.baseURL, "token secret").RoundTrip(req)
require.NoError(t, err)
resp.Body.Close()
assert.Empty(t, req.Header.Get("Authorization"))
}
}
+2
View File
@@ -54,6 +54,7 @@ func assertCommentsEqual(t *testing.T, expected, actual []*base.Comment) {
func assertLabelEqual(t *testing.T, expected, actual *base.Label) {
assert.Equal(t, expected.Name, actual.Name)
assert.Equal(t, expected.Exclusive, actual.Exclusive)
assert.Equal(t, expected.Archived, actual.Archived)
assert.Equal(t, expected.Color, actual.Color)
assert.Equal(t, expected.Description, actual.Description)
}
@@ -228,6 +229,7 @@ func assertReviewEqual(t *testing.T, expected, actual *base.Review) {
assert.Equal(t, expected.Content, actual.Content, "Content")
assert.WithinDuration(t, expected.CreatedAt, actual.CreatedAt, 10*time.Second)
assert.Equal(t, expected.State, actual.State, "State")
assert.Equal(t, expected.Dismissed, actual.Dismissed, "Dismissed")
assertReviewCommentsEqual(t, expected.Comments, actual.Comments)
}
+7 -4
View File
@@ -10,6 +10,7 @@ import (
"fmt"
"net/url"
"path/filepath"
"slices"
"strings"
repo_model "gitea.dev/models/repo"
@@ -293,14 +294,14 @@ func migrateRepository(ctx context.Context, doer *user_model.User, downloader ba
supportAllComments := downloader.SupportGetRepoComments()
mapInsertedIssueIndexes := container.Set[int64]{} // deletions on the source during the migration shift items into the next page
mapInsertedPRIndexes := container.Set[int64]{}
if opts.Issues {
log.Trace("migrating issues and comments")
messenger("repo.migrate.migrating_issues")
issueBatchSize := uploader.MaxBatchInsertSize("issue")
// because when the migrating is running, some issues maybe removed, so after the next page
// some of issue maybe duplicated, so we need to record the inserted issue indexes
mapInsertedIssueIndexes := container.Set[int64]{}
for i := 1; ; i++ {
issues, isEnd, err := downloader.GetIssues(ctx, i, issueBatchSize)
if err != nil {
@@ -363,7 +364,6 @@ func migrateRepository(ctx context.Context, doer *user_model.User, downloader ba
log.Trace("migrating pull requests and comments")
messenger("repo.migrate.migrating_pulls")
prBatchSize := uploader.MaxBatchInsertSize("pullrequest")
mapInsertedPRIndexes := container.Set[int64]{}
for i := 1; ; i++ {
prs, isEnd, err := downloader.GetPullRequests(ctx, i, prBatchSize)
if err != nil {
@@ -466,6 +466,9 @@ func migrateRepository(ctx context.Context, doer *user_model.User, downloader ba
if err != nil {
return err
}
comments = slices.DeleteFunc(comments, func(comment *base.Comment) bool {
return !mapInsertedIssueIndexes.Contains(comment.IssueIndex) && !mapInsertedPRIndexes.Contains(comment.IssueIndex)
})
if err := uploader.CreateComments(ctx, comments...); err != nil {
return err
+4 -15
View File
@@ -80,26 +80,15 @@ type OneDevDownloader struct {
}
// NewOneDevDownloader creates a new downloader
func NewOneDevDownloader(ctx context.Context, baseURL *url.URL, username, password, repoPath string) *OneDevDownloader {
httpTransport := NewMigrationHTTPTransport()
downloader := &OneDevDownloader{
func NewOneDevDownloader(_ context.Context, baseURL *url.URL, username, password, repoPath string) *OneDevDownloader {
return &OneDevDownloader{
baseURL: baseURL,
repoPath: repoPath,
client: &http.Client{
Transport: roundTripperFunc(
func(req *http.Request) (*http.Response, error) {
req = req.Clone(ctx)
if username != "" && password != "" {
req.SetBasicAuth(username, password)
}
return httpTransport.RoundTrip(req)
}),
},
client: newMigrationHTTPClient(baseURL.String(), basicAuthorization(username, password)),
userMap: make(map[int64]*onedevUser),
milestoneMap: make(map[int64]string),
}
return downloader
}
// String implements Stringer
+1 -1
View File
@@ -71,7 +71,7 @@ func cloneWiki(ctx context.Context, repo *repo_model.Repository, opts migration.
// MigrateRepositoryGitData starts migrating git related data after created migrating repository
func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
repo *repo_model.Repository, opts migration.MigrateOptions,
httpTransport *http.Transport,
httpTransport http.RoundTripper,
) (*repo_model.Repository, error) {
if u.IsOrganization() {
t, err := organization.OrgFromUser(u).GetOwnerTeam(ctx)
+1
View File
@@ -0,0 +1 @@
{{template "repo/migrate/gitea" .}}
+1 -1
View File
@@ -20,7 +20,7 @@
<div class="inline field">
<label>{{ctx.Locale.Tr "access_token"}}</label>
<input name="auth_token" type="password" autocomplete="new-password">
<a target="_blank" href="https://docs.gitea.com/development/api-usage">{{svg "octicon-question"}}</a>
<span class="help">{{ctx.Locale.Tr "repo.migrate.gitea_token_desc"}}</span>
</div>
{{template "repo/migrate/common-options" .}}
+1
View File
@@ -21,6 +21,7 @@
<label>{{ctx.Locale.Tr "access_token"}}</label>
<input name="auth_token" type="password" autocomplete="new-password">
<a target="_blank" href="https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html">{{svg "octicon-question"}}</a>
<span class="help">{{ctx.Locale.Tr "repo.migrate.gitlab_token_desc"}}</span>
</div>
{{template "repo/migrate/common-options" .}}
+7 -9
View File
@@ -3,25 +3,23 @@
<div class="ui middle very relaxed page grid">
<div class="column">
{{template "repo/migrate/helper" .}}
<div class="ui cards migrate-entries">
<div class="migrate-entries">
{{range .Services}}
<a class="ui card migrate-entry flex-text-block" href="{{AppSubUrl}}/repo/migrate?service_type={{.}}&org={{$.Org}}&mirror={{$.Mirror}}">
<a class="ui basic button migrate-entry" href="{{AppSubUrl}}/repo/migrate?service_type={{.}}&org={{$.Org}}&mirror={{$.Mirror}}">
{{if eq .Name "github"}}
{{svg "octicon-mark-github" 184 "tw-p-4"}}
{{else if eq .Name "gitlab"}}
{{svg "gitea-gitlab" 184 "tw-p-4"}}
{{else if eq .Name "bitbucketserver"}}
{{svg "gitea-bitbucket" 184 "tw-p-4"}}
{{else if eq .Name "gitbucket"}}
{{svg "gitea-gitbucket" 184 "tw-p-4"}}
{{else if or (eq .Name "gitlab") (eq .Name "gitbucket") (eq .Name "forgejo")}}
{{svg (printf "gitea-%s" .Name) 184 "tw-p-4"}}
{{else}}
{{svg (printf "gitea-%s" .Name) 184}}
{{end}}
<div class="content">
<div class="header tw-text-center">
<div class="migrate-entry-content">
<div class="migrate-entry-title">
{{.ProviderTitle}}
</div>
<div class="description tw-text-center tw-text-balance">
<div class="tw-text-balance">
{{ctx.Locale.Tr (printf "repo.migrate.%s.description" .Name)}}
</div>
</div>
+2 -21
View File
@@ -3360,17 +3360,6 @@
"type": "string",
"x-go-name": "IssueURL"
},
"original_author": {
"description": "OriginalAuthor is the original author name (for imported comments)",
"type": "string",
"x-go-name": "OriginalAuthor"
},
"original_author_id": {
"description": "OriginalAuthorID is the original author ID (for imported comments)",
"format": "int64",
"type": "integer",
"x-go-name": "OriginalAuthorID"
},
"pull_request_url": {
"description": "PRURL is the API URL for the pull request (if applicable)",
"format": "uri",
@@ -7486,15 +7475,6 @@
"type": "integer",
"x-go-name": "Index"
},
"original_author": {
"type": "string",
"x-go-name": "OriginalAuthor"
},
"original_author_id": {
"format": "int64",
"type": "integer",
"x-go-name": "OriginalAuthorID"
},
"pin_order": {
"format": "int64",
"type": "integer",
@@ -8108,7 +8088,8 @@
"gitbucket",
"codebase",
"codecommit",
"bitbucketserver"
"bitbucketserver",
"forgejo"
],
"type": "string",
"x-go-name": "Service"
+2 -21
View File
@@ -27258,17 +27258,6 @@
"type": "string",
"x-go-name": "IssueURL"
},
"original_author": {
"description": "OriginalAuthor is the original author name (for imported comments)",
"type": "string",
"x-go-name": "OriginalAuthor"
},
"original_author_id": {
"description": "OriginalAuthorID is the original author ID (for imported comments)",
"type": "integer",
"format": "int64",
"x-go-name": "OriginalAuthorID"
},
"pull_request_url": {
"description": "PRURL is the API URL for the pull request (if applicable)",
"type": "string",
@@ -31404,15 +31393,6 @@
"format": "int64",
"x-go-name": "Index"
},
"original_author": {
"type": "string",
"x-go-name": "OriginalAuthor"
},
"original_author_id": {
"type": "integer",
"format": "int64",
"x-go-name": "OriginalAuthorID"
},
"pin_order": {
"type": "integer",
"format": "int64",
@@ -32032,7 +32012,8 @@
"gitbucket",
"codebase",
"codecommit",
"bitbucketserver"
"bitbucketserver",
"forgejo"
],
"x-go-name": "Service"
},
@@ -1,3 +1,3 @@
Content-Type: application/json
[{"number":1,"title":"issue1","body":"","state":"open","user":{"id":689,"login":"6543","full_name":"","email":"6543@obermui.de","avatar_url":"","username":"6543"},"labels":[],"created_at":"2020-01-01T00:00:00Z","updated_at":"2020-01-01T00:00:00Z","is_locked":false},{"number":2,"title":"issue2","body":"","state":"open","user":{"id":689,"login":"6543","full_name":"","email":"6543@obermui.de","avatar_url":"","username":"6543"},"labels":[],"created_at":"2020-01-01T00:00:00Z","updated_at":"2020-01-01T00:00:00Z","is_locked":false},{"number":3,"title":"issue3","body":"","state":"open","user":{"id":689,"login":"6543","full_name":"","email":"6543@obermui.de","avatar_url":"","username":"6543"},"labels":[],"created_at":"2020-01-01T00:00:00Z","updated_at":"2020-01-01T00:00:00Z","is_locked":false},{"number":4,"title":"what is this repo about?","body":"","state":"closed","user":{"id":-1,"login":"Ghost","full_name":"","email":"","avatar_url":"","username":"Ghost"},"labels":[{"id":2,"name":"Question","color":"#d876e3"}],"milestone":{"id":1,"title":"V1"},"created_at":"2020-01-01T00:00:00Z","updated_at":"2020-01-01T00:00:00Z","closed_at":"2020-01-01T00:00:00Z","is_locked":true},{"number":5,"title":"issue5","body":"","state":"open","user":{"id":689,"login":"6543","full_name":"","email":"6543@obermui.de","avatar_url":"","username":"6543"},"labels":[],"created_at":"2020-01-01T00:00:00Z","updated_at":"2020-01-01T00:00:00Z","is_locked":false},{"number":6,"title":"issue6","body":"","state":"open","user":{"id":689,"login":"6543","full_name":"","email":"6543@obermui.de","avatar_url":"","username":"6543"},"labels":[],"created_at":"2020-01-01T00:00:00Z","updated_at":"2020-01-01T00:00:00Z","is_locked":false},{"number":7,"title":"issue7","body":"","state":"open","user":{"id":689,"login":"6543","full_name":"","email":"6543@obermui.de","avatar_url":"","username":"6543"},"labels":[],"created_at":"2020-01-01T00:00:00Z","updated_at":"2020-01-01T00:00:00Z","is_locked":false}]
[{"number":1,"title":"issue1","body":"","state":"open","user":{"id":689,"login":"6543","full_name":"","email":"6543@obermui.de","avatar_url":"","username":"6543"},"labels":[],"created_at":"2020-01-01T00:00:00Z","updated_at":"2020-01-01T00:00:00Z","is_locked":false},{"number":2,"title":"issue2","body":"","state":"open","user":{"id":689,"login":"6543","full_name":"","email":"6543@obermui.de","avatar_url":"","username":"6543"},"labels":[],"created_at":"2020-01-01T00:00:00Z","updated_at":"2020-01-01T00:00:00Z","is_locked":false},{"number":3,"title":"issue3","body":"","state":"open","user":{"id":689,"login":"6543","full_name":"","email":"6543@obermui.de","avatar_url":"","username":"6543"},"labels":[],"created_at":"2020-01-01T00:00:00Z","updated_at":"2020-01-01T00:00:00Z","is_locked":false},{"number":4,"title":"what is this repo about?","body":"","state":"closed","user":{"id":-1,"login":"Ghost","full_name":"","email":"","avatar_url":"","username":"Ghost"},"labels":[{"id":2,"name":"Question","color":"#d876e3"}],"milestone":{"id":1,"title":"V1"},"created_at":"2020-01-01T00:00:00Z","updated_at":"2020-01-01T00:00:00Z","closed_at":"2020-01-01T00:00:00Z","is_locked":true,"comments":2},{"number":5,"title":"issue5","body":"","state":"open","user":{"id":689,"login":"6543","full_name":"","email":"6543@obermui.de","avatar_url":"","username":"6543"},"labels":[],"created_at":"2020-01-01T00:00:00Z","updated_at":"2020-01-01T00:00:00Z","is_locked":false},{"number":6,"title":"issue6","body":"","state":"open","user":{"id":689,"login":"6543","full_name":"","email":"6543@obermui.de","avatar_url":"","username":"6543"},"labels":[],"created_at":"2020-01-01T00:00:00Z","updated_at":"2020-01-01T00:00:00Z","is_locked":false},{"number":7,"title":"issue7","body":"","state":"open","user":{"id":689,"login":"6543","full_name":"","email":"6543@obermui.de","avatar_url":"","username":"6543"},"labels":[],"created_at":"2020-01-01T00:00:00Z","updated_at":"2020-01-01T00:00:00Z","is_locked":false}]
@@ -1,2 +0,0 @@
Content-Type: text/plain
@@ -1,2 +0,0 @@
Content-Type: text/plain
@@ -1,2 +0,0 @@
Content-Type: text/plain
@@ -1,2 +0,0 @@
Content-Type: text/plain
@@ -1,2 +0,0 @@
Content-Type: text/plain
@@ -1,2 +0,0 @@
Content-Type: text/plain
+6
View File
@@ -17,15 +17,18 @@ import (
"gitea.dev/models/db"
access_model "gitea.dev/models/perm/access"
repo_model "gitea.dev/models/repo"
unit_model "gitea.dev/models/unit"
"gitea.dev/models/unittest"
user_model "gitea.dev/models/user"
"gitea.dev/modules/git"
"gitea.dev/modules/setting"
api "gitea.dev/modules/structs"
"gitea.dev/modules/test"
repo_service "gitea.dev/services/repository"
"gitea.dev/tests"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestAPIReleaseRead(t *testing.T) {
@@ -123,6 +126,9 @@ func testAPIListReleasesWithReadToken(t *testing.T) {
testAPIListReleasesTagNames(t, link+"?draft=false&pre-release=false", token, []string{"v1.1"}, "exclude drafts and pre-releases")
testAPIListReleasesTagNames(t, link+"?pre-release=true", token, []string{"v1.0"}, "only get pre-release")
testAPIListReleasesTagNames(t, link+"?draft=true&pre-release=true", token, nil, "there is no pre-release draft")
require.NoError(t, repo_service.UpdateRepositoryUnits(t.Context(), unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 2}), nil, []unit_model.Type{unit_model.TypeReleases}))
MakeRequest(t, NewRequest(t, "GET", "/api/v1/repos/user2/repo2/releases").AddTokenAuth(token), http.StatusNotFound)
}
func testAPIGetDraftRelease(t *testing.T) {
+19 -1
View File
@@ -6,6 +6,7 @@ package integration
import (
"errors"
"fmt"
"maps"
"net/url"
"os"
"path/filepath"
@@ -14,6 +15,8 @@ import (
"testing"
auth_model "gitea.dev/models/auth"
"gitea.dev/models/db"
issues_model "gitea.dev/models/issues"
repo_model "gitea.dev/models/repo"
"gitea.dev/models/unittest"
user_model "gitea.dev/models/user"
@@ -24,6 +27,7 @@ import (
"gitea.dev/services/migrations"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.yaml.in/yaml/v4"
)
@@ -45,6 +49,10 @@ func TestDumpRestore(t *testing.T) {
//
ctx := t.Context()
for bean, id := range map[any]int64{&issues_model.Issue{}: 1, &issues_model.Comment{}: 2, &issues_model.Reaction{}: 3, &issues_model.Reaction{}: 4} {
_, err := db.GetEngine(ctx).Table(bean).ID(id).Update(map[string]any{"original_author": "octocat", "original_author_id": 583231})
require.NoError(t, err)
}
opts := migrations.MigrateOptions{
GitServiceType: structs.GiteaService,
Issues: true,
@@ -66,6 +74,14 @@ func TestDumpRestore(t *testing.T) {
for _, f := range []string{"repo.yml", "topic.yml", "label.yml", "milestone.yml", "issue.yml"} {
assert.FileExists(t, filepath.Join(d, f))
}
var issues []*base.Issue
var comments []*base.Comment
require.NoError(t, base.Load(filepath.Join(d, "issue.yml"), &issues, false))
require.NoError(t, base.Load(filepath.Join(d, "comments/1.yml"), &comments, false))
for _, author := range []user_model.ExternalUserMigrated{issues[1], issues[1].Reactions[0], comments[0], comments[0].Reactions[0]} {
assert.Equal(t, user_model.GhostUserID, author.GetExternalID())
assert.Equal(t, "octocat", author.GetExternalName())
}
//
// Phase 2: restore from the filesystem to the Gitea instance in restoredrepo
@@ -184,10 +200,12 @@ func (c *compareDump) assertEquals(repoBefore, repoAfter *repo_model.Repository)
},
"CloneURL": {transform: c.replaceRepoName},
}
compareBaseBranch := maps.Clone(*comparePullRequestBranch)
compareBaseBranch["SHA"] = compareField{ignore: true}
prs, ok := c.assertEqual("pull_request.yml", []base.PullRequest{}, compareFields{
"Assignees": {ignore: true}, // not implemented yet
"Head": {nested: comparePullRequestBranch},
"Base": {nested: comparePullRequestBranch},
"Base": {nested: &compareBaseBranch},
"Labels": {ignore: true}, // because org labels are not handled properly
}).([]*base.PullRequest)
assert.True(c.t, ok)
+1 -1
View File
@@ -48,7 +48,7 @@ func TestRepoMigrate(t *testing.T) {
t.Run("ProviderEntries", func(t *testing.T) {
resp := session.MakeRequest(t, NewRequest(t, "GET", "/repo/migrate"), http.StatusOK)
entries := NewHTMLParser(t, resp.Body).Find(".migrate-entry .header").FilterFunction(func(_ int, s *goquery.Selection) bool {
entries := NewHTMLParser(t, resp.Body).Find(".migrate-entry-title").FilterFunction(func(_ int, s *goquery.Selection) bool {
return strings.Contains(s.Text(), "Bitbucket")
})
assert.Equal(t, 1, entries.Length())
+18 -12
View File
@@ -1601,10 +1601,9 @@ tbody.commit-list {
}
.migrate-entries {
display: grid !important;
display: grid;
grid-template-columns: repeat(3, 1fr);
gap: 25px;
margin: 0 !important;
}
@media (max-width: 767.98px) {
@@ -1613,18 +1612,25 @@ tbody.commit-list {
}
}
.migrate-entry {
transition: all 0.1s ease-in-out;
box-shadow: none !important;
border: 1px solid var(--color-secondary);
color: var(--color-text) !important;
width: auto !important;
margin: 0 !important;
.migrate-entry.ui.button {
flex-direction: column;
padding: 0;
margin: 0;
gap: 0;
white-space: normal;
line-height: var(--line-height-default);
}
.migrate-entry:hover {
transform: scale(105%);
box-shadow: 0 0.5rem 1rem var(--color-shadow) !important;
.migrate-entry-content {
align-self: stretch;
padding: 1em;
border-top: 1px solid var(--color-light-border);
}
.migrate-entry-title {
font-size: 18px;
font-weight: var(--font-weight-medium);
margin-bottom: 0.5em;
}
@media (max-width: 767.98px) {
+1
View File
@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 212 212"><g fill="none" transform="translate(6 6)"><path stroke="#f60" stroke-width="25" d="M58 168V70a50 50 0 0 1 50-50h20"/><path stroke="#d40000" stroke-width="25" d="M58 168v-30a50 50 0 0 1 50-50h20"/><circle cx="142" cy="20" r="18" stroke="#f60" stroke-width="15"/><circle cx="142" cy="88" r="18" stroke="#d40000" stroke-width="15"/><circle cx="58" cy="180" r="18" stroke="#d40000" stroke-width="15"/></g></svg>

After

Width:  |  Height:  |  Size: 470 B