feat(migrations): add migration from Bitbucket Server (Data Center) (#39913)

Add migration from Bitbucket Server (Data Center). It has no issues,
wiki, milestones or releases, so those are not offered.

Migrated: git data with tags and LFS, and pull requests with state,
draft flag, merge commit/base, comments, approvals and inline code
comments including resolved threads. Multi-line ranges become a `Lines
a-b:` text prefix, as Gitea anchors code comments to one line. Auth is
an HTTP access token (personal, project or repository) or
username/password; 429 responses are retried.

Changes outside the downloader:
- `ReviewComment.Resolved`, applied by the uploader.
- The uploader creates a fork PR's head branch from its head commit when
already in the repository, instead of fetching the fork (Bitbucket keeps
it in `refs/pull-requests/<id>/from`; private forks need credentials).

Tested with unit tests, responses recorded from Bitbucket 10.5.1, and an
end-to-end migration. Independent of
https://github.com/go-gitea/gitea/pull/38607 (Bitbucket Cloud); the
service type enum value needs adjusting for whichever lands second.

---------

Co-authored-by: bircni <bircni@icloud.com>
This commit is contained in:
Lunny Xiaoandbircni authored and GitHub committed 2026-10-23 19:25:29 +00:00
1 parent 8cd7721deb
commit a2da5cd7b5
25 files changed
+1575 -45

No files matched your search

@@ -0,0 +1,13 @@
Cache-Control: no-cache, no-transform
Content-Length: 583
Content-Security-Policy-Report-Only: script-src http://localhost:7990/bitbucket/s/ 'nonce-IWarugy5jB8gRNq1ZARFko1Q1VA='; default-src 'self'; base-uri 'self'; img-src data: *; style-src 'self' 'unsafe-inline'; frame-src 'self' https://www.youtube-nocookie.com/embed/ https://www.loom.com/embed/ https://player.vimeo.com/video/; connect-src 'self'; report-uri /bitbucket/rest/csp/latest/csp-report;
Content-Type: application/json;charset=UTF-8
Vary: X-AUSERNAME
Vary: X-AUSERID
Vary: Cookie
X-Arequestid: @8H2NK9x1265x325x0
X-Auserid: 2
X-Ausername: admin
X-Content-Type-Options: nosniff
{"slug":"demo","id":2,"name":"demo","hierarchyId":"01ee1752a4875aef8503","scmId":"git","state":"AVAILABLE","statusMessage":"Available","forkable":true,"project":{"key":"MIG","id":2,"name":"Migration Test","public":false,"type":"NORMAL","links":{"self":[{"href":"http://localhost:7990/bitbucket/projects/MIG"}]}},"public":false,"archived":false,"links":{"clone":[{"href":"ssh://git@localhost:7999/mig/demo.git","name":"ssh"},{"href":"http://localhost:7990/bitbucket/scm/mig/demo.git","name":"http"}],"self":[{"href":"http://localhost:7990/bitbucket/projects/MIG/repos/demo/browse"}]}}
@@ -0,0 +1,13 @@
Cache-Control: no-cache, no-transform
Content-Length: 59
Content-Security-Policy-Report-Only: script-src http://localhost:7990/bitbucket/s/ 'nonce-PiKZmxDnlrFC7QMnFC1SinlKErE='; default-src 'self'; base-uri 'self'; img-src data: *; style-src 'self' 'unsafe-inline'; frame-src 'self' https://www.youtube-nocookie.com/embed/ https://www.loom.com/embed/ https://player.vimeo.com/video/; connect-src 'self'; report-uri /bitbucket/rest/csp/latest/csp-report;
Content-Type: application/json;charset=UTF-8
Vary: X-AUSERNAME
Vary: X-AUSERID
Vary: Cookie
X-Arequestid: @8H2NK9x1265x326x0
X-Auserid: 2
X-Ausername: admin
X-Content-Type-Options: nosniff
{"id":"refs/heads/main","displayId":"main","type":"BRANCH"}
@@ -0,0 +1,10 @@
Cache-Control: no-cache, no-transform
Content-Security-Policy-Report-Only: script-src http://localhost:7990/bitbucket/s/ 'nonce-0W/hMaj8sXIdhG/WESDaZ6ESgmU='; default-src 'self'; base-uri 'self'; img-src data: *; style-src 'self' 'unsafe-inline'; frame-src 'self' https://www.youtube-nocookie.com/embed/ https://www.loom.com/embed/ https://player.vimeo.com/video/; connect-src 'self'; report-uri /bitbucket/rest/csp/latest/csp-report;
Content-Type: application/json;charset=UTF-8
Vary: x-ausername,x-auserid,cookie,accept-encoding
X-Arequestid: @8H2NK9x1265x330x0
X-Auserid: 2
X-Ausername: admin
X-Content-Type-Options: nosniff
{"size":9,"limit":100,"isLastPage":true,"values":[{"id":61,"createdDate":1792554902572,"user":{"name":"bob","emailAddress":"bob@example.com","active":true,"displayName":"bob","id":52,"slug":"bob","type":"NORMAL","links":{"self":[{"href":"http://localhost:7990/bitbucket/users/bob"}]}},"action":"COMMENTED","commentAction":"ADDED","comment":{"properties":{"repositoryId":2},"id":9,"version":0,"text":"Thread to resolve","author":{"name":"bob","emailAddress":"bob@example.com","active":true,"displayName":"bob","id":52,"slug":"bob","type":"NORMAL","links":{"self":[{"href":"http://localhost:7990/bitbucket/users/bob"}]}},"createdDate":1792554902569,"updatedDate":1792554902569,"comments":[{"properties":{"repositoryId":2},"id":10,"version":0,"text":"reply on resolved thread","author":{"name":"admin","emailAddress":"admin@example.com","active":true,"displayName":"Administrator","id":2,"slug":"admin","type":"NORMAL","links":{"self":[{"href":"http://localhost:7990/bitbucket/users/admin"}]}},"createdDate":1792554906416,"updatedDate":1792554906416,"comments":[],"anchor":{"fromHash":"a41991381bd1e6944b3422cd1dc933b65c74cf67","toHash":"810d0600716f4575691c804b583c1c6a6e6708db","line":6,"lineType":"ADDED","fileType":"TO","path":"a.txt","diffType":"EFFECTIVE","orphaned":false},"threadResolved":true,"severity":"NORMAL","state":"OPEN","permittedOperations":{"editable":true,"transitionable":true,"deletable":true}}],"anchor":{"fromHash":"a41991381bd1e6944b3422cd1dc933b65c74cf67","toHash":"810d0600716f4575691c804b583c1c6a6e6708db","line":6,"lineType":"ADDED","fileType":"TO","path":"a.txt","diffType":"EFFECTIVE","orphaned":false},"threadResolved":true,"severity":"NORMAL","state":"OPEN","permittedOperations":{"editable":false,"transitionable":true,"deletable":false}},"commentAnchor":{"fromHash":"a41991381bd1e6944b3422cd1dc933b65c74cf67","toHash":"810d0600716f4575691c804b583c1c6a6e6708db","line":6,"lineType":"ADDED","fileType":"TO","path":"a.txt","diffType":"EFFECTIVE","orphaned":false},"diff":{"destination":{"components":["a.txt"],"parent":"","name":"a.txt","extension":"txt","toString":"a.txt"},"hunks":[{"sourceLine":1,"sourceSpan":5,"destinationLine":1,"destinationSpan":6,"segments":[{"type":"CONTEXT","lines":[{"destination":1,"source":1,"line":"line1","truncated":false}],"truncated":false},{"type":"REMOVED","lines":[{"destination":2,"source":2,"line":"line2","truncated":false}],"truncated":false},{"type":"ADDED","lines":[{"destination":2,"source":3,"line":"line2 changed","truncated":false}],"truncated":false},{"type":"CONTEXT","lines":[{"destination":3,"source":3,"line":"line3","truncated":false},{"destination":4,"source":4,"line":"line4","truncated":false},{"destination":5,"source":5,"line":"line5","truncated":false}],"truncated":false},{"type":"ADDED","lines":[{"destination":6,"source":6,"line":"line6 new","truncated":false,"commentIds":[9]}],"truncated":false}],"truncated":false}],"truncated":false,"properties":{"toHash":"810d0600716f4575691c804b583c1c6a6e6708db","current":true,"fromHash":"a41991381bd1e6944b3422cd1dc933b65c74cf67"}}},{"id":60,"createdDate":1792554902532,"user":{"name":"bob","emailAddress":"bob@example.com","active":true,"displayName":"bob","id":52,"slug":"bob","type":"NORMAL","links":{"self":[{"href":"http://localhost:7990/bitbucket/users/bob"}]}},"action":"COMMENTED","commentAction":"ADDED","comment":{"properties":{"repositoryId":2},"id":8,"version":0,"text":"Multi line comment","author":{"name":"bob","emailAddress":"bob@example.com","active":true,"displayName":"bob","id":52,"slug":"bob","type":"NORMAL","links":{"self":[{"href":"http://localhost:7990/bitbucket/users/bob"}]}},"createdDate":1792554902528,"updatedDate":1792554902528,"comments":[],"anchor":{"fromHash":"a41991381bd1e6944b3422cd1dc933b65c74cf67","toHash":"810d0600716f4575691c804b583c1c6a6e6708db","line":4,"lineType":"CONTEXT","fileType":"TO","path":"a.txt","multilineMarker":{"startLine":2,"startLineType":"ADDED"},"multilineSpan":{"srcSpanStart":2,"srcSpanEnd":4,"dstSpanStart":2,"dstSpanEnd":4},"diffType":"EFFECTIVE","orphaned":false},"threadResolved":false,"severity":"NORMAL","state":"OPEN","permittedOperations":{"editable":false,"transitionable":true,"deletable":false}},"commentAnchor":{"fromHash":"a41991381bd1e6944b3422cd1dc933b65c74cf67","toHash":"810d0600716f4575691c804b583c1c6a6e6708db","line":4,"lineType":"CONTEXT","fileType":"TO","path":"a.txt","multilineMarker":{"startLine":2,"startLineType":"ADDED"},"multilineSpan":{"srcSpanStart":2,"srcSpanEnd":4,"dstSpanStart":2,"dstSpanEnd":4},"diffType":"EFFECTIVE","orphaned":false},"diff":{"destination":{"components":["a.txt"],"parent":"","name":"a.txt","extension":"txt","toString":"a.txt"},"hunks":[{"sourceLine":3,"sourceSpan":2,"destinationLine":2,"destinationSpan":3,"segments":[{"type":"ADDED","lines":[{"destination":2,"source":3,"line":"line2 changed","truncated":false}],"truncated":false},{"type":"CONTEXT","lines":[{"destination":3,"source":3,"line":"line3","truncated":false},{"destination":4,"sourLine truncated
@@ -0,0 +1,10 @@
Cache-Control: no-cache, no-transform
Content-Security-Policy-Report-Only: script-src http://localhost:7990/bitbucket/s/ 'nonce-3lbjhq7hBJksGbRCuPRjU0dVbfI='; default-src 'self'; base-uri 'self'; img-src data: *; style-src 'self' 'unsafe-inline'; frame-src 'self' https://www.youtube-nocookie.com/embed/ https://www.loom.com/embed/ https://player.vimeo.com/video/; connect-src 'self'; report-uri /bitbucket/rest/csp/latest/csp-report;
Content-Type: application/json;charset=UTF-8
Vary: x-ausername,x-auserid,cookie,accept-encoding
X-Arequestid: @8H2NK9x1265x328x0
X-Auserid: 2
X-Ausername: admin
X-Content-Type-Options: nosniff
{"size":2,"limit":100,"isLastPage":true,"values":[{"id":53,"createdDate":1792553814504,"user":{"name":"admin","emailAddress":"admin@example.com","active":true,"displayName":"Administrator","id":2,"slug":"admin","type":"NORMAL","links":{"self":[{"href":"http://localhost:7990/bitbucket/users/admin"}]}},"action":"MERGED","commit":{"id":"504e78ed1c8fcbc0e55f5f0b6d3c49a0bc1ae861","displayId":"504e78ed1c8","author":{"name":"admin","emailAddress":"admin@example.com","active":true,"displayName":"Administrator","id":2,"slug":"admin","type":"NORMAL","links":{"self":[{"href":"http://localhost:7990/bitbucket/users/admin"}]}},"authorTimestamp":1792553814000,"committer":{"name":"admin","emailAddress":"admin@example.com","active":true,"displayName":"Administrator","id":2,"slug":"admin","type":"NORMAL","links":{"self":[{"href":"http://localhost:7990/bitbucket/users/admin"}]}},"committerTimestamp":1792553814000,"message":"Merge pull request #2 in MIG/demo from merged to main\n\n* commit 'a84b8e075f3071e08052c7d9f8dbfd8d679b11c7':\n merged change","parents":[{"id":"a41991381bd1e6944b3422cd1dc933b65c74cf67","displayId":"a41991381bd","author":{"name":"admin","emailAddress":"admin@example.com"},"authorTimestamp":1792553776000,"committer":{"name":"admin","emailAddress":"admin@example.com"},"committerTimestamp":1792553776000,"message":"initial","parents":[]},{"id":"a84b8e075f3071e08052c7d9f8dbfd8d679b11c7","displayId":"a84b8e075f3","author":{"name":"admin","emailAddress":"admin@example.com"},"authorTimestamp":1792553779000,"committer":{"name":"admin","emailAddress":"admin@example.com"},"committerTimestamp":1792553779000,"message":"merged change","parents":[{"id":"a41991381bd1e6944b3422cd1dc933b65c74cf67","displayId":"a41991381bd"}]}]},"autoMerge":false,"strategyId":"no-ff"},{"id":51,"createdDate":1792553814274,"user":{"name":"admin","emailAddress":"admin@example.com","active":true,"displayName":"Administrator","id":2,"slug":"admin","type":"NORMAL","links":{"self":[{"href":"http://localhost:7990/bitbucket/users/admin"}]}},"action":"OPENED"}],"start":0}
@@ -0,0 +1,13 @@
Cache-Control: no-cache, no-transform
Content-Length: 628
Content-Security-Policy-Report-Only: script-src http://localhost:7990/bitbucket/s/ 'nonce-U5OUdU3Hi1/+nlNGoIe5d424Gfs='; default-src 'self'; base-uri 'self'; img-src data: *; style-src 'self' 'unsafe-inline'; frame-src 'self' https://www.youtube-nocookie.com/embed/ https://www.loom.com/embed/ https://player.vimeo.com/video/; connect-src 'self'; report-uri /bitbucket/rest/csp/latest/csp-report;
Content-Type: application/json;charset=UTF-8
Vary: X-AUSERNAME
Vary: X-AUSERID
Vary: Cookie
X-Arequestid: @8H2NK9x1265x329x0
X-Auserid: 2
X-Ausername: admin
X-Content-Type-Options: nosniff
{"values":[{"id":"a84b8e075f3071e08052c7d9f8dbfd8d679b11c7","displayId":"a84b8e075f3","author":{"name":"admin","emailAddress":"admin@example.com","active":true,"displayName":"Administrator","id":2,"slug":"admin","type":"NORMAL"},"authorTimestamp":1792553779000,"committer":{"name":"admin","emailAddress":"admin@example.com","active":true,"displayName":"Administrator","id":2,"slug":"admin","type":"NORMAL"},"committerTimestamp":1792553779000,"message":"merged change","parents":[{"id":"a41991381bd1e6944b3422cd1dc933b65c74cf67","displayId":"a41991381bd"}]}],"size":1,"isLastPage":true,"start":0,"limit":100,"nextPageStart":null}
@@ -0,0 +1,13 @@
Cache-Control: no-cache, no-transform
Content-Length: 1414
Content-Security-Policy-Report-Only: script-src http://localhost:7990/bitbucket/s/ 'nonce-e6rCQzX1Ipt1fa5qEaxaFaNwY1Y='; default-src 'self'; base-uri 'self'; img-src data: *; style-src 'self' 'unsafe-inline'; frame-src 'self' https://www.youtube-nocookie.com/embed/ https://www.loom.com/embed/ https://player.vimeo.com/video/; connect-src 'self'; report-uri /bitbucket/rest/csp/latest/csp-report;
Content-Type: application/json;charset=UTF-8
Vary: X-AUSERNAME
Vary: X-AUSERID
Vary: Cookie
X-Arequestid: @8H2NK9x1265x331x0
X-Auserid: 2
X-Ausername: admin
X-Content-Type-Options: nosniff
{"size":3,"limit":100,"isLastPage":true,"values":[{"id":56,"createdDate":1792553814846,"user":{"name":"admin","emailAddress":"admin@example.com","active":true,"displayName":"Administrator","id":2,"slug":"admin","type":"NORMAL","links":{"self":[{"href":"http://localhost:7990/bitbucket/users/admin"}]}},"action":"COMMENTED","commentAction":"ADDED","comment":{"properties":{"repositoryId":2},"id":7,"version":0,"text":"declining this","author":{"name":"admin","emailAddress":"admin@example.com","active":true,"displayName":"Administrator","id":2,"slug":"admin","type":"NORMAL","links":{"self":[{"href":"http://localhost:7990/bitbucket/users/admin"}]}},"createdDate":1792553814843,"updatedDate":1792553814843,"comments":[],"threadResolved":false,"severity":"NORMAL","state":"OPEN","permittedOperations":{"editable":true,"transitionable":true,"deletable":true}}},{"id":55,"createdDate":1792553814750,"user":{"name":"admin","emailAddress":"admin@example.com","active":true,"displayName":"Administrator","id":2,"slug":"admin","type":"NORMAL","links":{"self":[{"href":"http://localhost:7990/bitbucket/users/admin"}]}},"action":"DECLINED"},{"id":52,"createdDate":1792553814412,"user":{"name":"admin","emailAddress":"admin@example.com","active":true,"displayName":"Administrator","id":2,"slug":"admin","type":"NORMAL","links":{"self":[{"href":"http://localhost:7990/bitbucket/users/admin"}]}},"action":"OPENED"}],"start":0}
@@ -0,0 +1,10 @@
Cache-Control: no-cache, no-transform
Content-Security-Policy-Report-Only: script-src http://localhost:7990/bitbucket/s/ 'nonce-Jsnc2VX7MB+aw+YjfC4S+n6OUeg='; default-src 'self'; base-uri 'self'; img-src data: *; style-src 'self' 'unsafe-inline'; frame-src 'self' https://www.youtube-nocookie.com/embed/ https://www.loom.com/embed/ https://player.vimeo.com/video/; connect-src 'self'; report-uri /bitbucket/rest/csp/latest/csp-report;
Content-Type: application/json;charset=UTF-8
Vary: x-ausername,x-auserid,cookie,accept-encoding
X-Arequestid: @8H2NK9x1265x327x0
X-Auserid: 2
X-Ausername: admin
X-Content-Type-Options: nosniff
{"size":4,"limit":50,"isLastPage":true,"values":[{"id":1,"version":1,"title":"Open PR","description":"an open pull request","state":"OPEN","open":true,"closed":false,"draft":false,"createdDate":1792553804655,"updatedDate":1792553804655,"fromRef":{"id":"refs/heads/feature","displayId":"feature","latestCommit":"810d0600716f4575691c804b583c1c6a6e6708db","type":"BRANCH","repository":{"slug":"demo","id":2,"name":"demo","hierarchyId":"01ee1752a4875aef8503","scmId":"git","state":"AVAILABLE","statusMessage":"Available","forkable":true,"project":{"key":"MIG","id":2,"name":"Migration Test","public":false,"type":"NORMAL","links":{"self":[{"href":"http://localhost:7990/bitbucket/projects/MIG"}]}},"public":false,"archived":false,"links":{"clone":[{"href":"ssh://git@localhost:7999/mig/demo.git","name":"ssh"},{"href":"http://localhost:7990/bitbucket/scm/mig/demo.git","name":"http"}],"self":[{"href":"http://localhost:7990/bitbucket/projects/MIG/repos/demo/browse"}]}}},"toRef":{"id":"refs/heads/main","displayId":"main","latestCommit":"504e78ed1c8fcbc0e55f5f0b6d3c49a0bc1ae861","type":"BRANCH","repository":{"slug":"demo","id":2,"name":"demo","hierarchyId":"01ee1752a4875aef8503","scmId":"git","state":"AVAILABLE","statusMessage":"Available","forkable":true,"project":{"key":"MIG","id":2,"name":"Migration Test","public":false,"type":"NORMAL","links":{"self":[{"href":"http://localhost:7990/bitbucket/projects/MIG"}]}},"public":false,"archived":false,"links":{"clone":[{"href":"ssh://git@localhost:7999/mig/demo.git","name":"ssh"},{"href":"http://localhost:7990/bitbucket/scm/mig/demo.git","name":"http"}],"self":[{"href":"http://localhost:7990/bitbucket/projects/MIG/repos/demo/browse"}]}}},"locked":false,"author":{"user":{"name":"admin","emailAddress":"admin@example.com","active":true,"displayName":"Administrator","id":2,"slug":"admin","type":"NORMAL","links":{"self":[{"href":"http://localhost:7990/bitbucket/users/admin"}]}},"role":"AUTHOR","approved":false,"status":"UNAPPROVED"},"creator":{"user":{"name":"admin","emailAddress":"admin@example.com","active":true,"displayName":"Administrator","id":2,"slug":"admin","type":"NORMAL","links":{"self":[{"href":"http://localhost:7990/bitbucket/users/admin"}]}},"role":"CREATOR","approved":false,"status":"UNAPPROVED"},"reviewers":[{"user":{"name":"bob","emailAddress":"bob@example.com","active":true,"displayName":"bob","id":52,"slug":"bob","type":"NORMAL","links":{"self":[{"href":"http://localhost:7990/bitbucket/users/bob"}]}},"lastReviewedCommit":"810d0600716f4575691c804b583c1c6a6e6708db","role":"REVIEWER","approved":true,"status":"APPROVED"},{"user":{"name":"carol","emailAddress":"carol@example.com","active":true,"displayName":"carol","id":53,"slug":"carol","type":"NORMAL","links":{"self":[{"href":"http://localhost:7990/bitbucket/users/carol"}]}},"lastReviewedCommit":"810d0600716f4575691c804b583c1c6a6e6708db","role":"REVIEWER","approved":false,"status":"NEEDS_WORK"}],"participants":[],"properties":{"resolvedTaskCount":0,"isQueued":false,"commentCount":9,"openTaskCount":0},"links":{"self":[{"href":"http://localhost:7990/bitbucket/projects/MIG/repos/demo/pull-requests/1"}]}},{"id":2,"version":2,"title":"Merged PR","description":"Merged PR body","state":"MERGED","open":false,"closed":true,"draft":false,"createdDate":1792553814251,"updatedDate":1792553814504,"closedDate":1792553814504,"fromRef":{"id":"refs/heads/merged","displayId":"merged","latestCommit":"a84b8e075f3071e08052c7d9f8dbfd8d679b11c7","type":"BRANCH","repository":{"slug":"demo","id":2,"name":"demo","hierarchyId":"01ee1752a4875aef8503","scmId":"git","state":"AVAILABLE","statusMessage":"Available","forkable":true,"project":{"key":"MIG","id":2,"name":"Migration Test","public":false,"type":"NORMAL","links":{"self":[{"href":"http://localhost:7990/bitbucket/projects/MIG"}]}},"public":false,"archived":false,"links":{"clone":[{"href":"ssh://git@localhost:7999/mig/demo.git","name":"ssh"},{"href":"http://localhost:7990/bitbucket/scm/mig/demo.git","name":"http"}],"self":[{"href":"http://localhost:7990/bitbucket/projects/MIG/repos/demo/browse"}]}}},"toRef":{"id":"refs/heads/main","displayId":"main","latestCommit":"a41991381bd1e6944b3422cd1dc933b65c74cf67","type":"BRANCH","repository":{"slug":"demo","id":2,"name":"demo","hierarchyId":"01ee1752a4875aef8503","scmId":"git","state":"AVAILABLE","statusMessage":"Available","forkable":true,"project":{"key":"MIG","id":2,"name":"Migration Test","public":false,"type":"NORMAL","links":{"self":[{"href":"http://localhost:7990/bitbucket/projects/MIG"}]}},"public":false,"archived":false,"links":{"clone":[{"href":"ssh://git@localhost:7999/mig/demo.git","name":"ssh"},{"href":"http://localhost:7990/bitbucket/scm/mig/demo.git","name":"http"}],"self":[{"href":"http://localhost:7990/bitbucket/projects/MIG/repos/demo/browse"}]}}},"locked":false,"author":{"user":{"name":"admin","emailAddress":"admin@example.com","active":true,"displayName":"Administrator","id":2,"slug":"admin","type":"NORMAL","links":{"self":[{"href":"htLine truncated
+722
View File
@@ -0,0 +1,722 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package migrations
import (
"cmp"
"context"
"errors"
"fmt"
"io"
"maps"
"net/http"
"net/url"
"slices"
"strconv"
"strings"
"time"
"gitea.dev/modules/json"
"gitea.dev/modules/log"
base "gitea.dev/modules/migration"
"gitea.dev/modules/structs"
)
var (
_ base.Downloader = &BitbucketServerDownloader{}
_ base.DownloaderFactory = &BitbucketServerDownloaderFactory{}
)
const (
bitbucketServerMaxPageSize = 100
bitbucketServerMaxResponseSize = 64 * 1024 * 1024
bitbucketServerMaxAttempts = 5
bitbucketServerMaxRetryWait = time.Minute
)
func init() {
RegisterDownloaderFactory(&BitbucketServerDownloaderFactory{})
}
// BitbucketServerDownloaderFactory defines a downloader factory for Bitbucket Server (Data Center)
type BitbucketServerDownloaderFactory struct{}
// New returns a Downloader related to this factory according MigrateOptions
func (f *BitbucketServerDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
baseURL, projectKey, repoSlug, err := parseBitbucketServerURL(opts.CloneAddr)
if err != nil {
return nil, err
}
log.Trace("Create Bitbucket Server downloader. BaseURL: %s Project: %s Repo: %s", baseURL, projectKey, repoSlug)
return NewBitbucketServerDownloader(ctx, baseURL, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, projectKey, repoSlug), nil
}
// GitServiceType returns the type of git service
func (f *BitbucketServerDownloaderFactory) GitServiceType() structs.GitServiceType {
return structs.BitbucketServerService
}
// parseBitbucketServerURL splits a clone or browse URL into the base URL (with context path), project key and repo slug
func parseBitbucketServerURL(rawURL string) (baseURL, projectKey, repoSlug string, err error) {
u, err := url.Parse(rawURL)
if err != nil {
return "", "", "", err
}
if u.Scheme != "http" && u.Scheme != "https" {
return "", "", "", fmt.Errorf("unsupported scheme %q, an http(s) URL is required", u.Scheme)
}
segs := strings.Split(strings.Trim(u.Path, "/"), "/")
for i, seg := range segs {
rest := segs[i+1:]
switch {
case seg == "scm" && len(rest) >= 2:
projectKey, repoSlug = rest[0], rest[1]
case seg == "projects" && len(rest) >= 3 && rest[1] == "repos":
projectKey, repoSlug = rest[0], rest[2]
case seg == "users" && len(rest) >= 3 && rest[1] == "repos":
projectKey, repoSlug = "~"+rest[0], rest[2]
default:
continue
}
repoSlug = strings.TrimSuffix(repoSlug, ".git")
if projectKey == "" || repoSlug == "" {
break
}
root := url.URL{Scheme: u.Scheme, Host: u.Host, Path: "/" + strings.Join(segs[:i], "/")}
return strings.TrimSuffix(root.String(), "/"), projectKey, repoSlug, nil
}
return "", "", "", fmt.Errorf("invalid Bitbucket Server URL %q, expected .../scm/<project>/<repo>.git or .../projects/<project>/repos/<repo>", u.Redacted())
}
// BitbucketServerDownloader implements a Downloader for Bitbucket Server (Data Center) REST API 1.0.
// Bitbucket Server has no issues, wiki, labels, milestones or releases.
type BitbucketServerDownloader struct {
base.NullDownloader
client *http.Client
sleep func(context.Context, time.Duration) error
baseURL string
projectKey string
repoSlug string
username string
password string
token string
tokenUser string // user name git over HTTP needs together with the token as the basic-auth password
prNextStart int // offset of the next page of pull requests, pages are fetched in order
// activities of the PRs on the last fetched page, shared by comments, reviews and merge info
activities map[int64][]*bbsActivity
}
// NewBitbucketServerDownloader creates a Bitbucket Server downloader
func NewBitbucketServerDownloader(_ context.Context, baseURL, username, password, token, projectKey, repoSlug string) *BitbucketServerDownloader {
return &BitbucketServerDownloader{
client: getMigrationHTTPClient(),
sleep: bitbucketServerSleep,
baseURL: strings.TrimSuffix(baseURL, "/"),
projectKey: projectKey,
repoSlug: repoSlug,
username: username,
password: password,
token: token,
activities: map[int64][]*bbsActivity{},
}
}
// String implements Stringer
func (d *BitbucketServerDownloader) String() string {
return fmt.Sprintf("migration from bitbucket server %s %s/%s", d.baseURL, d.projectKey, d.repoSlug)
}
// LogString returns a log friendly description
func (d *BitbucketServerDownloader) LogString() string {
if d == nil {
return "<BitbucketServerDownloader nil>"
}
return fmt.Sprintf("<BitbucketServerDownloader %s %s/%s>", d.baseURL, d.projectKey, d.repoSlug)
}
// FormatCloneURL adds authentication into the remote URL
func (d *BitbucketServerDownloader) FormatCloneURL(opts base.MigrateOptions, remoteAddr string) (string, error) {
u, err := url.Parse(remoteAddr)
if err != nil {
return "", err
}
switch {
case opts.AuthToken != "":
// HTTP access tokens are accepted as the basic-auth password of their owner
u.User = url.UserPassword(cmp.Or(opts.AuthUsername, d.tokenUser, "token"), opts.AuthToken)
case opts.AuthUsername != "" || opts.AuthPassword != "":
u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword)
}
return u.String(), nil
}
type bbsUser struct {
ID int64 `json:"id"`
Name string `json:"name"`
Slug string `json:"slug"`
EmailAddress string `json:"emailAddress"`
}
func (u bbsUser) login() string {
return cmp.Or(u.Name, u.Slug)
}
type bbsRepository struct {
Slug string `json:"slug"`
Name string `json:"name"`
Description string `json:"description"`
Public bool `json:"public"`
Project struct {
Key string `json:"key"`
} `json:"project"`
}
type bbsRef struct {
DisplayID string `json:"displayId"`
LatestCommit string `json:"latestCommit"`
Repository *bbsRepository `json:"repository"`
}
type bbsPullRequest struct {
ID int64 `json:"id"`
Title string `json:"title"`
Description string `json:"description"`
State string `json:"state"` // OPEN, MERGED, DECLINED
Draft bool `json:"draft"`
CreatedDate int64 `json:"createdDate"`
UpdatedDate int64 `json:"updatedDate"`
ClosedDate int64 `json:"closedDate"`
FromRef bbsRef `json:"fromRef"`
ToRef bbsRef `json:"toRef"`
Author struct {
User bbsUser `json:"user"`
} `json:"author"`
}
type bbsAnchor struct {
Line int `json:"line"`
LineType string `json:"lineType"` // ADDED, REMOVED, CONTEXT
FileType string `json:"fileType"` // FROM, TO
Path string `json:"path"`
SrcPath string `json:"srcPath"`
FromHash string `json:"fromHash"`
ToHash string `json:"toHash"`
// set when the comment spans several lines, Line is then the last one
MultilineMarker struct {
StartLine int `json:"startLine"`
} `json:"multilineMarker"`
MultilineSpan struct {
SrcStart int `json:"srcSpanStart"`
SrcEnd int `json:"srcSpanEnd"`
DstStart int `json:"dstSpanStart"`
DstEnd int `json:"dstSpanEnd"`
} `json:"multilineSpan"`
}
// lineRange returns the first and last line of a multi-line comment, in the numbering of the commented side
func (a *bbsAnchor) lineRange() (start, end int, ok bool) {
start, end = a.MultilineSpan.DstStart, a.MultilineSpan.DstEnd
if a.FileType == "FROM" {
start, end = a.MultilineSpan.SrcStart, a.MultilineSpan.SrcEnd
}
if start == 0 || end == 0 {
start, end = a.MultilineMarker.StartLine, a.Line
}
return start, end, start > 0 && start < end
}
func (a *bbsAnchor) isInline() bool {
return a != nil && a.Path != "" && a.Line > 0
}
type bbsComment struct {
ID int64 `json:"id"`
Text string `json:"text"`
Author bbsUser `json:"author"`
CreatedDate int64 `json:"createdDate"`
UpdatedDate int64 `json:"updatedDate"`
Comments []*bbsComment `json:"comments"` // replies
ThreadResolved bool `json:"threadResolved"`
}
func (c *bbsComment) updated() time.Time {
return time.UnixMilli(cmp.Or(c.UpdatedDate, c.CreatedDate))
}
type bbsActivity struct {
Action string `json:"action"` // COMMENTED, APPROVED, UNAPPROVED, REVIEWED, DECLINED, REOPENED, MERGED, ...
CommentAction string `json:"commentAction"`
CreatedDate int64 `json:"createdDate"`
User bbsUser `json:"user"`
Comment *bbsComment `json:"comment"`
CommentAnchor *bbsAnchor `json:"commentAnchor"`
Commit *bbsCommit `json:"commit"` // merge commit of a MERGED activity
}
type bbsCommit struct {
ID string `json:"id"`
Parents []struct {
ID string `json:"id"`
} `json:"parents"`
}
type bbsPage[T any] struct {
Values []T `json:"values"`
IsLastPage bool `json:"isLastPage"`
NextPageStart int `json:"nextPageStart"`
}
func (p *bbsPage[T]) isLast(start int) bool {
return p.IsLastPage || len(p.Values) == 0 || p.NextPageStart <= start
}
type bbsStatusError struct {
Status int
URL string
}
func (e *bbsStatusError) isAuthError() bool {
return e.Status == http.StatusUnauthorized || e.Status == http.StatusForbidden
}
func (e *bbsStatusError) Error() string {
msg := fmt.Sprintf("bitbucket server API %s returned status %d", e.URL, e.Status)
if e.isAuthError() {
msg += ", check the credentials and the permission to read the repository"
}
return msg
}
func (d *BitbucketServerDownloader) repoPath() string {
return "/projects/" + url.PathEscape(d.projectKey) + "/repos/" + url.PathEscape(d.repoSlug)
}
func (d *BitbucketServerDownloader) cloneURL(projectKey, repoSlug string) string {
return d.baseURL + "/scm/" + url.PathEscape(projectKey) + "/" + url.PathEscape(repoSlug) + ".git"
}
func (d *BitbucketServerDownloader) browseURL() string {
if name, ok := strings.CutPrefix(d.projectKey, "~"); ok {
return d.baseURL + "/users/" + url.PathEscape(name) + "/repos/" + url.PathEscape(d.repoSlug)
}
return d.baseURL + d.repoPath()
}
// callAPI sends a GET request and returns the response headers, a rate limited (429) request is repeated after the
// time the server asks for
func (d *BitbucketServerDownloader) callAPI(ctx context.Context, path string, query url.Values, result any) (http.Header, error) {
endpoint := d.baseURL + "/rest/api/1.0" + path
apiURL := endpoint
if len(query) > 0 {
apiURL += "?" + query.Encode()
}
resp, err := d.doRequest(ctx, apiURL)
for attempt := 1; err == nil && resp.StatusCode == http.StatusTooManyRequests && attempt < bitbucketServerMaxAttempts; attempt++ {
wait := bitbucketServerRetryAfter(resp.Header.Get("Retry-After"), attempt)
resp.Body.Close()
log.Warn("Bitbucket Server rate limited the request to %s, retrying in %s", endpoint, wait)
if err := d.sleep(ctx, wait); err != nil {
return nil, err
}
resp, err = d.doRequest(ctx, apiURL)
}
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, &bbsStatusError{Status: resp.StatusCode, URL: endpoint}
}
data, err := io.ReadAll(io.LimitReader(resp.Body, bitbucketServerMaxResponseSize+1))
if err != nil {
return nil, err
}
if len(data) > bitbucketServerMaxResponseSize {
return nil, errors.New("bitbucket server API response is too large")
}
return resp.Header, json.Unmarshal(data, result)
}
func (d *BitbucketServerDownloader) doRequest(ctx context.Context, apiURL string) (*http.Response, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", "application/json")
if d.token != "" {
req.Header.Set("Authorization", "Bearer "+d.token)
} else if d.username != "" || d.password != "" {
req.SetBasicAuth(d.username, d.password)
}
return d.client.Do(req)
}
// bitbucketServerGitUser returns the user name to clone with for the user name an API response reports.
// Project and repository access tokens act as a technical user, which git does not accept, but "x-token-auth".
func bitbucketServerGitUser(reported string) string {
name, err := url.PathUnescape(reported)
if err != nil || name == "" {
return ""
}
if strings.HasPrefix(name, "access-token-user/") {
return "x-token-auth"
}
return name
}
// bitbucketServerRetryAfter returns the time to wait for a rate limited request: the Retry-After header,
// otherwise an exponential backoff.
func bitbucketServerRetryAfter(header string, attempt int) time.Duration {
wait := time.Duration(1<<(attempt-1)) * time.Second
if seconds, err := strconv.Atoi(strings.TrimSpace(header)); err == nil && seconds >= 0 {
wait = time.Duration(seconds) * time.Second
}
return min(wait, bitbucketServerMaxRetryWait)
}
func bitbucketServerSleep(ctx context.Context, d time.Duration) error {
timer := time.NewTimer(d)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return nil
}
}
func bbsGetPage[T any](ctx context.Context, d *BitbucketServerDownloader, path string, query url.Values, start, limit int) (*bbsPage[T], error) {
q := url.Values{}
maps.Copy(q, query)
q.Set("start", strconv.Itoa(start))
q.Set("limit", strconv.Itoa(limit))
var page bbsPage[T]
if _, err := d.callAPI(ctx, path, q, &page); err != nil {
return nil, err
}
return &page, nil
}
func bbsGetAll[T any](ctx context.Context, d *BitbucketServerDownloader, path string, query url.Values) ([]T, error) {
var all []T
start := 0
for {
page, err := bbsGetPage[T](ctx, d, path, query, start, bitbucketServerMaxPageSize)
if err != nil {
return nil, err
}
all = append(all, page.Values...)
if page.isLast(start) {
return all, nil
}
start = page.NextPageStart
}
}
// GetRepoInfo returns repository information
func (d *BitbucketServerDownloader) GetRepoInfo(ctx context.Context) (*base.Repository, error) {
var repo bbsRepository
header, err := d.callAPI(ctx, d.repoPath(), nil, &repo)
if err != nil {
return nil, err
}
if d.token != "" {
d.tokenUser = bitbucketServerGitUser(header.Get("X-AUSERNAME"))
}
// an empty repository has no default branch and the API answers 404
var branch struct {
DisplayID string `json:"displayId"`
}
if _, err := d.callAPI(ctx, d.repoPath()+"/default-branch", nil, &branch); err != nil {
if statusErr, ok := errors.AsType[*bbsStatusError](err); !ok || statusErr.Status != http.StatusNotFound {
return nil, err
}
}
return &base.Repository{
Name: repo.Slug,
Owner: repo.Project.Key,
IsPrivate: !repo.Public,
Description: repo.Description,
CloneURL: d.cloneURL(d.projectKey, d.repoSlug),
OriginalURL: d.browseURL(),
DefaultBranch: branch.DisplayID,
}, nil
}
// GetPullRequests returns a page of pull requests, oldest first
func (d *BitbucketServerDownloader) GetPullRequests(ctx context.Context, page, perPage int) ([]*base.PullRequest, bool, error) {
if page <= 1 {
d.prNextStart = 0
}
limit := min(perPage, bitbucketServerMaxPageSize)
if limit <= 0 {
limit = bitbucketServerMaxPageSize
}
// the server may cap the page size, so the offset is taken from the previous page
start := d.prNextStart
result, err := bbsGetPage[bbsPullRequest](ctx, d, d.repoPath()+"/pull-requests",
url.Values{"state": {"ALL"}, "order": {"oldest"}}, start, limit)
if err != nil {
return nil, false, err
}
d.prNextStart = result.NextPageStart
clear(d.activities)
prs := make([]*base.PullRequest, 0, len(result.Values))
for i := range result.Values {
prs = append(prs, d.convertPullRequest(ctx, &result.Values[i]))
}
return prs, result.isLast(start), nil
}
func (d *BitbucketServerDownloader) convertPullRequest(ctx context.Context, orig *bbsPullRequest) *base.PullRequest {
pr := &base.PullRequest{
Number: orig.ID,
ForeignIndex: orig.ID,
Title: orig.Title,
Content: orig.Description,
PosterID: orig.Author.User.ID,
PosterName: orig.Author.User.login(),
PosterEmail: orig.Author.User.EmailAddress,
State: "open",
Created: time.UnixMilli(orig.CreatedDate),
Updated: time.UnixMilli(orig.UpdatedDate),
IsDraft: orig.Draft,
Merged: orig.State == "MERGED",
Head: d.convertRef(&orig.FromRef),
Base: d.convertRef(&orig.ToRef),
}
// the merge base is computed by the uploader, the target branch tip given by the API may have moved on
pr.Base.SHA = ""
if orig.State != "OPEN" {
pr.State = "closed"
closed := time.UnixMilli(cmp.Or(orig.ClosedDate, orig.UpdatedDate))
pr.Closed = &closed
if pr.Merged {
pr.MergedTime = &closed
}
}
if pr.Merged {
d.completeMergedPullRequest(ctx, pr)
}
_ = CheckAndEnsureSafePR(pr, d.baseURL, d)
return pr
}
// completeMergedPullRequest adds the merge commit and the merge base, which the uploader cannot derive from
// the target branch as it already contains the merged head
func (d *BitbucketServerDownloader) completeMergedPullRequest(ctx context.Context, pr *base.PullRequest) {
activities, err := d.getActivities(ctx, pr.Number)
if err != nil {
log.Warn("Bitbucket Server: unable to load activities of pull request %d: %v", pr.Number, err)
}
for _, activity := range activities {
if activity.Action == "MERGED" && activity.Commit != nil {
pr.MergeCommitSHA = activity.Commit.ID
}
}
commits, err := bbsGetAll[bbsCommit](ctx, d, d.repoPath()+"/pull-requests/"+strconv.FormatInt(pr.Number, 10)+"/commits", nil)
if err != nil {
log.Warn("Bitbucket Server: unable to load commits of pull request %d: %v", pr.Number, err)
return
}
// listed newest first
if len(commits) > 0 && len(commits[len(commits)-1].Parents) > 0 {
pr.Base.SHA = commits[len(commits)-1].Parents[0].ID
}
}
func (d *BitbucketServerDownloader) convertRef(ref *bbsRef) base.PullRequestBranch {
branch := base.PullRequestBranch{Ref: ref.DisplayID, SHA: ref.LatestCommit}
if ref.Repository != nil {
branch.OwnerName = ref.Repository.Project.Key
branch.RepoName = ref.Repository.Slug
// built from the user supplied base URL instead of trusting the links of the response
branch.CloneURL = d.cloneURL(branch.OwnerName, branch.RepoName)
}
return branch
}
func (d *BitbucketServerDownloader) getActivities(ctx context.Context, prID int64) ([]*bbsActivity, error) {
if activities, ok := d.activities[prID]; ok {
return activities, nil
}
activities, err := bbsGetAll[*bbsActivity](ctx, d, d.repoPath()+"/pull-requests/"+strconv.FormatInt(prID, 10)+"/activities", nil)
if err != nil {
return nil, err
}
slices.Reverse(activities) // listed newest first
d.activities[prID] = activities
return activities, nil
}
// walkBitbucketServerComments visits each non-empty comment once, with the anchor of its thread.
// Deleted comments are skipped, also in the activity that added them.
func walkBitbucketServerComments(activities []*bbsActivity, visit func(c *bbsComment, anchor *bbsAnchor, isRoot bool)) {
seen := map[int64]bool{}
for _, activity := range activities {
if activity.CommentAction == "DELETED" && activity.Comment != nil {
seen[activity.Comment.ID] = true
}
}
var walk func(c *bbsComment, anchor *bbsAnchor, isRoot bool)
walk = func(c *bbsComment, anchor *bbsAnchor, isRoot bool) {
if c == nil || seen[c.ID] {
return
}
seen[c.ID] = true
if strings.TrimSpace(c.Text) != "" {
visit(c, anchor, isRoot)
}
for _, reply := range c.Comments {
walk(reply, anchor, false)
}
}
for _, activity := range activities {
if activity.Action == "COMMENTED" {
walk(activity.Comment, activity.CommentAnchor, true)
}
}
}
// GetComments returns the general and file comments and the decline/reopen events of a pull request
func (d *BitbucketServerDownloader) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) {
activities, err := d.getActivities(ctx, commentable.GetForeignIndex())
if err != nil {
return nil, false, err
}
var comments []*base.Comment
walkBitbucketServerComments(activities, func(c *bbsComment, anchor *bbsAnchor, _ bool) {
if anchor.isInline() {
return
}
content := c.Text
if anchor != nil && anchor.Path != "" {
content = fmt.Sprintf("Comment on `%s`:\n\n%s", anchor.Path, c.Text)
}
comments = append(comments, &base.Comment{
IssueIndex: commentable.GetLocalIndex(),
Index: c.ID,
PosterID: c.Author.ID,
PosterName: c.Author.login(),
PosterEmail: c.Author.EmailAddress,
Created: time.UnixMilli(c.CreatedDate),
Updated: c.updated(),
Content: content,
})
})
for _, activity := range activities {
var commentType string
switch activity.Action {
case "DECLINED":
commentType = "close"
case "REOPENED":
commentType = "reopen"
default:
continue
}
comments = append(comments, &base.Comment{
IssueIndex: commentable.GetLocalIndex(),
CommentType: commentType,
PosterID: activity.User.ID,
PosterName: activity.User.login(),
PosterEmail: activity.User.EmailAddress,
Created: time.UnixMilli(activity.CreatedDate),
Updated: time.UnixMilli(activity.CreatedDate),
})
}
slices.SortStableFunc(comments, func(a, b *base.Comment) int { return a.Created.Compare(b.Created) })
return comments, true, nil
}
// GetReviews returns the approvals, "needs work" votes and the inline code comments of a pull request
func (d *BitbucketServerDownloader) GetReviews(ctx context.Context, reviewable base.Reviewable) ([]*base.Review, error) {
activities, err := d.getActivities(ctx, reviewable.GetForeignIndex())
if err != nil {
return nil, err
}
var reviews []*base.Review
walkBitbucketServerComments(activities, func(c *bbsComment, anchor *bbsAnchor, isRoot bool) {
if !anchor.isInline() {
return
}
// every comment becomes its own review so that replies keep their author
reviews = append(reviews, &base.Review{
IssueIndex: reviewable.GetLocalIndex(),
ReviewerID: c.Author.ID,
ReviewerName: c.Author.login(),
CreatedAt: time.UnixMilli(c.CreatedDate),
State: base.ReviewStateCommented,
Comments: []*base.ReviewComment{c.toReviewComment(anchor, isRoot)},
})
})
lastVote := map[string]*bbsActivity{} // the last vote of every reviewer wins
for _, activity := range activities {
if activity.Action == "APPROVED" || activity.Action == "REVIEWED" || activity.Action == "UNAPPROVED" {
lastVote[activity.User.login()] = activity
}
}
for _, key := range slices.Sorted(maps.Keys(lastVote)) {
activity := lastVote[key]
if activity.Action == "UNAPPROVED" {
continue
}
state := base.ReviewStateApproved
if activity.Action == "REVIEWED" {
state = base.ReviewStateChangesRequested
}
reviews = append(reviews, &base.Review{
IssueIndex: reviewable.GetLocalIndex(),
ReviewerID: activity.User.ID,
ReviewerName: activity.User.login(),
CreatedAt: time.UnixMilli(activity.CreatedDate),
State: state,
})
}
slices.SortStableFunc(reviews, func(a, b *base.Review) int { return a.CreatedAt.Compare(b.CreatedAt) })
return reviews, nil
}
// toReviewComment converts a comment of a code thread, isRoot marks the first one, which carries the resolved state and line range
func (c *bbsComment) toReviewComment(anchor *bbsAnchor, isRoot bool) *base.ReviewComment {
rc := &base.ReviewComment{
ID: c.ID,
Content: c.Text,
TreePath: anchor.Path,
Line: anchor.Line,
CommitID: anchor.ToHash,
PosterID: c.Author.ID,
CreatedAt: time.UnixMilli(c.CreatedDate),
UpdatedAt: c.updated(),
Resolved: isRoot && c.ThreadResolved,
}
side := ""
if anchor.FileType == "FROM" {
side = " (before the change)"
rc.Line = -anchor.Line // a negative line addresses the old side of the diff
rc.CommitID = anchor.FromHash
}
// Gitea anchors a code comment to a single line only, so the range is kept in the text
if start, end, ok := anchor.lineRange(); isRoot && ok {
rc.Content = fmt.Sprintf("Lines %d-%d%s:\n\n%s", start, end, side, c.Text)
}
return rc
}
+511
View File
@@ -0,0 +1,511 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package migrations
import (
"cmp"
"context"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
"gitea.dev/models/unittest"
"gitea.dev/modules/json"
base "gitea.dev/modules/migration"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestParseBitbucketServerURL(t *testing.T) {
cases := []struct {
in string
baseURL, key, slug string
wantErr bool
}{
{in: "https://bb.example.com/scm/PROJ/repo.git", baseURL: "https://bb.example.com", key: "PROJ", slug: "repo"},
{in: "https://user:pass@bb.example.com:7990/scm/proj/repo", baseURL: "https://bb.example.com:7990", key: "proj", slug: "repo"},
{in: "https://bb.example.com/stash/scm/~jdoe/repo.git", baseURL: "https://bb.example.com/stash", key: "~jdoe", slug: "repo"},
{in: "https://bb.example.com/projects/PROJ/repos/repo/browse", baseURL: "https://bb.example.com", key: "PROJ", slug: "repo"},
{in: "https://bb.example.com/ctx/a/users/jdoe/repos/repo/pull-requests/1", baseURL: "https://bb.example.com/ctx/a", key: "~jdoe", slug: "repo"},
{in: "https://bb.example.com/projects/PROJ", wantErr: true},
{in: "https://bb.example.com/scm/PROJ", wantErr: true},
{in: "ssh://git@bb.example.com/PROJ/repo.git", wantErr: true},
{in: "/local/path/scm/PROJ/repo.git", wantErr: true},
}
for _, c := range cases {
baseURL, key, slug, err := parseBitbucketServerURL(c.in)
if c.wantErr {
assert.Error(t, err, c.in)
continue
}
require.NoError(t, err, c.in)
assert.Equal(t, c.baseURL, baseURL, c.in)
assert.Equal(t, c.key, key, c.in)
assert.Equal(t, c.slug, slug, c.in)
}
}
func TestBitbucketServerFormatCloneURL(t *testing.T) {
d := NewBitbucketServerDownloader(t.Context(), "https://bb.example.com", "", "", "", "PROJ", "repo")
const addr = "https://bb.example.com/scm/PROJ/repo.git"
got, err := d.FormatCloneURL(base.MigrateOptions{}, addr)
require.NoError(t, err)
assert.Equal(t, addr, got)
got, err = d.FormatCloneURL(base.MigrateOptions{AuthUsername: "u", AuthPassword: "p"}, addr)
require.NoError(t, err)
assert.Equal(t, "https://u:p@bb.example.com/scm/PROJ/repo.git", got)
got, err = d.FormatCloneURL(base.MigrateOptions{AuthToken: "tok"}, addr)
require.NoError(t, err)
assert.Equal(t, "https://token:tok@bb.example.com/scm/PROJ/repo.git", got)
// git needs the owner of the token as user name, which is learned from the API responses
d.tokenUser = "alice"
got, err = d.FormatCloneURL(base.MigrateOptions{AuthToken: "tok"}, addr)
require.NoError(t, err)
assert.Equal(t, "https://alice:tok@bb.example.com/scm/PROJ/repo.git", got)
got, err = d.FormatCloneURL(base.MigrateOptions{AuthToken: "tok", AuthUsername: "bob"}, addr)
require.NoError(t, err)
assert.Equal(t, "https://bob:tok@bb.example.com/scm/PROJ/repo.git", got)
assert.Equal(t, "j.doe", bitbucketServerGitUser("j.doe"))
// project and repository access tokens report a technical user
assert.Equal(t, "x-token-auth", bitbucketServerGitUser("access-token-user%2F1%2F2"))
assert.Equal(t, "x-token-auth", bitbucketServerGitUser("access-token-user/2/2"))
assert.Empty(t, bitbucketServerGitUser(""))
}
const bbsRepoJSON = `{"slug":"repo","name":"Repo","description":"a test repo","public":false,"project":{"key":"PROJ"}}`
func TestBitbucketServerRepoInfo(t *testing.T) {
var requests []*http.Request
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requests = append(requests, r)
w.Header().Set("X-AUSERNAME", "alice")
switch r.URL.Path {
case "/rest/api/1.0/projects/PROJ/repos/repo", "/rest/api/1.0/projects/~jdoe/repos/repo", "/rest/api/1.0/projects/DENIED/repos/repo":
_, _ = w.Write([]byte(bbsRepoJSON))
case "/rest/api/1.0/projects/~jdoe/repos/repo/default-branch":
_, _ = w.Write([]byte(`{"displayId":"main"}`))
case "/rest/api/1.0/projects/DENIED/repos/repo/default-branch":
w.WriteHeader(http.StatusUnauthorized)
default:
http.NotFound(w, r) // an empty repository has no default branch
}
}))
defer srv.Close()
d := NewBitbucketServerDownloader(t.Context(), srv.URL, "", "", "secret", "PROJ", "repo")
repo, err := d.GetRepoInfo(t.Context())
require.NoError(t, err)
assert.Equal(t, &base.Repository{
Name: "repo",
Owner: "PROJ",
IsPrivate: true,
Description: "a test repo",
CloneURL: srv.URL + "/scm/PROJ/repo.git",
OriginalURL: srv.URL + "/projects/PROJ/repos/repo",
}, repo)
assert.Equal(t, "Bearer secret", requests[0].Header.Get("Authorization"))
assert.Equal(t, "alice", d.tokenUser)
_, err = d.GetReleases(t.Context())
assert.True(t, base.IsErrNotSupported(err), "tags are synced from git")
requests = nil
repo, err = NewBitbucketServerDownloader(t.Context(), srv.URL, "u", "p", "", "~jdoe", "repo").GetRepoInfo(t.Context())
require.NoError(t, err)
assert.Equal(t, "main", repo.DefaultBranch)
assert.Equal(t, srv.URL+"/users/jdoe/repos/repo", repo.OriginalURL)
user, pass, ok := requests[0].BasicAuth()
assert.True(t, ok)
assert.Equal(t, "u", user)
assert.Equal(t, "p", pass)
_, err = NewBitbucketServerDownloader(t.Context(), srv.URL, "", "", "", "DENIED", "repo").GetRepoInfo(t.Context())
assert.True(t, IsAuthenticationError(err))
}
const bbsPullRequestsPage1JSON = `{"isLastPage":false,"nextPageStart":1,"values":[
{"id":1,"title":"merged","description":"d1","state":"MERGED","createdDate":1700000000000,"updatedDate":1700000300000,"closedDate":1700000200000,
"author":{"user":{"id":7,"name":"alice","emailAddress":"alice@example.com"}},
"fromRef":{"displayId":"feature","latestCommit":"HEAD_SHA","repository":{"slug":"repo","project":{"key":"PROJ"}}},
"toRef":{"displayId":"main","latestCommit":"BASE_SHA","repository":{"slug":"repo","project":{"key":"PROJ"}}}}
]}`
const bbsPullRequestsPage2JSON = `{"isLastPage":true,"values":[
{"id":2,"title":"fork pr","state":"OPEN","draft":true,"createdDate":1700001000000,"updatedDate":1700001000000,
"author":{"user":{"id":8,"slug":"bob"}},
"fromRef":{"displayId":"patch","latestCommit":"HEAD_SHA","repository":{"slug":"repo-fork","project":{"key":"~BOB"}}},
"toRef":{"displayId":"main","latestCommit":"BASE_SHA","repository":{"slug":"repo","project":{"key":"PROJ"}}}},
{"id":3,"title":"declined","state":"DECLINED","createdDate":1700002000000,"updatedDate":1700002500000,
"author":{"user":{"id":7,"name":"alice"}},
"fromRef":{"displayId":"x","latestCommit":"not-a-sha","repository":{"slug":"repo","project":{"key":"PROJ"}}},
"toRef":{"displayId":"main","latestCommit":"BASE_SHA","repository":{"slug":"repo","project":{"key":"PROJ"}}}}
]}`
func TestBitbucketServerPullRequests(t *testing.T) {
headSHA, baseSHA, mergeSHA, forkPoint := strings.Repeat("1", 40), strings.Repeat("2", 40), strings.Repeat("3", 40), strings.Repeat("4", 40)
const api = "/rest/api/1.0/projects/PROJ/repos/repo"
shas := strings.NewReplacer("HEAD_SHA", headSHA, "BASE_SHA", baseSHA)
var queries []string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case api + "/pull-requests/1/commits":
_, _ = w.Write([]byte(`{"isLastPage":true,"values":[{"id":"c2","parents":[{"id":"c1"}]},{"id":"c1","parents":[{"id":"` + forkPoint + `"}]}]}`))
case api + "/pull-requests/1/activities":
_, _ = w.Write([]byte(`{"isLastPage":true,"values":[{"action":"MERGED","commit":{"id":"` + mergeSHA + `"}}]}`))
case api + "/pull-requests":
queries = append(queries, r.URL.RawQuery)
body := bbsPullRequestsPage1JSON
if r.URL.Query().Get("start") != "0" {
body = bbsPullRequestsPage2JSON
}
_, _ = w.Write([]byte(shas.Replace(body)))
default:
assert.Failf(t, "unexpected request", "%s", r.URL.Path)
}
}))
defer srv.Close()
d := NewBitbucketServerDownloader(t.Context(), srv.URL, "", "", "", "PROJ", "repo")
prs, isEnd, err := d.GetPullRequests(t.Context(), 1, 500)
require.NoError(t, err)
assert.False(t, isEnd)
require.Len(t, prs, 1)
page2, isEnd, err := d.GetPullRequests(t.Context(), 2, 500)
require.NoError(t, err)
assert.True(t, isEnd)
prs = append(prs, page2...)
// the page size is capped, and the server answering with fewer values must not skip any
assert.Equal(t, []string{"limit=100&order=oldest&start=0&state=ALL", "limit=100&order=oldest&start=1&state=ALL"}, queries)
require.Len(t, prs, 3)
merged := prs[0]
assert.EqualValues(t, 1, merged.Number)
assert.EqualValues(t, 1, merged.GetForeignIndex())
assert.Equal(t, "closed", merged.State)
assert.True(t, merged.Merged)
assert.Equal(t, mergeSHA, merged.MergeCommitSHA)
assert.Equal(t, time.UnixMilli(1700000200000), *merged.MergedTime)
assert.Equal(t, time.UnixMilli(1700000200000), *merged.Closed)
assert.Equal(t, "alice", merged.PosterName)
assert.EqualValues(t, 7, merged.PosterID)
assert.Equal(t, "feature", merged.Head.Ref)
assert.Equal(t, headSHA, merged.Head.SHA)
assert.Equal(t, "main", merged.Base.Ref)
assert.Equal(t, forkPoint, merged.Base.SHA, "parent of the oldest PR commit, the target branch has moved on")
assert.False(t, merged.IsForkPullRequest())
assert.True(t, merged.EnsuredSafe)
fork := prs[1]
assert.Equal(t, "open", fork.State)
assert.True(t, fork.IsDraft)
assert.Equal(t, "bob", fork.PosterName)
assert.True(t, fork.IsForkPullRequest())
assert.Equal(t, srv.URL+"/scm/~BOB/repo-fork.git", fork.Head.CloneURL)
assert.Equal(t, "~BOB", fork.Head.OwnerName)
assert.Equal(t, "patch", fork.Head.Ref)
assert.Nil(t, fork.Closed)
declined := prs[2]
assert.Equal(t, "closed", declined.State)
assert.False(t, declined.Merged)
assert.Equal(t, time.UnixMilli(1700002500000), *declined.Closed, "falls back to the update time")
assert.Empty(t, declined.Head.SHA, "invalid SHA is dropped")
}
// newest first, like the API
const bbsActivitiesJSON = `{"isLastPage":false,"nextPageStart":10,"values":[
{"action":"REOPENED","createdDate":1700000900000,"user":{"id":7,"name":"alice"}},
{"action":"DECLINED","createdDate":1700000800000,"user":{"id":7,"name":"alice"}},
{"action":"UNAPPROVED","createdDate":1700000720000,"user":{"id":10,"name":"dave"}},
{"action":"APPROVED","createdDate":1700000710000,"user":{"id":10,"name":"dave"}},
{"action":"REVIEWED","createdDate":1700000700000,"user":{"id":9,"name":"carol"}},
{"action":"COMMENTED","commentAction":"DELETED","createdDate":1700000650000,"user":{"id":7,"name":"alice"},
"comment":{"id":16,"text":"deleted","author":{"id":7,"name":"alice"},"createdDate":1700000650000}},
{"action":"COMMENTED","commentAction":"ADDED","createdDate":1700000640000,"user":{"id":7,"name":"alice"},
"comment":{"id":16,"text":"deleted","author":{"id":7,"name":"alice"},"createdDate":1700000640000}},
{"action":"COMMENTED","commentAction":"ADDED","createdDate":1700000600000,"user":{"id":7,"name":"alice"},
"commentAnchor":{"path":"README.md","fileType":"TO"},
"comment":{"id":15,"text":"file comment","author":{"id":7,"name":"alice"},"createdDate":1700000600000,"updatedDate":1700000600000}},
{"action":"COMMENTED","commentAction":"ADDED","createdDate":1700000500000,"user":{"id":7,"name":"alice"},
"commentAnchor":{"line":5,"lineType":"REMOVED","fileType":"FROM","path":"src/b.go","srcPath":"src/old_b.go","fromHash":"fffffffffffffffffffffffffffffffffffffff0","toHash":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee0"},
"comment":{"id":14,"text":"inline old","author":{"id":7,"name":"alice"},"createdDate":1700000500000,"updatedDate":1700000500000}},
{"action":"COMMENTED","commentAction":"ADDED","createdDate":1700000400000,"user":{"id":8,"name":"bob"},
"commentAnchor":{"line":12,"lineType":"ADDED","fileType":"TO","path":"src/a.go","srcPath":"src/a.go","fromHash":"fffffffffffffffffffffffffffffffffffffff0","toHash":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee0"},
"comment":{"id":12,"text":"inline new","author":{"id":8,"name":"bob"},"createdDate":1700000400000,"updatedDate":1700000400000,
"comments":[{"id":13,"text":"inline reply","author":{"id":7,"name":"alice"},"createdDate":1700000450000,"updatedDate":1700000450000}]}}
]}`
const bbsActivitiesPage2JSON = `{"isLastPage":true,"values":[
{"action":"APPROVED","createdDate":1700000300000,"user":{"id":8,"name":"bob"}},
{"action":"COMMENTED","commentAction":"ADDED","createdDate":1700000100000,"user":{"id":8,"name":"bob"},
"comment":{"id":10,"text":"general comment","author":{"id":8,"name":"bob","emailAddress":"bob@example.com"},"createdDate":1700000100000,"updatedDate":1700000150000,
"comments":[{"id":11,"text":"a reply","author":{"id":7,"name":"alice"},"createdDate":1700000200000,"updatedDate":1700000200000}]}},
{"action":"OPENED","createdDate":1700000000000,"user":{"id":7,"name":"alice"}}
]}`
func TestBitbucketServerCommentsAndReviews(t *testing.T) {
var activityCalls int
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "/rest/api/1.0/projects/PROJ/repos/repo/pull-requests/4/activities", r.URL.Path)
activityCalls++
if r.URL.Query().Get("start") == "10" {
_, _ = w.Write([]byte(bbsActivitiesPage2JSON))
return
}
_, _ = w.Write([]byte(bbsActivitiesJSON))
}))
defer srv.Close()
d := NewBitbucketServerDownloader(t.Context(), srv.URL, "", "", "", "PROJ", "repo")
pr := &base.PullRequest{Number: 4, ForeignIndex: 4}
comments, isEnd, err := d.GetComments(t.Context(), pr)
require.NoError(t, err)
assert.True(t, isEnd)
require.Len(t, comments, 5)
assert.Equal(t, "general comment", comments[0].Content)
assert.Equal(t, "bob", comments[0].PosterName)
assert.Equal(t, "bob@example.com", comments[0].PosterEmail)
assert.EqualValues(t, 10, comments[0].Index)
assert.EqualValues(t, 4, comments[0].IssueIndex)
assert.Equal(t, time.UnixMilli(1700000150000), comments[0].Updated)
assert.Equal(t, "a reply", comments[1].Content)
assert.Equal(t, "alice", comments[1].PosterName)
assert.Equal(t, "Comment on `README.md`:\n\nfile comment", comments[2].Content)
assert.Equal(t, "close", comments[3].CommentType)
assert.Equal(t, "reopen", comments[4].CommentType)
// the deleted comment is skipped, inline comments belong to the reviews
for _, c := range comments {
assert.NotContains(t, c.Content, "deleted")
assert.NotContains(t, c.Content, "inline")
}
reviews, err := d.GetReviews(t.Context(), pr)
require.NoError(t, err)
assert.Equal(t, 2, activityCalls, "activities are paged and cached for both calls")
type summary struct {
reviewer, state, path string
line int
}
var got []summary
for _, r := range reviews {
s := summary{reviewer: r.ReviewerName, state: r.State}
if len(r.Comments) > 0 {
s.path, s.line = r.Comments[0].TreePath, r.Comments[0].Line
}
got = append(got, s)
}
assert.Equal(t, []summary{
{"bob", base.ReviewStateApproved, "", 0},
{"bob", base.ReviewStateCommented, "src/a.go", 12},
{"alice", base.ReviewStateCommented, "src/a.go", 12},
{"alice", base.ReviewStateCommented, "src/b.go", -5},
{"carol", base.ReviewStateChangesRequested, "", 0},
}, got, "dave's approval was withdrawn")
assert.Equal(t, "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee0", reviews[1].Comments[0].CommitID)
assert.Equal(t, "fffffffffffffffffffffffffffffffffffffff0", reviews[3].Comments[0].CommitID)
assert.Equal(t, "inline reply", reviews[2].Comments[0].Content)
assert.EqualValues(t, 4, reviews[0].IssueIndex)
}
// TestBitbucketServerDownloadRepo replays responses recorded from a Bitbucket Server 10.5.1 instance.
// To record again, set BITBUCKETSERVER_LIVE_TOKEN (an HTTP access token with project read permission) and
// optionally BITBUCKETSERVER_LIVE_URL (default http://localhost:7990/bitbucket) of an instance holding the
// project MIG with the repository demo.
func TestBitbucketServerDownloadRepo(t *testing.T) {
liveToken := os.Getenv("BITBUCKETSERVER_LIVE_TOKEN")
liveURL := cmp.Or(os.Getenv("BITBUCKETSERVER_LIVE_URL"), "http://localhost:7990/bitbucket")
_, callerFile, _, _ := runtime.Caller(0)
fixtureDir := filepath.Join(filepath.Dir(callerFile), "_mock_data/TestBitbucketServerDownloadRepo")
mockServer := unittest.NewMockWebServer(t, liveURL, fixtureDir, liveToken != "")
ctx := t.Context()
token := cmp.Or(liveToken, "replayed") // a token makes the downloader learn the name of its owner
d := NewBitbucketServerDownloader(ctx, mockServer.URL, "", "", token, "MIG", "demo")
repo, err := d.GetRepoInfo(ctx)
require.NoError(t, err)
assert.Equal(t, &base.Repository{
Name: "demo",
Owner: "MIG",
IsPrivate: true,
CloneURL: mockServer.URL + "/scm/MIG/demo.git",
OriginalURL: mockServer.URL + "/projects/MIG/repos/demo",
DefaultBranch: "main",
}, repo)
prs, isEnd, err := d.GetPullRequests(ctx, 1, 50)
require.NoError(t, err)
assert.True(t, isEnd)
require.Len(t, prs, 4)
// open PR from a branch of the same repository
open := prs[0]
assert.Equal(t, "Open PR", open.Title)
assert.Equal(t, "open", open.State)
assert.False(t, open.IsDraft)
assert.Equal(t, "admin", open.PosterName)
assert.Equal(t, "feature", open.Head.Ref)
assert.Equal(t, "810d0600716f4575691c804b583c1c6a6e6708db", open.Head.SHA)
assert.Equal(t, "main", open.Base.Ref)
assert.False(t, open.IsForkPullRequest())
merged := prs[1]
assert.Equal(t, "closed", merged.State)
assert.True(t, merged.Merged)
assert.Equal(t, "504e78ed1c8fcbc0e55f5f0b6d3c49a0bc1ae861", merged.MergeCommitSHA)
assert.Equal(t, "a41991381bd1e6944b3422cd1dc933b65c74cf67", merged.Base.SHA)
assert.NotNil(t, merged.MergedTime)
declined := prs[2]
assert.Equal(t, "closed", declined.State)
assert.False(t, declined.Merged)
assert.NotNil(t, declined.Closed)
// draft PR from a private fork
fork := prs[3]
assert.True(t, fork.IsDraft)
assert.Equal(t, "bob", fork.PosterName)
assert.Equal(t, "forkbranch", fork.Head.Ref)
assert.Equal(t, "aa537b277722995337c78d4d75018202091c0793", fork.Head.SHA)
assert.True(t, fork.IsForkPullRequest())
assert.Equal(t, mockServer.URL+"/scm/~BOB/demo.git", fork.Head.CloneURL)
comments, _, err := d.GetComments(ctx, open)
require.NoError(t, err)
var got []string
for _, c := range comments {
got = append(got, c.PosterName+": "+c.Content)
}
assert.Equal(t, []string{
"admin: General comment from admin",
"bob: Reply from bob",
"carol: Comment on `a.txt`:\n\nFile level comment",
}, got)
reviews, err := d.GetReviews(ctx, open)
require.NoError(t, err)
type summary struct {
reviewer, state, path, content string
line int
resolved bool
}
var reviewed []summary
for _, r := range reviews {
s := summary{reviewer: r.ReviewerName, state: r.State}
if len(r.Comments) > 0 {
c := r.Comments[0]
s.path, s.line, s.content, s.resolved = c.TreePath, c.Line, c.Content, c.Resolved
}
reviewed = append(reviewed, s)
}
assert.Equal(t, []summary{
{"bob", base.ReviewStateCommented, "a.txt", "Inline on added line", 2, false},
{"admin", base.ReviewStateCommented, "a.txt", "Inline reply by admin", 2, false},
{"carol", base.ReviewStateCommented, "a.txt", "Inline on removed line", -2, false},
{"bob", base.ReviewStateApproved, "", "", 0, false},
{"carol", base.ReviewStateChangesRequested, "", "", 0, false},
{"bob", base.ReviewStateCommented, "a.txt", "Lines 2-4:\n\nMulti line comment", 4, false},
{"bob", base.ReviewStateCommented, "a.txt", "Thread to resolve", 6, true},
{"admin", base.ReviewStateCommented, "a.txt", "reply on resolved thread", 6, false},
}, reviewed, "the resolved state is only set on the first comment of a thread")
declinedComments, _, err := d.GetComments(ctx, declined)
require.NoError(t, err)
require.Len(t, declinedComments, 2)
assert.Equal(t, "close", declinedComments[0].CommentType)
assert.Equal(t, "declining this", declinedComments[1].Content)
// a token is used to clone, with the name of its owner
cloneURL, err := d.FormatCloneURL(base.MigrateOptions{AuthToken: token}, repo.CloneURL)
require.NoError(t, err)
assert.Contains(t, cloneURL, "admin:")
}
func TestBitbucketServerInlineCommentRange(t *testing.T) {
var anchor bbsAnchor
require.NoError(t, json.Unmarshal([]byte(`{"line":9,"fileType":"FROM","path":"b","srcPath":"a",
"multilineMarker":{"startLine":7},"multilineSpan":{"srcSpanStart":3,"srcSpanEnd":5,"dstSpanStart":7,"dstSpanEnd":9}}`), &anchor))
c := &bbsComment{ID: 1, Text: "text", ThreadResolved: true}
// the span of the commented side wins, replies neither get the range nor the resolved state
rc := c.toReviewComment(&anchor, true)
assert.Equal(t, "Lines 3-5 (before the change):\n\ntext", rc.Content)
assert.Equal(t, -9, rc.Line)
assert.Equal(t, "b", rc.TreePath, "Gitea files old-side comments under the current path as well")
assert.True(t, rc.Resolved)
rc = c.toReviewComment(&anchor, false)
assert.Equal(t, "text", rc.Content)
assert.False(t, rc.Resolved)
// without a span the marker and the last line are used, a single line has no range
require.NoError(t, json.Unmarshal([]byte(`{"line":9,"fileType":"TO","path":"a","multilineMarker":{"startLine":7}}`), &anchor))
start, end, ok := anchor.lineRange()
assert.True(t, ok)
assert.Equal(t, [2]int{7, 9}, [2]int{start, end})
_, _, ok = (&bbsAnchor{Line: 9, FileType: "TO"}).lineRange()
assert.False(t, ok)
}
func TestBitbucketServerRateLimit(t *testing.T) {
var calls int
retryAfter, limitedUntil := "3", 2
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
if calls <= limitedUntil {
if retryAfter != "" {
w.Header().Set("Retry-After", retryAfter)
}
w.WriteHeader(http.StatusTooManyRequests)
return
}
_, _ = w.Write([]byte(bbsRepoJSON))
}))
defer srv.Close()
var waits []time.Duration
newDownloader := func() *BitbucketServerDownloader {
d := NewBitbucketServerDownloader(t.Context(), srv.URL, "", "", "", "PROJ", "repo")
d.sleep = func(_ context.Context, wait time.Duration) error {
waits = append(waits, wait)
return nil
}
return d
}
// the server is asked again after the time it names
var repo bbsRepository
_, err := newDownloader().callAPI(t.Context(), "/projects/PROJ/repos/repo", nil, &repo)
require.NoError(t, err)
assert.Equal(t, "repo", repo.Slug)
assert.Equal(t, []time.Duration{3 * time.Second, 3 * time.Second}, waits)
// without a hint the wait grows exponentially, and is capped
calls, limitedUntil, retryAfter, waits = 0, 2, "", nil
_, err = newDownloader().callAPI(t.Context(), "/projects/PROJ/repos/repo", nil, &repo)
require.NoError(t, err)
assert.Equal(t, []time.Duration{time.Second, 2 * time.Second}, waits)
assert.Equal(t, time.Minute, bitbucketServerRetryAfter("100000", 1))
// a server that never recovers is given up on
calls, limitedUntil, waits = 0, 100, nil
_, err = newDownloader().callAPI(t.Context(), "/projects/PROJ/repos/repo", nil, &repo)
var statusErr *bbsStatusError
require.ErrorAs(t, err, &statusErr)
assert.Equal(t, http.StatusTooManyRequests, statusErr.Status)
assert.Len(t, waits, bitbucketServerMaxAttempts-1)
assert.Equal(t, bitbucketServerMaxAttempts, calls)
}
+3
View File
@@ -33,6 +33,9 @@ func IsAuthenticationError(err error) bool {
if gitcmd.IsStderr(err, gitcmd.StderrAuthenticationFailed, gitcmd.StderrCouldNotReadUsername) {
return true
}
if bbsErr, ok := errors.AsType[*bbsStatusError](err); ok {
return bbsErr.isAuthError()
}
githubErr, ok := errors.AsType[*github.ErrorResponse](err)
return ok && githubErr.Response != nil && githubErr.Response.StatusCode == http.StatusUnauthorized
}
+37 -29
View File
@@ -597,6 +597,11 @@ func (g *GiteaLocalUploader) CreatePullRequests(ctx context.Context, prs ...*bas
return nil
}
func (g *GiteaLocalUploader) commitExists(ctx context.Context, sha string) bool {
_, _, err := gitcmd.NewCommand("rev-list", "--quiet", "-1").AddDynamicArguments(sha).WithRepo(g.repo).RunStdString(ctx)
return err == nil
}
func (g *GiteaLocalUploader) updateGitForPullRequest(ctx context.Context, pr *base.PullRequest) (head string, err error) {
// SECURITY: this pr must have been must have been ensured safe
if !pr.EnsuredSafe {
@@ -646,10 +651,14 @@ func (g *GiteaLocalUploader) updateGitForPullRequest(ctx context.Context, pr *ba
return head, nil
}
// ... the head commit may already be in the repository (e.g. fetched by the mirror clone as a hidden
// pull request ref), then no remote is needed and a private fork does not need credentials either
headIsLocal := pr.Head.SHA != "" && g.commitExists(ctx, pr.Head.SHA)
// 3. We need to create a remote for this clone url
// ... maybe we already have a name for this remote
remote, ok := g.prHeadCache[pr.Head.CloneURL+":"]
if !ok {
if !ok && !headIsLocal {
// ... let's try ownername as a reasonable name
remote = pr.Head.OwnerName
if !git.IsValidRefPattern(remote) {
@@ -657,16 +666,11 @@ func (g *GiteaLocalUploader) updateGitForPullRequest(ctx context.Context, pr *ba
remote = "head-pr-" + strconv.FormatInt(pr.Number, 10)
}
// ... now add the remote
err := g.gitRepo.AddRemote(ctx, remote, pr.Head.CloneURL, true)
if err != nil {
if err := g.gitRepo.AddRemote(ctx, remote, pr.Head.CloneURL, true); err != nil {
log.Error("PR #%d in %s/%s AddRemote[%s] failed: %v", pr.Number, g.repoOwner, g.repoName, remote, err)
} else {
g.prHeadCache[pr.Head.CloneURL+":"] = remote
ok = true
return head, nil
}
}
if !ok {
return head, nil
g.prHeadCache[pr.Head.CloneURL+":"] = remote
}
// 4. Check if we already have this ref?
@@ -690,15 +694,22 @@ func (g *GiteaLocalUploader) updateGitForPullRequest(ctx context.Context, pr *ba
}
}
fetchArg := pr.Head.Ref + ":" + git.BranchPrefix + localRef
if strings.HasPrefix(fetchArg, "-") {
fetchArg = git.BranchPrefix + fetchArg
}
if headIsLocal {
if err = git.UpdateRef(ctx, g.repo, git.BranchPrefix+localRef, pr.Head.SHA); err != nil {
log.Error("Creating branch %s for PR #%d in %s/%s failed: %v", localRef, pr.Number, g.repoOwner, g.repoName, err)
return head, nil
}
} else {
fetchArg := pr.Head.Ref + ":" + git.BranchPrefix + localRef
if strings.HasPrefix(fetchArg, "-") {
fetchArg = git.BranchPrefix + fetchArg
}
_, _, err = gitcmd.NewCommand("fetch", "--no-tags").AddDashesAndList(remote, fetchArg).WithRepo(g.repo).RunStdString(ctx)
if err != nil {
log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err)
return head, nil
_, _, err = gitcmd.NewCommand("fetch", "--no-tags").AddDashesAndList(remote, fetchArg).WithRepo(g.repo).RunStdString(ctx)
if err != nil {
log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err)
return head, nil
}
}
g.prHeadCache[pr.Head.CloneURL+":"+pr.Head.Ref] = localRef
head = localRef
@@ -729,18 +740,11 @@ func (g *GiteaLocalUploader) updateGitForPullRequest(ctx context.Context, pr *ba
if pr.Head.SHA == "" {
// The SHA is empty
log.Warn("Empty reference, no pull head for PR #%d in %s/%s", pr.Number, g.repoOwner, g.repoName)
} else {
_, _, err = gitcmd.NewCommand("rev-list", "--quiet", "-1").
AddDynamicArguments(pr.Head.SHA).WithRepo(g.repo).RunStdString(ctx)
if err != nil {
// Git update-ref remove bad references with a relative path
log.Warn("Deprecated local head %s for PR #%d in %s/%s, removing %s", pr.Head.SHA, pr.Number, g.repoOwner, g.repoName, pr.GetGitHeadRefName())
} else {
// set head information
if err = git.UpdateRef(ctx, g.repo, pr.GetGitHeadRefName(), pr.Head.SHA); err != nil {
log.Error("unable to set %s as the local head for PR #%d from %s in %s/%s. Error: %v", pr.Head.SHA, pr.Number, pr.Head.Ref, g.repoOwner, g.repoName, err)
}
}
} else if !g.commitExists(ctx, pr.Head.SHA) {
// Git update-ref remove bad references with a relative path
log.Warn("Deprecated local head %s for PR #%d in %s/%s, removing %s", pr.Head.SHA, pr.Number, g.repoOwner, g.repoName, pr.GetGitHeadRefName())
} else if err = git.UpdateRef(ctx, g.repo, pr.GetGitHeadRefName(), pr.Head.SHA); err != nil {
log.Error("unable to set %s as the local head for PR #%d from %s in %s/%s. Error: %v", pr.Head.SHA, pr.Number, pr.Head.Ref, g.repoOwner, g.repoName, err)
}
return head, nil
@@ -956,6 +960,10 @@ func (g *GiteaLocalUploader) CreateReviews(ctx context.Context, reviews ...*base
UpdatedUnix: timeutil.TimeStamp(comment.UpdatedAt.Unix()),
}
if comment.Resolved {
c.ResolveDoerID = g.doer.ID // the original resolver is unknown to the migration data
}
if err := g.remapUser(ctx, review, &c); err != nil {
return err
}
@@ -6,6 +6,7 @@ package migrations
import (
"strconv"
"strings"
"testing"
"time"
@@ -14,6 +15,8 @@ import (
repo_model "gitea.dev/models/repo"
"gitea.dev/models/unittest"
user_model "gitea.dev/models/user"
"gitea.dev/modules/git"
"gitea.dev/modules/git/gitcmd"
"gitea.dev/modules/graceful"
base "gitea.dev/modules/migration"
"gitea.dev/modules/optional"
@@ -224,6 +227,88 @@ func TestGiteaUploadRemapExternalUser(t *testing.T) {
assert.Equal(t, linkedUser.ID, target.GetUserID())
}
func TestGiteaUploadReviewCommentResolved(t *testing.T) {
unittest.PrepareTestEnv(t)
ctx := t.Context()
doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1})
pull := unittest.AssertExistsAndLoadBean(t, &issues_model.PullRequest{ID: 1})
require.NoError(t, pull.LoadIssue(ctx))
uploader := NewGiteaLocalUploader(ctx, doer, repo.OwnerName, repo.Name)
var err error
uploader.repo = repo
uploader.gitRepo, err = git.OpenRepository(ctx, repo)
require.NoError(t, err)
defer uploader.Close()
uploader.issues[pull.Issue.Index] = pull.Issue
headCommitID, err := uploader.gitRepo.GetBranchCommitID(ctx, repo.DefaultBranch)
require.NoError(t, err)
require.NoError(t, git.UpdateRef(ctx, repo, pull.GetGitHeadRefName(), headCommitID))
require.NoError(t, uploader.CreateReviews(ctx, &base.Review{
IssueIndex: pull.Issue.Index,
ReviewerID: doer.ID,
State: base.ReviewStateCommented,
Comments: []*base.ReviewComment{
{Content: "resolved thread", TreePath: "README.md", Line: 1, Resolved: true},
{Content: "open thread", TreePath: "README.md", Line: 1},
},
}))
comments, err := issues_model.FindComments(ctx, &issues_model.FindCommentsOptions{IssueID: pull.Issue.ID, Type: issues_model.CommentTypeCode})
require.NoError(t, err)
resolved := map[string]bool{}
for _, c := range comments {
if c.Content == "resolved thread" || c.Content == "open thread" {
resolved[c.Content] = c.ResolveDoerID == doer.ID
}
}
assert.Equal(t, map[string]bool{"resolved thread": true, "open thread": false}, resolved)
}
func TestGiteaUploadForkPullRequestHeadInRepository(t *testing.T) {
unittest.PrepareTestEnv(t)
ctx := t.Context()
doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1})
uploader := NewGiteaLocalUploader(ctx, doer, repo.OwnerName, repo.Name)
var err error
uploader.repo = repo
uploader.gitRepo, err = git.OpenRepository(ctx, repo)
require.NoError(t, err)
defer uploader.Close()
headCommitID, err := uploader.gitRepo.GetBranchCommitID(ctx, repo.DefaultBranch)
require.NoError(t, err)
// the fork cannot be reached, but its head commit is already in the repository
pr := &base.PullRequest{
Number: 99,
State: "open",
Head: base.PullRequestBranch{OwnerName: "fork-owner", RepoName: "repo", Ref: "feature", SHA: headCommitID, CloneURL: "https://fork.invalid/fork-owner/repo.git"},
Base: base.PullRequestBranch{OwnerName: repo.OwnerName, RepoName: repo.Name, Ref: repo.DefaultBranch},
}
pr.EnsuredSafe = true
require.True(t, pr.IsForkPullRequest())
head, err := uploader.updateGitForPullRequest(ctx, pr)
require.NoError(t, err)
assert.Equal(t, "fork-owner/feature", head)
assert.True(t, uploader.gitRepo.IsBranchExist(ctx, "fork-owner/feature"))
branchCommitID, err := uploader.gitRepo.GetBranchCommitID(ctx, "fork-owner/feature")
require.NoError(t, err)
assert.Equal(t, headCommitID, branchCommitID)
refCommitID, err := uploader.gitRepo.GetRefCommitID(ctx, pr.GetGitHeadRefName())
require.NoError(t, err)
assert.Equal(t, headCommitID, refCommitID)
remotes, _, err := gitcmd.NewCommand("remote").WithRepo(repo).RunStdString(ctx)
require.NoError(t, err)
assert.Empty(t, strings.TrimSpace(remotes), "no remote is added for a local head")
}
func TestGiteaUploadTimelineLabelAndMilestoneComments(t *testing.T) {
unittest.PrepareTestEnv(t)
doer := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
+3
View File
@@ -32,6 +32,9 @@ func TestIsAuthenticationError(t *testing.T) {
{"github unauthorized", true, util.SanitizeErrorCredentialURLs(&github.ErrorResponse{Response: &http.Response{StatusCode: http.StatusUnauthorized}})},
{"github other", false, &github.ErrorResponse{Response: &http.Response{StatusCode: http.StatusNotFound}}},
{"github nil response", false, &github.ErrorResponse{}},
{"bitbucket server unauthorized", true, fmt.Errorf("wrapped: %w", &bbsStatusError{Status: http.StatusUnauthorized})},
{"bitbucket server forbidden", true, &bbsStatusError{Status: http.StatusForbidden}},
{"bitbucket server not found", false, &bbsStatusError{Status: http.StatusNotFound}},
{"unrelated error", false, errDummy},
}
for _, c := range cases {