gitea/services/migrations/onedev.go

// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package migrations

import (
	"context"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"gitea.dev/modules/json"
	"gitea.dev/modules/log"
	base "gitea.dev/modules/migration"
	"gitea.dev/modules/structs"

	"github.com/hashicorp/go-version"
)

const (
	OneDevRequiredVersion        = "12.0.1"
	maxOneDevVersionResponseSize = 1024
)

var (
	_ base.Downloader        = &OneDevDownloader{}
	_ base.DownloaderFactory = &OneDevDownloaderFactory{}
)

func init() {
	RegisterDownloaderFactory(&OneDevDownloaderFactory{})
}

// OneDevDownloaderFactory defines a downloader factory
type OneDevDownloaderFactory struct{}

// New returns a downloader related to this factory according MigrateOptions
func (f *OneDevDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
	u, err := url.Parse(opts.CloneAddr)
	if err != nil {
		return nil, err
	}

	repoPath := strings.Trim(u.Path, "/")

	u.Path = ""
	u.Fragment = ""

	log.Trace("Create onedev downloader. BaseURL: %v RepoPath: %s", u, repoPath)

	return NewOneDevDownloader(ctx, u, opts.AuthUsername, opts.AuthPassword, repoPath), nil
}

// GitServiceType returns the type of git service
func (f *OneDevDownloaderFactory) GitServiceType() structs.GitServiceType {
	return structs.OneDevService
}

type onedevUser struct {
	ID    int64
	Name  string
	Email string
}

// OneDevDownloader implements a Downloader interface to get repository information
// from OneDev
type OneDevDownloader struct {
	base.NullDownloader
	client        *http.Client
	baseURL       *url.URL
	repoPath      string
	repoID        int64
	maxIssueIndex int64
	userMap       map[int64]*onedevUser
	milestoneMap  map[int64]string
}

// NewOneDevDownloader creates a new downloader
func NewOneDevDownloader(_ context.Context, baseURL *url.URL, username, password, repoPath string) *OneDevDownloader {
	return &OneDevDownloader{
		baseURL:  baseURL,
		repoPath: repoPath,

		client:       newMigrationHTTPClient(baseURL.String(), basicAuthorization(username, password)),
		userMap:      make(map[int64]*onedevUser),
		milestoneMap: make(map[int64]string),
	}
}

// String implements Stringer
func (d *OneDevDownloader) String() string {
	return fmt.Sprintf("migration from oneDev server %s [%d]/%s", d.baseURL, d.repoID, d.repoPath)
}

func (d *OneDevDownloader) LogString() string {
	if d == nil {
		return "<OneDevDownloader nil>"
	}
	return fmt.Sprintf("<OneDevDownloader %s [%d]/%s>", d.baseURL, d.repoID, d.repoPath)
}

func (d *OneDevDownloader) callAPI(ctx context.Context, endpoint string, parameter map[string]string, result any) error {
	u, err := d.baseURL.Parse(endpoint)
	if err != nil {
		return err
	}

	if parameter != nil {
		query := u.Query()
		for k, v := range parameter {
			query.Set(k, v)
		}
		u.RawQuery = query.Encode()
	}

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
	if err != nil {
		return err
	}

	resp, err := d.client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	// special case to read OneDev server version, which is not valid JSON
	if presult, ok := result.(**version.Version); ok {
		bytes, err := io.ReadAll(io.LimitReader(resp.Body, maxOneDevVersionResponseSize+1))
		if err != nil {
			return err
		}
		if len(bytes) > maxOneDevVersionResponseSize {
			return fmt.Errorf("OneDev server version response exceeds %d bytes", maxOneDevVersionResponseSize)
		}
		vers, err := version.NewVersion(string(bytes))
		if err != nil {
			return err
		}
		*presult = vers
		return nil
	}

	decoder := json.NewDecoder(resp.Body)
	return decoder.Decode(&result)
}

// GetRepoInfo returns repository information
func (d *OneDevDownloader) GetRepoInfo(ctx context.Context) (*base.Repository, error) {
	// check OneDev server version
	var serverVersion *version.Version
	err := d.callAPI(
		ctx,
		"/~api/version/server",
		nil,
		&serverVersion,
	)
	if err != nil {
		return nil, fmt.Errorf("failed to get OneDev server version; OneDev %s or newer required", OneDevRequiredVersion)
	}
	requiredVersion, _ := version.NewVersion(OneDevRequiredVersion)
	if serverVersion.LessThan(requiredVersion) {
		return nil, fmt.Errorf("OneDev %s or newer required; currently running OneDev %s", OneDevRequiredVersion, serverVersion)
	}

	info := make([]struct {
		ID          int64  `json:"id"`
		Name        string `json:"name"`
		Path        string `json:"path"`
		Description string `json:"description"`
	}, 0, 1)

	err = d.callAPI(
		ctx,
		"/~api/projects",
		map[string]string{
			"query":  `"Path" is "` + d.repoPath + `"`,
			"offset": "0",
			"count":  "1",
		},
		&info,
	)
	if err != nil {
		return nil, err
	}
	if len(info) != 1 {
		return nil, fmt.Errorf("Project %s not found", d.repoPath)
	}

	d.repoID = info[0].ID

	cloneURL, err := d.baseURL.Parse(info[0].Path)
	if err != nil {
		return nil, err
	}

	return &base.Repository{
		Name:        info[0].Name,
		Description: info[0].Description,
		CloneURL:    cloneURL.String(),
		OriginalURL: cloneURL.String(),
	}, nil
}

// GetMilestones returns milestones
func (d *OneDevDownloader) GetMilestones(ctx context.Context) ([]*base.Milestone, error) {
	endpoint := fmt.Sprintf("/~api/projects/%d/iterations", d.repoID)

	milestones := make([]*base.Milestone, 0, 100)
	offset := 0
	for {
		rawMilestones := make([]struct {
			ID          int64  `json:"id"`
			Name        string `json:"name"`
			Description string `json:"description"`
			DueDay      int64  `json:"dueDay"`
			Closed      bool   `json:"closed"`
		}, 0, 100)

		err := d.callAPI(
			ctx,
			endpoint,
			map[string]string{
				"offset": strconv.Itoa(offset),
				"count":  "100",
			},
			&rawMilestones,
		)
		if err != nil {
			return nil, err
		}
		if len(rawMilestones) == 0 {
			break
		}
		offset += 100

		for _, milestone := range rawMilestones {
			d.milestoneMap[milestone.ID] = milestone.Name

			var dueDate *time.Time
			if milestone.DueDay != 0 {
				d := time.Unix(milestone.DueDay*24*60*60, 0)
				dueDate = &d
			}

			var closedDate *time.Time
			state := "open"
			if milestone.Closed {
				closedDate = dueDate
				state = "closed"
			}

			milestones = append(milestones, &base.Milestone{
				Title:       milestone.Name,
				Description: milestone.Description,
				Deadline:    dueDate,
				Closed:      closedDate,
				State:       state,
			})
		}
	}
	return milestones, nil
}

// GetLabels returns labels
func (d *OneDevDownloader) GetLabels(_ context.Context) ([]*base.Label, error) {
	return []*base.Label{
		{
			Name:  "Bug",
			Color: "f64e60",
		},
		{
			Name:  "Build Failure",
			Color: "f64e60",
		},
		{
			Name:  "Discussion",
			Color: "8950fc",
		},
		{
			Name:  "Improvement",
			Color: "1bc5bd",
		},
		{
			Name:  "New Feature",
			Color: "1bc5bd",
		},
		{
			Name:  "Support Request",
			Color: "8950fc",
		},
	}, nil
}

type onedevIssueContext struct {
	IsPullRequest bool
}

// GetIssues returns issues
func (d *OneDevDownloader) GetIssues(ctx context.Context, page, perPage int) ([]*base.Issue, bool, error) {
	type Field struct {
		Name  string `json:"name"`
		Value string `json:"value"`
	}
	rawIssues := make([]struct {
		ID          int64     `json:"id"`
		Number      int64     `json:"number"`
		State       string    `json:"state"`
		Title       string    `json:"title"`
		Description string    `json:"description"`
		SubmitterID int64     `json:"submitterId"`
		SubmitDate  time.Time `json:"submitDate"`
		Fields      []Field   `json:"fields"`
	}, 0, perPage)

	err := d.callAPI(
		ctx,
		"/~api/issues",
		map[string]string{
			"query":      `"Project" is "` + d.repoPath + `"`,
			"offset":     strconv.Itoa((page - 1) * perPage),
			"count":      strconv.Itoa(perPage),
			"withFields": "true",
		},
		&rawIssues,
	)
	if err != nil {
		return nil, false, err
	}

	issues := make([]*base.Issue, 0, len(rawIssues))
	for _, issue := range rawIssues {
		var label *base.Label
		for _, field := range issue.Fields {
			if field.Name == "Type" {
				label = &base.Label{Name: field.Value}
				break
			}
		}

		milestones := make([]struct {
			ID   int64  `json:"id"`
			Name string `json:"name"`
		}, 0, 10)
		err = d.callAPI(
			ctx,
			fmt.Sprintf("/~api/issues/%d/iterations", issue.ID),
			nil,
			&milestones,
		)
		if err != nil {
			return nil, false, err
		}
		milestoneID := int64(0)
		if len(milestones) > 0 {
			milestoneID = milestones[0].ID
		}

		state := strings.ToLower(issue.State)
		if state == "released" {
			state = "closed"
		}
		poster := d.tryGetUser(ctx, issue.SubmitterID)
		issues = append(issues, &base.Issue{
			Title:        issue.Title,
			Number:       issue.Number,
			PosterName:   poster.Name,
			PosterEmail:  poster.Email,
			Content:      issue.Description,
			Milestone:    d.milestoneMap[milestoneID],
			State:        state,
			Created:      issue.SubmitDate,
			Updated:      issue.SubmitDate,
			Labels:       []*base.Label{label},
			ForeignIndex: issue.ID,
			Context:      onedevIssueContext{IsPullRequest: false},
		})

		if d.maxIssueIndex < issue.Number {
			d.maxIssueIndex = issue.Number
		}
	}

	return issues, len(issues) == 0, nil
}

// GetComments returns comments
func (d *OneDevDownloader) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) {
	context, ok := commentable.GetContext().(onedevIssueContext)
	if !ok {
		return nil, false, fmt.Errorf("unexpected context: %+v", commentable.GetContext())
	}

	rawComments := make([]struct {
		ID      int64     `json:"id"`
		Date    time.Time `json:"date"`
		UserID  int64     `json:"userId"`
		Content string    `json:"content"`
	}, 0, 100)

	var endpoint string
	if context.IsPullRequest {
		endpoint = fmt.Sprintf("/~api/pulls/%d/comments", commentable.GetForeignIndex())
	} else {
		endpoint = fmt.Sprintf("/~api/issues/%d/comments", commentable.GetForeignIndex())
	}

	err := d.callAPI(
		ctx,
		endpoint,
		nil,
		&rawComments,
	)
	if err != nil {
		return nil, false, err
	}

	rawChanges := make([]struct {
		Date   time.Time      `json:"date"`
		UserID int64          `json:"userId"`
		Data   map[string]any `json:"data"`
	}, 0, 100)

	if context.IsPullRequest {
		endpoint = fmt.Sprintf("/~api/pulls/%d/changes", commentable.GetForeignIndex())
	} else {
		endpoint = fmt.Sprintf("/~api/issues/%d/changes", commentable.GetForeignIndex())
	}

	err = d.callAPI(
		ctx,
		endpoint,
		nil,
		&rawChanges,
	)
	if err != nil {
		return nil, false, err
	}

	comments := make([]*base.Comment, 0, len(rawComments)+len(rawChanges))
	for _, comment := range rawComments {
		if len(comment.Content) == 0 {
			continue
		}
		poster := d.tryGetUser(ctx, comment.UserID)
		comments = append(comments, &base.Comment{
			IssueIndex:  commentable.GetLocalIndex(),
			Index:       comment.ID,
			PosterID:    poster.ID,
			PosterName:  poster.Name,
			PosterEmail: poster.Email,
			Content:     comment.Content,
			Created:     comment.Date,
			Updated:     comment.Date,
		})
	}
	for _, change := range rawChanges {
		contentV, ok := change.Data["content"]
		if !ok {
			contentV, ok = change.Data["comment"]
			if !ok {
				continue
			}
		}
		content, ok := contentV.(string)
		if !ok || len(content) == 0 {
			continue
		}

		poster := d.tryGetUser(ctx, change.UserID)
		comments = append(comments, &base.Comment{
			IssueIndex:  commentable.GetLocalIndex(),
			PosterID:    poster.ID,
			PosterName:  poster.Name,
			PosterEmail: poster.Email,
			Content:     content,
			Created:     change.Date,
			Updated:     change.Date,
		})
	}

	return comments, true, nil
}

// GetPullRequests returns pull requests
func (d *OneDevDownloader) GetPullRequests(ctx context.Context, page, perPage int) ([]*base.PullRequest, bool, error) {
	rawPullRequests := make([]struct {
		ID             int64      `json:"id"`
		Number         int64      `json:"number"`
		Title          string     `json:"title"`
		SubmitterID    int64      `json:"submitterId"`
		SubmitDate     time.Time  `json:"submitDate"`
		Description    string     `json:"description"`
		TargetBranch   string     `json:"targetBranch"`
		SourceBranch   string     `json:"sourceBranch"`
		BaseCommitHash string     `json:"baseCommitHash"`
		CloseDate      *time.Time `json:"closeDate"`
		Status         string     `json:"status"` // Possible values: OPEN, MERGED, DISCARDED
	}, 0, perPage)

	err := d.callAPI(
		ctx,
		"/~api/pulls",
		map[string]string{
			"query":  `"Target Project" is "` + d.repoPath + `"`,
			"offset": strconv.Itoa((page - 1) * perPage),
			"count":  strconv.Itoa(perPage),
		},
		&rawPullRequests,
	)
	if err != nil {
		return nil, false, err
	}

	pullRequests := make([]*base.PullRequest, 0, len(rawPullRequests))
	for _, pr := range rawPullRequests {
		var mergePreview struct {
			TargetHeadCommitHash string `json:"targetHeadCommitHash"`
			HeadCommitHash       string `json:"headCommitHash"`
			MergeStrategy        string `json:"mergeStrategy"`
			MergeCommitHash      string `json:"mergeCommitHash"`
		}
		err := d.callAPI(
			ctx,
			fmt.Sprintf("/~api/pulls/%d/merge-preview", pr.ID),
			nil,
			&mergePreview,
		)
		if err != nil {
			return nil, false, err
		}

		state := "open"
		merged := false
		var closeTime *time.Time
		var mergedTime *time.Time
		if pr.Status != "OPEN" {
			state = "closed"
			closeTime = pr.CloseDate
			if pr.Status == "MERGED" { // "DISCARDED"
				merged = true
				mergedTime = pr.CloseDate
			}
		}
		poster := d.tryGetUser(ctx, pr.SubmitterID)

		number := pr.Number + d.maxIssueIndex
		pullRequests = append(pullRequests, &base.PullRequest{
			Title:      pr.Title,
			Number:     number,
			PosterName: poster.Name,
			PosterID:   poster.ID,
			Content:    pr.Description,
			State:      state,
			Created:    pr.SubmitDate,
			Updated:    pr.SubmitDate,
			Closed:     closeTime,
			Merged:     merged,
			MergedTime: mergedTime,
			Head: base.PullRequestBranch{
				Ref:      pr.SourceBranch,
				SHA:      mergePreview.HeadCommitHash,
				RepoName: d.repoPath,
			},
			Base: base.PullRequestBranch{
				Ref:      pr.TargetBranch,
				SHA:      mergePreview.TargetHeadCommitHash,
				RepoName: d.repoPath,
			},
			ForeignIndex: pr.ID,
			Context:      onedevIssueContext{IsPullRequest: true},
		})

		// SECURITY: Ensure that the PR is safe
		_ = CheckAndEnsureSafePR(pullRequests[len(pullRequests)-1], d.baseURL.String(), d)
	}

	return pullRequests, len(pullRequests) == 0, nil
}

// GetReviews returns pull requests reviews
func (d *OneDevDownloader) GetReviews(ctx context.Context, reviewable base.Reviewable) ([]*base.Review, error) {
	rawReviews := make([]struct {
		ID     int64  `json:"id"`
		UserID int64  `json:"userId"`
		Status string `json:"status"` // Possible values: PENDING, APPROVED, REQUESTED_FOR_CHANGES, EXCLUDED
	}, 0, 100)

	err := d.callAPI(
		ctx,
		fmt.Sprintf("/~api/pulls/%d/reviews", reviewable.GetForeignIndex()),
		nil,
		&rawReviews,
	)
	if err != nil {
		return nil, err
	}

	reviews := make([]*base.Review, 0, len(rawReviews))
	for _, review := range rawReviews {
		state := base.ReviewStatePending
		content := ""
		switch review.Status {
		case "APPROVED":
			state = base.ReviewStateApproved
		case "REQUESTED_FOR_CHANGES":
			state = base.ReviewStateChangesRequested
		}

		poster := d.tryGetUser(ctx, review.UserID)
		reviews = append(reviews, &base.Review{
			IssueIndex:   reviewable.GetLocalIndex(),
			ReviewerID:   poster.ID,
			ReviewerName: poster.Name,
			Content:      content,
			State:        state,
		})
	}

	return reviews, nil
}

// GetTopics return repository topics
func (d *OneDevDownloader) GetTopics(_ context.Context) ([]string, error) {
	return []string{}, nil
}

func (d *OneDevDownloader) tryGetUser(ctx context.Context, userID int64) *onedevUser {
	user, ok := d.userMap[userID]
	if !ok {
		// get user name
		type RawUser struct {
			Name string `json:"name"`
		}
		var rawUser RawUser
		err := d.callAPI(
			ctx,
			fmt.Sprintf("/~api/users/%d", userID),
			nil,
			&rawUser,
		)
		var userName string
		if err == nil {
			userName = rawUser.Name
		} else {
			userName = fmt.Sprintf("User %d", userID)
		}

		// get (primary) user Email address
		rawEmailAddresses := make([]struct {
			Value   string `json:"value"`
			Primary bool   `json:"primary"`
		}, 0, 10)
		err = d.callAPI(
			ctx,
			fmt.Sprintf("/~api/users/%d/email-addresses", userID),
			nil,
			&rawEmailAddresses,
		)
		var userEmail string
		if err == nil {
			for _, email := range rawEmailAddresses {
				if userEmail == "" || email.Primary {
					userEmail = email.Value
				}
				if email.Primary {
					break
				}
			}
		}

		user = &onedevUser{
			ID:    userID,
			Name:  userName,
			Email: userEmail,
		}
		d.userMap[userID] = user
	}

	return user
}