Skip to content

Commit cee5f7c

Browse files
KN4CK3RKN4CK3R
authored
Add migrate from OneDev (#16356)
* Use context to simplify logic. * Added migration from OneDev. This PR adds [OneDev](https://code.onedev.io/) as migration source. Supported: - [x] Milestones - [x] Issues - [x] Pull Requests - [x] Comments - [x] Reviews - [x] Labels
1 parent 2d1935a commit cee5f7c

24 files changed

+1093
-92
lines changed

‎modules/convert/utils.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ func ToGitServiceType(value string) structs.GitServiceType {
3333
return structs.GitlabService
3434
case "gogs":
3535
return structs.GogsService
36+
case "onedev":
37+
return structs.OneDevService
3638
default:
3739
return structs.PlainGitService
3840
}

‎modules/migrations/base/downloader.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,9 @@ import (
1313

1414
// GetCommentOptions represents an options for get comment
1515
type GetCommentOptions struct {
16-
IssueNumber int64
17-
Page int
18-
PageSize int
16+
Context IssueContext
17+
Page int
18+
PageSize int
1919
}
2020

2121
// Downloader downloads the site repo information
@@ -30,7 +30,7 @@ type Downloader interface {
3030
GetComments(opts GetCommentOptions) ([]*Comment, bool, error)
3131
SupportGetRepoComments() bool
3232
GetPullRequests(page, perPage int) ([]*PullRequest, bool, error)
33-
GetReviews(pullRequestNumber int64) ([]*Review, error)
33+
GetReviews(pullRequestContext IssueContext) ([]*Review, error)
3434
FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error)
3535
}
3636

‎modules/migrations/base/issue.go

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,25 @@ package base
77

88
import "time"
99

10+
// IssueContext is used to map between local and foreign issue/PR ids.
11+
type IssueContext interface {
12+
LocalID() int64
13+
ForeignID() int64
14+
}
15+
16+
// BasicIssueContext is a 1:1 mapping between local and foreign ids.
17+
type BasicIssueContext int64
18+
19+
// LocalID gets the local id.
20+
func (c BasicIssueContext) LocalID() int64 {
21+
return int64(c)
22+
}
23+
24+
// ForeignID gets the foreign id.
25+
func (c BasicIssueContext) ForeignID() int64 {
26+
return int64(c)
27+
}
28+
1029
// Issue is a standard issue information
1130
type Issue struct {
1231
Number int64
@@ -25,4 +44,5 @@ type Issue struct {
2544
Labels []*Label
2645
Reactions []*Reaction
2746
Assignees []string
47+
Context IssueContext `yaml:"-"`
2848
}

‎modules/migrations/base/null_downloader.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ func (n NullDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) {
5050
return nil, false, &ErrNotSupported{Entity: "Issues"}
5151
}
5252

53-
// GetComments returns comments according issueNumber
53+
// GetComments returns comments according the options
5454
func (n NullDownloader) GetComments(GetCommentOptions) ([]*Comment, bool, error) {
5555
return nil, false, &ErrNotSupported{Entity: "Comments"}
5656
}
@@ -61,7 +61,7 @@ func (n NullDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool
6161
}
6262

6363
// GetReviews returns pull requests review
64-
func (n NullDownloader) GetReviews(pullRequestNumber int64) ([]*Review, error) {
64+
func (n NullDownloader) GetReviews(pullRequestContext IssueContext) ([]*Review, error) {
6565
return nil, &ErrNotSupported{Entity: "Reviews"}
6666
}
6767

‎modules/migrations/base/pullrequest.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ import (
1313
// PullRequest defines a standard pull request information
1414
type PullRequest struct {
1515
Number int64
16-
OriginalNumber int64 `yaml:"original_number"`
1716
Title string
1817
PosterName string `yaml:"poster_name"`
1918
PosterID int64 `yaml:"poster_id"`
@@ -34,6 +33,7 @@ type PullRequest struct {
3433
Assignees []string
3534
IsLocked bool `yaml:"is_locked"`
3635
Reactions []*Reaction
36+
Context IssueContext `yaml:"-"`
3737
}
3838

3939
// IsForkPullRequest returns true if the pull request from a forked repository but not the same repository

‎modules/migrations/base/retry_downloader.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -182,14 +182,14 @@ func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bo
182182
}
183183

184184
// GetReviews returns pull requests reviews
185-
func (d *RetryDownloader) GetReviews(pullRequestNumber int64) ([]*Review, error) {
185+
func (d *RetryDownloader) GetReviews(pullRequestContext IssueContext) ([]*Review, error) {
186186
var (
187187
reviews []*Review
188188
err error
189189
)
190190

191191
err = d.retry(func() error {
192-
reviews, err = d.Downloader.GetReviews(pullRequestNumber)
192+
reviews, err = d.Downloader.GetReviews(pullRequestContext)
193193
return err
194194
})
195195

‎modules/migrations/gitea_downloader.go

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -444,6 +444,7 @@ func (g *GiteaDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, err
444444
Labels: labels,
445445
Assignees: assignees,
446446
IsLocked: issue.IsLocked,
447+
Context: base.BasicIssueContext(issue.Index),
447448
})
448449
}
449450

@@ -466,26 +467,26 @@ func (g *GiteaDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comm
466467
default:
467468
}
468469

469-
comments, _, err := g.client.ListIssueComments(g.repoOwner, g.repoName, opts.IssueNumber, gitea_sdk.ListIssueCommentOptions{ListOptions: gitea_sdk.ListOptions{
470+
comments, _, err := g.client.ListIssueComments(g.repoOwner, g.repoName, opts.Context.ForeignID(), gitea_sdk.ListIssueCommentOptions{ListOptions: gitea_sdk.ListOptions{
470471
// PageSize: g.maxPerPage,
471472
// Page: i,
472473
}})
473474
if err != nil {
474-
return nil, false, fmt.Errorf("error while listing comments for issue #%d. Error: %v", opts.IssueNumber, err)
475+
return nil, false, fmt.Errorf("error while listing comments for issue #%d. Error: %v", opts.Context.ForeignID(), err)
475476
}
476477

477478
for _, comment := range comments {
478479
reactions, err := g.getCommentReactions(comment.ID)
479480
if err != nil {
480-
log.Warn("Unable to load comment reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.IssueNumber, comment.ID, g.repoOwner, g.repoName, err)
481+
log.Warn("Unable to load comment reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.Context.ForeignID(), comment.ID, g.repoOwner, g.repoName, err)
481482
if err2 := models.CreateRepositoryNotice(
482-
fmt.Sprintf("Unable to load reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.IssueNumber, comment.ID, g.repoOwner, g.repoName, err)); err2 != nil {
483+
fmt.Sprintf("Unable to load reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.Context.ForeignID(), comment.ID, g.repoOwner, g.repoName, err)); err2 != nil {
483484
log.Error("create repository notice failed: ", err2)
484485
}
485486
}
486487

487488
allComments = append(allComments, &base.Comment{
488-
IssueIndex: opts.IssueNumber,
489+
IssueIndex: opts.Context.LocalID(),
489490
PosterID: comment.Poster.ID,
490491
PosterName: comment.Poster.UserName,
491492
PosterEmail: comment.Poster.Email,
@@ -615,6 +616,7 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques
615616
RepoName: g.repoName,
616617
OwnerName: g.repoOwner,
617618
},
619+
Context: base.BasicIssueContext(pr.Index),
618620
})
619621
}
620622

@@ -626,7 +628,7 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques
626628
}
627629

628630
// GetReviews returns pull requests review
629-
func (g *GiteaDownloader) GetReviews(index int64) ([]*base.Review, error) {
631+
func (g *GiteaDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) {
630632
if err := g.client.CheckServerVersionConstraint(">=1.12"); err != nil {
631633
log.Info("GiteaDownloader: instance to old, skip GetReviews")
632634
return nil, nil
@@ -642,7 +644,7 @@ func (g *GiteaDownloader) GetReviews(index int64) ([]*base.Review, error) {
642644
default:
643645
}
644646

645-
prl, _, err := g.client.ListPullReviews(g.repoOwner, g.repoName, index, gitea_sdk.ListPullReviewsOptions{ListOptions: gitea_sdk.ListOptions{
647+
prl, _, err := g.client.ListPullReviews(g.repoOwner, g.repoName, context.ForeignID(), gitea_sdk.ListPullReviewsOptions{ListOptions: gitea_sdk.ListOptions{
646648
Page: i,
647649
PageSize: g.maxPerPage,
648650
}})
@@ -652,7 +654,7 @@ func (g *GiteaDownloader) GetReviews(index int64) ([]*base.Review, error) {
652654

653655
for _, pr := range prl {
654656

655-
rcl, _, err := g.client.ListPullReviewComments(g.repoOwner, g.repoName, index, pr.ID)
657+
rcl, _, err := g.client.ListPullReviewComments(g.repoOwner, g.repoName, context.ForeignID(), pr.ID)
656658
if err != nil {
657659
return nil, err
658660
}
@@ -678,7 +680,7 @@ func (g *GiteaDownloader) GetReviews(index int64) ([]*base.Review, error) {
678680

679681
allReviews = append(allReviews, &base.Review{
680682
ID: pr.ID,
681-
IssueIndex: index,
683+
IssueIndex: context.LocalID(),
682684
ReviewerID: pr.Reviewer.ID,
683685
ReviewerName: pr.Reviewer.UserName,
684686
Official: pr.Official,

‎modules/migrations/gitea_downloader_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -199,7 +199,7 @@ func TestGiteaDownloadRepo(t *testing.T) {
199199
}, issues)
200200

201201
comments, _, err := downloader.GetComments(base.GetCommentOptions{
202-
IssueNumber: 4,
202+
Context: base.BasicIssueContext(4),
203203
})
204204
assert.NoError(t, err)
205205
assertCommentsEqual(t, []*base.Comment{
@@ -265,7 +265,7 @@ func TestGiteaDownloadRepo(t *testing.T) {
265265
PatchURL: "https://gitea.com/gitea/test_repo/pulls/12.patch",
266266
}, prs[1])
267267

268-
reviews, err := downloader.GetReviews(7)
268+
reviews, err := downloader.GetReviews(base.BasicIssueContext(7))
269269
assert.NoError(t, err)
270270
assertReviewsEqual(t, []*base.Review{
271271
{

‎modules/migrations/gitea_uploader.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -609,6 +609,9 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
609609

610610
// download patch file
611611
err := func() error {
612+
if pr.PatchURL == "" {
613+
return nil
614+
}
612615
// pr.PatchURL maybe a local file
613616
ret, err := uri.Open(pr.PatchURL)
614617
if err != nil {

‎modules/migrations/github.go

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -428,6 +428,7 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool,
428428
Closed: issue.ClosedAt,
429429
IsLocked: issue.GetLocked(),
430430
Assignees: assignees,
431+
Context: base.BasicIssueContext(*issue.Number),
431432
})
432433
}
433434

@@ -441,15 +442,15 @@ func (g *GithubDownloaderV3) SupportGetRepoComments() bool {
441442

442443
// GetComments returns comments according issueNumber
443444
func (g *GithubDownloaderV3) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) {
444-
if opts.IssueNumber > 0 {
445-
comments, err := g.getComments(opts.IssueNumber)
445+
if opts.Context != nil {
446+
comments, err := g.getComments(opts.Context)
446447
return comments, false, err
447448
}
448449

449450
return g.GetAllComments(opts.Page, opts.PageSize)
450451
}
451452

452-
func (g *GithubDownloaderV3) getComments(issueNumber int64) ([]*base.Comment, error) {
453+
func (g *GithubDownloaderV3) getComments(issueContext base.IssueContext) ([]*base.Comment, error) {
453454
var (
454455
allComments = make([]*base.Comment, 0, g.maxPerPage)
455456
created = "created"
@@ -464,7 +465,7 @@ func (g *GithubDownloaderV3) getComments(issueNumber int64) ([]*base.Comment, er
464465
}
465466
for {
466467
g.sleep()
467-
comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueNumber), opt)
468+
comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueContext.ForeignID()), opt)
468469
if err != nil {
469470
return nil, fmt.Errorf("error while listing repos: %v", err)
470471
}
@@ -495,7 +496,7 @@ func (g *GithubDownloaderV3) getComments(issueNumber int64) ([]*base.Comment, er
495496
}
496497

497498
allComments = append(allComments, &base.Comment{
498-
IssueIndex: issueNumber,
499+
IssueIndex: issueContext.LocalID(),
499500
PosterID: comment.GetUser().GetID(),
500501
PosterName: comment.GetUser().GetLogin(),
501502
PosterEmail: comment.GetUser().GetEmail(),
@@ -661,6 +662,7 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
661662
},
662663
PatchURL: pr.GetPatchURL(),
663664
Reactions: reactions,
665+
Context: base.BasicIssueContext(*pr.Number),
664666
})
665667
}
666668

@@ -724,28 +726,28 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques
724726
}
725727

726728
// GetReviews returns pull requests review
727-
func (g *GithubDownloaderV3) GetReviews(pullRequestNumber int64) ([]*base.Review, error) {
729+
func (g *GithubDownloaderV3) GetReviews(context base.IssueContext) ([]*base.Review, error) {
728730
var allReviews = make([]*base.Review, 0, g.maxPerPage)
729731
opt := &github.ListOptions{
730732
PerPage: g.maxPerPage,
731733
}
732734
for {
733735
g.sleep()
734-
reviews, resp, err := g.client.PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(pullRequestNumber), opt)
736+
reviews, resp, err := g.client.PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), opt)
735737
if err != nil {
736738
return nil, fmt.Errorf("error while listing repos: %v", err)
737739
}
738740
g.rate = &resp.Rate
739741
for _, review := range reviews {
740742
r := convertGithubReview(review)
741-
r.IssueIndex = pullRequestNumber
743+
r.IssueIndex = context.LocalID()
742744
// retrieve all review comments
743745
opt2 := &github.ListOptions{
744746
PerPage: g.maxPerPage,
745747
}
746748
for {
747749
g.sleep()
748-
reviewComments, resp, err := g.client.PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(pullRequestNumber), review.GetID(), opt2)
750+
reviewComments, resp, err := g.client.PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), review.GetID(), opt2)
749751
if err != nil {
750752
return nil, fmt.Errorf("error while listing repos: %v", err)
751753
}

‎modules/migrations/github_test.go

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,7 @@ func TestGitHubDownloadRepo(t *testing.T) {
216216

217217
// downloader.GetComments()
218218
comments, _, err := downloader.GetComments(base.GetCommentOptions{
219-
IssueNumber: 2,
219+
Context: base.BasicIssueContext(2),
220220
})
221221
assert.NoError(t, err)
222222
assertCommentsEqual(t, []*base.Comment{
@@ -286,6 +286,7 @@ func TestGitHubDownloadRepo(t *testing.T) {
286286
Merged: true,
287287
MergedTime: timePtr(time.Date(2019, 11, 12, 21, 39, 27, 0, time.UTC)),
288288
MergeCommitSHA: "f32b0a9dfd09a60f616f29158f772cedd89942d2",
289+
Context: base.BasicIssueContext(3),
289290
},
290291
{
291292
Number: 4,
@@ -332,10 +333,11 @@ func TestGitHubDownloadRepo(t *testing.T) {
332333
Content: "+1",
333334
},
334335
},
336+
Context: base.BasicIssueContext(4),
335337
},
336338
}, prs)
337339

338-
reviews, err := downloader.GetReviews(3)
340+
reviews, err := downloader.GetReviews(base.BasicIssueContext(3))
339341
assert.NoError(t, err)
340342
assertReviewsEqual(t, []*base.Review{
341343
{
@@ -367,7 +369,7 @@ func TestGitHubDownloadRepo(t *testing.T) {
367369
},
368370
}, reviews)
369371

370-
reviews, err = downloader.GetReviews(4)
372+
reviews, err = downloader.GetReviews(base.BasicIssueContext(4))
371373
assert.NoError(t, err)
372374
assertReviewsEqual(t, []*base.Review{
373375
{

0 commit comments

Comments
 (0)