Merge branch 'main' into fix-16147

Author: lafriks

docs/content/doc/advanced/repo-mirror.en-us.md

@@ -0,0 +1,88 @@
+---
+date: "2021-05-13T00:00:00-00:00"
+title: "Repository Mirror"
+slug: "repo-mirror"
+weight: 45
+toc: false
+draft: false
+menu:
+  sidebar:
+    parent: "advanced"
+    name: "Repository Mirror"
+    weight: 45
+    identifier: "repo-mirror"
+---
+
+# Repository Mirror
+
+Repository mirroring allows for the mirroring of repositories to and from external sources. You can use it to mirror branches, tags, and commits between repositories.
+
+**Table of Contents**
+
+{{< toc >}}
+
+## Use cases
+
+The following are some possible use cases for repository mirroring:
+
+- You migrated to Gitea but still need to keep your project in another source. In that case, you can simply set it up to mirror to Gitea (pull) and all the essential history of commits, tags, and branches are available in your Gitea instance.
+- You have old projects in another source that you don’t use actively anymore, but don’t want to remove for archiving purposes. In that case, you can create a push mirror so that your active Gitea repository can push its changes to the old location.
+
+## Pulling from a remote repository
+
+For an existing remote repository, you can set up pull mirroring as follows:
+
+1. Select **New Migration** in the **Create...** menu on the top right.
+2. Select the remote repository service.
+3. Enter a repository URL.
+4. If the repository needs authentication fill in your authentication information.
+5. Check the box **This repository will be a mirror**.
+5. Select **Migrate repository** to save the configuration.
+
+The repository now gets mirrored periodically from the remote repository. You can force a sync by selecting **Synchronize Now** in the repository settings.
+
+## Pushing to a remote repository
+
+For an existing repository, you can set up push mirroring as follows:
+
+1. In your repository, go to **Settings** > **Repository**, and then the **Mirror Settings** section.
+2. Enter a repository URL.
+3. If the repository needs authentication expand the **Authorization** section and fill in your authentication information.
+4. Select **Add Push Mirror** to save the configuration.
+
+The repository now gets mirrored periodically to the remote repository. You can force a sync by selecting **Synchronize Now**. In case of an error a message displayed to help you resolve it.
+
+:exclamation::exclamation: **NOTE:** This will force push to the remote repository. This will overwrite any changes in the remote repository! :exclamation::exclamation:
+
+### Setting up a push mirror from Gitea to GitHub
+
+To set up a mirror from Gitea to GitHub, you need to follow these steps:
+
+1. Create a [GitHub personal access token](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token) with the *public_repo* box checked.
+2. Fill in the **Git Remote Repository URL**: `https://github.com/<your_github_group>/<your_github_project>.git`.
+3. Fill in the **Authorization** fields with your GitHub username and the personal access token.
+4. Select **Add Push Mirror** to save the configuration.
+
+The repository pushes shortly thereafter. To force a push, select the **Synchronize Now** button.
+
+### Setting up a push mirror from Gitea to GitLab
+
+To set up a mirror from Gitea to GitLab, you need to follow these steps:
+
+1. Create a [GitLab personal access token](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html) with *write_repository* scope.
+2. Fill in the **Git Remote Repository URL**: `https://<destination host>/<your_gitlab_group_or_name>/<your_gitlab_project>.git`.
+3. Fill in the **Authorization** fields with `oauth2` as **Username** and your GitLab personal access token as **Password**.
+4. Select **Add Push Mirror** to save the configuration.
+
+The repository pushes shortly thereafter. To force a push, select the **Synchronize Now** button.
+
+### Setting up a push mirror from Gitea to Bitbucket
+
+To set up a mirror from Gitea to Bitbucket, you need to follow these steps:
+
+1. Create a [Bitbucket app password](https://support.atlassian.com/bitbucket-cloud/docs/app-passwords/) with the *Repository Write* box checked.
+2. Fill in the **Git Remote Repository URL**: `https://bitbucket.org/<your_bitbucket_group_or_name>/<your_bitbucket_project>.git`.
+3. Fill in the **Authorization** fields with your Bitbucket username and the app password as **Password**.
+4. Select **Add Push Mirror** to save the configuration.
+
+The repository pushes shortly thereafter. To force a push, select the **Synchronize Now** button.

services/mirror/mirror_test.go renamed to integrations/mirror_pull_test.go

@@ -2,28 +2,24 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
-package mirror
+package integrations
 
 import (
 	"context"
-	"path/filepath"
 	"testing"
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/git"
 	migration "code.gitea.io/gitea/modules/migrations/base"
 	"code.gitea.io/gitea/modules/repository"
+	mirror_service "code.gitea.io/gitea/services/mirror"
 	release_service "code.gitea.io/gitea/services/release"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func TestMain(m *testing.M) {
-	models.MainTest(m, filepath.Join("..", ".."))
-}
-
-func TestRelease_MirrorDelete(t *testing.T) {
-	assert.NoError(t, models.PrepareTestDatabase())
+func TestMirrorPull(t *testing.T) {
+	defer prepareTestEnv(t)()
 
 	user := models.AssertExistsAndLoadBean(t, &models.User{ID: 2}).(*models.User)
 	repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository)
@@ -76,7 +72,7 @@ func TestRelease_MirrorDelete(t *testing.T) {
 	err = mirror.GetMirror()
 	assert.NoError(t, err)
 
-	_, ok := runSync(ctx, mirror.Mirror)
+	ok := mirror_service.SyncPullMirror(ctx, mirror.ID)
 	assert.True(t, ok)
 
 	count, err := models.GetReleaseCountByRepoID(mirror.ID, findOptions)
@@ -87,7 +83,7 @@ func TestRelease_MirrorDelete(t *testing.T) {
 	assert.NoError(t, err)
 	assert.NoError(t, release_service.DeleteReleaseByID(release.ID, user, true))
 
-	_, ok = runSync(ctx, mirror.Mirror)
+	ok = mirror_service.SyncPullMirror(ctx, mirror.ID)
 	assert.True(t, ok)
 
 	count, err = models.GetReleaseCountByRepoID(mirror.ID, findOptions)

integrations/mirror_push_test.go

@@ -0,0 +1,86 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package integrations
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"testing"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/repository"
+	"code.gitea.io/gitea/modules/setting"
+	mirror_service "code.gitea.io/gitea/services/mirror"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMirrorPush(t *testing.T) {
+	onGiteaRun(t, testMirrorPush)
+}
+
+func testMirrorPush(t *testing.T, u *url.URL) {
+	defer prepareTestEnv(t)()
+
+	setting.Migrations.AllowLocalNetworks = true
+
+	user := models.AssertExistsAndLoadBean(t, &models.User{ID: 2}).(*models.User)
+	srcRepo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository)
+
+	mirrorRepo, err := repository.CreateRepository(user, user, models.CreateRepoOptions{
+		Name: "test-push-mirror",
+	})
+	assert.NoError(t, err)
+
+	ctx := NewAPITestContext(t, user.LowerName, srcRepo.Name)
+
+	doCreatePushMirror(ctx, fmt.Sprintf("%s%s/%s", u.String(), url.PathEscape(ctx.Username), url.PathEscape(mirrorRepo.Name)), user.LowerName, userPassword)(t)
+
+	mirrors, err := models.GetPushMirrorsByRepoID(srcRepo.ID)
+	assert.NoError(t, err)
+	assert.Len(t, mirrors, 1)
+
+	ok := mirror_service.SyncPushMirror(context.Background(), mirrors[0].ID)
+	assert.True(t, ok)
+
+	srcGitRepo, err := git.OpenRepository(srcRepo.RepoPath())
+	assert.NoError(t, err)
+	defer srcGitRepo.Close()
+
+	srcCommit, err := srcGitRepo.GetBranchCommit("master")
+	assert.NoError(t, err)
+
+	mirrorGitRepo, err := git.OpenRepository(mirrorRepo.RepoPath())
+	assert.NoError(t, err)
+	defer mirrorGitRepo.Close()
+
+	mirrorCommit, err := mirrorGitRepo.GetBranchCommit("master")
+	assert.NoError(t, err)
+
+	assert.Equal(t, srcCommit.ID, mirrorCommit.ID)
+}
+
+func doCreatePushMirror(ctx APITestContext, address, username, password string) func(t *testing.T) {
+	return func(t *testing.T) {
+		csrf := GetCSRF(t, ctx.Session, fmt.Sprintf("/%s/%s/settings", url.PathEscape(ctx.Username), url.PathEscape(ctx.Reponame)))
+
+		req := NewRequestWithValues(t, "POST", fmt.Sprintf("/%s/%s/settings", url.PathEscape(ctx.Username), url.PathEscape(ctx.Reponame)), map[string]string{
+			"_csrf":                csrf,
+			"action":               "push-mirror-add",
+			"push_mirror_address":  address,
+			"push_mirror_username": username,
+			"push_mirror_password": password,
+			"push_mirror_interval": "0",
+		})
+		ctx.Session.MakeRequest(t, req, http.StatusFound)
+
+		flashCookie := ctx.Session.GetCookie("macaron_flash")
+		assert.NotNil(t, flashCookie)
+		assert.Contains(t, flashCookie.Value, "success")
+	}
+}

models/migrations/migrations.go

@@ -315,6 +315,8 @@ var migrations = []Migration{
 	NewMigration("Always save primary email on email address table", addPrimaryEmail2EmailAddress),
 	// v182 -> v183
 	NewMigration("Add issue resource index table", addIssueResourceIndexTable),
+	// v183 -> v184
+	NewMigration("Create PushMirror table", createPushMirrorTable),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/v180.go

@@ -5,9 +5,6 @@
 package migrations
 
 import (
-	"code.gitea.io/gitea/models"
-	"code.gitea.io/gitea/modules/migrations/base"
-	"code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/util"
 
 	jsoniter "github.com/json-iterator/go"
@@ -16,20 +13,38 @@ import (
 )
 
 func deleteMigrationCredentials(x *xorm.Engine) (err error) {
+	// Task represents a task
+	type Task struct {
+		ID             int64
+		DoerID         int64 `xorm:"index"` // operator
+		OwnerID        int64 `xorm:"index"` // repo owner id, when creating, the repoID maybe zero
+		RepoID         int64 `xorm:"index"`
+		Type           int
+		Status         int `xorm:"index"`
+		StartTime      int64
+		EndTime        int64
+		PayloadContent string `xorm:"TEXT"`
+		Errors         string `xorm:"TEXT"` // if task failed, saved the error reason
+		Created        int64  `xorm:"created"`
+	}
+
+	const TaskTypeMigrateRepo = 0
+	const TaskStatusStopped = 2
+
 	const batchSize = 100
 
 	// only match migration tasks, that are not pending or running
 	cond := builder.Eq{
-		"type": structs.TaskTypeMigrateRepo,
+		"type": TaskTypeMigrateRepo,
 	}.And(builder.Gte{
-		"status": structs.TaskStatusStopped,
+		"status": TaskStatusStopped,
 	})
 
 	sess := x.NewSession()
 	defer sess.Close()
 
 	for start := 0; ; start += batchSize {
-		tasks := make([]*models.Task, 0, batchSize)
+		tasks := make([]*Task, 0, batchSize)
 		if err = sess.Limit(batchSize, start).Where(cond, 0).Find(&tasks); err != nil {
 			return
 		}
@@ -55,7 +70,41 @@ func deleteMigrationCredentials(x *xorm.Engine) (err error) {
 }
 
 func removeCredentials(payload string) (string, error) {
-	var opts base.MigrateOptions
+	// MigrateOptions defines the way a repository gets migrated
+	// this is for internal usage by migrations module and func who interact with it
+	type MigrateOptions struct {
+		// required: true
+		CloneAddr             string `json:"clone_addr" binding:"Required"`
+		CloneAddrEncrypted    string `json:"clone_addr_encrypted,omitempty"`
+		AuthUsername          string `json:"auth_username"`
+		AuthPassword          string `json:"-"`
+		AuthPasswordEncrypted string `json:"auth_password_encrypted,omitempty"`
+		AuthToken             string `json:"-"`
+		AuthTokenEncrypted    string `json:"auth_token_encrypted,omitempty"`
+		// required: true
+		UID int `json:"uid" binding:"Required"`
+		// required: true
+		RepoName        string `json:"repo_name" binding:"Required"`
+		Mirror          bool   `json:"mirror"`
+		LFS             bool   `json:"lfs"`
+		LFSEndpoint     string `json:"lfs_endpoint"`
+		Private         bool   `json:"private"`
+		Description     string `json:"description"`
+		OriginalURL     string
+		GitServiceType  int
+		Wiki            bool
+		Issues          bool
+		Milestones      bool
+		Labels          bool
+		Releases        bool
+		Comments        bool
+		PullRequests    bool
+		ReleaseAssets   bool
+		MigrateToRepoID int64
+		MirrorInterval  string `json:"mirror_interval"`
+	}
+
+	var opts MigrateOptions
 	json := jsoniter.ConfigCompatibleWithStandardLibrary
 	err := json.Unmarshal([]byte(payload), &opts)
 	if err != nil {
@@ -64,7 +113,7 @@ func removeCredentials(payload string) (string, error) {
 
 	opts.AuthPassword = ""
 	opts.AuthToken = ""
-	opts.CloneAddr = util.SanitizeURLCredentials(opts.CloneAddr, true)
+	opts.CloneAddr = util.NewStringURLSanitizer(opts.CloneAddr, true).Replace(opts.CloneAddr)
 
 	confBytes, err := json.Marshal(opts)
 	if err != nil {

models/migrations/v183.go

@@ -0,0 +1,39 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"fmt"
+	"time"
+
+	"code.gitea.io/gitea/modules/timeutil"
+
+	"xorm.io/xorm"
+)
+
+func createPushMirrorTable(x *xorm.Engine) error {
+	type PushMirror struct {
+		ID         int64 `xorm:"pk autoincr"`
+		RepoID     int64 `xorm:"INDEX"`
+		RemoteName string
+
+		Interval       time.Duration
+		CreatedUnix    timeutil.TimeStamp `xorm:"created"`
+		LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"`
+		LastError      string             `xorm:"text"`
+	}
+
+	sess := x.NewSession()
+	defer sess.Close()
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+
+	if err := sess.Sync2(new(PushMirror)); err != nil {
+		return fmt.Errorf("Sync2: %v", err)
+	}
+
+	return sess.Commit()
+}

models/models.go

@@ -135,6 +135,7 @@ func init() {
 		new(Session),
 		new(RepoTransfer),
 		new(IssueIndex),
+		new(PushMirror),
 	)
 
 	gonicNames := []string{"SSL", "UID"}