migrations: add test for importing pull requests in gitea uploader (#18752)

* logs: add the buffer logger to inspect logs during testing

Signed-off-by: Loïc Dachary <[email protected]>

* migrations: add test for importing pull requests in gitea uploader

Signed-off-by: Loïc Dachary <[email protected]>

* for each git.OpenRepositoryCtx, call Close

* Content is expected to return the content of the log

* test for errors before defer

Co-authored-by: Loïc Dachary <[email protected]>
Co-authored-by: zeripath <[email protected]>
Co-authored-by: Lunny Xiao <[email protected]>

Author: 4 people

integrations/testlogger.go

@@ -181,6 +181,11 @@ func (log *TestLogger) Init(config string) error {
 	return nil
 }
 
+// Content returns the content accumulated in the content provider
+func (log *TestLogger) Content() (string, error) {
+	return "", fmt.Errorf("not supported")
+}
+
 // Flush when log should be flushed
 func (log *TestLogger) Flush() {
 }

models/migrations/testlogger_test.go

@@ -166,6 +166,11 @@ func (log *TestLogger) Init(config string) error {
 	return nil
 }
 
+// Content returns the content accumulated in the content provider
+func (log *TestLogger) Content() (string, error) {
+	return "", fmt.Errorf("not supported")
+}
+
 // Flush when log should be flushed
 func (log *TestLogger) Flush() {
 }

modules/log/buffer.go

@@ -0,0 +1,72 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package log
+
+import (
+	"bytes"
+	"sync"
+)
+
+type bufferWriteCloser struct {
+	mu     sync.Mutex
+	buffer bytes.Buffer
+}
+
+func (b *bufferWriteCloser) Write(p []byte) (int, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.buffer.Write(p)
+}
+
+func (b *bufferWriteCloser) Close() error {
+	return nil
+}
+
+func (b *bufferWriteCloser) String() string {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.buffer.String()
+}
+
+// BufferLogger implements LoggerProvider and writes messages in a buffer.
+type BufferLogger struct {
+	WriterLogger
+}
+
+// NewBufferLogger create BufferLogger returning as LoggerProvider.
+func NewBufferLogger() LoggerProvider {
+	log := &BufferLogger{}
+	log.NewWriterLogger(&bufferWriteCloser{})
+	return log
+}
+
+// Init inits connection writer
+func (log *BufferLogger) Init(string) error {
+	log.NewWriterLogger(log.out)
+	return nil
+}
+
+// Content returns the content accumulated in the content provider
+func (log *BufferLogger) Content() (string, error) {
+	return log.out.(*bufferWriteCloser).String(), nil
+}
+
+// Flush when log should be flushed
+func (log *BufferLogger) Flush() {
+}
+
+// ReleaseReopen does nothing
+func (log *BufferLogger) ReleaseReopen() error {
+	return nil
+}
+
+// GetName returns the default name for this implementation
+func (log *BufferLogger) GetName() string {
+	return "buffer"
+}
+
+func init() {
+	Register("buffer", NewBufferLogger)
+}

modules/log/buffer_test.go

@@ -0,0 +1,64 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package log
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBufferLogger(t *testing.T) {
+	logger := NewBufferLogger()
+	bufferLogger := logger.(*BufferLogger)
+	assert.NotNil(t, bufferLogger)
+
+	err := logger.Init("")
+	assert.NoError(t, err)
+
+	location, _ := time.LoadLocation("EST")
+	date := time.Date(2019, time.January, 13, 22, 3, 30, 15, location)
+
+	msg := "TEST MSG"
+	event := Event{
+		level:    INFO,
+		msg:      msg,
+		caller:   "CALLER",
+		filename: "FULL/FILENAME",
+		line:     1,
+		time:     date,
+	}
+	logger.LogEvent(&event)
+	content, err := bufferLogger.Content()
+	assert.NoError(t, err)
+	assert.Contains(t, content, msg)
+	logger.Close()
+}
+
+func TestBufferLoggerContent(t *testing.T) {
+	level := INFO
+	logger := NewLogger(0, "console", "console", fmt.Sprintf(`{"level":"%s"}`, level.String()))
+
+	logger.SetLogger("buffer", "buffer", "{}")
+	defer logger.DelLogger("buffer")
+
+	msg := "A UNIQUE MESSAGE"
+	Error(msg)
+
+	found := false
+	for i := 0; i < 30000; i++ {
+		content, err := logger.GetLoggerProviderContent("buffer")
+		assert.NoError(t, err)
+		if strings.Contains(content, msg) {
+			found = true
+			break
+		}
+		time.Sleep(1 * time.Millisecond)
+	}
+	assert.True(t, found)
+}

modules/log/conn.go

@@ -119,6 +119,11 @@ func (log *ConnLogger) Init(jsonconfig string) error {
 	return nil
 }
 
+// Content returns the content accumulated in the content provider
+func (log *ConnLogger) Content() (string, error) {
+	return "", fmt.Errorf("not supported")
+}
+
 // Flush does nothing for this implementation
 func (log *ConnLogger) Flush() {
 }

modules/log/console.go

@@ -66,6 +66,11 @@ func (log *ConsoleLogger) Init(config string) error {
 	return nil
 }
 
+// Content returns the content accumulated in the content provider
+func (log *ConsoleLogger) Content() (string, error) {
+	return "", fmt.Errorf("not supported")
+}
+
 // Flush when log should be flushed
 func (log *ConsoleLogger) Flush() {
 }

modules/log/event.go

@@ -216,6 +216,12 @@ func (m *MultiChannelledLog) GetEventLogger(name string) EventLogger {
 	return m.loggers[name]
 }
 
+// GetEventProvider returns a sub logger provider content from this MultiChannelledLog
+func (m *MultiChannelledLog) GetLoggerProviderContent(name string) (string, error) {
+	channelledLogger := m.GetEventLogger(name).(*ChannelledLog)
+	return channelledLogger.loggerProvider.Content()
+}
+
 // GetEventLoggerNames returns a list of names
 func (m *MultiChannelledLog) GetEventLoggerNames() []string {
 	m.rwmutex.RLock()

modules/log/file.go

@@ -243,6 +243,15 @@ func (log *FileLogger) deleteOldLog() {
 	})
 }
 
+// Content returns the content accumulated in the content provider
+func (log *FileLogger) Content() (string, error) {
+	b, err := os.ReadFile(log.Filename)
+	if err != nil {
+		return "", err
+	}
+	return string(b), nil
+}
+
 // Flush flush file logger.
 // there are no buffering messages in file logger in memory.
 // flush file means sync file from disk.

modules/log/provider.go

@@ -7,6 +7,7 @@ package log
 // LoggerProvider represents behaviors of a logger provider.
 type LoggerProvider interface {
 	Init(config string) error
+	Content() (string, error)
 	EventLogger
 }
 

modules/log/smtp.go

@@ -95,6 +95,11 @@ func (log *SMTPLogger) sendMail(p []byte) (int, error) {
 	)
 }
 
+// Content returns the content accumulated in the content provider
+func (log *SMTPLogger) Content() (string, error) {
+	return "", fmt.Errorf("not supported")
+}
+
 // Flush when log should be flushed
 func (log *SMTPLogger) Flush() {
 }

services/migrations/gitea_uploader.go

@@ -484,19 +484,9 @@ func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error
 	return nil
 }
 
-func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullRequest, error) {
-	var labels []*models.Label
-	for _, label := range pr.Labels {
-		lb, ok := g.labels[label.Name]
-		if ok {
-			labels = append(labels, lb)
-		}
-	}
-
-	milestoneID := g.milestones[pr.Milestone]
-
+func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head string, err error) {
 	// download patch file
-	err := func() error {
+	err = func() error {
 		if pr.PatchURL == "" {
 			return nil
 		}
@@ -519,25 +509,25 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
 		return err
 	}()
 	if err != nil {
-		return nil, err
+		return "", err
 	}
 
 	// set head information
 	pullHead := filepath.Join(g.repo.RepoPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number))
 	if err := os.MkdirAll(pullHead, os.ModePerm); err != nil {
-		return nil, err
+		return "", err
 	}
 	p, err := os.Create(filepath.Join(pullHead, "head"))
 	if err != nil {
-		return nil, err
+		return "", err
 	}
 	_, err = p.WriteString(pr.Head.SHA)
 	p.Close()
 	if err != nil {
-		return nil, err
+		return "", err
 	}
 
-	head := "unknown repository"
+	head = "unknown repository"
 	if pr.IsForkPullRequest() && pr.State != "closed" {
 		if pr.Head.OwnerName != "" {
 			remote := pr.Head.OwnerName
@@ -560,16 +550,16 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
 				} else {
 					headBranch := filepath.Join(g.repo.RepoPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref)
 					if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil {
-						return nil, err
+						return "", err
 					}
 					b, err := os.Create(headBranch)
 					if err != nil {
-						return nil, err
+						return "", err
 					}
 					_, err = b.WriteString(pr.Head.SHA)
 					b.Close()
 					if err != nil {
-						return nil, err
+						return "", err
 					}
 					head = pr.Head.OwnerName + "/" + pr.Head.Ref
 				}
@@ -595,6 +585,25 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
 		}
 	}
 
+	return head, nil
+}
+
+func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullRequest, error) {
+	var labels []*models.Label
+	for _, label := range pr.Labels {
+		lb, ok := g.labels[label.Name]
+		if ok {
+			labels = append(labels, lb)
+		}
+	}
+
+	milestoneID := g.milestones[pr.Milestone]
+
+	head, err := g.updateGitForPullRequest(pr)
+	if err != nil {
+		return nil, fmt.Errorf("updateGitForPullRequest: %w", err)
+	}
+
 	if pr.Created.IsZero() {
 		if pr.Closed != nil {
 			pr.Created = *pr.Closed