test: add db migration tests for postgres and sqlite

Author: ziggie1984

cmd_migrate_db_postgres_test.go

@@ -0,0 +1,144 @@
+//go:build kvdb_postgres
+
+package main
+
+import (
+	"context"
+	"crypto/rand"
+	"database/sql"
+	"encoding/hex"
+	"fmt"
+	"testing"
+
+	embeddedpostgres "github.com/fergusstrange/embedded-postgres"
+	"github.com/lightningnetwork/lnd/kvdb"
+	"github.com/lightningnetwork/lnd/kvdb/postgres"
+	"github.com/lightningnetwork/lnd/kvdb/sqlbase"
+	"github.com/lightningnetwork/lnd/lncfg"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	testMaxConnections = 100
+	testDsnTemplate    = "postgres://postgres:[email protected]:9877/%s?sslmode=disable"
+)
+
+// PostgresTestSetup holds the test configuration for Postgres.
+type PostgresTestSetup struct {
+	tempDir  string
+	dbName   string
+	postgres *embeddedpostgres.EmbeddedPostgres
+	stopFunc func() error
+}
+
+// setupEmbeddedPostgres initializes and starts the embedded postgres instance.
+func setupEmbeddedPostgres(t *testing.T) *PostgresTestSetup {
+	sqlbase.Init(testMaxConnections)
+
+	setup := &PostgresTestSetup{}
+
+	// Initialize embedded postgres
+	setup.postgres = embeddedpostgres.NewDatabase(
+		embeddedpostgres.DefaultConfig().
+			Port(9877).
+			StartParameters(map[string]string{
+				"max_connections": fmt.Sprintf("%d", testMaxConnections),
+			}),
+	)
+
+	// Start postgres
+	err := setup.postgres.Start()
+	require.NoError(t, err, "failed to start postgres")
+
+	setup.stopFunc = setup.postgres.Stop
+
+	return setup
+}
+
+// createTestDatabase creates a new random test database.
+func (p *PostgresTestSetup) createTestDatabase(t *testing.T) {
+	// Generate random database name
+	randBytes := make([]byte, 8)
+	_, err := rand.Read(randBytes)
+	require.NoError(t, err)
+	p.dbName = "test_" + hex.EncodeToString(randBytes)
+
+	// Create the database
+	dbConn, err := sql.Open("pgx", p.getDsn("postgres"))
+	require.NoError(t, err)
+	defer dbConn.Close()
+
+	_, err = dbConn.ExecContext(
+		context.Background(),
+		"CREATE DATABASE "+p.dbName,
+	)
+	require.NoError(t, err)
+}
+
+// setupTestDir creates and sets up the temporary test directory.
+func (p *PostgresTestSetup) setupTestDir(t *testing.T) {
+	p.tempDir = setupTestData(t)
+}
+
+// getDsn returns the DSN for the specified database.
+func (p *PostgresTestSetup) getDsn(dbName string) string {
+	return fmt.Sprintf(testDsnTemplate, dbName)
+}
+
+// cleanup performs necessary cleanup.
+func (p *PostgresTestSetup) cleanup() error {
+	if p.stopFunc != nil {
+		return p.stopFunc()
+	}
+	return nil
+}
+
+// getDBConfigs returns the source and destination DB configs.
+func (p *PostgresTestSetup) getDBConfigs() (*SourceDB, *DestDB) {
+	sourceDB := &SourceDB{
+		Backend: lncfg.BoltBackend,
+		Bolt: &Bolt{
+			DBTimeout: kvdb.DefaultDBTimeout,
+			DataDir:   p.tempDir,
+			TowerDir:  p.tempDir,
+		},
+	}
+
+	destDB := &DestDB{
+		Backend: lncfg.PostgresBackend,
+		Postgres: &postgres.Config{
+			Dsn: p.getDsn(p.dbName),
+		},
+	}
+
+	return sourceDB, destDB
+}
+
+// TestMigrateDBPostgres tests the migration of a database from Bolt to
+// Postgres.
+func TestMigrateDBPostgres(t *testing.T) {
+	t.Parallel()
+
+	// Setup postgres.
+	setup := setupEmbeddedPostgres(t)
+	defer func() {
+		require.NoError(t, setup.cleanup())
+	}()
+
+	// Setup test environment.
+	setup.setupTestDir(t)
+	setup.createTestDatabase(t)
+
+	sourceDB, destDB := setup.getDBConfigs()
+
+	// Create and run migration command.
+	cmd := &migrateDBCommand{
+		Source:    sourceDB,
+		Dest:      destDB,
+		Network:   "regtest",
+		ChunkSize: 1024,
+	}
+
+	err := cmd.Execute(nil)
+	require.NoError(t, err, "failed to execute migration")
+}

cmd_migrate_db_sqlite_test.go

@@ -0,0 +1,60 @@
+//go:build kvdb_sqlite
+
+package main
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/lightningnetwork/lnd/kvdb"
+	"github.com/lightningnetwork/lnd/kvdb/sqlite"
+	"github.com/lightningnetwork/lnd/lncfg"
+	"github.com/stretchr/testify/require"
+)
+
+// TestMigrateDBSqlite tests the migration of a database from Bolt to SQLite.
+func TestMigrateDBSqlite(t *testing.T) {
+	t.Parallel()
+
+	// Create temp dir for test databases.
+	tempDir := setupTestData(t)
+
+	fmt.Println("tempDir", tempDir)
+
+	// Copy entire test directory structure.
+	err := copyTestDataDir("testdata/data", tempDir)
+	require.NoError(t, err, "failed to copy test data")
+
+	// Set up source DB config (bolt).
+	sourceDB := &SourceDB{
+		Backend: lncfg.BoltBackend,
+		Bolt: &Bolt{
+			DBTimeout: kvdb.DefaultDBTimeout,
+			DataDir:   tempDir,
+			TowerDir:  tempDir,
+		},
+	}
+
+	// Set up destination DB config (sqlite).
+	destDB := &DestDB{
+		Backend: lncfg.SqliteBackend,
+		Sqlite: &Sqlite{
+			DataDir:  tempDir,
+			TowerDir: tempDir,
+			Config:   &sqlite.Config{},
+		},
+	}
+
+	// Create and run migration command.
+	cmd := &migrateDBCommand{
+		Source:  sourceDB,
+		Dest:    destDB,
+		Network: "regtest",
+		// Select a small chunk size to test the chunking.
+		ChunkSize: 1024,
+	}
+
+	err = cmd.Execute(nil)
+
+	require.NoError(t, err, "migration failed")
+}

cmd_migrate_db_test.go

@@ -0,0 +1,22 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// testDataPath is the source directory containing test data.
+const testDataPath = "testdata/data"
+
+// setupTestData creates a new temp directory and copies test data into it.
+// It returns the path to the new temp directory.
+func setupTestData(t *testing.T) string {
+	// Create unique temp dir for this test.
+	tempDir := t.TempDir()
+	err := copyTestDataDir(testDataPath, tempDir)
+
+	require.NoError(t, err, "failed to copy test data")
+
+	return tempDir
+}

migratekvdb/migration_test.go

@@ -105,7 +105,7 @@ func createTestDatabase(dbPath string) (kvdb.Backend, error) {
 	// Create test data structure.
 	err = db.Update(func(tx kvdb.RwTx) error {
 		fmt.Println("Creating test data structure...")
-		// Create root bucket "accounts"
+		// Create root bucket "accounts".
 		accounts, err := tx.CreateTopLevelBucket([]byte("accounts"))
 		if err != nil {
 			fmt.Print("bucket creation failed.")

testdata/data/chain/bitcoin/regtest/macaroons.db

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+)
+
+// copyTestDataDir copies the entire test directory structure so that we do not
+// alter any original files.
+func copyTestDataDir(src, dst string) error {
+	return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		relPath, err := filepath.Rel(src, path)
+		if err != nil {
+			return err
+		}
+
+		dstPath := filepath.Join(dst, relPath)
+
+		if info.IsDir() {
+			return os.MkdirAll(dstPath, info.Mode())
+		}
+
+		return copyFile(path, dstPath)
+	})
+}
+
+// copyFile copies a file from src to dst.
+func copyFile(src, dst string) error {
+	srcFile, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer srcFile.Close()
+
+	dstFile, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer dstFile.Close()
+
+	_, err = io.Copy(dstFile, srcFile)
+	return err
+}