Skip to content

armgit / memos

Public Imported from GitHub
0

Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.

test: improve migration tests stability and maintainability

Johnnyf4cf2e955996
diff --git a/go.mod b/go.mod
index cd7f5b5e1..fc85404e7 100644
--- a/go.mod
+++ b/go.mod
@@ -9,6 +9,7 @@ require (
github.com/aws/aws-sdk-go-v2/credentials v1.18.16
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.4
github.com/aws/aws-sdk-go-v2/service/s3 v1.87.3
+ github.com/docker/docker v28.5.1+incompatible
github.com/go-sql-driver/mysql v1.9.3
github.com/google/cel-go v0.26.1
github.com/google/uuid v1.6.0
@@ -50,7 +51,6 @@ require (
github.com/containerd/platforms v0.2.1 // indirect
github.com/cpuguy83/dockercfg v0.3.2 // indirect
github.com/distribution/reference v0.6.0 // indirect
- github.com/docker/docker v28.5.1+incompatible // indirect
github.com/docker/go-connections v0.6.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
diff --git a/store/test/Dockerfile b/store/test/Dockerfile
new file mode 100644
index 000000000..6b25a7079
--- /dev/null
+++ b/store/test/Dockerfile
@@ -0,0 +1,13 @@
+FROM golang:1.25-alpine AS backend
+WORKDIR /backend-build
+COPY . .
+RUN go build -o memos ./cmd/memos
+
+FROM alpine:latest
+WORKDIR /usr/local/memos
+COPY --from=backend /backend-build/memos /usr/local/memos/
+EXPOSE 5230
+RUN mkdir -p /var/opt/memos
+ENV MEMOS_MODE="prod"
+ENV MEMOS_PORT="5230"
+ENTRYPOINT ["./memos"]
diff --git a/store/test/containers.go b/store/test/containers.go
index bd65ea40e..c5e139306 100644
--- a/store/test/containers.go
+++ b/store/test/containers.go
@@ -10,10 +10,12 @@ import (
"testing"
"time"
+ "github.com/docker/docker/api/types/container"
"github.com/pkg/errors"
"github.com/testcontainers/testcontainers-go"
"github.com/testcontainers/testcontainers-go/modules/mysql"
"github.com/testcontainers/testcontainers-go/modules/postgres"
+ "github.com/testcontainers/testcontainers-go/network"
"github.com/testcontainers/testcontainers-go/wait"
// Database drivers for connection verification.
@@ -24,9 +26,21 @@ import (
const (
testUser = "root"
testPassword = "test"
+
+ // Memos container settings for migration testing.
+ MemosDockerImage = "neosmemo/memos"
+ StableMemosVersion = "stable"
)
var (
+ // MemosStartupWaitStrategy defines the wait strategy for Memos container startup.
+ // It waits for the "started" log message (compatible with both old and new versions)
+ // and checks if port 5230 is listening.
+ MemosStartupWaitStrategy = wait.ForAll(
+ wait.ForLog("started"),
+ wait.ForListeningPort("5230/tcp"),
+ ).WithDeadline(180 * time.Second)
+
mysqlContainer *mysql.MySQLContainer
postgresContainer *postgres.PostgresContainer
mysqlOnce sync.Once
@@ -34,13 +48,36 @@ var (
mysqlBaseDSN string
postgresBaseDSN string
dbCounter atomic.Int64
+
+ // Network for container communication.
+ testDockerNetwork *testcontainers.DockerNetwork
+ testNetworkOnce sync.Once
)
+// getTestNetwork creates or returns the shared Docker network for container communication.
+func getTestNetwork(ctx context.Context) (*testcontainers.DockerNetwork, error) {
+ var networkErr error
+ testNetworkOnce.Do(func() {
+ nw, err := network.New(ctx, network.WithDriver("bridge"))
+ if err != nil {
+ networkErr = err
+ return
+ }
+ testDockerNetwork = nw
+ })
+ return testDockerNetwork, networkErr
+}
+
// GetMySQLDSN starts a MySQL container (if not already running) and creates a fresh database for this test.
func GetMySQLDSN(t *testing.T) string {
ctx := context.Background()
mysqlOnce.Do(func() {
+ nw, err := getTestNetwork(ctx)
+ if err != nil {
+ t.Fatalf("failed to create test network: %v", err)
+ }
+
container, err := mysql.Run(ctx,
"mysql:8",
mysql.WithDatabase("init_db"),
@@ -55,6 +92,7 @@ func GetMySQLDSN(t *testing.T) string {
wait.ForListeningPort("3306/tcp"),
).WithDeadline(120*time.Second),
),
+ network.WithNetwork(nil, nw),
)
if err != nil {
t.Fatalf("failed to start MySQL container: %v", err)
@@ -130,6 +168,11 @@ func GetPostgresDSN(t *testing.T) string {
ctx := context.Background()
postgresOnce.Do(func() {
+ nw, err := getTestNetwork(ctx)
+ if err != nil {
+ t.Fatalf("failed to create test network: %v", err)
+ }
+
container, err := postgres.Run(ctx,
"postgres:18",
postgres.WithDatabase("init_db"),
@@ -141,6 +184,7 @@ func GetPostgresDSN(t *testing.T) string {
wait.ForListeningPort("5432/tcp"),
).WithDeadline(120*time.Second),
),
+ network.WithNetwork(nil, nw),
)
if err != nil {
t.Fatalf("failed to start PostgreSQL container: %v", err)
@@ -179,7 +223,106 @@ func GetPostgresDSN(t *testing.T) string {
return strings.Replace(postgresBaseDSN, "/init_db?", "/"+dbName+"?", 1)
}
-// TerminateContainers cleans up all running containers.
+// GetDedicatedMySQLDSN starts a dedicated MySQL container for migration testing.
+// This is needed because older Memos versions have bugs when connecting to a MySQL
+// server that has other initialized databases (they incorrectly query migration_history
+// on a fresh database without checking if the DB is initialized).
+// Returns: DSN for host access, container hostname for internal network access, cleanup function.
+func GetDedicatedMySQLDSN(t *testing.T) (dsn string, containerHost string, cleanup func()) {
+ ctx := context.Background()
+
+ nw, err := getTestNetwork(ctx)
+ if err != nil {
+ t.Fatalf("failed to create test network: %v", err)
+ }
+
+ container, err := mysql.Run(ctx,
+ "mysql:8",
+ mysql.WithDatabase("memos"),
+ mysql.WithUsername("root"),
+ mysql.WithPassword(testPassword),
+ testcontainers.WithEnv(map[string]string{
+ "MYSQL_ROOT_PASSWORD": testPassword,
+ }),
+ testcontainers.WithWaitStrategy(
+ wait.ForAll(
+ wait.ForLog("ready for connections").WithOccurrence(2),
+ wait.ForListeningPort("3306/tcp"),
+ ).WithDeadline(120*time.Second),
+ ),
+ network.WithNetwork(nil, nw),
+ )
+ if err != nil {
+ t.Fatalf("failed to start dedicated MySQL container: %v", err)
+ }
+
+ hostDSN, err := container.ConnectionString(ctx, "multiStatements=true")
+ if err != nil {
+ container.Terminate(ctx)
+ t.Fatalf("failed to get MySQL connection string: %v", err)
+ }
+
+ if err := waitForDB("mysql", hostDSN, 30*time.Second); err != nil {
+ container.Terminate(ctx)
+ t.Fatalf("MySQL not ready for connections: %v", err)
+ }
+
+ name, _ := container.Name(ctx)
+ host := strings.TrimPrefix(name, "/")
+
+ return hostDSN, host, func() {
+ container.Terminate(ctx)
+ }
+}
+
+// GetDedicatedPostgresDSN starts a dedicated PostgreSQL container for migration testing.
+// This is needed for isolation when testing migrations with older Memos versions.
+// Returns: DSN for host access, container hostname for internal network access, cleanup function.
+func GetDedicatedPostgresDSN(t *testing.T) (dsn string, containerHost string, cleanup func()) {
+ ctx := context.Background()
+
+ nw, err := getTestNetwork(ctx)
+ if err != nil {
+ t.Fatalf("failed to create test network: %v", err)
+ }
+
+ container, err := postgres.Run(ctx,
+ "postgres:18",
+ postgres.WithDatabase("memos"),
+ postgres.WithUsername(testUser),
+ postgres.WithPassword(testPassword),
+ testcontainers.WithWaitStrategy(
+ wait.ForAll(
+ wait.ForLog("database system is ready to accept connections").WithOccurrence(2),
+ wait.ForListeningPort("5432/tcp"),
+ ).WithDeadline(120*time.Second),
+ ),
+ network.WithNetwork(nil, nw),
+ )
+ if err != nil {
+ t.Fatalf("failed to start dedicated PostgreSQL container: %v", err)
+ }
+
+ hostDSN, err := container.ConnectionString(ctx, "sslmode=disable")
+ if err != nil {
+ container.Terminate(ctx)
+ t.Fatalf("failed to get PostgreSQL connection string: %v", err)
+ }
+
+ if err := waitForDB("postgres", hostDSN, 30*time.Second); err != nil {
+ container.Terminate(ctx)
+ t.Fatalf("PostgreSQL not ready for connections: %v", err)
+ }
+
+ name, _ := container.Name(ctx)
+ host := strings.TrimPrefix(name, "/")
+
+ return hostDSN, host, func() {
+ container.Terminate(ctx)
+ }
+}
+
+// TerminateContainers cleans up all running containers and network.
// This is typically called from TestMain.
func TerminateContainers() {
ctx := context.Background()
@@ -189,4 +332,101 @@ func TerminateContainers() {
if postgresContainer != nil {
_ = postgresContainer.Terminate(ctx)
}
+ if testDockerNetwork != nil {
+ _ = testDockerNetwork.Remove(ctx)
+ }
+}
+
+// GetMySQLContainerHost returns the MySQL container hostname for use within the Docker network.
+func GetMySQLContainerHost() string {
+ if mysqlContainer == nil {
+ return ""
+ }
+ name, _ := mysqlContainer.Name(context.Background())
+ // Remove leading slash from container name
+ return strings.TrimPrefix(name, "/")
+}
+
+// GetPostgresContainerHost returns the PostgreSQL container hostname for use within the Docker network.
+func GetPostgresContainerHost() string {
+ if postgresContainer == nil {
+ return ""
+ }
+ name, _ := postgresContainer.Name(context.Background())
+ return strings.TrimPrefix(name, "/")
+}
+
+// MemosContainerConfig holds configuration for starting a Memos container.
+type MemosContainerConfig struct {
+ Version string // Memos version tag (e.g., "0.25")
+ Driver string // Database driver: sqlite, mysql, postgres
+ DSN string // Database DSN (for mysql/postgres)
+ DataDir string // Host directory to mount for SQLite data
+}
+
+// StartMemosContainer starts a Memos container for migration testing.
+// For SQLite, it mounts the dataDir to /var/opt/memos.
+// For MySQL/PostgreSQL, it connects to the provided DSN via the test network.
+// If Version is "local", builds the image from the local Dockerfile.
+func StartMemosContainer(ctx context.Context, cfg MemosContainerConfig) (testcontainers.Container, error) {
+ env := map[string]string{
+ "MEMOS_MODE": "prod",
+ }
+
+ var mounts []testcontainers.ContainerMount
+ var opts []testcontainers.ContainerCustomizer
+
+ switch cfg.Driver {
+ case "sqlite":
+ env["MEMOS_DRIVER"] = "sqlite"
+ opts = append(opts, testcontainers.WithHostConfigModifier(func(hc *container.HostConfig) {
+ hc.Binds = append(hc.Binds, fmt.Sprintf("%s:%s", cfg.DataDir, "/var/opt/memos"))
+ }))
+ case "mysql":
+ env["MEMOS_DRIVER"] = "mysql"
+ env["MEMOS_DSN"] = cfg.DSN
+ opts = append(opts, network.WithNetwork(nil, testDockerNetwork))
+ case "postgres":
+ env["MEMOS_DRIVER"] = "postgres"
+ env["MEMOS_DSN"] = cfg.DSN
+ opts = append(opts, network.WithNetwork(nil, testDockerNetwork))
+ default:
+ return nil, errors.Errorf("unsupported driver: %s", cfg.Driver)
+ }
+
+ req := testcontainers.ContainerRequest{
+ Env: env,
+ Mounts: testcontainers.Mounts(mounts...),
+ ExposedPorts: []string{"5230/tcp"},
+ WaitingFor: MemosStartupWaitStrategy,
+ }
+
+ // Use local Dockerfile build or remote image
+ if cfg.Version == "local" {
+ req.FromDockerfile = testcontainers.FromDockerfile{
+ Context: "../../",
+ Dockerfile: "store/test/Dockerfile", // Simple Dockerfile without BuildKit requirements
+ }
+ } else {
+ req.Image = fmt.Sprintf("%s:%s", MemosDockerImage, cfg.Version)
+ }
+
+ genericReq := testcontainers.GenericContainerRequest{
+ ContainerRequest: req,
+ Started: true,
+ }
+
+ // Apply network options
+ for _, opt := range opts {
+ if err := opt.Customize(&genericReq); err != nil {
+ return nil, errors.Wrap(err, "failed to apply container option")
+ }
+ }
+
+ container, err := testcontainers.GenericContainer(ctx, genericReq)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to start memos container")
+ }
+
+ return container, nil
}
diff --git a/store/test/migrator_test.go b/store/test/migrator_test.go
index 946d9ce10..87bcb18ee 100644
--- a/store/test/migrator_test.go
+++ b/store/test/migrator_test.go
@@ -2,7 +2,10 @@ package test
import (
"context"
+ "fmt"
+ "strings"
"testing"
+ "time"
"github.com/stretchr/testify/require"
)
@@ -15,3 +18,120 @@ func TestGetCurrentSchemaVersion(t *testing.T) {
require.NoError(t, err)
require.Equal(t, "0.26.1", currentSchemaVersion)
}
+
+// TestFreshInstall verifies that LATEST.sql applies correctly on a fresh database.
+// This is essentially what NewTestingStore already does, but we make it explicit.
+func TestFreshInstall(t *testing.T) {
+ ctx := context.Background()
+
+ // NewTestingStore creates a fresh database and runs Migrate()
+ // which applies LATEST.sql for uninitialized databases
+ ts := NewTestingStore(ctx, t)
+
+ // Verify migration completed successfully
+ currentSchemaVersion, err := ts.GetCurrentSchemaVersion()
+ require.NoError(t, err)
+ require.NotEmpty(t, currentSchemaVersion, "schema version should be set after fresh install")
+
+ // Verify we can read instance settings (basic sanity check)
+ instanceSetting, err := ts.GetInstanceBasicSetting(ctx)
+ require.NoError(t, err)
+ require.Equal(t, currentSchemaVersion, instanceSetting.SchemaVersion)
+}
+
+// testMigration is a helper function that orchestrates the migration test flow.
+// It starts the stable version, waits for initialization, and then starts the local version.
+func testMigration(t *testing.T, driver string, prepareFunc func() (MemosContainerConfig, func())) {
+ if getDriverFromEnv() != driver {
+ t.Skipf("skipping %s migration test for non-%s driver", driver, driver)
+ }
+
+ ctx := context.Background()
+
+ // Prepare resources (temp dir or dedicated container)
+ cfg, cleanup := prepareFunc()
+ if cleanup != nil {
+ defer cleanup()
+ }
+
+ // 1. Start Old Memos container (Stable)
+ cfg.Version = StableMemosVersion
+ t.Logf("Starting Memos %s container to initialize %s database...", cfg.Version, driver)
+ oldContainer, err := StartMemosContainer(ctx, cfg)
+ require.NoError(t, err, "failed to start old memos container")
+
+ // Wait for database to be fully initialized
+ time.Sleep(5 * time.Second)
+
+ // Stop the old container
+ err = oldContainer.Terminate(ctx)
+ require.NoError(t, err, "failed to stop old memos container")
+
+ t.Log("Old Memos container stopped, starting new container with local build...")
+
+ // 2. Start New Memos container (Local)
+ cfg.Version = "local" // Triggers local build in StartMemosContainer
+ newContainer, err := StartMemosContainer(ctx, cfg)
+ require.NoError(t, err, "failed to start new memos container - migration may have failed")
+ defer newContainer.Terminate(ctx)
+
+ t.Logf("Migration successful: %s -> local build", StableMemosVersion)
+}
+
+// TestMigrationFromPreviousVersion_SQLite verifies that migrating from the previous
+// Memos version to the current version works correctly for SQLite.
+func TestMigrationFromPreviousVersion_SQLite(t *testing.T) {
+ testMigration(t, "sqlite", func() (MemosContainerConfig, func()) {
+ // Create a temp directory for SQLite data that persists across container restarts
+ dataDir := t.TempDir()
+ return MemosContainerConfig{
+ Driver: "sqlite",
+ DataDir: dataDir,
+ }, nil
+ })
+}
+
+// TestMigrationFromPreviousVersion_MySQL verifies that migrating from the previous
+// Memos version to the current version works correctly for MySQL.
+func TestMigrationFromPreviousVersion_MySQL(t *testing.T) {
+ testMigration(t, "mysql", func() (MemosContainerConfig, func()) {
+ // For migration testing, we need a dedicated MySQL container
+ dsn, containerHost, cleanup := GetDedicatedMySQLDSN(t)
+
+ // Extract database name from DSN
+ parts := strings.Split(dsn, "/")
+ dbNameWithParams := parts[len(parts)-1]
+ dbName := strings.Split(dbNameWithParams, "?")[0]
+
+ // Container DSN uses internal network hostname
+ containerDSN := fmt.Sprintf("%s:%s@tcp(%s:3306)/%s", testUser, testPassword, containerHost, dbName)
+
+ return MemosContainerConfig{
+ Driver: "mysql",
+ DSN: containerDSN,
+ }, cleanup
+ })
+}
+
+// TestMigrationFromPreviousVersion_Postgres verifies that migrating from the previous
+// Memos version to the current version works correctly for PostgreSQL.
+func TestMigrationFromPreviousVersion_Postgres(t *testing.T) {
+ testMigration(t, "postgres", func() (MemosContainerConfig, func()) {
+ // For migration testing, we need a dedicated PostgreSQL container
+ dsn, containerHost, cleanup := GetDedicatedPostgresDSN(t)
+
+ // Extract database name from DSN
+ parts := strings.Split(dsn, "/")
+ dbNameWithParams := parts[len(parts)-1]
+ dbName := strings.Split(dbNameWithParams, "?")[0]
+
+ // Container DSN uses internal network hostname
+ containerDSN := fmt.Sprintf("postgres://%s:%s@%s:5432/%s?sslmode=disable",
+ testUser, testPassword, containerHost, dbName)
+
+ return MemosContainerConfig{
+ Driver: "postgres",
+ DSN: containerDSN,
+ }, cleanup
+ })
+}