Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.
Go dockerfossgomarkdownmemomicroblognote-takingnotesopen-sourceown-your-datareactself-hostedsqlitetimeline
test: improve migration tests stability and maintainability
Johnnyf4cf2e955996
diff --git a/go.mod b/go.modindex cd7f5b5e1..fc85404e7 100644--- a/go.mod+++ b/go.mod@@ -9,6 +9,7 @@ require (github.com/aws/aws-sdk-go-v2/credentials v1.18.16github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.4github.com/aws/aws-sdk-go-v2/service/s3 v1.87.3+ github.com/docker/docker v28.5.1+incompatiblegithub.com/go-sql-driver/mysql v1.9.3github.com/google/cel-go v0.26.1github.com/google/uuid v1.6.0@@ -50,7 +51,6 @@ require (github.com/containerd/platforms v0.2.1 // indirectgithub.com/cpuguy83/dockercfg v0.3.2 // indirectgithub.com/distribution/reference v0.6.0 // indirect- github.com/docker/docker v28.5.1+incompatible // indirectgithub.com/docker/go-connections v0.6.0 // indirectgithub.com/docker/go-units v0.5.0 // indirectgithub.com/dustin/go-humanize v1.0.1 // indirectdiff --git a/store/test/Dockerfile b/store/test/Dockerfilenew file mode 100644index 000000000..6b25a7079--- /dev/null+++ b/store/test/Dockerfile@@ -0,0 +1,13 @@+FROM golang:1.25-alpine AS backend+WORKDIR /backend-build+COPY . .+RUN go build -o memos ./cmd/memos++FROM alpine:latest+WORKDIR /usr/local/memos+COPY --from=backend /backend-build/memos /usr/local/memos/+EXPOSE 5230+RUN mkdir -p /var/opt/memos+ENV MEMOS_MODE="prod"+ENV MEMOS_PORT="5230"+ENTRYPOINT ["./memos"]diff --git a/store/test/containers.go b/store/test/containers.goindex bd65ea40e..c5e139306 100644--- a/store/test/containers.go+++ b/store/test/containers.go@@ -10,10 +10,12 @@ import ("testing""time"+ "github.com/docker/docker/api/types/container""github.com/pkg/errors""github.com/testcontainers/testcontainers-go""github.com/testcontainers/testcontainers-go/modules/mysql""github.com/testcontainers/testcontainers-go/modules/postgres"+ "github.com/testcontainers/testcontainers-go/network""github.com/testcontainers/testcontainers-go/wait"// Database drivers for connection verification.@@ -24,9 +26,21 @@ import (const (testUser = "root"testPassword = "test"++ // Memos container settings for migration testing.+ MemosDockerImage = "neosmemo/memos"+ StableMemosVersion = "stable")var (+ // MemosStartupWaitStrategy defines the wait strategy for Memos container startup.+ // It waits for the "started" log message (compatible with both old and new versions)+ // and checks if port 5230 is listening.+ MemosStartupWaitStrategy = wait.ForAll(+ wait.ForLog("started"),+ wait.ForListeningPort("5230/tcp"),+ ).WithDeadline(180 * time.Second)+mysqlContainer *mysql.MySQLContainerpostgresContainer *postgres.PostgresContainermysqlOnce sync.Once@@ -34,13 +48,36 @@ var (mysqlBaseDSN stringpostgresBaseDSN stringdbCounter atomic.Int64++ // Network for container communication.+ testDockerNetwork *testcontainers.DockerNetwork+ testNetworkOnce sync.Once)+// getTestNetwork creates or returns the shared Docker network for container communication.+func getTestNetwork(ctx context.Context) (*testcontainers.DockerNetwork, error) {+ var networkErr error+ testNetworkOnce.Do(func() {+ nw, err := network.New(ctx, network.WithDriver("bridge"))+ if err != nil {+ networkErr = err+ return+ }+ testDockerNetwork = nw+ })+ return testDockerNetwork, networkErr+}+// GetMySQLDSN starts a MySQL container (if not already running) and creates a fresh database for this test.func GetMySQLDSN(t *testing.T) string {ctx := context.Background()mysqlOnce.Do(func() {+ nw, err := getTestNetwork(ctx)+ if err != nil {+ t.Fatalf("failed to create test network: %v", err)+ }+container, err := mysql.Run(ctx,"mysql:8",mysql.WithDatabase("init_db"),@@ -55,6 +92,7 @@ func GetMySQLDSN(t *testing.T) string {wait.ForListeningPort("3306/tcp"),).WithDeadline(120*time.Second),),+ network.WithNetwork(nil, nw),)if err != nil {t.Fatalf("failed to start MySQL container: %v", err)@@ -130,6 +168,11 @@ func GetPostgresDSN(t *testing.T) string {ctx := context.Background()postgresOnce.Do(func() {+ nw, err := getTestNetwork(ctx)+ if err != nil {+ t.Fatalf("failed to create test network: %v", err)+ }+container, err := postgres.Run(ctx,"postgres:18",postgres.WithDatabase("init_db"),@@ -141,6 +184,7 @@ func GetPostgresDSN(t *testing.T) string {wait.ForListeningPort("5432/tcp"),).WithDeadline(120*time.Second),),+ network.WithNetwork(nil, nw),)if err != nil {t.Fatalf("failed to start PostgreSQL container: %v", err)@@ -179,7 +223,106 @@ func GetPostgresDSN(t *testing.T) string {return strings.Replace(postgresBaseDSN, "/init_db?", "/"+dbName+"?", 1)}-// TerminateContainers cleans up all running containers.+// GetDedicatedMySQLDSN starts a dedicated MySQL container for migration testing.+// This is needed because older Memos versions have bugs when connecting to a MySQL+// server that has other initialized databases (they incorrectly query migration_history+// on a fresh database without checking if the DB is initialized).+// Returns: DSN for host access, container hostname for internal network access, cleanup function.+func GetDedicatedMySQLDSN(t *testing.T) (dsn string, containerHost string, cleanup func()) {+ ctx := context.Background()++ nw, err := getTestNetwork(ctx)+ if err != nil {+ t.Fatalf("failed to create test network: %v", err)+ }++ container, err := mysql.Run(ctx,+ "mysql:8",+ mysql.WithDatabase("memos"),+ mysql.WithUsername("root"),+ mysql.WithPassword(testPassword),+ testcontainers.WithEnv(map[string]string{+ "MYSQL_ROOT_PASSWORD": testPassword,+ }),+ testcontainers.WithWaitStrategy(+ wait.ForAll(+ wait.ForLog("ready for connections").WithOccurrence(2),+ wait.ForListeningPort("3306/tcp"),+ ).WithDeadline(120*time.Second),+ ),+ network.WithNetwork(nil, nw),+ )+ if err != nil {+ t.Fatalf("failed to start dedicated MySQL container: %v", err)+ }++ hostDSN, err := container.ConnectionString(ctx, "multiStatements=true")+ if err != nil {+ container.Terminate(ctx)+ t.Fatalf("failed to get MySQL connection string: %v", err)+ }++ if err := waitForDB("mysql", hostDSN, 30*time.Second); err != nil {+ container.Terminate(ctx)+ t.Fatalf("MySQL not ready for connections: %v", err)+ }++ name, _ := container.Name(ctx)+ host := strings.TrimPrefix(name, "/")++ return hostDSN, host, func() {+ container.Terminate(ctx)+ }+}++// GetDedicatedPostgresDSN starts a dedicated PostgreSQL container for migration testing.+// This is needed for isolation when testing migrations with older Memos versions.+// Returns: DSN for host access, container hostname for internal network access, cleanup function.+func GetDedicatedPostgresDSN(t *testing.T) (dsn string, containerHost string, cleanup func()) {+ ctx := context.Background()++ nw, err := getTestNetwork(ctx)+ if err != nil {+ t.Fatalf("failed to create test network: %v", err)+ }++ container, err := postgres.Run(ctx,+ "postgres:18",+ postgres.WithDatabase("memos"),+ postgres.WithUsername(testUser),+ postgres.WithPassword(testPassword),+ testcontainers.WithWaitStrategy(+ wait.ForAll(+ wait.ForLog("database system is ready to accept connections").WithOccurrence(2),+ wait.ForListeningPort("5432/tcp"),+ ).WithDeadline(120*time.Second),+ ),+ network.WithNetwork(nil, nw),+ )+ if err != nil {+ t.Fatalf("failed to start dedicated PostgreSQL container: %v", err)+ }++ hostDSN, err := container.ConnectionString(ctx, "sslmode=disable")+ if err != nil {+ container.Terminate(ctx)+ t.Fatalf("failed to get PostgreSQL connection string: %v", err)+ }++ if err := waitForDB("postgres", hostDSN, 30*time.Second); err != nil {+ container.Terminate(ctx)+ t.Fatalf("PostgreSQL not ready for connections: %v", err)+ }++ name, _ := container.Name(ctx)+ host := strings.TrimPrefix(name, "/")++ return hostDSN, host, func() {+ container.Terminate(ctx)+ }+}++// TerminateContainers cleans up all running containers and network.// This is typically called from TestMain.func TerminateContainers() {ctx := context.Background()@@ -189,4 +332,101 @@ func TerminateContainers() {if postgresContainer != nil {_ = postgresContainer.Terminate(ctx)}+ if testDockerNetwork != nil {+ _ = testDockerNetwork.Remove(ctx)+ }+}++// GetMySQLContainerHost returns the MySQL container hostname for use within the Docker network.+func GetMySQLContainerHost() string {+ if mysqlContainer == nil {+ return ""+ }+ name, _ := mysqlContainer.Name(context.Background())+ // Remove leading slash from container name+ return strings.TrimPrefix(name, "/")+}++// GetPostgresContainerHost returns the PostgreSQL container hostname for use within the Docker network.+func GetPostgresContainerHost() string {+ if postgresContainer == nil {+ return ""+ }+ name, _ := postgresContainer.Name(context.Background())+ return strings.TrimPrefix(name, "/")+}++// MemosContainerConfig holds configuration for starting a Memos container.+type MemosContainerConfig struct {+ Version string // Memos version tag (e.g., "0.25")+ Driver string // Database driver: sqlite, mysql, postgres+ DSN string // Database DSN (for mysql/postgres)+ DataDir string // Host directory to mount for SQLite data+}++// StartMemosContainer starts a Memos container for migration testing.+// For SQLite, it mounts the dataDir to /var/opt/memos.+// For MySQL/PostgreSQL, it connects to the provided DSN via the test network.+// If Version is "local", builds the image from the local Dockerfile.+func StartMemosContainer(ctx context.Context, cfg MemosContainerConfig) (testcontainers.Container, error) {+ env := map[string]string{+ "MEMOS_MODE": "prod",+ }++ var mounts []testcontainers.ContainerMount+ var opts []testcontainers.ContainerCustomizer++ switch cfg.Driver {+ case "sqlite":+ env["MEMOS_DRIVER"] = "sqlite"+ opts = append(opts, testcontainers.WithHostConfigModifier(func(hc *container.HostConfig) {+ hc.Binds = append(hc.Binds, fmt.Sprintf("%s:%s", cfg.DataDir, "/var/opt/memos"))+ }))+ case "mysql":+ env["MEMOS_DRIVER"] = "mysql"+ env["MEMOS_DSN"] = cfg.DSN+ opts = append(opts, network.WithNetwork(nil, testDockerNetwork))+ case "postgres":+ env["MEMOS_DRIVER"] = "postgres"+ env["MEMOS_DSN"] = cfg.DSN+ opts = append(opts, network.WithNetwork(nil, testDockerNetwork))+ default:+ return nil, errors.Errorf("unsupported driver: %s", cfg.Driver)+ }++ req := testcontainers.ContainerRequest{+ Env: env,+ Mounts: testcontainers.Mounts(mounts...),+ ExposedPorts: []string{"5230/tcp"},+ WaitingFor: MemosStartupWaitStrategy,+ }++ // Use local Dockerfile build or remote image+ if cfg.Version == "local" {+ req.FromDockerfile = testcontainers.FromDockerfile{+ Context: "../../",+ Dockerfile: "store/test/Dockerfile", // Simple Dockerfile without BuildKit requirements+ }+ } else {+ req.Image = fmt.Sprintf("%s:%s", MemosDockerImage, cfg.Version)+ }++ genericReq := testcontainers.GenericContainerRequest{+ ContainerRequest: req,+ Started: true,+ }++ // Apply network options+ for _, opt := range opts {+ if err := opt.Customize(&genericReq); err != nil {+ return nil, errors.Wrap(err, "failed to apply container option")+ }+ }++ container, err := testcontainers.GenericContainer(ctx, genericReq)+ if err != nil {+ return nil, errors.Wrap(err, "failed to start memos container")+ }++ return container, nil}diff --git a/store/test/migrator_test.go b/store/test/migrator_test.goindex 946d9ce10..87bcb18ee 100644--- a/store/test/migrator_test.go+++ b/store/test/migrator_test.go@@ -2,7 +2,10 @@ package testimport ("context"+ "fmt"+ "strings""testing"+ "time""github.com/stretchr/testify/require")@@ -15,3 +18,120 @@ func TestGetCurrentSchemaVersion(t *testing.T) {require.NoError(t, err)require.Equal(t, "0.26.1", currentSchemaVersion)}++// TestFreshInstall verifies that LATEST.sql applies correctly on a fresh database.+// This is essentially what NewTestingStore already does, but we make it explicit.+func TestFreshInstall(t *testing.T) {+ ctx := context.Background()++ // NewTestingStore creates a fresh database and runs Migrate()+ // which applies LATEST.sql for uninitialized databases+ ts := NewTestingStore(ctx, t)++ // Verify migration completed successfully+ currentSchemaVersion, err := ts.GetCurrentSchemaVersion()+ require.NoError(t, err)+ require.NotEmpty(t, currentSchemaVersion, "schema version should be set after fresh install")++ // Verify we can read instance settings (basic sanity check)+ instanceSetting, err := ts.GetInstanceBasicSetting(ctx)+ require.NoError(t, err)+ require.Equal(t, currentSchemaVersion, instanceSetting.SchemaVersion)+}++// testMigration is a helper function that orchestrates the migration test flow.+// It starts the stable version, waits for initialization, and then starts the local version.+func testMigration(t *testing.T, driver string, prepareFunc func() (MemosContainerConfig, func())) {+ if getDriverFromEnv() != driver {+ t.Skipf("skipping %s migration test for non-%s driver", driver, driver)+ }++ ctx := context.Background()++ // Prepare resources (temp dir or dedicated container)+ cfg, cleanup := prepareFunc()+ if cleanup != nil {+ defer cleanup()+ }++ // 1. Start Old Memos container (Stable)+ cfg.Version = StableMemosVersion+ t.Logf("Starting Memos %s container to initialize %s database...", cfg.Version, driver)+ oldContainer, err := StartMemosContainer(ctx, cfg)+ require.NoError(t, err, "failed to start old memos container")++ // Wait for database to be fully initialized+ time.Sleep(5 * time.Second)++ // Stop the old container+ err = oldContainer.Terminate(ctx)+ require.NoError(t, err, "failed to stop old memos container")++ t.Log("Old Memos container stopped, starting new container with local build...")++ // 2. Start New Memos container (Local)+ cfg.Version = "local" // Triggers local build in StartMemosContainer+ newContainer, err := StartMemosContainer(ctx, cfg)+ require.NoError(t, err, "failed to start new memos container - migration may have failed")+ defer newContainer.Terminate(ctx)++ t.Logf("Migration successful: %s -> local build", StableMemosVersion)+}++// TestMigrationFromPreviousVersion_SQLite verifies that migrating from the previous+// Memos version to the current version works correctly for SQLite.+func TestMigrationFromPreviousVersion_SQLite(t *testing.T) {+ testMigration(t, "sqlite", func() (MemosContainerConfig, func()) {+ // Create a temp directory for SQLite data that persists across container restarts+ dataDir := t.TempDir()+ return MemosContainerConfig{+ Driver: "sqlite",+ DataDir: dataDir,+ }, nil+ })+}++// TestMigrationFromPreviousVersion_MySQL verifies that migrating from the previous+// Memos version to the current version works correctly for MySQL.+func TestMigrationFromPreviousVersion_MySQL(t *testing.T) {+ testMigration(t, "mysql", func() (MemosContainerConfig, func()) {+ // For migration testing, we need a dedicated MySQL container+ dsn, containerHost, cleanup := GetDedicatedMySQLDSN(t)++ // Extract database name from DSN+ parts := strings.Split(dsn, "/")+ dbNameWithParams := parts[len(parts)-1]+ dbName := strings.Split(dbNameWithParams, "?")[0]++ // Container DSN uses internal network hostname+ containerDSN := fmt.Sprintf("%s:%s@tcp(%s:3306)/%s", testUser, testPassword, containerHost, dbName)++ return MemosContainerConfig{+ Driver: "mysql",+ DSN: containerDSN,+ }, cleanup+ })+}++// TestMigrationFromPreviousVersion_Postgres verifies that migrating from the previous+// Memos version to the current version works correctly for PostgreSQL.+func TestMigrationFromPreviousVersion_Postgres(t *testing.T) {+ testMigration(t, "postgres", func() (MemosContainerConfig, func()) {+ // For migration testing, we need a dedicated PostgreSQL container+ dsn, containerHost, cleanup := GetDedicatedPostgresDSN(t)++ // Extract database name from DSN+ parts := strings.Split(dsn, "/")+ dbNameWithParams := parts[len(parts)-1]+ dbName := strings.Split(dbNameWithParams, "?")[0]++ // Container DSN uses internal network hostname+ containerDSN := fmt.Sprintf("postgres://%s:%s@%s:5432/%s?sslmode=disable",+ testUser, testPassword, containerHost, dbName)++ return MemosContainerConfig{+ Driver: "postgres",+ DSN: containerDSN,+ }, cleanup+ })+}