Skip to content

armgit / memos

Public Imported from GitHub
0

Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.

refactor: update migration history methods

Steven95de5cc7004a
diff --git a/internal/version/version.go b/internal/version/version.go
index ca64424eb..ba57f3a2f 100644
--- a/internal/version/version.go
+++ b/internal/version/version.go
@@ -21,11 +21,15 @@ func GetCurrentVersion(mode string) string {
return Version
}
+// GetMinorVersion extracts the minor version (e.g., "0.25") from a full version string (e.g., "0.25.1").
+// Returns the minor version string or empty string if the version format is invalid.
+// Version format should be "major.minor.patch" (e.g., "0.25.1").
func GetMinorVersion(version string) string {
versionList := strings.Split(version, ".")
- if len(versionList) < 3 {
+ if len(versionList) < 2 {
return ""
}
+ // Return major.minor only (first two components)
return versionList[0] + "." + versionList[1]
}
diff --git a/store/db/mysql/migration_history.go b/store/db/mysql/migration_history.go
index bc6c89fc4..fbaf5a710 100644
--- a/store/db/mysql/migration_history.go
+++ b/store/db/mysql/migration_history.go
@@ -6,6 +6,8 @@ import (
"github.com/usememos/memos/store"
)
+// FindMigrationHistoryList retrieves all migration history records.
+// NOTE: This method is deprecated along with the migration_history table.
func (d *DB) FindMigrationHistoryList(ctx context.Context, _ *store.FindMigrationHistory) ([]*store.MigrationHistory, error) {
query := "SELECT `version`, UNIX_TIMESTAMP(`created_ts`) FROM `migration_history` ORDER BY `created_ts` DESC"
rows, err := d.db.QueryContext(ctx, query)
@@ -34,6 +36,11 @@ func (d *DB) FindMigrationHistoryList(ctx context.Context, _ *store.FindMigratio
return list, nil
}
+// UpsertMigrationHistory inserts or updates a migration history record.
+// NOTE: This method is deprecated along with the migration_history table.
+// This uses separate INSERT and SELECT queries instead of INSERT...RETURNING because
+// MySQL doesn't support RETURNING clause in the same way as PostgreSQL/SQLite.
+// This could have race conditions but is acceptable for deprecated transition code.
func (d *DB) UpsertMigrationHistory(ctx context.Context, upsert *store.UpsertMigrationHistory) (*store.MigrationHistory, error) {
stmt := "INSERT INTO `migration_history` (`version`) VALUES (?) ON DUPLICATE KEY UPDATE `version` = ?"
_, err := d.db.ExecContext(ctx, stmt, upsert.Version, upsert.Version)
diff --git a/store/db/postgres/migration_history.go b/store/db/postgres/migration_history.go
index 385b898b4..5464dfbca 100644
--- a/store/db/postgres/migration_history.go
+++ b/store/db/postgres/migration_history.go
@@ -6,6 +6,8 @@ import (
"github.com/usememos/memos/store"
)
+// FindMigrationHistoryList retrieves all migration history records.
+// NOTE: This method is deprecated along with the migration_history table.
func (d *DB) FindMigrationHistoryList(ctx context.Context, _ *store.FindMigrationHistory) ([]*store.MigrationHistory, error) {
query := "SELECT version, created_ts FROM migration_history ORDER BY created_ts DESC"
rows, err := d.db.QueryContext(ctx, query)
@@ -34,6 +36,8 @@ func (d *DB) FindMigrationHistoryList(ctx context.Context, _ *store.FindMigratio
return list, nil
}
+// UpsertMigrationHistory inserts or updates a migration history record.
+// NOTE: This method is deprecated along with the migration_history table.
func (d *DB) UpsertMigrationHistory(ctx context.Context, upsert *store.UpsertMigrationHistory) (*store.MigrationHistory, error) {
stmt := `
INSERT INTO migration_history (
diff --git a/store/db/sqlite/migration_history.go b/store/db/sqlite/migration_history.go
index b7ec01b24..3403bcfdb 100644
--- a/store/db/sqlite/migration_history.go
+++ b/store/db/sqlite/migration_history.go
@@ -6,6 +6,8 @@ import (
"github.com/usememos/memos/store"
)
+// FindMigrationHistoryList retrieves all migration history records.
+// NOTE: This method is deprecated along with the migration_history table.
func (d *DB) FindMigrationHistoryList(ctx context.Context, _ *store.FindMigrationHistory) ([]*store.MigrationHistory, error) {
query := "SELECT `version`, `created_ts` FROM `migration_history` ORDER BY `created_ts` DESC"
rows, err := d.db.QueryContext(ctx, query)
@@ -34,6 +36,8 @@ func (d *DB) FindMigrationHistoryList(ctx context.Context, _ *store.FindMigratio
return list, nil
}
+// UpsertMigrationHistory inserts or updates a migration history record.
+// NOTE: This method is deprecated along with the migration_history table.
func (d *DB) UpsertMigrationHistory(ctx context.Context, upsert *store.UpsertMigrationHistory) (*store.MigrationHistory, error) {
stmt := `
INSERT INTO migration_history (
diff --git a/store/driver.go b/store/driver.go
index bc35464f0..cb5d6eb9c 100644
--- a/store/driver.go
+++ b/store/driver.go
@@ -14,6 +14,9 @@ type Driver interface {
IsInitialized(ctx context.Context) (bool, error)
// MigrationHistory model related methods.
+ // NOTE: These methods are deprecated. The migration_history table is no longer used
+ // for tracking schema versions. Schema version is now stored in workspace_setting.
+ // These methods are kept for backward compatibility to migrate existing installations.
FindMigrationHistoryList(ctx context.Context, find *FindMigrationHistory) ([]*MigrationHistory, error)
UpsertMigrationHistory(ctx context.Context, upsert *UpsertMigrationHistory) (*MigrationHistory, error)
diff --git a/store/migration_history.go b/store/migration_history.go
index 693663bef..f25b31568 100644
--- a/store/migration_history.go
+++ b/store/migration_history.go
@@ -1,13 +1,21 @@
package store
+// MigrationHistory represents a record in the migration_history table.
+// NOTE: The migration_history table is deprecated in favor of storing schema version
+// in workspace_setting (BASIC setting). This is kept for backward compatibility only.
+// Migration from migration_history to workspace_setting happens automatically during startup.
type MigrationHistory struct {
Version string
CreatedTs int64
}
+// UpsertMigrationHistory is used to insert or update a migration history record.
+// NOTE: This is deprecated along with the migration_history table.
type UpsertMigrationHistory struct {
Version string
}
+// FindMigrationHistory is used to query migration history records.
+// NOTE: This is deprecated along with the migration_history table.
type FindMigrationHistory struct {
}
diff --git a/store/migrator.go b/store/migrator.go
index 3e7f10f1d..b2409aee5 100644
--- a/store/migrator.go
+++ b/store/migrator.go
@@ -18,6 +18,30 @@ import (
storepb "github.com/usememos/memos/proto/gen/store"
)
+// Migration System Overview:
+//
+// The migration system handles database schema versioning and upgrades.
+// Schema version is stored in workspace_setting (the new system).
+// The old migration_history table is deprecated but still supported for backward compatibility.
+//
+// Migration Flow:
+// 1. preMigrate: Check if DB is initialized. If not, apply LATEST.sql
+// 2. normalizeMigrationHistoryList: Normalize old migration_history records (for pre-0.22 installations)
+// 3. migrateSchemaVersionToSetting: Migrate version from migration_history to workspace_setting
+// 4. Migrate (prod mode): Apply incremental migrations from current to target version
+// 5. Migrate (demo mode): Seed database with demo data
+//
+// Version Tracking:
+// - New installations: Schema version set in workspace_setting immediately
+// - Old installations: Version migrated from migration_history to workspace_setting automatically
+// - Empty version: Treated as 0.0.0 and all migrations applied
+//
+// Migration Files:
+// - Location: store/migration/{driver}/{version}/NN__description.sql
+// - Naming: NN is zero-padded patch number, description is human-readable
+// - Ordering: Files sorted lexicographically and applied in order
+// - LATEST.sql: Full schema for new installations (faster than incremental migrations)
+
//go:embed migration
var migrationFS embed.FS
@@ -31,8 +55,59 @@ const (
// LatestSchemaFileName is the name of the latest schema file.
// This file is used to apply the latest schema when no migration history is found.
LatestSchemaFileName = "LATEST.sql"
+
+ // defaultSchemaVersion is used when schema version is empty or not set.
+ // This handles edge cases for old installations without version tracking.
+ defaultSchemaVersion = "0.0.0"
+
+ // migrationHistoryNormalizedVersion is the version where migration_history normalization was completed.
+ // Before 0.22, migration history had inconsistent versioning that needed normalization.
+ migrationHistoryNormalizedVersion = "0.22"
+
+ // Mode constants for profile mode
+ modeProd = "prod"
+ modeDemo = "demo"
)
+// getSchemaVersionOrDefault returns the schema version or default if empty.
+// This ensures safe version comparisons and handles old installations.
+func getSchemaVersionOrDefault(schemaVersion string) string {
+ if schemaVersion == "" {
+ return defaultSchemaVersion
+ }
+ return schemaVersion
+}
+
+// isVersionEmpty checks if the schema version is empty or the default value.
+func isVersionEmpty(schemaVersion string) bool {
+ return schemaVersion == "" || schemaVersion == defaultSchemaVersion
+}
+
+// shouldApplyMigration determines if a migration file should be applied.
+// It checks if the file's version is between the current DB version and target version.
+func shouldApplyMigration(fileVersion, currentDBVersion, targetVersion string) bool {
+ currentDBVersionSafe := getSchemaVersionOrDefault(currentDBVersion)
+ return version.IsVersionGreaterThan(fileVersion, currentDBVersionSafe) &&
+ version.IsVersionGreaterOrEqualThan(targetVersion, fileVersion)
+}
+
+// validateMigrationFileName checks if a migration file follows the expected naming convention.
+// Expected format: "NN__description.sql" where NN is a zero-padded number.
+func validateMigrationFileName(filename string) error {
+ if !strings.Contains(filename, MigrateFileNameSplit) {
+ return errors.Errorf("invalid migration filename format (missing %s): %s", MigrateFileNameSplit, filename)
+ }
+ parts := strings.Split(filename, MigrateFileNameSplit)
+ if len(parts) < 2 {
+ return errors.Errorf("invalid migration filename format: %s", filename)
+ }
+ // Check if first part is a number
+ if _, err := strconv.Atoi(parts[0]); err != nil {
+ return errors.Errorf("migration filename must start with a number: %s", filename)
+ }
+ return nil
+}
+
// Migrate migrates the database schema to the latest version.
// It checks the current schema version and applies any necessary migrations.
// It also seeds the database with initial data if in demo mode.
@@ -42,7 +117,7 @@ func (s *Store) Migrate(ctx context.Context) error {
}
switch s.profile.Mode {
- case "prod":
+ case modeProd:
workspaceBasicSetting, err := s.GetWorkspaceBasicSetting(ctx)
if err != nil {
return errors.Wrap(err, "failed to get workspace basic setting")
@@ -51,53 +126,21 @@ func (s *Store) Migrate(ctx context.Context) error {
if err != nil {
return errors.Wrap(err, "failed to get current schema version")
}
- if version.IsVersionGreaterThan(workspaceBasicSetting.SchemaVersion, currentSchemaVersion) {
+ // Check for downgrade (but skip if schema version is empty - that means fresh/old installation)
+ if !isVersionEmpty(workspaceBasicSetting.SchemaVersion) && version.IsVersionGreaterThan(workspaceBasicSetting.SchemaVersion, currentSchemaVersion) {
slog.Error("cannot downgrade schema version",
slog.String("databaseVersion", workspaceBasicSetting.SchemaVersion),
slog.String("currentVersion", currentSchemaVersion),
)
return errors.Errorf("cannot downgrade schema version from %s to %s", workspaceBasicSetting.SchemaVersion, currentSchemaVersion)
}
- if version.IsVersionGreaterThan(currentSchemaVersion, workspaceBasicSetting.SchemaVersion) {
- filePaths, err := fs.Glob(migrationFS, fmt.Sprintf("%s*/*.sql", s.getMigrationBasePath()))
- if err != nil {
- return errors.Wrap(err, "failed to read migration files")
- }
- sort.Strings(filePaths)
-
- // Start a transaction to apply the latest schema.
- tx, err := s.driver.GetDB().Begin()
- if err != nil {
- return errors.Wrap(err, "failed to start transaction")
- }
- defer tx.Rollback()
-
- slog.Info("start migration", slog.String("currentSchemaVersion", workspaceBasicSetting.SchemaVersion), slog.String("targetSchemaVersion", currentSchemaVersion))
- for _, filePath := range filePaths {
- fileSchemaVersion, err := s.getSchemaVersionOfMigrateScript(filePath)
- if err != nil {
- return errors.Wrap(err, "failed to get schema version of migrate script")
- }
- if version.IsVersionGreaterThan(fileSchemaVersion, workspaceBasicSetting.SchemaVersion) && version.IsVersionGreaterOrEqualThan(currentSchemaVersion, fileSchemaVersion) {
- bytes, err := migrationFS.ReadFile(filePath)
- if err != nil {
- return errors.Wrapf(err, "failed to read minor version migration file: %s", filePath)
- }
- stmt := string(bytes)
- if err := s.execute(ctx, tx, stmt); err != nil {
- return errors.Wrapf(err, "migrate error: %s", stmt)
- }
- }
- }
- if err := tx.Commit(); err != nil {
- return errors.Wrap(err, "failed to commit transaction")
- }
- slog.Info("end migrate")
- if err := s.updateCurrentSchemaVersion(ctx, currentSchemaVersion); err != nil {
- return errors.Wrap(err, "failed to update current schema version")
+ // Apply migrations if needed (including when schema version is empty)
+ if isVersionEmpty(workspaceBasicSetting.SchemaVersion) || version.IsVersionGreaterThan(currentSchemaVersion, workspaceBasicSetting.SchemaVersion) {
+ if err := s.applyMigrations(ctx, workspaceBasicSetting.SchemaVersion, currentSchemaVersion); err != nil {
+ return errors.Wrap(err, "failed to apply migrations")
}
}
- case "demo":
+ case modeDemo:
// In demo mode, we should seed the database.
if err := s.seed(ctx); err != nil {
return errors.Wrap(err, "failed to seed")
@@ -108,6 +151,78 @@ func (s *Store) Migrate(ctx context.Context) error {
return nil
}
+// applyMigrations applies all necessary migration files between current and target schema versions.
+// It runs all migrations in a single transaction for atomicity.
+func (s *Store) applyMigrations(ctx context.Context, currentSchemaVersion, targetSchemaVersion string) error {
+ filePaths, err := fs.Glob(migrationFS, fmt.Sprintf("%s*/*.sql", s.getMigrationBasePath()))
+ if err != nil {
+ return errors.Wrap(err, "failed to read migration files")
+ }
+ sort.Strings(filePaths)
+
+ // Start a transaction to apply migrations atomically
+ tx, err := s.driver.GetDB().Begin()
+ if err != nil {
+ return errors.Wrap(err, "failed to start transaction")
+ }
+ defer tx.Rollback()
+
+ // Use safe version for comparison (handles empty version case)
+ schemaVersionForComparison := getSchemaVersionOrDefault(currentSchemaVersion)
+ if isVersionEmpty(currentSchemaVersion) {
+ slog.Warn("schema version is empty, treating as default for migration comparison",
+ slog.String("defaultVersion", defaultSchemaVersion))
+ }
+
+ slog.Info("start migration",
+ slog.String("currentSchemaVersion", schemaVersionForComparison),
+ slog.String("targetSchemaVersion", targetSchemaVersion))
+
+ migrationsApplied := 0
+ for _, filePath := range filePaths {
+ fileSchemaVersion, err := s.getSchemaVersionOfMigrateScript(filePath)
+ if err != nil {
+ return errors.Wrap(err, "failed to get schema version of migrate script")
+ }
+
+ if shouldApplyMigration(fileSchemaVersion, currentSchemaVersion, targetSchemaVersion) {
+ // Validate migration filename before applying
+ filename := filepath.Base(filePath)
+ if err := validateMigrationFileName(filename); err != nil {
+ slog.Warn("migration file has invalid name but will be applied", slog.String("file", filePath), slog.String("error", err.Error()))
+ }
+
+ slog.Info("applying migration",
+ slog.String("file", filePath),
+ slog.String("version", fileSchemaVersion))
+
+ bytes, err := migrationFS.ReadFile(filePath)
+ if err != nil {
+ return errors.Wrapf(err, "failed to read migration file: %s", filePath)
+ }
+
+ stmt := string(bytes)
+ if err := s.execute(ctx, tx, stmt); err != nil {
+ return errors.Wrapf(err, "failed to execute migration %s: %s", filePath, err)
+ }
+ migrationsApplied++
+ }
+ }
+
+ if err := tx.Commit(); err != nil {
+ return errors.Wrap(err, "failed to commit migration transaction")
+ }
+
+ slog.Info("migration completed", slog.Int("migrationsApplied", migrationsApplied))
+
+ // Update schema version after successful migration
+ if err := s.updateCurrentSchemaVersion(ctx, targetSchemaVersion); err != nil {
+ return errors.Wrap(err, "failed to update current schema version")
+ }
+
+ return nil
+}
+
// preMigrate checks if the database is initialized and applies the latest schema if not.
func (s *Store) preMigrate(ctx context.Context) error {
initialized, err := s.driver.IsInitialized(ctx)
@@ -127,6 +242,7 @@ func (s *Store) preMigrate(ctx context.Context) error {
return errors.Wrap(err, "failed to start transaction")
}
defer tx.Rollback()
+ slog.Info("initializing new database with latest schema", slog.String("file", filePath))
if err := s.execute(ctx, tx, string(bytes)); err != nil {
return errors.Errorf("failed to execute SQL file %s, err %s", filePath, err)
}
@@ -139,12 +255,13 @@ func (s *Store) preMigrate(ctx context.Context) error {
if err != nil {
return errors.Wrap(err, "failed to get current schema version")
}
+ slog.Info("database initialized successfully", slog.String("schemaVersion", schemaVersion))
if err := s.updateCurrentSchemaVersion(ctx, schemaVersion); err != nil {
return errors.Wrap(err, "failed to update current schema version")
}
}
- if s.profile.Mode == "prod" {
+ if s.profile.Mode == modeProd {
if err := s.normalizeMigrationHistoryList(ctx); err != nil {
return errors.Wrap(err, "failed to normalize migration history list")
}
@@ -165,11 +282,11 @@ func (s *Store) getSeedBasePath() string {
// seed seeds the database with initial data.
// It reads all seed files from the embedded filesystem and executes them in order.
-// This is only supported for SQLite databases.
+// This is only supported for SQLite databases and is used in demo mode.
func (s *Store) seed(ctx context.Context) error {
- // Only seed for SQLite.
+ // Only seed for SQLite - other databases should use production data
if s.profile.Driver != "sqlite" {
- slog.Warn("seed is only supported for SQLite")
+ slog.Warn("seed is only supported for SQLite, skipping for other databases")
return nil
}
@@ -265,6 +382,8 @@ func (s *Store) updateCurrentSchemaVersion(ctx context.Context, schemaVersion st
// normalizeMigrationHistoryList normalizes the migration history list.
// It checks the existing migration history and updates it to the latest schema version if necessary.
+// NOTE: This is a transition function for backward compatibility with the deprecated migration_history table.
+// This ensures that old installations (< 0.22) have their migration_history normalized before migrating to workspace_setting.
func (s *Store) normalizeMigrationHistoryList(ctx context.Context) error {
migrationHistoryList, err := s.driver.FindMigrationHistoryList(ctx, &FindMigrationHistory{})
if err != nil {
@@ -281,9 +400,9 @@ func (s *Store) normalizeMigrationHistoryList(ctx context.Context) error {
latestVersion := versions[len(versions)-1]
latestMinorVersion := version.GetMinorVersion(latestVersion)
- // If the latest version is greater than 0.22, return.
- // As of 0.22, the migration history is already normalized.
- if version.IsVersionGreaterThan(latestMinorVersion, "0.22") {
+ // If the latest version is greater than migrationHistoryNormalizedVersion, return.
+ // As of that version, the migration history is already normalized.
+ if version.IsVersionGreaterThan(latestMinorVersion, migrationHistoryNormalizedVersion) {
return nil
}
@@ -318,6 +437,9 @@ func (s *Store) normalizeMigrationHistoryList(ctx context.Context) error {
// migrateSchemaVersionToSetting migrates the schema version from the migration history to the workspace basic setting.
// It retrieves the migration history, sorts the versions, and updates the workspace basic setting if necessary.
+// NOTE: This is a transition function for backward compatibility with the deprecated migration_history table.
+// The migration_history table is deprecated in favor of storing schema version in workspace_setting.
+// This handles upgrades from old installations that only have migration_history but no workspace_setting.
func (s *Store) migrateSchemaVersionToSetting(ctx context.Context) error {
migrationHistoryList, err := s.driver.FindMigrationHistoryList(ctx, &FindMigrationHistory{})
if err != nil {
@@ -337,7 +459,14 @@ func (s *Store) migrateSchemaVersionToSetting(ctx context.Context) error {
if err != nil {
return errors.Wrap(err, "failed to get workspace basic setting")
}
- if version.IsVersionGreaterOrEqualThan(workspaceBasicSetting.SchemaVersion, latestVersion) {
+
+ // If workspace_setting has no schema version (empty), or migration_history has a newer version, update workspace_setting.
+ // This handles upgrades from old installations where schema version was only tracked in migration_history.
+ if isVersionEmpty(workspaceBasicSetting.SchemaVersion) || version.IsVersionGreaterThan(latestVersion, workspaceBasicSetting.SchemaVersion) {
+ slog.Info("migrating schema version from migration_history to workspace_setting",
+ slog.String("from", workspaceBasicSetting.SchemaVersion),
+ slog.String("to", latestVersion),
+ )
if err := s.updateCurrentSchemaVersion(ctx, latestVersion); err != nil {
return errors.Wrap(err, "failed to update current schema version")
}