Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.
Go dockerfossgomarkdownmemomicroblognote-takingnotesopen-sourceown-your-datareactself-hostedsqlitetimeline
refactor: update migration history methods
Steven95de5cc7004a
diff --git a/internal/version/version.go b/internal/version/version.goindex ca64424eb..ba57f3a2f 100644--- a/internal/version/version.go+++ b/internal/version/version.go@@ -21,11 +21,15 @@ func GetCurrentVersion(mode string) string {return Version}+// GetMinorVersion extracts the minor version (e.g., "0.25") from a full version string (e.g., "0.25.1").+// Returns the minor version string or empty string if the version format is invalid.+// Version format should be "major.minor.patch" (e.g., "0.25.1").func GetMinorVersion(version string) string {versionList := strings.Split(version, ".")- if len(versionList) < 3 {+ if len(versionList) < 2 {return ""}+ // Return major.minor only (first two components)return versionList[0] + "." + versionList[1]}diff --git a/store/db/mysql/migration_history.go b/store/db/mysql/migration_history.goindex bc6c89fc4..fbaf5a710 100644--- a/store/db/mysql/migration_history.go+++ b/store/db/mysql/migration_history.go@@ -6,6 +6,8 @@ import ("github.com/usememos/memos/store")+// FindMigrationHistoryList retrieves all migration history records.+// NOTE: This method is deprecated along with the migration_history table.func (d *DB) FindMigrationHistoryList(ctx context.Context, _ *store.FindMigrationHistory) ([]*store.MigrationHistory, error) {query := "SELECT `version`, UNIX_TIMESTAMP(`created_ts`) FROM `migration_history` ORDER BY `created_ts` DESC"rows, err := d.db.QueryContext(ctx, query)@@ -34,6 +36,11 @@ func (d *DB) FindMigrationHistoryList(ctx context.Context, _ *store.FindMigratioreturn list, nil}+// UpsertMigrationHistory inserts or updates a migration history record.+// NOTE: This method is deprecated along with the migration_history table.+// This uses separate INSERT and SELECT queries instead of INSERT...RETURNING because+// MySQL doesn't support RETURNING clause in the same way as PostgreSQL/SQLite.+// This could have race conditions but is acceptable for deprecated transition code.func (d *DB) UpsertMigrationHistory(ctx context.Context, upsert *store.UpsertMigrationHistory) (*store.MigrationHistory, error) {stmt := "INSERT INTO `migration_history` (`version`) VALUES (?) ON DUPLICATE KEY UPDATE `version` = ?"_, err := d.db.ExecContext(ctx, stmt, upsert.Version, upsert.Version)diff --git a/store/db/postgres/migration_history.go b/store/db/postgres/migration_history.goindex 385b898b4..5464dfbca 100644--- a/store/db/postgres/migration_history.go+++ b/store/db/postgres/migration_history.go@@ -6,6 +6,8 @@ import ("github.com/usememos/memos/store")+// FindMigrationHistoryList retrieves all migration history records.+// NOTE: This method is deprecated along with the migration_history table.func (d *DB) FindMigrationHistoryList(ctx context.Context, _ *store.FindMigrationHistory) ([]*store.MigrationHistory, error) {query := "SELECT version, created_ts FROM migration_history ORDER BY created_ts DESC"rows, err := d.db.QueryContext(ctx, query)@@ -34,6 +36,8 @@ func (d *DB) FindMigrationHistoryList(ctx context.Context, _ *store.FindMigratioreturn list, nil}+// UpsertMigrationHistory inserts or updates a migration history record.+// NOTE: This method is deprecated along with the migration_history table.func (d *DB) UpsertMigrationHistory(ctx context.Context, upsert *store.UpsertMigrationHistory) (*store.MigrationHistory, error) {stmt := `INSERT INTO migration_history (diff --git a/store/db/sqlite/migration_history.go b/store/db/sqlite/migration_history.goindex b7ec01b24..3403bcfdb 100644--- a/store/db/sqlite/migration_history.go+++ b/store/db/sqlite/migration_history.go@@ -6,6 +6,8 @@ import ("github.com/usememos/memos/store")+// FindMigrationHistoryList retrieves all migration history records.+// NOTE: This method is deprecated along with the migration_history table.func (d *DB) FindMigrationHistoryList(ctx context.Context, _ *store.FindMigrationHistory) ([]*store.MigrationHistory, error) {query := "SELECT `version`, `created_ts` FROM `migration_history` ORDER BY `created_ts` DESC"rows, err := d.db.QueryContext(ctx, query)@@ -34,6 +36,8 @@ func (d *DB) FindMigrationHistoryList(ctx context.Context, _ *store.FindMigratioreturn list, nil}+// UpsertMigrationHistory inserts or updates a migration history record.+// NOTE: This method is deprecated along with the migration_history table.func (d *DB) UpsertMigrationHistory(ctx context.Context, upsert *store.UpsertMigrationHistory) (*store.MigrationHistory, error) {stmt := `INSERT INTO migration_history (diff --git a/store/driver.go b/store/driver.goindex bc35464f0..cb5d6eb9c 100644--- a/store/driver.go+++ b/store/driver.go@@ -14,6 +14,9 @@ type Driver interface {IsInitialized(ctx context.Context) (bool, error)// MigrationHistory model related methods.+ // NOTE: These methods are deprecated. The migration_history table is no longer used+ // for tracking schema versions. Schema version is now stored in workspace_setting.+ // These methods are kept for backward compatibility to migrate existing installations.FindMigrationHistoryList(ctx context.Context, find *FindMigrationHistory) ([]*MigrationHistory, error)UpsertMigrationHistory(ctx context.Context, upsert *UpsertMigrationHistory) (*MigrationHistory, error)diff --git a/store/migration_history.go b/store/migration_history.goindex 693663bef..f25b31568 100644--- a/store/migration_history.go+++ b/store/migration_history.go@@ -1,13 +1,21 @@package store+// MigrationHistory represents a record in the migration_history table.+// NOTE: The migration_history table is deprecated in favor of storing schema version+// in workspace_setting (BASIC setting). This is kept for backward compatibility only.+// Migration from migration_history to workspace_setting happens automatically during startup.type MigrationHistory struct {Version stringCreatedTs int64}+// UpsertMigrationHistory is used to insert or update a migration history record.+// NOTE: This is deprecated along with the migration_history table.type UpsertMigrationHistory struct {Version string}+// FindMigrationHistory is used to query migration history records.+// NOTE: This is deprecated along with the migration_history table.type FindMigrationHistory struct {}diff --git a/store/migrator.go b/store/migrator.goindex 3e7f10f1d..b2409aee5 100644--- a/store/migrator.go+++ b/store/migrator.go@@ -18,6 +18,30 @@ import (storepb "github.com/usememos/memos/proto/gen/store")+// Migration System Overview:+//+// The migration system handles database schema versioning and upgrades.+// Schema version is stored in workspace_setting (the new system).+// The old migration_history table is deprecated but still supported for backward compatibility.+//+// Migration Flow:+// 1. preMigrate: Check if DB is initialized. If not, apply LATEST.sql+// 2. normalizeMigrationHistoryList: Normalize old migration_history records (for pre-0.22 installations)+// 3. migrateSchemaVersionToSetting: Migrate version from migration_history to workspace_setting+// 4. Migrate (prod mode): Apply incremental migrations from current to target version+// 5. Migrate (demo mode): Seed database with demo data+//+// Version Tracking:+// - New installations: Schema version set in workspace_setting immediately+// - Old installations: Version migrated from migration_history to workspace_setting automatically+// - Empty version: Treated as 0.0.0 and all migrations applied+//+// Migration Files:+// - Location: store/migration/{driver}/{version}/NN__description.sql+// - Naming: NN is zero-padded patch number, description is human-readable+// - Ordering: Files sorted lexicographically and applied in order+// - LATEST.sql: Full schema for new installations (faster than incremental migrations)+//go:embed migrationvar migrationFS embed.FS@@ -31,8 +55,59 @@ const (// LatestSchemaFileName is the name of the latest schema file.// This file is used to apply the latest schema when no migration history is found.LatestSchemaFileName = "LATEST.sql"++ // defaultSchemaVersion is used when schema version is empty or not set.+ // This handles edge cases for old installations without version tracking.+ defaultSchemaVersion = "0.0.0"++ // migrationHistoryNormalizedVersion is the version where migration_history normalization was completed.+ // Before 0.22, migration history had inconsistent versioning that needed normalization.+ migrationHistoryNormalizedVersion = "0.22"++ // Mode constants for profile mode+ modeProd = "prod"+ modeDemo = "demo")+// getSchemaVersionOrDefault returns the schema version or default if empty.+// This ensures safe version comparisons and handles old installations.+func getSchemaVersionOrDefault(schemaVersion string) string {+ if schemaVersion == "" {+ return defaultSchemaVersion+ }+ return schemaVersion+}++// isVersionEmpty checks if the schema version is empty or the default value.+func isVersionEmpty(schemaVersion string) bool {+ return schemaVersion == "" || schemaVersion == defaultSchemaVersion+}++// shouldApplyMigration determines if a migration file should be applied.+// It checks if the file's version is between the current DB version and target version.+func shouldApplyMigration(fileVersion, currentDBVersion, targetVersion string) bool {+ currentDBVersionSafe := getSchemaVersionOrDefault(currentDBVersion)+ return version.IsVersionGreaterThan(fileVersion, currentDBVersionSafe) &&+ version.IsVersionGreaterOrEqualThan(targetVersion, fileVersion)+}++// validateMigrationFileName checks if a migration file follows the expected naming convention.+// Expected format: "NN__description.sql" where NN is a zero-padded number.+func validateMigrationFileName(filename string) error {+ if !strings.Contains(filename, MigrateFileNameSplit) {+ return errors.Errorf("invalid migration filename format (missing %s): %s", MigrateFileNameSplit, filename)+ }+ parts := strings.Split(filename, MigrateFileNameSplit)+ if len(parts) < 2 {+ return errors.Errorf("invalid migration filename format: %s", filename)+ }+ // Check if first part is a number+ if _, err := strconv.Atoi(parts[0]); err != nil {+ return errors.Errorf("migration filename must start with a number: %s", filename)+ }+ return nil+}+// Migrate migrates the database schema to the latest version.// It checks the current schema version and applies any necessary migrations.// It also seeds the database with initial data if in demo mode.@@ -42,7 +117,7 @@ func (s *Store) Migrate(ctx context.Context) error {}switch s.profile.Mode {- case "prod":+ case modeProd:workspaceBasicSetting, err := s.GetWorkspaceBasicSetting(ctx)if err != nil {return errors.Wrap(err, "failed to get workspace basic setting")@@ -51,53 +126,21 @@ func (s *Store) Migrate(ctx context.Context) error {if err != nil {return errors.Wrap(err, "failed to get current schema version")}- if version.IsVersionGreaterThan(workspaceBasicSetting.SchemaVersion, currentSchemaVersion) {+ // Check for downgrade (but skip if schema version is empty - that means fresh/old installation)+ if !isVersionEmpty(workspaceBasicSetting.SchemaVersion) && version.IsVersionGreaterThan(workspaceBasicSetting.SchemaVersion, currentSchemaVersion) {slog.Error("cannot downgrade schema version",slog.String("databaseVersion", workspaceBasicSetting.SchemaVersion),slog.String("currentVersion", currentSchemaVersion),)return errors.Errorf("cannot downgrade schema version from %s to %s", workspaceBasicSetting.SchemaVersion, currentSchemaVersion)}- if version.IsVersionGreaterThan(currentSchemaVersion, workspaceBasicSetting.SchemaVersion) {- filePaths, err := fs.Glob(migrationFS, fmt.Sprintf("%s*/*.sql", s.getMigrationBasePath()))- if err != nil {- return errors.Wrap(err, "failed to read migration files")- }- sort.Strings(filePaths)-- // Start a transaction to apply the latest schema.- tx, err := s.driver.GetDB().Begin()- if err != nil {- return errors.Wrap(err, "failed to start transaction")- }- defer tx.Rollback()-- slog.Info("start migration", slog.String("currentSchemaVersion", workspaceBasicSetting.SchemaVersion), slog.String("targetSchemaVersion", currentSchemaVersion))- for _, filePath := range filePaths {- fileSchemaVersion, err := s.getSchemaVersionOfMigrateScript(filePath)- if err != nil {- return errors.Wrap(err, "failed to get schema version of migrate script")- }- if version.IsVersionGreaterThan(fileSchemaVersion, workspaceBasicSetting.SchemaVersion) && version.IsVersionGreaterOrEqualThan(currentSchemaVersion, fileSchemaVersion) {- bytes, err := migrationFS.ReadFile(filePath)- if err != nil {- return errors.Wrapf(err, "failed to read minor version migration file: %s", filePath)- }- stmt := string(bytes)- if err := s.execute(ctx, tx, stmt); err != nil {- return errors.Wrapf(err, "migrate error: %s", stmt)- }- }- }- if err := tx.Commit(); err != nil {- return errors.Wrap(err, "failed to commit transaction")- }- slog.Info("end migrate")- if err := s.updateCurrentSchemaVersion(ctx, currentSchemaVersion); err != nil {- return errors.Wrap(err, "failed to update current schema version")+ // Apply migrations if needed (including when schema version is empty)+ if isVersionEmpty(workspaceBasicSetting.SchemaVersion) || version.IsVersionGreaterThan(currentSchemaVersion, workspaceBasicSetting.SchemaVersion) {+ if err := s.applyMigrations(ctx, workspaceBasicSetting.SchemaVersion, currentSchemaVersion); err != nil {+ return errors.Wrap(err, "failed to apply migrations")}}- case "demo":+ case modeDemo:// In demo mode, we should seed the database.if err := s.seed(ctx); err != nil {return errors.Wrap(err, "failed to seed")@@ -108,6 +151,78 @@ func (s *Store) Migrate(ctx context.Context) error {return nil}+// applyMigrations applies all necessary migration files between current and target schema versions.+// It runs all migrations in a single transaction for atomicity.+func (s *Store) applyMigrations(ctx context.Context, currentSchemaVersion, targetSchemaVersion string) error {+ filePaths, err := fs.Glob(migrationFS, fmt.Sprintf("%s*/*.sql", s.getMigrationBasePath()))+ if err != nil {+ return errors.Wrap(err, "failed to read migration files")+ }+ sort.Strings(filePaths)++ // Start a transaction to apply migrations atomically+ tx, err := s.driver.GetDB().Begin()+ if err != nil {+ return errors.Wrap(err, "failed to start transaction")+ }+ defer tx.Rollback()++ // Use safe version for comparison (handles empty version case)+ schemaVersionForComparison := getSchemaVersionOrDefault(currentSchemaVersion)+ if isVersionEmpty(currentSchemaVersion) {+ slog.Warn("schema version is empty, treating as default for migration comparison",+ slog.String("defaultVersion", defaultSchemaVersion))+ }++ slog.Info("start migration",+ slog.String("currentSchemaVersion", schemaVersionForComparison),+ slog.String("targetSchemaVersion", targetSchemaVersion))++ migrationsApplied := 0+ for _, filePath := range filePaths {+ fileSchemaVersion, err := s.getSchemaVersionOfMigrateScript(filePath)+ if err != nil {+ return errors.Wrap(err, "failed to get schema version of migrate script")+ }++ if shouldApplyMigration(fileSchemaVersion, currentSchemaVersion, targetSchemaVersion) {+ // Validate migration filename before applying+ filename := filepath.Base(filePath)+ if err := validateMigrationFileName(filename); err != nil {+ slog.Warn("migration file has invalid name but will be applied", slog.String("file", filePath), slog.String("error", err.Error()))+ }++ slog.Info("applying migration",+ slog.String("file", filePath),+ slog.String("version", fileSchemaVersion))++ bytes, err := migrationFS.ReadFile(filePath)+ if err != nil {+ return errors.Wrapf(err, "failed to read migration file: %s", filePath)+ }++ stmt := string(bytes)+ if err := s.execute(ctx, tx, stmt); err != nil {+ return errors.Wrapf(err, "failed to execute migration %s: %s", filePath, err)+ }+ migrationsApplied+++ }+ }++ if err := tx.Commit(); err != nil {+ return errors.Wrap(err, "failed to commit migration transaction")+ }++ slog.Info("migration completed", slog.Int("migrationsApplied", migrationsApplied))++ // Update schema version after successful migration+ if err := s.updateCurrentSchemaVersion(ctx, targetSchemaVersion); err != nil {+ return errors.Wrap(err, "failed to update current schema version")+ }++ return nil+}+// preMigrate checks if the database is initialized and applies the latest schema if not.func (s *Store) preMigrate(ctx context.Context) error {initialized, err := s.driver.IsInitialized(ctx)@@ -127,6 +242,7 @@ func (s *Store) preMigrate(ctx context.Context) error {return errors.Wrap(err, "failed to start transaction")}defer tx.Rollback()+ slog.Info("initializing new database with latest schema", slog.String("file", filePath))if err := s.execute(ctx, tx, string(bytes)); err != nil {return errors.Errorf("failed to execute SQL file %s, err %s", filePath, err)}@@ -139,12 +255,13 @@ func (s *Store) preMigrate(ctx context.Context) error {if err != nil {return errors.Wrap(err, "failed to get current schema version")}+ slog.Info("database initialized successfully", slog.String("schemaVersion", schemaVersion))if err := s.updateCurrentSchemaVersion(ctx, schemaVersion); err != nil {return errors.Wrap(err, "failed to update current schema version")}}- if s.profile.Mode == "prod" {+ if s.profile.Mode == modeProd {if err := s.normalizeMigrationHistoryList(ctx); err != nil {return errors.Wrap(err, "failed to normalize migration history list")}@@ -165,11 +282,11 @@ func (s *Store) getSeedBasePath() string {// seed seeds the database with initial data.// It reads all seed files from the embedded filesystem and executes them in order.-// This is only supported for SQLite databases.+// This is only supported for SQLite databases and is used in demo mode.func (s *Store) seed(ctx context.Context) error {- // Only seed for SQLite.+ // Only seed for SQLite - other databases should use production dataif s.profile.Driver != "sqlite" {- slog.Warn("seed is only supported for SQLite")+ slog.Warn("seed is only supported for SQLite, skipping for other databases")return nil}@@ -265,6 +382,8 @@ func (s *Store) updateCurrentSchemaVersion(ctx context.Context, schemaVersion st// normalizeMigrationHistoryList normalizes the migration history list.// It checks the existing migration history and updates it to the latest schema version if necessary.+// NOTE: This is a transition function for backward compatibility with the deprecated migration_history table.+// This ensures that old installations (< 0.22) have their migration_history normalized before migrating to workspace_setting.func (s *Store) normalizeMigrationHistoryList(ctx context.Context) error {migrationHistoryList, err := s.driver.FindMigrationHistoryList(ctx, &FindMigrationHistory{})if err != nil {@@ -281,9 +400,9 @@ func (s *Store) normalizeMigrationHistoryList(ctx context.Context) error {latestVersion := versions[len(versions)-1]latestMinorVersion := version.GetMinorVersion(latestVersion)- // If the latest version is greater than 0.22, return.- // As of 0.22, the migration history is already normalized.- if version.IsVersionGreaterThan(latestMinorVersion, "0.22") {+ // If the latest version is greater than migrationHistoryNormalizedVersion, return.+ // As of that version, the migration history is already normalized.+ if version.IsVersionGreaterThan(latestMinorVersion, migrationHistoryNormalizedVersion) {return nil}@@ -318,6 +437,9 @@ func (s *Store) normalizeMigrationHistoryList(ctx context.Context) error {// migrateSchemaVersionToSetting migrates the schema version from the migration history to the workspace basic setting.// It retrieves the migration history, sorts the versions, and updates the workspace basic setting if necessary.+// NOTE: This is a transition function for backward compatibility with the deprecated migration_history table.+// The migration_history table is deprecated in favor of storing schema version in workspace_setting.+// This handles upgrades from old installations that only have migration_history but no workspace_setting.func (s *Store) migrateSchemaVersionToSetting(ctx context.Context) error {migrationHistoryList, err := s.driver.FindMigrationHistoryList(ctx, &FindMigrationHistory{})if err != nil {@@ -337,7 +459,14 @@ func (s *Store) migrateSchemaVersionToSetting(ctx context.Context) error {if err != nil {return errors.Wrap(err, "failed to get workspace basic setting")}- if version.IsVersionGreaterOrEqualThan(workspaceBasicSetting.SchemaVersion, latestVersion) {++ // If workspace_setting has no schema version (empty), or migration_history has a newer version, update workspace_setting.+ // This handles upgrades from old installations where schema version was only tracked in migration_history.+ if isVersionEmpty(workspaceBasicSetting.SchemaVersion) || version.IsVersionGreaterThan(latestVersion, workspaceBasicSetting.SchemaVersion) {+ slog.Info("migrating schema version from migration_history to workspace_setting",+ slog.String("from", workspaceBasicSetting.SchemaVersion),+ slog.String("to", latestVersion),+ )if err := s.updateCurrentSchemaVersion(ctx, latestVersion); err != nil {return errors.Wrap(err, "failed to update current schema version")}