Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.
Go dockerfossgomarkdownmemomicroblognote-takingnotesopen-sourceown-your-datareactself-hostedsqlitetimeline
chore: greatly speed up migrator and lower memory usage (#2874)
Lincoln Nogueira279cba0e6b3e
diff --git a/store/migrator.go b/store/migrator.goindex a91131465..3065d94b2 100644--- a/store/migrator.go+++ b/store/migrator.go@@ -2,11 +2,11 @@ package storeimport ("context"+ "database/sql""fmt""os"- "time"-"strings"+ "time""github.com/lithammer/shortuuid/v4""github.com/pkg/errors"@@ -16,88 +16,242 @@ import (// MigrateResourceInternalPath migrates resource internal path from absolute path to relative path.func (s *Store) MigrateResourceInternalPath(ctx context.Context) error {- resources, err := s.ListResources(ctx, &FindResource{})- if err != nil {+ normalizedDataPath := strings.ReplaceAll(s.Profile.Data, `\`, "/")+ normalizedDataPath = strings.ReplaceAll(normalizedDataPath, `//`, "/")++ db := s.driver.GetDB()+ checkStmt := `+ SELECT id FROM resource+ WHERE+ internal_path LIKE ?+ OR internal_path LIKE ?+ OR internal_path LIKE ?+ LIMIT 1`+ rows := 0+ res := db.QueryRowContext(ctx, checkStmt, fmt.Sprintf("%s%%", s.Profile.Data), fmt.Sprintf("%s%%", normalizedDataPath), "%\\%")+ if err := res.Scan(&rows); err != nil {+ if rows == 0 || err == sql.ErrNoRows {+ log.Debug("Resource internal path migration is not required.")+ return nil+ }+ return errors.Wrap(err, "failed to check resource internal_path")+ }++ log.Info("Migrating resource internal paths. This may take a while.")++ listResourcesStmt := `+ SELECT id, internal_path FROM resource+ WHERE+ internal_path IS NOT ''+ AND internal_path LIKE ?+ OR internal_path LIKE ?+ `+ resources, err := db.QueryContext(ctx, listResourcesStmt, fmt.Sprintf("%s%%", s.Profile.Data), fmt.Sprintf("%s%%", normalizedDataPath))+ if err != nil || resources.Err() != nil {return errors.Wrap(err, "failed to list resources")}+ defer resources.Close()++ tx, err := db.BeginTx(ctx, nil)+ if err != nil {+ return errors.Wrap(err, "failed to start transaction")+ }+ defer tx.Rollback()++ updateStmt := `UPDATE resource SET internal_path = ? WHERE id = ?`+ preparedResourceUpdate, err := tx.PrepareContext(ctx, updateStmt)+ if err != nil {+ return errors.Wrap(err, "failed to prepare update statement")+ }+ defer preparedResourceUpdate.Close()- dataPath := strings.ReplaceAll(s.Profile.Data, `\`, "/")migrateStartTime := time.Now()migratedCount := 0- for _, resource := range resources {+ for resources.Next() {+ resource := Resource{}+ if err := resources.Scan(&resource.ID, &resource.InternalPath); err != nil {+ return errors.Wrap(err, "failed to parse resource data")+ }+if resource.InternalPath == "" {continue}internalPath := strings.ReplaceAll(resource.InternalPath, `\`, "/")- if !strings.HasPrefix(internalPath, dataPath) {+ if !strings.HasPrefix(internalPath, normalizedDataPath) {continue}- internalPath = strings.TrimPrefix(internalPath, dataPath)+ internalPath = strings.TrimPrefix(internalPath, normalizedDataPath)for os.IsPathSeparator(internalPath[0]) {internalPath = internalPath[1:]}- _, err := s.UpdateResource(ctx, &UpdateResource{- ID: resource.ID,- InternalPath: &internalPath,- })+ _, err := preparedResourceUpdate.ExecContext(ctx, internalPath, resource.ID)if err != nil {- return errors.Wrap(err, "failed to update local resource path")+ return errors.Wrap(err, "failed to update resource internal_path")}++ if migratedCount%500 == 0 {+ log.Info(fmt.Sprintf("[Running] Migrated %d local resource paths", migratedCount))+ }+migratedCount++}+ if err := tx.Commit(); err != nil {+ return errors.Wrap(err, "failed to commit transaction")+ }- if migratedCount > 0 && s.Profile.Mode == "prod" {- log.Info(fmt.Sprintf("migrated %d local resource paths in %s", migratedCount, time.Since(migrateStartTime)))+ if migratedCount > 0 {+ log.Info(fmt.Sprintf("Migrated %d local resource paths in %s", migratedCount, time.Since(migrateStartTime)))}return nil}// MigrateResourceName migrates resource name from other format to short UUID.func (s *Store) MigrateResourceName(ctx context.Context) error {- memos, err := s.ListMemos(ctx, &FindMemo{- ExcludeContent: true,- })+ db := s.driver.GetDB()++ checkStmt := `+ SELECT resource_name FROM resource+ WHERE+ resource_name = ''+ LIMIT 1`+ rows := 0+ res := db.QueryRowContext(ctx, checkStmt)+ if err := res.Scan(&rows); err != nil {+ if rows == 0 || err == sql.ErrNoRows {+ log.Debug("Resource migration to UUIDs is not required.")+ return nil+ }+ return errors.Wrap(err, "failed to check resource.resource_name")+ }++ log.Info("Migrating resource IDs to UUIDs. This may take a while.")++ listResourceStmt := "SELECT `id`, `resource_name` FROM `resource` WHERE `resource_name` = ''"+ resources, err := db.QueryContext(ctx, listResourceStmt)+ if err != nil || resources.Err() != nil {+ return errors.Wrap(err, "failed to list resources")+ }+ defer resources.Close()++ tx, err := db.BeginTx(ctx, nil)if err != nil {- return errors.Wrap(err, "failed to list memos")+ return errors.Wrap(err, "failed to start transaction")}- for _, memo := range memos {- if checkResourceName(memo.ResourceName) {+ defer tx.Rollback()++ updateResourceStmt := "UPDATE `resource` SET `resource_name` = ? WHERE `id` = ?"+ preparedResourceUpdate, err := tx.PrepareContext(ctx, updateResourceStmt)+ if err != nil {+ return errors.Wrap(err, "failed to prepare update statement")+ }+ defer preparedResourceUpdate.Close()++ migrateStartTime := time.Now()+ migratedCount := 0+ for resources.Next() {+ resource := Resource{}+ if err := resources.Scan(&resource.ID, &resource.ResourceName); err != nil {+ return errors.Wrap(err, "failed to parse resource data")+ }++ if checkResourceName(resource.ResourceName) {continue}resourceName := shortuuid.New()- err := s.UpdateMemo(ctx, &UpdateMemo{- ID: memo.ID,- ResourceName: &resourceName,- })- if err != nil {- return errors.Wrap(err, "failed to update memo")+ if _, err := preparedResourceUpdate.ExecContext(ctx, resourceName, resource.ID); err != nil {+ return errors.Wrap(err, "failed to update resource")+ }++ if migratedCount%500 == 0 {+ log.Info(fmt.Sprintf("[Running] Migrated %d local resources IDs", migratedCount))+ }+ migratedCount+++ }+ if err := tx.Commit(); err != nil {+ return errors.Wrap(err, "failed to commit transaction")+ }++ if migratedCount > 0 {+ log.Info(fmt.Sprintf("Migrated %d resource IDs to UUIDs in %s", migratedCount, time.Since(migrateStartTime)))+ }+ return nil+}++// MigrateResourceName migrates memo name from other format to short UUID.+func (s *Store) MigrateMemoName(ctx context.Context) error {+ db := s.driver.GetDB()++ checkStmt := `+ SELECT resource_name FROM memo+ WHERE+ resource_name = ''+ LIMIT 1`+ rows := 0+ res := db.QueryRowContext(ctx, checkStmt)+ if err := res.Scan(&rows); err != nil {+ if rows == 0 || err == sql.ErrNoRows {+ log.Debug("Memo migration to UUIDs is not required.")+ return nil}+ return errors.Wrap(err, "failed to check memo.resource_name")+ }++ log.Info("Migrating memo ids to uuids. This may take a while.")++ listMemoStmt := "SELECT `id`, `resource_name` FROM `memo` WHERE `resource_name` = ''"+ memos, err := db.QueryContext(ctx, listMemoStmt)+ if err != nil || memos.Err() != nil {+ return errors.Wrap(err, "failed to list memos")}+ defer memos.Close()- resources, err := s.ListResources(ctx, &FindResource{})+ tx, err := db.BeginTx(ctx, nil)if err != nil {- return errors.Wrap(err, "failed to list resources")+ return errors.Wrap(err, "failed to start transaction")}- for _, resource := range resources {- if checkResourceName(resource.ResourceName) {+ defer tx.Rollback()++ updateMemoStmt := "UPDATE `memo` SET `resource_name` = ? WHERE `id` = ?"+ preparedMemoUpdate, err := tx.PrepareContext(ctx, updateMemoStmt)+ if err != nil {+ return errors.Wrap(err, "failed to prepare update statement")+ }+ defer preparedMemoUpdate.Close()++ migrateStartTime := time.Now()+ migratedCount := 0+ for memos.Next() {+ memo := Memo{}+ if err := memos.Scan(&memo.ID, &memo.ResourceName); err != nil {+ return errors.Wrap(err, "failed to parse memo data")+ }++ if checkResourceName(memo.ResourceName) {continue}resourceName := shortuuid.New()- _, err := s.UpdateResource(ctx, &UpdateResource{- ID: resource.ID,- ResourceName: &resourceName,- })- if err != nil {- return errors.Wrap(err, "failed to update resource")+ if _, err := preparedMemoUpdate.ExecContext(ctx, resourceName, memo.ID); err != nil {+ return errors.Wrap(err, "failed to update memo")}++ if migratedCount%500 == 0 {+ log.Info(fmt.Sprintf("[Running] Migrated %d local resources IDs", migratedCount))+ }+ migratedCount+++ }+ if err := tx.Commit(); err != nil {+ return errors.Wrap(err, "failed to commit transaction")}+ if migratedCount > 0 {+ log.Info(fmt.Sprintf("Migrated %d memo ids to uuids in %s", migratedCount, time.Since(migrateStartTime)))+ }return nil}diff --git a/store/store.go b/store/store.goindex ddd3a903f..dd3450b87 100644--- a/store/store.go+++ b/store/store.go@@ -32,6 +32,9 @@ func (s *Store) MigrateManually(ctx context.Context) error {if err := s.MigrateResourceName(ctx); err != nil {return err}+ if err := s.MigrateMemoName(ctx); err != nil {+ return err+ }return nil}