Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.
Go dockerfossgomarkdownmemomicroblognote-takingnotesopen-sourceown-your-datareactself-hostedsqlitetimeline
chore: store vacuum and clean (#2293)
Athurg Goothca98367a0acd
diff --git a/cmd/memos.go b/cmd/memos.goindex 225470fef..0f0e06ad0 100644--- a/cmd/memos.go+++ b/cmd/memos.go@@ -56,8 +56,7 @@ var (}driver := sqlite.NewDriver(db.DBInstance)-- store := store.New(db.DBInstance, driver, profile)+ store := store.New(driver, profile)s, err := server.NewServer(ctx, profile, store)if err != nil {cancel()diff --git a/cmd/mvrss.go b/cmd/mvrss.goindex aec8385e7..310cc74ef 100644--- a/cmd/mvrss.go+++ b/cmd/mvrss.go@@ -51,8 +51,7 @@ var (}driver := sqlite.NewDriver(db.DBInstance)-- s := store.New(db.DBInstance, driver, profile)+ s := store.New(driver, profile)resources, err := s.ListResources(ctx, &store.FindResource{})if err != nil {fmt.Printf("failed to list resources, error: %+v\n", err)diff --git a/cmd/setup.go b/cmd/setup.goindex c491974c7..c1a36bec9 100644--- a/cmd/setup.go+++ b/cmd/setup.go@@ -48,8 +48,7 @@ var (}driver := sqlite.NewDriver(db.DBInstance)-- store := store.New(db.DBInstance, driver, profile)+ store := store.New(driver, profile)if err := ExecuteSetup(ctx, store, hostUsername, hostPassword); err != nil {fmt.Printf("failed to setup, error: %+v\n", err)returndiff --git a/server/server.go b/server/server.goindex f074a8efc..616860b4d 100644--- a/server/server.go+++ b/server/server.go@@ -162,7 +162,7 @@ func (s *Server) Shutdown(ctx context.Context) {}// Close database connection- if err := s.Store.GetDB().Close(); err != nil {+ if err := s.Store.Close(); err != nil {fmt.Printf("failed to close database, error: %v\n", err)}diff --git a/store/driver.go b/store/driver.goindex e10ae63a7..6cde96b19 100644--- a/store/driver.go+++ b/store/driver.go@@ -7,6 +7,10 @@ import ()type Driver interface {+ Vacuum(ctx context.Context) error+ BackupTo(ctx context.Context, filename string) error+ Close() error+CreateActivity(ctx context.Context, create *Activity) (*Activity, error)CreateResource(ctx context.Context, create *Resource) (*Resource, error)diff --git a/store/memo.go b/store/memo.goindex 3dab08027..a37d61b9e 100644--- a/store/memo.go+++ b/store/memo.go@@ -2,7 +2,6 @@ package storeimport ("context"- "database/sql")// Visibility is the type of a visibility.@@ -107,35 +106,9 @@ func (s *Store) UpdateMemo(ctx context.Context, update *UpdateMemo) error {}func (s *Store) DeleteMemo(ctx context.Context, delete *DeleteMemo) error {- if err := s.driver.DeleteMemo(ctx, delete); err != nil {- return err- }- if err := s.Vacuum(ctx); err != nil {- // Prevent linter warning.- return err- }- return nil+ return s.driver.DeleteMemo(ctx, delete)}func (s *Store) FindMemosVisibilityList(ctx context.Context, memoIDs []int32) ([]Visibility, error) {return s.driver.FindMemosVisibilityList(ctx, memoIDs)}--func vacuumMemo(ctx context.Context, tx *sql.Tx) error {- stmt := `- DELETE FROM- memo- WHERE- creator_id NOT IN (- SELECT- id- FROM- user- )`- _, err := tx.ExecContext(ctx, stmt)- if err != nil {- return err- }-- return nil-}diff --git a/store/memo_organizer.go b/store/memo_organizer.goindex 84a843c6c..5c153adae 100644--- a/store/memo_organizer.go+++ b/store/memo_organizer.go@@ -2,7 +2,6 @@ package storeimport ("context"- "database/sql")type MemoOrganizer struct {@@ -32,28 +31,3 @@ func (s *Store) GetMemoOrganizer(ctx context.Context, find *FindMemoOrganizer) (func (s *Store) DeleteMemoOrganizer(ctx context.Context, delete *DeleteMemoOrganizer) error {return s.driver.DeleteMemoOrganizer(ctx, delete)}--func vacuumMemoOrganizer(ctx context.Context, tx *sql.Tx) error {- stmt := `- DELETE FROM- memo_organizer- WHERE- memo_id NOT IN (- SELECT- id- FROM- memo- )- OR user_id NOT IN (- SELECT- id- FROM- user- )`- _, err := tx.ExecContext(ctx, stmt)- if err != nil {- return err- }-- return nil-}diff --git a/store/memo_relation.go b/store/memo_relation.goindex 351062520..5555c26ca 100644--- a/store/memo_relation.go+++ b/store/memo_relation.go@@ -2,7 +2,6 @@ package storeimport ("context"- "database/sql")type MemoRelationType string@@ -54,13 +53,3 @@ func (s *Store) GetMemoRelation(ctx context.Context, find *FindMemoRelation) (*Mfunc (s *Store) DeleteMemoRelation(ctx context.Context, delete *DeleteMemoRelation) error {return s.driver.DeleteMemoRelation(ctx, delete)}--func vacuumMemoRelations(ctx context.Context, tx *sql.Tx) error {- if _, err := tx.ExecContext(ctx, `- DELETE FROM memo_relation- WHERE memo_id NOT IN (SELECT id FROM memo) OR related_memo_id NOT IN (SELECT id FROM memo)- `); err != nil {- return err- }- return nil-}diff --git a/store/resource.go b/store/resource.goindex 8e66df93f..87c149a16 100644--- a/store/resource.go+++ b/store/resource.go@@ -2,7 +2,6 @@ package storeimport ("context"- "database/sql")type Resource struct {@@ -75,33 +74,5 @@ func (s *Store) UpdateResource(ctx context.Context, update *UpdateResource) (*Re}func (s *Store) DeleteResource(ctx context.Context, delete *DeleteResource) error {- err := s.driver.DeleteResource(ctx, delete)- if err != nil {- return err- }-- if err := s.Vacuum(ctx); err != nil {- // Prevent linter warning.- return err- }- return nil-}--func vacuumResource(ctx context.Context, tx *sql.Tx) error {- stmt := `- DELETE FROM- resource- WHERE- creator_id NOT IN (- SELECT- id- FROM- user- )`- _, err := tx.ExecContext(ctx, stmt)- if err != nil {- return err- }-- return nil+ return s.driver.DeleteResource(ctx, delete)}diff --git a/store/sqlite/memo.go b/store/sqlite/memo.goindex f408811d8..9459c3e19 100644--- a/store/sqlite/memo.go+++ b/store/sqlite/memo.go@@ -234,6 +234,10 @@ func (d *Driver) DeleteMemo(ctx context.Context, delete *store.DeleteMemo) errorreturn err}+ if err := d.Vacuum(ctx); err != nil {+ // Prevent linter warning.+ return err+ }return nil}@@ -268,3 +272,22 @@ func (d *Driver) FindMemosVisibilityList(ctx context.Context, memoIDs []int32) (return visibilityList, nil}++func vacuumMemo(ctx context.Context, tx *sql.Tx) error {+ stmt := `+ DELETE FROM+ memo+ WHERE+ creator_id NOT IN (+ SELECT+ id+ FROM+ user+ )`+ _, err := tx.ExecContext(ctx, stmt)+ if err != nil {+ return err+ }++ return nil+}diff --git a/store/sqlite/memo_organizer.go b/store/sqlite/memo_organizer.goindex f46421f9b..245e88a59 100644--- a/store/sqlite/memo_organizer.go+++ b/store/sqlite/memo_organizer.go@@ -2,6 +2,7 @@ package sqliteimport ("context"+ "database/sql""fmt""strings"@@ -80,3 +81,28 @@ func (d *Driver) DeleteMemoOrganizer(ctx context.Context, delete *store.DeleteMe}return nil}++func vacuumMemoOrganizer(ctx context.Context, tx *sql.Tx) error {+ stmt := `+ DELETE FROM+ memo_organizer+ WHERE+ memo_id NOT IN (+ SELECT+ id+ FROM+ memo+ )+ OR user_id NOT IN (+ SELECT+ id+ FROM+ user+ )`+ _, err := tx.ExecContext(ctx, stmt)+ if err != nil {+ return err+ }++ return nil+}diff --git a/store/sqlite/memo_relation.go b/store/sqlite/memo_relation.goindex 86f6de018..c0fcdd2e2 100644--- a/store/sqlite/memo_relation.go+++ b/store/sqlite/memo_relation.go@@ -2,6 +2,7 @@ package sqliteimport ("context"+ "database/sql""strings""github.com/usememos/memos/store"@@ -104,3 +105,13 @@ func (d *Driver) DeleteMemoRelation(ctx context.Context, delete *store.DeleteMem}return nil}++func vacuumMemoRelations(ctx context.Context, tx *sql.Tx) error {+ if _, err := tx.ExecContext(ctx, `+ DELETE FROM memo_relation+ WHERE memo_id NOT IN (SELECT id FROM memo) OR related_memo_id NOT IN (SELECT id FROM memo)+ `); err != nil {+ return err+ }+ return nil+}diff --git a/store/sqlite/resource.go b/store/sqlite/resource.goindex b71802a77..9095e3446 100644--- a/store/sqlite/resource.go+++ b/store/sqlite/resource.go@@ -181,5 +181,29 @@ func (d *Driver) DeleteResource(ctx context.Context, delete *store.DeleteResourcreturn err}+ if err := d.Vacuum(ctx); err != nil {+ // Prevent linter warning.+ return err+ }++ return nil+}++func vacuumResource(ctx context.Context, tx *sql.Tx) error {+ stmt := `+ DELETE FROM+ resource+ WHERE+ creator_id NOT IN (+ SELECT+ id+ FROM+ user+ )`+ _, err := tx.ExecContext(ctx, stmt)+ if err != nil {+ return err+ }+return nil}diff --git a/store/sqlite/sqlite.go b/store/sqlite/sqlite.goindex 69d5c3548..b9e53fdd0 100644--- a/store/sqlite/sqlite.go+++ b/store/sqlite/sqlite.go@@ -1,8 +1,12 @@package sqliteimport (+ "context""database/sql"+ "github.com/pkg/errors"+ "modernc.org/sqlite"+"github.com/usememos/memos/store")@@ -15,3 +19,91 @@ func NewDriver(db *sql.DB) store.Driver {db: db,}}++func (d *Driver) Vacuum(ctx context.Context) error {+ tx, err := d.db.BeginTx(ctx, nil)+ if err != nil {+ return err+ }+ defer tx.Rollback()++ if err := vacuumImpl(ctx, tx); err != nil {+ return err+ }++ if err := tx.Commit(); err != nil {+ return err+ }++ // Vacuum sqlite database file size after deleting resource.+ if _, err := d.db.Exec("VACUUM"); err != nil {+ return err+ }++ return nil+}++func vacuumImpl(ctx context.Context, tx *sql.Tx) error {+ if err := vacuumMemo(ctx, tx); err != nil {+ return err+ }+ if err := vacuumResource(ctx, tx); err != nil {+ return err+ }+ if err := vacuumUserSetting(ctx, tx); err != nil {+ return err+ }+ if err := vacuumMemoOrganizer(ctx, tx); err != nil {+ return err+ }+ if err := vacuumMemoRelations(ctx, tx); err != nil {+ return err+ }+ if err := vacuumTag(ctx, tx); err != nil {+ // Prevent revive warning.+ return err+ }++ return nil+}++func (d *Driver) BackupTo(ctx context.Context, filename string) error {+ conn, err := d.db.Conn(ctx)+ if err != nil {+ return errors.Errorf("fail to get conn %s", err)+ }+ defer conn.Close()++ err = conn.Raw(func(driverConn any) error {+ type backuper interface {+ NewBackup(string) (*sqlite.Backup, error)+ }+ backupConn, ok := driverConn.(backuper)+ if !ok {+ return errors.Errorf("db connection is not a sqlite backuper")+ }++ bck, err := backupConn.NewBackup(filename)+ if err != nil {+ return errors.Errorf("fail to create sqlite backup %s", err)+ }++ for more := true; more; {+ more, err = bck.Step(-1)+ if err != nil {+ return errors.Errorf("fail to execute sqlite backup %s", err)+ }+ }++ return bck.Finish()+ })+ if err != nil {+ return errors.Errorf("fail to backup %s", err)+ }++ return nil+}++func (d *Driver) Close() error {+ return d.db.Close()+}diff --git a/store/sqlite/tag.go b/store/sqlite/tag.goindex 73400e91e..0132b3f41 100644--- a/store/sqlite/tag.go+++ b/store/sqlite/tag.go@@ -2,6 +2,7 @@ package sqliteimport ("context"+ "database/sql""strings""github.com/usememos/memos/store"@@ -73,3 +74,22 @@ func (d *Driver) DeleteTag(ctx context.Context, delete *store.DeleteTag) error {}return nil}++func vacuumTag(ctx context.Context, tx *sql.Tx) error {+ stmt := `+ DELETE FROM+ tag+ WHERE+ creator_id NOT IN (+ SELECT+ id+ FROM+ user+ )`+ _, err := tx.ExecContext(ctx, stmt)+ if err != nil {+ return err+ }++ return nil+}diff --git a/store/sqlite/user.go b/store/sqlite/user.goindex e0d4c7b0f..1e97d9b50 100644--- a/store/sqlite/user.go+++ b/store/sqlite/user.go@@ -168,5 +168,11 @@ func (d *Driver) DeleteUser(ctx context.Context, delete *store.DeleteUser) errorif _, err := result.RowsAffected(); err != nil {return err}++ if err := d.Vacuum(ctx); err != nil {+ // Prevent linter warning.+ return err+ }+return nil}diff --git a/store/sqlite/user_setting.go b/store/sqlite/user_setting.goindex 75c7c3fec..f1916ec7f 100644--- a/store/sqlite/user_setting.go+++ b/store/sqlite/user_setting.go@@ -2,6 +2,7 @@ package sqliteimport ("context"+ "database/sql""errors""strings"@@ -153,3 +154,22 @@ func (d *Driver) ListUserSettingsV1(ctx context.Context, find *store.FindUserSetreturn userSettingList, nil}++func vacuumUserSetting(ctx context.Context, tx *sql.Tx) error {+ stmt := `+ DELETE FROM+ user_setting+ WHERE+ user_id NOT IN (+ SELECT+ id+ FROM+ user+ )`+ _, err := tx.ExecContext(ctx, stmt)+ if err != nil {+ return err+ }++ return nil+}diff --git a/store/store.go b/store/store.goindex d4287b252..91ef61757 100644--- a/store/store.go+++ b/store/store.go@@ -2,20 +2,14 @@ package storeimport ("context"- "database/sql""sync"- "modernc.org/sqlite"-- "github.com/pkg/errors"-"github.com/usememos/memos/server/profile")// Store provides database access to all raw objects.type Store struct {Profile *profile.Profile- db *sql.DBdriver DriversystemSettingCache sync.Map // map[string]*SystemSettinguserCache sync.Map // map[int]*User@@ -24,98 +18,21 @@ type Store struct {}// New creates a new instance of Store.-func New(db *sql.DB, driver Driver, profile *profile.Profile) *Store {+func New(driver Driver, profile *profile.Profile) *Store {return &Store{Profile: profile,- db: db,driver: driver,}}-func (s *Store) GetDB() *sql.DB {- return s.db-}-func (s *Store) BackupTo(ctx context.Context, filename string) error {- conn, err := s.db.Conn(ctx)- if err != nil {- return errors.Errorf("fail to get conn %s", err)- }- defer conn.Close()-- err = conn.Raw(func(driverConn any) error {- type backuper interface {- NewBackup(string) (*sqlite.Backup, error)- }- backupConn, ok := driverConn.(backuper)- if !ok {- return errors.Errorf("db connection is not a sqlite backuper")- }-- bck, err := backupConn.NewBackup(filename)- if err != nil {- return errors.Errorf("fail to create sqlite backup %s", err)- }-- for more := true; more; {- more, err = bck.Step(-1)- if err != nil {- return errors.Errorf("fail to execute sqlite backup %s", err)- }- }-- return bck.Finish()- })- if err != nil {- return errors.Errorf("fail to backup %s", err)- }-- return nil+ return s.driver.BackupTo(ctx, filename)}func (s *Store) Vacuum(ctx context.Context) error {- tx, err := s.db.BeginTx(ctx, nil)- if err != nil {- return err- }- defer tx.Rollback()-- if err := s.vacuumImpl(ctx, tx); err != nil {- return err- }-- if err := tx.Commit(); err != nil {- return err- }-- // Vacuum sqlite database file size after deleting resource.- if _, err := s.db.Exec("VACUUM"); err != nil {- return err- }-- return nil+ return s.driver.Vacuum(ctx)}-func (*Store) vacuumImpl(ctx context.Context, tx *sql.Tx) error {- if err := vacuumMemo(ctx, tx); err != nil {- return err- }- if err := vacuumResource(ctx, tx); err != nil {- return err- }- if err := vacuumUserSetting(ctx, tx); err != nil {- return err- }- if err := vacuumMemoOrganizer(ctx, tx); err != nil {- return err- }- if err := vacuumMemoRelations(ctx, tx); err != nil {- return err- }- if err := vacuumTag(ctx, tx); err != nil {- // Prevent revive warning.- return err- }-- return nil+func (s *Store) Close() error {+ return s.driver.Close()}diff --git a/store/tag.go b/store/tag.goindex afea07399..d5c38f6bf 100644--- a/store/tag.go+++ b/store/tag.go@@ -2,7 +2,6 @@ package storeimport ("context"- "database/sql")type Tag struct {@@ -30,22 +29,3 @@ func (s *Store) ListTags(ctx context.Context, find *FindTag) ([]*Tag, error) {func (s *Store) DeleteTag(ctx context.Context, delete *DeleteTag) error {return s.driver.DeleteTag(ctx, delete)}--func vacuumTag(ctx context.Context, tx *sql.Tx) error {- stmt := `- DELETE FROM- tag- WHERE- creator_id NOT IN (- SELECT- id- FROM- user- )`- _, err := tx.ExecContext(ctx, stmt)- if err != nil {- return err- }-- return nil-}diff --git a/store/user.go b/store/user.goindex 17948fa7a..6afd9dfb9 100644--- a/store/user.go+++ b/store/user.go@@ -130,10 +130,6 @@ func (s *Store) DeleteUser(ctx context.Context, delete *DeleteUser) error {return err}- if err := s.Vacuum(ctx); err != nil {- // Prevent linter warning.- return err- }s.userCache.Delete(delete.ID)return nil}diff --git a/store/user_setting.go b/store/user_setting.goindex 7726f7a83..98c4234c0 100644--- a/store/user_setting.go+++ b/store/user_setting.go@@ -2,7 +2,6 @@ package storeimport ("context"- "database/sql"storepb "github.com/usememos/memos/proto/gen/store")@@ -125,22 +124,3 @@ func (s *Store) GetUserAccessTokens(ctx context.Context, userID int32) ([]*storeaccessTokensUserSetting := userSetting.GetAccessTokens()return accessTokensUserSetting.AccessTokens, nil}--func vacuumUserSetting(ctx context.Context, tx *sql.Tx) error {- stmt := `- DELETE FROM- user_setting- WHERE- user_id NOT IN (- SELECT- id- FROM- user- )`- _, err := tx.ExecContext(ctx, stmt)- if err != nil {- return err- }-- return nil-}diff --git a/test/server/server.go b/test/server/server.goindex 5bf4daeb9..1bf400d13 100644--- a/test/server/server.go+++ b/test/server/server.go@@ -41,8 +41,7 @@ func NewTestingServer(ctx context.Context, t *testing.T) (*TestingServer, error)}driver := sqlite.NewDriver(db.DBInstance)-- store := store.New(db.DBInstance, driver, profile)+ store := store.New(driver, profile)server, err := server.NewServer(ctx, profile, store)if err != nil {return nil, errors.Wrap(err, "failed to create server")diff --git a/test/store/store.go b/test/store/store.goindex fdae9d888..d530fe685 100644--- a/test/store/store.go+++ b/test/store/store.go@@ -25,7 +25,6 @@ func NewTestingStore(ctx context.Context, t *testing.T) *store.Store {}driver := sqlite.NewDriver(db.DBInstance)-- store := store.New(db.DBInstance, driver, profile)+ store := store.New(driver, profile)return store}