Skip to content

armgit / memos

Public Imported from GitHub
0

Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.

chore: store vacuum and clean (#2293)

Athurg Goothca98367a0acd
diff --git a/cmd/memos.go b/cmd/memos.go
index 225470fef..0f0e06ad0 100644
--- a/cmd/memos.go
+++ b/cmd/memos.go
@@ -56,8 +56,7 @@ var (
}
driver := sqlite.NewDriver(db.DBInstance)
-
- store := store.New(db.DBInstance, driver, profile)
+ store := store.New(driver, profile)
s, err := server.NewServer(ctx, profile, store)
if err != nil {
cancel()
diff --git a/cmd/mvrss.go b/cmd/mvrss.go
index aec8385e7..310cc74ef 100644
--- a/cmd/mvrss.go
+++ b/cmd/mvrss.go
@@ -51,8 +51,7 @@ var (
}
driver := sqlite.NewDriver(db.DBInstance)
-
- s := store.New(db.DBInstance, driver, profile)
+ s := store.New(driver, profile)
resources, err := s.ListResources(ctx, &store.FindResource{})
if err != nil {
fmt.Printf("failed to list resources, error: %+v\n", err)
diff --git a/cmd/setup.go b/cmd/setup.go
index c491974c7..c1a36bec9 100644
--- a/cmd/setup.go
+++ b/cmd/setup.go
@@ -48,8 +48,7 @@ var (
}
driver := sqlite.NewDriver(db.DBInstance)
-
- store := store.New(db.DBInstance, driver, profile)
+ store := store.New(driver, profile)
if err := ExecuteSetup(ctx, store, hostUsername, hostPassword); err != nil {
fmt.Printf("failed to setup, error: %+v\n", err)
return
diff --git a/server/server.go b/server/server.go
index f074a8efc..616860b4d 100644
--- a/server/server.go
+++ b/server/server.go
@@ -162,7 +162,7 @@ func (s *Server) Shutdown(ctx context.Context) {
}
// Close database connection
- if err := s.Store.GetDB().Close(); err != nil {
+ if err := s.Store.Close(); err != nil {
fmt.Printf("failed to close database, error: %v\n", err)
}
diff --git a/store/driver.go b/store/driver.go
index e10ae63a7..6cde96b19 100644
--- a/store/driver.go
+++ b/store/driver.go
@@ -7,6 +7,10 @@ import (
)
type Driver interface {
+ Vacuum(ctx context.Context) error
+ BackupTo(ctx context.Context, filename string) error
+ Close() error
+
CreateActivity(ctx context.Context, create *Activity) (*Activity, error)
CreateResource(ctx context.Context, create *Resource) (*Resource, error)
diff --git a/store/memo.go b/store/memo.go
index 3dab08027..a37d61b9e 100644
--- a/store/memo.go
+++ b/store/memo.go
@@ -2,7 +2,6 @@ package store
import (
"context"
- "database/sql"
)
// Visibility is the type of a visibility.
@@ -107,35 +106,9 @@ func (s *Store) UpdateMemo(ctx context.Context, update *UpdateMemo) error {
}
func (s *Store) DeleteMemo(ctx context.Context, delete *DeleteMemo) error {
- if err := s.driver.DeleteMemo(ctx, delete); err != nil {
- return err
- }
- if err := s.Vacuum(ctx); err != nil {
- // Prevent linter warning.
- return err
- }
- return nil
+ return s.driver.DeleteMemo(ctx, delete)
}
func (s *Store) FindMemosVisibilityList(ctx context.Context, memoIDs []int32) ([]Visibility, error) {
return s.driver.FindMemosVisibilityList(ctx, memoIDs)
}
-
-func vacuumMemo(ctx context.Context, tx *sql.Tx) error {
- stmt := `
- DELETE FROM
- memo
- WHERE
- creator_id NOT IN (
- SELECT
- id
- FROM
- user
- )`
- _, err := tx.ExecContext(ctx, stmt)
- if err != nil {
- return err
- }
-
- return nil
-}
diff --git a/store/memo_organizer.go b/store/memo_organizer.go
index 84a843c6c..5c153adae 100644
--- a/store/memo_organizer.go
+++ b/store/memo_organizer.go
@@ -2,7 +2,6 @@ package store
import (
"context"
- "database/sql"
)
type MemoOrganizer struct {
@@ -32,28 +31,3 @@ func (s *Store) GetMemoOrganizer(ctx context.Context, find *FindMemoOrganizer) (
func (s *Store) DeleteMemoOrganizer(ctx context.Context, delete *DeleteMemoOrganizer) error {
return s.driver.DeleteMemoOrganizer(ctx, delete)
}
-
-func vacuumMemoOrganizer(ctx context.Context, tx *sql.Tx) error {
- stmt := `
- DELETE FROM
- memo_organizer
- WHERE
- memo_id NOT IN (
- SELECT
- id
- FROM
- memo
- )
- OR user_id NOT IN (
- SELECT
- id
- FROM
- user
- )`
- _, err := tx.ExecContext(ctx, stmt)
- if err != nil {
- return err
- }
-
- return nil
-}
diff --git a/store/memo_relation.go b/store/memo_relation.go
index 351062520..5555c26ca 100644
--- a/store/memo_relation.go
+++ b/store/memo_relation.go
@@ -2,7 +2,6 @@ package store
import (
"context"
- "database/sql"
)
type MemoRelationType string
@@ -54,13 +53,3 @@ func (s *Store) GetMemoRelation(ctx context.Context, find *FindMemoRelation) (*M
func (s *Store) DeleteMemoRelation(ctx context.Context, delete *DeleteMemoRelation) error {
return s.driver.DeleteMemoRelation(ctx, delete)
}
-
-func vacuumMemoRelations(ctx context.Context, tx *sql.Tx) error {
- if _, err := tx.ExecContext(ctx, `
- DELETE FROM memo_relation
- WHERE memo_id NOT IN (SELECT id FROM memo) OR related_memo_id NOT IN (SELECT id FROM memo)
- `); err != nil {
- return err
- }
- return nil
-}
diff --git a/store/resource.go b/store/resource.go
index 8e66df93f..87c149a16 100644
--- a/store/resource.go
+++ b/store/resource.go
@@ -2,7 +2,6 @@ package store
import (
"context"
- "database/sql"
)
type Resource struct {
@@ -75,33 +74,5 @@ func (s *Store) UpdateResource(ctx context.Context, update *UpdateResource) (*Re
}
func (s *Store) DeleteResource(ctx context.Context, delete *DeleteResource) error {
- err := s.driver.DeleteResource(ctx, delete)
- if err != nil {
- return err
- }
-
- if err := s.Vacuum(ctx); err != nil {
- // Prevent linter warning.
- return err
- }
- return nil
-}
-
-func vacuumResource(ctx context.Context, tx *sql.Tx) error {
- stmt := `
- DELETE FROM
- resource
- WHERE
- creator_id NOT IN (
- SELECT
- id
- FROM
- user
- )`
- _, err := tx.ExecContext(ctx, stmt)
- if err != nil {
- return err
- }
-
- return nil
+ return s.driver.DeleteResource(ctx, delete)
}
diff --git a/store/sqlite/memo.go b/store/sqlite/memo.go
index f408811d8..9459c3e19 100644
--- a/store/sqlite/memo.go
+++ b/store/sqlite/memo.go
@@ -234,6 +234,10 @@ func (d *Driver) DeleteMemo(ctx context.Context, delete *store.DeleteMemo) error
return err
}
+ if err := d.Vacuum(ctx); err != nil {
+ // Prevent linter warning.
+ return err
+ }
return nil
}
@@ -268,3 +272,22 @@ func (d *Driver) FindMemosVisibilityList(ctx context.Context, memoIDs []int32) (
return visibilityList, nil
}
+
+func vacuumMemo(ctx context.Context, tx *sql.Tx) error {
+ stmt := `
+ DELETE FROM
+ memo
+ WHERE
+ creator_id NOT IN (
+ SELECT
+ id
+ FROM
+ user
+ )`
+ _, err := tx.ExecContext(ctx, stmt)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
diff --git a/store/sqlite/memo_organizer.go b/store/sqlite/memo_organizer.go
index f46421f9b..245e88a59 100644
--- a/store/sqlite/memo_organizer.go
+++ b/store/sqlite/memo_organizer.go
@@ -2,6 +2,7 @@ package sqlite
import (
"context"
+ "database/sql"
"fmt"
"strings"
@@ -80,3 +81,28 @@ func (d *Driver) DeleteMemoOrganizer(ctx context.Context, delete *store.DeleteMe
}
return nil
}
+
+func vacuumMemoOrganizer(ctx context.Context, tx *sql.Tx) error {
+ stmt := `
+ DELETE FROM
+ memo_organizer
+ WHERE
+ memo_id NOT IN (
+ SELECT
+ id
+ FROM
+ memo
+ )
+ OR user_id NOT IN (
+ SELECT
+ id
+ FROM
+ user
+ )`
+ _, err := tx.ExecContext(ctx, stmt)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
diff --git a/store/sqlite/memo_relation.go b/store/sqlite/memo_relation.go
index 86f6de018..c0fcdd2e2 100644
--- a/store/sqlite/memo_relation.go
+++ b/store/sqlite/memo_relation.go
@@ -2,6 +2,7 @@ package sqlite
import (
"context"
+ "database/sql"
"strings"
"github.com/usememos/memos/store"
@@ -104,3 +105,13 @@ func (d *Driver) DeleteMemoRelation(ctx context.Context, delete *store.DeleteMem
}
return nil
}
+
+func vacuumMemoRelations(ctx context.Context, tx *sql.Tx) error {
+ if _, err := tx.ExecContext(ctx, `
+ DELETE FROM memo_relation
+ WHERE memo_id NOT IN (SELECT id FROM memo) OR related_memo_id NOT IN (SELECT id FROM memo)
+ `); err != nil {
+ return err
+ }
+ return nil
+}
diff --git a/store/sqlite/resource.go b/store/sqlite/resource.go
index b71802a77..9095e3446 100644
--- a/store/sqlite/resource.go
+++ b/store/sqlite/resource.go
@@ -181,5 +181,29 @@ func (d *Driver) DeleteResource(ctx context.Context, delete *store.DeleteResourc
return err
}
+ if err := d.Vacuum(ctx); err != nil {
+ // Prevent linter warning.
+ return err
+ }
+
+ return nil
+}
+
+func vacuumResource(ctx context.Context, tx *sql.Tx) error {
+ stmt := `
+ DELETE FROM
+ resource
+ WHERE
+ creator_id NOT IN (
+ SELECT
+ id
+ FROM
+ user
+ )`
+ _, err := tx.ExecContext(ctx, stmt)
+ if err != nil {
+ return err
+ }
+
return nil
}
diff --git a/store/sqlite/sqlite.go b/store/sqlite/sqlite.go
index 69d5c3548..b9e53fdd0 100644
--- a/store/sqlite/sqlite.go
+++ b/store/sqlite/sqlite.go
@@ -1,8 +1,12 @@
package sqlite
import (
+ "context"
"database/sql"
+ "github.com/pkg/errors"
+ "modernc.org/sqlite"
+
"github.com/usememos/memos/store"
)
@@ -15,3 +19,91 @@ func NewDriver(db *sql.DB) store.Driver {
db: db,
}
}
+
+func (d *Driver) Vacuum(ctx context.Context) error {
+ tx, err := d.db.BeginTx(ctx, nil)
+ if err != nil {
+ return err
+ }
+ defer tx.Rollback()
+
+ if err := vacuumImpl(ctx, tx); err != nil {
+ return err
+ }
+
+ if err := tx.Commit(); err != nil {
+ return err
+ }
+
+ // Vacuum sqlite database file size after deleting resource.
+ if _, err := d.db.Exec("VACUUM"); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func vacuumImpl(ctx context.Context, tx *sql.Tx) error {
+ if err := vacuumMemo(ctx, tx); err != nil {
+ return err
+ }
+ if err := vacuumResource(ctx, tx); err != nil {
+ return err
+ }
+ if err := vacuumUserSetting(ctx, tx); err != nil {
+ return err
+ }
+ if err := vacuumMemoOrganizer(ctx, tx); err != nil {
+ return err
+ }
+ if err := vacuumMemoRelations(ctx, tx); err != nil {
+ return err
+ }
+ if err := vacuumTag(ctx, tx); err != nil {
+ // Prevent revive warning.
+ return err
+ }
+
+ return nil
+}
+
+func (d *Driver) BackupTo(ctx context.Context, filename string) error {
+ conn, err := d.db.Conn(ctx)
+ if err != nil {
+ return errors.Errorf("fail to get conn %s", err)
+ }
+ defer conn.Close()
+
+ err = conn.Raw(func(driverConn any) error {
+ type backuper interface {
+ NewBackup(string) (*sqlite.Backup, error)
+ }
+ backupConn, ok := driverConn.(backuper)
+ if !ok {
+ return errors.Errorf("db connection is not a sqlite backuper")
+ }
+
+ bck, err := backupConn.NewBackup(filename)
+ if err != nil {
+ return errors.Errorf("fail to create sqlite backup %s", err)
+ }
+
+ for more := true; more; {
+ more, err = bck.Step(-1)
+ if err != nil {
+ return errors.Errorf("fail to execute sqlite backup %s", err)
+ }
+ }
+
+ return bck.Finish()
+ })
+ if err != nil {
+ return errors.Errorf("fail to backup %s", err)
+ }
+
+ return nil
+}
+
+func (d *Driver) Close() error {
+ return d.db.Close()
+}
diff --git a/store/sqlite/tag.go b/store/sqlite/tag.go
index 73400e91e..0132b3f41 100644
--- a/store/sqlite/tag.go
+++ b/store/sqlite/tag.go
@@ -2,6 +2,7 @@ package sqlite
import (
"context"
+ "database/sql"
"strings"
"github.com/usememos/memos/store"
@@ -73,3 +74,22 @@ func (d *Driver) DeleteTag(ctx context.Context, delete *store.DeleteTag) error {
}
return nil
}
+
+func vacuumTag(ctx context.Context, tx *sql.Tx) error {
+ stmt := `
+ DELETE FROM
+ tag
+ WHERE
+ creator_id NOT IN (
+ SELECT
+ id
+ FROM
+ user
+ )`
+ _, err := tx.ExecContext(ctx, stmt)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
diff --git a/store/sqlite/user.go b/store/sqlite/user.go
index e0d4c7b0f..1e97d9b50 100644
--- a/store/sqlite/user.go
+++ b/store/sqlite/user.go
@@ -168,5 +168,11 @@ func (d *Driver) DeleteUser(ctx context.Context, delete *store.DeleteUser) error
if _, err := result.RowsAffected(); err != nil {
return err
}
+
+ if err := d.Vacuum(ctx); err != nil {
+ // Prevent linter warning.
+ return err
+ }
+
return nil
}
diff --git a/store/sqlite/user_setting.go b/store/sqlite/user_setting.go
index 75c7c3fec..f1916ec7f 100644
--- a/store/sqlite/user_setting.go
+++ b/store/sqlite/user_setting.go
@@ -2,6 +2,7 @@ package sqlite
import (
"context"
+ "database/sql"
"errors"
"strings"
@@ -153,3 +154,22 @@ func (d *Driver) ListUserSettingsV1(ctx context.Context, find *store.FindUserSet
return userSettingList, nil
}
+
+func vacuumUserSetting(ctx context.Context, tx *sql.Tx) error {
+ stmt := `
+ DELETE FROM
+ user_setting
+ WHERE
+ user_id NOT IN (
+ SELECT
+ id
+ FROM
+ user
+ )`
+ _, err := tx.ExecContext(ctx, stmt)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
diff --git a/store/store.go b/store/store.go
index d4287b252..91ef61757 100644
--- a/store/store.go
+++ b/store/store.go
@@ -2,20 +2,14 @@ package store
import (
"context"
- "database/sql"
"sync"
- "modernc.org/sqlite"
-
- "github.com/pkg/errors"
-
"github.com/usememos/memos/server/profile"
)
// Store provides database access to all raw objects.
type Store struct {
Profile *profile.Profile
- db *sql.DB
driver Driver
systemSettingCache sync.Map // map[string]*SystemSetting
userCache sync.Map // map[int]*User
@@ -24,98 +18,21 @@ type Store struct {
}
// New creates a new instance of Store.
-func New(db *sql.DB, driver Driver, profile *profile.Profile) *Store {
+func New(driver Driver, profile *profile.Profile) *Store {
return &Store{
Profile: profile,
- db: db,
driver: driver,
}
}
-func (s *Store) GetDB() *sql.DB {
- return s.db
-}
-
func (s *Store) BackupTo(ctx context.Context, filename string) error {
- conn, err := s.db.Conn(ctx)
- if err != nil {
- return errors.Errorf("fail to get conn %s", err)
- }
- defer conn.Close()
-
- err = conn.Raw(func(driverConn any) error {
- type backuper interface {
- NewBackup(string) (*sqlite.Backup, error)
- }
- backupConn, ok := driverConn.(backuper)
- if !ok {
- return errors.Errorf("db connection is not a sqlite backuper")
- }
-
- bck, err := backupConn.NewBackup(filename)
- if err != nil {
- return errors.Errorf("fail to create sqlite backup %s", err)
- }
-
- for more := true; more; {
- more, err = bck.Step(-1)
- if err != nil {
- return errors.Errorf("fail to execute sqlite backup %s", err)
- }
- }
-
- return bck.Finish()
- })
- if err != nil {
- return errors.Errorf("fail to backup %s", err)
- }
-
- return nil
+ return s.driver.BackupTo(ctx, filename)
}
func (s *Store) Vacuum(ctx context.Context) error {
- tx, err := s.db.BeginTx(ctx, nil)
- if err != nil {
- return err
- }
- defer tx.Rollback()
-
- if err := s.vacuumImpl(ctx, tx); err != nil {
- return err
- }
-
- if err := tx.Commit(); err != nil {
- return err
- }
-
- // Vacuum sqlite database file size after deleting resource.
- if _, err := s.db.Exec("VACUUM"); err != nil {
- return err
- }
-
- return nil
+ return s.driver.Vacuum(ctx)
}
-func (*Store) vacuumImpl(ctx context.Context, tx *sql.Tx) error {
- if err := vacuumMemo(ctx, tx); err != nil {
- return err
- }
- if err := vacuumResource(ctx, tx); err != nil {
- return err
- }
- if err := vacuumUserSetting(ctx, tx); err != nil {
- return err
- }
- if err := vacuumMemoOrganizer(ctx, tx); err != nil {
- return err
- }
- if err := vacuumMemoRelations(ctx, tx); err != nil {
- return err
- }
- if err := vacuumTag(ctx, tx); err != nil {
- // Prevent revive warning.
- return err
- }
-
- return nil
+func (s *Store) Close() error {
+ return s.driver.Close()
}
diff --git a/store/tag.go b/store/tag.go
index afea07399..d5c38f6bf 100644
--- a/store/tag.go
+++ b/store/tag.go
@@ -2,7 +2,6 @@ package store
import (
"context"
- "database/sql"
)
type Tag struct {
@@ -30,22 +29,3 @@ func (s *Store) ListTags(ctx context.Context, find *FindTag) ([]*Tag, error) {
func (s *Store) DeleteTag(ctx context.Context, delete *DeleteTag) error {
return s.driver.DeleteTag(ctx, delete)
}
-
-func vacuumTag(ctx context.Context, tx *sql.Tx) error {
- stmt := `
- DELETE FROM
- tag
- WHERE
- creator_id NOT IN (
- SELECT
- id
- FROM
- user
- )`
- _, err := tx.ExecContext(ctx, stmt)
- if err != nil {
- return err
- }
-
- return nil
-}
diff --git a/store/user.go b/store/user.go
index 17948fa7a..6afd9dfb9 100644
--- a/store/user.go
+++ b/store/user.go
@@ -130,10 +130,6 @@ func (s *Store) DeleteUser(ctx context.Context, delete *DeleteUser) error {
return err
}
- if err := s.Vacuum(ctx); err != nil {
- // Prevent linter warning.
- return err
- }
s.userCache.Delete(delete.ID)
return nil
}
diff --git a/store/user_setting.go b/store/user_setting.go
index 7726f7a83..98c4234c0 100644
--- a/store/user_setting.go
+++ b/store/user_setting.go
@@ -2,7 +2,6 @@ package store
import (
"context"
- "database/sql"
storepb "github.com/usememos/memos/proto/gen/store"
)
@@ -125,22 +124,3 @@ func (s *Store) GetUserAccessTokens(ctx context.Context, userID int32) ([]*store
accessTokensUserSetting := userSetting.GetAccessTokens()
return accessTokensUserSetting.AccessTokens, nil
}
-
-func vacuumUserSetting(ctx context.Context, tx *sql.Tx) error {
- stmt := `
- DELETE FROM
- user_setting
- WHERE
- user_id NOT IN (
- SELECT
- id
- FROM
- user
- )`
- _, err := tx.ExecContext(ctx, stmt)
- if err != nil {
- return err
- }
-
- return nil
-}
diff --git a/test/server/server.go b/test/server/server.go
index 5bf4daeb9..1bf400d13 100644
--- a/test/server/server.go
+++ b/test/server/server.go
@@ -41,8 +41,7 @@ func NewTestingServer(ctx context.Context, t *testing.T) (*TestingServer, error)
}
driver := sqlite.NewDriver(db.DBInstance)
-
- store := store.New(db.DBInstance, driver, profile)
+ store := store.New(driver, profile)
server, err := server.NewServer(ctx, profile, store)
if err != nil {
return nil, errors.Wrap(err, "failed to create server")
diff --git a/test/store/store.go b/test/store/store.go
index fdae9d888..d530fe685 100644
--- a/test/store/store.go
+++ b/test/store/store.go
@@ -25,7 +25,6 @@ func NewTestingStore(ctx context.Context, t *testing.T) *store.Store {
}
driver := sqlite.NewDriver(db.DBInstance)
-
- store := store.New(db.DBInstance, driver, profile)
+ store := store.New(driver, profile)
return store
}