Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Initialize existing users' file counts

Neeraj Gupta8267e7f3fc0d
diff --git a/server/pkg/repo/usage_file_count_init.go b/server/pkg/repo/usage_file_count_init.go
new file mode 100644
index 0000000000..82d726eb1a
--- /dev/null
+++ b/server/pkg/repo/usage_file_count_init.go
@@ -0,0 +1,94 @@
+package repo
+
+import (
+ "context"
+ "errors"
+ "fmt"
+)
+
+var ErrFileCountIneligible = errors.New("file counts are ineligible")
+
+// InitializeFileCounts publishes both counters without holding a usage-row lock
+// while counting. All count-changing writers must be deployed before calling it.
+func (repo *UsageRepository) InitializeFileCounts(ctx context.Context, userID int64) (bool, error) {
+ counts, err := repo.readInitialFileCounts(ctx, userID)
+ if err != nil {
+ return false, err
+ }
+ if !counts.legacy {
+ return false, nil
+ }
+ if counts.ineligible != "" {
+ return false, fmt.Errorf("%w: %s", ErrFileCountIneligible, counts.ineligible)
+ }
+ return repo.publishInitialFileCounts(ctx, userID, counts)
+}
+
+type initialFileCounts struct {
+ photos, locker, version int64
+ legacy bool
+ ineligible string
+}
+
+func (repo *UsageRepository) readInitialFileCounts(ctx context.Context, userID int64) (initialFileCounts, error) {
+ var counts initialFileCounts
+ // One statement gives the source version, counts and checks the same snapshot.
+ // The per-file aggregate keeps membership lookups bounded to this owner's files.
+ err := repo.DB.QueryRowContext(ctx, `WITH source AS (
+ SELECT
+ COUNT(*) FILTER (WHERE memberships.photos) AS photos,
+ COUNT(*) FILTER (WHERE memberships.locker) AS locker,
+ BOOL_OR(memberships.app_count > 1 OR memberships.unsupported_app) AS ambiguous_app,
+ BOOL_OR(memberships.photos IS NOT TRUE AND memberships.locker IS NOT TRUE
+ AND t.file_id IS NULL) AS orphan,
+ BOOL_OR(t.file_id IS NOT NULL
+ AND (memberships.photos OR memberships.locker OR t.user_id <> $1)) AS inconsistent_trash
+ FROM files AS f
+ CROSS JOIN LATERAL (
+ SELECT
+ BOOL_OR(c.owner_id = $1 AND c.app = 'photos') AS photos,
+ BOOL_OR(c.owner_id = $1 AND c.app = 'locker') AS locker,
+ COUNT(DISTINCT c.app) AS app_count,
+ BOOL_OR(c.app NOT IN ('photos', 'locker')) AS unsupported_app
+ FROM collection_files AS cf
+ JOIN collections AS c ON c.collection_id = cf.collection_id
+ WHERE cf.file_id = f.file_id AND cf.is_deleted = FALSE
+ ) AS memberships
+ LEFT JOIN trash AS t ON t.file_id = f.file_id AND t.is_restored = FALSE
+ WHERE f.owner_id = $1
+ )
+ SELECT u.file_count_source_version,
+ u.photos_file_count IS NULL AND u.locker_file_count IS NULL,
+ source.photos, source.locker,
+ CASE
+ WHEN source.ambiguous_app THEN 'cross-app or unsupported app memberships'
+ WHEN source.orphan THEN 'untrashed file without an owned membership'
+ WHEN source.inconsistent_trash THEN 'active membership or wrong owner in Trash'
+ WHEN EXISTS (
+ SELECT 1
+ FROM collections AS c
+ JOIN collection_files AS cf ON cf.collection_id = c.collection_id
+ JOIN files AS f ON f.file_id = cf.file_id
+ WHERE c.owner_id = $1 AND c.app = 'locker' AND cf.is_deleted = FALSE
+ AND (cf.f_owner_id IS NOT DISTINCT FROM $1) <> (f.owner_id = $1)
+ ) THEN 'Locker ownership predicates disagree'
+ ELSE ''
+ END
+ FROM usage AS u CROSS JOIN source
+ WHERE u.user_id = $1`, userID).Scan(
+ &counts.version, &counts.legacy, &counts.photos, &counts.locker, &counts.ineligible)
+ return counts, err
+}
+
+func (repo *UsageRepository) publishInitialFileCounts(ctx context.Context, userID int64, counts initialFileCounts) (bool, error) {
+ result, err := repo.DB.ExecContext(ctx, `UPDATE usage
+ SET photos_file_count = $2, locker_file_count = $3
+ WHERE user_id = $1 AND file_count_source_version = $4
+ AND photos_file_count IS NULL AND locker_file_count IS NULL`,
+ userID, counts.photos, counts.locker, counts.version)
+ if err != nil {
+ return false, err
+ }
+ updated, err := result.RowsAffected()
+ return updated == 1, err
+}
diff --git a/server/pkg/repo/usage_file_count_init_test.go b/server/pkg/repo/usage_file_count_init_test.go
new file mode 100644
index 0000000000..72e61816f5
--- /dev/null
+++ b/server/pkg/repo/usage_file_count_init_test.go
@@ -0,0 +1,162 @@
+package repo
+
+import (
+ "errors"
+ "testing"
+
+ "github.com/ente/museum/ente"
+ "github.com/ente/museum/internal/testutil"
+ "github.com/ente/museum/pkg/repo/public"
+)
+
+func TestInitializeFileCountsUsesActiveOwnedMemberships(t *testing.T) {
+ _, db, userID := setupCollectionMembershipTest(t)
+ usageRepo := &UsageRepository{DB: db}
+ otherUserID := testutil.InsertUser(t, db, testutil.UserFixture{UserID: 2, Email: "[email protected]", CreationTime: 1})
+ photosFile := insertObjectTestFile(t, db, userID)
+ lockerFile := insertObjectTestFile(t, db, userID)
+ foreignFile := insertObjectTestFile(t, db, otherUserID)
+ for range 2 {
+ collectionID := insertObjectTestCollection(t, db, userID)
+ linkObjectTestFileToCollection(t, db, collectionID, photosFile, userID)
+ linkObjectTestFileToCollection(t, db, collectionID, foreignFile, otherUserID)
+ // Pre-60 Photos memberships have NULL denormalized owners. A collection
+ // scheduled for deletion remains countable until its memberships are trashed.
+ if _, err := db.Exec(`UPDATE collection_files SET c_owner_id = NULL, f_owner_id = NULL
+ WHERE collection_id = $1`, collectionID); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := db.Exec(`UPDATE collections SET is_deleted = TRUE WHERE collection_id = $1`, collectionID); err != nil {
+ t.Fatal(err)
+ }
+ }
+ lockerCollection := insertObjectTestCollection(t, db, userID)
+ if _, err := db.Exec(`UPDATE collections SET app = 'locker' WHERE collection_id = $1`, lockerCollection); err != nil {
+ t.Fatal(err)
+ }
+ linkObjectTestFileToCollection(t, db, lockerCollection, lockerFile, userID)
+ if _, err := db.Exec(`UPDATE usage SET storage_consumed = 17, file_count_source_version = 9 WHERE user_id = $1`, userID); err != nil {
+ t.Fatal(err)
+ }
+
+ for _, wantInitialized := range []bool{true, false} {
+ initialized, err := usageRepo.InitializeFileCounts(t.Context(), userID)
+ if err != nil || initialized != wantInitialized {
+ t.Fatalf("InitializeFileCounts() = (%t, %v), want %t", initialized, err, wantInitialized)
+ }
+ assertReadyFileCounts(t, db, userID, 1, 1, 9)
+ }
+ storage, err := usageRepo.GetUsage(userID)
+ if err != nil || storage != 17 {
+ t.Fatalf("storage = (%d, %v), want 17", storage, err)
+ }
+}
+
+func TestInitializeFileCountsRejectsInconsistentHistory(t *testing.T) {
+ for _, tt := range []struct {
+ name string
+ sql string
+ }{
+ {"orphan", `UPDATE collection_files SET is_deleted = TRUE WHERE file_id = 201`},
+ {"shared_only", `UPDATE collections SET owner_id = 2 WHERE collection_id = 101`},
+ {"cross_app", `INSERT INTO collection_files(collection_id, file_id, encrypted_key, key_decryption_nonce, updation_time)
+ VALUES (102, 201, 'key', 'nonce', 1)`},
+ {"active_trash", `INSERT INTO trash(file_id, user_id, collection_id, delete_by) VALUES (201, 1, 101, 1)`},
+ {"wrong_trash_owner", `UPDATE collection_files SET is_deleted = TRUE WHERE file_id = 201;
+ INSERT INTO trash(file_id, user_id, collection_id, delete_by) VALUES (201, 2, 101, 1)`},
+ {"locker_null_owner", `UPDATE collection_files SET f_owner_id = NULL WHERE file_id = 201`},
+ // The legacy and canonical totals both equal one; different files qualify.
+ {"locker_canceling_errors", `UPDATE collection_files
+ SET f_owner_id = CASE WHEN file_id = 201 THEN NULL ELSE 1 END WHERE collection_id = 101`},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ _, db, userID := setupCollectionMembershipTest(t)
+ testutil.InsertUser(t, db, testutil.UserFixture{UserID: 2, Email: "[email protected]", CreationTime: 1})
+ if _, err := db.Exec(`INSERT INTO collections
+ (collection_id, owner_id, encrypted_key, key_decryption_nonce, name, type, attributes, updation_time, app)
+ OVERRIDING SYSTEM VALUE VALUES
+ (101, 1, 'key', 'nonce', 'Locker', 'album', '{}', 1, 'locker'),
+ (102, 1, 'key', 'nonce', 'Photos', 'album', '{}', 1, 'photos');
+ INSERT INTO files(file_id, owner_id, file_decryption_header, thumbnail_decryption_header,
+ metadata_decryption_header, encrypted_metadata, updation_time)
+ OVERRIDING SYSTEM VALUE VALUES
+ (201, 1, 'header', 'header', 'header', 'metadata', 1),
+ (202, 2, 'header', 'header', 'header', 'metadata', 1);
+ INSERT INTO collection_files
+ (collection_id, file_id, encrypted_key, key_decryption_nonce, updation_time, c_owner_id, f_owner_id)
+ VALUES (101, 201, 'key', 'nonce', 1, 1, 1), (101, 202, 'key', 'nonce', 1, 1, 2)`); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := db.Exec(tt.sql); err != nil {
+ t.Fatal(err)
+ }
+ initialized, err := (&UsageRepository{DB: db}).InitializeFileCounts(t.Context(), userID)
+ if initialized || !errors.Is(err, ErrFileCountIneligible) {
+ t.Fatalf("InitializeFileCounts() = (%t, %v), want ineligible", initialized, err)
+ }
+ photos, locker, version := readFileCountState(t, db, userID)
+ if photos.Valid || locker.Valid || version != 0 {
+ t.Fatalf("ineligible user changed: (%v, %v, %d)", photos, locker, version)
+ }
+ })
+ }
+}
+
+func TestInitializeFileCountsAllowsEmptyAndPermanentlyDeletedLibraries(t *testing.T) {
+ for _, name := range []string{"empty", "permanently_deleted"} {
+ t.Run(name, func(t *testing.T) {
+ _, db, userID := setupCollectionMembershipTest(t)
+ if name == "permanently_deleted" {
+ collectionID := insertObjectTestCollection(t, db, userID)
+ fileID := insertObjectTestFile(t, db, userID)
+ linkObjectTestFileToCollection(t, db, collectionID, fileID, userID)
+ if _, err := db.Exec(`UPDATE collection_files SET is_deleted = TRUE WHERE file_id = $1`, fileID); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := db.Exec(`INSERT INTO trash(file_id, user_id, collection_id, delete_by, is_deleted)
+ VALUES ($1, $2, $3, 1, TRUE)`, fileID, userID, collectionID); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if initialized, err := (&UsageRepository{DB: db}).InitializeFileCounts(t.Context(), userID); err != nil || !initialized {
+ t.Fatalf("initialization = (%t, %v), want true", initialized, err)
+ }
+ assertReadyFileCounts(t, db, userID, 0, 0, 0)
+ })
+ }
+}
+
+func TestInitializeFileCountsSerializesWithWriters(t *testing.T) {
+ repository, db, userID := setupCollectionMembershipTest(t)
+ usageRepo := &UsageRepository{DB: db}
+ collectionID := insertObjectTestCollection(t, db, userID)
+ fileID := insertObjectTestFile(t, db, userID)
+ linkObjectTestFileToCollection(t, db, collectionID, fileID, userID)
+ repository.TrashRepo.FileLinkRepo = public.NewFileLinkRepo(db)
+
+ counts, err := usageRepo.readInitialFileCounts(t.Context(), userID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := repository.TrashRepo.TrashFiles(t.Context(), userID, ente.TrashRequest{
+ TrashItems: []ente.TrashItemRequest{{FileID: fileID, CollectionID: collectionID}},
+ }); err != nil {
+ t.Fatal(err)
+ }
+ if initialized, err := usageRepo.publishInitialFileCounts(t.Context(), userID, counts); err != nil || initialized {
+ t.Fatalf("stale publish = (%t, %v), want false", initialized, err)
+ }
+ photos, locker, version := readFileCountState(t, db, userID)
+ if photos.Valid || locker.Valid || version != 1 {
+ t.Fatalf("stale publish changed counters: (%v, %v, %d)", photos, locker, version)
+ }
+ if initialized, err := usageRepo.InitializeFileCounts(t.Context(), userID); err != nil || !initialized {
+ t.Fatalf("fresh initialization = (%t, %v), want true", initialized, err)
+ }
+ assertReadyFileCounts(t, db, userID, 0, 0, 1)
+ if err := repository.RestoreFiles(t.Context(), userID, collectionID,
+ []ente.CollectionFileItem{collectionMembershipTestItem(fileID)}); err != nil {
+ t.Fatal(err)
+ }
+ assertReadyFileCounts(t, db, userID, 1, 0, 2)
+}
diff --git a/server/tools/init-file-counts/README.md b/server/tools/init-file-counts/README.md
new file mode 100644
index 0000000000..682527f8a9
--- /dev/null
+++ b/server/tools/init-file-counts/README.md
@@ -0,0 +1,59 @@
+# Initialize existing users' file counts
+
+This operator-run command fills both NULL file-count columns in existing `usage`
+rows. It does not run on server startup, change API reads, repair source data,
+or modify storage usage. New accounts already start with initialized counters.
+
+Run only after **all count-aware writers are deployed and old processes have
+stopped**, including the stale-membership invalidation path. Use the primary DB.
+
+Build from `server/`:
+
+```sh
+go build -o init-file-counts ./tools/init-file-counts
+```
+
+Set the usual `PGHOST`, `PGPORT`, `PGDATABASE`, `PGUSER`, and TLS/authentication
+settings used by `psql`. Do not put passwords on the command line. The command
+does not read Museum's configuration or run schema migrations.
+
+For the first cohort, run one instance:
+
+```sh
+./init-file-counts -before-user-id 200
+```
+
+Both ID bounds are exclusive. Only paired-NULL rows qualify. The command uses
+one database connection, processes one user at a time, allows 10 seconds per
+attempt, and pauses one second between users. Override `-timeout` or `-pause`
+when needed. `-after-user-id` selects a later range; rerunning a range skips
+already initialized users. Stop with Ctrl-C/SIGTERM if DB load or query latency
+rises. A small ID range does not imply small libraries.
+
+Each attempt reads the version, distinct active owned-file counts, and
+eligibility in one SQL snapshot. It then publishes both counters only if they
+are still NULL and the version is unchanged. Counting takes no file or membership
+row locks; only publication briefly locks that user's usage row. Count-changing
+writers that commit before publication cause this attempt to be skipped; writers
+that follow it maintain the initialized counts. Publication does not change the
+source version.
+
+Unsafe histories stay NULL and are logged with a reason:
+
+- An owned file has no active owned membership and no unrestored Trash row.
+ Trash was added later: older cleanup used missing collection references to
+ delete objects directly, so missing references do not establish Trash state.
+- An unrestored Trash row conflicts with active owned membership or file ownership.
+- An owned file has active cross-app or unsupported-app memberships.
+- An active Locker membership's actual file ownership disagrees with the legacy
+ `f_owner_id` predicate. NULL denormalized owners on old Photos rows are allowed.
+
+Timeouts and version conflicts are also skipped for a later pass. Other database
+errors stop the command. Missing usage rows are never created. Do not reset a
+source version or write zero counts to get a user past a failed check.
+
+API reads remain source-based. Before expanding the cohort or switching reads,
+check the existing `museum_file_count_comparisons_total` match/mismatch metric
+and `file count mismatch` logs during real source reads and file activity.
+No observations is not a successful validation. The trusted-client same-app
+assumption still applies; this command does not add guards to AddFiles/MoveFiles.
diff --git a/server/tools/init-file-counts/main.go b/server/tools/init-file-counts/main.go
new file mode 100644
index 0000000000..64c611b4d8
--- /dev/null
+++ b/server/tools/init-file-counts/main.go
@@ -0,0 +1,79 @@
+package main
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "flag"
+ "fmt"
+ "log"
+ "os"
+ "os/signal"
+ "syscall"
+ "time"
+
+ "github.com/ente/museum/pkg/repo"
+ _ "github.com/lib/pq"
+)
+
+func main() {
+ if err := run(); err != nil && !errors.Is(err, context.Canceled) {
+ log.Fatal(err)
+ }
+}
+
+func run() error {
+ after := flag.Int64("after-user-id", 0, "exclusive lower user ID bound")
+ before := flag.Int64("before-user-id", 0, "exclusive upper user ID bound (required)")
+ timeout := flag.Duration("timeout", 10*time.Second, "maximum time per user")
+ pause := flag.Duration("pause", time.Second, "pause between users")
+ flag.Parse()
+ if *after < 0 || *before <= *after || *timeout <= 0 || *pause < 0 || flag.NArg() != 0 {
+ return fmt.Errorf("require 0 <= after-user-id < before-user-id, positive timeout and nonnegative pause")
+ }
+ if os.Getenv("PGDATABASE") == "" {
+ return fmt.Errorf("PGDATABASE must identify the primary database; connection settings use PG* environment variables")
+ }
+ ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+ defer stop()
+ db, err := sql.Open("postgres", "application_name=init-file-counts")
+ if err != nil {
+ return err
+ }
+ defer db.Close()
+ db.SetMaxOpenConns(1)
+ usageRepo := &repo.UsageRepository{DB: db}
+
+ for lastUserID := *after; ; {
+ attemptCtx, cancel := context.WithTimeout(ctx, *timeout)
+ var userID int64
+ err := db.QueryRowContext(attemptCtx, `SELECT user_id FROM usage
+ WHERE user_id > $1 AND user_id < $2
+ AND photos_file_count IS NULL AND locker_file_count IS NULL
+ ORDER BY user_id LIMIT 1`, lastUserID, *before).Scan(&userID)
+ if err != nil {
+ cancel()
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil
+ }
+ return err
+ }
+ lastUserID = userID
+ initialized, err := usageRepo.InitializeFileCounts(attemptCtx, userID)
+ timedOut := errors.Is(attemptCtx.Err(), context.DeadlineExceeded)
+ cancel()
+ if err != nil {
+ if !timedOut && !errors.Is(err, repo.ErrFileCountIneligible) {
+ return fmt.Errorf("user %d: %w", userID, err)
+ }
+ log.Printf("user_id=%d skipped: %v", userID, err)
+ } else {
+ log.Printf("user_id=%d initialized=%t", userID, initialized)
+ }
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-time.After(*pause):
+ }
+ }
+}