π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Initialize existing users' file counts
Neeraj Gupta8267e7f3fc0d
diff --git a/server/pkg/repo/usage_file_count_init.go b/server/pkg/repo/usage_file_count_init.gonew file mode 100644index 0000000000..82d726eb1a--- /dev/null+++ b/server/pkg/repo/usage_file_count_init.go@@ -0,0 +1,94 @@+package repo++import (+ "context"+ "errors"+ "fmt"+)++var ErrFileCountIneligible = errors.New("file counts are ineligible")++// InitializeFileCounts publishes both counters without holding a usage-row lock+// while counting. All count-changing writers must be deployed before calling it.+func (repo *UsageRepository) InitializeFileCounts(ctx context.Context, userID int64) (bool, error) {+ counts, err := repo.readInitialFileCounts(ctx, userID)+ if err != nil {+ return false, err+ }+ if !counts.legacy {+ return false, nil+ }+ if counts.ineligible != "" {+ return false, fmt.Errorf("%w: %s", ErrFileCountIneligible, counts.ineligible)+ }+ return repo.publishInitialFileCounts(ctx, userID, counts)+}++type initialFileCounts struct {+ photos, locker, version int64+ legacy bool+ ineligible string+}++func (repo *UsageRepository) readInitialFileCounts(ctx context.Context, userID int64) (initialFileCounts, error) {+ var counts initialFileCounts+ // One statement gives the source version, counts and checks the same snapshot.+ // The per-file aggregate keeps membership lookups bounded to this owner's files.+ err := repo.DB.QueryRowContext(ctx, `WITH source AS (+ SELECT+ COUNT(*) FILTER (WHERE memberships.photos) AS photos,+ COUNT(*) FILTER (WHERE memberships.locker) AS locker,+ BOOL_OR(memberships.app_count > 1 OR memberships.unsupported_app) AS ambiguous_app,+ BOOL_OR(memberships.photos IS NOT TRUE AND memberships.locker IS NOT TRUE+ AND t.file_id IS NULL) AS orphan,+ BOOL_OR(t.file_id IS NOT NULL+ AND (memberships.photos OR memberships.locker OR t.user_id <> $1)) AS inconsistent_trash+ FROM files AS f+ CROSS JOIN LATERAL (+ SELECT+ BOOL_OR(c.owner_id = $1 AND c.app = 'photos') AS photos,+ BOOL_OR(c.owner_id = $1 AND c.app = 'locker') AS locker,+ COUNT(DISTINCT c.app) AS app_count,+ BOOL_OR(c.app NOT IN ('photos', 'locker')) AS unsupported_app+ FROM collection_files AS cf+ JOIN collections AS c ON c.collection_id = cf.collection_id+ WHERE cf.file_id = f.file_id AND cf.is_deleted = FALSE+ ) AS memberships+ LEFT JOIN trash AS t ON t.file_id = f.file_id AND t.is_restored = FALSE+ WHERE f.owner_id = $1+ )+ SELECT u.file_count_source_version,+ u.photos_file_count IS NULL AND u.locker_file_count IS NULL,+ source.photos, source.locker,+ CASE+ WHEN source.ambiguous_app THEN 'cross-app or unsupported app memberships'+ WHEN source.orphan THEN 'untrashed file without an owned membership'+ WHEN source.inconsistent_trash THEN 'active membership or wrong owner in Trash'+ WHEN EXISTS (+ SELECT 1+ FROM collections AS c+ JOIN collection_files AS cf ON cf.collection_id = c.collection_id+ JOIN files AS f ON f.file_id = cf.file_id+ WHERE c.owner_id = $1 AND c.app = 'locker' AND cf.is_deleted = FALSE+ AND (cf.f_owner_id IS NOT DISTINCT FROM $1) <> (f.owner_id = $1)+ ) THEN 'Locker ownership predicates disagree'+ ELSE ''+ END+ FROM usage AS u CROSS JOIN source+ WHERE u.user_id = $1`, userID).Scan(+ &counts.version, &counts.legacy, &counts.photos, &counts.locker, &counts.ineligible)+ return counts, err+}++func (repo *UsageRepository) publishInitialFileCounts(ctx context.Context, userID int64, counts initialFileCounts) (bool, error) {+ result, err := repo.DB.ExecContext(ctx, `UPDATE usage+ SET photos_file_count = $2, locker_file_count = $3+ WHERE user_id = $1 AND file_count_source_version = $4+ AND photos_file_count IS NULL AND locker_file_count IS NULL`,+ userID, counts.photos, counts.locker, counts.version)+ if err != nil {+ return false, err+ }+ updated, err := result.RowsAffected()+ return updated == 1, err+}diff --git a/server/pkg/repo/usage_file_count_init_test.go b/server/pkg/repo/usage_file_count_init_test.gonew file mode 100644index 0000000000..72e61816f5--- /dev/null+++ b/server/pkg/repo/usage_file_count_init_test.go@@ -0,0 +1,162 @@+package repo++import (+ "errors"+ "testing"++ "github.com/ente/museum/ente"+ "github.com/ente/museum/internal/testutil"+ "github.com/ente/museum/pkg/repo/public"+)++func TestInitializeFileCountsUsesActiveOwnedMemberships(t *testing.T) {+ _, db, userID := setupCollectionMembershipTest(t)+ usageRepo := &UsageRepository{DB: db}+ otherUserID := testutil.InsertUser(t, db, testutil.UserFixture{UserID: 2, Email: "[email protected]", CreationTime: 1})+ photosFile := insertObjectTestFile(t, db, userID)+ lockerFile := insertObjectTestFile(t, db, userID)+ foreignFile := insertObjectTestFile(t, db, otherUserID)+ for range 2 {+ collectionID := insertObjectTestCollection(t, db, userID)+ linkObjectTestFileToCollection(t, db, collectionID, photosFile, userID)+ linkObjectTestFileToCollection(t, db, collectionID, foreignFile, otherUserID)+ // Pre-60 Photos memberships have NULL denormalized owners. A collection+ // scheduled for deletion remains countable until its memberships are trashed.+ if _, err := db.Exec(`UPDATE collection_files SET c_owner_id = NULL, f_owner_id = NULL+ WHERE collection_id = $1`, collectionID); err != nil {+ t.Fatal(err)+ }+ if _, err := db.Exec(`UPDATE collections SET is_deleted = TRUE WHERE collection_id = $1`, collectionID); err != nil {+ t.Fatal(err)+ }+ }+ lockerCollection := insertObjectTestCollection(t, db, userID)+ if _, err := db.Exec(`UPDATE collections SET app = 'locker' WHERE collection_id = $1`, lockerCollection); err != nil {+ t.Fatal(err)+ }+ linkObjectTestFileToCollection(t, db, lockerCollection, lockerFile, userID)+ if _, err := db.Exec(`UPDATE usage SET storage_consumed = 17, file_count_source_version = 9 WHERE user_id = $1`, userID); err != nil {+ t.Fatal(err)+ }++ for _, wantInitialized := range []bool{true, false} {+ initialized, err := usageRepo.InitializeFileCounts(t.Context(), userID)+ if err != nil || initialized != wantInitialized {+ t.Fatalf("InitializeFileCounts() = (%t, %v), want %t", initialized, err, wantInitialized)+ }+ assertReadyFileCounts(t, db, userID, 1, 1, 9)+ }+ storage, err := usageRepo.GetUsage(userID)+ if err != nil || storage != 17 {+ t.Fatalf("storage = (%d, %v), want 17", storage, err)+ }+}++func TestInitializeFileCountsRejectsInconsistentHistory(t *testing.T) {+ for _, tt := range []struct {+ name string+ sql string+ }{+ {"orphan", `UPDATE collection_files SET is_deleted = TRUE WHERE file_id = 201`},+ {"shared_only", `UPDATE collections SET owner_id = 2 WHERE collection_id = 101`},+ {"cross_app", `INSERT INTO collection_files(collection_id, file_id, encrypted_key, key_decryption_nonce, updation_time)+ VALUES (102, 201, 'key', 'nonce', 1)`},+ {"active_trash", `INSERT INTO trash(file_id, user_id, collection_id, delete_by) VALUES (201, 1, 101, 1)`},+ {"wrong_trash_owner", `UPDATE collection_files SET is_deleted = TRUE WHERE file_id = 201;+ INSERT INTO trash(file_id, user_id, collection_id, delete_by) VALUES (201, 2, 101, 1)`},+ {"locker_null_owner", `UPDATE collection_files SET f_owner_id = NULL WHERE file_id = 201`},+ // The legacy and canonical totals both equal one; different files qualify.+ {"locker_canceling_errors", `UPDATE collection_files+ SET f_owner_id = CASE WHEN file_id = 201 THEN NULL ELSE 1 END WHERE collection_id = 101`},+ } {+ t.Run(tt.name, func(t *testing.T) {+ _, db, userID := setupCollectionMembershipTest(t)+ testutil.InsertUser(t, db, testutil.UserFixture{UserID: 2, Email: "[email protected]", CreationTime: 1})+ if _, err := db.Exec(`INSERT INTO collections+ (collection_id, owner_id, encrypted_key, key_decryption_nonce, name, type, attributes, updation_time, app)+ OVERRIDING SYSTEM VALUE VALUES+ (101, 1, 'key', 'nonce', 'Locker', 'album', '{}', 1, 'locker'),+ (102, 1, 'key', 'nonce', 'Photos', 'album', '{}', 1, 'photos');+ INSERT INTO files(file_id, owner_id, file_decryption_header, thumbnail_decryption_header,+ metadata_decryption_header, encrypted_metadata, updation_time)+ OVERRIDING SYSTEM VALUE VALUES+ (201, 1, 'header', 'header', 'header', 'metadata', 1),+ (202, 2, 'header', 'header', 'header', 'metadata', 1);+ INSERT INTO collection_files+ (collection_id, file_id, encrypted_key, key_decryption_nonce, updation_time, c_owner_id, f_owner_id)+ VALUES (101, 201, 'key', 'nonce', 1, 1, 1), (101, 202, 'key', 'nonce', 1, 1, 2)`); err != nil {+ t.Fatal(err)+ }+ if _, err := db.Exec(tt.sql); err != nil {+ t.Fatal(err)+ }+ initialized, err := (&UsageRepository{DB: db}).InitializeFileCounts(t.Context(), userID)+ if initialized || !errors.Is(err, ErrFileCountIneligible) {+ t.Fatalf("InitializeFileCounts() = (%t, %v), want ineligible", initialized, err)+ }+ photos, locker, version := readFileCountState(t, db, userID)+ if photos.Valid || locker.Valid || version != 0 {+ t.Fatalf("ineligible user changed: (%v, %v, %d)", photos, locker, version)+ }+ })+ }+}++func TestInitializeFileCountsAllowsEmptyAndPermanentlyDeletedLibraries(t *testing.T) {+ for _, name := range []string{"empty", "permanently_deleted"} {+ t.Run(name, func(t *testing.T) {+ _, db, userID := setupCollectionMembershipTest(t)+ if name == "permanently_deleted" {+ collectionID := insertObjectTestCollection(t, db, userID)+ fileID := insertObjectTestFile(t, db, userID)+ linkObjectTestFileToCollection(t, db, collectionID, fileID, userID)+ if _, err := db.Exec(`UPDATE collection_files SET is_deleted = TRUE WHERE file_id = $1`, fileID); err != nil {+ t.Fatal(err)+ }+ if _, err := db.Exec(`INSERT INTO trash(file_id, user_id, collection_id, delete_by, is_deleted)+ VALUES ($1, $2, $3, 1, TRUE)`, fileID, userID, collectionID); err != nil {+ t.Fatal(err)+ }+ }+ if initialized, err := (&UsageRepository{DB: db}).InitializeFileCounts(t.Context(), userID); err != nil || !initialized {+ t.Fatalf("initialization = (%t, %v), want true", initialized, err)+ }+ assertReadyFileCounts(t, db, userID, 0, 0, 0)+ })+ }+}++func TestInitializeFileCountsSerializesWithWriters(t *testing.T) {+ repository, db, userID := setupCollectionMembershipTest(t)+ usageRepo := &UsageRepository{DB: db}+ collectionID := insertObjectTestCollection(t, db, userID)+ fileID := insertObjectTestFile(t, db, userID)+ linkObjectTestFileToCollection(t, db, collectionID, fileID, userID)+ repository.TrashRepo.FileLinkRepo = public.NewFileLinkRepo(db)++ counts, err := usageRepo.readInitialFileCounts(t.Context(), userID)+ if err != nil {+ t.Fatal(err)+ }+ if err := repository.TrashRepo.TrashFiles(t.Context(), userID, ente.TrashRequest{+ TrashItems: []ente.TrashItemRequest{{FileID: fileID, CollectionID: collectionID}},+ }); err != nil {+ t.Fatal(err)+ }+ if initialized, err := usageRepo.publishInitialFileCounts(t.Context(), userID, counts); err != nil || initialized {+ t.Fatalf("stale publish = (%t, %v), want false", initialized, err)+ }+ photos, locker, version := readFileCountState(t, db, userID)+ if photos.Valid || locker.Valid || version != 1 {+ t.Fatalf("stale publish changed counters: (%v, %v, %d)", photos, locker, version)+ }+ if initialized, err := usageRepo.InitializeFileCounts(t.Context(), userID); err != nil || !initialized {+ t.Fatalf("fresh initialization = (%t, %v), want true", initialized, err)+ }+ assertReadyFileCounts(t, db, userID, 0, 0, 1)+ if err := repository.RestoreFiles(t.Context(), userID, collectionID,+ []ente.CollectionFileItem{collectionMembershipTestItem(fileID)}); err != nil {+ t.Fatal(err)+ }+ assertReadyFileCounts(t, db, userID, 1, 0, 2)+}diff --git a/server/tools/init-file-counts/README.md b/server/tools/init-file-counts/README.mdnew file mode 100644index 0000000000..682527f8a9--- /dev/null+++ b/server/tools/init-file-counts/README.md@@ -0,0 +1,59 @@+# Initialize existing users' file counts++This operator-run command fills both NULL file-count columns in existing `usage`+rows. It does not run on server startup, change API reads, repair source data,+or modify storage usage. New accounts already start with initialized counters.++Run only after **all count-aware writers are deployed and old processes have+stopped**, including the stale-membership invalidation path. Use the primary DB.++Build from `server/`:++```sh+go build -o init-file-counts ./tools/init-file-counts+```++Set the usual `PGHOST`, `PGPORT`, `PGDATABASE`, `PGUSER`, and TLS/authentication+settings used by `psql`. Do not put passwords on the command line. The command+does not read Museum's configuration or run schema migrations.++For the first cohort, run one instance:++```sh+./init-file-counts -before-user-id 200+```++Both ID bounds are exclusive. Only paired-NULL rows qualify. The command uses+one database connection, processes one user at a time, allows 10 seconds per+attempt, and pauses one second between users. Override `-timeout` or `-pause`+when needed. `-after-user-id` selects a later range; rerunning a range skips+already initialized users. Stop with Ctrl-C/SIGTERM if DB load or query latency+rises. A small ID range does not imply small libraries.++Each attempt reads the version, distinct active owned-file counts, and+eligibility in one SQL snapshot. It then publishes both counters only if they+are still NULL and the version is unchanged. Counting takes no file or membership+row locks; only publication briefly locks that user's usage row. Count-changing+writers that commit before publication cause this attempt to be skipped; writers+that follow it maintain the initialized counts. Publication does not change the+source version.++Unsafe histories stay NULL and are logged with a reason:++- An owned file has no active owned membership and no unrestored Trash row.+ Trash was added later: older cleanup used missing collection references to+ delete objects directly, so missing references do not establish Trash state.+- An unrestored Trash row conflicts with active owned membership or file ownership.+- An owned file has active cross-app or unsupported-app memberships.+- An active Locker membership's actual file ownership disagrees with the legacy+ `f_owner_id` predicate. NULL denormalized owners on old Photos rows are allowed.++Timeouts and version conflicts are also skipped for a later pass. Other database+errors stop the command. Missing usage rows are never created. Do not reset a+source version or write zero counts to get a user past a failed check.++API reads remain source-based. Before expanding the cohort or switching reads,+check the existing `museum_file_count_comparisons_total` match/mismatch metric+and `file count mismatch` logs during real source reads and file activity.+No observations is not a successful validation. The trusted-client same-app+assumption still applies; this command does not add guards to AddFiles/MoveFiles.diff --git a/server/tools/init-file-counts/main.go b/server/tools/init-file-counts/main.gonew file mode 100644index 0000000000..64c611b4d8--- /dev/null+++ b/server/tools/init-file-counts/main.go@@ -0,0 +1,79 @@+package main++import (+ "context"+ "database/sql"+ "errors"+ "flag"+ "fmt"+ "log"+ "os"+ "os/signal"+ "syscall"+ "time"++ "github.com/ente/museum/pkg/repo"+ _ "github.com/lib/pq"+)++func main() {+ if err := run(); err != nil && !errors.Is(err, context.Canceled) {+ log.Fatal(err)+ }+}++func run() error {+ after := flag.Int64("after-user-id", 0, "exclusive lower user ID bound")+ before := flag.Int64("before-user-id", 0, "exclusive upper user ID bound (required)")+ timeout := flag.Duration("timeout", 10*time.Second, "maximum time per user")+ pause := flag.Duration("pause", time.Second, "pause between users")+ flag.Parse()+ if *after < 0 || *before <= *after || *timeout <= 0 || *pause < 0 || flag.NArg() != 0 {+ return fmt.Errorf("require 0 <= after-user-id < before-user-id, positive timeout and nonnegative pause")+ }+ if os.Getenv("PGDATABASE") == "" {+ return fmt.Errorf("PGDATABASE must identify the primary database; connection settings use PG* environment variables")+ }+ ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)+ defer stop()+ db, err := sql.Open("postgres", "application_name=init-file-counts")+ if err != nil {+ return err+ }+ defer db.Close()+ db.SetMaxOpenConns(1)+ usageRepo := &repo.UsageRepository{DB: db}++ for lastUserID := *after; ; {+ attemptCtx, cancel := context.WithTimeout(ctx, *timeout)+ var userID int64+ err := db.QueryRowContext(attemptCtx, `SELECT user_id FROM usage+ WHERE user_id > $1 AND user_id < $2+ AND photos_file_count IS NULL AND locker_file_count IS NULL+ ORDER BY user_id LIMIT 1`, lastUserID, *before).Scan(&userID)+ if err != nil {+ cancel()+ if errors.Is(err, sql.ErrNoRows) {+ return nil+ }+ return err+ }+ lastUserID = userID+ initialized, err := usageRepo.InitializeFileCounts(attemptCtx, userID)+ timedOut := errors.Is(attemptCtx.Err(), context.DeadlineExceeded)+ cancel()+ if err != nil {+ if !timedOut && !errors.Is(err, repo.ErrFileCountIneligible) {+ return fmt.Errorf("user %d: %w", userID, err)+ }+ log.Printf("user_id=%d skipped: %v", userID, err)+ } else {+ log.Printf("user_id=%d initialized=%t", userID, initialized)+ }+ select {+ case <-ctx.Done():+ return ctx.Err()+ case <-time.After(*pause):+ }+ }+}