π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Prevent recovery from racing account cleanup
Neeraj Gupta406539b0e1e2
diff --git a/server/pkg/controller/user/account_recovery.go b/server/pkg/controller/user/account_recovery.goindex 37d0c768be..2b3f237a07 100644--- a/server/pkg/controller/user/account_recovery.go+++ b/server/pkg/controller/user/account_recovery.go@@ -179,17 +179,29 @@ func (c *UserController) recoverAccount(ctx *gin.Context, req ente.RecoverAccounif err != nil {return "", stacktrace.Propagate(err, "")}- if err := c.UserRepo.UpdateEmail(req.UserID, encryptedEmail, emailHash); err != nil {+ transaction, err := c.UserRepo.DB.BeginTx(ctx, nil)+ if err != nil {+ return "", stacktrace.Propagate(err, "failed to start account recovery transaction")+ }+ defer transaction.Rollback()+ if err := c.DataCleanupRepo.LockScheduledDelete(ctx, transaction, req.UserID); errors.Is(err, sql.ErrNoRows) {+ return "", stacktrace.Propagate(ErrAccountRecoveryUnavailable, "scheduled deletion is no longer recoverable")+ } else if err != nil {+ return "", stacktrace.Propagate(err, "failed to lock scheduled deletion")+ }+ if err := c.UserRepo.UpdateEmailTx(ctx, transaction, req.UserID, encryptedEmail, emailHash); err != nil {if isEmailHashUniqueConstraint(err) {return "", stacktrace.Propagate(ErrAccountRecoveryEmailInUse, "email was claimed while recovering the account")}return "", stacktrace.Propagate(err, "failed to update email")}- c.touchContactsAfterEmailUpdate(ctx, req.UserID)- if err := c.DataCleanupRepo.RemoveScheduledDelete(ctx, req.UserID); err != nil {- logger.WithError(err).Error("failed to remove scheduled delete")- return "", stacktrace.Propagate(err, "")+ if err := c.DataCleanupRepo.RemoveScheduledDeleteTx(ctx, transaction, req.UserID); err != nil {+ return "", stacktrace.Propagate(err, "failed to remove scheduled delete")+ }+ if err := transaction.Commit(); err != nil {+ return "", stacktrace.Propagate(err, "failed to commit account recovery")}+ c.touchContactsAfterEmailUpdate(ctx, req.UserID)return ente.AccountRecoveryRecovered, nil}diff --git a/server/pkg/controller/user/user_recovery_test.go b/server/pkg/controller/user/user_recovery_test.goindex 9fa93e681f..6261c8f991 100644--- a/server/pkg/controller/user/user_recovery_test.go+++ b/server/pkg/controller/user/user_recovery_test.go@@ -7,6 +7,7 @@ import ("net/http/httptest""net/url""testing"+ stdtime "time""github.com/ente/museum/ente"contactmodel "github.com/ente/museum/ente/contact"@@ -248,6 +249,107 @@ func TestValidateSelfAccountRecoveryReportsUnavailableAfterKeyCleanup(t *testing}}+func TestAccountRecoveryLosesRaceToCleanupStageAdvance(t *testing.T) {+ testutil.WithServerRoot(t)+ db := testutil.RequireTestDB(t)+ testutil.ResetTables(t, db)+ t.Cleanup(func() { testutil.ResetTables(t, db) })++ userID := testutil.InsertUser(t, db, testutil.UserFixture{+ UserID: 82,+ Email: "[email protected]",+ CreationTime: 1,+ })+ insertKeyAttributes(t, db, userID)+ insertScheduledDelete(t, db, userID)+ userRepo := &repo.UserRepository{+ DB: db,+ SecretEncryptionKey: testutil.SecretEncryptionKey(),+ HashingKey: testutil.HashingKey(),+ }+ if err := userRepo.Delete(userID); err != nil {+ t.Fatalf("failed to delete user: %v", err)+ }+ controller := &UserController{+ UserRepo: userRepo,+ DataCleanupRepo: &cleanuprepo.Repository{DB: db},+ SecretEncryptionKey: testutil.SecretEncryptionKey(),+ HashingKey: testutil.HashingKey(),+ }++ cleanupTx, err := db.BeginTx(t.Context(), nil)+ if err != nil {+ t.Fatalf("failed to start cleanup transaction: %v", err)+ }+ defer cleanupTx.Rollback()+ if _, err := cleanupTx.ExecContext(t.Context(),+ `UPDATE data_cleanup SET stage = $1 WHERE user_id = $2`,+ cleanupentity.Collection,+ userID,+ ); err != nil {+ t.Fatalf("failed to advance cleanup: %v", err)+ }++ gin.SetMode(gin.TestMode)+ ctx, _ := gin.CreateTestContext(httptest.NewRecorder())+ recoveryDone := make(chan error, 1)+ go func() {+ recoveryDone <- controller.HandleAccountRecovery(ctx, ente.RecoverAccountRequest{+ UserID: userID,+ EmailID: "[email protected]",+ })+ }()+ waitForDataCleanupRowLock(t, db)+ if err := cleanupTx.Commit(); err != nil {+ t.Fatalf("failed to commit cleanup stage advance: %v", err)+ }++ select {+ case err := <-recoveryDone:+ if !errors.Is(err, ErrAccountRecoveryUnavailable) {+ t.Fatalf("HandleAccountRecovery() error = %v, want ErrAccountRecoveryUnavailable", err)+ }+ case <-stdtime.After(2 * stdtime.Second):+ t.Fatal("account recovery did not finish after cleanup committed")+ }+ if _, err := userRepo.Get(userID); !errors.Is(err, ente.ErrUserDeleted) {+ t.Fatalf("user state after lost recovery race = %v, want deleted", err)+ }+ var stage cleanupentity.Stage+ if err := db.QueryRow(`SELECT stage FROM data_cleanup WHERE user_id = $1`, userID).Scan(&stage); err != nil {+ t.Fatalf("failed to read cleanup stage: %v", err)+ }+ if stage != cleanupentity.Collection {+ t.Fatalf("cleanup stage = %q, want %q", stage, cleanupentity.Collection)+ }+}++func waitForDataCleanupRowLock(t *testing.T, db *sql.DB) {+ t.Helper()+ deadline := stdtime.Now().Add(2 * stdtime.Second)+ for {+ var waiting bool+ err := db.QueryRow(`SELECT EXISTS(+ SELECT 1 FROM pg_stat_activity+ WHERE datname = current_database()+ AND pid <> pg_backend_pid()+ AND wait_event_type = 'Lock'+ AND query LIKE '%data_cleanup%'+ AND query LIKE '%FOR UPDATE%'+ )`).Scan(&waiting)+ if err != nil {+ t.Fatalf("failed to inspect recovery lock wait: %v", err)+ }+ if waiting {+ return+ }+ if stdtime.Now().After(deadline) {+ t.Fatal("account recovery did not wait for the cleanup row lock")+ }+ stdtime.Sleep(10 * stdtime.Millisecond)+ }+}+func accountRecoveryToken(t *testing.T, controller *UserController, claim *enteJWT.WebCommonJWTClaim) string {t.Helper()token, err := controller.GetJWTTokenForClaim(claim)@@ -432,6 +534,11 @@ func insertKeyAttributes(t *testing.T, db *sql.DB, userID int64) {func insertScheduledDelete(t *testing.T, db *sql.DB, userID int64) {t.Helper()+ t.Cleanup(func() {+ if _, err := db.Exec(`DELETE FROM data_cleanup WHERE user_id = $1`, userID); err != nil {+ t.Errorf("failed to remove data_cleanup fixture for user %d: %v", userID, err)+ }+ })_, err := db.Exec(`INSERT INTO data_cleanup(user_id, stage, stage_schedule_time, stage_attempt_count)VALUES($1, $2, $3, $4)`,diff --git a/server/pkg/repo/datacleanup/repository.go b/server/pkg/repo/datacleanup/repository.goindex 60d41616c9..2e045fb280 100644--- a/server/pkg/repo/datacleanup/repository.go+++ b/server/pkg/repo/datacleanup/repository.go@@ -19,14 +19,32 @@ func (r *Repository) Insert(ctx context.Context, userID int64) error {return stacktrace.Propagate(err, "failed to insert")}+func (r *Repository) LockScheduledDelete(ctx context.Context, tx *sql.Tx, userID int64) error {+ var lockedUserID int64+ return tx.QueryRowContext(ctx, `SELECT user_id FROM data_cleanup+ WHERE user_id = $1 AND stage = $2 FOR UPDATE`, userID, entity.Scheduled).Scan(&lockedUserID)+}+func (r *Repository) RemoveScheduledDelete(ctx context.Context, userID int64) error {- res, execErr := r.DB.ExecContext(ctx, `DELETE from data_cleanup where user_id= $1 and stage = $2`, userID, entity.Scheduled)- if execErr != nil {- return execErr+ return removeScheduledDelete(ctx, r.DB, userID)+}++func (r *Repository) RemoveScheduledDeleteTx(ctx context.Context, tx *sql.Tx, userID int64) error {+ return removeScheduledDelete(ctx, tx, userID)+}++type scheduledDeleteExecutor interface {+ ExecContext(context.Context, string, ...any) (sql.Result, error)+}++func removeScheduledDelete(ctx context.Context, executor scheduledDeleteExecutor, userID int64) error {+ result, err := executor.ExecContext(ctx, `DELETE FROM data_cleanup WHERE user_id = $1 AND stage = $2`, userID, entity.Scheduled)+ if err != nil {+ return err}- affected, affErr := res.RowsAffected()- if affErr != nil {- return affErr+ affected, err := result.RowsAffected()+ if err != nil {+ return err}if affected != 1 {return fmt.Errorf("only one row should have been affected, got %d", affected)diff --git a/server/pkg/repo/user.go b/server/pkg/repo/user.goindex ffe8dcd527..c3d867d1cd 100644--- a/server/pkg/repo/user.go+++ b/server/pkg/repo/user.go@@ -41,6 +41,10 @@ type UserInactivityCandidate struct {LastActivity int64}+type emailUpdateExecutor interface {+ ExecContext(context.Context, string, ...any) (sql.Result, error)+}+// Get returns a user indicated by the userIDfunc (repo *UserRepository) Get(userID int64) (ente.User, error) {var user ente.User@@ -324,7 +328,15 @@ func (repo *UserRepository) UpdateDeleteFeedback(userID int64, feedback map[stri// UpdateEmail updates the email address of a userfunc (repo *UserRepository) UpdateEmail(userID int64, encryptedEmail ente.EncryptionResult, emailHash string) error {- _, err := repo.DB.Exec(`UPDATE users SET encrypted_email = $1, email_decryption_nonce = $2, email_hash = $3 WHERE user_id = $4`, encryptedEmail.Cipher, encryptedEmail.Nonce, emailHash, userID)+ return updateEmail(context.Background(), repo.DB, userID, encryptedEmail, emailHash)+}++func (repo *UserRepository) UpdateEmailTx(ctx context.Context, tx *sql.Tx, userID int64, encryptedEmail ente.EncryptionResult, emailHash string) error {+ return updateEmail(ctx, tx, userID, encryptedEmail, emailHash)+}++func updateEmail(ctx context.Context, executor emailUpdateExecutor, userID int64, encryptedEmail ente.EncryptionResult, emailHash string) error {+ _, err := executor.ExecContext(ctx, `UPDATE users SET encrypted_email = $1, email_decryption_nonce = $2, email_hash = $3 WHERE user_id = $4`, encryptedEmail.Cipher, encryptedEmail.Nonce, emailHash, userID)return stacktrace.Propagate(err, "")}