Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Prevent recovery from racing account cleanup

Neeraj Gupta406539b0e1e2
diff --git a/server/pkg/controller/user/account_recovery.go b/server/pkg/controller/user/account_recovery.go
index 37d0c768be..2b3f237a07 100644
--- a/server/pkg/controller/user/account_recovery.go
+++ b/server/pkg/controller/user/account_recovery.go
@@ -179,17 +179,29 @@ func (c *UserController) recoverAccount(ctx *gin.Context, req ente.RecoverAccoun
if err != nil {
return "", stacktrace.Propagate(err, "")
}
- if err := c.UserRepo.UpdateEmail(req.UserID, encryptedEmail, emailHash); err != nil {
+ transaction, err := c.UserRepo.DB.BeginTx(ctx, nil)
+ if err != nil {
+ return "", stacktrace.Propagate(err, "failed to start account recovery transaction")
+ }
+ defer transaction.Rollback()
+ if err := c.DataCleanupRepo.LockScheduledDelete(ctx, transaction, req.UserID); errors.Is(err, sql.ErrNoRows) {
+ return "", stacktrace.Propagate(ErrAccountRecoveryUnavailable, "scheduled deletion is no longer recoverable")
+ } else if err != nil {
+ return "", stacktrace.Propagate(err, "failed to lock scheduled deletion")
+ }
+ if err := c.UserRepo.UpdateEmailTx(ctx, transaction, req.UserID, encryptedEmail, emailHash); err != nil {
if isEmailHashUniqueConstraint(err) {
return "", stacktrace.Propagate(ErrAccountRecoveryEmailInUse, "email was claimed while recovering the account")
}
return "", stacktrace.Propagate(err, "failed to update email")
}
- c.touchContactsAfterEmailUpdate(ctx, req.UserID)
- if err := c.DataCleanupRepo.RemoveScheduledDelete(ctx, req.UserID); err != nil {
- logger.WithError(err).Error("failed to remove scheduled delete")
- return "", stacktrace.Propagate(err, "")
+ if err := c.DataCleanupRepo.RemoveScheduledDeleteTx(ctx, transaction, req.UserID); err != nil {
+ return "", stacktrace.Propagate(err, "failed to remove scheduled delete")
+ }
+ if err := transaction.Commit(); err != nil {
+ return "", stacktrace.Propagate(err, "failed to commit account recovery")
}
+ c.touchContactsAfterEmailUpdate(ctx, req.UserID)
return ente.AccountRecoveryRecovered, nil
}
diff --git a/server/pkg/controller/user/user_recovery_test.go b/server/pkg/controller/user/user_recovery_test.go
index 9fa93e681f..6261c8f991 100644
--- a/server/pkg/controller/user/user_recovery_test.go
+++ b/server/pkg/controller/user/user_recovery_test.go
@@ -7,6 +7,7 @@ import (
"net/http/httptest"
"net/url"
"testing"
+ stdtime "time"
"github.com/ente/museum/ente"
contactmodel "github.com/ente/museum/ente/contact"
@@ -248,6 +249,107 @@ func TestValidateSelfAccountRecoveryReportsUnavailableAfterKeyCleanup(t *testing
}
}
+func TestAccountRecoveryLosesRaceToCleanupStageAdvance(t *testing.T) {
+ testutil.WithServerRoot(t)
+ db := testutil.RequireTestDB(t)
+ testutil.ResetTables(t, db)
+ t.Cleanup(func() { testutil.ResetTables(t, db) })
+
+ userID := testutil.InsertUser(t, db, testutil.UserFixture{
+ UserID: 82,
+ Email: "[email protected]",
+ CreationTime: 1,
+ })
+ insertKeyAttributes(t, db, userID)
+ insertScheduledDelete(t, db, userID)
+ userRepo := &repo.UserRepository{
+ DB: db,
+ SecretEncryptionKey: testutil.SecretEncryptionKey(),
+ HashingKey: testutil.HashingKey(),
+ }
+ if err := userRepo.Delete(userID); err != nil {
+ t.Fatalf("failed to delete user: %v", err)
+ }
+ controller := &UserController{
+ UserRepo: userRepo,
+ DataCleanupRepo: &cleanuprepo.Repository{DB: db},
+ SecretEncryptionKey: testutil.SecretEncryptionKey(),
+ HashingKey: testutil.HashingKey(),
+ }
+
+ cleanupTx, err := db.BeginTx(t.Context(), nil)
+ if err != nil {
+ t.Fatalf("failed to start cleanup transaction: %v", err)
+ }
+ defer cleanupTx.Rollback()
+ if _, err := cleanupTx.ExecContext(t.Context(),
+ `UPDATE data_cleanup SET stage = $1 WHERE user_id = $2`,
+ cleanupentity.Collection,
+ userID,
+ ); err != nil {
+ t.Fatalf("failed to advance cleanup: %v", err)
+ }
+
+ gin.SetMode(gin.TestMode)
+ ctx, _ := gin.CreateTestContext(httptest.NewRecorder())
+ recoveryDone := make(chan error, 1)
+ go func() {
+ recoveryDone <- controller.HandleAccountRecovery(ctx, ente.RecoverAccountRequest{
+ UserID: userID,
+ EmailID: "[email protected]",
+ })
+ }()
+ waitForDataCleanupRowLock(t, db)
+ if err := cleanupTx.Commit(); err != nil {
+ t.Fatalf("failed to commit cleanup stage advance: %v", err)
+ }
+
+ select {
+ case err := <-recoveryDone:
+ if !errors.Is(err, ErrAccountRecoveryUnavailable) {
+ t.Fatalf("HandleAccountRecovery() error = %v, want ErrAccountRecoveryUnavailable", err)
+ }
+ case <-stdtime.After(2 * stdtime.Second):
+ t.Fatal("account recovery did not finish after cleanup committed")
+ }
+ if _, err := userRepo.Get(userID); !errors.Is(err, ente.ErrUserDeleted) {
+ t.Fatalf("user state after lost recovery race = %v, want deleted", err)
+ }
+ var stage cleanupentity.Stage
+ if err := db.QueryRow(`SELECT stage FROM data_cleanup WHERE user_id = $1`, userID).Scan(&stage); err != nil {
+ t.Fatalf("failed to read cleanup stage: %v", err)
+ }
+ if stage != cleanupentity.Collection {
+ t.Fatalf("cleanup stage = %q, want %q", stage, cleanupentity.Collection)
+ }
+}
+
+func waitForDataCleanupRowLock(t *testing.T, db *sql.DB) {
+ t.Helper()
+ deadline := stdtime.Now().Add(2 * stdtime.Second)
+ for {
+ var waiting bool
+ err := db.QueryRow(`SELECT EXISTS(
+ SELECT 1 FROM pg_stat_activity
+ WHERE datname = current_database()
+ AND pid <> pg_backend_pid()
+ AND wait_event_type = 'Lock'
+ AND query LIKE '%data_cleanup%'
+ AND query LIKE '%FOR UPDATE%'
+ )`).Scan(&waiting)
+ if err != nil {
+ t.Fatalf("failed to inspect recovery lock wait: %v", err)
+ }
+ if waiting {
+ return
+ }
+ if stdtime.Now().After(deadline) {
+ t.Fatal("account recovery did not wait for the cleanup row lock")
+ }
+ stdtime.Sleep(10 * stdtime.Millisecond)
+ }
+}
+
func accountRecoveryToken(t *testing.T, controller *UserController, claim *enteJWT.WebCommonJWTClaim) string {
t.Helper()
token, err := controller.GetJWTTokenForClaim(claim)
@@ -432,6 +534,11 @@ func insertKeyAttributes(t *testing.T, db *sql.DB, userID int64) {
func insertScheduledDelete(t *testing.T, db *sql.DB, userID int64) {
t.Helper()
+ t.Cleanup(func() {
+ if _, err := db.Exec(`DELETE FROM data_cleanup WHERE user_id = $1`, userID); err != nil {
+ t.Errorf("failed to remove data_cleanup fixture for user %d: %v", userID, err)
+ }
+ })
_, err := db.Exec(
`INSERT INTO data_cleanup(user_id, stage, stage_schedule_time, stage_attempt_count)
VALUES($1, $2, $3, $4)`,
diff --git a/server/pkg/repo/datacleanup/repository.go b/server/pkg/repo/datacleanup/repository.go
index 60d41616c9..2e045fb280 100644
--- a/server/pkg/repo/datacleanup/repository.go
+++ b/server/pkg/repo/datacleanup/repository.go
@@ -19,14 +19,32 @@ func (r *Repository) Insert(ctx context.Context, userID int64) error {
return stacktrace.Propagate(err, "failed to insert")
}
+func (r *Repository) LockScheduledDelete(ctx context.Context, tx *sql.Tx, userID int64) error {
+ var lockedUserID int64
+ return tx.QueryRowContext(ctx, `SELECT user_id FROM data_cleanup
+ WHERE user_id = $1 AND stage = $2 FOR UPDATE`, userID, entity.Scheduled).Scan(&lockedUserID)
+}
+
func (r *Repository) RemoveScheduledDelete(ctx context.Context, userID int64) error {
- res, execErr := r.DB.ExecContext(ctx, `DELETE from data_cleanup where user_id= $1 and stage = $2`, userID, entity.Scheduled)
- if execErr != nil {
- return execErr
+ return removeScheduledDelete(ctx, r.DB, userID)
+}
+
+func (r *Repository) RemoveScheduledDeleteTx(ctx context.Context, tx *sql.Tx, userID int64) error {
+ return removeScheduledDelete(ctx, tx, userID)
+}
+
+type scheduledDeleteExecutor interface {
+ ExecContext(context.Context, string, ...any) (sql.Result, error)
+}
+
+func removeScheduledDelete(ctx context.Context, executor scheduledDeleteExecutor, userID int64) error {
+ result, err := executor.ExecContext(ctx, `DELETE FROM data_cleanup WHERE user_id = $1 AND stage = $2`, userID, entity.Scheduled)
+ if err != nil {
+ return err
}
- affected, affErr := res.RowsAffected()
- if affErr != nil {
- return affErr
+ affected, err := result.RowsAffected()
+ if err != nil {
+ return err
}
if affected != 1 {
return fmt.Errorf("only one row should have been affected, got %d", affected)
diff --git a/server/pkg/repo/user.go b/server/pkg/repo/user.go
index ffe8dcd527..c3d867d1cd 100644
--- a/server/pkg/repo/user.go
+++ b/server/pkg/repo/user.go
@@ -41,6 +41,10 @@ type UserInactivityCandidate struct {
LastActivity int64
}
+type emailUpdateExecutor interface {
+ ExecContext(context.Context, string, ...any) (sql.Result, error)
+}
+
// Get returns a user indicated by the userID
func (repo *UserRepository) Get(userID int64) (ente.User, error) {
var user ente.User
@@ -324,7 +328,15 @@ func (repo *UserRepository) UpdateDeleteFeedback(userID int64, feedback map[stri
// UpdateEmail updates the email address of a user
func (repo *UserRepository) UpdateEmail(userID int64, encryptedEmail ente.EncryptionResult, emailHash string) error {
- _, err := repo.DB.Exec(`UPDATE users SET encrypted_email = $1, email_decryption_nonce = $2, email_hash = $3 WHERE user_id = $4`, encryptedEmail.Cipher, encryptedEmail.Nonce, emailHash, userID)
+ return updateEmail(context.Background(), repo.DB, userID, encryptedEmail, emailHash)
+}
+
+func (repo *UserRepository) UpdateEmailTx(ctx context.Context, tx *sql.Tx, userID int64, encryptedEmail ente.EncryptionResult, emailHash string) error {
+ return updateEmail(ctx, tx, userID, encryptedEmail, emailHash)
+}
+
+func updateEmail(ctx context.Context, executor emailUpdateExecutor, userID int64, encryptedEmail ente.EncryptionResult, emailHash string) error {
+ _, err := executor.ExecContext(ctx, `UPDATE users SET encrypted_email = $1, email_decryption_nonce = $2, email_hash = $3 WHERE user_id = $4`, encryptedEmail.Cipher, encryptedEmail.Nonce, emailHash, userID)
return stacktrace.Propagate(err, "")
}