π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Prune server file data comments
Manav Rathic4267eb476c8
diff --git a/server/ente/filedata/filedata.go b/server/ente/filedata/filedata.goindex 234c9cc4d1..8ee26ceaec 100644--- a/server/ente/filedata/filedata.go+++ b/server/ente/filedata/filedata.go@@ -6,21 +6,6 @@ import ("github.com/ente/museum/ente")-/*-We store three types of derived data from a file, whose information is stored in the file_data table.-Each derived data can have multiple objects, and each object is stored in the S3 bucket.-1) MLData: This is the derived data from the file that is used for machine learning purposes.There's only-one object for S3FileMetadata type.-2) PreviewVideo: This is the derived data from the file that is used for previewing the video. This contains two objects.-2.1) One object of type S3FileMetadata that contains the encrypted HLS playlist.-2.2) Second object contains the encrypted video. The objectKey for this object is derived via ObjectKey function. The OG size column in the file_data-contains sum of S3Metadata object size and the video object size. The object size is stored in the ObjectSize column.--3) PreviewImage: This is the derived data from the file that is used for previewing the image. This just contain one object.-The objectKey for this object is derived via ObjectKey function. We also store the nonce of the object in the ObjectNonce column.-ObjectNonce is not stored for PreviewVideo type as HLS playlist contains a random key, that's only used once to encrypt the video with default nonce.-*/-type Entity struct {FileID int64 `json:"fileID"`Type ente.ObjectType `json:"type"`@@ -44,7 +29,6 @@ type FDStatus struct {UpdatedAt int64 `json:"updatedAt" binding:"required"`}-// GetFilesData should only be used for getting the preview video playlist and derived metadata.type GetFilesData struct {FileIDs []int64 `json:"fileIDs" binding:"required"`Type ente.ObjectType `json:"type" binding:"required"`@@ -132,6 +116,10 @@ func (g *PreviewUploadUrlRequest) Validate() error {return nil}+// A row represents one derived asset. ML data and image previews each use one+// S3 object. A video preview uses two: encrypted HLS playlist metadata and the+// encrypted video. For videos, Size covers both and ObjectSize covers the+// video.type Row struct {FileID int64UserID int64diff --git a/server/ente/filedata/path.go b/server/ente/filedata/path.goindex e7b78a30f8..062081dbcb 100644--- a/server/ente/filedata/path.go+++ b/server/ente/filedata/path.go@@ -7,8 +7,7 @@ import ("github.com/ente/museum/ente/base")-// BasePrefix returns the base prefix for all objects related to a file. To check if the file data is deleted,-// ensure that there's no file in the S3 bucket with this prefix.+// File data is fully deleted only when no objects remain under this prefix.func BasePrefix(fileID int64, ownerID int64) string {return fmt.Sprintf("%d/file-data/%d/", ownerID, fileID)}diff --git a/server/ente/filedata/putfiledata.go b/server/ente/filedata/putfiledata.goindex e3df82906b..e45c23db24 100644--- a/server/ente/filedata/putfiledata.go+++ b/server/ente/filedata/putfiledata.go@@ -11,7 +11,7 @@ type PutFileDataRequest struct {EncryptedData *string `json:"encryptedData,omitempty"`DecryptionHeader *string `json:"decryptionHeader,omitempty"`Version *int `json:"version,omitempty"`- // Used to ensure that the client has correct state before it tries to update the metadata+ // Used to reject updates based on stale client state.LastUpdatedAt *int64 `json:"lastUpdatedAt,omitempty"`}diff --git a/server/ente/passkey.go b/server/ente/passkey.goindex 43c40dd5d9..2ab8b47908 100644--- a/server/ente/passkey.go+++ b/server/ente/passkey.go@@ -14,7 +14,8 @@ var MaxPasskeys = 10type SetPasskeyRecoveryRequest struct {Secret string `json:"secret" binding:"required"`- // SkipSecret encrypted with the user's recovery key.+ // The skip secret encrypted with the recovery key. Decrypting it during MFA+ // recovery proves possession of that key.UserSecretCipher string `json:"userSecretCipher" binding:"required"`UserSecretNonce string `json:"userSecretNonce" binding:"required"`}diff --git a/server/pkg/controller/filedata/delete.go b/server/pkg/controller/filedata/delete.goindex 693b88732e..39d7c97069 100644--- a/server/pkg/controller/filedata/delete.go+++ b/server/pkg/controller/filedata/delete.go@@ -31,8 +31,6 @@ func (c *Controller) delete(i int) {for {err := c.tryDelete()if err != nil {- // Sleep in proportion to the (arbitrary) index to space out the- // workers further.time.Sleep(time.Duration(i+5) * time.Minute)}}diff --git a/server/pkg/controller/filedata/replicate.go b/server/pkg/controller/filedata/replicate.goindex 8f311d675d..4141d40f57 100644--- a/server/pkg/controller/filedata/replicate.go+++ b/server/pkg/controller/filedata/replicate.go@@ -73,8 +73,6 @@ func (c *Controller) replicate(i int) {for {err := c.tryReplicate()if err != nil {- // Sleep in proportion to the (arbitrary) index to space out the- // workers further.time.Sleep(time.Duration(i+1) * time.Minute)}}diff --git a/server/pkg/repo/filedata/repository.go b/server/pkg/repo/filedata/repository.goindex fc3e17e059..402bee2df8 100644--- a/server/pkg/repo/filedata/repository.go+++ b/server/pkg/repo/filedata/repository.go@@ -26,8 +26,7 @@ const ()func (r *Repository) InsertOrUpdate(ctx context.Context, data filedata.Row) error {- // During insert, we set the sync_locked_till to 5 minutes in the future. This is to prevent- // immediate replication of the file data row, that can result in failure of update/retry requests+ // Delay replication to avoid failing immediate update retries.query := `INSERT INTO file_data(file_id, user_id, data_type, size, latest_bucket, sync_locked_till)@@ -228,8 +227,6 @@ func (r *Repository) MoveBetweenBuckets(row filedata.Row, bucketID string, sourcreturn nil}-// GetPendingSyncDataAndExtendLock in a transaction gets single file data row that has been deleted and pending sync is true and sync_lock_till is less than now_utc_micro_seconds() and extends the lock till newSyncLockTime-// This is used to lock the file data row for deletion and extendfunc (r *Repository) GetPendingSyncDataAndExtendLock(ctx context.Context, newSyncLockTime int64, forDeletion bool) (*filedata.Row, error) {if newSyncLockTime < time.Now().Add(5*time.Minute).UnixMicro() {return nil, stacktrace.NewError("newSyncLockTime should be at least 5min in the future")@@ -263,8 +260,6 @@ func (r *Repository) GetPendingSyncDataAndExtendLock(ctx context.Context, newSynreturn &fileData, nil}-// MarkReplicationAsDone marks the pending_sync as false for the file data row, while-// ensuring that the row is not deletedfunc (r *Repository) MarkReplicationAsDone(ctx context.Context, row filedata.Row) error {query := `UPDATE file_data SET pending_sync = false WHERE is_deleted=false and file_id = $1 AND data_type = $2 AND user_id = $3`_, err := r.DB.ExecContext(ctx, query, row.FileID, string(row.Type), row.UserID)@@ -284,8 +279,7 @@ func (r *Repository) RegisterReplicationAttempt(ctx context.Context, row filedatreturn nil}-// ResetSyncLock resets the sync_locked_till to now_utc_micro_seconds() for the file data row only if pending_sync is false and-// the input syncLockedTill is equal to the existing sync_locked_till. This is used to reset the lock after the replication is done+// Reset only after replication finishes and only if the lock has not changed.func (r *Repository) ResetSyncLock(ctx context.Context, row filedata.Row, syncLockedTill int64) error {query := `UPDATE file_data SET sync_locked_till = now_utc_micro_seconds() WHERE pending_sync = false and file_id = $1 AND data_type = $2 AND user_id = $3 AND sync_locked_till = $4`_, err := r.DB.ExecContext(ctx, query, row.FileID, string(row.Type), row.UserID, syncLockedTill)diff --git a/server/pkg/utils/s3config/s3config.go b/server/pkg/utils/s3config/s3config.goindex af32545116..a19ac6bbb1 100644--- a/server/pkg/utils/s3config/s3config.go+++ b/server/pkg/utils/s3config/s3config.go@@ -31,6 +31,15 @@ type S3Config struct {attachmentConfig AttachmentConfig}+// The primary object-replication path uses three provider-specific roles:+//+// Backblaze is primary hot storage. It is versioned, and the server rolls back+// client overwrites by deleting the newest object version.+//+// Wasabi is secondary hot storage. Compliance prevents overwrites and deletion+// for 21 days. Permanent deletion releases the hold before scheduled cleanup.+//+// Scaleway is cold storage. Replication uploads use the GLACIER storage class.var (dcB2EuropeCentral string = "b2-eu-cen"dcSCWEuropeFranceDeprecated string = "scw-eu-fr"