Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Prune server file data comments

Manav Rathic4267eb476c8
diff --git a/server/ente/filedata/filedata.go b/server/ente/filedata/filedata.go
index 234c9cc4d1..8ee26ceaec 100644
--- a/server/ente/filedata/filedata.go
+++ b/server/ente/filedata/filedata.go
@@ -6,21 +6,6 @@ import (
"github.com/ente/museum/ente"
)
-/*
-We store three types of derived data from a file, whose information is stored in the file_data table.
-Each derived data can have multiple objects, and each object is stored in the S3 bucket.
-1) MLData: This is the derived data from the file that is used for machine learning purposes.There's only
-one object for S3FileMetadata type.
-2) PreviewVideo: This is the derived data from the file that is used for previewing the video. This contains two objects.
-2.1) One object of type S3FileMetadata that contains the encrypted HLS playlist.
-2.2) Second object contains the encrypted video. The objectKey for this object is derived via ObjectKey function. The OG size column in the file_data
-contains sum of S3Metadata object size and the video object size. The object size is stored in the ObjectSize column.
-
-3) PreviewImage: This is the derived data from the file that is used for previewing the image. This just contain one object.
-The objectKey for this object is derived via ObjectKey function. We also store the nonce of the object in the ObjectNonce column.
-ObjectNonce is not stored for PreviewVideo type as HLS playlist contains a random key, that's only used once to encrypt the video with default nonce.
-*/
-
type Entity struct {
FileID int64 `json:"fileID"`
Type ente.ObjectType `json:"type"`
@@ -44,7 +29,6 @@ type FDStatus struct {
UpdatedAt int64 `json:"updatedAt" binding:"required"`
}
-// GetFilesData should only be used for getting the preview video playlist and derived metadata.
type GetFilesData struct {
FileIDs []int64 `json:"fileIDs" binding:"required"`
Type ente.ObjectType `json:"type" binding:"required"`
@@ -132,6 +116,10 @@ func (g *PreviewUploadUrlRequest) Validate() error {
return nil
}
+// A row represents one derived asset. ML data and image previews each use one
+// S3 object. A video preview uses two: encrypted HLS playlist metadata and the
+// encrypted video. For videos, Size covers both and ObjectSize covers the
+// video.
type Row struct {
FileID int64
UserID int64
diff --git a/server/ente/filedata/path.go b/server/ente/filedata/path.go
index e7b78a30f8..062081dbcb 100644
--- a/server/ente/filedata/path.go
+++ b/server/ente/filedata/path.go
@@ -7,8 +7,7 @@ import (
"github.com/ente/museum/ente/base"
)
-// BasePrefix returns the base prefix for all objects related to a file. To check if the file data is deleted,
-// ensure that there's no file in the S3 bucket with this prefix.
+// File data is fully deleted only when no objects remain under this prefix.
func BasePrefix(fileID int64, ownerID int64) string {
return fmt.Sprintf("%d/file-data/%d/", ownerID, fileID)
}
diff --git a/server/ente/filedata/putfiledata.go b/server/ente/filedata/putfiledata.go
index e3df82906b..e45c23db24 100644
--- a/server/ente/filedata/putfiledata.go
+++ b/server/ente/filedata/putfiledata.go
@@ -11,7 +11,7 @@ type PutFileDataRequest struct {
EncryptedData *string `json:"encryptedData,omitempty"`
DecryptionHeader *string `json:"decryptionHeader,omitempty"`
Version *int `json:"version,omitempty"`
- // Used to ensure that the client has correct state before it tries to update the metadata
+ // Used to reject updates based on stale client state.
LastUpdatedAt *int64 `json:"lastUpdatedAt,omitempty"`
}
diff --git a/server/ente/passkey.go b/server/ente/passkey.go
index 43c40dd5d9..2ab8b47908 100644
--- a/server/ente/passkey.go
+++ b/server/ente/passkey.go
@@ -14,7 +14,8 @@ var MaxPasskeys = 10
type SetPasskeyRecoveryRequest struct {
Secret string `json:"secret" binding:"required"`
- // SkipSecret encrypted with the user's recovery key.
+ // The skip secret encrypted with the recovery key. Decrypting it during MFA
+ // recovery proves possession of that key.
UserSecretCipher string `json:"userSecretCipher" binding:"required"`
UserSecretNonce string `json:"userSecretNonce" binding:"required"`
}
diff --git a/server/pkg/controller/filedata/delete.go b/server/pkg/controller/filedata/delete.go
index 693b88732e..39d7c97069 100644
--- a/server/pkg/controller/filedata/delete.go
+++ b/server/pkg/controller/filedata/delete.go
@@ -31,8 +31,6 @@ func (c *Controller) delete(i int) {
for {
err := c.tryDelete()
if err != nil {
- // Sleep in proportion to the (arbitrary) index to space out the
- // workers further.
time.Sleep(time.Duration(i+5) * time.Minute)
}
}
diff --git a/server/pkg/controller/filedata/replicate.go b/server/pkg/controller/filedata/replicate.go
index 8f311d675d..4141d40f57 100644
--- a/server/pkg/controller/filedata/replicate.go
+++ b/server/pkg/controller/filedata/replicate.go
@@ -73,8 +73,6 @@ func (c *Controller) replicate(i int) {
for {
err := c.tryReplicate()
if err != nil {
- // Sleep in proportion to the (arbitrary) index to space out the
- // workers further.
time.Sleep(time.Duration(i+1) * time.Minute)
}
}
diff --git a/server/pkg/repo/filedata/repository.go b/server/pkg/repo/filedata/repository.go
index fc3e17e059..402bee2df8 100644
--- a/server/pkg/repo/filedata/repository.go
+++ b/server/pkg/repo/filedata/repository.go
@@ -26,8 +26,7 @@ const (
)
func (r *Repository) InsertOrUpdate(ctx context.Context, data filedata.Row) error {
- // During insert, we set the sync_locked_till to 5 minutes in the future. This is to prevent
- // immediate replication of the file data row, that can result in failure of update/retry requests
+ // Delay replication to avoid failing immediate update retries.
query := `
INSERT INTO file_data
(file_id, user_id, data_type, size, latest_bucket, sync_locked_till)
@@ -228,8 +227,6 @@ func (r *Repository) MoveBetweenBuckets(row filedata.Row, bucketID string, sourc
return nil
}
-// GetPendingSyncDataAndExtendLock in a transaction gets single file data row that has been deleted and pending sync is true and sync_lock_till is less than now_utc_micro_seconds() and extends the lock till newSyncLockTime
-// This is used to lock the file data row for deletion and extend
func (r *Repository) GetPendingSyncDataAndExtendLock(ctx context.Context, newSyncLockTime int64, forDeletion bool) (*filedata.Row, error) {
if newSyncLockTime < time.Now().Add(5*time.Minute).UnixMicro() {
return nil, stacktrace.NewError("newSyncLockTime should be at least 5min in the future")
@@ -263,8 +260,6 @@ func (r *Repository) GetPendingSyncDataAndExtendLock(ctx context.Context, newSyn
return &fileData, nil
}
-// MarkReplicationAsDone marks the pending_sync as false for the file data row, while
-// ensuring that the row is not deleted
func (r *Repository) MarkReplicationAsDone(ctx context.Context, row filedata.Row) error {
query := `UPDATE file_data SET pending_sync = false WHERE is_deleted=false and file_id = $1 AND data_type = $2 AND user_id = $3`
_, err := r.DB.ExecContext(ctx, query, row.FileID, string(row.Type), row.UserID)
@@ -284,8 +279,7 @@ func (r *Repository) RegisterReplicationAttempt(ctx context.Context, row filedat
return nil
}
-// ResetSyncLock resets the sync_locked_till to now_utc_micro_seconds() for the file data row only if pending_sync is false and
-// the input syncLockedTill is equal to the existing sync_locked_till. This is used to reset the lock after the replication is done
+// Reset only after replication finishes and only if the lock has not changed.
func (r *Repository) ResetSyncLock(ctx context.Context, row filedata.Row, syncLockedTill int64) error {
query := `UPDATE file_data SET sync_locked_till = now_utc_micro_seconds() WHERE pending_sync = false and file_id = $1 AND data_type = $2 AND user_id = $3 AND sync_locked_till = $4`
_, err := r.DB.ExecContext(ctx, query, row.FileID, string(row.Type), row.UserID, syncLockedTill)
diff --git a/server/pkg/utils/s3config/s3config.go b/server/pkg/utils/s3config/s3config.go
index af32545116..a19ac6bbb1 100644
--- a/server/pkg/utils/s3config/s3config.go
+++ b/server/pkg/utils/s3config/s3config.go
@@ -31,6 +31,15 @@ type S3Config struct {
attachmentConfig AttachmentConfig
}
+// The primary object-replication path uses three provider-specific roles:
+//
+// Backblaze is primary hot storage. It is versioned, and the server rolls back
+// client overwrites by deleting the newest object version.
+//
+// Wasabi is secondary hot storage. Compliance prevents overwrites and deletion
+// for 21 days. Permanent deletion releases the hold before scheduled cleanup.
+//
+// Scaleway is cold storage. Replication uploads use the GLACIER storage class.
var (
dcB2EuropeCentral string = "b2-eu-cen"
dcSCWEuropeFranceDeprecated string = "scw-eu-fr"