💚 End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Prune server comment kernels
Manav Rathi9d3a0a1a171a
diff --git a/server/ente/trash.go b/server/ente/trash.goindex e92096a6a2..0a8c58023d 100644--- a/server/ente/trash.go+++ b/server/ente/trash.go@@ -11,33 +11,19 @@ type Trash struct {type DeleteTrashFilesRequest struct {FileIDs []int64 `json:"fileIDs" binding:"required"`- // OwnerID will be set based on the authenticated user+ // Set from the authenticated user after binding.OwnerID int64}type EmptyTrashRequest struct {- // LastUpdatedAt timestamp will be used to delete trashed files with updatedAt timestamp <= LastUpdatedAt- // User's trash will be cleaned up in an async manner. The timestamp is used to ensure that newly trashed files- // are not deleted due to delay in the async operation.+ // Empty only entries at or before this timestamp so queued work cannot+ // delete newly trashed files.LastUpdatedAt int64 `json:"lastUpdatedAt" binding:"required"`}-// In V3, while trashing/deleting any album, the user can decide to either keep or delete the all files which are-// present in to the trash. When user wants to keep the files, the clients are expected to move all the files from-// the underlying collection to any other collection owned by the user, inlcuding uncategorized.-// Note: Collection Delete Versions for DELETE /collections/V../ endpoint-// V1: All files which exclusively belong to the collections are deleted immediately.-// V2: All files which exclusively belong to the collections are moved to the trash.-// V3: All files which are still present in the collection (irrespective if they blong to another collection) will be moved to trash.-// V3 is introduced to avoid doing this booking on server, where we only delete a file when it's beling removed from the last collection it longs to.-// In theory above logic to delete when it's being removed from last collection sounds good. But,-// in practice it complicates the code (thus reducing its robustness) because of race conditions, and it's-// also hard to communicate it to the user. So, to simplify things, in V3, the files will be only deleted when user tell us to delete them.type TrashCollectionV3Request struct {CollectionID int64 `json:"collectionID" form:"collectionID" binding:"required"`- // When KeepFiles is false, then all the files which are present in the collection will be moved to trash.- // When KeepFiles is true, but the underlying collection still contains file, then the API call will fail.- // This is to ensure that before deleting the collection, the client has moved all relevant files to any other- // collection owned by the user, including Uncategorized.+ // If true, deletion requires an empty collection. Otherwise remaining files+ // move to trash.KeepFiles *bool `json:"keepFiles" form:"keepFiles" binding:"required"`}diff --git a/server/pkg/controller/collections/file_action.go b/server/pkg/controller/collections/file_action.goindex 84609aa223..627338bf95 100644--- a/server/pkg/controller/collections/file_action.go+++ b/server/pkg/controller/collections/file_action.go@@ -150,15 +150,6 @@ func (c *CollectionController) MoveFiles(ctx *gin.Context, req ente.MoveFilesReqreturn stacktrace.Propagate(err, "")}-// RemoveFilesV3 enforces all removal rules for shared collections:-// 1. accessCtrl must confirm the actor participates in the collection;-// 2. collaborators/viewers may only remove the files they added themselves;-// 3. the collection owner may remove files added by others but never their own;-// 4. admins can remove anyone's files, but a collection owner's files are only-// queued for removal (REMOVE action + pending collection_action entry) so-// the owner can act on them;-// 5. once the validations pass, non-owner files are deleted immediately via-// CollectionRepo.RemoveFilesV3.func (c *CollectionController) RemoveFilesV3(ctx *gin.Context, actorUserID int64, req ente.RemoveFilesV3Request) error {accessResp, err := c.AccessCtrl.GetCollection(ctx, &access.GetCollectionParams{CollectionID: req.CollectionID,@@ -221,14 +212,6 @@ func (c *CollectionController) RemoveFilesV3(ctx *gin.Context, actorUserID int64return nil}-// SuggestDeleteInSharedCollection allows collection owners/admins to nudge other-// participants to delete their files:-// 1. only OWNER/ADMIN roles pass the access check;-// 2. every file ID must belong to the collection and none may belong to the acting user;-// 3. the method internally reuses RemoveFilesV3 to enforce role-based rules and-// to actually detach the files from the collection;-// 4. each remote owner then receives DELETE_SUGGESTED actions so their clients-// can surface the pending delete request.func (c *CollectionController) SuggestDeleteInSharedCollection(ctx *gin.Context, req ente.SuggestDeleteRequest) error {actorUserID := auth.GetUserID(ctx.Request.Header)accessResp, err := c.AccessCtrl.GetCollection(ctx, &access.GetCollectionParams{diff --git a/server/pkg/controller/collections/files_diff.go b/server/pkg/controller/collections/files_diff.goindex 069bca5c2c..98e1131231 100644--- a/server/pkg/controller/collections/files_diff.go+++ b/server/pkg/controller/collections/files_diff.go@@ -29,9 +29,7 @@ func (c *CollectionController) GetDiffV2(ctx *gin.Context, cID int64, userID intif err != nil {return nil, false, stacktrace.Propagate(err, "")}- // hide private metadata before returning files info in difffor idx := range diff {- // Treat action markers as soft-deletes for non-ownersif diff[idx].Action != nil && !diff[idx].IsDeleted {if *diff[idx].Action == ente.ActionRemove || *diff[idx].Action == ente.ActionDeleteSuggested {if diff[idx].OwnerID != userID { // non-owner view: mask as deleted@@ -45,7 +43,7 @@ func (c *CollectionController) GetDiffV2(ctx *gin.Context, cID int64, userID intdiff[idx].ActionUserID = nil}if diff[idx].Metadata.EncryptedData == "-" && !diff[idx].IsDeleted {- // This indicates that the file is deleted, but we still have a stale entry in the collection+ // "-" marks a deleted file whose collection entry is stale.log.WithFields(log.Fields{"file_id": diff[idx].ID,"collection_id": cID,@@ -57,37 +55,22 @@ func (c *CollectionController) GetDiffV2(ctx *gin.Context, cID int64, userID intreturn diff, hasMore, nil}-// getDiff returns the diff in user's collection since a timestamp, along with hasMore bool flag.-// The function will never return partial result for a version. To maintain this promise, it will not be able to honor-// the limit parameter. Based on the db state, compared to the limit, the diff length can be-// less (case 1), more (case 2), or same (case 3, 4)-// Example: Assume we have 11 files with following versions: v0, v1, v1, v1, v1, v1, v1, v1, v2, v2, v2 (count = 7 v1, 3 v2)-// client has synced up till version v0.-// case 1: ( sinceTime: v0, limit = 8):-// The method will discard the entries with version v2 and return only 7 entries with version v1.-// case 2: (sinceTime: v0, limit 5):-// Instead of returning 5 entries with version V1, method will return all 7 entries with version v1.-// case 3: (sinceTime: v0, limit 7):-// The method will return all 7 entries with version V1.-// case 4: (sinceTime: v0, limit >=10):-// The method will all 10 entries in the diff+// Never split a version across pages. Results may be smaller or larger than+// limit so every row with the boundary timestamp stays together.func (c *CollectionController) getDiff(cID int64, sinceTime int64, limit int, logger *log.Entry) ([]ente.File, bool, error) {- // request for limit +1 filesdiffLimitPlusOne, err := c.CollectionRepo.GetDiff(cID, sinceTime, limit+1)if err != nil {return nil, false, stacktrace.Propagate(err, "")}if len(diffLimitPlusOne) <= limit {- // case 4: all files changed after sinceTime are included.return diffLimitPlusOne, false, nil}lastFileVersion := diffLimitPlusOne[limit].UpdationTimefilteredDiffs := c.removeFilesWithVersion(diffLimitPlusOne, lastFileVersion)filteredDiffLen := len(filteredDiffs)- if filteredDiffLen > 0 { // case 1 or case 3+ if filteredDiffLen > 0 {if filteredDiffLen < limit {- // logging case 1logger.WithField("last_file_version", lastFileVersion).WithField("filtered_diff_len", filteredDiffLen).@@ -95,7 +78,6 @@ func (c *CollectionController) getDiff(cID int64, sinceTime int64, limit int, lo}return filteredDiffs, true, nil}- // case 2diff, err := c.CollectionRepo.GetFilesWithVersion(cID, lastFileVersion)logger.WithField("last_file_version", lastFileVersion).@@ -107,13 +89,11 @@ func (c *CollectionController) getDiff(cID int64, sinceTime int64, limit int, loreturn diff, true, nil}-// removeFilesWithVersion returns filtered list of files are removing all files with given version.-// Important: The method assumes that files are sorted by increasing order of File.UpdationTime+// files must be sorted by increasing UpdationTime.func (c *CollectionController) removeFilesWithVersion(files []ente.File, version int64) []ente.File {var i = len(files) - 1for ; i >= 0; i-- {if files[i].UpdationTime != version {- // found index (from end) where file's version is different from given versionbreak}}diff --git a/server/pkg/controller/object_cleanup.go b/server/pkg/controller/object_cleanup.goindex b4c080e723..5335c3e8d3 100644--- a/server/pkg/controller/object_cleanup.go+++ b/server/pkg/controller/object_cleanup.go@@ -19,14 +19,6 @@ import (log "github.com/sirupsen/logrus")-// ObjectCleanupController exposes functions to remove temporary object storage-// entries that were never committed to the database.-//-// 1. We create presigned URLs for clients to upload their objects to. It might-// happen that the client is able to successfully upload to these URLs, but-// not tell museum about the successful upload.-//-// 2. During replication, we might have half-done multipart uploads.type ObjectCleanupController struct {Repo *repo.ObjectCleanupRepositoryObjectRepo *repo.ObjectRepository@@ -105,9 +97,7 @@ func (c *ObjectCleanupController) removeUnreportedObjects() int {logger.Infof("Removed %d objects", count)- // We always commit the transaction, even on errors for individual rows. To- // avoid object getting stuck in a loop, we increase their expiry times.-+ // Always commit the batch. skip() delays failed rows to prevent a tight loop.cerr := tx.Commit()if cerr != nil {cerr = stacktrace.Propagate(err, "Failed to commit transaction")@@ -171,12 +161,6 @@ func (c *ObjectCleanupController) removeUnreportedObject(tx *sql.Tx, t ente.Tempreturn nil}-// AddTempObjectKey creates a new temporary object entry.-//-// It persists a given object key as having been provided to a client for-// uploading. If a client does not successfully mark this object's upload as-// having completed within PreSignedRequestValidityDuration, this temp object-// will be cleaned up.func (c *ObjectCleanupController) AddTempObjectKey(objectKey string, dc string) error {expiry := time.Microseconds() + (2 * PreSignedRequestValidityDuration.Microseconds())return c.addCleanupEntryForObjectKey(objectKey, dc, expiry)@@ -298,7 +282,6 @@ func (c *ObjectCleanupController) abortMultipartUpload(objectKey string, uploadI})if err != nil {if isUnknownUploadError(err) {- // This is expected now, since we just aborted the uploadreturn nil}return stacktrace.Propagate(err, "")diff --git a/server/pkg/controller/replication3.go b/server/pkg/controller/replication3.goindex 8f681df4f9..bd724dde72 100644--- a/server/pkg/controller/replication3.go+++ b/server/pkg/controller/replication3.go@@ -32,19 +32,6 @@ const (replicationDelayForStaleObjects = 30)-// ReplicationController3 oversees version 3 of our object replication.-//-// The user's encrypted data starts off in 1 hot storage (Backblaze "b2"). This-// controller then takes over and replicates it the other two replicas. It keeps-// state in the object_copies table.-//-// Both v2 and v3 of object replication use the same hot storage (b2), but they-// replicate to different buckets thereafter.-//-// The current implementation only works if the hot storage is b2. This is not-// an inherent limitation, however the code has not yet been tested in other-// scenarios, so there is a safety check preventing the replication from-// happening if the current hot storage is not b2.type ReplicationController3 struct {S3Config *s3config.S3ConfigObjectRepo *repo.ObjectRepository@@ -66,17 +53,10 @@ type UploadDestination struct {Uploader *s3manager.UploaderBucket *stringLabel string- // If true, we should ignore Wasabi 403 errors. See "Reuploads".HasComplianceHold boolIsGlacier bool}-// StartReplication starts the background replication process.-//-// This method returns synchronously. ReplicationController3 will create-// suitable number of goroutines to parallelize and perform the replication-// asynchronously, as and when it notices new files that have not yet been-// replicated (it does this by querying the object_copies table).func (c *ReplicationController3) StartReplication() error {hotDC := c.S3Config.GetHotDataCenter()if hotDC != c.S3Config.GetHotBackblazeDC() {@@ -152,12 +132,6 @@ func (c *ReplicationController3) createTemporaryStorage() error {}func (c *ReplicationController3) createDestinations() {- // The s3manager.Uploader objects are safe for use concurrently. From the- // AWS docs:- //- // > The Uploader structure that calls Upload(). It is safe to call Upload()- // on this structure for multiple objects and across concurrent goroutines.- // Mutating the Uploader's properties is not safe to be done concurrently.// Scaleway limits multipart uploads to 1,000 parts.limitUploadParts := func(u *s3manager.Uploader) {u.MaxUploadParts = 1000@@ -189,8 +163,7 @@ func (c *ReplicationController3) createDestinations() {Uploader: s3manager.NewUploaderWithClient(&scwClient, limitUploadParts),Bucket: config.GetBucket(scwDC),Label: "scaleway",- // should be true, except when running in a local cluster (since minio doesn't- // support specifying the GLACIER storage class).+ // MinIO does not support the GLACIER storage class.IsGlacier: !config.AreLocalBuckets(),}}@@ -199,19 +172,12 @@ func (c *ReplicationController3) replicate(i int) {for {err := c.tryReplicate()if err != nil {- // Sleep in proportion to the (arbitrary) index to space out the- // workers further.+ // Stagger retries across workers.time.Sleep(time.Duration(i+1) * time.Minute)}}}-// Try to replicate an object.-//-// Return nil if something was replicated, otherwise return the error.-//-// A common and expected error is `sql.ErrNoRows`, which occurs if there are no-// objects left to replicate currently.func (c *ReplicationController3) tryReplicate() error {ctxWithTimeout, cancel := context.WithTimeout(context.Background(), 10*time.Second)defer cancel()@@ -267,9 +233,7 @@ func (c *ReplicationController3) tryReplicate() error {}if ob.IsFileDeleted || ob.IsUserDeleted {- // Update the object_copies to mark this object as not requiring further- // replication. The row in object_copies will get deleted when the next- // scheduled object deletion runs.+ // Scheduled object deletion removes the object_copies row later.err = c.ObjectCopiesRepo.UnmarkFromReplication(objectKey)if err != nil {return done(stacktrace.Propagate(err, "Failed to mark an object not requiring further replication"))@@ -281,12 +245,6 @@ func (c *ReplicationController3) tryReplicate() error {err = fileutil.EnsureSufficientSpace(ob.Size)if err != nil {- // We don't have free space right now, maybe because other big files are- // being downloaded simultanously, but we might get space later, so mark- // a failed attempt that'll get retried later.- //- // Log this error though, so that it gets noticed if it happens too- // frequently (the instance might need a bigger disk).return done(stacktrace.Propagate(err, ""))}@@ -440,30 +398,16 @@ func (c *ReplicationController3) replicateFile(in *UploadInput, dest *UploadDestreturn failure(stacktrace.Propagate(err, "Failed to verify upload"))}- // The update of the object_keys is not done in the transaction where the- // other updates to object_copies table are made. This is so that the- // object_keys table (which is what'll be used to delete objects) is- // (almost) always updated if the file gets uploaded successfully.- //- // The only time the update wouldn't happen is if museum gets restarted- // between the successful completion of the upload to the bucket and this- // query getting executed.- //- // While possible, that is a much smaller window as compared to the- // transaction for updating object_copies, which could easily span minutes- // as the transaction ends only after the object has been uploaded to all- // replicas.+ // Record each successful upload in object_keys before updating object_copies.+ // The object_copies transaction can span all replica uploads; a restart in+ // that window must not leave an uploaded replica absent from deletion records.rowsAffected, err := c.ObjectRepo.MarkObjectReplicated(in.ObjectKey, dest.DC)if err != nil {return failure(stacktrace.Propagate(err, "Failed to update object_keys to mark replication as completed"))}if rowsAffected != 1 {- // It is possible that this row was updated earlier, after an upload- // that got completed but before object_copies table could be updated in- // the transaction (See "Reuploads").- //- // So do not treat this as an error.+ // A previous interrupted attempt may already have recorded this replica.logger.Warnf("Expected 1 row to be updated, but got %d", rowsAffected)}@@ -476,22 +420,9 @@ func (c *ReplicationController3) replicateFile(in *UploadInput, dest *UploadDestreturn nil}-// # Reuploads-//-// It is possible that the object might already exist on remote. The known-// scenario where this might happen is if museum gets restarted after having-// completed the upload but before it got around to modifying the DB.-//-// The behaviour in this case is remote dependent.-//-// - Uploading an object with the same key on Scaleway would work normally.-//-// - But trying to add an object with the same key on the compliance locked-// Wasabi would return an HTTP 403.-//-// We intercept the Wasabi 403 in this case and move ahead. The subsequent-// object verification using the HEAD request will act as a sanity check for-// the object.+// A restart can leave an uploaded replica unrecorded in object_copies.+// Compliance-locked Wasabi returns 403 when that upload is retried, so allow+// the HEAD size check to verify the existing object.func (c *ReplicationController3) uploadFile(in *UploadInput, dest *UploadDestination) error {in.File.Seek(0, io.SeekStart)diff --git a/server/pkg/controller/trash.go b/server/pkg/controller/trash.goindex 4b4f9cc117..15f44e2fd4 100644--- a/server/pkg/controller/trash.go+++ b/server/pkg/controller/trash.go@@ -32,7 +32,6 @@ func (t *TrashController) GetDiff(userID int64, sinceTime int64, app ente.App) (if err != nil {return nil, false, err}- // hide private metadata before returning files info in difffor _, trashFile := range trashFilesDiff {if trashFile.IsDeleted {trashFile.File.MagicMetadata = nil@@ -44,36 +43,21 @@ func (t *TrashController) GetDiff(userID int64, sinceTime int64, app ente.App) (return trashFilesDiff, hasMore, err}-// GetDiff returns the diff in user's trash since a timestamp, along with hasMore bool flag.-// The function will never return partial result for a version. To maintain this promise, it will not be able to honor-// the limit parameter. Based on the db state, compared to the limit, the diff length can be-// less (case 1), more (case 2), or same (case 3, 4)-// Example: Assume we have 11 files with following versions: v0, v1, v1, v1, v1, v1, v1, v1, v2, v2, v2 (count = 7 v1, 3 v2)-// client has synced up till version v0.-// case 1: ( sinceTime: v0, limit = 8):-// The method will discard the entries with version v2 and return only 7 entries with version v1.-// case 2: (sinceTime: v0, limit 5):-// Instead of returning 5 entries with version V1, method will return all 7 entries with version v1.-// case 3: (sinceTime: v0, limit 7):-// The method will return all 7 entries with version V1.-// case 4: (sinceTime: v0, limit >=10):-// The method will all 10 entries in the diff+// Never split a version across pages. Results may be smaller or larger than+// limit so every row with the boundary timestamp stays together.func (t *TrashController) getDiff(userID int64, sinceTime int64, limit int, app ente.App) ([]ente.Trash, bool, error) {- // request for limit +1 filesdiffLimitPlusOne, err := t.TrashRepo.GetDiff(userID, sinceTime, limit+1, app)if err != nil {return nil, false, stacktrace.Propagate(err, "")}if len(diffLimitPlusOne) <= limit {- // case 4: all files changed after sinceTime are included.return diffLimitPlusOne, false, nil}lastFileVersion := diffLimitPlusOne[limit].UpdatedAtfilteredDiffs := t.removeFilesWithVersion(diffLimitPlusOne, lastFileVersion)- if len(filteredDiffs) > 0 { // case 1 or case 3+ if len(filteredDiffs) > 0 {return filteredDiffs, true, nil}- // case 2diff, err := t.TrashRepo.GetFilesWithVersion(userID, lastFileVersion, app)if err != nil {return nil, false, stacktrace.Propagate(err, "")@@ -81,7 +65,6 @@ func (t *TrashController) getDiff(userID int64, sinceTime int64, limit int, appreturn diff, true, nil}-// Delete files permanently, queues up the file for deletion & free up the space based on file's object sizefunc (t *TrashController) Delete(ctx context.Context, request ente.DeleteTrashFilesRequest) error {err := t.TrashRepo.Delete(ctx, request.OwnerID, request.FileIDs)if err != nil {@@ -198,13 +181,11 @@ func (t *TrashController) DeleteAgedTrashedFiles() {}}-// removeFilesWithVersion returns filtered list of trashedFiles are removing all files with given version.-// Important: The method assumes that trashedFiles are sorted by increasing order of Trash.UpdatedAt+// trashedFiles must be sorted by increasing UpdatedAt.func (t *TrashController) removeFilesWithVersion(trashedFiles []ente.Trash, version int64) []ente.Trash {var i = len(trashedFiles) - 1for ; i >= 0; i-- {if trashedFiles[i].UpdatedAt != version {- // found index (from end) where file's version is different from given versionbreak}}@@ -224,7 +205,8 @@ func (t *TrashController) trashCollection(item repo.QueueItem, queueName string,"queue": queueName,"flow": "trash_collection",})- // to avoid race conditions while finding exclusive files, lock at user level, instead of individual collection+ // File exclusivity spans collections, so lock the user rather than one+ // collection.lockName := fmt.Sprintf("CollectionTrash:%d", collection.Owner.ID)lockStatus, err := t.TaskLockRepo.AcquireLock(lockName, time.MicrosecondsAfterHours(1), t.HostName)if err != nil || !lockStatus {diff --git a/server/pkg/external/wasabi/compliance.go b/server/pkg/external/wasabi/compliance.goindex 78e116ce0d..a9645a230d 100644--- a/server/pkg/external/wasabi/compliance.go+++ b/server/pkg/external/wasabi/compliance.go@@ -1,21 +1,5 @@-// S3 service style operations for Wasabi specific compliance functionality.-//-// This file contains various service operations for interacting with the Wasabi-// compliance functions. These are based on standard operations templates taken-// from the source code in the AWS S3 Go SDK (v1), and modified to use the-// custom payloads expected by Wasabi.-//-// # Wasabi Compliance-//-// Wasabi supports a compliance policy that prevents the deletion of objects.-//-// Compliance is different from the object lock setting for a bucket, and is-// mutually exclusive with it - a particular bucket can have only one of these-// enabled at a time.-//-// There are compliance settings on a bucket level, which apply the policy that-// is applied to all objects added to that bucket. In addition, there are also-// compliance settings at the object level.+// These operations follow AWS SDK v1 service templates with Wasabi-specific+// payloads. Wasabi compliance and S3 Object Lock are mutually exclusive.package wasabiimport (@@ -180,52 +164,23 @@ type PutBucketComplianceInput struct {type BucketComplianceConfiguration struct {_ struct{} `type:"structure"`- // The compliance state of the bucket.- //- // Either "enabled" or "disabled" to turn compliance on and off,- // respectively. Enabling will immediately apply to all objects in the- // bucket.+ // Enabling compliance applies it immediately to every object in the bucket.Status *string `type:"string" enum:"BucketComplianceStatus"`- // The time at which the compliance settings are "locked".- //- // The time at which the compliance settings are "locked"— the settings- // cannot be reduced by any API call. Once the settings are locked, they- // cannot be unlocked without the intervention of Wasabi Customer Support.- // The lock time allows you to support two use cases:- //- // 1) testing that your software works properly before locking the- // compliance feature; or- //- // 2) never locking which means that data can be deleted with an additional- // step of an administrator turning compliance off.- //- // The lock time parameter may be:- //- // - an ISO date (for example, 2016-11-07T15:08:05Z),- //- // - the string "now" to force immediate locking, or- //- // - the string "off" to not lock the compliance settings. This is the default.+ // "now" locks immediately, "off" leaves settings unlocked, and an ISO time+ // schedules the lock. Once locked, settings cannot be reduced without Wasabi+ // support.LockTime *string `type:"string"`- // An integer for the minimum number of days that objects are always- // retained after their creation date or release from conditional hold. You- // can extend the retention date for any individual object, but may not- // shorten the date. This parameter is always required.+ // Required minimum retention after creation or release from conditional hold.+ // Per-object retention may be extended, but not shortened.RetentionDays *int64 `type:"integer"`- // A Boolean value indicating if newly created objects are placed on- // conditional hold, meaning that they cannot be deleted until the- // conditional hold is explicitly turned off. The default is false if this- // parameter is not given. Note that this setting may be changed even after- // the settings are locked.+ // Places new objects on hold. Defaults to false and may be changed after the+ // settings are locked.ConditionalHold *bool `type:"boolean"`- // A Boolean value indicating if the object should be deleted automatically- // at the end of the retention period. The default is to not delete objects- // after the retention period. Note that this setting may be changed even- // after the settings are locked.+ // Defaults to false and may be changed after the settings are locked.DeleteAfterRetention *bool `type:"boolean"`}@@ -473,22 +428,14 @@ type PutObjectComplianceInput struct {type ObjectComplianceConfiguration struct {_ struct{} `type:"structure"`- // An ISO time giving a new retention time for the object in which the- // object cannot be deleted before this time. Note that the new retention- // time must be past the retention period given by the bucket policy or an- // error is returned.+ // Must be later than the retention required by the bucket policy.RetentionTime *string `type:"string"`- // A Boolean value "false" to release the object from the conditional hold- // setting in the bucket policy. The retention period in days is started- // from the point when the conditional hold is released. Once the- // conditional hold is set false, it may not be returned to conditional- // hold.+ // Setting this to false releases the hold, starts the retention period, and+ // cannot be reversed.ConditionalHold *bool `type:"boolean"`- // A Boolean value "true" or "false" to set the legal hold status. When an- // object has a legal hold status of true, the object cannot be deleted- // regardless of the retention period.+ // A legal hold prevents deletion regardless of the retention period.LegalHold *bool `type:"boolean"`}diff --git a/server/pkg/repo/file.go b/server/pkg/repo/file.goindex fa1c293c46..87a22c5f5d 100644--- a/server/pkg/repo/file.go+++ b/server/pkg/repo/file.go@@ -188,17 +188,9 @@ func (repo *FileRepository) CreateMetaFile(return &file, stacktrace.Propagate(err, "")}-// markAsNeedingReplication inserts new entries in object_copies, setting the-// current hot DC as the source copy.-//-// The higher layer above us (file controller) would've already checked that the-// object exists in the current hot DC (See `c.sizeOf` in file controller). This-// would cover cases where the client fetched presigned upload URLs for say-// hotDC1, but by the time they connected to museum, museum switched to using-// hotDC2. So then when museum would try to fetch the file size from hotDC2, the-// object won't be found there, and the upload would fail (which is the-// behaviour we want, since hot DC swaps are not a frequent/expected operation,-// we just wish to guarantee correctness if they do happen).+// The controller first verifies the object in hotDC. This prevents recording an+// upload against another bucket if hot storage changes after the client receives+// its presigned URL.func (repo *FileRepository) markAsNeedingReplication(ctx context.Context, tx *sql.Tx, file ente.File, hotDC string) error {if hotDC == repo.S3Config.GetHotBackblazeDC() {err := repo.ObjectCopiesRepo.CreateNewB2Object(ctx, tx, file.File.ObjectKey, true, true)@@ -220,7 +212,6 @@ func (repo *FileRepository) markAsNeedingReplication(ctx context.Context, tx *sq}}-// See markAsNeedingReplication - this variant is for updating only thumbnails.func (repo *FileRepository) markThumbnailAsNeedingReplication(ctx context.Context, tx *sql.Tx, thumbnailObjectKey string, hotDC string) error {if hotDC == repo.S3Config.GetHotBackblazeDC() {err := repo.ObjectCopiesRepo.CreateNewB2Object(ctx, tx, thumbnailObjectKey, true, false)@@ -343,9 +334,8 @@ func (repo *FileRepository) Update(file ente.File, fileSize int64, thumbnailSizereturn stacktrace.Propagate(err, "")}if isDuplicateRequest {- // Skip markAsNeedingReplication for duplicate requests, it'd fail with- // pq: duplicate key value violates unique constraint \"object_copies_pkey\"- // and render our transaction uncommittable+ // Re-inserting object_copies on a duplicate request violates its primary+ // key and leaves the transaction uncommittable.log.Infof("Skipping update of object_copies for a duplicate request to update file %d", file.ID)} else {err = repo.markAsNeedingReplication(ctx, tx, file, hotDC)@@ -694,7 +684,6 @@ func (repo *FileRepository) GetFileAttributes(fileID int64) (*ente.File, error)}func (repo *FileRepository) DropFilesMetadata(ctx context.Context, fileIDs []int64) error {- // ensure that the fileIDs are not present in object_keysrows, err := repo.DB.QueryContext(ctx, `SELECT distinct(file_id) FROM object_keys WHERE file_id = ANY($1)`, pq.Array(fileIDs))if err != nil {return stacktrace.Propagate(err, "")diff --git a/server/pkg/repo/object.go b/server/pkg/repo/object.goindex 801e2216f7..49faa9cd9d 100644--- a/server/pkg/repo/object.go+++ b/server/pkg/repo/object.go@@ -87,15 +87,11 @@ func (repo *ObjectRepository) GetObjectWithDCs(fileID int64, objType ente.Objectreturn s3ObjectKey, datacenters, stacktrace.Propagate(err, "")}-// GetAccessibleObject verifies actor access and returns the S3 object key in a-// single latency-sensitive DB query.func (repo *ObjectRepository) GetAccessibleObject(ctx context.Context, fileID int64, actorUserID int64, objType ente.ObjectType) (ente.S3ObjectKey, error) {s3ObjectKey, _, err := repo.GetAccessibleObjectWithDCs(ctx, fileID, actorUserID, objType)return s3ObjectKey, err}-// GetAccessibleObjectWithDCs verifies actor access and returns the S3 object-// key along with replicated datacenters in a single latency-sensitive DB query.func (repo *ObjectRepository) GetAccessibleObjectWithDCs(ctx context.Context, fileID int64, actorUserID int64, objType ente.ObjectType) (ente.S3ObjectKey, []string, error) {row := repo.objectLookupDB().QueryRowContext(ctx, `SELECT@@ -153,15 +149,11 @@ func (repo *ObjectRepository) GetAccessibleObjectWithDCs(ctx context.Context, fireturn s3ObjectKey, datacenters, nil}-// GetOwnedObject verifies ownership and returns the S3 object key in a single-// latency-sensitive DB query.func (repo *ObjectRepository) GetOwnedObject(ctx context.Context, fileID int64, ownerID int64, objType ente.ObjectType) (ente.S3ObjectKey, error) {s3ObjectKey, _, err := repo.GetOwnedObjectWithDCs(ctx, fileID, ownerID, objType)return s3ObjectKey, err}-// GetOwnedObjectWithDCs verifies ownership and returns the S3 object key along-// with replicated datacenters in a single latency-sensitive DB query.func (repo *ObjectRepository) GetOwnedObjectWithDCs(ctx context.Context, fileID int64, ownerID int64, objType ente.ObjectType) (ente.S3ObjectKey, []string, error) {row := repo.objectLookupDB().QueryRowContext(ctx, `SELECT@@ -208,16 +200,11 @@ func (repo *ObjectRepository) GetOwnedObjectWithDCs(ctx context.Context, fileIDreturn s3ObjectKey, datacenters, nil}-// GetCollectionObject verifies collection membership and returns the S3 object-// key in a single latency-sensitive DB query.func (repo *ObjectRepository) GetCollectionObject(ctx context.Context, collectionID int64, fileID int64, objType ente.ObjectType) (ente.S3ObjectKey, error) {s3ObjectKey, _, err := repo.GetCollectionObjectWithDCs(ctx, collectionID, fileID, objType)return s3ObjectKey, err}-// GetCollectionObjectWithDCs verifies collection membership and returns the S3-// object key along with replicated datacenters in a single latency-sensitive DB-// query.func (repo *ObjectRepository) GetCollectionObjectWithDCs(ctx context.Context, collectionID int64, fileID int64, objType ente.ObjectType) (ente.S3ObjectKey, []string, error) {row := repo.objectLookupDB().QueryRowContext(ctx, `SELECT@@ -303,8 +290,6 @@ func (repo *ObjectRepository) RemoveObjectsForKey(objectKey string) error {return stacktrace.Propagate(err, "")}-// MarkObjectsAsDeletedForFileIDs marks the object keys corresponding to the given filesIDs as deleted-// The actual deletion happens later when the queue is processedfunc (repo *ObjectRepository) MarkObjectsAsDeletedForFileIDs(ctx context.Context, tx *sql.Tx, fileIDs []int64) ([]ente.S3ObjectKey, error) {rows, err := tx.QueryContext(ctx, `SELECT file_id, o_type, object_key, size FROM object_keysWHERE file_id = ANY($1) AND is_deleted=false FOR UPDATE`, pq.Array(fileIDs))@@ -437,11 +422,7 @@ func (repo *ObjectRepository) GetObjectReferenceStatuses(ctx context.Context, obreturn statuses, stacktrace.Propagate(rows.Err(), "")}-// GetObjectState returns various bits of information about an object that are-// useful in pre-flight checks during replication.-//-// Unknown objects (i.e. objectKeys for which there are no entries) are-// considered as deleted.+// Unknown object keys are treated as deleted.func (repo *ObjectRepository) GetObjectState(objectKey string) (ObjectState ente.ObjectState, err error) {row := repo.DB.QueryRow(`SELECT ok.is_deleted, u.encrypted_email IS NULL AS is_user_deleted, ok.sizediff --git a/server/pkg/repo/object_copies.go b/server/pkg/repo/object_copies.goindex 4d198cd522..389651fa19 100644--- a/server/pkg/repo/object_copies.go+++ b/server/pkg/repo/object_copies.go@@ -15,11 +15,6 @@ type ObjectCopiesRepository struct {DB *sql.DB}-// GetAndLockUnreplicatedObject gets an object which is not yet replicated to-// all the replicas. It also registers a replication to keep the row corresponding-// to that object to be blocked for 24h before next replication attemp.-//-// ObjectCopies is guaranteed to be nil if error is not nil.func (repo *ObjectCopiesRepository) GetAndLockUnreplicatedObject(ctx context.Context) (*ente.ObjectCopies, error) {tx, err := repo.DB.BeginTx(ctx, nil)if err != nil {@@ -71,13 +66,6 @@ func (repo *ObjectCopiesRepository) GetAndLockUnreplicatedObject(ctx context.Conreturn &r, nil}-// CreateNewB2Object creates a new entry for objectKey and marks it as having-// being replicated to B2. It then sets provided flags to mark this object as-// requiring replication where needed.-//-// This operation runs within the context of a transaction that creates the-// initial entry for the file in the database; thus, it gets passed ctx and tx-// which it uses to scope its own DB changes.func (repo *ObjectCopiesRepository) CreateNewB2Object(ctx context.Context, tx *sql.Tx, objectKey string, wantWasabi bool, wantScaleway bool) error {_, err := tx.ExecContext(ctx, `INSERT INTO object_copies (object_key, want_b2, b2, want_wasabi, want_scw)@@ -86,10 +74,6 @@ func (repo *ObjectCopiesRepository) CreateNewB2Object(ctx context.Context, tx *sreturn stacktrace.Propagate(err, "")}-// CreateNewWasabiObject creates a new entry for objectKey and marks it as having-// being replicated to Wasabi.-//-// See CreateNewB2Object for details.func (repo *ObjectCopiesRepository) CreateNewWasabiObject(ctx context.Context, tx *sql.Tx, objectKey string, wantB2 bool, wantScaleway bool) error {_, err := tx.ExecContext(ctx, `INSERT INTO object_copies (object_key, want_wasabi, wasabi, want_b2, want_scw)@@ -98,8 +82,6 @@ func (repo *ObjectCopiesRepository) CreateNewWasabiObject(ctx context.Context, treturn stacktrace.Propagate(err, "")}-// RegisterReplicationAttempt sets the last_attempt timestamp so that this row can-// be skipped over for the next day in case the replication was not succesful.func (repo *ObjectCopiesRepository) RegisterReplicationAttempt(tx *sql.Tx, ctx context.Context, objectKey string) error {_, err := tx.ExecContext(ctx, `UPDATE object_copiesdiff --git a/server/pkg/repo/trash.go b/server/pkg/repo/trash.goindex a89dcfca77..1dfe4842c9 100644--- a/server/pkg/repo/trash.go+++ b/server/pkg/repo/trash.go@@ -239,8 +239,6 @@ func (t *TrashRepository) Delete(ctx context.Context, userID int64, fileIDs []inreturn tx.Commit()}-// GetFilesInTrashState for a given userID and fileIDs, return the list of fileIDs which are actually present in-// trash and is not deleted or restored yet.func (t *TrashRepository) GetFilesInTrashState(ctx context.Context, userID int64, fileIDs []int64) ([]int64, bool, error) {rows, err := t.DB.Query(`SELECT file_id FROM trashWHERE user_id = $1 AND file_id = ANY ($2)@@ -264,11 +262,6 @@ func (t *TrashRepository) GetFilesInTrashState(ctx context.Context, userID int64return fileIDsInTrash, canRestoreOrDeleteAllFiles, nil}-// GetFilesInTrashOrDeleted returns the subset of fileIDs that are either in trash (not restored)-// or have been permanently deleted. This is useful for validation checks to prevent operations-// on files that are no longer in an active state.-// Unlike GetFilesInTrashState, this method does not log warnings when files are not found,-// making it suitable for validation checks where files are expected to NOT be in trash.func (t *TrashRepository) GetFilesInTrashOrDeleted(ctx context.Context, userID int64, fileIDs []int64) ([]int64, error) {rows, err := t.DB.QueryContext(ctx, `SELECT file_id FROM trashWHERE user_id = $1 AND file_id = ANY ($2)@@ -389,11 +382,8 @@ func (t *TrashRepository) GetUserIDToFileIDsMapForDeletion() (map[int64][]int64,return result, nil}-// GetFileIdsForDroppingMetadata retrieves file IDs of deleted files for metadata scrubbing.-// It returns files that were deleted after the provided timestamp (sinceUpdatedAt) and have been in the trash for at least 50 days.-// This delay ensures compliance with deletion locks.-// The method orders the results by the 'updated_at' field in ascending order and limits the results to 'TrashDiffLimit' + 1.-// If multiple files have the same 'updated_at' timestamp and are at the limit boundary, they are excluded to prevent partial scrubbing.+// Wait 50 days for compliance deletion locks, and never split an updated_at+// group across batches.func (t *TrashRepository) GetFileIdsForDroppingMetadata(sinceUpdatedAt int64) ([]FileWithUpdatedAt, error) {rows, err := t.DB.Query(`select file_id, updated_at from trash where is_deleted=true AND updated_at > $1@@ -422,9 +412,6 @@ order by updated_at ASC limit $2return fileWithUpdatedAt, nil}- // from the end ignore the fileIds from fileWithUpdatedAt that have the same updatedAt.- // this is to avoid scrubbing partial list of files that have same updatedAt as due to the limit not- // all files with the same updatedAt are returned.lastUpdatedAt := fileWithUpdatedAt[len(fileWithUpdatedAt)-1].UpdatedAtvar i = len(fileWithUpdatedAt) - 1for ; i >= 0; i-- {diff --git a/server/pkg/utils/s3config/s3config.go b/server/pkg/utils/s3config/s3config.goindex 6e20454bda..af32545116 100644--- a/server/pkg/utils/s3config/s3config.go+++ b/server/pkg/utils/s3config/s3config.go@@ -13,14 +13,8 @@ import ("github.com/spf13/viper")-// Objects are replicated in multiple "data centers". Each data center is an-// S3-compatible provider, and has an associated "bucket".-//-// The list of data centers are not arbitrarily configurable - while we pick the-// exact credentials, endpoint and the bucket from the runtime configuration-// file, the code has specific logic to deal with the quirks of specific data-// centers. So as such, the constants used to specify these data centers in the-// YAML configuration matter.+// Bucket IDs are fixed because provider-specific code depends on their exact+// values. Credentials, endpoints, and bucket names remain configurable.type S3Config struct {buckets map[string]stringhotDC string@@ -33,41 +27,10 @@ type S3Config struct {// debugging workarounds; not tested/intended for production.areLocalBuckets bool- // If for particular object type, the bucket is not specified, it will- // default to hotDC as the bucket with no replicas. Initially, this config won't support- // existing objectType (file, thumbnail) and will be used for new objectTypes. In the future,- // we can migrate existing objectTypes to this config.fileDataConfig FileDataConfig- // If no explicit config is present, attachments default to hot storage with no replicas.attachmentConfig AttachmentConfig}-// # Datacenters-// Note: We are now renaming datacenter names to bucketID. Till the migration is completed, you will see usage of both-// terminology.-// Below are some high level details about the three replicas ("data centers")-// that are in use. There are a few other legacy ones too.-//-// # Backblaze (dcB2EuropeCentral)-//-// - Primary hot storage-// - Versioned, but with extra code that undoes all overwrites-//-// # Wasabi (dcWasabiEuropeCentral_v3)-//-// - Secondary hot storage-// - Objects stay under compliance, which prevents them from being-// deleted/updated for 21 days.-// - Not versioned (versioning is not needed since objects cannot be overwritten)-// - When an user (permanently) deletes an object, we remove the compliance-// retention. It can then be deleted normally when the scheduled-// cleanup happens (as long as it happens after 21 days).-//-// # Scaleway (dcSCWEuropeFrance_v3)-//-// - Cold storage-// - Specify type GLACIER in API requests-var (dcB2EuropeCentral string = "b2-eu-cen"dcSCWEuropeFranceDeprecated string = "scw-eu-fr"@@ -163,7 +126,6 @@ func (config *S3Config) GetBucket(dcOrBucketID string) *string {return &bucket}-// GetBucketID returns the bucket ID for the given object type. Note: existing dc are renamed as bucketIDfunc (config *S3Config) GetBucketID(oType ente.ObjectType) string {if config.fileDataConfig.HasConfig(oType) {return config.fileDataConfig.GetPrimaryBucketID(oType)@@ -258,11 +220,6 @@ func (config *S3Config) GetColdScalewayDC() string {return dcSCWEuropeFrance_v3}-// ShouldDeleteFromDataCenter returns true if objects should be deleted from the-// given data center when permanently deleting these objects.-//-// There are some legacy / deprecated data center values which are no longer-// being used, and it returns false for such data centers.func (config *S3Config) ShouldDeleteFromDataCenter(dc string) bool {return dc != dcSCWEuropeFranceDeprecated && dc != dcSCWEuropeFranceLockedDeprecated && dc != dcWasabiEuropeCentralDeprecated}@@ -274,8 +231,6 @@ func (config *S3Config) WasabiComplianceDC() string {return ""}-// Return true if we're using local minio buckets. This can then be used to add-// various workarounds for debugging locally; not meant for production use.func (config *S3Config) AreLocalBuckets() bool {return config.areLocalBuckets}