Skip to content

armgit / ente

Public Imported from GitHub
0

💚 End-to-end encrypted cloud for everything.

Prune server comment kernels

Manav Rathi9d3a0a1a171a
diff --git a/server/ente/trash.go b/server/ente/trash.go
index e92096a6a2..0a8c58023d 100644
--- a/server/ente/trash.go
+++ b/server/ente/trash.go
@@ -11,33 +11,19 @@ type Trash struct {
type DeleteTrashFilesRequest struct {
FileIDs []int64 `json:"fileIDs" binding:"required"`
- // OwnerID will be set based on the authenticated user
+ // Set from the authenticated user after binding.
OwnerID int64
}
type EmptyTrashRequest struct {
- // LastUpdatedAt timestamp will be used to delete trashed files with updatedAt timestamp <= LastUpdatedAt
- // User's trash will be cleaned up in an async manner. The timestamp is used to ensure that newly trashed files
- // are not deleted due to delay in the async operation.
+ // Empty only entries at or before this timestamp so queued work cannot
+ // delete newly trashed files.
LastUpdatedAt int64 `json:"lastUpdatedAt" binding:"required"`
}
-// In V3, while trashing/deleting any album, the user can decide to either keep or delete the all files which are
-// present in to the trash. When user wants to keep the files, the clients are expected to move all the files from
-// the underlying collection to any other collection owned by the user, inlcuding uncategorized.
-// Note: Collection Delete Versions for DELETE /collections/V../ endpoint
-// V1: All files which exclusively belong to the collections are deleted immediately.
-// V2: All files which exclusively belong to the collections are moved to the trash.
-// V3: All files which are still present in the collection (irrespective if they blong to another collection) will be moved to trash.
-// V3 is introduced to avoid doing this booking on server, where we only delete a file when it's beling removed from the last collection it longs to.
-// In theory above logic to delete when it's being removed from last collection sounds good. But,
-// in practice it complicates the code (thus reducing its robustness) because of race conditions, and it's
-// also hard to communicate it to the user. So, to simplify things, in V3, the files will be only deleted when user tell us to delete them.
type TrashCollectionV3Request struct {
CollectionID int64 `json:"collectionID" form:"collectionID" binding:"required"`
- // When KeepFiles is false, then all the files which are present in the collection will be moved to trash.
- // When KeepFiles is true, but the underlying collection still contains file, then the API call will fail.
- // This is to ensure that before deleting the collection, the client has moved all relevant files to any other
- // collection owned by the user, including Uncategorized.
+ // If true, deletion requires an empty collection. Otherwise remaining files
+ // move to trash.
KeepFiles *bool `json:"keepFiles" form:"keepFiles" binding:"required"`
}
diff --git a/server/pkg/controller/collections/file_action.go b/server/pkg/controller/collections/file_action.go
index 84609aa223..627338bf95 100644
--- a/server/pkg/controller/collections/file_action.go
+++ b/server/pkg/controller/collections/file_action.go
@@ -150,15 +150,6 @@ func (c *CollectionController) MoveFiles(ctx *gin.Context, req ente.MoveFilesReq
return stacktrace.Propagate(err, "")
}
-// RemoveFilesV3 enforces all removal rules for shared collections:
-// 1. accessCtrl must confirm the actor participates in the collection;
-// 2. collaborators/viewers may only remove the files they added themselves;
-// 3. the collection owner may remove files added by others but never their own;
-// 4. admins can remove anyone's files, but a collection owner's files are only
-// queued for removal (REMOVE action + pending collection_action entry) so
-// the owner can act on them;
-// 5. once the validations pass, non-owner files are deleted immediately via
-// CollectionRepo.RemoveFilesV3.
func (c *CollectionController) RemoveFilesV3(ctx *gin.Context, actorUserID int64, req ente.RemoveFilesV3Request) error {
accessResp, err := c.AccessCtrl.GetCollection(ctx, &access.GetCollectionParams{
CollectionID: req.CollectionID,
@@ -221,14 +212,6 @@ func (c *CollectionController) RemoveFilesV3(ctx *gin.Context, actorUserID int64
return nil
}
-// SuggestDeleteInSharedCollection allows collection owners/admins to nudge other
-// participants to delete their files:
-// 1. only OWNER/ADMIN roles pass the access check;
-// 2. every file ID must belong to the collection and none may belong to the acting user;
-// 3. the method internally reuses RemoveFilesV3 to enforce role-based rules and
-// to actually detach the files from the collection;
-// 4. each remote owner then receives DELETE_SUGGESTED actions so their clients
-// can surface the pending delete request.
func (c *CollectionController) SuggestDeleteInSharedCollection(ctx *gin.Context, req ente.SuggestDeleteRequest) error {
actorUserID := auth.GetUserID(ctx.Request.Header)
accessResp, err := c.AccessCtrl.GetCollection(ctx, &access.GetCollectionParams{
diff --git a/server/pkg/controller/collections/files_diff.go b/server/pkg/controller/collections/files_diff.go
index 069bca5c2c..98e1131231 100644
--- a/server/pkg/controller/collections/files_diff.go
+++ b/server/pkg/controller/collections/files_diff.go
@@ -29,9 +29,7 @@ func (c *CollectionController) GetDiffV2(ctx *gin.Context, cID int64, userID int
if err != nil {
return nil, false, stacktrace.Propagate(err, "")
}
- // hide private metadata before returning files info in diff
for idx := range diff {
- // Treat action markers as soft-deletes for non-owners
if diff[idx].Action != nil && !diff[idx].IsDeleted {
if *diff[idx].Action == ente.ActionRemove || *diff[idx].Action == ente.ActionDeleteSuggested {
if diff[idx].OwnerID != userID { // non-owner view: mask as deleted
@@ -45,7 +43,7 @@ func (c *CollectionController) GetDiffV2(ctx *gin.Context, cID int64, userID int
diff[idx].ActionUserID = nil
}
if diff[idx].Metadata.EncryptedData == "-" && !diff[idx].IsDeleted {
- // This indicates that the file is deleted, but we still have a stale entry in the collection
+ // "-" marks a deleted file whose collection entry is stale.
log.WithFields(log.Fields{
"file_id": diff[idx].ID,
"collection_id": cID,
@@ -57,37 +55,22 @@ func (c *CollectionController) GetDiffV2(ctx *gin.Context, cID int64, userID int
return diff, hasMore, nil
}
-// getDiff returns the diff in user's collection since a timestamp, along with hasMore bool flag.
-// The function will never return partial result for a version. To maintain this promise, it will not be able to honor
-// the limit parameter. Based on the db state, compared to the limit, the diff length can be
-// less (case 1), more (case 2), or same (case 3, 4)
-// Example: Assume we have 11 files with following versions: v0, v1, v1, v1, v1, v1, v1, v1, v2, v2, v2 (count = 7 v1, 3 v2)
-// client has synced up till version v0.
-// case 1: ( sinceTime: v0, limit = 8):
-// The method will discard the entries with version v2 and return only 7 entries with version v1.
-// case 2: (sinceTime: v0, limit 5):
-// Instead of returning 5 entries with version V1, method will return all 7 entries with version v1.
-// case 3: (sinceTime: v0, limit 7):
-// The method will return all 7 entries with version V1.
-// case 4: (sinceTime: v0, limit >=10):
-// The method will all 10 entries in the diff
+// Never split a version across pages. Results may be smaller or larger than
+// limit so every row with the boundary timestamp stays together.
func (c *CollectionController) getDiff(cID int64, sinceTime int64, limit int, logger *log.Entry) ([]ente.File, bool, error) {
- // request for limit +1 files
diffLimitPlusOne, err := c.CollectionRepo.GetDiff(cID, sinceTime, limit+1)
if err != nil {
return nil, false, stacktrace.Propagate(err, "")
}
if len(diffLimitPlusOne) <= limit {
- // case 4: all files changed after sinceTime are included.
return diffLimitPlusOne, false, nil
}
lastFileVersion := diffLimitPlusOne[limit].UpdationTime
filteredDiffs := c.removeFilesWithVersion(diffLimitPlusOne, lastFileVersion)
filteredDiffLen := len(filteredDiffs)
- if filteredDiffLen > 0 { // case 1 or case 3
+ if filteredDiffLen > 0 {
if filteredDiffLen < limit {
- // logging case 1
logger.
WithField("last_file_version", lastFileVersion).
WithField("filtered_diff_len", filteredDiffLen).
@@ -95,7 +78,6 @@ func (c *CollectionController) getDiff(cID int64, sinceTime int64, limit int, lo
}
return filteredDiffs, true, nil
}
- // case 2
diff, err := c.CollectionRepo.GetFilesWithVersion(cID, lastFileVersion)
logger.
WithField("last_file_version", lastFileVersion).
@@ -107,13 +89,11 @@ func (c *CollectionController) getDiff(cID int64, sinceTime int64, limit int, lo
return diff, true, nil
}
-// removeFilesWithVersion returns filtered list of files are removing all files with given version.
-// Important: The method assumes that files are sorted by increasing order of File.UpdationTime
+// files must be sorted by increasing UpdationTime.
func (c *CollectionController) removeFilesWithVersion(files []ente.File, version int64) []ente.File {
var i = len(files) - 1
for ; i >= 0; i-- {
if files[i].UpdationTime != version {
- // found index (from end) where file's version is different from given version
break
}
}
diff --git a/server/pkg/controller/object_cleanup.go b/server/pkg/controller/object_cleanup.go
index b4c080e723..5335c3e8d3 100644
--- a/server/pkg/controller/object_cleanup.go
+++ b/server/pkg/controller/object_cleanup.go
@@ -19,14 +19,6 @@ import (
log "github.com/sirupsen/logrus"
)
-// ObjectCleanupController exposes functions to remove temporary object storage
-// entries that were never committed to the database.
-//
-// 1. We create presigned URLs for clients to upload their objects to. It might
-// happen that the client is able to successfully upload to these URLs, but
-// not tell museum about the successful upload.
-//
-// 2. During replication, we might have half-done multipart uploads.
type ObjectCleanupController struct {
Repo *repo.ObjectCleanupRepository
ObjectRepo *repo.ObjectRepository
@@ -105,9 +97,7 @@ func (c *ObjectCleanupController) removeUnreportedObjects() int {
logger.Infof("Removed %d objects", count)
- // We always commit the transaction, even on errors for individual rows. To
- // avoid object getting stuck in a loop, we increase their expiry times.
-
+ // Always commit the batch. skip() delays failed rows to prevent a tight loop.
cerr := tx.Commit()
if cerr != nil {
cerr = stacktrace.Propagate(err, "Failed to commit transaction")
@@ -171,12 +161,6 @@ func (c *ObjectCleanupController) removeUnreportedObject(tx *sql.Tx, t ente.Temp
return nil
}
-// AddTempObjectKey creates a new temporary object entry.
-//
-// It persists a given object key as having been provided to a client for
-// uploading. If a client does not successfully mark this object's upload as
-// having completed within PreSignedRequestValidityDuration, this temp object
-// will be cleaned up.
func (c *ObjectCleanupController) AddTempObjectKey(objectKey string, dc string) error {
expiry := time.Microseconds() + (2 * PreSignedRequestValidityDuration.Microseconds())
return c.addCleanupEntryForObjectKey(objectKey, dc, expiry)
@@ -298,7 +282,6 @@ func (c *ObjectCleanupController) abortMultipartUpload(objectKey string, uploadI
})
if err != nil {
if isUnknownUploadError(err) {
- // This is expected now, since we just aborted the upload
return nil
}
return stacktrace.Propagate(err, "")
diff --git a/server/pkg/controller/replication3.go b/server/pkg/controller/replication3.go
index 8f681df4f9..bd724dde72 100644
--- a/server/pkg/controller/replication3.go
+++ b/server/pkg/controller/replication3.go
@@ -32,19 +32,6 @@ const (
replicationDelayForStaleObjects = 30
)
-// ReplicationController3 oversees version 3 of our object replication.
-//
-// The user's encrypted data starts off in 1 hot storage (Backblaze "b2"). This
-// controller then takes over and replicates it the other two replicas. It keeps
-// state in the object_copies table.
-//
-// Both v2 and v3 of object replication use the same hot storage (b2), but they
-// replicate to different buckets thereafter.
-//
-// The current implementation only works if the hot storage is b2. This is not
-// an inherent limitation, however the code has not yet been tested in other
-// scenarios, so there is a safety check preventing the replication from
-// happening if the current hot storage is not b2.
type ReplicationController3 struct {
S3Config *s3config.S3Config
ObjectRepo *repo.ObjectRepository
@@ -66,17 +53,10 @@ type UploadDestination struct {
Uploader *s3manager.Uploader
Bucket *string
Label string
- // If true, we should ignore Wasabi 403 errors. See "Reuploads".
HasComplianceHold bool
IsGlacier bool
}
-// StartReplication starts the background replication process.
-//
-// This method returns synchronously. ReplicationController3 will create
-// suitable number of goroutines to parallelize and perform the replication
-// asynchronously, as and when it notices new files that have not yet been
-// replicated (it does this by querying the object_copies table).
func (c *ReplicationController3) StartReplication() error {
hotDC := c.S3Config.GetHotDataCenter()
if hotDC != c.S3Config.GetHotBackblazeDC() {
@@ -152,12 +132,6 @@ func (c *ReplicationController3) createTemporaryStorage() error {
}
func (c *ReplicationController3) createDestinations() {
- // The s3manager.Uploader objects are safe for use concurrently. From the
- // AWS docs:
- //
- // > The Uploader structure that calls Upload(). It is safe to call Upload()
- // on this structure for multiple objects and across concurrent goroutines.
- // Mutating the Uploader's properties is not safe to be done concurrently.
// Scaleway limits multipart uploads to 1,000 parts.
limitUploadParts := func(u *s3manager.Uploader) {
u.MaxUploadParts = 1000
@@ -189,8 +163,7 @@ func (c *ReplicationController3) createDestinations() {
Uploader: s3manager.NewUploaderWithClient(&scwClient, limitUploadParts),
Bucket: config.GetBucket(scwDC),
Label: "scaleway",
- // should be true, except when running in a local cluster (since minio doesn't
- // support specifying the GLACIER storage class).
+ // MinIO does not support the GLACIER storage class.
IsGlacier: !config.AreLocalBuckets(),
}
}
@@ -199,19 +172,12 @@ func (c *ReplicationController3) replicate(i int) {
for {
err := c.tryReplicate()
if err != nil {
- // Sleep in proportion to the (arbitrary) index to space out the
- // workers further.
+ // Stagger retries across workers.
time.Sleep(time.Duration(i+1) * time.Minute)
}
}
}
-// Try to replicate an object.
-//
-// Return nil if something was replicated, otherwise return the error.
-//
-// A common and expected error is `sql.ErrNoRows`, which occurs if there are no
-// objects left to replicate currently.
func (c *ReplicationController3) tryReplicate() error {
ctxWithTimeout, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
@@ -267,9 +233,7 @@ func (c *ReplicationController3) tryReplicate() error {
}
if ob.IsFileDeleted || ob.IsUserDeleted {
- // Update the object_copies to mark this object as not requiring further
- // replication. The row in object_copies will get deleted when the next
- // scheduled object deletion runs.
+ // Scheduled object deletion removes the object_copies row later.
err = c.ObjectCopiesRepo.UnmarkFromReplication(objectKey)
if err != nil {
return done(stacktrace.Propagate(err, "Failed to mark an object not requiring further replication"))
@@ -281,12 +245,6 @@ func (c *ReplicationController3) tryReplicate() error {
err = fileutil.EnsureSufficientSpace(ob.Size)
if err != nil {
- // We don't have free space right now, maybe because other big files are
- // being downloaded simultanously, but we might get space later, so mark
- // a failed attempt that'll get retried later.
- //
- // Log this error though, so that it gets noticed if it happens too
- // frequently (the instance might need a bigger disk).
return done(stacktrace.Propagate(err, ""))
}
@@ -440,30 +398,16 @@ func (c *ReplicationController3) replicateFile(in *UploadInput, dest *UploadDest
return failure(stacktrace.Propagate(err, "Failed to verify upload"))
}
- // The update of the object_keys is not done in the transaction where the
- // other updates to object_copies table are made. This is so that the
- // object_keys table (which is what'll be used to delete objects) is
- // (almost) always updated if the file gets uploaded successfully.
- //
- // The only time the update wouldn't happen is if museum gets restarted
- // between the successful completion of the upload to the bucket and this
- // query getting executed.
- //
- // While possible, that is a much smaller window as compared to the
- // transaction for updating object_copies, which could easily span minutes
- // as the transaction ends only after the object has been uploaded to all
- // replicas.
+ // Record each successful upload in object_keys before updating object_copies.
+ // The object_copies transaction can span all replica uploads; a restart in
+ // that window must not leave an uploaded replica absent from deletion records.
rowsAffected, err := c.ObjectRepo.MarkObjectReplicated(in.ObjectKey, dest.DC)
if err != nil {
return failure(stacktrace.Propagate(err, "Failed to update object_keys to mark replication as completed"))
}
if rowsAffected != 1 {
- // It is possible that this row was updated earlier, after an upload
- // that got completed but before object_copies table could be updated in
- // the transaction (See "Reuploads").
- //
- // So do not treat this as an error.
+ // A previous interrupted attempt may already have recorded this replica.
logger.Warnf("Expected 1 row to be updated, but got %d", rowsAffected)
}
@@ -476,22 +420,9 @@ func (c *ReplicationController3) replicateFile(in *UploadInput, dest *UploadDest
return nil
}
-// # Reuploads
-//
-// It is possible that the object might already exist on remote. The known
-// scenario where this might happen is if museum gets restarted after having
-// completed the upload but before it got around to modifying the DB.
-//
-// The behaviour in this case is remote dependent.
-//
-// - Uploading an object with the same key on Scaleway would work normally.
-//
-// - But trying to add an object with the same key on the compliance locked
-// Wasabi would return an HTTP 403.
-//
-// We intercept the Wasabi 403 in this case and move ahead. The subsequent
-// object verification using the HEAD request will act as a sanity check for
-// the object.
+// A restart can leave an uploaded replica unrecorded in object_copies.
+// Compliance-locked Wasabi returns 403 when that upload is retried, so allow
+// the HEAD size check to verify the existing object.
func (c *ReplicationController3) uploadFile(in *UploadInput, dest *UploadDestination) error {
in.File.Seek(0, io.SeekStart)
diff --git a/server/pkg/controller/trash.go b/server/pkg/controller/trash.go
index 4b4f9cc117..15f44e2fd4 100644
--- a/server/pkg/controller/trash.go
+++ b/server/pkg/controller/trash.go
@@ -32,7 +32,6 @@ func (t *TrashController) GetDiff(userID int64, sinceTime int64, app ente.App) (
if err != nil {
return nil, false, err
}
- // hide private metadata before returning files info in diff
for _, trashFile := range trashFilesDiff {
if trashFile.IsDeleted {
trashFile.File.MagicMetadata = nil
@@ -44,36 +43,21 @@ func (t *TrashController) GetDiff(userID int64, sinceTime int64, app ente.App) (
return trashFilesDiff, hasMore, err
}
-// GetDiff returns the diff in user's trash since a timestamp, along with hasMore bool flag.
-// The function will never return partial result for a version. To maintain this promise, it will not be able to honor
-// the limit parameter. Based on the db state, compared to the limit, the diff length can be
-// less (case 1), more (case 2), or same (case 3, 4)
-// Example: Assume we have 11 files with following versions: v0, v1, v1, v1, v1, v1, v1, v1, v2, v2, v2 (count = 7 v1, 3 v2)
-// client has synced up till version v0.
-// case 1: ( sinceTime: v0, limit = 8):
-// The method will discard the entries with version v2 and return only 7 entries with version v1.
-// case 2: (sinceTime: v0, limit 5):
-// Instead of returning 5 entries with version V1, method will return all 7 entries with version v1.
-// case 3: (sinceTime: v0, limit 7):
-// The method will return all 7 entries with version V1.
-// case 4: (sinceTime: v0, limit >=10):
-// The method will all 10 entries in the diff
+// Never split a version across pages. Results may be smaller or larger than
+// limit so every row with the boundary timestamp stays together.
func (t *TrashController) getDiff(userID int64, sinceTime int64, limit int, app ente.App) ([]ente.Trash, bool, error) {
- // request for limit +1 files
diffLimitPlusOne, err := t.TrashRepo.GetDiff(userID, sinceTime, limit+1, app)
if err != nil {
return nil, false, stacktrace.Propagate(err, "")
}
if len(diffLimitPlusOne) <= limit {
- // case 4: all files changed after sinceTime are included.
return diffLimitPlusOne, false, nil
}
lastFileVersion := diffLimitPlusOne[limit].UpdatedAt
filteredDiffs := t.removeFilesWithVersion(diffLimitPlusOne, lastFileVersion)
- if len(filteredDiffs) > 0 { // case 1 or case 3
+ if len(filteredDiffs) > 0 {
return filteredDiffs, true, nil
}
- // case 2
diff, err := t.TrashRepo.GetFilesWithVersion(userID, lastFileVersion, app)
if err != nil {
return nil, false, stacktrace.Propagate(err, "")
@@ -81,7 +65,6 @@ func (t *TrashController) getDiff(userID int64, sinceTime int64, limit int, app
return diff, true, nil
}
-// Delete files permanently, queues up the file for deletion & free up the space based on file's object size
func (t *TrashController) Delete(ctx context.Context, request ente.DeleteTrashFilesRequest) error {
err := t.TrashRepo.Delete(ctx, request.OwnerID, request.FileIDs)
if err != nil {
@@ -198,13 +181,11 @@ func (t *TrashController) DeleteAgedTrashedFiles() {
}
}
-// removeFilesWithVersion returns filtered list of trashedFiles are removing all files with given version.
-// Important: The method assumes that trashedFiles are sorted by increasing order of Trash.UpdatedAt
+// trashedFiles must be sorted by increasing UpdatedAt.
func (t *TrashController) removeFilesWithVersion(trashedFiles []ente.Trash, version int64) []ente.Trash {
var i = len(trashedFiles) - 1
for ; i >= 0; i-- {
if trashedFiles[i].UpdatedAt != version {
- // found index (from end) where file's version is different from given version
break
}
}
@@ -224,7 +205,8 @@ func (t *TrashController) trashCollection(item repo.QueueItem, queueName string,
"queue": queueName,
"flow": "trash_collection",
})
- // to avoid race conditions while finding exclusive files, lock at user level, instead of individual collection
+ // File exclusivity spans collections, so lock the user rather than one
+ // collection.
lockName := fmt.Sprintf("CollectionTrash:%d", collection.Owner.ID)
lockStatus, err := t.TaskLockRepo.AcquireLock(lockName, time.MicrosecondsAfterHours(1), t.HostName)
if err != nil || !lockStatus {
diff --git a/server/pkg/external/wasabi/compliance.go b/server/pkg/external/wasabi/compliance.go
index 78e116ce0d..a9645a230d 100644
--- a/server/pkg/external/wasabi/compliance.go
+++ b/server/pkg/external/wasabi/compliance.go
@@ -1,21 +1,5 @@
-// S3 service style operations for Wasabi specific compliance functionality.
-//
-// This file contains various service operations for interacting with the Wasabi
-// compliance functions. These are based on standard operations templates taken
-// from the source code in the AWS S3 Go SDK (v1), and modified to use the
-// custom payloads expected by Wasabi.
-//
-// # Wasabi Compliance
-//
-// Wasabi supports a compliance policy that prevents the deletion of objects.
-//
-// Compliance is different from the object lock setting for a bucket, and is
-// mutually exclusive with it - a particular bucket can have only one of these
-// enabled at a time.
-//
-// There are compliance settings on a bucket level, which apply the policy that
-// is applied to all objects added to that bucket. In addition, there are also
-// compliance settings at the object level.
+// These operations follow AWS SDK v1 service templates with Wasabi-specific
+// payloads. Wasabi compliance and S3 Object Lock are mutually exclusive.
package wasabi
import (
@@ -180,52 +164,23 @@ type PutBucketComplianceInput struct {
type BucketComplianceConfiguration struct {
_ struct{} `type:"structure"`
- // The compliance state of the bucket.
- //
- // Either "enabled" or "disabled" to turn compliance on and off,
- // respectively. Enabling will immediately apply to all objects in the
- // bucket.
+ // Enabling compliance applies it immediately to every object in the bucket.
Status *string `type:"string" enum:"BucketComplianceStatus"`
- // The time at which the compliance settings are "locked".
- //
- // The time at which the compliance settings are "locked"— the settings
- // cannot be reduced by any API call. Once the settings are locked, they
- // cannot be unlocked without the intervention of Wasabi Customer Support.
- // The lock time allows you to support two use cases:
- //
- // 1) testing that your software works properly before locking the
- // compliance feature; or
- //
- // 2) never locking which means that data can be deleted with an additional
- // step of an administrator turning compliance off.
- //
- // The lock time parameter may be:
- //
- // - an ISO date (for example, 2016-11-07T15:08:05Z),
- //
- // - the string "now" to force immediate locking, or
- //
- // - the string "off" to not lock the compliance settings. This is the default.
+ // "now" locks immediately, "off" leaves settings unlocked, and an ISO time
+ // schedules the lock. Once locked, settings cannot be reduced without Wasabi
+ // support.
LockTime *string `type:"string"`
- // An integer for the minimum number of days that objects are always
- // retained after their creation date or release from conditional hold. You
- // can extend the retention date for any individual object, but may not
- // shorten the date. This parameter is always required.
+ // Required minimum retention after creation or release from conditional hold.
+ // Per-object retention may be extended, but not shortened.
RetentionDays *int64 `type:"integer"`
- // A Boolean value indicating if newly created objects are placed on
- // conditional hold, meaning that they cannot be deleted until the
- // con­ditional hold is explicitly turned off. The default is false if this
- // parameter is not given. Note that this setting may be changed even after
- // the settings are locked.
+ // Places new objects on hold. Defaults to false and may be changed after the
+ // settings are locked.
ConditionalHold *bool `type:"boolean"`
- // A Boolean value indicating if the object should be deleted automatically
- // at the end of the retention period. The default is to not delete objects
- // after the reten­tion period. Note that this setting may be changed even
- // after the settings are locked.
+ // Defaults to false and may be changed after the settings are locked.
DeleteAfterRetention *bool `type:"boolean"`
}
@@ -473,22 +428,14 @@ type PutObjectComplianceInput struct {
type ObjectComplianceConfiguration struct {
_ struct{} `type:"structure"`
- // An ISO time giving a new retention time for the object in which the
- // object cannot be deleted before this time. Note that the new retention
- // time must be past the reten­tion period given by the bucket policy or an
- // error is returned.
+ // Must be later than the retention required by the bucket policy.
RetentionTime *string `type:"string"`
- // A Boolean value "false" to release the object from the conditional hold
- // setting in the bucket policy. The retention period in days is started
- // from the point when the con­ditional hold is released. Once the
- // conditional hold is set false, it may not be returned to conditional
- // hold.
+ // Setting this to false releases the hold, starts the retention period, and
+ // cannot be reversed.
ConditionalHold *bool `type:"boolean"`
- // A Boolean value "true" or "false" to set the legal hold status. When an
- // object has a legal hold status of true, the object cannot be deleted
- // regardless of the retention period.
+ // A legal hold prevents deletion regardless of the retention period.
LegalHold *bool `type:"boolean"`
}
diff --git a/server/pkg/repo/file.go b/server/pkg/repo/file.go
index fa1c293c46..87a22c5f5d 100644
--- a/server/pkg/repo/file.go
+++ b/server/pkg/repo/file.go
@@ -188,17 +188,9 @@ func (repo *FileRepository) CreateMetaFile(
return &file, stacktrace.Propagate(err, "")
}
-// markAsNeedingReplication inserts new entries in object_copies, setting the
-// current hot DC as the source copy.
-//
-// The higher layer above us (file controller) would've already checked that the
-// object exists in the current hot DC (See `c.sizeOf` in file controller). This
-// would cover cases where the client fetched presigned upload URLs for say
-// hotDC1, but by the time they connected to museum, museum switched to using
-// hotDC2. So then when museum would try to fetch the file size from hotDC2, the
-// object won't be found there, and the upload would fail (which is the
-// behaviour we want, since hot DC swaps are not a frequent/expected operation,
-// we just wish to guarantee correctness if they do happen).
+// The controller first verifies the object in hotDC. This prevents recording an
+// upload against another bucket if hot storage changes after the client receives
+// its presigned URL.
func (repo *FileRepository) markAsNeedingReplication(ctx context.Context, tx *sql.Tx, file ente.File, hotDC string) error {
if hotDC == repo.S3Config.GetHotBackblazeDC() {
err := repo.ObjectCopiesRepo.CreateNewB2Object(ctx, tx, file.File.ObjectKey, true, true)
@@ -220,7 +212,6 @@ func (repo *FileRepository) markAsNeedingReplication(ctx context.Context, tx *sq
}
}
-// See markAsNeedingReplication - this variant is for updating only thumbnails.
func (repo *FileRepository) markThumbnailAsNeedingReplication(ctx context.Context, tx *sql.Tx, thumbnailObjectKey string, hotDC string) error {
if hotDC == repo.S3Config.GetHotBackblazeDC() {
err := repo.ObjectCopiesRepo.CreateNewB2Object(ctx, tx, thumbnailObjectKey, true, false)
@@ -343,9 +334,8 @@ func (repo *FileRepository) Update(file ente.File, fileSize int64, thumbnailSize
return stacktrace.Propagate(err, "")
}
if isDuplicateRequest {
- // Skip markAsNeedingReplication for duplicate requests, it'd fail with
- // pq: duplicate key value violates unique constraint \"object_copies_pkey\"
- // and render our transaction uncommittable
+ // Re-inserting object_copies on a duplicate request violates its primary
+ // key and leaves the transaction uncommittable.
log.Infof("Skipping update of object_copies for a duplicate request to update file %d", file.ID)
} else {
err = repo.markAsNeedingReplication(ctx, tx, file, hotDC)
@@ -694,7 +684,6 @@ func (repo *FileRepository) GetFileAttributes(fileID int64) (*ente.File, error)
}
func (repo *FileRepository) DropFilesMetadata(ctx context.Context, fileIDs []int64) error {
- // ensure that the fileIDs are not present in object_keys
rows, err := repo.DB.QueryContext(ctx, `SELECT distinct(file_id) FROM object_keys WHERE file_id = ANY($1)`, pq.Array(fileIDs))
if err != nil {
return stacktrace.Propagate(err, "")
diff --git a/server/pkg/repo/object.go b/server/pkg/repo/object.go
index 801e2216f7..49faa9cd9d 100644
--- a/server/pkg/repo/object.go
+++ b/server/pkg/repo/object.go
@@ -87,15 +87,11 @@ func (repo *ObjectRepository) GetObjectWithDCs(fileID int64, objType ente.Object
return s3ObjectKey, datacenters, stacktrace.Propagate(err, "")
}
-// GetAccessibleObject verifies actor access and returns the S3 object key in a
-// single latency-sensitive DB query.
func (repo *ObjectRepository) GetAccessibleObject(ctx context.Context, fileID int64, actorUserID int64, objType ente.ObjectType) (ente.S3ObjectKey, error) {
s3ObjectKey, _, err := repo.GetAccessibleObjectWithDCs(ctx, fileID, actorUserID, objType)
return s3ObjectKey, err
}
-// GetAccessibleObjectWithDCs verifies actor access and returns the S3 object
-// key along with replicated datacenters in a single latency-sensitive DB query.
func (repo *ObjectRepository) GetAccessibleObjectWithDCs(ctx context.Context, fileID int64, actorUserID int64, objType ente.ObjectType) (ente.S3ObjectKey, []string, error) {
row := repo.objectLookupDB().QueryRowContext(ctx, `
SELECT
@@ -153,15 +149,11 @@ func (repo *ObjectRepository) GetAccessibleObjectWithDCs(ctx context.Context, fi
return s3ObjectKey, datacenters, nil
}
-// GetOwnedObject verifies ownership and returns the S3 object key in a single
-// latency-sensitive DB query.
func (repo *ObjectRepository) GetOwnedObject(ctx context.Context, fileID int64, ownerID int64, objType ente.ObjectType) (ente.S3ObjectKey, error) {
s3ObjectKey, _, err := repo.GetOwnedObjectWithDCs(ctx, fileID, ownerID, objType)
return s3ObjectKey, err
}
-// GetOwnedObjectWithDCs verifies ownership and returns the S3 object key along
-// with replicated datacenters in a single latency-sensitive DB query.
func (repo *ObjectRepository) GetOwnedObjectWithDCs(ctx context.Context, fileID int64, ownerID int64, objType ente.ObjectType) (ente.S3ObjectKey, []string, error) {
row := repo.objectLookupDB().QueryRowContext(ctx, `
SELECT
@@ -208,16 +200,11 @@ func (repo *ObjectRepository) GetOwnedObjectWithDCs(ctx context.Context, fileID
return s3ObjectKey, datacenters, nil
}
-// GetCollectionObject verifies collection membership and returns the S3 object
-// key in a single latency-sensitive DB query.
func (repo *ObjectRepository) GetCollectionObject(ctx context.Context, collectionID int64, fileID int64, objType ente.ObjectType) (ente.S3ObjectKey, error) {
s3ObjectKey, _, err := repo.GetCollectionObjectWithDCs(ctx, collectionID, fileID, objType)
return s3ObjectKey, err
}
-// GetCollectionObjectWithDCs verifies collection membership and returns the S3
-// object key along with replicated datacenters in a single latency-sensitive DB
-// query.
func (repo *ObjectRepository) GetCollectionObjectWithDCs(ctx context.Context, collectionID int64, fileID int64, objType ente.ObjectType) (ente.S3ObjectKey, []string, error) {
row := repo.objectLookupDB().QueryRowContext(ctx, `
SELECT
@@ -303,8 +290,6 @@ func (repo *ObjectRepository) RemoveObjectsForKey(objectKey string) error {
return stacktrace.Propagate(err, "")
}
-// MarkObjectsAsDeletedForFileIDs marks the object keys corresponding to the given filesIDs as deleted
-// The actual deletion happens later when the queue is processed
func (repo *ObjectRepository) MarkObjectsAsDeletedForFileIDs(ctx context.Context, tx *sql.Tx, fileIDs []int64) ([]ente.S3ObjectKey, error) {
rows, err := tx.QueryContext(ctx, `SELECT file_id, o_type, object_key, size FROM object_keys
WHERE file_id = ANY($1) AND is_deleted=false FOR UPDATE`, pq.Array(fileIDs))
@@ -437,11 +422,7 @@ func (repo *ObjectRepository) GetObjectReferenceStatuses(ctx context.Context, ob
return statuses, stacktrace.Propagate(rows.Err(), "")
}
-// GetObjectState returns various bits of information about an object that are
-// useful in pre-flight checks during replication.
-//
-// Unknown objects (i.e. objectKeys for which there are no entries) are
-// considered as deleted.
+// Unknown object keys are treated as deleted.
func (repo *ObjectRepository) GetObjectState(objectKey string) (ObjectState ente.ObjectState, err error) {
row := repo.DB.QueryRow(`
SELECT ok.is_deleted, u.encrypted_email IS NULL AS is_user_deleted, ok.size
diff --git a/server/pkg/repo/object_copies.go b/server/pkg/repo/object_copies.go
index 4d198cd522..389651fa19 100644
--- a/server/pkg/repo/object_copies.go
+++ b/server/pkg/repo/object_copies.go
@@ -15,11 +15,6 @@ type ObjectCopiesRepository struct {
DB *sql.DB
}
-// GetAndLockUnreplicatedObject gets an object which is not yet replicated to
-// all the replicas. It also registers a replication to keep the row corresponding
-// to that object to be blocked for 24h before next replication attemp.
-//
-// ObjectCopies is guaranteed to be nil if error is not nil.
func (repo *ObjectCopiesRepository) GetAndLockUnreplicatedObject(ctx context.Context) (*ente.ObjectCopies, error) {
tx, err := repo.DB.BeginTx(ctx, nil)
if err != nil {
@@ -71,13 +66,6 @@ func (repo *ObjectCopiesRepository) GetAndLockUnreplicatedObject(ctx context.Con
return &r, nil
}
-// CreateNewB2Object creates a new entry for objectKey and marks it as having
-// being replicated to B2. It then sets provided flags to mark this object as
-// requiring replication where needed.
-//
-// This operation runs within the context of a transaction that creates the
-// initial entry for the file in the database; thus, it gets passed ctx and tx
-// which it uses to scope its own DB changes.
func (repo *ObjectCopiesRepository) CreateNewB2Object(ctx context.Context, tx *sql.Tx, objectKey string, wantWasabi bool, wantScaleway bool) error {
_, err := tx.ExecContext(ctx, `
INSERT INTO object_copies (object_key, want_b2, b2, want_wasabi, want_scw)
@@ -86,10 +74,6 @@ func (repo *ObjectCopiesRepository) CreateNewB2Object(ctx context.Context, tx *s
return stacktrace.Propagate(err, "")
}
-// CreateNewWasabiObject creates a new entry for objectKey and marks it as having
-// being replicated to Wasabi.
-//
-// See CreateNewB2Object for details.
func (repo *ObjectCopiesRepository) CreateNewWasabiObject(ctx context.Context, tx *sql.Tx, objectKey string, wantB2 bool, wantScaleway bool) error {
_, err := tx.ExecContext(ctx, `
INSERT INTO object_copies (object_key, want_wasabi, wasabi, want_b2, want_scw)
@@ -98,8 +82,6 @@ func (repo *ObjectCopiesRepository) CreateNewWasabiObject(ctx context.Context, t
return stacktrace.Propagate(err, "")
}
-// RegisterReplicationAttempt sets the last_attempt timestamp so that this row can
-// be skipped over for the next day in case the replication was not succesful.
func (repo *ObjectCopiesRepository) RegisterReplicationAttempt(tx *sql.Tx, ctx context.Context, objectKey string) error {
_, err := tx.ExecContext(ctx, `
UPDATE object_copies
diff --git a/server/pkg/repo/trash.go b/server/pkg/repo/trash.go
index a89dcfca77..1dfe4842c9 100644
--- a/server/pkg/repo/trash.go
+++ b/server/pkg/repo/trash.go
@@ -239,8 +239,6 @@ func (t *TrashRepository) Delete(ctx context.Context, userID int64, fileIDs []in
return tx.Commit()
}
-// GetFilesInTrashState for a given userID and fileIDs, return the list of fileIDs which are actually present in
-// trash and is not deleted or restored yet.
func (t *TrashRepository) GetFilesInTrashState(ctx context.Context, userID int64, fileIDs []int64) ([]int64, bool, error) {
rows, err := t.DB.Query(`SELECT file_id FROM trash
WHERE user_id = $1 AND file_id = ANY ($2)
@@ -264,11 +262,6 @@ func (t *TrashRepository) GetFilesInTrashState(ctx context.Context, userID int64
return fileIDsInTrash, canRestoreOrDeleteAllFiles, nil
}
-// GetFilesInTrashOrDeleted returns the subset of fileIDs that are either in trash (not restored)
-// or have been permanently deleted. This is useful for validation checks to prevent operations
-// on files that are no longer in an active state.
-// Unlike GetFilesInTrashState, this method does not log warnings when files are not found,
-// making it suitable for validation checks where files are expected to NOT be in trash.
func (t *TrashRepository) GetFilesInTrashOrDeleted(ctx context.Context, userID int64, fileIDs []int64) ([]int64, error) {
rows, err := t.DB.QueryContext(ctx, `SELECT file_id FROM trash
WHERE user_id = $1 AND file_id = ANY ($2)
@@ -389,11 +382,8 @@ func (t *TrashRepository) GetUserIDToFileIDsMapForDeletion() (map[int64][]int64,
return result, nil
}
-// GetFileIdsForDroppingMetadata retrieves file IDs of deleted files for metadata scrubbing.
-// It returns files that were deleted after the provided timestamp (sinceUpdatedAt) and have been in the trash for at least 50 days.
-// This delay ensures compliance with deletion locks.
-// The method orders the results by the 'updated_at' field in ascending order and limits the results to 'TrashDiffLimit' + 1.
-// If multiple files have the same 'updated_at' timestamp and are at the limit boundary, they are excluded to prevent partial scrubbing.
+// Wait 50 days for compliance deletion locks, and never split an updated_at
+// group across batches.
func (t *TrashRepository) GetFileIdsForDroppingMetadata(sinceUpdatedAt int64) ([]FileWithUpdatedAt, error) {
rows, err := t.DB.Query(`
select file_id, updated_at from trash where is_deleted=true AND updated_at > $1
@@ -422,9 +412,6 @@ order by updated_at ASC limit $2
return fileWithUpdatedAt, nil
}
- // from the end ignore the fileIds from fileWithUpdatedAt that have the same updatedAt.
- // this is to avoid scrubbing partial list of files that have same updatedAt as due to the limit not
- // all files with the same updatedAt are returned.
lastUpdatedAt := fileWithUpdatedAt[len(fileWithUpdatedAt)-1].UpdatedAt
var i = len(fileWithUpdatedAt) - 1
for ; i >= 0; i-- {
diff --git a/server/pkg/utils/s3config/s3config.go b/server/pkg/utils/s3config/s3config.go
index 6e20454bda..af32545116 100644
--- a/server/pkg/utils/s3config/s3config.go
+++ b/server/pkg/utils/s3config/s3config.go
@@ -13,14 +13,8 @@ import (
"github.com/spf13/viper"
)
-// Objects are replicated in multiple "data centers". Each data center is an
-// S3-compatible provider, and has an associated "bucket".
-//
-// The list of data centers are not arbitrarily configurable - while we pick the
-// exact credentials, endpoint and the bucket from the runtime configuration
-// file, the code has specific logic to deal with the quirks of specific data
-// centers. So as such, the constants used to specify these data centers in the
-// YAML configuration matter.
+// Bucket IDs are fixed because provider-specific code depends on their exact
+// values. Credentials, endpoints, and bucket names remain configurable.
type S3Config struct {
buckets map[string]string
hotDC string
@@ -33,41 +27,10 @@ type S3Config struct {
// debugging workarounds; not tested/intended for production.
areLocalBuckets bool
- // If for particular object type, the bucket is not specified, it will
- // default to hotDC as the bucket with no replicas. Initially, this config won't support
- // existing objectType (file, thumbnail) and will be used for new objectTypes. In the future,
- // we can migrate existing objectTypes to this config.
fileDataConfig FileDataConfig
- // If no explicit config is present, attachments default to hot storage with no replicas.
attachmentConfig AttachmentConfig
}
-// # Datacenters
-// Note: We are now renaming datacenter names to bucketID. Till the migration is completed, you will see usage of both
-// terminology.
-// Below are some high level details about the three replicas ("data centers")
-// that are in use. There are a few other legacy ones too.
-//
-// # Backblaze (dcB2EuropeCentral)
-//
-// - Primary hot storage
-// - Versioned, but with extra code that undoes all overwrites
-//
-// # Wasabi (dcWasabiEuropeCentral_v3)
-//
-// - Secondary hot storage
-// - Objects stay under compliance, which prevents them from being
-// deleted/updated for 21 days.
-// - Not versioned (versioning is not needed since objects cannot be overwritten)
-// - When an user (permanently) deletes an object, we remove the compliance
-// retention. It can then be deleted normally when the scheduled
-// cleanup happens (as long as it happens after 21 days).
-//
-// # Scaleway (dcSCWEuropeFrance_v3)
-//
-// - Cold storage
-// - Specify type GLACIER in API requests
-
var (
dcB2EuropeCentral string = "b2-eu-cen"
dcSCWEuropeFranceDeprecated string = "scw-eu-fr"
@@ -163,7 +126,6 @@ func (config *S3Config) GetBucket(dcOrBucketID string) *string {
return &bucket
}
-// GetBucketID returns the bucket ID for the given object type. Note: existing dc are renamed as bucketID
func (config *S3Config) GetBucketID(oType ente.ObjectType) string {
if config.fileDataConfig.HasConfig(oType) {
return config.fileDataConfig.GetPrimaryBucketID(oType)
@@ -258,11 +220,6 @@ func (config *S3Config) GetColdScalewayDC() string {
return dcSCWEuropeFrance_v3
}
-// ShouldDeleteFromDataCenter returns true if objects should be deleted from the
-// given data center when permanently deleting these objects.
-//
-// There are some legacy / deprecated data center values which are no longer
-// being used, and it returns false for such data centers.
func (config *S3Config) ShouldDeleteFromDataCenter(dc string) bool {
return dc != dcSCWEuropeFranceDeprecated && dc != dcSCWEuropeFranceLockedDeprecated && dc != dcWasabiEuropeCentralDeprecated
}
@@ -274,8 +231,6 @@ func (config *S3Config) WasabiComplianceDC() string {
return ""
}
-// Return true if we're using local minio buckets. This can then be used to add
-// various workarounds for debugging locally; not meant for production use.
func (config *S3Config) AreLocalBuckets() bool {
return config.areLocalBuckets
}