π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
package controller
import (
"database/sql"
"errors"
"strings"
stime "time"
"github.com/aws/aws-sdk-go/aws"
"github.com/ente/museum/pkg/external/wasabi"
"github.com/ente/stacktrace"
"github.com/spf13/viper"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/ente/museum/ente"
"github.com/ente/museum/pkg/repo"
"github.com/ente/museum/pkg/utils/s3config"
"github.com/ente/museum/pkg/utils/time"
log "github.com/sirupsen/logrus"
)
type ObjectCleanupController struct {
Repo *repo.ObjectCleanupRepository
ObjectRepo *repo.ObjectRepository
S3Config *s3config.S3Config
}
const PreSignedRequestValidityDuration = 7 * 24 * stime.Hour
const PreSignedPartUploadRequestDuration = 7 * 24 * stime.Hour
func NewObjectCleanupController(
objectCleanupRepo *repo.ObjectCleanupRepository,
objectRepo *repo.ObjectRepository,
s3Config *s3config.S3Config,
) *ObjectCleanupController {
return &ObjectCleanupController{
Repo: objectCleanupRepo,
ObjectRepo: objectRepo,
S3Config: s3Config,
}
}
func (c *ObjectCleanupController) StartRemovingUnreportedObjects() {
// TODO: object_cleanup: This code is only currently tested for B2
if c.S3Config.GetHotDataCenter() != c.S3Config.GetHotBackblazeDC() {
log.Info("Skipping RemovingUnreportedObjects since the Hot DC is not B2")
return
}
workerCount := viper.GetInt("jobs.remove-unreported-objects.worker-count")
if workerCount == 0 {
workerCount = 1
}
log.Infof("Starting %d workers to remove-unreported-objects", workerCount)
for i := 0; i < workerCount; i++ {
go c.removeUnreportedObjectsWorker(i)
}
}
func (c *ObjectCleanupController) removeUnreportedObjectsWorker(i int) {
for {
count := c.removeUnreportedObjects()
if count == 0 {
stime.Sleep(stime.Duration(5+i) * stime.Minute)
} else {
stime.Sleep(stime.Second)
}
}
}
func (c *ObjectCleanupController) removeUnreportedObjects() int {
logger := log.WithFields(log.Fields{
"task": "remove-unreported-objects",
})
logger.Info("Removing unreported objects")
count := 0
tx, tempObjects, err := c.Repo.GetAndLockExpiredObjects()
if err != nil {
logger.Error(err)
return count
}
defer tx.Rollback()
for _, tempObject := range tempObjects {
err = c.removeUnreportedObject(tx, tempObject)
if err != nil {
continue
}
count += 1
}
logger.Infof("Removed %d objects", count)
// Always commit the batch. skip() delays failed rows to prevent a tight loop.
cerr := tx.Commit()
if cerr != nil {
cerr = stacktrace.Propagate(cerr, "Failed to commit transaction")
logger.Error(cerr)
}
return count
}
func (c *ObjectCleanupController) removeUnreportedObject(tx *sql.Tx, t ente.TempObject) error {
// TODO: object_cleanup
// This should use the DC from TempObject (once we start persisting it)
dc := t.BucketId
if dc == "" {
dc = c.S3Config.GetHotDataCenter()
}
logger := log.WithFields(log.Fields{
"task": "remove-unreported-objects",
"object_key": t.ObjectKey,
"data_center": dc,
"upload_id": t.UploadID,
})
skip := func(err error) error {
logger.Errorf("Clearing tempObject failed: %v", err)
newExpiry := time.MicrosecondsAfterDays(1)
serr := c.Repo.SetExpiryForTempObject(tx, t, newExpiry)
if serr != nil {
logger.Errorf("Updating expiry for failed temp object failed: %v", serr)
}
return err
}
logger.Info("Clearing tempObject")
exists, err := c.ObjectRepo.DoesObjectExist(tx, t.ObjectKey)
if err != nil {
return skip(stacktrace.Propagate(err, ""))
}
if exists {
err := errors.New("aborting attempt to delete an object which has a DB entry")
return skip(stacktrace.Propagate(err, ""))
}
if t.IsMultipart {
err = c.abortMultipartUpload(t.ObjectKey, t.UploadID, dc)
if err != nil {
return skip(err)
}
}
err = c.DeleteObjectFromDataCenter(t.ObjectKey, dc)
if err != nil {
return skip(err)
}
err = c.Repo.RemoveTempObject(tx, t)
if err != nil {
return skip(err)
}
return nil
}
func (c *ObjectCleanupController) AddTempObject(object ente.TempObject) error {
validity := PreSignedRequestValidityDuration
if object.IsMultipart {
validity = PreSignedPartUploadRequestDuration
}
expiry := time.Microseconds() + 2*validity.Microseconds()
err := c.Repo.AddTempObject(object, expiry)
return stacktrace.Propagate(err, "")
}
func (c *ObjectCleanupController) DeleteAllObjectsWithPrefix(prefix string, dc string) error {
s3Client := c.S3Config.GetS3Client(dc)
bucket := c.S3Config.GetBucket(dc)
var continuationToken *string
for {
output, err := s3Client.ListObjectsV2(&s3.ListObjectsV2Input{
Bucket: bucket,
Prefix: &prefix,
ContinuationToken: continuationToken,
})
if err != nil {
log.WithFields(log.Fields{
"prefix": prefix,
"dc": dc,
}).WithError(err).Error("Failed to list objects")
return stacktrace.Propagate(err, "")
}
for _, obj := range output.Contents {
err := c.DeleteObjectFromDataCenter(*obj.Key, dc)
if err != nil {
log.WithFields(log.Fields{
"object_key": *obj.Key,
"dc": dc,
}).WithError(err).Error("Failed to delete object")
return stacktrace.Propagate(err, "")
}
}
if !aws.BoolValue(output.IsTruncated) {
break
}
nextContinuationToken := output.NextContinuationToken
if aws.StringValue(nextContinuationToken) == "" ||
aws.StringValue(nextContinuationToken) == aws.StringValue(continuationToken) {
return errors.New("truncated object listing did not provide a new continuation token")
}
continuationToken = nextContinuationToken
}
return nil
}
func (c *ObjectCleanupController) DeleteObjectFromDataCenter(objectKey string, dc string) error {
log.Info("Deleting " + objectKey + " from " + dc)
var s3Client = c.S3Config.GetS3Client(dc)
bucket := c.S3Config.GetBucket(dc)
_, err := s3Client.DeleteObject(&s3.DeleteObjectInput{
Bucket: bucket,
Key: &objectKey,
})
if err != nil {
return stacktrace.Propagate(err, "")
}
err = s3Client.WaitUntilObjectNotExists(&s3.HeadObjectInput{
Bucket: bucket,
Key: &objectKey,
})
if err != nil {
return stacktrace.Propagate(err, "")
}
return nil
}
func (c *ObjectCleanupController) disableConditionalHoldIfPresent(dc string, objectKey string) error {
s3Client := c.S3Config.GetS3Client(dc)
bucket := c.S3Config.GetBucket(dc)
_, err := wasabi.PutObjectCompliance(&s3Client, &wasabi.PutObjectComplianceInput{
Bucket: bucket,
Key: aws.String(objectKey),
ObjectComplianceConfiguration: &wasabi.ObjectComplianceConfiguration{
ConditionalHold: aws.Bool(false),
},
})
if err != nil {
// Missing objects do not need compliance-hold cleanup. Outdated objects may not
// have reached Wasabi, or may already be gone there, while still needing cleanup
// from the other active data centers.
if strings.Contains(err.Error(), "NoSuchKey") || strings.Contains(err.Error(), "NotFound") {
return nil
}
return stacktrace.Propagate(err, "Failed to update ObjectCompliance for %s/%s", *bucket, objectKey)
}
return nil
}
func (c *ObjectCleanupController) abortMultipartUpload(objectKey string, uploadID string, dc string) error {
s3Client := c.S3Config.GetS3Client(dc)
bucket := c.S3Config.GetBucket(dc)
_, err := s3Client.AbortMultipartUpload(&s3.AbortMultipartUploadInput{
Bucket: bucket,
Key: &objectKey,
UploadId: &uploadID,
})
if err != nil {
if isUnknownUploadError(err) {
log.Info("Could not find upload for " + objectKey)
return nil
}
return stacktrace.Propagate(err, "")
}
r, err := s3Client.ListParts(&s3.ListPartsInput{
Bucket: bucket,
Key: &objectKey,
UploadId: &uploadID,
})
if err != nil {
if isUnknownUploadError(err) {
return nil
}
return stacktrace.Propagate(err, "")
}
if len(r.Parts) > 0 {
return stacktrace.NewError("abort Failed")
}
return nil
}
func isUnknownUploadError(err error) bool {
// B2, Wasabi
if strings.Contains(err.Error(), "NoSuchUpload") {
return true
}
// Scaleway
if strings.Contains(err.Error(), "NoSuchKey") {
return true
}
return false
}