Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

6.8 KB
Raw Blame
package filedata

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/ente/museum/ente"
	fileData "github.com/ente/museum/ente/filedata"
	"github.com/ente/museum/pkg/utils/file"
	"github.com/ente/stacktrace"
	log "github.com/sirupsen/logrus"
	"io"
	"os"
	"strings"
	stime "time"
)

const PreSignedRequestValidityDuration = 7 * 24 * stime.Hour

func (c *Controller) getUploadURL(object ente.TempObject) (string, error) {
	s3Client := c.S3Config.GetS3Client(object.BucketId)
	r, _ := s3Client.PutObjectRequest(&s3.PutObjectInput{
		Bucket:        c.S3Config.GetBucket(object.BucketId),
		Key:           &object.ObjectKey,
		ContentLength: object.ContentLength,
		ContentMD5:    object.ContentMD5,
	})
	url, err := r.Presign(PreSignedRequestValidityDuration)
	if err != nil {
		return "", stacktrace.Propagate(err, "")
	}
	err = c.ObjectCleanupController.AddTempObject(object)
	if err != nil {
		return "", stacktrace.Propagate(err, "")
	}
	return url, nil
}
func (c *Controller) getMultiPartUploadURL(object ente.TempObject, count int64, partLength int64, partMD5s []string) (*ente.MultipartUploadURLs, error) {
	s3Client := c.S3Config.GetS3Client(object.BucketId)
	bucket := c.S3Config.GetBucket(object.BucketId)
	r, err := s3Client.CreateMultipartUpload(&s3.CreateMultipartUploadInput{
		Bucket: bucket,
		Key:    &object.ObjectKey,
	})
	if err != nil {
		return nil, stacktrace.Propagate(err, "")
	}
	object.IsMultipart = true
	object.UploadID = *r.UploadId
	err = c.ObjectCleanupController.AddTempObject(object)
	if err != nil {
		return nil, stacktrace.Propagate(err, "")
	}
	multipartUploadURLs := ente.MultipartUploadURLs{ObjectKey: object.ObjectKey}
	urls := make([]string, 0)
	for i := int64(1); i <= count; i++ {
		input := &s3.UploadPartInput{
			Bucket:     bucket,
			Key:        &object.ObjectKey,
			UploadId:   r.UploadId,
			PartNumber: &i,
		}
		if object.ContentLength != nil {
			input.ContentLength = aws.Int64(min(partLength, *object.ContentLength-(i-1)*partLength))
			input.ContentMD5 = &partMD5s[i-1]
		}
		partReq, _ := s3Client.UploadPartRequest(input)
		partUrl, partUrlErr := partReq.Presign(PreSignedRequestValidityDuration)
		if partUrlErr != nil {
			return nil, stacktrace.Propagate(partUrlErr, "")
		}
		urls = append(urls, partUrl)
	}
	multipartUploadURLs.PartURLs = urls
	r2, _ := s3Client.CompleteMultipartUploadRequest(&s3.CompleteMultipartUploadInput{
		Bucket:   bucket,
		Key:      &object.ObjectKey,
		UploadId: r.UploadId,
	})
	url, err := r2.Presign(PreSignedRequestValidityDuration)
	if err != nil {
		return nil, stacktrace.Propagate(err, "")
	}
	multipartUploadURLs.CompleteURL = url
	return &multipartUploadURLs, nil
}

func (c *Controller) signedUrlGet(dc string, objectKey string) (*ente.UploadURL, error) {
	s3Client := c.S3Config.GetS3Client(dc)
	input := &s3.GetObjectInput{
		Bucket: c.S3Config.GetBucket(dc),
		Key:    &objectKey,
	}
	input.ResponseContentDisposition = aws.String("attachment")
	r, _ := s3Client.GetObjectRequest(input)
	url, err := r.Presign(PreSignedRequestValidityDuration)
	if err != nil {
		return nil, stacktrace.Propagate(err, "")
	}
	return &ente.UploadURL{ObjectKey: objectKey, URL: url}, nil
}

func (c *Controller) downloadObject(ctx context.Context, objectKey string, dc string) (fileData.S3FileMetadata, error) {
	var obj fileData.S3FileMetadata
	buff := &aws.WriteAtBuffer{}
	bucket := c.S3Config.GetBucket(dc)
	downloader := c.downloadManagerCache[dc]
	_, err := downloader.DownloadWithContext(ctx, buff, &s3.GetObjectInput{
		Bucket: bucket,
		Key:    &objectKey,
	})
	if err != nil {
		return obj, err
	}
	err = json.Unmarshal(buff.Bytes(), &obj)
	if err != nil {
		return obj, stacktrace.Propagate(err, "unmarshal failed")
	}
	return obj, nil
}

func (c *Controller) uploadObject(obj fileData.S3FileMetadata, objectKey string, dc string) (int64, error) {
	embeddingObj, _ := json.Marshal(obj)
	s3Client := c.S3Config.GetS3Client(dc)
	s3Bucket := c.S3Config.GetBucket(dc)
	uploader := s3manager.NewUploaderWithClient(&s3Client)
	up := s3manager.UploadInput{
		Bucket: s3Bucket,
		Key:    &objectKey,
		Body:   bytes.NewReader(embeddingObj),
	}
	var err error
	var result *s3manager.UploadOutput
	for range 3 {
		result, err = uploader.Upload(&up)
		if err == nil || !strings.Contains(err.Error(), "connection reset by peer") {
			break
		}
		stime.Sleep(50 * stime.Millisecond)
	}
	if err != nil {
		log.Error(err)
		return -1, stacktrace.Propagate(err, "metadata upload failed")
	}
	log.Infof("Uploaded to bucket %s", result.Location)
	return int64(len(embeddingObj)), nil
}

func (c *Controller) verifySize(bucketID string, objectKey string, expectedSize int64) error {
	s3Client := c.S3Config.GetS3Client(bucketID)
	bucket := c.S3Config.GetBucket(bucketID)
	res, err := s3Client.HeadObject(&s3.HeadObjectInput{
		Bucket: bucket,
		Key:    &objectKey,
	})
	if err != nil {
		return stacktrace.Propagate(err, "Fetching object info from bucket %s failed", *bucket)
	}

	if *res.ContentLength != expectedSize {
		err = fmt.Errorf("size of the uploaded file (%d) does not match the expected size (%d) in bucket %s",
			*res.ContentLength, expectedSize, *bucket)
		return stacktrace.Propagate(err, "")
	}
	return nil
}

type ReplicateObjectReq struct {
	ObjectKey    string
	SrcBucketID  string
	DestBucketID string
	ObjectSize   int64
}

func (c *Controller) replicateObject(ctx context.Context, req *ReplicateObjectReq) error {
	if err := file.EnsureSufficientSpace(req.ObjectSize); err != nil {
		return stacktrace.Propagate(err, "")
	}
	filePath, file, err := file.CreateTemporaryFile(c.tempStorage, req.ObjectKey)
	if err != nil {
		return stacktrace.Propagate(err, "Failed to create temporary file")
	}
	defer os.Remove(filePath)
	defer file.Close()
	bucket := c.S3Config.GetBucket(req.SrcBucketID)
	downloader := c.downloadManagerCache[req.SrcBucketID]
	_, err = downloader.DownloadWithContext(ctx, file, &s3.GetObjectInput{
		Bucket: bucket,
		Key:    &req.ObjectKey,
	})
	if err != nil {
		return stacktrace.Propagate(err, "Failed to download object from bucket %s", req.SrcBucketID)
	}
	if err := c.verifySize(req.SrcBucketID, req.ObjectKey, req.ObjectSize); err != nil {
		return stacktrace.Propagate(err, "")
	}
	dstClient := c.S3Config.GetS3Client(req.DestBucketID)
	uploader := s3manager.NewUploaderWithClient(&dstClient)
	file.Seek(0, io.SeekStart)
	up := s3manager.UploadInput{
		Bucket: c.S3Config.GetBucket(req.DestBucketID),
		Key:    aws.String(req.ObjectKey),
		Body:   file,
	}
	result, err := uploader.Upload(&up)
	if err != nil {
		return stacktrace.Propagate(err, "Failed to upload object to bucket %s", req.DestBucketID)
	}
	log.Infof("Uploaded to bucket %s", result.Location)
	if err := c.verifySize(req.DestBucketID, req.ObjectKey, req.ObjectSize); err != nil {
		return stacktrace.Propagate(err, "")
	}
	return nil
}