Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

29.9 KB
Raw Blame
package repo

import (
	"cmp"
	"context"
	"database/sql"
	"errors"
	"fmt"
	"slices"
	"strconv"
	"strings"

	"github.com/ente/stacktrace"
	log "github.com/sirupsen/logrus"

	"github.com/ente/museum/ente"
	"github.com/ente/museum/pkg/utils/dbmetrics"
	"github.com/ente/museum/pkg/utils/s3config"
	"github.com/ente/museum/pkg/utils/time"
	"github.com/lib/pq"
)

type FileRepository struct {
	DB                *sql.DB
	S3Config          *s3config.S3Config
	QueueRepo         *QueueRepository
	ObjectRepo        *ObjectRepository
	ObjectCleanupRepo *ObjectCleanupRepository
	ObjectCopiesRepo  *ObjectCopiesRepository
}

func (repo *FileRepository) Create(
	file ente.File,
	fileSize int64,
	thumbnailSize int64,
	usageDiff int64,
	collectionOwnerID int64,
	app ente.App,
) (ente.File, int64, error) {
	hotDC := repo.S3Config.GetHotDataCenter()
	dcsForNewEntry := pq.StringArray{hotDC}
	if file.OwnerID != collectionOwnerID {
		return file, -1, stacktrace.Propagate(errors.New("both file and collection should belong to same owner"), "")
	}

	ctx := context.Background()
	finishPhase := dbmetrics.Start(dbmetrics.FileCreate, "begin_with_checkout")
	tx, err := repo.DB.BeginTx(ctx, nil)
	finishPhase(err)
	if err != nil {
		return file, -1, stacktrace.Propagate(err, "")
	}
	defer tx.Rollback()
	var fileID int64
	finishPhase = dbmetrics.Start(dbmetrics.FileCreate, "insert_file")
	err = tx.QueryRowContext(ctx, `INSERT INTO files
			(owner_id, app, encrypted_metadata,
			file_decryption_header, thumbnail_decryption_header, metadata_decryption_header,
			magic_metadata, pub_magic_metadata, info, updation_time)
			VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING file_id`,
		file.OwnerID, app, file.Metadata.EncryptedData, file.File.DecryptionHeader,
		file.Thumbnail.DecryptionHeader, file.Metadata.DecryptionHeader,
		file.MagicMetadata, file.PublicMagicMetadata, file.Info,
		file.UpdationTime).Scan(&fileID)
	finishPhase(err)
	if err != nil {
		return file, -1, stacktrace.Propagate(err, "")
	}
	file.ID = fileID
	finishPhase = dbmetrics.Start(dbmetrics.FileCreate, "insert_membership")
	_, err = tx.ExecContext(ctx, `INSERT INTO collection_files
			(collection_id, file_id, encrypted_key, key_decryption_nonce, is_deleted, updation_time, c_owner_id, f_owner_id)
			VALUES($1, $2, $3, $4, $5, $6, $7, $8)`, file.CollectionID, file.ID,
		file.EncryptedKey, file.KeyDecryptionNonce, false, file.UpdationTime, file.OwnerID, collectionOwnerID)
	finishPhase(err)
	if err != nil {
		return file, -1, stacktrace.Propagate(err, "")
	}
	finishPhase = dbmetrics.Start(dbmetrics.FileCreate, "update_collection")
	_, err = tx.ExecContext(ctx, `UPDATE collections SET updation_time = $1
			WHERE collection_id = $2`, file.UpdationTime, file.CollectionID)
	finishPhase(err)
	if err != nil {
		return file, -1, stacktrace.Propagate(err, "")
	}
	finishPhase = dbmetrics.Start(dbmetrics.FileCreate, "insert_original_object")
	_, err = tx.ExecContext(ctx, `INSERT INTO object_keys(file_id, o_type, object_key, size, datacenters)
			VALUES($1, $2, $3, $4, $5)`, fileID, ente.FILE, file.File.ObjectKey, fileSize, dcsForNewEntry)
	finishPhase(err)
	if err != nil {
		if err.Error() == "pq: duplicate key value violates unique constraint \"object_keys_object_key_key\"" {
			return file, -1, ente.ErrDuplicateFileObjectFound
		}
		return file, -1, stacktrace.Propagate(err, "")
	}
	finishPhase = dbmetrics.Start(dbmetrics.FileCreate, "insert_thumbnail_object")
	_, err = tx.ExecContext(ctx, `INSERT INTO object_keys(file_id, o_type, object_key, size, datacenters)
			VALUES($1, $2, $3, $4, $5)`, fileID, ente.THUMBNAIL, file.Thumbnail.ObjectKey, thumbnailSize, dcsForNewEntry)
	finishPhase(err)
	if err != nil {
		if err.Error() == "pq: duplicate key value violates unique constraint \"object_keys_object_key_key\"" {
			return file, -1, ente.ErrDuplicateThumbnailObjectFound
		}
		return file, -1, stacktrace.Propagate(err, "")
	}

	finishPhase = dbmetrics.Start(dbmetrics.FileCreate, "consume_original_upload")
	err = repo.ObjectCleanupRepo.ConsumeFileUpload(ctx, tx, file.File.ObjectKey, file.OwnerID, app)
	finishPhase(err)
	if err != nil {
		return file, -1, stacktrace.Propagate(err, "")
	}
	finishPhase = dbmetrics.Start(dbmetrics.FileCreate, "consume_thumbnail_upload")
	err = repo.ObjectCleanupRepo.ConsumeFileUpload(ctx, tx, file.Thumbnail.ObjectKey, file.OwnerID, app)
	finishPhase(err)
	if err != nil {
		return file, -1, stacktrace.Propagate(err, "")
	}
	finishPhase = dbmetrics.Start(dbmetrics.FileCreate, "update_usage")
	usage, err := repo.updateUsageForFileCreation(ctx, tx, file.OwnerID, usageDiff, app)
	finishPhase(err)
	if err != nil {
		return file, -1, stacktrace.Propagate(err, "")
	}

	finishPhase = dbmetrics.Start(dbmetrics.FileCreate, "mark_replication")
	err = repo.markAsNeedingReplication(ctx, tx, file, hotDC)
	finishPhase(err)
	if err != nil {
		return file, -1, stacktrace.Propagate(err, "")
	}

	finishPhase = dbmetrics.Start(dbmetrics.FileCreate, "commit")
	err = tx.Commit()
	finishPhase(err)
	if err != nil {
		return file, -1, stacktrace.Propagate(err, "")
	}
	return file, usage, stacktrace.Propagate(err, "")
}

func (repo *FileRepository) CreateMetaFile(
	metaFile ente.MetaFile,
	collectionOwnerID int64,
	app ente.App,
) (*ente.File, error) {
	if metaFile.OwnerID != collectionOwnerID {
		return nil, stacktrace.Propagate(errors.New("both file and collection should belong to same owner"), "")
	}
	ctx := context.Background()
	tx, err := repo.DB.BeginTx(ctx, nil)
	if err != nil {
		return nil, stacktrace.Propagate(err, "")
	}
	defer tx.Rollback()

	var fileID int64
	info := &ente.FileInfo{
		FileSize:      0,
		ThumbnailSize: 0,
	}
	err = tx.QueryRowContext(ctx, `INSERT INTO files
			(owner_id, app, encrypted_metadata,
			file_decryption_header, thumbnail_decryption_header, metadata_decryption_header,
			magic_metadata, pub_magic_metadata, info, updation_time)
			VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING file_id`,
		metaFile.OwnerID, app, metaFile.Metadata.EncryptedData, "",
		"", metaFile.Metadata.DecryptionHeader,
		metaFile.MagicMetadata, metaFile.PublicMagicMetadata, info,
		metaFile.UpdationTime).Scan(&fileID)
	if err != nil {
		return nil, stacktrace.Propagate(err, "")
	}

	_, err = tx.ExecContext(ctx, `INSERT INTO collection_files
			(collection_id, file_id, encrypted_key, key_decryption_nonce, is_deleted, updation_time, c_owner_id, f_owner_id)
			VALUES($1, $2, $3, $4, $5, $6, $7, $8)`, metaFile.CollectionID, fileID,
		metaFile.EncryptedKey, metaFile.KeyDecryptionNonce, false, metaFile.UpdationTime, metaFile.OwnerID, collectionOwnerID)
	if err != nil {
		return nil, stacktrace.Propagate(err, "")
	}
	_, err = tx.ExecContext(ctx, `UPDATE collections SET updation_time = $1
			WHERE collection_id = $2`, metaFile.UpdationTime, metaFile.CollectionID)
	if err != nil {
		return nil, stacktrace.Propagate(err, "")
	}
	_, err = repo.updateUsageForFileCreation(ctx, tx, metaFile.OwnerID, 0, app)
	if err != nil {
		tx.Rollback()
		return nil, stacktrace.Propagate(err, "")
	}
	err = tx.Commit()
	if err != nil {
		return nil, stacktrace.Propagate(err, "")
	}
	var file ente.File = ente.File{
		ID:                  fileID,
		UpdationTime:        metaFile.UpdationTime,
		OwnerID:             metaFile.OwnerID,
		Metadata:            metaFile.Metadata,
		PublicMagicMetadata: metaFile.PublicMagicMetadata,
		MagicMetadata:       metaFile.MagicMetadata,
		EncryptedKey:        metaFile.EncryptedKey,
		KeyDecryptionNonce:  metaFile.KeyDecryptionNonce,
		Info:                info,
		CollectionID:        metaFile.CollectionID,
	}
	return &file, stacktrace.Propagate(err, "")
}

// The controller first verifies the object in hotDC. This prevents recording an
// upload against another bucket if hot storage changes after the client receives
// its presigned URL.
func (repo *FileRepository) markAsNeedingReplication(ctx context.Context, tx *sql.Tx, file ente.File, hotDC string) error {
	if hotDC == repo.S3Config.GetHotBackblazeDC() {
		err := repo.ObjectCopiesRepo.CreateNewB2Object(ctx, tx, file.File.ObjectKey, true, true)
		if err != nil {
			return stacktrace.Propagate(err, "")
		}
		err = repo.ObjectCopiesRepo.CreateNewB2Object(ctx, tx, file.Thumbnail.ObjectKey, true, false)
		return stacktrace.Propagate(err, "")
	} else if hotDC == repo.S3Config.GetHotWasabiDC() {
		err := repo.ObjectCopiesRepo.CreateNewWasabiObject(ctx, tx, file.File.ObjectKey, true, true)
		if err != nil {
			return stacktrace.Propagate(err, "")
		}
		err = repo.ObjectCopiesRepo.CreateNewWasabiObject(ctx, tx, file.Thumbnail.ObjectKey, true, false)
		return stacktrace.Propagate(err, "")
	} else {
		err := fmt.Errorf("only B2 and Wasabi DCs can be used for as the primary hot storage; instead, it was %s", hotDC)
		return stacktrace.Propagate(err, "")
	}
}

func (repo *FileRepository) markThumbnailAsNeedingReplication(ctx context.Context, tx *sql.Tx, thumbnailObjectKey string, hotDC string) error {
	if hotDC == repo.S3Config.GetHotBackblazeDC() {
		err := repo.ObjectCopiesRepo.CreateNewB2Object(ctx, tx, thumbnailObjectKey, true, false)
		return stacktrace.Propagate(err, "")
	} else if hotDC == repo.S3Config.GetHotWasabiDC() {
		err := repo.ObjectCopiesRepo.CreateNewWasabiObject(ctx, tx, thumbnailObjectKey, true, false)
		return stacktrace.Propagate(err, "")
	} else {
		err := fmt.Errorf("only B2 and Wasabi DCs can be used for as the primary hot storage; instead, it was %s", hotDC)
		return stacktrace.Propagate(err, "")
	}
}

func (repo *FileRepository) ResetNeedsReplication(file ente.File, hotDC string) error {
	if hotDC == repo.S3Config.GetHotBackblazeDC() {
		err := repo.ObjectCopiesRepo.ResetNeedsWasabiReplication(file.File.ObjectKey)
		if err != nil {
			return stacktrace.Propagate(err, "")
		}
		err = repo.ObjectCopiesRepo.ResetNeedsScalewayReplication(file.File.ObjectKey)
		if err != nil {
			return stacktrace.Propagate(err, "")
		}

		err = repo.ObjectCopiesRepo.ResetNeedsWasabiReplication(file.Thumbnail.ObjectKey)
		return stacktrace.Propagate(err, "")
	} else if hotDC == repo.S3Config.GetHotWasabiDC() {
		err := repo.ObjectCopiesRepo.ResetNeedsB2Replication(file.File.ObjectKey)
		if err != nil {
			return stacktrace.Propagate(err, "")
		}
		err = repo.ObjectCopiesRepo.ResetNeedsScalewayReplication(file.File.ObjectKey)
		if err != nil {
			return stacktrace.Propagate(err, "")
		}

		err = repo.ObjectCopiesRepo.ResetNeedsB2Replication(file.Thumbnail.ObjectKey)
		return stacktrace.Propagate(err, "")
	} else {
		err := fmt.Errorf("only B2 and Wasabi DCs can be used for as the primary hot storage; instead, it was %s", hotDC)
		return stacktrace.Propagate(err, "")
	}
}

func (repo *FileRepository) Update(file ente.File, fileSize int64, thumbnailSize int64, usageDiff int64, oldObjects []string, stagedObjects []string, app ente.App) error {
	hotDC := repo.S3Config.GetHotDataCenter()
	dcsForNewEntry := pq.StringArray{hotDC}
	// iOS may retry after backgrounding before receiving a successful response.
	// Only matching object keys and total size make the update a duplicate.
	isDuplicateRequest := len(stagedObjects) == 0 && usageDiff == 0

	ctx := context.Background()
	tx, err := repo.DB.BeginTx(ctx, nil)
	if err != nil {
		return stacktrace.Propagate(err, "")
	}
	defer tx.Rollback()
	_, err = tx.ExecContext(ctx, `UPDATE files SET encrypted_metadata = $1,
			file_decryption_header = $2, thumbnail_decryption_header = $3, 
			metadata_decryption_header = $4, updation_time = $5 , info = $6 WHERE file_id = $7`,
		file.Metadata.EncryptedData, file.File.DecryptionHeader,
		file.Thumbnail.DecryptionHeader, file.Metadata.DecryptionHeader,
		file.UpdationTime, file.Info, file.ID)
	if err != nil {
		return stacktrace.Propagate(err, "")
	}
	updatedRows, err := tx.QueryContext(ctx, `UPDATE collection_files 
			SET updation_time = $1 WHERE file_id = $2 RETURNING collection_id`, file.UpdationTime,
		file.ID)
	if err != nil {
		return stacktrace.Propagate(err, "")
	}
	defer updatedRows.Close()
	updatedCIDs := make([]int64, 0)
	for updatedRows.Next() {
		var cID int64
		err := updatedRows.Scan(&cID)
		if err != nil {
			return stacktrace.Propagate(err, "")
		}
		updatedCIDs = append(updatedCIDs, cID)
	}
	_, err = tx.ExecContext(ctx, `UPDATE collections SET updation_time = $1
			WHERE collection_id = ANY($2)`, file.UpdationTime, pq.Array(updatedCIDs))
	if err != nil {
		return stacktrace.Propagate(err, "")
	}
	_, err = tx.ExecContext(ctx, `DELETE FROM object_copies WHERE object_key = ANY($1)`,
		pq.Array(oldObjects))
	if err != nil {
		return stacktrace.Propagate(err, "")
	}
	_, err = tx.ExecContext(ctx, `UPDATE object_keys 
			SET object_key = $1, size = $2,
				datacenters = CASE WHEN object_key = $1 THEN datacenters ELSE $3 END
			WHERE file_id = $4 AND o_type = $5`,
		file.File.ObjectKey, fileSize, dcsForNewEntry, file.ID, ente.FILE)
	if err != nil {
		return stacktrace.Propagate(err, "")
	}
	_, err = tx.ExecContext(ctx, `UPDATE object_keys 
			SET object_key = $1, size = $2,
				datacenters = CASE WHEN object_key = $1 THEN datacenters ELSE $3 END
			WHERE file_id = $4 AND o_type = $5`,
		file.Thumbnail.ObjectKey, thumbnailSize, dcsForNewEntry, file.ID, ente.THUMBNAIL)
	if err != nil {
		return stacktrace.Propagate(err, "")
	}
	_, err = applyUsageChange(ctx, tx, file.OwnerID, usageChange{StorageDelta: usageDiff})
	if err != nil {
		return stacktrace.Propagate(err, "")
	}
	for _, objectKey := range stagedObjects {
		if err = repo.ObjectCleanupRepo.ConsumeFileUpload(ctx, tx, objectKey, file.OwnerID, app); err != nil {
			return stacktrace.Propagate(err, "")
		}
	}
	if isDuplicateRequest {
		// Re-inserting object_copies on a duplicate request violates its primary
		// key and leaves the transaction uncommittable.
		log.Infof("Skipping update of object_copies for a duplicate request to update file %d", file.ID)
	} else {
		err = repo.markAsNeedingReplication(ctx, tx, file, hotDC)
		if err != nil {
			return stacktrace.Propagate(err, "")
		}
	}
	err = repo.QueueRepo.AddItems(ctx, tx, OutdatedObjectsQueue, oldObjects)
	if err != nil {
		return stacktrace.Propagate(err, "")
	}
	err = tx.Commit()
	return stacktrace.Propagate(err, "")
}

func (repo *FileRepository) UpdateMagicAttributes(
	ctx context.Context,
	fileUpdates []ente.UpdateMagicMetadata,
	isPublicMetadata bool,
	skipVersion *bool,
) error {
	slices.SortStableFunc(fileUpdates, func(a, b ente.UpdateMagicMetadata) int {
		return cmp.Compare(a.ID, b.ID)
	})
	updationTime := time.Microseconds()
	tx, err := repo.DB.BeginTx(ctx, nil)
	if err != nil {
		return stacktrace.Propagate(err, "")
	}
	defer tx.Rollback()
	fileIDs := make([]int64, 0)
	for _, update := range fileUpdates {
		update.MagicMetadata.Version = update.MagicMetadata.Version + 1
		fileIDs = append(fileIDs, update.ID)
		if isPublicMetadata {
			_, err = tx.ExecContext(ctx, `UPDATE files SET pub_magic_metadata = $1,  updation_time = $2 WHERE file_id = $3`,
				update.MagicMetadata, updationTime, update.ID)
		} else {
			_, err = tx.ExecContext(ctx, `UPDATE files SET magic_metadata = $1,  updation_time = $2 WHERE file_id = $3`,
				update.MagicMetadata, updationTime, update.ID)
		}
		if err != nil {
			return stacktrace.Propagate(err, "")
		}
	}
	if skipVersion != nil && *skipVersion {
		return tx.Commit()
	}
	// todo: full table scan, need to add index (for discussion: add user_id and idx {user_id, file_id}).
	updatedRows, err := tx.QueryContext(ctx, `UPDATE collection_files 
			SET updation_time = $1 WHERE file_id = ANY($2) AND is_deleted= false RETURNING collection_id`, updationTime,
		pq.Array(fileIDs))
	if err != nil {
		return stacktrace.Propagate(err, "")
	}
	defer updatedRows.Close()
	updatedCIDs := make([]int64, 0)
	for updatedRows.Next() {
		var cID int64
		err := updatedRows.Scan(&cID)
		if err != nil {
			return stacktrace.Propagate(err, "")
		}
		updatedCIDs = append(updatedCIDs, cID)
	}
	_, err = tx.ExecContext(ctx, `UPDATE collections SET updation_time = $1
			WHERE collection_id = ANY($2)`, updationTime, pq.Array(updatedCIDs))
	if err != nil {
		return stacktrace.Propagate(err, "")
	}
	return tx.Commit()
}

func (repo *FileRepository) UpdateThumbnail(ctx context.Context, fileID int64, userID int64, thumbnail ente.FileAttributes, thumbnailSize int64, usageDiff int64, oldThumbnailObject *string, app ente.App) error {
	hotDC := repo.S3Config.GetHotDataCenter()
	dcsForNewEntry := pq.StringArray{hotDC}

	tx, err := repo.DB.BeginTx(ctx, nil)
	if err != nil {
		return stacktrace.Propagate(err, "")
	}
	defer tx.Rollback()
	updationTime := time.Microseconds()
	_, err = tx.ExecContext(ctx, `UPDATE files SET 
			thumbnail_decryption_header = $1, 
			updation_time = $2 WHERE file_id = $3`,
		thumbnail.DecryptionHeader,
		updationTime, fileID)
	if err != nil {
		return stacktrace.Propagate(err, "")
	}
	updatedRows, err := tx.QueryContext(ctx, `UPDATE collection_files 
			SET updation_time = $1 WHERE file_id = $2 RETURNING collection_id`, updationTime,
		fileID)
	if err != nil {
		return stacktrace.Propagate(err, "")
	}
	defer updatedRows.Close()
	updatedCIDs := make([]int64, 0)
	for updatedRows.Next() {
		var cID int64
		err := updatedRows.Scan(&cID)
		if err != nil {
			return stacktrace.Propagate(err, "")
		}
		updatedCIDs = append(updatedCIDs, cID)
	}
	_, err = tx.ExecContext(ctx, `UPDATE collections SET updation_time = $1
			WHERE collection_id = ANY($2)`, updationTime, pq.Array(updatedCIDs))
	if err != nil {
		return stacktrace.Propagate(err, "")
	}
	if oldThumbnailObject != nil {
		_, err = tx.ExecContext(ctx, `DELETE FROM object_copies WHERE object_key = $1`,
			*oldThumbnailObject)
		if err != nil {
			return stacktrace.Propagate(err, "")
		}
	}
	_, err = tx.ExecContext(ctx, `UPDATE object_keys 
			SET object_key = $1, size = $2, datacenters = $3 WHERE file_id = $4 AND o_type = $5`,
		thumbnail.ObjectKey, thumbnailSize, dcsForNewEntry, fileID, ente.THUMBNAIL)
	if err != nil {
		return stacktrace.Propagate(err, "")
	}
	_, err = applyUsageChange(ctx, tx, userID, usageChange{StorageDelta: usageDiff})
	if err != nil {
		return stacktrace.Propagate(err, "")
	}

	if oldThumbnailObject != nil {
		if err = repo.ObjectCleanupRepo.ConsumeFileUpload(ctx, tx, thumbnail.ObjectKey, userID, app); err != nil {
			return stacktrace.Propagate(err, "")
		}
	}
	err = repo.markThumbnailAsNeedingReplication(ctx, tx, thumbnail.ObjectKey, hotDC)
	if err != nil {
		return stacktrace.Propagate(err, "")
	}
	if oldThumbnailObject != nil {
		err = repo.QueueRepo.AddItems(ctx, tx, OutdatedObjectsQueue, []string{*oldThumbnailObject})
		if err != nil {
			return stacktrace.Propagate(err, "")
		}
	}
	err = tx.Commit()
	return stacktrace.Propagate(err, "")
}

func (repo *FileRepository) GetOwnerID(fileID int64) (int64, error) {
	row := repo.DB.QueryRow(`SELECT owner_id FROM files WHERE file_id = $1`,
		fileID)
	var ownerID int64
	err := row.Scan(&ownerID)
	return ownerID, stacktrace.Propagate(err, "failed to get file owner")
}

func (repo *FileRepository) GetOwnerAndApp(fileID int64) (int64, ente.App, error) {
	var ownerID int64
	var app sql.NullString
	err := repo.DB.QueryRow(`SELECT owner_id, app FROM files WHERE file_id = $1`, fileID).Scan(&ownerID, &app)
	return ownerID, ente.App(app.String), stacktrace.Propagate(err, "failed to get file owner and app")
}

func (repo *FileRepository) GetOwnerToFileCountMap(ctx context.Context, fileIDs []int64) (map[int64]int64, error) {
	rows, err := repo.DB.QueryContext(ctx, `SELECT owner_id, count(*) FROM files WHERE file_id = ANY($1) group by owner_id`,
		pq.Array(fileIDs))
	if err != nil {
		return nil, stacktrace.Propagate(err, "")
	}
	defer rows.Close()
	result := make(map[int64]int64, 0)
	for rows.Next() {
		var ownerID, count int64
		if err = rows.Scan(&ownerID, &count); err != nil {
			return nil, stacktrace.Propagate(err, "")
		}
		result[ownerID] = count
	}
	return result, nil
}

func (repo *FileRepository) GetOwnerToFileIDsMap(ctx context.Context, fileIDs []int64) (map[int64][]int64, error) {
	rows, err := repo.DB.QueryContext(ctx, `SELECT owner_id, file_id FROM files WHERE file_id = ANY($1)`,
		pq.Array(fileIDs))
	if err != nil {
		return nil, stacktrace.Propagate(err, "")
	}
	defer rows.Close()
	result := make(map[int64][]int64, 0)
	for rows.Next() {
		var ownerID, fileID int64
		if err = rows.Scan(&ownerID, &fileID); err != nil {
			return nil, stacktrace.Propagate(err, "")
		}
		if ownerFileIDs, ok := result[ownerID]; ok {
			result[ownerID] = append(ownerFileIDs, fileID)
		} else {
			result[ownerID] = []int64{fileID}
		}
	}
	return result, nil
}
func (repo *FileRepository) VerifyFileOwner(ctx context.Context, fileIDs []int64, ownerID int64, logger *log.Entry) error {
	countMap, err := repo.GetOwnerToFileCountMap(ctx, fileIDs)
	if err != nil {
		return stacktrace.Propagate(err, "failed to get owners info")
	}
	logger = logger.WithFields(log.Fields{
		"owner_id":   ownerID,
		"file_ids":   fileIDs,
		"owners_map": countMap,
	})
	if len(countMap) == 0 {
		logger.Error("all fileIDs are invalid")
		return stacktrace.Propagate(ente.ErrBadRequest, "")
	}
	if len(countMap) > 1 {
		logger.Error("files are owned by multiple users")
		return stacktrace.Propagate(ente.ErrPermissionDenied, "")
	}
	if filesOwned, ok := countMap[ownerID]; ok {
		if filesOwned != int64(len(fileIDs)) {
			logger.WithField("file_owned", filesOwned).Error("failed to find all fileIDs")
			return stacktrace.Propagate(ente.ErrBadRequest, "")
		}
		return nil
	} else {
		logger.Error("user is not an owner of any file")
		return stacktrace.Propagate(ente.ErrPermissionDenied, "")
	}
}

func (repo *FileRepository) GetOwnerAndMagicMetadata(fileID int64, publicMetadata bool) (int64, *ente.MagicMetadata, error) {
	var row *sql.Row
	if publicMetadata {
		row = repo.DB.QueryRow(`SELECT owner_id, pub_magic_metadata FROM files WHERE file_id = $1`,
			fileID)
	} else {
		row = repo.DB.QueryRow(`SELECT owner_id, magic_metadata FROM files WHERE file_id = $1`,
			fileID)
	}
	var ownerID int64
	var magicMetadata *ente.MagicMetadata
	err := row.Scan(&ownerID, &magicMetadata)
	return ownerID, magicMetadata, stacktrace.Propagate(err, "")
}

func (repo *FileRepository) GetSize(userID int64, fileIDs []int64) (int64, error) {
	row := repo.DB.QueryRow(`
			SELECT COALESCE(SUM(size), 0) FROM object_keys WHERE o_type = 'file' AND is_deleted = false AND file_id = ANY(SELECT file_id FROM files WHERE (file_id = ANY($1) AND owner_id = $2))`,
		pq.Array(fileIDs), userID)
	var size int64
	err := row.Scan(&size)
	if err != nil {
		return -1, stacktrace.Propagate(err, "")
	}
	return size, nil
}

func (repo *FileRepository) GetFileCountForUser(userID int64, app ente.App) (int64, error) {
	row := repo.DB.QueryRow(`SELECT count(distinct files.file_id) AS file_count
			FROM collection_files
			JOIN collections c on c.owner_id = $1 and c.collection_id = collection_files.collection_id 
			JOIN files ON 
			files.owner_id = $1 AND files.file_id = collection_files.file_id
			WHERE (c.app = $2 AND collection_files.is_deleted = false)`, userID, app)

	var fileCount int64
	err := row.Scan(&fileCount)
	if err != nil {
		return -1, stacktrace.Propagate(err, "")
	}
	return fileCount, nil
}

func (repo *FileRepository) GetFileAttributesFromObjectKey(objectKey string) (ente.File, error) {
	s3ObjectKeys, err := repo.ObjectRepo.GetAllFileObjectsByObjectKey(objectKey)
	if err != nil {
		return ente.File{}, stacktrace.Propagate(err, "")
	}
	if len(s3ObjectKeys) != 2 {
		return ente.File{}, stacktrace.Propagate(fmt.Errorf("unexpected file count: %d", len(s3ObjectKeys)), "")
	}

	var file ente.File
	file.ID = s3ObjectKeys[0].FileID // all file IDs should be same as per query in GetAllFileObjectsByObjectKey
	row := repo.DB.QueryRow(`SELECT owner_id, file_decryption_header, thumbnail_decryption_header, metadata_decryption_header, encrypted_metadata FROM files WHERE file_id = $1`, file.ID)
	err = row.Scan(&file.OwnerID,
		&file.File.DecryptionHeader, &file.Thumbnail.DecryptionHeader,
		&file.Metadata.DecryptionHeader,
		&file.Metadata.EncryptedData)
	if err != nil {
		return ente.File{}, err
	}
	for _, object := range s3ObjectKeys {
		if object.Type == ente.FILE {
			file.File.ObjectKey = object.ObjectKey
			file.File.Size = object.FileSize
		} else if object.Type == ente.THUMBNAIL {
			file.Thumbnail.ObjectKey = object.ObjectKey
			file.Thumbnail.Size = object.FileSize
		} else {
			err = fmt.Errorf("unexpted file type %s", object.Type)
			return ente.File{}, stacktrace.Propagate(err, "")
		}
	}
	return file, nil
}

func (repo *FileRepository) GetFileAttributesForCopy(fileIDs []int64) ([]ente.File, error) {
	result := make([]ente.File, 0)
	rows, err := repo.DB.Query(`SELECT file_id, owner_id, file_decryption_header, thumbnail_decryption_header, metadata_decryption_header, encrypted_metadata, pub_magic_metadata FROM files WHERE file_id = ANY($1)`, pq.Array(fileIDs))
	if err != nil {
		return nil, stacktrace.Propagate(err, "")
	}
	defer rows.Close()
	for rows.Next() {
		var file ente.File
		err := rows.Scan(&file.ID, &file.OwnerID, &file.File.DecryptionHeader, &file.Thumbnail.DecryptionHeader, &file.Metadata.DecryptionHeader, &file.Metadata.EncryptedData, &file.PublicMagicMetadata)
		if err != nil {
			return nil, stacktrace.Propagate(err, "")
		}
		result = append(result, file)
	}
	return result, nil
}

func (repo *FileRepository) GetFileAttributes(fileID int64) (*ente.File, error) {
	rows := repo.DB.QueryRow(`SELECT file_id, owner_id, file_decryption_header, thumbnail_decryption_header, metadata_decryption_header, encrypted_metadata, pub_magic_metadata, info FROM files WHERE file_id = $1`, fileID)
	var file ente.File
	err := rows.Scan(&file.ID, &file.OwnerID, &file.File.DecryptionHeader, &file.Thumbnail.DecryptionHeader, &file.Metadata.DecryptionHeader, &file.Metadata.EncryptedData, &file.PublicMagicMetadata, &file.Info)
	if err != nil {
		return nil, stacktrace.Propagate(err, "")
	}
	return &file, nil
}

func (repo *FileRepository) DropFilesMetadata(ctx context.Context, fileIDs []int64) error {
	rows, err := repo.DB.QueryContext(ctx, `SELECT distinct(file_id) FROM object_keys WHERE file_id = ANY($1)`, pq.Array(fileIDs))
	if err != nil {
		return stacktrace.Propagate(err, "")
	}
	defer rows.Close()
	fileIdsNotDeleted := make([]int64, 0)
	for rows.Next() {
		var fileID int64
		err := rows.Scan(&fileID)
		if err != nil {
			return stacktrace.Propagate(err, "")
		}
		fileIdsNotDeleted = append(fileIdsNotDeleted, fileID)
	}
	if len(fileIdsNotDeleted) > 0 {
		return stacktrace.Propagate(fmt.Errorf("fileIDs %v are still present in object_keys", fileIdsNotDeleted), "")
	}
	_, err = repo.DB.ExecContext(ctx, `
			UPDATE files SET encrypted_metadata = '-', 
			                 metadata_decryption_header = '-',
			                 file_decryption_header = '-', 
			                 thumbnail_decryption_header = '-',
			                 magic_metadata = NULL,
			                 pub_magic_metadata = NULL,
			                 info = NULL
             where file_id = ANY($1)`, pq.Array(fileIDs))
	return stacktrace.Propagate(err, "")
}

func (repo *FileRepository) GetDuplicateFiles(userID int64) ([]ente.DuplicateFiles, error) {
	rows, err := repo.DB.Query(`SELECT string_agg(o.file_id::character varying, ','), o.size FROM object_keys o JOIN files f ON f.file_id = o.file_id
											WHERE f.owner_id = $1 AND o.o_type = 'file' AND o.is_deleted = false
											GROUP BY size 
											HAVING count(*) > 1;`, userID)
	if err != nil {
		return nil, stacktrace.Propagate(err, "")
	}
	defer rows.Close()
	result := make([]ente.DuplicateFiles, 0)
	for rows.Next() {
		var res string
		var size int64
		err := rows.Scan(&res, &size)
		if err != nil {
			return result, stacktrace.Propagate(err, "")
		}
		fileIDs := make([]int64, 0)
		for fileIDStr := range strings.SplitSeq(res, ",") {
			fileID, err := strconv.ParseInt(fileIDStr, 10, 64)
			if err != nil {
				return result, stacktrace.Propagate(err, "")
			}
			fileIDs = append(fileIDs, fileID)
		}
		result = append(result, ente.DuplicateFiles{FileIDs: fileIDs, Size: size})
	}
	return result, nil
}

func (repo *FileRepository) GetTotalFileCount() (int64, error) {
	// 9,522,438 is the magic number that accommodates the bumping up of fileIDs
	// Doing this magic instead of count(*) since it's faster
	row := repo.DB.QueryRow(`select (select max(file_id) from files) - (select 9522438)`)
	var count int64
	err := row.Scan(&count)
	return count, stacktrace.Propagate(err, "")
}

func convertRowsToFiles(rows *sql.Rows) ([]ente.File, error) {
	defer rows.Close()
	files := make([]ente.File, 0)
	for rows.Next() {
		var (
			file         ente.File
			updationTime float64
			actionUser   sql.NullInt64
			action       sql.NullString
			addedAt      sql.NullInt64
		)
		err := rows.Scan(&file.ID, &file.OwnerID, &file.CollectionID, &file.CollectionOwnerID,
			&file.EncryptedKey, &file.KeyDecryptionNonce,
			&file.File.DecryptionHeader, &file.Thumbnail.DecryptionHeader,
			&file.Metadata.DecryptionHeader,
			&file.Metadata.EncryptedData, &file.MagicMetadata, &file.PublicMagicMetadata,
			&file.Info, &actionUser, &action, &file.IsDeleted, &updationTime, &addedAt)
		if err != nil {
			return files, stacktrace.Propagate(err, "")
		}
		if actionUser.Valid {
			v := actionUser.Int64
			file.ActionUserID = &v
		}
		if action.Valid {
			v := action.String
			file.Action = &v
		}
		file.UpdationTime = int64(updationTime)
		if addedAt.Valid {
			v := addedAt.Int64
			file.CollectionAddedAt = &v
		}
		files = append(files, file)
	}
	return files, nil
}

func (repo *FileRepository) scheduleDeletion(ctx context.Context, tx *sql.Tx, fileIDs []int64, userID int64) error {
	diff := int64(0)

	objectsToBeDeleted, err := repo.ObjectRepo.MarkObjectsAsDeletedForFileIDs(ctx, tx, fileIDs)
	if err != nil {
		return stacktrace.Propagate(err, "file object deletion failed for fileIDs: %v", fileIDs)
	}
	totalObjectSize := int64(0)
	for _, object := range objectsToBeDeleted {
		totalObjectSize += object.FileSize
	}
	diff = diff - (totalObjectSize)
	_, err = applyUsageChange(ctx, tx, userID, usageChange{StorageDelta: diff})
	return stacktrace.Propagate(err, "")
}

func (repo *FileRepository) updateUsageForFileCreation(ctx context.Context, tx *sql.Tx, userID, storageDiff int64, app ente.App) (int64, error) {
	photosFileCountDiff, lockerFileCountDiff, ok := fileCountDelta(app, 1)
	if !ok {
		return -1, stacktrace.Propagate(ente.ErrInvalidApp, "")
	}
	return applyUsageChange(ctx, tx, userID, usageChange{
		StorageDelta:    storageDiff,
		PhotosFileDelta: photosFileCountDiff,
		LockerFileDelta: lockerFileCountDiff,
	})
}