Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

8.8 KB
Raw Blame
package pkg

import (
	"archive/zip"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/ente/cli/internal/api"
	"github.com/ente/cli/pkg/mapper"
	"github.com/ente/cli/pkg/model"
	"github.com/ente/cli/pkg/model/export"
	"github.com/ente/cli/utils"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"
)

func (c *ClICtrl) syncFiles(ctx context.Context, account model.Account) error {
	log.Printf("Starting file download")
	exportRoot := account.ExportDir
	albumIDToMetaMap, err := readFolderMetadata(exportRoot)
	if err != nil {
		return err
	}
	albumsToSkip := make(map[int64]bool)
	filter := ctx.Value(model.FilterKey).(model.Filter)
	remoteAlbums, readAlbumErr := c.getRemoteAlbums(ctx)
	if readAlbumErr != nil {
		return readAlbumErr
	}
	for _, album := range remoteAlbums {
		if album.IsDeleted || filter.SkipAlbum(album, false) {
			albumsToSkip[album.ID] = true
		}
	}
	entries, err := c.getRemoteAlbumEntries(ctx)
	if err != nil {
		return err
	}
	log.Println("total entries", len(entries))
	model.SortAlbumFileEntry(entries)
	defer utils.TimeTrack(time.Now(), "process_files")
	var albumDiskInfo *albumDiskInfo
	for i, albumFileEntry := range entries {
		if albumFileEntry.SyncedLocally {
			continue
		}
		if _, ok := albumsToSkip[albumFileEntry.AlbumID]; ok {
			continue
		}
		albumInfo, ok := albumIDToMetaMap[albumFileEntry.AlbumID]
		if !ok {
			log.Printf("Album %d not found in local metadata", albumFileEntry.AlbumID)
			continue
		}
		if albumInfo.IsDeleted {
			putErr := c.DeleteAlbumEntry(ctx, albumFileEntry)
			if putErr != nil {
				return putErr
			}
			continue
		}

		if albumDiskInfo == nil || albumDiskInfo.AlbumMeta.ID != albumInfo.ID {
			albumDiskInfo, err = readFilesMetadata(exportRoot, albumInfo)
			if err != nil {
				return err
			}
		}
		fileBytes, err := c.GetValue(ctx, model.RemoteFiles, fmt.Appendf(nil, "%d", albumFileEntry.FileID))
		if err != nil {
			return err
		}
		if fileBytes != nil {
			var existingEntry *model.RemoteFile
			err = json.Unmarshal(fileBytes, &existingEntry)
			if err != nil {
				return err
			}
			log.Printf("[%d/%d] Sync %s for album %s", i, len(entries), existingEntry.GetTitle(), albumInfo.AlbumName)
			err = c.downloadEntry(ctx, albumDiskInfo, *existingEntry, albumFileEntry)
			if err != nil {
				if errors.Is(err, model.ErrDecryption) {
					continue
				} else if existingEntry.IsLivePhoto() && (errors.Is(err, zip.ErrFormat) || errors.Is(err, zip.ErrInsecurePath) || errors.Is(err, model.ErrLiveZip)) {
					log.Printf("err processing live photo %s (%d), %s", existingEntry.GetTitle(), existingEntry.ID, err.Error())
					continue
				} else if model.IsBadTimeStampError(err) {
					log.Printf("Skipping file due to error %s (%d)", existingEntry.GetTitle(), existingEntry.ID)
					log.Printf("CreationTime %v, ModidicationTime %v", existingEntry.GetCreationTime(), existingEntry.GetModificationTime())
					continue
				} else {
					return err
				}
			}
		} else {
			if albumFileEntry.IsDeleted {
				delErr := c.DeleteAlbumEntry(ctx, albumFileEntry)
				if delErr != nil {
					log.Fatalf("Error deleting album entry %d (deleted: %v)  %v", albumFileEntry.FileID, albumFileEntry.IsDeleted, delErr)
				}
			} else {
				log.Fatalf("Failed to find entry in db for file %d (deleted: %v)", albumFileEntry.FileID, albumFileEntry.IsDeleted)
			}
		}
	}

	return nil
}

func (c *ClICtrl) downloadEntry(ctx context.Context,
	diskInfo *albumDiskInfo,
	file model.RemoteFile,
	albumEntry *model.AlbumFileEntry,
) error {
	if !diskInfo.AlbumMeta.IsDeleted && albumEntry.IsDeleted {
		albumEntry.IsDeleted = true
		diskFileMeta := diskInfo.GetDiskFileMetadata(file)
		if diskFileMeta != nil {
			removeErr := removeDiskFile(diskFileMeta, diskInfo)
			if removeErr != nil {
				return removeErr
			}
		}
		delErr := c.DeleteAlbumEntry(ctx, albumEntry)
		if delErr != nil {
			return delErr
		}
		return nil
	}
	diskFileMeta := diskInfo.GetDiskFileMetadata(file)
	if diskFileMeta != nil {
		removeErr := removeDiskFile(diskFileMeta, diskInfo)
		if removeErr != nil {
			return removeErr
		}
	}
	if !diskInfo.IsFilePresent(file) {
		decrypt, err := c.downloadAndDecrypt(ctx, file, c.KeyHolder.DeviceKey)
		if err != nil {
			var downloadErr *api.ApiError
			if errors.As(err, &downloadErr) && (downloadErr.StatusCode == http.StatusBadRequest || downloadErr.StatusCode == http.StatusNotFound) {
				if _, membershipErr := c.Client.GetFile(ctx, albumEntry.AlbumID, file.ID); api.IsFileNotInAlbumError(membershipErr) {
					log.Printf("Skipping file %d for album %d: file is no longer in the album", file.ID, albumEntry.AlbumID)
					return c.DeleteAlbumEntry(ctx, albumEntry)
				}
			}
			return err
		}
		fileDiskMetadata := mapper.MapRemoteFileToDiskMetadata(file)
		extension := filepath.Ext(fileDiskMetadata.Title)
		baseFileName := strings.TrimSuffix(filepath.Clean(filepath.Base(fileDiskMetadata.Title)), extension)
		diskMetaFileName := diskInfo.GenerateUniqueMetaFileName(baseFileName, extension)
		if file.IsLivePhoto() {
			imagePath, videoPath, err := UnpackLive(*decrypt)
			if err != nil {
				return err
			}
			defer os.Remove(imagePath)
			defer os.Remove(videoPath)
			for _, componentPath := range []string{imagePath, videoPath} {
				fileName := diskInfo.GenerateUniqueFileName(baseFileName, filepath.Ext(componentPath))
				filePath := filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, fileName)
				if err := Move(componentPath, filePath); err != nil {
					return err
				}
				fileDiskMetadata.AddFileName(fileName)
			}
		} else {
			fileName := diskInfo.GenerateUniqueFileName(baseFileName, extension)
			filePath := filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, fileName)
			err = Move(*decrypt, filePath)
			if err != nil {
				return err
			}
			fileDiskMetadata.AddFileName(fileName)
		}

		fileDiskMetadata.MetaFileName = diskMetaFileName
		err = diskInfo.AddEntry(fileDiskMetadata)
		if err != nil {
			return err
		}

		err = writeJSONToFile(filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, ".meta", diskMetaFileName), fileDiskMetadata)
		if err != nil {
			return err
		}
		albumEntry.SyncedLocally = true
		putErr := c.UpsertAlbumEntry(ctx, albumEntry)
		if putErr != nil {
			return putErr
		}
	}
	return nil
}

func removeDiskFile(diskFileMeta *export.DiskFileMetadata, diskInfo *albumDiskInfo) error {
	for _, fileName := range diskFileMeta.Info.FileNames {
		if !filepath.IsLocal(fileName) || fileName == "." || filepath.Base(fileName) != fileName {
			return fmt.Errorf("invalid exported filename %q", fileName)
		}
	}
	log.Printf("Removing file %s from disk", diskFileMeta.MetaFileName)
	err := os.Remove(filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, ".meta", diskFileMeta.MetaFileName))
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	for _, fileName := range diskFileMeta.Info.FileNames {
		err = os.Remove(filepath.Join(diskInfo.ExportRoot, diskInfo.AlbumMeta.FolderName, fileName))
		if err != nil && !os.IsNotExist(err) {
			return err
		}
	}
	return diskInfo.RemoveEntry(diskFileMeta)
}

func readFilesMetadata(home string, albumMeta *export.AlbumMetadata) (*albumDiskInfo, error) {
	albumMetadataFolder := filepath.Join(home, albumMeta.FolderName, albumMetaFolder)
	albumPath := filepath.Join(home, albumMeta.FolderName)
	if _, err := os.Stat(albumMetadataFolder); err != nil {
		return nil, err
	}
	if _, err := os.Stat(albumPath); err != nil {
		return nil, err
	}
	result := make(map[string]*export.DiskFileMetadata)
	fileIdToMetadata := make(map[int64]*export.DiskFileMetadata)
	claimedFileName := make(map[string]bool)
	albumFileEntries, err := os.ReadDir(albumPath)
	if err != nil {
		return nil, err
	}
	for _, entry := range albumFileEntries {
		if !entry.IsDir() {
			claimedFileName[strings.ToLower(entry.Name())] = true
		}
	}
	metaEntries, err := os.ReadDir(albumMetadataFolder)
	if err != nil {
		return nil, err
	}
	for _, entry := range metaEntries {
		if !entry.IsDir() {
			fileName := entry.Name()
			if fileName == albumMetaFile {
				continue
			}
			if !strings.HasSuffix(fileName, ".json") {
				log.Printf("Skipping file %s as it is not a JSON file", fileName)
				continue
			}
			fileMetadataPath := filepath.Join(albumMetadataFolder, fileName)
			// Initialize as nil, will remain nil if JSON file is not found or not readable
			result[strings.ToLower(fileName)] = nil
			var metaData export.DiskFileMetadata
			metaDataBytes, err := os.ReadFile(fileMetadataPath)
			if err != nil {
				continue // Skip this entry if reading fails
			}
			if err := json.Unmarshal(metaDataBytes, &metaData); err == nil {
				metaData.MetaFileName = fileName
				result[strings.ToLower(fileName)] = &metaData
				fileIdToMetadata[metaData.Info.ID] = &metaData
			}
		}
	}
	return &albumDiskInfo{
		ExportRoot:                home,
		AlbumMeta:                 albumMeta,
		FileNames:                 &claimedFileName,
		MetaFileNameToDiskFileMap: &result,
		FileIdToDiskFileMap:       &fileIdToMetadata,
	}, nil
}