π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Bound Live Photo extraction in Go CLI
Neeraj Gupta1b09ec2c6622
diff --git a/cli/pkg/download.go b/cli/pkg/download.goindex bc8b6c1986..16fda35366 100644--- a/cli/pkg/download.go+++ b/cli/pkg/download.go@@ -1,18 +1,14 @@package pkgimport (- "archive/zip""context""fmt""github.com/ente/cli/internal/crypto""github.com/ente/cli/pkg/model""github.com/ente/cli/utils""github.com/ente/cli/utils/encoding"- "io""log""os"- "path/filepath"- "strings")func (c *ClICtrl) downloadAndDecrypt(@@ -41,62 +37,3 @@ func (c *ClICtrl) downloadAndDecrypt(}return &decryptedPath, nil}--func UnpackLive(src string) (imagePath, videoPath string, retErr error) {- var filenames []string- reader, err := zip.OpenReader(src)- if reader != nil {- defer reader.Close()- }- if err != nil {- retErr = err- return- }-- dest := filepath.Dir(src)-- for _, file := range reader.File {- if !filepath.IsLocal(file.Name) {- retErr = fmt.Errorf("invalid file path in live photo zip %s: %w", file.Name, zip.ErrInsecurePath)- return- }- destFilePath := filepath.Join(dest, file.Name)- filenames = append(filenames, destFilePath)-- destDir := filepath.Dir(destFilePath)- if err := os.MkdirAll(destDir, 0755); err != nil {- retErr = err- return- }-- destFile, err := os.Create(destFilePath)- if err != nil {- retErr = err- return- }- defer destFile.Close()-- srcFile, err := file.Open()- if err != nil {- retErr = err- return- }- defer srcFile.Close()-- _, err = io.Copy(destFile, srcFile)- if err != nil {- retErr = err- return- }- }- for _, filepath := range filenames {- if strings.Contains(strings.ToLower(filepath), "image") {- imagePath = filepath- } else if strings.Contains(strings.ToLower(filepath), "video") {- videoPath = filepath- } else {- retErr = fmt.Errorf("unexpcted file in zip %s", filepath)- }- }- return-}diff --git a/cli/pkg/live_photo.go b/cli/pkg/live_photo.gonew file mode 100644index 0000000000..a766fbe20f--- /dev/null+++ b/cli/pkg/live_photo.go@@ -0,0 +1,114 @@+package pkg++import (+ "archive/zip"+ "crypto/rand"+ "fmt"+ "io"+ "math"+ "os"+ "path/filepath"+ "regexp"++ "github.com/ente/cli/pkg/model"+)++const livePhotoExpansionRatio int64 = 20+const livePhotoExpansionOverhead int64 = 16 * 1024 * 1024++var livePhotoEntryName = regexp.MustCompile(`^(image|video)(\.[a-zA-Z0-9]{1,16})?$`)++func UnpackLive(src string) (imagePath, videoPath string, err error) {+ file, err := os.Open(src)+ if err != nil {+ return "", "", err+ }+ defer file.Close()+ info, err := file.Stat()+ if err != nil {+ return "", "", err+ }+ archive, err := zip.NewReader(file, info.Size())+ if err != nil {+ return "", "", err+ }+ limit := min(info.Size(), (math.MaxInt64-livePhotoExpansionOverhead)/livePhotoExpansionRatio)*livePhotoExpansionRatio + livePhotoExpansionOverhead+ return unpackLive(archive, filepath.Dir(src), limit)+}++func unpackLive(archive *zip.Reader, dest string, limit int64) (imagePath, videoPath string, err error) {+ if len(archive.File) != 2 {+ return "", "", fmt.Errorf("%w: expected one image and one video", model.ErrLiveZip)+ }+ var image, video *zip.File+ declaredRemaining := uint64(limit)+ for _, file := range archive.File {+ match := livePhotoEntryName.FindStringSubmatch(file.Name)+ if match == nil || file.FileInfo().IsDir() {+ return "", "", fmt.Errorf("%w: invalid component name", model.ErrLiveZip)+ }+ if match[1] == "image" {+ image = file+ } else {+ video = file+ }+ if file.UncompressedSize64 > declaredRemaining {+ return "", "", fmt.Errorf("%w: archive expands beyond limit", model.ErrLiveZip)+ }+ declaredRemaining -= file.UncompressedSize64+ }+ if image == nil || video == nil {+ return "", "", fmt.Errorf("%w: expected one image and one video", model.ErrLiveZip)+ }++ var outputs []string+ defer func() {+ if err != nil {+ for _, path := range outputs {+ _ = os.Remove(path)+ }+ imagePath, videoPath = "", ""+ }+ }()+ remaining := limit+ for _, entry := range []*zip.File{image, video} {+ input, openErr := entry.Open()+ if openErr != nil {+ return "", "", openErr+ }+ path := filepath.Join(dest, "live-photo-"+rand.Text()+filepath.Ext(entry.Name))+ output, createErr := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)+ if createErr != nil {+ input.Close()+ return "", "", createErr+ }+ outputs = append(outputs, output.Name())+ written, copyErr := copyLivePhotoComponent(output, input, remaining)+ input.Close()+ closeErr := output.Close()+ if copyErr != nil {+ return "", "", copyErr+ }+ if closeErr != nil {+ return "", "", closeErr+ }+ remaining -= written+ }+ return outputs[0], outputs[1], nil+}++func copyLivePhotoComponent(output io.Writer, input io.Reader, limit int64) (int64, error) {+ written, err := io.Copy(output, io.LimitReader(input, limit))+ if err != nil {+ return written, err+ }+ var extra [1]byte+ n, err := io.ReadFull(input, extra[:])+ if n != 0 {+ return written, fmt.Errorf("%w: archive expands beyond limit", model.ErrLiveZip)+ }+ if err == io.EOF {+ err = nil+ }+ return written, err+}diff --git a/cli/pkg/live_photo_test.go b/cli/pkg/live_photo_test.gonew file mode 100644index 0000000000..a7bfb5883f--- /dev/null+++ b/cli/pkg/live_photo_test.go@@ -0,0 +1,182 @@+package pkg++import (+ "archive/zip"+ "bytes"+ "errors"+ "io"+ "os"+ "path/filepath"+ "strings"+ "testing"++ "github.com/ente/cli/pkg/model"+)++func livePhotoArchive(t *testing.T, names []string, method uint16) []byte {+ t.Helper()+ var data bytes.Buffer+ writer := zip.NewWriter(&data)+ for _, name := range names {+ entry, err := writer.CreateHeader(&zip.FileHeader{Name: name, Method: method})+ if err != nil {+ t.Fatal(err)+ }+ if _, err := io.WriteString(entry, name); err != nil {+ t.Fatal(err)+ }+ }+ if err := writer.Close(); err != nil {+ t.Fatal(err)+ }+ return data.Bytes()+}++func livePhotoReader(t *testing.T, data []byte) *zip.Reader {+ t.Helper()+ reader, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))+ if err != nil {+ t.Fatal(err)+ }+ return reader+}++func TestUnpackLive(t *testing.T) {+ for _, method := range []uint16{zip.Store, zip.Deflate} {+ for _, names := range [][]string{{"image.heic", "video.mov"}, {"video", "image"}} {+ root := t.TempDir()+ src := filepath.Join(root, "photo.zip")+ if err := os.WriteFile(src, livePhotoArchive(t, names, method), 0666); err != nil {+ t.Fatal(err)+ }+ sourceInfo, err := os.Stat(src)+ if err != nil {+ t.Fatal(err)+ }+ image, video, err := UnpackLive(src)+ if err != nil {+ t.Fatal(err)+ }+ for _, path := range []string{image, video} {+ info, err := os.Stat(path)+ if err != nil {+ t.Fatal(err)+ }+ if info.Mode().Perm() != sourceInfo.Mode().Perm() {+ t.Fatalf("component mode %o differs from normal export mode %o", info.Mode().Perm(), sourceInfo.Mode().Perm())+ }+ data, err := os.ReadFile(path)+ if err != nil {+ t.Fatal(err)+ }+ if string(data) != names[0] && string(data) != names[1] {+ t.Fatalf("unexpected component bytes: %q", data)+ }+ if filepath.Ext(path) != filepath.Ext(string(data)) {+ t.Fatalf("component extension changed: %s", path)+ }+ }+ data, _ := os.ReadFile(image)+ if !strings.HasPrefix(string(data), "image") {+ t.Fatalf("image and video swapped: %q", data)+ }+ }+ }+}++func TestUnpackLiveValidatesBeforeCreatingOutput(t *testing.T) {+ for _, names := range [][]string{+ {"image.jpg", "video.mov", "extra.txt"},+ {"image.jpg", "image.png"},+ {"folder/image.jpg", "video.mov"},+ } {+ root := t.TempDir()+ _, _, err := unpackLive(livePhotoReader(t, livePhotoArchive(t, names, zip.Store)), root, 1024)+ if !errors.Is(err, model.ErrLiveZip) {+ t.Fatalf("names %v: expected Live Photo error, got %v", names, err)+ }+ entries, _ := os.ReadDir(root)+ if len(entries) != 0 {+ t.Fatalf("invalid archive created output: %v", entries)+ }+ }+}++func TestUnpackLiveCombinedBudget(t *testing.T) {+ names := []string{"image.jpg", "video.mov"}+ size := int64(len(names[0]) + len(names[1]))+ for _, limit := range []int64{size - 1, size} {+ root := t.TempDir()+ _, _, err := unpackLive(livePhotoReader(t, livePhotoArchive(t, names, zip.Store)), root, limit)+ if limit == size && err != nil {+ t.Fatalf("exact budget rejected: %v", err)+ }+ if limit < size {+ if !errors.Is(err, model.ErrLiveZip) {+ t.Fatalf("expected budget error, got %v", err)+ }+ entries, _ := os.ReadDir(root)+ if len(entries) != 0 {+ t.Fatal("budget rejection created output")+ }+ }+ }+}++func TestCopyLivePhotoComponentBudget(t *testing.T) {+ for _, value := range []string{"", "hello", "hello!"} {+ var output bytes.Buffer+ written, err := copyLivePhotoComponent(&output, strings.NewReader(value), 5)+ if len(value) > 5 {+ if !errors.Is(err, model.ErrLiveZip) {+ t.Fatalf("expected budget error, got %v", err)+ }+ } else if err != nil {+ t.Fatal(err)+ }+ if written != int64(min(len(value), 5)) || output.String() != value[:written] {+ t.Fatalf("unexpected bounded output: %q (%d bytes)", output.String(), written)+ }+ }+}++type failedLivePhotoRead struct{}++func (failedLivePhotoRead) Read([]byte) (int, error) { return 0, io.ErrUnexpectedEOF }++func TestCopyLivePhotoComponentChecksErrorsAtBoundary(t *testing.T) {+ input := io.MultiReader(strings.NewReader("hello"), failedLivePhotoRead{})+ written, err := copyLivePhotoComponent(io.Discard, input, 5)+ if written != 5 || !errors.Is(err, io.ErrUnexpectedEOF) {+ t.Fatalf("expected read failure at the exact boundary, got %d, %v", written, err)+ }+}++func TestUnpackLiveCleansUpFailedExtraction(t *testing.T) {+ root := t.TempDir()+ sentinel := filepath.Join(root, "image.jpg")+ if err := os.WriteFile(sentinel, []byte("existing"), 0600); err != nil {+ t.Fatal(err)+ }+ archive := livePhotoReader(t, livePhotoArchive(t, []string{"image.jpg", "video.mov"}, zip.Store))+ opened := 0+ archive.RegisterDecompressor(zip.Store, func(input io.Reader) io.ReadCloser {+ opened+++ if opened == 2 {+ return io.NopCloser(io.MultiReader(io.LimitReader(input, 1), failedLivePhotoRead{}))+ }+ return io.NopCloser(input)+ })+ image, video, err := unpackLive(archive, root, 1024)+ if !errors.Is(err, io.ErrUnexpectedEOF) || image != "" || video != "" {+ t.Fatalf("unexpected failed extraction result: %q %q %v", image, video, err)+ }+ entries, _ := os.ReadDir(root)+ if len(entries) != 1 || entries[0].Name() != "image.jpg" {+ t.Fatalf("partial components remain: %v", entries)+ }+ data, _ := os.ReadFile(sentinel)+ if string(data) != "existing" {+ t.Fatal("existing file changed")+ }+}diff --git a/cli/pkg/remote_to_disk_file.go b/cli/pkg/remote_to_disk_file.goindex b3a56a31b9..e1dba37c6e 100644--- a/cli/pkg/remote_to_disk_file.go+++ b/cli/pkg/remote_to_disk_file.go@@ -84,11 +84,9 @@ func (c *ClICtrl) syncFiles(ctx context.Context, account model.Account) error {if err != nil {if errors.Is(err, model.ErrDecryption) {continue- } else if existingEntry.IsLivePhoto() && (errors.Is(err, zip.ErrFormat) || errors.Is(err, zip.ErrInsecurePath)) {+ } else if existingEntry.IsLivePhoto() && (errors.Is(err, zip.ErrFormat) || errors.Is(err, zip.ErrInsecurePath) || errors.Is(err, model.ErrLiveZip)) {log.Printf("err processing live photo %s (%d), %s", existingEntry.GetTitle(), existingEntry.ID, err.Error())continue- } else if existingEntry.IsLivePhoto() && errors.Is(err, model.ErrLiveZip) {- continue} else if model.IsBadTimeStampError(err) {log.Printf("Skipping file due to error %s (%d)", existingEntry.GetTitle(), existingEntry.ID)log.Printf("CreationTime %v, ModidicationTime %v", existingEntry.GetCreationTime(), existingEntry.GetModificationTime())@@ -153,6 +151,8 @@ func (c *ClICtrl) downloadEntry(ctx context.Context,if err != nil {return err}+ defer os.Remove(imagePath)+ defer os.Remove(videoPath)if imagePath == "" && videoPath == "" {log.Printf("imagePath %s, videoPath %s", imagePath, videoPath)return model.ErrLiveZip