Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Bound Live Photo extraction in Go CLI

Neeraj Gupta1b09ec2c6622
diff --git a/cli/pkg/download.go b/cli/pkg/download.go
index bc8b6c1986..16fda35366 100644
--- a/cli/pkg/download.go
+++ b/cli/pkg/download.go
@@ -1,18 +1,14 @@
package pkg
import (
- "archive/zip"
"context"
"fmt"
"github.com/ente/cli/internal/crypto"
"github.com/ente/cli/pkg/model"
"github.com/ente/cli/utils"
"github.com/ente/cli/utils/encoding"
- "io"
"log"
"os"
- "path/filepath"
- "strings"
)
func (c *ClICtrl) downloadAndDecrypt(
@@ -41,62 +37,3 @@ func (c *ClICtrl) downloadAndDecrypt(
}
return &decryptedPath, nil
}
-
-func UnpackLive(src string) (imagePath, videoPath string, retErr error) {
- var filenames []string
- reader, err := zip.OpenReader(src)
- if reader != nil {
- defer reader.Close()
- }
- if err != nil {
- retErr = err
- return
- }
-
- dest := filepath.Dir(src)
-
- for _, file := range reader.File {
- if !filepath.IsLocal(file.Name) {
- retErr = fmt.Errorf("invalid file path in live photo zip %s: %w", file.Name, zip.ErrInsecurePath)
- return
- }
- destFilePath := filepath.Join(dest, file.Name)
- filenames = append(filenames, destFilePath)
-
- destDir := filepath.Dir(destFilePath)
- if err := os.MkdirAll(destDir, 0755); err != nil {
- retErr = err
- return
- }
-
- destFile, err := os.Create(destFilePath)
- if err != nil {
- retErr = err
- return
- }
- defer destFile.Close()
-
- srcFile, err := file.Open()
- if err != nil {
- retErr = err
- return
- }
- defer srcFile.Close()
-
- _, err = io.Copy(destFile, srcFile)
- if err != nil {
- retErr = err
- return
- }
- }
- for _, filepath := range filenames {
- if strings.Contains(strings.ToLower(filepath), "image") {
- imagePath = filepath
- } else if strings.Contains(strings.ToLower(filepath), "video") {
- videoPath = filepath
- } else {
- retErr = fmt.Errorf("unexpcted file in zip %s", filepath)
- }
- }
- return
-}
diff --git a/cli/pkg/live_photo.go b/cli/pkg/live_photo.go
new file mode 100644
index 0000000000..a766fbe20f
--- /dev/null
+++ b/cli/pkg/live_photo.go
@@ -0,0 +1,114 @@
+package pkg
+
+import (
+ "archive/zip"
+ "crypto/rand"
+ "fmt"
+ "io"
+ "math"
+ "os"
+ "path/filepath"
+ "regexp"
+
+ "github.com/ente/cli/pkg/model"
+)
+
+const livePhotoExpansionRatio int64 = 20
+const livePhotoExpansionOverhead int64 = 16 * 1024 * 1024
+
+var livePhotoEntryName = regexp.MustCompile(`^(image|video)(\.[a-zA-Z0-9]{1,16})?$`)
+
+func UnpackLive(src string) (imagePath, videoPath string, err error) {
+ file, err := os.Open(src)
+ if err != nil {
+ return "", "", err
+ }
+ defer file.Close()
+ info, err := file.Stat()
+ if err != nil {
+ return "", "", err
+ }
+ archive, err := zip.NewReader(file, info.Size())
+ if err != nil {
+ return "", "", err
+ }
+ limit := min(info.Size(), (math.MaxInt64-livePhotoExpansionOverhead)/livePhotoExpansionRatio)*livePhotoExpansionRatio + livePhotoExpansionOverhead
+ return unpackLive(archive, filepath.Dir(src), limit)
+}
+
+func unpackLive(archive *zip.Reader, dest string, limit int64) (imagePath, videoPath string, err error) {
+ if len(archive.File) != 2 {
+ return "", "", fmt.Errorf("%w: expected one image and one video", model.ErrLiveZip)
+ }
+ var image, video *zip.File
+ declaredRemaining := uint64(limit)
+ for _, file := range archive.File {
+ match := livePhotoEntryName.FindStringSubmatch(file.Name)
+ if match == nil || file.FileInfo().IsDir() {
+ return "", "", fmt.Errorf("%w: invalid component name", model.ErrLiveZip)
+ }
+ if match[1] == "image" {
+ image = file
+ } else {
+ video = file
+ }
+ if file.UncompressedSize64 > declaredRemaining {
+ return "", "", fmt.Errorf("%w: archive expands beyond limit", model.ErrLiveZip)
+ }
+ declaredRemaining -= file.UncompressedSize64
+ }
+ if image == nil || video == nil {
+ return "", "", fmt.Errorf("%w: expected one image and one video", model.ErrLiveZip)
+ }
+
+ var outputs []string
+ defer func() {
+ if err != nil {
+ for _, path := range outputs {
+ _ = os.Remove(path)
+ }
+ imagePath, videoPath = "", ""
+ }
+ }()
+ remaining := limit
+ for _, entry := range []*zip.File{image, video} {
+ input, openErr := entry.Open()
+ if openErr != nil {
+ return "", "", openErr
+ }
+ path := filepath.Join(dest, "live-photo-"+rand.Text()+filepath.Ext(entry.Name))
+ output, createErr := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)
+ if createErr != nil {
+ input.Close()
+ return "", "", createErr
+ }
+ outputs = append(outputs, output.Name())
+ written, copyErr := copyLivePhotoComponent(output, input, remaining)
+ input.Close()
+ closeErr := output.Close()
+ if copyErr != nil {
+ return "", "", copyErr
+ }
+ if closeErr != nil {
+ return "", "", closeErr
+ }
+ remaining -= written
+ }
+ return outputs[0], outputs[1], nil
+}
+
+func copyLivePhotoComponent(output io.Writer, input io.Reader, limit int64) (int64, error) {
+ written, err := io.Copy(output, io.LimitReader(input, limit))
+ if err != nil {
+ return written, err
+ }
+ var extra [1]byte
+ n, err := io.ReadFull(input, extra[:])
+ if n != 0 {
+ return written, fmt.Errorf("%w: archive expands beyond limit", model.ErrLiveZip)
+ }
+ if err == io.EOF {
+ err = nil
+ }
+ return written, err
+}
diff --git a/cli/pkg/live_photo_test.go b/cli/pkg/live_photo_test.go
new file mode 100644
index 0000000000..a7bfb5883f
--- /dev/null
+++ b/cli/pkg/live_photo_test.go
@@ -0,0 +1,182 @@
+package pkg
+
+import (
+ "archive/zip"
+ "bytes"
+ "errors"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/ente/cli/pkg/model"
+)
+
+func livePhotoArchive(t *testing.T, names []string, method uint16) []byte {
+ t.Helper()
+ var data bytes.Buffer
+ writer := zip.NewWriter(&data)
+ for _, name := range names {
+ entry, err := writer.CreateHeader(&zip.FileHeader{Name: name, Method: method})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := io.WriteString(entry, name); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := writer.Close(); err != nil {
+ t.Fatal(err)
+ }
+ return data.Bytes()
+}
+
+func livePhotoReader(t *testing.T, data []byte) *zip.Reader {
+ t.Helper()
+ reader, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
+ if err != nil {
+ t.Fatal(err)
+ }
+ return reader
+}
+
+func TestUnpackLive(t *testing.T) {
+ for _, method := range []uint16{zip.Store, zip.Deflate} {
+ for _, names := range [][]string{{"image.heic", "video.mov"}, {"video", "image"}} {
+ root := t.TempDir()
+ src := filepath.Join(root, "photo.zip")
+ if err := os.WriteFile(src, livePhotoArchive(t, names, method), 0666); err != nil {
+ t.Fatal(err)
+ }
+ sourceInfo, err := os.Stat(src)
+ if err != nil {
+ t.Fatal(err)
+ }
+ image, video, err := UnpackLive(src)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, path := range []string{image, video} {
+ info, err := os.Stat(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if info.Mode().Perm() != sourceInfo.Mode().Perm() {
+ t.Fatalf("component mode %o differs from normal export mode %o", info.Mode().Perm(), sourceInfo.Mode().Perm())
+ }
+ data, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if string(data) != names[0] && string(data) != names[1] {
+ t.Fatalf("unexpected component bytes: %q", data)
+ }
+ if filepath.Ext(path) != filepath.Ext(string(data)) {
+ t.Fatalf("component extension changed: %s", path)
+ }
+ }
+ data, _ := os.ReadFile(image)
+ if !strings.HasPrefix(string(data), "image") {
+ t.Fatalf("image and video swapped: %q", data)
+ }
+ }
+ }
+}
+
+func TestUnpackLiveValidatesBeforeCreatingOutput(t *testing.T) {
+ for _, names := range [][]string{
+ {"image.jpg", "video.mov", "extra.txt"},
+ {"image.jpg", "image.png"},
+ {"folder/image.jpg", "video.mov"},
+ } {
+ root := t.TempDir()
+ _, _, err := unpackLive(livePhotoReader(t, livePhotoArchive(t, names, zip.Store)), root, 1024)
+ if !errors.Is(err, model.ErrLiveZip) {
+ t.Fatalf("names %v: expected Live Photo error, got %v", names, err)
+ }
+ entries, _ := os.ReadDir(root)
+ if len(entries) != 0 {
+ t.Fatalf("invalid archive created output: %v", entries)
+ }
+ }
+}
+
+func TestUnpackLiveCombinedBudget(t *testing.T) {
+ names := []string{"image.jpg", "video.mov"}
+ size := int64(len(names[0]) + len(names[1]))
+ for _, limit := range []int64{size - 1, size} {
+ root := t.TempDir()
+ _, _, err := unpackLive(livePhotoReader(t, livePhotoArchive(t, names, zip.Store)), root, limit)
+ if limit == size && err != nil {
+ t.Fatalf("exact budget rejected: %v", err)
+ }
+ if limit < size {
+ if !errors.Is(err, model.ErrLiveZip) {
+ t.Fatalf("expected budget error, got %v", err)
+ }
+ entries, _ := os.ReadDir(root)
+ if len(entries) != 0 {
+ t.Fatal("budget rejection created output")
+ }
+ }
+ }
+}
+
+func TestCopyLivePhotoComponentBudget(t *testing.T) {
+ for _, value := range []string{"", "hello", "hello!"} {
+ var output bytes.Buffer
+ written, err := copyLivePhotoComponent(&output, strings.NewReader(value), 5)
+ if len(value) > 5 {
+ if !errors.Is(err, model.ErrLiveZip) {
+ t.Fatalf("expected budget error, got %v", err)
+ }
+ } else if err != nil {
+ t.Fatal(err)
+ }
+ if written != int64(min(len(value), 5)) || output.String() != value[:written] {
+ t.Fatalf("unexpected bounded output: %q (%d bytes)", output.String(), written)
+ }
+ }
+}
+
+type failedLivePhotoRead struct{}
+
+func (failedLivePhotoRead) Read([]byte) (int, error) { return 0, io.ErrUnexpectedEOF }
+
+func TestCopyLivePhotoComponentChecksErrorsAtBoundary(t *testing.T) {
+ input := io.MultiReader(strings.NewReader("hello"), failedLivePhotoRead{})
+ written, err := copyLivePhotoComponent(io.Discard, input, 5)
+ if written != 5 || !errors.Is(err, io.ErrUnexpectedEOF) {
+ t.Fatalf("expected read failure at the exact boundary, got %d, %v", written, err)
+ }
+}
+
+func TestUnpackLiveCleansUpFailedExtraction(t *testing.T) {
+ root := t.TempDir()
+ sentinel := filepath.Join(root, "image.jpg")
+ if err := os.WriteFile(sentinel, []byte("existing"), 0600); err != nil {
+ t.Fatal(err)
+ }
+ archive := livePhotoReader(t, livePhotoArchive(t, []string{"image.jpg", "video.mov"}, zip.Store))
+ opened := 0
+ archive.RegisterDecompressor(zip.Store, func(input io.Reader) io.ReadCloser {
+ opened++
+ if opened == 2 {
+ return io.NopCloser(io.MultiReader(io.LimitReader(input, 1), failedLivePhotoRead{}))
+ }
+ return io.NopCloser(input)
+ })
+ image, video, err := unpackLive(archive, root, 1024)
+ if !errors.Is(err, io.ErrUnexpectedEOF) || image != "" || video != "" {
+ t.Fatalf("unexpected failed extraction result: %q %q %v", image, video, err)
+ }
+ entries, _ := os.ReadDir(root)
+ if len(entries) != 1 || entries[0].Name() != "image.jpg" {
+ t.Fatalf("partial components remain: %v", entries)
+ }
+ data, _ := os.ReadFile(sentinel)
+ if string(data) != "existing" {
+ t.Fatal("existing file changed")
+ }
+}
diff --git a/cli/pkg/remote_to_disk_file.go b/cli/pkg/remote_to_disk_file.go
index b3a56a31b9..e1dba37c6e 100644
--- a/cli/pkg/remote_to_disk_file.go
+++ b/cli/pkg/remote_to_disk_file.go
@@ -84,11 +84,9 @@ func (c *ClICtrl) syncFiles(ctx context.Context, account model.Account) error {
if err != nil {
if errors.Is(err, model.ErrDecryption) {
continue
- } else if existingEntry.IsLivePhoto() && (errors.Is(err, zip.ErrFormat) || errors.Is(err, zip.ErrInsecurePath)) {
+ } else if existingEntry.IsLivePhoto() && (errors.Is(err, zip.ErrFormat) || errors.Is(err, zip.ErrInsecurePath) || errors.Is(err, model.ErrLiveZip)) {
log.Printf("err processing live photo %s (%d), %s", existingEntry.GetTitle(), existingEntry.ID, err.Error())
continue
- } else if existingEntry.IsLivePhoto() && errors.Is(err, model.ErrLiveZip) {
- continue
} else if model.IsBadTimeStampError(err) {
log.Printf("Skipping file due to error %s (%d)", existingEntry.GetTitle(), existingEntry.ID)
log.Printf("CreationTime %v, ModidicationTime %v", existingEntry.GetCreationTime(), existingEntry.GetModificationTime())
@@ -153,6 +151,8 @@ func (c *ClICtrl) downloadEntry(ctx context.Context,
if err != nil {
return err
}
+ defer os.Remove(imagePath)
+ defer os.Remove(videoPath)
if imagePath == "" && videoPath == "" {
log.Printf("imagePath %s, videoPath %s", imagePath, videoPath)
return model.ErrLiveZip