Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Bound Live Photo archive decoding

Neeraj Guptaf6ad6e23fdac
diff --git a/web/apps/photos/tests/live-photo.test.ts b/web/apps/photos/tests/live-photo.test.ts
new file mode 100644
index 0000000000..0c00931ab7
--- /dev/null
+++ b/web/apps/photos/tests/live-photo.test.ts
@@ -0,0 +1,156 @@
+import { decodeLivePhoto } from "ente-media/live-photo";
+import JSZip, { type JSZipObject } from "jszip";
+import { afterEach, describe, expect, test, vi } from "vitest";
+
+afterEach(() => vi.restoreAllMocks());
+
+const archive = async (names: string[], compression: "STORE" | "DEFLATE") => {
+ const zip = new JSZip();
+ for (const name of names) zip.file(name, name, { createFolders: false });
+ return new Blob([
+ await zip.generateAsync({ type: "arraybuffer", compression }),
+ ]);
+};
+
+describe("Live Photo decoding", () => {
+ test.each(["STORE", "DEFLATE"] as const)(
+ "decodes %s components in either order, including extensionless names",
+ async (compression) => {
+ for (const names of [
+ ["image.heic", "video.mov"],
+ ["video", "image"],
+ ]) {
+ const result = await decodeLivePhoto(
+ "holiday.photo.zip",
+ await archive(names, compression),
+ );
+ const imageName = names.find((name) =>
+ name.startsWith("image"),
+ )!;
+ const videoName = names.find((name) =>
+ name.startsWith("video"),
+ )!;
+ expect(result.imageFileName).toBe(
+ imageName.replace("image", "holiday.photo"),
+ );
+ expect(result.videoFileName).toBe(
+ videoName.replace("video", "holiday.photo"),
+ );
+ expect(new TextDecoder().decode(result.imageData)).toBe(
+ imageName,
+ );
+ expect(new TextDecoder().decode(result.videoData)).toBe(
+ videoName,
+ );
+ expect(result.imageData.buffer).toBeInstanceOf(ArrayBuffer);
+ expect(result.videoData.buffer).toBeInstanceOf(ArrayBuffer);
+ }
+ },
+ );
+
+ test.each([
+ [["image.jpg", "video.mov", "extra.txt"]],
+ [["image.jpg", "image.png"]],
+ [["folder/image.jpg", "video.mov"]],
+ [["./image.jpg", "video.mov"]],
+ [["image.jpg\n", "video.mov"]],
+ ])("rejects invalid component sets: %j", async (names) => {
+ await expect(
+ decodeLivePhoto("photo.zip", await archive(names, "STORE")),
+ ).rejects.toThrow();
+ });
+});
+
+const streamEntry = (name: string, chunks: Uint8Array[], error?: Error) => {
+ const listeners: Record<string, (value?: unknown) => void> = {};
+ const stream = {
+ on(event: string, listener: (value?: unknown) => void) {
+ listeners[event] = listener;
+ return this;
+ },
+ pause: vi.fn(),
+ resume: vi.fn(() => {
+ queueMicrotask(() => {
+ for (const chunk of chunks) listeners.data?.(chunk);
+ if (error) listeners.error?.(error);
+ else listeners.end?.();
+ });
+ return stream;
+ }),
+ };
+ const entry = {
+ name,
+ unsafeOriginalName: name,
+ dir: false,
+ internalStream: vi.fn(() => stream),
+ };
+ return { entry, stream };
+};
+
+const stubArchive = (...entries: ReturnType<typeof streamEntry>[]) => {
+ const zip = new JSZip();
+ zip.files = Object.fromEntries(
+ entries.map(({ entry }) => [
+ entry.name,
+ entry as unknown as JSZipObject,
+ ]),
+ );
+ vi.spyOn(JSZip, "loadAsync").mockResolvedValue(zip);
+};
+
+describe("Live Photo output budgets", () => {
+ const chunk = new Uint8Array(64 * 1024);
+ const chunks = (count: number) =>
+ Array.from({ length: count }, () => chunk);
+
+ test("accepts the exact combined budget", async () => {
+ const blob = new Blob(["archive"]);
+ const inputAllowance = new Uint8Array(blob.size * 20);
+ const image = streamEntry("image.jpg", chunks(128));
+ const video = streamEntry("video.mov", [
+ ...chunks(128),
+ inputAllowance,
+ ]);
+ stubArchive(image, video);
+ const result = await decodeLivePhoto("photo.zip", blob);
+ expect(result.imageData.length + result.videoData.length).toBe(
+ 16 * 1024 * 1024 + inputAllowance.length,
+ );
+ expect(image.stream.pause).not.toHaveBeenCalled();
+ expect(video.stream.pause).not.toHaveBeenCalled();
+
+ const oversizedVideo = streamEntry("video.mov", [
+ ...chunks(128),
+ inputAllowance,
+ new Uint8Array(1),
+ ]);
+ stubArchive(image, oversizedVideo);
+ await expect(decodeLivePhoto("photo.zip", blob)).rejects.toThrow(
+ "expands beyond limit",
+ );
+ expect(oversizedVideo.stream.pause).toHaveBeenCalledOnce();
+ });
+
+ test("stops an oversized image without starting the video", async () => {
+ const image = streamEntry("image.jpg", chunks(257));
+ const video = streamEntry("video.mov", []);
+ stubArchive(image, video);
+ await expect(decodeLivePhoto("photo.zip", new Blob())).rejects.toThrow(
+ "expands beyond limit",
+ );
+ expect(image.stream.pause).toHaveBeenCalledOnce();
+ expect(video.entry.internalStream).not.toHaveBeenCalled();
+ });
+
+ test("propagates read failures without starting the next component", async () => {
+ const failure = new Error("component read failed");
+ const image = streamEntry("image.jpg", [chunk], failure);
+ const video = streamEntry("video.mov", []);
+ stubArchive(image, video);
+ await expect(decodeLivePhoto("photo.zip", new Blob())).rejects.toBe(
+ failure,
+ );
+ expect(image.stream.pause).toHaveBeenCalledOnce();
+ expect(video.entry.internalStream).not.toHaveBeenCalled();
+ });
+});
diff --git a/web/packages/media/live-photo.ts b/web/packages/media/live-photo.ts
index 20dabd5ca2..8df0c436e7 100644
--- a/web/packages/media/live-photo.ts
+++ b/web/packages/media/live-photo.ts
@@ -4,9 +4,17 @@ import {
nameAndExtension,
} from "ente-base/file-name";
import { ensureArrayBufferBacked } from "ente-utils/bytes";
-import JSZip from "jszip";
+import JSZip, { type JSZipObject, type JSZipStreamHelper } from "jszip";
import { FileType } from "./file-type";
+const maxExpandedArchiveRatio = 20;
+const maxExpandedArchiveOverhead = 16 * 1024 * 1024;
+const livePhotoEntryPattern = /^(image|video)(?:\.[a-zA-Z0-9]{1,16})?$/;
+
+type StreamableZipObject = JSZipObject & {
+ internalStream(type: "uint8array"): JSZipStreamHelper<Uint8Array>;
+};
+
const potentialImageExtensions = [
"heic",
"heif",
@@ -57,36 +65,90 @@ export const decodeLivePhoto = async (
fileName: string,
zipBlob: Blob,
): Promise<LivePhoto> => {
- let imageFileName, videoFileName: string | undefined;
- let imageData, videoData: Uint8Array<ArrayBuffer> | undefined;
+ const zip = await JSZip.loadAsync(await zipBlob.arrayBuffer());
+ const entries = Object.values(zip.files);
+ let imageEntry, videoEntry: JSZipObject | undefined;
+ if (entries.length != 2)
+ throw new Error("Live Photo must contain one image and one video");
+ for (const entry of entries) {
+ const match = livePhotoEntryPattern.exec(entry.name);
+ if (
+ entry.dir ||
+ match?.[0] !== entry.name ||
+ entry.unsafeOriginalName !== entry.name
+ )
+ throw new Error("Invalid Live Photo component name");
+ if (match[1] === "image") imageEntry = entry;
+ else videoEntry = entry;
+ }
+ if (!imageEntry || !videoEntry)
+ throw new Error("Live Photo must contain one image and one video");
+ const limit =
+ zipBlob.size * maxExpandedArchiveRatio + maxExpandedArchiveOverhead;
+ const imageData = await readLivePhotoEntry(imageEntry, limit);
+ const videoData = await readLivePhotoEntry(
+ videoEntry,
+ limit - imageData.length,
+ );
const [name] = nameAndExtension(fileName);
- const zip = await JSZip.loadAsync(zipBlob, { createFolders: true });
-
- for (const zipFileName in zip.files) {
- if (zipFileName.startsWith("image")) {
- const [, imageExt] = nameAndExtension(zipFileName);
- imageFileName = fileNameFromComponents([name, imageExt]);
- const bytes = await zip.files[zipFileName]?.async("uint8array");
- imageData = bytes && ensureArrayBufferBacked(bytes);
- } else if (zipFileName.startsWith("video")) {
- const [, videoExt] = nameAndExtension(zipFileName);
- videoFileName = fileNameFromComponents([name, videoExt]);
- const bytes = await zip.files[zipFileName]?.async("uint8array");
- videoData = bytes && ensureArrayBufferBacked(bytes);
- }
- }
+ const [, imageExt] = nameAndExtension(imageEntry.name);
+ const [, videoExt] = nameAndExtension(videoEntry.name);
+ return {
+ imageFileName: fileNameFromComponents([name, imageExt]),
+ imageData,
+ videoFileName: fileNameFromComponents([name, videoExt]),
+ videoData,
+ };
+};
- if (!imageFileName || !imageData)
- throw new Error(
- `Decoded live photo ${fileName} does not have an image`,
+const readLivePhotoEntry = (
+ entry: JSZipObject,
+ limit: number,
+): Promise<Uint8Array<ArrayBuffer>> =>
+ new Promise((resolve, reject) => {
+ const stream = (entry as StreamableZipObject).internalStream(
+ "uint8array",
);
-
- if (!videoFileName || !videoData)
- throw new Error(`Decoded live photo ${fileName} does not have a video`);
-
- return { imageFileName, imageData, videoFileName, videoData };
-};
+ const chunks: Uint8Array[] = [];
+ let length = 0;
+ let settled = false;
+ const fail = (error: Error) => {
+ if (settled) return;
+ settled = true;
+ stream.pause();
+ chunks.length = 0;
+ reject(error);
+ };
+ stream
+ .on("data", (chunk) => {
+ if (settled) return;
+ if (chunk.length > limit - length) {
+ fail(new Error("Live Photo archive expands beyond limit"));
+ return;
+ }
+ length += chunk.length;
+ chunks.push(chunk);
+ })
+ .on("error", fail)
+ .on("end", () => {
+ if (settled) return;
+ try {
+ const data = new Uint8Array(length);
+ let offset = 0;
+ for (const chunk of chunks) {
+ data.set(chunk, offset);
+ offset += chunk.length;
+ }
+ chunks.length = 0;
+ settled = true;
+ resolve(data);
+ } catch (error) {
+ fail(error as Error);
+ }
+ })
+ .resume();
+ });
interface EncodeLivePhotoInput {
imageFileName: string;