π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Keep Space encoding local and preserve confirmed posts
Anand Baburajan62a5b0be210b
diff --git a/web/apps/space/package.json b/web/apps/space/package.jsonindex b358883fcf..0451bd4ae5 100644--- a/web/apps/space/package.json+++ b/web/apps/space/package.json@@ -7,6 +7,7 @@"test": "vitest --run"},"dependencies": {+ "@ffmpeg/ffmpeg": "0.12.15","@fontsource-variable/inter": "5.2.8","@fontsource/nunito": "5.2.7","@hugeicons/core-free-icons": "1.2.1",diff --git a/web/apps/space/src/screens/HomeScreen.tsx b/web/apps/space/src/screens/HomeScreen.tsxindex eafbeaa605..ea07925341 100644--- a/web/apps/space/src/screens/HomeScreen.tsx+++ b/web/apps/space/src/screens/HomeScreen.tsx@@ -1813,11 +1813,11 @@ export const HomeScreen: React.FC<HomeScreenProps> = ({Boolean(viewerSpaceId) && selectedPhotoFriendID == viewerSpaceId;const desiredFeedEntries = React.useMemo<HomeFeedEntry[]>(() => {const localResolvedPostIds = new Set(- localFeedPosts- .filter(- (item) => item.status == "posted" || item.status == "ready",- )- .map((item) => item.post.postId),+ localFeedPosts.map((item) =>+ item.status == "posted" || item.status == "ready"+ ? item.post.postId+ : item.postId,+ ),);return [...localFeedPosts.map(@@ -1825,7 +1825,9 @@ export const HomeScreen: React.FC<HomeScreenProps> = ({identity:item.status == "posted" || item.status == "ready"? `post:${item.post.postId}`- : `local:${item.id}`,+ : item.postId+ ? `post:${item.postId}`+ : `local:${item.id}`,item,kind: "local",renderKey: `local:${item.id}`,@@ -2071,6 +2073,11 @@ export const HomeScreen: React.FC<HomeScreenProps> = ({);}+ const fetchedPost = item.postId+ ? feedItems.find((post) => post.postId == item.postId)+ : undefined;+ if (fetchedPost) return feedItemFor(fetchedPost, item.id, "posted");+return (<FeedItemkey={item.id}@@ -2092,7 +2099,9 @@ export const HomeScreen: React.FC<HomeScreenProps> = ({? item.reason == "post-limit"? "post-limit": "failed"- : "posting"+ : item.postId+ ? "posted"+ : "posting"}timestampMs={item.timestampMs}viewerLiked={false}diff --git a/web/apps/space/src/services/space.ts b/web/apps/space/src/services/space.tsindex ad58bae494..3c9657af2f 100644--- a/web/apps/space/src/services/space.ts+++ b/web/apps/space/src/services/space.ts@@ -414,7 +414,7 @@ const accountPostAssetURL = (const cacheAccountPostAssetURL = async (photo: PostPhoto, blob: Blob) => {const key = postAssetCacheKey(postAssetFrom(photo));- await rememberCachedSpaceMediaBlobURL(key, blob);+ return rememberCachedSpaceMediaBlobURL(key, blob);};const postVideoFrom = (photo: PostPhoto): SpacePostVideo | undefined =>@@ -1216,14 +1216,30 @@ export const createCurrentMediaPost = async ({),);}- const created = await ctx.getPost(- spaceId,- BigInt(session.postId),- spaceId,+ return session.postId;+ } finally {+ releaseCurrentSpaceContext(ctx);+ }+};++export const loadCurrentCreatedPost = async (+ spaceId: string,+ postId: number,+ previews: File[],+) => {+ const ctx = await ensureCurrentSpaceContext();+ try {+ const created = await ctx.getPost(spaceId, BigInt(postId), spaceId);+ const imageURLs = await Promise.all(+ created.photos.map((photo, index) =>+ cacheAccountPostAssetURL(photo, previews[index]!),+ ),);- for (const [index, photo] of created.photos.entries())- await cacheAccountPostAssetURL(photo, images[index]!.file);- const post = await postFromAccountPost(ctx, created, true, spaceId);+ const post = await postFromAccountPost(ctx, created, false, spaceId);+ post.imageUrl = imageURLs[0];+ post.photos?.forEach((photo, index) => {+ photo.imageUrl = imageURLs[index];+ });await prependCachedSpaceFeedPost(spaceId, post);return post;} finally {diff --git a/web/apps/space/src/state/AppStateProvider.tsx b/web/apps/space/src/state/AppStateProvider.tsxindex 0776dd5cdb..f52612da7e 100644--- a/web/apps/space/src/state/AppStateProvider.tsx+++ b/web/apps/space/src/state/AppStateProvider.tsx@@ -1,4 +1,5 @@import { isNamedError } from "ente-base/error";+import { retryAsyncOperation } from "ente-base/http";import log from "ente-base/log";import React, {useCallback,@@ -24,7 +25,9 @@ import {clearSpaceFriendsCache,clearSpaceMediaURLCache,createCurrentMediaPost,+ loadCurrentCreatedPost,type PreparedSpacePostMedia,+ type SpacePostUploadSession,} from "services/space";import {initialFriends,@@ -85,6 +88,21 @@ export const SpaceAppStateProvider: React.FC<React.PropsWithChildren> = ({const profileRef = useRef<SetupProfile | null>(null);const profileLoadGenerationRef = useRef(0);const postPublishGenerationRef = useRef(0);+ const postReadbacksRef = useRef(new Map<string, () => Promise<void>>());++ useEffect(() => {+ const readbacks = postReadbacksRef.current;+ const retryReadbacks = () => {+ readbacks.forEach((readback) => void readback());+ };+ window.addEventListener("online", retryReadbacks);+ window.addEventListener("focus", retryReadbacks);+ return () => {+ window.removeEventListener("online", retryReadbacks);+ window.removeEventListener("focus", retryReadbacks);+ readbacks.clear();+ };+ }, []);const previewURLsRef = useRef(new Set<string>());useEffect(() => {@@ -180,22 +198,81 @@ export const SpaceAppStateProvider: React.FC<React.PropsWithChildren> = ({),);}- const post = await createCurrentMediaPost({+ const session: SpacePostUploadSession = { uploads: [] };+ const postId = await createCurrentMediaPost({caption,images: prepared,spaceId,- session: { uploads: [] },+ session,});- confirmLocalFeedPost(setLocalFeedPosts, localPostId, post);+ const posted = { ...publication.post, postId };+ if (profileRef.current?.spaceId != spaceId) return posted;+ setLocalFeedPosts((current) =>+ current.map((item) =>+ item.id == localPostId && item.status == "pending"+ ? { ...item, postId }+ : item,+ ),+ );if (postPublishGenerationRef.current == generation) {setPostPublication({...publication,phase: "posted",- post,+ post: posted,statusExpiresAtMs: Date.now() + postStatusDurationMs,});}- return post;+ const previews = prepared.map((image) => image.file);+ let reading = false;+ const readback = async () => {+ if (reading) return;+ reading = true;+ try {+ const post = await retryAsyncOperation(+ () => {+ if (!postReadbacksRef.current.has(localPostId))+ throw new DOMException(+ "Canceled",+ "AbortError",+ );+ return loadCurrentCreatedPost(+ spaceId,+ session.postId!,+ previews,+ );+ },+ {+ abortIfNeeded: (error) => {+ if (+ !postReadbacksRef.current.has(+ localPostId,+ )+ )+ throw error;+ },+ },+ );+ if (!postReadbacksRef.current.has(localPostId)) return;+ post.avatarUrl = profile.avatarUrl;+ postReadbacksRef.current.delete(localPostId);+ confirmLocalFeedPost(+ setLocalFeedPosts,+ localPostId,+ post,+ );+ if (postPublishGenerationRef.current == generation)+ setPostPublication(+ (current) => current && { ...current, post },+ );+ } catch (error) {+ log.warn("Failed to refresh a published post", error);+ } finally {+ reading = false;+ }+ };+ postReadbacksRef.current.set(localPostId, readback);+ void readback();+ return posted;} catch (error) {failLocalFeedPost(setLocalFeedPosts,@@ -381,6 +458,7 @@ export const SpaceAppStateProvider: React.FC<React.PropsWithChildren> = ({setProfileLoadError(undefined);setProfileLoadStatus("ready");postPublishGenerationRef.current += 1;+ postReadbacksRef.current.clear();setPostPublication(null);setPendingLoginCredentials(null);setPendingPasskeyVerification(null);diff --git a/web/apps/space/src/state/app-state.tsx b/web/apps/space/src/state/app-state.tsxindex 8d5b222fc9..1bceff0f3c 100644--- a/web/apps/space/src/state/app-state.tsx+++ b/web/apps/space/src/state/app-state.tsx@@ -37,6 +37,7 @@ interface PendingSpaceFeedPost {name: string;spaceId: string;photoCount: number;+ postId?: number;status: "pending";timestampMs: number;width?: number;diff --git a/web/apps/space/src/utils/post-video.ts b/web/apps/space/src/utils/post-video.tsindex d7c437f992..1148b34b76 100644--- a/web/apps/space/src/utils/post-video.ts+++ b/web/apps/space/src/utils/post-video.ts@@ -1,10 +1,10 @@import type { FFmpegCommand } from "ente-base/types/ipc";+import type { PreparedSpacePostImage } from "utils/post-image";+import { thumbHashBase64FromCanvas } from "utils/thumbhash";import {inputPathPlaceholder,outputPathPlaceholder,-} from "ente-media/ffmpeg/constants";-import type { PreparedSpacePostImage } from "utils/post-image";-import { thumbHashBase64FromCanvas } from "utils/thumbhash";+} from "./video-encoding/constants";export interface SpacePostVideoEdit {start: number;@@ -234,7 +234,7 @@ export const prepareSpaceVideo = async (let exported = exports.get(file);if (exported?.key != key) {const { transcodeVideoWeb, determineVideoDurationWeb } =- await import("ente-media/ffmpeg/web");+ await import("./video-encoding/web");const command: FFmpegCommand = {default: videoExportCommand(edit, false),hdr: videoExportCommand(edit, true),diff --git a/web/apps/space/src/utils/video-encoding/constants.ts b/web/apps/space/src/utils/video-encoding/constants.tsnew file mode 100644index 0000000000..2efe7e0a88--- /dev/null+++ b/web/apps/space/src/utils/video-encoding/constants.ts@@ -0,0 +1,2 @@+export const inputPathPlaceholder = "INPUT";+export const outputPathPlaceholder = "OUTPUT";diff --git a/web/packages/media/ffmpeg/web-codecs.ts b/web/apps/space/src/utils/video-encoding/web-codecs.tssimilarity index 100%rename from web/packages/media/ffmpeg/web-codecs.tsrename to web/apps/space/src/utils/video-encoding/web-codecs.tsdiff --git a/web/packages/media/ffmpeg/web.ts b/web/apps/space/src/utils/video-encoding/web.tssimilarity index 85%rename from web/packages/media/ffmpeg/web.tsrename to web/apps/space/src/utils/video-encoding/web.tsindex 32d1718cf8..22a042c945 100644--- a/web/packages/media/ffmpeg/web.ts+++ b/web/apps/space/src/utils/video-encoding/web.ts@@ -6,16 +6,11 @@ import type { FFmpegCommand } from "ente-base/types/ipc";import { ensureArrayBufferBacked } from "ente-utils/bytes";import { PromiseQueue } from "ente-utils/promise";import { z } from "zod";-import {- ffmpegPathPlaceholder,- inputPathPlaceholder,- outputPathPlaceholder,-} from "./constants";+import { inputPathPlaceholder, outputPathPlaceholder } from "./constants";import type { VideoTranscodeEdit } from "./web-codecs";let _ffmpeg: Promise<FFmpeg> | undefined;-// Interleaved ffmpeg.wasm calls can corrupt its memory.const _ffmpegTaskQueue = new PromiseQueue<unknown>();const ffmpegLazy = (signal?: AbortSignal): Promise<FFmpeg> =>@@ -41,24 +36,6 @@ const createFFmpeg = async (signal?: AbortSignal) => {}};-export const ffmpegExecWeb = async (- command: FFmpegCommand,- blob: Blob,- outputFileExtension: string,- options?: { signal?: AbortSignal; onProgress?: (seconds: number) => void },-): Promise<Uint8Array<ArrayBuffer>> => {- return runFFmpegTask(async (ffmpeg) => {- const progress = ({ time }: { time: number }) =>- options?.onProgress?.(time / 1_000_000);- ffmpeg.on("progress", progress);- try {- return await ffmpegExec(ffmpeg, command, outputFileExtension, blob);- } finally {- ffmpeg.off("progress", progress);- }- }, options?.signal);-};-export const determineVideoDurationWeb = async (blob: Blob,signal?: AbortSignal,@@ -184,7 +161,6 @@ const ffmpegExec = async (try {if (ffmpeg.loaded) await ffmpeg.deleteFile(outputPath);} catch (e) {- // A failed command may not create the output file.if (status === 0) {log.error(`Failed to remove output ${outputPath}`, e);}@@ -228,19 +204,15 @@ const substitutePlaceholders = (inputFilePath: string,outputFilePath: string,) =>- command- .map((segment) => {- if (segment == ffmpegPathPlaceholder) {- return undefined;- } else if (segment == inputPathPlaceholder) {- return inputFilePath;- } else if (segment == outputPathPlaceholder) {- return outputFilePath;- } else {- return segment;- }- })- .filter((s) => s !== undefined);+ command.map((segment) => {+ if (segment == inputPathPlaceholder) {+ return inputFilePath;+ } else if (segment == outputPathPlaceholder) {+ return outputFilePath;+ } else {+ return segment;+ }+ });const FFProbeOutputIsHDR = z.object({streams: z.array(z.object({ color_transfer: z.string().optional() })),@@ -254,7 +226,6 @@ const isHDRVideo = async (ffmpeg: FFmpeg, inputFilePath: string) => {[["-i", inputFilePath],"-show_streams",- // FFmpeg may auto-select another stream in multi-stream files.["-select_streams", "v:0"],["-of", "json"],["-o", "output.json"],@@ -286,7 +257,6 @@ const ffprobeOutput = async (try {status = await ffmpeg.ffprobe(cmd);- // ffmpeg.wasm currently returns -1 on success.if (status !== 0 && status != -1) {log.info(`[wasm] ffprobe command failed with exit code ${status}: ${cmd.join(" ")}`,@@ -302,8 +272,6 @@ const ffprobeOutput = async (try {if (ffmpeg.loaded) await ffmpeg.deleteFile(outputPath);} catch (e) {- // Output file might not even exist if the command did not succeed,- // so only log on success.if (status === 0 || status == -1) {log.error(`Failed to remove output ${outputPath}`, e);}@@ -317,7 +285,6 @@ const FFProbeOutputDuration = z.object({const ffprobeExecVideoDuration = async (ffmpeg: FFmpeg, blob: Blob) =>withInputMount(ffmpeg, blob, async (inputPath) => {- // Scalar output can contain extra lines; JSON is more reliable.const jsonString = await ffprobeOutput(ffmpeg,[diff --git a/web/apps/space/tests/post-publication.test.ts b/web/apps/space/tests/post-publication.test.tsnew file mode 100644index 0000000000..6dcba1f9b6--- /dev/null+++ b/web/apps/space/tests/post-publication.test.ts@@ -0,0 +1,144 @@+import { beforeEach, expect, test, vi } from "vitest";+import {+ createCurrentMediaPost,+ loadCurrentCreatedPost,+ type SpacePostUploadSession,+} from "../src/services/space";++const mocks = vi.hoisted(() => ({+ ctx: {+ generatePostKey: vi.fn(),+ uploadPostPhotoAsset: vi.fn(),+ createMediaPost: vi.fn(),+ getPost: vi.fn(),+ downloadPostAsset: vi.fn(),+ },+ release: vi.fn(),+ cacheMedia: vi.fn(),+ cachePost: vi.fn(),+}));++vi.mock("services/profile", () => ({+ ensureCurrentSpaceContext: () => Promise.resolve(mocks.ctx),+ releaseCurrentSpaceContext: mocks.release,+}));+vi.mock("services/feed-cache", () => ({+ prependCachedSpaceFeedPost: mocks.cachePost,+}));+vi.mock("services/media-cache", () => ({+ rememberCachedSpaceMediaBlobURL: mocks.cacheMedia,+ spacePostMediaCacheKey: (spaceId: string, key: string) =>+ `${spaceId}:${key}`,+ clearSpaceMediaURLCache: vi.fn(),+}));+vi.mock("ente-base/log", () => ({ default: { warn: vi.fn() } }));++const file = new File(["photo"], "photo.webp", { type: "image/webp" });+const images = [{ file, width: 1200, height: 800, thumbHash: "hash" }];+const asset = {+ objectKey: "photo-1",+ postId: 501,+ spaceId: "self",+ encryptedPostKey: "encrypted-key",+ keyVersion: 1,+};++beforeEach(() => {+ vi.resetAllMocks();+ mocks.ctx.generatePostKey.mockReturnValue(new Uint8Array(32));+ mocks.ctx.uploadPostPhotoAsset.mockResolvedValue({+ objectKey: "photo-1",+ size: 100,+ metadataCipher: "metadata",+ });+ mocks.ctx.createMediaPost.mockResolvedValue(501n);+ mocks.ctx.getPost.mockResolvedValue({+ postId: 501,+ spaceId: "self",+ caption: "Caption",+ createdAt: "2026-09-30T00:00:00Z",+ viewerLiked: false,+ author: {+ spaceId: "self",+ spaceSlug: "self",+ profile: { fullName: "Self" },+ avatar: { objectID: "avatar", keyVersion: 1 },+ },+ photos: [{ asset, width: 1200, height: 800, mediaType: "image/webp" }],+ });+ mocks.cacheMedia.mockResolvedValue("blob:preview");+});++test("confirms creation without waiting for a post readback", async () => {+ mocks.ctx.getPost.mockRejectedValue(new Error("Offline"));+ const session: SpacePostUploadSession = { uploads: [] };+ await expect(+ createCurrentMediaPost({+ images,+ caption: "Caption",+ spaceId: "self",+ session,+ }),+ ).resolves.toBe(501);+ expect(session.postId).toBe(501);+ expect(mocks.ctx.getPost).not.toHaveBeenCalled();+ expect(mocks.release).toHaveBeenCalledWith(mocks.ctx);+});++test("retains a committed session through a failed readback without creating another post", async () => {+ const session: SpacePostUploadSession = { uploads: [] };+ const request = { images, caption: "Caption", spaceId: "self", session };+ const postId = await createCurrentMediaPost(request);+ mocks.ctx.getPost.mockRejectedValueOnce(new Error("Offline"));+ await expect(+ loadCurrentCreatedPost("self", postId, [file]),+ ).rejects.toThrow("Offline");+ expect(await createCurrentMediaPost(request)).toBe(postId);+ const post = await loadCurrentCreatedPost("self", postId, [file]);+ expect(post.postId).toBe(postId);+ expect(mocks.ctx.createMediaPost).toHaveBeenCalledTimes(1);+ expect(mocks.ctx.uploadPostPhotoAsset).toHaveBeenCalledTimes(1);+ expect(mocks.ctx.getPost).toHaveBeenNthCalledWith(1, "self", 501n, "self");+ expect(mocks.ctx.getPost).toHaveBeenNthCalledWith(2, "self", 501n, "self");+});++test("hydrates a published post from its uploaded previews without another media download", async () => {+ const post = await loadCurrentCreatedPost("self", 501, [file]);+ expect(post.imageUrl).toBe("blob:preview");+ expect(post.photos?.[0]).toMatchObject({+ imageUrl: "blob:preview",+ imageAsset: asset,+ });+ expect(mocks.cacheMedia).toHaveBeenCalledWith("self:photo-1", file);+ expect(mocks.ctx.downloadPostAsset).not.toHaveBeenCalled();+ expect(mocks.cachePost).toHaveBeenCalledWith("self", post);+});++test("a cache failure after creation does not discard the saved post ID", async () => {+ const session: SpacePostUploadSession = { uploads: [] };+ const request = { images, caption: "Caption", spaceId: "self", session };+ const postId = await createCurrentMediaPost(request);+ mocks.cacheMedia.mockRejectedValueOnce(new Error("Cache unavailable"));+ await expect(+ loadCurrentCreatedPost("self", postId, [file]),+ ).rejects.toThrow("Cache unavailable");+ expect(await createCurrentMediaPost(request)).toBe(postId);+ expect(mocks.ctx.createMediaPost).toHaveBeenCalledTimes(1);+});++test("a rejected creation is still reported as a failure", async () => {+ mocks.ctx.createMediaPost.mockRejectedValue(+ new Error("Post limit reached"),+ );+ const session: SpacePostUploadSession = { uploads: [] };+ await expect(+ createCurrentMediaPost({+ images,+ caption: "Caption",+ spaceId: "self",+ session,+ }),+ ).rejects.toThrow("Post limit reached");+ expect(session.postId).toBeUndefined();+ expect(mocks.ctx.getPost).not.toHaveBeenCalled();+});diff --git a/web/apps/space/tests/video-export.test.ts b/web/apps/space/tests/video-export.test.tsindex 3c056bf945..3011b40ef7 100644--- a/web/apps/space/tests/video-export.test.ts+++ b/web/apps/space/tests/video-export.test.ts@@ -13,7 +13,7 @@ const mocks = vi.hoisted(() => ({vi.mock("ente-base/log", () => ({default: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },}));-vi.mock("ente-media/ffmpeg/web-codecs", () => ({+vi.mock("../src/utils/video-encoding/web-codecs", () => ({encodeVideoWithWebCodecs: mocks.encode,}));vi.mock("@ffmpeg/ffmpeg", () => ({@@ -50,7 +50,8 @@ beforeEach(() => {afterEach(() => vi.unstubAllGlobals());const exporter = async () => {- const { transcodeVideoWeb } = await import("ente-media/ffmpeg/web");+ const { transcodeVideoWeb } =+ await import("../src/utils/video-encoding/web");return (signal?: AbortSignal) =>transcodeVideoWeb(new Blob(["video"]),diff --git a/web/apps/space/tests/video-webcodecs.test.ts b/web/apps/space/tests/video-webcodecs.test.tsindex d7c915ee64..580342528c 100644--- a/web/apps/space/tests/video-webcodecs.test.ts+++ b/web/apps/space/tests/video-webcodecs.test.ts@@ -1,6 +1,6 @@import type { FFmpeg } from "@ffmpeg/ffmpeg";-import { encodeVideoWithWebCodecs } from "ente-media/ffmpeg/web-codecs";import { afterEach, beforeEach, expect, test, vi } from "vitest";+import { encodeVideoWithWebCodecs } from "../src/utils/video-encoding/web-codecs";let frames: Frame[];let encodedSources: number[];diff --git a/web/apps/space/vitest.config.ts b/web/apps/space/vitest.config.tsindex 02cd4c512d..391b873843 100644--- a/web/apps/space/vitest.config.ts+++ b/web/apps/space/vitest.config.ts@@ -8,6 +8,7 @@ export default defineConfig({resolve: {alias: {services: fileURLToPath(new URL("./src/services", import.meta.url)),+ utils: fileURLToPath(new URL("./src/utils", import.meta.url)),},},});diff --git a/web/package-lock.json b/web/package-lock.jsonindex 8458153d8c..a89f747f56 100644--- a/web/package-lock.json+++ b/web/package-lock.json@@ -340,6 +340,7 @@"apps/space": {"version": "0.0.0","dependencies": {+ "@ffmpeg/ffmpeg": "0.12.15","@fontsource-variable/inter": "5.2.8","@fontsource/nunito": "5.2.7","@hugeicons/core-free-icons": "1.2.1",@@ -12210,6 +12211,7 @@"name": "ente-gallery","version": "0.0.0","dependencies": {+ "@ffmpeg/ffmpeg": "0.12.15","@hugeicons/core-free-icons": "1.2.1","@hugeicons/react": "1.1.6","bs58": "6.0.0",@@ -12243,7 +12245,6 @@"name": "ente-media","version": "0.0.0","dependencies": {- "@ffmpeg/ffmpeg": "0.12.15","ente-base": "*","ente-utils": "*","file-type": "21.3.4",diff --git a/web/packages/gallery/package.json b/web/packages/gallery/package.jsonindex f7bbfa51e7..1efd2915ba 100644--- a/web/packages/gallery/package.json+++ b/web/packages/gallery/package.json@@ -3,6 +3,7 @@"version": "0.0.0","private": true,"dependencies": {+ "@ffmpeg/ffmpeg": "0.12.15","@hugeicons/core-free-icons": "1.2.1","@hugeicons/react": "1.1.6","bs58": "6.0.0",diff --git a/web/packages/gallery/services/ffmpeg/constants.ts b/web/packages/gallery/services/ffmpeg/constants.tsindex 0a098cf212..dae03a9976 100644--- a/web/packages/gallery/services/ffmpeg/constants.ts+++ b/web/packages/gallery/services/ffmpeg/constants.ts@@ -1,5 +1,5 @@-export {- ffmpegPathPlaceholder,- inputPathPlaceholder,- outputPathPlaceholder,-} from "ente-media/ffmpeg/constants";+/* Ditto in the desktop app's code (used by the native FFmpeg invocation). */++export const ffmpegPathPlaceholder = "FFMPEG";+export const inputPathPlaceholder = "INPUT";+export const outputPathPlaceholder = "OUTPUT";diff --git a/web/packages/gallery/services/ffmpeg/web.ts b/web/packages/gallery/services/ffmpeg/web.tsindex 6657ba5b1c..229ecb6c20 100644--- a/web/packages/gallery/services/ffmpeg/web.ts+++ b/web/packages/gallery/services/ffmpeg/web.ts@@ -1,4 +1,258 @@-export {- determineVideoDurationWeb,- ffmpegExecWeb,-} from "ente-media/ffmpeg/web";+import { FFFSType, FFmpeg } from "@ffmpeg/ffmpeg";+import { joinPath } from "ente-base/file-name";+import { newID } from "ente-base/id";+import log from "ente-base/log";+import type { FFmpegCommand } from "ente-base/types/ipc";+import { ensureArrayBufferBacked } from "ente-utils/bytes";+import { PromiseQueue } from "ente-utils/promise";+import { z } from "zod";+import {+ ffmpegPathPlaceholder,+ inputPathPlaceholder,+ outputPathPlaceholder,+} from "./constants";++let _ffmpeg: Promise<FFmpeg> | undefined;++// Interleaved ffmpeg.wasm calls can corrupt its memory.+const _ffmpegTaskQueue = new PromiseQueue<Uint8Array<ArrayBuffer> | number>();++const ffmpegLazy = (): Promise<FFmpeg> => (_ffmpeg ??= createFFmpeg());++const createFFmpeg = async () => {+ const ffmpeg = new FFmpeg();+ await ffmpeg.load({+ coreURL: "https://assets.ente.com/ffmpeg-core-0.12.10/ffmpeg-core.js",+ wasmURL: "https://assets.ente.com/ffmpeg-core-0.12.10/ffmpeg-core.wasm",+ });+ return ffmpeg;+};++export const ffmpegExecWeb = async (+ command: FFmpegCommand,+ blob: Blob,+ outputFileExtension: string,+): Promise<Uint8Array<ArrayBuffer>> => {+ const ffmpeg = await ffmpegLazy();+ return _ffmpegTaskQueue.add(() =>+ ffmpegExec(ffmpeg, command, outputFileExtension, blob),+ ) as Promise<Uint8Array<ArrayBuffer>>;+};++export const determineVideoDurationWeb = async (+ blob: Blob,+): Promise<number> => {+ const ffmpeg = await ffmpegLazy();+ return _ffmpegTaskQueue.add(() =>+ ffprobeExecVideoDuration(ffmpeg, blob),+ ) as Promise<number>;+};++const ffmpegExec = async (+ ffmpeg: FFmpeg,+ command: FFmpegCommand,+ outputFileExtension: string,+ blob: Blob,+): Promise<Uint8Array<ArrayBuffer>> => {+ const outputSuffix = outputFileExtension ? "." + outputFileExtension : "";+ const outputPath = newID("out_") + outputSuffix;++ let status: number | undefined;++ return withInputMount(ffmpeg, blob, async (inputPath) => {+ try {+ const startTime = Date.now();++ let resolvedCommand: string[];+ if (Array.isArray(command)) {+ resolvedCommand = command;+ } else {+ const isHDR = await isHDRVideo(ffmpeg, inputPath);+ resolvedCommand = isHDR ? command.hdr : command.default;+ }++ const cmd = substitutePlaceholders(+ resolvedCommand,+ inputPath,+ outputPath,+ );++ status = await ffmpeg.exec(cmd);+ if (status !== 0) {+ log.info(+ `[wasm] ffmpeg command failed with exit code ${status}: ${cmd.join(" ")}`,+ );+ throw new Error(+ `ffmpeg command failed with exit code ${status}`,+ );+ }++ const result = await ffmpeg.readFile(outputPath);+ if (typeof result == "string")+ throw new Error("Expected binary data");++ const ms = Date.now() - startTime;+ log.debug(() => `[wasm] ffmpeg ${cmd.join(" ")} (${ms} ms)`);+ return ensureArrayBufferBacked(result);+ } finally {+ try {+ await ffmpeg.deleteFile(outputPath);+ } catch (e) {+ // A failed command may not create the output file.+ if (status === 0) {+ log.error(`Failed to remove output ${outputPath}`, e);+ }+ }+ }+ });+};++const withInputMount = async <T>(+ ffmpeg: FFmpeg,+ blob: Blob,+ f: (inputPath: string) => Promise<T>,+): Promise<T> => {+ const mountDir = "/mount";+ const inputFileName = newID("in_");+ const inputPath = joinPath(mountDir, inputFileName);++ const inputFile = new File([blob], inputFileName);++ try {+ await ffmpeg.createDir(mountDir);+ await ffmpeg.mount(FFFSType.WORKERFS, { files: [inputFile] }, mountDir);++ return await f(inputPath);+ } finally {+ try {+ await ffmpeg.unmount(mountDir);+ } catch (e) {+ log.error(`Failed to remove mount ${mountDir}`, e);+ }+ try {+ await ffmpeg.deleteDir(mountDir);+ } catch (e) {+ log.error(`Failed to delete mount directory ${mountDir}`, e);+ }+ }+};++const substitutePlaceholders = (+ command: string[],+ inputFilePath: string,+ outputFilePath: string,+) =>+ command+ .map((segment) => {+ if (segment == ffmpegPathPlaceholder) {+ return undefined;+ } else if (segment == inputPathPlaceholder) {+ return inputFilePath;+ } else if (segment == outputPathPlaceholder) {+ return outputFilePath;+ } else {+ return segment;+ }+ })+ .filter((s) => s !== undefined);++const FFProbeOutputIsHDR = z.object({+ streams: z.array(z.object({ color_transfer: z.string().optional() })),+});++const isHDRVideo = async (ffmpeg: FFmpeg, inputFilePath: string) => {+ let jsonString: string | undefined;+ try {+ jsonString = await ffprobeOutput(+ ffmpeg,+ [+ ["-i", inputFilePath],+ "-show_streams",+ // FFmpeg may auto-select another stream in multi-stream files.+ ["-select_streams", "v:0"],+ ["-of", "json"],+ ["-o", "output.json"],+ ].flat(),+ "output.json",+ );++ const output = FFProbeOutputIsHDR.parse(JSON.parse(jsonString));+ switch (output.streams[0]?.color_transfer) {+ case "smpte2084":+ case "arib-std-b67":+ return true;+ default:+ return false;+ }+ } catch (e) {+ log.warn("Could not detect HDR status", e);+ if (jsonString) log.debug(() => ["ffprobe-output", jsonString]);+ return false;+ }+};++const ffprobeOutput = async (+ ffmpeg: FFmpeg,+ cmd: string[],+ outputPath: string,+) => {+ let status: number | undefined;++ try {+ status = await ffmpeg.ffprobe(cmd);+ // ffmpeg.wasm currently returns -1 on success.+ if (status !== 0 && status != -1) {+ log.info(+ `[wasm] ffprobe command failed with exit code ${status}: ${cmd.join(" ")}`,+ );+ throw new Error(`ffprobe command failed with exit code ${status}`);+ }++ const result = await ffmpeg.readFile(outputPath, "utf8");+ if (typeof result != "string") throw new Error("Expected text data");++ return result;+ } finally {+ try {+ await ffmpeg.deleteFile(outputPath);+ } catch (e) {+ // Output file might not even exist if the command did not succeed,+ // so only log on success.+ if (status === 0 || status == -1) {+ log.error(`Failed to remove output ${outputPath}`, e);+ }+ }+ }+};++const FFProbeOutputDuration = z.object({+ format: z.object({ duration: z.string() }),+});++const ffprobeExecVideoDuration = async (ffmpeg: FFmpeg, blob: Blob) =>+ withInputMount(ffmpeg, blob, async (inputPath) => {+ // Scalar output can contain extra lines; JSON is more reliable.+ const jsonString = await ffprobeOutput(+ ffmpeg,+ [+ ["-i", inputPath],+ ["-v", "error"],+ ["-show_entries", "format=duration"],+ ["-of", "json"],+ ["-o", "output.json"],+ ].flat(),+ "output.json",+ );++ const durationString = FFProbeOutputDuration.parse(+ JSON.parse(jsonString),+ ).format.duration;++ const duration = parseFloat(durationString);+ if (isNaN(duration)) {+ const msg = "Could not parse video duration";+ log.warn(msg, durationString);+ throw new Error(msg);+ }+ return duration;+ });diff --git a/web/packages/media/ffmpeg/constants.ts b/web/packages/media/ffmpeg/constants.tsdeleted file mode 100644index dae03a9976..0000000000--- a/web/packages/media/ffmpeg/constants.ts+++ /dev/null@@ -1,5 +0,0 @@-/* Ditto in the desktop app's code (used by the native FFmpeg invocation). */--export const ffmpegPathPlaceholder = "FFMPEG";-export const inputPathPlaceholder = "INPUT";-export const outputPathPlaceholder = "OUTPUT";diff --git a/web/packages/media/package.json b/web/packages/media/package.jsonindex 48630304cf..bfa9d5494b 100644--- a/web/packages/media/package.json+++ b/web/packages/media/package.json@@ -3,7 +3,6 @@"version": "0.0.0","private": true,"dependencies": {- "@ffmpeg/ffmpeg": "0.12.15","ente-base": "*","ente-utils": "*","file-type": "21.3.4",