π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
refractor(video):optimize-code
Aswin Asok03fd4b31d889
diff --git a/web/apps/photos/tests/video-streaming-percentage.test.ts b/web/apps/photos/tests/video-streaming-percentage.test.tsindex 77c37d9022..bcdbabd344 100644--- a/web/apps/photos/tests/video-streaming-percentage.test.ts+++ b/web/apps/photos/tests/video-streaming-percentage.test.ts@@ -57,30 +57,20 @@ vi.mock("ente-base/kv", async (importOriginal) => ({return Promise.resolve();},}));-vi.mock("ente-accounts/services/user", async (importOriginal) => ({- ...(await importOriginal<typeof import("ente-accounts/services/user")>()),+vi.mock("ente-accounts/services/user", () => ({ensureLocalUser: () => ({ id: 1 }),}));-vi.mock("ente-new/photos/services/photos-fdb", async (importOriginal) => ({- ...(await importOriginal<- typeof import("ente-new/photos/services/photos-fdb")- >()),+vi.mock("ente-new/photos/services/photos-fdb", () => ({savedCollectionFiles: async () => {mocks.collectionFilesReadCount++;await mocks.collectionFilesReadGate;return mocks.collectionFiles;},}));-vi.mock("ente-new/photos/services/trash", async (importOriginal) => ({- ...(await importOriginal<- typeof import("ente-new/photos/services/trash")- >()),+vi.mock("ente-new/photos/services/trash", () => ({savedTrashItemFileIDs: () => Promise.resolve(new Set<number>()),}));-vi.mock("ente-gallery/services/file-data", async (importOriginal) => ({- ...(await importOriginal<- typeof import("ente-gallery/services/file-data")- >()),+vi.mock("ente-gallery/services/file-data", () => ({syncUpdatedFileDataFileIDs: async (_type: string,_lastUpdatedAt: number,@@ -96,18 +86,13 @@ vi.mock("ente-gallery/services/file-data", async (importOriginal) => ({},fetchFileData: () => mocks.fetchFileDataResult,}));-vi.mock("ente-gallery/services/upload", async (importOriginal) => ({- ...(await importOriginal<typeof import("ente-gallery/services/upload")>()),+vi.mock("ente-gallery/services/upload", () => ({fileSystemUploadItemIfUnchanged: () => Promise.resolve({}),}));-vi.mock("ente-gallery/utils/native-stream", async (importOriginal) => ({- ...(await importOriginal<- typeof import("ente-gallery/utils/native-stream")- >()),+vi.mock("ente-gallery/utils/native-stream", () => ({initiateGenerateHLS: () => Promise.resolve(undefined),}));-vi.mock("ente-new/photos/services/file", async (importOriginal) => ({- ...(await importOriginal<typeof import("ente-new/photos/services/file")>()),+vi.mock("ente-new/photos/services/file", () => ({updateFilePublicMagicMetadata: () => Promise.resolve(),}));vi.mock("ente-utils/promise", async (importOriginal) => ({@@ -118,7 +103,6 @@ vi.mock("ente-utils/promise", async (importOriginal) => ({const {hlsGenerationStatusSnapshot,initVideoProcessing,- processedVideoFraction,processVideoNewUpload,resetVideoState,streamCandidateFiles,@@ -130,11 +114,13 @@ const {const MiB = 1024 * 1024;const expectProcessedFraction = (processedFraction: number) =>- vi.waitFor(() =>- expect(hlsGenerationStatusSnapshot()).toMatchObject({- enabled: true,- processedFraction,- }),+ vi.waitFor(+ () =>+ expect(hlsGenerationStatusSnapshot()).toMatchObject({+ enabled: true,+ processedFraction,+ }),+ { interval: 5 },);const file = (@@ -175,25 +161,36 @@ describe("video streaming percentage", () => {mocks.fetchFileDataResult = new Promise<undefined>(() => undefined);});- test("calculates the processed fraction for all Desktop candidates", () => {- expect(processedVideoFraction(new Set(), [])).toBe(1);- expect(processedVideoFraction(new Set(), [file(1), file(2)])).toBe(0);- expect(- processedVideoFraction(new Set([1, 3]), [+ test.each([+ { ids: [], files: [], fraction: 1 },+ { ids: [], files: [file(1), file(2)], fraction: 0 },+ {+ ids: [1, 3],+ files: [file(1), file(2), file(3), file(4)],+ fraction: 0.5,+ },+ {+ ids: [1],+ files: [file(1),- file(2),- file(3),- file(4),- ]),- ).toBe(0.5);-- const candidates = [- file(1),- file(2, { duration: 61 }),- file(3, { fileSize: 500 * MiB + 1 }),- ];- expect(processedVideoFraction(new Set([1]), candidates)).toBe(1 / 3);- });+ file(2, { duration: 61 }),+ file(3, { fileSize: 500 * MiB + 1 }),+ ],+ fraction: 1 / 3,+ },+ { ids: [1], files: [file(1), file(1), file(2)], fraction: 0.5 },+ { ids: [3], files: [], fraction: 1 },+ { ids: [1, 3], files: [file(1), file(2)], fraction: 0.5 },+ ])(+ "publishes the expected fraction: $fraction",+ async ({ ids, files, fraction }) => {+ mocks.kv.set("generateHLS", true);+ mocks.kv.set("videoPreviewProcessedFileIDs", ids);+ mocks.collectionFiles = files;+ await initVideoProcessing();+ await expectProcessedFraction(fraction);+ },+ );test("reuses Desktop's backfill population", () => {const candidates = streamCandidateFiles(@@ -223,7 +220,9 @@ describe("video streaming percentage", () => {await videoProcessingSyncIfNeeded();const toggle = toggleHLSGeneration();- await vi.waitFor(() => expect(mocks.previewStatusPullCount).toBe(1));+ await vi.waitFor(() => expect(mocks.previewStatusPullCount).toBe(1), {+ interval: 5,+ });expect(hlsGenerationStatusSnapshot()).toEqual({ enabled: true });expect(mocks.collectionFilesReadCount).toBe(0);@@ -238,7 +237,9 @@ describe("video streaming percentage", () => {mocks.previewStatusPullGate = previewStatusPull.promise;const enable = toggleHLSGeneration();- await vi.waitFor(() => expect(mocks.previewStatusPullCount).toBe(1));+ await vi.waitFor(() => expect(mocks.previewStatusPullCount).toBe(1), {+ interval: 5,+ });await toggleHLSGeneration();previewStatusPull.resolve(undefined);await enable;@@ -318,12 +319,65 @@ describe("video streaming percentage", () => {collectionFilesRead.resolve(undefined);- await vi.waitFor(() => {- expect(mocks.collectionFilesReadCount).toBe(2);- expect(hlsGenerationStatusSnapshot()).toMatchObject({- enabled: true,- processedFraction: 0,- });- });+ await vi.waitFor(+ () => {+ expect(mocks.collectionFilesReadCount).toBe(2);+ expect(hlsGenerationStatusSnapshot()).toMatchObject({+ enabled: true,+ processedFraction: 0,+ });+ },+ { interval: 5 },+ );+ });+ test("discards a refresh after generation is disabled", async () => {+ mocks.kv.set("generateHLS", true);+ mocks.collectionFiles = [file(1)];+ const gate = Promise.withResolvers<undefined>();+ mocks.collectionFilesReadGate = gate.promise;+ await initVideoProcessing();+ await toggleHLSGeneration();+ gate.resolve(undefined);+ await new Promise<void>(setImmediate);+ expect(hlsGenerationStatusSnapshot()).toEqual({ enabled: false });+ });++ test("does not publish an old account refresh after reset", async () => {+ mocks.kv.set("generateHLS", true);+ mocks.collectionFiles = [file(1)];+ const gate = Promise.withResolvers<undefined>();+ mocks.collectionFilesReadGate = gate.promise;+ await initVideoProcessing();+ resetVideoState();+ gate.resolve(undefined);+ await new Promise<void>(setImmediate);+ expect(hlsGenerationStatusSnapshot()).toBeUndefined();+ });++ test("includes completion recorded during an overlapping refresh", async () => {+ mocks.kv.set("generateHLS", true);+ mocks.collectionFiles = [file(1), file(2)];+ mocks.kv.set("videoPreviewProcessedFileIDs", [1]);+ const gate = Promise.withResolvers<undefined>();+ mocks.collectionFilesReadGate = gate.promise;+ await initVideoProcessing();+ mocks.kv.set("videoPreviewProcessedFileIDs", [1, 2]);+ await initVideoProcessing();+ gate.resolve(undefined);+ await expectProcessedFraction(1);+ expect(mocks.collectionFilesReadCount).toBe(2);+ });++ test("can refresh again after a storage failure", async () => {+ mocks.kv.set("generateHLS", true);+ mocks.collectionFiles = [file(1)];+ const gate = Promise.withResolvers<undefined>();+ mocks.collectionFilesReadGate = gate.promise;+ await initVideoProcessing();+ gate.reject(new Error("storage read failed"));+ await new Promise<void>(setImmediate);+ mocks.collectionFilesReadGate = undefined;+ await initVideoProcessing();+ await expectProcessedFraction(0);});});diff --git a/web/packages/gallery/services/video.ts b/web/packages/gallery/services/video.tsindex c4fb0bf658..3372943853 100644--- a/web/packages/gallery/services/video.ts+++ b/web/packages/gallery/services/video.ts@@ -65,8 +65,7 @@ class VideoState {hlsGenerationStatusSnapshot: HLSGenerationStatus | undefined;lastEnabledStatus: HLSGenerationEnabledStatus | undefined;processedFraction: number | undefined;- processedFractionRefresh: Promise<void> | undefined;- processedFractionRefreshRequests = 0;+ processedFractionRefresh: "running" | "pending" | undefined;candidateFileIDs: Set<number> | undefined;processedCandidateFileIDs: Set<number> | undefined;unsyncedUploadFiles = new Map<number, EnteFile>();@@ -318,17 +317,6 @@ export const videoPrunePermanentlyDeletedFileIDsIfNeeded = async (}};-export const processedVideoFraction = (- processedFileIDs: Set<number>,- candidateFiles: EnteFile[],-) => {- const candidateFileIDs = new Set(candidateFiles.map((f) => f.id));- const previewFileIDs = processedFileIDs.intersection(candidateFileIDs);- return candidateFileIDs.size == 0- ? 1- : previewFileIDs.size / candidateFileIDs.size;-};-const publishProcessedFraction = (state: VideoState) => {const candidateCount = state.candidateFileIDs!.size;const fraction =@@ -385,51 +373,36 @@ const refreshProcessedFractionIfNeeded = () => {if (!isHLSGenerationSupported || !isHLSGenerationEnabled()) return;const state = _state;- state.processedFractionRefreshRequests++;- if (state.processedFractionRefresh) return;-- let handledRequests = 0;- state.processedFractionRefresh = (async () => {- while (handledRequests != state.processedFractionRefreshRequests) {- const request = state.processedFractionRefreshRequests;- try {- const [candidates, processedFileIDs] = await Promise.all([- savedStreamCandidateFiles(- ensureLocalUser().id,- state.unsyncedUploadFiles,- state.locallySkippedFileIDs,- ),- savedProcessedVideoFileIDs(),- ]);- const fraction = processedVideoFraction(- processedFileIDs,- candidates,+ if (state.processedFractionRefresh) {+ state.processedFractionRefresh = "pending";+ return;+ }+ state.processedFractionRefresh = "running";+ void (async () => {+ try {+ const [candidates, processedFileIDs] = await Promise.all([+ savedStreamCandidateFiles(+ ensureLocalUser().id,+ state.unsyncedUploadFiles,+ state.locallySkippedFileIDs,+ ),+ savedProcessedVideoFileIDs(),+ ]);+ if (state != _state || !state.isHLSGenerationEnabled) return;+ if (state.processedFractionRefresh != "pending") {+ state.candidateFileIDs = new Set(candidates.map((f) => f.id));+ state.processedCandidateFileIDs = processedFileIDs.intersection(+ state.candidateFileIDs,);- if (state != _state || !state.isHLSGenerationEnabled) return;- if (request == state.processedFractionRefreshRequests) {- state.candidateFileIDs = new Set(- candidates.map((f) => f.id),- );- state.processedCandidateFileIDs =- processedFileIDs.intersection(state.candidateFileIDs);- if (fraction != state.processedFraction) {- state.processedFraction = fraction;- publishEnabledSnapshot();- }- }- } catch (e) {- log.error("Failed to compute video processed fraction", e);+ publishProcessedFraction(state);}- handledRequests = request;+ } catch (e) {+ log.error("Failed to compute video processed fraction", e);}})().finally(() => {+ const pending = state.processedFractionRefresh == "pending";state.processedFractionRefresh = undefined;- if (- state == _state &&- handledRequests != state.processedFractionRefreshRequests- ) {- refreshProcessedFractionIfNeeded();- }+ if (state == _state && pending) refreshProcessedFractionIfNeeded();});};@@ -563,8 +536,7 @@ const savedStreamCandidateFiles = async (savedCollectionFiles(),savedTrashItemFileIDs(),]);- const savedFileIDs = new Set(savedFiles.map((file) => file.id));- savedFileIDs.forEach((fileID) => unsyncedUploadFiles.delete(fileID));+ savedFiles.forEach(({ id }) => unsyncedUploadFiles.delete(id));return streamCandidateFiles([...savedFiles, ...unsyncedUploadFiles.values()],trashFileIDs,