Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Wait for video preview before viewer handoff

Anand Baburajan5a4fbf9514dc
diff --git a/web/apps/space/src/components/FileViewer.tsx b/web/apps/space/src/components/FileViewer.tsx
index e6f57b8063..d2a787b01e 100644
--- a/web/apps/space/src/components/FileViewer.tsx
+++ b/web/apps/space/src/components/FileViewer.tsx
@@ -287,7 +287,9 @@ const handOverOpenFlight = (
const image = content?.element;
const isShown =
content?.isError() ||
- (content?.data.type == "video" && image?.isConnected) ||
+ (content?.data.type == "video" &&
+ content.state == "loaded" &&
+ image?.isConnected) ||
(image instanceof HTMLImageElement &&
image.complete &&
image.naturalWidth > 0 &&
@@ -1539,13 +1541,15 @@ export const SpaceFileViewer: React.FC<SpaceFileViewerProps> = ({
pswp.on("contentLoad", (event) => {
if (event.content.data.type != "video") return;
event.preventDefault();
+ event.content.state = "loading";
videos.get(event.content)?.destroy();
const item = viewerPhotosRef.current[event.content.index]!;
- const video = createSpaceVideoContent(item, (asset) =>
- videoLoaderRef.current!(asset),
+ const video = createSpaceVideoContent(
+ item,
+ (asset) => videoLoaderRef.current!(asset),
+ { onReady: () => event.content.onLoaded() },
);
event.content.element = video.element;
- event.content.onLoaded();
videos.set(event.content, video);
});
pswp.on("contentActivate", ({ content }) =>
diff --git a/web/apps/space/src/components/PostVideoContent.ts b/web/apps/space/src/components/PostVideoContent.ts
index 3f48864201..e7ab0110c1 100644
--- a/web/apps/space/src/components/PostVideoContent.ts
+++ b/web/apps/space/src/components/PostVideoContent.ts
@@ -10,11 +10,13 @@ export const createSpaceVideoContent = (
inline = false,
onPlay,
onPause,
+ onReady,
onTogglePlayback,
}: {
inline?: boolean;
onPlay?: () => Promise<void> | void;
onPause?: () => void;
+ onReady?: () => void;
onTogglePlayback?: (play: boolean) => void;
} = {},
) => {
@@ -107,7 +109,24 @@ export const createSpaceVideoContent = (
}
supportMessage.style.marginTop = "2px";
errorMessage.append(errorTitle, supportMessage);
- element.append(video, button, loading, errorMessage);
+ const poster =
+ !inline && item.imageUrl ? document.createElement("img") : undefined;
+ element.append(video);
+ if (poster) {
+ poster.alt = "";
+ poster.src = item.imageUrl!;
+ poster.hidden = true;
+ Object.assign(poster.style, {
+ position: "absolute",
+ inset: "0",
+ width: "100%",
+ height: "100%",
+ objectFit: "contain",
+ pointerEvents: "none",
+ });
+ element.append(poster);
+ }
+ element.append(button, loading, errorMessage);
let generation = 0;
let sourceGeneration = 0;
let pendingLoad: Promise<void> | undefined;
@@ -116,9 +135,27 @@ export const createSpaceVideoContent = (
let active = false;
let autoplayBlocked = false;
let failure: "error" | "rate-limit" | undefined;
+ let posterReady = false;
+ let ready = false;
let retryTimer: ReturnType<typeof setTimeout> | undefined;
let loadingTimer: ReturnType<typeof setTimeout> | undefined;
let playbackTimeout: ReturnType<typeof setTimeout> | undefined;
+ const updatePreview = () => {
+ if (disposed) return;
+ const frameReady = video.readyState >= video.HAVE_CURRENT_DATA;
+ if (poster) poster.hidden = !posterReady || frameReady;
+ if (!ready && (posterReady || frameReady || failure)) {
+ ready = true;
+ onReady?.();
+ }
+ };
+ void poster?.decode().then(
+ () => {
+ posterReady = true;
+ updatePreview();
+ },
+ () => undefined,
+ );
const hideLoading = () => {
clearTimeout(loadingTimer);
loadingTimer = undefined;
@@ -166,6 +203,7 @@ export const createSpaceVideoContent = (
button.disabled = false;
button.setAttribute("aria-hidden", "true");
video.setAttribute("aria-label", "Pause video");
+ updatePreview();
};
const pause = () => {
generation++;
@@ -179,6 +217,7 @@ export const createSpaceVideoContent = (
pause();
video.removeAttribute("src");
video.load();
+ updatePreview();
};
const failPlayback = (
reason: "timeout" | "media" | "play",
@@ -315,6 +354,7 @@ export const createSpaceVideoContent = (
if (!video.paused) showLoading();
};
video.onplaying = showPlaying;
+ video.onloadeddata = updatePreview;
video.onerror = () => {
if (active) failPlayback("media");
else reset();
diff --git a/web/apps/space/tests/post-video-content.test.ts b/web/apps/space/tests/post-video-content.test.ts
index c72ba77765..62e47204a6 100644
--- a/web/apps/space/tests/post-video-content.test.ts
+++ b/web/apps/space/tests/post-video-content.test.ts
@@ -30,16 +30,20 @@ class Element extends EventTarget {
}
class Video extends Element {
+ HAVE_CURRENT_DATA = 2;
+ readyState = 0;
preload = "none";
paused = true;
ended = false;
currentTime = 0;
onpause?: () => void;
onplaying?: () => void;
+ onloadeddata?: () => void;
set src(value: string) {
this.setAttribute("src", value);
}
play = vi.fn(() => {
+ this.readyState = this.HAVE_CURRENT_DATA;
this.paused = false;
this.onplaying?.();
return Promise.resolve();
@@ -49,16 +53,24 @@ class Video extends Element {
this.paused = true;
this.onpause?.();
}
- load = vi.fn();
+ load = vi.fn(() => {
+ this.readyState = 0;
+ });
+}
+
+class Poster extends Element {
+ decode = () => decodePoster();
}
let contents: ReturnType<typeof createSpaceVideoContent>[];
let load: ReturnType<typeof vi.fn<SpacePostAssetURLLoader>>;
let revokeURL: ReturnType<typeof vi.spyOn>;
+let decodePoster: ReturnType<typeof vi.fn<() => Promise<void>>>;
beforeEach(() => {
vi.useFakeTimers();
contents = [];
+ decodePoster = vi.fn<() => Promise<void>>().mockResolvedValue(undefined);
load = vi.fn<SpacePostAssetURLLoader>().mockResolvedValue("blob:video");
revokeURL = vi
.spyOn(URL, "revokeObjectURL")
@@ -68,7 +80,11 @@ beforeEach(() => {
Object.assign(new EventTarget(), {
hidden: false,
createElement: (tag: string) =>
- tag == "video" ? new Video() : new Element(),
+ tag == "video"
+ ? new Video()
+ : tag == "img"
+ ? new Poster()
+ : new Element(),
}),
);
});
@@ -97,12 +113,15 @@ const create = (
{ ...options, inline },
);
contents.push(content);
- const [video, button, loading, error] = (
- content.element as unknown as Element
- ).children;
+ const children = (content.element as unknown as Element).children;
+ const poster = children.find((child) => child instanceof Poster);
+ const [video, button, loading, error] = children.filter(
+ (child) => child != poster,
+ );
return {
...content,
video: video as Video,
+ poster,
button: button!,
loading: loading!,
error: error!,
@@ -132,6 +151,90 @@ test("a video can download and play before its cover image is available", async
expect(content.video.getAttribute("poster")).toBeNull();
});
+test("viewer content waits for its cover to decode before handing over the flyer", async () => {
+ const decoded = Promise.withResolvers<undefined>();
+ decodePoster.mockReturnValue(decoded.promise);
+ const onReady = vi.fn(() => expect(content.poster?.hidden).toBe(false));
+ const content = create(false, "blob:poster", { onReady });
+ await vi.advanceTimersByTimeAsync(0);
+ expect(onReady).not.toHaveBeenCalled();
+ expect(content.poster?.hidden).toBe(true);
+
+ decoded.resolve(undefined);
+ await vi.advanceTimersByTimeAsync(0);
+ expect(onReady).toHaveBeenCalledOnce();
+ expect(content.video.readyState).toBe(0);
+ expect(content.video.play).not.toHaveBeenCalled();
+});
+
+test("a decoded video frame can hand over before its cover finishes loading", async () => {
+ const decoded = Promise.withResolvers<undefined>();
+ decodePoster.mockReturnValue(decoded.promise);
+ const onReady = vi.fn();
+ const content = create(false, "blob:poster", { onReady });
+ content.video.readyState = content.video.HAVE_CURRENT_DATA;
+ content.video.onloadeddata?.();
+ expect(onReady).toHaveBeenCalledOnce();
+ expect(content.poster?.hidden).toBe(true);
+
+ decoded.resolve(undefined);
+ await vi.advanceTimersByTimeAsync(0);
+ expect(onReady).toHaveBeenCalledOnce();
+ expect(content.poster?.hidden).toBe(true);
+});
+
+test("the viewer cover yields to playback and returns when the video resets", async () => {
+ const onReady = vi.fn();
+ const content = create(false, "blob:poster", { onReady });
+ await vi.advanceTimersByTimeAsync(0);
+ expect(content.poster?.hidden).toBe(false);
+ content.play();
+ await vi.advanceTimersByTimeAsync(0);
+ expect(content.poster?.hidden).toBe(true);
+
+ content.deactivate();
+ expect(content.poster?.hidden).toBe(false);
+ expect(onReady).toHaveBeenCalledOnce();
+});
+
+test("destroyed viewer content cannot hand over after late cover or video readiness", async () => {
+ const decoded = Promise.withResolvers<undefined>();
+ decodePoster.mockReturnValue(decoded.promise);
+ const onReady = vi.fn();
+ const content = create(false, "blob:poster", { onReady });
+ content.destroy();
+ decoded.resolve(undefined);
+ await vi.advanceTimersByTimeAsync(0);
+ content.video.readyState = content.video.HAVE_CURRENT_DATA;
+ content.video.onloadeddata?.();
+ expect(onReady).not.toHaveBeenCalled();
+ expect(content.poster?.hidden).toBe(true);
+});
+
+test("a failed cover waits for a video frame before handing over", async () => {
+ decodePoster.mockRejectedValue(new Error("decode failed"));
+ const onReady = vi.fn();
+ const content = create(false, "blob:poster", { onReady });
+ await vi.advanceTimersByTimeAsync(0);
+ expect(onReady).not.toHaveBeenCalled();
+
+ content.video.readyState = content.video.HAVE_CURRENT_DATA;
+ content.video.onloadeddata?.();
+ expect(onReady).toHaveBeenCalledOnce();
+ expect(content.poster?.hidden).toBe(true);
+});
+
+test("failed cover and video loading hand over to the visible error", async () => {
+ decodePoster.mockRejectedValue(new Error("decode failed"));
+ load.mockRejectedValue(new Error("Offline"));
+ const onReady = vi.fn(() => expect(content.error.hidden).toBe(false));
+ const content = create(false, "blob:poster", { onReady });
+ content.play();
+ await vi.advanceTimersByTimeAsync(0);
+ expect(onReady).toHaveBeenCalledOnce();
+ expect(content.errorTitle.textContent).toBe("Couldn't play video.");
+});
+
test("autoplay joins an in-flight preload without downloading the video again", async () => {
const download = Promise.withResolvers<string>();
load.mockReturnValue(download.promise);