Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

fix(preview): stop Chromium decoding a soundless recording's picture

EtienneLescotc593090c71da
diff --git a/src/components/ai-edition/VirtualPreview.tracks.test.ts b/src/components/ai-edition/VirtualPreview.tracks.test.ts
index 731861265..a1634a8be 100644
--- a/src/components/ai-edition/VirtualPreview.tracks.test.ts
+++ b/src/components/ai-edition/VirtualPreview.tracks.test.ts
@@ -24,12 +24,45 @@ describe("dropVideoTrack", () => {
});
// With neither track selected an element has no stream left to keep time with: measured,
- // its clock races to the end instead of playing.
- it("leaves a recording without sound decoding, so its clock still runs", () => {
- const { element: media, videoTracks } = element({ audio: 0, video: 1 });
+ // it jumps to its end on play. So it keeps time by the wall clock (issue #994).
+ it("deselects the picture of a recording without sound too, and keeps time by the wall clock", () => {
+ // Accessors on the prototype, where the element's own live.
+ class Media extends EventTarget {
+ duration = 10;
+ time = 0;
+ rate = 1;
+ get currentTime() {
+ return this.time;
+ }
+ set currentTime(sec: number) {
+ this.time = sec;
+ }
+ get playbackRate() {
+ return this.rate;
+ }
+ set playbackRate(rate: number) {
+ this.rate = rate;
+ }
+ get paused() {
+ return true;
+ }
+ play(): Promise<void> {
+ throw new Error("played for real");
+ }
+ pause(): void {
+ throw new Error("never playing for real, so never paused for real");
+ }
+ }
+ const videoTracks = [{ selected: true }];
+ const media = Object.assign(new Media(), {
+ audioTracks: { length: 0 },
+ videoTracks,
+ }) as unknown as HTMLMediaElement;
- expect(dropVideoTrack(media)).toBe(false);
- expect(videoTracks[0].selected).toBe(true);
+ expect(dropVideoTrack(media)).toBe(true);
+ expect(videoTracks[0].selected).toBe(false);
+ void media.play();
+ expect(media.paused).toBe(false);
});
it("does nothing where the track lists are absent (the Blink feature is off)", () => {
diff --git a/src/components/ai-edition/VirtualPreview.tsx b/src/components/ai-edition/VirtualPreview.tsx
index beda87868..c24d24191 100644
--- a/src/components/ai-edition/VirtualPreview.tsx
+++ b/src/components/ai-edition/VirtualPreview.tsx
@@ -51,6 +51,7 @@ import {
retryDelayMs,
} from "./mediaError";
import styles from "./VirtualPreview.module.css";
+import { keepTimeByWallClock } from "./wallClock";
export interface VideoSource {
id: string;
@@ -193,15 +194,18 @@ interface MediaTrackLists {
* deselected the element decodes no frame, and its clock, `playbackRate` and seeks behave
* the same.
*
- * Only when the element has a sound track: with neither track selected it has no stream
- * left to keep time with, and races to its end. A recording without sound keeps decoding,
- * as before. Returns whether the track was dropped.
+ * Without a sound track the element has no stream left to keep time with once its picture
+ * is deselected (measured: it jumps to its end on `play`), so it keeps time by the wall clock
+ * instead (`keepTimeByWallClock`, issue #994). Returns whether the track was dropped.
*/
export function dropVideoTrack(element: HTMLMediaElement): boolean {
const { audioTracks, videoTracks } = element as HTMLMediaElement & MediaTrackLists;
- if (!videoTracks?.length || !audioTracks?.length) {
+ if (!videoTracks?.length || !audioTracks) {
return false;
}
+ if (audioTracks.length === 0) {
+ keepTimeByWallClock(element);
+ }
for (let index = 0; index < videoTracks.length; index++) {
videoTracks[index].selected = false;
}
diff --git a/src/components/ai-edition/wallClock.test.ts b/src/components/ai-edition/wallClock.test.ts
new file mode 100644
index 000000000..e1e7cd1d0
--- /dev/null
+++ b/src/components/ai-edition/wallClock.test.ts
@@ -0,0 +1,154 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { keepTimeByWallClock } from "./wallClock";
+
+/** The slice of HTMLMediaElement the clock touches, playing for real when asked to. */
+class FakeMedia extends EventTarget {
+ duration = 40;
+ nativePlays = 0;
+ nativePauses = 0;
+ seeks: number[] = [];
+ #time = 0;
+ #rate = 1;
+ #paused = true;
+ get currentTime() {
+ return this.#time;
+ }
+ set currentTime(sec: number) {
+ this.seeks.push(sec);
+ this.#time = Math.min(sec, this.duration);
+ }
+ get playbackRate() {
+ return this.#rate;
+ }
+ set playbackRate(value: number) {
+ if (value > 16) throw new DOMException("rate", "NotSupportedError");
+ this.#rate = value;
+ }
+ get paused() {
+ return this.#paused;
+ }
+ play() {
+ this.nativePlays++;
+ this.#paused = false;
+ return Promise.resolve();
+ }
+ pause() {
+ this.nativePauses++;
+ this.#paused = true;
+ }
+}
+
+let nowMs = 0;
+const clock = () => nowMs;
+
+function clocked() {
+ const media = new FakeMedia();
+ keepTimeByWallClock(media as unknown as HTMLMediaElement, clock);
+ return media as unknown as HTMLMediaElement & FakeMedia;
+}
+
+function events(media: EventTarget) {
+ const seen: string[] = [];
+ for (const type of ["play", "playing", "pause", "ended"]) {
+ media.addEventListener(type, () => seen.push(type));
+ }
+ return seen;
+}
+
+beforeEach(() => {
+ nowMs = 0;
+ vi.useFakeTimers();
+});
+afterEach(() => vi.useRealTimers());
+
+describe("keepTimeByWallClock", () => {
+ it("advances at playbackRate times the wall clock while playing, without playing for real", async () => {
+ const media = clocked();
+ await media.play();
+ nowMs = 1000;
+ expect(media.currentTime).toBeCloseTo(1);
+ media.playbackRate = 3;
+ nowMs = 3000;
+ expect(media.currentTime).toBeCloseTo(7);
+ expect(media.paused).toBe(false);
+ expect(media.nativePlays).toBe(0);
+ });
+
+ it("holds still while paused, and resumes from there", async () => {
+ const media = clocked();
+ await media.play();
+ nowMs = 2000;
+ media.pause();
+ nowMs = 5000;
+ expect(media.currentTime).toBeCloseTo(2);
+ await media.play();
+ nowMs = 5500;
+ expect(media.currentTime).toBeCloseTo(2.5);
+ });
+
+ it("seeks the element, so seeking and seeked stay real, and runs on from the target", async () => {
+ const media = clocked();
+ await media.play();
+ nowMs = 1000;
+ media.currentTime = 10;
+ expect(media.seeks).toEqual([10]);
+ nowMs = 2000;
+ expect(media.currentTime).toBeCloseTo(11);
+ });
+
+ it("fires the events a playing element fires, queued", async () => {
+ const media = clocked();
+ const seen = events(media);
+ void media.play();
+ expect(seen).toEqual([]);
+ await Promise.resolve();
+ media.pause();
+ await Promise.resolve();
+ expect(seen).toEqual(["play", "playing", "pause"]);
+ });
+
+ it("stops at the end and says so, then starts over on play", async () => {
+ const media = clocked();
+ const seen = events(media);
+ media.currentTime = 38;
+ media.playbackRate = 2;
+ await media.play();
+ nowMs = 1000;
+ vi.advanceTimersByTime(1000);
+ await Promise.resolve();
+ expect(media.currentTime).toBe(40);
+ expect(media.paused).toBe(true);
+ expect(media.ended).toBe(true);
+ expect(seen).toEqual(["play", "playing", "pause", "ended"]);
+ await media.play();
+ expect(media.currentTime).toBe(0);
+ });
+
+ it("refuses a rate the element refuses, and keeps its own", async () => {
+ const media = clocked();
+ await media.play();
+ expect(() => {
+ media.playbackRate = 17;
+ }).toThrow();
+ nowMs = 1000;
+ expect(media.playbackRate).toBe(1);
+ expect(media.currentTime).toBeCloseTo(1);
+ });
+
+ it("stops an element that was already playing for real", () => {
+ const media = new FakeMedia();
+ void media.play();
+ keepTimeByWallClock(media as unknown as HTMLMediaElement, clock);
+ expect(media.nativePauses).toBe(1);
+ expect(media.paused).toBe(true);
+ });
+
+ it("starts over, paused at 0, when the element is reloaded", async () => {
+ const media = clocked();
+ await media.play();
+ nowMs = 3000;
+ media.dispatchEvent(new Event("emptied"));
+ expect(media.paused).toBe(true);
+ expect(media.currentTime).toBe(0);
+ });
+});
diff --git a/src/components/ai-edition/wallClock.ts b/src/components/ai-edition/wallClock.ts
new file mode 100644
index 000000000..bfa6ffcdc
--- /dev/null
+++ b/src/components/ai-edition/wallClock.ts
@@ -0,0 +1,136 @@
+/**
+ * Makes a media element keep time by the wall clock instead of by playing (issue #994).
+ *
+ * The preview's `<video>` and `<audio>` only keep time: the native compositor draws every
+ * pixel. A recording with sound keeps time on its sound track, its picture deselected
+ * (`dropVideoTrack`). A recording without sound has nothing else to keep time with, so
+ * Chromium decoded its picture on the GPU, twice (the `<video>` and the `<audio>`), at the
+ * speed region's rate, against the compositor's own decode of the same file. Measured on an
+ * RTX 4070 Ti with a hidden Electron playing a soundless 4K60 file the way the editor does,
+ * next to the compositor's free-run at 3x: the view fell 3.5 s of playback behind; with one
+ * Chromium decode left, 2.8 s; with this clock, 0.01 s, as with no Chromium at all.
+ *
+ * An element given this clock never plays for real: `play`, `pause`, `paused`, `ended`,
+ * `currentTime` and `playbackRate` are answered by a clock that runs `playbackRate` times the
+ * wall clock from where it was last placed, and it fires `play`, `playing`, `pause` and `ended`
+ * as a playing element would. A seek still reaches the element, so `seeking` and `seeked` stay
+ * real, and with no track selected it decodes nothing. Anything else is the element's own.
+ */
+
+const clocked = new WeakSet<HTMLMediaElement>();
+
+function nativeDescriptor(element: HTMLMediaElement, key: string): PropertyDescriptor {
+ for (let proto = Object.getPrototypeOf(element); proto; proto = Object.getPrototypeOf(proto)) {
+ const descriptor = Object.getOwnPropertyDescriptor(proto, key);
+ if (descriptor) return descriptor;
+ }
+ throw new Error(`media element without ${key}`);
+}
+
+export function keepTimeByWallClock(
+ element: HTMLMediaElement,
+ nowMs: () => number = () => performance.now(),
+): void {
+ if (clocked.has(element)) return;
+ clocked.add(element);
+ const time = nativeDescriptor(element, "currentTime");
+ const rateOf = nativeDescriptor(element, "playbackRate");
+ const nativePause = nativeDescriptor(element, "pause").value as () => void;
+
+ // Loaded and already playing for real (a `play()` that beat the metadata): it stops here,
+ // and says so with its own `pause` event.
+ if (!nativeDescriptor(element, "paused").get?.call(element)) {
+ nativePause.call(element);
+ }
+
+ let playing = false;
+ let rate: number = rateOf.get?.call(element);
+ let anchorSec: number = time.get?.call(element);
+ let anchorMs = nowMs();
+ let endTimer: ReturnType<typeof setTimeout> | undefined;
+
+ const position = () => {
+ const sec = playing ? anchorSec + ((nowMs() - anchorMs) / 1000) * rate : anchorSec;
+ const duration = element.duration;
+ return Math.max(0, Number.isFinite(duration) ? Math.min(sec, duration) : sec);
+ };
+ const atEnd = () => Number.isFinite(element.duration) && position() >= element.duration;
+ // Queued like the element's own events, never fired from inside the call.
+ const fire = (...types: string[]) =>
+ queueMicrotask(() => {
+ for (const type of types) element.dispatchEvent(new Event(type));
+ });
+ const anchor = (sec: number) => {
+ anchorSec = sec;
+ anchorMs = nowMs();
+ clearTimeout(endTimer);
+ const duration = element.duration;
+ if (!playing || rate <= 0 || !Number.isFinite(duration)) return;
+ endTimer = setTimeout(
+ () => {
+ if (!atEnd()) {
+ anchor(position());
+ return;
+ }
+ playing = false;
+ anchor(duration);
+ fire("pause", "ended");
+ },
+ ((duration - sec) / rate) * 1000,
+ );
+ };
+
+ Object.defineProperties(element, {
+ currentTime: {
+ configurable: true,
+ get: position,
+ set(sec: number) {
+ time.set?.call(element, sec);
+ anchor(time.get?.call(element));
+ },
+ },
+ playbackRate: {
+ configurable: true,
+ get: () => rate,
+ set(value: number) {
+ const sec = position();
+ // The element's own setter first: it is the one that refuses a rate (above 16).
+ rateOf.set?.call(element, value);
+ rate = value;
+ anchor(sec);
+ },
+ },
+ paused: { configurable: true, get: () => !playing },
+ ended: { configurable: true, get: () => !playing && atEnd() },
+ play: {
+ configurable: true,
+ value: () => {
+ if (!playing) {
+ // Like the element: playing again from the end starts over.
+ if (atEnd()) element.currentTime = 0;
+ playing = true;
+ anchor(anchorSec);
+ fire("play", "playing");
+ }
+ return Promise.resolve();
+ },
+ },
+ pause: {
+ configurable: true,
+ value: () => {
+ if (!playing) return;
+ const sec = position();
+ playing = false;
+ anchor(sec);
+ fire("pause");
+ },
+ },
+ });
+
+ // `load()` (a reload after an error) starts the element over, paused at 0.
+ element.addEventListener("emptied", () => {
+ playing = false;
+ rate = rateOf.get?.call(element);
+ anchor(0);
+ });
+}
diff --git a/technical-documentation/architecture/preview.md b/technical-documentation/architecture/preview.md
index 75be2fb7d..611b46289 100644
--- a/technical-documentation/architecture/preview.md
+++ b/technical-documentation/architecture/preview.md
@@ -42,8 +42,13 @@ once their metadata is in (`dropVideoTrack`, under the `AudioVideoTracks` Blink
editor window enables). Left alone, each decoded the whole recording a second and a third
time beside the compositor: ~6.7 % of an RTX 4070 Ti's decode engine apiece on a 1080p60
take, measured, and zero once the track is dropped, with the clock, `playbackRate` and seeks
-unchanged. A recording without sound keeps its picture: with neither track selected the
-element has nothing left to keep time with and races to its end. The `WebcamOverlay`
+unchanged. A recording without sound drops its picture too, but with neither track selected
+the element has nothing left to keep time with and jumps to its end on `play`, so it keeps
+time by the wall clock instead (`keepTimeByWallClock`, `wallClock.ts`): it never plays for
+real, and answers `play`, `pause`, `currentTime` and `playbackRate` from a clock that runs
+`playbackRate` times the wall clock. Before that, a soundless take was decoded twice by
+Chromium at the speed region's rate, against the compositor's own decode: at 4K and 3× the
+view fell 3.5 s of playback behind the playhead (#994). The `WebcamOverlay`
`<video>` is never played at all — it is read for the camera's size (`loadedmetadata`),
which shapes the PiP box in the scene.