Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(preview): stop Chromium decoding a soundless recording's picture
EtienneLescotc593090c71da
diff --git a/src/components/ai-edition/VirtualPreview.tracks.test.ts b/src/components/ai-edition/VirtualPreview.tracks.test.tsindex 731861265..a1634a8be 100644--- a/src/components/ai-edition/VirtualPreview.tracks.test.ts+++ b/src/components/ai-edition/VirtualPreview.tracks.test.ts@@ -24,12 +24,45 @@ describe("dropVideoTrack", () => {});// With neither track selected an element has no stream left to keep time with: measured,- // its clock races to the end instead of playing.- it("leaves a recording without sound decoding, so its clock still runs", () => {- const { element: media, videoTracks } = element({ audio: 0, video: 1 });+ // it jumps to its end on play. So it keeps time by the wall clock (issue #994).+ it("deselects the picture of a recording without sound too, and keeps time by the wall clock", () => {+ // Accessors on the prototype, where the element's own live.+ class Media extends EventTarget {+ duration = 10;+ time = 0;+ rate = 1;+ get currentTime() {+ return this.time;+ }+ set currentTime(sec: number) {+ this.time = sec;+ }+ get playbackRate() {+ return this.rate;+ }+ set playbackRate(rate: number) {+ this.rate = rate;+ }+ get paused() {+ return true;+ }+ play(): Promise<void> {+ throw new Error("played for real");+ }+ pause(): void {+ throw new Error("never playing for real, so never paused for real");+ }+ }+ const videoTracks = [{ selected: true }];+ const media = Object.assign(new Media(), {+ audioTracks: { length: 0 },+ videoTracks,+ }) as unknown as HTMLMediaElement;- expect(dropVideoTrack(media)).toBe(false);- expect(videoTracks[0].selected).toBe(true);+ expect(dropVideoTrack(media)).toBe(true);+ expect(videoTracks[0].selected).toBe(false);+ void media.play();+ expect(media.paused).toBe(false);});it("does nothing where the track lists are absent (the Blink feature is off)", () => {diff --git a/src/components/ai-edition/VirtualPreview.tsx b/src/components/ai-edition/VirtualPreview.tsxindex beda87868..c24d24191 100644--- a/src/components/ai-edition/VirtualPreview.tsx+++ b/src/components/ai-edition/VirtualPreview.tsx@@ -51,6 +51,7 @@ import {retryDelayMs,} from "./mediaError";import styles from "./VirtualPreview.module.css";+import { keepTimeByWallClock } from "./wallClock";export interface VideoSource {id: string;@@ -193,15 +194,18 @@ interface MediaTrackLists {* deselected the element decodes no frame, and its clock, `playbackRate` and seeks behave* the same.*- * Only when the element has a sound track: with neither track selected it has no stream- * left to keep time with, and races to its end. A recording without sound keeps decoding,- * as before. Returns whether the track was dropped.+ * Without a sound track the element has no stream left to keep time with once its picture+ * is deselected (measured: it jumps to its end on `play`), so it keeps time by the wall clock+ * instead (`keepTimeByWallClock`, issue #994). Returns whether the track was dropped.*/export function dropVideoTrack(element: HTMLMediaElement): boolean {const { audioTracks, videoTracks } = element as HTMLMediaElement & MediaTrackLists;- if (!videoTracks?.length || !audioTracks?.length) {+ if (!videoTracks?.length || !audioTracks) {return false;}+ if (audioTracks.length === 0) {+ keepTimeByWallClock(element);+ }for (let index = 0; index < videoTracks.length; index++) {videoTracks[index].selected = false;}diff --git a/src/components/ai-edition/wallClock.test.ts b/src/components/ai-edition/wallClock.test.tsnew file mode 100644index 000000000..e1e7cd1d0--- /dev/null+++ b/src/components/ai-edition/wallClock.test.ts@@ -0,0 +1,154 @@+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import { keepTimeByWallClock } from "./wallClock";++/** The slice of HTMLMediaElement the clock touches, playing for real when asked to. */+class FakeMedia extends EventTarget {+ duration = 40;+ nativePlays = 0;+ nativePauses = 0;+ seeks: number[] = [];+ #time = 0;+ #rate = 1;+ #paused = true;+ get currentTime() {+ return this.#time;+ }+ set currentTime(sec: number) {+ this.seeks.push(sec);+ this.#time = Math.min(sec, this.duration);+ }+ get playbackRate() {+ return this.#rate;+ }+ set playbackRate(value: number) {+ if (value > 16) throw new DOMException("rate", "NotSupportedError");+ this.#rate = value;+ }+ get paused() {+ return this.#paused;+ }+ play() {+ this.nativePlays++;+ this.#paused = false;+ return Promise.resolve();+ }+ pause() {+ this.nativePauses++;+ this.#paused = true;+ }+}++let nowMs = 0;+const clock = () => nowMs;++function clocked() {+ const media = new FakeMedia();+ keepTimeByWallClock(media as unknown as HTMLMediaElement, clock);+ return media as unknown as HTMLMediaElement & FakeMedia;+}++function events(media: EventTarget) {+ const seen: string[] = [];+ for (const type of ["play", "playing", "pause", "ended"]) {+ media.addEventListener(type, () => seen.push(type));+ }+ return seen;+}++beforeEach(() => {+ nowMs = 0;+ vi.useFakeTimers();+});+afterEach(() => vi.useRealTimers());++describe("keepTimeByWallClock", () => {+ it("advances at playbackRate times the wall clock while playing, without playing for real", async () => {+ const media = clocked();+ await media.play();+ nowMs = 1000;+ expect(media.currentTime).toBeCloseTo(1);+ media.playbackRate = 3;+ nowMs = 3000;+ expect(media.currentTime).toBeCloseTo(7);+ expect(media.paused).toBe(false);+ expect(media.nativePlays).toBe(0);+ });++ it("holds still while paused, and resumes from there", async () => {+ const media = clocked();+ await media.play();+ nowMs = 2000;+ media.pause();+ nowMs = 5000;+ expect(media.currentTime).toBeCloseTo(2);+ await media.play();+ nowMs = 5500;+ expect(media.currentTime).toBeCloseTo(2.5);+ });++ it("seeks the element, so seeking and seeked stay real, and runs on from the target", async () => {+ const media = clocked();+ await media.play();+ nowMs = 1000;+ media.currentTime = 10;+ expect(media.seeks).toEqual([10]);+ nowMs = 2000;+ expect(media.currentTime).toBeCloseTo(11);+ });++ it("fires the events a playing element fires, queued", async () => {+ const media = clocked();+ const seen = events(media);+ void media.play();+ expect(seen).toEqual([]);+ await Promise.resolve();+ media.pause();+ await Promise.resolve();+ expect(seen).toEqual(["play", "playing", "pause"]);+ });++ it("stops at the end and says so, then starts over on play", async () => {+ const media = clocked();+ const seen = events(media);+ media.currentTime = 38;+ media.playbackRate = 2;+ await media.play();+ nowMs = 1000;+ vi.advanceTimersByTime(1000);+ await Promise.resolve();+ expect(media.currentTime).toBe(40);+ expect(media.paused).toBe(true);+ expect(media.ended).toBe(true);+ expect(seen).toEqual(["play", "playing", "pause", "ended"]);+ await media.play();+ expect(media.currentTime).toBe(0);+ });++ it("refuses a rate the element refuses, and keeps its own", async () => {+ const media = clocked();+ await media.play();+ expect(() => {+ media.playbackRate = 17;+ }).toThrow();+ nowMs = 1000;+ expect(media.playbackRate).toBe(1);+ expect(media.currentTime).toBeCloseTo(1);+ });++ it("stops an element that was already playing for real", () => {+ const media = new FakeMedia();+ void media.play();+ keepTimeByWallClock(media as unknown as HTMLMediaElement, clock);+ expect(media.nativePauses).toBe(1);+ expect(media.paused).toBe(true);+ });++ it("starts over, paused at 0, when the element is reloaded", async () => {+ const media = clocked();+ await media.play();+ nowMs = 3000;+ media.dispatchEvent(new Event("emptied"));+ expect(media.paused).toBe(true);+ expect(media.currentTime).toBe(0);+ });+});diff --git a/src/components/ai-edition/wallClock.ts b/src/components/ai-edition/wallClock.tsnew file mode 100644index 000000000..bfa6ffcdc--- /dev/null+++ b/src/components/ai-edition/wallClock.ts@@ -0,0 +1,136 @@+/**+ * Makes a media element keep time by the wall clock instead of by playing (issue #994).+ *+ * The preview's `<video>` and `<audio>` only keep time: the native compositor draws every+ * pixel. A recording with sound keeps time on its sound track, its picture deselected+ * (`dropVideoTrack`). A recording without sound has nothing else to keep time with, so+ * Chromium decoded its picture on the GPU, twice (the `<video>` and the `<audio>`), at the+ * speed region's rate, against the compositor's own decode of the same file. Measured on an+ * RTX 4070 Ti with a hidden Electron playing a soundless 4K60 file the way the editor does,+ * next to the compositor's free-run at 3x: the view fell 3.5 s of playback behind; with one+ * Chromium decode left, 2.8 s; with this clock, 0.01 s, as with no Chromium at all.+ *+ * An element given this clock never plays for real: `play`, `pause`, `paused`, `ended`,+ * `currentTime` and `playbackRate` are answered by a clock that runs `playbackRate` times the+ * wall clock from where it was last placed, and it fires `play`, `playing`, `pause` and `ended`+ * as a playing element would. A seek still reaches the element, so `seeking` and `seeked` stay+ * real, and with no track selected it decodes nothing. Anything else is the element's own.+ */++const clocked = new WeakSet<HTMLMediaElement>();++function nativeDescriptor(element: HTMLMediaElement, key: string): PropertyDescriptor {+ for (let proto = Object.getPrototypeOf(element); proto; proto = Object.getPrototypeOf(proto)) {+ const descriptor = Object.getOwnPropertyDescriptor(proto, key);+ if (descriptor) return descriptor;+ }+ throw new Error(`media element without ${key}`);+}++export function keepTimeByWallClock(+ element: HTMLMediaElement,+ nowMs: () => number = () => performance.now(),+): void {+ if (clocked.has(element)) return;+ clocked.add(element);+ const time = nativeDescriptor(element, "currentTime");+ const rateOf = nativeDescriptor(element, "playbackRate");+ const nativePause = nativeDescriptor(element, "pause").value as () => void;++ // Loaded and already playing for real (a `play()` that beat the metadata): it stops here,+ // and says so with its own `pause` event.+ if (!nativeDescriptor(element, "paused").get?.call(element)) {+ nativePause.call(element);+ }++ let playing = false;+ let rate: number = rateOf.get?.call(element);+ let anchorSec: number = time.get?.call(element);+ let anchorMs = nowMs();+ let endTimer: ReturnType<typeof setTimeout> | undefined;++ const position = () => {+ const sec = playing ? anchorSec + ((nowMs() - anchorMs) / 1000) * rate : anchorSec;+ const duration = element.duration;+ return Math.max(0, Number.isFinite(duration) ? Math.min(sec, duration) : sec);+ };+ const atEnd = () => Number.isFinite(element.duration) && position() >= element.duration;+ // Queued like the element's own events, never fired from inside the call.+ const fire = (...types: string[]) =>+ queueMicrotask(() => {+ for (const type of types) element.dispatchEvent(new Event(type));+ });+ const anchor = (sec: number) => {+ anchorSec = sec;+ anchorMs = nowMs();+ clearTimeout(endTimer);+ const duration = element.duration;+ if (!playing || rate <= 0 || !Number.isFinite(duration)) return;+ endTimer = setTimeout(+ () => {+ if (!atEnd()) {+ anchor(position());+ return;+ }+ playing = false;+ anchor(duration);+ fire("pause", "ended");+ },+ ((duration - sec) / rate) * 1000,+ );+ };++ Object.defineProperties(element, {+ currentTime: {+ configurable: true,+ get: position,+ set(sec: number) {+ time.set?.call(element, sec);+ anchor(time.get?.call(element));+ },+ },+ playbackRate: {+ configurable: true,+ get: () => rate,+ set(value: number) {+ const sec = position();+ // The element's own setter first: it is the one that refuses a rate (above 16).+ rateOf.set?.call(element, value);+ rate = value;+ anchor(sec);+ },+ },+ paused: { configurable: true, get: () => !playing },+ ended: { configurable: true, get: () => !playing && atEnd() },+ play: {+ configurable: true,+ value: () => {+ if (!playing) {+ // Like the element: playing again from the end starts over.+ if (atEnd()) element.currentTime = 0;+ playing = true;+ anchor(anchorSec);+ fire("play", "playing");+ }+ return Promise.resolve();+ },+ },+ pause: {+ configurable: true,+ value: () => {+ if (!playing) return;+ const sec = position();+ playing = false;+ anchor(sec);+ fire("pause");+ },+ },+ });++ // `load()` (a reload after an error) starts the element over, paused at 0.+ element.addEventListener("emptied", () => {+ playing = false;+ rate = rateOf.get?.call(element);+ anchor(0);+ });+}diff --git a/technical-documentation/architecture/preview.md b/technical-documentation/architecture/preview.mdindex 75be2fb7d..611b46289 100644--- a/technical-documentation/architecture/preview.md+++ b/technical-documentation/architecture/preview.md@@ -42,8 +42,13 @@ once their metadata is in (`dropVideoTrack`, under the `AudioVideoTracks` Blinkeditor window enables). Left alone, each decoded the whole recording a second and a thirdtime beside the compositor: ~6.7 % of an RTX 4070 Ti's decode engine apiece on a 1080p60take, measured, and zero once the track is dropped, with the clock, `playbackRate` and seeks-unchanged. A recording without sound keeps its picture: with neither track selected the-element has nothing left to keep time with and races to its end. The `WebcamOverlay`+unchanged. A recording without sound drops its picture too, but with neither track selected+the element has nothing left to keep time with and jumps to its end on `play`, so it keeps+time by the wall clock instead (`keepTimeByWallClock`, `wallClock.ts`): it never plays for+real, and answers `play`, `pause`, `currentTime` and `playbackRate` from a clock that runs+`playbackRate` times the wall clock. Before that, a soundless take was decoded twice by+Chromium at the speed region's rate, against the compositor's own decode: at 4K and 3× the+view fell 3.5 s of playback behind the playhead (#994). The `WebcamOverlay``<video>` is never played at all — it is read for the camera's size (`loadedmetadata`),which shapes the PiP box in the scene.