Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(preview): count native drift in seconds of playback, and catch a frozen view
EtienneLescotb519f236ee45
diff --git a/src/components/ai-edition/NativeCompositorOverlay.test.tsx b/src/components/ai-edition/NativeCompositorOverlay.test.tsxindex 43d389e62..619bd84b3 100644--- a/src/components/ai-edition/NativeCompositorOverlay.test.tsx+++ b/src/components/ai-edition/NativeCompositorOverlay.test.tsx@@ -173,6 +173,26 @@ describe("NativeCompositorOverlay while playing", () => {expect(native.setActiveClip).toHaveBeenCalledWith(7, "/take.mp4", "", 0, 0, 2.52);});+ // At 16× a frame that takes 30 ms to arrive shows the playhead 0.48 s of programme ago.+ it("leaves alone a view keeping pace inside a speed region", async () => {+ useProjectStore.setState({+ document: {+ ...makeDocument(),+ legacyEditor: { speedRegions: [{ id: "speed_a", startMs: 0, endMs: 12000, speed: 16 }] },+ },+ currentTimeSec: 0.2,+ });+ await mountAtFirstClip();++ for (let frame = 0; frame < 8; frame++) {+ now += 30;+ publishNativePosition({ clipIndex: 0, sourceTimeSec: 0.2 + frame * 0.48 }, now);+ setPlayhead(0.2 + (frame + 1) * 0.48, true);+ }++ expect(native.setActiveClip).not.toHaveBeenCalled();+ });+it("still sends the clip on a change while paused", async () => {useProjectStore.setState({ playing: false });await mountAtFirstClip();diff --git a/src/components/ai-edition/NativeCompositorOverlay.tsx b/src/components/ai-edition/NativeCompositorOverlay.tsxindex e98e9e45b..d60bbcd20 100644--- a/src/components/ai-edition/NativeCompositorOverlay.tsx+++ b/src/components/ai-edition/NativeCompositorOverlay.tsx@@ -1,9 +1,11 @@import { useEffect, useMemo, useRef, useSyncExternalStore } from "react";import { useScopedT } from "@/contexts/I18nContext";+import { readSpeedRegions } from "@/lib/ai-edition/document/timeline";import { noteUiProbeClipSwitch } from "@/lib/ai-edition/perf/uiFrameProbe";import { getEditorSettings } from "@/lib/ai-edition/store/editorSettings";import { useProjectStore } from "@/lib/ai-edition/store/projectStore";import { assetCameraSource } from "@/lib/ai-edition/timeline/camera";+import { findActiveSpeedRegion, type SpeedRegion } from "@/lib/ai-edition/timeline/speed";import { resolveNativePosition } from "@/lib/ai-edition/timeline/timelineMap";import {pushAllNativeParams,@@ -191,6 +193,12 @@ export function NativeCompositorOverlay() {const activeClipId = activeClip?.id ?? null;const activeClipIndex = activePosition?.clipIndex ?? null;const activeSourceTimeSec = activePosition?.sourceTimeSec ?? null;+ // Matched on the raw ruler, like `currentTimeSec` and the regions' pills.+ const speedRegions = useMemo(+ () => (document ? readSpeedRegions<SpeedRegion>(document) : []),+ [document],+ );+ const playbackSpeed = findActiveSpeedRegion(speedRegions, currentTimeSec * 1000)?.speed ?? 1;const pendingTargetClipIdRef = useRef<string | null>(null);const playing = useProjectStore((s) => s.playing);@@ -243,6 +251,7 @@ export function NativeCompositorOverlay() {native,{ clipIndex: activeClipIndex, sourceTimeSec: activeSourceTimeSec },nativeClips,+ playbackSpeed,performance.now(),);// Une coupe franchie en lecture : la vue la franchit d'elle-même, au même endroit du@@ -297,6 +306,7 @@ export function NativeCompositorOverlay() {activeSourceTimeSec,playing,nativeClips,+ playbackSpeed,]);// Guet de dérive, en lecture : la vue tourne sur sa propre horloge, et n'est recalée que si@@ -322,6 +332,7 @@ export function NativeCompositorOverlay() {getNativePosition(),{ clipIndex: activeClipIndex, sourceTimeSec: activeSourceTimeSec },nativeClips,+ playbackSpeed,now,);const { watch, resync } = watchDrift(driftRef.current, lead, now);@@ -344,7 +355,16 @@ export function NativeCompositorOverlay() {).catch((error: unknown) => {console.warn("[compositor-view] re-anchoring the preview failed:", error);});- }, [viewId, document, playing, activeClip, activeClipIndex, activeSourceTimeSec, nativeClips]);+ }, [+ viewId,+ document,+ playing,+ activeClip,+ activeClipIndex,+ activeSourceTimeSec,+ nativeClips,+ playbackSpeed,+ ]);if (!ready) {return null;diff --git a/src/native/nativeSync.test.ts b/src/native/nativeSync.test.tsindex 224dd067f..3e6842e11 100644--- a/src/native/nativeSync.test.ts+++ b/src/native/nativeSync.test.ts@@ -3,6 +3,7 @@ import {getNativePosition,IDLE_DRIFT_WATCH,NATIVE_DRIFT_PERSIST_MS,+ NATIVE_FRAME_MAX_AGE_MS,NATIVE_RESYNC_COOLDOWN_MS,nativeLeadSec,programmeTimeSec,@@ -33,6 +34,7 @@ describe("nativeLeadSec", () => {{ clipIndex: 0, sourceTimeSec: 4.99, receivedAtMs: 1000 },{ clipIndex: 1, sourceTimeSec: 7.02 },SEGMENTS,+ 1,1000,);expect(lead).toBeCloseTo(-0.03, 5);@@ -43,23 +45,52 @@ describe("nativeLeadSec", () => {{ clipIndex: 1, sourceTimeSec: 8, receivedAtMs: 1000 },{ clipIndex: 1, sourceTimeSec: 8.05 },SEGMENTS,+ 1,1050,);expect(lead).toBeCloseTo(0, 5);});+ // At 16× the playhead covers 0.64 s of programme in the 40 ms a frame takes to arrive and+ // be read: in programme seconds that is a view far behind, to the eye it is in step.+ it("measures the gap in seconds of playback inside a speed region", () => {+ const lead = nativeLeadSec(+ { clipIndex: 1, sourceTimeSec: 8, receivedAtMs: 1000 },+ { clipIndex: 1, sourceTimeSec: 8.64 },+ SEGMENTS,+ 16,+ 1030,+ );+ expect(lead).toBeCloseTo(-0.01, 5);+ });+it("measures a view left behind by a stall", () => {const lead = nativeLeadSec({ clipIndex: 0, sourceTimeSec: 2, receivedAtMs: 1000 },{ clipIndex: 0, sourceTimeSec: 2.4 },SEGMENTS,+ 1,1000,);expect(lead).toBeCloseTo(-0.4, 5);});+ // Aging the last frame without bound would move a frozen view along with the playhead.+ it("catches a view that froze in step with the playhead", () => {+ const frozen = { clipIndex: 0, sourceTimeSec: 2, receivedAtMs: 1000 };+ let state = watchDrift(IDLE_DRIFT_WATCH, null, 1000);+ let resyncAtMs: number | null = null;+ for (let now = 1000; now <= 2000 && resyncAtMs === null; now += 16) {+ const playhead = { clipIndex: 0, sourceTimeSec: 2 + (now - 1000) / 1000 };+ state = watchDrift(state.watch, nativeLeadSec(frozen, playhead, SEGMENTS, 1, now), now);+ resyncAtMs = state.resync ? now : null;+ }+ expect(resyncAtMs).not.toBeNull();+ expect(resyncAtMs).toBeGreaterThan(1000 + NATIVE_FRAME_MAX_AGE_MS);+ });+it("says nothing without a reported position", () => {- expect(nativeLeadSec(null, { clipIndex: 0, sourceTimeSec: 1 }, SEGMENTS, 0)).toBeNull();+ expect(nativeLeadSec(null, { clipIndex: 0, sourceTimeSec: 1 }, SEGMENTS, 1, 0)).toBeNull();});});diff --git a/src/native/nativeSync.ts b/src/native/nativeSync.tsindex 84539572a..0db0b8bac 100644--- a/src/native/nativeSync.ts+++ b/src/native/nativeSync.ts@@ -35,6 +35,9 @@ export const NATIVE_DRIFT_TOLERANCE_SEC = 0.15;export const NATIVE_DRIFT_PERSIST_MS = 100;/** After a re-anchor the view needs a seek and a first frame before it reports the new place. */export const NATIVE_RESYNC_COOLDOWN_MS = 500;+/** A frame older than this is not late but stalled: aging it further would keep a frozen view+ * level with the playhead for good. Two frames of the CPU compositor's ~8 fps. */+export const NATIVE_FRAME_MAX_AGE_MS = 250;/** `sourceTimeSec` of segment `clipIndex`, on the programme timeline. `null` for a segment the* layout does not have (the scene and the document briefly disagree after an edit). */@@ -49,13 +52,16 @@ export function programmeTimeSec(return segment.timelineStartSec + (position.sourceTimeSec - segment.sourceStartSec);}-/** How far ahead of the playhead the view is now, in programme seconds (negative: behind).- * Its last frame is aged by the wall time since it arrived. `null` when either position- * cannot be placed. */+/** How far ahead of the playhead the view is now, in seconds of playback (negative: behind).+ * Inside a speed region the programme runs `speed` times faster than playback: a frame or+ * two of delay at 16× is a second of programme, and still only a frame or two to the eye.+ * The last frame is aged by the wall time since it arrived, up to `NATIVE_FRAME_MAX_AGE_MS`.+ * `null` when either position cannot be placed. */export function nativeLeadSec(native: ReportedNativePosition | null,app: NativePosition,segments: readonly ProgrammeSegment[],+ speed: number,nowMs: number,): number | null {if (!native) {@@ -66,7 +72,8 @@ export function nativeLeadSec(if (nativeSec === null || appSec === null) {return null;}- return nativeSec + Math.max(0, nowMs - native.receivedAtMs) / 1000 - appSec;+ const ageMs = Math.min(Math.max(0, nowMs - native.receivedAtMs), NATIVE_FRAME_MAX_AGE_MS);+ return (nativeSec - appSec) / speed + ageMs / 1000;}export interface DriftWatch {diff --git a/src/native/useNativePlaybackSync.test.ts b/src/native/useNativePlaybackSync.test.tsnew file mode 100644index 000000000..4629f2130--- /dev/null+++ b/src/native/useNativePlaybackSync.test.ts@@ -0,0 +1,85 @@+// @vitest-environment jsdom+/**+ * A jump inside the clip while playing. The overlay catches it from the position each frame+ * reports; an addon that reports none leaves it to this hook, against the wall clock at 1×.+ */+import { renderHook } from "@testing-library/react";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import type { AxcutClip } from "@/lib/ai-edition/schema";+import { publishNativePosition } from "./nativeSync";++const store = vi.hoisted(() => ({+ setNativePlaying: vi.fn(),+ setNativeTime: vi.fn(),+}));++vi.mock("./nativeCompositorStore", () => ({+ getCurrentNativeViewId: () => 7,+ setNativePlaying: store.setNativePlaying,+ setNativeTime: store.setNativeTime,+ subscribeNativeCompositor: () => () => undefined,+}));++import { useNativePlaybackSync } from "./useNativePlaybackSync";++const CLIPS: AxcutClip[] = [+ {+ id: "clip_a",+ assetId: "asset_take",+ sourceStartSec: 0,+ sourceEndSec: 10,+ timelineStartSec: 0,+ timelineEndSec: 10,+ wordRefs: [],+ origin: "system",+ reason: "",+ },+];++describe("useNativePlaybackSync while playing", () => {+ let now = 1000;++ beforeEach(() => {+ vi.clearAllMocks();+ now = 1000;+ vi.spyOn(performance, "now").mockImplementation(() => now);+ });++ afterEach(() => {+ vi.restoreAllMocks();+ publishNativePosition(null);+ });++ /** Plays from `fromSec` for a few frames, the playhead in step with the wall clock. */+ function play(fromSec: number) {+ const hook = renderHook(({ time }) => useNativePlaybackSync(true, time, CLIPS, CLIPS), {+ initialProps: { time: fromSec },+ });+ for (let frame = 1; frame <= 3; frame++) {+ now += 16;+ hook.rerender({ time: fromSec + frame * 0.016 });+ }+ return hook;+ }++ it("seeks the view on a jump inside the clip when the addon reports no position", () => {+ const hook = play(1);+ expect(store.setNativeTime).not.toHaveBeenCalled();++ now += 16;+ hook.rerender({ time: 6 });++ expect(store.setNativeTime).toHaveBeenCalledWith(6);+ });++ // The overlay watches the drift then; a guess at 1× here would re-seek inside speed regions.+ it("leaves the view to the overlay when the addon reports positions", () => {+ publishNativePosition({ clipIndex: 0, sourceTimeSec: 1 }, now);+ const hook = play(1);++ now += 16;+ hook.rerender({ time: 6 });++ expect(store.setNativeTime).not.toHaveBeenCalled();+ });+});diff --git a/src/native/useNativePlaybackSync.ts b/src/native/useNativePlaybackSync.tsindex 3ade18269..7ff2dd81f 100644--- a/src/native/useNativePlaybackSync.ts+++ b/src/native/useNativePlaybackSync.ts@@ -14,9 +14,9 @@* scrub/step interactions. Pausing also re-snaps native to the app playhead.** While playing, the two clocks are compared by `NativeCompositorOverlay`, from the- * position each native frame reports (`nativeSync.ts`), not here: this hook used to- * guess the drift from the wall clock at 1× speed, and re-seeked the view ten times a- * second inside a 2× speed region.+ * position each native frame reports (`nativeSync.ts`). This hook only guesses the drift+ * from the wall clock at 1× speed for an addon that reports no position: the guess+ * re-seeked the view ten times a second inside a 2× speed region.*/import { useEffect, useMemo, useRef, useSyncExternalStore } from "react";import type { AxcutClip } from "@/lib/ai-edition/schema";@@ -27,6 +27,7 @@ import {setNativeTime,subscribeNativeCompositor,} from "./nativeCompositorStore";+import { getNativePosition } from "./nativeSync";export function useNativePlaybackSync(playing: boolean,@@ -58,19 +59,43 @@ export function useNativePlaybackSync(setNativePlaying(playing);}, [active, playing]);- // Scrub/step while paused. A clip change is `setActiveClip`'s, in the overlay.+ // Scrub/step while paused, and drift while playing for an addon that reports no position.+ // A clip change is `setActiveClip`'s, in the overlay.+ const lastSyncedSourceTimeRef = useRef<number | null>(null);+ const lastSyncedWallTimeRef = useRef<number>(0);const lastActiveClipIdRef = useRef<string | null>(null);useEffect(() => {if (!active || sourceTimeSec === null || !activeClipId) {return;}+ const now = performance.now();if (lastActiveClipIdRef.current !== activeClipId) {lastActiveClipIdRef.current = activeClipId;+ lastSyncedSourceTimeRef.current = sourceTimeSec;+ lastSyncedWallTimeRef.current = now;return;}if (!playing) {setNativeTime(sourceTimeSec);+ lastSyncedSourceTimeRef.current = sourceTimeSec;+ lastSyncedWallTimeRef.current = now;+ return;+ }+ if (getNativePosition() !== null) {+ return;+ }+ if (lastSyncedSourceTimeRef.current === null || lastSyncedWallTimeRef.current === 0) {+ lastSyncedSourceTimeRef.current = sourceTimeSec;+ lastSyncedWallTimeRef.current = now;+ return;+ }+ const wallElapsedSec = (now - lastSyncedWallTimeRef.current) / 1000;+ const expectedSourceTimeSec = lastSyncedSourceTimeRef.current + wallElapsedSec;+ if (Math.abs(sourceTimeSec - expectedSourceTimeSec) > 0.1) {+ setNativeTime(sourceTimeSec);+ lastSyncedSourceTimeRef.current = sourceTimeSec;+ lastSyncedWallTimeRef.current = now;}}, [active, playing, activeClipId, sourceTimeSec]);}diff --git a/technical-documentation/architecture/preview.md b/technical-documentation/architecture/preview.mdindex 6df1823e1..b328307e7 100644--- a/technical-documentation/architecture/preview.md+++ b/technical-documentation/architecture/preview.md@@ -249,11 +249,14 @@ a new clip). The mapping sits incomposed it (`clipIndex`, `sourceTimeSec`, both transports), and`NativeCompositorOverlay` compares that with the playhead on the one timeline both share:programme time, the trim-compressed one, where a cut is no jump- ([`nativeSync.ts`](../../src/native/nativeSync.ts)). A gap over 150 ms that holds for- 100 ms re-anchors the view with `setActiveClip`, at most every 500 ms. That covers a stall- of the render thread and a jump by the user while playing. It replaced a guess from the- wall clock at 1× speed, which inside a 2× speed region re-seeked the view ten times a- second.+ ([`nativeSync.ts`](../../src/native/nativeSync.ts)). The gap is counted in seconds of+ playback, so a speed region divides it by its speed, and the last frame is aged by the+ time since it arrived, at most 250 ms, past which the view counts as stalled. A gap over+ 150 ms that holds for 100 ms re-anchors the view with `setActiveClip`, at most every+ 500 ms. That covers a stall of the render thread and a jump by the user while playing.+ It replaced a guess from the wall clock at 1× speed, which inside a 2× speed region+ re-seeked the view ten times a second; `useNativePlaybackSync` still makes that guess+ for an addon that reports no position.The overlay's rect is kept aligned with the DOM via the same primitives usedelsewhere in the renderer: