Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

fix(preview): count native drift in seconds of playback, and catch a frozen view

EtienneLescotb519f236ee45
diff --git a/src/components/ai-edition/NativeCompositorOverlay.test.tsx b/src/components/ai-edition/NativeCompositorOverlay.test.tsx
index 43d389e62..619bd84b3 100644
--- a/src/components/ai-edition/NativeCompositorOverlay.test.tsx
+++ b/src/components/ai-edition/NativeCompositorOverlay.test.tsx
@@ -173,6 +173,26 @@ describe("NativeCompositorOverlay while playing", () => {
expect(native.setActiveClip).toHaveBeenCalledWith(7, "/take.mp4", "", 0, 0, 2.52);
});
+ // At 16× a frame that takes 30 ms to arrive shows the playhead 0.48 s of programme ago.
+ it("leaves alone a view keeping pace inside a speed region", async () => {
+ useProjectStore.setState({
+ document: {
+ ...makeDocument(),
+ legacyEditor: { speedRegions: [{ id: "speed_a", startMs: 0, endMs: 12000, speed: 16 }] },
+ },
+ currentTimeSec: 0.2,
+ });
+ await mountAtFirstClip();
+
+ for (let frame = 0; frame < 8; frame++) {
+ now += 30;
+ publishNativePosition({ clipIndex: 0, sourceTimeSec: 0.2 + frame * 0.48 }, now);
+ setPlayhead(0.2 + (frame + 1) * 0.48, true);
+ }
+
+ expect(native.setActiveClip).not.toHaveBeenCalled();
+ });
+
it("still sends the clip on a change while paused", async () => {
useProjectStore.setState({ playing: false });
await mountAtFirstClip();
diff --git a/src/components/ai-edition/NativeCompositorOverlay.tsx b/src/components/ai-edition/NativeCompositorOverlay.tsx
index e98e9e45b..d60bbcd20 100644
--- a/src/components/ai-edition/NativeCompositorOverlay.tsx
+++ b/src/components/ai-edition/NativeCompositorOverlay.tsx
@@ -1,9 +1,11 @@
import { useEffect, useMemo, useRef, useSyncExternalStore } from "react";
import { useScopedT } from "@/contexts/I18nContext";
+import { readSpeedRegions } from "@/lib/ai-edition/document/timeline";
import { noteUiProbeClipSwitch } from "@/lib/ai-edition/perf/uiFrameProbe";
import { getEditorSettings } from "@/lib/ai-edition/store/editorSettings";
import { useProjectStore } from "@/lib/ai-edition/store/projectStore";
import { assetCameraSource } from "@/lib/ai-edition/timeline/camera";
+import { findActiveSpeedRegion, type SpeedRegion } from "@/lib/ai-edition/timeline/speed";
import { resolveNativePosition } from "@/lib/ai-edition/timeline/timelineMap";
import {
pushAllNativeParams,
@@ -191,6 +193,12 @@ export function NativeCompositorOverlay() {
const activeClipId = activeClip?.id ?? null;
const activeClipIndex = activePosition?.clipIndex ?? null;
const activeSourceTimeSec = activePosition?.sourceTimeSec ?? null;
+ // Matched on the raw ruler, like `currentTimeSec` and the regions' pills.
+ const speedRegions = useMemo(
+ () => (document ? readSpeedRegions<SpeedRegion>(document) : []),
+ [document],
+ );
+ const playbackSpeed = findActiveSpeedRegion(speedRegions, currentTimeSec * 1000)?.speed ?? 1;
const pendingTargetClipIdRef = useRef<string | null>(null);
const playing = useProjectStore((s) => s.playing);
@@ -243,6 +251,7 @@ export function NativeCompositorOverlay() {
native,
{ clipIndex: activeClipIndex, sourceTimeSec: activeSourceTimeSec },
nativeClips,
+ playbackSpeed,
performance.now(),
);
// Une coupe franchie en lecture : la vue la franchit d'elle-même, au même endroit du
@@ -297,6 +306,7 @@ export function NativeCompositorOverlay() {
activeSourceTimeSec,
playing,
nativeClips,
+ playbackSpeed,
]);
// Guet de dérive, en lecture : la vue tourne sur sa propre horloge, et n'est recalée que si
@@ -322,6 +332,7 @@ export function NativeCompositorOverlay() {
getNativePosition(),
{ clipIndex: activeClipIndex, sourceTimeSec: activeSourceTimeSec },
nativeClips,
+ playbackSpeed,
now,
);
const { watch, resync } = watchDrift(driftRef.current, lead, now);
@@ -344,7 +355,16 @@ export function NativeCompositorOverlay() {
).catch((error: unknown) => {
console.warn("[compositor-view] re-anchoring the preview failed:", error);
});
- }, [viewId, document, playing, activeClip, activeClipIndex, activeSourceTimeSec, nativeClips]);
+ }, [
+ viewId,
+ document,
+ playing,
+ activeClip,
+ activeClipIndex,
+ activeSourceTimeSec,
+ nativeClips,
+ playbackSpeed,
+ ]);
if (!ready) {
return null;
diff --git a/src/native/nativeSync.test.ts b/src/native/nativeSync.test.ts
index 224dd067f..3e6842e11 100644
--- a/src/native/nativeSync.test.ts
+++ b/src/native/nativeSync.test.ts
@@ -3,6 +3,7 @@ import {
getNativePosition,
IDLE_DRIFT_WATCH,
NATIVE_DRIFT_PERSIST_MS,
+ NATIVE_FRAME_MAX_AGE_MS,
NATIVE_RESYNC_COOLDOWN_MS,
nativeLeadSec,
programmeTimeSec,
@@ -33,6 +34,7 @@ describe("nativeLeadSec", () => {
{ clipIndex: 0, sourceTimeSec: 4.99, receivedAtMs: 1000 },
{ clipIndex: 1, sourceTimeSec: 7.02 },
SEGMENTS,
+ 1,
1000,
);
expect(lead).toBeCloseTo(-0.03, 5);
@@ -43,23 +45,52 @@ describe("nativeLeadSec", () => {
{ clipIndex: 1, sourceTimeSec: 8, receivedAtMs: 1000 },
{ clipIndex: 1, sourceTimeSec: 8.05 },
SEGMENTS,
+ 1,
1050,
);
expect(lead).toBeCloseTo(0, 5);
});
+ // At 16× the playhead covers 0.64 s of programme in the 40 ms a frame takes to arrive and
+ // be read: in programme seconds that is a view far behind, to the eye it is in step.
+ it("measures the gap in seconds of playback inside a speed region", () => {
+ const lead = nativeLeadSec(
+ { clipIndex: 1, sourceTimeSec: 8, receivedAtMs: 1000 },
+ { clipIndex: 1, sourceTimeSec: 8.64 },
+ SEGMENTS,
+ 16,
+ 1030,
+ );
+ expect(lead).toBeCloseTo(-0.01, 5);
+ });
+
it("measures a view left behind by a stall", () => {
const lead = nativeLeadSec(
{ clipIndex: 0, sourceTimeSec: 2, receivedAtMs: 1000 },
{ clipIndex: 0, sourceTimeSec: 2.4 },
SEGMENTS,
+ 1,
1000,
);
expect(lead).toBeCloseTo(-0.4, 5);
});
+ // Aging the last frame without bound would move a frozen view along with the playhead.
+ it("catches a view that froze in step with the playhead", () => {
+ const frozen = { clipIndex: 0, sourceTimeSec: 2, receivedAtMs: 1000 };
+ let state = watchDrift(IDLE_DRIFT_WATCH, null, 1000);
+ let resyncAtMs: number | null = null;
+ for (let now = 1000; now <= 2000 && resyncAtMs === null; now += 16) {
+ const playhead = { clipIndex: 0, sourceTimeSec: 2 + (now - 1000) / 1000 };
+ state = watchDrift(state.watch, nativeLeadSec(frozen, playhead, SEGMENTS, 1, now), now);
+ resyncAtMs = state.resync ? now : null;
+ }
+ expect(resyncAtMs).not.toBeNull();
+ expect(resyncAtMs).toBeGreaterThan(1000 + NATIVE_FRAME_MAX_AGE_MS);
+ });
+
it("says nothing without a reported position", () => {
- expect(nativeLeadSec(null, { clipIndex: 0, sourceTimeSec: 1 }, SEGMENTS, 0)).toBeNull();
+ expect(nativeLeadSec(null, { clipIndex: 0, sourceTimeSec: 1 }, SEGMENTS, 1, 0)).toBeNull();
});
});
diff --git a/src/native/nativeSync.ts b/src/native/nativeSync.ts
index 84539572a..0db0b8bac 100644
--- a/src/native/nativeSync.ts
+++ b/src/native/nativeSync.ts
@@ -35,6 +35,9 @@ export const NATIVE_DRIFT_TOLERANCE_SEC = 0.15;
export const NATIVE_DRIFT_PERSIST_MS = 100;
/** After a re-anchor the view needs a seek and a first frame before it reports the new place. */
export const NATIVE_RESYNC_COOLDOWN_MS = 500;
+/** A frame older than this is not late but stalled: aging it further would keep a frozen view
+ * level with the playhead for good. Two frames of the CPU compositor's ~8 fps. */
+export const NATIVE_FRAME_MAX_AGE_MS = 250;
/** `sourceTimeSec` of segment `clipIndex`, on the programme timeline. `null` for a segment the
* layout does not have (the scene and the document briefly disagree after an edit). */
@@ -49,13 +52,16 @@ export function programmeTimeSec(
return segment.timelineStartSec + (position.sourceTimeSec - segment.sourceStartSec);
}
-/** How far ahead of the playhead the view is now, in programme seconds (negative: behind).
- * Its last frame is aged by the wall time since it arrived. `null` when either position
- * cannot be placed. */
+/** How far ahead of the playhead the view is now, in seconds of playback (negative: behind).
+ * Inside a speed region the programme runs `speed` times faster than playback: a frame or
+ * two of delay at 16× is a second of programme, and still only a frame or two to the eye.
+ * The last frame is aged by the wall time since it arrived, up to `NATIVE_FRAME_MAX_AGE_MS`.
+ * `null` when either position cannot be placed. */
export function nativeLeadSec(
native: ReportedNativePosition | null,
app: NativePosition,
segments: readonly ProgrammeSegment[],
+ speed: number,
nowMs: number,
): number | null {
if (!native) {
@@ -66,7 +72,8 @@ export function nativeLeadSec(
if (nativeSec === null || appSec === null) {
return null;
}
- return nativeSec + Math.max(0, nowMs - native.receivedAtMs) / 1000 - appSec;
+ const ageMs = Math.min(Math.max(0, nowMs - native.receivedAtMs), NATIVE_FRAME_MAX_AGE_MS);
+ return (nativeSec - appSec) / speed + ageMs / 1000;
}
export interface DriftWatch {
diff --git a/src/native/useNativePlaybackSync.test.ts b/src/native/useNativePlaybackSync.test.ts
new file mode 100644
index 000000000..4629f2130
--- /dev/null
+++ b/src/native/useNativePlaybackSync.test.ts
@@ -0,0 +1,85 @@
+// @vitest-environment jsdom
+/**
+ * A jump inside the clip while playing. The overlay catches it from the position each frame
+ * reports; an addon that reports none leaves it to this hook, against the wall clock at 1×.
+ */
+import { renderHook } from "@testing-library/react";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import type { AxcutClip } from "@/lib/ai-edition/schema";
+import { publishNativePosition } from "./nativeSync";
+
+const store = vi.hoisted(() => ({
+ setNativePlaying: vi.fn(),
+ setNativeTime: vi.fn(),
+}));
+
+vi.mock("./nativeCompositorStore", () => ({
+ getCurrentNativeViewId: () => 7,
+ setNativePlaying: store.setNativePlaying,
+ setNativeTime: store.setNativeTime,
+ subscribeNativeCompositor: () => () => undefined,
+}));
+
+import { useNativePlaybackSync } from "./useNativePlaybackSync";
+
+const CLIPS: AxcutClip[] = [
+ {
+ id: "clip_a",
+ assetId: "asset_take",
+ sourceStartSec: 0,
+ sourceEndSec: 10,
+ timelineStartSec: 0,
+ timelineEndSec: 10,
+ wordRefs: [],
+ origin: "system",
+ reason: "",
+ },
+];
+
+describe("useNativePlaybackSync while playing", () => {
+ let now = 1000;
+
+ beforeEach(() => {
+ vi.clearAllMocks();
+ now = 1000;
+ vi.spyOn(performance, "now").mockImplementation(() => now);
+ });
+
+ afterEach(() => {
+ vi.restoreAllMocks();
+ publishNativePosition(null);
+ });
+
+ /** Plays from `fromSec` for a few frames, the playhead in step with the wall clock. */
+ function play(fromSec: number) {
+ const hook = renderHook(({ time }) => useNativePlaybackSync(true, time, CLIPS, CLIPS), {
+ initialProps: { time: fromSec },
+ });
+ for (let frame = 1; frame <= 3; frame++) {
+ now += 16;
+ hook.rerender({ time: fromSec + frame * 0.016 });
+ }
+ return hook;
+ }
+
+ it("seeks the view on a jump inside the clip when the addon reports no position", () => {
+ const hook = play(1);
+ expect(store.setNativeTime).not.toHaveBeenCalled();
+
+ now += 16;
+ hook.rerender({ time: 6 });
+
+ expect(store.setNativeTime).toHaveBeenCalledWith(6);
+ });
+
+ // The overlay watches the drift then; a guess at 1× here would re-seek inside speed regions.
+ it("leaves the view to the overlay when the addon reports positions", () => {
+ publishNativePosition({ clipIndex: 0, sourceTimeSec: 1 }, now);
+ const hook = play(1);
+
+ now += 16;
+ hook.rerender({ time: 6 });
+
+ expect(store.setNativeTime).not.toHaveBeenCalled();
+ });
+});
diff --git a/src/native/useNativePlaybackSync.ts b/src/native/useNativePlaybackSync.ts
index 3ade18269..7ff2dd81f 100644
--- a/src/native/useNativePlaybackSync.ts
+++ b/src/native/useNativePlaybackSync.ts
@@ -14,9 +14,9 @@
* scrub/step interactions. Pausing also re-snaps native to the app playhead.
*
* While playing, the two clocks are compared by `NativeCompositorOverlay`, from the
- * position each native frame reports (`nativeSync.ts`), not here: this hook used to
- * guess the drift from the wall clock at 1× speed, and re-seeked the view ten times a
- * second inside a 2× speed region.
+ * position each native frame reports (`nativeSync.ts`). This hook only guesses the drift
+ * from the wall clock at 1× speed for an addon that reports no position: the guess
+ * re-seeked the view ten times a second inside a 2× speed region.
*/
import { useEffect, useMemo, useRef, useSyncExternalStore } from "react";
import type { AxcutClip } from "@/lib/ai-edition/schema";
@@ -27,6 +27,7 @@ import {
setNativeTime,
subscribeNativeCompositor,
} from "./nativeCompositorStore";
+import { getNativePosition } from "./nativeSync";
export function useNativePlaybackSync(
playing: boolean,
@@ -58,19 +59,43 @@ export function useNativePlaybackSync(
setNativePlaying(playing);
}, [active, playing]);
- // Scrub/step while paused. A clip change is `setActiveClip`'s, in the overlay.
+ // Scrub/step while paused, and drift while playing for an addon that reports no position.
+ // A clip change is `setActiveClip`'s, in the overlay.
+ const lastSyncedSourceTimeRef = useRef<number | null>(null);
+ const lastSyncedWallTimeRef = useRef<number>(0);
const lastActiveClipIdRef = useRef<string | null>(null);
useEffect(() => {
if (!active || sourceTimeSec === null || !activeClipId) {
return;
}
+ const now = performance.now();
if (lastActiveClipIdRef.current !== activeClipId) {
lastActiveClipIdRef.current = activeClipId;
+ lastSyncedSourceTimeRef.current = sourceTimeSec;
+ lastSyncedWallTimeRef.current = now;
return;
}
if (!playing) {
setNativeTime(sourceTimeSec);
+ lastSyncedSourceTimeRef.current = sourceTimeSec;
+ lastSyncedWallTimeRef.current = now;
+ return;
+ }
+ if (getNativePosition() !== null) {
+ return;
+ }
+ if (lastSyncedSourceTimeRef.current === null || lastSyncedWallTimeRef.current === 0) {
+ lastSyncedSourceTimeRef.current = sourceTimeSec;
+ lastSyncedWallTimeRef.current = now;
+ return;
+ }
+ const wallElapsedSec = (now - lastSyncedWallTimeRef.current) / 1000;
+ const expectedSourceTimeSec = lastSyncedSourceTimeRef.current + wallElapsedSec;
+ if (Math.abs(sourceTimeSec - expectedSourceTimeSec) > 0.1) {
+ setNativeTime(sourceTimeSec);
+ lastSyncedSourceTimeRef.current = sourceTimeSec;
+ lastSyncedWallTimeRef.current = now;
}
}, [active, playing, activeClipId, sourceTimeSec]);
}
diff --git a/technical-documentation/architecture/preview.md b/technical-documentation/architecture/preview.md
index 6df1823e1..b328307e7 100644
--- a/technical-documentation/architecture/preview.md
+++ b/technical-documentation/architecture/preview.md
@@ -249,11 +249,14 @@ a new clip). The mapping sits in
composed it (`clipIndex`, `sourceTimeSec`, both transports), and
`NativeCompositorOverlay` compares that with the playhead on the one timeline both share:
programme time, the trim-compressed one, where a cut is no jump
- ([`nativeSync.ts`](../../src/native/nativeSync.ts)). A gap over 150 ms that holds for
- 100 ms re-anchors the view with `setActiveClip`, at most every 500 ms. That covers a stall
- of the render thread and a jump by the user while playing. It replaced a guess from the
- wall clock at 1× speed, which inside a 2× speed region re-seeked the view ten times a
- second.
+ ([`nativeSync.ts`](../../src/native/nativeSync.ts)). The gap is counted in seconds of
+ playback, so a speed region divides it by its speed, and the last frame is aged by the
+ time since it arrived, at most 250 ms, past which the view counts as stalled. A gap over
+ 150 ms that holds for 100 ms re-anchors the view with `setActiveClip`, at most every
+ 500 ms. That covers a stall of the render thread and a jump by the user while playing.
+ It replaced a guess from the wall clock at 1× speed, which inside a 2× speed region
+ re-seeked the view ten times a second; `useNativePlaybackSync` still makes that guess
+ for an addon that reports no position.
The overlay's rect is kept aligned with the DOM via the same primitives used
elsewhere in the renderer: