Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Improve playback for canonical layered document previews

[._.]/0ea9b0cb2505
diff --git a/client/src/components/Drawer.browser.test.js b/client/src/components/Drawer.browser.test.js
index 0549094946..0e17a60661 100644
--- a/client/src/components/Drawer.browser.test.js
+++ b/client/src/components/Drawer.browser.test.js
@@ -167,6 +167,8 @@ describe.skipIf(!chrome && !auditCdp)('modal native keyboard boundaries', () =>
expect(await summary.evaluate(el => el === document.activeElement)).toBe(true);
await page.keyboard.press('Space');
expect(await page.locator('details').evaluate(el => el.open)).toBe(true);
+ // Native toggle delivery and React's transcript commit follow details.open.
+ await page.getByText('Synthetic accessible reading passage.', { exact: true }).waitFor({ state: 'visible' });
expect(await page.getByText('Synthetic accessible reading passage.', { exact: true }).isVisible()).toBe(true);
expect(await page.getByRole('button', { name: 'Play', exact: true }).count()).toBe(1);
await page.keyboard.press('Escape');
diff --git a/client/src/components/musicVideo/CompositionPreviewPlayer.jsx b/client/src/components/musicVideo/CompositionPreviewPlayer.jsx
index b65bd24f5f..ff7a6d0115 100644
--- a/client/src/components/musicVideo/CompositionPreviewPlayer.jsx
+++ b/client/src/components/musicVideo/CompositionPreviewPlayer.jsx
@@ -1,6 +1,7 @@
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { Pause, Play } from 'lucide-react';
import { normalizeTitleCard } from '../../../../server/lib/musicVideoTitleCard.js';
+import useMounted from '../../hooks/useMounted.js';
import useGeneratedPreview from '../../hooks/useGeneratedPreview.js';
import ShotStepper, { shotSteps } from './ShotStepper.jsx';
import FrameStepper from './FrameStepper.jsx';
@@ -54,7 +55,16 @@ export default function CompositionPreviewPlayer({ project, audioUrl, seekReques
const [t, setT] = useState(0);
const [playing, setPlaying] = useState(false);
const audioRef = useRef(null);
- const wasPlaying = useRef(false);
+ const mounted = useMounted();
+ const intent = useRef(0);
+ const wantsPlay = useRef(false);
+ const nativePlaying = useRef(false);
+ const playbackSession = useRef(null);
+ const audioStartWait = useRef(null);
+ const playFromRef = useRef(null);
+ const resyncs = useRef([]);
+ const [preparing, setPreparing] = useState(false);
+ const [audioError, setAudioError] = useState('');
const appliedSeek = useRef(null);
const filmLook = lyrics ? null : (project.filmLook ?? null);
const titleCard = useMemo(() => normalizeTitleCard(project.titleCard), [project.titleCard]);
@@ -70,48 +80,193 @@ export default function CompositionPreviewPlayer({ project, audioUrl, seekReques
const duration = preview.durationSec || 0;
const fps = preview.fps || 24;
+ const cancelAudioStart = useCallback(() => {
+ const waiting = audioStartWait.current;
+ if (!waiting) return;
+ audioStartWait.current = null;
+ cancelAnimationFrame(waiting.frame);
+ clearTimeout(waiting.timer);
+ }, []);
+ const stopPlayback = useCallback((time = audioRef.current?.currentTime || 0) => {
+ const next = Math.min(Math.max(0, time), Math.max(0, duration - 1 / fps));
+ cancelAudioStart();
+ intent.current += 1;
+ wantsPlay.current = false;
+ nativePlaying.current = false;
+ audioRef.current?.pause?.();
+ preview.pausePlayback(next);
+ if (audioRef.current && audioRef.current.currentTime !== next) audioRef.current.currentTime = next;
+ setPreparing(false);
+ setPlaying(false);
+ setT(next);
+ }, [duration, fps, preview.pausePlayback, cancelAudioStart]);
+
+ const playFrom = useCallback((time) => {
+ const audio = audioRef.current;
+ if (!audio || !preview.ready) return;
+ cancelAudioStart();
+ const generation = ++intent.current;
+ const live = () => mounted.current && generation === intent.current && wantsPlay.current;
+ wantsPlay.current = true;
+ setAudioError('');
+ const failed = (error) => {
+ if (!live()) return;
+ stopPlayback(audio.currentTime || time);
+ setAudioError(error?.message || 'Audio playback could not start. Retry the preview.');
+ };
+ const startAudio = (native) => {
+ if (!live()) return;
+ const clockWindow = [];
+ const waiting = native ? { frame: 0, timer: 0 } : null;
+ if (waiting) {
+ audioStartWait.current = waiting;
+ waiting.timer = setTimeout(() => {
+ if (audioStartWait.current !== waiting) return;
+ cancelAudioStart();
+ failed(new Error('The audio clock did not start. Retry the preview.'));
+ }, 3000);
+ }
+ const started = audio.play?.();
+ if (!native) { setPreparing(false); setPlaying(true); }
+ Promise.resolve(started).then(() => {
+ if (!live()) return;
+ if (!native) { setPlaying(true); return; }
+ // play() can resolve while the media clock is still held at startup.
+ // Anchor only after its latest 100ms window advances steadily at 1x.
+ if (audioStartWait.current !== waiting) return;
+ const observe = () => {
+ if (audioStartWait.current !== waiting) return;
+ if (!live()) { cancelAudioStart(); return; }
+ const actualTime = audio.currentTime;
+ const observedAt = performance.now();
+ const previous = clockWindow.at(-1);
+ if (audio.paused || audio.seeking || !Number.isFinite(actualTime) || (previous && actualTime < previous.t)) {
+ clockWindow.length = 0;
+ } else {
+ clockWindow.push({ t: actualTime, at: observedAt });
+ // Retain the newest window that still spans at least 100ms.
+ while (clockWindow.length > 1 && observedAt - clockWindow[1].at >= 100) clockWindow.shift();
+ const spanMs = observedAt - clockWindow[0].at;
+ if (spanMs >= 100) {
+ let advances = 0;
+ for (let i = 1; i < clockWindow.length; i += 1) {
+ if (clockWindow[i].t > clockWindow[i - 1].t) advances += 1;
+ }
+ const slope = (actualTime - clockWindow[0].t) / (spanMs / 1000);
+ if (advances >= 2 && slope >= 0.95 && slope <= 1.05) {
+ cancelAudioStart();
+ nativePlaying.current = true;
+ preview.startPlayback(actualTime);
+ setPreparing(false);
+ setPlaying(true);
+ return;
+ }
+ }
+ }
+ waiting.frame = requestAnimationFrame(observe);
+ };
+ observe();
+ }).catch(failed);
+ };
+ if (preview.playback) {
+ audio.pause?.();
+ nativePlaying.current = false;
+ setPlaying(false);
+ setPreparing(true);
+ preview.preparePlayback(time).then((native) => { if (live() && native !== null) startAudio(native); }).catch(failed);
+ } else startAudio(false);
+ }, [mounted, preview.ready, preview.playback, preview.preparePlayback, preview.startPlayback, stopPlayback, cancelAudioStart]);
+ playFromRef.current = playFrom;
+
const seek = useCallback((time) => {
const next = Math.min(Math.max(0, time), Math.max(0, duration - 1 / fps));
+ const restart = preview.playback && wantsPlay.current;
+ if (restart) {
+ intent.current += 1;
+ audioRef.current?.pause?.();
+ nativePlaying.current = false;
+ setPlaying(false);
+ }
setT(next);
preview.seek(next);
- // Even a one-frame move must update audio before the pause repaint reads it.
if (audioRef.current && audioRef.current.currentTime !== next) audioRef.current.currentTime = next;
- }, [duration, fps, preview.seek]);
+ if (restart) playFromRef.current?.(next);
+ }, [duration, fps, preview.seek, preview.playback]);
+
+ useEffect(() => {
+ if (preview.ready && playbackSession.current === preview.sessionId) return;
+ playbackSession.current = preview.sessionId;
+ cancelAudioStart();
+ intent.current += 1;
+ wantsPlay.current = false;
+ nativePlaying.current = false;
+ audioRef.current?.pause?.();
+ setPreparing(false);
+ setPlaying(false);
+ }, [preview.ready, preview.sessionId, cancelAudioStart]);
useEffect(() => {
if (!seekRequest || duration <= 0 || appliedSeek.current === seekRequest.n) return;
appliedSeek.current = seekRequest.n;
- seek(seekRequest.t);
- if (seekRequest.play && audioRef.current) { audioRef.current.play?.()?.catch?.(() => {}); setPlaying(true); }
- else { audioRef.current?.pause?.(); setPlaying(false); }
- }, [seekRequest, duration, seek]);
+ stopPlayback(seekRequest.t);
+ if (seekRequest.play) playFromRef.current?.(audioRef.current?.currentTime || 0);
+ }, [seekRequest, duration, stopPlayback]);
useEffect(() => {
if (!playing) return undefined;
let handle = 0;
let posted = -1;
+ let anchored = performance.now();
const tick = () => {
+ if (!wantsPlay.current) return;
const time = audioRef.current?.currentTime || 0;
const frameIndex = Math.floor(time * fps + 1e-9);
- if (frameIndex !== posted) { posted = frameIndex; setT(time); preview.seek(time, { playing: true }); }
+ if (frameIndex !== posted) {
+ posted = frameIndex;
+ setT(time);
+ if (!nativePlaying.current) preview.seek(time, { playing: true });
+ }
+ if (nativePlaying.current && performance.now() - anchored >= 250) {
+ anchored = performance.now();
+ preview.syncPlayback(time);
+ }
handle = requestAnimationFrame(tick);
};
handle = requestAnimationFrame(tick);
return () => cancelAnimationFrame(handle);
- }, [playing, fps, preview.seek]);
+ }, [playing, fps, preview.seek, preview.syncPlayback]);
+
- // Pause/end repaints the current frame with the exact render filter.
useEffect(() => {
- if (wasPlaying.current && !playing && preview.ready) seek(audioRef.current?.currentTime || 0);
- wasPlaying.current = playing;
- }, [playing, preview.ready, seek]);
+ if (!preview.playbackResync || !wantsPlay.current) return;
+ if (preview.playbackResyncReason !== 'controls') {
+ const now = performance.now();
+ resyncs.current = resyncs.current.filter((at) => now - at < 10000);
+ if (resyncs.current.length >= 3) {
+ stopPlayback();
+ setAudioError('Native preview repeatedly lost synchronization. Retry the preview.');
+ return;
+ }
+ resyncs.current.push(now);
+ }
+ playFromRef.current?.(audioRef.current?.currentTime || 0);
+ }, [preview.playbackResync, preview.playbackResyncReason, stopPlayback]);
+
+ useEffect(() => {
+ const onVisibility = () => { if (document.visibilityState === 'hidden' && wantsPlay.current) stopPlayback(); };
+ document.addEventListener('visibilitychange', onVisibility);
+ return () => {
+ document.removeEventListener('visibilitychange', onVisibility);
+ cancelAudioStart();
+ intent.current += 1;
+ wantsPlay.current = false;
+ audioRef.current?.pause?.();
+ };
+ }, [stopPlayback, cancelAudioStart]);
const togglePlay = () => {
- const audio = audioRef.current;
- if (!audio) return;
- if (playing) { audio.pause(); setPlaying(false); return; }
- audio.play?.()?.catch?.(() => {});
- setPlaying(true);
+ if (playing || preparing) stopPlayback();
+ else playFrom(audioRef.current?.currentTime || 0);
};
if (!doc) return null;
@@ -119,7 +274,7 @@ export default function CompositionPreviewPlayer({ project, audioUrl, seekReques
const aspect = preview.width && preview.height ? `${preview.width} / ${preview.height}` : '16 / 9';
const lyric = lyricAt(project.lyricCues, t);
const shotIndex = shotNav ? shotSteps(project.scenes, t).index : 0;
- const failed = preview.status === 'error' || preview.status === 'unavailable';
+ const failed = preview.status === 'error' || preview.status === 'unavailable' || !!audioError;
const timeLabel = failed
? 'Preview unavailable'
: preview.ready
@@ -127,7 +282,8 @@ export default function CompositionPreviewPlayer({ project, audioUrl, seekReques
: !wanted ? 'Expand to load preview' : preview.status === 'idle' ? 'Preview paused' : 'Loading preview…';
return (
<div className="space-y-2" aria-label={lyrics ? 'Lyric timing playthrough' : 'Composition preview'}>
- {failed && <p className="text-xs text-port-error" role="alert">{preview.reason || 'Preview unavailable'}</p>}
+ {failed && <p className="text-xs text-port-error" role="alert">{audioError || preview.reason || 'Preview unavailable'}</p>}
+ {preview.playbackReason && <p className="text-xs text-port-text-muted" role="status">{preview.playbackReason}</p>}
<div title={title} className={`overflow-hidden rounded border border-port-border bg-black mx-auto ${collapsed ? 'max-xl:hidden' : ''}`}
style={{ aspectRatio: aspect, maxHeight: '70vh', maxWidth: '100%' }}>
{preview.frameUrl ? (
@@ -137,23 +293,23 @@ export default function CompositionPreviewPlayer({ project, audioUrl, seekReques
)}
</div>
<div className="flex flex-wrap items-center gap-2">
- {request && preview.status === 'idle' && <button type="button" className={buttonCls} onClick={preview.resume}>Resume preview</button>}
+ {request && (preview.status === 'idle' || failed) && <button type="button" className={buttonCls} onClick={() => { stopPlayback(); setAudioError(''); preview.resume(); }}>{failed ? 'Retry preview' : 'Resume preview'}</button>}
<button type="button" className={buttonCls} onClick={togglePlay} disabled={!preview.ready || !audioUrl || (failed && !playing)}>
- {playing ? <Pause size={14} /> : <Play size={14} />} {playing ? 'Pause' : 'Play'}
+ {playing || preparing ? <Pause size={14} /> : <Play size={14} />} {playing || preparing ? 'Pause' : 'Play'}
</button>
<label htmlFor={scrubId} className="sr-only">{lyrics ? 'Scrub the lyric playthrough' : draft ? 'Scrub the composition candidate' : 'Scrub the composition preview'}</label>
<input disabled={!preview.ready || failed} id={scrubId} type="range" min={0} max={duration || 0} step={1 / fps} value={Math.min(t, duration || 0)}
- onChange={(e) => { audioRef.current?.pause(); setPlaying(false); seek(Number(e.target.value)); }}
+ onChange={(e) => stopPlayback(Number(e.target.value))}
className="min-w-0 flex-1" />
<span className="text-xs text-port-text-muted tabular-nums">{timeLabel}</span>
</div>
- {shotNav && <ShotStepper scenes={project.scenes} time={t} onSeek={seek} onEditShot={onEditShot ? (sceneId) => { audioRef.current?.pause?.(); setPlaying(false); onEditShot(sceneId); } : null} disabled={!preview.ready || failed} keys />}
+ {shotNav && <ShotStepper scenes={project.scenes} time={t} onSeek={seek} onEditShot={onEditShot ? (sceneId) => { stopPlayback(); onEditShot(sceneId); } : null} disabled={!preview.ready || failed} keys />}
{shotNav && (
<FrameStepper
time={t}
fps={fps}
maxSec={duration || null}
- onSeek={(time) => { audioRef.current?.pause?.(); setPlaying(false); seek(time); }}
+ onSeek={stopPlayback}
disabled={!preview.ready || failed}
capture={() => (preview.frameUrl ? fetch(preview.frameUrl).then((res) => res.blob()) : Promise.resolve(null))}
label={[project.name, shotIndex ? `Shot ${shotIndex}` : null].filter(Boolean).join(' · ')}
@@ -161,7 +317,7 @@ export default function CompositionPreviewPlayer({ project, audioUrl, seekReques
/>
)}
{lyric && !lyrics && <p className="text-xs italic break-words" aria-live="off" data-testid="preview-lyric">♪ {lyric}</p>}
- {audioUrl && <audio ref={audioRef} src={audioUrl} preload="none" className="hidden" onEnded={() => setPlaying(false)} />}
+ {audioUrl && <audio ref={audioRef} src={audioUrl} preload="none" className="hidden" onEnded={() => stopPlayback()} />}
</div>
);
}
diff --git a/client/src/components/musicVideo/CompositionPreviewPlayer.test.jsx b/client/src/components/musicVideo/CompositionPreviewPlayer.test.jsx
index 35bf7a4451..dfb3d297d9 100644
--- a/client/src/components/musicVideo/CompositionPreviewPlayer.test.jsx
+++ b/client/src/components/musicVideo/CompositionPreviewPlayer.test.jsx
@@ -1,4 +1,4 @@
-import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { act, fireEvent, render, screen, waitFor } from '@testing-library/react';
const api = vi.hoisted(() => ({
@@ -418,3 +418,437 @@ describe('CompositionPreviewPlayer', () => {
fetchSpy.mockRestore();
});
});
+
+
+// Exercise the rendered player through the socket boundary. In particular,
+// action sequence numbers and capture frame sequence numbers are independent.
+describe('native composition playback', () => {
+ const receive = (event, payload) => {
+ for (const handler of socket.__handlers.get(`generated-preview:${event}`) || []) handler(payload);
+ };
+ const playbackMessages = (action) => socket.emit.mock.calls.filter(([event, payload]) => event === 'generated-preview:playback' && payload.action === action).map(([, payload]) => payload);
+ let advanceWall;
+ const installNative = ({ prepare = true, roundTrips = [0, 0, 0] } = {}) => {
+ let now = 100;
+ let probes = 0;
+ let opened;
+ advanceWall = (ms) => { now += ms; };
+ vi.spyOn(performance, 'now').mockImplementation(() => now);
+ socket.emit.mockImplementation((event, payload) => {
+ if (event === 'generated-preview:open') {
+ opened = { requestId: payload.requestId, sessionId: '10000000-0000-4000-8000-000000000001', durationSec: 10, fps: 30, playback: { version: 1 } };
+ receive('opened', opened);
+ receive('frame', { ...opened, generation: 0, data: btoa('initial') });
+ }
+ if (event !== 'generated-preview:playback') return;
+ if (payload.action === 'clock') {
+ const rtt = roundTrips[probes++ % roundTrips.length];
+ now += rtt;
+ receive('playback-state', { ...payload, workerTime: payload.clientTime + rtt / 2 + 5000 });
+ }
+ if (payload.action === 'prepare' && prepare) receive('playback-state', { ...payload, status: 'ready' });
+ });
+ return { advance: advanceWall, session: () => opened };
+ };
+ const media = ({ holdClock = false } = {}) => {
+ let audio = null;
+ let paused = true;
+ let startupFrames = 0;
+ let frameId = 0;
+ vi.spyOn(HTMLMediaElement.prototype, 'paused', 'get').mockImplementation(() => paused);
+ const play = vi.spyOn(HTMLMediaElement.prototype, 'play').mockImplementation(function () {
+ audio = this;
+ paused = false;
+ startupFrames = holdClock ? 0 : 12;
+ return Promise.resolve();
+ });
+ const pause = vi.spyOn(HTMLMediaElement.prototype, 'pause').mockImplementation(() => { paused = true; startupFrames = 0; });
+ const raf = vi.spyOn(window, 'requestAnimationFrame').mockImplementation((callback) => {
+ // Native tests model an advancing audio clock; legacy RAFs stay manual.
+ if (startupFrames > 0 && playbackMessages('prepare').length > playbackMessages('play').length) {
+ startupFrames -= 1;
+ queueMicrotask(() => { if (!paused) { advanceWall(1000 / 60); audio.currentTime += 1 / 60; callback(); } });
+ }
+ return ++frameId;
+ });
+ vi.spyOn(window, 'cancelAnimationFrame').mockImplementation(() => {});
+ return { play, pause, raf };
+ };
+ afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); });
+
+ it('waits for native preparation, anchors actual audio time, and sends bounded clock updates without frame seeks', async () => {
+ const native = installNative({ prepare: false, roundTrips: [180, 20, 120] });
+ const { play, raf } = media();
+ const { container } = render(<CompositionPreviewPlayer project={project} audioUrl='/song.mp3' />);
+ fireEvent.click(screen.getByRole('button', { name: 'Play' }));
+ await waitFor(() => expect(playbackMessages('prepare')).toHaveLength(1));
+ expect(play).not.toHaveBeenCalled();
+ expect(playbackMessages('clock')).toHaveLength(3);
+ const prepared = playbackMessages('prepare')[0];
+ expect(prepared.generation).toBe(0);
+ const audio = container.querySelector('audio');
+ audio.currentTime = 0.125;
+ await act(async () => receive('playback-state', { ...prepared, status: 'ready' }));
+ expect(play).toHaveBeenCalledOnce();
+ const anchor = playbackMessages('play')[0];
+ expect(anchor.t).toBeCloseTo(0.225, 8);
+ expect(anchor.t).toBe(audio.currentTime);
+ expect(anchor.workerTime).toBe(performance.now() + 5000);
+ for (let i = 1; i <= 60; i += 1) {
+ native.advance(1000 / 60);
+ audio.currentTime = i / 60;
+ act(() => raf.mock.calls.at(-1)[0]());
+ }
+ expect(playbackMessages('sync').length).toBeGreaterThanOrEqual(3);
+ expect(playbackMessages('sync').length).toBeLessThanOrEqual(4);
+ expect(socket.emit.mock.calls.filter(([event, payload]) => event === 'generated-preview:seek' && payload.playing)).toHaveLength(0);
+ expect(screen.getByText('1.00s / 10.0s')).toBeInTheDocument();
+ // Assert the client's exact wire projection here; authoritative schema
+ // admission is exercised through the server's public socket tests.
+ const identity = { requestId: opens()[0][1].requestId, sessionId: native.session().sessionId };
+ for (const id of Object.values(identity)) expect(id).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/);
+ const syncs = playbackMessages('sync');
+ const emitted = socket.emit.mock.calls.filter(([event]) => event === 'generated-preview:playback').map(([, payload]) => payload);
+ expect(emitted).toStrictEqual([
+ ...[100, 280, 300].map((clientTime, probeId) => ({ ...identity, action: 'clock', probeId, clientTime })),
+ { ...identity, action: 'prepare', epoch: 1, sequence: 1, generation: 0, t: 0 },
+ { ...identity, action: 'play', epoch: 1, sequence: 2, t: anchor.t, workerTime: anchor.workerTime },
+ ...syncs.map((payload, index) => ({ ...identity, action: 'sync', epoch: 1, sequence: index + 3, t: payload.t, workerTime: payload.workerTime })),
+ ]);
+ let previousTime = anchor.workerTime;
+ let previousPosition = anchor.t;
+ for (const payload of emitted) {
+ for (const field of ['clientTime', 't', 'workerTime']) {
+ if (field in payload) expect(Number.isFinite(payload[field]), `${payload.action}.${field}`).toBe(true);
+ }
+ if ('t' in payload) {
+ expect(payload.t).toBeGreaterThanOrEqual(0);
+ expect(payload.t).toBeLessThanOrEqual(10);
+ }
+ if (payload.action === 'sync') {
+ expect(payload.workerTime).toBeGreaterThan(previousTime);
+ expect(payload.t).toBeGreaterThan(previousPosition);
+ previousTime = payload.workerTime;
+ previousPosition = payload.t;
+ }
+ }
+
+ const frame = (extra) => receive('frame', { ...native.session(), playbackEpoch: anchor.epoch, sequence: 1, data: btoa('native'), ...extra });
+ act(() => frame({}));
+ const image = container.querySelector('img');
+ const accepted = image.src;
+ act(() => frame({ data: btoa('duplicate') }));
+ expect(image.src).toBe(accepted);
+ fireEvent.click(screen.getByRole('button', { name: 'Pause' }));
+ const exact = socket.emit.mock.calls.filter(([event]) => event === 'generated-preview:seek').at(-1)[1];
+ act(() => frame({ sequence: 2 }));
+ expect(image.src).toBe(accepted);
+ act(() => receive('frame', { ...native.session(), generation: exact.generation - 1, data: btoa('stale exact') }));
+ expect(image.src).toBe(accepted);
+ act(() => receive('frame', { ...native.session(), generation: exact.generation, data: btoa('paused exact') }));
+ expect(image.src).not.toBe(accepted);
+ });
+
+ it('waits through a resolved play promise and 120ms audio hold before anchoring a stable clock window', async () => {
+ const native = installNative();
+ const { play, raf } = media({ holdClock: true });
+ const { container } = render(<CompositionPreviewPlayer project={project} audioUrl='/song.mp3' />);
+ await act(async () => fireEvent.click(screen.getByRole('button', { name: 'Play' })));
+ const audio = container.querySelector('audio');
+ expect(play).toHaveBeenCalledOnce();
+ expect(playbackMessages('prepare')).toHaveLength(1);
+ expect(screen.getByRole('button', { name: 'Pause' })).toBeEnabled();
+ for (let i = 0; i < 6; i += 1) {
+ native.advance(20);
+ act(() => raf.mock.calls.at(-1)[0]());
+ }
+ expect(audio.currentTime).toBe(0);
+ expect(playbackMessages('play')).toHaveLength(0);
+ expect(playbackMessages('sync')).toHaveLength(0);
+ audio.currentTime = 0.01;
+ native.advance(10);
+ act(() => raf.mock.calls.at(-1)[0]());
+ expect(playbackMessages('play')).toHaveLength(0);
+ audio.currentTime = 0.02;
+ native.advance(10);
+ act(() => raf.mock.calls.at(-1)[0]());
+ expect(playbackMessages('play')).toHaveLength(0);
+ for (let i = 3; i <= 10; i += 1) {
+ audio.currentTime = i / 100;
+ native.advance(10);
+ act(() => raf.mock.calls.at(-1)[0]());
+ if (i < 10) expect(playbackMessages('play')).toHaveLength(0);
+ }
+ const anchor = playbackMessages('play')[0];
+ expect(anchor.t).toBe(0.1);
+ expect(anchor.workerTime).toBe(performance.now() + 5000);
+ audio.currentTime = 0.35;
+ native.advance(250);
+ act(() => raf.mock.calls.at(-1)[0]());
+ const sync = playbackMessages('sync')[0];
+ expect(sync.t - anchor.t).toBeCloseTo((sync.workerTime - anchor.workerTime) / 1000);
+ expect(socket.emit.mock.calls.filter(([event, payload]) => event === 'generated-preview:seek' && payload.playing)).toHaveLength(0);
+ });
+
+ it('rejects early 21ms clock advances followed by a 78ms hold until the latest window runs steadily at 1x', async () => {
+ const native = installNative();
+ const { raf } = media({ holdClock: true });
+ const { container } = render(<CompositionPreviewPlayer project={project} audioUrl='/song.mp3' />);
+ await act(async () => fireEvent.click(screen.getByRole('button', { name: 'Play' })));
+ const audio = container.querySelector('audio');
+ const observe = (wallMs, mediaTime) => {
+ native.advance(wallMs);
+ audio.currentTime = mediaTime;
+ act(() => raf.mock.calls.at(-1)[0]());
+ };
+ observe(10.5, 0.0105);
+ observe(10.5, 0.021);
+ expect(playbackMessages('play')).toHaveLength(0);
+ for (let i = 0; i < 3; i += 1) observe(26, 0.021);
+ expect(playbackMessages('play')).toHaveLength(0);
+ for (let i = 1; i <= 5; i += 1) {
+ observe(20, 0.021 + i * 0.02);
+ if (i < 5) expect(playbackMessages('play')).toHaveLength(0);
+ }
+ expect(playbackMessages('play')).toHaveLength(1);
+ expect(playbackMessages('play')[0].t).toBeCloseTo(0.121);
+ expect(playbackMessages('play')[0].workerTime).toBe(performance.now() + 5000);
+ expect(screen.queryByRole('alert')).toBeNull();
+ });
+
+ it('cancels the audio clock wait on scrub and unmount without allowing stale RAFs to start native playback', async () => {
+ installNative();
+ const { raf } = media({ holdClock: true });
+ const { container, unmount } = render(<CompositionPreviewPlayer project={project} audioUrl='/song.mp3' />);
+ vi.useFakeTimers();
+ await act(async () => fireEvent.click(screen.getByRole('button', { name: 'Play' })));
+ expect(vi.getTimerCount()).toBe(1);
+ const stale = raf.mock.calls.at(-1)[0];
+ fireEvent.change(screen.getByLabelText('Scrub the composition preview'), { target: { value: '2' } });
+ expect(vi.getTimerCount()).toBe(0);
+ container.querySelector('audio').currentTime = 3;
+ act(() => stale());
+ expect(playbackMessages('play')).toHaveLength(0);
+ expect(screen.getByRole('button', { name: 'Play' })).toBeEnabled();
+ await act(async () => fireEvent.click(screen.getByRole('button', { name: 'Play' })));
+ expect(vi.getTimerCount()).toBe(1);
+ const unmounted = raf.mock.calls.at(-1)[0];
+ unmount();
+ expect(vi.getTimerCount()).toBe(0);
+ const sent = socket.emit.mock.calls.length;
+ act(() => unmounted());
+ expect(socket.emit.mock.calls).toHaveLength(sent);
+ });
+
+ it('pauses with a visible error when a resolved play promise never advances the audio clock', async () => {
+ installNative();
+ const { pause } = media({ holdClock: true });
+ render(<CompositionPreviewPlayer project={project} audioUrl='/song.mp3' />);
+ vi.useFakeTimers();
+ await act(async () => fireEvent.click(screen.getByRole('button', { name: 'Play' })));
+ await act(async () => vi.advanceTimersByTimeAsync(3000));
+ expect(screen.getByRole('alert')).toHaveTextContent('audio clock did not start');
+ expect(playbackMessages('play')).toHaveLength(0);
+ expect(pause).toHaveBeenCalled();
+ expect(vi.getTimerCount()).toBe(0);
+ });
+
+ it('bounds a never-settling play promise and ignores its resolution after the deadline', async () => {
+ installNative();
+ const { play, raf } = media({ holdClock: true });
+ let resolvePlay;
+ play.mockImplementation(() => new Promise((resolve) => { resolvePlay = resolve; }));
+ render(<CompositionPreviewPlayer project={project} audioUrl='/song.mp3' />);
+ vi.useFakeTimers();
+ await act(async () => fireEvent.click(screen.getByRole('button', { name: 'Play' })));
+ expect(play).toHaveBeenCalledOnce();
+ expect(vi.getTimerCount()).toBe(1);
+ expect(raf).not.toHaveBeenCalled();
+ await act(async () => vi.advanceTimersByTimeAsync(3000));
+ expect(screen.getByRole('alert')).toHaveTextContent('audio clock did not start');
+ expect(vi.getTimerCount()).toBe(0);
+ await act(async () => resolvePlay());
+ expect(playbackMessages('play')).toHaveLength(0);
+ expect(raf).not.toHaveBeenCalled();
+ expect(vi.getTimerCount()).toBe(0);
+ expect(screen.getByRole('button', { name: 'Play' })).toBeDisabled();
+ });
+
+ it('cancels cold preparation on scrub and ignores its late ready and captured frames', async () => {
+ const native = installNative({ prepare: false });
+ const { play } = media();
+ const { container, unmount } = render(<CompositionPreviewPlayer project={project} audioUrl='/song.mp3' />);
+ fireEvent.click(screen.getByRole('button', { name: 'Play' }));
+ await waitFor(() => expect(playbackMessages('prepare')).toHaveLength(1));
+ const prepared = playbackMessages('prepare')[0];
+ const initial = container.querySelector('img').src;
+ fireEvent.change(screen.getByLabelText('Scrub the composition preview'), { target: { value: '3' } });
+ await act(async () => {
+ receive('playback-state', { ...prepared, status: 'ready' });
+ receive('frame', { ...native.session(), playbackEpoch: prepared.epoch, sequence: 50, data: btoa('late') });
+ });
+ expect(play).not.toHaveBeenCalled();
+ expect(playbackMessages('play')).toHaveLength(0);
+ expect(container.querySelector('img').src).toBe(initial);
+ expect(screen.getByText('3.00s / 10.0s')).toBeInTheDocument();
+ fireEvent.click(screen.getByRole('button', { name: 'Play' }));
+ await waitFor(() => expect(playbackMessages('prepare')).toHaveLength(2));
+ unmount();
+ const sent = socket.emit.mock.calls.length;
+ await act(async () => receive('playback-state', { ...playbackMessages('prepare')[1], status: 'ready' }));
+ expect(socket.emit.mock.calls).toHaveLength(sent);
+ expect(play).not.toHaveBeenCalled();
+ });
+
+ it('displays delayed fallback frames in order while playing, then rejects them across pause and resume', async () => {
+ const native = installNative({ roundTrips: [120, 160, 200] });
+ const { raf } = media();
+ const { container } = render(<CompositionPreviewPlayer project={project} audioUrl='/song.mp3' />);
+ fireEvent.click(screen.getByRole('button', { name: 'Play' }));
+ await screen.findByText(/Native playback clock could not be synchronized/);
+ expect(playbackMessages('prepare')).toHaveLength(0);
+ expect(playbackMessages('play')).toHaveLength(0);
+ const audio = container.querySelector('audio');
+ const image = container.querySelector('img');
+ const initial = image.src;
+ const frame = (generation) => receive('frame', { ...native.session(), generation, data: btoa(`fallback-${generation}`) });
+ audio.currentTime = 1;
+ act(() => raf.mock.calls.at(-1)[0]());
+ audio.currentTime = 2;
+ act(() => raf.mock.calls.at(-1)[0]());
+ expect(socket.emit).toHaveBeenLastCalledWith('generated-preview:seek', expect.objectContaining({ t: 2, generation: 2, playing: true }));
+ act(() => frame(1));
+ const first = image.src;
+ expect(first).not.toBe(initial);
+ act(() => frame(0));
+ expect(image.src).toBe(first);
+ act(() => frame(2));
+ const second = image.src;
+ expect(second).not.toBe(first);
+ fireEvent.click(screen.getByRole('button', { name: 'Pause' }));
+ expect(socket.emit).toHaveBeenLastCalledWith('generated-preview:seek', expect.objectContaining({ generation: 3, playing: false }));
+ act(() => { frame(1); frame(2); });
+ expect(image.src).toBe(second);
+ act(() => frame(3));
+ const paused = image.src;
+ expect(paused).not.toBe(second);
+ await act(async () => fireEvent.click(screen.getByRole('button', { name: 'Play' })));
+ audio.currentTime = 3;
+ act(() => raf.mock.calls.at(-1)[0]());
+ audio.currentTime = 4;
+ act(() => raf.mock.calls.at(-1)[0]());
+ act(() => frame(2));
+ expect(image.src).toBe(paused);
+ act(() => frame(4));
+ expect(image.src).not.toBe(paused);
+ });
+
+ it('resynchronizes clock drift at actual audio time and stops repeated clock failures with a visible error', async () => {
+ installNative();
+ media();
+ const { container } = render(<CompositionPreviewPlayer project={project} audioUrl='/song.mp3' />);
+ fireEvent.click(screen.getByRole('button', { name: 'Play' }));
+ await waitFor(() => expect(playbackMessages('play')).toHaveLength(1));
+ container.querySelector('audio').currentTime = 3;
+ for (let i = 1; i <= 3; i += 1) {
+ const previous = playbackMessages('play').at(-1);
+ await act(async () => receive('playback-state', { ...previous, action: 'sync', status: 'resync-required' }));
+ await waitFor(() => expect(playbackMessages('play')).toHaveLength(i + 1));
+ expect(playbackMessages('play').at(-1)).toEqual(expect.objectContaining({ t: container.querySelector('audio').currentTime, epoch: expect.any(Number) }));
+ expect(playbackMessages('play').at(-1).epoch).toBeGreaterThan(previous.epoch);
+ }
+ await act(async () => receive('playback-state', { ...playbackMessages('play').at(-1), action: 'sync', status: 'resync-required' }));
+ expect(screen.getByRole('alert')).toHaveTextContent('repeatedly lost synchronization');
+ expect(screen.getByRole('button', { name: 'Play' })).toBeDisabled();
+ expect(playbackMessages('prepare')).toHaveLength(4);
+ });
+
+ it('reprepares repeated intentional control edits without consuming the clock retry budget', async () => {
+ installNative();
+ media();
+ const { container } = render(<CompositionPreviewPlayer project={project} audioUrl='/song.mp3' />);
+ fireEvent.click(screen.getByRole('button', { name: 'Play' }));
+ await waitFor(() => expect(playbackMessages('play')).toHaveLength(1));
+ container.querySelector('audio').currentTime = 2;
+ for (let i = 1; i <= 5; i += 1) {
+ await act(async () => receive('playback-state', { ...playbackMessages('play').at(-1), action: 'sync', status: 'resync-required', reason: 'controls' }));
+ expect(playbackMessages('play')).toHaveLength(i + 1);
+ expect(playbackMessages('play').at(-1).t).toBe(container.querySelector('audio').currentTime);
+ }
+ expect(screen.queryByRole('alert')).toBeNull();
+ expect(screen.getByRole('button', { name: 'Pause' })).toBeEnabled();
+ expect(playbackMessages('prepare')).toHaveLength(6);
+ await act(async () => receive('playback-state', { ...playbackMessages('play').at(-1), action: 'sync', status: 'resync-required', reason: 'clock' }));
+ expect(playbackMessages('play')).toHaveLength(7);
+ expect(screen.queryByRole('alert')).toBeNull();
+ });
+
+ it('keeps native playback stopped when audio play fails', async () => {
+ installNative();
+ const { play, pause } = media();
+ play.mockRejectedValue(new Error('Audio permission denied'));
+ render(<CompositionPreviewPlayer project={project} audioUrl='/song.mp3' />);
+ fireEvent.click(screen.getByRole('button', { name: 'Play' }));
+ expect(await screen.findByRole('alert')).toHaveTextContent('Audio permission denied');
+ expect(pause).toHaveBeenCalled();
+ expect(playbackMessages('play')).toHaveLength(0);
+ expect(screen.getByRole('button', { name: 'Retry preview' })).toBeEnabled();
+ });
+
+ it('pauses with a visible error when native preparation never becomes ready', async () => {
+ installNative({ prepare: false });
+ const { play } = media();
+ render(<CompositionPreviewPlayer project={project} audioUrl='/song.mp3' />);
+ vi.useFakeTimers();
+ await act(async () => fireEvent.click(screen.getByRole('button', { name: 'Play' })));
+ expect(playbackMessages('prepare')).toHaveLength(1);
+ await act(async () => vi.advanceTimersByTimeAsync(10000));
+ expect(screen.getByRole('alert')).toHaveTextContent('Native preview preparation failed');
+ expect(play).not.toHaveBeenCalled();
+ expect(vi.getTimerCount()).toBe(0);
+ });
+
+ it('preserves play intent across shot navigation and gates queued frames when hidden', async () => {
+ const native = installNative();
+ const { play } = media();
+ const shots = { ...project, scenes: [{ sceneId: 'a', startSec: 0 }, { sceneId: 'b', startSec: 4 }] };
+ const { container } = render(<CompositionPreviewPlayer project={shots} audioUrl='/song.mp3' shotNav />);
+ fireEvent.click(screen.getByRole('button', { name: 'Play' }));
+ await waitFor(() => expect(playbackMessages('play')).toHaveLength(1));
+ const previous = playbackMessages('play')[0];
+ fireEvent.click(screen.getByRole('button', { name: 'Next shot' }));
+ await waitFor(() => expect(playbackMessages('play')).toHaveLength(2));
+ const anchor = playbackMessages('play')[1];
+ expect(anchor.t).toBeGreaterThanOrEqual(4.1);
+ expect(anchor.t).toBe(container.querySelector('audio').currentTime);
+ expect(anchor.epoch).toBeGreaterThan(previous.epoch);
+ expect(play).toHaveBeenCalledTimes(2);
+ expect(screen.getByRole('button', { name: 'Pause' })).toBeEnabled();
+ const image = container.querySelector('img');
+ const initial = image.src;
+ vi.spyOn(document, 'visibilityState', 'get').mockReturnValue('hidden');
+ act(() => document.dispatchEvent(new Event('visibilitychange')));
+ act(() => receive('frame', { ...native.session(), playbackEpoch: anchor.epoch, sequence: 100, data: btoa('hidden frame') }));
+ expect(image.src).toBe(initial);
+ expect(screen.getByRole('button', { name: 'Play' })).toBeEnabled();
+ expect(socket.emit).toHaveBeenLastCalledWith('generated-preview:seek', expect.objectContaining({ t: container.querySelector('audio').currentTime, playing: false }));
+ });
+
+ it('preserves a terminal error through closed and reconnect until explicit retry', async () => {
+ const native = installNative();
+ media();
+ render(<CompositionPreviewPlayer project={project} audioUrl='/song.mp3' />);
+ act(() => {
+ receive('error', { ...native.session(), message: 'Native worker failed' });
+ receive('closed', { ...native.session(), reason: 'error' });
+ receive('frame', { ...native.session(), generation: 0, data: btoa('late') });
+ for (const fn of socket.__handlers.get('disconnect') || []) fn();
+ for (const fn of socket.__handlers.get('connect') || []) fn();
+ });
+ expect(screen.getByRole('alert')).toHaveTextContent('Native worker failed');
+ expect(opens()).toHaveLength(1);
+ fireEvent.click(screen.getByRole('button', { name: 'Retry preview' }));
+ expect(opens()).toHaveLength(2);
+ expect(screen.queryByRole('alert')).toBeNull();
+ });
+});
diff --git a/client/src/hooks/useGeneratedPreview.js b/client/src/hooks/useGeneratedPreview.js
index e4bca3b7e0..d3ae592904 100644
--- a/client/src/hooks/useGeneratedPreview.js
+++ b/client/src/hooks/useGeneratedPreview.js
@@ -16,6 +16,10 @@ const EMPTY = {
audioStatus: 'none',
lastEvent: null,
recording: null,
+ playback: false,
+ playbackReason: '',
+ playbackResync: 0,
+ playbackResyncReason: '',
};
const newRequestId = () => uuidv4();
@@ -44,13 +48,31 @@ export default function useGeneratedPreview({ active, request, controls = null }
controlsRef.current = controls;
const [state, setState] = useState(EMPTY);
const requestKey = active && request ? JSON.stringify(request) : '';
- const resume = useCallback(() => openRef.current?.(), []);
+ const resume = useCallback(() => openRef.current?.(true), []);
+
+ const cancelPlayback = useCallback((current = pending.current) => {
+ if (!current) return;
+ current.epoch += 1;
+ current.mode = 'exact';
+ current.frameSequence = -1;
+ for (const finish of current.responses.values()) finish(null);
+ current.responses.clear();
+ }, []);
const seek = useCallback((t, { playing = false } = {}) => {
- lastSeek.current = { t, playing };
const current = pending.current;
+ const continuing = playing && current?.mode === 'legacy-playing';
+ if (!continuing) cancelPlayback(current);
+ lastSeek.current = { t, playing };
if (!socket.connected || !current?.sessionId) return;
current.generation += 1;
+ if (playing) {
+ if (!continuing) {
+ current.generationFloor = current.generation;
+ current.completedGeneration = current.generation - 1;
+ }
+ current.mode = 'legacy-playing';
+ }
socket.emit('generated-preview:seek', {
requestId: current.requestId,
sessionId: current.sessionId,
@@ -58,7 +80,7 @@ export default function useGeneratedPreview({ active, request, controls = null }
t,
playing,
});
- }, []);
+ }, [cancelPlayback]);
const command = useCallback((action, extra = {}) => {
const current = pending.current;
@@ -72,10 +94,84 @@ export default function useGeneratedPreview({ active, request, controls = null }
});
}, []);
+ const preparePlayback = useCallback(async (t) => {
+ const current = pending.current;
+ if (!socket.connected || !current?.playback) return false;
+ cancelPlayback(current);
+ const epoch = current.epoch;
+ current.mode = 'preparing';
+ const live = () => mounted.current && socket.connected && pending.current === current && current.epoch === epoch;
+ const exchange = (key, payload, timeoutMs) => new Promise((resolve) => {
+ const finish = (reply) => {
+ clearTimeout(timer);
+ current.responses.delete(key);
+ resolve(reply);
+ };
+ const timer = setTimeout(() => finish(null), timeoutMs);
+ current.responses.set(key, finish);
+ if (live()) socket.emit('generated-preview:playback', {
+ requestId: current.requestId, sessionId: current.sessionId, ...payload,
+ });
+ else finish(null);
+ });
+ if (current.offset === null) {
+ const samples = [];
+ for (let i = 0; i < 3 && live(); i += 1) {
+ const clientTime = performance.now();
+ const probeId = current.probeId++;
+ const reply = await exchange(probeId, { action: 'clock', probeId, clientTime }, 1000);
+ const received = performance.now();
+ if (reply && reply.clientTime === clientTime && Number.isFinite(reply.workerTime)) {
+ samples.push({ uncertainty: (received - clientTime) / 2, offset: reply.workerTime - (clientTime + received) / 2 });
+ }
+ }
+ if (!live()) return null;
+ samples.sort((a, b) => a.uncertainty - b.uncertainty);
+ if (!samples.length || samples[0].uncertainty > 50) {
+ current.mode = 'exact';
+ current.playback = false;
+ setState((prev) => ({ ...prev, playback: false, playbackReason: 'Native playback clock could not be synchronized; using exact frame playback.' }));
+ return false;
+ }
+ current.offset = samples[0].offset;
+ }
+ const sequence = ++current.sequence;
+ const reply = await exchange(`prepare-${epoch}-${sequence}`, {
+ action: 'prepare', epoch, sequence, t, generation: current.generation,
+ }, 10000);
+ if (!live()) return null;
+ if (reply?.status !== 'ready') {
+ current.mode = 'exact';
+ current.terminalError = true;
+ current.hasError = true;
+ setState((prev) => ({ ...prev, status: 'error', ready: false, reason: reply?.message || 'Native preview preparation failed. Retry the preview.' }));
+ throw new Error(reply?.message || 'Native preview preparation failed.');
+ }
+ current.mode = 'prepared';
+ return true;
+ }, [cancelPlayback, mounted]);
+
+ const playbackAnchor = useCallback((action, t) => {
+ const current = pending.current;
+ if (!socket.connected || !current?.playback || current.offset === null) return;
+ if (action === 'play' ? current.mode !== 'prepared' : current.mode !== 'playing') return;
+ current.mode = 'playing';
+ lastSeek.current = { t, playing: false };
+ socket.emit('generated-preview:playback', {
+ requestId: current.requestId, sessionId: current.sessionId,
+ epoch: current.epoch, sequence: ++current.sequence, action, t,
+ workerTime: performance.now() + current.offset,
+ });
+ }, []);
+ const startPlayback = useCallback((t) => playbackAnchor('play', t), [playbackAnchor]);
+ const syncPlayback = useCallback((t) => playbackAnchor('sync', t), [playbackAnchor]);
+ const pausePlayback = useCallback((t) => seek(t), [seek]);
+
useEffect(() => {
lastSeek.current = null;
if (!requestKey) {
const current = pending.current;
+ cancelPlayback(current);
pending.current = null;
if (current && socket.connected) {
socket.emit('generated-preview:close', {
@@ -91,13 +187,20 @@ export default function useGeneratedPreview({ active, request, controls = null }
let stoppedStatus = null;
// Sessions belong to one socket connection. Never buffer an open/command
// while offline: the server has already destroyed the previous session.
- const open = () => {
- if (!socket.connected || pending.current) return;
+ const open = (retry = false) => {
+ if (!socket.connected) return;
+ if (retry && pending.current) {
+ const previous = pending.current;
+ cancelPlayback(previous);
+ pending.current = null;
+ socket.emit('generated-preview:close', { requestId: previous.requestId, ...(previous.sessionId ? { sessionId: previous.sessionId } : {}) });
+ }
+ if (pending.current) return;
stoppedStatus = null;
requestId = newRequestId();
const openControls = { look: controlsRef.current?.look ?? null, card: controlsRef.current?.card ?? null };
- pending.current = { requestId, sessionId: null, generation: 0, controls: openControls };
- if (mounted.current) setState((prev) => ({ ...prev, status: 'opening', sessionId: null, ready: false, reason: '', lastEvent: null }));
+ pending.current = { requestId, sessionId: null, generation: 0, generationFloor: 0, completedGeneration: -1, controls: openControls, epoch: 0, sequence: 0, probeId: 0, frameSequence: -1, mode: 'exact', offset: null, playback: false, responses: new Map() };
+ if (mounted.current) setState((prev) => ({ ...prev, status: 'opening', sessionId: null, ready: false, reason: '', lastEvent: null, playback: false, playbackReason: '' }));
socket.emit('generated-preview:open', {
requestId,
...request,
@@ -106,6 +209,7 @@ export default function useGeneratedPreview({ active, request, controls = null }
});
};
const onDisconnect = () => {
+ cancelPlayback();
pending.current = null;
recordingChunks.clear();
if (mounted.current) setState((prev) => ({ ...prev, status: 'disconnected', sessionId: null, ready: false, lastEvent: null }));
@@ -119,7 +223,8 @@ export default function useGeneratedPreview({ active, request, controls = null }
const onOpened = (payload) => {
if (!mine(payload)) return;
stoppedStatus = null;
- pending.current = { ...pending.current, sessionId: payload.sessionId };
+ const capable = request.kind === 'music-video' && !request.lyrics && payload.playback?.version === 1;
+ Object.assign(pending.current, { sessionId: payload.sessionId, playback: capable, capable });
if (!mounted.current) return;
setState((prev) => ({
...prev,
@@ -132,6 +237,7 @@ export default function useGeneratedPreview({ active, request, controls = null }
width: Number(payload.width) || 0,
height: Number(payload.height) || 0,
audioStatus: payload.audio || 'none',
+ playback: capable,
}));
// A reconnect opens at time zero; restore the current scrub/playhead
// position using the new session, including seeks made while offline.
@@ -140,6 +246,19 @@ export default function useGeneratedPreview({ active, request, controls = null }
const onFrame = (payload) => {
if (!mine(payload) || !payload.data) return;
if (pending.current.sessionId && payload.sessionId && payload.sessionId !== pending.current.sessionId) return;
+ const current = pending.current;
+ if (current.terminalError) return;
+ if (payload.playbackEpoch !== undefined) {
+ if (current.mode !== 'playing' || payload.playbackEpoch !== current.epoch || !Number.isInteger(payload.sequence) || payload.sequence <= current.frameSequence) return;
+ current.frameSequence = payload.sequence;
+ } else if (current.capable) {
+ if (current.mode === 'legacy-playing') {
+ // A completed frame can trail newer playing requests. Display it
+ // once, in order, but never cross a pause/scrub/new-play boundary.
+ if (!Number.isInteger(payload.generation) || payload.generation < current.generationFloor || payload.generation > current.generation || payload.generation <= current.completedGeneration) return;
+ current.completedGeneration = payload.generation;
+ } else if (current.mode !== 'exact' || payload.generation !== current.generation) return;
+ }
let url = '';
try {
url = URL.createObjectURL(new Blob([decode(payload.data)], { type: 'image/jpeg' }));
@@ -153,12 +272,13 @@ export default function useGeneratedPreview({ active, request, controls = null }
// Screencasting can start before the open acknowledgement. Show its
// frame, but commands need the session identity established by opened.
const opened = !!pending.current.sessionId;
- if (opened) stoppedStatus = null;
+ if (opened) { stoppedStatus = null; current.hasError = false; }
setState((prev) => ({ ...prev, frameUrl: url, ...(opened ? { status: 'ready', ready: true } : {}) }));
};
const onUnavailable = (payload) => {
if (!mine(payload) || !mounted.current) return;
stoppedStatus = 'unavailable';
+ cancelPlayback();
setState((prev) => ({
...prev, status: 'unavailable', ready: false, reason: payload.message || 'Isolated preview is unavailable on this host.',
}));
@@ -169,10 +289,31 @@ export default function useGeneratedPreview({ active, request, controls = null }
// A command/seek failure must not prevent replacing a lost session.
// Only failures before the open handshake suppress reconnect retries.
stoppedStatus = pending.current.sessionId ? null : 'error';
+ pending.current.hasError = true;
+ if (pending.current.capable) pending.current.terminalError = true;
+ cancelPlayback();
setState((prev) => ({
...prev, status: 'error', ready: false, reason: payload.message || 'The isolated preview could not start.',
}));
};
+ const onPlaybackState = (payload) => {
+ if (!mine(payload) || payload.sessionId !== pending.current.sessionId) return;
+ const current = pending.current;
+ if (payload.action === 'clock') {
+ current.responses.get(payload.probeId)?.(payload);
+ return;
+ }
+ if (payload.epoch !== current.epoch) return;
+ if (payload.action === 'prepare') {
+ current.responses.get(`prepare-${payload.epoch}-${payload.sequence}`)?.(payload);
+ } else if (payload.status === 'resync-required') {
+ cancelPlayback();
+ setState((prev) => ({ ...prev, playbackResync: prev.playbackResync + 1, playbackResyncReason: payload.reason || 'clock' }));
+ } else if (payload.status === 'error') {
+ current.terminalError = true;
+ onError({ ...payload, message: payload.message || 'Native preview playback lost synchronization. Retry the preview.' });
+ }
+ };
const onEvent = (payload) => {
if (!mine(payload) || !mounted.current) return;
setState((prev) => ({ ...prev, lastEvent: payload }));
@@ -197,17 +338,20 @@ export default function useGeneratedPreview({ active, request, controls = null }
};
const onClosed = (payload) => {
if (!pending.current?.sessionId || payload?.sessionId !== pending.current.sessionId) return;
+ const failed = pending.current.hasError;
+ cancelPlayback();
pending.current = null;
recordingChunks.clear();
- stoppedStatus = 'idle';
+ stoppedStatus = failed ? 'error' : 'idle';
if (!mounted.current) return;
// Idle/capacity cleanup stays effective until the viewer explicitly
// resumes. Reopening here would keep idle renderers alive indefinitely.
- setState((prev) => ({ ...prev, status: 'idle', sessionId: null, ready: false, lastEvent: null }));
+ setState((prev) => ({ ...prev, status: stoppedStatus, sessionId: null, ready: false, lastEvent: null }));
};
socket.on('generated-preview:opened', onOpened);
socket.on('generated-preview:frame', onFrame);
+ socket.on('generated-preview:playback-state', onPlaybackState);
socket.on('generated-preview:unavailable', onUnavailable);
socket.on('generated-preview:error', onError);
socket.on('generated-preview:event', onEvent);
@@ -221,6 +365,7 @@ export default function useGeneratedPreview({ active, request, controls = null }
return () => {
socket.off('generated-preview:opened', onOpened);
socket.off('generated-preview:frame', onFrame);
+ socket.off('generated-preview:playback-state', onPlaybackState);
socket.off('generated-preview:unavailable', onUnavailable);
socket.off('generated-preview:error', onError);
socket.off('generated-preview:event', onEvent);
@@ -231,6 +376,7 @@ export default function useGeneratedPreview({ active, request, controls = null }
openRef.current = null;
const current = pending.current;
if (current?.requestId === requestId) {
+ cancelPlayback(current);
pending.current = null;
if (socket.connected) {
socket.emit('generated-preview:close', {
@@ -242,7 +388,7 @@ export default function useGeneratedPreview({ active, request, controls = null }
};
// `request` is read for the payload; requestKey is its identity.
// eslint-disable-next-line react-hooks/exhaustive-deps
- }, [requestKey, mounted, seek]);
+ }, [requestKey, mounted, seek, cancelPlayback]);
// Reconcile against what this session actually received. Controls can change
// while its open request is in flight; readiness must not mark those edits sent.
@@ -262,5 +408,5 @@ export default function useGeneratedPreview({ active, request, controls = null }
if (recordingUrl.current) URL.revokeObjectURL(recordingUrl.current);
}, []);
- return { ...state, seek, command, resume };
+ return { ...state, seek, command, resume, preparePlayback, startPlayback, syncPlayback, pausePlayback };
}
diff --git a/server/lib/README.md b/server/lib/README.md
index 0ed0e0819a..8501035673 100644
--- a/server/lib/README.md
+++ b/server/lib/README.md
@@ -35,6 +35,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
| `codeAnimationAcceptance.js` | Code Animation production acceptance: splits run evidence into technical/visual/temporal/sound (unmeasured stays unverified), decides promotability, freezes source/audio/render hashes and detects stale acceptance. Pure. |
| `codeAnimationContainment.js` | Code Animation production worker containment contract: per-platform enforced mechanism (macOS Seatbelt or Linux bubblewrap, otherwise refusal), deny-by-default profiles and a process-denying Linux seccomp filter, from-scratch worker environment, operator tool-root rules, tool configuration and worker limit schemas. Pure; the runner is `services/codeAnimation/containedWorker.js`. |
| `generatedPreviewSandbox.js` | OS network-isolation plan for generated preview browsers: macOS Seatbelt or Linux bubblewrap, otherwise refusal. The jail denies IP and the host DNS resolver; Chrome flags are defense in depth. Pure; the launcher is `services/htmlComposition/ownedBrowser.js`. |
+| `generatedPreviewPlaybackSchema.js` | Strict optional generated-preview clock, preparation, playback-anchor and pause message contracts; legacy open and seek payloads remain unchanged. |
| `codeAnimationPackage.js` | Versioned data-only Code Animation package schema, byte/path/reference integrity limits, SHA-256 revision binding, `createCodeAnimationPackage` and `summarizeCodeAnimationPackage`; validation never executes or stages code. |
| `socketEventContracts.js` | Payload contracts for modeled Socket.IO events, reusing live Zod validation for inbound events and documenting public outbound snapshots. |
| `socketEventInventory.js` | Cached source-derived Socket.IO event inventory: event names and directions only, with no checked-in manifest or positional source metadata. |
diff --git a/server/lib/generatedPreviewPlaybackSchema.js b/server/lib/generatedPreviewPlaybackSchema.js
new file mode 100644
index 0000000000..3458f51d7c
--- /dev/null
+++ b/server/lib/generatedPreviewPlaybackSchema.js
@@ -0,0 +1,18 @@
+/** Optional preview clock/playback messages. Legacy open and seek stay unchanged. */
+import { z } from 'zod';
+
+const identity = {
+ requestId: z.string().uuid(), sessionId: z.string().uuid(),
+};
+const anchor = {
+ ...identity, epoch: z.number().int().nonnegative(), sequence: z.number().int().nonnegative(),
+ t: z.number().min(0).max(86400), workerTime: z.number().finite(),
+};
+export const generatedPreviewPlaybackSchema = z.discriminatedUnion('action', [
+ z.object({ ...identity, action: z.literal('clock'), probeId: z.number().int().nonnegative(), clientTime: z.number().finite() }).strict(),
+ z.object({ ...identity, action: z.literal('prepare'), epoch: z.number().int().nonnegative(), sequence: z.number().int().nonnegative(), generation: z.number().int().nonnegative(), t: z.number().min(0).max(86400) }).strict(),
+ z.object({ ...anchor, action: z.literal('play') }).strict(),
+ z.object({ ...anchor, action: z.literal('sync') }).strict(),
+ z.object({ ...identity, action: z.literal('pause'), epoch: z.number().int().nonnegative(), generation: z.number().int().nonnegative(), t: z.number().min(0).max(86400) }).strict(),
+]);
+
diff --git a/server/lib/index.js b/server/lib/index.js
index 44cdda5af4..449d000a2a 100644
--- a/server/lib/index.js
+++ b/server/lib/index.js
@@ -23,6 +23,7 @@ export * from './codeAnimationPackage.js';
export * from './codeAnimationAcceptance.js';
export * from './codeAnimationContainment.js';
export * from './generatedPreviewSandbox.js';
+export * from './generatedPreviewPlaybackSchema.js';
export * from './autonomousJobTask.js';
export * from './asyncApiSpec.js';
export * from './avatarVariants.js';
diff --git a/server/lib/socketEventContracts.js b/server/lib/socketEventContracts.js
index 5326b09dc6..bca18e75ae 100644
--- a/server/lib/socketEventContracts.js
+++ b/server/lib/socketEventContracts.js
@@ -1,6 +1,7 @@
/** Runtime-backed payload contracts for modeled Socket.IO events. */
import { zodToOpenApiSchema } from './apiContractSchemas.js';
+import { generatedPreviewPlaybackSchema } from './generatedPreviewPlaybackSchema.js';
import {
appDeploySchema,
appStandardizeSchema,
@@ -44,6 +45,23 @@ const loomRunSnapshot = (production) => Object.freeze({
});
export const SOCKET_EVENT_CONTRACTS = Object.freeze({
+ 'generated-preview:playback': input(generatedPreviewPlaybackSchema, 'Calibrate or control an advertised cooperative document preview; audio remains in the client.'),
+ 'generated-preview:playback-state': {
+ direction: 'server-to-client',
+ summary: 'Correlated preview clock, preparation, playback and synchronization result.',
+ payloadSchema: {
+ type: 'object', required: ['requestId', 'sessionId', 'action'],
+ properties: {
+ requestId: { type: 'string', format: 'uuid' }, sessionId: { type: 'string', format: 'uuid' },
+ action: { type: 'string', enum: ['clock', 'prepare', 'play', 'sync'] },
+ probeId: { type: 'integer', minimum: 0 }, clientTime: { type: 'number' }, workerTime: { type: 'number' },
+ epoch: { type: 'integer', minimum: 0 }, sequence: { type: 'integer', minimum: 0 },
+ status: { type: 'string', enum: ['ready', 'playing', 'accepted', 'stale', 'resync-required', 'error'] },
+ message: { type: 'string' },
+ reason: { type: 'string', enum: ['controls', 'clock'] },
+ },
+ },
+ },
'messages:sync:completed': {
direction: 'server-to-client',
summary: 'Account cache persisted after sync; visible inbox consumers reconcile a bounded summary page. No message bodies.',
diff --git a/server/services/generatedPreview/cooperativePlayback.js b/server/services/generatedPreview/cooperativePlayback.js
new file mode 100644
index 0000000000..5516ecfd48
--- /dev/null
+++ b/server/services/generatedPreview/cooperativePlayback.js
@@ -0,0 +1,295 @@
+/**
+ * Optional document playback. Exact seeks and native capture lifetimes share
+ * one queue; only the page's cooperative producer advances playing frames.
+ */
+import { generatedPreviewPlaybackSchema as playbackSchema } from '../../lib/generatedPreviewPlaybackSchema.js';
+export { playbackSchema };
+
+const DRAIN_MS = 1500;
+const START_FENCE_MS = 750;
+const nativeCall = (page, method, args = {}) => page.evaluate(
+ `globalThis.portosComposition.previewPlayback.${method}(${JSON.stringify(args)})`,
+);
+
+export function createCooperativePlayback(session, { seekPage, capture, emitFrame, fail, closeSession }) {
+ let tail = Promise.resolve();
+ let intent = 0;
+ let epoch = 0;
+ let anchorSequence = -1;
+ let frameSequence = 0;
+ let mode = 'paused';
+ let stream = null;
+ let active = false;
+ let closed = false;
+ let time = 0;
+ let acceptedGeneration = 0;
+ let controlRevision = 0;
+ let exactBarrier = 0;
+ let resyncEpoch = null;
+
+ const enqueue = (operation) => {
+ const result = tail.catch(() => {}).then(() => closed ? undefined : operation());
+ tail = result;
+ return result;
+ };
+ const reply = (payload) => {
+ if (!closed && !session.closed) session.socket.emit('generated-preview:playback-state', {
+ requestId: session.requestId, sessionId: session.id, ...payload,
+ });
+ };
+ const gate = () => { if (stream) stream.forward = false; };
+ const terminate = (error) => {
+ if (closed || session.closed) return;
+ gate();
+ mode = 'error';
+ console.error(`❌ Cooperative preview failed: ${error?.message || error}`);
+ fail(session.socket, { requestId: session.requestId, sessionId: session.id, code: 'PREVIEW_PLAYBACK_FAILED', message: 'Live preview stopped. Retry the preview to continue.' });
+ closeSession(session.id, 'playback-failed').catch(() => {});
+ };
+
+ async function stopStream() {
+ const previous = stream;
+ if (!previous) return;
+ previous.forward = false;
+ previous.draining = true;
+ await Promise.all([...previous.acks]);
+ if (closed) return;
+ if (!previous.held) {
+ // A forced repaint is used only at a state transition. Holding the sole
+ // completed cap=1 event proves no older JPEG encoder can be relabelled
+ // by the next Start. ACK alone is not a raw-capture freshness barrier.
+ const waiting = new Promise((resolve, reject) => {
+ previous.resolveHeld = resolve;
+ previous.rejectHeld = reject;
+ previous.timer = setTimeout(() => reject(new Error('Native playback capture did not drain')), DRAIN_MS);
+ });
+ const repaint = capture(session.page);
+ try { await Promise.all([waiting, repaint]); }
+ finally { clearTimeout(previous.timer); previous.resolveHeld = null; previous.rejectHeld = null; }
+ }
+ if (closed) return;
+ await session.page.send('Page.stopScreencast');
+ // Deliberately retire the held credit without ACKing a stopped lifetime.
+ if (stream === previous) stream = null;
+ }
+
+ async function quiesce() {
+ gate();
+ if (stream) stream.draining = true;
+ await nativeCall(session.page, 'pause', { epoch });
+ await stopStream();
+ }
+
+ const beginClockResync = (action, sequence) => {
+ if (closed || session.closed || resyncEpoch === epoch) return;
+ const failedEpoch = epoch;
+ gate();
+ mode = 'paused';
+ resyncEpoch = failedEpoch;
+ // Queue retirement before notifying the master. Its next prepare must
+ // follow this drain, but its audio must stop without awaiting that drain.
+ // Do not return this tail from a currently queued operation: it would
+ // await itself through the next queue entry.
+ enqueue(() => quiesce()).catch(terminate);
+ reply({ action, epoch: failedEpoch, sequence, status: 'resync-required', reason: 'clock', message: 'Preview clock needs synchronization.' });
+ };
+
+ async function startStream() {
+ const next = { epoch, forward: false, draining: false, held: null, acks: new Set(), lifetime: null };
+ stream = next;
+ await session.page.send('Page.startScreencast', {
+ format: 'jpeg', quality: 55, everyNthFrame: 1, maxFramesInFlight: 1, sendLastFrame: false,
+ });
+ }
+
+ async function startAfterMotionFence(valid) {
+ let timer;
+ try {
+ return await Promise.race([
+ (async () => {
+ // Capture only on this transition, after motion mode is active. The
+ // next capturer must not bootstrap from the expensive exact surface.
+ await capture(session.page);
+ if (!valid()) return false;
+ await startStream();
+ return valid();
+ })(),
+ new Promise((_, reject) => { timer = setTimeout(() => reject(new Error('Native playback startup repaint timed out')), START_FENCE_MS); }),
+ ]);
+ } finally { clearTimeout(timer); }
+ }
+
+ const onFrame = (params) => {
+ const current = stream;
+ if (closed || !current || !params?.data) return;
+ const lifetime = params.sessionId;
+ if (!Number.isInteger(lifetime) || (current.lifetime !== null && current.lifetime !== lifetime)) {
+ terminate(new Error('Native capture lifetime changed unexpectedly'));
+ return;
+ }
+ current.lifetime = lifetime;
+ if (current.draining) {
+ if (current.held) { terminate(new Error('Native capture exceeded one in-flight frame')); return; }
+ current.held = params;
+ current.resolveHeld?.();
+ return;
+ }
+ if (current.forward && mode === 'playing' && current.epoch === epoch) {
+ emitFrame(session, null, params.data, { playbackEpoch: epoch, sequence: ++frameSequence });
+ }
+ const ack = session.page.send('Page.screencastFrameAck', { sessionId: lifetime });
+ current.acks.add(ack);
+ ack.then(() => current.acks.delete(ack), (error) => { current.acks.delete(ack); terminate(error); });
+ };
+
+ const seek = (generation, t, playing = false) => {
+ if (generation < acceptedGeneration) return Promise.resolve();
+ acceptedGeneration = generation;
+ time = t;
+ const current = ++intent;
+ const legacyPlaying = playing && !active;
+ if (!legacyPlaying || mode !== 'legacy-playing') exactBarrier += 1;
+ const barrier = exactBarrier;
+ gate();
+ mode = legacyPlaying ? 'legacy-playing' : 'paused';
+ const validCompletion = () => current === intent || legacyPlaying && mode === 'legacy-playing' && barrier === exactBarrier;
+ return enqueue(async () => {
+ await quiesce();
+ if (current !== intent) return;
+ await seekPage(session, t, playing);
+ if (!validCompletion() || closed) return;
+ const data = await capture(session.page);
+ if (!validCompletion() || closed) return;
+ session.appliedGeneration = generation;
+ emitFrame(session, generation, data);
+ });
+ };
+
+ const handle = (input) => {
+ if (input.requestId !== session.requestId) return Promise.resolve();
+ if (input.action === 'clock') return enqueue(async () => {
+ const workerTime = await nativeCall(session.page, 'clock');
+ reply({ action: 'clock', probeId: input.probeId, clientTime: input.clientTime, workerTime });
+ });
+ if (input.action === 'pause') {
+ if (input.epoch < epoch || input.generation < acceptedGeneration) return Promise.resolve();
+ epoch = input.epoch;
+ return seek(input.generation, input.t);
+ }
+ if (input.action === 'prepare') {
+ if (input.epoch <= epoch || input.generation < acceptedGeneration) return Promise.resolve();
+ acceptedGeneration = input.generation;
+ time = input.t;
+ active = true;
+ exactBarrier += 1;
+ epoch = input.epoch;
+ anchorSequence = input.sequence;
+ frameSequence = 0;
+ const current = ++intent;
+ const revision = controlRevision;
+ gate();
+ mode = 'preparing';
+ return enqueue(async () => {
+ await quiesce();
+ if (current !== intent) return;
+ await seekPage(session, input.t, false);
+ if (current !== intent) return;
+ const prepared = await nativeCall(session.page, 'prepare', { epoch: input.epoch, t: input.t });
+ if (current !== intent || revision !== controlRevision || prepared?.status === 'superseded') return;
+ const data = await capture(session.page);
+ if (current !== intent || revision !== controlRevision || closed) return;
+ time = input.t;
+ session.appliedGeneration = input.generation;
+ mode = 'prepared';
+ emitFrame(session, input.generation, data);
+ reply({ action: 'prepare', epoch, sequence: input.sequence, status: 'ready' });
+ });
+ }
+ if (input.epoch !== epoch || input.sequence <= anchorSequence || !['prepared', 'starting', 'playing'].includes(mode)) return Promise.resolve();
+ anchorSequence = input.sequence;
+ const current = intent;
+ const revision = controlRevision;
+ return enqueue(async () => {
+ if (current !== intent || revision !== controlRevision || input.epoch !== epoch) return;
+ if (input.action === 'play') {
+ if (mode !== 'prepared') return;
+ mode = 'starting';
+ const result = await nativeCall(session.page, 'play', { epoch: input.epoch, t: input.t, workerTime: input.workerTime, sequence: input.sequence });
+ if (current !== intent || revision !== controlRevision || mode !== 'starting' || closed) return;
+ if (result?.status === 'resync-required') {
+ beginClockResync('play', input.sequence);
+ return;
+ }
+ if (result?.status !== 'playing') throw new Error('Native playback did not start');
+ const valid = () => current === intent && revision === controlRevision && mode === 'starting' && input.epoch === epoch && !closed;
+ if (!await startAfterMotionFence(valid)) return;
+ mode = 'playing';
+ time = input.t;
+ if (stream) stream.forward = true;
+ reply({ action: 'play', epoch, sequence: input.sequence, status: 'playing' });
+ } else if (mode === 'playing') {
+ const result = await nativeCall(session.page, 'sync', { epoch: input.epoch, t: input.t, workerTime: input.workerTime, sequence: input.sequence });
+ if (current !== intent || revision !== controlRevision || mode !== 'playing' || closed) return;
+ time = input.t;
+ if (result?.status === 'resync-required') {
+ beginClockResync('sync', input.sequence);
+ return;
+ }
+ if (current !== intent || input.epoch !== epoch || closed) return;
+ reply({ action: 'sync', epoch: input.epoch, sequence: input.sequence, status: result?.status || 'error' });
+ }
+ });
+ };
+
+ const control = (operation) => {
+ const needsResync = ['preparing', 'prepared', 'starting', 'playing'].includes(mode);
+ const controlEpoch = epoch;
+ const controlSequence = anchorSequence;
+ const revision = ++controlRevision;
+ exactBarrier += 1;
+ const current = intent;
+ gate();
+ mode = 'paused';
+ const queued = enqueue(async () => {
+ await quiesce();
+ if (closed) return;
+ // Mutations retain queue order even when their now-obsolete image is
+ // superseded by a later control or seek.
+ await operation();
+ if (current !== intent || revision !== controlRevision || closed) return;
+ await seekPage(session, time, false);
+ if (current !== intent || revision !== controlRevision || closed) return;
+ const data = await capture(session.page);
+ if (current !== intent || revision !== controlRevision || closed) return;
+ session.appliedGeneration = acceptedGeneration;
+ emitFrame(session, acceptedGeneration, data);
+ });
+ // Queue the mutation first, then cancel the client's preparation/playing
+ // handshake. Its replacement prepare necessarily follows this mutation.
+ if (needsResync && resyncEpoch !== controlEpoch) {
+ resyncEpoch = controlEpoch;
+ reply({ action: 'sync', epoch: controlEpoch, sequence: controlSequence, status: 'resync-required', reason: 'controls', message: 'Preview controls changed.' });
+ }
+ return queued;
+ };
+
+ const onEvent = (params) => {
+ if (params?.epoch !== epoch || !active) return;
+ if (params.type === 'portos-mv:playback-error') terminate(new Error(params.message || 'Native playback failed'));
+ else if (params.type === 'portos-mv:playback-resync' && ['preparing', 'prepared', 'starting', 'playing'].includes(mode)) beginClockResync('sync', anchorSequence);
+ };
+
+ return {
+ get active() { return active; }, handle, seek, control, onFrame, onEvent, terminate,
+ close() {
+ closed = true;
+ intent += 1;
+ gate();
+ if (stream) {
+ clearTimeout(stream.timer);
+ stream.rejectHeld?.(new Error('Preview closed'));
+ }
+ },
+ async settled() { await Promise.allSettled([tail, ...(stream ? stream.acks : [])]); },
+ };
+}
diff --git a/server/services/generatedPreview/sessions.js b/server/services/generatedPreview/sessions.js
index 40daa0acd4..24dabeb446 100644
--- a/server/services/generatedPreview/sessions.js
+++ b/server/services/generatedPreview/sessions.js
@@ -12,6 +12,7 @@ import { join } from 'node:path';
import { z } from 'zod';
import { PATHS } from '../../lib/fileUtils.js';
import { PREVIEW_PAGE_CSP } from '../../lib/generatedPreviewSandbox.js';
+import { createCooperativePlayback, playbackSchema } from './cooperativePlayback.js';
const MAX_SESSIONS = 3;
const IDLE_MS = 120_000;
@@ -133,6 +134,7 @@ export async function closeSession(sessionId, reason = 'closed') {
session.closed = true;
sessions.delete(sessionId);
clearTimeout(session.idle);
+ session.playback?.close();
const ids = bySocket.get(session.socket.id);
if (ids) {
const next = ids.filter((id) => id !== sessionId);
@@ -143,6 +145,7 @@ export async function closeSession(sessionId, reason = 'closed') {
try { await session.page?.close(); } catch (error) {
console.error(`❌ Generated preview browser close failed: ${error.message}`);
}
+ await session.playback?.settled();
if (session.scratchRoot) {
await rm(session.scratchRoot, { recursive: true, force: true }).catch((error) => {
console.error(`❌ Generated preview scratch cleanup failed: ${error.message}`);
@@ -194,15 +197,20 @@ async function seekPage(session, t, playing = false) {
})()`);
}
-function emitFrame(session, generation, data) {
+function emitFrame(session, generation, data, playback = null) {
if (!data || session.closed) return;
session.socket.emit('generated-preview:frame', {
- requestId: session.requestId, sessionId: session.id, generation, mime: 'image/jpeg', data,
+ requestId: session.requestId, sessionId: session.id,
+ ...(playback || { generation }), mime: 'image/jpeg', data,
});
armIdle(session);
}
async function runSeek(session, generation, t, playing = false) {
+ if (session.playback) {
+ await session.playback.seek(generation, t, playing).catch(session.playback.terminate);
+ return;
+ }
if (session.seekTail) {
session.pendingSeek = { generation, t, playing };
return;
@@ -279,12 +287,14 @@ function attachEvents(session) {
if (session.closed) return;
if (message.method === 'Page.screencastFrame') {
const params = message.params || {};
+ if (session.playback) { session.playback.onFrame(params); return; }
emitFrame(session, session.appliedGeneration, params.data);
session.page?.send('Page.screencastFrameAck', { sessionId: params.sessionId }).catch(() => {});
return;
}
if (message.method === 'Portos.preview') {
const params = message.params || {};
+ session.playback?.onEvent(params);
session.socket.emit('generated-preview:event', {
requestId: session.requestId, sessionId: session.id, ...params,
});
@@ -415,6 +425,9 @@ async function startMusicVideo(session, input) {
});
session.page = opened.page;
session.scratchRoot = opened.scratchRoot;
+ if (opened.playback?.version === 1) {
+ session.playback = createCooperativePlayback(session, { seekPage, capture, emitFrame, fail, closeSession });
+ }
// The first frame is captured after this seek. Apply the live controls first
// so that frame is already graded and titled.
await applyJson(opened.page, '__portosFilmLook', input.look ?? null);
@@ -425,6 +438,7 @@ async function startMusicVideo(session, input) {
width: opened.width, height: opened.height, fps: opened.fps, durationSec: opened.durationSec,
revision: `${pointer?.directory || ''}|${pointer?.updatedAt || ''}`,
frame: await capture(opened.page),
+ ...(opened.playback ? { playback: opened.playback } : {}),
};
}
@@ -554,6 +568,7 @@ async function openGeneratedPreview(socket, data) {
durationSec: started.durationSec,
revision: started.revision,
audio: started.audio || 'none',
+ ...(started.playback ? { playback: started.playback } : {}),
});
emitFrame(session, 0, started.frame);
} catch (error) {
@@ -586,11 +601,18 @@ async function commandGeneratedPreview(socket, data) {
if (!session || session.socket.id !== socket.id || session.closed) return;
armIdle(session);
try {
- if (parsed.data.action === 'film-look') await applyJson(session.page, '__portosFilmLook', parsed.data.look ?? null);
- else if (parsed.data.action === 'title-card') await applyJson(session.page, '__portosTitleCard', parsed.data.card ?? null);
- else if (parsed.data.action === 'record' && parsed.data.message) {
- await session.page.evaluate(`window.postMessage({ type: ${JSON.stringify(parsed.data.message.type)} }, '*')`);
+ const apply = async () => {
+ if (parsed.data.action === 'film-look') await applyJson(session.page, '__portosFilmLook', parsed.data.look ?? null);
+ else if (parsed.data.action === 'title-card') await applyJson(session.page, '__portosTitleCard', parsed.data.card ?? null);
+ else if (parsed.data.action === 'record' && parsed.data.message) {
+ await session.page.evaluate(`window.postMessage({ type: ${JSON.stringify(parsed.data.message.type)} }, '*')`);
+ }
+ };
+ if (session.playback) {
+ await session.playback.control(apply).catch(session.playback.terminate);
+ return;
}
+ await apply();
if (session.kind !== 'code-animation') emitFrame(session, session.appliedGeneration, await capture(session.page));
} catch (error) {
console.error(`❌ Generated preview command failed: ${error.message}`);
@@ -598,18 +620,32 @@ async function commandGeneratedPreview(socket, data) {
}
}
+async function playbackGeneratedPreview(socket, data) {
+ const parsed = playbackSchema.safeParse(data);
+ if (!parsed.success) return;
+ const session = sessions.get(parsed.data.sessionId);
+ if (!session || session.socket.id !== socket.id || session.closed || !session.playback) return;
+ armIdle(session);
+ await session.playback.handle(parsed.data).catch(session.playback.terminate);
+}
+
function guard(socket, event, fn) {
socket.on(event, (data) => {
- Promise.resolve().then(() => fn(socket, data)).catch((error) => {
+ const rejected = (error) => {
console.error(`❌ Generated preview ${event} failed: ${error.message}`);
- fail(socket, { message: 'The isolated preview could not continue.' });
- });
+ fail(socket, { requestId: data?.requestId, sessionId: data?.sessionId, message: 'The isolated preview could not continue.' });
+ };
+ // Intent gates run in the receiving event turn, before another native
+ // frame can be forwarded. The async operation still reports rejections.
+ try { Promise.resolve(fn(socket, data)).catch(rejected); }
+ catch (error) { rejected(error); }
});
}
export function registerGeneratedPreviewHandlers(socket) {
guard(socket, 'generated-preview:open', openGeneratedPreview);
guard(socket, 'generated-preview:seek', seekGeneratedPreview);
+ guard(socket, 'generated-preview:playback', playbackGeneratedPreview);
guard(socket, 'generated-preview:command', commandGeneratedPreview);
guard(socket, 'generated-preview:close', async (current, data) => {
const parsed = closeSchema.safeParse(data || {});
diff --git a/server/services/generatedPreview/sessions.test.js b/server/services/generatedPreview/sessions.test.js
index b2c12a4675..c53d1d7b47 100644
--- a/server/services/generatedPreview/sessions.test.js
+++ b/server/services/generatedPreview/sessions.test.js
@@ -1,14 +1,16 @@
import { afterEach, describe, expect, it, vi } from 'vitest';
-const fixture = vi.hoisted(() => ({ page: null }));
+const fixture = vi.hoisted(() => ({ page: null, playback: null, onEvent: null }));
vi.mock('../musicVideo/projects.js', () => ({ getProject: async () => ({ composition: { document: { directory: 'example-document' } } }) }));
-vi.mock('../musicVideo/documentRender.js', () => ({ openIsolatedDocumentPreview: async () => ({
- page: fixture.page, scratchRoot: null, width: 390, height: 219, fps: 30, durationSec: 10,
-}) }));
+vi.mock('../musicVideo/documentRender.js', () => ({ openIsolatedDocumentPreview: async ({ onEvent }) => {
+ fixture.onEvent = onEvent;
+ return { page: fixture.page, scratchRoot: null, width: 390, height: 219, fps: 30, durationSec: 10,
+ ...(fixture.playback ? { playback: fixture.playback } : {}) };
+} }));
import { closeSession, registerGeneratedPreviewHandlers } from './sessions.js';
const socket = { id: 'preview-test-socket', handlers: new Map(), emit: vi.fn(), on(event, fn) { this.handlers.set(event, fn); } };
-afterEach(async () => { for (const [event, payload] of socket.emit.mock.calls) if (event === 'generated-preview:opened') await closeSession(payload.sessionId); socket.handlers.clear(); socket.emit.mockClear(); });
+afterEach(async () => { for (const [event, payload] of socket.emit.mock.calls) if (event === 'generated-preview:opened') await closeSession(payload.sessionId); socket.handlers.clear(); socket.emit.mockClear(); fixture.playback = null; });
// Catches playback state being lost at the queued worker boundary, especially the final pause behind an active seek.
describe('isolated document preview playback', () => {
@@ -53,3 +55,304 @@ describe('isolated document preview playback', () => {
expect(frame).toEqual({ t: 3, playing: false });
});
});
+
+// Uniquely catches capture resets on clock anchors, stale preparing epochs,
+// and old stream credits crossing an exact pause through the public socket.
+describe('cooperative playback through preview sockets', () => {
+ const requestId = '00000000-0000-4000-8000-000000000002';
+ const frames = () => socket.emit.mock.calls.filter(([name]) => name === 'generated-preview:frame').map(([, payload]) => payload);
+ const states = () => socket.emit.mock.calls.filter(([name]) => name === 'generated-preview:playback-state').map(([, payload]) => payload);
+ async function openedPage({ rejectPlay = false } = {}) {
+ let lifetime = 0;
+ let capturing = false;
+ let holdPrepare = null;
+ let heldSeek = null;
+ let heldCapture = null;
+ let heldPause = null;
+ let resyncMethod = null;
+ let resyncEvent = false;
+ let autoDrain = true;
+ const log = [];
+ const frame = { t: 0, playing: false };
+ const nativeFrame = (data = JSON.stringify(frame)) => fixture.onEvent({ method: 'Page.screencastFrame', params: { sessionId: lifetime, data } });
+ const globals = {
+ __portosFilmLook: { apply(value) { log.push(['look', value]); }, playback(on) { frame.playing = on; } }, __portosTitleCard: { apply(value) { log.push(['card', value]); } },
+ portosComposition: {
+ async seek(t) { log.push(['seek', t]); if (heldSeek?.t === t) await heldSeek.promise; frame.t = t; },
+ previewPlayback: {
+ clock: () => 123,
+ async pause() { log.push(['pause']); frame.playing = false; if (heldPause) await heldPause; },
+ async prepare({ epoch, t }) { log.push(['prepare', epoch, t]); if (holdPrepare) await holdPrepare; return { status: 'prepared', epoch }; },
+ async play({ epoch, sequence }) {
+ if (rejectPlay) throw new Error('Synthetic native play rejection'); log.push(['play', epoch]); frame.playing = true;
+ if (resyncMethod === 'play' && resyncEvent) fixture.onEvent({ method: 'Portos.preview', params: { type: 'portos-mv:playback-resync', epoch, sequence } });
+ return { status: resyncMethod === 'play' ? 'resync-required' : 'playing', epoch };
+ },
+ sync({ epoch, sequence }) {
+ log.push(['sync', epoch]);
+ if (resyncMethod === 'sync' && resyncEvent) fixture.onEvent({ method: 'Portos.preview', params: { type: 'portos-mv:playback-resync', epoch, sequence } });
+ return { status: resyncMethod === 'sync' ? 'resync-required' : 'accepted', epoch };
+ },
+ },
+ },
+ };
+ fixture.playback = { version: 1, fps: 30, presentation: 'atomic-canvas-v1', mediaTiming: 'staged-output-v1' };
+ fixture.page = {
+ evaluate: (source) => new Function('globalThis', 'window', 'return ' + source)(globals, { postMessage() {} }),
+ send: async (method) => {
+ log.push([method]);
+ if (method === 'Page.startScreencast') { lifetime += 1; capturing = true; }
+ if (method === 'Page.stopScreencast') capturing = false;
+ if (method === 'Page.captureScreenshot') { if (capturing && autoDrain) nativeFrame(); if (heldCapture) await heldCapture; return { data: JSON.stringify(frame) }; }
+ return {};
+ },
+ close: vi.fn(async () => { log.push(['close']); }),
+ };
+ registerGeneratedPreviewHandlers(socket);
+ socket.handlers.get('generated-preview:open')({ requestId, kind: 'music-video', projectId: 'example' });
+ await vi.waitFor(() => expect(socket.emit).toHaveBeenCalledWith('generated-preview:opened', expect.objectContaining({ playback: { ...fixture.playback } })));
+ const sessionId = socket.emit.mock.calls.find(([name]) => name === 'generated-preview:opened')[1].sessionId;
+ const send = (payload) => socket.handlers.get('generated-preview:playback')({ requestId, sessionId, ...payload });
+ return { sessionId, send, nativeFrame, frame, log, hold(promise) { holdPrepare = promise; }, holdSeek(t, promise) { heldSeek = { t, promise }; }, holdCapture(promise) { heldCapture = promise; }, holdPause(promise) { heldPause = promise; }, nativeResync(method, event = false) { resyncMethod = method; resyncEvent = event; }, blockDrain() { autoDrain = false; } };
+ }
+
+ it.each(['event', 'play-return', 'sync-return', 'play-race', 'sync-race'])('notifies the master before owned pause retires and deduplicates native resync (%s)', async scenario => {
+ const page = await openedPage();
+ page.send({ action: 'prepare', epoch: 1, sequence: 1, generation: 1, t: 1 });
+ await vi.waitFor(() => expect(states()).toContainEqual(expect.objectContaining({ status: 'ready' })));
+ const playFailure = scenario.startsWith('play');
+ if (!playFailure) {
+ page.send({ action: 'play', epoch: 1, sequence: 2, t: 1, workerTime: 123 });
+ await vi.waitFor(() => expect(states()).toContainEqual(expect.objectContaining({ status: 'playing' })));
+ }
+ const pauses = page.log.filter(([name]) => name === 'pause').length;
+ let release;
+ page.holdPause(new Promise(resolve => { release = resolve; }));
+ try {
+ if (scenario === 'event') fixture.onEvent({ method: 'Portos.preview', params: { type: 'portos-mv:playback-resync', epoch: 1, sequence: 2 } });
+ else {
+ const action = playFailure ? 'play' : 'sync';
+ page.nativeResync(action, scenario.endsWith('race'));
+ page.send({ action, epoch: 1, sequence: playFailure ? 2 : 3, t: 1, workerTime: 123 });
+ }
+ await vi.waitFor(() => expect(states().filter(state => state.status === 'resync-required')).toHaveLength(1));
+ expect(states().find(state => state.status === 'resync-required')).toMatchObject({ requestId, sessionId: page.sessionId, epoch: 1, reason: 'clock' });
+ await vi.waitFor(() => expect(page.log.filter(([name]) => name === 'pause')).toHaveLength(pauses + 1));
+ const count = frames().length;
+ page.nativeFrame('stale-after-resync');
+ fixture.onEvent({ method: 'Portos.preview', params: { type: 'portos-mv:playback-resync', epoch: 1, sequence: 3 } });
+ expect(frames()).toHaveLength(count);
+ expect(states().filter(state => state.status === 'resync-required')).toHaveLength(1);
+ page.send({ action: 'prepare', epoch: 2, sequence: 1, generation: 2, t: 2 });
+ await Promise.resolve();
+ expect(page.log).not.toContainEqual(['prepare', 2, 2]);
+ } finally { release(); }
+ await vi.waitFor(() => expect(states()).toContainEqual(expect.objectContaining({ action: 'prepare', epoch: 2, status: 'ready' })));
+ expect(page.log.filter(([name]) => name === 'pause')).toHaveLength(pauses + 2);
+ expect(states().filter(state => state.status === 'resync-required')).toHaveLength(1);
+ });
+
+ it('plays continuously between ordered anchors, drains once, and a pause supersedes asynchronous preparation', async () => {
+ const page = await openedPage();
+ page.send({ action: 'clock', probeId: 1, clientTime: 100 });
+ await vi.waitFor(() => expect(states()).toContainEqual(expect.objectContaining({ action: 'clock', workerTime: 123, clientTime: 100 })));
+ page.send({ action: 'prepare', epoch: 1, sequence: 1, generation: 1, t: 1 });
+ await vi.waitFor(() => expect(states()).toContainEqual(expect.objectContaining({ action: 'prepare', epoch: 1, status: 'ready' })));
+ page.send({ action: 'play', epoch: 1, sequence: 2, t: 1, workerTime: 123 });
+ await vi.waitFor(() => expect(states()).toContainEqual(expect.objectContaining({ action: 'play', status: 'playing' })));
+ const afterStart = page.log.length;
+ page.nativeFrame('complete-frame-a');
+ page.send({ action: 'sync', epoch: 1, sequence: 3, t: 1.25, workerTime: 373 });
+ page.send({ action: 'sync', epoch: 1, sequence: 2, t: 0, workerTime: 0 });
+ await vi.waitFor(() => expect(page.log).toContainEqual(['sync', 1]));
+ page.nativeFrame('complete-frame-b');
+ expect(frames().slice(-2)).toMatchObject([{ playbackEpoch: 1, sequence: 1, data: 'complete-frame-a' }, { playbackEpoch: 1, sequence: 2, data: 'complete-frame-b' }]);
+ expect(frames().slice(-2).every(frame => !('generation' in frame))).toBe(true);
+ expect(page.log.slice(afterStart).filter(([name]) => ['seek', 'Page.captureScreenshot', 'Page.startScreencast'].includes(name))).toEqual([]);
+ let release;
+ page.hold(new Promise(resolve => { release = resolve; }));
+ page.send({ action: 'prepare', epoch: 2, sequence: 1, generation: 2, t: 2 });
+ page.nativeFrame('queued-old-frame');
+ await vi.waitFor(() => expect(page.log).toContainEqual(['prepare', 2, 2]));
+ socket.handlers.get('generated-preview:seek')({ requestId, sessionId: page.sessionId, generation: 3, t: 3 });
+ await Promise.resolve();
+ release();
+ await vi.waitFor(() => expect(frames().at(-1)).toMatchObject({ generation: 3, data: JSON.stringify({ t: 3, playing: false }) }));
+ expect(frames().some(frame => frame.generation === 2 || frame.data === 'queued-old-frame')).toBe(false);
+ expect(page.log.filter(([name]) => name === 'Page.startScreencast')).toHaveLength(1);
+ expect(page.log.filter(([name]) => name === 'Page.stopScreencast')).toHaveLength(1);
+ });
+
+ it('closes a rejected native play with a correlated terminal error and no retry', async () => {
+ const page = await openedPage({ rejectPlay: true });
+ page.send({ action: 'prepare', epoch: 1, sequence: 1, generation: 1, t: 1 });
+ await vi.waitFor(() => expect(states()).toContainEqual(expect.objectContaining({ status: 'ready' })));
+ page.send({ action: 'play', epoch: 1, sequence: 2, t: 1, workerTime: 123 });
+ await vi.waitFor(() => expect(socket.emit).toHaveBeenCalledWith('generated-preview:closed', { sessionId: page.sessionId, reason: 'playback-failed' }));
+ expect(socket.emit).toHaveBeenCalledWith('generated-preview:error', expect.objectContaining({ requestId, sessionId: page.sessionId, code: 'PREVIEW_PLAYBACK_FAILED' }));
+ expect(page.log.filter(([name]) => name === 'Page.startScreencast')).toHaveLength(0);
+ expect(fixture.page.close).toHaveBeenCalledTimes(1);
+ });
+
+ it('never starts a stream when native resync cancels the startup repaint fence', async () => {
+ const page = await openedPage();
+ page.send({ action: 'prepare', epoch: 1, sequence: 1, generation: 1, t: 1 });
+ await vi.waitFor(() => expect(states()).toContainEqual(expect.objectContaining({ status: 'ready' })));
+ let release;
+ page.holdCapture(new Promise(resolve => { release = resolve; }));
+ const count = page.log.filter(([name]) => name === 'Page.captureScreenshot').length;
+ page.send({ action: 'play', epoch: 1, sequence: 2, t: 1, workerTime: 123 });
+ await vi.waitFor(() => expect(page.log.filter(([name]) => name === 'Page.captureScreenshot').length).toBe(count + 1));
+ fixture.onEvent({ method: 'Portos.preview', params: { type: 'portos-mv:playback-resync', epoch: 1, sequence: 2 } });
+ release();
+ await vi.waitFor(() => expect(states()).toContainEqual(expect.objectContaining({ status: 'resync-required', reason: 'clock' })));
+ expect(page.log.filter(([name]) => name === 'Page.startScreencast')).toHaveLength(0);
+ expect(states().some(state => state.status === 'playing')).toBe(false);
+ });
+
+ it('joins an earlier exact seek, retains both control mutations and rejects older pauses', async () => {
+ const page = await openedPage();
+ let release;
+ page.holdSeek(1, new Promise(resolve => { release = resolve; }));
+ const seek = (generation, t) => socket.handlers.get('generated-preview:seek')({ requestId, sessionId: page.sessionId, generation, t });
+ seek(1, 1);
+ await vi.waitFor(() => expect(page.log).toContainEqual(['seek', 1]));
+ page.send({ action: 'prepare', epoch: 1, sequence: 1, generation: 2, t: 2 });
+ await Promise.resolve();
+ expect(page.log.some(([name]) => name === 'prepare')).toBe(false);
+ release();
+ await vi.waitFor(() => expect(states()).toContainEqual(expect.objectContaining({ action: 'prepare', status: 'ready' })));
+ const command = (action, value) => socket.handlers.get('generated-preview:command')({ requestId, sessionId: page.sessionId, action, [action === 'film-look' ? 'look' : 'card']: value });
+ command('film-look', { exposure: 0.5 });
+ command('title-card', { enabled: true });
+ seek(3, 3);
+ page.send({ action: 'pause', epoch: 0, generation: 1, t: 0 });
+ page.send({ action: 'pause', epoch: 1, generation: 2, t: 0 });
+ await vi.waitFor(() => expect(frames().at(-1)).toMatchObject({ generation: 3, data: JSON.stringify({ t: 3, playing: false }) }));
+ expect(page.log).toContainEqual(['look', { exposure: 0.5 }]);
+ expect(page.log).toContainEqual(['card', { enabled: true }]);
+ expect(page.log.filter(([name]) => name === 'seek').at(-1)).toEqual(['seek', 3]);
+ });
+
+ it('retains an in-flight seek position when a control changes and cancels obsolete control images', async () => {
+ const page = await openedPage();
+ let release;
+ page.holdSeek(2, new Promise(resolve => { release = resolve; }));
+ const seek = (generation, t) => socket.handlers.get('generated-preview:seek')({ requestId, sessionId: page.sessionId, generation, t });
+ const look = () => socket.handlers.get('generated-preview:command')({ requestId, sessionId: page.sessionId, action: 'film-look', look: { exposure: 0.25 } });
+ seek(1, 2);
+ await vi.waitFor(() => expect(page.log).toContainEqual(['seek', 2]));
+ look();
+ release();
+ await vi.waitFor(() => expect(frames().at(-1)).toMatchObject({ generation: 1, data: JSON.stringify({ t: 2, playing: false }) }));
+ const screenshots = page.log.filter(([name]) => name === 'Page.captureScreenshot').length;
+ let releaseCapture;
+ page.holdCapture(new Promise(resolve => { releaseCapture = resolve; }));
+ look();
+ await vi.waitFor(() => expect(page.log.filter(([name]) => name === 'Page.captureScreenshot').length).toBe(screenshots + 1));
+ const count = frames().length;
+ seek(2, 3);
+ releaseCapture();
+ await vi.waitFor(() => expect(frames().at(-1)).toMatchObject({ generation: 2, data: JSON.stringify({ t: 3, playing: false }) }));
+ expect(frames().slice(count).map(frame => frame.generation)).toEqual([2]);
+ });
+
+ it('retains paused control mutations in order while skipping obsolete queued pictures', async () => {
+ const page = await openedPage();
+ const controlStart = page.log.length;
+ const command = (action, value) => socket.handlers.get('generated-preview:command')({ requestId, sessionId: page.sessionId, action, [action === 'film-look' ? 'look' : 'card']: value });
+ let release;
+ page.holdCapture(new Promise(resolve => { release = resolve; }));
+ const screenshots = page.log.filter(([name]) => name === 'Page.captureScreenshot').length, count = frames().length;
+ command('film-look', { exposure: 0.1 });
+ await vi.waitFor(() => expect(page.log.filter(([name]) => name === 'Page.captureScreenshot')).toHaveLength(screenshots + 1));
+ command('title-card', { enabled: true });
+ command('film-look', { exposure: 0.9 });
+ release();
+ await vi.waitFor(() => expect(frames()).toHaveLength(count + 1));
+ expect(page.log.slice(controlStart).filter(([name]) => name === 'look' || name === 'card')).toEqual([
+ ['look', { exposure: 0.1 }], ['card', { enabled: true }], ['look', { exposure: 0.9 }],
+ ]);
+ expect(page.log.filter(([name]) => name === 'Page.captureScreenshot')).toHaveLength(screenshots + 2);
+ });
+
+ it('preserves a newer preparation mode while an older queued control retires', async () => {
+ const page = await openedPage();
+ const controlStart = page.log.length;
+ const command = action => socket.handlers.get('generated-preview:command')({ requestId, sessionId: page.sessionId, action, ...(action === 'film-look' ? { look: { exposure: 0.25 } } : { card: { enabled: true } }) });
+ let releaseCapture, releasePrepare;
+ page.holdCapture(new Promise(resolve => { releaseCapture = resolve; }));
+ const screenshots = page.log.filter(([name]) => name === 'Page.captureScreenshot').length;
+ command('film-look');
+ await vi.waitFor(() => expect(page.log.filter(([name]) => name === 'Page.captureScreenshot')).toHaveLength(screenshots + 1));
+ command('title-card');
+ page.hold(new Promise(resolve => { releasePrepare = resolve; }));
+ page.send({ action: 'prepare', epoch: 1, sequence: 1, generation: 1, t: 1 });
+ releaseCapture();
+ try {
+ await vi.waitFor(() => expect(page.log).toContainEqual(['prepare', 1, 1]));
+ command('film-look');
+ expect(states()).toContainEqual(expect.objectContaining({ epoch: 1, status: 'resync-required', reason: 'controls' }));
+ page.send({ action: 'prepare', epoch: 2, sequence: 2, generation: 1, t: 1 });
+ } finally { releasePrepare(); }
+ await vi.waitFor(() => expect(states()).toContainEqual(expect.objectContaining({ action: 'prepare', epoch: 2, status: 'ready' })));
+ expect(states().some(state => state.action === 'prepare' && state.epoch === 1 && state.status === 'ready')).toBe(false);
+ expect(page.log.slice(controlStart).filter(([name]) => name === 'look' || name === 'card')).toEqual([
+ ['look', { exposure: 0.25 }], ['card', { enabled: true }], ['look', { exposure: 0.25 }],
+ ]);
+ });
+
+ it('resynchronizes a control edit during preparation before any obsolete ready acknowledgement', async () => {
+ const page = await openedPage();
+ let release;
+ page.hold(new Promise(resolve => { release = resolve; }));
+ page.send({ action: 'prepare', epoch: 1, sequence: 1, generation: 1, t: 1 });
+ await vi.waitFor(() => expect(page.log).toContainEqual(['prepare', 1, 1]));
+ socket.handlers.get('generated-preview:command')({ requestId, sessionId: page.sessionId, action: 'title-card', card: { enabled: true } });
+ expect(states()).toContainEqual(expect.objectContaining({ action: 'sync', epoch: 1, status: 'resync-required', reason: 'controls' }));
+ page.send({ action: 'prepare', epoch: 2, sequence: 2, generation: 1, t: 1 });
+ release();
+ await vi.waitFor(() => expect(states()).toContainEqual(expect.objectContaining({ action: 'prepare', epoch: 2, status: 'ready' })));
+ expect(states().some(state => state.action === 'prepare' && state.epoch === 1 && state.status === 'ready')).toBe(false);
+ expect(page.log.findIndex(([name, value]) => name === 'card' && value?.enabled)).toBeLessThan(page.log.findIndex(([name, value]) => name === 'prepare' && value === 2));
+ });
+
+ it('delivers complete legacy playing frames between requests but never crosses pause into a new interval', async () => {
+ const page = await openedPage();
+ let release;
+ page.holdSeek(1, new Promise(resolve => { release = resolve; }));
+ const seek = (generation, t, playing = false) => socket.handlers.get('generated-preview:seek')({ requestId, sessionId: page.sessionId, generation, t, playing });
+ seek(1, 1, true);
+ await vi.waitFor(() => expect(page.log).toContainEqual(['seek', 1]));
+ seek(2, 2, true);
+ release();
+ await vi.waitFor(() => expect(frames().at(-1)).toMatchObject({ generation: 2 }));
+ expect(frames()).toContainEqual(expect.objectContaining({ generation: 1, data: JSON.stringify({ t: 1, playing: true }) }));
+ let releaseAgain;
+ page.holdSeek(4, new Promise(resolve => { releaseAgain = resolve; }));
+ seek(3, 4, true);
+ await vi.waitFor(() => expect(page.log).toContainEqual(['seek', 4]));
+ seek(4, 2);
+ seek(5, 3, true);
+ releaseAgain();
+ await vi.waitFor(() => expect(frames().at(-1)).toMatchObject({ generation: 5 }));
+ expect(frames().some(frame => frame.generation === 3 || frame.generation === 4)).toBe(false);
+ });
+
+ it('closes visibly when the sole native capture cannot drain within the transition bound', async () => {
+ const page = await openedPage();
+ page.send({ action: 'prepare', epoch: 1, sequence: 1, generation: 1, t: 1 });
+ await vi.waitFor(() => expect(states()).toContainEqual(expect.objectContaining({ status: 'ready' })));
+ page.send({ action: 'play', epoch: 1, sequence: 2, t: 1, workerTime: 123 });
+ await vi.waitFor(() => expect(states()).toContainEqual(expect.objectContaining({ status: 'playing' })));
+ page.blockDrain();
+ vi.useFakeTimers();
+ try {
+ socket.handlers.get('generated-preview:seek')({ requestId, sessionId: page.sessionId, generation: 2, t: 2 });
+ await vi.advanceTimersByTimeAsync(1501);
+ expect(socket.emit).toHaveBeenCalledWith('generated-preview:closed', { sessionId: page.sessionId, reason: 'playback-failed' });
+ expect(socket.emit).toHaveBeenCalledWith('generated-preview:error', expect.objectContaining({ code: 'PREVIEW_PLAYBACK_FAILED' }));
+ expect(page.log.filter(([name]) => name === 'Page.startScreencast')).toHaveLength(1);
+ } finally { vi.useRealTimers(); }
+ });
+});
diff --git a/server/services/htmlComposition/browser.js b/server/services/htmlComposition/browser.js
index 08c66720c8..d8e53ca4db 100644
--- a/server/services/htmlComposition/browser.js
+++ b/server/services/htmlComposition/browser.js
@@ -105,6 +105,8 @@ const PREVIEW_BRIDGE_SCRIPT = `(() => {
const payload = JSON.stringify({
type: typeof data.type === 'string' ? data.type.slice(0, 80) : '',
t: typeof data.t === 'number' && Number.isFinite(data.t) ? data.t : null,
+ epoch: Number.isSafeInteger(data.epoch) && data.epoch >= 0 ? data.epoch : null,
+ sequence: Number.isSafeInteger(data.sequence) && data.sequence >= 0 ? data.sequence : null,
message: typeof data.message === 'string' ? data.message.slice(0, 300) : '',
hasBlob: data.blob instanceof Blob,
meta,
diff --git a/server/services/musicVideo/documentPreviewPlayback.js b/server/services/musicVideo/documentPreviewPlayback.js
new file mode 100644
index 0000000000..5b75bfad9f
--- /dev/null
+++ b/server/services/musicVideo/documentPreviewPlayback.js
@@ -0,0 +1,98 @@
+/** A narrowly recognized, disposable preview adaptation. Never used by exports. */
+import { createHash } from 'node:crypto';
+import { readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { installLayeredPreviewPlayback } from './previewPlaybackRuntime.js';
+
+const ROOT = fileURLToPath(new URL('./documentTemplates/', import.meta.url));
+const ENGINE_SHA256 = 'ecf110208a0861eb35a39ca559997eca647b922b49c1ca5d5bddc35b2f37bd61';
+const digest = (bytes) => createHash('sha256').update(bytes).digest('hex');
+const read = (directory, name) => readFile(join(directory, name));
+
+const BRIDGE = `
+ let previewBack = null;
+ (${installLayeredPreviewPlayback.toString()})({
+ ready, duration: DURATION, fps: FPS,
+ state: eventState, scene: sceneAt, source: sourceFor,
+ playback: on => globalThis.__portosFilmLook?.playback(on),
+ needsSource: scene => !cardFor(scene),
+ videos: () => [...videos.values()].map(entry => entry.el),
+ publish(songTime, t, scene, source, state, valid) {
+ if (!previewBack) previewBack = document.createElement('canvas');
+ if (previewBack.width !== W) previewBack.width = W;
+ if (previewBack.height !== H) previewBack.height = H;
+ const front = canvas, frontContext = ctx;
+ const backContext = previewBack.getContext('2d');
+ // Reset clip, transform and save stack even after a failed authored draw.
+ backContext.reset();
+ try { canvas = previewBack; ctx = backContext; render(t, scene, source, state); }
+ finally { canvas = front; ctx = frontContext; }
+ if (!valid()) return false;
+ // Fonts and sources are already prepared. Every visible update is in this
+ // synchronous task; no async work is permitted between these commits.
+ globalThis.__portosFilmLook?.frame(Math.round(songTime * FPS));
+ globalThis.__portosTitleCard?.draw(songTime);
+ frontContext.save();
+ try {
+ frontContext.resetTransform(); frontContext.globalAlpha = 1;
+ frontContext.globalCompositeOperation = 'copy'; frontContext.filter = 'none';
+ frontContext.drawImage(previewBack, 0, 0);
+ } finally { frontContext.restore(); }
+ return true;
+ },
+ dispose() {
+ for (const entry of videos.values()) { entry.el.pause(); entry.el.removeAttribute('src'); entry.el.load(); }
+ videos.clear(); images.clear();
+ if (previewBack) { previewBack.width = 0; previewBack.height = 0; previewBack = null; }
+ },
+ });
+`;
+
+/** `directory` is the caller's already-owned staged composition copy. */
+export async function adaptStagedDocumentPreview(directory, { scenes = [] } = {}) {
+ const declined = (reason) => ({ adapted: false, reason });
+ let original, index, template;
+ try {
+ [original, index, template] = await Promise.all([read(directory, 'engine.js'), read(directory, 'index.html'), read(join(ROOT, 'layered'), 'index.html')]);
+ } catch { return declined('unrecognized-entrypoint'); }
+ if (digest(original) !== ENGINE_SHA256) return declined('unrecognized-engine');
+ const marker = '<script src="engine.js"></script>';
+ const generatedIndex = template.toString().replace(marker, '<script src="generated.js"></script>\n' + marker);
+ if (!index.equals(template) && index.toString() !== generatedIndex) return declined('custom-entrypoint');
+ // General authored JavaScript is not a complete-frame publication contract.
+ // Generated/custom/older engines keep deterministic exact seek unchanged.
+ if (!index.equals(template)) return declined('generated-contract-not-admitted');
+ // Only the host's already-staged metadata is authority for source cadence.
+ // A source period >=100 ms cannot satisfy the live frame freshness bound.
+ if (scenes.some(scene => scene.media?.kind === 'video'
+ && (!Number.isFinite(scene.media.fps) || scene.media.fps <= 10))) return declined('unsupported-video-fps');
+ // The complete executable/style helper graph must match, not only engine.js.
+ const helpers = [
+ ['cameraRig.js', join(ROOT, 'layered')],
+ ['lyricType.js', join(ROOT, 'shared')], ['lyricType.css', join(ROOT, 'shared')],
+ ['musicVideoImageEffects.js', fileURLToPath(new URL('../../lib/', import.meta.url))],
+ ];
+ try {
+ for (const [name, root] of helpers) if (!(await read(directory, name)).equals(await read(root, name))) return declined('custom-helper');
+ // Font bytes affect exact parity too; only canonical entrypoint/CSS names apply.
+ const css = `${template}\n${await read(join(ROOT, 'shared'), 'lyricType.css')}`;
+ for (const match of css.matchAll(/url\(['"]?([^)'"\s]+)['"]?\)/g)) {
+ const name = match[1];
+ const root = name.includes('archivo') || name.includes('ibm-plex-mono-500') ? join(ROOT, 'shared') : join(ROOT, 'layered');
+ if (!(await read(directory, name)).equals(await read(root, name))) return declined('custom-font');
+ }
+ } catch { return declined('missing-helper'); }
+ let source = original.toString();
+ for (const [from, to] of [
+ [" const canvas = document.getElementById('stage');", " let canvas = document.getElementById('stage');"],
+ [" const ctx = canvas.getContext('2d');", " let ctx = canvas.getContext('2d');"],
+ ['\n})();\n', `${BRIDGE}\n})();\n`],
+ ]) {
+ if (source.split(from).length !== 2) return declined('bridge-site-mismatch');
+ source = source.replace(from, () => to);
+ }
+ await writeFile(join(directory, 'engine.js'), source);
+ return { adapted: true, adapterVersion: 1, originalEngineSha256: digest(original), stagedEngineSha256: digest(source),
+ presentation: 'atomic-canvas-v1', mediaTiming: 'staged-output-v1', fps: 30 };
+}
diff --git a/server/services/musicVideo/documentPreviewPlayback.test.js b/server/services/musicVideo/documentPreviewPlayback.test.js
new file mode 100644
index 0000000000..e44173ca4f
--- /dev/null
+++ b/server/services/musicVideo/documentPreviewPlayback.test.js
@@ -0,0 +1,612 @@
+import { afterAll, describe, expect, it, vi } from 'vitest';
+import { cp, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { runInNewContext } from 'node:vm';
+import { adaptStagedDocumentPreview } from './documentPreviewPlayback.js';
+import { installLayeredPreviewPlayback } from './previewPlaybackRuntime.js';
+
+const roots = [];
+const templates = fileURLToPath(new URL('./documentTemplates/', import.meta.url));
+afterAll(async () => { await Promise.all(roots.map(root => rm(root, { recursive: true, force: true }))); });
+async function staged() {
+ const root = await mkdtemp(join(tmpdir(), 'preview-playback-contract-')); roots.push(root);
+ await cp(join(templates, 'layered'), root, { recursive: true });
+ await cp(join(templates, 'shared'), root, { recursive: true });
+ await cp(new URL('../../lib/musicVideoImageEffects.js', import.meta.url), join(root, 'musicVideoImageEffects.js'));
+ return root;
+}
+
+describe('recognized staged composition playback capability', () => {
+ it('adapts only the disposable canonical copy, preserving the exact seek body and source template', async () => {
+ const root = await staged(), sourcePath = join(templates, 'layered/engine.js');
+ const original = await readFile(sourcePath, 'utf8'), index = await readFile(join(root, 'index.html'));
+ const receipt = await adaptStagedDocumentPreview(root);
+ expect(receipt).toMatchObject({ adapted: true, adapterVersion: 1, mediaTiming: 'staged-output-v1', fps: 30 });
+ const adapted = await readFile(join(root, 'engine.js'), 'utf8');
+ const exact = original.slice(original.indexOf(' async seek(t,'), original.lastIndexOf('\n})();'));
+ expect(adapted).toContain(exact);
+ expect(await readFile(sourcePath, 'utf8')).toBe(original);
+ expect(await readFile(join(root, 'index.html'))).toEqual(index);
+ expect(receipt.originalEngineSha256).not.toBe(receipt.stagedEngineSha256);
+ // The serialized bridge is a valid script without loading a browser.
+ expect(() => new Function(adapted)).not.toThrow();
+ });
+
+ it('preserves custom entrypoints/helpers and leaves every generated document on exact fallback', async () => {
+ const custom = await staged();
+ await writeFile(join(custom, 'cameraRig.js'), 'globalThis.custom = true;');
+ const before = await readFile(join(custom, 'engine.js'));
+ expect(await adaptStagedDocumentPreview(custom)).toMatchObject({ adapted: false, reason: 'custom-helper' });
+ expect(await readFile(join(custom, 'engine.js'))).toEqual(before);
+ const altered = await staged();
+ await writeFile(join(altered, 'index.html'), '<script src="engine.js"></script><script>sideEffect()</script>');
+ expect(await adaptStagedDocumentPreview(altered)).toMatchObject({ adapted: false, reason: 'custom-entrypoint' });
+ const generated = await staged();
+ const index = await readFile(join(generated, 'index.html'), 'utf8');
+ await writeFile(join(generated, 'index.html'), index.replace('<script src="engine.js"></script>', '<script src="generated.js"></script>\n<script src="engine.js"></script>'));
+ await writeFile(join(generated, 'generated.js'), 'window.PORTOS_MV_GENERATED = { sections: {} };');
+ expect(await adaptStagedDocumentPreview(generated)).toMatchObject({ adapted: false, reason: 'generated-contract-not-admitted' });
+ });
+
+ it('declines unknown or too-slow staged video cadence without rewriting its engine', async () => {
+ const root = await staged(), original = await readFile(join(root, 'engine.js'));
+ for (const fps of [null, 0, 10]) {
+ expect(await adaptStagedDocumentPreview(root, { scenes: [{ media: { kind: 'video', fps } }] }))
+ .toMatchObject({ adapted: false, reason: 'unsupported-video-fps' });
+ expect(await readFile(join(root, 'engine.js'))).toEqual(original);
+ }
+ expect(await adaptStagedDocumentPreview(root, { scenes: [{ media: { kind: 'video', fps: 24 } }, { media: { kind: 'image', fps: null } }] }))
+ .toMatchObject({ adapted: true });
+ });
+
+ it('runs the real staged engine with fail-atomic backbuffer publication and restores exact rendering after a failed draw', async () => {
+ const root = await staged();
+ const song = { sections: [{ id: 'intro', startSec: 0, endSec: 5 }] };
+ expect(await adaptStagedDocumentPreview(root)).toMatchObject({ adapted: true });
+ let failDraw = false;
+ const canvases = [];
+ const makeCanvas = () => {
+ const calls = [];
+ const context = new Proxy({ calls }, { get(target, key) {
+ if (key in target) return target[key];
+ if (key === 'createImageData' || key === 'getImageData') return (_x, _y, w, h) => ({ data: new Uint8ClampedArray((w || _x) * (h || _y) * 4) });
+ if (key === 'measureText') return text => ({ width: String(text).length * 10 });
+ if (key === 'createRadialGradient' || key === 'createLinearGradient') return () => ({ addColorStop() {} });
+ if (key === 'createPattern') return () => ({});
+ return (...args) => { if (key === 'fillRect' && failDraw) throw 1; calls.push([key, ...args]); };
+ } });
+ const canvas = { width: 1920, height: 1080, context, getContext: () => context };
+ canvases.push(canvas); return canvas;
+ };
+ const front = makeCanvas(), events = [], finish = [], titles = [];
+ const sandbox = { setTimeout, clearTimeout, performance: { now: () => 0 }, requestAnimationFrame: () => 1, cancelAnimationFrame() {},
+ document: { getElementById: () => front, createElement: () => makeCanvas(), fonts: { load: () => Promise.resolve() } },
+ PORTOS_MV: { render: { width: 1920, height: 1080, fps: 24, durationSec: 5 }, song, composition: {},
+ scenes: [{ sceneId: 'code', startSec: 0, endSec: 5, visualLayer: 'code' }] },
+ PORTOS_MV_EVENT_STATE: () => ({ t: 0, hold: true, activeEvents: [], reactiveGain: 0 }),
+ __portosFilmLook: { frame: t => finish.push(t), playback() {} }, __portosTitleCard: { draw: t => titles.push(t) },
+ PORTOS_LYRIC_TYPE: { createLyricType: () => ({ ready: Promise.resolve(), draw() {} }) }, postMessage: message => events.push(message) };
+ sandbox.window = sandbox;
+ runInNewContext(await readFile(join(root, 'engine.js'), 'utf8'), sandbox);
+ const composition = sandbox.portosComposition, api = composition.previewPlayback;
+ await api.prepare({ epoch: 1, t: 0 });
+ const copies = () => front.context.calls.filter(call => call[0] === 'drawImage').length;
+ expect(copies()).toBe(1);
+ failDraw = true;
+ await expect(api.prepare({ epoch: 2, t: 0.5 })).rejects.toBe(1);
+ failDraw = false;
+ expect(copies()).toBe(1);
+ await api.pause({ epoch: 3 });
+ await composition.seek(0);
+ expect(front.context.calls.some(call => call[0] === 'fillRect')).toBe(true);
+ await api.prepare({ epoch: 3, t: 0.75 });
+ expect(finish.at(-1)).toBe(18);
+ expect(titles.at(-1)).toBe(0.75);
+ expect(copies()).toBeGreaterThan(1);
+ const backs = canvases.filter(canvas => canvas.context.calls.some(call => call[0] === 'reset'));
+ expect(backs).toHaveLength(1);
+ expect(backs[0].context.calls.filter(call => call[0] === 'reset')).toHaveLength(3);
+ expect(events).toEqual([expect.objectContaining({ type: 'portos-mv:playback-error', epoch: 2 })]);
+ await api.dispose();
+ });
+});
+
+function player({ hold = false, quantized = false, inSec = 0, outSec = 5, cutAt = null, fps = 30, autoFirst = true } = {}) {
+ let clock = 0, frame = null, time = 0, writes = 0, readyResolve, playResolve, autoFrame = false, frameId = 0, presented = 0, drawMs = 0;
+ const callbacks = new Map();
+ const frameWaiters = [];
+ const events = [], paints = [], modes = [], sourceTimes = [], scenes = [{ startSec: 0, endSec: cutAt ?? 5, media: { kind: 'video', inSec, outSec, fps } },
+ { startSec: cutAt, endSec: 5, media: { kind: 'video', inSec: 0, outSec: 5, fps } }];
+ const present = (metadata = {}, callback = callbacks.values().next().value) => callback?.(clock,
+ { mediaTime: Math.floor(time * fps + 1e-6) / fps, presentedFrames: ++presented, ...metadata });
+ const video = { duration: 5, paused: true, playbackRate: 1, readyState: 4, seeking: false,
+ get currentTime() { return time; }, set currentTime(value) { time = value; writes++; },
+ pause() { this.paused = true; }, play() { this.paused = false; autoFrame = true; return playResolve ? new Promise(resolve => { playResolve.resolve = resolve; playResolve.enter(); }) : Promise.resolve(); },
+ requestVideoFrameCallback(callback) {
+ const id = ++frameId;
+ callbacks.set(id, (...args) => { callbacks.delete(id); callback(...args); });
+ for (const resolve of frameWaiters.splice(0)) resolve();
+ if (autoFirst && autoFrame) {
+ autoFrame = false;
+ Promise.resolve().then(() => { if (callbacks.has(id)) present({}, callbacks.get(id)); });
+ }
+ return id;
+ },
+ cancelVideoFrameCallback(id) { callbacks.delete(id); } };
+ const bridge = { duration: 5, fps: 24, ready: Promise.resolve(), videos: () => [video], scene: t => cutAt !== null && t >= cutAt ? scenes[1] : scenes[0], needsSource: () => true, playback: on => modes.push(on),
+ state: t => ({ t: hold && t >= 0.2 && t < 0.4 ? 0.2 : quantized ? Math.floor(t * 24) / 24 : t, hold: hold && t >= 0.2 && t < 0.4, activeEvents: [], reactiveGain: 0 }),
+ async source(scene, t) {
+ sourceTimes.push(t);
+ if (readyResolve) await new Promise(resolve => { readyResolve.resolve = resolve; readyResolve.enter(); });
+ video.currentTime = Math.min(Math.max(scene.media.inSec + t - scene.startSec, scene.media.inSec), scene.media.outSec - 0.5 / 30);
+ return video;
+ },
+ publish(songTime, t, _scene, _source, _state, valid) { if (drawMs) elapse(clock + drawMs); if (!valid()) return false; paints.push({ songTime, t }); return true; }, dispose() { video.pause(); } };
+ const sandbox = { setTimeout, clearTimeout, __bridge: bridge, performance: { now: () => clock }, requestAnimationFrame: callback => { frame = callback; return 1; }, cancelAnimationFrame: () => { frame = null; },
+ window: { postMessage: message => events.push(message) }, portosComposition: {} };
+ runInNewContext(`(${installLayeredPreviewPlayback.toString()})(__bridge)`, sandbox);
+ const elapse = (ms) => { const elapsed = (ms - clock) / 1000; if (!video.paused) time += elapsed * video.playbackRate; clock = ms; };
+ return { api: sandbox.portosComposition.previewPlayback, video, events, paints, modes, sourceTimes, present,
+ pendingCallback: () => callbacks.values().next().value,
+ waitForFrame: () => callbacks.size ? Promise.resolve() : new Promise(resolve => frameWaiters.push(resolve)),
+ get pendingCallbacks() { return callbacks.size; },
+ get writes() { return writes; },
+ elapse,
+ drawCost(ms) { drawMs = ms; },
+ async step(ms, { withVideo = true } = {}) { elapse(ms); const callback = frame; frame = null; callback?.(); if (withVideo && !video.paused) present(); await Promise.resolve(); },
+ deferSource() {
+ readyResolve = {};
+ const entered = new Promise(resolve => { readyResolve.enter = resolve; });
+ return { entered, release: () => { readyResolve.resolve?.(); readyResolve = null; } };
+ },
+ deferPlay() {
+ playResolve = {};
+ const entered = new Promise(resolve => { playResolve.enter = resolve; });
+ return Object.assign(() => { playResolve.resolve?.(); playResolve = null; }, { entered });
+ } };
+}
+
+describe('cooperative preview public runtime lifecycle', () => {
+ it('publishes each fresh video callback independent of RAF/audio phase and never republishes video on RAF alone', async () => {
+ const p = player();
+ await p.api.prepare({ epoch: 1, t: 0 });
+ await p.api.play({ epoch: 1, sequence: 1, t: 0, workerTime: 0 });
+ const first = p.paints.length;
+ await p.step(16, { withVideo: false }); await p.step(32, { withVideo: false });
+ expect(p.paints).toHaveLength(first);
+ for (const time of [35, 68, 101]) await p.step(time);
+ expect(p.paints).toHaveLength(first + 3);
+ expect(p.pendingCallbacks).toBe(1);
+ await p.api.pause({ epoch: 2 });
+ expect(p.pendingCallbacks).toBe(0);
+ });
+
+ it('caps a faster source by source-time bins while preserving native 24 fps and holds', async () => {
+ const p = player({ fps: 60 });
+ await p.api.prepare({ epoch: 1, t: 0 }); await p.api.play({ epoch: 1, sequence: 1, t: 0, workerTime: 0 });
+ const first = p.paints.length;
+ for (const time of [17, 34, 51, 68]) await p.step(time);
+ expect(p.paints).toHaveLength(first + 2);
+ await p.api.dispose();
+
+ const held = player({ fps: 24, hold: true });
+ await held.api.prepare({ epoch: 1, t: 0 }); await held.api.play({ epoch: 1, sequence: 1, t: 0, workerTime: 0 });
+ const before = held.paints.length;
+ await held.step(43); await held.step(85);
+ expect(held.paints).toHaveLength(before + 2);
+ const old = held.pendingCallback();
+ await held.step(210);
+ expect(held.video.paused).toBe(true);
+ expect(held.pendingCallbacks).toBe(0);
+ await held.step(250);
+ await vi.waitFor(async () => {
+ await held.step(300);
+ expect(held.paints.at(-1).t).toBe(0.2);
+ expect(held.paints.at(-1).songTime).toBeCloseTo(0.3);
+ }, { timeout: 500, interval: 1 });
+ expect(held.events).toHaveLength(0);
+ await vi.waitFor(async () => {
+ await held.step(410, { withVideo: false });
+ expect(held.pendingCallbacks).toBe(1);
+ expect(held.paints.at(-1).songTime).toBeCloseTo(0.41);
+ }, { timeout: 500, interval: 1 });
+ expect(held.video.paused).toBe(false);
+ expect(held.sourceTimes.at(-1)).toBe(0.41);
+ const afterHold = held.paints.length;
+ held.present({ mediaTime: 0.1, presentedFrames: 0 }, old);
+ expect(held.paints).toHaveLength(afterHold);
+ expect(held.pendingCallbacks).toBe(1);
+ await held.api.dispose();
+ });
+
+ it('requires a first qualified publication within its separate deadline and immediately settles cancellation', async () => {
+ vi.useFakeTimers();
+ try {
+ const p = player({ autoFirst: false });
+ await p.api.prepare({ epoch: 1, t: 0 });
+ const playing = p.api.play({ epoch: 1, sequence: 1, t: 0, workerTime: 0 });
+ await p.waitForFrame();
+ expect(p.paints).toHaveLength(1);
+ await vi.advanceTimersByTimeAsync(100);
+ expect(await playing).toMatchObject({ status: 'resync-required' });
+ expect(p.pendingCallbacks).toBe(0);
+ expect(p.video.paused).toBe(true);
+ expect(p.events).toHaveLength(1);
+ await p.api.prepare({ epoch: 2, t: 0 });
+ const second = p.api.play({ epoch: 2, sequence: 1, t: 0, workerTime: 0 });
+ await p.waitForFrame();
+ const late = p.pendingCallback();
+ const paused = p.api.pause({ epoch: 3 });
+ expect(await second).toMatchObject({ status: 'superseded' });
+ await paused;
+ const paints = p.paints.length;
+ p.present({ mediaTime: 0, presentedFrames: 1 }, late);
+ expect(p.paints).toHaveLength(paints);
+ expect(p.pendingCallbacks).toBe(0);
+ await p.api.prepare({ epoch: 3, t: 0 });
+ const third = p.api.play({ epoch: 3, sequence: 1, t: 0, workerTime: 0 });
+ await p.waitForFrame();
+ const disposed = p.api.dispose();
+ expect(await third).toMatchObject({ status: 'superseded' });
+ await disposed;
+ expect(p.pendingCallbacks).toBe(0);
+ } finally { vi.useRealTimers(); }
+ });
+
+ it('rejects stale metadata even when currentTime advances, and duplicate callbacks do not refresh silence', async () => {
+ const p = player();
+ await p.api.prepare({ epoch: 1, t: 0 }); await p.api.play({ epoch: 1, sequence: 1, t: 0, workerTime: 0 });
+ const first = p.paints.length;
+ await p.step(80, { withVideo: false });
+ p.present({ mediaTime: 0, presentedFrames: 1 });
+ expect(p.paints).toHaveLength(first);
+ await p.step(101, { withVideo: false });
+ expect(p.events).toEqual([expect.objectContaining({ type: 'portos-mv:playback-resync' })]);
+ expect(p.video.paused).toBe(true);
+ expect(p.pendingCallbacks).toBe(0);
+ await p.api.dispose();
+
+ const stale = player();
+ await stale.api.prepare({ epoch: 1, t: 0 }); await stale.api.play({ epoch: 1, sequence: 1, t: 0, workerTime: 0 });
+ await stale.step(90, { withVideo: false });
+ stale.present({ mediaTime: 0.001, presentedFrames: 2 });
+ await stale.step(180, { withVideo: false });
+ stale.present({ mediaTime: 0.002, presentedFrames: 3 });
+ expect(stale.events).toHaveLength(1); // currentTime is on target; decoded metadata is >100 ms old.
+ expect(stale.video.paused).toBe(true);
+ await stale.api.dispose();
+ });
+
+ it('rejects an expired first or steady callback even if fresh metadata arrives before delayed timers and RAF', async () => {
+ vi.useFakeTimers();
+ try {
+ for (const autoFirst of [false, true]) {
+ const p = player({ autoFirst });
+ await p.api.prepare({ epoch: 1, t: 0 });
+ const playing = p.api.play({ epoch: 1, sequence: 1, t: 0, workerTime: 0 });
+ await p.waitForFrame();
+ if (autoFirst) expect(await playing).toMatchObject({ status: 'playing' });
+ const paints = p.paints.length;
+ p.elapse(100); // No RAF or timer queue is flushed before this callback.
+ p.present({ mediaTime: 0.1, presentedFrames: 2 });
+ if (!autoFirst) expect(await playing).toMatchObject({ status: 'resync-required' });
+ expect(p.paints).toHaveLength(paints);
+ expect(p.events).toEqual([expect.objectContaining({ type: 'portos-mv:playback-resync' })]);
+ expect(p.pendingCallbacks).toBe(0);
+ expect(p.video.paused).toBe(true);
+ await p.api.dispose();
+ }
+ } finally { vi.useRealTimers(); }
+ });
+
+ it('enforces anchor silence in a video callback before an overdue RAF can run', async () => {
+ const p = player();
+ await p.api.prepare({ epoch: 1, t: 0 }); await p.api.play({ epoch: 1, sequence: 1, t: 0, workerTime: 0 });
+ for (let i = 1; i <= 29; i++) { p.elapse(i * 34); p.present(); }
+ const paints = p.paints.length;
+ expect(p.events).toHaveLength(0);
+ p.elapse(1020); p.present(); // Fresh source and callback gap34ms, but no accepted audio anchor for>1s.
+ expect(p.paints).toHaveLength(paints);
+ expect(p.events).toEqual([expect.objectContaining({ type: 'portos-mv:playback-resync' })]);
+ expect(p.pendingCallbacks).toBe(0);
+ expect(p.video.paused).toBe(true);
+ await p.api.dispose();
+ });
+
+ it('requests gated resync if RAF reaches another scene before the first qualified video publication', async () => {
+ const p = player({ autoFirst: false, cutAt: 0.05 });
+ await p.api.prepare({ epoch: 1, t: 0 });
+ const playing = p.api.play({ epoch: 1, sequence: 1, t: 0, workerTime: 0 });
+ await p.waitForFrame();
+ await p.step(60, { withVideo: false });
+ expect(await playing).toMatchObject({ status: 'resync-required' });
+ expect(p.sourceTimes).toEqual([0, 0]); // No hidden second alignment under the pending play.
+ expect(p.paints).toHaveLength(1);
+ expect(p.pendingCallbacks).toBe(0);
+ expect(p.video.paused).toBe(true);
+ await p.api.dispose();
+ });
+
+ it('rejects invalid metadata and retired callbacks while resetting baselines only for a new scene owner', async () => {
+ const p = player({ cutAt: 0.2, autoFirst: false });
+ await p.api.prepare({ epoch: 1, t: 0 });
+ const playing = p.api.play({ epoch: 1, sequence: 1, t: 0, workerTime: 0 });
+ await p.waitForFrame(); p.present({ mediaTime: 0, presentedFrames: 100 }); await playing;
+ const old = p.pendingCallback();
+ await p.step(210, { withVideo: false }); await p.waitForFrame();
+ p.present({ mediaTime: 0, presentedFrames: 1 });
+ const paints = p.paints.length;
+ p.present({ mediaTime: 0.2, presentedFrames: 101 }, old);
+ expect(p.paints).toHaveLength(paints);
+ expect(p.pendingCallbacks).toBe(1);
+ p.present({ mediaTime: 0.001, presentedFrames: 0 });
+ expect(p.events).toEqual([expect.objectContaining({ type: 'portos-mv:playback-resync' })]);
+ expect(p.pendingCallbacks).toBe(0);
+ await p.api.dispose();
+
+ const missing = player({ autoFirst: false });
+ await missing.api.prepare({ epoch: 1, t: 0 });
+ const first = missing.api.play({ epoch: 1, sequence: 1, t: 0, workerTime: 0 });
+ await missing.waitForFrame(); missing.present({ mediaTime: undefined });
+ expect(await first).toMatchObject({ status: 'resync-required' });
+ expect(missing.pendingCallbacks).toBe(0);
+ expect(missing.video.paused).toBe(true);
+ await missing.api.dispose();
+ });
+
+ it('does not report exact-safe pause when a superseded scene preparation times out', async () => {
+ vi.useFakeTimers();
+ try {
+ const p = player({ hold: true });
+ await p.api.prepare({ epoch: 1, t: 0 });
+ await p.api.play({ epoch: 1, sequence: 1, t: 0, workerTime: 0 });
+ const gate = p.deferSource();
+ await p.step(210); await gate.entered;
+ const paused = p.api.pause({ epoch: 2 });
+ const rejected = expect(paused).rejects.toMatchObject({ code: 'PLAYBACK_FAILED' });
+ await vi.advanceTimersByTimeAsync(3001); await rejected;
+ expect(() => p.api.prepare({ epoch: 3, t: 0 })).toThrow('PLAYBACK_FAILED');
+ expect(p.video.paused).toBe(true);
+ gate.release(); await p.api.dispose();
+ expect(p.events).toHaveLength(0); // Superseded epoch does not emit a false current-epoch event.
+ } finally { vi.useRealTimers(); }
+ });
+
+ it('resynchronizes a stalled transition at100ms despite fresh anchors while retaining owned source retirement', async () => {
+ vi.useFakeTimers();
+ try {
+ const p = player({ hold: true });
+ await p.api.prepare({ epoch: 1, t: 0 }); await p.api.play({ epoch: 1, sequence: 1, t: 0, workerTime: 0 });
+ const gate = p.deferSource(), paints = p.paints.length;
+ await p.step(210); await gate.entered;
+ p.elapse(260);
+ expect(p.api.sync({ epoch: 1, sequence: 2, t: 0.26, workerTime: 260 })).toMatchObject({ status: 'accepted' });
+ p.elapse(310);
+ await vi.advanceTimersByTimeAsync(100); // No RAF or source completion is needed to stop stale output.
+ expect(p.events).toEqual([expect.objectContaining({ type: 'portos-mv:playback-resync', epoch: 1, sequence: 2 })]);
+ expect(p.video.paused).toBe(true);
+ expect(p.pendingCallbacks).toBe(0);
+ let retired = false;
+ const paused = p.api.pause({ epoch: 2 }).then(result => { retired = true; return result; });
+ await Promise.resolve(); expect(retired).toBe(false);
+ gate.release(); expect(await paused).toMatchObject({ status: 'paused' });
+ expect(p.paints).toHaveLength(paints);
+ expect(await p.api.prepare({ epoch: 2, t: 0.4 })).toMatchObject({ status: 'prepared' });
+ await p.api.dispose();
+ } finally { vi.useRealTimers(); }
+ });
+
+ it('starts the shared transition deadline when a video callback observes a new mapping before RAF', async () => {
+ vi.useFakeTimers();
+ try {
+ const p = player({ cutAt: 0.2 });
+ await p.api.prepare({ epoch: 1, t: 0 }); await p.api.play({ epoch: 1, sequence: 1, t: 0, workerTime: 0 });
+ for (const time of [70, 140]) { p.elapse(time); p.present(); }
+ const paints = p.paints.length;
+ p.elapse(210); p.present(); // RAF never runs to adopt this mapping change.
+ expect(p.video.paused).toBe(true);
+ p.elapse(260);
+ expect(p.api.sync({ epoch: 1, sequence: 2, t: 0.26, workerTime: 260 })).toMatchObject({ status: 'accepted' });
+ p.elapse(310); await vi.advanceTimersByTimeAsync(100);
+ expect(p.events).toEqual([expect.objectContaining({ type: 'portos-mv:playback-resync' })]);
+ expect(p.paints).toHaveLength(paints);
+ expect(p.sourceTimes).toEqual([0, 0]);
+ expect(p.pendingCallbacks).toBe(0);
+ await p.api.dispose();
+ } finally { vi.useRealTimers(); }
+ });
+
+ it('keeps the original transition deadline through a new video first-frame wait', async () => {
+ const p = player({ cutAt: 0.2, autoFirst: false });
+ await p.api.prepare({ epoch: 1, t: 0 });
+ const playing = p.api.play({ epoch: 1, sequence: 1, t: 0, workerTime: 0 });
+ await p.waitForFrame(); p.present({ mediaTime: 0, presentedFrames: 1 }); await playing;
+ const gate = p.deferSource(), paints = p.paints.length;
+ await p.step(210, { withVideo: false }); await gate.entered;
+ p.elapse(290); gate.release(); await p.waitForFrame();
+ p.elapse(310); p.present({ mediaTime: 0.1, presentedFrames: 2 }); // Fresh callback20ms after arm, but transition age100ms.
+ expect(p.events).toEqual([expect.objectContaining({ type: 'portos-mv:playback-resync' })]);
+ expect(p.paints).toHaveLength(paints);
+ expect(p.pendingCallbacks).toBe(0);
+ await p.api.pause({ epoch: 2 }); await p.api.dispose();
+ });
+
+ it('refuses a late static transition commit after backbuffer drawing crosses the freshness deadline', async () => {
+ const p = player({ hold: true });
+ await p.api.prepare({ epoch: 1, t: 0 }); await p.api.play({ epoch: 1, sequence: 1, t: 0, workerTime: 0 });
+ const paints = p.paints.length;
+ p.drawCost(100);
+ await p.step(210, { withVideo: false });
+ await vi.waitFor(() => expect(p.events).toEqual([expect.objectContaining({ type: 'portos-mv:playback-resync' })]));
+ expect(p.paints).toHaveLength(paints);
+ expect(p.video.paused).toBe(true);
+ await p.api.pause({ epoch: 2 }); await p.api.dispose();
+ });
+
+ it('bounds an unsettled native play, poisons reuse, and pauses a late completion', async () => {
+ vi.useFakeTimers();
+ try {
+ const p = player();
+ await p.api.prepare({ epoch: 1, t: 0 });
+ const finish = p.deferPlay();
+ const playing = p.api.play({ epoch: 1, sequence: 1, t: 0, workerTime: 0 });
+ await finish.entered;
+ const rejected = expect(playing).rejects.toMatchObject({ code: 'PLAYBACK_TIMEOUT' });
+ await vi.advanceTimersByTimeAsync(1501); await rejected;
+ expect(p.video.paused).toBe(true);
+ expect(() => p.api.prepare({ epoch: 2, t: 0 })).toThrow('PLAYBACK_FAILED');
+ finish(); await p.api.dispose();
+ expect(p.video.paused).toBe(true);
+ expect(p.events).toEqual([expect.objectContaining({ type: 'portos-mv:playback-error', epoch: 1 })]);
+ } finally { vi.useRealTimers(); }
+ });
+
+ it('uses native forward playback without per-frame seeks, rejects old anchors, and stops when anchors disappear', async () => {
+ const p = player();
+ expect(await p.api.prepare({ epoch: 1, t: 0 })).toMatchObject({ status: 'prepared' });
+ expect(await p.api.play({ epoch: 1, sequence: 1, t: 0, workerTime: 0 })).toMatchObject({ status: 'playing' });
+ const initialWrites = p.writes;
+ for (const time of [34, 68, 102, 136]) await p.step(time);
+ expect(p.paints.length).toBeGreaterThanOrEqual(5);
+ expect(p.writes).toBe(initialWrites);
+ expect(p.modes).toEqual([false, true]);
+ expect(p.api.sync({ epoch: 0, sequence: 999, t: 1, workerTime: 136 })).toMatchObject({ status: 'stale' });
+ expect(p.api.sync({ epoch: 1, sequence: 2, t: 0.156, workerTime: 136 })).toMatchObject({ status: 'accepted' });
+ expect(p.writes).toBe(initialWrites);
+ await p.step(1137);
+ expect(p.video.paused).toBe(true);
+ expect(p.modes).toEqual([false, true, false]);
+ expect(p.events).toEqual([expect.objectContaining({ type: 'portos-mv:playback-resync', epoch: 1, sequence: 2 })]);
+ await p.step(1237); expect(p.events).toHaveLength(1);
+ });
+
+ it('retires late preparation/play before exact pause and permits the next client epoch', async () => {
+ const p = player(), gate = p.deferSource();
+ const preparing = p.api.prepare({ epoch: 2, t: 0.5 });
+ // Wait on the actual source-entry handshake rather than guessed microtask counts.
+ await gate.entered;
+ const paused = p.api.pause({ epoch: 3 }); gate.release();
+ expect(await preparing).toMatchObject({ status: 'superseded' });
+ expect(await paused).toMatchObject({ status: 'paused' });
+ expect(p.paints).toHaveLength(0);
+ expect(await p.api.prepare({ epoch: 3, t: 0.75 })).toMatchObject({ status: 'prepared' });
+ const finishPlay = p.deferPlay();
+ const playing = p.api.play({ epoch: 3, sequence: 1, t: 0.75, workerTime: 0 });
+ await finishPlay.entered;
+ const stopped = p.api.pause({ epoch: 4 }); finishPlay();
+ expect(await playing).toMatchObject({ status: 'superseded' }); await stopped;
+ expect(p.video.paused).toBe(true);
+ expect(p.modes).toEqual([false]); // A cancelled native play never re-enables motion.
+ expect(await p.api.prepare({ epoch: 4, t: 1 })).toMatchObject({ status: 'prepared' });
+ await p.api.dispose();
+ expect(() => p.api.prepare({ epoch: 5, t: 1 })).toThrow('PLAYBACK_DISPOSED');
+ });
+
+ it('aligns the warm live source once to running audio, then starts a fresh clock at native rate', async () => {
+ const p = player({ quantized: true, inSec: 1 });
+ await p.api.prepare({ epoch: 1, t: 0 });
+ const gate = p.deferSource();
+ const playing = p.api.play({ epoch: 1, sequence: 1, t: 0.18, workerTime: 0 });
+ await gate.entered; await p.step(25); gate.release();
+ expect(await playing).toMatchObject({ status: 'playing', t: 0.205 });
+ expect(p.sourceTimes).toEqual([0, 0.18]); // Source uses song time; drawing remains on its 24 fps grid.
+ expect(p.video.currentTime).toBeCloseTo(1.18);
+ expect(p.video.playbackRate).toBe(1);
+ expect(p.api.sync({ epoch: 1, sequence: 2, t: 0.205, workerTime: 25 })).toMatchObject({ status: 'accepted', driftSec: 0 });
+ const writes = p.writes;
+ await p.step(59); await p.step(93);
+ expect(p.writes).toBe(writes);
+ expect(p.video.playbackRate).toBe(1);
+ expect(p.paints.at(-1).songTime).toBeCloseTo(0.273);
+ expect(p.paints.at(-1).t).toBe(0.25);
+ await p.api.dispose();
+ });
+
+ it('rejects slow warm source work without publishing, starting media, or retrying the alignment', async () => {
+ const p = player();
+ await p.api.prepare({ epoch: 1, t: 0 });
+ const gate = p.deferSource();
+ const playing = p.api.play({ epoch: 1, sequence: 1, t: 0.18, workerTime: 0 });
+ await gate.entered; await p.step(1000); gate.release();
+ expect(await playing).toMatchObject({ status: 'resync-required' });
+ expect(p.sourceTimes).toEqual([0, 0.18]);
+ expect(p.paints).toHaveLength(1);
+ expect(p.video.paused).toBe(true);
+ expect(p.modes).toEqual([false]);
+ expect(p.events).toEqual([expect.objectContaining({ type: 'portos-mv:playback-resync', epoch: 1 })]);
+ await p.api.dispose();
+ });
+
+ it('requires the aligned scene and decoded media to remain usable through native play completion', async () => {
+ const p = player({ cutAt: 0.25 });
+ await p.api.prepare({ epoch: 1, t: 0 });
+ const finish = p.deferPlay();
+ const playing = p.api.play({ epoch: 1, sequence: 1, t: 0.24, workerTime: 0 });
+ await finish.entered; await p.step(25); finish();
+ expect(await playing).toMatchObject({ status: 'resync-required' });
+ expect(p.sourceTimes).toEqual([0, 0.24]);
+ expect(p.video.paused).toBe(true);
+ expect(p.modes).toEqual([false]);
+ expect(p.paints).toHaveLength(1);
+ await p.api.dispose();
+
+ const unavailable = player();
+ await unavailable.api.prepare({ epoch: 1, t: 0 });
+ unavailable.video.readyState = 1;
+ expect(await unavailable.api.play({ epoch: 1, sequence: 1, t: 0.18, workerTime: 0 })).toMatchObject({ status: 'resync-required' });
+ expect(unavailable.video.paused).toBe(true);
+ await unavailable.api.dispose();
+ });
+
+ it('retires a canceled warm alignment before exact pause and preserves frozen narrative source time', async () => {
+ const p = player({ hold: true, quantized: true });
+ await p.api.prepare({ epoch: 1, t: 0 });
+ const gate = p.deferSource();
+ const playing = p.api.play({ epoch: 1, sequence: 1, t: 0.18, workerTime: 0 });
+ await gate.entered;
+ const paused = p.api.pause({ epoch: 2 }); gate.release();
+ expect(await playing).toMatchObject({ status: 'superseded' });
+ expect(await paused).toMatchObject({ status: 'paused' });
+ expect(p.video.paused).toBe(true);
+ expect(p.modes).toEqual([false]);
+ await p.api.prepare({ epoch: 2, t: 0.25 });
+ expect(await p.api.play({ epoch: 2, sequence: 1, t: 0.25, workerTime: 0 })).toMatchObject({ status: 'playing' });
+ expect(p.sourceTimes.at(-1)).toBe(0.2);
+ expect(p.video.paused).toBe(true);
+ const writes = p.writes;
+ await p.step(34); await p.step(68);
+ expect(p.writes).toBe(writes);
+ expect(p.paints.at(-1)).toMatchObject({ t: 0.2 });
+ expect(p.paints.at(-1).songTime).toBeCloseTo(0.318);
+ await p.api.dispose();
+ });
+
+ it('uses live song time at the staged output boundary even before the next drawing frame', async () => {
+ const p = player({ quantized: true, outSec: 0.11 });
+ await p.api.prepare({ epoch: 1, t: 0 });
+ await p.api.play({ epoch: 1, sequence: 1, t: 0.085, workerTime: 0 });
+ expect(p.video.paused).toBe(false);
+ await p.step(10);
+ expect(p.video.paused).toBe(true);
+ expect(p.sourceTimes.at(-1)).toBeCloseTo(0.095);
+ expect(p.video.currentTime).toBeCloseTo(0.11 - 0.5 / 30);
+ await p.api.pause({ epoch: 2 });
+ expect(p.events).toHaveLength(0);
+ await p.api.dispose();
+ });
+
+ it('pauses native media for narrative holds and requires gated resync for a large audio mismatch', async () => {
+ const p = player({ hold: true });
+ await p.api.prepare({ epoch: 1, t: 0 }); await p.api.play({ epoch: 1, sequence: 1, t: 0, workerTime: 0 });
+ await p.step(210); await Promise.resolve();
+ expect(p.video.paused).toBe(true);
+ const writes = p.writes;
+ await p.step(250); await p.step(300);
+ expect(p.writes).toBe(writes);
+ expect(p.api.sync({ epoch: 1, sequence: 2, t: 1, workerTime: 300 })).toMatchObject({ status: 'resync-required' });
+ expect(p.writes).toBe(writes);
+ expect(p.video.paused).toBe(true);
+ });
+});
diff --git a/server/services/musicVideo/documentRender.js b/server/services/musicVideo/documentRender.js
index ed7d680ba7..85c2eb44bb 100644
--- a/server/services/musicVideo/documentRender.js
+++ b/server/services/musicVideo/documentRender.js
@@ -355,9 +355,15 @@ export async function openIsolatedDocumentPreview({ project, jobId, signal, onEv
});
let staged;
let page;
+ let playbackAdapter;
try {
+ const { adaptStagedDocumentPreview } = await import('./documentPreviewPlayback.js');
staged = await stageMusicVideoComposition(plan.directory, jobId, musicVideoSongDocument(project), {
- prepare: (dir) => stageDocumentData(dir, data, media, signal),
+ prepare: async (dir) => {
+ await stageDocumentData(dir, data, media, signal);
+ signal?.throwIfAborted();
+ playbackAdapter = await adaptStagedDocumentPreview(dir, { scenes: data.scenes });
+ },
});
signal?.throwIfAborted();
page = await openComposition(staged.directory, {
@@ -370,8 +376,30 @@ export async function openIsolatedDocumentPreview({ project, jobId, signal, onEv
width: plan.width, height: plan.height, deviceScaleFactor: 1, mobile: false,
});
signal?.throwIfAborted();
+ let playbackAvailable = false;
+ if (playbackAdapter?.adapted) {
+ let captureProtocolSupported = false;
+ try {
+ const version = await page.send('Browser.getVersion', {}, null);
+ const match = typeof version?.product === 'string' && /^(?:HeadlessChrome|Chrome)\/(\d+)\.\d+\.\d+\.\d+$/.exec(version.product);
+ const major = match ? Number(match[1]) : NaN;
+ // 155 is our verified cap=1 capture-lifetime baseline, not the first
+ // Chromium release to declare these optional screencast parameters.
+ captureProtocolSupported = Number.isSafeInteger(major) && major >= 155;
+ } catch { /* Unknown protocol support keeps the existing exact path. */ }
+ signal?.throwIfAborted();
+ const runtimeSupported = captureProtocolSupported && await page.evaluate(
+ `typeof HTMLVideoElement.prototype.requestVideoFrameCallback === 'function' && typeof HTMLVideoElement.prototype.cancelVideoFrameCallback === 'function'`,
+ );
+ signal?.throwIfAborted();
+ playbackAvailable = captureProtocolSupported && runtimeSupported;
+ playbackAdapter = { ...playbackAdapter, runtimeSupported, captureProtocolSupported,
+ ...(!captureProtocolSupported ? { reason: 'unsupported-capture-protocol' }
+ : !runtimeSupported ? { reason: 'unsupported-frame-callbacks' } : {}) };
+ }
return {
- page, scratchRoot: staged.scratchRoot,
+ page, scratchRoot: staged.scratchRoot, playbackAdapter,
+ ...(playbackAvailable ? { playback: { ...playbackAdapter, version: 1 } } : {}),
width: plan.width, height: plan.height, fps: plan.fps, durationSec: plan.durationSec,
};
} catch (error) {
diff --git a/server/services/musicVideo/documentRender.test.js b/server/services/musicVideo/documentRender.test.js
index b7ec079ef4..1b7a44cbf1 100644
--- a/server/services/musicVideo/documentRender.test.js
+++ b/server/services/musicVideo/documentRender.test.js
@@ -8,14 +8,14 @@
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { existsSync } from 'node:fs';
-import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises';
+import { cp, mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { cleanupTempDataRoots, lazyTempDataRoot, makePathsProxy } from '../../lib/mockPathsDataRoot.js';
vi.mock('../htmlComposition/encode.js', async (importOriginal) => ({ ...(await importOriginal()), encodeComposition: vi.fn(async () => { throw new Error('stop at encoder'); }) }));
const { encodeComposition } = await import('../htmlComposition/encode.js');
-const { browser } = vi.hoisted(() => ({ browser: { seen: null, contract: null, visibility: null, sampleTimes: [], send: null, closed: [] } }));
+const { browser } = vi.hoisted(() => ({ browser: { seen: null, contract: null, visibility: null, sampleTimes: [], send: null, closed: [], frameCallbacks: true, capabilityProbes: 0, version: 'Chrome/155.0.8059.41', versionCalls: [] } }));
vi.mock('../../lib/paths.js', async (importOriginal) => makePathsProxy(await importOriginal(), {
dataRoot: () => lazyTempDataRoot('portos-mv-document-stage-'),
}));
@@ -24,6 +24,7 @@ vi.mock('../../lib/paths.js', async (importOriginal) => makePathsProxy(await imp
vi.mock('../../lib/ffmpeg.js', async (importOriginal) => ({
...(await importOriginal()),
findFfmpeg: async () => 'ffmpeg',
+ probeVideoGeometry: vi.fn(async () => null),
runFfmpegProcess: vi.fn(async ({ args }) => {
if (args.includes('-n')) await writeFile(args.at(-1), await readFile(args[args.indexOf('-i') + 1]));
return { ok: true };
@@ -43,6 +44,13 @@ vi.mock('../htmlComposition/browser.js', () => ({
browser.seen = { directory, data: window.PORTOS_MV, media, song: JSON.parse(await read(joinPath(dir, 'song.json'), 'utf8')), initScripts: options?.initScripts ?? [] };
return {
async evaluate(expression) {
+ if (expression.includes('requestVideoFrameCallback')) {
+ browser.capabilityProbes++;
+ const prototype = {};
+ if (browser.frameCallbacks === true || browser.frameCallbacks === 'request-only') prototype.requestVideoFrameCallback = () => {};
+ if (browser.frameCallbacks === true || browser.frameCallbacks === 'cancel-only') prototype.cancelVideoFrameCallback = () => {};
+ return new Function('HTMLVideoElement', `return (${expression});`)({ prototype });
+ }
if (expression.includes('reviewFootage')) { browser.sampleTimes.push(Number(/c.seek\(([^,]+)/.exec(expression)[1])); return browser.visibility; }
if (expression.includes('canPlayType')) return 'probably';
if (browser.contract) return browser.contract;
@@ -50,17 +58,24 @@ vi.mock('../htmlComposition/browser.js', () => ({
},
check() {},
async close() { browser.closed.push({ directory, assetsPresent: (await list(dir)).includes('portos-mv.js') }); },
- async send(...args) { return browser.send?.(...args) ?? {}; },
+ async send(...args) {
+ if (args[0] === 'Browser.getVersion') {
+ browser.versionCalls.push(args);
+ if (browser.version instanceof Error) throw browser.version;
+ return browser.version == null ? {} : { product: browser.version };
+ }
+ return browser.send?.(...args) ?? {};
+ },
};
},
}));
-const { runFfmpegProcess } = await import('../../lib/ffmpeg.js');
+const { runFfmpegProcess, probeVideoGeometry } = await import('../../lib/ffmpeg.js');
const { PATHS } = await import('../../lib/paths.js');
const { encodeDocumentComposition, documentRenderClock, openIsolatedDocumentPreview, resolveSceneMedia } = await import('./documentRender.js');
afterAll(() => cleanupTempDataRoots());
-beforeEach(() => { runFfmpegProcess.mockClear(); browser.seen = null; browser.contract = null; browser.visibility = null; browser.sampleTimes = []; browser.send = null; browser.closed = []; });
+beforeEach(() => { runFfmpegProcess.mockClear(); probeVideoGeometry.mockReset(); probeVideoGeometry.mockResolvedValue(null); browser.seen = null; browser.contract = null; browser.visibility = null; browser.sampleTimes = []; browser.send = null; browser.closed = []; browser.frameCallbacks = true; browser.capabilityProbes = 0; browser.version = 'Chrome/155.0.8059.41'; browser.versionCalls = []; });
const performanceTake = (assetId) => ({ takeId: 'mvt-p', kind: 'video', assetId, status: 'candidate', shotInstruction: { shotMode: 'performance', edit: { inSec: 0.5, outSec: 3.5 } } });
@@ -95,8 +110,68 @@ async function fixture({ shipped = {} } = {}) {
const encode = (project, plan, jobId) => encodeDocumentComposition({ project, plan, jobId, audioPath: join(PATHS.data, 'none.wav'), outputPath: join(PATHS.videos, `${jobId}.mp4`) });
const scratchEntries = async () => (await readdir(join(PATHS.data, 'music-video-song-renders')).catch(() => []));
+async function canonicalPreview() {
+ const value = await fixture(), directory = join(PATHS.data, value.plan.directory);
+ await cp(new URL('./documentTemplates/layered/', import.meta.url), directory, { recursive: true });
+ await cp(new URL('./documentTemplates/shared/', import.meta.url), directory, { recursive: true });
+ await cp(new URL('../../lib/musicVideoImageEffects.js', import.meta.url), join(directory, 'musicVideoImageEffects.js'));
+ value.project.composition.document = { directory: value.plan.directory };
+ return value;
+}
describe('isolated document preview ownership', () => {
+ it.each([
+ { label: 'Chrome', product: 'Chrome/155.0.8059.41', supported: true },
+ { label: 'headless Chrome', product: 'HeadlessChrome/155.0.8059.41', supported: true },
+ { label: 'older baseline', product: 'Chrome/154.0.8037.57', supported: false },
+ { label: 'unknown product', product: 'OtherBrowser/155.0.8059.41', supported: false },
+ { label: 'missing product', product: null, supported: false },
+ { label: 'malformed version', product: 'Chrome/155oops', supported: false },
+ { label: 'unsafe major', product: 'Chrome/9007199254740992.0.0.0', supported: false },
+ { label: 'nonfinite major', product: `Chrome/${'9'.repeat(400)}.0.0.0`, supported: false },
+ { label: 'rejected version probe', product: new Error('Synthetic version probe failure'), supported: false },
+ ])('requires the verified owned capture protocol baseline before advertising playback ($label)', async ({ product, supported }) => {
+ const { project } = await canonicalPreview(); browser.version = product;
+ const preview = await openIsolatedDocumentPreview({ project, jobId: 'preview-protocol' });
+ try {
+ expect(browser.versionCalls).toEqual([['Browser.getVersion', {}, null]]);
+ expect(preview.playbackAdapter).toMatchObject({ adapted: true, captureProtocolSupported: supported });
+ if (supported) expect(preview.playback).toMatchObject({ version: 1 });
+ else {
+ expect(preview.playback).toBeUndefined();
+ expect(preview.playbackAdapter.reason).toBe('unsupported-capture-protocol');
+ expect(browser.capabilityProbes).toBe(0);
+ }
+ expect(browser.closed).toEqual([]); // Exact fallback retains the caller's usable page.
+ } finally { await preview.page.close(); await rm(preview.scratchRoot, { recursive: true, force: true }); }
+ });
+
+ it.each([{ mode: true, supported: true }, { mode: false, supported: false },
+ { mode: 'request-only', supported: false }, { mode: 'cancel-only', supported: false }])('advertises cooperative capability only after a supported browser probe ($mode)', async ({ mode, supported }) => {
+ const { project } = await canonicalPreview(); browser.frameCallbacks = mode;
+ const preview = await openIsolatedDocumentPreview({ project, jobId: `preview-capability-${mode}` });
+ try {
+ expect(browser.capabilityProbes).toBe(1);
+ expect(preview.playbackAdapter).toMatchObject({ adapted: true, runtimeSupported: supported });
+ if (supported) expect(preview.playback).toMatchObject({ version: 1 });
+ else expect(preview.playback).toBeUndefined();
+ expect(browser.seen.data.scenes.find(scene => scene.media?.kind === 'video').media.fps).toBe(24);
+ } finally { await preview.page.close(); await rm(preview.scratchRoot, { recursive: true, force: true }); }
+ });
+
+ it('keeps the actual open path on exact fallback when host staging cannot measure video fps', async () => {
+ const { project } = await canonicalPreview();
+ await writeFile(join(PATHS.data, 'video-history.json'), JSON.stringify([{ id: 'vh-a', filename: 'clip-a.mp4', width: 1280, height: 720 }]));
+ const preview = await openIsolatedDocumentPreview({ project, jobId: 'preview-unknown-fps' });
+ try {
+ expect(probeVideoGeometry).toHaveBeenCalled();
+ expect(preview.playback).toBeUndefined();
+ expect(preview.playbackAdapter).toMatchObject({ adapted: false, reason: 'unsupported-video-fps' });
+ expect(browser.capabilityProbes).toBe(0);
+ expect(browser.seen.data.scenes.find(scene => scene.media?.kind === 'video').media.fps).toBeNull();
+ } finally { await preview.page.close(); await rm(preview.scratchRoot, { recursive: true, force: true }); }
+ });
+
it.each(['metrics failure', 'cancellation'])('closes its opened page before removing staged assets after %s', async failure => {
const { project, plan } = await fixture();
project.composition.document = { directory: plan.directory };
diff --git a/server/services/musicVideo/previewPlaybackRuntime.js b/server/services/musicVideo/previewPlaybackRuntime.js
new file mode 100644
index 0000000000..856ad0da9f
--- /dev/null
+++ b/server/services/musicVideo/previewPlaybackRuntime.js
@@ -0,0 +1,361 @@
+/** Serialized into an admitted layered preview's private engine closure. */
+export function installLayeredPreviewPlayback(bridge) {
+ const FPS = 30;
+ const FRAME_FRESH_MS = 100;
+ let epoch = -1, lease = 0, sequence = -1, phase = 'idle', disposed = false, poisoned = false, motion = null;
+ let handle = null, active = null, transition = null, clockAt = 0, songTime = 0, rate = 1, correction = 0, posted = -1, lastAnchor = 0;
+ const pending = new Set();
+ const notified = new Set();
+ const now = performance.now.bind(performance);
+ let videoOwner = null;
+ let transitionOwner = null;
+ const clamp = (n, lo, hi) => Math.min(hi, Math.max(lo, n));
+ const error = (code) => Object.assign(new Error(code), { code });
+ const check = (input, anchor = false) => {
+ if (disposed) throw error('PLAYBACK_DISPOSED');
+ if (poisoned) throw error('PLAYBACK_FAILED');
+ if (!input || !Number.isSafeInteger(input.epoch) || input.epoch < 0
+ || !Number.isFinite(input.t) || input.t < 0 || input.t > bridge.duration) throw error('PLAYBACK_ARGUMENT');
+ if (anchor && (!Number.isSafeInteger(input.sequence) || input.sequence < 0 || !Number.isFinite(input.workerTime))) throw error('PLAYBACK_ANCHOR');
+ };
+ const stopMedia = () => { for (const video of bridge.videos()) video.pause(); };
+ const setMotion = (value) => {
+ if (motion === value) return;
+ motion = value;
+ bridge.playback(value);
+ };
+ const settleFirst = (owner, result) => {
+ clearTimeout(owner.timer);
+ owner.timer = null;
+ const resolve = owner.resolveFirst;
+ owner.resolveFirst = null;
+ resolve?.(result);
+ };
+ const retireVideo = (result = { epoch, status: 'superseded' }) => {
+ const owner = videoOwner;
+ videoOwner = null;
+ if (!owner) return;
+ try { if (owner.handle != null) owner.video.cancelVideoFrameCallback(owner.handle); }
+ finally { owner.handle = null; settleFirst(owner, result); }
+ };
+ const retireTransition = () => {
+ if (transitionOwner) clearTimeout(transitionOwner.timer);
+ transitionOwner = null;
+ };
+ const cancel = (result) => {
+ lease++;
+ retireTransition();
+ try { retireVideo(result); } catch { poisoned = true; }
+ if (handle != null) cancelAnimationFrame(handle);
+ handle = null; transition = null; phase = 'paused';
+ try { setMotion(false); } catch { poisoned = true; }
+ finally { stopMedia(); }
+ };
+ const tracked = (promise) => {
+ pending.add(promise);
+ promise.then(() => pending.delete(promise), () => pending.delete(promise));
+ return promise;
+ };
+ const bounded = (operation, ms) => {
+ let timer;
+ const expiry = new Promise((_resolve, reject) => { timer = setTimeout(() => {
+ poisoned = true; reject(error('PLAYBACK_TIMEOUT'));
+ }, ms); });
+ return Promise.race([operation, expiry]).finally(() => clearTimeout(timer));
+ };
+ const emit = (type, code) => {
+ const key = `${epoch}:${type}`;
+ if (notified.has(key)) return;
+ notified.add(key);
+ // Retain at most the current epoch's two possible terminal notifications.
+ for (const item of notified) if (!item.startsWith(`${epoch}:`)) notified.delete(item);
+ window.postMessage({ type, message: code, epoch, sequence }, '*');
+ };
+ const fail = (code) => { cancel(); emit('portos-mv:playback-error', code); };
+ const resync = (driftSec) => {
+ const result = { epoch, status: 'resync-required', driftSec };
+ cancel(result); emit('portos-mv:playback-resync', 'PLAYBACK_RESYNC');
+ return result;
+ };
+ const advance = (at) => {
+ const elapsed = Math.max(0, (at - clockAt) / 1000);
+ songTime = clamp(songTime + elapsed * rate, 0, bridge.duration);
+ correction -= elapsed * (rate - 1);
+ rate = 1 + clamp(correction / 2, -0.02, 0.02);
+ clockAt = at;
+ return songTime;
+ };
+ const stateAt = (t) => {
+ const state = bridge.state(t);
+ return { songTime: t, state, t: state.t, scene: bridge.scene(state.t) };
+ };
+ const videoOf = (item) => item?.scene?.media?.kind === 'video' && item.source ? item.source : null;
+ // Autonomous media advances on song time, while narrative holds deliberately
+ // freeze its source clock. Camera/code drawing still uses quantized item.t.
+ const sourceClock = (item) => item.state.hold ? item.t : item.songTime;
+ const sourceTarget = (item) => {
+ const video = videoOf(item), media = item.scene.media;
+ const end = Math.min(Number.isFinite(media.outSec) ? media.outSec : video.duration, video.duration);
+ const last = Math.max(0, end - 0.5 / (media.fps || bridge.fps));
+ return Math.min(Math.max((media.inSec || 0) + sourceClock(item) - item.scene.startSec, media.inSec || 0), last, Math.max(0, video.duration - 1e-3));
+ };
+ const endHold = (item) => {
+ const video = videoOf(item);
+ if (!video) return false;
+ const end = Math.min(Number.isFinite(item.scene.media.outSec) ? item.scene.media.outSec : video.duration, video.duration);
+ return Number.isFinite(end) && (item.scene.media.inSec || 0) + sourceClock(item) - item.scene.startSec >= end - 0.5 / (item.scene.media.fps || bridge.fps);
+ };
+ const startMedia = async (item, token) => {
+ const video = videoOf(item);
+ if (!video || item.state.hold || item.endHold) { stopMedia(); return; }
+ video.playbackRate = rate;
+ await video.play();
+ if (token !== lease || disposed) video.pause();
+ };
+ const load = async (t, token, live = false) => {
+ const item = stateAt(t);
+ const sourceTime = live && item.scene?.media?.kind === 'video' ? sourceClock(item) : item.t;
+ item.source = item.scene && bridge.needsSource(item.scene) ? await bridge.source(item.scene, sourceTime) : null;
+ if (token !== lease || disposed) return null;
+ item.endHold = endHold(item);
+ return item;
+ };
+ const freshSource = (aligned, latest, startedAt) => {
+ const video = videoOf(aligned), item = { ...latest, source: aligned.source };
+ const same = latest.scene === aligned.scene && latest.state.hold === aligned.state.hold
+ && (!latest.state.hold || latest.t === aligned.t) && endHold(item) === endHold(aligned);
+ const drift = video && latest.scene === aligned.scene ? video.currentTime - sourceTarget(item) : 0;
+ return { item, drift, valid: same && now() - startedAt < 100
+ && (!video || video.readyState >= 2 && !video.seeking && Number.isFinite(drift) && Math.abs(drift) < 0.1) };
+ };
+ const freshTransition = (owner) => {
+ if (transitionOwner !== owner || owner.token !== lease || owner.epoch !== epoch || disposed) return false;
+ const age = now() - owner.startedAt;
+ if (age >= FRAME_FRESH_MS) { resync(age / 1000); return false; }
+ return true;
+ };
+ const beginTransition = (token, startedAt) => {
+ if (transitionOwner) return transitionOwner;
+ retireVideo();
+ stopMedia();
+ const progress = { token, epoch, startedAt, timer: null };
+ transitionOwner = progress;
+ progress.timer = setTimeout(() => {
+ if (transitionOwner === progress && token === lease && progress.epoch === epoch && !disposed) resync(FRAME_FRESH_MS / 1000);
+ }, FRAME_FRESH_MS);
+ return progress;
+ };
+ const publish = (item, token, valid = () => true) => bridge.publish(item.songTime, item.t, item.scene, item.source, item.state,
+ () => token === lease && !disposed && valid());
+ const liveVideo = (item) => videoOf(item) && !item.state.hold && !item.endHold;
+ const ownVideo = (item, token, transitionLease = null) => {
+ retireVideo();
+ const video = videoOf(item);
+ const owner = { epoch, token, video, scene: item.scene, handle: null, timer: null, resolveFirst: null,
+ mediaTime: null, presentedFrames: null, lastFreshAt: now(), origin: null, bucket: -1, transitionLease };
+ videoOwner = owner;
+ const first = new Promise(resolve => { owner.resolveFirst = resolve; });
+ const valid = () => videoOwner === owner && owner.epoch === epoch && token === lease && !disposed
+ && phase === 'playing' && !transition && active?.source === video && active.scene === owner.scene && liveVideo(active);
+ const arm = () => {
+ if (!valid()) return;
+ owner.handle = video.requestVideoFrameCallback((_at, metadata) => {
+ owner.handle = null;
+ if (!valid()) return;
+ try {
+ const at = now();
+ if (owner.transitionLease && !freshTransition(owner.transitionLease)) return;
+ if (at - lastAnchor > 1000) { resync((at - lastAnchor) / 1000); return; }
+ const next = stateAt(advance(at));
+ if (at - owner.lastFreshAt >= FRAME_FRESH_MS) { resync((at - owner.lastFreshAt) / 1000); return; }
+ const item = { ...next, source: video };
+ item.endHold = endHold(item);
+ if (item.scene !== owner.scene || item.state.hold || item.endHold) {
+ // RAF owns the next scene/hold preparation. Never publish this old
+ // callback under that new mapping, including during initial play.
+ if (owner.resolveFirst) resync(0.1);
+ else beginTransition(token, at);
+ return;
+ }
+ const mediaTime = metadata?.mediaTime, frames = metadata?.presentedFrames;
+ if (!Number.isFinite(mediaTime) || mediaTime < 0 || !Number.isSafeInteger(frames) || frames < 0
+ || (owner.mediaTime != null && (mediaTime < owner.mediaTime || frames < owner.presentedFrames))) {
+ resync(0.1); return;
+ }
+ const duplicate = owner.mediaTime != null && (mediaTime === owner.mediaTime || frames === owner.presentedFrames);
+ owner.presentedFrames = frames;
+ if (duplicate) {
+ arm(); return; // Duplicates never extend the freshness deadline.
+ }
+ const drift = mediaTime - sourceTarget(item);
+ if (video.readyState < 2 || video.seeking || video.paused || Math.abs(drift) >= 0.1) { resync(drift); return; }
+ owner.mediaTime = mediaTime; owner.lastFreshAt = at;
+ // A <=30 fps source publishes every fresh callback, independent of
+ // audio/RAF phase. Faster sources share source-time-relative bins.
+ if (owner.origin == null) owner.origin = mediaTime;
+ const bucket = Math.floor((mediaTime - owner.origin) * FPS + 1e-6);
+ if (item.scene.media.fps > FPS && bucket === owner.bucket) { arm(); return; }
+ owner.bucket = bucket;
+ if (publish(item, token, () => !owner.transitionLease || freshTransition(owner.transitionLease))) {
+ active = item;
+ if (owner.transitionLease) { retireTransition(); owner.transitionLease = null; }
+ settleFirst(owner, { epoch, status: 'playing', t: item.songTime, workerTime: now() });
+ }
+ arm();
+ } catch { if (valid()) fail('PLAYBACK_FRAME'); }
+ });
+ };
+ // This deadline starts at post-native-play arming, separately from source
+ // alignment's 100 ms guard and the total 1.5 s play operation bound.
+ owner.timer = setTimeout(() => { if (valid()) resync(FRAME_FRESH_MS / 1000); }, FRAME_FRESH_MS);
+ try { arm(); } catch { fail('PLAYBACK_FRAME'); }
+ return first;
+ };
+ const tick = () => {
+ if (phase !== 'playing' || disposed) return;
+ const token = lease;
+ try {
+ const at = now();
+ if (transitionOwner && !freshTransition(transitionOwner)) return;
+ if (at - lastAnchor > 1000) { resync((at - lastAnchor) / 1000); return; }
+ const t = advance(at), next = stateAt(t);
+ const nextEndHold = active ? endHold({ ...next, source: active.source }) : false;
+ if (!active || active.scene !== next.scene || active.state.hold !== next.state.hold || active.endHold !== nextEndHold) {
+ if (!transition) {
+ if (videoOwner?.resolveFirst) { resync(0.1); return; }
+ const progress = beginTransition(token, at);
+ const task = bounded((async () => {
+ const startedAt = progress.startedAt;
+ const item = await load(t, token, true);
+ if (!item || !freshTransition(progress)) return;
+ const latest = stateAt(advance(now()));
+ // Slow preparation must not silently play media behind the master.
+ let fresh = freshSource(item, latest, startedAt);
+ if (!fresh.valid) { resync(fresh.drift || (now() - startedAt) / 1000); return; }
+ active = { ...fresh.item, endHold: endHold(fresh.item) };
+ await startMedia(active, token);
+ if (token !== lease || disposed) return;
+ if (!freshTransition(progress)) return;
+ fresh = freshSource(item, stateAt(advance(now())), startedAt);
+ if (!fresh.valid) { resync(fresh.drift || (now() - startedAt) / 1000); return; }
+ active = { ...fresh.item, endHold: endHold(fresh.item) };
+ if (liveVideo(active)) {
+ // The transition promise is still owned, but media preparation
+ // is finished. Release its gate before the first callback arms.
+ transition = null;
+ await ownVideo(active, token, progress);
+ } else if (publish(active, token, () => freshTransition(progress))) {
+ posted = Math.floor(active.songTime * FPS + 1e-9);
+ retireTransition();
+ }
+ })(), 3000);
+ transition = tracked(task);
+ task.then(() => { if (transition === task) transition = null; }, () => { if (token === lease) fail('PLAYBACK_MEDIA'); });
+ }
+ } else if (!transition) {
+ const video = videoOf(active);
+ if (video && !next.state.hold && !nextEndHold) {
+ const target = sourceTarget({ ...next, source: video });
+ if (Math.abs(video.currentTime - target) >= 0.1) { resync(video.currentTime - target); return; }
+ video.playbackRate = rate;
+ if (videoOwner && at - videoOwner.lastFreshAt >= FRAME_FRESH_MS) { resync((at - videoOwner.lastFreshAt) / 1000); return; }
+ // Fresh video frames are published only by their rVFC owner.
+ if (phase === 'playing' && token === lease) handle = requestAnimationFrame(tick);
+ return;
+ }
+ const frame = Math.floor(t * FPS + 1e-9);
+ if (frame !== posted) {
+ const item = { ...active, ...next };
+ if (publish(item, token)) posted = frame;
+ }
+ }
+ } catch { if (token === lease) fail('PLAYBACK_DRAW'); }
+ if (phase === 'playing' && token === lease) handle = requestAnimationFrame(tick);
+ };
+ const api = {
+ version: 1, fps: FPS, presentation: 'atomic-canvas-v1', mediaTiming: 'staged-output-v1',
+ clock: now,
+ prepare(input) {
+ check(input);
+ if (input.epoch < epoch) return Promise.resolve({ epoch: input.epoch, status: 'superseded' });
+ cancel(); epoch = input.epoch; sequence = -1; posted = -1;
+ const token = lease, previous = [...pending];
+ const task = bounded((async () => {
+ await Promise.allSettled(previous);
+ if (token !== lease || disposed) return { epoch: input.epoch, status: 'superseded' };
+ if (poisoned) throw error('PLAYBACK_FAILED');
+ await bridge.ready;
+ if (token !== lease || disposed) return { epoch: input.epoch, status: 'superseded' };
+ const item = await load(input.t, token);
+ if (!item) return { epoch: input.epoch, status: 'superseded' };
+ if (!publish(item, token)) return { epoch: input.epoch, status: 'superseded' };
+ active = item; phase = 'prepared';
+ return { epoch, status: 'prepared', t: input.t };
+ })(), 3000).catch((cause) => { if (token === lease) fail('PLAYBACK_PREPARE'); throw cause; });
+ return tracked(task);
+ },
+ play(input) {
+ check(input, true);
+ if (input.epoch !== epoch || input.sequence <= sequence) return Promise.resolve({ epoch: input.epoch, status: 'stale' });
+ if (phase !== 'prepared') throw error('PLAYBACK_NOT_PREPARED');
+ const at = now(), age = (at - input.workerTime) / 1000;
+ if (age < -0.1 || age > 1) return Promise.resolve(resync(age));
+ const token = lease;
+ sequence = input.sequence; songTime = clamp(input.t + age, 0, bridge.duration); clockAt = at;
+ lastAnchor = at;
+ correction = 0; rate = 1; phase = 'starting';
+ const task = bounded((async () => {
+ const project = () => stateAt(clamp(input.t + (now() - input.workerTime) / 1000, 0, bridge.duration));
+ const wanted = project(), prepared = active;
+ // Cold preparation precedes audio startup. Align its already-loaded
+ // source once to the now-running master; never repeat this seek in sync.
+ if (wanted.scene !== prepared.scene) return resync(wanted.songTime - prepared.songTime);
+ let aligned = { ...wanted, source: prepared.source };
+ if (videoOf(prepared)) {
+ aligned = await load(wanted.songTime, token, true);
+ if (token !== lease || disposed || !aligned) return { epoch: input.epoch, status: 'superseded' };
+ if (aligned.source !== prepared.source) return resync(0.1);
+ }
+ const alignment = () => freshSource(aligned, project(), at);
+ let fresh = alignment();
+ if (!fresh.valid) return resync(fresh.drift || (now() - at) / 1000);
+ active = { ...fresh.item, endHold: endHold(fresh.item) };
+ await startMedia(active, token);
+ if (token !== lease || disposed) return { epoch: input.epoch, status: 'superseded' };
+ fresh = alignment();
+ if (!fresh.valid) return resync(fresh.drift || (now() - at) / 1000);
+ active = { ...fresh.item, endHold: endHold(fresh.item) };
+ songTime = fresh.item.songTime; clockAt = now(); correction = 0; rate = 1;
+ setMotion(true);
+ phase = 'playing'; handle = requestAnimationFrame(tick);
+ return liveVideo(active) ? ownVideo(active, token) : { epoch, status: 'playing', t: songTime, workerTime: now() };
+ })(), 1500).catch((cause) => { if (token === lease) fail('PLAYBACK_MEDIA'); throw cause; });
+ return tracked(task);
+ },
+ sync(input) {
+ check(input, true);
+ if (input.epoch !== epoch || input.sequence <= sequence || phase !== 'playing') return { epoch: input.epoch, status: 'stale' };
+ const at = now(), age = (at - input.workerTime) / 1000;
+ const drift = input.t + age - advance(at);
+ sequence = input.sequence;
+ if (age < -0.1 || age > 1 || Math.abs(drift) >= 0.1) return resync(drift);
+ correction = drift;
+ lastAnchor = at;
+ return { epoch, status: 'accepted', driftSec: drift };
+ },
+ pause(input) {
+ if (!input || !Number.isSafeInteger(input.epoch) || input.epoch < 0) throw error('PLAYBACK_ARGUMENT');
+ if (input.epoch < epoch) return Promise.resolve({ epoch: input.epoch, status: 'stale' });
+ epoch = input.epoch; cancel();
+ return Promise.allSettled([...pending]).then(() => {
+ if (poisoned) throw error('PLAYBACK_FAILED');
+ return { epoch: input.epoch, status: 'paused' };
+ });
+ },
+ dispose() {
+ disposed = true; cancel();
+ return Promise.allSettled([...pending]).then(() => { bridge.dispose(); active = null; return { epoch, status: 'disposed' }; });
+ },
+ };
+ Object.defineProperty(globalThis.portosComposition, 'previewPlayback', { value: api });
+}