Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Code Animation: playhead and soundtrack playback in the preview

Claude06dbd35fe338
diff --git a/client/src/components/codeAnimation/CodeAnimationPreview.jsx b/client/src/components/codeAnimation/CodeAnimationPreview.jsx
index 6e5aa14431..e0a9ae2848 100644
--- a/client/src/components/codeAnimation/CodeAnimationPreview.jsx
+++ b/client/src/components/codeAnimation/CodeAnimationPreview.jsx
@@ -8,6 +8,7 @@ import { downloadBlob } from '../../lib/downloadBlob';
import { importsThree, inlineThreeVendor } from '../../lib/threeVendorImportMap';
import { formatLoudness } from '../../utils/formatters';
import useGeneratedPreview from '../../hooks/useGeneratedPreview.js';
+import CodeAnimationTransport from './CodeAnimationTransport';
// Seconds past the film's own duration before a silent recording is abandoned
// (a page that never answers the handshake must not leave Record spinning).
@@ -197,6 +198,7 @@ export default function CodeAnimationPreview({ html, audioUrl, messages, audioGl
}, [preview.recording]);
const duration = Number(meta?.duration) || frame?.durationSeconds || 0;
+ const fps = Number(meta?.fps) || frame?.fps || 24;
const fileBase = slug(title);
const handleRecord = () => {
@@ -269,7 +271,7 @@ export default function CodeAnimationPreview({ html, audioUrl, messages, audioGl
<div className="overflow-hidden rounded-lg border border-port-border bg-black" style={{ aspectRatio: aspect, maxHeight: '70vh' }}>
{request ? (
preview.frameUrl ? (
- <img title="Code animation preview" alt="" src={preview.frameUrl} className="h-full w-full border-0 object-contain" />
+ <img ref={preview.imageRef} onLoad={preview.onFrameLoad} title="Code animation preview" alt="" src={preview.frameUrl} className="h-full w-full border-0 object-contain" />
) : (
<div className="flex h-full items-center justify-center gap-2 p-5 text-center text-sm text-gray-400">
{preview.status === 'unavailable' || preview.status === 'error'
@@ -310,6 +312,11 @@ export default function CodeAnimationPreview({ html, audioUrl, messages, audioGl
)}
</div>
+ {request && (
+ <CodeAnimationTransport preview={preview} duration={duration} fps={fps} audioUrl={audioUrl}
+ disabled={recording} label={title || ''} />
+ )}
+
<div className="flex flex-wrap items-center gap-2">
{request && preview.status === 'idle' && <button type="button" className={BUTTON_SECONDARY} onClick={preview.resume}>Resume preview</button>}
<button
diff --git a/client/src/components/codeAnimation/CodeAnimationPreview.test.jsx b/client/src/components/codeAnimation/CodeAnimationPreview.test.jsx
index a62ebce58a..01b75c170a 100644
--- a/client/src/components/codeAnimation/CodeAnimationPreview.test.jsx
+++ b/client/src/components/codeAnimation/CodeAnimationPreview.test.jsx
@@ -1,5 +1,5 @@
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
-import { act, render, screen, waitFor } from '@testing-library/react';
+import { act, fireEvent, render, screen, waitFor } from '@testing-library/react';
import userEvent from '@testing-library/user-event';
import { MemoryRouter } from 'react-router';
@@ -277,6 +277,46 @@ describe('CodeAnimationPreview', () => {
});
});
+describe('CodeAnimationPreview playhead', () => {
+ beforeEach(() => {
+ globalThis.URL.createObjectURL = vi.fn(() => 'blob:frame');
+ globalThis.URL.revokeObjectURL = vi.fn();
+ socket.emit.mockReset();
+ socket.emit.mockImplementation(socket.reply);
+ socket.__handlers.clear();
+ socket.__sessions.clear();
+ socket.connected = true;
+ localStorage.clear();
+ });
+ afterEach(() => vi.restoreAllMocks());
+
+ const seeks = () => socket.emit.mock.calls.filter(([event]) => event === 'generated-preview:seek').map(([, payload]) => payload);
+
+ it('holds the film at the playhead, scrubs it, and plays the soundtrack here in sync', async () => {
+ const user = userEvent.setup();
+ const play = vi.spyOn(HTMLMediaElement.prototype, 'play').mockResolvedValue(undefined);
+ vi.spyOn(HTMLMediaElement.prototype, 'pause').mockImplementation(() => {});
+ renderPreview({ audioUrl: '/data/music/take.wav', jobId: '00000000-0000-4000-8000-000000000009' });
+ await user.click(screen.getByRole('button', { name: 'Preview without audio' }));
+ // The session opens on the film's own loop; the player takes it over at 0.
+ await waitFor(() => expect(seeks().at(-1)).toMatchObject({ t: 0, playing: false }));
+ expect(screen.getByText(/0\.00s \/ 5\.00s · f0/)).toBeTruthy();
+
+ const scrub = screen.getByLabelText('Playhead');
+ fireEvent.change(scrub, { target: { value: '2.5' } });
+ await waitFor(() => expect(seeks().at(-1)).toMatchObject({ t: 2.5, playing: false }));
+ expect(screen.getByText(/2\.50s \/ 5\.00s · f60/)).toBeTruthy();
+
+ await user.click(screen.getByRole('button', { name: 'Mute' }));
+ expect(JSON.parse(localStorage.getItem('portos.codeAnimation.previewAudio'))).toMatchObject({ muted: true });
+ await user.click(screen.getByRole('button', { name: 'Play' }));
+ expect(play).toHaveBeenCalled();
+ expect(screen.getByRole('button', { name: 'Pause' })).toBeTruthy();
+ await user.click(screen.getByRole('button', { name: 'Pause' }));
+ expect(screen.getByRole('button', { name: 'Play' })).toBeTruthy();
+ });
+});
+
describe('CodeAnimationPreview frame-exact export', () => {
it('retains sound limitations beside the completed export and clears them when switching animations', async () => {
const user = userEvent.setup();
diff --git a/client/src/components/codeAnimation/CodeAnimationTransport.jsx b/client/src/components/codeAnimation/CodeAnimationTransport.jsx
new file mode 100644
index 0000000000..31419d8b0b
--- /dev/null
+++ b/client/src/components/codeAnimation/CodeAnimationTransport.jsx
@@ -0,0 +1,170 @@
+import { useCallback, useEffect, useId, useRef, useState } from 'react';
+import { Pause, Play, Volume2, VolumeX } from 'lucide-react';
+import FrameStepper, { frameAt } from '../musicVideo/FrameStepper';
+import useMounted from '../../hooks/useMounted';
+import { safeReadJsonStorage, safeWriteJsonStorage } from '../../lib/safeStorage';
+
+// Mute and volume carry over between animations.
+export const TRANSPORT_AUDIO_KEY = 'portos.codeAnimation.previewAudio';
+
+const buttonCls = 'inline-flex min-h-[44px] min-w-[44px] items-center justify-center gap-1 rounded-lg border border-port-border bg-port-bg px-2 text-sm text-gray-200 hover:border-port-accent disabled:cursor-not-allowed disabled:opacity-40';
+
+const readAudioPrefs = () => {
+ const stored = safeReadJsonStorage(TRANSPORT_AUDIO_KEY, null);
+ const volume = Number(stored?.volume);
+ return { muted: stored?.muted === true, volume: Number.isFinite(volume) ? Math.min(1, Math.max(0, volume)) : 1 };
+};
+
+/**
+ * Play, pause and scrub a Code Animation preview. The film is drawn by the
+ * isolated worker at each requested time (`preview.seek`); the soundtrack plays
+ * here, in this browser, and is the clock while it plays. Without audio a
+ * wall clock drives the same frames.
+ */
+export default function CodeAnimationTransport({ preview, duration, fps = 24, audioUrl, disabled = false, label = '' }) {
+ const ids = useId();
+ const mounted = useMounted();
+ const audioRef = useRef(null);
+ const [t, setT] = useState(0);
+ const [playing, setPlaying] = useState(false);
+ const [audioError, setAudioError] = useState('');
+ const [prefs, setPrefs] = useState(readAudioPrefs);
+ const clockRef = useRef(null); // { startedAt, from } for the silent clock
+ const end = Math.max(0, duration - 1 / fps);
+ const { seek: seekPreview } = preview;
+
+ const now = useCallback(() => {
+ const audio = audioRef.current;
+ if (audio && audioUrl) return audio.currentTime || 0;
+ const clock = clockRef.current;
+ return clock ? clock.from + (performance.now() - clock.startedAt) / 1000 : t;
+ }, [audioUrl, t]);
+
+ const pause = useCallback((time) => {
+ const next = Math.min(Math.max(0, Number.isFinite(time) ? time : 0), end);
+ audioRef.current?.pause?.();
+ if (audioRef.current && audioUrl) audioRef.current.currentTime = next;
+ clockRef.current = null;
+ setPlaying(false);
+ setT(next);
+ seekPreview(next);
+ }, [end, audioUrl, seekPreview]);
+
+ // Effects below read the latest clock through this ref, so a tick or a new
+ // session does not re-run them.
+ const latest = useRef(null);
+ latest.current = { now, pause, end, duration, fps, seekPreview, playing };
+
+ // A new session opens on the film's own loop; take it over at the start.
+ useEffect(() => {
+ if (preview.ready) latest.current.pause(0);
+ }, [preview.ready, preview.sessionId]);
+
+ // A real-time record runs on the film's own clock.
+ useEffect(() => {
+ if (disabled && latest.current.playing) latest.current.pause(latest.current.now());
+ }, [disabled]);
+
+ const play = async () => {
+ const from = t >= end ? 0 : t;
+ setAudioError('');
+ const audio = audioRef.current;
+ if (audio && audioUrl) {
+ audio.currentTime = from;
+ const started = await Promise.resolve(audio.play?.()).then(() => true).catch((error) => {
+ if (mounted.current) setAudioError(error?.message ? `The soundtrack could not start: ${error.message}` : 'The soundtrack could not start.');
+ return false;
+ });
+ if (!started || !mounted.current) return;
+ } else {
+ clockRef.current = { startedAt: performance.now(), from };
+ }
+ setT(from);
+ setPlaying(true);
+ };
+
+ // Ask the worker for the picture once per film frame while playing.
+ useEffect(() => {
+ if (!playing) return undefined;
+ let handle = 0;
+ let posted = -1;
+ const tick = () => {
+ const clock = latest.current;
+ const time = clock.now();
+ if (time >= clock.duration) { clock.pause(clock.end); return; }
+ const frame = frameAt(time, clock.fps);
+ if (frame !== posted) {
+ posted = frame;
+ setT(time);
+ clock.seekPreview(time, { playing: true });
+ }
+ handle = requestAnimationFrame(tick);
+ };
+ handle = requestAnimationFrame(tick);
+ return () => cancelAnimationFrame(handle);
+ }, [playing]);
+
+ useEffect(() => {
+ const audio = audioRef.current;
+ if (!audio) return;
+ audio.muted = prefs.muted;
+ audio.volume = prefs.volume;
+ }, [prefs, audioUrl]);
+
+ useEffect(() => {
+ const onHidden = () => {
+ if (document.visibilityState === 'hidden' && latest.current.playing) latest.current.pause(latest.current.now());
+ };
+ const audio = audioRef.current;
+ document.addEventListener('visibilitychange', onHidden);
+ return () => {
+ document.removeEventListener('visibilitychange', onHidden);
+ audio?.pause?.();
+ };
+ }, []);
+
+ const updatePrefs = (patch) => setPrefs((current) => {
+ const next = { ...current, ...patch };
+ safeWriteJsonStorage(TRANSPORT_AUDIO_KEY, next);
+ return next;
+ });
+
+ const ready = preview.ready && duration > 0 && !disabled;
+ const frame = frameAt(t, fps);
+ return (
+ <div className="space-y-2">
+ <div className="flex flex-wrap items-center gap-2">
+ <button type="button" onClick={() => (playing ? pause(now()) : play())} disabled={!ready} className={buttonCls}
+ aria-label={playing ? 'Pause' : 'Play'}>
+ {playing ? <Pause className="h-4 w-4" /> : <Play className="h-4 w-4" />}
+ </button>
+ <label htmlFor={`${ids}-scrub`} className="sr-only">Playhead</label>
+ <input id={`${ids}-scrub`} type="range" min={0} max={end || 0} step={1 / fps} value={Math.min(t, end || 0)}
+ disabled={!ready} onChange={(event) => pause(Number(event.target.value))} className="min-h-[44px] min-w-0 flex-1" />
+ <span className="text-xs tabular-nums text-gray-400">
+ {t.toFixed(2)}s / {duration.toFixed(2)}s · f{frame}
+ </span>
+ </div>
+ <div className="flex flex-wrap items-center gap-2">
+ {audioUrl ? (
+ <>
+ <button type="button" onClick={() => updatePrefs({ muted: !prefs.muted })} className={buttonCls}
+ aria-label={prefs.muted ? 'Unmute' : 'Mute'} aria-pressed={prefs.muted}>
+ {prefs.muted ? <VolumeX className="h-4 w-4" /> : <Volume2 className="h-4 w-4" />}
+ </button>
+ <label htmlFor={`${ids}-volume`} className="sr-only">Volume</label>
+ <input id={`${ids}-volume`} type="range" min={0} max={1} step={0.05} value={prefs.volume}
+ onChange={(event) => updatePrefs({ volume: Number(event.target.value), muted: false })}
+ className="min-h-[44px] w-28" />
+ </>
+ ) : (
+ <span className="text-xs text-gray-500">No soundtrack yet. Add one under Soundtrack below.</span>
+ )}
+ <FrameStepper time={t} fps={fps} maxSec={duration || null} onSeek={pause} disabled={!ready}
+ capture={preview.captureFrame} label={label} />
+ </div>
+ {audioError && <p role="alert" className="text-xs text-port-error">{audioError}</p>}
+ {audioUrl && <audio ref={audioRef} src={audioUrl} preload="none" className="hidden" onEnded={() => pause(end)} />}
+ </div>
+ );
+}
diff --git a/server/services/generatedPreview/sessions.js b/server/services/generatedPreview/sessions.js
index 0b98466631..d50fb4dbbc 100644
--- a/server/services/generatedPreview/sessions.js
+++ b/server/services/generatedPreview/sessions.js
@@ -179,9 +179,43 @@ async function applyJson(page, globalName, value) {
await page.evaluate(`globalThis.${globalName} && globalThis.${globalName}.apply(${literal})`);
}
+// A Code Animation page runs its own requestAnimationFrame clock. A seek takes
+// that clock over: callbacks are held (not dropped) so releasing them, before a
+// real-time record, resumes the page's loop. One native frame passes after the
+// hold so a callback already queued cannot paint over the requested time.
+const holdCodeAnimationClock = (time) => `(async () => {
+ const w = window;
+ if (!w.__portosClock) {
+ const raf = w.requestAnimationFrame.bind(w);
+ const clock = { raf, held: [], on: false };
+ w.__portosClock = clock;
+ w.requestAnimationFrame = (callback) => {
+ if (!clock.on) return raf(callback);
+ clock.held.push(callback);
+ return 0;
+ };
+ }
+ const clock = w.__portosClock;
+ if (!clock.on) {
+ clock.on = true;
+ await Promise.race([new Promise((resolve) => clock.raf(() => resolve())), new Promise((resolve) => setTimeout(resolve, 100))]);
+ }
+ if (typeof w.renderFrame === 'function') w.renderFrame(${time});
+})()`;
+
+const RELEASE_CODE_ANIMATION_CLOCK = `(() => {
+ const clock = window.__portosClock;
+ if (!clock || !clock.on) return;
+ clock.on = false;
+ for (const callback of clock.held.splice(0)) clock.raf(callback);
+})()`;
+
async function seekPage(session, t, playing = false) {
const time = Number(t);
- if (session.kind === 'code-animation') return;
+ if (session.kind === 'code-animation') {
+ if (Number.isFinite(time)) await session.page.evaluate(holdCodeAnimationClock(Math.max(0, time)));
+ return;
+ }
if (session.lyrics) {
await session.page.evaluate(`window.postMessage({ type: 'portos-mv:seek', t: ${time} }, '*')`);
await session.page.evaluate('new Promise((resolve) => setTimeout(resolve, 40))');
@@ -226,7 +260,9 @@ async function runSeek(session, generation, t, playing = false) {
}
try {
await seekPage(session, current.t, current.playing);
- emitFrame(session, current.generation, await capture(session.page));
+ // A playing Code Animation's new canvas arrives through its screencast;
+ // a screenshot per frame would halve the rate.
+ if (!(session.kind === 'code-animation' && current.playing)) emitFrame(session, current.generation, await capture(session.page));
session.appliedGeneration = current.generation;
} catch (error) {
console.error(`❌ Generated preview seek failed: ${error.message}`);
@@ -578,6 +614,8 @@ async function commandGeneratedPreview(socket, data) {
if (parsed.data.action === 'film-look') await applyJson(session.page, '__portosFilmLook', parsed.data.look ?? null);
else if (parsed.data.action === 'title-card') await applyJson(session.page, '__portosTitleCard', parsed.data.card ?? null);
else if (parsed.data.action === 'record' && parsed.data.message) {
+ // A real-time recording runs on the page's own clock.
+ if (session.kind === 'code-animation') await session.page.evaluate(RELEASE_CODE_ANIMATION_CLOCK);
await session.page.evaluate(`window.postMessage({ type: ${JSON.stringify(parsed.data.message.type)} }, '*')`);
}
};
diff --git a/server/services/generatedPreview/sessions.test.js b/server/services/generatedPreview/sessions.test.js
index 0f13c58cfe..8f0ea4f6de 100644
--- a/server/services/generatedPreview/sessions.test.js
+++ b/server/services/generatedPreview/sessions.test.js
@@ -7,6 +7,11 @@ vi.mock('../musicVideo/documentRender.js', () => ({ openIsolatedDocumentPreview:
return { page: fixture.page, scratchRoot: null, width: 390, height: 219, fps: 30, durationSec: 10,
...(fixture.playback ? { playback: fixture.playback } : {}) };
} }));
+vi.mock('./codeAnimationPage.js', async (importOriginal) => ({
+ ...await importOriginal(),
+ stageCodeAnimationPage: async () => ({ scratchRoot: null, directory: 'example-animation' }),
+}));
+vi.mock('../htmlComposition/browser.js', () => ({ openComposition: async () => fixture.page }));
import { AUTHORED_PREVIEW_PLAYBACK_TARGET } from '../../lib/generatedPreviewPlaybackSchema.js';
import { closeSession, registerGeneratedPreviewHandlers } from './sessions.js';
@@ -568,3 +573,39 @@ describe('cooperative playback through preview sockets', () => {
});
});
+
+// Catches a scrub that the film's own requestAnimationFrame loop paints over,
+// and a record that can't run because the taken-over clock was never released.
+describe('code animation playhead', () => {
+ it('holds the page clock at the scrubbed time, then hands it back for a real-time record', async () => {
+ const drawn = [];
+ let clock = 100;
+ const win = {
+ requestAnimationFrame: (callback) => setTimeout(() => callback(performance.now()), 1),
+ renderFrame: (t) => drawn.push(t),
+ postMessage: vi.fn(),
+ };
+ const tick = () => { win.renderFrame(clock); clock += 1; win.requestAnimationFrame(tick); };
+ fixture.page = {
+ evaluate: (source) => new Function('window', 'return ' + source)(win),
+ send: async () => ({ data: 'frame' }),
+ close: async () => {},
+ };
+ registerGeneratedPreviewHandlers(socket);
+ const requestId = '00000000-0000-4000-8000-000000000002';
+ socket.handlers.get('generated-preview:open')({ requestId, kind: 'code-animation', html: '<canvas></canvas>' });
+ await vi.waitFor(() => expect(socket.emit).toHaveBeenCalledWith('generated-preview:opened', expect.any(Object)));
+ const sessionId = socket.emit.mock.calls.find(([event]) => event === 'generated-preview:opened')[1].sessionId;
+ tick();
+ await new Promise((resolve) => setTimeout(resolve, 20));
+
+ socket.handlers.get('generated-preview:seek')({ requestId, sessionId, generation: 1, t: 2.5 });
+ await vi.waitFor(() => expect(drawn.at(-1)).toBe(2.5));
+ await new Promise((resolve) => setTimeout(resolve, 30));
+ expect(drawn.at(-1)).toBe(2.5);
+
+ socket.handlers.get('generated-preview:command')({ requestId, sessionId, generation: 1, action: 'record', message: { type: 'example:record' } });
+ await vi.waitFor(() => expect(win.postMessage).toHaveBeenCalledWith({ type: 'example:record' }, '*'));
+ await vi.waitFor(() => expect(drawn.at(-1)).toBeGreaterThanOrEqual(100));
+ });
+});