Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Code Animation: playhead and soundtrack playback in the preview
Claude06dbd35fe338
diff --git a/client/src/components/codeAnimation/CodeAnimationPreview.jsx b/client/src/components/codeAnimation/CodeAnimationPreview.jsxindex 6e5aa14431..e0a9ae2848 100644--- a/client/src/components/codeAnimation/CodeAnimationPreview.jsx+++ b/client/src/components/codeAnimation/CodeAnimationPreview.jsx@@ -8,6 +8,7 @@ import { downloadBlob } from '../../lib/downloadBlob';import { importsThree, inlineThreeVendor } from '../../lib/threeVendorImportMap';import { formatLoudness } from '../../utils/formatters';import useGeneratedPreview from '../../hooks/useGeneratedPreview.js';+import CodeAnimationTransport from './CodeAnimationTransport';// Seconds past the film's own duration before a silent recording is abandoned// (a page that never answers the handshake must not leave Record spinning).@@ -197,6 +198,7 @@ export default function CodeAnimationPreview({ html, audioUrl, messages, audioGl}, [preview.recording]);const duration = Number(meta?.duration) || frame?.durationSeconds || 0;+ const fps = Number(meta?.fps) || frame?.fps || 24;const fileBase = slug(title);const handleRecord = () => {@@ -269,7 +271,7 @@ export default function CodeAnimationPreview({ html, audioUrl, messages, audioGl<div className="overflow-hidden rounded-lg border border-port-border bg-black" style={{ aspectRatio: aspect, maxHeight: '70vh' }}>{request ? (preview.frameUrl ? (- <img title="Code animation preview" alt="" src={preview.frameUrl} className="h-full w-full border-0 object-contain" />+ <img ref={preview.imageRef} onLoad={preview.onFrameLoad} title="Code animation preview" alt="" src={preview.frameUrl} className="h-full w-full border-0 object-contain" />) : (<div className="flex h-full items-center justify-center gap-2 p-5 text-center text-sm text-gray-400">{preview.status === 'unavailable' || preview.status === 'error'@@ -310,6 +312,11 @@ export default function CodeAnimationPreview({ html, audioUrl, messages, audioGl)}</div>+ {request && (+ <CodeAnimationTransport preview={preview} duration={duration} fps={fps} audioUrl={audioUrl}+ disabled={recording} label={title || ''} />+ )}+<div className="flex flex-wrap items-center gap-2">{request && preview.status === 'idle' && <button type="button" className={BUTTON_SECONDARY} onClick={preview.resume}>Resume preview</button>}<buttondiff --git a/client/src/components/codeAnimation/CodeAnimationPreview.test.jsx b/client/src/components/codeAnimation/CodeAnimationPreview.test.jsxindex a62ebce58a..01b75c170a 100644--- a/client/src/components/codeAnimation/CodeAnimationPreview.test.jsx+++ b/client/src/components/codeAnimation/CodeAnimationPreview.test.jsx@@ -1,5 +1,5 @@import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';-import { act, render, screen, waitFor } from '@testing-library/react';+import { act, fireEvent, render, screen, waitFor } from '@testing-library/react';import userEvent from '@testing-library/user-event';import { MemoryRouter } from 'react-router';@@ -277,6 +277,46 @@ describe('CodeAnimationPreview', () => {});});+describe('CodeAnimationPreview playhead', () => {+ beforeEach(() => {+ globalThis.URL.createObjectURL = vi.fn(() => 'blob:frame');+ globalThis.URL.revokeObjectURL = vi.fn();+ socket.emit.mockReset();+ socket.emit.mockImplementation(socket.reply);+ socket.__handlers.clear();+ socket.__sessions.clear();+ socket.connected = true;+ localStorage.clear();+ });+ afterEach(() => vi.restoreAllMocks());++ const seeks = () => socket.emit.mock.calls.filter(([event]) => event === 'generated-preview:seek').map(([, payload]) => payload);++ it('holds the film at the playhead, scrubs it, and plays the soundtrack here in sync', async () => {+ const user = userEvent.setup();+ const play = vi.spyOn(HTMLMediaElement.prototype, 'play').mockResolvedValue(undefined);+ vi.spyOn(HTMLMediaElement.prototype, 'pause').mockImplementation(() => {});+ renderPreview({ audioUrl: '/data/music/take.wav', jobId: '00000000-0000-4000-8000-000000000009' });+ await user.click(screen.getByRole('button', { name: 'Preview without audio' }));+ // The session opens on the film's own loop; the player takes it over at 0.+ await waitFor(() => expect(seeks().at(-1)).toMatchObject({ t: 0, playing: false }));+ expect(screen.getByText(/0\.00s \/ 5\.00s · f0/)).toBeTruthy();++ const scrub = screen.getByLabelText('Playhead');+ fireEvent.change(scrub, { target: { value: '2.5' } });+ await waitFor(() => expect(seeks().at(-1)).toMatchObject({ t: 2.5, playing: false }));+ expect(screen.getByText(/2\.50s \/ 5\.00s · f60/)).toBeTruthy();++ await user.click(screen.getByRole('button', { name: 'Mute' }));+ expect(JSON.parse(localStorage.getItem('portos.codeAnimation.previewAudio'))).toMatchObject({ muted: true });+ await user.click(screen.getByRole('button', { name: 'Play' }));+ expect(play).toHaveBeenCalled();+ expect(screen.getByRole('button', { name: 'Pause' })).toBeTruthy();+ await user.click(screen.getByRole('button', { name: 'Pause' }));+ expect(screen.getByRole('button', { name: 'Play' })).toBeTruthy();+ });+});+describe('CodeAnimationPreview frame-exact export', () => {it('retains sound limitations beside the completed export and clears them when switching animations', async () => {const user = userEvent.setup();diff --git a/client/src/components/codeAnimation/CodeAnimationTransport.jsx b/client/src/components/codeAnimation/CodeAnimationTransport.jsxnew file mode 100644index 0000000000..31419d8b0b--- /dev/null+++ b/client/src/components/codeAnimation/CodeAnimationTransport.jsx@@ -0,0 +1,170 @@+import { useCallback, useEffect, useId, useRef, useState } from 'react';+import { Pause, Play, Volume2, VolumeX } from 'lucide-react';+import FrameStepper, { frameAt } from '../musicVideo/FrameStepper';+import useMounted from '../../hooks/useMounted';+import { safeReadJsonStorage, safeWriteJsonStorage } from '../../lib/safeStorage';++// Mute and volume carry over between animations.+export const TRANSPORT_AUDIO_KEY = 'portos.codeAnimation.previewAudio';++const buttonCls = 'inline-flex min-h-[44px] min-w-[44px] items-center justify-center gap-1 rounded-lg border border-port-border bg-port-bg px-2 text-sm text-gray-200 hover:border-port-accent disabled:cursor-not-allowed disabled:opacity-40';++const readAudioPrefs = () => {+ const stored = safeReadJsonStorage(TRANSPORT_AUDIO_KEY, null);+ const volume = Number(stored?.volume);+ return { muted: stored?.muted === true, volume: Number.isFinite(volume) ? Math.min(1, Math.max(0, volume)) : 1 };+};++/**+ * Play, pause and scrub a Code Animation preview. The film is drawn by the+ * isolated worker at each requested time (`preview.seek`); the soundtrack plays+ * here, in this browser, and is the clock while it plays. Without audio a+ * wall clock drives the same frames.+ */+export default function CodeAnimationTransport({ preview, duration, fps = 24, audioUrl, disabled = false, label = '' }) {+ const ids = useId();+ const mounted = useMounted();+ const audioRef = useRef(null);+ const [t, setT] = useState(0);+ const [playing, setPlaying] = useState(false);+ const [audioError, setAudioError] = useState('');+ const [prefs, setPrefs] = useState(readAudioPrefs);+ const clockRef = useRef(null); // { startedAt, from } for the silent clock+ const end = Math.max(0, duration - 1 / fps);+ const { seek: seekPreview } = preview;++ const now = useCallback(() => {+ const audio = audioRef.current;+ if (audio && audioUrl) return audio.currentTime || 0;+ const clock = clockRef.current;+ return clock ? clock.from + (performance.now() - clock.startedAt) / 1000 : t;+ }, [audioUrl, t]);++ const pause = useCallback((time) => {+ const next = Math.min(Math.max(0, Number.isFinite(time) ? time : 0), end);+ audioRef.current?.pause?.();+ if (audioRef.current && audioUrl) audioRef.current.currentTime = next;+ clockRef.current = null;+ setPlaying(false);+ setT(next);+ seekPreview(next);+ }, [end, audioUrl, seekPreview]);++ // Effects below read the latest clock through this ref, so a tick or a new+ // session does not re-run them.+ const latest = useRef(null);+ latest.current = { now, pause, end, duration, fps, seekPreview, playing };++ // A new session opens on the film's own loop; take it over at the start.+ useEffect(() => {+ if (preview.ready) latest.current.pause(0);+ }, [preview.ready, preview.sessionId]);++ // A real-time record runs on the film's own clock.+ useEffect(() => {+ if (disabled && latest.current.playing) latest.current.pause(latest.current.now());+ }, [disabled]);++ const play = async () => {+ const from = t >= end ? 0 : t;+ setAudioError('');+ const audio = audioRef.current;+ if (audio && audioUrl) {+ audio.currentTime = from;+ const started = await Promise.resolve(audio.play?.()).then(() => true).catch((error) => {+ if (mounted.current) setAudioError(error?.message ? `The soundtrack could not start: ${error.message}` : 'The soundtrack could not start.');+ return false;+ });+ if (!started || !mounted.current) return;+ } else {+ clockRef.current = { startedAt: performance.now(), from };+ }+ setT(from);+ setPlaying(true);+ };++ // Ask the worker for the picture once per film frame while playing.+ useEffect(() => {+ if (!playing) return undefined;+ let handle = 0;+ let posted = -1;+ const tick = () => {+ const clock = latest.current;+ const time = clock.now();+ if (time >= clock.duration) { clock.pause(clock.end); return; }+ const frame = frameAt(time, clock.fps);+ if (frame !== posted) {+ posted = frame;+ setT(time);+ clock.seekPreview(time, { playing: true });+ }+ handle = requestAnimationFrame(tick);+ };+ handle = requestAnimationFrame(tick);+ return () => cancelAnimationFrame(handle);+ }, [playing]);++ useEffect(() => {+ const audio = audioRef.current;+ if (!audio) return;+ audio.muted = prefs.muted;+ audio.volume = prefs.volume;+ }, [prefs, audioUrl]);++ useEffect(() => {+ const onHidden = () => {+ if (document.visibilityState === 'hidden' && latest.current.playing) latest.current.pause(latest.current.now());+ };+ const audio = audioRef.current;+ document.addEventListener('visibilitychange', onHidden);+ return () => {+ document.removeEventListener('visibilitychange', onHidden);+ audio?.pause?.();+ };+ }, []);++ const updatePrefs = (patch) => setPrefs((current) => {+ const next = { ...current, ...patch };+ safeWriteJsonStorage(TRANSPORT_AUDIO_KEY, next);+ return next;+ });++ const ready = preview.ready && duration > 0 && !disabled;+ const frame = frameAt(t, fps);+ return (+ <div className="space-y-2">+ <div className="flex flex-wrap items-center gap-2">+ <button type="button" onClick={() => (playing ? pause(now()) : play())} disabled={!ready} className={buttonCls}+ aria-label={playing ? 'Pause' : 'Play'}>+ {playing ? <Pause className="h-4 w-4" /> : <Play className="h-4 w-4" />}+ </button>+ <label htmlFor={`${ids}-scrub`} className="sr-only">Playhead</label>+ <input id={`${ids}-scrub`} type="range" min={0} max={end || 0} step={1 / fps} value={Math.min(t, end || 0)}+ disabled={!ready} onChange={(event) => pause(Number(event.target.value))} className="min-h-[44px] min-w-0 flex-1" />+ <span className="text-xs tabular-nums text-gray-400">+ {t.toFixed(2)}s / {duration.toFixed(2)}s · f{frame}+ </span>+ </div>+ <div className="flex flex-wrap items-center gap-2">+ {audioUrl ? (+ <>+ <button type="button" onClick={() => updatePrefs({ muted: !prefs.muted })} className={buttonCls}+ aria-label={prefs.muted ? 'Unmute' : 'Mute'} aria-pressed={prefs.muted}>+ {prefs.muted ? <VolumeX className="h-4 w-4" /> : <Volume2 className="h-4 w-4" />}+ </button>+ <label htmlFor={`${ids}-volume`} className="sr-only">Volume</label>+ <input id={`${ids}-volume`} type="range" min={0} max={1} step={0.05} value={prefs.volume}+ onChange={(event) => updatePrefs({ volume: Number(event.target.value), muted: false })}+ className="min-h-[44px] w-28" />+ </>+ ) : (+ <span className="text-xs text-gray-500">No soundtrack yet. Add one under Soundtrack below.</span>+ )}+ <FrameStepper time={t} fps={fps} maxSec={duration || null} onSeek={pause} disabled={!ready}+ capture={preview.captureFrame} label={label} />+ </div>+ {audioError && <p role="alert" className="text-xs text-port-error">{audioError}</p>}+ {audioUrl && <audio ref={audioRef} src={audioUrl} preload="none" className="hidden" onEnded={() => pause(end)} />}+ </div>+ );+}diff --git a/server/services/generatedPreview/sessions.js b/server/services/generatedPreview/sessions.jsindex 0b98466631..d50fb4dbbc 100644--- a/server/services/generatedPreview/sessions.js+++ b/server/services/generatedPreview/sessions.js@@ -179,9 +179,43 @@ async function applyJson(page, globalName, value) {await page.evaluate(`globalThis.${globalName} && globalThis.${globalName}.apply(${literal})`);}+// A Code Animation page runs its own requestAnimationFrame clock. A seek takes+// that clock over: callbacks are held (not dropped) so releasing them, before a+// real-time record, resumes the page's loop. One native frame passes after the+// hold so a callback already queued cannot paint over the requested time.+const holdCodeAnimationClock = (time) => `(async () => {+ const w = window;+ if (!w.__portosClock) {+ const raf = w.requestAnimationFrame.bind(w);+ const clock = { raf, held: [], on: false };+ w.__portosClock = clock;+ w.requestAnimationFrame = (callback) => {+ if (!clock.on) return raf(callback);+ clock.held.push(callback);+ return 0;+ };+ }+ const clock = w.__portosClock;+ if (!clock.on) {+ clock.on = true;+ await Promise.race([new Promise((resolve) => clock.raf(() => resolve())), new Promise((resolve) => setTimeout(resolve, 100))]);+ }+ if (typeof w.renderFrame === 'function') w.renderFrame(${time});+})()`;++const RELEASE_CODE_ANIMATION_CLOCK = `(() => {+ const clock = window.__portosClock;+ if (!clock || !clock.on) return;+ clock.on = false;+ for (const callback of clock.held.splice(0)) clock.raf(callback);+})()`;+async function seekPage(session, t, playing = false) {const time = Number(t);- if (session.kind === 'code-animation') return;+ if (session.kind === 'code-animation') {+ if (Number.isFinite(time)) await session.page.evaluate(holdCodeAnimationClock(Math.max(0, time)));+ return;+ }if (session.lyrics) {await session.page.evaluate(`window.postMessage({ type: 'portos-mv:seek', t: ${time} }, '*')`);await session.page.evaluate('new Promise((resolve) => setTimeout(resolve, 40))');@@ -226,7 +260,9 @@ async function runSeek(session, generation, t, playing = false) {}try {await seekPage(session, current.t, current.playing);- emitFrame(session, current.generation, await capture(session.page));+ // A playing Code Animation's new canvas arrives through its screencast;+ // a screenshot per frame would halve the rate.+ if (!(session.kind === 'code-animation' && current.playing)) emitFrame(session, current.generation, await capture(session.page));session.appliedGeneration = current.generation;} catch (error) {console.error(`❌ Generated preview seek failed: ${error.message}`);@@ -578,6 +614,8 @@ async function commandGeneratedPreview(socket, data) {if (parsed.data.action === 'film-look') await applyJson(session.page, '__portosFilmLook', parsed.data.look ?? null);else if (parsed.data.action === 'title-card') await applyJson(session.page, '__portosTitleCard', parsed.data.card ?? null);else if (parsed.data.action === 'record' && parsed.data.message) {+ // A real-time recording runs on the page's own clock.+ if (session.kind === 'code-animation') await session.page.evaluate(RELEASE_CODE_ANIMATION_CLOCK);await session.page.evaluate(`window.postMessage({ type: ${JSON.stringify(parsed.data.message.type)} }, '*')`);}};diff --git a/server/services/generatedPreview/sessions.test.js b/server/services/generatedPreview/sessions.test.jsindex 0f13c58cfe..8f0ea4f6de 100644--- a/server/services/generatedPreview/sessions.test.js+++ b/server/services/generatedPreview/sessions.test.js@@ -7,6 +7,11 @@ vi.mock('../musicVideo/documentRender.js', () => ({ openIsolatedDocumentPreview:return { page: fixture.page, scratchRoot: null, width: 390, height: 219, fps: 30, durationSec: 10,...(fixture.playback ? { playback: fixture.playback } : {}) };} }));+vi.mock('./codeAnimationPage.js', async (importOriginal) => ({+ ...await importOriginal(),+ stageCodeAnimationPage: async () => ({ scratchRoot: null, directory: 'example-animation' }),+}));+vi.mock('../htmlComposition/browser.js', () => ({ openComposition: async () => fixture.page }));import { AUTHORED_PREVIEW_PLAYBACK_TARGET } from '../../lib/generatedPreviewPlaybackSchema.js';import { closeSession, registerGeneratedPreviewHandlers } from './sessions.js';@@ -568,3 +573,39 @@ describe('cooperative playback through preview sockets', () => {});});++// Catches a scrub that the film's own requestAnimationFrame loop paints over,+// and a record that can't run because the taken-over clock was never released.+describe('code animation playhead', () => {+ it('holds the page clock at the scrubbed time, then hands it back for a real-time record', async () => {+ const drawn = [];+ let clock = 100;+ const win = {+ requestAnimationFrame: (callback) => setTimeout(() => callback(performance.now()), 1),+ renderFrame: (t) => drawn.push(t),+ postMessage: vi.fn(),+ };+ const tick = () => { win.renderFrame(clock); clock += 1; win.requestAnimationFrame(tick); };+ fixture.page = {+ evaluate: (source) => new Function('window', 'return ' + source)(win),+ send: async () => ({ data: 'frame' }),+ close: async () => {},+ };+ registerGeneratedPreviewHandlers(socket);+ const requestId = '00000000-0000-4000-8000-000000000002';+ socket.handlers.get('generated-preview:open')({ requestId, kind: 'code-animation', html: '<canvas></canvas>' });+ await vi.waitFor(() => expect(socket.emit).toHaveBeenCalledWith('generated-preview:opened', expect.any(Object)));+ const sessionId = socket.emit.mock.calls.find(([event]) => event === 'generated-preview:opened')[1].sessionId;+ tick();+ await new Promise((resolve) => setTimeout(resolve, 20));++ socket.handlers.get('generated-preview:seek')({ requestId, sessionId, generation: 1, t: 2.5 });+ await vi.waitFor(() => expect(drawn.at(-1)).toBe(2.5));+ await new Promise((resolve) => setTimeout(resolve, 30));+ expect(drawn.at(-1)).toBe(2.5);++ socket.handlers.get('generated-preview:command')({ requestId, sessionId, generation: 1, action: 'record', message: { type: 'example:record' } });+ await vi.waitFor(() => expect(win.postMessage).toHaveBeenCalledWith({ type: 'example:record' }, '*'));+ await vi.waitFor(() => expect(drawn.at(-1)).toBeGreaterThanOrEqual(100));+ });+});