Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Apply shot speed and frame offset in live composition-document previews (#11406)
Adam Eivy5ef633d286b6
diff --git a/docs/HTML_COMPOSITIONS.md b/docs/HTML_COMPOSITIONS.mdindex 65f91c5391..0291af680c 100644--- a/docs/HTML_COMPOSITIONS.md+++ b/docs/HTML_COMPOSITIONS.md@@ -420,6 +420,11 @@ snapshot freezes (`server/services/musicVideo/documentRender.js`):width, height } | null }], textCues, composition: { mode, style, posterSec,overlay } }`. Load it with `<script src="portos-mv.js">`; the sandboxrefuses `fetch`.+ A shot's `speedFps`/`offsetFrames` are baked into the staged clip, so a render+ seeks `media.inSec + (t - startSec)`. The live preview bridges the original+ bytes instead: its `media` keeps source-time `inSec`/`outSec` and adds+ `timing: { rate, leadSec, startSec }` (only for a retimed shot), and the page+ seeks `startSec + max(0, (t - scene.startSec) - leadSec) * rate`.- `song.json` — the same song block the code-rendered mode reads.- `media/scene-<sceneId>.<ext>` — each scene's selected take (its clip whenit has one, else its still), named by `scenes[].media.src`.diff --git a/server/services/musicVideo/documentRender.browser.test.js b/server/services/musicVideo/documentRender.browser.test.jsindex 3ac574b3ce..56b625b935 100644--- a/server/services/musicVideo/documentRender.browser.test.js+++ b/server/services/musicVideo/documentRender.browser.test.js@@ -553,6 +553,50 @@ describe.skipIf(!canRun)('layered template with real Chrome and ffmpeg', () => {}}, 120000));+ // Uniquely catches the live preview bridging raw bytes without the shot's speed/offset (same samples as the render above).+ [+ { timing: { speedFps: 24, offsetFrames: 12 }, samples: [[0, 1], [5, 1], [6, 2]] },+ { timing: { speedFps: 24, offsetFrames: -6 }, samples: [[0, 0], [11, 0], [12, 1], [17, 1], [18, 2]] },+ { timing: { offsetFrames: -6 }, samples: [[0, 0], [17, 0], [18, 1], [29, 1], [30, 2]] },+ ].forEach(({ timing, samples }) => it(`plays shot speed and offset ${JSON.stringify(timing)} in the live preview`, async () => {+ await mkdir(PATHS.videos, { recursive: true });+ const clip = join(PATHS.videos, 'live-timed-rgb.webm');+ execFileSync(ffmpeg, ['-v', 'error', '-y',+ '-f', 'lavfi', '-i', 'color=c=red:s=64x36:r=24:d=0.5',+ '-f', 'lavfi', '-i', 'color=c=lime:s=64x36:r=24:d=0.5',+ '-f', 'lavfi', '-i', 'color=c=blue:s=64x36:r=24:d=0.5',+ '-filter_complex', '[0:v][1:v][2:v]concat=n=3:v=1:a=0', '-c:v', 'libvpx', clip]);+ const created = await projects.createProject({ name: 'Synthetic live shot timing' });+ await projects.mutateProjectRecord(created.id, current => ({ project: { ...current,+ audioAnalysis: { durationSec: 1.5, sections: [] },+ scenes: [{ sceneId: 'timed', startSec: 0, endSec: 1.5, videoHistoryId: 'vh-live', ...timing }],+ } }));+ await writeFile(join(PATHS.data, 'video-history.json'), JSON.stringify([{ id: 'vh-live', filename: 'live-timed-rgb.webm', numFrames: 36, fps: 24, width: 64, height: 36 }]));+ await importDocumentTemplate(created.id);+ const preview = await buildDocumentPreview(await projects.getProject(created.id));+ const page = await browser.newPage();+ await page.setContent(preview.html);+ const bytes = (await readFile(clip)).toString('base64');+ await page.evaluate(async (base64) => {+ const blob = new Blob([Uint8Array.from(atob(base64), c => c.charCodeAt(0))], { type: 'video/webm' });+ addEventListener('message', ({ data }) => { if (data?.type === 'portos-mv:request') postMessage({ type: 'portos-mv:asset', key: data.key, blob }, '*'); });+ postMessage({ type: 'portos-mv:manifest', keys: ['media/scene-timed.webm'] }, '*');+ await window.PORTOS_MV_ASSETS;+ }, bytes);+ for (const [frame, channel] of samples) {+ const rgb = await page.evaluate(async (frame) => {+ await window.portosComposition.seek(frame / 24);+ const stage = document.getElementById('stage');+ const probe = document.createElement('canvas'); probe.width = stage.width; probe.height = stage.height;+ const ctx = probe.getContext('2d'); ctx.drawImage(stage, 0, 0);+ return [...ctx.getImageData(stage.width >> 1, stage.height >> 1, 1, 1).data.slice(0, 3)];+ }, frame);+ expect(rgb[channel], `frame ${frame}`).toBeGreaterThan(180);+ rgb.forEach((value, i) => { if (i !== channel) expect(value, `frame ${frame}`).toBeLessThan(60); });+ }+ await page.close();+ }, 120000));+it('draws the selected take at SONG time in an excerpt, identically on every render, with the master muxed', async () => {const { cdpRequest } = await import('../browserService.js');const managedCalls = cdpRequest.mock.calls.length;diff --git a/server/services/musicVideo/documentRender.js b/server/services/musicVideo/documentRender.jsindex 1557ec5f7f..3dc2adef4e 100644--- a/server/services/musicVideo/documentRender.js+++ b/server/services/musicVideo/documentRender.js@@ -144,15 +144,31 @@ export async function resolveSceneMedia(project, { history = [], probe = async (const duration = measured.numFrames && measured.fps ? measured.numFrames / measured.fps : null;const edit = selectedPerformanceInstruction(scene)?.edit ?? null;const ext = (/\.([a-z0-9]{2,5})$/i.exec(entry.filename)?.[1] || 'mp4').toLowerCase();- const rate = retimed ? clipSpeedRate(scene) : 1;+ const rate = clipSpeedRate(scene);const originalIn = edit ? Math.min(edit.inSec, duration ?? edit.inSec) : 0;const originalOut = edit ? Math.min(edit.outSec, duration ?? edit.outSec) : duration;- const start = originalIn + (retimed ? clipOffsetSec(scene) : 0);+ const start = originalIn + clipOffsetSec(scene);const sourceIn = Math.min(duration ?? Infinity, Math.max(0, start));// Offset holds are song/output seconds, independent of playback rate:// clipTimeAt uses sourceIn + max(0, local + min(0, start)) * rate.const leadSec = Math.max(0, -start);const retime = rate !== 1 || leadSec > 0;+ // A raw-asset preview plays the original bytes, so it keeps source-time+ // in/out and carries the same timing for the page's seek to apply.+ if (!retimed) {+ out.set(scene.sceneId, {+ kind: 'video',+ path: videoPath,+ ext,+ inSec: round3(originalIn),+ outSec: originalOut == null ? null : round3(originalOut),+ ...(retime ? { timing: { rate, leadSec: round3(leadSec), startSec: round3(sourceIn) } } : {}),+ fps: finite(measured.fps),+ width: finite(measured.width),+ height: finite(measured.height),+ });+ continue;+ }out.set(scene.sceneId, {kind: 'video',path: videoPath,@@ -222,6 +238,8 @@ export function buildDocumentData(project, { media = new Map(), frame, clock, somedia: m ? {kind: m.kind, src: mediaSrc(scene.sceneId, m),inSec: m.inSec, outSec: m.outSec, fps: m.fps, width: m.width, height: m.height,+ // Raw-asset previews only: seek source time = startSec + max(0, local - leadSec) * rate.+ ...(m.timing ? { timing: m.timing } : {}),} : null,};});diff --git a/server/services/musicVideo/documentRender.test.js b/server/services/musicVideo/documentRender.test.jsindex 6271d256bc..f3466df5e5 100644--- a/server/services/musicVideo/documentRender.test.js+++ b/server/services/musicVideo/documentRender.test.js@@ -130,16 +130,18 @@ describe('isolated document preview ownership', () => {describe('composition document staging', () => {it.each([- { timing: { speedFps: 24, offsetFrames: 12 }, inSec: 0.25, outSec: 2, filter: 'setpts=(PTS-STARTPTS)/2,fps=24' },- { timing: { speedFps: 24, offsetFrames: -6 }, inSec: 0, outSec: 2.25, filter: 'setpts=(PTS-STARTPTS)/2,tpad=start_mode=clone:start_duration=0.25,fps=24' },- { timing: { offsetFrames: -6 }, inSec: 0, outSec: 4.25, filter: 'setpts=(PTS-STARTPTS)/1,tpad=start_mode=clone:start_duration=0.25,fps=24' },- ])('stages shot timing $timing on the document output timebase', async ({ timing, inSec, outSec, filter }) => {+ { timing: { speedFps: 24, offsetFrames: 12 }, live: { rate: 2, leadSec: 0, startSec: 0.5 }, inSec: 0.25, outSec: 2, filter: 'setpts=(PTS-STARTPTS)/2,fps=24' },+ { timing: { speedFps: 24, offsetFrames: -6 }, live: { rate: 2, leadSec: 0.25, startSec: 0 }, inSec: 0, outSec: 2.25, filter: 'setpts=(PTS-STARTPTS)/2,tpad=start_mode=clone:start_duration=0.25,fps=24' },+ { timing: { offsetFrames: -6 }, live: { rate: 1, leadSec: 0.25, startSec: 0 }, inSec: 0, outSec: 4.25, filter: 'setpts=(PTS-STARTPTS)/1,tpad=start_mode=clone:start_duration=0.25,fps=24' },+ ])('stages shot timing $timing on the document output timebase', async ({ timing, live, inSec, outSec, filter }) => {const { project, plan } = await fixture();Object.assign(project.scenes[0], timing);// The live preview bridges original bytes, so it must retain source time.const raw = await resolveSceneMedia(project, { history: JSON.parse(await readFile(join(PATHS.data, 'video-history.json'), 'utf8')) });expect(raw.get('a')).toMatchObject({ ext: 'mp4', inSec: 0, outSec: 4, fps: 24 });expect(raw.get('a').rate).toBeUndefined();+ // The live preview applies speed/offset at seek time (engine.js) from this source-time timing.+ expect(raw.get('a').timing).toEqual(live);const controller = new AbortController();await expect(encodeDocumentComposition({ project, plan, jobId: 'job-timing', signal: controller.signal })).rejects.toThrow('stop after staging');expect(browser.seen.data.scenes[0].media).toMatchObject({ inSec, outSec, fps: 24 });@@ -161,6 +163,14 @@ describe('composition document staging', () => {expect(browser.seen.data.scenes[0].media).toMatchObject({ inSec: 0, outSec: 4 });});+ it('keeps a performance in-point in the live preview timing and leaves untimed takes alone', async () => {+ const { project } = await fixture();+ Object.assign(project.scenes[3], { speedFps: -12, offsetFrames: 12 });+ const raw = await resolveSceneMedia(project, { history: JSON.parse(await readFile(join(PATHS.data, 'video-history.json'), 'utf8')) });+ expect(raw.get(project.scenes[3].sceneId)).toMatchObject({ inSec: 0.5, timing: { rate: 0.5, leadSec: 0, startSec: 1 } });+ expect(raw.get('a').timing).toBeUndefined();+ });+it('fails before opening the page and cleans scratch when shot retiming fails', async () => {const { project, plan } = await fixture();project.scenes[0].speedFps = 24;diff --git a/server/services/musicVideo/documentTemplates/layered/engine.js b/server/services/musicVideo/documentTemplates/layered/engine.jsindex 774c4355de..59f71cbfa6 100644--- a/server/services/musicVideo/documentTemplates/layered/engine.js+++ b/server/services/musicVideo/documentTemplates/layered/engine.js@@ -222,7 +222,11 @@if (next?.media?.src) (next.media.kind === 'video' ? loadVideo(next.media.src) : loadImage(next.media.src)).catch(() => {});if (media.kind === 'video') {const v = await loadVideo(media.src);- return seekVideo(v, media, (media.inSec || 0) + (t - scene.startSec));+ const local = t - scene.startSec;+ // Raw-asset previews carry the shot's speed/offset instead of a retimed file.+ const timing = media.timing;+ if (!timing) return seekVideo(v, media, (media.inSec || 0) + local);+ return seekVideo(v, { ...media, inSec: timing.startSec }, timing.startSec + Math.max(0, local - timing.leadSec) * timing.rate);}return loadImage(media.src);}