Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Apply shot speed and frame offset in live composition-document previews (#11406)

Adam Eivy5ef633d286b6
diff --git a/docs/HTML_COMPOSITIONS.md b/docs/HTML_COMPOSITIONS.md
index 65f91c5391..0291af680c 100644
--- a/docs/HTML_COMPOSITIONS.md
+++ b/docs/HTML_COMPOSITIONS.md
@@ -420,6 +420,11 @@ snapshot freezes (`server/services/musicVideo/documentRender.js`):
width, height } | null }], textCues, composition: { mode, style, posterSec,
overlay } }`. Load it with `<script src="portos-mv.js">`; the sandbox
refuses `fetch`.
+ A shot's `speedFps`/`offsetFrames` are baked into the staged clip, so a render
+ seeks `media.inSec + (t - startSec)`. The live preview bridges the original
+ bytes instead: its `media` keeps source-time `inSec`/`outSec` and adds
+ `timing: { rate, leadSec, startSec }` (only for a retimed shot), and the page
+ seeks `startSec + max(0, (t - scene.startSec) - leadSec) * rate`.
- `song.json` — the same song block the code-rendered mode reads.
- `media/scene-<sceneId>.<ext>` — each scene's selected take (its clip when
it has one, else its still), named by `scenes[].media.src`.
diff --git a/server/services/musicVideo/documentRender.browser.test.js b/server/services/musicVideo/documentRender.browser.test.js
index 3ac574b3ce..56b625b935 100644
--- a/server/services/musicVideo/documentRender.browser.test.js
+++ b/server/services/musicVideo/documentRender.browser.test.js
@@ -553,6 +553,50 @@ describe.skipIf(!canRun)('layered template with real Chrome and ffmpeg', () => {
}
}, 120000));
+ // Uniquely catches the live preview bridging raw bytes without the shot's speed/offset (same samples as the render above).
+ [
+ { timing: { speedFps: 24, offsetFrames: 12 }, samples: [[0, 1], [5, 1], [6, 2]] },
+ { timing: { speedFps: 24, offsetFrames: -6 }, samples: [[0, 0], [11, 0], [12, 1], [17, 1], [18, 2]] },
+ { timing: { offsetFrames: -6 }, samples: [[0, 0], [17, 0], [18, 1], [29, 1], [30, 2]] },
+ ].forEach(({ timing, samples }) => it(`plays shot speed and offset ${JSON.stringify(timing)} in the live preview`, async () => {
+ await mkdir(PATHS.videos, { recursive: true });
+ const clip = join(PATHS.videos, 'live-timed-rgb.webm');
+ execFileSync(ffmpeg, ['-v', 'error', '-y',
+ '-f', 'lavfi', '-i', 'color=c=red:s=64x36:r=24:d=0.5',
+ '-f', 'lavfi', '-i', 'color=c=lime:s=64x36:r=24:d=0.5',
+ '-f', 'lavfi', '-i', 'color=c=blue:s=64x36:r=24:d=0.5',
+ '-filter_complex', '[0:v][1:v][2:v]concat=n=3:v=1:a=0', '-c:v', 'libvpx', clip]);
+ const created = await projects.createProject({ name: 'Synthetic live shot timing' });
+ await projects.mutateProjectRecord(created.id, current => ({ project: { ...current,
+ audioAnalysis: { durationSec: 1.5, sections: [] },
+ scenes: [{ sceneId: 'timed', startSec: 0, endSec: 1.5, videoHistoryId: 'vh-live', ...timing }],
+ } }));
+ await writeFile(join(PATHS.data, 'video-history.json'), JSON.stringify([{ id: 'vh-live', filename: 'live-timed-rgb.webm', numFrames: 36, fps: 24, width: 64, height: 36 }]));
+ await importDocumentTemplate(created.id);
+ const preview = await buildDocumentPreview(await projects.getProject(created.id));
+ const page = await browser.newPage();
+ await page.setContent(preview.html);
+ const bytes = (await readFile(clip)).toString('base64');
+ await page.evaluate(async (base64) => {
+ const blob = new Blob([Uint8Array.from(atob(base64), c => c.charCodeAt(0))], { type: 'video/webm' });
+ addEventListener('message', ({ data }) => { if (data?.type === 'portos-mv:request') postMessage({ type: 'portos-mv:asset', key: data.key, blob }, '*'); });
+ postMessage({ type: 'portos-mv:manifest', keys: ['media/scene-timed.webm'] }, '*');
+ await window.PORTOS_MV_ASSETS;
+ }, bytes);
+ for (const [frame, channel] of samples) {
+ const rgb = await page.evaluate(async (frame) => {
+ await window.portosComposition.seek(frame / 24);
+ const stage = document.getElementById('stage');
+ const probe = document.createElement('canvas'); probe.width = stage.width; probe.height = stage.height;
+ const ctx = probe.getContext('2d'); ctx.drawImage(stage, 0, 0);
+ return [...ctx.getImageData(stage.width >> 1, stage.height >> 1, 1, 1).data.slice(0, 3)];
+ }, frame);
+ expect(rgb[channel], `frame ${frame}`).toBeGreaterThan(180);
+ rgb.forEach((value, i) => { if (i !== channel) expect(value, `frame ${frame}`).toBeLessThan(60); });
+ }
+ await page.close();
+ }, 120000));
+
it('draws the selected take at SONG time in an excerpt, identically on every render, with the master muxed', async () => {
const { cdpRequest } = await import('../browserService.js');
const managedCalls = cdpRequest.mock.calls.length;
diff --git a/server/services/musicVideo/documentRender.js b/server/services/musicVideo/documentRender.js
index 1557ec5f7f..3dc2adef4e 100644
--- a/server/services/musicVideo/documentRender.js
+++ b/server/services/musicVideo/documentRender.js
@@ -144,15 +144,31 @@ export async function resolveSceneMedia(project, { history = [], probe = async (
const duration = measured.numFrames && measured.fps ? measured.numFrames / measured.fps : null;
const edit = selectedPerformanceInstruction(scene)?.edit ?? null;
const ext = (/\.([a-z0-9]{2,5})$/i.exec(entry.filename)?.[1] || 'mp4').toLowerCase();
- const rate = retimed ? clipSpeedRate(scene) : 1;
+ const rate = clipSpeedRate(scene);
const originalIn = edit ? Math.min(edit.inSec, duration ?? edit.inSec) : 0;
const originalOut = edit ? Math.min(edit.outSec, duration ?? edit.outSec) : duration;
- const start = originalIn + (retimed ? clipOffsetSec(scene) : 0);
+ const start = originalIn + clipOffsetSec(scene);
const sourceIn = Math.min(duration ?? Infinity, Math.max(0, start));
// Offset holds are song/output seconds, independent of playback rate:
// clipTimeAt uses sourceIn + max(0, local + min(0, start)) * rate.
const leadSec = Math.max(0, -start);
const retime = rate !== 1 || leadSec > 0;
+ // A raw-asset preview plays the original bytes, so it keeps source-time
+ // in/out and carries the same timing for the page's seek to apply.
+ if (!retimed) {
+ out.set(scene.sceneId, {
+ kind: 'video',
+ path: videoPath,
+ ext,
+ inSec: round3(originalIn),
+ outSec: originalOut == null ? null : round3(originalOut),
+ ...(retime ? { timing: { rate, leadSec: round3(leadSec), startSec: round3(sourceIn) } } : {}),
+ fps: finite(measured.fps),
+ width: finite(measured.width),
+ height: finite(measured.height),
+ });
+ continue;
+ }
out.set(scene.sceneId, {
kind: 'video',
path: videoPath,
@@ -222,6 +238,8 @@ export function buildDocumentData(project, { media = new Map(), frame, clock, so
media: m ? {
kind: m.kind, src: mediaSrc(scene.sceneId, m),
inSec: m.inSec, outSec: m.outSec, fps: m.fps, width: m.width, height: m.height,
+ // Raw-asset previews only: seek source time = startSec + max(0, local - leadSec) * rate.
+ ...(m.timing ? { timing: m.timing } : {}),
} : null,
};
});
diff --git a/server/services/musicVideo/documentRender.test.js b/server/services/musicVideo/documentRender.test.js
index 6271d256bc..f3466df5e5 100644
--- a/server/services/musicVideo/documentRender.test.js
+++ b/server/services/musicVideo/documentRender.test.js
@@ -130,16 +130,18 @@ describe('isolated document preview ownership', () => {
describe('composition document staging', () => {
it.each([
- { timing: { speedFps: 24, offsetFrames: 12 }, inSec: 0.25, outSec: 2, filter: 'setpts=(PTS-STARTPTS)/2,fps=24' },
- { timing: { speedFps: 24, offsetFrames: -6 }, inSec: 0, outSec: 2.25, filter: 'setpts=(PTS-STARTPTS)/2,tpad=start_mode=clone:start_duration=0.25,fps=24' },
- { timing: { offsetFrames: -6 }, inSec: 0, outSec: 4.25, filter: 'setpts=(PTS-STARTPTS)/1,tpad=start_mode=clone:start_duration=0.25,fps=24' },
- ])('stages shot timing $timing on the document output timebase', async ({ timing, inSec, outSec, filter }) => {
+ { timing: { speedFps: 24, offsetFrames: 12 }, live: { rate: 2, leadSec: 0, startSec: 0.5 }, inSec: 0.25, outSec: 2, filter: 'setpts=(PTS-STARTPTS)/2,fps=24' },
+ { timing: { speedFps: 24, offsetFrames: -6 }, live: { rate: 2, leadSec: 0.25, startSec: 0 }, inSec: 0, outSec: 2.25, filter: 'setpts=(PTS-STARTPTS)/2,tpad=start_mode=clone:start_duration=0.25,fps=24' },
+ { timing: { offsetFrames: -6 }, live: { rate: 1, leadSec: 0.25, startSec: 0 }, inSec: 0, outSec: 4.25, filter: 'setpts=(PTS-STARTPTS)/1,tpad=start_mode=clone:start_duration=0.25,fps=24' },
+ ])('stages shot timing $timing on the document output timebase', async ({ timing, live, inSec, outSec, filter }) => {
const { project, plan } = await fixture();
Object.assign(project.scenes[0], timing);
// The live preview bridges original bytes, so it must retain source time.
const raw = await resolveSceneMedia(project, { history: JSON.parse(await readFile(join(PATHS.data, 'video-history.json'), 'utf8')) });
expect(raw.get('a')).toMatchObject({ ext: 'mp4', inSec: 0, outSec: 4, fps: 24 });
expect(raw.get('a').rate).toBeUndefined();
+ // The live preview applies speed/offset at seek time (engine.js) from this source-time timing.
+ expect(raw.get('a').timing).toEqual(live);
const controller = new AbortController();
await expect(encodeDocumentComposition({ project, plan, jobId: 'job-timing', signal: controller.signal })).rejects.toThrow('stop after staging');
expect(browser.seen.data.scenes[0].media).toMatchObject({ inSec, outSec, fps: 24 });
@@ -161,6 +163,14 @@ describe('composition document staging', () => {
expect(browser.seen.data.scenes[0].media).toMatchObject({ inSec: 0, outSec: 4 });
});
+ it('keeps a performance in-point in the live preview timing and leaves untimed takes alone', async () => {
+ const { project } = await fixture();
+ Object.assign(project.scenes[3], { speedFps: -12, offsetFrames: 12 });
+ const raw = await resolveSceneMedia(project, { history: JSON.parse(await readFile(join(PATHS.data, 'video-history.json'), 'utf8')) });
+ expect(raw.get(project.scenes[3].sceneId)).toMatchObject({ inSec: 0.5, timing: { rate: 0.5, leadSec: 0, startSec: 1 } });
+ expect(raw.get('a').timing).toBeUndefined();
+ });
+
it('fails before opening the page and cleans scratch when shot retiming fails', async () => {
const { project, plan } = await fixture();
project.scenes[0].speedFps = 24;
diff --git a/server/services/musicVideo/documentTemplates/layered/engine.js b/server/services/musicVideo/documentTemplates/layered/engine.js
index 774c4355de..59f71cbfa6 100644
--- a/server/services/musicVideo/documentTemplates/layered/engine.js
+++ b/server/services/musicVideo/documentTemplates/layered/engine.js
@@ -222,7 +222,11 @@
if (next?.media?.src) (next.media.kind === 'video' ? loadVideo(next.media.src) : loadImage(next.media.src)).catch(() => {});
if (media.kind === 'video') {
const v = await loadVideo(media.src);
- return seekVideo(v, media, (media.inSec || 0) + (t - scene.startSec));
+ const local = t - scene.startSec;
+ // Raw-asset previews carry the shot's speed/offset instead of a retimed file.
+ const timing = media.timing;
+ if (!timing) return seekVideo(v, media, (media.inSec || 0) + local);
+ return seekVideo(v, { ...media, inSec: timing.startSec }, timing.startSec + Math.max(0, local - timing.leadSec) * timing.rate);
}
return loadImage(media.src);
}