Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: apply shot speed and frame offset in document renders (#11393)

[._.]/ Adam Eivyce59aabf8ad2
diff --git a/server/services/musicVideo/documentRender.browser.test.js b/server/services/musicVideo/documentRender.browser.test.js
index f8c1891c9d..c4dc0bf161 100644
--- a/server/services/musicVideo/documentRender.browser.test.js
+++ b/server/services/musicVideo/documentRender.browser.test.js
@@ -517,6 +517,41 @@ describe.skipIf(!canRun)('layered template with real Chrome and ffmpeg', () => {
}
}, 120000);
+ // Uniquely catches correct timing metadata paired with an unretimed staged file.
+ [
+ { timing: { speedFps: 24, offsetFrames: 12 }, samples: [[0, 1], [5, 1], [6, 2]] },
+ { timing: { offsetFrames: -6 }, samples: [[0, 0], [17, 0], [18, 1], [29, 1], [30, 2]] },
+ ].forEach(({ timing, samples }) => it(`renders shot speed and offset ${JSON.stringify(timing)} into the document pixels`, async () => {
+ await mkdir(PATHS.videos, { recursive: true });
+ await mkdir(PATHS.music, { recursive: true });
+ const clip = join(PATHS.videos, 'timed-rgb.webm');
+ execFileSync(ffmpeg, ['-v', 'error', '-y',
+ '-f', 'lavfi', '-i', 'color=c=red:s=64x36:r=24:d=0.5',
+ '-f', 'lavfi', '-i', 'color=c=lime:s=64x36:r=24:d=0.5',
+ '-f', 'lavfi', '-i', 'color=c=blue:s=64x36:r=24:d=0.5',
+ '-filter_complex', '[0:v][1:v][2:v]concat=n=3:v=1:a=0', '-c:v', 'libvpx', clip]);
+ const master = join(PATHS.music, 'timed-master.wav');
+ execFileSync(ffmpeg, ['-v', 'error', '-y', '-f', 'lavfi', '-i', 'sine=frequency=220:duration=1.5', master]);
+ const created = await projects.createProject({ name: 'Synthetic shot timing' });
+ await projects.mutateProjectRecord(created.id, current => ({ project: { ...current,
+ audioAnalysis: { durationSec: 1.5, sections: [] },
+ scenes: [{ sceneId: 'timed', startSec: 0, endSec: 1.5, videoHistoryId: 'vh-timed', ...timing }],
+ } }));
+ await writeFile(join(PATHS.data, 'video-history.json'), JSON.stringify([{ id: 'vh-timed', filename: 'timed-rgb.webm', numFrames: 36, fps: 24, width: 64, height: 36 }]));
+ await importDocumentTemplate(created.id);
+ const project = await projects.getProject(created.id);
+ const outputPath = join(PATHS.videos, 'timed-output.mp4');
+ await encodeDocumentComposition({ project, plan: await prepareDocumentRender(project), jobId: 'timed-pixels', audioPath: master, outputPath });
+ const pixels = execFileSync(ffmpeg, ['-v', 'error', '-i', outputPath, '-vf', 'scale=64:36', '-f', 'rawvideo', '-pix_fmt', 'rgb24', '-']);
+ for (const [frame, channel] of samples) {
+ const at = (frame * 64 * 36 + 18 * 64 + 32) * 3;
+ const rgb = [...pixels.subarray(at, at + 3)];
+ expect(rgb).toHaveLength(3);
+ expect(rgb[channel], `frame ${frame}`).toBeGreaterThan(180);
+ rgb.forEach((value, i) => { if (i !== channel) expect(value, `frame ${frame}`).toBeLessThan(60); });
+ }
+ }, 120000));
+
it('draws the selected take at SONG time in an excerpt, identically on every render, with the master muxed', async () => {
const { cdpRequest } = await import('../browserService.js');
const managedCalls = cdpRequest.mock.calls.length;
diff --git a/server/services/musicVideo/documentRender.js b/server/services/musicVideo/documentRender.js
index f6fdff052b..136e9ab8d9 100644
--- a/server/services/musicVideo/documentRender.js
+++ b/server/services/musicVideo/documentRender.js
@@ -38,7 +38,7 @@ import { ServerError } from '../../lib/errorHandler.js';
import { PATHS } from '../../lib/fileUtils.js';
import { htmlCompositionContractSchemaFor } from '../../lib/validation.js';
import { selectedPerformanceInstruction } from '../../lib/musicVideoShotTiming.js';
-import { documentSceneVisualLayer, MUSIC_VIDEO_LYRIC_ROLES, MUSIC_VIDEO_TEXT_ZONES } from '../../lib/musicVideoLayers.js';
+import { clipSpeedRate, clipOffsetSec, documentSceneVisualLayer, MUSIC_VIDEO_LYRIC_ROLES, MUSIC_VIDEO_TEXT_ZONES } from '../../lib/musicVideoLayers.js';
import { musicVideoAspect } from '../../lib/musicVideoAspect.js';
import { documentDirectoryForRender } from './compositionDocument.js';
import { musicVideoSongDocument } from './compositionRender.js';
@@ -120,12 +120,14 @@ export function documentRenderClock(songDurationSec, fps = DOCUMENT_RENDER_FPS)
/**
* Each scene's selected take as a file to stage: `Map<sceneId, { kind, path,
- * ext, inSec, outSec, fps, width, height }>`. Video is preferred over the still;
+ * ext, inSec, outSec, fps, width, height }>`. `retimed` requests output-time
+ * in/out points and staging instructions; raw-asset previews keep source time.
+ * Video is preferred over the still;
* a scene whose selection is missing on disk gets no entry (media: null).
* `history` is the video history list; `probe(path)` measures a clip the
* history did not (hosted renders record no geometry).
*/
-export async function resolveSceneMedia(project, { history = [], probe = async () => null, strictLayers = false } = {}) {
+export async function resolveSceneMedia(project, { history = [], probe = async () => null, strictLayers = false, retimed = false } = {}) {
assertMusicVideoMediaSelections(project);
const byId = new Map((Array.isArray(history) ? history : []).map((entry) => [entry.id, entry]));
const out = new Map();
@@ -142,13 +144,21 @@ export async function resolveSceneMedia(project, { history = [], probe = async (
const duration = measured.numFrames && measured.fps ? measured.numFrames / measured.fps : null;
const edit = selectedPerformanceInstruction(scene)?.edit ?? null;
const ext = (/\.([a-z0-9]{2,5})$/i.exec(entry.filename)?.[1] || 'mp4').toLowerCase();
+ const rate = retimed ? clipSpeedRate(scene) : 1;
+ const originalIn = edit ? Math.min(edit.inSec, duration ?? edit.inSec) : 0;
+ const originalOut = edit ? Math.min(edit.outSec, duration ?? edit.outSec) : duration;
+ const start = originalIn + (retimed ? clipOffsetSec(scene) : 0);
+ const sourceIn = Math.min(duration ?? Infinity, Math.max(0, start));
+ const leadSec = Math.max(0, -start);
+ const retime = rate !== 1 || leadSec > 0;
out.set(scene.sceneId, {
kind: 'video',
path: videoPath,
- ext,
- inSec: edit ? round3(Math.min(edit.inSec, duration ?? edit.inSec)) : 0,
- outSec: edit ? round3(Math.min(edit.outSec, duration ?? edit.outSec)) : (duration ? round3(duration) : null),
- fps: finite(measured.fps),
+ ext: retime ? 'mp4' : ext,
+ inSec: round3(sourceIn / rate),
+ outSec: originalOut == null ? null : round3(leadSec + Math.max(sourceIn, originalOut) / rate),
+ ...(retime ? { rate, leadSec } : {}),
+ fps: retime ? DOCUMENT_RENDER_FPS : finite(measured.fps),
width: finite(measured.width),
height: finite(measured.height),
});
@@ -267,12 +277,34 @@ export async function prepareDocumentRender(project) {
}
/** Write portos-mv.js and copy the selected takes into a staged document folder. */
-async function stageDocumentData(compositionDir, data, media) {
+async function stageDocumentData(compositionDir, data, media, signal) {
await mkdir(join(compositionDir, 'media'), { recursive: true });
for (const scene of data.scenes) {
const m = scene.media && media.get(scene.sceneId);
if (!m) continue;
const target = join(compositionDir, scene.media.src);
+ signal?.throwIfAborted();
+ if (m.kind === 'video' && m.rate != null) {
+ // Documents seek output seconds themselves. Retime the staged copy so
+ // existing/custom documents need no new playback contract.
+ if (existsSync(target)) throw new ServerError(`The composition document ships ${scene.media.src}, a name PortOS writes at render time`, { status: 422, code: 'COMPOSITION_DOCUMENT_INVALID' });
+ const { findFfmpeg, runFfmpegProcess, H264_ENCODE_ARGS } = await import('../../lib/ffmpeg.js');
+ const bin = await findFfmpeg();
+ if (!bin) throw new Error('ffmpeg not found on PATH');
+ const filters = [
+ `setpts=(PTS-STARTPTS)/${Number(m.rate.toFixed(6))}`,
+ ...(m.leadSec > 0 ? [`tpad=start_mode=clone:start_duration=${round3(m.leadSec)}`] : []),
+ `fps=${DOCUMENT_RENDER_FPS}`,
+ ];
+ const result = await runFfmpegProcess({ bin, signal, args: [
+ '-hide_banner', '-loglevel', 'error', '-i', m.path,
+ '-map', '0:v:0', '-vf', filters.join(','), ...H264_ENCODE_ARGS,
+ '-an', '-movflags', '+faststart', '-n', target,
+ ] });
+ signal?.throwIfAborted();
+ if (!result.ok) throw new Error(`Shot timing failed: ${result.reason}`);
+ continue;
+ }
// Clone where the file system can (APFS), refuse to overwrite a file the
// document shipped under PortOS's reserved name.
await copyFile(m.path, target, fsConstants.COPYFILE_EXCL | fsConstants.COPYFILE_FICLONE).catch((error) => {
@@ -296,7 +328,7 @@ export async function openIsolatedDocumentPreview({ project, jobId, signal, onEv
const { loadHistory } = await import('../videoGen/history.js');
const generated = project.composition?.document?.source?.kind === 'generated';
const history = (project.scenes || []).some((scene) => scene?.videoHistoryId) ? await loadHistory() : [];
- const media = await resolveSceneMedia(project, { history, probe: probeVideoGeometry, strictLayers: generated });
+ const media = await resolveSceneMedia(project, { history, probe: probeVideoGeometry, retimed: true, strictLayers: generated });
const data = buildDocumentData(project, {
media, frame: plan.frame, clock: plan.clock, songDurationSec: plan.songDurationSec, generated,
});
@@ -304,7 +336,7 @@ export async function openIsolatedDocumentPreview({ project, jobId, signal, onEv
let page;
try {
staged = await stageMusicVideoComposition(plan.directory, jobId, musicVideoSongDocument(project), {
- prepare: (dir) => stageDocumentData(dir, data, media),
+ prepare: (dir) => stageDocumentData(dir, data, media, signal),
});
signal?.throwIfAborted();
page = await openComposition(staged.directory, {
@@ -397,7 +429,7 @@ export async function encodeDocumentComposition({
const ffmpeg = await findFfmpeg();
if (!ffmpeg) throw new ServerError('ffmpeg not found on PATH', { status: 500, code: 'FFMPEG_MISSING' });
const history = (project.scenes || []).some((s) => s?.videoHistoryId) ? await loadHistory() : [];
- const media = await resolveSceneMedia(project, { history, probe: probeVideoGeometry, strictLayers: project.composition?.document?.source?.kind === 'generated' });
+ const media = await resolveSceneMedia(project, { history, probe: probeVideoGeometry, retimed: true, strictLayers: project.composition?.document?.source?.kind === 'generated' });
const data = buildDocumentData(project, { media, frame: plan.frame, clock: plan.clock, songDurationSec: plan.songDurationSec,
generated: project.composition?.document?.source?.kind === 'generated' });
const silent = `${outputPath}.silent.mp4`;
@@ -405,7 +437,7 @@ export async function encodeDocumentComposition({
let page;
try {
staged = await stageMusicVideoComposition(plan.directory, jobId, musicVideoSongDocument(project), {
- prepare: (dir) => stageDocumentData(dir, data, media),
+ prepare: (dir) => stageDocumentData(dir, data, media, signal),
});
signal?.throwIfAborted();
const openPage = (pageSignal) => openComposition(staged.directory, { signal: pageSignal, streamMedia: true, mediaMode: musicVideoMediaMode(project), ownedBrowser: true,
@@ -536,13 +568,13 @@ export async function probeDocumentText({ project, jobId, signal, sampleTimes, o
const { default: sharp } = await import('sharp');
const generated = project.composition?.document?.source?.kind === 'generated';
const history = (project.scenes || []).some((s) => s?.videoHistoryId) ? await loadHistory() : [];
- const media = await resolveSceneMedia(project, { history, probe: probeVideoGeometry, strictLayers: generated });
+ const media = await resolveSceneMedia(project, { history, probe: probeVideoGeometry, retimed: true, strictLayers: generated });
const data = buildDocumentData(project, { media, frame: plan.frame, clock: plan.clock, songDurationSec: plan.songDurationSec, generated });
let staged;
let page;
try {
staged = await stageMusicVideoComposition(plan.directory, jobId, musicVideoSongDocument(project), {
- prepare: (dir) => stageDocumentData(dir, data, media),
+ prepare: (dir) => stageDocumentData(dir, data, media, signal),
});
signal?.throwIfAborted();
page = await openComposition(staged.directory, { signal, streamMedia: true, mediaMode: musicVideoMediaMode(project), ownedBrowser: true,
diff --git a/server/services/musicVideo/documentRender.test.js b/server/services/musicVideo/documentRender.test.js
index 30b4acc870..bbdd10cee3 100644
--- a/server/services/musicVideo/documentRender.test.js
+++ b/server/services/musicVideo/documentRender.test.js
@@ -24,7 +24,10 @@ vi.mock('../../lib/paths.js', async (importOriginal) => makePathsProxy(await imp
vi.mock('../../lib/ffmpeg.js', async (importOriginal) => ({
...(await importOriginal()),
findFfmpeg: async () => 'ffmpeg',
- runFfmpegProcess: async () => ({ ok: true }),
+ runFfmpegProcess: vi.fn(async ({ args }) => {
+ if (args.includes('-n')) await writeFile(args.at(-1), await readFile(args[args.indexOf('-i') + 1]));
+ return { ok: true };
+ }),
}));
// Capture the frozen job folder the page would open, then stop the render.
vi.mock('../htmlComposition/browser.js', () => ({
@@ -52,11 +55,12 @@ vi.mock('../htmlComposition/browser.js', () => ({
},
}));
+const { runFfmpegProcess } = await import('../../lib/ffmpeg.js');
const { PATHS } = await import('../../lib/paths.js');
-const { encodeDocumentComposition, documentRenderClock, openIsolatedDocumentPreview } = await import('./documentRender.js');
+const { encodeDocumentComposition, documentRenderClock, openIsolatedDocumentPreview, resolveSceneMedia } = await import('./documentRender.js');
afterAll(() => cleanupTempDataRoots());
-beforeEach(() => { browser.seen = null; browser.contract = null; browser.visibility = null; browser.sampleTimes = []; browser.send = null; browser.closed = []; });
+beforeEach(() => { runFfmpegProcess.mockClear(); browser.seen = null; browser.contract = null; browser.visibility = null; browser.sampleTimes = []; browser.send = null; browser.closed = []; });
const performanceTake = (assetId) => ({ takeId: 'mvt-p', kind: 'video', assetId, status: 'candidate', shotInstruction: { shotMode: 'performance', edit: { inSec: 0.5, outSec: 3.5 } } });
@@ -125,6 +129,46 @@ describe('isolated document preview ownership', () => {
});
describe('composition document staging', () => {
+ it.each([
+ { timing: { speedFps: 24, offsetFrames: 12 }, inSec: 0.25, outSec: 2, filter: 'setpts=(PTS-STARTPTS)/2,fps=24' },
+ { timing: { offsetFrames: -6 }, inSec: 0, outSec: 4.25, filter: 'setpts=(PTS-STARTPTS)/1,tpad=start_mode=clone:start_duration=0.25,fps=24' },
+ ])('stages shot timing $timing on the document output timebase', async ({ timing, inSec, outSec, filter }) => {
+ const { project, plan } = await fixture();
+ Object.assign(project.scenes[0], timing);
+ // The live preview bridges original bytes, so it must retain source time.
+ const raw = await resolveSceneMedia(project, { history: JSON.parse(await readFile(join(PATHS.data, 'video-history.json'), 'utf8')) });
+ expect(raw.get('a')).toMatchObject({ ext: 'mp4', inSec: 0, outSec: 4, fps: 24 });
+ expect(raw.get('a').rate).toBeUndefined();
+ const controller = new AbortController();
+ await expect(encodeDocumentComposition({ project, plan, jobId: 'job-timing', signal: controller.signal })).rejects.toThrow('stop after staging');
+ expect(browser.seen.data.scenes[0].media).toMatchObject({ inSec, outSec, fps: 24 });
+ expect(runFfmpegProcess).toHaveBeenCalledTimes(1);
+ const call = runFfmpegProcess.mock.calls[0][0];
+ expect(call.signal).toBe(controller.signal);
+ expect(call.args[call.args.indexOf('-vf') + 1]).toBe(filter);
+ expect(call.args).toContain('-an');
+ expect(browser.seen.media['scene-a.mp4']).toBe('video-a');
+ expect(await readFile(join(PATHS.videos, 'clip-a.mp4'), 'utf8')).toBe('video-a');
+ expect(await scratchEntries()).toEqual([]);
+ });
+
+ it('combines a performance in-point with offset and speed, without changing untimed takes', async () => {
+ const { project, plan } = await fixture();
+ Object.assign(project.scenes[3], { speedFps: -12, offsetFrames: 12 });
+ await expect(encode(project, plan, 'job-performance-timing')).rejects.toThrow('stop after staging');
+ expect(browser.seen.data.scenes[3].media).toMatchObject({ inSec: 2, outSec: 7 });
+ expect(browser.seen.data.scenes[0].media).toMatchObject({ inSec: 0, outSec: 4 });
+ });
+
+ it('fails before opening the page and cleans scratch when shot retiming fails', async () => {
+ const { project, plan } = await fixture();
+ project.scenes[0].speedFps = 24;
+ runFfmpegProcess.mockResolvedValueOnce({ ok: false, reason: 'synthetic retiming failure' });
+ await expect(encode(project, plan, 'job-timing-failure')).rejects.toThrow('Shot timing failed: synthetic retiming failure');
+ expect(browser.seen).toBeNull();
+ expect(await scratchEntries()).toEqual([]);
+ });
+
it('preserves the document shutter sampling contract through the shared encoder', async () => {
const { project, plan } = await fixture();
browser.contract = { durationSec: 30, fps: 24, width: 1920, height: 1080, motionBlur: 4, layout: false };
@@ -148,6 +192,7 @@ describe('composition document staging', () => {
expect(data.textCues.map((c) => c.emphasis)).toEqual(['hero']);
expect(data.composition).toMatchObject({ mode: 'document', overlay: { enabled: true, ticker: ['a'] } });
expect(media).toEqual({ 'scene-a.mp4': 'video-a', 'scene-b.png': 'image-b', 'scene-d.mp4': 'video-a' });
+ expect(runFfmpegProcess).not.toHaveBeenCalled();
expect(song).toMatchObject({ beats: [0, 0.5], downbeats: [0] });
// The source document is untouched and the job folder is gone.
expect(existsSync(join(PATHS.data, plan.directory, 'portos-mv.js'))).toBe(false);