Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: apply shot speed and frame offset in document renders (#11393)
[._.]/ Adam Eivyce59aabf8ad2
diff --git a/server/services/musicVideo/documentRender.browser.test.js b/server/services/musicVideo/documentRender.browser.test.jsindex f8c1891c9d..c4dc0bf161 100644--- a/server/services/musicVideo/documentRender.browser.test.js+++ b/server/services/musicVideo/documentRender.browser.test.js@@ -517,6 +517,41 @@ describe.skipIf(!canRun)('layered template with real Chrome and ffmpeg', () => {}}, 120000);+ // Uniquely catches correct timing metadata paired with an unretimed staged file.+ [+ { timing: { speedFps: 24, offsetFrames: 12 }, samples: [[0, 1], [5, 1], [6, 2]] },+ { timing: { offsetFrames: -6 }, samples: [[0, 0], [17, 0], [18, 1], [29, 1], [30, 2]] },+ ].forEach(({ timing, samples }) => it(`renders shot speed and offset ${JSON.stringify(timing)} into the document pixels`, async () => {+ await mkdir(PATHS.videos, { recursive: true });+ await mkdir(PATHS.music, { recursive: true });+ const clip = join(PATHS.videos, 'timed-rgb.webm');+ execFileSync(ffmpeg, ['-v', 'error', '-y',+ '-f', 'lavfi', '-i', 'color=c=red:s=64x36:r=24:d=0.5',+ '-f', 'lavfi', '-i', 'color=c=lime:s=64x36:r=24:d=0.5',+ '-f', 'lavfi', '-i', 'color=c=blue:s=64x36:r=24:d=0.5',+ '-filter_complex', '[0:v][1:v][2:v]concat=n=3:v=1:a=0', '-c:v', 'libvpx', clip]);+ const master = join(PATHS.music, 'timed-master.wav');+ execFileSync(ffmpeg, ['-v', 'error', '-y', '-f', 'lavfi', '-i', 'sine=frequency=220:duration=1.5', master]);+ const created = await projects.createProject({ name: 'Synthetic shot timing' });+ await projects.mutateProjectRecord(created.id, current => ({ project: { ...current,+ audioAnalysis: { durationSec: 1.5, sections: [] },+ scenes: [{ sceneId: 'timed', startSec: 0, endSec: 1.5, videoHistoryId: 'vh-timed', ...timing }],+ } }));+ await writeFile(join(PATHS.data, 'video-history.json'), JSON.stringify([{ id: 'vh-timed', filename: 'timed-rgb.webm', numFrames: 36, fps: 24, width: 64, height: 36 }]));+ await importDocumentTemplate(created.id);+ const project = await projects.getProject(created.id);+ const outputPath = join(PATHS.videos, 'timed-output.mp4');+ await encodeDocumentComposition({ project, plan: await prepareDocumentRender(project), jobId: 'timed-pixels', audioPath: master, outputPath });+ const pixels = execFileSync(ffmpeg, ['-v', 'error', '-i', outputPath, '-vf', 'scale=64:36', '-f', 'rawvideo', '-pix_fmt', 'rgb24', '-']);+ for (const [frame, channel] of samples) {+ const at = (frame * 64 * 36 + 18 * 64 + 32) * 3;+ const rgb = [...pixels.subarray(at, at + 3)];+ expect(rgb).toHaveLength(3);+ expect(rgb[channel], `frame ${frame}`).toBeGreaterThan(180);+ rgb.forEach((value, i) => { if (i !== channel) expect(value, `frame ${frame}`).toBeLessThan(60); });+ }+ }, 120000));+it('draws the selected take at SONG time in an excerpt, identically on every render, with the master muxed', async () => {const { cdpRequest } = await import('../browserService.js');const managedCalls = cdpRequest.mock.calls.length;diff --git a/server/services/musicVideo/documentRender.js b/server/services/musicVideo/documentRender.jsindex f6fdff052b..136e9ab8d9 100644--- a/server/services/musicVideo/documentRender.js+++ b/server/services/musicVideo/documentRender.js@@ -38,7 +38,7 @@ import { ServerError } from '../../lib/errorHandler.js';import { PATHS } from '../../lib/fileUtils.js';import { htmlCompositionContractSchemaFor } from '../../lib/validation.js';import { selectedPerformanceInstruction } from '../../lib/musicVideoShotTiming.js';-import { documentSceneVisualLayer, MUSIC_VIDEO_LYRIC_ROLES, MUSIC_VIDEO_TEXT_ZONES } from '../../lib/musicVideoLayers.js';+import { clipSpeedRate, clipOffsetSec, documentSceneVisualLayer, MUSIC_VIDEO_LYRIC_ROLES, MUSIC_VIDEO_TEXT_ZONES } from '../../lib/musicVideoLayers.js';import { musicVideoAspect } from '../../lib/musicVideoAspect.js';import { documentDirectoryForRender } from './compositionDocument.js';import { musicVideoSongDocument } from './compositionRender.js';@@ -120,12 +120,14 @@ export function documentRenderClock(songDurationSec, fps = DOCUMENT_RENDER_FPS)/*** Each scene's selected take as a file to stage: `Map<sceneId, { kind, path,- * ext, inSec, outSec, fps, width, height }>`. Video is preferred over the still;+ * ext, inSec, outSec, fps, width, height }>`. `retimed` requests output-time+ * in/out points and staging instructions; raw-asset previews keep source time.+ * Video is preferred over the still;* a scene whose selection is missing on disk gets no entry (media: null).* `history` is the video history list; `probe(path)` measures a clip the* history did not (hosted renders record no geometry).*/-export async function resolveSceneMedia(project, { history = [], probe = async () => null, strictLayers = false } = {}) {+export async function resolveSceneMedia(project, { history = [], probe = async () => null, strictLayers = false, retimed = false } = {}) {assertMusicVideoMediaSelections(project);const byId = new Map((Array.isArray(history) ? history : []).map((entry) => [entry.id, entry]));const out = new Map();@@ -142,13 +144,21 @@ export async function resolveSceneMedia(project, { history = [], probe = async (const duration = measured.numFrames && measured.fps ? measured.numFrames / measured.fps : null;const edit = selectedPerformanceInstruction(scene)?.edit ?? null;const ext = (/\.([a-z0-9]{2,5})$/i.exec(entry.filename)?.[1] || 'mp4').toLowerCase();+ const rate = retimed ? clipSpeedRate(scene) : 1;+ const originalIn = edit ? Math.min(edit.inSec, duration ?? edit.inSec) : 0;+ const originalOut = edit ? Math.min(edit.outSec, duration ?? edit.outSec) : duration;+ const start = originalIn + (retimed ? clipOffsetSec(scene) : 0);+ const sourceIn = Math.min(duration ?? Infinity, Math.max(0, start));+ const leadSec = Math.max(0, -start);+ const retime = rate !== 1 || leadSec > 0;out.set(scene.sceneId, {kind: 'video',path: videoPath,- ext,- inSec: edit ? round3(Math.min(edit.inSec, duration ?? edit.inSec)) : 0,- outSec: edit ? round3(Math.min(edit.outSec, duration ?? edit.outSec)) : (duration ? round3(duration) : null),- fps: finite(measured.fps),+ ext: retime ? 'mp4' : ext,+ inSec: round3(sourceIn / rate),+ outSec: originalOut == null ? null : round3(leadSec + Math.max(sourceIn, originalOut) / rate),+ ...(retime ? { rate, leadSec } : {}),+ fps: retime ? DOCUMENT_RENDER_FPS : finite(measured.fps),width: finite(measured.width),height: finite(measured.height),});@@ -267,12 +277,34 @@ export async function prepareDocumentRender(project) {}/** Write portos-mv.js and copy the selected takes into a staged document folder. */-async function stageDocumentData(compositionDir, data, media) {+async function stageDocumentData(compositionDir, data, media, signal) {await mkdir(join(compositionDir, 'media'), { recursive: true });for (const scene of data.scenes) {const m = scene.media && media.get(scene.sceneId);if (!m) continue;const target = join(compositionDir, scene.media.src);+ signal?.throwIfAborted();+ if (m.kind === 'video' && m.rate != null) {+ // Documents seek output seconds themselves. Retime the staged copy so+ // existing/custom documents need no new playback contract.+ if (existsSync(target)) throw new ServerError(`The composition document ships ${scene.media.src}, a name PortOS writes at render time`, { status: 422, code: 'COMPOSITION_DOCUMENT_INVALID' });+ const { findFfmpeg, runFfmpegProcess, H264_ENCODE_ARGS } = await import('../../lib/ffmpeg.js');+ const bin = await findFfmpeg();+ if (!bin) throw new Error('ffmpeg not found on PATH');+ const filters = [+ `setpts=(PTS-STARTPTS)/${Number(m.rate.toFixed(6))}`,+ ...(m.leadSec > 0 ? [`tpad=start_mode=clone:start_duration=${round3(m.leadSec)}`] : []),+ `fps=${DOCUMENT_RENDER_FPS}`,+ ];+ const result = await runFfmpegProcess({ bin, signal, args: [+ '-hide_banner', '-loglevel', 'error', '-i', m.path,+ '-map', '0:v:0', '-vf', filters.join(','), ...H264_ENCODE_ARGS,+ '-an', '-movflags', '+faststart', '-n', target,+ ] });+ signal?.throwIfAborted();+ if (!result.ok) throw new Error(`Shot timing failed: ${result.reason}`);+ continue;+ }// Clone where the file system can (APFS), refuse to overwrite a file the// document shipped under PortOS's reserved name.await copyFile(m.path, target, fsConstants.COPYFILE_EXCL | fsConstants.COPYFILE_FICLONE).catch((error) => {@@ -296,7 +328,7 @@ export async function openIsolatedDocumentPreview({ project, jobId, signal, onEvconst { loadHistory } = await import('../videoGen/history.js');const generated = project.composition?.document?.source?.kind === 'generated';const history = (project.scenes || []).some((scene) => scene?.videoHistoryId) ? await loadHistory() : [];- const media = await resolveSceneMedia(project, { history, probe: probeVideoGeometry, strictLayers: generated });+ const media = await resolveSceneMedia(project, { history, probe: probeVideoGeometry, retimed: true, strictLayers: generated });const data = buildDocumentData(project, {media, frame: plan.frame, clock: plan.clock, songDurationSec: plan.songDurationSec, generated,});@@ -304,7 +336,7 @@ export async function openIsolatedDocumentPreview({ project, jobId, signal, onEvlet page;try {staged = await stageMusicVideoComposition(plan.directory, jobId, musicVideoSongDocument(project), {- prepare: (dir) => stageDocumentData(dir, data, media),+ prepare: (dir) => stageDocumentData(dir, data, media, signal),});signal?.throwIfAborted();page = await openComposition(staged.directory, {@@ -397,7 +429,7 @@ export async function encodeDocumentComposition({const ffmpeg = await findFfmpeg();if (!ffmpeg) throw new ServerError('ffmpeg not found on PATH', { status: 500, code: 'FFMPEG_MISSING' });const history = (project.scenes || []).some((s) => s?.videoHistoryId) ? await loadHistory() : [];- const media = await resolveSceneMedia(project, { history, probe: probeVideoGeometry, strictLayers: project.composition?.document?.source?.kind === 'generated' });+ const media = await resolveSceneMedia(project, { history, probe: probeVideoGeometry, retimed: true, strictLayers: project.composition?.document?.source?.kind === 'generated' });const data = buildDocumentData(project, { media, frame: plan.frame, clock: plan.clock, songDurationSec: plan.songDurationSec,generated: project.composition?.document?.source?.kind === 'generated' });const silent = `${outputPath}.silent.mp4`;@@ -405,7 +437,7 @@ export async function encodeDocumentComposition({let page;try {staged = await stageMusicVideoComposition(plan.directory, jobId, musicVideoSongDocument(project), {- prepare: (dir) => stageDocumentData(dir, data, media),+ prepare: (dir) => stageDocumentData(dir, data, media, signal),});signal?.throwIfAborted();const openPage = (pageSignal) => openComposition(staged.directory, { signal: pageSignal, streamMedia: true, mediaMode: musicVideoMediaMode(project), ownedBrowser: true,@@ -536,13 +568,13 @@ export async function probeDocumentText({ project, jobId, signal, sampleTimes, oconst { default: sharp } = await import('sharp');const generated = project.composition?.document?.source?.kind === 'generated';const history = (project.scenes || []).some((s) => s?.videoHistoryId) ? await loadHistory() : [];- const media = await resolveSceneMedia(project, { history, probe: probeVideoGeometry, strictLayers: generated });+ const media = await resolveSceneMedia(project, { history, probe: probeVideoGeometry, retimed: true, strictLayers: generated });const data = buildDocumentData(project, { media, frame: plan.frame, clock: plan.clock, songDurationSec: plan.songDurationSec, generated });let staged;let page;try {staged = await stageMusicVideoComposition(plan.directory, jobId, musicVideoSongDocument(project), {- prepare: (dir) => stageDocumentData(dir, data, media),+ prepare: (dir) => stageDocumentData(dir, data, media, signal),});signal?.throwIfAborted();page = await openComposition(staged.directory, { signal, streamMedia: true, mediaMode: musicVideoMediaMode(project), ownedBrowser: true,diff --git a/server/services/musicVideo/documentRender.test.js b/server/services/musicVideo/documentRender.test.jsindex 30b4acc870..bbdd10cee3 100644--- a/server/services/musicVideo/documentRender.test.js+++ b/server/services/musicVideo/documentRender.test.js@@ -24,7 +24,10 @@ vi.mock('../../lib/paths.js', async (importOriginal) => makePathsProxy(await impvi.mock('../../lib/ffmpeg.js', async (importOriginal) => ({...(await importOriginal()),findFfmpeg: async () => 'ffmpeg',- runFfmpegProcess: async () => ({ ok: true }),+ runFfmpegProcess: vi.fn(async ({ args }) => {+ if (args.includes('-n')) await writeFile(args.at(-1), await readFile(args[args.indexOf('-i') + 1]));+ return { ok: true };+ }),}));// Capture the frozen job folder the page would open, then stop the render.vi.mock('../htmlComposition/browser.js', () => ({@@ -52,11 +55,12 @@ vi.mock('../htmlComposition/browser.js', () => ({},}));+const { runFfmpegProcess } = await import('../../lib/ffmpeg.js');const { PATHS } = await import('../../lib/paths.js');-const { encodeDocumentComposition, documentRenderClock, openIsolatedDocumentPreview } = await import('./documentRender.js');+const { encodeDocumentComposition, documentRenderClock, openIsolatedDocumentPreview, resolveSceneMedia } = await import('./documentRender.js');afterAll(() => cleanupTempDataRoots());-beforeEach(() => { browser.seen = null; browser.contract = null; browser.visibility = null; browser.sampleTimes = []; browser.send = null; browser.closed = []; });+beforeEach(() => { runFfmpegProcess.mockClear(); browser.seen = null; browser.contract = null; browser.visibility = null; browser.sampleTimes = []; browser.send = null; browser.closed = []; });const performanceTake = (assetId) => ({ takeId: 'mvt-p', kind: 'video', assetId, status: 'candidate', shotInstruction: { shotMode: 'performance', edit: { inSec: 0.5, outSec: 3.5 } } });@@ -125,6 +129,46 @@ describe('isolated document preview ownership', () => {});describe('composition document staging', () => {+ it.each([+ { timing: { speedFps: 24, offsetFrames: 12 }, inSec: 0.25, outSec: 2, filter: 'setpts=(PTS-STARTPTS)/2,fps=24' },+ { timing: { offsetFrames: -6 }, inSec: 0, outSec: 4.25, filter: 'setpts=(PTS-STARTPTS)/1,tpad=start_mode=clone:start_duration=0.25,fps=24' },+ ])('stages shot timing $timing on the document output timebase', async ({ timing, inSec, outSec, filter }) => {+ const { project, plan } = await fixture();+ Object.assign(project.scenes[0], timing);+ // The live preview bridges original bytes, so it must retain source time.+ const raw = await resolveSceneMedia(project, { history: JSON.parse(await readFile(join(PATHS.data, 'video-history.json'), 'utf8')) });+ expect(raw.get('a')).toMatchObject({ ext: 'mp4', inSec: 0, outSec: 4, fps: 24 });+ expect(raw.get('a').rate).toBeUndefined();+ const controller = new AbortController();+ await expect(encodeDocumentComposition({ project, plan, jobId: 'job-timing', signal: controller.signal })).rejects.toThrow('stop after staging');+ expect(browser.seen.data.scenes[0].media).toMatchObject({ inSec, outSec, fps: 24 });+ expect(runFfmpegProcess).toHaveBeenCalledTimes(1);+ const call = runFfmpegProcess.mock.calls[0][0];+ expect(call.signal).toBe(controller.signal);+ expect(call.args[call.args.indexOf('-vf') + 1]).toBe(filter);+ expect(call.args).toContain('-an');+ expect(browser.seen.media['scene-a.mp4']).toBe('video-a');+ expect(await readFile(join(PATHS.videos, 'clip-a.mp4'), 'utf8')).toBe('video-a');+ expect(await scratchEntries()).toEqual([]);+ });++ it('combines a performance in-point with offset and speed, without changing untimed takes', async () => {+ const { project, plan } = await fixture();+ Object.assign(project.scenes[3], { speedFps: -12, offsetFrames: 12 });+ await expect(encode(project, plan, 'job-performance-timing')).rejects.toThrow('stop after staging');+ expect(browser.seen.data.scenes[3].media).toMatchObject({ inSec: 2, outSec: 7 });+ expect(browser.seen.data.scenes[0].media).toMatchObject({ inSec: 0, outSec: 4 });+ });++ it('fails before opening the page and cleans scratch when shot retiming fails', async () => {+ const { project, plan } = await fixture();+ project.scenes[0].speedFps = 24;+ runFfmpegProcess.mockResolvedValueOnce({ ok: false, reason: 'synthetic retiming failure' });+ await expect(encode(project, plan, 'job-timing-failure')).rejects.toThrow('Shot timing failed: synthetic retiming failure');+ expect(browser.seen).toBeNull();+ expect(await scratchEntries()).toEqual([]);+ });+it('preserves the document shutter sampling contract through the shared encoder', async () => {const { project, plan } = await fixture();browser.contract = { durationSec: 30, fps: 24, width: 1920, height: 1080, motionBlur: 4, layout: false };@@ -148,6 +192,7 @@ describe('composition document staging', () => {expect(data.textCues.map((c) => c.emphasis)).toEqual(['hero']);expect(data.composition).toMatchObject({ mode: 'document', overlay: { enabled: true, ticker: ['a'] } });expect(media).toEqual({ 'scene-a.mp4': 'video-a', 'scene-b.png': 'image-b', 'scene-d.mp4': 'video-a' });+ expect(runFfmpegProcess).not.toHaveBeenCalled();expect(song).toMatchObject({ beats: [0, 0.5], downbeats: [0] });// The source document is untouched and the job folder is gone.expect(existsSync(join(PATHS.data, plan.directory, 'portos-mv.js'))).toBe(false);