Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Draft renders draw a stale lip-sync shot as a cutaway instead of refusing (#11147)

[._.]/7b552cc17480
diff --git a/server/services/musicVideo/excerptRender.js b/server/services/musicVideo/excerptRender.js
index 72e9c5e6f6..25a2805578 100644
--- a/server/services/musicVideo/excerptRender.js
+++ b/server/services/musicVideo/excerptRender.js
@@ -32,7 +32,7 @@ import { cancelMusicVideoRenderJob, isMusicVideoRenderCanceled } from './renderC
import { safeUnder } from '../../lib/ffmpeg.js';
import { encodeFileContactSheetAtTimes } from '../htmlComposition/encode.js';
import { getProject, listProjects, mutateProjectRecord } from './projects.js';
-import { assertCurrentPerformanceTakes } from './performanceShot.js';
+import { assertCurrentPerformanceTakes, withDraftPerformanceFallbacks } from './performanceShot.js';
import { assertCurrentClipDependencies, planMusicVideoRender, buildMusicVideoFfmpegArgs, excerptBoundaryTimes, isLocalRenderMark, resolveMasterAudioPath, resolveSoundBedPath } from './render.js';
import { encodeCodeComposition, prepareCodeRender, writeCodeProofSheet } from './codeRender.js';
import { encodeDocumentComposition, prepareDocumentRender } from './documentRender.js';
@@ -127,13 +127,14 @@ async function launchSeekedExcerpt({ projectId, project: stored, startSec, endSe
// A social cut renders the same composition at another frame (#9280): the
// re-framed view drives planning, staging and capture; the record keeps its own aspect.
const project = musicVideoAtAspect(stored, aspect);
- const prepared = await renderer.prepare(project);
- // Same order as a full render: a missing master throws before the
- // excerpt is marked rendering.
+ // A missing master throws before the excerpt is marked rendering.
const audioPath = await resolveMasterAudioPath(project);
+ // A draft draws a stale performance shot as a cutaway; the excerpt still records the real selection.
+ const drawn = renderer.performanceTakes ? await withDraftPerformanceFallbacks(project, audioPath) : project;
+ const prepared = await renderer.prepare(drawn);
if (renderer.performanceTakes) {
- assertCurrentClipDependencies(project);
- await assertCurrentPerformanceTakes(project, audioPath);
+ assertCurrentClipDependencies(drawn);
+ await assertCurrentPerformanceTakes(drawn, audioPath);
}
const soundBed = renderer.soundBed ? await resolveSoundBedPath(project) : null;
if (!excerptRangeFits(startSec, endSec, prepared.totalSec, prepared.plan?.fps ?? prepared.full?.fps)) {
@@ -180,7 +181,7 @@ async function launchSeekedExcerpt({ projectId, project: stored, startSec, endSe
return persisted;
};
Promise.resolve().then(() => renderer.encode({
- prepared, project, projectId, jobId, audioPath, soundBed, outputPath, signal, startSec, endSec: endClamped, fade,
+ prepared, project: drawn, projectId, jobId, audioPath, soundBed, outputPath, signal, startSec, endSec: endClamped, fade,
onProgress: (fraction) => broadcastSse(job, { type: 'progress', progress: fraction }),
})).then(async (encoded) => {
job.overlayAbort = null;
diff --git a/server/services/musicVideo/performanceShot.js b/server/services/musicVideo/performanceShot.js
index 47c022f99b..49a6918ddc 100644
--- a/server/services/musicVideo/performanceShot.js
+++ b/server/services/musicVideo/performanceShot.js
@@ -208,6 +208,23 @@ export async function findStalePerformanceTakes(project, masterPath) {
.map(({ sceneId, reason }) => ({ sceneId, reason }));
}
+/**
+ * A draft never waits on lip sync. Each performance (or supplied-audio) shot
+ * whose selected take is stale for the current song stands in as a cutaway:
+ * its newest cutaway clip (a video take with no performance or song-audio
+ * instruction), else its still. Final renders keep the strict gate below.
+ */
+export async function withDraftPerformanceFallbacks(project, masterPath) {
+ const stale = new Set((await findStalePerformanceTakes(project, masterPath)).map(({ sceneId }) => sceneId));
+ if (!stale.size) return project;
+ return { ...project, scenes: project.scenes.map((scene) => {
+ if (!stale.has(scene.sceneId)) return scene;
+ const cutaway = (scene.takes || []).filter((take) => take?.kind === 'video' && take.assetId
+ && take.shotInstruction?.shotMode !== 'performance' && !take.shotInstruction?.audioConditioning).at(-1);
+ return { ...scene, shotMode: 'cutaway', videoHistoryId: cutaway?.assetId || null };
+ }) };
+}
+
/**
* The one staleness gate every renderer that draws performance takes runs
* (concat, composed, document, and their excerpts): a performance take sings
diff --git a/server/services/musicVideo/performanceShot.test.js b/server/services/musicVideo/performanceShot.test.js
index 524958cba9..d8555a90a9 100644
--- a/server/services/musicVideo/performanceShot.test.js
+++ b/server/services/musicVideo/performanceShot.test.js
@@ -31,7 +31,7 @@ vi.mock('../videoGen/local.js', async () => {
const { PATHS } = await import('../../lib/fileUtils.js');
const { findFfmpeg, probeVideoStreamInfo } = await import('../../lib/ffmpeg.js');
-const { preparePerformanceShot, findStalePerformanceTakes } = await import('./performanceShot.js');
+const { preparePerformanceShot, findStalePerformanceTakes, withDraftPerformanceFallbacks } = await import('./performanceShot.js');
const fal = await import('../videoGen/fal.js');
const { videoGenEvents } = await import('../videoGen/events.js');
const { buildMusicVideoFfmpegArgs, beatSnapClips, resolveSceneClips } = await import('./render.js');
@@ -521,3 +521,22 @@ describe.skipIf(!ffmpeg)('accepted-prefix continuation render (#9348)', () => {
expect(prefix.videoHistoryId).toBe('accepted');
}, 60_000);
});
+
+// Uniquely catches a draft standing a stale lip-sync shot in with another lip-sync take, or dropping a cutaway clip it has.
+describe('draft fallbacks for stale performance shots', () => {
+ it('draws a stale performance shot as its newest cutaway clip, else its still', async () => {
+ const takes = [{ kind: 'video', assetId: 'cut-1' }, { kind: 'video', assetId: 'cut-2', shotInstruction: { shotMode: 'cutaway' } },
+ { kind: 'video', assetId: 'sync-1', shotInstruction: { shotMode: 'performance' } }];
+ const project = { scenes: [
+ { sceneId: 'a', shotMode: 'performance', startSec: 0, endSec: 2, videoHistoryId: 'sync-1', takes },
+ { sceneId: 'b', shotMode: 'performance', startSec: 2, endSec: 4, videoHistoryId: 'sync-2', referenceImageId: 'b.png',
+ takes: [{ kind: 'video', assetId: 'sync-2', shotInstruction: { shotMode: 'performance' } }] },
+ { sceneId: 'c', shotMode: 'cutaway', startSec: 4, endSec: 6, videoHistoryId: 'cut-3', takes: [] },
+ ] };
+ const drawn = await withDraftPerformanceFallbacks(project, '/no/master.wav');
+ expect(drawn.scenes.map(({ sceneId, shotMode, videoHistoryId }) => [sceneId, shotMode, videoHistoryId])).toEqual([
+ ['a', 'cutaway', 'cut-2'], ['b', 'cutaway', null], ['c', 'cutaway', 'cut-3'],
+ ]);
+ expect(project.scenes[0].videoHistoryId).toBe('sync-1');
+ });
+});
diff --git a/server/services/musicVideo/render.document.test.js b/server/services/musicVideo/render.document.test.js
index 3dd0140cda..01133cb66b 100644
--- a/server/services/musicVideo/render.document.test.js
+++ b/server/services/musicVideo/render.document.test.js
@@ -114,21 +114,25 @@ describe('composition-document render plan', () => {
}
});
- it('refuses a full render and an excerpt over a stale performance take, like a footage render (#9266)', async () => {
+ // Uniquely catches a draft blocked by a stale lip-sync take, or a final render drawing one.
+ it('refuses a full render over a stale performance take (#9266), while a draft draws it as a cutaway', async () => {
const id = await documentProject();
await importDocumentTemplate(id);
await projects.mutateProjectRecord(id, (current) => ({ project: {
...current,
- scenes: [{ sceneId: 'mvs-1', order: 0, shotMode: 'performance', startSec: 1, endSec: 3, videoHistoryId: 'clip-1', takes: [] }],
+ scenes: [{ sceneId: 'mvs-1', order: 0, shotMode: 'performance', startSec: 1, endSec: 3, videoHistoryId: 'clip-1', referenceImageId: 'still.png', takes: [] }],
} }));
encodeDocumentComposition.mockClear();
for (let attempt = 0; attempt < 2; attempt++) {
await expect(renderMusicVideo(id)).rejects.toMatchObject({
status: 422, code: 'STALE_PERFORMANCE_TAKES', context: { stale: [{ sceneId: 'mvs-1', reason: 'not-lip-synced' }] },
});
- await expect(startExcerptRender(id, { startSec: 1, endSec: 3 })).rejects.toMatchObject({ status: 422, code: 'STALE_PERFORMANCE_TAKES' });
}
expect(encodeDocumentComposition).not.toHaveBeenCalled();
+ await startExcerptRender(id, { startSec: 1, endSec: 3 });
+ await vi.waitFor(() => expect(encodeDocumentComposition).toHaveBeenCalledOnce());
+ expect(encodeDocumentComposition.mock.calls[0][0].project.scenes[0]).toMatchObject({ shotMode: 'cutaway', videoHistoryId: null, referenceImageId: 'still.png' });
+ expect((await projects.getProject(id)).scenes[0]).toMatchObject({ shotMode: 'performance', videoHistoryId: 'clip-1' });
});
});
diff --git a/server/services/musicVideo/render.js b/server/services/musicVideo/render.js
index a7cd39e768..61b43f3a2f 100644
--- a/server/services/musicVideo/render.js
+++ b/server/services/musicVideo/render.js
@@ -48,7 +48,7 @@ import { encodeDocumentComposition, prepareDocumentRender, sweepDocumentScratch
import { renderTypographyOverlays, removeCompositionScratch, sweepCompositionScratch } from './compositionRender.js';
import { selectedPerformanceInstruction } from '../../lib/musicVideoShotTiming.js';
import { captureMusicVideoEvidence, musicVideoTakeChanges } from '../../lib/musicVideoDependencies.js';
-import { assertCurrentPerformanceTakes } from './performanceShot.js';
+import { assertCurrentPerformanceTakes, withDraftPerformanceFallbacks } from './performanceShot.js';
import { ensureInstanceId } from '../instanceIdentity.js';
import { AUDIO_NORM, buildAudioBedMix } from '../videoTimeline/audioBedMix.js';
import { musicVideoEvents } from './events.js';
@@ -613,7 +613,8 @@ export async function planMusicVideoRender(project, { draft = false } = {}) {
const audioPath = await resolveMasterAudioPath(project);
// #8977/#9266: refuse a performance take whose scene was re-timed or whose
- // sung window changed in the current master.
+ // sung window changed in the current master. A draft shows a cutaway instead.
+ if (draft) project = await withDraftPerformanceFallbacks(project, audioPath);
await assertCurrentPerformanceTakes(project, audioPath);
// #8985: a composed render cuts still and card sections into the same
// timebase as the footage; plain concat renders footage only, as before.