Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Autopilot keeps a reused song's media choices and re-plans unfilmable performance shots (#11087)

[._.]/bb2fdd3f5b05
diff --git a/server/lib/musicVideoMediumPlan.js b/server/lib/musicVideoMediumPlan.js
index df0bc81c51..3b7890d878 100644
--- a/server/lib/musicVideoMediumPlan.js
+++ b/server/lib/musicVideoMediumPlan.js
@@ -154,7 +154,8 @@ export function codeFirstProductionAssets(project) {
steps.push({ sceneId: scene.sceneId, medium, action: 'generate-video' });
}
}
- return { steps, conflicts,
+ // Plan-level messages repeat once per shot; every caller joins them into one sentence.
+ return { steps, conflicts: [...new Set(conflicts)],
requiredRoutes: {
image: steps.some((step) => step.action === 'generate-image'),
video: steps.some((step) => step.action === 'generate-video'),
diff --git a/server/lib/musicVideoMediumPlan.test.js b/server/lib/musicVideoMediumPlan.test.js
index e626284bcc..101b5dfaed 100644
--- a/server/lib/musicVideoMediumPlan.test.js
+++ b/server/lib/musicVideoMediumPlan.test.js
@@ -46,3 +46,13 @@ describe('footage tools on a code-first project (#10450)', () => {
expect(codeFirstProductionAssets(p).conflicts).toEqual([]);
});
});
+
+describe('codeFirstProductionAssets conflicts', () => {
+ it('names a plan-level conflict once however many shots raise it', () => {
+ const scenes = [0, 1, 2].map((i) => ({ sceneId: `s${i}`, startSec: i * 5, endSec: i * 5 + 5, shotMode: 'cutaway' }));
+ const shotDirections = scenes.map((s) => ({ sceneId: s.sceneId, mode: 'performance', medium: 'procedural', mediumRationale: 'x' }));
+ const p = { productionPolicy: { strategy: 'code-first', maxGeneratedVideoPercent: 0 }, audioAnalysis: { durationSec: 15 }, scenes,
+ treatment: { revision: 1, appliedRevision: 1, shotDirections } };
+ expect(codeFirstProductionAssets(p).conflicts).toEqual([expect.stringMatching(/^Visible performance is unresolved/)]);
+ });
+});
diff --git a/server/services/musicVideo/autonomousService.js b/server/services/musicVideo/autonomousService.js
index 2114c428db..62a74f58fd 100644
--- a/server/services/musicVideo/autonomousService.js
+++ b/server/services/musicVideo/autonomousService.js
@@ -2,6 +2,8 @@ import { musicVideoMediaMode, musicVideoAllowsMedia } from '../../lib/musicVideo
import { prepareProductionReview } from './productionReviewService.js';
import { ALIGNMENT_UNVERIFIED_PROBLEM, assertProductionApproval, productionAlignmentBasis, productionReadiness, productionReviewBasis } from './productionReview.js';
import { CAST_SETS_WORKING } from './castAndSets.js';
+import { applyTreatmentPatch, applyTreatmentToProject } from './treatment.js';
+import { codeFirstProductionAssets, normalizeMusicVideoProductionPolicy } from '../../lib/musicVideoMediumPlan.js';
/**
* Fully-autonomous Music Video — the run orchestrator.
@@ -242,6 +244,11 @@ async function autoApproveStage(project, run, stage) {
if (!autoApproves(run, stage)) return project;
const readiness = productionReadiness(project);
if (readiness[stage].approved || readiness[stage].problems.length) return project;
+ const blocker = stageBlockers(project, run, stage)[0];
+ if (blocker) {
+ console.warn(`⚠️ Autonomous music video ${short(run.id)} left the ${stage} unapproved: ${trimTo(blocker, 200)}`);
+ return project;
+ }
await deps.approveProductionReview(project.id, { stage, basis: readiness.basis[stage], approvedBy: 'autopilot',
reviewer: { kind: 'autopilot', runId: run.id, authorizedBy: run.brief.autoApproveAuthorizedBy || null } });
console.log(`🤖 Autonomous music video ${short(run.id)} auto-approved ${stage} (brief.autoApprove)`);
@@ -641,8 +648,9 @@ async function orchestrateProductionStage(project, run, stage) {
project = await getProject(project.id);
const readiness = productionReadiness(project);
const why = review.verdict !== 'revise' ? '' : spent >= reviewLimit(run) ? 'Revision limit reached; accepted the latest version. ' : 'Nothing to revise was supplied; accepted as is. ';
- if (readiness[stage].problems.length) {
- await recordReview(project.id, reviewEntry(stage, review, { verdict: 'revise', notes: `Could not approve: ${trimTo(readiness[stage].problems[0], 300)} ${review.notes || ''}`.trim() }));
+ const problem = readiness[stage].problems[0] || stageBlockers(project, run, stage)[0];
+ if (problem) {
+ await recordReview(project.id, reviewEntry(stage, review, { verdict: 'revise', notes: `Could not approve: ${trimTo(problem, 300)} ${review.notes || ''}`.trim() }));
return project;
}
await deps.approveProductionReview(project.id, { stage, basis: readiness.basis[stage], approvedBy: 'orchestrator', reviewer: orchestratorIdentity(run, review.route) });
@@ -651,10 +659,52 @@ async function orchestrateProductionStage(project, run, stage) {
}
}
+/** The run renders its film from code alone rather than through production. */
+const rendersFromCode = (project, run) => musicVideoMediaMode(project) === 'code-only' || autonomousMedium(run.brief.tools) === 'code';
+
+/**
+ * What would stop production of an approved storyboard: the code-first plan's
+ * medium conflicts. Approving over them only moves the failure to
+ * production, so a run that approves for the director leaves the gate closed.
+ */
+const stageBlockers = (project, run, stage) => (stage === 'storyboard' && !rendersFromCode(project, run)
+ ? codeFirstProductionAssets(project)?.conflicts || [] : []);
+
+/**
+ * A performance direction planned without footage (no video tool, or no
+ * generated-video allowance left) blocks production until the plan is revised.
+ * A run that approves the storyboard itself makes that revision as the
+ * conflict asks: those shots become cutaways, and the plan is applied again.
+ * Any other conflict is left for stageBlockers to name.
+ */
+async function reviseUnfilmablePerformances(project, run) {
+ const ready = productionReadiness(project);
+ if (ready.storyboard.approved || !ready.art.approved || !(isOrchestratedRun(run) || autoApproves(run, 'storyboard'))) return project;
+ if (!stageBlockers(project, run, 'storyboard').length) return project;
+ const sceneIds = (project.treatment?.shotDirections || [])
+ .filter((d) => d.mode === 'performance' && !['existing-footage', 'generated-footage'].includes(d.medium)).map((d) => d.sceneId);
+ if (!sceneIds.length) return project;
+ const rationale = 'No footage is planned for this shot, so it is staged as a cutaway rather than a visible performance.';
+ const revised = await mutateProjectRecord(project.id, (current) => {
+ const patched = applyTreatmentPatch(current, { baseRevision: current.treatment?.revision ?? 0,
+ shotDirections: sceneIds.map((sceneId) => ({ sceneId, mode: 'cutaway', rationale })) });
+ return applyTreatmentToProject(patched, { revision: patched.treatment.revision });
+ }).then(({ project: next }) => next, (err) => {
+ console.warn(`⚠️ Autonomous music video ${short(run.id)} could not re-plan ${sceneIds.length} performance shot(s): ${err.message}`);
+ return null;
+ });
+ if (!revised) return project;
+ const note = `Re-planned ${sceneIds.length} performance shot${sceneIds.length === 1 ? '' : 's'} as cutaways: no footage is planned for them.`;
+ if (isOrchestratedRun(run)) await recordReview(project.id, { checkpoint: 'storyboard', verdict: 'revise', notes: note });
+ console.log(`🎬 Autonomous music video ${short(run.id)}: ${note}`);
+ return revised;
+}
+
/** Art and storyboard: the orchestrator when the run has one, else the operator's auto-approve grant. */
-const settleProductionStage = (project, run, stage) => (isOrchestratedRun(run)
- ? orchestrateProductionStage(project, run, stage)
- : autoApproveStage(project, run, stage));
+const settleProductionStage = async (project, run, stage) => {
+ if (stage === 'storyboard') project = await reviseUnfilmablePerformances(project, run);
+ return isOrchestratedRun(run) ? orchestrateProductionStage(project, run, stage) : autoApproveStage(project, run, stage);
+};
/**
* Watch the finished film and send flagged sections back, as a director would:
@@ -714,7 +764,7 @@ function issueWindow(project, issues) {
*/
async function startFinalRevision(projectId, run, issues) {
const project = await getProject(projectId);
- const code = musicVideoMediaMode(project) === 'code-only' || autonomousMedium(run.brief.tools) === 'code';
+ const code = rendersFromCode(project, run);
await patchRun(projectId, (r) => stagePatch(r, 'produce', { status: 'running', step: FINAL_REVISION_STEP }));
if (code) return reauthorFinal(projectId, run, issues).then(() => 'rerender');
const window = issueWindow(project, issues);
@@ -1057,7 +1107,7 @@ const STAGES = {
},
async produce({ project, run }) {
- const medium = musicVideoMediaMode(project) === 'code-only' ? 'code' : autonomousMedium(run.brief.tools);
+ const medium = rendersFromCode(project, run) ? 'code' : autonomousMedium(run.brief.tools);
if (medium === 'code' && !['code', 'document'].includes(project.composition?.mode)) {
await deps.updateProject(project.id, { composition: { mode: 'document', authoringRenderer: 'three' } });
}
@@ -1094,6 +1144,9 @@ const STAGES = {
const others = storyboard.problems.filter((p) => p !== ALIGNMENT_UNVERIFIED_PROBLEM);
throw runError(409, 'MUSIC_VIDEO_APPROVAL_REQUIRED', trimTo([timingHeld, ...others].join(' '), 500));
}
+ // A medium conflict the run could not revise is named, so the director knows what to change.
+ const blockers = storyboard.approved ? [] : stageBlockers(project, run, 'storyboard');
+ if (blockers.length) throw runError(409, 'MUSIC_VIDEO_APPROVAL_REQUIRED', trimTo(blockers.join(' '), 500));
assertProductionApproval(project, 'storyboard');
// The authoring stage's pin, else the run's code-authoring pin taken as
// given (production checks it exactly), else the direction LLM.
@@ -1279,6 +1332,8 @@ export async function startAutonomousVideo(input, { autoApproveAuthorized = fals
// No lyrics are written and no song is made, so those checkpoints have nothing to stop on.
Object.assign(brief, {
name: brief.name || trimTo(`${song.title} (Auto)`, 200),
+ // The source's effective media mode carries over unless the request names its own.
+ ...(input.mediaMode === undefined ? { mediaMode: musicVideoMediaMode(song.source) } : {}),
instrumental: !song.lyrics,
checkpoints: brief.checkpoints.filter((id) => id !== 'lyrics' && id !== 'song'),
songFrom: { projectId: song.source.id, title: song.title, style: song.style, verifiedAlignmentBasis: null },
@@ -1290,6 +1345,8 @@ export async function startAutonomousVideo(input, { autoApproveAuthorized = fals
mode: 'autonomous',
mediaMode: brief.mediaMode,
...(brief.mediaMode !== 'code-images-video' ? { productionPolicy: { strategy: 'code-first', maxGeneratedVideoPercent: 0 } } : {}),
+ // ...as does its production policy (a project with none is legacy), so a footage project stays one.
+ ...(song && input.mediaMode === undefined ? { productionPolicy: normalizeMusicVideoProductionPolicy(song.source.productionPolicy) } : {}),
...(song ? { trackId: song.source.trackId || null, uploadedAudioFilename: song.source.uploadedAudioFilename || null } : {}),
// The style id makes createProject cast its character and attach the sheet (styleSnapshots.js).
concept: { prompt: trimTo(brief.prompt, 8000), ...(brief.characterStyleId ? { characterStyleId: brief.characterStyleId } : {}),
diff --git a/server/services/musicVideo/autonomousService.test.js b/server/services/musicVideo/autonomousService.test.js
index bee0e99b24..9467e69b84 100644
--- a/server/services/musicVideo/autonomousService.test.js
+++ b/server/services/musicVideo/autonomousService.test.js
@@ -639,6 +639,8 @@ describe('starting on an existing project\'s song (fromProjectId)', () => {
expect(project).toMatchObject({ name: 'Example Song (Auto)', trackId: 'track-src', status: 'analyzed',
audioAnalysis: SONG_SOURCE.audioAnalysis, lyricCues: SONG_SOURCE.lyricCues, vocalStemFilename: 'vocals.wav', lyricAlignSource: 'vocal-stem' });
expect(project.concept).toEqual({ prompt: 'skatepark at golden hour', songStyle: 'post-punk, driving drums, -metal' });
+ // The source has no policy of its own, so it is a legacy footage project, and so is the copy.
+ expect(project).toMatchObject({ mediaMode: 'code-images-video', productionPolicy: { strategy: 'legacy', maxGeneratedVideoPercent: 100 } });
for (const key of ['visualSpec', 'castAndSets', 'treatment', 'filmLook']) expect(project[key]).toBeUndefined();
expect(project.scenes).toBeUndefined();
expect(run.brief).toMatchObject({ checkpoints: ['style'], instrumental: false, songFrom: { projectId: 'mv-src', title: 'Example Song' } });
@@ -1155,6 +1157,33 @@ describe('auto-approve the rest (brief.autoApprove)', () => {
expect(doubles.startProduction).not.toHaveBeenCalled();
});
+ // Uniquely catches a code-first plan with an unfilmable performance shot being approved and then failing production.
+ it('re-plans a performance shot with no footage as a cutaway before approving the storyboard', async () => {
+ reviewFixture();
+ const { buildTreatmentDraft } = await import('./treatmentDraft.js');
+ const { writeCompiledTreatment, applyTreatmentToProject, treatmentBasis } = await import('./treatment.js');
+ const { codeFirstProductionAssets } = await import('../../lib/musicVideoMediumPlan.js');
+ doubles.approveProductionReview.mockImplementation(async (id, input) => {
+ let project = clone(store.get(id));
+ if (input.stage === 'art' && !project.treatment) {
+ // What preparing the storyboard does for a code-first project: a deterministic plan, applied.
+ const compiled = writeCompiledTreatment(project, { baseRevision: 0, basis: treatmentBasis(project), draft: buildTreatmentDraft(project),
+ compiledWith: { source: 'deterministic', providerId: null, model: null } });
+ compiled.treatment.shotDirections = compiled.treatment.shotDirections.map((d) => ({ ...d, mode: 'performance', medium: 'procedural' }));
+ project = applyTreatmentToProject(compiled, { revision: compiled.treatment.revision }).project;
+ }
+ store.set(id, clone(actual.approveProductionStage(project, input)));
+ });
+ await service.startAutonomousVideo({ prompt: 'p', tools: ['image:local'], mediaMode: 'code-images', autoApprove: ['art', 'storyboard'] }, { autoApproveAuthorized: true });
+ await vi.waitFor(() => expect(doubles.approveProductionReview).toHaveBeenCalledTimes(2));
+ const project = store.get('mv-auto');
+ expect(project.productionPolicy).toMatchObject({ strategy: 'code-first', maxGeneratedVideoPercent: 0 });
+ expect(project.treatment.shotDirections.map((d) => d.mode)).toEqual(['cutaway']);
+ expect(project.treatment.appliedRevision).toBe(project.treatment.revision);
+ expect(codeFirstProductionAssets(project).conflicts).toEqual([]);
+ expect(approvals().storyboard).toMatchObject({ approvedBy: 'autopilot' });
+ });
+
it('ignores a legacy proof grant: code renders the final without rendering or approving a proof', async () => {
reviewFixture();
const { renderProductionProof } = await import('./productionReviewService.js');