Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Autopilot keeps a reused song's media choices and re-plans unfilmable performance shots (#11087)
[._.]/bb2fdd3f5b05
diff --git a/server/lib/musicVideoMediumPlan.js b/server/lib/musicVideoMediumPlan.jsindex df0bc81c51..3b7890d878 100644--- a/server/lib/musicVideoMediumPlan.js+++ b/server/lib/musicVideoMediumPlan.js@@ -154,7 +154,8 @@ export function codeFirstProductionAssets(project) {steps.push({ sceneId: scene.sceneId, medium, action: 'generate-video' });}}- return { steps, conflicts,+ // Plan-level messages repeat once per shot; every caller joins them into one sentence.+ return { steps, conflicts: [...new Set(conflicts)],requiredRoutes: {image: steps.some((step) => step.action === 'generate-image'),video: steps.some((step) => step.action === 'generate-video'),diff --git a/server/lib/musicVideoMediumPlan.test.js b/server/lib/musicVideoMediumPlan.test.jsindex e626284bcc..101b5dfaed 100644--- a/server/lib/musicVideoMediumPlan.test.js+++ b/server/lib/musicVideoMediumPlan.test.js@@ -46,3 +46,13 @@ describe('footage tools on a code-first project (#10450)', () => {expect(codeFirstProductionAssets(p).conflicts).toEqual([]);});});++describe('codeFirstProductionAssets conflicts', () => {+ it('names a plan-level conflict once however many shots raise it', () => {+ const scenes = [0, 1, 2].map((i) => ({ sceneId: `s${i}`, startSec: i * 5, endSec: i * 5 + 5, shotMode: 'cutaway' }));+ const shotDirections = scenes.map((s) => ({ sceneId: s.sceneId, mode: 'performance', medium: 'procedural', mediumRationale: 'x' }));+ const p = { productionPolicy: { strategy: 'code-first', maxGeneratedVideoPercent: 0 }, audioAnalysis: { durationSec: 15 }, scenes,+ treatment: { revision: 1, appliedRevision: 1, shotDirections } };+ expect(codeFirstProductionAssets(p).conflicts).toEqual([expect.stringMatching(/^Visible performance is unresolved/)]);+ });+});diff --git a/server/services/musicVideo/autonomousService.js b/server/services/musicVideo/autonomousService.jsindex 2114c428db..62a74f58fd 100644--- a/server/services/musicVideo/autonomousService.js+++ b/server/services/musicVideo/autonomousService.js@@ -2,6 +2,8 @@ import { musicVideoMediaMode, musicVideoAllowsMedia } from '../../lib/musicVideoimport { prepareProductionReview } from './productionReviewService.js';import { ALIGNMENT_UNVERIFIED_PROBLEM, assertProductionApproval, productionAlignmentBasis, productionReadiness, productionReviewBasis } from './productionReview.js';import { CAST_SETS_WORKING } from './castAndSets.js';+import { applyTreatmentPatch, applyTreatmentToProject } from './treatment.js';+import { codeFirstProductionAssets, normalizeMusicVideoProductionPolicy } from '../../lib/musicVideoMediumPlan.js';/*** Fully-autonomous Music Video — the run orchestrator.@@ -242,6 +244,11 @@ async function autoApproveStage(project, run, stage) {if (!autoApproves(run, stage)) return project;const readiness = productionReadiness(project);if (readiness[stage].approved || readiness[stage].problems.length) return project;+ const blocker = stageBlockers(project, run, stage)[0];+ if (blocker) {+ console.warn(`⚠️ Autonomous music video ${short(run.id)} left the ${stage} unapproved: ${trimTo(blocker, 200)}`);+ return project;+ }await deps.approveProductionReview(project.id, { stage, basis: readiness.basis[stage], approvedBy: 'autopilot',reviewer: { kind: 'autopilot', runId: run.id, authorizedBy: run.brief.autoApproveAuthorizedBy || null } });console.log(`🤖 Autonomous music video ${short(run.id)} auto-approved ${stage} (brief.autoApprove)`);@@ -641,8 +648,9 @@ async function orchestrateProductionStage(project, run, stage) {project = await getProject(project.id);const readiness = productionReadiness(project);const why = review.verdict !== 'revise' ? '' : spent >= reviewLimit(run) ? 'Revision limit reached; accepted the latest version. ' : 'Nothing to revise was supplied; accepted as is. ';- if (readiness[stage].problems.length) {- await recordReview(project.id, reviewEntry(stage, review, { verdict: 'revise', notes: `Could not approve: ${trimTo(readiness[stage].problems[0], 300)} ${review.notes || ''}`.trim() }));+ const problem = readiness[stage].problems[0] || stageBlockers(project, run, stage)[0];+ if (problem) {+ await recordReview(project.id, reviewEntry(stage, review, { verdict: 'revise', notes: `Could not approve: ${trimTo(problem, 300)} ${review.notes || ''}`.trim() }));return project;}await deps.approveProductionReview(project.id, { stage, basis: readiness.basis[stage], approvedBy: 'orchestrator', reviewer: orchestratorIdentity(run, review.route) });@@ -651,10 +659,52 @@ async function orchestrateProductionStage(project, run, stage) {}}+/** The run renders its film from code alone rather than through production. */+const rendersFromCode = (project, run) => musicVideoMediaMode(project) === 'code-only' || autonomousMedium(run.brief.tools) === 'code';++/**+ * What would stop production of an approved storyboard: the code-first plan's+ * medium conflicts. Approving over them only moves the failure to+ * production, so a run that approves for the director leaves the gate closed.+ */+const stageBlockers = (project, run, stage) => (stage === 'storyboard' && !rendersFromCode(project, run)+ ? codeFirstProductionAssets(project)?.conflicts || [] : []);++/**+ * A performance direction planned without footage (no video tool, or no+ * generated-video allowance left) blocks production until the plan is revised.+ * A run that approves the storyboard itself makes that revision as the+ * conflict asks: those shots become cutaways, and the plan is applied again.+ * Any other conflict is left for stageBlockers to name.+ */+async function reviseUnfilmablePerformances(project, run) {+ const ready = productionReadiness(project);+ if (ready.storyboard.approved || !ready.art.approved || !(isOrchestratedRun(run) || autoApproves(run, 'storyboard'))) return project;+ if (!stageBlockers(project, run, 'storyboard').length) return project;+ const sceneIds = (project.treatment?.shotDirections || [])+ .filter((d) => d.mode === 'performance' && !['existing-footage', 'generated-footage'].includes(d.medium)).map((d) => d.sceneId);+ if (!sceneIds.length) return project;+ const rationale = 'No footage is planned for this shot, so it is staged as a cutaway rather than a visible performance.';+ const revised = await mutateProjectRecord(project.id, (current) => {+ const patched = applyTreatmentPatch(current, { baseRevision: current.treatment?.revision ?? 0,+ shotDirections: sceneIds.map((sceneId) => ({ sceneId, mode: 'cutaway', rationale })) });+ return applyTreatmentToProject(patched, { revision: patched.treatment.revision });+ }).then(({ project: next }) => next, (err) => {+ console.warn(`⚠️ Autonomous music video ${short(run.id)} could not re-plan ${sceneIds.length} performance shot(s): ${err.message}`);+ return null;+ });+ if (!revised) return project;+ const note = `Re-planned ${sceneIds.length} performance shot${sceneIds.length === 1 ? '' : 's'} as cutaways: no footage is planned for them.`;+ if (isOrchestratedRun(run)) await recordReview(project.id, { checkpoint: 'storyboard', verdict: 'revise', notes: note });+ console.log(`🎬 Autonomous music video ${short(run.id)}: ${note}`);+ return revised;+}+/** Art and storyboard: the orchestrator when the run has one, else the operator's auto-approve grant. */-const settleProductionStage = (project, run, stage) => (isOrchestratedRun(run)- ? orchestrateProductionStage(project, run, stage)- : autoApproveStage(project, run, stage));+const settleProductionStage = async (project, run, stage) => {+ if (stage === 'storyboard') project = await reviseUnfilmablePerformances(project, run);+ return isOrchestratedRun(run) ? orchestrateProductionStage(project, run, stage) : autoApproveStage(project, run, stage);+};/*** Watch the finished film and send flagged sections back, as a director would:@@ -714,7 +764,7 @@ function issueWindow(project, issues) {*/async function startFinalRevision(projectId, run, issues) {const project = await getProject(projectId);- const code = musicVideoMediaMode(project) === 'code-only' || autonomousMedium(run.brief.tools) === 'code';+ const code = rendersFromCode(project, run);await patchRun(projectId, (r) => stagePatch(r, 'produce', { status: 'running', step: FINAL_REVISION_STEP }));if (code) return reauthorFinal(projectId, run, issues).then(() => 'rerender');const window = issueWindow(project, issues);@@ -1057,7 +1107,7 @@ const STAGES = {},async produce({ project, run }) {- const medium = musicVideoMediaMode(project) === 'code-only' ? 'code' : autonomousMedium(run.brief.tools);+ const medium = rendersFromCode(project, run) ? 'code' : autonomousMedium(run.brief.tools);if (medium === 'code' && !['code', 'document'].includes(project.composition?.mode)) {await deps.updateProject(project.id, { composition: { mode: 'document', authoringRenderer: 'three' } });}@@ -1094,6 +1144,9 @@ const STAGES = {const others = storyboard.problems.filter((p) => p !== ALIGNMENT_UNVERIFIED_PROBLEM);throw runError(409, 'MUSIC_VIDEO_APPROVAL_REQUIRED', trimTo([timingHeld, ...others].join(' '), 500));}+ // A medium conflict the run could not revise is named, so the director knows what to change.+ const blockers = storyboard.approved ? [] : stageBlockers(project, run, 'storyboard');+ if (blockers.length) throw runError(409, 'MUSIC_VIDEO_APPROVAL_REQUIRED', trimTo(blockers.join(' '), 500));assertProductionApproval(project, 'storyboard');// The authoring stage's pin, else the run's code-authoring pin taken as// given (production checks it exactly), else the direction LLM.@@ -1279,6 +1332,8 @@ export async function startAutonomousVideo(input, { autoApproveAuthorized = fals// No lyrics are written and no song is made, so those checkpoints have nothing to stop on.Object.assign(brief, {name: brief.name || trimTo(`${song.title} (Auto)`, 200),+ // The source's effective media mode carries over unless the request names its own.+ ...(input.mediaMode === undefined ? { mediaMode: musicVideoMediaMode(song.source) } : {}),instrumental: !song.lyrics,checkpoints: brief.checkpoints.filter((id) => id !== 'lyrics' && id !== 'song'),songFrom: { projectId: song.source.id, title: song.title, style: song.style, verifiedAlignmentBasis: null },@@ -1290,6 +1345,8 @@ export async function startAutonomousVideo(input, { autoApproveAuthorized = falsmode: 'autonomous',mediaMode: brief.mediaMode,...(brief.mediaMode !== 'code-images-video' ? { productionPolicy: { strategy: 'code-first', maxGeneratedVideoPercent: 0 } } : {}),+ // ...as does its production policy (a project with none is legacy), so a footage project stays one.+ ...(song && input.mediaMode === undefined ? { productionPolicy: normalizeMusicVideoProductionPolicy(song.source.productionPolicy) } : {}),...(song ? { trackId: song.source.trackId || null, uploadedAudioFilename: song.source.uploadedAudioFilename || null } : {}),// The style id makes createProject cast its character and attach the sheet (styleSnapshots.js).concept: { prompt: trimTo(brief.prompt, 8000), ...(brief.characterStyleId ? { characterStyleId: brief.characterStyleId } : {}),diff --git a/server/services/musicVideo/autonomousService.test.js b/server/services/musicVideo/autonomousService.test.jsindex bee0e99b24..9467e69b84 100644--- a/server/services/musicVideo/autonomousService.test.js+++ b/server/services/musicVideo/autonomousService.test.js@@ -639,6 +639,8 @@ describe('starting on an existing project\'s song (fromProjectId)', () => {expect(project).toMatchObject({ name: 'Example Song (Auto)', trackId: 'track-src', status: 'analyzed',audioAnalysis: SONG_SOURCE.audioAnalysis, lyricCues: SONG_SOURCE.lyricCues, vocalStemFilename: 'vocals.wav', lyricAlignSource: 'vocal-stem' });expect(project.concept).toEqual({ prompt: 'skatepark at golden hour', songStyle: 'post-punk, driving drums, -metal' });+ // The source has no policy of its own, so it is a legacy footage project, and so is the copy.+ expect(project).toMatchObject({ mediaMode: 'code-images-video', productionPolicy: { strategy: 'legacy', maxGeneratedVideoPercent: 100 } });for (const key of ['visualSpec', 'castAndSets', 'treatment', 'filmLook']) expect(project[key]).toBeUndefined();expect(project.scenes).toBeUndefined();expect(run.brief).toMatchObject({ checkpoints: ['style'], instrumental: false, songFrom: { projectId: 'mv-src', title: 'Example Song' } });@@ -1155,6 +1157,33 @@ describe('auto-approve the rest (brief.autoApprove)', () => {expect(doubles.startProduction).not.toHaveBeenCalled();});+ // Uniquely catches a code-first plan with an unfilmable performance shot being approved and then failing production.+ it('re-plans a performance shot with no footage as a cutaway before approving the storyboard', async () => {+ reviewFixture();+ const { buildTreatmentDraft } = await import('./treatmentDraft.js');+ const { writeCompiledTreatment, applyTreatmentToProject, treatmentBasis } = await import('./treatment.js');+ const { codeFirstProductionAssets } = await import('../../lib/musicVideoMediumPlan.js');+ doubles.approveProductionReview.mockImplementation(async (id, input) => {+ let project = clone(store.get(id));+ if (input.stage === 'art' && !project.treatment) {+ // What preparing the storyboard does for a code-first project: a deterministic plan, applied.+ const compiled = writeCompiledTreatment(project, { baseRevision: 0, basis: treatmentBasis(project), draft: buildTreatmentDraft(project),+ compiledWith: { source: 'deterministic', providerId: null, model: null } });+ compiled.treatment.shotDirections = compiled.treatment.shotDirections.map((d) => ({ ...d, mode: 'performance', medium: 'procedural' }));+ project = applyTreatmentToProject(compiled, { revision: compiled.treatment.revision }).project;+ }+ store.set(id, clone(actual.approveProductionStage(project, input)));+ });+ await service.startAutonomousVideo({ prompt: 'p', tools: ['image:local'], mediaMode: 'code-images', autoApprove: ['art', 'storyboard'] }, { autoApproveAuthorized: true });+ await vi.waitFor(() => expect(doubles.approveProductionReview).toHaveBeenCalledTimes(2));+ const project = store.get('mv-auto');+ expect(project.productionPolicy).toMatchObject({ strategy: 'code-first', maxGeneratedVideoPercent: 0 });+ expect(project.treatment.shotDirections.map((d) => d.mode)).toEqual(['cutaway']);+ expect(project.treatment.appliedRevision).toBe(project.treatment.revision);+ expect(codeFirstProductionAssets(project).conflicts).toEqual([]);+ expect(approvals().storyboard).toMatchObject({ approvedBy: 'autopilot' });+ });+it('ignores a legacy proof grant: code renders the final without rendering or approving a proof', async () => {reviewFixture();const { renderProductionProof } = await import('./productionReviewService.js');