Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: keep canceled music video runs from restarting production (#12064)

Adam Eivyb5c63de4f7bc
diff --git a/server/services/musicVideo/autonomousService.js b/server/services/musicVideo/autonomousService.js
index 01864e27bd..523abaa2c8 100644
--- a/server/services/musicVideo/autonomousService.js
+++ b/server/services/musicVideo/autonomousService.js
@@ -177,6 +177,7 @@ const runError = (status, code, message, context) =>
new ServerError(message, { status, code, ...(context ? { context } : {}) });
const projectAutonomousRun = (project) => project?.autonomousRun || null;
+const ownsRunningRun = (current, run) => current?.id === run.id && current.status === 'running' && current.processId === PROCESS_ID;
async function requireRun(projectId) {
const project = await getProject(projectId);
@@ -1233,6 +1234,9 @@ const STAGES = {
return { output: { renderJobId: render?.jobId || null }, wait: true, step: RENDER_STEP };
}
const pool = briefProductionPool(run.brief, project);
+ if (!ownsRunningRun(projectAutonomousRun(await getProject(project.id)), run)) {
+ throw runError(409, 'RUN_STOPPED', 'The run stopped before production could start');
+ }
const started = await deps.startProduction(project.id, {
directive: briefDirective(run.brief),
pool,
@@ -1244,6 +1248,11 @@ const STAGES = {
: { providerId: run.brief.llm?.providerId || null, model: run.brief.llm?.model || null },
authoring: input,
});
+ // A stop during preparation had no child id to stop; cancel the fresh child before handing it off.
+ if (!ownsRunningRun(projectAutonomousRun(await getProject(project.id)), run)) {
+ await settleChildStop([], project.id, run, 'cancel production', started.run.id, () => deps.cancelProduction(project.id, started.run.id));
+ throw runError(409, 'RUN_STOPPED', 'The run stopped while production was preparing');
+ }
return { output: { productionRunId: started.run.id }, wait: true };
},
};
@@ -1398,17 +1407,33 @@ async function advance(projectId) {
return;
}
const finishedAt = new Date().toISOString();
+ // Preserve useful late output, but only the still-owned run may continue the pipeline.
+ let owned = false;
if (result.goto) {
// The orchestrator sent the run back (a song retake): both stages run again.
const pending = { status: 'pending', startedAt: null, finishedAt: null, error: null, step: null };
- await patchRun(projectId, (r) => ({ output: result.output, stage: result.goto,
- stages: { [stage]: { ...r.stages[stage], ...pending }, [result.goto]: { ...r.stages[result.goto], ...pending } } }));
+ await patchRun(projectId, (r) => {
+ owned = ownsRunningRun(r, run);
+ return { output: result.output, ...(owned ? { stage: result.goto } : {}),
+ stages: { [stage]: { ...r.stages[stage], ...pending }, [result.goto]: { ...r.stages[result.goto], ...pending } } };
+ });
+ if (!owned) return;
if (result.unlinkTrack) await unlinkRunTrack(projectId, run.id, result.unlinkTrack);
console.log(`🎬 Autonomous music video ${short(run.id)} going back to ${result.goto}`);
continue;
}
if (result.wait) {
- await patchRun(projectId, (r) => ({ output: result.output, ...stagePatch(r, stage, { status: 'running', step: result.step || null }) }));
+ await patchRun(projectId, (r) => {
+ owned = ownsRunningRun(r, run);
+ return { output: result.output, ...stagePatch(r, stage, { status: 'running', step: result.step || null }) };
+ });
+ if (!owned) {
+ // Cancellation can win the serialized write after the executor's final ownership read.
+ const { productionRunId, renderJobId } = result.output || {};
+ if (productionRunId) await settleChildStop([], projectId, run, 'cancel production', productionRunId, () => deps.cancelProduction(projectId, productionRunId));
+ if (renderJobId) await settleChildStop([], projectId, run, 'cancel render', renderJobId, () => deps.cancelRender(renderJobId));
+ return;
+ }
console.log(`🎬 Autonomous music video ${short(run.id)} handed ${stage} to ${result.step === RENDER_STEP ? 'the final render' : result.step === CAST_STEP ? 'Cast & Sets' : 'production'}`);
// The render may have settled before its job id was stored on the run.
if (result.step === RENDER_STEP) await reconcileFinalRender(projectId);
@@ -1422,11 +1447,15 @@ async function advance(projectId) {
}
const next = nextAutonomousStage(stage);
const checkpoint = run.brief.checkpoints.includes(stage);
- await patchRun(projectId, (r) => ({
- output: result.output,
- ...stagePatch(r, stage, { status: 'done', finishedAt, step: null }),
- ...(checkpoint ? { status: 'awaiting-approval', awaiting: stage, stage: next } : next ? { stage: next } : { status: 'completed', stage }),
- }));
+ await patchRun(projectId, (r) => {
+ owned = ownsRunningRun(r, run);
+ return {
+ output: result.output,
+ ...stagePatch(r, stage, { status: 'done', finishedAt, step: null }),
+ ...(owned && (checkpoint ? { status: 'awaiting-approval', awaiting: stage, stage: next } : next ? { stage: next } : { status: 'completed', stage })),
+ };
+ });
+ if (!owned) return;
console.log(`🎬 Autonomous music video ${short(run.id)} finished ${stage}${checkpoint ? ' — waiting for approval' : ''}`);
if (checkpoint || !next) return;
}
@@ -1451,12 +1480,11 @@ function advanceInBackground(projectId) {
* lyrics. A failed unlink is not fatal: that same link replaces the old one.
*/
async function unlinkRunTrack(projectId, runId, trackId) {
- const project = await getProject(projectId);
- if (project?.trackId === trackId) {
- await deps.updateProject(projectId, { trackId: null })
- .catch((err) => console.warn(`⚠️ Autonomous music video ${short(runId)} could not unlink the retaken track: ${err.message}`));
- }
- console.log(`🎬 Autonomous music video ${short(runId)} retaking its song`);
+ const { applyProjectPatch } = await import('./projectsLogic.js');
+ // The track and run ownership must be checked in the same write as the unlink.
+ await mutateProjectRecord(projectId, (project) => ({ project: project.trackId === trackId && ownsRunningRun(projectAutonomousRun(project), { id: runId })
+ ? applyProjectPatch(project, { trackId: null }) : project }))
+ .catch((err) => console.warn(`⚠️ Autonomous music video ${short(runId)} could not unlink the retaken track: ${err.message}`));
}
// ---- director actions ------------------------------------------------------------
@@ -1861,11 +1889,12 @@ export async function stopAutonomousVideo(projectId) {
const { run } = await requireRun(projectId);
if (!['running', 'awaiting-approval', 'needs-human'].includes(run.status)) throw runError(409, 'NOT_RUNNING', `A ${run.status} run cannot be stopped`);
const out = await patchRun(projectId, () => ({ status: 'stopped' }));
+ const { productionRunId, finalAutoReviewId } = out.run.output;
const childStopFailures = [];
- if (run.output.productionRunId) {
- await settleChildStop(childStopFailures, projectId, run, 'stop production', run.output.productionRunId, () => deps.stopProduction(projectId, run.output.productionRunId));
+ if (productionRunId) {
+ await settleChildStop(childStopFailures, projectId, run, 'stop production', productionRunId, () => deps.stopProduction(projectId, productionRunId));
}
- if (run.output.finalAutoReviewId) await settleChildStop(childStopFailures, projectId, run, 'stop auto-review', run.output.finalAutoReviewId, () => deps.stopAutoReview(projectId, run.output.finalAutoReviewId));
+ if (finalAutoReviewId) await settleChildStop(childStopFailures, projectId, run, 'stop auto-review', finalAutoReviewId, () => deps.stopAutoReview(projectId, finalAutoReviewId));
sunoControllers.get(run.id)?.abort();
await cancelLocalSongJob(projectId, out.run, childStopFailures);
return { project: out.project, run: presentAutonomousRun(out.run), childStopFailures };
diff --git a/server/services/musicVideo/autonomousService.test.js b/server/services/musicVideo/autonomousService.test.js
index 3ccc6d1157..f790c72570 100644
--- a/server/services/musicVideo/autonomousService.test.js
+++ b/server/services/musicVideo/autonomousService.test.js
@@ -803,6 +803,95 @@ describe('cancel while the final render is preparing (#11768)', () => {
});
});
+describe('stage completion after ownership is released (#12064)', () => {
+ // Uniquely catches a successful late stage restoring approval/continuation after cancel, stop, or a process handoff.
+ it.each(['cancel', 'stop', 'handoff'])('keeps a checkpoint released by %s terminal and preserves its finished output', async (action) => {
+ const lyrics = Promise.withResolvers();
+ doubles.writeLyrics.mockImplementationOnce(() => lyrics.promise);
+ await service.startAutonomousVideo({ prompt: 'p', checkpoints: ['lyrics'] });
+ await vi.waitFor(() => expect(doubles.writeLyrics).toHaveBeenCalledOnce());
+ if (action === 'handoff') runOf().processId = 'proc-other';
+ else await service[`${action}AutonomousVideo`]('mv-auto');
+ const released = clone(runOf());
+ lyrics.resolve({ lyrics: 'finished words' });
+ await service.__testing.settleBackground();
+ expect(runOf()).toMatchObject({ status: released.status, stage: 'lyrics', awaiting: released.awaiting, processId: released.processId,
+ output: { lyrics: 'finished words' }, stages: { lyrics: { status: 'done' } } });
+ expect(doubles.createMoodBoard).not.toHaveBeenCalled();
+ expect(doubles.startProduction).not.toHaveBeenCalled();
+ });
+
+ // Uniquely catches production being dispatched after a cancel during the preceding preparation/review.
+ it('does not start production when preparation returns after cancellation', async () => {
+ const { prepareProductionReview } = await import('./productionReviewService.js');
+ const prepare = Promise.withResolvers();
+ prepareProductionReview.mockImplementationOnce(() => prepare.promise);
+ await service.startAutonomousVideo({ prompt: 'p' });
+ await vi.waitFor(() => expect(runOf()?.stage).toBe('produce'));
+ await service.cancelAutonomousVideo('mv-auto');
+ prepare.resolve();
+ await service.__testing.settleBackground();
+ expect(doubles.startProduction).not.toHaveBeenCalled();
+ expect(runOf()).toMatchObject({ status: 'canceled', stage: 'produce', errorCode: null });
+ });
+
+ // Uniquely catches a child created during cancellation running without an owned autonomous parent.
+ it.each(['cancel', 'stop', 'handoff'])('cancels a production returned after %s while preserving the released run', async (action) => {
+ const production = Promise.withResolvers();
+ doubles.startProduction.mockImplementationOnce(() => production.promise);
+ await service.startAutonomousVideo({ prompt: 'p' });
+ await vi.waitFor(() => expect(doubles.startProduction).toHaveBeenCalledOnce());
+ if (action === 'handoff') runOf().processId = 'proc-other';
+ else await service[`${action}AutonomousVideo`]('mv-auto');
+ const released = clone(runOf());
+ production.resolve({ run: { id: 'mvpr-late' } });
+ await service.__testing.settleBackground();
+ expect(doubles.cancelProduction).toHaveBeenCalledWith('mv-auto', 'mvpr-late');
+ expect(runOf()).toMatchObject({ status: released.status, stage: 'produce', processId: released.processId, errorCode: null });
+ expect(runOf().output.productionRunId).toBeUndefined();
+ });
+
+ // Uniquely catches cancellation winning the handoff write after the executor's last ownership read.
+ it('cancels production when cancellation reaches the serialized handoff first', async () => {
+ const projects = await import('./projects.js');
+ const getProject = projects.getProject;
+ let read;
+ doubles.startProduction.mockImplementationOnce(async () => {
+ read = vi.spyOn(projects, 'getProject').mockImplementationOnce(async (id) => {
+ const snapshot = await getProject(id);
+ writeTail = writeTail.then(() => { runOf().status = 'canceled'; });
+ return snapshot;
+ });
+ return { run: { id: 'mvpr-handoff' } };
+ });
+ try {
+ await service.startAutonomousVideo({ prompt: 'p' });
+ await service.__testing.settleBackground();
+ expect(doubles.cancelProduction).toHaveBeenCalledWith('mv-auto', 'mvpr-handoff');
+ expect(runOf()).toMatchObject({ status: 'canceled', stage: 'produce', output: { productionRunId: 'mvpr-handoff' } });
+ } finally {
+ read?.mockRestore();
+ }
+ });
+
+ // Uniquely catches stop reading a child id before an already-admitted write stores it.
+ it('stops children stored between the initial read and the serialized status change', async () => {
+ const lyrics = Promise.withResolvers();
+ doubles.writeLyrics.mockImplementationOnce(() => lyrics.promise);
+ doubles.stopAutoReview = vi.fn(async () => ({}));
+ service.__setAutonomousDepsForTests(doubles);
+ await service.startAutonomousVideo({ prompt: 'p' });
+ await vi.waitFor(() => expect(doubles.writeLyrics).toHaveBeenCalledOnce());
+ writeTail = writeTail.then(() => { Object.assign(runOf().output, { productionRunId: 'mvpr-just-stored', finalAutoReviewId: 'mvar-just-stored' }); });
+ await service.stopAutonomousVideo('mv-auto');
+ lyrics.resolve({ lyrics: 'finished words' });
+ await service.__testing.settleBackground();
+ expect(doubles.stopProduction).toHaveBeenCalledWith('mv-auto', 'mvpr-just-stored');
+ expect(doubles.stopAutoReview).toHaveBeenCalledWith('mv-auto', 'mvar-just-stored');
+ expect(runOf().status).toBe('stopped');
+ });
+});
+
describe('overlay text pass before the final render', () => {
const finding = (atSec, message, severity = 'error') => ({ id: `f${atSec}`, kind: severity === 'error' ? 'overlap' : 'contrast', severity, atSec, texts: ['A'], message, count: 1 });
const textEntries = () => (runOf()?.orchestration?.reviews || []).filter((r) => r.checkpoint === 'text');
@@ -898,7 +987,8 @@ describe('retaking the song at the song checkpoint', () => {
doubles.createTrack.mockResolvedValueOnce({ id: 'track-2' });
doubles.generateSunoSong.mockImplementationOnce(async (_fields, opts) => {
- // A fresh request: the rejected take's ids are not offered for reuse.
+ // A fresh request: the rejected take's ids and project link are no longer used.
+ expect(store.get('mv-auto').trackId).toBeNull();
expect(opts.songIds).toBeNull();
await opts.onSubmitted(['song-c', 'song-d']);
return { songId: 'song-d', songIds: ['song-c', 'song-d'], filename: 'music-song-d.m4a' };
@@ -918,8 +1008,7 @@ describe('retaking the song at the song checkpoint', () => {
expect(doubles.generateSunoSong).toHaveBeenLastCalledWith(expect.objectContaining({
lyrics: '[verse]\nnew words', style: 'darker synthwave', excludeStyles: 'metal', vocalGender: 'male', model: 'v6',
}), expect.any(Object));
- // The project was unlinked from the rejected track, then linked to the new one; analysis has not run.
- expect(doubles.updateProject).toHaveBeenCalledWith('mv-auto', { trackId: null });
+ // The project was linked to the new take; analysis has not run.
expect(store.get('mv-auto').trackId).toBe('track-2');
expect(calls).not.toContain('analyze');
@@ -1599,6 +1688,22 @@ describe('orchestrated mode (brief.orchestrator)', () => {
expect(doubles.orchestrate).toHaveBeenCalledWith(expect.objectContaining({ orchestrator: ORCHESTRATOR }));
});
+ // Uniquely catches a late song-retake verdict unlinking a canceled project's track or restarting song generation.
+ it('keeps the song linked when a retake review returns after cancellation', async () => {
+ const review = Promise.withResolvers();
+ const orchestrate = doubles.orchestrate.getMockImplementation();
+ doubles.orchestrate.mockImplementation((args) => checkpointOf(args.prompt) === 'song' ? review.promise : orchestrate(args));
+ await service.startAutonomousVideo({ prompt: 'p', songSource: 'local', orchestrator: ORCHESTRATOR }, { autoApproveAuthorized: true });
+ await vi.waitFor(() => expect(doubles.orchestrate.mock.calls.some(([args]) => checkpointOf(args.prompt) === 'song')).toBe(true));
+ await service.cancelAutonomousVideo('mv-auto');
+ review.resolve({ text: JSON.stringify({ verdict: 'retake', score: 3, notes: 'Try another take.' }), route: ORCHESTRATOR, visual: false });
+ await service.__testing.settleBackground();
+ expect(runOf()).toMatchObject({ status: 'canceled', stage: 'analyze' });
+ expect(store.get('mv-auto').trackId).toBe('track-1');
+ expect(doubles.generateLocalSong).toHaveBeenCalledOnce();
+ expect(doubles.startProduction).not.toHaveBeenCalled();
+ });
+
// Uniquely catches a run with no video tool spending on clips, the orchestrator approving stills it never saw,
// or a flagged still not going back to production with the fix.
it('stops production after the stills, regenerates the ones the orchestrator flags, then waits for the director', async () => {