Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: keep canceled music video runs from restarting production (#12064)
Adam Eivyb5c63de4f7bc
diff --git a/server/services/musicVideo/autonomousService.js b/server/services/musicVideo/autonomousService.jsindex 01864e27bd..523abaa2c8 100644--- a/server/services/musicVideo/autonomousService.js+++ b/server/services/musicVideo/autonomousService.js@@ -177,6 +177,7 @@ const runError = (status, code, message, context) =>new ServerError(message, { status, code, ...(context ? { context } : {}) });const projectAutonomousRun = (project) => project?.autonomousRun || null;+const ownsRunningRun = (current, run) => current?.id === run.id && current.status === 'running' && current.processId === PROCESS_ID;async function requireRun(projectId) {const project = await getProject(projectId);@@ -1233,6 +1234,9 @@ const STAGES = {return { output: { renderJobId: render?.jobId || null }, wait: true, step: RENDER_STEP };}const pool = briefProductionPool(run.brief, project);+ if (!ownsRunningRun(projectAutonomousRun(await getProject(project.id)), run)) {+ throw runError(409, 'RUN_STOPPED', 'The run stopped before production could start');+ }const started = await deps.startProduction(project.id, {directive: briefDirective(run.brief),pool,@@ -1244,6 +1248,11 @@ const STAGES = {: { providerId: run.brief.llm?.providerId || null, model: run.brief.llm?.model || null },authoring: input,});+ // A stop during preparation had no child id to stop; cancel the fresh child before handing it off.+ if (!ownsRunningRun(projectAutonomousRun(await getProject(project.id)), run)) {+ await settleChildStop([], project.id, run, 'cancel production', started.run.id, () => deps.cancelProduction(project.id, started.run.id));+ throw runError(409, 'RUN_STOPPED', 'The run stopped while production was preparing');+ }return { output: { productionRunId: started.run.id }, wait: true };},};@@ -1398,17 +1407,33 @@ async function advance(projectId) {return;}const finishedAt = new Date().toISOString();+ // Preserve useful late output, but only the still-owned run may continue the pipeline.+ let owned = false;if (result.goto) {// The orchestrator sent the run back (a song retake): both stages run again.const pending = { status: 'pending', startedAt: null, finishedAt: null, error: null, step: null };- await patchRun(projectId, (r) => ({ output: result.output, stage: result.goto,- stages: { [stage]: { ...r.stages[stage], ...pending }, [result.goto]: { ...r.stages[result.goto], ...pending } } }));+ await patchRun(projectId, (r) => {+ owned = ownsRunningRun(r, run);+ return { output: result.output, ...(owned ? { stage: result.goto } : {}),+ stages: { [stage]: { ...r.stages[stage], ...pending }, [result.goto]: { ...r.stages[result.goto], ...pending } } };+ });+ if (!owned) return;if (result.unlinkTrack) await unlinkRunTrack(projectId, run.id, result.unlinkTrack);console.log(`🎬 Autonomous music video ${short(run.id)} going back to ${result.goto}`);continue;}if (result.wait) {- await patchRun(projectId, (r) => ({ output: result.output, ...stagePatch(r, stage, { status: 'running', step: result.step || null }) }));+ await patchRun(projectId, (r) => {+ owned = ownsRunningRun(r, run);+ return { output: result.output, ...stagePatch(r, stage, { status: 'running', step: result.step || null }) };+ });+ if (!owned) {+ // Cancellation can win the serialized write after the executor's final ownership read.+ const { productionRunId, renderJobId } = result.output || {};+ if (productionRunId) await settleChildStop([], projectId, run, 'cancel production', productionRunId, () => deps.cancelProduction(projectId, productionRunId));+ if (renderJobId) await settleChildStop([], projectId, run, 'cancel render', renderJobId, () => deps.cancelRender(renderJobId));+ return;+ }console.log(`🎬 Autonomous music video ${short(run.id)} handed ${stage} to ${result.step === RENDER_STEP ? 'the final render' : result.step === CAST_STEP ? 'Cast & Sets' : 'production'}`);// The render may have settled before its job id was stored on the run.if (result.step === RENDER_STEP) await reconcileFinalRender(projectId);@@ -1422,11 +1447,15 @@ async function advance(projectId) {}const next = nextAutonomousStage(stage);const checkpoint = run.brief.checkpoints.includes(stage);- await patchRun(projectId, (r) => ({- output: result.output,- ...stagePatch(r, stage, { status: 'done', finishedAt, step: null }),- ...(checkpoint ? { status: 'awaiting-approval', awaiting: stage, stage: next } : next ? { stage: next } : { status: 'completed', stage }),- }));+ await patchRun(projectId, (r) => {+ owned = ownsRunningRun(r, run);+ return {+ output: result.output,+ ...stagePatch(r, stage, { status: 'done', finishedAt, step: null }),+ ...(owned && (checkpoint ? { status: 'awaiting-approval', awaiting: stage, stage: next } : next ? { stage: next } : { status: 'completed', stage })),+ };+ });+ if (!owned) return;console.log(`🎬 Autonomous music video ${short(run.id)} finished ${stage}${checkpoint ? ' — waiting for approval' : ''}`);if (checkpoint || !next) return;}@@ -1451,12 +1480,11 @@ function advanceInBackground(projectId) {* lyrics. A failed unlink is not fatal: that same link replaces the old one.*/async function unlinkRunTrack(projectId, runId, trackId) {- const project = await getProject(projectId);- if (project?.trackId === trackId) {- await deps.updateProject(projectId, { trackId: null })- .catch((err) => console.warn(`⚠️ Autonomous music video ${short(runId)} could not unlink the retaken track: ${err.message}`));- }- console.log(`🎬 Autonomous music video ${short(runId)} retaking its song`);+ const { applyProjectPatch } = await import('./projectsLogic.js');+ // The track and run ownership must be checked in the same write as the unlink.+ await mutateProjectRecord(projectId, (project) => ({ project: project.trackId === trackId && ownsRunningRun(projectAutonomousRun(project), { id: runId })+ ? applyProjectPatch(project, { trackId: null }) : project }))+ .catch((err) => console.warn(`⚠️ Autonomous music video ${short(runId)} could not unlink the retaken track: ${err.message}`));}// ---- director actions ------------------------------------------------------------@@ -1861,11 +1889,12 @@ export async function stopAutonomousVideo(projectId) {const { run } = await requireRun(projectId);if (!['running', 'awaiting-approval', 'needs-human'].includes(run.status)) throw runError(409, 'NOT_RUNNING', `A ${run.status} run cannot be stopped`);const out = await patchRun(projectId, () => ({ status: 'stopped' }));+ const { productionRunId, finalAutoReviewId } = out.run.output;const childStopFailures = [];- if (run.output.productionRunId) {- await settleChildStop(childStopFailures, projectId, run, 'stop production', run.output.productionRunId, () => deps.stopProduction(projectId, run.output.productionRunId));+ if (productionRunId) {+ await settleChildStop(childStopFailures, projectId, run, 'stop production', productionRunId, () => deps.stopProduction(projectId, productionRunId));}- if (run.output.finalAutoReviewId) await settleChildStop(childStopFailures, projectId, run, 'stop auto-review', run.output.finalAutoReviewId, () => deps.stopAutoReview(projectId, run.output.finalAutoReviewId));+ if (finalAutoReviewId) await settleChildStop(childStopFailures, projectId, run, 'stop auto-review', finalAutoReviewId, () => deps.stopAutoReview(projectId, finalAutoReviewId));sunoControllers.get(run.id)?.abort();await cancelLocalSongJob(projectId, out.run, childStopFailures);return { project: out.project, run: presentAutonomousRun(out.run), childStopFailures };diff --git a/server/services/musicVideo/autonomousService.test.js b/server/services/musicVideo/autonomousService.test.jsindex 3ccc6d1157..f790c72570 100644--- a/server/services/musicVideo/autonomousService.test.js+++ b/server/services/musicVideo/autonomousService.test.js@@ -803,6 +803,95 @@ describe('cancel while the final render is preparing (#11768)', () => {});});+describe('stage completion after ownership is released (#12064)', () => {+ // Uniquely catches a successful late stage restoring approval/continuation after cancel, stop, or a process handoff.+ it.each(['cancel', 'stop', 'handoff'])('keeps a checkpoint released by %s terminal and preserves its finished output', async (action) => {+ const lyrics = Promise.withResolvers();+ doubles.writeLyrics.mockImplementationOnce(() => lyrics.promise);+ await service.startAutonomousVideo({ prompt: 'p', checkpoints: ['lyrics'] });+ await vi.waitFor(() => expect(doubles.writeLyrics).toHaveBeenCalledOnce());+ if (action === 'handoff') runOf().processId = 'proc-other';+ else await service[`${action}AutonomousVideo`]('mv-auto');+ const released = clone(runOf());+ lyrics.resolve({ lyrics: 'finished words' });+ await service.__testing.settleBackground();+ expect(runOf()).toMatchObject({ status: released.status, stage: 'lyrics', awaiting: released.awaiting, processId: released.processId,+ output: { lyrics: 'finished words' }, stages: { lyrics: { status: 'done' } } });+ expect(doubles.createMoodBoard).not.toHaveBeenCalled();+ expect(doubles.startProduction).not.toHaveBeenCalled();+ });++ // Uniquely catches production being dispatched after a cancel during the preceding preparation/review.+ it('does not start production when preparation returns after cancellation', async () => {+ const { prepareProductionReview } = await import('./productionReviewService.js');+ const prepare = Promise.withResolvers();+ prepareProductionReview.mockImplementationOnce(() => prepare.promise);+ await service.startAutonomousVideo({ prompt: 'p' });+ await vi.waitFor(() => expect(runOf()?.stage).toBe('produce'));+ await service.cancelAutonomousVideo('mv-auto');+ prepare.resolve();+ await service.__testing.settleBackground();+ expect(doubles.startProduction).not.toHaveBeenCalled();+ expect(runOf()).toMatchObject({ status: 'canceled', stage: 'produce', errorCode: null });+ });++ // Uniquely catches a child created during cancellation running without an owned autonomous parent.+ it.each(['cancel', 'stop', 'handoff'])('cancels a production returned after %s while preserving the released run', async (action) => {+ const production = Promise.withResolvers();+ doubles.startProduction.mockImplementationOnce(() => production.promise);+ await service.startAutonomousVideo({ prompt: 'p' });+ await vi.waitFor(() => expect(doubles.startProduction).toHaveBeenCalledOnce());+ if (action === 'handoff') runOf().processId = 'proc-other';+ else await service[`${action}AutonomousVideo`]('mv-auto');+ const released = clone(runOf());+ production.resolve({ run: { id: 'mvpr-late' } });+ await service.__testing.settleBackground();+ expect(doubles.cancelProduction).toHaveBeenCalledWith('mv-auto', 'mvpr-late');+ expect(runOf()).toMatchObject({ status: released.status, stage: 'produce', processId: released.processId, errorCode: null });+ expect(runOf().output.productionRunId).toBeUndefined();+ });++ // Uniquely catches cancellation winning the handoff write after the executor's last ownership read.+ it('cancels production when cancellation reaches the serialized handoff first', async () => {+ const projects = await import('./projects.js');+ const getProject = projects.getProject;+ let read;+ doubles.startProduction.mockImplementationOnce(async () => {+ read = vi.spyOn(projects, 'getProject').mockImplementationOnce(async (id) => {+ const snapshot = await getProject(id);+ writeTail = writeTail.then(() => { runOf().status = 'canceled'; });+ return snapshot;+ });+ return { run: { id: 'mvpr-handoff' } };+ });+ try {+ await service.startAutonomousVideo({ prompt: 'p' });+ await service.__testing.settleBackground();+ expect(doubles.cancelProduction).toHaveBeenCalledWith('mv-auto', 'mvpr-handoff');+ expect(runOf()).toMatchObject({ status: 'canceled', stage: 'produce', output: { productionRunId: 'mvpr-handoff' } });+ } finally {+ read?.mockRestore();+ }+ });++ // Uniquely catches stop reading a child id before an already-admitted write stores it.+ it('stops children stored between the initial read and the serialized status change', async () => {+ const lyrics = Promise.withResolvers();+ doubles.writeLyrics.mockImplementationOnce(() => lyrics.promise);+ doubles.stopAutoReview = vi.fn(async () => ({}));+ service.__setAutonomousDepsForTests(doubles);+ await service.startAutonomousVideo({ prompt: 'p' });+ await vi.waitFor(() => expect(doubles.writeLyrics).toHaveBeenCalledOnce());+ writeTail = writeTail.then(() => { Object.assign(runOf().output, { productionRunId: 'mvpr-just-stored', finalAutoReviewId: 'mvar-just-stored' }); });+ await service.stopAutonomousVideo('mv-auto');+ lyrics.resolve({ lyrics: 'finished words' });+ await service.__testing.settleBackground();+ expect(doubles.stopProduction).toHaveBeenCalledWith('mv-auto', 'mvpr-just-stored');+ expect(doubles.stopAutoReview).toHaveBeenCalledWith('mv-auto', 'mvar-just-stored');+ expect(runOf().status).toBe('stopped');+ });+});+describe('overlay text pass before the final render', () => {const finding = (atSec, message, severity = 'error') => ({ id: `f${atSec}`, kind: severity === 'error' ? 'overlap' : 'contrast', severity, atSec, texts: ['A'], message, count: 1 });const textEntries = () => (runOf()?.orchestration?.reviews || []).filter((r) => r.checkpoint === 'text');@@ -898,7 +987,8 @@ describe('retaking the song at the song checkpoint', () => {doubles.createTrack.mockResolvedValueOnce({ id: 'track-2' });doubles.generateSunoSong.mockImplementationOnce(async (_fields, opts) => {- // A fresh request: the rejected take's ids are not offered for reuse.+ // A fresh request: the rejected take's ids and project link are no longer used.+ expect(store.get('mv-auto').trackId).toBeNull();expect(opts.songIds).toBeNull();await opts.onSubmitted(['song-c', 'song-d']);return { songId: 'song-d', songIds: ['song-c', 'song-d'], filename: 'music-song-d.m4a' };@@ -918,8 +1008,7 @@ describe('retaking the song at the song checkpoint', () => {expect(doubles.generateSunoSong).toHaveBeenLastCalledWith(expect.objectContaining({lyrics: '[verse]\nnew words', style: 'darker synthwave', excludeStyles: 'metal', vocalGender: 'male', model: 'v6',}), expect.any(Object));- // The project was unlinked from the rejected track, then linked to the new one; analysis has not run.- expect(doubles.updateProject).toHaveBeenCalledWith('mv-auto', { trackId: null });+ // The project was linked to the new take; analysis has not run.expect(store.get('mv-auto').trackId).toBe('track-2');expect(calls).not.toContain('analyze');@@ -1599,6 +1688,22 @@ describe('orchestrated mode (brief.orchestrator)', () => {expect(doubles.orchestrate).toHaveBeenCalledWith(expect.objectContaining({ orchestrator: ORCHESTRATOR }));});+ // Uniquely catches a late song-retake verdict unlinking a canceled project's track or restarting song generation.+ it('keeps the song linked when a retake review returns after cancellation', async () => {+ const review = Promise.withResolvers();+ const orchestrate = doubles.orchestrate.getMockImplementation();+ doubles.orchestrate.mockImplementation((args) => checkpointOf(args.prompt) === 'song' ? review.promise : orchestrate(args));+ await service.startAutonomousVideo({ prompt: 'p', songSource: 'local', orchestrator: ORCHESTRATOR }, { autoApproveAuthorized: true });+ await vi.waitFor(() => expect(doubles.orchestrate.mock.calls.some(([args]) => checkpointOf(args.prompt) === 'song')).toBe(true));+ await service.cancelAutonomousVideo('mv-auto');+ review.resolve({ text: JSON.stringify({ verdict: 'retake', score: 3, notes: 'Try another take.' }), route: ORCHESTRATOR, visual: false });+ await service.__testing.settleBackground();+ expect(runOf()).toMatchObject({ status: 'canceled', stage: 'analyze' });+ expect(store.get('mv-auto').trackId).toBe('track-1');+ expect(doubles.generateLocalSong).toHaveBeenCalledOnce();+ expect(doubles.startProduction).not.toHaveBeenCalled();+ });+// Uniquely catches a run with no video tool spending on clips, the orchestrator approving stills it never saw,// or a flagged still not going back to production with the fix.it('stops production after the stills, regenerates the ones the orchestrator flags, then waits for the director', async () => {