Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: preserve cancellation across late stage failures and child creation

[._.]/ Adam Eivy4660f628cce0
diff --git a/server/services/musicVideo/autonomousService.js b/server/services/musicVideo/autonomousService.js
index 523abaa2c8..4b957d6cba 100644
--- a/server/services/musicVideo/autonomousService.js
+++ b/server/services/musicVideo/autonomousService.js
@@ -1250,6 +1250,10 @@ const STAGES = {
});
// A stop during preparation had no child id to stop; cancel the fresh child before handing it off.
if (!ownsRunningRun(projectAutonomousRun(await getProject(project.id)), run)) {
+ // Persist the child before cancellation: a failed stop must leave it discoverable for recovery.
+ await patchRun(project.id, (current) => current.id === run.id && current.output.productionRunId === run.output.productionRunId
+ ? { output: { productionRunId: started.run.id } } : {})
+ .catch((err) => console.error(`❌ Autonomous music video ${short(run.id)} could not retain late production ${started.run.id}: ${err.message}`));
await settleChildStop([], project.id, run, 'cancel production', started.run.id, () => deps.cancelProduction(project.id, started.run.id));
throw runError(409, 'RUN_STOPPED', 'The run stopped while production was preparing');
}
@@ -1372,10 +1376,14 @@ async function reviewOrPark(projectId, run, productionRunId) {
const isLoginRequired = (err) => err?.code === 'PUBLISH_LOGIN_REQUIRED';
+function logParkedRun(run, status, error) {
+ const log = status === 'failed' ? console.error : status === 'completed' ? console.log : console.warn;
+ log(`${status === 'completed' ? '✅' : status === 'failed' ? '❌' : '⏸️'} Autonomous music video ${short(run.id)} ${status}${error ? `: ${error}` : ''}`);
+}
+
async function park(projectId, status, patch) {
const out = await patchRun(projectId, () => ({ status, ...patch }));
- const log = status === 'failed' ? console.error : status === 'completed' ? console.log : console.warn;
- log(`${status === 'completed' ? '✅' : status === 'failed' ? '❌' : '⏸️'} Autonomous music video ${short(out.run.id)} ${status}${patch.error ? `: ${patch.error}` : ''}`);
+ logParkedRun(out.run, status, patch.error);
return out;
}
@@ -1399,11 +1407,17 @@ async function advance(projectId) {
try {
result = await executor({ project, run, save: (patch) => patchRun(projectId, () => patch) });
} catch (err) {
- // A stop/cancel mid-stage (it cancels the song operation) must not become a failure.
- const latest = projectAutonomousRun(await getProject(projectId));
- if (latest?.status !== 'running' || latest.processId !== PROCESS_ID) return;
- await patchRun(projectId, (r) => stagePatch(r, stage, { status: 'failed', error: trimTo(err.message, 500), step: null }));
- await park(projectId, isLoginRequired(err) || err.code === 'MUSIC_VIDEO_APPROVAL_REQUIRED' ? 'needs-human' : 'failed', { error: trimTo(err.message, 500), errorCode: err.code || null });
+ // Recheck ownership inside the failure write, so an already queued stop/cancel wins.
+ let failed = false;
+ const out = await patchRun(projectId, (r) => {
+ failed = ownsRunningRun(r, run);
+ return failed ? {
+ ...stagePatch(r, stage, { status: 'failed', error: trimTo(err.message, 500), step: null }),
+ status: isLoginRequired(err) || err.code === 'MUSIC_VIDEO_APPROVAL_REQUIRED' ? 'needs-human' : 'failed',
+ error: trimTo(err.message, 500), errorCode: err.code || null,
+ } : {};
+ });
+ if (failed) logParkedRun(out.run, out.run.status, out.run.error);
return;
}
const finishedAt = new Date().toISOString();
diff --git a/server/services/musicVideo/autonomousService.test.js b/server/services/musicVideo/autonomousService.test.js
index d457c0fce0..4ccc62e915 100644
--- a/server/services/musicVideo/autonomousService.test.js
+++ b/server/services/musicVideo/autonomousService.test.js
@@ -804,6 +804,54 @@ describe('cancel while the final render is preparing (#11768)', () => {
});
describe('stage completion after ownership is released (#12064)', () => {
+ // A stop admitted ahead of failure persistence must win even while both writes wait for the store.
+ it.each([
+ ['cancel', 'canceled', null],
+ ['stop', 'stopped', 'MUSIC_VIDEO_APPROVAL_REQUIRED'],
+ ])('preserves %s when a stage failure queues behind its status write', async (action, status, code) => {
+ const projects = await import('./projects.js');
+ const lyrics = Promise.withResolvers();
+ doubles.writeLyrics.mockImplementationOnce(() => lyrics.promise);
+ await service.startAutonomousVideo({ prompt: 'p', checkpoints: ['lyrics'] });
+ await vi.waitFor(() => expect(doubles.writeLyrics).toHaveBeenCalledOnce());
+ const barrier = Promise.withResolvers();
+ writeTail = writeTail.then(() => barrier.promise);
+ const writes = vi.spyOn(projects, 'mutateProjectRecord');
+ let release;
+ try {
+ release = service[`${action}AutonomousVideo`]('mv-auto');
+ lyrics.reject(Object.assign(new Error('late stage failure'), { code }));
+ await vi.waitFor(() => expect(writes).toHaveBeenCalledTimes(2));
+ } finally {
+ barrier.resolve();
+ await release;
+ await service.__testing.settleBackground();
+ writes.mockRestore();
+ }
+ expect(runOf()).toMatchObject({ status, stage: 'lyrics', error: null, errorCode: null });
+ expect(doubles.createMoodBoard).not.toHaveBeenCalled();
+ });
+
+ // A failed cancellation must leave the newly created child discoverable on its released parent.
+ it('retains late production linkage when child cancellation fails', async () => {
+ const production = Promise.withResolvers();
+ doubles.startProduction.mockImplementationOnce(() => production.promise);
+ doubles.cancelProduction.mockRejectedValueOnce(new Error('production store unavailable'));
+ const errorLog = vi.spyOn(console, 'error').mockImplementation(() => {});
+ try {
+ await service.startAutonomousVideo({ prompt: 'p' });
+ await vi.waitFor(() => expect(doubles.startProduction).toHaveBeenCalledOnce());
+ await service.cancelAutonomousVideo('mv-auto');
+ production.resolve({ run: { id: 'mvpr-late-failed-cancel' } });
+ await service.__testing.settleBackground();
+ expect(doubles.cancelProduction).toHaveBeenCalledWith('mv-auto', 'mvpr-late-failed-cancel');
+ expect(runOf()).toMatchObject({ status: 'canceled', stage: 'produce', output: { productionRunId: 'mvpr-late-failed-cancel' } });
+ expect(errorLog).toHaveBeenCalledWith(expect.stringContaining('could not cancel production mvpr-late-failed-cancel'));
+ } finally {
+ errorLog.mockRestore();
+ }
+ });
+
// Uniquely catches a successful late stage restoring approval/continuation after cancel, stop, or a process handoff.
it.each(['cancel', 'stop', 'handoff'])('keeps a checkpoint released by %s terminal and preserves its finished output', async (action) => {
const lyrics = Promise.withResolvers();
@@ -821,6 +869,41 @@ describe('stage completion after ownership is released (#12064)', () => {
expect(doubles.startProduction).not.toHaveBeenCalled();
});
+ it('still cancels late production when saving its recovery linkage fails', async () => {
+ const projects = await import('./projects.js');
+ const production = Promise.withResolvers();
+ doubles.startProduction.mockImplementationOnce(() => production.promise);
+ const errorLog = vi.spyOn(console, 'error').mockImplementation(() => {});
+ let write;
+ try {
+ await service.startAutonomousVideo({ prompt: 'p' });
+ await vi.waitFor(() => expect(doubles.startProduction).toHaveBeenCalledOnce());
+ await service.cancelAutonomousVideo('mv-auto');
+ write = vi.spyOn(projects, 'mutateProjectRecord').mockRejectedValueOnce(new Error('link store unavailable'));
+ production.resolve({ run: { id: 'mvpr-late-unlinked' } });
+ await service.__testing.settleBackground();
+ expect(doubles.cancelProduction).toHaveBeenCalledWith('mv-auto', 'mvpr-late-unlinked');
+ expect(runOf().status).toBe('canceled');
+ expect(errorLog).toHaveBeenCalledWith(expect.stringContaining('link store unavailable'));
+ } finally {
+ write?.mockRestore();
+ errorLog.mockRestore();
+ }
+ });
+
+ it('preserves a replacement child when late production returns after a process handoff', async () => {
+ const production = Promise.withResolvers();
+ doubles.startProduction.mockImplementationOnce(() => production.promise);
+ await service.startAutonomousVideo({ prompt: 'p' });
+ await vi.waitFor(() => expect(doubles.startProduction).toHaveBeenCalledOnce());
+ runOf().processId = 'proc-other';
+ runOf().output.productionRunId = 'mvpr-replacement';
+ production.resolve({ run: { id: 'mvpr-obsolete' } });
+ await service.__testing.settleBackground();
+ expect(doubles.cancelProduction).toHaveBeenCalledWith('mv-auto', 'mvpr-obsolete');
+ expect(runOf()).toMatchObject({ status: 'running', processId: 'proc-other', output: { productionRunId: 'mvpr-replacement' } });
+ });
+
// Uniquely catches production being dispatched after a cancel during the preceding preparation/review.
it('does not start production when preparation returns after cancellation', async () => {
const { prepareProductionReview } = await import('./productionReviewService.js');
@@ -848,7 +931,7 @@ describe('stage completion after ownership is released (#12064)', () => {
await service.__testing.settleBackground();
expect(doubles.cancelProduction).toHaveBeenCalledWith('mv-auto', 'mvpr-late');
expect(runOf()).toMatchObject({ status: released.status, stage: 'produce', processId: released.processId, errorCode: null });
- expect(runOf().output.productionRunId).toBeUndefined();
+ expect(runOf().output.productionRunId).toBe('mvpr-late');
});
// Uniquely catches cancellation winning the handoff write after the executor's last ownership read.