Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: log child stop and cancel failures when an autonomous music video is stopped (#11456)

[._.]/ Adam Eivy3dcc539e3166
diff --git a/server/services/musicVideo/autonomousService.js b/server/services/musicVideo/autonomousService.js
index 049b70b86c..b2f5ddddca 100644
--- a/server/services/musicVideo/autonomousService.js
+++ b/server/services/musicVideo/autonomousService.js
@@ -1775,7 +1775,7 @@ async function resumeDelegatedProduction(projectId, run, { swapModels = false, l
if (!adopt) {
// Unlink first: the old run's own "canceled" event must no longer match this run.
const out = await patchRun(projectId, (r) => ({ output: { productionRunId: null }, ...stagePatch(r, 'produce', { status: 'pending', error: null, step: null }) }));
- if (current && PRODUCTION_LIVE.has(current.status)) await deps.cancelProduction(projectId, current.id).catch(() => {});
+ if (current && PRODUCTION_LIVE.has(current.status)) await settleChildStop([], projectId, run, 'cancel production', current.id, () => deps.cancelProduction(projectId, current.id));
console.log(`🎬 Autonomous music video ${short(run.id)} starting a new production run${swapModels ? ' with the new models' : ''}`);
advanceInBackground(projectId);
return { project: out.project, run: presentAutonomousRun(out.run) };
@@ -1810,36 +1810,54 @@ async function resumeDelegatedProduction(projectId, run, { swapModels = false, l
return { project: presentProjectAutonomousRun(latest), run: presentAutonomousRun(projectAutonomousRun(latest)) };
}
+/**
+ * Stop one delegated child after the run is already marked stopped/canceled. A 404/409 means
+ * the child is gone or already terminal (benign); anything else is logged and recorded in
+ * `failures` so the response can say work may still be running. Never throws.
+ */
+async function settleChildStop(failures, projectId, run, action, childId, call) {
+ try {
+ await call();
+ } catch (err) {
+ if (err?.status === 404 || err?.status === 409) return;
+ const message = err?.message || String(err);
+ console.error(`❌ Autonomous music video ${short(run.id)} (project ${projectId}) could not ${action} ${childId}: ${message}`);
+ failures.push({ kind: action, id: childId, error: message });
+ }
+}
+
// Read from the freshly patched record, so a job id stored a moment before the status flipped is seen.
-const cancelLocalSongJob = (run) => (run.stage === 'song' && run.output.localSongJobId
- ? deps.cancelLocalSong(run.output.localSongJobId).catch(() => {})
+const cancelLocalSongJob = (projectId, run, failures) => (run.stage === 'song' && run.output.localSongJobId
+ ? settleChildStop(failures, projectId, run, 'cancel song job', run.output.localSongJobId, () => deps.cancelLocalSong(run.output.localSongJobId))
: null);
export async function stopAutonomousVideo(projectId) {
const { run } = await requireRun(projectId);
if (!['running', 'awaiting-approval', 'needs-human'].includes(run.status)) throw runError(409, 'NOT_RUNNING', `A ${run.status} run cannot be stopped`);
const out = await patchRun(projectId, () => ({ status: 'stopped' }));
+ const childStopFailures = [];
if (run.output.productionRunId) {
- await deps.stopProduction(projectId, run.output.productionRunId).catch(() => {});
+ await settleChildStop(childStopFailures, projectId, run, 'stop production', run.output.productionRunId, () => deps.stopProduction(projectId, run.output.productionRunId));
}
- if (run.output.finalAutoReviewId) await deps.stopAutoReview(projectId, run.output.finalAutoReviewId).catch(() => {});
+ if (run.output.finalAutoReviewId) await settleChildStop(childStopFailures, projectId, run, 'stop auto-review', run.output.finalAutoReviewId, () => deps.stopAutoReview(projectId, run.output.finalAutoReviewId));
sunoControllers.get(run.id)?.abort();
- await cancelLocalSongJob(out.run);
- return { project: out.project, run: presentAutonomousRun(out.run) };
+ await cancelLocalSongJob(projectId, out.run, childStopFailures);
+ return { project: out.project, run: presentAutonomousRun(out.run), childStopFailures };
}
export async function cancelAutonomousVideo(projectId) {
const { run } = await requireRun(projectId);
if (!(AUTONOMOUS_LIVE_STATUSES.includes(run.status) || run.status === 'failed')) throw runError(409, 'NOT_CANCELABLE', `A ${run.status} run cannot be canceled`);
const out = await patchRun(projectId, () => ({ status: 'canceled', awaiting: null }));
- if (run.output.renderJobId) await deps.cancelRender(run.output.renderJobId).catch(() => {});
- if (run.output.finalAutoReviewId) await deps.cancelAutoReview(projectId, run.output.finalAutoReviewId).catch(() => {});
+ const childStopFailures = [];
+ if (run.output.renderJobId) await settleChildStop(childStopFailures, projectId, run, 'cancel render', run.output.renderJobId, () => deps.cancelRender(run.output.renderJobId));
+ if (run.output.finalAutoReviewId) await settleChildStop(childStopFailures, projectId, run, 'cancel auto-review', run.output.finalAutoReviewId, () => deps.cancelAutoReview(projectId, run.output.finalAutoReviewId));
if (run.output.productionRunId) {
- await deps.cancelProduction(projectId, run.output.productionRunId).catch(() => {});
+ await settleChildStop(childStopFailures, projectId, run, 'cancel production', run.output.productionRunId, () => deps.cancelProduction(projectId, run.output.productionRunId));
}
sunoControllers.get(run.id)?.abort();
- await cancelLocalSongJob(out.run);
- return { project: out.project, run: presentAutonomousRun(out.run) };
+ await cancelLocalSongJob(projectId, out.run, childStopFailures);
+ return { project: out.project, run: presentAutonomousRun(out.run), childStopFailures };
}
// ---- production completion -------------------------------------------------------
diff --git a/server/services/musicVideo/autonomousService.test.js b/server/services/musicVideo/autonomousService.test.js
index 8d21641956..83b49082bd 100644
--- a/server/services/musicVideo/autonomousService.test.js
+++ b/server/services/musicVideo/autonomousService.test.js
@@ -219,6 +219,33 @@ describe('startAutonomousVideo', () => {
expect(runOf().output).toMatchObject({ renderJobId: 'render-1' });
});
+ // Uniquely catches a failed child stop being swallowed while the delegated production keeps spending.
+ it.each([
+ ['stopAutonomousVideo', 'stopProduction', 'stopped', 'stop production'],
+ ['cancelAutonomousVideo', 'cancelProduction', 'canceled', 'cancel production'],
+ ])('%s reports a child failure but ignores an already-closed child (409/404)', async (action, dep, status, kind) => {
+ await service.startAutonomousVideo({ prompt: 'p' });
+ await vi.waitFor(() => expect(runOf()?.output.productionRunId).toBe('mvpr-1'));
+ const errorLog = vi.spyOn(console, 'error').mockImplementation(() => {});
+ try {
+ doubles[dep].mockRejectedValueOnce(Object.assign(new Error("already closed"), { status: 409 }));
+ const closed = await service[action]('mv-auto');
+ expect(closed.childStopFailures).toEqual([]);
+ expect(errorLog).not.toHaveBeenCalled();
+ expect(runOf().status).toBe(status);
+
+ store.get('mv-auto').autonomousRun.status = 'running';
+ doubles[dep].mockRejectedValueOnce(new Error('store write failed'));
+ const failed = await service[action]('mv-auto');
+ expect(failed.childStopFailures).toEqual([{ kind, id: 'mvpr-1', error: 'store write failed' }]);
+ expect(errorLog).toHaveBeenCalledWith(expect.stringContaining(`could not ${kind} mvpr-1: store write failed`));
+ expect(errorLog.mock.calls[0][0]).toContain('project mv-auto');
+ expect(runOf().status).toBe(status);
+ } finally {
+ errorLog.mockRestore();
+ }
+ });
+
it('finishes — and starts the final render — when the delegated production run completes', async () => {
await service.startAutonomousVideo({ prompt: 'p' });
await vi.waitFor(() => expect(runOf()?.output.productionRunId).toBe('mvpr-1'));
diff --git a/server/services/musicVideo/productionService.js b/server/services/musicVideo/productionService.js
index 0f77a3bcb5..fa927dfd26 100644
--- a/server/services/musicVideo/productionService.js
+++ b/server/services/musicVideo/productionService.js
@@ -718,6 +718,12 @@ export async function resumeProduction(projectId, runId, { limits, acceptBasis =
return response(fresh, findProductionRun(fresh, runId));
}
+// 404/409 = the review is gone or already closed; anything else leaves it running, so say so.
+function logReviewStopFailure(verb, projectId, reviewId, err) {
+ if (err?.status === 404 || err?.status === 409) return;
+ console.error(`❌ Music video production could not ${verb} auto-review ${reviewId} (project ${projectId}): ${err?.message || err}`);
+}
+
/** Stop: nothing new is dispatched and queued owned jobs are cancelled; running ones land. */
export async function stopProduction(projectId, runId) {
const out = await mutateProjectRecord(projectId, (current) => stopProductionOnProject(current, runId));
@@ -725,7 +731,7 @@ export async function stopProduction(projectId, runId) {
const owned = new Set([out.run.reviewRunId, ...(out.run.pilot?.scenes || []).map((p) => p.reviewRunId)]);
for (const review of (out.project.autoReviews || []).filter((r) => owned.has(r.id) && r.status === 'running')) {
const { stopAutoReview } = await deps.autoReview();
- await stopAutoReview(projectId, review.id).catch(() => {});
+ await stopAutoReview(projectId, review.id).catch((err) => logReviewStopFailure('stop', projectId, review.id, err));
}
const fresh = await requireProject(projectId);
const run = findProductionRun(fresh, runId);
@@ -741,7 +747,7 @@ export async function cancelProduction(projectId, runId) {
const owned = new Set([out.run.reviewRunId, ...(out.run.pilot?.scenes || []).map((p) => p.reviewRunId)]);
for (const review of (out.project.autoReviews || []).filter((r) => owned.has(r.id) && ['running', 'stopped', 'limit-reached'].includes(r.status))) {
const { cancelAutoReview } = await deps.autoReview();
- await cancelAutoReview(projectId, review.id).catch(() => {});
+ await cancelAutoReview(projectId, review.id).catch((err) => logReviewStopFailure('cancel', projectId, review.id, err));
}
const fresh = await requireProject(projectId);
const run = findProductionRun(fresh, runId);