Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: log child stop and cancel failures when an autonomous music video is stopped (#11456)
[._.]/ Adam Eivy3dcc539e3166
diff --git a/server/services/musicVideo/autonomousService.js b/server/services/musicVideo/autonomousService.jsindex 049b70b86c..b2f5ddddca 100644--- a/server/services/musicVideo/autonomousService.js+++ b/server/services/musicVideo/autonomousService.js@@ -1775,7 +1775,7 @@ async function resumeDelegatedProduction(projectId, run, { swapModels = false, lif (!adopt) {// Unlink first: the old run's own "canceled" event must no longer match this run.const out = await patchRun(projectId, (r) => ({ output: { productionRunId: null }, ...stagePatch(r, 'produce', { status: 'pending', error: null, step: null }) }));- if (current && PRODUCTION_LIVE.has(current.status)) await deps.cancelProduction(projectId, current.id).catch(() => {});+ if (current && PRODUCTION_LIVE.has(current.status)) await settleChildStop([], projectId, run, 'cancel production', current.id, () => deps.cancelProduction(projectId, current.id));console.log(`🎬 Autonomous music video ${short(run.id)} starting a new production run${swapModels ? ' with the new models' : ''}`);advanceInBackground(projectId);return { project: out.project, run: presentAutonomousRun(out.run) };@@ -1810,36 +1810,54 @@ async function resumeDelegatedProduction(projectId, run, { swapModels = false, lreturn { project: presentProjectAutonomousRun(latest), run: presentAutonomousRun(projectAutonomousRun(latest)) };}+/**+ * Stop one delegated child after the run is already marked stopped/canceled. A 404/409 means+ * the child is gone or already terminal (benign); anything else is logged and recorded in+ * `failures` so the response can say work may still be running. Never throws.+ */+async function settleChildStop(failures, projectId, run, action, childId, call) {+ try {+ await call();+ } catch (err) {+ if (err?.status === 404 || err?.status === 409) return;+ const message = err?.message || String(err);+ console.error(`❌ Autonomous music video ${short(run.id)} (project ${projectId}) could not ${action} ${childId}: ${message}`);+ failures.push({ kind: action, id: childId, error: message });+ }+}+// Read from the freshly patched record, so a job id stored a moment before the status flipped is seen.-const cancelLocalSongJob = (run) => (run.stage === 'song' && run.output.localSongJobId- ? deps.cancelLocalSong(run.output.localSongJobId).catch(() => {})+const cancelLocalSongJob = (projectId, run, failures) => (run.stage === 'song' && run.output.localSongJobId+ ? settleChildStop(failures, projectId, run, 'cancel song job', run.output.localSongJobId, () => deps.cancelLocalSong(run.output.localSongJobId)): null);export async function stopAutonomousVideo(projectId) {const { run } = await requireRun(projectId);if (!['running', 'awaiting-approval', 'needs-human'].includes(run.status)) throw runError(409, 'NOT_RUNNING', `A ${run.status} run cannot be stopped`);const out = await patchRun(projectId, () => ({ status: 'stopped' }));+ const childStopFailures = [];if (run.output.productionRunId) {- await deps.stopProduction(projectId, run.output.productionRunId).catch(() => {});+ await settleChildStop(childStopFailures, projectId, run, 'stop production', run.output.productionRunId, () => deps.stopProduction(projectId, run.output.productionRunId));}- if (run.output.finalAutoReviewId) await deps.stopAutoReview(projectId, run.output.finalAutoReviewId).catch(() => {});+ if (run.output.finalAutoReviewId) await settleChildStop(childStopFailures, projectId, run, 'stop auto-review', run.output.finalAutoReviewId, () => deps.stopAutoReview(projectId, run.output.finalAutoReviewId));sunoControllers.get(run.id)?.abort();- await cancelLocalSongJob(out.run);- return { project: out.project, run: presentAutonomousRun(out.run) };+ await cancelLocalSongJob(projectId, out.run, childStopFailures);+ return { project: out.project, run: presentAutonomousRun(out.run), childStopFailures };}export async function cancelAutonomousVideo(projectId) {const { run } = await requireRun(projectId);if (!(AUTONOMOUS_LIVE_STATUSES.includes(run.status) || run.status === 'failed')) throw runError(409, 'NOT_CANCELABLE', `A ${run.status} run cannot be canceled`);const out = await patchRun(projectId, () => ({ status: 'canceled', awaiting: null }));- if (run.output.renderJobId) await deps.cancelRender(run.output.renderJobId).catch(() => {});- if (run.output.finalAutoReviewId) await deps.cancelAutoReview(projectId, run.output.finalAutoReviewId).catch(() => {});+ const childStopFailures = [];+ if (run.output.renderJobId) await settleChildStop(childStopFailures, projectId, run, 'cancel render', run.output.renderJobId, () => deps.cancelRender(run.output.renderJobId));+ if (run.output.finalAutoReviewId) await settleChildStop(childStopFailures, projectId, run, 'cancel auto-review', run.output.finalAutoReviewId, () => deps.cancelAutoReview(projectId, run.output.finalAutoReviewId));if (run.output.productionRunId) {- await deps.cancelProduction(projectId, run.output.productionRunId).catch(() => {});+ await settleChildStop(childStopFailures, projectId, run, 'cancel production', run.output.productionRunId, () => deps.cancelProduction(projectId, run.output.productionRunId));}sunoControllers.get(run.id)?.abort();- await cancelLocalSongJob(out.run);- return { project: out.project, run: presentAutonomousRun(out.run) };+ await cancelLocalSongJob(projectId, out.run, childStopFailures);+ return { project: out.project, run: presentAutonomousRun(out.run), childStopFailures };}// ---- production completion -------------------------------------------------------diff --git a/server/services/musicVideo/autonomousService.test.js b/server/services/musicVideo/autonomousService.test.jsindex 8d21641956..83b49082bd 100644--- a/server/services/musicVideo/autonomousService.test.js+++ b/server/services/musicVideo/autonomousService.test.js@@ -219,6 +219,33 @@ describe('startAutonomousVideo', () => {expect(runOf().output).toMatchObject({ renderJobId: 'render-1' });});+ // Uniquely catches a failed child stop being swallowed while the delegated production keeps spending.+ it.each([+ ['stopAutonomousVideo', 'stopProduction', 'stopped', 'stop production'],+ ['cancelAutonomousVideo', 'cancelProduction', 'canceled', 'cancel production'],+ ])('%s reports a child failure but ignores an already-closed child (409/404)', async (action, dep, status, kind) => {+ await service.startAutonomousVideo({ prompt: 'p' });+ await vi.waitFor(() => expect(runOf()?.output.productionRunId).toBe('mvpr-1'));+ const errorLog = vi.spyOn(console, 'error').mockImplementation(() => {});+ try {+ doubles[dep].mockRejectedValueOnce(Object.assign(new Error("already closed"), { status: 409 }));+ const closed = await service[action]('mv-auto');+ expect(closed.childStopFailures).toEqual([]);+ expect(errorLog).not.toHaveBeenCalled();+ expect(runOf().status).toBe(status);++ store.get('mv-auto').autonomousRun.status = 'running';+ doubles[dep].mockRejectedValueOnce(new Error('store write failed'));+ const failed = await service[action]('mv-auto');+ expect(failed.childStopFailures).toEqual([{ kind, id: 'mvpr-1', error: 'store write failed' }]);+ expect(errorLog).toHaveBeenCalledWith(expect.stringContaining(`could not ${kind} mvpr-1: store write failed`));+ expect(errorLog.mock.calls[0][0]).toContain('project mv-auto');+ expect(runOf().status).toBe(status);+ } finally {+ errorLog.mockRestore();+ }+ });+it('finishes — and starts the final render — when the delegated production run completes', async () => {await service.startAutonomousVideo({ prompt: 'p' });await vi.waitFor(() => expect(runOf()?.output.productionRunId).toBe('mvpr-1'));diff --git a/server/services/musicVideo/productionService.js b/server/services/musicVideo/productionService.jsindex 0f77a3bcb5..fa927dfd26 100644--- a/server/services/musicVideo/productionService.js+++ b/server/services/musicVideo/productionService.js@@ -718,6 +718,12 @@ export async function resumeProduction(projectId, runId, { limits, acceptBasis =return response(fresh, findProductionRun(fresh, runId));}+// 404/409 = the review is gone or already closed; anything else leaves it running, so say so.+function logReviewStopFailure(verb, projectId, reviewId, err) {+ if (err?.status === 404 || err?.status === 409) return;+ console.error(`❌ Music video production could not ${verb} auto-review ${reviewId} (project ${projectId}): ${err?.message || err}`);+}+/** Stop: nothing new is dispatched and queued owned jobs are cancelled; running ones land. */export async function stopProduction(projectId, runId) {const out = await mutateProjectRecord(projectId, (current) => stopProductionOnProject(current, runId));@@ -725,7 +731,7 @@ export async function stopProduction(projectId, runId) {const owned = new Set([out.run.reviewRunId, ...(out.run.pilot?.scenes || []).map((p) => p.reviewRunId)]);for (const review of (out.project.autoReviews || []).filter((r) => owned.has(r.id) && r.status === 'running')) {const { stopAutoReview } = await deps.autoReview();- await stopAutoReview(projectId, review.id).catch(() => {});+ await stopAutoReview(projectId, review.id).catch((err) => logReviewStopFailure('stop', projectId, review.id, err));}const fresh = await requireProject(projectId);const run = findProductionRun(fresh, runId);@@ -741,7 +747,7 @@ export async function cancelProduction(projectId, runId) {const owned = new Set([out.run.reviewRunId, ...(out.run.pilot?.scenes || []).map((p) => p.reviewRunId)]);for (const review of (out.project.autoReviews || []).filter((r) => owned.has(r.id) && ['running', 'stopped', 'limit-reached'].includes(r.status))) {const { cancelAutoReview } = await deps.autoReview();- await cancelAutoReview(projectId, review.id).catch(() => {});+ await cancelAutoReview(projectId, review.id).catch((err) => logReviewStopFailure('cancel', projectId, review.id, err));}const fresh = await requireProject(projectId);const run = findProductionRun(fresh, runId);