Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: preserve canceled Music Video renders after ffmpeg exits (#10702)

Adam Eivy432f8b76440e
diff --git a/server/services/README.md b/server/services/README.md
index 2f9e67be21..4af5e9edb0 100644
--- a/server/services/README.md
+++ b/server/services/README.md
@@ -11,6 +11,7 @@ image-generation paths below are useful entry points when tracing render jobs.
| `databaseMaintenanceProducers.js` | Internal one-use coordinator stage: retain producer identity, stop this install's PM2 CoS/server, and verify fresh readback while admission remains fenced. |
| `databaseMaintenanceQuiescence.js` | Internal coordinator stage after producer shutdown: terminate verified running detached-writer groups, prove every registry record quiescent from process-table evidence, archive it with the operation, and refuse pending, legacy, orphaned, ambiguous or unregistered writers. |
| `imageGen/prepareParams.js` | Shared image request preparation and the canonical local image model selector. Use `selectLocalImageModelFromSettings()` for model identity and `resolveLocalImageModel()` when the caller must also enforce local runtime, hardware, and edit-image requirements. |
+| `musicVideo/renderCancellation.js` | Shared capture-abort and ffmpeg kill selection for full/excerpt renders; records explicit cancellation before signaling and classifies nonzero closes without releasing ownership. |
| `musicVideo/sharingCopy.js` | Source-hashed private sharing MP4s: queued 720p H.264/AAC export, strict decimal 100 MB cap, supervised cancellation, verified download and owned-file cleanup. |
| `mediaJobQueue/index.js` | Owns admission, lanes, durability and terminal effects; `videoHolds.js` returns local-video cohort preparation results and owns circuit breakers, `recoverySettlement.js` names the transient recovery evidence and the maintenance-permit release decision, while `videoGen/modelSelection.js` owns model-selection policy. |
| `imageGen/index.js` | Mode-aware image backend dispatcher and shared image-generation helpers. Direct queue consumers still resolve local model identity before enqueueing. |
diff --git a/server/services/musicVideo/excerptRender.js b/server/services/musicVideo/excerptRender.js
index ce4ddd9087..c9eabb42db 100644
--- a/server/services/musicVideo/excerptRender.js
+++ b/server/services/musicVideo/excerptRender.js
@@ -28,7 +28,7 @@ import { spawn } from '../../lib/childProcess.js';
import { safeChildProcessOptions } from '../../lib/processEnv.js';
import { broadcastSse, attachSseClient as attachSse, closeJobAfterDelay } from '../../lib/sseUtils.js';
import { attachFfmpegRenderGuard } from '../../lib/ffmpegRenderGuard.js';
-import { killWithEscalation } from '../../lib/killWithEscalation.js';
+import { cancelMusicVideoRenderJob, isMusicVideoRenderCanceled } from './renderCancellation.js';
import { safeUnder } from '../../lib/ffmpeg.js';
import { encodeFileContactSheetAtTimes } from '../htmlComposition/encode.js';
import { getProject, listProjects, mutateProjectRecord } from './projects.js';
@@ -54,21 +54,7 @@ const PENDING = Symbol('mv-excerpt-render-pending');
export const attachExcerptRenderSseClient = (jobId, res) => attachSse(jobs, jobId, res);
export function cancelExcerptRender(jobId) {
- const job = jobs.get(jobId);
- if (!job) return false;
- if (!job.process) {
- if (job.status !== 'running' || !job.overlayAbort || job.overlayAbort.signal.aborted) return false;
- job.overlayAbort.abort(new Error('Excerpt render cancelled'));
- return true;
- }
- // ffmpeg often intercepts SIGTERM and exits itself (nonzero code, `signal:
- // null` on the child's 'close' event) rather than dying FROM the signal —
- // `onClose` below can't tell that apart from a genuine encode failure by
- // signal alone, so record that a cancel was actually requested.
- job.cancelRequested = true;
- const proc = job.process;
- killWithEscalation(proc, { label: 'music-video excerpt render', stillRunning: () => job.process === proc });
- return true;
+ return cancelMusicVideoRenderJob(jobs.get(jobId), { label: 'music-video excerpt render' });
}
const round3 = (n) => Math.round(n * 1000) / 1000;
@@ -431,7 +417,7 @@ export async function startExcerptRender(projectId, { startSec, endSec, aspect =
onClose: async (code, signal) => {
job.process = null;
if (code !== 0) {
- const canceled = job.cancelRequested || signal === 'SIGTERM' || signal === 'SIGKILL';
+ const canceled = isMusicVideoRenderCanceled(job, signal);
job.status = canceled ? 'canceled' : 'error';
const reason = canceled ? 'Render cancelled' : signal ? `Killed by signal ${signal}` : `ffmpeg exit ${code}`;
job.lastError = reason;
diff --git a/server/services/musicVideo/excerptRender.test.js b/server/services/musicVideo/excerptRender.test.js
new file mode 100644
index 0000000000..6d8889f349
--- /dev/null
+++ b/server/services/musicVideo/excerptRender.test.js
@@ -0,0 +1,144 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+
+const h = vi.hoisted(() => {
+ const emitter = () => {
+ const listeners = {};
+ return {
+ on(event, fn) { (listeners[event] ||= []).push(fn); return this; },
+ emit(event, ...args) { for (const fn of listeners[event] || []) fn(...args); },
+ };
+ };
+ const procs = [];
+ const spawn = () => {
+ const proc = { ...emitter(), stderr: emitter(), kill: vi.fn() };
+ procs.push(proc);
+ return proc;
+ };
+ const store = new Map();
+ const overlays = { resolve: null };
+ return { procs, spawn, store, overlays };
+});
+
+vi.mock('../../lib/childProcess.js', () => ({ spawn: h.spawn, execFile: vi.fn(), exec: vi.fn() }));
+vi.mock('../../lib/processEnv.js', () => ({ safeChildProcessOptions: (options) => ({ ...options, env: {} }) }));
+vi.mock('fs/promises', () => ({ unlink: vi.fn(async () => {}) }));
+vi.mock('../../lib/fileUtils.js', () => ({
+ ensureDir: vi.fn(async () => {}),
+ PATHS: { videos: '/test/videos', videoThumbnails: '/test/thumbnails' },
+}));
+vi.mock('../../lib/sseUtils.js', () => ({ broadcastSse: vi.fn(), attachSseClient: vi.fn(), closeJobAfterDelay: vi.fn() }));
+vi.mock('../../lib/killWithEscalation.js', () => ({ killWithEscalation: vi.fn() }));
+vi.mock('../instanceIdentity.js', () => ({ ensureInstanceId: vi.fn(async () => 'test-instance') }));
+vi.mock('../htmlComposition/encode.js', () => ({ encodeFileContactSheetAtTimes: vi.fn(async () => {}) }));
+vi.mock('./projects.js', () => ({
+ getProject: vi.fn(async (id) => h.store.get(id)),
+ listProjects: vi.fn(async () => []),
+ mutateProjectRecord: vi.fn(async (id, transform) => {
+ const outcome = transform(h.store.get(id));
+ h.store.set(id, outcome.project);
+ return outcome;
+ }),
+}));
+vi.mock('./render.js', () => ({
+ planMusicVideoRender: vi.fn(async (project) => ({ ffmpeg: 'ffmpeg', audioPath: '/test/song.wav', clips: [], composed: !!project.composition, audioDurationSec: 4, soundBed: null })),
+ buildMusicVideoFfmpegArgs: vi.fn(() => ({ args: [], totalDuration: 4, canonW: 64, canonH: 64, fps: 24, sections: [] })),
+ excerptBoundaryTimes: vi.fn(() => [0]),
+ isLocalRenderMark: vi.fn(() => true),
+ assertCurrentClipDependencies: vi.fn(),
+ resolveMasterAudioPath: vi.fn(),
+ resolveSoundBedPath: vi.fn(),
+}));
+vi.mock('./compositionRender.js', () => ({
+ removeCompositionScratch: vi.fn(async () => {}),
+ renderTypographyOverlays: vi.fn(({ signal }) => new Promise((resolve, reject) => {
+ h.overlays.resolve = resolve;
+ signal.addEventListener('abort', () => reject(signal.reason), { once: true });
+ })),
+}));
+
+import { startExcerptRender, cancelExcerptRender } from './excerptRender.js';
+import { unlink } from 'fs/promises';
+import { broadcastSse } from '../../lib/sseUtils.js';
+import { removeCompositionScratch } from './compositionRender.js';
+import { musicVideoEvents } from './events.js';
+
+const prime = (id, composition = null) => h.store.set(id, { id, name: 'Test', scenes: [], excerpts: [], composition });
+const lastProc = () => h.procs.at(-1);
+const excerpt = (id, excerptId) => h.store.get(id).excerpts.find((record) => record.id === excerptId);
+const terminalFrames = (jobId) => broadcastSse.mock.calls.filter(([job, frame]) => job.id === jobId && ['canceled', 'error', 'complete'].includes(frame.type)).map(([, frame]) => frame);
+
+beforeEach(() => {
+ vi.clearAllMocks();
+ h.procs.length = 0;
+ h.store.clear();
+});
+
+describe('excerpt render cancellation', () => {
+ it('keeps a canceled encode owned until close, removes partial output, and settles once', async () => {
+ const id = 'excerpt-cancel';
+ prime(id);
+ const events = [];
+ const onRender = (event) => events.push(event);
+ musicVideoEvents.on('excerpt-render', onRender);
+ const { jobId, excerptId } = await startExcerptRender(id, { startSec: 0, endSec: 2 });
+ const partialFilename = excerpt(id, excerptId).partialFilename;
+ const proc = lastProc();
+ proc.emit('spawn');
+ expect(cancelExcerptRender(jobId)).toBe(true);
+ proc.emit('error', new Error('kill EPERM'));
+ expect(excerpt(id, excerptId).status).toBe('rendering');
+ await expect(startExcerptRender(id, { startSec: 0, endSec: 2 })).rejects.toMatchObject({ code: 'EXCERPT_RENDER_IN_PROGRESS' });
+ proc.emit('close', 255, null);
+ await vi.waitFor(() => expect(terminalFrames(jobId)).toHaveLength(1));
+ proc.emit('close', 255, null);
+ musicVideoEvents.off('excerpt-render', onRender);
+ expect(terminalFrames(jobId)).toEqual([{ type: 'canceled', error: 'Render cancelled' }]);
+ expect(excerpt(id, excerptId)).toMatchObject({ status: 'canceled', filename: null, partialFilename: null, jobId: null, renderingOn: null, error: null });
+ expect(unlink.mock.calls).toEqual([[`/test/videos/${partialFilename}`]]);
+ expect(events).toEqual([{ projectId: id, excerptId, status: 'canceled' }]);
+ expect(cancelExcerptRender(jobId)).toBe(false);
+ const again = await startExcerptRender(id, { startSec: 0, endSec: 2 });
+ lastProc().emit('close', 1, null);
+ await vi.waitFor(() => expect(terminalFrames(again.jobId)).toHaveLength(1));
+ });
+
+ it.each([{ code: 255, status: 'error' }, { code: 0, status: 'complete' }])('retains $status for an uncanceled close($code, null)', async ({ code, status }) => {
+ const id = `excerpt-close-${code}`;
+ prime(id);
+ const { jobId, excerptId } = await startExcerptRender(id, { startSec: 0, endSec: 2 });
+ lastProc().emit('spawn');
+ lastProc().emit('close', code, null);
+ await vi.waitFor(() => expect(terminalFrames(jobId)).toHaveLength(1));
+ expect(excerpt(id, excerptId).status).toBe(status);
+ expect(terminalFrames(jobId)[0].type).toBe(status);
+ });
+
+ it('retains zero-exit success after a cancellation request', async () => {
+ const id = 'excerpt-cancel-success';
+ prime(id);
+ const { jobId, excerptId } = await startExcerptRender(id, { startSec: 0, endSec: 2 });
+ lastProc().emit('spawn');
+ expect(cancelExcerptRender(jobId)).toBe(true);
+ lastProc().emit('close', 0, null);
+ await vi.waitFor(() => expect(terminalFrames(jobId)).toHaveLength(1));
+ expect(excerpt(id, excerptId).status).toBe('complete');
+ expect(terminalFrames(jobId)[0].type).toBe('complete');
+ });
+
+ it('aborts overlay capture without spawning and releases scratch and the excerpt slot', async () => {
+ const id = 'excerpt-capture';
+ prime(id, { mode: 'composed', textCues: [{ id: 'cue', text: 'Test', startSec: 0, endSec: 2 }], style: {} });
+ const { jobId, excerptId } = await startExcerptRender(id, { startSec: 0, endSec: 2 });
+ expect(h.procs).toHaveLength(0);
+ expect(cancelExcerptRender(jobId)).toBe(true);
+ expect(cancelExcerptRender(jobId)).toBe(false);
+ await vi.waitFor(() => expect(terminalFrames(jobId)).toHaveLength(1));
+ expect(excerpt(id, excerptId).status).toBe('canceled');
+ expect(removeCompositionScratch).toHaveBeenCalledExactlyOnceWith(jobId);
+ expect(h.procs).toHaveLength(0);
+ const again = await startExcerptRender(id, { startSec: 0, endSec: 2 });
+ expect(again.jobId).not.toBe(jobId);
+ cancelExcerptRender(again.jobId);
+ await vi.waitFor(() => expect(terminalFrames(again.jobId)).toHaveLength(1));
+ });
+});
diff --git a/server/services/musicVideo/render.js b/server/services/musicVideo/render.js
index 7d7ac63e20..b4e4799cba 100644
--- a/server/services/musicVideo/render.js
+++ b/server/services/musicVideo/render.js
@@ -35,7 +35,7 @@ import { ServerError } from '../../lib/errorHandler.js';
import { broadcastSse, attachSseClient as attachSse, closeJobAfterDelay } from '../../lib/sseUtils.js';
import { findFfmpeg, safeUnder, generateThumbnail, probeVideoDuration, probeVideoGeometry } from '../../lib/ffmpeg.js';
import { safeChildProcessOptions } from '../../lib/processEnv.js';
-import { killWithEscalation } from '../../lib/killWithEscalation.js';
+import { cancelMusicVideoRenderJob, isMusicVideoRenderCanceled } from './renderCancellation.js';
import { attachFfmpegRenderGuard } from '../../lib/ffmpegRenderGuard.js';
import { loadHistory, mutateVideoHistory } from '../videoGen/local.js';
import { getTrack } from '../tracks/index.js';
@@ -157,18 +157,7 @@ export function getRenderJobStatus(jobId) {
}
export function cancelRender(jobId) {
- const job = jobs.get(jobId);
- if (!job) return false;
- // A composed render spends its first phase capturing the typography overlay
- // (no ffmpeg yet); abort that capture instead.
- if (!job.process) {
- if (job.status !== 'running' || !job.overlayAbort || job.overlayAbort.signal.aborted) return false;
- job.overlayAbort.abort(new Error('Render cancelled'));
- return true;
- }
- const proc = job.process;
- killWithEscalation(proc, { label: 'music-video render', stillRunning: () => job.process === proc });
- return true;
+ return cancelMusicVideoRenderJob(jobs.get(jobId), { label: 'music-video render' });
}
// Resolve the project's source audio to a verified path under data/music/.
@@ -948,7 +937,7 @@ async function renderAdmittedMusicVideo(projectId, options, permit) {
onClose: async (code, signal) => {
job.process = null;
if (code !== 0) {
- const canceled = signal === 'SIGTERM' || signal === 'SIGKILL';
+ const canceled = isMusicVideoRenderCanceled(job, signal);
job.status = canceled ? 'canceled' : 'error';
const reason = canceled ? 'Render cancelled' : signal ? `Killed by signal ${signal}` : `ffmpeg exit ${code}`;
job.lastError = reason;
diff --git a/server/services/musicVideo/render.terminal.test.js b/server/services/musicVideo/render.terminal.test.js
index 39d3eb33cb..a3b366acb7 100644
--- a/server/services/musicVideo/render.terminal.test.js
+++ b/server/services/musicVideo/render.terminal.test.js
@@ -85,6 +85,9 @@ import { getTrack } from '../tracks/index.js';
import { getProject, listProjects, updateProject, mutateProjectRecord } from './projects.js';
import { unlink } from 'fs/promises';
import { musicVideoEvents } from './events.js';
+import { broadcastSse } from '../../lib/sseUtils.js';
+import { killWithEscalation } from '../../lib/killWithEscalation.js';
+import { maintenance } from '../../lib/maintenanceAdmission.js';
const tick = () => new Promise((r) => setTimeout(r, 0));
const lastProc = () => h.procs[h.procs.length - 1];
@@ -181,6 +184,55 @@ describe('renderMusicVideo terminal handling (#2386)', () => {
expect(getRenderJobStatus(jobId).status).toBe('error');
});
+ it('preserves a completed project when canceled ffmpeg exits 255 without a signal', async () => {
+ const pid = 'cancel-close';
+ prime(pid);
+ getProject.mockResolvedValue({ ...(await getProject(pid)), status: 'complete' });
+ const events = [];
+ const onRender = (event) => events.push(event);
+ musicVideoEvents.on('render', onRender);
+ const { jobId } = await renderMusicVideo(pid);
+ const proc = lastProc();
+ proc.emit('spawn');
+ expect(cancelRender(jobId)).toBe(true);
+ expect(killWithEscalation).toHaveBeenCalledWith(proc, expect.objectContaining({ label: 'music-video render' }));
+ proc.emit('error', new Error('kill EPERM'));
+ await tick();
+ expect(getRenderJobStatus(jobId).status).toBe('running');
+ expect(maintenance.status().blockers).toContainEqual(expect.objectContaining({ kind: 'music-video', resource: pid }));
+ await expect(renderMusicVideo(pid)).rejects.toMatchObject({ code: 'RENDER_IN_PROGRESS' });
+
+ proc.emit('close', 255, null);
+ await tick();
+ proc.emit('close', 255, null);
+ await tick();
+ musicVideoEvents.off('render', onRender);
+ expect(getRenderJobStatus(jobId)).toEqual({ status: 'canceled', error: 'Render cancelled' });
+ expect(events).toEqual([{ projectId: pid, jobId, status: 'canceled', error: 'Render cancelled' }]);
+ expect(broadcastSse.mock.calls.filter(([job, frame]) => job.id === jobId && ['canceled', 'error'].includes(frame.type)))
+ .toEqual([[expect.objectContaining({ id: jobId }), { type: 'canceled', error: 'Render cancelled' }]]);
+ expect(updateProject.mock.calls.filter(([id, patch]) => id === pid && patch.status === 'complete')).toEqual([[pid, settled('complete')]]);
+ expect(unlink).toHaveBeenCalledWith(expect.stringMatching(/music-video-.*\.mp4$/));
+ expect(maintenance.status().blockers).not.toContainEqual(expect.objectContaining({ resource: pid }));
+ expect(cancelRender(jobId)).toBe(false);
+ const again = await renderMusicVideo(pid);
+ expect(again.jobId).not.toBe(jobId);
+ lastProc().emit('error', new Error('spawn ENOENT'));
+ await tick();
+ });
+
+ it('retains zero-exit success when cancellation arrives just before close', async () => {
+ const pid = 'cancel-success';
+ prime(pid);
+ const { jobId } = await renderMusicVideo(pid);
+ lastProc().emit('spawn');
+ expect(cancelRender(jobId)).toBe(true);
+ lastProc().emit('close', 0, null);
+ await tick();
+ expect(getRenderJobStatus(jobId).status).toBe('complete');
+ expect(updateProject).toHaveBeenCalledWith(pid, expect.objectContaining(settled('complete', { renderHistoryId: jobId })));
+ });
+
it('a late stray error after a successful close does not clobber the completed job', async () => {
const pid = 'late-1';
prime(pid);
diff --git a/server/services/musicVideo/renderCancellation.js b/server/services/musicVideo/renderCancellation.js
new file mode 100644
index 0000000000..839044473d
--- /dev/null
+++ b/server/services/musicVideo/renderCancellation.js
@@ -0,0 +1,22 @@
+import { killWithEscalation } from '../../lib/killWithEscalation.js';
+
+/** Cancel capture or encoding without releasing ownership before settlement. */
+export function cancelMusicVideoRenderJob(job, { label }) {
+ if (!job) return false;
+ if (!job.process) {
+ if (job.status !== 'running' || !job.overlayAbort || job.overlayAbort.signal.aborted) return false;
+ job.cancelRequested = true;
+ job.overlayAbort.abort(new Error('Render cancelled'));
+ return true;
+ }
+ // ffmpeg can handle SIGTERM and exit nonzero with a null close-event signal.
+ // Record the request before kill, including a synchronous error/close race.
+ job.cancelRequested = true;
+ const proc = job.process;
+ killWithEscalation(proc, { label, stillRunning: () => job.process === proc });
+ return true;
+}
+
+/** Classify a nonzero close; a zero exit keeps the renderer's success policy. */
+export const isMusicVideoRenderCanceled = (job, signal) =>
+ job.cancelRequested === true || signal === 'SIGTERM' || signal === 'SIGKILL';