Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: preserve canceled Music Video renders after ffmpeg exits (#10702)
Adam Eivy432f8b76440e
diff --git a/server/services/README.md b/server/services/README.mdindex 2f9e67be21..4af5e9edb0 100644--- a/server/services/README.md+++ b/server/services/README.md@@ -11,6 +11,7 @@ image-generation paths below are useful entry points when tracing render jobs.| `databaseMaintenanceProducers.js` | Internal one-use coordinator stage: retain producer identity, stop this install's PM2 CoS/server, and verify fresh readback while admission remains fenced. || `databaseMaintenanceQuiescence.js` | Internal coordinator stage after producer shutdown: terminate verified running detached-writer groups, prove every registry record quiescent from process-table evidence, archive it with the operation, and refuse pending, legacy, orphaned, ambiguous or unregistered writers. || `imageGen/prepareParams.js` | Shared image request preparation and the canonical local image model selector. Use `selectLocalImageModelFromSettings()` for model identity and `resolveLocalImageModel()` when the caller must also enforce local runtime, hardware, and edit-image requirements. |+| `musicVideo/renderCancellation.js` | Shared capture-abort and ffmpeg kill selection for full/excerpt renders; records explicit cancellation before signaling and classifies nonzero closes without releasing ownership. || `musicVideo/sharingCopy.js` | Source-hashed private sharing MP4s: queued 720p H.264/AAC export, strict decimal 100 MB cap, supervised cancellation, verified download and owned-file cleanup. || `mediaJobQueue/index.js` | Owns admission, lanes, durability and terminal effects; `videoHolds.js` returns local-video cohort preparation results and owns circuit breakers, `recoverySettlement.js` names the transient recovery evidence and the maintenance-permit release decision, while `videoGen/modelSelection.js` owns model-selection policy. || `imageGen/index.js` | Mode-aware image backend dispatcher and shared image-generation helpers. Direct queue consumers still resolve local model identity before enqueueing. |diff --git a/server/services/musicVideo/excerptRender.js b/server/services/musicVideo/excerptRender.jsindex ce4ddd9087..c9eabb42db 100644--- a/server/services/musicVideo/excerptRender.js+++ b/server/services/musicVideo/excerptRender.js@@ -28,7 +28,7 @@ import { spawn } from '../../lib/childProcess.js';import { safeChildProcessOptions } from '../../lib/processEnv.js';import { broadcastSse, attachSseClient as attachSse, closeJobAfterDelay } from '../../lib/sseUtils.js';import { attachFfmpegRenderGuard } from '../../lib/ffmpegRenderGuard.js';-import { killWithEscalation } from '../../lib/killWithEscalation.js';+import { cancelMusicVideoRenderJob, isMusicVideoRenderCanceled } from './renderCancellation.js';import { safeUnder } from '../../lib/ffmpeg.js';import { encodeFileContactSheetAtTimes } from '../htmlComposition/encode.js';import { getProject, listProjects, mutateProjectRecord } from './projects.js';@@ -54,21 +54,7 @@ const PENDING = Symbol('mv-excerpt-render-pending');export const attachExcerptRenderSseClient = (jobId, res) => attachSse(jobs, jobId, res);export function cancelExcerptRender(jobId) {- const job = jobs.get(jobId);- if (!job) return false;- if (!job.process) {- if (job.status !== 'running' || !job.overlayAbort || job.overlayAbort.signal.aborted) return false;- job.overlayAbort.abort(new Error('Excerpt render cancelled'));- return true;- }- // ffmpeg often intercepts SIGTERM and exits itself (nonzero code, `signal:- // null` on the child's 'close' event) rather than dying FROM the signal —- // `onClose` below can't tell that apart from a genuine encode failure by- // signal alone, so record that a cancel was actually requested.- job.cancelRequested = true;- const proc = job.process;- killWithEscalation(proc, { label: 'music-video excerpt render', stillRunning: () => job.process === proc });- return true;+ return cancelMusicVideoRenderJob(jobs.get(jobId), { label: 'music-video excerpt render' });}const round3 = (n) => Math.round(n * 1000) / 1000;@@ -431,7 +417,7 @@ export async function startExcerptRender(projectId, { startSec, endSec, aspect =onClose: async (code, signal) => {job.process = null;if (code !== 0) {- const canceled = job.cancelRequested || signal === 'SIGTERM' || signal === 'SIGKILL';+ const canceled = isMusicVideoRenderCanceled(job, signal);job.status = canceled ? 'canceled' : 'error';const reason = canceled ? 'Render cancelled' : signal ? `Killed by signal ${signal}` : `ffmpeg exit ${code}`;job.lastError = reason;diff --git a/server/services/musicVideo/excerptRender.test.js b/server/services/musicVideo/excerptRender.test.jsnew file mode 100644index 0000000000..6d8889f349--- /dev/null+++ b/server/services/musicVideo/excerptRender.test.js@@ -0,0 +1,144 @@+import { describe, it, expect, vi, beforeEach } from 'vitest';++const h = vi.hoisted(() => {+ const emitter = () => {+ const listeners = {};+ return {+ on(event, fn) { (listeners[event] ||= []).push(fn); return this; },+ emit(event, ...args) { for (const fn of listeners[event] || []) fn(...args); },+ };+ };+ const procs = [];+ const spawn = () => {+ const proc = { ...emitter(), stderr: emitter(), kill: vi.fn() };+ procs.push(proc);+ return proc;+ };+ const store = new Map();+ const overlays = { resolve: null };+ return { procs, spawn, store, overlays };+});++vi.mock('../../lib/childProcess.js', () => ({ spawn: h.spawn, execFile: vi.fn(), exec: vi.fn() }));+vi.mock('../../lib/processEnv.js', () => ({ safeChildProcessOptions: (options) => ({ ...options, env: {} }) }));+vi.mock('fs/promises', () => ({ unlink: vi.fn(async () => {}) }));+vi.mock('../../lib/fileUtils.js', () => ({+ ensureDir: vi.fn(async () => {}),+ PATHS: { videos: '/test/videos', videoThumbnails: '/test/thumbnails' },+}));+vi.mock('../../lib/sseUtils.js', () => ({ broadcastSse: vi.fn(), attachSseClient: vi.fn(), closeJobAfterDelay: vi.fn() }));+vi.mock('../../lib/killWithEscalation.js', () => ({ killWithEscalation: vi.fn() }));+vi.mock('../instanceIdentity.js', () => ({ ensureInstanceId: vi.fn(async () => 'test-instance') }));+vi.mock('../htmlComposition/encode.js', () => ({ encodeFileContactSheetAtTimes: vi.fn(async () => {}) }));+vi.mock('./projects.js', () => ({+ getProject: vi.fn(async (id) => h.store.get(id)),+ listProjects: vi.fn(async () => []),+ mutateProjectRecord: vi.fn(async (id, transform) => {+ const outcome = transform(h.store.get(id));+ h.store.set(id, outcome.project);+ return outcome;+ }),+}));+vi.mock('./render.js', () => ({+ planMusicVideoRender: vi.fn(async (project) => ({ ffmpeg: 'ffmpeg', audioPath: '/test/song.wav', clips: [], composed: !!project.composition, audioDurationSec: 4, soundBed: null })),+ buildMusicVideoFfmpegArgs: vi.fn(() => ({ args: [], totalDuration: 4, canonW: 64, canonH: 64, fps: 24, sections: [] })),+ excerptBoundaryTimes: vi.fn(() => [0]),+ isLocalRenderMark: vi.fn(() => true),+ assertCurrentClipDependencies: vi.fn(),+ resolveMasterAudioPath: vi.fn(),+ resolveSoundBedPath: vi.fn(),+}));+vi.mock('./compositionRender.js', () => ({+ removeCompositionScratch: vi.fn(async () => {}),+ renderTypographyOverlays: vi.fn(({ signal }) => new Promise((resolve, reject) => {+ h.overlays.resolve = resolve;+ signal.addEventListener('abort', () => reject(signal.reason), { once: true });+ })),+}));++import { startExcerptRender, cancelExcerptRender } from './excerptRender.js';+import { unlink } from 'fs/promises';+import { broadcastSse } from '../../lib/sseUtils.js';+import { removeCompositionScratch } from './compositionRender.js';+import { musicVideoEvents } from './events.js';++const prime = (id, composition = null) => h.store.set(id, { id, name: 'Test', scenes: [], excerpts: [], composition });+const lastProc = () => h.procs.at(-1);+const excerpt = (id, excerptId) => h.store.get(id).excerpts.find((record) => record.id === excerptId);+const terminalFrames = (jobId) => broadcastSse.mock.calls.filter(([job, frame]) => job.id === jobId && ['canceled', 'error', 'complete'].includes(frame.type)).map(([, frame]) => frame);++beforeEach(() => {+ vi.clearAllMocks();+ h.procs.length = 0;+ h.store.clear();+});++describe('excerpt render cancellation', () => {+ it('keeps a canceled encode owned until close, removes partial output, and settles once', async () => {+ const id = 'excerpt-cancel';+ prime(id);+ const events = [];+ const onRender = (event) => events.push(event);+ musicVideoEvents.on('excerpt-render', onRender);+ const { jobId, excerptId } = await startExcerptRender(id, { startSec: 0, endSec: 2 });+ const partialFilename = excerpt(id, excerptId).partialFilename;+ const proc = lastProc();+ proc.emit('spawn');+ expect(cancelExcerptRender(jobId)).toBe(true);+ proc.emit('error', new Error('kill EPERM'));+ expect(excerpt(id, excerptId).status).toBe('rendering');+ await expect(startExcerptRender(id, { startSec: 0, endSec: 2 })).rejects.toMatchObject({ code: 'EXCERPT_RENDER_IN_PROGRESS' });+ proc.emit('close', 255, null);+ await vi.waitFor(() => expect(terminalFrames(jobId)).toHaveLength(1));+ proc.emit('close', 255, null);+ musicVideoEvents.off('excerpt-render', onRender);+ expect(terminalFrames(jobId)).toEqual([{ type: 'canceled', error: 'Render cancelled' }]);+ expect(excerpt(id, excerptId)).toMatchObject({ status: 'canceled', filename: null, partialFilename: null, jobId: null, renderingOn: null, error: null });+ expect(unlink.mock.calls).toEqual([[`/test/videos/${partialFilename}`]]);+ expect(events).toEqual([{ projectId: id, excerptId, status: 'canceled' }]);+ expect(cancelExcerptRender(jobId)).toBe(false);+ const again = await startExcerptRender(id, { startSec: 0, endSec: 2 });+ lastProc().emit('close', 1, null);+ await vi.waitFor(() => expect(terminalFrames(again.jobId)).toHaveLength(1));+ });++ it.each([{ code: 255, status: 'error' }, { code: 0, status: 'complete' }])('retains $status for an uncanceled close($code, null)', async ({ code, status }) => {+ const id = `excerpt-close-${code}`;+ prime(id);+ const { jobId, excerptId } = await startExcerptRender(id, { startSec: 0, endSec: 2 });+ lastProc().emit('spawn');+ lastProc().emit('close', code, null);+ await vi.waitFor(() => expect(terminalFrames(jobId)).toHaveLength(1));+ expect(excerpt(id, excerptId).status).toBe(status);+ expect(terminalFrames(jobId)[0].type).toBe(status);+ });++ it('retains zero-exit success after a cancellation request', async () => {+ const id = 'excerpt-cancel-success';+ prime(id);+ const { jobId, excerptId } = await startExcerptRender(id, { startSec: 0, endSec: 2 });+ lastProc().emit('spawn');+ expect(cancelExcerptRender(jobId)).toBe(true);+ lastProc().emit('close', 0, null);+ await vi.waitFor(() => expect(terminalFrames(jobId)).toHaveLength(1));+ expect(excerpt(id, excerptId).status).toBe('complete');+ expect(terminalFrames(jobId)[0].type).toBe('complete');+ });++ it('aborts overlay capture without spawning and releases scratch and the excerpt slot', async () => {+ const id = 'excerpt-capture';+ prime(id, { mode: 'composed', textCues: [{ id: 'cue', text: 'Test', startSec: 0, endSec: 2 }], style: {} });+ const { jobId, excerptId } = await startExcerptRender(id, { startSec: 0, endSec: 2 });+ expect(h.procs).toHaveLength(0);+ expect(cancelExcerptRender(jobId)).toBe(true);+ expect(cancelExcerptRender(jobId)).toBe(false);+ await vi.waitFor(() => expect(terminalFrames(jobId)).toHaveLength(1));+ expect(excerpt(id, excerptId).status).toBe('canceled');+ expect(removeCompositionScratch).toHaveBeenCalledExactlyOnceWith(jobId);+ expect(h.procs).toHaveLength(0);+ const again = await startExcerptRender(id, { startSec: 0, endSec: 2 });+ expect(again.jobId).not.toBe(jobId);+ cancelExcerptRender(again.jobId);+ await vi.waitFor(() => expect(terminalFrames(again.jobId)).toHaveLength(1));+ });+});diff --git a/server/services/musicVideo/render.js b/server/services/musicVideo/render.jsindex 7d7ac63e20..b4e4799cba 100644--- a/server/services/musicVideo/render.js+++ b/server/services/musicVideo/render.js@@ -35,7 +35,7 @@ import { ServerError } from '../../lib/errorHandler.js';import { broadcastSse, attachSseClient as attachSse, closeJobAfterDelay } from '../../lib/sseUtils.js';import { findFfmpeg, safeUnder, generateThumbnail, probeVideoDuration, probeVideoGeometry } from '../../lib/ffmpeg.js';import { safeChildProcessOptions } from '../../lib/processEnv.js';-import { killWithEscalation } from '../../lib/killWithEscalation.js';+import { cancelMusicVideoRenderJob, isMusicVideoRenderCanceled } from './renderCancellation.js';import { attachFfmpegRenderGuard } from '../../lib/ffmpegRenderGuard.js';import { loadHistory, mutateVideoHistory } from '../videoGen/local.js';import { getTrack } from '../tracks/index.js';@@ -157,18 +157,7 @@ export function getRenderJobStatus(jobId) {}export function cancelRender(jobId) {- const job = jobs.get(jobId);- if (!job) return false;- // A composed render spends its first phase capturing the typography overlay- // (no ffmpeg yet); abort that capture instead.- if (!job.process) {- if (job.status !== 'running' || !job.overlayAbort || job.overlayAbort.signal.aborted) return false;- job.overlayAbort.abort(new Error('Render cancelled'));- return true;- }- const proc = job.process;- killWithEscalation(proc, { label: 'music-video render', stillRunning: () => job.process === proc });- return true;+ return cancelMusicVideoRenderJob(jobs.get(jobId), { label: 'music-video render' });}// Resolve the project's source audio to a verified path under data/music/.@@ -948,7 +937,7 @@ async function renderAdmittedMusicVideo(projectId, options, permit) {onClose: async (code, signal) => {job.process = null;if (code !== 0) {- const canceled = signal === 'SIGTERM' || signal === 'SIGKILL';+ const canceled = isMusicVideoRenderCanceled(job, signal);job.status = canceled ? 'canceled' : 'error';const reason = canceled ? 'Render cancelled' : signal ? `Killed by signal ${signal}` : `ffmpeg exit ${code}`;job.lastError = reason;diff --git a/server/services/musicVideo/render.terminal.test.js b/server/services/musicVideo/render.terminal.test.jsindex 39d3eb33cb..a3b366acb7 100644--- a/server/services/musicVideo/render.terminal.test.js+++ b/server/services/musicVideo/render.terminal.test.js@@ -85,6 +85,9 @@ import { getTrack } from '../tracks/index.js';import { getProject, listProjects, updateProject, mutateProjectRecord } from './projects.js';import { unlink } from 'fs/promises';import { musicVideoEvents } from './events.js';+import { broadcastSse } from '../../lib/sseUtils.js';+import { killWithEscalation } from '../../lib/killWithEscalation.js';+import { maintenance } from '../../lib/maintenanceAdmission.js';const tick = () => new Promise((r) => setTimeout(r, 0));const lastProc = () => h.procs[h.procs.length - 1];@@ -181,6 +184,55 @@ describe('renderMusicVideo terminal handling (#2386)', () => {expect(getRenderJobStatus(jobId).status).toBe('error');});+ it('preserves a completed project when canceled ffmpeg exits 255 without a signal', async () => {+ const pid = 'cancel-close';+ prime(pid);+ getProject.mockResolvedValue({ ...(await getProject(pid)), status: 'complete' });+ const events = [];+ const onRender = (event) => events.push(event);+ musicVideoEvents.on('render', onRender);+ const { jobId } = await renderMusicVideo(pid);+ const proc = lastProc();+ proc.emit('spawn');+ expect(cancelRender(jobId)).toBe(true);+ expect(killWithEscalation).toHaveBeenCalledWith(proc, expect.objectContaining({ label: 'music-video render' }));+ proc.emit('error', new Error('kill EPERM'));+ await tick();+ expect(getRenderJobStatus(jobId).status).toBe('running');+ expect(maintenance.status().blockers).toContainEqual(expect.objectContaining({ kind: 'music-video', resource: pid }));+ await expect(renderMusicVideo(pid)).rejects.toMatchObject({ code: 'RENDER_IN_PROGRESS' });++ proc.emit('close', 255, null);+ await tick();+ proc.emit('close', 255, null);+ await tick();+ musicVideoEvents.off('render', onRender);+ expect(getRenderJobStatus(jobId)).toEqual({ status: 'canceled', error: 'Render cancelled' });+ expect(events).toEqual([{ projectId: pid, jobId, status: 'canceled', error: 'Render cancelled' }]);+ expect(broadcastSse.mock.calls.filter(([job, frame]) => job.id === jobId && ['canceled', 'error'].includes(frame.type)))+ .toEqual([[expect.objectContaining({ id: jobId }), { type: 'canceled', error: 'Render cancelled' }]]);+ expect(updateProject.mock.calls.filter(([id, patch]) => id === pid && patch.status === 'complete')).toEqual([[pid, settled('complete')]]);+ expect(unlink).toHaveBeenCalledWith(expect.stringMatching(/music-video-.*\.mp4$/));+ expect(maintenance.status().blockers).not.toContainEqual(expect.objectContaining({ resource: pid }));+ expect(cancelRender(jobId)).toBe(false);+ const again = await renderMusicVideo(pid);+ expect(again.jobId).not.toBe(jobId);+ lastProc().emit('error', new Error('spawn ENOENT'));+ await tick();+ });++ it('retains zero-exit success when cancellation arrives just before close', async () => {+ const pid = 'cancel-success';+ prime(pid);+ const { jobId } = await renderMusicVideo(pid);+ lastProc().emit('spawn');+ expect(cancelRender(jobId)).toBe(true);+ lastProc().emit('close', 0, null);+ await tick();+ expect(getRenderJobStatus(jobId).status).toBe('complete');+ expect(updateProject).toHaveBeenCalledWith(pid, expect.objectContaining(settled('complete', { renderHistoryId: jobId })));+ });+it('a late stray error after a successful close does not clobber the completed job', async () => {const pid = 'late-1';prime(pid);diff --git a/server/services/musicVideo/renderCancellation.js b/server/services/musicVideo/renderCancellation.jsnew file mode 100644index 0000000000..839044473d--- /dev/null+++ b/server/services/musicVideo/renderCancellation.js@@ -0,0 +1,22 @@+import { killWithEscalation } from '../../lib/killWithEscalation.js';++/** Cancel capture or encoding without releasing ownership before settlement. */+export function cancelMusicVideoRenderJob(job, { label }) {+ if (!job) return false;+ if (!job.process) {+ if (job.status !== 'running' || !job.overlayAbort || job.overlayAbort.signal.aborted) return false;+ job.cancelRequested = true;+ job.overlayAbort.abort(new Error('Render cancelled'));+ return true;+ }+ // ffmpeg can handle SIGTERM and exit nonzero with a null close-event signal.+ // Record the request before kill, including a synchronous error/close race.+ job.cancelRequested = true;+ const proc = job.process;+ killWithEscalation(proc, { label, stillRunning: () => job.process === proc });+ return true;+}++/** Classify a nonzero close; a zero exit keeps the renderer's success policy. */+export const isMusicVideoRenderCanceled = (job, signal) =>+ job.cancelRequested === true || signal === 'SIGTERM' || signal === 'SIGKILL';