Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: drain successful shot cleanup after failures and cancellation

[._.]/ Adam Eivyef85d43117ac
diff --git a/server/services/musicVideo/render.js b/server/services/musicVideo/render.js
index 4e70510bef..82b29443f0 100644
--- a/server/services/musicVideo/render.js
+++ b/server/services/musicVideo/render.js
@@ -155,14 +155,19 @@ function findCoverageShortfalls(clips, { toleranceSec = COVERAGE_TOLERANCE_SEC }
return out;
}
+function releaseProjectRender(projectId) {
+ if (projectRenders.delete(projectId)) musicVideoEvents.emit('render-cleanup-idle', { projectId });
+}
+
+// Preparation also owns the slot before a public job ID exists.
+export const isProjectRenderActive = (projectId) => projectRenders.has(projectId);
+
/**
* The project's live final-render job on THIS instance, or null — a render
* still preparing (no job id yet) or running on another machine reports null.
* Read-only: lets a reloaded page re-attach to the render's progress stream
* without POSTing /render, which would START one when none is live (#9940).
*/
-export const isProjectRenderActive = (projectId) => projectRenders.has(projectId);
-
export function getActiveRenderJobId(projectId) {
const existing = projectRenders.get(projectId);
return existing && existing !== PENDING && jobs.has(existing) ? existing : null;
@@ -963,7 +968,7 @@ async function renderSeekedMode(projectId, project, handOff, renderer, options =
});
// `written`: the success path already committed the row inside its backup lease.
const finish = async (patch, { written = false } = {}) => {
- projectRenders.delete(projectId);
+ releaseProjectRender(projectId);
if (!written) await writeSettled(patch);
emitRenderSettled(projectId, job);
if (options.productionRunId) musicVideoEvents.emit('document-render', { projectId, runId: options.productionRunId, attemptId: options.productionRenderAttemptId, jobId, status: patch.status === 'complete' ? 'completed' : 'failed', error: job.lastError || null });
@@ -1065,7 +1070,7 @@ async function renderFromShotSegments(projectId, project, handOff, options) {
console.log(`🎬 Rendering music video [${jobId.slice(0, 8)}]: project=${projectId.slice(0, 8)} segments=${all.length} cached=${cached} duration=${plan.songSec.toFixed(2)}s`);
const { signal } = job.overlayAbort;
const finish = (status) => {
- projectRenders.delete(projectId);
+ releaseProjectRender(projectId);
emitRenderSettled(projectId, job);
closeJobAfterDelay(jobs, jobId);
options.maintenancePermit?.finish();
@@ -1210,7 +1215,7 @@ async function renderAdmittedMusicVideo(projectId, options, permit) {
log(`${canceled ? '🛑' : '❌'} Music-video render ${canceled ? 'cancelled' : 'failed'} [${jobId.slice(0, 8)}]: ${reason}`);
await unlink(outputPath).catch(() => {});
broadcastSse(job, { type: canceled ? 'canceled' : 'error', error: reason });
- projectRenders.delete(projectId);
+ releaseProjectRender(projectId);
// A cancel restores the pre-render status (so a cancelled re-render of a
// 'complete' project stays 'complete'); a real failure marks it 'failed'.
const targetStatus = canceled ? priorStatus : 'failed';
@@ -1280,7 +1285,7 @@ async function renderAdmittedMusicVideo(projectId, options, permit) {
job.lastError = reason;
console.error(`❌ Music-video render spawn error [${jobId.slice(0, 8)}]: ${reason}`);
broadcastSse(job, { type: 'error', error: reason });
- projectRenders.delete(projectId);
+ releaseProjectRender(projectId);
await updateProject(projectId, settledRender('failed', { renderError: reason.slice(0, 2000) })).catch((updateErr) => {
maintenance.markCurrentUnsettled();
console.error(`❌ Music-video render [${jobId.slice(0, 8)}] project ${projectId.slice(0, 8)} status→failed write failed: ${updateErr.message}`);
@@ -1378,7 +1383,7 @@ async function renderAdmittedMusicVideo(projectId, options, permit) {
});
emitRenderSettled(projectId, job);
} finally {
- projectRenders.delete(projectId);
+ releaseProjectRender(projectId);
closeJobAfterDelay(jobs, jobId);
permit.finish();
}
@@ -1391,7 +1396,7 @@ async function renderAdmittedMusicVideo(projectId, options, permit) {
} finally {
// Prep threw (or we never handed off to the job lifecycle) — release the
// reserved slot so a stale PENDING can't 409 every future render.
- if (!handedOff) { projectRenders.delete(projectId); permit.finish(); }
+ if (!handedOff) { releaseProjectRender(projectId); permit.finish(); }
}
}
diff --git a/server/services/musicVideo/shotSegments.js b/server/services/musicVideo/shotSegments.js
index 369b759af5..276281a0e2 100644
--- a/server/services/musicVideo/shotSegments.js
+++ b/server/services/musicVideo/shotSegments.js
@@ -213,7 +213,11 @@ export function renderSegment(plan, segment, { signal, onProgress } = {}) {
const controller = new AbortController();
const fresh = { controller, waiters: 0, listeners: new Set() };
fresh.promise = renderSegmentFiles(plan, segment, { signal: controller.signal, onProgress: (f) => fresh.listeners.forEach((fn) => fn(f)) })
- .finally(() => { if (inFlight.get(segment.path) === fresh) inFlight.delete(segment.path); });
+ .finally(() => {
+ if (inFlight.get(segment.path) === fresh) inFlight.delete(segment.path);
+ // Cancellation releases its caller before the encoder/browser settles.
+ cleanupAfterShot(plan.project.id).catch(err => console.warn(`⚠️ Music-video shot cache cleanup failed: ${err.message}`));
+ });
inFlight.set(segment.path, fresh);
entry = fresh;
}
@@ -607,7 +611,10 @@ export async function startShotRender(projectId, sceneId) {
if (!segments.length) {
throw new ServerError(`${nameShots(project, [sceneId])}: not in the cut yet. Give it a clip, or a still with a start and end time`, { status: 422, code: 'SHOT_NOT_IN_CUT' });
}
- if (segments.every(segmentRendered)) return { sceneId, state: 'rendered' };
+ if (segments.every(segmentRendered)) {
+ await cleanupAfterShot(projectId, sceneId).catch(err => console.warn(`⚠️ Music-video shot cache cleanup failed: ${err.message}`));
+ return { sceneId, state: 'rendered' };
+ }
if (shotJobs.has(key)) return { sceneId, state: 'rendering' };
assertRenderCleanupIdle(projectId);
const controller = new AbortController();
@@ -641,7 +648,7 @@ export async function startShotRender(projectId, sceneId) {
emitShotRender({ projectId, sceneId, state: canceled ? 'canceled' : 'failed', error: canceled ? null : err.message });
}).finally(async () => {
shotJobs.delete(key);
- if (rendered) await cleanupAfterShot(projectId, sceneId).catch(err => console.warn(`⚠️ Music-video shot cache cleanup failed: ${err.message}`));
+ await cleanupAfterShot(projectId, rendered ? sceneId : null).catch(err => console.warn(`⚠️ Music-video shot cache cleanup failed: ${err.message}`));
permit.finish();
});
return { sceneId, state: 'rendering' };
diff --git a/server/services/musicVideo/shotSegments.test.js b/server/services/musicVideo/shotSegments.test.js
index 8cb5bfdd0b..ed2ac3d140 100644
--- a/server/services/musicVideo/shotSegments.test.js
+++ b/server/services/musicVideo/shotSegments.test.js
@@ -30,7 +30,7 @@ vi.mock('./render.js', async (importOriginal) => {
vi.mock('./filmLookFinish.js', () => ({ applyFilmLookFinish: vi.fn(async () => true) }));
vi.mock('./projects.js', () => ({ getProject: vi.fn(async (id) => (id === fixture.project?.id ? fixture.project : null)), withProjectMediaMutation: async work => work({}) }));
-const { planShotSegments, renderMissingSegments, renderSegment, segmentConcurrency, stitchSegments, pruneSegments, listFilmLooks, deleteFilmLook, buildRenderPreview, segmentRendering, cleanupShotSegments, getSegmentCleanup, startShotRender, shotSegmentDir, assertSegmentCleanupIdle } = await import('./shotSegments.js');
+const { planShotSegments, renderMissingSegments, renderSegment, segmentConcurrency, stitchSegments, pruneSegments, listFilmLooks, deleteFilmLook, buildRenderPreview, segmentRendering, cleanupShotSegments, getSegmentCleanup, startShotRender, shotSegmentDir, assertSegmentCleanupIdle, cancelShotRender } = await import('./shotSegments.js');
const { buildMusicVideoFfmpegArgs } = await import('./render.js');
const { assertRenderCleanupIdle } = await import('./renderCleanupActivity.js');
const { musicVideoEvents } = await import('./events.js');
@@ -263,8 +263,8 @@ describe.skipIf(!hasFfmpeg)('shot segments', () => {
}
});
- it('drains superseded segments for every successfully overlapping shot once the batch is idle', async () => {
- const id = 'mv-overlapping-replacements';
+ it.each(['rendered', 'failed', 'canceled'])('drains successful replacements after the last overlapping shot is %s', async (outcome) => {
+ const id = `mv-overlapping-${outcome}`;
fixture.project = project({ id, filmLook: { preset: 'vhs', grain: 0.1 } });
const originalClips = fixture.clips;
const before = await planShotSegments(fixture.project);
@@ -275,20 +275,28 @@ describe.skipIf(!hasFfmpeg)('shot segments', () => {
const started = {};
const held = Object.fromEntries(['a', 'b'].map(scene => [scene, new Promise(resolve => { release[scene] = resolve; })]));
const ready = ['a', 'b'].map(scene => new Promise(resolve => { started[scene] = resolve; }));
- applyFilmLookFinish.mockImplementation(async ({ label }) => {
+ applyFilmLookFinish.mockImplementation(async ({ label, signal }) => {
const scene = ['a', 'b'].find(key => label.endsWith(`shot ${key}`));
started[scene]();
await held[scene];
+ if (scene === 'b' && outcome === 'failed') throw new Error('Synthetic look failure');
+ if (signal.aborted) throw Object.assign(new Error('Synthetic cancellation'), { code: 'CANCELED' });
return true;
});
let onShot;
+ let settleLast;
+ const lastDone = new Promise(resolve => { settleLast = resolve; });
const firstDone = new Promise(resolve => {
- onShot = event => { if (event.projectId === id && event.sceneId === 'a' && event.state === 'rendered') resolve(); };
+ onShot = event => {
+ if (event.projectId !== id) return;
+ if (event.sceneId === 'a' && event.state === 'rendered') resolve();
+ if (event.sceneId === 'b' && ['rendered', 'failed', 'canceled'].includes(event.state)) settleLast(event.state);
+ };
});
let onCleanup;
const cleaned = new Promise(resolve => {
let count = 0;
- onCleanup = event => { if (event.projectId === id && ++count === 2) resolve(); };
+ onCleanup = event => { if (event.projectId === id && ++count === (outcome === 'rendered' ? 2 : 1)) resolve(); };
});
musicVideoEvents.on('shot-render', onShot);
musicVideoEvents.on('render-cleanup', onCleanup);
@@ -299,13 +307,15 @@ describe.skipIf(!hasFfmpeg)('shot segments', () => {
release.a();
await firstDone;
expect(existsSync(before.segments[0].path)).toBe(true);
+ if (outcome === 'canceled') expect(cancelShotRender(id, 'b')).toBe(true);
release.b();
+ expect(await lastDone).toBe(outcome);
await cleaned;
await vi.waitFor(() => { assertSegmentCleanupIdle(id); assertRenderCleanupIdle(id); });
- for (const index of [0, 1]) {
- expect(existsSync(before.segments[index].path)).toBe(false);
- expect(existsSync(after.segments[index].path)).toBe(true);
- }
+ expect(existsSync(before.segments[0].path)).toBe(false);
+ expect(existsSync(after.segments[0].path)).toBe(true);
+ expect(existsSync(before.segments[1].path)).toBe(outcome !== 'rendered');
+ expect(existsSync(after.segments[1].path)).toBe(outcome === 'rendered');
} finally {
release.a(); release.b();
await vi.waitFor(() => { assertSegmentCleanupIdle(id); assertRenderCleanupIdle(id); });