Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: drain shot cleanup after concurrent renders settle

[._.]/ Adam Eivy011096f9cba4
diff --git a/server/services/musicVideo/renderCleanupActivity.js b/server/services/musicVideo/renderCleanupActivity.js
index 41caa75f32..6eb1e088f6 100644
--- a/server/services/musicVideo/renderCleanupActivity.js
+++ b/server/services/musicVideo/renderCleanupActivity.js
@@ -1,4 +1,5 @@
/** Synchronous admission shared by derived-render cleanup and its consumers. */
+import { musicVideoEvents } from './events.js';
import { ServerError } from '../../lib/errorHandler.js';
const cleaning = new Set();
@@ -6,9 +7,9 @@ export function assertRenderCleanupIdle(projectId) {
if (cleaning.has(projectId)) throw new ServerError('Render cleanup is in progress. Try again when it finishes.', { status: 409, code: 'RENDER_CLEANUP_ACTIVE' });
}
-export async function withRenderCleanupReservation(projectId, work) {
+export async function withRenderCleanupReservation(projectId, work, { notify = true } = {}) {
assertRenderCleanupIdle(projectId);
cleaning.add(projectId);
- try { return await work(); } finally { cleaning.delete(projectId); }
+ try { return await work(); } finally { cleaning.delete(projectId); if (notify) musicVideoEvents.emit('render-cleanup-idle', { projectId }); }
}
diff --git a/server/services/musicVideo/shotSegments.js b/server/services/musicVideo/shotSegments.js
index a0ad705e21..369b759af5 100644
--- a/server/services/musicVideo/shotSegments.js
+++ b/server/services/musicVideo/shotSegments.js
@@ -49,6 +49,8 @@ export const SHOT_SEGMENT_DIR = 'music-video-segments';
// Bump when a segment's encode changes in a way its key cannot see.
export const SHOT_SEGMENT_VERSION = 1;
const segmentOwnerWrites = createKeyCachedQueue();
+const pendingShotCleanup = new Map();
+const shotCleanupWrites = createKeyCachedQueue();
const segmentOwnersFile = dir => join(dir, 'segments.json');
async function rememberSegmentOwner(plan, segment) {
@@ -470,10 +472,14 @@ export async function pruneSegments(plan, { reserved = false, current = false, .
return reserved ? prune() : withRenderCleanupReservation(plan.project.id, prune);
}
-export function assertSegmentCleanupIdle(projectId) {
+function segmentCleanupBusy(projectId) {
const dir = shotSegmentDir(projectId);
- if ([...inFlight.keys(), ...previewBuilds.keys()].some(path => path.startsWith(`${dir}${sep}`))
- || [...shotJobs.keys()].some(key => key.startsWith(`${projectId}:`))) {
+ return [...inFlight.keys(), ...previewBuilds.keys()].some(path => path.startsWith(`${dir}${sep}`))
+ || [...shotJobs.keys()].some(key => key.startsWith(`${projectId}:`));
+}
+
+export function assertSegmentCleanupIdle(projectId) {
+ if (segmentCleanupBusy(projectId)) {
throw new ServerError('Wait for shot renders and preview builds to finish before cleanup.', { status: 409, code: 'SHOT_RENDER_ACTIVE' });
}
}
@@ -508,7 +514,10 @@ export async function cleanupShotSegments(projectId, { unusedLooks = false, rese
}
/** Remove a project's whole segment cache (the project was deleted). */
-export const removeProjectSegments = (projectId) => rm(shotSegmentDir(projectId), { recursive: true, force: true });
+export const removeProjectSegments = (projectId) => {
+ pendingShotCleanup.delete(projectId);
+ return rm(shotSegmentDir(projectId), { recursive: true, force: true });
+};
/** Boot sweep: partial files from renders a restart interrupted. */
export async function sweepPartialSegments() {
@@ -807,7 +816,10 @@ export async function buildRenderPreview(projectId, { light = false } = {}) {
if (existsSync(path)) return result;
let build = previewBuilds.get(path);
if (!build) {
- build = writePreview(plan, lead, path, { light }).finally(() => previewBuilds.delete(path));
+ build = writePreview(plan, lead, path, { light }).finally(async () => {
+ previewBuilds.delete(path);
+ await cleanupAfterShot(projectId).catch(err => console.warn(`⚠️ Music-video shot cache cleanup failed: ${err.message}`));
+ });
previewBuilds.set(path, build);
}
await build;
@@ -837,17 +849,40 @@ async function writePreview(plan, lead, path, { light }) {
}
}
-async function cleanupAfterShot(projectId, sceneId) {
- const { isProjectRenderActive } = await import('./render.js');
- if (isProjectRenderActive(projectId)) return;
- assertSegmentCleanupIdle(projectId);
- const { withProjectMediaMutation } = await import('./projects.js');
- await withProjectMediaMutation(async () => {
- const { plan } = await planForLooks(projectId);
- if (!plan || plan.error || isProjectRenderActive(projectId)) return;
- const current = plan.segments.filter(segment => segment.sceneId === sceneId);
- if (!current.length || !current.every(segmentRendered)) return;
- assertSegmentCleanupIdle(projectId);
- await pruneSegments(plan, { sceneId });
+async function cleanupAfterShot(projectId, sceneId = null) {
+ if (sceneId) {
+ const pending = pendingShotCleanup.get(projectId) || new Set();
+ pending.add(sceneId);
+ pendingShotCleanup.set(projectId, pending);
+ }
+ if (!pendingShotCleanup.get(projectId)?.size) return;
+ return shotCleanupWrites(projectId, async () => {
+ const pending = pendingShotCleanup.get(projectId);
+ if (!pending?.size) return;
+ const { isProjectRenderActive } = await import('./render.js');
+ if (isProjectRenderActive(projectId) || segmentCleanupBusy(projectId)) return;
+ const { withProjectMediaMutation } = await import('./projects.js');
+ await withProjectMediaMutation(async () => {
+ if (isProjectRenderActive(projectId) || segmentCleanupBusy(projectId)) return;
+ await withRenderCleanupReservation(projectId, async () => {
+ const { plan } = await planForLooks(projectId);
+ if (!plan || plan.error) return;
+ for (const id of [...pending]) {
+ const current = plan.segments.filter(segment => segment.sceneId === id);
+ // A later edit supersedes the completed render: wait for its new success.
+ if (current.length && current.every(segmentRendered)) await pruneSegments(plan, { sceneId: id, reserved: true });
+ pending.delete(id);
+ }
+ if (!pending.size) pendingShotCleanup.delete(projectId);
+ }, { notify: false });
+ });
});
}
+
+// A deferred batch also resumes after preview/full-render readers or another
+// cleanup release their ownership. Never poll or drop another completed shot.
+const resumeShotCleanup = ({ projectId }) => {
+ cleanupAfterShot(projectId).catch(err => console.warn(`⚠️ Music-video shot cache cleanup failed: ${err.message}`));
+};
+musicVideoEvents.on('render', resumeShotCleanup);
+musicVideoEvents.on('render-cleanup-idle', resumeShotCleanup);
diff --git a/server/services/musicVideo/shotSegments.test.js b/server/services/musicVideo/shotSegments.test.js
index e643cfb5e3..8cb5bfdd0b 100644
--- a/server/services/musicVideo/shotSegments.test.js
+++ b/server/services/musicVideo/shotSegments.test.js
@@ -30,8 +30,9 @@ vi.mock('./render.js', async (importOriginal) => {
vi.mock('./filmLookFinish.js', () => ({ applyFilmLookFinish: vi.fn(async () => true) }));
vi.mock('./projects.js', () => ({ getProject: vi.fn(async (id) => (id === fixture.project?.id ? fixture.project : null)), withProjectMediaMutation: async work => work({}) }));
-const { planShotSegments, renderMissingSegments, renderSegment, segmentConcurrency, stitchSegments, pruneSegments, listFilmLooks, deleteFilmLook, buildRenderPreview, segmentRendering, cleanupShotSegments, getSegmentCleanup, startShotRender, shotSegmentDir } = await import('./shotSegments.js');
+const { planShotSegments, renderMissingSegments, renderSegment, segmentConcurrency, stitchSegments, pruneSegments, listFilmLooks, deleteFilmLook, buildRenderPreview, segmentRendering, cleanupShotSegments, getSegmentCleanup, startShotRender, shotSegmentDir, assertSegmentCleanupIdle } = await import('./shotSegments.js');
const { buildMusicVideoFfmpegArgs } = await import('./render.js');
+const { assertRenderCleanupIdle } = await import('./renderCleanupActivity.js');
const { musicVideoEvents } = await import('./events.js');
const { applyFilmLookFinish } = await import('./filmLookFinish.js');
@@ -262,6 +263,58 @@ describe.skipIf(!hasFfmpeg)('shot segments', () => {
}
});
+ it('drains superseded segments for every successfully overlapping shot once the batch is idle', async () => {
+ const id = 'mv-overlapping-replacements';
+ fixture.project = project({ id, filmLook: { preset: 'vhs', grain: 0.1 } });
+ const originalClips = fixture.clips;
+ const before = await planShotSegments(fixture.project);
+ await renderMissingSegments(before, { concurrency: 2 });
+ fixture.clips = originalClips.map(clip => ['a', 'b'].includes(clip.sceneId) ? { ...clip, inSec: 0.1, outSec: 1.1 } : clip);
+ const after = await planShotSegments(fixture.project);
+ const release = {};
+ const started = {};
+ const held = Object.fromEntries(['a', 'b'].map(scene => [scene, new Promise(resolve => { release[scene] = resolve; })]));
+ const ready = ['a', 'b'].map(scene => new Promise(resolve => { started[scene] = resolve; }));
+ applyFilmLookFinish.mockImplementation(async ({ label }) => {
+ const scene = ['a', 'b'].find(key => label.endsWith(`shot ${key}`));
+ started[scene]();
+ await held[scene];
+ return true;
+ });
+ let onShot;
+ const firstDone = new Promise(resolve => {
+ onShot = event => { if (event.projectId === id && event.sceneId === 'a' && event.state === 'rendered') resolve(); };
+ });
+ let onCleanup;
+ const cleaned = new Promise(resolve => {
+ let count = 0;
+ onCleanup = event => { if (event.projectId === id && ++count === 2) resolve(); };
+ });
+ musicVideoEvents.on('shot-render', onShot);
+ musicVideoEvents.on('render-cleanup', onCleanup);
+ try {
+ await startShotRender(id, 'a');
+ await startShotRender(id, 'b');
+ await Promise.all(ready);
+ release.a();
+ await firstDone;
+ expect(existsSync(before.segments[0].path)).toBe(true);
+ release.b();
+ await cleaned;
+ await vi.waitFor(() => { assertSegmentCleanupIdle(id); assertRenderCleanupIdle(id); });
+ for (const index of [0, 1]) {
+ expect(existsSync(before.segments[index].path)).toBe(false);
+ expect(existsSync(after.segments[index].path)).toBe(true);
+ }
+ } finally {
+ release.a(); release.b();
+ await vi.waitFor(() => { assertSegmentCleanupIdle(id); assertRenderCleanupIdle(id); });
+ musicVideoEvents.off('shot-render', onShot);
+ musicVideoEvents.off('render-cleanup', onCleanup);
+ fixture.clips = originalClips;
+ }
+ }, 60000);
+
it('tries a shot whose film look failed again before failing the render', async () => {
fixture.project = project({ id: 'mv-segments-retry-test', filmLook: { preset: 'vhs', grain: 0.7 } });
const plan = await planShotSegments(fixture.project);