Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: unwind local video startup failures before the render child owns the job (#12395)

[._.]/ Adam Eivy2a37a4f351fb
diff --git a/server/services/videoGen/generateVideo.js b/server/services/videoGen/generateVideo.js
index a6163cb82c..1fca716700 100644
--- a/server/services/videoGen/generateVideo.js
+++ b/server/services/videoGen/generateVideo.js
@@ -656,22 +656,14 @@ async function renderLocalVideo(preparation, { pythonPath, prompt, negativePromp
...sentRecord,
...(hidden ? { hidden: true } : {}),
};
- // Each render gets one private preview directory. Runners overwrite the
- // fixed `preview.png` path atomically, so a long render cannot accumulate
- // one PNG per step and a stale preview can never cross job boundaries.
- const stepwiseDir = await mkdtemp(join(tmpdir(), 'portos-video-stepwise-'));
- const job = { ...meta, clients: [], status: 'running' };
- videoJobState.jobs.set(jobId, job);
-
- // buildArgs now throws synchronously on multi-keyframe pixel-budget
- // overflow and a few other validation paths — without this guard the
- // job would stay "running" forever in the shared jobs map and the resized
- // temp files would leak (the spawn close-handler that normally cleans
- // them up never runs because we never spawned). Mirror the cleanup
- // logic of the spawn-error handler so failure modes converge.
- let bin, args;
- try {
- ({ bin, args } = buildArgs({
+ await startPreparedLocalVideo({
+ preparation,
+ jobId,
+ meta,
+ batch,
+ cleanupTempFiles,
+ displaySleep,
+ buildParams: {
pythonPath,
modelId,
model: loraCapableModel,
@@ -704,7 +696,6 @@ async function renderLocalVideo(preparation, { pythonPath, prompt, negativePromp
textEncoderRepo: actualTextEncoderRepo,
textEncoder: resolvedTextEncoder,
outputPath,
- previewDir: stepwiseDir,
loras: resolvedLoras,
icReferencePaths: resolvedIcReferencePaths,
icLoraWeightPath,
@@ -716,74 +707,96 @@ async function renderLocalVideo(preparation, { pythonPath, prompt, negativePromp
streamingMode,
ffmpegPath: ffmpeg,
ffprobePath: ffprobe,
- }));
+ },
+ watchParams: {
+ // Keeps this render's ETA samples in its own cost bucket — a speed profile
+ // changes the slope, not just the step count.
+ speedProfileId: speedProfile?.id ?? null,
+ outputPath,
+ filename,
+ actualSeed,
+ model,
+ modelId,
+ width: w,
+ height: h,
+ numFrames: parsedNumFrames,
+ steps: actualSteps,
+ // A native Extend delivers the original source, not the model's re-decode of
+ // it. Chain chunks (`hidden`) are skipped — the chain trims the echoed context
+ // itself and a window hop's "source" is a temp tail clip — except a chain's
+ // FIRST chunk (`keepExtendSource`), whose source is the user's own clip. That
+ // chunk is spliced in place at its rendered length, so the chain's trim math
+ // and the tail window it cuts for chunk 1 are unaffected. Only a visible
+ // render keeps an audio master: a hidden chunk is never extended again.
+ extendSplice: mode === 'extend' && extendFromVideoPath && (!hidden || keepExtendSource) && !batch && supportsContextWindow(model)
+ ? { sourcePath: extendFromVideoPath, extendLatents: extendLatentFrames(parsedNumFrames), keepMaster: !hidden }
+ : null,
+ },
+ });
+
+ return { jobId, generationId: jobId, filename, mode: 'local', model: modelId };
+}
+
+// Owns everything between "conditioning media exists" and "the watcher owns the
+// render": the preview directory, the running job record, argument construction
+// and the watcher handoff. A failure before `spawnAndWatchVideo` is entered
+// unwinds all of it here; once entered, the watcher is the single cleanup owner
+// (its busy-claim, setup-error and child-terminal paths each release the same
+// resources), so a throw out of it is rethrown untouched.
+async function startPreparedLocalVideo({
+ preparation, jobId, meta, batch, cleanupTempFiles, displaySleep, buildParams, watchParams,
+}) {
+ const job = { ...meta, clients: [], status: 'running' };
+ let stepwiseDir = null;
+ let handedOff = false;
+ try {
+ // Each render gets one private preview directory. Runners overwrite the
+ // fixed `preview.png` path atomically, so a long render cannot accumulate
+ // one PNG per step and a stale preview can never cross job boundaries.
+ stepwiseDir = await mkdtemp(join(tmpdir(), 'portos-video-stepwise-'));
+ videoJobState.jobs.set(jobId, job);
+
+ // buildArgs throws synchronously on multi-keyframe pixel-budget overflow and
+ // a few other validation paths; those failures converge with the allocation
+ // and weight-download failures in the catch below.
+ const { bin, args } = buildArgs({ ...buildParams, previewDir: stepwiseDir });
if (batch) args.push('--batch-seeds', JSON.stringify(batch.map((item) => item.seed)));
- if (model.runtime === 'minimax_h3') {
+ if (buildParams.model.runtime === 'minimax_h3') {
const { ensureMiniMaxH3Weights } = await import('./ensureWeights.js');
- await ensureMiniMaxH3Weights(model);
+ await ensureMiniMaxH3Weights(buildParams.model);
}
// Weights can take a long first-run download; honor a cancel that landed during it.
preparation.throwIfCanceled();
+ // This render's own choice (the `displaySleep` request field) wins over
+ // the install-wide settings default so a page reload/resume replays the
+ // choice the user actually made, not whatever Settings holds now.
+ const videoGenSettings = {
+ ...(await getSettings())?.videoGen,
+ ...(displaySleep != null ? { displaySleep } : {}),
+ };
+
+ handedOff = true;
+ await spawnAndWatchVideo({
+ jobId, preparation, batch, cleanupTempFiles, stepwiseDir, job, bin, args, meta, videoGenSettings, ...watchParams,
+ });
} catch (err) {
+ if (handedOff) throw err;
job.status = 'error';
const reason = err.message || 'Failed to prepare video generation';
console.error(`❌ Video generation preparation error [${jobId.slice(0, 8)}]: ${reason}`);
const cleanup = await Promise.allSettled([
cleanupTempFiles({ includeUploads: true, includeUntrackedAudio: true }),
- rmGuarded(stepwiseDir, { recursive: true, force: true }),
+ stepwiseDir ? rmGuarded(stepwiseDir, { recursive: true, force: true }) : undefined,
]);
if (cleanup.some(result => result.status === 'rejected')) {
maintenance.markCurrentUnsettled();
maintenance.markResourceUnsettled('media', jobId);
}
broadcastSse(job, { type: 'error', error: reason });
- videoGenEvents.emit('failed', { generationId: jobId, error: reason, failure: normalizeVideoFailure(err, { prompts: [prompt, negativePrompt] }) });
+ videoGenEvents.emit('failed', { generationId: jobId, error: reason, failure: normalizeVideoFailure(err, { prompts: [meta.prompt, meta.negativePrompt] }) });
closeJobAfterDelay(videoJobState.jobs, jobId);
throw err;
}
-
- await spawnAndWatchVideo({
- jobId,
- preparation,
- batch,
- // Keeps this render's ETA samples in its own cost bucket — a speed profile
- // changes the slope, not just the step count.
- speedProfileId: speedProfile?.id ?? null,
- cleanupTempFiles,
- stepwiseDir,
- job,
- bin,
- args,
- outputPath,
- filename,
- meta,
- actualSeed,
- model,
- modelId,
- width: w,
- height: h,
- numFrames: parsedNumFrames,
- steps: actualSteps,
- // This render's own choice (the `displaySleep` request field) wins over
- // the install-wide settings default so a page reload/resume replays the
- // choice the user actually made, not whatever Settings holds now.
- videoGenSettings: {
- ...(await getSettings())?.videoGen,
- ...(displaySleep != null ? { displaySleep } : {}),
- },
- // A native Extend delivers the original source, not the model's re-decode of
- // it. Chain chunks (`hidden`) are skipped — the chain trims the echoed context
- // itself and a window hop's "source" is a temp tail clip — except a chain's
- // FIRST chunk (`keepExtendSource`), whose source is the user's own clip. That
- // chunk is spliced in place at its rendered length, so the chain's trim math
- // and the tail window it cuts for chunk 1 are unaffected. Only a visible
- // render keeps an audio master: a hidden chunk is never extended again.
- extendSplice: mode === 'extend' && extendFromVideoPath && (!hidden || keepExtendSource) && !batch && supportsContextWindow(model)
- ? { sourcePath: extendFromVideoPath, extendLatents: extendLatentFrames(parsedNumFrames), keepMaster: !hidden }
- : null,
- });
-
- return { jobId, generationId: jobId, filename, mode: 'local', model: modelId };
}
// Pre-spawn work (weight download, memory preparation) runs with no child to
diff --git a/server/services/videoGen/local.test.js b/server/services/videoGen/local.test.js
index b4c67e6e41..0b7829e96d 100644
--- a/server/services/videoGen/local.test.js
+++ b/server/services/videoGen/local.test.js
@@ -2924,6 +2924,89 @@ describe('generateVideo — heavy claim setup cleanup (#4364)', () => {
});
});
+// Failures after conditioning returned but before the watcher owns the render:
+// the derived (resized) conditioning files, the preview directory and the
+// running job record must all unwind, and the original error must survive.
+describe.skipIf(process.platform === 'win32')('generateVideo — startup unwinds after conditioning', () => {
+ const jobIdFor = (name) => `startup-unwind-${name}`;
+ const render = (jobId) => generateVideo({
+ jobId,
+ pythonPath: '/usr/bin/python3',
+ modelId: 'ltx2_unified',
+ prompt: 'interpolate the two anchors',
+ width: 512, height: 512, numFrames: 25, fps: 24,
+ mode: 'fflf',
+ sourceImagePath: '/mock/uploads/start.png',
+ lastImagePath: '/mock/uploads/end.png',
+ });
+ const unlinkedPaths = async () => {
+ const { unlink } = await import('fs/promises');
+ return vi.mocked(unlink).mock.calls.map(([path]) => String(path));
+ };
+ const expectDerivedFilesReleased = async (jobId) => {
+ const unlinked = await unlinkedPaths();
+ expect(unlinked).toContain(join(tmpdir(), `resized-src-${jobId}.png`));
+ expect(unlinked).toContain(join(tmpdir(), `resized-last-${jobId}.png`));
+ expect(unlinked).not.toContain('/mock/uploads/start.png');
+ expect(unlinked).not.toContain('/mock/uploads/end.png');
+ };
+
+ it('releases derived conditioning files when preview-directory allocation rejects', async () => {
+ const { mkdtemp, unlink } = await import('fs/promises');
+ const { spawnDetached } = await import('../../lib/detachedSpawn.js');
+ const { claimHeavyLocalJob } = await import('../../lib/heavyJobClaim.js');
+ const { videoJobState } = await import('./jobState.js');
+ const jobId = jobIdFor('mkdtemp');
+ vi.mocked(unlink).mockClear();
+ vi.mocked(mkdtemp).mockRejectedValueOnce(new Error('EIO: preview directory'));
+
+ await expect(render(jobId)).rejects.toThrow('EIO: preview directory');
+
+ await expectDerivedFilesReleased(jobId);
+ expect(spawnDetached).not.toHaveBeenCalled();
+ expect(claimHeavyLocalJob).not.toHaveBeenCalled();
+ expect(videoJobState.jobs.get(jobId)?.status ?? 'absent').not.toBe('running');
+ });
+
+ it('releases files, preview directory and the running record when claim acquisition rejects', async () => {
+ const { rm, unlink } = await import('fs/promises');
+ const { spawnDetached } = await import('../../lib/detachedSpawn.js');
+ const { claimHeavyLocalJob } = await import('../../lib/heavyJobClaim.js');
+ const { videoJobState } = await import('./jobState.js');
+ const jobId = jobIdFor('claim');
+ vi.mocked(unlink).mockClear();
+ vi.mocked(rm).mockClear();
+ vi.mocked(claimHeavyLocalJob).mockRejectedValueOnce(new Error('EACCES: claim directory'));
+
+ await expect(render(jobId)).rejects.toThrow('EACCES: claim directory');
+
+ await expectDerivedFilesReleased(jobId);
+ expect(vi.mocked(rm).mock.calls.map(([path]) => String(path))).toContain(
+ join(tmpdir(), 'portos-video-stepwise-mock'),
+ );
+ expect(spawnDetached).not.toHaveBeenCalled();
+ expect(videoJobState.jobs.has(jobId)).toBe(false);
+ expect(heavyClaimRelease).not.toHaveBeenCalled();
+ });
+
+ it('keeps the acquisition error visible and records unsettled ownership when cleanup also fails', async () => {
+ const { unlink } = await import('fs/promises');
+ const { claimHeavyLocalJob } = await import('../../lib/heavyJobClaim.js');
+ const { maintenance } = await import('../../lib/maintenanceAdmission.js');
+ const unsettled = vi.spyOn(maintenance, 'markResourceUnsettled');
+ vi.mocked(unlink).mockRejectedValue(new Error('EBUSY: temp file'));
+ vi.mocked(claimHeavyLocalJob).mockRejectedValueOnce(new Error('EACCES: claim directory'));
+
+ try {
+ await expect(render(jobIdFor('cleanup-fails'))).rejects.toThrow('EACCES: claim directory');
+ expect(unsettled).toHaveBeenCalledWith('media', jobIdFor('cleanup-fails'));
+ } finally {
+ unsettled.mockRestore();
+ vi.mocked(unlink).mockReset().mockImplementation(async () => {});
+ }
+ });
+});
+
describe('generateVideo — close-handler resilience (issue #1334)', () => {
// A throw from finalizeGeneratedVideo inside proc.on('close') must NOT leak as
// an unhandled rejection (process-killing on Node ≥15) or strand the job
diff --git a/server/services/videoGen/spawnWatch.js b/server/services/videoGen/spawnWatch.js
index 9376b41fff..fc0d906c3f 100644
--- a/server/services/videoGen/spawnWatch.js
+++ b/server/services/videoGen/spawnWatch.js
@@ -84,11 +84,27 @@ export async function spawnAndWatchVideo({
maintenance.markCurrentUnsettled();
maintenance.markResourceUnsettled('media', jobId);
};
- const heavyClaim = await claimHeavyLocalJob({ kind: 'local video generation', id: jobId });
- if (!heavyClaim.ok) {
+ // No child and no claim exist yet, so this startup owner releases what the
+ // caller handed over. A refusal (busy holder) and a claim I/O rejection both
+ // end here, and neither touches another owner's claim.
+ const releaseUnclaimedStartup = async () => {
videoJobState.jobs.delete(jobId);
- await cleanupTempFiles({ includeUploads: true });
- await rmGuarded(stepwiseDir, { recursive: true, force: true });
+ const results = await Promise.allSettled([
+ cleanupTempFiles({ includeUploads: true }),
+ rmGuarded(stepwiseDir, { recursive: true, force: true }),
+ ]);
+ if (results.some(result => result.status === 'rejected')) retainOwnership();
+ };
+ let heavyClaim;
+ try {
+ heavyClaim = await claimHeavyLocalJob({ kind: 'local video generation', id: jobId });
+ } catch (err) {
+ console.error(`❌ Video generation claim acquisition [${jobId.slice(0, 8)}]: ${err.message}`);
+ await releaseUnclaimedStartup();
+ throw err;
+ }
+ if (!heavyClaim.ok) {
+ await releaseUnclaimedStartup();
throw new ServerError(heavyClaim.message, { status: 409, code: 'HEAVY_LOCAL_JOB_BUSY', context: { holder: heavyClaim.holder } });
}
const releaseHeavyClaim = () => heavyClaim.release().catch((err) => {