Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: unwind local video startup failures before the render child owns the job (#12395)
[._.]/ Adam Eivy2a37a4f351fb
diff --git a/server/services/videoGen/generateVideo.js b/server/services/videoGen/generateVideo.jsindex a6163cb82c..1fca716700 100644--- a/server/services/videoGen/generateVideo.js+++ b/server/services/videoGen/generateVideo.js@@ -656,22 +656,14 @@ async function renderLocalVideo(preparation, { pythonPath, prompt, negativePromp...sentRecord,...(hidden ? { hidden: true } : {}),};- // Each render gets one private preview directory. Runners overwrite the- // fixed `preview.png` path atomically, so a long render cannot accumulate- // one PNG per step and a stale preview can never cross job boundaries.- const stepwiseDir = await mkdtemp(join(tmpdir(), 'portos-video-stepwise-'));- const job = { ...meta, clients: [], status: 'running' };- videoJobState.jobs.set(jobId, job);-- // buildArgs now throws synchronously on multi-keyframe pixel-budget- // overflow and a few other validation paths — without this guard the- // job would stay "running" forever in the shared jobs map and the resized- // temp files would leak (the spawn close-handler that normally cleans- // them up never runs because we never spawned). Mirror the cleanup- // logic of the spawn-error handler so failure modes converge.- let bin, args;- try {- ({ bin, args } = buildArgs({+ await startPreparedLocalVideo({+ preparation,+ jobId,+ meta,+ batch,+ cleanupTempFiles,+ displaySleep,+ buildParams: {pythonPath,modelId,model: loraCapableModel,@@ -704,7 +696,6 @@ async function renderLocalVideo(preparation, { pythonPath, prompt, negativePromptextEncoderRepo: actualTextEncoderRepo,textEncoder: resolvedTextEncoder,outputPath,- previewDir: stepwiseDir,loras: resolvedLoras,icReferencePaths: resolvedIcReferencePaths,icLoraWeightPath,@@ -716,74 +707,96 @@ async function renderLocalVideo(preparation, { pythonPath, prompt, negativePrompstreamingMode,ffmpegPath: ffmpeg,ffprobePath: ffprobe,- }));+ },+ watchParams: {+ // Keeps this render's ETA samples in its own cost bucket — a speed profile+ // changes the slope, not just the step count.+ speedProfileId: speedProfile?.id ?? null,+ outputPath,+ filename,+ actualSeed,+ model,+ modelId,+ width: w,+ height: h,+ numFrames: parsedNumFrames,+ steps: actualSteps,+ // A native Extend delivers the original source, not the model's re-decode of+ // it. Chain chunks (`hidden`) are skipped — the chain trims the echoed context+ // itself and a window hop's "source" is a temp tail clip — except a chain's+ // FIRST chunk (`keepExtendSource`), whose source is the user's own clip. That+ // chunk is spliced in place at its rendered length, so the chain's trim math+ // and the tail window it cuts for chunk 1 are unaffected. Only a visible+ // render keeps an audio master: a hidden chunk is never extended again.+ extendSplice: mode === 'extend' && extendFromVideoPath && (!hidden || keepExtendSource) && !batch && supportsContextWindow(model)+ ? { sourcePath: extendFromVideoPath, extendLatents: extendLatentFrames(parsedNumFrames), keepMaster: !hidden }+ : null,+ },+ });++ return { jobId, generationId: jobId, filename, mode: 'local', model: modelId };+}++// Owns everything between "conditioning media exists" and "the watcher owns the+// render": the preview directory, the running job record, argument construction+// and the watcher handoff. A failure before `spawnAndWatchVideo` is entered+// unwinds all of it here; once entered, the watcher is the single cleanup owner+// (its busy-claim, setup-error and child-terminal paths each release the same+// resources), so a throw out of it is rethrown untouched.+async function startPreparedLocalVideo({+ preparation, jobId, meta, batch, cleanupTempFiles, displaySleep, buildParams, watchParams,+}) {+ const job = { ...meta, clients: [], status: 'running' };+ let stepwiseDir = null;+ let handedOff = false;+ try {+ // Each render gets one private preview directory. Runners overwrite the+ // fixed `preview.png` path atomically, so a long render cannot accumulate+ // one PNG per step and a stale preview can never cross job boundaries.+ stepwiseDir = await mkdtemp(join(tmpdir(), 'portos-video-stepwise-'));+ videoJobState.jobs.set(jobId, job);++ // buildArgs throws synchronously on multi-keyframe pixel-budget overflow and+ // a few other validation paths; those failures converge with the allocation+ // and weight-download failures in the catch below.+ const { bin, args } = buildArgs({ ...buildParams, previewDir: stepwiseDir });if (batch) args.push('--batch-seeds', JSON.stringify(batch.map((item) => item.seed)));- if (model.runtime === 'minimax_h3') {+ if (buildParams.model.runtime === 'minimax_h3') {const { ensureMiniMaxH3Weights } = await import('./ensureWeights.js');- await ensureMiniMaxH3Weights(model);+ await ensureMiniMaxH3Weights(buildParams.model);}// Weights can take a long first-run download; honor a cancel that landed during it.preparation.throwIfCanceled();+ // This render's own choice (the `displaySleep` request field) wins over+ // the install-wide settings default so a page reload/resume replays the+ // choice the user actually made, not whatever Settings holds now.+ const videoGenSettings = {+ ...(await getSettings())?.videoGen,+ ...(displaySleep != null ? { displaySleep } : {}),+ };++ handedOff = true;+ await spawnAndWatchVideo({+ jobId, preparation, batch, cleanupTempFiles, stepwiseDir, job, bin, args, meta, videoGenSettings, ...watchParams,+ });} catch (err) {+ if (handedOff) throw err;job.status = 'error';const reason = err.message || 'Failed to prepare video generation';console.error(`❌ Video generation preparation error [${jobId.slice(0, 8)}]: ${reason}`);const cleanup = await Promise.allSettled([cleanupTempFiles({ includeUploads: true, includeUntrackedAudio: true }),- rmGuarded(stepwiseDir, { recursive: true, force: true }),+ stepwiseDir ? rmGuarded(stepwiseDir, { recursive: true, force: true }) : undefined,]);if (cleanup.some(result => result.status === 'rejected')) {maintenance.markCurrentUnsettled();maintenance.markResourceUnsettled('media', jobId);}broadcastSse(job, { type: 'error', error: reason });- videoGenEvents.emit('failed', { generationId: jobId, error: reason, failure: normalizeVideoFailure(err, { prompts: [prompt, negativePrompt] }) });+ videoGenEvents.emit('failed', { generationId: jobId, error: reason, failure: normalizeVideoFailure(err, { prompts: [meta.prompt, meta.negativePrompt] }) });closeJobAfterDelay(videoJobState.jobs, jobId);throw err;}-- await spawnAndWatchVideo({- jobId,- preparation,- batch,- // Keeps this render's ETA samples in its own cost bucket — a speed profile- // changes the slope, not just the step count.- speedProfileId: speedProfile?.id ?? null,- cleanupTempFiles,- stepwiseDir,- job,- bin,- args,- outputPath,- filename,- meta,- actualSeed,- model,- modelId,- width: w,- height: h,- numFrames: parsedNumFrames,- steps: actualSteps,- // This render's own choice (the `displaySleep` request field) wins over- // the install-wide settings default so a page reload/resume replays the- // choice the user actually made, not whatever Settings holds now.- videoGenSettings: {- ...(await getSettings())?.videoGen,- ...(displaySleep != null ? { displaySleep } : {}),- },- // A native Extend delivers the original source, not the model's re-decode of- // it. Chain chunks (`hidden`) are skipped — the chain trims the echoed context- // itself and a window hop's "source" is a temp tail clip — except a chain's- // FIRST chunk (`keepExtendSource`), whose source is the user's own clip. That- // chunk is spliced in place at its rendered length, so the chain's trim math- // and the tail window it cuts for chunk 1 are unaffected. Only a visible- // render keeps an audio master: a hidden chunk is never extended again.- extendSplice: mode === 'extend' && extendFromVideoPath && (!hidden || keepExtendSource) && !batch && supportsContextWindow(model)- ? { sourcePath: extendFromVideoPath, extendLatents: extendLatentFrames(parsedNumFrames), keepMaster: !hidden }- : null,- });-- return { jobId, generationId: jobId, filename, mode: 'local', model: modelId };}// Pre-spawn work (weight download, memory preparation) runs with no child todiff --git a/server/services/videoGen/local.test.js b/server/services/videoGen/local.test.jsindex b4c67e6e41..0b7829e96d 100644--- a/server/services/videoGen/local.test.js+++ b/server/services/videoGen/local.test.js@@ -2924,6 +2924,89 @@ describe('generateVideo — heavy claim setup cleanup (#4364)', () => {});});+// Failures after conditioning returned but before the watcher owns the render:+// the derived (resized) conditioning files, the preview directory and the+// running job record must all unwind, and the original error must survive.+describe.skipIf(process.platform === 'win32')('generateVideo — startup unwinds after conditioning', () => {+ const jobIdFor = (name) => `startup-unwind-${name}`;+ const render = (jobId) => generateVideo({+ jobId,+ pythonPath: '/usr/bin/python3',+ modelId: 'ltx2_unified',+ prompt: 'interpolate the two anchors',+ width: 512, height: 512, numFrames: 25, fps: 24,+ mode: 'fflf',+ sourceImagePath: '/mock/uploads/start.png',+ lastImagePath: '/mock/uploads/end.png',+ });+ const unlinkedPaths = async () => {+ const { unlink } = await import('fs/promises');+ return vi.mocked(unlink).mock.calls.map(([path]) => String(path));+ };+ const expectDerivedFilesReleased = async (jobId) => {+ const unlinked = await unlinkedPaths();+ expect(unlinked).toContain(join(tmpdir(), `resized-src-${jobId}.png`));+ expect(unlinked).toContain(join(tmpdir(), `resized-last-${jobId}.png`));+ expect(unlinked).not.toContain('/mock/uploads/start.png');+ expect(unlinked).not.toContain('/mock/uploads/end.png');+ };++ it('releases derived conditioning files when preview-directory allocation rejects', async () => {+ const { mkdtemp, unlink } = await import('fs/promises');+ const { spawnDetached } = await import('../../lib/detachedSpawn.js');+ const { claimHeavyLocalJob } = await import('../../lib/heavyJobClaim.js');+ const { videoJobState } = await import('./jobState.js');+ const jobId = jobIdFor('mkdtemp');+ vi.mocked(unlink).mockClear();+ vi.mocked(mkdtemp).mockRejectedValueOnce(new Error('EIO: preview directory'));++ await expect(render(jobId)).rejects.toThrow('EIO: preview directory');++ await expectDerivedFilesReleased(jobId);+ expect(spawnDetached).not.toHaveBeenCalled();+ expect(claimHeavyLocalJob).not.toHaveBeenCalled();+ expect(videoJobState.jobs.get(jobId)?.status ?? 'absent').not.toBe('running');+ });++ it('releases files, preview directory and the running record when claim acquisition rejects', async () => {+ const { rm, unlink } = await import('fs/promises');+ const { spawnDetached } = await import('../../lib/detachedSpawn.js');+ const { claimHeavyLocalJob } = await import('../../lib/heavyJobClaim.js');+ const { videoJobState } = await import('./jobState.js');+ const jobId = jobIdFor('claim');+ vi.mocked(unlink).mockClear();+ vi.mocked(rm).mockClear();+ vi.mocked(claimHeavyLocalJob).mockRejectedValueOnce(new Error('EACCES: claim directory'));++ await expect(render(jobId)).rejects.toThrow('EACCES: claim directory');++ await expectDerivedFilesReleased(jobId);+ expect(vi.mocked(rm).mock.calls.map(([path]) => String(path))).toContain(+ join(tmpdir(), 'portos-video-stepwise-mock'),+ );+ expect(spawnDetached).not.toHaveBeenCalled();+ expect(videoJobState.jobs.has(jobId)).toBe(false);+ expect(heavyClaimRelease).not.toHaveBeenCalled();+ });++ it('keeps the acquisition error visible and records unsettled ownership when cleanup also fails', async () => {+ const { unlink } = await import('fs/promises');+ const { claimHeavyLocalJob } = await import('../../lib/heavyJobClaim.js');+ const { maintenance } = await import('../../lib/maintenanceAdmission.js');+ const unsettled = vi.spyOn(maintenance, 'markResourceUnsettled');+ vi.mocked(unlink).mockRejectedValue(new Error('EBUSY: temp file'));+ vi.mocked(claimHeavyLocalJob).mockRejectedValueOnce(new Error('EACCES: claim directory'));++ try {+ await expect(render(jobIdFor('cleanup-fails'))).rejects.toThrow('EACCES: claim directory');+ expect(unsettled).toHaveBeenCalledWith('media', jobIdFor('cleanup-fails'));+ } finally {+ unsettled.mockRestore();+ vi.mocked(unlink).mockReset().mockImplementation(async () => {});+ }+ });+});+describe('generateVideo — close-handler resilience (issue #1334)', () => {// A throw from finalizeGeneratedVideo inside proc.on('close') must NOT leak as// an unhandled rejection (process-killing on Node ≥15) or strand the jobdiff --git a/server/services/videoGen/spawnWatch.js b/server/services/videoGen/spawnWatch.jsindex 9376b41fff..fc0d906c3f 100644--- a/server/services/videoGen/spawnWatch.js+++ b/server/services/videoGen/spawnWatch.js@@ -84,11 +84,27 @@ export async function spawnAndWatchVideo({maintenance.markCurrentUnsettled();maintenance.markResourceUnsettled('media', jobId);};- const heavyClaim = await claimHeavyLocalJob({ kind: 'local video generation', id: jobId });- if (!heavyClaim.ok) {+ // No child and no claim exist yet, so this startup owner releases what the+ // caller handed over. A refusal (busy holder) and a claim I/O rejection both+ // end here, and neither touches another owner's claim.+ const releaseUnclaimedStartup = async () => {videoJobState.jobs.delete(jobId);- await cleanupTempFiles({ includeUploads: true });- await rmGuarded(stepwiseDir, { recursive: true, force: true });+ const results = await Promise.allSettled([+ cleanupTempFiles({ includeUploads: true }),+ rmGuarded(stepwiseDir, { recursive: true, force: true }),+ ]);+ if (results.some(result => result.status === 'rejected')) retainOwnership();+ };+ let heavyClaim;+ try {+ heavyClaim = await claimHeavyLocalJob({ kind: 'local video generation', id: jobId });+ } catch (err) {+ console.error(`❌ Video generation claim acquisition [${jobId.slice(0, 8)}]: ${err.message}`);+ await releaseUnclaimedStartup();+ throw err;+ }+ if (!heavyClaim.ok) {+ await releaseUnclaimedStartup();throw new ServerError(heavyClaim.message, { status: 409, code: 'HEAVY_LOCAL_JOB_BUSY', context: { holder: heavyClaim.holder } });}const releaseHeavyClaim = () => heavyClaim.release().catch((err) => {