Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: settle fal staged inputs before terminal publication (#11715)

[._.]/ddd9bd5c68e9
diff --git a/server/services/musicVideo/performanceShot.test.js b/server/services/musicVideo/performanceShot.test.js
index 6852854a58..2a6ab05e01 100644
--- a/server/services/musicVideo/performanceShot.test.js
+++ b/server/services/musicVideo/performanceShot.test.js
@@ -98,7 +98,7 @@ beforeEach(async () => {
videoGenEvents.removeAllListeners();
getProject.mockReset();
getTrack.mockReset();
- // Queue-owned slices are deleted by the media-job queue, which these tests bypass.
+ // Preparation-only cases leave slices behind; generated takes remove theirs before completion.
await rm(PATHS.uploads, { recursive: true, force: true });
await mkdir(PATHS.music, { recursive: true });
await writeFile(join(PATHS.music, 'song.wav'), songWav());
@@ -325,7 +325,8 @@ describe.skipIf(!ffmpeg)('music-video performance shot through the fal lip-sync
capability: { style: 'music', modelId: 'fal-ai/bytedance/omnihuman/v1.5', resolution: '1080p' } });
expect(si.audio.conditioning).toMatchObject({ source: 'master', filename: 'song.wav', selection: 'music-driven' });
expect(prepared).toMatchObject({ modelId: 'fal-ai/bytedance/omnihuman/v1.5', resolution: '1080p', enableTranscription: false });
- const samples = pcm(await readFile(prepared.audioFilePath));
+ const sliceWav = await readFile(prepared.audioFilePath);
+ const samples = pcm(sliceWav);
expect(samples.readInt16LE(0)).toBe(30000);
await mkdir(PATHS.images, { recursive: true });
@@ -368,7 +369,8 @@ describe.skipIf(!ffmpeg)('music-video performance shot through the fal lip-sync
});
expect(posts[0].body.image_url).toMatch(/^https:\/\/cdn\.example\.com\/.+-start\.[a-z]+$/);
expect(posts[0].body.audio_url).toMatch(/^https:\/\/cdn\.example\.com\/.+-audio\.wav$/);
- expect(uploads.get(posts[0].body.audio_url)).toEqual(await readFile(prepared.audioFilePath));
+ expect(uploads.get(posts[0].body.audio_url)).toEqual(sliceWav);
+ await expect(readFile(prepared.audioFilePath)).rejects.toMatchObject({ code: 'ENOENT' });
const { loadHistory } = await import('../videoGen/history.js');
// 1.5s at $0.16/s.
expect((await loadHistory()).find((h) => h.id === job.jobId)).toMatchObject({
@@ -394,6 +396,7 @@ describe.skipIf(!ffmpeg)('music-video performance shot through the fal lip-sync
expect(prepared.shotInstruction.audioWindow).toEqual(window);
expect(prepared.shotInstruction.edit).toEqual({ inSec: 20 - window.startSec, outSec: 21.5 - window.startSec, targetSec: 1.5 });
expect(prepared.modelId).toBe(url);
+ const sliceWav = await readFile(prepared.audioFilePath);
await mkdir(PATHS.images, { recursive: true });
await writeFile(join(PATHS.images, 'frame.png'), Buffer.from('89504e470d0a1a0a', 'hex'));
@@ -432,7 +435,8 @@ describe.skipIf(!ffmpeg)('music-video performance shot through the fal lip-sync
const start = frame === 'image_urls' ? posts[0].body.image_urls[0] : posts[0].body.image_url;
expect(start).toMatch(/^https:\/\/cdn\.example\.com\/.+-start\.[a-z]+$/);
expect(posts[0].body[audio]).toHaveLength(1);
- expect(uploads.get(posts[0].body[audio][0])).toEqual(await readFile(prepared.audioFilePath));
+ expect(uploads.get(posts[0].body[audio][0])).toEqual(sliceWav);
+ await expect(readFile(prepared.audioFilePath)).rejects.toMatchObject({ code: 'ENOENT' });
const { loadHistory } = await import('../videoGen/history.js');
expect((await loadHistory()).find((h) => h.id === job.jobId)).toMatchObject({ modelId: `fal:${url}`, estimatedCostUsd: costUsd, audioDriven: true });
}, 60_000);
@@ -576,7 +580,8 @@ describe.skipIf(!ffmpeg)('accepted-prefix continuation render (#9348)', () => {
expect(prepared.shotInstruction.audioWindow).toEqual({ startSec: 14, endSec: 24, durationSec: 10 });
expect(prepared.shotInstruction.edit).toEqual({ inSec: 0, outSec: 10, targetSec: 10 });
expect(prepared.shotInstruction.repair).toMatchObject({ sourceAssetId: 'accepted', sourceTakeId: 'accepted-take', role: 'continuation' });
- const sliced = pcm(await readFile(prepared.audioFilePath));
+ const sliceWav = await readFile(prepared.audioFilePath);
+ const sliced = pcm(sliceWav);
expect(sliced.length / 2).toBe(10 * RATE);
expect(sliced.readInt16LE((20 - 14) * RATE * 2)).toBe(30000);
// A source edit to a short interval cannot make preparation pad backwards
@@ -602,7 +607,8 @@ describe.skipIf(!ffmpeg)('accepted-prefix continuation render (#9348)', () => {
lipSync: { enableTranscription: prepared.enableTranscription }, shotInstruction: prepared.shotInstruction });
expect(await waitForTerminal(generated.jobId)).toMatchObject({ type: 'completed' });
expect(posts).toHaveLength(1);
- expect(Buffer.from(posts[0].audio_url.replace(/^data:audio\/wav;base64,/, ''), 'base64')).toEqual(await readFile(prepared.audioFilePath));
+ expect(Buffer.from(posts[0].audio_url.replace(/^data:audio\/wav;base64,/, ''), 'base64')).toEqual(sliceWav);
+ await expect(readFile(prepared.audioFilePath)).rejects.toMatchObject({ code: 'ENOENT' });
const landed = appendSceneTakes(repair.project, suffix.sceneId, [{ kind: 'video', assetId: generated.jobId, shotInstruction: prepared.shotInstruction }]).project;
const clips = beatSnapClips(await resolveSceneClips(landed), [], { scenes: landed.scenes });
expect(clips[0]).toMatchObject({ videoPath: acceptedPath, inSec: 0, outSec: 4, duration: 4, loop: false });
diff --git a/server/services/videoGen/fal.js b/server/services/videoGen/fal.js
index 7a3bad494e..ec8499a927 100644
--- a/server/services/videoGen/fal.js
+++ b/server/services/videoGen/fal.js
@@ -315,6 +315,26 @@ async function runFalVideo(job, jobId, {
}) {
const entry = createFalRequestEntry(apiKey);
const publication = {};
+ // Every terminal path shares the same cleanup attempt. A publication error
+ // can enter the catch after the success path has already settled its inputs.
+ let inputCleanup;
+ const settleInputs = () => (inputCleanup ??= (async () => {
+ const inputs = new Set([uploadedTempPath, audioFilePath, ...(Array.isArray(uploadedTempPaths) ? uploadedTempPaths : [])]);
+ for (const path of inputs) {
+ if (!path || !isStagedUpload(path)) continue;
+ await unlink(path).catch(error => {
+ if (error.code === 'ENOENT') return;
+ maintenance.markCurrentUnsettled();
+ maintenance.markResourceUnsettled('media', jobId);
+ console.error(`❌ fal staged-input cleanup failed [${jobId.slice(0, 8)}]: ${error.message}`);
+ });
+ }
+ })());
+ const finishFailure = async (reason) => {
+ await settleInputs();
+ if (entry.aborted) return finalizeCanceled(job, jobId);
+ return finalizeJobFailure(job, jobId, null, reason, { force: true });
+ };
activeRequests.set(jobId, entry);
const deadline = Date.now() + FAL_RENDER_TIMEOUT_MS;
try {
@@ -347,7 +367,7 @@ async function runFalVideo(job, jobId, {
: buildRequestBody({ prompt, negativePrompt, duration, aspectRatio, imageDataUri: imageUrl });
// Cancelled while the inputs were being read: nothing was submitted, so
// there is nothing to pay for or cancel remotely.
- if (entry.aborted) return finalizeCanceled(job, jobId);
+ if (entry.aborted) return finishFailure('Canceled');
const submitted = await submitFalRequest({ apiKey, modelId, body });
entry.cancelUrl = submitted.cancel_url;
const responseUrl = submitted.response_url;
@@ -362,8 +382,8 @@ async function runFalVideo(job, jobId, {
? emitCloudRenderStatus(job, jobId, CLOUD_RENDER_PHASE.RENDER, 'Rendering…')
: emitCloudRenderStatus(job, jobId, CLOUD_RENDER_PHASE.SUBMIT, 'Queued at fal.ai…')),
});
- if (polled.outcome === 'canceled') return finalizeCanceled(job, jobId);
- if (polled.outcome === 'failed') return finalizeJobFailure(job, jobId, null, polled.reason);
+ if (polled.outcome === 'canceled') return finishFailure('Canceled');
+ if (polled.outcome === 'failed') return finishFailure(polled.reason);
const buffer = await withAbortTimeout(FAL_DOWNLOAD_TIMEOUT_MS, async (timeoutSignal) => {
const signal = anyAbortSignal([timeoutSignal, entry.controller.signal]);
@@ -378,11 +398,11 @@ async function runFalVideo(job, jobId, {
signal, deadline: retrievalDeadline, binary: true, binaryTimeoutMs: FAL_DOWNLOAD_TIMEOUT_MS, label: 'video download',
}));
});
- if (entry.aborted) return finalizeCanceled(job, jobId);
+ if (entry.aborted) return finishFailure('Canceled');
await writeFile(outputPath, buffer);
if (entry.aborted) {
await unlink(outputPath);
- return finalizeCanceled(job, jobId);
+ return finishFailure('Canceled');
}
// Record what the provider actually delivered: fal reports no frame count,
@@ -394,6 +414,7 @@ async function runFalVideo(job, jobId, {
console.warn(`⚠️ fal source-audio take [${jobId.slice(0, 8)}] delivered ${measured.durationSec.toFixed(2)}s for a ${expectedSec.toFixed(2)}s audio window`);
}
activeRequests.delete(jobId);
+ await settleInputs();
activeJobs.delete(jobId);
await finalizeGeneratedVideo({ job, jobId, outputPath, filename, meta: { ...meta, ...measured }, actualSeed: null, mutateHistory: mutateVideoHistory, publication });
closeJobAfterDelay(jobs, jobId);
@@ -413,15 +434,7 @@ async function runFalVideo(job, jobId, {
console.error(`❌ fal video publication cleanup failed [${jobId.slice(0, 8)}]: ${error.message}`);
});
await cancelFalRequest(entry);
- if (entry.aborted) return finalizeCanceled(job, jobId);
- finalizeJobFailure(job, jobId, null, `fal.ai video generation failed: ${err?.message || err}`, { force: true });
- } finally {
- // The inline inputs are spent on every terminal outcome. Delete only
- // queue-owned uploads; gallery frames and the original song stay intact.
- const inputs = new Set([uploadedTempPath, audioFilePath, ...(Array.isArray(uploadedTempPaths) ? uploadedTempPaths : [])]);
- for (const path of inputs) {
- if (path && isStagedUpload(path)) await unlink(path).catch(() => {});
- }
+ return finishFailure(`fal.ai video generation failed: ${err?.message || err}`);
}
}
diff --git a/server/services/videoGen/fal.test.js b/server/services/videoGen/fal.test.js
index 0acd4dc284..02e826a0fe 100644
--- a/server/services/videoGen/fal.test.js
+++ b/server/services/videoGen/fal.test.js
@@ -1,8 +1,14 @@
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
-import { mkdir, rm, readFile, writeFile } from 'fs/promises';
+import { mkdir, rm, readFile, writeFile, unlink } from 'fs/promises';
import { join } from 'path';
import { tmpdir } from 'os';
+vi.mock('fs/promises', async (importOriginal) => {
+ const actual = await importOriginal();
+ return { ...actual, unlink: vi.fn(actual.unlink) };
+});
+const { unlink: realUnlink } = await vi.importActual('fs/promises');
+
const TEST_ROOT = join(tmpdir(), `portos-fal-video-test-${process.pid}-${Date.now()}`);
const FAKE_VIDEOS_DIR = join(TEST_ROOT, 'data-videos');
const FAKE_DATA_DIR = join(TEST_ROOT, 'data');
@@ -35,8 +41,9 @@ const fal = await import('./fal.js');
const ffmpeg = await import('../../lib/ffmpeg.js');
const { videoGenEvents } = await import('./events.js');
const { loadHistory } = await import('./history.js');
+const { maintenance } = await import('../../lib/maintenanceAdmission.js');
- // A returned job is not a settled run.
+// A returned job is not a settled run.
// Track from 'started', before generateVideo returns, through the terminal
// event after the real finalization/history tail. Keep mocks and files alive
// until every started job has reached that boundary, even if a test throws.
@@ -65,6 +72,7 @@ beforeEach(async () => {
await mkdir(FAKE_DATA_DIR, { recursive: true });
getSettingsMock.mockReset().mockResolvedValue({});
vi.restoreAllMocks();
+ unlink.mockReset().mockImplementation(realUnlink);
});
afterEach(async () => {
@@ -174,10 +182,8 @@ describe('videoGen/fal — generateVideo', () => {
expect(entry).toMatchObject({ sentPrompt: 'waves roll in', sentNegativePrompt: 'text' });
expect(entry.promptAdditions).toBeUndefined();
if (input === 'upload') {
- await vi.waitFor(async () => {
- await expect(readFile(framePath)).rejects.toMatchObject({ code: 'ENOENT' });
- await expect(readFile(endFramePath)).rejects.toMatchObject({ code: 'ENOENT' });
- });
+ await expect(readFile(framePath)).rejects.toMatchObject({ code: 'ENOENT' });
+ await expect(readFile(endFramePath)).rejects.toMatchObject({ code: 'ENOENT' });
} else {
expect(await readFile(endFramePath)).toEqual(endFrame);
expect(await readFile(framePath)).toEqual(Buffer.from('89504e470d0a1a0a', 'hex'));
@@ -754,6 +760,114 @@ describe('videoGen/fal — recover completed renders (#8564)', () => {
});
describe('standalone lip-sync take', () => {
+ // A terminal event releases the queue's ownership of staged inputs. Hold the
+ // actual unlink boundary, so the test fails even when the filesystem is fast.
+ it.each(['completed', 'provider failure', 'download failure', 'canceled'])(
+ 'settles owned inputs before the %s terminal event and SSE frame', async (outcome) => {
+ const { PATHS } = await import('../../lib/fileUtils.js');
+ await mkdir(PATHS.uploads, { recursive: true });
+ const staged = join(PATHS.uploads, 'voice.wav');
+ const stagedFrame = join(PATHS.uploads, 'end.png');
+ const outside = join(TEST_ROOT, 'original.png');
+ await writeFile(staged, 'voice');
+ await writeFile(stagedFrame, 'end frame');
+ await writeFile(outside, 'original');
+ const cleaning = Promise.withResolvers();
+ const release = Promise.withResolvers();
+ unlink.mockImplementation(async (path) => {
+ if (path === staged) {
+ cleaning.resolve();
+ await release.promise;
+ }
+ return realUnlink(path);
+ });
+ vi.stubGlobal('fetch', vi.fn(async (url) => {
+ if (url === 'https://queue.fal.run/fal-ai/y') return jsonResponse({ request_id: 'staged', status_url: 'https://queue.fal.run/s', response_url: 'https://queue.fal.run/r' });
+ if (url === 'https://queue.fal.run/s') return jsonResponse(outcome === 'provider failure' ? { status: 'ERROR', error: 'Provider refused input' } : { status: 'COMPLETED' });
+ if (url === 'https://queue.fal.run/r') return jsonResponse({ video: { url: 'https://cdn.example.com/video.mp4' } });
+ if (url === 'https://cdn.example.com/video.mp4') return new Response('mp4', { status: outcome === 'download failure' ? 403 : 200 });
+ throw new Error(`unexpected fetch: ${url}`);
+ }));
+ const job = await fal.generateVideo({
+ apiKey: 'test-key', modelId: 'fal-ai/y', sourceImagePath: outside, audioFilePath: staged,
+ uploadedTempPath: staged, uploadedTempPaths: [stagedFrame, outside],
+ });
+ const client = { writeHead: vi.fn(), write: vi.fn(), end: vi.fn(), req: { on: vi.fn() } };
+ fal.attachSseClient(job.jobId, client);
+ if (outcome === 'canceled') expect(fal.cancel(job.jobId)).toBe(true);
+ const frames = () => client.write.mock.calls.map(([msg]) => JSON.parse(msg.replace(/^data: /, '')));
+ try {
+ await cleaning.promise;
+ expect(terminalEvents.get(job.jobId)).toBeNull();
+ expect(frames().some(frame => ['complete', 'error'].includes(frame.type))).toBe(false);
+ expect(fal.getActiveJob()).toMatchObject({ generationId: job.jobId });
+ expect(await readFile(staged, 'utf8')).toBe('voice');
+ } finally {
+ release.resolve();
+ await waitForTerminal(job.jobId);
+ }
+ const succeeded = outcome === 'completed';
+ expect(terminalEvents.get(job.jobId)).toMatchObject({ type: succeeded ? 'completed' : 'failed',
+ ...(!succeeded ? { error: expect.stringMatching(outcome === 'canceled' ? /Canceled/ : outcome === 'provider failure' ? /Provider refused input/ : /403/) } : {}),
+ });
+ expect(frames().at(-1).type).toBe(succeeded ? 'complete' : 'error');
+ expect(fal.getActiveJob()).toBeNull();
+ await expect(readFile(staged)).rejects.toMatchObject({ code: 'ENOENT' });
+ await expect(readFile(stagedFrame)).rejects.toMatchObject({ code: 'ENOENT' });
+ expect(await readFile(outside, 'utf8')).toBe('original');
+ expect(unlink.mock.calls.filter(([path]) => path === staged)).toHaveLength(1);
+ if (succeeded) expect(await readFile(join(FAKE_VIDEOS_DIR, job.filename), 'utf8')).toBe('mp4');
+ else await expect(readFile(join(FAKE_VIDEOS_DIR, job.filename))).rejects.toMatchObject({ code: 'ENOENT' });
+ },
+ );
+
+ it.each([false, true])('reports uncertain cleanup and retains published video (notification failure: %s)', async (notificationFails) => {
+ const { PATHS } = await import('../../lib/fileUtils.js');
+ await mkdir(PATHS.uploads, { recursive: true });
+ const staged = join(PATHS.uploads, 'locked.wav');
+ const other = join(PATHS.uploads, 'other.png');
+ const missing = join(PATHS.uploads, 'already-removed.png');
+ const frame = join(TEST_ROOT, 'frame.png');
+ await writeFile(staged, 'voice');
+ await writeFile(other, 'png');
+ await writeFile(frame, 'png');
+ unlink.mockImplementation(path => path === staged
+ ? Promise.reject(Object.assign(new Error('Synthetic cleanup refusal'), { code: 'EACCES' }))
+ : realUnlink(path));
+ const currentUnsettled = vi.spyOn(maintenance, 'markCurrentUnsettled').mockImplementation(() => {});
+ const resourceUnsettled = vi.spyOn(maintenance, 'markResourceUnsettled').mockImplementation(() => {});
+ const logged = vi.spyOn(console, 'error').mockImplementation(() => {});
+ vi.stubGlobal('fetch', vi.fn(async (url) => {
+ if (url === 'https://queue.fal.run/fal-ai/y') return jsonResponse({ request_id: 'staged', status_url: 'https://queue.fal.run/s', response_url: 'https://queue.fal.run/r' });
+ if (url === 'https://queue.fal.run/s') return jsonResponse({ status: 'COMPLETED' });
+ if (url === 'https://queue.fal.run/r') return jsonResponse({ video: { url: 'https://cdn.example.com/video.mp4' } });
+ if (url === 'https://cdn.example.com/video.mp4') return new Response('committed video');
+ throw new Error(`unexpected fetch: ${url}`);
+ }));
+ const job = await fal.generateVideo({
+ apiKey: 'test-key', modelId: 'fal-ai/y', sourceImagePath: frame, audioFilePath: staged,
+ uploadedTempPaths: [other, missing],
+ });
+ if (notificationFails) {
+ const emit = videoGenEvents.emit.bind(videoGenEvents);
+ vi.spyOn(videoGenEvents, 'emit').mockImplementation((type, ...args) => {
+ if (type === 'completed') throw new Error('Synthetic notification failure');
+ return emit(type, ...args);
+ });
+ }
+ expect(await waitForTerminal(job.jobId)).toMatchObject(notificationFails
+ ? { type: 'failed', error: expect.stringContaining('Synthetic notification failure') }
+ : { type: 'completed' });
+ expect(currentUnsettled).toHaveBeenCalledOnce();
+ expect(resourceUnsettled).toHaveBeenCalledExactlyOnceWith('media', job.jobId);
+ expect(logged).toHaveBeenCalledWith(expect.stringContaining('staged-input cleanup failed'));
+ expect(await readFile(staged, 'utf8')).toBe('voice');
+ await expect(readFile(other)).rejects.toMatchObject({ code: 'ENOENT' });
+ expect(await readFile(join(FAKE_VIDEOS_DIR, job.filename), 'utf8')).toBe('committed video');
+ expect((await loadHistory()).filter(item => item.id === job.jobId)).toHaveLength(1);
+ expect(unlink.mock.calls.filter(([path]) => path === staged)).toHaveLength(1);
+ });
+
it('drops the staged voice clip once the take is published, but never a file outside data/uploads', async () => {
const { PATHS } = await import('../../lib/fileUtils.js');
await mkdir(PATHS.uploads, { recursive: true });