Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: settle fal staged inputs before terminal publication (#11715)
[._.]/ddd9bd5c68e9
diff --git a/server/services/musicVideo/performanceShot.test.js b/server/services/musicVideo/performanceShot.test.jsindex 6852854a58..2a6ab05e01 100644--- a/server/services/musicVideo/performanceShot.test.js+++ b/server/services/musicVideo/performanceShot.test.js@@ -98,7 +98,7 @@ beforeEach(async () => {videoGenEvents.removeAllListeners();getProject.mockReset();getTrack.mockReset();- // Queue-owned slices are deleted by the media-job queue, which these tests bypass.+ // Preparation-only cases leave slices behind; generated takes remove theirs before completion.await rm(PATHS.uploads, { recursive: true, force: true });await mkdir(PATHS.music, { recursive: true });await writeFile(join(PATHS.music, 'song.wav'), songWav());@@ -325,7 +325,8 @@ describe.skipIf(!ffmpeg)('music-video performance shot through the fal lip-synccapability: { style: 'music', modelId: 'fal-ai/bytedance/omnihuman/v1.5', resolution: '1080p' } });expect(si.audio.conditioning).toMatchObject({ source: 'master', filename: 'song.wav', selection: 'music-driven' });expect(prepared).toMatchObject({ modelId: 'fal-ai/bytedance/omnihuman/v1.5', resolution: '1080p', enableTranscription: false });- const samples = pcm(await readFile(prepared.audioFilePath));+ const sliceWav = await readFile(prepared.audioFilePath);+ const samples = pcm(sliceWav);expect(samples.readInt16LE(0)).toBe(30000);await mkdir(PATHS.images, { recursive: true });@@ -368,7 +369,8 @@ describe.skipIf(!ffmpeg)('music-video performance shot through the fal lip-sync});expect(posts[0].body.image_url).toMatch(/^https:\/\/cdn\.example\.com\/.+-start\.[a-z]+$/);expect(posts[0].body.audio_url).toMatch(/^https:\/\/cdn\.example\.com\/.+-audio\.wav$/);- expect(uploads.get(posts[0].body.audio_url)).toEqual(await readFile(prepared.audioFilePath));+ expect(uploads.get(posts[0].body.audio_url)).toEqual(sliceWav);+ await expect(readFile(prepared.audioFilePath)).rejects.toMatchObject({ code: 'ENOENT' });const { loadHistory } = await import('../videoGen/history.js');// 1.5s at $0.16/s.expect((await loadHistory()).find((h) => h.id === job.jobId)).toMatchObject({@@ -394,6 +396,7 @@ describe.skipIf(!ffmpeg)('music-video performance shot through the fal lip-syncexpect(prepared.shotInstruction.audioWindow).toEqual(window);expect(prepared.shotInstruction.edit).toEqual({ inSec: 20 - window.startSec, outSec: 21.5 - window.startSec, targetSec: 1.5 });expect(prepared.modelId).toBe(url);+ const sliceWav = await readFile(prepared.audioFilePath);await mkdir(PATHS.images, { recursive: true });await writeFile(join(PATHS.images, 'frame.png'), Buffer.from('89504e470d0a1a0a', 'hex'));@@ -432,7 +435,8 @@ describe.skipIf(!ffmpeg)('music-video performance shot through the fal lip-syncconst start = frame === 'image_urls' ? posts[0].body.image_urls[0] : posts[0].body.image_url;expect(start).toMatch(/^https:\/\/cdn\.example\.com\/.+-start\.[a-z]+$/);expect(posts[0].body[audio]).toHaveLength(1);- expect(uploads.get(posts[0].body[audio][0])).toEqual(await readFile(prepared.audioFilePath));+ expect(uploads.get(posts[0].body[audio][0])).toEqual(sliceWav);+ await expect(readFile(prepared.audioFilePath)).rejects.toMatchObject({ code: 'ENOENT' });const { loadHistory } = await import('../videoGen/history.js');expect((await loadHistory()).find((h) => h.id === job.jobId)).toMatchObject({ modelId: `fal:${url}`, estimatedCostUsd: costUsd, audioDriven: true });}, 60_000);@@ -576,7 +580,8 @@ describe.skipIf(!ffmpeg)('accepted-prefix continuation render (#9348)', () => {expect(prepared.shotInstruction.audioWindow).toEqual({ startSec: 14, endSec: 24, durationSec: 10 });expect(prepared.shotInstruction.edit).toEqual({ inSec: 0, outSec: 10, targetSec: 10 });expect(prepared.shotInstruction.repair).toMatchObject({ sourceAssetId: 'accepted', sourceTakeId: 'accepted-take', role: 'continuation' });- const sliced = pcm(await readFile(prepared.audioFilePath));+ const sliceWav = await readFile(prepared.audioFilePath);+ const sliced = pcm(sliceWav);expect(sliced.length / 2).toBe(10 * RATE);expect(sliced.readInt16LE((20 - 14) * RATE * 2)).toBe(30000);// A source edit to a short interval cannot make preparation pad backwards@@ -602,7 +607,8 @@ describe.skipIf(!ffmpeg)('accepted-prefix continuation render (#9348)', () => {lipSync: { enableTranscription: prepared.enableTranscription }, shotInstruction: prepared.shotInstruction });expect(await waitForTerminal(generated.jobId)).toMatchObject({ type: 'completed' });expect(posts).toHaveLength(1);- expect(Buffer.from(posts[0].audio_url.replace(/^data:audio\/wav;base64,/, ''), 'base64')).toEqual(await readFile(prepared.audioFilePath));+ expect(Buffer.from(posts[0].audio_url.replace(/^data:audio\/wav;base64,/, ''), 'base64')).toEqual(sliceWav);+ await expect(readFile(prepared.audioFilePath)).rejects.toMatchObject({ code: 'ENOENT' });const landed = appendSceneTakes(repair.project, suffix.sceneId, [{ kind: 'video', assetId: generated.jobId, shotInstruction: prepared.shotInstruction }]).project;const clips = beatSnapClips(await resolveSceneClips(landed), [], { scenes: landed.scenes });expect(clips[0]).toMatchObject({ videoPath: acceptedPath, inSec: 0, outSec: 4, duration: 4, loop: false });diff --git a/server/services/videoGen/fal.js b/server/services/videoGen/fal.jsindex 7a3bad494e..ec8499a927 100644--- a/server/services/videoGen/fal.js+++ b/server/services/videoGen/fal.js@@ -315,6 +315,26 @@ async function runFalVideo(job, jobId, {}) {const entry = createFalRequestEntry(apiKey);const publication = {};+ // Every terminal path shares the same cleanup attempt. A publication error+ // can enter the catch after the success path has already settled its inputs.+ let inputCleanup;+ const settleInputs = () => (inputCleanup ??= (async () => {+ const inputs = new Set([uploadedTempPath, audioFilePath, ...(Array.isArray(uploadedTempPaths) ? uploadedTempPaths : [])]);+ for (const path of inputs) {+ if (!path || !isStagedUpload(path)) continue;+ await unlink(path).catch(error => {+ if (error.code === 'ENOENT') return;+ maintenance.markCurrentUnsettled();+ maintenance.markResourceUnsettled('media', jobId);+ console.error(`❌ fal staged-input cleanup failed [${jobId.slice(0, 8)}]: ${error.message}`);+ });+ }+ })());+ const finishFailure = async (reason) => {+ await settleInputs();+ if (entry.aborted) return finalizeCanceled(job, jobId);+ return finalizeJobFailure(job, jobId, null, reason, { force: true });+ };activeRequests.set(jobId, entry);const deadline = Date.now() + FAL_RENDER_TIMEOUT_MS;try {@@ -347,7 +367,7 @@ async function runFalVideo(job, jobId, {: buildRequestBody({ prompt, negativePrompt, duration, aspectRatio, imageDataUri: imageUrl });// Cancelled while the inputs were being read: nothing was submitted, so// there is nothing to pay for or cancel remotely.- if (entry.aborted) return finalizeCanceled(job, jobId);+ if (entry.aborted) return finishFailure('Canceled');const submitted = await submitFalRequest({ apiKey, modelId, body });entry.cancelUrl = submitted.cancel_url;const responseUrl = submitted.response_url;@@ -362,8 +382,8 @@ async function runFalVideo(job, jobId, {? emitCloudRenderStatus(job, jobId, CLOUD_RENDER_PHASE.RENDER, 'Rendering…'): emitCloudRenderStatus(job, jobId, CLOUD_RENDER_PHASE.SUBMIT, 'Queued at fal.ai…')),});- if (polled.outcome === 'canceled') return finalizeCanceled(job, jobId);- if (polled.outcome === 'failed') return finalizeJobFailure(job, jobId, null, polled.reason);+ if (polled.outcome === 'canceled') return finishFailure('Canceled');+ if (polled.outcome === 'failed') return finishFailure(polled.reason);const buffer = await withAbortTimeout(FAL_DOWNLOAD_TIMEOUT_MS, async (timeoutSignal) => {const signal = anyAbortSignal([timeoutSignal, entry.controller.signal]);@@ -378,11 +398,11 @@ async function runFalVideo(job, jobId, {signal, deadline: retrievalDeadline, binary: true, binaryTimeoutMs: FAL_DOWNLOAD_TIMEOUT_MS, label: 'video download',}));});- if (entry.aborted) return finalizeCanceled(job, jobId);+ if (entry.aborted) return finishFailure('Canceled');await writeFile(outputPath, buffer);if (entry.aborted) {await unlink(outputPath);- return finalizeCanceled(job, jobId);+ return finishFailure('Canceled');}// Record what the provider actually delivered: fal reports no frame count,@@ -394,6 +414,7 @@ async function runFalVideo(job, jobId, {console.warn(`⚠️ fal source-audio take [${jobId.slice(0, 8)}] delivered ${measured.durationSec.toFixed(2)}s for a ${expectedSec.toFixed(2)}s audio window`);}activeRequests.delete(jobId);+ await settleInputs();activeJobs.delete(jobId);await finalizeGeneratedVideo({ job, jobId, outputPath, filename, meta: { ...meta, ...measured }, actualSeed: null, mutateHistory: mutateVideoHistory, publication });closeJobAfterDelay(jobs, jobId);@@ -413,15 +434,7 @@ async function runFalVideo(job, jobId, {console.error(`❌ fal video publication cleanup failed [${jobId.slice(0, 8)}]: ${error.message}`);});await cancelFalRequest(entry);- if (entry.aborted) return finalizeCanceled(job, jobId);- finalizeJobFailure(job, jobId, null, `fal.ai video generation failed: ${err?.message || err}`, { force: true });- } finally {- // The inline inputs are spent on every terminal outcome. Delete only- // queue-owned uploads; gallery frames and the original song stay intact.- const inputs = new Set([uploadedTempPath, audioFilePath, ...(Array.isArray(uploadedTempPaths) ? uploadedTempPaths : [])]);- for (const path of inputs) {- if (path && isStagedUpload(path)) await unlink(path).catch(() => {});- }+ return finishFailure(`fal.ai video generation failed: ${err?.message || err}`);}}diff --git a/server/services/videoGen/fal.test.js b/server/services/videoGen/fal.test.jsindex 0acd4dc284..02e826a0fe 100644--- a/server/services/videoGen/fal.test.js+++ b/server/services/videoGen/fal.test.js@@ -1,8 +1,14 @@import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';-import { mkdir, rm, readFile, writeFile } from 'fs/promises';+import { mkdir, rm, readFile, writeFile, unlink } from 'fs/promises';import { join } from 'path';import { tmpdir } from 'os';+vi.mock('fs/promises', async (importOriginal) => {+ const actual = await importOriginal();+ return { ...actual, unlink: vi.fn(actual.unlink) };+});+const { unlink: realUnlink } = await vi.importActual('fs/promises');+const TEST_ROOT = join(tmpdir(), `portos-fal-video-test-${process.pid}-${Date.now()}`);const FAKE_VIDEOS_DIR = join(TEST_ROOT, 'data-videos');const FAKE_DATA_DIR = join(TEST_ROOT, 'data');@@ -35,8 +41,9 @@ const fal = await import('./fal.js');const ffmpeg = await import('../../lib/ffmpeg.js');const { videoGenEvents } = await import('./events.js');const { loadHistory } = await import('./history.js');+const { maintenance } = await import('../../lib/maintenanceAdmission.js');- // A returned job is not a settled run.+// A returned job is not a settled run.// Track from 'started', before generateVideo returns, through the terminal// event after the real finalization/history tail. Keep mocks and files alive// until every started job has reached that boundary, even if a test throws.@@ -65,6 +72,7 @@ beforeEach(async () => {await mkdir(FAKE_DATA_DIR, { recursive: true });getSettingsMock.mockReset().mockResolvedValue({});vi.restoreAllMocks();+ unlink.mockReset().mockImplementation(realUnlink);});afterEach(async () => {@@ -174,10 +182,8 @@ describe('videoGen/fal — generateVideo', () => {expect(entry).toMatchObject({ sentPrompt: 'waves roll in', sentNegativePrompt: 'text' });expect(entry.promptAdditions).toBeUndefined();if (input === 'upload') {- await vi.waitFor(async () => {- await expect(readFile(framePath)).rejects.toMatchObject({ code: 'ENOENT' });- await expect(readFile(endFramePath)).rejects.toMatchObject({ code: 'ENOENT' });- });+ await expect(readFile(framePath)).rejects.toMatchObject({ code: 'ENOENT' });+ await expect(readFile(endFramePath)).rejects.toMatchObject({ code: 'ENOENT' });} else {expect(await readFile(endFramePath)).toEqual(endFrame);expect(await readFile(framePath)).toEqual(Buffer.from('89504e470d0a1a0a', 'hex'));@@ -754,6 +760,114 @@ describe('videoGen/fal — recover completed renders (#8564)', () => {});describe('standalone lip-sync take', () => {+ // A terminal event releases the queue's ownership of staged inputs. Hold the+ // actual unlink boundary, so the test fails even when the filesystem is fast.+ it.each(['completed', 'provider failure', 'download failure', 'canceled'])(+ 'settles owned inputs before the %s terminal event and SSE frame', async (outcome) => {+ const { PATHS } = await import('../../lib/fileUtils.js');+ await mkdir(PATHS.uploads, { recursive: true });+ const staged = join(PATHS.uploads, 'voice.wav');+ const stagedFrame = join(PATHS.uploads, 'end.png');+ const outside = join(TEST_ROOT, 'original.png');+ await writeFile(staged, 'voice');+ await writeFile(stagedFrame, 'end frame');+ await writeFile(outside, 'original');+ const cleaning = Promise.withResolvers();+ const release = Promise.withResolvers();+ unlink.mockImplementation(async (path) => {+ if (path === staged) {+ cleaning.resolve();+ await release.promise;+ }+ return realUnlink(path);+ });+ vi.stubGlobal('fetch', vi.fn(async (url) => {+ if (url === 'https://queue.fal.run/fal-ai/y') return jsonResponse({ request_id: 'staged', status_url: 'https://queue.fal.run/s', response_url: 'https://queue.fal.run/r' });+ if (url === 'https://queue.fal.run/s') return jsonResponse(outcome === 'provider failure' ? { status: 'ERROR', error: 'Provider refused input' } : { status: 'COMPLETED' });+ if (url === 'https://queue.fal.run/r') return jsonResponse({ video: { url: 'https://cdn.example.com/video.mp4' } });+ if (url === 'https://cdn.example.com/video.mp4') return new Response('mp4', { status: outcome === 'download failure' ? 403 : 200 });+ throw new Error(`unexpected fetch: ${url}`);+ }));+ const job = await fal.generateVideo({+ apiKey: 'test-key', modelId: 'fal-ai/y', sourceImagePath: outside, audioFilePath: staged,+ uploadedTempPath: staged, uploadedTempPaths: [stagedFrame, outside],+ });+ const client = { writeHead: vi.fn(), write: vi.fn(), end: vi.fn(), req: { on: vi.fn() } };+ fal.attachSseClient(job.jobId, client);+ if (outcome === 'canceled') expect(fal.cancel(job.jobId)).toBe(true);+ const frames = () => client.write.mock.calls.map(([msg]) => JSON.parse(msg.replace(/^data: /, '')));+ try {+ await cleaning.promise;+ expect(terminalEvents.get(job.jobId)).toBeNull();+ expect(frames().some(frame => ['complete', 'error'].includes(frame.type))).toBe(false);+ expect(fal.getActiveJob()).toMatchObject({ generationId: job.jobId });+ expect(await readFile(staged, 'utf8')).toBe('voice');+ } finally {+ release.resolve();+ await waitForTerminal(job.jobId);+ }+ const succeeded = outcome === 'completed';+ expect(terminalEvents.get(job.jobId)).toMatchObject({ type: succeeded ? 'completed' : 'failed',+ ...(!succeeded ? { error: expect.stringMatching(outcome === 'canceled' ? /Canceled/ : outcome === 'provider failure' ? /Provider refused input/ : /403/) } : {}),+ });+ expect(frames().at(-1).type).toBe(succeeded ? 'complete' : 'error');+ expect(fal.getActiveJob()).toBeNull();+ await expect(readFile(staged)).rejects.toMatchObject({ code: 'ENOENT' });+ await expect(readFile(stagedFrame)).rejects.toMatchObject({ code: 'ENOENT' });+ expect(await readFile(outside, 'utf8')).toBe('original');+ expect(unlink.mock.calls.filter(([path]) => path === staged)).toHaveLength(1);+ if (succeeded) expect(await readFile(join(FAKE_VIDEOS_DIR, job.filename), 'utf8')).toBe('mp4');+ else await expect(readFile(join(FAKE_VIDEOS_DIR, job.filename))).rejects.toMatchObject({ code: 'ENOENT' });+ },+ );++ it.each([false, true])('reports uncertain cleanup and retains published video (notification failure: %s)', async (notificationFails) => {+ const { PATHS } = await import('../../lib/fileUtils.js');+ await mkdir(PATHS.uploads, { recursive: true });+ const staged = join(PATHS.uploads, 'locked.wav');+ const other = join(PATHS.uploads, 'other.png');+ const missing = join(PATHS.uploads, 'already-removed.png');+ const frame = join(TEST_ROOT, 'frame.png');+ await writeFile(staged, 'voice');+ await writeFile(other, 'png');+ await writeFile(frame, 'png');+ unlink.mockImplementation(path => path === staged+ ? Promise.reject(Object.assign(new Error('Synthetic cleanup refusal'), { code: 'EACCES' }))+ : realUnlink(path));+ const currentUnsettled = vi.spyOn(maintenance, 'markCurrentUnsettled').mockImplementation(() => {});+ const resourceUnsettled = vi.spyOn(maintenance, 'markResourceUnsettled').mockImplementation(() => {});+ const logged = vi.spyOn(console, 'error').mockImplementation(() => {});+ vi.stubGlobal('fetch', vi.fn(async (url) => {+ if (url === 'https://queue.fal.run/fal-ai/y') return jsonResponse({ request_id: 'staged', status_url: 'https://queue.fal.run/s', response_url: 'https://queue.fal.run/r' });+ if (url === 'https://queue.fal.run/s') return jsonResponse({ status: 'COMPLETED' });+ if (url === 'https://queue.fal.run/r') return jsonResponse({ video: { url: 'https://cdn.example.com/video.mp4' } });+ if (url === 'https://cdn.example.com/video.mp4') return new Response('committed video');+ throw new Error(`unexpected fetch: ${url}`);+ }));+ const job = await fal.generateVideo({+ apiKey: 'test-key', modelId: 'fal-ai/y', sourceImagePath: frame, audioFilePath: staged,+ uploadedTempPaths: [other, missing],+ });+ if (notificationFails) {+ const emit = videoGenEvents.emit.bind(videoGenEvents);+ vi.spyOn(videoGenEvents, 'emit').mockImplementation((type, ...args) => {+ if (type === 'completed') throw new Error('Synthetic notification failure');+ return emit(type, ...args);+ });+ }+ expect(await waitForTerminal(job.jobId)).toMatchObject(notificationFails+ ? { type: 'failed', error: expect.stringContaining('Synthetic notification failure') }+ : { type: 'completed' });+ expect(currentUnsettled).toHaveBeenCalledOnce();+ expect(resourceUnsettled).toHaveBeenCalledExactlyOnceWith('media', job.jobId);+ expect(logged).toHaveBeenCalledWith(expect.stringContaining('staged-input cleanup failed'));+ expect(await readFile(staged, 'utf8')).toBe('voice');+ await expect(readFile(other)).rejects.toMatchObject({ code: 'ENOENT' });+ expect(await readFile(join(FAKE_VIDEOS_DIR, job.filename), 'utf8')).toBe('committed video');+ expect((await loadHistory()).filter(item => item.id === job.jobId)).toHaveLength(1);+ expect(unlink.mock.calls.filter(([path]) => path === staged)).toHaveLength(1);+ });+it('drops the staged voice clip once the take is published, but never a file outside data/uploads', async () => {const { PATHS } = await import('../../lib/fileUtils.js');await mkdir(PATHS.uploads, { recursive: true });