Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: honor end frames in fal video generation (#10700)
Adam Eivy8c7fcadfd05a
diff --git a/server/routes/videoGen.test.js b/server/routes/videoGen.test.jsindex 76db7693c9..23f453289c 100644--- a/server/routes/videoGen.test.js+++ b/server/routes/videoGen.test.js@@ -878,6 +878,87 @@ describe('videoGen routes', () => {tagged: { continueFromClipId: 'example-clip', seconds: 6, seed: 0, aspect: '9:16' } },];+ it.each([+ { input: 'gallery', falModelId: 'fal-ai/kling-video/v3/pro/image-to-video' },+ { input: 'upload', falModelId: 'fal-ai/kling-video/v3/pro/image-to-video' },+ { input: 'gallery', falModelId: undefined },+ ])('queues fal start/end frames from $input with model $falModelId and upload ownership intact', async ({ input, falModelId }) => {+ const { getSettings } = await import('../services/settings.js');+ getSettings.mockResolvedValueOnce(settings);+ if (input === 'upload') {+ setPendingUpload(+ { fieldname: 'sourceImage', path: '/tmp/fal-start.png', originalname: 'start.png' },+ { fieldname: 'lastImage', path: '/tmp/fal-end.jpg', originalname: 'end.jpg' },+ );+ }+ const musicVideo = { projectId: 'mv-example', sceneId: 'scene-example' };+ const r = await request(app).post('/api/video-gen/').send({+ backend: 'fal', falModelId, prompt: 'a fox',+ sourceImageFile: 'start.png', lastImageFile: 'end.jpg', musicVideo,+ });+ expect(r.status).toBe(200);+ const { params } = mediaJobQueue.enqueueJob.mock.calls[0][0];+ expect(params).toMatchObject({ mode: 'fal', videoMode: 'image', musicVideo });+ if (input === 'gallery') {+ expect(params).toMatchObject({ sourceImagePath: '/mock/images/start.png', lastImagePath: '/mock/images/end.jpg', uploadedTempPath: null });+ expect(params).not.toHaveProperty('uploadedTempPaths');+ } else {+ expect(params.sourceImagePath.split('\\').join('/')).toMatch(/^\/mock\/uploads\/video-source-.*\.png$/);+ expect(params.lastImagePath.split('\\').join('/')).toMatch(/^\/mock\/uploads\/video-last-.*\.jpg$/);+ expect(params.uploadedTempPath).toBe(params.sourceImagePath);+ expect(params.uploadedTempPaths).toEqual([params.lastImagePath]);+ expect(unlink).toHaveBeenCalledWith('/tmp/fal-start.png');+ expect(unlink).toHaveBeenCalledWith('/tmp/fal-end.jpg');+ }+ expect(unlink).not.toHaveBeenCalledWith('/mock/images/start.png');+ expect(unlink).not.toHaveBeenCalledWith('/mock/images/end.jpg');+ });++ it.each(['fal-ai/veo3.1/fast/image-to-video', 'fal-ai/uncatalogued-model'])('rejects an unsupported fal end frame for model %s before enqueueing', async (falModelId) => {+ const { getSettings } = await import('../services/settings.js');+ getSettings.mockResolvedValueOnce(settings);+ setPendingUpload({ fieldname: 'lastImage', path: '/tmp/fal-end.jpg', originalname: 'end.jpg' });+ const r = await request(app).post('/api/video-gen/').send({+ backend: 'fal', falModelId, prompt: 'a fox', sourceImageFile: 'start.png',+ });+ expect(r.status).toBe(400);+ expect(r.body).toMatchObject({ code: 'VALIDATION_ERROR', error: expect.stringMatching(/end.frame/i) });+ expect(mediaJobQueue.enqueueJob).not.toHaveBeenCalled();+ expect(copyFile).not.toHaveBeenCalled();+ expect(unlink).toHaveBeenCalledWith('/tmp/fal-end.jpg');+ });++ it('rejects a missing fal gallery end frame instead of silently rendering without it', async () => {+ const { getSettings } = await import('../services/settings.js');+ const { resolveGalleryImage } = await import('../lib/fileUtils.js');+ getSettings.mockResolvedValueOnce(settings);+ resolveGalleryImage.mockReturnValueOnce('/mock/images/start.png').mockReturnValueOnce(null);+ const r = await request(app).post('/api/video-gen/').send({+ backend: 'fal', falModelId: 'fal-ai/kling-video/v3/pro/image-to-video', prompt: 'a fox',+ sourceImageFile: 'start.png', lastImageFile: 'missing.jpg',+ });+ expect(r.status).toBe(400);+ expect(r.body.code).toBe('VALIDATION_ERROR');+ expect(mediaJobQueue.enqueueJob).not.toHaveBeenCalled();+ });++ it('rolls back both fal frame uploads when enqueueing fails', async () => {+ const { getSettings } = await import('../services/settings.js');+ getSettings.mockResolvedValueOnce(settings);+ mediaJobQueue.enqueueJob.mockImplementationOnce(() => { throw new Error('queue full'); });+ setPendingUpload(+ { fieldname: 'sourceImage', path: '/tmp/fal-start.png', originalname: 'start.png' },+ { fieldname: 'lastImage', path: '/tmp/fal-end.jpg', originalname: 'end.jpg' },+ );+ const r = await request(app).post('/api/video-gen/').send({+ backend: 'fal', falModelId: 'fal-ai/kling-video/v3/pro/image-to-video', prompt: 'a fox',+ });+ expect(r.status).toBe(500);+ const durablePaths = copyFile.mock.calls.map(([, path]) => path);+ expect(durablePaths).toHaveLength(2);+ for (const path of durablePaths) expect(unlink).toHaveBeenCalledWith(path);+ });+it.each(providers)('preserves $backend text defaults and exact optional-key presence', async ({ backend, minimal }) => {const { getSettings } = await import('../services/settings.js');getSettings.mockResolvedValueOnce(settings);diff --git a/server/services/videoGen/fal.js b/server/services/videoGen/fal.jsindex 60e5da7ec8..83f6ad3d93 100644--- a/server/services/videoGen/fal.js+++ b/server/services/videoGen/fal.js@@ -142,8 +142,9 @@ function buildRequestBody({ prompt, negativePrompt, duration, aspectRatio, imageexport async function generateVideo({apiKey: providedApiKey, settings, modelId: requestedModelId,prompt = '', negativePrompt, duration, aspectRatio, width, height,- sourceImagePath = null, jobId: providedJobId = null,+ sourceImagePath = null, lastImagePath = null, jobId: providedJobId = null,audioFilePath = null, lipSync = null, shotInstruction = null,+ uploadedTempPath = null, uploadedTempPaths = [],resolution = null, generateAudio = false,}) {await ensureDir(PATHS.videos);@@ -200,6 +201,7 @@ export async function generateVideo({planned = buildFalVideoRequest({...requestSpec,imageUrl: sourceImagePath ? 'pending:image' : null,+ endImageUrl: lastImagePath ? 'pending:end-image' : null,audioUrl: audioFilePath ? 'pending:audio' : null,});} catch (err) {@@ -207,6 +209,10 @@ export async function generateVideo({throw new ServerError(err.message, { status: 400, code: 'VALIDATION_ERROR' });}+ if (lastImagePath && !planned) {+ throw new ServerError('End frames require a fal.ai model with catalogued end-frame support', { status: 400, code: 'VALIDATION_ERROR' });+ }+const meta = {id: jobId,prompt: (prompt || '').trim(),@@ -242,8 +248,8 @@ export async function generateVideo({emitCloudRenderStatus(job, jobId, CLOUD_RENDER_PHASE.SUBMIT, 'Submitting to fal.ai…');runFalVideo(job, jobId, {- apiKey, modelId, prompt, negativePrompt, duration, aspectRatio: effectiveAspectRatio, sourceImagePath, outputPath, filename, meta,- audioFilePath, enableTranscription: lipSync?.enableTranscription === true,+ apiKey, modelId, prompt, negativePrompt, duration, aspectRatio: effectiveAspectRatio, sourceImagePath, lastImagePath, outputPath, filename, meta,+ audioFilePath, uploadedTempPath, uploadedTempPaths, enableTranscription: lipSync?.enableTranscription === true,requestSpec: planned ? requestSpec : null,}).catch((err) => {@@ -272,8 +278,8 @@ function audioMimeType(path) {}async function runFalVideo(job, jobId, {- apiKey, modelId, prompt, negativePrompt, duration, aspectRatio, sourceImagePath, outputPath, filename, meta,- audioFilePath = null, enableTranscription = false, requestSpec = null,+ apiKey, modelId, prompt, negativePrompt, duration, aspectRatio, sourceImagePath, lastImagePath, outputPath, filename, meta,+ audioFilePath = null, uploadedTempPath = null, uploadedTempPaths = [], enableTranscription = false, requestSpec = null,}) {const entry = createFalRequestEntry(apiKey);const publication = {};@@ -281,11 +287,12 @@ async function runFalVideo(job, jobId, {const deadline = Date.now() + FAL_RENDER_TIMEOUT_MS;try {const imageDataUri = sourceImagePath ? await fileToDataUri(sourceImagePath) : null;+ const endImageDataUri = lastImagePath ? await fileToDataUri(lastImagePath) : null;const audioDataUri = audioFilePath? `data:${audioMimeType(audioFilePath)};base64,${(await readFile(audioFilePath)).toString('base64')}`: null;const body = requestSpec- ? buildFalVideoRequest({ ...requestSpec, imageUrl: imageDataUri, audioUrl: audioDataUri }).body+ ? buildFalVideoRequest({ ...requestSpec, imageUrl: imageDataUri, endImageUrl: endImageDataUri, audioUrl: audioDataUri }).body: audioFilePath? buildLipSyncRequestBody({ imageDataUri, audioDataUri, enableTranscription }): buildRequestBody({ prompt, negativePrompt, duration, aspectRatio, imageDataUri });@@ -340,9 +347,6 @@ async function runFalVideo(job, jobId, {activeRequests.delete(jobId);activeJobs.delete(jobId);await finalizeGeneratedVideo({ job, jobId, outputPath, filename, meta: { ...meta, ...measured }, actualSeed: null, mutateHistory: mutateVideoHistory, publication });- // The staged voice clip is spent once the take is published. The queue removes it on failure,- // cancel and restart; on success it is ours to drop (only a clip staged under data/uploads).- if (audioFilePath && isStagedUpload(audioFilePath)) await unlink(audioFilePath).catch(() => {});closeJobAfterDelay(jobs, jobId);} catch (err) {// Best-effort: an unanticipated throw (e.g. from fetchFalResult or the@@ -362,6 +366,13 @@ async function runFalVideo(job, jobId, {await cancelFalRequest(entry);if (entry.aborted) return finalizeCanceled(job, jobId);finalizeJobFailure(job, jobId, null, `fal.ai video generation failed: ${err?.message || err}`, { force: true });+ } finally {+ // The inline inputs are spent on every terminal outcome. Delete only+ // queue-owned uploads; gallery frames and the original song stay intact.+ const inputs = new Set([uploadedTempPath, audioFilePath, ...(Array.isArray(uploadedTempPaths) ? uploadedTempPaths : [])]);+ for (const path of inputs) {+ if (path && isStagedUpload(path)) await unlink(path).catch(() => {});+ }}}diff --git a/server/services/videoGen/fal.test.js b/server/services/videoGen/fal.test.jsindex d90624eb0f..1b1b33882f 100644--- a/server/services/videoGen/fal.test.js+++ b/server/services/videoGen/fal.test.js@@ -134,9 +134,14 @@ describe('videoGen/fal — generateVideo', () => {// wire types per family — and the history records what was rendered and its// list-price estimate. One family end to end; the per-family matrix is// pinned in lib/falVideoModels.test.js.- it('builds a curated model\'s body through the catalog and records model, resolution and estimate', async () => {- const framePath = join(TEST_ROOT, 'frame.png');+ it.each(['gallery', 'upload'])('builds a curated model body with both %s anchor frames and records the estimate', async (input) => {+ const framesDir = input === 'upload' ? join(TEST_ROOT, 'uploads') : TEST_ROOT;+ await mkdir(framesDir, { recursive: true });+ const framePath = join(framesDir, 'frame.png');await writeFile(framePath, Buffer.from('89504e470d0a1a0a', 'hex'));+ const endFramePath = join(framesDir, 'end-frame.jpg');+ const endFrame = Buffer.from('ffd8ffe000104a464946000101', 'hex');+ await writeFile(endFramePath, endFrame);const posted = [];vi.stubGlobal('fetch', vi.fn(async (url, opts) => {if (opts?.method === 'POST') {@@ -151,18 +156,29 @@ describe('videoGen/fal — generateVideo', () => {const job = await fal.generateVideo({apiKey: 'test-key', modelId: 'fal-ai/kling-video/v3/pro/image-to-video',- prompt: 'waves roll in', negativePrompt: 'text', duration: 7.4, width: 1080, height: 1920, sourceImagePath: framePath,+ prompt: 'waves roll in', negativePrompt: 'text', duration: 7.4, width: 1080, height: 1920, sourceImagePath: framePath, lastImagePath: endFramePath,+ ...(input === 'upload' ? { uploadedTempPath: framePath, uploadedTempPaths: [endFramePath] } : {}),});expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'completed' });expect(posted).toHaveLength(1);expect(posted[0].url).toBe('https://queue.fal.run/fal-ai/kling-video/v3/pro/image-to-video');expect(posted[0].body).toEqual({prompt: 'waves roll in', negative_prompt: 'text', start_image_url: expect.stringMatching(/^data:image\/png;base64,/),+ end_image_url: `data:image/jpeg;base64,${endFrame.toString('base64')}`,duration: '8', generate_audio: false,});const [entry] = await loadHistory();// 8s (the covering length) × $0.112/s audio off.expect(entry).toMatchObject({ modelId: 'fal:fal-ai/kling-video/v3/pro/image-to-video', duration: 8, generateAudio: false, estimatedCostUsd: 0.896 });+ if (input === 'upload') {+ await vi.waitFor(async () => {+ await expect(readFile(framePath)).rejects.toMatchObject({ code: 'ENOENT' });+ await expect(readFile(endFramePath)).rejects.toMatchObject({ code: 'ENOENT' });+ });+ } else {+ expect(await readFile(endFramePath)).toEqual(endFrame);+ expect(await readFile(framePath)).toEqual(Buffer.from('89504e470d0a1a0a', 'hex'));+ }});it('refuses a start-frame-only model without a frame before submitting anything', async () => {@@ -173,6 +189,36 @@ describe('videoGen/fal — generateVideo', () => {expect(fetchMock).not.toHaveBeenCalled();});+ it.each(['fal-ai/veo3.1/fast/image-to-video', 'fal-ai/uncatalogued-model'])('refuses end frames for %s before reading media or submitting', async (modelId) => {+ const fetchMock = vi.fn();+ vi.stubGlobal('fetch', fetchMock);+ await expect(fal.generateVideo({+ apiKey: 'test-key', modelId, prompt: 'a fox',+ sourceImagePath: join(TEST_ROOT, 'missing-start.png'), lastImagePath: join(TEST_ROOT, 'missing-end.png'),+ })).rejects.toMatchObject({ status: 400, code: 'VALIDATION_ERROR', message: expect.stringMatching(/end.frame/i) });+ expect(fetchMock).not.toHaveBeenCalled();+ expect(terminalEvents.size).toBe(0);+ });++ it('cleans owned frames after an asynchronous input failure without submitting a paid request', async () => {+ const uploads = join(TEST_ROOT, 'uploads');+ await mkdir(uploads, { recursive: true });+ const frame = join(uploads, 'start.png');+ const end = join(uploads, 'missing-end.png');+ await writeFile(frame, Buffer.from('89504e470d0a1a0a', 'hex'));+ const fetchMock = vi.fn();+ vi.stubGlobal('fetch', fetchMock);+ const job = await fal.generateVideo({+ apiKey: 'test-key', modelId: 'fal-ai/kling-video/v3/pro/image-to-video', prompt: 'a fox',+ sourceImagePath: frame, lastImagePath: end, uploadedTempPath: frame, uploadedTempPaths: [end],+ });+ expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'failed' });+ await vi.waitFor(async () => {+ await expect(readFile(frame)).rejects.toMatchObject({ code: 'ENOENT' });+ });+ expect(fetchMock).not.toHaveBeenCalled();+ });+it('submits, polls to completion, downloads the result, and finalizes history', async () => {const requestId = 'req-123';const statusUrl = `https://queue.fal.run/fal-ai/x/requests/${requestId}/status`;diff --git a/server/services/videoGen/hostedSubmission.js b/server/services/videoGen/hostedSubmission.jsindex 9615208c79..cbb68763e3 100644--- a/server/services/videoGen/hostedSubmission.js+++ b/server/services/videoGen/hostedSubmission.js@@ -31,8 +31,10 @@ export const HOSTED_VIDEO_SUBMISSIONS = {// The model, resolution and audio flag are resolved against the curated// catalog by videoGen/fal.js (an unsupported resolution falls back to the// model's default there); provider audio is off unless asked for.- buildParams: (body) => ({+ buildParams: (body, prepared) => ({modelId: body.falModelId,+ ...(prepared.lastImagePath ? { lastImagePath: prepared.lastImagePath } : {}),+ ...(prepared.uploadedTempPaths?.length ? { uploadedTempPaths: prepared.uploadedTempPaths } : {}),aspectRatio: body.visualConditioning?.render?.parameters?.aspectRatio,width: body.width,height: body.height,diff --git a/server/services/videoGen/prepareParams.js b/server/services/videoGen/prepareParams.jsindex d6f57238d3..fcda7b05bc 100644--- a/server/services/videoGen/prepareParams.js+++ b/server/services/videoGen/prepareParams.js@@ -52,7 +52,7 @@ import { getSettings } from '../settings.js';import { getProject as getMusicVideoProject } from '../musicVideo/projects.js';import { getTrack } from '../tracks/index.js';import { VIDEO_GEN_MODE, resolveVideoMode } from './modes.js';-import { getFalVideoModel } from '../../lib/falVideoModels.js';+import { FAL_DEFAULT_IMAGE_VIDEO_MODEL, FAL_DEFAULT_TEXT_VIDEO_MODEL, getFalVideoModel } from '../../lib/falVideoModels.js';import { HOSTED_VIDEO_SUBMISSIONS } from './hostedSubmission.js';import { isDefaultI2vReferenceMode } from '../../lib/videoReferenceModes.js';import {@@ -808,8 +808,8 @@ async function resolvePreparedParams({}if (uploads.sourceImage?.path) await unlinkGuarded(uploads.sourceImage.path).catch(() => {});};- // Hosted workers consume the already-staged source frame, but none of the- // local last-frame/audio/IC machinery below. Keep availability checking here+ // Hosted workers consume staged frames and supported source audio, but none+ // of the local runtime/IC machinery below. Keep availability checking here// so failures retain source validation precedence and staged-file ownership.const hosted = HOSTED_VIDEO_SUBMISSIONS[backend];if (hosted) {@@ -818,6 +818,23 @@ async function resolvePreparedParams({await cleanupStaged();throw new ServerError(hosted.errorMessage, { status: 400, code: hosted.errorCode });}+ if (backend === VIDEO_GEN_MODE.FAL && (uploads.lastImage || body.lastImageFile)) {+ const modelId = body.falModelId || (sourceImagePath ? FAL_DEFAULT_IMAGE_VIDEO_MODEL : FAL_DEFAULT_TEXT_VIDEO_MODEL);+ const model = getFalVideoModel(modelId);+ if (!model?.endImage) {+ throw new ServerError(model ? `${model.label} takes no end frame` : 'End frames require a fal.ai model with catalogued end-frame support',+ { status: 400, code: 'VALIDATION_ERROR' });+ }+ if (uploads.lastImage) {+ lastImagePath = await stageUploadDurable(uploads.lastImage, 'last');+ extraUploadedTempPaths.push(lastImagePath);+ } else {+ lastImagePath = resolveGalleryImage(body.lastImageFile);+ if (!lastImagePath) {+ throw new ServerError('The end frame is missing or could not be resolved.', { status: 400, code: 'VALIDATION_ERROR' });+ }+ }+ }// A standalone fal lip-sync take: stage the voice clip durably like the frame. The queue removes// it if the job fails, is cancelled or is lost to a restart (uploadedTempPaths); the fal worker// removes it once the take is published.@@ -849,7 +866,10 @@ async function resolvePreparedParams({effectiveModel: { id: backend, supportedModes: ['text', 'image'] },sourceImagePath,uploadedTempPath,- ...(hostedAudioPath ? { audioFilePath: hostedAudioPath, uploadedTempPaths: [hostedAudioPath] } : {}),+ ...(lastImagePath ? { lastImagePath } : {}),+ ...(hostedAudioPath ? { audioFilePath: hostedAudioPath } : {}),+ ...((extraUploadedTempPaths.length || hostedAudioPath)+ ? { uploadedTempPaths: [...extraUploadedTempPaths, ...(hostedAudioPath ? [hostedAudioPath] : [])] } : {}),discardSourceImage,cleanupStaged,};