Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: honor end frames in fal video generation (#10700)

Adam Eivy8c7fcadfd05a
diff --git a/server/routes/videoGen.test.js b/server/routes/videoGen.test.js
index 76db7693c9..23f453289c 100644
--- a/server/routes/videoGen.test.js
+++ b/server/routes/videoGen.test.js
@@ -878,6 +878,87 @@ describe('videoGen routes', () => {
tagged: { continueFromClipId: 'example-clip', seconds: 6, seed: 0, aspect: '9:16' } },
];
+ it.each([
+ { input: 'gallery', falModelId: 'fal-ai/kling-video/v3/pro/image-to-video' },
+ { input: 'upload', falModelId: 'fal-ai/kling-video/v3/pro/image-to-video' },
+ { input: 'gallery', falModelId: undefined },
+ ])('queues fal start/end frames from $input with model $falModelId and upload ownership intact', async ({ input, falModelId }) => {
+ const { getSettings } = await import('../services/settings.js');
+ getSettings.mockResolvedValueOnce(settings);
+ if (input === 'upload') {
+ setPendingUpload(
+ { fieldname: 'sourceImage', path: '/tmp/fal-start.png', originalname: 'start.png' },
+ { fieldname: 'lastImage', path: '/tmp/fal-end.jpg', originalname: 'end.jpg' },
+ );
+ }
+ const musicVideo = { projectId: 'mv-example', sceneId: 'scene-example' };
+ const r = await request(app).post('/api/video-gen/').send({
+ backend: 'fal', falModelId, prompt: 'a fox',
+ sourceImageFile: 'start.png', lastImageFile: 'end.jpg', musicVideo,
+ });
+ expect(r.status).toBe(200);
+ const { params } = mediaJobQueue.enqueueJob.mock.calls[0][0];
+ expect(params).toMatchObject({ mode: 'fal', videoMode: 'image', musicVideo });
+ if (input === 'gallery') {
+ expect(params).toMatchObject({ sourceImagePath: '/mock/images/start.png', lastImagePath: '/mock/images/end.jpg', uploadedTempPath: null });
+ expect(params).not.toHaveProperty('uploadedTempPaths');
+ } else {
+ expect(params.sourceImagePath.split('\\').join('/')).toMatch(/^\/mock\/uploads\/video-source-.*\.png$/);
+ expect(params.lastImagePath.split('\\').join('/')).toMatch(/^\/mock\/uploads\/video-last-.*\.jpg$/);
+ expect(params.uploadedTempPath).toBe(params.sourceImagePath);
+ expect(params.uploadedTempPaths).toEqual([params.lastImagePath]);
+ expect(unlink).toHaveBeenCalledWith('/tmp/fal-start.png');
+ expect(unlink).toHaveBeenCalledWith('/tmp/fal-end.jpg');
+ }
+ expect(unlink).not.toHaveBeenCalledWith('/mock/images/start.png');
+ expect(unlink).not.toHaveBeenCalledWith('/mock/images/end.jpg');
+ });
+
+ it.each(['fal-ai/veo3.1/fast/image-to-video', 'fal-ai/uncatalogued-model'])('rejects an unsupported fal end frame for model %s before enqueueing', async (falModelId) => {
+ const { getSettings } = await import('../services/settings.js');
+ getSettings.mockResolvedValueOnce(settings);
+ setPendingUpload({ fieldname: 'lastImage', path: '/tmp/fal-end.jpg', originalname: 'end.jpg' });
+ const r = await request(app).post('/api/video-gen/').send({
+ backend: 'fal', falModelId, prompt: 'a fox', sourceImageFile: 'start.png',
+ });
+ expect(r.status).toBe(400);
+ expect(r.body).toMatchObject({ code: 'VALIDATION_ERROR', error: expect.stringMatching(/end.frame/i) });
+ expect(mediaJobQueue.enqueueJob).not.toHaveBeenCalled();
+ expect(copyFile).not.toHaveBeenCalled();
+ expect(unlink).toHaveBeenCalledWith('/tmp/fal-end.jpg');
+ });
+
+ it('rejects a missing fal gallery end frame instead of silently rendering without it', async () => {
+ const { getSettings } = await import('../services/settings.js');
+ const { resolveGalleryImage } = await import('../lib/fileUtils.js');
+ getSettings.mockResolvedValueOnce(settings);
+ resolveGalleryImage.mockReturnValueOnce('/mock/images/start.png').mockReturnValueOnce(null);
+ const r = await request(app).post('/api/video-gen/').send({
+ backend: 'fal', falModelId: 'fal-ai/kling-video/v3/pro/image-to-video', prompt: 'a fox',
+ sourceImageFile: 'start.png', lastImageFile: 'missing.jpg',
+ });
+ expect(r.status).toBe(400);
+ expect(r.body.code).toBe('VALIDATION_ERROR');
+ expect(mediaJobQueue.enqueueJob).not.toHaveBeenCalled();
+ });
+
+ it('rolls back both fal frame uploads when enqueueing fails', async () => {
+ const { getSettings } = await import('../services/settings.js');
+ getSettings.mockResolvedValueOnce(settings);
+ mediaJobQueue.enqueueJob.mockImplementationOnce(() => { throw new Error('queue full'); });
+ setPendingUpload(
+ { fieldname: 'sourceImage', path: '/tmp/fal-start.png', originalname: 'start.png' },
+ { fieldname: 'lastImage', path: '/tmp/fal-end.jpg', originalname: 'end.jpg' },
+ );
+ const r = await request(app).post('/api/video-gen/').send({
+ backend: 'fal', falModelId: 'fal-ai/kling-video/v3/pro/image-to-video', prompt: 'a fox',
+ });
+ expect(r.status).toBe(500);
+ const durablePaths = copyFile.mock.calls.map(([, path]) => path);
+ expect(durablePaths).toHaveLength(2);
+ for (const path of durablePaths) expect(unlink).toHaveBeenCalledWith(path);
+ });
+
it.each(providers)('preserves $backend text defaults and exact optional-key presence', async ({ backend, minimal }) => {
const { getSettings } = await import('../services/settings.js');
getSettings.mockResolvedValueOnce(settings);
diff --git a/server/services/videoGen/fal.js b/server/services/videoGen/fal.js
index 60e5da7ec8..83f6ad3d93 100644
--- a/server/services/videoGen/fal.js
+++ b/server/services/videoGen/fal.js
@@ -142,8 +142,9 @@ function buildRequestBody({ prompt, negativePrompt, duration, aspectRatio, image
export async function generateVideo({
apiKey: providedApiKey, settings, modelId: requestedModelId,
prompt = '', negativePrompt, duration, aspectRatio, width, height,
- sourceImagePath = null, jobId: providedJobId = null,
+ sourceImagePath = null, lastImagePath = null, jobId: providedJobId = null,
audioFilePath = null, lipSync = null, shotInstruction = null,
+ uploadedTempPath = null, uploadedTempPaths = [],
resolution = null, generateAudio = false,
}) {
await ensureDir(PATHS.videos);
@@ -200,6 +201,7 @@ export async function generateVideo({
planned = buildFalVideoRequest({
...requestSpec,
imageUrl: sourceImagePath ? 'pending:image' : null,
+ endImageUrl: lastImagePath ? 'pending:end-image' : null,
audioUrl: audioFilePath ? 'pending:audio' : null,
});
} catch (err) {
@@ -207,6 +209,10 @@ export async function generateVideo({
throw new ServerError(err.message, { status: 400, code: 'VALIDATION_ERROR' });
}
+ if (lastImagePath && !planned) {
+ throw new ServerError('End frames require a fal.ai model with catalogued end-frame support', { status: 400, code: 'VALIDATION_ERROR' });
+ }
+
const meta = {
id: jobId,
prompt: (prompt || '').trim(),
@@ -242,8 +248,8 @@ export async function generateVideo({
emitCloudRenderStatus(job, jobId, CLOUD_RENDER_PHASE.SUBMIT, 'Submitting to fal.ai…');
runFalVideo(job, jobId, {
- apiKey, modelId, prompt, negativePrompt, duration, aspectRatio: effectiveAspectRatio, sourceImagePath, outputPath, filename, meta,
- audioFilePath, enableTranscription: lipSync?.enableTranscription === true,
+ apiKey, modelId, prompt, negativePrompt, duration, aspectRatio: effectiveAspectRatio, sourceImagePath, lastImagePath, outputPath, filename, meta,
+ audioFilePath, uploadedTempPath, uploadedTempPaths, enableTranscription: lipSync?.enableTranscription === true,
requestSpec: planned ? requestSpec : null,
})
.catch((err) => {
@@ -272,8 +278,8 @@ function audioMimeType(path) {
}
async function runFalVideo(job, jobId, {
- apiKey, modelId, prompt, negativePrompt, duration, aspectRatio, sourceImagePath, outputPath, filename, meta,
- audioFilePath = null, enableTranscription = false, requestSpec = null,
+ apiKey, modelId, prompt, negativePrompt, duration, aspectRatio, sourceImagePath, lastImagePath, outputPath, filename, meta,
+ audioFilePath = null, uploadedTempPath = null, uploadedTempPaths = [], enableTranscription = false, requestSpec = null,
}) {
const entry = createFalRequestEntry(apiKey);
const publication = {};
@@ -281,11 +287,12 @@ async function runFalVideo(job, jobId, {
const deadline = Date.now() + FAL_RENDER_TIMEOUT_MS;
try {
const imageDataUri = sourceImagePath ? await fileToDataUri(sourceImagePath) : null;
+ const endImageDataUri = lastImagePath ? await fileToDataUri(lastImagePath) : null;
const audioDataUri = audioFilePath
? `data:${audioMimeType(audioFilePath)};base64,${(await readFile(audioFilePath)).toString('base64')}`
: null;
const body = requestSpec
- ? buildFalVideoRequest({ ...requestSpec, imageUrl: imageDataUri, audioUrl: audioDataUri }).body
+ ? buildFalVideoRequest({ ...requestSpec, imageUrl: imageDataUri, endImageUrl: endImageDataUri, audioUrl: audioDataUri }).body
: audioFilePath
? buildLipSyncRequestBody({ imageDataUri, audioDataUri, enableTranscription })
: buildRequestBody({ prompt, negativePrompt, duration, aspectRatio, imageDataUri });
@@ -340,9 +347,6 @@ async function runFalVideo(job, jobId, {
activeRequests.delete(jobId);
activeJobs.delete(jobId);
await finalizeGeneratedVideo({ job, jobId, outputPath, filename, meta: { ...meta, ...measured }, actualSeed: null, mutateHistory: mutateVideoHistory, publication });
- // The staged voice clip is spent once the take is published. The queue removes it on failure,
- // cancel and restart; on success it is ours to drop (only a clip staged under data/uploads).
- if (audioFilePath && isStagedUpload(audioFilePath)) await unlink(audioFilePath).catch(() => {});
closeJobAfterDelay(jobs, jobId);
} catch (err) {
// Best-effort: an unanticipated throw (e.g. from fetchFalResult or the
@@ -362,6 +366,13 @@ async function runFalVideo(job, jobId, {
await cancelFalRequest(entry);
if (entry.aborted) return finalizeCanceled(job, jobId);
finalizeJobFailure(job, jobId, null, `fal.ai video generation failed: ${err?.message || err}`, { force: true });
+ } finally {
+ // The inline inputs are spent on every terminal outcome. Delete only
+ // queue-owned uploads; gallery frames and the original song stay intact.
+ const inputs = new Set([uploadedTempPath, audioFilePath, ...(Array.isArray(uploadedTempPaths) ? uploadedTempPaths : [])]);
+ for (const path of inputs) {
+ if (path && isStagedUpload(path)) await unlink(path).catch(() => {});
+ }
}
}
diff --git a/server/services/videoGen/fal.test.js b/server/services/videoGen/fal.test.js
index d90624eb0f..1b1b33882f 100644
--- a/server/services/videoGen/fal.test.js
+++ b/server/services/videoGen/fal.test.js
@@ -134,9 +134,14 @@ describe('videoGen/fal — generateVideo', () => {
// wire types per family — and the history records what was rendered and its
// list-price estimate. One family end to end; the per-family matrix is
// pinned in lib/falVideoModels.test.js.
- it('builds a curated model\'s body through the catalog and records model, resolution and estimate', async () => {
- const framePath = join(TEST_ROOT, 'frame.png');
+ it.each(['gallery', 'upload'])('builds a curated model body with both %s anchor frames and records the estimate', async (input) => {
+ const framesDir = input === 'upload' ? join(TEST_ROOT, 'uploads') : TEST_ROOT;
+ await mkdir(framesDir, { recursive: true });
+ const framePath = join(framesDir, 'frame.png');
await writeFile(framePath, Buffer.from('89504e470d0a1a0a', 'hex'));
+ const endFramePath = join(framesDir, 'end-frame.jpg');
+ const endFrame = Buffer.from('ffd8ffe000104a464946000101', 'hex');
+ await writeFile(endFramePath, endFrame);
const posted = [];
vi.stubGlobal('fetch', vi.fn(async (url, opts) => {
if (opts?.method === 'POST') {
@@ -151,18 +156,29 @@ describe('videoGen/fal — generateVideo', () => {
const job = await fal.generateVideo({
apiKey: 'test-key', modelId: 'fal-ai/kling-video/v3/pro/image-to-video',
- prompt: 'waves roll in', negativePrompt: 'text', duration: 7.4, width: 1080, height: 1920, sourceImagePath: framePath,
+ prompt: 'waves roll in', negativePrompt: 'text', duration: 7.4, width: 1080, height: 1920, sourceImagePath: framePath, lastImagePath: endFramePath,
+ ...(input === 'upload' ? { uploadedTempPath: framePath, uploadedTempPaths: [endFramePath] } : {}),
});
expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'completed' });
expect(posted).toHaveLength(1);
expect(posted[0].url).toBe('https://queue.fal.run/fal-ai/kling-video/v3/pro/image-to-video');
expect(posted[0].body).toEqual({
prompt: 'waves roll in', negative_prompt: 'text', start_image_url: expect.stringMatching(/^data:image\/png;base64,/),
+ end_image_url: `data:image/jpeg;base64,${endFrame.toString('base64')}`,
duration: '8', generate_audio: false,
});
const [entry] = await loadHistory();
// 8s (the covering length) × $0.112/s audio off.
expect(entry).toMatchObject({ modelId: 'fal:fal-ai/kling-video/v3/pro/image-to-video', duration: 8, generateAudio: false, estimatedCostUsd: 0.896 });
+ if (input === 'upload') {
+ await vi.waitFor(async () => {
+ await expect(readFile(framePath)).rejects.toMatchObject({ code: 'ENOENT' });
+ await expect(readFile(endFramePath)).rejects.toMatchObject({ code: 'ENOENT' });
+ });
+ } else {
+ expect(await readFile(endFramePath)).toEqual(endFrame);
+ expect(await readFile(framePath)).toEqual(Buffer.from('89504e470d0a1a0a', 'hex'));
+ }
});
it('refuses a start-frame-only model without a frame before submitting anything', async () => {
@@ -173,6 +189,36 @@ describe('videoGen/fal — generateVideo', () => {
expect(fetchMock).not.toHaveBeenCalled();
});
+ it.each(['fal-ai/veo3.1/fast/image-to-video', 'fal-ai/uncatalogued-model'])('refuses end frames for %s before reading media or submitting', async (modelId) => {
+ const fetchMock = vi.fn();
+ vi.stubGlobal('fetch', fetchMock);
+ await expect(fal.generateVideo({
+ apiKey: 'test-key', modelId, prompt: 'a fox',
+ sourceImagePath: join(TEST_ROOT, 'missing-start.png'), lastImagePath: join(TEST_ROOT, 'missing-end.png'),
+ })).rejects.toMatchObject({ status: 400, code: 'VALIDATION_ERROR', message: expect.stringMatching(/end.frame/i) });
+ expect(fetchMock).not.toHaveBeenCalled();
+ expect(terminalEvents.size).toBe(0);
+ });
+
+ it('cleans owned frames after an asynchronous input failure without submitting a paid request', async () => {
+ const uploads = join(TEST_ROOT, 'uploads');
+ await mkdir(uploads, { recursive: true });
+ const frame = join(uploads, 'start.png');
+ const end = join(uploads, 'missing-end.png');
+ await writeFile(frame, Buffer.from('89504e470d0a1a0a', 'hex'));
+ const fetchMock = vi.fn();
+ vi.stubGlobal('fetch', fetchMock);
+ const job = await fal.generateVideo({
+ apiKey: 'test-key', modelId: 'fal-ai/kling-video/v3/pro/image-to-video', prompt: 'a fox',
+ sourceImagePath: frame, lastImagePath: end, uploadedTempPath: frame, uploadedTempPaths: [end],
+ });
+ expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'failed' });
+ await vi.waitFor(async () => {
+ await expect(readFile(frame)).rejects.toMatchObject({ code: 'ENOENT' });
+ });
+ expect(fetchMock).not.toHaveBeenCalled();
+ });
+
it('submits, polls to completion, downloads the result, and finalizes history', async () => {
const requestId = 'req-123';
const statusUrl = `https://queue.fal.run/fal-ai/x/requests/${requestId}/status`;
diff --git a/server/services/videoGen/hostedSubmission.js b/server/services/videoGen/hostedSubmission.js
index 9615208c79..cbb68763e3 100644
--- a/server/services/videoGen/hostedSubmission.js
+++ b/server/services/videoGen/hostedSubmission.js
@@ -31,8 +31,10 @@ export const HOSTED_VIDEO_SUBMISSIONS = {
// The model, resolution and audio flag are resolved against the curated
// catalog by videoGen/fal.js (an unsupported resolution falls back to the
// model's default there); provider audio is off unless asked for.
- buildParams: (body) => ({
+ buildParams: (body, prepared) => ({
modelId: body.falModelId,
+ ...(prepared.lastImagePath ? { lastImagePath: prepared.lastImagePath } : {}),
+ ...(prepared.uploadedTempPaths?.length ? { uploadedTempPaths: prepared.uploadedTempPaths } : {}),
aspectRatio: body.visualConditioning?.render?.parameters?.aspectRatio,
width: body.width,
height: body.height,
diff --git a/server/services/videoGen/prepareParams.js b/server/services/videoGen/prepareParams.js
index d6f57238d3..fcda7b05bc 100644
--- a/server/services/videoGen/prepareParams.js
+++ b/server/services/videoGen/prepareParams.js
@@ -52,7 +52,7 @@ import { getSettings } from '../settings.js';
import { getProject as getMusicVideoProject } from '../musicVideo/projects.js';
import { getTrack } from '../tracks/index.js';
import { VIDEO_GEN_MODE, resolveVideoMode } from './modes.js';
-import { getFalVideoModel } from '../../lib/falVideoModels.js';
+import { FAL_DEFAULT_IMAGE_VIDEO_MODEL, FAL_DEFAULT_TEXT_VIDEO_MODEL, getFalVideoModel } from '../../lib/falVideoModels.js';
import { HOSTED_VIDEO_SUBMISSIONS } from './hostedSubmission.js';
import { isDefaultI2vReferenceMode } from '../../lib/videoReferenceModes.js';
import {
@@ -808,8 +808,8 @@ async function resolvePreparedParams({
}
if (uploads.sourceImage?.path) await unlinkGuarded(uploads.sourceImage.path).catch(() => {});
};
- // Hosted workers consume the already-staged source frame, but none of the
- // local last-frame/audio/IC machinery below. Keep availability checking here
+ // Hosted workers consume staged frames and supported source audio, but none
+ // of the local runtime/IC machinery below. Keep availability checking here
// so failures retain source validation precedence and staged-file ownership.
const hosted = HOSTED_VIDEO_SUBMISSIONS[backend];
if (hosted) {
@@ -818,6 +818,23 @@ async function resolvePreparedParams({
await cleanupStaged();
throw new ServerError(hosted.errorMessage, { status: 400, code: hosted.errorCode });
}
+ if (backend === VIDEO_GEN_MODE.FAL && (uploads.lastImage || body.lastImageFile)) {
+ const modelId = body.falModelId || (sourceImagePath ? FAL_DEFAULT_IMAGE_VIDEO_MODEL : FAL_DEFAULT_TEXT_VIDEO_MODEL);
+ const model = getFalVideoModel(modelId);
+ if (!model?.endImage) {
+ throw new ServerError(model ? `${model.label} takes no end frame` : 'End frames require a fal.ai model with catalogued end-frame support',
+ { status: 400, code: 'VALIDATION_ERROR' });
+ }
+ if (uploads.lastImage) {
+ lastImagePath = await stageUploadDurable(uploads.lastImage, 'last');
+ extraUploadedTempPaths.push(lastImagePath);
+ } else {
+ lastImagePath = resolveGalleryImage(body.lastImageFile);
+ if (!lastImagePath) {
+ throw new ServerError('The end frame is missing or could not be resolved.', { status: 400, code: 'VALIDATION_ERROR' });
+ }
+ }
+ }
// A standalone fal lip-sync take: stage the voice clip durably like the frame. The queue removes
// it if the job fails, is cancelled or is lost to a restart (uploadedTempPaths); the fal worker
// removes it once the take is published.
@@ -849,7 +866,10 @@ async function resolvePreparedParams({
effectiveModel: { id: backend, supportedModes: ['text', 'image'] },
sourceImagePath,
uploadedTempPath,
- ...(hostedAudioPath ? { audioFilePath: hostedAudioPath, uploadedTempPaths: [hostedAudioPath] } : {}),
+ ...(lastImagePath ? { lastImagePath } : {}),
+ ...(hostedAudioPath ? { audioFilePath: hostedAudioPath } : {}),
+ ...((extraUploadedTempPaths.length || hostedAudioPath)
+ ? { uploadedTempPaths: [...extraUploadedTempPaths, ...(hostedAudioPath ? [hostedAudioPath] : [])] } : {}),
discardSourceImage,
cleanupStaged,
};