Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

OmniHuman takes upload their frame and audio to fal instead of failing

Claude0845a0cb7836
diff --git a/server/lib/falVideoModels.js b/server/lib/falVideoModels.js
index 34710d7249..05e18cd22d 100644
--- a/server/lib/falVideoModels.js
+++ b/server/lib/falVideoModels.js
@@ -165,6 +165,9 @@ export const FAL_VIDEO_MODELS = Object.freeze([
// the audio, not just the mouth. Expects a person in the frame.
id: FAL_AUDIO_VIDEO_MODEL, kind: 'a2v', label: 'OmniHuman v1.5 (audio-driven person)', family: 'omnihuman',
prompt: 'optional',
+ // Downloads its inputs: a data URI fails with "Failed to download the file"
+ // (#11381), so the frame and audio are hosted on fal's CDN first.
+ hostedInputs: true,
image: { param: 'image_url', required: true },
// "Audio must be under 30s long for 1080p generation and under 60s long for 720p".
audioInput: { param: 'audio_url', minSec: 1, maxSec: 30 },
diff --git a/server/services/falQueue.js b/server/services/falQueue.js
index d95d270e97..d835163606 100644
--- a/server/services/falQueue.js
+++ b/server/services/falQueue.js
@@ -56,6 +56,45 @@ export async function fileToDataUri(filePath) {
return `data:${mime};base64,${buf.toString('base64')}`;
}
+export const FAL_STORAGE_INITIATE_URL = 'https://rest.fal.ai/storage/upload/initiate?storage_type=fal-cdn-v3';
+// A hosted input only has to outlive the render that reads it.
+const FAL_HOSTED_INPUT_TTL_SEC = 24 * 60 * 60;
+const FAL_UPLOAD_TIMEOUT_MS = 60_000;
+const FILE_EXTENSIONS = { 'audio/mpeg': 'mp3', 'audio/mp4': 'm4a', 'image/jpeg': 'jpg' };
+
+/**
+ * Host a `data:` URI on fal.ai's CDN and return its https URL, for the models
+ * that download their inputs rather than reading them inline (OmniHuman v1.5
+ * answers a data URI with "Failed to download the file", #11381). Same two
+ * steps as fal's own client: initiate (names the upload and file URLs), then
+ * PUT the bytes. The file expires after a day.
+ */
+export async function hostFalDataUri({ apiKey, dataUri, fileName }) {
+ const match = /^data:([^;,]+);base64,(.*)$/s.exec(typeof dataUri === 'string' ? dataUri : '');
+ if (!match) throw new ServerError('fal.ai input is not a base64 data URI', { status: 500, code: 'FAL_UPLOAD_INPUT' });
+ const [, contentType, base64] = match;
+ const initiated = await fetchWithTimeout(FAL_STORAGE_INITIATE_URL, {
+ method: 'POST',
+ headers: {
+ Authorization: `Key ${apiKey}`,
+ 'Content-Type': 'application/json',
+ 'X-Fal-Object-Lifecycle': JSON.stringify({ expiration_duration_seconds: FAL_HOSTED_INPUT_TTL_SEC }),
+ },
+ body: JSON.stringify({ content_type: contentType, file_name: `${fileName}.${FILE_EXTENSIONS[contentType] || contentType.split('/')[1] || 'bin'}` }),
+ }, FAL_SUBMIT_TIMEOUT_MS);
+ const target = await initiated.json().catch(() => null);
+ if (!initiated.ok || typeof target?.upload_url !== 'string' || typeof target?.file_url !== 'string') {
+ throw new ServerError(`fal.ai upload could not start: HTTP ${initiated.status}`, { status: 502, code: 'FAL_UPLOAD_FAILED' });
+ }
+ const put = await fetchWithTimeout(target.upload_url, {
+ method: 'PUT',
+ headers: { 'Content-Type': contentType },
+ body: Buffer.from(base64, 'base64'),
+ }, FAL_UPLOAD_TIMEOUT_MS);
+ if (!put.ok) throw new ServerError(`fal.ai upload failed: HTTP ${put.status}`, { status: 502, code: 'FAL_UPLOAD_FAILED' });
+ return target.file_url;
+}
+
/**
* The per-request bookkeeping `cancelFalRequest` and `awaitFalCompletion`
* share. `cancelUrl` is filled in once the submit returns.
diff --git a/server/services/videoGen/fal.js b/server/services/videoGen/fal.js
index de9b3b0533..2fc87f7e9b 100644
--- a/server/services/videoGen/fal.js
+++ b/server/services/videoGen/fal.js
@@ -46,7 +46,7 @@ import { getSettings } from '../settings.js';
import { nearestAspectRatio } from '../imageGen/modes.js';
import { frameDataUriAtAspect } from './falFrameAspect.js';
import {
- FAL_QUEUE_BASE, awaitFalCompletion, cancelFalRequest, createFalRequestEntry, fileToDataUri,
+ FAL_QUEUE_BASE, awaitFalCompletion, cancelFalRequest, createFalRequestEntry, fileToDataUri, hostFalDataUri,
readCompletedFalResult, resolveFalApiKey, submitFalRequest,
} from '../falQueue.js';
import { carriedPromptAdditions, promptAddition, sentPromptRecord } from '../../lib/modelInput.js';
@@ -327,11 +327,17 @@ async function runFalVideo(job, jobId, {
const audioDataUri = audioFilePath
? `data:${audioMimeType(audioFilePath)};base64,${(await readFile(audioFilePath)).toString('base64')}`
: null;
+ // Most routes read inputs inline; a model that downloads them gets fal-hosted URLs.
+ const hosted = getFalVideoModel(modelId)?.hostedInputs === true;
+ const host = (dataUri, name) => (hosted && dataUri ? hostFalDataUri({ apiKey, dataUri, fileName: `${jobId}-${name}` }) : dataUri);
+ const [imageUrl, endImageUrl, audioUrl] = await Promise.all([
+ host(imageDataUri, 'start'), host(endImageDataUri, 'end'), host(audioDataUri, 'audio'),
+ ]);
const body = requestSpec
- ? buildFalVideoRequest({ ...requestSpec, imageUrl: imageDataUri, endImageUrl: endImageDataUri, audioUrl: audioDataUri }).body
+ ? buildFalVideoRequest({ ...requestSpec, imageUrl, endImageUrl, audioUrl }).body
: audioFilePath
- ? buildLipSyncRequestBody({ imageDataUri, audioDataUri, enableTranscription })
- : buildRequestBody({ prompt, negativePrompt, duration, aspectRatio, imageDataUri });
+ ? buildLipSyncRequestBody({ imageDataUri: imageUrl, audioDataUri: audioUrl, enableTranscription })
+ : buildRequestBody({ prompt, negativePrompt, duration, aspectRatio, imageDataUri: imageUrl });
// Cancelled while the inputs were being read: nothing was submitted, so
// there is nothing to pay for or cancel remotely.
if (entry.aborted) return finalizeCanceled(job, jobId);
diff --git a/server/services/videoGen/fal.test.js b/server/services/videoGen/fal.test.js
index 96ef7aa2fc..0acd4dc284 100644
--- a/server/services/videoGen/fal.test.js
+++ b/server/services/videoGen/fal.test.js
@@ -784,3 +784,39 @@ describe('standalone lip-sync take', () => {
expect(JSON.stringify(posted[1])).toContain('data:audio/webm;base64,');
});
});
+
+// #11381: OmniHuman v1.5 downloads its inputs and answers a data URI with
+// "Failed to download the file", so its frame and audio go up to fal's CDN first.
+describe('hosted inputs', () => {
+ it('uploads the frame and audio for a model that downloads its inputs and sends their URLs', async () => {
+ const frame = join(TEST_ROOT, 'singer.png');
+ const audio = join(TEST_ROOT, 'line.wav');
+ await writeFile(frame, 'png'); await writeFile(audio, 'wav');
+ const model = 'fal-ai/bytedance/omnihuman/v1.5';
+ const uploads = [];
+ let posted = null;
+ vi.stubGlobal('fetch', vi.fn(async (url, opts) => {
+ if (url.startsWith('https://rest.fal.ai/storage/upload/initiate')) {
+ const { content_type: type, file_name: name } = JSON.parse(opts.body);
+ uploads.push({ type, name, auth: opts.headers.Authorization });
+ return jsonResponse({ upload_url: `https://upload.example.com/${name}`, file_url: `https://cdn.example.com/${name}` });
+ }
+ if (url.startsWith('https://upload.example.com/')) return { ok: opts.method === 'PUT', status: 200 };
+ if (url === `https://queue.fal.run/${model}`) {
+ posted = JSON.parse(opts.body);
+ return jsonResponse({ request_id: 'req-hosted', status_url: 'https://queue.fal.run/s', response_url: 'https://queue.fal.run/r' });
+ }
+ if (url === 'https://queue.fal.run/s') return jsonResponse({ status: 'COMPLETED' });
+ if (url === 'https://queue.fal.run/r') return jsonResponse({ video: { url: 'https://cdn.fal.ai/hosted.mp4' } });
+ if (url === 'https://cdn.fal.ai/hosted.mp4') return { ok: true, status: 200, arrayBuffer: async () => Uint8Array.from(Buffer.from('mp4')).buffer };
+ throw new Error(`unexpected fetch: ${url}`);
+ }));
+ const job = await fal.generateVideo({ apiKey: 'test-key', modelId: model, prompt: 'she sings', sourceImagePath: frame, audioFilePath: audio });
+ expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'completed' });
+ expect(uploads.map((u) => u.type).sort()).toEqual(['audio/wav', 'image/png']);
+ expect(uploads.every((u) => u.auth === 'Key test-key')).toBe(true);
+ expect(posted.image_url).toMatch(/^https:\/\/cdn\.example\.com\/.+-start\.png$/);
+ expect(posted.audio_url).toMatch(/^https:\/\/cdn\.example\.com\/.+-audio\.wav$/);
+ expect(JSON.stringify(posted)).not.toContain('data:');
+ });
+});