Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
OmniHuman takes upload their frame and audio to fal instead of failing
Claude0845a0cb7836
diff --git a/server/lib/falVideoModels.js b/server/lib/falVideoModels.jsindex 34710d7249..05e18cd22d 100644--- a/server/lib/falVideoModels.js+++ b/server/lib/falVideoModels.js@@ -165,6 +165,9 @@ export const FAL_VIDEO_MODELS = Object.freeze([// the audio, not just the mouth. Expects a person in the frame.id: FAL_AUDIO_VIDEO_MODEL, kind: 'a2v', label: 'OmniHuman v1.5 (audio-driven person)', family: 'omnihuman',prompt: 'optional',+ // Downloads its inputs: a data URI fails with "Failed to download the file"+ // (#11381), so the frame and audio are hosted on fal's CDN first.+ hostedInputs: true,image: { param: 'image_url', required: true },// "Audio must be under 30s long for 1080p generation and under 60s long for 720p".audioInput: { param: 'audio_url', minSec: 1, maxSec: 30 },diff --git a/server/services/falQueue.js b/server/services/falQueue.jsindex d95d270e97..d835163606 100644--- a/server/services/falQueue.js+++ b/server/services/falQueue.js@@ -56,6 +56,45 @@ export async function fileToDataUri(filePath) {return `data:${mime};base64,${buf.toString('base64')}`;}+export const FAL_STORAGE_INITIATE_URL = 'https://rest.fal.ai/storage/upload/initiate?storage_type=fal-cdn-v3';+// A hosted input only has to outlive the render that reads it.+const FAL_HOSTED_INPUT_TTL_SEC = 24 * 60 * 60;+const FAL_UPLOAD_TIMEOUT_MS = 60_000;+const FILE_EXTENSIONS = { 'audio/mpeg': 'mp3', 'audio/mp4': 'm4a', 'image/jpeg': 'jpg' };++/**+ * Host a `data:` URI on fal.ai's CDN and return its https URL, for the models+ * that download their inputs rather than reading them inline (OmniHuman v1.5+ * answers a data URI with "Failed to download the file", #11381). Same two+ * steps as fal's own client: initiate (names the upload and file URLs), then+ * PUT the bytes. The file expires after a day.+ */+export async function hostFalDataUri({ apiKey, dataUri, fileName }) {+ const match = /^data:([^;,]+);base64,(.*)$/s.exec(typeof dataUri === 'string' ? dataUri : '');+ if (!match) throw new ServerError('fal.ai input is not a base64 data URI', { status: 500, code: 'FAL_UPLOAD_INPUT' });+ const [, contentType, base64] = match;+ const initiated = await fetchWithTimeout(FAL_STORAGE_INITIATE_URL, {+ method: 'POST',+ headers: {+ Authorization: `Key ${apiKey}`,+ 'Content-Type': 'application/json',+ 'X-Fal-Object-Lifecycle': JSON.stringify({ expiration_duration_seconds: FAL_HOSTED_INPUT_TTL_SEC }),+ },+ body: JSON.stringify({ content_type: contentType, file_name: `${fileName}.${FILE_EXTENSIONS[contentType] || contentType.split('/')[1] || 'bin'}` }),+ }, FAL_SUBMIT_TIMEOUT_MS);+ const target = await initiated.json().catch(() => null);+ if (!initiated.ok || typeof target?.upload_url !== 'string' || typeof target?.file_url !== 'string') {+ throw new ServerError(`fal.ai upload could not start: HTTP ${initiated.status}`, { status: 502, code: 'FAL_UPLOAD_FAILED' });+ }+ const put = await fetchWithTimeout(target.upload_url, {+ method: 'PUT',+ headers: { 'Content-Type': contentType },+ body: Buffer.from(base64, 'base64'),+ }, FAL_UPLOAD_TIMEOUT_MS);+ if (!put.ok) throw new ServerError(`fal.ai upload failed: HTTP ${put.status}`, { status: 502, code: 'FAL_UPLOAD_FAILED' });+ return target.file_url;+}+/*** The per-request bookkeeping `cancelFalRequest` and `awaitFalCompletion`* share. `cancelUrl` is filled in once the submit returns.diff --git a/server/services/videoGen/fal.js b/server/services/videoGen/fal.jsindex de9b3b0533..2fc87f7e9b 100644--- a/server/services/videoGen/fal.js+++ b/server/services/videoGen/fal.js@@ -46,7 +46,7 @@ import { getSettings } from '../settings.js';import { nearestAspectRatio } from '../imageGen/modes.js';import { frameDataUriAtAspect } from './falFrameAspect.js';import {- FAL_QUEUE_BASE, awaitFalCompletion, cancelFalRequest, createFalRequestEntry, fileToDataUri,+ FAL_QUEUE_BASE, awaitFalCompletion, cancelFalRequest, createFalRequestEntry, fileToDataUri, hostFalDataUri,readCompletedFalResult, resolveFalApiKey, submitFalRequest,} from '../falQueue.js';import { carriedPromptAdditions, promptAddition, sentPromptRecord } from '../../lib/modelInput.js';@@ -327,11 +327,17 @@ async function runFalVideo(job, jobId, {const audioDataUri = audioFilePath? `data:${audioMimeType(audioFilePath)};base64,${(await readFile(audioFilePath)).toString('base64')}`: null;+ // Most routes read inputs inline; a model that downloads them gets fal-hosted URLs.+ const hosted = getFalVideoModel(modelId)?.hostedInputs === true;+ const host = (dataUri, name) => (hosted && dataUri ? hostFalDataUri({ apiKey, dataUri, fileName: `${jobId}-${name}` }) : dataUri);+ const [imageUrl, endImageUrl, audioUrl] = await Promise.all([+ host(imageDataUri, 'start'), host(endImageDataUri, 'end'), host(audioDataUri, 'audio'),+ ]);const body = requestSpec- ? buildFalVideoRequest({ ...requestSpec, imageUrl: imageDataUri, endImageUrl: endImageDataUri, audioUrl: audioDataUri }).body+ ? buildFalVideoRequest({ ...requestSpec, imageUrl, endImageUrl, audioUrl }).body: audioFilePath- ? buildLipSyncRequestBody({ imageDataUri, audioDataUri, enableTranscription })- : buildRequestBody({ prompt, negativePrompt, duration, aspectRatio, imageDataUri });+ ? buildLipSyncRequestBody({ imageDataUri: imageUrl, audioDataUri: audioUrl, enableTranscription })+ : buildRequestBody({ prompt, negativePrompt, duration, aspectRatio, imageDataUri: imageUrl });// Cancelled while the inputs were being read: nothing was submitted, so// there is nothing to pay for or cancel remotely.if (entry.aborted) return finalizeCanceled(job, jobId);diff --git a/server/services/videoGen/fal.test.js b/server/services/videoGen/fal.test.jsindex 96ef7aa2fc..0acd4dc284 100644--- a/server/services/videoGen/fal.test.js+++ b/server/services/videoGen/fal.test.js@@ -784,3 +784,39 @@ describe('standalone lip-sync take', () => {expect(JSON.stringify(posted[1])).toContain('data:audio/webm;base64,');});});++// #11381: OmniHuman v1.5 downloads its inputs and answers a data URI with+// "Failed to download the file", so its frame and audio go up to fal's CDN first.+describe('hosted inputs', () => {+ it('uploads the frame and audio for a model that downloads its inputs and sends their URLs', async () => {+ const frame = join(TEST_ROOT, 'singer.png');+ const audio = join(TEST_ROOT, 'line.wav');+ await writeFile(frame, 'png'); await writeFile(audio, 'wav');+ const model = 'fal-ai/bytedance/omnihuman/v1.5';+ const uploads = [];+ let posted = null;+ vi.stubGlobal('fetch', vi.fn(async (url, opts) => {+ if (url.startsWith('https://rest.fal.ai/storage/upload/initiate')) {+ const { content_type: type, file_name: name } = JSON.parse(opts.body);+ uploads.push({ type, name, auth: opts.headers.Authorization });+ return jsonResponse({ upload_url: `https://upload.example.com/${name}`, file_url: `https://cdn.example.com/${name}` });+ }+ if (url.startsWith('https://upload.example.com/')) return { ok: opts.method === 'PUT', status: 200 };+ if (url === `https://queue.fal.run/${model}`) {+ posted = JSON.parse(opts.body);+ return jsonResponse({ request_id: 'req-hosted', status_url: 'https://queue.fal.run/s', response_url: 'https://queue.fal.run/r' });+ }+ if (url === 'https://queue.fal.run/s') return jsonResponse({ status: 'COMPLETED' });+ if (url === 'https://queue.fal.run/r') return jsonResponse({ video: { url: 'https://cdn.fal.ai/hosted.mp4' } });+ if (url === 'https://cdn.fal.ai/hosted.mp4') return { ok: true, status: 200, arrayBuffer: async () => Uint8Array.from(Buffer.from('mp4')).buffer };+ throw new Error(`unexpected fetch: ${url}`);+ }));+ const job = await fal.generateVideo({ apiKey: 'test-key', modelId: model, prompt: 'she sings', sourceImagePath: frame, audioFilePath: audio });+ expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'completed' });+ expect(uploads.map((u) => u.type).sort()).toEqual(['audio/wav', 'image/png']);+ expect(uploads.every((u) => u.auth === 'Key test-key')).toBe(true);+ expect(posted.image_url).toMatch(/^https:\/\/cdn\.example\.com\/.+-start\.png$/);+ expect(posted.audio_url).toMatch(/^https:\/\/cdn\.example\.com\/.+-audio\.wav$/);+ expect(JSON.stringify(posted)).not.toContain('data:');+ });+});