Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* fal.ai queue REST API — the transport shared by every fal.ai backend.
*
* fal.ai runs every model behind the same queue contract: POST the input to
* `queue.fal.run/{endpoint}`, poll the returned `status_url` until COMPLETED,
* then GET `response_url` for the output (which names a CDN URL for each file).
* `videoGen/fal.js` (#6213) grew this transport inline; the image backend
* (`imageGen/fal.js`) needs the exact same submit/poll/retrieve/cancel rules,
* so they live here once rather than as two copies free to drift on the
* retry/cancel semantics #8340 and #8564 had to fix.
*
* Only the transport is shared. Each caller keeps its own job map, SSE/event
* vocabulary and output finalization — the `onPoll`/`onStatus` hooks on
* `awaitFalCompletion` are how it reports progress in its own terms.
*/
import { readFile } from 'fs/promises';
import { maintenance } from '../lib/maintenanceAdmission.js';
import { ServerError } from '../lib/errorHandler.js';
import { describeFetchError } from '../lib/fetchErrorChain.js';
import { fetchWithTimeout } from '../lib/fetchWithTimeout.js';
import { detectImageFormat } from '../lib/mimeTypes.js';
export const FAL_QUEUE_BASE = 'https://queue.fal.run';
export const FAL_SUBMIT_TIMEOUT_MS = 30_000;
export const FAL_POLL_TIMEOUT_MS = 15_000;
export const FAL_POLL_INTERVAL_MS = 3000;
// A transient status-fetch failure (network blip, fal.ai 5xx) gets this many
// additional attempts — on the same FAL_POLL_INTERVAL_MS cadence, never
// resubmitting the paid generation — before the run is abandoned (#8340).
export const FAL_MAX_STATUS_RETRIES = 2;
const FAL_MAX_READ_RETRIES = 2;
/**
* Resolve the fal.ai API key: settings override, else the `FAL_KEY` env var
* (same settings-wins-over-env precedence as `loras.js`'s Civitai key).
*
* The settings value lives under `videoGen.fal.apiKey` because the video
* backend introduced it; `privateKeyStore.js` hydrates it there server-side.
* The image backend deliberately reuses the SAME key — one fal.ai account, one
* credential row — rather than growing an `imageGen.fal.apiKey` twin.
*/
export function resolveFalApiKey(settings) {
const fromSettings = (settings?.videoGen?.fal?.apiKey || '').trim();
if (fromSettings) return fromSettings;
const fromEnv = (process.env.FAL_KEY || '').trim();
return fromEnv || null;
}
/** Read a local file into a `data:` URI — how fal.ai accepts inline inputs. */
export async function fileToDataUri(filePath) {
const buf = await readFile(filePath);
const detected = detectImageFormat(buf);
const mime = detected?.mime || 'image/png';
return `data:${mime};base64,${buf.toString('base64')}`;
}
export const FAL_STORAGE_INITIATE_URL = 'https://rest.fal.ai/storage/upload/initiate?storage_type=fal-cdn-v3';
// A hosted input only has to outlive the render that reads it.
const FAL_HOSTED_INPUT_TTL_SEC = 24 * 60 * 60;
const FAL_UPLOAD_TIMEOUT_MS = 60_000;
const FILE_EXTENSIONS = { 'audio/mpeg': 'mp3', 'audio/mp4': 'm4a', 'image/jpeg': 'jpg' };
/**
* Host a `data:` URI on fal.ai's CDN and return its https URL, for the models
* that download their inputs rather than reading them inline (OmniHuman v1.5
* answers a data URI with "Failed to download the file", #11381). Same two
* steps as fal's own client: initiate (names the upload and file URLs), then
* PUT the bytes. The file expires after a day.
*/
export async function hostFalDataUri({ apiKey, dataUri, fileName }) {
const match = /^data:([^;,]+);base64,(.*)$/s.exec(typeof dataUri === 'string' ? dataUri : '');
if (!match) throw new ServerError('fal.ai input is not a base64 data URI', { status: 500, code: 'FAL_UPLOAD_INPUT' });
const [, contentType, base64] = match;
const initiated = await fetchWithTimeout(FAL_STORAGE_INITIATE_URL, {
method: 'POST',
headers: {
Authorization: `Key ${apiKey}`,
'Content-Type': 'application/json',
'X-Fal-Object-Lifecycle': JSON.stringify({ expiration_duration_seconds: FAL_HOSTED_INPUT_TTL_SEC }),
},
body: JSON.stringify({ content_type: contentType, file_name: `${fileName}.${FILE_EXTENSIONS[contentType] || contentType.split('/')[1] || 'bin'}` }),
}, FAL_SUBMIT_TIMEOUT_MS);
const target = await initiated.json().catch(() => null);
if (!initiated.ok || typeof target?.upload_url !== 'string' || typeof target?.file_url !== 'string') {
throw new ServerError(`fal.ai upload could not start: HTTP ${initiated.status}`, { status: 502, code: 'FAL_UPLOAD_FAILED' });
}
const put = await fetchWithTimeout(target.upload_url, {
method: 'PUT',
headers: { 'Content-Type': contentType },
body: Buffer.from(base64, 'base64'),
}, FAL_UPLOAD_TIMEOUT_MS);
if (!put.ok) throw new ServerError(`fal.ai upload failed: HTTP ${put.status}`, { status: 502, code: 'FAL_UPLOAD_FAILED' });
return target.file_url;
}
/**
* The per-request bookkeeping `cancelFalRequest` and `awaitFalCompletion`
* share. `cancelUrl` is filled in once the submit returns.
*/
export const createFalRequestEntry = (apiKey) => ({
apiKey, controller: new AbortController(), aborted: false, cancelUrl: null, canceledRemote: false, remoteTerminal: false,
});
// Best-effort, idempotent cancellation of the remote fal.ai request — shared
// by explicit user cancellation (cancel()/cancelAll()) and every local
// abandonment path (exhausted status retries, the render deadline) so an
// already-known cancel_url is never left unsent (#8340). A caught failure is
// logged for diagnosis but never rethrown: the local job still finalizes as
// failed/canceled either way. Guarded so it never re-sends once fired, and
// never fires once the remote request already reached a fal-reported
// terminal state (COMPLETED/ERROR) — there is nothing left to cancel there.
export async function cancelFalRequest(entry) {
if (!entry) return;
if (entry.cancelPromise) await entry.cancelPromise;
else {
if (entry.remoteTerminal || !entry.cancelUrl || !entry.apiKey) return;
entry.cancelPromise = sendFalCancel(entry);
await entry.cancelPromise;
}
// An accepted cancel request does not prove the paid remote render stopped.
if (!entry.remoteTerminal) maintenance.markCurrentUnsettled();
}
async function sendFalCancel(entry) {
entry.canceledRemote = true;
try {
const res = await fetchWithTimeout(entry.cancelUrl, {
method: 'PUT',
headers: { Authorization: `Key ${entry.apiKey}` },
}, FAL_POLL_TIMEOUT_MS);
if (!res.ok) console.error(`❌ fal.ai cancellation request failed: HTTP ${res.status}`);
await res.body?.cancel?.();
} catch (err) {
console.error(`❌ fal.ai cancellation request failed: ${err?.message || err}`);
}
}
/**
* Submit one paid request. Resolves to the queue receipt with every URL the
* rest of the run needs, falling back to the documented URL shapes when a
* receipt omits one.
*/
export async function submitFalRequest({ apiKey, modelId, body }) {
const res = await fetchWithTimeout(`${FAL_QUEUE_BASE}/${modelId}`, {
method: 'POST',
headers: { Authorization: `Key ${apiKey}`, 'Content-Type': 'application/json' },
body: JSON.stringify(body),
}, FAL_SUBMIT_TIMEOUT_MS).catch(err => {
// A lost POST response is not proof that the paid request was rejected.
maintenance.markCurrentUnsettled();
throw err;
});
const payload = await res.json().catch(() => null);
if (!res.ok || !payload?.request_id) {
if (res.ok || res.status >= 500) maintenance.markCurrentUnsettled();
const reason = payload?.detail ? JSON.stringify(payload.detail) : `HTTP ${res.status}`;
throw new ServerError(`fal.ai rejected the request: ${reason}`, { status: 502, code: 'FAL_SUBMIT_FAILED' });
}
const requestBase = `${FAL_QUEUE_BASE}/${modelId}/requests/${payload.request_id}`;
return {
...payload,
cancel_url: payload.cancel_url || `${requestBase}/cancel`,
status_url: payload.status_url || `${requestBase}/status`,
response_url: payload.response_url || requestBase,
};
}
async function pollFalStatus({ statusUrl, apiKey }) {
const res = await fetchWithTimeout(statusUrl, {
headers: { Authorization: `Key ${apiKey}` },
}, FAL_POLL_TIMEOUT_MS);
if (!res.ok) throw new ServerError(`fal.ai status check failed: HTTP ${res.status}`, { status: 502, code: 'FAL_STATUS_FAILED' });
return res.json();
}
/**
* Poll a submitted request until fal.ai reports it terminal, the caller
* cancels it (`entry.aborted`), or `deadline` passes. Resolves to
* `{ outcome: 'completed' | 'canceled' | 'failed', reason? }` and never
* throws — every abandonment path has already sent the remote cancel.
*
* - `onPoll()` fires before every status request (the video lane uses it as
* its watchdog activity heartbeat).
* - `onStatus(status)` fires for every non-terminal status fal.ai reports
* (`IN_QUEUE` / `IN_PROGRESS`), for the caller's progress wording.
* - `timeoutMs` only names the budget in the deadline message.
*/
export async function awaitFalCompletion({
entry, statusUrl, apiKey, deadline, timeoutMs, onPoll = () => {}, onStatus = () => {},
}) {
let statusFailures = 0;
while (Date.now() < deadline) {
if (entry.aborted) {
// The cancel may have landed while submit was in flight, before the
// receipt's cancel_url existed — send it now rather than leave a paid render running.
await cancelFalRequest(entry);
return { outcome: 'canceled' };
}
onPoll();
let status;
try {
status = await pollFalStatus({ statusUrl, apiKey });
} catch (err) {
statusFailures += 1;
if (statusFailures > FAL_MAX_STATUS_RETRIES) {
await cancelFalRequest(entry);
return { outcome: 'failed', reason: `fal.ai status checks failed ${statusFailures} times in a row: ${err?.message || err}` };
}
await new Promise((r) => setTimeout(r, FAL_POLL_INTERVAL_MS));
continue;
}
statusFailures = 0;
if (status.status === 'COMPLETED') {
// A poll that started inside the budget and returned COMPLETED is a
// finished paid render. `remoteTerminal` is set before the break so the
// post-loop deadline check cannot discard it when the round trip
// straddles `deadline` (#12053). Cancel still wins if the caller aborted
// while this read was in flight.
entry.remoteTerminal = true;
break;
}
if (status.status === 'ERROR') {
entry.remoteTerminal = true;
await cancelFalRequest(entry);
return { outcome: 'failed', reason: `fal.ai render failed: ${status.error || 'unknown error'}` };
}
onStatus(status.status);
await new Promise((r) => setTimeout(r, FAL_POLL_INTERVAL_MS));
}
if (entry.aborted) {
await cancelFalRequest(entry);
return { outcome: 'canceled' };
}
// Reaching here with remoteTerminal means the COMPLETED branch broke out.
// ERROR returns inside the loop, so this is not a failed render.
if (entry.remoteTerminal) return { outcome: 'completed' };
if (Date.now() >= deadline) {
await cancelFalRequest(entry);
return { outcome: 'failed', reason: `fal.ai did not finish within ${Math.round(timeoutMs / 1000)}s` };
}
return { outcome: 'completed' };
}
/** A short, single-line reason from a fal error response body (`detail` string or validation list), or ''. */
async function falErrorDetail(response) {
const text = typeof response.text === 'function' ? await response.text().catch(() => '') : '';
if (!text) return '';
let reason = text;
try {
const body = JSON.parse(text);
const detail = body?.detail ?? body?.error ?? body?.message;
reason = Array.isArray(detail)
? detail.map((d) => [d?.loc?.slice?.(-1)?.[0], d?.msg || d?.type].filter(Boolean).join(': ')).join('; ')
: typeof detail === 'string' ? detail : JSON.stringify(detail ?? body);
} catch { /* not JSON: keep the text */ }
return reason.replace(/\s+/g, ' ').trim().slice(0, 300);
}
/**
* Read the result of an already-paid render — the `response_url` JSON, or
* (with `binary: true`) an output file's bytes. Retries only transient reads,
* including body consumption; schema/JSON errors and permanent HTTP failures
* must not enter this loop. `apiKey` is omitted for CDN downloads, which are
* public URLs.
*/
export async function readCompletedFalResult(url, { apiKey, signal, deadline, binary = false, binaryTimeoutMs, label }) {
const what = label || (binary ? 'download' : 'result retrieval');
for (let attempt = 0; ; attempt += 1) {
signal.throwIfAborted();
const remaining = deadline - Date.now();
if (remaining <= 0) throw new Error('fal.ai retrieval deadline exceeded');
let response;
try {
response = await fetchWithTimeout(url, {
...(apiKey ? { headers: { Authorization: `Key ${apiKey}` } } : {}),
signal,
}, Math.min(remaining, binary ? binaryTimeoutMs : FAL_POLL_TIMEOUT_MS));
if (!response.ok) {
const transientRead = response.status === 408 || response.status === 429 || (response.status >= 500 && response.status <= 599);
// A permanent failure on the result read is how fal reports a job that
// failed on its side (a 422 with a `detail` naming the input or policy
// violation). Keep that reason — without it the job just says "HTTP 422".
const detail = !transientRead && !binary ? await falErrorDetail(response) : '';
const error = new Error(`fal.ai ${what} failed: HTTP ${response.status}${detail ? ` — ${detail}` : ''}`);
error.transientRead = transientRead;
throw error;
}
const value = await (binary ? response.arrayBuffer() : response.json());
signal.throwIfAborted();
return value;
} catch (err) {
// Release error responses without waiting for an error body to download.
// A rejected consumer has already released its fetch deadline.
if (response?.body && !response.bodyUsed) void response.body.cancel().catch(() => {});
signal.throwIfAborted();
if (err instanceof SyntaxError) throw new Error('fal.ai did not return valid result JSON');
const transient = err.transientRead ?? (err.name === 'AbortError' || err.name === 'TimeoutError' ||
/ECONNRESET|ECONNREFUSED|EPIPE|ETIMEDOUT|EAI_AGAIN|UND_ERR_(SOCKET|CONNECT_TIMEOUT|HEADERS_TIMEOUT|BODY_TIMEOUT)|fetch failed|network error|socket hang up/i.test(describeFetchError(err)));
if (!transient || attempt >= FAL_MAX_READ_RETRIES) throw err;
await new Promise((resolve) => setTimeout(resolve, Math.min(500 * (2 ** attempt), Math.max(0, deadline - Date.now()))));
}
}
}