Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Music Video production run (#9066) — the allowed provider/model pool.
*
* A pool route is `{ kind: 'image'|'video', mode, model }`: a render backend
* from the shared alphabets (lib/generationModes.js) plus, where the backend
* takes one, a model id. Eligibility is re-derived from LIVE settings and
* catalogs every time — at Start (every route must be eligible) and again at
* dispatch for the route a step picked — so a backend disabled, a key removed
* or a local model uninstalled mid-run refuses that step before enqueue
* instead of silently falling back to something the director did not allow.
*
* Capability is per scene: a frame conditioned on the visual spec's reference
* images needs a backend (and local model) that consumes reference images, and
* a performance shot needs a source-audio lip-sync video provider. A route
* that cannot serve a scene is skipped for that scene with the reason
* recorded; when no pool route can, the run halts with every reason listed.
*
* `metered` routes spend money or remote quota. fal.ai video routes are priced
* from the curated fal catalog (lib/falVideoModels.js via `falSceneTake`): the
* run's start-time price is a cutaway take on the project's fal model, and each
* step is charged its own scene's estimate (a lip-sync take costs by its audio
* window and resolution). Other metered routes have no price catalog, so their
* price is unknown (null) and a run with a dollar cap refuses them; local
* routes cost $0.
*/
import { ServerError } from '../../lib/errorHandler.js';
import { FAL_IMAGE_FAMILIES, buildFalImageRequest, falImageFamily } from '../../lib/falImageModels.js';
import { resolveFalApiKey } from '../falQueue.js';
import { QUEUEABLE_IMAGE_MODES, VIDEO_GEN_MODES } from '../../lib/generationModes.js';
import { musicVideoConditioningReferences } from '../../lib/musicVideoConditioning.js';
import { MUSIC_VIDEO_AUTOMATION_TOOLS } from '../../lib/musicVideoAutomation.js';
import { falSceneTake, isPerformanceScene, performanceBlockedReason, runwaySceneTake } from '../../lib/musicVideoShotTiming.js';
import { maxInputImages, supportsCloudModelOverride } from '../../lib/imageGenCapabilities.js';
import { resolveVideoSupportedModes } from '../../lib/videoModeProfiles.js';
import { isHardwareCompatible } from '../../lib/systemCapabilities.js';
import { RUNNER_FAMILIES } from '../../lib/runners.js';
import { poolHasRoute, routeKey } from './production.js';
import { supportsSourceAudioWindow } from '../federatedMedia/sourceAudio.js';
async function localImageModel(route, env) {
if (route.model) return (env.imageModels || []).find((model) => model.id === route.model);
const { selectLocalImageModelFromSettings } = await import('../imageGen/prepareParams.js');
return selectLocalImageModelFromSettings(env.settings, '', env.imageModels || []);
}
const describe = (route) => `${route.kind} ${route.mode}${route.model ? ` (${route.model})` : ''}`;
const isMetered = (route) => MUSIC_VIDEO_AUTOMATION_TOOLS.some((t) => t.id === `${route.kind}:${route.mode}` && t.metered);
/**
* The render settings a fal route renders a scene with: the project's fal pins,
* with a route that names its own model overriding the cutaway model.
*/
export const falRouteVideoSettings = (project, route) => ({
...(project?.videoSettings || {}),
...(route?.model ? { falModelId: route.model } : {}),
});
/** The render settings a Runway route renders a scene with: the project's Runway pins, a route's own model winning. */
export const runwayRouteVideoSettings = (project, route) => ({
...(project?.videoSettings || {}),
...(route?.model ? { runwayModelId: route.model } : {}),
});
/**
* The price per generation this install can vouch for: 0 for local, the
* catalog estimate of a default-length cutaway take for a fal video route
* (null when its model is uncurated), null (unknown) for other metered routes.
*/
// A fal image route renders on the configured (or per-route) catalog model; when
// the route names none the configured one is unknown here, so price at the
// dearest catalog model — a dollar cap then bounds the worst case.
function falImageRoutePriceUsd(modelId) {
const estimate = (id) => buildFalImageRequest({ modelId: id, prompt: 'x' }).estimatedCostUsd;
if (falImageFamily(modelId)) return estimate(modelId);
return Math.max(...FAL_IMAGE_FAMILIES.map((f) => estimate(f.textEndpoint)));
}
function routePriceUsd(route, project = null) {
if (!isMetered(route)) return 0;
if (route.kind === 'video' && route.mode === 'fal') {
return falSceneTake({ scene: null, videoSettings: falRouteVideoSettings(project, route) }).costUsd;
}
if (route.kind === 'image' && route.mode === 'fal') return falImageRoutePriceUsd(route.model);
if (route.kind === 'video' && route.mode === 'runway') {
return runwaySceneTake({ scene: null, videoSettings: runwayRouteVideoSettings(project, route) }).costUsd;
}
return null;
}
/**
* What ONE step on `route` for `scene` should cost — the figure a step is
* charged against the dollar cap. For a fal clip it is that scene's own take
* (a performance by its audio window and lip-sync resolution, a cutaway by the
* length that covers the shot); otherwise the route's flat price.
*/
export function stepPriceUsd({ route, project, scene, stepKind }) {
if (stepKind !== 'frame' && route?.kind === 'video' && route.mode === 'runway' && scene) {
return runwaySceneTake({ scene, videoSettings: runwayRouteVideoSettings(project, route) }).costUsd;
}
if (stepKind !== 'frame' && route?.kind === 'video' && route.mode === 'fal' && scene) {
return falSceneTake({
scene,
videoSettings: falRouteVideoSettings(project, route),
songDurationSec: project?.audioAnalysis?.durationSec ?? null,
}).costUsd;
}
return routePriceUsd(route, project);
}
/**
* The paid model a step's price is for, recorded on the step so project spend
* can be broken down by model: the fal clip model `stepPriceUsd` priced (an
* end-frame upgrade included), a fal image route's own model, else null.
*/
export function stepCostModelId({ route, project, scene, stepKind }) {
if (route?.mode === 'runway' && route.kind === 'video') {
return scene ? runwaySceneTake({ scene, videoSettings: runwayRouteVideoSettings(project, route) }).modelId : route.model || null;
}
if (route?.mode !== 'fal') return null;
if (stepKind !== 'frame' && route.kind === 'video' && scene) {
return falSceneTake({ scene, videoSettings: falRouteVideoSettings(project, route) }).modelId;
}
return route.model || null;
}
/** routeKey → price map for a pool (fal routes priced for `project`). */
export const poolPricing = (pool, project = null) => Object.fromEntries(pool.map((r) => [routeKey(r), routePriceUsd(r, project)]));
/** Shared scene-aware frame conditioning, with the legacy capped fallback. */
export const conditioningReferences = musicVideoConditioningReferences;
/** What a scene's `kind` generation needs from a route. `castMedia` is loadSceneCastMedia's result for the scene. */
export function sceneRequirement(project, scene, stepKind, castMedia = null) {
return stepKind === 'frame'
? { kind: 'image', conditioning: conditioningReferences(project, scene, castMedia).length }
: { kind: 'video', performance: isPerformanceScene(scene) };
}
/**
* The live facts eligibility reads. Loaded lazily (the catalogs are heavy)
* and injectable for tests.
*/
export async function loadPoolEnv() {
const [{ getSettings }, { getImageModels }, { resolveVideoModelSelection }] = await Promise.all([
import('../settings.js'),
import('../../lib/mediaModels.js'),
import('../videoGen/modelSelection.js'),
]);
const { isVideoModeUsable } = await import('../videoGen/modes.js');
return {
settings: await getSettings(),
imageModels: getImageModels(),
resolveVideoModel: (modelId) => resolveVideoModelSelection(modelId),
isVideoModeUsable,
};
}
/**
* Is `route` runnable on this install right now? Returns `{ ok, reason }`.
* Covers the explicit opt-in toggles (a cloud CLI spends quota only when
* enabled), configured API keys, installed + hardware-compatible local
* models, and whether a named cloud model can actually be selected.
*/
async function routeEligibility(route, env) {
const { settings } = env;
if (route.kind === 'image') {
if (!QUEUEABLE_IMAGE_MODES.includes(route.mode)) {
return { ok: false, reason: `${describe(route)} renders synchronously and cannot run unattended; choose a queued image backend` };
}
if (route.mode === 'local') {
if (!settings?.imageGen?.local?.pythonPath) return { ok: false, reason: 'The local image runtime is not configured in Settings' };
const model = await localImageModel(route, env);
if (!model) return { ok: false, reason: `Local image model "${route.model || '(none)'}" is not installed` };
if (!isHardwareCompatible(model.hardwareCompatibility)) return { ok: false, reason: `Local image model "${model.id}" cannot run on this hardware` };
return { ok: true, reason: null };
}
if (settings?.imageGen?.[route.mode]?.enabled !== true) {
return { ok: false, reason: `${describe(route)} is not enabled in Settings → Image Gen` };
}
if (route.mode === 'fal' && !resolveFalApiKey(settings)) {
return { ok: false, reason: 'fal.ai has no API key — add one in Settings → Image Gen or set FAL_KEY' };
}
const configured = settings.imageGen[route.mode].model || null;
if (route.model && route.model !== configured && !supportsCloudModelOverride(route.mode)) {
return { ok: false, reason: `${route.mode} image gen cannot select model "${route.model}"; it renders with its configured model only` };
}
return { ok: true, reason: null };
}
if (!VIDEO_GEN_MODES.includes(route.mode)) return { ok: false, reason: `Unknown video backend "${route.mode}"` };
if (!env.isVideoModeUsable(settings, route.mode)) {
return { ok: false, reason: `${describe(route)} is not usable — enable it or configure its API key in Settings` };
}
if (route.mode === 'local') {
if (!settings?.imageGen?.local?.pythonPath) return { ok: false, reason: 'The local video runtime is not configured in Settings' };
// A blank pin is the install default. The renderer resolves that same
// omission, so refusing it here made "Install default" fail at Start.
const requested = typeof route.model === 'string' ? route.model.trim() : '';
const { model, modelId } = await env.resolveVideoModel(requested);
const label = requested || modelId || '(none)';
if (!model) return { ok: false, reason: `Local video model "${label}" is not installed` };
if (!isHardwareCompatible(model.hardwareCompatibility)) return { ok: false, reason: `Local video model "${label}" cannot run on this hardware` };
// Footage always animates a generated or chosen frame, so the model must take an image.
if (!resolveVideoSupportedModes(model).includes('image')) {
return { ok: false, reason: `Local video model "${model.name || label}" does not support image-to-video, which footage production needs — choose another video model` };
}
}
return { ok: true, reason: null };
}
/** Why `route` cannot serve this scene requirement, or null when it can. */
async function routeIncapableReason(route, requirement, env) {
if (route.kind !== requirement.kind) return `${describe(route)} does not generate ${requirement.kind === 'image' ? 'frames' : 'clips'}`;
if (requirement.kind === 'image' && requirement.conditioning > 0) {
const limit = route.mode === 'local' ? null : maxInputImages(route.mode);
if (limit != null && requirement.conditioning > limit) {
return `${describe(route)} takes at most ${limit} reference image${limit === 1 ? '' : 's'}; the visual spec conditions on ${requirement.conditioning}`;
}
if (route.mode === 'local') {
const model = await localImageModel(route, env);
if (model?.runner !== RUNNER_FAMILIES.FLUX2 && model?.pipelineClass !== 'QwenImage21Pipeline') {
return `Local model "${model?.id || route.model || '(none)'}" cannot use reference images (FLUX.2 and Qwen Image 2.1 only)`;
}
}
}
if (requirement.kind === 'video' && requirement.performance) {
if (route.mode === 'local') {
// Local lip-sync is LTX audio-to-video, which only some local models run.
const requested = typeof route.model === 'string' ? route.model.trim() : '';
const { model, modelId } = await env.resolveVideoModel(requested);
if (!supportsSourceAudioWindow(model)) {
return `Local video model "${model?.name || requested || modelId || '(none)'}" cannot lip-sync to the song; choose an LTX-2.5 (or LTX-2.3 dgrauet) model`;
}
}
return performanceBlockedReason(route.mode);
}
return null;
}
/**
* Why `route` cannot generate `requirement` on this install right now, or null
* — the eligibility + capability checks a production step passes, for a caller
* with no pool to choose from (a standalone auto-review run, #10014).
*/
export async function routeUnavailableReason(route, requirement, env) {
const { ok, reason } = await routeEligibility(route, env);
return ok ? routeIncapableReason(route, requirement, env) : reason;
}
/**
* Refuse a pinned local video model that cannot animate a frame, so a text-only
* model fails the start request instead of the first clip. Narrower than
* assertPoolEligible: it ignores install state (keys, toggles) the run may still fix.
*/
export async function assertFootageVideoModelsCapable(pool, env) {
for (const route of pool) {
const pinned = route.kind === 'video' && route.mode === 'local' && typeof route.model === 'string' ? route.model.trim() : '';
if (!pinned) continue;
const { model } = await env.resolveVideoModel(pinned);
if (model && !resolveVideoSupportedModes(model).includes('image')) {
throw new ServerError(`Local video model "${model.name || pinned}" does not support image-to-video, which footage production needs — choose another video model`, { status: 400, code: 'VIDEO_MODEL_TEXT_ONLY', context: { model: pinned } });
}
}
}
/** Start-time check: every route in the pool must be eligible now. Throws 409 with every reason. */
export async function assertPoolEligible(pool, env) {
const problems = [];
for (const route of pool) {
const { ok, reason } = await routeEligibility(route, env);
if (!ok) problems.push(reason);
}
if (problems.length) {
throw new ServerError(`Some allowed routes cannot run: ${problems.join('; ')}`, { status: 409, code: 'PRODUCTION_ROUTE_INELIGIBLE', context: { problems } });
}
}
/**
* Dispatch-time enforcement for ONE chosen route: it must be a literal member
* of the run's pool, eligible right now, and capable of this scene. Throws
* 409 before anything is enqueued — there is never a fallback to another route.
*/
export async function assertRouteAllowed(run, route, requirement, env) {
if (!poolHasRoute(run, route)) {
throw new ServerError(`${route ? describe(route) : 'That route'} is not in this run's allowed pool`, { status: 409, code: 'PRODUCTION_ROUTE_NOT_ALLOWED' });
}
const { ok, reason } = await routeEligibility(route, env);
if (!ok) throw new ServerError(reason, { status: 409, code: 'PRODUCTION_ROUTE_INELIGIBLE' });
const incapable = await routeIncapableReason(route, requirement, env);
if (incapable) throw new ServerError(incapable, { status: 409, code: 'PRODUCTION_ROUTE_INCAPABLE' });
}
/**
* The default chooser: the first route in the director's pool order that is
* eligible and capable. Returns `{ route, rationale }`, or `{ route: null,
* reasons }` listing why every candidate was passed over.
*/
export async function chooseProductionRoute(run, requirement, env) {
const reasons = [];
const candidates = run.pool.filter((r) => r.kind === requirement.kind);
if (!candidates.length) return { route: null, reasons: [`The allowed pool has no ${requirement.kind} route`] };
for (const route of candidates) {
const { ok, reason } = await routeEligibility(route, env);
if (!ok) { reasons.push(reason); continue; }
const incapable = await routeIncapableReason(route, requirement, env);
if (incapable) { reasons.push(incapable); continue; }
const skipped = reasons.length ? ` (skipped ${reasons.length} earlier route${reasons.length === 1 ? '' : 's'}: ${reasons.join('; ')})` : '';
const need = requirement.kind === 'image'
? (requirement.conditioning ? `frame conditioned on ${requirement.conditioning} reference image${requirement.conditioning === 1 ? '' : 's'}` : 'unconditioned frame')
: (requirement.performance ? 'lip-synced performance shot' : 'cutaway clip');
return { route, rationale: `First eligible ${requirement.kind} route in the allowed pool for ${/^[aeiou]/.test(need) ? 'an' : 'a'} ${need}${skipped}` };
}
return { route: null, reasons };
}