Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Add Runway as a paid video renderer (#12344)

[._.]/240a5545fbe0
diff --git a/.env.example b/.env.example
index 784ec40971..6c862bcc8a 100644
--- a/.env.example
+++ b/.env.example
@@ -151,6 +151,11 @@ PGPASSWORD=portos
# (videoGen/reactor.js; default: 1200000 / 20 minutes) — same reasoning as
# FAL_VIDEO_TIMEOUT_MS above.
# REACTOR_VIDEO_TIMEOUT_MS=1200000
+
+# Maximum Runway video-generation runtime in milliseconds (videoGen/runway.js;
+# default: 1200000 / 20 minutes) — a task can wait THROTTLED behind the
+# account's concurrency limit, so this bounds the whole submit→poll→download run.
+# RUNWAY_VIDEO_TIMEOUT_MS=1200000
# Optional Python executable in an environment with reactor-sdk installed.
# Defaults to data/venvs/reactor created by npm run setup:reactor.
# REACTOR_PYTHON_PATH=
@@ -203,6 +208,10 @@ PGPASSWORD=portos
# page, FableLoom). Settings > Video Gen's stored key wins when both are set.
# REACTOR_API_KEY=your_reactor_api_key_here
+# Runway developer API key for the Runway video backend (Video Gen page, Music
+# Video shots). Settings > Video Gen's stored key wins when both are set.
+# RUNWAYML_API_SECRET=your_runway_api_key_here
+
# Artificial Analysis API key for synchronizing model quality, speed, and pricing comparison benchmarks
# ARTIFICIAL_ANALYSIS_API_KEY=your_artificial_analysis_api_key_here
diff --git a/client/src/components/musicVideo/SceneEndFrame.jsx b/client/src/components/musicVideo/SceneEndFrame.jsx
index 6a2ae5e7f9..bf46ada18f 100644
--- a/client/src/components/musicVideo/SceneEndFrame.jsx
+++ b/client/src/components/musicVideo/SceneEndFrame.jsx
@@ -1,5 +1,5 @@
import { sceneEndFramePlan } from '../../../../server/lib/musicVideoEndFrame.js';
-import { falSceneTake } from '../../lib/musicVideoShotTiming.js';
+import { falSceneTake, runwaySceneTake } from '../../lib/musicVideoShotTiming.js';
/**
* Whether a clip rendered with `settings` (the project's saved renderer, or a
@@ -12,8 +12,9 @@ import { falSceneTake } from '../../lib/musicVideoShotTiming.js';
export function sceneEndFramePlanFor(scene, settings = {}, { models = [], defaultModel = '' } = {}) {
const mode = settings.generationMode === 'suppliedAudio' || (settings.backend === 'local' && settings.generationMode === 'audioReactive') ? 'a2v' : 'image';
const falModelId = settings.backend === 'fal' ? falSceneTake({ scene, videoSettings: settings }).modelId : null;
+ const runwayModelId = settings.backend === 'runway' ? runwaySceneTake({ scene, videoSettings: settings }).modelId : null;
const localRuntime = settings.backend === 'local'
? models.find((model) => model.id === (settings.modelId || defaultModel))?.runtime ?? null
: null;
- return sceneEndFramePlan(scene, { backend: settings.backend || null, falModelId, mode, localRuntime });
+ return sceneEndFramePlan(scene, { backend: settings.backend || null, falModelId, runwayModelId, mode, localRuntime });
}
diff --git a/client/src/components/musicVideo/SceneRenderInputs.jsx b/client/src/components/musicVideo/SceneRenderInputs.jsx
index 0dd28931b1..d7d549dacf 100644
--- a/client/src/components/musicVideo/SceneRenderInputs.jsx
+++ b/client/src/components/musicVideo/SceneRenderInputs.jsx
@@ -1,5 +1,6 @@
import { falHearsAudioReference, getFalVideoModel } from '../../lib/falVideoModels.js';
-import { falSceneTake } from '../../lib/musicVideoShotTiming.js';
+import { falSceneTake, runwaySceneTake } from '../../lib/musicVideoShotTiming.js';
+import { getRunwayVideoModel } from '../../lib/runwayVideoModels.js';
import { shotActionContractProblem } from '../../../../server/lib/musicVideoActionContract.js';
import { authoredShotPrompt, musicVideoShotModelInput } from '../../../../server/lib/musicVideoShotPrompt.js';
import { localModelTakesSongAudio } from '../../../../server/lib/musicVideoLocalModels.js';
@@ -33,6 +34,7 @@ export function shotCardRenderRequest({
project = null, scene, settings = {}, performance = false, endFramePlan = null, models = [], defaultModel = '', inputs = {}, plan = null,
}) {
const falModel = settings.backend === 'fal' ? getFalVideoModel(falSceneTake({ scene, videoSettings: settings }).modelId) : null;
+ const runwayModel = settings.backend === 'runway' ? getRunwayVideoModel(runwaySceneTake({ scene, videoSettings: settings }).modelId) : null;
const localModel = settings.backend === 'local'
? models.find((model) => model.id === (settings.modelId || defaultModel)) || null : null;
const span = `${clock(scene?.startSec)}–${clock(scene?.endSec)}`;
@@ -82,7 +84,7 @@ export function shotCardRenderRequest({
prompt,
promptAdditions,
promptUsed: falModel?.prompt !== 'none',
- maxChars: falModel?.promptMaxChars ?? null,
+ maxChars: falModel?.promptMaxChars ?? runwayModel?.promptMaxChars ?? null,
endFrame,
songAudio,
wholeLine,
diff --git a/client/src/components/musicVideo/SceneRenderTarget.jsx b/client/src/components/musicVideo/SceneRenderTarget.jsx
index 9ccc710acf..22d2ff4e64 100644
--- a/client/src/components/musicVideo/SceneRenderTarget.jsx
+++ b/client/src/components/musicVideo/SceneRenderTarget.jsx
@@ -2,6 +2,9 @@ import {
FAL_DEFAULT_IMAGE_VIDEO_MODEL, FAL_IMAGE_VIDEO_MODELS, coerceFalVideoSeconds, describeFalVideoRate, falVideoResolutions, getFalVideoModel,
} from '../../lib/falVideoModels.js';
import { GROK_VIDEO_DURATIONS } from '../../lib/grokVideoClip.js';
+import {
+ RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL, RUNWAY_VIDEO_MODELS, coerceRunwayVideoSeconds, describeRunwayVideoRate, getRunwayVideoModel, runwayVideoLengths, runwayVideoResolutions,
+} from '../../lib/runwayVideoModels.js';
import { SOURCE_AUDIO_MUSIC, grokCoverage, performanceCapability } from '../../lib/musicVideoShotTiming.js';
import { localMusicVideoModels } from '../../../../server/lib/musicVideoLocalModels.js';
import { frameOptionsForModel } from '../../lib/videoGenParams.js';
@@ -13,20 +16,22 @@ import { safeReadJsonStorage, safeWriteJsonStorage } from '../../lib/safeStorage
* offers only its curated start-frame models; a lip-synced shot renders on
* fal's lip-sync route, so fal is one choice there, a music-driven shot offers
* each music-driven model (the song slice drives the picture), and Grok (no
- * audio) renders cutaways only.
- * The pick is a value string: '' (saved), 'local', 'grok', 'fal' or 'fal:<modelId>'.
+ * audio) and Runway (image-to-video only here) render cutaways only.
+ * The pick is a value string: '' (saved), 'local', 'grok', 'fal', 'fal:<modelId>'
+ * or 'runway:<modelId>'.
*
* A picked lane shows its own options, from the same definitions the Video Gen
* form and the project's render settings read: the local model and clip length
* (frame counts), Grok's clip lengths, and a fal model's lengths and
- * resolutions. Each length defaults to the shortest one that covers the shot.
- * `options` holds only what the director changed: `{ modelId, numFrames,
- * grokDuration, falDuration, falResolution }`.
+ * resolutions (a Runway model's likewise). Each length defaults to the
+ * shortest one that covers the shot. `options` holds only what the director
+ * changed: `{ modelId, numFrames, grokDuration, falDuration, falResolution,
+ * runwayDuration, runwayResolution }`.
*
* A card opens on the last pick made on this device (lastRenderTarget), with
* its local model or fal size; lengths stay sized to each shot.
*/
-const BACKEND_LABELS = { local: 'Local LTX', grok: 'Grok', fal: 'fal.ai' };
+const BACKEND_LABELS = { local: 'Local LTX', grok: 'Grok', fal: 'fal.ai', runway: 'Runway' };
// The local render's frame rate when the request names none (generateVideo's default).
const LOCAL_FPS = 24;
const SELECT = 'min-w-0 max-w-full bg-port-bg border border-port-border rounded px-1 py-1 text-xs min-h-[44px] sm:min-h-0 disabled:opacity-50';
@@ -42,6 +47,9 @@ function savedLabel(saved, performance, scene) {
if (saved.backend === 'fal') {
return `fal.ai · ${performance ? falPerformanceLabel(saved, scene) : getFalVideoModel(saved.falModelId || FAL_DEFAULT_IMAGE_VIDEO_MODEL)?.label || 'video'}`;
}
+ if (saved.backend === 'runway' && !performance) {
+ return `Runway · ${getRunwayVideoModel(saved.runwayModelId || RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL)?.label || 'video'}`;
+ }
return BACKEND_LABELS[saved.backend] || 'install default';
}
@@ -63,6 +71,13 @@ export function renderTargetOptions({ performance, saved, endFrame = false, scen
const rate = describeFalVideoRate(model.id);
return { value: `fal:${model.id}`, label: `fal.ai · ${model.label}${rate ? ` · ${rate}` : ''}${endFrame && model.endImage ? ' · ends on end frame' : ''}` };
})),
+ // Runway takes no song audio, so a performance shot cannot render there.
+ ...(performance
+ ? [{ value: 'runway', label: 'Runway (cutaways only)', disabled: true }]
+ : RUNWAY_VIDEO_MODELS.map((model) => {
+ const rate = describeRunwayVideoRate(model.id);
+ return { value: `runway:${model.id}`, label: `Runway · ${model.label}${rate ? ` · ${rate}` : ''}${endFrame && model.endImage ? ' · ends on end frame' : ''}` };
+ })),
];
}
@@ -70,7 +85,9 @@ const LAST_PICK_KEY = 'portos.musicVideo.lastRenderOn';
/** Remember a card's pick for the next card opened on this device. */
export function rememberRenderTarget(pick, options = {}) {
- safeWriteJsonStorage(LAST_PICK_KEY, { pick, modelId: options.modelId || null, falResolution: options.falResolution || null });
+ safeWriteJsonStorage(LAST_PICK_KEY, {
+ pick, modelId: options.modelId || null, falResolution: options.falResolution || null, runwayResolution: options.runwayResolution || null,
+ });
}
/**
@@ -84,8 +101,10 @@ export function lastRenderTarget({ performance, saved }) {
if (performance && pick.startsWith('fal:')) pick = 'fal';
if (!renderTargetOptions({ performance, saved }).some((option) => option.value === pick && !option.disabled)) pick = '';
const falModel = pick.startsWith('fal:') ? getFalVideoModel(pick.slice(4)) : null;
+ const runwayModel = pick.startsWith('runway:') ? getRunwayVideoModel(pick.slice(7)) : null;
const options = pick === 'local' && last.modelId ? { modelId: last.modelId }
- : falModel && falVideoResolutions(falModel).includes(last.falResolution) ? { falResolution: last.falResolution } : {};
+ : falModel && falVideoResolutions(falModel).includes(last.falResolution) ? { falResolution: last.falResolution }
+ : runwayModel && runwayVideoResolutions(runwayModel).includes(last.runwayResolution) ? { runwayResolution: last.runwayResolution } : {};
return { pick, options };
}
@@ -147,6 +166,18 @@ export function renderTargetOverrides(value, saved, options = {}, context = {})
if (value === 'grok') {
return { backend: 'grok', ...(options.grokDuration || spanSec != null ? { grokDuration: options.grokDuration || grokCoverage(spanSec).requestSec } : {}) };
}
+ if (value.startsWith('runway')) {
+ const runwayModelId = value.slice(7) || null;
+ if (!runwayModelId) return { backend: 'runway' };
+ // Lengths and sizes are per model, so the saved ones only carry over to the saved model.
+ const sameModel = runwayModelId === (saved.runwayModelId || RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL);
+ return {
+ backend: 'runway', runwayModelId,
+ ...(sameModel ? {} : { runwayDuration: null, runwayResolution: null }),
+ ...(options.runwayDuration ? { runwayDuration: options.runwayDuration } : {}),
+ ...(options.runwayResolution ? { runwayResolution: options.runwayResolution } : {}),
+ };
+ }
const falModelId = value.slice(4) || null;
if (!falModelId) return { backend: 'fal' };
// A music-driven shot's pick is its music model, at that model's own default size.
@@ -188,6 +219,7 @@ export default function SceneRenderTarget({
const localChoices = localModelChoices(models, performance);
const localModel = value === 'local' ? models.find((model) => model.id === overrides.modelId) : null;
const falModel = value.startsWith('fal:') ? getFalVideoModel(overrides.falModelId) : null;
+ const runwayModel = value.startsWith('runway:') ? getRunwayVideoModel(overrides.runwayModelId) : null;
const coversLabel = (sec) => (spanSec != null ? `${formatSec(sec)} · covers shot` : `${formatSec(sec)} · default`);
return (
<>
@@ -249,6 +281,26 @@ export default function SceneRenderTarget({
)}
</>
)}
+ {runwayModel && !performance && (
+ <>
+ {runwayVideoLengths(runwayModel).length > 0 && (
+ <OptionSelect id={`${id}-length`} label="Length" value={options.runwayDuration || ''} disabled={disabled}
+ onChange={(sec) => set({ runwayDuration: Number(sec) || null })}>
+ <option value="">{coversLabel(coerceRunwayVideoSeconds(runwayModel, spanSec))}</option>
+ {runwayVideoLengths(runwayModel).map((sec) => <option key={sec} value={sec}>{formatSec(sec)}</option>)}
+ </OptionSelect>
+ )}
+ {runwayVideoResolutions(runwayModel).length > 1 && (
+ <OptionSelect id={`${id}-resolution`} label="Size" value={options.runwayResolution || ''} disabled={disabled}
+ onChange={(runwayResolution) => set({ runwayResolution: runwayResolution || null })}>
+ <option value="">{`${runwayModel.size.default} · default`}</option>
+ {runwayVideoResolutions(runwayModel).filter((res) => res !== runwayModel.size.default).map((res) => (
+ <option key={res} value={res}>{`${res} · ${describeRunwayVideoRate(runwayModel.id, res)}`}</option>
+ ))}
+ </OptionSelect>
+ )}
+ </>
+ )}
</>
);
}
diff --git a/client/src/components/musicVideo/SceneTimingPlan.jsx b/client/src/components/musicVideo/SceneTimingPlan.jsx
index 5d02d18fba..1b20303f7e 100644
--- a/client/src/components/musicVideo/SceneTimingPlan.jsx
+++ b/client/src/components/musicVideo/SceneTimingPlan.jsx
@@ -1,7 +1,7 @@
import { useState } from 'react';
import { AlertTriangle, Merge, Scissors } from 'lucide-react';
import { formatDurationSec } from '../../utils/formatters.js';
-import { getFalVideoModel } from '../../lib/falVideoModels.js';
+import { meteredModelLabel } from '../../lib/musicVideoShotTiming.js';
const SPLIT_BUTTON = 'inline-flex items-center gap-1 rounded bg-port-bg border border-port-border hover:bg-port-border/40 px-2 py-1 text-xs min-h-[44px] sm:min-h-0';
// The shortest piece the server leaves on either side of a split (projectsLogic MIN_SPLIT_PIECE_SEC).
@@ -58,8 +58,8 @@ export default function SceneTimingPlan({ timing, sceneId, lipSyncBackend, onSpl
)}
{falTake && !falTake.performance && (
<p className="text-[11px] text-port-text-muted break-words" data-testid="fal-cutaway-plan">
- fal.ai renders {falTake.seconds != null ? `a ${falTake.seconds}s ` : 'a '}clip on {getFalVideoModel(falTake.modelId)?.label || falTake.modelId}
- {' · '}{falCost || 'cost unknown — billed to your fal.ai account'}
+ {lipSyncBackend === 'runway' ? 'Runway' : 'fal.ai'} renders {falTake.seconds != null ? `a ${falTake.seconds}s ` : 'a '}clip on {meteredModelLabel(falTake.modelId) || falTake.modelId}
+ {' · '}{falCost || `cost unknown — billed to your ${lipSyncBackend === 'runway' ? 'Runway' : 'fal.ai'} account`}
</p>
)}
{grokPlan && (
diff --git a/client/src/components/musicVideo/VideoRenderSettings.jsx b/client/src/components/musicVideo/VideoRenderSettings.jsx
index 5c9d14bf6d..e0cd7ed094 100644
--- a/client/src/components/musicVideo/VideoRenderSettings.jsx
+++ b/client/src/components/musicVideo/VideoRenderSettings.jsx
@@ -3,6 +3,9 @@ import { GROK_VIDEO_DURATIONS } from '../../lib/grokVideoClip.js';
import {
FAL_DEFAULT_IMAGE_VIDEO_MODEL, FAL_MUSIC_VIDEO_DEFAULT_MODEL, FAL_IMAGE_VIDEO_MODELS, describeFalVideoRate, falImageVideoQuotes, falVideoResolutions, getFalVideoModel,
} from '../../lib/falVideoModels.js';
+import {
+ RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL, RUNWAY_VIDEO_MODELS, describeRunwayVideoRate, getRunwayVideoModel, runwayVideoLengths, runwayVideoResolutions,
+} from '../../lib/runwayVideoModels.js';
import { projectFalSpend } from '../../lib/musicVideoStages.js';
import { formatUsd } from '../../utils/formatters.js';
import { SOURCE_AUDIO_LIPSYNC, SOURCE_AUDIO_MUSIC, performanceCapability } from '../../lib/musicVideoShotTiming.js';
@@ -13,6 +16,10 @@ const falOptionLabel = (model) => {
const rate = describeFalVideoRate(model.id);
return rate ? `${model.label} · ${rate}` : model.label;
};
+const runwayOptionLabel = (model, resolution) => {
+ const rate = describeRunwayVideoRate(model.id, resolution);
+ return rate ? `${model.label} · ${rate}` : model.label;
+};
// The project's saved scene-video render pins (backend → generation mode →
// model → audio-reactive LoRA/strength, or Grok clip duration). Every control
@@ -46,6 +53,12 @@ export default function VideoRenderSettings({ videoSettings, generating, project
// only; each option shows what a typical short (2 s) shot bills.
const endFrameQuotes = falImageVideoQuotes({ seconds: 2, endFrame: true });
const falSpend = project ? projectFalSpend(project) : null;
+ // Runway is image-to-video only here, so it renders cutaways; performance
+ // shots stay on the lip-sync lane. Length and size alphabets are per model,
+ // so a model change clears both pins.
+ const runwayModel = getRunwayVideoModel(settings.runwayModelId || RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL);
+ const runwayResolutions = runwayVideoResolutions(runwayModel);
+ const runwayLengths = runwayVideoLengths(runwayModel);
return (
<>
<RenderPoolPicker settings={settings} change={change} disabled={locked} />
@@ -64,6 +77,8 @@ export default function VideoRenderSettings({ videoSettings, generating, project
performance (lip-sync) shots need fal.ai (#8977). */}
<option value="grok">Grok video (cutaway only)</option>
<option value="fal">fal.ai video</option>
+ {/* Runway is image-to-video only, so like Grok it renders cutaways. */}
+ <option value="runway">Runway video (cutaway only)</option>
</select>
{settings.backend === 'local' && (
<>
@@ -302,9 +317,66 @@ export default function VideoRenderSettings({ videoSettings, generating, project
</select>
</>
)}
+ {settings.backend === 'runway' && (
+ <>
+ <label htmlFor="mv-runway-model" className="sr-only">Runway cutaway model</label>
+ <select
+ id="mv-runway-model"
+ value={settings.runwayModelId || ''}
+ onChange={(e) => change({ runwayModelId: e.target.value || null, runwayDuration: null, runwayResolution: null })}
+ disabled={locked}
+ title="Runway image-to-video model for this project's cutaway scenes (list price per generated second)"
+ className={`${falSelectCls} sm:max-w-[280px]`}
+ >
+ <option value="">{`Default · ${runwayOptionLabel(getRunwayVideoModel(RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL))}`}</option>
+ {RUNWAY_VIDEO_MODELS.filter((model) => model.id !== RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL).map((model) => (
+ <option key={model.id} value={model.id}>{runwayOptionLabel(model)}</option>
+ ))}
+ {settings.runwayModelId && !runwayModel && (
+ <option value={settings.runwayModelId}>{`${settings.runwayModelId} · cost unknown`}</option>
+ )}
+ </select>
+ {runwayResolutions.length > 1 && (
+ <>
+ <label htmlFor="mv-runway-resolution" className="sr-only">Runway cutaway resolution</label>
+ <select
+ id="mv-runway-resolution"
+ value={runwayResolutions.includes(settings.runwayResolution) ? settings.runwayResolution : ''}
+ onChange={(e) => change({ runwayResolution: e.target.value || null })}
+ disabled={locked}
+ title="Output resolution for Runway cutaway renders — higher can cost more per second"
+ className={falSelectCls}
+ >
+ <option value="">{`${runwayModel.size.default} (model default)`}</option>
+ {runwayResolutions.map((res) => (
+ <option key={res} value={res}>{`${res} · ${describeRunwayVideoRate(runwayModel.id, res)}`}</option>
+ ))}
+ </select>
+ </>
+ )}
+ {runwayLengths.length > 0 && (
+ <>
+ <label htmlFor="mv-runway-duration" className="sr-only">Runway scene clip length</label>
+ <select
+ id="mv-runway-duration"
+ value={runwayLengths.includes(settings.runwayDuration) ? settings.runwayDuration : ''}
+ onChange={(e) => change({ runwayDuration: e.target.value === '' ? null : Number(e.target.value) })}
+ disabled={locked}
+ title="Clip length for this project's Runway cutaway renders. Shot length renders the shortest length the model offers that covers each shot."
+ className={falSelectCls}
+ >
+ <option value="">Shot length</option>
+ {runwayLengths.map((sec) => (
+ <option key={sec} value={sec}>{`${sec}s clips`}</option>
+ ))}
+ </select>
+ </>
+ )}
+ </>
+ )}
{falSpend?.totalUsd > 0 && (
- <p className="w-full min-w-0 break-words text-xs text-port-text-muted" title="Estimated from the price each fal render was quoted at; fal's own invoice is the final word">
- {`fal spend (est.) ${formatUsd(falSpend.totalUsd)}: `}
+ <p className="w-full min-w-0 break-words text-xs text-port-text-muted" title="Estimated from the price each paid render was quoted at; the provider's own invoice is the final word">
+ {`Paid video spend (est.) ${formatUsd(falSpend.totalUsd)}: `}
{falSpend.models.map((row) => `${row.label} ${formatUsd(row.usd)} (${row.count})`).join(' · ')}
</p>
)}
diff --git a/client/src/components/settings/ImageGenBackendSections.jsx b/client/src/components/settings/ImageGenBackendSections.jsx
index 8993e829ff..89a41cb5ca 100644
--- a/client/src/components/settings/ImageGenBackendSections.jsx
+++ b/client/src/components/settings/ImageGenBackendSections.jsx
@@ -42,6 +42,11 @@ const VIDEO_KEY_ROWS = [
hint: 'Enables the reactor.inc fast-h3 video backend on the Video Gen page and in FableLoom, or set the REACTOR_API_KEY environment variable instead.',
missing: 'No reactor.inc key configured — the reactor.inc video backend will be unavailable.',
},
+ {
+ id: 'runway', label: 'Runway API key', placeholder: 'key_...', envLabel: 'RUNWAYML_API_SECRET',
+ hint: 'Enables the Runway video backend on the Video Gen page and in Music Video shots. Get a key at dev.runwayml.com, or set the RUNWAYML_API_SECRET environment variable instead.',
+ missing: 'No Runway key configured — the Runway video backend will be unavailable.',
+ },
];
function VideoKeyRow({ row, spec, busy, onSave, onClear }) {
diff --git a/client/src/components/settings/ImageGenTab.jsx b/client/src/components/settings/ImageGenTab.jsx
index 0d73e70ed4..33f8583242 100644
--- a/client/src/components/settings/ImageGenTab.jsx
+++ b/client/src/components/settings/ImageGenTab.jsx
@@ -272,7 +272,7 @@ export function ImageGenTab() {
// (GET /api/settings/credentials, same one CredentialsTab renders) and
// saves/clears through the write-only PUT /api/settings/credentials/:id
// setter — never through the slice PATCHes this tab otherwise uses.
- const [credentialRows, setCredentialRows] = useState({ fal: null, reactor: null });
+ const [credentialRows, setCredentialRows] = useState({ fal: null, reactor: null, runway: null });
// One busy flag for both keys since only one save/clear can be in flight
// from this panel at a time; the value names which button is spinning.
const [videoCredentialBusy, setVideoCredentialBusy] = useState(null);
@@ -284,6 +284,7 @@ export function ImageGenTab() {
setCredentialRows({
fal: rows.find((row) => row.id === 'fal') || null,
reactor: rows.find((row) => row.id === 'reactor') || null,
+ runway: rows.find((row) => row.id === 'runway') || null,
});
})
.catch(() => {});
diff --git a/client/src/hooks/useMusicVideoModelSettings.js b/client/src/hooks/useMusicVideoModelSettings.js
index 1a36a6986f..5973e31a13 100644
--- a/client/src/hooks/useMusicVideoModelSettings.js
+++ b/client/src/hooks/useMusicVideoModelSettings.js
@@ -27,6 +27,7 @@ export default function useMusicVideoModelSettings({ project, onProjectPatch } =
const [loras, setLoras] = useState([]);
const [defaultModel, setDefaultModel] = useState('');
const [falEnabled, setFalEnabled] = useState(false);
+ const [runwayEnabled, setRunwayEnabled] = useState(false);
const [saving, setSaving] = useState(false);
const [framePinSaving, setFramePinSaving] = useState(false);
@@ -43,6 +44,7 @@ export default function useMusicVideoModelSettings({ project, onProjectPatch } =
// isVideoModeUsable() check a render itself is gated on (mirrors the
// general Video Gen page's `falEnabled`).
setFalEnabled(status?.falEnabled === true);
+ setRunwayEnabled(status?.runwayEnabled === true);
setModelsLoading(false);
})
.catch(() => {
@@ -82,6 +84,12 @@ export default function useMusicVideoModelSettings({ project, onProjectPatch } =
// null means FAL_MUSIC_VIDEO_DEFAULT_MODEL (MiniMax H3 Max reference) at its own default size, resolved by falSceneTake.
falMusicModelId: project?.videoSettings?.falMusicModelId ?? null,
falMusicResolution: project?.videoSettings?.falMusicResolution ?? null,
+ // Runway cutaway model / clip length / size — null means the catalog
+ // default (RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL, the shot's covering length,
+ // the project aspect), resolved by runwaySceneTake.
+ runwayModelId: project?.videoSettings?.runwayModelId ?? null,
+ runwayDuration: project?.videoSettings?.runwayDuration ?? null,
+ runwayResolution: project?.videoSettings?.runwayResolution ?? null,
generationMode: project?.videoSettings?.generationMode || 'image',
audioReactiveLora: project?.videoSettings?.audioReactiveLora || '',
audioReactiveScale: project?.videoSettings?.audioReactiveScale ?? 1.2,
@@ -112,7 +120,9 @@ export default function useMusicVideoModelSettings({ project, onProjectPatch } =
? 'Audio-reactive generation requires an installed LTX-2.3 audio-reactive LoRA and an LTX-2.3 local model'
: pick.backend === 'fal' && !falEnabled
? 'No fal.ai API key configured — set it in Settings → Video Gen (or the FAL_KEY env var) first'
- : null);
+ : pick.backend === 'runway' && !runwayEnabled
+ ? 'No Runway API key configured — set it in Settings → Video Gen (or the RUNWAYML_API_SECRET env var) first'
+ : null);
const videoBlockedReason = blockedReasonFor(settings);
const change = (patch) => {
@@ -168,6 +178,7 @@ export default function useMusicVideoModelSettings({ project, onProjectPatch } =
modelsLoading,
defaultModel,
falEnabled,
+ runwayEnabled,
saving,
framePinSaving,
settings,
diff --git a/client/src/hooks/useMusicVideoSceneMedia.js b/client/src/hooks/useMusicVideoSceneMedia.js
index 36637d6dea..837b70e6ab 100644
--- a/client/src/hooks/useMusicVideoSceneMedia.js
+++ b/client/src/hooks/useMusicVideoSceneMedia.js
@@ -14,7 +14,7 @@ import { isLtx2FamilyRuntime } from '../lib/runnerFamilies';
import { isLayeredComposition, isSelfDrawnLayer, sceneVisualLayer } from '../lib/musicVideoLayers.js';
import { musicVideoFrameGenSize } from '../lib/musicVideoAspect.js';
import {
- falSceneTake, falTakeRequestFields, grokCoverage, isPerformanceScene, performanceBlockedReason,
+ falSceneTake, falTakeRequestFields, grokCoverage, isPerformanceScene, performanceBlockedReason, runwaySceneTake, runwayTakeRequestFields,
} from '../lib/musicVideoShotTiming.js';
// The image backends accept at most four reference images for most models
@@ -357,6 +357,10 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applyS
const falFields = settings.backend === 'fal'
? falTakeRequestFields(falSceneTake({ scene, videoSettings: settings, songDurationSec: project?.audioAnalysis?.durationSec ?? null }))
: null;
+ // Runway — image-to-video cutaways, the take the scene card prices (runwaySceneTake).
+ const runwayFields = settings.backend === 'runway'
+ ? runwayTakeRequestFields(runwaySceneTake({ scene, videoSettings: settings }))
+ : null;
videoLane.startScene(scene.sceneId);
return generateVideo({
prompt,
@@ -372,8 +376,8 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applyS
// pinned length) and its resolution; a performance clip's length
// follows its song slice (server-side), so it sends only the lip-sync
// resolution.
- : falFields
- ? falFields
+ : falFields || runwayFields
+ ? falFields || runwayFields
: settings.backend === 'local'
// A scene card's pick names its model and length; else the saved model at the model's default length.
? { modelId: settings.modelId || undefined, ...(target?.numFrames ? { numFrames: target.numFrames } : {}), disableAudio: true }
@@ -386,7 +390,7 @@ export default function useMusicVideoSceneMedia({ project, videoSettings, applyS
// A style-break end frame (#11173), when this fal or local model takes one
// (a local LTX clip then renders first+last frame, #11258).
...(card && (card.endFrame === false || songAudio) ? {} : sceneEndFrameRequestFields(scene, {
- backend: settings.backend, falModelId: falFields?.falModelId, mode, // A card's local pick names its own model; else the active one.
+ backend: settings.backend, falModelId: falFields?.falModelId, runwayModelId: runwayFields?.runwayModelId, mode, // A card's local pick names its own model; else the active one.
localRuntime: (target?.modelId ? (videoSettings.models || []).find((m) => m.id === target.modelId)?.runtime : videoSettings.activeModel?.runtime) ?? null,
})),
...(audioReactiveSelected ? {
diff --git a/client/src/hooks/useVideoGenFieldState.js b/client/src/hooks/useVideoGenFieldState.js
index f64789134a..370c428b5b 100644
--- a/client/src/hooks/useVideoGenFieldState.js
+++ b/client/src/hooks/useVideoGenFieldState.js
@@ -27,6 +27,10 @@ export function useVideoGenFieldState({
const [falModelId, setFalModelId] = useState('');
const [falResolution, setFalResolution] = useState('');
const [falGenerateAudio, setFalGenerateAudio] = useState(false);
+ // Runway: '' = the catalog default model / the model's default length and size.
+ const [runwayModelId, setRunwayModelId] = useState('');
+ const [runwayDuration, setRunwayDuration] = useState('');
+ const [runwayResolution, setRunwayResolution] = useState('');
const [reactorClipId, setReactorClipId] = useState('');
const [reactorSeconds, setReactorSeconds] = useState(REACTOR_DEFAULT_CLIP_LENGTH);
const [reactorSeed, setReactorSeed] = useState('');
@@ -100,6 +104,9 @@ export function useVideoGenFieldState({
falModelId, setFalModelId,
falResolution, setFalResolution,
falGenerateAudio, setFalGenerateAudio,
+ runwayModelId, setRunwayModelId,
+ runwayDuration, setRunwayDuration,
+ runwayResolution, setRunwayResolution,
reactorClipId, setReactorClipId,
reactorSeconds, setReactorSeconds,
reactorSeed, setReactorSeed,
diff --git a/client/src/hooks/useVideoGenForm.js b/client/src/hooks/useVideoGenForm.js
index eec65b53a2..a1cdedf074 100644
--- a/client/src/hooks/useVideoGenForm.js
+++ b/client/src/hooks/useVideoGenForm.js
@@ -78,7 +78,7 @@ const editableRemixModel = (models, defaultModelId) => {
* only to decide whether to show the control at all.
*/
export function useVideoGenForm({
- models, modelContext, availableLoras, grokEnabled, falEnabled = false, reactorEnabled = false,
+ models, modelContext, availableLoras, grokEnabled, falEnabled = false, reactorEnabled = false, runwayEnabled = false,
remoteSubmissionFields = null, displaySleepEnabled = false,
}) {
const [searchParams, setSearchParams] = useSearchParams();
@@ -105,6 +105,9 @@ export function useVideoGenForm({
falModelId, setFalModelId,
falResolution, setFalResolution,
falGenerateAudio, setFalGenerateAudio,
+ runwayModelId, setRunwayModelId,
+ runwayDuration, setRunwayDuration,
+ runwayResolution, setRunwayResolution,
reactorClipId, setReactorClipId,
reactorSeconds, setReactorSeconds,
reactorSeed, setReactorSeed,
@@ -160,6 +163,15 @@ export function useVideoGenForm({
setFalGenerateAudio(false);
}, [falModelId]);
+ // Each Runway model has its own lengths and sizes.
+ const runwayModelPickRef = useRef(runwayModelId);
+ useEffect(() => {
+ if (runwayModelPickRef.current === runwayModelId) return;
+ runwayModelPickRef.current = runwayModelId;
+ setRunwayDuration('');
+ setRunwayResolution('');
+ }, [runwayModelId]);
+
// Music renders live in the shared library rather than in the browser's
// local file picker. Turn a render's deep-link filename into the same File
// object the upload panel produces, so the existing multipart submit path,
@@ -643,7 +655,9 @@ export function useVideoGenForm({
// (`reactorEnabled`) — same short-circuit shape as fal, plus native
// clip-to-clip chaining via continue_from_clip_id.
const isReactor = reactorEnabled && backend === 'reactor';
- const referenceModeApplies = mode === 'image' && !isGrok && !isFal && !isReactor;
+ // Runway is usability-gated the same way, and renders image-to-video only.
+ const isRunway = runwayEnabled && backend === 'runway';
+ const referenceModeApplies = mode === 'image' && !isGrok && !isFal && !isReactor && !isRunway;
// The strength the render will actually use, for the slider readout — an
// untouched slider under Inspire still resolves to the contract's low default
// rather than the pipeline's 1.0, and the panel must say so.
@@ -927,7 +941,9 @@ export function useVideoGenForm({
// clip survives the switch and reappears if the user flips back.
const handleBackendChange = (id) => {
setBackend(id);
- if ((id === 'grok' || id === 'fal' || id === 'reactor') && mode !== 'text' && mode !== 'image') {
+ if (id === 'runway' && mode !== 'image') {
+ handleModeChange('image');
+ } else if ((id === 'grok' || id === 'fal' || id === 'reactor') && mode !== 'text' && mode !== 'image') {
handleModeChange((sourceImageFile || sourceImageUpload) ? 'image' : 'text');
}
};
@@ -1227,6 +1243,16 @@ export function useVideoGenForm({
setMode(p.videoMode === 'image' ? 'image' : 'text');
if (p.duration) setFalDuration(p.duration);
if (p.modelId) setFalModelId(p.modelId);
+ } else if (p.mode === 'runway') {
+ // Runway job: same discriminator shape as fal above; image-to-video only.
+ setBackend('runway');
+ setMode('image');
+ if (p.modelId) {
+ runwayModelPickRef.current = p.modelId;
+ setRunwayModelId(p.modelId);
+ }
+ if (p.duration) setRunwayDuration(String(p.duration));
+ if (p.resolution) setRunwayResolution(p.resolution);
} else if (p.mode === 'reactor') {
// reactor.inc job: same discriminator shape as grok/fal above.
setBackend('reactor');
@@ -1309,6 +1335,7 @@ export function useVideoGenForm({
const submissionState = {
isGrok, grokDuration, isFal, falDuration, falModelId, falResolution, falGenerateAudio,
isReactor, reactorClipId, reactorSeconds, reactorSeed, reactorAspect, remoteSubmissionFields,
+ isRunway, runwayModelId, runwayDuration, runwayResolution,
displaySleepEnabled,
prompt, negativePrompt, stylePreset, selectedUniverse,
width, height, mode, sourceImageFile, sourceImageUpload,
@@ -1325,12 +1352,15 @@ export function useVideoGenForm({
return {
// Backend + mode
- backend, isGrok, isFal, isReactor, handleBackendChange,
+ backend, isGrok, isFal, isReactor, isRunway, handleBackendChange,
grokDuration, setGrokDuration,
falDuration, setFalDuration,
falModelId, setFalModelId,
falResolution, setFalResolution,
falGenerateAudio, setFalGenerateAudio,
+ runwayModelId, setRunwayModelId,
+ runwayDuration, setRunwayDuration,
+ runwayResolution, setRunwayResolution,
reactorClipId, setReactorClipId,
reactorSeconds, setReactorSeconds,
reactorSeed, setReactorSeed,
diff --git a/client/src/lib/README.md b/client/src/lib/README.md
index d83fd39a0a..96f6b7bbfe 100644
--- a/client/src/lib/README.md
+++ b/client/src/lib/README.md
@@ -69,7 +69,7 @@ grep -i "what you want to do" client/src/lib/README.md
| `musicVideoCoverOverlay.js` | Cover-art lettering preview (#10345): re-export of `server/lib/musicVideoCoverOverlay.js` (the design vocabulary, `normalizeCoverDesign`, `coverOverlaySvg`) plus `canvasCoverWidths(design, texts, fonts)`, which measures text with a canvas so the live preview wraps and sizes the title the way the server's measured render does. |
| `musicVideoSceneTiming.js` | `buildSceneTimingPresentation`: pure provider timing, cost, coverage and split/merge projection for the Music Video scene card, reusing canonical shot-timing policy. |
| `musicVideoShotTiming.js` | Re-export of `server/lib/musicVideoShotTiming.js` (#8977) — shot modes and performance styles, the verified source-audio lip-sync and music-driven capability tables, `performanceBlockedReason`, `planPerformanceRepair` (measured accepted-prefix boundary, suffix interval and next spend), `planPerformanceWindow`, `grokCoverage`, `approximateMotionCues`, `shotSplitLimit` (when a scene offers a lyric-boundary split), `falSceneTake` (a fal take's model, length, resolution and estimated cost). `clipAudioOffsetSec` (how far into an aligned take's clip its audio window starts, #11210). Used by `hooks/useMusicVideoSceneMedia.js` and `components/musicVideo/SceneCard.jsx`. |
-| `musicVideoStages.js` | Music Video page stage model: `MUSIC_VIDEO_STAGES` (six steps: Song `setup` → Look `cast-sets` → Storyboard `board` → Make `produce` → Final render `review` → Publish; a retired `compose` link resolves to Make), `resolveStageParam(value)` for the `:stage` route param, `deriveStages(project)` (per-stage `done`/`active`/`blocked`/`todo` + `current`, the first incomplete step; a live production run pins Make), `deriveNextAction(project, ctx)` (the one primary action the sticky header offers — approve Cast & Sets, verify lyric timing, stop/resume production, render final, …), `describeProjectStatus` (the header line and the step that needs the director), `autopilotStatus` (the header Autopilot button), `projectSpend(project)` (paid generation used vs. cap), `projectShotSummary(project)` (document shots or scene counts), `stageChecklist(stageId, project, readiness)` (what one step still needs, with the open approval's reason and a scroll anchor; an `optional` item, such as Make's animated proof, is listed but not counted), `summarizeStoryboardProblems(problems)` (folds the server's one-sentence-per-shot storyboard problems into one count line per kind), `stepNotes(project, readiness, publish)` (one fact per step for the step list), `listPreviewSources` (final, live composition, drafts, then the storyboard animatic), `projectServicesSummary(project)` (the saved image/video render services), `compareMusicVideoProjectsNewestFirst(a, b)` (sort projects newest created / updated first for project cards and pickers), and `currentProductionRun`; `productionReviewStopGuidance` separates current approval blockers from historical run stops. Used by `components/musicVideo/MusicVideoLayout.jsx` and `pages/MusicVideo.jsx`. `projectFalSpend(project)` (estimated fal.ai spend by model across paid takes and production steps). |
+| `musicVideoStages.js` | Music Video page stage model: `MUSIC_VIDEO_STAGES` (six steps: Song `setup` → Look `cast-sets` → Storyboard `board` → Make `produce` → Final render `review` → Publish; a retired `compose` link resolves to Make), `resolveStageParam(value)` for the `:stage` route param, `deriveStages(project)` (per-stage `done`/`active`/`blocked`/`todo` + `current`, the first incomplete step; a live production run pins Make), `deriveNextAction(project, ctx)` (the one primary action the sticky header offers — approve Cast & Sets, verify lyric timing, stop/resume production, render final, …), `describeProjectStatus` (the header line and the step that needs the director), `autopilotStatus` (the header Autopilot button), `projectSpend(project)` (paid generation used vs. cap), `projectShotSummary(project)` (document shots or scene counts), `stageChecklist(stageId, project, readiness)` (what one step still needs, with the open approval's reason and a scroll anchor; an `optional` item, such as Make's animated proof, is listed but not counted), `summarizeStoryboardProblems(problems)` (folds the server's one-sentence-per-shot storyboard problems into one count line per kind), `stepNotes(project, readiness, publish)` (one fact per step for the step list), `listPreviewSources` (final, live composition, drafts, then the storyboard animatic), `projectServicesSummary(project)` (the saved image/video render services), `compareMusicVideoProjectsNewestFirst(a, b)` (sort projects newest created / updated first for project cards and pickers), and `currentProductionRun`; `productionReviewStopGuidance` separates current approval blockers from historical run stops. Used by `components/musicVideo/MusicVideoLayout.jsx` and `pages/MusicVideo.jsx`. `projectFalSpend(project)` (estimated fal.ai and Runway spend by model across paid takes and production steps). |
| `musicVideoAttention.js` | Music Video "Needs attention" model (#9940): `deriveAttentionItems(project, live)` returns the banner rows for server-held states a client-orchestrated workflow can strand — an open/rendering revision, an interrupted Cast & Sets stage, a running auto-review waiting on its revised sections, a final render nobody is watching — derived from the persisted record so they survive a reload (healthy work this tab can see progressing is not flagged); `openRevisionOf(project)`; `ATTENTION_ANCHOR_ID` + `revealAttention()` scroll a refusal toast to the banner. Used by `components/musicVideo/NeedsAttentionBanner.jsx` and `pages/MusicVideo.jsx`. |
| `musicVideoProjectList.js` | Music Video project index model (#10167): `projectRunPill(project)` (Running / Needs you / Interrupted / Done, from the same next-action and attention logic as the header), `groupMusicVideoProjects(projects, query)` (collapse forks under their root, newest version first, name filter, groups ordered by last touch), and `compareMusicVideoProjectsRecentlyTouched`. Used by `pages/MusicVideo.jsx` and `components/musicVideo/MusicVideoProjectCard.jsx`. |
| `musicVideoPreview.js` | Pick the representative produced asset for a Music Video project card (`selectMusicVideoPreview`: final video → latest excerpt → latest rendered scene clip → latest scene reference frame → visual spec reference) for the main project grid. Pure, computed off the project record, with image path and fallback helpers (`musicVideoImageSrc`, `musicVideoImageFallback`, `scenePosterSrc`: a scene's frame or its clip's poster). |
@@ -91,6 +91,7 @@ grep -i "what you want to do" client/src/lib/README.md
| `glbFailure.js` | Turns a 3D asset failure into an actionable sentence. `glbFailureHint(error)` matches the message against a shared table (an HTML body reaching the glTF parser, a WebGL context failure, a 404) and returns `null` when nothing recognizes it — so a caller can tell "we know the cause" from "we don't" and fall back to the raw text. `glbErrorText(error)` reads a message off whatever shape was thrown. Shared because the CoS avatars load remote GLBs too: every caught error there used to be reported as "this display has no WebGL", sending the user to change an unrelated setting (#4688). Consumed by `media/GlbViewer.jsx` and `cos/CoSCanvasGuard.jsx`. |
| `grokVideoClip.js` | Re-export of `server/lib/grokVideoClip.js` — `GROK_VIDEO_DURATIONS` (`[6, 10, 15]`) + `GROK_VIDEO_DEFAULT_DURATION`, the clip lengths grok's `image_to_video` actually delivers. |
| `reactorVideoClip.js` | Re-export of `server/lib/reactorVideoClip.js` — `REACTOR_MAX_PROMPT_LENGTH` (800, counted in the VideoGen prompt field before submit), `REACTOR_MIN_CLIP_SECONDS` / `REACTOR_MAX_CLIP_SECONDS`, the picker list `REACTOR_CLIP_LENGTHS` + `REACTOR_DEFAULT_CLIP_LENGTH` + `reactorClipLengthLabel()`, and the canvas contract `REACTOR_CANVASES` / `REACTOR_ASPECTS` / `REACTOR_DEFAULT_ASPECT` / `reactorCanvas()` / `nearestReactorAspect()`. |
+| `runwayVideoModels.js` | Re-export of `server/lib/runwayVideoModels.js` — the curated Runway video model catalog, per-model request rules and `estimateRunwayVideoCostUsd()`. Used by the Music Video Runway pickers and the Video Gen page's Runway backend. |
| `imageCleaners.js` | Re-exports `resolveCleanersFromConfig` from the shared pure `server/lib/imageCleanDefaults.js` policy. Reads `{cleanC2PA, denoise}` off a per-mode settings record. `cleanC2PA` defaults are mode-aware (on for `codex` + `external` — the backends that emit C2PA chunks today — off otherwise, as an allow-list rather than a deny-list); `denoise` defaults off everywhere (lossy, opt-in only). |
| `reviewerModels.js` | `reviewerModelsFromDefaults` / `reviewerModelsToDefaults` / `reviewerEffortsFromDefaults` / `reviewerEffortsToDefaults` — adapt the Code Review Defaults' per-reviewer SCALARS (`codexModel`, `ollamaModel`, `claudeEffort`, …) to and from the token-keyed maps `ReviewerPicker`'s Model and Effort columns take. Mirrors `reviewerModelsFromDefaults` / `reviewerEffortsFromDefaults` in `server/lib/reviewerConfig.js`. Also preserves configured-provider model and effort pins in the additive `providerModels` / `providerEfforts` maps, keyed by `provider:<id>`; empty maps clear overrides without changing builtin scalars. |
| `reviewerPins.js` | Client mirror of the whole reviewer vocabulary — the roster (`REVIEWER_VALUES`), `REVIEWER_ALIASES`, `DEFAULT_REVIEWER(S)`, `normalizeReviewers`, the review-username pattern/cap (`MAX_REVIEW_USERNAMES` / `cleanReviewUsername` / `normalizeReviewUsernames`), `MAX_REVIEWER_MAX_ROUNDS`, `REVIEW_STOP_MODES` / `DEFAULT_REVIEW_STOP_MODE`, plus the per-reviewer pin vocabularies (`MODEL_SELECTABLE_REVIEWERS` / `EFFORT_SELECTABLE_REVIEWERS` / `REVIEWER_EFFORT_LEVELS` / `reviewerEffortLevels` / `normalizeReviewerSlug` / `sanitizeReviewerModelInput`) and their bounds. Dependency-free on purpose: the server suite imports it to pin the mirror against `server/lib/reviewerConfig.js`, so it must not reach for a client-only package. Re-exported by `components/cos/constants.js`. `isProviderReviewer` recognizes the shared `provider:<id>` identity alongside legacy harness slugs. |
diff --git a/client/src/lib/imageGenModes.js b/client/src/lib/imageGenModes.js
index b5a090a91a..522358edf1 100644
--- a/client/src/lib/imageGenModes.js
+++ b/client/src/lib/imageGenModes.js
@@ -180,6 +180,7 @@ export const MODE_LABELS = Object.freeze({
[IMAGE_GEN_MODE.EXTERNAL]: 'External',
[VIDEO_GEN_MODE.FAL]: 'fal.ai',
[VIDEO_GEN_MODE.REACTOR]: 'Reactor.inc',
+ [VIDEO_GEN_MODE.RUNWAY]: 'Runway',
});
// The fal.ai model picker rows — one per catalog family, keyed by its
diff --git a/client/src/lib/index.js b/client/src/lib/index.js
index 4f3a3a2bcb..17aec8739b 100644
--- a/client/src/lib/index.js
+++ b/client/src/lib/index.js
@@ -52,6 +52,7 @@ export * from './fableLoomReadiness.js';
export * from './glbFailure.js';
export * from './grokVideoClip.js';
export * from './reactorVideoClip.js';
+export * from './runwayVideoModels.js';
export * from './imageCleaners.js';
export * from './imageGenBackends.js';
// Namespace export: imageGenBackends re-exports every imageGenModes symbol
diff --git a/client/src/lib/musicVideoBatchPlan.js b/client/src/lib/musicVideoBatchPlan.js
index 63178d6e97..ae0d196186 100644
--- a/client/src/lib/musicVideoBatchPlan.js
+++ b/client/src/lib/musicVideoBatchPlan.js
@@ -1,5 +1,4 @@
-import { falSceneTake } from './musicVideoShotTiming.js';
-import { getFalVideoModel } from './falVideoModels.js';
+import { meteredModelLabel, meteredSceneTake } from './musicVideoShotTiming.js';
import { formatUsd } from '../utils/formatters.js';
/**
@@ -29,23 +28,23 @@ export function videosButtonLabel({ renderableCount, footageCount, waiting }) {
}
/**
- * The confirm line for a clip batch. fal takes are priced per scene with the
- * same `falSceneTake` the scene card shows; a scene it cannot price is counted
- * as unpriced. Local and Grok renders show the count only.
+ * The confirm line for a clip batch. fal and Runway takes are priced per scene
+ * with the same `meteredSceneTake` the scene card shows; a scene it cannot
+ * price is counted as unpriced. Local and Grok renders show the count only.
*/
export function videoBatchPreview({ scenes, backend, videoSettings, songDurationSec = null }) {
const count = scenes.length;
- if (backend !== 'fal') return { count, backend, text: `Generate ${plural(count, 'clip')} on ${backend === 'grok' ? 'Grok' : 'the local renderer'}` };
- const takes = scenes.map((scene) => falSceneTake({ scene, videoSettings, songDurationSec }));
+ if (backend !== 'fal' && backend !== 'runway') return { count, backend, text: `Generate ${plural(count, 'clip')} on ${backend === 'grok' ? 'Grok' : 'the local renderer'}` };
+ const takes = scenes.map((scene) => meteredSceneTake(backend, { scene, videoSettings, songDurationSec }));
const priced = takes.filter((take) => take.costUsd != null);
const costUsd = priced.reduce((sum, take) => sum + take.costUsd, 0);
const modelIds = [...new Set(takes.map((take) => take.modelId))];
- const model = modelIds.length === 1 ? (getFalVideoModel(modelIds[0])?.label || modelIds[0]) : `${modelIds.length} models`;
+ const model = modelIds.length === 1 ? (meteredModelLabel(modelIds[0]) || modelIds[0]) : `${modelIds.length} models`;
const resolutions = [...new Set(takes.map((take) => take.resolution).filter(Boolean))];
const unpriced = takes.length - priced.length;
return {
count, backend, costUsd, unpriced,
- text: `Generate ${plural(count, 'clip')} on fal / ${model}${resolutions.length === 1 ? ` ${resolutions[0]}` : ''}`
+ text: `Generate ${plural(count, 'clip')} on ${backend === 'runway' ? 'Runway' : 'fal'} / ${model}${resolutions.length === 1 ? ` ${resolutions[0]}` : ''}`
+ ` · est. ${formatUsd(costUsd)}${unpriced > 0 ? ` (${unpriced} unpriced)` : ''}`,
};
}
diff --git a/client/src/lib/musicVideoSceneTiming.js b/client/src/lib/musicVideoSceneTiming.js
index 8a8658a2da..4521afb12d 100644
--- a/client/src/lib/musicVideoSceneTiming.js
+++ b/client/src/lib/musicVideoSceneTiming.js
@@ -1,6 +1,6 @@
import { formatUsd } from '../utils/formatters.js';
import {
- falSceneTake, grokCoverage, isPerformanceScene, performanceBlockedReason, performanceCapability, planPerformanceWindow, shotSplitLimit,
+ grokCoverage, isPerformanceScene, meteredSceneTake, performanceBlockedReason, performanceCapability, planPerformanceWindow, shotSplitLimit,
} from './musicVideoShotTiming.js';
// "est. $0.81 (5.05s at 1080P)" — or null when the take cannot be priced.
@@ -28,8 +28,9 @@ export function buildSceneTimingPresentation({
? (capability ? (plan.ok ? null : plan.message) : performanceBlockedReason(lipSyncBackend))
: null;
const grokPlan = !performance && lipSyncBackend === 'grok' && timedSpan != null ? grokCoverage(timedSpan) : null;
- const falTake = lipSyncBackend === 'fal' && layer === 'footage'
- ? falSceneTake({ scene, videoSettings: falVideoSettings || {}, songDurationSec, lineCues })
+ // The priced take on a paid lane (fal.ai or Runway); `falTake` keeps its name for the fal-shaped consumers.
+ const falTake = layer === 'footage'
+ ? meteredSceneTake(lipSyncBackend, { scene, videoSettings: falVideoSettings || {}, songDurationSec, lineCues })
: null;
const falCost = falEstimate(falTake);
const performanceCost = falTake?.performance ? (falCost || capability?.costLabel) : capability?.costLabel;
diff --git a/client/src/lib/musicVideoShotTiming.js b/client/src/lib/musicVideoShotTiming.js
index 6c1529ba3c..91f8ea0e65 100644
--- a/client/src/lib/musicVideoShotTiming.js
+++ b/client/src/lib/musicVideoShotTiming.js
@@ -16,10 +16,14 @@ export {
falTakeRequestFields,
grokCoverage,
isPerformanceScene,
+ meteredModelLabel,
+ meteredSceneTake,
performanceBlockedReason,
performanceCapability,
planPerformanceWindow,
planPerformanceRepair,
+ runwaySceneTake,
+ runwayTakeRequestFields,
selectedPerformanceInstruction,
shotSplitLimit,
} from '../../../server/lib/musicVideoShotTiming.js';
diff --git a/client/src/lib/musicVideoStages.js b/client/src/lib/musicVideoStages.js
index ba8a148cbb..56b5eb747f 100644
--- a/client/src/lib/musicVideoStages.js
+++ b/client/src/lib/musicVideoStages.js
@@ -11,9 +11,8 @@ import {
import { formatCount, formatDurationSec, formatTimecode } from '../utils/formatters.js';
import { isNonBlankStr } from './textUtils';
import { modeLabel } from './imageGenModes.js';
-import { falSceneTake } from './musicVideoShotTiming.js';
+import { meteredModelLabel, meteredSceneTake } from './musicVideoShotTiming.js';
import { unlockedShots } from './musicVideoSceneAttention.js';
-import { getFalVideoModel } from './falVideoModels.js';
import { latestMusicVideoReviewDraft } from '../../../server/lib/musicVideoReviewDraft.js';
import { supersededStillsCheckpoint } from '../../../server/lib/musicVideoAutonomous.js';
import { FINISHED_OUTSIDE_RERENDER_NOTE, finishedOutside, finishedOutsideCovers } from '../../../server/lib/musicVideoFinishedOutside.js';
@@ -133,7 +132,7 @@ export function projectFalSpend(project) {
const add = (modelId, usd) => {
if (!(Number.isFinite(usd) && usd > 0)) return;
const key = modelId || null;
- const row = byModel.get(key) || { modelId: key, label: key ? getFalVideoModel(key)?.label || key : 'Model not recorded', usd: 0, count: 0 };
+ const row = byModel.get(key) || { modelId: key, label: key ? meteredModelLabel(key) || key : 'Model not recorded', usd: 0, count: 0 };
row.usd += usd;
row.count += 1;
byModel.set(key, row);
@@ -145,7 +144,7 @@ export function projectFalSpend(project) {
}
for (const run of Array.isArray(project?.productionRuns) ? project.productionRuns : []) {
for (const step of Array.isArray(run?.steps) ? run.steps : []) {
- if (step?.route?.mode === 'fal' && step.status !== 'refused') add(step.costModelId || step.route.model, Number(step.costUsd));
+ if ((step?.route?.mode === 'fal' || step?.route?.mode === 'runway') && step.status !== 'refused') add(step.costModelId || step.route.model, Number(step.costUsd));
}
}
const models = [...byModel.values()].map((row) => ({ ...row, usd: Math.round(row.usd * 1e4) / 1e4 })).sort((a, b) => b.usd - a.usd);
@@ -169,9 +168,7 @@ export function boardJobEstimate(project) {
let knownUsd = 0;
let unpriced = missingFrames;
for (const scene of missingClips) {
- const cost = settings.backend === 'fal'
- ? falSceneTake({ scene, videoSettings: settings, songDurationSec: project?.audioAnalysis?.durationSec ?? null }).costUsd
- : null;
+ const cost = meteredSceneTake(settings.backend, { scene, videoSettings: settings, songDurationSec: project?.audioAnalysis?.durationSec ?? null })?.costUsd ?? null;
if (Number.isFinite(cost)) knownUsd += cost;
else unpriced += 1;
}
diff --git a/client/src/lib/runwayVideoModels.js b/client/src/lib/runwayVideoModels.js
new file mode 100644
index 0000000000..bee2abe540
--- /dev/null
+++ b/client/src/lib/runwayVideoModels.js
@@ -0,0 +1,18 @@
+/**
+ * Curated Runway video model catalog.
+ *
+ * Re-export of `server/lib/runwayVideoModels.js` — the one definition of each
+ * Runway model's request rules and list price, imported rather than copied so
+ * the board's cost estimate and the server's request body cannot drift.
+ */
+export {
+ RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL,
+ RUNWAY_VIDEO_MODELS,
+ coerceRunwayVideoSeconds,
+ describeRunwayVideoRate,
+ estimateRunwayVideoCostUsd,
+ getRunwayVideoModel,
+ runwayVideoClipQuote,
+ runwayVideoLengths,
+ runwayVideoResolutions,
+} from '../../../server/lib/runwayVideoModels.js';
diff --git a/client/src/lib/videoGenSubmission.js b/client/src/lib/videoGenSubmission.js
index cc415f158f..080b4523bd 100644
--- a/client/src/lib/videoGenSubmission.js
+++ b/client/src/lib/videoGenSubmission.js
@@ -63,7 +63,7 @@ export function videoPromptAdditions({ prompt, negativePrompt, stylePreset, sele
];
}
-// Hosted lanes (Grok, fal, reactor) take only text or image mode, and the
+// Hosted lanes (Grok, fal, reactor, Runway) take only text or image mode, and the
// source pair is meaningful only in image mode. '' is the wire value otherwise.
const hostedImageFields = (mode, sourceImageFile, sourceImageUpload) => {
const isImage = mode === 'image';
@@ -115,6 +115,25 @@ function falVideoSubmission({
};
}
+// Runway renders image-to-video only: a start frame is required (the form
+// offers Image mode alone). The negative prompt is sent; the server passes it
+// natively or as an "Avoid:" line, listed as a chip in the sent-prompt view.
+function runwayVideoSubmission({
+ composed, runwayModelId, runwayDuration, runwayResolution,
+ width, height, mode, sourceImageFile, sourceImageUpload,
+}) {
+ return {
+ backend: 'runway',
+ prompt: composed.prompt,
+ negativePrompt: composed.negativePrompt,
+ runwayModelId: runwayModelId || undefined,
+ ...(runwayDuration ? { runwayDuration: Number(runwayDuration) } : {}),
+ ...(runwayResolution ? { runwayResolution } : {}),
+ ...clampedSize(width, height),
+ ...hostedImageFields(mode, sourceImageFile, sourceImageUpload),
+ };
+}
+
function reactorVideoSubmission({
composed, reactorClipId, reactorSeconds, reactorSeed, reactorAspect,
mode, sourceImageFile, sourceImageUpload,
@@ -305,7 +324,7 @@ const liveBeats = (chainingActive, chunkPrompts, chunks, withEnvelope) => (
export function buildVideoGenSubmission(args) {
const {
- isGrok, isFal, isReactor, remoteSubmissionFields,
+ isGrok, isFal, isReactor, isRunway, remoteSubmissionFields,
prompt, negativePrompt, stylePreset, selectedUniverse,
currentModel, disableAudio, noMusic, chainingActive, chunks, chunkPrompts,
} = args;
@@ -315,11 +334,12 @@ export function buildVideoGenSubmission(args) {
});
const shared = { ...args, composed };
- const isLocal = !isGrok && !isFal && !isReactor && !remoteSubmissionFields;
+ const isLocal = !isGrok && !isFal && !isReactor && !isRunway && !remoteSubmissionFields;
const additions = videoPromptAdditions({ ...args, isLocal });
const payload = isGrok ? grokVideoSubmission(shared)
: isFal ? falVideoSubmission(shared)
: isReactor ? reactorVideoSubmission(shared)
+ : isRunway ? runwayVideoSubmission(shared)
: remoteSubmissionFields ? remoteVideoSubmission(shared)
: localVideoSubmission(args, {
composed,
diff --git a/client/src/pages/VideoGen.jsx b/client/src/pages/VideoGen.jsx
index 4b6ab1d162..36339a9b80 100644
--- a/client/src/pages/VideoGen.jsx
+++ b/client/src/pages/VideoGen.jsx
@@ -105,6 +105,10 @@ import {
FAL_DEFAULT_IMAGE_VIDEO_MODEL, FAL_DEFAULT_TEXT_VIDEO_MODEL, FAL_VIDEO_MODELS, getFalVideoModel, falVideoResolutions, estimateFalVideoCostUsd,
} from '../lib/falVideoModels.js';
import { REACTOR_MAX_PROMPT_LENGTH } from '../lib/reactorVideoClip.js';
+import {
+ RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL, RUNWAY_VIDEO_MODELS, describeRunwayVideoRate, estimateRunwayVideoCostUsd, getRunwayVideoModel,
+ runwayVideoLengths, runwayVideoResolutions,
+} from '../lib/runwayVideoModels.js';
import { videoGenEnqueuePolicy } from '../lib/videoGenEnqueuePolicy.js';
import { styledVideoPrompt } from '../lib/videoGenSubmission.js';
import ReactorPanel from '../components/videoGen/ReactorPanel';
@@ -198,6 +202,7 @@ export default function VideoGen() {
// entry) still surfaces the backend switcher below.
const falEnabled = status?.falEnabled === true;
const reactorEnabled = status?.reactorEnabled === true;
+ const runwayEnabled = status?.runwayEnabled === true;
const models = useMemo(() => modelContext?.models || [], [modelContext]);
const refreshGrokEnabled = useCallback(() => {
getSettings({ silent: true })
@@ -235,7 +240,8 @@ export default function VideoGen() {
// Every field the form submits, plus the payload builder both submit paths
// share. See client/src/hooks/useVideoGenForm.js.
const {
- backend, isGrok, isFal, isReactor, handleBackendChange, grokDuration, setGrokDuration,
+ backend, isGrok, isFal, isReactor, isRunway, handleBackendChange, grokDuration, setGrokDuration,
+ runwayModelId, setRunwayModelId, runwayDuration, setRunwayDuration, runwayResolution, setRunwayResolution,
falDuration, setFalDuration, falModelId, setFalModelId,
falResolution, setFalResolution, falGenerateAudio, setFalGenerateAudio,
reactorClipId, setReactorClipId, reactorSeconds, setReactorSeconds, reactorSeed, setReactorSeed,
@@ -276,7 +282,7 @@ export default function VideoGen() {
icStrength, setIcStrength, icSkipStage2, setIcSkipStage2,
applyRemix, applyFinish, applyResumedParams, buildGeneratePayload,
} = useVideoGenForm({
- models, modelContext, availableLoras, grokEnabled, falEnabled, reactorEnabled,
+ models, modelContext, availableLoras, grokEnabled, falEnabled, reactorEnabled, runwayEnabled,
remoteSubmissionFields: remoteTarget.isRemote ? remoteTarget.submissionFields : null,
displaySleepEnabled,
});
@@ -311,6 +317,7 @@ export default function VideoGen() {
['the Grok backend', isGrok],
['the fal.ai backend', isFal],
['the reactor.inc backend', isReactor],
+ ['the Runway backend', isRunway],
// Each remaining pipeline semantic has its own input listed below, but the
// mode can be set before that input is filled — so gate the mode too
// rather than letting an a2v render reach the peer as plain text-to-video.
@@ -337,7 +344,7 @@ export default function VideoGen() {
return `${model?.modelName || 'The selected peer model'} renders only from a source image — add a start frame, or pick a text-to-video model.`;
}
return null;
- }, [remoteTarget.isRemote, remoteTarget.model, remoteTarget.acceptsInput, isGrok, isFal, isReactor, mode, sourceImageFile, sourceImageUpload,
+ }, [remoteTarget.isRemote, remoteTarget.model, remoteTarget.acceptsInput, isGrok, isFal, isReactor, isRunway, mode, sourceImageFile, sourceImageUpload,
lastImageFile, lastImageUpload, keyframesActive, extendFromVideoId, audioFile, icReferenceFile,
icReferenceVideoId, icReferenceImageFiles, selectedLoras, chunks]);
// One reading for the Generate button, the enqueue guard and the caption.
@@ -355,7 +362,7 @@ export default function VideoGen() {
// short submit / render / fetch ladder instead. A federated PEER is
// deliberately excluded: it really does load the weights and run the sampler,
// it just does so out of view.
- const rendersOffMachine = isGrok || isFal || isReactor;
+ const rendersOffMachine = isGrok || isFal || isReactor || isRunway;
const rendersOnThisMachine = !rendersOffMachine && !remoteTarget.isRemote;
// Can THIS render put the display to sleep at all? The model says whether
@@ -960,6 +967,8 @@ export default function VideoGen() {
const modelFieldVisible = !modelsLoading || modelContextLoading;
const falModel = getFalVideoModel(falModelId);
const falResolutions = falVideoResolutions(falModel);
+ const runwayModel = getRunwayVideoModel(runwayModelId || RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL);
+ const runwayResolutions = runwayVideoResolutions(runwayModel);
const notConnected = !!status && status.connected === false && !needsByovProbe;
@@ -968,13 +977,17 @@ export default function VideoGen() {
// the counter measures the same string buildVideoGenSubmission builds rather
// than the textarea. Gating Generate on it turns a post-click 400 into a
// number the user can watch while writing.
+ // Runway caps the prompt per model; the server refuses an over-cap prompt
+ // with its count rather than shortening it.
+ const promptCap = isReactor ? REACTOR_MAX_PROMPT_LENGTH : isRunway ? runwayModel?.promptMaxChars ?? null : null;
+ const promptCapLabel = isReactor ? 'Reactor' : 'Runway';
const submittedPromptLength = useMemo(
- () => (isReactor
+ () => (promptCap != null
? styledVideoPrompt(prompt, { negativePrompt, stylePreset, selectedUniverse }).length
: prompt.length),
- [isReactor, prompt, negativePrompt, stylePreset, selectedUniverse],
+ [promptCap, prompt, negativePrompt, stylePreset, selectedUniverse],
);
- const promptOverLimit = isReactor && submittedPromptLength > REACTOR_MAX_PROMPT_LENGTH;
+ const promptOverLimit = promptCap != null && submittedPromptLength > promptCap;
// Budget the AI enhancer writes inside. It's the backend cap MINUS the style
// preset / universe prefix, because that prefix is part of what PortOS
@@ -983,11 +996,11 @@ export default function VideoGen() {
// degenerate case where the prefix alone already fills the allowance, which
// the counter above is already flagging).
const enhancePromptBudget = useMemo(() => {
- if (!isReactor) return undefined;
+ if (promptCap == null) return undefined;
const overhead = submittedPromptLength - prompt.length;
- const budget = REACTOR_MAX_PROMPT_LENGTH - Math.max(0, overhead);
+ const budget = promptCap - Math.max(0, overhead);
return budget > 0 ? budget : undefined;
- }, [isReactor, submittedPromptLength, prompt]);
+ }, [promptCap, submittedPromptLength, prompt]);
// Only grok folds a negative prompt into its request (as an "Avoid:" line);
// fal's queue body and reactor's enqueue command have no such field, and a
@@ -995,7 +1008,7 @@ export default function VideoGen() {
// dead field that quietly does nothing. A federated render is the PEER's
// model to judge, so the local selection's gate must not take away a field
// that lane does submit.
- const negativePromptSupported = isGrok || remoteTarget.isRemote
+ const negativePromptSupported = isGrok || isRunway || remoteTarget.isRemote
|| (!isFal && !isReactor && currentModel?.supportsNegativePrompt !== false);
// Every completed reactor render stamps its fast-h3 clip id on the history
@@ -1133,13 +1146,14 @@ export default function VideoGen() {
cloud backend's image-to-video only supports text (image-first) and
image modes, so switching to one snaps an unsupported mode back to
the nearest one. */}
- {(grokEnabled || falEnabled || reactorEnabled) && (
+ {(grokEnabled || falEnabled || reactorEnabled || runwayEnabled) && (
<div className="bg-port-card border border-port-border rounded-xl p-1 flex gap-1" role="group" aria-label="Video generation backend">
{[
{ id: 'local', label: 'Local' },
...(grokEnabled ? [{ id: 'grok', label: 'Grok' }] : []),
...(falEnabled ? [{ id: 'fal', label: 'fal.ai' }] : []),
...(reactorEnabled ? [{ id: 'reactor', label: 'Reactor.inc' }] : []),
+ ...(runwayEnabled ? [{ id: 'runway', label: 'Runway' }] : []),
].map(({ id, label }) => (
<button
key={id}
@@ -1155,7 +1169,9 @@ export default function VideoGen() {
? 'Render via the fal.ai queue API. Counts against your fal.ai balance.'
: id === 'reactor'
? 'Render via the reactor.inc fast-h3 API. Counts against your reactor.inc balance.'
- : 'Render on this machine with the local runtimes.'}
+ : id === 'runway'
+ ? 'Animate a start frame via the Runway API. Counts against your Runway credits.'
+ : 'Render on this machine with the local runtimes.'}
>
{label}
</button>
@@ -1169,7 +1185,8 @@ export default function VideoGen() {
WAI-ARIA Tabs, since the mode-specific inputs aren't structured as
tabpanels and we don't implement roving-tabindex/arrow-key focus. */}
<div className="bg-port-card border border-port-border rounded-xl p-1 flex flex-wrap gap-1" role="group" aria-label="Video generation mode">
- {(rendersOffMachine ? MODES.filter((m) => m.id === 'text' || m.id === 'image') : MODES).map(({ id, label, icon: Icon, desc }) => {
+ {(isRunway ? MODES.filter((m) => m.id === 'image')
+ : rendersOffMachine ? MODES.filter((m) => m.id === 'text' || m.id === 'image') : MODES).map(({ id, label, icon: Icon, desc }) => {
const active = mode === id;
return (
<button
@@ -1277,21 +1294,21 @@ export default function VideoGen() {
pushes a short-looking prompt past reactor's cap is visible
here instead of surfacing as a 400 after Generate. */}
<p className={`mt-1 text-[11px] leading-snug ${promptOverLimit ? 'text-port-error' : 'text-gray-500'}`}>
- {isReactor
- ? `${formatCount(submittedPromptLength)} / ${formatCount(REACTOR_MAX_PROMPT_LENGTH)} characters`
+ {promptCap != null
+ ? `${formatCount(submittedPromptLength)} / ${formatCount(promptCap)} characters`
: `${formatCount(submittedPromptLength)} characters`}
</p>
{/* Only crossing the cap is worth announcing; the count itself
changes on every keystroke and would be pure noise. */}
{promptOverLimit && (
<p className="mt-1 text-[11px] text-port-error leading-snug" role="status">
- Reactor will reject this — shorten the prompt. Style presets count toward the limit.
+ {promptCapLabel} will reject this — shorten the prompt. Style presets count toward the limit.
</p>
)}
<div className="mt-2">
<ModelInputPreview preview={modelInputPreview} prompt={prompt} />
</div>
- {!negativePromptSupported && !isFal && !isReactor && (
+ {!negativePromptSupported && !isFal && !isReactor && !isRunway && (
<p className="mt-1 text-[11px] text-gray-500 leading-snug">
This model does not use a negative prompt.
</p>
@@ -1641,6 +1658,65 @@ export default function VideoGen() {
})()}
</p>
</div>
+ ) : isRunway ? (
+ <div className="grid grid-cols-2 gap-3">
+ <FormField className="col-span-2" label="Runway model" labelClassName="block text-xs font-medium text-gray-400 mb-1">
+ <select
+ id="runway-video-model"
+ value={runwayModel?.id || runwayModelId}
+ onChange={(e) => setRunwayModelId(e.target.value)}
+ className="w-full bg-port-bg border border-port-border rounded-lg px-2 py-2 text-sm text-white focus:outline-none focus:border-port-accent"
+ >
+ {RUNWAY_VIDEO_MODELS.map((m) => (
+ <option key={m.id} value={m.id}>
+ {`${m.label} · ${describeRunwayVideoRate(m.id)}${m.id === RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL ? ' (default)' : ''}`}
+ </option>
+ ))}
+ </select>
+ </FormField>
+ <FormField label="Clip length" labelClassName="block text-xs font-medium text-gray-400 mb-1">
+ <select
+ id="runway-video-length"
+ value={runwayDuration}
+ onChange={(e) => setRunwayDuration(e.target.value)}
+ className="w-full bg-port-bg border border-port-border rounded-lg px-2 py-2 text-sm text-white focus:outline-none focus:border-port-accent"
+ >
+ <option value="">{`${runwayModel?.duration.defaultSec}s (default)`}</option>
+ {runwayVideoLengths(runwayModel).map((sec) => <option key={sec} value={String(sec)}>{sec} seconds</option>)}
+ </select>
+ </FormField>
+ <ResolutionField
+ presets={VIDEO_RESOLUTIONS}
+ width={width}
+ height={height}
+ onChange={handleResolutionChange}
+ {...VIDEO_EDGE_BOUNDS}
+ snapOnBlur
+ note="Sets the shape: Runway renders the closest aspect ratio it offers."
+ />
+ {runwayResolutions.length > 1 && (
+ <FormField label="Output size" labelClassName="block text-xs font-medium text-gray-400 mb-1">
+ <select
+ id="runway-video-resolution"
+ value={runwayResolution}
+ onChange={(e) => setRunwayResolution(e.target.value)}
+ className="w-full bg-port-bg border border-port-border rounded-lg px-2 py-2 text-sm text-white focus:outline-none focus:border-port-accent"
+ >
+ <option value="">{`${runwayModel.size.default} (default)`}</option>
+ {runwayResolutions.map((res) => <option key={res} value={res}>{`${res} · ${describeRunwayVideoRate(runwayModel.id, res)}`}</option>)}
+ </select>
+ </FormField>
+ )}
+ <p className="col-span-2 text-[11px] text-gray-500 leading-snug">
+ Animates your start frame on the Runway API. Counts against your Runway credits
+ {(() => {
+ const estimate = estimateRunwayVideoCostUsd({
+ modelId: runwayModel?.id, seconds: runwayDuration === '' ? null : Number(runwayDuration), resolution: runwayResolution,
+ });
+ return estimate == null ? ' — cost unknown for this model.' : ` — about ${formatUsd(estimate)} at list price.`;
+ })()}
+ </p>
+ </div>
) : isReactor ? (
<ReactorPanel
clipId={reactorClipId}
diff --git a/server/lib/README.md b/server/lib/README.md
index 7a5523777a..7b3f9657d6 100644
--- a/server/lib/README.md
+++ b/server/lib/README.md
@@ -339,6 +339,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
| `musicVideoShotTiming.js` | Music Video shot modes (#8977): `MUSIC_VIDEO_SHOT_MODES` (`cutaway`/`performance`), `isPerformanceScene`, `MUSIC_VIDEO_PERFORMANCE_STYLES` (`lipsync`/`music`) + `isLipSyncScene`, `selectedPerformanceInstruction(scene)` (the selected take's shot instruction, performance scenes only), `clipAudioOffsetSec(instruction)` (provider lead padding a landed take's edit points were moved by, #11210), `SOURCE_AUDIO_LIPSYNC` (verified source-audio lip-sync lanes — only fal.ai MiniMax H3 lip-sync, 5–30s per take, default 1080P), `SOURCE_AUDIO_MUSIC` (music-driven lanes — fal.ai audio-to-video models, picked by `videoSettings.falMusicModelId`, default OmniHuman v1.5 at 1080p), `falSceneTake()` (the fal model/length/resolution a scene take requests and its estimated cost — shared by the board, the submit path and the autopilot spend cap) + `falTakeRequestFields(take)` (that take as `/api/video-gen` fields), `performanceCapability(backend, { scene, videoSettings })` (lip-sync lane, or the music-driven one for a music-driven scene), `performanceBlockedReason(backend)` (Grok is cutaway-only), `planPerformanceWindow()` (provider-bounded audio window + clip-relative edit in/out points; refuses over-long shots instead of relying on provider truncation), `planPerformanceRepair()` (complete measured suffix evidence, safe word-gap boundary, source/clip timebases and next spend), `clipRelativeCues()`, `grokCoverage(spanSec)` (covering 6/10/15 s request, flags spans needing a split), `approximateMotionCues()`, `shotSplitLimit(scene, backend)` (per-take limit: lip-sync window or Grok's longest clip), `planShotSplit()` (contiguous pieces cut at lyric pauses → line/phrase boundaries → beats; one extra piece at most to avoid cutting a sung line). Dependency-free; the client re-exports it. |
| `reactorStartingFrame.js` | Fits a reactor.inc `starting_frame` to the fast-h3 session canvas before it is uploaded. `readImageSize(path)` → `{width,height}` or `null` (never a guessed dimension), and `prepareReactorStartingFrame(sourceImagePath, requestedAspect, outputPath)` → `{ aspect, canvas, framePath, fittedPath }`: an absent `requestedAspect` DERIVES the canvas from the frame's own shape (`nearestReactorAspect`), then centre cover-crops the frame to that canvas's exact pixel size so reactor's own "fitted to the session canvas" step is a no-op. `fittedPath` is the temp file the caller deletes; `null` means the original was uploaded unchanged. A frame sharp cannot decode (or a resize that fails) passes through untouched on the default canvas rather than failing a render PortOS merely could not measure — reactor decodes more containers than sharp does. |
| `reactorVideoClip.js` | The reactor.inc `fast-h3` clip contract, dependency-free so the route schema, the FableLoom prompt compiler and the VideoGen picker all read one copy — `REACTOR_MAX_PROMPT_LENGTH` (800; the API rejects a longer prompt rather than truncating it), `REACTOR_MIN_CLIP_SECONDS` / `REACTOR_MAX_CLIP_SECONDS` (5.167–14.375, i.e. 124–345 frames at the `REACTOR_CLIP_FPS` 24 it delivers), `REACTOR_MAX_CLIP_ID_LENGTH`, and the picker list `REACTOR_CLIP_LENGTHS` + `REACTOR_DEFAULT_CLIP_LENGTH` + `reactorClipLengthLabel()` (the two exact endpoints plus every whole second between them, since the accepted range is continuous but starts and ends off a round number). Also the CANVAS contract `set_canvas` accepts: `REACTOR_CANVASES` (`16:9` 1344×768, `4:3` 1024×768, `1:1` 768×768, `9:16` 768×1344 — every one a 768px short edge, so the aspect string is the whole choice), `REACTOR_ASPECTS`, `REACTOR_DEFAULT_ASPECT`, `reactorCanvas(aspect)` and `nearestReactorAspect(width, height)` (closest canvas in log-ratio space; anything unmeasurable answers the default rather than deriving one). Mirrored to `client/src/lib/reactorVideoClip.js`, parity pinned by `reactorVideoClip.test.js`. |
+| `runwayVideoModels.js` | Curated Runway VIDEO model catalog for the developer API's `/v1/image_to_video` — `RUNWAY_VIDEO_MODELS` (Gen-4.5 default, Gen-4 Turbo, Veo 3.1 / Fast, Kling 3.0 Standard / Pro, Kling 2.5 Turbo Pro, MiniMax H3 Max, WAN 3.0) with per-model prompt caps, duration rules, output sizes (`ratio` or `resolution`), end-frame support and list prices verified on `RUNWAY_VIDEO_PRICING_VERIFIED_AT`; `getRunwayVideoModel()`, `coerceRunwayVideoSeconds()` (round UP to a covering length), `resolveRunwayVideoResolution()`, `runwayVideoLengths()`, `buildRunwayVideoRequest()` (the exact body; refuses an uncurated model, a missing frame or an over-cap prompt, never cuts it), `runwayVideoClipQuote()` / `estimateRunwayVideoCostUsd()`, `describeRunwayVideoRate()`. Dependency-free; the client re-exports it. |
| `kimi.js` | Moonshot AI Kimi Code (`kimi`) provider helpers — id constants (`KIMI_CLI_ID`/`KIMI_TUI_ID`), `isKimiCommand`/`isKimiCliProvider`/`isKimiTuiProvider` predicates, `ensureKimiHeadlessArgs` (model flag only — non-interactive mode is implicit in `--prompt`, and kimi has no `--print`/`--afk` and refuses approval flags alongside `--prompt`) / `ensureKimiTuiArgs` (`--yolo`) argv builders (model selection uses the `KIMI_CONFIGURED_DEFAULT` sentinel in `providerModels.js` so PortOS omits `--model` like Grok/Antigravity), and `prepareKimiPrompt` (delivers the prompt as the `--prompt <value>` argv, not stdin). |
| `runners.js` | Image-runner family constants (`RUNNER_FAMILIES`, `loraCompatKey`), video LoRA families, MiniMax H3 runtime predicates, and the shared `AUDIO_TO_VIDEO_RUNTIMES` / `isAudioToVideoRuntime` capability gate (LTX plus H3 Ref2VA). Mirrored to `client/src/lib/runnerFamilies.js`. |
| `codexAccount.js` | The ChatGPT-subscription half of the Codex providers (#5589), pure. The fixed `codex app-server` invocation, the `account/*` method + notification names, `CODEX_ACCOUNT_STATUS` (`runtime-missing` / `unknown` / `signed-out` / `login-pending` / `ready` / `quota-exhausted` / `reauth-required`) and `CODEX_ERROR_CODES`, `isCodexSubscriptionProvider` (re-exported from `providerModels.js`; keyed on the `codex` COMMAND, so a renamed clone still resolves), the `normalizeCodexAccount` / `normalizeCodexRateLimits` / `normalizeCodexLoginStart` payload normalizers, `redactCodexPayload` (every log/error/response path runs through it, so "PortOS never writes down a Codex token" is one function rather than a rule per call site), and `deriveCodexAccountStatus`. Sentinels are load-bearing: `runtimeInstalled: null` = NOT PROBED, `accountFetched: false` ≠ signed out, `rateLimits: null` (failed fetch) ≠ a fetched window with nothing to report. The process half is `services/codexAppServer.js`. |
diff --git a/server/lib/credentialRegistry.js b/server/lib/credentialRegistry.js
index 3e04680fc0..0a77181b76 100644
--- a/server/lib/credentialRegistry.js
+++ b/server/lib/credentialRegistry.js
@@ -44,6 +44,12 @@ export const CREDENTIALS = Object.freeze([
getUrl: 'https://reactor.inc/', envVars: Object.freeze(['REACTOR_API_KEY']),
settingsPath: 'videoGen.reactor.apiKey', configurePath: '/settings/credentials',
}),
+ Object.freeze({
+ id: 'runway', label: 'Runway', privateStore: true,
+ unlocks: 'Video generation through Runway\'s developer API.', tier: 'metered',
+ getUrl: 'https://dev.runwayml.com/', envVars: Object.freeze(['RUNWAYML_API_SECRET']),
+ settingsPath: 'videoGen.runway.apiKey', configurePath: '/settings/credentials',
+ }),
Object.freeze({
id: 'github',
label: 'GitHub',
diff --git a/server/lib/generationModes.js b/server/lib/generationModes.js
index 4a9b57c781..b5b4b672ec 100644
--- a/server/lib/generationModes.js
+++ b/server/lib/generationModes.js
@@ -38,18 +38,20 @@ export const QUEUEABLE_IMAGE_MODES = Object.freeze([
// Video deliberately shares the image backend discriminator namespace. Local
// video's text/image/fflf modes are a separate semantic value carried elsewhere.
-// REACTOR has no image-gen counterpart (issue #6214 is video-only), so it is a
-// video-only literal rather than a re-export of an IMAGE_GEN_MODE entry.
+// REACTOR and RUNWAY have no image-gen counterpart, so they are video-only
+// literals rather than re-exports of an IMAGE_GEN_MODE entry.
export const VIDEO_GEN_MODE = Object.freeze({
LOCAL: IMAGE_GEN_MODE.LOCAL,
GROK: IMAGE_GEN_MODE.GROK,
FAL: IMAGE_GEN_MODE.FAL,
REACTOR: 'reactor',
+ // Runway's developer API (services/videoGen/runway.js).
+ RUNWAY: 'runway',
});
export const VIDEO_GEN_MODES = Object.freeze(Object.values(VIDEO_GEN_MODE));
export const CLOUD_VIDEO_GEN_MODES = Object.freeze([
- VIDEO_GEN_MODE.GROK, VIDEO_GEN_MODE.FAL, VIDEO_GEN_MODE.REACTOR,
+ VIDEO_GEN_MODE.GROK, VIDEO_GEN_MODE.FAL, VIDEO_GEN_MODE.REACTOR, VIDEO_GEN_MODE.RUNWAY,
]);
/**
diff --git a/server/lib/generationModes.test.js b/server/lib/generationModes.test.js
index 3cdee1ec5e..8c47736d0f 100644
--- a/server/lib/generationModes.test.js
+++ b/server/lib/generationModes.test.js
@@ -22,12 +22,12 @@ describe('generation mode alphabets', () => {
].every(Object.isFrozen)).toBe(true);
});
- it('keeps the video backend alphabet in the image discriminator namespace, plus reactor as a video-only extra', () => {
+ it('keeps the video backend alphabet in the image discriminator namespace, plus reactor and runway as video-only extras', () => {
expect(VIDEO_GEN_MODE).toEqual({
- LOCAL: IMAGE_GEN_MODE.LOCAL, GROK: IMAGE_GEN_MODE.GROK, FAL: IMAGE_GEN_MODE.FAL, REACTOR: 'reactor',
+ LOCAL: IMAGE_GEN_MODE.LOCAL, GROK: IMAGE_GEN_MODE.GROK, FAL: IMAGE_GEN_MODE.FAL, REACTOR: 'reactor', RUNWAY: 'runway',
});
- expect(VIDEO_GEN_MODES).toEqual(['local', 'grok', 'fal', 'reactor']);
- expect(CLOUD_VIDEO_GEN_MODES).toEqual(['grok', 'fal', 'reactor']);
+ expect(VIDEO_GEN_MODES).toEqual(['local', 'grok', 'fal', 'reactor', 'runway']);
+ expect(CLOUD_VIDEO_GEN_MODES).toEqual(['grok', 'fal', 'reactor', 'runway']);
expect([VIDEO_GEN_MODE, VIDEO_GEN_MODES, CLOUD_VIDEO_GEN_MODES].every(Object.isFrozen)).toBe(true);
});
});
diff --git a/server/lib/index.js b/server/lib/index.js
index 1723fc8f3e..107639361b 100644
--- a/server/lib/index.js
+++ b/server/lib/index.js
@@ -213,6 +213,7 @@ export * from './fontMetadata.js';
export * from './musicVideoCoverOverlay.js';
export * from './musicVideoShotTiming.js';
export * from './reactorStartingFrame.js';
+export * from './runwayVideoModels.js';
export * from './reactorVideoClip.js';
export * from './heavyJobClaim.js';
export * from './hfErrors.js';
diff --git a/server/lib/musicVideoAutomation.js b/server/lib/musicVideoAutomation.js
index 1b43379f46..9d4e430619 100644
--- a/server/lib/musicVideoAutomation.js
+++ b/server/lib/musicVideoAutomation.js
@@ -31,6 +31,7 @@ const VIDEO_TOOL_LABELS = {
[VIDEO_GEN_MODE.GROK]: 'Grok video',
[VIDEO_GEN_MODE.FAL]: 'fal.ai video',
[VIDEO_GEN_MODE.REACTOR]: 'Reactor video',
+ [VIDEO_GEN_MODE.RUNWAY]: 'Runway video',
};
// Metered = spends money or remote quota (the cloud lists); the budget cap is
diff --git a/server/lib/musicVideoEndFrame.js b/server/lib/musicVideoEndFrame.js
index b913834e72..7bc0dae01e 100644
--- a/server/lib/musicVideoEndFrame.js
+++ b/server/lib/musicVideoEndFrame.js
@@ -10,6 +10,7 @@
* and the production dispatcher make the same call.
*/
import { FAL_DEFAULT_IMAGE_VIDEO_MODEL, getFalVideoModel } from './falVideoModels.js';
+import { RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL, getRunwayVideoModel } from './runwayVideoModels.js';
import { isPerformanceScene } from './musicVideoShotTiming.js';
import { isLastFrameAnchoredRuntime } from './runners.js';
import { VIDEO_RUNTIME_MODES } from './videoModeProfiles.js';
@@ -23,13 +24,13 @@ const localRuntimeEndsOnFrame = (runtime) => isLastFrameAnchoredRuntime(runtime)
const endImageOf = (scene) => (typeof scene?.endImageId === 'string' && scene.endImageId.trim() ? scene.endImageId : null);
/**
- * Whether a clip for `scene` on `backend` (with `falModelId` for fal, the local
+ * Whether a clip for `scene` on `backend` (with `falModelId` for fal, `runwayModelId` for Runway, the local
* model's `localRuntime` for local, and the request's video `mode`,
* image-to-video by default) sends the scene's end frame. Returns
* `{ endImageId, used, reason }`: `reason` names why a selected end frame is
* ignored, and is null when it is used or absent.
*/
-export function sceneEndFramePlan(scene, { backend = null, falModelId = null, mode = 'image', localRuntime = null } = {}) {
+export function sceneEndFramePlan(scene, { backend = null, falModelId = null, runwayModelId = null, mode = 'image', localRuntime = null } = {}) {
const endImageId = endImageOf(scene);
if (!endImageId) return { endImageId: null, used: false, reason: null };
const ignored = (reason) => ({ endImageId, used: false, reason });
@@ -40,7 +41,13 @@ export function sceneEndFramePlan(scene, { backend = null, falModelId = null, mo
? { endImageId, used: true, reason: null }
: ignored('This local model animates from the start frame only; local LTX-2.x and MiniMax H3 end on the end frame');
}
- if (backend !== 'fal') return ignored('Only fal.ai models that take an end frame and local LTX use it; this renderer ignores it');
+ if (backend === 'runway') {
+ const model = getRunwayVideoModel(runwayModelId || RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL);
+ return model?.endImage
+ ? { endImageId, used: true, reason: null }
+ : ignored(`${model?.label || 'This Runway model'} takes no end frame; choose one that does`);
+ }
+ if (backend !== 'fal') return ignored('Only fal.ai and Runway models that take an end frame and local LTX use it; this renderer ignores it');
const model = getFalVideoModel(falModelId || FAL_DEFAULT_IMAGE_VIDEO_MODEL);
if (!model?.endImage) return ignored(`${model?.label || 'This fal.ai model'} takes no end frame; choose one that does`);
return { endImageId, used: true, reason: null };
diff --git a/server/lib/musicVideoShotTiming.js b/server/lib/musicVideoShotTiming.js
index c23b871f5d..d8d95e0c6b 100644
--- a/server/lib/musicVideoShotTiming.js
+++ b/server/lib/musicVideoShotTiming.js
@@ -35,8 +35,11 @@
* take makes on the fal lane and its estimated cost, shared by the board,
* the submit path and the production autopilot's spend cap.
*
- * Dependency-free (only grokVideoClip.js and falVideoModels.js, themselves
- * dependency-free) so the client
+ * - `runwaySceneTake` — the same answer for a cutaway on the Runway lane;
+ * `meteredSceneTake` picks the paid lane's take for a backend.
+ *
+ * Dependency-free (only grokVideoClip.js, falVideoModels.js and
+ * runwayVideoModels.js, themselves dependency-free) so the client
* re-exports it from `client/src/lib/musicVideoShotTiming.js` and the two sides
* cannot drift.
*/
@@ -54,6 +57,13 @@ import {
getFalVideoModel,
resolveFalVideoResolution,
} from './falVideoModels.js';
+import {
+ RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL,
+ coerceRunwayVideoSeconds,
+ estimateRunwayVideoCostUsd,
+ getRunwayVideoModel,
+ resolveRunwayVideoResolution,
+} from './runwayVideoModels.js';
export const MUSIC_VIDEO_SHOT_MODES = Object.freeze(['cutaway', 'performance']);
@@ -218,8 +228,8 @@ export function performanceCapability(backend, { scene = null, videoSettings = n
*/
export function performanceBlockedReason(backend) {
if (performanceCapability(backend)) return null;
- if (backend === 'grok') {
- return 'Grok video is cutaway-only: it cannot synchronize a singer to the song. Pin fal.ai for performance shots, or switch this scene to Cutaway.';
+ if (backend === 'grok' || backend === 'runway') {
+ return `${backend === 'grok' ? 'Grok' : 'Runway'} video is cutaway-only: it cannot synchronize a singer to the song. Pin fal.ai for performance shots, or switch this scene to Cutaway.`;
}
return 'Performance shots need a source-audio lip-sync provider — pin fal.ai as this project\'s video backend.';
}
@@ -579,6 +589,51 @@ export const falTakeRequestFields = (take) => (take.performance
...(take.resolution ? { falResolution: take.resolution } : {}),
});
+/**
+ * The Runway request one cutaway take of `scene` makes, and its estimated cost:
+ * `videoSettings.runwayModelId` (default Gen-4.5) at `runwayResolution`, for
+ * the pinned `runwayDuration` or else the shot's span rounded UP to a length the
+ * model delivers. A shot with an end frame stays on its model; the end frame is
+ * sent only when that model takes one (musicVideoEndFrame.js). Returns
+ * `{ performance: false, modelId, seconds, resolution, costUsd }`; Runway has
+ * no performance lane, so a performance scene is priced as nothing (null).
+ */
+export function runwaySceneTake({ scene, videoSettings = {} } = {}) {
+ const settings = videoSettings || {};
+ const modelId = settings.runwayModelId || RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL;
+ const model = getRunwayVideoModel(modelId);
+ if (isPerformanceScene(scene) || !model) {
+ return { performance: isPerformanceScene(scene), modelId, seconds: null, resolution: null, costUsd: null };
+ }
+ const spanSec = typeof scene?.startSec === 'number' && typeof scene?.endSec === 'number' && scene.endSec > scene.startSec
+ ? scene.endSec - scene.startSec
+ : null;
+ const seconds = coerceRunwayVideoSeconds(model, settings.runwayDuration || spanSec);
+ const resolution = resolveRunwayVideoResolution(model, settings.runwayResolution);
+ return { performance: false, modelId, seconds, resolution, costUsd: estimateRunwayVideoCostUsd({ modelId, seconds, resolution }) };
+}
+
+/** A `runwaySceneTake` as `/api/video-gen` fields. */
+export const runwayTakeRequestFields = (take) => ({
+ runwayModelId: take.modelId,
+ ...(take.seconds != null ? { runwayDuration: take.seconds } : {}),
+ ...(take.resolution ? { runwayResolution: take.resolution } : {}),
+});
+
+/**
+ * The priced take of `scene` on a paid lane (`fal` or `runway`), or null for a
+ * lane PortOS cannot price (local, Grok): the one answer the board's cost line,
+ * the batch estimate, the submit path and the autopilot's cap read.
+ */
+export function meteredSceneTake(backend, options = {}) {
+ if (backend === 'fal') return falSceneTake(options);
+ if (backend === 'runway') return runwaySceneTake(options);
+ return null;
+}
+
+/** The label of a paid take's model, from whichever catalog holds it. */
+export const meteredModelLabel = (modelId) => getFalVideoModel(modelId)?.label || getRunwayVideoModel(modelId)?.label || null;
+
/** A repair needs measured complete coverage and a genuine pause between words.
* Returned times distinguish song, excerpt and provider-clip timebases.
* No padding: accepted audio must never be sent for generation again.
diff --git a/server/lib/musicVideoValidation.js b/server/lib/musicVideoValidation.js
index d49207f91e..ad958e3874 100644
--- a/server/lib/musicVideoValidation.js
+++ b/server/lib/musicVideoValidation.js
@@ -145,8 +145,9 @@ export const musicVideoVideoSettingsSchema = z.object({
// null clears the per-project pin so this install's configured default wins.
// 'fal' is the metered fal.ai queue REST backend (server/services/videoGen/fal.js,
// #8968) — image-to-video only here; the audio-reactive lane stays local-only
- // (see generationMode below).
- backend: z.enum(['local', 'grok', 'fal']).nullable().optional(),
+ // (see generationMode below). 'runway' is Runway's developer API
+ // (services/videoGen/runway.js), cutaways only.
+ backend: z.enum(['local', 'grok', 'fal', 'runway']).nullable().optional(),
modelId: z.string().max(64).nullable().optional(),
grokDuration: z.union(GROK_VIDEO_DURATIONS.map((d) => z.literal(d))).optional(),
// fal.ai clip length in seconds. Loosely bounded (not a closed enum) for the
@@ -177,6 +178,12 @@ export const musicVideoVideoSettingsSchema = z.object({
// falls back to the default (OmniHuman v1.5).
falMusicModelId: z.string().min(1).max(200).nullable().optional(),
falMusicResolution: z.string().min(1).max(16).nullable().optional(),
+ // Runway cutaway model (lib/runwayVideoModels.js), clip length and output
+ // size. Loosely validated like falModelId; null/omitted = the default model,
+ // the length covering each shot, that model's default size.
+ runwayModelId: z.string().min(1).max(64).nullable().optional(),
+ runwayDuration: z.number().min(1).max(60).nullable().optional(),
+ runwayResolution: z.string().min(1).max(16).nullable().optional(),
generationMode: z.enum(['image', 'audioReactive', 'suppliedAudio']).optional(),
audioReactiveLora: z.string().max(255).regex(/^[^/\\]+\.safetensors$/i).nullable().optional(),
audioReactiveScale: z.number().min(0).max(2).optional(),
@@ -1614,7 +1621,7 @@ export const musicVideoSceneReorderSchema = z.object({
// Split an over-long shot on lyric/phrase boundaries (#8977). `backend` is
// the lane the director will render with; omitted/null = the project's pin.
export const musicVideoSceneSplitSchema = z.object({
- backend: z.enum(['local', 'grok', 'fal']).nullable().optional(),
+ backend: z.enum(['local', 'grok', 'fal', 'runway']).nullable().optional(),
// Split the shot in two at this song time instead of by the backend's longest take.
atSec: z.number().finite().nonnegative().optional(),
}).strict();
diff --git a/server/lib/runwayVideoModels.js b/server/lib/runwayVideoModels.js
new file mode 100644
index 0000000000..b2793cbfd7
--- /dev/null
+++ b/server/lib/runwayVideoModels.js
@@ -0,0 +1,275 @@
+/**
+ * Curated Runway VIDEO model catalog — what PortOS knows about each model it
+ * offers through Runway's developer API (`POST /v1/image_to_video`): the
+ * request fields each model takes (the body is one shape per `model`), its
+ * duration and output-size rules, whether it ends on a last frame, and its
+ * published price.
+ *
+ * Every row was checked against Runway's API reference and pricing guide
+ * (docs.dev.runwayml.com: api.md, guides/pricing.md, assets/inputs.md) on
+ * RUNWAY_VIDEO_PRICING_VERIFIED_AT. Runway changes these; when a render is
+ * refused with a 400 or a bill disagrees with an estimate, re-check that
+ * model's section and edit its row.
+ *
+ * Veo is deliberately not offered (a project choice), though
+ * Runway serves it; the native `negativePrompt` row field stays for a model
+ * that takes one.
+ *
+ * Image-to-video only: every row animates a start frame, and the rows marked
+ * `endImage` also end on a last frame. Provider audio is always requested off
+ * (`audio: false`) where a model has the field, since a PortOS clip is silent
+ * unless the caller asks (a Music Video's master song is its only soundtrack),
+ * and the silent rate is what is priced.
+ *
+ * Runway bills credits at $0.01 each; prices below are credits per second ÷ 100.
+ *
+ * Dependency-free so `client/src/lib/runwayVideoModels.js` can re-export it and
+ * the board, the provider and the autopilot price a take identically.
+ */
+
+/** Date the schemas and prices below were last verified against Runway's docs. */
+export const RUNWAY_VIDEO_PRICING_VERIFIED_AT = '2026-10-19';
+
+/** Runway's own Gen-4.5: the default when a request names no model. */
+export const RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL = 'gen4.5';
+
+const GEN4_RATIOS = Object.freeze(['1280:720', '720:1280', '1104:832', '960:960', '832:1104', '1584:672']);
+
+/** The curated catalog, one frozen row per Runway model. */
+export const RUNWAY_VIDEO_MODELS = Object.freeze([
+ {
+ id: 'gen4.5', label: 'Runway Gen-4.5', family: 'runway-gen4',
+ prompt: 'required', promptMaxChars: 1000,
+ endImage: false,
+ duration: { kind: 'range', min: 2, max: 10, defaultSec: 5 },
+ // `ratio` is required and names the output canvas.
+ size: { param: 'ratio', required: true, byResolution: { '720p': GEN4_RATIOS }, default: '720p' },
+ audio: false, seed: true, negativePrompt: false,
+ pricing: { perSecond: 0.12 },
+ },
+ {
+ id: 'gen4_turbo', label: 'Runway Gen-4 Turbo', family: 'runway-gen4',
+ prompt: 'optional', promptMaxChars: 1000,
+ endImage: false,
+ duration: { kind: 'range', min: 2, max: 10, defaultSec: 5 },
+ size: { param: 'ratio', required: true, byResolution: { '720p': GEN4_RATIOS }, default: '720p' },
+ audio: false, seed: true, negativePrompt: false,
+ pricing: { perSecond: 0.05 },
+ },
+ {
+ id: 'kling3.0_standard', label: 'Kling 3.0 Standard', family: 'kling-3.0',
+ prompt: 'optional', promptMaxChars: 2500,
+ endImage: true,
+ duration: { kind: 'range', min: 3, max: 15, defaultSec: 5 },
+ // Optional; with a last frame the canvas follows the first frame.
+ size: { param: 'ratio', required: false, byResolution: { '720p': ['1280:720', '720:1280', '960:960'] }, default: '720p' },
+ // 13 credits/s with audio (Runway's default), 9 with audio off.
+ audio: true, seed: false, negativePrompt: false,
+ pricing: { perSecond: 0.09 },
+ },
+ {
+ id: 'kling3.0_pro', label: 'Kling 3.0 Pro', family: 'kling-3.0',
+ prompt: 'optional', promptMaxChars: 2500,
+ endImage: true,
+ duration: { kind: 'range', min: 3, max: 15, defaultSec: 5 },
+ size: { param: 'ratio', required: false, byResolution: { '1080p': ['1920:1080', '1080:1920', '1440:1440'] }, default: '1080p' },
+ // 17 credits/s with audio, 12 with audio off.
+ audio: true, seed: false, negativePrompt: false,
+ pricing: { perSecond: 0.12 },
+ },
+ {
+ id: 'kling2.5_turbo_pro', label: 'Kling 2.5 Turbo Pro', family: 'kling-2.5',
+ prompt: 'optional', promptMaxChars: 2500,
+ endImage: true,
+ duration: { kind: 'enum', values: [5, 10], defaultSec: 5 },
+ // No ratio field: the canvas follows the start frame.
+ size: null,
+ audio: false, seed: false, negativePrompt: false,
+ pricing: { perSecond: 0.12 },
+ },
+ {
+ id: 'h3_max', label: 'MiniMax H3 Max', family: 'minimax-h3',
+ prompt: 'required', promptMaxChars: 6000,
+ endImage: true,
+ duration: { kind: 'range', min: 5, max: 15, defaultSec: 5 },
+ // A resolution, not a ratio: the canvas follows the start frame.
+ size: { param: 'resolution', required: false, byResolution: { '480p': null, '768p': null }, default: '768p' },
+ audio: false, seed: true, negativePrompt: false,
+ pricing: { perSecond: { '480p': 0.05, '768p': 0.08 } },
+ },
+ {
+ id: 'wan3', label: 'WAN 3.0', family: 'wan-3',
+ prompt: 'required', promptMaxChars: 20000,
+ endImage: true,
+ duration: { kind: 'range', min: 2, max: 30, defaultSec: 5 },
+ // A keyframe render must use an `auto_*` ratio: the canvas follows the first frame.
+ size: { param: 'ratio', required: false, byResolution: { '480p': ['auto_480p'], '720p': ['auto_720p'], '1080p': ['auto_1080p'] }, default: '720p' },
+ audio: true, seed: false, negativePrompt: false,
+ pricing: { perSecond: { '480p': 0.05, '720p': 0.10, '1080p': 0.20 } },
+ },
+].map((model) => Object.freeze(model)));
+
+const BY_ID = new Map(RUNWAY_VIDEO_MODELS.map((m) => [m.id, m]));
+
+/** The catalog row for `modelId`, or null for a model PortOS has not curated. */
+export const getRunwayVideoModel = (modelId) => (typeof modelId === 'string' ? BY_ID.get(modelId.trim()) || null : null);
+
+/** Output sizes a model offers (empty when it has no size parameter). */
+export const runwayVideoResolutions = (model) => (model?.size ? Object.keys(model.size.byResolution) : []);
+
+/** The model's spelling of `requested` (case-insensitive), else its default; null without a size parameter. */
+export function resolveRunwayVideoResolution(model, requested) {
+ if (!model?.size) return null;
+ const want = typeof requested === 'string' ? requested.trim().toLowerCase() : '';
+ return runwayVideoResolutions(model).find((r) => r.toLowerCase() === want) || model.size.default;
+}
+
+/**
+ * The clip length `model` renders for a request of `seconds`: the shortest
+ * length that still COVERS it, else the longest (a Music Video shot must not
+ * be under-covered, and nothing loops or stretches footage). An absent or
+ * invalid request gets the model's default.
+ */
+export function coerceRunwayVideoSeconds(model, seconds) {
+ const rule = model?.duration;
+ if (!rule) return null;
+ const n = Number(seconds);
+ if (seconds == null || seconds === '' || !Number.isFinite(n) || n <= 0) return rule.defaultSec;
+ if (rule.kind === 'enum') return rule.values.find((v) => v >= n - 1e-6) ?? rule.values[rule.values.length - 1];
+ return Math.min(rule.max, Math.max(rule.min, Math.ceil(n - 1e-6)));
+}
+
+/** Clip lengths a model renders, for a length picker. */
+export function runwayVideoLengths(model) {
+ const rule = model?.duration;
+ if (rule?.kind === 'enum') return [...rule.values];
+ if (rule?.kind === 'range') return Array.from({ length: rule.max - rule.min + 1 }, (_, i) => rule.min + i);
+ return [];
+}
+
+const ratioOf = (r) => { const [w, h] = String(r).split(':').map(Number); return w / h; };
+const aspectOf = ({ aspectRatio, width, height }) => {
+ const w = Number(width);
+ const h = Number(height);
+ if (w > 0 && h > 0) return w / h;
+ return /^\d+:\d+$/.test(aspectRatio || '') ? ratioOf(aspectRatio) : null;
+};
+
+/** The size value sent for `resolution`: the closest listed ratio to the target aspect, or the resolution itself. */
+function pickSizeValue(model, resolution, target) {
+ const options = model.size.byResolution[resolution];
+ if (options == null) return resolution;
+ if (options.length === 1) return options[0];
+ // No target: a landscape canvas, Runway's own documented example.
+ const want = target ?? 16 / 9;
+ return options.reduce((best, option) => (Math.abs(ratioOf(option) - want) < Math.abs(ratioOf(best) - want) ? option : best));
+}
+
+const perSecondRate = (model, resolution) => {
+ const { perSecond } = model.pricing;
+ return typeof perSecond === 'number' ? perSecond : perSecond?.[resolution] ?? null;
+};
+
+const round4 = (n) => Math.round(n * 1e4) / 1e4;
+
+/**
+ * What one Runway render costs and why, or null for an uncurated model:
+ * `{ modelId, label, requestedSec, billedSec, minBilledSec, resolution, perSec,
+ * costUsd }`. `billedSec` is the length Runway renders and bills, coerced
+ * exactly as the request body will be.
+ */
+export function runwayVideoClipQuote({ modelId, seconds, resolution } = {}) {
+ const model = getRunwayVideoModel(modelId);
+ if (!model) return null;
+ const billedSec = coerceRunwayVideoSeconds(model, seconds);
+ const res = resolveRunwayVideoResolution(model, resolution);
+ const perSec = perSecondRate(model, res);
+ if (perSec == null) return null;
+ const requested = Number(seconds);
+ const rule = model.duration;
+ return {
+ modelId: model.id,
+ label: model.label,
+ requestedSec: Number.isFinite(requested) && requested > 0 ? requested : null,
+ billedSec,
+ minBilledSec: rule.kind === 'enum' ? rule.values[0] : rule.min,
+ resolution: res,
+ perSec: round4(perSec),
+ costUsd: round4(perSec * billedSec),
+ };
+}
+
+/** Estimated USD for one Runway render, or null when it cannot be priced. */
+export const estimateRunwayVideoCostUsd = (options = {}) => runwayVideoClipQuote(options)?.costUsd ?? null;
+
+/** A one-line human rate, e.g. "$0.12/s · bills 2s min" or "$0.08/s at 768p · bills 5s min". */
+export function describeRunwayVideoRate(modelId, resolution) {
+ const model = getRunwayVideoModel(modelId);
+ if (!model) return null;
+ const res = resolveRunwayVideoResolution(model, resolution);
+ const perSec = perSecondRate(model, res);
+ if (perSec == null) return null;
+ const multiRate = typeof model.pricing.perSecond !== 'number';
+ const rule = model.duration;
+ return `$${perSec}/s${multiRate && res ? ` at ${res}` : ''} · bills ${rule.kind === 'enum' ? rule.values[0] : rule.min}s min`;
+}
+
+/**
+ * Build the `/v1/image_to_video` body for a curated model. Returns `{ body,
+ * model, seconds, resolution, estimatedCostUsd }`. A missing required input, an
+ * input the model cannot take, an uncurated model, or an over-cap prompt throws
+ * an Error with `code = 'RUNWAY_MODEL_INPUT'` so the caller refuses before
+ * anything is paid for. A prompt is never shortened (AGENTS.md, #11317).
+ */
+export function buildRunwayVideoRequest({
+ modelId, prompt = '', negativePrompt = '', seconds, aspectRatio, width, height, resolution,
+ imageUri = null, endImageUri = null, seed,
+} = {}) {
+ const refuse = (message) => Object.assign(new Error(message), { code: 'RUNWAY_MODEL_INPUT' });
+ const model = getRunwayVideoModel(modelId || RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL);
+ if (!model) throw refuse(`"${modelId}" is not a Runway model PortOS offers`);
+ const text = typeof prompt === 'string' ? prompt.trim() : '';
+ const avoid = typeof negativePrompt === 'string' ? negativePrompt.trim() : '';
+ if (model.prompt === 'required' && !text) throw refuse(`${model.label} needs a prompt`);
+ // No native field: the negative prompt rides in the prompt as an "Avoid:"
+ // clause, shown to the user as a labeled addition (same fallback as fal.ai).
+ const sent = !model.negativePrompt && avoid ? `${text}\nAvoid: ${avoid}`.trim() : text;
+ if (sent.length > model.promptMaxChars) {
+ throw refuse(`The prompt is ${sent.length.toLocaleString('en-US')} characters${sent === text ? '' : ' with its "Avoid:" clause'}; ${model.label} takes ${model.promptMaxChars.toLocaleString('en-US')}. Shorten it to render.`);
+ }
+ if (model.negativePrompt && avoid.length > model.negativePromptMaxChars) {
+ throw refuse(`The negative prompt is ${avoid.length.toLocaleString('en-US')} characters; ${model.label} takes ${model.negativePromptMaxChars.toLocaleString('en-US')}. Shorten it to render.`);
+ }
+ if (!imageUri) throw refuse(`${model.label} renders from a start frame. Supply a source image.`);
+ if (endImageUri && !model.endImage) throw refuse(`${model.label} takes no end frame; choose a model that does`);
+
+ const body = { model: model.id };
+ body.promptImage = endImageUri
+ ? [{ uri: imageUri, position: 'first' }, { uri: endImageUri, position: 'last' }]
+ : imageUri;
+ if (sent) body.promptText = sent;
+ if (model.negativePrompt && avoid) body.negativePrompt = avoid;
+ const coercedSec = coerceRunwayVideoSeconds(model, seconds);
+ body.duration = coercedSec;
+ const res = resolveRunwayVideoResolution(model, resolution);
+ if (model.size) {
+ const target = aspectOf({ aspectRatio, width, height });
+ // An optional ratio is sent only when there is a target to match and no
+ // last frame (a keyframe render's canvas follows its first frame);
+ // otherwise Runway sizes from the frame.
+ if (model.size.required || model.size.param === 'resolution' || model.size.byResolution[res]?.length === 1
+ || (target != null && !endImageUri)) {
+ body[model.size.param] = pickSizeValue(model, res, target);
+ }
+ }
+ if (model.audio) body.audio = false;
+ if (model.seed && Number.isInteger(seed) && seed >= 0 && seed <= 4294967295) body.seed = seed;
+
+ return {
+ body,
+ model,
+ seconds: coercedSec,
+ resolution: res,
+ estimatedCostUsd: estimateRunwayVideoCostUsd({ modelId: model.id, seconds: coercedSec, resolution: res }),
+ };
+}
diff --git a/server/lib/runwayVideoModels.test.js b/server/lib/runwayVideoModels.test.js
new file mode 100644
index 0000000000..7b1c8d792a
--- /dev/null
+++ b/server/lib/runwayVideoModels.test.js
@@ -0,0 +1,97 @@
+import { describe, it, expect } from 'vitest';
+import {
+ RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL,
+ RUNWAY_VIDEO_MODELS,
+ buildRunwayVideoRequest,
+ coerceRunwayVideoSeconds,
+ describeRunwayVideoRate,
+ getRunwayVideoModel,
+ runwayVideoClipQuote,
+} from './runwayVideoModels.js';
+
+// The catalog is the wire boundary for a paid API that validates every field
+// per model: a wrong key or type is a 400 after the user pressed Render, and a
+// wrong price is a bill the board never showed. These pin the bodies against
+// Runway's API reference and the quotes against its pricing guide
+// (RUNWAY_VIDEO_PRICING_VERIFIED_AT).
+
+const FRAME = 'data:image/png;base64,AA==';
+const END = 'data:image/jpeg;base64,BB==';
+const build = (modelId, extra = {}) => buildRunwayVideoRequest({ modelId, prompt: 'a lighthouse at dusk', imageUri: FRAME, ...extra });
+
+describe('buildRunwayVideoRequest — per-family wire bodies', () => {
+ it('Gen-4.5 (the default): required ratio matched to the target shape, covering duration, prompt Avoid fold', () => {
+ const { body, seconds, estimatedCostUsd } = buildRunwayVideoRequest({
+ prompt: 'a lighthouse at dusk', negativePrompt: 'text', imageUri: FRAME, seconds: 3.2, width: 1080, height: 1920, seed: 7,
+ });
+ expect(body).toEqual({
+ model: RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL, promptImage: FRAME, promptText: 'a lighthouse at dusk\nAvoid: text',
+ duration: 4, ratio: '720:1280', seed: 7,
+ });
+ expect(seconds).toBe(4);
+ expect(estimatedCostUsd).toBe(0.48);
+ });
+
+ it('Kling 3.0 Pro: first+last frames, audio off, no ratio on a keyframe render', () => {
+ expect(build('kling3.0_pro', { seconds: 5, negativePrompt: 'blur', endImageUri: END, width: 1920, height: 1080 }).body).toEqual({
+ model: 'kling3.0_pro',
+ promptImage: [{ uri: FRAME, position: 'first' }, { uri: END, position: 'last' }],
+ promptText: 'a lighthouse at dusk\nAvoid: blur', duration: 5, audio: false,
+ });
+ });
+
+ it('Kling 3.0: optional ratio sent only for a start-frame render with a target shape', () => {
+ expect(build('kling3.0_standard', { width: 1280, height: 720 }).body).toMatchObject({ ratio: '1280:720', audio: false, duration: 5 });
+ expect(build('kling3.0_standard', {}).body).not.toHaveProperty('ratio');
+ expect(build('kling3.0_standard', { width: 1280, height: 720, endImageUri: END }).body).not.toHaveProperty('ratio');
+ });
+
+ it('MiniMax H3 / WAN 3.0: size sent as a resolution or an auto ratio, priced per resolution', () => {
+ const h3 = build('h3_max', { seconds: 2, resolution: '480P' });
+ expect(h3.body).toEqual({ model: 'h3_max', promptImage: FRAME, promptText: 'a lighthouse at dusk', duration: 5, resolution: '480p' });
+ expect(h3.estimatedCostUsd).toBe(0.25);
+ expect(build('wan3', { seconds: 12, endImageUri: END }).body).toMatchObject({ ratio: 'auto_720p', duration: 12, audio: false });
+ // Kling 2.5 has no size field and no audio field at all.
+ expect(build('kling2.5_turbo_pro', { width: 1280, height: 720 }).body).toEqual({
+ model: 'kling2.5_turbo_pro', promptImage: FRAME, promptText: 'a lighthouse at dusk', duration: 5,
+ });
+ });
+});
+
+describe('buildRunwayVideoRequest — refusals before anything is paid for', () => {
+ it.each([
+ ['an uncurated model', { modelId: 'gen9' }, /not a Runway model/],
+ ['a missing start frame', { imageUri: null }, /start frame/],
+ ['an end frame on a start-frame-only model', { modelId: 'gen4.5', endImageUri: END }, /takes no end frame/],
+ ['a missing required prompt', { modelId: 'gen4.5', prompt: ' ' }, /needs a prompt/],
+ ])('refuses %s', (_label, extra, message) => {
+ expect(() => build(extra.modelId || 'gen4.5', extra)).toThrow(message);
+ });
+
+ it('refuses an over-cap prompt with its count, counting the Avoid fold, and never shortens it', () => {
+ const prompt = 'x'.repeat(995);
+ expect(() => build('gen4.5', { prompt })).not.toThrow();
+ let err;
+ try { build('gen4.5', { prompt, negativePrompt: 'blurry' }); } catch (e) { err = e; }
+ expect(err).toMatchObject({ code: 'RUNWAY_MODEL_INPUT', message: expect.stringMatching(/1,009 characters with its "Avoid:" clause; Runway Gen-4.5 takes 1,000/) });
+ });
+});
+
+describe('Runway pricing and lengths', () => {
+ it('rounds a shot up to the shortest length that covers it, capped at the longest', () => {
+ expect(coerceRunwayVideoSeconds(getRunwayVideoModel('kling2.5_turbo_pro'), 5.01)).toBe(10);
+ expect(coerceRunwayVideoSeconds(getRunwayVideoModel('kling2.5_turbo_pro'), 20)).toBe(10);
+ expect(getRunwayVideoModel('veo3.1')).toBeNull();
+ expect(coerceRunwayVideoSeconds(getRunwayVideoModel('gen4_turbo'), 1)).toBe(2);
+ expect(coerceRunwayVideoSeconds(getRunwayVideoModel('gen4_turbo'), null)).toBe(5);
+ });
+
+ it('quotes every curated model and describes its rate', () => {
+ for (const model of RUNWAY_VIDEO_MODELS) {
+ expect(runwayVideoClipQuote({ modelId: model.id, seconds: 4 })?.costUsd).toBeGreaterThan(0);
+ expect(describeRunwayVideoRate(model.id)).toMatch(/^\$[\d.]+\/s/);
+ }
+ expect(runwayVideoClipQuote({ modelId: 'wan3', seconds: 4, resolution: '1080p' })).toMatchObject({ billedSec: 4, perSec: 0.2, costUsd: 0.8 });
+ expect(runwayVideoClipQuote({ modelId: 'unknown' })).toBeNull();
+ });
+});
diff --git a/server/lib/validation.js b/server/lib/validation.js
index e02703e857..e60253f117 100644
--- a/server/lib/validation.js
+++ b/server/lib/validation.js
@@ -2145,6 +2145,10 @@ export const videoGenSettingsSchema = z.object({
reactor: z.object({
apiKey: z.preprocess((v) => (v === '' ? undefined : v), z.string().trim().max(200).optional()),
}).optional(),
+ // Runway developer API — same settings-wins-over-env (RUNWAYML_API_SECRET) precedence.
+ runway: z.object({
+ apiKey: z.preprocess((v) => (v === '' ? undefined : v), z.string().trim().max(200).optional()),
+ }).optional(),
});
// POST /api/video-gen/model-terms — record (or withdraw) the acknowledgement of
diff --git a/server/routes/mediaJobs.hostedRetry.test.js b/server/routes/mediaJobs.hostedRetry.test.js
index 92ac9aacc0..a385311b78 100644
--- a/server/routes/mediaJobs.hostedRetry.test.js
+++ b/server/routes/mediaJobs.hostedRetry.test.js
@@ -52,6 +52,7 @@ const HOSTED_JOB_PARAMS = {
grok: { prompt: 'a fox', mode: 'grok', videoMode: 'text', grokPath: '/bin/grok', aspectRatio: '16:9', width: 1280, height: 720, duration: 6 },
fal: { prompt: 'a fox', mode: 'fal', videoMode: 'text', modelId: 'fal-ai/provider-model', aspectRatio: '16:9', width: 1280, height: 720, duration: 5 },
reactor: { prompt: 'a fox', mode: 'reactor', videoMode: 'text', seconds: 4, seed: 7, aspect: '16:9' },
+ runway: { prompt: 'a fox', mode: 'runway', videoMode: 'image', sourceImagePath: '/data/images/frame.png', modelId: 'gen4.5', width: 1280, height: 720, duration: 5 },
};
const seedJob = (id, params) => jobStore.set(id, { id, kind: 'video', owner: null, status: 'failed', params });
const retry = (id, body = {}) => request(makeApp()).post(`/api/media-jobs/${id}/retry`).send(body);
diff --git a/server/routes/settings.js b/server/routes/settings.js
index ae7fb3da88..e655f0e8bc 100644
--- a/server/routes/settings.js
+++ b/server/routes/settings.js
@@ -105,6 +105,10 @@ const redactExternalTokens = (settings) => {
const { apiKey, ...rest } = videoGen.reactor;
videoGen.reactor = rest;
}
+ if (isPlainObject(videoGen.runway)) {
+ const { apiKey, ...rest } = videoGen.runway;
+ videoGen.runway = rest;
+ }
next.videoGen = videoGen;
}
return next;
@@ -159,6 +163,7 @@ const preserveExternallyOwnedKeys = (next, current) => {
carryOver('videoGen', 'acceptedModelTerms', { alwaysStored: true });
carryOverNested('videoGen', 'fal', 'apiKey');
carryOverNested('videoGen', 'reactor', 'apiKey');
+ carryOverNested('videoGen', 'runway', 'apiKey');
// beeperSettingsSchema is `.strict()` with no `token`/`tokenExpiresAt`
// field, so a valid `beeper` PUT through this route can never carry either
// — without this, the generic top-level shallow merge (`{ ...current,
diff --git a/server/routes/videoGen.js b/server/routes/videoGen.js
index 303fbcce06..d72207f37c 100644
--- a/server/routes/videoGen.js
+++ b/server/routes/videoGen.js
@@ -290,7 +290,7 @@ const generateBodySchema = z.object({
// ignore the local-only knobs below; they read prompt/negativePrompt,
// width/height (mapped to an aspect ratio), sourceImageFile/sourceImage,
// and their own duration field.
- backend: z.enum(['local', 'grok', 'fal', 'reactor']).optional(),
+ backend: z.enum(['local', 'grok', 'fal', 'reactor', 'runway']).optional(),
// Grok image_to_video clip length in seconds — the shared schema (see
// lib/grokVideoClip.js for which lengths grok actually delivers). Multipart
// bodies arrive as strings, so coerce first.
@@ -318,6 +318,16 @@ const generateBodySchema = z.object({
(v) => (v === 'true' ? true : v === 'false' || v === '' ? undefined : v),
z.boolean().optional(),
),
+ // Runway model id and clip length (lib/runwayVideoModels.js). Loosely
+ // validated like falModelId: the provider refuses an uncatalogued id and
+ // rounds a length up to one the model delivers.
+ runwayModelId: z.string().min(1).max(64).optional(),
+ runwayDuration: z.preprocess(
+ (v) => (v == null || v === '' ? undefined : Number(v)),
+ optionalNum(1, 60, 'runwayDuration'),
+ ),
+ // Output size ('720p', '1080p'); resolved per model, falling back to its default.
+ runwayResolution: z.preprocess((v) => (v === '' ? undefined : v), z.string().min(1).max(16).optional()),
// reactor.inc fast-h3 (#6214): the clip id to chain from
// (continue_from_clip_id — frame-accurate continuation, unlike fal/grok
// which start a fresh render each time) and clip length in seconds. The
@@ -591,6 +601,7 @@ router.get('/status', asyncHandler(async (_req, res) => {
// settings object alone missed that case entirely.
falEnabled: isVideoModeUsable(s, VIDEO_GEN_MODE.FAL),
reactorEnabled: isVideoModeUsable(s, VIDEO_GEN_MODE.REACTOR),
+ runwayEnabled: isVideoModeUsable(s, VIDEO_GEN_MODE.RUNWAY),
});
}));
diff --git a/server/services/mediaJobQueue/index.js b/server/services/mediaJobQueue/index.js
index 625999dc8b..00aeeacc8e 100644
--- a/server/services/mediaJobQueue/index.js
+++ b/server/services/mediaJobQueue/index.js
@@ -200,6 +200,7 @@ const MEDIA_JOB_RUNTIME = Object.freeze({
[IMAGE_GEN_MODE.GROK]: () => import('../videoGen/grok.js'),
[VIDEO_GEN_MODE.FAL]: () => import('../videoGen/fal.js'),
[VIDEO_GEN_MODE.REACTOR]: () => import('../videoGen/reactor.js'),
+ [VIDEO_GEN_MODE.RUNWAY]: () => import('../videoGen/runway.js'),
default: () => import('../videoGen/local.js'),
},
invoke: (mod, request, { chunks }) => (chunks > 1 ? mod.generateChainedVideo(request) : mod.generateVideo(request)),
@@ -2173,7 +2174,7 @@ export async function cancelJob(jobId) {
if (isRemoteMediaJob(runningJob)) await persist();
return { ok: false, code: 'ALREADY_TERMINAL', status: runningJob.status, error: 'Job is already finishing' };
}
- if (runningJob.params?.videoProduction && ['reactor', 'fal', 'grok'].includes(runningJob.params.mode)) {
+ if (runningJob.params?.videoProduction && ['reactor', 'fal', 'grok', 'runway'].includes(runningJob.params.mode)) {
runningJob.params.videoProduction = { ...runningJob.params.videoProduction, submissionUncertain: true };
await persist();
}
diff --git a/server/services/musicVideo/productionDispatch.js b/server/services/musicVideo/productionDispatch.js
index dceb3843f0..f038d64ffe 100644
--- a/server/services/musicVideo/productionDispatch.js
+++ b/server/services/musicVideo/productionDispatch.js
@@ -29,10 +29,10 @@ import { withMusicVideoStyle } from './styleReferences.js';
import { ServerError } from '../../lib/errorHandler.js';
import { resolveGalleryImage } from '../../lib/pathSafety.js';
import { RENDER_TARGET } from '../../lib/renderTargets.js';
-import { falSceneTake, falTakeRequestFields, grokCoverage } from '../../lib/musicVideoShotTiming.js';
+import { falSceneTake, falTakeRequestFields, grokCoverage, runwaySceneTake, runwayTakeRequestFields } from '../../lib/musicVideoShotTiming.js';
import { authoredShotPrompt, musicVideoFrameModelInput } from '../../lib/musicVideoShotPrompt.js';
import { sceneEndFrameRequestFields } from '../../lib/musicVideoEndFrame.js';
-import { conditioningReferences, falRouteVideoSettings } from './productionPool.js';
+import { conditioningReferences, falRouteVideoSettings, runwayRouteVideoSettings } from './productionPool.js';
import { musicVideoFrameGenSize } from '../../lib/musicVideoAspect.js';
const unprompted = (scene, what) => new ServerError(
@@ -121,6 +121,7 @@ async function dispatchClip({ project, scene, route, tag }) {
grok: { grokDuration: spanSec != null ? grokCoverage(spanSec).requestSec : (project.videoSettings?.grokDuration || 10), disableAudio: true },
fal: falClipParams(project, scene, route),
reactor: { disableAudio: true },
+ runway: runwayTakeRequestFields(runwaySceneTake({ scene, videoSettings: runwayRouteVideoSettings(project, route) })),
};
// A local model ends on the end frame only when its runtime anchors the last
// frame (#11258), so the plan needs the model this route actually renders on.
@@ -138,7 +139,7 @@ async function dispatchClip({ project, scene, route, tag }) {
...(byBackend[route.mode] || {}),
// A style-break end frame (#11173), when this fal or local model takes one.
...sceneEndFrameRequestFields(scene, {
- backend: route.mode, falModelId: byBackend.fal?.falModelId, mode: byBackend[route.mode]?.mode || 'image', localRuntime,
+ backend: route.mode, falModelId: byBackend.fal?.falModelId, runwayModelId: byBackend.runway?.runwayModelId, mode: byBackend[route.mode]?.mode || 'image', localRuntime,
}),
musicVideo: { ...tag, compose: true },
...(fixes ? { _clipFixes: fixes } : {}),
diff --git a/server/services/musicVideo/productionPool.js b/server/services/musicVideo/productionPool.js
index ca6227a7b7..e80e3b8a2a 100644
--- a/server/services/musicVideo/productionPool.js
+++ b/server/services/musicVideo/productionPool.js
@@ -30,7 +30,7 @@ import { resolveFalApiKey } from '../falQueue.js';
import { QUEUEABLE_IMAGE_MODES, VIDEO_GEN_MODES } from '../../lib/generationModes.js';
import { musicVideoConditioningReferences } from '../../lib/musicVideoConditioning.js';
import { MUSIC_VIDEO_AUTOMATION_TOOLS } from '../../lib/musicVideoAutomation.js';
-import { falSceneTake, isPerformanceScene, performanceBlockedReason } from '../../lib/musicVideoShotTiming.js';
+import { falSceneTake, isPerformanceScene, performanceBlockedReason, runwaySceneTake } from '../../lib/musicVideoShotTiming.js';
import { maxInputImages, supportsCloudModelOverride } from '../../lib/imageGenCapabilities.js';
import { resolveVideoSupportedModes } from '../../lib/videoModeProfiles.js';
import { isHardwareCompatible } from '../../lib/systemCapabilities.js';
@@ -56,6 +56,12 @@ export const falRouteVideoSettings = (project, route) => ({
...(route?.model ? { falModelId: route.model } : {}),
});
+/** The render settings a Runway route renders a scene with: the project's Runway pins, a route's own model winning. */
+export const runwayRouteVideoSettings = (project, route) => ({
+ ...(project?.videoSettings || {}),
+ ...(route?.model ? { runwayModelId: route.model } : {}),
+});
+
/**
* The price per generation this install can vouch for: 0 for local, the
* catalog estimate of a default-length cutaway take for a fal video route
@@ -76,6 +82,9 @@ function routePriceUsd(route, project = null) {
return falSceneTake({ scene: null, videoSettings: falRouteVideoSettings(project, route) }).costUsd;
}
if (route.kind === 'image' && route.mode === 'fal') return falImageRoutePriceUsd(route.model);
+ if (route.kind === 'video' && route.mode === 'runway') {
+ return runwaySceneTake({ scene: null, videoSettings: runwayRouteVideoSettings(project, route) }).costUsd;
+ }
return null;
}
@@ -86,6 +95,9 @@ function routePriceUsd(route, project = null) {
* length that covers the shot); otherwise the route's flat price.
*/
export function stepPriceUsd({ route, project, scene, stepKind }) {
+ if (stepKind !== 'frame' && route?.kind === 'video' && route.mode === 'runway' && scene) {
+ return runwaySceneTake({ scene, videoSettings: runwayRouteVideoSettings(project, route) }).costUsd;
+ }
if (stepKind !== 'frame' && route?.kind === 'video' && route.mode === 'fal' && scene) {
return falSceneTake({
scene,
@@ -102,6 +114,9 @@ export function stepPriceUsd({ route, project, scene, stepKind }) {
* end-frame upgrade included), a fal image route's own model, else null.
*/
export function stepCostModelId({ route, project, scene, stepKind }) {
+ if (route?.mode === 'runway' && route.kind === 'video') {
+ return scene ? runwaySceneTake({ scene, videoSettings: runwayRouteVideoSettings(project, route) }).modelId : route.model || null;
+ }
if (route?.mode !== 'fal') return null;
if (stepKind !== 'frame' && route.kind === 'video' && scene) {
return falSceneTake({ scene, videoSettings: falRouteVideoSettings(project, route) }).modelId;
diff --git a/server/services/musicVideo/renderPool.js b/server/services/musicVideo/renderPool.js
index fcf04f286a..689b9886ea 100644
--- a/server/services/musicVideo/renderPool.js
+++ b/server/services/musicVideo/renderPool.js
@@ -63,7 +63,7 @@ export async function applyProjectRenderPool(body) {
|| capability.memory.requiredGb > capability.memory.freeGb || !capability.inputAssets?.roles?.includes('sourceImage')
|| !capability.supportedModes?.includes(requestedMode)
|| (suppliedAudio && !capability.sourceAudio)) reasons.push('This Mac: model, memory capacity or supplied-audio support is unavailable');
- else candidates.push({ modelId, peerId: null, load: jobs.filter((job) => !job.params?.remoteMedia && !['grok', 'fal', 'reactor', 'codex', 'agy'].includes(job.params?.mode)).length });
+ else candidates.push({ modelId, peerId: null, load: jobs.filter((job) => !job.params?.remoteMedia && !['grok', 'fal', 'reactor', 'runway', 'codex', 'agy'].includes(job.params?.mode)).length });
}
}
// Probe only explicitly selected and already allowlisted peers. Status alone
diff --git a/server/services/runwayApi.js b/server/services/runwayApi.js
new file mode 100644
index 0000000000..2899dc56b6
--- /dev/null
+++ b/server/services/runwayApi.js
@@ -0,0 +1,164 @@
+/**
+ * Runway developer API — the task transport for the Runway video backend.
+ *
+ * Runway runs every generation as a task: POST the input to
+ * `/v1/image_to_video` (or `/v1/text_to_video`), which answers `{ id }`; poll
+ * `GET /v1/tasks/{id}` until SUCCEEDED / FAILED / CANCELLED, then download
+ * `output[0]` (a signed URL that expires, so it is fetched at once).
+ * `DELETE /v1/tasks/{id}` cancels a task that is still running.
+ *
+ * Same submit/poll/cancel rules as `falQueue.js`: a lost submit response marks
+ * the run unsettled (it may still bill), a transient status failure is retried
+ * on the poll cadence without resubmitting, and every abandonment path sends
+ * the remote cancel once.
+ */
+
+import { maintenance } from '../lib/maintenanceAdmission.js';
+import { ServerError } from '../lib/errorHandler.js';
+import { fetchWithTimeout } from '../lib/fetchWithTimeout.js';
+
+export const RUNWAY_API_BASE = 'https://api.dev.runwayml.com';
+// Every request names the API version it was written against.
+export const RUNWAY_API_VERSION = '2024-11-06';
+
+export const RUNWAY_SUBMIT_TIMEOUT_MS = 60_000;
+export const RUNWAY_POLL_TIMEOUT_MS = 15_000;
+// Runway asks for polling no faster than every 5 seconds.
+export const RUNWAY_POLL_INTERVAL_MS = 5000;
+export const RUNWAY_MAX_STATUS_RETRIES = 2;
+
+/** The Runway API key: the settings-stored key, else the `RUNWAYML_API_SECRET` env var. */
+export function resolveRunwayApiKey(settings) {
+ const fromSettings = (settings?.videoGen?.runway?.apiKey || '').trim();
+ if (fromSettings) return fromSettings;
+ return (process.env.RUNWAYML_API_SECRET || '').trim() || null;
+}
+
+const headers = (apiKey, json = false) => ({
+ Authorization: `Bearer ${apiKey}`,
+ 'X-Runway-Version': RUNWAY_API_VERSION,
+ ...(json ? { 'Content-Type': 'application/json' } : {}),
+});
+
+/** A short, single-line reason from a Runway error body (`error`, `issues`), or ''. */
+function runwayErrorDetail(payload) {
+ if (!payload || typeof payload !== 'object') return '';
+ const issues = Array.isArray(payload.issues)
+ ? payload.issues.map((issue) => [Array.isArray(issue?.path) ? issue.path.join('.') : null, issue?.message].filter(Boolean).join(': ')).filter(Boolean)
+ : [];
+ const reason = [typeof payload.error === 'string' ? payload.error : null, ...issues].filter(Boolean).join(' — ');
+ return reason.replace(/\s+/g, ' ').trim().slice(0, 300);
+}
+
+/** Per-request bookkeeping shared by `cancelRunwayTask` and `awaitRunwayTask`. */
+export const createRunwayTaskEntry = (apiKey) => ({
+ apiKey, controller: new AbortController(), aborted: false, taskId: null, remoteTerminal: false, cancelPromise: null,
+});
+
+/**
+ * Best-effort, idempotent cancellation of the remote task. Never rethrows: the
+ * local job still finalizes as failed/canceled either way. Never fires once
+ * Runway reported a terminal status (DELETE would then delete the finished task).
+ */
+export async function cancelRunwayTask(entry) {
+ if (!entry) return;
+ if (!entry.cancelPromise) {
+ if (entry.remoteTerminal || !entry.taskId || !entry.apiKey) return;
+ entry.cancelPromise = fetchWithTimeout(`${RUNWAY_API_BASE}/v1/tasks/${encodeURIComponent(entry.taskId)}`, {
+ method: 'DELETE', headers: headers(entry.apiKey),
+ }, RUNWAY_POLL_TIMEOUT_MS)
+ .then(async (res) => {
+ if (!res.ok && res.status !== 404) console.error(`❌ Runway cancellation request failed: HTTP ${res.status}`);
+ await res.body?.cancel?.();
+ })
+ .catch((err) => console.error(`❌ Runway cancellation request failed: ${err?.message || err}`));
+ }
+ await entry.cancelPromise;
+ // An accepted cancel does not prove the paid render stopped.
+ if (!entry.remoteTerminal) maintenance.markCurrentUnsettled();
+}
+
+/** Submit one paid generation to `endpoint` ('image_to_video' | 'text_to_video'). Resolves to the task id. */
+export async function submitRunwayTask({ apiKey, endpoint, body }) {
+ const res = await fetchWithTimeout(`${RUNWAY_API_BASE}/v1/${endpoint}`, {
+ method: 'POST', headers: headers(apiKey, true), body: JSON.stringify(body),
+ }, RUNWAY_SUBMIT_TIMEOUT_MS).catch((err) => {
+ // A lost POST response is not proof the paid task was rejected.
+ maintenance.markCurrentUnsettled();
+ throw err;
+ });
+ const payload = await res.json().catch(() => null);
+ if (!res.ok || typeof payload?.id !== 'string') {
+ if (res.ok || res.status >= 500) maintenance.markCurrentUnsettled();
+ const detail = runwayErrorDetail(payload);
+ throw new ServerError(`Runway rejected the request: HTTP ${res.status}${detail ? ` — ${detail}` : ''}`, { status: 502, code: 'RUNWAY_SUBMIT_FAILED' });
+ }
+ return payload.id;
+}
+
+async function getRunwayTask({ apiKey, taskId }) {
+ const res = await fetchWithTimeout(`${RUNWAY_API_BASE}/v1/tasks/${encodeURIComponent(taskId)}`, {
+ headers: headers(apiKey),
+ }, RUNWAY_POLL_TIMEOUT_MS);
+ if (!res.ok) throw new ServerError(`Runway status check failed: HTTP ${res.status}`, { status: 502, code: 'RUNWAY_STATUS_FAILED' });
+ return res.json();
+}
+
+/**
+ * Poll a submitted task until Runway reports it terminal, the caller cancels it
+ * (`entry.aborted`), or `deadline` passes. Resolves to `{ outcome: 'completed',
+ * outputUrl }` or `{ outcome: 'canceled' | 'failed', reason? }` and never
+ * throws; every abandonment path has already sent the remote cancel.
+ * `onStatus(status, progress)` reports each non-terminal status (PENDING,
+ * THROTTLED, RUNNING) for the caller's progress wording.
+ */
+export async function awaitRunwayTask({
+ entry, apiKey, deadline, timeoutMs, onPoll = () => {}, onStatus = () => {}, pollIntervalMs = RUNWAY_POLL_INTERVAL_MS,
+}) {
+ let statusFailures = 0;
+ // A cancel (entry.controller.abort()) ends the wait at once rather than after the next interval.
+ const wait = () => new Promise((resolve) => {
+ const signal = entry.controller.signal;
+ if (signal.aborted) return resolve();
+ const done = () => { clearTimeout(timer); signal.removeEventListener('abort', done); resolve(); };
+ const timer = setTimeout(done, pollIntervalMs);
+ signal.addEventListener('abort', done, { once: true });
+ });
+ while (Date.now() < deadline) {
+ if (entry.aborted) {
+ await cancelRunwayTask(entry);
+ return { outcome: 'canceled' };
+ }
+ onPoll();
+ let task;
+ try {
+ task = await getRunwayTask({ apiKey, taskId: entry.taskId });
+ } catch (err) {
+ statusFailures += 1;
+ if (statusFailures > RUNWAY_MAX_STATUS_RETRIES) {
+ await cancelRunwayTask(entry);
+ return { outcome: 'failed', reason: `Runway status checks failed ${statusFailures} times in a row: ${err?.message || err}` };
+ }
+ await wait();
+ continue;
+ }
+ statusFailures = 0;
+ if (task?.status === 'SUCCEEDED') {
+ entry.remoteTerminal = true;
+ if (entry.aborted) return { outcome: 'canceled' };
+ const outputUrl = Array.isArray(task.output) ? task.output.find((url) => typeof url === 'string' && url.trim()) : null;
+ return outputUrl ? { outcome: 'completed', outputUrl } : { outcome: 'failed', reason: 'Runway finished but returned no video URL' };
+ }
+ if (task?.status === 'FAILED' || task?.status === 'CANCELLED' || task?.status === 'CANCELED') {
+ entry.remoteTerminal = true;
+ if (entry.aborted) return { outcome: 'canceled' };
+ const code = typeof task.failureCode === 'string' ? ` (${task.failureCode})` : '';
+ return { outcome: 'failed', reason: `Runway render failed: ${task.failure || task.status.toLowerCase()}${code}` };
+ }
+ onStatus(task?.status, Number.isFinite(task?.progress) ? task.progress : null);
+ await wait();
+ }
+ await cancelRunwayTask(entry);
+ if (entry.aborted) return { outcome: 'canceled' };
+ return { outcome: 'failed', reason: `Runway did not finish within ${Math.round(timeoutMs / 1000)}s` };
+}
diff --git a/server/services/videoGen/backendPin.js b/server/services/videoGen/backendPin.js
index 6ecab442f7..553dabd650 100644
--- a/server/services/videoGen/backendPin.js
+++ b/server/services/videoGen/backendPin.js
@@ -30,6 +30,7 @@
import { RENDER_TARGET, normalizeRenderPinValue } from '../../lib/renderTargets.js';
import { nearestGrokDuration } from '../../lib/grokVideoClip.js';
import { nearestReactorAspect } from '../../lib/reactorVideoClip.js';
+import { getRunwayVideoModel } from '../../lib/runwayVideoModels.js';
import { ServerError } from '../../lib/errorHandler.js';
import { renderTargetDefaults } from '../imageGen/cloudProviderConfig.js';
import { VIDEO_GEN_MODE, VIDEO_GEN_MODES, resolveVideoMode, hasVideoPin, isVideoModeUsable } from './modes.js';
@@ -67,15 +68,19 @@ export function resolveVideoBackendPin(project, settings, { target = RENDER_TARG
}
const resolvedMode = project?.workspace === 'video' ? strictMode : resolveVideoMode(mode, settings, { target });
// The target videoModel and project.modelId are local catalog choices. fal
- // accepts an endpoint model only from its own explicit project pin; Reactor
- // and Grok choose their models in the renderer. A lapsed fal pin must not
- // pass its endpoint id into the legacy local fallback either.
- const cloudModelPin = mode === VIDEO_GEN_MODE.FAL || mode === VIDEO_GEN_MODE.REACTOR;
+ // and Runway accept a model only from their own explicit project pin (Runway
+ // only a curated catalog id); Reactor and Grok choose their models in the
+ // renderer. A lapsed fal or Runway pin must not pass its model id into the
+ // legacy local fallback either.
+ const cloudModelPin = mode === VIDEO_GEN_MODE.FAL || mode === VIDEO_GEN_MODE.REACTOR || mode === VIDEO_GEN_MODE.RUNWAY;
const localModelId = (!cloudModelPin && normalizeRenderPinValue(raw?.modelId))
|| targetDefaults.videoModel || null;
const modelId = resolvedMode === VIDEO_GEN_MODE.LOCAL ? localModelId
: resolvedMode === VIDEO_GEN_MODE.FAL && mode === VIDEO_GEN_MODE.FAL
- ? normalizeRenderPinValue(raw?.modelId) : null;
+ ? normalizeRenderPinValue(raw?.modelId)
+ : resolvedMode === VIDEO_GEN_MODE.RUNWAY && mode === VIDEO_GEN_MODE.RUNWAY
+ ? getRunwayVideoModel(normalizeRenderPinValue(raw?.modelId))?.id || null
+ : null;
return {
// A model pin counts as a pin even with no mode beside it. Naming a local
// video model IS a configured choice, and the resolved mode for an
@@ -192,6 +197,15 @@ export function videoBackendJobParams(settings, pin, params = {}, { durationSeco
...(duration != null ? { duration } : {}),
};
}
+ if (pin.mode === VIDEO_GEN_MODE.RUNWAY) {
+ return {
+ ...base,
+ mode: VIDEO_GEN_MODE.RUNWAY,
+ videoMode: sourceImagePath ? 'image' : 'text',
+ ...(pin.modelId ? { modelId: pin.modelId } : {}),
+ ...(duration != null ? { duration } : {}),
+ };
+ }
if (pin.mode === VIDEO_GEN_MODE.REACTOR) {
const seconds = params.seconds ?? duration;
const aspect = Number(params.width) > 0 && Number(params.height) > 0
diff --git a/server/services/videoGen/backendPin.test.js b/server/services/videoGen/backendPin.test.js
index 9a673d694c..dce27ac6b2 100644
--- a/server/services/videoGen/backendPin.test.js
+++ b/server/services/videoGen/backendPin.test.js
@@ -1,5 +1,5 @@
import { describe, it, expect } from 'vitest';
-import { resolveVideoBackendPin, grokVideoJobParams } from './backendPin.js';
+import { resolveVideoBackendPin, grokVideoJobParams, videoBackendJobParams } from './backendPin.js';
import { VIDEO_GEN_MODE } from './modes.js';
import { RENDER_TARGET } from '../../lib/renderTargets.js';
@@ -106,6 +106,24 @@ describe('resolveVideoBackendPin', () => {
pinned: false, requested: null, mode: VIDEO_GEN_MODE.LOCAL, modelId: null,
});
});
+
+ // A Runway model pin renders on the pinned catalog model (its price and
+ // end-frame support differ from the Gen-4.5 default), and never leaks into
+ // the local fallback when the Runway key goes away.
+ it('carries a curated Runway model pin into the Runway job and drops it from a local fallback', () => {
+ const runwayOn = { videoGen: { runway: { apiKey: 'key_test' } } };
+ const pin = resolveVideoBackendPin(project({ mode: 'runway', modelId: 'kling2.5_turbo_pro' }), runwayOn);
+ expect(pin).toMatchObject({ mode: VIDEO_GEN_MODE.RUNWAY, modelId: 'kling2.5_turbo_pro' });
+ expect(videoBackendJobParams(runwayOn, pin, { sourceImagePath: '/tmp/frame.png', duration: 5 }))
+ .toMatchObject({ mode: VIDEO_GEN_MODE.RUNWAY, modelId: 'kling2.5_turbo_pro', videoMode: 'image', duration: 5 });
+ expect(resolveVideoBackendPin(project({ mode: 'runway', modelId: 'not-a-runway-model' }), runwayOn).modelId).toBeNull();
+
+ const prev = process.env.RUNWAYML_API_SECRET;
+ delete process.env.RUNWAYML_API_SECRET;
+ const lapsed = resolveVideoBackendPin(project({ mode: 'runway', modelId: 'kling2.5_turbo_pro' }), {});
+ if (prev !== undefined) process.env.RUNWAYML_API_SECRET = prev;
+ expect(lapsed).toMatchObject({ requested: 'runway', mode: VIDEO_GEN_MODE.LOCAL, modelId: null });
+ });
});
describe('grokVideoJobParams', () => {
diff --git a/server/services/videoGen/hostedSubmission.js b/server/services/videoGen/hostedSubmission.js
index 2595c8814c..54b4325308 100644
--- a/server/services/videoGen/hostedSubmission.js
+++ b/server/services/videoGen/hostedSubmission.js
@@ -7,6 +7,7 @@ import { ServerError } from '../../lib/errorHandler.js';
import { resolveGalleryImage } from '../../lib/fileUtils.js';
import { probeVideoDuration } from '../../lib/ffmpeg.js';
import { FAL_DEFAULT_IMAGE_VIDEO_MODEL, FAL_DEFAULT_TEXT_VIDEO_MODEL, getFalVideoModel } from '../../lib/falVideoModels.js';
+import { RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL, getRunwayVideoModel } from '../../lib/runwayVideoModels.js';
import { isDefaultI2vReferenceMode } from '../../lib/videoReferenceModes.js';
import { VIDEO_GEN_MODE, isVideoModeUsable } from './modes.js';
@@ -48,6 +49,33 @@ async function stageFalEndFrame({ body, uploads, sourceImagePath, stageUploadDur
return { path: resolved, owned: false };
}
+// Stage an end frame durably (an upload) or resolve a gallery pick.
+async function stageEndFrame({ body, uploads, stageUploadDurable }) {
+ if (uploads.lastImage) return { path: await stageUploadDurable(uploads.lastImage, 'last'), owned: true };
+ const resolved = resolveGalleryImage(body.lastImageFile);
+ if (!resolved) {
+ throw new ServerError('The end frame is missing or could not be resolved.', { status: 400, code: 'VALIDATION_ERROR' });
+ }
+ return { path: resolved, owned: false };
+}
+
+// Runway renders from a start frame only, with an end frame on the models that take one.
+async function stageRunwayInputs(ctx) {
+ const { body, uploads, sourceImagePath, cleanupStaged } = ctx;
+ const refuse = async (message) => {
+ await cleanupStaged();
+ throw new ServerError(message, { status: 400, code: 'VALIDATION_ERROR' });
+ };
+ if (uploads.audioFile) await refuse('Runway renders cutaways only; it takes no source audio.');
+ if (!sourceImagePath) await refuse('Runway renders from a start frame — supply a source image.');
+ const model = getRunwayVideoModel(body.runwayModelId || RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL);
+ if (!model) await refuse(`"${body.runwayModelId}" is not a Runway model PortOS offers`);
+ if (!uploads.lastImage && !body.lastImageFile) return {};
+ if (!model.endImage) await refuse(`${model.label} takes no end frame`);
+ const endFrame = await stageEndFrame(ctx);
+ return { lastImagePath: endFrame.path, ...(endFrame.owned ? { uploadedTempPaths: [endFrame.path] } : {}) };
+}
+
// A standalone fal source-audio take: stage the audio clip durably like the frame. The queue removes
// it if the job fails, is cancelled or is lost to a restart (uploadedTempPaths); the fal worker
// removes it once the take is published.
@@ -130,6 +158,27 @@ export const HOSTED_VIDEO_SUBMISSIONS = {
...(body.falGenerateAudio === true ? { generateAudio: true } : {}),
}),
},
+ [VIDEO_GEN_MODE.RUNWAY]: {
+ prepare: (settings) => ({
+ usable: isVideoModeUsable(settings, VIDEO_GEN_MODE.RUNWAY),
+ extras: {},
+ }),
+ stageInputs: stageRunwayInputs,
+ validateRetry: (params) => assertAnchoredReference(VIDEO_GEN_MODE.RUNWAY, params),
+ errorCode: 'RUNWAY_NOT_CONFIGURED',
+ errorMessage: 'No Runway API key configured — set it in Settings → Video Gen (or the RUNWAYML_API_SECRET env var) first',
+ // The model, length and size are resolved against the curated catalog by videoGen/runway.js.
+ buildParams: (body, prepared) => ({
+ modelId: body.runwayModelId,
+ ...(prepared.lastImagePath ? { lastImagePath: prepared.lastImagePath } : {}),
+ ...(prepared.uploadedTempPaths?.length ? { uploadedTempPaths: prepared.uploadedTempPaths } : {}),
+ aspectRatio: body.visualConditioning?.render?.parameters?.aspectRatio,
+ width: body.width,
+ height: body.height,
+ duration: body.runwayDuration,
+ ...(body.runwayResolution ? { resolution: body.runwayResolution } : {}),
+ }),
+ },
[VIDEO_GEN_MODE.REACTOR]: {
prepare: (settings) => ({
usable: isVideoModeUsable(settings, VIDEO_GEN_MODE.REACTOR),
diff --git a/server/services/videoGen/modes.js b/server/services/videoGen/modes.js
index 2a837e64c1..ddccf9fff4 100644
--- a/server/services/videoGen/modes.js
+++ b/server/services/videoGen/modes.js
@@ -52,6 +52,10 @@ export function isVideoModeUsable(settings, mode) {
if (mode === VIDEO_GEN_MODE.REACTOR) {
return Boolean((settings?.videoGen?.reactor?.apiKey || '').trim() || (process.env.REACTOR_API_KEY || '').trim());
}
+ // Runway's developer API — gated on a key in settings or RUNWAYML_API_SECRET.
+ if (mode === VIDEO_GEN_MODE.RUNWAY) {
+ return Boolean((settings?.videoGen?.runway?.apiKey || '').trim() || (process.env.RUNWAYML_API_SECRET || '').trim());
+ }
return mode === VIDEO_GEN_MODE.LOCAL;
}
diff --git a/server/services/videoGen/modes.test.js b/server/services/videoGen/modes.test.js
index 73e7125cca..89af0259ae 100644
--- a/server/services/videoGen/modes.test.js
+++ b/server/services/videoGen/modes.test.js
@@ -11,8 +11,8 @@ describe('VIDEO_GEN_MODE', () => {
// rather than re-typing is what keeps a schema from drifting from dispatch.
expect(VIDEO_GEN_MODE.LOCAL).toBe(IMAGE_GEN_MODE.LOCAL);
expect(VIDEO_GEN_MODE.GROK).toBe(IMAGE_GEN_MODE.GROK);
- expect(VIDEO_GEN_MODES).toEqual(['local', 'grok', 'fal', 'reactor']);
- expect(CLOUD_VIDEO_GEN_MODES).toEqual(['grok', 'fal', 'reactor']);
+ expect(VIDEO_GEN_MODES).toEqual(['local', 'grok', 'fal', 'reactor', 'runway']);
+ expect(CLOUD_VIDEO_GEN_MODES).toEqual(['grok', 'fal', 'reactor', 'runway']);
expect(Object.isFrozen(VIDEO_GEN_MODE)).toBe(true);
});
});
diff --git a/server/services/videoGen/prepareParams.js b/server/services/videoGen/prepareParams.js
index 2ce844ff17..2610ef34e7 100644
--- a/server/services/videoGen/prepareParams.js
+++ b/server/services/videoGen/prepareParams.js
@@ -55,7 +55,7 @@ import { musicVideoLocalClipSize } from '../../lib/musicVideoAspect.js';
import { localModelRendersCutaway, localModelRendersPerformance } from '../../lib/musicVideoLocalModels.js';
import { isPerformanceScene } from '../../lib/musicVideoShotTiming.js';
import { getTrack } from '../tracks/index.js';
-import { VIDEO_GEN_MODE, resolveVideoMode } from './modes.js';
+import { CLOUD_VIDEO_GEN_MODES, VIDEO_GEN_MODE, resolveVideoMode } from './modes.js';
import { getFalVideoModel } from '../../lib/falVideoModels.js';
import { HOSTED_VIDEO_SUBMISSIONS } from './hostedSubmission.js';
import { isDefaultI2vReferenceMode } from '../../lib/videoReferenceModes.js';
@@ -401,7 +401,7 @@ export async function prepareVideoGenParams({ body, uploads, localOnlyParamKeys
// in sync.
const runtimeBringsOwnVenv = effectiveModel && BYOV_VIDEO_RUNTIMES.has(effectiveModel.runtime);
if (!pythonPath && !runtimeBringsOwnVenv
- && backend !== VIDEO_GEN_MODE.GROK && backend !== VIDEO_GEN_MODE.FAL && backend !== VIDEO_GEN_MODE.REACTOR) {
+ && !CLOUD_VIDEO_GEN_MODES.includes(backend)) {
await cleanupMultipartTemp(uploads);
throw new ServerError(
'Local video generation is not configured (settings.imageGen.local.pythonPath is missing).',
@@ -693,7 +693,7 @@ async function resolvePreparedParams({
// plain grok render with the reference clip silently dropped. The client's
// mode bar snaps grok back to text/image, but reject explicitly so a direct
// caller gets an error instead of a wrong-looking clip.
- if (body.backend === VIDEO_GEN_MODE.GROK || body.backend === VIDEO_GEN_MODE.FAL || body.backend === VIDEO_GEN_MODE.REACTOR) {
+ if (CLOUD_VIDEO_GEN_MODES.includes(body.backend)) {
await cleanupStaged();
throw new ServerError(
`${icSpec.label} mode runs on the local ltx2 runtime — it isn't available on the ${body.backend} backend.`,
diff --git a/server/services/videoGen/promptPreview.js b/server/services/videoGen/promptPreview.js
index 321c9e02f6..9689b21828 100644
--- a/server/services/videoGen/promptPreview.js
+++ b/server/services/videoGen/promptPreview.js
@@ -66,6 +66,14 @@ export async function previewVideoPrompt({ previewUploads = [], ...body }) {
hasImage: hasSourceImage, hasEndImage: !!uploads.lastImage || !!body.lastImageFile, hasAudio: !!uploads.audioFile,
}).record };
}
+ if (backend === VIDEO_GEN_MODE.RUNWAY) {
+ const { runwayVideoSentPrompt } = await import('./runway.js');
+ return { backend, ...runwayVideoSentPrompt({
+ modelId: body.runwayModelId, prompt: body.prompt, negativePrompt: body.negativePrompt,
+ seconds: body.runwayDuration, width: body.width, height: body.height, resolution: body.runwayResolution, promptAdditions,
+ hasImage: hasSourceImage, hasEndImage: !!uploads.lastImage || !!body.lastImageFile,
+ }).record };
+ }
if (backend === VIDEO_GEN_MODE.LOCAL) {
// The local runner's modules load only for a local preview.
const [{ defaultVideoModelId, listVideoModels }, { resolveVideoLoras }, { captureSystemCapabilities }] = await Promise.all([
diff --git a/server/services/videoGen/runway.js b/server/services/videoGen/runway.js
new file mode 100644
index 0000000000..b8ed7e3f13
--- /dev/null
+++ b/server/services/videoGen/runway.js
@@ -0,0 +1,306 @@
+/**
+ * Video Gen — Runway developer API provider.
+ *
+ * A cloud lane like `videoGen/fal.js`: the same job-map/SSE contract, so
+ * `mediaJobQueue` dispatches to it on `params.mode === 'runway'`. Flow: POST
+ * the start frame (and an optional last frame) as data URIs to
+ * `/v1/image_to_video`, poll the task until it SUCCEEDS, download `output[0]`
+ * and hand it to the shared `finalizeGeneratedVideo` helper.
+ *
+ * The request body is built per model from the curated catalog
+ * (lib/runwayVideoModels.js). Unlike fal, a model id outside the catalog is
+ * refused: Runway validates every field per model, so there is no legacy body
+ * that would render on an unknown model.
+ */
+
+import { createMediaJobPreparation } from '../../lib/mediaJobPreparation.js';
+import { randomUUID } from 'crypto';
+import { writeFile, unlink } from 'fs/promises';
+import { join, relative, resolve, isAbsolute } from 'path';
+import sharp from 'sharp';
+import { ensureDir, PATHS } from '../../lib/fileUtils.js';
+import { ServerError } from '../../lib/errorHandler.js';
+import { fetchWithTimeout } from '../../lib/fetchWithTimeout.js';
+import { anyAbortSignal } from '../../lib/requestAbort.js';
+import { withAbortTimeout } from '../../lib/abortTimeout.js';
+import { RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL, buildRunwayVideoRequest } from '../../lib/runwayVideoModels.js';
+import { probeVideoGeometry } from '../../lib/ffmpeg.js';
+import { attachSseClient as attachSse, closeJobAfterDelay, createJobFailureFinalizer } from '../../lib/sseUtils.js';
+import { videoGenEvents } from './events.js';
+import { discardUnpublishedVideo, finalizeGeneratedVideo, emitCloudRenderStatus, CLOUD_RENDER_PHASE } from './generateVideoHelpers.js';
+import { maintenance } from '../../lib/maintenanceAdmission.js';
+import { mutateVideoHistory } from './history.js';
+import { getSettings } from '../settings.js';
+import { frameDataUriAtAspect } from './falFrameAspect.js';
+import {
+ awaitRunwayTask, cancelRunwayTask, createRunwayTaskEntry, resolveRunwayApiKey, submitRunwayTask,
+} from '../runwayApi.js';
+import { carriedPromptAdditions, promptAddition, sentPromptRecord } from '../../lib/modelInput.js';
+
+export { resolveRunwayApiKey };
+
+const RUNWAY_DOWNLOAD_TIMEOUT_MS = 10 * 60 * 1000;
+// Runway queues a task behind the account's concurrency limit (THROTTLED), so
+// the render budget is generous, the same order as fal's.
+const RUNWAY_RENDER_TIMEOUT_MS = (() => {
+ const n = Number(process.env.RUNWAY_VIDEO_TIMEOUT_MS);
+ return Number.isFinite(n) && n > 0 ? n : 20 * 60 * 1000;
+})();
+// Runway takes an inline image of at most 5 MB as an encoded data URI.
+const RUNWAY_DATA_URI_MAX_BYTES = 5 * 1024 * 1024;
+const RUNWAY_FRAME_MAX_SIDE = 2048;
+
+const jobs = new Map();
+const activeRequests = new Map();
+const activeJobs = new Map();
+
+export const getActiveJob = () => {
+ const entries = [...activeJobs.values()];
+ return entries.length ? entries[entries.length - 1] : null;
+};
+
+export const attachSseClient = (jobId, res) => attachSse(jobs, jobId, res);
+
+const preparations = createMediaJobPreparation();
+
+export const cancel = (jobId) => {
+ if (!jobId) {
+ throw new Error('videoGen/runway.cancel requires a jobId — use cancelAll() to terminate every in-flight render');
+ }
+ const pending = preparations.cancel(jobId);
+ if (pending) return pending;
+ const entry = activeRequests.get(jobId);
+ if (!entry) return false;
+ entry.aborted = true;
+ entry.controller.abort();
+ cancelRunwayTask(entry);
+ return true;
+};
+
+export const cancelAll = () => {
+ const ids = [...preparations.ids(), ...activeRequests.keys()];
+ if (ids.length === 0) return false;
+ for (const id of ids) cancel(id);
+ return true;
+};
+
+/**
+ * Dry-run the request body with placeholder frames before anything is read or
+ * paid for: a missing input or an over-cap prompt refuses here as a 400. Returns
+ * the plan and the record of the text Runway receives (#11317); the Video Gen
+ * form's preview calls it with the same inputs.
+ */
+export function runwayVideoSentPrompt({ hasImage = false, hasEndImage = false, promptAdditions = [], ...spec }) {
+ let planned;
+ try {
+ planned = buildRunwayVideoRequest({
+ ...spec,
+ imageUri: hasImage ? 'pending:image' : null,
+ endImageUri: hasEndImage ? 'pending:end-image' : null,
+ });
+ } catch (err) {
+ if (err?.code !== 'RUNWAY_MODEL_INPUT') throw err;
+ throw new ServerError(err.message, { status: 400, code: 'VALIDATION_ERROR' });
+ }
+ const record = sentPromptRecord({
+ sentPrompt: planned.body.promptText ?? '',
+ sentNegativePrompt: planned.body.negativePrompt ?? null,
+ carried: carriedPromptAdditions({ promptAdditions }),
+ added: [planned.model.negativePrompt ? null : promptAddition('avoid', 'Negative prompt, sent as "Avoid:"', spec.negativePrompt)],
+ });
+ return { planned, record };
+}
+
+/**
+ * A frame as a data URI Runway accepts: cropped to the project's aspect when
+ * one is named (see falFrameAspect.js), then, only if the encoded URI would
+ * pass Runway's 5 MB inline limit, re-encoded as a high-quality JPEG no larger
+ * than 2048px on its long side. Returns `{ dataUri, conformed, reencoded }`.
+ */
+async function runwayFrameDataUri(filePath, aspect) {
+ const framed = await frameDataUriAtAspect(filePath, aspect);
+ if (Buffer.byteLength(framed.dataUri) <= RUNWAY_DATA_URI_MAX_BYTES) return { ...framed, reencoded: null };
+ const source = Buffer.from(framed.dataUri.slice(framed.dataUri.indexOf(',') + 1), 'base64');
+ const buffer = await sharp(source)
+ .resize(RUNWAY_FRAME_MAX_SIDE, RUNWAY_FRAME_MAX_SIDE, { fit: 'inside', withoutEnlargement: true })
+ .jpeg({ quality: 92 })
+ .toBuffer();
+ const dataUri = `data:image/jpeg;base64,${buffer.toString('base64')}`;
+ if (Buffer.byteLength(dataUri) > RUNWAY_DATA_URI_MAX_BYTES) {
+ throw new Error('The frame is still over Runway\'s 5 MB inline limit after re-encoding');
+ }
+ return { ...framed, dataUri, reencoded: 'jpeg' };
+}
+
+export const generateVideo = (params) => preparations.run(params, generateVideoPrepared);
+
+async function generateVideoPrepared({
+ apiKey: providedApiKey, settings, modelId: requestedModelId,
+ prompt = '', negativePrompt, duration, aspectRatio, width, height,
+ sourceImagePath = null, lastImagePath = null, jobId: providedJobId = null,
+ uploadedTempPath = null, uploadedTempPaths = [], audioFilePath = null,
+ resolution = null, seed, frameAspect = null, shotInstruction = null, promptAdditions = [],
+}) {
+ await ensureDir(PATHS.videos);
+ const renderStartedAtMs = Date.now();
+ if (audioFilePath) {
+ throw new ServerError('Runway renders cutaways only; it takes no source audio', { status: 400, code: 'VALIDATION_ERROR' });
+ }
+ const modelId = requestedModelId || RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL;
+ const effectiveSettings = settings || (providedApiKey ? null : await getSettings().catch(() => null));
+ // Re-resolved from live settings, never threaded through persisted job params.
+ const apiKey = providedApiKey || resolveRunwayApiKey(effectiveSettings);
+ if (!apiKey) {
+ throw new ServerError('No Runway API key configured — set it in Settings → Video Gen (or the RUNWAYML_API_SECRET env var)', { status: 400, code: 'RUNWAY_NOT_CONFIGURED' });
+ }
+ const requestSpec = {
+ modelId, prompt, negativePrompt, seconds: duration, aspectRatio: frameAspect || aspectRatio, width, height, resolution,
+ ...(Number.isInteger(seed) ? { seed } : {}),
+ };
+ const { planned, record: sentRecord } = runwayVideoSentPrompt({
+ ...requestSpec, promptAdditions, hasImage: !!sourceImagePath, hasEndImage: !!lastImagePath,
+ });
+
+ const jobId = providedJobId || randomUUID();
+ const filename = `${jobId}.mp4`;
+ const outputPath = join(PATHS.videos, filename);
+ const meta = {
+ id: jobId,
+ prompt: (prompt || '').trim(),
+ negativePrompt: negativePrompt || '',
+ modelId: `runway:${planned.model.id}`,
+ duration: planned.seconds,
+ ...(planned.body.ratio ? { aspectRatio: planned.body.ratio } : {}),
+ ...(planned.resolution ? { resolution: planned.resolution } : {}),
+ // What PortOS expected this render to cost at Runway's list price.
+ estimatedCostUsd: planned.estimatedCostUsd,
+ ...sentRecord,
+ filename,
+ createdAt: new Date().toISOString(),
+ mode: 'image',
+ ...(shotInstruction?.audioWindow ? { audioWindow: shotInstruction.audioWindow } : {}),
+ };
+ const job = { ...meta, clients: [], status: 'running', renderStartedAtMs };
+ jobs.set(jobId, job);
+
+ const costNote = meta.estimatedCostUsd != null ? ` ~$${meta.estimatedCostUsd.toFixed(2)}` : ' cost unknown';
+ console.log(`🎬 Generating video [${jobId.slice(0, 8)}] runway (${planned.model.id}${meta.resolution ? ` ${meta.resolution}` : ''}${costNote}): ${(prompt || '(no prompt)').slice(0, 60)}…`);
+ videoGenEvents.emit('started', { generationId: jobId, totalSteps: 1, ...meta });
+ activeJobs.set(jobId, { ...meta, generationId: jobId, totalSteps: 1, step: 0, progress: 0 });
+ emitCloudRenderStatus(job, jobId, CLOUD_RENDER_PHASE.SUBMIT, 'Submitting to Runway…');
+
+ runRunwayVideo(job, jobId, {
+ apiKey, requestSpec, sourceImagePath, lastImagePath, outputPath, filename, meta,
+ uploadedTempPath, uploadedTempPaths, frameAspect,
+ }).catch((err) => {
+ console.error(`❌ Runway video run failed [${jobId.slice(0, 8)}]: ${err?.message}`);
+ });
+
+ return {
+ jobId, filename, path: `/data/videos/${filename}`, generationId: jobId,
+ mode: 'runway',
+ status: 'running',
+ };
+}
+
+const isStagedUpload = (path) => {
+ const rel = relative(PATHS.uploads, resolve(path));
+ return !!rel && !rel.startsWith('..') && !isAbsolute(rel);
+};
+
+async function downloadRunwayOutput(url, signal) {
+ const res = await fetchWithTimeout(url, { signal }, RUNWAY_DOWNLOAD_TIMEOUT_MS);
+ if (!res.ok) {
+ await res.body?.cancel?.().catch(() => {});
+ throw new Error(`Runway video download failed: HTTP ${res.status}`);
+ }
+ return Buffer.from(await res.arrayBuffer());
+}
+
+async function runRunwayVideo(job, jobId, {
+ apiKey, requestSpec, sourceImagePath, lastImagePath, outputPath, filename, meta,
+ uploadedTempPath = null, uploadedTempPaths = [], frameAspect = null,
+}) {
+ const entry = createRunwayTaskEntry(apiKey);
+ entry.aborted = preparations.take(jobId);
+ const publication = {};
+ let inputCleanup;
+ const settleInputs = () => (inputCleanup ??= (async () => {
+ const inputs = new Set([uploadedTempPath, ...(Array.isArray(uploadedTempPaths) ? uploadedTempPaths : [])]);
+ for (const path of inputs) {
+ if (!path || !isStagedUpload(path)) continue;
+ await unlink(path).catch((error) => {
+ if (error.code === 'ENOENT') return;
+ maintenance.markCurrentUnsettled();
+ maintenance.markResourceUnsettled('media', jobId);
+ console.error(`❌ Runway staged-input cleanup failed [${jobId.slice(0, 8)}]: ${error.message}`);
+ });
+ }
+ })());
+ const finishFailure = async (reason) => {
+ await settleInputs();
+ if (entry.aborted) return finalizeCanceled(job, jobId);
+ return finalizeJobFailure(job, jobId, null, reason, { force: true });
+ };
+ activeRequests.set(jobId, entry);
+ const deadline = Date.now() + RUNWAY_RENDER_TIMEOUT_MS;
+ try {
+ const frame = await runwayFrameDataUri(sourceImagePath, frameAspect);
+ const endFrame = lastImagePath ? await runwayFrameDataUri(lastImagePath, frameAspect) : null;
+ for (const [label, fit] of [['start', frame], ['end', endFrame]]) {
+ if (fit?.conformed) console.log(`✂️ Cropped Runway ${label} frame [${jobId.slice(0, 8)}] ${fit.conformed.from} → ${fit.conformed.to} for ${frameAspect}`);
+ if (fit?.reencoded) console.log(`🗜️ Re-encoded Runway ${label} frame [${jobId.slice(0, 8)}] as JPEG to fit the 5 MB inline limit`);
+ }
+ if (frame.conformed || endFrame?.conformed) {
+ meta.frameConformed = { aspect: frameAspect, ...(frame.conformed ? { start: frame.conformed } : {}), ...(endFrame?.conformed ? { end: endFrame.conformed } : {}) };
+ }
+ const { body } = buildRunwayVideoRequest({ ...requestSpec, imageUri: frame.dataUri, endImageUri: endFrame?.dataUri ?? null });
+ if (entry.aborted) return finishFailure('Canceled');
+ entry.taskId = await submitRunwayTask({ apiKey, endpoint: 'image_to_video', body });
+
+ const polled = await awaitRunwayTask({
+ entry, apiKey, deadline, timeoutMs: RUNWAY_RENDER_TIMEOUT_MS,
+ onPoll: () => videoGenEvents.emit('activity', { generationId: jobId }),
+ onStatus: (status) => (status === 'RUNNING'
+ ? emitCloudRenderStatus(job, jobId, CLOUD_RENDER_PHASE.RENDER, 'Rendering…')
+ : emitCloudRenderStatus(job, jobId, CLOUD_RENDER_PHASE.SUBMIT, status === 'THROTTLED' ? 'Waiting for a Runway slot…' : 'Queued at Runway…')),
+ });
+ if (polled.outcome === 'canceled') return finishFailure('Canceled');
+ if (polled.outcome === 'failed') return finishFailure(polled.reason);
+
+ emitCloudRenderStatus(job, jobId, CLOUD_RENDER_PHASE.FETCH, 'Downloading video…');
+ const buffer = await withAbortTimeout(RUNWAY_DOWNLOAD_TIMEOUT_MS, (timeoutSignal) => downloadRunwayOutput(
+ polled.outputUrl, anyAbortSignal([timeoutSignal, entry.controller.signal]),
+ ));
+ if (entry.aborted) return finishFailure('Canceled');
+ await writeFile(outputPath, buffer);
+ if (entry.aborted) {
+ await unlink(outputPath);
+ return finishFailure('Canceled');
+ }
+ // Record what Runway actually delivered: the Music Video renderer trims every clip by its measured length.
+ const measured = (await probeVideoGeometry(outputPath)) || {};
+ activeRequests.delete(jobId);
+ await settleInputs();
+ activeJobs.delete(jobId);
+ await finalizeGeneratedVideo({ job, jobId, outputPath, filename, meta: { ...meta, ...measured }, actualSeed: null, mutateHistory: mutateVideoHistory, publication });
+ closeJobAfterDelay(jobs, jobId);
+ } catch (err) {
+ if (!publication.committed) await discardUnpublishedVideo({ jobId, outputPath }).catch((error) => {
+ maintenance.markCurrentUnsettled();
+ maintenance.markResourceUnsettled('media', jobId);
+ console.error(`❌ Runway video publication cleanup failed [${jobId.slice(0, 8)}]: ${error.message}`);
+ });
+ await cancelRunwayTask(entry);
+ return finishFailure(`Runway video generation failed: ${err?.message || err}`);
+ }
+}
+
+const finalizeJobFailure = createJobFailureFinalizer({
+ jobs,
+ activeJobs,
+ activeSlots: activeRequests,
+ label: 'Runway video generation',
+ events: videoGenEvents,
+});
+const finalizeCanceled = finalizeJobFailure.canceled;
diff --git a/server/services/videoGen/runway.test.js b/server/services/videoGen/runway.test.js
new file mode 100644
index 0000000000..550b702af0
--- /dev/null
+++ b/server/services/videoGen/runway.test.js
@@ -0,0 +1,152 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { mkdir, rm, readFile } from 'fs/promises';
+import { join } from 'path';
+import { tmpdir } from 'os';
+import sharp from 'sharp';
+
+const TEST_ROOT = join(tmpdir(), `portos-runway-video-test-${process.pid}-${Date.now()}`);
+const FAKE_VIDEOS_DIR = join(TEST_ROOT, 'data-videos');
+const FAKE_DATA_DIR = join(TEST_ROOT, 'data');
+
+vi.mock('../../lib/fileUtils.js', async () => {
+ const actual = await vi.importActual('../../lib/fileUtils.js');
+ actual.PATHS.videos = FAKE_VIDEOS_DIR;
+ actual.PATHS.data = FAKE_DATA_DIR;
+ actual.PATHS.videoThumbnails = join(TEST_ROOT, 'thumbnails');
+ actual.PATHS.uploads = join(TEST_ROOT, 'uploads');
+ return {
+ ...actual,
+ ensureDir: vi.fn(async (dir) => mkdir(dir, { recursive: true })),
+ };
+});
+
+vi.mock('../../lib/ffmpeg.js', async () => {
+ const actual = await vi.importActual('../../lib/ffmpeg.js');
+ return {
+ ...actual,
+ optimizeForStreaming: vi.fn(async () => {}),
+ generateThumbnail: vi.fn(async (_p, jobId) => `${jobId}.jpg`),
+ probeVideoGeometry: vi.fn(async () => ({ duration: 4 })),
+ };
+});
+
+const getSettingsMock = vi.fn();
+vi.mock('../settings.js', () => ({ getSettings: getSettingsMock }));
+
+const runway = await import('./runway.js');
+const { RUNWAY_API_VERSION } = await import('../runwayApi.js');
+const { videoGenEvents } = await import('./events.js');
+const { loadHistory } = await import('./history.js');
+
+const terminalEvents = new Map();
+const waitForTerminal = async (jobId) => {
+ await vi.waitFor(() => expect(terminalEvents.get(jobId)).toBeTruthy(), { timeout: 5000 });
+ return terminalEvents.get(jobId);
+};
+
+const jsonResponse = (body, ok = true, status = 200) => ({ ok, status, json: async () => body, body: { cancel: async () => {} } });
+const TASKS = 'https://api.dev.runwayml.com/v1/tasks/task-1';
+
+let framePath;
+beforeEach(async () => {
+ videoGenEvents.removeAllListeners();
+ terminalEvents.clear();
+ videoGenEvents.on('started', ({ generationId }) => terminalEvents.set(generationId, null));
+ for (const type of ['completed', 'failed']) {
+ videoGenEvents.on(type, (event) => terminalEvents.set(event.generationId, { type, ...event }));
+ }
+ await rm(TEST_ROOT, { recursive: true, force: true }).catch(() => {});
+ await mkdir(FAKE_DATA_DIR, { recursive: true });
+ framePath = join(TEST_ROOT, 'frame.png');
+ await sharp({ create: { width: 64, height: 36, channels: 3, background: '#336699' } }).png().toFile(framePath);
+ getSettingsMock.mockReset().mockResolvedValue({});
+});
+
+afterEach(async () => {
+ await vi.waitFor(() => expect([...terminalEvents.values()].filter((e) => !e)).toEqual([]), { timeout: 5000 });
+ vi.unstubAllGlobals();
+ videoGenEvents.removeAllListeners();
+ await rm(TEST_ROOT, { recursive: true, force: true }).catch(() => {});
+});
+
+describe('videoGen/runway — generateVideo', () => {
+ it('submits the catalog body, polls to SUCCEEDED, downloads the clip and records the estimate', async () => {
+ const calls = [];
+ vi.stubGlobal('fetch', vi.fn(async (url, opts = {}) => {
+ calls.push({ url, method: opts.method || 'GET', headers: opts.headers, body: opts.body ? JSON.parse(opts.body) : null });
+ if (url === 'https://api.dev.runwayml.com/v1/image_to_video') return jsonResponse({ id: 'task-1' });
+ if (url === TASKS) return jsonResponse({ id: 'task-1', status: 'SUCCEEDED', output: ['https://cdn.example.com/out.mp4'] });
+ if (url === 'https://cdn.example.com/out.mp4') return { ok: true, status: 200, arrayBuffer: async () => Uint8Array.from(Buffer.from('mp4')).buffer };
+ throw new Error(`unexpected fetch: ${url}`);
+ }));
+
+ const job = await runway.generateVideo({
+ apiKey: 'key_test', modelId: 'gen4_turbo', prompt: 'waves roll in', negativePrompt: 'text',
+ duration: 3.4, width: 1280, height: 720, sourceImagePath: framePath,
+ });
+ expect(job).toMatchObject({ mode: 'runway', status: 'running' });
+ expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'completed' });
+
+ const submit = calls.find((c) => c.method === 'POST');
+ expect(submit.headers).toMatchObject({ Authorization: 'Bearer key_test', 'X-Runway-Version': RUNWAY_API_VERSION });
+ expect(submit.body).toEqual({
+ model: 'gen4_turbo', promptImage: expect.stringMatching(/^data:image\/png;base64,/),
+ promptText: 'waves roll in\nAvoid: text', duration: 4, ratio: '1280:720',
+ });
+ // A finished task is never DELETEd (that would delete it, not cancel it).
+ expect(calls.some((c) => c.method === 'DELETE')).toBe(false);
+ expect(await readFile(join(FAKE_VIDEOS_DIR, job.filename), 'utf8')).toBe('mp4');
+ const [entry] = await loadHistory();
+ expect(entry).toMatchObject({ modelId: 'runway:gen4_turbo', duration: 4, aspectRatio: '1280:720', estimatedCostUsd: 0.2 });
+ // The negative prompt rides in the prompt, so the record lists it as an addition (#11317).
+ expect(entry.sentPrompt).toBe('waves roll in\nAvoid: text');
+ expect(entry.promptAdditions).toEqual([expect.objectContaining({ id: 'avoid', text: 'text' })]);
+ });
+
+ it('fails the job with Runway\'s reason when the task FAILS', async () => {
+ vi.stubGlobal('fetch', vi.fn(async (url, opts = {}) => {
+ if (opts.method === 'POST') return jsonResponse({ id: 'task-1' });
+ if (url === TASKS) return jsonResponse({ id: 'task-1', status: 'FAILED', failure: 'Content moderation', failureCode: 'SAFETY.INPUT.IMAGE' });
+ throw new Error(`unexpected fetch: ${url} ${opts.method}`);
+ }));
+ const job = await runway.generateVideo({ apiKey: 'key_test', prompt: 'a fox', sourceImagePath: framePath });
+ expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'failed', error: expect.stringMatching(/Content moderation \(SAFETY\.INPUT\.IMAGE\)/) });
+ });
+
+ it.each([
+ ['no start frame', { prompt: 'a fox' }, /start frame/],
+ ['an over-cap prompt', { prompt: 'x'.repeat(1001), sourceImagePath: 'unused.png' }, /1,001 characters/],
+ ['source audio', { prompt: 'a fox', sourceImagePath: 'unused.png', audioFilePath: 'song.wav' }, /cutaways only/],
+ ])('refuses %s before reading media or submitting', async (_label, params, message) => {
+ const fetchMock = vi.fn();
+ vi.stubGlobal('fetch', fetchMock);
+ await expect(runway.generateVideo({ apiKey: 'key_test', ...params }))
+ .rejects.toMatchObject({ status: 400, message: expect.stringMatching(message) });
+ expect(fetchMock).not.toHaveBeenCalled();
+ });
+
+ it('refuses without a key, naming where to set it', async () => {
+ const prev = process.env.RUNWAYML_API_SECRET;
+ delete process.env.RUNWAYML_API_SECRET;
+ await expect(runway.generateVideo({ prompt: 'a fox', sourceImagePath: framePath }))
+ .rejects.toMatchObject({ code: 'RUNWAY_NOT_CONFIGURED' });
+ if (prev !== undefined) process.env.RUNWAYML_API_SECRET = prev;
+ });
+
+ it('cancels a running task with one DELETE and finishes the job canceled', async () => {
+ const deletes = [];
+ let polled;
+ const firstPoll = new Promise((resolve) => { polled = resolve; });
+ vi.stubGlobal('fetch', vi.fn(async (url, opts = {}) => {
+ if (opts.method === 'POST') return jsonResponse({ id: 'task-1' });
+ if (opts.method === 'DELETE') { deletes.push(url); return jsonResponse({}, true, 204); }
+ if (url === TASKS) { polled(); return jsonResponse({ id: 'task-1', status: 'RUNNING', progress: 0.2 }); }
+ throw new Error(`unexpected fetch: ${url}`);
+ }));
+ const job = await runway.generateVideo({ apiKey: 'key_test', prompt: 'a fox', sourceImagePath: framePath });
+ await firstPoll;
+ expect(runway.cancel(job.jobId)).toBe(true);
+ expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'failed' });
+ expect(deletes).toEqual([TASKS]);
+ });
+});
diff --git a/server/services/videoGen/submitJob.js b/server/services/videoGen/submitJob.js
index ac1b35ca33..b50c4adc61 100644
--- a/server/services/videoGen/submitJob.js
+++ b/server/services/videoGen/submitJob.js
@@ -375,6 +375,14 @@ const submitValidatedVideoGenJob = async (body, uploads) => {
// #10157: a board-started (manual or auto-review) fal clip records the
// estimate it was priced at so project spend counts it. A production-run
// step is already charged to the run's own usage, so it is skipped.
+ if (scene && backend === VIDEO_GEN_MODE.RUNWAY && !body.musicVideo.productionRunId) {
+ const { runwaySceneTake } = await import('../../lib/musicVideoShotTiming.js');
+ const { costUsd, modelId: costModelId } = runwaySceneTake({
+ scene,
+ videoSettings: { runwayModelId: body.runwayModelId, runwayDuration: body.runwayDuration, runwayResolution: body.runwayResolution },
+ });
+ if (Number.isFinite(costUsd)) Object.assign(params, { musicVideoCostUsd: costUsd, musicVideoCostModelId: costModelId });
+ }
if (scene && backend === VIDEO_GEN_MODE.FAL && !body.musicVideo.productionRunId) {
const { falSceneTake } = await import('../../lib/musicVideoShotTiming.js');
const { costUsd, modelId: costModelId } = falSceneTake({
@@ -434,7 +442,7 @@ const submitValidatedVideoGenJob = async (body, uploads) => {
// the model takes an aspect ratio, and its frames cropped to it where the
// output follows the start frame (falFrameAspect.js). An explicit
// visual-conditioning aspect (FableLoom) still wins.
- const projectAspect = backend === VIDEO_GEN_MODE.FAL && body.musicVideo && !body.visualConditioning?.render?.parameters?.aspectRatio
+ const projectAspect = (backend === VIDEO_GEN_MODE.FAL || backend === VIDEO_GEN_MODE.RUNWAY) && body.musicVideo && !body.visualConditioning?.render?.parameters?.aspectRatio
? await musicVideoProjectAspect(body.musicVideo.projectId).catch(async (error) => { await cleanupStaged(); throw error; })
: null;
const { jobId, position, status } = await enqueue({