Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

feat: add Runway video editing and character performance workflows

wrenchwren73ae0f702430
diff --git a/client/src/components/videoGen/RunwayVideoInputPanel.jsx b/client/src/components/videoGen/RunwayVideoInputPanel.jsx
new file mode 100644
index 0000000000..b665e3be75
--- /dev/null
+++ b/client/src/components/videoGen/RunwayVideoInputPanel.jsx
@@ -0,0 +1,35 @@
+import { RUNWAY_ACT_TWO_RATIOS, RUNWAY_ALEPH_ASPECTS } from '../../../../server/lib/runwaySpecialVideoModels.js';
+
+/** Inputs owned by the page's form state; switching modes clears their distinct meanings. */
+export default function RunwayVideoInputPanel({ mode, value, onChange }) {
+ const performance = mode === 'character-performance';
+ const change = (key, next) => onChange({ ...value, [key]: next });
+ return <div className="col-span-2 space-y-3 text-sm">
+ <label className="block">{performance ? 'Performance video' : 'Source video'}
+ <input type="file" accept="video/*" className="block w-full mt-1" onChange={(e) => change('video', e.target.files?.[0] || null)} />
+ </label>
+ <p className="text-xs text-port-text-muted">{performance ? 'Use a 3–30 second performance with a visible face. Choose a character image below or a character video here.' : 'Use a 2–30 second video at 30 FPS or lower. An optional guidance image below anchors the edit at the beginning.'} Videos must be 512 bytes–200 MB. Files are uploaded privately to Runway when the job starts; uploads expire after 24 hours.</p>
+ {performance ? <>
+ <label className="block">Character video (instead of an image)
+ <input type="file" accept="video/*" className="block w-full mt-1" onChange={(e) => change('characterVideo', e.target.files?.[0] || null)} />
+ </label>
+ <label className="block">Output canvas
+ <select value={value.ratio || ''} onChange={(e) => change('ratio', e.target.value)} className="block bg-port-bg border border-port-border rounded p-2">
+ <option value="">Runway default</option>{RUNWAY_ACT_TWO_RATIOS.map((r) => <option key={r} value={r}>{r.replace(':', ' × ')}</option>)}
+ </select>
+ </label>
+ <label className="flex gap-2"><input type="checkbox" checked={value.bodyControl ?? true} onChange={(e) => change('bodyControl', e.target.checked)} />Apply body movements and gestures</label>
+ <label className="block">Expression intensity
+ <input type="number" min="1" max="5" step="1" value={value.expressionIntensity ?? 3} onChange={(e) => change('expressionIntensity', Number(e.target.value))} className="block bg-port-bg border border-port-border rounded p-2" />
+ </label>
+ <p className="text-xs text-port-text-muted">Act-Two follows the performance, including its audio; it does not accept a text prompt.</p>
+ </> : <label className="block">Expand aspect ratio
+ <select value={value.targetAspectRatio || ''} onChange={(e) => change('targetAspectRatio', e.target.value)} className="block bg-port-bg border border-port-border rounded p-2">
+ <option value="">Preserve source framing</option>{RUNWAY_ALEPH_ASPECTS.map((r) => <option key={r} value={r}>{r}</option>)}
+ </select>
+ </label>}
+ <label className="block">Runway seed (optional)
+ <input type="number" min="0" max="4294967295" step="1" value={value.seed ?? ''} onChange={(e) => change('seed', e.target.value)} className="block bg-port-bg border border-port-border rounded p-2" />
+ </label>
+ </div>;
+}
diff --git a/client/src/hooks/useRunwayCatalog.js b/client/src/hooks/useRunwayCatalog.js
index 0129d5a455..a8a73ea0f2 100644
--- a/client/src/hooks/useRunwayCatalog.js
+++ b/client/src/hooks/useRunwayCatalog.js
@@ -2,14 +2,17 @@ import { useEffect, useSyncExternalStore } from 'react';
import { getRunwayCatalogSnapshot, subscribeRunwayCatalog, loadRunwayCatalog } from '../services/apiRunwayCatalog.js';
import { RUNWAY_IMAGE_MODELS } from '../lib/imageGenModes.js';
import { RUNWAY_VIDEO_MODELS } from '../lib/runwayVideoModels.js';
+import { RUNWAY_SPECIAL_VIDEO_MODELS } from '../../../server/lib/runwaySpecialVideoModels.js';
export function useRunwayCatalog(enabled = true, videoMode = 'image') {
const catalog = useSyncExternalStore(subscribeRunwayCatalog, getRunwayCatalogSnapshot);
useEffect(() => { if (enabled) void loadRunwayCatalog(); }, [enabled]);
+ const special = RUNWAY_SPECIAL_VIDEO_MODELS.find((m) => m.mode === videoMode);
+ const endpoint = special?.endpoint || (videoMode === 'text' ? 'text_to_video' : 'image_to_video');
return {
...catalog,
imageModels: RUNWAY_IMAGE_MODELS.filter((m) => catalog.available.text_to_image.includes(m.id)),
- videoModels: RUNWAY_VIDEO_MODELS.filter((m) => catalog.available[videoMode === 'text' ? 'text_to_video' : 'image_to_video'].includes(m.id)),
+ videoModels: (special ? RUNWAY_SPECIAL_VIDEO_MODELS : RUNWAY_VIDEO_MODELS).filter((m) => (catalog.available[endpoint] || []).includes(m.id)),
refresh: () => loadRunwayCatalog({ refresh: true }),
};
}
diff --git a/client/src/hooks/useRunwayCatalog.test.jsx b/client/src/hooks/useRunwayCatalog.test.jsx
index c5b25b5dfb..d1f471a94c 100644
--- a/client/src/hooks/useRunwayCatalog.test.jsx
+++ b/client/src/hooks/useRunwayCatalog.test.jsx
@@ -5,7 +5,7 @@ import { request } from '../services/apiCore.js';
import { useRunwayCatalog } from './useRunwayCatalog.js';
const initial = {
- available: { text_to_image: ['gen4_image', 'gpt_image_2'], image_to_video: ['gen4.5', 'gen4_turbo', 'veo3.1'], text_to_video: ['gen4.5', 'veo3.1'] },
+ available: { text_to_image: ['gen4_image', 'gpt_image_2'], image_to_video: ['gen4.5', 'gen4_turbo', 'veo3.1'], text_to_video: ['gen4.5', 'veo3.1'], video_to_video: ['aleph2'], character_performance: ['act_two'] },
unsupported: [{ id: 'act_two', reason: 'Requires performance inputs' }], stale: false,
};
describe('Runway form catalog lifecycle', () => {
@@ -13,10 +13,14 @@ describe('Runway form catalog lifecycle', () => {
request.mockResolvedValue(initial);
const image = renderHook(() => useRunwayCatalog());
const video = renderHook(() => useRunwayCatalog(true, 'text'));
+ const edit = renderHook(() => useRunwayCatalog(true, 'video-edit'));
+ const performance = renderHook(() => useRunwayCatalog(true, 'character-performance'));
await waitFor(() => expect(image.result.current.stale).toBe(false));
expect(request).toHaveBeenCalledTimes(1);
expect(image.result.current.imageModels.map((m) => m.id)).toEqual(['gen4_image', 'gpt_image_2']);
expect(video.result.current.videoModels.map((m) => m.id)).toEqual(['gen4.5', 'veo3.1']);
+ expect(edit.result.current.videoModels.map((m) => m.id)).toEqual(['aleph2']);
+ expect(performance.result.current.videoModels.map((m) => m.id)).toEqual(['act_two']);
image.unmount();
const reopened = renderHook(() => useRunwayCatalog());
expect(request).toHaveBeenCalledTimes(1);
diff --git a/client/src/hooks/useVideoGenFieldState.js b/client/src/hooks/useVideoGenFieldState.js
index 370c428b5b..0ef06391a0 100644
--- a/client/src/hooks/useVideoGenFieldState.js
+++ b/client/src/hooks/useVideoGenFieldState.js
@@ -31,6 +31,7 @@ export function useVideoGenFieldState({
const [runwayModelId, setRunwayModelId] = useState('');
const [runwayDuration, setRunwayDuration] = useState('');
const [runwayResolution, setRunwayResolution] = useState('');
+ const [runwayInputs, setRunwayInputs] = useState({});
const [reactorClipId, setReactorClipId] = useState('');
const [reactorSeconds, setReactorSeconds] = useState(REACTOR_DEFAULT_CLIP_LENGTH);
const [reactorSeed, setReactorSeed] = useState('');
@@ -107,6 +108,7 @@ export function useVideoGenFieldState({
runwayModelId, setRunwayModelId,
runwayDuration, setRunwayDuration,
runwayResolution, setRunwayResolution,
+ runwayInputs, setRunwayInputs,
reactorClipId, setReactorClipId,
reactorSeconds, setReactorSeconds,
reactorSeed, setReactorSeed,
diff --git a/client/src/hooks/useVideoGenForm.js b/client/src/hooks/useVideoGenForm.js
index fb2b9b8c42..f57f8102cf 100644
--- a/client/src/hooks/useVideoGenForm.js
+++ b/client/src/hooks/useVideoGenForm.js
@@ -25,6 +25,7 @@ import {
} from '../lib/videoGenParams.js';
import { videoStreamingModeFromRecord } from '../lib/videoStreamingMode.js';
import { RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL, getRunwayVideoModel, runwayModelTakesText } from '../lib/runwayVideoModels.js';
+import { RUNWAY_SPECIAL_VIDEO_MODELS, getRunwaySpecialVideoModel, RUNWAY_SPECIAL_VIDEO_MODES } from '../../../server/lib/runwaySpecialVideoModels.js';
import { useVideoGenFieldState } from './useVideoGenFieldState.js';
import { useVideoGenSubmitFlow } from './useVideoGenSubmitFlow.js';
import { useVideoGenValidation } from './useVideoGenValidation.js';
@@ -109,6 +110,7 @@ export function useVideoGenForm({
runwayModelId, setRunwayModelId,
runwayDuration, setRunwayDuration,
runwayResolution, setRunwayResolution,
+ runwayInputs, setRunwayInputs,
reactorClipId, setReactorClipId,
reactorSeconds, setReactorSeconds,
reactorSeed, setReactorSeed,
@@ -868,6 +870,14 @@ export function useVideoGenForm({
// carry over because they apply to all modes.)
const handleModeChange = (next) => {
setMode(next);
+ if (next !== mode) setRunwayInputs({});
+ if (isRunway) setRunwayModelId(RUNWAY_SPECIAL_VIDEO_MODELS.find((m) => m.mode === next)?.id || (getRunwaySpecialVideoModel(runwayModelId) ? '' : runwayModelId));
+ if (RUNWAY_SPECIAL_VIDEO_MODES.includes(next)) {
+ clearSourceImage();
+ clearLastImage();
+ setExtendFromVideoId('');
+ setChunks(1);
+ }
// Audio is only meaningful in a2v mode — drop it on every other switch
// so a stale upload from a prior pick doesn't sneak into a non-a2v post.
if (next !== 'a2v') setAudioFile(null);
@@ -951,6 +961,7 @@ export function useVideoGenForm({
// clip survives the switch and reappears if the user flips back.
const handleBackendChange = (id) => {
setBackend(id);
+ if (id !== 'runway' && RUNWAY_SPECIAL_VIDEO_MODES.includes(mode)) handleModeChange('text');
if ((id === 'grok' || id === 'fal' || id === 'reactor' || id === 'runway') && mode !== 'text' && mode !== 'image') {
handleModeChange((sourceImageFile || sourceImageUpload) ? 'image' : 'text');
}
@@ -1254,7 +1265,8 @@ export function useVideoGenForm({
} else if (p.mode === 'runway') {
// Runway job: same discriminator shape as fal above.
setBackend('runway');
- setMode(p.videoMode === 'text' ? 'text' : 'image');
+ setMode(RUNWAY_SPECIAL_VIDEO_MODES.includes(p.videoMode) ? p.videoMode : p.videoMode === 'text' ? 'text' : 'image');
+ setRunwayInputs({ targetAspectRatio: p.targetAspectRatio, ratio: p.ratio, bodyControl: p.bodyControl, expressionIntensity: p.expressionIntensity, seed: p.seed });
if (p.modelId) {
runwayModelPickRef.current = p.modelId;
setRunwayModelId(p.modelId);
@@ -1343,7 +1355,7 @@ export function useVideoGenForm({
const submissionState = {
hostedLane, isGrok, grokDuration, isFal, falDuration, falModelId, falResolution, falGenerateAudio,
isReactor, reactorClipId, reactorSeconds, reactorSeed, reactorAspect, remoteSubmissionFields,
- isRunway, runwayModelId, runwayDuration, runwayResolution,
+ isRunway, runwayModelId, runwayDuration, runwayResolution, runwayInputs,
displaySleepEnabled,
prompt, negativePrompt, stylePreset, selectedUniverse,
width, height, mode, sourceImageFile, sourceImageUpload,
@@ -1369,6 +1381,7 @@ export function useVideoGenForm({
runwayModelId, setRunwayModelId,
runwayDuration, setRunwayDuration,
runwayResolution, setRunwayResolution,
+ runwayInputs, setRunwayInputs,
reactorClipId, setReactorClipId,
reactorSeconds, setReactorSeconds,
reactorSeed, setReactorSeed,
diff --git a/client/src/lib/videoGenEnqueuePolicy.js b/client/src/lib/videoGenEnqueuePolicy.js
index 0ba4f42234..31a5244edc 100644
--- a/client/src/lib/videoGenEnqueuePolicy.js
+++ b/client/src/lib/videoGenEnqueuePolicy.js
@@ -27,7 +27,7 @@ const byovReason = ({ byovRuntimeMissing, byovLabel, byovRuntime }) => (byovRunt
: `Checking ${byovRuntime} runtime status…`);
const ENQUEUE_ROWS = [
- { scope: 'always', active: (c) => !c.prompt.trim() },
+ { scope: 'always', active: (c) => !c.promptOptional && !c.prompt.trim() },
{ scope: 'always', active: (c) => c.remixHandoffPending },
{ scope: 'always', active: (c) => c.promptOverLimit },
{ scope: 'remote', active: (c) => !!c.remoteBlocked, reason: (c) => c.remoteBlocked },
diff --git a/client/src/lib/videoGenSubmission.js b/client/src/lib/videoGenSubmission.js
index aed469cddc..f942183917 100644
--- a/client/src/lib/videoGenSubmission.js
+++ b/client/src/lib/videoGenSubmission.js
@@ -119,9 +119,22 @@ function falVideoSubmission({
// text-to-video rule, from the prompt alone (Text mode). The negative prompt is sent; the server passes it
// natively or as an "Avoid:" line, listed as a chip in the sent-prompt view.
function runwayVideoSubmission({
- composed, runwayModelId, runwayDuration, runwayResolution,
+ composed, runwayModelId, runwayDuration, runwayResolution, runwayInputs = {},
width, height, mode, sourceImageFile, sourceImageUpload,
}) {
+ if (mode === 'video-edit' || mode === 'character-performance') return {
+ backend: 'runway', mode, runwayModelId,
+ prompt: mode === 'character-performance' ? '' : composed.prompt,
+ negativePrompt: mode === 'character-performance' ? '' : composed.negativePrompt,
+ runwayVideo: runwayInputs.video || '',
+ runwayCharacterVideo: mode === 'character-performance' ? runwayInputs.characterVideo || '' : '',
+ sourceImageFile: runwayInputs.characterVideo ? '' : sourceImageFile || '',
+ sourceImage: runwayInputs.characterVideo ? '' : sourceImageUpload || '',
+ ...(runwayInputs.targetAspectRatio ? { runwayTargetAspect: runwayInputs.targetAspectRatio } : {}),
+ ...(runwayInputs.ratio ? { runwayRatio: runwayInputs.ratio } : {}),
+ ...(mode === 'character-performance' ? { runwayBodyControl: runwayInputs.bodyControl ?? true, runwayExpressionIntensity: runwayInputs.expressionIntensity ?? 3 } : {}),
+ ...(runwayInputs.seed != null && runwayInputs.seed !== '' ? { runwaySeed: Number(runwayInputs.seed) } : {}),
+ };
return {
backend: 'runway',
prompt: composed.prompt,
@@ -343,7 +356,8 @@ export function buildVideoGenSubmission(args) {
const shared = { ...args, composed };
const isLocal = !hostedLane && !remoteSubmissionFields;
- const additions = videoPromptAdditions({ ...args, isLocal });
+ const additions = hostedLane === 'runway' && args.mode === 'character-performance'
+ ? [] : videoPromptAdditions({ ...args, isLocal });
const payload = hostedLane ? HOSTED_VIDEO_SUBMISSIONS[hostedLane](shared)
: remoteSubmissionFields ? remoteVideoSubmission(shared)
: localVideoSubmission(args, {
diff --git a/client/src/lib/videoGenSubmission.test.js b/client/src/lib/videoGenSubmission.test.js
index 87c08358a4..2468d42418 100644
--- a/client/src/lib/videoGenSubmission.test.js
+++ b/client/src/lib/videoGenSubmission.test.js
@@ -255,6 +255,15 @@ describe('buildVideoGenSubmission — local IC-LoRA, chunks and misc', () => {
describe('buildVideoGenSubmission — prompt additions (#11317)', () => {
const preset = { label: 'Film noir', prompt: 'cinematic noir', negativePrompt: '' };
+ it('drops carried style metadata when switching to Runway character performance', () => {
+ const payload = buildVideoGenSubmission(baseArgs({
+ hostedLane: 'runway', mode: 'character-performance', runwayModelId: 'act_two',
+ stylePreset: preset, negativePrompt: 'blur', noMusic: true,
+ }));
+ expect(payload).toMatchObject({ prompt: '', negativePrompt: '' });
+ expect(payload).not.toHaveProperty('promptAdditions');
+ });
+
it('lists the style preset, then the No music line the local lane appends', () => {
const payload = buildVideoGenSubmission(baseArgs({ currentModel: {}, stylePreset: preset, noMusic: true }));
expect(payload.prompt).toBe('cinematic noir. a fox running through a forest\n\nno music, no soundtrack');
diff --git a/client/src/pages/VideoGen.jsx b/client/src/pages/VideoGen.jsx
index 3bd632b8c0..fe9582f0bb 100644
--- a/client/src/pages/VideoGen.jsx
+++ b/client/src/pages/VideoGen.jsx
@@ -1,5 +1,7 @@
import { useRunwayCatalog } from '../hooks/useRunwayCatalog.js';
import RunwayCatalogStatus from '../components/imageGen/RunwayCatalogStatus.jsx';
+import RunwayVideoInputPanel from '../components/videoGen/RunwayVideoInputPanel.jsx';
+import { RUNWAY_SPECIAL_VIDEO_MODES, getRunwaySpecialVideoModel } from '../../../server/lib/runwaySpecialVideoModels.js';
/**
* Video Generation page (LTX models via mlx_video on macOS, diffusers on
* Windows), with local, Grok, and federated render targets.
@@ -246,7 +248,7 @@ export default function VideoGen() {
// share. See client/src/hooks/useVideoGenForm.js.
const {
backend, hostedLane, isGrok, isFal, isReactor, isRunway, handleBackendChange, grokDuration, setGrokDuration,
- runwayModelId, setRunwayModelId, runwayDuration, setRunwayDuration, runwayResolution, setRunwayResolution,
+ runwayModelId, setRunwayModelId, runwayDuration, setRunwayDuration, runwayResolution, setRunwayResolution, runwayInputs, setRunwayInputs,
falDuration, setFalDuration, falModelId, setFalModelId,
falResolution, setFalResolution, falGenerateAudio, setFalGenerateAudio,
reactorClipId, setReactorClipId, reactorSeconds, setReactorSeconds, reactorSeed, setReactorSeed,
@@ -999,7 +1001,9 @@ export default function VideoGen() {
const falModel = getFalVideoModel(falModelId);
const falResolutions = falVideoResolutions(falModel);
const runwayCatalog = useRunwayCatalog(isRunway, mode);
- const runwayModel = getRunwayVideoModel(runwayModelId || RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL);
+ const runwaySpecial = isRunway && RUNWAY_SPECIAL_VIDEO_MODES.includes(mode);
+ const runwayPerformance = isRunway && mode === 'character-performance';
+ const runwayModel = getRunwaySpecialVideoModel(runwayModelId) || getRunwayVideoModel(runwayModelId || RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL);
const runwaySizeModel = mode === 'text' && runwayModel?.textToVideo ? { ...runwayModel, size: runwayModel.textToVideo.size } : runwayModel;
const runwayResolutions = runwayVideoResolutions(runwaySizeModel);
@@ -1020,7 +1024,7 @@ export default function VideoGen() {
: prompt.length),
[promptCap, prompt, negativePrompt, stylePreset, selectedUniverse],
);
- const promptOverLimit = promptCap != null && submittedPromptLength > promptCap;
+ const promptOverLimit = !runwayPerformance && promptCap != null && submittedPromptLength > promptCap;
// Budget the AI enhancer writes inside. It's the backend cap MINUS the style
// preset / universe prefix, because that prefix is part of what PortOS
@@ -1075,6 +1079,7 @@ export default function VideoGen() {
const { canEnqueue: backendCanEnqueue, generateTitle, queueTitle, optionsHint } = videoGenEnqueuePolicy({
prompt,
+ promptOptional: runwaySpecial,
remixHandoffPending,
promptOverLimit,
isRemote: remoteTarget.isRemote,
@@ -1095,7 +1100,8 @@ export default function VideoGen() {
keyframesBlocked,
keyframesError,
});
- const canEnqueue = backendCanEnqueue && (!isRunway || runwayCatalog.videoModels.some((m) => m.id === runwayModel?.id));
+ const specialInputsReady = !runwaySpecial || (!!runwayInputs.video && (!runwayPerformance || !!(runwayInputs.characterVideo || sourceImageFile || sourceImageUpload)));
+ const canEnqueue = backendCanEnqueue && specialInputsReady && (!isRunway || runwayCatalog.videoModels.some((m) => m.id === runwayModel?.id));
return (
<div className="space-y-3">
@@ -1219,7 +1225,7 @@ export default function VideoGen() {
WAI-ARIA Tabs, since the mode-specific inputs aren't structured as
tabpanels and we don't implement roving-tabindex/arrow-key focus. */}
<div className="bg-port-card border border-port-border rounded-xl p-1 flex flex-wrap gap-1" role="group" aria-label="Video generation mode">
- {(rendersOffMachine ? MODES.filter((m) => m.id === 'text' || m.id === 'image') : MODES).map(({ id, label, icon: Icon, desc }) => {
+ {(isRunway ? [...MODES.filter((m) => m.id === 'text' || m.id === 'image'), { id: 'video-edit', label: 'Video edit', icon: SlidersHorizontal, desc: 'Edit a source video with Aleph 2' }, { id: 'character-performance', label: 'Character performance', icon: SlidersHorizontal, desc: 'Apply a performance to a character with Act-Two' }] : rendersOffMachine ? MODES.filter((m) => m.id === 'text' || m.id === 'image') : MODES).map(({ id, label, icon: Icon, desc }) => {
const active = mode === id;
return (
<button
@@ -1315,7 +1321,7 @@ export default function VideoGen() {
repairing={modelDownload.repairing}
/>
))}
- <FormField label="Prompt" labelClassName="block text-xs font-medium text-gray-400 mb-1">
+ {!runwayPerformance && <FormField label="Prompt" labelClassName="block text-xs font-medium text-gray-400 mb-1">
<AutoSizeTextarea
value={prompt}
onChange={(e) => setPrompt(e.target.value)}
@@ -1347,7 +1353,7 @@ export default function VideoGen() {
This model does not use a negative prompt.
</p>
)}
- </FormField>
+ </FormField>}
<div
className="sticky top-0 z-10 -mx-4 flex flex-wrap items-center gap-2 border-y border-port-border bg-port-card/95 px-4 py-2 backdrop-blur lg:static lg:mx-0 lg:border-0 lg:bg-transparent lg:p-0 lg:backdrop-blur-none"
@@ -1417,7 +1423,7 @@ export default function VideoGen() {
</summary>
<div className="mt-3 grid min-w-0 grid-cols-1 gap-4 lg:mt-0 @3xl/video-form:grid-cols-2">
<section aria-label="Prompt styling and reference media" className="min-w-0 space-y-3">
- <div className="grid min-w-0 grid-cols-1 gap-3 @5xl/video-form:grid-cols-2">
+ {!runwayPerformance && <div className="grid min-w-0 grid-cols-1 gap-3 @5xl/video-form:grid-cols-2">
<UniverseStylePicker
value={selectedUniverse?.id || ''}
onChange={setSelectedUniverse}
@@ -1426,8 +1432,8 @@ export default function VideoGen() {
value={stylePreset?.id || ''}
onChange={setStylePreset}
/>
- </div>
- {negativePromptSupported && (
+ </div>}
+ {!runwayPerformance && negativePromptSupported && (
<FormField label="Negative Prompt" labelClassName="block text-xs font-medium text-gray-400 mb-1">
<AutoSizeTextarea
value={negativePrompt}
@@ -1442,7 +1448,7 @@ export default function VideoGen() {
{/* Keep Enhance live while a render is in flight so the next clip
can be composed and submitted to its server queue. Generate
itself becomes Cancel. */}
- <PromptEnhancer
+ {!runwayPerformance && <PromptEnhancer
kind="video"
prompt={prompt}
setPrompt={setPrompt}
@@ -1450,7 +1456,7 @@ export default function VideoGen() {
setNegativePrompt={negativePromptSupported ? setNegativePrompt : undefined}
renderConfig={{ stylePreset: stylePreset?.id, mode, model: modelId }}
maxPromptLength={enhancePromptBudget}
- />
+ />}
{mode === 'fflf' && keyframesSupported && (
<KeyframePanel
@@ -1467,10 +1473,10 @@ export default function VideoGen() {
/>
)}
- {(mode === 'image' || (mode === 'fflf' && !keyframesActive)) && (
+ {(mode === 'image' || (runwaySpecial && !runwayInputs.characterVideo) || (mode === 'fflf' && !keyframesActive)) && (
<div className={`grid gap-2 ${mode === 'fflf' ? 'grid-cols-1 sm:grid-cols-2' : 'grid-cols-1'}`}>
<FramePanel
- label={mode === 'fflf' ? 'First frame' : 'Source image'}
+ label={runwayPerformance ? 'Character image' : runwaySpecial ? 'Guidance image (optional)' : mode === 'fflf' ? 'First frame' : 'Source image'}
file={sourceImageFile}
upload={sourceImageUpload}
uploadUrl={sourceUploadUrl}
@@ -1710,6 +1716,7 @@ export default function VideoGen() {
))}
</select>
</FormField>
+ {runwaySpecial ? <RunwayVideoInputPanel key={mode} mode={mode} value={runwayInputs} onChange={setRunwayInputs} /> : <>
<FormField label="Clip length" labelClassName="block text-xs font-medium text-gray-400 mb-1">
<select
id="runway-video-length"
@@ -1743,8 +1750,9 @@ export default function VideoGen() {
</select>
</FormField>
)}
+ </>}
<p className="col-span-2 text-[11px] text-gray-500 leading-snug">
- {mode === 'image' ? 'Animates your start frame' : 'Renders from your prompt'} on the Runway API. Counts against your Runway credits
+ {runwaySpecial ? 'Uses your video inputs' : mode === 'image' ? 'Animates your start frame' : 'Renders from your prompt'} on the Runway API. Counts against your Runway credits
{(() => {
const estimate = estimateRunwayVideoCostUsd({
modelId: runwayModel?.id, seconds: runwayDuration === '' ? null : Number(runwayDuration), resolution: runwayResolution,
diff --git a/client/src/pages/VideoGen.runway.test.jsx b/client/src/pages/VideoGen.runway.test.jsx
index 6ec753c9c4..abf4bbc91b 100644
--- a/client/src/pages/VideoGen.runway.test.jsx
+++ b/client/src/pages/VideoGen.runway.test.jsx
@@ -18,6 +18,52 @@ describe('VideoGen Runway text lane', () => {
state.generateVideo.mockResolvedValue({ jobId: 'job-1' });
});
+ it.each([
+ ['Video edit', 'aleph2', 'Source video'],
+ ['Character performance', 'act_two', 'Performance video'],
+ ])('submits %s through its own model and video inputs without inventing a prompt', async (label, modelId, videoLabel) => {
+ await renderVideoGenPage();
+ fireEvent.click(await screen.findByRole('button', { name: 'Runway' }));
+ fireEvent.click(within(screen.getByRole('group', { name: 'Video generation mode' })).getByRole('button', { name: label }));
+ const model = await screen.findByLabelText('Runway model');
+ expect([...model.options].map((o) => o.value)).toEqual([modelId]);
+ const source = new File([new Uint8Array(1024)], 'performance.mp4', { type: 'video/mp4' });
+ expect(screen.getByRole('button', { name: /Add to queue/ })).toBeDisabled();
+ fireEvent.change(screen.getByLabelText(videoLabel), { target: { files: [source] } });
+ if (modelId === 'act_two') {
+ expect(screen.queryByLabelText('Prompt')).toBeNull();
+ expect(screen.getByRole('button', { name: /Add to queue/ })).toBeDisabled();
+ fireEvent.change(screen.getByLabelText('Character video (instead of an image)'), { target: { files: [source] } });
+ fireEvent.change(screen.getByLabelText('Expression intensity'), { target: { value: '4' } });
+ }
+ fireEvent.click(await findEnabledByRole('button', { name: /Add to queue/ }));
+ await waitFor(() => expect(state.generateVideo).toHaveBeenCalled());
+ expect(state.generateVideo.mock.calls[0][0]).toMatchObject({ backend: 'runway', runwayModelId: modelId, runwayVideo: source, prompt: '', mode: modelId === 'aleph2' ? 'video-edit' : 'character-performance' });
+ if (modelId === 'act_two') expect(state.generateVideo.mock.calls[0][0]).toMatchObject({ runwayCharacterVideo: source, runwayExpressionIntensity: 4 });
+ });
+
+ it('clears file selections when switching special workflows and allows reselecting the same file', async () => {
+ await renderVideoGenPage();
+ fireEvent.click(await screen.findByRole('button', { name: 'Runway' }));
+ const modes = within(screen.getByRole('group', { name: 'Video generation mode' }));
+ fireEvent.click(modes.getByRole('button', { name: 'Video edit' }));
+ const firstInput = screen.getByLabelText('Source video');
+ const source = new File([new Uint8Array(1024)], 'source.mp4', { type: 'video/mp4' });
+ fireEvent.change(firstInput, { target: { files: [source] } });
+ await findEnabledByRole('button', { name: /Add to queue/ });
+ fireEvent.click(modes.getByRole('button', { name: 'Character performance' }));
+ expect(screen.getByLabelText('Performance video')).not.toBe(firstInput);
+ expect(screen.getByLabelText('Performance video').files).toHaveLength(0);
+ fireEvent.click(modes.getByRole('button', { name: 'Video edit' }));
+ const resetInput = screen.getByLabelText('Source video');
+ expect(resetInput.files).toHaveLength(0);
+ expect(screen.getByRole('button', { name: /Add to queue/ })).toBeDisabled();
+ fireEvent.change(resetInput, { target: { files: [source] } });
+ fireEvent.click(await findEnabledByRole('button', { name: /Add to queue/ }));
+ await waitFor(() => expect(state.generateVideo).toHaveBeenCalled());
+ expect(state.generateVideo.mock.calls[0][0]).toMatchObject({ runwayVideo: source });
+ });
+
it('switches from an image-only model to a priced text-capable model and submits without a frame', async () => {
await renderVideoGenPage();
fireEvent.click(await screen.findByRole('button', { name: 'Runway' }));
diff --git a/docs/RUNWAY_MODELS.md b/docs/RUNWAY_MODELS.md
index ca7bdc1067..4f83f592e7 100644
--- a/docs/RUNWAY_MODELS.md
+++ b/docs/RUNWAY_MODELS.md
@@ -9,6 +9,8 @@ needs no API key.
The bundled capability snapshot was verified on 2026-10-22. It offers 19 image
models and 24 image-to-video models; 23 of the video models also accept text alone.
+Video edit adds Aleph 2 and Character performance adds Act-Two, for 26 distinct
+video models across these workflows.
Selectors filter by the active operation. Gen-4 Image Turbo requires a reference;
Gen-4 Turbo video requires a start frame. The builder validates each model's prompt,
reference and seed constraints and emits supported parameters only. Models accepting
@@ -49,12 +51,30 @@ before a paid POST. Generation does not perform a separate discovery lookup.
## Workflow limits
-Aleph 2 (`aleph2`) needs video-to-video editing inputs and Act-Two (`act_two`) needs
-character/performance inputs. They are identified in the catalog UI but are not
-runnable through the existing text/image forms. These are PortOS workflow limits,
-not missing Runway APIs. Upscaling, HDR, avatars, source-video/audio references and
-other specialized controls are also outside these forms. This integration does not
-claim to expose every Runway API operation.
+Video Gen's Runway lane offers **Video edit** (`aleph2`) and **Character performance**
+(`act_two`) alongside Text and Image. Aleph takes a 2–30 second source video at up
+to 30 FPS, an optional prompt (1000 characters including the negative prompt), an
+optional guidance image at the start, and optional expand aspect ratio. It uses
+the default MP4 delivery and preserves source resolution up to 1080p. Act-Two takes
+a 3–30 second performance clip plus one character image or video, body-control and
+expression-intensity controls, and an optional supported canvas. It follows the
+performance rather than a text prompt. Video durations and streams are probed
+before upload; invalid or unreadable inputs fail before generation.
+
+Uploads are staged durably using the normal queue ownership/cleanup rules. The
+worker creates [ephemeral Runway uploads](https://docs.dev.runwayml.com/assets/uploads/)
+at dispatch time, propagates cancellation through registration and storage POST,
+and submits only after every upload succeeds. Video files must be 512 bytes–200 MB.
+Only local input paths are queued; 24-hour upload URIs are never persisted as retry
+inputs, and replay creates fresh uploads. Storage POSTs receive signed upload fields,
+not the Runway API key. Local owned inputs are cleaned on success/failure/cancel;
+Runway's remote uploads expire automatically. Reloaded forms require video files to
+be selected again before a new submission.
+
+Specialized models stay out of text/image-only Music Video and Creative Director
+selectors. Other models' advanced video-reference controls, multiple timed Aleph
+keyframes, non-MP4 delivery, upscaling, HDR and avatars remain outside these forms.
+This integration does not claim to expose every Runway API operation.
Tests use public schema excerpts and mocked transport, including refresh, cache
restart/expiry, source failures and capability-specific request bodies. No paid
diff --git a/server/lib/README.md b/server/lib/README.md
index 6a1da14625..a3a74bedaf 100644
--- a/server/lib/README.md
+++ b/server/lib/README.md
@@ -972,3 +972,5 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
- `manuscriptLocate.js`: `locateFind` / `locateFindSpan` — the manuscript `find` locator (nearest-to-`anchorQuote` occurrence, then whitespace-tolerant regex fallback). The one implementation behind both the server accept path (`services/pipeline/manuscriptFix.js`) and the client highlight/impact preview (`client/src/lib/manuscriptAnchors.js` re-exports it), so the preview selects the span an accept replaces. Pure / browser-safe.
| `runwayCatalogSnapshot.js` | Shared Runway model catalog capabilities and cached model selection. |
+
+| `runwaySpecialVideoModels.js` | Aleph 2 video-edit and Act-Two character-performance capabilities, schema compatibility and request validation. |
diff --git a/server/lib/index.js b/server/lib/index.js
index 133631f589..8129f94c71 100644
--- a/server/lib/index.js
+++ b/server/lib/index.js
@@ -958,3 +958,5 @@ export * from './errorLogFingerprint.js';
export * from './manuscriptLocate.js';
export * from './runwayCatalogSnapshot.js';
+
+export * from './runwaySpecialVideoModels.js';
diff --git a/server/lib/runwayCatalogSnapshot.js b/server/lib/runwayCatalogSnapshot.js
index e544c80cfd..31e2e55427 100644
--- a/server/lib/runwayCatalogSnapshot.js
+++ b/server/lib/runwayCatalogSnapshot.js
@@ -73,5 +73,7 @@ export const RUNWAY_CATALOG_SNAPSHOT = {
{"id": "h3_max","promptRequired": true,"promptMin": 1,"promptMax": 6000,"ratios": [],"ratioRequired": false,"resolutions": ["480p","768p"],"seed": true,"audio": false,"endImage": false,"duration": {"minimum": 5,"maximum": 15},"minReferences": 0,"maxReferences": 0,"referenceTags": false,"seedMax": 4294967295,"negativePromptMax": 0},
{"id": "seedance2_5","promptRequired": false,"promptMin": 1,"promptMax": 15000,"ratios": ["992:432","854:480","752:560","640:640","560:752","480:854","1470:630","1280:720","1112:834","960:960","834:1112","720:1280","2206:946","1920:1080","1664:1248","1440:1440","1248:1664","1080:1920"],"ratioRequired": false,"resolutions": [],"seed": true,"audio": true,"endImage": false,"duration": {"minimum": 4,"maximum": 30},"minReferences": 0,"maxReferences": 0,"referenceTags": false,"seedMax": 4294967295,"negativePromptMax": 0},
{"id": "grok_imagine_1_5_lite","promptRequired": true,"promptMin": 1,"promptMax": 2500,"ratios": ["848:480","480:848","480:480","640:480","480:640","720:480","480:720","1280:720","720:1280","720:720","960:720","720:960","1088:720","720:1088","1904:1072","1072:1904","1424:1424","1648:1232","1232:1648","1744:1152","1152:1744"],"ratioRequired": false,"resolutions": [],"seed": false,"audio": false,"endImage": false,"duration": {"minimum": 1,"maximum": 15},"minReferences": 0,"maxReferences": 0,"referenceTags": false,"seedMax": null,"negativePromptMax": 0}
- ]
+ ],
+ "video_to_video": [{"id":"aleph2","contract":{"additionalProperties":false,"properties":{"contentModeration":{"additionalProperties":false,"properties":{"publicFigureThreshold":{"enum":["auto","low"],"type":"string"}},"type":"object"},"keyframes":{"items":{"anyOf":[{"additionalProperties":false,"properties":{"range":{"additionalProperties":false,"properties":{"end_seconds":{"exclusiveMinimum":0,"maximum":9007199254740991,"type":"integer"},"start_seconds":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["start_seconds","end_seconds"],"type":"object"},"seconds":{"maximum":30,"minimum":0,"type":"number"},"uri":{"anyOf":[{"maxLength":2048,"minLength":13,"pattern":"^https:\\/\\/.*","type":"string"},{"maxLength":5000,"minLength":13,"pattern":"^runway:\\/\\/.*","type":"string"},{"maxLength":5242880,"minLength":13,"pattern":"^data:image\\/.*","type":"string"}]}},"required":["uri","seconds"],"type":"object"},{"additionalProperties":false,"properties":{"at":{"maximum":1,"minimum":0,"type":"number"},"range":{"additionalProperties":false,"properties":{"end_seconds":{"exclusiveMinimum":0,"maximum":9007199254740991,"type":"integer"},"start_seconds":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["start_seconds","end_seconds"],"type":"object"},"uri":{"anyOf":[{"maxLength":2048,"minLength":13,"pattern":"^https:\\/\\/.*","type":"string"},{"maxLength":5000,"minLength":13,"pattern":"^runway:\\/\\/.*","type":"string"},{"maxLength":5242880,"minLength":13,"pattern":"^data:image\\/.*","type":"string"}]}},"required":["uri","at"],"type":"object"}]},"maxItems":5,"minItems":1,"type":"array"},"model":{"const":"aleph2","type":"string"},"outputFormat":{"enum":["mp4","prores","png_sequence","sdr_rec709_10bit"],"type":"string"},"promptText":{"maxLength":1000,"minLength":1,"type":"string"},"proresProfile":{"enum":["422","4444","422 Proxy","422 LT","422 HQ","4444 XQ"],"type":"string"},"ratio":{"type":"string"},"seed":{"maximum":4294967295,"minimum":0,"type":"integer"},"targetAspectRatio":{"enum":["16:9","4:3","3:2","1:1","2:3","3:4","9:16","21:9"],"type":"string"},"videoUri":{"anyOf":[{"maxLength":2048,"minLength":13,"pattern":"^https:\\/\\/.*","type":"string"},{"maxLength":5000,"minLength":13,"pattern":"^runway:\\/\\/.*","type":"string"},{"maxLength":5242880,"minLength":13,"pattern":"^data:video\\/.*","type":"string"}]}},"required":["videoUri","model"],"type":"object"}}],
+ "character_performance": [{"id":"act_two","contract":{"properties":{"bodyControl":{"type":"boolean"},"character":{"oneOf":[{"properties":{"type":{"const":"image","type":"string"},"uri":{"anyOf":[{"maxLength":2048,"minLength":13,"pattern":"^https:\\/\\/.*","type":"string"},{"maxLength":5000,"minLength":13,"pattern":"^runway:\\/\\/.*","type":"string"},{"maxLength":5242880,"minLength":13,"pattern":"^data:image\\/.*","type":"string"}]}},"required":["type","uri"],"type":"object"},{"properties":{"type":{"const":"video","type":"string"},"uri":{"anyOf":[{"maxLength":2048,"minLength":13,"pattern":"^https:\\/\\/.*","type":"string"},{"maxLength":5000,"minLength":13,"pattern":"^runway:\\/\\/.*","type":"string"},{"maxLength":5242880,"minLength":13,"pattern":"^data:video\\/.*","type":"string"}]}},"required":["type","uri"],"type":"object"}]},"contentModeration":{"additionalProperties":false,"properties":{"publicFigureThreshold":{"enum":["auto","low"],"type":"string"}},"type":"object"},"expressionIntensity":{"maximum":5,"minimum":1,"type":"integer"},"model":{"const":"act_two","type":"string"},"ratio":{"enum":["1280:720","720:1280","960:960","1104:832","832:1104","1584:672"],"type":"string"},"reference":{"oneOf":[{"properties":{"type":{"const":"video","type":"string"},"uri":{"anyOf":[{"maxLength":2048,"minLength":13,"pattern":"^https:\\/\\/.*","type":"string"},{"maxLength":5000,"minLength":13,"pattern":"^runway:\\/\\/.*","type":"string"},{"maxLength":5242880,"minLength":13,"pattern":"^data:video\\/.*","type":"string"}]}},"required":["type","uri"],"type":"object"}]},"seed":{"maximum":4294967295,"minimum":0,"type":"integer"}},"required":["character","reference","model"],"type":"object"}}]
};
diff --git a/server/lib/runwaySpecialVideoModels.js b/server/lib/runwaySpecialVideoModels.js
new file mode 100644
index 0000000000..8a35bdf6ca
--- /dev/null
+++ b/server/lib/runwaySpecialVideoModels.js
@@ -0,0 +1,56 @@
+/** Runway workflows with video inputs, separate from text/start-frame model choices. */
+export const RUNWAY_SPECIAL_VIDEO_MODELS = Object.freeze([
+ { id: 'aleph2', label: 'Aleph 2', mode: 'video-edit', endpoint: 'video_to_video', promptMaxChars: 1000, prompt: 'optional', minSec: 2, maxSec: 30 },
+ { id: 'act_two', label: 'Act-Two', mode: 'character-performance', endpoint: 'character_performance', promptMaxChars: 0, prompt: 'unsupported', minSec: 3, maxSec: 30 },
+]);
+export const getRunwaySpecialVideoModel = (id) => RUNWAY_SPECIAL_VIDEO_MODELS.find((m) => m.id === id) || null;
+export const RUNWAY_SPECIAL_VIDEO_MODES = RUNWAY_SPECIAL_VIDEO_MODELS.map((m) => m.mode);
+export const RUNWAY_ACT_TWO_RATIOS = ['1280:720', '720:1280', '960:960', '1104:832', '832:1104', '1584:672'];
+export const RUNWAY_ALEPH_ASPECTS = ['16:9', '4:3', '3:2', '1:1', '2:3', '3:4', '9:16', '21:9'];
+
+/** Semantic schema only; documentation wording/order must not retire a compatible model. */
+export function runwaySpecialSchemaContract(value) {
+ if (Array.isArray(value)) return value.map(runwaySpecialSchemaContract);
+ if (!value || typeof value !== 'object') return value;
+ const ignored = ['$schema', 'description', 'title', 'example', 'examples', 'default', 'deprecated', 'discriminator'];
+ return Object.fromEntries(Object.keys(value).sort().filter((key) => !ignored.includes(key) && !key.startsWith('x-')).map((key) => [key, runwaySpecialSchemaContract(value[key])]));
+}
+
+export function buildRunwaySpecialVideoRequest({ modelId, videoMode, videoUri, characterVideoUri, imageUri, endImageUri,
+ prompt = '', negativePrompt = '', seed, targetAspectRatio, ratio, bodyControl = true, expressionIntensity = 3, sourceDuration } = {}) {
+ const fail = (message) => { throw Object.assign(new Error(message), { status: 400, code: 'RUNWAY_MODEL_INPUT' }); };
+ const model = getRunwaySpecialVideoModel(modelId);
+ if (!model || (videoMode && videoMode !== model.mode)) fail('Select a Runway model matching the video workflow.');
+ if (!videoUri) fail(`${model.label} needs a ${model.id === 'act_two' ? 'performance' : 'source'} video.`);
+ if (endImageUri) fail(`${model.label} does not accept an end frame in this workflow.`);
+ const body = { model: model.id };
+ if (seed != null && seed !== '') {
+ if (!Number.isInteger(seed) || seed < 0 || seed > 4294967295) fail('Runway seed must be an integer between 0 and 4294967295.');
+ body.seed = seed;
+ }
+ if (model.id === 'aleph2') {
+ if (characterVideoUri) fail('Aleph 2 takes one source video, not a character video.');
+ if (ratio) fail('Aleph 2 preserves source resolution; use an expand aspect ratio instead.');
+ const text = [prompt.trim(), negativePrompt.trim() ? `Avoid: ${negativePrompt.trim()}` : ''].filter(Boolean).join('\n');
+ if (text.length > 1000) fail('Aleph 2 takes at most 1000 prompt characters, including the negative prompt.');
+ body.videoUri = videoUri;
+ if (text) body.promptText = text;
+ if (imageUri) body.keyframes = [{ uri: imageUri, at: 0 }];
+ if (targetAspectRatio) {
+ if (!RUNWAY_ALEPH_ASPECTS.includes(targetAspectRatio)) fail('Unsupported Aleph 2 expand aspect ratio.');
+ body.targetAspectRatio = targetAspectRatio;
+ }
+ } else {
+ if (prompt.trim() || negativePrompt.trim()) fail('Act-Two follows the performance video and does not accept prompts.');
+ if (!!imageUri === !!characterVideoUri) fail('Act-Two needs exactly one character image or character video.');
+ if (targetAspectRatio) fail('Act-Two does not accept an expand aspect ratio.');
+ if (typeof bodyControl !== 'boolean' || !Number.isInteger(expressionIntensity) || expressionIntensity < 1 || expressionIntensity > 5) fail('Act-Two expression intensity must be 1–5 and body control must be boolean.');
+ if (ratio && !RUNWAY_ACT_TWO_RATIOS.includes(ratio)) fail('Unsupported Act-Two output canvas.');
+ body.character = { type: characterVideoUri ? 'video' : 'image', uri: characterVideoUri || imageUri };
+ body.reference = { type: 'video', uri: videoUri };
+ body.bodyControl = bodyControl;
+ body.expressionIntensity = expressionIntensity;
+ if (ratio) body.ratio = ratio;
+ }
+ return { model, endpoint: model.endpoint, body, seconds: sourceDuration ?? null, resolution: ratio || null, estimatedCostUsd: null };
+}
diff --git a/server/routes/videoGen.js b/server/routes/videoGen.js
index 4a04bbb351..4d5093fd36 100644
--- a/server/routes/videoGen.js
+++ b/server/routes/videoGen.js
@@ -98,6 +98,7 @@ import { detectSystemCapabilities, withHardwareCompatibility } from '../lib/syst
import { isDisplaySleepEnabled } from '../services/videoGen/displayPower.js';
import { promptAdditionsSchema } from '../lib/modelInput.js';
import { VIDEO_TILING_MODES } from '../lib/videoTiling.js';
+import { RUNWAY_SPECIAL_VIDEO_MODES, RUNWAY_ACT_TWO_RATIOS, RUNWAY_ALEPH_ASPECTS } from '../lib/runwaySpecialVideoModels.js';
const router = Router();
@@ -173,11 +174,11 @@ export const isAudioMime = (mime, filename) => {
// upload (icReference). Files stream to disk without a file-size cap.
// Per-fieldname mime filtering rejects mismatched parts
// up-front so a stray .mp4 drag-drop can't get staged under these fields.
-const frameImageUpload = uploadFields(['sourceImage', 'lastImage', 'audioFile', 'icReference'], {
+const frameImageUpload = uploadFields(['sourceImage', 'lastImage', 'audioFile', 'icReference', 'runwayVideo', 'runwayCharacterVideo'], {
fileFilter: (_req, file, cb) => {
const isImageField = file.fieldname === 'sourceImage' || file.fieldname === 'lastImage';
const isAudioField = file.fieldname === 'audioFile';
- const isVideoField = file.fieldname === 'icReference';
+ const isVideoField = ['icReference', 'runwayVideo', 'runwayCharacterVideo'].includes(file.fieldname);
const okImage = isImageField && file.mimetype.startsWith('image/');
const okAudio = isAudioField && isAudioMime(file.mimetype, file.originalname);
// IC-LoRA references are clips — the control weight reads structure/motion
@@ -324,6 +325,11 @@ const generateBodySchema = z.object({
// validated like falModelId: the provider refuses an uncatalogued id and
// rounds a length up to one the model delivers.
runwayModelId: z.string().min(1).max(64).optional(),
+ runwayTargetAspect: z.enum(RUNWAY_ALEPH_ASPECTS).optional(),
+ runwayRatio: z.enum(RUNWAY_ACT_TWO_RATIOS).optional(),
+ runwayBodyControl: z.preprocess((v) => v === 'true' ? true : v === 'false' ? false : v, z.boolean().optional()),
+ runwayExpressionIntensity: optionalNum(1, 5, 'runwayExpressionIntensity'),
+ runwaySeed: optionalNum(0, 4294967295, 'runwaySeed'),
runwayDuration: z.preprocess(
(v) => (v == null || v === '' ? undefined : Number(v)),
optionalNum(1, 60, 'runwayDuration'),
@@ -354,7 +360,7 @@ const generateBodySchema = z.object({
(v) => (v === '' ? undefined : v),
z.enum(REACTOR_ASPECTS).optional(),
),
- prompt: z.string().min(1).max(20000),
+ prompt: z.string().max(20000),
negativePrompt: z.string().max(8000).optional(),
modelId: z.string().max(64).optional(),
width: optionalNum(64, 2048, 'width'),
@@ -378,7 +384,7 @@ const generateBodySchema = z.object({
// falls back to inferring (sourceImage→i2v, no source→t2v).
// IC-LoRA remix modes (`ic-control`, …) come from the weight registry so the
// enum can never drift from what's actually installable.
- mode: z.enum(['text', 'image', 'fflf', 'extend', 'a2v', ...IC_LORA_MODE_VALUES]).optional(),
+ mode: z.enum(['text', 'image', 'fflf', 'extend', 'a2v', ...IC_LORA_MODE_VALUES, ...RUNWAY_SPECIAL_VIDEO_MODES]).optional(),
// Chain N renders end-to-end: each chunk's last frame becomes the next
// chunk's start frame, then ffmpeg concats them into one clip. 1..8 to
// keep the worst-case wall time bounded (8 × ~5min ≈ 40min on M3 Max).
@@ -546,6 +552,8 @@ const generateBodySchema = z.object({
// docs/decisions/2026-08-20-federated-visual-prompts.md
mediaProviderPeerId: z.string().guid().optional(),
mediaProviderEngine: z.string().trim().min(1).max(80).optional(),
+}).refine((body) => body.prompt.length > 0 || (body.backend === 'runway' && RUNWAY_SPECIAL_VIDEO_MODES.includes(body.mode)), {
+ message: 'A prompt is required for this workflow.', path: ['prompt'],
});
// The Video Gen form's preview (#11317): the generate body as the form would
@@ -1081,6 +1089,7 @@ const ACTIVE_JOB_PARAM_FIELDS = [
// 'grok' discriminator for them) and the clip duration — both plain
// values, safe to echo for the reloading page's form restore.
'videoMode', 'duration',
+ 'targetAspectRatio', 'ratio', 'bodyControl', 'expressionIntensity',
// reactor.inc jobs (#6214): the clip to chain from, the clip length, and the
// fast-h3 session canvas — all plain scalars (no filesystem path), safe to
// echo for the reloading page's form restore. `seed` above already covers
diff --git a/server/routes/videoGen.test.js b/server/routes/videoGen.test.js
index 44e63e757a..a083bcfc36 100644
--- a/server/routes/videoGen.test.js
+++ b/server/routes/videoGen.test.js
@@ -12,6 +12,8 @@ const compiledVisual = vi.hoisted(() => ({
}));
const probeVideoDuration = vi.hoisted(() => vi.fn(async () => 41.041281));
const probeVideoGeometry = vi.hoisted(() => vi.fn());
+const validateRunwayVideoInput = vi.hoisted(() => vi.fn(async () => ({ durationSec: 4 })));
+vi.mock('../services/videoGen/runwayVideoInputs.js', () => ({ validateRunwayVideoInput }));
vi.mock('../lib/ffmpeg.js', async (importOriginal) => ({
...(await importOriginal()),
probeVideoDuration,
@@ -876,6 +878,45 @@ describe('videoGen routes', () => {
imageGen: { grok: { enabled: true, grokPath: '/example/grok', aspectRatio: '16:9' } },
videoGen: { fal: { apiKey: 'example-fal-key' }, reactor: { apiKey: 'example-reactor-key' } },
};
+ it.each(['aleph2', 'act_two'])('stages %s inputs durably and preserves the workflow in queued params', async (modelId) => {
+ const { getSettings } = await import('../services/settings.js');
+ getSettings.mockResolvedValueOnce({ videoGen: { runway: { apiKey: 'example-key' } } });
+ setPendingUpload({ fieldname: 'runwayVideo', path: '/tmp/performance.mp4', originalname: 'performance.mp4' });
+ const mode = modelId === 'aleph2' ? 'video-edit' : 'character-performance';
+ const response = await request(app).post('/api/video-gen/').send({ backend: 'runway', mode, runwayModelId: modelId, prompt: '',
+ ...(modelId === 'act_two' ? { sourceImageFile: 'character.png', runwayBodyControl: 'false', runwayExpressionIntensity: '4' } : { runwayTargetAspect: '9:16' }),
+ });
+ expect(response.status).toBe(200);
+ const params = mediaJobQueue.enqueueJob.mock.calls[0][0].params;
+ expect(params).toMatchObject({ mode: 'runway', videoMode: mode, modelId, runwayVideoPath: expect.stringContaining('video-runway-video-') });
+ expect(params.uploadedTempPaths).toContain(params.runwayVideoPath);
+ expect(validateRunwayVideoInput).toHaveBeenCalledWith(params.runwayVideoPath, expect.objectContaining({ id: modelId }));
+ expect(JSON.stringify(params)).not.toContain('example-key');
+ });
+
+ it('rolls back the durable video if validation fails and refuses a video workflow on another backend', async () => {
+ const { getSettings } = await import('../services/settings.js');
+ getSettings.mockResolvedValueOnce({ videoGen: { runway: { apiKey: 'example-key' } } });
+ validateRunwayVideoInput.mockRejectedValueOnce(Object.assign(new Error('Video is too long'), { status: 400 }));
+ setPendingUpload({ fieldname: 'runwayVideo', path: '/tmp/source.mp4', originalname: 'source.mp4' });
+ const response = await request(app).post('/api/video-gen/').send({ backend: 'runway', mode: 'video-edit', runwayModelId: 'aleph2', prompt: '' });
+ expect(response.status).toBe(400);
+ expect(unlink).toHaveBeenCalledWith(copyFile.mock.calls[0][1]);
+ expect(mediaJobQueue.enqueueJob).not.toHaveBeenCalled();
+ const mismatch = await request(app).post('/api/video-gen/').send({ backend: 'local', mode: 'character-performance', prompt: 'test' });
+ expect(mismatch.status).toBe(400);
+ });
+
+ it('does not silently drop an unresolved Aleph guidance image', async () => {
+ const { getSettings } = await import('../services/settings.js');
+ getSettings.mockResolvedValueOnce({ videoGen: { runway: { apiKey: 'example-key' } } });
+ resolveGalleryImage.mockReturnValueOnce(null);
+ setPendingUpload({ fieldname: 'runwayVideo', path: '/tmp/source.mp4', originalname: 'source.mp4' });
+ const response = await request(app).post('/api/video-gen/').send({ backend: 'runway', mode: 'video-edit', runwayModelId: 'aleph2', prompt: '', sourceImageFile: 'missing.png' });
+ expect(response.status).toBe(400);
+ expect(mediaJobQueue.enqueueJob).not.toHaveBeenCalled();
+ expect(copyFile).not.toHaveBeenCalled();
+ });
const providers = [
{ backend: 'grok', input: { grokDuration: 10 },
minimal: { grokPath: '/example/grok', aspectRatio: '16:9', width: undefined, height: undefined, duration: undefined },
diff --git a/server/services/fixtures/runwayOpenApi.json b/server/services/fixtures/runwayOpenApi.json
index 96977ed5c5..52c439b388 100644
--- a/server/services/fixtures/runwayOpenApi.json
+++ b/server/services/fixtures/runwayOpenApi.json
@@ -781,6 +781,522 @@
}
}
}
+ },
+ "/v1/video_to_video": {
+ "post": {
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "oneOf": [
+ {
+ "type": "object",
+ "properties": {
+ "promptText": {
+ "description": "A non-empty and optional string describing what should appear in the output.",
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 1000
+ },
+ "keyframes": {
+ "description": "Timed guidance images placed at specific points in the input video.\nUp to 5 keyframes.",
+ "title": "Keyframes",
+ "minItems": 1,
+ "maxItems": 5,
+ "type": "array",
+ "items": {
+ "title": "Keyframe",
+ "x-stainless-variantName": "Keyframe",
+ "anyOf": [
+ {
+ "title": "KeyframeWithSeconds",
+ "x-stainless-variantName": "KeyframeWithSeconds",
+ "type": "object",
+ "properties": {
+ "uri": {
+ "description": "A HTTPS URL, Runway or data URI containing an encoded image.\nSee [our docs](/assets/inputs#images) on image inputs for more information.",
+ "example": "https://example.com/image.jpg",
+ "anyOf": [
+ {
+ "description": "A HTTPS URL, Runway upload URI, or base64 data URI (e.g. `data:image/png;base64,...`, up to 5MB) containing an encoded image. See [our docs](/assets/inputs#images) on image inputs for more information.",
+ "example": "https://example.com/image.jpg",
+ "type": "string",
+ "minLength": 13,
+ "maxLength": 2048,
+ "pattern": "^https:\\/\\/.*"
+ },
+ {
+ "description": "A Runway upload URI.\nSee https://docs.dev.runwayml.com/assets/uploads for more information.",
+ "example": "runway://<token>",
+ "type": "string",
+ "minLength": 13,
+ "maxLength": 5000,
+ "pattern": "^runway:\\/\\/.*"
+ },
+ {
+ "description": "A data URI containing encoded media, up to 5MB.",
+ "example": "data:image/jpeg;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",
+ "type": "string",
+ "minLength": 13,
+ "maxLength": 5242880,
+ "pattern": "^data:image\\/.*"
+ }
+ ]
+ },
+ "seconds": {
+ "description": "Absolute timestamp in seconds from the start of the input video when this guidance image should apply.",
+ "type": "number",
+ "minimum": 0,
+ "maximum": 30
+ },
+ "range": {
+ "description": "Optional edit window.\nWhen set, the edit applies only to this time range and the keyframe timestamp must fall within it.\nAll keyframes must either set a range or none may.",
+ "title": "KeyframeRange",
+ "type": "object",
+ "properties": {
+ "start_seconds": {
+ "description": "Start of the edit window in whole seconds from the start of the input video.",
+ "type": "integer",
+ "minimum": 0,
+ "maximum": 9007199254740991
+ },
+ "end_seconds": {
+ "description": "End of the edit window (exclusive) in whole seconds from the start of the input video.",
+ "type": "integer",
+ "exclusiveMinimum": 0,
+ "maximum": 9007199254740991
+ }
+ },
+ "required": [
+ "start_seconds",
+ "end_seconds"
+ ],
+ "additionalProperties": false
+ }
+ },
+ "required": [
+ "uri",
+ "seconds"
+ ],
+ "additionalProperties": false
+ },
+ {
+ "title": "KeyframeWithAt",
+ "x-stainless-variantName": "KeyframeWithAt",
+ "type": "object",
+ "properties": {
+ "uri": {
+ "description": "A HTTPS URL, Runway or data URI containing an encoded image.\nSee [our docs](/assets/inputs#images) on image inputs for more information.",
+ "example": "https://example.com/image.jpg",
+ "anyOf": [
+ {
+ "description": "A HTTPS URL, Runway upload URI, or base64 data URI (e.g. `data:image/png;base64,...`, up to 5MB) containing an encoded image. See [our docs](/assets/inputs#images) on image inputs for more information.",
+ "example": "https://example.com/image.jpg",
+ "type": "string",
+ "minLength": 13,
+ "maxLength": 2048,
+ "pattern": "^https:\\/\\/.*"
+ },
+ {
+ "description": "A Runway upload URI.\nSee https://docs.dev.runwayml.com/assets/uploads for more information.",
+ "example": "runway://<token>",
+ "type": "string",
+ "minLength": 13,
+ "maxLength": 5000,
+ "pattern": "^runway:\\/\\/.*"
+ },
+ {
+ "description": "A data URI containing encoded media, up to 5MB.",
+ "example": "data:image/jpeg;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",
+ "type": "string",
+ "minLength": 13,
+ "maxLength": 5242880,
+ "pattern": "^data:image\\/.*"
+ }
+ ]
+ },
+ "at": {
+ "description": "Position as a fraction [0.0, 1.0] of the input video duration when this guidance image should apply.",
+ "type": "number",
+ "minimum": 0,
+ "maximum": 1
+ },
+ "range": {
+ "description": "Optional edit window.\nWhen set, the edit applies only to this time range and the keyframe timestamp must fall within it.\nAll keyframes must either set a range or none may.",
+ "title": "KeyframeRange",
+ "type": "object",
+ "properties": {
+ "start_seconds": {
+ "description": "Start of the edit window in whole seconds from the start of the input video.",
+ "type": "integer",
+ "minimum": 0,
+ "maximum": 9007199254740991
+ },
+ "end_seconds": {
+ "description": "End of the edit window (exclusive) in whole seconds from the start of the input video.",
+ "type": "integer",
+ "exclusiveMinimum": 0,
+ "maximum": 9007199254740991
+ }
+ },
+ "required": [
+ "start_seconds",
+ "end_seconds"
+ ],
+ "additionalProperties": false
+ }
+ },
+ "required": [
+ "uri",
+ "at"
+ ],
+ "additionalProperties": false
+ }
+ ]
+ }
+ },
+ "videoUri": {
+ "description": "The input video to edit.\nMust be 30 seconds or shorter.",
+ "example": "https://example.com/video.mp4",
+ "anyOf": [
+ {
+ "description": "A HTTPS URL, Runway upload URI, or base64 data URI (e.g. `data:video/mp4;base64,...`, up to 5MB) containing an encoded video. See [our docs](/assets/inputs#videos) on video inputs for more information.",
+ "example": "https://example.com/video.mp4",
+ "type": "string",
+ "minLength": 13,
+ "maxLength": 2048,
+ "pattern": "^https:\\/\\/.*"
+ },
+ {
+ "description": "A Runway upload URI.\nSee https://docs.dev.runwayml.com/assets/uploads for more information.",
+ "example": "runway://<token>",
+ "type": "string",
+ "minLength": 13,
+ "maxLength": 5000,
+ "pattern": "^runway:\\/\\/.*"
+ },
+ {
+ "description": "A data URI containing encoded media, up to 5MB.",
+ "example": "data:image/jpeg;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",
+ "type": "string",
+ "minLength": 13,
+ "maxLength": 5242880,
+ "pattern": "^data:video\\/.*"
+ }
+ ]
+ },
+ "seed": {
+ "description": "If unspecified, a random number is chosen.\nVarying the seed integer is a way to get different results for the same other request parameters. Using the same seed integer for an identical request will produce similar results.",
+ "type": "integer",
+ "minimum": 0,
+ "maximum": 4294967295
+ },
+ "ratio": {
+ "deprecated": true,
+ "type": "string"
+ },
+ "targetAspectRatio": {
+ "description": "Target aspect ratio for expand/outpaint.\nLetterboxes the input video and keyframes before generation.",
+ "type": "string",
+ "enum": [
+ "16:9",
+ "4:3",
+ "3:2",
+ "1:1",
+ "2:3",
+ "3:4",
+ "9:16",
+ "21:9"
+ ]
+ },
+ "contentModeration": {
+ "description": "Settings that affect the behavior of the content moderation system.",
+ "type": "object",
+ "properties": {
+ "publicFigureThreshold": {
+ "description": "When set to `low`, the content moderation system will be less strict about preventing generations that include recognizable public figures.",
+ "type": "string",
+ "enum": [
+ "auto",
+ "low"
+ ]
+ }
+ },
+ "additionalProperties": false
+ },
+ "outputFormat": {
+ "description": "The container/encoding of the output.\n`mp4` (default) returns an H.264 .mp4. `prores` returns a ProRes .mov. `png_sequence` returns a .zip of PNG frames. `sdr_rec709_10bit` returns a 10-bit Rec.709 HEVC .mp4 for SDR grading pipelines.\nNon-mp4 formats incur an additional surcharge: 5 credits per second for `prores` and `png_sequence`, and 20 credits per second for `sdr_rec709_10bit` — 40 credits per second when the output is larger than 4 megapixels — includes anything larger than 1440p, up through 4K.",
+ "type": "string",
+ "enum": [
+ "mp4",
+ "prores",
+ "png_sequence",
+ "sdr_rec709_10bit"
+ ]
+ },
+ "proresProfile": {
+ "description": "The ProRes profile to use. Only valid when `outputFormat` is `prores`. Defaults to `4444`.\nNote: generated content contains no transparency — the alpha channel in `4444` / `4444 XQ` outputs is present but fully opaque; choose these tiers for 12-bit 4:4:4 color fidelity, not for mattes.",
+ "type": "string",
+ "enum": [
+ "422",
+ "4444",
+ "422 Proxy",
+ "422 LT",
+ "422 HQ",
+ "4444 XQ"
+ ]
+ },
+ "model": {
+ "type": "string",
+ "const": "aleph2"
+ }
+ },
+ "required": [
+ "videoUri",
+ "model"
+ ],
+ "additionalProperties": false
+ }
+ ]
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/character_performance": {
+ "post": {
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "oneOf": [
+ {
+ "title": "act_two",
+ "type": "object",
+ "properties": {
+ "seed": {
+ "description": "If unspecified, a random number is chosen.\nVarying the seed integer is a way to get different results for the same other request parameters. Using the same seed integer for an identical request will produce similar results.",
+ "type": "integer",
+ "minimum": 0,
+ "maximum": 4294967295
+ },
+ "character": {
+ "description": "The character to control. You can either provide a video or an image. A visually recognizable face must be visible and stay within the frame.",
+ "oneOf": [
+ {
+ "title": "CharacterImage",
+ "description": "An image of your character.\nIn the output, the character will use the reference video performance in its original static environment.",
+ "type": "object",
+ "properties": {
+ "type": {
+ "type": "string",
+ "const": "image"
+ },
+ "uri": {
+ "description": "A HTTPS URL, Runway or data URI containing an encoded image. See [our docs](/assets/inputs#images) on image inputs for more information.",
+ "example": "https://example.com/image.jpg",
+ "anyOf": [
+ {
+ "description": "A HTTPS URL, Runway upload URI, or base64 data URI (e.g.\n`data:image/png;base64,...`, up to 5MB) containing an encoded image. See [our docs](/assets/inputs#images) on image inputs for more information.",
+ "example": "https://example.com/image.jpg",
+ "type": "string",
+ "minLength": 13,
+ "maxLength": 2048,
+ "pattern": "^https:\\/\\/.*"
+ },
+ {
+ "description": "A Runway upload URI.\nSee https://docs.dev.runwayml.com/assets/uploads for more information.",
+ "example": "runway://<token>",
+ "type": "string",
+ "minLength": 13,
+ "maxLength": 5000,
+ "pattern": "^runway:\\/\\/.*"
+ },
+ {
+ "description": "A data URI containing encoded media, up to 5MB.",
+ "example": "data:image/jpeg;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",
+ "type": "string",
+ "minLength": 13,
+ "maxLength": 5242880,
+ "pattern": "^data:image\\/.*"
+ }
+ ]
+ }
+ },
+ "required": [
+ "type",
+ "uri"
+ ]
+ },
+ {
+ "title": "CharacterVideo",
+ "description": "A video of your character.\nIn the output, the character will use the reference video performance in its original animated environment and some of the character's own movements.",
+ "type": "object",
+ "properties": {
+ "type": {
+ "type": "string",
+ "const": "video"
+ },
+ "uri": {
+ "description": "A HTTPS URL, Runway or data URI containing an encoded video. See [our docs](/assets/inputs#videos) on video inputs for more information.",
+ "example": "https://example.com/video.mp4",
+ "anyOf": [
+ {
+ "description": "A HTTPS URL, Runway upload URI, or base64 data URI (e.g.\n`data:video/mp4;base64,...`, up to 5MB) containing an encoded video. See [our docs](/assets/inputs#videos) on video inputs for more information.",
+ "example": "https://example.com/video.mp4",
+ "type": "string",
+ "minLength": 13,
+ "maxLength": 2048,
+ "pattern": "^https:\\/\\/.*"
+ },
+ {
+ "description": "A Runway upload URI.\nSee https://docs.dev.runwayml.com/assets/uploads for more information.",
+ "example": "runway://<token>",
+ "type": "string",
+ "minLength": 13,
+ "maxLength": 5000,
+ "pattern": "^runway:\\/\\/.*"
+ },
+ {
+ "description": "A data URI containing encoded media, up to 5MB.",
+ "example": "data:image/jpeg;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",
+ "type": "string",
+ "minLength": 13,
+ "maxLength": 5242880,
+ "pattern": "^data:video\\/.*"
+ }
+ ]
+ }
+ },
+ "required": [
+ "type",
+ "uri"
+ ]
+ }
+ ],
+ "discriminator": {
+ "propertyName": "type"
+ }
+ },
+ "reference": {
+ "description": "The reference video containing the performance to apply to the character.",
+ "example": {
+ "type": "video",
+ "uri": "https://example.com/reference-performance.mp4"
+ },
+ "oneOf": [
+ {
+ "title": "CharacterReferenceVideo",
+ "description": "A video of a person performing in the manner that you would like your character to perform.\nThe video must be between 3 and 30 seconds in duration.",
+ "type": "object",
+ "properties": {
+ "type": {
+ "type": "string",
+ "const": "video"
+ },
+ "uri": {
+ "description": "A video of a person performing in the manner that you would like your character to perform. The video must be between 3 and 30 seconds in duration.\nSee [our docs](/assets/inputs#videos) on video inputs for more information.",
+ "example": "https://example.com/reference-performance.mp4",
+ "anyOf": [
+ {
+ "description": "A HTTPS URL, Runway upload URI, or base64 data URI (e.g. `data:video/mp4;base64,...`, up to 5MB) containing an encoded video.\nSee [our docs](/assets/inputs#videos) on video inputs for more information.",
+ "example": "https://example.com/video.mp4",
+ "type": "string",
+ "minLength": 13,
+ "maxLength": 2048,
+ "pattern": "^https:\\/\\/.*"
+ },
+ {
+ "description": "A Runway upload URI.\nSee https://docs.dev.runwayml.com/assets/uploads for more information.",
+ "example": "runway://<token>",
+ "type": "string",
+ "minLength": 13,
+ "maxLength": 5000,
+ "pattern": "^runway:\\/\\/.*"
+ },
+ {
+ "description": "A data URI containing encoded media, up to 5MB.",
+ "example": "data:image/jpeg;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",
+ "type": "string",
+ "minLength": 13,
+ "maxLength": 5242880,
+ "pattern": "^data:video\\/.*"
+ }
+ ]
+ }
+ },
+ "required": [
+ "type",
+ "uri"
+ ]
+ }
+ ],
+ "discriminator": {
+ "propertyName": "type"
+ }
+ },
+ "bodyControl": {
+ "description": "A boolean indicating whether to enable body control.\nWhen enabled, non-facial movements and gestures will be applied to the character in addition to facial expressions.",
+ "example": true,
+ "type": "boolean"
+ },
+ "expressionIntensity": {
+ "description": "An integer between 1 and 5 (inclusive).\nA larger value increases the intensity of the character's expression.",
+ "example": 3,
+ "default": 3,
+ "type": "integer",
+ "minimum": 1,
+ "maximum": 5
+ },
+ "ratio": {
+ "description": "The resolution of the output video.",
+ "example": "1280:720",
+ "type": "string",
+ "enum": [
+ "1280:720",
+ "720:1280",
+ "960:960",
+ "1104:832",
+ "832:1104",
+ "1584:672"
+ ]
+ },
+ "contentModeration": {
+ "description": "Settings that affect the behavior of the content moderation system.",
+ "example": {
+ "publicFigureThreshold": "auto"
+ },
+ "type": "object",
+ "properties": {
+ "publicFigureThreshold": {
+ "description": "When set to `low`, the content moderation system will be less strict about preventing generations that include recognizable public figures.",
+ "type": "string",
+ "enum": [
+ "auto",
+ "low"
+ ]
+ }
+ },
+ "additionalProperties": false
+ },
+ "model": {
+ "type": "string",
+ "const": "act_two"
+ }
+ },
+ "required": [
+ "character",
+ "reference",
+ "model"
+ ]
+ }
+ ]
+ }
+ }
+ }
+ }
+ }
}
}
-}
\ No newline at end of file
+}
diff --git a/server/services/runwayApi.js b/server/services/runwayApi.js
index 0df950bfe7..e7c9c71e89 100644
--- a/server/services/runwayApi.js
+++ b/server/services/runwayApi.js
@@ -17,6 +17,8 @@ import { assertRunwayCatalogModel } from './runwayModelCatalog.js';
import { maintenance } from '../lib/maintenanceAdmission.js';
import { ServerError } from '../lib/errorHandler.js';
import { fetchWithTimeout } from '../lib/fetchWithTimeout.js';
+import { openAsBlob } from 'node:fs';
+import { basename } from 'node:path';
export const RUNWAY_API_BASE = 'https://api.dev.runwayml.com';
// Every request names the API version it was written against.
@@ -41,6 +43,33 @@ const headers = (apiKey, json = false) => ({
...(json ? { 'Content-Type': 'application/json' } : {}),
});
+/** Upload just before dispatch: ephemeral URIs expire after 24h and never enter queue params. */
+export async function uploadRunwayVideo({ apiKey, filePath, signal }) {
+ signal?.throwIfAborted();
+ const file = await openAsBlob(filePath);
+ if (file.size < 512 || file.size > 200 * 1024 * 1024) throw new Error('Runway uploads must be 512 bytes–200 MB.');
+ const registration = await fetchWithTimeout(`${RUNWAY_API_BASE}/v1/uploads`, {
+ method: 'POST', headers: headers(apiKey, true), signal,
+ body: JSON.stringify({ filename: basename(filePath), type: 'ephemeral' }),
+ }, RUNWAY_SUBMIT_TIMEOUT_MS);
+ if (!registration.ok) throw new Error(`Runway upload registration failed: HTTP ${registration.status}`);
+ const data = await registration.json();
+ if (typeof data.uploadUrl !== 'string' || !data.uploadUrl.startsWith('https://')
+ || typeof data.runwayUri !== 'string' || !data.runwayUri.startsWith('runway://')
+ || !data.fields || typeof data.fields !== 'object' || Array.isArray(data.fields)
+ || Object.values(data.fields).some((value) => typeof value !== 'string')) throw new Error('Invalid Runway upload registration.');
+ signal?.throwIfAborted();
+ const form = new FormData();
+ for (const [key, value] of Object.entries(data.fields)) form.append(key, value);
+ form.append('file', file, basename(filePath));
+ // The signed storage fields authorize this POST. Never send the API key to storage.
+ const uploaded = await fetchWithTimeout(data.uploadUrl, { method: 'POST', body: form, signal }, 10 * 60 * 1000);
+ await uploaded.body?.cancel?.();
+ if (!uploaded.ok) throw new Error(`Runway media upload failed: HTTP ${uploaded.status}`);
+ signal?.throwIfAborted();
+ return data.runwayUri;
+}
+
/** A short, single-line reason from a Runway error body (`error`, `issues`), or ''. */
function runwayErrorDetail(payload) {
if (!payload || typeof payload !== 'object') return '';
diff --git a/server/services/runwayModelCatalog.js b/server/services/runwayModelCatalog.js
index 342669923f..b454d56249 100644
--- a/server/services/runwayModelCatalog.js
+++ b/server/services/runwayModelCatalog.js
@@ -4,6 +4,7 @@ import { createHash } from 'node:crypto';
import { PATHS } from '../lib/paths.js';
import { fetchWithTimeout } from '../lib/fetchWithTimeout.js';
import { RUNWAY_CATALOG_SNAPSHOT } from '../lib/runwayCatalogSnapshot.js';
+import { runwaySpecialSchemaContract } from '../lib/runwaySpecialVideoModels.js';
export const RUNWAY_CATALOG_SOURCE = 'https://docs.dev.runwayml.com/openapi.json';
const TTL = 24 * 60 * 60 * 1000;
@@ -11,12 +12,10 @@ const RETRY = 5 * 60 * 1000;
// A new shipped snapshot invalidates even a fresh disk cache from an older install.
const VERSION = createHash('sha256').update(JSON.stringify(RUNWAY_CATALOG_SNAPSHOT)).digest('hex');
const fallback = Object.fromEntries(Object.entries(RUNWAY_CATALOG_SNAPSHOT).map(([endpoint, rows]) => [endpoint, rows.map(({ id }) => id)]));
-const limitations = [
- { id: 'aleph2', reason: 'Requires video-to-video editing inputs; not supported by these text/image forms.' },
- { id: 'act_two', reason: 'Requires character and reference performance inputs; not supported by these forms.' },
-];
+const limitations = [];
function compatibleSchema(row, expected) {
+ if (expected.contract) return JSON.stringify(runwaySpecialSchemaContract(row)) === JSON.stringify(expected.contract);
const p = row.properties;
const required = row.required || [];
const allowed = ['model', 'promptText', 'ratio', 'duration', 'promptImage', 'referenceImages'];
@@ -67,7 +66,7 @@ function parseCatalog(spec) {
/** Public documentation, never an invented /models API or an authenticated generation probe.
* One bounded, machine-local disposable cache; no user data, credentials or federation.
*/
-export function createRunwayModelCatalog({
+export function _createRunwayModelCatalog({
fetchSchema = () => fetchWithTimeout(RUNWAY_CATALOG_SOURCE, {}, 15_000),
cachePath = join(PATHS.installRoot, '.cache', 'runway-model-catalog.json'), now = Date.now,
} = {}) {
@@ -127,7 +126,7 @@ export function createRunwayModelCatalog({
},
};
}
-const catalog = createRunwayModelCatalog();
+const catalog = _createRunwayModelCatalog();
export const getRunwayModelCatalog = (options) => catalog.get(options);
/** Refreshed removals/schema changes fail before a paid submission. No catalog lookup per render. */
diff --git a/server/services/runwayModelCatalog.test.js b/server/services/runwayModelCatalog.test.js
index 7690312e06..4d66280b97 100644
--- a/server/services/runwayModelCatalog.test.js
+++ b/server/services/runwayModelCatalog.test.js
@@ -2,7 +2,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest';
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { tmpdir } from 'node:os';
-import { createRunwayModelCatalog } from './runwayModelCatalog.js';
+import { _createRunwayModelCatalog } from './runwayModelCatalog.js';
// Real public OpenAPI excerpts (2026-10-22), descriptions removed; no generation endpoint is called.
const fixture = JSON.parse(await readFile(new URL('./fixtures/runwayOpenApi.json', import.meta.url), 'utf8'));
@@ -15,7 +15,7 @@ async function setup() {
let time = 1_000_000;
const fetchSchema = vi.fn(async () => ({ ok: true, json: async () => structuredClone(fixture) }));
const options = { cachePath, fetchSchema, now: () => time };
- return { cache: createRunwayModelCatalog(options), options, fetchSchema, advance: (ms) => { time += ms; } };
+ return { cache: _createRunwayModelCatalog(options), options, fetchSchema, advance: (ms) => { time += ms; } };
}
describe('Runway public catalog cache', () => {
it('coalesces form loads, survives restart, expires and honors explicit refresh/removals', async () => {
@@ -24,9 +24,11 @@ describe('Runway public catalog cache', () => {
expect(first).toEqual(concurrent);
expect(first.available.text_to_image).toEqual(['gen4_image', 'gpt_image_2']);
expect(first.available.text_to_video).not.toContain('gen4_turbo');
+ expect(first.available.video_to_video).toEqual(['aleph2']);
+ expect(first.available.character_performance).toEqual(['act_two']);
expect(fetchSchema).toHaveBeenCalledTimes(1);
await cache.get();
- await createRunwayModelCatalog(options).get();
+ await _createRunwayModelCatalog(options).get();
expect(fetchSchema).toHaveBeenCalledTimes(1);
advance(24 * 60 * 60 * 1000);
await cache.get();
@@ -63,7 +65,7 @@ describe('Runway public catalog cache', () => {
expect(fetchSchema).toHaveBeenCalledTimes(1);
await writeFile(options.cachePath, JSON.stringify({ version: 'old', fetchedAt: 1_000_000, catalog: { available: {} } }));
fetchSchema.mockResolvedValue({ ok: true, json: async () => fixture });
- expect(await createRunwayModelCatalog(options).get()).toMatchObject({ bundled: false, stale: false });
+ expect(await _createRunwayModelCatalog(options).get()).toMatchObject({ bundled: false, stale: false });
});
it('does not claim unknown or changed request schemas are runnable and preserves forced refresh during disk load', async () => {
const { cache, options, fetchSchema } = await setup();
@@ -71,9 +73,10 @@ describe('Runway public catalog cache', () => {
const changed = structuredClone(fixture);
const rows = changed.paths['/v1/text_to_image'].post.requestBody.content['application/json'].schema.oneOf;
delete rows[0].properties.referenceImages.items.properties.tag;
+ changed.paths['/v1/character_performance'].post.requestBody.content['application/json'].schema.oneOf[0].properties.expressionIntensity.maximum = 4;
rows.push({ properties: { model: { const: 'future_image' } } });
fetchSchema.mockResolvedValue({ ok: true, json: async () => changed });
- const restarted = createRunwayModelCatalog(options);
+ const restarted = _createRunwayModelCatalog(options);
const load = restarted.get();
const forced = restarted.get({ refresh: true });
const forcedAgain = restarted.get({ refresh: true });
@@ -82,6 +85,7 @@ describe('Runway public catalog cache', () => {
expect(await forcedAgain).toEqual(updated);
expect(fetchSchema).toHaveBeenCalledTimes(2);
expect(updated.available.text_to_image).toEqual(['gpt_image_2']);
- expect(updated.unsupported.map((r) => r.id)).toEqual(expect.arrayContaining(['gen4_image', 'future_image', 'aleph2', 'act_two']));
+ expect(updated.available.character_performance).toEqual([]);
+ expect(updated.unsupported.map((r) => r.id)).toEqual(expect.arrayContaining(['gen4_image', 'future_image']));
});
});
diff --git a/server/services/videoGen/hostedSubmission.js b/server/services/videoGen/hostedSubmission.js
index 5c39c97556..5094b392c4 100644
--- a/server/services/videoGen/hostedSubmission.js
+++ b/server/services/videoGen/hostedSubmission.js
@@ -10,6 +10,8 @@ import { FAL_DEFAULT_IMAGE_VIDEO_MODEL, FAL_DEFAULT_TEXT_VIDEO_MODEL, getFalVide
import { RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL, getRunwayVideoModel, runwayModelTakesText } from '../../lib/runwayVideoModels.js';
import { isDefaultI2vReferenceMode } from '../../lib/videoReferenceModes.js';
import { VIDEO_GEN_MODE, isVideoModeUsable } from './modes.js';
+import { getRunwaySpecialVideoModel, buildRunwaySpecialVideoRequest } from '../../lib/runwaySpecialVideoModels.js';
+import { validateRunwayVideoInput } from './runwayVideoInputs.js';
// Every hosted backend anchors a reference image as frame one, so a loose
// reference mode is a local-runtime promise it can never keep. Submission
@@ -68,6 +70,24 @@ async function stageRunwayInputs(ctx) {
throw new ServerError(message, { status: 400, code: 'VALIDATION_ERROR' });
};
if (uploads.audioFile) await refuse('Runway renders cutaways only; it takes no source audio.');
+ const special = getRunwaySpecialVideoModel(body.runwayModelId);
+ if (special) {
+ if ((uploads.sourceImage || body.sourceImageFile) && !sourceImagePath) await refuse('The requested Runway guidance or character image could not be resolved.');
+ buildRunwaySpecialVideoRequest({
+ modelId: special.id, videoMode: body.mode, prompt: body.prompt, negativePrompt: body.negativePrompt,
+ videoUri: uploads.runwayVideo ? 'pending:video' : null,
+ characterVideoUri: uploads.runwayCharacterVideo ? 'pending:character' : null,
+ imageUri: sourceImagePath, endImageUri: uploads.lastImage || body.lastImageFile,
+ bodyControl: body.runwayBodyControl, expressionIntensity: body.runwayExpressionIntensity,
+ targetAspectRatio: body.runwayTargetAspect, ratio: body.runwayRatio, seed: body.runwaySeed,
+ });
+ const runwayVideoPath = await ctx.stageUploadDurable(uploads.runwayVideo, 'runway-video');
+ await validateRunwayVideoInput(runwayVideoPath, special);
+ const runwayCharacterVideoPath = uploads.runwayCharacterVideo ? await ctx.stageUploadDurable(uploads.runwayCharacterVideo, 'runway-character') : null;
+ if (runwayCharacterVideoPath) await validateRunwayVideoInput(runwayCharacterVideoPath, special, { character: true });
+ return { runwayVideoPath, runwayCharacterVideoPath, uploadedTempPaths: [runwayVideoPath, runwayCharacterVideoPath].filter(Boolean) };
+ }
+ if (uploads.runwayVideo || uploads.runwayCharacterVideo || !['text', 'image', undefined].includes(body.mode)) await refuse('Choose the Runway model matching these video inputs.');
const model = getRunwayVideoModel(body.runwayModelId || RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL);
if (!model) await refuse(`"${body.runwayModelId}" is not a Runway model PortOS offers`);
if (!sourceImagePath && !runwayModelTakesText(model)) await refuse(`${model.label} renders from a start frame — supply a source image.`);
@@ -172,6 +192,10 @@ export const HOSTED_VIDEO_SUBMISSIONS = {
// The model, length and size are resolved against the curated catalog by videoGen/runway.js.
buildParams: (body, prepared) => ({
modelId: body.runwayModelId,
+ videoMode: body.mode,
+ ...(prepared.runwayVideoPath ? { runwayVideoPath: prepared.runwayVideoPath, runwayCharacterVideoPath: prepared.runwayCharacterVideoPath } : {}),
+ targetAspectRatio: body.runwayTargetAspect, ratio: body.runwayRatio,
+ bodyControl: body.runwayBodyControl, expressionIntensity: body.runwayExpressionIntensity, seed: body.runwaySeed,
...(prepared.lastImagePath ? { lastImagePath: prepared.lastImagePath } : {}),
...(prepared.uploadedTempPaths?.length ? { uploadedTempPaths: prepared.uploadedTempPaths } : {}),
aspectRatio: body.visualConditioning?.render?.parameters?.aspectRatio,
diff --git a/server/services/videoGen/prepareParams.js b/server/services/videoGen/prepareParams.js
index 7769a413ce..02770716fc 100644
--- a/server/services/videoGen/prepareParams.js
+++ b/server/services/videoGen/prepareParams.js
@@ -79,6 +79,7 @@ import { videoModeContractError, videoChainUnsupportedError, videoReferenceModeE
import { resolveByovRuntimeLoraCapable, videoLoraUnsupportedError } from './runtimes.js';
import { validateVideoBatch } from './batch.js';
import { audioDurationToFrames } from './audioDuration.js';
+import { RUNWAY_SPECIAL_VIDEO_MODES } from '../../lib/runwaySpecialVideoModels.js';
// Retries reuse persisted worker parameters instead of passing through the
// multipart preparation path. Keep the model/mode gates here so a model edit
@@ -286,6 +287,11 @@ export function resolveRequestedVideoBackend({ body, uploads, settings, localOnl
export async function prepareVideoGenParams({ body, uploads, localOnlyParamKeys }) {
const settings = await getSettings();
const backend = resolveRequestedVideoBackend({ body, uploads, settings, localOnlyParamKeys });
+ const runwaySpecial = backend === VIDEO_GEN_MODE.RUNWAY && RUNWAY_SPECIAL_VIDEO_MODES.includes(body.mode);
+ if (!runwaySpecial && (RUNWAY_SPECIAL_VIDEO_MODES.includes(body.mode) || uploads.runwayVideo || uploads.runwayCharacterVideo)) {
+ await cleanupMultipartTemp(uploads);
+ throw new ServerError('Runway video inputs require the matching Runway video workflow.', { status: 400, code: 'VALIDATION_ERROR' });
+ }
// A loose reference (#4874) is a LOCAL-runtime capability. The model gate
// further down asks whether the chosen model's runtime can honor it — but an
// explicit `backend: 'grok'` never reaches that runtime at all: grok's
@@ -427,7 +433,7 @@ export async function prepareVideoGenParams({ body, uploads, localOnlyParamKeys
// rejection point at once, and stays correct as new awaits are added.
return withStagedRollback(cleanupStaged, () => resolvePreparedParams({
body, uploads, settings, backend, pythonPath,
- effectiveModel, effectiveModelId, stagedDurablePaths, cleanupStaged, musicVideoContractMode,
+ effectiveModel: runwaySpecial ? null : effectiveModel, effectiveModelId, stagedDurablePaths, cleanupStaged, musicVideoContractMode,
}));
}
@@ -435,7 +441,7 @@ export async function prepareVideoGenParams({ body, uploads, localOnlyParamKeys
// try/catch so a permission/disk failure here still cleans up the
// multipart temp uploads instead of leaking them in the OS temp dir.
async function ensureUploadsDir({ uploads, cleanupStaged }) {
- if (!(uploads.sourceImage || uploads.lastImage || uploads.audioFile || uploads.icReference)) return;
+ if (!(uploads.sourceImage || uploads.lastImage || uploads.audioFile || uploads.icReference || uploads.runwayVideo || uploads.runwayCharacterVideo)) return;
try {
await ensureDir(PATHS.uploads);
} catch (err) {
diff --git a/server/services/videoGen/promptPreview.js b/server/services/videoGen/promptPreview.js
index 9689b21828..192fdc4ee6 100644
--- a/server/services/videoGen/promptPreview.js
+++ b/server/services/videoGen/promptPreview.js
@@ -18,7 +18,7 @@ import { VIDEO_GEN_LOCAL_ONLY_FIELD_NAMES } from './requestFields.js';
import { grokVideoSentPrompt } from './grok.js';
import { falVideoSentPrompt } from './fal.js';
-export const PREVIEW_UPLOAD_FIELDS = ['sourceImage', 'lastImage', 'audioFile', 'icReference'];
+export const PREVIEW_UPLOAD_FIELDS = ['sourceImage', 'lastImage', 'audioFile', 'icReference', 'runwayVideo', 'runwayCharacterVideo'];
const UPLOADED_SOURCE_IMAGE = '(uploaded image)';
const asWritten = (body, extra = {}) => ({
@@ -70,6 +70,9 @@ export async function previewVideoPrompt({ previewUploads = [], ...body }) {
const { runwayVideoSentPrompt } = await import('./runway.js');
return { backend, ...runwayVideoSentPrompt({
modelId: body.runwayModelId, prompt: body.prompt, negativePrompt: body.negativePrompt,
+ videoMode: body.mode, hasVideo: !!uploads.runwayVideo, hasCharacterVideo: !!uploads.runwayCharacterVideo,
+ targetAspectRatio: body.runwayTargetAspect, ratio: body.runwayRatio,
+ bodyControl: body.runwayBodyControl, expressionIntensity: body.runwayExpressionIntensity, seed: body.runwaySeed,
seconds: body.runwayDuration, width: body.width, height: body.height, resolution: body.runwayResolution, promptAdditions,
hasImage: hasSourceImage, hasEndImage: !!uploads.lastImage || !!body.lastImageFile,
}).record };
diff --git a/server/services/videoGen/runway.js b/server/services/videoGen/runway.js
index 47a9e34619..d5ea3d81e4 100644
--- a/server/services/videoGen/runway.js
+++ b/server/services/videoGen/runway.js
@@ -25,6 +25,8 @@ import { fetchWithTimeout } from '../../lib/fetchWithTimeout.js';
import { anyAbortSignal } from '../../lib/requestAbort.js';
import { withAbortTimeout } from '../../lib/abortTimeout.js';
import { RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL, buildRunwayVideoRequest } from '../../lib/runwayVideoModels.js';
+import { getRunwaySpecialVideoModel, buildRunwaySpecialVideoRequest } from '../../lib/runwaySpecialVideoModels.js';
+import { validateRunwayVideoInput } from './runwayVideoInputs.js';
import { probeVideoGeometry } from '../../lib/ffmpeg.js';
import { attachSseClient as attachSse, closeJobAfterDelay, createJobFailureFinalizer } from '../../lib/sseUtils.js';
import { videoGenEvents } from './events.js';
@@ -34,7 +36,7 @@ import { mutateVideoHistory } from './history.js';
import { getSettings } from '../settings.js';
import { frameDataUriAtAspect } from './falFrameAspect.js';
import {
- awaitRunwayTask, cancelRunwayTask, createRunwayTaskEntry, resolveRunwayApiKey, submitRunwayTask,
+ awaitRunwayTask, cancelRunwayTask, createRunwayTaskEntry, resolveRunwayApiKey, submitRunwayTask, uploadRunwayVideo,
} from '../runwayApi.js';
import { carriedPromptAdditions, promptAddition, sentPromptRecord } from '../../lib/modelInput.js';
@@ -91,11 +93,14 @@ export const cancelAll = () => {
* the plan and the record of the text Runway receives (#11317); the Video Gen
* form's preview calls it with the same inputs.
*/
-export function runwayVideoSentPrompt({ hasImage = false, hasEndImage = false, promptAdditions = [], ...spec }) {
+export function runwayVideoSentPrompt({ hasImage = false, hasEndImage = false, hasVideo = false, hasCharacterVideo = false, promptAdditions = [], ...spec }) {
let planned;
try {
- planned = buildRunwayVideoRequest({
+ const build = getRunwaySpecialVideoModel(spec.modelId) ? buildRunwaySpecialVideoRequest : buildRunwayVideoRequest;
+ planned = build({
...spec,
+ videoUri: hasVideo ? 'pending:video' : null,
+ characterVideoUri: hasCharacterVideo ? 'pending:character-video' : null,
imageUri: hasImage ? 'pending:image' : null,
endImageUri: hasEndImage ? 'pending:end-image' : null,
});
@@ -141,6 +146,7 @@ async function generateVideoPrepared({
sourceImagePath = null, lastImagePath = null, jobId: providedJobId = null,
uploadedTempPath = null, uploadedTempPaths = [], audioFilePath = null,
resolution = null, seed, frameAspect = null, shotInstruction = null, promptAdditions = [],
+ videoMode, runwayVideoPath = null, runwayCharacterVideoPath = null, targetAspectRatio, ratio, bodyControl, expressionIntensity,
}) {
await ensureDir(PATHS.videos);
const renderStartedAtMs = Date.now();
@@ -148,6 +154,8 @@ async function generateVideoPrepared({
throw new ServerError('Runway renders cutaways only; it takes no source audio', { status: 400, code: 'VALIDATION_ERROR' });
}
const modelId = requestedModelId || RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL;
+ const special = getRunwaySpecialVideoModel(modelId);
+ if (!special && (runwayVideoPath || runwayCharacterVideoPath)) throw new ServerError('This Runway model does not take video inputs.', { status: 400, code: 'VALIDATION_ERROR' });
const effectiveSettings = settings || (providedApiKey ? null : await getSettings().catch(() => null));
// Re-resolved from live settings, never threaded through persisted job params.
const apiKey = providedApiKey || resolveRunwayApiKey(effectiveSettings);
@@ -156,10 +164,11 @@ async function generateVideoPrepared({
}
const requestSpec = {
modelId, prompt, negativePrompt, seconds: duration, aspectRatio: frameAspect || aspectRatio, width, height, resolution,
+ videoMode, targetAspectRatio, ratio, bodyControl, expressionIntensity,
...(Number.isInteger(seed) ? { seed } : {}),
};
const { planned, record: sentRecord } = runwayVideoSentPrompt({
- ...requestSpec, promptAdditions, hasImage: !!sourceImagePath, hasEndImage: !!lastImagePath,
+ ...requestSpec, promptAdditions, hasImage: !!sourceImagePath, hasEndImage: !!lastImagePath, hasVideo: !!runwayVideoPath, hasCharacterVideo: !!runwayCharacterVideoPath,
});
const jobId = providedJobId || randomUUID();
@@ -178,7 +187,7 @@ async function generateVideoPrepared({
...sentRecord,
filename,
createdAt: new Date().toISOString(),
- mode: sourceImagePath ? 'image' : 'text',
+ mode: special?.mode || (sourceImagePath ? 'image' : 'text'),
...(shotInstruction?.audioWindow ? { audioWindow: shotInstruction.audioWindow } : {}),
};
const job = { ...meta, clients: [], status: 'running', renderStartedAtMs };
@@ -193,6 +202,7 @@ async function generateVideoPrepared({
runRunwayVideo(job, jobId, {
apiKey, requestSpec, sourceImagePath, lastImagePath, outputPath, filename, meta,
uploadedTempPath, uploadedTempPaths, frameAspect,
+ runwayVideoPath, runwayCharacterVideoPath,
}).catch((err) => {
console.error(`❌ Runway video run failed [${jobId.slice(0, 8)}]: ${err?.message}`);
});
@@ -221,6 +231,7 @@ async function downloadRunwayOutput(url, signal) {
async function runRunwayVideo(job, jobId, {
apiKey, requestSpec, sourceImagePath, lastImagePath, outputPath, filename, meta,
uploadedTempPath = null, uploadedTempPaths = [], frameAspect = null,
+ runwayVideoPath = null, runwayCharacterVideoPath = null,
}) {
const entry = createRunwayTaskEntry(apiKey);
entry.aborted = preparations.take(jobId);
@@ -246,6 +257,14 @@ async function runRunwayVideo(job, jobId, {
activeRequests.set(jobId, entry);
const deadline = Date.now() + RUNWAY_RENDER_TIMEOUT_MS;
try {
+ if (entry.aborted) return finishFailure('Canceled');
+ const special = getRunwaySpecialVideoModel(requestSpec.modelId);
+ if (special) {
+ const geometry = await validateRunwayVideoInput(runwayVideoPath, special);
+ if (runwayCharacterVideoPath) await validateRunwayVideoInput(runwayCharacterVideoPath, special, { character: true });
+ meta.duration = geometry.durationSec;
+ job.duration = geometry.durationSec;
+ }
const frame = sourceImagePath ? await runwayFrameDataUri(sourceImagePath, frameAspect) : null;
const endFrame = lastImagePath ? await runwayFrameDataUri(lastImagePath, frameAspect) : null;
for (const [label, fit] of [['start', frame], ['end', endFrame]]) {
@@ -255,7 +274,10 @@ async function runRunwayVideo(job, jobId, {
if (frame?.conformed || endFrame?.conformed) {
meta.frameConformed = { aspect: frameAspect, ...(frame?.conformed ? { start: frame.conformed } : {}), ...(endFrame?.conformed ? { end: endFrame.conformed } : {}) };
}
- const { body, endpoint } = buildRunwayVideoRequest({ ...requestSpec, imageUri: frame?.dataUri ?? null, endImageUri: endFrame?.dataUri ?? null });
+ const videoUri = runwayVideoPath ? await uploadRunwayVideo({ apiKey, filePath: runwayVideoPath, signal: entry.controller.signal }) : null;
+ const characterVideoUri = runwayCharacterVideoPath ? await uploadRunwayVideo({ apiKey, filePath: runwayCharacterVideoPath, signal: entry.controller.signal }) : null;
+ const build = special ? buildRunwaySpecialVideoRequest : buildRunwayVideoRequest;
+ const { body, endpoint } = build({ ...requestSpec, videoUri, characterVideoUri, imageUri: frame?.dataUri ?? null, endImageUri: endFrame?.dataUri ?? null });
if (entry.aborted) return finishFailure('Canceled');
entry.taskId = await submitRunwayTask({ apiKey, endpoint, body });
diff --git a/server/services/videoGen/runway.test.js b/server/services/videoGen/runway.test.js
index 9d7587f462..1542cc5d82 100644
--- a/server/services/videoGen/runway.test.js
+++ b/server/services/videoGen/runway.test.js
@@ -1,5 +1,5 @@
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
-import { mkdir, rm, readFile } from 'fs/promises';
+import { mkdir, rm, readFile, writeFile, stat } from 'fs/promises';
import { join } from 'path';
import { tmpdir } from 'os';
import sharp from 'sharp';
@@ -26,7 +26,7 @@ vi.mock('../../lib/ffmpeg.js', async () => {
...actual,
optimizeForStreaming: vi.fn(async () => {}),
generateThumbnail: vi.fn(async (_p, jobId) => `${jobId}.jpg`),
- probeVideoGeometry: vi.fn(async () => ({ duration: 4 })),
+ probeVideoGeometry: vi.fn(async () => ({ width: 1280, height: 720, fps: 24, durationSec: 4, numFrames: 96 })),
};
});
@@ -37,6 +37,7 @@ const runway = await import('./runway.js');
const { RUNWAY_API_VERSION } = await import('../runwayApi.js');
const { videoGenEvents } = await import('./events.js');
const { loadHistory } = await import('./history.js');
+const { probeVideoGeometry } = await import('../../lib/ffmpeg.js');
const terminalEvents = new Map();
const waitForTerminal = async (jobId) => {
@@ -60,6 +61,149 @@ beforeEach(async () => {
framePath = join(TEST_ROOT, 'frame.png');
await sharp({ create: { width: 64, height: 36, channels: 3, background: '#336699' } }).png().toFile(framePath);
getSettingsMock.mockReset().mockResolvedValue({});
+ probeVideoGeometry.mockReset().mockResolvedValue({ width: 1280, height: 720, fps: 24, durationSec: 4, numFrames: 96 });
+});
+
+describe('Runway video-input workflows', () => {
+ const stagedVideo = async () => {
+ await mkdir(join(TEST_ROOT, 'uploads'), { recursive: true });
+ const path = join(TEST_ROOT, 'uploads', 'source.mp4');
+ await writeFile(path, Buffer.alloc(1024));
+ return path;
+ };
+ it('cancels during preparation without uploading private inputs', async () => {
+ const video = await stagedVideo();
+ let releaseSettings;
+ getSettingsMock.mockImplementationOnce(() => new Promise((resolve) => { releaseSettings = resolve; }));
+ vi.stubGlobal('fetch', vi.fn());
+ const starting = runway.generateVideo({ jobId: 'preparation-cancel', modelId: 'aleph2', runwayVideoPath: video, uploadedTempPaths: [video] });
+ await vi.waitFor(() => expect(releaseSettings).toBeTypeOf('function'));
+ expect(runway.cancel('preparation-cancel')).toBe('pending');
+ releaseSettings({ videoGen: { runway: { apiKey: 'test-key' } } });
+ const job = await starting;
+ expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'failed' });
+ expect(fetch).not.toHaveBeenCalled();
+ await expect(stat(video)).rejects.toMatchObject({ code: 'ENOENT' });
+ });
+ it.each(['aleph2', 'act_two', 'act_two_video'])('uploads and publishes %s with the endpoint-specific fields, preserving input ownership', async (variant) => {
+ const video = await stagedVideo();
+ const modelId = variant === 'act_two_video' ? 'act_two' : variant;
+ const durationSec = variant === 'act_two' ? 3 : variant === 'act_two_video' ? 30 : 4;
+ probeVideoGeometry.mockResolvedValue({ width: 1280, height: 720, fps: 24, durationSec, numFrames: durationSec * 24 });
+ const calls = [];
+ let registrations = 0;
+ vi.stubGlobal('fetch', vi.fn(async (url, options = {}) => {
+ calls.push({ url, ...options });
+ if (url.endsWith('/v1/uploads')) return jsonResponse({ uploadUrl: 'https://storage.example/upload', fields: { key: 'signed-object', policy: 'signed-policy' }, runwayUri: `runway://upload-${++registrations}` });
+ if (url === 'https://storage.example/upload') return jsonResponse({}, true, 204);
+ if (url.endsWith('/v1/video_to_video') || url.endsWith('/v1/character_performance')) return jsonResponse({ id: 'task-1' });
+ if (url === TASKS) return jsonResponse({ status: 'SUCCEEDED', output: ['https://cdn.example.com/out.mp4'] });
+ if (url === 'https://cdn.example.com/out.mp4') return { ok: true, arrayBuffer: async () => Uint8Array.from([1, 2, 3]).buffer };
+ throw new Error(`Unexpected fetch ${url}`);
+ }));
+ const job = await runway.generateVideo({ apiKey: 'test-key', modelId, videoMode: modelId === 'aleph2' ? 'video-edit' : 'character-performance',
+ prompt: modelId === 'aleph2' ? 'Make it snow' : '', runwayVideoPath: video,
+ ...(variant === 'act_two_video' ? { runwayCharacterVideoPath: video } : { sourceImagePath: framePath }),
+ uploadedTempPaths: [video], bodyControl: false, expressionIntensity: 5,
+ });
+ expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'completed' });
+ const storage = calls.find((c) => c.url === 'https://storage.example/upload');
+ expect(storage.headers?.Authorization).toBeUndefined();
+ expect([...storage.body.keys()]).toEqual(['key', 'policy', 'file']);
+ expect(storage.body.get('file').size).toBe(1024);
+ const submitted = calls.find((c) => c.url.endsWith(modelId === 'aleph2' ? '/v1/video_to_video' : '/v1/character_performance'));
+ const body = JSON.parse(submitted.body);
+ expect(body).not.toHaveProperty('duration');
+ expect(body).not.toHaveProperty('audio');
+ if (modelId === 'aleph2') expect(body).toMatchObject({ model: modelId, videoUri: 'runway://upload-1', promptText: 'Make it snow', keyframes: [{ at: 0, uri: expect.stringMatching(/^data:image\//) }] });
+ else {
+ expect(body).not.toHaveProperty('promptText');
+ expect(body).toMatchObject({ model: modelId, character: { type: variant === 'act_two_video' ? 'video' : 'image' }, reference: { type: 'video', uri: 'runway://upload-1' }, bodyControl: false, expressionIntensity: 5 });
+ expect(registrations).toBe(variant === 'act_two_video' ? 2 : 1);
+ }
+ await expect(stat(video)).rejects.toMatchObject({ code: 'ENOENT' });
+ expect(await stat(framePath)).toBeTruthy();
+ expect((await loadHistory())[0]).toMatchObject({ modelId: `runway:${modelId}`, duration: durationSec, mode: modelId === 'aleph2' ? 'video-edit' : 'character-performance' });
+ });
+
+ it.each([
+ [{ modelId: 'aleph2' }, /source video/],
+ [{ modelId: 'act_two', runwayVideoPath: 'unused.mp4' }, /exactly one character/],
+ [{ modelId: 'act_two', runwayVideoPath: 'unused.mp4', sourceImagePath: 'unused.png', prompt: 'ignored text' }, /does not accept prompts/],
+ [{ modelId: 'aleph2', runwayVideoPath: 'unused.mp4', videoMode: 'text' }, /matching/],
+ [{ modelId: 'act_two', runwayVideoPath: 'unused.mp4', sourceImagePath: 'unused.png', expressionIntensity: 6 }, /intensity/],
+ [{ modelId: 'aleph2', runwayVideoPath: 'unused.mp4', seed: -1 }, /seed/],
+ ])('rejects incompatible inputs before upload', async (params, message) => {
+ vi.stubGlobal('fetch', vi.fn());
+ await expect(runway.generateVideo({ apiKey: 'test-key', ...params })).rejects.toMatchObject({ status: 400, message: expect.stringMatching(message) });
+ expect(fetch).not.toHaveBeenCalled();
+ });
+
+ it.each([
+ ['aleph2', { durationSec: 1.99, fps: 24 }], ['aleph2', { durationSec: 30.01, fps: 24 }],
+ ['aleph2', { durationSec: 4, fps: 31 }], ['act_two', { durationSec: 2.99, fps: 24 }], ['act_two', { durationSec: 30.01, fps: 24 }], ['act_two', null],
+ ])('rejects unreadable/out-of-range %s media and cleans staged input', async (modelId, geometry) => {
+ const video = await stagedVideo();
+ probeVideoGeometry.mockResolvedValue(geometry);
+ vi.stubGlobal('fetch', vi.fn());
+ const job = await runway.generateVideo({ apiKey: 'test-key', modelId, runwayVideoPath: video, sourceImagePath: framePath, uploadedTempPaths: [video] });
+ expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'failed' });
+ expect(fetch).not.toHaveBeenCalled();
+ await expect(stat(video)).rejects.toMatchObject({ code: 'ENOENT' });
+ });
+
+ it.each(['registration', 'storage', 'malformed'])('does not submit generation after %s failure', async (failure) => {
+ const video = await stagedVideo();
+ const calls = [];
+ vi.stubGlobal('fetch', vi.fn(async (url) => {
+ calls.push(url);
+ if (url.endsWith('/v1/uploads')) return failure === 'registration' ? jsonResponse({}, false, 503) : jsonResponse(failure === 'malformed' ? {} : { uploadUrl: 'https://storage.example/upload', fields: {}, runwayUri: 'runway://upload-1' });
+ return jsonResponse({}, false, 503);
+ }));
+ const job = await runway.generateVideo({ apiKey: 'test-key', modelId: 'aleph2', runwayVideoPath: video, uploadedTempPaths: [video] });
+ expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'failed', error: expect.stringMatching(/upload/i) });
+ expect(calls).toEqual(failure === 'storage' ? ['https://api.dev.runwayml.com/v1/uploads', 'https://storage.example/upload'] : ['https://api.dev.runwayml.com/v1/uploads']);
+ await expect(stat(video)).rejects.toMatchObject({ code: 'ENOENT' });
+ });
+
+ it.each(['registration', 'storage'])('cancels during %s without a paid task and cleans input', async (phase) => {
+ const video = await stagedVideo();
+ let uploading;
+ const started = new Promise((resolve) => { uploading = resolve; });
+ const calls = [];
+ vi.stubGlobal('fetch', vi.fn(async (url, options) => {
+ calls.push(url);
+ if (phase === 'storage' && url.endsWith('/v1/uploads')) return jsonResponse({ uploadUrl: 'https://storage.example/upload', fields: {}, runwayUri: 'runway://upload-1' });
+ uploading();
+ return new Promise((_resolve, reject) => options.signal.addEventListener('abort', () => reject(options.signal.reason), { once: true }));
+ }));
+ const job = await runway.generateVideo({ apiKey: 'test-key', modelId: 'aleph2', runwayVideoPath: video, uploadedTempPaths: [video] });
+ await started;
+ runway.cancel(job.jobId);
+ expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'failed' });
+ expect(calls.some((url) => url.endsWith('/v1/video_to_video'))).toBe(false);
+ await expect(stat(video)).rejects.toMatchObject({ code: 'ENOENT' });
+ });
+
+ it('replays local input through fresh ephemeral uploads after a failed remote task', async () => {
+ const video = await stagedVideo();
+ const bodies = [];
+ let registrations = 0;
+ vi.stubGlobal('fetch', vi.fn(async (url, options = {}) => {
+ if (url.endsWith('/v1/uploads')) return jsonResponse({ uploadUrl: 'https://storage.example/upload', fields: {}, runwayUri: `runway://fresh-${++registrations}` });
+ if (url === 'https://storage.example/upload') return jsonResponse({}, true, 204);
+ if (url.endsWith('/v1/video_to_video')) { bodies.push(JSON.parse(options.body)); return jsonResponse({ id: 'task-1' }); }
+ if (url === TASKS) return jsonResponse({ status: 'FAILED', failure: 'Unable to render' });
+ throw new Error(`Unexpected fetch ${url}`);
+ }));
+ for (const durationSec of [2, 30]) {
+ probeVideoGeometry.mockResolvedValue({ width: 1280, height: 720, fps: 30, durationSec });
+ const job = await runway.generateVideo({ apiKey: 'test-key', modelId: 'aleph2', runwayVideoPath: video });
+ expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'failed', error: expect.stringContaining('Unable to render') });
+ }
+ expect(bodies.map((body) => body.videoUri)).toEqual(['runway://fresh-1', 'runway://fresh-2']);
+ expect(await stat(video)).toBeTruthy(); // externally owned inputs survive failure/replay
+ });
});
afterEach(async () => {
diff --git a/server/services/videoGen/runwayVideoInputs.js b/server/services/videoGen/runwayVideoInputs.js
new file mode 100644
index 0000000000..8d6e11170f
--- /dev/null
+++ b/server/services/videoGen/runwayVideoInputs.js
@@ -0,0 +1,16 @@
+import { stat } from 'node:fs/promises';
+import { probeVideoGeometry } from '../../lib/ffmpeg.js';
+import { ServerError } from '../../lib/errorHandler.js';
+
+/** Refuse unreadable/unsupported clips before upload or a paid task, including queue replay. */
+export async function validateRunwayVideoInput(path, model, { character = false } = {}) {
+ const fail = (message) => { throw new ServerError(message, { status: 400, code: 'RUNWAY_VIDEO_INPUT' }); };
+ const file = await stat(path);
+ if (file.size < 512 || file.size > 200 * 1024 * 1024) fail('Runway videos must be between 512 bytes and 200 MB.');
+ const geometry = await probeVideoGeometry(path);
+ if (!geometry) fail('Could not read the video stream. Choose a valid video file; ffprobe must be available.');
+ const min = character ? 0 : model.minSec;
+ if (geometry.durationSec <= 0 || (!character && (geometry.durationSec < min || geometry.durationSec > model.maxSec))) fail(`${model.label} requires a video between ${model.minSec} and ${model.maxSec} seconds.`);
+ if (model.id === 'aleph2' && geometry.fps > 30) fail('Aleph 2 input video must be 30 FPS or lower.');
+ return geometry;
+}
diff --git a/server/services/videoGen/submitJob.js b/server/services/videoGen/submitJob.js
index 3d07a3a519..21c3913dff 100644
--- a/server/services/videoGen/submitJob.js
+++ b/server/services/videoGen/submitJob.js
@@ -373,7 +373,7 @@ const submitHostedLane = async ({ body, prepared, backend, hosted, performancePa
const { jobId, position, status } = await enqueue({
// Hosted jobs use mode for queue dispatch and videoMode for text/image.
mode: backend,
- videoMode: sourceImagePath ? 'image' : 'text',
+ videoMode: body.mode || (sourceImagePath ? 'image' : 'text'),
prompt: body.prompt,
negativePrompt: body.negativePrompt || '',
sourceImagePath,