Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
feat: add Runway video editing and character performance workflows
wrenchwren73ae0f702430
diff --git a/client/src/components/videoGen/RunwayVideoInputPanel.jsx b/client/src/components/videoGen/RunwayVideoInputPanel.jsxnew file mode 100644index 0000000000..b665e3be75--- /dev/null+++ b/client/src/components/videoGen/RunwayVideoInputPanel.jsx@@ -0,0 +1,35 @@+import { RUNWAY_ACT_TWO_RATIOS, RUNWAY_ALEPH_ASPECTS } from '../../../../server/lib/runwaySpecialVideoModels.js';++/** Inputs owned by the page's form state; switching modes clears their distinct meanings. */+export default function RunwayVideoInputPanel({ mode, value, onChange }) {+ const performance = mode === 'character-performance';+ const change = (key, next) => onChange({ ...value, [key]: next });+ return <div className="col-span-2 space-y-3 text-sm">+ <label className="block">{performance ? 'Performance video' : 'Source video'}+ <input type="file" accept="video/*" className="block w-full mt-1" onChange={(e) => change('video', e.target.files?.[0] || null)} />+ </label>+ <p className="text-xs text-port-text-muted">{performance ? 'Use a 3–30 second performance with a visible face. Choose a character image below or a character video here.' : 'Use a 2–30 second video at 30 FPS or lower. An optional guidance image below anchors the edit at the beginning.'} Videos must be 512 bytes–200 MB. Files are uploaded privately to Runway when the job starts; uploads expire after 24 hours.</p>+ {performance ? <>+ <label className="block">Character video (instead of an image)+ <input type="file" accept="video/*" className="block w-full mt-1" onChange={(e) => change('characterVideo', e.target.files?.[0] || null)} />+ </label>+ <label className="block">Output canvas+ <select value={value.ratio || ''} onChange={(e) => change('ratio', e.target.value)} className="block bg-port-bg border border-port-border rounded p-2">+ <option value="">Runway default</option>{RUNWAY_ACT_TWO_RATIOS.map((r) => <option key={r} value={r}>{r.replace(':', ' × ')}</option>)}+ </select>+ </label>+ <label className="flex gap-2"><input type="checkbox" checked={value.bodyControl ?? true} onChange={(e) => change('bodyControl', e.target.checked)} />Apply body movements and gestures</label>+ <label className="block">Expression intensity+ <input type="number" min="1" max="5" step="1" value={value.expressionIntensity ?? 3} onChange={(e) => change('expressionIntensity', Number(e.target.value))} className="block bg-port-bg border border-port-border rounded p-2" />+ </label>+ <p className="text-xs text-port-text-muted">Act-Two follows the performance, including its audio; it does not accept a text prompt.</p>+ </> : <label className="block">Expand aspect ratio+ <select value={value.targetAspectRatio || ''} onChange={(e) => change('targetAspectRatio', e.target.value)} className="block bg-port-bg border border-port-border rounded p-2">+ <option value="">Preserve source framing</option>{RUNWAY_ALEPH_ASPECTS.map((r) => <option key={r} value={r}>{r}</option>)}+ </select>+ </label>}+ <label className="block">Runway seed (optional)+ <input type="number" min="0" max="4294967295" step="1" value={value.seed ?? ''} onChange={(e) => change('seed', e.target.value)} className="block bg-port-bg border border-port-border rounded p-2" />+ </label>+ </div>;+}diff --git a/client/src/hooks/useRunwayCatalog.js b/client/src/hooks/useRunwayCatalog.jsindex 0129d5a455..a8a73ea0f2 100644--- a/client/src/hooks/useRunwayCatalog.js+++ b/client/src/hooks/useRunwayCatalog.js@@ -2,14 +2,17 @@ import { useEffect, useSyncExternalStore } from 'react';import { getRunwayCatalogSnapshot, subscribeRunwayCatalog, loadRunwayCatalog } from '../services/apiRunwayCatalog.js';import { RUNWAY_IMAGE_MODELS } from '../lib/imageGenModes.js';import { RUNWAY_VIDEO_MODELS } from '../lib/runwayVideoModels.js';+import { RUNWAY_SPECIAL_VIDEO_MODELS } from '../../../server/lib/runwaySpecialVideoModels.js';export function useRunwayCatalog(enabled = true, videoMode = 'image') {const catalog = useSyncExternalStore(subscribeRunwayCatalog, getRunwayCatalogSnapshot);useEffect(() => { if (enabled) void loadRunwayCatalog(); }, [enabled]);+ const special = RUNWAY_SPECIAL_VIDEO_MODELS.find((m) => m.mode === videoMode);+ const endpoint = special?.endpoint || (videoMode === 'text' ? 'text_to_video' : 'image_to_video');return {...catalog,imageModels: RUNWAY_IMAGE_MODELS.filter((m) => catalog.available.text_to_image.includes(m.id)),- videoModels: RUNWAY_VIDEO_MODELS.filter((m) => catalog.available[videoMode === 'text' ? 'text_to_video' : 'image_to_video'].includes(m.id)),+ videoModels: (special ? RUNWAY_SPECIAL_VIDEO_MODELS : RUNWAY_VIDEO_MODELS).filter((m) => (catalog.available[endpoint] || []).includes(m.id)),refresh: () => loadRunwayCatalog({ refresh: true }),};}diff --git a/client/src/hooks/useRunwayCatalog.test.jsx b/client/src/hooks/useRunwayCatalog.test.jsxindex c5b25b5dfb..d1f471a94c 100644--- a/client/src/hooks/useRunwayCatalog.test.jsx+++ b/client/src/hooks/useRunwayCatalog.test.jsx@@ -5,7 +5,7 @@ import { request } from '../services/apiCore.js';import { useRunwayCatalog } from './useRunwayCatalog.js';const initial = {- available: { text_to_image: ['gen4_image', 'gpt_image_2'], image_to_video: ['gen4.5', 'gen4_turbo', 'veo3.1'], text_to_video: ['gen4.5', 'veo3.1'] },+ available: { text_to_image: ['gen4_image', 'gpt_image_2'], image_to_video: ['gen4.5', 'gen4_turbo', 'veo3.1'], text_to_video: ['gen4.5', 'veo3.1'], video_to_video: ['aleph2'], character_performance: ['act_two'] },unsupported: [{ id: 'act_two', reason: 'Requires performance inputs' }], stale: false,};describe('Runway form catalog lifecycle', () => {@@ -13,10 +13,14 @@ describe('Runway form catalog lifecycle', () => {request.mockResolvedValue(initial);const image = renderHook(() => useRunwayCatalog());const video = renderHook(() => useRunwayCatalog(true, 'text'));+ const edit = renderHook(() => useRunwayCatalog(true, 'video-edit'));+ const performance = renderHook(() => useRunwayCatalog(true, 'character-performance'));await waitFor(() => expect(image.result.current.stale).toBe(false));expect(request).toHaveBeenCalledTimes(1);expect(image.result.current.imageModels.map((m) => m.id)).toEqual(['gen4_image', 'gpt_image_2']);expect(video.result.current.videoModels.map((m) => m.id)).toEqual(['gen4.5', 'veo3.1']);+ expect(edit.result.current.videoModels.map((m) => m.id)).toEqual(['aleph2']);+ expect(performance.result.current.videoModels.map((m) => m.id)).toEqual(['act_two']);image.unmount();const reopened = renderHook(() => useRunwayCatalog());expect(request).toHaveBeenCalledTimes(1);diff --git a/client/src/hooks/useVideoGenFieldState.js b/client/src/hooks/useVideoGenFieldState.jsindex 370c428b5b..0ef06391a0 100644--- a/client/src/hooks/useVideoGenFieldState.js+++ b/client/src/hooks/useVideoGenFieldState.js@@ -31,6 +31,7 @@ export function useVideoGenFieldState({const [runwayModelId, setRunwayModelId] = useState('');const [runwayDuration, setRunwayDuration] = useState('');const [runwayResolution, setRunwayResolution] = useState('');+ const [runwayInputs, setRunwayInputs] = useState({});const [reactorClipId, setReactorClipId] = useState('');const [reactorSeconds, setReactorSeconds] = useState(REACTOR_DEFAULT_CLIP_LENGTH);const [reactorSeed, setReactorSeed] = useState('');@@ -107,6 +108,7 @@ export function useVideoGenFieldState({runwayModelId, setRunwayModelId,runwayDuration, setRunwayDuration,runwayResolution, setRunwayResolution,+ runwayInputs, setRunwayInputs,reactorClipId, setReactorClipId,reactorSeconds, setReactorSeconds,reactorSeed, setReactorSeed,diff --git a/client/src/hooks/useVideoGenForm.js b/client/src/hooks/useVideoGenForm.jsindex fb2b9b8c42..f57f8102cf 100644--- a/client/src/hooks/useVideoGenForm.js+++ b/client/src/hooks/useVideoGenForm.js@@ -25,6 +25,7 @@ import {} from '../lib/videoGenParams.js';import { videoStreamingModeFromRecord } from '../lib/videoStreamingMode.js';import { RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL, getRunwayVideoModel, runwayModelTakesText } from '../lib/runwayVideoModels.js';+import { RUNWAY_SPECIAL_VIDEO_MODELS, getRunwaySpecialVideoModel, RUNWAY_SPECIAL_VIDEO_MODES } from '../../../server/lib/runwaySpecialVideoModels.js';import { useVideoGenFieldState } from './useVideoGenFieldState.js';import { useVideoGenSubmitFlow } from './useVideoGenSubmitFlow.js';import { useVideoGenValidation } from './useVideoGenValidation.js';@@ -109,6 +110,7 @@ export function useVideoGenForm({runwayModelId, setRunwayModelId,runwayDuration, setRunwayDuration,runwayResolution, setRunwayResolution,+ runwayInputs, setRunwayInputs,reactorClipId, setReactorClipId,reactorSeconds, setReactorSeconds,reactorSeed, setReactorSeed,@@ -868,6 +870,14 @@ export function useVideoGenForm({// carry over because they apply to all modes.)const handleModeChange = (next) => {setMode(next);+ if (next !== mode) setRunwayInputs({});+ if (isRunway) setRunwayModelId(RUNWAY_SPECIAL_VIDEO_MODELS.find((m) => m.mode === next)?.id || (getRunwaySpecialVideoModel(runwayModelId) ? '' : runwayModelId));+ if (RUNWAY_SPECIAL_VIDEO_MODES.includes(next)) {+ clearSourceImage();+ clearLastImage();+ setExtendFromVideoId('');+ setChunks(1);+ }// Audio is only meaningful in a2v mode — drop it on every other switch// so a stale upload from a prior pick doesn't sneak into a non-a2v post.if (next !== 'a2v') setAudioFile(null);@@ -951,6 +961,7 @@ export function useVideoGenForm({// clip survives the switch and reappears if the user flips back.const handleBackendChange = (id) => {setBackend(id);+ if (id !== 'runway' && RUNWAY_SPECIAL_VIDEO_MODES.includes(mode)) handleModeChange('text');if ((id === 'grok' || id === 'fal' || id === 'reactor' || id === 'runway') && mode !== 'text' && mode !== 'image') {handleModeChange((sourceImageFile || sourceImageUpload) ? 'image' : 'text');}@@ -1254,7 +1265,8 @@ export function useVideoGenForm({} else if (p.mode === 'runway') {// Runway job: same discriminator shape as fal above.setBackend('runway');- setMode(p.videoMode === 'text' ? 'text' : 'image');+ setMode(RUNWAY_SPECIAL_VIDEO_MODES.includes(p.videoMode) ? p.videoMode : p.videoMode === 'text' ? 'text' : 'image');+ setRunwayInputs({ targetAspectRatio: p.targetAspectRatio, ratio: p.ratio, bodyControl: p.bodyControl, expressionIntensity: p.expressionIntensity, seed: p.seed });if (p.modelId) {runwayModelPickRef.current = p.modelId;setRunwayModelId(p.modelId);@@ -1343,7 +1355,7 @@ export function useVideoGenForm({const submissionState = {hostedLane, isGrok, grokDuration, isFal, falDuration, falModelId, falResolution, falGenerateAudio,isReactor, reactorClipId, reactorSeconds, reactorSeed, reactorAspect, remoteSubmissionFields,- isRunway, runwayModelId, runwayDuration, runwayResolution,+ isRunway, runwayModelId, runwayDuration, runwayResolution, runwayInputs,displaySleepEnabled,prompt, negativePrompt, stylePreset, selectedUniverse,width, height, mode, sourceImageFile, sourceImageUpload,@@ -1369,6 +1381,7 @@ export function useVideoGenForm({runwayModelId, setRunwayModelId,runwayDuration, setRunwayDuration,runwayResolution, setRunwayResolution,+ runwayInputs, setRunwayInputs,reactorClipId, setReactorClipId,reactorSeconds, setReactorSeconds,reactorSeed, setReactorSeed,diff --git a/client/src/lib/videoGenEnqueuePolicy.js b/client/src/lib/videoGenEnqueuePolicy.jsindex 0ba4f42234..31a5244edc 100644--- a/client/src/lib/videoGenEnqueuePolicy.js+++ b/client/src/lib/videoGenEnqueuePolicy.js@@ -27,7 +27,7 @@ const byovReason = ({ byovRuntimeMissing, byovLabel, byovRuntime }) => (byovRunt: `Checking ${byovRuntime} runtime status…`);const ENQUEUE_ROWS = [- { scope: 'always', active: (c) => !c.prompt.trim() },+ { scope: 'always', active: (c) => !c.promptOptional && !c.prompt.trim() },{ scope: 'always', active: (c) => c.remixHandoffPending },{ scope: 'always', active: (c) => c.promptOverLimit },{ scope: 'remote', active: (c) => !!c.remoteBlocked, reason: (c) => c.remoteBlocked },diff --git a/client/src/lib/videoGenSubmission.js b/client/src/lib/videoGenSubmission.jsindex aed469cddc..f942183917 100644--- a/client/src/lib/videoGenSubmission.js+++ b/client/src/lib/videoGenSubmission.js@@ -119,9 +119,22 @@ function falVideoSubmission({// text-to-video rule, from the prompt alone (Text mode). The negative prompt is sent; the server passes it// natively or as an "Avoid:" line, listed as a chip in the sent-prompt view.function runwayVideoSubmission({- composed, runwayModelId, runwayDuration, runwayResolution,+ composed, runwayModelId, runwayDuration, runwayResolution, runwayInputs = {},width, height, mode, sourceImageFile, sourceImageUpload,}) {+ if (mode === 'video-edit' || mode === 'character-performance') return {+ backend: 'runway', mode, runwayModelId,+ prompt: mode === 'character-performance' ? '' : composed.prompt,+ negativePrompt: mode === 'character-performance' ? '' : composed.negativePrompt,+ runwayVideo: runwayInputs.video || '',+ runwayCharacterVideo: mode === 'character-performance' ? runwayInputs.characterVideo || '' : '',+ sourceImageFile: runwayInputs.characterVideo ? '' : sourceImageFile || '',+ sourceImage: runwayInputs.characterVideo ? '' : sourceImageUpload || '',+ ...(runwayInputs.targetAspectRatio ? { runwayTargetAspect: runwayInputs.targetAspectRatio } : {}),+ ...(runwayInputs.ratio ? { runwayRatio: runwayInputs.ratio } : {}),+ ...(mode === 'character-performance' ? { runwayBodyControl: runwayInputs.bodyControl ?? true, runwayExpressionIntensity: runwayInputs.expressionIntensity ?? 3 } : {}),+ ...(runwayInputs.seed != null && runwayInputs.seed !== '' ? { runwaySeed: Number(runwayInputs.seed) } : {}),+ };return {backend: 'runway',prompt: composed.prompt,@@ -343,7 +356,8 @@ export function buildVideoGenSubmission(args) {const shared = { ...args, composed };const isLocal = !hostedLane && !remoteSubmissionFields;- const additions = videoPromptAdditions({ ...args, isLocal });+ const additions = hostedLane === 'runway' && args.mode === 'character-performance'+ ? [] : videoPromptAdditions({ ...args, isLocal });const payload = hostedLane ? HOSTED_VIDEO_SUBMISSIONS[hostedLane](shared): remoteSubmissionFields ? remoteVideoSubmission(shared): localVideoSubmission(args, {diff --git a/client/src/lib/videoGenSubmission.test.js b/client/src/lib/videoGenSubmission.test.jsindex 87c08358a4..2468d42418 100644--- a/client/src/lib/videoGenSubmission.test.js+++ b/client/src/lib/videoGenSubmission.test.js@@ -255,6 +255,15 @@ describe('buildVideoGenSubmission — local IC-LoRA, chunks and misc', () => {describe('buildVideoGenSubmission — prompt additions (#11317)', () => {const preset = { label: 'Film noir', prompt: 'cinematic noir', negativePrompt: '' };+ it('drops carried style metadata when switching to Runway character performance', () => {+ const payload = buildVideoGenSubmission(baseArgs({+ hostedLane: 'runway', mode: 'character-performance', runwayModelId: 'act_two',+ stylePreset: preset, negativePrompt: 'blur', noMusic: true,+ }));+ expect(payload).toMatchObject({ prompt: '', negativePrompt: '' });+ expect(payload).not.toHaveProperty('promptAdditions');+ });+it('lists the style preset, then the No music line the local lane appends', () => {const payload = buildVideoGenSubmission(baseArgs({ currentModel: {}, stylePreset: preset, noMusic: true }));expect(payload.prompt).toBe('cinematic noir. a fox running through a forest\n\nno music, no soundtrack');diff --git a/client/src/pages/VideoGen.jsx b/client/src/pages/VideoGen.jsxindex 3bd632b8c0..fe9582f0bb 100644--- a/client/src/pages/VideoGen.jsx+++ b/client/src/pages/VideoGen.jsx@@ -1,5 +1,7 @@import { useRunwayCatalog } from '../hooks/useRunwayCatalog.js';import RunwayCatalogStatus from '../components/imageGen/RunwayCatalogStatus.jsx';+import RunwayVideoInputPanel from '../components/videoGen/RunwayVideoInputPanel.jsx';+import { RUNWAY_SPECIAL_VIDEO_MODES, getRunwaySpecialVideoModel } from '../../../server/lib/runwaySpecialVideoModels.js';/*** Video Generation page (LTX models via mlx_video on macOS, diffusers on* Windows), with local, Grok, and federated render targets.@@ -246,7 +248,7 @@ export default function VideoGen() {// share. See client/src/hooks/useVideoGenForm.js.const {backend, hostedLane, isGrok, isFal, isReactor, isRunway, handleBackendChange, grokDuration, setGrokDuration,- runwayModelId, setRunwayModelId, runwayDuration, setRunwayDuration, runwayResolution, setRunwayResolution,+ runwayModelId, setRunwayModelId, runwayDuration, setRunwayDuration, runwayResolution, setRunwayResolution, runwayInputs, setRunwayInputs,falDuration, setFalDuration, falModelId, setFalModelId,falResolution, setFalResolution, falGenerateAudio, setFalGenerateAudio,reactorClipId, setReactorClipId, reactorSeconds, setReactorSeconds, reactorSeed, setReactorSeed,@@ -999,7 +1001,9 @@ export default function VideoGen() {const falModel = getFalVideoModel(falModelId);const falResolutions = falVideoResolutions(falModel);const runwayCatalog = useRunwayCatalog(isRunway, mode);- const runwayModel = getRunwayVideoModel(runwayModelId || RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL);+ const runwaySpecial = isRunway && RUNWAY_SPECIAL_VIDEO_MODES.includes(mode);+ const runwayPerformance = isRunway && mode === 'character-performance';+ const runwayModel = getRunwaySpecialVideoModel(runwayModelId) || getRunwayVideoModel(runwayModelId || RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL);const runwaySizeModel = mode === 'text' && runwayModel?.textToVideo ? { ...runwayModel, size: runwayModel.textToVideo.size } : runwayModel;const runwayResolutions = runwayVideoResolutions(runwaySizeModel);@@ -1020,7 +1024,7 @@ export default function VideoGen() {: prompt.length),[promptCap, prompt, negativePrompt, stylePreset, selectedUniverse],);- const promptOverLimit = promptCap != null && submittedPromptLength > promptCap;+ const promptOverLimit = !runwayPerformance && promptCap != null && submittedPromptLength > promptCap;// Budget the AI enhancer writes inside. It's the backend cap MINUS the style// preset / universe prefix, because that prefix is part of what PortOS@@ -1075,6 +1079,7 @@ export default function VideoGen() {const { canEnqueue: backendCanEnqueue, generateTitle, queueTitle, optionsHint } = videoGenEnqueuePolicy({prompt,+ promptOptional: runwaySpecial,remixHandoffPending,promptOverLimit,isRemote: remoteTarget.isRemote,@@ -1095,7 +1100,8 @@ export default function VideoGen() {keyframesBlocked,keyframesError,});- const canEnqueue = backendCanEnqueue && (!isRunway || runwayCatalog.videoModels.some((m) => m.id === runwayModel?.id));+ const specialInputsReady = !runwaySpecial || (!!runwayInputs.video && (!runwayPerformance || !!(runwayInputs.characterVideo || sourceImageFile || sourceImageUpload)));+ const canEnqueue = backendCanEnqueue && specialInputsReady && (!isRunway || runwayCatalog.videoModels.some((m) => m.id === runwayModel?.id));return (<div className="space-y-3">@@ -1219,7 +1225,7 @@ export default function VideoGen() {WAI-ARIA Tabs, since the mode-specific inputs aren't structured astabpanels and we don't implement roving-tabindex/arrow-key focus. */}<div className="bg-port-card border border-port-border rounded-xl p-1 flex flex-wrap gap-1" role="group" aria-label="Video generation mode">- {(rendersOffMachine ? MODES.filter((m) => m.id === 'text' || m.id === 'image') : MODES).map(({ id, label, icon: Icon, desc }) => {+ {(isRunway ? [...MODES.filter((m) => m.id === 'text' || m.id === 'image'), { id: 'video-edit', label: 'Video edit', icon: SlidersHorizontal, desc: 'Edit a source video with Aleph 2' }, { id: 'character-performance', label: 'Character performance', icon: SlidersHorizontal, desc: 'Apply a performance to a character with Act-Two' }] : rendersOffMachine ? MODES.filter((m) => m.id === 'text' || m.id === 'image') : MODES).map(({ id, label, icon: Icon, desc }) => {const active = mode === id;return (<button@@ -1315,7 +1321,7 @@ export default function VideoGen() {repairing={modelDownload.repairing}/>))}- <FormField label="Prompt" labelClassName="block text-xs font-medium text-gray-400 mb-1">+ {!runwayPerformance && <FormField label="Prompt" labelClassName="block text-xs font-medium text-gray-400 mb-1"><AutoSizeTextareavalue={prompt}onChange={(e) => setPrompt(e.target.value)}@@ -1347,7 +1353,7 @@ export default function VideoGen() {This model does not use a negative prompt.</p>)}- </FormField>+ </FormField>}<divclassName="sticky top-0 z-10 -mx-4 flex flex-wrap items-center gap-2 border-y border-port-border bg-port-card/95 px-4 py-2 backdrop-blur lg:static lg:mx-0 lg:border-0 lg:bg-transparent lg:p-0 lg:backdrop-blur-none"@@ -1417,7 +1423,7 @@ export default function VideoGen() {</summary><div className="mt-3 grid min-w-0 grid-cols-1 gap-4 lg:mt-0 @3xl/video-form:grid-cols-2"><section aria-label="Prompt styling and reference media" className="min-w-0 space-y-3">- <div className="grid min-w-0 grid-cols-1 gap-3 @5xl/video-form:grid-cols-2">+ {!runwayPerformance && <div className="grid min-w-0 grid-cols-1 gap-3 @5xl/video-form:grid-cols-2"><UniverseStylePickervalue={selectedUniverse?.id || ''}onChange={setSelectedUniverse}@@ -1426,8 +1432,8 @@ export default function VideoGen() {value={stylePreset?.id || ''}onChange={setStylePreset}/>- </div>- {negativePromptSupported && (+ </div>}+ {!runwayPerformance && negativePromptSupported && (<FormField label="Negative Prompt" labelClassName="block text-xs font-medium text-gray-400 mb-1"><AutoSizeTextareavalue={negativePrompt}@@ -1442,7 +1448,7 @@ export default function VideoGen() {{/* Keep Enhance live while a render is in flight so the next clipcan be composed and submitted to its server queue. Generateitself becomes Cancel. */}- <PromptEnhancer+ {!runwayPerformance && <PromptEnhancerkind="video"prompt={prompt}setPrompt={setPrompt}@@ -1450,7 +1456,7 @@ export default function VideoGen() {setNegativePrompt={negativePromptSupported ? setNegativePrompt : undefined}renderConfig={{ stylePreset: stylePreset?.id, mode, model: modelId }}maxPromptLength={enhancePromptBudget}- />+ />}{mode === 'fflf' && keyframesSupported && (<KeyframePanel@@ -1467,10 +1473,10 @@ export default function VideoGen() {/>)}- {(mode === 'image' || (mode === 'fflf' && !keyframesActive)) && (+ {(mode === 'image' || (runwaySpecial && !runwayInputs.characterVideo) || (mode === 'fflf' && !keyframesActive)) && (<div className={`grid gap-2 ${mode === 'fflf' ? 'grid-cols-1 sm:grid-cols-2' : 'grid-cols-1'}`}><FramePanel- label={mode === 'fflf' ? 'First frame' : 'Source image'}+ label={runwayPerformance ? 'Character image' : runwaySpecial ? 'Guidance image (optional)' : mode === 'fflf' ? 'First frame' : 'Source image'}file={sourceImageFile}upload={sourceImageUpload}uploadUrl={sourceUploadUrl}@@ -1710,6 +1716,7 @@ export default function VideoGen() {))}</select></FormField>+ {runwaySpecial ? <RunwayVideoInputPanel key={mode} mode={mode} value={runwayInputs} onChange={setRunwayInputs} /> : <><FormField label="Clip length" labelClassName="block text-xs font-medium text-gray-400 mb-1"><selectid="runway-video-length"@@ -1743,8 +1750,9 @@ export default function VideoGen() {</select></FormField>)}+ </>}<p className="col-span-2 text-[11px] text-gray-500 leading-snug">- {mode === 'image' ? 'Animates your start frame' : 'Renders from your prompt'} on the Runway API. Counts against your Runway credits+ {runwaySpecial ? 'Uses your video inputs' : mode === 'image' ? 'Animates your start frame' : 'Renders from your prompt'} on the Runway API. Counts against your Runway credits{(() => {const estimate = estimateRunwayVideoCostUsd({modelId: runwayModel?.id, seconds: runwayDuration === '' ? null : Number(runwayDuration), resolution: runwayResolution,diff --git a/client/src/pages/VideoGen.runway.test.jsx b/client/src/pages/VideoGen.runway.test.jsxindex 6ec753c9c4..abf4bbc91b 100644--- a/client/src/pages/VideoGen.runway.test.jsx+++ b/client/src/pages/VideoGen.runway.test.jsx@@ -18,6 +18,52 @@ describe('VideoGen Runway text lane', () => {state.generateVideo.mockResolvedValue({ jobId: 'job-1' });});+ it.each([+ ['Video edit', 'aleph2', 'Source video'],+ ['Character performance', 'act_two', 'Performance video'],+ ])('submits %s through its own model and video inputs without inventing a prompt', async (label, modelId, videoLabel) => {+ await renderVideoGenPage();+ fireEvent.click(await screen.findByRole('button', { name: 'Runway' }));+ fireEvent.click(within(screen.getByRole('group', { name: 'Video generation mode' })).getByRole('button', { name: label }));+ const model = await screen.findByLabelText('Runway model');+ expect([...model.options].map((o) => o.value)).toEqual([modelId]);+ const source = new File([new Uint8Array(1024)], 'performance.mp4', { type: 'video/mp4' });+ expect(screen.getByRole('button', { name: /Add to queue/ })).toBeDisabled();+ fireEvent.change(screen.getByLabelText(videoLabel), { target: { files: [source] } });+ if (modelId === 'act_two') {+ expect(screen.queryByLabelText('Prompt')).toBeNull();+ expect(screen.getByRole('button', { name: /Add to queue/ })).toBeDisabled();+ fireEvent.change(screen.getByLabelText('Character video (instead of an image)'), { target: { files: [source] } });+ fireEvent.change(screen.getByLabelText('Expression intensity'), { target: { value: '4' } });+ }+ fireEvent.click(await findEnabledByRole('button', { name: /Add to queue/ }));+ await waitFor(() => expect(state.generateVideo).toHaveBeenCalled());+ expect(state.generateVideo.mock.calls[0][0]).toMatchObject({ backend: 'runway', runwayModelId: modelId, runwayVideo: source, prompt: '', mode: modelId === 'aleph2' ? 'video-edit' : 'character-performance' });+ if (modelId === 'act_two') expect(state.generateVideo.mock.calls[0][0]).toMatchObject({ runwayCharacterVideo: source, runwayExpressionIntensity: 4 });+ });++ it('clears file selections when switching special workflows and allows reselecting the same file', async () => {+ await renderVideoGenPage();+ fireEvent.click(await screen.findByRole('button', { name: 'Runway' }));+ const modes = within(screen.getByRole('group', { name: 'Video generation mode' }));+ fireEvent.click(modes.getByRole('button', { name: 'Video edit' }));+ const firstInput = screen.getByLabelText('Source video');+ const source = new File([new Uint8Array(1024)], 'source.mp4', { type: 'video/mp4' });+ fireEvent.change(firstInput, { target: { files: [source] } });+ await findEnabledByRole('button', { name: /Add to queue/ });+ fireEvent.click(modes.getByRole('button', { name: 'Character performance' }));+ expect(screen.getByLabelText('Performance video')).not.toBe(firstInput);+ expect(screen.getByLabelText('Performance video').files).toHaveLength(0);+ fireEvent.click(modes.getByRole('button', { name: 'Video edit' }));+ const resetInput = screen.getByLabelText('Source video');+ expect(resetInput.files).toHaveLength(0);+ expect(screen.getByRole('button', { name: /Add to queue/ })).toBeDisabled();+ fireEvent.change(resetInput, { target: { files: [source] } });+ fireEvent.click(await findEnabledByRole('button', { name: /Add to queue/ }));+ await waitFor(() => expect(state.generateVideo).toHaveBeenCalled());+ expect(state.generateVideo.mock.calls[0][0]).toMatchObject({ runwayVideo: source });+ });+it('switches from an image-only model to a priced text-capable model and submits without a frame', async () => {await renderVideoGenPage();fireEvent.click(await screen.findByRole('button', { name: 'Runway' }));diff --git a/docs/RUNWAY_MODELS.md b/docs/RUNWAY_MODELS.mdindex ca7bdc1067..4f83f592e7 100644--- a/docs/RUNWAY_MODELS.md+++ b/docs/RUNWAY_MODELS.md@@ -9,6 +9,8 @@ needs no API key.The bundled capability snapshot was verified on 2026-10-22. It offers 19 imagemodels and 24 image-to-video models; 23 of the video models also accept text alone.+Video edit adds Aleph 2 and Character performance adds Act-Two, for 26 distinct+video models across these workflows.Selectors filter by the active operation. Gen-4 Image Turbo requires a reference;Gen-4 Turbo video requires a start frame. The builder validates each model's prompt,reference and seed constraints and emits supported parameters only. Models accepting@@ -49,12 +51,30 @@ before a paid POST. Generation does not perform a separate discovery lookup.## Workflow limits-Aleph 2 (`aleph2`) needs video-to-video editing inputs and Act-Two (`act_two`) needs-character/performance inputs. They are identified in the catalog UI but are not-runnable through the existing text/image forms. These are PortOS workflow limits,-not missing Runway APIs. Upscaling, HDR, avatars, source-video/audio references and-other specialized controls are also outside these forms. This integration does not-claim to expose every Runway API operation.+Video Gen's Runway lane offers **Video edit** (`aleph2`) and **Character performance**+(`act_two`) alongside Text and Image. Aleph takes a 2–30 second source video at up+to 30 FPS, an optional prompt (1000 characters including the negative prompt), an+optional guidance image at the start, and optional expand aspect ratio. It uses+the default MP4 delivery and preserves source resolution up to 1080p. Act-Two takes+a 3–30 second performance clip plus one character image or video, body-control and+expression-intensity controls, and an optional supported canvas. It follows the+performance rather than a text prompt. Video durations and streams are probed+before upload; invalid or unreadable inputs fail before generation.++Uploads are staged durably using the normal queue ownership/cleanup rules. The+worker creates [ephemeral Runway uploads](https://docs.dev.runwayml.com/assets/uploads/)+at dispatch time, propagates cancellation through registration and storage POST,+and submits only after every upload succeeds. Video files must be 512 bytes–200 MB.+Only local input paths are queued; 24-hour upload URIs are never persisted as retry+inputs, and replay creates fresh uploads. Storage POSTs receive signed upload fields,+not the Runway API key. Local owned inputs are cleaned on success/failure/cancel;+Runway's remote uploads expire automatically. Reloaded forms require video files to+be selected again before a new submission.++Specialized models stay out of text/image-only Music Video and Creative Director+selectors. Other models' advanced video-reference controls, multiple timed Aleph+keyframes, non-MP4 delivery, upscaling, HDR and avatars remain outside these forms.+This integration does not claim to expose every Runway API operation.Tests use public schema excerpts and mocked transport, including refresh, cacherestart/expiry, source failures and capability-specific request bodies. No paiddiff --git a/server/lib/README.md b/server/lib/README.mdindex 6a1da14625..a3a74bedaf 100644--- a/server/lib/README.md+++ b/server/lib/README.md@@ -972,3 +972,5 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub- `manuscriptLocate.js`: `locateFind` / `locateFindSpan` — the manuscript `find` locator (nearest-to-`anchorQuote` occurrence, then whitespace-tolerant regex fallback). The one implementation behind both the server accept path (`services/pipeline/manuscriptFix.js`) and the client highlight/impact preview (`client/src/lib/manuscriptAnchors.js` re-exports it), so the preview selects the span an accept replaces. Pure / browser-safe.| `runwayCatalogSnapshot.js` | Shared Runway model catalog capabilities and cached model selection. |++| `runwaySpecialVideoModels.js` | Aleph 2 video-edit and Act-Two character-performance capabilities, schema compatibility and request validation. |diff --git a/server/lib/index.js b/server/lib/index.jsindex 133631f589..8129f94c71 100644--- a/server/lib/index.js+++ b/server/lib/index.js@@ -958,3 +958,5 @@ export * from './errorLogFingerprint.js';export * from './manuscriptLocate.js';export * from './runwayCatalogSnapshot.js';++export * from './runwaySpecialVideoModels.js';diff --git a/server/lib/runwayCatalogSnapshot.js b/server/lib/runwayCatalogSnapshot.jsindex e544c80cfd..31e2e55427 100644--- a/server/lib/runwayCatalogSnapshot.js+++ b/server/lib/runwayCatalogSnapshot.js@@ -73,5 +73,7 @@ export const RUNWAY_CATALOG_SNAPSHOT = {{"id": "h3_max","promptRequired": true,"promptMin": 1,"promptMax": 6000,"ratios": [],"ratioRequired": false,"resolutions": ["480p","768p"],"seed": true,"audio": false,"endImage": false,"duration": {"minimum": 5,"maximum": 15},"minReferences": 0,"maxReferences": 0,"referenceTags": false,"seedMax": 4294967295,"negativePromptMax": 0},{"id": "seedance2_5","promptRequired": false,"promptMin": 1,"promptMax": 15000,"ratios": ["992:432","854:480","752:560","640:640","560:752","480:854","1470:630","1280:720","1112:834","960:960","834:1112","720:1280","2206:946","1920:1080","1664:1248","1440:1440","1248:1664","1080:1920"],"ratioRequired": false,"resolutions": [],"seed": true,"audio": true,"endImage": false,"duration": {"minimum": 4,"maximum": 30},"minReferences": 0,"maxReferences": 0,"referenceTags": false,"seedMax": 4294967295,"negativePromptMax": 0},{"id": "grok_imagine_1_5_lite","promptRequired": true,"promptMin": 1,"promptMax": 2500,"ratios": ["848:480","480:848","480:480","640:480","480:640","720:480","480:720","1280:720","720:1280","720:720","960:720","720:960","1088:720","720:1088","1904:1072","1072:1904","1424:1424","1648:1232","1232:1648","1744:1152","1152:1744"],"ratioRequired": false,"resolutions": [],"seed": false,"audio": false,"endImage": false,"duration": {"minimum": 1,"maximum": 15},"minReferences": 0,"maxReferences": 0,"referenceTags": false,"seedMax": null,"negativePromptMax": 0}- ]+ ],+ "video_to_video": [{"id":"aleph2","contract":{"additionalProperties":false,"properties":{"contentModeration":{"additionalProperties":false,"properties":{"publicFigureThreshold":{"enum":["auto","low"],"type":"string"}},"type":"object"},"keyframes":{"items":{"anyOf":[{"additionalProperties":false,"properties":{"range":{"additionalProperties":false,"properties":{"end_seconds":{"exclusiveMinimum":0,"maximum":9007199254740991,"type":"integer"},"start_seconds":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["start_seconds","end_seconds"],"type":"object"},"seconds":{"maximum":30,"minimum":0,"type":"number"},"uri":{"anyOf":[{"maxLength":2048,"minLength":13,"pattern":"^https:\\/\\/.*","type":"string"},{"maxLength":5000,"minLength":13,"pattern":"^runway:\\/\\/.*","type":"string"},{"maxLength":5242880,"minLength":13,"pattern":"^data:image\\/.*","type":"string"}]}},"required":["uri","seconds"],"type":"object"},{"additionalProperties":false,"properties":{"at":{"maximum":1,"minimum":0,"type":"number"},"range":{"additionalProperties":false,"properties":{"end_seconds":{"exclusiveMinimum":0,"maximum":9007199254740991,"type":"integer"},"start_seconds":{"maximum":9007199254740991,"minimum":0,"type":"integer"}},"required":["start_seconds","end_seconds"],"type":"object"},"uri":{"anyOf":[{"maxLength":2048,"minLength":13,"pattern":"^https:\\/\\/.*","type":"string"},{"maxLength":5000,"minLength":13,"pattern":"^runway:\\/\\/.*","type":"string"},{"maxLength":5242880,"minLength":13,"pattern":"^data:image\\/.*","type":"string"}]}},"required":["uri","at"],"type":"object"}]},"maxItems":5,"minItems":1,"type":"array"},"model":{"const":"aleph2","type":"string"},"outputFormat":{"enum":["mp4","prores","png_sequence","sdr_rec709_10bit"],"type":"string"},"promptText":{"maxLength":1000,"minLength":1,"type":"string"},"proresProfile":{"enum":["422","4444","422 Proxy","422 LT","422 HQ","4444 XQ"],"type":"string"},"ratio":{"type":"string"},"seed":{"maximum":4294967295,"minimum":0,"type":"integer"},"targetAspectRatio":{"enum":["16:9","4:3","3:2","1:1","2:3","3:4","9:16","21:9"],"type":"string"},"videoUri":{"anyOf":[{"maxLength":2048,"minLength":13,"pattern":"^https:\\/\\/.*","type":"string"},{"maxLength":5000,"minLength":13,"pattern":"^runway:\\/\\/.*","type":"string"},{"maxLength":5242880,"minLength":13,"pattern":"^data:video\\/.*","type":"string"}]}},"required":["videoUri","model"],"type":"object"}}],+ "character_performance": [{"id":"act_two","contract":{"properties":{"bodyControl":{"type":"boolean"},"character":{"oneOf":[{"properties":{"type":{"const":"image","type":"string"},"uri":{"anyOf":[{"maxLength":2048,"minLength":13,"pattern":"^https:\\/\\/.*","type":"string"},{"maxLength":5000,"minLength":13,"pattern":"^runway:\\/\\/.*","type":"string"},{"maxLength":5242880,"minLength":13,"pattern":"^data:image\\/.*","type":"string"}]}},"required":["type","uri"],"type":"object"},{"properties":{"type":{"const":"video","type":"string"},"uri":{"anyOf":[{"maxLength":2048,"minLength":13,"pattern":"^https:\\/\\/.*","type":"string"},{"maxLength":5000,"minLength":13,"pattern":"^runway:\\/\\/.*","type":"string"},{"maxLength":5242880,"minLength":13,"pattern":"^data:video\\/.*","type":"string"}]}},"required":["type","uri"],"type":"object"}]},"contentModeration":{"additionalProperties":false,"properties":{"publicFigureThreshold":{"enum":["auto","low"],"type":"string"}},"type":"object"},"expressionIntensity":{"maximum":5,"minimum":1,"type":"integer"},"model":{"const":"act_two","type":"string"},"ratio":{"enum":["1280:720","720:1280","960:960","1104:832","832:1104","1584:672"],"type":"string"},"reference":{"oneOf":[{"properties":{"type":{"const":"video","type":"string"},"uri":{"anyOf":[{"maxLength":2048,"minLength":13,"pattern":"^https:\\/\\/.*","type":"string"},{"maxLength":5000,"minLength":13,"pattern":"^runway:\\/\\/.*","type":"string"},{"maxLength":5242880,"minLength":13,"pattern":"^data:video\\/.*","type":"string"}]}},"required":["type","uri"],"type":"object"}]},"seed":{"maximum":4294967295,"minimum":0,"type":"integer"}},"required":["character","reference","model"],"type":"object"}}]};diff --git a/server/lib/runwaySpecialVideoModels.js b/server/lib/runwaySpecialVideoModels.jsnew file mode 100644index 0000000000..8a35bdf6ca--- /dev/null+++ b/server/lib/runwaySpecialVideoModels.js@@ -0,0 +1,56 @@+/** Runway workflows with video inputs, separate from text/start-frame model choices. */+export const RUNWAY_SPECIAL_VIDEO_MODELS = Object.freeze([+ { id: 'aleph2', label: 'Aleph 2', mode: 'video-edit', endpoint: 'video_to_video', promptMaxChars: 1000, prompt: 'optional', minSec: 2, maxSec: 30 },+ { id: 'act_two', label: 'Act-Two', mode: 'character-performance', endpoint: 'character_performance', promptMaxChars: 0, prompt: 'unsupported', minSec: 3, maxSec: 30 },+]);+export const getRunwaySpecialVideoModel = (id) => RUNWAY_SPECIAL_VIDEO_MODELS.find((m) => m.id === id) || null;+export const RUNWAY_SPECIAL_VIDEO_MODES = RUNWAY_SPECIAL_VIDEO_MODELS.map((m) => m.mode);+export const RUNWAY_ACT_TWO_RATIOS = ['1280:720', '720:1280', '960:960', '1104:832', '832:1104', '1584:672'];+export const RUNWAY_ALEPH_ASPECTS = ['16:9', '4:3', '3:2', '1:1', '2:3', '3:4', '9:16', '21:9'];++/** Semantic schema only; documentation wording/order must not retire a compatible model. */+export function runwaySpecialSchemaContract(value) {+ if (Array.isArray(value)) return value.map(runwaySpecialSchemaContract);+ if (!value || typeof value !== 'object') return value;+ const ignored = ['$schema', 'description', 'title', 'example', 'examples', 'default', 'deprecated', 'discriminator'];+ return Object.fromEntries(Object.keys(value).sort().filter((key) => !ignored.includes(key) && !key.startsWith('x-')).map((key) => [key, runwaySpecialSchemaContract(value[key])]));+}++export function buildRunwaySpecialVideoRequest({ modelId, videoMode, videoUri, characterVideoUri, imageUri, endImageUri,+ prompt = '', negativePrompt = '', seed, targetAspectRatio, ratio, bodyControl = true, expressionIntensity = 3, sourceDuration } = {}) {+ const fail = (message) => { throw Object.assign(new Error(message), { status: 400, code: 'RUNWAY_MODEL_INPUT' }); };+ const model = getRunwaySpecialVideoModel(modelId);+ if (!model || (videoMode && videoMode !== model.mode)) fail('Select a Runway model matching the video workflow.');+ if (!videoUri) fail(`${model.label} needs a ${model.id === 'act_two' ? 'performance' : 'source'} video.`);+ if (endImageUri) fail(`${model.label} does not accept an end frame in this workflow.`);+ const body = { model: model.id };+ if (seed != null && seed !== '') {+ if (!Number.isInteger(seed) || seed < 0 || seed > 4294967295) fail('Runway seed must be an integer between 0 and 4294967295.');+ body.seed = seed;+ }+ if (model.id === 'aleph2') {+ if (characterVideoUri) fail('Aleph 2 takes one source video, not a character video.');+ if (ratio) fail('Aleph 2 preserves source resolution; use an expand aspect ratio instead.');+ const text = [prompt.trim(), negativePrompt.trim() ? `Avoid: ${negativePrompt.trim()}` : ''].filter(Boolean).join('\n');+ if (text.length > 1000) fail('Aleph 2 takes at most 1000 prompt characters, including the negative prompt.');+ body.videoUri = videoUri;+ if (text) body.promptText = text;+ if (imageUri) body.keyframes = [{ uri: imageUri, at: 0 }];+ if (targetAspectRatio) {+ if (!RUNWAY_ALEPH_ASPECTS.includes(targetAspectRatio)) fail('Unsupported Aleph 2 expand aspect ratio.');+ body.targetAspectRatio = targetAspectRatio;+ }+ } else {+ if (prompt.trim() || negativePrompt.trim()) fail('Act-Two follows the performance video and does not accept prompts.');+ if (!!imageUri === !!characterVideoUri) fail('Act-Two needs exactly one character image or character video.');+ if (targetAspectRatio) fail('Act-Two does not accept an expand aspect ratio.');+ if (typeof bodyControl !== 'boolean' || !Number.isInteger(expressionIntensity) || expressionIntensity < 1 || expressionIntensity > 5) fail('Act-Two expression intensity must be 1–5 and body control must be boolean.');+ if (ratio && !RUNWAY_ACT_TWO_RATIOS.includes(ratio)) fail('Unsupported Act-Two output canvas.');+ body.character = { type: characterVideoUri ? 'video' : 'image', uri: characterVideoUri || imageUri };+ body.reference = { type: 'video', uri: videoUri };+ body.bodyControl = bodyControl;+ body.expressionIntensity = expressionIntensity;+ if (ratio) body.ratio = ratio;+ }+ return { model, endpoint: model.endpoint, body, seconds: sourceDuration ?? null, resolution: ratio || null, estimatedCostUsd: null };+}diff --git a/server/routes/videoGen.js b/server/routes/videoGen.jsindex 4a04bbb351..4d5093fd36 100644--- a/server/routes/videoGen.js+++ b/server/routes/videoGen.js@@ -98,6 +98,7 @@ import { detectSystemCapabilities, withHardwareCompatibility } from '../lib/systimport { isDisplaySleepEnabled } from '../services/videoGen/displayPower.js';import { promptAdditionsSchema } from '../lib/modelInput.js';import { VIDEO_TILING_MODES } from '../lib/videoTiling.js';+import { RUNWAY_SPECIAL_VIDEO_MODES, RUNWAY_ACT_TWO_RATIOS, RUNWAY_ALEPH_ASPECTS } from '../lib/runwaySpecialVideoModels.js';const router = Router();@@ -173,11 +174,11 @@ export const isAudioMime = (mime, filename) => {// upload (icReference). Files stream to disk without a file-size cap.// Per-fieldname mime filtering rejects mismatched parts// up-front so a stray .mp4 drag-drop can't get staged under these fields.-const frameImageUpload = uploadFields(['sourceImage', 'lastImage', 'audioFile', 'icReference'], {+const frameImageUpload = uploadFields(['sourceImage', 'lastImage', 'audioFile', 'icReference', 'runwayVideo', 'runwayCharacterVideo'], {fileFilter: (_req, file, cb) => {const isImageField = file.fieldname === 'sourceImage' || file.fieldname === 'lastImage';const isAudioField = file.fieldname === 'audioFile';- const isVideoField = file.fieldname === 'icReference';+ const isVideoField = ['icReference', 'runwayVideo', 'runwayCharacterVideo'].includes(file.fieldname);const okImage = isImageField && file.mimetype.startsWith('image/');const okAudio = isAudioField && isAudioMime(file.mimetype, file.originalname);// IC-LoRA references are clips — the control weight reads structure/motion@@ -324,6 +325,11 @@ const generateBodySchema = z.object({// validated like falModelId: the provider refuses an uncatalogued id and// rounds a length up to one the model delivers.runwayModelId: z.string().min(1).max(64).optional(),+ runwayTargetAspect: z.enum(RUNWAY_ALEPH_ASPECTS).optional(),+ runwayRatio: z.enum(RUNWAY_ACT_TWO_RATIOS).optional(),+ runwayBodyControl: z.preprocess((v) => v === 'true' ? true : v === 'false' ? false : v, z.boolean().optional()),+ runwayExpressionIntensity: optionalNum(1, 5, 'runwayExpressionIntensity'),+ runwaySeed: optionalNum(0, 4294967295, 'runwaySeed'),runwayDuration: z.preprocess((v) => (v == null || v === '' ? undefined : Number(v)),optionalNum(1, 60, 'runwayDuration'),@@ -354,7 +360,7 @@ const generateBodySchema = z.object({(v) => (v === '' ? undefined : v),z.enum(REACTOR_ASPECTS).optional(),),- prompt: z.string().min(1).max(20000),+ prompt: z.string().max(20000),negativePrompt: z.string().max(8000).optional(),modelId: z.string().max(64).optional(),width: optionalNum(64, 2048, 'width'),@@ -378,7 +384,7 @@ const generateBodySchema = z.object({// falls back to inferring (sourceImage→i2v, no source→t2v).// IC-LoRA remix modes (`ic-control`, …) come from the weight registry so the// enum can never drift from what's actually installable.- mode: z.enum(['text', 'image', 'fflf', 'extend', 'a2v', ...IC_LORA_MODE_VALUES]).optional(),+ mode: z.enum(['text', 'image', 'fflf', 'extend', 'a2v', ...IC_LORA_MODE_VALUES, ...RUNWAY_SPECIAL_VIDEO_MODES]).optional(),// Chain N renders end-to-end: each chunk's last frame becomes the next// chunk's start frame, then ffmpeg concats them into one clip. 1..8 to// keep the worst-case wall time bounded (8 × ~5min ≈ 40min on M3 Max).@@ -546,6 +552,8 @@ const generateBodySchema = z.object({// docs/decisions/2026-08-20-federated-visual-prompts.mdmediaProviderPeerId: z.string().guid().optional(),mediaProviderEngine: z.string().trim().min(1).max(80).optional(),+}).refine((body) => body.prompt.length > 0 || (body.backend === 'runway' && RUNWAY_SPECIAL_VIDEO_MODES.includes(body.mode)), {+ message: 'A prompt is required for this workflow.', path: ['prompt'],});// The Video Gen form's preview (#11317): the generate body as the form would@@ -1081,6 +1089,7 @@ const ACTIVE_JOB_PARAM_FIELDS = [// 'grok' discriminator for them) and the clip duration — both plain// values, safe to echo for the reloading page's form restore.'videoMode', 'duration',+ 'targetAspectRatio', 'ratio', 'bodyControl', 'expressionIntensity',// reactor.inc jobs (#6214): the clip to chain from, the clip length, and the// fast-h3 session canvas — all plain scalars (no filesystem path), safe to// echo for the reloading page's form restore. `seed` above already coversdiff --git a/server/routes/videoGen.test.js b/server/routes/videoGen.test.jsindex 44e63e757a..a083bcfc36 100644--- a/server/routes/videoGen.test.js+++ b/server/routes/videoGen.test.js@@ -12,6 +12,8 @@ const compiledVisual = vi.hoisted(() => ({}));const probeVideoDuration = vi.hoisted(() => vi.fn(async () => 41.041281));const probeVideoGeometry = vi.hoisted(() => vi.fn());+const validateRunwayVideoInput = vi.hoisted(() => vi.fn(async () => ({ durationSec: 4 })));+vi.mock('../services/videoGen/runwayVideoInputs.js', () => ({ validateRunwayVideoInput }));vi.mock('../lib/ffmpeg.js', async (importOriginal) => ({...(await importOriginal()),probeVideoDuration,@@ -876,6 +878,45 @@ describe('videoGen routes', () => {imageGen: { grok: { enabled: true, grokPath: '/example/grok', aspectRatio: '16:9' } },videoGen: { fal: { apiKey: 'example-fal-key' }, reactor: { apiKey: 'example-reactor-key' } },};+ it.each(['aleph2', 'act_two'])('stages %s inputs durably and preserves the workflow in queued params', async (modelId) => {+ const { getSettings } = await import('../services/settings.js');+ getSettings.mockResolvedValueOnce({ videoGen: { runway: { apiKey: 'example-key' } } });+ setPendingUpload({ fieldname: 'runwayVideo', path: '/tmp/performance.mp4', originalname: 'performance.mp4' });+ const mode = modelId === 'aleph2' ? 'video-edit' : 'character-performance';+ const response = await request(app).post('/api/video-gen/').send({ backend: 'runway', mode, runwayModelId: modelId, prompt: '',+ ...(modelId === 'act_two' ? { sourceImageFile: 'character.png', runwayBodyControl: 'false', runwayExpressionIntensity: '4' } : { runwayTargetAspect: '9:16' }),+ });+ expect(response.status).toBe(200);+ const params = mediaJobQueue.enqueueJob.mock.calls[0][0].params;+ expect(params).toMatchObject({ mode: 'runway', videoMode: mode, modelId, runwayVideoPath: expect.stringContaining('video-runway-video-') });+ expect(params.uploadedTempPaths).toContain(params.runwayVideoPath);+ expect(validateRunwayVideoInput).toHaveBeenCalledWith(params.runwayVideoPath, expect.objectContaining({ id: modelId }));+ expect(JSON.stringify(params)).not.toContain('example-key');+ });++ it('rolls back the durable video if validation fails and refuses a video workflow on another backend', async () => {+ const { getSettings } = await import('../services/settings.js');+ getSettings.mockResolvedValueOnce({ videoGen: { runway: { apiKey: 'example-key' } } });+ validateRunwayVideoInput.mockRejectedValueOnce(Object.assign(new Error('Video is too long'), { status: 400 }));+ setPendingUpload({ fieldname: 'runwayVideo', path: '/tmp/source.mp4', originalname: 'source.mp4' });+ const response = await request(app).post('/api/video-gen/').send({ backend: 'runway', mode: 'video-edit', runwayModelId: 'aleph2', prompt: '' });+ expect(response.status).toBe(400);+ expect(unlink).toHaveBeenCalledWith(copyFile.mock.calls[0][1]);+ expect(mediaJobQueue.enqueueJob).not.toHaveBeenCalled();+ const mismatch = await request(app).post('/api/video-gen/').send({ backend: 'local', mode: 'character-performance', prompt: 'test' });+ expect(mismatch.status).toBe(400);+ });++ it('does not silently drop an unresolved Aleph guidance image', async () => {+ const { getSettings } = await import('../services/settings.js');+ getSettings.mockResolvedValueOnce({ videoGen: { runway: { apiKey: 'example-key' } } });+ resolveGalleryImage.mockReturnValueOnce(null);+ setPendingUpload({ fieldname: 'runwayVideo', path: '/tmp/source.mp4', originalname: 'source.mp4' });+ const response = await request(app).post('/api/video-gen/').send({ backend: 'runway', mode: 'video-edit', runwayModelId: 'aleph2', prompt: '', sourceImageFile: 'missing.png' });+ expect(response.status).toBe(400);+ expect(mediaJobQueue.enqueueJob).not.toHaveBeenCalled();+ expect(copyFile).not.toHaveBeenCalled();+ });const providers = [{ backend: 'grok', input: { grokDuration: 10 },minimal: { grokPath: '/example/grok', aspectRatio: '16:9', width: undefined, height: undefined, duration: undefined },diff --git a/server/services/fixtures/runwayOpenApi.json b/server/services/fixtures/runwayOpenApi.jsonindex 96977ed5c5..52c439b388 100644--- a/server/services/fixtures/runwayOpenApi.json+++ b/server/services/fixtures/runwayOpenApi.json@@ -781,6 +781,522 @@}}}+ },+ "/v1/video_to_video": {+ "post": {+ "requestBody": {+ "content": {+ "application/json": {+ "schema": {+ "oneOf": [+ {+ "type": "object",+ "properties": {+ "promptText": {+ "description": "A non-empty and optional string describing what should appear in the output.",+ "type": "string",+ "minLength": 1,+ "maxLength": 1000+ },+ "keyframes": {+ "description": "Timed guidance images placed at specific points in the input video.\nUp to 5 keyframes.",+ "title": "Keyframes",+ "minItems": 1,+ "maxItems": 5,+ "type": "array",+ "items": {+ "title": "Keyframe",+ "x-stainless-variantName": "Keyframe",+ "anyOf": [+ {+ "title": "KeyframeWithSeconds",+ "x-stainless-variantName": "KeyframeWithSeconds",+ "type": "object",+ "properties": {+ "uri": {+ "description": "A HTTPS URL, Runway or data URI containing an encoded image.\nSee [our docs](/assets/inputs#images) on image inputs for more information.",+ "example": "https://example.com/image.jpg",+ "anyOf": [+ {+ "description": "A HTTPS URL, Runway upload URI, or base64 data URI (e.g. `data:image/png;base64,...`, up to 5MB) containing an encoded image. See [our docs](/assets/inputs#images) on image inputs for more information.",+ "example": "https://example.com/image.jpg",+ "type": "string",+ "minLength": 13,+ "maxLength": 2048,+ "pattern": "^https:\\/\\/.*"+ },+ {+ "description": "A Runway upload URI.\nSee https://docs.dev.runwayml.com/assets/uploads for more information.",+ "example": "runway://<token>",+ "type": "string",+ "minLength": 13,+ "maxLength": 5000,+ "pattern": "^runway:\\/\\/.*"+ },+ {+ "description": "A data URI containing encoded media, up to 5MB.",+ "example": "data:image/jpeg;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",+ "type": "string",+ "minLength": 13,+ "maxLength": 5242880,+ "pattern": "^data:image\\/.*"+ }+ ]+ },+ "seconds": {+ "description": "Absolute timestamp in seconds from the start of the input video when this guidance image should apply.",+ "type": "number",+ "minimum": 0,+ "maximum": 30+ },+ "range": {+ "description": "Optional edit window.\nWhen set, the edit applies only to this time range and the keyframe timestamp must fall within it.\nAll keyframes must either set a range or none may.",+ "title": "KeyframeRange",+ "type": "object",+ "properties": {+ "start_seconds": {+ "description": "Start of the edit window in whole seconds from the start of the input video.",+ "type": "integer",+ "minimum": 0,+ "maximum": 9007199254740991+ },+ "end_seconds": {+ "description": "End of the edit window (exclusive) in whole seconds from the start of the input video.",+ "type": "integer",+ "exclusiveMinimum": 0,+ "maximum": 9007199254740991+ }+ },+ "required": [+ "start_seconds",+ "end_seconds"+ ],+ "additionalProperties": false+ }+ },+ "required": [+ "uri",+ "seconds"+ ],+ "additionalProperties": false+ },+ {+ "title": "KeyframeWithAt",+ "x-stainless-variantName": "KeyframeWithAt",+ "type": "object",+ "properties": {+ "uri": {+ "description": "A HTTPS URL, Runway or data URI containing an encoded image.\nSee [our docs](/assets/inputs#images) on image inputs for more information.",+ "example": "https://example.com/image.jpg",+ "anyOf": [+ {+ "description": "A HTTPS URL, Runway upload URI, or base64 data URI (e.g. `data:image/png;base64,...`, up to 5MB) containing an encoded image. See [our docs](/assets/inputs#images) on image inputs for more information.",+ "example": "https://example.com/image.jpg",+ "type": "string",+ "minLength": 13,+ "maxLength": 2048,+ "pattern": "^https:\\/\\/.*"+ },+ {+ "description": "A Runway upload URI.\nSee https://docs.dev.runwayml.com/assets/uploads for more information.",+ "example": "runway://<token>",+ "type": "string",+ "minLength": 13,+ "maxLength": 5000,+ "pattern": "^runway:\\/\\/.*"+ },+ {+ "description": "A data URI containing encoded media, up to 5MB.",+ "example": "data:image/jpeg;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",+ "type": "string",+ "minLength": 13,+ "maxLength": 5242880,+ "pattern": "^data:image\\/.*"+ }+ ]+ },+ "at": {+ "description": "Position as a fraction [0.0, 1.0] of the input video duration when this guidance image should apply.",+ "type": "number",+ "minimum": 0,+ "maximum": 1+ },+ "range": {+ "description": "Optional edit window.\nWhen set, the edit applies only to this time range and the keyframe timestamp must fall within it.\nAll keyframes must either set a range or none may.",+ "title": "KeyframeRange",+ "type": "object",+ "properties": {+ "start_seconds": {+ "description": "Start of the edit window in whole seconds from the start of the input video.",+ "type": "integer",+ "minimum": 0,+ "maximum": 9007199254740991+ },+ "end_seconds": {+ "description": "End of the edit window (exclusive) in whole seconds from the start of the input video.",+ "type": "integer",+ "exclusiveMinimum": 0,+ "maximum": 9007199254740991+ }+ },+ "required": [+ "start_seconds",+ "end_seconds"+ ],+ "additionalProperties": false+ }+ },+ "required": [+ "uri",+ "at"+ ],+ "additionalProperties": false+ }+ ]+ }+ },+ "videoUri": {+ "description": "The input video to edit.\nMust be 30 seconds or shorter.",+ "example": "https://example.com/video.mp4",+ "anyOf": [+ {+ "description": "A HTTPS URL, Runway upload URI, or base64 data URI (e.g. `data:video/mp4;base64,...`, up to 5MB) containing an encoded video. See [our docs](/assets/inputs#videos) on video inputs for more information.",+ "example": "https://example.com/video.mp4",+ "type": "string",+ "minLength": 13,+ "maxLength": 2048,+ "pattern": "^https:\\/\\/.*"+ },+ {+ "description": "A Runway upload URI.\nSee https://docs.dev.runwayml.com/assets/uploads for more information.",+ "example": "runway://<token>",+ "type": "string",+ "minLength": 13,+ "maxLength": 5000,+ "pattern": "^runway:\\/\\/.*"+ },+ {+ "description": "A data URI containing encoded media, up to 5MB.",+ "example": "data:image/jpeg;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",+ "type": "string",+ "minLength": 13,+ "maxLength": 5242880,+ "pattern": "^data:video\\/.*"+ }+ ]+ },+ "seed": {+ "description": "If unspecified, a random number is chosen.\nVarying the seed integer is a way to get different results for the same other request parameters. Using the same seed integer for an identical request will produce similar results.",+ "type": "integer",+ "minimum": 0,+ "maximum": 4294967295+ },+ "ratio": {+ "deprecated": true,+ "type": "string"+ },+ "targetAspectRatio": {+ "description": "Target aspect ratio for expand/outpaint.\nLetterboxes the input video and keyframes before generation.",+ "type": "string",+ "enum": [+ "16:9",+ "4:3",+ "3:2",+ "1:1",+ "2:3",+ "3:4",+ "9:16",+ "21:9"+ ]+ },+ "contentModeration": {+ "description": "Settings that affect the behavior of the content moderation system.",+ "type": "object",+ "properties": {+ "publicFigureThreshold": {+ "description": "When set to `low`, the content moderation system will be less strict about preventing generations that include recognizable public figures.",+ "type": "string",+ "enum": [+ "auto",+ "low"+ ]+ }+ },+ "additionalProperties": false+ },+ "outputFormat": {+ "description": "The container/encoding of the output.\n`mp4` (default) returns an H.264 .mp4. `prores` returns a ProRes .mov. `png_sequence` returns a .zip of PNG frames. `sdr_rec709_10bit` returns a 10-bit Rec.709 HEVC .mp4 for SDR grading pipelines.\nNon-mp4 formats incur an additional surcharge: 5 credits per second for `prores` and `png_sequence`, and 20 credits per second for `sdr_rec709_10bit` — 40 credits per second when the output is larger than 4 megapixels — includes anything larger than 1440p, up through 4K.",+ "type": "string",+ "enum": [+ "mp4",+ "prores",+ "png_sequence",+ "sdr_rec709_10bit"+ ]+ },+ "proresProfile": {+ "description": "The ProRes profile to use. Only valid when `outputFormat` is `prores`. Defaults to `4444`.\nNote: generated content contains no transparency — the alpha channel in `4444` / `4444 XQ` outputs is present but fully opaque; choose these tiers for 12-bit 4:4:4 color fidelity, not for mattes.",+ "type": "string",+ "enum": [+ "422",+ "4444",+ "422 Proxy",+ "422 LT",+ "422 HQ",+ "4444 XQ"+ ]+ },+ "model": {+ "type": "string",+ "const": "aleph2"+ }+ },+ "required": [+ "videoUri",+ "model"+ ],+ "additionalProperties": false+ }+ ]+ }+ }+ }+ }+ }+ },+ "/v1/character_performance": {+ "post": {+ "requestBody": {+ "content": {+ "application/json": {+ "schema": {+ "oneOf": [+ {+ "title": "act_two",+ "type": "object",+ "properties": {+ "seed": {+ "description": "If unspecified, a random number is chosen.\nVarying the seed integer is a way to get different results for the same other request parameters. Using the same seed integer for an identical request will produce similar results.",+ "type": "integer",+ "minimum": 0,+ "maximum": 4294967295+ },+ "character": {+ "description": "The character to control. You can either provide a video or an image. A visually recognizable face must be visible and stay within the frame.",+ "oneOf": [+ {+ "title": "CharacterImage",+ "description": "An image of your character.\nIn the output, the character will use the reference video performance in its original static environment.",+ "type": "object",+ "properties": {+ "type": {+ "type": "string",+ "const": "image"+ },+ "uri": {+ "description": "A HTTPS URL, Runway or data URI containing an encoded image. See [our docs](/assets/inputs#images) on image inputs for more information.",+ "example": "https://example.com/image.jpg",+ "anyOf": [+ {+ "description": "A HTTPS URL, Runway upload URI, or base64 data URI (e.g.\n`data:image/png;base64,...`, up to 5MB) containing an encoded image. See [our docs](/assets/inputs#images) on image inputs for more information.",+ "example": "https://example.com/image.jpg",+ "type": "string",+ "minLength": 13,+ "maxLength": 2048,+ "pattern": "^https:\\/\\/.*"+ },+ {+ "description": "A Runway upload URI.\nSee https://docs.dev.runwayml.com/assets/uploads for more information.",+ "example": "runway://<token>",+ "type": "string",+ "minLength": 13,+ "maxLength": 5000,+ "pattern": "^runway:\\/\\/.*"+ },+ {+ "description": "A data URI containing encoded media, up to 5MB.",+ "example": "data:image/jpeg;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",+ "type": "string",+ "minLength": 13,+ "maxLength": 5242880,+ "pattern": "^data:image\\/.*"+ }+ ]+ }+ },+ "required": [+ "type",+ "uri"+ ]+ },+ {+ "title": "CharacterVideo",+ "description": "A video of your character.\nIn the output, the character will use the reference video performance in its original animated environment and some of the character's own movements.",+ "type": "object",+ "properties": {+ "type": {+ "type": "string",+ "const": "video"+ },+ "uri": {+ "description": "A HTTPS URL, Runway or data URI containing an encoded video. See [our docs](/assets/inputs#videos) on video inputs for more information.",+ "example": "https://example.com/video.mp4",+ "anyOf": [+ {+ "description": "A HTTPS URL, Runway upload URI, or base64 data URI (e.g.\n`data:video/mp4;base64,...`, up to 5MB) containing an encoded video. See [our docs](/assets/inputs#videos) on video inputs for more information.",+ "example": "https://example.com/video.mp4",+ "type": "string",+ "minLength": 13,+ "maxLength": 2048,+ "pattern": "^https:\\/\\/.*"+ },+ {+ "description": "A Runway upload URI.\nSee https://docs.dev.runwayml.com/assets/uploads for more information.",+ "example": "runway://<token>",+ "type": "string",+ "minLength": 13,+ "maxLength": 5000,+ "pattern": "^runway:\\/\\/.*"+ },+ {+ "description": "A data URI containing encoded media, up to 5MB.",+ "example": "data:image/jpeg;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",+ "type": "string",+ "minLength": 13,+ "maxLength": 5242880,+ "pattern": "^data:video\\/.*"+ }+ ]+ }+ },+ "required": [+ "type",+ "uri"+ ]+ }+ ],+ "discriminator": {+ "propertyName": "type"+ }+ },+ "reference": {+ "description": "The reference video containing the performance to apply to the character.",+ "example": {+ "type": "video",+ "uri": "https://example.com/reference-performance.mp4"+ },+ "oneOf": [+ {+ "title": "CharacterReferenceVideo",+ "description": "A video of a person performing in the manner that you would like your character to perform.\nThe video must be between 3 and 30 seconds in duration.",+ "type": "object",+ "properties": {+ "type": {+ "type": "string",+ "const": "video"+ },+ "uri": {+ "description": "A video of a person performing in the manner that you would like your character to perform. The video must be between 3 and 30 seconds in duration.\nSee [our docs](/assets/inputs#videos) on video inputs for more information.",+ "example": "https://example.com/reference-performance.mp4",+ "anyOf": [+ {+ "description": "A HTTPS URL, Runway upload URI, or base64 data URI (e.g. `data:video/mp4;base64,...`, up to 5MB) containing an encoded video.\nSee [our docs](/assets/inputs#videos) on video inputs for more information.",+ "example": "https://example.com/video.mp4",+ "type": "string",+ "minLength": 13,+ "maxLength": 2048,+ "pattern": "^https:\\/\\/.*"+ },+ {+ "description": "A Runway upload URI.\nSee https://docs.dev.runwayml.com/assets/uploads for more information.",+ "example": "runway://<token>",+ "type": "string",+ "minLength": 13,+ "maxLength": 5000,+ "pattern": "^runway:\\/\\/.*"+ },+ {+ "description": "A data URI containing encoded media, up to 5MB.",+ "example": "data:image/jpeg;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",+ "type": "string",+ "minLength": 13,+ "maxLength": 5242880,+ "pattern": "^data:video\\/.*"+ }+ ]+ }+ },+ "required": [+ "type",+ "uri"+ ]+ }+ ],+ "discriminator": {+ "propertyName": "type"+ }+ },+ "bodyControl": {+ "description": "A boolean indicating whether to enable body control.\nWhen enabled, non-facial movements and gestures will be applied to the character in addition to facial expressions.",+ "example": true,+ "type": "boolean"+ },+ "expressionIntensity": {+ "description": "An integer between 1 and 5 (inclusive).\nA larger value increases the intensity of the character's expression.",+ "example": 3,+ "default": 3,+ "type": "integer",+ "minimum": 1,+ "maximum": 5+ },+ "ratio": {+ "description": "The resolution of the output video.",+ "example": "1280:720",+ "type": "string",+ "enum": [+ "1280:720",+ "720:1280",+ "960:960",+ "1104:832",+ "832:1104",+ "1584:672"+ ]+ },+ "contentModeration": {+ "description": "Settings that affect the behavior of the content moderation system.",+ "example": {+ "publicFigureThreshold": "auto"+ },+ "type": "object",+ "properties": {+ "publicFigureThreshold": {+ "description": "When set to `low`, the content moderation system will be less strict about preventing generations that include recognizable public figures.",+ "type": "string",+ "enum": [+ "auto",+ "low"+ ]+ }+ },+ "additionalProperties": false+ },+ "model": {+ "type": "string",+ "const": "act_two"+ }+ },+ "required": [+ "character",+ "reference",+ "model"+ ]+ }+ ]+ }+ }+ }+ }+ }}}-}\ No newline at end of file+}diff --git a/server/services/runwayApi.js b/server/services/runwayApi.jsindex 0df950bfe7..e7c9c71e89 100644--- a/server/services/runwayApi.js+++ b/server/services/runwayApi.js@@ -17,6 +17,8 @@ import { assertRunwayCatalogModel } from './runwayModelCatalog.js';import { maintenance } from '../lib/maintenanceAdmission.js';import { ServerError } from '../lib/errorHandler.js';import { fetchWithTimeout } from '../lib/fetchWithTimeout.js';+import { openAsBlob } from 'node:fs';+import { basename } from 'node:path';export const RUNWAY_API_BASE = 'https://api.dev.runwayml.com';// Every request names the API version it was written against.@@ -41,6 +43,33 @@ const headers = (apiKey, json = false) => ({...(json ? { 'Content-Type': 'application/json' } : {}),});+/** Upload just before dispatch: ephemeral URIs expire after 24h and never enter queue params. */+export async function uploadRunwayVideo({ apiKey, filePath, signal }) {+ signal?.throwIfAborted();+ const file = await openAsBlob(filePath);+ if (file.size < 512 || file.size > 200 * 1024 * 1024) throw new Error('Runway uploads must be 512 bytes–200 MB.');+ const registration = await fetchWithTimeout(`${RUNWAY_API_BASE}/v1/uploads`, {+ method: 'POST', headers: headers(apiKey, true), signal,+ body: JSON.stringify({ filename: basename(filePath), type: 'ephemeral' }),+ }, RUNWAY_SUBMIT_TIMEOUT_MS);+ if (!registration.ok) throw new Error(`Runway upload registration failed: HTTP ${registration.status}`);+ const data = await registration.json();+ if (typeof data.uploadUrl !== 'string' || !data.uploadUrl.startsWith('https://')+ || typeof data.runwayUri !== 'string' || !data.runwayUri.startsWith('runway://')+ || !data.fields || typeof data.fields !== 'object' || Array.isArray(data.fields)+ || Object.values(data.fields).some((value) => typeof value !== 'string')) throw new Error('Invalid Runway upload registration.');+ signal?.throwIfAborted();+ const form = new FormData();+ for (const [key, value] of Object.entries(data.fields)) form.append(key, value);+ form.append('file', file, basename(filePath));+ // The signed storage fields authorize this POST. Never send the API key to storage.+ const uploaded = await fetchWithTimeout(data.uploadUrl, { method: 'POST', body: form, signal }, 10 * 60 * 1000);+ await uploaded.body?.cancel?.();+ if (!uploaded.ok) throw new Error(`Runway media upload failed: HTTP ${uploaded.status}`);+ signal?.throwIfAborted();+ return data.runwayUri;+}+/** A short, single-line reason from a Runway error body (`error`, `issues`), or ''. */function runwayErrorDetail(payload) {if (!payload || typeof payload !== 'object') return '';diff --git a/server/services/runwayModelCatalog.js b/server/services/runwayModelCatalog.jsindex 342669923f..b454d56249 100644--- a/server/services/runwayModelCatalog.js+++ b/server/services/runwayModelCatalog.js@@ -4,6 +4,7 @@ import { createHash } from 'node:crypto';import { PATHS } from '../lib/paths.js';import { fetchWithTimeout } from '../lib/fetchWithTimeout.js';import { RUNWAY_CATALOG_SNAPSHOT } from '../lib/runwayCatalogSnapshot.js';+import { runwaySpecialSchemaContract } from '../lib/runwaySpecialVideoModels.js';export const RUNWAY_CATALOG_SOURCE = 'https://docs.dev.runwayml.com/openapi.json';const TTL = 24 * 60 * 60 * 1000;@@ -11,12 +12,10 @@ const RETRY = 5 * 60 * 1000;// A new shipped snapshot invalidates even a fresh disk cache from an older install.const VERSION = createHash('sha256').update(JSON.stringify(RUNWAY_CATALOG_SNAPSHOT)).digest('hex');const fallback = Object.fromEntries(Object.entries(RUNWAY_CATALOG_SNAPSHOT).map(([endpoint, rows]) => [endpoint, rows.map(({ id }) => id)]));-const limitations = [- { id: 'aleph2', reason: 'Requires video-to-video editing inputs; not supported by these text/image forms.' },- { id: 'act_two', reason: 'Requires character and reference performance inputs; not supported by these forms.' },-];+const limitations = [];function compatibleSchema(row, expected) {+ if (expected.contract) return JSON.stringify(runwaySpecialSchemaContract(row)) === JSON.stringify(expected.contract);const p = row.properties;const required = row.required || [];const allowed = ['model', 'promptText', 'ratio', 'duration', 'promptImage', 'referenceImages'];@@ -67,7 +66,7 @@ function parseCatalog(spec) {/** Public documentation, never an invented /models API or an authenticated generation probe.* One bounded, machine-local disposable cache; no user data, credentials or federation.*/-export function createRunwayModelCatalog({+export function _createRunwayModelCatalog({fetchSchema = () => fetchWithTimeout(RUNWAY_CATALOG_SOURCE, {}, 15_000),cachePath = join(PATHS.installRoot, '.cache', 'runway-model-catalog.json'), now = Date.now,} = {}) {@@ -127,7 +126,7 @@ export function createRunwayModelCatalog({},};}-const catalog = createRunwayModelCatalog();+const catalog = _createRunwayModelCatalog();export const getRunwayModelCatalog = (options) => catalog.get(options);/** Refreshed removals/schema changes fail before a paid submission. No catalog lookup per render. */diff --git a/server/services/runwayModelCatalog.test.js b/server/services/runwayModelCatalog.test.jsindex 7690312e06..4d66280b97 100644--- a/server/services/runwayModelCatalog.test.js+++ b/server/services/runwayModelCatalog.test.js@@ -2,7 +2,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest';import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';import { join } from 'node:path';import { tmpdir } from 'node:os';-import { createRunwayModelCatalog } from './runwayModelCatalog.js';+import { _createRunwayModelCatalog } from './runwayModelCatalog.js';// Real public OpenAPI excerpts (2026-10-22), descriptions removed; no generation endpoint is called.const fixture = JSON.parse(await readFile(new URL('./fixtures/runwayOpenApi.json', import.meta.url), 'utf8'));@@ -15,7 +15,7 @@ async function setup() {let time = 1_000_000;const fetchSchema = vi.fn(async () => ({ ok: true, json: async () => structuredClone(fixture) }));const options = { cachePath, fetchSchema, now: () => time };- return { cache: createRunwayModelCatalog(options), options, fetchSchema, advance: (ms) => { time += ms; } };+ return { cache: _createRunwayModelCatalog(options), options, fetchSchema, advance: (ms) => { time += ms; } };}describe('Runway public catalog cache', () => {it('coalesces form loads, survives restart, expires and honors explicit refresh/removals', async () => {@@ -24,9 +24,11 @@ describe('Runway public catalog cache', () => {expect(first).toEqual(concurrent);expect(first.available.text_to_image).toEqual(['gen4_image', 'gpt_image_2']);expect(first.available.text_to_video).not.toContain('gen4_turbo');+ expect(first.available.video_to_video).toEqual(['aleph2']);+ expect(first.available.character_performance).toEqual(['act_two']);expect(fetchSchema).toHaveBeenCalledTimes(1);await cache.get();- await createRunwayModelCatalog(options).get();+ await _createRunwayModelCatalog(options).get();expect(fetchSchema).toHaveBeenCalledTimes(1);advance(24 * 60 * 60 * 1000);await cache.get();@@ -63,7 +65,7 @@ describe('Runway public catalog cache', () => {expect(fetchSchema).toHaveBeenCalledTimes(1);await writeFile(options.cachePath, JSON.stringify({ version: 'old', fetchedAt: 1_000_000, catalog: { available: {} } }));fetchSchema.mockResolvedValue({ ok: true, json: async () => fixture });- expect(await createRunwayModelCatalog(options).get()).toMatchObject({ bundled: false, stale: false });+ expect(await _createRunwayModelCatalog(options).get()).toMatchObject({ bundled: false, stale: false });});it('does not claim unknown or changed request schemas are runnable and preserves forced refresh during disk load', async () => {const { cache, options, fetchSchema } = await setup();@@ -71,9 +73,10 @@ describe('Runway public catalog cache', () => {const changed = structuredClone(fixture);const rows = changed.paths['/v1/text_to_image'].post.requestBody.content['application/json'].schema.oneOf;delete rows[0].properties.referenceImages.items.properties.tag;+ changed.paths['/v1/character_performance'].post.requestBody.content['application/json'].schema.oneOf[0].properties.expressionIntensity.maximum = 4;rows.push({ properties: { model: { const: 'future_image' } } });fetchSchema.mockResolvedValue({ ok: true, json: async () => changed });- const restarted = createRunwayModelCatalog(options);+ const restarted = _createRunwayModelCatalog(options);const load = restarted.get();const forced = restarted.get({ refresh: true });const forcedAgain = restarted.get({ refresh: true });@@ -82,6 +85,7 @@ describe('Runway public catalog cache', () => {expect(await forcedAgain).toEqual(updated);expect(fetchSchema).toHaveBeenCalledTimes(2);expect(updated.available.text_to_image).toEqual(['gpt_image_2']);- expect(updated.unsupported.map((r) => r.id)).toEqual(expect.arrayContaining(['gen4_image', 'future_image', 'aleph2', 'act_two']));+ expect(updated.available.character_performance).toEqual([]);+ expect(updated.unsupported.map((r) => r.id)).toEqual(expect.arrayContaining(['gen4_image', 'future_image']));});});diff --git a/server/services/videoGen/hostedSubmission.js b/server/services/videoGen/hostedSubmission.jsindex 5c39c97556..5094b392c4 100644--- a/server/services/videoGen/hostedSubmission.js+++ b/server/services/videoGen/hostedSubmission.js@@ -10,6 +10,8 @@ import { FAL_DEFAULT_IMAGE_VIDEO_MODEL, FAL_DEFAULT_TEXT_VIDEO_MODEL, getFalVideimport { RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL, getRunwayVideoModel, runwayModelTakesText } from '../../lib/runwayVideoModels.js';import { isDefaultI2vReferenceMode } from '../../lib/videoReferenceModes.js';import { VIDEO_GEN_MODE, isVideoModeUsable } from './modes.js';+import { getRunwaySpecialVideoModel, buildRunwaySpecialVideoRequest } from '../../lib/runwaySpecialVideoModels.js';+import { validateRunwayVideoInput } from './runwayVideoInputs.js';// Every hosted backend anchors a reference image as frame one, so a loose// reference mode is a local-runtime promise it can never keep. Submission@@ -68,6 +70,24 @@ async function stageRunwayInputs(ctx) {throw new ServerError(message, { status: 400, code: 'VALIDATION_ERROR' });};if (uploads.audioFile) await refuse('Runway renders cutaways only; it takes no source audio.');+ const special = getRunwaySpecialVideoModel(body.runwayModelId);+ if (special) {+ if ((uploads.sourceImage || body.sourceImageFile) && !sourceImagePath) await refuse('The requested Runway guidance or character image could not be resolved.');+ buildRunwaySpecialVideoRequest({+ modelId: special.id, videoMode: body.mode, prompt: body.prompt, negativePrompt: body.negativePrompt,+ videoUri: uploads.runwayVideo ? 'pending:video' : null,+ characterVideoUri: uploads.runwayCharacterVideo ? 'pending:character' : null,+ imageUri: sourceImagePath, endImageUri: uploads.lastImage || body.lastImageFile,+ bodyControl: body.runwayBodyControl, expressionIntensity: body.runwayExpressionIntensity,+ targetAspectRatio: body.runwayTargetAspect, ratio: body.runwayRatio, seed: body.runwaySeed,+ });+ const runwayVideoPath = await ctx.stageUploadDurable(uploads.runwayVideo, 'runway-video');+ await validateRunwayVideoInput(runwayVideoPath, special);+ const runwayCharacterVideoPath = uploads.runwayCharacterVideo ? await ctx.stageUploadDurable(uploads.runwayCharacterVideo, 'runway-character') : null;+ if (runwayCharacterVideoPath) await validateRunwayVideoInput(runwayCharacterVideoPath, special, { character: true });+ return { runwayVideoPath, runwayCharacterVideoPath, uploadedTempPaths: [runwayVideoPath, runwayCharacterVideoPath].filter(Boolean) };+ }+ if (uploads.runwayVideo || uploads.runwayCharacterVideo || !['text', 'image', undefined].includes(body.mode)) await refuse('Choose the Runway model matching these video inputs.');const model = getRunwayVideoModel(body.runwayModelId || RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL);if (!model) await refuse(`"${body.runwayModelId}" is not a Runway model PortOS offers`);if (!sourceImagePath && !runwayModelTakesText(model)) await refuse(`${model.label} renders from a start frame — supply a source image.`);@@ -172,6 +192,10 @@ export const HOSTED_VIDEO_SUBMISSIONS = {// The model, length and size are resolved against the curated catalog by videoGen/runway.js.buildParams: (body, prepared) => ({modelId: body.runwayModelId,+ videoMode: body.mode,+ ...(prepared.runwayVideoPath ? { runwayVideoPath: prepared.runwayVideoPath, runwayCharacterVideoPath: prepared.runwayCharacterVideoPath } : {}),+ targetAspectRatio: body.runwayTargetAspect, ratio: body.runwayRatio,+ bodyControl: body.runwayBodyControl, expressionIntensity: body.runwayExpressionIntensity, seed: body.runwaySeed,...(prepared.lastImagePath ? { lastImagePath: prepared.lastImagePath } : {}),...(prepared.uploadedTempPaths?.length ? { uploadedTempPaths: prepared.uploadedTempPaths } : {}),aspectRatio: body.visualConditioning?.render?.parameters?.aspectRatio,diff --git a/server/services/videoGen/prepareParams.js b/server/services/videoGen/prepareParams.jsindex 7769a413ce..02770716fc 100644--- a/server/services/videoGen/prepareParams.js+++ b/server/services/videoGen/prepareParams.js@@ -79,6 +79,7 @@ import { videoModeContractError, videoChainUnsupportedError, videoReferenceModeEimport { resolveByovRuntimeLoraCapable, videoLoraUnsupportedError } from './runtimes.js';import { validateVideoBatch } from './batch.js';import { audioDurationToFrames } from './audioDuration.js';+import { RUNWAY_SPECIAL_VIDEO_MODES } from '../../lib/runwaySpecialVideoModels.js';// Retries reuse persisted worker parameters instead of passing through the// multipart preparation path. Keep the model/mode gates here so a model edit@@ -286,6 +287,11 @@ export function resolveRequestedVideoBackend({ body, uploads, settings, localOnlexport async function prepareVideoGenParams({ body, uploads, localOnlyParamKeys }) {const settings = await getSettings();const backend = resolveRequestedVideoBackend({ body, uploads, settings, localOnlyParamKeys });+ const runwaySpecial = backend === VIDEO_GEN_MODE.RUNWAY && RUNWAY_SPECIAL_VIDEO_MODES.includes(body.mode);+ if (!runwaySpecial && (RUNWAY_SPECIAL_VIDEO_MODES.includes(body.mode) || uploads.runwayVideo || uploads.runwayCharacterVideo)) {+ await cleanupMultipartTemp(uploads);+ throw new ServerError('Runway video inputs require the matching Runway video workflow.', { status: 400, code: 'VALIDATION_ERROR' });+ }// A loose reference (#4874) is a LOCAL-runtime capability. The model gate// further down asks whether the chosen model's runtime can honor it — but an// explicit `backend: 'grok'` never reaches that runtime at all: grok's@@ -427,7 +433,7 @@ export async function prepareVideoGenParams({ body, uploads, localOnlyParamKeys// rejection point at once, and stays correct as new awaits are added.return withStagedRollback(cleanupStaged, () => resolvePreparedParams({body, uploads, settings, backend, pythonPath,- effectiveModel, effectiveModelId, stagedDurablePaths, cleanupStaged, musicVideoContractMode,+ effectiveModel: runwaySpecial ? null : effectiveModel, effectiveModelId, stagedDurablePaths, cleanupStaged, musicVideoContractMode,}));}@@ -435,7 +441,7 @@ export async function prepareVideoGenParams({ body, uploads, localOnlyParamKeys// try/catch so a permission/disk failure here still cleans up the// multipart temp uploads instead of leaking them in the OS temp dir.async function ensureUploadsDir({ uploads, cleanupStaged }) {- if (!(uploads.sourceImage || uploads.lastImage || uploads.audioFile || uploads.icReference)) return;+ if (!(uploads.sourceImage || uploads.lastImage || uploads.audioFile || uploads.icReference || uploads.runwayVideo || uploads.runwayCharacterVideo)) return;try {await ensureDir(PATHS.uploads);} catch (err) {diff --git a/server/services/videoGen/promptPreview.js b/server/services/videoGen/promptPreview.jsindex 9689b21828..192fdc4ee6 100644--- a/server/services/videoGen/promptPreview.js+++ b/server/services/videoGen/promptPreview.js@@ -18,7 +18,7 @@ import { VIDEO_GEN_LOCAL_ONLY_FIELD_NAMES } from './requestFields.js';import { grokVideoSentPrompt } from './grok.js';import { falVideoSentPrompt } from './fal.js';-export const PREVIEW_UPLOAD_FIELDS = ['sourceImage', 'lastImage', 'audioFile', 'icReference'];+export const PREVIEW_UPLOAD_FIELDS = ['sourceImage', 'lastImage', 'audioFile', 'icReference', 'runwayVideo', 'runwayCharacterVideo'];const UPLOADED_SOURCE_IMAGE = '(uploaded image)';const asWritten = (body, extra = {}) => ({@@ -70,6 +70,9 @@ export async function previewVideoPrompt({ previewUploads = [], ...body }) {const { runwayVideoSentPrompt } = await import('./runway.js');return { backend, ...runwayVideoSentPrompt({modelId: body.runwayModelId, prompt: body.prompt, negativePrompt: body.negativePrompt,+ videoMode: body.mode, hasVideo: !!uploads.runwayVideo, hasCharacterVideo: !!uploads.runwayCharacterVideo,+ targetAspectRatio: body.runwayTargetAspect, ratio: body.runwayRatio,+ bodyControl: body.runwayBodyControl, expressionIntensity: body.runwayExpressionIntensity, seed: body.runwaySeed,seconds: body.runwayDuration, width: body.width, height: body.height, resolution: body.runwayResolution, promptAdditions,hasImage: hasSourceImage, hasEndImage: !!uploads.lastImage || !!body.lastImageFile,}).record };diff --git a/server/services/videoGen/runway.js b/server/services/videoGen/runway.jsindex 47a9e34619..d5ea3d81e4 100644--- a/server/services/videoGen/runway.js+++ b/server/services/videoGen/runway.js@@ -25,6 +25,8 @@ import { fetchWithTimeout } from '../../lib/fetchWithTimeout.js';import { anyAbortSignal } from '../../lib/requestAbort.js';import { withAbortTimeout } from '../../lib/abortTimeout.js';import { RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL, buildRunwayVideoRequest } from '../../lib/runwayVideoModels.js';+import { getRunwaySpecialVideoModel, buildRunwaySpecialVideoRequest } from '../../lib/runwaySpecialVideoModels.js';+import { validateRunwayVideoInput } from './runwayVideoInputs.js';import { probeVideoGeometry } from '../../lib/ffmpeg.js';import { attachSseClient as attachSse, closeJobAfterDelay, createJobFailureFinalizer } from '../../lib/sseUtils.js';import { videoGenEvents } from './events.js';@@ -34,7 +36,7 @@ import { mutateVideoHistory } from './history.js';import { getSettings } from '../settings.js';import { frameDataUriAtAspect } from './falFrameAspect.js';import {- awaitRunwayTask, cancelRunwayTask, createRunwayTaskEntry, resolveRunwayApiKey, submitRunwayTask,+ awaitRunwayTask, cancelRunwayTask, createRunwayTaskEntry, resolveRunwayApiKey, submitRunwayTask, uploadRunwayVideo,} from '../runwayApi.js';import { carriedPromptAdditions, promptAddition, sentPromptRecord } from '../../lib/modelInput.js';@@ -91,11 +93,14 @@ export const cancelAll = () => {* the plan and the record of the text Runway receives (#11317); the Video Gen* form's preview calls it with the same inputs.*/-export function runwayVideoSentPrompt({ hasImage = false, hasEndImage = false, promptAdditions = [], ...spec }) {+export function runwayVideoSentPrompt({ hasImage = false, hasEndImage = false, hasVideo = false, hasCharacterVideo = false, promptAdditions = [], ...spec }) {let planned;try {- planned = buildRunwayVideoRequest({+ const build = getRunwaySpecialVideoModel(spec.modelId) ? buildRunwaySpecialVideoRequest : buildRunwayVideoRequest;+ planned = build({...spec,+ videoUri: hasVideo ? 'pending:video' : null,+ characterVideoUri: hasCharacterVideo ? 'pending:character-video' : null,imageUri: hasImage ? 'pending:image' : null,endImageUri: hasEndImage ? 'pending:end-image' : null,});@@ -141,6 +146,7 @@ async function generateVideoPrepared({sourceImagePath = null, lastImagePath = null, jobId: providedJobId = null,uploadedTempPath = null, uploadedTempPaths = [], audioFilePath = null,resolution = null, seed, frameAspect = null, shotInstruction = null, promptAdditions = [],+ videoMode, runwayVideoPath = null, runwayCharacterVideoPath = null, targetAspectRatio, ratio, bodyControl, expressionIntensity,}) {await ensureDir(PATHS.videos);const renderStartedAtMs = Date.now();@@ -148,6 +154,8 @@ async function generateVideoPrepared({throw new ServerError('Runway renders cutaways only; it takes no source audio', { status: 400, code: 'VALIDATION_ERROR' });}const modelId = requestedModelId || RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL;+ const special = getRunwaySpecialVideoModel(modelId);+ if (!special && (runwayVideoPath || runwayCharacterVideoPath)) throw new ServerError('This Runway model does not take video inputs.', { status: 400, code: 'VALIDATION_ERROR' });const effectiveSettings = settings || (providedApiKey ? null : await getSettings().catch(() => null));// Re-resolved from live settings, never threaded through persisted job params.const apiKey = providedApiKey || resolveRunwayApiKey(effectiveSettings);@@ -156,10 +164,11 @@ async function generateVideoPrepared({}const requestSpec = {modelId, prompt, negativePrompt, seconds: duration, aspectRatio: frameAspect || aspectRatio, width, height, resolution,+ videoMode, targetAspectRatio, ratio, bodyControl, expressionIntensity,...(Number.isInteger(seed) ? { seed } : {}),};const { planned, record: sentRecord } = runwayVideoSentPrompt({- ...requestSpec, promptAdditions, hasImage: !!sourceImagePath, hasEndImage: !!lastImagePath,+ ...requestSpec, promptAdditions, hasImage: !!sourceImagePath, hasEndImage: !!lastImagePath, hasVideo: !!runwayVideoPath, hasCharacterVideo: !!runwayCharacterVideoPath,});const jobId = providedJobId || randomUUID();@@ -178,7 +187,7 @@ async function generateVideoPrepared({...sentRecord,filename,createdAt: new Date().toISOString(),- mode: sourceImagePath ? 'image' : 'text',+ mode: special?.mode || (sourceImagePath ? 'image' : 'text'),...(shotInstruction?.audioWindow ? { audioWindow: shotInstruction.audioWindow } : {}),};const job = { ...meta, clients: [], status: 'running', renderStartedAtMs };@@ -193,6 +202,7 @@ async function generateVideoPrepared({runRunwayVideo(job, jobId, {apiKey, requestSpec, sourceImagePath, lastImagePath, outputPath, filename, meta,uploadedTempPath, uploadedTempPaths, frameAspect,+ runwayVideoPath, runwayCharacterVideoPath,}).catch((err) => {console.error(`❌ Runway video run failed [${jobId.slice(0, 8)}]: ${err?.message}`);});@@ -221,6 +231,7 @@ async function downloadRunwayOutput(url, signal) {async function runRunwayVideo(job, jobId, {apiKey, requestSpec, sourceImagePath, lastImagePath, outputPath, filename, meta,uploadedTempPath = null, uploadedTempPaths = [], frameAspect = null,+ runwayVideoPath = null, runwayCharacterVideoPath = null,}) {const entry = createRunwayTaskEntry(apiKey);entry.aborted = preparations.take(jobId);@@ -246,6 +257,14 @@ async function runRunwayVideo(job, jobId, {activeRequests.set(jobId, entry);const deadline = Date.now() + RUNWAY_RENDER_TIMEOUT_MS;try {+ if (entry.aborted) return finishFailure('Canceled');+ const special = getRunwaySpecialVideoModel(requestSpec.modelId);+ if (special) {+ const geometry = await validateRunwayVideoInput(runwayVideoPath, special);+ if (runwayCharacterVideoPath) await validateRunwayVideoInput(runwayCharacterVideoPath, special, { character: true });+ meta.duration = geometry.durationSec;+ job.duration = geometry.durationSec;+ }const frame = sourceImagePath ? await runwayFrameDataUri(sourceImagePath, frameAspect) : null;const endFrame = lastImagePath ? await runwayFrameDataUri(lastImagePath, frameAspect) : null;for (const [label, fit] of [['start', frame], ['end', endFrame]]) {@@ -255,7 +274,10 @@ async function runRunwayVideo(job, jobId, {if (frame?.conformed || endFrame?.conformed) {meta.frameConformed = { aspect: frameAspect, ...(frame?.conformed ? { start: frame.conformed } : {}), ...(endFrame?.conformed ? { end: endFrame.conformed } : {}) };}- const { body, endpoint } = buildRunwayVideoRequest({ ...requestSpec, imageUri: frame?.dataUri ?? null, endImageUri: endFrame?.dataUri ?? null });+ const videoUri = runwayVideoPath ? await uploadRunwayVideo({ apiKey, filePath: runwayVideoPath, signal: entry.controller.signal }) : null;+ const characterVideoUri = runwayCharacterVideoPath ? await uploadRunwayVideo({ apiKey, filePath: runwayCharacterVideoPath, signal: entry.controller.signal }) : null;+ const build = special ? buildRunwaySpecialVideoRequest : buildRunwayVideoRequest;+ const { body, endpoint } = build({ ...requestSpec, videoUri, characterVideoUri, imageUri: frame?.dataUri ?? null, endImageUri: endFrame?.dataUri ?? null });if (entry.aborted) return finishFailure('Canceled');entry.taskId = await submitRunwayTask({ apiKey, endpoint, body });diff --git a/server/services/videoGen/runway.test.js b/server/services/videoGen/runway.test.jsindex 9d7587f462..1542cc5d82 100644--- a/server/services/videoGen/runway.test.js+++ b/server/services/videoGen/runway.test.js@@ -1,5 +1,5 @@import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';-import { mkdir, rm, readFile } from 'fs/promises';+import { mkdir, rm, readFile, writeFile, stat } from 'fs/promises';import { join } from 'path';import { tmpdir } from 'os';import sharp from 'sharp';@@ -26,7 +26,7 @@ vi.mock('../../lib/ffmpeg.js', async () => {...actual,optimizeForStreaming: vi.fn(async () => {}),generateThumbnail: vi.fn(async (_p, jobId) => `${jobId}.jpg`),- probeVideoGeometry: vi.fn(async () => ({ duration: 4 })),+ probeVideoGeometry: vi.fn(async () => ({ width: 1280, height: 720, fps: 24, durationSec: 4, numFrames: 96 })),};});@@ -37,6 +37,7 @@ const runway = await import('./runway.js');const { RUNWAY_API_VERSION } = await import('../runwayApi.js');const { videoGenEvents } = await import('./events.js');const { loadHistory } = await import('./history.js');+const { probeVideoGeometry } = await import('../../lib/ffmpeg.js');const terminalEvents = new Map();const waitForTerminal = async (jobId) => {@@ -60,6 +61,149 @@ beforeEach(async () => {framePath = join(TEST_ROOT, 'frame.png');await sharp({ create: { width: 64, height: 36, channels: 3, background: '#336699' } }).png().toFile(framePath);getSettingsMock.mockReset().mockResolvedValue({});+ probeVideoGeometry.mockReset().mockResolvedValue({ width: 1280, height: 720, fps: 24, durationSec: 4, numFrames: 96 });+});++describe('Runway video-input workflows', () => {+ const stagedVideo = async () => {+ await mkdir(join(TEST_ROOT, 'uploads'), { recursive: true });+ const path = join(TEST_ROOT, 'uploads', 'source.mp4');+ await writeFile(path, Buffer.alloc(1024));+ return path;+ };+ it('cancels during preparation without uploading private inputs', async () => {+ const video = await stagedVideo();+ let releaseSettings;+ getSettingsMock.mockImplementationOnce(() => new Promise((resolve) => { releaseSettings = resolve; }));+ vi.stubGlobal('fetch', vi.fn());+ const starting = runway.generateVideo({ jobId: 'preparation-cancel', modelId: 'aleph2', runwayVideoPath: video, uploadedTempPaths: [video] });+ await vi.waitFor(() => expect(releaseSettings).toBeTypeOf('function'));+ expect(runway.cancel('preparation-cancel')).toBe('pending');+ releaseSettings({ videoGen: { runway: { apiKey: 'test-key' } } });+ const job = await starting;+ expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'failed' });+ expect(fetch).not.toHaveBeenCalled();+ await expect(stat(video)).rejects.toMatchObject({ code: 'ENOENT' });+ });+ it.each(['aleph2', 'act_two', 'act_two_video'])('uploads and publishes %s with the endpoint-specific fields, preserving input ownership', async (variant) => {+ const video = await stagedVideo();+ const modelId = variant === 'act_two_video' ? 'act_two' : variant;+ const durationSec = variant === 'act_two' ? 3 : variant === 'act_two_video' ? 30 : 4;+ probeVideoGeometry.mockResolvedValue({ width: 1280, height: 720, fps: 24, durationSec, numFrames: durationSec * 24 });+ const calls = [];+ let registrations = 0;+ vi.stubGlobal('fetch', vi.fn(async (url, options = {}) => {+ calls.push({ url, ...options });+ if (url.endsWith('/v1/uploads')) return jsonResponse({ uploadUrl: 'https://storage.example/upload', fields: { key: 'signed-object', policy: 'signed-policy' }, runwayUri: `runway://upload-${++registrations}` });+ if (url === 'https://storage.example/upload') return jsonResponse({}, true, 204);+ if (url.endsWith('/v1/video_to_video') || url.endsWith('/v1/character_performance')) return jsonResponse({ id: 'task-1' });+ if (url === TASKS) return jsonResponse({ status: 'SUCCEEDED', output: ['https://cdn.example.com/out.mp4'] });+ if (url === 'https://cdn.example.com/out.mp4') return { ok: true, arrayBuffer: async () => Uint8Array.from([1, 2, 3]).buffer };+ throw new Error(`Unexpected fetch ${url}`);+ }));+ const job = await runway.generateVideo({ apiKey: 'test-key', modelId, videoMode: modelId === 'aleph2' ? 'video-edit' : 'character-performance',+ prompt: modelId === 'aleph2' ? 'Make it snow' : '', runwayVideoPath: video,+ ...(variant === 'act_two_video' ? { runwayCharacterVideoPath: video } : { sourceImagePath: framePath }),+ uploadedTempPaths: [video], bodyControl: false, expressionIntensity: 5,+ });+ expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'completed' });+ const storage = calls.find((c) => c.url === 'https://storage.example/upload');+ expect(storage.headers?.Authorization).toBeUndefined();+ expect([...storage.body.keys()]).toEqual(['key', 'policy', 'file']);+ expect(storage.body.get('file').size).toBe(1024);+ const submitted = calls.find((c) => c.url.endsWith(modelId === 'aleph2' ? '/v1/video_to_video' : '/v1/character_performance'));+ const body = JSON.parse(submitted.body);+ expect(body).not.toHaveProperty('duration');+ expect(body).not.toHaveProperty('audio');+ if (modelId === 'aleph2') expect(body).toMatchObject({ model: modelId, videoUri: 'runway://upload-1', promptText: 'Make it snow', keyframes: [{ at: 0, uri: expect.stringMatching(/^data:image\//) }] });+ else {+ expect(body).not.toHaveProperty('promptText');+ expect(body).toMatchObject({ model: modelId, character: { type: variant === 'act_two_video' ? 'video' : 'image' }, reference: { type: 'video', uri: 'runway://upload-1' }, bodyControl: false, expressionIntensity: 5 });+ expect(registrations).toBe(variant === 'act_two_video' ? 2 : 1);+ }+ await expect(stat(video)).rejects.toMatchObject({ code: 'ENOENT' });+ expect(await stat(framePath)).toBeTruthy();+ expect((await loadHistory())[0]).toMatchObject({ modelId: `runway:${modelId}`, duration: durationSec, mode: modelId === 'aleph2' ? 'video-edit' : 'character-performance' });+ });++ it.each([+ [{ modelId: 'aleph2' }, /source video/],+ [{ modelId: 'act_two', runwayVideoPath: 'unused.mp4' }, /exactly one character/],+ [{ modelId: 'act_two', runwayVideoPath: 'unused.mp4', sourceImagePath: 'unused.png', prompt: 'ignored text' }, /does not accept prompts/],+ [{ modelId: 'aleph2', runwayVideoPath: 'unused.mp4', videoMode: 'text' }, /matching/],+ [{ modelId: 'act_two', runwayVideoPath: 'unused.mp4', sourceImagePath: 'unused.png', expressionIntensity: 6 }, /intensity/],+ [{ modelId: 'aleph2', runwayVideoPath: 'unused.mp4', seed: -1 }, /seed/],+ ])('rejects incompatible inputs before upload', async (params, message) => {+ vi.stubGlobal('fetch', vi.fn());+ await expect(runway.generateVideo({ apiKey: 'test-key', ...params })).rejects.toMatchObject({ status: 400, message: expect.stringMatching(message) });+ expect(fetch).not.toHaveBeenCalled();+ });++ it.each([+ ['aleph2', { durationSec: 1.99, fps: 24 }], ['aleph2', { durationSec: 30.01, fps: 24 }],+ ['aleph2', { durationSec: 4, fps: 31 }], ['act_two', { durationSec: 2.99, fps: 24 }], ['act_two', { durationSec: 30.01, fps: 24 }], ['act_two', null],+ ])('rejects unreadable/out-of-range %s media and cleans staged input', async (modelId, geometry) => {+ const video = await stagedVideo();+ probeVideoGeometry.mockResolvedValue(geometry);+ vi.stubGlobal('fetch', vi.fn());+ const job = await runway.generateVideo({ apiKey: 'test-key', modelId, runwayVideoPath: video, sourceImagePath: framePath, uploadedTempPaths: [video] });+ expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'failed' });+ expect(fetch).not.toHaveBeenCalled();+ await expect(stat(video)).rejects.toMatchObject({ code: 'ENOENT' });+ });++ it.each(['registration', 'storage', 'malformed'])('does not submit generation after %s failure', async (failure) => {+ const video = await stagedVideo();+ const calls = [];+ vi.stubGlobal('fetch', vi.fn(async (url) => {+ calls.push(url);+ if (url.endsWith('/v1/uploads')) return failure === 'registration' ? jsonResponse({}, false, 503) : jsonResponse(failure === 'malformed' ? {} : { uploadUrl: 'https://storage.example/upload', fields: {}, runwayUri: 'runway://upload-1' });+ return jsonResponse({}, false, 503);+ }));+ const job = await runway.generateVideo({ apiKey: 'test-key', modelId: 'aleph2', runwayVideoPath: video, uploadedTempPaths: [video] });+ expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'failed', error: expect.stringMatching(/upload/i) });+ expect(calls).toEqual(failure === 'storage' ? ['https://api.dev.runwayml.com/v1/uploads', 'https://storage.example/upload'] : ['https://api.dev.runwayml.com/v1/uploads']);+ await expect(stat(video)).rejects.toMatchObject({ code: 'ENOENT' });+ });++ it.each(['registration', 'storage'])('cancels during %s without a paid task and cleans input', async (phase) => {+ const video = await stagedVideo();+ let uploading;+ const started = new Promise((resolve) => { uploading = resolve; });+ const calls = [];+ vi.stubGlobal('fetch', vi.fn(async (url, options) => {+ calls.push(url);+ if (phase === 'storage' && url.endsWith('/v1/uploads')) return jsonResponse({ uploadUrl: 'https://storage.example/upload', fields: {}, runwayUri: 'runway://upload-1' });+ uploading();+ return new Promise((_resolve, reject) => options.signal.addEventListener('abort', () => reject(options.signal.reason), { once: true }));+ }));+ const job = await runway.generateVideo({ apiKey: 'test-key', modelId: 'aleph2', runwayVideoPath: video, uploadedTempPaths: [video] });+ await started;+ runway.cancel(job.jobId);+ expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'failed' });+ expect(calls.some((url) => url.endsWith('/v1/video_to_video'))).toBe(false);+ await expect(stat(video)).rejects.toMatchObject({ code: 'ENOENT' });+ });++ it('replays local input through fresh ephemeral uploads after a failed remote task', async () => {+ const video = await stagedVideo();+ const bodies = [];+ let registrations = 0;+ vi.stubGlobal('fetch', vi.fn(async (url, options = {}) => {+ if (url.endsWith('/v1/uploads')) return jsonResponse({ uploadUrl: 'https://storage.example/upload', fields: {}, runwayUri: `runway://fresh-${++registrations}` });+ if (url === 'https://storage.example/upload') return jsonResponse({}, true, 204);+ if (url.endsWith('/v1/video_to_video')) { bodies.push(JSON.parse(options.body)); return jsonResponse({ id: 'task-1' }); }+ if (url === TASKS) return jsonResponse({ status: 'FAILED', failure: 'Unable to render' });+ throw new Error(`Unexpected fetch ${url}`);+ }));+ for (const durationSec of [2, 30]) {+ probeVideoGeometry.mockResolvedValue({ width: 1280, height: 720, fps: 30, durationSec });+ const job = await runway.generateVideo({ apiKey: 'test-key', modelId: 'aleph2', runwayVideoPath: video });+ expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'failed', error: expect.stringContaining('Unable to render') });+ }+ expect(bodies.map((body) => body.videoUri)).toEqual(['runway://fresh-1', 'runway://fresh-2']);+ expect(await stat(video)).toBeTruthy(); // externally owned inputs survive failure/replay+ });});afterEach(async () => {diff --git a/server/services/videoGen/runwayVideoInputs.js b/server/services/videoGen/runwayVideoInputs.jsnew file mode 100644index 0000000000..8d6e11170f--- /dev/null+++ b/server/services/videoGen/runwayVideoInputs.js@@ -0,0 +1,16 @@+import { stat } from 'node:fs/promises';+import { probeVideoGeometry } from '../../lib/ffmpeg.js';+import { ServerError } from '../../lib/errorHandler.js';++/** Refuse unreadable/unsupported clips before upload or a paid task, including queue replay. */+export async function validateRunwayVideoInput(path, model, { character = false } = {}) {+ const fail = (message) => { throw new ServerError(message, { status: 400, code: 'RUNWAY_VIDEO_INPUT' }); };+ const file = await stat(path);+ if (file.size < 512 || file.size > 200 * 1024 * 1024) fail('Runway videos must be between 512 bytes and 200 MB.');+ const geometry = await probeVideoGeometry(path);+ if (!geometry) fail('Could not read the video stream. Choose a valid video file; ffprobe must be available.');+ const min = character ? 0 : model.minSec;+ if (geometry.durationSec <= 0 || (!character && (geometry.durationSec < min || geometry.durationSec > model.maxSec))) fail(`${model.label} requires a video between ${model.minSec} and ${model.maxSec} seconds.`);+ if (model.id === 'aleph2' && geometry.fps > 30) fail('Aleph 2 input video must be 30 FPS or lower.');+ return geometry;+}diff --git a/server/services/videoGen/submitJob.js b/server/services/videoGen/submitJob.jsindex 3d07a3a519..21c3913dff 100644--- a/server/services/videoGen/submitJob.js+++ b/server/services/videoGen/submitJob.js@@ -373,7 +373,7 @@ const submitHostedLane = async ({ body, prepared, backend, hosted, performancePaconst { jobId, position, status } = await enqueue({// Hosted jobs use mode for queue dispatch and videoMode for text/image.mode: backend,- videoMode: sourceImagePath ? 'image' : 'text',+ videoMode: body.mode || (sourceImagePath ? 'image' : 'text'),prompt: body.prompt,negativePrompt: body.negativePrompt || '',sourceImagePath,