Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

feat: add Runway text-to-video and creative workflow lanes (#12558)

[._.]/6101dc0d9ba5
diff --git a/client/src/components/creative-director/VideoDraftDrawer.jsx b/client/src/components/creative-director/VideoDraftDrawer.jsx
index 636eb949b9..4e9fd22695 100644
--- a/client/src/components/creative-director/VideoDraftDrawer.jsx
+++ b/client/src/components/creative-director/VideoDraftDrawer.jsx
@@ -1,5 +1,6 @@
import { VIDEO_REVIEW_CHECKPOINTS } from '../../../../server/lib/creativeDirectorPresets.js';
import { VIDEO_RENDER_MODES, modeLabel } from '../../lib/imageGenBackends.js';
+import { RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL, RUNWAY_VIDEO_MODELS, getRunwayVideoModel } from '../../lib/runwayVideoModels.js';
import { useEffect, useState } from 'react';
import Drawer from '../Drawer.jsx';
import useDrawerTab from '../../hooks/useDrawerTab.js';
@@ -81,6 +82,7 @@ export default function VideoDraftDrawer({ open, onClose, project, onSaved, cata
<label htmlFor="video-draft-videoMode" className="block text-sm">Media backend<select id="video-draft-videoMode" className={fieldClass} value={form.videoMode || 'local'} onChange={e => { change('videoMode', e.target.value); change('backendModelId', ''); }}>{!VIDEO_RENDER_MODES.includes(form.videoMode) && form.videoMode && <option value={form.videoMode}>{form.videoMode} (saved backend)</option>}{VIDEO_RENDER_MODES.map(mode => <option key={mode} value={mode}>{modeLabel(mode)}</option>)}</select></label>
{form.videoMode === 'local' ? select('modelId', 'Media model', [['', 'Choose before production'], ...(form.modelId && !models.some(m => m.id === form.modelId) ? [[form.modelId, `${form.modelId} (saved model unavailable)`]] : []), ...models.map(m => [m.id, m.name || m.id])])
: form.videoMode === 'fal' ? input('backendModelId', 'fal.ai model (optional)', { placeholder: 'Use configured fal.ai default', maxLength: 64 })
+ : form.videoMode === 'runway' ? select('backendModelId', 'Runway model', [['', `${getRunwayVideoModel(RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL).label} (backend default)`], ...(form.backendModelId && !getRunwayVideoModel(form.backendModelId) ? [[form.backendModelId, `${form.backendModelId} (saved model unavailable)`]] : []), ...RUNWAY_VIDEO_MODELS.map(m => [m.id, m.label])])
: select('backendModelId', 'Media model', [['', form.videoMode === 'reactor' ? 'fast-h3 (backend default)' : 'Backend default'], ...(form.backendModelId ? [[form.backendModelId, `${form.backendModelId} (saved model)`]] : [])])}
<p className="text-sm text-port-text-muted">Backend selections are saved without running providers. Cloud renders may incur charges when production is enabled.</p>
{select('transition', 'Shot joins', [['cut', 'Straight cuts'], ['fade', 'Fade through black (no overlap)']])}
diff --git a/client/src/components/fableloom/LoomSettingsDrawer.jsx b/client/src/components/fableloom/LoomSettingsDrawer.jsx
index aa72628998..b336fc0e5f 100644
--- a/client/src/components/fableloom/LoomSettingsDrawer.jsx
+++ b/client/src/components/fableloom/LoomSettingsDrawer.jsx
@@ -54,6 +54,7 @@ const VIDEO_RENDER_BACKENDS = [
{ id: 'grok', label: 'Grok', icon: Cloud },
{ id: 'fal', label: 'fal.ai', icon: Cloud },
{ id: 'reactor', label: 'Reactor.inc', icon: Cloud },
+ { id: 'runway', label: 'Runway', icon: Cloud },
];
const modelOptions = (models, selected) => {
diff --git a/client/src/hooks/useVideoGenForm.js b/client/src/hooks/useVideoGenForm.js
index 31d59b42fa..fb2b9b8c42 100644
--- a/client/src/hooks/useVideoGenForm.js
+++ b/client/src/hooks/useVideoGenForm.js
@@ -24,6 +24,7 @@ import {
resolveDraftDecodeForModel, draftDecodeFromRecord,
} from '../lib/videoGenParams.js';
import { videoStreamingModeFromRecord } from '../lib/videoStreamingMode.js';
+import { RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL, getRunwayVideoModel, runwayModelTakesText } from '../lib/runwayVideoModels.js';
import { useVideoGenFieldState } from './useVideoGenFieldState.js';
import { useVideoGenSubmitFlow } from './useVideoGenSubmitFlow.js';
import { useVideoGenValidation } from './useVideoGenValidation.js';
@@ -659,8 +660,13 @@ export function useVideoGenForm({
// (`reactorEnabled`) — same short-circuit shape as fal, plus native
// clip-to-clip chaining via continue_from_clip_id.
const isReactor = hostedLane === 'reactor';
- // Runway is usability-gated the same way, and renders image-to-video only.
+ // Runway is usability-gated the same way; text mode is offered for the models that render from a prompt alone.
const isRunway = hostedLane === 'runway';
+ // A model that needs a start frame cannot stay picked in Text mode: fall back to the catalog default (which renders from text).
+ const runwayTextBlocked = isRunway && mode === 'text' && !runwayModelTakesText(getRunwayVideoModel(runwayModelId || RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL));
+ useEffect(() => {
+ if (runwayTextBlocked) setRunwayModelId('');
+ }, [runwayTextBlocked, setRunwayModelId]);
const referenceModeApplies = mode === 'image' && !hostedLane;
// The strength the render will actually use, for the slider readout — an
// untouched slider under Inspire still resolves to the contract's low default
@@ -945,9 +951,7 @@ export function useVideoGenForm({
// clip survives the switch and reappears if the user flips back.
const handleBackendChange = (id) => {
setBackend(id);
- if (id === 'runway' && mode !== 'image') {
- handleModeChange('image');
- } else if ((id === 'grok' || id === 'fal' || id === 'reactor') && mode !== 'text' && mode !== 'image') {
+ if ((id === 'grok' || id === 'fal' || id === 'reactor' || id === 'runway') && mode !== 'text' && mode !== 'image') {
handleModeChange((sourceImageFile || sourceImageUpload) ? 'image' : 'text');
}
};
@@ -1248,9 +1252,9 @@ export function useVideoGenForm({
if (p.duration) setFalDuration(p.duration);
if (p.modelId) setFalModelId(p.modelId);
} else if (p.mode === 'runway') {
- // Runway job: same discriminator shape as fal above; image-to-video only.
+ // Runway job: same discriminator shape as fal above.
setBackend('runway');
- setMode('image');
+ setMode(p.videoMode === 'text' ? 'text' : 'image');
if (p.modelId) {
runwayModelPickRef.current = p.modelId;
setRunwayModelId(p.modelId);
diff --git a/client/src/hooks/useVideoGenForm.test.jsx b/client/src/hooks/useVideoGenForm.test.jsx
index 701ecb40f3..0530da4b5a 100644
--- a/client/src/hooks/useVideoGenForm.test.jsx
+++ b/client/src/hooks/useVideoGenForm.test.jsx
@@ -942,6 +942,21 @@ describe('useVideoGenForm', () => {
expect(result.current.modelId).toBe(LTX2.id);
});
+ it('restores a queued Runway text render without changing it to image mode', () => {
+ const { result, rerender } = render();
+ rerender({ models: MODELS, modelContext: MODEL_CONTEXT, availableLoras: [], runwayEnabled: true });
+ act(() => result.current.applyResumedParams({
+ mode: 'runway', videoMode: 'text', modelId: 'gen4.5', duration: 7, resolution: '720p',
+ prompt: 'a fox watches the rain',
+ }));
+ expect(result.current.hostedLane).toBe('runway');
+ expect(result.current.mode).toBe('text');
+ expect(result.current.runwayModelId).toBe('gen4.5');
+ expect(result.current.runwayDuration).toBe('7');
+ expect(result.current.runwayResolution).toBe('720p');
+ expect(result.current.prompt).toBe('a fox watches the rain');
+ });
+
it('resolves a stored hosted backend whose key is no longer configured to the local lane', async () => {
const { result, rerender } = render({ grokEnabled: false });
const props = { models: MODELS, modelContext: MODEL_CONTEXT, availableLoras: [], grokEnabled: false };
diff --git a/client/src/lib/runwayVideoModels.js b/client/src/lib/runwayVideoModels.js
index bee2abe540..8552d47f2a 100644
--- a/client/src/lib/runwayVideoModels.js
+++ b/client/src/lib/runwayVideoModels.js
@@ -13,6 +13,7 @@ export {
estimateRunwayVideoCostUsd,
getRunwayVideoModel,
runwayVideoClipQuote,
+ runwayModelTakesText,
runwayVideoLengths,
runwayVideoResolutions,
} from '../../../server/lib/runwayVideoModels.js';
diff --git a/client/src/lib/videoGenSubmission.js b/client/src/lib/videoGenSubmission.js
index d1b476a69c..aed469cddc 100644
--- a/client/src/lib/videoGenSubmission.js
+++ b/client/src/lib/videoGenSubmission.js
@@ -115,8 +115,8 @@ function falVideoSubmission({
};
}
-// Runway renders image-to-video only: a start frame is required (the form
-// offers Image mode alone). The negative prompt is sent; the server passes it
+// Runway renders from a start frame (Image mode) or, on a model with a
+// text-to-video rule, from the prompt alone (Text mode). The negative prompt is sent; the server passes it
// natively or as an "Avoid:" line, listed as a chip in the sent-prompt view.
function runwayVideoSubmission({
composed, runwayModelId, runwayDuration, runwayResolution,
diff --git a/client/src/pages/VideoGen.jsx b/client/src/pages/VideoGen.jsx
index c94df74519..b6a6063124 100644
--- a/client/src/pages/VideoGen.jsx
+++ b/client/src/pages/VideoGen.jsx
@@ -107,7 +107,7 @@ import {
import { REACTOR_MAX_PROMPT_LENGTH } from '../lib/reactorVideoClip.js';
import {
RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL, RUNWAY_VIDEO_MODELS, describeRunwayVideoRate, estimateRunwayVideoCostUsd, getRunwayVideoModel,
- runwayVideoLengths, runwayVideoResolutions,
+ runwayModelTakesText, runwayVideoLengths, runwayVideoResolutions,
} from '../lib/runwayVideoModels.js';
import { videoGenEnqueuePolicy } from '../lib/videoGenEnqueuePolicy.js';
import { styledVideoPrompt } from '../lib/videoGenSubmission.js';
@@ -1170,7 +1170,7 @@ export default function VideoGen() {
: id === 'reactor'
? 'Render via the reactor.inc fast-h3 API. Counts against your reactor.inc balance.'
: id === 'runway'
- ? 'Animate a start frame via the Runway API. Counts against your Runway credits.'
+ ? 'Render from text or animate a start frame via the Runway API. Counts against your Runway credits.'
: 'Render on this machine with the local runtimes.'}
>
{label}
@@ -1185,8 +1185,7 @@ export default function VideoGen() {
WAI-ARIA Tabs, since the mode-specific inputs aren't structured as
tabpanels and we don't implement roving-tabindex/arrow-key focus. */}
<div className="bg-port-card border border-port-border rounded-xl p-1 flex flex-wrap gap-1" role="group" aria-label="Video generation mode">
- {(isRunway ? MODES.filter((m) => m.id === 'image')
- : rendersOffMachine ? MODES.filter((m) => m.id === 'text' || m.id === 'image') : MODES).map(({ id, label, icon: Icon, desc }) => {
+ {(rendersOffMachine ? MODES.filter((m) => m.id === 'text' || m.id === 'image') : MODES).map(({ id, label, icon: Icon, desc }) => {
const active = mode === id;
return (
<button
@@ -1668,7 +1667,7 @@ export default function VideoGen() {
onChange={(e) => setRunwayModelId(e.target.value)}
className="w-full bg-port-bg border border-port-border rounded-lg px-2 py-2 text-sm text-white focus:outline-none focus:border-port-accent"
>
- {RUNWAY_VIDEO_MODELS.map((m) => (
+ {RUNWAY_VIDEO_MODELS.filter((m) => mode !== 'text' || runwayModelTakesText(m)).map((m) => (
<option key={m.id} value={m.id}>
{`${m.label} · ${describeRunwayVideoRate(m.id)}${m.id === RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL ? ' (default)' : ''}`}
</option>
@@ -1709,7 +1708,7 @@ export default function VideoGen() {
</FormField>
)}
<p className="col-span-2 text-[11px] text-gray-500 leading-snug">
- Animates your start frame on the Runway API. Counts against your Runway credits
+ {mode === 'image' ? 'Animates your start frame' : 'Renders from your prompt'} on the Runway API. Counts against your Runway credits
{(() => {
const estimate = estimateRunwayVideoCostUsd({
modelId: runwayModel?.id, seconds: runwayDuration === '' ? null : Number(runwayDuration), resolution: runwayResolution,
diff --git a/client/src/pages/VideoGen.runway.test.jsx b/client/src/pages/VideoGen.runway.test.jsx
new file mode 100644
index 0000000000..30d9ae6cd0
--- /dev/null
+++ b/client/src/pages/VideoGen.runway.test.jsx
@@ -0,0 +1,41 @@
+import { beforeEach, describe, expect, it } from 'vitest';
+import { fireEvent, screen, waitFor, within } from '@testing-library/react';
+import { findEnabledByRole } from '../test/enabledBarrier.js';
+import {
+ loadVideoGenPage, renderVideoGenPage, resetVideoGenMockState,
+ state, videoGenModel, videoGenModelContext, videoGenStatus,
+} from '../test/videoGenPageMocks.jsx';
+
+const MODEL = videoGenModel('local-model');
+await loadVideoGenPage();
+
+describe('VideoGen Runway text lane', () => {
+ beforeEach(() => {
+ localStorage.clear();
+ resetVideoGenMockState();
+ state.getVideoGenModelContext.mockResolvedValue(videoGenModelContext([MODEL]));
+ state.getVideoGenStatus.mockResolvedValue(videoGenStatus([MODEL], { runwayEnabled: true }));
+ state.generateVideo.mockResolvedValue({ jobId: 'job-1' });
+ });
+
+ it('switches from an image-only model to a priced text-capable model and submits without a frame', async () => {
+ await renderVideoGenPage();
+ fireEvent.click(await screen.findByRole('button', { name: 'Runway' }));
+ const modes = within(screen.getByRole('group', { name: 'Video generation mode' }));
+ fireEvent.click(modes.getByRole('button', { name: 'Image' }));
+ const model = await screen.findByLabelText('Runway model');
+ fireEvent.change(model, { target: { value: 'gen4_turbo' } });
+ expect(model).toHaveValue('gen4_turbo');
+ fireEvent.click(modes.getByRole('button', { name: 'Text' }));
+ await waitFor(() => expect(model).toHaveValue('gen4.5'));
+ expect([...model.options].map(option => option.value)).toEqual(['gen4.5']);
+ expect(screen.getByText(/about \$0.60 at list price/)).toBeInTheDocument();
+ fireEvent.change(screen.getByLabelText('Prompt'), { target: { value: 'a fox watches the rain' } });
+ fireEvent.click(await findEnabledByRole('button', { name: /Add to queue/ }));
+ await waitFor(() => expect(state.generateVideo).toHaveBeenCalled());
+ const payload = state.generateVideo.mock.calls[0][0];
+ expect(payload).toMatchObject({ backend: 'runway', prompt: 'a fox watches the rain' });
+ expect(payload.sourceImageFile).toBe('');
+ expect(payload.sourceImage).toBe('');
+ });
+});
diff --git a/server/lib/README.md b/server/lib/README.md
index 987ab0a5a8..eb7ffa6b46 100644
--- a/server/lib/README.md
+++ b/server/lib/README.md
@@ -344,7 +344,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
| `musicVideoShotTiming.js` | Music Video shot modes (#8977): `MUSIC_VIDEO_SHOT_MODES` (`cutaway`/`performance`), `isPerformanceScene`, `MUSIC_VIDEO_PERFORMANCE_STYLES` (`lipsync`/`music`) + `isLipSyncScene`, `selectedPerformanceInstruction(scene)` (the selected take's shot instruction, performance scenes only), `clipAudioOffsetSec(instruction)` (provider lead padding a landed take's edit points were moved by, #11210), `SOURCE_AUDIO_LIPSYNC` (verified source-audio lip-sync lanes — only fal.ai MiniMax H3 lip-sync, 5–30s per take, default 1080P), `SOURCE_AUDIO_MUSIC` (music-driven lanes — fal.ai audio-to-video models, picked by `videoSettings.falMusicModelId`, default OmniHuman v1.5 at 1080p), `falSceneTake()` (the fal model/length/resolution a scene take requests and its estimated cost — shared by the board, the submit path and the autopilot spend cap) + `falTakeRequestFields(take)` (that take as `/api/video-gen` fields), `performanceCapability(backend, { scene, videoSettings })` (lip-sync lane, or the music-driven one for a music-driven scene), `performanceBlockedReason(backend)` (Grok is cutaway-only), `planPerformanceWindow()` (provider-bounded audio window + clip-relative edit in/out points; refuses over-long shots instead of relying on provider truncation), `planPerformanceRepair()` (complete measured suffix evidence, safe word-gap boundary, source/clip timebases and next spend), `clipRelativeCues()`, `grokCoverage(spanSec)` (covering 6/10/15 s request, flags spans needing a split), `approximateMotionCues()`, `shotSplitLimit(scene, backend)` (per-take limit: lip-sync window or Grok's longest clip), `planShotSplit()` (contiguous pieces cut at lyric pauses → line/phrase boundaries → beats; one extra piece at most to avoid cutting a sung line). Dependency-free; the client re-exports it. |
| `reactorStartingFrame.js` | Fits a reactor.inc `starting_frame` to the fast-h3 session canvas before it is uploaded. `readImageSize(path)` → `{width,height}` or `null` (never a guessed dimension), and `prepareReactorStartingFrame(sourceImagePath, requestedAspect, outputPath)` → `{ aspect, canvas, framePath, fittedPath }`: an absent `requestedAspect` DERIVES the canvas from the frame's own shape (`nearestReactorAspect`), then centre cover-crops the frame to that canvas's exact pixel size so reactor's own "fitted to the session canvas" step is a no-op. `fittedPath` is the temp file the caller deletes; `null` means the original was uploaded unchanged. A frame sharp cannot decode (or a resize that fails) passes through untouched on the default canvas rather than failing a render PortOS merely could not measure — reactor decodes more containers than sharp does. |
| `reactorVideoClip.js` | The reactor.inc `fast-h3` clip contract, dependency-free so the route schema, the FableLoom prompt compiler and the VideoGen picker all read one copy — `REACTOR_MAX_PROMPT_LENGTH` (800; the API rejects a longer prompt rather than truncating it), `REACTOR_MIN_CLIP_SECONDS` / `REACTOR_MAX_CLIP_SECONDS` (5.167–14.375, i.e. 124–345 frames at the `REACTOR_CLIP_FPS` 24 it delivers), `REACTOR_MAX_CLIP_ID_LENGTH`, and the picker list `REACTOR_CLIP_LENGTHS` + `REACTOR_DEFAULT_CLIP_LENGTH` + `reactorClipLengthLabel()` (the two exact endpoints plus every whole second between them, since the accepted range is continuous but starts and ends off a round number). Also the CANVAS contract `set_canvas` accepts: `REACTOR_CANVASES` (`16:9` 1344×768, `4:3` 1024×768, `1:1` 768×768, `9:16` 768×1344 — every one a 768px short edge, so the aspect string is the whole choice), `REACTOR_ASPECTS`, `REACTOR_DEFAULT_ASPECT`, `reactorCanvas(aspect)` and `nearestReactorAspect(width, height)` (closest canvas in log-ratio space; anything unmeasurable answers the default rather than deriving one). Mirrored to `client/src/lib/reactorVideoClip.js`, parity pinned by `reactorVideoClip.test.js`. |
-| `runwayVideoModels.js` | Curated Runway VIDEO model catalog for the developer API's `/v1/image_to_video` — `RUNWAY_VIDEO_MODELS` (Gen-4.5 default, Gen-4 Turbo, Veo 3.1 / Fast, Kling 3.0 Standard / Pro, Kling 2.5 Turbo Pro, MiniMax H3 Max, WAN 3.0) with per-model prompt caps, duration rules, output sizes (`ratio` or `resolution`), end-frame support and list prices verified on `RUNWAY_VIDEO_PRICING_VERIFIED_AT`; `getRunwayVideoModel()`, `coerceRunwayVideoSeconds()` (round UP to a covering length), `resolveRunwayVideoResolution()`, `runwayVideoLengths()`, `buildRunwayVideoRequest()` (the exact body; refuses an uncurated model, a missing frame or an over-cap prompt, never cuts it), `runwayVideoClipQuote()` / `estimateRunwayVideoCostUsd()`, `describeRunwayVideoRate()`. Dependency-free; the client re-exports it. |
+| `runwayVideoModels.js` | Curated Runway VIDEO model catalog for the developer API's `/v1/image_to_video` and (Gen-4.5) `/v1/text_to_video` — `RUNWAY_VIDEO_MODELS` (Gen-4.5 default, Gen-4 Turbo, Kling 3.0 Standard / Pro, Kling 2.5 Turbo Pro, MiniMax H3 Max, WAN 3.0) with per-model prompt caps, duration rules, output sizes (`ratio` or `resolution`), end-frame support and list prices verified on `RUNWAY_VIDEO_PRICING_VERIFIED_AT`; `getRunwayVideoModel()`, `coerceRunwayVideoSeconds()` (round UP to a covering length), `resolveRunwayVideoResolution()`, `runwayVideoLengths()`, `buildRunwayVideoRequest()` (the endpoint and exact body, text-only when no frame and the model has a `textToVideo` rule, `runwayModelTakesText()`; refuses an uncurated model, a missing frame or an over-cap prompt, never cuts it), `runwayVideoClipQuote()` / `estimateRunwayVideoCostUsd()`, `describeRunwayVideoRate()`. Dependency-free; the client re-exports it. |
| `kimi.js` | Moonshot AI Kimi Code (`kimi`) provider helpers — id constants (`KIMI_CLI_ID`/`KIMI_TUI_ID`), `isKimiCommand`/`isKimiCliProvider`/`isKimiTuiProvider` predicates, `ensureKimiHeadlessArgs` (model flag only — non-interactive mode is implicit in `--prompt`, and kimi has no `--print`/`--afk` and refuses approval flags alongside `--prompt`) / `ensureKimiTuiArgs` (`--yolo`) argv builders (model selection uses the `KIMI_CONFIGURED_DEFAULT` sentinel in `providerModels.js` so PortOS omits `--model` like Grok/Antigravity), and `prepareKimiPrompt` (delivers the prompt as the `--prompt <value>` argv, not stdin). |
| `runners.js` | Image-runner family constants (`RUNNER_FAMILIES`, `loraCompatKey`), video LoRA families, MiniMax H3 runtime predicates, and the shared `AUDIO_TO_VIDEO_RUNTIMES` / `isAudioToVideoRuntime` capability gate (LTX plus H3 Ref2VA). Mirrored to `client/src/lib/runnerFamilies.js`. |
| `codexAccount.js` | The ChatGPT-subscription half of the Codex providers (#5589), pure. The fixed `codex app-server` invocation, the `account/*` method + notification names, `CODEX_ACCOUNT_STATUS` (`runtime-missing` / `unknown` / `signed-out` / `login-pending` / `ready` / `quota-exhausted` / `reauth-required`) and `CODEX_ERROR_CODES`, `isCodexSubscriptionProvider` (re-exported from `providerModels.js`; keyed on the `codex` COMMAND, so a renamed clone still resolves), the `normalizeCodexAccount` / `normalizeCodexRateLimits` / `normalizeCodexLoginStart` payload normalizers, `redactCodexPayload` (every log/error/response path runs through it, so "PortOS never writes down a Codex token" is one function rather than a rule per call site), and `deriveCodexAccountStatus`. Sentinels are load-bearing: `runtimeInstalled: null` = NOT PROBED, `accountFetched: false` ≠ signed out, `rateLimits: null` (failed fetch) ≠ a fetched window with nothing to report. The process half is `services/codexAppServer.js`. |
diff --git a/server/lib/runwayVideoModels.js b/server/lib/runwayVideoModels.js
index b2793cbfd7..27b17375a9 100644
--- a/server/lib/runwayVideoModels.js
+++ b/server/lib/runwayVideoModels.js
@@ -1,6 +1,7 @@
/**
* Curated Runway VIDEO model catalog — what PortOS knows about each model it
- * offers through Runway's developer API (`POST /v1/image_to_video`): the
+ * offers through Runway's developer API (`POST /v1/image_to_video`, and
+ * `POST /v1/text_to_video` for a row with a `textToVideo` rule): the
* request fields each model takes (the body is one shape per `model`), its
* duration and output-size rules, whether it ends on a last frame, and its
* published price.
@@ -15,8 +16,11 @@
* Runway serves it; the native `negativePrompt` row field stays for a model
* that takes one.
*
- * Image-to-video only: every row animates a start frame, and the rows marked
- * `endImage` also end on a last frame. Provider audio is always requested off
+ * Every row animates a start frame, and the rows marked `endImage` also end on
+ * a last frame. Only Gen-4.5 also renders from text alone; its `textToVideo`
+ * rule lists the ratios `/v1/text_to_video` takes (the duration, prompt cap,
+ * seed and price are the row's own). The other curated rows stay image-only
+ * until a text-to-video need for them is checked against the API reference. Provider audio is always requested off
* (`audio: false`) where a model has the field, since a PortOS clip is silent
* unless the caller asks (a Music Video's master song is its only soundtrack),
* and the silent rate is what is priced.
@@ -45,6 +49,8 @@ export const RUNWAY_VIDEO_MODELS = Object.freeze([
// `ratio` is required and names the output canvas.
size: { param: 'ratio', required: true, byResolution: { '720p': GEN4_RATIOS }, default: '720p' },
audio: false, seed: true, negativePrompt: false,
+ // `ratio` is required on /v1/text_to_video and offers only these two.
+ textToVideo: { ratios: Object.freeze(['1280:720', '720:1280']) },
pricing: { perSecond: 0.12 },
},
{
@@ -114,6 +120,9 @@ const BY_ID = new Map(RUNWAY_VIDEO_MODELS.map((m) => [m.id, m]));
/** The catalog row for `modelId`, or null for a model PortOS has not curated. */
export const getRunwayVideoModel = (modelId) => (typeof modelId === 'string' ? BY_ID.get(modelId.trim()) || null : null);
+/** True when `model` renders from a prompt alone (`/v1/text_to_video`). */
+export const runwayModelTakesText = (model) => !!model?.textToVideo;
+
/** Output sizes a model offers (empty when it has no size parameter). */
export const runwayVideoResolutions = (model) => (model?.size ? Object.keys(model.size.byResolution) : []);
@@ -215,8 +224,10 @@ export function describeRunwayVideoRate(modelId, resolution) {
}
/**
- * Build the `/v1/image_to_video` body for a curated model. Returns `{ body,
- * model, seconds, resolution, estimatedCostUsd }`. A missing required input, an
+ * Build the Runway request for a curated model: `/v1/image_to_video` with a
+ * start frame, `/v1/text_to_video` without one (a model with a `textToVideo`
+ * rule). Returns `{ endpoint, body, model, seconds, resolution,
+ * estimatedCostUsd }`. A missing required input, an
* input the model cannot take, an uncurated model, or an over-cap prompt throws
* an Error with `code = 'RUNWAY_MODEL_INPUT'` so the caller refuses before
* anything is paid for. A prompt is never shortened (AGENTS.md, #11317).
@@ -240,19 +251,27 @@ export function buildRunwayVideoRequest({
if (model.negativePrompt && avoid.length > model.negativePromptMaxChars) {
throw refuse(`The negative prompt is ${avoid.length.toLocaleString('en-US')} characters; ${model.label} takes ${model.negativePromptMaxChars.toLocaleString('en-US')}. Shorten it to render.`);
}
- if (!imageUri) throw refuse(`${model.label} renders from a start frame. Supply a source image.`);
+ const textOnly = !imageUri;
+ if (textOnly && !runwayModelTakesText(model)) throw refuse(`${model.label} renders from a start frame. Supply a source image.`);
+ if (textOnly && endImageUri) throw refuse('An end frame needs a start frame. Supply a source image.');
if (endImageUri && !model.endImage) throw refuse(`${model.label} takes no end frame; choose a model that does`);
const body = { model: model.id };
- body.promptImage = endImageUri
- ? [{ uri: imageUri, position: 'first' }, { uri: endImageUri, position: 'last' }]
- : imageUri;
+ if (!textOnly) {
+ body.promptImage = endImageUri
+ ? [{ uri: imageUri, position: 'first' }, { uri: endImageUri, position: 'last' }]
+ : imageUri;
+ }
if (sent) body.promptText = sent;
if (model.negativePrompt && avoid) body.negativePrompt = avoid;
const coercedSec = coerceRunwayVideoSeconds(model, seconds);
body.duration = coercedSec;
const res = resolveRunwayVideoResolution(model, resolution);
- if (model.size) {
+ if (textOnly) {
+ // No frame to follow, so the canvas is always named: the offered ratio closest to the target.
+ const want = aspectOf({ aspectRatio, width, height }) ?? 16 / 9;
+ body.ratio = model.textToVideo.ratios.reduce((best, r) => (Math.abs(ratioOf(r) - want) < Math.abs(ratioOf(best) - want) ? r : best));
+ } else if (model.size) {
const target = aspectOf({ aspectRatio, width, height });
// An optional ratio is sent only when there is a target to match and no
// last frame (a keyframe render's canvas follows its first frame);
@@ -266,6 +285,7 @@ export function buildRunwayVideoRequest({
if (model.seed && Number.isInteger(seed) && seed >= 0 && seed <= 4294967295) body.seed = seed;
return {
+ endpoint: textOnly ? 'text_to_video' : 'image_to_video',
body,
model,
seconds: coercedSec,
diff --git a/server/lib/runwayVideoModels.test.js b/server/lib/runwayVideoModels.test.js
index 7b1c8d792a..ec46d2e1be 100644
--- a/server/lib/runwayVideoModels.test.js
+++ b/server/lib/runwayVideoModels.test.js
@@ -58,10 +58,31 @@ describe('buildRunwayVideoRequest — per-family wire bodies', () => {
});
});
+describe('buildRunwayVideoRequest — text-to-video', () => {
+ it('Gen-4.5 with no start frame: /v1/text_to_video body, ratio matched to the shape, priced, Avoid folded in', () => {
+ const portrait = buildRunwayVideoRequest({
+ prompt: 'a lighthouse at dusk', negativePrompt: 'text', seconds: 3.2, width: 1080, height: 1920, seed: 7,
+ });
+ expect(portrait.endpoint).toBe('text_to_video');
+ expect(portrait.body).toEqual({
+ model: 'gen4.5', promptText: 'a lighthouse at dusk\nAvoid: text', duration: 4, ratio: '720:1280', seed: 7,
+ });
+ expect(portrait.estimatedCostUsd).toBe(0.48);
+ // No target shape: Runway's documented landscape canvas.
+ expect(buildRunwayVideoRequest({ prompt: 'a fox' }).body.ratio).toBe('1280:720');
+ expect(build('gen4.5').endpoint).toBe('image_to_video');
+ });
+
+ it('refuses an over-cap text prompt with its count and never shortens it', () => {
+ expect(() => buildRunwayVideoRequest({ prompt: 'x'.repeat(1001) })).toThrow(/1,001 characters/);
+ });
+});
+
describe('buildRunwayVideoRequest — refusals before anything is paid for', () => {
it.each([
['an uncurated model', { modelId: 'gen9' }, /not a Runway model/],
- ['a missing start frame', { imageUri: null }, /start frame/],
+ ['a missing start frame on an image-only model', { modelId: 'gen4_turbo', imageUri: null }, /start frame/],
+ ['an end frame without a start frame', { imageUri: null, endImageUri: END }, /end frame needs a start frame/],
['an end frame on a start-frame-only model', { modelId: 'gen4.5', endImageUri: END }, /takes no end frame/],
['a missing required prompt', { modelId: 'gen4.5', prompt: ' ' }, /needs a prompt/],
])('refuses %s', (_label, extra, message) => {
diff --git a/server/services/fableLoom/production.js b/server/services/fableLoom/production.js
index 6ec29cefff..573c0fc705 100644
--- a/server/services/fableLoom/production.js
+++ b/server/services/fableLoom/production.js
@@ -481,6 +481,9 @@ function resolveVideoBackend(settings, render, recordedConditioning = null) {
if (requested === VIDEO_GEN_MODE.REACTOR && mode !== VIDEO_GEN_MODE.REACTOR) {
throw new ServerError('Reactor video is not configured — set an API key in Settings → Video Gen.', { status: 400, code: 'REACTOR_NOT_CONFIGURED' });
}
+ if (requested === VIDEO_GEN_MODE.RUNWAY && mode !== VIDEO_GEN_MODE.RUNWAY) {
+ throw new ServerError('No Runway API key configured — set it in Settings → Video Gen first', { status: 400, code: 'RUNWAY_NOT_CONFIGURED' });
+ }
return mode;
}
@@ -624,7 +627,9 @@ async function prepareVideoJob(run, asset) {
? { id: 'fal-video', supportedModes: ['text', 'image'] }
: backend === VIDEO_GEN_MODE.REACTOR
? { id: 'reactor-video', supportedModes: ['text', 'image'] }
- : { id: 'grok-video', supportedModes: ['image'] },
+ : backend === VIDEO_GEN_MODE.RUNWAY
+ ? { id: 'runway-video', supportedModes: ['text', 'image'] }
+ : { id: 'grok-video', supportedModes: ['image'] },
});
const requestedSourceImagePath = imageInputForNode(run, asset);
const conditioned = await conditionRequest(run, asset, 'video', capability, requestedSourceImagePath);
@@ -714,6 +719,30 @@ async function prepareVideoJob(run, asset) {
};
}
+ if (backend === VIDEO_GEN_MODE.RUNWAY) {
+ // As with fal, the model is the lane's catalog default: a cloud video backend carries no model pick here.
+ return {
+ kind: 'video',
+ provider: backend,
+ modelId: null,
+ modelRevision: capability.modelRevision || null,
+ params: {
+ mode: VIDEO_GEN_MODE.RUNWAY,
+ videoMode: sourceImagePath ? 'image' : 'text',
+ prompt: conditioned?.prompt || asset.prompt,
+ ...(conditioned?.promptAdditions?.length ? { promptAdditions: conditioned.promptAdditions } : {}),
+ negativePrompt: conditioned?.negativePrompt || '',
+ width: geometry.width,
+ height: geometry.height,
+ aspectRatio: geometry.aspectRatio,
+ ...replayParameters,
+ sourceImagePath,
+ visualConditioning: conditioned?.visualConditioning || null,
+ fableLoom: productionTag(run, asset),
+ },
+ };
+ }
+
const cloud = resolveRenderTargetConfig(settings, RENDER_TARGET.FABLELOOM_PRODUCTION, {
mode: IMAGE_GEN_MODE.GROK,
model: recordedConditioning?.capability?.modelId || null,
diff --git a/server/services/fableLoom/production.test.js b/server/services/fableLoom/production.test.js
index 3e228f138e..f257bcd73b 100644
--- a/server/services/fableLoom/production.test.js
+++ b/server/services/fableLoom/production.test.js
@@ -330,6 +330,27 @@ describe('fableLoom production service', () => {
expect(snapshots.every(snapshot => !('tail' in snapshot) && !('runtime' in snapshot))).toBe(true);
});
+ it('queues the Runway text lane with the compiled prompt and no invented source frame', async () => {
+ const loom = structuredClone(sampleLoom);
+ loom.episodes[0].nodes.forEach(node => { node.image = 'example-frame.png'; });
+ records.getLoom.mockResolvedValue(loom);
+ settingsMocks.getSettings.mockResolvedValue({ videoGen: { runway: { apiKey: 'example-key' } } });
+ const prompt = 'A fox crosses the clearing at dawn.';
+ visualConditioningMocks.compileFableLoomVisualRequest.mockResolvedValue({
+ prompt, sourceImagePath: null, visualConditioning: { compiledPrompt: prompt },
+ });
+ await startEpisodeProductionBatch('loom-1', 'ep-1', {
+ videoMode: 'runway', assetTypes: ['video_entry'], nodeIds: ['node-1'],
+ });
+ await vi.waitFor(() => expect(queueMocks.enqueueJob).toHaveBeenCalled());
+ expect(queueMocks.enqueueJob).toHaveBeenCalledWith(expect.objectContaining({
+ kind: 'video', params: expect.objectContaining({
+ mode: 'runway', videoMode: 'text', prompt, sourceImagePath: null,
+ visualConditioning: expect.objectContaining({ compiledPrompt: prompt }),
+ }),
+ }));
+ });
+
it('queues Reactor with its compact compiled script and fails oversized scripts before queueing', async () => {
const loom = structuredClone(sampleLoom);
loom.episodes[0].nodes.forEach((node) => { node.image = 'example-frame.png'; });
diff --git a/server/services/videoGen/hostedSubmission.js b/server/services/videoGen/hostedSubmission.js
index 54b4325308..5c39c97556 100644
--- a/server/services/videoGen/hostedSubmission.js
+++ b/server/services/videoGen/hostedSubmission.js
@@ -7,7 +7,7 @@ import { ServerError } from '../../lib/errorHandler.js';
import { resolveGalleryImage } from '../../lib/fileUtils.js';
import { probeVideoDuration } from '../../lib/ffmpeg.js';
import { FAL_DEFAULT_IMAGE_VIDEO_MODEL, FAL_DEFAULT_TEXT_VIDEO_MODEL, getFalVideoModel } from '../../lib/falVideoModels.js';
-import { RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL, getRunwayVideoModel } from '../../lib/runwayVideoModels.js';
+import { RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL, getRunwayVideoModel, runwayModelTakesText } from '../../lib/runwayVideoModels.js';
import { isDefaultI2vReferenceMode } from '../../lib/videoReferenceModes.js';
import { VIDEO_GEN_MODE, isVideoModeUsable } from './modes.js';
@@ -59,7 +59,8 @@ async function stageEndFrame({ body, uploads, stageUploadDurable }) {
return { path: resolved, owned: false };
}
-// Runway renders from a start frame only, with an end frame on the models that take one.
+// Runway renders from a start frame (an end frame on the models that take one),
+// or from text alone on a model with a text-to-video rule.
async function stageRunwayInputs(ctx) {
const { body, uploads, sourceImagePath, cleanupStaged } = ctx;
const refuse = async (message) => {
@@ -67,9 +68,10 @@ async function stageRunwayInputs(ctx) {
throw new ServerError(message, { status: 400, code: 'VALIDATION_ERROR' });
};
if (uploads.audioFile) await refuse('Runway renders cutaways only; it takes no source audio.');
- if (!sourceImagePath) await refuse('Runway renders from a start frame — supply a source image.');
const model = getRunwayVideoModel(body.runwayModelId || RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL);
if (!model) await refuse(`"${body.runwayModelId}" is not a Runway model PortOS offers`);
+ if (!sourceImagePath && !runwayModelTakesText(model)) await refuse(`${model.label} renders from a start frame — supply a source image.`);
+ if (!sourceImagePath && (uploads.lastImage || body.lastImageFile)) await refuse('An end frame needs a start frame — supply a source image.');
if (!uploads.lastImage && !body.lastImageFile) return {};
if (!model.endImage) await refuse(`${model.label} takes no end frame`);
const endFrame = await stageEndFrame(ctx);
diff --git a/server/services/videoGen/runway.js b/server/services/videoGen/runway.js
index b8ed7e3f13..20625f0bc0 100644
--- a/server/services/videoGen/runway.js
+++ b/server/services/videoGen/runway.js
@@ -4,7 +4,8 @@
* A cloud lane like `videoGen/fal.js`: the same job-map/SSE contract, so
* `mediaJobQueue` dispatches to it on `params.mode === 'runway'`. Flow: POST
* the start frame (and an optional last frame) as data URIs to
- * `/v1/image_to_video`, poll the task until it SUCCEEDS, download `output[0]`
+ * `/v1/image_to_video` — or just the prompt to `/v1/text_to_video` when no frame
+ * is given and the model renders from text — poll the task until it SUCCEEDS, download `output[0]`
* and hand it to the shared `finalizeGeneratedVideo` helper.
*
* The request body is built per model from the curated catalog
@@ -177,7 +178,7 @@ async function generateVideoPrepared({
...sentRecord,
filename,
createdAt: new Date().toISOString(),
- mode: 'image',
+ mode: sourceImagePath ? 'image' : 'text',
...(shotInstruction?.audioWindow ? { audioWindow: shotInstruction.audioWindow } : {}),
};
const job = { ...meta, clients: [], status: 'running', renderStartedAtMs };
@@ -245,18 +246,18 @@ async function runRunwayVideo(job, jobId, {
activeRequests.set(jobId, entry);
const deadline = Date.now() + RUNWAY_RENDER_TIMEOUT_MS;
try {
- const frame = await runwayFrameDataUri(sourceImagePath, frameAspect);
+ const frame = sourceImagePath ? await runwayFrameDataUri(sourceImagePath, frameAspect) : null;
const endFrame = lastImagePath ? await runwayFrameDataUri(lastImagePath, frameAspect) : null;
for (const [label, fit] of [['start', frame], ['end', endFrame]]) {
if (fit?.conformed) console.log(`✂️ Cropped Runway ${label} frame [${jobId.slice(0, 8)}] ${fit.conformed.from} → ${fit.conformed.to} for ${frameAspect}`);
if (fit?.reencoded) console.log(`🗜️ Re-encoded Runway ${label} frame [${jobId.slice(0, 8)}] as JPEG to fit the 5 MB inline limit`);
}
- if (frame.conformed || endFrame?.conformed) {
- meta.frameConformed = { aspect: frameAspect, ...(frame.conformed ? { start: frame.conformed } : {}), ...(endFrame?.conformed ? { end: endFrame.conformed } : {}) };
+ if (frame?.conformed || endFrame?.conformed) {
+ meta.frameConformed = { aspect: frameAspect, ...(frame?.conformed ? { start: frame.conformed } : {}), ...(endFrame?.conformed ? { end: endFrame.conformed } : {}) };
}
- const { body } = buildRunwayVideoRequest({ ...requestSpec, imageUri: frame.dataUri, endImageUri: endFrame?.dataUri ?? null });
+ const { body, endpoint } = buildRunwayVideoRequest({ ...requestSpec, imageUri: frame?.dataUri ?? null, endImageUri: endFrame?.dataUri ?? null });
if (entry.aborted) return finishFailure('Canceled');
- entry.taskId = await submitRunwayTask({ apiKey, endpoint: 'image_to_video', body });
+ entry.taskId = await submitRunwayTask({ apiKey, endpoint, body });
const polled = await awaitRunwayTask({
entry, apiKey, deadline, timeoutMs: RUNWAY_RENDER_TIMEOUT_MS,
diff --git a/server/services/videoGen/runway.test.js b/server/services/videoGen/runway.test.js
index 550b702af0..9d7587f462 100644
--- a/server/services/videoGen/runway.test.js
+++ b/server/services/videoGen/runway.test.js
@@ -103,6 +103,24 @@ describe('videoGen/runway — generateVideo', () => {
expect(entry.promptAdditions).toEqual([expect.objectContaining({ id: 'avoid', text: 'text' })]);
});
+ it('renders Gen-4.5 from text alone: posts /v1/text_to_video and records a text-mode take', async () => {
+ const calls = [];
+ vi.stubGlobal('fetch', vi.fn(async (url, opts = {}) => {
+ calls.push({ url, method: opts.method || 'GET', body: opts.body ? JSON.parse(opts.body) : null });
+ if (url === 'https://api.dev.runwayml.com/v1/text_to_video') return jsonResponse({ id: 'task-1' });
+ if (url === TASKS) return jsonResponse({ id: 'task-1', status: 'SUCCEEDED', output: ['https://cdn.example.com/out.mp4'] });
+ if (url === 'https://cdn.example.com/out.mp4') return { ok: true, status: 200, arrayBuffer: async () => Uint8Array.from(Buffer.from('mp4')).buffer };
+ throw new Error(`unexpected fetch: ${url}`);
+ }));
+ const job = await runway.generateVideo({ apiKey: 'key_test', prompt: 'a fox at dawn', duration: 5, width: 720, height: 1280 });
+ expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'completed' });
+ expect(calls.find((c) => c.method === 'POST').body).toEqual({
+ model: 'gen4.5', promptText: 'a fox at dawn', duration: 5, ratio: '720:1280',
+ });
+ const [entry] = await loadHistory();
+ expect(entry).toMatchObject({ modelId: 'runway:gen4.5', mode: 'text', aspectRatio: '720:1280', estimatedCostUsd: 0.6 });
+ });
+
it('fails the job with Runway\'s reason when the task FAILS', async () => {
vi.stubGlobal('fetch', vi.fn(async (url, opts = {}) => {
if (opts.method === 'POST') return jsonResponse({ id: 'task-1' });
@@ -114,7 +132,7 @@ describe('videoGen/runway — generateVideo', () => {
});
it.each([
- ['no start frame', { prompt: 'a fox' }, /start frame/],
+ ['no start frame on an image-only model', { modelId: 'gen4_turbo', prompt: 'a fox' }, /start frame/],
['an over-cap prompt', { prompt: 'x'.repeat(1001), sourceImagePath: 'unused.png' }, /1,001 characters/],
['source audio', { prompt: 'a fox', sourceImagePath: 'unused.png', audioFilePath: 'song.wav' }, /cutaways only/],
])('refuses %s before reading media or submitting', async (_label, params, message) => {
diff --git a/server/services/videoGen/submitJob.js b/server/services/videoGen/submitJob.js
index 527a1f5eef..8d96df3091 100644
--- a/server/services/videoGen/submitJob.js
+++ b/server/services/videoGen/submitJob.js
@@ -214,7 +214,9 @@ const submitValidatedVideoGenJob = async (body, uploads, inputs) => {
? { id: 'fal-video', supportedModes: ['text', 'image'] }
: backend === VIDEO_GEN_MODE.REACTOR
? { id: 'reactor-video', supportedModes: ['text', 'image'] }
- : prepared.effectiveModel;
+ : backend === VIDEO_GEN_MODE.RUNWAY
+ ? { id: 'runway-video', supportedModes: ['text', 'image'] }
+ : prepared.effectiveModel;
const compiled = await compileFableLoomVisualRequest({
tag: body.fableLoom,
kind: 'video',