Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
feat: add Runway text-to-video and creative workflow lanes (#12558)
[._.]/6101dc0d9ba5
diff --git a/client/src/components/creative-director/VideoDraftDrawer.jsx b/client/src/components/creative-director/VideoDraftDrawer.jsxindex 636eb949b9..4e9fd22695 100644--- a/client/src/components/creative-director/VideoDraftDrawer.jsx+++ b/client/src/components/creative-director/VideoDraftDrawer.jsx@@ -1,5 +1,6 @@import { VIDEO_REVIEW_CHECKPOINTS } from '../../../../server/lib/creativeDirectorPresets.js';import { VIDEO_RENDER_MODES, modeLabel } from '../../lib/imageGenBackends.js';+import { RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL, RUNWAY_VIDEO_MODELS, getRunwayVideoModel } from '../../lib/runwayVideoModels.js';import { useEffect, useState } from 'react';import Drawer from '../Drawer.jsx';import useDrawerTab from '../../hooks/useDrawerTab.js';@@ -81,6 +82,7 @@ export default function VideoDraftDrawer({ open, onClose, project, onSaved, cata<label htmlFor="video-draft-videoMode" className="block text-sm">Media backend<select id="video-draft-videoMode" className={fieldClass} value={form.videoMode || 'local'} onChange={e => { change('videoMode', e.target.value); change('backendModelId', ''); }}>{!VIDEO_RENDER_MODES.includes(form.videoMode) && form.videoMode && <option value={form.videoMode}>{form.videoMode} (saved backend)</option>}{VIDEO_RENDER_MODES.map(mode => <option key={mode} value={mode}>{modeLabel(mode)}</option>)}</select></label>{form.videoMode === 'local' ? select('modelId', 'Media model', [['', 'Choose before production'], ...(form.modelId && !models.some(m => m.id === form.modelId) ? [[form.modelId, `${form.modelId} (saved model unavailable)`]] : []), ...models.map(m => [m.id, m.name || m.id])]): form.videoMode === 'fal' ? input('backendModelId', 'fal.ai model (optional)', { placeholder: 'Use configured fal.ai default', maxLength: 64 })+ : form.videoMode === 'runway' ? select('backendModelId', 'Runway model', [['', `${getRunwayVideoModel(RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL).label} (backend default)`], ...(form.backendModelId && !getRunwayVideoModel(form.backendModelId) ? [[form.backendModelId, `${form.backendModelId} (saved model unavailable)`]] : []), ...RUNWAY_VIDEO_MODELS.map(m => [m.id, m.label])]): select('backendModelId', 'Media model', [['', form.videoMode === 'reactor' ? 'fast-h3 (backend default)' : 'Backend default'], ...(form.backendModelId ? [[form.backendModelId, `${form.backendModelId} (saved model)`]] : [])])}<p className="text-sm text-port-text-muted">Backend selections are saved without running providers. Cloud renders may incur charges when production is enabled.</p>{select('transition', 'Shot joins', [['cut', 'Straight cuts'], ['fade', 'Fade through black (no overlap)']])}diff --git a/client/src/components/fableloom/LoomSettingsDrawer.jsx b/client/src/components/fableloom/LoomSettingsDrawer.jsxindex aa72628998..b336fc0e5f 100644--- a/client/src/components/fableloom/LoomSettingsDrawer.jsx+++ b/client/src/components/fableloom/LoomSettingsDrawer.jsx@@ -54,6 +54,7 @@ const VIDEO_RENDER_BACKENDS = [{ id: 'grok', label: 'Grok', icon: Cloud },{ id: 'fal', label: 'fal.ai', icon: Cloud },{ id: 'reactor', label: 'Reactor.inc', icon: Cloud },+ { id: 'runway', label: 'Runway', icon: Cloud },];const modelOptions = (models, selected) => {diff --git a/client/src/hooks/useVideoGenForm.js b/client/src/hooks/useVideoGenForm.jsindex 31d59b42fa..fb2b9b8c42 100644--- a/client/src/hooks/useVideoGenForm.js+++ b/client/src/hooks/useVideoGenForm.js@@ -24,6 +24,7 @@ import {resolveDraftDecodeForModel, draftDecodeFromRecord,} from '../lib/videoGenParams.js';import { videoStreamingModeFromRecord } from '../lib/videoStreamingMode.js';+import { RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL, getRunwayVideoModel, runwayModelTakesText } from '../lib/runwayVideoModels.js';import { useVideoGenFieldState } from './useVideoGenFieldState.js';import { useVideoGenSubmitFlow } from './useVideoGenSubmitFlow.js';import { useVideoGenValidation } from './useVideoGenValidation.js';@@ -659,8 +660,13 @@ export function useVideoGenForm({// (`reactorEnabled`) — same short-circuit shape as fal, plus native// clip-to-clip chaining via continue_from_clip_id.const isReactor = hostedLane === 'reactor';- // Runway is usability-gated the same way, and renders image-to-video only.+ // Runway is usability-gated the same way; text mode is offered for the models that render from a prompt alone.const isRunway = hostedLane === 'runway';+ // A model that needs a start frame cannot stay picked in Text mode: fall back to the catalog default (which renders from text).+ const runwayTextBlocked = isRunway && mode === 'text' && !runwayModelTakesText(getRunwayVideoModel(runwayModelId || RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL));+ useEffect(() => {+ if (runwayTextBlocked) setRunwayModelId('');+ }, [runwayTextBlocked, setRunwayModelId]);const referenceModeApplies = mode === 'image' && !hostedLane;// The strength the render will actually use, for the slider readout — an// untouched slider under Inspire still resolves to the contract's low default@@ -945,9 +951,7 @@ export function useVideoGenForm({// clip survives the switch and reappears if the user flips back.const handleBackendChange = (id) => {setBackend(id);- if (id === 'runway' && mode !== 'image') {- handleModeChange('image');- } else if ((id === 'grok' || id === 'fal' || id === 'reactor') && mode !== 'text' && mode !== 'image') {+ if ((id === 'grok' || id === 'fal' || id === 'reactor' || id === 'runway') && mode !== 'text' && mode !== 'image') {handleModeChange((sourceImageFile || sourceImageUpload) ? 'image' : 'text');}};@@ -1248,9 +1252,9 @@ export function useVideoGenForm({if (p.duration) setFalDuration(p.duration);if (p.modelId) setFalModelId(p.modelId);} else if (p.mode === 'runway') {- // Runway job: same discriminator shape as fal above; image-to-video only.+ // Runway job: same discriminator shape as fal above.setBackend('runway');- setMode('image');+ setMode(p.videoMode === 'text' ? 'text' : 'image');if (p.modelId) {runwayModelPickRef.current = p.modelId;setRunwayModelId(p.modelId);diff --git a/client/src/hooks/useVideoGenForm.test.jsx b/client/src/hooks/useVideoGenForm.test.jsxindex 701ecb40f3..0530da4b5a 100644--- a/client/src/hooks/useVideoGenForm.test.jsx+++ b/client/src/hooks/useVideoGenForm.test.jsx@@ -942,6 +942,21 @@ describe('useVideoGenForm', () => {expect(result.current.modelId).toBe(LTX2.id);});+ it('restores a queued Runway text render without changing it to image mode', () => {+ const { result, rerender } = render();+ rerender({ models: MODELS, modelContext: MODEL_CONTEXT, availableLoras: [], runwayEnabled: true });+ act(() => result.current.applyResumedParams({+ mode: 'runway', videoMode: 'text', modelId: 'gen4.5', duration: 7, resolution: '720p',+ prompt: 'a fox watches the rain',+ }));+ expect(result.current.hostedLane).toBe('runway');+ expect(result.current.mode).toBe('text');+ expect(result.current.runwayModelId).toBe('gen4.5');+ expect(result.current.runwayDuration).toBe('7');+ expect(result.current.runwayResolution).toBe('720p');+ expect(result.current.prompt).toBe('a fox watches the rain');+ });+it('resolves a stored hosted backend whose key is no longer configured to the local lane', async () => {const { result, rerender } = render({ grokEnabled: false });const props = { models: MODELS, modelContext: MODEL_CONTEXT, availableLoras: [], grokEnabled: false };diff --git a/client/src/lib/runwayVideoModels.js b/client/src/lib/runwayVideoModels.jsindex bee2abe540..8552d47f2a 100644--- a/client/src/lib/runwayVideoModels.js+++ b/client/src/lib/runwayVideoModels.js@@ -13,6 +13,7 @@ export {estimateRunwayVideoCostUsd,getRunwayVideoModel,runwayVideoClipQuote,+ runwayModelTakesText,runwayVideoLengths,runwayVideoResolutions,} from '../../../server/lib/runwayVideoModels.js';diff --git a/client/src/lib/videoGenSubmission.js b/client/src/lib/videoGenSubmission.jsindex d1b476a69c..aed469cddc 100644--- a/client/src/lib/videoGenSubmission.js+++ b/client/src/lib/videoGenSubmission.js@@ -115,8 +115,8 @@ function falVideoSubmission({};}-// Runway renders image-to-video only: a start frame is required (the form-// offers Image mode alone). The negative prompt is sent; the server passes it+// Runway renders from a start frame (Image mode) or, on a model with a+// text-to-video rule, from the prompt alone (Text mode). The negative prompt is sent; the server passes it// natively or as an "Avoid:" line, listed as a chip in the sent-prompt view.function runwayVideoSubmission({composed, runwayModelId, runwayDuration, runwayResolution,diff --git a/client/src/pages/VideoGen.jsx b/client/src/pages/VideoGen.jsxindex c94df74519..b6a6063124 100644--- a/client/src/pages/VideoGen.jsx+++ b/client/src/pages/VideoGen.jsx@@ -107,7 +107,7 @@ import {import { REACTOR_MAX_PROMPT_LENGTH } from '../lib/reactorVideoClip.js';import {RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL, RUNWAY_VIDEO_MODELS, describeRunwayVideoRate, estimateRunwayVideoCostUsd, getRunwayVideoModel,- runwayVideoLengths, runwayVideoResolutions,+ runwayModelTakesText, runwayVideoLengths, runwayVideoResolutions,} from '../lib/runwayVideoModels.js';import { videoGenEnqueuePolicy } from '../lib/videoGenEnqueuePolicy.js';import { styledVideoPrompt } from '../lib/videoGenSubmission.js';@@ -1170,7 +1170,7 @@ export default function VideoGen() {: id === 'reactor'? 'Render via the reactor.inc fast-h3 API. Counts against your reactor.inc balance.': id === 'runway'- ? 'Animate a start frame via the Runway API. Counts against your Runway credits.'+ ? 'Render from text or animate a start frame via the Runway API. Counts against your Runway credits.': 'Render on this machine with the local runtimes.'}>{label}@@ -1185,8 +1185,7 @@ export default function VideoGen() {WAI-ARIA Tabs, since the mode-specific inputs aren't structured astabpanels and we don't implement roving-tabindex/arrow-key focus. */}<div className="bg-port-card border border-port-border rounded-xl p-1 flex flex-wrap gap-1" role="group" aria-label="Video generation mode">- {(isRunway ? MODES.filter((m) => m.id === 'image')- : rendersOffMachine ? MODES.filter((m) => m.id === 'text' || m.id === 'image') : MODES).map(({ id, label, icon: Icon, desc }) => {+ {(rendersOffMachine ? MODES.filter((m) => m.id === 'text' || m.id === 'image') : MODES).map(({ id, label, icon: Icon, desc }) => {const active = mode === id;return (<button@@ -1668,7 +1667,7 @@ export default function VideoGen() {onChange={(e) => setRunwayModelId(e.target.value)}className="w-full bg-port-bg border border-port-border rounded-lg px-2 py-2 text-sm text-white focus:outline-none focus:border-port-accent">- {RUNWAY_VIDEO_MODELS.map((m) => (+ {RUNWAY_VIDEO_MODELS.filter((m) => mode !== 'text' || runwayModelTakesText(m)).map((m) => (<option key={m.id} value={m.id}>{`${m.label} · ${describeRunwayVideoRate(m.id)}${m.id === RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL ? ' (default)' : ''}`}</option>@@ -1709,7 +1708,7 @@ export default function VideoGen() {</FormField>)}<p className="col-span-2 text-[11px] text-gray-500 leading-snug">- Animates your start frame on the Runway API. Counts against your Runway credits+ {mode === 'image' ? 'Animates your start frame' : 'Renders from your prompt'} on the Runway API. Counts against your Runway credits{(() => {const estimate = estimateRunwayVideoCostUsd({modelId: runwayModel?.id, seconds: runwayDuration === '' ? null : Number(runwayDuration), resolution: runwayResolution,diff --git a/client/src/pages/VideoGen.runway.test.jsx b/client/src/pages/VideoGen.runway.test.jsxnew file mode 100644index 0000000000..30d9ae6cd0--- /dev/null+++ b/client/src/pages/VideoGen.runway.test.jsx@@ -0,0 +1,41 @@+import { beforeEach, describe, expect, it } from 'vitest';+import { fireEvent, screen, waitFor, within } from '@testing-library/react';+import { findEnabledByRole } from '../test/enabledBarrier.js';+import {+ loadVideoGenPage, renderVideoGenPage, resetVideoGenMockState,+ state, videoGenModel, videoGenModelContext, videoGenStatus,+} from '../test/videoGenPageMocks.jsx';++const MODEL = videoGenModel('local-model');+await loadVideoGenPage();++describe('VideoGen Runway text lane', () => {+ beforeEach(() => {+ localStorage.clear();+ resetVideoGenMockState();+ state.getVideoGenModelContext.mockResolvedValue(videoGenModelContext([MODEL]));+ state.getVideoGenStatus.mockResolvedValue(videoGenStatus([MODEL], { runwayEnabled: true }));+ state.generateVideo.mockResolvedValue({ jobId: 'job-1' });+ });++ it('switches from an image-only model to a priced text-capable model and submits without a frame', async () => {+ await renderVideoGenPage();+ fireEvent.click(await screen.findByRole('button', { name: 'Runway' }));+ const modes = within(screen.getByRole('group', { name: 'Video generation mode' }));+ fireEvent.click(modes.getByRole('button', { name: 'Image' }));+ const model = await screen.findByLabelText('Runway model');+ fireEvent.change(model, { target: { value: 'gen4_turbo' } });+ expect(model).toHaveValue('gen4_turbo');+ fireEvent.click(modes.getByRole('button', { name: 'Text' }));+ await waitFor(() => expect(model).toHaveValue('gen4.5'));+ expect([...model.options].map(option => option.value)).toEqual(['gen4.5']);+ expect(screen.getByText(/about \$0.60 at list price/)).toBeInTheDocument();+ fireEvent.change(screen.getByLabelText('Prompt'), { target: { value: 'a fox watches the rain' } });+ fireEvent.click(await findEnabledByRole('button', { name: /Add to queue/ }));+ await waitFor(() => expect(state.generateVideo).toHaveBeenCalled());+ const payload = state.generateVideo.mock.calls[0][0];+ expect(payload).toMatchObject({ backend: 'runway', prompt: 'a fox watches the rain' });+ expect(payload.sourceImageFile).toBe('');+ expect(payload.sourceImage).toBe('');+ });+});diff --git a/server/lib/README.md b/server/lib/README.mdindex 987ab0a5a8..eb7ffa6b46 100644--- a/server/lib/README.md+++ b/server/lib/README.md@@ -344,7 +344,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub| `musicVideoShotTiming.js` | Music Video shot modes (#8977): `MUSIC_VIDEO_SHOT_MODES` (`cutaway`/`performance`), `isPerformanceScene`, `MUSIC_VIDEO_PERFORMANCE_STYLES` (`lipsync`/`music`) + `isLipSyncScene`, `selectedPerformanceInstruction(scene)` (the selected take's shot instruction, performance scenes only), `clipAudioOffsetSec(instruction)` (provider lead padding a landed take's edit points were moved by, #11210), `SOURCE_AUDIO_LIPSYNC` (verified source-audio lip-sync lanes — only fal.ai MiniMax H3 lip-sync, 5–30s per take, default 1080P), `SOURCE_AUDIO_MUSIC` (music-driven lanes — fal.ai audio-to-video models, picked by `videoSettings.falMusicModelId`, default OmniHuman v1.5 at 1080p), `falSceneTake()` (the fal model/length/resolution a scene take requests and its estimated cost — shared by the board, the submit path and the autopilot spend cap) + `falTakeRequestFields(take)` (that take as `/api/video-gen` fields), `performanceCapability(backend, { scene, videoSettings })` (lip-sync lane, or the music-driven one for a music-driven scene), `performanceBlockedReason(backend)` (Grok is cutaway-only), `planPerformanceWindow()` (provider-bounded audio window + clip-relative edit in/out points; refuses over-long shots instead of relying on provider truncation), `planPerformanceRepair()` (complete measured suffix evidence, safe word-gap boundary, source/clip timebases and next spend), `clipRelativeCues()`, `grokCoverage(spanSec)` (covering 6/10/15 s request, flags spans needing a split), `approximateMotionCues()`, `shotSplitLimit(scene, backend)` (per-take limit: lip-sync window or Grok's longest clip), `planShotSplit()` (contiguous pieces cut at lyric pauses → line/phrase boundaries → beats; one extra piece at most to avoid cutting a sung line). Dependency-free; the client re-exports it. || `reactorStartingFrame.js` | Fits a reactor.inc `starting_frame` to the fast-h3 session canvas before it is uploaded. `readImageSize(path)` → `{width,height}` or `null` (never a guessed dimension), and `prepareReactorStartingFrame(sourceImagePath, requestedAspect, outputPath)` → `{ aspect, canvas, framePath, fittedPath }`: an absent `requestedAspect` DERIVES the canvas from the frame's own shape (`nearestReactorAspect`), then centre cover-crops the frame to that canvas's exact pixel size so reactor's own "fitted to the session canvas" step is a no-op. `fittedPath` is the temp file the caller deletes; `null` means the original was uploaded unchanged. A frame sharp cannot decode (or a resize that fails) passes through untouched on the default canvas rather than failing a render PortOS merely could not measure — reactor decodes more containers than sharp does. || `reactorVideoClip.js` | The reactor.inc `fast-h3` clip contract, dependency-free so the route schema, the FableLoom prompt compiler and the VideoGen picker all read one copy — `REACTOR_MAX_PROMPT_LENGTH` (800; the API rejects a longer prompt rather than truncating it), `REACTOR_MIN_CLIP_SECONDS` / `REACTOR_MAX_CLIP_SECONDS` (5.167–14.375, i.e. 124–345 frames at the `REACTOR_CLIP_FPS` 24 it delivers), `REACTOR_MAX_CLIP_ID_LENGTH`, and the picker list `REACTOR_CLIP_LENGTHS` + `REACTOR_DEFAULT_CLIP_LENGTH` + `reactorClipLengthLabel()` (the two exact endpoints plus every whole second between them, since the accepted range is continuous but starts and ends off a round number). Also the CANVAS contract `set_canvas` accepts: `REACTOR_CANVASES` (`16:9` 1344×768, `4:3` 1024×768, `1:1` 768×768, `9:16` 768×1344 — every one a 768px short edge, so the aspect string is the whole choice), `REACTOR_ASPECTS`, `REACTOR_DEFAULT_ASPECT`, `reactorCanvas(aspect)` and `nearestReactorAspect(width, height)` (closest canvas in log-ratio space; anything unmeasurable answers the default rather than deriving one). Mirrored to `client/src/lib/reactorVideoClip.js`, parity pinned by `reactorVideoClip.test.js`. |-| `runwayVideoModels.js` | Curated Runway VIDEO model catalog for the developer API's `/v1/image_to_video` — `RUNWAY_VIDEO_MODELS` (Gen-4.5 default, Gen-4 Turbo, Veo 3.1 / Fast, Kling 3.0 Standard / Pro, Kling 2.5 Turbo Pro, MiniMax H3 Max, WAN 3.0) with per-model prompt caps, duration rules, output sizes (`ratio` or `resolution`), end-frame support and list prices verified on `RUNWAY_VIDEO_PRICING_VERIFIED_AT`; `getRunwayVideoModel()`, `coerceRunwayVideoSeconds()` (round UP to a covering length), `resolveRunwayVideoResolution()`, `runwayVideoLengths()`, `buildRunwayVideoRequest()` (the exact body; refuses an uncurated model, a missing frame or an over-cap prompt, never cuts it), `runwayVideoClipQuote()` / `estimateRunwayVideoCostUsd()`, `describeRunwayVideoRate()`. Dependency-free; the client re-exports it. |+| `runwayVideoModels.js` | Curated Runway VIDEO model catalog for the developer API's `/v1/image_to_video` and (Gen-4.5) `/v1/text_to_video` — `RUNWAY_VIDEO_MODELS` (Gen-4.5 default, Gen-4 Turbo, Kling 3.0 Standard / Pro, Kling 2.5 Turbo Pro, MiniMax H3 Max, WAN 3.0) with per-model prompt caps, duration rules, output sizes (`ratio` or `resolution`), end-frame support and list prices verified on `RUNWAY_VIDEO_PRICING_VERIFIED_AT`; `getRunwayVideoModel()`, `coerceRunwayVideoSeconds()` (round UP to a covering length), `resolveRunwayVideoResolution()`, `runwayVideoLengths()`, `buildRunwayVideoRequest()` (the endpoint and exact body, text-only when no frame and the model has a `textToVideo` rule, `runwayModelTakesText()`; refuses an uncurated model, a missing frame or an over-cap prompt, never cuts it), `runwayVideoClipQuote()` / `estimateRunwayVideoCostUsd()`, `describeRunwayVideoRate()`. Dependency-free; the client re-exports it. || `kimi.js` | Moonshot AI Kimi Code (`kimi`) provider helpers — id constants (`KIMI_CLI_ID`/`KIMI_TUI_ID`), `isKimiCommand`/`isKimiCliProvider`/`isKimiTuiProvider` predicates, `ensureKimiHeadlessArgs` (model flag only — non-interactive mode is implicit in `--prompt`, and kimi has no `--print`/`--afk` and refuses approval flags alongside `--prompt`) / `ensureKimiTuiArgs` (`--yolo`) argv builders (model selection uses the `KIMI_CONFIGURED_DEFAULT` sentinel in `providerModels.js` so PortOS omits `--model` like Grok/Antigravity), and `prepareKimiPrompt` (delivers the prompt as the `--prompt <value>` argv, not stdin). || `runners.js` | Image-runner family constants (`RUNNER_FAMILIES`, `loraCompatKey`), video LoRA families, MiniMax H3 runtime predicates, and the shared `AUDIO_TO_VIDEO_RUNTIMES` / `isAudioToVideoRuntime` capability gate (LTX plus H3 Ref2VA). Mirrored to `client/src/lib/runnerFamilies.js`. || `codexAccount.js` | The ChatGPT-subscription half of the Codex providers (#5589), pure. The fixed `codex app-server` invocation, the `account/*` method + notification names, `CODEX_ACCOUNT_STATUS` (`runtime-missing` / `unknown` / `signed-out` / `login-pending` / `ready` / `quota-exhausted` / `reauth-required`) and `CODEX_ERROR_CODES`, `isCodexSubscriptionProvider` (re-exported from `providerModels.js`; keyed on the `codex` COMMAND, so a renamed clone still resolves), the `normalizeCodexAccount` / `normalizeCodexRateLimits` / `normalizeCodexLoginStart` payload normalizers, `redactCodexPayload` (every log/error/response path runs through it, so "PortOS never writes down a Codex token" is one function rather than a rule per call site), and `deriveCodexAccountStatus`. Sentinels are load-bearing: `runtimeInstalled: null` = NOT PROBED, `accountFetched: false` ≠ signed out, `rateLimits: null` (failed fetch) ≠ a fetched window with nothing to report. The process half is `services/codexAppServer.js`. |diff --git a/server/lib/runwayVideoModels.js b/server/lib/runwayVideoModels.jsindex b2793cbfd7..27b17375a9 100644--- a/server/lib/runwayVideoModels.js+++ b/server/lib/runwayVideoModels.js@@ -1,6 +1,7 @@/*** Curated Runway VIDEO model catalog — what PortOS knows about each model it- * offers through Runway's developer API (`POST /v1/image_to_video`): the+ * offers through Runway's developer API (`POST /v1/image_to_video`, and+ * `POST /v1/text_to_video` for a row with a `textToVideo` rule): the* request fields each model takes (the body is one shape per `model`), its* duration and output-size rules, whether it ends on a last frame, and its* published price.@@ -15,8 +16,11 @@* Runway serves it; the native `negativePrompt` row field stays for a model* that takes one.*- * Image-to-video only: every row animates a start frame, and the rows marked- * `endImage` also end on a last frame. Provider audio is always requested off+ * Every row animates a start frame, and the rows marked `endImage` also end on+ * a last frame. Only Gen-4.5 also renders from text alone; its `textToVideo`+ * rule lists the ratios `/v1/text_to_video` takes (the duration, prompt cap,+ * seed and price are the row's own). The other curated rows stay image-only+ * until a text-to-video need for them is checked against the API reference. Provider audio is always requested off* (`audio: false`) where a model has the field, since a PortOS clip is silent* unless the caller asks (a Music Video's master song is its only soundtrack),* and the silent rate is what is priced.@@ -45,6 +49,8 @@ export const RUNWAY_VIDEO_MODELS = Object.freeze([// `ratio` is required and names the output canvas.size: { param: 'ratio', required: true, byResolution: { '720p': GEN4_RATIOS }, default: '720p' },audio: false, seed: true, negativePrompt: false,+ // `ratio` is required on /v1/text_to_video and offers only these two.+ textToVideo: { ratios: Object.freeze(['1280:720', '720:1280']) },pricing: { perSecond: 0.12 },},{@@ -114,6 +120,9 @@ const BY_ID = new Map(RUNWAY_VIDEO_MODELS.map((m) => [m.id, m]));/** The catalog row for `modelId`, or null for a model PortOS has not curated. */export const getRunwayVideoModel = (modelId) => (typeof modelId === 'string' ? BY_ID.get(modelId.trim()) || null : null);+/** True when `model` renders from a prompt alone (`/v1/text_to_video`). */+export const runwayModelTakesText = (model) => !!model?.textToVideo;+/** Output sizes a model offers (empty when it has no size parameter). */export const runwayVideoResolutions = (model) => (model?.size ? Object.keys(model.size.byResolution) : []);@@ -215,8 +224,10 @@ export function describeRunwayVideoRate(modelId, resolution) {}/**- * Build the `/v1/image_to_video` body for a curated model. Returns `{ body,- * model, seconds, resolution, estimatedCostUsd }`. A missing required input, an+ * Build the Runway request for a curated model: `/v1/image_to_video` with a+ * start frame, `/v1/text_to_video` without one (a model with a `textToVideo`+ * rule). Returns `{ endpoint, body, model, seconds, resolution,+ * estimatedCostUsd }`. A missing required input, an* input the model cannot take, an uncurated model, or an over-cap prompt throws* an Error with `code = 'RUNWAY_MODEL_INPUT'` so the caller refuses before* anything is paid for. A prompt is never shortened (AGENTS.md, #11317).@@ -240,19 +251,27 @@ export function buildRunwayVideoRequest({if (model.negativePrompt && avoid.length > model.negativePromptMaxChars) {throw refuse(`The negative prompt is ${avoid.length.toLocaleString('en-US')} characters; ${model.label} takes ${model.negativePromptMaxChars.toLocaleString('en-US')}. Shorten it to render.`);}- if (!imageUri) throw refuse(`${model.label} renders from a start frame. Supply a source image.`);+ const textOnly = !imageUri;+ if (textOnly && !runwayModelTakesText(model)) throw refuse(`${model.label} renders from a start frame. Supply a source image.`);+ if (textOnly && endImageUri) throw refuse('An end frame needs a start frame. Supply a source image.');if (endImageUri && !model.endImage) throw refuse(`${model.label} takes no end frame; choose a model that does`);const body = { model: model.id };- body.promptImage = endImageUri- ? [{ uri: imageUri, position: 'first' }, { uri: endImageUri, position: 'last' }]- : imageUri;+ if (!textOnly) {+ body.promptImage = endImageUri+ ? [{ uri: imageUri, position: 'first' }, { uri: endImageUri, position: 'last' }]+ : imageUri;+ }if (sent) body.promptText = sent;if (model.negativePrompt && avoid) body.negativePrompt = avoid;const coercedSec = coerceRunwayVideoSeconds(model, seconds);body.duration = coercedSec;const res = resolveRunwayVideoResolution(model, resolution);- if (model.size) {+ if (textOnly) {+ // No frame to follow, so the canvas is always named: the offered ratio closest to the target.+ const want = aspectOf({ aspectRatio, width, height }) ?? 16 / 9;+ body.ratio = model.textToVideo.ratios.reduce((best, r) => (Math.abs(ratioOf(r) - want) < Math.abs(ratioOf(best) - want) ? r : best));+ } else if (model.size) {const target = aspectOf({ aspectRatio, width, height });// An optional ratio is sent only when there is a target to match and no// last frame (a keyframe render's canvas follows its first frame);@@ -266,6 +285,7 @@ export function buildRunwayVideoRequest({if (model.seed && Number.isInteger(seed) && seed >= 0 && seed <= 4294967295) body.seed = seed;return {+ endpoint: textOnly ? 'text_to_video' : 'image_to_video',body,model,seconds: coercedSec,diff --git a/server/lib/runwayVideoModels.test.js b/server/lib/runwayVideoModels.test.jsindex 7b1c8d792a..ec46d2e1be 100644--- a/server/lib/runwayVideoModels.test.js+++ b/server/lib/runwayVideoModels.test.js@@ -58,10 +58,31 @@ describe('buildRunwayVideoRequest — per-family wire bodies', () => {});});+describe('buildRunwayVideoRequest — text-to-video', () => {+ it('Gen-4.5 with no start frame: /v1/text_to_video body, ratio matched to the shape, priced, Avoid folded in', () => {+ const portrait = buildRunwayVideoRequest({+ prompt: 'a lighthouse at dusk', negativePrompt: 'text', seconds: 3.2, width: 1080, height: 1920, seed: 7,+ });+ expect(portrait.endpoint).toBe('text_to_video');+ expect(portrait.body).toEqual({+ model: 'gen4.5', promptText: 'a lighthouse at dusk\nAvoid: text', duration: 4, ratio: '720:1280', seed: 7,+ });+ expect(portrait.estimatedCostUsd).toBe(0.48);+ // No target shape: Runway's documented landscape canvas.+ expect(buildRunwayVideoRequest({ prompt: 'a fox' }).body.ratio).toBe('1280:720');+ expect(build('gen4.5').endpoint).toBe('image_to_video');+ });++ it('refuses an over-cap text prompt with its count and never shortens it', () => {+ expect(() => buildRunwayVideoRequest({ prompt: 'x'.repeat(1001) })).toThrow(/1,001 characters/);+ });+});+describe('buildRunwayVideoRequest — refusals before anything is paid for', () => {it.each([['an uncurated model', { modelId: 'gen9' }, /not a Runway model/],- ['a missing start frame', { imageUri: null }, /start frame/],+ ['a missing start frame on an image-only model', { modelId: 'gen4_turbo', imageUri: null }, /start frame/],+ ['an end frame without a start frame', { imageUri: null, endImageUri: END }, /end frame needs a start frame/],['an end frame on a start-frame-only model', { modelId: 'gen4.5', endImageUri: END }, /takes no end frame/],['a missing required prompt', { modelId: 'gen4.5', prompt: ' ' }, /needs a prompt/],])('refuses %s', (_label, extra, message) => {diff --git a/server/services/fableLoom/production.js b/server/services/fableLoom/production.jsindex 6ec29cefff..573c0fc705 100644--- a/server/services/fableLoom/production.js+++ b/server/services/fableLoom/production.js@@ -481,6 +481,9 @@ function resolveVideoBackend(settings, render, recordedConditioning = null) {if (requested === VIDEO_GEN_MODE.REACTOR && mode !== VIDEO_GEN_MODE.REACTOR) {throw new ServerError('Reactor video is not configured — set an API key in Settings → Video Gen.', { status: 400, code: 'REACTOR_NOT_CONFIGURED' });}+ if (requested === VIDEO_GEN_MODE.RUNWAY && mode !== VIDEO_GEN_MODE.RUNWAY) {+ throw new ServerError('No Runway API key configured — set it in Settings → Video Gen first', { status: 400, code: 'RUNWAY_NOT_CONFIGURED' });+ }return mode;}@@ -624,7 +627,9 @@ async function prepareVideoJob(run, asset) {? { id: 'fal-video', supportedModes: ['text', 'image'] }: backend === VIDEO_GEN_MODE.REACTOR? { id: 'reactor-video', supportedModes: ['text', 'image'] }- : { id: 'grok-video', supportedModes: ['image'] },+ : backend === VIDEO_GEN_MODE.RUNWAY+ ? { id: 'runway-video', supportedModes: ['text', 'image'] }+ : { id: 'grok-video', supportedModes: ['image'] },});const requestedSourceImagePath = imageInputForNode(run, asset);const conditioned = await conditionRequest(run, asset, 'video', capability, requestedSourceImagePath);@@ -714,6 +719,30 @@ async function prepareVideoJob(run, asset) {};}+ if (backend === VIDEO_GEN_MODE.RUNWAY) {+ // As with fal, the model is the lane's catalog default: a cloud video backend carries no model pick here.+ return {+ kind: 'video',+ provider: backend,+ modelId: null,+ modelRevision: capability.modelRevision || null,+ params: {+ mode: VIDEO_GEN_MODE.RUNWAY,+ videoMode: sourceImagePath ? 'image' : 'text',+ prompt: conditioned?.prompt || asset.prompt,+ ...(conditioned?.promptAdditions?.length ? { promptAdditions: conditioned.promptAdditions } : {}),+ negativePrompt: conditioned?.negativePrompt || '',+ width: geometry.width,+ height: geometry.height,+ aspectRatio: geometry.aspectRatio,+ ...replayParameters,+ sourceImagePath,+ visualConditioning: conditioned?.visualConditioning || null,+ fableLoom: productionTag(run, asset),+ },+ };+ }+const cloud = resolveRenderTargetConfig(settings, RENDER_TARGET.FABLELOOM_PRODUCTION, {mode: IMAGE_GEN_MODE.GROK,model: recordedConditioning?.capability?.modelId || null,diff --git a/server/services/fableLoom/production.test.js b/server/services/fableLoom/production.test.jsindex 3e228f138e..f257bcd73b 100644--- a/server/services/fableLoom/production.test.js+++ b/server/services/fableLoom/production.test.js@@ -330,6 +330,27 @@ describe('fableLoom production service', () => {expect(snapshots.every(snapshot => !('tail' in snapshot) && !('runtime' in snapshot))).toBe(true);});+ it('queues the Runway text lane with the compiled prompt and no invented source frame', async () => {+ const loom = structuredClone(sampleLoom);+ loom.episodes[0].nodes.forEach(node => { node.image = 'example-frame.png'; });+ records.getLoom.mockResolvedValue(loom);+ settingsMocks.getSettings.mockResolvedValue({ videoGen: { runway: { apiKey: 'example-key' } } });+ const prompt = 'A fox crosses the clearing at dawn.';+ visualConditioningMocks.compileFableLoomVisualRequest.mockResolvedValue({+ prompt, sourceImagePath: null, visualConditioning: { compiledPrompt: prompt },+ });+ await startEpisodeProductionBatch('loom-1', 'ep-1', {+ videoMode: 'runway', assetTypes: ['video_entry'], nodeIds: ['node-1'],+ });+ await vi.waitFor(() => expect(queueMocks.enqueueJob).toHaveBeenCalled());+ expect(queueMocks.enqueueJob).toHaveBeenCalledWith(expect.objectContaining({+ kind: 'video', params: expect.objectContaining({+ mode: 'runway', videoMode: 'text', prompt, sourceImagePath: null,+ visualConditioning: expect.objectContaining({ compiledPrompt: prompt }),+ }),+ }));+ });+it('queues Reactor with its compact compiled script and fails oversized scripts before queueing', async () => {const loom = structuredClone(sampleLoom);loom.episodes[0].nodes.forEach((node) => { node.image = 'example-frame.png'; });diff --git a/server/services/videoGen/hostedSubmission.js b/server/services/videoGen/hostedSubmission.jsindex 54b4325308..5c39c97556 100644--- a/server/services/videoGen/hostedSubmission.js+++ b/server/services/videoGen/hostedSubmission.js@@ -7,7 +7,7 @@ import { ServerError } from '../../lib/errorHandler.js';import { resolveGalleryImage } from '../../lib/fileUtils.js';import { probeVideoDuration } from '../../lib/ffmpeg.js';import { FAL_DEFAULT_IMAGE_VIDEO_MODEL, FAL_DEFAULT_TEXT_VIDEO_MODEL, getFalVideoModel } from '../../lib/falVideoModels.js';-import { RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL, getRunwayVideoModel } from '../../lib/runwayVideoModels.js';+import { RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL, getRunwayVideoModel, runwayModelTakesText } from '../../lib/runwayVideoModels.js';import { isDefaultI2vReferenceMode } from '../../lib/videoReferenceModes.js';import { VIDEO_GEN_MODE, isVideoModeUsable } from './modes.js';@@ -59,7 +59,8 @@ async function stageEndFrame({ body, uploads, stageUploadDurable }) {return { path: resolved, owned: false };}-// Runway renders from a start frame only, with an end frame on the models that take one.+// Runway renders from a start frame (an end frame on the models that take one),+// or from text alone on a model with a text-to-video rule.async function stageRunwayInputs(ctx) {const { body, uploads, sourceImagePath, cleanupStaged } = ctx;const refuse = async (message) => {@@ -67,9 +68,10 @@ async function stageRunwayInputs(ctx) {throw new ServerError(message, { status: 400, code: 'VALIDATION_ERROR' });};if (uploads.audioFile) await refuse('Runway renders cutaways only; it takes no source audio.');- if (!sourceImagePath) await refuse('Runway renders from a start frame — supply a source image.');const model = getRunwayVideoModel(body.runwayModelId || RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL);if (!model) await refuse(`"${body.runwayModelId}" is not a Runway model PortOS offers`);+ if (!sourceImagePath && !runwayModelTakesText(model)) await refuse(`${model.label} renders from a start frame — supply a source image.`);+ if (!sourceImagePath && (uploads.lastImage || body.lastImageFile)) await refuse('An end frame needs a start frame — supply a source image.');if (!uploads.lastImage && !body.lastImageFile) return {};if (!model.endImage) await refuse(`${model.label} takes no end frame`);const endFrame = await stageEndFrame(ctx);diff --git a/server/services/videoGen/runway.js b/server/services/videoGen/runway.jsindex b8ed7e3f13..20625f0bc0 100644--- a/server/services/videoGen/runway.js+++ b/server/services/videoGen/runway.js@@ -4,7 +4,8 @@* A cloud lane like `videoGen/fal.js`: the same job-map/SSE contract, so* `mediaJobQueue` dispatches to it on `params.mode === 'runway'`. Flow: POST* the start frame (and an optional last frame) as data URIs to- * `/v1/image_to_video`, poll the task until it SUCCEEDS, download `output[0]`+ * `/v1/image_to_video` — or just the prompt to `/v1/text_to_video` when no frame+ * is given and the model renders from text — poll the task until it SUCCEEDS, download `output[0]`* and hand it to the shared `finalizeGeneratedVideo` helper.** The request body is built per model from the curated catalog@@ -177,7 +178,7 @@ async function generateVideoPrepared({...sentRecord,filename,createdAt: new Date().toISOString(),- mode: 'image',+ mode: sourceImagePath ? 'image' : 'text',...(shotInstruction?.audioWindow ? { audioWindow: shotInstruction.audioWindow } : {}),};const job = { ...meta, clients: [], status: 'running', renderStartedAtMs };@@ -245,18 +246,18 @@ async function runRunwayVideo(job, jobId, {activeRequests.set(jobId, entry);const deadline = Date.now() + RUNWAY_RENDER_TIMEOUT_MS;try {- const frame = await runwayFrameDataUri(sourceImagePath, frameAspect);+ const frame = sourceImagePath ? await runwayFrameDataUri(sourceImagePath, frameAspect) : null;const endFrame = lastImagePath ? await runwayFrameDataUri(lastImagePath, frameAspect) : null;for (const [label, fit] of [['start', frame], ['end', endFrame]]) {if (fit?.conformed) console.log(`✂️ Cropped Runway ${label} frame [${jobId.slice(0, 8)}] ${fit.conformed.from} → ${fit.conformed.to} for ${frameAspect}`);if (fit?.reencoded) console.log(`🗜️ Re-encoded Runway ${label} frame [${jobId.slice(0, 8)}] as JPEG to fit the 5 MB inline limit`);}- if (frame.conformed || endFrame?.conformed) {- meta.frameConformed = { aspect: frameAspect, ...(frame.conformed ? { start: frame.conformed } : {}), ...(endFrame?.conformed ? { end: endFrame.conformed } : {}) };+ if (frame?.conformed || endFrame?.conformed) {+ meta.frameConformed = { aspect: frameAspect, ...(frame?.conformed ? { start: frame.conformed } : {}), ...(endFrame?.conformed ? { end: endFrame.conformed } : {}) };}- const { body } = buildRunwayVideoRequest({ ...requestSpec, imageUri: frame.dataUri, endImageUri: endFrame?.dataUri ?? null });+ const { body, endpoint } = buildRunwayVideoRequest({ ...requestSpec, imageUri: frame?.dataUri ?? null, endImageUri: endFrame?.dataUri ?? null });if (entry.aborted) return finishFailure('Canceled');- entry.taskId = await submitRunwayTask({ apiKey, endpoint: 'image_to_video', body });+ entry.taskId = await submitRunwayTask({ apiKey, endpoint, body });const polled = await awaitRunwayTask({entry, apiKey, deadline, timeoutMs: RUNWAY_RENDER_TIMEOUT_MS,diff --git a/server/services/videoGen/runway.test.js b/server/services/videoGen/runway.test.jsindex 550b702af0..9d7587f462 100644--- a/server/services/videoGen/runway.test.js+++ b/server/services/videoGen/runway.test.js@@ -103,6 +103,24 @@ describe('videoGen/runway — generateVideo', () => {expect(entry.promptAdditions).toEqual([expect.objectContaining({ id: 'avoid', text: 'text' })]);});+ it('renders Gen-4.5 from text alone: posts /v1/text_to_video and records a text-mode take', async () => {+ const calls = [];+ vi.stubGlobal('fetch', vi.fn(async (url, opts = {}) => {+ calls.push({ url, method: opts.method || 'GET', body: opts.body ? JSON.parse(opts.body) : null });+ if (url === 'https://api.dev.runwayml.com/v1/text_to_video') return jsonResponse({ id: 'task-1' });+ if (url === TASKS) return jsonResponse({ id: 'task-1', status: 'SUCCEEDED', output: ['https://cdn.example.com/out.mp4'] });+ if (url === 'https://cdn.example.com/out.mp4') return { ok: true, status: 200, arrayBuffer: async () => Uint8Array.from(Buffer.from('mp4')).buffer };+ throw new Error(`unexpected fetch: ${url}`);+ }));+ const job = await runway.generateVideo({ apiKey: 'key_test', prompt: 'a fox at dawn', duration: 5, width: 720, height: 1280 });+ expect(await waitForTerminal(job.jobId)).toMatchObject({ type: 'completed' });+ expect(calls.find((c) => c.method === 'POST').body).toEqual({+ model: 'gen4.5', promptText: 'a fox at dawn', duration: 5, ratio: '720:1280',+ });+ const [entry] = await loadHistory();+ expect(entry).toMatchObject({ modelId: 'runway:gen4.5', mode: 'text', aspectRatio: '720:1280', estimatedCostUsd: 0.6 });+ });+it('fails the job with Runway\'s reason when the task FAILS', async () => {vi.stubGlobal('fetch', vi.fn(async (url, opts = {}) => {if (opts.method === 'POST') return jsonResponse({ id: 'task-1' });@@ -114,7 +132,7 @@ describe('videoGen/runway — generateVideo', () => {});it.each([- ['no start frame', { prompt: 'a fox' }, /start frame/],+ ['no start frame on an image-only model', { modelId: 'gen4_turbo', prompt: 'a fox' }, /start frame/],['an over-cap prompt', { prompt: 'x'.repeat(1001), sourceImagePath: 'unused.png' }, /1,001 characters/],['source audio', { prompt: 'a fox', sourceImagePath: 'unused.png', audioFilePath: 'song.wav' }, /cutaways only/],])('refuses %s before reading media or submitting', async (_label, params, message) => {diff --git a/server/services/videoGen/submitJob.js b/server/services/videoGen/submitJob.jsindex 527a1f5eef..8d96df3091 100644--- a/server/services/videoGen/submitJob.js+++ b/server/services/videoGen/submitJob.js@@ -214,7 +214,9 @@ const submitValidatedVideoGenJob = async (body, uploads, inputs) => {? { id: 'fal-video', supportedModes: ['text', 'image'] }: backend === VIDEO_GEN_MODE.REACTOR? { id: 'reactor-video', supportedModes: ['text', 'image'] }- : prepared.effectiveModel;+ : backend === VIDEO_GEN_MODE.RUNWAY+ ? { id: 'runway-video', supportedModes: ['text', 'image'] }+ : prepared.effectiveModel;const compiled = await compileFableLoomVisualRequest({tag: body.fableLoom,kind: 'video',