Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

22.1 KB
Raw Blame
import { useRunwayCatalog } from '../../hooks/useRunwayCatalog.js';
import RenderPoolPicker from './RenderPoolPicker.jsx';
import ListLoadError from '../ui/ListLoadError.jsx';
import { GROK_VIDEO_DURATIONS } from '../../lib/grokVideoClip.js';
import {
  FAL_DEFAULT_IMAGE_VIDEO_MODEL, FAL_MUSIC_VIDEO_DEFAULT_MODEL, FAL_IMAGE_VIDEO_MODELS, describeFalVideoRate, falImageVideoQuotes, falVideoResolutions, getFalVideoModel,
} from '../../lib/falVideoModels.js';
import {
  RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL, describeRunwayVideoRate, getRunwayVideoModel, runwayVideoLengths, runwayVideoResolutions,
} from '../../lib/runwayVideoModels.js';
import { projectFalSpend } from '../../lib/musicVideoStages.js';
import { formatUsd } from '../../utils/formatters.js';
import { SOURCE_AUDIO_LIPSYNC, SOURCE_AUDIO_MUSIC, performanceCapability } from '../../lib/musicVideoShotTiming.js';
import { localModelRendersCutaway } from '../../../../server/lib/musicVideoLocalModels.js';

const falSelectCls = 'w-full max-w-full bg-port-bg border border-port-border rounded px-1.5 py-1.5 text-sm disabled:opacity-50 sm:w-auto min-h-[44px] sm:min-h-0';
const falOptionLabel = (model) => {
  const rate = describeFalVideoRate(model.id);
  return rate ? `${model.label} · ${rate}` : model.label;
};
const runwayOptionLabel = (model, resolution) => {
  const rate = describeRunwayVideoRate(model.id, resolution);
  return rate ? `${model.label} · ${rate}` : model.label;
};

// The project's saved scene-video render pins (backend → generation mode →
// model → audio-reactive LoRA/strength, or Grok clip duration). Every control
// persists through useMusicVideoModelSettings' `change`, and locks while a save
// is in flight or any scene clip is generating, so the job payload and the
// board's displayed setting cannot disagree.
export default function VideoRenderSettings({ videoSettings, generating, project = null }) {
  const runwayCatalog = useRunwayCatalog(videoSettings?.settings?.backend === 'runway');
  const {
    settings, saving, models, modelsLoading, catalogStatus, retryCatalog, defaultModel, effectiveModelId,
    audioReactiveModels, audioReactiveLoras, detectedAudioReactiveLora,
    audioReactiveReady, audioReactiveSelected, change,
  } = videoSettings;
  // fal.ai (#8968) is image-to-video only here — every scene render already
  // starts from the director's chosen reference frame, so the model picker
  // offers the curated start-frame models (lib/falVideoModels.js), each with
  // its list rate; blank is the default (Hailuo-02 image-to-video). The
  // resolution alphabet is per model, so a model change clears the pin.
  // Performance (lip-sync) takes always render on the lip-sync route, with
  // their own resolution pin (default 1080P).
  // The audio-reactive lane stays local-only (root AGENTS.md's environmental-
  // motion contract needs an independently verified provider capability).
  const locked = saving || generating;
  const falModel = getFalVideoModel(settings.falModelId || FAL_DEFAULT_IMAGE_VIDEO_MODEL);
  const falResolutions = falVideoResolutions(falModel);
  const lipSync = SOURCE_AUDIO_LIPSYNC.fal;
  // Music-driven performance takes (scene Shot › Performance (music-driven)):
  // an audio-to-video model where the song slice drives the whole frame.
  const music = performanceCapability('fal', { scene: { performanceStyle: 'music' }, videoSettings: settings });
  const musicModels = Object.values(SOURCE_AUDIO_MUSIC.fal);
  // A dearer start+end-frame model is an explicit upgrade for end-frame shots
  // only; each option shows what a typical short (2 s) shot bills.
  const endFrameQuotes = falImageVideoQuotes({ seconds: 2, endFrame: true });
  const falSpend = project ? projectFalSpend(project) : null;
  // Runway is image-to-video only here, so it renders cutaways; performance
  // shots stay on the lip-sync lane. Length and size alphabets are per model,
  // so a model change clears both pins.
  const runwayModel = getRunwayVideoModel(settings.runwayModelId || RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL);
  const runwayResolutions = runwayVideoResolutions(runwayModel);
  const runwayLengths = runwayVideoLengths(runwayModel);
  return (
    <>
      {catalogStatus === 'error' && (
        <ListLoadError what="Video Gen status" onRetry={retryCatalog} className="w-full" />
      )}
      <RenderPoolPicker settings={settings} change={change} disabled={locked} />
      <label htmlFor="mv-video-backend" className="sr-only">Scene video renderer</label>
      <select
        id="mv-video-backend"
        value={settings.backend}
        onChange={(e) => change({ backend: e.target.value || null })}
        disabled={locked}
        title="Saved renderer for this project's scene videos"
        className="w-full max-w-full bg-port-bg border border-port-border rounded px-1.5 py-1.5 text-sm disabled:opacity-50 min-h-[44px] sm:min-h-0 sm:w-auto"
      >
        <option value="">Install default</option>
        <option value="local">Local video</option>
        {/* Grok's CLI lane takes no audio, so it renders cutaways only —
            performance (lip-sync) shots need fal.ai (#8977). */}
        <option value="grok">Grok video (cutaway only)</option>
        <option value="fal">fal.ai video</option>
        {/* Runway is image-to-video only, so like Grok it renders cutaways. */}
        <option value="runway">Runway video (cutaway only)</option>
      </select>
      {settings.backend === 'local' && (
        <>
          <label htmlFor="mv-generation-mode" className="sr-only">Scene generation mode</label>
          <select
            id="mv-generation-mode"
            value={settings.generationMode}
            onChange={(e) => {
              const generationMode = e.target.value;
              const compatibleModel = audioReactiveModels.find((model) => model.id === effectiveModelId)
                || audioReactiveModels[0];
              change({
                generationMode,
                ...(generationMode === 'audioReactive' && detectedAudioReactiveLora
                  ? { audioReactiveLora: detectedAudioReactiveLora.filename }
                  : {}),
                ...(generationMode === 'audioReactive' && compatibleModel
                  ? { modelId: compatibleModel.id }
                  : {}),
              });
            }}
            disabled={locked}
            title="Prompt motion uses the reference frame; audio reactive also conditions motion on this scene's song segment"
            className="w-full max-w-full bg-port-bg border border-port-border rounded px-1.5 py-1.5 text-sm disabled:opacity-50 min-h-[44px] sm:min-h-0 sm:w-auto"
          >
            <option value="image">Prompt motion</option>
            <option value="suppliedAudio">Supplied song audio (cutaway only)</option>
            <option value="audioReactive" disabled={!detectedAudioReactiveLora}>Audio reactive</option>
          </select>
          <label htmlFor="mv-video-model" className="sr-only">Local video model</label>
          <select
            id="mv-video-model"
            value={settings.modelId}
            onChange={(e) => change({ modelId: e.target.value })}
            disabled={locked || models.length === 0}
            title="Saved local image-to-video model for this project"
            className="w-full max-w-full bg-port-bg border border-port-border rounded px-1.5 py-1.5 text-sm disabled:opacity-50 min-h-[44px] sm:min-h-0 sm:w-auto sm:max-w-[240px]"
          >
            <option value="">
              {defaultModel
                ? `Local default · ${models.find((model) => model.id === defaultModel)?.name || defaultModel}`
                : 'Local default model'}
            </option>
            {/* Cutaways animate their still, so an audio-only model (Ref2VA) is a per-shot pick for performance shots. */}
            {(audioReactiveSelected ? audioReactiveModels : models.filter((model) => localModelRendersCutaway(model) || model.id === settings.modelId)).map((model) => (
              <option key={model.id} value={model.id}>{model.name || model.id}</option>
            ))}
          </select>
          {audioReactiveSelected && (
            <>
              <label htmlFor="mv-audio-reactive-lora" className="sr-only">Audio reactive LoRA</label>
              <select
                id="mv-audio-reactive-lora"
                value={settings.audioReactiveLora || detectedAudioReactiveLora?.filename || ''}
                onChange={(e) => change({ audioReactiveLora: e.target.value })}
                disabled={locked || audioReactiveLoras.length === 0}
                title="Saved audio-reactive LoRA version for this project"
                className="w-full max-w-full bg-port-bg border border-port-border rounded px-1.5 py-1.5 text-sm disabled:opacity-50 min-h-[44px] sm:min-h-0 sm:w-auto sm:max-w-[220px]"
              >
                {audioReactiveLoras.length === 0 && (
                  <option value="">{catalogStatus === 'ready' ? 'No audio-reactive LoRA installed' : 'Audio-reactive LoRAs not loaded'}</option>
                )}
                {audioReactiveLoras.map((lora) => (
                  <option key={lora.filename} value={lora.filename}>
                    {lora.name || lora.filename}
                  </option>
                ))}
              </select>
              <label htmlFor="mv-audio-reactive-scale" className="sr-only">Audio reactive LoRA strength</label>
              <select
                id="mv-audio-reactive-scale"
                value={settings.audioReactiveScale}
                onChange={(e) => change({ audioReactiveScale: Number(e.target.value) })}
                disabled={locked}
                title="How strongly the song drives visible motion"
                className="w-full max-w-full bg-port-bg border border-port-border rounded px-1.5 py-1.5 text-sm disabled:opacity-50 min-h-[44px] sm:min-h-0 sm:w-auto"
              >
                <option value={1}>Reactive 1.0×</option>
                <option value={1.2}>Reactive 1.2×</option>
                <option value={1.5}>Reactive 1.5×</option>
              </select>
              <span
                className={`text-[10px] px-1.5 py-0.5 rounded ${
                  audioReactiveReady
                    ? 'bg-port-success/20 text-port-success'
                    : (modelsLoading || catalogStatus !== 'ready' ? 'bg-port-warning/20 text-port-warning' : 'bg-port-error/20 text-port-error')
                }`}
                title={detectedAudioReactiveLora?.filename || 'Audio-reactive LoRA not installed'}
              >
                {audioReactiveReady
                  ? 'song-conditioned · no vocals'
                  : (catalogStatus === 'error' ? 'status unavailable' : modelsLoading || catalogStatus === 'loading' ? 'checking local runtime…' : 'audio-reactive unavailable')}
              </span>
            </>
          )}
        </>
      )}
      {settings.backend === 'grok' && (
        <>
          <label htmlFor="mv-grok-duration" className="sr-only">Grok scene clip duration</label>
          <select
            id="mv-grok-duration"
            value={settings.grokDuration}
            onChange={(e) => change({ grokDuration: Number(e.target.value) })}
            disabled={locked}
            title="Native duration for each Grok scene clip"
            className="w-full max-w-full bg-port-bg border border-port-border rounded px-1.5 py-1.5 text-sm disabled:opacity-50 min-h-[44px] sm:min-h-0 sm:w-auto"
          >
            {GROK_VIDEO_DURATIONS.map((duration) => (
              <option key={duration} value={duration}>{duration}s clips</option>
            ))}
          </select>
        </>
      )}
      {settings.backend === 'fal' && (
        <>
          <label htmlFor="mv-fal-model" className="sr-only">fal.ai cutaway model</label>
          <select
            id="mv-fal-model"
            value={settings.falModelId || ''}
            onChange={(e) => change({ falModelId: e.target.value || null, falResolution: null })}
            disabled={locked}
            title="fal.ai image-to-video model for this project's cutaway scenes (list price per generated second)"
            className={`${falSelectCls} sm:max-w-[280px]`}
          >
            <option value="">{`Default · ${falOptionLabel(getFalVideoModel(FAL_DEFAULT_IMAGE_VIDEO_MODEL))}`}</option>
            {FAL_IMAGE_VIDEO_MODELS.filter((model) => model.id !== FAL_DEFAULT_IMAGE_VIDEO_MODEL).map((model) => (
              <option key={model.id} value={model.id}>{falOptionLabel(model)}</option>
            ))}
            {/* A model pinned on a peer with a newer catalog: kept, but it
                cannot be priced here. */}
            {settings.falModelId && !falModel && (
              <option value={settings.falModelId}>{`${settings.falModelId} · cost unknown`}</option>
            )}
          </select>
          {falResolutions.length > 0 && (
            <>
              <label htmlFor="mv-fal-resolution" className="sr-only">fal.ai cutaway resolution</label>
              <select
                id="mv-fal-resolution"
                value={falResolutions.includes(settings.falResolution) ? settings.falResolution : ''}
                onChange={(e) => change({ falResolution: e.target.value || null })}
                disabled={locked}
                title="Output resolution for fal.ai cutaway renders — higher costs more per second"
                className={falSelectCls}
              >
                <option value="">{`${falModel.resolution.default} (model default)`}</option>
                {falResolutions.map((res) => (
                  <option key={res} value={res}>{`${res} · ${describeFalVideoRate(falModel.id, res)}`}</option>
                ))}
              </select>
            </>
          )}
          <label htmlFor="mv-fal-end-frame-model" className="sr-only">fal.ai model for end-frame shots</label>
          <select
            id="mv-fal-end-frame-model"
            value={settings.falEndFrameModelId || ''}
            onChange={(e) => change({ falEndFrameModelId: e.target.value || null })}
            disabled={locked}
            title="Model for shots with a style-break end frame only. Every other shot keeps the cutaway model above."
            className={`${falSelectCls} sm:max-w-[280px]`}
          >
            <option value="">End-frame shots: same model</option>
            {endFrameQuotes.map((quote) => (
              <option key={quote.modelId} value={quote.modelId}>{`End-frame shots: ${quote.label} · ${formatUsd(quote.costUsd)} per short shot`}</option>
            ))}
          </select>
          <label htmlFor="mv-fal-duration" className="sr-only">fal.ai scene clip duration</label>
          <input
            id="mv-fal-duration"
            type="number"
            min={1}
            max={60}
            value={settings.falDuration ?? ''}
            onChange={(e) => change({ falDuration: e.target.value === '' ? null : Number(e.target.value) })}
            disabled={locked}
            placeholder="shot length"
            title="Clip length in seconds for this project's fal.ai cutaway renders. Blank renders the shortest length the model offers that covers each shot (the model default for an untimed scene)."
            className="w-full max-w-full bg-port-bg border border-port-border rounded px-1.5 py-1.5 text-sm disabled:opacity-50 min-h-[44px] sm:min-h-0 sm:w-24"
          />
          <label htmlFor="mv-fal-lipsync-resolution" className="sr-only">fal.ai lip-sync resolution</label>
          <select
            id="mv-fal-lipsync-resolution"
            value={settings.falLipSyncResolution || ''}
            onChange={(e) => change({ falLipSyncResolution: e.target.value || null })}
            disabled={locked}
            title={`Output resolution for performance (lip-sync) takes on ${lipSync.label}`}
            className={falSelectCls}
          >
            <option value="">{`Lip-sync ${lipSync.defaultResolution} (default)`}</option>
            {lipSync.resolutions.map((res) => (
              <option key={res} value={res}>{`Lip-sync ${res} · ${describeFalVideoRate(lipSync.modelId, res)}`}</option>
            ))}
          </select>
          <label className="flex items-center gap-1.5 text-xs text-port-text-muted min-h-[44px] sm:min-h-0"
            title="Let the provider transcribe the sung audio to guide mouth shapes. Turn off if it mishears lyrics and shapes the wrong words (e.g. 'swarm' mouthed as 'sworn'); the take then follows the audio alone.">
            <input
              type="checkbox"
              checked={settings.falLipSyncTranscription !== false}
              onChange={(e) => change({ falLipSyncTranscription: e.target.checked ? null : false })}
              disabled={locked}
            />
            Transcript-guided lip-sync
          </label>
          {/* The model picker appears once there is a choice; the resolution picker always does. */}
          {musicModels.length > 1 && (
            <>
              <label htmlFor="mv-fal-music-model" className="sr-only">fal.ai music-driven model</label>
              <select
                id="mv-fal-music-model"
                value={settings.falMusicModelId || ''}
                onChange={(e) => change({ falMusicModelId: e.target.value || null, falMusicResolution: null })}
                disabled={locked}
                title="Audio-to-video model for music-driven performance shots: the scene's song slice drives the whole frame's motion, not just a mouth"
                className={`${falSelectCls} sm:max-w-[280px]`}
              >
                {musicModels.map((lane) => (
                  <option key={lane.modelId} value={lane.modelId === FAL_MUSIC_VIDEO_DEFAULT_MODEL ? '' : lane.modelId}>
                    {`Music-driven · ${getFalVideoModel(lane.modelId).label} · ${describeFalVideoRate(lane.modelId, lane.defaultResolution)}${lane.modelId === FAL_MUSIC_VIDEO_DEFAULT_MODEL ? ' (default)' : ''}`}
                  </option>
                ))}
              </select>
            </>
          )}
          <label htmlFor="mv-fal-music-resolution" className="sr-only">fal.ai music-driven resolution</label>
          <select
            id="mv-fal-music-resolution"
            value={music.resolutions.includes(settings.falMusicResolution) ? settings.falMusicResolution : ''}
            onChange={(e) => change({ falMusicResolution: e.target.value || null })}
            disabled={locked}
            title={`Output resolution for music-driven takes on ${music.label}`}
            className={falSelectCls}
          >
            <option value="">{`Music-driven ${music.defaultResolution} (default)`}</option>
            {music.resolutions.map((res) => (
              <option key={res} value={res}>{`Music-driven ${res} · ${describeFalVideoRate(music.modelId, res)}`}</option>
            ))}
          </select>
        </>
      )}
      {settings.backend === 'runway' && (
        <>
          <label htmlFor="mv-runway-model" className="sr-only">Runway cutaway model</label>
          <select
            id="mv-runway-model"
            value={settings.runwayModelId || ''}
            onChange={(e) => change({ runwayModelId: e.target.value || null, runwayDuration: null, runwayResolution: null })}
            disabled={locked}
            title="Runway image-to-video model for this project's cutaway scenes (list price per generated second)"
            className={`${falSelectCls} sm:max-w-[280px]`}
          >
            <option value="">{`Default · ${runwayOptionLabel(getRunwayVideoModel(RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL))}${runwayCatalog.videoModels.some((m) => m.id === RUNWAY_DEFAULT_IMAGE_VIDEO_MODEL) ? '' : ' (unavailable)'}`}</option>
            {runwayCatalog.videoModels.map((model) => (
              <option key={model.id} value={model.id}>{runwayOptionLabel(model)}</option>
            ))}
            {settings.runwayModelId && !runwayCatalog.videoModels.some((m) => m.id === settings.runwayModelId) && (
              <option value={settings.runwayModelId}>{`${settings.runwayModelId} (unavailable)`}</option>
            )}
          </select>
          {runwayResolutions.length > 1 && (
            <>
              <label htmlFor="mv-runway-resolution" className="sr-only">Runway cutaway resolution</label>
              <select
                id="mv-runway-resolution"
                value={runwayResolutions.includes(settings.runwayResolution) ? settings.runwayResolution : ''}
                onChange={(e) => change({ runwayResolution: e.target.value || null })}
                disabled={locked}
                title="Output resolution for Runway cutaway renders — higher can cost more per second"
                className={falSelectCls}
              >
                <option value="">{`${runwayModel.size.default} (model default)`}</option>
                {runwayResolutions.map((res) => (
                  <option key={res} value={res}>{`${res} · ${describeRunwayVideoRate(runwayModel.id, res)}`}</option>
                ))}
              </select>
            </>
          )}
          {runwayLengths.length > 0 && (
            <>
              <label htmlFor="mv-runway-duration" className="sr-only">Runway scene clip length</label>
              <select
                id="mv-runway-duration"
                value={runwayLengths.includes(settings.runwayDuration) ? settings.runwayDuration : ''}
                onChange={(e) => change({ runwayDuration: e.target.value === '' ? null : Number(e.target.value) })}
                disabled={locked}
                title="Clip length for this project's Runway cutaway renders. Shot length renders the shortest length the model offers that covers each shot."
                className={falSelectCls}
              >
                <option value="">Shot length</option>
                {runwayLengths.map((sec) => (
                  <option key={sec} value={sec}>{`${sec}s clips`}</option>
                ))}
              </select>
            </>
          )}
        </>
      )}
      {falSpend?.totalUsd > 0 && (
        <p className="w-full min-w-0 break-words text-xs text-port-text-muted" title="Estimated from the price each paid render was quoted at; the provider's own invoice is the final word">
          {`Paid video spend (est.) ${formatUsd(falSpend.totalUsd)}: `}
          {falSpend.models.map((row) => `${row.label} ${formatUsd(row.usd)} (${row.count})`).join(' · ')}
        </p>
      )}
    </>
  );
}