Skip to content

AI media harness for planning, generating, and editing locally

TypeScript
33.7 KB
Raw Blame
import { clipPlaybackRate } from "./clipTiming";
import { invoke } from "@tauri-apps/api/core";
import { cancelAgentCaptures, listenForAgentCaptures } from "./agentCapture";
import type { AgentGenerateCommand } from "./agentGeneration";
import { applyImageDraftCommand, type ImageDraftCommand } from "./imageDraftCommands";
import { saveImageDraft } from "./imageHistory";
import { executeGeneratorCommands, generatorContext, type GeneratorCommand } from "./agentGenerators";
import { createImageScene, imageScenePromptText } from "./imageScene";
import { applyImageCommand, type ImageCommand } from "./imageCommands";
import { isTauri } from "./persistence";
import type { TemplateLora } from "./loras";
import { refreshDownloadedLoras, refreshDownloadedWeights } from "./weightDownloads";
import { sceneArchiveSegments, sceneMediaDurationMs } from "./continuationMedia";
import { bridgeGapFrames } from "./longShot";
import {
  actionReferenceIds, createDraftGenerationJob, GENERATION_FRAME_RATE, parseProjectConfig,
  sceneBriefText, type AgentMessage, type GenerationJob, type ProjectAsset, type ProjectConfig,
  type ProjectRecord, type SceneShot, type TimelineClip,
} from "./project";
import {
  agentEndpointProviderSettings, endpointProviderSetting, engineProviderSetting, defaultGeneratorTemplate,
  generationStepsWithLoras, loadDefaultGenerationSteps, loadEngineSettings, withAgentEndpointSettings,
  type AttentionMode, type EndpointProviderId, type EndpointProviderSettings, type EnginePathField, type GeneratorTemplate,
} from "./settings";

export type ProviderId = "claude" | "codex" | EndpointProviderId;
export interface ProviderStatus {
  id: ProviderId;
  label: string;
  /* "checking" is FRONTEND-ONLY and never comes back from Rust: it is what the
     dock shows while the probe is in flight. It used to borrow "unavailable"
     for that, so a slow or failed probe left every provider reading
     "Unavailable" — including a Claude Code that was installed and working. */
  state: "checking" | "ready" | "disabled" | "notInstalled" | "authRequired" | "unavailable";
  executable: string | null;
  version: string | null;
  detail: string;
}
export interface ModelStatus { id: string; configured: boolean; available: boolean; path: string | null }
export interface SlopfabStatus {
  state: "ready" | "modelsMissing" | "runtimeMissing" | "incompatible" | "demo";
  dllPath: string;
  version: string | null;
  platform: string | null;
  cudaAvailable?: boolean;
  cudaDeviceNames?: string[];
  detail: string;
  models: ModelStatus[];
}
export interface RuntimeStatus { providers: ProviderStatus[]; slopfab: SlopfabStatus }

export type ProjectCommand =
  | { op: "video.music.set"; video: string; song: string }
  | { op: "music.add"; id: string; title: string; style: string; settings?: Partial<import("./music").MusicSettings>; folderId?: string; steps?: number; seed?: number }
  | { op: "music.set"; id: string; title?: string; style?: string; settings?: Partial<import("./music").MusicSettings>; folderId?: string | null; steps?: number; seed?: number }
  | { op: "motion.add"; id: string; name: string; motion: import("./motion").Motion }
  | { op: "motion.set"; id: string; name?: string; motion?: import("./motion").Motion }
  | { op: "motion.sync"; id: string; song: string; bpm: number; beatOffsetSeconds?: number; beatsPerBar?: number; clip?: string; audioClip?: string }
  | ImageCommand
  | ImageDraftCommand
  | { op: "project.set"; name?: string; prompt?: string; targetSeconds?: number; aspectRatio?: string; resolution?: string; frameRate?: number; backgroundColor?: string }
  | { op: "ref.add"; id: string; name: string; text: string; use: string[] }
  | { op: "ref.set"; id: string; name?: string; text?: string; use?: string[] }
  | { op: "ref.remove"; id: string }
  | { op: "scene.add"; id: string; title: string; seconds: number; steps?: number; seed?: number; sound?: string; music?: string; startFrame?: string; endFrame?: string; usePreviousSceneLastFrame?: boolean }
  | { op: "scene.set"; id: string; title?: string; seconds?: number; steps?: number; seed?: number; sound?: string | null; music?: string | null; startFrame?: string | null; endFrame?: string | null; usePreviousSceneLastFrame?: boolean }
  | { op: "scene.remove"; id: string }
  | { op: "scene.move"; id: string; before?: string | null }
  | { op: "shot.add"; scene: string; id: string; at: number; action: string; name?: string; speech?: string; language?: string; settings?: Record<string, string[]> }
  | { op: "shot.set"; scene: string; id: string; at?: number; action?: string; name?: string | null; speech?: string | null; language?: string | null; settings?: Record<string, string[]> | null }
  | { op: "shot.remove"; scene: string; id: string }
  | { op: "clip.add"; id: string; scene?: string; asset?: string; track?: string; at?: number; seconds?: number; sourceAt?: number; label?: string }
  | { op: "clip.set"; id: string; track?: string; at?: number; seconds?: number; sourceAt?: number; label?: string }
  | { op: "clip.remove"; id: string };

export type AgentTurnResult =
  | { kind: "generation"; summary: string; command: AgentGenerateCommand; prepare?: ProjectCommand[] }
  | { kind: "answer"; content: string }
  | { kind: "question"; content: string }
  | { kind: "generatorCommands"; summary: string; commands: GeneratorCommand[] }
  | { kind: "commands"; summary: string; commands: ProjectCommand[] };

export type AgentCommand = ProjectCommand | GeneratorCommand;

export type AgentTurnEvent =
  | { type: "started"; provider: ProviderId }
  | { type: "message" | "diagnostic"; text: string }
  | { type: "validation"; round: number; maxRounds: number; text: string }
  | { type: "completed" };

export interface AgentTurnEventPayload {
  requestId: string;
  event: AgentTurnEvent;
}

export const AGENT_TURN_EVENT = "agent-turn-event";

export interface AgentTurnResponse {
  result: AgentTurnResult;
  events: AgentTurnEvent[];
  messages: AgentMessage[];
}

export interface SlopfabGenerationRequest {
  loraSnapshot?: ReturnType<typeof import("./loras").loraSnapshot>;
  contextSceneId?: string;
  keystoneRelativePath?: string;
  singleShot?: { endingPrompt: string };
  music?: import("./music").MusicSettings;
  jobId: string;
  videoTransition?: "extend" | "bridge";
  latentBridge?: import("./project").SceneGenerationInput["latentBridge"];
  prompt: string;
  frames: number;
  stillImage?: boolean;
  imageEdit?: { sourceRelativePath: string; regionSize?: { width: number; height: number }; edits: import("./imageEditing").ImageEditStep[] };
  steps: number;
  audioSteps?: number;
  guidanceScale?: number;
  negativePrompt?: string;
  generationEffects?: import("./generationEffects").GenerationEffects;
  videoSigmaShift?: number;
  audioSigmaShift?: number;
  latentUpscale?: boolean;
  latentUpscaleSource?: { path: string; scale: number };
  seed: number;
  canvasWidth: number;
  canvasHeight: number;
  referencePaths: string[];
  refmods?: Array<{ path: string; strength: number; copies: number }>;
  referenceVideos?: Array<{ name: string; relativePath?: string | null; sourcePath?: string | null; startSeconds: number; durationSeconds?: number; includeAudio: boolean }>;
  referenceAudios?: Array<{ name: string; relativePath?: string | null; sourcePath?: string | null; startSeconds: number; durationSeconds: number }>;
  /** Temporary native handles, populated by the work queue and never saved. */
  referenceVideoIds?: string[];
  referenceAudioIds?: string[];
  /** Resolved by the app queue before native submission. */
  previousSceneId?: string;
  continuationRelativePath?: string;
  continuationOverlapFrames?: number;
  continuationFrom?: "start" | "end";
  continuationLockOverlap?: boolean;
  continuationSourceNoise?: number;
  continuationSourceFrames?: number;
}

export interface ResolvedPlan {
  canvasWidth: number;
  canvasHeight: number;
  alignedFrames: number;
  durationSeconds: number;
  modelEvaluations: number;
  sequenceRowsWithoutText: number;
  latentFrames: number;
  latentWidth: number;
  latentHeight: number;
  description: string;
  boundary: string;
}

const DEMO_STATUS: RuntimeStatus = {
  providers: [
    { id: "claude", label: "Claude Code", state: "ready", executable: null, version: "Browser demo", detail: "Deterministic local demo provider." },
    { id: "codex", label: "Codex", state: "ready", executable: null, version: "Browser demo", detail: "Deterministic local demo provider." },
  ],
  slopfab: { state: "demo", dllPath: "Browser demo", version: null, platform: null, detail: "Deterministic plans only. No model runs or media files are created in the browser.", models: [] },
};

/* Every slopfab command is handed the project config with THIS machine's engine
   paths merged in (see lib/settings.ts). The merged copy is passed straight to
   the command and dropped; it is never the config that gets saved. */

/** What is installed on this computer. Takes no project: the agent CLIs and
 *  the video engine belong to the machine, so the app probes this ONCE at
 *  startup and every project shares the answer. */
export async function getRuntimeStatus(): Promise<RuntimeStatus> {
  if (!isTauri()) return DEMO_STATUS;
  await refreshDownloadedWeights();
  await refreshDownloadedLoras();
  return invoke<RuntimeStatus>("runtime_status", {
    settings: { slopfab: engineProviderSetting(loadEngineSettings()), ...agentEndpointProviderSettings() },
  });
}

/** The placeholder every caller shows until the startup probe lands. */
export const CHECKING_PROVIDERS: ProviderStatus[] = [
  { id: "claude", label: "Claude Code", state: "checking", executable: null, version: null, detail: "Looking for Claude Code on this computer…" },
  { id: "codex", label: "Codex", state: "checking", executable: null, version: null, detail: "Looking for Codex on this computer…" },
];

/** Probes the engine paths on their own, with no project in hand — what the
 *  settings screen shows. */
export async function getEngineStatus(settings = loadEngineSettings(), attention?: AttentionMode, loras?: TemplateLora[], mode?: GeneratorTemplate["mode"], additionalSafetensors?: GeneratorTemplate["additionalSafetensors"], motionCache?: boolean, sources?: GeneratorTemplate["sources"], modelType = defaultGeneratorTemplate().modelType): Promise<SlopfabStatus> {
  if (!isTauri()) return DEMO_STATUS.slopfab;
  let slopfab;
  let adapterError: string | undefined;
  try {
    slopfab = engineProviderSetting(settings, undefined, attention, loras, mode, additionalSafetensors, motionCache, sources, { modelType });
  } catch (reason) {
    // A missing adapter must not prevent checking the individual model files.
    adapterError = reason instanceof Error ? reason.message : String(reason);
    slopfab = engineProviderSetting(settings, undefined, attention, [], mode, additionalSafetensors, motionCache, sources, { modelType });
  }
  const status = await invoke<SlopfabStatus>("slopfab_status", { settings: { slopfab } });
  return adapterError ? { ...status, state: status.state === "ready" ? "modelsMissing" : status.state,
    detail: `${status.detail} ${adapterError}` } : status;
}

/** Opens the OS picker for one engine path. Returns null when the user cancels. */
export async function chooseEnginePath(field: EnginePathField): Promise<string | null> {
  if (!isTauri()) return null;
  return invoke<string | null>("choose_engine_path", {
    title: `Select ${field.label}`,
    directory: field.directory,
    extensions: field.extensions,
  });
}

export async function getAgentModels(provider: EndpointProviderId, settings: EndpointProviderSettings): Promise<string[]> {
  if (!isTauri()) return [];
  return invoke<string[]>("list_agent_models", { provider, setting: endpointProviderSetting(settings) });
}

export type AgentScope = "project" | "settings";

export async function runAgentTurn(record: ProjectRecord, provider: ProviderId, prompt: string, requestId: string, conversation: AgentMessage[] = [], signal?: AbortSignal, scope: AgentScope = "project"): Promise<AgentTurnResponse> {
  signal?.throwIfAborted();
  const createdAt = new Date().toISOString();
  const userMessageId = crypto.randomUUID();
  const assistantMessageId = crypto.randomUUID();
  if (isTauri()) {
    const stopCaptures = scope === "settings" ? () => {} : await listenForAgentCaptures(record, requestId);
    try {
      signal?.throwIfAborted();
      const response = await invoke<Omit<AgentTurnResponse, "messages">>("run_agent_turn", { request: {
        requestId, folderPath: record.folderPath, provider, prompt, config: withAgentEndpointSettings(record.config),
        conversation, generators: generatorContext(), scope,
      } });
      const assistantContent = "summary" in response.result ? response.result.summary : response.result.content;
      return {
        ...response,
        messages: [
          ...conversation,
          { id: userMessageId, role: "user", content: prompt.trim(), createdAt },
          { id: assistantMessageId, role: "assistant", content: assistantContent, createdAt },
        ],
      };
    } finally { stopCaptures(); }
  }
  const result = scope === "settings"
    ? { kind: "answer" as const, content: "I can help configure generator templates, weight paths and LoRAs in the desktop app. Open Settings > Generator to review them in this browser preview." }
    : demoTurn(record.config, prompt);
  const assistantContent = "summary" in result ? result.summary : result.content;
  return {
    result,
    events: [{ type: "started", provider }, { type: "completed" }],
    messages: [
      ...conversation,
      { id: userMessageId, role: "user", content: prompt.trim(), createdAt },
      { id: assistantMessageId, role: "assistant", content: assistantContent, createdAt },
    ],
  };
}

/** The provider plans against its request snapshot, then this applies the
 * accepted commands to the editor's latest state. Rust is authoritative in the
 * desktop app; this mirror keeps the deterministic browser preview useful. */
export async function executeAgentCommands(config: ProjectConfig, commands: AgentCommand[]): Promise<ProjectConfig> {
  const generatorCommands = commands.filter((command): command is GeneratorCommand => command.op.startsWith("generator."));
  if (generatorCommands.length) {
    if (generatorCommands.length !== commands.length) throw new Error("Generator and project edits must be separate command batches.");
    if (!isTauri()) throw new Error("Generator commands require the desktop application.");
    await executeGeneratorCommands(generatorCommands);
    return config;
  }
  const projectCommands = commands as ProjectCommand[];
  if (isTauri()) {
    const next = await invoke<ProjectConfig>("execute_agent_commands", { config, commands: projectCommands, executedAt: new Date().toISOString() });
    return parseProjectConfig(next);
  }
  return executeDemoCommands(config, projectCommands);
}

export async function cancelAgentTurn(requestId: string): Promise<boolean> {
  cancelAgentCaptures(requestId);
  return isTauri() ? invoke<boolean>("cancel_agent_turn", { requestId }) : true;
}

export async function resolveSlopfabPlan(request: SlopfabGenerationRequest, config: ProjectConfig, folderPath?: string): Promise<ResolvedPlan> {
  request = { ...request, steps: generationStepsWithLoras(request.steps, config) };
  if (isTauri()) return invoke<ResolvedPlan>("resolve_slopfab_plan", { request, config, folderPath });
  const source = config.generationJobs.find((job) => job.latentRelativePath === request.continuationRelativePath);
  const tail = source && sceneArchiveSegments(config, source)?.at(-1);
  const prefix = request.continuationFrom === "start" ? request.continuationOverlapFrames ?? 22
    : tail ? tail.startFrame + tail.frameCount : request.continuationSourceFrames ?? 22;
  const alignedFrames = request.latentBridge ? (request.latentBridge.leftFrames ?? 0) + bridgeGapFrames(request.frames) + (request.latentBridge.rightFrames ?? 0)
    : request.continuationRelativePath ? prefix + Math.ceil(request.frames / 17) * 17
    : Math.ceil(Math.max(5, request.frames - 5) / 17) * 17 + 5;
  return {
    canvasWidth: request.canvasWidth,
    canvasHeight: request.canvasHeight,
    alignedFrames,
    durationSeconds: alignedFrames / GENERATION_FRAME_RATE,
    modelEvaluations: request.steps - 1,
    sequenceRowsWithoutText: 0,
    latentFrames: Math.ceil(alignedFrames / 17), latentWidth: 0, latentHeight: 0,
    description: `Browser demo plan: ${alignedFrames} aligned frames at ${GENERATION_FRAME_RATE} fps.`,
    boundary: "Deterministic plan only. No weights were opened and no media file was created.",
  };
}

export async function enqueueSlopfabGeneration(request: SlopfabGenerationRequest, config: ProjectConfig, folderPath: string): Promise<void> {
  request = { ...request, steps: generationStepsWithLoras(request.steps, config) };
  if (isTauri()) await invoke("enqueue_slopfab_generation", { request, config, folderPath });
}

export async function cancelSlopfabGeneration(jobId: string): Promise<boolean> {
  return isTauri() ? invoke<boolean>("cancel_slopfab_generation", { jobId }) : true;
}

export async function saveReferenceIcon(folderPath: string, jobId: string): Promise<string> {
  return invoke<string>("save_reference_icon", { folderPath, jobId });
}

function demoTurn(_config: ProjectConfig, prompt: string): AgentTurnResult {
  const clean = prompt.trim();
  if (/\b(which|what)\b.*\?/i.test(clean)) {
    return { kind: "question", content: "Should I apply that direction to the selected shot or to the full sequence?" };
  }
  if (/\b(add|create|queue|generate)\b.*\b(shot|clip)\b/i.test(clean)) {
    const id = `job-demo-${Date.now()}`;
    return {
      kind: "commands",
      summary: "Added a schema-valid draft scene. No media was generated.",
      commands: [
        { op: "scene.add", id, title: clean.split(/\s+/).slice(0, 5).join(" "), seconds: 6 },
        { op: "shot.add", scene: id, id: `${id}-shot-1`, at: 0, action: clean },
        { op: "clip.add", id: `clip-${id}`, scene: id },
      ],
    };
  }
  return { kind: "answer", content: `I reviewed the project context. A safe next step is to refine “${clean.slice(0, 72)}” into a shot brief before generation.` };
}

function executeDemoCommands(config: ProjectConfig, commands: ProjectCommand[]): ProjectConfig {
  const next = structuredClone(config);
  const now = new Date().toISOString();
  const scene = (id: string): GenerationJob => {
    const found = next.generationJobs.find((job) => job.id === id);
    if (!found) throw new Error(`Scene '${id}' was not found.`);
    return found;
  };
  const milliseconds = (seconds: number, field: string, allowZero: boolean) => {
    if (!Number.isFinite(seconds) || seconds < 0 || (!allowZero && seconds <= 0)) throw new Error(`${field} must be ${allowZero ? "non-negative" : "positive"} finite seconds.`);
    const value = Math.round(seconds * 1000);
    if (!Number.isSafeInteger(value) || (!allowZero && value < 1)) throw new Error(`${field} is outside the supported range.`);
    return value;
  };
  const clipLocation = (id: string) => next.timeline.tracks.flatMap((track, trackIndex) =>
    track.clips.map((clip, clipIndex) => ({ track, trackIndex, clip, clipIndex })))
    .find((item) => item.clip.id === id);
  const targetTrack = (id?: string) => {
    const track = id
      ? next.timeline.tracks.find((candidate) => candidate.id === id)
      : next.timeline.tracks.find((candidate) => candidate.kind === "video" && !candidate.locked);
    if (!track) throw new Error(id ? `Track '${id}' was not found.` : "No unlocked video track is available.");
    if (track.locked) throw new Error(`Track '${track.id}' is locked.`);
    return track;
  };
  const validateSource = (clip: TimelineClip) => {
    const asset = next.assets.find((candidate) => candidate.id === clip.assetId);
    if (!asset) throw new Error(`Asset '${clip.assetId}' was not found.`);
    const durationMs = sceneMediaDurationMs(asset);
    if (asset.kind !== "image" && durationMs && clip.sourceStartMs + clip.durationMs * clipPlaybackRate(clip) > durationMs + 1) {
      throw new Error(`Clip source range extends past asset '${asset.id}'.`);
    }
  };
  for (const command of commands) {
    switch (command.op) {
      case "video.music.set": case "music.add": case "music.set": case "motion.add": case "motion.set": case "motion.sync":
        throw new Error("Music and Motion agent commands require the desktop app.");
      case "image.draft.add": case "image.draft.select": {
        Object.assign(next, applyImageDraftCommand(next, command, now));
        break;
      }
      case "image.set": case "image.configure": case "image.node.add": case "image.node.set": case "image.node.move": case "image.node.duplicate": case "image.node.remove": {
        const scene = applyImageCommand(next.imageScene ?? createImageScene(next.brief.prompt), command);
        for (const id of scene.referenceIds) if (!next.references.some((reference) => reference.id === id && ["image", "text"].includes(reference.kind))) throw new Error(`Image reference '${id}' must name an existing image or text reference.`);
        for (const id of actionReferenceIds(imageScenePromptText(scene))) if (!next.references.some((reference) => reference.id === id && ["image", "text"].includes(reference.kind))) throw new Error(`The image cites '@[ref:${id}]', which must name an existing image or text reference.`);
        next.imageScene = scene;
        Object.assign(next, saveImageDraft(next));
        break;
      }
      case "project.set":
        if (command.name !== undefined) next.name = command.name;
        if (command.prompt !== undefined) next.brief.prompt = command.prompt;
        if (command.targetSeconds !== undefined) next.brief.targetDurationSeconds = command.targetSeconds;
        if (command.aspectRatio !== undefined) {
          next.settings.aspectRatio = command.aspectRatio as ProjectConfig["settings"]["aspectRatio"];
          next.brief.aspectRatio = command.aspectRatio as ProjectConfig["brief"]["aspectRatio"];
        }
        if (command.resolution !== undefined) {
          next.settings.resolution = command.resolution as ProjectConfig["settings"]["resolution"];
          next.brief.resolution = command.resolution as ProjectConfig["brief"]["resolution"];
        }
        if (command.frameRate !== undefined) next.settings.frameRate = command.frameRate as ProjectConfig["settings"]["frameRate"];
        if (command.backgroundColor !== undefined) next.settings.backgroundColor = command.backgroundColor;
        break;
      case "ref.add":
        if (next.references.some((reference) => reference.id === command.id)) throw new Error(`Reference '${command.id}' already exists.`);
        next.references.push({ id: command.id, kind: "text", name: command.name, description: command.text, intendedUse: command.use as ProjectConfig["references"][number]["intendedUse"], createdAt: now });
        break;
      case "ref.set": {
        const reference = next.references.find((candidate) => candidate.id === command.id);
        if (!reference) throw new Error(`Reference '${command.id}' was not found.`);
        if (command.name !== undefined) reference.name = command.name;
        if (command.text !== undefined) { reference.description = command.text; reference.content = null; }
        if (command.use !== undefined) reference.intendedUse = command.use as typeof reference.intendedUse;
        break;
      }
      case "ref.remove": {
        const token = `@[ref:${command.id}]`;
        if (next.imageScene && (next.imageScene.referenceIds.includes(command.id) || imageScenePromptText(next.imageScene).includes(token))) throw new Error(`Reference '${command.id}' is still used by the image; remove its citations before removing it.`);
        if (next.generationJobs.some((job) => job.startFrameReferenceId === command.id || job.endFrameReferenceId === command.id || job.shots?.some((shot) => shot.action.includes(token)))) {
          throw new Error(`Reference '${command.id}' is still used by a scene.`);
        }
        const at = next.references.findIndex((reference) => reference.id === command.id);
        if (at < 0) throw new Error(`Reference '${command.id}' was not found.`);
        next.references.splice(at, 1);
        next.generationJobs.forEach((job) => { job.referenceIds = job.referenceIds.filter((id) => id !== command.id); });
        break;
      }
      case "scene.add": {
        if (next.generationJobs.some((job) => job.id === command.id)) throw new Error(`Scene '${command.id}' already exists.`);
        const job = createDraftGenerationJob("", { id: command.id, title: command.title, durationSeconds: command.seconds, steps: command.steps ?? loadDefaultGenerationSteps(), seed: command.seed, now });
        job.shots = [];
        job.prompt = "";
        job.creativeBrief = "";
        job.soundscape = command.sound;
        job.music = command.music;
        job.startFrameReferenceId = command.startFrame;
        job.endFrameReferenceId = command.endFrame;
        job.usePreviousSceneLastFrame = command.usePreviousSceneLastFrame;
        next.generationJobs.push(job);
        break;
      }
      case "scene.set": {
        const job = scene(command.id);
        if (command.title !== undefined) job.title = command.title;
        if (command.seconds !== undefined) job.durationSeconds = command.seconds;
        if (command.steps !== undefined) job.steps = command.steps;
        if (command.seed !== undefined) job.seed = command.seed;
        if ("sound" in command) job.soundscape = command.sound;
        if ("music" in command) job.music = command.music;
        if ("startFrame" in command) {
          job.startFrameReferenceId = command.startFrame;
          job.usePreviousSceneLastFrame = false;
          if (job.sceneType === "continue" || (job.sceneType === "prompt" && command.startFrame)) job.sceneType = "first-last-frame";
        }
        if ("endFrame" in command) {
          job.endFrameReferenceId = command.endFrame;
          if (job.sceneType === "prompt" && command.endFrame) job.sceneType = "first-last-frame";
        }
        if ("usePreviousSceneLastFrame" in command) {
          job.sceneType = "first-last-frame";
          job.usePreviousSceneLastFrame = command.usePreviousSceneLastFrame;
          if (command.usePreviousSceneLastFrame) job.startFrameReferenceId = undefined;
        }
        job.updatedAt = now;
        break;
      }
      case "scene.remove": {
        const at = next.generationJobs.findIndex((job) => job.id === command.id);
        if (at < 0) throw new Error(`Scene '${command.id}' was not found.`);
        next.generationJobs.splice(at, 1);
        break;
      }
      case "scene.move": {
        const at = next.generationJobs.findIndex((job) => job.id === command.id);
        if (at < 0) throw new Error(`Scene '${command.id}' was not found.`);
        const [moving] = next.generationJobs.splice(at, 1);
        const destination = command.before == null ? next.generationJobs.length : next.generationJobs.findIndex((job) => job.id === command.before);
        if (destination < 0) throw new Error(`Destination scene '${command.before}' was not found.`);
        next.generationJobs.splice(destination, 0, moving);
        break;
      }
      case "shot.add": {
        const job = scene(command.scene);
        const shots = job.shots ?? (job.shots = []);
        if (shots.some((shot) => shot.id === command.id)) throw new Error(`Shot '${command.id}' already exists.`);
        shots.push({ id: command.id, name: command.name, startSeconds: command.at, action: command.action, speech: command.speech, speechLanguage: command.language, settings: command.settings });
        job.updatedAt = now;
        break;
      }
      case "shot.set": {
        const job = scene(command.scene);
        const shot = job.shots?.find((candidate) => candidate.id === command.id);
        if (!shot) throw new Error(`Shot '${command.id}' was not found in scene '${command.scene}'.`);
        if (command.at !== undefined) shot.startSeconds = command.at;
        if (command.action !== undefined) shot.action = command.action;
        if ("name" in command) shot.name = command.name;
        if ("speech" in command) shot.speech = command.speech;
        if ("language" in command) shot.speechLanguage = command.language;
        if ("settings" in command) shot.settings = command.settings;
        job.updatedAt = now;
        break;
      }
      case "shot.remove": {
        const job = scene(command.scene);
        const at = job.shots?.findIndex((shot) => shot.id === command.id) ?? -1;
        if (at < 0) throw new Error(`Shot '${command.id}' was not found in scene '${command.scene}'.`);
        job.shots!.splice(at, 1);
        job.updatedAt = now;
        break;
      }
      case "clip.add": {
        if (clipLocation(command.id)) throw new Error(`Clip '${command.id}' already exists.`);
        if (Boolean(command.scene) === Boolean(command.asset)) throw new Error("Exactly one of scene or asset is required.");
        let asset: ProjectAsset;
        let defaultDurationMs: number;
        let status: TimelineClip["status"];
        if (command.scene) {
          const job = scene(command.scene);
          const plannedId = `asset-${job.id}`;
          const existing = next.assets.find((candidate) => candidate.id === plannedId)
            ?? next.assets.find((candidate) => Boolean(job.outputRelativePath) && candidate.kind === "generated" && candidate.relativePath === job.outputRelativePath);
          defaultDurationMs = Math.max(Math.round(1000 / next.settings.frameRate), Math.round((job.durationSeconds ?? 6) * 1000));
          if (existing?.sceneSegments?.length) defaultDurationMs = sceneMediaDurationMs(existing)!;
          asset = existing ?? {
            id: plannedId,
            kind: "generated",
            name: job.title,
            relativePath: job.outputRelativePath ?? null,
            sourcePath: null,
            mimeType: "video/mp4",
            durationMs: defaultDurationMs,
            width: null,
            height: null,
            createdAt: job.createdAt,
          };
          if (!existing) next.assets.push(asset);
          status = job.status === "completed" && Boolean(job.outputRelativePath) ? "generated" : "draft";
        } else {
          const existing = next.assets.find((candidate) => candidate.id === command.asset);
          if (!existing) throw new Error(`Asset '${command.asset}' was not found.`);
          asset = existing;
          if (asset.kind === "caption") throw new Error(`Asset '${asset.id}' cannot be placed on an audiovisual track.`);
          defaultDurationMs = Math.max(1, sceneMediaDurationMs(asset) ?? 5_000);
          status = "approved";
        }
        const track = targetTrack(command.track);
        if (command.scene && track.kind !== "video") throw new Error(`Scene clips require a video track, not '${track.id}'.`);
        const startMs = command.at === undefined
          ? track.clips.reduce((end, clip) => Math.max(end, clip.startMs + clip.durationMs), 0)
          : milliseconds(command.at, "Clip start", true);
        const clip: TimelineClip = {
          id: command.id,
          assetId: asset.id,
          trackId: track.id,
          startMs,
          durationMs: command.seconds === undefined ? defaultDurationMs : milliseconds(command.seconds, "Clip duration", false),
          sourceStartMs: command.sourceAt === undefined ? 0 : milliseconds(command.sourceAt, "Clip source start", true),
          label: command.label ?? asset.name,
          color: null,
          status,
        };
        if (!clip.label.trim()) throw new Error("Clip label cannot be empty.");
        validateSource(clip);
        track.clips.push(clip);
        break;
      }
      case "clip.set": {
        if (command.track === undefined && command.at === undefined && command.seconds === undefined && command.sourceAt === undefined && command.label === undefined) throw new Error("Clip command has no fields to change.");
        const location = clipLocation(command.id);
        if (!location) throw new Error(`Clip '${command.id}' was not found.`);
        if (location.track.locked) throw new Error(`Track '${location.track.id}' is locked.`);
        const track = command.track === undefined ? location.track : targetTrack(command.track);
        const clip = { ...location.clip };
        if (command.at !== undefined) clip.startMs = milliseconds(command.at, "Clip start", true);
        if (command.seconds !== undefined) clip.durationMs = milliseconds(command.seconds, "Clip duration", false);
        if (command.sourceAt !== undefined) clip.sourceStartMs = milliseconds(command.sourceAt, "Clip source start", true);
        if (command.label !== undefined) clip.label = command.label;
        if (!clip.label.trim()) throw new Error("Clip label cannot be empty.");
        validateSource(clip);
        clip.trackId = track.id;
        location.track.clips.splice(location.clipIndex, 1);
        track.clips.push(clip);
        break;
      }
      case "clip.remove": {
        const location = clipLocation(command.id);
        if (!location) throw new Error(`Clip '${command.id}' was not found.`);
        if (location.track.locked) throw new Error(`Track '${location.track.id}' is locked.`);
        location.track.clips.splice(location.clipIndex, 1);
        next.generationJobs.forEach((job) => { if (job.clipId === command.id) job.clipId = null; });
        break;
      }
    }
  }
  const knownReferences = new Set(next.references.map((reference) => reference.id));
  const changedScenes = new Set(commands.flatMap((command) =>
    command.op === "shot.add" || command.op === "shot.set" || command.op === "shot.remove" ? [command.scene] : []));
  next.generationJobs.forEach((job) => {
    if (!changedScenes.has(job.id) || !job.shots) return;
    job.shots.sort((left, right) => left.startSeconds - right.startSeconds);
    const text = sceneBriefText(job.shots as SceneShot[]);
    job.prompt = text;
    job.creativeBrief = text;
    job.shotTags = null;
    job.shots.flatMap((shot) => actionReferenceIds(shot.action)).forEach((id) => {
      if (knownReferences.has(id) && !job.referenceIds.includes(id)) job.referenceIds.push(id);
    });
  });
  return parseProjectConfig(next);
}