Skip to content

AI media harness for planning, generating, and editing locally

TypeScript
10.3 KB
Raw Blame
import { FIBO_MODEL_TYPE, MOSS_MODEL_TYPE, KREA_MODEL_TYPE, generatorModel } from "./settings";
import { imageSettings } from "./imageSettings";
import { createImageScene, imageScenePromptParts, imageScenePromptText } from "./imageScene";
import { actionReferenceIds, activeReferenceRefmods, isVideoReference, referenceDefinition, referenceImages, splitActionText, type ProjectConfig } from "./project";
import { SHOT_TAG_GROUPS } from "./shot-tags";

/** Shared by generation and Debug Prompt. H3's base three-field / reference
 * six-section contracts still apply to a single silent image. */
export function compileImagePrompt(config: ProjectConfig, options: { modelType?: string; referenceTransformations?: Readonly<Record<string, string>>; sourceTreatment?: "outpaint" } = {}) {
  const scene = config.imageScene ?? createImageScene(config.brief.prompt);
  // Only the references the prompt cites as @[ref:<id>] go to the generation.
  const references = actionReferenceIds(imageScenePromptText(scene)).map((id) => {
    const reference = config.references.find((candidate) => candidate.id === id);
    if (!reference) throw new Error("The prompt cites a reference that no longer exists. Swap or remove it before generating.");
    if (isVideoReference(reference) || reference.kind === "audio") throw new Error(`Still images accept image and text references; the prompt cites the ${reference.kind} reference '${reference.name}'.`);
    return reference;
  });
  const model = generatorModel(options.modelType);
  if (!model.imageEditing && scene.rootType === "image") throw new Error(`${model.label} does not support image editing.`);
  if (!model.imageReferences && references.some((reference) => referenceImages(reference).length || activeReferenceRefmods(reference).length)) {
    throw new Error(`${model.label} does not support reference images or refmods. Choose a reference generator.`);
  }
  if (options.modelType === "h3_image") {
    if (references.some((reference) => activeReferenceRefmods(reference).length)) throw new Error("MiniMax H3 Image does not support refmods.");
    const editing = scene.rootType === "image" && Boolean(scene.sourceImage);
    let picture = editing ? 1 : 0;
    const descriptions = new Map(references.map((reference) => {
      const labels = referenceImages(reference).map(() => `<Picture ${++picture}>`);
      return [reference.id, `${reference.name}${labels.length ? ` (${labels.join(", ")})` : ""}${referenceDefinition(reference) ? `: ${referenceDefinition(reference)}` : ""}`];
    }));
    if (picture > model.maxImageReferences) throw new Error("MiniMax H3 Image supports at most 16 pictures including the edit source.");
    const look = SHOT_TAG_GROUPS.find((group) => group.id === "visualStyle")?.options.find((option) => option.id === imageSettings(config).defaultLook)?.label;
    const parts = imageScenePromptParts(scene, look);
    const authored = parts ? [parts.style, parts.composition].filter(Boolean).join("\n") : "";
    const prompt = splitActionText(authored).map((part) => part.kind === "text" ? part.value : descriptions.get(part.value)!).join("");
    return { prompt: editing && prompt ? `Edit <Picture 1>. ${prompt}` : prompt, references };
  }
  if (options.modelType === FIBO_MODEL_TYPE) {
    const pictures = references.flatMap(referenceImages);
    if (references.some((reference) => activeReferenceRefmods(reference).length)) throw new Error("Fibo 2.0 does not support refmods.");
    if (scene.rootType === "image" && pictures.length) throw new Error("Fibo edits use the source image only. Remove additional image references from this edit prompt.");
    if (pictures.length > model.maxImageReferences) throw new Error(`Fibo 2.0 supports at most ${model.maxImageReferences} image references.`);
    const parts = imageScenePromptParts(scene);
    const rootPrompt = scene.nodes.find((node) => node.kind === "root")?.description.trim() ?? "";
    // Preserve optional structured edit JSON verbatim instead of adding prose punctuation.
    const structured = rootPrompt.startsWith("{") && scene.nodes.length === 1 && !scene.background.trim() && scene.style.mode === "none";
    const authored = structured ? rootPrompt : parts ? [parts.style, parts.composition].filter(Boolean).join("\n") : "";
    let imageIndex = 0;
    const descriptions = new Map(references.map((reference) => {
      const numbers = referenceImages(reference).map(() => `image ${++imageIndex}`);
      return [reference.id, `${reference.name}${numbers.length ? ` (${numbers.join(", ")})` : ""}${referenceDefinition(reference) ? `: ${referenceDefinition(reference)}` : ""}`];
    }));
    return { prompt: splitActionText(authored).map((part) => part.kind === "text" ? part.value : descriptions.get(part.value)!).join(""), references };
  }
  if (options.modelType === MOSS_MODEL_TYPE || options.modelType === KREA_MODEL_TYPE) {
    const look = SHOT_TAG_GROUPS.find((group) => group.id === "visualStyle")?.options.find((option) => option.id === imageSettings(config).defaultLook)?.label;
    const parts = imageScenePromptParts(scene, look);
    const prompt = parts ? [parts.style, parts.composition].filter(Boolean).join("\n") : "";
    return { prompt: splitActionText(prompt).map((part) => {
      if (part.kind === "text") return part.value;
      const reference = references.find((entry) => entry.id === part.value)!;
      return `${reference.name}: ${referenceDefinition(reference) || reference.name}`;
    }).join(""), references };
  }
  // Ordinary edits submit the source before the selected reference pictures.
  // The runtime supplies the preserved outpaint crop to Qwen as "Source scene"
  // and its positioned canvas to denoising; it does not consume a Picture slot.
  const editing = scene.rootType === "image" && Boolean(scene.sourceImage);
  const outpainting = editing && options.sourceTreatment === "outpaint";
  const sourcePicture = editing && !outpainting;
  let picture = sourcePicture ? 1 : 0;
  const subjects = references.flatMap((reference) => {
    const labels = referenceImages(reference).map(() => `<Picture ${++picture}>`);
    const definition = referenceDefinition(reference);
    const refmods = activeReferenceRefmods(reference);
    if (reference.refmods?.length && !refmods.length && !labels.length && !definition) return [];
    const content = definition || reference.name;
    const description = /[.!?。!?]$/.test(content) ? content : `${content}.`;
    const role = reference.intendedUse.includes("character") || reference.intendedUse.includes("animal") ? "identity and appearance" : reference.intendedUse.includes("style") ? "visual style" : "appearance";
    const sources = [...labels, ...(refmods.length ? ["the supplied reference conditioning"] : [])];
    return [{ id: reference.id, name: reference.name, role, description, source: sources.length ? ` from ${sources.join(" and ")}` : "" }];
  });
  if (picture > 9) throw new Error("MiniMax H3 supports at most nine reference images per generation.");
  const look = SHOT_TAG_GROUPS.find((group) => group.id === "visualStyle")?.options.find((option) => option.id === imageSettings(config).defaultLook)?.label;
  const parts = imageScenePromptParts(scene, look);
  if (!parts) return { prompt: "", references };
  const label = (index: number) => `<Subject ${index + 1}>`;
  // A citation reads as the subject it defines, or by name when it defines none.
  const cite = (text: string) => splitActionText(text).map((part) => {
    if (part.kind === "text") return part.value;
    const index = subjects.findIndex((subject) => subject.id === part.value);
    return index < 0 ? references.find((reference) => reference.id === part.value)!.name : label(index);
  }).join("");
  const visual = { style: cite(parts.style), composition: cite(parts.composition) };
  const style = outpainting
    ? "Match the source scene's visual style."
    : editing ? "The still image retains the visual medium, lighting, palette and perspective of <Picture 1>." : visual.style;
  const audio = ["overall_soundscape: N/A", "non_diegetic_music: N/A"];
  if (!subjects.length && !sourcePicture) return {
    prompt: [`integrated_multimodal_description: [Shot 1] ${[style, visual.composition].filter(Boolean).join("\n")}`, ...audio].join("\n\n"), references,
  };
  const definitions = subjects.map((subject, index) => `${label(index)} is ${subject.name}, providing ${subject.role}${subject.source}. ${subject.description}`);
  const retention = subjects.map((subject, index) => {
    const transformation = options.referenceTransformations?.[subject.id];
    return transformation
      ? `${label(index)} (appears in [Shot 1]): partially_preserved - ${transformation}`
      : `${label(index)} (appears in [Shot 1]): fully_preserved - retain the referenced ${subject.role} of ${subject.name} while following the requested composition and styling.`;
  });
  if (sourcePicture) {
    definitions.unshift("<Picture 1> is the original source image and composition anchor for the edited still keyframe in [Shot 1], providing the framing, perspective, environment, lighting and visual style.");
    retention.unshift("<Picture 1> ([Shot 1] edited keyframe): partially_preserved - retain its composition and visual characteristics except for the described change. Preserve all other content, including any previously completed edits.");
  }
  const summary = outpainting
    ? "[keyframe completion + reference generation] A single still image fills the masked extension of the existing canvas with a coherent continuation of the scene."
    : editing
    ? `[keyframe completion${subjects.length ? " + reference generation" : ""}] The target is a single edited still keyframe based on <Picture 1>. Apply the described change while preserving all other content.${subjects.length ? ` Use ${subjects.map((_, index) => label(index)).join(", ")} for the specified reference attributes.` : ""}`
    : `[reference generation] A single still image uses ${subjects.map((_, index) => label(index)).join(", ")} in the requested composition.`;
  return { prompt: [
    `subject_definitions:\n${definitions.join("\n")}`,
    `summary:\n${summary}`,
    `retention_analysis:\n${retention.join("\n")}`,
    `detailed_description:\n${style ? `${style}\n` : ""}[Shot 1] ${visual.composition}\n${subjects.map((subject, index) => `Use ${label(index)} for ${subject.name}'s ${subject.role}${subject.source}. ${subject.description}`).join("\n")}`,
    ...audio,
  ].join("\n\n"), references };
}