Skip to content

Open source CapCut alternative.

TypeScript
8.0 KB
Raw Blame
"use client";

import { geminiModelRoleNames } from "@/lib/inference/gemini-models";
import type { AssetRef } from "./assetRef";
import { hostedPost } from "./hosted";
import { readRefText, refsToParts, type InlineImage } from "./refMedia";

// Reference-aware prompt composition, shared by video and image generation.
//
// The generators read their input images literally: the video model plays its one input
// image as the video's first frame, and the image model edits the pixels it is
// given. So a prompt that asks to *transform* a reference ("turn this into a
// korean woman") loses to the picture, and text-file references (a script,
// notes) can't ride as images at all. Before rendering, a small multimodal
// Gemini call reads every reference — video frames, images, text contents —
// plus the user's request, rewrites the prompt to stand alone, and picks which
// pictures should still ride to the generator. Best-effort throughout: any
// failure returns null and the caller falls back to the raw prompt + refs.

const VIDEO_INSTRUCTIONS = `You prepare prompts for a text+image-to-video model. The numbered images and attached files are the user's references; their request comes last. The model accepts at most one input image and plays it as the video's literal first frame — it can animate what is pictured but cannot change who or what is in it.
Reply with JSON {"prompt": string, "keepImages": number[]}:
- keepImages lists at most one image number: the one whose exact pixels should open the video. Keep an image only when the request uses its subject as-is (animate it, continue the shot, have them speak). Keep none when the request transforms or replaces what any reference shows.
- prompt must stand alone: fold in everything the video needs from every reference — setting, framing, lighting, wardrobe, camera, mood, action, any script or text from attached files — with the requested changes applied. Describe subjects concretely; if anyone speaks, give them a matching voice and the words to say.
- A frame labelled as pinned is the user's chosen moment; frames sampled near it are the same moment a beat earlier/later. Treat the pinned frame as the reference, and when one of them should ride, keep the one that shows the subject best.
- The video model never hears attached audio. Fold what it carries into the prompt: speech becomes the words a speaker says, music or ambience becomes a concrete description of the video's own sound.`;

const IMAGE_INSTRUCTIONS = `You prepare prompts for an image generation and editing model. The numbered images and attached files are the user's references; their request comes last. The model receives the kept images as inputs it can edit, combine, and draw from.
Reply with JSON {"prompt": string, "keepImages": number[]}:
- keepImages lists the image numbers the model should receive: keep the ones the request edits, combines, or borrows a subject or style from; drop ones whose contribution is better said in words.
- prompt must stand alone given only the kept images: refer to them by their content, and fold in whatever matters from the dropped references and any script or text from attached files, with the requested changes applied.
- A frame labelled as pinned is the user's chosen moment; frames sampled near it are the same moment a beat earlier/later. Treat the pinned frame as the reference, and when one of them should ride, keep the one that shows the subject best.
- The image model never hears attached audio. Fold what it carries into the prompt: transcribe speech that should appear, translate music or ambience into the mood and setting it implies.`;

const MUSIC_INSTRUCTIONS = `You prepare prompts for a text-to-music model that generates a track from a text description alone — it never hears or sees the references. The attachments are what the user wants the music to match: an audio track to emulate, or video/images whose mood the score should carry. Their request comes last.
Reply with JSON {"prompt": string}:
- prompt must fully describe the SOUND to generate so a new track can match the reference: genre, instrumentation, vocal style and gender (or that it is instrumental), mood, energy, tempo, and production feel. For an audio reference, describe what you actually hear — groove, arrangement, key feel — closely enough to recreate it; for visual references, translate their tone and pacing into music. Keep any language the lyrics should be in.
- Describe everything by its musical traits. Never name a real artist, band, song, album, or brand — the model rejects those; state the characteristics instead.`;

export interface ComposedGen {
  prompt: string;
  /** The reference pictures the generator should receive, in kept order. */
  images: InlineImage[];
}

/** Compose a generation prompt from the user's request and their references.
 * Null on any failure (or when no ref contributes anything) — callers fall
 * back to sending the raw prompt with the visual refs as-is. */
export async function composeGenPrompt(
  target: "video" | "image",
  prompt: string,
  refs: AssetRef[]
): Promise<ComposedGen | null> {
  try {
    const { parts, visuals } = await refsToParts(refs);
    if (parts.length === 0) return null;
    const res = await hostedPost("/api/inference/responses", {
      donkeyProvider: "gemini",
      model: geminiModelRoleNames.chat,
      instructions: target === "video" ? VIDEO_INSTRUCTIONS : IMAGE_INSTRUCTIONS,
      response_format: { type: "json_object" },
      input: [{ role: "user", content: [...parts, { text: `Request: ${prompt}` }] }],
    });
    if (!res.ok) return null;
    const body = (await res.json()) as { output_text?: string };
    const parsed = JSON.parse(body.output_text ?? "") as {
      prompt?: unknown;
      keepImages?: unknown;
    };
    if (typeof parsed.prompt !== "string" || !parsed.prompt.trim()) return null;
    const keep = Array.isArray(parsed.keepImages)
      ? parsed.keepImages.filter(
          (n): n is number => typeof n === "number" && Number.isInteger(n) && n >= 1 && n <= visuals.length
        )
      : [];
    return { prompt: parsed.prompt.trim(), images: keep.map((n) => visuals[n - 1]) };
  } catch {
    return null;
  }
}

/** Compose a music-generation prompt from the user's request and their media
 * references. The music model reads only text, so a multimodal pass listens to
 * each audio reference and looks at each visual one, then folds a description
 * of the target sound into a stand-alone prompt. Null on any failure (or when
 * no ref contributes anything) — the caller falls back to the raw prompt. */
export async function composeMusicPrompt(
  prompt: string,
  refs: AssetRef[]
): Promise<string | null> {
  try {
    const { parts } = await refsToParts(refs);
    if (parts.length === 0) return null;
    const res = await hostedPost("/api/inference/responses", {
      donkeyProvider: "gemini",
      model: geminiModelRoleNames.chat,
      instructions: MUSIC_INSTRUCTIONS,
      response_format: { type: "json_object" },
      input: [{ role: "user", content: [...parts, { text: `Request: ${prompt}` }] }],
    });
    if (!res.ok) return null;
    const body = (await res.json()) as { output_text?: string };
    const parsed = JSON.parse(body.output_text ?? "") as { prompt?: unknown };
    if (typeof parsed.prompt !== "string" || !parsed.prompt.trim()) return null;
    return parsed.prompt.trim();
  } catch {
    return null;
  }
}

/** Fallback text folding for when the compose call fails: append each text
 * ref's contents to the prompt so a dropped script still reaches the
 * generator. Unreadable files are skipped. */
export async function foldTextRefs(prompt: string, refs: AssetRef[]): Promise<string> {
  const blocks = await Promise.all(
    refs
      .filter((r) => r.kind === "text")
      .map((r) =>
        readRefText(r).then(
          (contents) => `Contents of "${r.name}":\n${contents}`,
          () => null
        )
      )
  );
  const kept = blocks.filter((b): b is string => b !== null);
  return kept.length > 0 ? `${prompt}\n\n${kept.join("\n\n")}` : prompt;
}