Skip to content

Open source CapCut alternative.

TypeScript
4.2 KB
Raw Blame
"use client";

import { bytesFromBase64 } from "./bytes";
import { readHostedFailure } from "./hostedError";
import { hostedPost } from "./hosted";
import { importFileToProject } from "./media";
import { mediaSlug, type MediaAsset } from "./types";
import { UserError } from "./userError";
import type { MusicLength } from "@/lib/credits/price-sheet";

// Client side of AI music generation: Gemini/Lyria on Donkey's hosted inference
// routes, with the user's Donkey sign-in and credits (same-origin on the cut
// hosts, like image/video/voiceover generation). The prompt describes the mood
// and genre; the model renders a track — a full song with sung vocals, or an
// instrumental bed when asked — that comes back inline and saves into the
// project through the local engine like any other media file.

export type MusicVariant = MusicLength;

/** The length choices the Music generator offers — the clip model renders a
 * short bed, the pro model a full-length track. */
export const MUSIC_VARIANTS: { id: MusicVariant; label: string; hint: string }[] = [
  { id: "clip", label: "Clip", hint: "~30 sec" },
  { id: "song", label: "Full track", hint: "~2 min" },
];

const readFailure = (res: Response, fallback: UserError) =>
  readHostedFailure(res, new UserError("sign_in_to_generate_music", "Sign in to Donkey to generate music."), fallback);

/** Display name for the track: the prompt, tidied and capped. */
function musicName(prompt: string) {
  const line = prompt.trim().replace(/\s+/g, " ");
  return line.length > 60 ? `${line.slice(0, 57)}…` : line;
}

/** Generate one music track and save it into the project as a generated audio
 * asset. By default the model may sing (a full song); pass `instrumental` for a
 * vocal-free bed. Returns the asset (unplaced) — callers register it in the store
 * and decide placement, exactly like synthesizeSpeech. */
export async function synthesizeMusic(
  projectId: string,
  prompt: string,
  opts?: { variant?: MusicVariant; instrumental?: boolean; name?: string }
): Promise<MediaAsset> {
  // Lyria sings by default; the explicit clause is how you get an instrumental
  // bed (the model has no separate flag — it reads it from the prompt).
  const sent = opts?.instrumental ? `${prompt.trim()} Instrumental only, no vocals.` : prompt;
  const res = await hostedPost("/api/inference/assets", {
    kind: "music",
    prompt: sent,
    // Route to Lyria; the neutral `variant` picks the length (the model ids stay
    // server-side). Omit for the default clip.
    provider: "gemini-music",
    ...(opts?.variant === "song" ? { parameters: { variant: "song" } } : {}),
  });
  if (!res.ok) {
    throw await readFailure(res, new UserError("music_generation_failed", "Music generation failed."));
  }
  const gen = (await res.json()) as {
    outputs?: { dataBase64?: string; url?: string; contentType?: string; filename?: string }[];
  };
  const out = gen.outputs?.find((o) => o.dataBase64) ?? gen.outputs?.find((o) => o.url);

  // Cap whatever the label comes from — a caller-supplied name (the user's own
  // words when the prompt was composed from references) or the prompt itself.
  const label = musicName(opts?.name?.trim() || prompt);
  const fileName = `ai-${mediaSlug(label, "ai-music")}.mp3`;
  let file: File;
  if (out?.dataBase64) {
    file = new File([bytesFromBase64(out.dataBase64)], out.filename ?? fileName, {
      type: out.contentType ?? "audio/mpeg",
    });
  } else if (out?.url) {
    const dl = await fetch(out.url);
    if (!dl.ok) {
      throw new UserError("music_download_failed", "Could not download the generated music.");
    }
    file = new File([await dl.arrayBuffer()], fileName, {
      type: out.contentType ?? "audio/mpeg",
    });
  } else {
    throw new UserError("music_output_missing", "The provider returned no music.");
  }

  const asset = await importFileToProject(projectId, file);
  if (!asset) {
    throw new UserError("music_save_failed", "Could not save the generated music into the project.");
  }
  asset.name = label;
  // Generated (not a user import): stays out of the Media panel and lists in the
  // Audio tab's Music generator. A chat-made track is re-tagged to its thread by
  // the caller (tagChatAsset), like every other generated asset.
  asset.origin = "generated";
  return asset;
}