Skip to content

Open source CapCut alternative.

TypeScript
8.2 KB
Raw Blame
import { parseFolderLink } from "@/cut/lib/folderReference";
import { copyFile } from "node:fs/promises";
import { UI_TOOLS } from "@/cut/lib/projectCommands";
import { PAGE_ONLY_TOOLS, runProjectCommand } from "../../lib/aiTools";
import { projectOperation } from "../../lib/projectOperation";
import { localBackend } from "../../lib/backend/local";
import { DETACHED_MEDIA_ERROR, DETACHED_SESSION_ERROR, DETACHED_UI_NOTE } from "../../lib/chatResume";
import { mp4NameFor, setHostMediaStore } from "../../lib/mediaConvert";
import { setHostDocStore } from "../../lib/projectReference";
import { serializeDoc, useEditor } from "../../lib/store";
import type { MediaAsset, ProjectDoc } from "../../lib/types";
import { convertToMp4 } from "../convert";
import { probeDuration } from "../frames";
import { deleteMedia, listProjects, mediaPath, readProject, writeProject } from "../projects";
import { uniqueName } from "../util";
import { withFileColors } from "../fileColor";
import { drawnLists } from "../../lib/itemKinds";

// The engine's own tool executor, for a chat whose editor tab is gone: the
// project doc hydrates into the same store the page uses, tools run against
// it, and every mutation lands back on disk. A turn survives its tab.
//
// Scope: doc edits. Tools that need the page (preview reads, media decodes)
// or the page's hosted sign-in (generation, library sync) answer with a
// typed refusal so the model reports honestly and moves on.

/** Tools that decode or rasterize media. The engine runs on Bun with the
 * bundled tools and no canvas, decoders or Web Audio, so these answer with a
 * typed refusal here; the cloud runner installs those primitives and runs the
 * same tools for real. */
const PAGE_MEDIA_TOOLS: ReadonlySet<string> = new Set([
  "watch_video",
  "listen_audio",
  "measure_level",
  "detect_silence",
  "detect_beats",
  "refine_speech_cuts",
  "sync_audio",
  "freeze_frame",
  "capture_frame",
  "compare_to_source",
  "find_picture",
  "detect_cuts",
  "sample_color",
  "track_element",
  "track_camera",
  "suggest_font",
  "render_preview",
  "create_sticker",
  "subtitles_generate",
  "captions_generate",
  "subtitles_from_visuals",
  "stock_add",
  "read_color_stats",
  "match_color_grade",
  ...PAGE_ONLY_TOOLS,
]);

const PAGE_SESSION_TOOLS: ReadonlySet<string> = new Set([
  "generate_image",
  "generate_video",
  "generate_character_video",
  "generate_scene",
  "approve_scene",
  "cancel_scene",
  "regenerate_shot",
  "recut_scene",
  "restyle_scene",
  "generate_music",
  "voiceover_generate",
  "read_subtitles_aloud",
  "subtitles_translate_track",
  "wait_for_renders",
  "library_list",
  "library_add",
  "library_organize",
  "library_share",
  "notes_list",
  "read_note",
  "note_save",
  "template_add",
  "file_asset",
  "import_url",
  "check_facts",
  "set_interface_language",
]);

// Converting media is the one media job this process can do: ffmpeg is right
// here, and the file is in the project folder. Going through the client's
// backend seam would have the engine call itself over HTTP, so the converter
// works the folder directly and the shared client code never notices.
setHostMediaStore({
  async convert(projectId, source, opts) {
    const src = mediaPath(projectId, source.fileName);
    const fileName = await uniqueName(mp4NameFor(source.fileName), (n) => mediaPath(projectId, n));
    const outPath = mediaPath(projectId, fileName);
    const handle = { tmpDir: "", outPath, progress: 0, log: [] as string[] };
    const outcome = await convertToMp4(handle, src, outPath, { maxHeight: opts.maxHeight, mode: opts.mode });
    return {
      fileName: outcome.unchanged ? source.fileName : fileName,
      ...(outcome.unchanged ? { unchanged: true } : {}),
      reencoded: outcome.transcodedVideo || outcome.transcodedAudio,
      duration: await probeDuration(outcome.unchanged ? src : outPath),
      ...(outcome.width !== undefined ? { width: outcome.width, height: outcome.height } : {}),
    };
  },
  drop(projectId, fileName) {
    return deleteMedia(projectId, fileName);
  },
});

// Another project on this Mac, listed and read as a reference: the doc off
// disk and a file copied folder to folder. A cloud or shared reference needs the page's
// sign-in, and the reader says so.
setHostDocStore({
  listDocs: () => listProjects(),
  readDoc: (projectId) => readProject(projectId),
  async copyMedia(srcProjectId, fileName, dstProjectId) {
    const target = await uniqueName(fileName, (n) => mediaPath(dstProjectId, n));
    await copyFile(mediaPath(srcProjectId, fileName), mediaPath(dstProjectId, target));
    return target;
  },
});

interface OpenDoc {
  projectId: string;
  stored: ProjectDoc;
}

let open: OpenDoc | null = null;

async function ensureOpen(projectId: string): Promise<OpenDoc | null> {
  // Read the doc on every call: the page saves through writeProject between
  // turns, and flushing a store hydrated from an older read would revert
  // those edits. The store is reused only while the disk copy still matches
  // the one it was hydrated from.
  const stored = await readProject(projectId);
  if (!stored) return null;
  const s = useEditor.getState();
  if (
    open?.projectId === projectId &&
    s.projectId === projectId &&
    s.loaded &&
    stored.updatedAt === open.stored.updatedAt
  ) {
    return open;
  }
  // Footage imported before its color was read gets it here, off the file,
  // so what a tool renders and what the doc saves both read it right.
  const assets: MediaAsset[] = await withFileColors(
    (stored.assets ?? []).map((a) => ({
      ...a,
      url: `/api/cut/projects/${encodeURIComponent(projectId)}/media/${encodeURIComponent(a.fileName)}`,
    })),
    drawnLists(stored),
    (fileName) => mediaPath(projectId, fileName)
  );
  await useEditor.getState().openProjectDoc(projectId, stored, assets);
  open = { projectId, stored };
  return open;
}

async function flush(doc: OpenDoc): Promise<void> {
  const merged = { ...doc.stored, ...serializeDoc(useEditor.getState()) } as ProjectDoc;
  await writeProject(doc.projectId, merged);
  doc.stored = merged;
}

/** True when a tool input carries svg markup to draw. */
function hasSvgMarkup(input: unknown): boolean {
  const svg = (input as { svg?: unknown } | null)?.svg;
  return typeof svg === "string" && svg.trim() !== "";
}

/** Run one tool call against the engine-held store for `projectId`,
 * persisting the doc after it. The bridge calls this when no editor tab is
 * attached to the chat session. */
async function executeHeadlessTool(
  projectId: string,
  toolName: string,
  input: unknown
): Promise<{ output?: unknown; errorText?: string }> {
  if (UI_TOOLS.has(toolName))
    return {
      output: { noEditor: true, note: DETACHED_UI_NOTE },
    };
  if (PAGE_MEDIA_TOOLS.has(toolName)) return { errorText: DETACHED_MEDIA_ERROR };

  // A sticker drawn from svg markup rasterizes it, and the engine has no
  // canvas; placing an existing sticker asset is a doc edit and runs here.
  if (toolName === "add_sticker" && hasSvgMarkup(input)) {
    return { errorText: DETACHED_MEDIA_ERROR };
  }
  if (toolName === "read_folder") {
    const args = (input ?? {}) as { reference?: { scope?: string }; link?: string };
    const ref = args.reference ?? (args.link ? parseFolderLink(args.link) : null);
    if (ref?.scope !== "project") return { errorText: DETACHED_SESSION_ERROR };
  }
  if (PAGE_SESSION_TOOLS.has(toolName)) return { errorText: DETACHED_SESSION_ERROR };
  const doc = await ensureOpen(projectId);
  if (!doc) return { errorText: "Project not found on this Mac." };
  try {
    const output = await runProjectCommand(projectOperation(projectId, localBackend), toolName, (input ?? {}) as Record<string, unknown>);
    await flush(doc);
    return { output: output ?? null };
  } catch (err) {
    return { errorText: err instanceof Error ? err.message : String(err) };
  }
}

let toolsIdle: Promise<unknown> = Promise.resolve();
/** The engine's editor store belongs to one tool invocation at a time. */
export function callHeadlessTool(projectId: string, toolName: string, input: unknown): Promise<{ output?: unknown; errorText?: string }> {
  const result = toolsIdle.then(() => executeHeadlessTool(projectId, toolName, input));
  toolsIdle = result.catch(() => {});
  return result;
}