Skip to content

Open source CapCut alternative.

TypeScript
4.0 KB
Raw Blame
import type { TSchema } from "typebox";
import type { AgentTool, AgentToolResult } from "@earendil-works/pi-agent-core";
import type { AiToolDef } from "../aiToolDef";
import { splitToolDisplay } from "../toolDisplay";
import type { DonkeyToolDetails, WirePart } from "./donkeyStream";

// The tool bridge: the shared AiToolDef catalog as pi AgentTools. Execution is
// injected — production serves skills locally and runs everything else on the
// live editor store; the eval serves sims and stubs — so this file carries no
// environment of its own.

export type ExecTool = (name: string, args: Record<string, unknown>) => Promise<unknown>;

const DATA_URL = /^data:([^;,]+);base64,(.+)$/;

/** A tool's raw output split for the wire: its `display` markup goes to the
 * UI alone, media (frames, audio) moves to its own following user turn —
 * Gemini degenerates when media shares a turn with a functionResponse — and
 * megabyte payloads stay out of the JSON the model and the UI transcript carry. */
export function toToolResult(name: string, output: unknown): AgentToolResult<DonkeyToolDetails> {
  const split = splitToolDisplay(output);
  const result = forModel(name, split.output);
  return split.display ? { ...result, details: { ...result.details, display: split.display } } : result;
}

/** The output the chat UI shows: the model's response with the display
 * markup put back. */
export function uiToolOutput(details: DonkeyToolDetails | undefined): unknown {
  const response = details?.response ?? null;
  if (!details?.display || !response || typeof response !== "object" || Array.isArray(response)) return response;
  return { ...response, display: details.display };
}

function forModel(name: string, output: unknown): AgentToolResult<DonkeyToolDetails> {
  if (output && typeof output === "object" && "audio" in output) {
    const { audio, ...rest } = output as { audio?: unknown; name?: unknown };
    const m = typeof audio === "string" ? DATA_URL.exec(audio) : null;
    if (m) {
      const assetName = typeof rest.name === "string" ? rest.name : name;
      return {
        content: [{ type: "text", text: JSON.stringify(rest) }],
        details: {
          response: rest,
          mediaParts: [
            { text: `Attached audio "${assetName}":` },
            { type: "input_audio", dataBase64: m[2], mimeType: m[1] },
          ],
        },
      };
    }
  }
  if (output && typeof output === "object" && ("image" in output || "images" in output)) {
    const { image, images, ...rest } = output as {
      image?: unknown;
      images?: unknown;
      source?: { name?: unknown };
    };
    const parsed = [
      ...(typeof image === "string" ? [image] : []),
      ...(Array.isArray(images) ? images.filter((u): u is string => typeof u === "string") : []),
    ]
      .map((u) => DATA_URL.exec(u))
      .filter((m): m is RegExpExecArray => m !== null);
    if (parsed.length > 0) {
      const sourceName = typeof rest.source?.name === "string" ? rest.source.name : null;
      const mediaParts: WirePart[] = [
        { text: sourceName ? `Frames from ${name} of "${sourceName}":` : `Frames from ${name}:` },
        ...parsed.map((m) => ({ type: "input_image", dataBase64: m[2], mimeType: m[1] })),
      ];
      return {
        content: [{ type: "text", text: JSON.stringify(rest) }],
        details: { response: rest, mediaParts },
      };
    }
  }
  const response =
    output && typeof output === "object" && !Array.isArray(output)
      ? output
      : { result: output ?? null };
  return {
    content: [{ type: "text", text: JSON.stringify(response) }],
    details: { response },
  };
}

export function toAgentTools(defs: AiToolDef[], execTool: ExecTool): AgentTool[] {
  return defs.map((def) => ({
    name: def.name,
    label: def.name,
    description: def.description,
    parameters: def.inputSchema as unknown as TSchema,
    execute: async (_toolCallId, params) =>
      toToolResult(def.name, await execTool(def.name, params as Record<string, unknown>)),
  }));
}