Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

9.8 KB
Raw Blame
/**
 * The conversation AG-UI carries, as LangChain's message classes.
 *
 * Its own module so it can be tested without starting a server: `index.ts` calls `serve()` at module
 * scope, so importing it to reach one pure function binds a port. `agent-computer/src/control.ts`
 * was split out for the same reason, to keep state-machine tests away from a browser.
 */
import type { RunAgentInput } from "@ag-ui/core";
import {
  AIMessage,
  type BaseMessage,
  HumanMessage,
  SystemMessage,
  ToolMessage,
} from "@langchain/core/messages";
import { computerGuidance, NO_ANSWER_CAME } from "../../shared/bot-prompt";
import { researchRoleOfToolName } from "../../shared/research";
import { userContent } from "../../shared/user-content";

/*
 * Re-exported so this module's own tests and callers keep reading it from here, while the wording
 * itself lives in `shared` where the other Bot can reach it. Both have to say the same thing.
 */
export { NO_ANSWER_CAME };

/**
 * The providers that take one system prompt, and take it first.
 *
 * Anthropic's Messages API has the system prompt as a field of the request rather than as a turn,
 * and Gemini has it as `systemInstruction`, so neither integration has anywhere to put a second one:
 * `@langchain/anthropic` throws "System messages are only permitted as the first passed message."
 * and `@langchain/google-genai` throws "System message should be the first one", both before any
 * request is made. OpenAI takes a system turn anywhere in a conversation.
 */
const ONE_SYSTEM_PROMPT = new Set(["anthropic", "google"]);

/** Translate the conversation AG-UI carries into LangChain's message classes. */
export function toLangChainMessages(
  input: RunAgentInput,
  provider = "openai",
): BaseMessage[] {
  const messages: BaseMessage[] = [
    // Worded for the tools this run offers; see `computerGuidance` and agent-bot's history.ts.
    new SystemMessage(
      computerGuidance({
        search: (input.tools ?? []).some(
          (tool) => researchRoleOfToolName(tool.name) === "search",
        ),
      }),
    ),
    // AG-UI carries application context separately from conversation history. CopilotKit puts
    // the A2UI catalog and tool instructions here; dropping it leaves the model guessing the
    // component schema and can strand the renderer on an invalid, never-painted surface.
    ...(input.context ?? []).map(
      ({ description, value }) => new SystemMessage(`${description}\n${value}`),
    ),
  ];

  /*
   * Which calls in this history were ever answered.
   *
   * A tool call the surface owns ends the run without a result on purpose: the surface draws it, or
   * puts it to a person, and starts the next run carrying the answer. When nobody answers — a Bot
   * asks for the wheel to get past a sign-in and the person decides they do not need it after all —
   * no answer is ever carried, and the call stays in the history with nothing following it.
   *
   * OpenAI rejects that outright on the NEXT turn: "an assistant message with 'tool_calls' must be
   * followed by tool messages responding to each 'tool_call_id'". So the conversation was not merely
   * stuck on that request, it was finished. Every later message failed the same way, and the only
   * escape was starting a new one, which loses it.
   *
   * Collected up front because an answer arrives as a later message than the call it answers.
   */
  const answered = new Set(
    input.messages
      .filter((message) => message.role === "tool")
      .map((message) => (message as { toolCallId?: string }).toolCallId)
      .filter((id): id is string => Boolean(id)),
  );

  /*
   * Tool results, by the call they answer.
   *
   * The history is not guaranteed to arrive with a result after the call it belongs to. Read back
   * from the durable thread store it arrives the other way round, result first, which is a payload
   * no provider accepts: a tool message with no preceding call, and then a call with nothing
   * following it. The model answers that with silence rather than an error, which is the worst of
   * both, so the pairing is rebuilt here instead of trusted.
   */
  const resultsByCall = new Map<string, string>();
  for (const message of input.messages) {
    if (message.role !== "tool") continue;
    const id = (message as { toolCallId?: string }).toolCallId;
    if (id) resultsByCall.set(id, String(message.content ?? ""));
  }

  for (const message of input.messages) {
    // Placed with the call they answer, below, rather than wherever they arrived.
    if (message.role === "tool") continue;
    if (message.role === "user") {
      messages.push(
        new HumanMessage({ content: userContent(message.content) }),
      );
      continue;
    }
    if (message.role === "system" || message.role === "developer") {
      messages.push(new SystemMessage(String(message.content ?? "")));
      continue;
    }
    if (message.role === "assistant") {
      messages.push(
        new AIMessage({
          content: message.content ?? "",
          tool_calls:
            message.toolCalls?.map((call) => ({
              id: call.id,
              name: callDetails(call).name,
              // LangChain wants parsed arguments where AG-UI carries the raw string. A call whose
              // arguments did not parse is passed as empty rather than dropped: the model needs to
              // see that it made the call, or it makes it again.
              args: parseArguments(callDetails(call).arguments),
            })) ?? [],
        }),
      );

      /*
       * Every call this message made, answered, immediately after it.
       *
       * The real result where there is one, wherever it arrived in the input, and `NO_ANSWER_CAME`
       * where there is not. Both cases are the same requirement: a call must be followed by its
       * result, and the provider rejects the message outright otherwise.
       */
      for (const call of message.toolCalls ?? []) {
        if (!call.id) continue;
        messages.push(
          new ToolMessage({
            tool_call_id: call.id,
            content: answered.has(call.id)
              ? (resultsByCall.get(call.id) ?? "")
              : NO_ANSWER_CAME,
            name: callDetails(call).name,
          }),
        );
      }
    }
  }

  /*
   * A run that carries no human turn is answered by OpenAI and refused by the strict providers.
   *
   * OpenAI tolerates a history that opens on an assistant or tool message. Anthropic and the strict
   * OpenAI-compatible providers (z.ai GLM among them) require the first non-system message to be a
   * human one, and will not answer a history that is only deltas: an assistant turn and its tool
   * results with nothing a person said to respond to. A follow-up run continuing after a tool result
   * is exactly that shape, so on those providers it came back empty and the run ended in silence.
   *
   * A neutral continuation turn gives them one to answer. It is appended only when the history holds
   * no human turn at all, so a normal conversation is untouched, and OpenAI — which already answered
   * the same history — sees no change beyond one trailing line asking it to continue.
   */
  const hasHumanTurn = messages.some(
    (message): message is HumanMessage => message instanceof HumanMessage,
  );
  if (!hasHumanTurn) {
    messages.push(new HumanMessage(CONTINUE_TURN));
  }

  /*
   * Every run holds more than one system message, so on those providers every run failed.
   *
   * The computer guidance opens this list and the caller's context follows it, and the server puts a
   * coworker's standing role at the head of `input.messages` on every run it sends a remote Bot. A
   * skill somebody picks arrives as a system turn ahead of their message, too. So a Bot set to
   * `BOT_PROVIDER=anthropic` or `google` answered nothing at all: each run ended in the integration's
   * refusal, before the model was asked.
   *
   * Folded into one, in the order given, and only for those providers. On OpenAI a skill's
   * instruction stays beside the message it was picked for.
   */
  return ONE_SYSTEM_PROMPT.has(provider)
    ? withOneSystemPrompt(messages)
    : messages;
}

/** The continuation a strict provider needs when a run carries only deltas. See toLangChainMessages. */
const CONTINUE_TURN = "Continue from where the conversation above left off.";

/** Every system message's text as one system message at the top, and the rest as they were. */
function withOneSystemPrompt(messages: BaseMessage[]): BaseMessage[] {
  const system = messages.filter((message) => message instanceof SystemMessage);
  return [
    new SystemMessage(
      system.map((message) => String(message.content)).join("\n\n"),
    ),
    ...messages.filter((message) => !(message instanceof SystemMessage)),
  ];
}

function parseArguments(raw: string): Record<string, unknown> {
  try {
    const parsed = JSON.parse(raw || "{}");
    return typeof parsed === "object" && parsed !== null
      ? (parsed as Record<string, unknown>)
      : {};
  } catch {
    return {};
  }
}

/**
 * A tool call's name and arguments, in whichever dialect it arrived in.
 *
 * TWO SPELLINGS, ONE CALL. AG-UI describes `{id, type: "function", function: {name, arguments}}` and
 * the history store writes `{id, name, args}`. Read back from a thread, every call arrives in the
 * second, so `call.function.name` here did not merely degrade: it threw, and took the run with it.
 */
function callDetails(call: {
  function?: { name?: unknown; arguments?: unknown };
  name?: unknown;
  args?: unknown;
}): { name: string; arguments: string } {
  const name = call.function?.name ?? call.name;
  const args = call.function?.arguments ?? call.args;
  return {
    name: typeof name === "string" && name ? name : "tool",
    arguments:
      typeof args === "string"
        ? args
        : args === undefined || args === null
          ? "{}"
          : JSON.stringify(args),
  };
}