Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

12.0 KB
Raw Blame
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { AGUIMock } from "@copilotkit/aimock/agui";
import { testAgentConnection } from "../src/agents/connection-test";

/**
 * The connection test, against something that really speaks AG-UI.
 *
 * Uses a real AG-UI mock rather than a stubbed fetch. The other tests for this hand it a function that returns a hand-written
 * SSE body, which proves the parsing and nothing about the protocol: every one of them would still
 * pass if AG-UI changed its wire format underneath us, or if we had misread it in the first place.
 * The whole promise is that somebody else's agent works if it speaks this protocol, so the
 * test that matters is one where an implementation we did not write answers.
 *
 * `@copilotkit/aimock` is ours, which is the point: it is the org's deterministic backend for
 * exactly this, it tracks the protocol as the protocol moves, and using it here means OpenBot finds
 * out about a drift in the same week as everything else that depends on it rather than in a
 * customer's integration.
 *
 * Deterministic and offline. No key, no network, no spend, and the same answer on every run, so
 * this belongs in CI, which is where an agent-facing contract most needs watching.
 */

describe("registering an agent that really answers", () => {
  const mock = new AGUIMock();
  let url = "";

  beforeAll(async () => {
    // The events a well-behaved agent sends for a trivial run: it starts, says something, finishes.
    mock.onRun(/.*/, [
      { type: "RUN_STARTED", threadId: "t", runId: "r" },
      { type: "TEXT_MESSAGE_START", messageId: "m", role: "assistant" },
      { type: "TEXT_MESSAGE_CONTENT", messageId: "m", delta: "ready" },
      { type: "TEXT_MESSAGE_END", messageId: "m" },
      { type: "RUN_FINISHED", threadId: "t", runId: "r" },
    ] as never);
    url = await mock.start();
  });

  afterAll(async () => {
    await mock.stop?.();
  });

  test("an agent speaking real AG-UI is reported as working, with what it sent", async () => {
    const result = await testAgentConnection(url, { allowPrivateHosts: true });

    expect(result.ok).toBe(true);
    if (!result.ok) return;
    // The events are shown to the person registering it, so that they can see it really answered
    // rather than being told it did.
    expect(result.events).toContain("RUN_STARTED");
    expect(result.events).toContain("RUN_FINISHED");
  });

  test("the address is still checked before anything is dialled", async () => {
    // Same guard as registration: the button must not become a way to probe the internal network,
    // and a live server on the other end does not change that.
    const result = await testAgentConnection("http://169.254.169.254/", {
      allowPrivateHosts: true,
    });
    expect(result.ok).toBe(false);
  });

  test("a redirect cannot carry the request somewhere the check refuses", async () => {
    // The check only ever sees the URL a person typed. Following a redirect blindly makes that check
    // decorative: anything registrable can bounce the server at the metadata endpoint.
    const bounced: string[] = [];
    const redirector = Bun.serve({
      port: 0,
      fetch: (request) => {
        bounced.push(request.url);
        return new Response(null, {
          status: 307,
          headers: { location: "http://169.254.169.254/latest/meta-data/" },
        });
      },
    });

    try {
      const result = await testAgentConnection(
        `http://127.0.0.1:${redirector.port}/ag-ui`,
        { allowPrivateHosts: true, timeoutMs: 4_000 },
      );

      expect(result.ok).toBe(false);
      if (!result.ok) expect(result.reason).toMatch(/redirect/i);
      // The first hop is the registered address and is expected. Nothing beyond it should have been
      // dialled, which is what the refusal is for.
      expect(bounced.length).toBe(1);
    } finally {
      redirector.stop(true);
    }
  });

  test("a redirect to an address the check permits is still followed", async () => {
    // A deployment that puts its agent behind a redirect, http to https being the ordinary case, has
    // done nothing wrong. Refusing every redirect would break it, so the destination is checked
    // rather than the hop count.
    const redirector = Bun.serve({
      port: 0,
      fetch: () =>
        new Response(null, { status: 307, headers: { location: url } }),
    });

    try {
      const result = await testAgentConnection(
        `http://127.0.0.1:${redirector.port}/ag-ui`,
        { allowPrivateHosts: true, timeoutMs: 8_000 },
      );

      expect(result.ok).toBe(true);
      if (result.ok) expect(result.events).toContain("RUN_STARTED");
    } finally {
      redirector.stop(true);
    }
  });

  test("a redirect that never arrives anywhere gives up rather than looping", async () => {
    const looper = Bun.serve({
      port: 0,
      fetch: (request) =>
        new Response(null, { status: 307, headers: { location: request.url } }),
    });

    try {
      const result = await testAgentConnection(
        `http://127.0.0.1:${looper.port}/ag-ui`,
        { allowPrivateHosts: true, timeoutMs: 8_000 },
      );
      expect(result.ok).toBe(false);
    } finally {
      looper.stop(true);
    }
  });

  test("a port with nothing on it reports the direction of the connection", async () => {
    // The server dials the agent, so localhost must be tested from the server side.
    const dead = await testAgentConnection("http://127.0.0.1:9/", {
      allowPrivateHosts: true,
      timeoutMs: 2_000,
    });
    expect(dead.ok).toBe(false);
    if (!dead.ok) {
      expect(dead.reason).toMatch(/reachable from|did not answer/);
    }
  });
});

/**
 * The shapes a real agent turns up in, beyond the happy one.
 *
 * The connection test is what somebody sees when they register a colleague's agent, and the verdict
 * decides whether they go looking at their own service or at ours. A stubbed SSE body proves we parse
 * the string we wrote; these put a real AG-UI implementation on the other end and make it misbehave
 * in the ways a real one does.
 */
describe("registering an agent that answers badly", () => {
  test("an agent that starts a run and never finishes is still reported as working", async () => {
    /*
     * A run that produces events is a working endpoint. Whether the agent finishes its work is the
     * agent's business and not something a registration check should refuse over: the alternative is
     * a person unable to register a slow agent, told their perfectly good service is broken.
     */
    const partial = new AGUIMock();
    partial.onRun(/.*/, [
      { type: "RUN_STARTED", threadId: "t", runId: "r" },
      { type: "TEXT_MESSAGE_START", messageId: "m", role: "assistant" },
    ] as never);
    const partialUrl = await partial.start();

    try {
      const result = await testAgentConnection(partialUrl, {
        allowPrivateHosts: true,
      });

      expect(result.ok).toBe(true);
      if (!result.ok) return;
      expect(result.events).toContain("RUN_STARTED");
    } finally {
      await partial.stop?.();
    }
  });

  test("an agent that reports its own error still counts as speaking the protocol", async () => {
    // `RUN_ERROR` is the agent telling us something went wrong, which means it is there and it
    // speaks AG-UI. Refusing registration here would send somebody to debug the wrong end.
    const failing = new AGUIMock();
    failing.onRun(/.*/, [
      { type: "RUN_STARTED", threadId: "t", runId: "r" },
      { type: "RUN_ERROR", message: "the model is unavailable" },
    ] as never);
    const failingUrl = await failing.start();

    try {
      const result = await testAgentConnection(failingUrl, {
        allowPrivateHosts: true,
      });

      expect(result.ok).toBe(true);
      if (!result.ok) return;
      expect(result.events).toContain("RUN_ERROR");
    } finally {
      await failing.stop?.();
    }
  });

  test("a tool call in the stream is reported, because that is what a governed Bot does", async () => {
    /*
     * The event that matters most for this product. A remote Bot yields a tool call and OpenBot
     * decides it, so an endpoint that emits one is exactly the shape the gateway is built for, and
     * somebody registering it should see that it did.
     */
    const calling = new AGUIMock();
    calling.onRun(/.*/, [
      { type: "RUN_STARTED", threadId: "t", runId: "r" },
      {
        type: "TOOL_CALL_START",
        toolCallId: "call_1",
        toolCallName: "search_issues",
      },
      { type: "TOOL_CALL_ARGS", toolCallId: "call_1", delta: '{"q":"x"}' },
      { type: "TOOL_CALL_END", toolCallId: "call_1" },
      { type: "RUN_FINISHED", threadId: "t", runId: "r" },
    ] as never);
    const callingUrl = await calling.start();

    try {
      const result = await testAgentConnection(callingUrl, {
        allowPrivateHosts: true,
      });

      expect(result.ok).toBe(true);
      if (!result.ok) return;
      expect(result.events).toContain("TOOL_CALL_START");
    } finally {
      await calling.stop?.();
    }
  });

  test("nothing listening is a refusal a person can act on", async () => {
    // The commonest registration mistake: a typo, or a service that is not up yet. It has to read as
    // "we could not reach it" rather than as a protocol complaint.
    const result = await testAgentConnection("http://127.0.0.1:9/ag-ui", {
      allowPrivateHosts: true,
    });

    expect(result.ok).toBe(false);
    if (result.ok) return;
    expect(result.reason.length).toBeGreaterThan(0);
  });
});

/**
 * An agent that keeps talking.
 *
 * The check needs the opening of the stream and says so: the cap beside it is named for how much of
 * the answer is enough to prove the far end is an agent. What decides whether that cap means
 * anything is whether the read stops there, and the person registering an agent that streams for a
 * while is the one who finds out.
 */
describe("registering an agent that streams a long answer", () => {
  /** The events, and then a run that goes on writing: a Bot working through a long document. */
  function talkativeAgent() {
    return Bun.serve({
      port: 0,
      fetch: () =>
        new Response(
          new ReadableStream<Uint8Array>({
            async start(controller) {
              const encoder = new TextEncoder();
              controller.enqueue(
                encoder.encode(
                  'event: RUN_STARTED\ndata: {"type":"RUN_STARTED"}\n\n' +
                    'event: TEXT_MESSAGE_START\ndata: {"type":"TEXT_MESSAGE_START"}\n\n',
                ),
              );
              // A kilobyte at a time, slowly, the way a model streams. Nothing here closes: the run
              // is still going, which is the whole point.
              for (let chunk = 0; chunk < 4_000; chunk += 1) {
                controller.enqueue(
                  encoder.encode(
                    `event: TEXT_MESSAGE_CONTENT\ndata: {"type":"TEXT_MESSAGE_CONTENT","delta":"${"x".repeat(960)}"}\n\n`,
                  ),
                );
                await Bun.sleep(20);
              }
              controller.close();
            },
          }),
          { headers: { "content-type": "text/event-stream" } },
        ),
    });
  }

  test("is reported as working, without waiting for it to finish", async () => {
    const agent = talkativeAgent();

    try {
      const started = Date.now();
      const result = await testAgentConnection(
        `http://127.0.0.1:${agent.port}/ag-ui`,
        { allowPrivateHosts: true, timeoutMs: 3_000 },
      );
      const took = Date.now() - started;

      // It answered, immediately and correctly, with events this check can read. Anything else is
      // this deployment describing a working agent as one that may not be reachable.
      expect(result.ok).toBe(true);
      if (!result.ok) return;
      expect(result.events).toContain("RUN_STARTED");
      // Well inside the timeout. The check has what it needs after the opening of the stream, and
      // an answer that arrives only when the agent stops talking is an answer that depends on the
      // agent stopping.
      expect(took).toBeLessThan(2_000);
    } finally {
      agent.stop(true);
    }
  });
});