Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { AGUIMock } from "@copilotkit/aimock/agui";
import { testAgentConnection } from "../src/agents/connection-test";
/**
* The connection test, against something that really speaks AG-UI.
*
* Uses a real AG-UI mock rather than a stubbed fetch. The other tests for this hand it a function that returns a hand-written
* SSE body, which proves the parsing and nothing about the protocol: every one of them would still
* pass if AG-UI changed its wire format underneath us, or if we had misread it in the first place.
* The whole promise is that somebody else's agent works if it speaks this protocol, so the
* test that matters is one where an implementation we did not write answers.
*
* `@copilotkit/aimock` is ours, which is the point: it is the org's deterministic backend for
* exactly this, it tracks the protocol as the protocol moves, and using it here means OpenBot finds
* out about a drift in the same week as everything else that depends on it rather than in a
* customer's integration.
*
* Deterministic and offline. No key, no network, no spend, and the same answer on every run, so
* this belongs in CI, which is where an agent-facing contract most needs watching.
*/
describe("registering an agent that really answers", () => {
const mock = new AGUIMock();
let url = "";
beforeAll(async () => {
// The events a well-behaved agent sends for a trivial run: it starts, says something, finishes.
mock.onRun(/.*/, [
{ type: "RUN_STARTED", threadId: "t", runId: "r" },
{ type: "TEXT_MESSAGE_START", messageId: "m", role: "assistant" },
{ type: "TEXT_MESSAGE_CONTENT", messageId: "m", delta: "ready" },
{ type: "TEXT_MESSAGE_END", messageId: "m" },
{ type: "RUN_FINISHED", threadId: "t", runId: "r" },
] as never);
url = await mock.start();
});
afterAll(async () => {
await mock.stop?.();
});
test("an agent speaking real AG-UI is reported as working, with what it sent", async () => {
const result = await testAgentConnection(url, { allowPrivateHosts: true });
expect(result.ok).toBe(true);
if (!result.ok) return;
// The events are shown to the person registering it, so that they can see it really answered
// rather than being told it did.
expect(result.events).toContain("RUN_STARTED");
expect(result.events).toContain("RUN_FINISHED");
});
test("the address is still checked before anything is dialled", async () => {
// Same guard as registration: the button must not become a way to probe the internal network,
// and a live server on the other end does not change that.
const result = await testAgentConnection("http://169.254.169.254/", {
allowPrivateHosts: true,
});
expect(result.ok).toBe(false);
});
test("a redirect cannot carry the request somewhere the check refuses", async () => {
// The check only ever sees the URL a person typed. Following a redirect blindly makes that check
// decorative: anything registrable can bounce the server at the metadata endpoint.
const bounced: string[] = [];
const redirector = Bun.serve({
port: 0,
fetch: (request) => {
bounced.push(request.url);
return new Response(null, {
status: 307,
headers: { location: "http://169.254.169.254/latest/meta-data/" },
});
},
});
try {
const result = await testAgentConnection(
`http://127.0.0.1:${redirector.port}/ag-ui`,
{ allowPrivateHosts: true, timeoutMs: 4_000 },
);
expect(result.ok).toBe(false);
if (!result.ok) expect(result.reason).toMatch(/redirect/i);
// The first hop is the registered address and is expected. Nothing beyond it should have been
// dialled, which is what the refusal is for.
expect(bounced.length).toBe(1);
} finally {
redirector.stop(true);
}
});
test("a redirect to an address the check permits is still followed", async () => {
// A deployment that puts its agent behind a redirect, http to https being the ordinary case, has
// done nothing wrong. Refusing every redirect would break it, so the destination is checked
// rather than the hop count.
const redirector = Bun.serve({
port: 0,
fetch: () =>
new Response(null, { status: 307, headers: { location: url } }),
});
try {
const result = await testAgentConnection(
`http://127.0.0.1:${redirector.port}/ag-ui`,
{ allowPrivateHosts: true, timeoutMs: 8_000 },
);
expect(result.ok).toBe(true);
if (result.ok) expect(result.events).toContain("RUN_STARTED");
} finally {
redirector.stop(true);
}
});
test("a redirect that never arrives anywhere gives up rather than looping", async () => {
const looper = Bun.serve({
port: 0,
fetch: (request) =>
new Response(null, { status: 307, headers: { location: request.url } }),
});
try {
const result = await testAgentConnection(
`http://127.0.0.1:${looper.port}/ag-ui`,
{ allowPrivateHosts: true, timeoutMs: 8_000 },
);
expect(result.ok).toBe(false);
} finally {
looper.stop(true);
}
});
test("a port with nothing on it reports the direction of the connection", async () => {
// The server dials the agent, so localhost must be tested from the server side.
const dead = await testAgentConnection("http://127.0.0.1:9/", {
allowPrivateHosts: true,
timeoutMs: 2_000,
});
expect(dead.ok).toBe(false);
if (!dead.ok) {
expect(dead.reason).toMatch(/reachable from|did not answer/);
}
});
});
/**
* The shapes a real agent turns up in, beyond the happy one.
*
* The connection test is what somebody sees when they register a colleague's agent, and the verdict
* decides whether they go looking at their own service or at ours. A stubbed SSE body proves we parse
* the string we wrote; these put a real AG-UI implementation on the other end and make it misbehave
* in the ways a real one does.
*/
describe("registering an agent that answers badly", () => {
test("an agent that starts a run and never finishes is still reported as working", async () => {
/*
* A run that produces events is a working endpoint. Whether the agent finishes its work is the
* agent's business and not something a registration check should refuse over: the alternative is
* a person unable to register a slow agent, told their perfectly good service is broken.
*/
const partial = new AGUIMock();
partial.onRun(/.*/, [
{ type: "RUN_STARTED", threadId: "t", runId: "r" },
{ type: "TEXT_MESSAGE_START", messageId: "m", role: "assistant" },
] as never);
const partialUrl = await partial.start();
try {
const result = await testAgentConnection(partialUrl, {
allowPrivateHosts: true,
});
expect(result.ok).toBe(true);
if (!result.ok) return;
expect(result.events).toContain("RUN_STARTED");
} finally {
await partial.stop?.();
}
});
test("an agent that reports its own error still counts as speaking the protocol", async () => {
// `RUN_ERROR` is the agent telling us something went wrong, which means it is there and it
// speaks AG-UI. Refusing registration here would send somebody to debug the wrong end.
const failing = new AGUIMock();
failing.onRun(/.*/, [
{ type: "RUN_STARTED", threadId: "t", runId: "r" },
{ type: "RUN_ERROR", message: "the model is unavailable" },
] as never);
const failingUrl = await failing.start();
try {
const result = await testAgentConnection(failingUrl, {
allowPrivateHosts: true,
});
expect(result.ok).toBe(true);
if (!result.ok) return;
expect(result.events).toContain("RUN_ERROR");
} finally {
await failing.stop?.();
}
});
test("a tool call in the stream is reported, because that is what a governed Bot does", async () => {
/*
* The event that matters most for this product. A remote Bot yields a tool call and OpenBot
* decides it, so an endpoint that emits one is exactly the shape the gateway is built for, and
* somebody registering it should see that it did.
*/
const calling = new AGUIMock();
calling.onRun(/.*/, [
{ type: "RUN_STARTED", threadId: "t", runId: "r" },
{
type: "TOOL_CALL_START",
toolCallId: "call_1",
toolCallName: "search_issues",
},
{ type: "TOOL_CALL_ARGS", toolCallId: "call_1", delta: '{"q":"x"}' },
{ type: "TOOL_CALL_END", toolCallId: "call_1" },
{ type: "RUN_FINISHED", threadId: "t", runId: "r" },
] as never);
const callingUrl = await calling.start();
try {
const result = await testAgentConnection(callingUrl, {
allowPrivateHosts: true,
});
expect(result.ok).toBe(true);
if (!result.ok) return;
expect(result.events).toContain("TOOL_CALL_START");
} finally {
await calling.stop?.();
}
});
test("nothing listening is a refusal a person can act on", async () => {
// The commonest registration mistake: a typo, or a service that is not up yet. It has to read as
// "we could not reach it" rather than as a protocol complaint.
const result = await testAgentConnection("http://127.0.0.1:9/ag-ui", {
allowPrivateHosts: true,
});
expect(result.ok).toBe(false);
if (result.ok) return;
expect(result.reason.length).toBeGreaterThan(0);
});
});
/**
* An agent that keeps talking.
*
* The check needs the opening of the stream and says so: the cap beside it is named for how much of
* the answer is enough to prove the far end is an agent. What decides whether that cap means
* anything is whether the read stops there, and the person registering an agent that streams for a
* while is the one who finds out.
*/
describe("registering an agent that streams a long answer", () => {
/** The events, and then a run that goes on writing: a Bot working through a long document. */
function talkativeAgent() {
return Bun.serve({
port: 0,
fetch: () =>
new Response(
new ReadableStream<Uint8Array>({
async start(controller) {
const encoder = new TextEncoder();
controller.enqueue(
encoder.encode(
'event: RUN_STARTED\ndata: {"type":"RUN_STARTED"}\n\n' +
'event: TEXT_MESSAGE_START\ndata: {"type":"TEXT_MESSAGE_START"}\n\n',
),
);
// A kilobyte at a time, slowly, the way a model streams. Nothing here closes: the run
// is still going, which is the whole point.
for (let chunk = 0; chunk < 4_000; chunk += 1) {
controller.enqueue(
encoder.encode(
`event: TEXT_MESSAGE_CONTENT\ndata: {"type":"TEXT_MESSAGE_CONTENT","delta":"${"x".repeat(960)}"}\n\n`,
),
);
await Bun.sleep(20);
}
controller.close();
},
}),
{ headers: { "content-type": "text/event-stream" } },
),
});
}
test("is reported as working, without waiting for it to finish", async () => {
const agent = talkativeAgent();
try {
const started = Date.now();
const result = await testAgentConnection(
`http://127.0.0.1:${agent.port}/ag-ui`,
{ allowPrivateHosts: true, timeoutMs: 3_000 },
);
const took = Date.now() - started;
// It answered, immediately and correctly, with events this check can read. Anything else is
// this deployment describing a working agent as one that may not be reachable.
expect(result.ok).toBe(true);
if (!result.ok) return;
expect(result.events).toContain("RUN_STARTED");
// Well inside the timeout. The check has what it needs after the opening of the stream, and
// an answer that arrives only when the agent stops talking is an answer that depends on the
// agent stopping.
expect(took).toBeLessThan(2_000);
} finally {
agent.stop(true);
}
});
});