Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

10.1 KB
Raw Blame
// Recorded real AG-UI traffic (tests/fixtures/agent-traffic, recorded by scripts/agent-traffic/record.ts from a real
// CopilotKit runtime and real OpenAI agents) turned into what the network capture hands the agent events: a run's
// request (unwrapped from CopilotKit's single-route envelope), each AG-UI event of its SSE response or of its
// Intelligence WebSocket channel, and each stream's end, at the times they arrived. Beside it, what CopilotKit itself
// emitted for the same traffic through its legacy sink.
import { readdirSync, readFileSync } from "node:fs";
import { join } from "node:path";
import type { CapturedEvent } from "@openbot/shared/recorder-masking";
import {
  type AgentEvents,
  createAgentEvents,
} from "../src/background/agent/events";
import type { AgentObservation } from "../src/background/agent/observation";

const ROOT = join(import.meta.dir, "fixtures/agent-traffic");
/** Recorded times are offsets from the start of each conversation; observations carry real epoch times. */
export const CLOCK = 1_792_000_000_000;

type Exchange = {
  at: number;
  url: string;
  body: string | null;
  status: number;
  contentType: string | null;
  chunks: { at: number; text: string }[];
  end: { at: number; outcome: "complete" | "failed" } | null;
};
type Frame = {
  at: number;
  socket: number;
  direction: "in" | "out";
  joinRef: string | null;
  topic: string;
  event: string;
  payload?: unknown;
};
type SinkBatch = { events: CapturedEvent[] };
export type Fixture = {
  name: string;
  transport: "sse" | "intelligence";
  exchanges: Exchange[];
  frames: Frame[];
  sink: SinkBatch[];
  replaySink?: SinkBatch[];
  replayFromAt?: number;
};

export const fixtureNames = (transport: Fixture["transport"]): string[] =>
  readdirSync(join(ROOT, transport))
    .filter((file) => file.endsWith(".json"))
    .map((file) => file.replace(/\.json$/, ""))
    .sort();

export const loadFixture = (
  transport: Fixture["transport"],
  name: string,
): Fixture =>
  JSON.parse(readFileSync(join(ROOT, transport, `${name}.json`), "utf8"));

/** The AG-UI events of an SSE body: each `data:` field of each block, as JSON. */
export function sseEvents(text: string): unknown[] {
  const events: unknown[] = [];
  for (const block of text.split(/\r?\n\r?\n/)) {
    const data = block
      .split(/\r?\n/)
      .filter((line) => line.startsWith("data:"))
      .map((line) => line.slice(5).replace(/^ /, ""))
      .join("\n");
    if (data) events.push(JSON.parse(data));
  }
  return events;
}

const envelope = (body: string | null) => {
  const parsed = JSON.parse(body ?? "{}") as {
    method?: string;
    params?: { agentId?: string };
    body?: unknown;
  };
  return parsed;
};

/**
 * The observations the network capture makes of a fixture's traffic, in arrival order. `between` keeps only traffic that
 * started in [from, until): a recording that began, or ended, part-way through.
 */
export function observationsOf(
  fixture: Fixture,
  between: { from?: number; until?: number } = {},
  tabId = 1,
): AgentObservation[] {
  const { from = 0, until = Number.POSITIVE_INFINITY } = between;
  const within = (at: number) => at >= from && at < until;
  const out: AgentObservation[] = [];
  const agents = new Map<string, string | null>();
  fixture.exchanges.forEach((exchange, index) => {
    if (!within(exchange.at)) return;
    const { method, params, body } = envelope(exchange.body);
    const agentId = params?.agentId ?? null;
    const streamId = `http-${index}`;
    const input = body as { threadId?: string };
    if (input?.threadId) agents.set(input.threadId, agentId);
    out.push({
      kind: "request",
      tabId,
      streamId,
      at: CLOCK + exchange.at,
      url: exchange.url,
      agentId,
      method: method as "agent/run" | "agent/connect",
      input: body,
    });
    if (exchange.contentType?.includes("text/event-stream")) {
      // A chunk can end inside an event: events are cut at block boundaries across chunks, each at the chunk that
      // completed it.
      let buffer = "";
      for (const chunk of exchange.chunks) {
        buffer += chunk.text;
        const cut = buffer.lastIndexOf("\n\n");
        if (cut === -1) continue;
        for (const event of sseEvents(buffer.slice(0, cut)))
          out.push({
            kind: "event",
            tabId,
            streamId,
            at: CLOCK + chunk.at,
            agentId,
            threadId: input?.threadId ?? null,
            event,
          });
        buffer = buffer.slice(cut + 2);
      }
    }
    if (exchange.end)
      out.push({
        kind: "end",
        tabId,
        streamId,
        at: CLOCK + exchange.end.at,
        outcome: exchange.status >= 400 ? "failed" : exchange.end.outcome,
      });
  });
  const channels = new Map<string, string>();
  for (const frame of fixture.frames) {
    const key = `${frame.socket}:${frame.topic}`;
    if (frame.event === "phx_join" && frame.direction === "out") {
      channels.set(key, `ws-${key}:${frame.joinRef}`);
      continue;
    }
    const streamId = channels.get(key);
    if (!streamId || !within(frame.at)) continue;
    const threadId = frame.topic.replace(/^thread:/, "");
    if (frame.event === "ag_ui_event")
      out.push({
        kind: "event",
        tabId,
        streamId,
        at: CLOCK + frame.at,
        agentId: agents.get(threadId) ?? null,
        threadId,
        event: frame.payload,
      });
    else if (frame.event === "phx_leave" || frame.event === "phx_close") {
      out.push({
        kind: "end",
        tabId,
        streamId,
        at: CLOCK + frame.at,
        outcome: "complete",
      });
      channels.delete(key);
    }
  }
  return out.sort((a, b) => a.at - b.at);
}

/** A conversation recorded on its own, as the same tab has it after another: its times moved by `by`, its streams its
 * own. */
export const later = (
  observations: AgentObservation[],
  by: number,
  streams: string,
): AgentObservation[] =>
  observations.map((observation) => ({
    ...observation,
    at: observation.at + by,
    streamId: `${streams}:${observation.streamId}`,
  }));

/** Two conversations of one agent, each recorded on its own, as one page has them when its chat moves from the first's
 * Thread to a new one (the second's): the second's first request comes at `at`, an epoch time after the first ended. */
export function movedTo(
  first: Fixture,
  second: Fixture,
  at: number,
): AgentObservation[] {
  const next = observationsOf(second);
  const asked = next.find(({ kind }) => kind === "request")?.at ?? at;
  return [...observationsOf(first), ...later(next, at - asked, "moved")].sort(
    (a, b) => a.at - b.at,
  );
}

/** End-to-end runs 1 and 2 on CopilotKit's learning deal page (fixtures/e2e-new-thread.json): what CopilotKit's own
 * legacy sink stored as the person moved the chat to a new Thread, the edit step the Recorder captured as they typed the
 * question for it, and when the page asked for the run on it. */
export type NewThreadRun = {
  name: string;
  copilotKit: CapturedEvent[];
  typed: CapturedEvent;
  secondRunRequestAt: number;
};

export const newThreadRuns = (): NewThreadRun[] =>
  (
    JSON.parse(
      readFileSync(
        join(import.meta.dir, "fixtures/e2e-new-thread.json"),
        "utf8",
      ),
    ) as { runs: NewThreadRun[] }
  ).runs;

/** The events CopilotKit emitted, in order, with the collector's own sequence number left out. */
export function sinkEvents(batches: SinkBatch[]): CapturedEvent[] {
  return batches.flatMap((batch) => batch.events);
}

/** The Thread of a conversation's first run, as CopilotKit named it. */
export function firstThreadOf(fixture: Fixture): string {
  const run = sinkEvents(fixture.sink).find(
    (event) => event.name === "agent.run",
  );
  const threadId = (run?.value as { threadId?: unknown } | undefined)?.threadId;
  if (typeof threadId !== "string") throw new Error(`${fixture.name}: no run`);
  return threadId;
}

/** CopilotKit's agent rows: `agent.run`, `agent.message` and `tool.call`, a frontend tool's executing and completed among
 * them. (`thread.linked` is CopilotKit's only where a chat opened the Thread: agent-events-parity.test.ts compares it on
 * the conversation that had one.) */
export const observable = (event: CapturedEvent): boolean =>
  ["agent.run", "agent.message", "tool.call"].includes(event.name);

export const withoutSeq = (event: CapturedEvent) => {
  const { seq: _seq, ...value } = event.value as Record<string, unknown>;
  return { name: event.name, value };
};

/** A row without how long a frontend tool ran: the page times the tool itself, the network sees the gap from the run's
 * end to the request that carries the tool's result (agent-events-parity.test.ts holds that gap). */
export const withoutDuration = (event: {
  name: string;
  value: Record<string, unknown>;
}) => {
  const { durationMs: _duration, ...value } = event.value;
  return { name: event.name, value };
};

/** Lets the library apply everything fed so far: its subscribers run in promise jobs. */
export const settle = async () => {
  for (let round = 0; round < 5; round++)
    await new Promise((resolve) => setTimeout(resolve, 0));
};

/** Hands observations to the agent events as the service worker does: each in a task of its own. */
export async function feed(
  agentEvents: AgentEvents,
  observations: AgentObservation[],
): Promise<void> {
  for (const observation of observations) {
    agentEvents.observe(observation);
    await new Promise((resolve) => setImmediate(resolve));
  }
  await settle();
}

/** New agent events, and what they emitted. */
export function agentEventsFor(digests: ReadonlySet<string> = new Set()) {
  const emitted: CapturedEvent[] = [];
  const agentEvents = createAgentEvents({
    emit: (_tabId, event) => emitted.push(event),
    digests: () => digests,
    now: () => CLOCK,
  });
  return { emitted, agentEvents };
}

/** Runs the agent events over observations, and returns what they emitted. */
export async function record(
  observations: AgentObservation[],
  digests?: ReadonlySet<string>,
) {
  const { emitted, agentEvents } = agentEventsFor(digests);
  await feed(agentEvents, observations);
  return { emitted, agentEvents };
}