Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

22.3 KB
Raw Blame
import { describe, expect, test } from "bun:test";
import { readRunAssertion } from "../src/agents/callback-token";
import { ThreadBusyError } from "../src/agents/handoff-delivery";
import {
  createHandoffRunner,
  type HandoffWork,
} from "../src/agents/handoff-runner";
import { signHandoffDeliveryRun } from "../src/agents/handoff-signing";
import type { AuditStore } from "../src/audit";
import type { WorkItem, WorkQueue } from "../src/work/queue";

/**
 * Delivering a hop, and the three ways it must not go wrong.
 *
 * Running the other Bot twice for one hop. Finishing work that is no longer this replica's. And
 * letting a lease lapse in the middle of a run, which is the same as the first with extra steps.
 */

const KEY = "test-encryption-key-not-a-real-one";

const WORK: HandoffWork = {
  fromBotId: "assistant",
  toBotId: "researcher",
  actorId: "user-1",
  threadId: "thread-1",
  runId: "run-1",
  depth: 1,
  task: "find the outage window",
  expecting: "a date range",
};

function runner(options?: {
  claimed?: WorkItem[];
  deliver?: (input: {
    work: HandoffWork;
    message: string;
    shown?: string;
  }) => Promise<void>;
  /** What the delivery says the Bot answered. Null — nothing worth relaying — unless a test cares. */
  answer?: string | null;
  onSettled?: () => Promise<void>;
}) {
  const calls: Array<{ verb: string; key: string; owner?: string }> = [];
  const events: string[] = [];
  const written: Array<{
    eventType: string;
    payload: Record<string, unknown>;
  }> = [];
  const delivered: Array<{ message: string; assertion: string }> = [];
  const offered: HandoffWork[] = [];
  const delays: number[] = [];
  /** Each time the asking conversation was told the queue moved, with what the queue had seen by then. */
  const settled: Array<{ threadId: string; after: string[] }> = [];

  const queue = {
    claim: async () =>
      options?.claimed ?? [
        { kind: "bot.message", key: "run-1:abc", payload: WORK, attempts: 1 },
      ],
    renew: async () => true,
    finish: async ({ key, owner }: { key: string; owner: string }) => {
      calls.push({ verb: "finish", key, owner });
      return true;
    },
    release: async ({
      key,
      owner,
      delayMs,
    }: {
      key: string;
      owner: string;
      delayMs: number;
    }) => {
      calls.push({ verb: "release", key, owner });
      delays.push(delayMs);
      return true;
    },
    offer: async ({
      key,
      payload,
    }: {
      key: string;
      payload?: Record<string, unknown>;
    }) => {
      calls.push({ verb: "offer", key });
      offered.push(payload as unknown as HandoffWork);
    },
  } as unknown as WorkQueue;

  const auditStore: AuditStore = {
    insert: async (event) => {
      events.push(event.eventType);
      written.push({
        eventType: event.eventType,
        payload: (event.payload ?? {}) as Record<string, unknown>,
      });
    },
  };

  return {
    calls,
    events,
    written,
    delivered,
    offered,
    delays,
    settled,
    runner: createHandoffRunner({
      queue,
      owner: "replica-a",
      sign: (work, claim) =>
        signHandoffDeliveryRun(work, KEY, "delivery-run", claim),
      auditStore,
      onSettled: async (work) => {
        settled.push({
          threadId: work.threadId,
          after: calls.map((call) => call.verb),
        });
        await options?.onSettled?.();
      },
      delivery: {
        deliver: async ({ work, message, shown, assertion }) => {
          delivered.push({ message, assertion });
          await options?.deliver?.({ work, message, shown });
          return { answer: options?.answer ?? null };
        },
      },
    }),
  };
}

describe("delivering a hop", () => {
  test("runs the addressed Bot and finishes the work as its owner", async () => {
    const { runner: sweep, calls, delivered } = runner();

    const report = await sweep.sweep();

    expect(report.delivered).toEqual(["researcher"]);
    expect(calls).toEqual([
      { verb: "finish", key: "run-1:abc", owner: "replica-a" },
    ]);
    expect(delivered).toHaveLength(1);
  });

  /*
   * Who is asking is stamped by the deployment, from the row it wrote. A Bot able to write its own
   * attribution is a Bot able to claim to be another one.
   */
  test("the addressed Bot is told who asked, and what for, in parts", async () => {
    const { runner: sweep, delivered } = runner();

    await sweep.sweep();

    const message = delivered[0]?.message ?? "";
    expect(message).toContain("assistant");
    expect(message).toContain("Task: find the outage window");
    // The parts stay parts: the asking model was made to name them so this one need not infer them.
    expect(message).toContain("What a good answer looks like: a date range");
  });

  test("the run it starts carries the depth this hop reached", async () => {
    const { runner: sweep, delivered } = runner();

    await sweep.sweep();

    expect(readRunAssertion(delivered[0]?.assertion, KEY)).toMatchObject({
      botId: "researcher",
      depth: 1,
      handoff: { key: "run-1:abc", owner: "replica-a" },
    });
  });

  test("the delivery assertion preserves the run initiator across the queue", async () => {
    const { runner: sweep, delivered } = runner({
      claimed: [
        {
          kind: "bot.message",
          key: "run-1:abc",
          payload: {
            ...WORK,
            initiator: { kind: "routine", id: "routine_7" },
          },
          attempts: 1,
        },
      ],
    });

    await sweep.sweep();

    expect(readRunAssertion(delivered[0]?.assertion, KEY)).toMatchObject({
      botId: "researcher",
      actorId: "user-1",
      threadId: "thread-1",
      depth: 1,
      initiator: { kind: "routine", id: "routine_7" },
    });
  });

  test("a legacy queued hop without an initiator is still read as a person's", async () => {
    const { runner: sweep, delivered } = runner();

    await sweep.sweep();

    expect(readRunAssertion(delivered[0]?.assertion, KEY)?.initiator).toEqual({
      kind: "person",
    });
  });

  /*
   * A Bot that answered has answered, whatever it said. Retrying a delivery because the answer was
   * unhelpful would ask it the same question again and bill for the same non-answer.
   */
  test("a delivery that fails is released rather than finished", async () => {
    const {
      runner: sweep,
      calls,
      events,
    } = runner({
      deliver: async () => {
        throw new Error("the gateway was unreachable");
      },
    });

    const report = await sweep.sweep();

    expect(report.delivered).toEqual([]);
    expect(calls).toEqual([
      { verb: "release", key: "run-1:abc", owner: "replica-a" },
    ]);
    expect(events).toContain("agent.handoff_failed");
  });

  test("an answer that finds the asking Bot still talking is retried in seconds, then a minute", async () => {
    const first = runner({
      deliver: async () => {
        throw new ThreadBusyError("thread-1 is busy with another run");
      },
    });
    await first.runner.sweep();
    expect(first.delays).toEqual([5_000]);

    const later = runner({
      claimed: [
        { kind: "bot.message", key: "run-1:abc", payload: WORK, attempts: 2 },
      ] as unknown as WorkItem[],
      deliver: async () => {
        throw new ThreadBusyError("thread-1 is busy with another run");
      },
    });
    await later.runner.sweep();
    expect(later.delays).toEqual([60_000]);

    const other = runner({
      deliver: async () => {
        throw new Error("the gateway was unreachable");
      },
    });
    await other.runner.sweep();
    expect(other.delays).toEqual([60_000]);
  });

  /*
   * A second attempt may already have run that Bot, spent a model call and posted an answer before
   * its owner died. Somebody reading two similar answers should be able to tell which happened.
   */
  test("a second attempt says so, before it runs anything", async () => {
    const { runner: sweep, events } = runner({
      claimed: [
        { kind: "bot.message", key: "run-1:abc", payload: WORK, attempts: 2 },
      ],
    });

    await sweep.sweep();

    expect(events[0]).toBe("agent.handoff_retried");
    expect(events).toContain("agent.handoff_delivered");
  });

  /*
   * The Audit screen's Bot column reads `payload.bot` and renders a dash without it, so a delivery
   * that names the Bot only under `from` is a row saying a handoff happened and not who did it.
   * Its sibling `agent.handoff_offered` is asserted the same way in `agent-handoff.test.ts`.
   */
  test("a delivery names the Bot that handed the work over", async () => {
    const { runner: sweep, written } = runner();

    await sweep.sweep();

    const delivery = written.find(
      (event) => event.eventType === "agent.handoff_delivered",
    );
    expect(delivery?.payload).toMatchObject({
      bot: WORK.fromBotId,
      from: WORK.fromBotId,
      to: WORK.toBotId,
    });
  });

  /*
   * And so do the rows either side of it, which is the half the assertion above did not reach.
   *
   * A delivery is the outcome that is also visible in the transcript. A hop that was retried or
   * that failed is visible nowhere else at all, so those are the rows somebody actually comes to
   * this screen for — and they were the ones rendering a dash where the Bot's name belongs.
   */
  test("a hop that was retried or that failed names the Bot too", async () => {
    const retried = runner({
      claimed: [
        { kind: "bot.message", key: "run-1:abc", payload: WORK, attempts: 2 },
      ],
    });
    await retried.runner.sweep();

    expect(
      retried.written.find(
        (event) => event.eventType === "agent.handoff_retried",
      )?.payload,
    ).toMatchObject({ bot: WORK.fromBotId, from: WORK.fromBotId });

    const failed = runner({
      deliver: async () => {
        throw new Error("the gateway was unreachable");
      },
    });
    await failed.runner.sweep();

    expect(
      failed.written.find((event) => event.eventType === "agent.handoff_failed")
        ?.payload,
    ).toMatchObject({ bot: WORK.fromBotId, from: WORK.fromBotId });
  });

  /* Releasing an unusable row would put it back on the queue for ever. */
  test("a row that is not a hop is finished rather than released", async () => {
    const { runner: sweep, calls } = runner({
      claimed: [
        { kind: "bot.message", key: "run-1:junk", payload: {}, attempts: 1 },
      ],
    });

    const report = await sweep.sweep();

    expect(report.skipped).toEqual([
      { key: "run-1:junk", reason: "not a hop" },
    ]);
    expect(calls).toEqual([
      { verb: "finish", key: "run-1:junk", owner: "replica-a" },
    ]);
  });
});

/**
 * A hop that will not be tried again.
 *
 * The person was told their question had been handed on. If nothing ever comes back and nothing ever
 * says so, they cannot tell a slow Bot from a broken one, and the conversation simply stops.
 */
describe("a hop that failed for good", () => {
  test("the Bot that asked is sent back to tell the person", async () => {
    const { runner: sweeper, offered } = runner({
      claimed: [
        {
          kind: "bot.message",
          key: "run-1:abc",
          payload: { ...WORK, initiator: { kind: "routine", id: "routine_7" } },
          attempts: 5,
        },
      ] as unknown as WorkItem[],
      deliver: async () => {
        throw new Error("researcher did not finish within 300s");
      },
    });

    await sweeper.sweep();

    expect(offered).toHaveLength(1);
    // Back to the Bot that asked, in the conversation the person is watching.
    expect(offered[0]).toMatchObject({
      fromBotId: "researcher",
      toBotId: "assistant",
      answerIn: "thread-1",
      threadId: "thread-1",
      initiator: { kind: "routine", id: "routine_7" },
    });
    expect(offered[0]?.task).toContain("did not finish within 300s");
  });

  test("a hop the deployment's controls refuse ends on its first try, and the person is told", async () => {
    const {
      runner: sweeper,
      offered,
      calls,
    } = runner({
      deliver: async () => {
        const error = new Error("Use Bots is turned off for you.");
        error.name = "CapabilityRefusedError";
        throw error;
      },
    });

    await sweeper.sweep();

    // Retrying cannot change the answer, so the work ends now rather than five minutes from now.
    expect(calls.filter((call) => call.verb === "release")).toEqual([]);
    expect(calls).toContainEqual({
      verb: "finish",
      key: "run-1:abc",
      owner: "replica-a",
    });
    expect(offered).toHaveLength(1);
    expect(offered[0]?.task).toContain("Use Bots is turned off");
  });

  test("a hop to an address this deployment will not dial ends on its first try", async () => {
    const {
      runner: sweeper,
      offered,
      calls,
    } = runner({
      // As it arrives: the AG-UI client rebuilds the refusal from its RUN_ERROR, so only the
      // message survives.
      deliver: async () => {
        throw new Error(
          "This deployment will not dial http://localhost:4201/ag-ui: that address is inside OpenBot's own network, so an agent may not live there.",
        );
      },
    });

    await sweeper.sweep();

    expect(calls.filter((call) => call.verb === "release")).toEqual([]);
    expect(calls).toContainEqual({
      verb: "finish",
      key: "run-1:abc",
      owner: "replica-a",
    });
    expect(offered).toHaveLength(1);
    expect(offered[0]?.task).toContain("will not dial");
  });

  /*
   * The fan-out cap counts every row whose key starts with the run's own prefix. A notice is not one
   * of the Bots this run asked for, and a run long enough to see a hop fail for good is exactly the
   * run that still has asking to do.
   */
  test("its key is outside the run's own prefix, so it costs no fan-out budget", async () => {
    const {
      runner: sweeper,
      calls,
      offered,
    } = runner({
      claimed: [
        { kind: "bot.message", key: "run-1:abc", payload: WORK, attempts: 5 },
      ] as unknown as WorkItem[],
      deliver: async () => {
        throw new Error("nope");
      },
    });

    await sweeper.sweep();

    const key = calls.find((call) => call.verb === "offer")?.key ?? "";
    expect(key.startsWith("run-1:")).toBe(false);
    expect(key).toContain("run-1:abc");
    expect(offered).toHaveLength(1);
  });

  /*
   * One run may legally ask the same Bot two different things. Keyed on the Bot alone both notices
   * are the same work to `offer`, the second is dropped on conflict, and nothing purges this kind —
   * so the person hears about one of their two lost questions, for good.
   */
  test("two lost questions to one Bot leave two notices", async () => {
    const { runner: sweeper, calls } = runner({
      claimed: [
        { kind: "bot.message", key: "run-1:aaa", payload: WORK, attempts: 5 },
        { kind: "bot.message", key: "run-1:bbb", payload: WORK, attempts: 5 },
      ] as unknown as WorkItem[],
      deliver: async () => {
        throw new Error("nope");
      },
    });

    await sweeper.sweep();

    const keys = calls
      .filter((call) => call.verb === "offer")
      .map((call) => call.key);
    expect(keys).toHaveLength(2);
    expect(new Set(keys).size).toBe(2);
  });

  /*
   * Otherwise a Bot nobody can reach produces a notice that cannot be delivered either, which
   * produces a notice, for ever.
   */
  test("a notice that fails is not itself noticed", async () => {
    const { runner: sweeper, offered } = runner({
      claimed: [
        {
          kind: "bot.message",
          key: "run-1:notice:researcher",
          payload: { ...WORK, answerIn: "thread-1" },
          attempts: 5,
        },
      ] as unknown as WorkItem[],
      deliver: async () => {
        throw new Error("nope");
      },
    });

    await sweeper.sweep();

    expect(offered).toEqual([]);
  });

  test("a hop with tries left is simply released", async () => {
    const {
      runner: sweeper,
      offered,
      calls,
    } = runner({
      claimed: [
        { kind: "bot.message", key: "run-1:abc", payload: WORK, attempts: 2 },
      ] as unknown as WorkItem[],
      deliver: async () => {
        throw new Error("busy");
      },
    });

    await sweeper.sweep();

    expect(offered).toEqual([]);
    expect(calls.map((call) => call.verb)).toContain("release");
  });
});

/**
 * The answer coming home.
 *
 * The addressed Bot's turn runs in a scratch thread nobody is shown, so its words reach the person
 * one way: a backwards hop that runs the asking Bot, in the conversation being watched, with the
 * answer in its prompt. Attributed by the deployment, in the asking Bot's voice — the only voice
 * that thread admits.
 */
describe("relaying the answer home", () => {
  test("a delivered hop sends the answer back through the Bot that asked", async () => {
    const { runner: sweeper, offered } = runner({
      claimed: [
        {
          kind: "bot.message",
          key: "run-1:abc",
          payload: { ...WORK, initiator: { kind: "routine", id: "routine_7" } },
          attempts: 1,
        },
      ],
      answer: "The outage was Tuesday, 02:10 to 02:45.",
    });

    await sweeper.sweep();

    expect(offered).toHaveLength(1);
    expect(offered[0]).toMatchObject({
      fromBotId: "researcher",
      toBotId: "assistant",
      answerIn: "thread-1",
      threadId: "thread-1",
      depth: 1,
      initiator: { kind: "routine", id: "routine_7" },
    });
    expect(offered[0]?.task).toContain("find the outage window");
    expect(offered[0]?.task).toContain(
      "The outage was Tuesday, 02:10 to 02:45.",
    );
  });

  test("its key is outside the run's own prefix, like the notice", async () => {
    const { runner: sweeper, calls } = runner({ answer: "Tuesday." });

    await sweeper.sweep();

    const key = calls.find((call) => call.verb === "offer")?.key ?? "";
    expect(key.startsWith("run-1:")).toBe(false);
    expect(key).toContain("run-1:abc");
  });

  /* A relay of a relay is a loop. The `answerIn` marker that stops a notice stops this too. */
  test("a relay is not itself relayed", async () => {
    const { runner: sweeper, offered } = runner({
      claimed: [
        {
          kind: "bot.message",
          key: "relay:run-1:abc",
          payload: { ...WORK, answerIn: "thread-1" },
          attempts: 1,
        },
      ] as unknown as WorkItem[],
      answer: "Understood, telling them now.",
    });

    await sweeper.sweep();

    expect(offered).toEqual([]);
  });

  test("a turn that said nothing sends nothing home", async () => {
    const { runner: sweeper, offered } = runner({ answer: null });

    await sweeper.sweep();

    expect(offered).toEqual([]);
  });

  /*
   * The answer rides inside the relaying run's prompt, and a Bot that came back with a book would
   * spend that run's whole context window repeating it.
   */
  test("an answer the length of a book is clipped, and says so", async () => {
    const { runner: sweeper, offered } = runner({
      answer: "x".repeat(20_000),
    });

    await sweeper.sweep();

    expect(offered[0]?.task).toContain("[…the answer was cut here for length]");
    expect((offered[0]?.task ?? "").length).toBeLessThan(14_000);
  });

  test("an answer cut inside a character loses the whole character, not half of it", async () => {
    // 12,000 code units are kept, and an emoji is two: its high half on the last kept unit would
    // reach the relaying run's prompt as a lone surrogate.
    const { runner: sweeper, offered } = runner({
      answer: `${"x".repeat(11_999)}😀tail`,
    });

    await sweeper.sweep();

    expect(offered[0]?.task).toContain(
      `${"x".repeat(11_999)}\n\n[…the answer was cut here for length]`,
    );
  });
});

/**
 * What a notice leaves in the transcript.
 *
 * Nothing. The asking Bot's own sentence is the whole message; the text that prompted it is an
 * instruction to a model, and kept it appears as something the person typed and had read back.
 */
describe("what a notice shows", () => {
  test("the instruction that produced it is not shown to anybody", async () => {
    const shownTexts: Array<string | undefined> = [];
    const { runner: sweeper } = runner({
      claimed: [
        {
          kind: "bot.message",
          key: "run-1:notice:researcher",
          payload: { ...WORK, answerIn: "thread-1" },
          attempts: 1,
        },
      ] as unknown as WorkItem[],
      deliver: async (input) => {
        shownTexts.push(input.shown);
      },
    });

    await sweeper.sweep();

    expect(shownTexts).toEqual([undefined]);
  });

  test("an ordinary hop shows who asked and what for", async () => {
    const shownTexts: Array<string | undefined> = [];
    const { runner: sweeper } = runner({
      claimed: [
        {
          kind: "bot.message",
          key: "run-1:abc",
          payload: { ...WORK, fromName: "Assistant", toName: "Researcher" },
          attempts: 1,
        },
      ] as unknown as WorkItem[],
      deliver: async (input) => {
        shownTexts.push(input.shown);
      },
    });

    await sweeper.sweep();

    expect(shownTexts[0]).toBe(
      "Assistant asked Researcher for this on your behalf: find the outage window",
    );
  });
});

/*
 * The asking conversation reads what it is waiting on from the queue, so it is told after the queue
 * moved and not before. Told from inside the delivery, the row would still say the hop was in
 * flight, and "Asking Ana…" would stay on screen for a hop that had finished.
 */
describe("telling the asking conversation", () => {
  test("after a delivered hop is finished and its answer is on its way back", async () => {
    const { runner: sweep, settled } = runner({
      answer: "Tuesday 02:00 to 04:00.",
    });

    await sweep.sweep();

    expect(settled).toEqual([
      { threadId: WORK.threadId, after: ["finish", "offer"] },
    ]);
  });

  test("after a failed delivery is released for another try", async () => {
    const { runner: sweep, settled } = runner({
      deliver: async () => {
        throw new Error("the model stopped");
      },
    });

    await sweep.sweep();

    expect(settled).toEqual([{ threadId: WORK.threadId, after: ["release"] }]);
  });

  test("a conversation that cannot be told does not fail the hop", async () => {
    const { runner: sweep, calls } = runner({
      onSettled: async () => {
        throw new Error("the socket is down");
      },
    });

    const report = await sweep.sweep();

    expect(report.delivered).toEqual(["researcher"]);
    expect(calls).toEqual([
      { verb: "finish", key: "run-1:abc", owner: "replica-a" },
    ]);
  });
});