Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

7.7 KB
Raw Blame
import { describe, expect, test } from "bun:test";
import {
  type StalledStream,
  TurnWatchdog,
} from "../src/channels/turn-watchdog";

/**
 * These test the distinction the watchdog exists to draw, not the plumbing around it.
 *
 * Every case is one a deployment can actually be in: a Bot that answers slowly and is fine, a Bot
 * that answered once and then stopped, a Bot that accepted the connection and never wrote a byte.
 * Time is injected rather than waited for, so a two-minute timeout is tested in no time at all and
 * the assertions are about the rule rather than about how fast the machine running them is.
 *
 * There is no exemption for a frontend tool call and therefore no test of one. The run ends before
 * the browser executes a tool and a second run carries the result back, so no stream is ever held
 * open across one; see the module comment on turn-watchdog.ts for what was measured.
 */

/** A clock a test moves by hand. */
function clock(): { now: () => number; advance: (ms: number) => void } {
  let current = 0;
  return {
    now: () => current,
    advance: (ms: number) => {
      current += ms;
    },
  };
}

function watching(stallMs: number) {
  const time = clock();
  const stalled: StalledStream[] = [];
  const watchdog = new TurnWatchdog({
    stallMs,
    now: time.now,
    onStall: (stream) => stalled.push(stream),
  });
  return { time, stalled, watchdog };
}

describe("a turn is judged on activity, not on how long it has run", () => {
  test("a stream that has said nothing for the timeout is given up on", () => {
    const { time, stalled, watchdog } = watching(60_000);
    watchdog.open({ id: "stream-1", botId: "risk-analyst" });

    time.advance(59_999);
    expect(watchdog.sweep()).toBe(0);

    time.advance(1);
    expect(watchdog.sweep()).toBe(1);
    expect(stalled).toHaveLength(1);
    expect(stalled[0]?.botId).toBe("risk-analyst");
    expect(stalled[0]?.silentForMs).toBe(60_000);
    // Zero chunks is the fact worth reading: the Bot answered the request and then said nothing.
    expect(stalled[0]?.chunks).toBe(0);
  });

  test("a stream that keeps talking is never given up on, however long it runs", () => {
    const { time, stalled, watchdog } = watching(60_000);
    watchdog.open({ id: "stream-1", botId: "risk-analyst" });

    // An hour of work, a chunk every half minute. A duration limit would have ended this repeatedly.
    for (let minute = 0; minute < 120; minute++) {
      time.advance(30_000);
      watchdog.record("stream-1");
      expect(watchdog.sweep()).toBe(0);
    }

    expect(stalled).toHaveLength(0);
    expect(watchdog.watching).toBe(1);
  });

  test("a stream that spoke and then stopped is given up on from its last chunk", () => {
    const { time, stalled, watchdog } = watching(60_000);
    watchdog.open({ id: "stream-1", botId: "risk-analyst" });

    time.advance(50_000);
    watchdog.record("stream-1");
    watchdog.record("stream-1");
    time.advance(50_000);
    // A hundred seconds since it opened, fifty since it last said anything. Silence is what counts.
    expect(watchdog.sweep()).toBe(0);

    time.advance(10_000);
    expect(watchdog.sweep()).toBe(1);
    expect(stalled[0]?.silentForMs).toBe(60_000);
    expect(stalled[0]?.chunks).toBe(2);
  });

  test("silence while the relay's reader has not taken delivery is not charged to the Bot", () => {
    const { time, stalled, watchdog } = watching(60_000);
    watchdog.open({ id: "stream-1", botId: "risk-analyst" });

    // The Bot spoke and this process is now holding that chunk out to whoever reads the relayed
    // stream. In this deployment that is the Intelligence runner, publishing over the network.
    watchdog.record("stream-1");
    watchdog.pause("stream-1");
    time.advance(600_000);
    expect(watchdog.sweep()).toBe(0);
    expect(stalled).toHaveLength(0);

    // Ten minutes of it, and the Bot still gets its whole timeout from the moment it is waited on
    // again. The wait that just ended was not its.
    watchdog.resume("stream-1");
    time.advance(59_999);
    expect(watchdog.sweep()).toBe(0);

    time.advance(1);
    expect(watchdog.sweep()).toBe(1);
    expect(stalled[0]?.silentForMs).toBe(60_000);
    expect(stalled[0]?.chunks).toBe(1);
  });

  test("pausing or resuming a stream nobody is watching is harmless", () => {
    const { watchdog } = watching(60_000);
    expect(() => watchdog.pause("never-opened")).not.toThrow();
    expect(() => watchdog.resume("never-opened")).not.toThrow();
    expect(watchdog.watching).toBe(0);
  });

  test("a stream that ends is forgotten and is never reported", () => {
    const { time, stalled, watchdog } = watching(60_000);
    watchdog.open({ id: "stream-1", botId: "risk-analyst" });
    watchdog.close("stream-1");

    time.advance(600_000);
    expect(watchdog.sweep()).toBe(0);
    expect(stalled).toHaveLength(0);
    expect(watchdog.watching).toBe(0);
  });

  test("closing a stream that was never watched is harmless", () => {
    const { watchdog } = watching(60_000);
    expect(() => watchdog.close("never-opened")).not.toThrow();
    expect(watchdog.watching).toBe(0);
  });
});

describe("a stalled stream is reported exactly once", () => {
  test("sweeping again finds nothing, because the entry went with the callback", () => {
    const { time, stalled, watchdog } = watching(1_000);
    watchdog.open({ id: "stream-1", botId: "risk-analyst" });

    time.advance(5_000);
    expect(watchdog.sweep()).toBe(1);
    expect(watchdog.sweep()).toBe(0);
    expect(watchdog.sweep()).toBe(0);
    expect(stalled).toHaveLength(1);
  });

  test("a late chunk from a Bot already given up on does not start a second watch", () => {
    const { time, stalled, watchdog } = watching(1_000);
    watchdog.open({ id: "stream-1", botId: "risk-analyst" });

    time.advance(2_000);
    expect(watchdog.sweep()).toBe(1);

    // The endpoint wakes up and writes. There is no turn left to end, and reviving it here would
    // stall again and report the same wedged turn twice.
    watchdog.record("stream-1");
    time.advance(60_000);
    expect(watchdog.sweep()).toBe(0);
    expect(stalled).toHaveLength(1);
  });

  test("one wedged Bot's callback throwing does not spare the others", () => {
    const time = clock();
    const reported: string[] = [];
    const watchdog = new TurnWatchdog({
      stallMs: 1_000,
      now: time.now,
      onStall: (stream) => {
        reported.push(stream.id);
        if (stream.id === "stream-1") throw new Error("the surface blew up");
      },
    });

    watchdog.open({ id: "stream-1", botId: "risk-analyst" });
    watchdog.open({ id: "stream-2", botId: "general-assistant" });
    time.advance(2_000);

    expect(watchdog.sweep()).toBe(2);
    expect(reported).toEqual(["stream-1", "stream-2"]);
    expect(watchdog.watching).toBe(0);
  });
});

describe("a deployment that has not asked for a watchdog does not get one", () => {
  test("zero watches nothing, whatever is opened", () => {
    const { time, stalled, watchdog } = watching(0);
    watchdog.open({ id: "stream-1", botId: "risk-analyst" });

    expect(watchdog.enabled).toBe(false);
    expect(watchdog.watching).toBe(0);

    time.advance(3_600_000);
    expect(watchdog.sweep()).toBe(0);
    expect(stalled).toHaveLength(0);
  });

  test("a positive timeout is what switches it on", () => {
    const { watchdog } = watching(1);
    expect(watchdog.enabled).toBe(true);
  });
});

describe("opening a stream twice", () => {
  test("restarts its clock rather than raising", () => {
    const { time, stalled, watchdog } = watching(1_000);
    watchdog.open({ id: "stream-1", botId: "risk-analyst" });

    time.advance(900);
    watchdog.open({ id: "stream-1", botId: "risk-analyst" });
    time.advance(900);

    expect(watchdog.sweep()).toBe(0);
    expect(watchdog.watching).toBe(1);
    expect(stalled).toHaveLength(0);
  });
});