Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

13.2 KB
Raw Blame
import { describe, expect, test } from "bun:test";
import type { TurnRunner } from "@openbot/toolkit/run-turn";
import type { AgentActor } from "../src/agents/profile-types";
import type {
  AgentChannel,
  ChannelActivity,
  ChannelStore,
} from "../src/channels/routes";
import { createRoutineRunner } from "../src/routines/runner";
import type {
  RoutineRunContext,
  RoutineRunOutcome,
  RoutineStore,
} from "../src/routines/store";

/**
 * A routine's turn with nobody's browser open, asserted without a database and without a model.
 *
 * The runner is the one place a headless firing does what a browser normally does — record the
 * reply as channel activity — so what is under test is the sequence and the counts: exactly one
 * activity per successful turn, exactly one run row closed, and the fatigue rule speaking once and
 * then shutting up. `runTurn` is injected, which is why nothing here dials a model, and the store
 * halves are recording stubs, which is why nothing here needs Postgres.
 *
 * The person-facing half of RoutineStore throws on contact: the runner acts as the owner but is not
 * the owner asking a question, and reaching for `create` or `listFor` from here would be a bug that
 * a test asserting only outcomes would not see.
 */

const OWNER: AgentActor = { id: "user_owner", role: "user" };

const CONTEXT: RoutineRunContext = {
  routineId: "routine_1",
  ownerUserId: "user_owner",
  agentId: "bot_helper",
  channelId: "channel_1",
  instruction: "Post the standup summary.",
};

const CHANNEL: AgentChannel = {
  id: "channel_1",
  name: "Standup",
  agentIds: ["bot_helper"],
  threadId: "thread_owner_channel_1",
  active: true,
};

const RUN_ID = "routine_run_1";

type Recorded = {
  finished: { runId: string; status: RoutineRunOutcome; error?: string }[];
  activity: {
    actor: AgentActor;
    channelId: string;
    activity: ChannelActivity;
  }[];
  enabled: { ownerUserId: string; id: string; enabled: boolean }[];
  turns: Parameters<TurnRunner>[0][];
  /** Each busy signal, with how many run rows had been closed when it was sent. */
  busy: { threadId: string; busy: boolean; finishedBefore: number }[];
};

function unreachable(method: string): never {
  throw new Error(`the runner must not call ${method}`);
}

function harness(options: {
  context?: RoutineRunContext | null;
  channel?: AgentChannel | null;
  failures?: number;
  runTurn?: TurnRunner;
  recordActivity?: () => Promise<void>;
  signalBusy?: () => Promise<void>;
}) {
  const recorded: Recorded = {
    finished: [],
    activity: [],
    enabled: [],
    turns: [],
    busy: [],
  };

  const routineStore: RoutineStore = {
    create: () => unreachable("create"),
    listFor: () => unreachable("listFor"),
    update: () => unreachable("update"),
    remove: () => unreachable("remove"),
    dueRoutines: () => unreachable("dueRoutines"),
    advanceNextRun: () => unreachable("advanceNextRun"),
    insertRun: () => unreachable("insertRun"),

    async runContext(runId) {
      expect(runId).toBe(RUN_ID);
      return options.context === undefined ? CONTEXT : options.context;
    },
    async finishRun(runId, status, error) {
      recorded.finished.push({ runId, status, error });
    },
    async consecutiveFailures(routineId) {
      expect(routineId).toBe(CONTEXT.routineId);
      // Read AFTER the failure is on the row, or the tenth failure reads as the ninth: hoisting
      // this read above `finishRun` in the runner would keep every other assertion here green.
      expect(recorded.finished).toHaveLength(1);
      expect(recorded.finished[0]).toMatchObject({
        runId: RUN_ID,
        status: "failed",
      });
      return options.failures ?? 0;
    },
    async setEnabled(ownerUserId, id, enabled) {
      recorded.enabled.push({ ownerUserId, id, enabled });
    },
  };

  const channelStore: ChannelStore = {
    create: () => unreachable("channels.create"),
    list: () => unreachable("channels.list"),
    setPinned: () => unreachable("channels.setPinned"),
    markRead: () => unreachable("channels.markRead"),
    softDelete: () => unreachable("channels.softDelete"),
    direct: () => unreachable("channels.direct"),
    signalChannelBusy: () => unreachable("channels.signalChannelBusy"),
    signalWork: () => unreachable("channels.signalWork"),
    workIn: () => unreachable("channels.workIn"),

    async get(actor, channelId) {
      expect(actor).toEqual(OWNER);
      expect(channelId).toBe(CONTEXT.channelId);
      return options.channel === undefined ? CHANNEL : options.channel;
    },
    async recordActivity(actor, channelId, activity) {
      recorded.activity.push({ actor, channelId, activity });
      if (options.recordActivity) await options.recordActivity();
    },
    async signalBusy(threadId, busy) {
      recorded.busy.push({
        threadId,
        busy,
        finishedBefore: recorded.finished.length,
      });
      if (options.signalBusy) await options.signalBusy();
    },
  };

  const runTurn: TurnRunner = async (input) => {
    recorded.turns.push(input);
    if (options.runTurn) return await options.runTurn(input);
    return { replyText: "Three people are blocked." };
  };

  return {
    recorded,
    runner: createRoutineRunner({ routineStore, channelStore, runTurn }),
  };
}

const throwingTurn: TurnRunner = async () => {
  throw new Error("the model refused");
};

describe("createRoutineRunner", () => {
  test("runs the owner's thread and records the reply exactly once", async () => {
    const { runner, recorded } = harness({});

    await runner.run(RUN_ID);

    expect(recorded.turns).toEqual([
      {
        ownerUserId: CONTEXT.ownerUserId,
        // Carried so the turn's audit rows say a routine ran this, not that the owner did.
        routineId: CONTEXT.routineId,
        agentId: CONTEXT.agentId,
        threadId: CHANNEL.threadId,
        instruction: CONTEXT.instruction,
      },
    ]);
    // The count, not merely that one happened: a second record would ring a second unread dot.
    expect(recorded.activity).toHaveLength(1);
    expect(recorded.activity[0]?.actor).toEqual(OWNER);
    expect(recorded.activity[0]?.channelId).toBe(CONTEXT.channelId);
    expect(recorded.activity[0]?.activity.text).toBe(
      "Three people are blocked.",
    );
    expect(recorded.activity[0]?.activity.agentId).toBe(CONTEXT.agentId);
    expect(recorded.finished).toEqual([
      { runId: RUN_ID, status: "succeeded", error: undefined },
    ]);
    expect(recorded.enabled).toEqual([]);
  });

  test("records a thrown turn as a failure carrying its message", async () => {
    const { runner, recorded } = harness({
      runTurn: throwingTurn,
      failures: 1,
    });

    await runner.run(RUN_ID);

    expect(recorded.finished).toEqual([
      { runId: RUN_ID, status: "failed", error: "the model refused" },
    ]);
  });

  test("says so once on the first failure after a success", async () => {
    const { runner, recorded } = harness({
      runTurn: throwingTurn,
      failures: 1,
    });

    await runner.run(RUN_ID);

    expect(recorded.activity).toHaveLength(1);
    expect(recorded.activity[0]?.activity.text).toBe(
      "This routine failed: the model refused",
    );
    expect(recorded.activity[0]?.activity.agentId).toBe(CONTEXT.agentId);
    expect(recorded.enabled).toEqual([]);
  });

  test("says nothing on the second consecutive failure", async () => {
    const { runner, recorded } = harness({
      runTurn: throwingTurn,
      failures: 2,
    });

    await runner.run(RUN_ID);

    expect(recorded.activity).toEqual([]);
    expect(recorded.enabled).toEqual([]);
    expect(recorded.finished).toHaveLength(1);
  });

  test("switches the routine off after ten consecutive failures", async () => {
    const { runner, recorded } = harness({
      runTurn: throwingTurn,
      failures: 10,
    });

    await runner.run(RUN_ID);

    expect(recorded.enabled).toEqual([
      {
        ownerUserId: CONTEXT.ownerUserId,
        id: CONTEXT.routineId,
        enabled: false,
      },
    ]);
    expect(recorded.activity).toHaveLength(1);
    expect(recorded.activity[0]?.activity.text).toBe(
      "This routine has failed ten times in a row, so I have switched it off. Ask me to turn it back on when whatever it needs is working.",
    );
  });

  test("skips a firing whose channel is gone without running a turn", async () => {
    const { runner, recorded } = harness({ channel: null });

    await runner.run(RUN_ID);

    expect(recorded.turns).toEqual([]);
    expect(recorded.activity).toEqual([]);
    expect(recorded.finished).toEqual([
      { runId: RUN_ID, status: "skipped", error: "the channel is gone" },
    ]);
  });

  test("does nothing at all for a run row that is gone", async () => {
    const { runner, recorded } = harness({ context: null });

    await runner.run(RUN_ID);

    expect(recorded).toEqual({
      finished: [],
      activity: [],
      enabled: [],
      turns: [],
      busy: [],
    });
  });

  test("keeps a succeeded run recorded when the channel cannot be written to", async () => {
    const { runner, recorded } = harness({
      recordActivity: async () => {
        throw new Error("that channel is gone");
      },
    });

    await runner.run(RUN_ID);

    expect(recorded.finished).toEqual([
      { runId: RUN_ID, status: "succeeded", error: undefined },
    ]);
  });

  test("keeps a failed run recorded when the notification cannot be posted", async () => {
    const { runner, recorded } = harness({
      runTurn: throwingTurn,
      failures: 1,
      recordActivity: async () => {
        throw new Error("that channel is gone");
      },
    });

    await runner.run(RUN_ID);

    expect(recorded.finished).toEqual([
      { runId: RUN_ID, status: "failed", error: "the model refused" },
    ]);
  });

  test("truncates a failure reason at the code-point cap without splitting an emoji", async () => {
    // 170 code points: an ASCII run long enough to push the cut (at code point 159) into the
    // middle of the run of astral emoji that follows, so the cut has to land between code points,
    // never inside one of their surrogate pairs.
    const longReason = `${"a".repeat(150)}${"\u{1F600}".repeat(20)}`;
    const { runner, recorded } = harness({
      runTurn: async () => {
        throw new Error(longReason);
      },
      failures: 1,
    });

    await runner.run(RUN_ID);

    expect(recorded.activity).toHaveLength(1);
    const posted = recorded.activity[0]?.activity.text ?? "";
    expect(posted.startsWith("This routine failed: ")).toBe(true);
    const reasonPart = posted.slice("This routine failed: ".length);

    // Capped at MAX_NOTIFIED_REASON (160) code points: 159 kept, plus the implementation's own
    // ellipsis character — read runner.ts's `shorten` rather than assume a format here.
    expect(Array.from(reasonPart)).toHaveLength(160);
    expect(reasonPart).toBe(`${"a".repeat(150)}${"\u{1F600}".repeat(9)}…`);

    // The cut is measured in code points, not UTF-16 units, so no emoji is split into a lone
    // surrogate: the string stays well-formed and a code-point split round-trips cleanly.
    expect(reasonPart.isWellFormed()).toBe(true);
    expect(Array.from(reasonPart).join("")).toBe(reasonPart);
  });

  test("posts a thrown non-Error value with String(error)", async () => {
    const { runner, recorded } = harness({
      runTurn: async () => {
        throw "not an Error object";
      },
      failures: 1,
    });

    await runner.run(RUN_ID);

    expect(recorded.finished).toEqual([
      { runId: RUN_ID, status: "failed", error: "not an Error object" },
    ]);
    expect(recorded.activity).toHaveLength(1);
    expect(recorded.activity[0]?.activity.text).toBe(
      "This routine failed: not an Error object",
    );
  });
  /*
   * The conversation reads "a routine is running" from the open run row, so the signal that says
   * it is over has to come after the row is closed. Sent before, the event would still list the
   * routine and the row would stay on screen until something else happened in the conversation.
   */
  test("shows the conversation working while it fires, and clears it after the run is closed", async () => {
    const { runner, recorded } = harness({});

    await runner.run(RUN_ID);

    expect(recorded.busy).toEqual([
      { threadId: CHANNEL.threadId, busy: true, finishedBefore: 0 },
      { threadId: CHANNEL.threadId, busy: false, finishedBefore: 1 },
    ]);
  });

  test("clears the working signal when the turn fails", async () => {
    const { runner, recorded } = harness({
      runTurn: throwingTurn,
      failures: 2,
    });

    await runner.run(RUN_ID);

    expect(recorded.busy.map((signal) => signal.busy)).toEqual([true, false]);
    expect(recorded.busy.at(-1)?.finishedBefore).toBe(1);
  });

  test("still runs and records the firing when the working signal cannot be sent", async () => {
    const { runner, recorded } = harness({
      signalBusy: async () => {
        throw new Error("the socket is down");
      },
    });

    await runner.run(RUN_ID);

    expect(recorded.turns).toHaveLength(1);
    expect(recorded.finished).toEqual([{ runId: RUN_ID, status: "succeeded" }]);
  });

  test("signals nothing for a firing that never runs", async () => {
    const { runner, recorded } = harness({ channel: null });

    await runner.run(RUN_ID);

    expect(recorded.busy).toEqual([]);
  });
});