Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

15.0 KB
Raw Blame
import { afterAll, beforeEach, describe, expect, test } from "bun:test";
import { randomUUID } from "node:crypto";
import { and, eq } from "drizzle-orm";
import { createDatabase } from "../src/db/client";
import { workItems } from "../src/db/schema";
import { createWorkQueue } from "../src/work/queue";
import { TEST_POOL, testDatabaseUrl } from "./support/database";

/**
 * The one mechanism suspending idle computers, running routines and handing work between Bots all
 * need, driven against a real PostgreSQL rather than a fake.
 *
 * A fake cannot answer the only question worth asking here. `for update skip locked` is a promise
 * the database makes about two transactions racing, and a stub that returns rows in order would pass
 * every test below while the real thing handed one item to two replicas.
 */
const database = createDatabase(testDatabaseUrl(), TEST_POOL);
const queue = createWorkQueue(database);
const kind = `test.${randomUUID().slice(0, 8)}`;

afterAll(async () => {
  await database.delete(workItems).where(eq(workItems.kind, kind));
  await database.$client.end({ timeout: 5 });
});

beforeEach(async () => {
  await database.delete(workItems).where(eq(workItems.kind, kind));
});

describe("claiming durable work", () => {
  test("one replica takes a due item, and it comes back with what it is about", async () => {
    await queue.offer({ kind, key: "bot-a", payload: { botId: "bot-a" } });

    const claimed = await queue.claim({
      kind,
      owner: "replica-1",
      leaseMs: 30_000,
    });

    expect(claimed).toHaveLength(1);
    expect(claimed[0]?.key).toBe("bot-a");
    expect(claimed[0]?.payload).toEqual({ botId: "bot-a" });
    // First time out, so whatever runs this knows it has certainly not run before.
    expect(claimed[0]?.attempts).toBe(1);
  });

  test("a second replica does not get an item the first is holding", async () => {
    await queue.offer({ kind, key: "bot-a" });

    const first = await queue.claim({
      kind,
      owner: "replica-1",
      leaseMs: 30_000,
    });
    const second = await queue.claim({
      kind,
      owner: "replica-2",
      leaseMs: 30_000,
    });

    expect(first).toHaveLength(1);
    expect(second).toHaveLength(0);
  });

  /*
   * THE TEST THIS FILE EXISTS FOR.
   *
   * Ten replicas reaching for ten items at the same moment must between them take each item once.
   * Anything less than `skip locked` fails here: plain `for update` serialises and one transaction
   * waits behind another, and no locking at all hands the same row to several claimants, which for a
   * routine is the same run billed N times.
   */
  test("ten replicas racing for ten items take each of them exactly once", async () => {
    const keys = Array.from({ length: 10 }, (_, index) => `bot-${index}`);
    for (const key of keys) await queue.offer({ kind, key });

    const results = await Promise.all(
      Array.from({ length: 10 }, (_, index) =>
        queue.claim({
          kind,
          owner: `replica-${index}`,
          leaseMs: 30_000,
          limit: 3,
        }),
      ),
    );

    const taken = results.flat().map((item) => item.key);
    expect(taken).toHaveLength(10);
    expect(new Set(taken).size).toBe(10);
  });

  test("a lease that stopped being renewed comes back to whoever asks next", async () => {
    await queue.offer({ kind, key: "bot-a" });
    // Claimed by a replica that then dies: nothing renews, and the lease is already in the past.
    await queue.claim({ kind, owner: "replica-1", leaseMs: -1 });

    const recovered = await queue.claim({
      kind,
      owner: "replica-2",
      leaseMs: 30_000,
    });

    expect(recovered).toHaveLength(1);
    /*
     * Second time out, which is the number that matters. Whatever picks this up has to be able to
     * tell "this never started" from "this started and we lost the process", because the second may
     * already have called a tool and spent money.
     */
    expect(recovered[0]?.attempts).toBe(2);
  });

  test("renewing keeps a claim, and cannot steal one back after it was lost", async () => {
    await queue.offer({ kind, key: "bot-a" });
    await queue.claim({ kind, owner: "replica-1", leaseMs: -1 });
    await queue.claim({ kind, owner: "replica-2", leaseMs: 30_000 });

    // Replica 1 wakes up and tries to keep a claim that is no longer its own.
    const stale = await queue.renew({
      kind,
      key: "bot-a",
      owner: "replica-1",
      leaseMs: 30_000,
    });
    const current = await queue.renew({
      kind,
      key: "bot-a",
      owner: "replica-2",
      leaseMs: 30_000,
    });

    expect(stale).toBe(false);
    expect(current).toBe(true);
  });

  test("offering the same work twice leaves one item", async () => {
    /*
     * Idempotence, which is where every recovery path stops being a duplicate-run path. A routine
     * due at 07:00 is offered by every replica that wakes; the key carries the minute, so they are
     * all offering the same thing.
     */
    await queue.offer({ kind, key: "routine:daily:2026-08-24T07:00" });
    await queue.offer({ kind, key: "routine:daily:2026-08-24T07:00" });
    await queue.offer({ kind, key: "routine:daily:2026-08-24T07:00" });

    const rows = await database
      .select({ key: workItems.key })
      .from(workItems)
      .where(eq(workItems.kind, kind));
    expect(rows).toHaveLength(1);
  });

  test("an item that is not due yet is not claimed", async () => {
    await queue.offer({
      kind,
      key: "later",
      runAt: new Date(Date.now() + 60_000),
    });

    expect(
      await queue.claim({ kind, owner: "replica-1", leaseMs: 30_000 }),
    ).toHaveLength(0);
  });

  /*
   * The idempotence this table promises has to survive completion.
   *
   * Finishing used to delete the row, so the insert a re-offer was supposed to collide with had
   * nothing left to collide with: a routine due at 07:00 that had already run was handed straight
   * back to the next replica to wake late, and ran twice. Every recovery path was a duplicate-run
   * path.
   */
  test("a finished item stays, so re-offering the same key runs nothing", async () => {
    await queue.offer({ kind, key: "bot-a" });
    await queue.claim({ kind, owner: "replica-1", leaseMs: 30_000 });
    expect(await queue.finish({ kind, key: "bot-a", owner: "replica-1" })).toBe(
      true,
    );

    await queue.offer({ kind, key: "bot-a" });

    expect(
      await queue.claim({ kind, owner: "replica-2", leaseMs: 30_000 }),
    ).toHaveLength(0);
  });

  test("finished rows are swept once they are past their retention", async () => {
    await queue.offer({ kind, key: "bot-a" });
    await queue.claim({ kind, owner: "replica-1", leaseMs: 30_000 });
    await queue.finish({ kind, key: "bot-a", owner: "replica-1" });

    expect(await queue.purge({ kind, olderThanMs: 60_000 })).toBe(0);
    expect(await queue.purge({ kind, olderThanMs: 0 })).toBe(1);
  });

  test("releasing frees the item and holds it back for a while", async () => {
    await queue.offer({ kind, key: "bot-a" });
    await queue.claim({ kind, owner: "replica-1", leaseMs: 30_000 });
    await queue.release({
      kind,
      key: "bot-a",
      owner: "replica-1",
      delayMs: 60_000,
    });

    // Free, but not yet due, so nobody picks it straight back up and spins on it.
    expect(
      await queue.claim({ kind, owner: "replica-2", leaseMs: 30_000 }),
    ).toHaveLength(0);
  });

  /*
   * Both of these are the same bug from two ends: the lease says who may act on an item, and only
   * `renew` used to ask. A replica whose lease had quietly gone could delete or reschedule work
   * another replica was in the middle of doing.
   */
  test("a replica that lost its lease cannot finish somebody else's work", async () => {
    await queue.offer({ kind, key: "bot-a" });
    await queue.claim({ kind, owner: "replica-1", leaseMs: 1 });
    await Bun.sleep(30);
    await queue.claim({ kind, owner: "replica-2", leaseMs: 30_000 });

    expect(await queue.finish({ kind, key: "bot-a", owner: "replica-1" })).toBe(
      false,
    );

    const [row] = await database
      .select({ by: workItems.claimedBy, done: workItems.finishedAt })
      .from(workItems)
      .where(and(eq(workItems.kind, kind), eq(workItems.key, "bot-a")));
    expect(row?.by).toBe("replica-2");
    expect(row?.done).toBeNull();
  });

  test("a replica that lost its lease cannot reschedule somebody else's work", async () => {
    await queue.offer({ kind, key: "bot-a" });
    await queue.claim({ kind, owner: "replica-1", leaseMs: 1 });
    await Bun.sleep(30);
    await queue.claim({ kind, owner: "replica-2", leaseMs: 30_000 });

    expect(
      await queue.release({
        kind,
        key: "bot-a",
        owner: "replica-1",
        delayMs: 60_000,
      }),
    ).toBe(false);

    const [row] = await database
      .select({ by: workItems.claimedBy })
      .from(workItems)
      .where(and(eq(workItems.kind, kind), eq(workItems.key, "bot-a")));
    expect(row?.by).toBe("replica-2");
  });

  /*
   * Two clocks pretending to be one.
   *
   * The lease was computed as `Date.now() + leaseMs` on the replica and compared against `now()` in
   * Postgres. A node behind the database wrote a live lease that arrived already expired, and the
   * next replica to look took the item out from under it. Both ran it. Every moment is named in SQL
   * now, so a wrong local clock cannot produce one.
   */
  test("a replica whose clock is behind still holds a real lease", async () => {
    await queue.offer({ kind, key: "bot-a" });

    const realNow = Date.now;
    Date.now = () => realNow() - 90_000;
    try {
      await queue.claim({ kind, owner: "replica-1", leaseMs: 60_000 });
    } finally {
      Date.now = realNow;
    }

    expect(
      await queue.claim({ kind, owner: "replica-2", leaseMs: 60_000 }),
    ).toHaveLength(0);
  });

  /*
   * A permanently failing item has to stop somewhere a person can see, rather than retrying until
   * somebody notices, which on a queue with no dashboard is never.
   */
  test("an item stops being offered once it runs out of attempts", async () => {
    await queue.offer({ kind, key: "bot-a" });

    for (let attempt = 0; attempt < 3; attempt += 1) {
      const [item] = await queue.claim({
        kind,
        owner: "replica-1",
        leaseMs: 30_000,
        maxAttempts: 3,
      });
      expect(item?.attempts).toBe(attempt + 1);
      await queue.release({
        kind,
        key: "bot-a",
        owner: "replica-1",
        delayMs: 0,
        reason: "the cluster said no",
      });
    }

    expect(
      await queue.claim({
        kind,
        owner: "replica-1",
        leaseMs: 30_000,
        maxAttempts: 3,
      }),
    ).toHaveLength(0);

    // Still here, with its count and its reason, which is the terminal state rather than a silence.
    const [row] = await database
      .select({ attempts: workItems.attempts, why: workItems.lastError })
      .from(workItems)
      .where(and(eq(workItems.kind, kind), eq(workItems.key, "bot-a")));
    expect(row?.attempts).toBe(3);
    expect(row?.why).toBe("the cluster said no");
  });

  /*
   * The two kinds of done, on their own clocks.
   *
   * They were one number, which a queue whose work repeats cannot afford: a finished row only has to
   * outlast a sweep, and a row that gave up is kept because that window is also the backoff before
   * anything tries the work again. Sharing it meant the culler either wedged its own key for a day
   * or retried a broken suspension every few minutes.
   */
  test("a finished row and one that gave up are swept on separate windows", async () => {
    await queue.offer({ kind, key: "done" });
    await queue.claim({ kind, owner: "replica-1", leaseMs: 30_000 });
    await queue.finish({ kind, key: "done", owner: "replica-1" });

    await queue.offer({ kind, key: "gave-up" });
    for (let attempt = 0; attempt < 3; attempt += 1) {
      await queue.claim({
        kind,
        owner: "replica-1",
        leaseMs: 30_000,
        maxAttempts: 3,
      });
      await queue.release({
        kind,
        key: "gave-up",
        owner: "replica-1",
        delayMs: 0,
        reason: "the cluster said no",
      });
    }

    // The finished one goes on its own short window; the one that gave up keeps its long one.
    expect(
      await queue.purge({
        kind,
        olderThanMs: 60_000,
        finishedOlderThanMs: 0,
        maxAttempts: 3,
      }),
    ).toBe(1);
    const [left] = await database
      .select({ key: workItems.key })
      .from(workItems)
      .where(eq(workItems.kind, kind));
    expect(left?.key).toBe("gave-up");

    // And omitting it leaves both halves on the one window, which is what every other caller does.
    expect(await queue.purge({ kind, olderThanMs: 0, maxAttempts: 3 })).toBe(1);
  });
});

/**
 * The fan-out cap, under the only conditions that matter.
 *
 * A model asked to do several things emits several tool calls in one turn and they run at once. A
 * cap checked before the write holds only while nothing else is writing, so all of them pass it:
 * each reads a count taken before any of the others had committed. This needs no cluster and no
 * unusual timing, which is why it must be driven against a real database rather than a stub that
 * awaits one call at a time.
 */
describe("offering at most so many under one prefix", () => {
  test("five at once cannot get past a cap of three", async () => {
    const run = `${randomUUID()}:`;

    const results = await Promise.all(
      ["one", "two", "three", "four", "five"].map((word) =>
        queue.offer({
          kind,
          key: `${run}${word}`,
          atMost: { keyPrefix: run, max: 3 },
        }),
      ),
    );

    expect(results.filter((result) => result === "queued")).toHaveLength(3);
    expect(results.filter((result) => result === "refused")).toHaveLength(2);
    const written = await database
      .select({ key: workItems.key })
      .from(workItems)
      .where(eq(workItems.kind, kind));
    expect(written).toHaveLength(3);
  });

  /*
   * A retried offer of work that is already queued is not a new hop, and must not be reported as
   * refused by the cap: the caller asked for it to be on the queue and it is.
   */
  test("the same key again is not counted against the cap", async () => {
    const run = `${randomUUID()}:`;
    const cap = { keyPrefix: run, max: 1 };

    expect(await queue.offer({ kind, key: `${run}a`, atMost: cap })).toBe(
      "queued",
    );
    // The same key again is work already queued, not a second piece of work — and not something the
    // cap should refuse either. A caller that reports it as new promises an answer nobody will give.
    expect(await queue.offer({ kind, key: `${run}a`, atMost: cap })).toBe(
      "already",
    );
    expect(await queue.offer({ kind, key: `${run}b`, atMost: cap })).toBe(
      "refused",
    );
  });

  test("without a cap nothing is refused", async () => {
    const run = `${randomUUID()}:`;
    const results = await Promise.all(
      [1, 2, 3, 4, 5].map((n) => queue.offer({ kind, key: `${run}${n}` })),
    );
    expect(results.every((result) => result === "queued")).toBe(true);
  });
});