Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

29.1 KB
Raw Blame
/**
 * Delivering a hop: running the Bot that was addressed, and putting its answer in the conversation.
 *
 * The other half of `handoff.ts`. Deciding happens inside somebody's run and has to be quick and
 * fail closed; delivering is a whole agent turn against a model, and it has to survive the pod it
 * started on. So the two are separated by the queue rather than by a function call.
 *
 * CLAIMED, NOT ASSIGNED. Any replica may take any hop, which is what makes this work on a cluster
 * where the Bot being addressed is very unlikely to be on the pod that addressed it. The lease is
 * renewed for as long as the run takes, because a run is minutes and a lease that lapses mid-answer
 * hands the same hop to a second replica and bills for it twice.
 */
import {
  type AuditInitiator,
  type AuditStore,
  recordAuditEvent,
} from "../audit";
import { cutAtCodeUnits } from "../channels/text";
import { DEFAULT_MAX_ATTEMPTS, type WorkQueue } from "../work/queue";
import { isEndpointRefusal } from "./endpoint";
import { HANDOFF_KIND } from "./handoff";
import { ThreadBusyError } from "./handoff-delivery";
import { BOT_PAUSED_REASON, isBotPaused } from "./lifecycle";

/** What a hop carries, as `handoff.ts` wrote it. */
export type HandoffWork = {
  fromBotId: string;
  toBotId: string;
  actorId: string;
  threadId: string;
  runId: string;
  depth: number;
  /** What started the original run, preserved across queued delivery and relay hops. */
  initiator?: AuditInitiator;
  task: string;
  constraints?: string;
  expecting?: string;
  /** The asking Bot's display name, for the line a person reads. Absent falls back to its id. */
  fromName?: string;
  /** The addressed Bot's display name, for the same reason. */
  toName?: string;
  /**
   * Where the answer belongs, when it is not the addressed Bot's own conversation.
   *
   * Set on the one kind of hop that goes backwards: telling the asking Bot, in the conversation the
   * person is actually watching, that the Bot it asked never answered. That conversation belongs to
   * the asking Bot, which is why it can speak in it at all.
   */
  answerIn?: string;
};

export type HandoffDelivery = {
  /**
   * Run the addressed Bot against the conversation, and resolve when its turn is on record.
   *
   * Rejecting means the hop did not happen and is worth another go. Resolving means it did, whatever
   * the Bot said: a Bot that answers "I could not find that" has answered, and retrying would ask it
   * the same question again and bill for the same non-answer.
   *
   * Resolves with what the Bot said, because its turn runs in a scratch thread nobody is shown:
   * the words it comes back with exist for the relay or not at all. Null means a turn of nothing
   * but tool calls, which is a turn that happened and nothing worth carrying back.
   */
  deliver: (input: {
    work: HandoffWork;
    /** The message the addressed Bot sees, already attributed by the deployment. */
    message: string;
    /**
     * The one line of it that belongs in the transcript, if any.
     *
     * TWO TEXTS, because they have two readers. The model needs the envelope: who is asking, the
     * task, its constraints, what a good answer looks like, and an instruction about who to write
     * for. A person scrolling their conversation with the addressed Bot needs to know why it
     * suddenly said something, in one sentence. Persisting the envelope puts a paragraph of
     * machine instructions in their transcript, in a bubble that looks like something they wrote.
     *
     * Absent means nothing is kept, which is right for a Bot going back to its own conversation to
     * report a failure: what it says already explains why it spoke, and the instruction that made it
     * speak is addressed to a model.
     */
    shown?: string;
    /** The signed statement of the run it is starting, carrying its depth. */
    assertion: string;
  }) => Promise<{ answer: string | null }>;
};

export type HandoffRunReport = {
  delivered: string[];
  skipped: { key: string; reason: string }[];
};

/**
 * How often a claim is refreshed while a hop is being delivered.
 *
 * Comfortably inside the lease, because a renewal that lands after it has lapsed is not a renewal:
 * the item has already gone to somebody else, and this one is now the second replica running it.
 */
const RENEW_EVERY_MS = 20_000;

/**
 * How long a hop that is over is kept before it is dropped.
 *
 * Long past the point where its key still has to stop a duplicate — that is the asking run's own
 * lifetime, minutes — and short enough that this table holds about a day of work rather than all of
 * it. Both the finished ones and the ones that ran out of attempts: the second is a terminal state
 * somebody can query, and a day is long enough to query it in.
 */
const REAP_OLDER_THAN_MS = 24 * 60 * 60 * 1_000;

export function createHandoffRunner(options: {
  queue: WorkQueue;
  delivery: HandoffDelivery;
  /** Who this replica is, for the lease. */
  owner: string;
  /** How the deployment signs what the addressed Bot's run is. */
  sign: (work: HandoffWork, claim: { key: string; owner: string }) => string;
  auditStore: AuditStore;
  /** How long a claim lasts before anything may take it back. */
  leaseMs?: number;
  /** How many hops one sweep will take. */
  limit?: number;
  /** After how many tries a hop is given up on. Told to `claim` as well as gating the notice. */
  maxAttempts?: number;
  /**
   * How often a claim is refreshed. Comfortably inside the lease.
   *
   * Injectable so the thing it protects against can be driven in a test in milliseconds rather than
   * in minutes. What it protects against is a batch whose tail expires while its head is delivering,
   * which is a matter of one duration outrunning another and does not care about the scale.
   */
  renewEveryMs?: number;
  /**
   * The queue moved for a hop: finished, released for another try, or dropped. Told after the row
   * changed, with the relay already queued, so the conversation that asked can be told what it is
   * waiting on now. The delivery's own busy signal fires inside the run, while the row still says
   * the hop is in flight, so it cannot be the one that says it is over. Best-effort: never thrown.
   */
  onSettled?: (work: HandoffWork) => Promise<void>;
}) {
  const {
    queue,
    delivery,
    owner,
    sign,
    auditStore,
    leaseMs = 60_000,
    limit = 5,
    maxAttempts = DEFAULT_MAX_ATTEMPTS,
    renewEveryMs = RENEW_EVERY_MS,
    onSettled,
  } = options;

  const settled = async (work: HandoffWork) => {
    try {
      await onSettled?.(work);
    } catch {
      // A conversation that has not heard is corrected by its next event or its next read; a hop
      // reported as failed for it would be run again.
    }
  };

  /**
   * Put the failure in front of the person, by running the Bot that asked in the conversation they
   * are watching.
   *
   * THROUGH THE SAME QUEUE, not by writing a line somewhere. The asking Bot is the only thing that
   * can speak in that conversation, and what the person needs is a sentence in its voice saying who
   * it asked and that nothing came back. A row written past the Bot would be a message from nobody.
   *
   * Marked with `answerIn`, which is also what stops this recursing: a notice that fails is not
   * itself worth a notice, and the check above skips any hop that carries one.
   */
  /**
   * Put the answer in front of the person, the same way a failure is: by running the Bot that
   * asked, in the conversation they are watching.
   *
   * THE ADDRESSED BOT NEVER SPEAKS THERE — the platform gives a thread exactly one agent — so its
   * words come home in the asking Bot's voice, attributed. The same `answerIn` marker that stops a
   * notice recursing stops a relay relaying: a hop that carries one enqueues nothing when it lands.
   *
   * The answer is clipped rather than trusted to be a paragraph. It rides inside the prompt of the
   * relaying run, and a Bot that came back with a report the length of a book would otherwise spend
   * the relay's whole context window repeating it.
   */
  const relay = (work: HandoffWork, key: string, answer: string) =>
    queue.offer({
      kind: HANDOFF_KIND,
      // Outside the run's fan-out prefix and keyed on the hop, for the same two reasons as the
      // notice below: a relay is not a Bot this run asked for, and one run may legally ask the
      // same Bot two different things.
      key: `relay:${key}`,
      payload: {
        fromBotId: work.toBotId,
        toBotId: work.fromBotId,
        actorId: work.actorId,
        threadId: work.threadId,
        runId: work.runId,
        depth: work.depth,
        ...(work.initiator ? { initiator: work.initiator } : {}),
        answerIn: work.threadId,
        task: `You asked ${work.toName ?? work.toBotId} to help with this: ${work.task}\n\nIt answered:\n\n${clip(answer)}\n\nGive the person the outcome. Keep what matters, drop the pleasantries, and say it came from ${work.toName ?? work.toBotId}.`,
      } as unknown as Record<string, unknown>,
    });

  const tell = (work: HandoffWork, key: string, reason: string) =>
    queue.offer({
      kind: HANDOFF_KIND,
      /*
       * OUTSIDE THE RUN'S OWN PREFIX, and carrying the failed hop's key.
       *
       * Outside, because the fan-out cap counts every row whose key starts with `${runId}:` and a
       * notice is not one of the Bots this run asked for. A hop that failed for good while the run
       * was still going would otherwise spend a third of a three-Bot budget on the message saying
       * so, and the run's next legitimate ask would be refused with "this turn has already asked 3
       * Bots" after asking two.
       *
       * Carrying the hop's key, because one run may legally ask the same Bot two different things.
       * Keyed on the Bot alone both notices are the same work to `offer`, the second is dropped on
       * conflict, and the person hears about one of their two lost questions with the other's
       * reason — for a whole day, until `reap` drops the row that is blocking it.
       */
      key: `notice:${key}`,
      payload: {
        fromBotId: work.toBotId,
        toBotId: work.fromBotId,
        actorId: work.actorId,
        threadId: work.threadId,
        runId: work.runId,
        depth: work.depth,
        ...(work.initiator ? { initiator: work.initiator } : {}),
        answerIn: work.threadId,
        task: `You asked ${work.toBotId} to help with this and it never answered: ${forThePerson(reason)}. Tell the person plainly that it did not come back, say what you had asked it for, and offer what you can do yourself.`,
      } as unknown as Record<string, unknown>,
    });

  return {
    /**
     * Drop hops that are over, long after they were.
     *
     * NOTHING ELSE REAPS THIS KIND. A finished hop is kept rather than deleted, because a key that
     * is still there is what makes `offer` idempotent and stops a retried delivery running the other
     * Bot twice. Kept for ever, though, the table only grows — and the fan-out cap counts rows under
     * a run's prefix with a `LIKE` that no index serves, so every offer pays for every hop the
     * deployment has ever made.
     *
     * The window is what keeps both true at once. Idempotency only has to hold while the asking run
     * could still offer the same hop again, which is minutes; a day is far past that and still short
     * enough that the table reflects roughly a day's work.
     */
    async reap(): Promise<number> {
      return queue.purge({
        kind: HANDOFF_KIND,
        olderThanMs: REAP_OLDER_THAN_MS,
        maxAttempts,
      });
    },

    /** Deliver whatever this replica can claim. */
    async sweep(): Promise<HandoffRunReport> {
      const claimed = await queue.claim({
        kind: HANDOFF_KIND,
        owner,
        leaseMs,
        limit,
        /*
         * The same ceiling the notice below is gated on, because two different ceilings is two
         * different ideas of when a hop is over.
         *
         * `claim` stops serving a row at its own cutoff. Set higher here than there and the row is
         * never handed out again, `attempts` never reaches this number, and the notice that exists
         * to stop a person waiting for ever is never sent: the silent stop, arriving through the
         * feature built to prevent it. Set lower and the person is told it failed for good while
         * the queue keeps handing it out, so the Bot may answer after they were told it would not.
         */
        maxAttempts,
      });
      const report: HandoffRunReport = { delivered: [], skipped: [] };

      /*
       * EVERY CLAIMED HOP IS RENEWED, not just the one being delivered.
       *
       * A claim leases the whole batch from one moment and this loop delivers them one at a time, so
       * a heartbeat started per item leaves the rest of the batch on a lease that is quietly running
       * out while the first delivery runs. A delivery is minutes and the lease is one, so the tail of
       * every batch expired, was claimed by another replica, and was delivered twice: two model
       * calls, two answers in the person's conversation, and both replicas reporting success.
       *
       * Reproduced against a real PostgreSQL with two replicas, which is the only way this shows up:
       * the item in flight is fine, and the ones waiting behind it are not.
       */
      const ours = new Set(claimed.map((item) => item.key));
      const heartbeat = setInterval(() => {
        for (const key of ours) {
          void queue
            .renew({ kind: HANDOFF_KIND, key, owner, leaseMs })
            .then((kept) => {
              // False means it went to somebody else. Dropped rather than renewed again, so the
              // loop below knows not to spend a model call on work it no longer holds.
              if (!kept) ours.delete(key);
            })
            .catch(() => {});
        }
      }, renewEveryMs);

      try {
        for (const item of claimed) {
          const work = item.payload as unknown as HandoffWork;
          if (!work?.toBotId || !work.threadId) {
            /*
             * A hop nothing can be done with. Finished rather than released, because releasing it puts
             * the same unusable row back on the queue for ever.
             */
            await queue.finish({ kind: HANDOFF_KIND, key: item.key, owner });
            report.skipped.push({ key: item.key, reason: "not a hop" });
            continue;
          }
          // A hop to a Bot its person paused is dropped, not delivered. See agents/lifecycle.ts.
          if (await isBotPaused(work.actorId, work.toBotId)) {
            await queue.finish({ kind: HANDOFF_KIND, key: item.key, owner });
            ours.delete(item.key);
            report.skipped.push({ key: item.key, reason: BOT_PAUSED_REASON });
            await settled(work);
            continue;
          }

          /*
           * A hop that has already been tried is not a fresh one, and the difference matters here more
           * than anywhere else this queue is used: a first attempt has certainly not run the other
           * Bot, while a second may already have run it, spent a model call and posted an answer
           * before its owner died. Recorded rather than guessed at, so somebody reading the trail can
           * tell a duplicate answer from a mystery.
           */
          if (item.attempts > 1) {
            await recordAuditEvent(auditStore, {
              eventType: "agent.handoff_retried",
              targetType: "agent",
              targetId: work.toBotId,
              ...(work.actorId ? { actorUserId: work.actorId } : {}),
              initiator: { kind: "handoff", id: work.fromBotId },
              payload: {
                // See the same key on `agent.handoff_delivered` below: the Audit screen's Bot
                // column reads `payload.bot`, so a row without it names no Bot.
                bot: work.fromBotId,
                from: work.fromBotId,
                to: work.toBotId,
                run: work.runId,
                attempt: item.attempts,
                note: "A previous attempt may already have run this Bot.",
              },
            });
          }

          /*
           * How long the hop took, recorded either way. A hop is a run nobody is watching, so the
           * trail is the only place its duration is visible: "delivered in 4s" and "delivered in 4m"
           * are the same row otherwise, and the second is what a person waiting was actually shown.
           */
          const startedAt = Date.now();
          /*
           * Still ours, ASKED OF THE DATABASE, immediately before a model call rather than after it.
           *
           * Consulting the heartbeat's own set would only catch a renewal that had been attempted
           * and refused. A process paused long enough for the lease to lapse never attempted one, so
           * its set still says the hop is his, and he delivers it on top of whoever has since taken
           * it. The renewal is the question and the answer at once, and it puts a fresh lease under
           * the delivery about to start, which is the moment one is most needed.
           *
           * Running a hop that is no longer ours is the expensive half of a duplicate: a whole agent
           * turn, billed, ending in a second answer in somebody's conversation.
           */
          const stillOurs = await queue.renew({
            kind: HANDOFF_KIND,
            key: item.key,
            owner,
            leaseMs,
          });
          if (!stillOurs) {
            ours.delete(item.key);
            report.skipped.push({
              key: item.key,
              reason: "the lease went elsewhere",
            });
            continue;
          }

          try {
            const shown = summarise(work);
            const { answer } = await delivery.deliver({
              work,
              message: attribute(work),
              ...(shown ? { shown } : {}),
              assertion: sign(work, { key: item.key, owner }),
            });
            const kept = await queue.finish({
              kind: HANDOFF_KIND,
              key: item.key,
              owner,
            });
            ours.delete(item.key);
            /*
             * `finish` answering false means the lease went elsewhere while this ran, so another
             * replica may have delivered the same hop. The turn happened either way and the trail has
             * to say so; what it must not say is that this replica finished the work, because it did
             * not, and a person reading two similar answers would have nothing to tell a duplicate
             * from a mystery.
             */
            if (!kept) {
              report.skipped.push({
                key: item.key,
                reason: "delivered, but the lease had gone elsewhere",
              });
              await recordAuditEvent(auditStore, {
                eventType: "agent.handoff_retried",
                targetType: "agent",
                targetId: work.toBotId,
                ...(work.actorId ? { actorUserId: work.actorId } : {}),
                initiator: { kind: "handoff", id: work.fromBotId },
                payload: {
                  // See the same key on `agent.handoff_delivered` below.
                  bot: work.fromBotId,
                  from: work.fromBotId,
                  to: work.toBotId,
                  run: work.runId,
                  attempt: item.attempts,
                  note: "This replica delivered a hop whose lease had already gone elsewhere. Another may have delivered it too.",
                },
              });
              continue;
            }
            report.delivered.push(work.toBotId);
            /*
             * The answer goes home through the queue, like the turn that produced it: durable, so a
             * pod dying between the turn and the relay loses the relay to a retry rather than for
             * ever. Only for a forward hop with words to carry — a relay of a relay is the loop the
             * `answerIn` check exists to stop, and a wordless turn has nothing to say.
             */
            if (!work.answerIn && answer) {
              await relay(work, item.key, answer).catch((failure) => {
                // The turn happened and is on record; a relay that cannot be queued must not undo
                // that by failing the hop into a retry and a second turn.
                console.warn(
                  "Could not queue the relay for a delivered hop.",
                  failure,
                );
              });
            }
            // After the relay is queued, so the conversation reads as still waiting on the answer.
            await settled(work);
            await recordAuditEvent(auditStore, {
              eventType: "agent.handoff_delivered",
              targetType: "agent",
              targetId: work.toBotId,
              ...(work.actorId ? { actorUserId: work.actorId } : {}),
              initiator: { kind: "handoff", id: work.fromBotId },
              payload: {
                // See the same key on `agent.handoff_offered`: the Audit screen's Bot column reads
                // `payload.bot`, so a row without it names no Bot.
                bot: work.fromBotId,
                from: work.fromBotId,
                to: work.toBotId,
                run: work.runId,
                depth: work.depth,
                ms: Date.now() - startedAt,
              },
            });
          } catch (error) {
            const reason =
              error instanceof Error ? error.message : "could not be delivered";
            /*
             * The last try, so the person is told rather than left waiting.
             *
             * Enqueued before the release, because the release is what makes this attempt the last
             * one: after it the row will never be claimed again and nothing else will ever look at
             * this hop. A person who was told their question had been handed on, and then hears
             * nothing for ever, has no way to tell a slow Bot from a broken one.
             */
            // The deployment's controls refused it (Use Bots, the model allowlist), or the other
            // Bot's address is one this deployment will not dial. Trying again in a minute gets the
            // same answer, so this is the last try, whatever the count says. Retrying a refused
            // address left the person watching "Asking …" for five minutes before being told.
            const refused =
              (error instanceof Error &&
                error.name === "CapabilityRefusedError") ||
              isEndpointRefusal(error);
            if ((refused || item.attempts >= maxAttempts) && !work.answerIn) {
              await tell(work, item.key, reason).catch((failure) => {
                // A notice that cannot be queued must not take the release with it: leaving the row
                // claimed would be worse than a hop nobody was told about.
                console.warn(
                  "Could not queue the notice for a hop that failed for good.",
                  failure,
                );
              });
            }
            /*
             * Released and pushed out rather than dropped. The work still wants doing, and whatever
             * refused it once will probably refuse it again in the next second. A refusal by the
             * controls is the exception: it is final, so the work ends here.
             */
            if (refused)
              await queue.finish({ kind: HANDOFF_KIND, key: item.key, owner });
            else
              await queue.release({
                kind: HANDOFF_KIND,
                key: item.key,
                owner,
                // A busy conversation on the first try is almost always the asking Bot finishing its
                // own reply, a second or two from free. Later tries keep the minute, so a person
                // mid-conversation still gets the same few minutes of patience.
                delayMs:
                  error instanceof ThreadBusyError && item.attempts <= 1
                    ? 5_000
                    : 60_000,
                reason,
              });
            ours.delete(item.key);
            report.skipped.push({ key: item.key, reason });
            await settled(work);
            await recordAuditEvent(auditStore, {
              eventType: "agent.handoff_failed",
              targetType: "agent",
              targetId: work.toBotId,
              ...(work.actorId ? { actorUserId: work.actorId } : {}),
              initiator: { kind: "handoff", id: work.fromBotId },
              payload: {
                // See the same key on `agent.handoff_delivered` above. This row is the one a
                // person's unanswered question ends on, so a Bot column showing a dash on it is
                // the worst place in the set to have one.
                bot: work.fromBotId,
                from: work.fromBotId,
                to: work.toBotId,
                run: work.runId,
                attempt: item.attempts,
                reason,
                ms: Date.now() - startedAt,
              },
            });
          }
        }
      } finally {
        clearInterval(heartbeat);
      }

      return report;
    },
  };
}

/**
 * How much of an answer one relay will carry.
 *
 * Generous, because with the answer living nowhere a person is shown, what the relay drops is gone:
 * the scratch thread that holds the rest is never mapped to a channel. The cap exists for the Bot
 * that comes back with a book — an answer that size swamps the relaying run's prompt, and the
 * asking Bot was told what a good answer looks like precisely so this stays a paragraph.
 */
const RELAY_ANSWER_LIMIT = 12_000;

function clip(answer: string): string {
  if (answer.length <= RELAY_ANSWER_LIMIT) return answer;
  return `${cutAtCodeUnits(answer, RELAY_ANSWER_LIMIT)}\n\n[…the answer was cut here for length]`;
}

/**
 * The same failure, in words that can be said out loud.
 *
 * The reason on a failed hop is whatever threw, and one of the things that throws is the platform
 * client, whose message is `Intelligence platform error 409: {"error":{...}}` — a response body,
 * verbatim. That reason is interpolated into the notice a Bot then paraphrases to a person, so an
 * internal error envelope ends up in somebody's chat. The trail keeps the whole thing; the sentence
 * gets the shape of the problem.
 */
function forThePerson(reason: string): string {
  const platform = reason.match(/^Intelligence platform error (\d{3})\b/);
  if (platform) {
    return `the platform answered ${platform[1]} (the full response is in the trail)`;
  }
  return reason;
}

/**
 * What the addressed Bot is shown.
 *
 * WHO IS ASKING IS STAMPED HERE, from the row this deployment wrote, and never taken from anything a
 * model produced. A Bot able to write its own attribution is a Bot able to claim to be another one,
 * and the whole point of naming the sender is that the answer can be trusted to say who wanted it.
 *
 * The parts stay parts. The asking model was made to name the task, its constraints and what a good
 * answer looks like precisely so the receiving one does not have to infer them out of a paragraph,
 * and flattening them back into prose here would throw that away at the last step.
 */
function attribute(work: HandoffWork): string {
  /*
   * A notice is not a request for help, and must not read as one.
   *
   * This one goes to the Bot that ASKED, in the conversation it is already in, and its whole content
   * is what became of the hop. Dressed in the wording below it would tell a Bot that the Bot it
   * asked has now asked it for something, which is the beginning of a loop rather than the end of
   * one.
   */
  if (work.answerIn) {
    return `${work.task}\n\nSay this in your own words to the person in this conversation, in a sentence or two. Do not hand it to another Bot.`;
  }
  const lines = [
    `${work.fromBotId} has asked you to help with this, on behalf of the person in this conversation.`,
    "",
    `Task: ${work.task}`,
  ];
  if (work.constraints) lines.push(`Constraints: ${work.constraints}`);
  if (work.expecting)
    lines.push(`What a good answer looks like: ${work.expecting}`);
  lines.push(
    "",
    "Answer in this conversation as yourself. The person can see it, so write it for them rather than for the Bot that asked.",
  );
  return lines.join("\n");
}

/**
 * The same hop, in one line, for the person who will scroll past it.
 *
 * They did not send this and it is not addressed to them: their conversation with one Bot has a
 * message in it because a different Bot asked for something. So it says exactly that, and leaves the
 * constraints and the shape-of-answer notes out. Those are instructions to a model, and reading
 * somebody else's instructions to a model is how a transcript stops being a conversation.
 */
function summarise(work: HandoffWork): string | null {
  /*
   * Nothing, for a Bot going back to its own conversation to say a hop failed. Its own sentence is
   * the whole message; the text that prompted it is an instruction to a model, and shown here it
   * appears as something the person typed and then had read back to them.
   */
  if (work.answerIn) return null;
  return `${work.fromName ?? work.fromBotId} asked ${work.toName ?? work.toBotId} for this on your behalf: ${work.task}`;
}