Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
Notice when a Bot stops talking, instead of spinning forever (#19)
Jerel Velarde6b9e4cc0edea
diff --git a/.env.example b/.env.exampleindex 229792f4..f25c4b7c 100644--- a/.env.example+++ b/.env.example@@ -49,6 +49,29 @@ INTELLIGENCE_GATEWAY_WS_URL=wss://realtime.intelligence.copilotkit.aiINTELLIGENCE_API_KEY=COPILOTKIT_LICENSE_TOKEN=+# How long a Bot's stream may say nothing before this deployment gives up on the turn, in+# milliseconds. A Bot is any AG-UI endpoint, which means it will be redeployed mid-answer, its own+# upstream will time out, and it will sometimes accept a connection and then write nothing at all.+# Without this the channel stays busy, the composer stays locked, and the only way out is a reload.+#+# Silence, not duration. A turn may legitimately run for an hour while events keep arriving; what is+# measured here is the gap between them. Nothing about how long a Bot is allowed to work changes.+#+# A minute, because this deadline is in a race it has to win. Every Bot in this repository serves on+# Bun with `idleTimeout: 120`, and Bun tears a wedged streaming response down at roughly a second+# past that. A watchdog set to the same two minutes lands within a second of the socket dying, and+# whichever gets there first decides what the person sees: this deployment's sentence naming the Bot+# and saying the turn was ended, or "The socket connection was closed unexpectedly" and no audit row+# at all. Half the Bot's own idle timeout is far enough clear that the answer is always the first.+#+# A minute is still far longer than any real silence inside a run. The longest legitimate one is the+# wait for a model's first token, which is seconds. Browser tool calls do not need allowing for:+# they run between turns, not during one, because the run ends before the browser executes the tool+# and a second run carries the result back.+#+# 0, or leaving this unset, switches the watchdog off. Nothing is watched and no turn is ever ended.+AGENT_STALL_TIMEOUT_MS=60000+# Model key. Required by the proof-of-concept Bot, which speaks OpenAI's API directly, and by the# framework Bot unless you point it at another provider below.OPENAI_API_KEY=diff --git a/app/src/components/channels/channel-chat.tsx b/app/src/components/channels/channel-chat.tsxindex d16b8350..dabb8ba9 100644--- a/app/src/components/channels/channel-chat.tsx+++ b/app/src/components/channels/channel-chat.tsx@@ -19,6 +19,7 @@ import type { AgentChannel } from "@/lib/channels/queries";import { useActiveBot } from "@/lib/copilot/active-bot";import { ConversationProvider } from "@/lib/copilot/conversation";import { repairUnansweredToolCalls } from "@/lib/copilot/repair-history";+import { stoppedReason } from "@/lib/copilot/stopped-turn";import { useSkillCommands } from "@/lib/plugins/skill-commands";/**@@ -271,14 +272,10 @@ export function ChannelChat({setRunError(message);};const subscription = agent.subscribe?.({- onRunErrorEvent: ({ event }) =>- fail(event?.message ?? "The Bot stopped without saying why."),- onRunFailed: ({ error }) =>- fail(- error instanceof Error- ? error.message- : "The Bot stopped without saying why.",- ),+ // Both surfaces fall back to the same sentence, from the same place, so a person who uses+ // both is not told two different things about the same silence.+ onRunErrorEvent: ({ event }) => fail(stoppedReason(event?.message)),+ onRunFailed: ({ error }) => fail(stoppedReason(error)),onRunFinishedEvent: () => {const wasOurs = awaitingReply.current;awaitingReply.current = false;@@ -339,23 +336,12 @@ export function ChannelChat({disabled={!channel.active}messages={transcriptMessages(agent.messages, seed)}notice={- <>- {runError ? (- <p- className="pb-2 text-sm text-destructive"- data-testid="channel-run-error"- role="alert"- >- {runError}- </p>- ) : null}- {channel.active ? null : (- <p className="pb-2 text-sm text-muted-foreground" role="status">- This coworker has been deleted. The conversation stays readable,- but it can no longer reply.- </p>- )}- </>+ channel.active ? null : (+ <p className="pb-2 text-sm text-muted-foreground" role="status">+ This coworker has been deleted. The conversation stays readable,+ but it can no longer reply.+ </p>+ )}onSubmit={async (draft) => {// `draft.agentId` carries the @mentioned coworker, but nothing routes on it yet: this@@ -405,6 +391,17 @@ export function ChannelChat({* this is the one place the narrower fact is the honest one to draw a button from.*/stoppable={agent.isRunning || runsInFlight > 0}+ /*+ * At the END OF THE TRANSCRIPT rather than above the composer, which is where this used to+ * be. A turn that ends without an answer leaves a gap exactly where the reply was going to+ * appear, and the person is already looking at it; an explanation in the composer area is a+ * different part of the screen from the thing it explains.+ *+ * `runError` carries whatever ended the turn, in that thing's own words. A Bot that stopped+ * streaming says so, because the deployment's stall watchdog writes that sentence into the+ * run before closing it; see server/src/channels/stall-guard.ts.+ */+ stopped={runError ?? undefined}/></ConversationProvider>);diff --git a/app/src/components/channels/chat-transcript.tsx b/app/src/components/channels/chat-transcript.tsxindex 89406dc1..42b3ebfc 100644--- a/app/src/components/channels/chat-transcript.tsx+++ b/app/src/components/channels/chat-transcript.tsx@@ -38,6 +38,14 @@ type ChatTranscriptProps = {queued?: readonly QueuedMessage[];/** Take one back before it runs. Without it a queued line is shown but cannot be undone. */onRemoveQueued?: (id: string) => void;+ /**+ * Why the last turn ended without an answer, if it did.+ *+ * A sentence rather than a flag, because the reasons are not interchangeable: a Bot that refused,+ * a Bot whose endpoint is down and a Bot that simply stopped talking are three different things to+ * be told, and only the thing that ended the turn knows which one happened.+ */+ stopped?: string;};/** One shared empty array, so a screen without a queue does not hand down a new one per render. */@@ -91,6 +99,31 @@ function Thinking() {);}+/**+ * The turn ended and no answer came.+ *+ * In the same slot as `Thinking`, and for the same reason it is there: the person is looking at the+ * bottom of the transcript, immediately under their own message, because that is where the answer+ * was going to appear. Saying so above the composer put the explanation in a different part of the+ * screen from the gap it explains, and left the last thing in the conversation looking unfinished.+ *+ * NOT A MESSAGE, deliberately. It has no id, is never anchored, and is gone the moment the next turn+ * starts. Making it a transcript row would put a sentence into the conversation that nobody said,+ * and the conversation is sent back to the model on the next turn, so the Bot would then read its+ * own obituary as something it had written.+ */+function Stopped({ reason }: { reason: string }) {+ return (+ <p+ className="text-destructive text-sm"+ data-testid="transcript-stopped"+ role="alert"+ >+ {reason}+ </p>+ );+}+/*** Something the person said while the Bot was working, waiting its turn.*@@ -455,6 +488,7 @@ export function ChatTranscript({messages,onRemoveQueued,queued = EMPTY_QUEUE,+ stopped,}: ChatTranscriptProps) {/** NOT MEMOISED, AND THAT IS DELIBERATE. `useMemo` keyed on `messages` looks obviously right and@@ -545,11 +579,18 @@ export function ChatTranscript({),)}{/*- * Outside the item list, so it is not a message. It has no id, is never anchored, and- * disappears the moment the answer starts — giving it a `MessageScrollerItem` would ask+ * Outside the item list, so neither of these is a message. Each has no id, is never+ * anchored, and is gone by the next turn — giving one a `MessageScrollerItem` would ask* the scroller to measure and anchor something that exists for a second and a half.+ *+ * One or the other, never both: a turn that ended has stopped being in flight, and a+ * shimmering "Thinking" under a line saying the Bot stopped would contradict it.*/}- {waitingOnFirstToken ? <Thinking /> : null}+ {stopped ? (+ <Stopped reason={stopped} />+ ) : waitingOnFirstToken ? (+ <Thinking />+ ) : null}{/** Below the thinking line, and outside the item list for the same reason it is: these* are not yet turns. They have ids of their own, but they are this tab's ids and not thediff --git a/app/src/components/channels/conversation-view.tsx b/app/src/components/channels/conversation-view.tsxindex 9dd6ae58..13ac5135 100644--- a/app/src/components/channels/conversation-view.tsx+++ b/app/src/components/channels/conversation-view.tsx@@ -25,6 +25,7 @@ export function ConversationView({commands,disabled = false,pending = false,+ stopped,stoppable,queueWhileBusy = false,onSubmit,@@ -50,6 +51,8 @@ export function ConversationView({* drains on this falling.*/pending?: boolean;+ /** Why the last turn ended without an answer. Drawn at the end of the transcript, not here. */+ stopped?: string;/*** There is a run for Stop to abort, which is a narrower fact than `pending` and is the honest one* to draw a Stop button from. Defaults to `pending` for a caller with no gap between the two.@@ -209,6 +212,7 @@ export function ConversationView({apply({ id, type: "remove" });}}queued={queued}+ {...(stopped ? { stopped } : {})}/></div><div className="max-w-2xl mx-auto w-full px-0 pb-4 shrink-0">diff --git a/app/src/lib/audit/silence.ts b/app/src/lib/audit/silence.tsnew file mode 100644index 00000000..0a1e378c--- /dev/null+++ b/app/src/lib/audit/silence.ts@@ -0,0 +1,29 @@+/**+ * How a stalled turn reads on the audit page.+ *+ * The row for a Bot that stopped talking carries two numbers nothing else in the trail carries: how+ * long its stream had been quiet when the deployment gave up on it, and how much it had managed to+ * say first. They are the whole reason that row exists. An endpoint that dies halfway through an+ * answer and one that accepts a connection and never writes are different faults with different+ * fixes, and on the page they are the same line unless these two are drawn.+ *+ * Chunks, not events, because that is what was counted: one chunk can carry several AG-UI events and+ * the boundaries are the network's. Saying "events" here would be a number that looks precise and is+ * not. What a reader needs from it is whether it is zero, and that it says plainly.+ *+ * Returns null rather than a placeholder when the payload does not carry both. An older row written+ * before this was recorded should show nothing, not "0 chunks", which would be a claim about a Bot+ * that nobody ever measured.+ */+export function silenceOf(payload: Record<string, unknown>): string | null {+ const silentForMs = payload.silentForMs;+ const chunks = payload.chunks;+ if (typeof silentForMs !== "number" || typeof chunks !== "number") {+ return null;+ }++ const seconds = Math.max(1, Math.round(silentForMs / 1000));+ const quiet = `Silent for ${seconds}s`;+ if (chunks === 0) return `${quiet}, having said nothing at all`;+ return `${quiet}, after ${chunks} ${chunks === 1 ? "chunk" : "chunks"}`;+}diff --git a/app/src/lib/copilot/stopped-turn.ts b/app/src/lib/copilot/stopped-turn.tsnew file mode 100644index 00000000..adabd35f--- /dev/null+++ b/app/src/lib/copilot/stopped-turn.ts@@ -0,0 +1,60 @@+import { useAgent } from "@copilotkit/react-core/v2";+import { useEffect, useState } from "react";++/**+ * Why the last turn ended without an answer, for a surface that has to say so itself.+ *+ * A run can end three ways. It finishes, which needs no explanation. It fails in the browser, which+ * arrives as an error. Or the Bot's own stream stops producing anything and this deployment ends the+ * turn for it, which arrives as a RUN_ERROR carrying the sentence the server wrote (see+ * server/src/channels/stall-guard.ts). The last two both leave the same hole on screen: the composer+ * unlocks, the spinner disappears, and nothing says what happened.+ *+ * The reason is kept as a sentence rather than a flag because the reasons are not interchangeable. A+ * Bot that refused, a Bot whose endpoint is down and a Bot that simply stopped talking are three+ * different things to be told, and only the thing that ended the turn knows which one it was.+ */++/**+ * The sentence to show, in the words of whatever ended the turn.+ *+ * Falls back only when there is genuinely nothing to pass on. Saying "the Bot stopped without saying+ * why" is honest about that; inventing a cause would not be, and this is the one moment a person has+ * no other way to find out what went wrong.+ */+export function stoppedReason(reported: unknown): string {+ const said =+ reported instanceof Error+ ? reported.message+ : typeof reported === "string"+ ? reported+ : "";+ return said.trim() || "The Bot stopped without saying why.";+}++/**+ * Watch one Bot's runs and hold on to the reason the last one ended, if it ended badly.+ *+ * Bound by agent id rather than handed an agent, so a caller that only renders the packaged chat+ * does not have to reach for one: `useAgent` returns the same shared instance the chat itself binds+ * to, so this watches exactly the runs that chat starts.+ *+ * Cleared when the next run begins rather than on a timer. A sentence about a turn that is over+ * should stay until there is something newer to look at, and the person deciding when that is is the+ * one who sends the next message.+ */+export function useStoppedTurn(agentId: string): string | null {+ const { agent } = useAgent({ agentId });+ const [stopped, setStopped] = useState<string | null>(null);++ useEffect(() => {+ const subscription = agent.subscribe?.({+ onRunInitialized: () => setStopped(null),+ onRunErrorEvent: ({ event }) => setStopped(stoppedReason(event?.message)),+ onRunFailed: ({ error }) => setStopped(stoppedReason(error)),+ });+ return () => subscription?.unsubscribe();+ }, [agent]);++ return stopped;+}diff --git a/app/src/routes/_authed/_app/bot.tsx b/app/src/routes/_authed/_app/bot.tsxindex a0d56770..a3dd9022 100644--- a/app/src/routes/_authed/_app/bot.tsx+++ b/app/src/routes/_authed/_app/bot.tsx@@ -2,6 +2,7 @@ import { CopilotChat } from "@copilotkit/react-core/v2";import { createFileRoute } from "@tanstack/react-router";import { useActiveBot } from "@/lib/copilot/active-bot";import { useBotThread } from "@/lib/copilot/bot-thread";+import { useStoppedTurn } from "@/lib/copilot/stopped-turn";export const Route = createFileRoute("/_authed/_app/bot")({component: RouteComponent,@@ -18,6 +19,15 @@ function RouteComponent() {useActiveBot(agentId);// Minted by this deployment rather than by the chat, and the same one on the next visit.const threadId = useBotThread(agentId);+ /*+ * A turn that ends without an answer has to be said out loud here, because the packaged chat says+ * nothing. It reports a failed run to an `onError` prop and otherwise carries on as though the+ * turn simply finished: the composer unlocks, the spinner goes, and the transcript keeps the+ * person's own message with nothing under it. The banner that would have explained it belongs to+ * a provider this app does not mount.+ */+ const stopped = useStoppedTurn(agentId);+return (<div className="flex h-screen flex-col"><header className="border-b px-6 py-3">@@ -26,6 +36,23 @@ function RouteComponent() {Ask it to open a page and watch it work.</p></header>+ {/*+ * Under the header rather than at the end of the transcript, which is where the missing answer+ * was going to be and where the channel draws its own version of this. The packaged chat owns+ * that list and virtualises it, so reaching into it means replacing the whole message view and+ * taking on its scrolling. The cost of putting the sentence here instead is that it is not+ * beside the gap it explains; what it buys is that it is always on screen, whatever the+ * transcript has been scrolled to, and that it survives the next release of the chat.+ */}+ {stopped ? (+ <p+ className="border-b bg-destructive/10 px-6 py-2 text-destructive text-sm"+ data-testid="bot-chat-stopped"+ role="alert"+ >+ {stopped}+ </p>+ ) : null}<div className="min-h-0 flex-1">{/* Remount when switching Bots so chat state stays bound to the selected agent. */}{threadId ? (diff --git a/app/src/routes/_authed/admin/audit.tsx b/app/src/routes/_authed/admin/audit.tsxindex 182ec683..a6051ab0 100644--- a/app/src/routes/_authed/admin/audit.tsx+++ b/app/src/routes/_authed/admin/audit.tsx@@ -10,6 +10,7 @@ import {import { Button } from "@/components/ui/button";import { useBotNames } from "@/lib/agents/bot-names";import { auditEventsQueryOptions } from "@/lib/audit/queries";+import { silenceOf } from "@/lib/audit/silence";/*** Read surface for policy, computer, component, MCP, and credential audit events.@@ -38,7 +39,12 @@ const FILTERS = [search:"?eventType=computer.action_refused,mcp.call_rejected,component.refused,component.function_refused",},- { label: "Did not happen", search: "?eventType=computer.action_failed" },+ {+ label: "Did not happen",+ // A stalled stream belongs here. It is the same complaint as an action that was allowed and then+ // did not take: nothing was refused, and nothing came of it either.+ search: "?eventType=computer.action_failed,agent.stream_stalled",+ },{// Its own filter rather than a place in "Blocked". A Bot repeating itself has not been stopped by// anything, and putting it beside the refusals would make the refusals look less real.@@ -143,8 +149,12 @@ function Row({event.eventType === "component.refused" ||event.eventType === "component.function_refused" ||event.eventType === "mcp.call_rejected";- // Allowed by policy but not carried out.- const failed = event.eventType === "computer.action_failed";+ const stalled = event.eventType === "agent.stream_stalled";+ // Allowed by policy but not carried out. A stalled turn belongs in the same family: the Bot was+ // asked and the answer never arrived. Colour is how this table is read, and a row left in the+ // muted foreground reads as "Allowed", which a turn nobody ever got an answer to was not.+ const failed = event.eventType === "computer.action_failed" || stalled;+ const silence = stalled ? silenceOf(payload) : null;return (<tr className="border-border border-t align-top">@@ -244,6 +254,14 @@ function Row({{payload.failure}</div>) : null}+ {/*+ * The two numbers the stall row is worth reading for. Without them every stalled turn looks+ * the same, and the difference between an endpoint that dies halfway through an answer and+ * one that never begins is the difference between a slow Bot and a dead one.+ */}+ {silence ? (+ <div className="mt-0.5 text-xs text-muted-foreground">{silence}</div>+ ) : null}{/* Show concrete policy rules, but suppress the uninformative default `true` allow rule. */}{decision.rule && decision.rule !== "true" ? (<div className="mt-0.5 font-mono text-xs text-muted-foreground">@@ -275,6 +293,9 @@ const NAMED_TARGETS = new Set([const DECISIONS: Record<string, string> = {"bot.declined": "The Bot declined",+ // Not a refusal, so not the refusal colour: nothing was blocked. The Bot was asked and never+ // answered, which is the same complaint as an action that was allowed and then did not happen.+ "agent.stream_stalled": "The Bot stopped responding","computer.policy_loaded": "Boundary at start-up","computer.isolation_loaded": "Isolation at start-up","computer.control_taken": "A person took the wheel",diff --git a/app/tests/audit-silence.test.ts b/app/tests/audit-silence.test.tsnew file mode 100644index 00000000..089f1a30--- /dev/null+++ b/app/tests/audit-silence.test.ts@@ -0,0 +1,44 @@+import { describe, expect, test } from "bun:test";+import { silenceOf } from "../src/lib/audit/silence";++/**+ * The line that tells a stalled-turn row apart from every other stalled-turn row.+ *+ * The distinction the audit trail exists to draw here is between a Bot that dies partway through an+ * answer and one that accepts the request and never writes, so the zero case is the one that has to+ * read unmistakably.+ */++describe("what a stalled turn says on the audit page", () => {+ test("names the silence and how much came before it", () => {+ expect(silenceOf({ silentForMs: 120_000, chunks: 14 })).toBe(+ "Silent for 120s, after 14 chunks",+ );+ });++ test("says outright when nothing ever arrived", () => {+ expect(silenceOf({ silentForMs: 60_000, chunks: 0 })).toBe(+ "Silent for 60s, having said nothing at all",+ );+ });++ test("counts one chunk as one", () => {+ expect(silenceOf({ silentForMs: 60_000, chunks: 1 })).toBe(+ "Silent for 60s, after 1 chunk",+ );+ });++ test("never says nought seconds, whatever the deployment configured", () => {+ expect(silenceOf({ silentForMs: 60, chunks: 0 })).toBe(+ "Silent for 1s, having said nothing at all",+ );+ });++ test("draws nothing for a row that does not carry the numbers", () => {+ // An older row, written before these were recorded. Showing "0 chunks" there would be a claim+ // about a Bot nobody measured.+ expect(silenceOf({ bot: "risk-analyst" })).toBeNull();+ expect(silenceOf({ silentForMs: 60_000 })).toBeNull();+ expect(silenceOf({ silentForMs: "60000", chunks: 0 })).toBeNull();+ });+});diff --git a/app/tests/stopped-turn.test.ts b/app/tests/stopped-turn.test.tsnew file mode 100644index 00000000..6842b940--- /dev/null+++ b/app/tests/stopped-turn.test.ts@@ -0,0 +1,41 @@+import { describe, expect, test } from "bun:test";+import { stoppedReason } from "../src/lib/copilot/stopped-turn";++/**+ * What a person is told when a turn ends and no answer came.+ *+ * The cases are the three things that actually arrive: the sentence the deployment's stall watchdog+ * wrote into the run, an error thrown in the browser, and nothing at all.+ */++describe("the reason a turn ended", () => {+ test("passes on what ended the turn, in its own words", () => {+ expect(+ stoppedReason(+ "Risk Analyst stopped responding. Nothing arrived from it for 2 minutes, so this turn was ended. Ask again, or check that the Bot is running.",+ ),+ ).toContain("Risk Analyst stopped responding");+ });++ test("reads an Error the same way, because a failed run carries one", () => {+ expect(+ stoppedReason(new Error("The endpoint refused the connection")),+ ).toBe("The endpoint refused the connection");+ });++ test("says so plainly when nothing was reported, rather than inventing a cause", () => {+ // This is the one moment a person has no other way to find out what went wrong, so a guess here+ // would be worse than an admission.+ expect(stoppedReason(undefined)).toBe(+ "The Bot stopped without saying why.",+ );+ expect(stoppedReason("")).toBe("The Bot stopped without saying why.");+ expect(stoppedReason(" ")).toBe("The Bot stopped without saying why.");+ expect(stoppedReason(new Error(""))).toBe(+ "The Bot stopped without saying why.",+ );+ expect(stoppedReason({ message: "not a string or an Error" })).toBe(+ "The Bot stopped without saying why.",+ );+ });+});diff --git a/server/src/audit.ts b/server/src/audit.tsindex 405ea43f..3a826e7b 100644--- a/server/src/audit.ts+++ b/server/src/audit.ts@@ -41,6 +41,20 @@ export const auditEventTypes = ["connector.sync_failed","knowledge.searched","agent.invoked",+ /**+ * A Bot's stream stopped producing anything and the turn was ended for it.+ *+ * Recorded because a Bot is somebody else's infrastructure and this is the failure it has that+ * nothing else in the trail can show. Every other row here is something that happened; this one is+ * the absence of anything happening, which leaves no trace of its own.+ *+ * It is also the sort of thing nobody notices is happening repeatedly. One hung turn reads as a+ * bad afternoon and gets a shrug; the same Bot hanging twice a day for a month is a fact about an+ * endpoint, and it only becomes visible when somebody can count it. The row names the Bot, how+ * long its stream was silent, and how many chunks it managed first, so a reader can tell an+ * endpoint that dies mid-answer from one that never answers at all.+ */+ "agent.stream_stalled","mcp.call_succeeded","mcp.call_rejected",// Every action a Bot takes on its computer, allowed or refused. Both, always: a trail that recordsdiff --git a/server/src/channels/stall-guard.ts b/server/src/channels/stall-guard.tsnew file mode 100644index 00000000..b6aca710--- /dev/null+++ b/server/src/channels/stall-guard.ts@@ -0,0 +1,392 @@+/**+ * The watch on a Bot's own stream, and what happens when it stops saying anything.+ *+ * A Bot is any AG-UI endpoint, which makes it infrastructure this deployment does not run: it will+ * be redeployed mid-run, its upstream will time out, it will accept a connection and then write+ * nothing. None of that is exceptional. What was missing was anybody noticing: an open stream that+ * never produces another byte leaves the channel busy, the composer locked and a person watching a+ * Bot that appears to be thinking and is not.+ *+ * The seam is the Bot's own HTTP response, wrapped on the way in. In Intelligence mode this server's+ * reply to the browser is a JSON envelope, not a stream, and the AG-UI events reach the browser over+ * a WebSocket to the gateway (see the module comment on turn-watchdog.ts for what was measured).+ * The Bot's response, however, is an ordinary streaming HTTP body that this process reads to the+ * end, so it is both the thing that stalls and the only place a stall can be seen.+ *+ * A PASS-THROUGH RELAY IS THE ONLY ACCEPTABLE SHAPE HERE, and the relay below is an identity+ * transform with nothing in it at all. It must not buffer, because a Bot's answer is streamed a+ * token at a time and holding chunks back to inspect them would turn a live answer into a paragraph+ * that lands all at once. It must not await anything, because a chunk delayed here is a chunk+ * delayed on somebody's screen. And it must not decide anything about the bytes, because the moment+ * it parses them it can be wrong about them, and a watchdog that can misread a working run into a+ * broken one is worse than the failure it was added for.+ *+ * THE CLOCK IS KEPT BY THE PUMP AND NOT BY THE RELAY, which is the difference between watching the+ * Bot and watching whoever is reading it. A `TransformStream` runs its transform only once the+ * readable side is being pulled, so timestamping in there would record when the CONSUMER took a+ * chunk rather than when the Bot produced one; a consumer that paused for longer than the timeout+ * would then be reported as a Bot that had gone silent, on a run that was streaming the whole time.+ * The pump times the resolution of each read from the Bot instead, and stops the clock entirely+ * while it waits for the consumer to take delivery, so the only quiet ever counted is quiet on the+ * wire. In this deployment that consumer is the Intelligence runner publishing every event on to+ * the gateway over the network, which is exactly the sort of thing that pauses.+ *+ * On a stall it writes one RUN_ERROR event into the same stream and closes it. RUN_ERROR is the+ * event both surfaces already subscribe to, so nothing downstream had to learn a new one. What+ * neither of them had was anywhere to put it, and both now draw the sentence themselves: see+ * app/src/components/channels/chat-transcript.tsx for the channel and+ * app/src/routes/_authed/_app/bot.tsx for the direct Bot chat. The packaged chat draws nothing of+ * its own for a failed run — the banner that would have done it belongs to the v1 provider this app+ * does not mount, and is suppressed even there unless the dev console is switched on. AG-UI permits+ * RUN_ERROR at any point in a stream, including as the very first event, which is what a Bot that+ * never spoke produces.+ */+import { type AuditStore, recordAuditEvent } from "../audit";+import { type StalledStream, TurnWatchdog } from "./turn-watchdog";++/** The fetch an `HttpAgent` uses, as @ag-ui/client 0.0.57 declares it. */+export type AgentFetch = (+ url: string,+ requestInit: RequestInit,+) => Promise<Response>;++/** Which Bot a watched stream belongs to. The name is for the sentence a person reads. */+export type WatchedBot = { id: string; name: string };++export type StallGuardOptions = {+ /** Silence this long ends the turn. Zero or less leaves every stream untouched. */+ stallMs: number;+ /** Absent leaves the trail without stall rows; the recovery still happens. */+ auditStore?: AuditStore;+ now?: () => number;+};++export type StallGuard = {+ /**+ * Wrap a fetch so that every stream it opens for this Bot is watched.+ *+ * Returns the fetch it was given, unchanged, when the watchdog is off. A deployment that has not+ * configured a timeout gets the code path it had before this existed, rather than a wrapper that+ * happens never to fire.+ */+ watch: (bot: WatchedBot, fetchImplementation?: AgentFetch) => AgentFetch;+ /** Stop sweeping. For orderly shutdown and for tests that must not leave a timer behind. */+ stop: () => void;+};++/**+ * The one framing a sentence must never be written into.+ *+ * @ag-ui/client 0.0.57 picks its parser by comparing the response's content type to exactly this+ * media type: this value means protocol buffers, and every other value, including one it has never+ * seen, is read as server-sent events. The rule here is that same rule, deliberately, because the+ * question being asked is not "what did the Bot mean by this header" but "what will the thing at the+ * other end of this relay try to parse". Testing for `text/event-stream` instead was stricter than+ * the client and silently worse: a Bot serving SSE bytes under some other content type had its+ * stream closed with nothing in it, so the run ended, the composer unlocked, and nobody was told+ * anything on either surface.+ *+ * Guessing the other way is the failure worth avoiding, and this still avoids it. Appending SSE+ * bytes to a protobuf stream would corrupt a run that was merely slow.+ */+const PROTOBUF_CONTENT_TYPE = "application/vnd.ag-ui.event+proto";++const ENCODER = new TextEncoder();++/**+ * How often to look, derived from the deadline rather than fixed.+ *+ * A quarter of the timeout means a stall is reported within a quarter of the deadline of it+ * happening, which is proportionate: nobody watching a two-minute limit cares about thirty seconds+ * of reporting lag, and nobody testing a sixty-millisecond limit can wait a second for it. Bounded+ * at both ends so a very long timeout still sweeps every second and a very short one does not spin.+ */+function sweepIntervalFor(stallMs: number): number {+ return Math.min(1_000, Math.max(50, Math.floor(stallMs / 4)));+}++/** What one watched stream needs in order to be ended from outside it. */+type OpenStream = {+ bot: WatchedBot;+ writer: WritableStreamDefaultWriter<Uint8Array>;+ /** Cancels the Bot's side, so a wedged endpoint does not keep a socket here forever. */+ cancelUpstream: () => void;+ /** Whether an event may be written into this stream. See PROTOBUF_CONTENT_TYPE. */+ sse: boolean;+ /**+ * The request body, held as the string it was already serialised to.+ *+ * Read only when a stall has to name the turn it ended, so the thread and run identifiers cost a+ * JSON parse on the rare path rather than on every run. Nothing else in the request is wanted, and+ * nothing from it is recorded beyond those two ids.+ */+ requestBody: string | null;+ /** Set by whichever of the pump or the stall gets there first. */+ finished: boolean;+};++export function createStallGuard(options: StallGuardOptions): StallGuard {+ const streams = new Map<string, OpenStream>();+ const sweepEveryMs = sweepIntervalFor(options.stallMs);+ let sweeper: ReturnType<typeof setInterval> | undefined;++ const watchdog = new TurnWatchdog({+ stallMs: options.stallMs,+ ...(options.now ? { now: options.now } : {}),+ onStall: (stalled) => {+ void giveUp(stalled);+ },+ });++ /**+ * The sweep runs only while something is being watched.+ *+ * An idle deployment therefore holds no timer at all, and the one it holds while a Bot is+ * answering is unreferenced, so it can never be the reason a process refuses to exit.+ */+ function arm(): void {+ if (sweeper !== undefined) return;+ sweeper = setInterval(() => {+ watchdog.sweep();+ if (watchdog.watching === 0) disarm();+ }, sweepEveryMs);+ sweeper.unref();+ }++ function disarm(): void {+ if (sweeper === undefined) return;+ clearInterval(sweeper);+ sweeper = undefined;+ }++ /**+ * Take a stream off the watch, exactly once.+ *+ * Both the pump and the stall want to be the one that finishes a stream, and which of them gets+ * there first depends on timing nobody controls. Returning the stream only to the first caller is+ * what stops a stalled stream also being closed by its own pump a moment later, and stops a+ * cancelled read being reported as a stall.+ */+ function release(id: string): OpenStream | undefined {+ const stream = streams.get(id);+ if (!stream || stream.finished) return undefined;+ stream.finished = true;+ streams.delete(id);+ watchdog.close(id);+ return stream;+ }++ /**+ * End a turn whose Bot has stopped talking.+ *+ * NOTHING THE PERSON NEEDS WAITS ON ANYTHING THAT CAN FAIL TO SETTLE. Every step of the recovery+ * below is either synchronous or queued and deliberately unawaited, so the socket is released, the+ * sentence is queued and the stream is closed within this tick whatever else in the deployment is+ * unwell. A wedged Bot may also have a consumer that has stopped reading, in which case those+ * writes never settle; awaiting them would leave the spinner and the locked composer this whole+ * file exists to end.+ *+ * The audit row comes last, after the close, and the same argument is what puts it there. The+ * store is a bare insert against the pool every other write in this deployment shares (see+ * `createAuditStore`) with no deadline of its own, and a Bot is most likely to hang in exactly the+ * conditions where that pool is saturated or Postgres is unreachable. Recovering first and+ * recording second costs a row if the process dies in between, and a lost row is a smaller loss+ * than a watchdog that fires and then fails open.+ */+ async function giveUp(stalled: StalledStream): Promise<void> {+ const stream = release(stalled.id);+ if (!stream) return;++ const turn = turnOf(stream.requestBody);+ console.error(+ JSON.stringify({+ type: "agent-stream-stalled",+ bot: stream.bot.id,+ silentForMs: stalled.silentForMs,+ chunks: stalled.chunks,+ ...(turn ? { thread: turn.threadId, run: turn.runId } : {}),+ note: "The Bot's stream produced nothing for the configured timeout, so the turn was ended.",+ }),+ );++ // The Bot's side goes first, so a socket into an endpoint that will never answer is released+ // whatever the browser's side of the stream is doing.+ stream.cancelUpstream();++ if (stream.sse) {+ // The write is queued ahead of the close, so it either lands in order or neither does.+ void stream.writer+ .write(stalledEvent(stream.bot.name, options.stallMs))+ .catch(() => undefined);+ }+ void stream.writer.close().catch(() => undefined);++ if (options.auditStore) {+ await recordAuditEvent(options.auditStore, {+ eventType: "agent.stream_stalled",+ targetType: "agent",+ targetId: stream.bot.id,+ payload: {+ bot: stream.bot.id,+ silentForMs: stalled.silentForMs,+ // Chunks, not events. One chunk can carry several AG-UI events and the boundaries are the+ // network's, so saying "events" here would be a number that looks precise and is not. Zero+ // is the fact worth reading: the Bot answered the request and then said nothing at all.+ chunks: stalled.chunks,+ ...(turn ? { thread: turn.threadId, run: turn.runId } : {}),+ },+ }).catch(() => undefined);+ }+ }++ function watch(+ bot: WatchedBot,+ fetchImplementation?: AgentFetch,+ ): AgentFetch {+ const inner: AgentFetch =+ fetchImplementation ?? ((url, requestInit) => fetch(url, requestInit));+ if (!watchdog.enabled) return inner;++ return async (url, requestInit) => {+ const response = await inner(url, requestInit);+ const body = response.body;+ // A refusal or an empty body is not a stream and has already ended. Passing it through+ // untouched keeps every error path exactly as it was, which is the point of the pass-through.+ if (!response.ok || body === null) return response;++ const id = crypto.randomUUID();+ // An identity transform, with no transform in it. All it is here for is a writable end this+ // process can put one last event into and close from outside the pump; see `giveUp`.+ const relay = new TransformStream<Uint8Array, Uint8Array>();+ const reader = body.getReader();+ const writer = relay.writable.getWriter();++ streams.set(id, {+ bot,+ writer,+ cancelUpstream: () => {+ void reader.cancel().catch(() => undefined);+ },+ sse: response.headers.get("content-type") !== PROTOBUF_CONTENT_TYPE,+ requestBody:+ typeof requestInit.body === "string" ? requestInit.body : null,+ finished: false,+ });+ watchdog.open({ id, botId: bot.id });+ arm();++ void pump(id, reader, writer);++ // The same response, with the body replaced. Status and headers are carried over because the+ // AG-UI client reads the content type off this response to choose its parser.+ return new Response(relay.readable, {+ status: response.status,+ statusText: response.statusText,+ headers: response.headers,+ });+ };+ }++ /**+ * Move bytes across, keep the clock, and end the stream the way the Bot ended it.+ *+ * This loop is the only place that knows which of its two waits is which, which is why the timing+ * lives here. A read that resolves is the Bot having said something and is the only evidence the+ * endpoint is alive. A write that has not resolved yet is the far side of the relay not having+ * taken delivery, and says nothing about the Bot at all, so the clock is stopped across it and+ * restarted when the handover completes. Timing the relay instead would have reported a paused+ * consumer as a silent Bot, which is the one mistake this must not make.+ *+ * A read that throws is passed on as an abort rather than as a clean close. A broken connection+ * carries a real reason, and relabelling it as an ending would file a transport failure as a Bot+ * that merely stopped, which is exactly the distinction this file exists to be able to draw.+ */+ async function pump(+ id: string,+ reader: ReadableStreamDefaultReader<Uint8Array>,+ writer: WritableStreamDefaultWriter<Uint8Array>,+ ): Promise<void> {+ try {+ for (;;) {+ const { done, value } = await reader.read();+ if (done) break;+ watchdog.record(id);+ watchdog.pause(id);+ await writer.write(value);+ watchdog.resume(id);+ }+ if (release(id)) await writer.close().catch(() => undefined);+ } catch (error) {+ if (release(id)) await writer.abort(error).catch(() => undefined);+ }+ }++ return {+ watch,+ stop: disarm,+ };+}++/**+ * The turn, read from the request the stalled stream was answering.+ *+ * Returns null rather than throwing for anything unexpected. A stall is already a failure, and a+ * malformed body is no reason to lose the row that records it; the Bot is named either way, and the+ * Bot is what a deployment counts these by.+ */+function turnOf(+ body: string | null,+): { threadId: string; runId: string } | null {+ if (!body) return null;+ try {+ const parsed: unknown = JSON.parse(body);+ if (!parsed || typeof parsed !== "object") return null;+ const { threadId, runId } = parsed as {+ threadId?: unknown;+ runId?: unknown;+ };+ return typeof threadId === "string" && typeof runId === "string"+ ? { threadId, runId }+ : null;+ } catch {+ return null;+ }+}++/**+ * The one event a stalled stream gets, as a server-sent event.+ *+ * Written by hand rather than through an encoder because the framing is two lines and adding a+ * dependency to produce them would be the larger change. The AG-UI client parses `data:` lines as+ * JSON and validates them against its own schemas, so this is the same shape a Bot would have sent.+ *+ * The wording is the whole of what a person gets. Both surfaces draw this message and nothing else+ * around it, so it names the Bot, says what was observed rather than what was concluded, says the+ * turn is over, and says what to do next. No identifiers and no milliseconds: a run id in a sentence+ * is a thing the reader has to decide to ignore, and it is not what they came to find out.+ */+function stalledEvent(botName: string, stallMs: number): Uint8Array {+ const event = {+ type: "RUN_ERROR",+ message:+ `${botName} stopped responding. Nothing arrived from it for ${inWords(stallMs)}, ` ++ "so this turn was ended. Ask again, or check that the Bot is running.",+ code: "AGENT_STREAM_STALLED",+ };+ return ENCODER.encode(`data: ${JSON.stringify(event)}\n\n`);+}++/**+ * The timeout as a person would say it, because the sentence above is read by one.+ *+ * Floored at a second. A timeout below one is only ever a test's, and "nothing arrived from it for+ * 0 seconds" is a sentence that makes a reader doubt everything else on the screen.+ */+function inWords(ms: number): string {+ if (ms >= 60_000 && ms % 60_000 === 0) {+ const minutes = ms / 60_000;+ return minutes === 1 ? "a minute" : `${minutes} minutes`;+ }+ const seconds = Math.max(1, Math.round(ms / 1_000));+ return seconds === 1 ? "a second" : `${seconds} seconds`;+}diff --git a/server/src/channels/turn-watchdog.ts b/server/src/channels/turn-watchdog.tsnew file mode 100644index 00000000..72d6ce6d--- /dev/null+++ b/server/src/channels/turn-watchdog.ts@@ -0,0 +1,240 @@+/**+ * Whether a turn has gone quiet, which is not the same question as whether it has taken a long time.+ *+ * A turn may legitimately run for an hour. A Bot reading a long document, or a model writing slowly,+ * produces events the whole way and there is nothing wrong with it. A turn whose stream has produced+ * nothing at all for `stallMs` is wedged. Telling those apart by activity rather than by duration is+ * the whole design, and the difference matters: a duration limit puts a ceiling on how much work a+ * Bot is allowed to do, which nobody asked for, whereas a silence limit puts a ceiling on how long a+ * person is asked to watch a spinner, which is the actual complaint.+ *+ * SILENCE ONLY MEANS ANYTHING WHILE THIS PROCESS IS WAITING TO HEAR FROM THE BOT. The Bot's stream+ * is relayed onward as it arrives, and whoever reads the relayed copy is entitled to take its time:+ * in Intelligence mode that reader is publishing every event on to the gateway over the network. A+ * chunk sitting in that handover is quiet that says nothing about the endpoint, and counting it+ * would end a run that was streaming perfectly and put the blame on a Bot for a pause on this side+ * of the wire. That is the one outcome a watchdog must never produce, so the clock is stopped for+ * the handover and started again when it completes, which is what `pause` and `resume` are for.+ *+ * WHAT THE STREAM ACTUALLY DOES IN THIS PRODUCT, measured before any of this was designed, because+ * the answer decides whether a long tool call would be reported as a stall.+ *+ * OpenBot runs in Intelligence mode and has no other mode, so `POST /api/copilotkit/agent/:id/run`+ * does not stream at all. It answers `Content-Type: application/json` with a fixed Content-Length in+ * about a second, carrying a join token, and the AG-UI events reach the browser over a WebSocket to+ * the Intelligence gateway. Wrapping that response body would watch a JSON envelope go past. The+ * stream that can actually stall is the one on the other side of this server: the Bot's own AG-UI+ * response, read here, event by event, and republished to the gateway. That is the stream this+ * watchdog is pointed at, and stall-guard.ts is where it is wrapped.+ *+ * The computer tools are frontend tools, executed in the browser, and the run ENDS before the+ * browser runs one. The Bot emits TOOL_CALL_START, ARGS and END and then RUN_FINISHED, its stream+ * closes, the browser executes the tool, and the CopilotKit client opens a SECOND run carrying the+ * result. Verified in @copilotkit/core 1.67.1, whose `processAgentResult` executes frontend tools+ * only after `agent.runAgent` has resolved and then re-enters `runAgent`, and against `agent-bot` in+ * this repository, which writes its tool calls followed by RUN_FINISHED and closes the stream.+ *+ * So no exemption for tool calls exists here, and none is needed. A browser tool that takes ten+ * minutes holds no stream open, because there is no stream open while it runs. An exemption would be+ * a hole in the only thing this watchdog does, opened for a case that does not occur.+ *+ * It holds no timer of its own. The clock is injected and `sweep` is called by whoever owns the+ * schedule, so a test can move time by a minute without waiting one, and so the decision about when+ * to look is made in one place rather than hidden in here.+ */++/** Milliseconds, from whatever source the caller trusts. Injected so tests need not sleep. */+export type Clock = () => number;++/**+ * What the watch is told about a stream, which is deliberately almost nothing: a way to find it+ * again and the Bot on the far end. Never the bytes, because nothing in here is allowed to read+ * them, and never the response, because holding one would make this a party to the stream rather+ * than an observer of it.+ */+export type WatchedStream = {+ /**+ * How the caller finds this stream again. Minted per stream rather than taken from the run,+ * because the id has to exist before the first byte arrives and the run identifiers are inside the+ * request the stream is answering.+ */+ id: string;+ /** The Bot on the far end, so a stall names what went quiet rather than which socket it was. */+ botId: string;+};++export type StalledStream = WatchedStream & {+ /** How long it had been silent when it was given up on. Never less than `stallMs`. */+ silentForMs: number;+ /** Chunks that arrived before the silence. Zero means the Bot never said anything at all. */+ chunks: number;+};++export type TurnWatchdogOptions = {+ /**+ * Silence this long means the stream is wedged.+ *+ * Zero, or anything below it, switches the watchdog off completely: `open` records nothing and+ * `sweep` has nothing to find. A deployment that has not asked for a watchdog gets the behaviour+ * it had before there was one, rather than a watchdog with a surprising number in it.+ */+ stallMs: number;+ /** Called once per stalled stream, after the stream has been taken off the watch. */+ onStall: (stream: StalledStream) => void;+ now?: Clock;+};++type OpenStream = WatchedStream & {+ lastChunkAt: number;+ chunks: number;+ /** While true the clock is not running, because the wait is not on the Bot. See `pause`. */+ paused: boolean;+};++export class TurnWatchdog {+ private readonly stallMs: number;+ private readonly onStall: (stream: StalledStream) => void;+ private readonly now: Clock;+ private readonly streams = new Map<string, OpenStream>();++ constructor(options: TurnWatchdogOptions) {+ this.stallMs = options.stallMs;+ this.onStall = options.onStall;+ this.now = options.now ?? (() => Date.now());+ }++ /** Whether this deployment asked for a watchdog at all. */+ get enabled(): boolean {+ return this.stallMs > 0;+ }++ /**+ * Read by whoever owns the sweep, so a process with nothing to watch can stop looking rather than+ * hold a timer that ticks for the life of the deployment.+ */+ get watching(): number {+ return this.streams.size;+ }++ /**+ * Start watching a stream.+ *+ * The clock starts here rather than at the first chunk, so a Bot that accepts a connection and+ * then says nothing at all is caught by the same rule as one that stops halfway. That case is the+ * common one: an endpoint that has been redeployed answers the request and never writes.+ *+ * Opening an id that is already open restarts its clock rather than raising. A second open for the+ * same stream can only mean the caller believes it is alive, and the safe reading of that is to+ * believe them.+ */+ open(stream: WatchedStream): void {+ if (!this.enabled) return;+ this.streams.set(stream.id, {+ ...stream,+ lastChunkAt: this.now(),+ chunks: 0,+ paused: false,+ });+ }++ /**+ * Note that something arrived from the Bot.+ *+ * Called when a read off the Bot's own body resolves, which is the only moment this process learns+ * anything about the endpoint. Anywhere later in the relay would be measuring how promptly the+ * next component took delivery instead.+ *+ * An id that is not being watched is ignored rather than started, and that is what makes the+ * callback fire exactly once per stream: `sweep` removes a stream as it reports it, so a chunk+ * that arrives from a Bot after it was given up on cannot quietly begin a second watch which would+ * stall again and report the same wedged turn twice.+ */+ record(id: string): void {+ const stream = this.streams.get(id);+ if (!stream) return;+ stream.lastChunkAt = this.now();+ stream.chunks += 1;+ }++ /**+ * Stop counting silence against a stream, because the wait has stopped being the Bot's.+ *+ * Between one chunk and the next, this process is not waiting on the endpoint at all: it is+ * waiting for whoever reads the relayed stream to take the chunk it already has. That wait can be+ * long for reasons that have nothing to do with the Bot, and a Bot that is streaming steadily must+ * not be ended because the far side of the relay went to sleep. Counting only the stretches spent+ * waiting on the endpoint is what keeps the number in the audit row true to its own name.+ *+ * The cost is that a stream whose reader stops for good is never reported. It is not silent from+ * the Bot, so nothing here ends it, and it stays in the watch until the relay errors and the pump+ * releases it. That is the right way round: believing a healthy Bot is broken ends somebody's+ * turn, and believing a broken reader is healthy costs an entry in a map.+ */+ pause(id: string): void {+ const stream = this.streams.get(id);+ if (!stream) return;+ stream.paused = true;+ }++ /**+ * Start counting again, from now rather than from the last chunk.+ *+ * The stretch that just ended was not the Bot's, so it must not be carried forward into the next+ * deadline; a handover that took most of the timeout would otherwise leave the Bot a sliver of it.+ */+ resume(id: string): void {+ const stream = this.streams.get(id);+ if (!stream) return;+ stream.lastChunkAt = this.now();+ stream.paused = false;+ }++ /** Stop watching. Unknown ids are ignored, so closing a stream that already stalled is harmless. */+ close(id: string): void {+ this.streams.delete(id);+ }++ /**+ * Reports every stream that has gone quiet, and returns how many there were.+ *+ * Each one is removed before its callback runs, so a callback that throws still leaves the+ * watchdog consistent, and a slow callback cannot be entered twice for the same stream.+ */+ sweep(): number {+ if (!this.enabled) return 0;+ const now = this.now();+ let stalled = 0;++ for (const [id, stream] of this.streams) {+ if (stream.paused) continue;+ const silentForMs = now - stream.lastChunkAt;+ if (silentForMs < this.stallMs) continue;++ this.streams.delete(id);+ stalled += 1;+ try {+ this.onStall({+ id,+ botId: stream.botId,+ silentForMs,+ chunks: stream.chunks,+ });+ } catch (error) {+ // One Bot's failure must not stop the others being closed. This is the same blast-radius+ // argument the process-level rejection handler in index.ts makes: a wedged Bot is somebody+ // else's infrastructure, and the sweep it happens to be in is holding every other person's+ // stuck turn. Logged loudly, because a watchdog that fails silently is worse than none.+ console.error(+ JSON.stringify({+ type: "turn-watchdog-callback-error",+ stream: id,+ bot: stream.botId,+ error: String(error),+ }),+ );+ }+ }++ return stalled;+ }+}diff --git a/server/src/config.ts b/server/src/config.tsindex d9f8bd9f..18ad61fb 100644--- a/server/src/config.ts+++ b/server/src/config.ts@@ -34,6 +34,15 @@ export type DeploymentConfig = {deploymentId: string | undefined;tenantPackageDirectory: string;runtime: RuntimeCapabilities;+ /**+ * How long a Bot's stream may say nothing before this deployment ends the turn, in milliseconds.+ *+ * Zero means no watchdog, and an unset variable means zero. A turn that is ended is a turn+ * somebody loses, so a deployment that has not said it wants that gets the behaviour it already+ * had. `.env.example` ships a value, so a new clone starts with the watch on and an upgraded+ * deployment does not acquire it without being asked.+ */+ agentStallTimeoutMs: number;oauth: {google?: { clientId: string; clientSecret: string };};@@ -348,6 +357,31 @@ function actionPolicy(environment: Environment): ActionPolicy | undefined {return result.policy;}+/**+ * How long silence on a Bot's stream is allowed to last.+ *+ * Refuses to start on anything that is not a whole number of milliseconds, rather than falling back+ * to the default. Same reasoning as the action policy above it: an operator who meant to write a+ * two-minute timeout and typed something else would otherwise get a running deployment with a+ * silently different boundary, and no indication that anything was wrong.+ *+ * Zero is a legitimate value and means off. It is not the same as a malformed one.+ */+function agentStallTimeoutMs(environment: Environment): number {+ const raw = optional(environment, "AGENT_STALL_TIMEOUT_MS");+ if (!raw) {+ return 0;+ }++ const milliseconds = Number(raw);+ if (!Number.isInteger(milliseconds) || milliseconds < 0) {+ throw new Error(+ "AGENT_STALL_TIMEOUT_MS must be a whole number of milliseconds, or 0 to switch the watchdog off",+ );+ }+ return milliseconds;+}+export function loadConfig(environment: Environment = process.env,): DeploymentConfig {@@ -364,6 +398,7 @@ export function loadConfig(tenantPackageDirectory:optional(environment, "TENANT_PACKAGE_DIR") ?? "../examples/fintech",runtime: runtimeCapabilities(environment),+ agentStallTimeoutMs: agentStallTimeoutMs(environment),oauth: { google },auth: authConfig(environment, google),devNoAuth: devAuthEnabled(environment),diff --git a/server/src/copilot.ts b/server/src/copilot.tsindex 5c00e36c..24258237 100644--- a/server/src/copilot.ts+++ b/server/src/copilot.ts@@ -7,6 +7,7 @@ import {} from "@copilotkit/runtime/v2";import { createCopilotHonoHandler } from "@copilotkit/runtime/v2/hono";import type { AgentActor } from "./agents/profile-types";+import type { StallGuard } from "./channels/stall-guard";import type { DeploymentConfig } from "./config";/**@@ -193,9 +194,14 @@ export function buildAgents(agents: RegisteredAgent[],model: RuntimeModel,apiKey: string | null,+ /** Absent leaves every stream unwatched, which is what an unconfigured timeout means. */+ stallGuard?: StallGuard,): Record<string, AbstractAgent> {return Object.fromEntries(- agents.map((agent) => [agent.id, buildAgent(agent, model, apiKey)]),+ agents.map((agent) => [+ agent.id,+ buildAgent(agent, model, apiKey, stallGuard),+ ]),);}@@ -203,6 +209,7 @@ function buildAgent(agent: RegisteredAgent,model: RuntimeModel,apiKey: string | null,+ stallGuard?: StallGuard,): AbstractAgent {if (agent.type === "built_in") {return new BuiltInAgent(builtInAgentConfiguration(agent, model, apiKey));@@ -210,7 +217,7 @@ function buildAgent(if (agent.type === "unavailable") {return new UnavailableAgent(agent);}- return remoteAgentWithStandingRole(agent);+ return remoteAgentWithStandingRole(agent, stallGuard);}/**@@ -220,14 +227,24 @@ function buildAgent(* so the same coworker works against any endpoint that speaks the protocol. Any copy of the standing* message already in the conversation is dropped: the endpoint must receive exactly one, first,* however many times the thread has been replayed.+ *+ * The stall watch goes on the fetch rather than into that middleware, because the middleware works+ * in AG-UI events and a stall is the absence of one. The thing that has to be watched is the+ * response body, and the fetch is where this deployment still holds it.*/-function remoteAgentWithStandingRole(agent: RegisteredRemoteAgent) {+function remoteAgentWithStandingRole(+ agent: RegisteredRemoteAgent,+ stallGuard?: StallGuard,+) {const remote = new HttpAgent({url: agent.endpoint,agentId: agent.id,// The customer's own key, if their agent sits behind one. `HttpAgentConfig` is// `{ url, headers?, fetch? }`, verified against @ag-ui/client 0.0.57....(agent.headers ? { headers: agent.headers } : {}),+ ...(stallGuard+ ? { fetch: stallGuard.watch({ id: agent.id, name: agent.name }) }+ : {}),});remote.use((input, next) =>next.run({@@ -262,6 +279,7 @@ export async function resolveRuntimeAgents(loadAgents: () => Promise<RegisteredAgent[]>,model: RuntimeModel,resolveModelApiKey: () => Promise<string | null>,+ stallGuard?: StallGuard,): Promise<Record<string, AbstractAgent>> {const registered = await loadAgents();if (registered.length === 0) {@@ -273,7 +291,7 @@ export async function resolveRuntimeAgents(const apiKey = registered.some((agent) => agent.type === "built_in")? await resolveModelApiKey(): null;- return buildAgents(registered, model, apiKey);+ return buildAgents(registered, model, apiKey, stallGuard);}/** Who is asking. Agent visibility is decided per person, so a run has to know this first. */@@ -296,6 +314,12 @@ export function createRequestAgents(loadAgents: LoadAgentsForActor,model: RuntimeModel,resolveModelApiKey: () => Promise<string | null>,+ /**+ * Shared across every request rather than built per run, because it is the thing that has to+ * outlive one: the sweep that notices a silent stream has to still be running after the request+ * that opened it has been answered.+ */+ stallGuard?: StallGuard,) {return async ({ request }: { request: Request }) => {const actor = await identifyActor(request);@@ -303,6 +327,7 @@ export function createRequestAgents(() => loadAgents(actor),model,resolveModelApiKey,+ stallGuard,);};}@@ -321,6 +346,12 @@ export function mountCopilotRuntime(resolveModelApiKey: () => Promise<string | null>,identifyUser: IdentifyUser,identifyActor: IdentifyActor,+ /**+ * The watch on Bot streams. Not optional, unlike the parameter it forwards to: a guard built from+ * a timeout of zero already watches nothing, so an unconfigured deployment has one to hand and+ * there is no reason for a caller to have to say `undefined` here to reach `basePath`.+ */+ stallGuard: StallGuard,basePath = "/api/copilotkit",) {const { intelligence } = config.runtime;@@ -345,6 +376,7 @@ export function mountCopilotRuntime(loadAgents,model,resolveModelApiKey,+ stallGuard,) as never,});diff --git a/server/src/index.ts b/server/src/index.tsindex cb9f7f0f..45dcaa25 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -12,6 +12,7 @@ import {startChannelActivityListener,} from "./channels/events";import { createChannelStore } from "./channels/routes";+import { createStallGuard } from "./channels/stall-guard";import { createThreadIdentity } from "./channels/thread-identity";import { websocket as channelSocket } from "./channels/socket";import { createSandboxedStore } from "./components/sandboxed";@@ -282,6 +283,20 @@ process.on("unhandledRejection", (reason) => {);});+/**+ * The watch on Bot streams, built once and shared by every run.+ *+ * It has to outlive the request that opens a stream: the sweep that notices a silent one is still+ * running long after the run request has been answered, because in Intelligence mode that request is+ * answered in about a second and the Bot keeps writing for as long as it has something to say.+ *+ * The same audit store as everything else, so a Bot that hangs is recorded beside what Bots do.+ */+const stallGuard = createStallGuard({+ stallMs: config.agentStallTimeoutMs,+ auditStore: bootAuditStore,+});+const app = createApp(config,auth,@@ -318,6 +333,7 @@ const app = createApp(}),identifyUser,identifyActor,+ stallGuard,),computerClient,// The only path to an acting call.diff --git a/server/tests/config.test.ts b/server/tests/config.test.tsindex 9e7b3126..d784e5d1 100644--- a/server/tests/config.test.ts+++ b/server/tests/config.test.ts@@ -154,6 +154,39 @@ describe("deployment configuration", () => {).toThrow("Google authentication requires BETTER_AUTH_SECRET");});+ // A turn that is ended is a turn somebody loses, so an unset variable leaves every stream alone+ // rather than acquiring a timeout the deployment never asked for. `.env.example` ships a value.+ test("leaves the stall watchdog off when nothing is configured", () => {+ expect(loadConfig(baseEnvironment).agentStallTimeoutMs).toBe(0);+ });++ test("takes a timeout in milliseconds, and zero as switching it off", () => {+ expect(+ loadConfig({ ...baseEnvironment, AGENT_STALL_TIMEOUT_MS: "120000" })+ .agentStallTimeoutMs,+ ).toBe(120_000);+ expect(+ loadConfig({ ...baseEnvironment, AGENT_STALL_TIMEOUT_MS: "0" })+ .agentStallTimeoutMs,+ ).toBe(0);+ });++ // Refused rather than defaulted, for the same reason a malformed policy is: an operator who meant+ // to write a boundary and mistyped it would otherwise get a deployment enforcing something else.+ test.each(["two minutes", "-1", "1.5", ""])(+ "refuses to start on AGENT_STALL_TIMEOUT_MS=%p",+ (value) => {+ const attempt = () =>+ loadConfig({ ...baseEnvironment, AGENT_STALL_TIMEOUT_MS: value });+ if (value === "") {+ // An empty value is an absent one, which is the off case rather than a malformed one.+ expect(attempt().agentStallTimeoutMs).toBe(0);+ return;+ }+ expect(attempt).toThrow("AGENT_STALL_TIMEOUT_MS");+ },+ );+test("takes a widened repetition window, and leaves it absent when nobody set one", () => {expect(loadConfig({diff --git a/server/tests/copilot.test.ts b/server/tests/copilot.test.tsindex a35764d5..a2951ba0 100644--- a/server/tests/copilot.test.ts+++ b/server/tests/copilot.test.ts@@ -163,6 +163,81 @@ describe("registered Copilot agents", () => {expect(agents.risk).toBeInstanceOf(HttpAgent);});+ /*+ * The watch goes on the fetch of a remote Bot and nowhere else.+ *+ * A built-in agent talks to a model provider through the AI SDK rather than over an AG-UI stream,+ * so there is no response body here to watch and nothing for the guard to be given. Asserting the+ * Bot's own name reaches it matters because that name is what the person is shown when its stream+ * goes quiet, and a guard handed the wrong one would say so convincingly.+ */+ test("hands a remote Bot's fetch to the stall guard, and a built-in Bot none", () => {+ const watched: { id: string; name: string }[] = [];+ const stallGuard = {+ watch: (bot: { id: string; name: string }) => {+ watched.push(bot);+ return async () => new Response(null);+ },+ stop: () => undefined,+ };++ const agents = buildAgents(+ [+ {+ id: "general-assistant",+ name: "General Assistant",+ type: "built_in",+ systemPrompt: "Be helpful.",+ },+ {+ id: "risk",+ name: "Risk",+ type: "remote_ag_ui",+ endpoint: "http://risk.internal/ag-ui",+ },+ ],+ { provider: "openai", defaultModel: "gpt-4.1" },+ "openai-secret",+ stallGuard,+ );++ expect(watched).toEqual([{ id: "risk", name: "Risk" }]);+ expect(agents.risk).toBeInstanceOf(HttpAgent);+ });++ /*+ * Told apart by a sentinel, because nothing else tells them apart.+ *+ * @ag-ui/client fills `fetch` in with a wrapper of its own whenever the config does not carry one,+ * so a remote Bot always has a function there and asserting that it does asserts nothing at all.+ * The same registration is built twice, with a guard whose watch returns a fetch nothing else+ * could have produced and then without one, and the two are compared.+ */+ test("leaves a remote Bot's fetch alone when no timeout is configured", () => {+ const sentinel = async () => new Response(null);+ const registered = [+ {+ id: "risk",+ name: "Risk",+ type: "remote_ag_ui" as const,+ endpoint: "http://risk.internal/ag-ui",+ },+ ];+ const model = { provider: "openai" as const, defaultModel: "gpt-4.1" };++ const guarded = buildAgents(registered, model, null, {+ watch: () => sentinel,+ stop: () => undefined,+ }).risk;+ const unguarded = buildAgents(registered, model, null).risk;+ if (!(guarded instanceof HttpAgent) || !(unguarded instanceof HttpAgent)) {+ throw new Error("Expected the remote agent");+ }++ expect(guarded.fetch).toBe(sentinel);+ expect(unguarded.fetch).not.toBe(sentinel);+ });+test("resolves fresh built-in agents and credentials for every request", async () => {const registered = [{diff --git a/server/tests/stall-guard.test.ts b/server/tests/stall-guard.test.tsnew file mode 100644index 00000000..9148db18--- /dev/null+++ b/server/tests/stall-guard.test.ts@@ -0,0 +1,280 @@+import { describe, expect, test } from "bun:test";+import type { AuditEventInput } from "../src/audit";+import { createStallGuard } from "../src/channels/stall-guard";++/**+ * These drive real streams, because the properties that matter are properties of a stream.+ *+ * The timeouts are tens of milliseconds rather than the two minutes a deployment runs, since the+ * sweep interval is derived from the timeout: a test asserting the rule should not have to wait the+ * length of the rule. Nothing else is faked. The bodies below are `ReadableStream`s and the wrapper+ * under test is the same one an `HttpAgent` is handed.+ */++const BOT = { id: "risk-analyst", name: "Risk Analyst" };++/** A body that arrives and closes, like a Bot that answered. */+function speaks(frames: string[]): ReadableStream<Uint8Array> {+ const encoder = new TextEncoder();+ return new ReadableStream<Uint8Array>({+ start(controller) {+ for (const frame of frames) controller.enqueue(encoder.encode(frame));+ controller.close();+ },+ });+}++/** A body that is accepted and never written to, like a Bot whose own upstream has hung. */+function saysNothing(): ReadableStream<Uint8Array> {+ return new ReadableStream<Uint8Array>({+ start() {+ // Deliberately empty. The whole point is that nothing ever arrives and nothing ever closes.+ },+ });+}++function sse(body: ReadableStream<Uint8Array>, status = 200): Response {+ return new Response(body, {+ status,+ headers: { "content-type": "text/event-stream" },+ });+}++const RUN_REQUEST: RequestInit = {+ method: "POST",+ body: JSON.stringify({ threadId: "thread-7", runId: "run-9" }),+};++function collecting(): {+ store: { insert: (event: AuditEventInput) => Promise<void> };+ rows: AuditEventInput[];+} {+ const rows: AuditEventInput[] = [];+ return {+ rows,+ store: {+ insert: async (event) => {+ rows.push(event);+ },+ },+ };+}++describe("a Bot that stops streaming", () => {+ test("is told to the person in the stream they are already reading, and the stream ends", async () => {+ const audit = collecting();+ const guard = createStallGuard({ stallMs: 60, auditStore: audit.store });+ const watched = guard.watch(BOT, async () => sse(saysNothing()));++ const response = await watched("http://bot.internal/ag-ui", RUN_REQUEST);+ // Reading to the end returns only because the guard closed it. A stream nobody ends never does.+ const body = await new Response(response.body).text();+ guard.stop();++ expect(body).toContain('"RUN_ERROR"');+ expect(body).toContain("Risk Analyst stopped responding");+ expect(body).toContain("AGENT_STREAM_STALLED");+ // The framing has to be one an AG-UI client parses, which is a `data:` line and a blank line.+ expect(body.startsWith("data: ")).toBe(true);+ expect(body.endsWith("\n\n")).toBe(true);+ });++ test("leaves a row naming the Bot, the turn and how long the silence was", async () => {+ const audit = collecting();+ const guard = createStallGuard({ stallMs: 60, auditStore: audit.store });+ const watched = guard.watch(BOT, async () => sse(saysNothing()));++ const response = await watched("http://bot.internal/ag-ui", RUN_REQUEST);+ await new Response(response.body).text();+ guard.stop();++ const row = audit.rows.find(+ (event) => event.eventType === "agent.stream_stalled",+ );+ expect(row).toBeDefined();+ expect(row?.targetId).toBe("risk-analyst");+ expect(row?.payload.bot).toBe("risk-analyst");+ expect(row?.payload.thread).toBe("thread-7");+ expect(row?.payload.run).toBe("run-9");+ expect(row?.payload.chunks).toBe(0);+ expect(Number(row?.payload.silentForMs)).toBeGreaterThanOrEqual(60);+ });+});++describe("the sentence a person is left with", () => {+ test("is the whole explanation, because nothing is drawn around it", async () => {+ const guard = createStallGuard({ stallMs: 60 });+ const watched = guard.watch(BOT, async () => sse(saysNothing()));++ const response = await watched("http://bot.internal/ag-ui", RUN_REQUEST);+ const body = await new Response(response.body).text();+ guard.stop();++ const message = String(+ (JSON.parse(body.slice("data: ".length)) as { message: unknown }).message,+ );+ // Both surfaces draw this message and nothing else: no banner, no heading, no error code beside+ // it. So it has to name what went quiet, say the turn is over, and say what to do about it.+ expect(message).toContain("Risk Analyst");+ expect(message).toContain("this turn was ended");+ expect(message).toContain("Ask again");+ // Said in words a person reads, not in the milliseconds a deployment configured, and never as+ // "0 seconds": the timeout here is a test's, and a floor keeps the sentence sane at any value.+ expect(message).toContain("for a second");+ // No identifiers. The thread and run are in the audit row, where somebody is looking for them.+ expect(message).not.toContain("thread-7");+ expect(message).not.toContain("run-9");+ });+});++describe("a Bot that is answering", () => {+ test("has its bytes passed through untouched and nothing added", async () => {+ const audit = collecting();+ const guard = createStallGuard({ stallMs: 5_000, auditStore: audit.store });+ const frames = [+ 'data: {"type":"RUN_STARTED"}\n\n',+ 'data: {"type":"TEXT_MESSAGE_CHUNK","delta":"hello"}\n\n',+ 'data: {"type":"RUN_FINISHED"}\n\n',+ ];+ const watched = guard.watch(BOT, async () => sse(speaks(frames)));++ const response = await watched("http://bot.internal/ag-ui", RUN_REQUEST);+ const body = await new Response(response.body).text();+ guard.stop();++ expect(body).toBe(frames.join(""));+ expect(audit.rows).toHaveLength(0);+ });++ test("is not given up on because whoever reads the relay paused", async () => {+ const audit = collecting();+ // Sixty milliseconds of silence ends a turn here, and the reader below waits five times that+ // before it takes its first chunk. A watch pointed at the wrong end of the relay reports this+ // as a Bot that never said anything, on a run that had already finished streaming.+ const guard = createStallGuard({ stallMs: 60, auditStore: audit.store });+ const frames = [+ 'data: {"type":"RUN_STARTED"}\n\n',+ 'data: {"type":"TEXT_MESSAGE_CHUNK","delta":"hello"}\n\n',+ 'data: {"type":"RUN_FINISHED"}\n\n',+ ];+ const watched = guard.watch(BOT, async () => sse(speaks(frames)));++ const response = await watched("http://bot.internal/ag-ui", RUN_REQUEST);+ await Bun.sleep(300);+ const body = await new Response(response.body).text();+ guard.stop();++ expect(body).toBe(frames.join(""));+ expect(audit.rows).toHaveLength(0);+ });++ test("keeps the status and the content type the parser chooses on", async () => {+ const guard = createStallGuard({ stallMs: 5_000 });+ const watched = guard.watch(BOT, async () => sse(speaks(["data: {}\n\n"])));++ const response = await watched("http://bot.internal/ag-ui", RUN_REQUEST);+ await new Response(response.body).text();+ guard.stop();++ expect(response.status).toBe(200);+ expect(response.headers.get("content-type")).toBe("text/event-stream");+ });+});++describe("what the watch refuses to touch", () => {+ test("a refusal passes straight through, body and all", async () => {+ const audit = collecting();+ const guard = createStallGuard({ stallMs: 60, auditStore: audit.store });+ const refusal = new Response("no", { status: 502 });+ const watched = guard.watch(BOT, async () => refusal);++ const response = await watched("http://bot.internal/ag-ui", RUN_REQUEST);+ guard.stop();++ expect(response).toBe(refusal);+ expect(await response.text()).toBe("no");+ expect(audit.rows).toHaveLength(0);+ });++ test("a protobuf stream is closed rather than written into", async () => {+ const audit = collecting();+ const guard = createStallGuard({ stallMs: 60, auditStore: audit.store });+ const watched = guard.watch(+ BOT,+ async () =>+ new Response(saysNothing(), {+ headers: { "content-type": "application/vnd.ag-ui.event+proto" },+ }),+ );++ const response = await watched("http://bot.internal/ag-ui", RUN_REQUEST);+ const body = await new Response(response.body).arrayBuffer();+ guard.stop();++ // Nothing written in: SSE bytes in a binary stream would corrupt a run that was merely slow.+ expect(body.byteLength).toBe(0);+ expect(+ audit.rows.some((event) => event.eventType === "agent.stream_stalled"),+ ).toBe(true);+ });++ test("a Bot serving events under some other content type is still told", async () => {+ const guard = createStallGuard({ stallMs: 60 });+ // Not `text/event-stream`, and not the protobuf media type either. The AG-UI client parses+ // anything but the latter as server-sent events, so this is a stream a sentence reaches, and a+ // rule stricter than the client's would have ended this run in silence on both surfaces.+ const watched = guard.watch(+ BOT,+ async () =>+ new Response(saysNothing(), {+ headers: { "content-type": "application/json" },+ }),+ );++ const response = await watched("http://bot.internal/ag-ui", RUN_REQUEST);+ const body = await new Response(response.body).text();+ guard.stop();++ expect(body).toContain("Risk Analyst stopped responding");+ });+});++describe("the recovery a person is waiting on", () => {+ test("does not wait on the database it also writes a row to", async () => {+ // An audit store whose insert never settles, which is what a saturated pool or an unreachable+ // Postgres looks like from here. It is also the condition a Bot is most likely to hang in, so a+ // recovery sequenced behind this write would fail open in exactly the case it exists for.+ const guard = createStallGuard({+ stallMs: 60,+ auditStore: { insert: () => new Promise<void>(() => undefined) },+ });+ const watched = guard.watch(BOT, async () => sse(saysNothing()));++ const response = await watched("http://bot.internal/ag-ui", RUN_REQUEST);+ // This returns only if the stream was closed. Nothing else ever closes it.+ const body = await new Response(response.body).text();+ guard.stop();++ expect(body).toContain("Risk Analyst stopped responding");+ });+});++describe("a deployment with no timeout configured", () => {+ test("gets the fetch it handed in, unwrapped", () => {+ const guard = createStallGuard({ stallMs: 0 });+ const inner = async () => sse(speaks([]));+ expect(guard.watch(BOT, inner)).toBe(inner);+ guard.stop();+ });++ test("and its streams are never ended for it", async () => {+ const audit = collecting();+ const guard = createStallGuard({ stallMs: 0, auditStore: audit.store });+ const watched = guard.watch(BOT, async () => sse(speaks(["data: {}\n\n"])));++ const response = await watched("http://bot.internal/ag-ui", RUN_REQUEST);+ expect(await new Response(response.body).text()).toBe("data: {}\n\n");+ expect(audit.rows).toHaveLength(0);+ guard.stop();+ });+});diff --git a/server/tests/turn-watchdog.test.ts b/server/tests/turn-watchdog.test.tsnew file mode 100644index 00000000..f3924ca1--- /dev/null+++ b/server/tests/turn-watchdog.test.ts@@ -0,0 +1,221 @@+import { describe, expect, test } from "bun:test";+import {+ type StalledStream,+ TurnWatchdog,+} from "../src/channels/turn-watchdog";++/**+ * These test the distinction the watchdog exists to draw, not the plumbing around it.+ *+ * Every case is one a deployment can actually be in: a Bot that answers slowly and is fine, a Bot+ * that answered once and then stopped, a Bot that accepted the connection and never wrote a byte.+ * Time is injected rather than waited for, so a two-minute timeout is tested in no time at all and+ * the assertions are about the rule rather than about how fast the machine running them is.+ *+ * There is no exemption for a frontend tool call and therefore no test of one. The run ends before+ * the browser executes a tool and a second run carries the result back, so no stream is ever held+ * open across one; see the module comment on turn-watchdog.ts for what was measured.+ */++/** A clock a test moves by hand. */+function clock(): { now: () => number; advance: (ms: number) => void } {+ let current = 0;+ return {+ now: () => current,+ advance: (ms: number) => {+ current += ms;+ },+ };+}++function watching(stallMs: number) {+ const time = clock();+ const stalled: StalledStream[] = [];+ const watchdog = new TurnWatchdog({+ stallMs,+ now: time.now,+ onStall: (stream) => stalled.push(stream),+ });+ return { time, stalled, watchdog };+}++describe("a turn is judged on activity, not on how long it has run", () => {+ test("a stream that has said nothing for the timeout is given up on", () => {+ const { time, stalled, watchdog } = watching(60_000);+ watchdog.open({ id: "stream-1", botId: "risk-analyst" });++ time.advance(59_999);+ expect(watchdog.sweep()).toBe(0);++ time.advance(1);+ expect(watchdog.sweep()).toBe(1);+ expect(stalled).toHaveLength(1);+ expect(stalled[0]?.botId).toBe("risk-analyst");+ expect(stalled[0]?.silentForMs).toBe(60_000);+ // Zero chunks is the fact worth reading: the Bot answered the request and then said nothing.+ expect(stalled[0]?.chunks).toBe(0);+ });++ test("a stream that keeps talking is never given up on, however long it runs", () => {+ const { time, stalled, watchdog } = watching(60_000);+ watchdog.open({ id: "stream-1", botId: "risk-analyst" });++ // An hour of work, a chunk every half minute. A duration limit would have ended this repeatedly.+ for (let minute = 0; minute < 120; minute++) {+ time.advance(30_000);+ watchdog.record("stream-1");+ expect(watchdog.sweep()).toBe(0);+ }++ expect(stalled).toHaveLength(0);+ expect(watchdog.watching).toBe(1);+ });++ test("a stream that spoke and then stopped is given up on from its last chunk", () => {+ const { time, stalled, watchdog } = watching(60_000);+ watchdog.open({ id: "stream-1", botId: "risk-analyst" });++ time.advance(50_000);+ watchdog.record("stream-1");+ watchdog.record("stream-1");+ time.advance(50_000);+ // A hundred seconds since it opened, fifty since it last said anything. Silence is what counts.+ expect(watchdog.sweep()).toBe(0);++ time.advance(10_000);+ expect(watchdog.sweep()).toBe(1);+ expect(stalled[0]?.silentForMs).toBe(60_000);+ expect(stalled[0]?.chunks).toBe(2);+ });++ test("silence while the relay's reader has not taken delivery is not charged to the Bot", () => {+ const { time, stalled, watchdog } = watching(60_000);+ watchdog.open({ id: "stream-1", botId: "risk-analyst" });++ // The Bot spoke and this process is now holding that chunk out to whoever reads the relayed+ // stream. In this deployment that is the Intelligence runner, publishing over the network.+ watchdog.record("stream-1");+ watchdog.pause("stream-1");+ time.advance(600_000);+ expect(watchdog.sweep()).toBe(0);+ expect(stalled).toHaveLength(0);++ // Ten minutes of it, and the Bot still gets its whole timeout from the moment it is waited on+ // again. The wait that just ended was not its.+ watchdog.resume("stream-1");+ time.advance(59_999);+ expect(watchdog.sweep()).toBe(0);++ time.advance(1);+ expect(watchdog.sweep()).toBe(1);+ expect(stalled[0]?.silentForMs).toBe(60_000);+ expect(stalled[0]?.chunks).toBe(1);+ });++ test("pausing or resuming a stream nobody is watching is harmless", () => {+ const { watchdog } = watching(60_000);+ expect(() => watchdog.pause("never-opened")).not.toThrow();+ expect(() => watchdog.resume("never-opened")).not.toThrow();+ expect(watchdog.watching).toBe(0);+ });++ test("a stream that ends is forgotten and is never reported", () => {+ const { time, stalled, watchdog } = watching(60_000);+ watchdog.open({ id: "stream-1", botId: "risk-analyst" });+ watchdog.close("stream-1");++ time.advance(600_000);+ expect(watchdog.sweep()).toBe(0);+ expect(stalled).toHaveLength(0);+ expect(watchdog.watching).toBe(0);+ });++ test("closing a stream that was never watched is harmless", () => {+ const { watchdog } = watching(60_000);+ expect(() => watchdog.close("never-opened")).not.toThrow();+ expect(watchdog.watching).toBe(0);+ });+});++describe("a stalled stream is reported exactly once", () => {+ test("sweeping again finds nothing, because the entry went with the callback", () => {+ const { time, stalled, watchdog } = watching(1_000);+ watchdog.open({ id: "stream-1", botId: "risk-analyst" });++ time.advance(5_000);+ expect(watchdog.sweep()).toBe(1);+ expect(watchdog.sweep()).toBe(0);+ expect(watchdog.sweep()).toBe(0);+ expect(stalled).toHaveLength(1);+ });++ test("a late chunk from a Bot already given up on does not start a second watch", () => {+ const { time, stalled, watchdog } = watching(1_000);+ watchdog.open({ id: "stream-1", botId: "risk-analyst" });++ time.advance(2_000);+ expect(watchdog.sweep()).toBe(1);++ // The endpoint wakes up and writes. There is no turn left to end, and reviving it here would+ // stall again and report the same wedged turn twice.+ watchdog.record("stream-1");+ time.advance(60_000);+ expect(watchdog.sweep()).toBe(0);+ expect(stalled).toHaveLength(1);+ });++ test("one wedged Bot's callback throwing does not spare the others", () => {+ const time = clock();+ const reported: string[] = [];+ const watchdog = new TurnWatchdog({+ stallMs: 1_000,+ now: time.now,+ onStall: (stream) => {+ reported.push(stream.id);+ if (stream.id === "stream-1") throw new Error("the surface blew up");+ },+ });++ watchdog.open({ id: "stream-1", botId: "risk-analyst" });+ watchdog.open({ id: "stream-2", botId: "general-assistant" });+ time.advance(2_000);++ expect(watchdog.sweep()).toBe(2);+ expect(reported).toEqual(["stream-1", "stream-2"]);+ expect(watchdog.watching).toBe(0);+ });+});++describe("a deployment that has not asked for a watchdog does not get one", () => {+ test("zero watches nothing, whatever is opened", () => {+ const { time, stalled, watchdog } = watching(0);+ watchdog.open({ id: "stream-1", botId: "risk-analyst" });++ expect(watchdog.enabled).toBe(false);+ expect(watchdog.watching).toBe(0);++ time.advance(3_600_000);+ expect(watchdog.sweep()).toBe(0);+ expect(stalled).toHaveLength(0);+ });++ test("a positive timeout is what switches it on", () => {+ const { watchdog } = watching(1);+ expect(watchdog.enabled).toBe(true);+ });+});++describe("opening a stream twice", () => {+ test("restarts its clock rather than raising", () => {+ const { time, stalled, watchdog } = watching(1_000);+ watchdog.open({ id: "stream-1", botId: "risk-analyst" });++ time.advance(900);+ watchdog.open({ id: "stream-1", botId: "risk-analyst" });+ time.advance(900);++ expect(watchdog.sweep()).toBe(0);+ expect(watchdog.watching).toBe(1);+ expect(stalled).toHaveLength(0);+ });+});