AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
test(frontend/copilot): check what the chat shows after every way its connection can end (#15405)
Reinier van der Leerff1487615891
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/stream/__tests__/faultMatrix/faultMatrix.test.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/stream/__tests__/faultMatrix/faultMatrix.test.tsnew file mode 100644index 0000000000..a31eec60af--- /dev/null+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/stream/__tests__/faultMatrix/faultMatrix.test.ts@@ -0,0 +1,687 @@+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";++import { FaultServer } from "./faultServer";+import { Tab, type Screen } from "./tab";++// A case the runtime still gets wrong runs as `it.fails`; fixing it fails the+// case here until its entry goes.+const KNOWN_WRONG: Record<string, string> = {+ F1: "a server-started turn that registers after the probes is not followed",+ F3: "an idle visible tab is not told of a turn another device starts",+};+const CASES = new Set<string>();++const P1 = "What does example.com say?";+const P2 = "And what about example.org?";+const ANSWER = "assistant: Let me fetch that page.It says Example Domain.";+const FIRST_BLOCK = "assistant: Let me fetch that page.";+const SIGN_IN_AGAIN = "Authentication failed — please sign in again.";++const auth = vi.hoisted(() => ({ signedIn: true }));+vi.mock("@/lib/auth/actions", () => ({+ getWebSocketToken: async () =>+ auth.signedIn ? { token: "token" } : { token: null, error: "No session" },+}));++let server: FaultServer;+let tab: Tab;+const tabs: Tab[] = [];++beforeEach(() => {+ vi.useFakeTimers();+ setVisibility("visible");+ auth.signedIn = true;+ server = new FaultServer();+ tab = openTab();+});++afterEach(() => {+ tabs.splice(0).forEach((t) => t.dispose());+ vi.restoreAllMocks();+ vi.useRealTimers();+ setVisibility("visible");+});++it("every KNOWN_WRONG id names a case", () => {+ expect(Object.keys(KNOWN_WRONG).filter((id) => !CASES.has(id))).toEqual([]);+});++describe("a send", () => {+ cell("S1", "refused 500 with JSON is reported", async () => {+ server.postMode = { status: 500, body: { detail: "Internal error" } };+ tab.send(P1);+ await advance(10_000);+ expectScreen({+ error: '{"detail":"Internal error"}',+ notice: null,+ toasts: 0,+ });+ });++ cell(+ "S2",+ "refused by a proxy's HTML page is reported in words",+ async () => {+ server.postMode = { status: 502, body: "<html><body>502</body></html>" };+ tab.send(P1);+ await advance(10_000);+ expect(tab.screen().error).not.toMatch(/^</);+ },+ );++ cell("S3", "that never reached the server is reported", async () => {+ server.postMode = "lost-before-send";+ tab.send(P1);+ await advance(15_000);+ expectScreen({+ error: "Failed to fetch",+ notice: null,+ phase: "failed",+ toasts: 0,+ });+ });++ cell(+ "S4",+ "whose response was lost while its turn runs is followed",+ async () => {+ server.postMode = "lost-after-receive";+ tab.send(P1);+ await advance(0);+ server.publish(9);+ await advance(2_000);+ expectScreen({+ error: null,+ phase: "live",+ chat: [`user: ${P1}`, FIRST_BLOCK],+ });+ },+ );++ cell(+ "S5",+ "whose response was lost after its turn ended shows the answer",+ async () => {+ server.postMode = "lost-after-receive";+ tab.send(P1);+ await advance(0);+ server.publish();+ await advance(15_000);+ expectScreen({+ error: null,+ notice: null,+ chat: [`user: ${P1}`, ANSWER],+ });+ },+ );++ cell(+ "S6",+ "lost while hidden is followed into its running turn on return",+ async () => {+ server.postMode = "lost-after-receive";+ setVisibility("hidden");+ tab.send(P1);+ await advance(60_000);+ server.publish(9);+ setVisibility("visible");+ await advance(2_000);+ expectScreen({+ error: null,+ phase: "live",+ chat: [`user: ${P1}`, FIRST_BLOCK],+ });+ },+ );++ cell(+ "S7",+ "lost while hidden shows the answer its turn finished meanwhile",+ async () => {+ server.postMode = "lost-after-receive";+ setVisibility("hidden");+ tab.send(P1);+ await advance(30_000);+ server.publish();+ await advance(30_000);+ setVisibility("visible");+ await advance(15_000);+ expectScreen({+ error: null,+ notice: null,+ chat: [`user: ${P1}`, ANSWER],+ });+ },+ );++ cell(+ "S8",+ "on a frozen page whose socket died meanwhile shows the finished answer",+ async () => {+ server.postMode = "silent-after-receive";+ tab.send(P1);+ await advance(0);+ setVisibility("hidden");+ server.publish();+ await advance(60_000);+ server.connections[0].fail();+ setVisibility("visible");+ await advance(15_000);+ expectScreen({+ error: null,+ notice: null,+ phase: "idle",+ chat: [`user: ${P1}`, ANSWER],+ });+ },+ );++ cell(+ "S9",+ "on a frozen page is followed into its running turn on return",+ async () => {+ server.postMode = "silent-after-receive";+ tab.send(P1);+ await advance(0);+ setVisibility("hidden");+ server.publish(9);+ await advance(60_000);+ server.connections[0].fail();+ setVisibility("visible");+ await advance(2_000);+ expectScreen({+ error: null,+ phase: "live",+ chat: [`user: ${P1}`, FIRST_BLOCK],+ });+ },+ );++ cell(+ "S9b",+ "on a frozen page with its socket still open shows the answer on return",+ async () => {+ server.postMode = "silent-after-receive";+ tab.send(P1);+ await advance(0);+ setVisibility("hidden");+ server.publish();+ await advance(60_000);+ setVisibility("visible");+ await advance(3_000);+ expectScreen({ error: null, chat: [`user: ${P1}`, ANSWER] });+ },+ );++ cell(+ "S10",+ "lost with its first session reads failing shows the answer and no notice",+ async () => {+ server.postMode = "lost-after-receive";+ server.failViews = 2;+ tab.send(P1);+ await advance(0);+ server.publish();+ await advance(30_000);+ expectScreen({+ error: null,+ notice: null,+ chat: [`user: ${P1}`, ANSWER],+ });+ },+ );++ cell(+ "S10h",+ "lost while hidden, first reads on return failing, shows the answer and no notice",+ async () => {+ server.postMode = "lost-after-receive";+ setVisibility("hidden");+ tab.send(P1);+ await advance(0);+ server.publish();+ await advance(60_000);+ server.failViews = 2;+ setVisibility("visible");+ await advance(30_000);+ expectScreen({+ error: null,+ notice: null,+ chat: [`user: ${P1}`, ANSWER],+ });+ },+ );++ cell(+ "S11",+ "that never arrived, first reads failing, is still reported",+ async () => {+ server.postMode = "lost-before-send";+ server.failViews = 2;+ tab.send(P1);+ await advance(30_000);+ expectScreen({ error: "Failed to fetch", notice: null, phase: "failed" });+ },+ );++ cell(+ "S11h",+ "that never arrived while hidden, first reads on return failing, is still reported",+ async () => {+ server.postMode = "lost-before-send";+ setVisibility("hidden");+ tab.send(P1);+ await advance(60_000);+ server.failViews = 2;+ setVisibility("visible");+ await advance(30_000);+ expectScreen({ error: "Failed to fetch", notice: null, phase: "failed" });+ },+ );++ cell(+ "S12",+ "lost while another read attaches its turn shows no error",+ async () => {+ tab.dispose();+ tab = openTab(300);+ server.postMode = "lost-after-receive";+ tab.send(P1);+ await advance(0);+ tab.runtime.observe(server.view());+ server.publish(9);+ await advance(20_000);+ expectScreen({ error: null, phase: "live" });+ server.publish();+ await advance(15_000);+ expectScreen({+ error: null,+ phase: "finished",+ chat: [`user: ${P1}`, ANSWER],+ });+ },+ );++ cell(+ "S12b",+ "sent just after an answer, lost, while that turn's settle read attaches it, shows no error",+ async () => {+ tab.dispose();+ tab = openTab(300);+ tab.send(P1);+ await advance(0);+ server.publish();+ // The finished turn's settle read starts 500 ms later and answers 300 ms after that.+ await advance(550);+ server.postMode = "lost-after-receive";+ tab.send(P2);+ await advance(400);+ server.publish(9);+ await advance(20_000);+ expectScreen({ error: null, phase: "live" });+ server.publish();+ await advance(15_000);+ expectScreen({+ error: null,+ chat: [`user: ${P1}`, ANSWER, `user: ${P2}`, ANSWER],+ });+ },+ );++ cell(+ "S13",+ "lost, then sent again during the recovery, shows both turns",+ async () => {+ server.postMode = () =>+ server.turns.length === 0 ? "lost-after-receive" : "stream";+ tab.send(P1);+ await advance(0);+ tab.send(P2);+ await advance(0);+ server.publish();+ await advance(5_000);+ server.publish();+ await advance(30_000);+ expectScreen({+ error: null,+ chat: [`user: ${P1}`, ANSWER, `user: ${P2}`, ANSWER],+ });+ },+ );++ cell(+ "S14",+ "answered by the route's own error frame is followed into its turn",+ async () => {+ server.postMode = "route-error-after-receive";+ tab.send(P1);+ await advance(0);+ server.publish(9);+ await advance(3_000);+ expectScreen({ error: null, phase: "live" });+ },+ );++ cell("S15", "while offline shows the offline notice", async () => {+ server.postMode = "lost-before-send";+ setOnline(false);+ tab.send(P1);+ await advance(10_000);+ expectScreen({ error: null, notice: "offline" });+ });++ cell(+ "S15b",+ "while offline is reported once the network returns",+ async () => {+ server.postMode = "lost-before-send";+ setOnline(false);+ tab.send(P1);+ await advance(10_000);+ setOnline(true);+ await advance(15_000);+ expectScreen({ error: "Failed to fetch", notice: null });+ },+ );++ cell(+ "S16",+ "queued at the running cap, its response lost, is followed into its turn",+ async () => {+ server.queueing = true;+ server.postMode = "lost-before-send";+ server.startTurn(P1, "queued");+ tab.send(P1);+ await advance(10_000);+ server.startQueued();+ server.publish(9);+ await advance(25_000);+ expectScreen({ error: null, phase: "live" });+ },+ );++ cell(+ "S17",+ "while signed out reports it, without the reconnect loop",+ async () => {+ auth.signedIn = false;+ server.viewsDown = true;+ tab.send(P1);+ await advance(120_000);+ expectScreen({ error: SIGN_IN_AGAIN, notice: null, toasts: 0 });+ },+ );+});++describe("a resume", () => {+ cell(+ "R1",+ "refused 401 ends with the sign-in error, without the connection toast",+ async () => {+ await streamFirstBlock();+ server.resumeMode = { status: 401, body: { detail: "Unauthorized" } };+ server.connections[0].fail();+ await advance(180_000);+ expectScreen({ error: SIGN_IN_AGAIN, toasts: 0 });+ },+ );++ cell("R2", "refused 404 ends, without the connection toast", async () => {+ await streamFirstBlock();+ server.resumeMode = { status: 404, body: { detail: "Session not found" } };+ server.connections[0].fail();+ await advance(180_000);+ expectScreen({ phase: "failed", toasts: 0 });+ });++ cell(+ "R3",+ "signed out mid-turn ends with the sign-in error, without the connection toast",+ async () => {+ await streamFirstBlock();+ auth.signedIn = false;+ server.viewsDown = true;+ server.connections[0].fail();+ await advance(180_000);+ expectScreen({ error: SIGN_IN_AGAIN, toasts: 0 });+ },+ );++ cell(+ "R4",+ "after a drop, while the turn ended meanwhile, shows the answer",+ async () => {+ await streamFirstBlock();+ server.connections[0].fail();+ server.publish();+ await advance(5_000);+ expectScreen({+ error: null,+ notice: null,+ toasts: 0,+ phase: "finished",+ chat: [`user: ${P1}`, ANSWER],+ });+ },+ );++ cell(+ "R5",+ "of a turn stopped elsewhere while hidden ends it inline",+ async () => {+ await streamFirstBlock();+ setVisibility("hidden");+ server.connections[0].fail();+ server.stopWithoutFinish();+ await advance(30_000);+ setVisibility("visible");+ await advance(30_000);+ expectScreen({ error: null, notice: null, toasts: 0, phase: "finished" });+ },+ );+});++describe("a turn this tab did not start", () => {+ cell(+ "F1",+ "an approval wake whose turn starts 6 s later is followed",+ async () => {+ await finishFirstTurn();+ tab.runtime.followBackendTurn();+ await advance(6_000);+ server.startTurn("(approval answered)");+ server.publish(9);+ await advance(10_000);+ expectScreen({ phase: "live" });+ },+ );++ cell(+ "F2",+ "started by the server after a failed send clears the send's error",+ async () => {+ server.postMode = "lost-before-send";+ tab.send(P1);+ await advance(15_000);+ tab.runtime.followBackendTurn();+ server.startTurn("(server-started)");+ server.publish(9);+ await advance(3_000);+ expectScreen({ phase: "live", error: null });+ },+ );++ cell(+ "F3",+ "started on another device while this tab is idle and visible is followed",+ async () => {+ await finishFirstTurn();+ server.startTurn(P2);+ server.publish(9);+ await advance(60_000);+ expectScreen({ phase: "live" });+ },+ );++ cell(+ "F3h",+ "started elsewhere while this tab was hidden is followed on return",+ async () => {+ await finishFirstTurn();+ setVisibility("hidden");+ await advance(60_000);+ server.startTurn(P2);+ server.publish(9);+ await advance(60_000);+ setVisibility("visible");+ await advance(5_000);+ expectScreen({ phase: "live" });+ },+ );+});++describe("two tabs on one chat", () => {+ cell(+ "M1",+ "a send from a stale tab while the other tab's turn runs keeps both prompts in order",+ async () => {+ const stale = openTab();+ tab.send(P1);+ await advance(0);+ server.publish(9);+ await advance(100);+ stale.send(P2);+ await advance(3_000);+ server.publish();+ await advance(5_000);+ server.publish();+ await advance(30_000);+ expectScreen(+ { error: null, chat: [`user: ${P1}`, ANSWER, `user: ${P2}`, ANSWER] },+ stale,+ );+ },+ );++ cell(+ "M2",+ "a tab opened onto the other tab's running turn draws it once",+ async () => {+ tab.send(P1);+ await advance(0);+ server.publish(9);+ await advance(100);+ const second = openTab();+ await advance(100);+ server.publish();+ await advance(5_000);+ expectScreen({ error: null, chat: [`user: ${P1}`, ANSWER] }, second);+ },+ );+});++describe("the page lifecycle", () => {+ cell(+ "L1",+ "a back/forward-cache restore after the stream died shows the answer",+ async () => {+ await streamFirstBlock();+ setVisibility("hidden");+ server.connections[0].fail();+ server.publish();+ await advance(120_000);+ Object.defineProperty(document, "visibilityState", {+ value: "visible",+ configurable: true,+ });+ window.dispatchEvent(new Event("pageshow"));+ await advance(5_000);+ expectScreen({+ error: null,+ toasts: 0,+ phase: "finished",+ chat: [`user: ${P1}`, ANSWER],+ });+ },+ );++ cell("L2", "a network switch mid-turn recovers silently", async () => {+ await streamFirstBlock();+ server.resumeMode = "network-error";+ server.connections[0].fail();+ await advance(3_000);+ server.resumeMode = "stream";+ await advance(10_000);+ server.publish();+ await advance(5_000);+ expectScreen({ error: null, notice: null, toasts: 0, phase: "finished" });+ });++ cell(+ "L3",+ "a send whose page froze for 20 s shows the answer soon after return",+ async () => {+ server.postMode = "silent-after-receive";+ tab.send(P1);+ await advance(0);+ setVisibility("hidden");+ server.publish();+ await advance(20_000);+ setVisibility("visible");+ await advance(35_000);+ expectScreen({ chat: [`user: ${P1}`, ANSWER] });+ },+ );+});++function cell(id: string, title: string, run: () => Promise<void>) {+ CASES.add(id);+ const wrong = KNOWN_WRONG[id];+ if (wrong) it.fails(`${id}: ${title} (known wrong: ${wrong})`, run);+ else it(`${id}: ${title}`, run);+}++function expectScreen(should: Partial<Screen>, on: Tab = tab) {+ const screen = on.screen();+ const seen = Object.fromEntries(+ Object.keys(should).map((key) => [key, screen[key as keyof Screen]]),+ );+ expect(seen).toEqual(should);+}++function openTab(viewLatencyMs = 0) {+ const opened = new Tab(server, viewLatencyMs);+ tabs.push(opened);+ return opened;+}++async function streamFirstBlock() {+ tab.send(P1);+ await advance(0);+ server.publish(9);+ await advance(100);+}++async function finishFirstTurn() {+ tab.send(P1);+ await advance(0);+ server.publish();+ await advance(3_000);+}++async function advance(ms: number) {+ await vi.advanceTimersByTimeAsync(ms);+}++function setVisibility(state: "hidden" | "visible") {+ Object.defineProperty(document, "visibilityState", {+ value: state,+ configurable: true,+ });+ document.dispatchEvent(new Event("visibilitychange"));+}++function setOnline(online: boolean) {+ vi.spyOn(navigator, "onLine", "get").mockReturnValue(online);+ window.dispatchEvent(new Event(online ? "online" : "offline"));+}diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/stream/__tests__/faultMatrix/faultServer.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/stream/__tests__/faultMatrix/faultServer.tsnew file mode 100644index 0000000000..f5c80060a5--- /dev/null+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/stream/__tests__/faultMatrix/faultServer.ts@@ -0,0 +1,360 @@+import { compareEntryIds } from "../../turnConverter";+import type { PersistedRow } from "../../turnLog";+import type { SessionView } from "../../turnRuntime";+import { loadRecordedTurn } from "../recordedTurns";++const FIXTURE = loadRecordedTurn("baseline-tool-turn");+const HEARTBEAT_MS = 10_000;+const LISTENER_READ_MS = 5_000;+const encoder = new TextEncoder();+const QUEUED_STREAM = emptyStream();+const FOLLOW_UP_STREAM = emptyStream({+ type: "data-follow-up-queued",+ data: {},+ transient: true,+});++// The session GET's rows carry the message id the renderer keys a prompt on.+type Row = PersistedRow & { id: string };++/** How the next send is answered, or lost, between the client and the server. */+export type PostMode =+ | "stream"+ | "lost-before-send"+ | "lost-after-receive"+ // The server started the turn; the response stays open and silent, as a frozen page sees it.+ | "silent-after-receive"+ | "route-error-after-receive"+ | { status: number; body: unknown };++export type ResumeMode =+ | "stream"+ | "network-error"+ | { status: number; body: unknown };++export interface FaultConnection {+ method: string;+ url: URL;+ closed: boolean;+ aborted: boolean;+ /** Fail the response mid-body, as a dropped socket does. */+ fail(): void;+}++interface FaultTurn {+ id: string;+ frames: { id: string; sse: string }[];+ /** The turn's own rows after its prompt, with absolute sequences. */+ rows: Row[];+ published: number;+ state: "queued" | "running" | "ended" | "stopped";+}++/**+ * The chat backend as the turn runtime meets it, over several turns: a send+ * persists its prompt and registers its turn before the response starts, as+ * `stream_chat_post` does; a send while a turn runs joins its pending queue+ * and becomes the next turn; a resume serves a turn's entries after a cursor.+ * Every turn replays the recorded `baseline-tool-turn` under its own id.+ */+export class FaultServer {+ history: Row[] = [];+ turns: FaultTurn[] = [];+ connections: FaultConnection[] = [];+ requests: { method: string; url: URL }[] = [];+ postMode: PostMode | (() => PostMode) = "stream";+ resumeMode: ResumeMode = "stream";+ /** Session reads fail while this is above zero; each read takes one. */+ failViews = 0;+ /** Session reads fail while this holds, as for a signed-out user. */+ viewsDown = false;+ /** The user is at the running cap: the next send is queued. */+ queueing = false;+ viewReads = 0;+ private pending: string[] = [];+ private pumps = new Set<() => void>();++ fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {+ const url = new URL(String(input));+ const method = init?.method ?? "GET";+ this.requests.push({ method, url });+ return method === "POST" ? this.post(url, init) : this.resume(url, init);+ }) as typeof fetch;++ view(): SessionView {+ this.viewReads += 1;+ if (this.viewsDown) throw new TypeError("Failed to fetch");+ if (this.failViews > 0) {+ this.failViews -= 1;+ throw new TypeError("Failed to fetch");+ }+ const turn = this.current();+ return {+ messages: this.history.map((row) => ({ ...row })),+ has_more_messages: false,+ active_stream:+ turn?.state === "running"+ ? { turn_id: turn.id, checkpoint: this.checkpoint(turn) }+ : null,+ chat_status:+ turn?.state === "queued"+ ? "queued"+ : turn?.state === "running"+ ? "running"+ : "idle",+ };+ }++ /** Persist a prompt and register its turn, as `schedule_chat_turn` does. */+ startTurn(text: string, state: FaultTurn["state"] = "running") {+ const id = `turn-${this.turns.length + 1}`;+ const promptSequence = this.history.length;+ const turn: FaultTurn = {+ id,+ frames: FIXTURE.sse.map((sse, i) => ({+ id: `${i + 1}-0`,+ sse: sse+ .replace("id: drift-turn:", `id: ${id}:`)+ .replace('"sequence": 1,', `"sequence": ${promptSequence + 1},`),+ })),+ rows: FIXTURE.rows.slice(1).map((row, i) => ({+ ...row,+ id: `row-${promptSequence + 1 + i}`,+ sequence: promptSequence + 1 + i,+ })),+ published: 0,+ state,+ };+ this.turns.push(turn);+ this.history.push({+ ...FIXTURE.rows[0],+ id: `row-${promptSequence}`,+ content: text,+ sequence: promptSequence,+ });+ // The route publishes "Message received…" before it answers.+ if (state === "running") this.publish(1);+ return turn;+ }++ /** Publish the running turn's next `count` entries (default: to its end). */+ publish(count = Infinity) {+ const turn = this.running();+ if (!turn) return;+ const before = persistedRows(turn);+ turn.published = Math.min(turn.frames.length, turn.published + count);+ this.history.push(...turn.rows.slice(before, persistedRows(turn)));+ if (turn.published === turn.frames.length) {+ turn.state = "ended";+ if (this.pending.length)+ this.startTurn(this.pending.splice(0).join("\n"));+ }+ this.pumps.forEach((pump) => pump());+ }++ /** The queued turn gets its slot. */+ startQueued() {+ const turn = this.current();+ if (turn?.state !== "queued") return;+ turn.state = "running";+ this.publish(1);+ }++ /** The turn is stopped elsewhere; no finish is stored. */+ stopWithoutFinish() {+ const turn = this.running();+ if (!turn) return;+ turn.state = "stopped";+ this.pumps.forEach((pump) => pump());+ }++ private async post(url: URL, init?: RequestInit) {+ const mode =+ typeof this.postMode === "function" ? this.postMode() : this.postMode;+ if (mode === "lost-before-send") throw new TypeError("Failed to fetch");+ if (typeof mode === "object") return refusal(mode.status, mode.body);+ const { message } = JSON.parse(String(init?.body)) as { message: string };+ if (this.running()) {+ this.pending.push(message);+ return sse(FOLLOW_UP_STREAM);+ }+ const turn = this.startTurn(message, this.queueing ? "queued" : "running");+ if (turn.state === "queued") return sse(QUEUED_STREAM);+ if (mode === "lost-after-receive") throw new TypeError("Failed to fetch");+ if (mode === "route-error-after-receive")+ return sse(+ [+ { type: "error", errorText: "An error occurred. Please try again." },+ { type: "finish" },+ ]+ .map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`)+ .concat("data: [DONE]\n\n")+ .join(""),+ );+ return sse(+ this.open(+ "POST",+ url,+ turn,+ "0-0",+ init?.signal,+ mode === "silent-after-receive",+ ),+ );+ }++ private async resume(url: URL, init?: RequestInit) {+ const mode = this.resumeMode;+ if (mode === "network-error") throw new TypeError("Failed to fetch");+ if (typeof mode === "object") return refusal(mode.status, mode.body);+ const turn = this.turns.find((t) => t.id === url.searchParams.get("turn"));+ if (!turn) return refusal(410, { reason: "expired" });+ const after = url.searchParams.get("after") ?? "0-0";+ return sse(this.open("GET", url, turn, after, init?.signal));+ }++ private open(+ method: string,+ url: URL,+ turn: FaultTurn,+ after: string,+ signal: AbortSignal | null | undefined,+ silent = false,+ ) {+ let index = turn.frames.findIndex((f) => compareEntryIds(f.id, after) > 0);+ if (index === -1) index = turn.frames.length;+ let controller!: ReadableStreamDefaultController<Uint8Array>;+ let heartbeat: ReturnType<typeof setInterval> | null = null;+ let done: ReturnType<typeof setTimeout> | null = null;+ const pumps = this.pumps;+ const conn: FaultConnection = {+ method,+ url,+ closed: false,+ aborted: false,+ fail: () => end(() => controller.error(new TypeError("network lost"))),+ };+ function end(finish: () => void) {+ if (conn.closed) return;+ conn.closed = true;+ if (heartbeat) clearInterval(heartbeat);+ if (done) clearTimeout(done);+ pumps.delete(pump);+ finish();+ }+ function pump() {+ if (silent) return;+ while (!conn.closed && index < turn.published) {+ const frame = turn.frames[index++];+ controller.enqueue(encoder.encode(frame.sse));+ if (isFinish(frame.sse)) {+ controller.enqueue(encoder.encode("data: [DONE]\n\n"));+ end(() => controller.close());+ }+ }+ // The listener ends a read of a turn that is no longer running.+ if (!conn.closed && !done && turn.state !== "running") {+ done = setTimeout(() => {+ controller.enqueue(encoder.encode("data: [DONE]\n\n"));+ end(() => controller.close());+ }, LISTENER_READ_MS);+ }+ }+ this.connections.push(conn);+ signal?.addEventListener("abort", () => {+ conn.aborted = true;+ end(() => {});+ });+ return new ReadableStream<Uint8Array>({+ start(c) {+ controller = c;+ if (!silent)+ heartbeat = setInterval(() => {+ if (!conn.closed) c.enqueue(encoder.encode(": heartbeat\n\n"));+ }, HEARTBEAT_MS);+ pumps.add(pump);+ pump();+ },+ cancel() {+ conn.aborted = true;+ end(() => {});+ },+ });+ }++ private current(): FaultTurn | null {+ return this.turns[this.turns.length - 1] ?? null;+ }++ private running(): FaultTurn | null {+ const turn = this.current();+ return turn?.state === "running" ? turn : null;+ }++ private checkpoint(turn: FaultTurn) {+ for (let i = turn.published - 1; i >= 0; i--) {+ const data = dataOf(turn.frames[i].sse);+ if (data?.type === "data-checkpoint") {+ return {+ entry_id: turn.frames[i].id,+ rows: Number(data.data.rows),+ sequence: Number(data.data.sequence),+ };+ }+ }+ return null;+ }+}++// A checkpoint names how many of the turn's rows are persisted; the finish, all of them.+function persistedRows(turn: FaultTurn) {+ let rows = 0;+ for (const frame of turn.frames.slice(0, turn.published)) {+ if (isFinish(frame.sse)) return turn.rows.length;+ const data = dataOf(frame.sse);+ if (data?.type === "data-checkpoint") rows = Number(data.data.rows);+ }+ return rows;+}++function isFinish(sse: string) {+ return dataOf(sse)?.type === "finish";+}++function dataOf(sse: string) {+ const line = sse.split("\n").find((l) => l.startsWith("data: "));+ return line+ ? (JSON.parse(line.slice("data: ".length)) as {+ type: string;+ data: Record<string, unknown>;+ })+ : null;+}++function emptyStream(...parts: object[]) {+ return [+ { type: "start", messageId: "empty" },+ { type: "start-step" },+ ...parts,+ { type: "finish-step" },+ { type: "finish" },+ ]+ .map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`)+ .concat("data: [DONE]\n\n")+ .join("");+}++function sse(body: string | ReadableStream<Uint8Array>) {+ return new Response(body, {+ status: 200,+ headers: { "content-type": "text/event-stream" },+ });+}++function refusal(status: number, body: unknown) {+ const html = typeof body === "string";+ return new Response(html ? body : JSON.stringify(body), {+ status,+ headers: { "content-type": html ? "text/html" : "application/json" },+ });+}diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/stream/__tests__/faultMatrix/tab.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/stream/__tests__/faultMatrix/tab.tsnew file mode 100644index 0000000000..66b850ba59--- /dev/null+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/stream/__tests__/faultMatrix/tab.ts@@ -0,0 +1,105 @@+import type { UIMessage } from "ai";+import { vi } from "vitest";++import { getCopilotAuthHeaders } from "../../../helpers";+import { convertChatSessionMessagesToUiMessages } from "../../../helpers/convertChatSessionToUiMessages";+import { createAssistantRunJoiner, createTailRenderer } from "../../renderTail";+import {+ TurnRuntime,+ type RuntimeDeps,+ type SessionView,+} from "../../turnRuntime";+import type { FaultServer } from "./faultServer";++const SESSION = "session-1";++/** What the user sees: the error box, the inline notice, the status, the toast count, and the chat. */+export interface Screen {+ error: string | null;+ notice: string | null;+ phase: string;+ toasts: number;+ chat: string[];+}++/**+ * One browser tab on the chat, bound as `useCopilotRuntimeStream` binds the+ * runtime: the runtime's session reads are the page's refetches, and the+ * page observes each view it gets on the next macrotask, as React Query+ * notifies. The chat is the view's history above the runtime's tail, both+ * through the renderers the page uses.+ */+export class Tab {+ readonly runtime: TurnRuntime;+ private toast = vi.fn<RuntimeDeps["toast"]>();+ private lastView: SessionView | null = null;+ private renderTail = createTailRenderer(SESSION);+ private joinRuns = createAssistantRunJoiner();++ constructor(+ private readonly server: FaultServer,+ private readonly viewLatencyMs = 0,+ ) {+ this.runtime = new TurnRuntime(SESSION, {+ baseUrl: () => "http://backend.test",+ headers: getCopilotAuthHeaders,+ fetch: server.fetch,+ fetchSession: () => this.refetch(),+ toast: this.toast,+ report: vi.fn(),+ });+ this.runtime.bind({}, () => this.refetch());+ this.lastView = server.view();+ this.runtime.observe(this.lastView);+ }++ send(text: string) {+ void this.runtime.send({ text }, undefined);+ }++ screen(): Screen {+ const snapshot = this.runtime.getSnapshot();+ const cut = snapshot.ownedFrom;+ const history = (+ (this.lastView?.messages ?? []) as { sequence?: number }[]+ ).filter((row) => cut === null || (row.sequence ?? 0) < cut);+ const messages = this.joinRuns([+ ...convertChatSessionMessagesToUiMessages(SESSION, history, {+ isComplete: !this.lastView?.active_stream,+ }).messages,+ ...this.renderTail(snapshot).messages,+ ]);+ return {+ error: snapshot.error?.message ?? null,+ notice: snapshot.notice,+ phase: snapshot.phase,+ toasts: this.toast.mock.calls.length,+ chat: messages.map(textOf),+ };+ }++ dispose() {+ this.runtime.dispose();+ }++ private async refetch(): Promise<SessionView | null> {+ if (this.viewLatencyMs)+ await new Promise((resolve) => setTimeout(resolve, this.viewLatencyMs));+ let view: SessionView;+ try {+ view = this.server.view();+ } catch {+ return null;+ }+ this.lastView = view;+ setTimeout(() => this.runtime.observe(view), 0);+ return view;+ }+}++function textOf(message: UIMessage) {+ const text = message.parts+ .map((part) => (part.type === "text" ? part.text : ""))+ .join("");+ return `${message.role}: ${text}`;+}diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/stream/turnRuntime.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/stream/turnRuntime.tsindex 70b2ff962f..1743ab2c75 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/stream/turnRuntime.ts+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/stream/turnRuntime.ts@@ -328,7 +328,10 @@ export class TurnRuntime {this.emit();}- /** The backend refused the send before persisting it (a 429). */+ /**+ * Drop the prompt drawn at send that will not be its own turn: the backend+ * refused it (a 429) or queued it behind the running turn.+ */dropUnsentUserRow() {const last = this.segments[this.segments.length - 1];if (last?.kind !== "user" || last.origin !== "sent") return;