Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(frontend/copilot): start streaming a queued turn once the backend runs it, without a reload (#15346)

Reinier van der Leer11ad2e5abfa1
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/stream/__tests__/fakeBackend.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/stream/__tests__/fakeBackend.ts
index 7305f054d5..faa7933f76 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/stream/__tests__/fakeBackend.ts
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/stream/__tests__/fakeBackend.ts
@@ -7,6 +7,16 @@ const HEARTBEAT_MS = 10_000;
// The route's listener checks the meta after each empty read of this length.
const LISTENER_READ_MS = 5_000;
const encoder = new TextEncoder();
+// What the route answers a send it queued: framing, no ids, no turn.
+const QUEUED_STREAM = [
+ { type: "start", messageId: "queued" },
+ { type: "start-step" },
+ { type: "finish-step" },
+ { type: "finish" },
+]
+ .map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`)
+ .concat("data: [DONE]\n\n")
+ .join("");
export interface FakeConnection {
method: string;
@@ -41,6 +51,9 @@ export function fakeBackend(turn: RecordedTurn) {
let running = false;
let started = false;
let stoppedWithoutFinish = false;
+ // "queued" until a running turn ends; "claimed" between the backend marking
+ // it running and its stream existing.
+ let queue: "none" | "queued" | "claimed" | "left" = "none";
const connections: FakeConnection[] = [];
// Every request, refused ones included.
const requests: { method: string; url: URL }[] = [];
@@ -70,6 +83,12 @@ export function fakeBackend(turn: RecordedTurn) {
headers: { "content-type": "application/json" },
});
}
+ if (method === "POST" && queue === "queued") {
+ return new Response(QUEUED_STREAM, {
+ status: 200,
+ headers: { "content-type": "text/event-stream" },
+ });
+ }
if (method === "POST") {
started = true;
running = true;
@@ -175,11 +194,24 @@ export function fakeBackend(turn: RecordedTurn) {
stoppedWithoutFinish = true;
pumps.forEach((pump) => pump());
},
- /** The turn is already running when the page loads. */
+ /** The turn is already running when the page loads, or leaves the queue for a slot. */
beginRunning() {
+ if (queue === "queued" || queue === "claimed") queue = "left";
started = true;
running = true;
},
+ /** The user is at the running cap: the next send is persisted and queued. */
+ queueNextTurn() {
+ queue = "queued";
+ },
+ /** The backend marks the queued turn running before its stream exists. */
+ claimQueued() {
+ queue = "claimed";
+ },
+ /** The user cancels the queued turn; its prompt row stays. */
+ cancelQueued() {
+ queue = "left";
+ },
respond(next: Responder) {
responder = next;
},
@@ -200,13 +232,21 @@ export function fakeBackend(turn: RecordedTurn) {
view(rows?: PersistedRow[]): SessionView {
const shown = started
? (rows ?? turn.rows.slice(0, persistedCount()))
- : [];
+ : queue !== "none"
+ ? turn.rows.slice(0, 1)
+ : [];
return {
messages: shown.map((row, sequence) => ({ ...row, sequence })),
has_more_messages: false,
active_stream: running
? { turn_id: turnId, checkpoint: this.lastCheckpoint() }
: null,
+ chat_status:
+ queue === "queued"
+ ? "queued"
+ : running || queue === "claimed"
+ ? "running"
+ : "idle",
};
},
open() {
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/stream/__tests__/turnRuntime.test.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/stream/__tests__/turnRuntime.test.ts
index 9f9870d2f1..73dfd78e4d 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/stream/__tests__/turnRuntime.test.ts
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/stream/__tests__/turnRuntime.test.ts
@@ -315,6 +315,96 @@ describe("recovery", () => {
PROMPT,
]);
});
+
+ it("a send the backend queued behind the running cap is followed into its turn", async () => {
+ backend.queueNextTurn();
+ void runtime.send({ text: PROMPT }, undefined);
+ await advance(10_000);
+ expect(snapshot().phase).toBe("idle");
+
+ // Another of the user's turns ends, and the backend starts this one.
+ backend.beginRunning();
+ backend.publish(FIRST_BLOCK_DONE);
+ await advance(15_000);
+ expect(backend.connections.at(-1)?.url.searchParams.get("turn")).toBe(
+ backend.turnId,
+ );
+ expect(snapshot().phase).toBe("live");
+ expect(textsOf(render())).toEqual([PROMPT, FIRST_TEXT]);
+ });
+
+ it("a queued turn marked running before its stream exists is still followed into it", async () => {
+ backend.queueNextTurn();
+ void runtime.send({ text: PROMPT }, undefined);
+ await advance(10_000);
+ backend.claimQueued();
+ await advance(30_000);
+
+ backend.beginRunning();
+ backend.publish(FIRST_BLOCK_DONE);
+ await advance(15_000);
+ expect(snapshot().phase).toBe("live");
+ expect(textsOf(render())).toEqual([PROMPT, FIRST_TEXT]);
+ });
+
+ it("gives up on a claimed queued turn whose stream never appears, and reports it", async () => {
+ backend.queueNextTurn();
+ void runtime.send({ text: PROMPT }, undefined);
+ await advance(10_000);
+ backend.claimQueued();
+ await advance(3 * 60_000);
+
+ expect(report).toHaveBeenCalledWith(
+ "queue_start_stalled",
+ expect.anything(),
+ );
+ const views = vi.spyOn(backend, "view");
+ await advance(60_000);
+ expect(views).not.toHaveBeenCalled();
+ });
+
+ it("does not follow a queued turn the user stopped", async () => {
+ backend.queueNextTurn();
+ void runtime.send({ text: PROMPT }, undefined);
+ await advance(10_000);
+ runtime.stop();
+ // The page's own refetch still shows the session queued.
+ const queuedView = backend.view();
+ runtime.observe(queuedView);
+
+ const views = vi.spyOn(backend, "view");
+ await advance(60_000);
+ expect(views).not.toHaveBeenCalled();
+ });
+
+ it("reads a queued session one request at a time", async () => {
+ backend.queueNextTurn();
+ void runtime.send({ text: PROMPT }, undefined);
+ await advance(10_000);
+ let inFlight = 0;
+ let most = 0;
+ runtime.bind({}, async () => {
+ most = Math.max(most, ++inFlight);
+ await new Promise((resolve) => setTimeout(resolve, 25_000));
+ inFlight -= 1;
+ return backend.view();
+ });
+
+ await advance(120_000);
+ expect(most).toBe(1);
+ });
+
+ it("stops reading the session once its queued turn is cancelled", async () => {
+ backend.queueNextTurn();
+ void runtime.send({ text: PROMPT }, undefined);
+ await advance(10_000);
+ backend.cancelQueued();
+ await advance(10_000);
+
+ const views = vi.spyOn(backend, "view");
+ await advance(60_000);
+ expect(views).not.toHaveBeenCalled();
+ });
});
describe("one connection slot", () => {
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/stream/turnRuntime.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/stream/turnRuntime.ts
index 980a0194cf..fc645158f2 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/stream/turnRuntime.ts
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/stream/turnRuntime.ts
@@ -53,6 +53,10 @@ const CATCHING_UP_AFTER_MS = 2_000;
const TICK_MS = 5_000;
const EMIT_THROTTLE_MS = 30;
const FROZEN_POLL_MS = 10_000;
+const QUEUED_POLL_MS = 10_000;
+// The backend marks a queued turn running a moment before its stream exists;
+// past this, the stream is not coming.
+const QUEUE_START_GRACE_MS = 2 * 60_000;
const FINISH_PROBE_MS = 500;
// A server-started continuation (engine switch, approval wake) is dispatched
// after the turn ends; its meta can lag the finish by a few seconds.
@@ -97,6 +101,7 @@ export interface RuntimeSnapshot {
export interface SessionView {
messages?: readonly unknown[] | null;
has_more_messages?: boolean;
+ chat_status?: string | null;
active_stream?: {
turn_id: string;
checkpoint?: TurnCheckpoint | null;
@@ -142,6 +147,11 @@ type ConnectionEnd =
| { kind: "refused"; status: number; body: unknown }
| { kind: "failed"; error: unknown };
+interface QueueWatch {
+ timer: ReturnType<typeof setTimeout> | null;
+ claimedAt: number | null;
+}
+
interface TurnBookkeeping {
verified: number;
reportedErrors: number;
@@ -178,6 +188,9 @@ export class TurnRuntime {
private retryTimer: ReturnType<typeof setTimeout> | null = null;
private tickTimer: ReturnType<typeof setInterval> | null = null;
private frozenTimer: ReturnType<typeof setInterval> | null = null;
+ private queueWatch: QueueWatch | null = null;
+ // Stopped while queued: its turn is not followed until the queue lets go.
+ private queueDeclined = false;
private emitTimer: ReturnType<typeof setTimeout> | null = null;
private reveal = new TextReveal(
() => this.segments,
@@ -243,6 +256,7 @@ export class TurnRuntime {
if (this.isPostPending()) return;
this.stopped = false;
this.suppressAttach = false;
+ this.queueDeclined = false;
this.error = null;
this.providerFailure = null;
const message = userMessage(`local:${uuidv4({})}`, input);
@@ -264,6 +278,8 @@ export class TurnRuntime {
this.clearRetry();
this.stopped = true;
this.suppressAttach = true;
+ this.queueDeclined = true;
+ this.endQueueWatch();
this.notice = null;
this.reveal.clear();
if (running) {
@@ -324,6 +340,7 @@ export class TurnRuntime {
this.ensureConnected();
}
if (!running && !this.isPostPending() && !active) this.settleTail(view);
+ this.followQueue(view);
this.emitNow();
}
@@ -356,6 +373,7 @@ export class TurnRuntime {
this.abortConnections();
this.clearRetry();
this.reveal.dispose();
+ this.endQueueWatch();
for (const timer of [this.tickTimer, this.frozenTimer]) {
if (timer) clearInterval(timer);
}
@@ -700,6 +718,53 @@ export class TurnRuntime {
}
}
+ // A turn queued behind the user's running cap starts when another of their
+ // turns ends, which nothing tells this tab; watch the view until it runs.
+ private followQueue(view: SessionView) {
+ const status = view.chat_status;
+ if (status !== "queued" && status !== "running") this.queueDeclined = false;
+ const queued = status === "queued" && !view.active_stream;
+ const claimed =
+ status === "running" && !view.active_stream && !!this.queueWatch;
+ if (this.queueDeclined || (!queued && !claimed)) {
+ this.endQueueWatch();
+ return;
+ }
+ const watch = this.queueWatch ?? this.startQueueWatch();
+ if (!claimed) {
+ watch.claimedAt = null;
+ return;
+ }
+ watch.claimedAt ??= Date.now();
+ const waited = Date.now() - watch.claimedAt;
+ if (waited < QUEUE_START_GRACE_MS) return;
+ this.deps.report("queue_start_stalled", { waitedMs: waited });
+ this.endQueueWatch();
+ }
+
+ private startQueueWatch() {
+ const watch: QueueWatch = { timer: null, claimedAt: null };
+ this.queueWatch = watch;
+ this.pollQueue(watch);
+ return watch;
+ }
+
+ // Scheduled after each read settles, so a slow read never overlaps the next.
+ private pollQueue(watch: QueueWatch) {
+ watch.timer = setTimeout(async () => {
+ watch.timer = null;
+ const view = await this.fetchView();
+ if (this.queueWatch !== watch || this.disposed) return;
+ if (view) this.observe(view);
+ if (this.queueWatch === watch) this.pollQueue(watch);
+ }, QUEUED_POLL_MS);
+ }
+
+ private endQueueWatch() {
+ if (this.queueWatch?.timer) clearTimeout(this.queueWatch.timer);
+ this.queueWatch = null;
+ }
+
/**
* Rebuild the turn from the DB rows at a checkpoint and tail the stream
* from there: a trimmed stream, a protocol error, a digest mismatch. A