AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(frontend/copilot): drift reports name the fields that drifted (#15646)
Reinier van der Leercb77a2a73bb1
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/stream/__tests__/driftSentry.test.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/stream/__tests__/driftSentry.test.tsnew file mode 100644index 0000000000..c800c112cd--- /dev/null+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/stream/__tests__/driftSentry.test.ts@@ -0,0 +1,152 @@+import * as Sentry from "@sentry/nextjs";+import {+ afterEach,+ beforeAll,+ beforeEach,+ describe,+ expect,+ it,+ vi,+} from "vitest";++import { getTurnRuntime, resetTurnRuntimes } from "../turnRuntime";+import {+ compareShadowWithSession,+ createShadowFetch,+ getShadowLog,+ resetShadows,+ setStreamShadowRate,+} from "../turnShadow";+import { fakeBackend, type FakeBackend } from "./fakeBackend";+import { loadRecordedTurn } from "./recordedTurns";++// The real SDK, so each report is normalised as Sentry stores it: arrays nested+// past `normalizeDepth` (3) arrive as "[Array]".+const events: Sentry.ErrorEvent[] = [];+const toolTurn = loadRecordedTurn("baseline-tool-turn");+const SECOND_TEXT = "It says Example Domain.";++beforeAll(() => {+ Sentry.init({+ dsn: "https://[email protected]/0",+ beforeSend(event) {+ events.push(event);+ return null;+ },+ });+});++beforeEach(() => {+ events.length = 0;+});++describe("a drift report reaching Sentry", () => {+ let backend: FakeBackend;++ beforeEach(() => {+ vi.useFakeTimers();+ backend = fakeBackend(toolTurn);+ getTurnRuntime("session-1", {+ baseUrl: () => "http://backend.test",+ headers: async () => ({}),+ fetch: backend.fetch,+ fetchSession: async () => backend.view(),+ toast: vi.fn(),+ });+ });++ afterEach(() => {+ resetTurnRuntimes();+ vi.restoreAllMocks();+ vi.useRealTimers();+ });++ it("names the fields of a turn whose saved rows differ from the stream", async () => {+ void getTurnRuntime("session-1").send({ text: "hi" }, undefined);+ await vi.advanceTimersByTimeAsync(0);+ backend.publish(toolTurn.sse.length - 1);+ await vi.advanceTimersByTimeAsync(100);+ const rows = toolTurn.rows.map((row) =>+ row.content === SECOND_TEXT ? { ...row, content: "It says Other." } : row,+ );+ vi.spyOn(backend, "view").mockImplementation(() => ({+ messages: rows.map((row, sequence) => ({ ...row, sequence })),+ has_more_messages: false,+ active_stream: null,+ }));+ backend.publish();+ await vi.advanceTimersByTimeAsync(1_000);++ await vi.waitFor(() =>+ expect(driftExtras("finish")).toEqual([+ expect.objectContaining({+ sessionId: "session-1",+ diffs: ["2: content"],+ }),+ ]),+ );+ });++ it("names the field a refused send failed validation on, without its input", async () => {+ backend.respond(() => ({+ status: 422,+ body: {+ message: "Invalid data for POST /api/chat/sessions/session-1/stream",+ detail: [+ {+ type: "missing",+ loc: ["body", "message"],+ msg: "Field required",+ input: { text: "the user's words" },+ },+ ],+ },+ }));+ void getTurnRuntime("session-1").send({ text: "hi" }, undefined);+ await vi.advanceTimersByTimeAsync(1_000);++ await vi.waitFor(() =>+ expect(driftExtras("refused")).toEqual([+ expect.objectContaining({+ status: 422,+ body: expect.objectContaining({+ detail: ["body.message: Field required"],+ }),+ }),+ ]),+ );+ expect(JSON.stringify(events)).not.toContain("the user's words");+ });+});++describe("a shadow drift report reaching Sentry", () => {+ beforeEach(() => {+ resetShadows();+ setStreamShadowRate(1);+ });++ it("names the fields of a turn whose saved rows differ from the stream", async () => {+ const body = toolTurn.sse.join("") + "data: [DONE]\n\n";+ const shadowFetch = createShadowFetch("s1", async () => new Response(body));+ await (await shadowFetch("http://x/stream", { method: "POST" })).text();+ await vi.waitFor(() => expect(getShadowLog("s1")?.status).toBe("finished"));+ compareShadowWithSession("s1", {+ messages: toolTurn.rows.map((row) =>+ row.content === SECOND_TEXT ? { ...row, content: "Other." } : row,+ ),+ active_stream: null,+ });++ await vi.waitFor(() =>+ expect(driftExtras("finish")).toEqual([+ expect.objectContaining({ diffs: ["2: content"] }),+ ]),+ );+ });+});++function driftExtras(kind: string) {+ return events+ .filter((event) => event.tags?.stream_drift === kind)+ .map((event) => event.extra);+}diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/stream/__tests__/turnRuntime.test.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/stream/__tests__/turnRuntime.test.tsindex c3542b448c..b8dbe61f64 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/stream/__tests__/turnRuntime.test.ts+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/stream/__tests__/turnRuntime.test.ts@@ -1129,7 +1129,7 @@ describe("the end-of-turn reconcile", () => {expect(textsOf(render())).toEqual([PROMPT, `${FIRST_TEXT}It says Other.`]);expect(report).toHaveBeenCalledWith("finish",- expect.objectContaining({ diffs: [{ index: 2, fields: ["content"] }] }),+ expect.objectContaining({ diffs: ["2: content"] }),);});});diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/stream/turnRuntime.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/stream/turnRuntime.tsindex 5c99078b83..a31063dd4a 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/stream/turnRuntime.ts+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/stream/turnRuntime.ts@@ -18,7 +18,7 @@ import { fetchSse, type SseFrame } from "./sseClient";import { TextReveal } from "./textReveal";import { applyEntry, type StreamEntry, type WireChunk } from "./turnConverter";import { seedTurnLog } from "./turnLog";-import { diffRows, sha256Hex } from "./turnShadow";+import { diffRows, formatDiffs, sha256Hex } from "./turnShadow";import {checkpointOf,closeBlocks,@@ -712,7 +712,10 @@ export class TurnRuntime {// The banner tells the user we were notified of these.private reportIfUnexpected(end: Extract<ConnectionEnd, { kind: "refused" }>) {if (!isUnexpectedRefusal(end.status)) return;- this.deps.report("refused", { status: end.status, body: end.body });+ this.deps.report("refused", {+ status: end.status,+ body: reportableBody(end.body),+ });}/** A refusal no retry can change: end the turn where it stands and say why. */@@ -1058,7 +1061,7 @@ export class TurnRuntime {this.deps.report("finish", {turnId: seg.log.turnId,cursor: seg.log.cursor,- diffs: drift,+ diffs: formatDiffs(drift),});}this.updateTurn(seg.key, (s) => reconcileTurn(s, persisted));@@ -1518,6 +1521,20 @@ export class SendRefusedError extends Error {}}+// Sentry flattens a 422's error objects to "[Object]"; keep each one's place and+// reason as a line, and leave out its `input`, which can hold the user's message.+function reportableBody(body: unknown) {+ const detail = (body as { detail?: unknown } | null)?.detail;+ if (!Array.isArray(detail)) return body;+ return {+ ...(body as object),+ detail: detail.map(+ (item: { loc?: unknown; msg?: unknown } | null) =>+ `${Array.isArray(item?.loc) ? item.loc.join(".") : ""}: ${String(item?.msg)}`,+ ),+ };+}+// FastAPI's 422 for a message over the request model's max_length; the composer has no cap.function isMessageTooLong(body: unknown) {const detail = (body as { detail?: unknown } | null)?.detail;diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/stream/turnShadow.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/stream/turnShadow.tsindex e33213a0d9..708e4186f5 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/stream/turnShadow.ts+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/stream/turnShadow.ts@@ -135,6 +135,11 @@ export function diffRows(return diffs;}+// Sentry flattens arrays nested this deep in `extra` to "[Array]"; a string survives.+export function formatDiffs(diffs: ReturnType<typeof diffRows>) {+ return diffs.map(({ index, fields }) => `${index}: ${fields.join(", ")}`);+}+interface SessionView {messages?: readonly unknown[] | null;active_stream?: { turn_id: string } | null;@@ -226,7 +231,8 @@ function compareTurn(shadow: ShadowTurn, session: SessionView) {.slice(0, last.rows).map(logRowFromPersisted);const diffs = diffRows(log.rows, turnRows);- if (diffs.length > 0) reportDrift("finish", log, { diffs });+ if (diffs.length > 0)+ reportDrift("finish", log, { diffs: formatDiffs(diffs) });}// A replay that opens past the turn's `start` (a trimmed stream) is a tail,