Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix silent data corruption when a query errors mid-stream (#75586)

Romeo Van Snick1b0e9e91d418
diff --git a/e2e/test/scenarios/sharing/downloads/mid-stream-download-failure.cy.spec.ts b/e2e/test/scenarios/sharing/downloads/mid-stream-download-failure.cy.spec.ts
new file mode 100644
index 00000000000..baf7f2c0fd5
--- /dev/null
+++ b/e2e/test/scenarios/sharing/downloads/mid-stream-download-failure.cy.spec.ts
@@ -0,0 +1,47 @@
+const { H } = cy;
+
+// Reproduction for the streaming-download corruption bug: when a query fails
+// *after* the download stream has started, the server used to append a JSON error
+// blob and close the connection cleanly. The client couldn't tell, so it saved a
+// corrupt file and reported success. The server now aborts the connection
+// mid-stream and the client surfaces a download error instead.
+//
+// To fail *mid-stream* (not before the response commits) the query has to start
+// streaming and then blow up. This native query previews fine — the query builder
+// caps results at 2000 rows, below the division-by-zero at row 5000 — but the CSV
+// export runs unbounded, streams past the failing row, and aborts.
+describe("scenarios > sharing > downloads > mid-stream failure", () => {
+ beforeEach(() => {
+ H.restore();
+ cy.signInAsAdmin();
+ });
+
+ it("surfaces a download error when a query fails after the stream has started", () => {
+ H.createNativeQuestion({
+ name: "Fails mid-stream",
+ native: {
+ query: "SELECT 100 / (x - 5000) AS v FROM SYSTEM_RANGE(1, 10000)",
+ },
+ }).then(({ body: card }) => {
+ cy.intercept("POST", `/api/card/${card.id}/query`).as("preview");
+ H.visitQuestion(card.id);
+ });
+
+ cy.log("Preview loads because it is capped below the failing row");
+ cy.wait("@preview");
+ cy.findByLabelText("Download results").should("be.visible").click();
+
+ cy.log("Trigger a full CSV export, which streams past the failing row");
+ H.popover().within(() => {
+ cy.findByText(".csv").click();
+ cy.findByTestId("download-results-button").click();
+ });
+
+ cy.log(
+ "The aborted stream is surfaced as a download error, not a silent success",
+ );
+ cy.findByTestId("status-root-container", { timeout: 15000 })
+ .should("contain", "Download error")
+ .and("not.contain", "Download completed");
+ });
+});
diff --git a/frontend/src/metabase/api/client/client.ts b/frontend/src/metabase/api/client/client.ts
index e20cb5edd14..6cc9770edaa 100644
--- a/frontend/src/metabase/api/client/client.ts
+++ b/frontend/src/metabase/api/client/client.ts
@@ -309,7 +309,7 @@ export class ApiClient extends EventEmitter<EventMap> {
/**
* Resolve with the raw `Response`, leaving everything after the network to the
- * caller: unlike `request`, it does not read the body, recover a 202 `_status`,
+ * caller: unlike `request`, it does not read the body,
* emit events, or throw on a non-2xx status. For streaming callers (e.g. SSE)
* that consume `response.body` and do their own error handling, while still
* getting the client pipeline — middleware, client headers, anti-CSRF, and
diff --git a/frontend/src/metabase/api/client/errors.ts b/frontend/src/metabase/api/client/errors.ts
index 5cafc1aac2c..a097fcd984c 100644
--- a/frontend/src/metabase/api/client/errors.ts
+++ b/frontend/src/metabase/api/client/errors.ts
@@ -15,6 +15,28 @@ export function isNetworkError(error: unknown): error is NetworkError {
return error instanceof NetworkError;
}
+/**
+ * Thrown when a response *was* received — its status line and some body bytes
+ * already committed — but reading the body then failed partway through. A
+ * streamed query or export that errors mid-stream aborts the connection without
+ * a clean terminator, which rejects the body read. This is deliberately distinct
+ * from `NetworkError` (where the transport never delivered a response at all):
+ * the server answered and then the stream broke, so the UI must not blame the
+ * user's connectivity or imply the server is down.
+ */
+export class StreamInterruptedError extends Error {
+ constructor(message = "Stream interrupted") {
+ super(message);
+ this.name = "StreamInterruptedError";
+ }
+}
+
+export function isStreamInterruptedError(
+ error: unknown,
+): error is StreamInterruptedError {
+ return error instanceof StreamInterruptedError;
+}
+
/**
* `true` for the standard `DOMException` of name `"AbortError"` that
* `fetch()` rejects with when its signal aborts. Use this in place of the
diff --git a/frontend/src/metabase/api/client/utils.ts b/frontend/src/metabase/api/client/utils.ts
index e013c6c4864..05abb396dcb 100644
--- a/frontend/src/metabase/api/client/utils.ts
+++ b/frontend/src/metabase/api/client/utils.ts
@@ -1,3 +1,5 @@
+import { StreamInterruptedError } from "./errors";
+
type ResponseResult = {
ok: boolean;
status: number;
@@ -59,10 +61,6 @@ function isJson(response: Response): boolean {
);
}
-function isOk(status: number): boolean {
- return 200 <= status && status <= 299;
-}
-
/**
* Parse a JSON body stream natively — `response.json()` never materializes the
* payload as a JS string the way `text()` + `JSON.parse` would. A body that
@@ -79,108 +77,69 @@ async function parseJsonOrNull(response: Response): Promise<unknown> {
}
/**
- * Interpret a fetched `Response` into a `{ ok, status, body }` result, without
- * emitting or throwing — the caller decides what to do with a non-ok status.
+ * Read a response body by its `Content-Type`: JSON natively (never materializing
+ * an intermediate string), otherwise text. A successful JSON body must be
+ * parseable, so a parse failure throws; a failed response's body is error data,
+ * so an empty or garbage body is swallowed to `null` rather than masking the
+ * status with a `SyntaxError`.
*
- * The body is only read when it'll actually be used — as the result of a normal
- * request, as error data, or to recover the real status of a 202 streaming
- * response (whose status hides in a `{_status}` body). A `rawResponse` caller
- * whose non-202 request succeeded gets the `Response` itself as `body`, unread,
- * so a binary payload like a map tile is never decoded as text.
+ * The response is already committed by the time we read it, so a read that
+ * rejects with a `TypeError` is the body stream breaking mid-flight — a streamed
+ * query/export that errored after committing aborts the connection without a
+ * clean terminator. Surface that as a typed `StreamInterruptedError` so the UI
+ * can distinguish it from a genuine connectivity failure (where no response
+ * arrives at all) instead of showing a misleading "server issues" message. A
+ * `SyntaxError` from a complete-but-malformed body is a real parse problem and
+ * propagates unchanged.
*/
-export async function handleResponse(
- response: Response,
- rawResponse?: boolean,
-): Promise<ResponseResult> {
- // Hand back the raw Response only when the caller asked for it and the request
- // succeeded. A failure falls through below so its body can be read as error
- // data; a 202 counts as ok here and is sorted out next.
- if (rawResponse && response.ok) {
- if (response.status === 202) {
- // A 202's HTTP status can't be trusted (a streamed export commits it
- // before the work can fail), so read the body to learn the real status.
- // The caller still needs an unread Response, so clone before reading.
- const unreadResponse = response.clone();
- const info = await readBody(response);
- if (!info.ok) {
- return info;
- }
- return { ...info, body: unreadResponse };
+async function readBody(response: Response): Promise<unknown> {
+ try {
+ if (!isJson(response)) {
+ return await response.text();
}
-
- // A non-202 status is authoritative, so skip reading the body entirely and
- // return the untouched Response — this is what keeps a binary payload like a
- // map tile from being decoded as text.
- return { ok: true, status: response.status, body: response };
- }
-
- // Either the caller doesn't want the raw Response (the parsed body IS the
- // result) or the request failed (the body is the error payload). Read it and
- // let the caller inspect `ok`.
- return await readBody(response);
-}
-
-/**
- * A streamed 202 signals its real status in a `{_status: N}` body.
- */
-function streamedStatus(body: unknown): number | undefined {
- if (
- body &&
- typeof body === "object" &&
- "_status" in body &&
- typeof body._status === "number" &&
- body._status > 0
- ) {
- return body._status;
- }
- return undefined;
-}
-
-/**
- * Read the body by its `Content-Type`: JSON natively (never materializing an
- * intermediate string), otherwise text. A successful JSON body must be
- * parseable, so a parse failure throws; on a failed response the body is error
- * data, so an empty or garbage body is swallowed to `null` rather than masking
- * the status with a `SyntaxError`.
- */
-async function readResponseBody(
- response: Response,
- ok: boolean,
-): Promise<unknown> {
- if (!isJson(response)) {
- return response.text();
+ return response.ok
+ ? await response.json()
+ : await parseJsonOrNull(response);
+ } catch (error) {
+ if (error instanceof TypeError) {
+ throw new StreamInterruptedError(error.message);
+ }
+ throw error;
}
- return ok ? response.json() : parseJsonOrNull(response);
}
/**
- * Read and interpret a response into `{ ok, status, body }`, routed by status:
+ * Interpret a fetched `Response` into a `{ ok, status, body }` result. It does
+ * not throw on a non-ok status — the caller decides what to do with it — but a
+ * body that fails to read (e.g. a streamed response aborted mid-flight) still
+ * rejects, which is how a mid-stream failure surfaces.
*
- * - 202: a streamed query/download commits the 202 before its work can fail, so
- * read the body to recover the real status from a `{_status}` payload. The
- * read stays tolerant — a non-`_status` body (even a non-JSON export) is a
- * success streamed straight through.
- * TODO: a JSON error emitted mid-stream parses to a non-`_status` object and
- * is silently treated as success.
- * - 204: no content.
- * - otherwise: the status line is authoritative; see `readResponseBody`.
- *
- * Does not throw on a non-2xx status — the caller inspects `ok`.
+ * The body is only read when it'll actually be used — as the result of a normal
+ * request, or as error data. A `rawResponse` caller whose request succeeded gets
+ * the `Response` itself as `body`, unread, so a binary payload like a map tile or
+ * a streamed export is never decoded as text.
*/
-async function readBody(response: Response): Promise<ResponseResult> {
- if (response.status === 202) {
- const body = await parseJsonOrNull(response);
- const status = streamedStatus(body) ?? 202;
- return { ok: isOk(status), status, body };
+export async function handleResponse(
+ response: Response,
+ rawResponse?: boolean,
+): Promise<ResponseResult> {
+ // The status is authoritative, so skip reading the body and return the untouched
+ // Response — this keeps a binary payload like a map tile or a streamed export
+ // from being decoded as text. A streamed export that fails mid-flight aborts the
+ // connection, so a body that reads to completion is a genuine success; there is
+ // no error blob to recover from the body here. A failed rawResponse request
+ // falls through so its body can be read as error data.
+ if (rawResponse && response.ok) {
+ return { ok: true, status: response.status, body: response };
}
if (response.status === 204) {
+ // Empty response: body is null
return { ok: true, status: 204, body: null };
}
- const { status, ok } = response;
- const body = await readResponseBody(response, ok);
- return { ok, status, body };
+ const { ok, status } = response;
+ return { ok, status, body: await readBody(response) };
}
// `:tag` placeholders are URL-encoded (slashes become %2F).
diff --git a/frontend/src/metabase/api/client/utils.unit.spec.ts b/frontend/src/metabase/api/client/utils.unit.spec.ts
index 3b56c93fe79..e6e22e5a86f 100644
--- a/frontend/src/metabase/api/client/utils.unit.spec.ts
+++ b/frontend/src/metabase/api/client/utils.unit.spec.ts
@@ -1,4 +1,134 @@
-import { substituteUrlTags } from "./utils";
+import { StreamInterruptedError } from "./errors";
+import { handleResponse, substituteUrlTags } from "./utils";
+
+type MockResponseOptions = {
+ ok?: boolean;
+ status?: number;
+ contentType?: string | null;
+ json?: () => Promise<unknown>;
+ text?: () => Promise<string>;
+};
+
+const mockResponse = ({
+ ok = true,
+ status = 200,
+ contentType = "application/json",
+ json = () => Promise.resolve({}),
+ text = () => Promise.resolve(""),
+}: MockResponseOptions): Response =>
+ ({
+ ok,
+ status,
+ headers: {
+ get: (name: string) =>
+ name.toLowerCase() === "content-type" ? contentType : null,
+ },
+ json,
+ text,
+ }) as unknown as Response;
+
+describe("handleResponse", () => {
+ it("parses a successful JSON body", async () => {
+ const body = { id: 1 };
+ const result = await handleResponse(
+ mockResponse({ json: () => Promise.resolve(body) }),
+ );
+ expect(result).toEqual({ ok: true, status: 200, body });
+ });
+
+ it("returns a null body for an empty 204", async () => {
+ const result = await handleResponse(
+ mockResponse({ status: 204, contentType: null }),
+ );
+ expect(result).toEqual({ ok: true, status: 204, body: null });
+ });
+
+ it("reads a non-JSON body as text", async () => {
+ const result = await handleResponse(
+ mockResponse({
+ contentType: "text/csv",
+ text: () => Promise.resolve("a,b"),
+ }),
+ );
+ expect(result).toEqual({ ok: true, status: 200, body: "a,b" });
+ });
+
+ it("parses a non-ok JSON error body", async () => {
+ const body = { message: "boom" };
+ const result = await handleResponse(
+ mockResponse({
+ ok: false,
+ status: 400,
+ json: () => Promise.resolve(body),
+ }),
+ );
+ expect(result).toEqual({ ok: false, status: 400, body });
+ });
+
+ it("swallows an unparseable error body to null rather than throwing", async () => {
+ const result = await handleResponse(
+ mockResponse({
+ ok: false,
+ status: 500,
+ json: () =>
+ Promise.reject(new SyntaxError("Unexpected end of JSON input")),
+ }),
+ );
+ expect(result).toEqual({ ok: false, status: 500, body: null });
+ });
+
+ // The dropped 202 special-casing: a streamed success whose body fails to read
+ // (the server aborted the connection mid-stream) must reject, not resolve — that
+ // rejection is how the truncated response surfaces as an error. A mid-stream
+ // abort rejects the read with a TypeError, which becomes a typed
+ // StreamInterruptedError so the UI doesn't misread it as a connectivity failure.
+ it("rejects with a StreamInterruptedError when a body fails to read mid-stream", async () => {
+ await expect(
+ handleResponse(
+ mockResponse({
+ status: 202,
+ json: () => Promise.reject(new TypeError("network error")),
+ }),
+ ),
+ ).rejects.toBeInstanceOf(StreamInterruptedError);
+ });
+
+ // A complete-but-malformed JSON body is a parse problem, not a stream abort —
+ // it must NOT be reclassified as a StreamInterruptedError.
+ it("lets a SyntaxError from a malformed JSON body propagate unchanged", async () => {
+ const syntaxError = new SyntaxError("Unexpected token");
+ await expect(
+ handleResponse(mockResponse({ json: () => Promise.reject(syntaxError) })),
+ ).rejects.toBe(syntaxError);
+ });
+
+ describe("rawResponse", () => {
+ it("returns the untouched Response as the body on success without reading it", async () => {
+ const json = jest.fn();
+ const text = jest.fn();
+ const response = mockResponse({ json, text });
+
+ const result = await handleResponse(response, true);
+
+ expect(result).toEqual({ ok: true, status: 200, body: response });
+ expect(json).not.toHaveBeenCalled();
+ expect(text).not.toHaveBeenCalled();
+ });
+
+ it("falls through to read the body as error data on failure", async () => {
+ const body = { message: "nope" };
+ const result = await handleResponse(
+ mockResponse({
+ ok: false,
+ status: 403,
+ json: () => Promise.resolve(body),
+ }),
+ true,
+ );
+ expect(result).toEqual({ ok: false, status: 403, body });
+ });
+ });
+});
describe("substituteUrlTags", () => {
it("returns the URL unchanged when no tags are present", () => {
diff --git a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/VisualizationError.tsx b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/VisualizationError.tsx
index e8a064e6663..4061cca57dc 100644
--- a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/VisualizationError.tsx
+++ b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/VisualizationError.tsx
@@ -2,7 +2,7 @@ import cx from "classnames";
import { getIn } from "icepick";
import { t } from "ttag";
-import { isNetworkError } from "metabase/api/client";
+import { isNetworkError, isStreamInterruptedError } from "metabase/api/client";
import { EmptyState } from "metabase/common/components/EmptyState";
import { ErrorDetails } from "metabase/common/components/ErrorDetails/ErrorDetails";
import { ErrorMessage } from "metabase/common/components/ErrorMessage";
@@ -46,6 +46,23 @@ export function VisualizationError({
const showMetabaseLinks = useSelector(getShowMetabaseLinks);
const isNative = question && Lib.queryDisplayInfo(query).isNative;
+ // The response committed and then the stream broke partway through — typically
+ // a query that errored after results started streaming, aborting the
+ // connection. We can't recover the reason, but it is NOT necessarily a
+ // connectivity/server-outage problem, so don't show the "server issues"
+ // message that implies the user should wait for the server to recover.
+ if (isStreamInterruptedError(error)) {
+ return (
+ <ErrorMessage
+ className={className}
+ type="serverError"
+ title={t`This question didn't finish loading`}
+ message={t`The results stopped before the query finished. This can happen when a query runs into an error after it starts returning data. Try running it again.`}
+ action={<AdminEmail />}
+ />
+ );
+ }
+
// Treat transport-level failures (server dropped connection, offline, etc.)
// the same as an HTTP error with a status — the user just needs to know the
// server isn't reachable, not see a stack trace.
diff --git a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/common.unit.spec.ts b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/common.unit.spec.ts
index 0491a620bda..a2ed87a17a0 100644
--- a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/common.unit.spec.ts
+++ b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/common.unit.spec.ts
@@ -1,9 +1,26 @@
import { screen } from "__support__/ui";
+import { StreamInterruptedError } from "metabase/api/client";
import { createMockCard, createMockDatabase } from "metabase-types/api/mocks";
import { setup } from "./setup";
describe("VisualizationError (OSS)", () => {
+ describe("interrupted stream", () => {
+ it("shows a query-didn't-finish message, not a connectivity/server-issues one", () => {
+ setup({ error: new StreamInterruptedError("network error") });
+
+ expect(
+ screen.getByText("This question didn't finish loading"),
+ ).toBeInTheDocument();
+ // It must not be misattributed to a server outage / connectivity problem
+ expect(
+ screen.queryByText("We're experiencing server issues"),
+ ).not.toBeInTheDocument();
+ // ...and the raw transport message is not leaked to the user
+ expect(screen.queryByText(/network error/i)).not.toBeInTheDocument();
+ });
+ });
+
describe("SQL databases", () => {
const database = createMockDatabase({
engine: "postgres",
diff --git a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/setup.tsx b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/setup.tsx
index 59f42379f9d..a4142483432 100644
--- a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/setup.tsx
+++ b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/setup.tsx
@@ -6,7 +6,12 @@ import { createMockEntitiesState } from "__support__/store";
import { renderWithProviders } from "__support__/ui";
import { createMockState } from "metabase/redux/store/mocks";
import { checkNotNull } from "metabase/utils/types";
-import type { Card, Database, TokenFeatures } from "metabase-types/api";
+import type {
+ Card,
+ Database,
+ DatasetError,
+ TokenFeatures,
+} from "metabase-types/api";
import {
createMockCard,
createMockDatabase,
@@ -18,6 +23,9 @@ import { VisualizationError } from "../VisualizationError";
export interface SetupOpts {
database?: Database;
card?: Card;
+ // `DatasetError` doesn't model it, but at runtime the component is also handed
+ // thrown `Error` instances (network/stream failures), so allow them here too.
+ error?: DatasetError | Error;
showMetabaseLinks?: boolean;
tokenFeatures?: Partial<TokenFeatures>;
enterprisePlugins?: Parameters<typeof setupEnterpriseOnlyPlugin>[0][];
@@ -26,6 +34,7 @@ export interface SetupOpts {
export const setup = ({
database = createMockDatabase(),
card = createMockCard(),
+ error = "An error occurred",
showMetabaseLinks = true,
tokenFeatures = {},
enterprisePlugins = [],
@@ -57,7 +66,7 @@ export const setup = ({
<VisualizationError
question={question}
duration={0}
- error="An error occurred"
+ error={error as DatasetError}
via={[]}
/>,
{ storeInitialState: state },
diff --git a/frontend/src/metabase/redux/downloads.ts b/frontend/src/metabase/redux/downloads.ts
index 92ddd132802..0e8c542611c 100644
--- a/frontend/src/metabase/redux/downloads.ts
+++ b/frontend/src/metabase/redux/downloads.ts
@@ -240,6 +240,24 @@ export const downloadQueryResults = createAsyncThunk(
},
);
+/**
+ * Read a download response body as a Blob. When a query fails *after* the
+ * download has started streaming, the server has already committed the HTTP
+ * status, so it signals the failure by aborting the connection. That truncates
+ * the response and makes `blob()` reject — surface a clean, localized error
+ * instead of a raw network error like "Failed to fetch", so the user knows the
+ * file did not download completely.
+ */
+export const readDownloadBlob = async (response: Response): Promise<Blob> => {
+ try {
+ return await response.blob();
+ } catch {
+ throw new Error(
+ t`The download was interrupted and the file may be incomplete. Please try again.`,
+ );
+ }
+};
+
export const downloadDataset = createAsyncThunk(
"metabase/downloads/downloadDataset",
async (
@@ -257,7 +275,7 @@ export const downloadDataset = createAsyncThunk(
try {
const response = await promise.unwrap();
const fileName = getDatasetFileName(response.headers, opts.type);
- const fileContent = await response.blob();
+ const fileContent = await readDownloadBlob(response);
openSaveDialog(fileName, fileContent);
return { id, fileName };
diff --git a/frontend/src/metabase/redux/downloads.unit.spec.ts b/frontend/src/metabase/redux/downloads.unit.spec.ts
index 70d3e2f4dca..a21014de2b0 100644
--- a/frontend/src/metabase/redux/downloads.unit.spec.ts
+++ b/frontend/src/metabase/redux/downloads.unit.spec.ts
@@ -8,6 +8,7 @@ import {
getChartFileName,
getDatasetDownloadUrl,
getDatasetParams,
+ readDownloadBlob,
} from "./downloads";
describe("getDatasetResponse", () => {
@@ -150,6 +151,25 @@ describe("getDatasetParams - public question (uuid-based)", () => {
});
});
+describe("readDownloadBlob", () => {
+ it("returns the blob when the response reads to completion", async () => {
+ const blob = new Blob(["a,b,c"], { type: "text/csv" });
+ const response = { blob: () => Promise.resolve(blob) } as Response;
+
+ await expect(readDownloadBlob(response)).resolves.toBe(blob);
+ });
+
+ it("surfaces a localized error when the stream was aborted mid-download", async () => {
+ const response = {
+ blob: () => Promise.reject(new TypeError("Failed to fetch")),
+ } as unknown as Response;
+
+ await expect(readDownloadBlob(response)).rejects.toThrow(
+ "The download was interrupted and the file may be incomplete. Please try again.",
+ );
+ });
+});
+
describe("getChartFileName", () => {
beforeEach(() => {
jest.useFakeTimers();
diff --git a/src/metabase/metabot/api.clj b/src/metabase/metabot/api.clj
index aff9da7a023..e955908c1bf 100644
--- a/src/metabase/metabot/api.clj
+++ b/src/metabase/metabot/api.clj
@@ -140,7 +140,19 @@
(log/error t "Native agent stream failed"
{:conversation-id conversation-id
:assistant-msg-id assistant-msg-id
- :external-id external-id}))
+ :external-id external-id})
+ ;; Stream a well-formed AI SDK error part so the client surfaces the failure
+ ;; instead of treating the truncated stream as a silent success. Unlike binary
+ ;; downloads (which abort the connection), an event stream carries its own error
+ ;; framing, so we emit the error event and then let the body fn return to close
+ ;; the socket cleanly — aborting here would deny the client this very event.
+ (try
+ (let [error-line (self.core/format-error-line
+ {:error (metabot.persistence/throwable->error-payload t)})]
+ (.write os (.getBytes (str error-line "\n") "UTF-8"))
+ (.flush os))
+ (catch org.eclipse.jetty.io.EofException _
+ (vreset! canceled? true))))
(finally
(try
(let [combined-parts (into [] (metabot.persistence/combine-text-parts-xf) @parts-atom)
diff --git a/src/metabase/server/streaming_response.clj b/src/metabase/server/streaming_response.clj
index 401c1cdaa35..978730f4ff9 100644
--- a/src/metabase/server/streaming_response.clj
+++ b/src/metabase/server/streaming_response.clj
@@ -26,7 +26,7 @@
(java.util.concurrent Future)
(java.util.concurrent.atomic AtomicBoolean)
(java.util.zip GZIPOutputStream)
- (org.eclipse.jetty.ee9.nested Request)
+ (org.eclipse.jetty.ee9.nested HttpChannel Request)
(org.eclipse.jetty.io EofException SocketChannelEndPoint)))
(set! *warn-on-reflection* true)
@@ -40,13 +40,8 @@
([^OutputStream os ^bytes ba ^Integer offset ^Integer len]
(.write os ba offset len)))
-(defn- ex-status-code [e]
- (or (some #((some-fn :status-code :status) (ex-data %))
- (take-while some? (iterate ex-cause e)))
- 500))
-
(defn- format-exception [e]
- (cond-> (assoc (Throwable->map e) :_status (ex-status-code e))
+ (cond-> (Throwable->map e)
(server.settings/hide-stacktraces) (dissoc :via :trace)))
(def ^:dynamic *response*
@@ -56,6 +51,13 @@
[[set-content-type!]] to interact with it."
nil)
+(def ^:dynamic *request*
+ "The Jetty `Request` for the current streaming response.
+ Bound automatically inside `streaming-response` bodies in the Jetty async path.
+ Used by [[abort-connection!]] to tear down the underlying connection when an error
+ occurs after the response has already been committed."
+ nil)
+
(def ^:dynamic *completed?*
"An `AtomicBoolean` that is set to `true` when the async context has been completed,
either by the worker thread or by Jetty's timeout/error callbacks. When `true`, the
@@ -126,8 +128,54 @@
(dissoc :export-format)
(cond-> (server.settings/hide-stacktraces) (dissoc :stacktrace :trace :via))))
+(defn- abort-connection!
+ "Abort the underlying Jetty connection for an *already-committed* streaming response, so it
+ terminates without a clean chunked/gzip terminator. The client's `fetch`/`blob()` then
+ rejects with a network error instead of silently accepting a truncated body that has a
+ JSON error blob appended to it. Marks the async context completed so the worker thread's
+ normal `.complete` call is skipped — the abort tears the connection down itself."
+ []
+ (when (and *request* *completed?*
+ (.compareAndSet ^AtomicBoolean *completed?* false true))
+ (try
+ (let [^HttpChannel channel (.getHttpChannel ^Request *request*)]
+ (.abort channel (EofException. "Aborting streaming response after a mid-stream error")))
+ (catch Throwable e
+ ;; The abort failed, so it won't tear the connection down. Revert the flag we
+ ;; optimistically set so the worker thread's `finally` in `do-f-async` can still
+ ;; `.complete` the async context, instead of leaving the request to hang until
+ ;; Jetty's async timeout fires.
+ (.set ^AtomicBoolean *completed?* false)
+ (log/error e "Error aborting streaming connection after mid-stream error")))))
+
+(defn- write-error-to-stream!
+ "Serialize `obj` as a JSON error body onto `os` and close the stream. Used when there is no
+ committed HTTP response to abort — either an uncommitted error response, or a plain output
+ stream with no Ring response bound (e.g. [[metabase.query-processor.streaming/do-with-streaming-rff]]
+ used directly)."
+ [^OutputStream os obj export-format]
+ (with-open [os os]
+ (log/trace (u/pprint-to-str (list 'write-error! obj)))
+ (try
+ (let [obj (sanitize-error-obj obj export-format)]
+ (with-open [writer (BufferedWriter. (OutputStreamWriter. os StandardCharsets/UTF_8))]
+ (json/encode-to obj writer {})))
+ (catch EofException _)
+ (catch Throwable e
+ (log/error e "Error writing error to output stream" obj)))))
+
(defn write-error!
- "Write an error to the output stream, formatting it nicely. Closes output stream afterwards.
+ "Handle an error that occurred while producing a streaming response.
+
+ - When the HTTP response is *not yet committed*, send a normal error response: set the
+ status code and `application/json` content type, then write the error as a JSON body.
+ - When the response is *already committed* (the error happened mid-stream, after the status
+ line and some body bytes are on the wire), abort the connection via [[abort-connection!]].
+ We can no longer signal failure through the status code or a parseable trailer, so a clean
+ close with an appended error blob would be silently accepted by the client as a successful
+ (but corrupt) download. Aborting makes the client's `fetch`/`blob()` reject instead.
+ - When no HTTP response is bound (a plain output stream), just write the JSON error body.
+
No-op if the async context has already been completed (response and stream may be recycled)."
([os obj export-format]
(write-error! os obj export-format nil))
@@ -142,20 +190,19 @@
(instance? Throwable obj)
(recur os (format-exception obj) export-format status-code)
+ (nil? *response*)
+ (write-error-to-stream! os obj export-format)
+
+ (committed?)
+ (do
+ (log/trace "Streaming response already committed; aborting connection to signal mid-stream error")
+ (abort-connection!))
+
:else
(do
- (when (and *response* (not (committed?)))
- (set-status! (or status-code 500))
- (set-content-type! "application/json"))
- (with-open [os os]
- (log/trace (u/pprint-to-str (list 'write-error! obj)))
- (try
- (let [obj (sanitize-error-obj obj export-format)]
- (with-open [writer (BufferedWriter. (OutputStreamWriter. os StandardCharsets/UTF_8))]
- (json/encode-to obj writer {})))
- (catch EofException _)
- (catch Throwable e
- (log/error e "Error writing error to output stream" obj))))))))
+ (set-status! (or status-code 500))
+ (set-content-type! "application/json")
+ (write-error-to-stream! os obj export-format)))))
(defn- do-f* [f ^OutputStream os _finished-chan canceled-chan]
(try
@@ -186,10 +233,11 @@
"Runs `f` asynchronously on the streaming response `thread-pool`, returning immediately. When `f` finishes,
completes (i.e., closes) Jetty `async-context`. `completed?` is an `AtomicBoolean` used to coordinate with
Jetty's timeout/error callbacks so that only one path calls `.complete`."
- [^AsyncContext async-context response f ^OutputStream os finished-chan canceled-chan ^AtomicBoolean completed?]
+ [^AsyncContext async-context response ^Request request f ^OutputStream os finished-chan canceled-chan ^AtomicBoolean completed?]
{:pre [(some? os)]}
(let [task (^:once fn* []
(binding [*response* response
+ *request* request
*completed?* completed?]
(try
(do-f* f os finished-chan canceled-chan)
@@ -350,13 +398,22 @@
;; by the client because of `start-async-cancel-loop!`. The latter tries to read a
;; byte from the input stream at some interval, and that may/will cause corruption
;; of the subsequent requests that come through the reused connection (see #46071).
- "Connection" "close")
+ "Connection" "close"
+ ;; Force chunked transfer encoding so the body has a positive terminator (the
+ ;; final zero-length chunk). Without it, `Connection: close` makes Jetty delimit
+ ;; the body by connection close, which is indistinguishable from a truncated
+ ;; stream — so a mid-stream failure that aborts the connection (see
+ ;; [[abort-connection!]]) would be silently accepted by the client as a complete
+ ;; body. With chunked framing the client gets a protocol error instead.
+ ;; (HTTP/2 has no chunked encoding, but an aborted stream becomes an RST_STREAM,
+ ;; which the client surfaces as an error just the same.)
+ "Transfer-Encoding" "chunked")
gzip? (assoc "Content-Encoding" "gzip"))]
(#'servlet/set-headers response headers)
(let [output-stream-delay (output-stream-delay gzip? response)
delay-os (delay-output-stream output-stream-delay)]
(start-async-cancel-loop! request finished-chan canceled-chan)
- (do-f-async async-context response f delay-os finished-chan canceled-chan completed?)))
+ (do-f-async async-context response request f delay-os finished-chan canceled-chan completed?)))
(catch Throwable e
(log/error e "Unexpected exception in do-f-async")
(try
diff --git a/test/metabase/metabot/api_test.clj b/test/metabase/metabot/api_test.clj
index 7b5703a3444..8d810824c1d 100644
--- a/test/metabase/metabot/api_test.clj
+++ b/test/metabase/metabot/api_test.clj
@@ -182,16 +182,21 @@
(mt/with-model-cleanup [:model/MetabotMessage
[:model/MetabotConversation :created_at]]
(let [conversation-id (str (random-uuid))
- body (mt/user-real-request :rasta :post 202 "metabot/agent-streaming"
- {:request-options {:as :stream
- :decompress-body false}}
- {:message "Test closure"
- :context {}
- :conversation_id conversation-id
- :history []
- :state {}})]
- (.read ^java.io.InputStream body) ;; start the handler
- (.close ^java.io.Closeable body)
+ response (mt/user-real-request-full-response
+ :rasta :post 202 "metabot/agent-streaming"
+ {:request-options {:as :stream
+ :decompress-body false}}
+ {:message "Test closure"
+ :context {}
+ :conversation_id conversation-id
+ :history []
+ :state {}})]
+ (.read ^java.io.InputStream (:body response)) ;; start the handler
+ ;; Close the underlying client, not the body stream: closing the body would
+ ;; make clj-http drain the (now chunked) response to completion, which looks
+ ;; like a normal finish rather than a disconnect. Closing the client aborts
+ ;; the connection, which is what the server's cancel loop detects.
+ (.close ^java.io.Closeable (:http-client response))
(u/poll {:thunk #(deref stored-parts)
:done? some?
:interval-ms 10
@@ -229,25 +234,28 @@
(reset! stored-kwargs (apply hash-map kwargs)))]
(mt/with-model-cleanup [:model/MetabotMessage
[:model/MetabotConversation :created_at]]
- (mt/user-http-request :rasta :post 202 "metabot/agent-streaming"
- {:message "go"
- :context {}
- :conversation_id (str (random-uuid))
- :history []
- :state {}})
- (u/poll {:thunk #(deref stored-kwargs)
- :done? some?
- :interval-ms 10
- :timeout-ms 3000})
- (is (some? @stored-kwargs)
- "finalize-assistant-turn! is called from the finally even when setup threw")
- (is (true? (:finished? @stored-kwargs))
- "a thrown turn is :finished? true — `finished=false` is reserved for client aborts")
- (is (=? {:message #"(?i)agent setup exploded"
- :type "clojure.lang.ExceptionInfo"
- :data {:status 503 :provider :test}}
- (:error @stored-kwargs))
- "the throwable becomes a structured error payload"))))))))
+ (let [response (mt/user-http-request :rasta :post 202 "metabot/agent-streaming"
+ {:message "go"
+ :context {}
+ :conversation_id (str (random-uuid))
+ :history []
+ :state {}})]
+ (u/poll {:thunk #(deref stored-kwargs)
+ :done? some?
+ :interval-ms 10
+ :timeout-ms 3000})
+ (is (some? @stored-kwargs)
+ "finalize-assistant-turn! is called from the finally even when setup threw")
+ (is (true? (:finished? @stored-kwargs))
+ "a thrown turn is :finished? true — `finished=false` is reserved for client aborts")
+ (is (=? {:message #"(?i)agent setup exploded"
+ :type "clojure.lang.ExceptionInfo"
+ :data {:status 503 :provider :test}}
+ (:error @stored-kwargs))
+ "the throwable becomes a structured error payload")
+ (testing "the failure is streamed to the client as an AI SDK error part (3:...) rather than a silent close"
+ (is (some #(str/starts-with? % "3:") (str/split-lines response)))
+ (is (re-find #"(?i)agent setup exploded" response)))))))))))
(deftest settings-get-returns-live-models-test
(mt/with-temporary-setting-values [metabot.settings/llm-metabot-provider "anthropic/claude-haiku-4-5"
diff --git a/test/metabase/server/streaming_response_test.clj b/test/metabase/server/streaming_response_test.clj
index 32af553eaaf..6bdc7cd6bea 100644
--- a/test/metabase/server/streaming_response_test.clj
+++ b/test/metabase/server/streaming_response_test.clj
@@ -203,6 +203,35 @@
(finally
(.stop server)))))
+(deftest abort-on-committed-error-test
+ (testing "An error after the response is committed aborts the connection so the client cannot read a complete body"
+ (let [handler (fn [req respond _raise]
+ (respond
+ (compojure.response/render
+ (streaming-response/streaming-response {:content-type "text/csv"} [os _canceled-chan]
+ ;; write + flush some bytes so the response commits, then fail mid-stream
+ (.write os (.getBytes "a,b,c\n1,2,3\n" "UTF-8"))
+ (.flush os)
+ (streaming-response/write-error! os {:error "boom"} :csv 500))
+ req)))
+ server (doto (server.instance/create-server handler {:port 0 :join? false})
+ .start)
+ url (str "http://localhost:" (.. server getURI getPort))
+ consume (fn [] (let [res (http/request {:method :get, :url url, :as :stream, :decompress-body false})]
+ [res (slurp (:body res))]))]
+ (try
+ (testing "the response is chunked so a missing terminator is detectable"
+ ;; can't read the body cleanly, so just open a request to inspect the headers
+ (let [res (http/request {:method :get, :url url, :as :stream, :decompress-body false})]
+ (is (= "chunked" (get-in res [:headers "transfer-encoding"])))
+ (u/ignore-exceptions (.close ^InputStream (:body res)))))
+ (testing "consuming the whole body throws because the stream was aborted without a clean chunk terminator"
+ (is (thrown? Exception (consume))))
+ (testing "no JSON error blob is appended to the body"
+ (is (not (re-find #"boom" (try (second (consume)) (catch Exception _ ""))))))
+ (finally
+ (.stop server))))))
+
(def ^:private ^:dynamic *number-of-cans* nil)
(deftest ^:parallel preserve-bindings-test
@@ -249,7 +278,7 @@
(is (= "application/json" @content-type-called))))))
(deftest write-error-committed-response-test
- (testing "write-error! should not set status or content type when response is committed"
+ (testing "write-error! aborts instead of appending an error blob when the response is already committed"
(let [os (ByteArrayOutputStream.)
status-called (atom nil)
content-type-called (atom nil)]
@@ -262,7 +291,9 @@
(testing "Status should not be set when response is committed"
(is (nil? @status-called)))
(testing "Content type should not be set when response is committed"
- (is (nil? @content-type-called))))))
+ (is (nil? @content-type-called)))
+ (testing "No error blob should be appended to an already-committed stream"
+ (is (zero? (.size os)))))))
(deftest write-error-no-response-test
(testing "write-error! should not attempt to set status when no *response* is bound"
@@ -380,9 +411,7 @@
(is (not (contains? error-response :via))
"Response should not contain :via key")
(is (= "test-value" (get-in error-response [:data :custom-data]))
- "Response should include custom data from ex-info")
- (is (contains? error-response :_status)
- "Response should include :_status")))))))
+ "Response should include custom data from ex-info")))))))
(deftest write-error-nested-exception-with-stacktraces-disabled-test
(testing "write-error! includes nested exception details when hide-stacktraces is false"
@@ -532,6 +561,7 @@
(complete [_]
(deliver complete-promise true)))
nil
+ nil
(fn [_os _canceled-chan]
(deliver task-started? true)
(try
@@ -598,6 +628,7 @@
(throw (NullPointerException. "recycled"))))
(setContentType [_ _] (when (.get completed?)
(throw (NullPointerException. "recycled")))))
+ nil
(fn [_os _canceled-chan]
(.countDown task-started)
(.await task-can-finish 5 TimeUnit/SECONDS)
diff --git a/test/metabase/test.clj b/test/metabase/test.clj
index 00e18ca8c94..3edb160afbc 100644
--- a/test/metabase/test.clj
+++ b/test/metabase/test.clj
@@ -228,6 +228,7 @@
user-http-request
user-http-request-full-response
user-real-request
+ user-real-request-full-response
with-group
with-group-for-user
with-test-user]
diff --git a/test/metabase/test/data/users.clj b/test/metabase/test/data/users.clj
index e2a18702f28..0499b9cb395 100644
--- a/test/metabase/test/data/users.clj
+++ b/test/metabase/test/data/users.clj
@@ -242,6 +242,13 @@
"Like [[user-http-request]] but instead of calling the app handler, this makes an actual http request."
(partial user-request client/real-client))
+(def ^{:arglists '([test-user-name-or-user-or-id method expected-status-code? endpoint
+ request-options? http-body-map? & {:as query-params}])} user-real-request-full-response
+ "Like [[user-real-request]] but returns the full HTTP response map instead of just the body. Useful for
+ tests that need the underlying `:http-client` (e.g. to forcibly close the connection and simulate a client
+ disconnect on a streaming response)."
+ (partial user-request client/client-real-response))
+
(defn do-with-test-user [user-kwd thunk]
(t/testing (format "\nwith test user %s\n" user-kwd)
(request/with-current-user (some-> user-kwd user->id)