The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix silent data corruption when a query errors mid-stream (#75586)
Romeo Van Snick1b0e9e91d418
diff --git a/e2e/test/scenarios/sharing/downloads/mid-stream-download-failure.cy.spec.ts b/e2e/test/scenarios/sharing/downloads/mid-stream-download-failure.cy.spec.tsnew file mode 100644index 00000000000..baf7f2c0fd5--- /dev/null+++ b/e2e/test/scenarios/sharing/downloads/mid-stream-download-failure.cy.spec.ts@@ -0,0 +1,47 @@+const { H } = cy;++// Reproduction for the streaming-download corruption bug: when a query fails+// *after* the download stream has started, the server used to append a JSON error+// blob and close the connection cleanly. The client couldn't tell, so it saved a+// corrupt file and reported success. The server now aborts the connection+// mid-stream and the client surfaces a download error instead.+//+// To fail *mid-stream* (not before the response commits) the query has to start+// streaming and then blow up. This native query previews fine — the query builder+// caps results at 2000 rows, below the division-by-zero at row 5000 — but the CSV+// export runs unbounded, streams past the failing row, and aborts.+describe("scenarios > sharing > downloads > mid-stream failure", () => {+ beforeEach(() => {+ H.restore();+ cy.signInAsAdmin();+ });++ it("surfaces a download error when a query fails after the stream has started", () => {+ H.createNativeQuestion({+ name: "Fails mid-stream",+ native: {+ query: "SELECT 100 / (x - 5000) AS v FROM SYSTEM_RANGE(1, 10000)",+ },+ }).then(({ body: card }) => {+ cy.intercept("POST", `/api/card/${card.id}/query`).as("preview");+ H.visitQuestion(card.id);+ });++ cy.log("Preview loads because it is capped below the failing row");+ cy.wait("@preview");+ cy.findByLabelText("Download results").should("be.visible").click();++ cy.log("Trigger a full CSV export, which streams past the failing row");+ H.popover().within(() => {+ cy.findByText(".csv").click();+ cy.findByTestId("download-results-button").click();+ });++ cy.log(+ "The aborted stream is surfaced as a download error, not a silent success",+ );+ cy.findByTestId("status-root-container", { timeout: 15000 })+ .should("contain", "Download error")+ .and("not.contain", "Download completed");+ });+});diff --git a/frontend/src/metabase/api/client/client.ts b/frontend/src/metabase/api/client/client.tsindex e20cb5edd14..6cc9770edaa 100644--- a/frontend/src/metabase/api/client/client.ts+++ b/frontend/src/metabase/api/client/client.ts@@ -309,7 +309,7 @@ export class ApiClient extends EventEmitter<EventMap> {/*** Resolve with the raw `Response`, leaving everything after the network to the- * caller: unlike `request`, it does not read the body, recover a 202 `_status`,+ * caller: unlike `request`, it does not read the body,* emit events, or throw on a non-2xx status. For streaming callers (e.g. SSE)* that consume `response.body` and do their own error handling, while still* getting the client pipeline — middleware, client headers, anti-CSRF, anddiff --git a/frontend/src/metabase/api/client/errors.ts b/frontend/src/metabase/api/client/errors.tsindex 5cafc1aac2c..a097fcd984c 100644--- a/frontend/src/metabase/api/client/errors.ts+++ b/frontend/src/metabase/api/client/errors.ts@@ -15,6 +15,28 @@ export function isNetworkError(error: unknown): error is NetworkError {return error instanceof NetworkError;}+/**+ * Thrown when a response *was* received — its status line and some body bytes+ * already committed — but reading the body then failed partway through. A+ * streamed query or export that errors mid-stream aborts the connection without+ * a clean terminator, which rejects the body read. This is deliberately distinct+ * from `NetworkError` (where the transport never delivered a response at all):+ * the server answered and then the stream broke, so the UI must not blame the+ * user's connectivity or imply the server is down.+ */+export class StreamInterruptedError extends Error {+ constructor(message = "Stream interrupted") {+ super(message);+ this.name = "StreamInterruptedError";+ }+}++export function isStreamInterruptedError(+ error: unknown,+): error is StreamInterruptedError {+ return error instanceof StreamInterruptedError;+}+/*** `true` for the standard `DOMException` of name `"AbortError"` that* `fetch()` rejects with when its signal aborts. Use this in place of thediff --git a/frontend/src/metabase/api/client/utils.ts b/frontend/src/metabase/api/client/utils.tsindex e013c6c4864..05abb396dcb 100644--- a/frontend/src/metabase/api/client/utils.ts+++ b/frontend/src/metabase/api/client/utils.ts@@ -1,3 +1,5 @@+import { StreamInterruptedError } from "./errors";+type ResponseResult = {ok: boolean;status: number;@@ -59,10 +61,6 @@ function isJson(response: Response): boolean {);}-function isOk(status: number): boolean {- return 200 <= status && status <= 299;-}-/*** Parse a JSON body stream natively — `response.json()` never materializes the* payload as a JS string the way `text()` + `JSON.parse` would. A body that@@ -79,108 +77,69 @@ async function parseJsonOrNull(response: Response): Promise<unknown> {}/**- * Interpret a fetched `Response` into a `{ ok, status, body }` result, without- * emitting or throwing — the caller decides what to do with a non-ok status.+ * Read a response body by its `Content-Type`: JSON natively (never materializing+ * an intermediate string), otherwise text. A successful JSON body must be+ * parseable, so a parse failure throws; a failed response's body is error data,+ * so an empty or garbage body is swallowed to `null` rather than masking the+ * status with a `SyntaxError`.*- * The body is only read when it'll actually be used — as the result of a normal- * request, as error data, or to recover the real status of a 202 streaming- * response (whose status hides in a `{_status}` body). A `rawResponse` caller- * whose non-202 request succeeded gets the `Response` itself as `body`, unread,- * so a binary payload like a map tile is never decoded as text.+ * The response is already committed by the time we read it, so a read that+ * rejects with a `TypeError` is the body stream breaking mid-flight — a streamed+ * query/export that errored after committing aborts the connection without a+ * clean terminator. Surface that as a typed `StreamInterruptedError` so the UI+ * can distinguish it from a genuine connectivity failure (where no response+ * arrives at all) instead of showing a misleading "server issues" message. A+ * `SyntaxError` from a complete-but-malformed body is a real parse problem and+ * propagates unchanged.*/-export async function handleResponse(- response: Response,- rawResponse?: boolean,-): Promise<ResponseResult> {- // Hand back the raw Response only when the caller asked for it and the request- // succeeded. A failure falls through below so its body can be read as error- // data; a 202 counts as ok here and is sorted out next.- if (rawResponse && response.ok) {- if (response.status === 202) {- // A 202's HTTP status can't be trusted (a streamed export commits it- // before the work can fail), so read the body to learn the real status.- // The caller still needs an unread Response, so clone before reading.- const unreadResponse = response.clone();- const info = await readBody(response);- if (!info.ok) {- return info;- }- return { ...info, body: unreadResponse };+async function readBody(response: Response): Promise<unknown> {+ try {+ if (!isJson(response)) {+ return await response.text();}-- // A non-202 status is authoritative, so skip reading the body entirely and- // return the untouched Response — this is what keeps a binary payload like a- // map tile from being decoded as text.- return { ok: true, status: response.status, body: response };- }-- // Either the caller doesn't want the raw Response (the parsed body IS the- // result) or the request failed (the body is the error payload). Read it and- // let the caller inspect `ok`.- return await readBody(response);-}--/**- * A streamed 202 signals its real status in a `{_status: N}` body.- */-function streamedStatus(body: unknown): number | undefined {- if (- body &&- typeof body === "object" &&- "_status" in body &&- typeof body._status === "number" &&- body._status > 0- ) {- return body._status;- }- return undefined;-}--/**- * Read the body by its `Content-Type`: JSON natively (never materializing an- * intermediate string), otherwise text. A successful JSON body must be- * parseable, so a parse failure throws; on a failed response the body is error- * data, so an empty or garbage body is swallowed to `null` rather than masking- * the status with a `SyntaxError`.- */-async function readResponseBody(- response: Response,- ok: boolean,-): Promise<unknown> {- if (!isJson(response)) {- return response.text();+ return response.ok+ ? await response.json()+ : await parseJsonOrNull(response);+ } catch (error) {+ if (error instanceof TypeError) {+ throw new StreamInterruptedError(error.message);+ }+ throw error;}- return ok ? response.json() : parseJsonOrNull(response);}/**- * Read and interpret a response into `{ ok, status, body }`, routed by status:+ * Interpret a fetched `Response` into a `{ ok, status, body }` result. It does+ * not throw on a non-ok status — the caller decides what to do with it — but a+ * body that fails to read (e.g. a streamed response aborted mid-flight) still+ * rejects, which is how a mid-stream failure surfaces.*- * - 202: a streamed query/download commits the 202 before its work can fail, so- * read the body to recover the real status from a `{_status}` payload. The- * read stays tolerant — a non-`_status` body (even a non-JSON export) is a- * success streamed straight through.- * TODO: a JSON error emitted mid-stream parses to a non-`_status` object and- * is silently treated as success.- * - 204: no content.- * - otherwise: the status line is authoritative; see `readResponseBody`.- *- * Does not throw on a non-2xx status — the caller inspects `ok`.+ * The body is only read when it'll actually be used — as the result of a normal+ * request, or as error data. A `rawResponse` caller whose request succeeded gets+ * the `Response` itself as `body`, unread, so a binary payload like a map tile or+ * a streamed export is never decoded as text.*/-async function readBody(response: Response): Promise<ResponseResult> {- if (response.status === 202) {- const body = await parseJsonOrNull(response);- const status = streamedStatus(body) ?? 202;- return { ok: isOk(status), status, body };+export async function handleResponse(+ response: Response,+ rawResponse?: boolean,+): Promise<ResponseResult> {+ // The status is authoritative, so skip reading the body and return the untouched+ // Response — this keeps a binary payload like a map tile or a streamed export+ // from being decoded as text. A streamed export that fails mid-flight aborts the+ // connection, so a body that reads to completion is a genuine success; there is+ // no error blob to recover from the body here. A failed rawResponse request+ // falls through so its body can be read as error data.+ if (rawResponse && response.ok) {+ return { ok: true, status: response.status, body: response };}if (response.status === 204) {+ // Empty response: body is nullreturn { ok: true, status: 204, body: null };}- const { status, ok } = response;- const body = await readResponseBody(response, ok);- return { ok, status, body };+ const { ok, status } = response;+ return { ok, status, body: await readBody(response) };}// `:tag` placeholders are URL-encoded (slashes become %2F).diff --git a/frontend/src/metabase/api/client/utils.unit.spec.ts b/frontend/src/metabase/api/client/utils.unit.spec.tsindex 3b56c93fe79..e6e22e5a86f 100644--- a/frontend/src/metabase/api/client/utils.unit.spec.ts+++ b/frontend/src/metabase/api/client/utils.unit.spec.ts@@ -1,4 +1,134 @@-import { substituteUrlTags } from "./utils";+import { StreamInterruptedError } from "./errors";+import { handleResponse, substituteUrlTags } from "./utils";++type MockResponseOptions = {+ ok?: boolean;+ status?: number;+ contentType?: string | null;+ json?: () => Promise<unknown>;+ text?: () => Promise<string>;+};++const mockResponse = ({+ ok = true,+ status = 200,+ contentType = "application/json",+ json = () => Promise.resolve({}),+ text = () => Promise.resolve(""),+}: MockResponseOptions): Response =>+ ({+ ok,+ status,+ headers: {+ get: (name: string) =>+ name.toLowerCase() === "content-type" ? contentType : null,+ },+ json,+ text,+ }) as unknown as Response;++describe("handleResponse", () => {+ it("parses a successful JSON body", async () => {+ const body = { id: 1 };+ const result = await handleResponse(+ mockResponse({ json: () => Promise.resolve(body) }),+ );+ expect(result).toEqual({ ok: true, status: 200, body });+ });++ it("returns a null body for an empty 204", async () => {+ const result = await handleResponse(+ mockResponse({ status: 204, contentType: null }),+ );+ expect(result).toEqual({ ok: true, status: 204, body: null });+ });++ it("reads a non-JSON body as text", async () => {+ const result = await handleResponse(+ mockResponse({+ contentType: "text/csv",+ text: () => Promise.resolve("a,b"),+ }),+ );+ expect(result).toEqual({ ok: true, status: 200, body: "a,b" });+ });++ it("parses a non-ok JSON error body", async () => {+ const body = { message: "boom" };+ const result = await handleResponse(+ mockResponse({+ ok: false,+ status: 400,+ json: () => Promise.resolve(body),+ }),+ );+ expect(result).toEqual({ ok: false, status: 400, body });+ });++ it("swallows an unparseable error body to null rather than throwing", async () => {+ const result = await handleResponse(+ mockResponse({+ ok: false,+ status: 500,+ json: () =>+ Promise.reject(new SyntaxError("Unexpected end of JSON input")),+ }),+ );+ expect(result).toEqual({ ok: false, status: 500, body: null });+ });++ // The dropped 202 special-casing: a streamed success whose body fails to read+ // (the server aborted the connection mid-stream) must reject, not resolve — that+ // rejection is how the truncated response surfaces as an error. A mid-stream+ // abort rejects the read with a TypeError, which becomes a typed+ // StreamInterruptedError so the UI doesn't misread it as a connectivity failure.+ it("rejects with a StreamInterruptedError when a body fails to read mid-stream", async () => {+ await expect(+ handleResponse(+ mockResponse({+ status: 202,+ json: () => Promise.reject(new TypeError("network error")),+ }),+ ),+ ).rejects.toBeInstanceOf(StreamInterruptedError);+ });++ // A complete-but-malformed JSON body is a parse problem, not a stream abort —+ // it must NOT be reclassified as a StreamInterruptedError.+ it("lets a SyntaxError from a malformed JSON body propagate unchanged", async () => {+ const syntaxError = new SyntaxError("Unexpected token");+ await expect(+ handleResponse(mockResponse({ json: () => Promise.reject(syntaxError) })),+ ).rejects.toBe(syntaxError);+ });++ describe("rawResponse", () => {+ it("returns the untouched Response as the body on success without reading it", async () => {+ const json = jest.fn();+ const text = jest.fn();+ const response = mockResponse({ json, text });++ const result = await handleResponse(response, true);++ expect(result).toEqual({ ok: true, status: 200, body: response });+ expect(json).not.toHaveBeenCalled();+ expect(text).not.toHaveBeenCalled();+ });++ it("falls through to read the body as error data on failure", async () => {+ const body = { message: "nope" };+ const result = await handleResponse(+ mockResponse({+ ok: false,+ status: 403,+ json: () => Promise.resolve(body),+ }),+ true,+ );+ expect(result).toEqual({ ok: false, status: 403, body });+ });+ });+});describe("substituteUrlTags", () => {it("returns the URL unchanged when no tags are present", () => {diff --git a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/VisualizationError.tsx b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/VisualizationError.tsxindex e8a064e6663..4061cca57dc 100644--- a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/VisualizationError.tsx+++ b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/VisualizationError.tsx@@ -2,7 +2,7 @@ import cx from "classnames";import { getIn } from "icepick";import { t } from "ttag";-import { isNetworkError } from "metabase/api/client";+import { isNetworkError, isStreamInterruptedError } from "metabase/api/client";import { EmptyState } from "metabase/common/components/EmptyState";import { ErrorDetails } from "metabase/common/components/ErrorDetails/ErrorDetails";import { ErrorMessage } from "metabase/common/components/ErrorMessage";@@ -46,6 +46,23 @@ export function VisualizationError({const showMetabaseLinks = useSelector(getShowMetabaseLinks);const isNative = question && Lib.queryDisplayInfo(query).isNative;+ // The response committed and then the stream broke partway through — typically+ // a query that errored after results started streaming, aborting the+ // connection. We can't recover the reason, but it is NOT necessarily a+ // connectivity/server-outage problem, so don't show the "server issues"+ // message that implies the user should wait for the server to recover.+ if (isStreamInterruptedError(error)) {+ return (+ <ErrorMessage+ className={className}+ type="serverError"+ title={t`This question didn't finish loading`}+ message={t`The results stopped before the query finished. This can happen when a query runs into an error after it starts returning data. Try running it again.`}+ action={<AdminEmail />}+ />+ );+ }+// Treat transport-level failures (server dropped connection, offline, etc.)// the same as an HTTP error with a status — the user just needs to know the// server isn't reachable, not see a stack trace.diff --git a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/common.unit.spec.ts b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/common.unit.spec.tsindex 0491a620bda..a2ed87a17a0 100644--- a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/common.unit.spec.ts+++ b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/common.unit.spec.ts@@ -1,9 +1,26 @@import { screen } from "__support__/ui";+import { StreamInterruptedError } from "metabase/api/client";import { createMockCard, createMockDatabase } from "metabase-types/api/mocks";import { setup } from "./setup";describe("VisualizationError (OSS)", () => {+ describe("interrupted stream", () => {+ it("shows a query-didn't-finish message, not a connectivity/server-issues one", () => {+ setup({ error: new StreamInterruptedError("network error") });++ expect(+ screen.getByText("This question didn't finish loading"),+ ).toBeInTheDocument();+ // It must not be misattributed to a server outage / connectivity problem+ expect(+ screen.queryByText("We're experiencing server issues"),+ ).not.toBeInTheDocument();+ // ...and the raw transport message is not leaked to the user+ expect(screen.queryByText(/network error/i)).not.toBeInTheDocument();+ });+ });+describe("SQL databases", () => {const database = createMockDatabase({engine: "postgres",diff --git a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/setup.tsx b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/setup.tsxindex 59f42379f9d..a4142483432 100644--- a/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/setup.tsx+++ b/frontend/src/metabase/querying/components/QueryVisualization/VisualizationError/tests/setup.tsx@@ -6,7 +6,12 @@ import { createMockEntitiesState } from "__support__/store";import { renderWithProviders } from "__support__/ui";import { createMockState } from "metabase/redux/store/mocks";import { checkNotNull } from "metabase/utils/types";-import type { Card, Database, TokenFeatures } from "metabase-types/api";+import type {+ Card,+ Database,+ DatasetError,+ TokenFeatures,+} from "metabase-types/api";import {createMockCard,createMockDatabase,@@ -18,6 +23,9 @@ import { VisualizationError } from "../VisualizationError";export interface SetupOpts {database?: Database;card?: Card;+ // `DatasetError` doesn't model it, but at runtime the component is also handed+ // thrown `Error` instances (network/stream failures), so allow them here too.+ error?: DatasetError | Error;showMetabaseLinks?: boolean;tokenFeatures?: Partial<TokenFeatures>;enterprisePlugins?: Parameters<typeof setupEnterpriseOnlyPlugin>[0][];@@ -26,6 +34,7 @@ export interface SetupOpts {export const setup = ({database = createMockDatabase(),card = createMockCard(),+ error = "An error occurred",showMetabaseLinks = true,tokenFeatures = {},enterprisePlugins = [],@@ -57,7 +66,7 @@ export const setup = ({<VisualizationErrorquestion={question}duration={0}- error="An error occurred"+ error={error as DatasetError}via={[]}/>,{ storeInitialState: state },diff --git a/frontend/src/metabase/redux/downloads.ts b/frontend/src/metabase/redux/downloads.tsindex 92ddd132802..0e8c542611c 100644--- a/frontend/src/metabase/redux/downloads.ts+++ b/frontend/src/metabase/redux/downloads.ts@@ -240,6 +240,24 @@ export const downloadQueryResults = createAsyncThunk(},);+/**+ * Read a download response body as a Blob. When a query fails *after* the+ * download has started streaming, the server has already committed the HTTP+ * status, so it signals the failure by aborting the connection. That truncates+ * the response and makes `blob()` reject — surface a clean, localized error+ * instead of a raw network error like "Failed to fetch", so the user knows the+ * file did not download completely.+ */+export const readDownloadBlob = async (response: Response): Promise<Blob> => {+ try {+ return await response.blob();+ } catch {+ throw new Error(+ t`The download was interrupted and the file may be incomplete. Please try again.`,+ );+ }+};+export const downloadDataset = createAsyncThunk("metabase/downloads/downloadDataset",async (@@ -257,7 +275,7 @@ export const downloadDataset = createAsyncThunk(try {const response = await promise.unwrap();const fileName = getDatasetFileName(response.headers, opts.type);- const fileContent = await response.blob();+ const fileContent = await readDownloadBlob(response);openSaveDialog(fileName, fileContent);return { id, fileName };diff --git a/frontend/src/metabase/redux/downloads.unit.spec.ts b/frontend/src/metabase/redux/downloads.unit.spec.tsindex 70d3e2f4dca..a21014de2b0 100644--- a/frontend/src/metabase/redux/downloads.unit.spec.ts+++ b/frontend/src/metabase/redux/downloads.unit.spec.ts@@ -8,6 +8,7 @@ import {getChartFileName,getDatasetDownloadUrl,getDatasetParams,+ readDownloadBlob,} from "./downloads";describe("getDatasetResponse", () => {@@ -150,6 +151,25 @@ describe("getDatasetParams - public question (uuid-based)", () => {});});+describe("readDownloadBlob", () => {+ it("returns the blob when the response reads to completion", async () => {+ const blob = new Blob(["a,b,c"], { type: "text/csv" });+ const response = { blob: () => Promise.resolve(blob) } as Response;++ await expect(readDownloadBlob(response)).resolves.toBe(blob);+ });++ it("surfaces a localized error when the stream was aborted mid-download", async () => {+ const response = {+ blob: () => Promise.reject(new TypeError("Failed to fetch")),+ } as unknown as Response;++ await expect(readDownloadBlob(response)).rejects.toThrow(+ "The download was interrupted and the file may be incomplete. Please try again.",+ );+ });+});+describe("getChartFileName", () => {beforeEach(() => {jest.useFakeTimers();diff --git a/src/metabase/metabot/api.clj b/src/metabase/metabot/api.cljindex aff9da7a023..e955908c1bf 100644--- a/src/metabase/metabot/api.clj+++ b/src/metabase/metabot/api.clj@@ -140,7 +140,19 @@(log/error t "Native agent stream failed"{:conversation-id conversation-id:assistant-msg-id assistant-msg-id- :external-id external-id}))+ :external-id external-id})+ ;; Stream a well-formed AI SDK error part so the client surfaces the failure+ ;; instead of treating the truncated stream as a silent success. Unlike binary+ ;; downloads (which abort the connection), an event stream carries its own error+ ;; framing, so we emit the error event and then let the body fn return to close+ ;; the socket cleanly — aborting here would deny the client this very event.+ (try+ (let [error-line (self.core/format-error-line+ {:error (metabot.persistence/throwable->error-payload t)})]+ (.write os (.getBytes (str error-line "\n") "UTF-8"))+ (.flush os))+ (catch org.eclipse.jetty.io.EofException _+ (vreset! canceled? true))))(finally(try(let [combined-parts (into [] (metabot.persistence/combine-text-parts-xf) @parts-atom)diff --git a/src/metabase/server/streaming_response.clj b/src/metabase/server/streaming_response.cljindex 401c1cdaa35..978730f4ff9 100644--- a/src/metabase/server/streaming_response.clj+++ b/src/metabase/server/streaming_response.clj@@ -26,7 +26,7 @@(java.util.concurrent Future)(java.util.concurrent.atomic AtomicBoolean)(java.util.zip GZIPOutputStream)- (org.eclipse.jetty.ee9.nested Request)+ (org.eclipse.jetty.ee9.nested HttpChannel Request)(org.eclipse.jetty.io EofException SocketChannelEndPoint)))(set! *warn-on-reflection* true)@@ -40,13 +40,8 @@([^OutputStream os ^bytes ba ^Integer offset ^Integer len](.write os ba offset len)))-(defn- ex-status-code [e]- (or (some #((some-fn :status-code :status) (ex-data %))- (take-while some? (iterate ex-cause e)))- 500))-(defn- format-exception [e]- (cond-> (assoc (Throwable->map e) :_status (ex-status-code e))+ (cond-> (Throwable->map e)(server.settings/hide-stacktraces) (dissoc :via :trace)))(def ^:dynamic *response*@@ -56,6 +51,13 @@[[set-content-type!]] to interact with it."nil)+(def ^:dynamic *request*+ "The Jetty `Request` for the current streaming response.+ Bound automatically inside `streaming-response` bodies in the Jetty async path.+ Used by [[abort-connection!]] to tear down the underlying connection when an error+ occurs after the response has already been committed."+ nil)+(def ^:dynamic *completed?*"An `AtomicBoolean` that is set to `true` when the async context has been completed,either by the worker thread or by Jetty's timeout/error callbacks. When `true`, the@@ -126,8 +128,54 @@(dissoc :export-format)(cond-> (server.settings/hide-stacktraces) (dissoc :stacktrace :trace :via))))+(defn- abort-connection!+ "Abort the underlying Jetty connection for an *already-committed* streaming response, so it+ terminates without a clean chunked/gzip terminator. The client's `fetch`/`blob()` then+ rejects with a network error instead of silently accepting a truncated body that has a+ JSON error blob appended to it. Marks the async context completed so the worker thread's+ normal `.complete` call is skipped — the abort tears the connection down itself."+ []+ (when (and *request* *completed?*+ (.compareAndSet ^AtomicBoolean *completed?* false true))+ (try+ (let [^HttpChannel channel (.getHttpChannel ^Request *request*)]+ (.abort channel (EofException. "Aborting streaming response after a mid-stream error")))+ (catch Throwable e+ ;; The abort failed, so it won't tear the connection down. Revert the flag we+ ;; optimistically set so the worker thread's `finally` in `do-f-async` can still+ ;; `.complete` the async context, instead of leaving the request to hang until+ ;; Jetty's async timeout fires.+ (.set ^AtomicBoolean *completed?* false)+ (log/error e "Error aborting streaming connection after mid-stream error")))))++(defn- write-error-to-stream!+ "Serialize `obj` as a JSON error body onto `os` and close the stream. Used when there is no+ committed HTTP response to abort — either an uncommitted error response, or a plain output+ stream with no Ring response bound (e.g. [[metabase.query-processor.streaming/do-with-streaming-rff]]+ used directly)."+ [^OutputStream os obj export-format]+ (with-open [os os]+ (log/trace (u/pprint-to-str (list 'write-error! obj)))+ (try+ (let [obj (sanitize-error-obj obj export-format)]+ (with-open [writer (BufferedWriter. (OutputStreamWriter. os StandardCharsets/UTF_8))]+ (json/encode-to obj writer {})))+ (catch EofException _)+ (catch Throwable e+ (log/error e "Error writing error to output stream" obj)))))+(defn write-error!- "Write an error to the output stream, formatting it nicely. Closes output stream afterwards.+ "Handle an error that occurred while producing a streaming response.++ - When the HTTP response is *not yet committed*, send a normal error response: set the+ status code and `application/json` content type, then write the error as a JSON body.+ - When the response is *already committed* (the error happened mid-stream, after the status+ line and some body bytes are on the wire), abort the connection via [[abort-connection!]].+ We can no longer signal failure through the status code or a parseable trailer, so a clean+ close with an appended error blob would be silently accepted by the client as a successful+ (but corrupt) download. Aborting makes the client's `fetch`/`blob()` reject instead.+ - When no HTTP response is bound (a plain output stream), just write the JSON error body.+No-op if the async context has already been completed (response and stream may be recycled)."([os obj export-format](write-error! os obj export-format nil))@@ -142,20 +190,19 @@(instance? Throwable obj)(recur os (format-exception obj) export-format status-code)+ (nil? *response*)+ (write-error-to-stream! os obj export-format)++ (committed?)+ (do+ (log/trace "Streaming response already committed; aborting connection to signal mid-stream error")+ (abort-connection!))+:else(do- (when (and *response* (not (committed?)))- (set-status! (or status-code 500))- (set-content-type! "application/json"))- (with-open [os os]- (log/trace (u/pprint-to-str (list 'write-error! obj)))- (try- (let [obj (sanitize-error-obj obj export-format)]- (with-open [writer (BufferedWriter. (OutputStreamWriter. os StandardCharsets/UTF_8))]- (json/encode-to obj writer {})))- (catch EofException _)- (catch Throwable e- (log/error e "Error writing error to output stream" obj))))))))+ (set-status! (or status-code 500))+ (set-content-type! "application/json")+ (write-error-to-stream! os obj export-format)))))(defn- do-f* [f ^OutputStream os _finished-chan canceled-chan](try@@ -186,10 +233,11 @@"Runs `f` asynchronously on the streaming response `thread-pool`, returning immediately. When `f` finishes,completes (i.e., closes) Jetty `async-context`. `completed?` is an `AtomicBoolean` used to coordinate withJetty's timeout/error callbacks so that only one path calls `.complete`."- [^AsyncContext async-context response f ^OutputStream os finished-chan canceled-chan ^AtomicBoolean completed?]+ [^AsyncContext async-context response ^Request request f ^OutputStream os finished-chan canceled-chan ^AtomicBoolean completed?]{:pre [(some? os)]}(let [task (^:once fn* [](binding [*response* response+ *request* request*completed?* completed?](try(do-f* f os finished-chan canceled-chan)@@ -350,13 +398,22 @@;; by the client because of `start-async-cancel-loop!`. The latter tries to read a;; byte from the input stream at some interval, and that may/will cause corruption;; of the subsequent requests that come through the reused connection (see #46071).- "Connection" "close")+ "Connection" "close"+ ;; Force chunked transfer encoding so the body has a positive terminator (the+ ;; final zero-length chunk). Without it, `Connection: close` makes Jetty delimit+ ;; the body by connection close, which is indistinguishable from a truncated+ ;; stream — so a mid-stream failure that aborts the connection (see+ ;; [[abort-connection!]]) would be silently accepted by the client as a complete+ ;; body. With chunked framing the client gets a protocol error instead.+ ;; (HTTP/2 has no chunked encoding, but an aborted stream becomes an RST_STREAM,+ ;; which the client surfaces as an error just the same.)+ "Transfer-Encoding" "chunked")gzip? (assoc "Content-Encoding" "gzip"))](#'servlet/set-headers response headers)(let [output-stream-delay (output-stream-delay gzip? response)delay-os (delay-output-stream output-stream-delay)](start-async-cancel-loop! request finished-chan canceled-chan)- (do-f-async async-context response f delay-os finished-chan canceled-chan completed?)))+ (do-f-async async-context response request f delay-os finished-chan canceled-chan completed?)))(catch Throwable e(log/error e "Unexpected exception in do-f-async")(trydiff --git a/test/metabase/metabot/api_test.clj b/test/metabase/metabot/api_test.cljindex 7b5703a3444..8d810824c1d 100644--- a/test/metabase/metabot/api_test.clj+++ b/test/metabase/metabot/api_test.clj@@ -182,16 +182,21 @@(mt/with-model-cleanup [:model/MetabotMessage[:model/MetabotConversation :created_at]](let [conversation-id (str (random-uuid))- body (mt/user-real-request :rasta :post 202 "metabot/agent-streaming"- {:request-options {:as :stream- :decompress-body false}}- {:message "Test closure"- :context {}- :conversation_id conversation-id- :history []- :state {}})]- (.read ^java.io.InputStream body) ;; start the handler- (.close ^java.io.Closeable body)+ response (mt/user-real-request-full-response+ :rasta :post 202 "metabot/agent-streaming"+ {:request-options {:as :stream+ :decompress-body false}}+ {:message "Test closure"+ :context {}+ :conversation_id conversation-id+ :history []+ :state {}})]+ (.read ^java.io.InputStream (:body response)) ;; start the handler+ ;; Close the underlying client, not the body stream: closing the body would+ ;; make clj-http drain the (now chunked) response to completion, which looks+ ;; like a normal finish rather than a disconnect. Closing the client aborts+ ;; the connection, which is what the server's cancel loop detects.+ (.close ^java.io.Closeable (:http-client response))(u/poll {:thunk #(deref stored-parts):done? some?:interval-ms 10@@ -229,25 +234,28 @@(reset! stored-kwargs (apply hash-map kwargs)))](mt/with-model-cleanup [:model/MetabotMessage[:model/MetabotConversation :created_at]]- (mt/user-http-request :rasta :post 202 "metabot/agent-streaming"- {:message "go"- :context {}- :conversation_id (str (random-uuid))- :history []- :state {}})- (u/poll {:thunk #(deref stored-kwargs)- :done? some?- :interval-ms 10- :timeout-ms 3000})- (is (some? @stored-kwargs)- "finalize-assistant-turn! is called from the finally even when setup threw")- (is (true? (:finished? @stored-kwargs))- "a thrown turn is :finished? true — `finished=false` is reserved for client aborts")- (is (=? {:message #"(?i)agent setup exploded"- :type "clojure.lang.ExceptionInfo"- :data {:status 503 :provider :test}}- (:error @stored-kwargs))- "the throwable becomes a structured error payload"))))))))+ (let [response (mt/user-http-request :rasta :post 202 "metabot/agent-streaming"+ {:message "go"+ :context {}+ :conversation_id (str (random-uuid))+ :history []+ :state {}})]+ (u/poll {:thunk #(deref stored-kwargs)+ :done? some?+ :interval-ms 10+ :timeout-ms 3000})+ (is (some? @stored-kwargs)+ "finalize-assistant-turn! is called from the finally even when setup threw")+ (is (true? (:finished? @stored-kwargs))+ "a thrown turn is :finished? true — `finished=false` is reserved for client aborts")+ (is (=? {:message #"(?i)agent setup exploded"+ :type "clojure.lang.ExceptionInfo"+ :data {:status 503 :provider :test}}+ (:error @stored-kwargs))+ "the throwable becomes a structured error payload")+ (testing "the failure is streamed to the client as an AI SDK error part (3:...) rather than a silent close"+ (is (some #(str/starts-with? % "3:") (str/split-lines response)))+ (is (re-find #"(?i)agent setup exploded" response)))))))))))(deftest settings-get-returns-live-models-test(mt/with-temporary-setting-values [metabot.settings/llm-metabot-provider "anthropic/claude-haiku-4-5"diff --git a/test/metabase/server/streaming_response_test.clj b/test/metabase/server/streaming_response_test.cljindex 32af553eaaf..6bdc7cd6bea 100644--- a/test/metabase/server/streaming_response_test.clj+++ b/test/metabase/server/streaming_response_test.clj@@ -203,6 +203,35 @@(finally(.stop server)))))+(deftest abort-on-committed-error-test+ (testing "An error after the response is committed aborts the connection so the client cannot read a complete body"+ (let [handler (fn [req respond _raise]+ (respond+ (compojure.response/render+ (streaming-response/streaming-response {:content-type "text/csv"} [os _canceled-chan]+ ;; write + flush some bytes so the response commits, then fail mid-stream+ (.write os (.getBytes "a,b,c\n1,2,3\n" "UTF-8"))+ (.flush os)+ (streaming-response/write-error! os {:error "boom"} :csv 500))+ req)))+ server (doto (server.instance/create-server handler {:port 0 :join? false})+ .start)+ url (str "http://localhost:" (.. server getURI getPort))+ consume (fn [] (let [res (http/request {:method :get, :url url, :as :stream, :decompress-body false})]+ [res (slurp (:body res))]))]+ (try+ (testing "the response is chunked so a missing terminator is detectable"+ ;; can't read the body cleanly, so just open a request to inspect the headers+ (let [res (http/request {:method :get, :url url, :as :stream, :decompress-body false})]+ (is (= "chunked" (get-in res [:headers "transfer-encoding"])))+ (u/ignore-exceptions (.close ^InputStream (:body res)))))+ (testing "consuming the whole body throws because the stream was aborted without a clean chunk terminator"+ (is (thrown? Exception (consume))))+ (testing "no JSON error blob is appended to the body"+ (is (not (re-find #"boom" (try (second (consume)) (catch Exception _ ""))))))+ (finally+ (.stop server))))))+(def ^:private ^:dynamic *number-of-cans* nil)(deftest ^:parallel preserve-bindings-test@@ -249,7 +278,7 @@(is (= "application/json" @content-type-called))))))(deftest write-error-committed-response-test- (testing "write-error! should not set status or content type when response is committed"+ (testing "write-error! aborts instead of appending an error blob when the response is already committed"(let [os (ByteArrayOutputStream.)status-called (atom nil)content-type-called (atom nil)]@@ -262,7 +291,9 @@(testing "Status should not be set when response is committed"(is (nil? @status-called)))(testing "Content type should not be set when response is committed"- (is (nil? @content-type-called))))))+ (is (nil? @content-type-called)))+ (testing "No error blob should be appended to an already-committed stream"+ (is (zero? (.size os)))))))(deftest write-error-no-response-test(testing "write-error! should not attempt to set status when no *response* is bound"@@ -380,9 +411,7 @@(is (not (contains? error-response :via))"Response should not contain :via key")(is (= "test-value" (get-in error-response [:data :custom-data]))- "Response should include custom data from ex-info")- (is (contains? error-response :_status)- "Response should include :_status")))))))+ "Response should include custom data from ex-info")))))))(deftest write-error-nested-exception-with-stacktraces-disabled-test(testing "write-error! includes nested exception details when hide-stacktraces is false"@@ -532,6 +561,7 @@(complete [_](deliver complete-promise true)))nil+ nil(fn [_os _canceled-chan](deliver task-started? true)(try@@ -598,6 +628,7 @@(throw (NullPointerException. "recycled"))))(setContentType [_ _] (when (.get completed?)(throw (NullPointerException. "recycled")))))+ nil(fn [_os _canceled-chan](.countDown task-started)(.await task-can-finish 5 TimeUnit/SECONDS)diff --git a/test/metabase/test.clj b/test/metabase/test.cljindex 00e18ca8c94..3edb160afbc 100644--- a/test/metabase/test.clj+++ b/test/metabase/test.clj@@ -228,6 +228,7 @@user-http-requestuser-http-request-full-responseuser-real-request+ user-real-request-full-responsewith-groupwith-group-for-userwith-test-user]diff --git a/test/metabase/test/data/users.clj b/test/metabase/test/data/users.cljindex e2a18702f28..0499b9cb395 100644--- a/test/metabase/test/data/users.clj+++ b/test/metabase/test/data/users.clj@@ -242,6 +242,13 @@"Like [[user-http-request]] but instead of calling the app handler, this makes an actual http request."(partial user-request client/real-client))+(def ^{:arglists '([test-user-name-or-user-or-id method expected-status-code? endpoint+ request-options? http-body-map? & {:as query-params}])} user-real-request-full-response+ "Like [[user-real-request]] but returns the full HTTP response map instead of just the body. Useful for+ tests that need the underlying `:http-client` (e.g. to forcibly close the connection and simulate a client+ disconnect on a streaming response)."+ (partial user-request client/client-real-response))+(defn do-with-test-user [user-kwd thunk](t/testing (format "\nwith test user %s\n" user-kwd)(request/with-current-user (some-> user-kwd user->id)