Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(clickhouse): count only caller aborts as client_closed (#19626)

Niklas Semmlerc0486847783a
diff --git a/packages/shared/src/server/clickhouse/queryOutcome.ts b/packages/shared/src/server/clickhouse/queryOutcome.ts
index 89e8b5b497..673d66c20b 100644
--- a/packages/shared/src/server/clickhouse/queryOutcome.ts
+++ b/packages/shared/src/server/clickhouse/queryOutcome.ts
@@ -25,8 +25,9 @@ export const CLICKHOUSE_QUERY_ELAPSED_MS_METRIC =
"langfuse.clickhouse.query.elapsed_ms";
/**
- * `client_closed` is not a ClickHouse failure: the caller aborted or stopped
- * reading a streamed result, e.g. a browser closing a dashboard stream.
+ * `client_closed` is not a ClickHouse failure: the caller aborted its signal for
+ * a reason other than a deadline, e.g. a browser closing a dashboard stream. A
+ * deadline abort is an `error` with `error_code:client_timeout`.
*/
export type ClickHouseQueryOutcome =
| "success"
diff --git a/packages/shared/src/server/repositories/clickhouse.test.ts b/packages/shared/src/server/repositories/clickhouse.test.ts
index c0495dfb39..5642fd918c 100644
--- a/packages/shared/src/server/repositories/clickhouse.test.ts
+++ b/packages/shared/src/server/repositories/clickhouse.test.ts
@@ -17,6 +17,7 @@ vi.mock("../clickhouse/queryOutcome", async (importOriginal) => ({
recordClickHouseQueryElapsed: mocks.recordElapsed,
}));
+import { clickHouseQueryErrorCode } from "../clickhouse/queryOutcome";
import {
ClickHouseQueryAbortedError,
isProgressRow,
@@ -130,9 +131,9 @@ describe("queryClickhouseWithProgress", () => {
);
it.each([
- [{ row: { id: "row-1" } }, "client_closed"],
+ [{ row: { id: "row-1" } }, "error"],
[{ exception: "Code: 159. DB::Exception: Timeout exceeded" }, "timeout"],
- ])("counts early consumer closure once: %j", async (event, outcome) => {
+ ])("counts closure without an abort once: %j", async (event, outcome) => {
mocks.query.mockResolvedValueOnce(response([event]));
const stream = queryClickhouseWithProgress({ query: "SELECT 1" });
await stream.next();
@@ -142,6 +143,66 @@ describe("queryClickhouseWithProgress", () => {
expect(mocks.recordElapsed).not.toHaveBeenCalled();
});
+ it("counts an abort followed by an early return as client_closed", async () => {
+ const controller = new AbortController();
+ mocks.query.mockResolvedValueOnce(
+ response([{ row: { id: 0 } }, { row: { id: 1 } }]),
+ );
+ const stream = queryClickhouseWithProgress({
+ query: "SELECT 1",
+ abortSignal: controller.signal,
+ });
+ await stream.next();
+ controller.abort();
+ await stream.return(undefined);
+ expect(mocks.recordOutcome).toHaveBeenCalledOnce();
+ expect(mocks.recordOutcome.mock.calls[0][0]).toBe("client_closed");
+ });
+
+ it("counts an abort with a timeout reason as a client timeout", async () => {
+ const controller = new AbortController();
+ mocks.query.mockResolvedValueOnce(
+ response([{ row: { id: 0 } }, { row: { id: 1 } }]),
+ );
+ const stream = queryClickhouseWithProgress({
+ query: "SELECT 1",
+ abortSignal: controller.signal,
+ });
+ await stream.next();
+ controller.abort(
+ new DOMException(
+ "The operation was aborted due to timeout",
+ "TimeoutError",
+ ),
+ );
+ await expect(stream.next()).rejects.toBeInstanceOf(
+ ClickHouseQueryAbortedError,
+ );
+ const [outcome, , , , , error] = mocks.recordOutcome.mock.calls[0];
+ expect(outcome).toBe("error");
+ expect(clickHouseQueryErrorCode(error)).toBe("client_timeout");
+ });
+
+ it("keeps a resource error that arrives together with an abort", async () => {
+ const controller = new AbortController();
+ mocks.query.mockResolvedValueOnce(
+ response([{ row: { id: 0 } }], () => {
+ controller.abort();
+ throw new Error("Code: 159. DB::Exception: Timeout exceeded");
+ }),
+ );
+ const stream = queryClickhouseWithProgress({
+ query: "SELECT 1",
+ abortSignal: controller.signal,
+ });
+ await stream.next();
+ await expect(stream.next()).rejects.toBeInstanceOf(
+ ClickHouseQueryAbortedError,
+ );
+ expect(mocks.recordOutcome).toHaveBeenCalledOnce();
+ expect(mocks.recordOutcome.mock.calls[0][0]).toBe("timeout");
+ });
+
it("does not retry after the first data row has been emitted", async () => {
mocks.query.mockResolvedValueOnce(
response([{ row: { id: "row-1" } }], () => {
diff --git a/packages/shared/src/server/repositories/clickhouse.ts b/packages/shared/src/server/repositories/clickhouse.ts
index 3bcf2dec47..53b2646d49 100644
--- a/packages/shared/src/server/repositories/clickhouse.ts
+++ b/packages/shared/src/server/repositories/clickhouse.ts
@@ -700,11 +700,26 @@ function isRetryableError(error: unknown): boolean {
);
}
+/** `cause` is the abort signal's reason. */
export class ClickHouseQueryAbortedError extends Error {
constructor(cause?: unknown) {
super("ClickHouse query aborted", { cause });
this.name = "ClickHouseQueryAbortedError";
}
+
+ /** Aborted by a deadline such as `AbortSignal.timeout()`, not by a client leaving. */
+ get isClientTimeout(): boolean {
+ return this.cause instanceof Error && this.cause.name === "TimeoutError";
+ }
+}
+
+function isResourceError(error: unknown): boolean {
+ return (
+ error instanceof ClickHouseResourceError ||
+ (error instanceof Error &&
+ ClickHouseResourceError.wrapIfResourceError(error) instanceof
+ ClickHouseResourceError)
+ );
}
export async function queryClickhouse<T>(
@@ -883,8 +898,14 @@ async function* runClickHouseProgressAttempts<T>(
return;
} catch (error) {
if (opts.abortSignal?.aborted) {
- const aborted = new ClickHouseQueryAbortedError(error);
- if (!outcomeRecorded) recordError(aborted, queryId);
+ const aborted = new ClickHouseQueryAbortedError(
+ opts.abortSignal.reason,
+ );
+ // A deadline reason reads like a CH timeout, so only a different error is checked.
+ const resourceError =
+ error !== opts.abortSignal.reason && isResourceError(error);
+ if (!outcomeRecorded)
+ recordError(resourceError ? error : aborted, queryId);
throw aborted;
}
if (outcomeRecorded) throw error;
@@ -910,8 +931,19 @@ async function* runClickHouseProgressAttempts<T>(
}
}
} finally {
- /** Consumer return()/break closes the stream without reaching EOF or catch. */
- if (!outcomeRecorded) recordError(new ClickHouseQueryAbortedError());
+ /**
+ * Consumer return()/break, or a throw in its loop body, closes the stream
+ * without reaching EOF or catch. A consumer that stops on purpose aborts
+ * its signal first; without an abort this path counts as an error.
+ */
+ if (!outcomeRecorded)
+ recordError(
+ opts.abortSignal?.aborted
+ ? new ClickHouseQueryAbortedError(opts.abortSignal.reason)
+ : new Error(
+ "ClickHouse stream closed by its consumer without an abort",
+ ),
+ );
}
}
@@ -1068,7 +1100,9 @@ function recordClickHouseProgressError(
context.normalizedTags,
);
const fallbackOutcome =
- error instanceof ClickHouseQueryAbortedError ? "client_closed" : "error";
+ error instanceof ClickHouseQueryAbortedError && !error.isClientTimeout
+ ? "client_closed"
+ : "error";
recordClickHouseQueryOutcome(
wrapped instanceof ClickHouseResourceError
? CLICKHOUSE_RESOURCE_ERROR_OUTCOMES[wrapped.errorType]
diff --git a/web/src/pages/api/dashboard/execute-query-stream.ts b/web/src/pages/api/dashboard/execute-query-stream.ts
index 664b2160b0..10c7b6f245 100644
--- a/web/src/pages/api/dashboard/execute-query-stream.ts
+++ b/web/src/pages/api/dashboard/execute-query-stream.ts
@@ -130,19 +130,17 @@ export default async function handler(
res.flushHeaders();
// A closed client should also stop the ClickHouse query, not just this
- // response: the `break` below tears down the CH stream (its socket closes
- // within one progress event), and cancel_http_readonly_queries_on_client_close
+ // response: aborting the signal tears down the CH request (and counts the
+ // query as client_closed), and cancel_http_readonly_queries_on_client_close
// makes ClickHouse kill the query on that close instead of running the
// abandoned aggregation to completion. max_execution_time still bounds the
// stragglers either way.
- let aborted = false;
+ const clientClosed = new AbortController();
// `res`, not `req`: the request stream can end once the POST body is
// consumed, while the response lives for the whole stream — its close
// fires on a client disconnect mid-stream (and only after res.end() on
// the happy path, where the loop has already finished).
- res.on("close", () => {
- aborted = true;
- });
+ res.on("close", () => clientClosed.abort());
try {
const prepared = await prepareExecuteQuery({
@@ -157,6 +155,7 @@ export default async function handler(
Record<string, unknown>
>({
...chOpts,
+ abortSignal: clientClosed.signal,
tags: {
...chOpts.tags,
surface: "internalapi",
@@ -167,8 +166,6 @@ export default async function handler(
cancel_http_readonly_queries_on_client_close: 1,
},
})) {
- if (aborted) break;
-
if (isProgressRow(event)) {
res.write(
formatSSEEvent({ type: "progress", progress: event.progress }),
@@ -192,11 +189,11 @@ export default async function handler(
}
}
- if (!aborted) {
+ if (!clientClosed.signal.aborted) {
res.write(formatSSEEvent({ type: "done" }));
}
} catch (error) {
- if (!aborted) {
+ if (!clientClosed.signal.aborted) {
logger.error("[execute-query-stream] Query failed", {
error: error instanceof Error ? error.message : String(error),
projectId,