🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(clickhouse): count only caller aborts as client_closed (#19626)
Niklas Semmlerc0486847783a
diff --git a/packages/shared/src/server/clickhouse/queryOutcome.ts b/packages/shared/src/server/clickhouse/queryOutcome.tsindex 89e8b5b497..673d66c20b 100644--- a/packages/shared/src/server/clickhouse/queryOutcome.ts+++ b/packages/shared/src/server/clickhouse/queryOutcome.ts@@ -25,8 +25,9 @@ export const CLICKHOUSE_QUERY_ELAPSED_MS_METRIC ="langfuse.clickhouse.query.elapsed_ms";/**- * `client_closed` is not a ClickHouse failure: the caller aborted or stopped- * reading a streamed result, e.g. a browser closing a dashboard stream.+ * `client_closed` is not a ClickHouse failure: the caller aborted its signal for+ * a reason other than a deadline, e.g. a browser closing a dashboard stream. A+ * deadline abort is an `error` with `error_code:client_timeout`.*/export type ClickHouseQueryOutcome =| "success"diff --git a/packages/shared/src/server/repositories/clickhouse.test.ts b/packages/shared/src/server/repositories/clickhouse.test.tsindex c0495dfb39..5642fd918c 100644--- a/packages/shared/src/server/repositories/clickhouse.test.ts+++ b/packages/shared/src/server/repositories/clickhouse.test.ts@@ -17,6 +17,7 @@ vi.mock("../clickhouse/queryOutcome", async (importOriginal) => ({recordClickHouseQueryElapsed: mocks.recordElapsed,}));+import { clickHouseQueryErrorCode } from "../clickhouse/queryOutcome";import {ClickHouseQueryAbortedError,isProgressRow,@@ -130,9 +131,9 @@ describe("queryClickhouseWithProgress", () => {);it.each([- [{ row: { id: "row-1" } }, "client_closed"],+ [{ row: { id: "row-1" } }, "error"],[{ exception: "Code: 159. DB::Exception: Timeout exceeded" }, "timeout"],- ])("counts early consumer closure once: %j", async (event, outcome) => {+ ])("counts closure without an abort once: %j", async (event, outcome) => {mocks.query.mockResolvedValueOnce(response([event]));const stream = queryClickhouseWithProgress({ query: "SELECT 1" });await stream.next();@@ -142,6 +143,66 @@ describe("queryClickhouseWithProgress", () => {expect(mocks.recordElapsed).not.toHaveBeenCalled();});+ it("counts an abort followed by an early return as client_closed", async () => {+ const controller = new AbortController();+ mocks.query.mockResolvedValueOnce(+ response([{ row: { id: 0 } }, { row: { id: 1 } }]),+ );+ const stream = queryClickhouseWithProgress({+ query: "SELECT 1",+ abortSignal: controller.signal,+ });+ await stream.next();+ controller.abort();+ await stream.return(undefined);+ expect(mocks.recordOutcome).toHaveBeenCalledOnce();+ expect(mocks.recordOutcome.mock.calls[0][0]).toBe("client_closed");+ });++ it("counts an abort with a timeout reason as a client timeout", async () => {+ const controller = new AbortController();+ mocks.query.mockResolvedValueOnce(+ response([{ row: { id: 0 } }, { row: { id: 1 } }]),+ );+ const stream = queryClickhouseWithProgress({+ query: "SELECT 1",+ abortSignal: controller.signal,+ });+ await stream.next();+ controller.abort(+ new DOMException(+ "The operation was aborted due to timeout",+ "TimeoutError",+ ),+ );+ await expect(stream.next()).rejects.toBeInstanceOf(+ ClickHouseQueryAbortedError,+ );+ const [outcome, , , , , error] = mocks.recordOutcome.mock.calls[0];+ expect(outcome).toBe("error");+ expect(clickHouseQueryErrorCode(error)).toBe("client_timeout");+ });++ it("keeps a resource error that arrives together with an abort", async () => {+ const controller = new AbortController();+ mocks.query.mockResolvedValueOnce(+ response([{ row: { id: 0 } }], () => {+ controller.abort();+ throw new Error("Code: 159. DB::Exception: Timeout exceeded");+ }),+ );+ const stream = queryClickhouseWithProgress({+ query: "SELECT 1",+ abortSignal: controller.signal,+ });+ await stream.next();+ await expect(stream.next()).rejects.toBeInstanceOf(+ ClickHouseQueryAbortedError,+ );+ expect(mocks.recordOutcome).toHaveBeenCalledOnce();+ expect(mocks.recordOutcome.mock.calls[0][0]).toBe("timeout");+ });+it("does not retry after the first data row has been emitted", async () => {mocks.query.mockResolvedValueOnce(response([{ row: { id: "row-1" } }], () => {diff --git a/packages/shared/src/server/repositories/clickhouse.ts b/packages/shared/src/server/repositories/clickhouse.tsindex 3bcf2dec47..53b2646d49 100644--- a/packages/shared/src/server/repositories/clickhouse.ts+++ b/packages/shared/src/server/repositories/clickhouse.ts@@ -700,11 +700,26 @@ function isRetryableError(error: unknown): boolean {);}+/** `cause` is the abort signal's reason. */export class ClickHouseQueryAbortedError extends Error {constructor(cause?: unknown) {super("ClickHouse query aborted", { cause });this.name = "ClickHouseQueryAbortedError";}++ /** Aborted by a deadline such as `AbortSignal.timeout()`, not by a client leaving. */+ get isClientTimeout(): boolean {+ return this.cause instanceof Error && this.cause.name === "TimeoutError";+ }+}++function isResourceError(error: unknown): boolean {+ return (+ error instanceof ClickHouseResourceError ||+ (error instanceof Error &&+ ClickHouseResourceError.wrapIfResourceError(error) instanceof+ ClickHouseResourceError)+ );}export async function queryClickhouse<T>(@@ -883,8 +898,14 @@ async function* runClickHouseProgressAttempts<T>(return;} catch (error) {if (opts.abortSignal?.aborted) {- const aborted = new ClickHouseQueryAbortedError(error);- if (!outcomeRecorded) recordError(aborted, queryId);+ const aborted = new ClickHouseQueryAbortedError(+ opts.abortSignal.reason,+ );+ // A deadline reason reads like a CH timeout, so only a different error is checked.+ const resourceError =+ error !== opts.abortSignal.reason && isResourceError(error);+ if (!outcomeRecorded)+ recordError(resourceError ? error : aborted, queryId);throw aborted;}if (outcomeRecorded) throw error;@@ -910,8 +931,19 @@ async function* runClickHouseProgressAttempts<T>(}}} finally {- /** Consumer return()/break closes the stream without reaching EOF or catch. */- if (!outcomeRecorded) recordError(new ClickHouseQueryAbortedError());+ /**+ * Consumer return()/break, or a throw in its loop body, closes the stream+ * without reaching EOF or catch. A consumer that stops on purpose aborts+ * its signal first; without an abort this path counts as an error.+ */+ if (!outcomeRecorded)+ recordError(+ opts.abortSignal?.aborted+ ? new ClickHouseQueryAbortedError(opts.abortSignal.reason)+ : new Error(+ "ClickHouse stream closed by its consumer without an abort",+ ),+ );}}@@ -1068,7 +1100,9 @@ function recordClickHouseProgressError(context.normalizedTags,);const fallbackOutcome =- error instanceof ClickHouseQueryAbortedError ? "client_closed" : "error";+ error instanceof ClickHouseQueryAbortedError && !error.isClientTimeout+ ? "client_closed"+ : "error";recordClickHouseQueryOutcome(wrapped instanceof ClickHouseResourceError? CLICKHOUSE_RESOURCE_ERROR_OUTCOMES[wrapped.errorType]diff --git a/web/src/pages/api/dashboard/execute-query-stream.ts b/web/src/pages/api/dashboard/execute-query-stream.tsindex 664b2160b0..10c7b6f245 100644--- a/web/src/pages/api/dashboard/execute-query-stream.ts+++ b/web/src/pages/api/dashboard/execute-query-stream.ts@@ -130,19 +130,17 @@ export default async function handler(res.flushHeaders();// A closed client should also stop the ClickHouse query, not just this- // response: the `break` below tears down the CH stream (its socket closes- // within one progress event), and cancel_http_readonly_queries_on_client_close+ // response: aborting the signal tears down the CH request (and counts the+ // query as client_closed), and cancel_http_readonly_queries_on_client_close// makes ClickHouse kill the query on that close instead of running the// abandoned aggregation to completion. max_execution_time still bounds the// stragglers either way.- let aborted = false;+ const clientClosed = new AbortController();// `res`, not `req`: the request stream can end once the POST body is// consumed, while the response lives for the whole stream — its close// fires on a client disconnect mid-stream (and only after res.end() on// the happy path, where the loop has already finished).- res.on("close", () => {- aborted = true;- });+ res.on("close", () => clientClosed.abort());try {const prepared = await prepareExecuteQuery({@@ -157,6 +155,7 @@ export default async function handler(Record<string, unknown>>({...chOpts,+ abortSignal: clientClosed.signal,tags: {...chOpts.tags,surface: "internalapi",@@ -167,8 +166,6 @@ export default async function handler(cancel_http_readonly_queries_on_client_close: 1,},})) {- if (aborted) break;-if (isProgressRow(event)) {res.write(formatSSEEvent({ type: "progress", progress: event.progress }),@@ -192,11 +189,11 @@ export default async function handler(}}- if (!aborted) {+ if (!clientClosed.signal.aborted) {res.write(formatSSEEvent({ type: "done" }));}} catch (error) {- if (!aborted) {+ if (!clientClosed.signal.aborted) {logger.error("[execute-query-stream] Query failed", {error: error instanceof Error ? error.message : String(error),projectId,