Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(clickhouse): support cancellable streaming queries with safe retries (#19090)

Valery Meleshkind0f4283216bd
diff --git a/packages/shared/src/server/clickhouse/queryOutcome.ts b/packages/shared/src/server/clickhouse/queryOutcome.ts
index 0761aa02f2..c8212b1302 100644
--- a/packages/shared/src/server/clickhouse/queryOutcome.ts
+++ b/packages/shared/src/server/clickhouse/queryOutcome.ts
@@ -26,6 +26,7 @@ export const CLICKHOUSE_QUERY_ELAPSED_MS_METRIC =
export type ClickHouseQueryOutcome =
| "success"
+ | "cancelled"
| "timeout"
| "memory_limit"
| "overcommit"
diff --git a/packages/shared/src/server/repositories/clickhouse.test.ts b/packages/shared/src/server/repositories/clickhouse.test.ts
new file mode 100644
index 0000000000..c5b9084009
--- /dev/null
+++ b/packages/shared/src/server/repositories/clickhouse.test.ts
@@ -0,0 +1,208 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+ query: vi.fn(),
+ recordOutcome: vi.fn(),
+ recordElapsed: vi.fn(),
+}));
+
+vi.mock("../clickhouse/client", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("../clickhouse/client")>()),
+ clickhouseClient: () => ({ query: mocks.query }),
+}));
+
+vi.mock("../clickhouse/queryOutcome", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("../clickhouse/queryOutcome")>()),
+ recordClickHouseQueryOutcome: mocks.recordOutcome,
+ recordClickHouseQueryElapsed: mocks.recordElapsed,
+}));
+
+import {
+ ClickHouseQueryAbortedError,
+ isProgressRow,
+ queryClickhouseWithProgress,
+} from "./clickhouse";
+
+const summary = { elapsed_ns: "1000000" };
+
+function response(events: unknown[], afterStream?: () => void) {
+ return {
+ query_id: "query-id",
+ response_headers: { "x-clickhouse-summary": JSON.stringify(summary) },
+ stream: async function* () {
+ yield events.map((event) => ({ json: () => event }));
+ afterStream?.();
+ },
+ };
+}
+
+async function collect(
+ opts: Parameters<typeof queryClickhouseWithProgress>[0] = {
+ query: "SELECT 1",
+ },
+) {
+ const events = [];
+ for await (const event of queryClickhouseWithProgress(opts)) {
+ events.push(event);
+ }
+ return events;
+}
+
+beforeEach(() => vi.resetAllMocks());
+
+describe("queryClickhouseWithProgress", () => {
+ it("retries a request failure and records success once", async () => {
+ mocks.query
+ .mockRejectedValueOnce(new Error("socket hang up"))
+ .mockResolvedValueOnce(response([{ row: { id: "row-1" } }]));
+
+ expect(await collect()).toEqual([{ row: { id: "row-1" } }]);
+ expect(mocks.query).toHaveBeenCalledTimes(2);
+ expect(mocks.recordOutcome).toHaveBeenCalledOnce();
+ expect(mocks.recordOutcome.mock.calls[0][0]).toBe("success");
+ expect(mocks.recordElapsed).toHaveBeenCalledOnce();
+ expect(mocks.recordElapsed.mock.calls[0][0]).toEqual(summary);
+ });
+
+ it.each([
+ ["10", "13"],
+ ["0", "0"],
+ [undefined, undefined],
+ ])("preserves progress totals on retry: %s", async (total, retriedTotal) => {
+ const first = Object.freeze({
+ read_rows: "3",
+ read_bytes: "30",
+ elapsed_ns: "10",
+ total_rows_to_read: total,
+ });
+ const second = Object.freeze({
+ ...first,
+ read_rows: "8",
+ read_bytes: "80",
+ elapsed_ns: "20",
+ });
+ mocks.query
+ .mockResolvedValueOnce(
+ response([{ progress: first }], () => {
+ throw new Error("connection reset");
+ }),
+ )
+ .mockResolvedValueOnce(response([{ progress: second }]));
+
+ const events = await collect();
+
+ expect(mocks.query).toHaveBeenCalledTimes(2);
+ expect(events.filter(isProgressRow).map((event) => event.progress)).toEqual(
+ [
+ first,
+ {
+ ...second,
+ read_rows: "11",
+ read_bytes: "110",
+ elapsed_ns: "30",
+ total_rows_to_read: retriedTotal,
+ },
+ ],
+ );
+ });
+
+ it.each([1, 3])(
+ "honors abort with %i buffered rows, including at EOF",
+ async (count) => {
+ const controller = new AbortController();
+ mocks.query.mockResolvedValueOnce(
+ response(Array.from({ length: count }, (_, id) => ({ row: { id } }))),
+ );
+ const stream = queryClickhouseWithProgress({
+ query: "SELECT 1",
+ abortSignal: controller.signal,
+ });
+ expect((await stream.next()).value).toEqual({ row: { id: 0 } });
+ controller.abort();
+ await expect(stream.next()).rejects.toBeInstanceOf(
+ ClickHouseQueryAbortedError,
+ );
+ expect(mocks.query).toHaveBeenCalledOnce();
+ expect(mocks.recordOutcome).toHaveBeenCalledOnce();
+ expect(mocks.recordOutcome.mock.calls[0][0]).toBe("cancelled");
+ expect(mocks.recordElapsed).not.toHaveBeenCalled();
+ },
+ );
+
+ it.each([
+ [{ row: { id: "row-1" } }, "cancelled"],
+ [{ exception: "Code: 159. DB::Exception: Timeout exceeded" }, "timeout"],
+ ])("counts early consumer closure once: %j", async (event, outcome) => {
+ mocks.query.mockResolvedValueOnce(response([event]));
+ const stream = queryClickhouseWithProgress({ query: "SELECT 1" });
+ await stream.next();
+ await stream.return(undefined);
+ expect(mocks.recordOutcome).toHaveBeenCalledOnce();
+ expect(mocks.recordOutcome.mock.calls[0][0]).toBe(outcome);
+ expect(mocks.recordElapsed).not.toHaveBeenCalled();
+ });
+
+ it("does not retry after the first data row has been emitted", async () => {
+ mocks.query.mockResolvedValueOnce(
+ response([{ row: { id: "row-1" } }], () => {
+ throw new Error("socket hang up");
+ }),
+ );
+
+ await expect(collect()).rejects.toThrow("socket hang up");
+ expect(mocks.query).toHaveBeenCalledTimes(1);
+ });
+
+ it("does not retry an aborted request", async () => {
+ const abortController = new AbortController();
+ mocks.query.mockResolvedValueOnce(
+ response(
+ [{ progress: { read_rows: "3", read_bytes: "30", elapsed_ns: "1" } }],
+ () => {
+ abortController.abort();
+ throw new Error("socket hang up");
+ },
+ ),
+ );
+
+ await expect(
+ collect({ query: "SELECT 1", abortSignal: abortController.signal }),
+ ).rejects.toBeInstanceOf(ClickHouseQueryAbortedError);
+ expect(mocks.query).toHaveBeenCalledTimes(1);
+ });
+
+ it.each([false, true])(
+ "records an exception frame once when followed by abort: %s",
+ async (abortAfterException) => {
+ const controller = new AbortController();
+ mocks.query.mockResolvedValueOnce(
+ response(
+ [{ exception: "Code: 159. DB::Exception: Timeout exceeded" }],
+ () => {
+ if (abortAfterException) {
+ controller.abort();
+ throw new Error("socket hang up");
+ }
+ },
+ ),
+ );
+
+ const result = collect({
+ query: "SELECT 1",
+ abortSignal: controller.signal,
+ });
+ if (abortAfterException) {
+ await expect(result).rejects.toBeInstanceOf(
+ ClickHouseQueryAbortedError,
+ );
+ } else {
+ await result;
+ }
+
+ expect(mocks.query).toHaveBeenCalledOnce();
+ expect(mocks.recordOutcome).toHaveBeenCalledOnce();
+ expect(mocks.recordOutcome.mock.calls[0][0]).toBe("timeout");
+ expect(mocks.recordElapsed).not.toHaveBeenCalled();
+ },
+ );
+});
diff --git a/packages/shared/src/server/repositories/clickhouse.ts b/packages/shared/src/server/repositories/clickhouse.ts
index d1485a3d4f..34f55c613c 100644
--- a/packages/shared/src/server/repositories/clickhouse.ts
+++ b/packages/shared/src/server/repositories/clickhouse.ts
@@ -20,8 +20,12 @@ import { createEventUploadStorageService } from "../s3";
import { buildEventBucketPrefix } from "../ingestion/eventBucketPath";
import {
ClickHouseSettings,
+ type ProgressRow,
type RowOrProgress,
type DataFormat,
+ isException,
+ isProgressRow,
+ isRow,
} from "@clickhouse/client";
import { RESOURCE_LIMIT_ERROR_MESSAGE } from "../../errors/errorMessages";
import {
@@ -650,6 +654,7 @@ async function sendClickhouseQuery<F extends DataFormat>(opts: {
format: F;
span: Span;
queryId?: string;
+ abortSignal?: AbortSignal;
}) {
const normalizedTags = normalizeClickHouseQueryTags(opts.tags);
const res = await clickhouseClient(
@@ -661,6 +666,7 @@ async function sendClickhouseQuery<F extends DataFormat>(opts: {
query_params: opts.params,
use_multipart_params_auto: opts.useMultipartParamsAuto,
...(opts.queryId ? { query_id: opts.queryId } : {}),
+ ...(opts.abortSignal ? { abort_signal: opts.abortSignal } : {}),
clickhouse_settings: {
...opts.clickhouseSettings,
log_comment: JSON.stringify(normalizedTags),
@@ -694,6 +700,13 @@ function isRetryableError(error: unknown): boolean {
);
}
+export class ClickHouseQueryAbortedError extends Error {
+ constructor(cause?: unknown) {
+ super("ClickHouse query aborted", { cause });
+ this.name = "ClickHouseQueryAbortedError";
+ }
+}
+
export async function queryClickhouse<T>(
opts: ClickhouseQueryOpts,
): Promise<T[]> {
@@ -797,70 +810,295 @@ export async function queryClickhouse<T>(
}
export async function* queryClickhouseWithProgress<T>(
- opts: ClickhouseQueryOpts,
+ opts: ClickhouseQueryOpts & { abortSignal?: AbortSignal },
): AsyncGenerator<RowOrProgress<T>> {
if (!opts.allowLegacyEventsRead) assertNoLegacyEventsRead(opts.query);
const normalizedTags = normalizeClickHouseQueryTags(opts.tags);
+ const table = clickHouseQueryTableLabel(opts.query);
+ const shape = clickHouseQueryShape(opts.query, table);
+ const clickhouseService = resolveClickhouseService(
+ opts.preferredClickhouseService,
+ );
+ const progressContext = {
+ normalizedTags,
+ table,
+ shape,
+ clickhouseService,
+ };
const tracer = getTracer("clickhouse-query-progress");
const span = tracer.startSpan("clickhouse-query-progress", {
kind: SpanKind.CLIENT,
});
- // Client-generated so failures before/without a response still carry a
- // query_id on errors and spans; system.query_log stays pollable by id.
- const queryId = randomUUID();
-
try {
setSpanQueryAttributes(span, opts.query);
- span.setAttribute("ch.queryId", queryId);
+ yield* runClickHouseProgressAttempts<T>(opts, span, progressContext);
+ } finally {
+ span.end();
+ }
+}
- const res = await context
- .with(trace.setSpan(context.active(), span), () =>
- sendClickhouseQuery({
- ...opts,
- clickhouseSettings: {
- asterisk_include_alias_columns: 1,
- asterisk_include_materialized_columns: 1,
- ...opts.clickhouseSettings,
- },
- format: "JSONEachRowWithProgress",
+async function* runClickHouseProgressAttempts<T>(
+ opts: ClickhouseQueryOpts & { abortSignal?: AbortSignal },
+ span: Span,
+ progressContext: ClickHouseProgressQueryContext,
+): AsyncGenerator<RowOrProgress<T>> {
+ let emittedDataRow = false;
+ let completed = { read_rows: 0, read_bytes: 0, elapsed_ns: 0 };
+ let outcomeRecorded = false;
+ const recordError = (error: unknown, queryId?: string) => {
+ outcomeRecorded = true;
+ return recordClickHouseProgressError(error, queryId, progressContext);
+ };
+ const maxAttempts = Math.max(1, env.LANGFUSE_CLICKHOUSE_QUERY_MAX_ATTEMPTS);
+
+ try {
+ for (let attemptNumber = 1; attemptNumber <= maxAttempts; attemptNumber++) {
+ // Client-generated so failures before/without a response still carry a
+ // query_id on errors and spans; system.query_log stays pollable by id.
+ const queryId = randomUUID();
+ const progress = createClickHouseProgressAccumulator(completed);
+ try {
+ const { response, summary } = await startClickHouseProgressAttempt(
+ opts,
span,
+ progressContext.normalizedTags,
queryId,
- }),
- )
- .catch((error) => {
- throw ClickHouseResourceError.wrapIfResourceError(
- error as Error,
- normalizedTags,
);
- });
-
- for await (const rows of res.stream()) {
- for (const row of rows) {
- yield row.json() as RowOrProgress<T>;
- }
- }
- } catch (error) {
- if (error instanceof ClickHouseResourceError) {
- const enriched = enrichWithQueryId(error, queryId);
- throw enriched === error
- ? error
- : new ClickHouseResourceError(
- error.errorType,
- enriched,
- normalizedTags,
+ for await (const event of streamClickHouseProgressResponse<T>(
+ response,
+ )) {
+ opts.abortSignal?.throwIfAborted();
+ if (isException(event))
+ recordError(new Error(event.exception), queryId);
+ if (isRow(event)) emittedDataRow = true;
+ yield isProgressRow(event) ? progress.add(event) : event;
+ }
+ opts.abortSignal?.throwIfAborted();
+ if (!outcomeRecorded) {
+ outcomeRecorded = true;
+ recordClickHouseProgressSuccess(summary, progressContext);
+ }
+ return;
+ } catch (error) {
+ if (opts.abortSignal?.aborted) {
+ const aborted = new ClickHouseQueryAbortedError(error);
+ if (!outcomeRecorded) recordError(aborted, queryId);
+ throw aborted;
+ }
+ if (outcomeRecorded) throw error;
+ if (
+ shouldRetryClickHouseProgress(
+ error,
+ emittedDataRow,
+ attemptNumber,
+ maxAttempts,
+ )
+ ) {
+ completed = progress.totals;
+ await retryClickHouseProgress(
+ error,
+ attemptNumber,
+ maxAttempts,
+ progressContext.normalizedTags,
+ span,
);
+ continue;
+ }
+ throw recordError(error, queryId);
+ }
}
- throw ClickHouseResourceError.wrapIfResourceError(
- enrichWithQueryId(error as Error, queryId),
- normalizedTags,
- );
} finally {
- span.end();
+ /** Consumer return()/break closes the stream without reaching EOF or catch. */
+ if (!outcomeRecorded) recordError(new ClickHouseQueryAbortedError());
+ }
+}
+
+async function startClickHouseProgressAttempt(
+ opts: ClickhouseQueryOpts & { abortSignal?: AbortSignal },
+ span: Span,
+ normalizedTags: NormalizedClickHouseQueryTags,
+ queryId: string,
+): Promise<{
+ response: ClickHouseProgressResponse;
+ summary: Record<string, string> | undefined;
+}> {
+ opts.abortSignal?.throwIfAborted();
+ span.setAttribute("ch.queryId", queryId);
+ const response = await context
+ .with(trace.setSpan(context.active(), span), () =>
+ sendClickhouseQuery({
+ ...opts,
+ clickhouseSettings: {
+ asterisk_include_alias_columns: 1,
+ asterisk_include_materialized_columns: 1,
+ ...opts.clickhouseSettings,
+ },
+ format: "JSONEachRowWithProgress",
+ span,
+ queryId,
+ }),
+ )
+ .catch((error) => {
+ throw ClickHouseResourceError.wrapIfResourceError(
+ error as Error,
+ normalizedTags,
+ );
+ });
+
+ return {
+ response,
+ summary: parseClickHouseSummary(response.response_headers),
+ };
+}
+
+async function* streamClickHouseProgressResponse<T>(
+ response: ClickHouseProgressResponse,
+): AsyncGenerator<RowOrProgress<T>> {
+ for await (const rows of response.stream()) {
+ for (const row of rows) yield row.json() as RowOrProgress<T>;
}
}
+/** Own cumulative maxima; completed work is counted once across retries. */
+function createClickHouseProgressAccumulator(
+ completed: ClickHouseProgressCounters,
+) {
+ const totals = { ...completed };
+ return {
+ totals,
+ add: ({ progress }: ProgressRow): ProgressRow => {
+ const result = { ...progress };
+ for (const key of ["read_rows", "read_bytes", "elapsed_ns"] as const) {
+ const value = Number(progress[key]);
+ if (Number.isFinite(value)) {
+ totals[key] = Math.max(totals[key], completed[key] + value);
+ }
+ result[key] = String(totals[key]);
+ }
+ const estimate = Number(progress.total_rows_to_read);
+ if (estimate > 0) {
+ result.total_rows_to_read = String(
+ Math.max(
+ totals.read_rows,
+ completed.read_rows + (Number.isFinite(estimate) ? estimate : 0),
+ ),
+ );
+ }
+ return { progress: result };
+ },
+ };
+}
+
+function recordClickHouseProgressSuccess(
+ summary: Record<string, string> | undefined,
+ context: ClickHouseProgressQueryContext,
+): void {
+ recordClickHouseQueryOutcome(
+ "success",
+ context.normalizedTags,
+ context.table,
+ context.shape,
+ context.clickhouseService,
+ );
+ if (summary) {
+ recordClickHouseQueryElapsed(
+ summary,
+ context.normalizedTags,
+ context.table,
+ context.shape,
+ context.clickhouseService,
+ );
+ }
+}
+
+function shouldRetryClickHouseProgress(
+ error: unknown,
+ emittedDataRow: boolean,
+ attemptNumber: number,
+ maxAttempts: number,
+): boolean {
+ return (
+ !emittedDataRow &&
+ attemptNumber < maxAttempts &&
+ !(error instanceof ClickHouseQueryAbortedError) &&
+ isRetryableError(error)
+ );
+}
+
+async function retryClickHouseProgress(
+ error: unknown,
+ attemptNumber: number,
+ maxAttempts: number,
+ normalizedTags: NormalizedClickHouseQueryTags,
+ span: Span,
+): Promise<void> {
+ const message = error instanceof Error ? error.message : String(error);
+ logger.warn(
+ `ClickHouse query failed with retryable error (attempt ${attemptNumber}/${maxAttempts}): ${message}`,
+ {
+ error: message,
+ attemptNumber,
+ tags: normalizedTags,
+ },
+ );
+ span.addEvent("clickhouse-query-retry", {
+ "retry.attempt": attemptNumber,
+ "retry.error": message,
+ });
+ await new Promise((resolve) => setTimeout(resolve, 100));
+}
+
+function recordClickHouseProgressError(
+ error: unknown,
+ queryId: string | undefined,
+ context: ClickHouseProgressQueryContext,
+): Error {
+ const originalError = error as Error;
+ const wrapped =
+ originalError instanceof ClickHouseResourceError
+ ? new ClickHouseResourceError(
+ originalError.errorType,
+ enrichWithQueryId(originalError, queryId),
+ context.normalizedTags,
+ )
+ : ClickHouseResourceError.wrapIfResourceError(
+ enrichWithQueryId(originalError, queryId),
+ context.normalizedTags,
+ );
+ const fallbackOutcome =
+ error instanceof ClickHouseQueryAbortedError ? "cancelled" : "error";
+ recordClickHouseQueryOutcome(
+ wrapped instanceof ClickHouseResourceError
+ ? CLICKHOUSE_RESOURCE_ERROR_OUTCOMES[wrapped.errorType]
+ : fallbackOutcome,
+ context.normalizedTags,
+ context.table,
+ context.shape,
+ context.clickhouseService,
+ error,
+ );
+ return wrapped;
+}
+
+type ClickHouseProgressCounters = {
+ read_rows: number;
+ read_bytes: number;
+ elapsed_ns: number;
+};
+
+type ClickHouseProgressResponse = {
+ stream: () => AsyncIterable<ReadonlyArray<{ json: () => unknown }>>;
+};
+
+type ClickHouseProgressQueryContext = {
+ normalizedTags: NormalizedClickHouseQueryTags;
+ table: ReturnType<typeof clickHouseQueryTableLabel>;
+ shape: ReturnType<typeof clickHouseQueryShape>;
+ clickhouseService: ReturnType<typeof resolveClickhouseService>;
+};
+
export async function commandClickhouse(opts: {
query: string;
params?: Record<string, unknown> | undefined;
diff --git a/web/src/__tests__/server/repositories/clickhouse-resource-errors.servertest.ts b/web/src/__tests__/server/repositories/clickhouse-resource-errors.servertest.ts
index c71b0ef072..38195642c4 100644
--- a/web/src/__tests__/server/repositories/clickhouse-resource-errors.servertest.ts
+++ b/web/src/__tests__/server/repositories/clickhouse-resource-errors.servertest.ts
@@ -5,6 +5,9 @@ import {
queryClickhouseExecRaw,
queryClickhouseWithProgress,
ClickHouseResourceError,
+ ClickHouseQueryAbortedError,
+ isRow,
+ isProgressRow,
} from "@langfuse/shared/src/server";
import { fail } from "assert";
import { randomUUID } from "node:crypto";
@@ -203,6 +206,43 @@ describe("ClickHouse Resource Error Handling", () => {
});
describe("queryClickhouseWithProgress", () => {
+ it("streams the same rows and reports scan counters", async () => {
+ const query = "SELECT toUInt32(number) AS n FROM numbers(5)";
+ const rows: { n: number }[] = [];
+ let readRows = 0;
+ let readBytes = 0;
+ for await (const item of queryClickhouseWithProgress<{ n: number }>({
+ query,
+ clickhouseSettings: { interactive_delay: "0", max_block_size: "1" },
+ })) {
+ if (isRow(item)) rows.push(item.row);
+ if (isProgressRow(item)) {
+ readRows = Number(item.progress.read_rows);
+ readBytes = Number(item.progress.read_bytes);
+ }
+ }
+ expect(rows).toEqual(await queryClickhouse({ query }));
+ expect(readRows).toBeGreaterThanOrEqual(5);
+ expect(readBytes).toBeGreaterThan(0);
+ });
+
+ it.each(["before", "during"])(
+ "returns a typed cancellation error when aborted %s a query",
+ async (when) => {
+ const stream = queryClickhouseWithProgress({
+ query: "SELECT sleepEachRow(0.02) FROM numbers(100)",
+ abortSignal:
+ when === "before" ? AbortSignal.abort() : AbortSignal.timeout(50),
+ clickhouseSettings: {
+ cancel_http_readonly_queries_on_client_close: 1,
+ },
+ });
+ await expect(Array.fromAsync(stream)).rejects.toBeInstanceOf(
+ ClickHouseQueryAbortedError,
+ );
+ },
+ );
+
it("should include query_id exactly once in errors thrown before streaming starts", async () => {
const generator = queryClickhouseWithProgress({
query: `SELECT * FROM non_existent_table_xyz123`,