🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(clickhouse): support cancellable streaming queries with safe retries (#19090)
Valery Meleshkind0f4283216bd
diff --git a/packages/shared/src/server/clickhouse/queryOutcome.ts b/packages/shared/src/server/clickhouse/queryOutcome.tsindex 0761aa02f2..c8212b1302 100644--- a/packages/shared/src/server/clickhouse/queryOutcome.ts+++ b/packages/shared/src/server/clickhouse/queryOutcome.ts@@ -26,6 +26,7 @@ export const CLICKHOUSE_QUERY_ELAPSED_MS_METRIC =export type ClickHouseQueryOutcome =| "success"+ | "cancelled"| "timeout"| "memory_limit"| "overcommit"diff --git a/packages/shared/src/server/repositories/clickhouse.test.ts b/packages/shared/src/server/repositories/clickhouse.test.tsnew file mode 100644index 0000000000..c5b9084009--- /dev/null+++ b/packages/shared/src/server/repositories/clickhouse.test.ts@@ -0,0 +1,208 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++const mocks = vi.hoisted(() => ({+ query: vi.fn(),+ recordOutcome: vi.fn(),+ recordElapsed: vi.fn(),+}));++vi.mock("../clickhouse/client", async (importOriginal) => ({+ ...(await importOriginal<typeof import("../clickhouse/client")>()),+ clickhouseClient: () => ({ query: mocks.query }),+}));++vi.mock("../clickhouse/queryOutcome", async (importOriginal) => ({+ ...(await importOriginal<typeof import("../clickhouse/queryOutcome")>()),+ recordClickHouseQueryOutcome: mocks.recordOutcome,+ recordClickHouseQueryElapsed: mocks.recordElapsed,+}));++import {+ ClickHouseQueryAbortedError,+ isProgressRow,+ queryClickhouseWithProgress,+} from "./clickhouse";++const summary = { elapsed_ns: "1000000" };++function response(events: unknown[], afterStream?: () => void) {+ return {+ query_id: "query-id",+ response_headers: { "x-clickhouse-summary": JSON.stringify(summary) },+ stream: async function* () {+ yield events.map((event) => ({ json: () => event }));+ afterStream?.();+ },+ };+}++async function collect(+ opts: Parameters<typeof queryClickhouseWithProgress>[0] = {+ query: "SELECT 1",+ },+) {+ const events = [];+ for await (const event of queryClickhouseWithProgress(opts)) {+ events.push(event);+ }+ return events;+}++beforeEach(() => vi.resetAllMocks());++describe("queryClickhouseWithProgress", () => {+ it("retries a request failure and records success once", async () => {+ mocks.query+ .mockRejectedValueOnce(new Error("socket hang up"))+ .mockResolvedValueOnce(response([{ row: { id: "row-1" } }]));++ expect(await collect()).toEqual([{ row: { id: "row-1" } }]);+ expect(mocks.query).toHaveBeenCalledTimes(2);+ expect(mocks.recordOutcome).toHaveBeenCalledOnce();+ expect(mocks.recordOutcome.mock.calls[0][0]).toBe("success");+ expect(mocks.recordElapsed).toHaveBeenCalledOnce();+ expect(mocks.recordElapsed.mock.calls[0][0]).toEqual(summary);+ });++ it.each([+ ["10", "13"],+ ["0", "0"],+ [undefined, undefined],+ ])("preserves progress totals on retry: %s", async (total, retriedTotal) => {+ const first = Object.freeze({+ read_rows: "3",+ read_bytes: "30",+ elapsed_ns: "10",+ total_rows_to_read: total,+ });+ const second = Object.freeze({+ ...first,+ read_rows: "8",+ read_bytes: "80",+ elapsed_ns: "20",+ });+ mocks.query+ .mockResolvedValueOnce(+ response([{ progress: first }], () => {+ throw new Error("connection reset");+ }),+ )+ .mockResolvedValueOnce(response([{ progress: second }]));++ const events = await collect();++ expect(mocks.query).toHaveBeenCalledTimes(2);+ expect(events.filter(isProgressRow).map((event) => event.progress)).toEqual(+ [+ first,+ {+ ...second,+ read_rows: "11",+ read_bytes: "110",+ elapsed_ns: "30",+ total_rows_to_read: retriedTotal,+ },+ ],+ );+ });++ it.each([1, 3])(+ "honors abort with %i buffered rows, including at EOF",+ async (count) => {+ const controller = new AbortController();+ mocks.query.mockResolvedValueOnce(+ response(Array.from({ length: count }, (_, id) => ({ row: { id } }))),+ );+ const stream = queryClickhouseWithProgress({+ query: "SELECT 1",+ abortSignal: controller.signal,+ });+ expect((await stream.next()).value).toEqual({ row: { id: 0 } });+ controller.abort();+ await expect(stream.next()).rejects.toBeInstanceOf(+ ClickHouseQueryAbortedError,+ );+ expect(mocks.query).toHaveBeenCalledOnce();+ expect(mocks.recordOutcome).toHaveBeenCalledOnce();+ expect(mocks.recordOutcome.mock.calls[0][0]).toBe("cancelled");+ expect(mocks.recordElapsed).not.toHaveBeenCalled();+ },+ );++ it.each([+ [{ row: { id: "row-1" } }, "cancelled"],+ [{ exception: "Code: 159. DB::Exception: Timeout exceeded" }, "timeout"],+ ])("counts early consumer closure once: %j", async (event, outcome) => {+ mocks.query.mockResolvedValueOnce(response([event]));+ const stream = queryClickhouseWithProgress({ query: "SELECT 1" });+ await stream.next();+ await stream.return(undefined);+ expect(mocks.recordOutcome).toHaveBeenCalledOnce();+ expect(mocks.recordOutcome.mock.calls[0][0]).toBe(outcome);+ expect(mocks.recordElapsed).not.toHaveBeenCalled();+ });++ it("does not retry after the first data row has been emitted", async () => {+ mocks.query.mockResolvedValueOnce(+ response([{ row: { id: "row-1" } }], () => {+ throw new Error("socket hang up");+ }),+ );++ await expect(collect()).rejects.toThrow("socket hang up");+ expect(mocks.query).toHaveBeenCalledTimes(1);+ });++ it("does not retry an aborted request", async () => {+ const abortController = new AbortController();+ mocks.query.mockResolvedValueOnce(+ response(+ [{ progress: { read_rows: "3", read_bytes: "30", elapsed_ns: "1" } }],+ () => {+ abortController.abort();+ throw new Error("socket hang up");+ },+ ),+ );++ await expect(+ collect({ query: "SELECT 1", abortSignal: abortController.signal }),+ ).rejects.toBeInstanceOf(ClickHouseQueryAbortedError);+ expect(mocks.query).toHaveBeenCalledTimes(1);+ });++ it.each([false, true])(+ "records an exception frame once when followed by abort: %s",+ async (abortAfterException) => {+ const controller = new AbortController();+ mocks.query.mockResolvedValueOnce(+ response(+ [{ exception: "Code: 159. DB::Exception: Timeout exceeded" }],+ () => {+ if (abortAfterException) {+ controller.abort();+ throw new Error("socket hang up");+ }+ },+ ),+ );++ const result = collect({+ query: "SELECT 1",+ abortSignal: controller.signal,+ });+ if (abortAfterException) {+ await expect(result).rejects.toBeInstanceOf(+ ClickHouseQueryAbortedError,+ );+ } else {+ await result;+ }++ expect(mocks.query).toHaveBeenCalledOnce();+ expect(mocks.recordOutcome).toHaveBeenCalledOnce();+ expect(mocks.recordOutcome.mock.calls[0][0]).toBe("timeout");+ expect(mocks.recordElapsed).not.toHaveBeenCalled();+ },+ );+});diff --git a/packages/shared/src/server/repositories/clickhouse.ts b/packages/shared/src/server/repositories/clickhouse.tsindex d1485a3d4f..34f55c613c 100644--- a/packages/shared/src/server/repositories/clickhouse.ts+++ b/packages/shared/src/server/repositories/clickhouse.ts@@ -20,8 +20,12 @@ import { createEventUploadStorageService } from "../s3";import { buildEventBucketPrefix } from "../ingestion/eventBucketPath";import {ClickHouseSettings,+ type ProgressRow,type RowOrProgress,type DataFormat,+ isException,+ isProgressRow,+ isRow,} from "@clickhouse/client";import { RESOURCE_LIMIT_ERROR_MESSAGE } from "../../errors/errorMessages";import {@@ -650,6 +654,7 @@ async function sendClickhouseQuery<F extends DataFormat>(opts: {format: F;span: Span;queryId?: string;+ abortSignal?: AbortSignal;}) {const normalizedTags = normalizeClickHouseQueryTags(opts.tags);const res = await clickhouseClient(@@ -661,6 +666,7 @@ async function sendClickhouseQuery<F extends DataFormat>(opts: {query_params: opts.params,use_multipart_params_auto: opts.useMultipartParamsAuto,...(opts.queryId ? { query_id: opts.queryId } : {}),+ ...(opts.abortSignal ? { abort_signal: opts.abortSignal } : {}),clickhouse_settings: {...opts.clickhouseSettings,log_comment: JSON.stringify(normalizedTags),@@ -694,6 +700,13 @@ function isRetryableError(error: unknown): boolean {);}+export class ClickHouseQueryAbortedError extends Error {+ constructor(cause?: unknown) {+ super("ClickHouse query aborted", { cause });+ this.name = "ClickHouseQueryAbortedError";+ }+}+export async function queryClickhouse<T>(opts: ClickhouseQueryOpts,): Promise<T[]> {@@ -797,70 +810,295 @@ export async function queryClickhouse<T>(}export async function* queryClickhouseWithProgress<T>(- opts: ClickhouseQueryOpts,+ opts: ClickhouseQueryOpts & { abortSignal?: AbortSignal },): AsyncGenerator<RowOrProgress<T>> {if (!opts.allowLegacyEventsRead) assertNoLegacyEventsRead(opts.query);const normalizedTags = normalizeClickHouseQueryTags(opts.tags);+ const table = clickHouseQueryTableLabel(opts.query);+ const shape = clickHouseQueryShape(opts.query, table);+ const clickhouseService = resolveClickhouseService(+ opts.preferredClickhouseService,+ );+ const progressContext = {+ normalizedTags,+ table,+ shape,+ clickhouseService,+ };const tracer = getTracer("clickhouse-query-progress");const span = tracer.startSpan("clickhouse-query-progress", {kind: SpanKind.CLIENT,});- // Client-generated so failures before/without a response still carry a- // query_id on errors and spans; system.query_log stays pollable by id.- const queryId = randomUUID();-try {setSpanQueryAttributes(span, opts.query);- span.setAttribute("ch.queryId", queryId);+ yield* runClickHouseProgressAttempts<T>(opts, span, progressContext);+ } finally {+ span.end();+ }+}- const res = await context- .with(trace.setSpan(context.active(), span), () =>- sendClickhouseQuery({- ...opts,- clickhouseSettings: {- asterisk_include_alias_columns: 1,- asterisk_include_materialized_columns: 1,- ...opts.clickhouseSettings,- },- format: "JSONEachRowWithProgress",+async function* runClickHouseProgressAttempts<T>(+ opts: ClickhouseQueryOpts & { abortSignal?: AbortSignal },+ span: Span,+ progressContext: ClickHouseProgressQueryContext,+): AsyncGenerator<RowOrProgress<T>> {+ let emittedDataRow = false;+ let completed = { read_rows: 0, read_bytes: 0, elapsed_ns: 0 };+ let outcomeRecorded = false;+ const recordError = (error: unknown, queryId?: string) => {+ outcomeRecorded = true;+ return recordClickHouseProgressError(error, queryId, progressContext);+ };+ const maxAttempts = Math.max(1, env.LANGFUSE_CLICKHOUSE_QUERY_MAX_ATTEMPTS);++ try {+ for (let attemptNumber = 1; attemptNumber <= maxAttempts; attemptNumber++) {+ // Client-generated so failures before/without a response still carry a+ // query_id on errors and spans; system.query_log stays pollable by id.+ const queryId = randomUUID();+ const progress = createClickHouseProgressAccumulator(completed);+ try {+ const { response, summary } = await startClickHouseProgressAttempt(+ opts,span,+ progressContext.normalizedTags,queryId,- }),- )- .catch((error) => {- throw ClickHouseResourceError.wrapIfResourceError(- error as Error,- normalizedTags,);- });-- for await (const rows of res.stream()) {- for (const row of rows) {- yield row.json() as RowOrProgress<T>;- }- }- } catch (error) {- if (error instanceof ClickHouseResourceError) {- const enriched = enrichWithQueryId(error, queryId);- throw enriched === error- ? error- : new ClickHouseResourceError(- error.errorType,- enriched,- normalizedTags,+ for await (const event of streamClickHouseProgressResponse<T>(+ response,+ )) {+ opts.abortSignal?.throwIfAborted();+ if (isException(event))+ recordError(new Error(event.exception), queryId);+ if (isRow(event)) emittedDataRow = true;+ yield isProgressRow(event) ? progress.add(event) : event;+ }+ opts.abortSignal?.throwIfAborted();+ if (!outcomeRecorded) {+ outcomeRecorded = true;+ recordClickHouseProgressSuccess(summary, progressContext);+ }+ return;+ } catch (error) {+ if (opts.abortSignal?.aborted) {+ const aborted = new ClickHouseQueryAbortedError(error);+ if (!outcomeRecorded) recordError(aborted, queryId);+ throw aborted;+ }+ if (outcomeRecorded) throw error;+ if (+ shouldRetryClickHouseProgress(+ error,+ emittedDataRow,+ attemptNumber,+ maxAttempts,+ )+ ) {+ completed = progress.totals;+ await retryClickHouseProgress(+ error,+ attemptNumber,+ maxAttempts,+ progressContext.normalizedTags,+ span,);+ continue;+ }+ throw recordError(error, queryId);+ }}- throw ClickHouseResourceError.wrapIfResourceError(- enrichWithQueryId(error as Error, queryId),- normalizedTags,- );} finally {- span.end();+ /** Consumer return()/break closes the stream without reaching EOF or catch. */+ if (!outcomeRecorded) recordError(new ClickHouseQueryAbortedError());+ }+}++async function startClickHouseProgressAttempt(+ opts: ClickhouseQueryOpts & { abortSignal?: AbortSignal },+ span: Span,+ normalizedTags: NormalizedClickHouseQueryTags,+ queryId: string,+): Promise<{+ response: ClickHouseProgressResponse;+ summary: Record<string, string> | undefined;+}> {+ opts.abortSignal?.throwIfAborted();+ span.setAttribute("ch.queryId", queryId);+ const response = await context+ .with(trace.setSpan(context.active(), span), () =>+ sendClickhouseQuery({+ ...opts,+ clickhouseSettings: {+ asterisk_include_alias_columns: 1,+ asterisk_include_materialized_columns: 1,+ ...opts.clickhouseSettings,+ },+ format: "JSONEachRowWithProgress",+ span,+ queryId,+ }),+ )+ .catch((error) => {+ throw ClickHouseResourceError.wrapIfResourceError(+ error as Error,+ normalizedTags,+ );+ });++ return {+ response,+ summary: parseClickHouseSummary(response.response_headers),+ };+}++async function* streamClickHouseProgressResponse<T>(+ response: ClickHouseProgressResponse,+): AsyncGenerator<RowOrProgress<T>> {+ for await (const rows of response.stream()) {+ for (const row of rows) yield row.json() as RowOrProgress<T>;}}+/** Own cumulative maxima; completed work is counted once across retries. */+function createClickHouseProgressAccumulator(+ completed: ClickHouseProgressCounters,+) {+ const totals = { ...completed };+ return {+ totals,+ add: ({ progress }: ProgressRow): ProgressRow => {+ const result = { ...progress };+ for (const key of ["read_rows", "read_bytes", "elapsed_ns"] as const) {+ const value = Number(progress[key]);+ if (Number.isFinite(value)) {+ totals[key] = Math.max(totals[key], completed[key] + value);+ }+ result[key] = String(totals[key]);+ }+ const estimate = Number(progress.total_rows_to_read);+ if (estimate > 0) {+ result.total_rows_to_read = String(+ Math.max(+ totals.read_rows,+ completed.read_rows + (Number.isFinite(estimate) ? estimate : 0),+ ),+ );+ }+ return { progress: result };+ },+ };+}++function recordClickHouseProgressSuccess(+ summary: Record<string, string> | undefined,+ context: ClickHouseProgressQueryContext,+): void {+ recordClickHouseQueryOutcome(+ "success",+ context.normalizedTags,+ context.table,+ context.shape,+ context.clickhouseService,+ );+ if (summary) {+ recordClickHouseQueryElapsed(+ summary,+ context.normalizedTags,+ context.table,+ context.shape,+ context.clickhouseService,+ );+ }+}++function shouldRetryClickHouseProgress(+ error: unknown,+ emittedDataRow: boolean,+ attemptNumber: number,+ maxAttempts: number,+): boolean {+ return (+ !emittedDataRow &&+ attemptNumber < maxAttempts &&+ !(error instanceof ClickHouseQueryAbortedError) &&+ isRetryableError(error)+ );+}++async function retryClickHouseProgress(+ error: unknown,+ attemptNumber: number,+ maxAttempts: number,+ normalizedTags: NormalizedClickHouseQueryTags,+ span: Span,+): Promise<void> {+ const message = error instanceof Error ? error.message : String(error);+ logger.warn(+ `ClickHouse query failed with retryable error (attempt ${attemptNumber}/${maxAttempts}): ${message}`,+ {+ error: message,+ attemptNumber,+ tags: normalizedTags,+ },+ );+ span.addEvent("clickhouse-query-retry", {+ "retry.attempt": attemptNumber,+ "retry.error": message,+ });+ await new Promise((resolve) => setTimeout(resolve, 100));+}++function recordClickHouseProgressError(+ error: unknown,+ queryId: string | undefined,+ context: ClickHouseProgressQueryContext,+): Error {+ const originalError = error as Error;+ const wrapped =+ originalError instanceof ClickHouseResourceError+ ? new ClickHouseResourceError(+ originalError.errorType,+ enrichWithQueryId(originalError, queryId),+ context.normalizedTags,+ )+ : ClickHouseResourceError.wrapIfResourceError(+ enrichWithQueryId(originalError, queryId),+ context.normalizedTags,+ );+ const fallbackOutcome =+ error instanceof ClickHouseQueryAbortedError ? "cancelled" : "error";+ recordClickHouseQueryOutcome(+ wrapped instanceof ClickHouseResourceError+ ? CLICKHOUSE_RESOURCE_ERROR_OUTCOMES[wrapped.errorType]+ : fallbackOutcome,+ context.normalizedTags,+ context.table,+ context.shape,+ context.clickhouseService,+ error,+ );+ return wrapped;+}++type ClickHouseProgressCounters = {+ read_rows: number;+ read_bytes: number;+ elapsed_ns: number;+};++type ClickHouseProgressResponse = {+ stream: () => AsyncIterable<ReadonlyArray<{ json: () => unknown }>>;+};++type ClickHouseProgressQueryContext = {+ normalizedTags: NormalizedClickHouseQueryTags;+ table: ReturnType<typeof clickHouseQueryTableLabel>;+ shape: ReturnType<typeof clickHouseQueryShape>;+ clickhouseService: ReturnType<typeof resolveClickhouseService>;+};+export async function commandClickhouse(opts: {query: string;params?: Record<string, unknown> | undefined;diff --git a/web/src/__tests__/server/repositories/clickhouse-resource-errors.servertest.ts b/web/src/__tests__/server/repositories/clickhouse-resource-errors.servertest.tsindex c71b0ef072..38195642c4 100644--- a/web/src/__tests__/server/repositories/clickhouse-resource-errors.servertest.ts+++ b/web/src/__tests__/server/repositories/clickhouse-resource-errors.servertest.ts@@ -5,6 +5,9 @@ import {queryClickhouseExecRaw,queryClickhouseWithProgress,ClickHouseResourceError,+ ClickHouseQueryAbortedError,+ isRow,+ isProgressRow,} from "@langfuse/shared/src/server";import { fail } from "assert";import { randomUUID } from "node:crypto";@@ -203,6 +206,43 @@ describe("ClickHouse Resource Error Handling", () => {});describe("queryClickhouseWithProgress", () => {+ it("streams the same rows and reports scan counters", async () => {+ const query = "SELECT toUInt32(number) AS n FROM numbers(5)";+ const rows: { n: number }[] = [];+ let readRows = 0;+ let readBytes = 0;+ for await (const item of queryClickhouseWithProgress<{ n: number }>({+ query,+ clickhouseSettings: { interactive_delay: "0", max_block_size: "1" },+ })) {+ if (isRow(item)) rows.push(item.row);+ if (isProgressRow(item)) {+ readRows = Number(item.progress.read_rows);+ readBytes = Number(item.progress.read_bytes);+ }+ }+ expect(rows).toEqual(await queryClickhouse({ query }));+ expect(readRows).toBeGreaterThanOrEqual(5);+ expect(readBytes).toBeGreaterThan(0);+ });++ it.each(["before", "during"])(+ "returns a typed cancellation error when aborted %s a query",+ async (when) => {+ const stream = queryClickhouseWithProgress({+ query: "SELECT sleepEachRow(0.02) FROM numbers(100)",+ abortSignal:+ when === "before" ? AbortSignal.abort() : AbortSignal.timeout(50),+ clickhouseSettings: {+ cancel_http_readonly_queries_on_client_close: 1,+ },+ });+ await expect(Array.fromAsync(stream)).rejects.toBeInstanceOf(+ ClickHouseQueryAbortedError,+ );+ },+ );+it("should include query_id exactly once in errors thrown before streaming starts", async () => {const generator = queryClickhouseWithProgress({query: `SELECT * FROM non_existent_table_xyz123`,