🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(worker): await in-flight ClickHouse writes during shutdown (#17836)
Valery Meleshkinbe2279ff733f
diff --git a/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts b/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.tsindex f90420d36f..c24cd60dc5 100644--- a/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts+++ b/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts@@ -227,6 +227,42 @@ describe("ClickhouseWriter", () => {);});+ it.each(["batch-size", "interval"] as const)(+ "waits for an in-flight %s flush before shutdown completes",+ async (flushTrigger) => {+ let resolveInsert!: () => void;+ const mockInsert = vi+ .spyOn(clickhouseClientMock, "insert")+ .mockImplementation(+ () => new Promise<void>((resolve) => (resolveInsert = resolve)),+ );+ if (flushTrigger === "batch-size") writer.batchSize = 1;+ writer.addToQueue(TableName.Traces, { id: "1", name: "trace" } as any);+ if (flushTrigger === "interval") {+ await vi.advanceTimersByTimeAsync(writer.writeInterval);+ }++ let shutdown: Promise<void> | undefined;+ try {+ await vi.waitFor(() => expect(mockInsert).toHaveBeenCalledTimes(1));+ let shutdownComplete = false;+ shutdown = writer.shutdown().then(() => {+ shutdownComplete = true;+ });+ await vi.advanceTimersByTimeAsync(0);++ expect(shutdownComplete).toBe(false);++ resolveInsert();+ await shutdown;+ expect(shutdownComplete).toBe(true);+ } finally {+ resolveInsert?.();+ await shutdown;+ }+ },+ );+it("should handle multiple table types", async () => {const mockInsert = vi.spyOn(clickhouseClientMock, "insert")diff --git a/worker/src/services/ClickhouseWriter/index.ts b/worker/src/services/ClickhouseWriter/index.tsindex 7b200f269a..76dffadc85 100644--- a/worker/src/services/ClickhouseWriter/index.ts+++ b/worker/src/services/ClickhouseWriter/index.ts@@ -36,6 +36,7 @@ const MULTI_PROJECT_LOG_COMMENT_PROJECT_ID = "MULTI_PROJECT";export class ClickhouseWriter {private static instance: ClickhouseWriter | null = null;private static client: ClickhouseClientType | null = null;+ private readonly activeFlushes = new Set<Promise<void>>();batchSize: number;writeInterval: number;maxAttempts: number;@@ -105,32 +106,47 @@ export class ClickhouseWriter {this.intervalId = null;}+ while (this.activeFlushes.size > 0) {+ await Promise.all([...this.activeFlushes]);+ }+await this.flushAll(true);logger.info("ClickhouseWriter shutdown complete.");}public async flushAll(fullQueue = false) {- return instrumentAsync(- {- name: "write-to-clickhouse",- },- async () => {- recordIncrement("langfuse.queue.clickhouse_writer.request");- await Promise.all([- this.flush(TableName.Traces, fullQueue),- this.flush(TableName.TracesNull, fullQueue),- this.flush(TableName.Scores, fullQueue),- this.flush(TableName.Observations, fullQueue),- this.flush(TableName.ObservationsBatchStaging, fullQueue),- this.flush(TableName.BlobStorageFileLog, fullQueue),- this.flush(TableName.DatasetRunItems, fullQueue),- this.flush(TableName.EventsFull, fullQueue),- ]).catch((err) => {- logger.error("ClickhouseWriter.flushAll", err);- });- },+ return this.trackActiveFlush(+ instrumentAsync(+ {+ name: "write-to-clickhouse",+ },+ async () => {+ recordIncrement("langfuse.queue.clickhouse_writer.request");+ await Promise.all([+ this.flush(TableName.Traces, fullQueue),+ this.flush(TableName.TracesNull, fullQueue),+ this.flush(TableName.Scores, fullQueue),+ this.flush(TableName.Observations, fullQueue),+ this.flush(TableName.ObservationsBatchStaging, fullQueue),+ this.flush(TableName.BlobStorageFileLog, fullQueue),+ this.flush(TableName.DatasetRunItems, fullQueue),+ this.flush(TableName.EventsFull, fullQueue),+ ]).catch((err) => {+ logger.error("ClickhouseWriter.flushAll", err);+ });+ },+ ),+ );+ }++ private trackActiveFlush(flush: Promise<void>): Promise<void> {+ this.activeFlushes.add(flush);+ flush.then(+ () => this.activeFlushes.delete(flush),+ () => this.activeFlushes.delete(flush),);+ return flush;}private isRetryableError(error: unknown): boolean {@@ -586,7 +602,7 @@ export class ClickhouseWriter {if (entityQueue.length >= this.batchSize) {logger.debug(`Queue is full. Flushing ${tableName}...`);- this.flush(tableName).catch((err) => {+ this.trackActiveFlush(this.flush(tableName)).catch((err) => {logger.error("ClickhouseWriter.addToQueue flush", err);});}