Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(worker): await in-flight ClickHouse writes during shutdown (#17836)

Valery Meleshkinbe2279ff733f
diff --git a/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts b/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts
index f90420d36f..c24cd60dc5 100644
--- a/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts
+++ b/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts
@@ -227,6 +227,42 @@ describe("ClickhouseWriter", () => {
);
});
+ it.each(["batch-size", "interval"] as const)(
+ "waits for an in-flight %s flush before shutdown completes",
+ async (flushTrigger) => {
+ let resolveInsert!: () => void;
+ const mockInsert = vi
+ .spyOn(clickhouseClientMock, "insert")
+ .mockImplementation(
+ () => new Promise<void>((resolve) => (resolveInsert = resolve)),
+ );
+ if (flushTrigger === "batch-size") writer.batchSize = 1;
+ writer.addToQueue(TableName.Traces, { id: "1", name: "trace" } as any);
+ if (flushTrigger === "interval") {
+ await vi.advanceTimersByTimeAsync(writer.writeInterval);
+ }
+
+ let shutdown: Promise<void> | undefined;
+ try {
+ await vi.waitFor(() => expect(mockInsert).toHaveBeenCalledTimes(1));
+ let shutdownComplete = false;
+ shutdown = writer.shutdown().then(() => {
+ shutdownComplete = true;
+ });
+ await vi.advanceTimersByTimeAsync(0);
+
+ expect(shutdownComplete).toBe(false);
+
+ resolveInsert();
+ await shutdown;
+ expect(shutdownComplete).toBe(true);
+ } finally {
+ resolveInsert?.();
+ await shutdown;
+ }
+ },
+ );
+
it("should handle multiple table types", async () => {
const mockInsert = vi
.spyOn(clickhouseClientMock, "insert")
diff --git a/worker/src/services/ClickhouseWriter/index.ts b/worker/src/services/ClickhouseWriter/index.ts
index 7b200f269a..76dffadc85 100644
--- a/worker/src/services/ClickhouseWriter/index.ts
+++ b/worker/src/services/ClickhouseWriter/index.ts
@@ -36,6 +36,7 @@ const MULTI_PROJECT_LOG_COMMENT_PROJECT_ID = "MULTI_PROJECT";
export class ClickhouseWriter {
private static instance: ClickhouseWriter | null = null;
private static client: ClickhouseClientType | null = null;
+ private readonly activeFlushes = new Set<Promise<void>>();
batchSize: number;
writeInterval: number;
maxAttempts: number;
@@ -105,32 +106,47 @@ export class ClickhouseWriter {
this.intervalId = null;
}
+ while (this.activeFlushes.size > 0) {
+ await Promise.all([...this.activeFlushes]);
+ }
+
await this.flushAll(true);
logger.info("ClickhouseWriter shutdown complete.");
}
public async flushAll(fullQueue = false) {
- return instrumentAsync(
- {
- name: "write-to-clickhouse",
- },
- async () => {
- recordIncrement("langfuse.queue.clickhouse_writer.request");
- await Promise.all([
- this.flush(TableName.Traces, fullQueue),
- this.flush(TableName.TracesNull, fullQueue),
- this.flush(TableName.Scores, fullQueue),
- this.flush(TableName.Observations, fullQueue),
- this.flush(TableName.ObservationsBatchStaging, fullQueue),
- this.flush(TableName.BlobStorageFileLog, fullQueue),
- this.flush(TableName.DatasetRunItems, fullQueue),
- this.flush(TableName.EventsFull, fullQueue),
- ]).catch((err) => {
- logger.error("ClickhouseWriter.flushAll", err);
- });
- },
+ return this.trackActiveFlush(
+ instrumentAsync(
+ {
+ name: "write-to-clickhouse",
+ },
+ async () => {
+ recordIncrement("langfuse.queue.clickhouse_writer.request");
+ await Promise.all([
+ this.flush(TableName.Traces, fullQueue),
+ this.flush(TableName.TracesNull, fullQueue),
+ this.flush(TableName.Scores, fullQueue),
+ this.flush(TableName.Observations, fullQueue),
+ this.flush(TableName.ObservationsBatchStaging, fullQueue),
+ this.flush(TableName.BlobStorageFileLog, fullQueue),
+ this.flush(TableName.DatasetRunItems, fullQueue),
+ this.flush(TableName.EventsFull, fullQueue),
+ ]).catch((err) => {
+ logger.error("ClickhouseWriter.flushAll", err);
+ });
+ },
+ ),
+ );
+ }
+
+ private trackActiveFlush(flush: Promise<void>): Promise<void> {
+ this.activeFlushes.add(flush);
+ flush.then(
+ () => this.activeFlushes.delete(flush),
+ () => this.activeFlushes.delete(flush),
);
+ return flush;
}
private isRetryableError(error: unknown): boolean {
@@ -586,7 +602,7 @@ export class ClickhouseWriter {
if (entityQueue.length >= this.batchSize) {
logger.debug(`Queue is full. Flushing ${tableName}...`);
- this.flush(tableName).catch((err) => {
+ this.trackActiveFlush(this.flush(tableName)).catch((err) => {
logger.error("ClickhouseWriter.addToQueue flush", err);
});
}