🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
chore(worker): log the row a ClickHouse parse error names (#18577)
Niklas Semmlera416d22f9534
diff --git a/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts b/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.tsindex 299ad80588..846350b7e1 100644--- a/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts+++ b/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts@@ -437,6 +437,33 @@ describe("ClickhouseWriter", () => {expect(nativeWriter["queue"][TableName.EventsFull]).toHaveLength(0);});+ it("logs the row a ClickHouse parse error names, with its S3 file", async () => {+ const rows = ["e-1", "e-2", "e-3"].map((id) => ({+ id,+ project_id: "project-1",+ trace_id: "trace-1",+ blob_storage_file_path: `project-1/observation/${id}/event.json`,+ }));+ vi.spyOn(clickhouseClientMock, "insert").mockRejectedValue(+ new Error(+ "Cannot parse input: expected ',' before: 'abc' (at row 2) : While executing WaitForAsyncInsert",+ ),+ );++ rows.forEach((row) => writer.addToQueue(TableName.EventsFull, row as any));+ await writer["flushAll"](true);++ expect(logger.error).toHaveBeenCalledWith(+ expect.stringContaining("ClickHouse rejected events_full row"),+ {+ project_id: "project-1",+ trace_id: "trace-1",+ id: "e-2",+ blobStorageFilePath: "project-1/observation/e-2/event.json",+ },+ );+ });+it("should mark writer insert log comments as multi-project", async () => {const mockInsert = vi.spyOn(clickhouseClientMock, "insert")diff --git a/worker/src/services/ClickhouseWriter/index.ts b/worker/src/services/ClickhouseWriter/index.tsindex 99f58bdb7f..29c8fb891d 100644--- a/worker/src/services/ClickhouseWriter/index.ts+++ b/worker/src/services/ClickhouseWriter/index.ts@@ -472,6 +472,8 @@ export class ClickhouseWriter<err,);+ this.logRejectedRow(tableName, err, queueItems, writeStrategy);+// Re-add the records to the queue with incremented attemptsconst reason = classifyJobFailure(err);let droppedCount = 0;@@ -514,6 +516,27 @@ export class ClickhouseWriter<}}+ /** Log the row a ClickHouse parse error names, so its S3 event file can be found. */+ private logRejectedRow<T extends TableName>(+ tableName: T,+ err: unknown,+ queueItems: ClickhouseWriterQueueItem<PayloadMap[T]>[],+ strategy: ClickhouseWriteStrategy<PayloadMap[TableName]>,+ ) {+ const row = queueItems[failedRowNumber(err) - 1]?.data;+ if (!row) return;+ this.logger.error(+ `${this.logPrefix}ClickhouseWriter: ClickHouse rejected ${tableName} row`,+ {+ ...strategy.droppedId(row),+ blobStorageFilePath:+ "blob_storage_file_path" in row+ ? row.blob_storage_file_path+ : undefined,+ },+ );+ }+public addToQueue<T extends TableName>(tableName: T, data: PayloadMap[T]) {if (this.allowedTable && tableName !== this.allowedTable) {throw new Error(@@ -577,6 +600,12 @@ export class ClickhouseWriter<}}+/** 1-based row from ClickHouse parse errors, e.g. "Cannot parse input: ... (at row 11)"; NaN if absent. */+function failedRowNumber(err: unknown): number {+ const message = err instanceof Error ? err.message : "";+ return Number(/\(at row (\d+)\)/.exec(message)?.[1]);+}+type WriterPayloadMap = { [T in TableName]: object };type JsonWriterPayloadMap = {