Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

chore(worker): log the row a ClickHouse parse error names (#18577)

Niklas Semmlera416d22f9534
diff --git a/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts b/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts
index 299ad80588..846350b7e1 100644
--- a/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts
+++ b/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts
@@ -437,6 +437,33 @@ describe("ClickhouseWriter", () => {
expect(nativeWriter["queue"][TableName.EventsFull]).toHaveLength(0);
});
+ it("logs the row a ClickHouse parse error names, with its S3 file", async () => {
+ const rows = ["e-1", "e-2", "e-3"].map((id) => ({
+ id,
+ project_id: "project-1",
+ trace_id: "trace-1",
+ blob_storage_file_path: `project-1/observation/${id}/event.json`,
+ }));
+ vi.spyOn(clickhouseClientMock, "insert").mockRejectedValue(
+ new Error(
+ "Cannot parse input: expected ',' before: 'abc' (at row 2) : While executing WaitForAsyncInsert",
+ ),
+ );
+
+ rows.forEach((row) => writer.addToQueue(TableName.EventsFull, row as any));
+ await writer["flushAll"](true);
+
+ expect(logger.error).toHaveBeenCalledWith(
+ expect.stringContaining("ClickHouse rejected events_full row"),
+ {
+ project_id: "project-1",
+ trace_id: "trace-1",
+ id: "e-2",
+ blobStorageFilePath: "project-1/observation/e-2/event.json",
+ },
+ );
+ });
+
it("should mark writer insert log comments as multi-project", async () => {
const mockInsert = vi
.spyOn(clickhouseClientMock, "insert")
diff --git a/worker/src/services/ClickhouseWriter/index.ts b/worker/src/services/ClickhouseWriter/index.ts
index 99f58bdb7f..29c8fb891d 100644
--- a/worker/src/services/ClickhouseWriter/index.ts
+++ b/worker/src/services/ClickhouseWriter/index.ts
@@ -472,6 +472,8 @@ export class ClickhouseWriter<
err,
);
+ this.logRejectedRow(tableName, err, queueItems, writeStrategy);
+
// Re-add the records to the queue with incremented attempts
const reason = classifyJobFailure(err);
let droppedCount = 0;
@@ -514,6 +516,27 @@ export class ClickhouseWriter<
}
}
+ /** Log the row a ClickHouse parse error names, so its S3 event file can be found. */
+ private logRejectedRow<T extends TableName>(
+ tableName: T,
+ err: unknown,
+ queueItems: ClickhouseWriterQueueItem<PayloadMap[T]>[],
+ strategy: ClickhouseWriteStrategy<PayloadMap[TableName]>,
+ ) {
+ const row = queueItems[failedRowNumber(err) - 1]?.data;
+ if (!row) return;
+ this.logger.error(
+ `${this.logPrefix}ClickhouseWriter: ClickHouse rejected ${tableName} row`,
+ {
+ ...strategy.droppedId(row),
+ blobStorageFilePath:
+ "blob_storage_file_path" in row
+ ? row.blob_storage_file_path
+ : undefined,
+ },
+ );
+ }
+
public addToQueue<T extends TableName>(tableName: T, data: PayloadMap[T]) {
if (this.allowedTable && tableName !== this.allowedTable) {
throw new Error(
@@ -577,6 +600,12 @@ export class ClickhouseWriter<
}
}
+/** 1-based row from ClickHouse parse errors, e.g. "Cannot parse input: ... (at row 11)"; NaN if absent. */
+function failedRowNumber(err: unknown): number {
+ const message = err instanceof Error ? err.message : "";
+ return Number(/\(at row (\d+)\)/.exec(message)?.[1]);
+}
+
type WriterPayloadMap = { [T in TableName]: object };
type JsonWriterPayloadMap = {