Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(worker): register Topics tables in the ClickHouse writer (#19389)

Lysander Kiesel9c6205d9b73d
diff --git a/packages/shared/src/server/topics/clickhouse.ts b/packages/shared/src/server/topics/clickhouse.ts
index 76045b3f81..1d946d50c8 100644
--- a/packages/shared/src/server/topics/clickhouse.ts
+++ b/packages/shared/src/server/topics/clickhouse.ts
@@ -722,7 +722,33 @@ export async function writeTopicSummaries(rows: TopicSummary[]): Promise<void> {
if (!rows.length) return;
if (!rows.every(isValidTopicSummary))
throw new Error("Invalid Topics summary result.");
- await insertTopicRows("topic_facet_summaries", rows, (row) => ({
+ await insertTopicRows("topic_facet_summaries", rows, topicSummaryRecord);
+}
+
+export async function writeTopicAssignments(
+ rows: TopicAssignment[],
+): Promise<void> {
+ if (!rows.length) return;
+ for (const row of rows) {
+ if (
+ !topicSourceSchema.safeParse(row).success ||
+ typeof row.environment !== "string" ||
+ (row.topicId === null) !== (row.topicVersionId === null) ||
+ (row.distance !== null && !Number.isFinite(row.distance)) ||
+ (row.runnerUpDistance !== null &&
+ !Number.isFinite(row.runnerUpDistance)) ||
+ (row.coordinates !== null &&
+ (!row.runId ||
+ row.coordinates.length !== 2 ||
+ !row.coordinates.every(Number.isFinite)))
+ )
+ throw new Error("Invalid Topics assignment.");
+ }
+ await insertTopicRows("topic_assignments", rows, topicAssignmentRecord);
+}
+
+function topicSummaryRecord(row: TopicSummary) {
+ return {
project_id: row.projectId,
facet_id: row.facetId,
facet_version: row.facetVersion,
@@ -747,29 +773,16 @@ export async function writeTopicSummaries(rows: TopicSummary[]): Promise<void> {
cost_details: row.costDetails,
processed_at: convertDateToClickhouseDateTime(new Date(row.processedAt)),
metadata: JSON.stringify(row.metadata),
- }));
+ };
}
-export async function writeTopicAssignments(
- rows: TopicAssignment[],
-): Promise<void> {
- if (!rows.length) return;
- for (const row of rows) {
- if (
- !topicSourceSchema.safeParse(row).success ||
- typeof row.environment !== "string" ||
- (row.topicId === null) !== (row.topicVersionId === null) ||
- (row.distance !== null && !Number.isFinite(row.distance)) ||
- (row.runnerUpDistance !== null &&
- !Number.isFinite(row.runnerUpDistance)) ||
- (row.coordinates !== null &&
- (!row.runId ||
- row.coordinates.length !== 2 ||
- !row.coordinates.every(Number.isFinite)))
- )
- throw new Error("Invalid Topics assignment.");
- }
- await insertTopicRows("topic_assignments", rows, (row) => ({
+/** A `topic_facet_summaries` row as inserted with JSONEachRow. */
+export type TopicFacetSummaryRecordInsertType = ReturnType<
+ typeof topicSummaryRecord
+>;
+
+function topicAssignmentRecord(row: TopicAssignment) {
+ return {
project_id: row.projectId,
facet_id: row.facetId,
facet_version: row.facetVersion,
@@ -791,9 +804,14 @@ export async function writeTopicAssignments(
origin: row.origin,
coordinates: row.coordinates ?? [],
assigned_at: convertDateToClickhouseDateTime(new Date(row.assignedAt)),
- }));
+ };
}
+/** A `topic_assignments` row as inserted with JSONEachRow. */
+export type TopicAssignmentRecordInsertType = ReturnType<
+ typeof topicAssignmentRecord
+>;
+
const assignmentColumns = `project_id AS projectId, facet_id AS facetId,
facet_version AS facetVersion, trace_id AS traceId,
session_id AS sessionId,
diff --git a/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts b/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts
index 606d0360f6..5a5330b9d9 100644
--- a/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts
+++ b/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts
@@ -571,6 +571,20 @@ describe("ClickhouseWriter", () => {
);
});
+ it("leaves a Topics metadata JSON string untouched when truncating", () => {
+ const row = {
+ project_id: "project-1",
+ facet_id: "facet-1",
+ facet_version: 1,
+ trace_id: "t-1",
+ session_id: "",
+ metadata: '{"a":1}',
+ } as any;
+ expect(
+ truncateOversizedRecord(TableName.TopicFacetSummaries, row).metadata,
+ ).toBe('{"a":1}');
+ });
+
it("should retry client request timeouts within the same flush", async () => {
const mockInsert = vi
.spyOn(clickhouseClientMock, "insert")
diff --git a/worker/src/services/ClickhouseWriter/index.ts b/worker/src/services/ClickhouseWriter/index.ts
index b58d57e857..1d718a9433 100644
--- a/worker/src/services/ClickhouseWriter/index.ts
+++ b/worker/src/services/ClickhouseWriter/index.ts
@@ -88,6 +88,8 @@ export class ClickhouseWriter<
[TableName.BlobStorageFileLog]: [],
[TableName.DatasetRunItems]: [],
[TableName.EventsFull]: [],
+ [TableName.TopicFacetSummaries]: [],
+ [TableName.TopicAssignments]: [],
};
this.start();
@@ -174,16 +176,7 @@ export class ClickhouseWriter<
public async flushAll(fullQueue = false) {
const tables = this.allowedTable
? [this.allowedTable]
- : [
- TableName.Traces,
- TableName.TracesNull,
- TableName.Scores,
- TableName.Observations,
- TableName.ObservationsBatchStaging,
- TableName.BlobStorageFileLog,
- TableName.DatasetRunItems,
- TableName.EventsFull,
- ];
+ : Object.values(TableName);
return this.trackActiveFlush(
instrumentAsync(
{
diff --git a/worker/src/services/ClickhouseWriter/jsonRecords.ts b/worker/src/services/ClickhouseWriter/jsonRecords.ts
index 2b1368bbd8..bd65043ab5 100644
--- a/worker/src/services/ClickhouseWriter/jsonRecords.ts
+++ b/worker/src/services/ClickhouseWriter/jsonRecords.ts
@@ -9,6 +9,27 @@ const DECIMAL_64_12_LIMIT = new Decimal("1e6");
const DECIMAL_64_12_MAX_NUM = 999_999.999_999;
const DECIMAL_64_12_MIN_NUM = -DECIMAL_64_12_MAX_NUM;
+/** Identifies a dropped row in logs. Topics rows have no `id` column. */
+export function recordIdentity(record: RecordInsertType<TableName>): {
+ project_id: string;
+ trace_id: string | null | undefined;
+ id: string;
+} {
+ if ("id" in record) {
+ return {
+ project_id: record.project_id,
+ trace_id:
+ ("trace_id" in record ? record.trace_id : undefined) ?? record.id,
+ id: record.id,
+ };
+ }
+ return {
+ project_id: record.project_id,
+ trace_id: record.trace_id || null,
+ id: `${record.facet_id}:${record.facet_version}:${record.trace_id || record.session_id}`,
+ };
+}
+
export function truncateOversizedRecord<
Row extends RecordInsertType<TableName>,
>(tableName: TableName, record: Row): Row {
@@ -53,15 +74,19 @@ export function truncateOversizedRecord<
);
}
- // Truncate metadata field if present
- if ("metadata" in record && record.metadata) {
+ // Truncate map metadata; JSON-string metadata (Topics) can't be cut without breaking it
+ if (
+ "metadata" in record &&
+ record.metadata &&
+ typeof record.metadata === "object"
+ ) {
const metadata = record.metadata;
const truncatedMetadata: Record<string, string> = {};
for (const [key, value] of Object.entries(metadata)) {
if (value && value.length > maxFieldSize) {
truncatedMetadata[key] = truncateField(value) || "";
logger.info(
- `Truncated oversized metadata for record ${record.id} of type ${tableName} and key ${key}`,
+ `Truncated oversized metadata for record ${recordIdentity(record).id} of type ${tableName} and key ${key}`,
{
projectId: record.project_id,
},
@@ -127,12 +152,16 @@ export function clampDecimal64Fields<Row extends RecordInsertType<TableName>>(
tableName: TableName,
record: Row,
): Row {
- const ctx = { recordId: record.id, projectId: record.project_id };
+ const ctx = {
+ recordId: recordIdentity(record).id,
+ projectId: record.project_id,
+ };
switch (tableName) {
case TableName.Observations:
case TableName.ObservationsBatchStaging:
- case TableName.EventsFull: {
+ case TableName.EventsFull:
+ case TableName.TopicFacetSummaries: {
if ("provided_cost_details" in record) {
record.provided_cost_details = clampDecimal64Map(
record.provided_cost_details,
diff --git a/worker/src/services/ClickhouseWriter/types.ts b/worker/src/services/ClickhouseWriter/types.ts
index 25633f5370..bfbc264f66 100644
--- a/worker/src/services/ClickhouseWriter/types.ts
+++ b/worker/src/services/ClickhouseWriter/types.ts
@@ -8,6 +8,10 @@ import type {
DatasetRunItemRecordInsertType,
EventRecordInsertType,
} from "@langfuse/shared/src/server";
+import type {
+ TopicAssignmentRecordInsertType,
+ TopicFacetSummaryRecordInsertType,
+} from "@langfuse/shared/topics/server";
export enum TableName {
Traces = "traces",
@@ -18,6 +22,8 @@ export enum TableName {
BlobStorageFileLog = "blob_storage_file_log",
DatasetRunItems = "dataset_run_items_rmt",
EventsFull = "events_full", // Primary write target - MV auto-populates events_core
+ TopicFacetSummaries = "topic_facet_summaries",
+ TopicAssignments = "topic_assignments",
}
export type RecordInsertType<T extends TableName> = T extends TableName.Scores
@@ -36,4 +42,8 @@ export type RecordInsertType<T extends TableName> = T extends TableName.Scores
? DatasetRunItemRecordInsertType
: T extends TableName.EventsFull
? EventRecordInsertType
- : never;
+ : T extends TableName.TopicFacetSummaries
+ ? TopicFacetSummaryRecordInsertType
+ : T extends TableName.TopicAssignments
+ ? TopicAssignmentRecordInsertType
+ : never;
diff --git a/worker/src/services/ClickhouseWriter/writeStrategies.ts b/worker/src/services/ClickhouseWriter/writeStrategies.ts
index d236fb1963..58bb8f176f 100644
--- a/worker/src/services/ClickhouseWriter/writeStrategies.ts
+++ b/worker/src/services/ClickhouseWriter/writeStrategies.ts
@@ -8,7 +8,11 @@ import {
import type { ClickhouseClientType } from "@langfuse/shared/src/server";
import type { RecordInsertType, TableName } from "./types";
-import { clampDecimal64Fields, truncateOversizedRecord } from "./jsonRecords";
+import {
+ clampDecimal64Fields,
+ recordIdentity,
+ truncateOversizedRecord,
+} from "./jsonRecords";
export type ClickhouseWriteStrategy<Row> = {
write(
@@ -47,14 +51,7 @@ const jsonWriteStrategy: ClickhouseWriteStrategy<RecordInsertType<TableName>> =
},
prepare: clampDecimal64Fields,
truncate: truncateOversizedRecord,
- droppedId(record) {
- return {
- project_id: record.project_id,
- trace_id:
- ("trace_id" in record ? record.trace_id : undefined) ?? record.id,
- id: record.id,
- };
- },
+ droppedId: recordIdentity,
};
export const jsonWriteStrategyFactory: ClickhouseWriteStrategyFactory<