🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(worker): register Topics tables in the ClickHouse writer (#19389)
Lysander Kiesel9c6205d9b73d
diff --git a/packages/shared/src/server/topics/clickhouse.ts b/packages/shared/src/server/topics/clickhouse.tsindex 76045b3f81..1d946d50c8 100644--- a/packages/shared/src/server/topics/clickhouse.ts+++ b/packages/shared/src/server/topics/clickhouse.ts@@ -722,7 +722,33 @@ export async function writeTopicSummaries(rows: TopicSummary[]): Promise<void> {if (!rows.length) return;if (!rows.every(isValidTopicSummary))throw new Error("Invalid Topics summary result.");- await insertTopicRows("topic_facet_summaries", rows, (row) => ({+ await insertTopicRows("topic_facet_summaries", rows, topicSummaryRecord);+}++export async function writeTopicAssignments(+ rows: TopicAssignment[],+): Promise<void> {+ if (!rows.length) return;+ for (const row of rows) {+ if (+ !topicSourceSchema.safeParse(row).success ||+ typeof row.environment !== "string" ||+ (row.topicId === null) !== (row.topicVersionId === null) ||+ (row.distance !== null && !Number.isFinite(row.distance)) ||+ (row.runnerUpDistance !== null &&+ !Number.isFinite(row.runnerUpDistance)) ||+ (row.coordinates !== null &&+ (!row.runId ||+ row.coordinates.length !== 2 ||+ !row.coordinates.every(Number.isFinite)))+ )+ throw new Error("Invalid Topics assignment.");+ }+ await insertTopicRows("topic_assignments", rows, topicAssignmentRecord);+}++function topicSummaryRecord(row: TopicSummary) {+ return {project_id: row.projectId,facet_id: row.facetId,facet_version: row.facetVersion,@@ -747,29 +773,16 @@ export async function writeTopicSummaries(rows: TopicSummary[]): Promise<void> {cost_details: row.costDetails,processed_at: convertDateToClickhouseDateTime(new Date(row.processedAt)),metadata: JSON.stringify(row.metadata),- }));+ };}-export async function writeTopicAssignments(- rows: TopicAssignment[],-): Promise<void> {- if (!rows.length) return;- for (const row of rows) {- if (- !topicSourceSchema.safeParse(row).success ||- typeof row.environment !== "string" ||- (row.topicId === null) !== (row.topicVersionId === null) ||- (row.distance !== null && !Number.isFinite(row.distance)) ||- (row.runnerUpDistance !== null &&- !Number.isFinite(row.runnerUpDistance)) ||- (row.coordinates !== null &&- (!row.runId ||- row.coordinates.length !== 2 ||- !row.coordinates.every(Number.isFinite)))- )- throw new Error("Invalid Topics assignment.");- }- await insertTopicRows("topic_assignments", rows, (row) => ({+/** A `topic_facet_summaries` row as inserted with JSONEachRow. */+export type TopicFacetSummaryRecordInsertType = ReturnType<+ typeof topicSummaryRecord+>;++function topicAssignmentRecord(row: TopicAssignment) {+ return {project_id: row.projectId,facet_id: row.facetId,facet_version: row.facetVersion,@@ -791,9 +804,14 @@ export async function writeTopicAssignments(origin: row.origin,coordinates: row.coordinates ?? [],assigned_at: convertDateToClickhouseDateTime(new Date(row.assignedAt)),- }));+ };}+/** A `topic_assignments` row as inserted with JSONEachRow. */+export type TopicAssignmentRecordInsertType = ReturnType<+ typeof topicAssignmentRecord+>;+const assignmentColumns = `project_id AS projectId, facet_id AS facetId,facet_version AS facetVersion, trace_id AS traceId,session_id AS sessionId,diff --git a/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts b/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.tsindex 606d0360f6..5a5330b9d9 100644--- a/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts+++ b/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts@@ -571,6 +571,20 @@ describe("ClickhouseWriter", () => {);});+ it("leaves a Topics metadata JSON string untouched when truncating", () => {+ const row = {+ project_id: "project-1",+ facet_id: "facet-1",+ facet_version: 1,+ trace_id: "t-1",+ session_id: "",+ metadata: '{"a":1}',+ } as any;+ expect(+ truncateOversizedRecord(TableName.TopicFacetSummaries, row).metadata,+ ).toBe('{"a":1}');+ });+it("should retry client request timeouts within the same flush", async () => {const mockInsert = vi.spyOn(clickhouseClientMock, "insert")diff --git a/worker/src/services/ClickhouseWriter/index.ts b/worker/src/services/ClickhouseWriter/index.tsindex b58d57e857..1d718a9433 100644--- a/worker/src/services/ClickhouseWriter/index.ts+++ b/worker/src/services/ClickhouseWriter/index.ts@@ -88,6 +88,8 @@ export class ClickhouseWriter<[TableName.BlobStorageFileLog]: [],[TableName.DatasetRunItems]: [],[TableName.EventsFull]: [],+ [TableName.TopicFacetSummaries]: [],+ [TableName.TopicAssignments]: [],};this.start();@@ -174,16 +176,7 @@ export class ClickhouseWriter<public async flushAll(fullQueue = false) {const tables = this.allowedTable? [this.allowedTable]- : [- TableName.Traces,- TableName.TracesNull,- TableName.Scores,- TableName.Observations,- TableName.ObservationsBatchStaging,- TableName.BlobStorageFileLog,- TableName.DatasetRunItems,- TableName.EventsFull,- ];+ : Object.values(TableName);return this.trackActiveFlush(instrumentAsync({diff --git a/worker/src/services/ClickhouseWriter/jsonRecords.ts b/worker/src/services/ClickhouseWriter/jsonRecords.tsindex 2b1368bbd8..bd65043ab5 100644--- a/worker/src/services/ClickhouseWriter/jsonRecords.ts+++ b/worker/src/services/ClickhouseWriter/jsonRecords.ts@@ -9,6 +9,27 @@ const DECIMAL_64_12_LIMIT = new Decimal("1e6");const DECIMAL_64_12_MAX_NUM = 999_999.999_999;const DECIMAL_64_12_MIN_NUM = -DECIMAL_64_12_MAX_NUM;+/** Identifies a dropped row in logs. Topics rows have no `id` column. */+export function recordIdentity(record: RecordInsertType<TableName>): {+ project_id: string;+ trace_id: string | null | undefined;+ id: string;+} {+ if ("id" in record) {+ return {+ project_id: record.project_id,+ trace_id:+ ("trace_id" in record ? record.trace_id : undefined) ?? record.id,+ id: record.id,+ };+ }+ return {+ project_id: record.project_id,+ trace_id: record.trace_id || null,+ id: `${record.facet_id}:${record.facet_version}:${record.trace_id || record.session_id}`,+ };+}+export function truncateOversizedRecord<Row extends RecordInsertType<TableName>,>(tableName: TableName, record: Row): Row {@@ -53,15 +74,19 @@ export function truncateOversizedRecord<);}- // Truncate metadata field if present- if ("metadata" in record && record.metadata) {+ // Truncate map metadata; JSON-string metadata (Topics) can't be cut without breaking it+ if (+ "metadata" in record &&+ record.metadata &&+ typeof record.metadata === "object"+ ) {const metadata = record.metadata;const truncatedMetadata: Record<string, string> = {};for (const [key, value] of Object.entries(metadata)) {if (value && value.length > maxFieldSize) {truncatedMetadata[key] = truncateField(value) || "";logger.info(- `Truncated oversized metadata for record ${record.id} of type ${tableName} and key ${key}`,+ `Truncated oversized metadata for record ${recordIdentity(record).id} of type ${tableName} and key ${key}`,{projectId: record.project_id,},@@ -127,12 +152,16 @@ export function clampDecimal64Fields<Row extends RecordInsertType<TableName>>(tableName: TableName,record: Row,): Row {- const ctx = { recordId: record.id, projectId: record.project_id };+ const ctx = {+ recordId: recordIdentity(record).id,+ projectId: record.project_id,+ };switch (tableName) {case TableName.Observations:case TableName.ObservationsBatchStaging:- case TableName.EventsFull: {+ case TableName.EventsFull:+ case TableName.TopicFacetSummaries: {if ("provided_cost_details" in record) {record.provided_cost_details = clampDecimal64Map(record.provided_cost_details,diff --git a/worker/src/services/ClickhouseWriter/types.ts b/worker/src/services/ClickhouseWriter/types.tsindex 25633f5370..bfbc264f66 100644--- a/worker/src/services/ClickhouseWriter/types.ts+++ b/worker/src/services/ClickhouseWriter/types.ts@@ -8,6 +8,10 @@ import type {DatasetRunItemRecordInsertType,EventRecordInsertType,} from "@langfuse/shared/src/server";+import type {+ TopicAssignmentRecordInsertType,+ TopicFacetSummaryRecordInsertType,+} from "@langfuse/shared/topics/server";export enum TableName {Traces = "traces",@@ -18,6 +22,8 @@ export enum TableName {BlobStorageFileLog = "blob_storage_file_log",DatasetRunItems = "dataset_run_items_rmt",EventsFull = "events_full", // Primary write target - MV auto-populates events_core+ TopicFacetSummaries = "topic_facet_summaries",+ TopicAssignments = "topic_assignments",}export type RecordInsertType<T extends TableName> = T extends TableName.Scores@@ -36,4 +42,8 @@ export type RecordInsertType<T extends TableName> = T extends TableName.Scores? DatasetRunItemRecordInsertType: T extends TableName.EventsFull? EventRecordInsertType- : never;+ : T extends TableName.TopicFacetSummaries+ ? TopicFacetSummaryRecordInsertType+ : T extends TableName.TopicAssignments+ ? TopicAssignmentRecordInsertType+ : never;diff --git a/worker/src/services/ClickhouseWriter/writeStrategies.ts b/worker/src/services/ClickhouseWriter/writeStrategies.tsindex d236fb1963..58bb8f176f 100644--- a/worker/src/services/ClickhouseWriter/writeStrategies.ts+++ b/worker/src/services/ClickhouseWriter/writeStrategies.ts@@ -8,7 +8,11 @@ import {import type { ClickhouseClientType } from "@langfuse/shared/src/server";import type { RecordInsertType, TableName } from "./types";-import { clampDecimal64Fields, truncateOversizedRecord } from "./jsonRecords";+import {+ clampDecimal64Fields,+ recordIdentity,+ truncateOversizedRecord,+} from "./jsonRecords";export type ClickhouseWriteStrategy<Row> = {write(@@ -47,14 +51,7 @@ const jsonWriteStrategy: ClickhouseWriteStrategy<RecordInsertType<TableName>> =},prepare: clampDecimal64Fields,truncate: truncateOversizedRecord,- droppedId(record) {- return {- project_id: record.project_id,- trace_id:- ("trace_id" in record ? record.trace_id : undefined) ?? record.id,- id: record.id,- };- },+ droppedId: recordIdentity,};export const jsonWriteStrategyFactory: ClickhouseWriteStrategyFactory<