🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { isRootObservation } from "../../eventsTable";
import { EventsQueryBuilder } from "../queries/clickhouse-sql/event-query-builder";
import {
queryClickhouse,
queryClickhouseStream,
} from "../repositories/clickhouse";
import { convertDateToClickhouseDateTime } from "../clickhouse/client";
import { assembleTranscript } from "../transcript/transcript";
import { orderObservations } from "../transcript/ordering";
import type { TranscriptObservation } from "../transcript/types";
/** Historical summaries are reusable only with the same transcript producer. */
export const TOPICS_TRANSCRIPT_VERSION = "shared-transcript-v3";
const MAX_OBSERVATIONS = 2_000;
const MAX_SNAPSHOT_BYTES = 10 * 1024 * 1024;
/** The trace is above the transcript limits, whether ClickHouse or the reader stopped it. */
export class TopicTraceTooLarge extends Error {
readonly name = "TopicTraceTooLarge";
constructor() {
super("Trace exceeds the Topics PoC limit (2,000 observations or 10 MiB)");
}
}
type SnapshotRow = {
project_id: string;
trace_id: string;
session_id: string;
environment: string;
trace_name: string;
span_id: string;
parent_span_id: string | null;
is_app_root: boolean;
start_time: string;
end_time: string | null;
type: TranscriptObservation["type"];
name: string;
input: string;
output: string;
metadata: Record<string, string>;
};
const utcTimestamp = (value: string) =>
value.includes("T") ? value : `${value.replace(" ", "T")}Z`;
/** Load a complete project-scoped v4 snapshot and assemble its shared transcript. */
export async function loadTopicTranscript(params: {
projectId: string;
traceId: string;
}) {
const { projectId, traceId } = params;
// ID-only requests first discover source bounds without reading transcript payloads.
const boundsQuery = new EventsQueryBuilder({ projectId })
.selectRaw(
"count() AS count",
"min(toUnixTimestamp64Milli(e.start_time)) AS minStart",
"max(toUnixTimestamp64Milli(e.start_time)) AS maxStart",
)
.forceFullTable()
.whereRaw("e.trace_id = {traceId: String}", { traceId })
.whereRaw("xxHash32(e.trace_id) = xxHash32({traceId: String})", { traceId })
.buildWithParams();
const [bounds] = await queryClickhouse<{
count: string;
minStart: string;
maxStart: string;
}>({
...boundsQuery,
preferredClickhouseService: "EventsReadOnly",
tags: { projectId },
clickhouseSettings: { max_threads: 2, max_execution_time: 30 },
});
if (!bounds || Number(bounds.count) === 0)
throw new Error(
"Trace not found in this project's v4 events; legacy-only traces are not supported by this PoC",
);
const builder = new EventsQueryBuilder({ projectId })
.selectRaw(
"e.project_id",
"e.trace_id",
"e.session_id",
"e.environment",
"e.trace_name",
"e.is_app_root",
"e.span_id",
"e.parent_span_id",
"e.start_time",
"e.end_time",
"e.type",
"e.name",
)
.selectIO(false)
.selectMetadataExpanded()
.forceFullTable()
.whereRaw("e.trace_id = {traceId: String}", { traceId })
.whereRaw("xxHash32(e.trace_id) = xxHash32({traceId: String})", { traceId })
.whereRaw(
"e.start_time >= {from: DateTime64(3)} AND e.start_time < {to: DateTime64(3)}",
{
from: convertDateToClickhouseDateTime(
new Date(Number(bounds.minStart)),
),
to: convertDateToClickhouseDateTime(
new Date(Number(bounds.maxStart) + 1),
),
},
)
.orderByColumns([
{ column: "e.event_ts", direction: "DESC" },
/** Use the same environment tie-breakers as Topics root discovery. */
{ column: "e.span_id", direction: "DESC" },
{ column: "e.environment", direction: "DESC" },
// Equal storage timestamps need a stable choice; they do not establish causal order.
{
column:
"cityHash64(tuple(e.environment, e.trace_name, e.is_app_root, e.parent_span_id, e.start_time, e.end_time, e.type, e.name, e.level, e.status_message, e.input, e.output, e.metadata_names, e.metadata_values))",
direction: "DESC",
},
])
.limitBy("e.span_id", "e.project_id")
.limit(MAX_OBSERVATIONS + 1);
const { query, params: queryParams } = builder.buildWithParams();
const observations: TranscriptObservation[] = [];
let sessionId: string | null = null;
let environment = "";
let traceName = "";
let rootName = "";
let bytes = 0;
try {
for await (const row of queryClickhouseStream<SnapshotRow>({
query,
params: queryParams,
preferredClickhouseService: "EventsReadOnly",
tags: { projectId },
clickhouseSettings: {
max_threads: 2,
max_execution_time: 30,
timeout_overflow_mode: "throw",
max_result_bytes: String(MAX_SNAPSHOT_BYTES),
result_overflow_mode: "throw",
},
})) {
if (row.project_id !== projectId || row.trace_id !== traceId)
throw new Error("Trace snapshot scope mismatch");
if (!observations.length) environment = row.environment;
// Rows arrive newest first; retain the latest non-empty session and name metadata.
if (!sessionId && row.session_id) sessionId = row.session_id;
if (!traceName && row.trace_name) traceName = row.trace_name;
if (
!rootName &&
row.name &&
isRootObservation({
parentObservationId: row.parent_span_id,
isAppRoot: row.is_app_root,
})
)
rootName = row.name;
bytes += Buffer.byteLength(JSON.stringify(row));
if (bytes > MAX_SNAPSHOT_BYTES || observations.length >= MAX_OBSERVATIONS)
throw new TopicTraceTooLarge();
observations.push({
id: row.span_id,
traceId,
parentObservationId: row.parent_span_id || null,
startTime: new Date(utcTimestamp(row.start_time)),
endTime:
row.end_time === null ? null : new Date(utcTimestamp(row.end_time)),
type: row.type,
name: row.name,
input: row.input,
output: row.output,
metadata: row.metadata,
});
}
} catch (error) {
if (isResultOverflow(error)) throw new TopicTraceTooLarge();
throw error;
}
if (!observations.length)
throw new Error(
"Trace not found in this project's v4 events; legacy-only traces are not supported by this PoC",
);
if (new Set(observations.map(({ id }) => id)).size !== observations.length)
throw new Error("Trace snapshot contains duplicate observation IDs");
// Storage order determines metadata freshness; transcript ties use stable IDs.
observations.sort((a, b) => {
const timeDifference = a.startTime.getTime() - b.startTime.getTime();
if (timeDifference) return timeDifference;
if (a.id === b.id) return 0;
return a.id < b.id ? -1 : 1;
});
return {
unitStartTime: observations[0].startTime.toISOString(),
sessionId,
environment,
traceName: traceName || rootName,
// Uncapped, like trace-batch; the summary input limit rejects oversized traces.
transcript: assembleTranscript(orderObservations(observations)),
};
}
/** ClickHouse stops oversized results before the reader can count them. */
function isResultOverflow(error: unknown): boolean {
if (typeof error !== "object" || error === null) return false;
return (
("type" in error && error.type === "TOO_MANY_ROWS_OR_BYTES") ||
(error instanceof Error &&
error.message.includes("Limit for result exceeded"))
);
}