Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

7.4 KB
Raw Blame
import { isRootObservation } from "../../eventsTable";
import { EventsQueryBuilder } from "../queries/clickhouse-sql/event-query-builder";
import {
  queryClickhouse,
  queryClickhouseStream,
} from "../repositories/clickhouse";
import { convertDateToClickhouseDateTime } from "../clickhouse/client";
import { assembleTranscript } from "../transcript/transcript";
import { orderObservations } from "../transcript/ordering";
import type { TranscriptObservation } from "../transcript/types";

/** Historical summaries are reusable only with the same transcript producer. */
export const TOPICS_TRANSCRIPT_VERSION = "shared-transcript-v3";

const MAX_OBSERVATIONS = 2_000;
const MAX_SNAPSHOT_BYTES = 10 * 1024 * 1024;

/** The trace is above the transcript limits, whether ClickHouse or the reader stopped it. */
export class TopicTraceTooLarge extends Error {
  readonly name = "TopicTraceTooLarge";
  constructor() {
    super("Trace exceeds the Topics PoC limit (2,000 observations or 10 MiB)");
  }
}

type SnapshotRow = {
  project_id: string;
  trace_id: string;
  session_id: string;
  environment: string;
  trace_name: string;
  span_id: string;
  parent_span_id: string | null;
  is_app_root: boolean;
  start_time: string;
  end_time: string | null;
  type: TranscriptObservation["type"];
  name: string;
  input: string;
  output: string;
  metadata: Record<string, string>;
};

const utcTimestamp = (value: string) =>
  value.includes("T") ? value : `${value.replace(" ", "T")}Z`;

/** Load a complete project-scoped v4 snapshot and assemble its shared transcript. */
export async function loadTopicTranscript(params: {
  projectId: string;
  traceId: string;
}) {
  const { projectId, traceId } = params;
  // ID-only requests first discover source bounds without reading transcript payloads.
  const boundsQuery = new EventsQueryBuilder({ projectId })
    .selectRaw(
      "count() AS count",
      "min(toUnixTimestamp64Milli(e.start_time)) AS minStart",
      "max(toUnixTimestamp64Milli(e.start_time)) AS maxStart",
    )
    .forceFullTable()
    .whereRaw("e.trace_id = {traceId: String}", { traceId })
    .whereRaw("xxHash32(e.trace_id) = xxHash32({traceId: String})", { traceId })
    .buildWithParams();
  const [bounds] = await queryClickhouse<{
    count: string;
    minStart: string;
    maxStart: string;
  }>({
    ...boundsQuery,
    preferredClickhouseService: "EventsReadOnly",
    tags: { projectId },
    clickhouseSettings: { max_threads: 2, max_execution_time: 30 },
  });
  if (!bounds || Number(bounds.count) === 0)
    throw new Error(
      "Trace not found in this project's v4 events; legacy-only traces are not supported by this PoC",
    );
  const builder = new EventsQueryBuilder({ projectId })
    .selectRaw(
      "e.project_id",
      "e.trace_id",
      "e.session_id",
      "e.environment",
      "e.trace_name",
      "e.is_app_root",
      "e.span_id",
      "e.parent_span_id",
      "e.start_time",
      "e.end_time",
      "e.type",
      "e.name",
    )
    .selectIO(false)
    .selectMetadataExpanded()
    .forceFullTable()
    .whereRaw("e.trace_id = {traceId: String}", { traceId })
    .whereRaw("xxHash32(e.trace_id) = xxHash32({traceId: String})", { traceId })
    .whereRaw(
      "e.start_time >= {from: DateTime64(3)} AND e.start_time < {to: DateTime64(3)}",
      {
        from: convertDateToClickhouseDateTime(
          new Date(Number(bounds.minStart)),
        ),
        to: convertDateToClickhouseDateTime(
          new Date(Number(bounds.maxStart) + 1),
        ),
      },
    )
    .orderByColumns([
      { column: "e.event_ts", direction: "DESC" },
      /** Use the same environment tie-breakers as Topics root discovery. */
      { column: "e.span_id", direction: "DESC" },
      { column: "e.environment", direction: "DESC" },
      // Equal storage timestamps need a stable choice; they do not establish causal order.
      {
        column:
          "cityHash64(tuple(e.environment, e.trace_name, e.is_app_root, e.parent_span_id, e.start_time, e.end_time, e.type, e.name, e.level, e.status_message, e.input, e.output, e.metadata_names, e.metadata_values))",
        direction: "DESC",
      },
    ])
    .limitBy("e.span_id", "e.project_id")
    .limit(MAX_OBSERVATIONS + 1);
  const { query, params: queryParams } = builder.buildWithParams();
  const observations: TranscriptObservation[] = [];
  let sessionId: string | null = null;
  let environment = "";
  let traceName = "";
  let rootName = "";
  let bytes = 0;
  try {
    for await (const row of queryClickhouseStream<SnapshotRow>({
      query,
      params: queryParams,
      preferredClickhouseService: "EventsReadOnly",
      tags: { projectId },
      clickhouseSettings: {
        max_threads: 2,
        max_execution_time: 30,
        timeout_overflow_mode: "throw",
        max_result_bytes: String(MAX_SNAPSHOT_BYTES),
        result_overflow_mode: "throw",
      },
    })) {
      if (row.project_id !== projectId || row.trace_id !== traceId)
        throw new Error("Trace snapshot scope mismatch");
      if (!observations.length) environment = row.environment;
      // Rows arrive newest first; retain the latest non-empty session and name metadata.
      if (!sessionId && row.session_id) sessionId = row.session_id;
      if (!traceName && row.trace_name) traceName = row.trace_name;
      if (
        !rootName &&
        row.name &&
        isRootObservation({
          parentObservationId: row.parent_span_id,
          isAppRoot: row.is_app_root,
        })
      )
        rootName = row.name;
      bytes += Buffer.byteLength(JSON.stringify(row));
      if (bytes > MAX_SNAPSHOT_BYTES || observations.length >= MAX_OBSERVATIONS)
        throw new TopicTraceTooLarge();
      observations.push({
        id: row.span_id,
        traceId,
        parentObservationId: row.parent_span_id || null,
        startTime: new Date(utcTimestamp(row.start_time)),
        endTime:
          row.end_time === null ? null : new Date(utcTimestamp(row.end_time)),
        type: row.type,
        name: row.name,
        input: row.input,
        output: row.output,
        metadata: row.metadata,
      });
    }
  } catch (error) {
    if (isResultOverflow(error)) throw new TopicTraceTooLarge();
    throw error;
  }
  if (!observations.length)
    throw new Error(
      "Trace not found in this project's v4 events; legacy-only traces are not supported by this PoC",
    );
  if (new Set(observations.map(({ id }) => id)).size !== observations.length)
    throw new Error("Trace snapshot contains duplicate observation IDs");
  // Storage order determines metadata freshness; transcript ties use stable IDs.
  observations.sort((a, b) => {
    const timeDifference = a.startTime.getTime() - b.startTime.getTime();
    if (timeDifference) return timeDifference;
    if (a.id === b.id) return 0;
    return a.id < b.id ? -1 : 1;
  });
  return {
    unitStartTime: observations[0].startTime.toISOString(),
    sessionId,
    environment,
    traceName: traceName || rootName,
    // Uncapped, like trace-batch; the summary input limit rejects oversized traces.
    transcript: assembleTranscript(orderObservations(observations)),
  };
}

/** ClickHouse stops oversized results before the reader can count them. */
function isResultOverflow(error: unknown): boolean {
  if (typeof error !== "object" || error === null) return false;
  return (
    ("type" in error && error.type === "TOO_MANY_ROWS_OR_BYTES") ||
    (error instanceof Error &&
      error.message.includes("Limit for result exceeded"))
  );
}