Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

35.8 KB
Raw Blame
import {
  clickhouseClient,
  convertDateToClickhouseDateTime,
} from "../clickhouse/client";
import { buildClickHouseLogComment } from "../clickhouse/queryTags";
import {
  queryClickhouse,
  queryClickhouseStream,
} from "../repositories/clickhouse";
import type {
  TopicAssignment,
  TopicDefinition,
  TopicEmbeddingConfig,
  TopicFacetRef,
  TopicSummary,
  TopicTimeRange,
} from "../../topics";
import { topicSourceSchema, topicTimeRangeSchema } from "../../topics";
import { prisma } from "../../db";
import { logger } from "../logger";
import { traceException } from "../instrumentation";
import { redis } from "../redis/redis";
import { chunk } from "lodash";
import { Readable } from "node:stream";
import { finished } from "node:stream/promises";
import {
  INTERNAL_EVAL_ENVIRONMENT_PREFIX,
  isInternalEvalEnvironment,
  PUBLIC_LLM_JUDGE_ENVIRONMENT,
} from "../llm/isInternalEvalEnvironment";

const sourceKeySql =
  "project_id, facet_id, facet_version, trace_id, if(trace_id = '', session_id, '')";

const unitTimeFilterSql =
  "unit_start_time >= {from:DateTime64(3)} AND unit_start_time < {to:DateTime64(3)}";

const summaryVersionKeySql =
  "facet_id, facet_version, trace_id, if(trace_id = '', session_id, ''), unit_start_time, processed_at";
const summarySourceKeySql =
  "facet_id, trace_id, if(trace_id = '', session_id, '')";

// Payloads are time-bounded; source-index metadata must include versions that moved
// outside that window. State, model and dimension filters apply after this check.
// Candidate keys scope the metadata scan; a scalar facet_version predicate there
// prevents ClickHouse from selecting the latest_source_versions projection.
function latestSummaryFilterSql(scopeSql: string, latestKeySql = sourceKeySql) {
  return `(${summaryVersionKeySql}) IN (
    SELECT ${summaryVersionKeySql} FROM topic_facet_summaries
    WHERE project_id = {projectId:String}
      AND (${summarySourceKeySql}) IN (
        SELECT ${summarySourceKeySql} FROM topic_facet_summaries
        WHERE project_id = {projectId:String} AND ${unitTimeFilterSql} AND ${scopeSql}
        GROUP BY ${summarySourceKeySql}
      )
    ORDER BY facet_version DESC, processed_at DESC
    LIMIT 1 BY ${latestKeySql}
  )`;
}

function timeRangeParams(timeRange: TopicTimeRange) {
  const { from, to } = topicTimeRangeSchema.parse(timeRange);
  return {
    from: convertDateToClickhouseDateTime(from),
    to: convertDateToClickhouseDateTime(to),
  };
}

const LOOKUP_BATCH_SIZE = 1000;
const INSERT_BATCH_SIZE = 10_000;
const INSERT_BATCH_BYTES = 8 * 1024 * 1024;

const sourceFilterSql =
  "(trace_id, if(trace_id = '', session_id, '')) IN arrayZip({traceIds:Array(String)}, {sessionIds:Array(String)})";

function sourceBatches(sources: Pick<TopicSummary, "traceId" | "sessionId">[]) {
  const keys = new Map<string, { traceId: string; sessionId: string }>();
  for (const value of sources) {
    const source = topicSourceSchema.parse(value);
    const traceId = source.traceId ?? "";
    const sessionId = source.traceId === null ? source.sessionId : "";
    keys.set(JSON.stringify([traceId, sessionId]), { traceId, sessionId });
  }
  return chunk([...keys.values()], LOOKUP_BATCH_SIZE).map((batch) => ({
    traceIds: batch.map((source) => source.traceId),
    sessionIds: batch.map((source) => source.sessionId),
  }));
}

async function insertTopicRows<T extends { projectId: string }>(
  table: "topic_facet_summaries" | "topic_assignments",
  rows: T[],
  serialize: (row: T) => Record<string, unknown>,
): Promise<void> {
  let values: Record<string, unknown>[] = [];
  let bytes = 0;
  const flush = async () => {
    if (!values.length) return;
    await clickhouseClient().insert({
      table,
      format: "JSONEachRow",
      values,
      clickhouse_settings: {
        async_insert: 1,
        wait_for_async_insert: 1,
        log_comment: buildClickHouseLogComment({
          surface: "worker",
          route: {
            topic_facet_summaries: "topics-summaries",
            topic_assignments: "topics-assignments",
          }[table],
          projectId: rows[0].projectId,
        }),
      },
    });
    values = [];
    bytes = 0;
  };
  for (const row of rows) {
    const value = serialize(row);
    const size = Buffer.byteLength(JSON.stringify(value), "utf8") + 1;
    if (size > INSERT_BATCH_BYTES)
      throw new Error("A Topics result exceeds the maximum insert row size.");
    if (values.length >= INSERT_BATCH_SIZE || bytes + size > INSERT_BATCH_BYTES)
      await flush();
    values.push(value);
    bytes += size;
  }
  await flush();
}

export async function getTopicDefinitions(
  projectId: string,
  topicVersionIds: string[],
  timeRange: TopicTimeRange,
): Promise<TopicDefinition[]> {
  const definitions: TopicDefinition[] = [];
  for (const ids of chunk([...new Set(topicVersionIds)], LOOKUP_BATCH_SIZE)) {
    const rows = await queryClickhouse<
      Omit<
        TopicDefinition,
        "createdAt" | "metadata" | "representativeSummaries"
      > & {
        createdAtMs: string;
        metadataJson: string;
        representativeSummaries: {
          facet_id: string;
          facet_version: number;
          trace_id: string;
          session_id: string;
        }[];
      }
    >({
      query: `SELECT project_id AS projectId, id AS topicVersionId,
        stable_id AS topicId, created_by_run_id AS createdByRunId,
        toUnixTimestamp64Milli(created_at) AS createdAtMs, name, description,
        centroid, radius, tags, representative_summaries AS representativeSummaries,
        metadata AS metadataJson
        FROM topics
        WHERE project_id = {projectId:String} AND id IN {ids:Array(String)}
          AND created_at >= {from:DateTime64(3)} AND created_at < {to:DateTime64(3)}
        ORDER BY created_at DESC LIMIT 1 BY project_id, id`,
      params: { projectId, ids, ...timeRangeParams(timeRange) },
      tags: { route: "topics-definitions", projectId },
    });
    definitions.push(
      ...rows.map(({ createdAtMs, metadataJson, ...row }) => ({
        ...row,
        createdAt: new Date(Number(createdAtMs)).toISOString(),
        representativeSummaries: row.representativeSummaries.map((source) => ({
          facetId: source.facet_id,
          facetVersion: source.facet_version,
          traceId: source.trace_id || null,
          sessionId: source.session_id || null,
        })),
        metadata: JSON.parse(metadataJson) as Record<string, unknown>,
      })),
    );
  }
  return definitions;
}

/** RowBinary column order matches the explicit definition INSERT below. */
function encodeTopicDefinition(row: TopicDefinition): Buffer {
  const parts: Buffer[] = [];
  const length = (value: number) => {
    const bytes: number[] = [];
    do {
      const byte = value % 128;
      value = Math.floor(value / 128);
      bytes.push(byte + (value ? 128 : 0));
    } while (value);
    parts.push(Buffer.from(bytes));
  };
  const string = (value: string) => {
    const bytes = Buffer.from(value, "utf8");
    length(bytes.length);
    parts.push(bytes);
  };
  const strings = (values: string[]) => {
    length(values.length);
    for (const value of values) string(value);
  };

  string(row.projectId);
  string(row.topicVersionId);
  string(row.topicId);
  string(row.createdByRunId);
  const createdAt = Buffer.allocUnsafe(8);
  createdAt.writeBigInt64LE(BigInt(Date.parse(row.createdAt)));
  parts.push(createdAt);
  string(row.name);
  string(row.description);
  length(row.centroid.length);
  const centroid = Buffer.allocUnsafe(row.centroid.length * 8);
  for (let index = 0; index < row.centroid.length; index++)
    centroid.writeDoubleLE(row.centroid[index], index * 8);
  parts.push(centroid);
  const radius = Buffer.allocUnsafe(8);
  radius.writeDoubleLE(row.radius);
  parts.push(radius);
  strings(row.tags);
  length(row.representativeSummaries.length);
  for (const source of row.representativeSummaries) {
    string(source.facetId);
    const facetVersion = Buffer.allocUnsafe(4);
    facetVersion.writeUInt32LE(source.facetVersion);
    parts.push(facetVersion);
    string(source.traceId ?? "");
    string(source.traceId === null ? source.sessionId! : "");
  }
  string(JSON.stringify(row.metadata));
  return Buffer.concat(parts);
}

export async function writeTopicDefinitions(
  rows: TopicDefinition[],
): Promise<void> {
  for (const row of rows) {
    if (
      !row.projectId ||
      !row.topicVersionId ||
      !row.topicId ||
      !row.createdByRunId ||
      !Number.isFinite(Date.parse(row.createdAt)) ||
      !row.centroid.length ||
      !Number.isFinite(row.radius) ||
      row.radius < 0
    )
      throw new Error("Invalid topic definition.");
    for (const value of row.centroid)
      if (!Number.isFinite(value)) throw new Error("Invalid topic definition.");
  }
  let values: Buffer[] = [];
  let bytes = 0;
  const flush = async () => {
    if (!values.length) return;
    const result = await clickhouseClient().exec({
      query: `INSERT INTO topics (project_id, id, stable_id, created_by_run_id,
        created_at, name, description, centroid, radius, tags,
        representative_summaries, metadata) FORMAT RowBinary`,
      values: Readable.from(values),
      clickhouse_settings: {
        async_insert: 1,
        wait_for_async_insert: 1,
        log_comment: buildClickHouseLogComment({
          surface: "worker",
          route: "topics-definitions",
          projectId: rows[0].projectId,
        }),
      },
    });
    await finished(result.stream.resume(), { cleanup: true });
    values = [];
    bytes = 0;
  };
  for (const row of rows) {
    const value = encodeTopicDefinition(row);
    if (value.length > INSERT_BATCH_BYTES)
      throw new Error("A Topics result exceeds the maximum insert row size.");
    if (
      values.length >= INSERT_BATCH_SIZE ||
      bytes + value.length > INSERT_BATCH_BYTES
    )
      await flush();
    values.push(value);
    bytes += value.length;
  }
  await flush();
}

const summaryColumns = `project_id AS projectId, facet_id AS facetId,
  facet_version AS facetVersion,
  trace_id AS traceId, session_id AS sessionId, trigger_type AS triggerType,
  environment, trace_name AS traceName,
  toUnixTimestamp64Milli(unit_start_time) AS unitStartTimeMs,
  processing_state AS state, summary, embedding,
  transcript_id AS transcriptId, transcript_version AS transcriptVersion,
  summary_model AS summaryModel, embedding_model AS embeddingModel,
  provided_usage_details AS providedUsageDetails, usage_details AS usageDetails,
  provided_cost_details AS providedCostDetails, cost_details AS costDetails,
  toUnixTimestamp64Milli(processed_at) AS processedAtMs, metadata AS metadataJson`;

type SummaryRow = Omit<
  TopicSummary,
  "metadata" | "unitStartTime" | "processedAt"
> & { metadataJson: string; unitStartTimeMs: string; processedAtMs: string };
const numberMap = (values: Record<string, number>) =>
  Object.fromEntries(
    Object.entries(values).map(([key, value]) => [key, Number(value)]),
  );
function summarySourceAndTimestamps(
  row: Pick<
    SummaryRow,
    "traceId" | "sessionId" | "unitStartTimeMs" | "processedAtMs"
  >,
) {
  return {
    ...topicSourceSchema.parse({
      traceId: row.traceId || null,
      sessionId: row.sessionId || null,
    }),
    unitStartTime: new Date(Number(row.unitStartTimeMs)).toISOString(),
    processedAt: new Date(Number(row.processedAtMs)).toISOString(),
  };
}

function summaryResult(row: SummaryRow): TopicSummary {
  const { metadataJson, unitStartTimeMs, processedAtMs, ...rest } = row;
  return {
    ...rest,
    ...summarySourceAndTimestamps(row),
    providedUsageDetails: numberMap(row.providedUsageDetails),
    usageDetails: numberMap(row.usageDetails),
    providedCostDetails: numberMap(row.providedCostDetails),
    costDetails: numberMap(row.costDetails),
    metadata: JSON.parse(metadataJson),
  };
}

export async function listTopicSummaries(
  projectId: string,
  filter: (
    | {
        sources: Pick<TopicSummary, "traceId" | "sessionId">[];
        traceIds?: never;
      }
    | { traceIds: string[]; sources?: never }
  ) & { facetId: string; facetVersion: number },
  timeRange: TopicTimeRange,
): Promise<TopicSummary[]> {
  const rows: SummaryRow[] = [];
  const scopeSql = `facet_id = {facetId:String} AND facet_version = {facetVersion:UInt32}
    AND ${sourceFilterSql}`;
  const sources =
    filter.sources ??
    filter.traceIds.map((traceId) => ({ traceId, sessionId: null }));
  for (const batch of sourceBatches(sources)) {
    rows.push(
      ...(await queryClickhouse<SummaryRow>({
        query: `SELECT ${summaryColumns} FROM topic_facet_summaries
      WHERE project_id = {projectId:String} AND ${unitTimeFilterSql}
        AND ${scopeSql} AND ${latestSummaryFilterSql(scopeSql)}
      ORDER BY processed_at DESC LIMIT 1 BY ${sourceKeySql}`,
        params: {
          projectId,
          ...timeRangeParams(timeRange),
          facetId: filter.facetId,
          facetVersion: filter.facetVersion,
          ...batch,
        },
        tags: { route: "topics-summaries", projectId },
      })),
    );
  }
  rows.sort((a, b) => Number(b.processedAtMs) - Number(a.processedAtMs));
  return rows
    .filter((row) => !isInternalEvalEnvironment(row.environment))
    .map(summaryResult);
}
/** Display fields only. Without a version filter, use each source's newest processed facet version. */
export async function getLatestFacetSummaries(
  projectId: string,
  facetId: string,
  facetVersion: number | undefined,
  timeRange: TopicTimeRange,
  traceIds?: string[],
) {
  const scopeSql = `facet_id = {facetId:String}
    ${facetVersion ? "AND facet_version = {facetVersion:UInt32}" : ""}
    ${traceIds ? "AND trace_id IN {traceIds:Array(String)}" : ""}`;
  const rows = await queryClickhouse<
    Pick<
      TopicSummary,
      | "projectId"
      | "facetId"
      | "facetVersion"
      | "traceId"
      | "sessionId"
      | "state"
      | "summary"
      | "environment"
    > & { unitStartTimeMs: string; processedAtMs: string }
  >({
    query: `SELECT project_id AS projectId, facet_id AS facetId,
      facet_version AS facetVersion, trace_id AS traceId, session_id AS sessionId,
      processing_state AS state, summary, environment,
      toUnixTimestamp64Milli(unit_start_time) AS unitStartTimeMs,
      toUnixTimestamp64Milli(processed_at) AS processedAtMs
      FROM topic_facet_summaries
      WHERE project_id = {projectId:String} AND ${unitTimeFilterSql} AND ${scopeSql}
        AND ${latestSummaryFilterSql(scopeSql, facetVersion ? sourceKeySql : summarySourceKeySql)}
      ORDER BY ${facetVersion ? "" : "facet_version DESC, "}processed_at DESC
      LIMIT 1 BY ${facetVersion ? sourceKeySql : "project_id, trace_id, if(trace_id = '', session_id, '')"}`,
    params: {
      projectId,
      ...timeRangeParams(timeRange),
      facetId,
      ...(facetVersion ? { facetVersion } : {}),
      ...(traceIds ? { traceIds } : {}),
    },
    tags: { route: "topics-latest-summaries", projectId },
  });
  return rows
    .filter((row) => !isInternalEvalEnvironment(row.environment))
    .map((row) => ({
      projectId: row.projectId,
      facetId: row.facetId,
      facetVersion: row.facetVersion,
      state: row.state,
      summary: row.summary,
      ...summarySourceAndTimestamps(row),
    }));
}

const clusteringSummaryScopeSql = `trace_id != '' AND facet_id = {facetId:String}
  AND facet_version = {facetVersion:UInt32}`;

function clusteringSummariesSql(columns: string, rowColumns = "*") {
  return `SELECT ${columns} FROM (
      SELECT ${rowColumns}, length(embedding) AS dimensions FROM topic_facet_summaries
      WHERE project_id = {projectId:String} AND ${unitTimeFilterSql} AND ${clusteringSummaryScopeSql}
        AND ${latestSummaryFilterSql(clusteringSummaryScopeSql)}
      ORDER BY processed_at DESC
      LIMIT 1 BY ${sourceKeySql}
    ) WHERE processing_state = 'complete' AND embedding_model = {embeddingModel:String}
      AND dimensions = {embeddingDimensions:UInt32}
      AND NOT startsWith(environment, {internalEnvironmentPrefix:String})
      AND environment != {publicLlmJudgeEnvironment:String}`;
}

/** Returns aggregate admission estimates; summary payloads stay in ClickHouse. */
export async function getTopicClusteringSummaryStats(
  projectId: string,
  facetId: string,
  facetVersion: number,
  embeddingConfig: TopicEmbeddingConfig,
  timeRange: TopicTimeRange,
  limits: { maxRows: number; maxBytes: number },
): Promise<{ count: number; bytes: number }> {
  const [{ count }] = await getTopicSummaryCounts(
    projectId,
    [{ facetId, version: facetVersion }],
    embeddingConfig,
    timeRange,
  );
  const vectorBytes = count * embeddingConfig.embeddingDimensions * 8;
  // Narrow oversized windows using metadata before inspecting their text payloads.
  if (!count || count > limits.maxRows || vectorBytes > limits.maxBytes)
    return { count, bytes: vectorBytes };
  // UTF-8 JSON bytes bound UTF-16 code units, including escaping and map keys.
  // Allow for field names and row overhead; this is not a total RSS estimate.
  const estimatedBytesSql = `length(embedding) * 8 + 2 * length(toJSONString(tuple(
    project_id, facet_id, facet_version, trace_id, session_id, trigger_type,
    environment, trace_name, toUnixTimestamp64Milli(unit_start_time),
    processing_state, summary, transcript_id, transcript_version, summary_model,
    embedding_model, provided_usage_details, usage_details,
    provided_cost_details, cost_details, toUnixTimestamp64Milli(processed_at), metadata
  ))) + 2048`;
  const [row] = await queryClickhouse<{ count: string; bytes: string }>({
    query: clusteringSummariesSql(
      "count() AS count, sum(estimated_bytes) AS bytes",
      `processing_state, embedding_model, environment, ${estimatedBytesSql} AS estimated_bytes`,
    ),
    params: {
      projectId,
      facetId,
      facetVersion,
      ...embeddingConfig,
      ...timeRangeParams(timeRange),
      internalEnvironmentPrefix: INTERNAL_EVAL_ENVIRONMENT_PREFIX,
      publicLlmJudgeEnvironment: PUBLIC_LLM_JUDGE_ENVIRONMENT,
    },
    tags: { route: "topics-clustering-summary-stats", projectId },
  });
  return { count: Number(row?.count ?? 0), bytes: Number(row?.bytes ?? 0) };
}

export const TOPIC_CLUSTERING_CAPACITY_ERROR =
  "Topics clustering exceeds this worker's capacity. Choose a smaller time window; no partial cohort was used.";

export async function getTopicClusteringSummaries(
  projectId: string,
  facetId: string,
  facetVersion: number,
  embeddingConfig: TopicEmbeddingConfig,
  timeRange: TopicTimeRange,
  limits: { maxRows: number; maxBytes: number },
): Promise<TopicSummary[]> {
  const capacityError = () => new Error(TOPIC_CLUSTERING_CAPACITY_ERROR);
  if (
    !Number.isSafeInteger(limits.maxRows) ||
    limits.maxRows < 1 ||
    limits.maxRows >= Number.MAX_SAFE_INTEGER ||
    !Number.isSafeInteger(limits.maxBytes) ||
    limits.maxBytes < 1
  )
    throw new Error("Topics clustering limits must be positive safe integers.");
  const [{ count }] = await getTopicSummaryCounts(
    projectId,
    [{ facetId, version: facetVersion }],
    embeddingConfig,
    timeRange,
  );
  if (
    count > limits.maxRows ||
    count * embeddingConfig.embeddingDimensions * 8 > limits.maxBytes
  )
    throw capacityError();

  const summaries: TopicSummary[] = [];
  let bytes = 0;
  try {
    for await (const row of queryClickhouseStream<SummaryRow>({
      query: `${clusteringSummariesSql(summaryColumns)}
        ORDER BY trace_id LIMIT {maximumRows:UInt64}`,
      params: {
        projectId,
        facetId,
        facetVersion,
        ...embeddingConfig,
        ...timeRangeParams(timeRange),
        internalEnvironmentPrefix: INTERNAL_EVAL_ENVIRONMENT_PREFIX,
        publicLlmJudgeEnvironment: PUBLIC_LLM_JUDGE_ENVIRONMENT,
        maximumRows: limits.maxRows + 1,
      },
      tags: { route: "topics-clustering-summaries", projectId },
      clickhouseSettings: {
        max_result_bytes: String(limits.maxBytes),
        result_overflow_mode: "throw",
      },
    })) {
      const { embedding, ...fields } = row;
      // Budget decoded doubles and UTF-16 text separately from transport bytes.
      // Object headers, parsed metadata, and numerical-fit copies need extra RAM.
      bytes += embedding.length * 8 + JSON.stringify(fields).length * 2 + 256;
      if (summaries.length >= limits.maxRows || bytes > limits.maxBytes)
        throw capacityError();
      summaries.push(summaryResult(row));
    }
  } catch (error) {
    if (
      error instanceof Error &&
      /TOO_MANY_ROWS_OR_BYTES|Limit for result exceeded/.test(error.message)
    )
      throw capacityError();
    throw error;
  }
  return summaries;
}

export async function hasTopicEmbeddings(projectId: string): Promise<boolean> {
  const rows = await queryClickhouse<{ found: number }>({
    query: `SELECT 1 AS found FROM topic_facet_summaries
      WHERE project_id = {projectId:String} AND processing_state = 'complete'
      LIMIT 1`,
    params: { projectId },
    tags: { route: "topics-has-embeddings", projectId },
  });
  return rows.length > 0;
}

/** Project onboarding readiness, cached for 30 minutes when ready and 30 seconds when empty. */
export async function hasStoredTopicSummaries(
  projectId: string,
): Promise<boolean> {
  const cacheKey = `langfuse:topics:has-summaries:${projectId}`;
  try {
    const cached = await redis?.get(cacheKey);
    if (cached === "1") return true;
    if (cached === "0") return false;
  } catch (error) {
    traceException(error);
    logger.warn("Failed to read cached Topics summary readiness", {
      projectId,
      error,
    });
  }
  const rows = await queryClickhouse<{ found: number }>({
    query: `SELECT 1 AS found FROM topic_facet_summaries
      WHERE project_id = {projectId:String}
        AND trace_id != '' AND processing_state = 'complete'
        AND NOT startsWith(environment, {internalEnvironmentPrefix:String})
        AND environment != {publicLlmJudgeEnvironment:String}
      LIMIT 1`,
    params: {
      projectId,
      internalEnvironmentPrefix: INTERNAL_EVAL_ENVIRONMENT_PREFIX,
      publicLlmJudgeEnvironment: PUBLIC_LLM_JUDGE_ENVIRONMENT,
    },
    tags: { route: "topics-has-summaries", projectId },
    clickhouseSettings: { max_threads: 1 },
  });
  const hasSummaries = rows.length > 0;
  try {
    await redis?.set(
      cacheKey,
      hasSummaries ? "1" : "0",
      "EX",
      hasSummaries ? 1800 : 30,
    );
  } catch (error) {
    traceException(error);
    logger.warn("Failed to cache Topics summary readiness", {
      projectId,
      error,
    });
  }
  return hasSummaries;
}

/** Approximate costs; retry inserts can overcount until background merges deduplicate them. */
export async function getTopicExecutionCosts(
  projectId: string,
  executionIds: string[],
): Promise<Record<string, number>> {
  if (executionIds.length === 0) return {};
  const rows = await queryClickhouse<{ executionId: string; cost: string }>({
    query: `SELECT JSONExtractString(metadata, 'executionId') AS executionId,
        sum(calculated_total_cost) AS cost
      FROM topic_facet_summaries
      WHERE project_id = {projectId:String}
        AND executionId IN {executionIds:Array(String)}
        AND notEmpty(cost_details)
      GROUP BY executionId`,
    params: { projectId, executionIds },
    tags: { route: "topics-execution-costs", projectId },
  }).catch((error: unknown) => {
    traceException(error);
    logger.warn("Failed to estimate Topics execution costs", {
      projectId,
      error,
    });
    return [];
  });
  return Object.fromEntries(
    rows.map((row) => [row.executionId, Number(row.cost)]),
  );
}

/** Count current compatible summaries without loading summary text or vectors. */
export async function getTopicSummaryCounts(
  projectId: string,
  facets: TopicFacetRef[],
  embeddingConfig: TopicEmbeddingConfig,
  timeRange: TopicTimeRange,
): Promise<{ facetId: string; facetVersion: number; count: number }[]> {
  if (!facets.length) return [];
  const scopeSql = `trace_id != ''
    AND (facet_id, facet_version) IN arrayZip({facetIds:Array(String)}, {facetVersions:Array(UInt32)})`;
  const rows = await queryClickhouse<{
    facetId: string;
    facetVersion: number;
    count: string;
  }>({
    query: `SELECT facet_id AS facetId, facet_version AS facetVersion, count() AS count FROM (
      SELECT facet_id, facet_version, processing_state, embedding_model, length(embedding) AS dimensions, environment
      FROM topic_facet_summaries
      WHERE project_id = {projectId:String} AND ${unitTimeFilterSql} AND ${scopeSql}
        AND ${latestSummaryFilterSql(scopeSql)}
      ORDER BY processed_at DESC
      LIMIT 1 BY ${sourceKeySql}
    ) WHERE processing_state = 'complete' AND embedding_model = {embeddingModel:String}
      AND dimensions = {embeddingDimensions:UInt32}
      AND NOT startsWith(environment, {internalEnvironmentPrefix:String})
      AND environment != {publicLlmJudgeEnvironment:String}
    GROUP BY facet_id, facet_version`,
    params: {
      projectId,
      ...timeRangeParams(timeRange),
      facetIds: facets.map(({ facetId }) => facetId),
      facetVersions: facets.map(({ version }) => version),
      ...embeddingConfig,
      internalEnvironmentPrefix: INTERNAL_EVAL_ENVIRONMENT_PREFIX,
      publicLlmJudgeEnvironment: PUBLIC_LLM_JUDGE_ENVIRONMENT,
    },
    tags: { route: "topics-summary-counts", projectId },
  });
  return facets.map(({ facetId, version }) => ({
    facetId,
    facetVersion: version,
    count: Number(
      rows.find(
        (row) => row.facetId === facetId && row.facetVersion === version,
      )?.count ?? 0,
    ),
  }));
}

/** writeTopicSummaries rejects the whole insert if any row fails this check. */
export function isValidTopicSummary(row: TopicSummary): boolean {
  return (
    topicSourceSchema.safeParse(row).success &&
    typeof row.environment === "string" &&
    row.embedding.every(Number.isFinite) &&
    !(row.state === "complete" && row.embedding.length === 0) &&
    row.state !== "summarized"
  );
}

export async function writeTopicSummaries(rows: TopicSummary[]): Promise<void> {
  if (!rows.length) return;
  if (!rows.every(isValidTopicSummary))
    throw new Error("Invalid Topics summary result.");
  await insertTopicRows("topic_facet_summaries", rows, topicSummaryRecord);
}

export async function writeTopicAssignments(
  rows: TopicAssignment[],
): Promise<void> {
  if (!rows.length) return;
  for (const row of rows) {
    if (
      !topicSourceSchema.safeParse(row).success ||
      typeof row.environment !== "string" ||
      (row.topicId === null) !== (row.topicVersionId === null) ||
      (row.distance !== null && !Number.isFinite(row.distance)) ||
      (row.runnerUpDistance !== null &&
        !Number.isFinite(row.runnerUpDistance)) ||
      (row.coordinates !== null &&
        (!row.runId ||
          row.coordinates.length !== 2 ||
          !row.coordinates.every(Number.isFinite)))
    )
      throw new Error("Invalid Topics assignment.");
  }
  await insertTopicRows("topic_assignments", rows, topicAssignmentRecord);
}

function topicSummaryRecord(row: TopicSummary) {
  return {
    project_id: row.projectId,
    facet_id: row.facetId,
    facet_version: row.facetVersion,
    trace_id: row.traceId ?? "",
    session_id: row.sessionId ?? "",
    trigger_type: row.triggerType,
    environment: row.environment,
    trace_name: row.traceId ? row.traceName : "",
    unit_start_time: convertDateToClickhouseDateTime(
      new Date(row.unitStartTime),
    ),
    processing_state: row.state,
    summary: row.summary,
    embedding: row.embedding,
    transcript_id: row.transcriptId,
    transcript_version: row.transcriptVersion,
    summary_model: row.summaryModel,
    embedding_model: row.embeddingModel,
    provided_usage_details: row.providedUsageDetails,
    usage_details: row.usageDetails,
    provided_cost_details: row.providedCostDetails,
    cost_details: row.costDetails,
    processed_at: convertDateToClickhouseDateTime(new Date(row.processedAt)),
    metadata: JSON.stringify(row.metadata),
  };
}

/** A `topic_facet_summaries` row as inserted with JSONEachRow. */
export type TopicFacetSummaryRecordInsertType = ReturnType<
  typeof topicSummaryRecord
>;

function topicAssignmentRecord(row: TopicAssignment) {
  return {
    project_id: row.projectId,
    facet_id: row.facetId,
    facet_version: row.facetVersion,
    trace_id: row.traceId ?? "",
    session_id: row.sessionId ?? "",
    unit_start_time: convertDateToClickhouseDateTime(
      new Date(row.unitStartTime),
    ),
    summary_processed_at: convertDateToClickhouseDateTime(
      new Date(row.summaryProcessedAt),
    ),
    environment: row.environment,
    trace_name: row.traceId ? row.traceName : "",
    clustering_run_id: row.runId ?? "",
    topic_id: row.topicId ?? "",
    topic_version_id: row.topicVersionId ?? "",
    distance: row.distance,
    runner_up_distance: row.runnerUpDistance,
    origin: row.origin,
    coordinates: row.coordinates ?? [],
    assigned_at: convertDateToClickhouseDateTime(new Date(row.assignedAt)),
  };
}

/** A `topic_assignments` row as inserted with JSONEachRow. */
export type TopicAssignmentRecordInsertType = ReturnType<
  typeof topicAssignmentRecord
>;

const assignmentColumns = `project_id AS projectId, facet_id AS facetId,
      facet_version AS facetVersion, trace_id AS traceId,
      session_id AS sessionId,
      environment, trace_name AS traceName,
      toUnixTimestamp64Milli(unit_start_time) AS unitStartTimeMs,
      coordinates,
      toUnixTimestamp64Milli(summary_processed_at) AS summaryProcessedAtMs,
      clustering_run_id AS runId,
      topic_id AS topicId, topic_version_id AS topicVersionId, distance,
      runner_up_distance AS runnerUpDistance, origin,
      toUnixTimestamp64Milli(assigned_at) AS assignedAtMs`;
type AssignmentRow = Omit<
  TopicAssignment,
  | "unitStartTime"
  | "assignedAt"
  | "summaryProcessedAt"
  | "runId"
  | "coordinates"
> & {
  unitStartTimeMs: string;
  summaryProcessedAtMs: string;
  assignedAtMs: string;
  runId: string;
  coordinates: number[];
};
function assignmentResult({
  unitStartTimeMs,
  summaryProcessedAtMs,
  assignedAtMs,
  ...row
}: AssignmentRow): TopicAssignment {
  return {
    ...row,
    ...topicSourceSchema.parse({
      traceId: row.traceId || null,
      sessionId: row.sessionId || null,
    }),
    runId: row.runId || null,
    topicId: row.topicId || null,
    topicVersionId: row.topicVersionId || null,
    coordinates:
      row.coordinates.length === 2
        ? [row.coordinates[0], row.coordinates[1]]
        : null,
    unitStartTime: new Date(Number(unitStartTimeMs)).toISOString(),
    summaryProcessedAt: new Date(Number(summaryProcessedAtMs)).toISOString(),
    assignedAt: new Date(Number(assignedAtMs)).toISOString(),
  };
}

/** A published map's cohort excludes later online assignments to the same map. */
export async function readTopicRunTraceIds(
  projectId: string,
  facet: TopicFacetRef,
  runId: string,
  timeRange: TopicTimeRange,
  maxRows: number,
): Promise<string[]> {
  if (
    !Number.isSafeInteger(maxRows) ||
    maxRows < 1 ||
    maxRows >= Number.MAX_SAFE_INTEGER
  )
    throw new Error("Topics continuity limit must be a positive safe integer.");
  const rows = await queryClickhouse<{ traceId: string }>({
    query: `SELECT DISTINCT trace_id AS traceId FROM topic_assignments
      WHERE project_id = {projectId:String} AND ${unitTimeFilterSql} AND clustering_run_id = {runId:String}
        AND facet_id = {facetId:String} AND facet_version = {facetVersion:UInt32}
        AND origin = 'initial'
        AND trace_id != ''
      ORDER BY traceId LIMIT {maximumRows:UInt64}`,
    params: {
      projectId,
      ...timeRangeParams(timeRange),
      runId,
      facetId: facet.facetId,
      facetVersion: facet.version,
      maximumRows: maxRows + 1,
    },
    tags: { route: "topics-run-summaries", projectId },
  });
  if (rows.length > maxRows)
    throw new Error(
      "Previous Topics map exceeds this worker's clustering capacity; its continuity cohort cannot be loaded.",
    );
  return rows.map((row) => row.traceId);
}

/** Initial membership includes rows with missing coordinates, excluding online assignments. */
export async function readTopicMapAssignments(
  projectId: string,
  facet: TopicFacetRef,
  runId: string,
  timeRange: TopicTimeRange,
): Promise<TopicAssignment[]> {
  const rows = await queryClickhouse<AssignmentRow>({
    query: `SELECT ${assignmentColumns} FROM topic_assignments
      WHERE project_id = {projectId:String} AND ${unitTimeFilterSql}
        AND facet_id = {facetId:String} AND facet_version = {facetVersion:UInt32}
        AND clustering_run_id = {runId:String}
        AND origin = 'initial' AND trace_id != ''
      ORDER BY assigned_at DESC
      LIMIT 1 BY ${sourceKeySql}`,
    params: {
      projectId,
      ...timeRangeParams(timeRange),
      runId,
      facetId: facet.facetId,
      facetVersion: facet.version,
    },
    tags: { route: "topics-map", projectId },
  });
  return rows.map(assignmentResult);
}

export async function readTopicAssignments(
  projectId: string,
  facet: TopicFacetRef,
  sources: Pick<TopicSummary, "traceId" | "sessionId">[],
  runId: string,
  timeRange: TopicTimeRange,
): Promise<TopicAssignment[]> {
  const rows: AssignmentRow[] = [];
  for (const batch of sourceBatches(sources)) {
    rows.push(
      ...(await queryClickhouse<AssignmentRow>({
        query: `SELECT ${assignmentColumns}
      FROM topic_assignments WHERE project_id = {projectId:String} AND ${unitTimeFilterSql}
        AND facet_id = {facetId:String} AND facet_version = {facetVersion:UInt32}
        AND clustering_run_id = {runId:String} AND ${sourceFilterSql}
      ORDER BY assigned_at DESC, origin DESC
      LIMIT 1 BY ${sourceKeySql}`,
        params: {
          projectId,
          ...timeRangeParams(timeRange),
          facetId: facet.facetId,
          facetVersion: facet.version,
          ...batch,
          runId,
        },
        tags: { route: "topics-assignments", projectId },
      })),
    );
  }
  return rows.map(assignmentResult);
}

/** Current membership prefers the newest facet version, including ad-hoc attempts. */
export async function readLatestTopicAssignments(
  projectId: string,
  facetId: string,
  timeRange: TopicTimeRange,
  traceIds?: string[],
): Promise<TopicAssignment[]> {
  const publishedRuns = await prisma.topicClusteringRun.findMany({
    where: {
      projectId,
      facetId,
      status: "completed",
    },
    select: { id: true },
  });
  const rows = await queryClickhouse<AssignmentRow>({
    query: `SELECT ${assignmentColumns} FROM topic_assignments
      WHERE project_id = {projectId:String} AND ${unitTimeFilterSql} AND facet_id = {facetId:String}
        AND (clustering_run_id = '' OR clustering_run_id IN ({publishedRunIds:Array(String)}))
        ${traceIds ? "AND trace_id IN {traceIds:Array(String)}" : ""}
      ORDER BY facet_version DESC, assigned_at DESC, clustering_run_id DESC, origin DESC
      LIMIT 1 BY project_id, facet_id, trace_id, if(trace_id = '', session_id, '')`,
    params: {
      projectId,
      ...timeRangeParams(timeRange),
      facetId,
      publishedRunIds: publishedRuns.map(({ id }) => id),
      ...(traceIds ? { traceIds } : {}),
    },
    tags: { route: "topics-current-assignments", projectId },
  });
  return rows.map(assignmentResult);
}

/** Source time span of one trace's summaries, served by the latest_source_versions projection; null when never summarized. */
export async function getTopicTraceUnitTimeRange(
  projectId: string,
  traceId: string,
  timeRange: TopicTimeRange,
): Promise<TopicTimeRange | null> {
  const [row] = await queryClickhouse<{
    found: string;
    fromMs: string;
    toMs: string;
  }>({
    query: `SELECT count() AS found,
      toUnixTimestamp64Milli(min(unit_start_time)) AS fromMs,
      toUnixTimestamp64Milli(max(unit_start_time)) AS toMs
      FROM topic_facet_summaries
      WHERE project_id = {projectId:String} AND ${unitTimeFilterSql} AND trace_id = {traceId:String}`,
    params: { projectId, ...timeRangeParams(timeRange), traceId },
    tags: { route: "topics-trace-unit-time", projectId },
  });
  if (!row || Number(row.found) === 0) return null;
  return {
    from: new Date(Number(row.fromMs)),
    to: new Date(Number(row.toMs) + 1),
  };
}