Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

19.5 KB
Raw Blame
import { prisma, Prisma } from "../../db";
import {
  DEFAULT_TOPIC_FACETS,
  topicRuleSettingsSchema,
  topicTimeRangeSchema,
  sameTopicGeometry,
  type TopicFacet,
  type TopicFacetRef,
  type TopicFacetVersion,
  type TopicRun,
  type TopicRule,
  type TopicDefinition,
  type TopicEmbeddingConfig,
} from "../../topics";
import { isEqual } from "lodash";
import { getTopicDefinitions, writeTopicDefinitions } from "./clickhouse";
import { InvalidRequestError } from "../../errors/InvalidRequestError";
import { TopicsTargetObject } from "../../features/evals/types";
import {
  invalidateTopicsProcessingActive,
  topicRuleWhere,
} from "./processing-gate";

type FacetVersionRow = Prisma.EvaluatorVersionGetPayload<object>;
type FacetRow = Prisma.EvaluatorGetPayload<{ include: { versions: true } }>;
type RunRow = Prisma.TopicClusteringRunGetPayload<object>;

function facetVersion(
  row: FacetVersionRow,
  projectId: string,
): TopicFacetVersion {
  if (row.prompt === null) throw new Error("Facet version has no prompt.");
  return {
    projectId,
    facetId: row.evaluatorId,
    version: row.version,
    prompt: row.prompt,
    createdAt: row.createdAt.toISOString(),
  };
}

function topicFacet(row: FacetRow): TopicFacet {
  return {
    id: row.id,
    projectId: row.projectId,
    name: row.name,
    description: row.description ?? "",
    isBuiltIn: row.isBuiltIn,
    versions: row.versions.map((version) =>
      facetVersion(version, row.projectId),
    ),
  };
}

export async function listTopicFacets(
  projectId: string,
): Promise<TopicFacet[]> {
  const rows = await prisma.evaluator.findMany({
    where: { projectId, type: "FACET" },
    include: { versions: { orderBy: { version: "desc" } } },
    // A stable order keeps the Topics summary prompt (and its cache) identical across traces.
    orderBy: [{ createdAt: "asc" }, { id: "asc" }],
  });
  return rows.map(topicFacet);
}

export async function getTopicFacetVersion(
  projectId: string,
  facetId: string,
  version: number,
): Promise<TopicFacetVersion | null> {
  const row = await prisma.evaluatorVersion.findFirst({
    where: {
      evaluatorId: facetId,
      version,
      evaluator: { projectId, type: "FACET" },
    },
  });
  return row ? facetVersion(row, projectId) : null;
}

interface FacetVersionInput {
  projectId: string;
  prompt: string;
}

export async function createTopicFacet(
  input: FacetVersionInput & { name: string; description: string },
): Promise<TopicFacet> {
  const row = await prisma.evaluator.create({
    data: {
      projectId: input.projectId,
      type: "FACET",
      name: input.name.trim(),
      description: input.description.trim(),
      versions: { create: { version: 1, prompt: input.prompt.trim() } },
    },
    include: { versions: true },
  });
  return topicFacet(row);
}

export async function createTopicFacetVersion(
  input: FacetVersionInput & { facetId: string },
): Promise<TopicFacetVersion> {
  const row = await prisma.$transaction(async (tx) => {
    const facets = await tx.$queryRaw<
      { id: string; isBuiltIn: boolean }[]
    >`SELECT id, is_built_in AS "isBuiltIn" FROM evaluators
      WHERE project_id = ${input.projectId} AND id = ${input.facetId} AND type = 'FACET'
      FOR UPDATE`;
    if (!facets.length)
      throw new InvalidRequestError("Facet not found in this project.");
    if (facets[0].isBuiltIn)
      throw new InvalidRequestError("Built-in facets cannot be changed.");
    const last = await tx.evaluatorVersion.findFirst({
      where: { evaluatorId: input.facetId },
      orderBy: { version: "desc" },
    });
    if (last?.prompt === input.prompt.trim()) return last;
    return tx.evaluatorVersion.create({
      data: {
        evaluatorId: input.facetId,
        version: (last?.version ?? 0) + 1,
        prompt: input.prompt.trim(),
      },
    });
  });
  return facetVersion(row, input.projectId);
}

export async function ensureDefaultTopicFacets(
  projectId: string,
): Promise<TopicFacet[]> {
  const presets = DEFAULT_TOPIC_FACETS;
  const facets = await listTopicFacets(projectId);
  if (
    presets.every((preset) =>
      facets.some((facet) => facet.isBuiltIn && facet.name === preset.name),
    )
  )
    return facets;
  await prisma.$transaction(async (tx) => {
    const projects = await tx.$queryRaw<{ id: string }[]>`
      SELECT id FROM projects WHERE id = ${projectId} FOR UPDATE`;
    if (!projects.length) throw new InvalidRequestError("Project not found.");
    const existing = await tx.evaluator.findMany({
      where: { projectId, type: "FACET", isBuiltIn: true },
      select: { name: true },
    });
    for (const { prompt, ...preset } of presets) {
      if (!existing.some((facet) => facet.name === preset.name))
        await tx.evaluator.create({
          data: {
            projectId,
            ...preset,
            type: "FACET",
            isBuiltIn: true,
            versions: { create: { version: 1, prompt } },
          },
        });
    }
  });
  return listTopicFacets(projectId);
}

type RuleRow = Prisma.EvaluationRuleGetPayload<{
  include: { assignments: true };
}>;
const topicRule = (row: RuleRow): TopicRule => ({
  id: row.id,
  projectId: row.projectId,
  name: row.name,
  ...topicRuleSettingsSchema.parse({
    filter: row.filter,
    sampling: Number(row.sampling),
  }),
  facetIds: row.assignments.map((assignment) => assignment.evaluatorId),
  updatedAt: row.updatedAt.toISOString(),
});

export async function listTopicRules(projectId: string): Promise<TopicRule[]> {
  return (
    await prisma.evaluationRule.findMany({
      where: topicRuleWhere(projectId),
      include: { assignments: true },
      orderBy: { updatedAt: "desc" },
    })
  ).map(topicRule);
}

export async function getTopicRule(
  projectId: string,
  id: string,
): Promise<TopicRule | null> {
  const row = await prisma.evaluationRule.findFirst({
    where: { ...topicRuleWhere(projectId), id },
    include: { assignments: true },
  });
  return row ? topicRule(row) : null;
}

/**
 * A project has at most one Topics rule: its trace filter, sampling, and idle
 * time apply to every facet it runs, because trace readiness keeps one idle
 * time per trace. Postgres has no constraint for this, so writers hold this
 * per-project lock while they look up the rule; otherwise two first saves
 * could each find none and create a rule.
 */
async function lockProjectTopicRule(
  tx: Prisma.TransactionClient,
  projectId: string,
) {
  await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtextextended(${`topics-rule:${projectId}`}, 0))`;
}

export type TopicRuleInput = Omit<
  TopicRule,
  "id" | "name" | "updatedAt" | "sampling"
> &
  Partial<Pick<TopicRule, "name" | "sampling">> & {
    id?: string;
    /** The user saving the rule; recorded when it is created. */
    userId?: string | null;
  };

export async function saveTopicRule(input: TopicRuleInput): Promise<TopicRule> {
  const rule = await prisma.$transaction((tx) => writeTopicRule(tx, input));
  await invalidateTopicsProcessingActive(input.projectId);
  return rule;
}

/** Writes the project's rule inside the caller's transaction, under its lock. */
export async function writeTopicRule(
  tx: Prisma.TransactionClient,
  input: TopicRuleInput,
): Promise<TopicRule> {
  const facetIds = [...new Set(input.facetIds)];
  await lockProjectTopicRule(tx, input.projectId);
  const facetCount = await tx.evaluator.count({
    where: {
      projectId: input.projectId,
      type: "FACET",
      id: { in: facetIds },
    },
  });
  if (!facetIds.length || facetCount !== facetIds.length)
    throw new InvalidRequestError("Select facets from this project.");
  const rules = await tx.evaluationRule.findMany({
    where: topicRuleWhere(input.projectId),
    select: { id: true, name: true, sampling: true },
  });
  if (rules.length > 1)
    throw new InvalidRequestError(
      "This project has more than one Topics rule.",
    );
  const ruleId = rules[0]?.id;
  const name = input.name?.trim() || rules[0]?.name || "Topics";
  if (ruleId && !input.id)
    throw new InvalidRequestError(
      "This project already has a Topics rule. Reload and save that rule instead.",
    );
  if (input.id && input.id !== ruleId)
    throw new InvalidRequestError("Topic rule not found in this project.");
  // Sampling belongs to automatic processing; a write without it keeps the rate.
  const config = topicRuleSettingsSchema.parse({
    ...input,
    sampling:
      input.sampling ?? (rules[0] ? Number(rules[0].sampling) : undefined),
  });
  const data = {
    name,
    filter: JSON.parse(JSON.stringify(config.filter)) as Prisma.InputJsonValue,
    sampling: config.sampling,
    delay: 0,
    targetObject: TopicsTargetObject,
    timeScope: ["NEW"],
    status: "ACTIVE" as const,
  };
  const assignments = facetIds.map((evaluatorId) => ({
    projectId: input.projectId,
    evaluatorId,
  }));
  const row = ruleId
    ? await tx.evaluationRule.update({
        where: { ...topicRuleWhere(input.projectId), id: ruleId },
        data: {
          ...data,
          assignments: { deleteMany: {}, create: assignments },
        },
        include: { assignments: true },
      })
    : await tx.evaluationRule.create({
        data: {
          ...data,
          projectId: input.projectId,
          createdByUserId: input.userId ?? null,
          assignments: { create: assignments },
        },
        include: { assignments: true },
      });
  return topicRule(row);
}

function runResult(
  row: RunRow,
  definitions: Map<string, TopicDefinition>,
): TopicRun {
  return {
    id: row.id,
    projectId: row.projectId,
    facetId: row.facetId,
    facetVersion: row.facetVersion,
    status: row.status as TopicRun["status"],
    createdAt: row.createdAt.toISOString(),
    finishedAt: row.finishedAt?.toISOString() ?? null,
    startedAt: row.startedAt?.toISOString() ?? null,
    config: row.config as Record<string, unknown>,
    error: row.error,
    topics: row.topicVersionIds.map((id) => {
      const topic = definitions.get(id);
      if (!topic || topic.projectId !== row.projectId)
        throw new Error("Topic run references a missing definition.");
      return topic;
    }),
  };
}

async function hydrateRuns(
  projectId: string,
  rows: RunRow[],
): Promise<TopicRun[]> {
  const ids = rows.flatMap((row) => row.topicVersionIds);
  if (!ids.length) return rows.map((row) => runResult(row, new Map()));
  const ranges = rows
    .filter((row) => row.topicVersionIds.length)
    .map((row) =>
      topicTimeRangeSchema.parse(
        (row.config as Record<string, unknown>).definitionTimeRange,
      ),
    );
  const definitions = new Map(
    (
      await getTopicDefinitions(projectId, ids, {
        from: new Date(Math.min(...ranges.map(({ from }) => from.getTime()))),
        to: new Date(Math.max(...ranges.map(({ to }) => to.getTime()))),
      })
    ).map((topic) => [topic.topicVersionId, topic]),
  );
  return rows.map((row) => runResult(row, definitions));
}

export async function getTopicRun(
  projectId: string,
  id: string,
): Promise<TopicRun | null> {
  const row = await prisma.topicClusteringRun.findFirst({
    where: { projectId, id },
  });
  return row ? (await hydrateRuns(projectId, [row]))[0] : null;
}

export async function getTopicRuns(
  projectId: string,
  ids: string[],
): Promise<TopicRun[]> {
  if (!ids.length) return [];
  const rows = await prisma.topicClusteringRun.findMany({
    where: { projectId, id: { in: ids } },
  });
  return hydrateRuns(projectId, rows);
}

// Higher facet versions win; a late-finishing older run cannot replace a newer map.
const servingMapOrder = [
  { facetVersion: "desc" },
  { createdAt: "desc" },
  { id: "desc" },
] satisfies Prisma.TopicClusteringRunOrderByWithRelationInput[];

export async function getPublishedTopicRun(
  projectId: string,
  facetId: string,
): Promise<TopicRun | null> {
  const row = await prisma.topicClusteringRun.findFirst({
    where: { projectId, facetId, status: "completed" },
    orderBy: servingMapOrder,
  });
  return row ? (await hydrateRuns(projectId, [row]))[0] : null;
}

/** Capture compatible serving-map IDs without reading their ClickHouse cohorts. */
export async function getTopicProcessingMapIds(
  projectId: string,
  facets: TopicFacetRef[],
  embeddingConfig: TopicEmbeddingConfig,
): Promise<{ facetId: string; facetVersion: number; runId: string | null }[]> {
  if (!facets.length) return [];
  const runs = await prisma.topicClusteringRun.findMany({
    where: {
      projectId,
      facetId: { in: [...new Set(facets.map((facet) => facet.facetId))] },
      status: "completed",
    },
    orderBy: servingMapOrder,
    distinct: ["facetId"],
    select: { id: true, facetId: true, facetVersion: true, config: true },
  });
  return facets.map(({ facetId, version }) => {
    const run = runs.find((run) => run.facetId === facetId);
    const config = run?.config as Record<string, unknown> | undefined;
    return {
      facetId,
      facetVersion: version,
      runId:
        run?.facetVersion === version &&
        config?.embeddingModel === embeddingConfig.embeddingModel &&
        config?.dimensions === embeddingConfig.embeddingDimensions
          ? run.id
          : null,
    };
  });
}

export async function createTopicRun(input: {
  projectId: string;
  facetId: string;
  facetVersion: number;
  config: Record<string, unknown>;
}): Promise<TopicRun> {
  if (
    !(await getTopicFacetVersion(
      input.projectId,
      input.facetId,
      input.facetVersion,
    ))
  )
    throw new InvalidRequestError("Facet version not found in this project.");
  const row = await prisma.topicClusteringRun.create({
    data: {
      projectId: input.projectId,
      facetId: input.facetId,
      facetVersion: input.facetVersion,
      config: input.config as Prisma.InputJsonValue,
    },
  });
  return runResult(row, new Map());
}

function sameTopicDefinition(
  stored: TopicDefinition | undefined,
  candidate: TopicDefinition,
): boolean {
  if (!stored || !sameTopicGeometry(stored, candidate)) return false;
  return isEqual(
    { ...stored, centroid: candidate.centroid, radius: candidate.radius },
    candidate,
  );
}

export async function saveTopicRun(run: TopicRun): Promise<TopicRun> {
  const stored = await prisma.topicClusteringRun.findFirst({
    where: { projectId: run.projectId, id: run.id },
  });
  if (
    !stored ||
    stored.facetId !== run.facetId ||
    stored.facetVersion !== run.facetVersion
  )
    throw new Error("Run not found or facet version mismatch.");
  if (stored.status === "completed" || stored.status === "skipped")
    return (await hydrateRuns(run.projectId, [stored]))[0];
  const topicVersionIds = run.topics.map((topic) => topic.topicVersionId);
  if (
    new Set(topicVersionIds).size !== topicVersionIds.length ||
    new Set(run.topics.map((topic) => topic.topicId)).size !== run.topics.length
  )
    throw new Error("Topic run contains duplicate definitions.");
  const definitionTimes = run.topics.map((topic) =>
    Date.parse(topic.createdAt),
  );
  const definitionTimeRange = definitionTimes.length
    ? topicTimeRangeSchema.parse({
        from: new Date(Math.min(...definitionTimes)),
        to: new Date(Math.max(...definitionTimes) + 1),
      })
    : null;
  const config = { ...run.config };
  if (definitionTimeRange)
    config.definitionTimeRange = {
      from: definitionTimeRange.from.toISOString(),
      to: definitionTimeRange.to.toISOString(),
    };
  else delete config.definitionTimeRange;
  const saved = new Map(
    (definitionTimeRange
      ? await getTopicDefinitions(
          run.projectId,
          topicVersionIds,
          definitionTimeRange,
        )
      : []
    ).map((topic) => [topic.topicVersionId, topic]),
  );
  const missing: TopicDefinition[] = [];
  for (const topic of run.topics) {
    if (topic.projectId !== run.projectId)
      throw new Error("Invalid topic definition.");
    const existing = saved.get(topic.topicVersionId);
    if (existing) {
      if (!sameTopicDefinition(existing, topic))
        throw new Error("Topic definitions cannot change.");
    } else {
      if (topic.createdByRunId !== run.id)
        throw new Error("Topic run references a missing definition.");
      missing.push(topic);
    }
  }
  if (missing.length) {
    // Definitions must be readable before Postgres can publish their membership.
    await writeTopicDefinitions(missing);
    const inserted = await getTopicDefinitions(
      run.projectId,
      missing.map((topic) => topic.topicVersionId),
      definitionTimeRange!,
    );
    for (const topic of inserted) saved.set(topic.topicVersionId, topic);
    assertDefinitionsReadBack(missing, saved);
  }
  const row = await prisma.$transaction(async (tx) => {
    await tx.$queryRaw`SELECT id FROM topic_clustering_runs WHERE project_id = ${run.projectId} AND id = ${run.id} FOR UPDATE`;
    const existing = await tx.topicClusteringRun.findFirst({
      where: { projectId: run.projectId, id: run.id },
    });
    if (
      !existing ||
      existing.facetId !== run.facetId ||
      existing.facetVersion !== run.facetVersion
    )
      throw new Error("Run not found or facet version mismatch.");
    if (existing.status === "completed" || existing.status === "skipped")
      return existing;
    return tx.topicClusteringRun.update({
      where: { id: run.id, projectId: run.projectId },
      data: {
        topicVersionIds,
        status: run.status,
        config: config as Prisma.InputJsonValue,
        error: run.error,
        startedAt:
          existing.startedAt ??
          (run.startedAt ? new Date(run.startedAt) : null),
        finishedAt: run.finishedAt ? new Date(run.finishedAt) : null,
      },
    });
  });
  return row.topicVersionIds.every((id) => saved.has(id))
    ? runResult(row, saved)
    : (await hydrateRuns(run.projectId, [row]))[0];
}

/** Throws unless every new definition reads back identically, naming what differs. */
function assertDefinitionsReadBack(
  missing: TopicDefinition[],
  saved: Map<string, TopicDefinition>,
): void {
  const mismatched = missing.filter(
    (topic) => !sameTopicDefinition(saved.get(topic.topicVersionId), topic),
  );
  if (mismatched.length === 0) return;
  const differences = new Set(
    mismatched.flatMap((topic) =>
      definitionDifferences(saved.get(topic.topicVersionId), topic),
    ),
  );
  throw new TopicDefinitionReadbackError(
    `Topic definition readback did not match for ${mismatched.length} of ${missing.length} new definitions (${[...differences].join(", ")}).`,
  );
}

/** A written definition did not read back identically; the message names fields, never values. */
export class TopicDefinitionReadbackError extends Error {
  readonly name = "TopicDefinitionReadbackError";
}

/** Names what differs between a stored definition and its candidate, including metadata keys. */
function definitionDifferences(
  stored: TopicDefinition | undefined,
  candidate: TopicDefinition,
): string[] {
  if (!stored) return ["not readable"];
  const differs = (left: object, right: object, key: string) =>
    key in left !== key in right ||
    !isEqual(
      (left as Record<string, unknown>)[key],
      (right as Record<string, unknown>)[key],
    );
  const differences = sameTopicGeometry(stored, candidate) ? [] : ["geometry"];
  const fields = new Set([...Object.keys(stored), ...Object.keys(candidate)]);
  for (const field of fields) {
    if (field === "centroid" || field === "radius" || field === "metadata")
      continue;
    if (differs(stored, candidate, field)) differences.push(field);
  }
  const keys = new Set([
    ...Object.keys(stored.metadata),
    ...Object.keys(candidate.metadata),
  ]);
  for (const key of keys)
    if (differs(stored.metadata, candidate.metadata, key))
      differences.push(`metadata.${key}`);
  return differences;
}