Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

11.8 KB
Raw Blame
import { createHash } from "node:crypto";
import { prisma, type Prisma } from "../../db";
import { getTopicProcessingMapIds } from "./postgres";
import { InvalidRequestError } from "../../errors/InvalidRequestError";
import { BatchActionStatus } from "../../features/batchAction/types";
import {
  topicExecutionInputSchema,
  topicIdSchema,
  topicTimeRangeSchema,
  type TopicExecution,
  type TopicExecutionInput,
  type TopicExecutionSummary,
  type TopicFacetProgress,
} from "../../topics";

export const TOPICS_ACTION = "topics";
// Actor of executions Topics starts on its own, such as automatic discovery.
export const TOPICS_SYSTEM_USER_ID = "system:topics";
type BatchMetadata = {
  inputSettings:
    | Extract<TopicExecutionSummary["input"], { operation: "process" }>
    | (Omit<
        Extract<TopicExecutionSummary["input"], { operation: "update" }>,
        "timeRange"
      > & { timeRange: { from: string; to: string } });
  facets: TopicFacetProgress[];
  phase: string;
  inputHash: string;
  requestHash: string;
  progressVersion?: number;
};
type BatchRow = Prisma.BatchActionGetPayload<object>;
const statuses = {
  queued: BatchActionStatus.Queued,
  running: BatchActionStatus.Processing,
  completed: BatchActionStatus.Completed,
  completed_with_errors: BatchActionStatus.Partial,
  failed: BatchActionStatus.Failed,
};
const hash = (value: unknown) =>
  createHash("sha256").update(JSON.stringify(value)).digest("hex");
const executionIdForRequest = (projectId: string, requestId: string) =>
  createHash("sha256")
    .update(`${projectId}:${requestId}`)
    .digest("hex")
    .slice(0, 32);
const batchMetadata = (row: BatchRow) => row.config as unknown as BatchMetadata;
const json = (value: unknown) => value as Prisma.InputJsonValue;
const facetKey = (facet: { facetId: string; facetVersion: number }) =>
  JSON.stringify([facet.facetId, facet.facetVersion]);

async function lockExecution(
  tx: Prisma.TransactionClient,
  projectId: string,
  executionId: string,
) {
  await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtextextended(${`topics:${projectId}:${executionId}`}, 0))`;
  await tx.$queryRaw`SELECT id FROM batch_actions WHERE project_id = ${projectId} AND id = ${executionId} FOR UPDATE`;
}

async function storedExecution(
  projectId: string,
  executionId: string,
  db: Pick<Prisma.TransactionClient, "batchAction"> = prisma,
): Promise<BatchRow | null> {
  topicIdSchema.parse(projectId);
  topicIdSchema.parse(executionId);
  return db.batchAction.findFirst({
    where: { projectId, id: executionId, actionType: TOPICS_ACTION },
  });
}

function checkRequest(
  state: BatchMetadata,
  input: TopicExecutionInput,
  requestHash: string,
) {
  if (state.inputHash !== hash(input) || state.requestHash !== requestHash)
    throw new InvalidRequestError(
      "This request ID already belongs to a different Topics request.",
    );
}

// JSON storage turns time ranges into strings; restore them as dates.
function storedInput(
  settings: BatchMetadata["inputSettings"],
): TopicExecutionSummary["input"] {
  if (settings.operation === "update")
    return {
      ...settings,
      timeRange: topicTimeRangeSchema.parse(settings.timeRange),
    };
  if (!settings.thenUpdate) return settings;
  return {
    ...settings,
    thenUpdate: {
      timeRange: topicTimeRangeSchema.parse(settings.thenUpdate.timeRange),
    },
  };
}

function summaryResult(row: BatchRow): TopicExecutionSummary {
  const state = batchMetadata(row);
  const status = (
    Object.keys(statuses) as TopicExecutionSummary["status"][]
  ).find((candidate) => statuses[candidate] === row.status);
  if (!status) throw new Error("Invalid Topics execution status.");
  return {
    id: row.id,
    projectId: row.projectId,
    status,
    phase: state.phase,
    createdAt: row.createdAt.toISOString(),
    updatedAt: row.updatedAt.toISOString(),
    error: row.log,
    input: storedInput(state.inputSettings),
    facets: state.inputSettings.facets.map(({ facetId, version }) => {
      const facet = state.facets.find(
        (candidate) =>
          candidate.facetId === facetId && candidate.facetVersion === version,
      );
      if (!facet)
        throw new Error("Topics execution is missing a selected facet.");
      return facet;
    }),
  };
}

/** The user who started an execution, so follow-up runs stay in their History. */
export async function readTopicExecutionUserId(
  projectId: string,
  executionId: string,
): Promise<string | null> {
  const row = await storedExecution(projectId, executionId);
  return row?.userId ?? null;
}

/** Postgres owns compact execution settings and progress, never per-trace payloads. */
export async function createTopicExecution(
  rawInput: TopicExecutionInput,
  originalRequestHash?: string,
  userId?: string,
): Promise<TopicExecution> {
  const input = topicExecutionInputSchema.parse(rawInput);
  if (!userId)
    throw new InvalidRequestError("A user is required to run Topics manually.");
  input.facets = [
    ...new Map(
      input.facets.map((facet) => [
        JSON.stringify([facet.facetId, facet.version]),
        facet,
      ]),
    ).values(),
  ];
  if (input.operation === "process")
    input.traceIds = [...new Set(input.traceIds)];
  const id = executionIdForRequest(input.projectId, input.requestId);
  const requestHash = originalRequestHash ?? hash(input);
  const existing = await storedExecution(input.projectId, id);
  if (existing) {
    checkRequest(batchMetadata(existing), input, requestHash);
    return { ...summaryResult(existing), input, traceErrors: [] };
  }
  const { projectId } = input;
  const { ids, inputSettings } =
    input.operation === "process"
      ? (() => {
          const { traceIds, ...compact } = input;
          return { ids: traceIds, inputSettings: compact };
        })()
      : { ids: [], inputSettings: input };
  const mapIds =
    input.operation === "process"
      ? await getTopicProcessingMapIds(
          projectId,
          input.facets,
          input.embeddingConfig,
        )
      : [];
  const row = await prisma.$transaction(async (tx) => {
    await lockExecution(tx, projectId, id);
    const current = await storedExecution(projectId, id, tx);
    if (current) {
      checkRequest(batchMetadata(current), input, requestHash);
      return current;
    }
    const versionCount = await tx.evaluatorVersion.count({
      where: {
        evaluator: { projectId, type: "FACET" },
        OR: input.facets.map(({ facetId, version }) => ({
          evaluatorId: facetId,
          version,
        })),
      },
    });
    if (versionCount !== input.facets.length)
      throw new InvalidRequestError("Facet version not found in this project.");
    const facets: TopicFacetProgress[] = input.facets.map(
      ({ facetId, version }) => ({
        facetId,
        facetVersion: version,
        outcome: "pending",
        runId:
          input.operation === "update"
            ? hash([id, facetId, version, "run"]).slice(0, 48)
            : (mapIds.find(
                (map) =>
                  map.facetId === facetId && map.facetVersion === version,
              )?.runId ?? null),
        error: null,
        counts: {
          requested: ids.length,
          complete: 0,
          nonApplicable: 0,
          insufficientInput: 0,
          failed: 0,
          assigned: 0,
          outlier: 0,
        },
      }),
    );
    if (input.operation === "update") {
      await tx.topicClusteringRun.createMany({
        data: facets.map((facet) => ({
          id: facet.runId!,
          projectId,
          facetId: facet.facetId,
          facetVersion: facet.facetVersion,
          status: "pending",
          config: {
            timeRange: {
              from: input.timeRange.from.toISOString(),
              to: input.timeRange.to.toISOString(),
            },
          },
        })),
      });
    }
    return tx.batchAction.create({
      data: {
        id,
        projectId,
        userId,
        actionType: TOPICS_ACTION,
        tableName: "traces",
        status: BatchActionStatus.Queued,
        query: {},
        config: json({
          inputSettings:
            inputSettings.operation === "update"
              ? {
                  ...inputSettings,
                  timeRange: {
                    from: inputSettings.timeRange.from.toISOString(),
                    to: inputSettings.timeRange.to.toISOString(),
                  },
                }
              : inputSettings,
          facets,
          phase: "queued",
          inputHash: hash(input),
          requestHash,
        } satisfies BatchMetadata),
        totalCount: input.operation === "process" ? ids.length : facets.length,
        processedCount: 0,
        failedCount: 0,
      },
    });
  });
  return { ...summaryResult(row), input, traceErrors: [] };
}

export async function readTopicExecutionForRequest(
  projectId: string,
  requestId: string,
  requestHash: string,
): Promise<TopicExecutionSummary | null> {
  topicIdSchema.parse(requestId);
  const id = executionIdForRequest(projectId, requestId);
  const stored = await storedExecution(projectId, id);
  if (!stored) return null;
  if (batchMetadata(stored).requestHash !== requestHash)
    throw new InvalidRequestError(
      "This request ID already belongs to a different Topics request.",
    );
  return summaryResult(stored);
}

export async function readTopicExecutionSummary(
  projectId: string,
  executionId: string,
): Promise<TopicExecutionSummary | null> {
  const stored = await storedExecution(projectId, executionId);
  return stored ? summaryResult(stored) : null;
}

export async function listTopicExecutions(
  projectId: string,
): Promise<TopicExecutionSummary[]> {
  topicIdSchema.parse(projectId);
  const batches = await prisma.batchAction.findMany({
    where: { projectId, actionType: TOPICS_ACTION },
    orderBy: { createdAt: "desc" },
    take: 100,
  });
  return batches.map(summaryResult);
}

export async function writeTopicExecution(
  execution: Pick<
    TopicExecutionSummary,
    "id" | "projectId" | "status" | "phase" | "facets" | "error"
  >,
  progressVersion?: number,
): Promise<void> {
  const { projectId, id } = execution;
  await prisma.$transaction(async (tx) => {
    await lockExecution(tx, projectId, id);
    const current = await storedExecution(projectId, id, tx);
    if (!current) throw new Error("Topics execution does not exist.");
    const state = batchMetadata(current);
    if (
      progressVersion !== undefined &&
      progressVersion <= (state.progressVersion ?? 0)
    )
      return;
    const facetKeys = new Set(execution.facets.map(facetKey));
    if (
      state.facets.length !== execution.facets.length ||
      facetKeys.size !== execution.facets.length ||
      state.facets.some((facet) => !facetKeys.has(facetKey(facet)))
    )
      throw new Error("Topics execution facets cannot change.");
    const terminal =
      execution.status !== "queued" && execution.status !== "running";
    const { facets } = execution;
    let processedCount =
      execution.status === "completed" ||
      execution.status === "completed_with_errors"
        ? current.totalCount
        : null;
    let failedCount = facets.every((facet) => facet.counts.failed === 0)
      ? 0
      : null;
    if (state.inputSettings.operation === "update") {
      processedCount = facets.filter(
        (facet) => facet.outcome !== "pending",
      ).length;
      failedCount = facets.filter((facet) => facet.outcome === "failed").length;
    }
    await tx.batchAction.update({
      where: { projectId, id },
      data: {
        config: json({
          ...state,
          phase: execution.phase,
          ...(progressVersion === undefined ? {} : { progressVersion }),
          facets,
        } satisfies BatchMetadata),
        status: statuses[execution.status],
        processedCount,
        failedCount,
        finishedAt: terminal ? (current.finishedAt ?? new Date()) : null,
        log: execution.error,
      },
    });
  });
}