Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

7.8 KB
Raw Blame
import type {
  TopicAssignment,
  TopicSummary,
  TopicTimeRange,
} from "@langfuse/shared/topics";
import { OBSERVATIONS_TO_TRACE_INTERVAL_DAYS } from "@langfuse/shared/src/server";
import {
  getLatestFacetSummaries,
  getPublishedTopicRun,
  getTopicRuns,
  getTopicTraceUnitTimeRange,
  hasStoredTopicSummaries,
  listTopicFacets,
  listTopicRules,
  readLatestTopicAssignments,
} from "@langfuse/shared/topics/server";

function resolveTopicResult(
  summary: Pick<
    TopicSummary,
    | "projectId"
    | "facetId"
    | "facetVersion"
    | "traceId"
    | "unitStartTime"
    | "processedAt"
    | "state"
  >,
  stored?: TopicAssignment,
) {
  const assignment =
    summary.state === "complete" &&
    stored?.projectId === summary.projectId &&
    stored.facetId === summary.facetId &&
    stored.facetVersion === summary.facetVersion &&
    stored.traceId === summary.traceId &&
    stored.unitStartTime === summary.unitStartTime &&
    stored.summaryProcessedAt === summary.processedAt
      ? stored
      : undefined;
  let outcome: string = summary.state;
  if (summary.state === "complete") outcome = "awaiting_map";
  if (assignment) outcome = assignment.topicId ? "assigned" : "outlier";
  return { assignment, outcome };
}

/** Current state is resolved per trace and facet, independently of execution batches. */
export async function currentTopicResults(
  projectId: string,
  timeRange: TopicTimeRange,
  { facetIds, traceIds }: { facetIds?: string[]; traceIds?: string[] } = {},
) {
  const facets = (await listTopicFacets(projectId)).filter(
    (facet) => !facetIds || facetIds.includes(facet.id),
  );
  const hasStoredSummaries =
    facets.length > 0 && (await hasStoredTopicSummaries(projectId));
  if (!hasStoredSummaries) return { hasStoredSummaries, facets: [] };
  // Only facets enabled on the project's rule get tabs; without a rule, all show.
  const [rule] = await listTopicRules(projectId);
  const enabledFacetIds = rule ? new Set(rule.facetIds) : null;
  const currentFacets = await Promise.all(
    facets
      .filter((facet) => !enabledFacetIds || enabledFacetIds.has(facet.id))
      .map(async (facet) => {
        const version = facet.versions[0];
        const [storedAssignments, storedSummaries, run] = await Promise.all([
          readLatestTopicAssignments(projectId, facet.id, timeRange, traceIds),
          getLatestFacetSummaries(
            projectId,
            facet.id,
            undefined,
            timeRange,
            traceIds,
          ),
          getPublishedTopicRun(projectId, facet.id),
        ]);
        const latestSummaries = storedSummaries.filter(
          (row) => row.traceId !== null,
        );
        const assignmentByTrace = new Map(
          storedAssignments.map((row) => [row.traceId, row]),
        );
        const current = latestSummaries
          .sort(
            (a, b) =>
              b.unitStartTime.localeCompare(a.unitStartTime) ||
              a.traceId.localeCompare(b.traceId),
          )
          .map((summary) => ({
            summary,
            ...resolveTopicResult(
              summary,
              assignmentByTrace.get(summary.traceId),
            ),
          }));
        const assignmentRuns = await getTopicRuns(projectId, [
          ...new Set(
            current.flatMap(({ assignment }) =>
              assignment?.topicId &&
              assignment.runId &&
              assignment.runId !== run?.id
                ? [assignment.runId]
                : [],
            ),
          ),
        ]);
        const definitions = [
          ...(run?.topics ?? []),
          ...assignmentRuns.flatMap((item) => item.topics),
        ];
        const topicByVersion = new Map(
          definitions.map((topic) => [topic.topicVersionId, topic]),
        );
        const topics = new Map<
          string,
          { assignment: TopicAssignment; count: number }
        >();
        const rows = current.map(({ summary, assignment, outcome }) => {
          const definition = assignment?.topicVersionId
            ? topicByVersion.get(assignment.topicVersionId)
            : undefined;
          if (assignment?.topicId) {
            const topic = topics.get(assignment.topicId);
            topics.set(assignment.topicId, {
              assignment:
                !topic || assignment.assignedAt > topic.assignment.assignedAt
                  ? assignment
                  : topic.assignment,
              count: (topic?.count ?? 0) + 1,
            });
          }
          return {
            facetVersion: summary.facetVersion,
            traceId: summary.traceId,
            unitStartTime: summary.unitStartTime,
            summary: summary.summary,
            outcome,
            topicId: assignment?.topicId ?? null,
            topicName: definition?.name ?? null,
          };
        });
        // Current map names take precedence; retired IDs keep their latest referenced name.
        const currentTopics = new Map(
          run?.topics.map((topic) => [topic.topicId, topic]),
        );
        return {
          facetId: facet.id,
          name: facet.name,
          facetVersion: version?.version ?? null,
          usableCount: latestSummaries.filter(
            (row) =>
              row.facetVersion === version?.version && row.state === "complete",
          ).length,
          awaitingCount: rows.filter((row) => row.outcome === "awaiting_map")
            .length,
          topics: [...topics]
            .sort(
              ([, a], [, b]) =>
                b.count - a.count ||
                b.assignment.assignedAt.localeCompare(a.assignment.assignedAt),
            )
            .map(([id, topic]) => {
              const definition =
                currentTopics.get(id) ??
                topicByVersion.get(topic.assignment.topicVersionId ?? "");
              return {
                id,
                versionId:
                  definition?.topicVersionId ??
                  topic.assignment.topicVersionId ??
                  "",
                runId:
                  definition?.createdByRunId ?? topic.assignment.runId ?? "",
                name: definition?.name ?? "Unavailable topic",
                description: definition?.description ?? "",
                count: topic.count,
              };
            }),
          rows,
          map: run
            ? {
                runId: run.id,
                topics: run.topics.map((topic) => ({
                  id: topic.topicId,
                  versionId: topic.topicVersionId,
                  runId: topic.createdByRunId,
                  name: topic.name,
                  description: topic.description,
                })),
              }
            : null,
        };
      }),
  );
  return { hasStoredSummaries, facets: currentFacets };
}

/** Current result per facet for one trace; summaries start within a few days of the trace. */
export async function traceFacetResults(
  projectId: string,
  traceId: string,
  timestamp: Date,
) {
  const slackMs = OBSERVATIONS_TO_TRACE_INTERVAL_DAYS * 86_400_000;
  // Narrow to the trace's exact source time so assignment reads prune to its minute.
  const unitTimeRange = await getTopicTraceUnitTimeRange(projectId, traceId, {
    from: new Date(timestamp.getTime() - slackMs),
    to: new Date(timestamp.getTime() + slackMs),
  });
  if (!unitTimeRange) return [];
  const { facets } = await currentTopicResults(projectId, unitTimeRange, {
    traceIds: [traceId],
  });
  return facets.map((facet) => {
    const row = facet.rows.find((item) => item.traceId === traceId);
    return {
      facetId: facet.facetId,
      name: facet.name,
      summary: row?.summary ?? null,
      outcome: row?.outcome ?? null,
      topicId: row?.topicId ?? null,
      topicName: row?.topicName ?? null,
      // Slot in the published map; picks the topic's color.
      topicIndex:
        facet.map?.topics.findIndex((topic) => topic.id === row?.topicId) ?? -1,
    };
  });
}