Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

49.3 KB
Raw Blame
import { setImmediate as yieldToEventLoop } from "node:timers/promises";
import {
  getCurrentSpan,
  instrumentAsync,
  isInternalEvalEnvironment,
  logger,
  traceException,
} from "@langfuse/shared/src/server";
import {
  isTopicsEnabled,
  readTopicExecutionSummary,
  writeTopicExecution,
  getTopicFacetVersion,
  listTopicFacets,
  listTopicSummaries,
  writeTopicAssignments,
  readTopicAssignments,
  readTopicRunTraceIds,
  createTopicRun,
  getTopicRun,
  getPublishedTopicRun,
  readTopicMapAssignments,
  getTopicClusteringSummaries,
  saveTopicRun,
  loadTopicTranscript,
  stageTopicSummary,
  deleteStagedTopicSummary,
  readStagedTopicSummaries,
  readStagedTopicSummary,
  enqueueTopicEmbeddingBatch,
  TOPIC_EMBEDDING_EXPIRED_ERROR,
  TOPICS_TRANSCRIPT_VERSION,
  getTopicsModels,
  type TopicEmbeddingRef,
  type TopicsModels,
} from "@langfuse/shared/topics/server";
import {
  type TopicExecution,
  type TopicFacetVersion,
  type TopicSummary,
  type TopicRun,
  type TopicFacetProgress,
  type TopicDefinition,
  type TopicProcessBatchState,
  type TopicTimeRange,
  sameTopicGeometry,
  topicTimeRangeSchema,
} from "@langfuse/shared/topics";
import pLimit from "p-limit";
import { env } from "../../env";
import { nameTopicGroup, prepareTopicNamingGroup } from "./models";
import {
  ASSEMBLED_TRANSCRIPT_ID,
  baseTopicSummary,
  summarizeTraceFacets,
  type TraceFacet,
} from "./summarizeAssembledTrace";
import {
  isRejectedConnection,
  pauseForRejectedConnection,
  TopicsProviderUnavailable,
} from "./provider-error";
import {
  buildTopicPrototypes,
  buildNamingEvidence,
  classifyTopic,
  retainPopulatedTopics,
  topicHash,
} from "./classifier";
import { topicAssignmentRows } from "./assignments";
import {
  runTopicClustering,
  topicClusterSettings,
  TOPICS_NUMERIC_VERSION,
} from "./numeric";
import { matchTopicContinuity } from "./continuity";
import { createEventLoopYielder } from "./event-loop";
import { TopicMetrics } from "./metrics";
import { splitTopicModelUsage } from "./summaryResult";

type ProcessExecution = TopicExecution & {
  input: Extract<TopicExecution["input"], { operation: "process" }>;
};
type UpdateExecution = TopicExecution & {
  input: Extract<TopicExecution["input"], { operation: "update" }>;
};

class PendingTopicEmbeddings extends Error {}
class TopicEmbeddingFailure extends Error {}

const errorMessage = (error: unknown) =>
  error instanceof Error ? error.message : "Topics processing failed.";
const isExecutionFailure = (error: unknown) =>
  error instanceof TopicsProviderUnavailable ||
  error instanceof TopicEmbeddingFailure;
// Like trace-batch, a rejected input or unusable output fails only that trace.
const isTraceModelFailure = (error: unknown) =>
  error instanceof TopicsProviderUnavailable &&
  (error.reason === "invalid_output" ||
    error.reason === "invalid_input" ||
    error.reason === "content_filter");

async function saveProgress(
  metrics: TopicMetrics,
  execution: TopicExecution,
  phase: string,
) {
  execution.phase = phase;
  await metrics.measure(
    "storage",
    () => writeTopicExecution(execution),
    "storage",
  );
}

type TraceResults = Map<TraceFacet, TopicSummary | Error>;
type TraceInput = Awaited<ReturnType<typeof loadTopicTranscript>>;

/**
 * Summarizes a trace's pending facets with the trace-batch request: one call for
 * every facet that has no accepted or reusable summary. Results follow `facets`;
 * a trace-level failure applies only to unresolved facets, execution-level failures throw.
 */
async function summarizeTrace(
  metrics: TopicMetrics,
  models: TopicsModels,
  execution: ProcessExecution,
  facets: TraceFacet[],
  traceId: string,
  staged: Map<TopicFacetVersion, TopicSummary[]>,
  getTranscript: () => Promise<TraceInput>,
): Promise<(TopicSummary | Error)[]> {
  const results = acceptedSummaries(metrics, facets, traceId, staged);
  const unresolved = () => facets.filter((facet) => !results.has(facet));
  if (unresolved().length > 0) {
    try {
      const input = await getTranscript();
      if (isInternalEvalEnvironment(input.environment))
        throw new Error("Topics does not process internal Langfuse traces.");
      const base = (facet: TraceFacet) =>
        baseTopicSummary({
          projectId: execution.projectId,
          traceId,
          sessionId: input.sessionId,
          unitStartTime: input.unitStartTime,
          environment: input.environment,
          traceName: input.traceName,
          facet: facet.version,
          summaryModel: execution.input.processingConfig.summaryModel!,
          embeddingModel: execution.input.embeddingConfig.embeddingModel!,
          triggerType: "manual",
          metadata: { executionId: execution.id },
        });
      // Each step records a facet as soon as it is staged, so a later failure keeps it.
      await reuseSummaries(metrics, execution, unresolved(), base, results);
      await summarizePending(
        metrics,
        models,
        execution,
        traceId,
        unresolved(),
        base,
        input,
        results,
      );
    } catch (error) {
      if (isExecutionFailure(error) && !isTraceModelFailure(error)) throw error;
      metrics.error("summary", error);
      logger.warn("Topics summary failed for trace", {
        "langfuse.trace.id": traceId,
        "langfuse.topics.execution_id": execution.id,
        "langfuse.topics.error.reason":
          error instanceof TopicsProviderUnavailable ? error.reason : "other",
        error: errorMessage(error),
      });
      for (const facet of unresolved()) results.set(facet, error as Error);
    }
  }
  return facets.map((facet) => results.get(facet)!);
}

/** Summaries an earlier attempt of this execution already staged. */
function acceptedSummaries(
  metrics: TopicMetrics,
  facets: TraceFacet[],
  traceId: string,
  staged: Map<TopicFacetVersion, TopicSummary[]>,
): TraceResults {
  const results: TraceResults = new Map();
  for (const facet of facets) {
    const accepted = staged
      .get(facet.version)!
      .find((row) => row.traceId === traceId);
    if (!accepted) continue;
    if (isInternalEvalEnvironment(accepted.environment))
      throw new Error("Topics does not process internal Langfuse traces.");
    metrics.result("summary", "cached");
    // An accepted payload may expire after the cache read; never renew its TTL.
    results.set(facet, accepted);
  }
  return results;
}

/** Stages compatible stored summaries when the execution opted into reuse. */
async function reuseSummaries(
  metrics: TopicMetrics,
  execution: ProcessExecution,
  facets: TraceFacet[],
  base: (facet: TraceFacet) => TopicSummary,
  results: TraceResults,
): Promise<void> {
  if (!execution.input.reuseExistingSummaries) return;
  for (const facet of facets) {
    const summary = base(facet);
    const reusable = (
      await listTopicSummaries(
        execution.projectId,
        {
          facetId: summary.facetId,
          facetVersion: summary.facetVersion,
          traceIds: [summary.traceId!],
        },
        summaryTimeRange([summary]),
      )
    ).find(
      (row) =>
        row.summaryModel === execution.input.processingConfig.summaryModel &&
        row.transcriptId === ASSEMBLED_TRANSCRIPT_ID &&
        row.transcriptVersion === TOPICS_TRANSCRIPT_VERSION,
    );
    if (!reusable) continue;
    metrics.result("summary", "cached");
    const reuseEmbedding =
      reusable.state === "complete" &&
      reusable.embeddingModel ===
        execution.input.embeddingConfig.embeddingModel &&
      reusable.embedding.length ===
        execution.input.embeddingConfig.embeddingDimensions;
    results.set(
      facet,
      await stageSummary(metrics, execution, {
        ...summary,
        state:
          reusable.state === "complete" && !reuseEmbedding
            ? "summarized"
            : reusable.state,
        summary: reusable.summary,
        embedding: reuseEmbedding ? reusable.embedding : [],
        metadata: {
          ...summary.metadata,
          summaryReusedFromProcessedAt: reusable.processedAt,
          ...(reuseEmbedding
            ? { embeddingReusedFromProcessedAt: reusable.processedAt }
            : {}),
        },
      }),
    );
  }
}

/** Summarizes the remaining facets in one shared call and stages each result. */
async function summarizePending(
  metrics: TopicMetrics,
  models: TopicsModels,
  execution: ProcessExecution,
  traceId: string,
  facets: TraceFacet[],
  base: (facet: TraceFacet) => TopicSummary,
  input: TraceInput,
  results: TraceResults,
): Promise<void> {
  if (facets.length === 0) return;
  const summarized = await summarizeTraceFacets({
    models,
    facets,
    transcript: input.transcript,
    config: execution.input.processingConfig,
    metadata: {
      source_trace_id: traceId,
      source_environment: input.environment,
      execution_id: execution.id,
    },
    metrics,
  });
  const rows: { facet: TraceFacet; summary: TopicSummary }[] = [];
  for (const [index, facet] of facets.entries()) {
    const result = summarized?.results[index];
    if (result && "error" in result) {
      // An invalid facet output fails only this trace, not the execution.
      results.set(facet, new Error(result.error.message));
      continue;
    }
    const output = result?.output;
    rows.push({
      facet,
      summary: {
        ...base(facet),
        ...(output && {
          state: output.status === "applicable" ? "summarized" : output.status,
          summary: output.status === "applicable" ? output.summary.trim() : "",
        }),
      },
    });
  }
  // One call summarized every facet; each staged row carries an equal share of its usage.
  const shares = summarized
    ? splitTopicModelUsage(summarized.usage, rows.length)
    : [];
  for (const [index, { facet, summary }] of rows.entries())
    results.set(
      facet,
      await stageSummary(metrics, execution, { ...summary, ...shares[index] }),
    );
}

async function stageSummary(
  metrics: TopicMetrics,
  execution: TopicExecution,
  summary: TopicSummary,
): Promise<TopicSummary> {
  try {
    return await metrics.measure(
      "storage",
      () =>
        stageTopicSummary(
          { projectId: execution.projectId, executionId: execution.id },
          summary,
          execution.input.embeddingConfig,
        ),
      "storage",
    );
  } catch (error) {
    throw new TopicEmbeddingFailure(errorMessage(error));
  }
}

/** Frozen references survive payload expiry without silently regenerating paid work. */
async function awaitEmbeddingBatch(
  execution: TopicExecution,
  batchId: string,
  summaries: TopicEmbeddingRef[],
  retryFailed: boolean,
): Promise<void> {
  if (!summaries.length) return;
  const scope = { projectId: execution.projectId, executionId: execution.id };
  try {
    const status = await enqueueTopicEmbeddingBatch(
      { ...scope, batchId, summaries },
      { retryFailed },
    );
    if (status === "pending") throw new PendingTopicEmbeddings();
  } catch (error) {
    if (error instanceof PendingTopicEmbeddings) throw error;
    throw new TopicEmbeddingFailure(errorMessage(error));
  }
}

/** Terminal BullMQ state is the completion receipt; Redis payload cleanup is best effort. */
async function releaseCompletedSummaries(state: TopicProcessBatchState) {
  if (
    state.execution.facets.some(
      (facet) => facet.outcome === "pending" || facet.outcome === "failed",
    )
  )
    return;
  try {
    const refs = state.summaries.concat(
      state.failedTraceIds.flatMap(({ facetId, facetVersion, traceIds }) =>
        traceIds.map((traceId) => ({ facetId, facetVersion, traceId })),
      ),
    );
    await Promise.all(
      refs.map((ref) =>
        deleteStagedTopicSummary(
          {
            projectId: state.execution.projectId,
            executionId: state.execution.id,
          },
          ref,
        ),
      ),
    );
  } catch (error) {
    logger.warn(
      "Topics completed payload cleanup failed; Redis TTL will expire it.",
      { error },
    );
  }
}

function countSummaries(
  progress: TopicFacetProgress,
  summaries: TopicSummary[],
) {
  progress.counts.complete = summaries.filter(
    (row) => row.state === "complete",
  ).length;
  progress.counts.nonApplicable = summaries.filter(
    (row) => row.state === "not_applicable",
  ).length;
  progress.counts.insufficientInput = summaries.filter(
    (row) => row.state === "insufficient_input",
  ).length;
}

function summaryTimeRange(summaries: TopicSummary[]): TopicTimeRange {
  let from = Infinity;
  let to = -Infinity;
  for (const summary of summaries) {
    const timestamp = Date.parse(summary.unitStartTime);
    from = Math.min(from, timestamp);
    to = Math.max(to, timestamp + 1);
  }
  return topicTimeRangeSchema.parse({ from: new Date(from), to: new Date(to) });
}

async function assignSummaries(
  metrics: TopicMetrics,
  execution: TopicExecution,
  facet: TopicFacetVersion,
  progress: TopicFacetProgress,
  summaries: TopicSummary[],
  run: TopicRun,
  coordinates: Map<string, [number, number]> = new Map(),
  assignedAt = new Date().toISOString(),
) {
  return instrumentAsync(
    { name: "topics.assignment", traceScope: "topics" },
    (span) => {
      span.setAttribute("langfuse.topics.summaries", summaries.length);
      return metrics.measure("assignment", async () => {
        let assignedCount = 0;
        // Keep classification and readback scratch bounded independently of cohort size.
        for (let start = 0; start < summaries.length; start += 1000) {
          const batch = summaries.slice(start, start + 1000);
          const rows: ReturnType<typeof topicAssignmentRows> = [];
          for (let offset = 0; offset < batch.length; offset += 64) {
            rows.push(
              ...topicAssignmentRows({
                projectId: execution.projectId,
                facet: { facetId: facet.facetId, version: facet.version },
                embeddingConfig: execution.input.embeddingConfig,
                summaries: batch.slice(offset, offset + 64),
                run,
                origin:
                  execution.input.operation === "process"
                    ? "online"
                    : "initial",
                assignedAt,
                coordinates,
              }),
            );
            await yieldToEventLoop();
          }
          await metrics.measure(
            "storage",
            () => writeTopicAssignments(rows),
            "storage",
          );
          if (execution.input.operation === "update") {
            const visible = await readTopicAssignments(
              execution.projectId,
              { facetId: facet.facetId, version: facet.version },
              batch,
              run.id,
              summaryTimeRange(batch),
            );
            const visibleByTrace = new Map(
              visible.map((row) => [row.traceId, row]),
            );
            if (
              rows.some((row) => {
                const persisted = visibleByTrace.get(row.traceId);
                return (
                  !persisted ||
                  persisted.topicId !== row.topicId ||
                  persisted.topicVersionId !== row.topicVersionId ||
                  persisted.summaryProcessedAt !== row.summaryProcessedAt
                );
              })
            )
              throw new Error(
                "Topic assignments are not fully visible; map publication is deferred.",
              );
          }
          assignedCount += rows.filter((row) => row.topicId !== null).length;
          await yieldToEventLoop();
        }
        progress.counts.assigned = assignedCount;
        progress.counts.outlier = summaries.length - assignedCount;
        metrics.result("assignment", "assigned", progress.counts.assigned);
        metrics.result("assignment", "outlier", progress.counts.outlier);
      });
    },
  );
}

async function clusterFacet(
  metrics: TopicMetrics,
  models: TopicsModels,
  execution: UpdateExecution,
  facet: TopicFacetVersion,
  facetName: string | undefined,
  progress: TopicFacetProgress,
  summaries: TopicSummary[],
  attempt: TopicRun | null,
  previous: TopicRun | null,
) {
  const numericConfig = {
    ...topicClusterSettings(
      execution.input.exploratory,
      execution.input.minimumTraceCount,
    ),
    numericVersion: TOPICS_NUMERIC_VERSION,
  };
  const config = {
    ...numericConfig,
    exploratory: execution.input.exploratory,
    embeddingModel: execution.input.embeddingConfig.embeddingModel!,
    dimensions: execution.input.embeddingConfig.embeddingDimensions,
    classifierVersion: "original-cosine-loo95-rival05-eps1e-12-v2",
    timeRange: {
      from: execution.input.timeRange.from.toISOString(),
      to: execution.input.timeRange.to.toISOString(),
    },
  };
  let run =
    attempt?.status === "pending"
      ? await saveTopicRun({ ...attempt, config })
      : await createTopicRun({
          projectId: execution.projectId,
          facetId: facet.facetId,
          facetVersion: facet.version,
          config,
        });
  progress.runId = run.id;
  // Persist the attempt and selected cohort before any terminal result or inference.
  await saveProgress(metrics, execution, "clustering");
  if (summaries.length < numericConfig.minimumCount) {
    progress.outcome = summaries.length
      ? "insufficient_data"
      : "no_applicable_summaries";
    await saveTopicRun({
      ...run,
      status: "skipped",
      finishedAt: new Date().toISOString(),
    });
    metrics.result("clustering", progress.outcome);
    return;
  }
  run = await saveTopicRun({
    ...run,
    status: "running",
    startedAt: run.startedAt ?? new Date().toISOString(),
    finishedAt: null,
    error: null,
  });
  try {
    metrics.clusteringWorkload(
      summaries.length,
      execution.input.embeddingConfig.embeddingDimensions,
    );
    const numeric = await instrumentAsync(
      { name: "topics.clustering", traceScope: "topics" },
      async (span) => {
        span.setAttributes({
          "langfuse.topics.clustering.vectors": summaries.length,
          "langfuse.topics.clustering.dimensions":
            execution.input.embeddingConfig.embeddingDimensions,
        });
        const result = await metrics.measure(
          "clustering",
          () =>
            runTopicClustering(
              summaries.map((row) => row.embedding),
              numericConfig,
            ),
          "numerical",
        );
        span.setAttribute("langfuse.topics.clustering.status", result.status);
        return result;
      },
    );
    const coordinates = new Map(
      summaries
        .map((row, index) => [row.traceId, numeric.coordinates[index]])
        .filter(
          (entry): entry is [string, [number, number]] =>
            typeof entry[0] === "string" && Array.isArray(entry[1]),
        ),
    );
    const prototypes =
      numeric.status === "no_topics"
        ? []
        : await retainPopulatedTopics(
            summaries,
            (await buildTopicPrototypes(summaries, numeric.labels)).map(
              (prototype) => ({
                ...prototype,
                id: topicHash([run.id, prototype.id]).slice(0, 48),
              }),
            ),
          );
    if (!prototypes.length) {
      metrics.result("clustering", "no_topics");
      await assignSummaries(
        metrics,
        execution,
        facet,
        progress,
        summaries,
        run,
        coordinates,
      );
      await saveTopicRun({
        ...run,
        status: "completed",
        finishedAt: new Date().toISOString(),
      });
      progress.outcome = "no_topics";
      return;
    }
    metrics.result("clustering", "topics_found");
    let candidates = prototypes.map((prototype) => ({
      topicVersionId: prototype.id,
      topicId: prototype.id,
      centroid: prototype.centroid,
      radius: prototype.radius,
      metadata: {},
    }));
    const compatiblePrevious =
      previous !== null && compatibleMap(execution, facet, previous);
    if (previous) {
      const facetRef = { facetId: facet.facetId, version: facet.version };
      const previousTimeRange = topicTimeRangeSchema.parse(
        previous.config.timeRange,
      );
      const previousTraceIds = await readTopicRunTraceIds(
        execution.projectId,
        facetRef,
        previous.id,
        previousTimeRange,
        env.LANGFUSE_TOPICS_MAX_CLUSTERING_ROWS,
      );
      const previousMemberships: {
        traceId: string;
        topicVersionId: string | null;
      }[] = [];
      for (let start = 0; start < previousTraceIds.length; start += 1000) {
        const traceIds = previousTraceIds.slice(start, start + 1000);
        const [previousSummaries, previousAssignments] = await Promise.all([
          listTopicSummaries(
            execution.projectId,
            { facetId: facet.facetId, facetVersion: facet.version, traceIds },
            previousTimeRange,
          ),
          readTopicAssignments(
            execution.projectId,
            facetRef,
            traceIds.map((traceId) => ({ traceId, sessionId: null })),
            previous.id,
            previousTimeRange,
          ),
        ]);
        const eligible = new Set(previousSummaries.map((row) => row.traceId));
        for (const row of previousAssignments) {
          if (row.traceId !== null && eligible.has(row.traceId))
            previousMemberships.push({
              traceId: row.traceId,
              topicVersionId: row.topicVersionId,
            });
        }
      }
      const candidateMemberships: {
        traceId: string;
        topicVersionId: string | null;
      }[] = [];
      const yieldIfNeeded = createEventLoopYielder();
      for (const summary of summaries) {
        if (summary.traceId === null)
          throw new Error("Topics processing requires a trace summary.");
        candidateMemberships.push({
          traceId: summary.traceId,
          topicVersionId: classifyTopic(summary.embedding, prototypes).topicId,
        });
        const pause = yieldIfNeeded();
        if (pause) await pause;
      }
      candidates = matchTopicContinuity({
        previousTopics: previous.topics,
        candidateTopics: candidates,
        previousMemberships,
        candidateMemberships,
        exploratory: execution.input.exploratory,
        compatibleEmbeddingSpace: compatiblePrevious,
      });
    }
    const reused = new Map<string, TopicDefinition>();
    candidates = candidates.map((candidate) => {
      const old = previous?.topics.find(
        (topic) => topic.topicId === candidate.topicId,
      );
      if (
        compatiblePrevious &&
        old &&
        !old.metadata.namingFailure &&
        sameTopicGeometry(old, candidate)
      ) {
        reused.set(old.topicVersionId, old);
        return {
          ...candidate,
          topicVersionId: old.topicVersionId,
          centroid: old.centroid,
          radius: old.radius,
        };
      }
      return candidate;
    });
    // Final version IDs also determine the serving classifier's distance ties.
    const evidence = await buildNamingEvidence(
      summaries,
      await retainPopulatedTopics(
        summaries,
        candidates.map((candidate) => ({
          id: candidate.topicVersionId,
          centroid: candidate.centroid,
          radius: candidate.radius,
        })),
      ),
    );
    await saveProgress(metrics, execution, "naming");
    const topics: TopicDefinition[] = [];
    const named: NamedTopics = new Map();
    for (const group of evidence) {
      const retained = reused.get(group.id);
      if (retained)
        named.set(retained.name.toLowerCase(), {
          name: retained.name,
          description: retained.description,
          summaries: group.members.slice(0, 3).map((member) => member.summary),
        });
    }
    for (const group of evidence) {
      const retained = reused.get(group.id);
      if (retained) {
        topics.push(retained);
        continue;
      }
      const label = await instrumentAsync(
        { name: "topics.naming", traceScope: "topics" },
        (span) => {
          span.setAttributes({
            "langfuse.topics.group_id": group.id,
            "langfuse.topics.group_members": group.members.length,
          });
          return metrics.measure("naming", async () => {
            const prepared = await prepareTopicNamingGroup(
              {
                members: group.members,
                contrasts: group.contrasts.map((member, index) => ({
                  id: `c${index + 1}`,
                  summary: member.summary,
                })),
              },
              topicHash([run.id, group.id]),
            );
            const label = await nameGroupAvoidingConflicts(
              metrics,
              models,
              named,
              { ...prepared, facetName },
              {
                facet_id: facet.facetId,
                facet_name: facetName,
                facet_version: facet.version,
                execution_id: execution.id,
                run_id: run.id,
                group_id: group.id,
                effective_member_count: group.count,
                selected_member_count: prepared.members.length,
              },
            );
            return {
              ...label,
              selectedMemberCount: prepared.members.length,
              namingSampled: prepared.sampled,
              representativeSummaries: label.evidenceSummaryIds.map((id) => {
                const { facetId, facetVersion, traceId, sessionId } =
                  prepared.members.find((member) => member.id === id)!.source;
                return {
                  facetId,
                  facetVersion,
                  traceId,
                  sessionId: traceId === null ? sessionId : null,
                };
              }),
            };
          });
        },
      ).catch((error: unknown) => {
        if (
          error instanceof TopicsProviderUnavailable &&
          error.reason === "content_filter"
        )
          return null;
        throw error;
      });
      const candidate = candidates.find(
        (topic) => topic.topicVersionId === group.id,
      )!;
      topics.push({
        ...candidate,
        projectId: execution.projectId,
        createdByRunId: run.id,
        createdAt: run.createdAt,
        tags: [],
        name: label?.name ?? `Unnamed topic ${topics.length + 1}`,
        description:
          label?.description ??
          "The model's content filter prevented naming this topic. Its traces and assignments were preserved.",
        representativeSummaries: label?.representativeSummaries ?? [],
        metadata: {
          ...candidate.metadata,
          effectiveMemberCount: group.count,
          ...(label && {
            selectedMemberCount: label.selectedMemberCount,
            namingSampled: label.namingSampled,
          }),
          namingModel: models.clustering.model,
          ...(!label && { namingFailure: "content_filter" }),
        },
      });
      if (label) metrics.result("naming", "generated");
    }
    run = await saveTopicRun({ ...run, topics });
    // Readback returns the persisted classifier precision before creating any memberships.
    await assignSummaries(
      metrics,
      execution,
      facet,
      progress,
      summaries,
      run,
      coordinates,
    );
    const unnamed = run.topics.filter(
      (topic) => topic.metadata.namingFailure === "content_filter",
    ).length;
    run = await saveTopicRun({
      ...run,
      status: "completed",
      error: unnamed
        ? `The model's content filter prevented naming ${unnamed} topic${unnamed === 1 ? "" : "s"}. Traces and assignments were preserved. Choose another clustering model, then run Update topics to retry naming.`
        : null,
      finishedAt: new Date().toISOString(),
    });
    progress.outcome = "published";
    progress.error = run.error;
  } catch (error) {
    await saveTopicRun({
      ...run,
      status: "failed",
      error: errorMessage(error),
      finishedAt: new Date().toISOString(),
    });
    throw error;
  }
}

type NamedTopics = Map<
  string,
  { name: string; description: string; summaries: string[] }
>;

/** Groups are named independently, so a name taken in the run gets one conflict-aware retry. */
async function nameGroupAvoidingConflicts(
  metrics: TopicMetrics,
  models: TopicsModels,
  named: NamedTopics,
  group: Parameters<typeof nameTopicGroup>[1],
  metadata: Record<string, unknown>,
) {
  let { output: label } = await nameTopicGroup(models, group, metadata);
  const conflict = named.get(label.name.toLowerCase());
  if (conflict) {
    metrics.result("naming", "name_conflict");
    // The first name is valid, so a failed rename keeps it.
    label = await nameTopicGroup(models, group, metadata, conflict).then(
      ({ output }) => output,
      (error: unknown) => {
        metrics.error("naming", error);
        return label;
      },
    );
  }
  if (named.has(label.name.toLowerCase()))
    metrics.result("naming", "duplicate_name");
  else
    named.set(label.name.toLowerCase(), {
      name: label.name,
      description: label.description,
      summaries: label.evidenceSummaryIds.map(
        (id) => group.members.find((member) => member.id === id)!.summary,
      ),
    });
  return label;
}

function compatibleMap(
  execution: TopicExecution,
  facet: TopicFacetVersion,
  run: TopicRun,
): boolean {
  return (
    run.facetId === facet.facetId &&
    run.facetVersion === facet.version &&
    run.config.embeddingModel ===
      execution.input.embeddingConfig.embeddingModel &&
    run.config.dimensions ===
      execution.input.embeddingConfig.embeddingDimensions
  );
}

type PendingFacet = {
  facet: TopicFacetVersion;
  progress: TopicFacetProgress;
  name: string;
  builtIn: boolean;
};

async function pendingFacets(
  execution: TopicExecution,
): Promise<PendingFacet[]> {
  const facets: PendingFacet[] = [];
  const definitions = await listTopicFacets(execution.projectId);
  for (const progress of execution.facets) {
    if (progress.outcome !== "pending") continue;
    const facet = await getTopicFacetVersion(
      execution.projectId,
      progress.facetId,
      progress.facetVersion,
    );
    const definition = definitions.find(({ id }) => id === progress.facetId);
    if (!facet || !definition)
      throw new Error("Facet version not found in this project.");
    progress.counts.failed = 0;
    facets.push({
      facet,
      progress,
      name: definition.name,
      builtIn: definition.isBuiltIn,
    });
  }
  return facets;
}

/** Processes one bounded queue batch; the job owns retry state until completion. */
async function processTraces(
  metrics: TopicMetrics,
  models: TopicsModels,
  execution: ProcessExecution,
  state: TopicProcessBatchState,
  batchId: string,
  retryFailed: boolean,
  save: (phase: string) => Promise<void>,
) {
  const facets: (PendingFacet & {
    run: TopicRun | null;
  })[] = [];
  for (const pending of await pendingFacets(execution)) {
    const { facet, progress } = pending;
    progress.counts.failed =
      state.failedTraceIds.find(
        (entry) =>
          entry.facetId === facet.facetId &&
          entry.facetVersion === facet.version,
      )?.traceIds.length ?? 0;
    const runId = progress.runId;
    const run = runId ? await getTopicRun(execution.projectId, runId) : null;
    if (
      runId &&
      (run?.status !== "completed" || !compatibleMap(execution, facet, run))
    )
      throw new Error("The selected topic map is unavailable or incompatible.");
    facets.push({ ...pending, run });
  }
  if (!facets.length) return;
  if (!state.summarized) {
    const cached = new Map<TopicFacetVersion, TopicSummary[]>();
    for (const { facet } of facets)
      cached.set(
        facet,
        await readStagedTopicSummaries(
          execution.projectId,
          execution.id,
          facet.facetId,
          facet.version,
          execution.input.traceIds,
        ),
      );
    for (const { facet } of facets) {
      const available = new Set(cached.get(facet)!.map((row) => row.traceId));
      if (
        state.summaries.some(
          (ref) =>
            ref.facetId === facet.facetId &&
            ref.facetVersion === facet.version &&
            !available.has(ref.traceId),
        )
      )
        throw new TopicEmbeddingFailure(TOPIC_EMBEDDING_EXPIRED_ERROR);
    }
    // Traces run concurrently; a trace's pending facets share one transcript and summary call.
    const limit = pLimit(env.LANGFUSE_TOPICS_TRACE_CONCURRENCY);
    let saving = Promise.resolve();
    // Saves write the whole state, so they must not land out of order.
    const saveInOrder = () => {
      const saved = saving.then(() => save("summarizing"));
      saving = saved.catch(() => {});
      return saved;
    };
    let aborted = false;
    const results = await Promise.allSettled(
      execution.input.traceIds.map((traceId) =>
        limit(async () => {
          if (aborted) return;
          let input: ReturnType<typeof loadTopicTranscript> | undefined;
          const getTranscript = () => {
            if (!input)
              input = metrics.measure(
                "transcript",
                () =>
                  loadTopicTranscript({
                    projectId: execution.projectId,
                    traceId,
                  }),
                "trace_load",
              );
            return input;
          };
          const todo = facets.filter(
            ({ facet }) =>
              !state.summaries.some(
                (ref) =>
                  ref.facetId === facet.facetId &&
                  ref.facetVersion === facet.version &&
                  ref.traceId === traceId,
              ) &&
              !state.failedTraceIds.some(
                (entry) =>
                  entry.facetId === facet.facetId &&
                  entry.facetVersion === facet.version &&
                  entry.traceIds.includes(traceId),
              ),
          );
          if (todo.length === 0) return;
          let outcomes: (TopicSummary | Error)[];
          try {
            outcomes = await instrumentAsync(
              { name: "topics-trace-summary", traceScope: "topics" },
              async (span) => {
                span.setAttributes({
                  "langfuse.trace.id": traceId,
                  "langfuse.topics.execution_id": execution.id,
                });
                const results = await summarizeTrace(
                  metrics,
                  models,
                  execution,
                  todo.map(({ facet, name, builtIn }) => ({
                    version: facet,
                    name,
                    builtIn,
                  })),
                  traceId,
                  cached,
                  getTranscript,
                );
                span.setAttribute(
                  "langfuse.topics.trace_outcome",
                  results.some((result) => result instanceof Error)
                    ? "failed"
                    : "summarized",
                );
                return results;
              },
            );
          } catch (error) {
            metrics.error("summary", error);
            if (isExecutionFailure(error) && !isTraceModelFailure(error)) {
              aborted = true;
              throw error;
            }
            outcomes = todo.map(() => error as Error);
          }
          for (const [index, { facet, progress }] of todo.entries()) {
            const outcome = outcomes[index];
            const ref = {
              facetId: facet.facetId,
              facetVersion: facet.version,
              traceId,
            };
            // Invalid facet outputs are already counted by the shared summary call.
            if (outcome instanceof Error)
              recordTraceFailure(state, progress, ref, outcome);
            else state.summaries.push(ref);
          }
          await saveInOrder();
        }),
      ),
    );
    const failure = results.find((result) => result.status === "rejected");
    if (failure) throw failure.reason;
    state.summarized = true;
    await save("embedding");
  }
  if (!state.assignedAt) {
    await awaitEmbeddingBatch(execution, batchId, state.summaries, retryFailed);
    // This saved timestamp acknowledges the summary insert and fixes assignment
    // ordering across retries, even if the completed embedding job is removed.
    state.assignedAt = new Date().toISOString();
    await save("assigning");
  }
  const stagedSummaries = await Promise.all(
    state.summaries
      .filter((ref) =>
        facets.some(
          ({ facet }) =>
            ref.facetId === facet.facetId && ref.facetVersion === facet.version,
        ),
      )
      .map(async (ref) => {
        const staged = await readStagedTopicSummary(
          { projectId: execution.projectId, executionId: execution.id },
          ref,
        );
        if (!staged)
          throw new TopicEmbeddingFailure(TOPIC_EMBEDDING_EXPIRED_ERROR);
        return { ref, summary: staged.summary };
      }),
  );
  const summaries: TopicSummary[] = [];
  for (const { ref, summary } of stagedSummaries) {
    if (isInternalEvalEnvironment(summary.environment)) {
      const { progress } = facets.find(
        ({ facet }) =>
          facet.facetId === ref.facetId && facet.version === ref.facetVersion,
      )!;
      const error = new Error(
        "Topics does not process internal Langfuse traces.",
      );
      metrics.error("summary", error);
      recordTraceFailure(state, progress, ref, error);
      state.summaries = state.summaries.filter((entry) => entry !== ref);
      continue;
    }
    if (summary.state === "summarized")
      throw new TopicEmbeddingFailure(
        "Topics embedding batch has an unfinished result.",
      );
    summaries.push(summary);
  }
  for (const { facet, progress, run } of facets) {
    try {
      const selected = summaries.filter(
        (row) =>
          row.facetId === facet.facetId && row.facetVersion === facet.version,
      );
      countSummaries(progress, selected);
      const complete = selected.filter((row) => row.state === "complete");
      if (!complete.length) progress.outcome = "no_applicable_summaries";
      else if (!run) {
        progress.outcome = "awaiting_topics";
        metrics.result("assignment", "awaiting_topics", complete.length);
      } else {
        progress.runId = run.id;
        await save("assigning");
        await assignSummaries(
          metrics,
          execution,
          facet,
          progress,
          complete,
          run,
          undefined,
          state.assignedAt,
        );
        progress.outcome = "assigned";
      }
    } catch (error) {
      failFacet(metrics, execution, facet, progress, error);
    }
    await save("processing");
  }
}

function recordTraceFailure(
  state: TopicProcessBatchState,
  progress: TopicFacetProgress,
  { facetId, facetVersion, traceId }: TopicEmbeddingRef,
  error: unknown,
) {
  let failed = state.failedTraceIds.find(
    (entry) => entry.facetId === facetId && entry.facetVersion === facetVersion,
  );
  if (!failed) {
    failed = { facetId, facetVersion, traceIds: [] };
    state.failedTraceIds.push(failed);
  }
  if (!failed.traceIds.includes(traceId)) failed.traceIds.push(traceId);
  progress.counts.failed = failed.traceIds.length;
  const message = errorMessage(error);
  if (
    !state.execution.traceErrors.some(
      (item) => item.traceId === traceId && item.error === message,
    )
  )
    state.execution.traceErrors.push({ traceId, error: message });
}

/** Each unpublished attempt fits the current compatible summaries from scratch. */
async function updateTopics(
  metrics: TopicMetrics,
  models: TopicsModels,
  execution: UpdateExecution,
) {
  const facetNames = new Map(
    (await listTopicFacets(execution.projectId)).map((facet) => [
      facet.id,
      facet.name,
    ]),
  );
  for (const { facet, progress } of await pendingFacets(execution)) {
    await instrumentAsync(
      { name: "topics-facet-update", traceScope: "topics" },
      async (span) => {
        span.setAttributes({
          "langfuse.topics.execution_id": execution.id,
          "langfuse.topics.facet_id": facet.facetId,
          "langfuse.topics.facet_version": facet.version,
        });
        try {
          const accepted = progress.runId
            ? await getTopicRun(execution.projectId, progress.runId)
            : null;
          if (
            accepted &&
            (accepted.projectId !== execution.projectId ||
              accepted.facetId !== facet.facetId ||
              accepted.facetVersion !== facet.version)
          )
            throw new Error(
              "Clustering run does not match this facet version.",
            );
          if (accepted?.status === "completed") {
            progress.error = accepted.error;
            const assignments = await readTopicMapAssignments(
              execution.projectId,
              { facetId: facet.facetId, version: facet.version },
              accepted.id,
              topicTimeRangeSchema.parse(accepted.config.timeRange),
            );
            progress.counts.requested = assignments.length;
            progress.counts.complete = assignments.length;
            progress.counts.assigned = assignments.filter(
              (row) => row.topicId !== null,
            ).length;
            progress.counts.outlier =
              assignments.length - progress.counts.assigned;
            progress.outcome = accepted.topics.length
              ? "published"
              : "no_topics";
            await saveProgress(metrics, execution, "processing");
            return;
          }
          if (accepted?.status === "skipped") {
            progress.outcome = progress.counts.requested
              ? "insufficient_data"
              : "no_applicable_summaries";
            await saveProgress(metrics, execution, "processing");
            return;
          }
          const current = await getPublishedTopicRun(
            execution.projectId,
            facet.facetId,
          );
          const previous =
            current?.facetVersion === facet.version ? current : null;
          const summaries = await getTopicClusteringSummaries(
            execution.projectId,
            facet.facetId,
            facet.version,
            execution.input.embeddingConfig,
            execution.input.timeRange,
            {
              maxRows: env.LANGFUSE_TOPICS_MAX_CLUSTERING_ROWS,
              maxBytes: env.LANGFUSE_TOPICS_MAX_CLUSTERING_BYTES,
            },
          ).catch(async (error: unknown) => {
            if (accepted?.status === "pending")
              await saveTopicRun({
                ...accepted,
                status: "failed",
                error: errorMessage(error),
                finishedAt: new Date().toISOString(),
              });
            throw error;
          });
          countSummaries(progress, summaries);
          progress.counts.requested = summaries.length;
          await clusterFacet(
            metrics,
            models,
            execution,
            facet,
            facetNames.get(facet.facetId),
            progress,
            summaries,
            accepted,
            previous,
          );
        } catch (error) {
          failFacet(metrics, execution, facet, progress, error);
        }
        await saveProgress(metrics, execution, "processing");
        span.setAttribute("langfuse.topics.facet_outcome", progress.outcome);
      },
    );
  }
}

/** Provider and configuration failures are the customer's to fix; anything else is ours. */
function failFacet(
  metrics: TopicMetrics,
  execution: TopicExecution,
  facet: TopicFacetVersion,
  progress: TopicFacetProgress,
  error: unknown,
) {
  metrics.error("execution", error);
  progress.outcome = "failed";
  progress.error = errorMessage(error);
  const log =
    error instanceof TopicsProviderUnavailable ? logger.warn : logger.error;
  log("Topics facet failed", {
    "langfuse.topics.execution_id": execution.id,
    "langfuse.topics.operation": execution.input.operation,
    "langfuse.topics.facet_id": facet.facetId,
    "langfuse.topics.facet_version": facet.version,
    error: progress.error,
  });
  if (isExecutionFailure(error)) throw error;
  // Handled, but still a failed facet for the user; the enclosing span reports it.
  traceException(error);
}

/** Queue batches own trace retry state; Postgres owns compact update progress. */
export async function processTopicsExecution({
  projectId,
  executionId,
  traceIds,
  batchId,
  batchState,
  saveBatchState,
}: {
  projectId: string;
  executionId: string;
  traceIds?: string[];
  batchId?: string;
  batchState?: TopicProcessBatchState;
  saveBatchState?: (state: TopicProcessBatchState) => Promise<void>;
}): Promise<void> {
  if (!isTopicsEnabled()) throw new Error("Topics processing is not enabled.");
  const metadata = await readTopicExecutionSummary(projectId, executionId);
  if (!metadata) throw new Error("Topics execution not found.");
  const processing = metadata.input.operation === "process";
  if (
    processing &&
    (!traceIds?.length || traceIds.length > 100 || !batchId || !saveBatchState)
  )
    throw new Error(
      "Trace processing requires a queue batch of 1–100 trace IDs and its retry-state writer.",
    );
  if (
    batchState &&
    (batchState.execution.projectId !== projectId ||
      batchState.execution.id !== executionId ||
      batchState.execution.input.operation !== "process" ||
      JSON.stringify(batchState.execution.input.traceIds) !==
        JSON.stringify(traceIds))
  )
    throw new Error("Topics batch state does not match its queue payload.");
  const execution: TopicExecution = batchState?.execution ?? {
    ...metadata,
    input:
      metadata.input.operation === "process"
        ? { ...metadata.input, traceIds: traceIds! }
        : metadata.input,
    status: processing ? "queued" : metadata.status,
    phase: processing ? "queued" : metadata.phase,
    facets: metadata.facets.map((facet) => ({
      ...facet,
      ...(processing
        ? {
            outcome: "pending" as const,
            error: null,
            counts: {
              requested: traceIds!.length,
              complete: 0,
              nonApplicable: 0,
              insufficientInput: 0,
              failed: 0,
              assigned: 0,
              outlier: 0,
            },
          }
        : {}),
    })),
    traceErrors: [],
    error: processing ? null : metadata.error,
  };
  if (
    ["completed", "completed_with_errors"].includes(execution.status) &&
    !execution.facets.some(
      (facet) => facet.outcome === "pending" || facet.outcome === "failed",
    )
  ) {
    if (batchState) await releaseCompletedSummaries(batchState);
    return;
  }
  const state: TopicProcessBatchState = batchState ?? {
    execution,
    summaries: [],
    failedTraceIds: [],
    summarized: false,
  };
  const metrics = new TopicMetrics();
  const save = async (phase: string) => {
    execution.phase = phase;
    if (!processing) return saveProgress(metrics, execution, phase);
    try {
      await saveBatchState!(state);
    } catch (error) {
      throw new TopicEmbeddingFailure(errorMessage(error));
    }
  };
  const retryFailed =
    execution.status === "failed" ||
    execution.status === "completed_with_errors";
  if (execution.status !== "running") metrics.execution("started");
  execution.status = "running";
  execution.error = null;
  for (const progress of execution.facets) {
    if (progress.outcome === "failed") {
      progress.outcome = "pending";
      progress.error = null;
    }
  }
  const phase = execution.phase === "embedding" ? "embedding" : "summarizing";
  await save(processing ? phase : "selecting");
  try {
    const models = await getTopicsModels(projectId);
    if (!models)
      throw new TopicsProviderUnavailable(
        "Choose Topics models for this project before running Topics.",
        "configuration",
      );
    if (
      execution.input.embeddingConfig.embeddingModel !== models.embedding.model
    )
      throw new TopicsProviderUnavailable(
        "The project's embedding model changed after this execution was created; start a new Topics execution.",
        "configuration",
      );
    if (execution.input.operation === "process") {
      if (
        execution.input.processingConfig.summaryModel !== models.summary.model
      )
        throw new TopicsProviderUnavailable(
          "The project's summary model changed after this execution was created; start a new Topics execution.",
          "configuration",
        );
      await processTraces(
        metrics,
        models,
        execution as ProcessExecution,
        state,
        batchId!,
        retryFailed,
        save,
      );
    } else {
      await updateTopics(metrics, models, execution as UpdateExecution);
    }
    execution.status = execution.facets.some(
      (facet) =>
        facet.outcome === "failed" ||
        facet.counts.failed > 0 ||
        facet.error !== null,
    )
      ? "completed_with_errors"
      : "completed";
    await save("completed");
    if (processing) await releaseCompletedSummaries(state);
    metrics.execution(execution.status);
  } catch (error) {
    if (error instanceof PendingTopicEmbeddings) {
      await save("embedding");
      return;
    }
    if (isRejectedConnection(error))
      await pauseForRejectedConnection(projectId, error);
    metrics.error("execution", error);
    metrics.execution("failed");
    execution.status = "failed";
    execution.error = errorMessage(error);
    await save("failed");
  } finally {
    getCurrentSpan()?.setAttributes({
      "langfuse.project.id": projectId,
      "langfuse.topics.execution_id": executionId,
      "langfuse.topics.operation": execution.input.operation,
      "langfuse.topics.execution_status": execution.status,
      "langfuse.topics.execution_phase": execution.phase,
    });
  }
}