Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

6.2 KB
Raw Blame
import type { TopicDefinition } from "@langfuse/shared/topics";
import { normalizeVector } from "./classifier";

type Membership = {
  traceId: string;
  topicVersionId: string | null;
};

function membershipIndex(
  rows: readonly Membership[],
  topics: readonly Pick<TopicDefinition, "topicVersionId">[],
) {
  const ids = new Set(topics.map((topic) => topic.topicVersionId));
  const result = new Map<string, Membership>();
  for (const row of rows) {
    if (row.topicVersionId !== null && !ids.has(row.topicVersionId))
      throw new Error("Topic membership references an unknown version");
    const previous = result.get(row.traceId);
    if (previous && previous.topicVersionId !== row.topicVersionId)
      throw new Error("Topic cohort contains conflicting memberships");
    result.set(row.traceId, row);
  }
  return result;
}

function cosineDistance(left: number[], right: number[]) {
  if (left.length !== right.length)
    throw new Error("Topic embedding dimensions differ");
  const a = normalizeVector(left);
  const b = normalizeVector(right);
  return Math.max(
    0,
    Math.min(2, 1 - a.reduce((sum, x, i) => sum + x * b[i], 0)),
  );
}

/**
 * Matches final serving memberships, independently of cluster labels and names.
 * These conservative PoC thresholds are provisional: at least 80% of both common
 * populations must agree, with 10 anchors (3 exploratory) and half of the old
 * population still represented. Material branches need 5 anchors (3 exploratory)
 * and 20% of either common population; all split/merge children start new IDs.
 */
export function matchTopicContinuity<
  T extends Pick<
    TopicDefinition,
    "topicVersionId" | "topicId" | "centroid" | "metadata"
  >,
>(input: {
  previousTopics: readonly TopicDefinition[];
  candidateTopics: readonly T[];
  previousMemberships: readonly Membership[];
  candidateMemberships: readonly Membership[];
  exploratory: boolean;
  compatibleEmbeddingSpace: boolean;
}): T[] {
  const previous = membershipIndex(
    input.previousMemberships,
    input.previousTopics,
  );
  const candidate = membershipIndex(
    input.candidateMemberships,
    input.candidateTopics,
  );
  const oldTotal = new Map<string, number>();
  const oldCommon = new Map<string, number>();
  const newCommon = new Map<string, number>();
  const overlaps = new Map<string, Map<string, number>>();
  const increment = (counts: Map<string, number>, id: string | null) => {
    if (id !== null) counts.set(id, (counts.get(id) ?? 0) + 1);
  };
  for (const [key, row] of previous) {
    increment(oldTotal, row.topicVersionId);
    const current = candidate.get(key);
    if (!current) continue;
    increment(oldCommon, row.topicVersionId);
    increment(newCommon, current.topicVersionId);
    if (row.topicVersionId === null || current.topicVersionId === null)
      continue;
    const counts =
      overlaps.get(current.topicVersionId) ?? new Map<string, number>();
    increment(counts, row.topicVersionId);
    overlaps.set(current.topicVersionId, counts);
  }
  const minimumShared = input.exploratory ? 3 : 10;
  const minimumBranch = input.exploratory ? 3 : 5;
  const pairs = input.candidateTopics.flatMap((current) =>
    input.previousTopics.flatMap((old) => {
      const sharedCount =
        overlaps.get(current.topicVersionId)?.get(old.topicVersionId) ?? 0;
      if (!sharedCount) return [];
      const previousCoverage = sharedCount / oldCommon.get(old.topicVersionId)!;
      const candidateCoverage =
        sharedCount / newCommon.get(current.topicVersionId)!;
      return [
        {
          previousTopic: old,
          candidateId: current.topicVersionId,
          sharedCount,
          previousCoverage,
          candidateCoverage,
          commonAnchorCoverage:
            oldCommon.get(old.topicVersionId)! /
            oldTotal.get(old.topicVersionId)!,
          centroidDistance: input.compatibleEmbeddingSpace
            ? cosineDistance(old.centroid, current.centroid)
            : null,
          material:
            sharedCount >= minimumBranch &&
            (previousCoverage >= 0.2 || candidateCoverage >= 0.2),
        },
      ];
    }),
  );
  const materialSuccessors = new Map<string, number>();
  for (const pair of pairs)
    if (pair.material)
      increment(materialSuccessors, pair.previousTopic.topicVersionId);

  return input.candidateTopics.map((current) => {
    const candidates = pairs
      .filter((pair) => pair.candidateId === current.topicVersionId)
      .sort(
        (a, b) =>
          b.sharedCount - a.sharedCount ||
          a.previousTopic.topicVersionId.localeCompare(
            b.previousTopic.topicVersionId,
          ),
      );
    const predecessors = candidates.filter((pair) => pair.material);
    const merged = predecessors.length > 1;
    const split = predecessors.some(
      (pair) =>
        (materialSuccessors.get(pair.previousTopic.topicVersionId) ?? 0) > 1,
    );
    const best = candidates[0];
    // Reciprocal 80% support implies a unique best match in both directions.
    const continued =
      !merged &&
      !split &&
      best &&
      best.sharedCount >= minimumShared &&
      best.previousCoverage >= 0.8 &&
      best.candidateCoverage >= 0.8 &&
      best.commonAnchorCoverage >= 0.5 &&
      (best.centroidDistance === null || best.centroidDistance <= 0.15);
    let status = "new";
    if (continued) status = "continued";
    else if (merged && split) status = "ambiguous";
    else if (merged) status = "merge";
    else if (split) status = "split";
    return {
      ...current,
      topicId: continued ? best.previousTopic.topicId : current.topicId,
      metadata: {
        ...current.metadata,
        predecessorTopicIds: predecessors
          .map((pair) => pair.previousTopic.topicId)
          .sort(),
        continuity: {
          version: 1,
          status,
          sharedCount: best?.sharedCount ?? 0,
          previousCoverage: best?.previousCoverage ?? 0,
          candidateCoverage: best?.candidateCoverage ?? 0,
          commonAnchorCoverage: best?.commonAnchorCoverage ?? 0,
          centroidDistance: best?.centroidDistance ?? null,
          predecessorTopicVersionIds: predecessors
            .map((pair) => pair.previousTopic.topicVersionId)
            .sort(),
        },
      },
    };
  });
}