Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

33.6 KB
Raw Blame
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { encrypt } from "@langfuse/shared/encryption";

const state = vi.hoisted(() => ({
  env: { LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS: ["project"] },
  call: vi.fn(),
  embed: vi.fn(),
  increment: vi.fn(),
  project: vi.fn(),
  traceSink: vi.fn(),
  responseMetadata: vi.fn(),
  spanAttributes: vi.fn(),
  encoding: null as {
    encode: ReturnType<typeof vi.fn>;
    free: ReturnType<typeof vi.fn>;
  } | null,
}));
vi.mock("tiktoken", async (importOriginal) => {
  const actual = await importOriginal<typeof import("tiktoken")>();
  return {
    ...actual,
    get_encoding: (...args: Parameters<typeof actual.get_encoding>) =>
      state.encoding ?? actual.get_encoding(...args),
  };
});
vi.mock("@langfuse/shared/src/env", () => ({ env: state.env }));
vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({
  ...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),
  getLangfuseAITraceSinkParams: state.traceSink,
  LangfuseInternalTraceEnvironment: { Topics: "langfuse-topics" },
  logger: { warn: vi.fn() },
  recordIncrement: state.increment,
  instrumentAsync: async (
    _options: unknown,
    call: (span: { setAttributes: typeof state.spanAttributes }) => unknown,
  ) => call({ setAttributes: state.spanAttributes }),
}));
vi.mock("@langfuse/shared/src/db", () => ({
  prisma: { project: { findFirst: state.project } },
}));
vi.mock("@langfuse/shared/topics/server", async (importOriginal) => ({
  topicMessages: (
    await importOriginal<typeof import("@langfuse/shared/topics/server")>()
  ).topicMessages,
  generateTopicText: (...args: unknown[]) => state.call(...args),
  generateTopicEmbeddings: (...args: unknown[]) => state.embed(...args),
  getTopicTextResponseMetadata: (error: unknown) =>
    state.responseMetadata(error),
}));

// Token counters only; every call also records call and request-byte counters.
const tokenCalls = () =>
  state.increment.mock.calls.filter(([name]) =>
    String(name).startsWith("langfuse.topics.token"),
  );

import {
  embedTopicSummaries,
  nameTopicGroup as namePreparedTopicGroup,
  prepareTopicNamingGroup,
  summarizeTopicTraceFacets,
} from "./models";
import {
  topicProcessingConfigSchema,
  type TopicProcessingConfig,
} from "@langfuse/shared/topics";
import { LLMAdapter } from "@langfuse/shared";
import { logger } from "@langfuse/shared/src/server";
import { recordTopicModelCall } from "./metrics";
import type { TopicsModel, TopicsModels } from "@langfuse/shared/topics/server";

async function nameTopicGroup(
  models: TopicsModels,
  group: Parameters<typeof prepareTopicNamingGroup>[0],
) {
  return namePreparedTopicGroup(
    models,
    await prepareTopicNamingGroup(group, "naming-test"),
  );
}

function slot(slot: TopicsModel["slot"], model: string): TopicsModel {
  return {
    slot,
    provider: "bedrock",
    adapter: LLMAdapter.Bedrock,
    model,
    connection: { secretKey: "encrypted", config: { region: "eu-west-1" } },
  };
}

function topicsModels(summaryModel = "us.openai.gpt-5.6-luna"): TopicsModels {
  return {
    projectId: "project",
    enabled: true,
    summary: slot("summary", summaryModel),
    embedding: {
      ...slot("embedding", "eu.cohere.embed-v4:0"),
      dimensions: 256,
    },
    clustering: slot("clustering", "us.openai.gpt-5.6-terra"),
  };
}

const facet = {
  projectId: "project",
  facetId: "facet",
  version: 1,
  prompt: "Describe the user's request.",
  createdAt: "2026-09-16T00:00:00Z",
};

const evidence = {
  members: [
    { id: "m1", summary: "The user requests an invoice." },
    { id: "m2", summary: "The user requests billing corrections." },
  ],
  contrasts: [{ id: "contrast", summary: "The user requests translation." }],
};

beforeEach(() => {
  state.env.LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS = ["project"];
  state.call.mockReset();
  state.embed.mockReset();
  state.increment.mockReset();
  state.responseMetadata.mockReset();
  state.spanAttributes.mockReset();
  vi.mocked(logger.warn).mockClear();
  state.project.mockReset().mockResolvedValue({
    organization: { aiFeaturesEnabled: true, aiTelemetryEnabled: true },
  });
  state.encoding = null;
  state.traceSink.mockReset().mockImplementation((params) => ({
    targetProjectId: "ai-features",
    traceId: "a".repeat(32),
    traceName: params.traceName,
    environment: "production",
    aiFeatureOtelIngestion: true,
    metadata: {
      langfuse_ai_feature: params.feature,
      langfuse_project_id: params.projectId,
      ...params.metadata,
    },
  }));
});

afterEach(() => {
  vi.unstubAllGlobals();
});

// One built-in facet through the shared multi-facet request.
const summarizeOne = (
  models: TopicsModels,
  text: string,
  config: TopicProcessingConfig,
) =>
  summarizeTopicTraceFacets(
    models,
    [{ key: "intent", facet, builtIn: true }],
    text,
    config,
  );
const keyed = (output: Record<string, unknown>) => ({
  intent: { notes: "Evidence.", ...output },
});

describe("Topics naming boundary", () => {
  it("keeps invalid cached Bedrock output per-trace and records paid usage", async () => {
    const { generateTopicText } = await vi.importActual<
      typeof import("@langfuse/shared/topics/server")
    >("@langfuse/shared/topics/server");
    state.call.mockImplementation(generateTopicText);
    state.env.LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS = [];
    vi.stubGlobal(
      "fetch",
      vi.fn(async () =>
        Response.json({
          choices: [
            {
              finish_reason: "stop",
              message: {
                content: JSON.stringify(
                  keyed({ summary: 42, status: "applicable" }),
                ),
              },
            },
          ],
          usage: { prompt_tokens: 12, completion_tokens: 4, total_tokens: 16 },
        }),
      ),
    );
    const models = topicsModels();
    models.summary.connection.secretKey = encrypt(
      JSON.stringify({ accessKeyId: "AKIATOPICS", secretAccessKey: "test" }),
    );

    await expect(
      summarizeOne(
        models,
        "Trace evidence.",
        topicProcessingConfigSchema.parse({
          summaryModel: models.summary.model,
        }),
      ),
    ).rejects.toMatchObject({ name: "ZodError" });
    expect(state.increment).toHaveBeenCalledWith("langfuse.topics.tokens", 12, {
      stage: "summary",
      direction: "input",
    });
    expect(state.increment).toHaveBeenCalledWith("langfuse.topics.tokens", 4, {
      stage: "summary",
      direction: "output",
    });
  });

  it("classifies text after a valid cached Bedrock object as invalid output", async () => {
    const { generateTopicText } = await vi.importActual<
      typeof import("@langfuse/shared/topics/server")
    >("@langfuse/shared/topics/server");
    state.call.mockImplementation(generateTopicText);
    state.env.LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS = [];
    vi.stubGlobal(
      "fetch",
      vi.fn(async () =>
        Response.json({
          choices: [
            {
              finish_reason: "stop",
              message: {
                content: `${JSON.stringify(keyed({ summary: "A request.", status: "applicable" }))} assistant {"intent_1":`,
              },
            },
          ],
          usage: { prompt_tokens: 12, completion_tokens: 4, total_tokens: 16 },
        }),
      ),
    );
    const models = topicsModels();
    models.summary.connection.secretKey = encrypt(
      JSON.stringify({ accessKeyId: "AKIATOPICS", secretAccessKey: "test" }),
    );

    await expect(
      summarizeOne(
        models,
        "Trace evidence.",
        topicProcessingConfigSchema.parse({
          summaryModel: models.summary.model,
        }),
      ),
    ).rejects.toMatchObject({ reason: "invalid_output" });
  });

  it.each([
    ["us.openai.gpt-5.6-luna", 0.00002, 0.000036],
    ["us.openai.gpt-6-luna", 0.000011, 0.0000165],
    ["global.openai.gpt-6-luna", 0.00001, 0.000015],
  ])(
    "summarizes a trace and prices %s usage",
    async (model, inputCost, outputCost) => {
      state.call.mockResolvedValue({
        output: keyed({ summary: "A billing request.", status: "applicable" }),
        usage: { inputTokens: 100, outputTokens: 30 },
      });
      const result = await summarizeOne(
        topicsModels(model),
        "RAW_TRANSCRIPT_SENTINEL",
        topicProcessingConfigSchema.parse({
          summaryModel: model,
        }),
      );
      expect(result.output).toEqual(
        keyed({ summary: "A billing request.", status: "applicable" }),
      );
      expect(result.providedUsageDetails).toEqual({
        summary_input: 100,
        summary_output: 30,
      });
      expect(result.usageDetails).toEqual({
        summary_input: 100,
        summary_output: 30,
        total: 130,
      });
      expect(result.providedCostDetails).toEqual({});
      expect(result.costDetails.summary_input).toBeCloseTo(inputCost, 10);
      expect(result.costDetails.summary_output).toBeCloseTo(outputCost, 10);
      expect(result.costDetails.total).toBeCloseTo(inputCost + outputCost, 10);
      const request = state.call.mock.calls[0][0];
      expect(request.model).toMatchObject({ slot: "summary", model });
      expect(request.trace).toMatchObject({
        targetProjectId: "ai-features",
        traceName: "topics-summary",
        environment: "langfuse-topics",
        aiFeatureOtelIngestion: false,
        metadata: {
          langfuse_project_id: "project",
          facets: [{ key: "intent", facet_id: "facet", facet_version: 1 }],
        },
      });
      expect(request.system).toEqual([
        { text: expect.stringContaining(facet.prompt), cache: true },
      ]);
      expect(request.input).toBe(
        "<transcript>\nRAW_TRANSCRIPT_SENTINEL\n</transcript>\n\nWrite the summaries now, one per facet key, each in its facet's format.",
      );
    },
  );

  it("rejects a request frozen on another summary model before calling the provider", async () => {
    await expect(
      summarizeOne(
        topicsModels("us.openai.gpt-6-luna"),
        "Trace evidence.",
        topicProcessingConfigSchema.parse({
          summaryModel: "us.openai.gpt-5.6-luna",
        }),
      ),
    ).rejects.toMatchObject({ reason: "configuration" });
    expect(state.call).not.toHaveBeenCalled();
  });

  it.each([
    ["secret error code", "invalid_input"],
    ["content_filter", "content_filter"],
  ])(
    "classifies HTTP 400 naming errors with code %s without logging provider content",
    async (code, reason) => {
      // The AI SDK identifies an APICallError by these markers.
      const rejection = Object.assign(
        new Error("Invalid input: secret transcript"),
        {
          [Symbol.for("vercel.ai.error")]: true,
          [Symbol.for("vercel.ai.error.AI_APICallError")]: true,
          name: "AI_APICallError",
          statusCode: 400,
          isRetryable: false,
          requestBodyValues: { messages: "secret transcript" },
          responseBody: '{"error":{"message":"secret echo"}}',
          data: {
            error: {
              type: "secret error type",
              code,
              param: "secret input value",
            },
          },
        },
      );
      state.call.mockRejectedValue(rejection);
      await expect(
        nameTopicGroup(topicsModels(), evidence),
      ).rejects.toMatchObject({ reason });
      const [message, fields] = vi.mocked(logger.warn).mock.calls.at(-1)!;
      expect(message).toBe("Topics model request failed");
      expect(fields).toMatchObject({
        "http.status_code": 400,
      });
      expect(JSON.stringify(fields)).not.toMatch(/secret/);
      expect(JSON.stringify(state.spanAttributes.mock.calls)).not.toMatch(
        /secret/,
      );
      expect(state.increment).toHaveBeenCalledWith(
        "langfuse.topics.llm_calls",
        1,
        {
          stage: "naming",
          adapter: LLMAdapter.Bedrock,
          status: "400",
        },
      );
    },
  );

  it.each([
    ["other", "refusal", "invalid_output"],
    ["content-filter", "content_filtered", "content_filter"],
  ])(
    "reports %s naming output without logging provider content",
    async (finishReason, rawFinishReason, reason) => {
      const rejection = Object.assign(new Error("secret response text"), {
        name: "AI_NoOutputGeneratedError",
        responseBody: "secret response body",
        text: "secret model output",
      });
      const response = {
        "langfuse.topics.response.finish_reason": finishReason,
        "langfuse.topics.response.raw_finish_reason": rawFinishReason,
        "langfuse.topics.response.text_characters": 0,
        "langfuse.topics.response.output_tokens": 0,
      };
      state.responseMetadata.mockImplementation((error) =>
        error === rejection ? response : undefined,
      );
      state.call.mockRejectedValue(rejection);

      const failure = await nameTopicGroup(topicsModels(), evidence).catch(
        (error: unknown) => error,
      );
      expect(failure).toMatchObject({
        reason,
        message: expect.stringMatching(
          reason === "content_filter"
            ? /filter/i
            : /returned no usable structured output/,
        ),
      });
      expect((failure as Error).message).not.toMatch(/credentials|secret/);
      expect(logger.warn).toHaveBeenCalledOnce();
      const [message, fields] = vi.mocked(logger.warn).mock.calls[0];
      expect(message).toBe("Topics model request failed");
      expect(fields).toMatchObject({
        ...response,
        "langfuse.topics.error.reason": reason,
        "langfuse.topics.error.type": "AI_NoOutputGeneratedError",
      });
      expect(state.spanAttributes).toHaveBeenCalledWith(fields);
      expect(JSON.stringify(fields)).not.toMatch(/secret/);
      expect(JSON.stringify(state.spanAttributes.mock.calls)).not.toMatch(
        /secret/,
      );
      expect(state.increment).toHaveBeenCalledWith(
        "langfuse.topics.llm_calls",
        1,
        {
          stage: "naming",
          adapter: LLMAdapter.Bedrock,
          status: reason,
        },
      );
    },
  );

  it.each([
    [{}, "ok"],
    [{ httpStatus: 429, reason: "rate_limit" }, "429"],
    [{ httpStatus: 404, reason: "invalid_input" }, "4xx"],
    [{ httpStatus: 503, reason: "provider_error" }, "5xx"],
    [{ reason: "timeout" }, "timeout"],
  ])("tags a model call outcome %o as %s", (outcome, status) => {
    recordTopicModelCall("summary", LLMAdapter.OpenAI, outcome);
    expect(state.increment).toHaveBeenCalledWith(
      "langfuse.topics.llm_calls",
      1,
      {
        stage: "summary",
        adapter: LLMAdapter.OpenAI,
        status,
      },
    );
  });

  it("logs only the status for an error whose message carries the response body", async () => {
    // The direct Bedrock call throws a plain error with the response text.
    state.call.mockRejectedValue(
      Object.assign(
        new Error('Bedrock InvokeModel failed with 400: {"secret": "echo"}'),
        { status: 400 },
      ),
    );
    await expect(
      summarizeOne(
        topicsModels(),
        "Trace evidence.",
        topicProcessingConfigSchema.parse({
          summaryModel: "us.openai.gpt-5.6-luna",
        }),
      ),
    ).rejects.toMatchObject({ reason: "invalid_input" });
    const [, fields] = vi.mocked(logger.warn).mock.calls.at(-1)!;
    expect(fields).toMatchObject({ "http.status_code": 400 });
    expect(JSON.stringify(fields)).not.toMatch(/secret/);
  });

  it("keeps fallback token counts out of provider-reported usage", async () => {
    state.call.mockResolvedValue({
      output: keyed({ summary: "A billing request.", status: "applicable" }),
      usage: { inputTokens: 100 },
    });
    const config = topicProcessingConfigSchema.parse({
      summaryModel: "us.openai.gpt-5.6-luna",
    });
    const result = await summarizeOne(
      topicsModels(),
      "A request for an invoice.",
      config,
    );
    expect(result.providedUsageDetails).toEqual({ summary_input: 100 });
    expect(tokenCalls()).toEqual([
      ["langfuse.topics.tokens", 100, { stage: "summary", direction: "input" }],
      [
        "langfuse.topics.token_usage_missing",
        1,
        { stage: "summary", direction: "output" },
      ],
    ]);
    expect(result.usageDetails).toEqual({
      summary_input: 100,
      summary_output: config.maxOutputTokens,
      total: 100 + config.maxOutputTokens,
    });
  });
  it("rejects an oversized shared transcript before calling the provider", async () => {
    await expect(
      summarizeOne(
        topicsModels(),
        "Trace evidence. ".repeat(1000),
        topicProcessingConfigSchema.parse({
          maxInputTokens: 256,
          summaryModel: "us.openai.gpt-5.6-luna",
        }),
      ),
    ).rejects.toThrow(
      /are \d+ tokens, above this run's 256-token input limit\. No model call was made; the transcript is never shortened per facet\./,
    );
    expect(state.call).not.toHaveBeenCalled();
  });

  it("prices cache reads and writes at their cache rates", async () => {
    state.call.mockResolvedValue({
      output: keyed({ summary: "A billing request.", status: "applicable" }),
      usage: {
        inputTokens: 4000,
        outputTokens: 300,
        totalTokens: 4300,
        cacheReadTokens: 2840,
        cacheWriteTokens: 100,
      },
    });
    const result = await summarizeOne(
      topicsModels("us.openai.gpt-6-luna"),
      "RAW_TRANSCRIPT_SENTINEL",
      topicProcessingConfigSchema.parse({
        summaryModel: "us.openai.gpt-6-luna",
      }),
    );
    expect(result.usageDetails).toMatchObject({
      summary_input: 4000,
      summary_input_cache_read: 2840,
    });
    expect(result.usageDetails.summary_input_cache_write).toBe(100);
    // 1,060 uncached at $0.11, 2,840 cache reads at $0.011 and 100 cache writes at $0.1375 per million.
    expect(result.costDetails.summary_input).toBeCloseTo(
      (1060 * 0.11 + 2840 * 0.011 + 100 * 0.1375) / 1_000_000,
      12,
    );
  });

  it("summarizes all facets of a trace in one request and checks its size first", async () => {
    state.call.mockResolvedValue({
      output: {
        intent_1: {
          notes: "n",
          summary: "Export sales.",
          status: "applicable",
        },
        issues_2: { notes: "n", summary: "", status: "not_applicable" },
      },
      usage: { inputTokens: 100, outputTokens: 30 },
    });
    const config = topicProcessingConfigSchema.parse({
      summaryModel: "us.openai.gpt-5.6-luna",
    });
    const issues = {
      ...facet,
      facetId: "issues",
      prompt: "Describe the main problem.",
    };
    const facets = [
      { key: "intent_1", facet, builtIn: true },
      { key: "issues_2", facet: issues, builtIn: false },
    ];
    await summarizeTopicTraceFacets(
      topicsModels(),
      facets,
      "RAW_TRANSCRIPT_SENTINEL",
      config,
    );
    expect(state.call).toHaveBeenCalledOnce();
    const request = state.call.mock.calls[0][0];
    // Built-in facets end the first cached prefix, custom facets the second; the transcript is uncached.
    expect(request.system).toEqual([
      {
        text: expect.stringContaining(
          `<facet key="intent_1">\n${facet.prompt}\n</facet>`,
        ),
        cache: true,
      },
      {
        text: `<facet key="issues_2">\n${issues.prompt}\n</facet>`,
        cache: true,
      },
    ]);
    expect(request.system[0].text).not.toContain("issues_2");
    expect(request.input).toContain("RAW_TRANSCRIPT_SENTINEL");
    expect(request.maxOutputTokens).toBe(config.maxOutputTokens * 2);
    expect(request.trace).toMatchObject({
      traceName: "topics-summary",
      environment: "langfuse-topics",
      metadata: {
        langfuse_project_id: "project",
        facets: [
          { key: "intent_1", facet_id: "facet", facet_version: 1 },
          { key: "issues_2", facet_id: "issues", facet_version: 1 },
        ],
      },
    });
    expect(state.project).toHaveBeenCalledWith({
      where: { id: "project", deletedAt: null },
      select: {
        organization: {
          select: { aiFeaturesEnabled: true, aiTelemetryEnabled: true },
        },
      },
    });
    // Every facet key is required in the structured output.
    expect(
      request.schema.safeParse({
        intent_1: {
          notes: "n",
          summary: "Export sales.",
          status: "applicable",
        },
      }).success,
    ).toBe(false);

    state.call.mockClear();
    await expect(
      summarizeTopicTraceFacets(
        topicsModels(),
        facets,
        "Trace evidence. ".repeat(1000),
        topicProcessingConfigSchema.parse({
          maxInputTokens: 256,
          summaryModel: "us.openai.gpt-5.6-luna",
        }),
      ),
    ).rejects.toThrow(/above this run's 256-token input limit/);
    expect(state.call).not.toHaveBeenCalled();
  });

  it("names the complete cohort without shortening member summaries", async () => {
    const members = Array.from({ length: 400 }, (_, index) => ({
      id: `m${index + 1}`,
      summary: `Request ${index}: ${"Invoice request details. ".repeat(60)} END-${index}`,
    }));
    const summaries = members.map(({ summary }) => summary);
    state.call.mockResolvedValue({
      output: {
        name: "Invoice assistance",
        description: "Requests involving invoices.",
        evidenceSummaryIds: ["m400"],
      },
      usage: { inputTokens: 16000, outputTokens: 30 },
    });
    const result = await nameTopicGroup(topicsModels(), {
      members,
      contrasts: [],
    });
    expect(result.output).toMatchObject({
      name: "Invoice assistance",
      evidenceSummaryIds: ["m400"],
    });
    expect(state.call).toHaveBeenCalledOnce();
    expect(state.call.mock.calls[0][0].trace).toMatchObject({
      traceName: "topics-naming",
      environment: "langfuse-topics",
      metadata: { langfuse_project_id: "project" },
    });
    expect(state.call.mock.calls[0][0].model).toMatchObject({
      slot: "clustering",
      model: "us.openai.gpt-5.6-terra",
    });
    expect(result.costDetails.total).toBeCloseTo(0.03236, 10);
    const submitted = JSON.parse(state.call.mock.calls[0][0].input);
    expect(
      submitted.members
        .map(({ summary }: { summary: string }) => summary)
        .sort(),
    ).toEqual(summaries.sort());
    expect(state.call.mock.calls[0][0].trace.metadata.naming_sampled).toBe(
      false,
    );
  });

  it("names complete sampled summaries, reuses them for renames and rejects omitted citations", async () => {
    state.encoding = {
      encode: vi.fn((text: string) => ({
        length:
          (text.match(/NAMING_MEMBER/g)?.length ?? 0) * 220_000 + text.length,
      })),
      free: vi.fn(),
    };
    const makeGroup = () => ({
      members: Array.from({ length: 12 }, (_, index) => ({
        summary: `NAMING_MEMBER invoice request ${index}.`,
        traceId: `trace-${index}`,
      })),
      contrasts: [],
    });
    const group = makeGroup();
    const sources = [...group.members];
    const prepared = await prepareTopicNamingGroup(group, "run-a/group-a");
    expect(prepared.sampled).toBe(true);
    expect(prepared.members.length).toBeLessThan(group.members.length);
    expect(new Set(prepared.members.map(({ id }) => id)).size).toBe(
      prepared.members.length,
    );
    expect(prepared.members).toEqual(
      (await prepareTopicNamingGroup(makeGroup(), "run-a/group-a")).members,
    );
    expect(prepared.members).not.toEqual(
      (await prepareTopicNamingGroup(makeGroup(), "run-b/group-a")).members,
    );
    for (const member of prepared.members) {
      expect(sources.find(({ summary }) => summary === member.summary)).toBe(
        member.source,
      );
    }
    group.members.reverse();
    state.call.mockImplementation(async ({ input }) => ({
      output: {
        name: "Invoice requests",
        description: "Requests involving invoices.",
        evidenceSummaryIds: [JSON.parse(input).members[0].id],
      },
      usage: {},
    }));
    const first = await namePreparedTopicGroup(topicsModels(), prepared);
    const conflict = {
      name: "Billing support",
      description: "Support for billing problems.",
      summaries: [
        "NAMING_MEMBER too large for the retry reserve.",
        "A billing question.",
      ],
    };
    const renamed = await namePreparedTopicGroup(
      topicsModels(),
      prepared,
      undefined,
      conflict,
    );
    const [initialCall, renameCall] = state.call.mock.calls.map(
      ([call]) => call,
    );
    const [initialRequest, renameRequest] = state.call.mock.calls.map(
      ([request]) => JSON.parse(request.input),
    );
    expect(initialRequest.members).toEqual(
      prepared.members.map(({ id, summary }) => ({ id, summary })),
    );
    expect(renameRequest.members).toEqual(initialRequest.members);
    expect(renameRequest.conflictingTopic).toEqual({
      ...conflict,
      summaries: ["A billing question."],
    });
    expect(initialCall.trace.traceName).toBe("topics-naming");
    expect(renameCall.trace.traceName).toBe("topics-naming-conflict");
    expect(renameCall.system[0].text).toContain(initialCall.system[0].text);
    expect(renamed.usageDetails.naming_input).toBeGreaterThan(
      first.usageDetails.naming_input!,
    );
    expect(renamed.usageDetails.naming_input).toBeLessThanOrEqual(900_000);

    const omittedId = Array.from(
      { length: group.members.length },
      (_, index) => `m${index + 1}`,
    ).find((id) => !prepared.members.some((member) => member.id === id));
    state.call.mockResolvedValueOnce({
      output: {
        name: "Invoice requests",
        description: "Requests involving invoices.",
        evidenceSummaryIds: [omittedId],
      },
      usage: { inputTokens: 100, outputTokens: 30 },
    });
    await expect(
      namePreparedTopicGroup(topicsModels(), prepared),
    ).rejects.toThrow("Evidence must cite a submitted cluster member.");
  });

  it("skips individually oversized members and fails before inference when none fit", async () => {
    state.encoding = {
      encode: vi.fn((text: string) => ({
        length: text.includes("OVERSIZED") ? 1_000_000 : text.length,
      })),
      free: vi.fn(),
    };
    const prepared = await prepareTopicNamingGroup(
      {
        members: [
          { summary: "OVERSIZED" },
          { summary: "A complete invoice question." },
        ],
        contrasts: [{ id: "contrast", summary: "OVERSIZED" }],
      },
      "oversized",
    );
    expect(prepared.members.map(({ summary }) => summary)).toEqual([
      "A complete invoice question.",
    ]);
    expect(JSON.parse(prepared.input).contrasts).toEqual([]);
    await expect(
      prepareTopicNamingGroup(
        { members: [{ summary: "OVERSIZED" }], contrasts: [] },
        "oversized",
      ),
    ).rejects.toThrow("No complete cluster member summary fits");
    expect(state.call).not.toHaveBeenCalled();
    expect(state.encoding.free).toHaveBeenCalledTimes(2);
  });

  it("bounds serialized bytes independently of token estimates", async () => {
    state.encoding = {
      encode: vi.fn(() => ({ length: 1 })),
      free: vi.fn(),
    };
    const summary = "x".repeat(1024 * 1024);
    const group = {
      members: Array.from({ length: 17 }, () => ({ summary })),
      contrasts: [],
    };
    const prepared = await prepareTopicNamingGroup(group, "byte-budget");
    expect(prepared.sampled).toBe(true);
    expect(prepared.members.length).toBeGreaterThan(0);
    expect(Buffer.byteLength(prepared.input)).toBeLessThanOrEqual(
      16 * 1024 * 1024,
    );
    expect(prepared.members.every((member) => member.summary === summary)).toBe(
      true,
    );
  });

  it("yields while preparing escaped summaries and preserves complete text", async () => {
    state.encoding = {
      encode: vi.fn((text: string) => ({ length: text.length })),
      free: vi.fn(),
    };
    let elapsed = 0;
    const time = vi
      .spyOn(performance, "now")
      .mockImplementation(() => (elapsed += 2));
    let ticked = false;
    const heartbeat = setImmediate(() => {
      ticked = true;
    });
    const summary = `${"x".repeat(4095)}😀\u0000\n\"\\${"y".repeat(12_000)}`;
    try {
      const prepared = await prepareTopicNamingGroup(
        { members: [{ summary }], contrasts: [] },
        "yield",
      );
      expect(ticked).toBe(true);
      expect(JSON.parse(prepared.input).members).toEqual([
        { id: "m1", summary },
      ]);
      expect(
        state.encoding.encode.mock.calls.every(([text]) => text.length <= 4096),
      ).toBe(true);
    } finally {
      clearImmediate(heartbeat);
      time.mockRestore();
    }
  });

  it("trims names and rejects whitespace-only descriptions", async () => {
    state.call.mockResolvedValueOnce({
      output: {
        name: " Invoice assistance ",
        description: "Invoice requests.",
        evidenceSummaryIds: ["m1"],
      },
      usage: {},
    });
    expect((await nameTopicGroup(topicsModels(), evidence)).output.name).toBe(
      "Invoice assistance",
    );
    state.call.mockResolvedValueOnce({
      output: {
        name: "Invoice assistance",
        description: "   ",
        evidenceSummaryIds: ["m1"],
      },
      usage: {},
    });
    await expect(nameTopicGroup(topicsModels(), evidence)).rejects.toThrow();
  });

  it.each([["contrast"], ["m1", "m2", "m1", "m2"]])(
    "rejects contrast or excess evidence: %j",
    async (...evidenceSummaryIds) => {
      state.call.mockResolvedValue({
        output: {
          name: "Invoice assistance",
          description: "Invoice requests.",
          evidenceSummaryIds,
        },
        usage: { inputTokens: 100, outputTokens: 30 },
      });
      await expect(nameTopicGroup(topicsModels(), evidence)).rejects.toThrow();
      // Provider work is billable even when local output validation rejects it.
      expect(tokenCalls()).toEqual([
        [
          "langfuse.topics.tokens",
          100,
          { stage: "naming", direction: "input" },
        ],
        [
          "langfuse.topics.tokens",
          30,
          { stage: "naming", direction: "output" },
        ],
      ]);
    },
  );

  it.each([
    { aiFeaturesEnabled: true, aiTelemetryEnabled: false },
    { aiFeaturesEnabled: false, aiTelemetryEnabled: true },
    null,
    new Error("database unavailable"),
  ])(
    "keeps inference untraced without organization consent: %j",
    async (organization) => {
      if (organization instanceof Error)
        state.project.mockRejectedValue(organization);
      else
        state.project.mockResolvedValue(organization ? { organization } : null);
      state.call
        .mockResolvedValueOnce({
          output: keyed({
            summary: "A billing request.",
            status: "applicable",
          }),
          usage: {},
        })
        .mockResolvedValueOnce({
          output: {
            name: "Billing",
            description: "Billing requests.",
            evidenceSummaryIds: ["m1"],
          },
          usage: {},
        });
      await summarizeOne(
        topicsModels(),
        "A billing request.",
        topicProcessingConfigSchema.parse({
          summaryModel: "us.openai.gpt-5.6-luna",
        }),
      );
      await nameTopicGroup(topicsModels(), evidence);
      state.call.mockResolvedValueOnce({
        output: {
          intent: {
            notes: "Invoice request.",
            summary: "Request invoice.",
            status: "applicable",
          },
        },
        usage: {},
      });
      await summarizeTopicTraceFacets(
        topicsModels(),
        [{ key: "intent", facet, builtIn: true }],
        "Request invoice.",
        topicProcessingConfigSchema.parse({
          summaryModel: "us.openai.gpt-5.6-luna",
        }),
      );
      expect(state.call).toHaveBeenCalledTimes(3);
      for (const [request] of state.call.mock.calls)
        expect(request.trace).toBeUndefined();
    },
  );

  it.each(["unconfigured destination", "empty allowlist", "unlisted source"])(
    "keeps inference untraced and skips the consent lookup with %s",
    async (reason) => {
      if (reason === "unconfigured destination")
        state.traceSink.mockReturnValue(undefined);
      else
        state.env.LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS =
          reason === "empty allowlist" ? [] : ["ai-features", "project-other"];
      state.call.mockResolvedValue({
        output: keyed({ summary: "A billing request.", status: "applicable" }),
        usage: {},
      });
      await summarizeOne(
        topicsModels(),
        "A billing request.",
        topicProcessingConfigSchema.parse({
          summaryModel: "us.openai.gpt-5.6-luna",
        }),
      );
      expect(state.project).not.toHaveBeenCalled();
      expect(state.call).toHaveBeenCalledOnce();
      expect(state.call.mock.calls[0][0].trace).toBeUndefined();
      if (reason !== "unconfigured destination")
        expect(state.traceSink).not.toHaveBeenCalled();
    },
  );

  it("records embedding input usage even when the returned vector is invalid", async () => {
    state.embed.mockResolvedValue({ embeddings: [[]], tokens: 12 });
    await expect(
      embedTopicSummaries(
        topicsModels().embedding,
        ["An invoice request."],
        256,
        "project",
      ),
    ).rejects.toMatchObject({
      reason: "invalid_output",
    });
    expect(tokenCalls()).toEqual([
      [
        "langfuse.topics.tokens",
        12,
        { stage: "embedding", direction: "input" },
      ],
    ]);
  });

  it.each([undefined, -1, Number.NaN])(
    "does not invent missing embedding usage: %s",
    async (tokens) => {
      state.embed.mockResolvedValue({
        embeddings: [Array(256).fill(0.25)],
        tokens,
      });
      await embedTopicSummaries(
        topicsModels().embedding,
        ["An invoice request."],
        256,
        "project",
      );
      expect(tokenCalls()).toEqual([
        [
          "langfuse.topics.token_usage_missing",
          1,
          { stage: "embedding", direction: "input" },
        ],
      ]);
    },
  );
});