Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(topics): trace summary and naming calls with org analytics opt-in (#18346)

Nimar3eb71e89be4f
diff --git a/packages/shared/AGENTS.md b/packages/shared/AGENTS.md
index 212d41d7bb..2c3232990f 100644
--- a/packages/shared/AGENTS.md
+++ b/packages/shared/AGENTS.md
@@ -93,8 +93,10 @@
Summary references carry facet/version/source fields within project and
execution scope; exact storage reads also require facet/version scope.
- `text.ts` and `embeddings.ts`: model transport through the project's LLM
- connections (`generateLLMText` / `embedLLMText`); worker model calls own
- usage, cost and vector validation.
+ connections (`generateLLMText` / `embedLLMText`), with cached Bedrock text
+ calls using InvokeModel. Text accepts optional internal generation tracing;
+ worker model calls own tracing allowlisting, organization consent, usage,
+ cost and vector validation.
- `model-config.ts`: per-project `topic_configs` (three connection +
model slots and the automatic-processing switch). Saving tests every slot
with a real call; Anthropic works for summaries and clustering but fails the
@@ -134,7 +136,8 @@
for shared backend services, repositories, queue helpers/contracts, Redis and
ClickHouse helpers, auth helpers, logger/instrumentation, ingestion helpers,
AI SDK-native LLM execution helpers (`generateLLMText`, `streamLLMText`, and
- `embedLLMText`), Bedrock default-credential provider auth
+ `embedLLMText`), `isInternalEvalEnvironment` for shared eval/Topics exclusions,
+ Bedrock default-credential provider auth
(`createDefaultBedrockProviderAuth`), and server test utilities. Langfuse AI
callers select their local profile through `getLangfuseAIAwsProfile`;
generic Bedrock auth only uses an explicitly supplied profile or the normal
diff --git a/packages/shared/src/env.ts b/packages/shared/src/env.ts
index 324f213738..628ebdca33 100644
--- a/packages/shared/src/env.ts
+++ b/packages/shared/src/env.ts
@@ -53,6 +53,15 @@ const EnvSchema = z.object({
.map((id) => id.trim())
.filter(Boolean),
),
+ LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS: z
+ .string()
+ .default("")
+ .transform((value) =>
+ value
+ .split(",")
+ .map((id) => id.trim())
+ .filter(Boolean),
+ ),
LANGFUSE_TOPICS_REDIS_TTL_SECONDS: z.coerce
.number()
.int()
diff --git a/packages/shared/src/server/index.ts b/packages/shared/src/server/index.ts
index b17fad518c..d1bf952d56 100644
--- a/packages/shared/src/server/index.ts
+++ b/packages/shared/src/server/index.ts
@@ -58,6 +58,7 @@ export * from "./llm/utils";
export * from "./llm/types";
export * from "./llm/promptToolConfig";
export * from "./llm/internalTraceEvents";
+export { isInternalEvalEnvironment } from "./llm/isInternalEvalEnvironment";
export * from "./llm/langfuseAiCompletion";
export * from "./utils/baseUrl";
export * from "./utils/productUrl";
diff --git a/packages/shared/src/server/llm/isInternalEvalEnvironment.ts b/packages/shared/src/server/llm/isInternalEvalEnvironment.ts
new file mode 100644
index 0000000000..06cfeb8515
--- /dev/null
+++ b/packages/shared/src/server/llm/isInternalEvalEnvironment.ts
@@ -0,0 +1,14 @@
+export const INTERNAL_EVAL_ENVIRONMENT_PREFIX = "langfuse";
+
+// Public ingestion strips the reserved `langfuse-` prefix from environments
+// originating outside Langfuse, including OpenRouter Broadcast callbacks.
+export const PUBLIC_LLM_JUDGE_ENVIRONMENT = "llm-as-a-judge";
+
+export function isInternalEvalEnvironment(
+ environment: string | null | undefined,
+): boolean {
+ return (
+ environment?.startsWith(INTERNAL_EVAL_ENVIRONMENT_PREFIX) === true ||
+ environment === PUBLIC_LLM_JUDGE_ENVIRONMENT
+ );
+}
diff --git a/packages/shared/src/server/llm/types.ts b/packages/shared/src/server/llm/types.ts
index f74ac0080b..ccd645f325 100644
--- a/packages/shared/src/server/llm/types.ts
+++ b/packages/shared/src/server/llm/types.ts
@@ -558,6 +558,7 @@ export enum LangfuseInternalTraceEnvironment {
CodeEval = "langfuse-code-eval",
NaturalLanguageFilter = "langfuse-natural-language-filter",
InAppAgent = "langfuse-in-app-agent",
+ Topics = "langfuse-topics",
}
export type ProcessedTraceEvent = {
diff --git a/packages/shared/src/server/otel/internalTraceOtelWriter.ts b/packages/shared/src/server/otel/internalTraceOtelWriter.ts
index 7c5509b90b..c83ea68bd3 100644
--- a/packages/shared/src/server/otel/internalTraceOtelWriter.ts
+++ b/packages/shared/src/server/otel/internalTraceOtelWriter.ts
@@ -260,7 +260,7 @@ export async function publishInternalOtelSpans(params: {
/**
* Exactly the `InternalTraceEventInput` fields this writer maps onto OTel
- * spans. Narrowed so a caller with richer inputs (usage, prompt, experiment
+ * spans. Narrowed so a caller with richer inputs (cost, prompt, experiment
* linkage, ...) gets a compile-time surface mismatch instead of silent drops.
*/
export type InternalOtelSpanInput = Pick<
@@ -280,6 +280,12 @@ export type InternalOtelSpanInput = Pick<
| "output"
| "metadata"
| "evaluationContext"
+ | "type"
+ | "modelName"
+ | "modelParameters"
+ | "providedUsageDetails"
+ | "userId"
+ | "sessionId"
>;
/**
diff --git a/packages/shared/src/server/topics/clickhouse.ts b/packages/shared/src/server/topics/clickhouse.ts
index 283768f5a1..8ce66060ca 100644
--- a/packages/shared/src/server/topics/clickhouse.ts
+++ b/packages/shared/src/server/topics/clickhouse.ts
@@ -17,6 +17,11 @@ import { prisma } from "../../db";
import { chunk } from "lodash";
import { Readable } from "node:stream";
import { finished } from "node:stream/promises";
+import {
+ INTERNAL_EVAL_ENVIRONMENT_PREFIX,
+ isInternalEvalEnvironment,
+ PUBLIC_LLM_JUDGE_ENVIRONMENT,
+} from "../llm/isInternalEvalEnvironment";
const sourceKeySql =
"project_id, facet_id, facet_version, trace_id, if(trace_id = '', session_id, '')";
@@ -352,7 +357,9 @@ export async function listTopicSummaries(
);
}
rows.sort((a, b) => Number(b.processedAtMs) - Number(a.processedAtMs));
- return rows.map(summaryResult);
+ return rows
+ .filter((row) => !isInternalEvalEnvironment(row.environment))
+ .map(summaryResult);
}
/** Without a version filter, use each source's newest processed facet version. */
export async function getLatestFacetSummaries(
@@ -377,7 +384,9 @@ export async function getLatestFacetSummaries(
},
tags: { route: "topics-latest-summaries", projectId },
});
- return rows.map(summaryResult);
+ return rows
+ .filter((row) => !isInternalEvalEnvironment(row.environment))
+ .map(summaryResult);
}
export async function getTopicClusteringSummaries(
@@ -398,6 +407,8 @@ export async function getTopicClusteringSummaries(
LIMIT 1 BY ${sourceKeySql}
) WHERE processing_state = 'complete' AND embedding_model = {embeddingModel:String}
AND length(embedding) = {embeddingDimensions:UInt32}
+ AND NOT startsWith(environment, {internalEnvironmentPrefix:String})
+ AND environment != {publicLlmJudgeEnvironment:String}
ORDER BY trace_id`,
params: {
projectId,
@@ -405,6 +416,8 @@ export async function getTopicClusteringSummaries(
facetVersion,
...embeddingConfig,
...timeRangeParams(timeRange),
+ internalEnvironmentPrefix: INTERNAL_EVAL_ENVIRONMENT_PREFIX,
+ publicLlmJudgeEnvironment: PUBLIC_LLM_JUDGE_ENVIRONMENT,
},
tags: { route: "topics-clustering-summaries", projectId },
});
@@ -438,7 +451,7 @@ export async function getTopicSummaryCounts(
count: string;
}>({
query: `SELECT facet_id AS facetId, facet_version AS facetVersion, count() AS count FROM (
- SELECT facet_id, facet_version, processing_state, embedding_model, length(embedding) AS dimensions
+ SELECT facet_id, facet_version, processing_state, embedding_model, length(embedding) AS dimensions, environment
FROM topic_facet_summaries
WHERE project_id = {projectId:String} AND ${unitTimeFilterSql} AND ${scopeSql}
AND ${latestSummaryFilterSql(scopeSql)}
@@ -446,6 +459,8 @@ export async function getTopicSummaryCounts(
LIMIT 1 BY ${sourceKeySql}
) WHERE processing_state = 'complete' AND embedding_model = {embeddingModel:String}
AND dimensions = {embeddingDimensions:UInt32}
+ AND NOT startsWith(environment, {internalEnvironmentPrefix:String})
+ AND environment != {publicLlmJudgeEnvironment:String}
GROUP BY facet_id, facet_version`,
params: {
projectId,
@@ -453,6 +468,8 @@ export async function getTopicSummaryCounts(
facetIds: facets.map(({ facetId }) => facetId),
facetVersions: facets.map(({ version }) => version),
...embeddingConfig,
+ internalEnvironmentPrefix: INTERNAL_EVAL_ENVIRONMENT_PREFIX,
+ publicLlmJudgeEnvironment: PUBLIC_LLM_JUDGE_ENVIRONMENT,
},
tags: { route: "topics-summary-counts", projectId },
});
diff --git a/packages/shared/src/server/topics/config.test.ts b/packages/shared/src/server/topics/config.test.ts
index e9fcfb39ac..f2345fe8af 100644
--- a/packages/shared/src/server/topics/config.test.ts
+++ b/packages/shared/src/server/topics/config.test.ts
@@ -4,6 +4,7 @@ beforeEach(() => {
vi.stubEnv("DOCKER_BUILD", undefined);
vi.stubEnv("LANGFUSE_TOPICS_ENABLED", undefined);
vi.stubEnv("LANGFUSE_TOPICS_ENABLED_PROJECT_IDS", undefined);
+ vi.stubEnv("LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS", undefined);
});
afterEach(() => {
@@ -11,6 +12,35 @@ afterEach(() => {
vi.resetModules();
});
+describe("Topics tracing allowlist", () => {
+ it.each([undefined, "", " , "])(
+ "disables tracing when its allowlist has no IDs: %s",
+ async (value) => {
+ vi.stubEnv("LANGFUSE_TOPICS_ENABLED_PROJECT_IDS", "project-a");
+ vi.stubEnv("LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS", value);
+ const { env } = await import("../../env.js");
+ expect(env.LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS).toEqual([]);
+ },
+ );
+
+ it("parses tracing IDs independently from the processing allowlist", async () => {
+ vi.stubEnv("LANGFUSE_TOPICS_ENABLED_PROJECT_IDS", "project-a,project-b");
+ vi.stubEnv(
+ "LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS",
+ " project-b, ,project-c ",
+ );
+ const { env } = await import("../../env.js");
+ expect(env.LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS).toEqual([
+ "project-b",
+ "project-c",
+ ]);
+ expect(env.LANGFUSE_TOPICS_ENABLED_PROJECT_IDS).toEqual([
+ "project-a",
+ "project-b",
+ ]);
+ });
+});
+
describe("Topics project allowlist", () => {
it("stays disabled when Docker builds skip environment defaults", async () => {
vi.stubEnv("DOCKER_BUILD", "1");
diff --git a/packages/shared/src/server/topics/text.test.ts b/packages/shared/src/server/topics/text.test.ts
index c35d7b7241..14e7fe3a25 100644
--- a/packages/shared/src/server/topics/text.test.ts
+++ b/packages/shared/src/server/topics/text.test.ts
@@ -3,11 +3,74 @@ import { z } from "zod";
import { encrypt } from "../../encryption";
import { env } from "../../env";
import { BEDROCK_USE_DEFAULT_CREDENTIALS } from "../../interfaces/customLLMProviderConfigSchemas";
-import { LLMAdapter } from "../llm/types";
+import { LLMAdapter, type TraceSinkParams } from "../llm/types";
+import type { ResourceSpan } from "../otel/OtelIngestionProcessor";
import { generateTopicEmbeddings } from "./embeddings";
import type { TopicsModel } from "./model-config";
import { generateTopicText } from "./text";
+const { publishToOtelIngestionQueue, processorConstructor } = vi.hoisted(
+ () => ({
+ publishToOtelIngestionQueue:
+ vi.fn<(spans: ResourceSpan[]) => Promise<void>>(),
+ processorConstructor: vi.fn(),
+ }),
+);
+
+vi.mock("../otel/OtelIngestionProcessor", () => ({
+ OtelIngestionProcessor: class {
+ constructor(config: unknown) {
+ processorConstructor(config);
+ }
+ publishToOtelIngestionQueue = publishToOtelIngestionQueue;
+ },
+}));
+
+const trace: TraceSinkParams = {
+ targetProjectId: "ai-features-project",
+ traceId: "0af7651916cd43dd8448eb211c80319c",
+ traceName: "topics-summary",
+ environment: "langfuse-topics",
+ metadata: { project_id: "source-project" },
+};
+
+function tracedGeneration() {
+ expect(processorConstructor).toHaveBeenCalledWith(
+ expect.objectContaining({
+ projectId: trace.targetProjectId,
+ isLangfuseInternal: true,
+ ingestionVersion: "4",
+ }),
+ );
+ expect(publishToOtelIngestionQueue).toHaveBeenCalledTimes(1);
+ const spans = publishToOtelIngestionQueue.mock.calls[0][0].flatMap(
+ (resource) =>
+ resource.scopeSpans?.flatMap((scope) => scope.spans ?? []) ?? [],
+ );
+ expect(spans).toHaveLength(1);
+ const span = spans[0];
+ expect(span.traceId).toBe(trace.traceId);
+ expect(span.parentSpanId ?? "").toBe("");
+ expect(span.name).toBe(trace.traceName);
+ const attributes = Object.fromEntries(
+ (span.attributes ?? []).map(({ key, value }) => [
+ key,
+ value.stringValue ?? value,
+ ]),
+ );
+ expect(attributes).toMatchObject({
+ "langfuse.observation.type": "generation",
+ "langfuse.environment": "langfuse-topics",
+ "langfuse.trace.metadata": JSON.stringify(trace.metadata),
+ });
+ return attributes;
+}
+
+beforeEach(() => {
+ vi.clearAllMocks();
+ publishToOtelIngestionQueue.mockResolvedValue(undefined);
+});
+
afterEach(() => {
vi.unstubAllGlobals();
vi.unstubAllEnvs();
@@ -29,51 +92,85 @@ function bedrockModel(slot: TopicsModel["slot"], model: string): TopicsModel {
}
describe("generateTopicText", () => {
- it("sends structured output through the project's Bedrock connection", async () => {
- const output = { summary: "Account access" };
- const fetch = vi.fn<typeof globalThis.fetch>(async () =>
- Response.json({
- output: {
- message: {
- role: "assistant",
- content: [
- { toolUse: { toolUseId: "json-1", name: "json", input: output } },
- ],
+ it.each(["us.openai.gpt-6-luna", "openai.gpt-oss-120b"])(
+ "sends structured output through the project's Bedrock connection: %s",
+ async (modelId) => {
+ const output = { summary: "Account access" };
+ const fetch = vi.fn<typeof globalThis.fetch>(async () =>
+ Response.json({
+ output: {
+ message: {
+ role: "assistant",
+ content: [
+ {
+ toolUse: { toolUseId: "json-1", name: "json", input: output },
+ },
+ ],
+ },
},
- },
- stopReason: "tool_use",
- usage: { inputTokens: 12, outputTokens: 4, totalTokens: 16 },
- metrics: { latencyMs: 1 },
- }),
- );
- vi.stubGlobal("fetch", fetch);
- // Connection credentials must win over ambient worker credentials.
- vi.stubEnv("AWS_BEARER_TOKEN_BEDROCK", "ambient-bearer-token");
-
- const result = await generateTopicText({
- model: bedrockModel("summary", "us.openai.gpt-6-luna"),
- system: [{ text: "Summarize the user's request." }],
- input: "I cannot sign in.",
- schema: z.object({ summary: z.string() }),
- maxOutputTokens: 256,
- });
+ stopReason: "tool_use",
+ usage: { inputTokens: 12, outputTokens: 4, totalTokens: 16 },
+ metrics: { latencyMs: 1 },
+ }),
+ );
+ vi.stubGlobal("fetch", fetch);
+ // Connection credentials must win over ambient worker credentials.
+ vi.stubEnv("AWS_BEARER_TOKEN_BEDROCK", "ambient-bearer-token");
+
+ const result = await generateTopicText({
+ model: bedrockModel("summary", modelId),
+ system: [
+ {
+ text: "Summarize the user's request.",
+ cache: modelId.includes("oss"),
+ },
+ ],
+ input: "I cannot sign in.",
+ schema: z.object({ summary: z.string() }),
+ maxOutputTokens: 256,
+ trace,
+ });
- expect(fetch).toHaveBeenCalledTimes(1);
- const request = new Request(...fetch.mock.calls[0]);
- expect(request.url).toBe(
- "https://bedrock-runtime.eu-central-1.amazonaws.com/model/us.openai.gpt-6-luna/converse",
- );
- expect(request.headers.get("authorization")).toMatch(
- /^AWS4-HMAC-SHA256 Credential=AKIATOPICS\//,
- );
- expect(await request.json()).toMatchObject({
- system: [{ text: "Summarize the user's request." }],
- inferenceConfig: { maxTokens: 256 },
- additionalModelRequestFields: { reasoning: { effort: "none" } },
- toolConfig: { toolChoice: { any: {} } },
- });
- expect(result.output).toEqual(output);
- });
+ expect(fetch).toHaveBeenCalledTimes(1);
+ const request = new Request(...fetch.mock.calls[0]);
+ expect(request.url).toBe(
+ `https://bedrock-runtime.eu-central-1.amazonaws.com/model/${modelId}/converse`,
+ );
+ expect(request.headers.get("authorization")).toMatch(
+ /^AWS4-HMAC-SHA256 Credential=AKIATOPICS\//,
+ );
+ const body = await request.json();
+ expect(body).toMatchObject({
+ system: [{ text: "Summarize the user's request." }],
+ inferenceConfig: { maxTokens: 256 },
+ toolConfig: { toolChoice: { any: {} } },
+ });
+ expect(result.output).toEqual(output);
+ if (modelId.includes("oss")) {
+ expect(body).not.toHaveProperty("additionalModelRequestFields");
+ } else {
+ expect(body).toHaveProperty(
+ "additionalModelRequestFields.reasoning.effort",
+ "none",
+ );
+ }
+ expect(tracedGeneration()).toMatchObject({
+ "langfuse.observation.input": JSON.stringify([
+ { role: "system", content: "Summarize the user's request." },
+ { role: "user", content: "I cannot sign in." },
+ ]),
+ "langfuse.observation.output": JSON.stringify(output),
+ "langfuse.observation.model.name": modelId,
+ "langfuse.observation.usage_details": JSON.stringify({
+ input: 12,
+ output: 4,
+ total: 16,
+ input_cached_tokens: 0,
+ input_cache_creation: 0,
+ }),
+ });
+ },
+ );
it("sends cached prompts through InvokeModel with the connection's credentials and region", async () => {
const fetch = vi.fn<typeof globalThis.fetch>(async () =>
@@ -103,6 +200,7 @@ describe("generateTopicText", () => {
input: "I cannot sign in.",
schema: z.object({ summary: z.string() }),
maxOutputTokens: 256,
+ trace,
});
const request = new Request(...fetch.mock.calls[0]);
@@ -132,6 +230,18 @@ describe("generateTopicText", () => {
{ role: "user", content: [{ type: "text", text: "I cannot sign in." }] },
]);
expect(body.prompt_cache_options).toEqual({ mode: "explicit", ttl: "30m" });
+ expect(tracedGeneration()).toMatchObject({
+ "langfuse.observation.output": JSON.stringify({
+ summary: "Account access",
+ }),
+ "langfuse.observation.usage_details": JSON.stringify({
+ input: 813,
+ output: 275,
+ total: 3928,
+ input_cached_tokens: 2840,
+ input_cache_creation: 0,
+ }),
+ });
expect(result.usage).toEqual({
inputTokens: 3653,
outputTokens: 275,
@@ -153,11 +263,17 @@ describe("generateTopicText", () => {
input: "Hi",
schema: z.object({ summary: z.string() }),
maxOutputTokens: 256,
+ trace,
}),
).rejects.toMatchObject({ status: 429 });
+ expect(tracedGeneration()).toMatchObject({
+ "langfuse.observation.level": "ERROR",
+ "langfuse.observation.status_message": expect.stringContaining("429"),
+ });
});
- it("reports cached prompt tokens from providers that cache automatically", async () => {
+ it("reports automatic cache usage and keeps inference successful when tracing fails", async () => {
+ publishToOtelIngestionQueue.mockRejectedValueOnce(new Error("queue down"));
vi.stubGlobal(
"fetch",
vi.fn(async () =>
@@ -198,13 +314,149 @@ describe("generateTopicText", () => {
input: "I cannot sign in.",
schema: z.object({ summary: z.string() }),
maxOutputTokens: 256,
+ trace,
});
+ expect(tracedGeneration()).toMatchObject({
+ "langfuse.observation.output": JSON.stringify({
+ summary: "Account access",
+ }),
+ "langfuse.observation.model.name": "gpt-6-luna",
+ "langfuse.observation.usage_details": JSON.stringify({
+ input: 440,
+ output: 200,
+ total: 3200,
+ input_cached_tokens: 2560,
+ }),
+ });
expect(result.usage).toMatchObject({
inputTokens: 3000,
cacheReadTokens: 2560,
});
});
+ it("retains SDK usage when no output is generated", async () => {
+ vi.stubGlobal(
+ "fetch",
+ vi.fn(async () =>
+ Response.json({
+ id: "chatcmpl-empty",
+ object: "chat.completion",
+ created: 1,
+ model: "gpt-6-luna",
+ choices: [
+ {
+ index: 0,
+ finish_reason: "length",
+ message: { role: "assistant", content: null },
+ },
+ ],
+ usage: { prompt_tokens: 12, completion_tokens: 4, total_tokens: 16 },
+ }),
+ ),
+ );
+
+ await expect(
+ generateTopicText({
+ model: {
+ slot: "summary",
+ provider: "openai-topics",
+ adapter: LLMAdapter.OpenAI,
+ model: "gpt-6-luna",
+ connection: { secretKey: encrypt("sk-test") },
+ },
+ system: [{ text: "Summarize." }],
+ input: "Hi",
+ schema: z.object({ summary: z.string() }),
+ maxOutputTokens: 256,
+ trace,
+ }),
+ ).rejects.toMatchObject({ name: "AI_NoOutputGeneratedError" });
+ expect(tracedGeneration()).toMatchObject({
+ "langfuse.observation.level": "ERROR",
+ "langfuse.observation.usage_details": JSON.stringify({
+ input: 12,
+ output: 4,
+ total: 16,
+ input_cached_tokens: 0,
+ }),
+ });
+ });
+
+ it.each([false, true])(
+ "traces invalid structured output with its paid usage (cached: %s)",
+ async (cache) => {
+ const output = { summary: 42 };
+ vi.stubGlobal(
+ "fetch",
+ vi.fn(async () =>
+ Response.json(
+ cache
+ ? {
+ choices: [
+ {
+ finish_reason: "stop",
+ message: { content: JSON.stringify(output) },
+ },
+ ],
+ usage: {
+ prompt_tokens: 12,
+ completion_tokens: 4,
+ total_tokens: 16,
+ },
+ }
+ : {
+ output: {
+ message: {
+ role: "assistant",
+ content: [
+ {
+ toolUse: {
+ toolUseId: "json-1",
+ name: "json",
+ input: output,
+ },
+ },
+ ],
+ },
+ },
+ stopReason: "tool_use",
+ usage: { inputTokens: 12, outputTokens: 4, totalTokens: 16 },
+ metrics: { latencyMs: 1 },
+ },
+ ),
+ ),
+ );
+ const response = generateTopicText({
+ model: bedrockModel("summary", "us.openai.gpt-6-luna"),
+ system: [{ text: "Summarize.", cache }],
+ input: "Hi",
+ schema: z.object({ summary: z.string() }),
+ maxOutputTokens: 256,
+ trace,
+ });
+ if (cache) {
+ await expect(response).resolves.toMatchObject({
+ output,
+ usage: { inputTokens: 12, outputTokens: 4, totalTokens: 16 },
+ });
+ } else {
+ await expect(response).rejects.toThrow();
+ }
+ const attributes = tracedGeneration();
+ expect(attributes).toMatchObject({
+ "langfuse.observation.level": "ERROR",
+ "langfuse.observation.usage_details": JSON.stringify({
+ input: 12,
+ output: 4,
+ total: 16,
+ ...(cache ? {} : { input_cached_tokens: 0, input_cache_creation: 0 }),
+ }),
+ });
+ expect(
+ JSON.parse(attributes["langfuse.observation.output"] as string),
+ ).toEqual(cache ? output : JSON.stringify(output));
+ },
+ );
});
describe("generateTopicEmbeddings", () => {
@@ -240,6 +492,7 @@ describe("generateTopicEmbeddings", () => {
output_dimension: 512,
});
expect(result.embeddings).toEqual([vector, other]);
+ expect(publishToOtelIngestionQueue).not.toHaveBeenCalled();
});
it("asks OpenAI for any configured size", async () => {
@@ -418,6 +671,7 @@ describe("allowlisted Cloud Bedrock runtime credentials", () => {
generateTopicText({ ...input, model: runtimeModel("summary", "other") }),
).rejects.toThrow("Invalid Bedrock credentials");
expect(fetch).toHaveBeenCalledTimes(1);
+ expect(publishToOtelIngestionQueue).not.toHaveBeenCalled();
});
it("uses the existing project allowlist for embeddings", async () => {
diff --git a/packages/shared/src/server/topics/text.ts b/packages/shared/src/server/topics/text.ts
index 81294b07a8..7f31ad2d4b 100644
--- a/packages/shared/src/server/topics/text.ts
+++ b/packages/shared/src/server/topics/text.ts
@@ -1,6 +1,6 @@
import { Sha256 } from "@aws-crypto/sha256-js";
import { SignatureV4 } from "@smithy/signature-v4";
-import { Output } from "ai";
+import { NoObjectGeneratedError, Output, type LanguageModelUsage } from "ai";
import { z } from "zod";
import { decrypt } from "../../encryption";
import { BedrockConfigSchema } from "../../interfaces/customLLMProviderConfigSchemas";
@@ -10,62 +10,160 @@ import {
resolveBedrockProviderAuth,
} from "../llm/ai-sdk/providers/bedrock";
import { generateLLMText } from "../llm/llmText";
-import { LLMAdapter } from "../llm/types";
+import { LLMAdapter, type TraceSinkParams } from "../llm/types";
+import { logger } from "../logger";
+import { writeInternalTraceViaOtelIngestion } from "../otel/internalTraceOtelWriter";
import type { TopicsModel } from "./model-config";
-/** A system prompt segment; `cache` ends a reusable prefix with a cache breakpoint. */
-export type TopicPromptPart = { text: string; cache?: boolean };
-
-type TopicTextResult = {
- output: unknown;
- usage: {
- inputTokens?: number;
- outputTokens?: number;
- totalTokens?: number;
- cacheReadTokens?: number;
- cacheWriteTokens?: number;
- };
-};
-
const TIMEOUT_MS = 60_000;
-const isBedrockOpenAI = (model: TopicsModel) =>
- model.adapter === LLMAdapter.Bedrock && /(^|\.)openai\./.test(model.model);
-
/**
* Structured Topics text through the project's LLM connection. GPT-5.6+ models
* on Bedrock reuse a shared prompt prefix only with explicit cache breakpoints,
* which Converse does not accept for them, so cached prompts on those models
* use InvokeModel at the same regional endpoint and model ID.
*/
-export async function generateTopicText<T>(params: {
- model: TopicsModel;
- system: TopicPromptPart[];
- input: string;
- schema: z.ZodType<T>;
- maxOutputTokens: number;
-}): Promise<TopicTextResult> {
- if (isBedrockOpenAI(params.model) && params.system.some((p) => p.cache))
- return invokeWithCache(params);
+export async function generateTopicText<T>(
+ params: TopicTextParams<T>,
+): Promise<TopicTextResult> {
+ const startTimeISO = new Date().toISOString();
+ let result: TopicTextResult | undefined;
+ let failure: unknown;
+ const recordResponse = (response: TopicTextResult) => {
+ result = response;
+ };
+ try {
+ result =
+ isBedrockGPT(params.model) && params.system.some((part) => part.cache)
+ ? await invokeWithCache(params, recordResponse)
+ : await generateWithConnection(params, recordResponse);
+ return result;
+ } catch (error) {
+ failure = error;
+ if (NoObjectGeneratedError.isInstance(error)) {
+ result = { output: error.text, usage: topicUsage(error.usage) };
+ }
+ throw error;
+ } finally {
+ await writeTopicTrace(params, startTimeISO, result, failure);
+ }
+}
+
+async function writeTopicTrace<T>(
+ params: TopicTextParams<T>,
+ startTimeISO: string,
+ result: TopicTextResult | undefined,
+ failure: unknown,
+) {
+ const { trace } = params;
+ if (!trace) return;
+ try {
+ /** Validation marks the trace; the caller validates after accounting for usage. */
+ const validation = params.schema.safeParse(result?.output);
+ await writeInternalTraceViaOtelIngestion({
+ rootSpanId: trace.traceId,
+ eventInputs: [
+ {
+ projectId: trace.targetProjectId,
+ traceId: trace.traceId,
+ spanId: trace.traceId,
+ name: trace.traceName,
+ traceName: trace.traceName,
+ type: "GENERATION",
+ environment: trace.environment,
+ startTimeISO,
+ endTimeISO: new Date().toISOString(),
+ modelName: params.model.model,
+ modelParameters: { maxOutputTokens: params.maxOutputTokens },
+ input: JSON.stringify(topicMessages(params)),
+ output: JSON.stringify(result?.output),
+ providedUsageDetails: topicTraceUsage(result?.usage),
+ metadata: trace.metadata ?? {},
+ userId: trace.userId,
+ sessionId: trace.sessionId,
+ ...topicTraceError(failure ?? validation.error),
+ },
+ ],
+ });
+ } catch (error) {
+ logger.error("Failed to publish Topics model trace", {
+ error,
+ traceId: trace.traceId,
+ });
+ }
+}
+
+function topicTraceUsage(usage: TopicTextResult["usage"] | undefined) {
+ if (!usage) return undefined;
+ const details: Record<string, number | undefined> = {
+ input: undefined,
+ output: usage.outputTokens,
+ total: usage.totalTokens,
+ input_cached_tokens: usage.cacheReadTokens,
+ input_cache_creation: usage.cacheWriteTokens,
+ };
+ /** Standard input excludes cached tokens, which have their own prices. */
+ if (usage.inputTokens !== undefined) {
+ details.input = Math.max(
+ 0,
+ usage.inputTokens -
+ (usage.cacheReadTokens ?? 0) -
+ (usage.cacheWriteTokens ?? 0),
+ );
+ }
+ return Object.fromEntries(
+ Object.entries(details).filter(
+ (entry): entry is [string, number] => entry[1] !== undefined,
+ ),
+ );
+}
+
+function topicTraceError(failure: unknown) {
+ if (failure === undefined) return {};
+ return {
+ level: "ERROR" as const,
+ statusMessage: failure instanceof Error ? failure.message : String(failure),
+ };
+}
+function topicMessages(params: { system: TopicPromptPart[]; input: string }) {
+ return [
+ {
+ role: "system" as const,
+ content: params.system.map((part) => part.text).join("\n\n"),
+ },
+ { role: "user" as const, content: params.input },
+ ];
+}
+
+function topicUsage(
+ usage: LanguageModelUsage | undefined,
+): TopicTextResult["usage"] {
+ return {
+ inputTokens: usage?.inputTokens,
+ outputTokens: usage?.outputTokens,
+ totalTokens: usage?.totalTokens,
+ cacheReadTokens: usage?.inputTokenDetails?.cacheReadTokens,
+ cacheWriteTokens: usage?.inputTokenDetails?.cacheWriteTokens,
+ };
+}
+
+async function generateWithConnection<T>(
+ params: TopicTextParams<T>,
+ recordResponse: (result: TopicTextResult) => void,
+): Promise<TopicTextResult> {
const result = await generateLLMText({
model: { adapter: params.model.adapter, id: params.model.model },
connection: params.model.connection,
// The static system prompt comes first so providers with automatic prefix
// caching can reuse it across traces.
- messages: [
- {
- role: "system",
- content: params.system.map((part) => part.text).join("\n\n"),
- },
- { role: "user", content: params.input },
- ],
+ messages: topicMessages(params),
output: Output.object({ schema: params.schema }),
maxOutputTokens: params.maxOutputTokens,
reasoning: "none",
// Bedrock ignores portable reasoning for OpenAI models; their default effort
// is not none.
- ...(isBedrockOpenAI(params.model)
+ ...(isBedrockGPT(params.model)
? {
providerOptions: {
bedrock: {
@@ -78,18 +176,28 @@ export async function generateTopicText<T>(params: {
maxRetries: 2,
timeout: TIMEOUT_MS,
});
+ const usage = topicUsage(result.usage);
+ // The SDK's output getter can throw after a response with billable usage.
+ recordResponse({ output: result.text, usage });
return {
output: result.output,
- usage: {
- inputTokens: result.usage.inputTokens,
- outputTokens: result.usage.outputTokens,
- totalTokens: result.usage.totalTokens,
- cacheReadTokens: result.usage.inputTokenDetails?.cacheReadTokens,
- cacheWriteTokens: result.usage.inputTokenDetails?.cacheWriteTokens,
- },
+ usage,
};
}
+function isBedrockGPT(model: TopicsModel): boolean {
+ if (model.adapter !== LLMAdapter.Bedrock) return false;
+ /** Opaque application inference profiles cannot identify the underlying model. */
+ const version = /(?:^|[/.])openai\.gpt-(\d+)(?:\.(\d+))?(?:[-:]|$)/.exec(
+ model.model,
+ );
+ return Boolean(
+ version &&
+ (Number(version[1]) > 5 ||
+ (Number(version[1]) === 5 && Number(version[2] ?? 0) >= 6)),
+ );
+}
+
function signingCredentials(
auth: ReturnType<typeof resolveBedrockProviderAuth>,
) {
@@ -153,13 +261,10 @@ async function bedrockRequestHeaders(
return signed.headers;
}
-async function invokeWithCache(params: {
- model: TopicsModel;
- system: TopicPromptPart[];
- input: string;
- schema: z.ZodType;
- maxOutputTokens: number;
-}): Promise<TopicTextResult> {
+async function invokeWithCache(
+ params: TopicTextParams<unknown>,
+ recordResponse: (result: TopicTextResult) => void,
+): Promise<TopicTextResult> {
const { region } = BedrockConfigSchema.parse(params.model.connection.config);
assertValidBedrockRegion(region);
const { $schema: _, ...schema } = z.toJSONSchema(params.schema);
@@ -219,14 +324,8 @@ async function invokeWithCache(params: {
};
};
const message = json.choices?.[0]?.message;
- if (!message?.content || json.choices?.[0]?.finish_reason === "length")
- throw new Error(
- message?.refusal
- ? `The model refused: ${message.refusal}`
- : "The model returned no complete output.",
- );
- return {
- output: JSON.parse(message.content) as unknown,
+ const result: TopicTextResult = {
+ output: message?.content ?? message?.refusal,
usage: {
inputTokens: json.usage?.prompt_tokens,
outputTokens: json.usage?.completion_tokens,
@@ -235,4 +334,37 @@ async function invokeWithCache(params: {
cacheWriteTokens: json.usage?.prompt_tokens_details?.cache_write_tokens,
},
};
+ // Retain paid usage and the original response if parsing or validation fails.
+ recordResponse(result);
+
+ if (!message?.content || json.choices?.[0]?.finish_reason === "length")
+ throw new Error(
+ message?.refusal
+ ? `The model refused: ${message.refusal}`
+ : "The model returned no complete output.",
+ );
+ return { ...result, output: JSON.parse(message.content) as unknown };
}
+
+/** A system prompt segment; `cache` ends a reusable prefix with a cache breakpoint. */
+export type TopicPromptPart = { text: string; cache?: boolean };
+
+type TopicTextResult = {
+ output: unknown;
+ usage: {
+ inputTokens?: number;
+ outputTokens?: number;
+ totalTokens?: number;
+ cacheReadTokens?: number;
+ cacheWriteTokens?: number;
+ };
+};
+
+type TopicTextParams<T> = {
+ model: TopicsModel;
+ system: TopicPromptPart[];
+ input: string;
+ schema: z.ZodType<T>;
+ maxOutputTokens: number;
+ trace?: TraceSinkParams;
+};
diff --git a/packages/shared/src/server/topics/trace-input.ts b/packages/shared/src/server/topics/trace-input.ts
index a507975621..9d3475870d 100644
--- a/packages/shared/src/server/topics/trace-input.ts
+++ b/packages/shared/src/server/topics/trace-input.ts
@@ -102,6 +102,9 @@ export async function loadTopicTranscript(params: {
)
.orderByColumns([
{ column: "e.event_ts", direction: "DESC" },
+ /** Use the same environment tie-breakers as Topics root discovery. */
+ { column: "e.span_id", direction: "DESC" },
+ { column: "e.environment", direction: "DESC" },
// Equal storage timestamps need a stable choice; they do not establish causal order.
{
column:
diff --git a/packages/shared/src/server/topics/trace-selection.ts b/packages/shared/src/server/topics/trace-selection.ts
index 4f3cf274d1..61660f9c4e 100644
--- a/packages/shared/src/server/topics/trace-selection.ts
+++ b/packages/shared/src/server/topics/trace-selection.ts
@@ -6,6 +6,10 @@ import { applyCommentFilters } from "../services/commentFilterService";
import { CTEQueryBuilder } from "../queries/clickhouse-sql/event-query-builder";
import { buildEventsObservationRowSelection } from "../queries/clickhouse-sql/events-observation-row-selection";
import { queryClickhouse } from "../repositories/clickhouse";
+import {
+ INTERNAL_EVAL_ENVIRONMENT_PREFIX,
+ PUBLIC_LLM_JUDGE_ENVIRONMENT,
+} from "../llm/isInternalEvalEnvironment";
type SelectionRows = {
ids: { id: string };
@@ -24,7 +28,7 @@ type SelectionRows = {
};
};
-/** Matching observations select trace identities; processing reads each full trace. */
+/** Select trace identities from matching root observations. */
export async function selectTopicTraceRows<Mode extends keyof SelectionRows>(
input: z.infer<typeof topicTraceSelectionCriteriaSchema> & {
projectId: string;
@@ -68,14 +72,10 @@ export async function selectTopicTraceRows<Mode extends keyof SelectionRows>(
.selectRaw(
"e.trace_id AS id",
"e.start_time AS start_time",
- ...(preview
- ? [
- "e.span_id AS span_id",
- "e.event_ts AS event_ts",
- `${eventsTableTraceNameSelectSql} AS trace_name`,
- "e.environment AS environment",
- ]
- : []),
+ "e.span_id AS span_id",
+ "e.event_ts AS event_ts",
+ "e.environment AS environment",
+ ...(preview ? [`${eventsTableTraceNameSelectSql} AS trace_name`] : []),
)
.whereRaw("e.trace_id != ''")
.buildWithParams();
@@ -85,9 +85,10 @@ export async function selectTopicTraceRows<Mode extends keyof SelectionRows>(
schema: [
"id",
"start_time",
- ...(preview
- ? (["span_id", "event_ts", "trace_name", "environment"] as const)
- : []),
+ "span_id",
+ "event_ts",
+ "environment",
+ ...(preview ? (["trace_name"] as const) : []),
] as const,
})
.from("matching_observations", "m")
@@ -95,14 +96,21 @@ export async function selectTopicTraceRows<Mode extends keyof SelectionRows>(
"m.id AS id",
"min(m.start_time) AS timestamp",
"max(m.start_time) AS latest_match",
+ "argMax(m.environment, tuple(m.event_ts, m.span_id, m.environment)) AS environment",
...(preview
? [
"argMaxIf(m.trace_name, tuple(m.event_ts, m.span_id, m.trace_name), m.trace_name != '') AS name",
- "argMax(m.environment, tuple(m.event_ts, m.span_id, m.environment)) AS environment",
]
: []),
)
.groupBy("m.id")
+ .havingRaw(
+ "NOT startsWith(environment, {internalEnvironmentPrefix:String}) AND environment != {publicLlmJudgeEnvironment:String}",
+ {
+ internalEnvironmentPrefix: INTERNAL_EVAL_ENVIRONMENT_PREFIX,
+ publicLlmJudgeEnvironment: PUBLIC_LLM_JUDGE_ENVIRONMENT,
+ },
+ )
.buildWithParams();
const selection = new CTEQueryBuilder()
.withCTE("matching_traces", {
@@ -111,7 +119,8 @@ export async function selectTopicTraceRows<Mode extends keyof SelectionRows>(
"id",
"timestamp",
"latest_match",
- ...(preview ? (["name", "environment"] as const) : []),
+ "environment",
+ ...(preview ? (["name"] as const) : []),
] as const,
})
.from("matching_traces", "t")
diff --git a/worker/AGENTS.md b/worker/AGENTS.md
index 09901f5311..9e4a647b3b 100644
--- a/worker/AGENTS.md
+++ b/worker/AGENTS.md
@@ -33,6 +33,11 @@
Model calls use `generateTopicText` and `generateTopicEmbeddings` from
`@langfuse/shared/topics/server` with the project's resolved `TopicsModels`
(summary, embedding, clustering LLM connections from `topic_configs`).
+ Text tracing requires the source project in
+ `LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS` (empty by default) and checks
+ organization `aiFeaturesEnabled` and `aiTelemetryEnabled` in `models.ts`;
+ embeddings are untraced. Use `isInternalEvalEnvironment`
+ to exclude internal sources.
Pass models explicitly; never put them on execution or batch state, which is
persisted. Trace-batch jobs load enabled settings for all allowlisted batch
projects in one query (`getEnabledTopicsModels`) and pause a project on
diff --git a/worker/src/features/evaluation/isEvalTargetEnvironmentAllowed.ts b/worker/src/features/evaluation/isEvalTargetEnvironmentAllowed.ts
index 42ddd939fa..700e7bab7d 100644
--- a/worker/src/features/evaluation/isEvalTargetEnvironmentAllowed.ts
+++ b/worker/src/features/evaluation/isEvalTargetEnvironmentAllowed.ts
@@ -1,17 +1,9 @@
-import { LangfuseInternalTraceEnvironment } from "@langfuse/shared/src/server";
+import {
+ LangfuseInternalTraceEnvironment,
+ isInternalEvalEnvironment,
+} from "@langfuse/shared/src/server";
-// Public ingestion strips the reserved `langfuse-` prefix from environments
-// originating outside Langfuse, including OpenRouter Broadcast callbacks.
-const PUBLIC_LLM_JUDGE_ENVIRONMENT = "llm-as-a-judge";
-
-export function isInternalEvalEnvironment(
- environment: string | null | undefined,
-): boolean {
- return (
- environment?.startsWith("langfuse") === true ||
- environment === PUBLIC_LLM_JUDGE_ENVIRONMENT
- );
-}
+export { isInternalEvalEnvironment } from "@langfuse/shared/src/server";
/**
* Final, fail-closed loop safeguard at eval EXECUTION time.
diff --git a/worker/src/features/topics/README.md b/worker/src/features/topics/README.md
index 6e9a378e3a..25408bc21f 100644
--- a/worker/src/features/topics/README.md
+++ b/worker/src/features/topics/README.md
@@ -65,10 +65,13 @@ Processing is unavailable until all three are set. Any connection whose save-tim
test call succeeds can serve summaries and clustering, including Anthropic.
Embeddings need OpenAI (including OpenAI-compatible base URLs), Azure OpenAI,
Amazon Bedrock, Google AI Studio or Vertex AI; Anthropic has no embeddings API,
-so its embedding test call fails. Summary and clustering calls go through `generateLLMText`;
-embeddings go through `embedLLMText`. Both retry 429 and 5xx responses twice
-with backoff. On Cloud, Bedrock and Vertex connections need explicit
-credentials; the default credential chain is self-hosted only.
+so its embedding test call fails. Summary and clustering calls normally go through
+`generateLLMText`; embeddings go through `embedLLMText`. Both retry 429 and 5xx
+responses twice with backoff. Cached Bedrock GPT-5.6+ summaries use a single
+InvokeModel request with explicit cache breakpoints. On Cloud, Bedrock and Vertex
+connections need explicit credentials, except operator-provisioned Bedrock
+connections in the runtime-credential allowlist; self-hosted deployments can use
+the default credential chain.
Saving the settings makes one real call per configured slot (structured output
for summaries and clustering, one embedding whose length must equal the
@@ -82,6 +85,28 @@ fails and must be started again. Deleting a connection that a slot uses turns
automatic processing off and records why. Usage is retained for any model, but
cost estimates are only available for the model IDs priced in
[models.ts](models.ts).
+
+Summary and naming calls emit one generation each to the configured internal
+AI-features project (`LANGFUSE_AI_FEATURES_PROJECT_ID`) only when the source
+project is listed in `LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS` and its
+organization enables AI features and AI analytics. The tracing allowlist is
+comma-separated, trims whitespace, and defaults to empty (tracing off). It is
+independent of the processing allowlist; projects can run Topics without tracing.
+Multi-facet summaries share one generation. Both connection-based SDK calls and
+cached Bedrock InvokeModel calls record inputs, outputs, models, usage, and errors;
+embeddings are untraced.
+Trace costs use the destination project's model catalog. Unmatched connection
+model IDs retain their usage but have no trace cost, independently of Topics'
+own estimates in `models.ts`.
+Traces use `langfuse-topics`. Topics excludes all `langfuse`-prefixed environments
+and `llm-as-a-judge` through the shared eval exclusion predicate, preventing loops
+from these internal traces. Manual selection checks matching root observations
+within the requested time range; workers also check the source environment they
+load. If an OpenRouter connection broadcasts requests back
+to a Topics-enabled project, use a dedicated Topics API key excluded from that
+Broadcast destination. OpenRouter's separate public traces do not retain the
+reserved internal environment and can otherwise trigger more Topics calls.
+
Numerical fitting uses the worker's existing Node runtime and compiled
`@langfuse/native` addon, with no extra runtime or service.
Summary and assignment records use `trace_id` as their source when present;
diff --git a/worker/src/features/topics/backfillTopicTraces.test.ts b/worker/src/features/topics/backfillTopicTraces.test.ts
index 942cdfac6b..d52781326c 100644
--- a/worker/src/features/topics/backfillTopicTraces.test.ts
+++ b/worker/src/features/topics/backfillTopicTraces.test.ts
@@ -15,7 +15,10 @@ import {
createEventsCh,
} from "@langfuse/shared/src/server";
import { prisma } from "@langfuse/shared/src/db";
-import { selectTopicTraceRows } from "@langfuse/shared/topics/server";
+import {
+ loadTopicTranscript,
+ selectTopicTraceRows,
+} from "@langfuse/shared/topics/server";
const enabled = vi.hoisted(() => vi.fn(() => true));
@@ -74,13 +77,21 @@ describe("Topics trace selection and backfill", () => {
[projectId, "trace-old", "2026-09-23T08:00:00Z"],
[projectId, "trace-outside", "2026-09-24T00:00:00Z"],
[randomUUID(), "trace-foreign", "2026-09-23T10:00:00Z"],
- ].map(([project, trace, timestamp]) =>
+ [
+ projectId,
+ "trace-internal",
+ "2026-09-23T11:00:00Z",
+ "langfuse-topics",
+ ],
+ [projectId, "trace-judge", "2026-09-23T12:00:00Z", "llm-as-a-judge"],
+ [projectId, "trace-reserved", "2026-09-23T13:00:00Z", "langfuse"],
+ ].map(([project, trace, timestamp, environment]) =>
createEvent({
project_id: project,
trace_id: trace,
start_time: new Date(timestamp),
trace_name: "agent-turn",
- environment: "test",
+ environment: environment ?? "test",
tags: ["billing"],
}),
),
@@ -130,6 +141,84 @@ describe("Topics trace selection and backfill", () => {
expect(options.jobId).toMatch(/^[a-f0-9]{64}$/);
});
+ it("filters latest matching root environments before counting and limiting", async () => {
+ const mixedProject = randomUUID();
+ const source = {
+ project_id: mixedProject,
+ trace_name: "agent-turn",
+ input: JSON.stringify({ role: "user", content: "Send an invoice" }),
+ output: null,
+ };
+ await createEventsCh(
+ [
+ ["public-root", "production", "langfuse-topics"],
+ ["equal-timestamp", "production", "langfuse-topics"],
+ ["internal-root", "langfuse-topics", "production"],
+ ]
+ .flatMap(([traceId, rootEnvironment, latestEnvironment], index) => [
+ createEvent({
+ ...source,
+ trace_id: traceId,
+ span_id: "a-root",
+ start_time: new Date(Date.UTC(2026, 8, 23, 9 + index)),
+ event_ts: new Date("2026-09-24T12:00:00Z"),
+ environment: rootEnvironment,
+ }),
+ createEvent({
+ ...source,
+ trace_id: traceId,
+ span_id: "z-child",
+ parent_span_id: "a-root",
+ start_time: new Date(Date.UTC(2026, 8, 24, 9 + index)),
+ event_ts: new Date(
+ traceId === "equal-timestamp"
+ ? "2026-09-24T12:00:00Z"
+ : "2026-09-24T13:00:00Z",
+ ),
+ environment: latestEnvironment,
+ }),
+ ])
+ .concat(
+ ["production", "langfuse-topics"].map((environment, index) =>
+ createEvent({
+ ...source,
+ trace_id: "updated-root",
+ span_id: "a-root",
+ start_time: new Date("2026-09-23T12:00:00Z"),
+ event_ts: new Date(Date.UTC(2026, 8, 24, 12 + index)),
+ environment,
+ }),
+ ),
+ ),
+ );
+ const input = { ...selection, projectId: mixedProject, limit: 1 };
+ const [preview, ids, backfill] = await Promise.all([
+ selectTopicTraceRows(input, prisma, "preview"),
+ selectTopicTraceRows(input, prisma, "ids"),
+ selectTopicTraceRows(input, prisma, "backfill"),
+ ]);
+ expect(preview).toEqual([
+ expect.objectContaining({
+ id: "equal-timestamp",
+ environment: "production",
+ }),
+ ]);
+ expect(Number(preview[0].matchedTraceCount)).toBe(2);
+ expect(ids).toEqual([{ id: "equal-timestamp" }]);
+ expect(backfill.map(({ id }) => id)).toEqual(["equal-timestamp"]);
+ expect(Number(backfill[0].matchedTraceCount)).toBe(2);
+ for (const [traceId, environment] of [
+ ["public-root", "langfuse-topics"],
+ ["internal-root", "production"],
+ ["equal-timestamp", "langfuse-topics"],
+ ["updated-root", "langfuse-topics"],
+ ]) {
+ expect(
+ await loadTopicTranscript({ projectId: mixedProject, traceId }),
+ ).toMatchObject({ environment });
+ }
+ });
+
it("keeps preview, ID-only triggers and backfill sampling aligned", async () => {
const input = {
...selection,
diff --git a/worker/src/features/topics/models.test.ts b/worker/src/features/topics/models.test.ts
index 48fb49b73b..d1eed6493f 100644
--- a/worker/src/features/topics/models.test.ts
+++ b/worker/src/features/topics/models.test.ts
@@ -1,14 +1,24 @@
-import { beforeEach, describe, expect, it, vi } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { encrypt } from "@langfuse/shared/encryption";
const state = vi.hoisted(() => ({
+ env: { LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS: ["project"] },
call: vi.fn(),
embed: vi.fn(),
increment: vi.fn(),
+ project: vi.fn(),
+ traceSink: vi.fn(),
}));
+vi.mock("@langfuse/shared/src/env", () => ({ env: state.env }));
vi.mock("@langfuse/shared/src/server", () => ({
+ getLangfuseAITraceSinkParams: state.traceSink,
+ LangfuseInternalTraceEnvironment: { Topics: "langfuse-topics" },
logger: { warn: vi.fn() },
recordIncrement: state.increment,
}));
+vi.mock("@langfuse/shared/src/db", () => ({
+ prisma: { project: { findFirst: state.project } },
+}));
vi.mock("@langfuse/shared/topics/server", () => ({
generateTopicText: (...args: unknown[]) => state.call(...args),
generateTopicEmbeddings: (...args: unknown[]) => state.embed(...args),
@@ -64,12 +74,79 @@ const evidence = {
};
beforeEach(() => {
+ state.env.LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS = ["project"];
state.call.mockReset();
state.embed.mockReset();
state.increment.mockReset();
+ state.project.mockReset().mockResolvedValue({
+ organization: { aiFeaturesEnabled: true, aiTelemetryEnabled: true },
+ });
+ state.traceSink.mockReset().mockImplementation((params) => ({
+ targetProjectId: "ai-features",
+ traceId: "a".repeat(32),
+ traceName: params.traceName,
+ environment: "production",
+ aiFeatureOtelIngestion: true,
+ metadata: {
+ langfuse_ai_feature: params.feature,
+ langfuse_project_id: params.projectId,
+ ...params.metadata,
+ },
+ }));
+});
+
+afterEach(() => {
+ vi.unstubAllGlobals();
});
describe("Topics naming boundary", () => {
+ it("keeps invalid cached Bedrock output per-trace and records paid usage", async () => {
+ const { generateTopicText } = await vi.importActual<
+ typeof import("@langfuse/shared/topics/server")
+ >("@langfuse/shared/topics/server");
+ state.call.mockImplementation(generateTopicText);
+ state.env.LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS = [];
+ vi.stubGlobal(
+ "fetch",
+ vi.fn(async () =>
+ Response.json({
+ choices: [
+ {
+ finish_reason: "stop",
+ message: {
+ content: JSON.stringify({ summary: 42, status: "applicable" }),
+ },
+ },
+ ],
+ usage: { prompt_tokens: 12, completion_tokens: 4, total_tokens: 16 },
+ }),
+ ),
+ );
+ const models = topicsModels();
+ models.summary.connection.secretKey = encrypt(
+ JSON.stringify({ accessKeyId: "AKIATOPICS", secretAccessKey: "test" }),
+ );
+
+ await expect(
+ summarizeTopicTrace(
+ models,
+ facet,
+ "Trace evidence.",
+ topicProcessingConfigSchema.parse({
+ summaryModel: models.summary.model,
+ }),
+ ),
+ ).rejects.toMatchObject({ name: "ZodError" });
+ expect(state.increment).toHaveBeenCalledWith("langfuse.topics.tokens", 12, {
+ stage: "summary",
+ direction: "input",
+ });
+ expect(state.increment).toHaveBeenCalledWith("langfuse.topics.tokens", 4, {
+ stage: "summary",
+ direction: "output",
+ });
+ });
+
it.each([
["us.openai.gpt-5.6-luna", 0.00002, 0.000036],
["us.openai.gpt-6-luna", 0.000011, 0.0000165],
@@ -108,6 +185,17 @@ describe("Topics naming boundary", () => {
expect(result.costDetails.total).toBeCloseTo(inputCost + outputCost, 10);
const request = state.call.mock.calls[0][0];
expect(request.model).toMatchObject({ slot: "summary", model });
+ expect(request.trace).toMatchObject({
+ targetProjectId: "ai-features",
+ traceName: "topics-summary",
+ environment: "langfuse-topics",
+ aiFeatureOtelIngestion: false,
+ metadata: {
+ langfuse_project_id: "project",
+ facet_id: "facet",
+ facet_version: 1,
+ },
+ });
expect(request.system).toEqual([
{ text: expect.stringContaining(facet.prompt), cache: true },
]);
@@ -256,6 +344,25 @@ describe("Topics naming boundary", () => {
expect(request.system[0].text).not.toContain("issues_2");
expect(request.input).toContain("RAW_TRANSCRIPT_SENTINEL");
expect(request.maxOutputTokens).toBe(config.maxOutputTokens * 2);
+ expect(request.trace).toMatchObject({
+ traceName: "topics-summary",
+ environment: "langfuse-topics",
+ metadata: {
+ langfuse_project_id: "project",
+ facets: [
+ { key: "intent_1", facet_id: "facet", facet_version: 1 },
+ { key: "issues_2", facet_id: "issues", facet_version: 1 },
+ ],
+ },
+ });
+ expect(state.project).toHaveBeenCalledWith({
+ where: { id: "project", deletedAt: null },
+ select: {
+ organization: {
+ select: { aiFeaturesEnabled: true, aiTelemetryEnabled: true },
+ },
+ },
+ });
// Every facet key is required in the structured output.
expect(
request.schema.safeParse({
@@ -304,6 +411,11 @@ describe("Topics naming boundary", () => {
evidenceSummaryIds: [members[399].id],
});
expect(state.call).toHaveBeenCalledOnce();
+ expect(state.call.mock.calls[0][0].trace).toMatchObject({
+ traceName: "topics-naming",
+ environment: "langfuse-topics",
+ metadata: { langfuse_project_id: "project" },
+ });
expect(state.call.mock.calls[0][0].model).toMatchObject({
slot: "clustering",
model: "us.openai.gpt-5.6-terra",
@@ -341,6 +453,92 @@ describe("Topics naming boundary", () => {
},
);
+ it.each([
+ { aiFeaturesEnabled: true, aiTelemetryEnabled: false },
+ { aiFeaturesEnabled: false, aiTelemetryEnabled: true },
+ null,
+ new Error("database unavailable"),
+ ])(
+ "keeps inference untraced without organization consent: %j",
+ async (organization) => {
+ if (organization instanceof Error)
+ state.project.mockRejectedValue(organization);
+ else
+ state.project.mockResolvedValue(organization ? { organization } : null);
+ state.call
+ .mockResolvedValueOnce({
+ output: { summary: "A billing request.", status: "applicable" },
+ usage: {},
+ })
+ .mockResolvedValueOnce({
+ output: {
+ name: "Billing",
+ description: "Billing requests.",
+ evidenceSummaryIds: ["member-a"],
+ },
+ usage: {},
+ });
+ await summarizeTopicTrace(
+ topicsModels(),
+ facet,
+ "A billing request.",
+ topicProcessingConfigSchema.parse({
+ summaryModel: "us.openai.gpt-5.6-luna",
+ }),
+ );
+ await nameTopicGroup(topicsModels(), evidence);
+ state.call.mockResolvedValueOnce({
+ output: {
+ intent: {
+ notes: "Invoice request.",
+ summary: "Request invoice.",
+ status: "applicable",
+ },
+ },
+ usage: {},
+ });
+ await summarizeTopicTraceFacets(
+ topicsModels(),
+ [{ key: "intent", facet, builtIn: true }],
+ "Request invoice.",
+ topicProcessingConfigSchema.parse({
+ summaryModel: "us.openai.gpt-5.6-luna",
+ }),
+ );
+ expect(state.call).toHaveBeenCalledTimes(3);
+ for (const [request] of state.call.mock.calls)
+ expect(request.trace).toBeUndefined();
+ },
+ );
+
+ it.each(["unconfigured destination", "empty allowlist", "unlisted source"])(
+ "keeps inference untraced and skips the consent lookup with %s",
+ async (reason) => {
+ if (reason === "unconfigured destination")
+ state.traceSink.mockReturnValue(undefined);
+ else
+ state.env.LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS =
+ reason === "empty allowlist" ? [] : ["ai-features", "project-other"];
+ state.call.mockResolvedValue({
+ output: { summary: "A billing request.", status: "applicable" },
+ usage: {},
+ });
+ await summarizeTopicTrace(
+ topicsModels(),
+ facet,
+ "A billing request.",
+ topicProcessingConfigSchema.parse({
+ summaryModel: "us.openai.gpt-5.6-luna",
+ }),
+ );
+ expect(state.project).not.toHaveBeenCalled();
+ expect(state.call).toHaveBeenCalledOnce();
+ expect(state.call.mock.calls[0][0].trace).toBeUndefined();
+ if (reason !== "unconfigured destination")
+ expect(state.traceSink).not.toHaveBeenCalled();
+ },
+ );
+
it("records embedding input usage even when the returned vector is invalid", async () => {
state.embed.mockResolvedValue({ embeddings: [[]], tokens: 12 });
await expect(
diff --git a/worker/src/features/topics/models.ts b/worker/src/features/topics/models.ts
index 7d520ea354..21c6e925cf 100644
--- a/worker/src/features/topics/models.ts
+++ b/worker/src/features/topics/models.ts
@@ -1,6 +1,12 @@
import { get_encoding } from "tiktoken";
import { z } from "zod";
-import { logger } from "@langfuse/shared/src/server";
+import { prisma } from "@langfuse/shared/src/db";
+import { env } from "@langfuse/shared/src/env";
+import {
+ logger,
+ getLangfuseAITraceSinkParams,
+ LangfuseInternalTraceEnvironment,
+} from "@langfuse/shared/src/server";
import {
generateTopicEmbeddings,
generateTopicText,
@@ -89,6 +95,8 @@ async function structuredCall<T>(
outputLimit: number,
stage: "summary" | "naming",
model: TopicsModel,
+ projectId: string,
+ metadata?: Record<string, unknown>,
): Promise<ModelResult<T>> {
// Unknown model IDs record usage without cost.
const rates = (tokens: number): Rates | undefined =>
@@ -101,6 +109,7 @@ async function structuredCall<T>(
input,
schema,
maxOutputTokens: outputLimit,
+ trace: await topicTraceSink(stage, projectId, metadata),
}).catch((error: unknown) => {
logger.warn("Topics model request failed", {
model: model.model,
@@ -162,6 +171,45 @@ async function structuredCall<T>(
return accepted;
}
+async function topicTraceSink(
+ stage: "summary" | "naming",
+ projectId: string,
+ metadata?: Record<string, unknown>,
+) {
+ if (!env.LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS?.includes(projectId))
+ return undefined;
+ const trace = getLangfuseAITraceSinkParams({
+ feature: "topics",
+ projectId,
+ traceName: `topics-${stage}`,
+ metadata,
+ });
+ if (!trace) return undefined;
+ const project = await prisma.project
+ .findFirst({
+ where: { id: projectId, deletedAt: null },
+ select: {
+ organization: {
+ select: { aiFeaturesEnabled: true, aiTelemetryEnabled: true },
+ },
+ },
+ })
+ .catch((error: unknown) => {
+ logger.warn("Failed to check Topics tracing consent", { error });
+ return null;
+ });
+ if (
+ !project?.organization.aiFeaturesEnabled ||
+ !project.organization.aiTelemetryEnabled
+ )
+ return undefined;
+ return {
+ ...trace,
+ environment: LangfuseInternalTraceEnvironment.Topics,
+ aiFeatureOtelIngestion: false,
+ };
+}
+
const SUMMARY_SYSTEM_PROMPT = `You describe one facet of a recorded run of an LLM application. Your description is embedded and clustered together with descriptions of many other runs. Runs that share a pattern should get similar descriptions; runs that differ in a way that matters should not.
<transcript_format>
@@ -202,6 +250,7 @@ export async function summarizeTopicTrace(
facet: TopicFacetVersion,
text: string,
config: TopicProcessingConfig,
+ metadata?: Record<string, unknown>,
) {
requireFrozenSummaryModel(models, config);
const system = [
@@ -227,6 +276,8 @@ export async function summarizeTopicTrace(
config.maxOutputTokens,
"summary",
models.summary,
+ models.projectId,
+ { ...metadata, facet_id: facet.facetId, facet_version: facet.version },
);
}
@@ -242,6 +293,7 @@ export async function summarizeTopicTraceFacets(
facets: { key: string; facet: TopicFacetVersion; builtIn: boolean }[],
text: string,
config: TopicProcessingConfig,
+ metadata?: Record<string, unknown>,
) {
requireFrozenSummaryModel(models, config);
const entry = z.object({
@@ -294,6 +346,15 @@ ${facetBlock(true)}`,
config.maxOutputTokens * facets.length,
"summary",
models.summary,
+ models.projectId,
+ {
+ ...metadata,
+ facets: facets.map(({ key, facet }) => ({
+ key,
+ facet_id: facet.facetId,
+ facet_version: facet.version,
+ })),
+ },
);
}
@@ -303,6 +364,7 @@ export async function nameTopicGroup(
members: { id: string; summary: string }[];
contrasts: { id: string; summary: string }[];
},
+ metadata?: Record<string, unknown>,
) {
if (!group.members.length)
throw new Error("Naming requires one non-empty effective group.");
@@ -345,6 +407,8 @@ export async function nameTopicGroup(
1000,
"naming",
models.clustering,
+ models.projectId,
+ metadata,
);
}
diff --git a/worker/src/features/topics/processTopicEmbeddingBatch.test.ts b/worker/src/features/topics/processTopicEmbeddingBatch.test.ts
index edb5a29855..ec7901e951 100644
--- a/worker/src/features/topics/processTopicEmbeddingBatch.test.ts
+++ b/worker/src/features/topics/processTopicEmbeddingBatch.test.ts
@@ -35,11 +35,16 @@ vi.mock("@langfuse/shared/topics/server", () => ({
TOPIC_EMBEDDING_EXPIRED_ERROR:
"Topics staged results expired before processing completed. Start a new execution with stored-summary reuse to recover persisted results.",
}));
-vi.mock("@langfuse/shared/src/server", () => ({
- logger: { error: vi.fn() },
- recordIncrement: vi.fn(),
- recordDistribution: vi.fn(),
-}));
+vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
+ const { isInternalEvalEnvironment } =
+ await importOriginal<typeof import("@langfuse/shared/src/server")>();
+ return {
+ isInternalEvalEnvironment,
+ logger: { error: vi.fn() },
+ recordIncrement: vi.fn(),
+ recordDistribution: vi.fn(),
+ };
+});
vi.mock("./models", () => ({
embedTopicSummaries: mocks.embed,
}));
@@ -122,6 +127,36 @@ beforeEach(() => {
});
describe("Topics embedding handoff", () => {
+ it.each(["summarized", "complete"] as const)(
+ "skips internal staged summaries in a mixed batch (%s)",
+ async (state) => {
+ const row: TopicSummary = {
+ ...summary(),
+ state,
+ environment: "langfuse-topics",
+ embedding: state === "complete" ? Array(256).fill(0.25) : [],
+ };
+ const allowed = summary("allowed");
+ staged.set(topicSourceKey(row), row);
+ staged.set(topicSourceKey(allowed), allowed);
+
+ await processTopicEmbeddingBatch(batch(row, allowed));
+
+ expect(mocks.embed).toHaveBeenCalledExactlyOnceWith(
+ expect.anything(),
+ [allowed.summary],
+ 256,
+ );
+ expect(mocks.update).toHaveBeenCalledTimes(1);
+ expect(mocks.write).toHaveBeenCalledExactlyOnceWith([
+ expect.objectContaining({
+ traceId: allowed.traceId,
+ state: "complete",
+ }),
+ ]);
+ },
+ );
+
it("retries combined results from Redis after a failed ClickHouse write", async () => {
const applicable = summary();
const nonApplicable: TopicSummary = {
diff --git a/worker/src/features/topics/processTopicEmbeddingBatch.ts b/worker/src/features/topics/processTopicEmbeddingBatch.ts
index 1e6d7bbfd1..b085d7f8fe 100644
--- a/worker/src/features/topics/processTopicEmbeddingBatch.ts
+++ b/worker/src/features/topics/processTopicEmbeddingBatch.ts
@@ -1,4 +1,5 @@
import { UnrecoverableError } from "bullmq";
+import { isInternalEvalEnvironment } from "@langfuse/shared/src/server";
import pLimit from "p-limit";
import type { TopicSummary } from "@langfuse/shared/topics";
import {
@@ -20,7 +21,7 @@ import {
TopicsProviderUnavailable,
} from "./provider-error";
-/** Acknowledging a batch means every result is durably stored in ClickHouse. */
+/** Acknowledging a batch means every eligible result is durably stored in ClickHouse. */
export async function processTopicEmbeddingBatch(
batch: TopicEmbeddingBatch,
): Promise<void> {
@@ -42,6 +43,7 @@ export async function processTopicEmbeddingBatch(
for (const ref of batch.summaries) {
const staged = await readStagedTopicSummary(batch, ref);
if (!staged) throw new UnrecoverableError(TOPIC_EMBEDDING_EXPIRED_ERROR);
+ if (isInternalEvalEnvironment(staged.summary.environment)) continue;
if (staged.embeddingConfig.embeddingModel !== models.embedding.model)
throw new TopicsProviderUnavailable(
"The project's embedding model changed after this batch was created. Start a new Topics execution.",
diff --git a/worker/src/features/topics/processTopicsExecution.test.ts b/worker/src/features/topics/processTopicsExecution.test.ts
index 785641ff97..97ea55f3fb 100644
--- a/worker/src/features/topics/processTopicsExecution.test.ts
+++ b/worker/src/features/topics/processTopicsExecution.test.ts
@@ -51,10 +51,11 @@ const state = vi.hoisted(() => ({
numeric: vi.fn(),
}));
vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
- const { assembleTranscript } =
+ const { assembleTranscript, isInternalEvalEnvironment } =
await importOriginal<typeof import("@langfuse/shared/src/server")>();
return {
assembleTranscript,
+ isInternalEvalEnvironment,
recordIncrement: vi.fn(),
recordDistribution: vi.fn(),
};
@@ -576,6 +577,88 @@ beforeEach(() => {
});
describe("Topics execution", () => {
+ it.each([false, true])(
+ "rejects internal traces supplied by ID (cached: %s)",
+ async (cached) => {
+ const blocked = execution("blocked", 1);
+ if (cached) {
+ await processSelection("source", 1);
+ const summary = [...state.summaries.values()][0];
+ state.staged.set(`blocked/${topicSourceKey(summary)}`, {
+ summary: { ...summary, environment: "langfuse-topics" },
+ embeddingConfig: blocked.input.embeddingConfig,
+ });
+ state.summaries.clear();
+ state.summarize.mockClear();
+ state.embed.mockClear();
+ } else {
+ state.loadTranscript.mockResolvedValue({
+ ...traceInput("trace0"),
+ environment: "langfuse-topics",
+ });
+ }
+ state.executions.set("blocked", blocked);
+ await processTopicsExecution({
+ projectId: "project",
+ executionId: "blocked",
+ });
+ expect(state.summarize).not.toHaveBeenCalled();
+ expect(state.embed).not.toHaveBeenCalled();
+ expect(state.summaries.size).toBe(0);
+ expect(state.executions.get("blocked")?.facets[0].counts.failed).toBe(1);
+ },
+ );
+
+ it.each([false, true])(
+ "resumes mixed internal staged summaries without repeating paid work (embedded: %s)",
+ async (embedded) => {
+ state.deferEmbeddings = true;
+ await processSelection("resumed", 2);
+ const saved = state.batches.get("resumed")!;
+ expect(saved.summarized).toBe(true);
+ const staged = [...state.staged.values()];
+ staged[0].summary.environment = "langfuse-topics";
+ if (embedded) {
+ for (const row of staged) {
+ row.summary.state = "complete";
+ row.summary.embedding = Array(256).fill(0.25);
+ }
+ saved.assignedAt = "2026-01-01T00:01:00.000Z";
+ }
+ state.deferEmbeddings = false;
+ state.summarize.mockClear();
+ state.embed.mockClear();
+
+ for (let attempt = 0; attempt < 2; attempt++) {
+ await processTopicsExecution({
+ projectId: "project",
+ executionId: "resumed",
+ });
+ expect(state.executions.get("resumed")).toMatchObject({
+ status: "completed_with_errors",
+ error: null,
+ facets: [{ counts: { complete: 1, failed: 1 } }],
+ traceErrors: [
+ {
+ traceId: staged[0].summary.traceId,
+ error: "Topics does not process internal Langfuse traces.",
+ },
+ ],
+ });
+ expect(state.batches.get("resumed")?.summaries).toEqual([
+ expect.objectContaining({ traceId: staged[1].summary.traceId }),
+ ]);
+ expect(state.staged.size).toBe(0);
+ }
+ expect(state.summarize).not.toHaveBeenCalled();
+ expect(state.embed).toHaveBeenCalledTimes(embedded ? 0 : 1);
+ if (!embedded)
+ expect([...state.summaries.values()]).toMatchObject([
+ { traceId: staged[1].summary.traceId, state: "complete" },
+ ]);
+ },
+ );
+
it("records null transcripts as insufficient input without inference", async () => {
state.loadTranscript.mockResolvedValueOnce(traceInput("trace0", null));
await processSelection("empty-transcript", 1);
diff --git a/worker/src/features/topics/processTopicsExecution.ts b/worker/src/features/topics/processTopicsExecution.ts
index 1c0247d507..b9a1b57449 100644
--- a/worker/src/features/topics/processTopicsExecution.ts
+++ b/worker/src/features/topics/processTopicsExecution.ts
@@ -1,4 +1,4 @@
-import { logger } from "@langfuse/shared/src/server";
+import { isInternalEvalEnvironment, logger } from "@langfuse/shared/src/server";
import {
isTopicsEnabled,
readTopicExecutionSummary,
@@ -113,12 +113,16 @@ async function summarizeTrace(
): Promise<TopicSummary> {
const accepted = staged.find((row) => row.traceId === traceId);
if (accepted) {
+ if (isInternalEvalEnvironment(accepted.environment))
+ throw new Error("Topics does not process internal Langfuse traces.");
metrics.result("summary", "cached");
// An accepted payload may expire after the cache read; never renew its TTL.
return accepted;
}
const { unitStartTime, sessionId, environment, traceName, transcript } =
await getTranscript();
+ if (isInternalEvalEnvironment(environment))
+ throw new Error("Topics does not process internal Langfuse traces.");
const source = {
projectId: execution.projectId,
facetId: facet.facetId,
@@ -193,6 +197,11 @@ async function summarizeTrace(
facet,
JSON.stringify(transcript),
execution.input.processingConfig,
+ {
+ source_trace_id: traceId,
+ source_environment: environment,
+ execution_id: execution.id,
+ },
);
const error = topicSummaryOutputError(result.output);
if (error) throw invalidOutput(metrics, "summary", error);
@@ -276,8 +285,13 @@ async function releaseCompletedSummaries(state: TopicProcessBatchState) {
)
return;
try {
+ const refs = state.summaries.concat(
+ state.failedTraceIds.flatMap(({ facetId, facetVersion, traceIds }) =>
+ traceIds.map((traceId) => ({ facetId, facetVersion, traceId })),
+ ),
+ );
await Promise.all(
- state.summaries.map((ref) =>
+ refs.map((ref) =>
deleteStagedTopicSummary(
{
projectId: state.execution.projectId,
@@ -646,16 +660,26 @@ async function clusterFacet(
const members = new Map(
group.members.map((summary, index) => [`m${index + 1}`, summary]),
);
- const { output: label } = await nameTopicGroup(models, {
- members: Array.from(members, ([id, { summary }]) => ({
- id,
- summary,
- })),
- contrasts: group.contrasts.map((member, index) => ({
- id: `c${index + 1}`,
- summary: member.summary,
- })),
- });
+ const { output: label } = await nameTopicGroup(
+ models,
+ {
+ members: Array.from(members, ([id, { summary }]) => ({
+ id,
+ summary,
+ })),
+ contrasts: group.contrasts.map((member, index) => ({
+ id: `c${index + 1}`,
+ summary: member.summary,
+ })),
+ },
+ {
+ facet_id: facet.facetId,
+ facet_version: facet.version,
+ execution_id: execution.id,
+ run_id: run.id,
+ group_id: group.id,
+ },
+ );
if (
!label.name.trim() ||
!label.description.trim() ||
@@ -888,28 +912,16 @@ async function processTraces(
aborted = true;
throw error;
}
- let failed = state.failedTraceIds.find(
- (entry) =>
- entry.facetId === facet.facetId &&
- entry.facetVersion === facet.version,
- );
- if (!failed) {
- failed = {
+ recordTraceFailure(
+ state,
+ progress,
+ {
facetId: facet.facetId,
facetVersion: facet.version,
- traceIds: [],
- };
- state.failedTraceIds.push(failed);
- }
- failed.traceIds.push(traceId);
- progress.counts.failed = failed.traceIds.length;
- const message = errorMessage(error);
- if (
- !execution.traceErrors.some(
- (item) => item.traceId === traceId && item.error === message,
- )
- )
- execution.traceErrors.push({ traceId, error: message });
+ traceId,
+ },
+ error,
+ );
await saveInOrder();
}
}
@@ -928,7 +940,7 @@ async function processTraces(
state.assignedAt = new Date().toISOString();
await save("assigning");
}
- const summaries = await Promise.all(
+ const stagedSummaries = await Promise.all(
state.summaries
.filter((ref) =>
facets.some(
@@ -943,13 +955,30 @@ async function processTraces(
);
if (!staged)
throw new TopicEmbeddingFailure(TOPIC_EMBEDDING_EXPIRED_ERROR);
- if (staged.summary.state === "summarized")
- throw new TopicEmbeddingFailure(
- "Topics embedding batch has an unfinished result.",
- );
- return staged.summary;
+ return { ref, summary: staged.summary };
}),
);
+ const summaries: TopicSummary[] = [];
+ for (const { ref, summary } of stagedSummaries) {
+ if (isInternalEvalEnvironment(summary.environment)) {
+ const { progress } = facets.find(
+ ({ facet }) =>
+ facet.facetId === ref.facetId && facet.version === ref.facetVersion,
+ )!;
+ const error = new Error(
+ "Topics does not process internal Langfuse traces.",
+ );
+ metrics.error("summary", error);
+ recordTraceFailure(state, progress, ref, error);
+ state.summaries = state.summaries.filter((entry) => entry !== ref);
+ continue;
+ }
+ if (summary.state === "summarized")
+ throw new TopicEmbeddingFailure(
+ "Topics embedding batch has an unfinished result.",
+ );
+ summaries.push(summary);
+ }
for (const { facet, progress, run } of facets) {
try {
const selected = summaries.filter(
@@ -987,6 +1016,30 @@ async function processTraces(
}
}
+function recordTraceFailure(
+ state: TopicProcessBatchState,
+ progress: TopicFacetProgress,
+ { facetId, facetVersion, traceId }: TopicEmbeddingRef,
+ error: unknown,
+) {
+ let failed = state.failedTraceIds.find(
+ (entry) => entry.facetId === facetId && entry.facetVersion === facetVersion,
+ );
+ if (!failed) {
+ failed = { facetId, facetVersion, traceIds: [] };
+ state.failedTraceIds.push(failed);
+ }
+ if (!failed.traceIds.includes(traceId)) failed.traceIds.push(traceId);
+ progress.counts.failed = failed.traceIds.length;
+ const message = errorMessage(error);
+ if (
+ !state.execution.traceErrors.some(
+ (item) => item.traceId === traceId && item.error === message,
+ )
+ )
+ state.execution.traceErrors.push({ traceId, error: message });
+}
+
/** Each unpublished attempt fits the current compatible summaries from scratch. */
async function updateTopics(
metrics: TopicMetrics,
diff --git a/worker/src/features/topics/storage.integration.test.ts b/worker/src/features/topics/storage.integration.test.ts
index 27853b45f0..994eef2cf3 100644
--- a/worker/src/features/topics/storage.integration.test.ts
+++ b/worker/src/features/topics/storage.integration.test.ts
@@ -595,6 +595,62 @@ describe("Topics definition persistence", () => {
expect(
await getLatestFacetSummaries(projectId, "facet", 1, newRange),
).toEqual([moved]);
+
+ const internalFacet = "internal-environments";
+ const internalTraces = [
+ "langfuse-topics",
+ "llm-as-a-judge",
+ "langfuse",
+ ].map((environment) => ({ traceId: randomUUID(), environment }));
+ await writeTopicSummaries([
+ ...internalTraces.flatMap(({ traceId, environment }) => {
+ return [
+ { ...summary, facetId: internalFacet, traceId },
+ {
+ ...latest,
+ facetId: internalFacet,
+ traceId,
+ environment,
+ },
+ ];
+ }),
+ { ...summary, facetId: internalFacet, traceId: "allowed-trace" },
+ ]);
+ const [eligible, counts, listed, latestDisplayed] = await Promise.all([
+ getTopicClusteringSummaries(
+ projectId,
+ internalFacet,
+ 1,
+ embedding,
+ oldRange,
+ ),
+ getTopicSummaryCounts(
+ projectId,
+ [{ facetId: internalFacet, version: 1 }],
+ embedding,
+ oldRange,
+ ),
+ listTopicSummaries(
+ projectId,
+ {
+ facetId: internalFacet,
+ facetVersion: 1,
+ traceIds: ["allowed-trace"].concat(
+ internalTraces.map(({ traceId }) => traceId),
+ ),
+ },
+ oldRange,
+ ),
+ getLatestFacetSummaries(projectId, internalFacet, undefined, oldRange),
+ ]);
+ expect(eligible.map(({ traceId }) => traceId)).toEqual(["allowed-trace"]);
+ expect(listed.map(({ traceId }) => traceId)).toEqual(["allowed-trace"]);
+ expect(latestDisplayed.map(({ traceId }) => traceId)).toEqual([
+ "allowed-trace",
+ ]);
+ expect(counts).toEqual([
+ { facetId: internalFacet, facetVersion: 1, count: 1 },
+ ]);
});
it("round-trips Float64 geometry and normalized source references through ClickHouse", async () => {
diff --git a/worker/src/features/topics/summarizeAssembledTrace.test.ts b/worker/src/features/topics/summarizeAssembledTrace.test.ts
index 00cc8d47c0..cf4c28ae12 100644
--- a/worker/src/features/topics/summarizeAssembledTrace.test.ts
+++ b/worker/src/features/topics/summarizeAssembledTrace.test.ts
@@ -21,7 +21,10 @@ vi.mock("@langfuse/shared/topics/server", () => ({
writeTopicSummaries: (...args: unknown[]) => state.write(...args),
TOPICS_TRANSCRIPT_VERSION: "shared-transcript-v3",
}));
-vi.mock("@langfuse/shared/src/server", () => ({
+vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({
+ isInternalEvalEnvironment: (
+ await importOriginal<typeof import("@langfuse/shared/src/server")>()
+ ).isInternalEvalEnvironment,
recordDistribution: vi.fn(),
recordIncrement: vi.fn(),
instrumentAsync: async (
@@ -133,6 +136,25 @@ beforeEach(() => {
});
describe("summarizeAssembledTrace", () => {
+ it("skips internal traces before reading summaries, calling models or writing", async () => {
+ const result = await summarizeAssembledTrace({
+ projectId: "project-a",
+ traceId: "trace-1",
+ traceTimestamp: "2026-09-22T12:00:00.000Z",
+ jobEnqueuedAt: new Date("2026-09-22T12:01:00.000Z"),
+ environment: "langfuse-topics",
+ traceName: "topics-summary",
+ transcript,
+ models,
+ facets: [facet],
+ });
+ expect(result).toBe("disabled");
+ expect(state.stored).not.toHaveBeenCalled();
+ expect(state.summarize).not.toHaveBeenCalled();
+ expect(state.embed).not.toHaveBeenCalled();
+ expect(state.write).not.toHaveBeenCalled();
+ });
+
it("skips a finished job retry but summarizes a later redispatch", async () => {
await summarizeAssembledTrace({
projectId: "project-a",
@@ -153,6 +175,10 @@ describe("summarizeAssembledTrace", () => {
]);
expect(submitted[2]).toContain("Export monthly sales");
expect(submitted[2]).not.toContain("observation");
+ expect(submitted[4]).toEqual({
+ source_trace_id: "trace-1",
+ source_environment: "default",
+ });
const written = state.write.mock.calls[0][0][0];
expect(written).toMatchObject({
projectId: "project-a",
diff --git a/worker/src/features/topics/summarizeAssembledTrace.ts b/worker/src/features/topics/summarizeAssembledTrace.ts
index 035485f898..3928f8d184 100644
--- a/worker/src/features/topics/summarizeAssembledTrace.ts
+++ b/worker/src/features/topics/summarizeAssembledTrace.ts
@@ -12,6 +12,7 @@ import {
} from "@langfuse/shared/topics";
import {
instrumentAsync,
+ isInternalEvalEnvironment,
recordIncrement,
type Transcript,
} from "@langfuse/shared/src/server";
@@ -55,7 +56,8 @@ type AssembledTraceInput = {
*/
export async function summarizeAssembledTrace(
input: AssembledTraceInput,
-): Promise<"unchanged" | "summarized"> {
+): Promise<"disabled" | "unchanged" | "summarized"> {
+ if (isInternalEvalEnvironment(input.environment)) return "disabled";
return instrumentAsync(
{ name: "topics-trace-summary", traceScope: "topics" },
async (span) => {
@@ -148,6 +150,10 @@ async function summarizeEnabledTrace(
keyed,
prepared.text,
config,
+ {
+ source_trace_id: input.traceId,
+ source_environment: input.environment,
+ },
);
const outputs = generated.output as Record<string, FacetOutput>;
for (const { key } of keyed) {
diff --git a/worker/src/queues/__tests__/traceBatchQueue.test.ts b/worker/src/queues/__tests__/traceBatchQueue.test.ts
index 0134d2a231..ec8605a8a9 100644
--- a/worker/src/queues/__tests__/traceBatchQueue.test.ts
+++ b/worker/src/queues/__tests__/traceBatchQueue.test.ts
@@ -920,17 +920,17 @@ describe("trace batch queue", () => {
);
});
- it("assembles each tenant's trace at the boundary and EOF", async () => {
+ it("assembles each tenant's trace with its latest environment at the boundary and EOF", async () => {
const userMessage = { role: "user", content: "first question" };
const answer = { role: "assistant", content: "first answer" };
const row = {
project_id: "a",
trace_id: "shared-trace",
- environment: "production",
+ environment: "langfuse-topics",
span_id: "first",
parent_span_id: null,
start_time: "2026-09-11 00:00:00.000000",
- event_ts: "2026-09-11 00:00:00.000000",
+ event_ts: "2026-09-11 00:00:00.000002",
type: "GENERATION",
name: "generation",
input: JSON.stringify([userMessage]),
@@ -947,6 +947,8 @@ describe("trace batch queue", () => {
yield {
...row,
span_id: "second",
+ environment: "production",
+ event_ts: "2026-09-11 00:00:00.000001",
start_time: "2026-09-11 00:01:00.000000",
input: JSON.stringify([
userMessage,
@@ -955,18 +957,19 @@ describe("trace batch queue", () => {
]),
output: JSON.stringify({ role: "assistant", content: "second answer" }),
};
+ // The newer internal event has an earlier start time and smaller span ID.
yield row;
expect(tokenCountAsync).not.toHaveBeenCalled();
yield {
...row,
project_id: "b",
- environment: "staging",
+ environment: "langfuse-topics",
type: "SPAN",
};
yield {
...row,
project_id: "b",
- environment: "staging",
+ environment: "production",
type: "SPAN",
span_id: "second",
};
@@ -994,8 +997,8 @@ describe("trace batch queue", () => {
environment: input.environment,
})),
).toEqual([
- { projectId: "a", environment: "production" },
- { projectId: "b", environment: "staging" },
+ { projectId: "a", environment: "langfuse-topics" },
+ { projectId: "b", environment: "production" },
]);
const estimates = vi.mocked(tokenCountAsync).mock.calls;
expect(estimates).toHaveLength(5);
diff --git a/worker/src/queues/traceBatchQueue.ts b/worker/src/queues/traceBatchQueue.ts
index b94cdee21b..13ae83e8db 100644
--- a/worker/src/queues/traceBatchQueue.ts
+++ b/worker/src/queues/traceBatchQueue.ts
@@ -36,6 +36,8 @@ import {
import { summarizeAssembledTrace } from "../features/topics/summarizeAssembledTrace";
import { recordTraceBatchTranscript } from "../features/traceBatching/traceBatchTranscript";
+type TraceBatchObservation = Observation & { eventTimestamp: string };
+
type TraceOutcome =
| { outcome: "disabled" | "unchanged" | "summarized" }
| { outcome: "failed"; reason: string };
@@ -97,7 +99,7 @@ async function loadProjectRule(projectId: string) {
}
async function summarizeTraceBatch(
- observations: Observation[],
+ observations: TraceBatchObservation[],
transcript: Transcript | null,
topics: TopicsBatchProjects,
jobEnqueuedAt: Date,
@@ -117,6 +119,13 @@ async function summarizeTraceBatch(
const traceTimestamp = new Date(
Math.min(...observations.map((row) => row.startTime.getTime())),
).toISOString();
+ // Match manual processing and discovery, preserving event_ts microseconds.
+ const latest = observations.reduce((current, row) => {
+ if (row.eventTimestamp !== current.eventTimestamp)
+ return row.eventTimestamp > current.eventTimestamp ? row : current;
+ if (row.id !== current.id) return row.id > current.id ? row : current;
+ return row.environment > current.environment ? row : current;
+ });
try {
const project = await topics.get(first.projectId);
if (
@@ -133,7 +142,7 @@ async function summarizeTraceBatch(
traceId,
traceTimestamp,
jobEnqueuedAt,
- environment: first.environment,
+ environment: latest.environment,
traceName: first.name ?? "",
transcript,
models: project.models,
@@ -306,7 +315,10 @@ export const traceBatchQueueProcessor: Processor<
let pendingSummaryBytes = 0;
let summaryFailed = false;
let summaryError: unknown;
- const enqueueSummary = (observations: Observation[], bytes: number) => {
+ const enqueueSummary = (
+ observations: TraceBatchObservation[],
+ bytes: number,
+ ) => {
pendingSummaryBytes += bytes;
const enqueuedAt = performance.now();
const transcribed = transcribing.then(async () => {
@@ -352,7 +364,7 @@ export const traceBatchQueueProcessor: Processor<
activeReads++;
try {
recordTraceBatchActiveReads();
- let traceObservations: Observation[] = [];
+ let traceObservations: TraceBatchObservation[] = [];
let traceBytes = 0;
for await (const event of getTraceBatchEventStream(batch, queryOptions)) {
observationCount++;
@@ -394,8 +406,8 @@ export const traceBatchQueueProcessor: Processor<
if (summaryFailed) break;
}
traceBytes += eventBytes;
- traceObservations.push(
- convertObservation({
+ traceObservations.push({
+ ...convertObservation({
...event,
id: event.span_id,
parent_observation_id: event.parent_span_id,
@@ -408,7 +420,8 @@ export const traceBatchQueueProcessor: Processor<
usage_details: {},
cost_details: {},
}),
- );
+ eventTimestamp: event.event_ts,
+ });
}
// Reaching EOF completes the last trace; a failed stream must not flush it.
if (traceObservations.length && !summaryFailed)