🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(topics): trace summary and naming calls with org analytics opt-in (#18346)
Nimar3eb71e89be4f
diff --git a/packages/shared/AGENTS.md b/packages/shared/AGENTS.mdindex 212d41d7bb..2c3232990f 100644--- a/packages/shared/AGENTS.md+++ b/packages/shared/AGENTS.md@@ -93,8 +93,10 @@Summary references carry facet/version/source fields within project andexecution scope; exact storage reads also require facet/version scope.- `text.ts` and `embeddings.ts`: model transport through the project's LLM- connections (`generateLLMText` / `embedLLMText`); worker model calls own- usage, cost and vector validation.+ connections (`generateLLMText` / `embedLLMText`), with cached Bedrock text+ calls using InvokeModel. Text accepts optional internal generation tracing;+ worker model calls own tracing allowlisting, organization consent, usage,+ cost and vector validation.- `model-config.ts`: per-project `topic_configs` (three connection +model slots and the automatic-processing switch). Saving tests every slotwith a real call; Anthropic works for summaries and clustering but fails the@@ -134,7 +136,8 @@for shared backend services, repositories, queue helpers/contracts, Redis andClickHouse helpers, auth helpers, logger/instrumentation, ingestion helpers,AI SDK-native LLM execution helpers (`generateLLMText`, `streamLLMText`, and- `embedLLMText`), Bedrock default-credential provider auth+ `embedLLMText`), `isInternalEvalEnvironment` for shared eval/Topics exclusions,+ Bedrock default-credential provider auth(`createDefaultBedrockProviderAuth`), and server test utilities. Langfuse AIcallers select their local profile through `getLangfuseAIAwsProfile`;generic Bedrock auth only uses an explicitly supplied profile or the normaldiff --git a/packages/shared/src/env.ts b/packages/shared/src/env.tsindex 324f213738..628ebdca33 100644--- a/packages/shared/src/env.ts+++ b/packages/shared/src/env.ts@@ -53,6 +53,15 @@ const EnvSchema = z.object({.map((id) => id.trim()).filter(Boolean),),+ LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS: z+ .string()+ .default("")+ .transform((value) =>+ value+ .split(",")+ .map((id) => id.trim())+ .filter(Boolean),+ ),LANGFUSE_TOPICS_REDIS_TTL_SECONDS: z.coerce.number().int()diff --git a/packages/shared/src/server/index.ts b/packages/shared/src/server/index.tsindex b17fad518c..d1bf952d56 100644--- a/packages/shared/src/server/index.ts+++ b/packages/shared/src/server/index.ts@@ -58,6 +58,7 @@ export * from "./llm/utils";export * from "./llm/types";export * from "./llm/promptToolConfig";export * from "./llm/internalTraceEvents";+export { isInternalEvalEnvironment } from "./llm/isInternalEvalEnvironment";export * from "./llm/langfuseAiCompletion";export * from "./utils/baseUrl";export * from "./utils/productUrl";diff --git a/packages/shared/src/server/llm/isInternalEvalEnvironment.ts b/packages/shared/src/server/llm/isInternalEvalEnvironment.tsnew file mode 100644index 0000000000..06cfeb8515--- /dev/null+++ b/packages/shared/src/server/llm/isInternalEvalEnvironment.ts@@ -0,0 +1,14 @@+export const INTERNAL_EVAL_ENVIRONMENT_PREFIX = "langfuse";++// Public ingestion strips the reserved `langfuse-` prefix from environments+// originating outside Langfuse, including OpenRouter Broadcast callbacks.+export const PUBLIC_LLM_JUDGE_ENVIRONMENT = "llm-as-a-judge";++export function isInternalEvalEnvironment(+ environment: string | null | undefined,+): boolean {+ return (+ environment?.startsWith(INTERNAL_EVAL_ENVIRONMENT_PREFIX) === true ||+ environment === PUBLIC_LLM_JUDGE_ENVIRONMENT+ );+}diff --git a/packages/shared/src/server/llm/types.ts b/packages/shared/src/server/llm/types.tsindex f74ac0080b..ccd645f325 100644--- a/packages/shared/src/server/llm/types.ts+++ b/packages/shared/src/server/llm/types.ts@@ -558,6 +558,7 @@ export enum LangfuseInternalTraceEnvironment {CodeEval = "langfuse-code-eval",NaturalLanguageFilter = "langfuse-natural-language-filter",InAppAgent = "langfuse-in-app-agent",+ Topics = "langfuse-topics",}export type ProcessedTraceEvent = {diff --git a/packages/shared/src/server/otel/internalTraceOtelWriter.ts b/packages/shared/src/server/otel/internalTraceOtelWriter.tsindex 7c5509b90b..c83ea68bd3 100644--- a/packages/shared/src/server/otel/internalTraceOtelWriter.ts+++ b/packages/shared/src/server/otel/internalTraceOtelWriter.ts@@ -260,7 +260,7 @@ export async function publishInternalOtelSpans(params: {/*** Exactly the `InternalTraceEventInput` fields this writer maps onto OTel- * spans. Narrowed so a caller with richer inputs (usage, prompt, experiment+ * spans. Narrowed so a caller with richer inputs (cost, prompt, experiment* linkage, ...) gets a compile-time surface mismatch instead of silent drops.*/export type InternalOtelSpanInput = Pick<@@ -280,6 +280,12 @@ export type InternalOtelSpanInput = Pick<| "output"| "metadata"| "evaluationContext"+ | "type"+ | "modelName"+ | "modelParameters"+ | "providedUsageDetails"+ | "userId"+ | "sessionId">;/**diff --git a/packages/shared/src/server/topics/clickhouse.ts b/packages/shared/src/server/topics/clickhouse.tsindex 283768f5a1..8ce66060ca 100644--- a/packages/shared/src/server/topics/clickhouse.ts+++ b/packages/shared/src/server/topics/clickhouse.ts@@ -17,6 +17,11 @@ import { prisma } from "../../db";import { chunk } from "lodash";import { Readable } from "node:stream";import { finished } from "node:stream/promises";+import {+ INTERNAL_EVAL_ENVIRONMENT_PREFIX,+ isInternalEvalEnvironment,+ PUBLIC_LLM_JUDGE_ENVIRONMENT,+} from "../llm/isInternalEvalEnvironment";const sourceKeySql ="project_id, facet_id, facet_version, trace_id, if(trace_id = '', session_id, '')";@@ -352,7 +357,9 @@ export async function listTopicSummaries();}rows.sort((a, b) => Number(b.processedAtMs) - Number(a.processedAtMs));- return rows.map(summaryResult);+ return rows+ .filter((row) => !isInternalEvalEnvironment(row.environment))+ .map(summaryResult);}/** Without a version filter, use each source's newest processed facet version. */export async function getLatestFacetSummaries(@@ -377,7 +384,9 @@ export async function getLatestFacetSummaries(},tags: { route: "topics-latest-summaries", projectId },});- return rows.map(summaryResult);+ return rows+ .filter((row) => !isInternalEvalEnvironment(row.environment))+ .map(summaryResult);}export async function getTopicClusteringSummaries(@@ -398,6 +407,8 @@ export async function getTopicClusteringSummaries(LIMIT 1 BY ${sourceKeySql}) WHERE processing_state = 'complete' AND embedding_model = {embeddingModel:String}AND length(embedding) = {embeddingDimensions:UInt32}+ AND NOT startsWith(environment, {internalEnvironmentPrefix:String})+ AND environment != {publicLlmJudgeEnvironment:String}ORDER BY trace_id`,params: {projectId,@@ -405,6 +416,8 @@ export async function getTopicClusteringSummaries(facetVersion,...embeddingConfig,...timeRangeParams(timeRange),+ internalEnvironmentPrefix: INTERNAL_EVAL_ENVIRONMENT_PREFIX,+ publicLlmJudgeEnvironment: PUBLIC_LLM_JUDGE_ENVIRONMENT,},tags: { route: "topics-clustering-summaries", projectId },});@@ -438,7 +451,7 @@ export async function getTopicSummaryCounts(count: string;}>({query: `SELECT facet_id AS facetId, facet_version AS facetVersion, count() AS count FROM (- SELECT facet_id, facet_version, processing_state, embedding_model, length(embedding) AS dimensions+ SELECT facet_id, facet_version, processing_state, embedding_model, length(embedding) AS dimensions, environmentFROM topic_facet_summariesWHERE project_id = {projectId:String} AND ${unitTimeFilterSql} AND ${scopeSql}AND ${latestSummaryFilterSql(scopeSql)}@@ -446,6 +459,8 @@ export async function getTopicSummaryCounts(LIMIT 1 BY ${sourceKeySql}) WHERE processing_state = 'complete' AND embedding_model = {embeddingModel:String}AND dimensions = {embeddingDimensions:UInt32}+ AND NOT startsWith(environment, {internalEnvironmentPrefix:String})+ AND environment != {publicLlmJudgeEnvironment:String}GROUP BY facet_id, facet_version`,params: {projectId,@@ -453,6 +468,8 @@ export async function getTopicSummaryCounts(facetIds: facets.map(({ facetId }) => facetId),facetVersions: facets.map(({ version }) => version),...embeddingConfig,+ internalEnvironmentPrefix: INTERNAL_EVAL_ENVIRONMENT_PREFIX,+ publicLlmJudgeEnvironment: PUBLIC_LLM_JUDGE_ENVIRONMENT,},tags: { route: "topics-summary-counts", projectId },});diff --git a/packages/shared/src/server/topics/config.test.ts b/packages/shared/src/server/topics/config.test.tsindex e9fcfb39ac..f2345fe8af 100644--- a/packages/shared/src/server/topics/config.test.ts+++ b/packages/shared/src/server/topics/config.test.ts@@ -4,6 +4,7 @@ beforeEach(() => {vi.stubEnv("DOCKER_BUILD", undefined);vi.stubEnv("LANGFUSE_TOPICS_ENABLED", undefined);vi.stubEnv("LANGFUSE_TOPICS_ENABLED_PROJECT_IDS", undefined);+ vi.stubEnv("LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS", undefined);});afterEach(() => {@@ -11,6 +12,35 @@ afterEach(() => {vi.resetModules();});+describe("Topics tracing allowlist", () => {+ it.each([undefined, "", " , "])(+ "disables tracing when its allowlist has no IDs: %s",+ async (value) => {+ vi.stubEnv("LANGFUSE_TOPICS_ENABLED_PROJECT_IDS", "project-a");+ vi.stubEnv("LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS", value);+ const { env } = await import("../../env.js");+ expect(env.LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS).toEqual([]);+ },+ );++ it("parses tracing IDs independently from the processing allowlist", async () => {+ vi.stubEnv("LANGFUSE_TOPICS_ENABLED_PROJECT_IDS", "project-a,project-b");+ vi.stubEnv(+ "LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS",+ " project-b, ,project-c ",+ );+ const { env } = await import("../../env.js");+ expect(env.LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS).toEqual([+ "project-b",+ "project-c",+ ]);+ expect(env.LANGFUSE_TOPICS_ENABLED_PROJECT_IDS).toEqual([+ "project-a",+ "project-b",+ ]);+ });+});+describe("Topics project allowlist", () => {it("stays disabled when Docker builds skip environment defaults", async () => {vi.stubEnv("DOCKER_BUILD", "1");diff --git a/packages/shared/src/server/topics/text.test.ts b/packages/shared/src/server/topics/text.test.tsindex c35d7b7241..14e7fe3a25 100644--- a/packages/shared/src/server/topics/text.test.ts+++ b/packages/shared/src/server/topics/text.test.ts@@ -3,11 +3,74 @@ import { z } from "zod";import { encrypt } from "../../encryption";import { env } from "../../env";import { BEDROCK_USE_DEFAULT_CREDENTIALS } from "../../interfaces/customLLMProviderConfigSchemas";-import { LLMAdapter } from "../llm/types";+import { LLMAdapter, type TraceSinkParams } from "../llm/types";+import type { ResourceSpan } from "../otel/OtelIngestionProcessor";import { generateTopicEmbeddings } from "./embeddings";import type { TopicsModel } from "./model-config";import { generateTopicText } from "./text";+const { publishToOtelIngestionQueue, processorConstructor } = vi.hoisted(+ () => ({+ publishToOtelIngestionQueue:+ vi.fn<(spans: ResourceSpan[]) => Promise<void>>(),+ processorConstructor: vi.fn(),+ }),+);++vi.mock("../otel/OtelIngestionProcessor", () => ({+ OtelIngestionProcessor: class {+ constructor(config: unknown) {+ processorConstructor(config);+ }+ publishToOtelIngestionQueue = publishToOtelIngestionQueue;+ },+}));++const trace: TraceSinkParams = {+ targetProjectId: "ai-features-project",+ traceId: "0af7651916cd43dd8448eb211c80319c",+ traceName: "topics-summary",+ environment: "langfuse-topics",+ metadata: { project_id: "source-project" },+};++function tracedGeneration() {+ expect(processorConstructor).toHaveBeenCalledWith(+ expect.objectContaining({+ projectId: trace.targetProjectId,+ isLangfuseInternal: true,+ ingestionVersion: "4",+ }),+ );+ expect(publishToOtelIngestionQueue).toHaveBeenCalledTimes(1);+ const spans = publishToOtelIngestionQueue.mock.calls[0][0].flatMap(+ (resource) =>+ resource.scopeSpans?.flatMap((scope) => scope.spans ?? []) ?? [],+ );+ expect(spans).toHaveLength(1);+ const span = spans[0];+ expect(span.traceId).toBe(trace.traceId);+ expect(span.parentSpanId ?? "").toBe("");+ expect(span.name).toBe(trace.traceName);+ const attributes = Object.fromEntries(+ (span.attributes ?? []).map(({ key, value }) => [+ key,+ value.stringValue ?? value,+ ]),+ );+ expect(attributes).toMatchObject({+ "langfuse.observation.type": "generation",+ "langfuse.environment": "langfuse-topics",+ "langfuse.trace.metadata": JSON.stringify(trace.metadata),+ });+ return attributes;+}++beforeEach(() => {+ vi.clearAllMocks();+ publishToOtelIngestionQueue.mockResolvedValue(undefined);+});+afterEach(() => {vi.unstubAllGlobals();vi.unstubAllEnvs();@@ -29,51 +92,85 @@ function bedrockModel(slot: TopicsModel["slot"], model: string): TopicsModel {}describe("generateTopicText", () => {- it("sends structured output through the project's Bedrock connection", async () => {- const output = { summary: "Account access" };- const fetch = vi.fn<typeof globalThis.fetch>(async () =>- Response.json({- output: {- message: {- role: "assistant",- content: [- { toolUse: { toolUseId: "json-1", name: "json", input: output } },- ],+ it.each(["us.openai.gpt-6-luna", "openai.gpt-oss-120b"])(+ "sends structured output through the project's Bedrock connection: %s",+ async (modelId) => {+ const output = { summary: "Account access" };+ const fetch = vi.fn<typeof globalThis.fetch>(async () =>+ Response.json({+ output: {+ message: {+ role: "assistant",+ content: [+ {+ toolUse: { toolUseId: "json-1", name: "json", input: output },+ },+ ],+ },},- },- stopReason: "tool_use",- usage: { inputTokens: 12, outputTokens: 4, totalTokens: 16 },- metrics: { latencyMs: 1 },- }),- );- vi.stubGlobal("fetch", fetch);- // Connection credentials must win over ambient worker credentials.- vi.stubEnv("AWS_BEARER_TOKEN_BEDROCK", "ambient-bearer-token");-- const result = await generateTopicText({- model: bedrockModel("summary", "us.openai.gpt-6-luna"),- system: [{ text: "Summarize the user's request." }],- input: "I cannot sign in.",- schema: z.object({ summary: z.string() }),- maxOutputTokens: 256,- });+ stopReason: "tool_use",+ usage: { inputTokens: 12, outputTokens: 4, totalTokens: 16 },+ metrics: { latencyMs: 1 },+ }),+ );+ vi.stubGlobal("fetch", fetch);+ // Connection credentials must win over ambient worker credentials.+ vi.stubEnv("AWS_BEARER_TOKEN_BEDROCK", "ambient-bearer-token");++ const result = await generateTopicText({+ model: bedrockModel("summary", modelId),+ system: [+ {+ text: "Summarize the user's request.",+ cache: modelId.includes("oss"),+ },+ ],+ input: "I cannot sign in.",+ schema: z.object({ summary: z.string() }),+ maxOutputTokens: 256,+ trace,+ });- expect(fetch).toHaveBeenCalledTimes(1);- const request = new Request(...fetch.mock.calls[0]);- expect(request.url).toBe(- "https://bedrock-runtime.eu-central-1.amazonaws.com/model/us.openai.gpt-6-luna/converse",- );- expect(request.headers.get("authorization")).toMatch(- /^AWS4-HMAC-SHA256 Credential=AKIATOPICS\//,- );- expect(await request.json()).toMatchObject({- system: [{ text: "Summarize the user's request." }],- inferenceConfig: { maxTokens: 256 },- additionalModelRequestFields: { reasoning: { effort: "none" } },- toolConfig: { toolChoice: { any: {} } },- });- expect(result.output).toEqual(output);- });+ expect(fetch).toHaveBeenCalledTimes(1);+ const request = new Request(...fetch.mock.calls[0]);+ expect(request.url).toBe(+ `https://bedrock-runtime.eu-central-1.amazonaws.com/model/${modelId}/converse`,+ );+ expect(request.headers.get("authorization")).toMatch(+ /^AWS4-HMAC-SHA256 Credential=AKIATOPICS\//,+ );+ const body = await request.json();+ expect(body).toMatchObject({+ system: [{ text: "Summarize the user's request." }],+ inferenceConfig: { maxTokens: 256 },+ toolConfig: { toolChoice: { any: {} } },+ });+ expect(result.output).toEqual(output);+ if (modelId.includes("oss")) {+ expect(body).not.toHaveProperty("additionalModelRequestFields");+ } else {+ expect(body).toHaveProperty(+ "additionalModelRequestFields.reasoning.effort",+ "none",+ );+ }+ expect(tracedGeneration()).toMatchObject({+ "langfuse.observation.input": JSON.stringify([+ { role: "system", content: "Summarize the user's request." },+ { role: "user", content: "I cannot sign in." },+ ]),+ "langfuse.observation.output": JSON.stringify(output),+ "langfuse.observation.model.name": modelId,+ "langfuse.observation.usage_details": JSON.stringify({+ input: 12,+ output: 4,+ total: 16,+ input_cached_tokens: 0,+ input_cache_creation: 0,+ }),+ });+ },+ );it("sends cached prompts through InvokeModel with the connection's credentials and region", async () => {const fetch = vi.fn<typeof globalThis.fetch>(async () =>@@ -103,6 +200,7 @@ describe("generateTopicText", () => {input: "I cannot sign in.",schema: z.object({ summary: z.string() }),maxOutputTokens: 256,+ trace,});const request = new Request(...fetch.mock.calls[0]);@@ -132,6 +230,18 @@ describe("generateTopicText", () => {{ role: "user", content: [{ type: "text", text: "I cannot sign in." }] },]);expect(body.prompt_cache_options).toEqual({ mode: "explicit", ttl: "30m" });+ expect(tracedGeneration()).toMatchObject({+ "langfuse.observation.output": JSON.stringify({+ summary: "Account access",+ }),+ "langfuse.observation.usage_details": JSON.stringify({+ input: 813,+ output: 275,+ total: 3928,+ input_cached_tokens: 2840,+ input_cache_creation: 0,+ }),+ });expect(result.usage).toEqual({inputTokens: 3653,outputTokens: 275,@@ -153,11 +263,17 @@ describe("generateTopicText", () => {input: "Hi",schema: z.object({ summary: z.string() }),maxOutputTokens: 256,+ trace,}),).rejects.toMatchObject({ status: 429 });+ expect(tracedGeneration()).toMatchObject({+ "langfuse.observation.level": "ERROR",+ "langfuse.observation.status_message": expect.stringContaining("429"),+ });});- it("reports cached prompt tokens from providers that cache automatically", async () => {+ it("reports automatic cache usage and keeps inference successful when tracing fails", async () => {+ publishToOtelIngestionQueue.mockRejectedValueOnce(new Error("queue down"));vi.stubGlobal("fetch",vi.fn(async () =>@@ -198,13 +314,149 @@ describe("generateTopicText", () => {input: "I cannot sign in.",schema: z.object({ summary: z.string() }),maxOutputTokens: 256,+ trace,});+ expect(tracedGeneration()).toMatchObject({+ "langfuse.observation.output": JSON.stringify({+ summary: "Account access",+ }),+ "langfuse.observation.model.name": "gpt-6-luna",+ "langfuse.observation.usage_details": JSON.stringify({+ input: 440,+ output: 200,+ total: 3200,+ input_cached_tokens: 2560,+ }),+ });expect(result.usage).toMatchObject({inputTokens: 3000,cacheReadTokens: 2560,});});+ it("retains SDK usage when no output is generated", async () => {+ vi.stubGlobal(+ "fetch",+ vi.fn(async () =>+ Response.json({+ id: "chatcmpl-empty",+ object: "chat.completion",+ created: 1,+ model: "gpt-6-luna",+ choices: [+ {+ index: 0,+ finish_reason: "length",+ message: { role: "assistant", content: null },+ },+ ],+ usage: { prompt_tokens: 12, completion_tokens: 4, total_tokens: 16 },+ }),+ ),+ );++ await expect(+ generateTopicText({+ model: {+ slot: "summary",+ provider: "openai-topics",+ adapter: LLMAdapter.OpenAI,+ model: "gpt-6-luna",+ connection: { secretKey: encrypt("sk-test") },+ },+ system: [{ text: "Summarize." }],+ input: "Hi",+ schema: z.object({ summary: z.string() }),+ maxOutputTokens: 256,+ trace,+ }),+ ).rejects.toMatchObject({ name: "AI_NoOutputGeneratedError" });+ expect(tracedGeneration()).toMatchObject({+ "langfuse.observation.level": "ERROR",+ "langfuse.observation.usage_details": JSON.stringify({+ input: 12,+ output: 4,+ total: 16,+ input_cached_tokens: 0,+ }),+ });+ });++ it.each([false, true])(+ "traces invalid structured output with its paid usage (cached: %s)",+ async (cache) => {+ const output = { summary: 42 };+ vi.stubGlobal(+ "fetch",+ vi.fn(async () =>+ Response.json(+ cache+ ? {+ choices: [+ {+ finish_reason: "stop",+ message: { content: JSON.stringify(output) },+ },+ ],+ usage: {+ prompt_tokens: 12,+ completion_tokens: 4,+ total_tokens: 16,+ },+ }+ : {+ output: {+ message: {+ role: "assistant",+ content: [+ {+ toolUse: {+ toolUseId: "json-1",+ name: "json",+ input: output,+ },+ },+ ],+ },+ },+ stopReason: "tool_use",+ usage: { inputTokens: 12, outputTokens: 4, totalTokens: 16 },+ metrics: { latencyMs: 1 },+ },+ ),+ ),+ );+ const response = generateTopicText({+ model: bedrockModel("summary", "us.openai.gpt-6-luna"),+ system: [{ text: "Summarize.", cache }],+ input: "Hi",+ schema: z.object({ summary: z.string() }),+ maxOutputTokens: 256,+ trace,+ });+ if (cache) {+ await expect(response).resolves.toMatchObject({+ output,+ usage: { inputTokens: 12, outputTokens: 4, totalTokens: 16 },+ });+ } else {+ await expect(response).rejects.toThrow();+ }+ const attributes = tracedGeneration();+ expect(attributes).toMatchObject({+ "langfuse.observation.level": "ERROR",+ "langfuse.observation.usage_details": JSON.stringify({+ input: 12,+ output: 4,+ total: 16,+ ...(cache ? {} : { input_cached_tokens: 0, input_cache_creation: 0 }),+ }),+ });+ expect(+ JSON.parse(attributes["langfuse.observation.output"] as string),+ ).toEqual(cache ? output : JSON.stringify(output));+ },+ );});describe("generateTopicEmbeddings", () => {@@ -240,6 +492,7 @@ describe("generateTopicEmbeddings", () => {output_dimension: 512,});expect(result.embeddings).toEqual([vector, other]);+ expect(publishToOtelIngestionQueue).not.toHaveBeenCalled();});it("asks OpenAI for any configured size", async () => {@@ -418,6 +671,7 @@ describe("allowlisted Cloud Bedrock runtime credentials", () => {generateTopicText({ ...input, model: runtimeModel("summary", "other") }),).rejects.toThrow("Invalid Bedrock credentials");expect(fetch).toHaveBeenCalledTimes(1);+ expect(publishToOtelIngestionQueue).not.toHaveBeenCalled();});it("uses the existing project allowlist for embeddings", async () => {diff --git a/packages/shared/src/server/topics/text.ts b/packages/shared/src/server/topics/text.tsindex 81294b07a8..7f31ad2d4b 100644--- a/packages/shared/src/server/topics/text.ts+++ b/packages/shared/src/server/topics/text.ts@@ -1,6 +1,6 @@import { Sha256 } from "@aws-crypto/sha256-js";import { SignatureV4 } from "@smithy/signature-v4";-import { Output } from "ai";+import { NoObjectGeneratedError, Output, type LanguageModelUsage } from "ai";import { z } from "zod";import { decrypt } from "../../encryption";import { BedrockConfigSchema } from "../../interfaces/customLLMProviderConfigSchemas";@@ -10,62 +10,160 @@ import {resolveBedrockProviderAuth,} from "../llm/ai-sdk/providers/bedrock";import { generateLLMText } from "../llm/llmText";-import { LLMAdapter } from "../llm/types";+import { LLMAdapter, type TraceSinkParams } from "../llm/types";+import { logger } from "../logger";+import { writeInternalTraceViaOtelIngestion } from "../otel/internalTraceOtelWriter";import type { TopicsModel } from "./model-config";-/** A system prompt segment; `cache` ends a reusable prefix with a cache breakpoint. */-export type TopicPromptPart = { text: string; cache?: boolean };--type TopicTextResult = {- output: unknown;- usage: {- inputTokens?: number;- outputTokens?: number;- totalTokens?: number;- cacheReadTokens?: number;- cacheWriteTokens?: number;- };-};-const TIMEOUT_MS = 60_000;-const isBedrockOpenAI = (model: TopicsModel) =>- model.adapter === LLMAdapter.Bedrock && /(^|\.)openai\./.test(model.model);-/*** Structured Topics text through the project's LLM connection. GPT-5.6+ models* on Bedrock reuse a shared prompt prefix only with explicit cache breakpoints,* which Converse does not accept for them, so cached prompts on those models* use InvokeModel at the same regional endpoint and model ID.*/-export async function generateTopicText<T>(params: {- model: TopicsModel;- system: TopicPromptPart[];- input: string;- schema: z.ZodType<T>;- maxOutputTokens: number;-}): Promise<TopicTextResult> {- if (isBedrockOpenAI(params.model) && params.system.some((p) => p.cache))- return invokeWithCache(params);+export async function generateTopicText<T>(+ params: TopicTextParams<T>,+): Promise<TopicTextResult> {+ const startTimeISO = new Date().toISOString();+ let result: TopicTextResult | undefined;+ let failure: unknown;+ const recordResponse = (response: TopicTextResult) => {+ result = response;+ };+ try {+ result =+ isBedrockGPT(params.model) && params.system.some((part) => part.cache)+ ? await invokeWithCache(params, recordResponse)+ : await generateWithConnection(params, recordResponse);+ return result;+ } catch (error) {+ failure = error;+ if (NoObjectGeneratedError.isInstance(error)) {+ result = { output: error.text, usage: topicUsage(error.usage) };+ }+ throw error;+ } finally {+ await writeTopicTrace(params, startTimeISO, result, failure);+ }+}++async function writeTopicTrace<T>(+ params: TopicTextParams<T>,+ startTimeISO: string,+ result: TopicTextResult | undefined,+ failure: unknown,+) {+ const { trace } = params;+ if (!trace) return;+ try {+ /** Validation marks the trace; the caller validates after accounting for usage. */+ const validation = params.schema.safeParse(result?.output);+ await writeInternalTraceViaOtelIngestion({+ rootSpanId: trace.traceId,+ eventInputs: [+ {+ projectId: trace.targetProjectId,+ traceId: trace.traceId,+ spanId: trace.traceId,+ name: trace.traceName,+ traceName: trace.traceName,+ type: "GENERATION",+ environment: trace.environment,+ startTimeISO,+ endTimeISO: new Date().toISOString(),+ modelName: params.model.model,+ modelParameters: { maxOutputTokens: params.maxOutputTokens },+ input: JSON.stringify(topicMessages(params)),+ output: JSON.stringify(result?.output),+ providedUsageDetails: topicTraceUsage(result?.usage),+ metadata: trace.metadata ?? {},+ userId: trace.userId,+ sessionId: trace.sessionId,+ ...topicTraceError(failure ?? validation.error),+ },+ ],+ });+ } catch (error) {+ logger.error("Failed to publish Topics model trace", {+ error,+ traceId: trace.traceId,+ });+ }+}++function topicTraceUsage(usage: TopicTextResult["usage"] | undefined) {+ if (!usage) return undefined;+ const details: Record<string, number | undefined> = {+ input: undefined,+ output: usage.outputTokens,+ total: usage.totalTokens,+ input_cached_tokens: usage.cacheReadTokens,+ input_cache_creation: usage.cacheWriteTokens,+ };+ /** Standard input excludes cached tokens, which have their own prices. */+ if (usage.inputTokens !== undefined) {+ details.input = Math.max(+ 0,+ usage.inputTokens -+ (usage.cacheReadTokens ?? 0) -+ (usage.cacheWriteTokens ?? 0),+ );+ }+ return Object.fromEntries(+ Object.entries(details).filter(+ (entry): entry is [string, number] => entry[1] !== undefined,+ ),+ );+}++function topicTraceError(failure: unknown) {+ if (failure === undefined) return {};+ return {+ level: "ERROR" as const,+ statusMessage: failure instanceof Error ? failure.message : String(failure),+ };+}+function topicMessages(params: { system: TopicPromptPart[]; input: string }) {+ return [+ {+ role: "system" as const,+ content: params.system.map((part) => part.text).join("\n\n"),+ },+ { role: "user" as const, content: params.input },+ ];+}++function topicUsage(+ usage: LanguageModelUsage | undefined,+): TopicTextResult["usage"] {+ return {+ inputTokens: usage?.inputTokens,+ outputTokens: usage?.outputTokens,+ totalTokens: usage?.totalTokens,+ cacheReadTokens: usage?.inputTokenDetails?.cacheReadTokens,+ cacheWriteTokens: usage?.inputTokenDetails?.cacheWriteTokens,+ };+}++async function generateWithConnection<T>(+ params: TopicTextParams<T>,+ recordResponse: (result: TopicTextResult) => void,+): Promise<TopicTextResult> {const result = await generateLLMText({model: { adapter: params.model.adapter, id: params.model.model },connection: params.model.connection,// The static system prompt comes first so providers with automatic prefix// caching can reuse it across traces.- messages: [- {- role: "system",- content: params.system.map((part) => part.text).join("\n\n"),- },- { role: "user", content: params.input },- ],+ messages: topicMessages(params),output: Output.object({ schema: params.schema }),maxOutputTokens: params.maxOutputTokens,reasoning: "none",// Bedrock ignores portable reasoning for OpenAI models; their default effort// is not none.- ...(isBedrockOpenAI(params.model)+ ...(isBedrockGPT(params.model)? {providerOptions: {bedrock: {@@ -78,18 +176,28 @@ export async function generateTopicText<T>(params: {maxRetries: 2,timeout: TIMEOUT_MS,});+ const usage = topicUsage(result.usage);+ // The SDK's output getter can throw after a response with billable usage.+ recordResponse({ output: result.text, usage });return {output: result.output,- usage: {- inputTokens: result.usage.inputTokens,- outputTokens: result.usage.outputTokens,- totalTokens: result.usage.totalTokens,- cacheReadTokens: result.usage.inputTokenDetails?.cacheReadTokens,- cacheWriteTokens: result.usage.inputTokenDetails?.cacheWriteTokens,- },+ usage,};}+function isBedrockGPT(model: TopicsModel): boolean {+ if (model.adapter !== LLMAdapter.Bedrock) return false;+ /** Opaque application inference profiles cannot identify the underlying model. */+ const version = /(?:^|[/.])openai\.gpt-(\d+)(?:\.(\d+))?(?:[-:]|$)/.exec(+ model.model,+ );+ return Boolean(+ version &&+ (Number(version[1]) > 5 ||+ (Number(version[1]) === 5 && Number(version[2] ?? 0) >= 6)),+ );+}+function signingCredentials(auth: ReturnType<typeof resolveBedrockProviderAuth>,) {@@ -153,13 +261,10 @@ async function bedrockRequestHeaders(return signed.headers;}-async function invokeWithCache(params: {- model: TopicsModel;- system: TopicPromptPart[];- input: string;- schema: z.ZodType;- maxOutputTokens: number;-}): Promise<TopicTextResult> {+async function invokeWithCache(+ params: TopicTextParams<unknown>,+ recordResponse: (result: TopicTextResult) => void,+): Promise<TopicTextResult> {const { region } = BedrockConfigSchema.parse(params.model.connection.config);assertValidBedrockRegion(region);const { $schema: _, ...schema } = z.toJSONSchema(params.schema);@@ -219,14 +324,8 @@ async function invokeWithCache(params: {};};const message = json.choices?.[0]?.message;- if (!message?.content || json.choices?.[0]?.finish_reason === "length")- throw new Error(- message?.refusal- ? `The model refused: ${message.refusal}`- : "The model returned no complete output.",- );- return {- output: JSON.parse(message.content) as unknown,+ const result: TopicTextResult = {+ output: message?.content ?? message?.refusal,usage: {inputTokens: json.usage?.prompt_tokens,outputTokens: json.usage?.completion_tokens,@@ -235,4 +334,37 @@ async function invokeWithCache(params: {cacheWriteTokens: json.usage?.prompt_tokens_details?.cache_write_tokens,},};+ // Retain paid usage and the original response if parsing or validation fails.+ recordResponse(result);++ if (!message?.content || json.choices?.[0]?.finish_reason === "length")+ throw new Error(+ message?.refusal+ ? `The model refused: ${message.refusal}`+ : "The model returned no complete output.",+ );+ return { ...result, output: JSON.parse(message.content) as unknown };}++/** A system prompt segment; `cache` ends a reusable prefix with a cache breakpoint. */+export type TopicPromptPart = { text: string; cache?: boolean };++type TopicTextResult = {+ output: unknown;+ usage: {+ inputTokens?: number;+ outputTokens?: number;+ totalTokens?: number;+ cacheReadTokens?: number;+ cacheWriteTokens?: number;+ };+};++type TopicTextParams<T> = {+ model: TopicsModel;+ system: TopicPromptPart[];+ input: string;+ schema: z.ZodType<T>;+ maxOutputTokens: number;+ trace?: TraceSinkParams;+};diff --git a/packages/shared/src/server/topics/trace-input.ts b/packages/shared/src/server/topics/trace-input.tsindex a507975621..9d3475870d 100644--- a/packages/shared/src/server/topics/trace-input.ts+++ b/packages/shared/src/server/topics/trace-input.ts@@ -102,6 +102,9 @@ export async function loadTopicTranscript(params: {).orderByColumns([{ column: "e.event_ts", direction: "DESC" },+ /** Use the same environment tie-breakers as Topics root discovery. */+ { column: "e.span_id", direction: "DESC" },+ { column: "e.environment", direction: "DESC" },// Equal storage timestamps need a stable choice; they do not establish causal order.{column:diff --git a/packages/shared/src/server/topics/trace-selection.ts b/packages/shared/src/server/topics/trace-selection.tsindex 4f3cf274d1..61660f9c4e 100644--- a/packages/shared/src/server/topics/trace-selection.ts+++ b/packages/shared/src/server/topics/trace-selection.ts@@ -6,6 +6,10 @@ import { applyCommentFilters } from "../services/commentFilterService";import { CTEQueryBuilder } from "../queries/clickhouse-sql/event-query-builder";import { buildEventsObservationRowSelection } from "../queries/clickhouse-sql/events-observation-row-selection";import { queryClickhouse } from "../repositories/clickhouse";+import {+ INTERNAL_EVAL_ENVIRONMENT_PREFIX,+ PUBLIC_LLM_JUDGE_ENVIRONMENT,+} from "../llm/isInternalEvalEnvironment";type SelectionRows = {ids: { id: string };@@ -24,7 +28,7 @@ type SelectionRows = {};};-/** Matching observations select trace identities; processing reads each full trace. */+/** Select trace identities from matching root observations. */export async function selectTopicTraceRows<Mode extends keyof SelectionRows>(input: z.infer<typeof topicTraceSelectionCriteriaSchema> & {projectId: string;@@ -68,14 +72,10 @@ export async function selectTopicTraceRows<Mode extends keyof SelectionRows>(.selectRaw("e.trace_id AS id","e.start_time AS start_time",- ...(preview- ? [- "e.span_id AS span_id",- "e.event_ts AS event_ts",- `${eventsTableTraceNameSelectSql} AS trace_name`,- "e.environment AS environment",- ]- : []),+ "e.span_id AS span_id",+ "e.event_ts AS event_ts",+ "e.environment AS environment",+ ...(preview ? [`${eventsTableTraceNameSelectSql} AS trace_name`] : []),).whereRaw("e.trace_id != ''").buildWithParams();@@ -85,9 +85,10 @@ export async function selectTopicTraceRows<Mode extends keyof SelectionRows>(schema: ["id","start_time",- ...(preview- ? (["span_id", "event_ts", "trace_name", "environment"] as const)- : []),+ "span_id",+ "event_ts",+ "environment",+ ...(preview ? (["trace_name"] as const) : []),] as const,}).from("matching_observations", "m")@@ -95,14 +96,21 @@ export async function selectTopicTraceRows<Mode extends keyof SelectionRows>("m.id AS id","min(m.start_time) AS timestamp","max(m.start_time) AS latest_match",+ "argMax(m.environment, tuple(m.event_ts, m.span_id, m.environment)) AS environment",...(preview? ["argMaxIf(m.trace_name, tuple(m.event_ts, m.span_id, m.trace_name), m.trace_name != '') AS name",- "argMax(m.environment, tuple(m.event_ts, m.span_id, m.environment)) AS environment",]: []),).groupBy("m.id")+ .havingRaw(+ "NOT startsWith(environment, {internalEnvironmentPrefix:String}) AND environment != {publicLlmJudgeEnvironment:String}",+ {+ internalEnvironmentPrefix: INTERNAL_EVAL_ENVIRONMENT_PREFIX,+ publicLlmJudgeEnvironment: PUBLIC_LLM_JUDGE_ENVIRONMENT,+ },+ ).buildWithParams();const selection = new CTEQueryBuilder().withCTE("matching_traces", {@@ -111,7 +119,8 @@ export async function selectTopicTraceRows<Mode extends keyof SelectionRows>("id","timestamp","latest_match",- ...(preview ? (["name", "environment"] as const) : []),+ "environment",+ ...(preview ? (["name"] as const) : []),] as const,}).from("matching_traces", "t")diff --git a/worker/AGENTS.md b/worker/AGENTS.mdindex 09901f5311..9e4a647b3b 100644--- a/worker/AGENTS.md+++ b/worker/AGENTS.md@@ -33,6 +33,11 @@Model calls use `generateTopicText` and `generateTopicEmbeddings` from`@langfuse/shared/topics/server` with the project's resolved `TopicsModels`(summary, embedding, clustering LLM connections from `topic_configs`).+ Text tracing requires the source project in+ `LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS` (empty by default) and checks+ organization `aiFeaturesEnabled` and `aiTelemetryEnabled` in `models.ts`;+ embeddings are untraced. Use `isInternalEvalEnvironment`+ to exclude internal sources.Pass models explicitly; never put them on execution or batch state, which ispersisted. Trace-batch jobs load enabled settings for all allowlisted batchprojects in one query (`getEnabledTopicsModels`) and pause a project ondiff --git a/worker/src/features/evaluation/isEvalTargetEnvironmentAllowed.ts b/worker/src/features/evaluation/isEvalTargetEnvironmentAllowed.tsindex 42ddd939fa..700e7bab7d 100644--- a/worker/src/features/evaluation/isEvalTargetEnvironmentAllowed.ts+++ b/worker/src/features/evaluation/isEvalTargetEnvironmentAllowed.ts@@ -1,17 +1,9 @@-import { LangfuseInternalTraceEnvironment } from "@langfuse/shared/src/server";+import {+ LangfuseInternalTraceEnvironment,+ isInternalEvalEnvironment,+} from "@langfuse/shared/src/server";-// Public ingestion strips the reserved `langfuse-` prefix from environments-// originating outside Langfuse, including OpenRouter Broadcast callbacks.-const PUBLIC_LLM_JUDGE_ENVIRONMENT = "llm-as-a-judge";--export function isInternalEvalEnvironment(- environment: string | null | undefined,-): boolean {- return (- environment?.startsWith("langfuse") === true ||- environment === PUBLIC_LLM_JUDGE_ENVIRONMENT- );-}+export { isInternalEvalEnvironment } from "@langfuse/shared/src/server";/*** Final, fail-closed loop safeguard at eval EXECUTION time.diff --git a/worker/src/features/topics/README.md b/worker/src/features/topics/README.mdindex 6e9a378e3a..25408bc21f 100644--- a/worker/src/features/topics/README.md+++ b/worker/src/features/topics/README.md@@ -65,10 +65,13 @@ Processing is unavailable until all three are set. Any connection whose save-timtest call succeeds can serve summaries and clustering, including Anthropic.Embeddings need OpenAI (including OpenAI-compatible base URLs), Azure OpenAI,Amazon Bedrock, Google AI Studio or Vertex AI; Anthropic has no embeddings API,-so its embedding test call fails. Summary and clustering calls go through `generateLLMText`;-embeddings go through `embedLLMText`. Both retry 429 and 5xx responses twice-with backoff. On Cloud, Bedrock and Vertex connections need explicit-credentials; the default credential chain is self-hosted only.+so its embedding test call fails. Summary and clustering calls normally go through+`generateLLMText`; embeddings go through `embedLLMText`. Both retry 429 and 5xx+responses twice with backoff. Cached Bedrock GPT-5.6+ summaries use a single+InvokeModel request with explicit cache breakpoints. On Cloud, Bedrock and Vertex+connections need explicit credentials, except operator-provisioned Bedrock+connections in the runtime-credential allowlist; self-hosted deployments can use+the default credential chain.Saving the settings makes one real call per configured slot (structured outputfor summaries and clustering, one embedding whose length must equal the@@ -82,6 +85,28 @@ fails and must be started again. Deleting a connection that a slot uses turnsautomatic processing off and records why. Usage is retained for any model, butcost estimates are only available for the model IDs priced in[models.ts](models.ts).++Summary and naming calls emit one generation each to the configured internal+AI-features project (`LANGFUSE_AI_FEATURES_PROJECT_ID`) only when the source+project is listed in `LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS` and its+organization enables AI features and AI analytics. The tracing allowlist is+comma-separated, trims whitespace, and defaults to empty (tracing off). It is+independent of the processing allowlist; projects can run Topics without tracing.+Multi-facet summaries share one generation. Both connection-based SDK calls and+cached Bedrock InvokeModel calls record inputs, outputs, models, usage, and errors;+embeddings are untraced.+Trace costs use the destination project's model catalog. Unmatched connection+model IDs retain their usage but have no trace cost, independently of Topics'+own estimates in `models.ts`.+Traces use `langfuse-topics`. Topics excludes all `langfuse`-prefixed environments+and `llm-as-a-judge` through the shared eval exclusion predicate, preventing loops+from these internal traces. Manual selection checks matching root observations+within the requested time range; workers also check the source environment they+load. If an OpenRouter connection broadcasts requests back+to a Topics-enabled project, use a dedicated Topics API key excluded from that+Broadcast destination. OpenRouter's separate public traces do not retain the+reserved internal environment and can otherwise trigger more Topics calls.+Numerical fitting uses the worker's existing Node runtime and compiled`@langfuse/native` addon, with no extra runtime or service.Summary and assignment records use `trace_id` as their source when present;diff --git a/worker/src/features/topics/backfillTopicTraces.test.ts b/worker/src/features/topics/backfillTopicTraces.test.tsindex 942cdfac6b..d52781326c 100644--- a/worker/src/features/topics/backfillTopicTraces.test.ts+++ b/worker/src/features/topics/backfillTopicTraces.test.ts@@ -15,7 +15,10 @@ import {createEventsCh,} from "@langfuse/shared/src/server";import { prisma } from "@langfuse/shared/src/db";-import { selectTopicTraceRows } from "@langfuse/shared/topics/server";+import {+ loadTopicTranscript,+ selectTopicTraceRows,+} from "@langfuse/shared/topics/server";const enabled = vi.hoisted(() => vi.fn(() => true));@@ -74,13 +77,21 @@ describe("Topics trace selection and backfill", () => {[projectId, "trace-old", "2026-09-23T08:00:00Z"],[projectId, "trace-outside", "2026-09-24T00:00:00Z"],[randomUUID(), "trace-foreign", "2026-09-23T10:00:00Z"],- ].map(([project, trace, timestamp]) =>+ [+ projectId,+ "trace-internal",+ "2026-09-23T11:00:00Z",+ "langfuse-topics",+ ],+ [projectId, "trace-judge", "2026-09-23T12:00:00Z", "llm-as-a-judge"],+ [projectId, "trace-reserved", "2026-09-23T13:00:00Z", "langfuse"],+ ].map(([project, trace, timestamp, environment]) =>createEvent({project_id: project,trace_id: trace,start_time: new Date(timestamp),trace_name: "agent-turn",- environment: "test",+ environment: environment ?? "test",tags: ["billing"],}),),@@ -130,6 +141,84 @@ describe("Topics trace selection and backfill", () => {expect(options.jobId).toMatch(/^[a-f0-9]{64}$/);});+ it("filters latest matching root environments before counting and limiting", async () => {+ const mixedProject = randomUUID();+ const source = {+ project_id: mixedProject,+ trace_name: "agent-turn",+ input: JSON.stringify({ role: "user", content: "Send an invoice" }),+ output: null,+ };+ await createEventsCh(+ [+ ["public-root", "production", "langfuse-topics"],+ ["equal-timestamp", "production", "langfuse-topics"],+ ["internal-root", "langfuse-topics", "production"],+ ]+ .flatMap(([traceId, rootEnvironment, latestEnvironment], index) => [+ createEvent({+ ...source,+ trace_id: traceId,+ span_id: "a-root",+ start_time: new Date(Date.UTC(2026, 8, 23, 9 + index)),+ event_ts: new Date("2026-09-24T12:00:00Z"),+ environment: rootEnvironment,+ }),+ createEvent({+ ...source,+ trace_id: traceId,+ span_id: "z-child",+ parent_span_id: "a-root",+ start_time: new Date(Date.UTC(2026, 8, 24, 9 + index)),+ event_ts: new Date(+ traceId === "equal-timestamp"+ ? "2026-09-24T12:00:00Z"+ : "2026-09-24T13:00:00Z",+ ),+ environment: latestEnvironment,+ }),+ ])+ .concat(+ ["production", "langfuse-topics"].map((environment, index) =>+ createEvent({+ ...source,+ trace_id: "updated-root",+ span_id: "a-root",+ start_time: new Date("2026-09-23T12:00:00Z"),+ event_ts: new Date(Date.UTC(2026, 8, 24, 12 + index)),+ environment,+ }),+ ),+ ),+ );+ const input = { ...selection, projectId: mixedProject, limit: 1 };+ const [preview, ids, backfill] = await Promise.all([+ selectTopicTraceRows(input, prisma, "preview"),+ selectTopicTraceRows(input, prisma, "ids"),+ selectTopicTraceRows(input, prisma, "backfill"),+ ]);+ expect(preview).toEqual([+ expect.objectContaining({+ id: "equal-timestamp",+ environment: "production",+ }),+ ]);+ expect(Number(preview[0].matchedTraceCount)).toBe(2);+ expect(ids).toEqual([{ id: "equal-timestamp" }]);+ expect(backfill.map(({ id }) => id)).toEqual(["equal-timestamp"]);+ expect(Number(backfill[0].matchedTraceCount)).toBe(2);+ for (const [traceId, environment] of [+ ["public-root", "langfuse-topics"],+ ["internal-root", "production"],+ ["equal-timestamp", "langfuse-topics"],+ ["updated-root", "langfuse-topics"],+ ]) {+ expect(+ await loadTopicTranscript({ projectId: mixedProject, traceId }),+ ).toMatchObject({ environment });+ }+ });+it("keeps preview, ID-only triggers and backfill sampling aligned", async () => {const input = {...selection,diff --git a/worker/src/features/topics/models.test.ts b/worker/src/features/topics/models.test.tsindex 48fb49b73b..d1eed6493f 100644--- a/worker/src/features/topics/models.test.ts+++ b/worker/src/features/topics/models.test.ts@@ -1,14 +1,24 @@-import { beforeEach, describe, expect, it, vi } from "vitest";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import { encrypt } from "@langfuse/shared/encryption";const state = vi.hoisted(() => ({+ env: { LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS: ["project"] },call: vi.fn(),embed: vi.fn(),increment: vi.fn(),+ project: vi.fn(),+ traceSink: vi.fn(),}));+vi.mock("@langfuse/shared/src/env", () => ({ env: state.env }));vi.mock("@langfuse/shared/src/server", () => ({+ getLangfuseAITraceSinkParams: state.traceSink,+ LangfuseInternalTraceEnvironment: { Topics: "langfuse-topics" },logger: { warn: vi.fn() },recordIncrement: state.increment,}));+vi.mock("@langfuse/shared/src/db", () => ({+ prisma: { project: { findFirst: state.project } },+}));vi.mock("@langfuse/shared/topics/server", () => ({generateTopicText: (...args: unknown[]) => state.call(...args),generateTopicEmbeddings: (...args: unknown[]) => state.embed(...args),@@ -64,12 +74,79 @@ const evidence = {};beforeEach(() => {+ state.env.LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS = ["project"];state.call.mockReset();state.embed.mockReset();state.increment.mockReset();+ state.project.mockReset().mockResolvedValue({+ organization: { aiFeaturesEnabled: true, aiTelemetryEnabled: true },+ });+ state.traceSink.mockReset().mockImplementation((params) => ({+ targetProjectId: "ai-features",+ traceId: "a".repeat(32),+ traceName: params.traceName,+ environment: "production",+ aiFeatureOtelIngestion: true,+ metadata: {+ langfuse_ai_feature: params.feature,+ langfuse_project_id: params.projectId,+ ...params.metadata,+ },+ }));+});++afterEach(() => {+ vi.unstubAllGlobals();});describe("Topics naming boundary", () => {+ it("keeps invalid cached Bedrock output per-trace and records paid usage", async () => {+ const { generateTopicText } = await vi.importActual<+ typeof import("@langfuse/shared/topics/server")+ >("@langfuse/shared/topics/server");+ state.call.mockImplementation(generateTopicText);+ state.env.LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS = [];+ vi.stubGlobal(+ "fetch",+ vi.fn(async () =>+ Response.json({+ choices: [+ {+ finish_reason: "stop",+ message: {+ content: JSON.stringify({ summary: 42, status: "applicable" }),+ },+ },+ ],+ usage: { prompt_tokens: 12, completion_tokens: 4, total_tokens: 16 },+ }),+ ),+ );+ const models = topicsModels();+ models.summary.connection.secretKey = encrypt(+ JSON.stringify({ accessKeyId: "AKIATOPICS", secretAccessKey: "test" }),+ );++ await expect(+ summarizeTopicTrace(+ models,+ facet,+ "Trace evidence.",+ topicProcessingConfigSchema.parse({+ summaryModel: models.summary.model,+ }),+ ),+ ).rejects.toMatchObject({ name: "ZodError" });+ expect(state.increment).toHaveBeenCalledWith("langfuse.topics.tokens", 12, {+ stage: "summary",+ direction: "input",+ });+ expect(state.increment).toHaveBeenCalledWith("langfuse.topics.tokens", 4, {+ stage: "summary",+ direction: "output",+ });+ });+it.each([["us.openai.gpt-5.6-luna", 0.00002, 0.000036],["us.openai.gpt-6-luna", 0.000011, 0.0000165],@@ -108,6 +185,17 @@ describe("Topics naming boundary", () => {expect(result.costDetails.total).toBeCloseTo(inputCost + outputCost, 10);const request = state.call.mock.calls[0][0];expect(request.model).toMatchObject({ slot: "summary", model });+ expect(request.trace).toMatchObject({+ targetProjectId: "ai-features",+ traceName: "topics-summary",+ environment: "langfuse-topics",+ aiFeatureOtelIngestion: false,+ metadata: {+ langfuse_project_id: "project",+ facet_id: "facet",+ facet_version: 1,+ },+ });expect(request.system).toEqual([{ text: expect.stringContaining(facet.prompt), cache: true },]);@@ -256,6 +344,25 @@ describe("Topics naming boundary", () => {expect(request.system[0].text).not.toContain("issues_2");expect(request.input).toContain("RAW_TRANSCRIPT_SENTINEL");expect(request.maxOutputTokens).toBe(config.maxOutputTokens * 2);+ expect(request.trace).toMatchObject({+ traceName: "topics-summary",+ environment: "langfuse-topics",+ metadata: {+ langfuse_project_id: "project",+ facets: [+ { key: "intent_1", facet_id: "facet", facet_version: 1 },+ { key: "issues_2", facet_id: "issues", facet_version: 1 },+ ],+ },+ });+ expect(state.project).toHaveBeenCalledWith({+ where: { id: "project", deletedAt: null },+ select: {+ organization: {+ select: { aiFeaturesEnabled: true, aiTelemetryEnabled: true },+ },+ },+ });// Every facet key is required in the structured output.expect(request.schema.safeParse({@@ -304,6 +411,11 @@ describe("Topics naming boundary", () => {evidenceSummaryIds: [members[399].id],});expect(state.call).toHaveBeenCalledOnce();+ expect(state.call.mock.calls[0][0].trace).toMatchObject({+ traceName: "topics-naming",+ environment: "langfuse-topics",+ metadata: { langfuse_project_id: "project" },+ });expect(state.call.mock.calls[0][0].model).toMatchObject({slot: "clustering",model: "us.openai.gpt-5.6-terra",@@ -341,6 +453,92 @@ describe("Topics naming boundary", () => {},);+ it.each([+ { aiFeaturesEnabled: true, aiTelemetryEnabled: false },+ { aiFeaturesEnabled: false, aiTelemetryEnabled: true },+ null,+ new Error("database unavailable"),+ ])(+ "keeps inference untraced without organization consent: %j",+ async (organization) => {+ if (organization instanceof Error)+ state.project.mockRejectedValue(organization);+ else+ state.project.mockResolvedValue(organization ? { organization } : null);+ state.call+ .mockResolvedValueOnce({+ output: { summary: "A billing request.", status: "applicable" },+ usage: {},+ })+ .mockResolvedValueOnce({+ output: {+ name: "Billing",+ description: "Billing requests.",+ evidenceSummaryIds: ["member-a"],+ },+ usage: {},+ });+ await summarizeTopicTrace(+ topicsModels(),+ facet,+ "A billing request.",+ topicProcessingConfigSchema.parse({+ summaryModel: "us.openai.gpt-5.6-luna",+ }),+ );+ await nameTopicGroup(topicsModels(), evidence);+ state.call.mockResolvedValueOnce({+ output: {+ intent: {+ notes: "Invoice request.",+ summary: "Request invoice.",+ status: "applicable",+ },+ },+ usage: {},+ });+ await summarizeTopicTraceFacets(+ topicsModels(),+ [{ key: "intent", facet, builtIn: true }],+ "Request invoice.",+ topicProcessingConfigSchema.parse({+ summaryModel: "us.openai.gpt-5.6-luna",+ }),+ );+ expect(state.call).toHaveBeenCalledTimes(3);+ for (const [request] of state.call.mock.calls)+ expect(request.trace).toBeUndefined();+ },+ );++ it.each(["unconfigured destination", "empty allowlist", "unlisted source"])(+ "keeps inference untraced and skips the consent lookup with %s",+ async (reason) => {+ if (reason === "unconfigured destination")+ state.traceSink.mockReturnValue(undefined);+ else+ state.env.LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS =+ reason === "empty allowlist" ? [] : ["ai-features", "project-other"];+ state.call.mockResolvedValue({+ output: { summary: "A billing request.", status: "applicable" },+ usage: {},+ });+ await summarizeTopicTrace(+ topicsModels(),+ facet,+ "A billing request.",+ topicProcessingConfigSchema.parse({+ summaryModel: "us.openai.gpt-5.6-luna",+ }),+ );+ expect(state.project).not.toHaveBeenCalled();+ expect(state.call).toHaveBeenCalledOnce();+ expect(state.call.mock.calls[0][0].trace).toBeUndefined();+ if (reason !== "unconfigured destination")+ expect(state.traceSink).not.toHaveBeenCalled();+ },+ );+it("records embedding input usage even when the returned vector is invalid", async () => {state.embed.mockResolvedValue({ embeddings: [[]], tokens: 12 });await expect(diff --git a/worker/src/features/topics/models.ts b/worker/src/features/topics/models.tsindex 7d520ea354..21c6e925cf 100644--- a/worker/src/features/topics/models.ts+++ b/worker/src/features/topics/models.ts@@ -1,6 +1,12 @@import { get_encoding } from "tiktoken";import { z } from "zod";-import { logger } from "@langfuse/shared/src/server";+import { prisma } from "@langfuse/shared/src/db";+import { env } from "@langfuse/shared/src/env";+import {+ logger,+ getLangfuseAITraceSinkParams,+ LangfuseInternalTraceEnvironment,+} from "@langfuse/shared/src/server";import {generateTopicEmbeddings,generateTopicText,@@ -89,6 +95,8 @@ async function structuredCall<T>(outputLimit: number,stage: "summary" | "naming",model: TopicsModel,+ projectId: string,+ metadata?: Record<string, unknown>,): Promise<ModelResult<T>> {// Unknown model IDs record usage without cost.const rates = (tokens: number): Rates | undefined =>@@ -101,6 +109,7 @@ async function structuredCall<T>(input,schema,maxOutputTokens: outputLimit,+ trace: await topicTraceSink(stage, projectId, metadata),}).catch((error: unknown) => {logger.warn("Topics model request failed", {model: model.model,@@ -162,6 +171,45 @@ async function structuredCall<T>(return accepted;}+async function topicTraceSink(+ stage: "summary" | "naming",+ projectId: string,+ metadata?: Record<string, unknown>,+) {+ if (!env.LANGFUSE_TOPICS_TRACING_ENABLED_PROJECT_IDS?.includes(projectId))+ return undefined;+ const trace = getLangfuseAITraceSinkParams({+ feature: "topics",+ projectId,+ traceName: `topics-${stage}`,+ metadata,+ });+ if (!trace) return undefined;+ const project = await prisma.project+ .findFirst({+ where: { id: projectId, deletedAt: null },+ select: {+ organization: {+ select: { aiFeaturesEnabled: true, aiTelemetryEnabled: true },+ },+ },+ })+ .catch((error: unknown) => {+ logger.warn("Failed to check Topics tracing consent", { error });+ return null;+ });+ if (+ !project?.organization.aiFeaturesEnabled ||+ !project.organization.aiTelemetryEnabled+ )+ return undefined;+ return {+ ...trace,+ environment: LangfuseInternalTraceEnvironment.Topics,+ aiFeatureOtelIngestion: false,+ };+}+const SUMMARY_SYSTEM_PROMPT = `You describe one facet of a recorded run of an LLM application. Your description is embedded and clustered together with descriptions of many other runs. Runs that share a pattern should get similar descriptions; runs that differ in a way that matters should not.<transcript_format>@@ -202,6 +250,7 @@ export async function summarizeTopicTrace(facet: TopicFacetVersion,text: string,config: TopicProcessingConfig,+ metadata?: Record<string, unknown>,) {requireFrozenSummaryModel(models, config);const system = [@@ -227,6 +276,8 @@ export async function summarizeTopicTrace(config.maxOutputTokens,"summary",models.summary,+ models.projectId,+ { ...metadata, facet_id: facet.facetId, facet_version: facet.version },);}@@ -242,6 +293,7 @@ export async function summarizeTopicTraceFacets(facets: { key: string; facet: TopicFacetVersion; builtIn: boolean }[],text: string,config: TopicProcessingConfig,+ metadata?: Record<string, unknown>,) {requireFrozenSummaryModel(models, config);const entry = z.object({@@ -294,6 +346,15 @@ ${facetBlock(true)}`,config.maxOutputTokens * facets.length,"summary",models.summary,+ models.projectId,+ {+ ...metadata,+ facets: facets.map(({ key, facet }) => ({+ key,+ facet_id: facet.facetId,+ facet_version: facet.version,+ })),+ },);}@@ -303,6 +364,7 @@ export async function nameTopicGroup(members: { id: string; summary: string }[];contrasts: { id: string; summary: string }[];},+ metadata?: Record<string, unknown>,) {if (!group.members.length)throw new Error("Naming requires one non-empty effective group.");@@ -345,6 +407,8 @@ export async function nameTopicGroup(1000,"naming",models.clustering,+ models.projectId,+ metadata,);}diff --git a/worker/src/features/topics/processTopicEmbeddingBatch.test.ts b/worker/src/features/topics/processTopicEmbeddingBatch.test.tsindex edb5a29855..ec7901e951 100644--- a/worker/src/features/topics/processTopicEmbeddingBatch.test.ts+++ b/worker/src/features/topics/processTopicEmbeddingBatch.test.ts@@ -35,11 +35,16 @@ vi.mock("@langfuse/shared/topics/server", () => ({TOPIC_EMBEDDING_EXPIRED_ERROR:"Topics staged results expired before processing completed. Start a new execution with stored-summary reuse to recover persisted results.",}));-vi.mock("@langfuse/shared/src/server", () => ({- logger: { error: vi.fn() },- recordIncrement: vi.fn(),- recordDistribution: vi.fn(),-}));+vi.mock("@langfuse/shared/src/server", async (importOriginal) => {+ const { isInternalEvalEnvironment } =+ await importOriginal<typeof import("@langfuse/shared/src/server")>();+ return {+ isInternalEvalEnvironment,+ logger: { error: vi.fn() },+ recordIncrement: vi.fn(),+ recordDistribution: vi.fn(),+ };+});vi.mock("./models", () => ({embedTopicSummaries: mocks.embed,}));@@ -122,6 +127,36 @@ beforeEach(() => {});describe("Topics embedding handoff", () => {+ it.each(["summarized", "complete"] as const)(+ "skips internal staged summaries in a mixed batch (%s)",+ async (state) => {+ const row: TopicSummary = {+ ...summary(),+ state,+ environment: "langfuse-topics",+ embedding: state === "complete" ? Array(256).fill(0.25) : [],+ };+ const allowed = summary("allowed");+ staged.set(topicSourceKey(row), row);+ staged.set(topicSourceKey(allowed), allowed);++ await processTopicEmbeddingBatch(batch(row, allowed));++ expect(mocks.embed).toHaveBeenCalledExactlyOnceWith(+ expect.anything(),+ [allowed.summary],+ 256,+ );+ expect(mocks.update).toHaveBeenCalledTimes(1);+ expect(mocks.write).toHaveBeenCalledExactlyOnceWith([+ expect.objectContaining({+ traceId: allowed.traceId,+ state: "complete",+ }),+ ]);+ },+ );+it("retries combined results from Redis after a failed ClickHouse write", async () => {const applicable = summary();const nonApplicable: TopicSummary = {diff --git a/worker/src/features/topics/processTopicEmbeddingBatch.ts b/worker/src/features/topics/processTopicEmbeddingBatch.tsindex 1e6d7bbfd1..b085d7f8fe 100644--- a/worker/src/features/topics/processTopicEmbeddingBatch.ts+++ b/worker/src/features/topics/processTopicEmbeddingBatch.ts@@ -1,4 +1,5 @@import { UnrecoverableError } from "bullmq";+import { isInternalEvalEnvironment } from "@langfuse/shared/src/server";import pLimit from "p-limit";import type { TopicSummary } from "@langfuse/shared/topics";import {@@ -20,7 +21,7 @@ import {TopicsProviderUnavailable,} from "./provider-error";-/** Acknowledging a batch means every result is durably stored in ClickHouse. */+/** Acknowledging a batch means every eligible result is durably stored in ClickHouse. */export async function processTopicEmbeddingBatch(batch: TopicEmbeddingBatch,): Promise<void> {@@ -42,6 +43,7 @@ export async function processTopicEmbeddingBatch(for (const ref of batch.summaries) {const staged = await readStagedTopicSummary(batch, ref);if (!staged) throw new UnrecoverableError(TOPIC_EMBEDDING_EXPIRED_ERROR);+ if (isInternalEvalEnvironment(staged.summary.environment)) continue;if (staged.embeddingConfig.embeddingModel !== models.embedding.model)throw new TopicsProviderUnavailable("The project's embedding model changed after this batch was created. Start a new Topics execution.",diff --git a/worker/src/features/topics/processTopicsExecution.test.ts b/worker/src/features/topics/processTopicsExecution.test.tsindex 785641ff97..97ea55f3fb 100644--- a/worker/src/features/topics/processTopicsExecution.test.ts+++ b/worker/src/features/topics/processTopicsExecution.test.ts@@ -51,10 +51,11 @@ const state = vi.hoisted(() => ({numeric: vi.fn(),}));vi.mock("@langfuse/shared/src/server", async (importOriginal) => {- const { assembleTranscript } =+ const { assembleTranscript, isInternalEvalEnvironment } =await importOriginal<typeof import("@langfuse/shared/src/server")>();return {assembleTranscript,+ isInternalEvalEnvironment,recordIncrement: vi.fn(),recordDistribution: vi.fn(),};@@ -576,6 +577,88 @@ beforeEach(() => {});describe("Topics execution", () => {+ it.each([false, true])(+ "rejects internal traces supplied by ID (cached: %s)",+ async (cached) => {+ const blocked = execution("blocked", 1);+ if (cached) {+ await processSelection("source", 1);+ const summary = [...state.summaries.values()][0];+ state.staged.set(`blocked/${topicSourceKey(summary)}`, {+ summary: { ...summary, environment: "langfuse-topics" },+ embeddingConfig: blocked.input.embeddingConfig,+ });+ state.summaries.clear();+ state.summarize.mockClear();+ state.embed.mockClear();+ } else {+ state.loadTranscript.mockResolvedValue({+ ...traceInput("trace0"),+ environment: "langfuse-topics",+ });+ }+ state.executions.set("blocked", blocked);+ await processTopicsExecution({+ projectId: "project",+ executionId: "blocked",+ });+ expect(state.summarize).not.toHaveBeenCalled();+ expect(state.embed).not.toHaveBeenCalled();+ expect(state.summaries.size).toBe(0);+ expect(state.executions.get("blocked")?.facets[0].counts.failed).toBe(1);+ },+ );++ it.each([false, true])(+ "resumes mixed internal staged summaries without repeating paid work (embedded: %s)",+ async (embedded) => {+ state.deferEmbeddings = true;+ await processSelection("resumed", 2);+ const saved = state.batches.get("resumed")!;+ expect(saved.summarized).toBe(true);+ const staged = [...state.staged.values()];+ staged[0].summary.environment = "langfuse-topics";+ if (embedded) {+ for (const row of staged) {+ row.summary.state = "complete";+ row.summary.embedding = Array(256).fill(0.25);+ }+ saved.assignedAt = "2026-01-01T00:01:00.000Z";+ }+ state.deferEmbeddings = false;+ state.summarize.mockClear();+ state.embed.mockClear();++ for (let attempt = 0; attempt < 2; attempt++) {+ await processTopicsExecution({+ projectId: "project",+ executionId: "resumed",+ });+ expect(state.executions.get("resumed")).toMatchObject({+ status: "completed_with_errors",+ error: null,+ facets: [{ counts: { complete: 1, failed: 1 } }],+ traceErrors: [+ {+ traceId: staged[0].summary.traceId,+ error: "Topics does not process internal Langfuse traces.",+ },+ ],+ });+ expect(state.batches.get("resumed")?.summaries).toEqual([+ expect.objectContaining({ traceId: staged[1].summary.traceId }),+ ]);+ expect(state.staged.size).toBe(0);+ }+ expect(state.summarize).not.toHaveBeenCalled();+ expect(state.embed).toHaveBeenCalledTimes(embedded ? 0 : 1);+ if (!embedded)+ expect([...state.summaries.values()]).toMatchObject([+ { traceId: staged[1].summary.traceId, state: "complete" },+ ]);+ },+ );+it("records null transcripts as insufficient input without inference", async () => {state.loadTranscript.mockResolvedValueOnce(traceInput("trace0", null));await processSelection("empty-transcript", 1);diff --git a/worker/src/features/topics/processTopicsExecution.ts b/worker/src/features/topics/processTopicsExecution.tsindex 1c0247d507..b9a1b57449 100644--- a/worker/src/features/topics/processTopicsExecution.ts+++ b/worker/src/features/topics/processTopicsExecution.ts@@ -1,4 +1,4 @@-import { logger } from "@langfuse/shared/src/server";+import { isInternalEvalEnvironment, logger } from "@langfuse/shared/src/server";import {isTopicsEnabled,readTopicExecutionSummary,@@ -113,12 +113,16 @@ async function summarizeTrace(): Promise<TopicSummary> {const accepted = staged.find((row) => row.traceId === traceId);if (accepted) {+ if (isInternalEvalEnvironment(accepted.environment))+ throw new Error("Topics does not process internal Langfuse traces.");metrics.result("summary", "cached");// An accepted payload may expire after the cache read; never renew its TTL.return accepted;}const { unitStartTime, sessionId, environment, traceName, transcript } =await getTranscript();+ if (isInternalEvalEnvironment(environment))+ throw new Error("Topics does not process internal Langfuse traces.");const source = {projectId: execution.projectId,facetId: facet.facetId,@@ -193,6 +197,11 @@ async function summarizeTrace(facet,JSON.stringify(transcript),execution.input.processingConfig,+ {+ source_trace_id: traceId,+ source_environment: environment,+ execution_id: execution.id,+ },);const error = topicSummaryOutputError(result.output);if (error) throw invalidOutput(metrics, "summary", error);@@ -276,8 +285,13 @@ async function releaseCompletedSummaries(state: TopicProcessBatchState) {)return;try {+ const refs = state.summaries.concat(+ state.failedTraceIds.flatMap(({ facetId, facetVersion, traceIds }) =>+ traceIds.map((traceId) => ({ facetId, facetVersion, traceId })),+ ),+ );await Promise.all(- state.summaries.map((ref) =>+ refs.map((ref) =>deleteStagedTopicSummary({projectId: state.execution.projectId,@@ -646,16 +660,26 @@ async function clusterFacet(const members = new Map(group.members.map((summary, index) => [`m${index + 1}`, summary]),);- const { output: label } = await nameTopicGroup(models, {- members: Array.from(members, ([id, { summary }]) => ({- id,- summary,- })),- contrasts: group.contrasts.map((member, index) => ({- id: `c${index + 1}`,- summary: member.summary,- })),- });+ const { output: label } = await nameTopicGroup(+ models,+ {+ members: Array.from(members, ([id, { summary }]) => ({+ id,+ summary,+ })),+ contrasts: group.contrasts.map((member, index) => ({+ id: `c${index + 1}`,+ summary: member.summary,+ })),+ },+ {+ facet_id: facet.facetId,+ facet_version: facet.version,+ execution_id: execution.id,+ run_id: run.id,+ group_id: group.id,+ },+ );if (!label.name.trim() ||!label.description.trim() ||@@ -888,28 +912,16 @@ async function processTraces(aborted = true;throw error;}- let failed = state.failedTraceIds.find(- (entry) =>- entry.facetId === facet.facetId &&- entry.facetVersion === facet.version,- );- if (!failed) {- failed = {+ recordTraceFailure(+ state,+ progress,+ {facetId: facet.facetId,facetVersion: facet.version,- traceIds: [],- };- state.failedTraceIds.push(failed);- }- failed.traceIds.push(traceId);- progress.counts.failed = failed.traceIds.length;- const message = errorMessage(error);- if (- !execution.traceErrors.some(- (item) => item.traceId === traceId && item.error === message,- )- )- execution.traceErrors.push({ traceId, error: message });+ traceId,+ },+ error,+ );await saveInOrder();}}@@ -928,7 +940,7 @@ async function processTraces(state.assignedAt = new Date().toISOString();await save("assigning");}- const summaries = await Promise.all(+ const stagedSummaries = await Promise.all(state.summaries.filter((ref) =>facets.some(@@ -943,13 +955,30 @@ async function processTraces();if (!staged)throw new TopicEmbeddingFailure(TOPIC_EMBEDDING_EXPIRED_ERROR);- if (staged.summary.state === "summarized")- throw new TopicEmbeddingFailure(- "Topics embedding batch has an unfinished result.",- );- return staged.summary;+ return { ref, summary: staged.summary };}),);+ const summaries: TopicSummary[] = [];+ for (const { ref, summary } of stagedSummaries) {+ if (isInternalEvalEnvironment(summary.environment)) {+ const { progress } = facets.find(+ ({ facet }) =>+ facet.facetId === ref.facetId && facet.version === ref.facetVersion,+ )!;+ const error = new Error(+ "Topics does not process internal Langfuse traces.",+ );+ metrics.error("summary", error);+ recordTraceFailure(state, progress, ref, error);+ state.summaries = state.summaries.filter((entry) => entry !== ref);+ continue;+ }+ if (summary.state === "summarized")+ throw new TopicEmbeddingFailure(+ "Topics embedding batch has an unfinished result.",+ );+ summaries.push(summary);+ }for (const { facet, progress, run } of facets) {try {const selected = summaries.filter(@@ -987,6 +1016,30 @@ async function processTraces(}}+function recordTraceFailure(+ state: TopicProcessBatchState,+ progress: TopicFacetProgress,+ { facetId, facetVersion, traceId }: TopicEmbeddingRef,+ error: unknown,+) {+ let failed = state.failedTraceIds.find(+ (entry) => entry.facetId === facetId && entry.facetVersion === facetVersion,+ );+ if (!failed) {+ failed = { facetId, facetVersion, traceIds: [] };+ state.failedTraceIds.push(failed);+ }+ if (!failed.traceIds.includes(traceId)) failed.traceIds.push(traceId);+ progress.counts.failed = failed.traceIds.length;+ const message = errorMessage(error);+ if (+ !state.execution.traceErrors.some(+ (item) => item.traceId === traceId && item.error === message,+ )+ )+ state.execution.traceErrors.push({ traceId, error: message });+}+/** Each unpublished attempt fits the current compatible summaries from scratch. */async function updateTopics(metrics: TopicMetrics,diff --git a/worker/src/features/topics/storage.integration.test.ts b/worker/src/features/topics/storage.integration.test.tsindex 27853b45f0..994eef2cf3 100644--- a/worker/src/features/topics/storage.integration.test.ts+++ b/worker/src/features/topics/storage.integration.test.ts@@ -595,6 +595,62 @@ describe("Topics definition persistence", () => {expect(await getLatestFacetSummaries(projectId, "facet", 1, newRange),).toEqual([moved]);++ const internalFacet = "internal-environments";+ const internalTraces = [+ "langfuse-topics",+ "llm-as-a-judge",+ "langfuse",+ ].map((environment) => ({ traceId: randomUUID(), environment }));+ await writeTopicSummaries([+ ...internalTraces.flatMap(({ traceId, environment }) => {+ return [+ { ...summary, facetId: internalFacet, traceId },+ {+ ...latest,+ facetId: internalFacet,+ traceId,+ environment,+ },+ ];+ }),+ { ...summary, facetId: internalFacet, traceId: "allowed-trace" },+ ]);+ const [eligible, counts, listed, latestDisplayed] = await Promise.all([+ getTopicClusteringSummaries(+ projectId,+ internalFacet,+ 1,+ embedding,+ oldRange,+ ),+ getTopicSummaryCounts(+ projectId,+ [{ facetId: internalFacet, version: 1 }],+ embedding,+ oldRange,+ ),+ listTopicSummaries(+ projectId,+ {+ facetId: internalFacet,+ facetVersion: 1,+ traceIds: ["allowed-trace"].concat(+ internalTraces.map(({ traceId }) => traceId),+ ),+ },+ oldRange,+ ),+ getLatestFacetSummaries(projectId, internalFacet, undefined, oldRange),+ ]);+ expect(eligible.map(({ traceId }) => traceId)).toEqual(["allowed-trace"]);+ expect(listed.map(({ traceId }) => traceId)).toEqual(["allowed-trace"]);+ expect(latestDisplayed.map(({ traceId }) => traceId)).toEqual([+ "allowed-trace",+ ]);+ expect(counts).toEqual([+ { facetId: internalFacet, facetVersion: 1, count: 1 },+ ]);});it("round-trips Float64 geometry and normalized source references through ClickHouse", async () => {diff --git a/worker/src/features/topics/summarizeAssembledTrace.test.ts b/worker/src/features/topics/summarizeAssembledTrace.test.tsindex 00cc8d47c0..cf4c28ae12 100644--- a/worker/src/features/topics/summarizeAssembledTrace.test.ts+++ b/worker/src/features/topics/summarizeAssembledTrace.test.ts@@ -21,7 +21,10 @@ vi.mock("@langfuse/shared/topics/server", () => ({writeTopicSummaries: (...args: unknown[]) => state.write(...args),TOPICS_TRANSCRIPT_VERSION: "shared-transcript-v3",}));-vi.mock("@langfuse/shared/src/server", () => ({+vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({+ isInternalEvalEnvironment: (+ await importOriginal<typeof import("@langfuse/shared/src/server")>()+ ).isInternalEvalEnvironment,recordDistribution: vi.fn(),recordIncrement: vi.fn(),instrumentAsync: async (@@ -133,6 +136,25 @@ beforeEach(() => {});describe("summarizeAssembledTrace", () => {+ it("skips internal traces before reading summaries, calling models or writing", async () => {+ const result = await summarizeAssembledTrace({+ projectId: "project-a",+ traceId: "trace-1",+ traceTimestamp: "2026-09-22T12:00:00.000Z",+ jobEnqueuedAt: new Date("2026-09-22T12:01:00.000Z"),+ environment: "langfuse-topics",+ traceName: "topics-summary",+ transcript,+ models,+ facets: [facet],+ });+ expect(result).toBe("disabled");+ expect(state.stored).not.toHaveBeenCalled();+ expect(state.summarize).not.toHaveBeenCalled();+ expect(state.embed).not.toHaveBeenCalled();+ expect(state.write).not.toHaveBeenCalled();+ });+it("skips a finished job retry but summarizes a later redispatch", async () => {await summarizeAssembledTrace({projectId: "project-a",@@ -153,6 +175,10 @@ describe("summarizeAssembledTrace", () => {]);expect(submitted[2]).toContain("Export monthly sales");expect(submitted[2]).not.toContain("observation");+ expect(submitted[4]).toEqual({+ source_trace_id: "trace-1",+ source_environment: "default",+ });const written = state.write.mock.calls[0][0][0];expect(written).toMatchObject({projectId: "project-a",diff --git a/worker/src/features/topics/summarizeAssembledTrace.ts b/worker/src/features/topics/summarizeAssembledTrace.tsindex 035485f898..3928f8d184 100644--- a/worker/src/features/topics/summarizeAssembledTrace.ts+++ b/worker/src/features/topics/summarizeAssembledTrace.ts@@ -12,6 +12,7 @@ import {} from "@langfuse/shared/topics";import {instrumentAsync,+ isInternalEvalEnvironment,recordIncrement,type Transcript,} from "@langfuse/shared/src/server";@@ -55,7 +56,8 @@ type AssembledTraceInput = {*/export async function summarizeAssembledTrace(input: AssembledTraceInput,-): Promise<"unchanged" | "summarized"> {+): Promise<"disabled" | "unchanged" | "summarized"> {+ if (isInternalEvalEnvironment(input.environment)) return "disabled";return instrumentAsync({ name: "topics-trace-summary", traceScope: "topics" },async (span) => {@@ -148,6 +150,10 @@ async function summarizeEnabledTrace(keyed,prepared.text,config,+ {+ source_trace_id: input.traceId,+ source_environment: input.environment,+ },);const outputs = generated.output as Record<string, FacetOutput>;for (const { key } of keyed) {diff --git a/worker/src/queues/__tests__/traceBatchQueue.test.ts b/worker/src/queues/__tests__/traceBatchQueue.test.tsindex 0134d2a231..ec8605a8a9 100644--- a/worker/src/queues/__tests__/traceBatchQueue.test.ts+++ b/worker/src/queues/__tests__/traceBatchQueue.test.ts@@ -920,17 +920,17 @@ describe("trace batch queue", () => {);});- it("assembles each tenant's trace at the boundary and EOF", async () => {+ it("assembles each tenant's trace with its latest environment at the boundary and EOF", async () => {const userMessage = { role: "user", content: "first question" };const answer = { role: "assistant", content: "first answer" };const row = {project_id: "a",trace_id: "shared-trace",- environment: "production",+ environment: "langfuse-topics",span_id: "first",parent_span_id: null,start_time: "2026-09-11 00:00:00.000000",- event_ts: "2026-09-11 00:00:00.000000",+ event_ts: "2026-09-11 00:00:00.000002",type: "GENERATION",name: "generation",input: JSON.stringify([userMessage]),@@ -947,6 +947,8 @@ describe("trace batch queue", () => {yield {...row,span_id: "second",+ environment: "production",+ event_ts: "2026-09-11 00:00:00.000001",start_time: "2026-09-11 00:01:00.000000",input: JSON.stringify([userMessage,@@ -955,18 +957,19 @@ describe("trace batch queue", () => {]),output: JSON.stringify({ role: "assistant", content: "second answer" }),};+ // The newer internal event has an earlier start time and smaller span ID.yield row;expect(tokenCountAsync).not.toHaveBeenCalled();yield {...row,project_id: "b",- environment: "staging",+ environment: "langfuse-topics",type: "SPAN",};yield {...row,project_id: "b",- environment: "staging",+ environment: "production",type: "SPAN",span_id: "second",};@@ -994,8 +997,8 @@ describe("trace batch queue", () => {environment: input.environment,})),).toEqual([- { projectId: "a", environment: "production" },- { projectId: "b", environment: "staging" },+ { projectId: "a", environment: "langfuse-topics" },+ { projectId: "b", environment: "production" },]);const estimates = vi.mocked(tokenCountAsync).mock.calls;expect(estimates).toHaveLength(5);diff --git a/worker/src/queues/traceBatchQueue.ts b/worker/src/queues/traceBatchQueue.tsindex b94cdee21b..13ae83e8db 100644--- a/worker/src/queues/traceBatchQueue.ts+++ b/worker/src/queues/traceBatchQueue.ts@@ -36,6 +36,8 @@ import {import { summarizeAssembledTrace } from "../features/topics/summarizeAssembledTrace";import { recordTraceBatchTranscript } from "../features/traceBatching/traceBatchTranscript";+type TraceBatchObservation = Observation & { eventTimestamp: string };+type TraceOutcome =| { outcome: "disabled" | "unchanged" | "summarized" }| { outcome: "failed"; reason: string };@@ -97,7 +99,7 @@ async function loadProjectRule(projectId: string) {}async function summarizeTraceBatch(- observations: Observation[],+ observations: TraceBatchObservation[],transcript: Transcript | null,topics: TopicsBatchProjects,jobEnqueuedAt: Date,@@ -117,6 +119,13 @@ async function summarizeTraceBatch(const traceTimestamp = new Date(Math.min(...observations.map((row) => row.startTime.getTime())),).toISOString();+ // Match manual processing and discovery, preserving event_ts microseconds.+ const latest = observations.reduce((current, row) => {+ if (row.eventTimestamp !== current.eventTimestamp)+ return row.eventTimestamp > current.eventTimestamp ? row : current;+ if (row.id !== current.id) return row.id > current.id ? row : current;+ return row.environment > current.environment ? row : current;+ });try {const project = await topics.get(first.projectId);if (@@ -133,7 +142,7 @@ async function summarizeTraceBatch(traceId,traceTimestamp,jobEnqueuedAt,- environment: first.environment,+ environment: latest.environment,traceName: first.name ?? "",transcript,models: project.models,@@ -306,7 +315,10 @@ export const traceBatchQueueProcessor: Processor<let pendingSummaryBytes = 0;let summaryFailed = false;let summaryError: unknown;- const enqueueSummary = (observations: Observation[], bytes: number) => {+ const enqueueSummary = (+ observations: TraceBatchObservation[],+ bytes: number,+ ) => {pendingSummaryBytes += bytes;const enqueuedAt = performance.now();const transcribed = transcribing.then(async () => {@@ -352,7 +364,7 @@ export const traceBatchQueueProcessor: Processor<activeReads++;try {recordTraceBatchActiveReads();- let traceObservations: Observation[] = [];+ let traceObservations: TraceBatchObservation[] = [];let traceBytes = 0;for await (const event of getTraceBatchEventStream(batch, queryOptions)) {observationCount++;@@ -394,8 +406,8 @@ export const traceBatchQueueProcessor: Processor<if (summaryFailed) break;}traceBytes += eventBytes;- traceObservations.push(- convertObservation({+ traceObservations.push({+ ...convertObservation({...event,id: event.span_id,parent_observation_id: event.parent_span_id,@@ -408,7 +420,8 @@ export const traceBatchQueueProcessor: Processor<usage_details: {},cost_details: {},}),- );+ eventTimestamp: event.event_ts,+ });}// Reaching EOF completes the last trace; a failed stream must not flush it.if (traceObservations.length && !summaryFailed)