🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(topics): process traces concurrently and batch embeddings (#18698)
Lysander Kieselecd847d518c7
diff --git a/packages/shared/src/server/llm/llmText.ts b/packages/shared/src/server/llm/llmText.tsindex 55153911fa..e4e179eef6 100644--- a/packages/shared/src/server/llm/llmText.ts+++ b/packages/shared/src/server/llm/llmText.ts@@ -2,7 +2,7 @@ import { type ZodType } from "zod";import {Output,- embed,+ embedMany,generateText,jsonSchema,streamText,@@ -11,7 +11,7 @@ import {type GenerateTextOnAbortCallback,type GenerateTextOnEndCallback,type GenerateTextResult,- type EmbedResult,+ type EmbedManyResult,type Experimental_DownloadFunction,type JSONValue,type LanguageModelCallOptions,@@ -377,15 +377,16 @@ function openLLMConnection(connection: LLMConnection) {return { apiKey, extraHeaders, createFetch };}-/** Embeds one value with a persisted connection; calls are not traced. */+/** Embeds values with a persisted connection, batched per provider limit; calls are not traced. */export async function embedLLMText(options: {model: LLMModelRef;connection: LLMConnection;- value: string;+ values: string[];providerOptions?: ProviderOptions;maxRetries?: number;+ maxParallelCalls?: number;abortSignal?: AbortSignal;-}): Promise<EmbedResult> {+}): Promise<EmbedManyResult> {resolveAiSdkModelConfig({model: options.model,connectionConfig: options.connection.config,@@ -405,11 +406,12 @@ export async function embedLLMText(options: {credentialSource: "user",createFetch,});- return embed({+ return embedMany({model: embeddingModel,- value: options.value,+ values: options.values,providerOptions: options.providerOptions,maxRetries: options.maxRetries,+ maxParallelCalls: options.maxParallelCalls,abortSignal: options.abortSignal,});}diff --git a/packages/shared/src/server/topics/embeddings.ts b/packages/shared/src/server/topics/embeddings.tsindex 50b06471d8..fdc4475611 100644--- a/packages/shared/src/server/topics/embeddings.ts+++ b/packages/shared/src/server/topics/embeddings.ts@@ -20,10 +20,10 @@ function bedrockEmbeddingOptions(return { dimensions };}-/** Embeds one summary through the project's LLM connection. */-export async function generateTopicEmbedding(params: {+/** Embeds summaries through the project's LLM connection in as few requests as the provider allows. */+export async function generateTopicEmbeddings(params: {model: TopicsModel;- summary: string;+ summaries: string[];dimensions: number;}) {const { embeddingDimensions } = topicEmbeddingConfigSchema.parse({@@ -32,7 +32,7 @@ export async function generateTopicEmbedding(params: {const result = await embedLLMText({model: { adapter: params.model.adapter, id: params.model.model },connection: params.model.connection,- value: params.summary,+ values: params.summaries,// Discovery and later centroid assignment must share one vector space, so// every provider is asked for clustering vectors of the configured size.providerOptions:@@ -55,7 +55,9 @@ export async function generateTopicEmbedding(params: {},},maxRetries: 2,+ // Single-text models (Bedrock Titan, Nova) split a batch into one request per text.+ maxParallelCalls: 10,abortSignal: AbortSignal.timeout(60_000),});- return { embedding: result.embedding, tokens: result.usage.tokens };+ return { embeddings: result.embeddings, tokens: result.usage.tokens };}diff --git a/packages/shared/src/server/topics/model-config.test.ts b/packages/shared/src/server/topics/model-config.test.tsindex 51db82ba95..28edc0a24f 100644--- a/packages/shared/src/server/topics/model-config.test.ts+++ b/packages/shared/src/server/topics/model-config.test.ts@@ -17,7 +17,7 @@ const mocks = vi.hoisted(() => ({}));vi.mock("./clickhouse", () => ({ hasTopicEmbeddings: mocks.hasEmbeddings }));vi.mock("../llm/testModelCall", () => ({ testModelCall: mocks.testCall }));-vi.mock("./embeddings", () => ({ generateTopicEmbedding: mocks.embed }));+vi.mock("./embeddings", () => ({ generateTopicEmbeddings: mocks.embed }));vi.mock("../../db", () => ({prisma: {llmApiKeys: { findMany: mocks.keys },@@ -65,8 +65,8 @@ beforeEach(() => {mocks.current.mockResolvedValue(null);mocks.hasEmbeddings.mockResolvedValue(false);mocks.testCall.mockResolvedValue(undefined);- mocks.embed.mockImplementation(async ({ dimensions }) => ({- embedding: Array(dimensions).fill(0.1),+ mocks.embed.mockImplementation(async ({ summaries, dimensions }) => ({+ embeddings: summaries.map(() => Array(dimensions).fill(0.1)),}));});@@ -100,7 +100,9 @@ describe("prepareTopicsModelSettings", () => {if (model === "gpt-5.6-terra")throw new Error("Model not found (HTTP 404)");});- mocks.embed.mockResolvedValue({ embedding: Array(1536).fill(0.1) });+ mocks.embed.mockResolvedValue({+ embeddings: [Array(1536).fill(0.1), Array(1536).fill(0.1)],+ });await expect(saveTopicsModelSettings("project", settings)).rejects.toThrow("Embeddings (text-embedding-3-small): The model returned 1536 dimensions instead of 1024. Choose a size this model supports.; Topic clustering (gpt-5.6-terra): Model not found (HTTP 404)",diff --git a/packages/shared/src/server/topics/model-config.ts b/packages/shared/src/server/topics/model-config.tsindex 5a9035220f..4313fada82 100644--- a/packages/shared/src/server/topics/model-config.ts+++ b/packages/shared/src/server/topics/model-config.ts@@ -19,7 +19,7 @@ import {type TopicsModelSlotName,} from "../../topics";import { hasTopicEmbeddings } from "./clickhouse";-import { generateTopicEmbedding } from "./embeddings";+import { generateTopicEmbeddings } from "./embeddings";import { invalidateTopicsProcessingActive } from "./processing-gate";import { z } from "zod";@@ -337,14 +337,22 @@ async function testTopicsModelSlots(errors[name] = "LLM connection not found in this project.";return;}- const { embedding } = await generateTopicEmbedding({+ // Two texts, because processing embeds a trace's facets in one call.+ const { embeddings } = await generateTopicEmbeddings({model: resolved,- summary: "Topics model check",+ summaries: ["Topics model check", "Topics batch check"],dimensions: settings.embeddingDimensions,});- if (embedding.length !== settings.embeddingDimensions)+ if (embeddings.length !== 2)throw new Error(- `The model returned ${embedding.length} dimensions instead of ${settings.embeddingDimensions}. Choose a size this model supports.`,+ "The model must embed several texts in one request. Choose an embedding model or endpoint that supports batched input.",+ );+ const wrong = embeddings.find(+ (vector) => vector.length !== settings.embeddingDimensions,+ );+ if (wrong)+ throw new Error(+ `The model returned ${wrong.length} dimensions instead of ${settings.embeddingDimensions}. Choose a size this model supports.`,);} catch (error) {errors[name] =diff --git a/packages/shared/src/server/topics/text.test.ts b/packages/shared/src/server/topics/text.test.tsindex 5c6db7e120..f6ed7ce1aa 100644--- a/packages/shared/src/server/topics/text.test.ts+++ b/packages/shared/src/server/topics/text.test.ts@@ -4,7 +4,7 @@ import { encrypt } from "../../encryption";import { env } from "../../env";import { BEDROCK_USE_DEFAULT_CREDENTIALS } from "../../interfaces/customLLMProviderConfigSchemas";import { LLMAdapter } from "../llm/types";-import { generateTopicEmbedding } from "./embeddings";+import { generateTopicEmbeddings } from "./embeddings";import type { TopicsModel } from "./model-config";import { generateTopicText } from "./text";@@ -207,34 +207,39 @@ describe("generateTopicText", () => {});});-describe("generateTopicEmbedding", () => {- it("requests clustering vectors of the configured size from Cohere on Bedrock", async () => {+describe("generateTopicEmbeddings", () => {+ it("embeds every summary in one Cohere request on Bedrock", async () => {const vector = Array.from({ length: 512 }, (_, index) => index + 1);+ const other = vector.map((value) => -value);const fetch = vi.fn<typeof globalThis.fetch>(async () =>Response.json(- { embeddings: { float: [vector] }, texts: ["Account access"] },+ {+ embeddings: { float: [vector, other] },+ texts: ["Account access", "Refund request"],+ },{ headers: { "x-amzn-bedrock-input-token-count": "3" } },),);vi.stubGlobal("fetch", fetch);- const result = await generateTopicEmbedding({+ const result = await generateTopicEmbeddings({model: bedrockModel("embedding", "eu.cohere.embed-v4:0"),- summary: "Account access",+ summaries: ["Account access", "Refund request"],dimensions: 512,});+ expect(fetch).toHaveBeenCalledTimes(1);const request = new Request(...fetch.mock.calls[0]);expect(decodeURIComponent(request.url)).toBe("https://bedrock-runtime.eu-central-1.amazonaws.com/model/eu.cohere.embed-v4:0/invoke",);expect(await request.json()).toEqual({input_type: "clustering",- texts: ["Account access"],+ texts: ["Account access", "Refund request"],truncate: "NONE",output_dimension: 512,});- expect(result.embedding).toEqual(vector);+ expect(result.embeddings).toEqual([vector, other]);});it("asks OpenAI for any configured size", async () => {@@ -247,7 +252,7 @@ describe("generateTopicEmbedding", () => {);vi.stubGlobal("fetch", fetch);- const result = await generateTopicEmbedding({+ const result = await generateTopicEmbeddings({model: {slot: "embedding",provider: "openai-topics",@@ -255,7 +260,7 @@ describe("generateTopicEmbedding", () => {model: "text-embedding-3-small",connection: { secretKey: encrypt("sk-test") },},- summary: "Account access",+ summaries: ["Account access"],dimensions: 768,});@@ -266,7 +271,7 @@ describe("generateTopicEmbedding", () => {input: ["Account access"],dimensions: 768,});- expect(result.embedding).toHaveLength(768);+ expect(result.embeddings[0]).toHaveLength(768);});});@@ -343,15 +348,15 @@ describe("allowlisted Cloud Bedrock runtime credentials", () => {),);vi.stubGlobal("fetch", fetch);- const input = { summary: "Account access", dimensions: 512 };+ const input = { summaries: ["Account access"], dimensions: 512 };- await generateTopicEmbedding({+ await generateTopicEmbeddings({...input,model: runtimeModel("embedding", "allowed"),});expect(fetch).toHaveBeenCalledTimes(1);await expect(- generateTopicEmbedding({+ generateTopicEmbeddings({...input,model: runtimeModel("embedding", "other"),}),diff --git a/worker/AGENTS.md b/worker/AGENTS.mdindex b78cd131eb..09901f5311 100644--- a/worker/AGENTS.md+++ b/worker/AGENTS.md@@ -30,7 +30,7 @@Transcript assembly is shared with web through `loadTopicTranscript`, returning`Transcript | null`; serialize it for inference and skip inference on null.Token counting and its WASM dependency stay in `src/features/topics/models.ts`.- Model calls use `generateTopicText` and `generateTopicEmbedding` from+ Model calls use `generateTopicText` and `generateTopicEmbeddings` from`@langfuse/shared/topics/server` with the project's resolved `TopicsModels`(summary, embedding, clustering LLM connections from `topic_configs`).Pass models explicitly; never put them on execution or batch state, which isdiff --git a/worker/src/__tests__/traceBatching.test.ts b/worker/src/__tests__/traceBatching.test.tsindex 2e1cf38fd7..a229aa01d1 100644--- a/worker/src/__tests__/traceBatching.test.ts+++ b/worker/src/__tests__/traceBatching.test.ts@@ -52,7 +52,7 @@ import {} from "@langfuse/shared/topics/server";import {summarizeTopicTraceFacets,- embedTopicSummary,+ embedTopicSummaries,} from "../features/topics/models";vi.mock("@langfuse/shared/src/server", async (importOriginal) => {@@ -112,7 +112,7 @@ vi.mock("@langfuse/shared/topics/server", async (importOriginal) => ({vi.mock("../features/topics/models", () => ({summarizeTopicTraceFacets: vi.fn(),- embedTopicSummary: vi.fn(),+ embedTopicSummaries: vi.fn(),}));vi.mock("../features/evaluation/observationEval", async (importOriginal) => ({@@ -1451,9 +1451,11 @@ describe("trace micro-batch scheduling with Redis", () => {...usage,}),);- vi.mocked(embedTopicSummary).mockImplementation(- async (_model, _text, dimensions) => ({- embedding: Array.from({ length: dimensions }, () => 0.1),+ vi.mocked(embedTopicSummaries).mockImplementation(+ async (_model, summaries, dimensions) => ({+ embeddings: summaries.map(() =>+ Array.from({ length: dimensions }, () => 0.1),+ ),...usage,}),);diff --git a/worker/src/env.ts b/worker/src/env.tsindex 3e0fef80fb..d0122c2fbc 100644--- a/worker/src/env.ts+++ b/worker/src/env.ts@@ -170,6 +170,12 @@ const EnvSchema = z.object({.int().positive().default(100 * 1024 * 1024),+ // Traces summarized and embedded at once per Topics job, in both trace-batch and manual processing.+ LANGFUSE_TOPICS_TRACE_CONCURRENCY: z.coerce+ .number()+ .int()+ .positive()+ .default(20),LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE: z.coerce.number().int()diff --git a/worker/src/features/topics/README.md b/worker/src/features/topics/README.mdindex 32bbb19809..9bf39deff9 100644--- a/worker/src/features/topics/README.md+++ b/worker/src/features/topics/README.md@@ -115,8 +115,12 @@ residency matters. Every provider is asked for clustering vectors of theconfigured size (Cohere `clustering` input type with truncation disabled, Google`CLUSTERING` task type, OpenAI `dimensions`). Default: 1,024dimensions; any whole number from 64 to 4,096 is accepted, and a provider that-returns a vector of another length fails the call. Calls embed one summary at a-time, retaining per-summary Redis checkpointing.+returns a vector of another length fails the call. Embeddings are batched per trace: each trace's+applicable facets are embedded in one call, in trace-batch and in embedding jobs+(which run up to `LANGFUSE_TOPICS_TRACE_CONCURRENCY` traces at once). The call's+usage is recorded on the trace's first row; one failure fails that trace's call,+and a retry embeds the traces that are left. Saving the models requires the+embedding model to embed two texts in one call.Embedding settings are independent of immutable facet versions; changing themwith stored-summary reuse enabled regenerates vectors without summarization inference. This PoC doesnot use `aiEmbed` or change ingestion. It accepts traces already stored in v4@@ -138,7 +142,8 @@ call's usage is recorded on the first facet's row. It does not load the tracefrom ClickHouse again. Projects outside`LANGFUSE_TOPICS_ENABLED_PROJECT_IDS` are skipped. A finished facet version isskipped on retry. A failing trace does not stop the batch: every trace is-processed and outcomes are counted once per job (`langfuse.topics.trace_outcomes`+processed, up to `LANGFUSE_TOPICS_TRACE_CONCURRENCY` traces at once (default+20; manual processing uses the same limit per queue job), and outcomes are counted once per job (`langfuse.topics.trace_outcomes`by outcome and reason). Failed traces are not retried, because a retry wouldread the shared batch from ClickHouse again. Only a failed batch read fails thejob. Summaries and embeddings use the same project models as manualdiff --git a/worker/src/features/topics/models.test.ts b/worker/src/features/topics/models.test.tsindex 11bff393d5..48fb49b73b 100644--- a/worker/src/features/topics/models.test.ts+++ b/worker/src/features/topics/models.test.ts@@ -11,11 +11,11 @@ vi.mock("@langfuse/shared/src/server", () => ({}));vi.mock("@langfuse/shared/topics/server", () => ({generateTopicText: (...args: unknown[]) => state.call(...args),- generateTopicEmbedding: (...args: unknown[]) => state.embed(...args),+ generateTopicEmbeddings: (...args: unknown[]) => state.embed(...args),}));import {- embedTopicSummary,+ embedTopicSummaries,nameTopicGroup,summarizeTopicTrace,summarizeTopicTraceFacets,@@ -342,9 +342,13 @@ describe("Topics naming boundary", () => {);it("records embedding input usage even when the returned vector is invalid", async () => {- state.embed.mockResolvedValue({ embedding: [], tokens: 12 });+ state.embed.mockResolvedValue({ embeddings: [[]], tokens: 12 });await expect(- embedTopicSummary(topicsModels().embedding, "An invoice request.", 256),+ embedTopicSummaries(+ topicsModels().embedding,+ ["An invoice request."],+ 256,+ ),).rejects.toMatchObject({reason: "invalid_output",});@@ -361,12 +365,12 @@ describe("Topics naming boundary", () => {"does not invent missing embedding usage: %s",async (tokens) => {state.embed.mockResolvedValue({- embedding: Array(256).fill(0.25),+ embeddings: [Array(256).fill(0.25)],tokens,});- await embedTopicSummary(+ await embedTopicSummaries(topicsModels().embedding,- "An invoice request.",+ ["An invoice request."],256,);expect(state.increment.mock.calls).toEqual([diff --git a/worker/src/features/topics/models.ts b/worker/src/features/topics/models.tsindex 6026f13c50..7d520ea354 100644--- a/worker/src/features/topics/models.ts+++ b/worker/src/features/topics/models.ts@@ -2,7 +2,7 @@ import { get_encoding } from "tiktoken";import { z } from "zod";import { logger } from "@langfuse/shared/src/server";import {- generateTopicEmbedding,+ generateTopicEmbeddings,generateTopicText,type TopicPromptPart,type TopicsModel,@@ -348,13 +348,15 @@ export async function nameTopicGroup();}-export async function embedTopicSummary(+/** Embeds summaries in one batched call; the usage covers every returned vector. */+export async function embedTopicSummaries(model: TopicsModel,- summary: string,+ summaries: string[],dimensions: number,-): Promise<TopicModelUsage & { embedding: number[] }> {+): Promise<TopicModelUsage & { embeddings: number[][] }> {if (- !summary.trim() ||+ summaries.length === 0 ||+ summaries.some((summary) => !summary.trim()) ||!topicEmbeddingConfigSchema.safeParse({ embeddingDimensions: dimensions }).success)@@ -362,20 +364,26 @@ export async function embedTopicSummary("Topics embeddings require non-empty text and 64 to 4,096 dimensions.","invalid_input",);- const result = await generateTopicEmbedding({+ const result = await generateTopicEmbeddings({model,- summary,+ summaries,dimensions,}).catch((error: unknown) => {throw topicProviderError(error, model);});recordTopicTokenUsage("embedding", { input: result.tokens });// ClickHouse stores Float32; calibration and future classification must use those same vectors.- const embedding = Array.from(new Float32Array(result.embedding));+ const embeddings = result.embeddings.map((vector) =>+ Array.from(new Float32Array(vector)),+ );if (- embedding.length !== dimensions ||- !embedding.every(Number.isFinite) ||- !embedding.some((value) => value !== 0)+ embeddings.length !== summaries.length ||+ embeddings.some(+ (embedding) =>+ embedding.length !== dimensions ||+ !embedding.every(Number.isFinite) ||+ !embedding.some((value) => value !== 0),+ ))throw new TopicsProviderUnavailable("Topics embedding provider returned an invalid vector.",@@ -394,7 +402,7 @@ export async function embedTopicSummary(});}return {- embedding,+ embeddings,providedUsageDetails: usageDetails,usageDetails,providedCostDetails: {},diff --git a/worker/src/features/topics/processTopicEmbeddingBatch.test.ts b/worker/src/features/topics/processTopicEmbeddingBatch.test.tsindex 6cd81853b8..edb5a29855 100644--- a/worker/src/features/topics/processTopicEmbeddingBatch.test.ts+++ b/worker/src/features/topics/processTopicEmbeddingBatch.test.ts@@ -41,7 +41,7 @@ vi.mock("@langfuse/shared/src/server", () => ({recordDistribution: vi.fn(),}));vi.mock("./models", () => ({- embedTopicSummary: mocks.embed,+ embedTopicSummaries: mocks.embed,}));import { processTopicEmbeddingBatch } from "./processTopicEmbeddingBatch";@@ -89,8 +89,7 @@ const batch = (...rows: TopicSummary[]): TopicEmbeddingBatch => ({traceId: row.traceId,})),});-const embeddingResult = {- embedding: Array(256).fill(0.25),+const embeddingUsage = {providedUsageDetails: { embedding_input: 8, total: 8 },usageDetails: { embedding_input: 8, total: 8 },providedCostDetails: {},@@ -116,7 +115,10 @@ beforeEach(() => {staged.set(topicSourceKey({ ..._batch, ...ref, sessionId: null }), row);return row;});- mocks.embed.mockResolvedValue(embeddingResult);+ mocks.embed.mockImplementation(async (_model, summaries: string[]) => ({+ ...embeddingUsage,+ embeddings: summaries.map(() => Array(256).fill(0.25)),+ }));});describe("Topics embedding handoff", () => {@@ -198,26 +200,58 @@ describe("Topics embedding handoff", () => {expect(mocks.write).not.toHaveBeenCalled();});- it("persists completed rows before retrying a later provider failure", async () => {- const rows = [summary("first"), summary("second")];- for (const row of rows) staged.set(topicSourceKey(row), row);- mocks.embed.mockResolvedValueOnce(embeddingResult);+ it("embeds each trace's facets in one call and keeps finished traces when another fails", async () => {+ const intent = summary("first");+ const outcome: TopicSummary = {+ ...intent,+ facetId: "outcome",+ summary: "An outcome of first.",+ };+ const other = summary("second");+ for (const row of [intent, outcome, other])+ staged.set(topicSourceKey(row), row);const failure = new TopicsProviderUnavailable("Topics provider call failed (HTTP 429).","rate_limit",);- mocks.embed.mockRejectedValueOnce(failure);- await expect(processTopicEmbeddingBatch(batch(...rows))).rejects.toBe(- failure,- );+ mocks.embed.mockImplementation(async (_model, summaries: string[]) => {+ if (summaries.includes("A summary of second.")) throw failure;+ return {+ ...embeddingUsage,+ embeddings: summaries.map(() => Array(256).fill(0.25)),+ };+ });+ await expect(+ processTopicEmbeddingBatch(batch(intent, outcome, other)),+ ).rejects.toBe(failure);+ expect(mocks.embed).toHaveBeenCalledTimes(2);+ expect(mocks.embed.mock.calls.map((call) => call[1])).toContainEqual([+ "A summary of first.",+ "An outcome of first.",+ ]);+ // The finished trace is stored with the call's usage recorded once.expect(mocks.write.mock.calls[0][0]).toMatchObject([- { traceId: "trace-first", state: "complete" },+ { facetId: "facet", state: "complete", usageDetails: { total: 38 } },+ { facetId: "outcome", state: "complete", usageDetails: { total: 30 } },]);- expect(staged.get(topicSourceKey(rows[0]))?.state).toBe("complete");- expect(staged.get(topicSourceKey(rows[1]))?.state).toBe("summarized");- await processTopicEmbeddingBatch(batch(...rows));- expect(mocks.embed).toHaveBeenCalledTimes(3);- expect(staged.get(topicSourceKey(rows[1]))?.state).toBe("complete");+ expect(staged.get(topicSourceKey(other))?.state).toBe("summarized");+ });++ it("stops on a permanent failure even when another trace failed retryably", async () => {+ const rows = [summary("first"), summary("second")];+ for (const row of rows) staged.set(topicSourceKey(row), row);+ mocks.embed.mockImplementation(async (_model, summaries: string[]) => {+ throw summaries.includes("A summary of first.")+ ? new TopicsProviderUnavailable("Rate limited.", "rate_limit")+ : new TopicsProviderUnavailable(+ "Check worker credentials.",+ "authentication",+ );+ });+ await expect(processTopicEmbeddingBatch(batch(...rows))).rejects.toThrow(+ UnrecoverableError,+ );+ expect(mocks.pause).toHaveBeenCalledOnce();});it("does not retry provider authentication failures", async () => {@@ -257,7 +291,7 @@ describe("Topics embedding handoff", () => {staged.set(topicSourceKey(row), row);mocks.embed.mockImplementation(async () => {staged.delete(topicSourceKey(row));- return embeddingResult;+ return { ...embeddingUsage, embeddings: [Array(256).fill(0.25)] };});await expect(processTopicEmbeddingBatch(batch(row))).rejects.toThrow(UnrecoverableError,diff --git a/worker/src/features/topics/processTopicEmbeddingBatch.ts b/worker/src/features/topics/processTopicEmbeddingBatch.tsindex ee6ec65c69..1e6d7bbfd1 100644--- a/worker/src/features/topics/processTopicEmbeddingBatch.ts+++ b/worker/src/features/topics/processTopicEmbeddingBatch.ts@@ -1,4 +1,5 @@import { UnrecoverableError } from "bullmq";+import pLimit from "p-limit";import type { TopicSummary } from "@langfuse/shared/topics";import {getTopicsModels,@@ -8,7 +9,8 @@ import {writeTopicSummaries,type TopicEmbeddingBatch,} from "@langfuse/shared/topics/server";-import { embedTopicSummary } from "./models";+import { env } from "../../env";+import { embedTopicSummaries } from "./models";import { TopicMetrics } from "./metrics";import { mergeTopicModelUsage } from "./summaryResult";import {@@ -23,7 +25,11 @@ export async function processTopicEmbeddingBatch(batch: TopicEmbeddingBatch,): Promise<void> {const metrics = new TopicMetrics();- const pending: TopicSummary[] = [];+ // Batch order; rows still `summarized` are not persisted.+ const rows: {+ ref: TopicEmbeddingBatch["summaries"][number];+ summary: TopicSummary;+ }[] = [];try {const models = await getTopicsModels(batch.projectId);if (!models)@@ -31,6 +37,8 @@ export async function processTopicEmbeddingBatch("Choose Topics models for this project, then start a new Topics execution.","configuration",);+ // Every summary in a batch belongs to one execution's embedding configuration.+ let dimensions = models.embedding.dimensions;for (const ref of batch.summaries) {const staged = await readStagedTopicSummary(batch, ref);if (!staged) throw new UnrecoverableError(TOPIC_EMBEDDING_EXPIRED_ERROR);@@ -39,54 +47,77 @@ export async function processTopicEmbeddingBatch("The project's embedding model changed after this batch was created. Start a new Topics execution.","configuration",);- let { summary } = staged;- if (summary.state === "summarized") {- const result = await metrics.measure("embedding", async () => {- try {- return await embedTopicSummary(- models.embedding,- summary.summary,- staged.embeddingConfig.embeddingDimensions,- );- } catch (error) {- throw error instanceof TopicsProviderUnavailable- ? error- : topicProviderError(error, models.embedding);- }- });- summary = {- ...summary,- state: "complete",- embedding: result.embedding,- ...mergeTopicModelUsage(summary, result),- processedAt: new Date().toISOString(),- };- metrics.result("embedding", "generated");- // All overlapping attempts must persist the same accepted result.- const accepted = await updateStagedTopicSummary(batch, ref, summary);- if (!accepted)- throw new UnrecoverableError(TOPIC_EMBEDDING_EXPIRED_ERROR);- pending.push(accepted);- } else {- if (summary.state === "complete") metrics.result("embedding", "cached");- pending.push(summary);- }+ dimensions = staged.embeddingConfig.embeddingDimensions;+ if (staged.summary.state === "complete")+ metrics.result("embedding", "cached");+ rows.push({ ref, summary: staged.summary });}+ const unembedded = rows.filter(+ ({ summary }) => summary.state === "summarized",+ );+ // One call per trace (at most one text per facet), so each call keeps its own+ // deadline and finished traces stay staged when another trace fails.+ const limit = pLimit(env.LANGFUSE_TOPICS_TRACE_CONCURRENCY);+ const results = await Promise.allSettled(+ [...Map.groupBy(unembedded, ({ ref }) => ref.traceId).values()].map(+ (traceRows) =>+ limit(async () => {+ const result = await metrics.measure("embedding", async () => {+ try {+ return await embedTopicSummaries(+ models.embedding,+ traceRows.map(({ summary }) => summary.summary),+ dimensions,+ );+ } catch (error) {+ throw error instanceof TopicsProviderUnavailable+ ? error+ : topicProviderError(error, models.embedding);+ }+ });+ const processedAt = new Date().toISOString();+ for (const [index, row] of traceRows.entries()) {+ const embedded: TopicSummary = {+ ...row.summary,+ state: "complete",+ embedding: result.embeddings[index],+ // The call's usage is recorded once, on the trace's first summary.+ ...(index === 0+ ? mergeTopicModelUsage(row.summary, result)+ : {}),+ processedAt,+ };+ metrics.result("embedding", "generated");+ // All overlapping attempts must persist the same accepted result.+ const accepted = await updateStagedTopicSummary(+ batch,+ row.ref,+ embedded,+ );+ if (!accepted)+ throw new UnrecoverableError(TOPIC_EMBEDDING_EXPIRED_ERROR);+ row.summary = accepted;+ }+ }),+ ),+ );+ const failures = results.flatMap((result) =>+ result.status === "rejected" ? [result.reason] : [],+ );+ // A permanent failure must stop the job and pause Topics even if another+ // trace failed with a retryable error first.+ if (failures.length > 0)+ throw failures.find(isPermanentFailure) ?? failures[0];} catch (error) {if (isRejectedConnection(error))await pauseForRejectedConnection(batch.projectId, error);- if (- error instanceof TopicsProviderUnavailable &&- [- "authentication",- "configuration",- "invalid_input",- "invalid_output",- ].includes(error.reason)- )+ if (error instanceof TopicsProviderUnavailable && isPermanentFailure(error))throw new UnrecoverableError(error.message);throw error;} finally {+ const pending = rows+ .map(({ summary }) => summary)+ .filter((summary) => summary.state !== "summarized");if (pending.length)await metrics.measure("storage",@@ -97,3 +128,16 @@ export async function processTopicEmbeddingBatch(// Retrying a partial insert reuses these vectors and their processing time.}}++function isPermanentFailure(error: unknown): boolean {+ return (+ error instanceof UnrecoverableError ||+ (error instanceof TopicsProviderUnavailable &&+ [+ "authentication",+ "configuration",+ "invalid_input",+ "invalid_output",+ ].includes(error.reason))+ );+}diff --git a/worker/src/features/topics/processTopicsExecution.test.ts b/worker/src/features/topics/processTopicsExecution.test.tsindex 670d6908dd..785641ff97 100644--- a/worker/src/features/topics/processTopicsExecution.test.ts+++ b/worker/src/features/topics/processTopicsExecution.test.ts@@ -315,8 +315,17 @@ vi.mock("@langfuse/shared/topics/server", async (importOriginal) => {vi.mock("./models", () => ({summarizeTopicTrace: (_models: unknown, ...args: unknown[]) =>state.summarize(...args),- embedTopicSummary: (_model: unknown, ...args: unknown[]) =>- state.embed(...args),+ // `state.embed` sees each summary; one failure fails the whole batched call.+ embedTopicSummaries: async (+ _model: unknown,+ summaries: string[],+ dimensions: number,+ ) => {+ const results = [];+ for (const summary of summaries)+ results.push(await state.embed(summary, dimensions));+ return { ...results[0], embeddings: results.map((r) => r.embedding) };+ },nameTopicGroup: (_models: unknown, ...args: unknown[]) => state.name(...args),}));vi.mock("./numeric", async (importOriginal) => ({@@ -816,7 +825,8 @@ describe("Topics execution", () => {),);await processSelection("partial", 3);- expect(state.batches.get("partial")?.summaries).toHaveLength(1);+ // Traces run concurrently, so the third finishes while the second fails.+ expect(state.batches.get("partial")?.summaries).toHaveLength(2);expect(state.executions.get("partial")?.status).toBe("failed");const accepted = [...state.staged.values()][0].summary;accepted.transcriptVersion = "older-transcript";diff --git a/worker/src/features/topics/processTopicsExecution.ts b/worker/src/features/topics/processTopicsExecution.tsindex c5ca4da92d..1c0247d507 100644--- a/worker/src/features/topics/processTopicsExecution.ts+++ b/worker/src/features/topics/processTopicsExecution.ts@@ -39,6 +39,8 @@ import {sameTopicGeometry,topicTimeRangeSchema,} from "@langfuse/shared/topics";+import pLimit from "p-limit";+import { env } from "../../env";import { summarizeTopicTrace, nameTopicGroup } from "./models";import {isRejectedConnection,@@ -820,79 +822,102 @@ async function processTraces()throw new TopicEmbeddingFailure(TOPIC_EMBEDDING_EXPIRED_ERROR);}- for (const traceId of execution.input.traceIds) {- let input: ReturnType<typeof loadTopicTranscript> | undefined;- const getTranscript = () => {- if (!input)- input = metrics.measure(- "transcript",- () =>- loadTopicTranscript({ projectId: execution.projectId, traceId }),- "trace_load",- );- return input;- };- for (const { facet, progress } of facets) {- if (- state.summaries.some(- (ref) =>- ref.facetId === facet.facetId &&- ref.facetVersion === facet.version &&- ref.traceId === traceId,- ) ||- state.failedTraceIds.some(- (entry) =>- entry.facetId === facet.facetId &&- entry.facetVersion === facet.version &&- entry.traceIds.includes(traceId),- )- )- continue;- try {- await summarizeTrace(- metrics,- models,- execution,- facet,- traceId,- cached.get(facet)!,- getTranscript,- );- state.summaries.push({- facetId: facet.facetId,- facetVersion: facet.version,- traceId,- });- await save("summarizing");- } catch (error) {- metrics.error("summary", error);- if (isExecutionFailure(error)) throw error;- let failed = state.failedTraceIds.find(- (entry) =>- entry.facetId === facet.facetId &&- entry.facetVersion === facet.version,- );- if (!failed) {- failed = {- facetId: facet.facetId,- facetVersion: facet.version,- traceIds: [],- };- state.failedTraceIds.push(failed);- }- failed.traceIds.push(traceId);- progress.counts.failed = failed.traceIds.length;- const message = errorMessage(error);- if (- !execution.traceErrors.some(- (item) => item.traceId === traceId && item.error === message,+ // Traces run concurrently; a trace's facets share its transcript and run in order.+ const limit = pLimit(env.LANGFUSE_TOPICS_TRACE_CONCURRENCY);+ let saving = Promise.resolve();+ // Saves write the whole state, so they must not land out of order.+ const saveInOrder = () => {+ const saved = saving.then(() => save("summarizing"));+ saving = saved.catch(() => {});+ return saved;+ };+ let aborted = false;+ const results = await Promise.allSettled(+ execution.input.traceIds.map((traceId) =>+ limit(async () => {+ if (aborted) return;+ let input: ReturnType<typeof loadTopicTranscript> | undefined;+ const getTranscript = () => {+ if (!input)+ input = metrics.measure(+ "transcript",+ () =>+ loadTopicTranscript({+ projectId: execution.projectId,+ traceId,+ }),+ "trace_load",+ );+ return input;+ };+ for (const { facet, progress } of facets) {+ if (+ state.summaries.some(+ (ref) =>+ ref.facetId === facet.facetId &&+ ref.facetVersion === facet.version &&+ ref.traceId === traceId,+ ) ||+ state.failedTraceIds.some(+ (entry) =>+ entry.facetId === facet.facetId &&+ entry.facetVersion === facet.version &&+ entry.traceIds.includes(traceId),+ ))- )- execution.traceErrors.push({ traceId, error: message });- await save("summarizing");- }- }- }+ continue;+ try {+ await summarizeTrace(+ metrics,+ models,+ execution,+ facet,+ traceId,+ cached.get(facet)!,+ getTranscript,+ );+ state.summaries.push({+ facetId: facet.facetId,+ facetVersion: facet.version,+ traceId,+ });+ await saveInOrder();+ } catch (error) {+ metrics.error("summary", error);+ if (isExecutionFailure(error)) {+ aborted = true;+ throw error;+ }+ let failed = state.failedTraceIds.find(+ (entry) =>+ entry.facetId === facet.facetId &&+ entry.facetVersion === facet.version,+ );+ if (!failed) {+ failed = {+ facetId: facet.facetId,+ facetVersion: facet.version,+ traceIds: [],+ };+ state.failedTraceIds.push(failed);+ }+ failed.traceIds.push(traceId);+ progress.counts.failed = failed.traceIds.length;+ const message = errorMessage(error);+ if (+ !execution.traceErrors.some(+ (item) => item.traceId === traceId && item.error === message,+ )+ )+ execution.traceErrors.push({ traceId, error: message });+ await saveInOrder();+ }+ }+ }),+ ),+ );+ const failure = results.find((result) => result.status === "rejected");+ if (failure) throw failure.reason;state.summarized = true;await save("embedding");}diff --git a/worker/src/features/topics/summarizeAssembledTrace.test.ts b/worker/src/features/topics/summarizeAssembledTrace.test.tsindex 40c6f5804e..00cc8d47c0 100644--- a/worker/src/features/topics/summarizeAssembledTrace.test.ts+++ b/worker/src/features/topics/summarizeAssembledTrace.test.ts@@ -34,7 +34,7 @@ vi.mock("@langfuse/shared/src/server", () => ({}));vi.mock("./models", () => ({summarizeTopicTraceFacets: (...args: unknown[]) => state.summarize(...args),- embedTopicSummary: (...args: unknown[]) => state.embed(...args),+ embedTopicSummaries: (...args: unknown[]) => state.embed(...args),}));import { summarizeAssembledTrace } from "./summarizeAssembledTrace";@@ -123,13 +123,13 @@ beforeEach(() => {},...usage,});- state.embed.mockResolvedValue({- embedding: [0.25],+ state.embed.mockImplementation(async (_model, summaries: string[]) => ({+ embeddings: summaries.map(() => [0.25]),providedUsageDetails: { embedding_input: 4 },usageDetails: { embedding_input: 4, total: 4 },providedCostDetails: {},costDetails: { embedding_input: 0.00000048, total: 0.00000048 },- });+ }));});describe("summarizeAssembledTrace", () => {@@ -221,18 +221,25 @@ describe("summarizeAssembledTrace", () => {expect(state.stored).toHaveBeenCalledTimes(3);});- it("summarizes all pending facets in one call and records its usage once", async () => {+ it("summarizes and embeds all pending facets in one call each and records their usage once", async () => {const issues: TopicFacet = {...facet,id: "facet-2",name: "Issues",versions: [{ ...facet.versions[0], facetId: "facet-2", version: 1 }],};- state.facets.mockReturnValue([facet, issues]);+ const outcome: TopicFacet = {+ ...facet,+ id: "facet-3",+ name: "Outcome",+ versions: [{ ...facet.versions[0], facetId: "facet-3", version: 1 }],+ };+ state.facets.mockReturnValue([facet, issues, outcome]);state.summarize.mockResolvedValue({output: {intent_1: { summary: "Export monthly sales.", status: "applicable" },issues_2: { summary: "", status: "not_applicable" },+ outcome_3: { summary: "Export completed.", status: "applicable" },},...usage,});diff --git a/worker/src/features/topics/summarizeAssembledTrace.ts b/worker/src/features/topics/summarizeAssembledTrace.tsindex 79b7646957..035485f898 100644--- a/worker/src/features/topics/summarizeAssembledTrace.ts+++ b/worker/src/features/topics/summarizeAssembledTrace.ts@@ -17,7 +17,7 @@ import {} from "@langfuse/shared/src/server";import { prepareAssembledTopicTranscript } from "./assembledTranscript";import { TopicMetrics } from "./metrics";-import { embedTopicSummary, summarizeTopicTraceFacets } from "./models";+import { embedTopicSummaries, summarizeTopicTraceFacets } from "./models";import { TopicsProviderUnavailable } from "./provider-error";import {mergeTopicModelUsage,@@ -178,25 +178,53 @@ async function summarizeEnabledTrace(providedCostDetails: result.providedCostDetails,costDetails: result.costDetails,};- rows = [];- // A failed facet (invalid output, embedding error) must not discard the others.- for (const [index, { key }] of keyed.entries()) {- if (invalidFacets.has(key)) continue;+ const accepted = keyed.flatMap(({ key }, index) =>+ invalidFacets.has(key)+ ? []+ : [{ base: bases[index], output: outputs[key] }],+ );+ const applicable = accepted+ .filter(({ output }) => output.status === "applicable")+ .map(({ output }) => output.summary.trim());+ let embedded: Awaited<ReturnType<typeof embedTopicSummaries>> | undefined;+ if (applicable.length > 0) {try {- rows.push(- await finalizeSummary(- bases[index],- outputs[key],- // One call serves every facet of the trace; record its usage once.- rows.length ? NO_USAGE : usage,- models,- metrics,+ embedded = await metrics.measure("embedding", () =>+ embedTopicSummaries(+ models.embedding,+ applicable,+ models.embedding.dimensions,),);} catch (error) {+ // Non-applicable facets are still persisted when the embedding fails.if (failure === undefined) failure = error;}}+ // One call summarizes and one embeds every facet of the trace; record their usage once.+ const traceUsage = embedded ? mergeTopicModelUsage(usage, embedded) : usage;+ rows = [];+ let embeddingIndex = 0;+ for (const { base, output } of accepted) {+ const rowUsage = rows.length > 0 ? NO_USAGE : traceUsage;+ if (output.status !== "applicable")+ rows.push({+ ...base,+ state:+ output.status === "not_applicable"+ ? "not_applicable"+ : "insufficient_input",+ ...rowUsage,+ });+ else if (embedded)+ rows.push({+ ...base,+ state: "complete",+ summary: output.summary.trim(),+ embedding: embedded.embeddings[embeddingIndex++],+ ...rowUsage,+ });+ }}if (rows.length)await metrics.measure("storage", () => writeTopicSummaries(rows));@@ -240,32 +268,3 @@ function baseSummary(metadata: { input: "assembled-transcript" },};}--async function finalizeSummary(- base: TopicSummary,- output: FacetOutput,- usage: TopicModelUsage,- models: TopicsModels,- metrics: TopicMetrics,-): Promise<TopicSummary> {- if (output.status !== "applicable")- return {- ...base,- state:- output.status === "not_applicable"- ? "not_applicable"- : "insufficient_input",- ...usage,- };- const summary = output.summary.trim();- const embedded = await metrics.measure("embedding", () =>- embedTopicSummary(models.embedding, summary, models.embedding.dimensions),- );- return {- ...base,- state: "complete",- summary,- embedding: embedded.embedding,- ...mergeTopicModelUsage(usage, embedded),- };-}diff --git a/worker/src/features/traceBatching/README.md b/worker/src/features/traceBatching/README.mdindex e414f1421c..c3f276daa3 100644--- a/worker/src/features/traceBatching/README.md+++ b/worker/src/features/traceBatching/README.md@@ -50,6 +50,7 @@ templates. The local dev template sets the idle time below.| `LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE` | unset | Optional positive `max_block_size` hint; unset inherits the server profile. || `LANGFUSE_TRACE_BATCH_EXPERIMENT_ID` | unset | Optional 1–64 character attempt/query label: letters, digits, `.`, `_`, `-`; first character alphanumeric. || `LANGFUSE_TRACE_BATCH_CONCURRENCY` | `2` | Concurrent batch jobs per worker process. |+| `LANGFUSE_TOPICS_TRACE_CONCURRENCY` | `20` | Topics traces summarized and embedded at once per batch job (positive integer). Tokenization stays one trace at a time. || `LANGFUSE_TRACE_BATCH_IDLE_MS` | `600000` | Default idle time before a trace is ready. Unset and `DEV` uses `120000` (2 minutes); every other region uses 10 minutes. A project's `tracing_readiness_settings.trace_idle_ms` (60 seconds to 60 minutes) overrides it, capped at the pending TTL minus the dispatch interval. The project value is cached in Redis for 5 minutes; saving replaces it, and a failed read uses the default without caching. || `LANGFUSE_TRACE_BATCH_PENDING_TTL_MS` | `7200000` | Recovery grace after a trace becomes due (2 hours). Pending keys expire no earlier than the latest required due-plus-grace deadline. || `LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS` | `86400000` | Historical event-start bounds and size estimates retained after activity or dispatch (24 hours; at most 48 hours). Internal and cloud-gated. |diff --git a/worker/src/queues/__tests__/traceBatchQueue.test.ts b/worker/src/queues/__tests__/traceBatchQueue.test.tsindex 014a143efd..0134d2a231 100644--- a/worker/src/queues/__tests__/traceBatchQueue.test.ts+++ b/worker/src/queues/__tests__/traceBatchQueue.test.ts@@ -594,7 +594,7 @@ describe("trace batch queue", () => {}},);- it("resumes streaming below the payload cap while a later Topics summary is pending", async () => {+ it("summarizes traces concurrently and resumes streaming when any summary brings payloads below the cap", async () => {const originalCap = env.LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES;env.LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES = 1_000;const payload = "x".repeat(600);@@ -666,15 +666,16 @@ describe("trace batch queue", () => {);expect(reachedEnd).toBe(false);await vi.waitFor(() =>- expect(summarizeAssembledTrace).toHaveBeenCalledTimes(1),+ expect(summarizeAssembledTrace).toHaveBeenCalledTimes(2),);- resolveFirst();+ // The later trace finishing first releases its payload.+ resolveSecond();await vi.waitFor(() => expect(reachedEnd).toBe(true));await vi.waitFor(() =>- expect(summarizeAssembledTrace).toHaveBeenCalledTimes(2),+ expect(summarizeAssembledTrace).toHaveBeenCalledTimes(3),);expect(settled).toBe(false);- resolveSecond();+ resolveFirst();await expect(processing).resolves.toEqual(expect.objectContaining({ traceCount: 3 }),);diff --git a/worker/src/queues/traceBatchQueue.ts b/worker/src/queues/traceBatchQueue.tsindex 8adea51fdd..b94cdee21b 100644--- a/worker/src/queues/traceBatchQueue.ts+++ b/worker/src/queues/traceBatchQueue.ts@@ -1,4 +1,5 @@import { type Processor } from "bullmq";+import pLimit from "p-limit";import { randomUUID } from "node:crypto";import { type Observation } from "@langfuse/shared";import {@@ -12,6 +13,7 @@ import {TraceBatchEventSchema,type QueueName,type TQueueJobTypes,+ type Transcript,} from "@langfuse/shared/src/server";import {getEnabledTopicsModels,@@ -96,6 +98,7 @@ async function loadProjectRule(projectId: string) {async function summarizeTraceBatch(observations: Observation[],+ transcript: Transcript | null,topics: TopicsBatchProjects,jobEnqueuedAt: Date,): Promise<TraceOutcome> {@@ -114,46 +117,41 @@ async function summarizeTraceBatch(const traceTimestamp = new Date(Math.min(...observations.map((row) => row.startTime.getTime())),).toISOString();- let outcome: TraceOutcome = { outcome: "disabled" };- await recordTraceBatchTranscript(observations, async (transcript) => {- try {- const project = await topics.get(first.projectId);- if (- !project ||- !shouldSampleEvaluation({- samplingValue: getDeterministicSamplingValue(traceId),- samplingRate: project.sampling,- })- )- return;- outcome = {- outcome: await summarizeAssembledTrace({- projectId: first.projectId,- traceId,- traceTimestamp,- jobEnqueuedAt,- environment: first.environment,- traceName: first.name ?? "",- transcript,- models: project.models,- facets: project.facets,- }),- };- } catch (error) {- // One trace's failure must not fail or re-read the shared batch.- const reason =- error instanceof TopicsProviderUnavailable ? error.reason : "other";- if (isRejectedConnection(error))- await topics.pause(first.projectId, error);- outcome = { outcome: "failed", reason };- logger.warn("Topics summary failed for trace", {+ try {+ const project = await topics.get(first.projectId);+ if (+ !project ||+ !shouldSampleEvaluation({+ samplingValue: getDeterministicSamplingValue(traceId),+ samplingRate: project.sampling,+ })+ )+ return { outcome: "disabled" };+ return {+ outcome: await summarizeAssembledTrace({projectId: first.projectId,traceId,- reason,- });- }- });- return outcome;+ traceTimestamp,+ jobEnqueuedAt,+ environment: first.environment,+ traceName: first.name ?? "",+ transcript,+ models: project.models,+ facets: project.facets,+ }),+ };+ } catch (error) {+ // One trace's failure must not fail or re-read the shared batch.+ const reason =+ error instanceof TopicsProviderUnavailable ? error.reason : "other";+ if (isRejectedConnection(error)) await topics.pause(first.projectId, error);+ logger.warn("Topics summary failed for trace", {+ projectId: first.projectId,+ traceId,+ reason,+ });+ return { outcome: "failed", reason };+ }}/** Failed traces are counted, not retried. */@@ -299,27 +297,47 @@ export const traceBatchQueueProcessor: Processor<// ClickHouse sockets time out when idle, so the reader never waits on// transcript or Topics work unless queued payloads exceed the cap.- // Traces are still summarized one at a time per batch to keep at most- // one pending request in the tokenizer pool shared with ingestion.- const pendingSummaries: Promise<void>[] = [];+ // Transcripts are tokenized one trace at a time to keep at most one pending+ // request in the tokenizer pool shared with ingestion; Topics model calls+ // then run for up to LANGFUSE_TOPICS_TRACE_CONCURRENCY traces at once.+ const pendingSummaries = new Set<Promise<void>>();+ const topicsLimit = pLimit(env.LANGFUSE_TOPICS_TRACE_CONCURRENCY);+ let transcribing: Promise<unknown> = Promise.resolve();let pendingSummaryBytes = 0;let summaryFailed = false;let summaryError: unknown;const enqueueSummary = (observations: Observation[], bytes: number) => {pendingSummaryBytes += bytes;const enqueuedAt = performance.now();- const summary = (pendingSummaries.at(-1) ?? Promise.resolve())- .then(async () => {- // Let socket reads run between traces' synchronous assembly.- await new Promise((resolve) => setImmediate(resolve));- recordDistribution(- "langfuse.trace_batch.summary_queue_wait_ms",- performance.now() - enqueuedAt,- );- if (!summaryFailed)- topicsOutcomes.push(- await summarizeTraceBatch(observations, topics, event.timestamp),- );+ const transcribed = transcribing.then(async () => {+ // Let socket reads run between traces' synchronous assembly.+ await new Promise((resolve) => setImmediate(resolve));+ recordDistribution(+ "langfuse.trace_batch.summary_queue_wait_ms",+ performance.now() - enqueuedAt,+ );+ if (summaryFailed) return null;+ let transcript: Transcript | null = null;+ await recordTraceBatchTranscript(observations, async (assembled) => {+ transcript = assembled;+ });+ return { transcript };+ });+ transcribing = transcribed.catch(() => {});+ const summary: Promise<void> = transcribed+ .then(async (assembled) => {+ if (!assembled) return;+ await topicsLimit(async () => {+ if (!summaryFailed)+ topicsOutcomes.push(+ await summarizeTraceBatch(+ observations,+ assembled.transcript,+ topics,+ event.timestamp,+ ),+ );+ });}).catch((error: unknown) => {if (!summaryFailed) summaryError = error;@@ -327,9 +345,9 @@ export const traceBatchQueueProcessor: Processor<}).finally(() => {pendingSummaryBytes -= bytes;- pendingSummaries.shift();+ pendingSummaries.delete(summary);});- pendingSummaries.push(summary);+ pendingSummaries.add(summary);};activeReads++;try {@@ -371,7 +389,7 @@ export const traceBatchQueueProcessor: Processor<pendingSummaryBytes >env.LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES)- await pendingSummaries[0];+ await Promise.race(pendingSummaries);}if (summaryFailed) break;}@@ -398,7 +416,7 @@ export const traceBatchQueueProcessor: Processor<} finally {try {// Drain accepted summaries even when the stream fails.- await pendingSummaries.at(-1);+ await Promise.all(pendingSummaries);} finally {activeReads--;recordTraceBatchActiveReads();