Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(topics): process traces concurrently and batch embeddings (#18698)

Lysander Kieselecd847d518c7
diff --git a/packages/shared/src/server/llm/llmText.ts b/packages/shared/src/server/llm/llmText.ts
index 55153911fa..e4e179eef6 100644
--- a/packages/shared/src/server/llm/llmText.ts
+++ b/packages/shared/src/server/llm/llmText.ts
@@ -2,7 +2,7 @@ import { type ZodType } from "zod";
import {
Output,
- embed,
+ embedMany,
generateText,
jsonSchema,
streamText,
@@ -11,7 +11,7 @@ import {
type GenerateTextOnAbortCallback,
type GenerateTextOnEndCallback,
type GenerateTextResult,
- type EmbedResult,
+ type EmbedManyResult,
type Experimental_DownloadFunction,
type JSONValue,
type LanguageModelCallOptions,
@@ -377,15 +377,16 @@ function openLLMConnection(connection: LLMConnection) {
return { apiKey, extraHeaders, createFetch };
}
-/** Embeds one value with a persisted connection; calls are not traced. */
+/** Embeds values with a persisted connection, batched per provider limit; calls are not traced. */
export async function embedLLMText(options: {
model: LLMModelRef;
connection: LLMConnection;
- value: string;
+ values: string[];
providerOptions?: ProviderOptions;
maxRetries?: number;
+ maxParallelCalls?: number;
abortSignal?: AbortSignal;
-}): Promise<EmbedResult> {
+}): Promise<EmbedManyResult> {
resolveAiSdkModelConfig({
model: options.model,
connectionConfig: options.connection.config,
@@ -405,11 +406,12 @@ export async function embedLLMText(options: {
credentialSource: "user",
createFetch,
});
- return embed({
+ return embedMany({
model: embeddingModel,
- value: options.value,
+ values: options.values,
providerOptions: options.providerOptions,
maxRetries: options.maxRetries,
+ maxParallelCalls: options.maxParallelCalls,
abortSignal: options.abortSignal,
});
}
diff --git a/packages/shared/src/server/topics/embeddings.ts b/packages/shared/src/server/topics/embeddings.ts
index 50b06471d8..fdc4475611 100644
--- a/packages/shared/src/server/topics/embeddings.ts
+++ b/packages/shared/src/server/topics/embeddings.ts
@@ -20,10 +20,10 @@ function bedrockEmbeddingOptions(
return { dimensions };
}
-/** Embeds one summary through the project's LLM connection. */
-export async function generateTopicEmbedding(params: {
+/** Embeds summaries through the project's LLM connection in as few requests as the provider allows. */
+export async function generateTopicEmbeddings(params: {
model: TopicsModel;
- summary: string;
+ summaries: string[];
dimensions: number;
}) {
const { embeddingDimensions } = topicEmbeddingConfigSchema.parse({
@@ -32,7 +32,7 @@ export async function generateTopicEmbedding(params: {
const result = await embedLLMText({
model: { adapter: params.model.adapter, id: params.model.model },
connection: params.model.connection,
- value: params.summary,
+ values: params.summaries,
// Discovery and later centroid assignment must share one vector space, so
// every provider is asked for clustering vectors of the configured size.
providerOptions:
@@ -55,7 +55,9 @@ export async function generateTopicEmbedding(params: {
},
},
maxRetries: 2,
+ // Single-text models (Bedrock Titan, Nova) split a batch into one request per text.
+ maxParallelCalls: 10,
abortSignal: AbortSignal.timeout(60_000),
});
- return { embedding: result.embedding, tokens: result.usage.tokens };
+ return { embeddings: result.embeddings, tokens: result.usage.tokens };
}
diff --git a/packages/shared/src/server/topics/model-config.test.ts b/packages/shared/src/server/topics/model-config.test.ts
index 51db82ba95..28edc0a24f 100644
--- a/packages/shared/src/server/topics/model-config.test.ts
+++ b/packages/shared/src/server/topics/model-config.test.ts
@@ -17,7 +17,7 @@ const mocks = vi.hoisted(() => ({
}));
vi.mock("./clickhouse", () => ({ hasTopicEmbeddings: mocks.hasEmbeddings }));
vi.mock("../llm/testModelCall", () => ({ testModelCall: mocks.testCall }));
-vi.mock("./embeddings", () => ({ generateTopicEmbedding: mocks.embed }));
+vi.mock("./embeddings", () => ({ generateTopicEmbeddings: mocks.embed }));
vi.mock("../../db", () => ({
prisma: {
llmApiKeys: { findMany: mocks.keys },
@@ -65,8 +65,8 @@ beforeEach(() => {
mocks.current.mockResolvedValue(null);
mocks.hasEmbeddings.mockResolvedValue(false);
mocks.testCall.mockResolvedValue(undefined);
- mocks.embed.mockImplementation(async ({ dimensions }) => ({
- embedding: Array(dimensions).fill(0.1),
+ mocks.embed.mockImplementation(async ({ summaries, dimensions }) => ({
+ embeddings: summaries.map(() => Array(dimensions).fill(0.1)),
}));
});
@@ -100,7 +100,9 @@ describe("prepareTopicsModelSettings", () => {
if (model === "gpt-5.6-terra")
throw new Error("Model not found (HTTP 404)");
});
- mocks.embed.mockResolvedValue({ embedding: Array(1536).fill(0.1) });
+ mocks.embed.mockResolvedValue({
+ embeddings: [Array(1536).fill(0.1), Array(1536).fill(0.1)],
+ });
await expect(saveTopicsModelSettings("project", settings)).rejects.toThrow(
"Embeddings (text-embedding-3-small): The model returned 1536 dimensions instead of 1024. Choose a size this model supports.; Topic clustering (gpt-5.6-terra): Model not found (HTTP 404)",
diff --git a/packages/shared/src/server/topics/model-config.ts b/packages/shared/src/server/topics/model-config.ts
index 5a9035220f..4313fada82 100644
--- a/packages/shared/src/server/topics/model-config.ts
+++ b/packages/shared/src/server/topics/model-config.ts
@@ -19,7 +19,7 @@ import {
type TopicsModelSlotName,
} from "../../topics";
import { hasTopicEmbeddings } from "./clickhouse";
-import { generateTopicEmbedding } from "./embeddings";
+import { generateTopicEmbeddings } from "./embeddings";
import { invalidateTopicsProcessingActive } from "./processing-gate";
import { z } from "zod";
@@ -337,14 +337,22 @@ async function testTopicsModelSlots(
errors[name] = "LLM connection not found in this project.";
return;
}
- const { embedding } = await generateTopicEmbedding({
+ // Two texts, because processing embeds a trace's facets in one call.
+ const { embeddings } = await generateTopicEmbeddings({
model: resolved,
- summary: "Topics model check",
+ summaries: ["Topics model check", "Topics batch check"],
dimensions: settings.embeddingDimensions,
});
- if (embedding.length !== settings.embeddingDimensions)
+ if (embeddings.length !== 2)
throw new Error(
- `The model returned ${embedding.length} dimensions instead of ${settings.embeddingDimensions}. Choose a size this model supports.`,
+ "The model must embed several texts in one request. Choose an embedding model or endpoint that supports batched input.",
+ );
+ const wrong = embeddings.find(
+ (vector) => vector.length !== settings.embeddingDimensions,
+ );
+ if (wrong)
+ throw new Error(
+ `The model returned ${wrong.length} dimensions instead of ${settings.embeddingDimensions}. Choose a size this model supports.`,
);
} catch (error) {
errors[name] =
diff --git a/packages/shared/src/server/topics/text.test.ts b/packages/shared/src/server/topics/text.test.ts
index 5c6db7e120..f6ed7ce1aa 100644
--- a/packages/shared/src/server/topics/text.test.ts
+++ b/packages/shared/src/server/topics/text.test.ts
@@ -4,7 +4,7 @@ import { encrypt } from "../../encryption";
import { env } from "../../env";
import { BEDROCK_USE_DEFAULT_CREDENTIALS } from "../../interfaces/customLLMProviderConfigSchemas";
import { LLMAdapter } from "../llm/types";
-import { generateTopicEmbedding } from "./embeddings";
+import { generateTopicEmbeddings } from "./embeddings";
import type { TopicsModel } from "./model-config";
import { generateTopicText } from "./text";
@@ -207,34 +207,39 @@ describe("generateTopicText", () => {
});
});
-describe("generateTopicEmbedding", () => {
- it("requests clustering vectors of the configured size from Cohere on Bedrock", async () => {
+describe("generateTopicEmbeddings", () => {
+ it("embeds every summary in one Cohere request on Bedrock", async () => {
const vector = Array.from({ length: 512 }, (_, index) => index + 1);
+ const other = vector.map((value) => -value);
const fetch = vi.fn<typeof globalThis.fetch>(async () =>
Response.json(
- { embeddings: { float: [vector] }, texts: ["Account access"] },
+ {
+ embeddings: { float: [vector, other] },
+ texts: ["Account access", "Refund request"],
+ },
{ headers: { "x-amzn-bedrock-input-token-count": "3" } },
),
);
vi.stubGlobal("fetch", fetch);
- const result = await generateTopicEmbedding({
+ const result = await generateTopicEmbeddings({
model: bedrockModel("embedding", "eu.cohere.embed-v4:0"),
- summary: "Account access",
+ summaries: ["Account access", "Refund request"],
dimensions: 512,
});
+ expect(fetch).toHaveBeenCalledTimes(1);
const request = new Request(...fetch.mock.calls[0]);
expect(decodeURIComponent(request.url)).toBe(
"https://bedrock-runtime.eu-central-1.amazonaws.com/model/eu.cohere.embed-v4:0/invoke",
);
expect(await request.json()).toEqual({
input_type: "clustering",
- texts: ["Account access"],
+ texts: ["Account access", "Refund request"],
truncate: "NONE",
output_dimension: 512,
});
- expect(result.embedding).toEqual(vector);
+ expect(result.embeddings).toEqual([vector, other]);
});
it("asks OpenAI for any configured size", async () => {
@@ -247,7 +252,7 @@ describe("generateTopicEmbedding", () => {
);
vi.stubGlobal("fetch", fetch);
- const result = await generateTopicEmbedding({
+ const result = await generateTopicEmbeddings({
model: {
slot: "embedding",
provider: "openai-topics",
@@ -255,7 +260,7 @@ describe("generateTopicEmbedding", () => {
model: "text-embedding-3-small",
connection: { secretKey: encrypt("sk-test") },
},
- summary: "Account access",
+ summaries: ["Account access"],
dimensions: 768,
});
@@ -266,7 +271,7 @@ describe("generateTopicEmbedding", () => {
input: ["Account access"],
dimensions: 768,
});
- expect(result.embedding).toHaveLength(768);
+ expect(result.embeddings[0]).toHaveLength(768);
});
});
@@ -343,15 +348,15 @@ describe("allowlisted Cloud Bedrock runtime credentials", () => {
),
);
vi.stubGlobal("fetch", fetch);
- const input = { summary: "Account access", dimensions: 512 };
+ const input = { summaries: ["Account access"], dimensions: 512 };
- await generateTopicEmbedding({
+ await generateTopicEmbeddings({
...input,
model: runtimeModel("embedding", "allowed"),
});
expect(fetch).toHaveBeenCalledTimes(1);
await expect(
- generateTopicEmbedding({
+ generateTopicEmbeddings({
...input,
model: runtimeModel("embedding", "other"),
}),
diff --git a/worker/AGENTS.md b/worker/AGENTS.md
index b78cd131eb..09901f5311 100644
--- a/worker/AGENTS.md
+++ b/worker/AGENTS.md
@@ -30,7 +30,7 @@
Transcript assembly is shared with web through `loadTopicTranscript`, returning
`Transcript | null`; serialize it for inference and skip inference on null.
Token counting and its WASM dependency stay in `src/features/topics/models.ts`.
- Model calls use `generateTopicText` and `generateTopicEmbedding` from
+ Model calls use `generateTopicText` and `generateTopicEmbeddings` from
`@langfuse/shared/topics/server` with the project's resolved `TopicsModels`
(summary, embedding, clustering LLM connections from `topic_configs`).
Pass models explicitly; never put them on execution or batch state, which is
diff --git a/worker/src/__tests__/traceBatching.test.ts b/worker/src/__tests__/traceBatching.test.ts
index 2e1cf38fd7..a229aa01d1 100644
--- a/worker/src/__tests__/traceBatching.test.ts
+++ b/worker/src/__tests__/traceBatching.test.ts
@@ -52,7 +52,7 @@ import {
} from "@langfuse/shared/topics/server";
import {
summarizeTopicTraceFacets,
- embedTopicSummary,
+ embedTopicSummaries,
} from "../features/topics/models";
vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
@@ -112,7 +112,7 @@ vi.mock("@langfuse/shared/topics/server", async (importOriginal) => ({
vi.mock("../features/topics/models", () => ({
summarizeTopicTraceFacets: vi.fn(),
- embedTopicSummary: vi.fn(),
+ embedTopicSummaries: vi.fn(),
}));
vi.mock("../features/evaluation/observationEval", async (importOriginal) => ({
@@ -1451,9 +1451,11 @@ describe("trace micro-batch scheduling with Redis", () => {
...usage,
}),
);
- vi.mocked(embedTopicSummary).mockImplementation(
- async (_model, _text, dimensions) => ({
- embedding: Array.from({ length: dimensions }, () => 0.1),
+ vi.mocked(embedTopicSummaries).mockImplementation(
+ async (_model, summaries, dimensions) => ({
+ embeddings: summaries.map(() =>
+ Array.from({ length: dimensions }, () => 0.1),
+ ),
...usage,
}),
);
diff --git a/worker/src/env.ts b/worker/src/env.ts
index 3e0fef80fb..d0122c2fbc 100644
--- a/worker/src/env.ts
+++ b/worker/src/env.ts
@@ -170,6 +170,12 @@ const EnvSchema = z.object({
.int()
.positive()
.default(100 * 1024 * 1024),
+ // Traces summarized and embedded at once per Topics job, in both trace-batch and manual processing.
+ LANGFUSE_TOPICS_TRACE_CONCURRENCY: z.coerce
+ .number()
+ .int()
+ .positive()
+ .default(20),
LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE: z.coerce
.number()
.int()
diff --git a/worker/src/features/topics/README.md b/worker/src/features/topics/README.md
index 32bbb19809..9bf39deff9 100644
--- a/worker/src/features/topics/README.md
+++ b/worker/src/features/topics/README.md
@@ -115,8 +115,12 @@ residency matters. Every provider is asked for clustering vectors of the
configured size (Cohere `clustering` input type with truncation disabled, Google
`CLUSTERING` task type, OpenAI `dimensions`). Default: 1,024
dimensions; any whole number from 64 to 4,096 is accepted, and a provider that
-returns a vector of another length fails the call. Calls embed one summary at a
-time, retaining per-summary Redis checkpointing.
+returns a vector of another length fails the call. Embeddings are batched per trace: each trace's
+applicable facets are embedded in one call, in trace-batch and in embedding jobs
+(which run up to `LANGFUSE_TOPICS_TRACE_CONCURRENCY` traces at once). The call's
+usage is recorded on the trace's first row; one failure fails that trace's call,
+and a retry embeds the traces that are left. Saving the models requires the
+embedding model to embed two texts in one call.
Embedding settings are independent of immutable facet versions; changing them
with stored-summary reuse enabled regenerates vectors without summarization inference. This PoC does
not use `aiEmbed` or change ingestion. It accepts traces already stored in v4
@@ -138,7 +142,8 @@ call's usage is recorded on the first facet's row. It does not load the trace
from ClickHouse again. Projects outside
`LANGFUSE_TOPICS_ENABLED_PROJECT_IDS` are skipped. A finished facet version is
skipped on retry. A failing trace does not stop the batch: every trace is
-processed and outcomes are counted once per job (`langfuse.topics.trace_outcomes`
+processed, up to `LANGFUSE_TOPICS_TRACE_CONCURRENCY` traces at once (default
+20; manual processing uses the same limit per queue job), and outcomes are counted once per job (`langfuse.topics.trace_outcomes`
by outcome and reason). Failed traces are not retried, because a retry would
read the shared batch from ClickHouse again. Only a failed batch read fails the
job. Summaries and embeddings use the same project models as manual
diff --git a/worker/src/features/topics/models.test.ts b/worker/src/features/topics/models.test.ts
index 11bff393d5..48fb49b73b 100644
--- a/worker/src/features/topics/models.test.ts
+++ b/worker/src/features/topics/models.test.ts
@@ -11,11 +11,11 @@ vi.mock("@langfuse/shared/src/server", () => ({
}));
vi.mock("@langfuse/shared/topics/server", () => ({
generateTopicText: (...args: unknown[]) => state.call(...args),
- generateTopicEmbedding: (...args: unknown[]) => state.embed(...args),
+ generateTopicEmbeddings: (...args: unknown[]) => state.embed(...args),
}));
import {
- embedTopicSummary,
+ embedTopicSummaries,
nameTopicGroup,
summarizeTopicTrace,
summarizeTopicTraceFacets,
@@ -342,9 +342,13 @@ describe("Topics naming boundary", () => {
);
it("records embedding input usage even when the returned vector is invalid", async () => {
- state.embed.mockResolvedValue({ embedding: [], tokens: 12 });
+ state.embed.mockResolvedValue({ embeddings: [[]], tokens: 12 });
await expect(
- embedTopicSummary(topicsModels().embedding, "An invoice request.", 256),
+ embedTopicSummaries(
+ topicsModels().embedding,
+ ["An invoice request."],
+ 256,
+ ),
).rejects.toMatchObject({
reason: "invalid_output",
});
@@ -361,12 +365,12 @@ describe("Topics naming boundary", () => {
"does not invent missing embedding usage: %s",
async (tokens) => {
state.embed.mockResolvedValue({
- embedding: Array(256).fill(0.25),
+ embeddings: [Array(256).fill(0.25)],
tokens,
});
- await embedTopicSummary(
+ await embedTopicSummaries(
topicsModels().embedding,
- "An invoice request.",
+ ["An invoice request."],
256,
);
expect(state.increment.mock.calls).toEqual([
diff --git a/worker/src/features/topics/models.ts b/worker/src/features/topics/models.ts
index 6026f13c50..7d520ea354 100644
--- a/worker/src/features/topics/models.ts
+++ b/worker/src/features/topics/models.ts
@@ -2,7 +2,7 @@ import { get_encoding } from "tiktoken";
import { z } from "zod";
import { logger } from "@langfuse/shared/src/server";
import {
- generateTopicEmbedding,
+ generateTopicEmbeddings,
generateTopicText,
type TopicPromptPart,
type TopicsModel,
@@ -348,13 +348,15 @@ export async function nameTopicGroup(
);
}
-export async function embedTopicSummary(
+/** Embeds summaries in one batched call; the usage covers every returned vector. */
+export async function embedTopicSummaries(
model: TopicsModel,
- summary: string,
+ summaries: string[],
dimensions: number,
-): Promise<TopicModelUsage & { embedding: number[] }> {
+): Promise<TopicModelUsage & { embeddings: number[][] }> {
if (
- !summary.trim() ||
+ summaries.length === 0 ||
+ summaries.some((summary) => !summary.trim()) ||
!topicEmbeddingConfigSchema.safeParse({ embeddingDimensions: dimensions })
.success
)
@@ -362,20 +364,26 @@ export async function embedTopicSummary(
"Topics embeddings require non-empty text and 64 to 4,096 dimensions.",
"invalid_input",
);
- const result = await generateTopicEmbedding({
+ const result = await generateTopicEmbeddings({
model,
- summary,
+ summaries,
dimensions,
}).catch((error: unknown) => {
throw topicProviderError(error, model);
});
recordTopicTokenUsage("embedding", { input: result.tokens });
// ClickHouse stores Float32; calibration and future classification must use those same vectors.
- const embedding = Array.from(new Float32Array(result.embedding));
+ const embeddings = result.embeddings.map((vector) =>
+ Array.from(new Float32Array(vector)),
+ );
if (
- embedding.length !== dimensions ||
- !embedding.every(Number.isFinite) ||
- !embedding.some((value) => value !== 0)
+ embeddings.length !== summaries.length ||
+ embeddings.some(
+ (embedding) =>
+ embedding.length !== dimensions ||
+ !embedding.every(Number.isFinite) ||
+ !embedding.some((value) => value !== 0),
+ )
)
throw new TopicsProviderUnavailable(
"Topics embedding provider returned an invalid vector.",
@@ -394,7 +402,7 @@ export async function embedTopicSummary(
});
}
return {
- embedding,
+ embeddings,
providedUsageDetails: usageDetails,
usageDetails,
providedCostDetails: {},
diff --git a/worker/src/features/topics/processTopicEmbeddingBatch.test.ts b/worker/src/features/topics/processTopicEmbeddingBatch.test.ts
index 6cd81853b8..edb5a29855 100644
--- a/worker/src/features/topics/processTopicEmbeddingBatch.test.ts
+++ b/worker/src/features/topics/processTopicEmbeddingBatch.test.ts
@@ -41,7 +41,7 @@ vi.mock("@langfuse/shared/src/server", () => ({
recordDistribution: vi.fn(),
}));
vi.mock("./models", () => ({
- embedTopicSummary: mocks.embed,
+ embedTopicSummaries: mocks.embed,
}));
import { processTopicEmbeddingBatch } from "./processTopicEmbeddingBatch";
@@ -89,8 +89,7 @@ const batch = (...rows: TopicSummary[]): TopicEmbeddingBatch => ({
traceId: row.traceId,
})),
});
-const embeddingResult = {
- embedding: Array(256).fill(0.25),
+const embeddingUsage = {
providedUsageDetails: { embedding_input: 8, total: 8 },
usageDetails: { embedding_input: 8, total: 8 },
providedCostDetails: {},
@@ -116,7 +115,10 @@ beforeEach(() => {
staged.set(topicSourceKey({ ..._batch, ...ref, sessionId: null }), row);
return row;
});
- mocks.embed.mockResolvedValue(embeddingResult);
+ mocks.embed.mockImplementation(async (_model, summaries: string[]) => ({
+ ...embeddingUsage,
+ embeddings: summaries.map(() => Array(256).fill(0.25)),
+ }));
});
describe("Topics embedding handoff", () => {
@@ -198,26 +200,58 @@ describe("Topics embedding handoff", () => {
expect(mocks.write).not.toHaveBeenCalled();
});
- it("persists completed rows before retrying a later provider failure", async () => {
- const rows = [summary("first"), summary("second")];
- for (const row of rows) staged.set(topicSourceKey(row), row);
- mocks.embed.mockResolvedValueOnce(embeddingResult);
+ it("embeds each trace's facets in one call and keeps finished traces when another fails", async () => {
+ const intent = summary("first");
+ const outcome: TopicSummary = {
+ ...intent,
+ facetId: "outcome",
+ summary: "An outcome of first.",
+ };
+ const other = summary("second");
+ for (const row of [intent, outcome, other])
+ staged.set(topicSourceKey(row), row);
const failure = new TopicsProviderUnavailable(
"Topics provider call failed (HTTP 429).",
"rate_limit",
);
- mocks.embed.mockRejectedValueOnce(failure);
- await expect(processTopicEmbeddingBatch(batch(...rows))).rejects.toBe(
- failure,
- );
+ mocks.embed.mockImplementation(async (_model, summaries: string[]) => {
+ if (summaries.includes("A summary of second.")) throw failure;
+ return {
+ ...embeddingUsage,
+ embeddings: summaries.map(() => Array(256).fill(0.25)),
+ };
+ });
+ await expect(
+ processTopicEmbeddingBatch(batch(intent, outcome, other)),
+ ).rejects.toBe(failure);
+ expect(mocks.embed).toHaveBeenCalledTimes(2);
+ expect(mocks.embed.mock.calls.map((call) => call[1])).toContainEqual([
+ "A summary of first.",
+ "An outcome of first.",
+ ]);
+ // The finished trace is stored with the call's usage recorded once.
expect(mocks.write.mock.calls[0][0]).toMatchObject([
- { traceId: "trace-first", state: "complete" },
+ { facetId: "facet", state: "complete", usageDetails: { total: 38 } },
+ { facetId: "outcome", state: "complete", usageDetails: { total: 30 } },
]);
- expect(staged.get(topicSourceKey(rows[0]))?.state).toBe("complete");
- expect(staged.get(topicSourceKey(rows[1]))?.state).toBe("summarized");
- await processTopicEmbeddingBatch(batch(...rows));
- expect(mocks.embed).toHaveBeenCalledTimes(3);
- expect(staged.get(topicSourceKey(rows[1]))?.state).toBe("complete");
+ expect(staged.get(topicSourceKey(other))?.state).toBe("summarized");
+ });
+
+ it("stops on a permanent failure even when another trace failed retryably", async () => {
+ const rows = [summary("first"), summary("second")];
+ for (const row of rows) staged.set(topicSourceKey(row), row);
+ mocks.embed.mockImplementation(async (_model, summaries: string[]) => {
+ throw summaries.includes("A summary of first.")
+ ? new TopicsProviderUnavailable("Rate limited.", "rate_limit")
+ : new TopicsProviderUnavailable(
+ "Check worker credentials.",
+ "authentication",
+ );
+ });
+ await expect(processTopicEmbeddingBatch(batch(...rows))).rejects.toThrow(
+ UnrecoverableError,
+ );
+ expect(mocks.pause).toHaveBeenCalledOnce();
});
it("does not retry provider authentication failures", async () => {
@@ -257,7 +291,7 @@ describe("Topics embedding handoff", () => {
staged.set(topicSourceKey(row), row);
mocks.embed.mockImplementation(async () => {
staged.delete(topicSourceKey(row));
- return embeddingResult;
+ return { ...embeddingUsage, embeddings: [Array(256).fill(0.25)] };
});
await expect(processTopicEmbeddingBatch(batch(row))).rejects.toThrow(
UnrecoverableError,
diff --git a/worker/src/features/topics/processTopicEmbeddingBatch.ts b/worker/src/features/topics/processTopicEmbeddingBatch.ts
index ee6ec65c69..1e6d7bbfd1 100644
--- a/worker/src/features/topics/processTopicEmbeddingBatch.ts
+++ b/worker/src/features/topics/processTopicEmbeddingBatch.ts
@@ -1,4 +1,5 @@
import { UnrecoverableError } from "bullmq";
+import pLimit from "p-limit";
import type { TopicSummary } from "@langfuse/shared/topics";
import {
getTopicsModels,
@@ -8,7 +9,8 @@ import {
writeTopicSummaries,
type TopicEmbeddingBatch,
} from "@langfuse/shared/topics/server";
-import { embedTopicSummary } from "./models";
+import { env } from "../../env";
+import { embedTopicSummaries } from "./models";
import { TopicMetrics } from "./metrics";
import { mergeTopicModelUsage } from "./summaryResult";
import {
@@ -23,7 +25,11 @@ export async function processTopicEmbeddingBatch(
batch: TopicEmbeddingBatch,
): Promise<void> {
const metrics = new TopicMetrics();
- const pending: TopicSummary[] = [];
+ // Batch order; rows still `summarized` are not persisted.
+ const rows: {
+ ref: TopicEmbeddingBatch["summaries"][number];
+ summary: TopicSummary;
+ }[] = [];
try {
const models = await getTopicsModels(batch.projectId);
if (!models)
@@ -31,6 +37,8 @@ export async function processTopicEmbeddingBatch(
"Choose Topics models for this project, then start a new Topics execution.",
"configuration",
);
+ // Every summary in a batch belongs to one execution's embedding configuration.
+ let dimensions = models.embedding.dimensions;
for (const ref of batch.summaries) {
const staged = await readStagedTopicSummary(batch, ref);
if (!staged) throw new UnrecoverableError(TOPIC_EMBEDDING_EXPIRED_ERROR);
@@ -39,54 +47,77 @@ export async function processTopicEmbeddingBatch(
"The project's embedding model changed after this batch was created. Start a new Topics execution.",
"configuration",
);
- let { summary } = staged;
- if (summary.state === "summarized") {
- const result = await metrics.measure("embedding", async () => {
- try {
- return await embedTopicSummary(
- models.embedding,
- summary.summary,
- staged.embeddingConfig.embeddingDimensions,
- );
- } catch (error) {
- throw error instanceof TopicsProviderUnavailable
- ? error
- : topicProviderError(error, models.embedding);
- }
- });
- summary = {
- ...summary,
- state: "complete",
- embedding: result.embedding,
- ...mergeTopicModelUsage(summary, result),
- processedAt: new Date().toISOString(),
- };
- metrics.result("embedding", "generated");
- // All overlapping attempts must persist the same accepted result.
- const accepted = await updateStagedTopicSummary(batch, ref, summary);
- if (!accepted)
- throw new UnrecoverableError(TOPIC_EMBEDDING_EXPIRED_ERROR);
- pending.push(accepted);
- } else {
- if (summary.state === "complete") metrics.result("embedding", "cached");
- pending.push(summary);
- }
+ dimensions = staged.embeddingConfig.embeddingDimensions;
+ if (staged.summary.state === "complete")
+ metrics.result("embedding", "cached");
+ rows.push({ ref, summary: staged.summary });
}
+ const unembedded = rows.filter(
+ ({ summary }) => summary.state === "summarized",
+ );
+ // One call per trace (at most one text per facet), so each call keeps its own
+ // deadline and finished traces stay staged when another trace fails.
+ const limit = pLimit(env.LANGFUSE_TOPICS_TRACE_CONCURRENCY);
+ const results = await Promise.allSettled(
+ [...Map.groupBy(unembedded, ({ ref }) => ref.traceId).values()].map(
+ (traceRows) =>
+ limit(async () => {
+ const result = await metrics.measure("embedding", async () => {
+ try {
+ return await embedTopicSummaries(
+ models.embedding,
+ traceRows.map(({ summary }) => summary.summary),
+ dimensions,
+ );
+ } catch (error) {
+ throw error instanceof TopicsProviderUnavailable
+ ? error
+ : topicProviderError(error, models.embedding);
+ }
+ });
+ const processedAt = new Date().toISOString();
+ for (const [index, row] of traceRows.entries()) {
+ const embedded: TopicSummary = {
+ ...row.summary,
+ state: "complete",
+ embedding: result.embeddings[index],
+ // The call's usage is recorded once, on the trace's first summary.
+ ...(index === 0
+ ? mergeTopicModelUsage(row.summary, result)
+ : {}),
+ processedAt,
+ };
+ metrics.result("embedding", "generated");
+ // All overlapping attempts must persist the same accepted result.
+ const accepted = await updateStagedTopicSummary(
+ batch,
+ row.ref,
+ embedded,
+ );
+ if (!accepted)
+ throw new UnrecoverableError(TOPIC_EMBEDDING_EXPIRED_ERROR);
+ row.summary = accepted;
+ }
+ }),
+ ),
+ );
+ const failures = results.flatMap((result) =>
+ result.status === "rejected" ? [result.reason] : [],
+ );
+ // A permanent failure must stop the job and pause Topics even if another
+ // trace failed with a retryable error first.
+ if (failures.length > 0)
+ throw failures.find(isPermanentFailure) ?? failures[0];
} catch (error) {
if (isRejectedConnection(error))
await pauseForRejectedConnection(batch.projectId, error);
- if (
- error instanceof TopicsProviderUnavailable &&
- [
- "authentication",
- "configuration",
- "invalid_input",
- "invalid_output",
- ].includes(error.reason)
- )
+ if (error instanceof TopicsProviderUnavailable && isPermanentFailure(error))
throw new UnrecoverableError(error.message);
throw error;
} finally {
+ const pending = rows
+ .map(({ summary }) => summary)
+ .filter((summary) => summary.state !== "summarized");
if (pending.length)
await metrics.measure(
"storage",
@@ -97,3 +128,16 @@ export async function processTopicEmbeddingBatch(
// Retrying a partial insert reuses these vectors and their processing time.
}
}
+
+function isPermanentFailure(error: unknown): boolean {
+ return (
+ error instanceof UnrecoverableError ||
+ (error instanceof TopicsProviderUnavailable &&
+ [
+ "authentication",
+ "configuration",
+ "invalid_input",
+ "invalid_output",
+ ].includes(error.reason))
+ );
+}
diff --git a/worker/src/features/topics/processTopicsExecution.test.ts b/worker/src/features/topics/processTopicsExecution.test.ts
index 670d6908dd..785641ff97 100644
--- a/worker/src/features/topics/processTopicsExecution.test.ts
+++ b/worker/src/features/topics/processTopicsExecution.test.ts
@@ -315,8 +315,17 @@ vi.mock("@langfuse/shared/topics/server", async (importOriginal) => {
vi.mock("./models", () => ({
summarizeTopicTrace: (_models: unknown, ...args: unknown[]) =>
state.summarize(...args),
- embedTopicSummary: (_model: unknown, ...args: unknown[]) =>
- state.embed(...args),
+ // `state.embed` sees each summary; one failure fails the whole batched call.
+ embedTopicSummaries: async (
+ _model: unknown,
+ summaries: string[],
+ dimensions: number,
+ ) => {
+ const results = [];
+ for (const summary of summaries)
+ results.push(await state.embed(summary, dimensions));
+ return { ...results[0], embeddings: results.map((r) => r.embedding) };
+ },
nameTopicGroup: (_models: unknown, ...args: unknown[]) => state.name(...args),
}));
vi.mock("./numeric", async (importOriginal) => ({
@@ -816,7 +825,8 @@ describe("Topics execution", () => {
),
);
await processSelection("partial", 3);
- expect(state.batches.get("partial")?.summaries).toHaveLength(1);
+ // Traces run concurrently, so the third finishes while the second fails.
+ expect(state.batches.get("partial")?.summaries).toHaveLength(2);
expect(state.executions.get("partial")?.status).toBe("failed");
const accepted = [...state.staged.values()][0].summary;
accepted.transcriptVersion = "older-transcript";
diff --git a/worker/src/features/topics/processTopicsExecution.ts b/worker/src/features/topics/processTopicsExecution.ts
index c5ca4da92d..1c0247d507 100644
--- a/worker/src/features/topics/processTopicsExecution.ts
+++ b/worker/src/features/topics/processTopicsExecution.ts
@@ -39,6 +39,8 @@ import {
sameTopicGeometry,
topicTimeRangeSchema,
} from "@langfuse/shared/topics";
+import pLimit from "p-limit";
+import { env } from "../../env";
import { summarizeTopicTrace, nameTopicGroup } from "./models";
import {
isRejectedConnection,
@@ -820,79 +822,102 @@ async function processTraces(
)
throw new TopicEmbeddingFailure(TOPIC_EMBEDDING_EXPIRED_ERROR);
}
- for (const traceId of execution.input.traceIds) {
- let input: ReturnType<typeof loadTopicTranscript> | undefined;
- const getTranscript = () => {
- if (!input)
- input = metrics.measure(
- "transcript",
- () =>
- loadTopicTranscript({ projectId: execution.projectId, traceId }),
- "trace_load",
- );
- return input;
- };
- for (const { facet, progress } of facets) {
- if (
- state.summaries.some(
- (ref) =>
- ref.facetId === facet.facetId &&
- ref.facetVersion === facet.version &&
- ref.traceId === traceId,
- ) ||
- state.failedTraceIds.some(
- (entry) =>
- entry.facetId === facet.facetId &&
- entry.facetVersion === facet.version &&
- entry.traceIds.includes(traceId),
- )
- )
- continue;
- try {
- await summarizeTrace(
- metrics,
- models,
- execution,
- facet,
- traceId,
- cached.get(facet)!,
- getTranscript,
- );
- state.summaries.push({
- facetId: facet.facetId,
- facetVersion: facet.version,
- traceId,
- });
- await save("summarizing");
- } catch (error) {
- metrics.error("summary", error);
- if (isExecutionFailure(error)) throw error;
- let failed = state.failedTraceIds.find(
- (entry) =>
- entry.facetId === facet.facetId &&
- entry.facetVersion === facet.version,
- );
- if (!failed) {
- failed = {
- facetId: facet.facetId,
- facetVersion: facet.version,
- traceIds: [],
- };
- state.failedTraceIds.push(failed);
- }
- failed.traceIds.push(traceId);
- progress.counts.failed = failed.traceIds.length;
- const message = errorMessage(error);
- if (
- !execution.traceErrors.some(
- (item) => item.traceId === traceId && item.error === message,
+ // Traces run concurrently; a trace's facets share its transcript and run in order.
+ const limit = pLimit(env.LANGFUSE_TOPICS_TRACE_CONCURRENCY);
+ let saving = Promise.resolve();
+ // Saves write the whole state, so they must not land out of order.
+ const saveInOrder = () => {
+ const saved = saving.then(() => save("summarizing"));
+ saving = saved.catch(() => {});
+ return saved;
+ };
+ let aborted = false;
+ const results = await Promise.allSettled(
+ execution.input.traceIds.map((traceId) =>
+ limit(async () => {
+ if (aborted) return;
+ let input: ReturnType<typeof loadTopicTranscript> | undefined;
+ const getTranscript = () => {
+ if (!input)
+ input = metrics.measure(
+ "transcript",
+ () =>
+ loadTopicTranscript({
+ projectId: execution.projectId,
+ traceId,
+ }),
+ "trace_load",
+ );
+ return input;
+ };
+ for (const { facet, progress } of facets) {
+ if (
+ state.summaries.some(
+ (ref) =>
+ ref.facetId === facet.facetId &&
+ ref.facetVersion === facet.version &&
+ ref.traceId === traceId,
+ ) ||
+ state.failedTraceIds.some(
+ (entry) =>
+ entry.facetId === facet.facetId &&
+ entry.facetVersion === facet.version &&
+ entry.traceIds.includes(traceId),
+ )
)
- )
- execution.traceErrors.push({ traceId, error: message });
- await save("summarizing");
- }
- }
- }
+ continue;
+ try {
+ await summarizeTrace(
+ metrics,
+ models,
+ execution,
+ facet,
+ traceId,
+ cached.get(facet)!,
+ getTranscript,
+ );
+ state.summaries.push({
+ facetId: facet.facetId,
+ facetVersion: facet.version,
+ traceId,
+ });
+ await saveInOrder();
+ } catch (error) {
+ metrics.error("summary", error);
+ if (isExecutionFailure(error)) {
+ aborted = true;
+ throw error;
+ }
+ let failed = state.failedTraceIds.find(
+ (entry) =>
+ entry.facetId === facet.facetId &&
+ entry.facetVersion === facet.version,
+ );
+ if (!failed) {
+ failed = {
+ facetId: facet.facetId,
+ facetVersion: facet.version,
+ traceIds: [],
+ };
+ state.failedTraceIds.push(failed);
+ }
+ failed.traceIds.push(traceId);
+ progress.counts.failed = failed.traceIds.length;
+ const message = errorMessage(error);
+ if (
+ !execution.traceErrors.some(
+ (item) => item.traceId === traceId && item.error === message,
+ )
+ )
+ execution.traceErrors.push({ traceId, error: message });
+ await saveInOrder();
+ }
+ }
+ }),
+ ),
+ );
+ const failure = results.find((result) => result.status === "rejected");
+ if (failure) throw failure.reason;
state.summarized = true;
await save("embedding");
}
diff --git a/worker/src/features/topics/summarizeAssembledTrace.test.ts b/worker/src/features/topics/summarizeAssembledTrace.test.ts
index 40c6f5804e..00cc8d47c0 100644
--- a/worker/src/features/topics/summarizeAssembledTrace.test.ts
+++ b/worker/src/features/topics/summarizeAssembledTrace.test.ts
@@ -34,7 +34,7 @@ vi.mock("@langfuse/shared/src/server", () => ({
}));
vi.mock("./models", () => ({
summarizeTopicTraceFacets: (...args: unknown[]) => state.summarize(...args),
- embedTopicSummary: (...args: unknown[]) => state.embed(...args),
+ embedTopicSummaries: (...args: unknown[]) => state.embed(...args),
}));
import { summarizeAssembledTrace } from "./summarizeAssembledTrace";
@@ -123,13 +123,13 @@ beforeEach(() => {
},
...usage,
});
- state.embed.mockResolvedValue({
- embedding: [0.25],
+ state.embed.mockImplementation(async (_model, summaries: string[]) => ({
+ embeddings: summaries.map(() => [0.25]),
providedUsageDetails: { embedding_input: 4 },
usageDetails: { embedding_input: 4, total: 4 },
providedCostDetails: {},
costDetails: { embedding_input: 0.00000048, total: 0.00000048 },
- });
+ }));
});
describe("summarizeAssembledTrace", () => {
@@ -221,18 +221,25 @@ describe("summarizeAssembledTrace", () => {
expect(state.stored).toHaveBeenCalledTimes(3);
});
- it("summarizes all pending facets in one call and records its usage once", async () => {
+ it("summarizes and embeds all pending facets in one call each and records their usage once", async () => {
const issues: TopicFacet = {
...facet,
id: "facet-2",
name: "Issues",
versions: [{ ...facet.versions[0], facetId: "facet-2", version: 1 }],
};
- state.facets.mockReturnValue([facet, issues]);
+ const outcome: TopicFacet = {
+ ...facet,
+ id: "facet-3",
+ name: "Outcome",
+ versions: [{ ...facet.versions[0], facetId: "facet-3", version: 1 }],
+ };
+ state.facets.mockReturnValue([facet, issues, outcome]);
state.summarize.mockResolvedValue({
output: {
intent_1: { summary: "Export monthly sales.", status: "applicable" },
issues_2: { summary: "", status: "not_applicable" },
+ outcome_3: { summary: "Export completed.", status: "applicable" },
},
...usage,
});
diff --git a/worker/src/features/topics/summarizeAssembledTrace.ts b/worker/src/features/topics/summarizeAssembledTrace.ts
index 79b7646957..035485f898 100644
--- a/worker/src/features/topics/summarizeAssembledTrace.ts
+++ b/worker/src/features/topics/summarizeAssembledTrace.ts
@@ -17,7 +17,7 @@ import {
} from "@langfuse/shared/src/server";
import { prepareAssembledTopicTranscript } from "./assembledTranscript";
import { TopicMetrics } from "./metrics";
-import { embedTopicSummary, summarizeTopicTraceFacets } from "./models";
+import { embedTopicSummaries, summarizeTopicTraceFacets } from "./models";
import { TopicsProviderUnavailable } from "./provider-error";
import {
mergeTopicModelUsage,
@@ -178,25 +178,53 @@ async function summarizeEnabledTrace(
providedCostDetails: result.providedCostDetails,
costDetails: result.costDetails,
};
- rows = [];
- // A failed facet (invalid output, embedding error) must not discard the others.
- for (const [index, { key }] of keyed.entries()) {
- if (invalidFacets.has(key)) continue;
+ const accepted = keyed.flatMap(({ key }, index) =>
+ invalidFacets.has(key)
+ ? []
+ : [{ base: bases[index], output: outputs[key] }],
+ );
+ const applicable = accepted
+ .filter(({ output }) => output.status === "applicable")
+ .map(({ output }) => output.summary.trim());
+ let embedded: Awaited<ReturnType<typeof embedTopicSummaries>> | undefined;
+ if (applicable.length > 0) {
try {
- rows.push(
- await finalizeSummary(
- bases[index],
- outputs[key],
- // One call serves every facet of the trace; record its usage once.
- rows.length ? NO_USAGE : usage,
- models,
- metrics,
+ embedded = await metrics.measure("embedding", () =>
+ embedTopicSummaries(
+ models.embedding,
+ applicable,
+ models.embedding.dimensions,
),
);
} catch (error) {
+ // Non-applicable facets are still persisted when the embedding fails.
if (failure === undefined) failure = error;
}
}
+ // One call summarizes and one embeds every facet of the trace; record their usage once.
+ const traceUsage = embedded ? mergeTopicModelUsage(usage, embedded) : usage;
+ rows = [];
+ let embeddingIndex = 0;
+ for (const { base, output } of accepted) {
+ const rowUsage = rows.length > 0 ? NO_USAGE : traceUsage;
+ if (output.status !== "applicable")
+ rows.push({
+ ...base,
+ state:
+ output.status === "not_applicable"
+ ? "not_applicable"
+ : "insufficient_input",
+ ...rowUsage,
+ });
+ else if (embedded)
+ rows.push({
+ ...base,
+ state: "complete",
+ summary: output.summary.trim(),
+ embedding: embedded.embeddings[embeddingIndex++],
+ ...rowUsage,
+ });
+ }
}
if (rows.length)
await metrics.measure("storage", () => writeTopicSummaries(rows));
@@ -240,32 +268,3 @@ function baseSummary(
metadata: { input: "assembled-transcript" },
};
}
-
-async function finalizeSummary(
- base: TopicSummary,
- output: FacetOutput,
- usage: TopicModelUsage,
- models: TopicsModels,
- metrics: TopicMetrics,
-): Promise<TopicSummary> {
- if (output.status !== "applicable")
- return {
- ...base,
- state:
- output.status === "not_applicable"
- ? "not_applicable"
- : "insufficient_input",
- ...usage,
- };
- const summary = output.summary.trim();
- const embedded = await metrics.measure("embedding", () =>
- embedTopicSummary(models.embedding, summary, models.embedding.dimensions),
- );
- return {
- ...base,
- state: "complete",
- summary,
- embedding: embedded.embedding,
- ...mergeTopicModelUsage(usage, embedded),
- };
-}
diff --git a/worker/src/features/traceBatching/README.md b/worker/src/features/traceBatching/README.md
index e414f1421c..c3f276daa3 100644
--- a/worker/src/features/traceBatching/README.md
+++ b/worker/src/features/traceBatching/README.md
@@ -50,6 +50,7 @@ templates. The local dev template sets the idle time below.
| `LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE` | unset | Optional positive `max_block_size` hint; unset inherits the server profile. |
| `LANGFUSE_TRACE_BATCH_EXPERIMENT_ID` | unset | Optional 1–64 character attempt/query label: letters, digits, `.`, `_`, `-`; first character alphanumeric. |
| `LANGFUSE_TRACE_BATCH_CONCURRENCY` | `2` | Concurrent batch jobs per worker process. |
+| `LANGFUSE_TOPICS_TRACE_CONCURRENCY` | `20` | Topics traces summarized and embedded at once per batch job (positive integer). Tokenization stays one trace at a time. |
| `LANGFUSE_TRACE_BATCH_IDLE_MS` | `600000` | Default idle time before a trace is ready. Unset and `DEV` uses `120000` (2 minutes); every other region uses 10 minutes. A project's `tracing_readiness_settings.trace_idle_ms` (60 seconds to 60 minutes) overrides it, capped at the pending TTL minus the dispatch interval. The project value is cached in Redis for 5 minutes; saving replaces it, and a failed read uses the default without caching. |
| `LANGFUSE_TRACE_BATCH_PENDING_TTL_MS` | `7200000` | Recovery grace after a trace becomes due (2 hours). Pending keys expire no earlier than the latest required due-plus-grace deadline. |
| `LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS` | `86400000` | Historical event-start bounds and size estimates retained after activity or dispatch (24 hours; at most 48 hours). Internal and cloud-gated. |
diff --git a/worker/src/queues/__tests__/traceBatchQueue.test.ts b/worker/src/queues/__tests__/traceBatchQueue.test.ts
index 014a143efd..0134d2a231 100644
--- a/worker/src/queues/__tests__/traceBatchQueue.test.ts
+++ b/worker/src/queues/__tests__/traceBatchQueue.test.ts
@@ -594,7 +594,7 @@ describe("trace batch queue", () => {
}
},
);
- it("resumes streaming below the payload cap while a later Topics summary is pending", async () => {
+ it("summarizes traces concurrently and resumes streaming when any summary brings payloads below the cap", async () => {
const originalCap = env.LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES;
env.LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES = 1_000;
const payload = "x".repeat(600);
@@ -666,15 +666,16 @@ describe("trace batch queue", () => {
);
expect(reachedEnd).toBe(false);
await vi.waitFor(() =>
- expect(summarizeAssembledTrace).toHaveBeenCalledTimes(1),
+ expect(summarizeAssembledTrace).toHaveBeenCalledTimes(2),
);
- resolveFirst();
+ // The later trace finishing first releases its payload.
+ resolveSecond();
await vi.waitFor(() => expect(reachedEnd).toBe(true));
await vi.waitFor(() =>
- expect(summarizeAssembledTrace).toHaveBeenCalledTimes(2),
+ expect(summarizeAssembledTrace).toHaveBeenCalledTimes(3),
);
expect(settled).toBe(false);
- resolveSecond();
+ resolveFirst();
await expect(processing).resolves.toEqual(
expect.objectContaining({ traceCount: 3 }),
);
diff --git a/worker/src/queues/traceBatchQueue.ts b/worker/src/queues/traceBatchQueue.ts
index 8adea51fdd..b94cdee21b 100644
--- a/worker/src/queues/traceBatchQueue.ts
+++ b/worker/src/queues/traceBatchQueue.ts
@@ -1,4 +1,5 @@
import { type Processor } from "bullmq";
+import pLimit from "p-limit";
import { randomUUID } from "node:crypto";
import { type Observation } from "@langfuse/shared";
import {
@@ -12,6 +13,7 @@ import {
TraceBatchEventSchema,
type QueueName,
type TQueueJobTypes,
+ type Transcript,
} from "@langfuse/shared/src/server";
import {
getEnabledTopicsModels,
@@ -96,6 +98,7 @@ async function loadProjectRule(projectId: string) {
async function summarizeTraceBatch(
observations: Observation[],
+ transcript: Transcript | null,
topics: TopicsBatchProjects,
jobEnqueuedAt: Date,
): Promise<TraceOutcome> {
@@ -114,46 +117,41 @@ async function summarizeTraceBatch(
const traceTimestamp = new Date(
Math.min(...observations.map((row) => row.startTime.getTime())),
).toISOString();
- let outcome: TraceOutcome = { outcome: "disabled" };
- await recordTraceBatchTranscript(observations, async (transcript) => {
- try {
- const project = await topics.get(first.projectId);
- if (
- !project ||
- !shouldSampleEvaluation({
- samplingValue: getDeterministicSamplingValue(traceId),
- samplingRate: project.sampling,
- })
- )
- return;
- outcome = {
- outcome: await summarizeAssembledTrace({
- projectId: first.projectId,
- traceId,
- traceTimestamp,
- jobEnqueuedAt,
- environment: first.environment,
- traceName: first.name ?? "",
- transcript,
- models: project.models,
- facets: project.facets,
- }),
- };
- } catch (error) {
- // One trace's failure must not fail or re-read the shared batch.
- const reason =
- error instanceof TopicsProviderUnavailable ? error.reason : "other";
- if (isRejectedConnection(error))
- await topics.pause(first.projectId, error);
- outcome = { outcome: "failed", reason };
- logger.warn("Topics summary failed for trace", {
+ try {
+ const project = await topics.get(first.projectId);
+ if (
+ !project ||
+ !shouldSampleEvaluation({
+ samplingValue: getDeterministicSamplingValue(traceId),
+ samplingRate: project.sampling,
+ })
+ )
+ return { outcome: "disabled" };
+ return {
+ outcome: await summarizeAssembledTrace({
projectId: first.projectId,
traceId,
- reason,
- });
- }
- });
- return outcome;
+ traceTimestamp,
+ jobEnqueuedAt,
+ environment: first.environment,
+ traceName: first.name ?? "",
+ transcript,
+ models: project.models,
+ facets: project.facets,
+ }),
+ };
+ } catch (error) {
+ // One trace's failure must not fail or re-read the shared batch.
+ const reason =
+ error instanceof TopicsProviderUnavailable ? error.reason : "other";
+ if (isRejectedConnection(error)) await topics.pause(first.projectId, error);
+ logger.warn("Topics summary failed for trace", {
+ projectId: first.projectId,
+ traceId,
+ reason,
+ });
+ return { outcome: "failed", reason };
+ }
}
/** Failed traces are counted, not retried. */
@@ -299,27 +297,47 @@ export const traceBatchQueueProcessor: Processor<
// ClickHouse sockets time out when idle, so the reader never waits on
// transcript or Topics work unless queued payloads exceed the cap.
- // Traces are still summarized one at a time per batch to keep at most
- // one pending request in the tokenizer pool shared with ingestion.
- const pendingSummaries: Promise<void>[] = [];
+ // Transcripts are tokenized one trace at a time to keep at most one pending
+ // request in the tokenizer pool shared with ingestion; Topics model calls
+ // then run for up to LANGFUSE_TOPICS_TRACE_CONCURRENCY traces at once.
+ const pendingSummaries = new Set<Promise<void>>();
+ const topicsLimit = pLimit(env.LANGFUSE_TOPICS_TRACE_CONCURRENCY);
+ let transcribing: Promise<unknown> = Promise.resolve();
let pendingSummaryBytes = 0;
let summaryFailed = false;
let summaryError: unknown;
const enqueueSummary = (observations: Observation[], bytes: number) => {
pendingSummaryBytes += bytes;
const enqueuedAt = performance.now();
- const summary = (pendingSummaries.at(-1) ?? Promise.resolve())
- .then(async () => {
- // Let socket reads run between traces' synchronous assembly.
- await new Promise((resolve) => setImmediate(resolve));
- recordDistribution(
- "langfuse.trace_batch.summary_queue_wait_ms",
- performance.now() - enqueuedAt,
- );
- if (!summaryFailed)
- topicsOutcomes.push(
- await summarizeTraceBatch(observations, topics, event.timestamp),
- );
+ const transcribed = transcribing.then(async () => {
+ // Let socket reads run between traces' synchronous assembly.
+ await new Promise((resolve) => setImmediate(resolve));
+ recordDistribution(
+ "langfuse.trace_batch.summary_queue_wait_ms",
+ performance.now() - enqueuedAt,
+ );
+ if (summaryFailed) return null;
+ let transcript: Transcript | null = null;
+ await recordTraceBatchTranscript(observations, async (assembled) => {
+ transcript = assembled;
+ });
+ return { transcript };
+ });
+ transcribing = transcribed.catch(() => {});
+ const summary: Promise<void> = transcribed
+ .then(async (assembled) => {
+ if (!assembled) return;
+ await topicsLimit(async () => {
+ if (!summaryFailed)
+ topicsOutcomes.push(
+ await summarizeTraceBatch(
+ observations,
+ assembled.transcript,
+ topics,
+ event.timestamp,
+ ),
+ );
+ });
})
.catch((error: unknown) => {
if (!summaryFailed) summaryError = error;
@@ -327,9 +345,9 @@ export const traceBatchQueueProcessor: Processor<
})
.finally(() => {
pendingSummaryBytes -= bytes;
- pendingSummaries.shift();
+ pendingSummaries.delete(summary);
});
- pendingSummaries.push(summary);
+ pendingSummaries.add(summary);
};
activeReads++;
try {
@@ -371,7 +389,7 @@ export const traceBatchQueueProcessor: Processor<
pendingSummaryBytes >
env.LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES
)
- await pendingSummaries[0];
+ await Promise.race(pendingSummaries);
}
if (summaryFailed) break;
}
@@ -398,7 +416,7 @@ export const traceBatchQueueProcessor: Processor<
} finally {
try {
// Drain accepted summaries even when the stream fails.
- await pendingSummaries.at(-1);
+ await Promise.all(pendingSummaries);
} finally {
activeReads--;
recordTraceBatchActiveReads();