🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { beforeEach, describe, expect, it, vi } from "vitest";
import { UnrecoverableError } from "bullmq";
import {
topicSourceKey,
type TopicEmbeddingConfig,
type TopicSummary,
} from "@langfuse/shared/topics";
import type { TopicEmbeddingBatch } from "@langfuse/shared/topics/server";
const mocks = vi.hoisted(() => ({
write: vi.fn(),
staged: vi.fn(),
update: vi.fn(),
embed: vi.fn(),
pause: vi.fn(),
embeddingModel: "eu.cohere.embed-v4:0",
}));
vi.mock("@langfuse/shared/topics/server", () => ({
getTopicTextResponseMetadata: () => undefined,
pauseTopicsModels: mocks.pause,
writeTopicSummaries: mocks.write,
readStagedTopicSummary: mocks.staged,
updateStagedTopicSummary: mocks.update,
getTopicsModels: async () => ({
projectId: "project",
enabled: true,
embedding: {
slot: "embedding",
provider: "bedrock",
adapter: "bedrock",
model: mocks.embeddingModel,
connection: { secretKey: "encrypted" },
dimensions: 256,
},
}),
TOPIC_EMBEDDING_EXPIRED_ERROR:
"Topics staged results expired before processing completed. Start a new execution with stored-summary reuse to recover persisted results.",
}));
vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
const { isInternalEvalEnvironment } =
await importOriginal<typeof import("@langfuse/shared/src/server")>();
return {
...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),
isInternalEvalEnvironment,
logger: { error: vi.fn() },
recordIncrement: vi.fn(),
recordDistribution: vi.fn(),
};
});
vi.mock("./models", () => ({
embedTopicSummaries: mocks.embed,
}));
import { processTopicEmbeddingBatch } from "./processTopicEmbeddingBatch";
import { TopicsProviderUnavailable } from "./provider-error";
const embeddingConfig: TopicEmbeddingConfig = {
embeddingModel: "eu.cohere.embed-v4:0",
embeddingDimensions: 256,
};
const summary = (id = "summary"): TopicSummary => ({
projectId: "project",
facetId: "facet",
facetVersion: 1,
traceId: `trace-${id}`,
sessionId: null,
triggerType: "manual_poc",
unitStartTime: "2026-09-18T00:00:00.000Z",
environment: "production",
traceName: "Customer support",
processedAt: "2026-09-18T00:00:00.000Z",
state: "summarized",
summary: `A summary of ${id}.`,
embedding: [],
transcriptId: "poc",
transcriptVersion: "poc",
summaryModel: "gpt-4.1-nano",
embeddingModel: embeddingConfig.embeddingModel,
providedUsageDetails: { summary_input: 20, summary_output: 10 },
usageDetails: { summary_input: 20, summary_output: 10, total: 30 },
providedCostDetails: {},
costDetails: {
summary_input: 0.00006,
summary_output: 0.00004,
total: 0.0001,
},
metadata: {},
});
const batch = (...rows: TopicSummary[]): TopicEmbeddingBatch => ({
projectId: "project",
executionId: "execution",
batchId: "batch",
summaries: rows.map((row) => ({
facetId: row.facetId,
facetVersion: row.facetVersion,
traceId: row.traceId,
})),
});
const embeddingUsage = {
providedUsageDetails: { embedding_input: 8, total: 8 },
usageDetails: { embedding_input: 8, total: 8 },
providedCostDetails: {},
costDetails: { embedding_input: 0.000001, total: 0.000001 },
};
let staged: Map<string, TopicSummary>;
beforeEach(() => {
vi.resetAllMocks();
staged = new Map();
mocks.write.mockResolvedValue(undefined);
mocks.staged.mockImplementation(async (_batch, ref) => {
const row = staged.get(
topicSourceKey({ ..._batch, ...ref, sessionId: null }),
);
return row ? { summary: row, embeddingConfig } : null;
});
mocks.update.mockImplementation(async (_batch, ref, row) => {
const existing = staged.get(
topicSourceKey({ ..._batch, ...ref, sessionId: null }),
);
if (!existing) return null;
if (existing.state !== "summarized") return existing;
staged.set(topicSourceKey({ ..._batch, ...ref, sessionId: null }), row);
return row;
});
mocks.embed.mockImplementation(async (_model, summaries: string[]) => ({
...embeddingUsage,
embeddings: summaries.map(() => Array(256).fill(0.25)),
}));
});
describe("Topics embedding handoff", () => {
it.each(["summarized", "complete"] as const)(
"skips internal staged summaries in a mixed batch (%s)",
async (state) => {
const row: TopicSummary = {
...summary(),
state,
environment: "langfuse-topics",
embedding: state === "complete" ? Array(256).fill(0.25) : [],
};
const allowed = summary("allowed");
staged.set(topicSourceKey(row), row);
staged.set(topicSourceKey(allowed), allowed);
await processTopicEmbeddingBatch(batch(row, allowed));
expect(mocks.embed).toHaveBeenCalledExactlyOnceWith(
expect.anything(),
[allowed.summary],
256,
"project",
);
expect(mocks.update).toHaveBeenCalledTimes(1);
expect(mocks.write).toHaveBeenCalledExactlyOnceWith([
expect.objectContaining({
traceId: allowed.traceId,
state: "complete",
}),
]);
},
);
it("retries combined results from Redis after a failed ClickHouse write", async () => {
const applicable = summary();
const nonApplicable: TopicSummary = {
...summary("empty"),
state: "not_applicable",
summary: "",
};
for (const row of [applicable, nonApplicable])
staged.set(topicSourceKey(row), row);
mocks.write.mockRejectedValueOnce(new Error("ClickHouse unavailable"));
await expect(
processTopicEmbeddingBatch(batch(applicable, nonApplicable)),
).rejects.toThrow("ClickHouse unavailable");
expect(mocks.write.mock.calls[0][0]).toMatchObject([
{
traceId: applicable.traceId,
state: "complete",
providedUsageDetails: {
summary_input: 20,
summary_output: 10,
embedding_input: 8,
total: 38,
},
usageDetails: {
summary_input: 20,
summary_output: 10,
embedding_input: 8,
total: 38,
},
providedCostDetails: {},
costDetails: {
summary_input: 0.00006,
summary_output: 0.00004,
embedding_input: 0.000001,
total: 0.000101,
},
},
{
traceId: nonApplicable.traceId,
state: "not_applicable",
embedding: [],
},
]);
expect(staged.get(topicSourceKey(applicable))?.state).toBe("complete");
const completed = structuredClone(mocks.write.mock.calls[0][0]);
await processTopicEmbeddingBatch(batch(applicable, nonApplicable));
expect(mocks.embed).toHaveBeenCalledOnce();
expect(mocks.write).toHaveBeenCalledTimes(2);
expect(mocks.write.mock.calls[1][0]).toEqual(completed);
expect(staged.size).toBe(2);
});
it("persists the canonical result returned by Redis", async () => {
const row = summary();
staged.set(topicSourceKey(row), row);
const accepted: TopicSummary = {
...row,
state: "complete",
embedding: Array(256).fill(0.5),
processedAt: "2026-09-18T00:01:00.000Z",
};
mocks.update.mockResolvedValueOnce(accepted);
await processTopicEmbeddingBatch(batch(row));
expect(mocks.write).toHaveBeenCalledExactlyOnceWith([accepted]);
});
it("reports expired payloads as unrecoverable instead of regenerating summaries", async () => {
const error = await processTopicEmbeddingBatch(batch(summary())).catch(
(error: unknown) => error,
);
expect(error).toBeInstanceOf(UnrecoverableError);
expect(error).toMatchObject({
message: expect.stringContaining("Start a new execution"),
});
expect(mocks.embed).not.toHaveBeenCalled();
expect(mocks.write).not.toHaveBeenCalled();
});
it("embeds each trace's facets in one call and keeps finished traces when another fails", async () => {
const intent = summary("first");
const outcome: TopicSummary = {
...intent,
facetId: "outcome",
summary: "An outcome of first.",
};
const other = summary("second");
for (const row of [intent, outcome, other])
staged.set(topicSourceKey(row), row);
const failure = new TopicsProviderUnavailable(
"Topics provider call failed (HTTP 429).",
"rate_limit",
);
mocks.embed.mockImplementation(async (_model, summaries: string[]) => {
if (summaries.includes("A summary of second.")) throw failure;
return {
...embeddingUsage,
embeddings: summaries.map(() => Array(256).fill(0.25)),
};
});
await expect(
processTopicEmbeddingBatch(batch(intent, outcome, other)),
).rejects.toBe(failure);
expect(mocks.embed).toHaveBeenCalledTimes(2);
expect(mocks.embed.mock.calls.map((call) => call[1])).toContainEqual([
"A summary of first.",
"An outcome of first.",
]);
// The finished trace is stored with the call's usage recorded once.
expect(mocks.write.mock.calls[0][0]).toMatchObject([
{ facetId: "facet", state: "complete", usageDetails: { total: 38 } },
{ facetId: "outcome", state: "complete", usageDetails: { total: 30 } },
]);
expect(staged.get(topicSourceKey(other))?.state).toBe("summarized");
});
it("stops on a permanent failure even when another trace failed retryably", async () => {
const rows = [summary("first"), summary("second")];
for (const row of rows) staged.set(topicSourceKey(row), row);
mocks.embed.mockImplementation(async (_model, summaries: string[]) => {
throw summaries.includes("A summary of first.")
? new TopicsProviderUnavailable("Rate limited.", "rate_limit")
: new TopicsProviderUnavailable(
"Check worker credentials.",
"authentication",
);
});
await expect(processTopicEmbeddingBatch(batch(...rows))).rejects.toThrow(
UnrecoverableError,
);
expect(mocks.pause).toHaveBeenCalledOnce();
});
it("does not retry provider authentication failures", async () => {
const row = summary();
staged.set(topicSourceKey(row), row);
mocks.embed.mockRejectedValue(
new TopicsProviderUnavailable(
"Check worker credentials.",
"authentication",
),
);
await expect(processTopicEmbeddingBatch(batch(row))).rejects.toThrow(
UnrecoverableError,
);
expect(mocks.write).not.toHaveBeenCalled();
expect(staged.get(topicSourceKey(row))?.state).toBe("summarized");
expect(mocks.pause).toHaveBeenCalledExactlyOnceWith("project", {
blockReason: "LLM_CONNECTION_AUTH_INVALID",
blockMessage: expect.stringContaining("Check worker credentials."),
});
});
it("does not pause Topics when the saved embedding model changed", async () => {
const row = summary();
staged.set(topicSourceKey(row), row);
mocks.embeddingModel = "text-embedding-3-small";
await expect(processTopicEmbeddingBatch(batch(row))).rejects.toThrow(
UnrecoverableError,
);
mocks.embeddingModel = "eu.cohere.embed-v4:0";
expect(mocks.embed).not.toHaveBeenCalled();
expect(mocks.pause).not.toHaveBeenCalled();
});
it("fences an embedding when its Redis payload disappears during the call", async () => {
const row = summary();
staged.set(topicSourceKey(row), row);
mocks.embed.mockImplementation(async () => {
staged.delete(topicSourceKey(row));
return { ...embeddingUsage, embeddings: [Array(256).fill(0.25)] };
});
await expect(processTopicEmbeddingBatch(batch(row))).rejects.toThrow(
UnrecoverableError,
);
expect(mocks.write).not.toHaveBeenCalled();
expect(staged.size).toBe(0);
});
});