🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
/* eslint-disable @repo/no-exotic-operators */
import { randomBytes, randomUUID } from "node:crypto";
import { Queue, QueueEvents, Worker } from "bullmq";
import {
afterAll,
afterEach,
beforeEach,
describe,
expect,
it,
vi,
} from "vitest";
import {
createNewRedisInstance,
getQueuePrefix,
getS3EventStorageClient,
getTraceBatchEventStream,
QueueJobs,
QueueName,
recordDistribution,
recordGauge,
recordIncrement,
redis,
TraceBatchEventSchema,
TraceBatchQueue,
type TQueueJobTypes,
} from "@langfuse/shared/src/server";
import * as shared from "@langfuse/shared/src/server";
import { prisma } from "@langfuse/shared/src/db";
import { env } from "../env";
import { TraceBatchMetricsRunner } from "../features/traceBatching/TraceBatchMetricsRunner";
import {
getTraceBatchFirstKeys,
getTraceBatchReadinessKeys,
getTraceBatchShard,
} from "../features/traceBatching/traceBatchReadiness";
import {
trackTraceBatchActivity,
TraceBatchDispatcher,
} from "../features/traceBatching/traceBatching";
import {
prepareLocalityPartials,
selectTraceBatches,
type PendingTrace,
} from "../features/traceBatching/traceBatchSelection";
import { otelIngestionQueueProcessorBuilder } from "../queues/otelIngestionQueue";
import { traceBatchQueueProcessor } from "../queues/traceBatchQueue";
import { ClickhouseWriter } from "../services/ClickhouseWriter";
import {
ensureDefaultTopicFacets,
isTopicsProcessingActive,
listTopicSummaries,
type TopicsModels,
} from "@langfuse/shared/topics/server";
import {
summarizeTopicTraceFacets,
embedTopicSummaries,
} from "../features/topics/models";
vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
const original =
await importOriginal<typeof import("@langfuse/shared/src/server")>();
return {
...original,
redis: original.createNewRedisInstance({
keyPrefix: `trace-batch-test-${randomUUID()}:`,
}),
recordDistribution: vi.fn(),
recordGauge: vi.fn(),
recordIncrement: vi.fn(),
getS3EventStorageClient: vi.fn(),
getTracingReadinessSettings: vi.fn(original.getTracingReadinessSettings),
};
});
const topicsProjects = vi.hoisted(() => new Set<string>());
vi.mock("@langfuse/shared/topics/server", async (importOriginal) => ({
...(await importOriginal<typeof import("@langfuse/shared/topics/server")>()),
isTopicsProjectEnabled: (projectId: string) => topicsProjects.has(projectId),
isTopicsProcessingActive: vi.fn(async (projectId: string) =>
topicsProjects.has(projectId),
),
getEnabledTopicsModels: async (projectIds: string[]) =>
new Map(
projectIds
.filter((id) => topicsProjects.has(id))
.map((projectId) => {
const slot = (
name: "summary" | "embedding" | "clustering",
model: string,
) => ({
slot: name,
provider: "bedrock",
adapter: "bedrock",
model,
connection: { secretKey: "encrypted" },
});
return [
projectId,
{
projectId,
enabled: true,
summary: slot("summary", "us.openai.gpt-5.6-luna"),
embedding: {
...slot("embedding", "eu.cohere.embed-v4:0"),
dimensions: 1024,
},
clustering: slot("clustering", "us.openai.gpt-5.6-terra"),
} as TopicsModels,
] as const;
}),
),
}));
vi.mock("../features/topics/models", () => ({
summarizeTopicTraceFacets: vi.fn(),
embedTopicSummaries: vi.fn(),
}));
vi.mock(
"../features/evaluation/observationEval/fetchObservationEvalRules",
async (importOriginal) => ({
...(await importOriginal<
typeof import("../features/evaluation/observationEval/fetchObservationEvalRules")
>()),
fetchObservationEvalRules: vi.fn().mockResolvedValue([]),
}),
);
describe("trace batch selection", () => {
const minute = 60_000;
const pendingTrace = (
projectId: string,
traceId: string,
due: number,
minStart: number,
maxStart: number,
): PendingTrace => ({
member: JSON.stringify([projectId, traceId]),
due,
trace: {
projectId,
traceId,
minStart,
maxStart,
revision: `revision-${projectId}-${traceId}`,
},
});
const ids = (batches: PendingTrace[][]) =>
batches.map((batch) => batch.map(({ member }) => member));
it("orders batches by the events table locality key", () => {
const projectB = pendingTrace("project-b", "trace-1", 1, 0, minute);
const laterMinute = pendingTrace(
"project-a",
"trace-3",
2,
2 * minute,
3 * minute,
);
const widerRange = pendingTrace(
"project-a",
"trace-4",
3,
10_000,
2 * minute,
);
const trace0 = pendingTrace("project-a", "trace-0", 4, 20_000, 50_000);
const trace1 = pendingTrace("project-a", "trace-1", 5, 10_000, 50_000);
const batches = selectTraceBatches(
[projectB, laterMinute, widerRange, trace0, trace1],
2,
"locality",
);
expect(ids(batches)).toEqual([
[trace1.member, trace0.member],
[widerRange.member, laterMinute.member],
[projectB.member],
]);
});
it("is independent of readiness and input order", () => {
const candidates = [
pendingTrace("project-b", "trace-3", 1, 10 * minute, 11 * minute),
pendingTrace("project-a", "trace-0", 4, 10 * minute, 11 * minute),
pendingTrace("project-a", "trace-1", 3, 10 * minute, 11 * minute),
pendingTrace("project-a", "trace-4", 2, 10 * minute, 11 * minute),
];
const expected = selectTraceBatches(candidates, 3, "locality");
expect(expected.flat().map(({ trace }) => trace.traceId)).toEqual([
"trace-1",
"trace-0",
"trace-4",
"trace-3",
]);
expect(selectTraceBatches(candidates.toReversed(), 3, "locality")).toEqual(
expected,
);
});
it("moves a batch boundary instead of bridging distant time ranges", () => {
const early = Array.from({ length: 59 }, (_, index) =>
pendingTrace("project", `early-${index}`, index, 0, minute),
);
const late = Array.from({ length: 60 }, (_, index) =>
pendingTrace(
"project",
`late-${index}`,
100 + index,
10_000 * minute,
10_001 * minute,
),
);
const batches = selectTraceBatches([...late, ...early], 60, "locality");
expect(batches.map((batch) => batch.length)).toEqual([59, 60]);
expect(
batches[0].every(({ trace }) => trace.traceId.startsWith("early-")),
).toBe(true);
expect(
batches[1].every(({ trace }) => trace.traceId.startsWith("late-")),
).toBe(true);
});
it("avoids a cross-project fill when the same job count permits it", () => {
const projectA = Array.from({ length: 59 }, (_, index) =>
pendingTrace("project-a", `trace-${index}`, index, 0, minute),
);
const projectB = Array.from({ length: 60 }, (_, index) =>
pendingTrace("project-b", `trace-${index}`, 100 + index, 0, minute),
);
const batches = selectTraceBatches(
[...projectB, ...projectA],
60,
"locality",
);
expect(batches.map((batch) => batch.length)).toEqual([59, 60]);
expect(
batches.map(
(batch) => new Set(batch.map(({ trace }) => trace.projectId)).size,
),
).toEqual([1, 1]);
});
it("cuts at the largest trace-hash gap within one time range", () => {
const candidates = Array.from({ length: 5 }, (_, index) =>
pendingTrace("project", `trace-${index}`, index, 0, minute),
);
const batches = selectTraceBatches(candidates.toReversed(), 3, "locality");
expect(
batches.map((batch) => batch.map(({ trace }) => trace.traceId)),
).toEqual([
["trace-1", "trace-0"],
["trace-4", "trace-2", "trace-3"],
]);
});
it("splits distant narrow traces rather than sharing one envelope", () => {
const candidates = Array.from({ length: 3 }, (_, index) =>
pendingTrace(
"project",
`day-${index}`,
index,
index * 3 * 60 * minute,
index * 3 * 60 * minute + minute,
),
);
expect(ids(selectTraceBatches(candidates, 60, "locality"))).toEqual(
candidates.map((candidate) => [candidate.member]),
);
});
it("preserves uneven cohorts separated by mandatory envelope boundaries", () => {
const cohorts = [2, 1, 4, 2].map((size, cohort) =>
Array.from({ length: size }, (_, index) => {
const start = (cohort * 120 + index) * minute;
return pendingTrace(
"project",
`cohort-${cohort}-${index}`,
index,
start,
start + minute,
);
}),
);
for (const cap of [4, 9]) {
expect(
ids(selectTraceBatches(cohorts.flat().toReversed(), cap, "locality")),
).toEqual(ids(cohorts));
}
});
it("bounds chained wide-trace expansion relative to the batch seed", () => {
const seed = pendingTrace("project", "seed", 1, 0, 120 * minute);
const compatible = pendingTrace(
"project",
"compatible",
2,
0,
144 * minute,
);
const wouldChain = pendingTrace(
"project",
"would-chain",
3,
0,
168 * minute,
);
const batches = selectTraceBatches(
[wouldChain, compatible, seed],
60,
"locality",
);
expect(batches.flat()).toHaveLength(3);
expect(
batches.some((batch) => {
const ids = new Set(batch.map(({ member }) => member));
return ids.has(seed.member) && ids.has(wouldChain.member);
}),
).toBe(false);
expect(
batches.every((batch) => {
const union =
Math.max(...batch.map(({ trace }) => trace.maxStart)) -
Math.min(...batch.map(({ trace }) => trace.minStart));
const seedSpan = batch[0].trace.maxStart - batch[0].trace.minStart;
return union <= Math.max(60 * minute, seedSpan * 1.25);
}),
).toBe(true);
});
it("coalesces overlapping same-project partials deterministically", () => {
const early = [
pendingTrace("project", "early-1", 1, 0, minute),
pendingTrace("project", "early-2", 2, minute, 2 * minute),
];
const intervening = [
pendingTrace("project", "late-1", 3, 24 * 60 * minute, 24 * 60 * minute),
];
const compatible = [
pendingTrace("project", "early-3", 4, 3 * minute, 3 * minute),
];
const forward = prepareLocalityPartials(
[early, intervening, compatible],
6,
);
const reversed = prepareLocalityPartials(
[compatible, intervening, early],
6,
);
expect(ids(forward.carry)).toEqual(ids(reversed.carry));
expect(ids(forward.dispatch)).toEqual(ids(reversed.dispatch));
expect(forward.dispatch).toEqual([]);
expect(
forward.carry.map((batch) =>
batch.map(({ trace }) => trace.traceId).sort(),
),
).toEqual([["early-1", "early-2", "early-3"], ["late-1"]]);
});
it("bounds locality partial carry without splitting batches", () => {
const oldest = [
pendingTrace("project", "oldest-1", 1, 0, 0),
pendingTrace("project", "oldest-2", 2, 0, 0),
pendingTrace("project", "oldest-3", 3, 0, 0),
];
const newer = [
pendingTrace("project", "newer-1", 4, 24 * 60 * minute, 24 * 60 * minute),
pendingTrace("project", "newer-2", 5, 24 * 60 * minute, 24 * 60 * minute),
pendingTrace("project", "newer-3", 6, 24 * 60 * minute, 24 * 60 * minute),
];
const result = prepareLocalityPartials([newer, oldest], 6);
expect(
result.dispatch.map((batch) =>
batch.map(({ trace }) => trace.traceId).sort(),
),
).toEqual([oldest.map(({ trace }) => trace.traceId)]);
expect(
result.carry.map((batch) =>
batch.map(({ trace }) => trace.traceId).sort(),
),
).toEqual([newer.map(({ trace }) => trace.traceId)]);
expect(result.carry.flat()).toHaveLength(3);
expect(result.carry.flat().length).toBeLessThanOrEqual(5);
});
it("breaks equal-increase coalescing ties by canonical order", () => {
const candidates = [
pendingTrace("project", "trace-1", 1, 0, 0),
pendingTrace("project", "trace-2", 1, 0, 0),
pendingTrace("project", "trace-3", 1, 0, 0),
];
const canonical = selectTraceBatches(candidates, 3, "locality")[0];
const result = prepareLocalityPartials(
candidates.toReversed().map((candidate) => [candidate]),
2,
);
expect(result.dispatch).toHaveLength(1);
expect(ids(result.dispatch)).toEqual([
canonical.slice(0, 2).map(({ member }) => member),
]);
expect(ids(result.carry)).toEqual([
canonical.slice(2).map(({ member }) => member),
]);
});
it("does not coalesce incompatible partials or exceed the trace cap", () => {
const wide = [
pendingTrace("project", "wide", 1, 0, 120 * minute),
pendingTrace("project", "wide-neighbor", 2, 121 * minute, 121 * minute),
];
const otherProject = [
pendingTrace("other", "same-window", 3, 121 * minute, 121 * minute),
];
const distant = [
pendingTrace("project", "distant", 4, 24 * 60 * minute, 24 * 60 * minute),
];
const atCapacity = Array.from({ length: 5 }, (_, index) =>
pendingTrace("capacity", `trace-${index}`, 10 + index, 0, 0),
);
const wouldOverflow = [
pendingTrace("capacity", "overflow", 20, minute, minute),
];
const result = prepareLocalityPartials(
[otherProject, distant, wouldOverflow, wide, atCapacity],
5,
);
const all = [...result.dispatch, ...result.carry];
expect(
all.some(
(batch) =>
batch.some(({ trace }) => trace.traceId === "wide") &&
batch.some(({ trace }) => trace.traceId === "wide-neighbor"),
),
).toBe(true);
expect(
all.some(
(batch) =>
batch.some(({ trace }) => trace.traceId === "wide") &&
batch.some(({ trace }) => trace.projectId === "other"),
),
).toBe(false);
expect(
all.some(
(batch) =>
batch.some(({ trace }) => trace.traceId === "wide") &&
batch.some(({ trace }) => trace.traceId === "distant"),
),
).toBe(false);
expect(
all.some(
(batch) =>
batch.some(({ trace }) => trace.traceId === "overflow") &&
batch.some(({ trace }) => trace.traceId === "trace-0"),
),
).toBe(false);
expect(all.every((batch) => batch.length <= 5)).toBe(true);
expect(new Set(all.flat().map(({ member }) => member)).size).toBe(10);
});
describe("weight limits", () => {
const weighted = (
traceId: string,
eventUpdateCount: number,
serializedEventBytes = eventUpdateCount * 1_000,
): PendingTrace => ({
...pendingTrace("project", traceId, 1, 0, minute),
estimates: { eventUpdateCount, serializedEventBytes },
});
const limits = { maxEventUpdates: 10, maxSerializedEventBytes: 50_000 };
const weightOf = (batch: PendingTrace[]) =>
batch.reduce(
(total, { estimates }) => ({
events: total.events + (estimates?.eventUpdateCount ?? 0),
bytes: total.bytes + (estimates?.serializedEventBytes ?? 0),
}),
{ events: 0, bytes: 0 },
);
it.each(["project", "locality"] as const)(
"closes %s batches before the event or byte budget and isolates overweight traces",
(strategy) => {
const candidates = [
weighted("a", 4),
weighted("b", 4),
weighted("c", 4),
weighted("heavy", 25),
weighted("bytes-1", 1, 30_000),
weighted("bytes-2", 1, 30_000),
{ ...pendingTrace("project", "unknown", 1, 0, minute) },
];
const batches = selectTraceBatches(candidates, 60, strategy, limits);
expect(batches.flat()).toHaveLength(candidates.length);
for (const batch of batches) {
if (batch.length === 1) continue;
const { events, bytes } = weightOf(batch);
expect(events).toBeLessThanOrEqual(limits.maxEventUpdates);
expect(bytes).toBeLessThanOrEqual(limits.maxSerializedEventBytes);
}
expect(
batches.find((batch) =>
batch.some(({ trace }) => trace.traceId === "heavy"),
),
).toHaveLength(1);
expect(
batches.some(
(batch) =>
batch.some(({ trace }) => trace.traceId === "bytes-1") &&
batch.some(({ trace }) => trace.traceId === "bytes-2"),
),
).toBe(false);
// Without budgets the same candidates fit one batch.
expect(selectTraceBatches(candidates, 60, strategy)).toHaveLength(1);
},
);
it("packs light locality neighbors of a heavy trace into full batches", () => {
const at = (
traceId: string,
start: number,
eventUpdateCount: number,
): PendingTrace => ({
...pendingTrace("project", traceId, 1, start, start),
estimates: {
eventUpdateCount,
serializedEventBytes: eventUpdateCount * 1_000,
},
});
// Locality order: two light traces, the heavy trace, six light traces.
const candidates = [
at("light-0", 0, 1),
at("light-1", 0, 1),
at("heavy", minute, 9),
...Array.from({ length: 6 }, (_, index) =>
at(`light-${index + 2}`, 2 * minute, 1),
),
];
const batches = selectTraceBatches(candidates, 4, "locality", limits);
expect(batches.flat()).toHaveLength(candidates.length);
expect(batches).toHaveLength(3);
expect(
batches.find((batch) =>
batch.some(({ trace }) => trace.traceId === "heavy"),
),
).toHaveLength(1);
});
it("does not coalesce locality partials past the weight budget", () => {
const result = prepareLocalityPartials(
[[weighted("a", 6)], [weighted("b", 6)], [weighted("c", 4)]],
60,
limits,
);
const all = [...result.dispatch, ...result.carry];
expect(all.flat()).toHaveLength(3);
expect(all.every((batch) => weightOf(batch).events <= 10)).toBe(true);
expect(all).toHaveLength(2);
});
});
});
describe("trace micro-batch scheduling with Redis", () => {
const { due: dueKey, state: stateKey } = getTraceBatchReadinessKeys(0);
const originalEnabled = env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED;
const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
const originalDispatcherEnabled = env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED;
const originalConsumerEnabled =
env.QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED;
const originalReadEnabled = env.LANGFUSE_TRACE_BATCH_READ_ENABLED;
const originalSamplingRate = env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE;
const originalStrategy = env.LANGFUSE_TRACE_BATCH_STRATEGY;
const originalMaxSize = env.LANGFUSE_TRACE_BATCH_MAX_SIZE;
const originalPendingTtl = env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS;
const originalIdle = env.LANGFUSE_TRACE_BATCH_IDLE_MS;
const originalFirstTtl = env.LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS;
const originalShards = env.LANGFUSE_TRACE_BATCH_SHARD_COUNT;
const originalMaxTraceEventUpdates =
env.LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES;
const originalMaxTraceBytes =
env.LANGFUSE_TRACE_BATCH_MAX_TRACE_SERIALIZED_BYTES;
let queue: Queue<TQueueJobTypes[QueueName.TraceBatch]>;
let connection: NonNullable<ReturnType<typeof createNewRedisInstance>>;
const runners: TraceBatchDispatcher[] = [];
const client = () => {
if (!redis) throw new Error("Redis is required for this test");
return redis;
};
const member = (projectId: string, traceId: string) =>
JSON.stringify([projectId, traceId]);
const event = (
traceId: string,
start = 1_000_000,
serializedEventBytes = 100,
) => ({
traceId,
startTimeISO: new Date(start).toISOString(),
serializedEventBytes,
});
const runner = () => {
const instance = new TraceBatchDispatcher();
runners.push(instance);
return instance;
};
async function makeDue(projectId: string, ...traceIds: string[]) {
for (const traceId of traceIds)
await client().zadd(
dueKey,
Date.now() - 10_000,
member(projectId, traceId),
);
}
async function clearReadiness() {
for (let shard = 0; shard < env.LANGFUSE_TRACE_BATCH_SHARD_COUNT; shard++) {
const keys = getTraceBatchReadinessKeys(shard);
await client().del(
keys.due,
keys.state,
...getTraceBatchFirstKeys(shard, Date.now()).keys,
);
}
await client().del("{trace-batch}:dispatcher");
}
beforeEach(async () => {
env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";
env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED = "true";
env.QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED = "false";
env.LANGFUSE_TRACE_BATCH_READ_ENABLED = "true";
env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "true";
env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = 1;
env.LANGFUSE_TRACE_BATCH_STRATEGY = "project";
env.LANGFUSE_TRACE_BATCH_MAX_SIZE = 60;
env.LANGFUSE_TRACE_BATCH_IDLE_MS = 600_000;
env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 7_200_000;
env.LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS = 86_400_000;
env.LANGFUSE_TRACE_BATCH_SHARD_COUNT = 1;
// The unset default is 2 minutes on DEV; pin the production idle time.
env.LANGFUSE_TRACE_BATCH_IDLE_MS = 600_000;
await clearReadiness();
const redisConnection = createNewRedisInstance();
if (!redisConnection) throw new Error("Redis is required for this test");
connection = redisConnection;
const name = `trace-batch-test-${randomUUID()}`;
queue = new Queue(name, { connection, prefix: getQueuePrefix(name) });
await queue.waitUntilReady();
vi.spyOn(TraceBatchQueue, "getInstance").mockReturnValue(queue);
});
afterEach(async () => {
for (const dispatcher of runners) await dispatcher.drain();
runners.length = 0;
vi.restoreAllMocks();
vi.clearAllMocks();
await clearReadiness();
env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = originalEnabled;
env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalCloudRegion;
env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED = originalDispatcherEnabled;
env.QUEUE_CONSUMER_TRACE_BATCH_QUEUE_IS_ENABLED = originalConsumerEnabled;
env.LANGFUSE_TRACE_BATCH_READ_ENABLED = originalReadEnabled;
env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = originalSamplingRate;
env.LANGFUSE_TRACE_BATCH_STRATEGY = originalStrategy;
env.LANGFUSE_TRACE_BATCH_MAX_SIZE = originalMaxSize;
env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = originalPendingTtl;
env.LANGFUSE_TRACE_BATCH_IDLE_MS = originalIdle;
env.LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS = originalFirstTtl;
env.LANGFUSE_TRACE_BATCH_SHARD_COUNT = originalShards;
env.LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES =
originalMaxTraceEventUpdates;
env.LANGFUSE_TRACE_BATCH_MAX_TRACE_SERIALIZED_BYTES = originalMaxTraceBytes;
await queue.obliterate({ force: true });
await queue.close();
connection.disconnect();
});
afterAll(() => client().disconnect());
it("re-dispatches late arrivals with the earliest start of the whole trace", async () => {
env.LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS = 600;
const start = Date.now() - 20 * 60_000;
await trackTraceBatchActivity("project", [event("late-trace", start)]);
const trackedAt = Date.now();
await vi.waitFor(() =>
expect(Date.now() - trackedAt).toBeGreaterThanOrEqual(400),
);
await makeDue("project", "late-trace");
await runner().processBatch();
const [first] = await queue.getJobs(["wait"]);
expect(first.data.payload.traces[0].minStart).toBe(start);
expect(await client().zcard(dueKey)).toBe(0);
const dispatchedAt = Date.now();
await vi.waitFor(() =>
expect(Date.now() - dispatchedAt).toBeGreaterThanOrEqual(300),
);
const lateStart = start + 10 * 60_000;
await trackTraceBatchActivity("project", [event("late-trace", lateStart)]);
await makeDue("project", "late-trace");
await runner().processBatch();
const jobs = await queue.getJobs(["wait"]);
expect(jobs).toHaveLength(2);
const later = jobs.find((job) => job.id !== first.id)!;
expect(later.data.payload.traces[0]).toMatchObject({
minStart: start,
maxStart: lateStart,
});
expect(later.data.payload.traces[0].revision).not.toBe(
first.data.payload.traces[0].revision,
);
const backdatedStart = start + 3 * 60_000;
await trackTraceBatchActivity("project", [
event("late-trace", backdatedStart),
]);
await makeDue("project", "late-trace");
await runner().processBatch();
const backdated = (await queue.getJobs(["wait"])).find(
(job) => job.id !== first.id && job.id !== later.id,
)!;
expect(backdated.data.payload.traces[0]).toMatchObject({
minStart: start,
maxStart: lateStart,
});
});
it("keeps abandoned pending keys through idle time and the recovery grace", async () => {
env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 200;
await trackTraceBatchActivity("project", [event("long-idle")], 500);
const due = Number(
await client().zscore(dueKey, member("project", "long-idle")),
);
for (const key of [dueKey, stateKey])
expect(Number(await client().pexpiretime(key))).toBeGreaterThanOrEqual(
due + 200,
);
await vi.waitFor(() =>
expect(Date.now()).toBeGreaterThanOrEqual(due + 100),
);
expect(await client().exists(dueKey, stateKey)).toBe(2);
await vi.waitFor(async () =>
expect(await client().exists(dueKey, stateKey)).toBe(0),
);
});
it("expires first-seen history natively while keeping pending state independently", async () => {
env.LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS = 150;
const first = Date.now() - 600_000;
await trackTraceBatchActivity("project", [event("history", first)]);
const firstKeys = getTraceBatchFirstKeys(0, Date.now()).keys;
expect(
(await Promise.all(firstKeys.map((key) => client().hlen(key)))).reduce(
(sum, count) => sum + count,
0,
),
).toBe(1);
await makeDue("project", "history");
await runner().processBatch();
await vi.waitFor(async () => {
expect(await client().exists(...firstKeys)).toBe(0);
});
const next = first + 300_000;
await trackTraceBatchActivity("project", [event("history", next)]);
const state = JSON.parse(
(await client().hget(stateKey, member("project", "history")))!,
);
expect(state).toMatchObject({
minStart: next,
eventUpdateCount: 1,
serializedEventBytes: 100,
});
expect(await client().pttl(dueKey)).toBeGreaterThan(150);
});
it("distributes skewed projects by trace and dispatches every shard with single-slot scripts", async () => {
const shardCount = 64;
env.LANGFUSE_TRACE_BATCH_SHARD_COUNT = shardCount;
const evaluate = client().eval.bind(client());
const slots: string[][] = [];
vi.spyOn(client(), "eval").mockImplementation(((
script: string,
count: number,
...args: (string | number | Buffer)[]
) => {
slots.push(args.slice(0, count).map(String));
return evaluate(script, count, ...args);
}) as typeof evaluate);
const evaluateCached = client().evalsha.bind(client());
vi.spyOn(client(), "evalsha").mockImplementation(((
scriptHash: string,
count: number,
...args: (string | number | Buffer)[]
) => {
slots.push(args.slice(0, count).map(String));
return evaluateCached(scriptHash, count, ...args);
}) as typeof evaluateCached);
const count = 2_000;
for (let project = 0; project < 11; project++) {
const size = project === 0 ? 600 : 140;
await trackTraceBatchActivity(
`project-${project}`,
Array.from({ length: size }, (_, i) => event(`trace-${project}-${i}`)),
);
}
const counts = [];
for (let shard = 0; shard < shardCount; shard++) {
const { due } = getTraceBatchReadinessKeys(shard);
const members = await client().zrange(due, 0, -1);
counts.push(members.length);
for (const entry of members)
expect(getTraceBatchShard(entry)).toBe(shard);
if (members.length)
await client().zadd(
due,
...members.flatMap((entry) => [Date.now() - 1_000, entry]),
);
}
expect(counts.reduce((sum, n) => sum + n, 0)).toBe(count);
expect(Math.max(...counts)).toBeLessThanOrEqual((1.5 * count) / shardCount);
await runner().processBatch();
const dispatched = (await queue.getJobs(["wait"])).flatMap(
(job) => job.data.payload.traces,
);
expect(dispatched).toHaveLength(count);
expect(
new Set(
dispatched.map(({ projectId, traceId }) => member(projectId, traceId)),
).size,
).toBe(count);
for (const keys of slots) {
expect(new Set(keys.map((key) => key.match(/\{[^}]+\}/)?.[0])).size).toBe(
1,
);
}
});
it.each(["project", "locality"] as const)(
"packs one project's traces across readiness shards with %s",
async (strategy) => {
env.LANGFUSE_TRACE_BATCH_SHARD_COUNT = 16;
env.LANGFUSE_TRACE_BATCH_STRATEGY = strategy;
const traceIds = Array.from(
{ length: 60 },
(_, index) => `cross-shard-${index}`,
);
await trackTraceBatchActivity(
"project",
traceIds.map((traceId) => event(traceId)),
);
for (const traceId of traceIds) {
const entry = member("project", traceId);
await client().zadd(
getTraceBatchReadinessKeys(getTraceBatchShard(entry)).due,
Date.now() - 1_000,
entry,
);
}
await runner().processBatch();
const jobs = await queue.getJobs(["wait"]);
expect(jobs).toHaveLength(1);
expect(jobs[0].data.payload.traces).toHaveLength(60);
for (let shard = 0; shard < 16; shard++) {
const { due, state } = getTraceBatchReadinessKeys(shard);
expect(await client().zcard(due)).toBe(0);
expect(await client().hlen(state)).toBe(0);
}
},
);
it("acknowledges enqueued batches by shard across a hydration window", async () => {
env.LANGFUSE_TRACE_BATCH_SHARD_COUNT = 16;
env.LANGFUSE_TRACE_BATCH_MAX_SIZE = 30;
const traceIds = Array.from(
{ length: 120 },
(_, index) => `trace-${index}`,
);
await trackTraceBatchActivity(
"project",
traceIds.map((traceId) => event(traceId)),
);
for (const traceId of traceIds) {
const entry = member("project", traceId);
await client().zadd(
getTraceBatchReadinessKeys(getTraceBatchShard(entry)).due,
Date.now() - 1_000,
entry,
);
}
const evaluate = vi.spyOn(client(), "evalsha");
await runner().processBatch();
expect(await queue.getWaitingCount()).toBe(4);
const acknowledgements = evaluate.mock.calls.filter(
([, keyCount]) => Number(keyCount) > 2,
);
expect(acknowledgements.length).toBeLessThanOrEqual(16);
for (let shard = 0; shard < 16; shard++) {
const { due, state } = getTraceBatchReadinessKeys(shard);
expect(await client().zcard(due)).toBe(0);
expect(await client().hlen(state)).toBe(0);
}
});
it("samples prefixed map memory and paused/delayed queue work, then reports drained zeros", async () => {
const now = Date.now();
const clock = vi.spyOn(Date, "now").mockReturnValue(now);
await trackTraceBatchActivity("project", [event("trace")]);
const data = {
id: randomUUID(),
name: QueueJobs.TraceBatch,
timestamp: new Date(now),
payload: { traces: [] },
};
const waiting = await queue.add(QueueJobs.TraceBatch, data, {
timestamp: now - 5_000,
});
await queue.add(QueueJobs.TraceBatch, data, { timestamp: now - 1_000 });
await queue.add(QueueJobs.TraceBatch, data, { delay: 120_000 });
await queue.pause();
// Age collection must not fetch or parse the batch payload.
await connection.hset(queue.toKey(waiting.id!), "data", "invalid JSON");
const metrics = new TraceBatchMetricsRunner();
const evaluate = vi.spyOn(client(), "eval");
await metrics["execute"]();
for (const key of ["due", "state", "first"]) {
expect(recordGauge).toHaveBeenCalledWith(
"langfuse.trace_batch.redis_key_entries",
1,
{ key, shard: "total", unit: "records" },
);
const sample = vi
.mocked(recordGauge)
.mock.calls.find(
([name, , tags]) =>
name === "langfuse.trace_batch.redis_key_bytes" &&
tags?.key === key,
);
expect(Number(sample?.[1])).toBeGreaterThan(0);
}
expect(recordGauge).toHaveBeenCalledWith(
"langfuse.trace_batch.queue_depth",
1,
{ type: "delayed", unit: "records" },
);
expect(recordGauge).not.toHaveBeenCalledWith(
"langfuse.trace_batch.queue_depth",
expect.anything(),
expect.objectContaining({ type: "waiting" }),
);
expect(recordGauge).toHaveBeenCalledWith(
"langfuse.trace_batch.queue_waiting_head_age_ms",
5_000,
{ unit: "milliseconds" },
);
clock.mockReturnValue(now + 30_000);
await metrics["execute"]();
expect(evaluate).toHaveBeenCalledTimes(1);
await clearReadiness();
await queue.drain(true);
vi.mocked(recordGauge).mockClear();
clock.mockReturnValue(now + 60_000);
await metrics["execute"]();
expect(evaluate).toHaveBeenCalledTimes(2);
expect(recordGauge).toHaveBeenCalledWith(
"langfuse.trace_batch.active_reads",
0,
);
expect(vi.mocked(recordGauge).mock.calls).toHaveLength(9);
expect(
vi.mocked(recordGauge).mock.calls.every(([, value]) => value === 0),
).toBe(true);
});
it("awaits failed telemetry collections while preserving independent queue measurements", async () => {
const metrics = new TraceBatchMetricsRunner();
const pendingMemory = Promise.withResolvers<number[]>();
vi.spyOn(client(), "eval").mockImplementationOnce(
() => pendingMemory.promise,
);
vi.spyOn(queue, "getJobCounts").mockRejectedValueOnce(
new Error("queue unavailable"),
);
let completed = false;
const collection = metrics["execute"]().then(() => {
completed = true;
});
await vi.waitFor(() =>
expect(recordGauge).toHaveBeenCalledWith(
"langfuse.trace_batch.queue_waiting_head_age_ms",
0,
{ unit: "milliseconds" },
),
);
expect(completed).toBe(false);
pendingMemory.reject(new Error("memory unavailable"));
await collection;
expect(vi.mocked(recordGauge).mock.calls).toHaveLength(2);
});
it("dispatches with locality selection and records bounded selector measurements", async () => {
env.LANGFUSE_TRACE_BATCH_STRATEGY = "locality";
const minute = 60_000;
await trackTraceBatchActivity("project-a", [
event("short-1", 10 * minute),
event("short-1", 12 * minute),
event("wide", 0),
event("wide", 8 * 60 * minute),
]);
await trackTraceBatchActivity("project-b", [
event("short-2", 11 * minute),
event("short-2", 13 * minute),
]);
const due = Date.now() - 1_000;
await client().zadd(
dueKey,
due,
member("project-a", "short-1"),
due + 1,
member("project-a", "wide"),
due + 2,
member("project-b", "short-2"),
);
await runner().processBatch();
const jobs = await queue.getJobs(["wait"]);
expect(
jobs
.map((job) =>
job.data.payload.traces
.map(({ traceId }) => traceId)
.sort()
.join(","),
)
.sort(),
).toEqual(["short-1,short-2,wide"]);
expect(recordDistribution).toHaveBeenCalledWith(
"langfuse.trace_batch.candidate_buffer_size",
3,
{ strategy: "locality" },
);
expect(recordDistribution).toHaveBeenCalledWith(
"langfuse.trace_batch.event_time_envelope_ms",
29_040_000,
{ strategy: "locality" },
);
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.trace_batch.dispatched_batches",
1,
{ strategy: "locality", fill: "partial" },
);
expect(
vi
.mocked(recordDistribution)
.mock.calls.some(
([name, value, tags]) =>
name === "langfuse.trace_batch.selector_duration_ms" &&
Number(value) >= 0 &&
tags?.strategy === "locality",
),
).toBe(true);
});
it("excludes oversized traces from dispatch, removes their state, and reports their IDs", async () => {
env.LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES = 3;
env.LANGFUSE_TRACE_BATCH_MAX_TRACE_SERIALIZED_BYTES = 1_000;
const warn = vi.spyOn(shared.logger, "warn");
await trackTraceBatchActivity("project", [
...Array.from({ length: 4 }, (_, index) =>
event("many-updates", 1_000_000 + index),
),
event("large-input", 1_000_000, 5_000),
event("small", 1_000_000),
]);
await makeDue("project", "many-updates", "large-input", "small");
await runner().processBatch();
const jobs = await queue.getJobs(["wait"]);
expect(
jobs.map((job) => job.data.payload.traces.map(({ traceId }) => traceId)),
).toEqual([["small"]]);
expect(await client().zcard(dueKey)).toBe(0);
expect(await client().hlen(stateKey)).toBe(0);
for (const [traceId, reason] of [
["many-updates", "event_updates"],
["large-input", "serialized_bytes"],
] as const) {
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.trace_batch.excluded_traces",
1,
{ reason, strategy: "project" },
);
expect(warn).toHaveBeenCalledWith(
"Trace batch excluded oversized trace",
expect.objectContaining({ projectId: "project", traceId, reason }),
);
}
await trackTraceBatchActivity("project", [
event("many-updates", 1_000_004),
event("large-input", 1_000_001),
]);
for (const [traceId, eventUpdateCount, serializedEventBytes] of [
["many-updates", 5, 500],
["large-input", 2, 5_100],
] as const)
expect(
JSON.parse(
(await client().hget(stateKey, member("project", traceId)))!,
),
).toMatchObject({ eventUpdateCount, serializedEventBytes });
await makeDue("project", "many-updates", "large-input");
await runner().processBatch();
expect(await queue.getWaitingCount()).toBe(1);
expect(await client().zcard(dueKey)).toBe(0);
expect(await client().hlen(stateKey)).toBe(0);
});
it("uses retained full-trace estimates for batch budgets after successful dispatch", async () => {
const original = env.LANGFUSE_TRACE_BATCH_MAX_BATCH_EVENT_UPDATES;
env.LANGFUSE_TRACE_BATCH_MAX_BATCH_EVENT_UPDATES = 6;
try {
const ingest = async (start: number) => {
await trackTraceBatchActivity("project", [
event("a", start),
event("a", start),
event("b", start),
event("b", start),
]);
await makeDue("project", "a", "b");
await runner().processBatch();
};
await ingest(1_000_000);
const [first] = await queue.getJobs(["wait"]);
expect(first.data.payload.traces).toHaveLength(2);
await ingest(1_000_001);
const lateJobs = (await queue.getJobs(["wait"])).filter(
(job) => job.id !== first.id,
);
expect(lateJobs).toHaveLength(2);
expect(lateJobs.flatMap((job) => job.data.payload.traces)).toHaveLength(
2,
);
for (const job of lateJobs) {
expect(job.data.payload.traces).toHaveLength(1);
expect(job.data.payload.traces[0].minStart).toBe(1_000_000);
}
expect(recordDistribution).toHaveBeenCalledWith(
"langfuse.trace_batch.estimated_event_update_count",
4,
{ scope: "trace", strategy: "project" },
);
} finally {
env.LANGFUSE_TRACE_BATCH_MAX_BATCH_EVENT_UPDATES = original;
}
});
it("does not report an oversized trace as excluded when new activity keeps it pending", async () => {
env.LANGFUSE_TRACE_BATCH_MAX_TRACE_EVENT_UPDATES = 3;
const warn = vi.spyOn(shared.logger, "warn");
await trackTraceBatchActivity(
"project",
Array.from({ length: 4 }, (_, index) =>
event("reactivated", 1_000_000 + index),
),
);
await makeDue("project", "reactivated");
const evaluate = client().evalsha.bind(client());
let reactivated = false;
vi.spyOn(client(), "evalsha").mockImplementation((async (
scriptHash: string,
keyCount: number,
...args: (string | number)[]
) => {
if (
!reactivated &&
keyCount > 2 &&
Number(args[keyCount]) === env.LANGFUSE_TRACE_BATCH_FIRST_SEEN_TTL_MS
) {
reactivated = true;
const traceMember = member("project", "reactivated");
const state = JSON.parse((await client().hget(stateKey, traceMember))!);
await client().hset(
stateKey,
traceMember,
JSON.stringify({ ...state, revision: randomUUID() }),
);
}
return evaluate(scriptHash, keyCount, ...args);
}) as typeof evaluate);
await runner().processBatch();
expect(reactivated).toBe(true);
expect(
await client().hexists(stateKey, member("project", "reactivated")),
).toBe(1);
expect(recordIncrement).not.toHaveBeenCalledWith(
"langfuse.trace_batch.excluded_traces",
expect.anything(),
expect.anything(),
);
expect(warn).not.toHaveBeenCalledWith(
"Trace batch excluded oversized trace",
expect.anything(),
);
expect(recordIncrement).not.toHaveBeenCalledWith(
"langfuse.trace_batch.reactivated_traces",
1,
);
});
it("sets the due time from the project's trace idle time, capped below the pending TTL", async () => {
const orgId = `org-${randomUUID()}`;
const projectId = `project-${randomUUID()}`;
await prisma.organization.create({
data: { id: orgId, name: "Idle time org" },
});
await prisma.project.create({
data: { id: projectId, orgId, name: "Idle time project" },
});
try {
await shared.saveTracingReadinessSettings(projectId, {
traceIdleMs: 3_600_000,
});
const now = Date.now();
// The Lua script reads Redis time, so allow for clock skew.
const dueOffset = async (traceId: string) =>
Number(await client().zscore(dueKey, member(projectId, traceId))) - now;
await trackTraceBatchActivity(projectId, [event("custom")]);
await trackTraceBatchActivity("project", [event("default")]);
expect(await dueOffset("custom")).toBeGreaterThan(3_590_000);
expect(await dueOffset("custom")).toBeLessThan(3_610_000);
expect(
Number(await client().zscore(dueKey, member("project", "default"))) -
now,
).toBeLessThan(610_000);
env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 1_800_000;
await trackTraceBatchActivity(projectId, [event("capped")]);
expect(await dueOffset("capped")).toBeLessThan(1_780_000);
expect(await dueOffset("capped")).toBeGreaterThan(1_760_000);
} finally {
await prisma.organization.delete({ where: { id: orgId } });
}
});
it("caches the project's settings in Redis with an expiry and replaces them on save", async () => {
const orgId = `org-${randomUUID()}`;
const projectId = `project-${randomUUID()}`;
const cacheKey = `langfuse:tracing-readiness-settings:${projectId}`;
await prisma.organization.create({
data: { id: orgId, name: "Idle time cache org" },
});
await prisma.project.create({
data: { id: projectId, orgId, name: "Idle time cache project" },
});
try {
await trackTraceBatchActivity(projectId, [event("uncached")]);
expect(JSON.parse((await connection.get(cacheKey))!)).toEqual({
traceIdleMs: null,
});
const ttl = await connection.ttl(cacheKey);
expect(ttl).toBeGreaterThan(0);
expect(ttl).toBeLessThanOrEqual(300);
await shared.saveTracingReadinessSettings(projectId, {
traceIdleMs: 300_000,
});
expect(JSON.parse((await connection.get(cacheKey))!)).toEqual({
traceIdleMs: 300_000,
});
await trackTraceBatchActivity(projectId, [event("saved")]);
const offset =
Number(await client().zscore(dueKey, member(projectId, "saved"))) -
Date.now();
expect(offset).toBeGreaterThan(290_000);
expect(offset).toBeLessThan(310_000);
} finally {
await prisma.organization.delete({ where: { id: orgId } });
await connection.del(cacheKey);
}
});
it("does not let an earlier database read overwrite newly saved settings", async () => {
const orgId = `org-${randomUUID()}`;
const projectId = `project-${randomUUID()}`;
const cacheKey = `langfuse:tracing-readiness-settings:${projectId}`;
await prisma.organization.create({
data: { id: orgId, name: "Idle time race org" },
});
await prisma.project.create({
data: { id: projectId, orgId, name: "Idle time race project" },
});
try {
await shared.saveTracingReadinessSettings(projectId, {
traceIdleMs: 120_000,
});
await connection.del(cacheKey);
let releaseFill!: () => void;
const fillHeld = new Promise<void>((resolve) => {
releaseFill = resolve;
});
let signalFill!: () => void;
const fillStarted = new Promise<void>((resolve) => {
signalFill = resolve;
});
const actualShared = await vi.importActual<
typeof import("@langfuse/shared/src/server")
>("@langfuse/shared/src/server");
if (!actualShared.redis)
throw new Error("Redis is required for this test");
const originalSet = actualShared.redis.set.bind(actualShared.redis);
vi.spyOn(actualShared.redis, "set").mockImplementationOnce(
async (...args) => {
signalFill();
await fillHeld;
return originalSet(...args);
},
);
const earlierRead = shared.getTracingReadinessSettings(projectId);
await fillStarted;
try {
await shared.saveTracingReadinessSettings(projectId, {
traceIdleMs: 300_000,
});
} finally {
releaseFill();
}
await earlierRead;
expect(await shared.getTracingReadinessSettings(projectId)).toEqual({
traceIdleMs: 300_000,
});
} finally {
await prisma.organization.delete({ where: { id: orgId } });
await connection.del(cacheKey);
}
});
it("uses and caches the default for an out-of-range stored idle time", async () => {
const orgId = `org-${randomUUID()}`;
const projectId = `project-${randomUUID()}`;
const cacheKey = `langfuse:tracing-readiness-settings:${projectId}`;
await prisma.organization.create({
data: { id: orgId, name: "Invalid idle time org" },
});
await prisma.project.create({
data: { id: projectId, orgId, name: "Invalid idle time project" },
});
try {
await prisma.tracingReadinessSettings.create({
data: { projectId, traceIdleMs: 1 },
});
expect(await shared.getTracingReadinessSettings(projectId)).toEqual({
traceIdleMs: null,
});
expect(JSON.parse((await connection.get(cacheKey))!)).toEqual({
traceIdleMs: null,
});
} finally {
await prisma.organization.delete({ where: { id: orgId } });
await connection.del(cacheKey);
}
});
it("uses the default when the idle time read fails", async () => {
const projectId = `project-${randomUUID()}`;
vi.mocked(shared.getTracingReadinessSettings).mockRejectedValueOnce(
new Error("Postgres unavailable"),
);
await trackTraceBatchActivity(projectId, [event("failed")]);
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.trace_batch.idle_time_read_failures",
1,
);
const offset =
Number(await client().zscore(dueKey, member(projectId, "failed"))) -
Date.now();
expect(offset).toBeGreaterThan(590_000);
expect(offset).toBeLessThan(610_000);
});
it("does not track or dispatch on self-hosted deployments even with experiment flags enabled", async () => {
env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;
const evaluate = vi.spyOn(client(), "eval");
const lock = vi.spyOn(client(), "set");
await trackTraceBatchActivity("project", [event("trace")]);
await runner()["execute"]();
const metrics = new TraceBatchMetricsRunner();
await metrics["execute"]();
env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "DEV";
env.LANGFUSE_TRACE_BATCH_DISPATCHER_ENABLED = "false";
await metrics["execute"]();
expect(evaluate).not.toHaveBeenCalled();
expect(lock).not.toHaveBeenCalled();
expect(await client().exists(dueKey, stateKey)).toBe(0);
expect(await queue.count()).toBe(0);
});
it("ingests through BullMQ, postpones readiness on new activity, and consumes full ClickHouse payloads once due", async () => {
const orgId = randomUUID();
const projectId = randomUUID();
await prisma.organization.create({
data: {
id: orgId,
name: "Trace batch late arrivals",
projects: { create: { id: projectId, name: "Trace batch" } },
},
});
const usage = {
providedUsageDetails: {},
usageDetails: {},
providedCostDetails: {},
costDetails: {},
};
vi.mocked(summarizeTopicTraceFacets).mockImplementation(
async (_models, facets) => ({
output: Object.fromEntries(
facets.map(({ key }) => [
key,
{
notes: "The complete trace was read.",
summary: `summary-${vi.mocked(summarizeTopicTraceFacets).mock.calls.length}`,
status: "applicable" as const,
},
]),
),
...usage,
}),
);
vi.mocked(embedTopicSummaries).mockImplementation(
async (_model, summaries, dimensions) => ({
embeddings: summaries.map(() =>
Array.from({ length: dimensions }, () => 0.1),
),
...usage,
}),
);
const original = {
LANGFUSE_TRACE_BATCH_IDLE_MS: env.LANGFUSE_TRACE_BATCH_IDLE_MS,
LANGFUSE_MIGRATION_V4_WRITE_MODE: env.LANGFUSE_MIGRATION_V4_WRITE_MODE,
LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED:
env.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED,
LANGFUSE_OBSERVATION_FIELD_OVERFLOW_ENABLED:
env.LANGFUSE_OBSERVATION_FIELD_OVERFLOW_ENABLED,
};
env.LANGFUSE_TRACE_BATCH_IDLE_MS = 4_000;
env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = 0.1;
env.LANGFUSE_MIGRATION_V4_WRITE_MODE = "events_only";
env.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED = "false";
env.LANGFUSE_OBSERVATION_FIELD_OVERFLOW_ENABLED = "false";
const ingestionName = `trace-batch-ingestion-test-${randomUUID()}`;
const ingestionQueue = new Queue<
TQueueJobTypes[QueueName.OtelIngestionQueue]
>(ingestionName, { connection, prefix: getQueuePrefix(ingestionName) });
const ingestionEvents = new QueueEvents(ingestionName, {
connection,
prefix: ingestionQueue.opts.prefix,
});
const ingestionWorker = new Worker(
ingestionName,
otelIngestionQueueProcessorBuilder(false),
{ connection, prefix: ingestionQueue.opts.prefix },
);
const batchEvents = new QueueEvents(queue.name, {
connection,
prefix: queue.opts.prefix,
});
const batchWorker = new Worker(queue.name, traceBatchQueueProcessor, {
connection,
prefix: queue.opts.prefix,
});
const writer = ClickhouseWriter.getInstance();
try {
await Promise.all([
ingestionWorker.waitUntilReady(),
ingestionEvents.waitUntilReady(),
batchWorker.waitUntilReady(),
batchEvents.waitUntilReady(),
]);
// The evaluator sampler assigns these scores 0.0593, 0.0291 and 0.7116.
const traceId = "00000000000000000000000000000001";
const otherTraceId = "00000000000000000000000000000003";
const excludedTraceId = "00000000000000000000000000000002";
const input = JSON.stringify([
{ role: "user", content: "input".repeat(1_000) },
]);
const output = JSON.stringify({
role: "assistant",
content: "output".repeat(1_000),
});
const metadata = JSON.stringify({ context: "metadata".repeat(1_000) });
const nano = BigInt(Date.now() - 20 * 60_000) * 1_000_000n;
async function ingest(traceIds: string[], start = nano) {
const timestamp = {
low: Number(start & 0xffffffffn),
high: Number(start >> 32n),
unsigned: true,
};
// S3 transport and unrelated evaluator configuration are stubbed;
// OTLP conversion, writer, Redis scripts and both queue workers are real.
vi.mocked(getS3EventStorageClient).mockReturnValue({
download: vi.fn().mockResolvedValue(
JSON.stringify([
{
resource: { attributes: [] },
scopeSpans: [
{
scope: { name: "langfuse-sdk", version: "4.0.0" },
spans: traceIds.map((id) => ({
traceId: Buffer.from(id, "hex").toJSON(),
spanId: randomBytes(8).toJSON(),
name: "trace-batch-integration",
kind: 1,
startTimeUnixNano: timestamp,
endTimeUnixNano: timestamp,
status: { code: 2, message: "tool call failed" },
attributes: Object.entries({
"langfuse.observation.type": "generation",
"langfuse.observation.input": input,
"langfuse.observation.output": output,
"langfuse.observation.metadata": metadata,
}).map(([key, value]) => ({
key,
value: { stringValue: value },
})),
})),
},
],
},
]),
),
} as unknown as ReturnType<typeof getS3EventStorageClient>);
const job = await ingestionQueue.add(QueueJobs.OtelIngestionJob, {
id: randomUUID(),
timestamp: new Date(),
name: QueueJobs.OtelIngestionJob,
payload: {
data: { fileKey: "trace-batch-integration.json" },
authCheck: {
validKey: true,
scope: { projectId, orgId: randomUUID(), accessLevel: "project" },
},
ingestionVersion: "4",
},
});
await job.waitUntilFinished(ingestionEvents, 10_000);
}
async function waitUntilRedisTime(deadline: number) {
await expect
.poll(
async () => {
const [seconds, microseconds] = await client().time();
return (
Number(seconds) * 1_000 +
Math.floor(Number(microseconds) / 1_000)
);
},
{ interval: 25, timeout: 10_000 },
)
.toBeGreaterThanOrEqual(deadline);
}
await ingest([traceId, excludedTraceId]);
expect(await client().hlen(stateKey)).toBe(1);
const firstDue = Number(
await client().zscore(dueKey, member(projectId, traceId)),
);
await waitUntilRedisTime(firstDue - 2_000);
// Leave ample time to check the old deadline even on a busy CI worker.
env.LANGFUSE_TRACE_BATCH_IDLE_MS = 10_000;
await ingest([traceId, otherTraceId, excludedTraceId]);
const renewedDue = Number(
await client().zscore(dueKey, member(projectId, traceId)),
);
expect(renewedDue).toBeGreaterThan(firstDue);
expect(await client().hlen(stateKey)).toBe(2);
const dispatcher = runner();
await waitUntilRedisTime(firstDue);
await dispatcher.processBatch();
expect(await queue.getJobCounts("wait", "active", "completed")).toEqual({
wait: 0,
active: 0,
completed: 0,
});
await writer.flushAll(true);
// Sampling only gates readiness; excluded observations are still ingested.
let excludedObservations = 0;
for await (const observation of getTraceBatchEventStream({
traces: [
{
projectId,
traceId: excludedTraceId,
minStart: Number(nano / 1_000_000n),
maxStart: Number(nano / 1_000_000n),
},
],
})) {
expect(observation).toMatchObject({
trace_id: excludedTraceId,
level: "ERROR",
status_message: "tool call failed",
});
excludedObservations++;
}
expect(excludedObservations).toBe(2);
topicsProjects.add(projectId);
const facets = await ensureDefaultTopicFacets(projectId);
env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "false";
env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = 0;
await waitUntilRedisTime(renewedDue);
await dispatcher.processBatch();
const [batch] = await queue.getJobs(["wait", "active", "completed"]);
expect(
batch.data.payload.traces.map((trace) => trace.traceId).sort(),
).toEqual([traceId, otherTraceId].sort());
const result = await batch.waitUntilFinished(batchEvents, 10_000);
expect(result).toMatchObject({
observationCount: 3,
traceCount: 2,
projectCount: 1,
});
// All three observations must carry their untruncated I/O and metadata.
expect(result.ioMetadataBytes).toBeGreaterThanOrEqual(
3 * (Buffer.byteLength(input) + Buffer.byteLength(output) + 8_000),
);
expect(recordDistribution).toHaveBeenCalledWith(
"langfuse.trace_batch.size",
2,
);
expect(recordDistribution).not.toHaveBeenCalledWith(
"langfuse.trace_batch.io_metadata_bytes",
expect.anything(),
);
expect(await client().zcard(dueKey)).toBe(0);
expect(await client().hlen(stateKey)).toBe(0);
await dispatcher.processBatch();
expect(await queue.getCompletedCount()).toBe(1);
expect(summarizeTopicTraceFacets).toHaveBeenCalledTimes(2);
const summaries = async () =>
(
await Promise.all(
facets.map((facet) =>
listTopicSummaries(
projectId,
{
traceIds: [traceId, otherTraceId],
facetId: facet.id,
facetVersion: facet.versions[0].version,
},
{
from: new Date(Number(nano / 1_000_000n) - 1),
to: new Date(),
},
),
),
)
).flat();
const firstSummaries = await summaries();
expect(firstSummaries).toHaveLength(8);
// One late event stays within the original buffer, while the other is
// ten minutes away and requires the retained earliest start.
env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "true";
env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = 1;
await ingest([traceId], nano + 60_000n * 1_000_000n);
await ingest([otherTraceId], nano + 10n * 60_000n * 1_000_000n);
await writer.flushAll(true);
await makeDue(projectId, traceId, otherTraceId);
await dispatcher.processBatch();
const lateBatch = (
await queue.getJobs(["wait", "active", "completed"])
).find((job) => job.id !== batch.id)!;
for (const trace of lateBatch.data.payload.traces)
expect(trace.minStart).toBe(Number(nano / 1_000_000n));
const lateResult = await lateBatch.waitUntilFinished(batchEvents, 10_000);
expect(lateResult).toMatchObject({ observationCount: 5, traceCount: 2 });
expect(summarizeTopicTraceFacets).toHaveBeenCalledTimes(4);
await ingest([otherTraceId], nano + 3n * 60_000n * 1_000_000n);
await writer.flushAll(true);
await makeDue(projectId, otherTraceId);
await dispatcher.processBatch();
const backdatedBatch = (
await queue.getJobs(["wait", "active", "completed"])
).find((job) => job.id !== batch.id && job.id !== lateBatch.id)!;
expect(backdatedBatch.data.payload.traces).toMatchObject([
{
minStart: Number(nano / 1_000_000n),
maxStart: Number(nano / 1_000_000n) + 10 * 60_000,
},
]);
expect(
await backdatedBatch.waitUntilFinished(batchEvents, 10_000),
).toMatchObject({ observationCount: 3, traceCount: 1 });
expect(summarizeTopicTraceFacets).toHaveBeenCalledTimes(5);
const latest = await summaries();
expect(latest).toHaveLength(8);
for (const summary of latest) {
expect(summary.unitStartTime).toBe(
firstSummaries.find((row) => row.traceId === summary.traceId)!
.unitStartTime,
);
expect(summary.summary).not.toBe(
firstSummaries.find(
(row) =>
row.traceId === summary.traceId &&
row.facetId === summary.facetId,
)!.summary,
);
}
} finally {
try {
await Promise.all([ingestionWorker.close(), batchWorker.close()]);
await Promise.all([ingestionEvents.close(), batchEvents.close()]);
await ingestionQueue.obliterate({ force: true });
await ingestionQueue.close();
await writer.shutdown();
} finally {
topicsProjects.delete(projectId);
await prisma.organization.deleteMany({ where: { id: orgId } });
Object.assign(env, original);
}
}
}, 30_000);
it("at rate 0 tracks only Topics-enabled projects and skips all work for others", async () => {
const samplingRate = env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE;
env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = 0;
topicsProjects.add("topics-project");
topicsProjects.add("inactive-project");
try {
vi.mocked(recordIncrement).mockClear();
await trackTraceBatchActivity("other-project", [event("other-trace")]);
vi.mocked(isTopicsProcessingActive).mockResolvedValueOnce(false);
await trackTraceBatchActivity("inactive-project", [
event("inactive-trace"),
]);
expect(recordIncrement).not.toHaveBeenCalled();
await trackTraceBatchActivity("topics-project", [event("topics-trace")]);
// An unreadable gate falls back to the allowlist.
vi.mocked(isTopicsProcessingActive).mockRejectedValueOnce(
new Error("postgres down"),
);
await trackTraceBatchActivity("topics-project", [
event("fallback-trace"),
]);
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.trace_batch.topics_gate_failures",
1,
);
expect((await client().hkeys(stateKey)).sort()).toEqual(
[
member("topics-project", "topics-trace"),
member("topics-project", "fallback-trace"),
].sort(),
);
} finally {
topicsProjects.clear();
env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = samplingRate;
}
});
it("does not load idle settings when every trace is sampled out", async () => {
env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = Number.MIN_VALUE;
vi.mocked(shared.getTracingReadinessSettings).mockClear();
await trackTraceBatchActivity("project", [event("sampled-out")]);
expect(shared.getTracingReadinessSettings).not.toHaveBeenCalled();
expect(await client().hlen(stateKey)).toBe(0);
});
it("keeps stable nested trace samples across replay and counts decisions once per trace in each ingestion batch", async () => {
const traceIds = Array.from({ length: 1_000 }, (_, i) =>
i.toString(16).padStart(32, "0"),
);
let previous: string[] = [];
// Fixed fixtures pin the evaluator's sampling cohort, including both endpoints.
for (const [rate, expectedCount] of [
[0.1, 78],
[0.1, 78],
[0.5, 478],
[1, 1_000],
]) {
await client().del(dueKey, stateKey);
vi.mocked(recordIncrement).mockClear();
env.LANGFUSE_TRACE_BATCH_SAMPLING_RATE = rate;
traceIds.reverse();
await trackTraceBatchActivity(
"project",
traceIds.flatMap((id) => [
event(id, 100),
event(id, 200, 200),
event(id, 150, 50),
]),
);
const selected = (await client().hkeys(stateKey)).sort();
expect(selected).toHaveLength(expectedCount);
expect(await client().zcard(dueKey)).toBe(expectedCount);
expect(selected).toEqual(expect.arrayContaining(previous));
if (previous.length === expectedCount) expect(selected).toEqual(previous);
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.trace_batch.sampling_decisions",
expectedCount,
{ decision: "selected" },
);
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.trace_batch.sampling_decisions",
1_000 - expectedCount,
{ decision: "excluded" },
);
expect(recordIncrement).not.toHaveBeenCalledWith(
"langfuse.trace_batch.event_updates",
expect.anything(),
{ stage: "sampled" },
);
for (const [stage, count] of [
["eligible", 1_000],
...(expectedCount > 0 ? [["recorded", expectedCount] as const] : []),
] as const) {
for (const [metric, total] of [
["event_updates", count * 3],
["serialized_event_bytes", count * 350],
] as const)
expect(
vi
.mocked(recordIncrement)
.mock.calls.filter(
([name, , tags]) =>
name === `langfuse.trace_batch.${metric}` &&
tags?.stage === stage,
)
.reduce((sum, [, value]) => sum + value, 0),
).toBe(total);
}
previous = selected;
}
});
it("counts only acknowledged update volume when tracking fails after a chunk", async () => {
await trackTraceBatchActivity("warm-up", [event("trace")]);
await clearReadiness();
vi.mocked(recordIncrement).mockClear();
const fallback = vi.spyOn(client(), "eval");
const pipeline = client().pipeline.bind(client());
let commands = 0;
vi.spyOn(client(), "pipeline").mockImplementation(() => {
const batch = pipeline();
const evaluate = batch.evalsha.bind(batch);
vi.spyOn(batch, "evalsha").mockImplementation(((
scriptHash: string,
keyCount: number,
...args: (string | number | Buffer)[]
) => {
// Fail one command before mutation; the other pipelined chunks still execute.
if (++commands === 2) args[1] = args[0];
// Simulate a cache miss for one sibling without flushing shared Redis scripts.
if (commands === 3) scriptHash = "0".repeat(40);
return evaluate(scriptHash, keyCount, ...args);
}) as typeof evaluate);
return batch;
});
await expect(
trackTraceBatchActivity("project", [
...Array.from({ length: 1_001 }, (_, index) =>
event(`trace-${index}`, 100, 7),
),
{ ...event("invalid"), startTimeISO: "not-a-date" },
]),
).resolves.toBeUndefined();
const recorded = await client().hlen(stateKey);
expect(recorded).toBe(150);
expect(commands).toBe(4);
expect(fallback).toHaveBeenCalledOnce();
for (const state of await client().hvals(stateKey))
expect(JSON.parse(state)).toMatchObject({
eventUpdateCount: 1,
serializedEventBytes: 7,
});
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.trace_batch.event_updates",
1_001,
{ stage: "eligible" },
);
expect(recordIncrement).not.toHaveBeenCalledWith(
"langfuse.trace_batch.event_updates",
expect.anything(),
{ stage: "sampled" },
);
for (const [metric, count] of [
["event_updates", recorded],
["serialized_event_bytes", recorded * 7],
] as const)
expect(
vi
.mocked(recordIncrement)
.mock.calls.filter(
([name, , tags]) =>
name === `langfuse.trace_batch.${metric}` &&
tags?.stage === "recorded",
)
.reduce((sum, [, value]) => sum + value, 0),
).toBe(count);
expect(recordIncrement).not.toHaveBeenCalledWith(
"langfuse.trace_batch.event_updates",
1,
{ stage: "recorded" },
);
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.trace_batch.tracking_errors",
1,
);
});
it("preserves min/max across concurrent out-of-order arrivals, resets readiness, and separates tenants", async () => {
await Promise.all(
[30, 0, 20, 10].map((start) =>
trackTraceBatchActivity("project", [
event("trace", start, 100 + start),
event("trace", start, 200 + start),
]),
),
);
const state = JSON.parse(
(await client().hget(stateKey, member("project", "trace")))!,
);
expect(state).toMatchObject({
minStart: 0,
maxStart: 30,
eventUpdateCount: 8,
serializedEventBytes: 1_320,
});
await makeDue("project", "trace");
await trackTraceBatchActivity("project", [
event("trace", 15, 25),
event("trace", 15, 25),
]);
expect(
JSON.parse((await client().hget(stateKey, member("project", "trace")))!),
).toMatchObject({ eventUpdateCount: 10, serializedEventBytes: 1_370 });
expect(
Number(await client().zscore(dueKey, member("project", "trace"))),
).toBeGreaterThan(Date.now() + 590_000);
await trackTraceBatchActivity("other", [event("trace", 100, 7)]);
expect(
JSON.parse((await client().hget(stateKey, member("other", "trace")))!),
).toMatchObject({
minStart: 100,
maxStart: 100,
eventUpdateCount: 1,
serializedEventBytes: 7,
});
await runner().processBatch();
expect(await queue.getWaitingCount()).toBe(0);
await makeDue("project", "trace");
await makeDue("other", "trace");
await runner().processBatch();
for (const [count, bytes] of [
[10, 1_370],
[1, 7],
]) {
expect(recordDistribution).toHaveBeenCalledWith(
"langfuse.trace_batch.estimated_event_update_count",
count,
{ scope: "trace", strategy: "project" },
);
expect(recordDistribution).toHaveBeenCalledWith(
"langfuse.trace_batch.estimated_serialized_event_bytes",
bytes,
{ scope: "trace", strategy: "project" },
);
}
});
it.each([
{ maxSize: 60, sizes: [22, 60, 60, 60], projectCounts: [1, 1, 1, 2] },
{ maxSize: 80, sizes: [42, 80, 80], projectCounts: [1, 1, 2] },
])(
"dispatches cross-project batches capped at $maxSize, records their distribution, and drains with intake off",
async ({ maxSize, sizes, projectCounts }) => {
env.LANGFUSE_TRACE_BATCH_MAX_SIZE = maxSize;
const traces = Array.from(
{ length: 201 },
(_, index) => `trace-${index}`,
);
await trackTraceBatchActivity(
"project",
traces.map((traceId, index) => event(traceId, 1_000_000, index + 1)),
);
await trackTraceBatchActivity("project", [
event("trace-0", 1_000_000, 5),
]);
await trackTraceBatchActivity("other", [
event("trace-0", 1_000_000, 10_000),
]);
await client().zadd(
dueKey,
...traces.flatMap((traceId, index) => [
Date.now() - 1_000 + index,
member("project", traceId),
]),
);
await makeDue("other", "trace-0");
const add = vi.spyOn(queue, "add");
env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "false";
await trackTraceBatchActivity("disabled", [event("ignored")]);
await runner().processBatch();
const jobs = await queue.getJobs(["wait"]);
for (const job of jobs) TraceBatchEventSchema.parse(job.data);
expect(
jobs.map((job) => job.data.payload.traces.length).sort((a, b) => a - b),
).toEqual(sizes);
expect(
jobs
.flatMap((job) =>
job.data.payload.traces.map((trace) =>
member(trace.projectId, trace.traceId),
),
)
.sort(),
).toEqual(
[
...traces.map((traceId) => member("project", traceId)),
member("other", "trace-0"),
].sort(),
);
expect(
add.mock.calls.flatMap(([, job]) =>
job.payload.traces.map((trace) =>
member(trace.projectId, trace.traceId),
),
),
).toEqual([
member("other", "trace-0"),
...traces.map((traceId) => member("project", traceId)),
]);
expect(await client().zcard(dueKey)).toBe(0);
expect(await client().hlen(stateKey)).toBe(0);
expect(
vi
.mocked(recordDistribution)
.mock.calls.filter(([name]) => name === "langfuse.trace_batch.size")
.map(([, value]) => value)
.sort((a, b) => Number(a) - Number(b)),
).toEqual(sizes);
expect(
vi
.mocked(recordDistribution)
.mock.calls.filter(
([name]) => name === "langfuse.trace_batch.project_count",
)
.map(([, value]) => value)
.sort((a, b) => Number(a) - Number(b)),
).toEqual(projectCounts);
for (const [metric, estimate] of [
["estimated_event_update_count", "count"],
["estimated_serialized_event_bytes", "bytes"],
] as const) {
const expected = add.mock.calls.map(([, job]) =>
job.payload.traces.reduce((total, trace) => {
if (trace.projectId === "other")
return total + (estimate === "count" ? 1 : 10_000);
const index = Number(trace.traceId.replace("trace-", ""));
const estimates =
index === 0
? { count: 2, bytes: 6 }
: { count: 1, bytes: index + 1 };
return total + estimates[estimate];
}, 0),
);
expect(
vi
.mocked(recordDistribution)
.mock.calls.filter(
([name, , tags]) =>
name === `langfuse.trace_batch.${metric}` &&
tags?.scope === "batch",
)
.map(([, value, tags]) => ({ value, tags })),
).toEqual(
expected.map((value) => ({
value,
tags: { scope: "batch", strategy: "project" },
})),
);
}
for (const [, job] of add.mock.calls) {
for (const trace of job.payload.traces) {
expect(Object.keys(trace).sort()).toEqual([
"maxStart",
"minStart",
"projectId",
"revision",
"traceId",
]);
}
}
await runner().processBatch();
expect(await queue.getWaitingCount()).toBe(sizes.length);
},
);
it("keeps unknown estimates through updates and late-arrival dispatch", async () => {
const legacyStates = [
{},
{ eventUpdateCount: 4 },
{ serializedEventBytes: 400 },
];
const legacyIds = legacyStates.map((_, index) => `legacy-${index}`);
for (const [index, estimates] of legacyStates.entries()) {
await client().hset(
stateKey,
member("project", legacyIds[index]),
JSON.stringify({
minStart: 1_000_000,
maxStart: 1_000_000,
revision: randomUUID(),
...estimates,
}),
);
await client().zadd(
dueKey,
Date.now() + 600_000,
member("project", legacyIds[index]),
);
}
await trackTraceBatchActivity("project", [
...legacyIds.map((traceId) => event(traceId, 1_000_001, 20)),
event("known", 1_000_000, 30),
]);
await makeDue("project", ...legacyIds, "known");
const dispatcher = runner();
await dispatcher.processBatch();
const [job] = await queue.getJobs(["wait"]);
expect(
job.data.payload.traces.map(({ traceId }) => traceId).sort(),
).toEqual(["known", ...legacyIds].sort());
const estimateSamples = vi
.mocked(recordDistribution)
.mock.calls.filter(([name]) =>
name.startsWith("langfuse.trace_batch.estimated_"),
);
expect(estimateSamples).toHaveLength(2);
expect(estimateSamples).toEqual(
expect.arrayContaining([
[
"langfuse.trace_batch.estimated_event_update_count",
1,
{ scope: "trace", strategy: "project" },
],
[
"langfuse.trace_batch.estimated_serialized_event_bytes",
30,
{ scope: "trace", strategy: "project" },
],
]),
);
for (const [scope, count] of [
["trace", 3],
["batch", 1],
] as const) {
expect(
vi
.mocked(recordIncrement)
.mock.calls.filter(
([name, , tags]) =>
name === "langfuse.trace_batch.estimates_unavailable" &&
tags?.scope === scope &&
tags?.strategy === "project",
)
.reduce((total, [, value]) => total + Number(value), 0),
).toBe(count);
}
expect(await client().hlen(stateKey)).toBe(0);
vi.clearAllMocks();
await trackTraceBatchActivity(
"project",
legacyIds.map((traceId) => event(traceId, 1_000_002, 20)),
);
for (const traceId of legacyIds) {
const state = JSON.parse(
(await client().hget(stateKey, member("project", traceId)))!,
);
expect(state.eventUpdateCount).toBeUndefined();
expect(state.serializedEventBytes).toBeUndefined();
}
await makeDue("project", ...legacyIds);
await dispatcher.processBatch();
expect(
vi
.mocked(recordDistribution)
.mock.calls.filter(([name]) =>
name.startsWith("langfuse.trace_batch.estimated_"),
),
).toHaveLength(0);
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.trace_batch.estimates_unavailable",
1,
{ scope: "batch", strategy: "project" },
);
});
it.each(["project", "locality"] as const)(
"dispatches the complete due cohort across hydration chunks with %s",
async (strategy) => {
env.LANGFUSE_TRACE_BATCH_STRATEGY = strategy;
const due = Date.now() - 100_000;
const traces = Array.from({ length: 10_061 }, (_, i) => ({
projectId: ["project-c", "project-a", "project-b"][i % 3],
traceId: `trace-${String(i).padStart(5, "0")}`,
due: due + Math.floor(i / 2_000) * 1_000,
}));
for (const projectId of ["project-c", "project-a", "project-b"]) {
await trackTraceBatchActivity(
projectId,
traces
.filter((trace) => trace.projectId === projectId)
.map((trace) => event(trace.traceId)),
);
}
await client().zadd(
dueKey,
...traces.flatMap((trace) => [
trace.due,
member(trace.projectId, trace.traceId),
]),
);
const add = vi.spyOn(queue, "add");
await runner().processBatch();
const batches = add.mock.calls.map(([, job]) => job.payload.traces);
const assigned = batches
.flat()
.map((trace) => member(trace.projectId, trace.traceId));
expect(assigned).toHaveLength(traces.length);
expect(new Set(assigned)).toEqual(
new Set(traces.map((trace) => member(trace.projectId, trace.traceId))),
);
expect(batches.every((batch) => batch.length <= 60)).toBe(true);
const candidateSizes = vi
.mocked(recordDistribution)
.mock.calls.filter(
([name]) => name === "langfuse.trace_batch.candidate_buffer_size",
)
.map(([, value]) => Number(value));
expect(Math.max(...candidateSizes)).toBeLessThanOrEqual(1059);
if (strategy === "project") {
expect(batches.map((batch) => batch.length)).toEqual([
...Array(167).fill(60),
41,
]);
const expected = traces.sort(
(a, b) =>
a.projectId.localeCompare(b.projectId) ||
a.due - b.due ||
a.traceId.localeCompare(b.traceId),
);
expect(
batches.flat().map((trace) => member(trace.projectId, trace.traceId)),
).toEqual(
expected.map((trace) => member(trace.projectId, trace.traceId)),
);
} else {
expect(batches.length).toBeGreaterThanOrEqual(
Math.ceil(traces.length / 60),
);
expect(batches.length).toBeLessThanOrEqual(
Math.ceil(traces.length / 1000) * Math.ceil(1000 / 60),
);
}
expect(await client().zcard(dueKey)).toBe(0);
expect(await client().hlen(stateKey)).toBe(0);
},
);
it("keeps project locality and a bounded partial tail across hydration chunks", async () => {
env.LANGFUSE_TRACE_BATCH_STRATEGY = "locality";
const traces = Array.from({ length: 2_001 }, (_, index) => ({
projectId: index % 2 === 0 ? "project-a" : "project-b",
traceId: `trace-${String(index).padStart(4, "0")}`,
}));
for (const projectId of ["project-a", "project-b"]) {
await trackTraceBatchActivity(
projectId,
traces
.filter((trace) => trace.projectId === projectId)
.map((trace) => event(trace.traceId)),
);
}
const due = Date.now() - 1_000;
await client().zadd(
dueKey,
...traces.flatMap((trace) => [
due,
member(trace.projectId, trace.traceId),
]),
);
const add = vi.spyOn(queue, "add");
await runner().processBatch();
const batches = add.mock.calls.map(([, job]) => job.payload.traces);
expect(batches.length).toBeGreaterThanOrEqual(
Math.ceil(traces.length / 60),
);
expect(batches.length).toBeLessThanOrEqual(2 * Math.ceil(1_000 / 60) + 1);
expect(
batches.every((batch) => batch.length > 0 && batch.length <= 60),
).toBe(true);
expect(
batches.filter(
(batch) => new Set(batch.map((trace) => trace.projectId)).size > 1,
),
).toHaveLength(0);
const dispatchedMembers = batches
.flat()
.map((trace) => member(trace.projectId, trace.traceId));
expect(dispatchedMembers).toHaveLength(traces.length);
expect(new Set(dispatchedMembers)).toEqual(
new Set(traces.map((trace) => member(trace.projectId, trace.traceId))),
);
expect(
vi
.mocked(recordIncrement)
.mock.calls.filter(
([name]) => name === "langfuse.trace_batch.skipped_traces",
)
.map(([, value]) => value),
).toEqual([0, 0, 0]);
expect(await queue.getWaitingCount()).toBe(batches.length);
expect(await client().zcard(dueKey)).toBe(0);
expect(await client().hlen(stateKey)).toBe(0);
});
it("coalesces a non-final compatible partial across a hydration boundary", async () => {
env.LANGFUSE_TRACE_BATCH_STRATEGY = "locality";
const minute = 60_000;
const early = Array.from({ length: 30 }, (_, index) => ({
traceId: `early-${String(index).padStart(2, "0")}`,
start: 10 * minute,
}));
const filler = Array.from({ length: 960 }, (_, index) => ({
traceId: `filler-${String(index).padStart(3, "0")}`,
start: 12 * 60 * minute,
}));
const late = Array.from({ length: 12 }, (_, index) => ({
traceId: `late-${String(index).padStart(2, "0")}`,
start: 24 * 60 * minute,
}));
const firstWindow = [...early.slice(0, 28), ...filler, ...late];
const traces = [...firstWindow, ...early.slice(28)];
await trackTraceBatchActivity(
"project",
traces.map(({ traceId, start }) => event(traceId, start)),
);
const due = Date.now() - 10_000;
await client().zadd(
dueKey,
...traces.flatMap(({ traceId }, index) => [
due + index,
member("project", traceId),
]),
);
const targetMember = member("project", early[0].traceId);
const hydratedRevision = JSON.parse(
(await client().hget(stateKey, targetMember))!,
).revision;
const evaluate = client().evalsha.bind(client());
let reactivated = false;
vi.spyOn(client(), "evalsha").mockImplementation(async (...args) => {
const result = await evaluate(...args);
if (!reactivated && Number(args[1]) === 2 && args.length > 6) {
reactivated = true;
await trackTraceBatchActivity("project", [
event(early[0].traceId, 11 * minute),
]);
}
return result;
});
const add = vi.spyOn(queue, "add");
await runner().processBatch();
expect(reactivated).toBe(true);
const batches = add.mock.calls.map(([, job]) => job.payload.traces);
expect(
batches
.filter((batch) =>
batch.some(({ traceId }) => traceId.startsWith("early-")),
)
.map((batch) => batch.map(({ traceId }) => traceId).sort()),
).toEqual([early.map(({ traceId }) => traceId)]);
expect(batches.map((batch) => batch.length).sort((a, b) => a - b)).toEqual([
12,
30,
...Array(16).fill(60),
]);
const assigned = batches
.flatMap((batch) => batch)
.map(({ projectId, traceId }) => member(projectId, traceId));
expect(assigned).toHaveLength(traces.length);
expect(new Set(assigned)).toEqual(
new Set(traces.map(({ traceId }) => member("project", traceId))),
);
const candidateBufferSizes = vi
.mocked(recordDistribution)
.mock.calls.filter(
([name]) => name === "langfuse.trace_batch.candidate_buffer_size",
)
.map(([, value]) => Number(value));
expect(candidateBufferSizes).toContain(42);
expect(Math.max(...candidateBufferSizes)).toBeLessThanOrEqual(1_059);
const current = JSON.parse((await client().hget(stateKey, targetMember))!);
const dispatched = batches
.flat()
.find(({ traceId }) => traceId === early[0].traceId);
expect(dispatched?.revision).toBe(hydratedRevision);
expect(current.revision).not.toBe(hydratedRevision);
expect(current.maxStart).toBe(11 * minute);
expect(await client().zcard(dueKey)).toBe(1);
expect(await client().hlen(stateKey)).toBe(1);
await makeDue("project", early[0].traceId);
await runner().processBatch();
expect(await queue.getWaitingCount()).toBe(batches.length + 1);
expect(await client().zcard(dueKey)).toBe(0);
expect(await client().hlen(stateKey)).toBe(0);
});
it("revalidates the due ID list after deletion, missing state, reactivation and new arrivals", async () => {
const traces = Array.from(
{ length: 2_001 },
(_, i) => `trace-${String(i).padStart(5, "0")}`,
);
await trackTraceBatchActivity(
"project",
traces.map((id) => event(id)),
);
const due = Date.now() - 1_000;
await client().zadd(
dueKey,
...traces.flatMap((id) => [due, member("project", id)]),
);
const range = client().zrange.bind(client());
let changed = false;
vi.spyOn(client(), "zrange").mockImplementation(async (...args) => {
const result = await range(...args);
if (!changed && args.includes("BYSCORE")) {
changed = true;
// Change Redis after the complete ID list has been read but before hydration.
await client().zrem(dueKey, member("project", traces[0]));
await client().hdel(stateKey, member("project", traces[0]));
await client().hdel(stateKey, member("project", traces[1_000]));
env.LANGFUSE_TRACE_BATCH_IDLE_MS = 1;
await trackTraceBatchActivity("project", [
event(traces[999]),
event("new-after-cutoff"),
]);
}
return result;
});
const add = vi.spyOn(queue, "add");
await runner().processBatch();
expect(changed).toBe(true);
expect(
add.mock.calls.flatMap(([, job]) =>
job.payload.traces.map((trace) => trace.traceId),
),
).toEqual(traces.filter((_, i) => i !== 0 && i !== 999 && i !== 1_000));
expect(
await client().zscore(dueKey, member("project", traces[1_000])),
).toBeNull();
expect((await client().hkeys(stateKey)).sort()).toEqual(
[
member("project", "new-after-cutoff"),
member("project", traces[999]),
].sort(),
);
});
it("expires abandoned pending keys natively without another ingestion or dispatcher run", async () => {
env.LANGFUSE_TRACE_BATCH_IDLE_MS = 100;
env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 300;
await trackTraceBatchActivity("project", [event("abandoned")]);
for (const key of [dueKey, stateKey]) {
expect(await client().pttl(key)).toBeGreaterThan(0);
expect(await client().pttl(key)).toBeLessThanOrEqual(400);
}
await vi.waitFor(async () => {
expect(await client().exists(dueKey, stateKey)).toBe(0);
});
expect(await queue.getWaitingCount()).toBe(0);
});
it("extends native expiry for later activity and never shortens another pending trace's deadline", async () => {
env.LANGFUSE_TRACE_BATCH_IDLE_MS = 10_000;
env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 60_000;
await trackTraceBatchActivity("project", [event("long-idle")]);
const initialDeadline = Number(await client().pexpiretime(dueKey));
await vi.waitFor(() =>
expect(Date.now()).toBeGreaterThan(initialDeadline - 70_000 + 10),
);
env.LANGFUSE_TRACE_BATCH_IDLE_MS = 20_000;
await trackTraceBatchActivity("project", [event("later")]);
const extendedDeadline = Number(await client().pexpiretime(dueKey));
expect(extendedDeadline).toBeGreaterThan(initialDeadline);
expect(Number(await client().pexpiretime(stateKey))).toBe(extendedDeadline);
env.LANGFUSE_TRACE_BATCH_IDLE_MS = 1_000;
env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 1_000;
await trackTraceBatchActivity("project", [event("short-idle")]);
expect(Number(await client().pexpiretime(dueKey))).toBeGreaterThanOrEqual(
extendedDeadline,
);
expect(Number(await client().pexpiretime(stateKey))).toBe(
Number(await client().pexpiretime(dueKey)),
);
expect(await client().zcard(dueKey)).toBe(3);
expect(await client().hlen(stateKey)).toBe(3);
const roundedDeadline =
Math.ceil((Date.now() + 4 * 60 * 60_000) / 100_000_000) * 100_000_000;
for (const key of [dueKey, stateKey])
await client().pexpireat(key, roundedDeadline);
vi.mocked(recordIncrement).mockClear();
await trackTraceBatchActivity("project", [event("rounded-expiry")]);
expect(recordIncrement).not.toHaveBeenCalledWith(
"langfuse.trace_batch.tracking_errors",
expect.anything(),
);
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.trace_batch.event_updates",
1,
{ stage: "recorded" },
);
for (const key of [dueKey, stateKey])
expect(Number(await client().pexpiretime(key))).toBe(roundedDeadline);
});
it("drains and removes queued jobs without queries when reads are disabled", async () => {
env.LANGFUSE_TRACE_BATCH_READ_ENABLED = "false";
env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "false";
const read = vi
.spyOn(shared, "getTraceBatchEventStream")
.mockImplementation(async function* () {
throw new Error("Drain mode must not query ClickHouse");
});
const payload = {
traces: [
{
projectId: "project",
traceId: "trace",
minStart: 0,
maxStart: 1,
revision: "r",
},
],
};
const current = await queue.add(
QueueJobs.TraceBatch,
{
id: "current",
timestamp: new Date(),
name: QueueJobs.TraceBatch,
payload,
},
{ jobId: "current", removeOnComplete: { age: 3600, count: 10000 } },
);
const old = await queue.add(
QueueJobs.TraceBatch,
{
id: "old",
timestamp: new Date(Date.now() - 3 * 60 * 60_000),
name: QueueJobs.TraceBatch,
payload,
},
{ jobId: "old", removeOnComplete: { age: 3600, count: 10000 } },
);
const worker = new Worker(queue.name, traceBatchQueueProcessor, {
connection,
prefix: getQueuePrefix(queue.name),
});
try {
await vi.waitFor(async () => {
expect(await queue.getJob(current.id!)).toBeUndefined();
expect(await queue.getJob(old.id!)).toBeUndefined();
});
expect(
await queue.getJobCounts("wait", "active", "completed", "failed"),
).toEqual({ wait: 0, active: 0, completed: 0, failed: 0 });
expect(read).not.toHaveBeenCalled();
} finally {
await worker.close();
}
});
it("expires bounded pending state during ingestion without dispatch and preserves refreshed traces", async () => {
env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 60_000;
const expired = Array.from({ length: 1_001 }, (_, i) =>
event(`expired-${i}`),
);
await trackTraceBatchActivity("project", [
...expired,
event("refreshed"),
event("recent"),
]);
const cutoff = Date.now() - 70_000;
await client().zadd(
dueKey,
...expired.flatMap(({ traceId }) => [cutoff, member("project", traceId)]),
cutoff,
member("project", "refreshed"),
);
await makeDue("project", "recent");
await trackTraceBatchActivity("project", [event("refreshed", 2_000_000)]);
const pending = await client().zcard(dueKey);
expect(pending).toBeGreaterThan(2);
expect(pending).toBeLessThan(expired.length + 2);
expect(await client().hlen(stateKey)).toBe(pending);
expect(
await client().hget(stateKey, member("project", "recent")),
).not.toBeNull();
expect(
Number(await client().zscore(dueKey, member("project", "refreshed"))),
).toBeGreaterThan(Date.now());
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.trace_batch.expired_traces",
expired.length + 2 - pending,
{ source: "ingestion" },
);
await vi.waitFor(async () => {
await trackTraceBatchActivity("project", [event("refreshed", 3_000_000)]);
expect(await client().zcard(dueKey)).toBe(2);
});
expect((await client().hkeys(stateKey)).sort()).toEqual([
member("project", "recent"),
member("project", "refreshed"),
]);
expect(await client().zcard(dueKey)).toBe(2);
const expirySamples = vi
.mocked(recordIncrement)
.mock.calls.filter(
([name, , tags]) =>
name === "langfuse.trace_batch.expired_traces" &&
tags?.source === "ingestion",
)
.map(([, count]) => count);
expect(expirySamples.reduce((sum, count) => sum + count, 0)).toBe(
expired.length,
);
expect(expirySamples.every((count) => count < expired.length)).toBe(true);
expect(await queue.getWaitingCount()).toBe(0);
});
it("cleans an expired backlog in bounded pages without queueing expired traces", async () => {
env.LANGFUSE_TRACE_BATCH_PENDING_TTL_MS = 60_000;
const expired = Array.from({ length: 10_001 }, (_, i) =>
event(`expired-${i}`),
);
await trackTraceBatchActivity("project", [...expired, event("ready")]);
await client().zadd(
dueKey,
...expired.flatMap(({ traceId }) => [
Date.now() - 70_000,
member("project", traceId),
]),
);
await makeDue("project", "ready");
env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED = "false";
const dispatcher = runner();
await dispatcher.processBatch();
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.trace_batch.expired_traces",
expect.any(Number),
{ source: "dispatcher" },
);
expect(
vi
.mocked(recordIncrement)
.mock.calls.filter(
([name]) => name === "langfuse.trace_batch.expired_traces",
)
.every(([, count]) => count <= 1_000),
).toBe(true);
const jobs = await queue.getJobs(["wait"]);
expect(
jobs.flatMap((job) =>
job.data.payload.traces.map((trace) => trace.traceId),
),
).toEqual(["ready"]);
expect(await client().hlen(stateKey)).toBe(0);
expect(await client().zcard(dueKey)).toBe(0);
expect(
vi
.mocked(recordIncrement)
.mock.calls.filter(
([name]) => name === "langfuse.trace_batch.expired_traces",
)
.reduce((sum, [, value]) => sum + value, 0),
).toBe(10_001);
});
it.each(["project", "locality"] as const)(
"keeps due state on enqueue failure and retries it with %s",
async (strategy) => {
env.LANGFUSE_TRACE_BATCH_STRATEGY = strategy;
await trackTraceBatchActivity("project", [event("trace")]);
await makeDue("project", "trace");
vi.spyOn(queue, "add").mockRejectedValueOnce(
new Error("enqueue unavailable"),
);
const dispatcher = runner();
await dispatcher.processBatch();
expect(await client().zcard(dueKey)).toBe(1);
expect(await client().hlen(stateKey)).toBe(1);
await dispatcher.processBatch();
expect(await queue.getWaitingCount()).toBe(1);
expect(await client().zcard(dueKey)).toBe(0);
},
);
it("acknowledges successful jobs when a later enqueue fails", async () => {
env.LANGFUSE_TRACE_BATCH_MAX_SIZE = 1;
await trackTraceBatchActivity("project", [event("first"), event("second")]);
await makeDue("project", "first", "second");
const add = queue.add.bind(queue);
vi.spyOn(queue, "add")
.mockImplementationOnce((...args) => add(...args))
.mockRejectedValueOnce(new Error("enqueue unavailable"));
const dispatcher = runner();
await dispatcher.processBatch();
expect(await queue.getWaitingCount()).toBe(1);
expect(await client().zcard(dueKey)).toBe(1);
expect(await client().hlen(stateKey)).toBe(1);
await dispatcher.processBatch();
expect(await queue.getWaitingCount()).toBe(2);
expect(await client().zcard(dueKey)).toBe(0);
expect(await client().hlen(stateKey)).toBe(0);
});
it.each(["updated", "deleted", "expired"] as const)(
"preserves arrivals during enqueue, including deleted/recreated state: %s",
async (mode) => {
await trackTraceBatchActivity("project", [event("trace")]);
await makeDue("project", "trace");
const add = queue.add.bind(queue);
vi.spyOn(queue, "add").mockImplementationOnce(async (...args) => {
const job = await add(...args);
if (mode === "deleted")
await client().hdel(stateKey, member("project", "trace"));
if (mode === "expired") {
await client().zadd(dueKey, 0, member("project", "trace"));
await trackTraceBatchActivity("other", [event("trigger")]);
expect(
await client().hget(stateKey, member("project", "trace")),
).toBeNull();
await client().zrem(dueKey, member("other", "trigger"));
await client().hdel(stateKey, member("other", "trigger"));
}
await trackTraceBatchActivity("project", [event("trace", 2_000_000)]);
return job;
});
const dispatcher = runner();
await dispatcher.processBatch();
expect(await queue.getWaitingCount()).toBe(1);
expect(await client().zcard(dueKey)).toBe(1);
const current = JSON.parse(
(await client().hget(stateKey, member("project", "trace")))!,
);
const [job] = await queue.getJobs(["wait"]);
expect(current.revision).not.toBe(job.data.payload.traces[0].revision);
expect(current.maxStart).toBe(2_000_000);
expect(current).toMatchObject({
eventUpdateCount: 2,
serializedEventBytes: 200,
});
await makeDue("project", "trace");
await dispatcher.processBatch();
expect(await queue.getWaitingCount()).toBe(2);
expect(await client().hlen(stateKey)).toBe(0);
},
);
it("reuses a queued batch when its acknowledgement fails", async () => {
await trackTraceBatchActivity("project", [event("trace")]);
await makeDue("project", "trace");
await trackTraceBatchActivity("other", [event("trace")]);
await makeDue("other", "trace");
const add = queue.add.bind(queue);
vi.spyOn(queue, "add").mockImplementationOnce(async (...args) => {
const job = await add(...args);
// The next Redis operation is the post-enqueue acknowledgement.
vi.spyOn(client(), "evalsha").mockRejectedValueOnce(
new Error("ACK unavailable"),
);
return job;
});
const dispatcher = runner();
await dispatcher.processBatch();
expect(await queue.getWaitingCount()).toBe(1);
expect(await client().hlen(stateKey)).toBe(2);
await dispatcher.processBatch();
expect(await queue.getWaitingCount()).toBe(1);
expect(await client().hlen(stateKey)).toBe(0);
const [job] = await queue.getJobs(["wait"]);
expect(
job.data.payload.traces
.map((trace) => member(trace.projectId, trace.traceId))
.sort(),
).toEqual([member("other", "trace"), member("project", "trace")]);
});
it("subtracts dispatcher runtime from the next delay and catches up after an overrun", async () => {
let clock = Date.now();
const interval = env.LANGFUSE_TRACE_BATCH_DISPATCH_INTERVAL_MS;
let elapsed = Math.floor(interval / 2);
vi.spyOn(Date, "now").mockImplementation(() => clock);
const range = client().zrange.bind(client());
vi.spyOn(client(), "zrange").mockImplementation(async (...args) => {
const result = await range(...args);
if (args.includes("BYSCORE")) clock += elapsed;
return result;
});
const dispatcher = runner();
expect(await dispatcher.processBatch()).toBe(interval - elapsed);
elapsed = interval + 1_000;
expect(await dispatcher.processBatch()).toBe(0);
});
it("bounds shutdown when an in-flight enqueue stalls and remains stopped after recovery", async () => {
await trackTraceBatchActivity("project", [event("trace")]);
await makeDue("project", "trace");
const entered = Promise.withResolvers<void>();
const release = Promise.withResolvers<void>();
const add = queue.add.bind(queue);
vi.spyOn(queue, "add").mockImplementationOnce(async (...args) => {
entered.resolve();
await release.promise;
return add(...args);
});
const dispatcher = runner();
const active = dispatcher.processBatch();
await entered.promise;
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
let drained = false;
const drain = dispatcher.drain().then(() => {
drained = true;
});
try {
await vi.advanceTimersByTimeAsync(4_999);
expect(drained).toBe(false);
await vi.advanceTimersByTimeAsync(1);
expect(drained).toBe(true);
} finally {
vi.useRealTimers();
release.resolve();
await Promise.all([active, drain]);
}
await trackTraceBatchActivity("project", [event("next")]);
await makeDue("project", "next");
await dispatcher.processBatch();
expect(queue.add).toHaveBeenCalledTimes(1);
expect(await client().hlen(stateKey)).toBe(1);
});
it("allows only one dispatcher and waits for its in-flight enqueue before shutdown", async () => {
env.LANGFUSE_TRACE_BATCH_MAX_SIZE = 1;
const traces = ["trace"];
await trackTraceBatchActivity(
"project",
traces.map((traceId) => event(traceId)),
);
await makeDue("project", ...traces);
await trackTraceBatchActivity("other", [event("trace")]);
await makeDue("other", "trace");
const entered = Promise.withResolvers<void>();
const release = Promise.withResolvers<void>();
const add = queue.add.bind(queue);
vi.spyOn(queue, "add").mockImplementationOnce(async (...args) => {
entered.resolve();
await release.promise;
return add(...args);
});
const first = runner();
const active = first.processBatch();
await entered.promise;
await runner().processBatch();
expect(queue.add).toHaveBeenCalledTimes(1);
let drained = false;
const drain = first.drain().then(() => {
drained = true;
});
await Promise.resolve();
expect(drained).toBe(false);
release.resolve();
await Promise.all([active, drain]);
expect(drained).toBe(true);
expect(await queue.getWaitingCount()).toBe(1);
expect(await client().hlen(stateKey)).toBe(1);
await first.processBatch();
expect(queue.add).toHaveBeenCalledTimes(1);
});
});