🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
handlers: new Map<string, (...args: unknown[]) => void>(),
processor: undefined as
| ((job: {
data: unknown;
timestamp: number;
attemptsMade?: number;
attemptsStarted?: number;
}) => Promise<unknown>)
| undefined,
legacyRecordGauge: vi.fn(),
legacyRecordHistogram: vi.fn(),
recordDistribution: vi.fn(),
recordIncrement: vi.fn(),
}));
vi.mock("bullmq", () => ({
Job: class {},
Worker: class {
constructor(
_queueName: string,
processor: (job: {
data: unknown;
timestamp: number;
}) => Promise<unknown>,
) {
mocks.processor = processor;
}
close = vi.fn(async () => undefined);
isRunning = vi.fn(() => true);
on = vi.fn((event: string, handler: (...args: unknown[]) => void) => {
mocks.handlers.set(event, handler);
});
},
}));
vi.mock("@opentelemetry/api", () => ({
context: {
with: (_context: unknown, callback: () => unknown) => callback(),
},
createContextKey: (description: string) => Symbol.for(description),
}));
vi.mock("@langfuse/shared/src/server", async () => {
const { isS3SlowDownError } =
await import("../../../packages/shared/src/server/services/s3ThrottleError");
return {
QueueName: {
TraceDelete: "trace-delete",
},
contextWithLangfuseProps: vi.fn(() => ({})),
convertQueueNameToMetricName: (queueName: string) =>
`langfuse.queue.${queueName.replace(/-/g, "_").replace(/_queue$/, "")}`,
createBullMQWorkerOptionsWithRedis: vi.fn(() => ({ connection: {} })),
isS3SlowDownError,
logger: {
debug: vi.fn(),
error: vi.fn(),
info: vi.fn(),
warn: vi.fn(),
},
recordGauge: mocks.legacyRecordGauge,
recordHistogram: mocks.legacyRecordHistogram,
recordDistribution: mocks.recordDistribution,
recordIncrement: mocks.recordIncrement,
traceException: vi.fn(),
};
});
vi.mock("../queues/shardedQueueRegistry", () => ({
resolveQueueInstance: vi.fn(() => ({
getActiveCount: vi.fn(async () => 0),
getFailed: vi.fn(async () => []),
getFailedCount: vi.fn(async () => 0),
getWaitingCount: vi.fn(async () => 0),
})),
SHARDED_QUEUE_BASE_NAMES: new Set(),
}));
vi.mock("../utils/hostId", () => ({
WORKER_HOST_ID: "test-host",
}));
import { logger } from "@langfuse/shared/src/server";
import { WorkerManager } from "../queues/workerManager";
describe("WorkerManager queue metrics", () => {
beforeEach(() => {
mocks.handlers.clear();
mocks.processor = undefined;
vi.clearAllMocks();
});
it("emits only tagged rate and time metrics", async () => {
WorkerManager.register("trace-delete" as never, async () => "processed");
expect(mocks.processor).toBeDefined();
await expect(
mocks.processor!({
data: { payload: { projectId: "project-id" } },
timestamp: Date.now() - 50,
}),
).resolves.toBe("processed");
await Promise.resolve();
expect(mocks.recordIncrement.mock.calls).toEqual([
[
"langfuse.queue.trace_delete.rate",
1,
{ type: "request", attempt: "first" },
],
]);
expect(
mocks.recordDistribution.mock.calls.map(([metric, _value, tags]) => [
metric,
tags,
]),
).toEqual([
[
"langfuse.queue.trace_delete.time_distribution",
{ type: "wait", attempt: "first", unit: "milliseconds" },
],
[
"langfuse.queue.trace_delete.time_distribution",
{ type: "processing", unit: "milliseconds" },
],
]);
expect(mocks.legacyRecordGauge).not.toHaveBeenCalled();
expect(mocks.legacyRecordHistogram).not.toHaveBeenCalled();
mocks.recordIncrement.mockClear();
mocks.handlers.get("failed")?.(
{ id: "job-id", name: "job" },
new Error("failed"),
);
mocks.handlers.get("error")?.(new Error("errored"));
mocks.handlers.get("stalled")?.("job-id");
expect(mocks.recordIncrement.mock.calls).toEqual([
[
"langfuse.queue.trace_delete.rate",
1,
{ type: "failed", reason: "other" },
],
["langfuse.queue.trace_delete.rate", 1, { type: "error" }],
["langfuse.queue.trace_delete.rate", 1, { type: "stalled" }],
]);
});
it("tags a retry's request and records no wait sample for it", async () => {
// Own queue name: register() no-ops on the statically registered one above.
WorkerManager.register("dataset-delete" as never, async () => "processed");
await mocks.processor!({
data: { payload: { projectId: "project-id" } },
timestamp: Date.now() - 3_600_000,
attemptsMade: 1,
attemptsStarted: 2,
});
expect(mocks.recordIncrement.mock.calls).toEqual([
[
"langfuse.queue.dataset_delete.rate",
1,
{ type: "request", attempt: "retry" },
],
]);
expect(
mocks.recordDistribution.mock.calls.map(([, , tags]) => tags.type),
).toEqual(["processing"]);
});
it("emits a rate counter for completed jobs but not for active ones", () => {
// Own queue name: register() no-ops on the statically registered one above.
WorkerManager.register("trace-upsert" as never, async () => "processed");
mocks.handlers.get("active")?.({ id: "job-id", name: "job" });
mocks.handlers.get("completed")?.({ id: "job-id", name: "job" });
expect(mocks.recordIncrement.mock.calls).toEqual([
["langfuse.queue.trace_upsert.rate", 1, { type: "completed" }],
]);
});
it("counts failed_terminal only once BullMQ moved the job to the failed set", () => {
WorkerManager.register("project-delete" as never, async () => undefined);
// BullMQ sets finishedOn only when it stops retrying.
mocks.handlers.get("failed")?.(
{ id: "job-id", name: "job", attemptsMade: 1, opts: { attempts: 5 } },
new Error("retrying"),
);
mocks.handlers.get("failed")?.(
{
id: "job-id",
name: "job",
attemptsMade: 5,
opts: { attempts: 5 },
finishedOn: Date.now(),
},
new Error("exhausted"),
);
expect(mocks.recordIncrement.mock.calls).toEqual([
[
"langfuse.queue.project_delete.rate",
1,
{ type: "failed", reason: "other" },
],
[
"langfuse.queue.project_delete.rate",
1,
{ type: "failed", reason: "other" },
],
[
"langfuse.queue.project_delete.rate",
1,
{ type: "failed_terminal", reason: "other" },
],
]);
});
it("logs retryable S3 SlowDown as a warning until retries are exhausted", () => {
WorkerManager.register("score-delete" as never, async () => undefined);
const slowDown = new Error("Failed to download file from S3", {
cause: Object.assign(new Error("Please reduce your request rate."), {
name: "SlowDown",
}),
});
vi.mocked(logger.warn).mockClear();
vi.mocked(logger.error).mockClear();
mocks.handlers.get("failed")?.(
{
id: "job-id",
name: "job",
attemptsMade: 1,
opts: { attempts: 6 },
},
slowDown,
);
expect(logger.warn).toHaveBeenCalled();
expect(logger.error).not.toHaveBeenCalled();
expect(mocks.recordIncrement).toHaveBeenCalledWith(
"langfuse.queue.score_delete.rate",
1,
{ type: "failed", reason: "s3_slowdown" },
);
mocks.handlers.get("failed")?.(
{
id: "job-id",
name: "job",
attemptsMade: 6,
opts: { attempts: 6 },
},
slowDown,
);
expect(logger.error).toHaveBeenCalled();
});
});