🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { Job } from "bullmq";
const {
mockCronJobsUpsert,
mockCronJobsUpdate,
mockOrganizationFindMany,
mockMeterEventsCreate,
mockUsageMeteringQueueAdd,
mockSpendAlertQueueAdd,
mockObservationCounts,
mockTraceCounts,
mockScoreCounts,
mockLoggerInfo,
mockLoggerWarn,
mockLoggerError,
mockRecordIncrement,
mockTraceException,
mockCompareWithBillingLedger,
mockEnv,
} = vi.hoisted(() => ({
mockCronJobsUpsert: vi.fn(),
mockCronJobsUpdate: vi.fn(),
mockOrganizationFindMany: vi.fn(),
mockMeterEventsCreate: vi.fn(),
mockUsageMeteringQueueAdd: vi.fn(),
mockSpendAlertQueueAdd: vi.fn(),
mockObservationCounts: vi.fn(),
mockTraceCounts: vi.fn(),
mockScoreCounts: vi.fn(),
mockLoggerInfo: vi.fn(),
mockLoggerWarn: vi.fn(),
mockLoggerError: vi.fn(),
mockRecordIncrement: vi.fn(),
mockTraceException: vi.fn(),
mockCompareWithBillingLedger: vi.fn(),
mockEnv: {
STRIPE_SECRET_KEY: "sk_test",
LANGFUSE_CLOUD_BILLING_CHB_CUTOFF_DATE: undefined,
LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED: "false",
},
}));
vi.mock("@langfuse/shared", () => ({
getBillingProvider: () => "stripe",
parseDbOrg: (org: unknown) => org,
Prisma: { DbNull: "DbNull" },
}));
vi.mock("@langfuse/shared/src/db", () => ({
prisma: {
cronJobs: {
upsert: mockCronJobsUpsert,
update: mockCronJobsUpdate,
},
organization: {
findMany: mockOrganizationFindMany,
},
},
}));
vi.mock("@langfuse/shared/src/server", () => ({
CloudUsageMeteringQueue: {
getInstance: () => ({ add: mockUsageMeteringQueueAdd }),
},
CloudSpendAlertQueue: {
getInstance: () => ({ add: mockSpendAlertQueueAdd }),
},
compareWithBillingLedger: mockCompareWithBillingLedger,
getObservationCountsByProjectInCreationInterval: mockObservationCounts,
getTraceCountsByProjectInCreationInterval: mockTraceCounts,
getScoreCountsByProjectInCreationInterval: mockScoreCounts,
logger: {
info: mockLoggerInfo,
warn: mockLoggerWarn,
error: mockLoggerError,
},
QueueJobs: {
CloudUsageMeteringJob: "cloud-usage-metering-job",
CloudSpendAlertJob: "cloud-spend-alert-job",
},
recordIncrement: mockRecordIncrement,
traceException: mockTraceException,
}));
vi.mock("stripe", () => ({
default: class {
billing = { meterEvents: { create: mockMeterEventsCreate } };
},
}));
vi.mock("../../../env", () => ({ env: mockEnv }));
import { handleCloudUsageMeteringJob } from "../handleCloudUsageMeteringJob";
import { CloudUsageMeteringDbCronJobStates } from "../constants";
// Mirrors the prod-eu incident: a run claims the 08:00-09:00 interval, dies
// partway through, and the next hourly tick takes the same interval over.
const NOW = new Date("2026-08-03T10:30:00.000Z");
const INTERVAL_START = new Date("2026-08-03T08:00:00.000Z");
const createJob = () =>
({
updateProgress: vi.fn(),
}) as unknown as Job;
const identifiersFromCalls = () =>
mockMeterEventsCreate.mock.calls.map(([params]) => params.identifier);
const callsForMeter = (eventName: string) =>
mockMeterEventsCreate.mock.calls.filter(
([params]) => params.event_name === eventName,
);
const closingUpdate = () =>
mockCronJobsUpdate.mock.calls.find(
([args]) => args.data.lastRun !== undefined,
);
// Shape taken from the prod-eu rejection: a 400 invalid_request_error whose only
// machine-readable marker is the message. Stripe sets stripe-should-retry: false.
const stripeInvalidRequestError = (message: string) =>
Object.assign(new Error(message), {
type: "StripeInvalidRequestError",
rawType: "invalid_request_error",
statusCode: 400,
});
const duplicateIdentifierError = (identifier: string) =>
stripeInvalidRequestError(
`An event already exists with identifier ${identifier}.`,
);
describe("handleCloudUsageMeteringJob", () => {
beforeEach(() => {
vi.clearAllMocks();
vi.useFakeTimers({ shouldAdvanceTime: true });
vi.setSystemTime(NOW);
mockEnv.LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED = "false";
mockCronJobsUpsert.mockResolvedValue({
name: "cloud_usage_metering",
lastRun: INTERVAL_START,
state: CloudUsageMeteringDbCronJobStates.Queued,
jobStartedAt: null,
});
mockCronJobsUpdate.mockResolvedValue({});
mockMeterEventsCreate.mockResolvedValue({});
mockOrganizationFindMany.mockResolvedValue([
{
id: "org-1",
cloudConfig: { stripe: { customerId: "cus_1" } },
projects: [{ id: "project-1" }],
cloudSpendAlerts: [],
},
]);
mockObservationCounts.mockResolvedValue([
{ projectId: "project-1", count: 5 },
]);
mockTraceCounts.mockResolvedValue([{ projectId: "project-1", count: 3 }]);
mockScoreCounts.mockResolvedValue([{ projectId: "project-1", count: 2 }]);
});
afterEach(() => {
vi.useRealTimers();
});
it("resolves the same meter event identifier when an interval is replayed", async () => {
await handleCloudUsageMeteringJob(createJob());
const firstRun = identifiersFromCalls();
expect(firstRun).toHaveLength(2);
expect(
firstRun.every((id) => typeof id === "string" && id.length > 0),
).toBe(true);
// lastRun never advanced, so the replay recomputes the same interval from
// the top. Identical identifiers are what lets Stripe drop the duplicates
// instead of summing them into the customer's usage.
mockMeterEventsCreate.mockClear();
await handleCloudUsageMeteringJob(createJob());
expect(identifiersFromCalls()).toEqual(firstRun);
});
it("derives the identifier from the meter, the org and the interval", async () => {
await handleCloudUsageMeteringJob(createJob());
const byMeter = new Map(
mockMeterEventsCreate.mock.calls.map(([params]) => [
params.event_name,
params.identifier,
]),
);
// Both meters are reported for the same org and interval, so event_name has
// to be part of the key or Stripe would dedup them against each other and
// silently drop one.
expect(byMeter.get("tracing_observations")).not.toEqual(
byMeter.get("tracing_events"),
);
for (const [eventName, identifier] of byMeter) {
expect(identifier).toContain(eventName);
expect(identifier).toContain("org-1");
expect(identifier).toContain(String(INTERVAL_START.getTime()));
}
});
it("advances lastRun under the lease it claimed", async () => {
await handleCloudUsageMeteringJob(createJob());
const closingCall = mockCronJobsUpdate.mock.calls.find(
([args]) => args.data.lastRun !== undefined,
);
const claimCall = mockCronJobsUpdate.mock.calls.find(
([args]) =>
args.data.state === CloudUsageMeteringDbCronJobStates.Processing,
);
expect(claimCall).toBeDefined();
expect(closingCall?.[0].where).toMatchObject({
state: CloudUsageMeteringDbCronJobStates.Processing,
jobStartedAt: claimCall?.[0].data.jobStartedAt,
});
});
it("does not advance lastRun or re-enqueue when the interval was taken over", async () => {
// The 20 minute stale-job takeover handed this interval to another run, so
// the closing update no longer matches the lease this run claimed.
mockCronJobsUpdate.mockImplementation(async ({ data }) => {
if (data.lastRun !== undefined) {
throw new Error("Record to update not found.");
}
return {};
});
await expect(
handleCloudUsageMeteringJob(createJob()),
).resolves.toBeUndefined();
// Throwing here would make the wrapper reset the state and replay the
// interval a third time.
expect(mockUsageMeteringQueueAdd).not.toHaveBeenCalled();
expect(mockLoggerWarn).toHaveBeenCalled();
});
it("skips a meter event Stripe already holds and still sends the other meter", async () => {
// The replayed run died between the two meters last time, so Stripe holds
// tracing_observations for this interval but never received tracing_events.
mockMeterEventsCreate.mockImplementation(
async ({ event_name, identifier }) => {
if (event_name === "tracing_observations") {
throw duplicateIdentifierError(identifier);
}
return {};
},
);
await expect(
handleCloudUsageMeteringJob(createJob()),
).resolves.toBeUndefined();
// Skipping the whole org on the first duplicate would leave this meter
// unsent for good, turning the over-bill into an under-bill.
expect(callsForMeter("tracing_events")).toHaveLength(1);
// A rejection reaching the caller would strand the interval: lastRun only
// advances after the loop, so the next tick replays it and fails again.
expect(closingUpdate()?.[0].data.lastRun).toEqual(
new Date(INTERVAL_START.getTime() + 3600000),
);
expect(mockRecordIncrement).toHaveBeenCalledWith(
"langfuse.queue.cloud_usage_metering_queue.duplicate_meter_events",
1,
{ unit: "events" },
);
expect(mockLoggerInfo).toHaveBeenCalledWith(
expect.stringContaining("tracing_observations:org-1"),
);
});
it("does not retry a meter event Stripe rejected as a duplicate", async () => {
mockMeterEventsCreate.mockImplementation(async ({ identifier }) => {
throw duplicateIdentifierError(identifier);
});
await handleCloudUsageMeteringJob(createJob());
// Stripe answers stripe-should-retry: false, so backOff must not spend its
// three attempts re-posting a request that can never succeed.
expect(callsForMeter("tracing_observations")).toHaveLength(1);
expect(callsForMeter("tracing_events")).toHaveLength(1);
});
it("fails the job when Stripe rejects a meter event for another reason", async () => {
// The >35 day backdating rejection is the same status and rawType, and it
// means the usage was never delivered - swallowing it would lose billing.
mockMeterEventsCreate.mockRejectedValue(
stripeInvalidRequestError(
"Timestamp must be no more than 35 days in the past.",
),
);
await expect(handleCloudUsageMeteringJob(createJob())).rejects.toThrow(
/35 days/,
);
expect(closingUpdate()).toBeUndefined();
expect(mockRecordIncrement).not.toHaveBeenCalledWith(
"langfuse.queue.cloud_usage_metering_queue.duplicate_meter_events",
expect.anything(),
expect.anything(),
);
});
it("compares the interval with the billing ledger after sending it to Stripe", async () => {
mockEnv.LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED = "true";
await handleCloudUsageMeteringJob(createJob());
expect(mockCompareWithBillingLedger).toHaveBeenCalledTimes(1);
expect(mockCompareWithBillingLedger).toHaveBeenCalledWith({
source: "stripe_metering",
from: INTERVAL_START,
to: new Date(INTERVAL_START.getTime() + 3600000),
legacy: {
traces: [{ projectId: "project-1", count: 3 }],
observations: [{ projectId: "project-1", count: 5 }],
scores: [{ projectId: "project-1", count: 2 }],
},
orgIdByProjectId: new Map([["project-1", "org-1"]]),
});
// The comparison must not hold up metering: it starts only once Stripe
// has the interval and the lease is released.
const comparedAt = mockCompareWithBillingLedger.mock.invocationCallOrder[0];
expect(comparedAt).toBeGreaterThan(
Math.max(...mockMeterEventsCreate.mock.invocationCallOrder),
);
expect(comparedAt).toBeGreaterThan(
mockCronJobsUpdate.mock.invocationCallOrder.at(-1) ?? Infinity,
);
});
it("does not read the billing ledger unless enabled", async () => {
await handleCloudUsageMeteringJob(createJob());
expect(mockCompareWithBillingLedger).not.toHaveBeenCalled();
});
});