Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(billing): compare legacy billing counts with the billable units ledger (#18955)

Steffen Reuther677c24839b14
diff --git a/.env.dev.example b/.env.dev.example
index ffb09ecdf1..bf62943476 100644
--- a/.env.dev.example
+++ b/.env.dev.example
@@ -475,3 +475,6 @@ CLICKHOUSE_ENABLE_SKIP_INDEXES_ON_DATA_READ=auto
# CLICKHOUSE_BILLING_AUTH0_AUDIENCE=
# HMAC secret for inbound CHB webhooks; unset = the endpoint 500s.
# CLICKHOUSE_BILLING_WEBHOOK_SIGNING_SECRET=
+# Compare Stripe metering (worker) and GET /api/billing/metrics (web) with the
+# billable_events ledger and report the gap; requires the ledger table.
+# LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED=false
diff --git a/.env.prod.example b/.env.prod.example
index 0e8826ccb2..02f992bf88 100644
--- a/.env.prod.example
+++ b/.env.prod.example
@@ -600,6 +600,9 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true
# CLICKHOUSE_BILLING_AUTH0_AUDIENCE=
# HMAC secret for inbound CHB webhooks; unset = the endpoint 500s.
# CLICKHOUSE_BILLING_WEBHOOK_SIGNING_SECRET=
+# Compare Stripe metering (worker) and GET /api/billing/metrics (web) with the
+# billable_events ledger and report the gap; requires the ledger table.
+# LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED=false
# Rust AI gateway. Use the trusted HTTPS Web URL including any NEXT_PUBLIC_BASE_PATH.
# Without a Web URL the gateway starts, but readiness and inference return 503.
diff --git a/packages/shared/AGENTS.md b/packages/shared/AGENTS.md
index f7348c94ca..9932bf012b 100644
--- a/packages/shared/AGENTS.md
+++ b/packages/shared/AGENTS.md
@@ -32,6 +32,9 @@
`clickhouse/scripts/dev-tables.sh`, applied by hand on Cloud, not
migration-managed) read via `src/server/repositories/billable-events.ts`.
Keep it out of every delete, retention and cleaner path.
+ `src/server/billing/billingLedgerShadow.ts` compares legacy billing counts
+ with the ledger and only reports (metrics plus one log line); it never
+ throws. Callers gate it on `LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED`.
- Queue payload schemas: `src/server/queues.ts`
- Queue helpers: `src/server/redis/*`
`getShardIndex` from `src/server/redis/sharding.ts` is exported by the server
diff --git a/packages/shared/src/server/billing/billingLedgerShadow.test.ts b/packages/shared/src/server/billing/billingLedgerShadow.test.ts
new file mode 100644
index 0000000000..c0c6bd336d
--- /dev/null
+++ b/packages/shared/src/server/billing/billingLedgerShadow.test.ts
@@ -0,0 +1,207 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+ logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+ recordIncrement: vi.fn(),
+ ledgerCounts: vi.fn(),
+}));
+
+vi.mock("../logger", () => ({ logger: mocks.logger }));
+vi.mock("../instrumentation", () => ({
+ recordIncrement: mocks.recordIncrement,
+}));
+vi.mock("../repositories/billable-events", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("../repositories/billable-events")>()),
+ getBillableUnitCountsByProjectInWindow: mocks.ledgerCounts,
+}));
+
+import { compareWithBillingLedger } from "./billingLedgerShadow";
+
+const FROM = new Date("2026-10-16T08:00:00.000Z");
+const TO = new Date("2026-10-16T09:00:00.000Z");
+
+const incrementsOf = (stat: string) =>
+ mocks.recordIncrement.mock.calls
+ .filter(([name]) => name === stat)
+ .map(([, value, tags]) => ({ value, tags }));
+
+describe("compareWithBillingLedger", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ });
+
+ it("reports unit totals and logs the projects with the largest gaps", async () => {
+ mocks.ledgerCounts.mockResolvedValue([
+ { projectId: "p-diverging", eventType: "observation", count: 200 },
+ { projectId: "p-busy", eventType: "observation", count: 99_000 },
+ { projectId: "p-steady", eventType: "observation", count: 9_990 },
+ { projectId: "p-ledger-only", eventType: "score", count: 150 },
+ ]);
+
+ await compareWithBillingLedger({
+ source: "stripe_metering",
+ from: FROM,
+ to: TO,
+ legacy: {
+ traces: [],
+ observations: [
+ { projectId: "p-diverging", count: 1_000 },
+ // Below the unit threshold, but 100% of its usage, and one-sided.
+ { projectId: "p-small", count: 50 },
+ // Only 1% of its usage, but 1,000 units apart.
+ { projectId: "p-busy", count: 100_000 },
+ // 10 units and 0.1% apart: not diverging.
+ { projectId: "p-steady", count: 10_000 },
+ ],
+ scores: [],
+ },
+ orgIdByProjectId: new Map([["p-diverging", "org-1"]]),
+ });
+
+ expect(mocks.ledgerCounts).toHaveBeenCalledWith({
+ windowFrom: FROM,
+ windowTo: TO,
+ projectId: undefined,
+ });
+ expect(incrementsOf("langfuse.billing.ledger_shadow.units")).toEqual(
+ expect.arrayContaining([
+ {
+ value: 111_050,
+ tags: {
+ source: "stripe_metering",
+ side: "legacy",
+ unit_type: "observation",
+ },
+ },
+ {
+ value: 109_190,
+ tags: {
+ source: "stripe_metering",
+ side: "ledger",
+ unit_type: "observation",
+ },
+ },
+ {
+ value: 150,
+ tags: {
+ source: "stripe_metering",
+ side: "ledger",
+ unit_type: "score",
+ },
+ },
+ ]),
+ );
+ expect(
+ incrementsOf("langfuse.billing.ledger_shadow.diverging_projects"),
+ ).toEqual([{ value: 4, tags: { source: "stripe_metering" } }]);
+ expect(
+ incrementsOf("langfuse.billing.ledger_shadow.one_sided_projects"),
+ ).toEqual([
+ { value: 1, tags: { source: "stripe_metering", missing_from: "ledger" } },
+ { value: 1, tags: { source: "stripe_metering", missing_from: "legacy" } },
+ ]);
+
+ expect(mocks.logger.info).toHaveBeenCalledTimes(1);
+ const [, logged] = mocks.logger.info.mock.calls[0];
+ expect(
+ logged.projects.map(({ projectId, orgId }: Record<string, unknown>) => ({
+ projectId,
+ orgId,
+ })),
+ ).toEqual([
+ { projectId: "p-busy", orgId: undefined },
+ { projectId: "p-diverging", orgId: "org-1" },
+ { projectId: "p-ledger-only", orgId: undefined },
+ { projectId: "p-small", orgId: undefined },
+ ]);
+ });
+
+ it("flags a project whose unit types diverge in opposite directions", async () => {
+ mocks.ledgerCounts.mockResolvedValue([
+ { projectId: "p-offsetting", eventType: "trace", count: 300 },
+ { projectId: "p-offsetting", eventType: "score", count: 200 },
+ ]);
+
+ await compareWithBillingLedger({
+ source: "stripe_metering",
+ from: FROM,
+ to: TO,
+ legacy: {
+ traces: [{ projectId: "p-offsetting", count: 500 }],
+ observations: [],
+ scores: [],
+ },
+ });
+
+ expect(
+ incrementsOf("langfuse.billing.ledger_shadow.diverging_projects"),
+ ).toEqual([{ value: 1, tags: { source: "stripe_metering" } }]);
+ const [, logged] = mocks.logger.info.mock.calls[0];
+ expect(logged.projects[0].delta).toEqual({
+ trace: -200,
+ observation: 0,
+ score: 200,
+ });
+ });
+
+ it("does not throw when the ledger read fails", async () => {
+ mocks.ledgerCounts.mockRejectedValue(
+ new Error("Table billable_events does not exist"),
+ );
+
+ await expect(
+ compareWithBillingLedger({
+ source: "chb_metrics_api",
+ from: FROM,
+ to: new Date("2026-10-16T08:15:00.000Z"),
+ projectId: "p-1",
+ legacy: { traces: [], observations: [], scores: [] },
+ }),
+ ).resolves.toBeUndefined();
+
+ expect(
+ incrementsOf("langfuse.billing.ledger_shadow.failed_comparisons"),
+ ).toEqual([{ value: 1, tags: { source: "chb_metrics_api" } }]);
+ expect(mocks.logger.warn).toHaveBeenCalled();
+ });
+
+ it("reads the ledger for the exact range it is given", async () => {
+ mocks.ledgerCounts.mockResolvedValue([]);
+ const from = new Date("2026-10-16T08:05:00.000Z");
+ const to = new Date("2026-10-16T08:20:00.000Z");
+
+ await compareWithBillingLedger({
+ source: "chb_metrics_api",
+ from,
+ to,
+ projectId: "p-1",
+ legacy: { traces: [], observations: [], scores: [] },
+ });
+
+ expect(mocks.ledgerCounts).toHaveBeenCalledWith({
+ windowFrom: from,
+ windowTo: to,
+ projectId: "p-1",
+ });
+ });
+
+ it("skips a range longer than an hour", async () => {
+ await compareWithBillingLedger({
+ source: "chb_metrics_api",
+ from: FROM,
+ to: new Date("2026-10-16T09:00:01.000Z"),
+ projectId: "p-1",
+ legacy: { traces: [], observations: [], scores: [] },
+ });
+
+ expect(mocks.ledgerCounts).not.toHaveBeenCalled();
+ expect(
+ incrementsOf("langfuse.billing.ledger_shadow.skipped_comparisons"),
+ ).toEqual([
+ {
+ value: 1,
+ tags: { source: "chb_metrics_api", reason: "unsupported_range" },
+ },
+ ]);
+ });
+});
diff --git a/packages/shared/src/server/billing/billingLedgerShadow.ts b/packages/shared/src/server/billing/billingLedgerShadow.ts
new file mode 100644
index 0000000000..f39db750d1
--- /dev/null
+++ b/packages/shared/src/server/billing/billingLedgerShadow.ts
@@ -0,0 +1,203 @@
+import { recordIncrement } from "../instrumentation";
+import { logger } from "../logger";
+import {
+ type BillableEventType,
+ getBillableUnitCountsByProjectInWindow,
+} from "../repositories/billable-events";
+
+/** Billing consumer whose legacy counts the ledger is compared against. */
+export type BillingLedgerShadowSource = "stripe_metering" | "chb_metrics_api";
+
+type ProjectCount = { projectId: string; count: number };
+type Units = Record<BillableEventType, number>;
+
+// Bounds the ledger scan of one comparison. Stripe meters one hour, and the
+// CHB endpoint accepts ranges of up to 35 days.
+const MAX_COMPARED_RANGE_MS = 60 * 60 * 1000;
+
+// A project diverges when one of its unit types is at least 100 units or 5% of
+// its legacy count apart. Unit types are compared separately, so a gap in one
+// cannot hide a gap of the opposite sign in another.
+const DIVERGENCE_MIN_UNITS = 100;
+const DIVERGENCE_MIN_RATIO = 0.05;
+const LOGGED_PROJECTS = 10;
+
+const UNIT_TYPES: readonly BillableEventType[] = [
+ "trace",
+ "observation",
+ "score",
+];
+
+const zeroUnits = (): Units => ({ trace: 0, observation: 0, score: 0 });
+
+const totalUnits = (units: Units) =>
+ units.trace + units.observation + units.score;
+
+/** Ledger minus legacy, per unit type. */
+const deltaUnits = ({
+ legacy,
+ ledger,
+}: {
+ legacy: Units;
+ ledger: Units;
+}): Units => ({
+ trace: ledger.trace - legacy.trace,
+ observation: ledger.observation - legacy.observation,
+ score: ledger.score - legacy.score,
+});
+
+const largestGap = (delta: Units) =>
+ Math.max(...UNIT_TYPES.map((unitType) => Math.abs(delta[unitType])));
+
+const isDiverging = ({ legacy, delta }: { legacy: Units; delta: Units }) =>
+ UNIT_TYPES.some((unitType) => {
+ const gap = Math.abs(delta[unitType]);
+ return (
+ gap > 0 &&
+ (gap >= DIVERGENCE_MIN_UNITS ||
+ gap >= DIVERGENCE_MIN_RATIO * legacy[unitType])
+ );
+ });
+
+/**
+ * Compares the legacy billing counts of `[from, to)` with the billable_events
+ * ledger per project and reports the result. Metrics carry unit totals per
+ * side and type, and the number of compared, diverging and one-sided projects.
+ * One log line lists the projects with the largest gaps.
+ *
+ * Shadow only: it never changes what is billed and never throws. The legacy
+ * score count leaves CORRECTION scores out and the ledger does not, so a
+ * project with CORRECTION scores shows a score gap of that size.
+ */
+export const compareWithBillingLedger = async ({
+ source,
+ from,
+ to,
+ projectId,
+ legacy,
+ orgIdByProjectId,
+}: {
+ source: BillingLedgerShadowSource;
+ from: Date;
+ to: Date;
+ /** Restricts the ledger read to one project, as the legacy counts are. */
+ projectId?: string;
+ legacy: {
+ traces: ProjectCount[];
+ observations: ProjectCount[];
+ scores: ProjectCount[];
+ };
+ orgIdByProjectId?: Map<string, string>;
+}): Promise<void> => {
+ const rangeMs = to.getTime() - from.getTime();
+ if (rangeMs <= 0 || rangeMs > MAX_COMPARED_RANGE_MS) {
+ recordIncrement("langfuse.billing.ledger_shadow.skipped_comparisons", 1, {
+ source,
+ reason: "unsupported_range",
+ });
+ return;
+ }
+
+ try {
+ const ledger = await getBillableUnitCountsByProjectInWindow({
+ windowFrom: from,
+ windowTo: to,
+ projectId,
+ });
+
+ const projects = new Map<string, { legacy: Units; ledger: Units }>();
+ const unitsOf = (id: string) => {
+ let units = projects.get(id);
+ if (!units) {
+ units = { legacy: zeroUnits(), ledger: zeroUnits() };
+ projects.set(id, units);
+ }
+ return units;
+ };
+ const legacyByType: Array<[BillableEventType, ProjectCount[]]> = [
+ ["trace", legacy.traces],
+ ["observation", legacy.observations],
+ ["score", legacy.scores],
+ ];
+ for (const [eventType, rows] of legacyByType) {
+ for (const row of rows)
+ unitsOf(row.projectId).legacy[eventType] += row.count;
+ }
+ for (const row of ledger) {
+ unitsOf(row.projectId).ledger[row.eventType] += row.count;
+ }
+
+ const comparisons = [...projects].map(([id, units]) => ({
+ projectId: id,
+ orgId: orgIdByProjectId?.get(id),
+ legacy: units.legacy,
+ ledger: units.ledger,
+ delta: deltaUnits(units),
+ }));
+ const diverging = comparisons
+ .filter(isDiverging)
+ .sort((a, b) => largestGap(b.delta) - largestGap(a.delta));
+
+ const totals = { legacy: zeroUnits(), ledger: zeroUnits() };
+ for (const comparison of comparisons) {
+ for (const unitType of UNIT_TYPES) {
+ totals.legacy[unitType] += comparison.legacy[unitType];
+ totals.ledger[unitType] += comparison.ledger[unitType];
+ }
+ }
+ for (const unitType of UNIT_TYPES) {
+ for (const side of ["legacy", "ledger"] as const) {
+ recordIncrement(
+ "langfuse.billing.ledger_shadow.units",
+ totals[side][unitType],
+ { source, side, unit_type: unitType },
+ );
+ }
+ }
+ recordIncrement(
+ "langfuse.billing.ledger_shadow.compared_projects",
+ comparisons.length,
+ { source },
+ );
+ recordIncrement(
+ "langfuse.billing.ledger_shadow.diverging_projects",
+ diverging.length,
+ { source },
+ );
+ // A project with usage on only one side points at a broken write path,
+ // such as a materialized view that stopped filling the ledger.
+ for (const side of ["ledger", "legacy"] as const) {
+ const other = side === "ledger" ? "legacy" : "ledger";
+ recordIncrement(
+ "langfuse.billing.ledger_shadow.one_sided_projects",
+ comparisons.filter(
+ (comparison) =>
+ totalUnits(comparison[side]) === 0 &&
+ totalUnits(comparison[other]) > 0,
+ ).length,
+ { source, missing_from: side },
+ );
+ }
+
+ if (diverging.length > 0) {
+ logger.info(
+ `[BILLING LEDGER SHADOW] ${diverging.length} of ${comparisons.length} projects diverge for ${source} in ${from.toISOString()} - ${to.toISOString()}`,
+ {
+ source,
+ from: from.toISOString(),
+ to: to.toISOString(),
+ totals,
+ projects: diverging.slice(0, LOGGED_PROJECTS),
+ },
+ );
+ }
+ } catch (error) {
+ logger.warn(
+ `[BILLING LEDGER SHADOW] Comparison for ${source} in ${from.toISOString()} - ${to.toISOString()} failed`,
+ { error },
+ );
+ recordIncrement("langfuse.billing.ledger_shadow.failed_comparisons", 1, {
+ source,
+ });
+ }
+};
diff --git a/packages/shared/src/server/index.ts b/packages/shared/src/server/index.ts
index 23187b079e..75d482ef2e 100644
--- a/packages/shared/src/server/index.ts
+++ b/packages/shared/src/server/index.ts
@@ -159,6 +159,7 @@ export * from "./queries";
export * from "./queries/clickhouse-sql/orderby-factory";
export * from "./queries/clickhouse-sql/query-options";
export * from "./repositories";
+export * from "./billing/billingLedgerShadow";
export { SCORE_TO_TRACE_OBSERVATIONS_INTERVAL } from "./repositories/constants";
export { scoreDomainToV3 } from "./repositories/scores";
export * from "./repositories/observations";
diff --git a/web/src/__tests__/server/unit/chb-metrics-api-handler.servertest.ts b/web/src/__tests__/server/unit/chb-metrics-api-handler.servertest.ts
index 11ebde7908..6b7b437c18 100644
--- a/web/src/__tests__/server/unit/chb-metrics-api-handler.servertest.ts
+++ b/web/src/__tests__/server/unit/chb-metrics-api-handler.servertest.ts
@@ -6,12 +6,14 @@ const mocks = vi.hoisted(() => ({
env: {
NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "eu" as string | undefined,
CLICKHOUSE_BILLING_METRICS_API_KEY: "chb-secret" as string | undefined,
+ LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED: "false",
},
findProject: vi.fn(),
traceCounts: vi.fn(),
observationCounts: vi.fn(),
scoreCounts: vi.fn(),
recordIncrement: vi.fn(),
+ compareWithBillingLedger: vi.fn(),
}));
vi.mock("@/src/env.mjs", () => ({ env: mocks.env }));
@@ -31,6 +33,7 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
getObservationCountsByProjectInCreationInterval: mocks.observationCounts,
getScoreCountsByProjectInCreationInterval: mocks.scoreCounts,
recordIncrement: mocks.recordIncrement,
+ compareWithBillingLedger: mocks.compareWithBillingLedger,
};
});
@@ -70,6 +73,7 @@ describe("chbMetricsApiHandler", () => {
vi.clearAllMocks();
mocks.env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "eu";
mocks.env.CLICKHOUSE_BILLING_METRICS_API_KEY = "chb-secret";
+ mocks.env.LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED = "false";
mocks.findProject.mockResolvedValue({ id: PROJECT_ID });
mocks.traceCounts.mockResolvedValue([{ projectId: PROJECT_ID, count: 7 }]);
mocks.observationCounts.mockResolvedValue([
@@ -212,4 +216,44 @@ describe("chbMetricsApiHandler", () => {
});
expect(mocks.traceCounts).not.toHaveBeenCalled();
});
+
+ it("compares the poll with the billing ledger without waiting for it", async () => {
+ mocks.env.LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED = "true";
+ // Never settles: awaiting the comparison would hang the poll.
+ mocks.compareWithBillingLedger.mockReturnValue(new Promise(() => {}));
+
+ const response = await chbMetricsApiHandler(
+ createRequest({
+ startTime: "2026-07-01T00:00:00Z",
+ endTime: "2026-07-01T00:15:00Z",
+ }),
+ );
+
+ expect(response.status).toBe(200);
+ expect(await response.json()).toEqual({
+ metrics: {
+ traces: { sum: 7 },
+ observations: { sum: 11 },
+ scores: { sum: 3 },
+ },
+ });
+ expect(mocks.compareWithBillingLedger).toHaveBeenCalledWith({
+ source: "chb_metrics_api",
+ from: new Date("2026-07-01T00:00:00Z"),
+ to: new Date("2026-07-01T00:15:00Z"),
+ projectId: PROJECT_ID,
+ legacy: {
+ traces: [{ projectId: PROJECT_ID, count: 7 }],
+ observations: [{ projectId: PROJECT_ID, count: 11 }],
+ scores: [{ projectId: PROJECT_ID, count: 3 }],
+ },
+ });
+ });
+
+ it("does not read the billing ledger unless enabled", async () => {
+ const response = await chbMetricsApiHandler(createRequest());
+
+ expect(response.status).toBe(200);
+ expect(mocks.compareWithBillingLedger).not.toHaveBeenCalled();
+ });
});
diff --git a/web/src/ee/features/billing/server/chb/chbMetricsApiHandler.ts b/web/src/ee/features/billing/server/chb/chbMetricsApiHandler.ts
index aca5bd9122..2796431174 100644
--- a/web/src/ee/features/billing/server/chb/chbMetricsApiHandler.ts
+++ b/web/src/ee/features/billing/server/chb/chbMetricsApiHandler.ts
@@ -7,6 +7,7 @@ import { z } from "zod";
import { env } from "@/src/env.mjs";
import { prisma } from "@langfuse/shared/src/db";
import {
+ compareWithBillingLedger,
getObservationCountsByProjectInCreationInterval,
getScoreCountsByProjectInCreationInterval,
getTraceCountsByProjectInCreationInterval,
@@ -149,6 +150,22 @@ export async function chbMetricsApiHandler(req: NextRequest) {
}),
]);
+ // Not awaited, so the comparison never delays CHB's poll, and it never
+ // throws. Windows longer than an hour are skipped.
+ if (env.LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED === "true") {
+ compareWithBillingLedger({
+ source: "chb_metrics_api",
+ from: start,
+ to: end,
+ projectId: resourceId,
+ legacy: {
+ traces: traceCounts,
+ observations: observationCounts,
+ scores: scoreCounts,
+ },
+ });
+ }
+
return NextResponse.json(
{
metrics: {
diff --git a/web/src/env.mjs b/web/src/env.mjs
index dcc60f0932..7fcb78fb25 100644
--- a/web/src/env.mjs
+++ b/web/src/env.mjs
@@ -522,6 +522,12 @@ export const env = createEnv({
// Bearer secret CHB's metering pipeline presents to GET /api/billing/metrics.
// Unset disables the endpoint (it 500s) rather than leaving it open.
CLICKHOUSE_BILLING_METRICS_API_KEY: z.string().optional(),
+ // Compares each GET /api/billing/metrics poll with the billable_events
+ // ledger and reports the gap. Requires the ledger table, which only
+ // Langfuse Cloud has; keep in sync with worker/src/env.ts.
+ LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED: z
+ .enum(["true", "false"])
+ .default("false"),
// ARN of CHB's cross-account EventBridge bus we publish project lifecycle
// events to. Unset = no events are emitted. The ARN carries the bus region,
// so it is the only input needed; authentication is SigV4 from the task
@@ -1219,6 +1225,8 @@ export const env = createEnv({
process.env.LANGFUSE_CLOUD_BILLING_CHB_OPT_IN_ORG_IDS,
CLICKHOUSE_BILLING_METRICS_API_KEY:
process.env.CLICKHOUSE_BILLING_METRICS_API_KEY,
+ LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED:
+ process.env.LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED,
CLICKHOUSE_BILLING_EVENT_BUS_ARN:
process.env.CLICKHOUSE_BILLING_EVENT_BUS_ARN,
CLICKHOUSE_BILLING_BASE_URL: process.env.CLICKHOUSE_BILLING_BASE_URL,
diff --git a/worker/src/ee/cloudUsageMetering/__tests__/handleCloudUsageMeteringJob.test.ts b/worker/src/ee/cloudUsageMetering/__tests__/handleCloudUsageMeteringJob.test.ts
index 95c7babaa8..7a607c32fb 100644
--- a/worker/src/ee/cloudUsageMetering/__tests__/handleCloudUsageMeteringJob.test.ts
+++ b/worker/src/ee/cloudUsageMetering/__tests__/handleCloudUsageMeteringJob.test.ts
@@ -16,6 +16,8 @@ const {
mockLoggerError,
mockRecordIncrement,
mockTraceException,
+ mockCompareWithBillingLedger,
+ mockEnv,
} = vi.hoisted(() => ({
mockCronJobsUpsert: vi.fn(),
mockCronJobsUpdate: vi.fn(),
@@ -31,6 +33,12 @@ const {
mockLoggerError: vi.fn(),
mockRecordIncrement: vi.fn(),
mockTraceException: vi.fn(),
+ mockCompareWithBillingLedger: vi.fn(),
+ mockEnv: {
+ STRIPE_SECRET_KEY: "sk_test",
+ LANGFUSE_CLOUD_BILLING_CHB_CUTOFF_DATE: undefined,
+ LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED: "false",
+ },
}));
vi.mock("@langfuse/shared", () => ({
@@ -58,6 +66,7 @@ vi.mock("@langfuse/shared/src/server", () => ({
CloudSpendAlertQueue: {
getInstance: () => ({ add: mockSpendAlertQueueAdd }),
},
+ compareWithBillingLedger: mockCompareWithBillingLedger,
getObservationCountsByProjectInCreationInterval: mockObservationCounts,
getTraceCountsByProjectInCreationInterval: mockTraceCounts,
getScoreCountsByProjectInCreationInterval: mockScoreCounts,
@@ -80,12 +89,7 @@ vi.mock("stripe", () => ({
},
}));
-vi.mock("../../../env", () => ({
- env: {
- STRIPE_SECRET_KEY: "sk_test",
- LANGFUSE_CLOUD_BILLING_CHB_CUTOFF_DATE: undefined,
- },
-}));
+vi.mock("../../../env", () => ({ env: mockEnv }));
import { handleCloudUsageMeteringJob } from "../handleCloudUsageMeteringJob";
import { CloudUsageMeteringDbCronJobStates } from "../constants";
@@ -132,6 +136,7 @@ describe("handleCloudUsageMeteringJob", () => {
vi.clearAllMocks();
vi.useFakeTimers({ shouldAdvanceTime: true });
vi.setSystemTime(NOW);
+ mockEnv.LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED = "false";
mockCronJobsUpsert.mockResolvedValue({
name: "cloud_usage_metering",
@@ -140,6 +145,7 @@ describe("handleCloudUsageMeteringJob", () => {
jobStartedAt: null,
});
mockCronJobsUpdate.mockResolvedValue({});
+ mockMeterEventsCreate.mockResolvedValue({});
mockOrganizationFindMany.mockResolvedValue([
{
id: "org-1",
@@ -305,4 +311,38 @@ describe("handleCloudUsageMeteringJob", () => {
expect.anything(),
);
});
+
+ it("compares the interval with the billing ledger after sending it to Stripe", async () => {
+ mockEnv.LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED = "true";
+
+ await handleCloudUsageMeteringJob(createJob());
+
+ expect(mockCompareWithBillingLedger).toHaveBeenCalledTimes(1);
+ expect(mockCompareWithBillingLedger).toHaveBeenCalledWith({
+ source: "stripe_metering",
+ from: INTERVAL_START,
+ to: new Date(INTERVAL_START.getTime() + 3600000),
+ legacy: {
+ traces: [{ projectId: "project-1", count: 3 }],
+ observations: [{ projectId: "project-1", count: 5 }],
+ scores: [{ projectId: "project-1", count: 2 }],
+ },
+ orgIdByProjectId: new Map([["project-1", "org-1"]]),
+ });
+ // The comparison must not hold up metering: it starts only once Stripe
+ // has the interval and the lease is released.
+ const comparedAt = mockCompareWithBillingLedger.mock.invocationCallOrder[0];
+ expect(comparedAt).toBeGreaterThan(
+ Math.max(...mockMeterEventsCreate.mock.invocationCallOrder),
+ );
+ expect(comparedAt).toBeGreaterThan(
+ mockCronJobsUpdate.mock.invocationCallOrder.at(-1) ?? Infinity,
+ );
+ });
+
+ it("does not read the billing ledger unless enabled", async () => {
+ await handleCloudUsageMeteringJob(createJob());
+
+ expect(mockCompareWithBillingLedger).not.toHaveBeenCalled();
+ });
});
diff --git a/worker/src/ee/cloudUsageMetering/handleCloudUsageMeteringJob.ts b/worker/src/ee/cloudUsageMetering/handleCloudUsageMeteringJob.ts
index 9f0b8cceb1..f9fa0c49be 100644
--- a/worker/src/ee/cloudUsageMetering/handleCloudUsageMeteringJob.ts
+++ b/worker/src/ee/cloudUsageMetering/handleCloudUsageMeteringJob.ts
@@ -5,6 +5,7 @@ import { env } from "../../env";
import {
CloudUsageMeteringQueue,
CloudSpendAlertQueue,
+ compareWithBillingLedger,
getObservationCountsByProjectInCreationInterval,
getScoreCountsByProjectInCreationInterval,
getTraceCountsByProjectInCreationInterval,
@@ -433,6 +434,27 @@ export const handleCloudUsageMeteringJob = async (job: Job) => {
},
);
+ // Runs after the interval is metered and the lease released, and never
+ // throws, so it cannot delay or change what Stripe receives. It compares
+ // every project with usage, not only the Stripe-billed ones.
+ if (env.LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED === "true") {
+ await compareWithBillingLedger({
+ source: "stripe_metering",
+ from: meterIntervalStart,
+ to: meterIntervalEnd,
+ legacy: {
+ traces: traceCountsByProject,
+ observations: observationCountsByProject,
+ scores: scoreCountsByProject,
+ },
+ orgIdByProjectId: new Map(
+ organizations.flatMap((org) =>
+ org.projectIds.map((projectId) => [projectId, org.id] as const),
+ ),
+ ),
+ });
+ }
+
if (meterIntervalEnd.getTime() + delayFromStartOfInterval < Date.now()) {
logger.info(
`[CLOUD USAGE METERING] Enqueueing next Cloud Usage Metering Job to catch up `,
diff --git a/worker/src/env.ts b/worker/src/env.ts
index 09f87e93fa..9c97b0a369 100644
--- a/worker/src/env.ts
+++ b/worker/src/env.ts
@@ -64,6 +64,12 @@ const EnvSchema = z.object({
// CHB's resource-server identifier. Defaulted because it is the same value
// in every CHB tenant, and overridable in case that stops being true.
CLICKHOUSE_BILLING_AUTH0_AUDIENCE: z.string().default("billing-api"),
+ // Compares each metered Stripe interval with the billable_events ledger and
+ // reports the gap. Requires the ledger table, which only Langfuse Cloud has;
+ // keep in sync with web/src/env.mjs.
+ LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED: z
+ .enum(["true", "false"])
+ .default("false"),
LANGFUSE_CACHE_AUTOMATIONS_ENABLED: z.enum(["true", "false"]).default("true"),
LANGFUSE_CACHE_AUTOMATIONS_TTL_SECONDS: z.coerce.number().default(60),