🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(billing): compare legacy billing counts with the billable units ledger (#18955)
Steffen Reuther677c24839b14
diff --git a/.env.dev.example b/.env.dev.exampleindex ffb09ecdf1..bf62943476 100644--- a/.env.dev.example+++ b/.env.dev.example@@ -475,3 +475,6 @@ CLICKHOUSE_ENABLE_SKIP_INDEXES_ON_DATA_READ=auto# CLICKHOUSE_BILLING_AUTH0_AUDIENCE=# HMAC secret for inbound CHB webhooks; unset = the endpoint 500s.# CLICKHOUSE_BILLING_WEBHOOK_SIGNING_SECRET=+# Compare Stripe metering (worker) and GET /api/billing/metrics (web) with the+# billable_events ledger and report the gap; requires the ledger table.+# LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED=falsediff --git a/.env.prod.example b/.env.prod.exampleindex 0e8826ccb2..02f992bf88 100644--- a/.env.prod.example+++ b/.env.prod.example@@ -600,6 +600,9 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true# CLICKHOUSE_BILLING_AUTH0_AUDIENCE=# HMAC secret for inbound CHB webhooks; unset = the endpoint 500s.# CLICKHOUSE_BILLING_WEBHOOK_SIGNING_SECRET=+# Compare Stripe metering (worker) and GET /api/billing/metrics (web) with the+# billable_events ledger and report the gap; requires the ledger table.+# LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED=false# Rust AI gateway. Use the trusted HTTPS Web URL including any NEXT_PUBLIC_BASE_PATH.# Without a Web URL the gateway starts, but readiness and inference return 503.diff --git a/packages/shared/AGENTS.md b/packages/shared/AGENTS.mdindex f7348c94ca..9932bf012b 100644--- a/packages/shared/AGENTS.md+++ b/packages/shared/AGENTS.md@@ -32,6 +32,9 @@`clickhouse/scripts/dev-tables.sh`, applied by hand on Cloud, notmigration-managed) read via `src/server/repositories/billable-events.ts`.Keep it out of every delete, retention and cleaner path.+ `src/server/billing/billingLedgerShadow.ts` compares legacy billing counts+ with the ledger and only reports (metrics plus one log line); it never+ throws. Callers gate it on `LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED`.- Queue payload schemas: `src/server/queues.ts`- Queue helpers: `src/server/redis/*``getShardIndex` from `src/server/redis/sharding.ts` is exported by the serverdiff --git a/packages/shared/src/server/billing/billingLedgerShadow.test.ts b/packages/shared/src/server/billing/billingLedgerShadow.test.tsnew file mode 100644index 0000000000..c0c6bd336d--- /dev/null+++ b/packages/shared/src/server/billing/billingLedgerShadow.test.ts@@ -0,0 +1,207 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++const mocks = vi.hoisted(() => ({+ logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },+ recordIncrement: vi.fn(),+ ledgerCounts: vi.fn(),+}));++vi.mock("../logger", () => ({ logger: mocks.logger }));+vi.mock("../instrumentation", () => ({+ recordIncrement: mocks.recordIncrement,+}));+vi.mock("../repositories/billable-events", async (importOriginal) => ({+ ...(await importOriginal<typeof import("../repositories/billable-events")>()),+ getBillableUnitCountsByProjectInWindow: mocks.ledgerCounts,+}));++import { compareWithBillingLedger } from "./billingLedgerShadow";++const FROM = new Date("2026-10-16T08:00:00.000Z");+const TO = new Date("2026-10-16T09:00:00.000Z");++const incrementsOf = (stat: string) =>+ mocks.recordIncrement.mock.calls+ .filter(([name]) => name === stat)+ .map(([, value, tags]) => ({ value, tags }));++describe("compareWithBillingLedger", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ });++ it("reports unit totals and logs the projects with the largest gaps", async () => {+ mocks.ledgerCounts.mockResolvedValue([+ { projectId: "p-diverging", eventType: "observation", count: 200 },+ { projectId: "p-busy", eventType: "observation", count: 99_000 },+ { projectId: "p-steady", eventType: "observation", count: 9_990 },+ { projectId: "p-ledger-only", eventType: "score", count: 150 },+ ]);++ await compareWithBillingLedger({+ source: "stripe_metering",+ from: FROM,+ to: TO,+ legacy: {+ traces: [],+ observations: [+ { projectId: "p-diverging", count: 1_000 },+ // Below the unit threshold, but 100% of its usage, and one-sided.+ { projectId: "p-small", count: 50 },+ // Only 1% of its usage, but 1,000 units apart.+ { projectId: "p-busy", count: 100_000 },+ // 10 units and 0.1% apart: not diverging.+ { projectId: "p-steady", count: 10_000 },+ ],+ scores: [],+ },+ orgIdByProjectId: new Map([["p-diverging", "org-1"]]),+ });++ expect(mocks.ledgerCounts).toHaveBeenCalledWith({+ windowFrom: FROM,+ windowTo: TO,+ projectId: undefined,+ });+ expect(incrementsOf("langfuse.billing.ledger_shadow.units")).toEqual(+ expect.arrayContaining([+ {+ value: 111_050,+ tags: {+ source: "stripe_metering",+ side: "legacy",+ unit_type: "observation",+ },+ },+ {+ value: 109_190,+ tags: {+ source: "stripe_metering",+ side: "ledger",+ unit_type: "observation",+ },+ },+ {+ value: 150,+ tags: {+ source: "stripe_metering",+ side: "ledger",+ unit_type: "score",+ },+ },+ ]),+ );+ expect(+ incrementsOf("langfuse.billing.ledger_shadow.diverging_projects"),+ ).toEqual([{ value: 4, tags: { source: "stripe_metering" } }]);+ expect(+ incrementsOf("langfuse.billing.ledger_shadow.one_sided_projects"),+ ).toEqual([+ { value: 1, tags: { source: "stripe_metering", missing_from: "ledger" } },+ { value: 1, tags: { source: "stripe_metering", missing_from: "legacy" } },+ ]);++ expect(mocks.logger.info).toHaveBeenCalledTimes(1);+ const [, logged] = mocks.logger.info.mock.calls[0];+ expect(+ logged.projects.map(({ projectId, orgId }: Record<string, unknown>) => ({+ projectId,+ orgId,+ })),+ ).toEqual([+ { projectId: "p-busy", orgId: undefined },+ { projectId: "p-diverging", orgId: "org-1" },+ { projectId: "p-ledger-only", orgId: undefined },+ { projectId: "p-small", orgId: undefined },+ ]);+ });++ it("flags a project whose unit types diverge in opposite directions", async () => {+ mocks.ledgerCounts.mockResolvedValue([+ { projectId: "p-offsetting", eventType: "trace", count: 300 },+ { projectId: "p-offsetting", eventType: "score", count: 200 },+ ]);++ await compareWithBillingLedger({+ source: "stripe_metering",+ from: FROM,+ to: TO,+ legacy: {+ traces: [{ projectId: "p-offsetting", count: 500 }],+ observations: [],+ scores: [],+ },+ });++ expect(+ incrementsOf("langfuse.billing.ledger_shadow.diverging_projects"),+ ).toEqual([{ value: 1, tags: { source: "stripe_metering" } }]);+ const [, logged] = mocks.logger.info.mock.calls[0];+ expect(logged.projects[0].delta).toEqual({+ trace: -200,+ observation: 0,+ score: 200,+ });+ });++ it("does not throw when the ledger read fails", async () => {+ mocks.ledgerCounts.mockRejectedValue(+ new Error("Table billable_events does not exist"),+ );++ await expect(+ compareWithBillingLedger({+ source: "chb_metrics_api",+ from: FROM,+ to: new Date("2026-10-16T08:15:00.000Z"),+ projectId: "p-1",+ legacy: { traces: [], observations: [], scores: [] },+ }),+ ).resolves.toBeUndefined();++ expect(+ incrementsOf("langfuse.billing.ledger_shadow.failed_comparisons"),+ ).toEqual([{ value: 1, tags: { source: "chb_metrics_api" } }]);+ expect(mocks.logger.warn).toHaveBeenCalled();+ });++ it("reads the ledger for the exact range it is given", async () => {+ mocks.ledgerCounts.mockResolvedValue([]);+ const from = new Date("2026-10-16T08:05:00.000Z");+ const to = new Date("2026-10-16T08:20:00.000Z");++ await compareWithBillingLedger({+ source: "chb_metrics_api",+ from,+ to,+ projectId: "p-1",+ legacy: { traces: [], observations: [], scores: [] },+ });++ expect(mocks.ledgerCounts).toHaveBeenCalledWith({+ windowFrom: from,+ windowTo: to,+ projectId: "p-1",+ });+ });++ it("skips a range longer than an hour", async () => {+ await compareWithBillingLedger({+ source: "chb_metrics_api",+ from: FROM,+ to: new Date("2026-10-16T09:00:01.000Z"),+ projectId: "p-1",+ legacy: { traces: [], observations: [], scores: [] },+ });++ expect(mocks.ledgerCounts).not.toHaveBeenCalled();+ expect(+ incrementsOf("langfuse.billing.ledger_shadow.skipped_comparisons"),+ ).toEqual([+ {+ value: 1,+ tags: { source: "chb_metrics_api", reason: "unsupported_range" },+ },+ ]);+ });+});diff --git a/packages/shared/src/server/billing/billingLedgerShadow.ts b/packages/shared/src/server/billing/billingLedgerShadow.tsnew file mode 100644index 0000000000..f39db750d1--- /dev/null+++ b/packages/shared/src/server/billing/billingLedgerShadow.ts@@ -0,0 +1,203 @@+import { recordIncrement } from "../instrumentation";+import { logger } from "../logger";+import {+ type BillableEventType,+ getBillableUnitCountsByProjectInWindow,+} from "../repositories/billable-events";++/** Billing consumer whose legacy counts the ledger is compared against. */+export type BillingLedgerShadowSource = "stripe_metering" | "chb_metrics_api";++type ProjectCount = { projectId: string; count: number };+type Units = Record<BillableEventType, number>;++// Bounds the ledger scan of one comparison. Stripe meters one hour, and the+// CHB endpoint accepts ranges of up to 35 days.+const MAX_COMPARED_RANGE_MS = 60 * 60 * 1000;++// A project diverges when one of its unit types is at least 100 units or 5% of+// its legacy count apart. Unit types are compared separately, so a gap in one+// cannot hide a gap of the opposite sign in another.+const DIVERGENCE_MIN_UNITS = 100;+const DIVERGENCE_MIN_RATIO = 0.05;+const LOGGED_PROJECTS = 10;++const UNIT_TYPES: readonly BillableEventType[] = [+ "trace",+ "observation",+ "score",+];++const zeroUnits = (): Units => ({ trace: 0, observation: 0, score: 0 });++const totalUnits = (units: Units) =>+ units.trace + units.observation + units.score;++/** Ledger minus legacy, per unit type. */+const deltaUnits = ({+ legacy,+ ledger,+}: {+ legacy: Units;+ ledger: Units;+}): Units => ({+ trace: ledger.trace - legacy.trace,+ observation: ledger.observation - legacy.observation,+ score: ledger.score - legacy.score,+});++const largestGap = (delta: Units) =>+ Math.max(...UNIT_TYPES.map((unitType) => Math.abs(delta[unitType])));++const isDiverging = ({ legacy, delta }: { legacy: Units; delta: Units }) =>+ UNIT_TYPES.some((unitType) => {+ const gap = Math.abs(delta[unitType]);+ return (+ gap > 0 &&+ (gap >= DIVERGENCE_MIN_UNITS ||+ gap >= DIVERGENCE_MIN_RATIO * legacy[unitType])+ );+ });++/**+ * Compares the legacy billing counts of `[from, to)` with the billable_events+ * ledger per project and reports the result. Metrics carry unit totals per+ * side and type, and the number of compared, diverging and one-sided projects.+ * One log line lists the projects with the largest gaps.+ *+ * Shadow only: it never changes what is billed and never throws. The legacy+ * score count leaves CORRECTION scores out and the ledger does not, so a+ * project with CORRECTION scores shows a score gap of that size.+ */+export const compareWithBillingLedger = async ({+ source,+ from,+ to,+ projectId,+ legacy,+ orgIdByProjectId,+}: {+ source: BillingLedgerShadowSource;+ from: Date;+ to: Date;+ /** Restricts the ledger read to one project, as the legacy counts are. */+ projectId?: string;+ legacy: {+ traces: ProjectCount[];+ observations: ProjectCount[];+ scores: ProjectCount[];+ };+ orgIdByProjectId?: Map<string, string>;+}): Promise<void> => {+ const rangeMs = to.getTime() - from.getTime();+ if (rangeMs <= 0 || rangeMs > MAX_COMPARED_RANGE_MS) {+ recordIncrement("langfuse.billing.ledger_shadow.skipped_comparisons", 1, {+ source,+ reason: "unsupported_range",+ });+ return;+ }++ try {+ const ledger = await getBillableUnitCountsByProjectInWindow({+ windowFrom: from,+ windowTo: to,+ projectId,+ });++ const projects = new Map<string, { legacy: Units; ledger: Units }>();+ const unitsOf = (id: string) => {+ let units = projects.get(id);+ if (!units) {+ units = { legacy: zeroUnits(), ledger: zeroUnits() };+ projects.set(id, units);+ }+ return units;+ };+ const legacyByType: Array<[BillableEventType, ProjectCount[]]> = [+ ["trace", legacy.traces],+ ["observation", legacy.observations],+ ["score", legacy.scores],+ ];+ for (const [eventType, rows] of legacyByType) {+ for (const row of rows)+ unitsOf(row.projectId).legacy[eventType] += row.count;+ }+ for (const row of ledger) {+ unitsOf(row.projectId).ledger[row.eventType] += row.count;+ }++ const comparisons = [...projects].map(([id, units]) => ({+ projectId: id,+ orgId: orgIdByProjectId?.get(id),+ legacy: units.legacy,+ ledger: units.ledger,+ delta: deltaUnits(units),+ }));+ const diverging = comparisons+ .filter(isDiverging)+ .sort((a, b) => largestGap(b.delta) - largestGap(a.delta));++ const totals = { legacy: zeroUnits(), ledger: zeroUnits() };+ for (const comparison of comparisons) {+ for (const unitType of UNIT_TYPES) {+ totals.legacy[unitType] += comparison.legacy[unitType];+ totals.ledger[unitType] += comparison.ledger[unitType];+ }+ }+ for (const unitType of UNIT_TYPES) {+ for (const side of ["legacy", "ledger"] as const) {+ recordIncrement(+ "langfuse.billing.ledger_shadow.units",+ totals[side][unitType],+ { source, side, unit_type: unitType },+ );+ }+ }+ recordIncrement(+ "langfuse.billing.ledger_shadow.compared_projects",+ comparisons.length,+ { source },+ );+ recordIncrement(+ "langfuse.billing.ledger_shadow.diverging_projects",+ diverging.length,+ { source },+ );+ // A project with usage on only one side points at a broken write path,+ // such as a materialized view that stopped filling the ledger.+ for (const side of ["ledger", "legacy"] as const) {+ const other = side === "ledger" ? "legacy" : "ledger";+ recordIncrement(+ "langfuse.billing.ledger_shadow.one_sided_projects",+ comparisons.filter(+ (comparison) =>+ totalUnits(comparison[side]) === 0 &&+ totalUnits(comparison[other]) > 0,+ ).length,+ { source, missing_from: side },+ );+ }++ if (diverging.length > 0) {+ logger.info(+ `[BILLING LEDGER SHADOW] ${diverging.length} of ${comparisons.length} projects diverge for ${source} in ${from.toISOString()} - ${to.toISOString()}`,+ {+ source,+ from: from.toISOString(),+ to: to.toISOString(),+ totals,+ projects: diverging.slice(0, LOGGED_PROJECTS),+ },+ );+ }+ } catch (error) {+ logger.warn(+ `[BILLING LEDGER SHADOW] Comparison for ${source} in ${from.toISOString()} - ${to.toISOString()} failed`,+ { error },+ );+ recordIncrement("langfuse.billing.ledger_shadow.failed_comparisons", 1, {+ source,+ });+ }+};diff --git a/packages/shared/src/server/index.ts b/packages/shared/src/server/index.tsindex 23187b079e..75d482ef2e 100644--- a/packages/shared/src/server/index.ts+++ b/packages/shared/src/server/index.ts@@ -159,6 +159,7 @@ export * from "./queries";export * from "./queries/clickhouse-sql/orderby-factory";export * from "./queries/clickhouse-sql/query-options";export * from "./repositories";+export * from "./billing/billingLedgerShadow";export { SCORE_TO_TRACE_OBSERVATIONS_INTERVAL } from "./repositories/constants";export { scoreDomainToV3 } from "./repositories/scores";export * from "./repositories/observations";diff --git a/web/src/__tests__/server/unit/chb-metrics-api-handler.servertest.ts b/web/src/__tests__/server/unit/chb-metrics-api-handler.servertest.tsindex 11ebde7908..6b7b437c18 100644--- a/web/src/__tests__/server/unit/chb-metrics-api-handler.servertest.ts+++ b/web/src/__tests__/server/unit/chb-metrics-api-handler.servertest.ts@@ -6,12 +6,14 @@ const mocks = vi.hoisted(() => ({env: {NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "eu" as string | undefined,CLICKHOUSE_BILLING_METRICS_API_KEY: "chb-secret" as string | undefined,+ LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED: "false",},findProject: vi.fn(),traceCounts: vi.fn(),observationCounts: vi.fn(),scoreCounts: vi.fn(),recordIncrement: vi.fn(),+ compareWithBillingLedger: vi.fn(),}));vi.mock("@/src/env.mjs", () => ({ env: mocks.env }));@@ -31,6 +33,7 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => {getObservationCountsByProjectInCreationInterval: mocks.observationCounts,getScoreCountsByProjectInCreationInterval: mocks.scoreCounts,recordIncrement: mocks.recordIncrement,+ compareWithBillingLedger: mocks.compareWithBillingLedger,};});@@ -70,6 +73,7 @@ describe("chbMetricsApiHandler", () => {vi.clearAllMocks();mocks.env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "eu";mocks.env.CLICKHOUSE_BILLING_METRICS_API_KEY = "chb-secret";+ mocks.env.LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED = "false";mocks.findProject.mockResolvedValue({ id: PROJECT_ID });mocks.traceCounts.mockResolvedValue([{ projectId: PROJECT_ID, count: 7 }]);mocks.observationCounts.mockResolvedValue([@@ -212,4 +216,44 @@ describe("chbMetricsApiHandler", () => {});expect(mocks.traceCounts).not.toHaveBeenCalled();});++ it("compares the poll with the billing ledger without waiting for it", async () => {+ mocks.env.LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED = "true";+ // Never settles: awaiting the comparison would hang the poll.+ mocks.compareWithBillingLedger.mockReturnValue(new Promise(() => {}));++ const response = await chbMetricsApiHandler(+ createRequest({+ startTime: "2026-07-01T00:00:00Z",+ endTime: "2026-07-01T00:15:00Z",+ }),+ );++ expect(response.status).toBe(200);+ expect(await response.json()).toEqual({+ metrics: {+ traces: { sum: 7 },+ observations: { sum: 11 },+ scores: { sum: 3 },+ },+ });+ expect(mocks.compareWithBillingLedger).toHaveBeenCalledWith({+ source: "chb_metrics_api",+ from: new Date("2026-07-01T00:00:00Z"),+ to: new Date("2026-07-01T00:15:00Z"),+ projectId: PROJECT_ID,+ legacy: {+ traces: [{ projectId: PROJECT_ID, count: 7 }],+ observations: [{ projectId: PROJECT_ID, count: 11 }],+ scores: [{ projectId: PROJECT_ID, count: 3 }],+ },+ });+ });++ it("does not read the billing ledger unless enabled", async () => {+ const response = await chbMetricsApiHandler(createRequest());++ expect(response.status).toBe(200);+ expect(mocks.compareWithBillingLedger).not.toHaveBeenCalled();+ });});diff --git a/web/src/ee/features/billing/server/chb/chbMetricsApiHandler.ts b/web/src/ee/features/billing/server/chb/chbMetricsApiHandler.tsindex aca5bd9122..2796431174 100644--- a/web/src/ee/features/billing/server/chb/chbMetricsApiHandler.ts+++ b/web/src/ee/features/billing/server/chb/chbMetricsApiHandler.ts@@ -7,6 +7,7 @@ import { z } from "zod";import { env } from "@/src/env.mjs";import { prisma } from "@langfuse/shared/src/db";import {+ compareWithBillingLedger,getObservationCountsByProjectInCreationInterval,getScoreCountsByProjectInCreationInterval,getTraceCountsByProjectInCreationInterval,@@ -149,6 +150,22 @@ export async function chbMetricsApiHandler(req: NextRequest) {}),]);+ // Not awaited, so the comparison never delays CHB's poll, and it never+ // throws. Windows longer than an hour are skipped.+ if (env.LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED === "true") {+ compareWithBillingLedger({+ source: "chb_metrics_api",+ from: start,+ to: end,+ projectId: resourceId,+ legacy: {+ traces: traceCounts,+ observations: observationCounts,+ scores: scoreCounts,+ },+ });+ }+return NextResponse.json({metrics: {diff --git a/web/src/env.mjs b/web/src/env.mjsindex dcc60f0932..7fcb78fb25 100644--- a/web/src/env.mjs+++ b/web/src/env.mjs@@ -522,6 +522,12 @@ export const env = createEnv({// Bearer secret CHB's metering pipeline presents to GET /api/billing/metrics.// Unset disables the endpoint (it 500s) rather than leaving it open.CLICKHOUSE_BILLING_METRICS_API_KEY: z.string().optional(),+ // Compares each GET /api/billing/metrics poll with the billable_events+ // ledger and reports the gap. Requires the ledger table, which only+ // Langfuse Cloud has; keep in sync with worker/src/env.ts.+ LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED: z+ .enum(["true", "false"])+ .default("false"),// ARN of CHB's cross-account EventBridge bus we publish project lifecycle// events to. Unset = no events are emitted. The ARN carries the bus region,// so it is the only input needed; authentication is SigV4 from the task@@ -1219,6 +1225,8 @@ export const env = createEnv({process.env.LANGFUSE_CLOUD_BILLING_CHB_OPT_IN_ORG_IDS,CLICKHOUSE_BILLING_METRICS_API_KEY:process.env.CLICKHOUSE_BILLING_METRICS_API_KEY,+ LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED:+ process.env.LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED,CLICKHOUSE_BILLING_EVENT_BUS_ARN:process.env.CLICKHOUSE_BILLING_EVENT_BUS_ARN,CLICKHOUSE_BILLING_BASE_URL: process.env.CLICKHOUSE_BILLING_BASE_URL,diff --git a/worker/src/ee/cloudUsageMetering/__tests__/handleCloudUsageMeteringJob.test.ts b/worker/src/ee/cloudUsageMetering/__tests__/handleCloudUsageMeteringJob.test.tsindex 95c7babaa8..7a607c32fb 100644--- a/worker/src/ee/cloudUsageMetering/__tests__/handleCloudUsageMeteringJob.test.ts+++ b/worker/src/ee/cloudUsageMetering/__tests__/handleCloudUsageMeteringJob.test.ts@@ -16,6 +16,8 @@ const {mockLoggerError,mockRecordIncrement,mockTraceException,+ mockCompareWithBillingLedger,+ mockEnv,} = vi.hoisted(() => ({mockCronJobsUpsert: vi.fn(),mockCronJobsUpdate: vi.fn(),@@ -31,6 +33,12 @@ const {mockLoggerError: vi.fn(),mockRecordIncrement: vi.fn(),mockTraceException: vi.fn(),+ mockCompareWithBillingLedger: vi.fn(),+ mockEnv: {+ STRIPE_SECRET_KEY: "sk_test",+ LANGFUSE_CLOUD_BILLING_CHB_CUTOFF_DATE: undefined,+ LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED: "false",+ },}));vi.mock("@langfuse/shared", () => ({@@ -58,6 +66,7 @@ vi.mock("@langfuse/shared/src/server", () => ({CloudSpendAlertQueue: {getInstance: () => ({ add: mockSpendAlertQueueAdd }),},+ compareWithBillingLedger: mockCompareWithBillingLedger,getObservationCountsByProjectInCreationInterval: mockObservationCounts,getTraceCountsByProjectInCreationInterval: mockTraceCounts,getScoreCountsByProjectInCreationInterval: mockScoreCounts,@@ -80,12 +89,7 @@ vi.mock("stripe", () => ({},}));-vi.mock("../../../env", () => ({- env: {- STRIPE_SECRET_KEY: "sk_test",- LANGFUSE_CLOUD_BILLING_CHB_CUTOFF_DATE: undefined,- },-}));+vi.mock("../../../env", () => ({ env: mockEnv }));import { handleCloudUsageMeteringJob } from "../handleCloudUsageMeteringJob";import { CloudUsageMeteringDbCronJobStates } from "../constants";@@ -132,6 +136,7 @@ describe("handleCloudUsageMeteringJob", () => {vi.clearAllMocks();vi.useFakeTimers({ shouldAdvanceTime: true });vi.setSystemTime(NOW);+ mockEnv.LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED = "false";mockCronJobsUpsert.mockResolvedValue({name: "cloud_usage_metering",@@ -140,6 +145,7 @@ describe("handleCloudUsageMeteringJob", () => {jobStartedAt: null,});mockCronJobsUpdate.mockResolvedValue({});+ mockMeterEventsCreate.mockResolvedValue({});mockOrganizationFindMany.mockResolvedValue([{id: "org-1",@@ -305,4 +311,38 @@ describe("handleCloudUsageMeteringJob", () => {expect.anything(),);});++ it("compares the interval with the billing ledger after sending it to Stripe", async () => {+ mockEnv.LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED = "true";++ await handleCloudUsageMeteringJob(createJob());++ expect(mockCompareWithBillingLedger).toHaveBeenCalledTimes(1);+ expect(mockCompareWithBillingLedger).toHaveBeenCalledWith({+ source: "stripe_metering",+ from: INTERVAL_START,+ to: new Date(INTERVAL_START.getTime() + 3600000),+ legacy: {+ traces: [{ projectId: "project-1", count: 3 }],+ observations: [{ projectId: "project-1", count: 5 }],+ scores: [{ projectId: "project-1", count: 2 }],+ },+ orgIdByProjectId: new Map([["project-1", "org-1"]]),+ });+ // The comparison must not hold up metering: it starts only once Stripe+ // has the interval and the lease is released.+ const comparedAt = mockCompareWithBillingLedger.mock.invocationCallOrder[0];+ expect(comparedAt).toBeGreaterThan(+ Math.max(...mockMeterEventsCreate.mock.invocationCallOrder),+ );+ expect(comparedAt).toBeGreaterThan(+ mockCronJobsUpdate.mock.invocationCallOrder.at(-1) ?? Infinity,+ );+ });++ it("does not read the billing ledger unless enabled", async () => {+ await handleCloudUsageMeteringJob(createJob());++ expect(mockCompareWithBillingLedger).not.toHaveBeenCalled();+ });});diff --git a/worker/src/ee/cloudUsageMetering/handleCloudUsageMeteringJob.ts b/worker/src/ee/cloudUsageMetering/handleCloudUsageMeteringJob.tsindex 9f0b8cceb1..f9fa0c49be 100644--- a/worker/src/ee/cloudUsageMetering/handleCloudUsageMeteringJob.ts+++ b/worker/src/ee/cloudUsageMetering/handleCloudUsageMeteringJob.ts@@ -5,6 +5,7 @@ import { env } from "../../env";import {CloudUsageMeteringQueue,CloudSpendAlertQueue,+ compareWithBillingLedger,getObservationCountsByProjectInCreationInterval,getScoreCountsByProjectInCreationInterval,getTraceCountsByProjectInCreationInterval,@@ -433,6 +434,27 @@ export const handleCloudUsageMeteringJob = async (job: Job) => {},);+ // Runs after the interval is metered and the lease released, and never+ // throws, so it cannot delay or change what Stripe receives. It compares+ // every project with usage, not only the Stripe-billed ones.+ if (env.LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED === "true") {+ await compareWithBillingLedger({+ source: "stripe_metering",+ from: meterIntervalStart,+ to: meterIntervalEnd,+ legacy: {+ traces: traceCountsByProject,+ observations: observationCountsByProject,+ scores: scoreCountsByProject,+ },+ orgIdByProjectId: new Map(+ organizations.flatMap((org) =>+ org.projectIds.map((projectId) => [projectId, org.id] as const),+ ),+ ),+ });+ }+if (meterIntervalEnd.getTime() + delayFromStartOfInterval < Date.now()) {logger.info(`[CLOUD USAGE METERING] Enqueueing next Cloud Usage Metering Job to catch up `,diff --git a/worker/src/env.ts b/worker/src/env.tsindex 09f87e93fa..9c97b0a369 100644--- a/worker/src/env.ts+++ b/worker/src/env.ts@@ -64,6 +64,12 @@ const EnvSchema = z.object({// CHB's resource-server identifier. Defaulted because it is the same value// in every CHB tenant, and overridable in case that stops being true.CLICKHOUSE_BILLING_AUTH0_AUDIENCE: z.string().default("billing-api"),+ // Compares each metered Stripe interval with the billable_events ledger and+ // reports the gap. Requires the ledger table, which only Langfuse Cloud has;+ // keep in sync with web/src/env.mjs.+ LANGFUSE_BILLING_LEDGER_SHADOW_ENABLED: z+ .enum(["true", "false"])+ .default("false"),LANGFUSE_CACHE_AUTOMATIONS_ENABLED: z.enum(["true", "false"]).default("true"),LANGFUSE_CACHE_AUTOMATIONS_TTL_SECONDS: z.coerce.number().default(60),