🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(billing): count the plan fee in spend alerts for ClickHouse-billed orgs (#18739)
Steffen Reuther0d85902af3b0
diff --git a/packages/shared/src/server/ee/clickhouseBilling/chbApiClient.test.ts b/packages/shared/src/server/ee/clickhouseBilling/chbApiClient.test.tsindex 3e77503db6..c954b46f91 100644--- a/packages/shared/src/server/ee/clickhouseBilling/chbApiClient.test.ts+++ b/packages/shared/src/server/ee/clickhouseBilling/chbApiClient.test.ts@@ -18,6 +18,7 @@ import {ChbApiClient,ChbApiError,ChbPaymentRequiredError,+ chbBillPlanAmountUSD,chbBillUsageAmountUSD,} from "./chbApiClient";@@ -216,7 +217,7 @@ describe("chbApiClient", () => {const { url, init } = lastChbCall();expect(init.method).toBe("POST");expect(url.pathname).toBe("/api/v1/bills/query");- expect(url.searchParams.get("fields")).toBe("usage");+ expect(url.searchParams.get("fields")).toBe("usage,plan");expect(JSON.parse(init.body as string)).toEqual({organizationIds: [CH_ORG_ID],});@@ -457,6 +458,12 @@ describe("chbApiClient", () => {currency: "USD",asOf: "2026-09-29T12:00:00.000Z",},+ plan: {+ amount: 199,+ currency: "USD",+ startDate: "2026-10-01T00:00:00.000Z",+ endDate: "2026-11-01T00:00:00.000Z",+ },},{ id: "bill_2", organizationId: "other-org", extra: true },]),@@ -468,7 +475,9 @@ describe("chbApiClient", () => {expect(bills.map((bill) => bill.id)).toEqual(["bill_1", "bill_2"]);expect(bills[0]!.usage?.amount).toBe(1250.68);+ expect(bills[0]!.plan?.amount).toBe(199);expect(bills[1]!.usage).toBeUndefined();+ expect(bills[1]!.plan).toBeUndefined();});it("reads the portal URL from CHB's portalUrl field", async () => {@@ -623,4 +632,38 @@ describe("chbApiClient", () => {).toBeNull();});});++ /**+ * The plan fee shares the usage amount's major-unit/USD contract; what it+ * adds is that CHB leaves `plan` out for a bill without a standing charge.+ */+ describe("chbBillPlanAmountUSD", () => {+ const withPlan = (plan: unknown) =>+ ({+ id: "bill_1",+ organizationId: CH_ORG_ID,+ startDate: "2026-09-01T00:00:00.000Z",+ plan,+ }) as Parameters<typeof chbBillPlanAmountUSD>[0];++ it("returns the amount unchanged as USD", () => {+ expect(+ chbBillPlanAmountUSD(withPlan({ amount: 499.9, currency: "USD" })),+ ).toBe(499.9);+ });++ /** No standing charge is a fee of 0; a fee without a figure is no reading. */+ it("distinguishes an absent plan from a plan without a usable amount", () => {+ expect(+ chbBillPlanAmountUSD({ id: "bill_1", organizationId: CH_ORG_ID }),+ ).toBe(0);+ expect(chbBillPlanAmountUSD(withPlan(null))).toBe(0);+ expect(+ chbBillPlanAmountUSD(withPlan({ amount: null, currency: "USD" })),+ ).toBeNull();+ expect(+ chbBillPlanAmountUSD(withPlan({ amount: 199, currency: "EUR" })),+ ).toBeNull();+ });+ });});diff --git a/packages/shared/src/server/ee/clickhouseBilling/chbApiClient.ts b/packages/shared/src/server/ee/clickhouseBilling/chbApiClient.tsindex 437529b9c2..076045795e 100644--- a/packages/shared/src/server/ee/clickhouseBilling/chbApiClient.ts+++ b/packages/shared/src/server/ee/clickhouseBilling/chbApiClient.ts@@ -99,11 +99,12 @@ const ChbBillSchema = z.object({// Exclusive, like the attached plan's period end.endDate: z.string().nullish(),// Accrued metered usage of the open bill: CHB counts only its USAGE line- // items, so the plan's fixed fee (STANDING_CHARGE) and prorated upgrade- // deltas (DEBIT_MEMO) are not in it. `amount` is a **major-unit** USD figure- // (1250.68 means $1,250.68) -- unlike `ChbInvoice.amount`, which is minor- // units. Read it through `chbBillUsageAmountUSD`, which is the one place- // that encodes this. `asOf` is when CHB last exported the bill.+ // items, so the plan's fixed fee (STANDING_CHARGE, reported as `plan`) and+ // prorated upgrade deltas (DEBIT_MEMO, invoiced on their own) are not in it.+ // `amount` is a **major-unit** USD figure (1250.68 means $1,250.68) --+ // unlike `ChbInvoice.amount`, which is minor units. Read it through+ // `chbBillUsageAmountUSD`, which is the one place that encodes this. `asOf`+ // is when CHB last exported the bill.usage: z.object({amount: z.number().nullish(),@@ -111,6 +112,19 @@ const ChbBillSchema = z.object({asOf: z.string().nullish(),}).nullish(),+ // The plan's fixed fee on the open bill: the total of its STANDING_CHARGE+ // line items, in the same major-unit USD as `usage`. The fee is billed in+ // advance, so `startDate`/`endDate` are the period it covers, which can be+ // the one after the bill's own. Absent when the bill has no standing charge.+ // Read it through `chbBillPlanAmountUSD`.+ plan: z+ .object({+ amount: z.number().nullish(),+ currency: z.string().nullish(),+ startDate: z.string().nullish(),+ endDate: z.string().nullish(),+ })+ .nullish(),});export type ChbBill = z.infer<typeof ChbBillSchema>;@@ -401,12 +415,12 @@ export class ChbApiClient {}/**- * The open bill of each organization, with its accrued usage. CHB serves this- * from its billing warehouse rather than from m3ter, so it can be polled- * without spending m3ter's rate limit, at the cost of lagging by up to one- * warehouse export (`usage.asOf`). Not scoped by header: the organizations- * are the request body, 1 to 100 of them. An organization without an open- * bill is absent from the result rather than an error.+ * The open bill of each organization, with its accrued usage and plan fee.+ * CHB serves this from its billing warehouse rather than from m3ter, so it+ * can be polled without spending m3ter's rate limit, at the cost of lagging+ * by up to one warehouse export (`usage.asOf`). Not scoped by header: the+ * organizations are the request body, 1 to 100 of them. An organization+ * without an open bill is absent from the result rather than an error.*/async queryBills(params: {chOrganizationIds: string[];@@ -417,7 +431,7 @@ export class ChbApiClient {method: "POST",path: "bills/query",body: { organizationIds: params.chOrganizationIds },- searchParams: { fields: "usage" },+ searchParams: { fields: "usage,plan" },});return ChbBillListSchema.parse(body);}@@ -454,31 +468,49 @@ export class ChbApiClient {}}-/** The currency CHB reports accrued usage in; it rejects every other one. */+/** The currency CHB reports bill amounts in; it rejects every other one. */export const CHB_USAGE_CURRENCY = "USD";/**- * The open bill's accrued usage in USD, or null when CHB did not report a- * usable figure. Returning null rather than 0 keeps "no reading" distinct from- * "nothing spent yet" -- a caller comparing against a spend threshold must skip- * the former and evaluate the latter.- *- * A currency other than USD is treated as no reading: CHB asserts USD when it- * computes the amount, so anything else means the contract moved and the number- * can no longer be compared against a USD threshold.+ * A bill amount in USD, or null when it is not a usable figure. A currency+ * other than USD is treated as no reading: CHB asserts USD when it computes+ * the amount, so anything else means the contract moved and the number can no+ * longer be compared against a USD threshold.*/-export const chbBillUsageAmountUSD = (bill: ChbBill): number | null => {- const usage = bill.usage;- if (usage?.amount == null || !Number.isFinite(usage.amount)) return null;+const chbAmountUSD = (+ money:+ | { amount?: number | null; currency?: string | null }+ | null+ | undefined,+): number | null => {+ if (money?.amount == null || !Number.isFinite(money.amount)) return null;if (- usage.currency != null &&- usage.currency.toUpperCase() !== CHB_USAGE_CURRENCY+ money.currency != null &&+ money.currency.toUpperCase() !== CHB_USAGE_CURRENCY) {return null;}- return usage.amount;+ return money.amount;};+/**+ * The open bill's accrued usage in USD, or null when CHB did not report a+ * usable figure. Returning null rather than 0 keeps "no reading" distinct from+ * "nothing spent yet" -- a caller comparing against a spend threshold must skip+ * the former and evaluate the latter.+ */+export const chbBillUsageAmountUSD = (bill: ChbBill): number | null =>+ chbAmountUSD(bill.usage);++/**+ * The open bill's plan fee in USD: 0 when the bill has no standing charge+ * (CHB then leaves `plan` out), null when CHB reported one without a usable+ * figure. Like `chbBillUsageAmountUSD`, null is "no reading", which a caller+ * must not fold into the total as 0.+ */+export const chbBillPlanAmountUSD = (bill: ChbBill): number | null =>+ bill.plan == null ? 0 : chbAmountUSD(bill.plan);+/*** The CHB REST configuration an app reads from its own environment. Optional* everywhere except the audience, which is defaulted in both env schemas.diff --git a/worker/src/ee/cloudSpendAlerts/__tests__/handleCloudSpendAlertJob.test.ts b/worker/src/ee/cloudSpendAlerts/__tests__/handleCloudSpendAlertJob.test.tsindex ff3326c38f..6555b50fef 100644--- a/worker/src/ee/cloudSpendAlerts/__tests__/handleCloudSpendAlertJob.test.ts+++ b/worker/src/ee/cloudSpendAlerts/__tests__/handleCloudSpendAlertJob.test.ts@@ -63,8 +63,8 @@ vi.mock("@langfuse/shared/src/db", () => ({/*** Only the side-effecting exports (logging, metrics, email) are replaced. The- * CHB helpers stay real: `chbBillUsageAmountUSD` encodes the major-unit/USD-only- * contract these cases exist to pin, and `ChbApiError` is what the job's status+ * CHB helpers stay real: `chbBillUsageAmountUSD` and `chbBillPlanAmountUSD`+ * encode the major-unit/USD-only contract these cases exist to pin, and `ChbApiError` is what the job's status* branching uses `instanceof` against.*/vi.mock("@langfuse/shared/src/server", () => ({@@ -135,11 +135,16 @@ const chbCloudConfig = () => ({},});-/** A `POST /bills/query` result holding the org's one open bill. */+/**+ * A `POST /bills/query` result holding the org's one open bill. Without a+ * `plan`, the bill carries no standing charge, which CHB reports by leaving the+ * field out.+ */const openBill = (amount: number | null,currency: string | null = "USD",startDate: string | null = "2026-09-01T00:00:00.000Z",+ plan?: { amount: number | null; currency: string | null },) => [{id: "bill_1",@@ -147,6 +152,14 @@ const openBill = (startDate,endDate: "2026-10-01T00:00:00.000Z",usage: { amount, currency, asOf: "2026-09-29T12:00:00.000Z" },+ ...(plan && {+ plan: {+ ...plan,+ // Billed in advance: the fee on this bill covers the next period.+ startDate: "2026-10-01T00:00:00.000Z",+ endDate: "2026-11-01T00:00:00.000Z",+ },+ }),},];@@ -264,6 +277,41 @@ describe("handleCloudSpendAlertJob", () => {});});+ /**+ * The alert is set against the upcoming invoice, base fee included, the+ * same total Stripe's preview invoice reports.+ */+ it("counts the plan fee toward the threshold", async () => {+ mockOrganizationFindFirst.mockResolvedValue(createOrg(chbCloudConfig()));+ mockQueryBills.mockResolvedValue(+ openBill(150.5, "USD", undefined, { amount: 59, currency: "USD" }),+ );++ await handleCloudSpendAlertJob(createJob());++ expect(mockSendCloudSpendAlertEmail).toHaveBeenCalledTimes(1);+ expect(mockSendCloudSpendAlertEmail.mock.calls[0]![0]).toMatchObject({+ currentSpend: 209.5,+ threshold: 200,+ });+ });++ /**+ * Comparing usage alone would report less than the org owes and alert+ * late, so a plan fee CHB cannot express in USD skips the org instead.+ */+ it("skips when CHB reports a plan fee it cannot express in USD", async () => {+ mockOrganizationFindFirst.mockResolvedValue(createOrg(chbCloudConfig()));+ mockQueryBills.mockResolvedValue(+ openBill(250, "USD", undefined, { amount: 59, currency: "EUR" }),+ );++ await handleCloudSpendAlertJob(createJob());++ expect(mockSendCloudSpendAlertEmail).not.toHaveBeenCalled();+ expect(skipReasons()).toContain("missing_chb_plan_amount");+ });+it("does not alert below the threshold", async () => {mockOrganizationFindFirst.mockResolvedValue(createOrg(chbCloudConfig()));mockQueryBills.mockResolvedValue(openBill(199.99));diff --git a/worker/src/ee/cloudSpendAlerts/handleCloudSpendAlertJob.ts b/worker/src/ee/cloudSpendAlerts/handleCloudSpendAlertJob.tsindex fd2b395d86..89a406fb47 100644--- a/worker/src/ee/cloudSpendAlerts/handleCloudSpendAlertJob.ts+++ b/worker/src/ee/cloudSpendAlerts/handleCloudSpendAlertJob.ts@@ -6,6 +6,7 @@ import { logger } from "@langfuse/shared/src/server";import {CHB_USAGE_CURRENCY,ChbApiError,+ chbBillPlanAmountUSD,chbBillUsageAmountUSD,} from "@langfuse/shared/src/server/ee/clickhouseBilling";import { recordIncrement, traceException } from "@langfuse/shared/src/server";@@ -28,6 +29,7 @@ type MissingBillingConfigReason =| "chb_not_configured"| "chb_open_bill_not_found"| "missing_chb_usage_amount"+ | "missing_chb_plan_amount"| "missing_chb_period_start";const recordMissingBillingConfigSkip = (reason: MissingBillingConfigReason) => {@@ -127,12 +129,15 @@ const readStripeSpend = async (};/**- * CHB's reading: the accrued usage on the org's open bill. This is the CHB- * equivalent of Stripe's preview invoice — CHB issues no preview invoice, but- * its billing warehouse reports what the open bill has accrued so far, in- * major-unit USD (see `chbBillUsageAmountUSD`). The warehouse, not m3ter,- * serves this read, which is what makes hourly polling affordable on CHB's- * side; the price is a reading up to one warehouse export old.+ * CHB's reading: the accrued usage plus the plan fee on the org's open bill.+ * This is the CHB equivalent of Stripe's preview invoice — CHB issues no+ * preview invoice, but its billing warehouse reports what the open bill has+ * accrued so far and the plan fee it carries, both in major-unit USD (see+ * `chbBillUsageAmountUSD` and `chbBillPlanAmountUSD`). Like Stripe's, that fee+ * is billed in advance, so it can cover the next period rather than this one.+ * The warehouse, not m3ter, serves this read, which is what makes hourly+ * polling affordable on CHB's side; the price is a reading up to one warehouse+ * export old.*/const readClickhouseSpend = async (org: OrgWithAlerts,@@ -197,11 +202,11 @@ const readClickhouseSpend = async (// reading of CHB's amount is observable in production, so a contract change// shows up here rather than as alerts that quietly stop matching reality.logger.info(- `[CLOUD SPEND ALERTS] Org ${org.id} CHB bill ${bill.id} reports accrued usage ${bill.usage?.amount} ${bill.usage?.currency ?? CHB_USAGE_CURRENCY} as of ${bill.usage?.asOf} for period ${bill.startDate} - ${bill.endDate}`,+ `[CLOUD SPEND ALERTS] Org ${org.id} CHB bill ${bill.id} reports accrued usage ${bill.usage?.amount} ${bill.usage?.currency ?? CHB_USAGE_CURRENCY} as of ${bill.usage?.asOf} and plan fee ${bill.plan ? `${bill.plan.amount} ${bill.plan.currency ?? CHB_USAGE_CURRENCY}` : "none"} for period ${bill.startDate} - ${bill.endDate}`,);- const currentSpendUSD = chbBillUsageAmountUSD(bill);- if (currentSpendUSD === null) {+ const usageUSD = chbBillUsageAmountUSD(bill);+ if (usageUSD === null) {logger.warn(`[CLOUD SPEND ALERTS] CHB reported no usable ${CHB_USAGE_CURRENCY} usage amount for org ${org.id}, skipping`,);@@ -209,6 +214,16 @@ const readClickhouseSpend = async (return null;}+ const planUSD = chbBillPlanAmountUSD(bill);+ if (planUSD === null) {+ // Usage alone would understate what the org owes and alert late.+ logger.warn(+ `[CLOUD SPEND ALERTS] CHB reported no usable ${CHB_USAGE_CURRENCY} plan fee for org ${org.id}, skipping`,+ );+ recordMissingBillingConfigSkip("missing_chb_plan_amount");+ return null;+ }+const periodStart = bill.startDate ? new Date(bill.startDate) : null;if (!periodStart || Number.isNaN(periodStart.getTime())) {// Without a period start an alert cannot be held to once per cycle, and@@ -220,7 +235,10 @@ const readClickhouseSpend = async (return null;}- return { currentSpendUSD, currentPeriodStart: periodStart };+ return {+ currentSpendUSD: usageUSD + planUSD,+ currentPeriodStart: periodStart,+ };};export const handleCloudSpendAlertJob = async (job: Job<{ orgId: string }>) => {