πͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(billing): send spend alerts for ClickHouse-billed organizations (#17615)
Steffen Reuther74acb1ab2f8c
diff --git a/packages/shared/package.json b/packages/shared/package.jsonindex 5f4adbd914..21723a2bc1 100644--- a/packages/shared/package.json+++ b/packages/shared/package.json@@ -91,6 +91,10 @@"import": "./dist/src/server/ee/licenseCheck/index.js","require": "./dist/src/server/ee/licenseCheck/index.js"},+ "./src/server/ee/clickhouseBilling": {+ "import": "./dist/src/server/ee/clickhouseBilling/index.js",+ "require": "./dist/src/server/ee/clickhouseBilling/index.js"+ },"./query": {"import": "./dist/src/features/query/index.js","require": "./dist/src/features/query/index.js"diff --git a/web/src/__tests__/server/unit/chbAccessToken.servertest.ts b/packages/shared/src/server/ee/clickhouseBilling/chbAccessToken.test.tssimilarity index 94%rename from web/src/__tests__/server/unit/chbAccessToken.servertest.tsrename to packages/shared/src/server/ee/clickhouseBilling/chbAccessToken.test.tsindex 079dd455a4..c3dfd29606 100644--- a/web/src/__tests__/server/unit/chbAccessToken.servertest.ts+++ b/packages/shared/src/server/ee/clickhouseBilling/chbAccessToken.test.ts@@ -1,19 +1,12 @@-import type * as SharedServer from "@langfuse/shared/src/server";import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";const mocks = vi.hoisted(() => ({logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },}));-vi.mock("@langfuse/shared/src/server", async (importOriginal) => {- const actual = await importOriginal<typeof SharedServer>();- return { ...actual, logger: mocks.logger };-});+vi.mock("../../logger", () => ({ logger: mocks.logger }));-import {- ChbAccessTokenProvider,- ChbAuthError,-} from "@/src/ee/features/billing/server/chb/chbAccessToken";+import { ChbAccessTokenProvider, ChbAuthError } from "./chbAccessToken";const AUTH0_DOMAIN = "chb-tenant.eu.auth0.com";const TOKEN_URL = `https://${AUTH0_DOMAIN}/oauth/token`;diff --git a/web/src/ee/features/billing/server/chb/chbAccessToken.ts b/packages/shared/src/server/ee/clickhouseBilling/chbAccessToken.tssimilarity index 98%rename from web/src/ee/features/billing/server/chb/chbAccessToken.tsrename to packages/shared/src/server/ee/clickhouseBilling/chbAccessToken.tsindex 38a7c7d6d0..2921e55c90 100644--- a/web/src/ee/features/billing/server/chb/chbAccessToken.ts+++ b/packages/shared/src/server/ee/clickhouseBilling/chbAccessToken.ts@@ -1,7 +1,8 @@import { SpanKind, type Span } from "@opentelemetry/api";import { z } from "zod";-import { instrumentAsync, logger } from "@langfuse/shared/src/server";+import { instrumentAsync } from "../../instrumentation";+import { logger } from "../../logger";/*** Auth0 client-credentials (M2M) token source for the ClickHouse Billing (CHB)diff --git a/web/src/__tests__/server/unit/chbApiClient.servertest.ts b/packages/shared/src/server/ee/clickhouseBilling/chbApiClient.test.tssimilarity index 81%rename from web/src/__tests__/server/unit/chbApiClient.servertest.tsrename to packages/shared/src/server/ee/clickhouseBilling/chbApiClient.test.tsindex f7b2b7a204..3e77503db6 100644--- a/web/src/__tests__/server/unit/chbApiClient.servertest.ts+++ b/packages/shared/src/server/ee/clickhouseBilling/chbApiClient.test.ts@@ -1,4 +1,3 @@-import type * as SharedServer from "@langfuse/shared/src/server";import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";const mocks = vi.hoisted(() => ({@@ -12,21 +11,21 @@ const mocks = vi.hoisted(() => ({logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },}));-vi.mock("@/src/env.mjs", () => ({ env: mocks.env }));--vi.mock("@langfuse/shared/src/server", async (importOriginal) => {- const actual = await importOriginal<typeof SharedServer>();- return { ...actual, logger: mocks.logger };-});+vi.mock("../../logger", () => ({ logger: mocks.logger }));import {- buildChbApiClientFromEnv,+ buildChbApiClient,ChbApiClient,ChbApiError,ChbPaymentRequiredError,- getChbApiClient,- resetChbApiClientForTests,-} from "@/src/ee/features/billing/server/chb/chbApiClient";+ chbBillUsageAmountUSD,+} from "./chbApiClient";++/**+ * The env-reading wrapper each app keeps for itself; exercised here against the+ * shared fail-closed rule it delegates to.+ */+const buildChbApiClientFromEnv = () => buildChbApiClient(mocks.env);const CH_ORG_ID = "6dd6ab1d-9e8d-4c1a-8b4f-9a3d1e2c4b5a";const AUTH0_DOMAIN = "chb-tenant.eu.auth0.com";@@ -209,6 +208,21 @@ describe("chbApiClient", () => {expect(url.searchParams.get("organizationId")).toBe(CH_ORG_ID);});+ it("queries bills by organization ids in the body, not by header", async () => {+ onChb(jsonResponse(200, []));++ await client().queryBills({ chOrganizationIds: [CH_ORG_ID] });++ const { url, init } = lastChbCall();+ expect(init.method).toBe("POST");+ expect(url.pathname).toBe("/api/v1/bills/query");+ expect(url.searchParams.get("fields")).toBe("usage");+ expect(JSON.parse(init.body as string)).toEqual({+ organizationIds: [CH_ORG_ID],+ });+ expect(headersOf(init)["CH-Organization-Id"]).toBeUndefined();+ });+it("always sends a checkout idempotency key, generating one when the caller has none", async () => {const session = {checkoutUrl: "https://pay.example.com/c/1",@@ -430,6 +444,33 @@ describe("chbApiClient", () => {expect(attachedPlan.scheduled).toBeUndefined();});+ it("parses a bill without usage instead of failing the whole batch", async () => {+ onChb(+ jsonResponse(200, [+ {+ id: "bill_1",+ organizationId: CH_ORG_ID,+ startDate: "2026-09-01T00:00:00.000Z",+ endDate: "2026-10-01T00:00:00.000Z",+ usage: {+ amount: 1250.68,+ currency: "USD",+ asOf: "2026-09-29T12:00:00.000Z",+ },+ },+ { id: "bill_2", organizationId: "other-org", extra: true },+ ]),+ );++ const bills = await client().queryBills({+ chOrganizationIds: [CH_ORG_ID, "other-org"],+ });++ expect(bills.map((bill) => bill.id)).toEqual(["bill_1", "bill_2"]);+ expect(bills[0]!.usage?.amount).toBe(1250.68);+ expect(bills[1]!.usage).toBeUndefined();+ });+it("reads the portal URL from CHB's portalUrl field", async () => {onChb(jsonResponse(200, { portalUrl: "https://pay.example.com/s/1" }));@@ -517,59 +558,69 @@ describe("chbApiClient", () => {});});- describe("getChbApiClient", () => {- const setAll = () => {- mocks.env.CLICKHOUSE_BILLING_BASE_URL = "https://chb.example.com";- mocks.env.CLICKHOUSE_BILLING_AUTH0_DOMAIN = AUTH0_DOMAIN;- mocks.env.CLICKHOUSE_BILLING_AUTH0_CLIENT_ID = "client-id";- mocks.env.CLICKHOUSE_BILLING_AUTH0_CLIENT_SECRET = "client-secret";- };-- beforeEach(() => resetChbApiClientForTests());- afterEach(() => resetChbApiClientForTests());+ /**+ * CHB reports this amount in USD and refuses any other currency, as a+ * major-unit figure β unlike `ChbInvoice.amount`, which is minor units β so+ * a caller comparing it against a USD spend threshold uses it as-is.+ */+ describe("chbBillUsageAmountUSD", () => {+ const withUsage = (usage: unknown) =>+ ({+ id: "bill_1",+ organizationId: CH_ORG_ID,+ startDate: "2026-09-01T00:00:00.000Z",+ usage,+ }) as Parameters<typeof chbBillUsageAmountUSD>[0];- /**- * The dispatch layer resolves a billing service per tRPC request. If that- * handed back a fresh client each time, every billing call would mint a new- * Auth0 token β the client's token cache and single-flight only do anything- * when one instance is shared.- */- it("hands every caller the same client", () => {- setAll();- const first = getChbApiClient();- expect(first).toBeInstanceOf(ChbApiClient);- expect(getChbApiClient()).toBe(first);- expect(getChbApiClient()).toBe(first);+ it("returns the amount unchanged as USD", () => {+ expect(+ chbBillUsageAmountUSD(withUsage({ amount: 1234.56, currency: "USD" })),+ ).toBe(1234.56);});- it("mints one token across calls made through the shared client", async () => {- setAll();- onChb(jsonResponse(200, { portalUrl: "https://chb.example.com/portal" }));- onChb(jsonResponse(200, { portalUrl: "https://chb.example.com/portal" }));+ it("accepts a lowercase currency", () => {+ expect(+ chbBillUsageAmountUSD(withUsage({ amount: 10, currency: "usd" })),+ ).toBe(10);+ });- // Two separate resolutions, as two billing procedures in one page load.- await getChbApiClient()!.createPortalSession({- chOrganizationId: CH_ORG_ID,- returnUrl: "https://cloud.langfuse.com/return",- });- await getChbApiClient()!.createPortalSession({- chOrganizationId: CH_ORG_ID,- returnUrl: "https://cloud.langfuse.com/return",- });+ it("treats an absent currency as USD, which is all CHB emits", () => {+ expect(chbBillUsageAmountUSD(withUsage({ amount: 10 }))).toBe(10);+ });- expect(chbCalls()).toHaveLength(2);- expect(tokenCalls()).toHaveLength(1);+ /** Nothing spent yet is a reading; no reading is not. */+ it("distinguishes zero spend from a missing amount", () => {+ expect(+ chbBillUsageAmountUSD(withUsage({ amount: 0, currency: "USD" })),+ ).toBe(0);+ expect(+ chbBillUsageAmountUSD(withUsage({ amount: null, currency: "USD" })),+ ).toBeNull();+ expect(chbBillUsageAmountUSD(withUsage(null))).toBeNull();+ expect(+ chbBillUsageAmountUSD({ id: "bill_1", organizationId: CH_ORG_ID }),+ ).toBeNull();});- it("caches the unconfigured verdict instead of re-reading env", () => {- setAll();- mocks.env.CLICKHOUSE_BILLING_BASE_URL = undefined;- expect(getChbApiClient()).toBeNull();+ /**+ * A non-USD currency means the contract moved; the number can no longer be+ * held against a USD threshold, so it must not be guessed at.+ */+ it("refuses a currency it cannot compare against a USD threshold", () => {+ expect(+ chbBillUsageAmountUSD(withUsage({ amount: 1000, currency: "EUR" })),+ ).toBeNull();+ });- // Env cannot change under a running process; a later call must not- // suddenly start talking to CHB.- setAll();- expect(getChbApiClient()).toBeNull();+ it("refuses a non-finite amount", () => {+ expect(+ chbBillUsageAmountUSD(+ withUsage({ amount: Number.POSITIVE_INFINITY, currency: "USD" }),+ ),+ ).toBeNull();+ expect(+ chbBillUsageAmountUSD(withUsage({ amount: NaN, currency: "USD" })),+ ).toBeNull();});});});diff --git a/packages/shared/src/server/ee/clickhouseBilling/chbApiClient.ts b/packages/shared/src/server/ee/clickhouseBilling/chbApiClient.tsnew file mode 100644index 0000000000..8c86c1de27--- /dev/null+++ b/packages/shared/src/server/ee/clickhouseBilling/chbApiClient.ts@@ -0,0 +1,522 @@+import { randomUUID } from "crypto";++import { SpanKind, type Span } from "@opentelemetry/api";+import { z } from "zod";++import { instrumentAsync } from "../../instrumentation";+import { logger } from "../../logger";++import {+ ChbAccessTokenProvider,+ type ChbAccessTokenConfig,+} from "./chbAccessToken";++/**+ * Thin fetch wrapper for the ClickHouse Billing (CHB) REST API.+ */++export class ChbApiError extends Error {+ constructor(+ message: string,+ public readonly status: number,+ public readonly body?: unknown,+ ) {+ super(message);+ this.name = "ChbApiError";+ }+}++/**+ * CHB returns 409 Conflict on attached-plan mutations when the organization has no+ * active payment method. Callers translate this into the same "needs checkout"+ * UX path the billing dialog already handles.+ */+export class ChbPaymentRequiredError extends ChbApiError {+ constructor(body?: unknown) {+ super(+ "CHB rejected the mutation: no active payment method on the organization",+ 409,+ body,+ );+ this.name = "ChbPaymentRequiredError";+ }+}++// A pending change on the attached plan, as GET /attachedplan reports it.+// Kept permissive (no discriminated union) because this is a read path that+// renders the billing page: an unknown type must degrade, not throw.+const ChbAttachedPlanScheduledSchema = z.object({+ type: z.string(), // "upgrade" | "downgrade" | "cancel"+ planCode: z.string().nullish(), // upgrade / downgrade target+ startDate: z.string().nullish(), // upgrade / downgrade effective date+ endDate: z.string().nullish(), // cancel: when the plan ends+});++const ChbAttachedPlanSchema = z.object({+ id: z.string(),+ plan: z+ .object({+ code: z.string().nullish(),+ })+ .nullish(),+ period: z+ .object({+ startDate: z.string().nullish(),+ // CHB's period end is exclusive (`endDateExclusive` on its side).+ endDate: z.string().nullish(),+ })+ .nullish(),+ payment: z+ .object({+ status: z.string().nullish(), // "active" | "past-due" | "failed"+ provider: z+ .object({+ name: z.string().nullish(),+ customerId: z.string().nullish(),+ })+ .nullish(),+ })+ .nullish(),+ scheduled: ChbAttachedPlanScheduledSchema.nullish(),+});+export type ChbAttachedPlan = z.infer<typeof ChbAttachedPlanSchema>;++const ChbCheckoutSessionSchema = z.object({+ checkoutUrl: z.url(),+ // ClickHouse Organization ID β persisted on the Langfuse org right away so a+ // checkout retry recovers the same CH org instead of orphaning one.+ organizationId: z.uuid(),+});+export type ChbCheckoutSession = z.infer<typeof ChbCheckoutSessionSchema>;++// An organization's open bill, as POST /bills/query reports it. Permissive for+// the same reason as the attached plan: a field CHB leaves out must surface as+// "no reading" to the caller, not fail the whole batch it arrived in.+const ChbBillSchema = z.object({+ id: z.string(),+ organizationId: z.string(),+ startDate: z.string().nullish(),+ // Exclusive, like the attached plan's period end.+ endDate: z.string().nullish(),+ // Accrued metered usage of the open bill: CHB counts only its USAGE line+ // items, so the plan's fixed fee (STANDING_CHARGE) and prorated upgrade+ // deltas (DEBIT_MEMO) are not in it. `amount` is a **major-unit** USD figure+ // (1250.68 means $1,250.68) -- unlike `ChbInvoice.amount`, which is minor+ // units. Read it through `chbBillUsageAmountUSD`, which is the one place+ // that encodes this. `asOf` is when CHB last exported the bill.+ usage: z+ .object({+ amount: z.number().nullish(),+ currency: z.string().nullish(),+ asOf: z.string().nullish(),+ })+ .nullish(),+});+export type ChbBill = z.infer<typeof ChbBillSchema>;++const ChbBillListSchema = z.array(ChbBillSchema);++const ChbInvoiceSchema = z.object({+ id: z.string().nullish(),+ number: z.string().nullish(),+ status: z.string().nullish(), // "draft" | "open" | "paid" | "void" | "uncollectible"+ currency: z.string().nullish(),+ createdAt: z.string().nullish(),+ // Minor units (cents for USD)+ amount: z.number().nullish(),+ hostedUrl: z.string().nullish(),+ pdfUrl: z.string().nullish(),+});+export type ChbInvoice = z.infer<typeof ChbInvoiceSchema>;++const ChbInvoiceListSchema = z.object({+ invoices: z.array(ChbInvoiceSchema).default([]),+});++const ChbPortalSessionSchema = z.object({+ portalUrl: z.string(),+});++const REQUEST_TIMEOUT_MS = 15_000;++type ChbRequestOptions = {+ /**+ * APM span name for the CHB operation, e.g. `chb.checkout_session.create`.+ * Named after the operation rather than the route so a path carrying an id+ * does not fan one operation out into unrelated APM resources.+ */+ operation: string;+ method: "GET" | "POST" | "PUT" | "DELETE";+ path: string;+ /** CH-Organization-Id header β required for org-scoped endpoints */+ chOrganizationId?: string;+ body?: unknown;+ idempotencyKey?: string;+ searchParams?: Record<string, string>;+};++export class ChbApiClient {+ private readonly tokenProvider: ChbAccessTokenProvider;++ constructor(+ private readonly config: {+ baseUrl: string;+ auth: ChbAccessTokenConfig;+ },+ ) {+ this.tokenProvider = new ChbAccessTokenProvider(config.auth);+ }++ private requestUrl(opts: ChbRequestOptions): URL {+ // Relative join against a trailing slash, so a base url carrying a path+ // prefix keeps it instead of having it replaced.+ const url = new URL(+ opts.path.replace(/^\//, ""),+ `${this.config.baseUrl.replace(/\/$/, "")}/`,+ );+ for (const [key, value] of Object.entries(opts.searchParams ?? {})) {+ url.searchParams.set(key, value);+ }+ return url;+ }++ /**+ * One span per HTTP attempt, so a token replay shows as two attempt spans β+ * each with the token mint it triggered nested underneath β rather than as+ * one opaque slow call.+ */+ private async send(+ url: URL,+ opts: ChbRequestOptions,+ attempt: number,+ ): Promise<Response> {+ return await instrumentAsync(+ { name: "chb.api.request", spanKind: SpanKind.CLIENT },+ async (span) => {+ span.setAttributes({+ "http.method": opts.method,+ "http.route": opts.path,+ "peer.hostname": url.hostname,+ "chb.operation": opts.operation,+ "chb.attempt": attempt,+ });++ const response = await fetch(url, {+ method: opts.method,+ headers: {+ authorization: `Bearer ${await this.tokenProvider.getToken()}`,+ ...(opts.chOrganizationId+ ? { "CH-Organization-Id": opts.chOrganizationId }+ : {}),+ ...(opts.body !== undefined+ ? { "content-type": "application/json" }+ : {}),+ ...(opts.idempotencyKey+ ? { "Idempotency-Key": opts.idempotencyKey }+ : {}),+ },+ body: opts.body !== undefined ? JSON.stringify(opts.body) : undefined,+ // The base URL is operator-configured; an unexpected redirect is an+ // error, not something to follow with a bearer token attached.+ redirect: "error",+ signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),+ });++ span.setAttribute("http.status_code", response.status);+ return response;+ },+ );+ }++ private async request(opts: ChbRequestOptions): Promise<unknown> {+ const outcome = await instrumentAsync(+ { name: opts.operation, spanKind: SpanKind.CLIENT },+ async (span) => {+ span.setAttributes({+ "http.method": opts.method,+ "http.route": opts.path,+ ...(opts.chOrganizationId+ ? { "chb.ch_organization_id": opts.chOrganizationId }+ : {}),+ ...(opts.idempotencyKey+ ? { "chb.idempotency_key": opts.idempotencyKey }+ : {}),+ });++ try {+ return {+ thrown: undefined,+ body: await this.sendWithReplay(opts, span),+ };+ } catch (error) {+ // A 409 is the "organization needs a payment method" UX branch, not+ // an incident β which is why it is deliberately not logged as an+ // error either. Carrying it out of the span instead of throwing+ // through it keeps instrumentAsync from marking the operation as an+ // APM error, so a routine checkout prompt cannot inflate the error+ // rate this instrumentation exists to make readable.+ if (error instanceof ChbPaymentRequiredError) {+ span.setAttribute("chb.payment_required", true);+ return { thrown: error, body: undefined };+ }+ throw error;+ }+ },+ );++ // Rethrown outside the span so the caller contract is unchanged.+ if (outcome.thrown) throw outcome.thrown;+ return outcome.body;+ }++ private async sendWithReplay(+ opts: ChbRequestOptions,+ span: Span,+ ): Promise<unknown> {+ const url = this.requestUrl(opts);++ let response = await this.send(url, opts, 1);+ if (response.status === 401) {+ // CHB rejected a token we still considered fresh β revoked, or the Auth0+ // signing key rotated. A 401 means CHB did no work, and the retry carries+ // the same idempotency key, so replaying once with a new token is safe.+ logger.warn("[CHB API] Token rejected, retrying with a fresh one", {+ method: opts.method,+ path: opts.path,+ });+ span.setAttribute("chb.token_replayed", true);+ this.tokenProvider.invalidate();+ response = await this.send(url, opts, 2);+ }++ span.setAttribute("http.status_code", response.status);++ const responseBody = await response+ .json()+ .catch(() => undefined as unknown);++ if (response.status === 409) {+ throw new ChbPaymentRequiredError(responseBody);+ }+ if (!response.ok) {+ logger.error("[CHB API] Request failed", {+ method: opts.method,+ path: opts.path,+ status: response.status,+ body: responseBody,+ });+ throw new ChbApiError(+ `CHB API ${opts.method} ${opts.path} failed with status ${response.status}`,+ response.status,+ responseBody,+ );+ }++ return responseBody;+ }++ async createCheckoutSession(params: {+ organizationId?: string;+ /** Names the CH organization when checkout creates it. */+ name?: string;+ email: string;+ planCode: string;+ returnUrl: string;+ idempotencyKey?: string;+ }): Promise<ChbCheckoutSession> {+ // CHB rejects a blank name (400) but defaults one when the field is+ // absent, so a whitespace-only name is dropped rather than sent.+ const name = params.name?.trim();+ const body = await this.request({+ operation: "chb.checkout_session.create",+ method: "POST",+ path: "checkout-sessions",+ body: {+ ...(params.organizationId+ ? { organizationId: params.organizationId }+ : {}),+ ...(name ? { name } : {}),+ email: params.email,+ planCode: params.planCode,+ returnUrl: params.returnUrl,+ // Required by CHB. Without an opId there is nothing to dedupe against,+ // so a fresh key makes the call unique rather than rejected.+ idempotencyKey: params.idempotencyKey ?? randomUUID(),+ },+ idempotencyKey: params.idempotencyKey,+ });+ return ChbCheckoutSessionSchema.parse(body);+ }++ /**+ * The organization's current attached plan. CHB scopes the attached-plan+ * routes by the CH-Organization-Id header, there is no id in the path; a 404+ * means the organization has none.+ */+ async getAttachedPlan(params: {+ chOrganizationId: string;+ }): Promise<ChbAttachedPlan> {+ const body = await this.request({+ operation: "chb.attachedplan.get",+ method: "GET",+ path: "attachedplan",+ chOrganizationId: params.chOrganizationId,+ searchParams: { fields: "plan,period,payment,scheduled" },+ });+ return ChbAttachedPlanSchema.parse(body);+ }++ /** Upgrade now, schedule a downgrade for the cycle end, or cancel the plan. */+ async setScheduledChange(params: {+ chOrganizationId: string;+ change: {+ type: "upgrade" | "downgrade" | "cancel";+ when: "immediate" | "billing_cycle_end";+ planCode?: string;+ };+ idempotencyKey?: string;+ }): Promise<void> {+ await this.request({+ operation: "chb.attachedplan.scheduled.set",+ method: "PUT",+ path: "attachedplan/scheduled",+ chOrganizationId: params.chOrganizationId,+ body: params.change,+ idempotencyKey: params.idempotencyKey,+ });+ }++ /** Clear a pending scheduled change β reactivate / undo plan switch. */+ async clearScheduledChange(params: {+ chOrganizationId: string;+ idempotencyKey?: string;+ }): Promise<void> {+ await this.request({+ operation: "chb.attachedplan.scheduled.clear",+ method: "DELETE",+ path: "attachedplan/scheduled",+ chOrganizationId: params.chOrganizationId,+ idempotencyKey: params.idempotencyKey,+ });+ }++ /**+ * The open bill of each organization, with its accrued usage. CHB serves this+ * from its billing warehouse rather than from m3ter, so it can be polled+ * without spending m3ter's rate limit, at the cost of lagging by up to one+ * warehouse export (`usage.asOf`). Not scoped by header: the organizations+ * are the request body, 1 to 100 of them. An organization without an open+ * bill is absent from the result rather than an error.+ */+ async queryBills(params: {+ chOrganizationIds: string[];+ }): Promise<ChbBill[]> {+ const body = await this.request({+ operation: "chb.bills.query",+ // POST because CHB's API toolkit has no QUERY method yet.+ method: "POST",+ path: "bills/query",+ body: { organizationIds: params.chOrganizationIds },+ searchParams: { fields: "usage" },+ });+ return ChbBillListSchema.parse(body);+ }++ /**+ * Issued invoices, most recent first. Unlike the attached-plan routes this+ * one is scoped by an `organizationId` query parameter.+ */+ async listInvoices(params: {+ chOrganizationId: string;+ }): Promise<ChbInvoice[]> {+ const body = await this.request({+ operation: "chb.invoices.list",+ method: "GET",+ path: "invoices",+ chOrganizationId: params.chOrganizationId,+ searchParams: { organizationId: params.chOrganizationId },+ });+ return ChbInvoiceListSchema.parse(body).invoices;+ }++ async createPortalSession(params: {+ chOrganizationId: string;+ returnUrl: string;+ }): Promise<string> {+ const body = await this.request({+ operation: "chb.portal_session.create",+ method: "POST",+ path: "portal-sessions",+ chOrganizationId: params.chOrganizationId,+ body: { returnUrl: params.returnUrl },+ });+ return ChbPortalSessionSchema.parse(body).portalUrl;+ }+}++/** The currency CHB reports accrued usage in; it rejects every other one. */+export const CHB_USAGE_CURRENCY = "USD";++/**+ * The open bill's accrued usage in USD, or null when CHB did not report a+ * usable figure. Returning null rather than 0 keeps "no reading" distinct from+ * "nothing spent yet" -- a caller comparing against a spend threshold must skip+ * the former and evaluate the latter.+ *+ * A currency other than USD is treated as no reading: CHB asserts USD when it+ * computes the amount, so anything else means the contract moved and the number+ * can no longer be compared against a USD threshold.+ */+export const chbBillUsageAmountUSD = (bill: ChbBill): number | null => {+ const usage = bill.usage;+ if (usage?.amount == null || !Number.isFinite(usage.amount)) return null;+ if (+ usage.currency != null &&+ usage.currency.toUpperCase() !== CHB_USAGE_CURRENCY+ ) {+ return null;+ }+ return usage.amount;+};++/**+ * The CHB REST configuration an app reads from its own environment. Optional+ * everywhere except the audience, which is defaulted in both env schemas.+ */+export type ChbApiClientEnv = {+ CLICKHOUSE_BILLING_BASE_URL?: string;+ CLICKHOUSE_BILLING_AUTH0_DOMAIN?: string;+ CLICKHOUSE_BILLING_AUTH0_CLIENT_ID?: string;+ CLICKHOUSE_BILLING_AUTH0_CLIENT_SECRET?: string;+ CLICKHOUSE_BILLING_AUTH0_AUDIENCE: string;+};++/**+ * Build a client from an app's env, or null when the CHB REST surface is not+ * fully configured. Callers treat null as "CHB unavailable" and fail closed β a+ * partially configured deployment must not reach CHB unauthenticated.+ *+ * The env is passed in rather than read here: web and worker each validate+ * their own environment, and this module is the transport both of them share.+ */+export const buildChbApiClient = (+ env: ChbApiClientEnv,+): ChbApiClient | null => {+ if (+ !env.CLICKHOUSE_BILLING_BASE_URL ||+ !env.CLICKHOUSE_BILLING_AUTH0_DOMAIN ||+ !env.CLICKHOUSE_BILLING_AUTH0_CLIENT_ID ||+ !env.CLICKHOUSE_BILLING_AUTH0_CLIENT_SECRET+ ) {+ return null;+ }+ return new ChbApiClient({+ baseUrl: env.CLICKHOUSE_BILLING_BASE_URL,+ auth: {+ auth0Domain: env.CLICKHOUSE_BILLING_AUTH0_DOMAIN,+ clientId: env.CLICKHOUSE_BILLING_AUTH0_CLIENT_ID,+ clientSecret: env.CLICKHOUSE_BILLING_AUTH0_CLIENT_SECRET,+ audience: env.CLICKHOUSE_BILLING_AUTH0_AUDIENCE,+ },+ });+};diff --git a/packages/shared/src/server/ee/clickhouseBilling/index.ts b/packages/shared/src/server/ee/clickhouseBilling/index.tsnew file mode 100644index 0000000000..10adaa8762--- /dev/null+++ b/packages/shared/src/server/ee/clickhouseBilling/index.ts@@ -0,0 +1,2 @@+export * from "./chbAccessToken";+export * from "./chbApiClient";diff --git a/packages/shared/src/server/queues.ts b/packages/shared/src/server/queues.tsindex 236decbb0c..1533abb8ad 100644--- a/packages/shared/src/server/queues.ts+++ b/packages/shared/src/server/queues.ts@@ -450,6 +450,7 @@ export enum QueueJobs {BatchExportJob = "batch-export-job",CloudUsageMeteringJob = "cloud-usage-metering-job",CloudSpendAlertJob = "cloud-spend-alert-job",+ CloudSpendAlertFanOutJob = "cloud-spend-alert-fan-out-job",CloudFreeTierUsageThresholdJob = "cloud-free-tier-usage-threshold-job",OtelIngestionJob = "otel-ingestion-job",IngestionJob = "ingestion-job",@@ -703,12 +704,21 @@ export type TQueueJobTypes = {payload: EntityChangeEventType;name: QueueJobs.EntityChangeJob;};- [QueueName.CloudSpendAlertQueue]: {- timestamp: Date;- id: string;- payload: CloudSpendAlertJobType;- name: QueueJobs.CloudSpendAlertJob;- };+ // Per-org evaluation, plus the recurring fan-out that produces those jobs+ // for ClickHouse-billed orgs (Stripe-billed ones are enqueued by the usage+ // metering job, which only iterates Stripe customers).+ [QueueName.CloudSpendAlertQueue]:+ | {+ timestamp: Date;+ id: string;+ payload: CloudSpendAlertJobType;+ name: QueueJobs.CloudSpendAlertJob;+ }+ | {+ timestamp: Date;+ id: string;+ name: QueueJobs.CloudSpendAlertFanOutJob;+ };[QueueName.CloudFreeTierUsageThresholdQueue]: {timestamp: Date;id: string;diff --git a/packages/shared/src/server/redis/cloudSpendAlertQueue.ts b/packages/shared/src/server/redis/cloudSpendAlertQueue.tsindex 3a61606caf..8e0b15a484 100644--- a/packages/shared/src/server/redis/cloudSpendAlertQueue.ts+++ b/packages/shared/src/server/redis/cloudSpendAlertQueue.ts@@ -1,14 +1,17 @@import { Queue } from "bullmq";import { env } from "../../env";-import { QueueName } from "../queues";+import { QueueName, QueueJobs } from "../queues";import { createBullMQQueueOptionsWithRedis } from "./redis";+import { scheduleRecurringJob } from "./scheduleRecurringJob";import { logger } from "../logger";export class CloudSpendAlertQueue {private static instance: Queue | null = null;public static getInstance(): Queue | null {- if (!env.STRIPE_SECRET_KEY) {+ // Cloud-only, but no longer Stripe-only: ClickHouse-billed orgs run their+ // alerts through the same queue, and a CHB deployment has no Stripe key.+ if (!env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION) {return null;}@@ -38,8 +41,18 @@ export class CloudSpendAlertQueue {logger.error("CloudSpendAlertQueue error", err);});- // Note: Jobs are triggered by the metering job with 5-minute delays- // No automatic scheduling needed+ if (CloudSpendAlertQueue.instance) {+ // Stripe-billed orgs are enqueued by the usage metering job (with a+ // 5-minute delay, once it has pushed the hour's meter events). That job+ // only iterates orgs carrying a Stripe customer id, so CHB-billed orgs+ // need their own trigger: this hourly fan-out. It runs at :15 β after+ // metering at :05 and the alert jobs it delays to ~:10, before the+ // free-tier threshold job at :35.+ scheduleRecurringJob(CloudSpendAlertQueue.instance, {+ jobName: QueueJobs.CloudSpendAlertFanOutJob,+ pattern: "15 * * * *",+ });+ }return CloudSpendAlertQueue.instance;}diff --git a/web/src/__tests__/server/createDefaultSpendAlerts.servertest.ts b/web/src/__tests__/server/createDefaultSpendAlerts.servertest.tsindex 3267a10336..b357c926d9 100644--- a/web/src/__tests__/server/createDefaultSpendAlerts.servertest.ts+++ b/web/src/__tests__/server/createDefaultSpendAlerts.servertest.ts@@ -1,7 +1,11 @@import { prisma } from "@langfuse/shared/src/db";import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";-import { createDefaultSpendAlerts } from "@/src/ee/features/billing/server/stripe/stripeWebhookHandler";+import {+ createDefaultSpendAlerts,+ createDefaultSpendAlertsForStripeProduct,+} from "@/src/ee/features/billing/server/defaultSpendAlerts";import { stripeProducts } from "@/src/ee/features/billing/utils/stripeCatalogue";+import { chbPlanCodeToPlan } from "@langfuse/shared";describe("createDefaultSpendAlerts", () => {it("creates alerts with correct thresholds for core plan", async () => {@@ -10,7 +14,7 @@ describe("createDefaultSpendAlerts", () => {(p) => p.mappedPlan === "cloud:core",)!;- await createDefaultSpendAlerts({+ await createDefaultSpendAlertsForStripeProduct({orgId,productId: coreProduct.stripeProductId,});@@ -33,7 +37,7 @@ describe("createDefaultSpendAlerts", () => {(p) => p.mappedPlan === "cloud:pro",)!;- await createDefaultSpendAlerts({+ await createDefaultSpendAlertsForStripeProduct({orgId,productId: proProduct.stripeProductId,});@@ -54,7 +58,7 @@ describe("createDefaultSpendAlerts", () => {(p) => p.mappedPlan === "cloud:enterprise",)!;- await createDefaultSpendAlerts({+ await createDefaultSpendAlertsForStripeProduct({orgId,productId: enterpriseProduct.stripeProductId,});@@ -85,7 +89,7 @@ describe("createDefaultSpendAlerts", () => {(p) => p.mappedPlan === "cloud:core",)!;- await createDefaultSpendAlerts({+ await createDefaultSpendAlertsForStripeProduct({orgId,productId: coreProduct.stripeProductId,});@@ -103,7 +107,7 @@ describe("createDefaultSpendAlerts", () => {const { orgId } = await createOrgProjectAndApiKey();// Should not throw- await createDefaultSpendAlerts({+ await createDefaultSpendAlertsForStripeProduct({orgId,productId: "prod_unknown_id",});@@ -114,4 +118,72 @@ describe("createDefaultSpendAlerts", () => {expect(alerts).toHaveLength(0);});++ /**+ * The CHB path seeds through the same function with a plan resolved from the+ * attached plan's code, so a ClickHouse-billed org must end up with exactly+ * the thresholds a Stripe-billed one on the same plan gets.+ */+ describe("ClickHouse-billed organizations", () => {+ it.each([+ ["LANGFUSE_CORE", 200],+ ["LANGFUSE_PRO", 1000],+ ["LANGFUSE_PRO_TEAMS", 1000],+ ["LANGFUSE_ENTERPRISE", 2000],+ ] as const)(+ "creates the plan threshold plus the universal one for %s",+ async (planCode, expectedThreshold) => {+ const { orgId } = await createOrgProjectAndApiKey();++ await createDefaultSpendAlerts({+ orgId,+ plan: chbPlanCodeToPlan[planCode],+ source: "clickhouse",+ });++ const alerts = await prisma.cloudSpendAlert.findMany({+ where: { orgId },+ orderBy: { threshold: "asc" },+ });++ expect(alerts.map((alert) => alert.threshold.toNumber())).toEqual(+ [...new Set([expectedThreshold, 4000])].sort((a, b) => a - b),+ );+ },+ );++ it("does not seed alerts for a plan without spend alerts", async () => {+ const { orgId } = await createOrgProjectAndApiKey();++ // Should not throw+ await createDefaultSpendAlerts({+ orgId,+ plan: "cloud:hobby",+ source: "clickhouse",+ });++ expect(+ await prisma.cloudSpendAlert.findMany({ where: { orgId } }),+ ).toHaveLength(0);+ });++ it("attributes the seeded alerts to the ClickHouse webhook in the audit log", async () => {+ const { orgId } = await createOrgProjectAndApiKey();++ await createDefaultSpendAlerts({+ orgId,+ plan: "cloud:core",+ source: "clickhouse",+ });++ const auditLogs = await prisma.auditLog.findMany({+ where: { orgId, resourceType: "cloudSpendAlert" },+ });++ expect(auditLogs).toHaveLength(2);+ expect(+ auditLogs.every((log) => log.userId === "clickhouse-webhook"),+ ).toBe(true);+ });+ });});diff --git a/web/src/__tests__/server/unit/chbApiClientSingleton.servertest.ts b/web/src/__tests__/server/unit/chbApiClientSingleton.servertest.tsnew file mode 100644index 0000000000..5cfd445d2c--- /dev/null+++ b/web/src/__tests__/server/unit/chbApiClientSingleton.servertest.ts@@ -0,0 +1,122 @@+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";++/**+ * Web's per-process CHB client handle. The transport itself is tested in+ * `packages/shared/src/server/ee/clickhouseBilling/chbApiClient.test.ts`; what is+ * web-specific β and what these cases pin β is that one client is shared across+ * billing requests so its token cache does any work at all.+ */++const mocks = vi.hoisted(() => ({+ env: {+ CLICKHOUSE_BILLING_BASE_URL: undefined as string | undefined,+ CLICKHOUSE_BILLING_AUTH0_DOMAIN: undefined as string | undefined,+ CLICKHOUSE_BILLING_AUTH0_CLIENT_ID: undefined as string | undefined,+ CLICKHOUSE_BILLING_AUTH0_CLIENT_SECRET: undefined as string | undefined,+ CLICKHOUSE_BILLING_AUTH0_AUDIENCE: "billing-api",+ },+}));++vi.mock("@/src/env.mjs", () => ({ env: mocks.env }));++import {+ ChbApiClient,+ getChbApiClient,+ resetChbApiClientForTests,+} from "@/src/ee/features/billing/server/chb/chbApiClient";++const CH_ORG_ID = "6dd6ab1d-9e8d-4c1a-8b4f-9a3d1e2c4b5a";+const AUTH0_DOMAIN = "chb-tenant.eu.auth0.com";+const TOKEN_URL = `https://${AUTH0_DOMAIN}/oauth/token`;++const jsonResponse = (status: number, body: unknown) => ({+ status,+ ok: status >= 200 && status < 300,+ json: async () => body,+});++let tokenCounter = 0;++const fetchMock = vi.fn(async (url: URL | string) => {+ if (String(url) === TOKEN_URL) {+ tokenCounter += 1;+ return jsonResponse(200, {+ access_token: `chb-access-token-${tokenCounter}`,+ expires_in: 86_400,+ });+ }+ return jsonResponse(200, { portalUrl: "https://chb.example.com/portal" });+});++const tokenCalls = () =>+ fetchMock.mock.calls.filter((call) => String(call[0]) === TOKEN_URL);++const chbCalls = () =>+ fetchMock.mock.calls.filter((call) => String(call[0]) !== TOKEN_URL);++const setAll = () => {+ mocks.env.CLICKHOUSE_BILLING_BASE_URL = "https://chb.example.com";+ mocks.env.CLICKHOUSE_BILLING_AUTH0_DOMAIN = AUTH0_DOMAIN;+ mocks.env.CLICKHOUSE_BILLING_AUTH0_CLIENT_ID = "client-id";+ mocks.env.CLICKHOUSE_BILLING_AUTH0_CLIENT_SECRET = "client-secret";+};++describe("getChbApiClient", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ tokenCounter = 0;+ mocks.env.CLICKHOUSE_BILLING_BASE_URL = undefined;+ mocks.env.CLICKHOUSE_BILLING_AUTH0_DOMAIN = undefined;+ mocks.env.CLICKHOUSE_BILLING_AUTH0_CLIENT_ID = undefined;+ mocks.env.CLICKHOUSE_BILLING_AUTH0_CLIENT_SECRET = undefined;+ vi.stubGlobal("fetch", fetchMock);+ resetChbApiClientForTests();+ });++ afterEach(() => {+ resetChbApiClientForTests();+ vi.unstubAllGlobals();+ });++ /**+ * The dispatch layer resolves a billing service per tRPC request. If that+ * handed back a fresh client each time, every billing call would mint a new+ * Auth0 token β the client's token cache and single-flight only do anything+ * when one instance is shared.+ */+ it("hands every caller the same client", () => {+ setAll();+ const first = getChbApiClient();+ expect(first).toBeInstanceOf(ChbApiClient);+ expect(getChbApiClient()).toBe(first);+ expect(getChbApiClient()).toBe(first);+ });++ it("mints one token across calls made through the shared client", async () => {+ setAll();++ // Two separate resolutions, as two billing procedures in one page load.+ await getChbApiClient()!.createPortalSession({+ chOrganizationId: CH_ORG_ID,+ returnUrl: "https://cloud.langfuse.com/return",+ });+ await getChbApiClient()!.createPortalSession({+ chOrganizationId: CH_ORG_ID,+ returnUrl: "https://cloud.langfuse.com/return",+ });++ expect(chbCalls()).toHaveLength(2);+ expect(tokenCalls()).toHaveLength(1);+ });++ it("caches the unconfigured verdict instead of re-reading env", () => {+ setAll();+ mocks.env.CLICKHOUSE_BILLING_BASE_URL = undefined;+ expect(getChbApiClient()).toBeNull();++ // Env cannot change under a running process; a later call must not+ // suddenly start talking to CHB.+ setAll();+ expect(getChbApiClient()).toBeNull();+ });+});diff --git a/web/src/__tests__/server/unit/chbWebhookHandler.servertest.ts b/web/src/__tests__/server/unit/chbWebhookHandler.servertest.tsindex ed35fd15a3..81acbd111c 100644--- a/web/src/__tests__/server/unit/chbWebhookHandler.servertest.ts+++ b/web/src/__tests__/server/unit/chbWebhookHandler.servertest.ts@@ -20,6 +20,8 @@ const mocks = vi.hoisted(() => ({traceException: vi.fn(),recordIncrement: vi.fn(),auditLog: vi.fn(),+ findSpendAlert: vi.fn(),+ createSpendAlert: vi.fn(),sendChbProjectEvent: vi.fn(),getChbApiClient: vi.fn(),getAttachedPlan: vi.fn(),@@ -31,6 +33,10 @@ vi.mock("@langfuse/shared/src/db", () => ({prisma: {organization: { findFirst: mocks.findOrg, update: mocks.updateOrg },project: { findMany: mocks.findProjects },+ cloudSpendAlert: {+ findFirst: mocks.findSpendAlert,+ create: mocks.createSpendAlert,+ },},}));@@ -266,6 +272,13 @@ describe("chbWebhookHandler", () => {// test rejects has to be pinned back here or it leaks into the next test.mocks.invalidateCachedOrgApiKeys.mockResolvedValue(undefined);mocks.auditLog.mockResolvedValue(undefined);+ mocks.findSpendAlert.mockResolvedValue(null);+ mocks.createSpendAlert.mockImplementation(+ async ({ data }: { data: Record<string, unknown> }) => ({+ id: `alert-${data.threshold}`,+ ...data,+ }),+ );mocks.getChbApiClient.mockReturnValue({getAttachedPlan: mocks.getAttachedPlan,});@@ -486,7 +499,14 @@ describe("chbWebhookHandler", () => {// Post-commit like the cache invalidation: the state change is applied,// so the missing audit row is logged and paged rather than retried.expect(response.status).toBe(200);- expect(mocks.auditLog).toHaveBeenCalledTimes(1);+ // The organization row's audit write is the one that must have been+ // attempted; a created event also audits each seeded spend alert, so the+ // action is the assertion rather than a call count.+ expect(mocks.auditLog).toHaveBeenCalledWith(+ expect.objectContaining({+ action: "BillingService.chb.BILLING_ATTACHEDPLAN_CREATED",+ }),+ );expect(mocks.redisDel).not.toHaveBeenCalled();expect(mocks.traceException).toHaveBeenCalledWith(expect.objectContaining({ message: "connection reset" }),@@ -646,4 +666,63 @@ describe("chbWebhookHandler", () => {});expect(orgColumnsOfUpdate()).toHaveProperty("cloudBillingCycleAnchor");});++ /**+ * The Stripe path seeds default spend alerts on a first subscription. A+ * CHB-billed org has to start out with the same thresholds, or its billing+ * page offers spend alerts that nobody ever configured.+ */+ describe("default spend alerts", () => {+ const seededThresholds = () =>+ mocks.createSpendAlert.mock.calls.map(+ (call) => (call[0] as { data: { threshold: number } }).data.threshold,+ );++ it("seeds the plan and universal thresholds on BILLING_ATTACHEDPLAN_CREATED", async () => {+ const response = await chbWebhookHandler(post(attachedPlanCreated()));++ expect(response.status).toBe(200);+ // LANGFUSE_CORE -> cloud:core -> $200, plus the universal $4000.+ expect(seededThresholds()).toEqual([200, 4000]);+ });++ it("seeds from the plan CHB reports, not the code the event announced", async () => {+ mocks.getAttachedPlan.mockResolvedValue(+ attachedPlan({ plan: { code: "LANGFUSE_ENTERPRISE" } }),+ );++ await chbWebhookHandler(post(attachedPlanCreated()));++ expect(seededThresholds()).toEqual([2000, 4000]);+ });++ it("leaves an org's own alerts alone", async () => {+ mocks.findSpendAlert.mockResolvedValue({ id: "existing" });++ await chbWebhookHandler(post(attachedPlanCreated()));++ expect(mocks.createSpendAlert).not.toHaveBeenCalled();+ });++ it("does not seed on an update or a cancellation", async () => {+ await chbWebhookHandler(post(chbEvent("BILLING_ATTACHEDPLAN_UPDATED")));+ await chbWebhookHandler(post(chbEvent("BILLING_ATTACHEDPLAN_CANCELLED")));++ expect(mocks.createSpendAlert).not.toHaveBeenCalled();+ });++ /**+ * Seeding runs after the organization update has committed, so a failure+ * there must not turn into a 500 that asks CHB to replay an event this+ * handler has already applied.+ */+ it("still answers 200 when seeding fails", async () => {+ mocks.createSpendAlert.mockRejectedValue(new Error("db down"));++ const response = await chbWebhookHandler(post(attachedPlanCreated()));++ expect(response.status).toBe(200);+ expect(mocks.traceException).toHaveBeenCalled();+ });+ });});diff --git a/web/src/ee/features/billing/server/chb/chbApiClient.ts b/web/src/ee/features/billing/server/chb/chbApiClient.tsindex 5ec03f6e40..8da95e2b17 100644--- a/web/src/ee/features/billing/server/chb/chbApiClient.ts+++ b/web/src/ee/features/billing/server/chb/chbApiClient.ts@@ -1,436 +1,26 @@-import { randomUUID } from "crypto";--import { SpanKind, type Span } from "@opentelemetry/api";-import { z } from "zod";--import { env } from "@/src/env.mjs";-import { instrumentAsync, logger } from "@langfuse/shared/src/server";--import {- ChbAccessTokenProvider,- type ChbAccessTokenConfig,-} from "./chbAccessToken";-/**- * Thin fetch wrapper for the ClickHouse Billing (CHB) REST API.- */--export class ChbApiError extends Error {- constructor(- message: string,- public readonly status: number,- public readonly body?: unknown,- ) {- super(message);- this.name = "ChbApiError";- }-}--/**- * CHB returns 409 Conflict on attached-plan mutations when the organization has no- * active payment method. Callers translate this into the same "needs checkout"- * UX path the billing dialog already handles.+ * Web's env-bound handle on the ClickHouse Billing (CHB) REST client.+ *+ * The transport, its schemas, its error types and the fail-closed build rule+ * live in `@langfuse/shared/src/server/ee/clickhouseBilling` so the worker can+ * reach CHB as well β the spend-alert job reads the open bill's accrued usage+ * from there. Only+ * the env reading and the process-wide singleton are per-app, because each app+ * validates its own environment.*/-export class ChbPaymentRequiredError extends ChbApiError {- constructor(body?: unknown) {- super(- "CHB rejected the mutation: no active payment method on the organization",- 409,- body,- );- this.name = "ChbPaymentRequiredError";- }-}--// A pending change on the attached plan, as GET /attachedplan reports it.-// Kept permissive (no discriminated union) because this is a read path that-// renders the billing page: an unknown type must degrade, not throw.-const ChbAttachedPlanScheduledSchema = z.object({- type: z.string(), // "upgrade" | "downgrade" | "cancel"- planCode: z.string().nullish(), // upgrade / downgrade target- startDate: z.string().nullish(), // upgrade / downgrade effective date- endDate: z.string().nullish(), // cancel: when the plan ends-});--const ChbAttachedPlanSchema = z.object({- id: z.string(),- plan: z- .object({- code: z.string().nullish(),- })- .nullish(),- period: z- .object({- startDate: z.string().nullish(),- endDate: z.string().nullish(),- })- .nullish(),- payment: z- .object({- status: z.string().nullish(), // "active" | "past-due" | "failed"- provider: z- .object({- name: z.string().nullish(),- customerId: z.string().nullish(),- })- .nullish(),- })- .nullish(),- scheduled: ChbAttachedPlanScheduledSchema.nullish(),-});-export type ChbAttachedPlan = z.infer<typeof ChbAttachedPlanSchema>;--const ChbCheckoutSessionSchema = z.object({- checkoutUrl: z.url(),- // ClickHouse Organization ID β persisted on the Langfuse org right away so a- // checkout retry recovers the same CH org instead of orphaning one.- organizationId: z.uuid(),-});-export type ChbCheckoutSession = z.infer<typeof ChbCheckoutSessionSchema>;--const ChbInvoiceSchema = z.object({- id: z.string().nullish(),- number: z.string().nullish(),- status: z.string().nullish(), // "draft" | "open" | "paid" | "void" | "uncollectible"- currency: z.string().nullish(),- createdAt: z.string().nullish(),- // Minor units (cents for USD)- amount: z.number().nullish(),- hostedUrl: z.string().nullish(),- pdfUrl: z.string().nullish(),-});-export type ChbInvoice = z.infer<typeof ChbInvoiceSchema>;--const ChbInvoiceListSchema = z.object({- invoices: z.array(ChbInvoiceSchema).default([]),-});--const ChbPortalSessionSchema = z.object({- portalUrl: z.string(),-});--const REQUEST_TIMEOUT_MS = 15_000;--type ChbRequestOptions = {- /**- * APM span name for the CHB operation, e.g. `chb.checkout_session.create`.- * Named after the operation rather than the route so a path carrying an id- * does not fan one operation out into unrelated APM resources.- */- operation: string;- method: "GET" | "POST" | "PUT" | "DELETE";- path: string;- /** CH-Organization-Id header β required for org-scoped endpoints */- chOrganizationId?: string;- body?: unknown;- idempotencyKey?: string;- searchParams?: Record<string, string>;-};--export class ChbApiClient {- private readonly tokenProvider: ChbAccessTokenProvider;-- constructor(- private readonly config: {- baseUrl: string;- auth: ChbAccessTokenConfig;- },- ) {- this.tokenProvider = new ChbAccessTokenProvider(config.auth);- }-- private requestUrl(opts: ChbRequestOptions): URL {- // Relative join against a trailing slash, so a base url carrying a path- // prefix keeps it instead of having it replaced.- const url = new URL(- opts.path.replace(/^\//, ""),- `${this.config.baseUrl.replace(/\/$/, "")}/`,- );- for (const [key, value] of Object.entries(opts.searchParams ?? {})) {- url.searchParams.set(key, value);- }- return url;- }-- /**- * One span per HTTP attempt, so a token replay shows as two attempt spans β- * each with the token mint it triggered nested underneath β rather than as- * one opaque slow call.- */- private async send(- url: URL,- opts: ChbRequestOptions,- attempt: number,- ): Promise<Response> {- return await instrumentAsync(- { name: "chb.api.request", spanKind: SpanKind.CLIENT },- async (span) => {- span.setAttributes({- "http.method": opts.method,- "http.route": opts.path,- "peer.hostname": url.hostname,- "chb.operation": opts.operation,- "chb.attempt": attempt,- });-- const response = await fetch(url, {- method: opts.method,- headers: {- authorization: `Bearer ${await this.tokenProvider.getToken()}`,- ...(opts.chOrganizationId- ? { "CH-Organization-Id": opts.chOrganizationId }- : {}),- ...(opts.body !== undefined- ? { "content-type": "application/json" }- : {}),- ...(opts.idempotencyKey- ? { "Idempotency-Key": opts.idempotencyKey }- : {}),- },- body: opts.body !== undefined ? JSON.stringify(opts.body) : undefined,- // The base URL is operator-configured; an unexpected redirect is an- // error, not something to follow with a bearer token attached.- redirect: "error",- signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),- });-- span.setAttribute("http.status_code", response.status);- return response;- },- );- }- private async request(opts: ChbRequestOptions): Promise<unknown> {- const outcome = await instrumentAsync(- { name: opts.operation, spanKind: SpanKind.CLIENT },- async (span) => {- span.setAttributes({- "http.method": opts.method,- "http.route": opts.path,- ...(opts.chOrganizationId- ? { "chb.ch_organization_id": opts.chOrganizationId }- : {}),- ...(opts.idempotencyKey- ? { "chb.idempotency_key": opts.idempotencyKey }- : {}),- });-- try {- return {- thrown: undefined,- body: await this.sendWithReplay(opts, span),- };- } catch (error) {- // A 409 is the "organization needs a payment method" UX branch, not- // an incident β which is why it is deliberately not logged as an- // error either. Carrying it out of the span instead of throwing- // through it keeps instrumentAsync from marking the operation as an- // APM error, so a routine checkout prompt cannot inflate the error- // rate this instrumentation exists to make readable.- if (error instanceof ChbPaymentRequiredError) {- span.setAttribute("chb.payment_required", true);- return { thrown: error, body: undefined };- }- throw error;- }- },- );-- // Rethrown outside the span so the caller contract is unchanged.- if (outcome.thrown) throw outcome.thrown;- return outcome.body;- }-- private async sendWithReplay(- opts: ChbRequestOptions,- span: Span,- ): Promise<unknown> {- const url = this.requestUrl(opts);-- let response = await this.send(url, opts, 1);- if (response.status === 401) {- // CHB rejected a token we still considered fresh β revoked, or the Auth0- // signing key rotated. A 401 means CHB did no work, and the retry carries- // the same idempotency key, so replaying once with a new token is safe.- logger.warn("[CHB API] Token rejected, retrying with a fresh one", {- method: opts.method,- path: opts.path,- });- span.setAttribute("chb.token_replayed", true);- this.tokenProvider.invalidate();- response = await this.send(url, opts, 2);- }-- span.setAttribute("http.status_code", response.status);-- const responseBody = await response- .json()- .catch(() => undefined as unknown);-- if (response.status === 409) {- throw new ChbPaymentRequiredError(responseBody);- }- if (!response.ok) {- logger.error("[CHB API] Request failed", {- method: opts.method,- path: opts.path,- status: response.status,- body: responseBody,- });- throw new ChbApiError(- `CHB API ${opts.method} ${opts.path} failed with status ${response.status}`,- response.status,- responseBody,- );- }-- return responseBody;- }-- async createCheckoutSession(params: {- organizationId?: string;- /** Names the CH organization when checkout creates it. */- name?: string;- email: string;- planCode: string;- returnUrl: string;- idempotencyKey?: string;- }): Promise<ChbCheckoutSession> {- // CHB rejects a blank name (400) but defaults one when the field is- // absent, so a whitespace-only name is dropped rather than sent.- const name = params.name?.trim();- const body = await this.request({- operation: "chb.checkout_session.create",- method: "POST",- path: "checkout-sessions",- body: {- ...(params.organizationId- ? { organizationId: params.organizationId }- : {}),- ...(name ? { name } : {}),- email: params.email,- planCode: params.planCode,- returnUrl: params.returnUrl,- // Required by CHB. Without an opId there is nothing to dedupe against,- // so a fresh key makes the call unique rather than rejected.- idempotencyKey: params.idempotencyKey ?? randomUUID(),- },- idempotencyKey: params.idempotencyKey,- });- return ChbCheckoutSessionSchema.parse(body);- }-- /**- * The organization's current attached plan. CHB scopes the attached-plan- * routes by the CH-Organization-Id header, there is no id in the path; a 404- * means the organization has none.- */- async getAttachedPlan(params: {- chOrganizationId: string;- }): Promise<ChbAttachedPlan> {- const body = await this.request({- operation: "chb.attachedplan.get",- method: "GET",- path: "attachedplan",- chOrganizationId: params.chOrganizationId,- searchParams: { fields: "plan,period,payment,scheduled" },- });- return ChbAttachedPlanSchema.parse(body);- }-- /** Upgrade now, schedule a downgrade for the cycle end, or cancel the plan. */- async setScheduledChange(params: {- chOrganizationId: string;- change: {- type: "upgrade" | "downgrade" | "cancel";- when: "immediate" | "billing_cycle_end";- planCode?: string;- };- idempotencyKey?: string;- }): Promise<void> {- await this.request({- operation: "chb.attachedplan.scheduled.set",- method: "PUT",- path: "attachedplan/scheduled",- chOrganizationId: params.chOrganizationId,- body: params.change,- idempotencyKey: params.idempotencyKey,- });- }-- /** Clear a pending scheduled change β reactivate / undo plan switch. */- async clearScheduledChange(params: {- chOrganizationId: string;- idempotencyKey?: string;- }): Promise<void> {- await this.request({- operation: "chb.attachedplan.scheduled.clear",- method: "DELETE",- path: "attachedplan/scheduled",- chOrganizationId: params.chOrganizationId,- idempotencyKey: params.idempotencyKey,- });- }-- /**- * Issued invoices, most recent first. Unlike the attached-plan routes this- * one is scoped by an `organizationId` query parameter.- */- async listInvoices(params: {- chOrganizationId: string;- }): Promise<ChbInvoice[]> {- const body = await this.request({- operation: "chb.invoices.list",- method: "GET",- path: "invoices",- chOrganizationId: params.chOrganizationId,- searchParams: { organizationId: params.chOrganizationId },- });- return ChbInvoiceListSchema.parse(body).invoices;- }-- async createPortalSession(params: {- chOrganizationId: string;- returnUrl: string;- }): Promise<string> {- const body = await this.request({- operation: "chb.portal_session.create",- method: "POST",- path: "portal-sessions",- chOrganizationId: params.chOrganizationId,- body: { returnUrl: params.returnUrl },- });- return ChbPortalSessionSchema.parse(body).portalUrl;- }-}+import { env } from "@/src/env.mjs";+import {+ buildChbApiClient,+ type ChbApiClient,+} from "@langfuse/shared/src/server/ee/clickhouseBilling";-/**- * Build a client from env, or null when the CHB REST surface is not fully- * configured. Callers treat null as "CHB unavailable" and fail closed β a- * partially configured deployment must not reach CHB unauthenticated.- *- * Exported for the env-validation tests only; call `getChbApiClient` instead,- * which shares one client (and therefore one token cache) process-wide.- */-export const buildChbApiClientFromEnv = (): ChbApiClient | null => {- if (- !env.CLICKHOUSE_BILLING_BASE_URL ||- !env.CLICKHOUSE_BILLING_AUTH0_DOMAIN ||- !env.CLICKHOUSE_BILLING_AUTH0_CLIENT_ID ||- !env.CLICKHOUSE_BILLING_AUTH0_CLIENT_SECRET- ) {- return null;- }- return new ChbApiClient({- baseUrl: env.CLICKHOUSE_BILLING_BASE_URL,- auth: {- auth0Domain: env.CLICKHOUSE_BILLING_AUTH0_DOMAIN,- clientId: env.CLICKHOUSE_BILLING_AUTH0_CLIENT_ID,- clientSecret: env.CLICKHOUSE_BILLING_AUTH0_CLIENT_SECRET,- audience: env.CLICKHOUSE_BILLING_AUTH0_AUDIENCE,- },- });-};+export {+ ChbApiClient,+ ChbPaymentRequiredError,+ type ChbAttachedPlan,+ type ChbCheckoutSession,+} from "@langfuse/shared/src/server/ee/clickhouseBilling";class ChbApiClientSingleton {private static instance: ChbApiClient | null;@@ -440,7 +30,7 @@ class ChbApiClientSingleton {if (!ChbApiClientSingleton.built) {// Cached even when null: an unconfigured deployment must not re-check// env on every billing request either.- ChbApiClientSingleton.instance = buildChbApiClientFromEnv();+ ChbApiClientSingleton.instance = buildChbApiClient(env);ChbApiClientSingleton.built = true;}return ChbApiClientSingleton.instance;diff --git a/web/src/ee/features/billing/server/chb/chbWebhookHandler.ts b/web/src/ee/features/billing/server/chb/chbWebhookHandler.tsindex 238afac006..6b4993c3cc 100644--- a/web/src/ee/features/billing/server/chb/chbWebhookHandler.ts+++ b/web/src/ee/features/billing/server/chb/chbWebhookHandler.ts@@ -6,7 +6,9 @@ import { z } from "zod";import { env } from "@/src/env.mjs";import { auditLog } from "@/src/features/audit-logs/server";import {+ type ChbPlanCode,chbPlanCodes,+ chbPlanCodeToPlan,CloudConfigSchema,parseDbOrg,type ParsedOrganization,@@ -21,6 +23,7 @@ import {traceException,} from "@langfuse/shared/src/server";+import { createDefaultSpendAlerts } from "../defaultSpendAlerts";import { type ChbAttachedPlan, getChbApiClient } from "./chbApiClient";import { sendChbProjectEvent } from "./chbProjectEvents";@@ -504,16 +507,19 @@ async function readAttachedPlanState(// The pipeline already refused an unknown code on the event itself, so a// rejection here means CHB reports a code that has no mapping yet. Same// answer: the 500 keeps the event retryable until the mapping ships.- const planCode =+ const reportedPlanCode =attachedPlan.plan?.code ?? event.data.payload.planCode ?? null;if (- planCode != null &&- !(chbPlanCodes as readonly string[]).includes(planCode)+ reportedPlanCode != null &&+ !(chbPlanCodes as readonly string[]).includes(reportedPlanCode)) {throw new Error(- `Attached plan ${attachedPlan.id} for org ${parsedOrg.id} carries unknown plan code ${planCode}`,+ `Attached plan ${attachedPlan.id} for org ${parsedOrg.id} carries unknown plan code ${reportedPlanCode}`,);}+ // Narrowed against the membership check above, which is the runtime source+ // of truth; callers need the code typed to map it onto a Langfuse plan.+ const planCode = reportedPlanCode as ChbPlanCode | null;return {attachedPlan,@@ -563,9 +569,24 @@ async function handleAttachedPlanCreated(},});- // No default spend alerts for CHB orgs. The spend-alert job computes current- // spend from the org's own Stripe subscription, which a CHB org does not- // have, so seeded alerts would show in billing settings and never fire.+ // Seed the plan's default spend alerts, as the Stripe path does on a first+ // subscription. Post-commit and best-effort: the plan is already applied, and+ // an org without seeded alerts can still configure its own.+ if (clickhouse.planCode) {+ try {+ await createDefaultSpendAlerts({+ orgId: parsedOrg.id,+ plan: chbPlanCodeToPlan[clickhouse.planCode],+ source: "clickhouse",+ });+ } catch (error) {+ logger.error(+ `[CHB Webhook] Failed to create default spend alerts for org ${parsedOrg.id}`,+ error,+ );+ traceException(error);+ }+ }// Backfill-emit LANGFUSE_PROJECT_CREATED for all existing projects: projects// created before checkout would otherwise be invisible to CHB metering.diff --git a/web/src/ee/features/billing/server/defaultSpendAlerts.ts b/web/src/ee/features/billing/server/defaultSpendAlerts.tsnew file mode 100644index 0000000000..5f288ecca0--- /dev/null+++ b/web/src/ee/features/billing/server/defaultSpendAlerts.ts@@ -0,0 +1,123 @@+import { type Plan } from "@langfuse/shared";+import { prisma } from "@langfuse/shared/src/db";+import { logger } from "@langfuse/shared/src/server";++import { auditLog } from "@/src/features/audit-logs/server";+import { mapStripeProductIdToPlan } from "@/src/ee/features/billing/utils/stripeCatalogue";++/**+ * Seeding of the default spend alerts a paid organization starts out with.+ *+ * Provider-neutral on purpose: both webhooks call in here with a resolved plan,+ * so a CHB-billed org gets the same starting thresholds a Stripe-billed one+ * does. Which provider then evaluates them is the spend-alert job's business.+ */++type PlanWithoutSpendAlerts =+ | "oss"+ | "cloud:hobby"+ | "self-hosted:pro"+ | "self-hosted:enterprise";++type PlanWithSpendAlerts = Exclude<Plan, PlanWithoutSpendAlerts>;++const DEFAULT_SPEND_ALERT_THRESHOLDS: Record<PlanWithSpendAlerts, number> = {+ "cloud:core": 200,+ "cloud:pro": 1000,+ "cloud:team": 1000,+ "cloud:enterprise": 2000,+};++// Universal threshold applied to all plans in addition to the plan-specific threshold+const UNIVERSAL_SPEND_ALERT_THRESHOLD = 4000;++/** Which webhook is seeding, for the audit actor and the log prefix. */+export type SpendAlertSource = "stripe" | "clickhouse";++const logPrefixes: Record<SpendAlertSource, string> = {+ stripe: "[Stripe Webhook]",+ clickhouse: "[CHB Webhook]",+};++export async function createDefaultSpendAlerts({+ orgId,+ plan,+ source,+}: {+ orgId: string;+ plan: Plan;+ source: SpendAlertSource;+}) {+ const logPrefix = logPrefixes[source];++ const planThreshold =+ DEFAULT_SPEND_ALERT_THRESHOLDS[plan as PlanWithSpendAlerts];+ if (!planThreshold) {+ logger.error(+ `${logPrefix} createDefaultSpendAlerts: No spend alerts configured for plan ${plan}, skipping`,+ );+ return;+ }++ // Skip if org already has spend alerts (idempotency / don't overwrite user-configured alerts)+ const existingAlerts = await prisma.cloudSpendAlert.findFirst({+ where: { orgId },+ select: { id: true },+ });+ if (existingAlerts) {+ logger.info(+ `${logPrefix} createDefaultSpendAlerts: Org ${orgId} already has spend alerts, skipping`,+ );+ return;+ }++ // Create plan-specific alert plus the universal $4K alert (deduplicated)+ const thresholds = [+ ...new Set([planThreshold, UNIVERSAL_SPEND_ALERT_THRESHOLD]),+ ];++ for (const threshold of thresholds) {+ const alert = await prisma.cloudSpendAlert.create({+ data: {+ orgId,+ title: `Default Spend alert ($${threshold})`,+ threshold,+ },+ });++ await auditLog({+ session: {+ user: { id: `${source}-webhook` },+ orgId,+ },+ orgId,+ resourceType: "cloudSpendAlert",+ resourceId: alert.id,+ action: "create",+ after: alert,+ });++ logger.info(+ `${logPrefix} createDefaultSpendAlerts: Created default alert over $${threshold} for org ${orgId} on plan ${plan}`,+ );+ }+}++/** Stripe's entry point: it knows the subscription's product, not the plan. */+export async function createDefaultSpendAlertsForStripeProduct({+ orgId,+ productId,+}: {+ orgId: string;+ productId: string;+}) {+ const plan = mapStripeProductIdToPlan(productId);+ if (!plan) {+ logger.error(+ `[Stripe Webhook] createDefaultSpendAlerts: Unknown product ID ${productId}, skipping`,+ );+ return;+ }++ await createDefaultSpendAlerts({ orgId, plan, source: "stripe" });+}diff --git a/web/src/ee/features/billing/server/stripe/stripeWebhookHandler.ts b/web/src/ee/features/billing/server/stripe/stripeWebhookHandler.tsindex 89ccc64c41..6f81e239d5 100644--- a/web/src/ee/features/billing/server/stripe/stripeWebhookHandler.ts+++ b/web/src/ee/features/billing/server/stripe/stripeWebhookHandler.ts@@ -13,7 +13,6 @@ import {InternalServerError,type Organization,type ParsedOrganization,- type Plan,parseDbOrg,} from "@langfuse/shared";import {@@ -24,8 +23,8 @@ import {} from "@langfuse/shared/src/server";import { auditLog } from "@/src/features/audit-logs/server";import { type StripeSubscriptionMetadata } from "@/src/ee/features/billing/utils/stripeSubscriptionMetadata";-import { mapStripeProductIdToPlan } from "@/src/ee/features/billing/utils/stripeCatalogue";import { syncOrgPlanChangeToSfdc } from "@/src/ee/features/sfdc-sync/server";+import { createDefaultSpendAlertsForStripeProduct } from "@/src/ee/features/billing/server/defaultSpendAlerts";/*** Stripe webhook handler for managing subscription events, billing alerts, and invoice notifications.@@ -462,95 +461,6 @@ async function updateOrgBillingCycleAnchor(orgId: string, anchor?: Date) {});}-type PlanWithoutSpendAlerts =- | "oss"- | "cloud:hobby"- | "self-hosted:pro"- | "self-hosted:enterprise";--const DEFAULT_SPEND_ALERT_THRESHOLDS: Record<- Exclude<Plan, PlanWithoutSpendAlerts>,- number-> = {- "cloud:core": 200,- "cloud:pro": 1000,- "cloud:team": 1000,- "cloud:enterprise": 2000,-};--// Universal threshold applied to all plans in addition to the plan-specific threshold-const UNIVERSAL_SPEND_ALERT_THRESHOLD = 4000;--export async function createDefaultSpendAlerts({- orgId,- productId,-}: {- orgId: string;- productId: string;-}) {- const plan = mapStripeProductIdToPlan(productId) as Exclude<- Plan,- PlanWithoutSpendAlerts- >;- if (!plan) {- logger.error(- `[Stripe Webhook] createDefaultSpendAlerts: Unknown product ID ${productId}, skipping`,- );- return;- }-- const planThreshold = DEFAULT_SPEND_ALERT_THRESHOLDS[plan];- if (!planThreshold) {- logger.error(- `[Stripe Webhook] createDefaultSpendAlerts: No spend alerts configured for plan ${plan}, skipping`,- );- return;- }-- // Skip if org already has spend alerts (idempotency / don't overwrite user-configured alerts)- const existingAlerts = await prisma.cloudSpendAlert.findFirst({- where: { orgId },- select: { id: true },- });- if (existingAlerts) {- logger.info(- `[Stripe Webhook] createDefaultSpendAlerts: Org ${orgId} already has spend alerts, skipping`,- );- return;- }-- // Create plan-specific alert plus the universal $4K alert (deduplicated)- const thresholds = [- ...new Set([planThreshold, UNIVERSAL_SPEND_ALERT_THRESHOLD]),- ];-- for (const threshold of thresholds) {- const alert = await prisma.cloudSpendAlert.create({- data: {- orgId,- title: `Default Spend alert ($${threshold})`,- threshold,- },- });-- await auditLog({- session: {- user: { id: "stripe-webhook" },- orgId,- },- orgId,- resourceType: "cloudSpendAlert",- resourceId: alert.id,- action: "create",- after: alert,- });-- logger.info(- `[Stripe Webhook] createDefaultSpendAlerts: Created default alert over $${threshold} for org ${orgId} on plan ${plan}`,- );- }-}-export async function handleSubscriptionChanged(subscription: Stripe.Subscription,action: "created" | "deleted" | "updated",@@ -751,7 +661,7 @@ export async function handleSubscriptionChanged(// Auto-create default spend alerts for new subscriptions (best-effort)if (action === "created") {try {- await createDefaultSpendAlerts({+ await createDefaultSpendAlertsForStripeProduct({orgId: parsedOrg.id,productId: productId,});diff --git a/worker/src/app.ts b/worker/src/app.tsindex 6fcc63ebc3..f5ae962ac3 100644--- a/worker/src/app.ts+++ b/worker/src/app.ts@@ -20,6 +20,7 @@ import { installProcessErrorHandlers } from "@langfuse/shared/src/server";import helmet from "helmet";import { cloudUsageMeteringQueueProcessor } from "./queues/cloudUsageMeteringQueue";import { cloudSpendAlertQueueProcessor } from "./queues/cloudSpendAlertQueue";+import { isChbConfigured } from "./ee/cloudSpendAlerts/chbApiClient";import { cloudFreeTierUsageThresholdQueueProcessor } from "./queues/cloudFreeTierUsageThresholdQueue";import { monitorQueueProcessor } from "./queues/monitorQueue";import { inAppAgentRunQueueProcessor } from "./queues/inAppAgentRunQueue";@@ -40,6 +41,7 @@ import {SecondaryOtelIngestionQueue,TraceUpsertQueue,CloudFreeTierUsageThresholdQueue,+ CloudSpendAlertQueue,CloudUsageMeteringQueue,V4LegacyApiUsageQueue,EventPropagationQueue,@@ -503,11 +505,19 @@ if (inAppAgentDlqRetryRunner.start();}-// Cloud Spend Alert Queue: Only enable in cloud environment with Stripe+// Cloud Spend Alert Queue: cloud only, and only where at least one billing+// provider can be reached β Stripe for legacy orgs, ClickHouse Billing for+// orgs billed through CHB.if (env.QUEUE_CONSUMER_CLOUD_SPEND_ALERT_QUEUE_IS_ENABLED === "true" &&- env.STRIPE_SECRET_KEY+ (env.STRIPE_SECRET_KEY || isChbConfigured())) {+ // Instantiate the queue to trigger scheduled jobs β this is what installs+ // the hourly CHB fan-out. Without it the schedule only ever appeared as a+ // side effect of the Stripe metering job touching the same queue, so a+ // CHB-only deployment produced no fan-out at all.+ CloudSpendAlertQueue.getInstance();+WorkerManager.register(QueueName.CloudSpendAlertQueue,cloudSpendAlertQueueProcessor,diff --git a/worker/src/ee/cloudSpendAlerts/__tests__/handleCloudSpendAlertFanOutJob.test.ts b/worker/src/ee/cloudSpendAlerts/__tests__/handleCloudSpendAlertFanOutJob.test.tsnew file mode 100644index 0000000000..85c1278fdb--- /dev/null+++ b/worker/src/ee/cloudSpendAlerts/__tests__/handleCloudSpendAlertFanOutJob.test.ts@@ -0,0 +1,130 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++const {+ mockOrganizationFindMany,+ mockQueueAdd,+ mockGetQueueInstance,+ mockIsChbConfigured,+ mockLoggerInfo,+ mockLoggerError,+ mockLoggerDebug,+ mockRecordIncrement,+} = vi.hoisted(() => ({+ mockOrganizationFindMany: vi.fn(),+ mockQueueAdd: vi.fn(),+ mockGetQueueInstance: vi.fn(),+ mockIsChbConfigured: vi.fn(),+ mockLoggerInfo: vi.fn(),+ mockLoggerError: vi.fn(),+ mockLoggerDebug: vi.fn(),+ mockRecordIncrement: vi.fn(),+}));++vi.mock("@langfuse/shared/src/db", () => ({+ prisma: { organization: { findMany: mockOrganizationFindMany } },+ Prisma: { DbNull: Symbol("DbNull") },+}));++vi.mock("@langfuse/shared/src/server", () => ({+ CloudSpendAlertQueue: { getInstance: mockGetQueueInstance },+ QueueJobs: { CloudSpendAlertJob: "cloud-spend-alert-job" },+ logger: {+ info: mockLoggerInfo,+ error: mockLoggerError,+ debug: mockLoggerDebug,+ warn: vi.fn(),+ },+ recordIncrement: mockRecordIncrement,+}));++vi.mock("../chbApiClient", () => ({+ isChbConfigured: mockIsChbConfigured,+}));++import { Prisma } from "@langfuse/shared/src/db";++import { handleCloudSpendAlertFanOutJob } from "../handleCloudSpendAlertFanOutJob";++describe("handleCloudSpendAlertFanOutJob", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ mockIsChbConfigured.mockReturnValue(true);+ mockGetQueueInstance.mockReturnValue({ add: mockQueueAdd });+ mockQueueAdd.mockResolvedValue({});+ mockOrganizationFindMany.mockResolvedValue([]);+ });++ it("does nothing on a deployment without CHB", async () => {+ mockIsChbConfigured.mockReturnValue(false);++ await handleCloudSpendAlertFanOutJob();++ expect(mockOrganizationFindMany).not.toHaveBeenCalled();+ expect(mockQueueAdd).not.toHaveBeenCalled();+ });++ /**+ * Scoping matters in both directions: an org without an attached plan has+ * nothing accruing, and a Stripe-billed org is already enqueued by the usage+ * metering job β including it here would double its preview-invoice calls.+ */+ it("selects only ClickHouse-billed orgs that have alerts configured", async () => {+ await handleCloudSpendAlertFanOutJob();++ expect(mockOrganizationFindMany).toHaveBeenCalledWith({+ where: {+ cloudConfig: {+ path: ["clickhouse", "attachedPlanId"],+ not: Prisma.DbNull,+ },+ cloudSpendAlerts: { some: {} },+ },+ select: { id: true },+ });+ });++ it("enqueues one evaluation job per org", async () => {+ mockOrganizationFindMany.mockResolvedValue([+ { id: "org-1" },+ { id: "org-2" },+ ]);++ await handleCloudSpendAlertFanOutJob();++ expect(mockQueueAdd.mock.calls).toEqual([+ ["cloud-spend-alert-job", { orgId: "org-1" }],+ ["cloud-spend-alert-job", { orgId: "org-2" }],+ ]);+ expect(mockRecordIncrement).toHaveBeenCalledWith(+ "langfuse.queue.cloud_spend_alert_queue.fanned_out_orgs",+ 2,+ { unit: "organizations" },+ );+ });++ it("keeps going when one org fails to enqueue", async () => {+ mockOrganizationFindMany.mockResolvedValue([+ { id: "org-1" },+ { id: "org-2" },+ { id: "org-3" },+ ]);+ mockQueueAdd.mockRejectedValueOnce(new Error("redis blip"));++ await expect(handleCloudSpendAlertFanOutJob()).resolves.toBeUndefined();++ expect(mockQueueAdd).toHaveBeenCalledTimes(3);+ expect(mockLoggerError).toHaveBeenCalled();+ // Only the orgs actually enqueued are counted.+ expect(mockRecordIncrement).toHaveBeenCalledWith(+ "langfuse.queue.cloud_spend_alert_queue.fanned_out_orgs",+ 2,+ { unit: "organizations" },+ );+ });++ it("does not touch the queue when there is nothing to fan out", async () => {+ await handleCloudSpendAlertFanOutJob();++ expect(mockGetQueueInstance).not.toHaveBeenCalled();+ });+});diff --git a/worker/src/ee/cloudSpendAlerts/__tests__/handleCloudSpendAlertJob.test.ts b/worker/src/ee/cloudSpendAlerts/__tests__/handleCloudSpendAlertJob.test.tsindex a9b9889d80..ff3326c38f 100644--- a/worker/src/ee/cloudSpendAlerts/__tests__/handleCloudSpendAlertJob.test.ts+++ b/worker/src/ee/cloudSpendAlerts/__tests__/handleCloudSpendAlertJob.test.ts@@ -3,6 +3,8 @@ import type { Job } from "bullmq";const {mockOrganizationFindFirst,+ mockMembershipFindMany,+ mockSpendAlertUpdate,mockLoggerInfo,mockLoggerWarn,mockLoggerError,@@ -10,8 +12,12 @@ const {mockRecordIncrement,mockTraceException,mockSendCloudSpendAlertEmail,+ mockQueryBills,+ mockGetChbApiClient,} = vi.hoisted(() => ({mockOrganizationFindFirst: vi.fn(),+ mockMembershipFindMany: vi.fn(),+ mockSpendAlertUpdate: vi.fn(),mockLoggerInfo: vi.fn(),mockLoggerWarn: vi.fn(),mockLoggerError: vi.fn(),@@ -19,15 +25,27 @@ const {mockRecordIncrement: vi.fn(),mockTraceException: vi.fn(),mockSendCloudSpendAlertEmail: vi.fn(),+ mockQueryBills: vi.fn(),+ mockGetChbApiClient: vi.fn(),}));-vi.mock("@langfuse/shared", () => ({- parseDbOrg: (org: unknown) => org,- Role: {- ADMIN: "ADMIN",- OWNER: "OWNER",- },-}));+/**+ * The real `getBillingProvider`, not a stub: which provider a cloudConfig+ * resolves to is exactly what decides whether the job previews a Stripe+ * invoice or reads CHB's accrued usage, so stubbing it would test nothing.+ */+vi.mock("@langfuse/shared", async () => {+ const { getBillingProvider } =+ await import("../../../../../packages/shared/src/interfaces/billingProvider");+ return {+ getBillingProvider,+ parseDbOrg: (org: unknown) => org,+ Role: {+ ADMIN: "ADMIN",+ OWNER: "OWNER",+ },+ };+});vi.mock("@langfuse/shared/src/db", () => ({prisma: {@@ -35,14 +53,20 @@ vi.mock("@langfuse/shared/src/db", () => ({findFirst: mockOrganizationFindFirst,},organizationMembership: {- findMany: vi.fn(),+ findMany: mockMembershipFindMany,},cloudSpendAlert: {- update: vi.fn(),+ update: mockSpendAlertUpdate,},},}));+/**+ * Only the side-effecting exports (logging, metrics, email) are replaced. The+ * CHB helpers stay real: `chbBillUsageAmountUSD` encodes the major-unit/USD-only+ * contract these cases exist to pin, and `ChbApiError` is what the job's status+ * branching uses `instanceof` against.+ */vi.mock("@langfuse/shared/src/server", () => ({logger: {info: mockLoggerInfo,@@ -55,12 +79,19 @@ vi.mock("@langfuse/shared/src/server", () => ({sendCloudSpendAlertEmail: mockSendCloudSpendAlertEmail,}));+vi.mock("../chbApiClient", () => ({+ getChbApiClient: mockGetChbApiClient,+}));+vi.mock("../../../env", () => ({env: {STRIPE_SECRET_KEY: "sk_test",+ LANGFUSE_CLOUD_BILLING_CHB_CUTOFF_DATE: null,},}));+import { ChbApiError } from "@langfuse/shared/src/server/ee/clickhouseBilling";+import { handleCloudSpendAlertJob } from "../handleCloudSpendAlertJob";const createJob = (orgId = "org-1") =>@@ -68,23 +99,76 @@ const createJob = (orgId = "org-1") =>data: { orgId },}) as Job<{ orgId: string }>;-const createOrg = (cloudConfig: Record<string, unknown>) => ({+const createOrg = (+ cloudConfig: Record<string, unknown>,+ alertOverrides: Partial<{+ id: string;+ title: string;+ threshold: string;+ triggeredAt: Date | null;+ }> = {},+) => ({id: "org-1",name: "Test Org",cloudConfig,cloudSpendAlerts: [{- id: "alert-1",- title: "Default Spend alert ($200)",- threshold: { toString: () => "200" },- triggeredAt: null,+ id: alertOverrides.id ?? "alert-1",+ title: alertOverrides.title ?? "Default Spend alert ($200)",+ threshold: { toString: () => alertOverrides.threshold ?? "200" },+ triggeredAt:+ alertOverrides.triggeredAt === undefined+ ? null+ : alertOverrides.triggeredAt,},],});+const CH_ORG_ID = "6dd6ab1d-9e8d-4c1a-8b4f-9a3d1e2c4b5a";++/** A CHB cloudConfig: attached plan present, no Stripe state at all. */+const chbCloudConfig = () => ({+ clickhouse: {+ organizationId: CH_ORG_ID,+ attachedPlanId: "ap_1",+ planCode: "LANGFUSE_CORE",+ },+});++/** A `POST /bills/query` result holding the org's one open bill. */+const openBill = (+ amount: number | null,+ currency: string | null = "USD",+ startDate: string | null = "2026-09-01T00:00:00.000Z",+) => [+ {+ id: "bill_1",+ organizationId: CH_ORG_ID,+ startDate,+ endDate: "2026-10-01T00:00:00.000Z",+ usage: { amount, currency, asOf: "2026-09-29T12:00:00.000Z" },+ },+];++const skipReasons = () =>+ mockRecordIncrement.mock.calls+ .filter(+ (call) =>+ call[0] ===+ "langfuse.queue.cloud_spend_alert_queue.skipped_orgs_missing_billing_config",+ )+ .map((call) => (call[2] as { reason: string }).reason);+describe("handleCloudSpendAlertJob", () => {beforeEach(() => {vi.clearAllMocks();+ mockMembershipFindMany.mockResolvedValue([+ { user: { email: "[email protected]" } },+ ]);+ mockSpendAlertUpdate.mockResolvedValue({});+ mockGetChbApiClient.mockReturnValue({+ queryBills: mockQueryBills,+ });});it("skips with a healthy span when the Stripe customer id is missing", async () => {@@ -142,4 +226,170 @@ describe("handleCloudSpendAlertJob", () => {},);});++ describe("ClickHouse-billed organizations", () => {+ it("never reaches for Stripe state", async () => {+ mockOrganizationFindFirst.mockResolvedValue(createOrg(chbCloudConfig()));+ mockQueryBills.mockResolvedValue(openBill(10));++ await handleCloudSpendAlertJob(createJob());++ // The Stripe branch's skip reasons are what a CHB org used to collect.+ expect(skipReasons()).toEqual([]);+ expect(mockQueryBills).toHaveBeenCalledWith({+ chOrganizationIds: [CH_ORG_ID],+ });+ });++ /**+ * CHB reports the accrued amount in major-unit USD (1250.68 means+ * $1,250.68). Reading it as minor units would put every threshold 100x out+ * of reach.+ */+ it("compares the accrued amount against the threshold as whole USD", async () => {+ mockOrganizationFindFirst.mockResolvedValue(createOrg(chbCloudConfig()));+ mockQueryBills.mockResolvedValue(openBill(250.75));++ await handleCloudSpendAlertJob(createJob());++ expect(mockSendCloudSpendAlertEmail).toHaveBeenCalledTimes(1);+ expect(mockSendCloudSpendAlertEmail.mock.calls[0]![0]).toMatchObject({+ currentSpend: 250.75,+ threshold: 200,+ recipients: ["[email protected]"],+ });+ expect(mockSpendAlertUpdate).toHaveBeenCalledWith({+ where: { id: "alert-1" },+ data: { triggeredAt: expect.any(Date) },+ });+ });++ it("does not alert below the threshold", async () => {+ mockOrganizationFindFirst.mockResolvedValue(createOrg(chbCloudConfig()));+ mockQueryBills.mockResolvedValue(openBill(199.99));++ await handleCloudSpendAlertJob(createJob());++ expect(mockSendCloudSpendAlertEmail).not.toHaveBeenCalled();+ expect(mockSpendAlertUpdate).not.toHaveBeenCalled();+ });++ it("alerts once per billing period", async () => {+ mockOrganizationFindFirst.mockResolvedValue(+ createOrg(chbCloudConfig(), {+ // Already fired inside the period the open bill reports.+ triggeredAt: new Date("2026-09-10T00:00:00.000Z"),+ }),+ );+ mockQueryBills.mockResolvedValue(openBill(250));++ await handleCloudSpendAlertJob(createJob());++ expect(mockSendCloudSpendAlertEmail).not.toHaveBeenCalled();+ expect(mockSpendAlertUpdate).not.toHaveBeenCalled();+ });++ it("alerts again once the period has rolled over", async () => {+ mockOrganizationFindFirst.mockResolvedValue(+ createOrg(chbCloudConfig(), {+ // Fired in the previous period, which the bill's startDate ended.+ triggeredAt: new Date("2026-08-10T00:00:00.000Z"),+ }),+ );+ mockQueryBills.mockResolvedValue(openBill(250));++ await handleCloudSpendAlertJob(createJob());++ expect(mockSendCloudSpendAlertEmail).toHaveBeenCalledTimes(1);+ });++ it("refuses a currency other than USD rather than comparing it", async () => {+ mockOrganizationFindFirst.mockResolvedValue(createOrg(chbCloudConfig()));+ mockQueryBills.mockResolvedValue(openBill(5000, "EUR"));++ await handleCloudSpendAlertJob(createJob());++ expect(mockSendCloudSpendAlertEmail).not.toHaveBeenCalled();+ expect(skipReasons()).toContain("missing_chb_usage_amount");+ });++ it("skips when CHB reports no period start to scope the alert to", async () => {+ mockOrganizationFindFirst.mockResolvedValue(createOrg(chbCloudConfig()));+ mockQueryBills.mockResolvedValue(openBill(250, "USD", null));++ await handleCloudSpendAlertJob(createJob());++ expect(mockSendCloudSpendAlertEmail).not.toHaveBeenCalled();+ expect(skipReasons()).toContain("missing_chb_period_start");+ });++ it("skips an org CHB reports no open bill for, without failing", async () => {+ mockOrganizationFindFirst.mockResolvedValue(createOrg(chbCloudConfig()));+ // CHB omits an org without an open bill instead of erroring.+ mockQueryBills.mockResolvedValue([]);++ await expect(+ handleCloudSpendAlertJob(createJob()),+ ).resolves.toBeUndefined();++ expect(mockSendCloudSpendAlertEmail).not.toHaveBeenCalled();+ expect(mockTraceException).not.toHaveBeenCalled();+ expect(skipReasons()).toContain("chb_open_bill_not_found");+ });++ /**+ * Unlike the attached-plan route, `/bills/query` has no "not found" of its+ * own: a 4xx means the request itself was refused (a route CHB has not+ * deployed, a token without access), which must fail visibly.+ */+ it("fails on a CHB 4xx without spending the retry budget", async () => {+ mockOrganizationFindFirst.mockResolvedValue(createOrg(chbCloudConfig()));+ mockQueryBills.mockRejectedValue(new ChbApiError("not found", 404));++ await expect(handleCloudSpendAlertJob(createJob())).rejects.toThrow(+ "not found",+ );++ expect(mockQueryBills).toHaveBeenCalledTimes(1);+ expect(mockTraceException).toHaveBeenCalled();+ });++ it("skips an org whose attached plan was cancelled locally", async () => {+ mockOrganizationFindFirst.mockResolvedValue(+ createOrg({ clickhouse: { organizationId: CH_ORG_ID } }),+ );++ await handleCloudSpendAlertJob(createJob());++ expect(mockQueryBills).not.toHaveBeenCalled();+ expect(skipReasons()).toContain("missing_chb_attached_plan");+ });++ it("skips when CHB is not configured on this deployment", async () => {+ mockOrganizationFindFirst.mockResolvedValue(createOrg(chbCloudConfig()));+ mockGetChbApiClient.mockReturnValue(null);++ await handleCloudSpendAlertJob(createJob());++ expect(skipReasons()).toContain("chb_not_configured");+ expect(mockTraceException).not.toHaveBeenCalled();+ });++ it("retries and rethrows a CHB server error", async () => {+ mockOrganizationFindFirst.mockResolvedValue(createOrg(chbCloudConfig()));+ mockQueryBills.mockRejectedValue(new ChbApiError("boom", 500));++ await expect(handleCloudSpendAlertJob(createJob())).rejects.toThrow(+ "boom",+ );++ expect(mockQueryBills).toHaveBeenCalledTimes(3);+ expect(mockTraceException).toHaveBeenCalled();+ expect(mockRecordIncrement).toHaveBeenCalledWith(+ "langfuse.queue.cloud_spend_alert_queue.skipped_orgs_with_errors",+ 1,+ { unit: "organizations" },+ );+ });+ });});diff --git a/worker/src/ee/cloudSpendAlerts/chbApiClient.ts b/worker/src/ee/cloudSpendAlerts/chbApiClient.tsnew file mode 100644index 0000000000..957beca8ad--- /dev/null+++ b/worker/src/ee/cloudSpendAlerts/chbApiClient.ts@@ -0,0 +1,34 @@+import {+ buildChbApiClient,+ type ChbApiClient,+} from "@langfuse/shared/src/server/ee/clickhouseBilling";++import { env } from "../../env";++/**+ * The worker's env-bound handle on the ClickHouse Billing REST client β the+ * twin of `web/src/ee/features/billing/server/chb/chbApiClient.ts`. Shared+ * owns the transport and the fail-closed build rule; the singleton is per-app+ * so one Auth0 token is minted per process rather than per job.+ */+class ChbApiClientSingleton {+ private static instance: ChbApiClient | null;+ private static built = false;++ public static getInstance(): ChbApiClient | null {+ if (!ChbApiClientSingleton.built) {+ // Cached even when null: an unconfigured deployment must not re-check+ // env on every job either.+ ChbApiClientSingleton.instance = buildChbApiClient(env);+ ChbApiClientSingleton.built = true;+ }+ return ChbApiClientSingleton.instance;+ }+}++/** The CHB client for this process, or null when CHB is not fully configured. */+export const getChbApiClient = (): ChbApiClient | null =>+ ChbApiClientSingleton.getInstance();++/** Whether this deployment can reach CHB at all. */+export const isChbConfigured = (): boolean => getChbApiClient() !== null;diff --git a/worker/src/ee/cloudSpendAlerts/handleCloudSpendAlertFanOutJob.ts b/worker/src/ee/cloudSpendAlerts/handleCloudSpendAlertFanOutJob.tsnew file mode 100644index 0000000000..bd6c83d87f--- /dev/null+++ b/worker/src/ee/cloudSpendAlerts/handleCloudSpendAlertFanOutJob.ts@@ -0,0 +1,85 @@+import { Prisma, prisma } from "@langfuse/shared/src/db";+import {+ CloudSpendAlertQueue,+ logger,+ QueueJobs,+ recordIncrement,+} from "@langfuse/shared/src/server";++import { isChbConfigured } from "./chbApiClient";++/**+ * Hourly trigger for ClickHouse-billed organizations' spend alerts.+ *+ * Stripe-billed orgs get theirs from the usage metering job, which enqueues a+ * per-org alert job after pushing the hour's meter events. That job iterates+ * orgs by Stripe customer id, so it never sees a CHB org β CHB meters by+ * polling our metrics API instead, and there is no corresponding local loop.+ * This fan-out is that loop: it finds CHB orgs with alerts configured and+ * enqueues one evaluation job each.+ *+ * Scoped to CHB orgs on purpose. Stripe orgs are already covered, and+ * including them here would double their preview-invoice calls.+ */+export const handleCloudSpendAlertFanOutJob = async () => {+ if (!isChbConfigured()) {+ // A Stripe-only deployment still runs this schedule; there is simply+ // nothing for it to do.+ logger.debug(+ "[CLOUD SPEND ALERTS] CHB is not configured, nothing to fan out",+ );+ return;+ }++ const orgs = await prisma.organization.findMany({+ where: {+ // Carrying an attached plan is what makes an org CHB-billed and paying.+ // A cancelled org keeps `clickhouse.organizationId` but loses this.+ cloudConfig: {+ path: ["clickhouse", "attachedPlanId"],+ not: Prisma.DbNull,+ },+ cloudSpendAlerts: { some: {} },+ },+ select: { id: true },+ });++ if (orgs.length === 0) {+ logger.debug(+ "[CLOUD SPEND ALERTS] No ClickHouse-billed orgs with spend alerts",+ );+ return;+ }++ logger.info(+ `[CLOUD SPEND ALERTS] Fanning out to ${orgs.length} ClickHouse-billed organizations`,+ );++ const queue = CloudSpendAlertQueue.getInstance();+ if (!queue) {+ logger.error(+ "[CLOUD SPEND ALERTS] Spend alert queue unavailable, cannot fan out",+ );+ return;+ }++ let enqueued = 0;+ for (const org of orgs) {+ try {+ await queue.add(QueueJobs.CloudSpendAlertJob, { orgId: org.id });+ enqueued += 1;+ } catch (error) {+ // One org failing to enqueue must not cost the rest their hour.+ logger.error(+ `[CLOUD SPEND ALERTS] Failed to enqueue spend alert job for org ${org.id}`,+ error,+ );+ }+ }++ recordIncrement(+ "langfuse.queue.cloud_spend_alert_queue.fanned_out_orgs",+ enqueued,+ { unit: "organizations" },+ );+};diff --git a/worker/src/ee/cloudSpendAlerts/handleCloudSpendAlertJob.ts b/worker/src/ee/cloudSpendAlerts/handleCloudSpendAlertJob.tsindex adc40ca824..fd2b395d86 100644--- a/worker/src/ee/cloudSpendAlerts/handleCloudSpendAlertJob.ts+++ b/worker/src/ee/cloudSpendAlerts/handleCloudSpendAlertJob.ts@@ -1,16 +1,36 @@-import { parseDbOrg, Role } from "@langfuse/shared";+import { getBillingProvider, parseDbOrg, Role } from "@langfuse/shared";import { prisma } from "@langfuse/shared/src/db";import Stripe from "stripe";import { env } from "../../env";import { logger } from "@langfuse/shared/src/server";+import {+ CHB_USAGE_CURRENCY,+ ChbApiError,+ chbBillUsageAmountUSD,+} from "@langfuse/shared/src/server/ee/clickhouseBilling";import { recordIncrement, traceException } from "@langfuse/shared/src/server";import { Job } from "bullmq";import { backOff } from "exponential-backoff";import { sendCloudSpendAlertEmail } from "@langfuse/shared/src/server";-const recordMissingBillingConfigSkip = (- reason: "missing_stripe_customer_id" | "missing_stripe_subscription_id",-) => {+import { getChbApiClient } from "./chbApiClient";++/**+ * Why a spend reading was unavailable. Every reason is a configuration or+ * lifecycle state the job cannot act on, never an error β the org is counted+ * and skipped so a gap in alerting is visible without paging.+ */+type MissingBillingConfigReason =+ | "missing_stripe_customer_id"+ | "missing_stripe_subscription_id"+ | "missing_chb_organization_id"+ | "missing_chb_attached_plan"+ | "chb_not_configured"+ | "chb_open_bill_not_found"+ | "missing_chb_usage_amount"+ | "missing_chb_period_start";++const recordMissingBillingConfigSkip = (reason: MissingBillingConfigReason) => {recordIncrement("langfuse.queue.cloud_spend_alert_queue.skipped_orgs_missing_billing_config",1,@@ -21,16 +41,193 @@ const recordMissingBillingConfigSkip = ();};-export const handleCloudSpendAlertJob = async (job: Job<{ orgId: string }>) => {- const { orgId } = job.data;+/**+ * What both providers have to produce for an alert to be evaluated: the spend+ * accrued so far and the start of the period it accrued in. The period start is+ * not decoration β it is what makes an alert fire once per cycle instead of on+ * every run, and what resets it when the next cycle opens.+ */+type SpendReading = {+ currentSpendUSD: number;+ currentPeriodStart: Date;+};- logger.info(`[CLOUD SPEND ALERTS] Processing org ${orgId}`);+type OrgWithAlerts = ReturnType<typeof parseDbOrg>;- if (!env.STRIPE_SECRET_KEY) {+/** Stripe's reading: the total of the subscription's preview invoice. */+const readStripeSpend = async (+ org: OrgWithAlerts,+): Promise<SpendReading | null> => {+ const stripeSecretKey = env.STRIPE_SECRET_KEY;+ if (!stripeSecretKey) {+ // A Stripe-billed org in a deployment without a Stripe key is a+ // misconfiguration, not a state to skip past: throw so the job retries and+ // the failure stays visible.logger.warn("[CLOUD SPEND ALERTS] Stripe secret key not found");throw new Error("Stripe secret key not found");}+ const stripeCustomerId = org.cloudConfig?.stripe?.customerId;+ if (!stripeCustomerId) {+ logger.warn(+ `[CLOUD SPEND ALERTS] Stripe customer id not found for org ${org.id}`,+ );+ recordMissingBillingConfigSkip("missing_stripe_customer_id");+ return null;+ }+ const stripeSubscriptionId = org.cloudConfig?.stripe?.activeSubscriptionId;+ if (!stripeSubscriptionId) {+ logger.warn(+ `[CLOUD SPEND ALERTS] Stripe subscription id not found for org ${org.id}`,+ );+ recordMissingBillingConfigSkip("missing_stripe_subscription_id");+ return null;+ }++ const stripe = new Stripe(stripeSecretKey);++ // Get subscription to check billing cycle+ const subscription =+ await stripe.subscriptions.retrieve(stripeSubscriptionId);++ // Create preview invoice to calculate current spend+ const canCreateInvoicePreview = [+ "active",+ "past_due",+ "trialing",+ "unpaid",+ ].includes(subscription.status);++ if (!canCreateInvoicePreview) {+ logger.warn(+ `[CLOUD SPEND ALERTS] Cannot create invoice preview for org ${org.id} - subscription status: ${subscription.status}`,+ );+ return null;+ }++ const previewInvoice = await backOff(+ async () =>+ await stripe.invoices.createPreview({+ customer:+ typeof subscription.customer === "string"+ ? subscription.customer+ : subscription.customer?.id,+ subscription: stripeSubscriptionId,+ }),+ {+ numOfAttempts: 3,+ },+ );++ return {+ // Stripe reports invoice totals in minor units.+ currentSpendUSD: (previewInvoice.total ?? 0) / 100,+ currentPeriodStart: new Date(subscription.current_period_start * 1000),+ };+};++/**+ * CHB's reading: the accrued usage on the org's open bill. This is the CHB+ * equivalent of Stripe's preview invoice β CHB issues no preview invoice, but+ * its billing warehouse reports what the open bill has accrued so far, in+ * major-unit USD (see `chbBillUsageAmountUSD`). The warehouse, not m3ter,+ * serves this read, which is what makes hourly polling affordable on CHB's+ * side; the price is a reading up to one warehouse export old.+ */+const readClickhouseSpend = async (+ org: OrgWithAlerts,+): Promise<SpendReading | null> => {+ const chOrganizationId = org.cloudConfig?.clickhouse?.organizationId;+ if (!chOrganizationId) {+ logger.warn(+ `[CLOUD SPEND ALERTS] ClickHouse organization id not found for org ${org.id}`,+ );+ recordMissingBillingConfigSkip("missing_chb_organization_id");+ return null;+ }+ if (!org.cloudConfig?.clickhouse?.attachedPlanId) {+ // Cancelled or never subscribed: nothing accrues, so nothing to alert on.+ logger.info(+ `[CLOUD SPEND ALERTS] Org ${org.id} has no CHB attached plan, skipping`,+ );+ recordMissingBillingConfigSkip("missing_chb_attached_plan");+ return null;+ }++ const client = getChbApiClient();+ if (!client) {+ logger.warn(+ `[CLOUD SPEND ALERTS] CHB REST client is not configured, cannot read spend for org ${org.id}`,+ );+ recordMissingBillingConfigSkip("chb_not_configured");+ return null;+ }++ const bills = await backOff(+ async () =>+ await client.queryBills({ chOrganizationIds: [chOrganizationId] }),+ {+ numOfAttempts: 3,+ // A 4xx is CHB's verdict on the request, not a blip: retrying it just+ // spends the budget. 5xx and transport failures still get the retries.+ retry: (error: unknown) =>+ !(+ error instanceof ChbApiError &&+ error.status >= 400 &&+ error.status < 500+ ),+ },+ );++ const bill = bills.find(+ (candidate) => candidate.organizationId === chOrganizationId,+ );+ if (!bill) {+ // CHB leaves an org without an open bill out of the result: its plan ended+ // after the fan-out read it, or it is newer than the warehouse's last+ // export. Either way nothing has accrued that we could compare.+ logger.info(+ `[CLOUD SPEND ALERTS] CHB reports no open bill for org ${org.id}, skipping`,+ );+ recordMissingBillingConfigSkip("chb_open_bill_not_found");+ return null;+ }++ // Logged raw as well as converted: this is the one place the major-unit+ // reading of CHB's amount is observable in production, so a contract change+ // shows up here rather than as alerts that quietly stop matching reality.+ logger.info(+ `[CLOUD SPEND ALERTS] Org ${org.id} CHB bill ${bill.id} reports accrued usage ${bill.usage?.amount} ${bill.usage?.currency ?? CHB_USAGE_CURRENCY} as of ${bill.usage?.asOf} for period ${bill.startDate} - ${bill.endDate}`,+ );++ const currentSpendUSD = chbBillUsageAmountUSD(bill);+ if (currentSpendUSD === null) {+ logger.warn(+ `[CLOUD SPEND ALERTS] CHB reported no usable ${CHB_USAGE_CURRENCY} usage amount for org ${org.id}, skipping`,+ );+ recordMissingBillingConfigSkip("missing_chb_usage_amount");+ return null;+ }++ const periodStart = bill.startDate ? new Date(bill.startDate) : null;+ if (!periodStart || Number.isNaN(periodStart.getTime())) {+ // Without a period start an alert cannot be held to once per cycle, and+ // every run past the threshold would email the org's admins again.+ logger.warn(+ `[CLOUD SPEND ALERTS] CHB reported no usable period start for org ${org.id}, skipping`,+ );+ recordMissingBillingConfigSkip("missing_chb_period_start");+ return null;+ }++ return { currentSpendUSD, currentPeriodStart: periodStart };+};++export const handleCloudSpendAlertJob = async (job: Job<{ orgId: string }>) => {+ const { orgId } = job.data;++ logger.info(`[CLOUD SPEND ALERTS] Processing org ${orgId}`);+// Fetch organization with spend alertsconst orgData = await prisma.organization.findFirst({where: {@@ -65,72 +262,26 @@ export const handleCloudSpendAlertJob = async (job: Job<{ orgId: string }>) => {return;}- // Get Stripe customer ID- const stripeCustomerId = org.cloudConfig?.stripe?.customerId;- if (!stripeCustomerId) {- logger.warn(- `[CLOUD SPEND ALERTS] Stripe customer id not found for org ${orgId}`,- );- recordMissingBillingConfigSkip("missing_stripe_customer_id");- return;- }- // Get Stripe subscription ID- const stripeSubscriptionId = org.cloudConfig?.stripe?.activeSubscriptionId;- if (!stripeSubscriptionId) {- logger.warn(- `[CLOUD SPEND ALERTS] Stripe subscription id not found for org ${orgId}`,- );- recordMissingBillingConfigSkip("missing_stripe_subscription_id");- return;- }-- // Setup stripe client- const stripe = new Stripe(env.STRIPE_SECRET_KEY);+ // Which provider bills this org decides where its accrued spend is read+ // from. A CHB org has no Stripe subscription to preview, and a Stripe org is+ // unknown to CHB, so this is a hard fork rather than a fallback chain.+ const billingProvider = getBillingProvider(org, {+ cutoff: env.LANGFUSE_CLOUD_BILLING_CHB_CUTOFF_DATE,+ });try {- // Get subscription to check billing cycle- const subscription =- await stripe.subscriptions.retrieve(stripeSubscriptionId);-- const currentPeriodStart = new Date(- subscription.current_period_start * 1000,- );+ const spend =+ billingProvider === "clickhouse"+ ? await readClickhouseSpend(org)+ : await readStripeSpend(org);- // Create preview invoice to calculate current spend- const canCreateInvoicePreview = [- "active",- "past_due",- "trialing",- "unpaid",- ].includes(subscription.status);-- if (!canCreateInvoicePreview) {- logger.warn(- `[CLOUD SPEND ALERTS] Cannot create invoice preview for org ${orgId} - subscription status: ${subscription.status}`,- );- return;- }-- const previewInvoice = await backOff(- async () =>- await stripe.invoices.createPreview({- customer:- typeof subscription.customer === "string"- ? subscription.customer- : subscription.customer?.id,- subscription: stripeSubscriptionId,- }),- {- numOfAttempts: 3,- },- );+ // The reader already logged why and counted the skip.+ if (!spend) return;- // Calculate current spend in USD- const currentSpendCents = previewInvoice.total ?? 0;- const currentSpendUSD = currentSpendCents / 100;+ const { currentSpendUSD, currentPeriodStart } = spend;logger.info(- `[CLOUD SPEND ALERTS] Org ${orgId} current spend: $${currentSpendUSD.toFixed(2)}`,+ `[CLOUD SPEND ALERTS] Org ${orgId} current spend: $${currentSpendUSD.toFixed(2)} (provider: ${billingProvider})`,);// Get org admins and owners for email notifications (fetch once for all alerts)@@ -212,6 +363,7 @@ export const handleCloudSpendAlertJob = async (job: Job<{ orgId: string }>) => {1,{unit: "alerts",+ billing_provider: billingProvider,},);} else {@@ -242,6 +394,7 @@ export const handleCloudSpendAlertJob = async (job: Job<{ orgId: string }>) => {1,{unit: "organizations",+ billing_provider: billingProvider,},);diff --git a/worker/src/env.ts b/worker/src/env.tsindex a23b80adcc..ea09e5f1dc 100644--- a/worker/src/env.ts+++ b/worker/src/env.ts@@ -44,6 +44,16 @@ const EnvSchema = z.object({.optional().transform((date) => (date ? new Date(date) : null)),+ // CHB REST credentials, mirroring web/src/env.mjs. The worker needs them for+ // one read: the spend-alert job asks for the open period's accrued usage.+ CLICKHOUSE_BILLING_BASE_URL: z.url().optional(),+ CLICKHOUSE_BILLING_AUTH0_DOMAIN: z.string().optional(),+ CLICKHOUSE_BILLING_AUTH0_CLIENT_ID: z.string().optional(),+ CLICKHOUSE_BILLING_AUTH0_CLIENT_SECRET: z.string().optional(),+ // CHB's resource-server identifier. Defaulted because it is the same value+ // in every CHB tenant, and overridable in case that stops being true.+ CLICKHOUSE_BILLING_AUTH0_AUDIENCE: z.string().default("billing-api"),+LANGFUSE_CACHE_AUTOMATIONS_ENABLED: z.enum(["true", "false"]).default("true"),LANGFUSE_CACHE_AUTOMATIONS_TTL_SECONDS: z.coerce.number().default(60),LANGFUSE_S3_BATCH_EXPORT_ENABLED: z.enum(["true", "false"]).default("false"),diff --git a/worker/src/queues/cloudSpendAlertQueue.ts b/worker/src/queues/cloudSpendAlertQueue.tsindex 7971144d0f..3bc5d704b9 100644--- a/worker/src/queues/cloudSpendAlertQueue.ts+++ b/worker/src/queues/cloudSpendAlertQueue.ts@@ -1,10 +1,15 @@import { Processor } from "bullmq";import { logger, QueueJobs } from "@langfuse/shared/src/server";import { handleCloudSpendAlertJob } from "../ee/cloudSpendAlerts/handleCloudSpendAlertJob";+import { handleCloudSpendAlertFanOutJob } from "../ee/cloudSpendAlerts/handleCloudSpendAlertFanOutJob";export const cloudSpendAlertQueueProcessor: Processor = async (job) => {if (job.name === QueueJobs.CloudSpendAlertJob) {logger.info("Executing Cloud Spend Alert Job", job.data);return await handleCloudSpendAlertJob(job);}+ if (job.name === QueueJobs.CloudSpendAlertFanOutJob) {+ logger.info("Executing Cloud Spend Alert Fan Out Job");+ return await handleCloudSpendAlertFanOutJob();+ }};