🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(billing): CHB inbound webhook processor (#15463)
Steffen Reuther56b53659a504
diff --git a/.env.dev.example b/.env.dev.exampleindex 1087c02f1f..a63880163a 100644--- a/.env.dev.example+++ b/.env.dev.example@@ -350,3 +350,5 @@ CLICKHOUSE_ENABLE_SKIP_INDEXES_ON_DATA_READ=auto# CLICKHOUSE_BILLING_AUTH0_CLIENT_SECRET=# CHB's resource-server identifier; defaults to billing-api.# CLICKHOUSE_BILLING_AUTH0_AUDIENCE=+# HMAC secret for inbound CHB webhooks; unset = the endpoint 500s.+# CLICKHOUSE_BILLING_WEBHOOK_SIGNING_SECRET=diff --git a/.env.prod.example b/.env.prod.exampleindex 9400f5d93d..946e180156 100644--- a/.env.prod.example+++ b/.env.prod.example@@ -496,6 +496,8 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true# CLICKHOUSE_BILLING_AUTH0_CLIENT_SECRET=# CHB's resource-server identifier; defaults to billing-api.# CLICKHOUSE_BILLING_AUTH0_AUDIENCE=+# HMAC secret for inbound CHB webhooks; unset = the endpoint 500s.+# CLICKHOUSE_BILLING_WEBHOOK_SIGNING_SECRET=# Rust AI gateway. Use the trusted HTTPS Web URL including any NEXT_PUBLIC_BASE_PATH.# Without a Web URL the gateway starts, but readiness and inference return 503.diff --git a/packages/shared/src/interfaces/cloudConfigSchema.ts b/packages/shared/src/interfaces/cloudConfigSchema.tsindex 7cd5756a73..972c3a39c9 100644--- a/packages/shared/src/interfaces/cloudConfigSchema.ts+++ b/packages/shared/src/interfaces/cloudConfigSchema.ts@@ -53,13 +53,14 @@ export const CloudConfigSchema = z.object({// attachedplan.* events). Support tooling only — routing, plan resolution, and// the worker jobs never read it.stripeCustomerId: z.string().nullish(),- // Snapshot of a pending scheduled change (downgrade/cancel) for the UI.+ // Snapshot of a pending scheduled change (upgrade/downgrade/cancel).scheduled: z.object({type: z.string(), // "upgrade" | "downgrade" | "cancel"when: z.string(), // "immediate" | "billing_cycle_end" | ISO date- planCode: z.string().nullish(),- startDate: z.string().nullish(),+ planCode: z.string().nullish(), // upgrade / downgrade target+ startDate: z.string().nullish(), // upgrade / downgrade effective date+ endDate: z.string().nullish(), // cancel: when the plan ends}).nullish(),// Monotonic guard against out-of-order webhook delivery.diff --git a/web/src/__tests__/server/unit/chbWebhookHandler.servertest.ts b/web/src/__tests__/server/unit/chbWebhookHandler.servertest.tsnew file mode 100644index 0000000000..9a3ec9fe38--- /dev/null+++ b/web/src/__tests__/server/unit/chbWebhookHandler.servertest.ts@@ -0,0 +1,512 @@+import crypto from "crypto";++import { NextRequest } from "next/server";+import type * as SharedServer from "@langfuse/shared/src/server";+import { beforeEach, describe, expect, it, vi } from "vitest";++const mocks = vi.hoisted(() => ({+ env: {+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "eu" as string | undefined,+ CLICKHOUSE_BILLING_WEBHOOK_SIGNING_SECRET: "test-signing-secret" as+ | string+ | undefined,+ },+ findOrg: vi.fn(),+ updateOrg: vi.fn(),+ findProjects: vi.fn(),+ redisSet: vi.fn(),+ redisDel: vi.fn(),+ invalidateCachedOrgApiKeys: vi.fn(),+ traceException: vi.fn(),+ recordIncrement: vi.fn(),+ auditLog: vi.fn(),+ sendChbProjectEvent: vi.fn(),+}));++vi.mock("@/src/env.mjs", () => ({ env: mocks.env }));++vi.mock("@langfuse/shared/src/db", () => ({+ prisma: {+ organization: { findFirst: mocks.findOrg, update: mocks.updateOrg },+ project: { findMany: mocks.findProjects },+ },+}));++vi.mock("@langfuse/shared/src/server", async (importOriginal) => {+ const actual = await importOriginal<typeof SharedServer>();++ return {+ ...actual,+ // status "end" keeps the shared teardown hook from trying to disconnect+ // this stand-in.+ redis: { set: mocks.redisSet, del: mocks.redisDel, status: "end" },+ invalidateCachedOrgApiKeys: mocks.invalidateCachedOrgApiKeys,+ traceException: mocks.traceException,+ recordIncrement: mocks.recordIncrement,+ };+});++vi.mock("@/src/features/audit-logs/auditLog", () => ({+ auditLog: mocks.auditLog,+}));++vi.mock("@/src/ee/features/billing/server/chb/chbProjectEvents", () => ({+ sendChbProjectEvent: mocks.sendChbProjectEvent,+}));++import {+ chbWebhookHandler,+ verifyChbSignature,+} from "@/src/ee/features/billing/server/chb/chbWebhookHandler";+import { logger } from "@langfuse/shared/src/server";++vi.spyOn(logger, "error").mockImplementation((() => {}) as never);+vi.spyOn(logger, "warn").mockImplementation((() => {}) as never);+vi.spyOn(logger, "info").mockImplementation((() => {}) as never);++const SECRET = "test-signing-secret";++const hmacHex = (rawBody: string, timestamp: string, secret: string = SECRET) =>+ crypto+ .createHmac("sha256", secret)+ .update(`${timestamp}.${rawBody}`)+ .digest("hex");++/** `X-CHB-Signature` as the control-plane dispatcher emits it: `t=` first, then+ * one `v1=` per signing key it currently holds (two during a rotation). */+const signatureHeader = (+ rawBody: string,+ timestamp: string,+ secrets: string[] = [SECRET],+) =>+ [+ `t=${timestamp}`,+ ...secrets.map((secret) => `v1=${hmacHex(rawBody, timestamp, secret)}`),+ ].join(",");++describe("verifyChbSignature", () => {+ const nowMs = 1_753_200_000_000; // fixed reference time+ const timestamp = String(Math.floor(nowMs / 1000));+ const rawBody = JSON.stringify({+ eventId: "evt_1",+ type: "attachedplan.created",+ });++ const verify = (header: string | null, body: string = rawBody) =>+ verifyChbSignature({+ rawBody: body,+ signatureHeader: header,+ secret: SECRET,+ nowMs,+ });++ it("accepts a valid signature within the skew window", () => {+ expect(verify(signatureHeader(rawBody, timestamp))).toEqual({+ valid: true,+ });+ });++ it("accepts the request when any v1 signature matches during a key rotation", () => {+ // The dispatcher signs with every key it holds; we hold only one of them.+ expect(+ verify(signatureHeader(rawBody, timestamp, ["retired-secret", SECRET])),+ ).toEqual({ valid: true });+ expect(+ verify(signatureHeader(rawBody, timestamp, [SECRET, "next-secret"])),+ ).toEqual({ valid: true });+ });++ it("rejects when none of several v1 signatures matches", () => {+ const result = verify(+ signatureHeader(rawBody, timestamp, ["retired-secret", "next-secret"]),+ );+ expect(result).toEqual({ valid: false, reason: "signature mismatch" });+ });++ it("rejects a tampered body", () => {+ const result = verify(+ signatureHeader(rawBody, timestamp),+ rawBody + "tampered",+ );+ expect(result).toEqual({ valid: false, reason: "signature mismatch" });+ });++ it("rejects a signature made with the wrong secret", () => {+ const result = verify(+ signatureHeader(rawBody, timestamp, ["other-secret"]),+ );+ expect(result).toEqual({ valid: false, reason: "signature mismatch" });+ });++ it("rejects a timestamp outside the 5 minute skew window", () => {+ const stale = String(Math.floor(nowMs / 1000) - 6 * 60);+ const result = verify(signatureHeader(rawBody, stale));+ expect(result).toEqual({+ valid: false,+ reason: "timestamp outside allowed clock skew",+ });+ });++ it("rejects a signature computed over a different timestamp than the header carries", () => {+ const signedOver = String(Math.floor(nowMs / 1000) - 30);+ const header = `t=${timestamp},v1=${hmacHex(rawBody, signedOver)}`;+ expect(verify(header)).toEqual({+ valid: false,+ reason: "signature mismatch",+ });+ });++ it("rejects a missing header", () => {+ expect(verify(null)).toEqual({+ valid: false,+ reason: "missing signature header",+ });+ });++ it.each([+ ["no t= element", `v1=${hmacHex(rawBody, timestamp)}`],+ ["no v1= element", `t=${timestamp}`],+ ["a bare signature", hmacHex(rawBody, timestamp)],+ ["an empty value", ""],+ ])("rejects a header with %s", (_label, header) => {+ expect(verify(header)).toEqual({+ valid: false,+ reason: "malformed signature header",+ });+ });++ it("rejects a malformed timestamp", () => {+ const result = verify(signatureHeader(rawBody, "not-a-number"));+ expect(result).toEqual({ valid: false, reason: "malformed timestamp" });+ });+});++describe("chbWebhookHandler", () => {+ const CHB_ORG_ID = "3f7c1b0a-2d5e-4c8b-9a1f-8e6d4c2b1a09";+ const ORG_ID = "org-1";++ const orgRow = (clickhouse: Record<string, unknown>, stripe?: unknown) => ({+ id: ORG_ID,+ name: "Org",+ cloudConfig: { clickhouse, ...(stripe ? { stripe } : {}) },+ });++ const post = (event: Record<string, unknown>) => {+ const rawBody = JSON.stringify(event);+ const timestamp = String(Math.floor(Date.now() / 1000));+ return new NextRequest("http://localhost/api/billing/clickhouse-webhook", {+ method: "POST",+ body: rawBody,+ headers: { "x-chb-signature": signatureHeader(rawBody, timestamp) },+ });+ };++ const attachedPlanCreated = (payment?: Record<string, unknown>) => ({+ eventId: "evt_1",+ type: "attachedplan.created",+ occurredAt: "2026-07-01T00:00:00Z",+ data: {+ organizationId: CHB_ORG_ID,+ id: "plan_1",+ planCode: "LANGFUSE_PRO",+ startDate: "2026-07-01T00:00:00Z",+ ...(payment ? { payment } : {}),+ },+ });++ beforeEach(() => {+ vi.clearAllMocks();+ mocks.env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "eu";+ mocks.env.CLICKHOUSE_BILLING_WEBHOOK_SIGNING_SECRET = SECRET;+ mocks.findOrg.mockResolvedValue(orgRow({ organizationId: CHB_ORG_ID }));+ mocks.updateOrg.mockResolvedValue({});+ mocks.findProjects.mockResolvedValue([]);+ // ioredis SET NX returns "OK" on a fresh key, null when the key existed.+ mocks.redisSet.mockResolvedValue("OK");+ mocks.redisDel.mockResolvedValue(1);+ mocks.sendChbProjectEvent.mockResolvedValue(undefined);+ // clearAllMocks keeps mockRejectedValue implementations, so every mock a+ // test rejects has to be pinned back here or it leaks into the next test.+ mocks.invalidateCachedOrgApiKeys.mockResolvedValue(undefined);+ mocks.auditLog.mockResolvedValue(undefined);+ });++ const orgColumnsOfUpdate = () =>+ mocks.updateOrg.mock.calls[0]?.[0]?.data ?? {};++ it("un-suspends on attachedplan.created only once payment is active", async () => {+ const response = await chbWebhookHandler(+ post(attachedPlanCreated({ status: "active" })),+ );++ expect(response.status).toBe(200);+ expect(orgColumnsOfUpdate()).toMatchObject({+ cloudFreeTierUsageThresholdState: null,+ });+ });++ it.each([+ ["a pending initial payment", { status: "pending" }],+ ["no payment block at all", undefined],+ ])(+ "leaves the free-tier suspension in place on attachedplan.created with %s",+ async (_label, payment) => {+ // A failed or pending first payment must not un-block ingestion for an org+ // suspended at the free-tier limit; attachedplan.updated lifts it when the+ // payment goes active.+ const response = await chbWebhookHandler(+ post(attachedPlanCreated(payment)),+ );++ expect(response.status).toBe(200);+ expect(orgColumnsOfUpdate()).not.toHaveProperty(+ "cloudFreeTierUsageThresholdState",+ );+ // The plan itself is still recorded, anchor included.+ expect(orgColumnsOfUpdate()).toHaveProperty("cloudBillingCycleAnchor");+ },+ );++ it("releases the dedupe claim when applying the event fails", async () => {+ mocks.updateOrg.mockRejectedValue(new Error("could not serialize access"));++ const response = await chbWebhookHandler(+ post(attachedPlanCreated({ status: "active" })),+ );++ // Without the release, CHB's retry hits the claim this request took and is+ // dropped as a duplicate — the event is lost for the whole 24h TTL.+ expect(response.status).toBe(500);+ expect(mocks.redisDel).toHaveBeenCalledWith("chb-webhook-event:evt_1");+ expect(await response.json()).toEqual({+ message: "Failed to apply webhook event",+ });+ });++ it("keeps the dedupe claim when the event applied cleanly", async () => {+ await chbWebhookHandler(post(attachedPlanCreated({ status: "active" })));++ expect(mocks.redisDel).not.toHaveBeenCalled();+ });++ it("refuses to apply CHB state onto an org with a live Stripe subscription", async () => {+ mocks.findOrg.mockResolvedValue(+ orgRow(+ { organizationId: CHB_ORG_ID },+ { customerId: "cus_1", activeSubscriptionId: "sub_1" },+ ),+ );++ const response = await chbWebhookHandler(+ post(attachedPlanCreated({ status: "active" })),+ );++ // Double-billed org: applying more CHB state would bury the contradiction.+ expect(response.status).toBe(200);+ expect(mocks.updateOrg).not.toHaveBeenCalled();+ expect(mocks.traceException).toHaveBeenCalled();+ });++ it("skips an event whose id was already claimed", async () => {+ mocks.redisSet.mockResolvedValue(null);++ const response = await chbWebhookHandler(+ post(attachedPlanCreated({ status: "active" })),+ );++ expect(response.status).toBe(200);+ expect(mocks.findOrg).not.toHaveBeenCalled();+ expect(mocks.updateOrg).not.toHaveBeenCalled();+ });++ it("ignores an org that belongs to another region", async () => {+ mocks.findOrg.mockResolvedValue(null);++ const response = await chbWebhookHandler(+ post(attachedPlanCreated({ status: "active" })),+ );++ expect(response.status).toBe(200);+ expect(mocks.updateOrg).not.toHaveBeenCalled();+ });++ it("rejects a request without the signature header", async () => {+ const response = await chbWebhookHandler(+ new NextRequest("http://localhost/api/billing/clickhouse-webhook", {+ method: "POST",+ body: JSON.stringify(attachedPlanCreated({ status: "active" })),+ }),+ );++ expect(response.status).toBe(400);+ expect(await response.json()).toEqual({+ message: "Webhook error: missing signature header",+ });+ // Nothing downstream runs on an unverified body, not even the dedupe claim.+ expect(mocks.redisSet).not.toHaveBeenCalled();+ });++ it("rejects an envelope without data.organizationId", async () => {+ const event = attachedPlanCreated({ status: "active" });+ const response = await chbWebhookHandler(+ post({ ...event, data: { ...event.data, organizationId: undefined } }),+ );++ expect(response.status).toBe(400);+ expect(await response.json()).toEqual({+ message: "Webhook error: invalid payload",+ });+ expect(mocks.findOrg).not.toHaveBeenCalled();+ });++ it("persists data.organizationId and occurredAt on the clickhouse block", async () => {+ await chbWebhookHandler(post(attachedPlanCreated({ status: "active" })));++ expect(orgColumnsOfUpdate().cloudConfig).toMatchObject({+ clickhouse: {+ organizationId: CHB_ORG_ID,+ attachedPlanId: "plan_1",+ planCode: "LANGFUSE_PRO",+ lastEventCreatedAt: "2026-07-01T00:00:00Z",+ },+ });+ });++ it("drops an event that occurred at or before the last applied one", async () => {+ mocks.findOrg.mockResolvedValue(+ orgRow({+ organizationId: CHB_ORG_ID,+ lastEventCreatedAt: "2026-07-01T00:00:00Z",+ }),+ );++ const response = await chbWebhookHandler(+ post(attachedPlanCreated({ status: "active" })),+ );++ expect(response.status).toBe(200);+ expect(mocks.updateOrg).not.toHaveBeenCalled();+ });++ it("keeps a failed project backfill out of the retry path", async () => {+ mocks.findProjects.mockRejectedValue(new Error("connection reset"));++ const response = await chbWebhookHandler(+ post(attachedPlanCreated({ status: "active" })),+ );++ // The plan is already persisted, so the ordering guard would drop a+ // retry as already applied; a 500 here only asks CHB for a retry that can+ // never reach the backfill. Log and page instead.+ expect(response.status).toBe(200);+ expect(mocks.updateOrg).toHaveBeenCalledTimes(1);+ expect(mocks.redisDel).not.toHaveBeenCalled();+ expect(mocks.traceException).toHaveBeenCalled();+ });++ it("keeps a failed API-key cache invalidation out of the retry path", async () => {+ mocks.invalidateCachedOrgApiKeys.mockRejectedValue(+ new Error("connection reset"),+ );++ const response = await chbWebhookHandler(+ post(attachedPlanCreated({ status: "active" })),+ );++ // Same commit-point rule; the stale cache entry expires with its TTL.+ expect(response.status).toBe(200);+ expect(mocks.redisDel).not.toHaveBeenCalled();+ expect(mocks.traceException).toHaveBeenCalled();+ });++ it("keeps a failed audit log write out of the retry path", async () => {+ mocks.auditLog.mockRejectedValue(new Error("connection reset"));++ const response = await chbWebhookHandler(+ post(attachedPlanCreated({ status: "active" })),+ );++ // Post-commit like the cache invalidation: the state change is applied,+ // so the missing audit row is logged and paged rather than retried.+ expect(response.status).toBe(200);+ expect(mocks.auditLog).toHaveBeenCalledTimes(1);+ expect(mocks.redisDel).not.toHaveBeenCalled();+ expect(mocks.traceException).toHaveBeenCalledWith(+ expect.objectContaining({ message: "connection reset" }),+ );+ });++ it("maps payment.dueDate and the provider customer onto the stored block", async () => {+ await chbWebhookHandler(+ post(+ attachedPlanCreated({+ status: "active",+ dueDate: "2026-08-01T00:00:00Z",+ provider: { name: "stripe", customerId: "cus_1" },+ }),+ ),+ );++ expect(orgColumnsOfUpdate().cloudConfig).toMatchObject({+ clickhouse: {+ paymentStatus: "active",+ nextPaymentDate: "2026-08-01T00:00:00Z",+ stripeCustomerId: "cus_1",+ },+ });+ });++ it("refuses an unknown plan code and keeps the event retryable", async () => {+ const event = attachedPlanCreated({ status: "active" });+ const response = await chbWebhookHandler(+ post({ ...event, data: { ...event.data, planCode: "LANGFUSE_ULTRA" } }),+ );++ // Storing the code would null the plan and drop a paying org to hobby.+ // The 500 keeps the event in the dispatcher's retry budget until the+ // mapping ships, so the dedupe claim has to go back.+ expect(response.status).toBe(500);+ expect(mocks.updateOrg).not.toHaveBeenCalled();+ expect(mocks.redisDel).toHaveBeenCalledWith("chb-webhook-event:evt_1");+ expect(mocks.traceException).toHaveBeenCalled();+ });++ const scheduledEvent = (scheduled: Record<string, unknown>) => ({+ eventId: "evt_2",+ type: "attachedplan.scheduled",+ occurredAt: "2026-07-02T00:00:00Z",+ data: { organizationId: CHB_ORG_ID, id: "plan_1", scheduled },+ });++ it("persists a scheduled cancellation with its end date", async () => {+ mocks.findOrg.mockResolvedValue(+ orgRow({ organizationId: CHB_ORG_ID, attachedPlanId: "plan_1" }),+ );++ const response = await chbWebhookHandler(+ post(+ scheduledEvent({+ type: "cancel",+ when: "billing_cycle_end",+ endDate: "2026-08-01T00:00:00Z",+ }),+ ),+ );++ expect(response.status).toBe(200);+ expect(orgColumnsOfUpdate().cloudConfig.clickhouse.scheduled).toEqual({+ type: "cancel",+ when: "billing_cycle_end",+ endDate: "2026-08-01T00:00:00Z",+ });+ });++ it("rejects a scheduled change of an unknown type", async () => {+ const response = await chbWebhookHandler(+ post(scheduledEvent({ type: "pause", when: "immediate" })),+ );++ expect(response.status).toBe(400);+ expect(mocks.updateOrg).not.toHaveBeenCalled();+ });+});diff --git a/web/src/app/api/billing/clickhouse-webhook/route.ts b/web/src/app/api/billing/clickhouse-webhook/route.tsnew file mode 100644index 0000000000..3d1e353b35--- /dev/null+++ b/web/src/app/api/billing/clickhouse-webhook/route.ts@@ -0,0 +1,5 @@+import { chbWebhookHandler } from "@/src/ee/features/billing/server/chb/chbWebhookHandler";++export const dynamic = "force-dynamic";++export const POST = chbWebhookHandler;diff --git a/web/src/ee/features/billing/server/chb/chbWebhookHandler.ts b/web/src/ee/features/billing/server/chb/chbWebhookHandler.tsnew file mode 100644index 0000000000..a99afc7222--- /dev/null+++ b/web/src/ee/features/billing/server/chb/chbWebhookHandler.ts@@ -0,0 +1,657 @@+import crypto from "crypto";++import { type NextRequest, NextResponse } from "next/server";+import { z } from "zod";++import { env } from "@/src/env.mjs";+import { auditLog } from "@/src/features/audit-logs/auditLog";+import {+ chbPlanCodes,+ CloudConfigSchema,+ parseDbOrg,+ type ParsedOrganization,+} from "@langfuse/shared";+import { prisma } from "@langfuse/shared/src/db";+import {+ invalidateCachedOrgApiKeys,+ logger,+ recordIncrement,+ redis,+ startOfDayUTC,+ traceException,+} from "@langfuse/shared/src/server";++import { sendChbProjectEvent } from "./chbProjectEvents";++/**+ * ClickHouse Billing webhook handler — structural twin of+ * stripeWebhookHandler. CHB is the source of truth for attached-plan state;+ * this handler is the single writer of `cloudConfig.clickhouse` (plus the+ * checkout-session write of `organizationId`).+ *+ * Pipeline: verify HMAC → dedupe → resolve org (region fan-out) → ordering+ * guard → per-event effect → invalidateCachedOrgApiKeys + auditLog.+ */++const CHB_SIGNATURE_HEADER = "x-chb-signature";+const MAX_CLOCK_SKEW_SECONDS = 5 * 60;+const DEDUPE_TTL_SECONDS = 24 * 60 * 60;++// Envelope as the ClickHouse control plane's signed-webhook dispatcher emits+// it: `{eventId, type, occurredAt, data}`. `data` is the attached-plan payload+// of the CHB webhook contract and names the owning ClickHouse organization.+// The body is signed as-is, so this schema only ever sees bytes that already+// passed verification. Objects stay permissive about unknown fields so CHB can+// extend the payload without a deploy here.++// A pending change on the attached plan. `when` is "immediate",+// "billing_cycle_end", or an ISO date. The plan itself flips only when the+// terminal attachedplan.updated / attachedplan.cancelled event lands.+const ChbWebhookScheduledChangeSchema = z.discriminatedUnion("type", [+ z.object({+ type: z.literal("cancel"),+ when: z.string(),+ endDate: z.string().nullish(),+ }),+ z.object({+ type: z.literal("downgrade"),+ when: z.string(),+ planCode: z.string(),+ startDate: z.string().nullish(),+ }),+ z.object({+ type: z.literal("upgrade"),+ when: z.string(),+ planCode: z.string(),+ startDate: z.string().nullish(),+ }),+]);++const ChbWebhookEventSchema = z.object({+ eventId: z.string().min(1),+ type: z.string(),+ occurredAt: z.iso.datetime({ offset: true }),+ data: z.object({+ // ClickHouse Organization ID owning the attached plan+ organizationId: z.uuid(),+ // Attached plan id, stored as `clickhouse.attachedPlanId`+ id: z.string().nullish(),+ planCode: z.string().nullish(),+ startDate: z.string().nullish(),+ payment: z+ .object({+ status: z.string().nullish(),+ dueDate: z.string().nullish(),+ provider: z+ .object({+ name: z.string().nullish(),+ customerId: z.string().nullish(),+ })+ .nullish(),+ })+ .nullish(),+ scheduled: ChbWebhookScheduledChangeSchema.nullish(),+ }),+});+export type ChbWebhookEvent = z.infer<typeof ChbWebhookEventSchema>;++/**+ * `X-CHB-Signature: t=<unix seconds>,v1=<hex>[,v1=<hex>...]`, as emitted by+ * the control plane's signed-webhook dispatcher. The timestamp stays a string+ * because it is signed byte for byte. Several `v1` values appear while CHB+ * rotates its signing key: the dispatcher signs with every key it holds, and a+ * receiver accepts the request if any of them matches the key it holds.+ */+function parseChbSignatureHeader(+ header: string,+): { timestamp: string; signatures: string[] } | null {+ let timestamp: string | undefined;+ const signatures: string[] = [];++ for (const part of header.split(",")) {+ const separator = part.indexOf("=");+ if (separator === -1) continue;+ const key = part.slice(0, separator);+ const value = part.slice(separator + 1);+ if (key === "t") timestamp = value;+ else if (key === "v1") signatures.push(value);+ }++ if (timestamp === undefined || signatures.length === 0) return null;+ return { timestamp, signatures };+}++/** Constant-time compare of two hex digests. `timingSafeEqual` throws on a+ * length mismatch and the candidate is attacker-controlled; the length is not+ * secret, so checking it first is fine. */+function digestsMatch(expectedHex: string, candidateHex: string): boolean {+ const expected = Buffer.from(expectedHex, "hex");+ const candidate = Buffer.from(candidateHex, "hex");+ return (+ expected.length === candidate.length &&+ crypto.timingSafeEqual(expected, candidate)+ );+}++/**+ * Verify the CHB webhook signature: HMAC-SHA256 over `${t}.${rawBody}`, hex+ * encoded, compared in constant time, with a ±5 minute skew window on `t`.+ * The dispatcher re-signs on every delivery attempt, so the short window stays+ * compatible with its multi-day retry budget. One local secret is enough for a+ * rotation: while CHB signs with both keys, any `v1` matching ours passes.+ */+export function verifyChbSignature(params: {+ rawBody: string | Buffer;+ signatureHeader: string | null;+ secret: string;+ nowMs?: number;+}): { valid: boolean; reason?: string } {+ const { rawBody, signatureHeader, secret } = params;+ if (signatureHeader === null) {+ return { valid: false, reason: "missing signature header" };+ }+ const parsed = parseChbSignatureHeader(signatureHeader);+ if (!parsed) {+ return { valid: false, reason: "malformed signature header" };+ }+ if (!/^\d+$/.test(parsed.timestamp)) {+ return { valid: false, reason: "malformed timestamp" };+ }++ const nowSeconds = Math.floor((params.nowMs ?? Date.now()) / 1000);+ if (+ Math.abs(nowSeconds - Number(parsed.timestamp)) > MAX_CLOCK_SKEW_SECONDS+ ) {+ return { valid: false, reason: "timestamp outside allowed clock skew" };+ }++ const expected = crypto+ .createHmac("sha256", secret)+ .update(`${parsed.timestamp}.`)+ .update(rawBody)+ .digest("hex");+ const matches = parsed.signatures.some((candidate) =>+ digestsMatch(expected, candidate),+ );+ return matches+ ? { valid: true }+ : { valid: false, reason: "signature mismatch" };+}++async function getOrgByChbOrganizationId(+ chbOrganizationId: string,+): Promise<ParsedOrganization | null> {+ const org = await prisma.organization.findFirst({+ where: {+ cloudConfig: {+ path: ["clickhouse", "organizationId"],+ equals: chbOrganizationId,+ },+ },+ });+ return org ? parseDbOrg(org) : null;+}++const dedupeKey = (eventId: string) => `chb-webhook-event:${eventId}`;++/** Best-effort Redis dedupe; every handler is also idempotent, so a Redis+ * flush cannot corrupt state. Returns true when the event was seen before.+ *+ * Claiming the id is the same operation as checking it, so a claim that is not+ * followed by a successful apply has to be released — see releaseEventClaim. */+async function isDuplicateEvent(eventId: string): Promise<boolean> {+ try {+ const result = await redis?.set(+ dedupeKey(eventId),+ "1",+ "EX",+ DEDUPE_TTL_SECONDS,+ "NX",+ );+ // ioredis returns null when NX prevented the write → key existed+ return redis ? result === null : false;+ } catch (error) {+ logger.warn("[CHB Webhook] Redis dedupe check failed, processing event", {+ eventId,+ error,+ });+ return false;+ }+}++/**+ * Drop the dedupe claim for an event that failed to apply. Without this, the+ * 500 we return sends CHB into a retry that its own claim now suppresses as a+ * duplicate, and the event is lost for the whole 24h TTL — long after CHB has+ * stopped retrying. Losing one means a missing cloudConfig.clickhouse block, a+ * still-suspended paying org, or an unmetered project, all needing manual+ * reconciliation.+ */+async function releaseEventClaim(eventId: string): Promise<void> {+ try {+ await redis?.del(dedupeKey(eventId));+ } catch (error) {+ // Nothing better to do than page: the event is now un-retryable.+ logger.error(+ `[CHB Webhook] Failed to release the dedupe claim for event ${eventId}; a CHB retry will be dropped as a duplicate`,+ error,+ );+ traceException(error);+ }+}++export async function chbWebhookHandler(req: NextRequest) {+ if (req.method !== "POST") {+ return NextResponse.json(+ { message: "Method not allowed" },+ { status: 405 },+ );+ }++ if (!env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION) {+ logger.error("[CHB Webhook] Endpoint only available in Langfuse Cloud");+ return NextResponse.json(+ { message: "CHB webhook endpoint only available in Langfuse Cloud" },+ { status: 500 },+ );+ }+ if (!env.CLICKHOUSE_BILLING_WEBHOOK_SIGNING_SECRET) {+ logger.error(+ "[CHB Webhook] CLICKHOUSE_BILLING_WEBHOOK_SIGNING_SECRET is not configured",+ );+ return NextResponse.json(+ { message: "CHB webhook signing secret not found" },+ { status: 500 },+ );+ }++ // The bytes as received: the signature covers the body exactly as CHB+ // serialized it, so nothing may decode or re-encode it before verification.+ const rawBody = Buffer.from(await req.arrayBuffer());+ const verification = verifyChbSignature({+ rawBody,+ signatureHeader: req.headers.get(CHB_SIGNATURE_HEADER),+ secret: env.CLICKHOUSE_BILLING_WEBHOOK_SIGNING_SECRET,+ });+ if (!verification.valid) {+ logger.error(+ `[CHB Webhook] Signature verification failed: ${verification.reason}`,+ );+ return NextResponse.json(+ { message: `Webhook error: ${verification.reason}` },+ { status: 400 },+ );+ }++ let event: ChbWebhookEvent;+ try {+ event = ChbWebhookEventSchema.parse(JSON.parse(rawBody.toString("utf8")));+ } catch (error) {+ logger.error("[CHB Webhook] Failed to parse event payload", error);+ return NextResponse.json(+ { message: "Webhook error: invalid payload" },+ { status: 400 },+ );+ }++ logger.info(`[CHB Webhook] Start ${event.type}`, { payload: event });++ if (await isDuplicateEvent(event.eventId)) {+ logger.info(+ `[CHB Webhook] Duplicate event ${event.eventId} (${event.type}), skipping`,+ );+ return NextResponse.json({ received: true }, { status: 200 });+ }++ // Region fan-out: CHB pings all Langfuse regions; exactly one owns the org.+ const parsedOrg = await getOrgByChbOrganizationId(event.data.organizationId);+ if (!parsedOrg) {+ logger.info(+ `[CHB Webhook] No org for CHB organization ${event.data.organizationId} in this region, ignoring`,+ );+ return NextResponse.json({ received: true }, { status: 200 });+ }++ // Interlock (must never happen): an org bills through exactly one provider.+ // Checking getBillingProvider here would be dead code -- it returns+ // "clickhouse" for anything carrying a clickhouse.organizationId, which is+ // precisely how this org was just resolved. The state that actually needs+ // catching is the contradiction: a live Stripe subscription alongside CHB+ // state means the customer is being billed twice, and applying more CHB state+ // would bury that. Refuse and page instead; a retry cannot fix it.+ if (parsedOrg.cloudConfig?.stripe?.activeSubscriptionId) {+ const message = `[CHB Webhook] Org ${parsedOrg.id} carries an active Stripe subscription alongside CHB state, refusing to apply ${event.type}`;+ logger.error(message);+ traceException(message);+ return NextResponse.json({ received: true }, { status: 200 });+ }++ // Ordering guard: drop events at or before the last applied one (protects+ // against retries and out-of-order delivery).+ const lastEventCreatedAt =+ parsedOrg.cloudConfig?.clickhouse?.lastEventCreatedAt;+ if (+ lastEventCreatedAt &&+ Date.parse(event.occurredAt) <= Date.parse(lastEventCreatedAt)+ ) {+ logger.info(+ `[CHB Webhook] Out-of-order event ${event.eventId} (${event.type}) for org ${parsedOrg.id}, skipping`,+ { occurredAt: event.occurredAt, lastEventCreatedAt },+ );+ return NextResponse.json({ received: true }, { status: 200 });+ }++ // An unknown plan code means CHB shipped a tier before its mapping is+ // deployed here. Storing it would null the plan and resolve a paying org to+ // cloud:hobby, so refuse and page instead. The 500 keeps the event inside+ // the dispatcher's retry budget: once the mapping ships, the retry applies it.+ if (+ event.data.planCode != null &&+ !(chbPlanCodes as readonly string[]).includes(event.data.planCode)+ ) {+ await releaseEventClaim(event.eventId);+ const message = `[CHB Webhook] Unknown plan code ${event.data.planCode} on ${event.type} for org ${parsedOrg.id}, refusing until a mapping ships`;+ logger.error(message);+ traceException(message);+ return NextResponse.json(+ { message: "Webhook error: unknown plan code" },+ { status: 500 },+ );+ }++ try {+ switch (event.type) {+ case "attachedplan.created":+ await handleAttachedPlanCreated(parsedOrg, event);+ break;+ case "attachedplan.updated":+ await handleAttachedPlanUpdated(parsedOrg, event);+ break;+ case "attachedplan.scheduled":+ await handleAttachedPlanScheduled(parsedOrg, event);+ break;+ case "attachedplan.cancelled":+ await handleAttachedPlanCancelled(parsedOrg, event);+ break;+ default:+ logger.warn(`[CHB Webhook] Unhandled event type ${event.type}`);+ }+ } catch (error) {+ // The dedupe claim was taken before dispatch, so it has to go back before we+ // ask CHB to retry -- otherwise the retry is dropped as a duplicate and the+ // event is lost for good.+ await releaseEventClaim(event.eventId);+ logger.error(+ `[CHB Webhook] Failed to apply ${event.type} (${event.eventId}) for org ${parsedOrg.id}`,+ error,+ );+ traceException(error);+ return NextResponse.json(+ { message: "Failed to apply webhook event" },+ { status: 500 },+ );+ }++ return NextResponse.json({ received: true }, { status: 200 });+}++/**+ * Persist a new clickhouse block + org columns, then propagate: the resolved+ * plan and suspension flag are baked into the Redis-cached API-key record, so+ * every write ends with a cache invalidation.+ *+ * The organization update is the commit point. It persists this event's+ * occurredAt as lastEventCreatedAt, after which the ordering guard drops any+ * retry of the same event as already applied. Nothing that runs after it may+ * fail the request: a 500 would ask CHB for a retry that can never reach that+ * code again. Post-commit steps -- here and in the per-event handlers -- are+ * best-effort: log, page, and let the request succeed.+ */+async function persistAndPropagate(params: {+ parsedOrg: ParsedOrganization;+ event: ChbWebhookEvent;+ clickhouse: unknown;+ orgColumns?: {+ cloudBillingCycleAnchor?: Date;+ cloudFreeTierUsageThresholdState?: null;+ };+}) {+ const { parsedOrg, event } = params;++ // Validate through the stored schema so a bad write can never poison+ // parseDbOrg for the whole cloudConfig.+ const updatedCloudConfig = {+ ...parsedOrg.cloudConfig,+ clickhouse: CloudConfigSchema.shape.clickhouse.parse(params.clickhouse),+ };++ await prisma.organization.update({+ where: { id: parsedOrg.id },+ data: {+ cloudConfig: updatedCloudConfig,+ ...(params.orgColumns ?? {}),+ },+ });++ // Post-commit. Bounded: a stale cached plan expires with the API-key cache+ // TTL.+ try {+ await invalidateCachedOrgApiKeys(parsedOrg.id);+ } catch (error) {+ logger.error(+ `[CHB Webhook] Failed to invalidate cached API keys for org ${parsedOrg.id} after applying ${event.type}`,+ error,+ );+ traceException(error);+ }++ // Post-commit as well: the state change stands, a missing audit row is+ // surfaced rather than retried.+ try {+ await auditLog({+ session: {+ user: { id: "clickhouse-webhook" },+ orgId: parsedOrg.id,+ },+ orgId: parsedOrg.id,+ resourceType: "organization",+ resourceId: parsedOrg.id,+ action: `BillingService.chb.${event.type}`,+ before: parsedOrg.cloudConfig,+ after: updatedCloudConfig,+ });+ } catch (error) {+ logger.error(+ `[CHB Webhook] Failed to write the audit log for ${event.type} on org ${parsedOrg.id}`,+ error,+ );+ traceException(error);+ }++ return updatedCloudConfig;+}++async function handleAttachedPlanCreated(+ parsedOrg: ParsedOrganization,+ event: ChbWebhookEvent,+) {+ const { data } = event;+ const existing = parsedOrg.cloudConfig?.clickhouse;++ if (!data.id) {+ logger.error(+ `[CHB Webhook] attachedplan.created without an attached plan id for org ${parsedOrg.id}, skipping`,+ );+ traceException(+ `[CHB Webhook] attachedplan.created without an attached plan id for org ${parsedOrg.id}`,+ );+ return;+ }++ // Same gate as attachedplan.updated and the Stripe path: only clear the+ // free-tier suspension once payment is credibly current. A plan whose initial+ // payment is pending or failed must not un-block ingestion for an org that+ // was suspended at the free-tier limit -- attachedplan.updated lifts it when+ // the payment goes active.+ const isPaidAndCurrent = data.payment?.status === "active";++ await persistAndPropagate({+ parsedOrg,+ event,+ // Fresh block: a re-subscription must not inherit scheduled state from a+ // previously cancelled plan.+ clickhouse: {+ organizationId: event.data.organizationId,+ attachedPlanId: data.id,+ planCode: data.planCode,+ paymentStatus: data.payment?.status,+ nextPaymentDate: data.payment?.dueDate,+ stripeCustomerId:+ data.payment?.provider?.customerId ?? existing?.stripeCustomerId,+ lastEventCreatedAt: event.occurredAt,+ },+ orgColumns: {+ // First paid subscription anchors the billing cycle on the plan start+ cloudBillingCycleAnchor: data.startDate+ ? new Date(data.startDate)+ : startOfDayUTC(new Date()),+ ...(isPaidAndCurrent ? { cloudFreeTierUsageThresholdState: null } : {}),+ },+ });++ // No default spend alerts for CHB orgs. The spend-alert job computes current+ // spend from the org's own Stripe subscription, which a CHB org does not+ // have, so seeded alerts would show in billing settings and never fire.++ // Backfill-emit LANGFUSE_PROJECT_CREATED for all existing projects: projects+ // created before checkout would otherwise be invisible to CHB metering.+ // Post-commit and best-effort -- CHB's own backfill pipeline is the backstop.+ try {+ await backfillProjectEvents(parsedOrg, event.data.organizationId);+ } catch (error) {+ logger.error(+ `[CHB Webhook] Project backfill failed for org ${parsedOrg.id}`,+ error,+ );+ traceException(error);+ }+}++async function backfillProjectEvents(+ parsedOrg: ParsedOrganization,+ chbOrganizationId: string,+) {+ const projects = await prisma.project.findMany({+ where: { orgId: parsedOrg.id, deletedAt: null },+ select: { id: true },+ });+ const results = await Promise.allSettled(+ projects.map((project) =>+ sendChbProjectEvent({+ type: "LANGFUSE_PROJECT_CREATED",+ chbOrganizationId,+ projectId: project.id,+ }),+ ),+ );+ const failed = results.filter((result) => result.status === "rejected");+ if (failed.length > 0) {+ recordIncrement("langfuse.billing_events.emit_failed", failed.length, {+ unit: "events",+ });+ logger.error(+ `[CHB Webhook] Project backfill emit failed for ${failed.length}/${projects.length} projects of org ${parsedOrg.id}`,+ );+ }+}++async function handleAttachedPlanUpdated(+ parsedOrg: ParsedOrganization,+ event: ChbWebhookEvent,+) {+ const { data } = event;+ const existing = parsedOrg.cloudConfig?.clickhouse;++ if (!existing?.attachedPlanId && !data.id) {+ logger.error(+ `[CHB Webhook] attachedplan.updated for org ${parsedOrg.id} without any attached plan id, skipping`,+ );+ return;+ }++ // Mirror of the Stripe active|trialing logic: only clear the free-tier+ // suspension when payment is credibly current.+ const isPaidAndCurrent = data.payment?.status === "active";++ await persistAndPropagate({+ parsedOrg,+ event,+ // Merge semantics: absent fields keep their stored value; explicit null+ // clears (e.g. scheduled: null after a scheduled change executed).+ clickhouse: {+ ...existing,+ organizationId: event.data.organizationId,+ ...(data.id !== undefined ? { attachedPlanId: data.id } : {}),+ ...(data.planCode !== undefined ? { planCode: data.planCode } : {}),+ ...(data.payment?.status !== undefined+ ? { paymentStatus: data.payment.status }+ : {}),+ ...(data.payment?.dueDate !== undefined+ ? { nextPaymentDate: data.payment.dueDate }+ : {}),+ ...(data.payment?.provider?.customerId+ ? { stripeCustomerId: data.payment.provider.customerId }+ : {}),+ ...(data.scheduled !== undefined ? { scheduled: data.scheduled } : {}),+ lastEventCreatedAt: event.occurredAt,+ },+ orgColumns: isPaidAndCurrent+ ? { cloudFreeTierUsageThresholdState: null }+ : undefined,+ });+}++async function handleAttachedPlanScheduled(+ parsedOrg: ParsedOrganization,+ event: ChbWebhookEvent,+) {+ const existing = parsedOrg.cloudConfig?.clickhouse;++ // Snapshot only — no plan change. The plan flips when attachedplan.updated or+ // attachedplan.cancelled lands. We never execute scheduled changes locally on+ // a timer: CHB owns the terminal event, and a local timer would race it.+ await persistAndPropagate({+ parsedOrg,+ event,+ clickhouse: {+ ...existing,+ organizationId: event.data.organizationId,+ scheduled: event.data.scheduled ?? null,+ lastEventCreatedAt: event.occurredAt,+ },+ });+}++async function handleAttachedPlanCancelled(+ parsedOrg: ParsedOrganization,+ event: ChbWebhookEvent,+) {+ const existing = parsedOrg.cloudConfig?.clickhouse;++ await persistAndPropagate({+ parsedOrg,+ event,+ // Keep organizationId (the customer and CH org survive cancellation) and+ // stripeCustomerId (support tooling); drop the plan so the org resolves+ // back to cloud:hobby — same semantics as Stripe subscription.deleted.+ clickhouse: {+ organizationId: event.data.organizationId,+ stripeCustomerId: existing?.stripeCustomerId,+ lastEventCreatedAt: event.occurredAt,+ },+ orgColumns: {+ // Reset billing cycle anchor on downgrade to hobby to start of today+ cloudBillingCycleAnchor: startOfDayUTC(new Date()),+ },+ });+}diff --git a/web/src/env.mjs b/web/src/env.mjsindex eac294591b..2fdfbc26d9 100644--- a/web/src/env.mjs+++ b/web/src/env.mjs@@ -487,6 +487,9 @@ export const env = createEnv({// CHB's resource-server identifier. Defaulted because it is the same value// in every CHB tenant, and overridable in case that stops being true.CLICKHOUSE_BILLING_AUTH0_AUDIENCE: z.string().default("billing-api"),+ // HMAC secret for inbound CHB webhooks. Unset makes the endpoint 500 rather+ // than accept unverified events.+ CLICKHOUSE_BILLING_WEBHOOK_SIGNING_SECRET: z.string().optional(),SENTRY_AUTH_TOKEN: z.string().optional(),SENTRY_CSP_REPORT_URI: z.string().optional(),LANGFUSE_RATE_LIMITS_ENABLED: z.enum(["true", "false"]).default("true"),@@ -1111,6 +1114,8 @@ export const env = createEnv({process.env.CLICKHOUSE_BILLING_AUTH0_CLIENT_SECRET,CLICKHOUSE_BILLING_AUTH0_AUDIENCE:process.env.CLICKHOUSE_BILLING_AUTH0_AUDIENCE,+ CLICKHOUSE_BILLING_WEBHOOK_SIGNING_SECRET:+ process.env.CLICKHOUSE_BILLING_WEBHOOK_SIGNING_SECRET,SENTRY_AUTH_TOKEN: process.env.SENTRY_AUTH_TOKEN,SENTRY_CSP_REPORT_URI: process.env.SENTRY_CSP_REPORT_URI,LANGFUSE_RATE_LIMITS_ENABLED: process.env.LANGFUSE_RATE_LIMITS_ENABLED,