🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
chore: adjust CHB event envelope and debug log incoming hooks (#17543)
Steffen Reuther53640f4ca74b
diff --git a/packages/shared/src/interfaces/cloudConfigSchema.ts b/packages/shared/src/interfaces/cloudConfigSchema.tsindex 972c3a39c9..f152da7d40 100644--- a/packages/shared/src/interfaces/cloudConfigSchema.ts+++ b/packages/shared/src/interfaces/cloudConfigSchema.ts@@ -53,11 +53,13 @@ export const CloudConfigSchema = z.object({// attachedplan.* events). Support tooling only — routing, plan resolution, and// the worker jobs never read it.stripeCustomerId: z.string().nullish(),- // Snapshot of a pending scheduled change (upgrade/downgrade/cancel).+ // Snapshot of a pending scheduled change (upgrade/downgrade/cancel), as+ // GET /attachedplan reports it.scheduled: z.object({type: z.string(), // "upgrade" | "downgrade" | "cancel"- when: z.string(), // "immediate" | "billing_cycle_end" | ISO date+ // "immediate" | "billing_cycle_end" | ISO date, when CHB supplies one+ when: z.string().nullish(),planCode: z.string().nullish(), // upgrade / downgrade targetstartDate: z.string().nullish(), // upgrade / downgrade effective dateendDate: z.string().nullish(), // cancel: when the plan endsdiff --git a/web/src/__tests__/server/unit/chb-project-events.servertest.ts b/web/src/__tests__/server/unit/chb-project-events.servertest.tsindex d47a0eb4c3..4dd127c89d 100644--- a/web/src/__tests__/server/unit/chb-project-events.servertest.ts+++ b/web/src/__tests__/server/unit/chb-project-events.servertest.ts@@ -118,18 +118,25 @@ describe("chbProjectEvents", () => {expect(entry.Source).toBe("langfuse");// The event carries CHB's organization id, not ours -- CHB's registry is// keyed by its own org id.+ // Detail mirrors CHB's own bus record: an envelope with the event body+ // nested under `payload`.const detail = JSON.parse(entry.Detail);expect(detail).toMatchObject({- type: "LANGFUSE_PROJECT_CREATED",- organizationId: CHB_ORG_ID,- projectId: PROJECT_ID,- // Where this deployment runs: the provider region it lives in plus our- // own cell name, lowercased.- cloudProvider: "aws",- region: "eu-west-1",- cell: "eu",+ id: expect.any(String),+ source: "langfuse",+ timestamp: expect.any(Number),+ payload: {+ eventType: "LANGFUSE_PROJECT_CREATED",+ organizationId: CHB_ORG_ID,+ projectId: PROJECT_ID,+ // Where this deployment runs: the provider region it lives in plus our+ // own cell name, lowercased.+ cloudProvider: "aws",+ region: "eu-west-1",+ cell: "eu",+ },});- expect(detail).not.toHaveProperty("regionId");+ expect(detail.payload).not.toHaveProperty("regionId");});// CHB locates a deployment by provider region plus cell, not by our cell@@ -150,7 +157,7 @@ describe("chbProjectEvents", () => {projectId: PROJECT_ID,});- expect(publishedDetails()[0]).toMatchObject({+ expect(publishedDetails()[0].payload).toMatchObject({cloudProvider: "aws",region: awsRegion,cell,@@ -274,14 +281,13 @@ describe("chbProjectEvents", () => {where: { orgId: ORG_ID, deletedAt: null },select: { id: true },});- expect(publishedDetails().map((detail) => detail.projectId)).toEqual([- "project-a",- "project-b",- ]);+ expect(+ publishedDetails().map((detail) => detail.payload.projectId),+ ).toEqual(["project-a", "project-b"]);// Batched: both projects ride one PutEvents call, not one call each.expect(mocks.send).toHaveBeenCalledTimes(1);expect(publishedEntries()[0].DetailType).toBe("LANGFUSE_PROJECT_CREATED");- expect(publishedDetails()[0]).toMatchObject({+ expect(publishedDetails()[0].payload).toMatchObject({organizationId: CHB_ORG_ID,});});@@ -316,7 +322,7 @@ describe("chbProjectEvents", () => {mocks.send.mockImplementation((command) => {const results = command.input.Entries.map((entry: { Detail: string }) =>- JSON.parse(entry.Detail).projectId === "project-a"+ JSON.parse(entry.Detail).payload.projectId === "project-a"? { ErrorCode: "InternalException" }: { EventId: "ok" },);@@ -338,7 +344,8 @@ describe("chbProjectEvents", () => {.slice(1).flatMap(([command]) => command.input.Entries).map(- (entry: { Detail: string }) => JSON.parse(entry.Detail).projectId,+ (entry: { Detail: string }) =>+ JSON.parse(entry.Detail).payload.projectId,),).toEqual(["project-a", "project-a"]);diff --git a/web/src/__tests__/server/unit/chbWebhookHandler.servertest.ts b/web/src/__tests__/server/unit/chbWebhookHandler.servertest.tsindex 9a3ec9fe38..d5eb6e55db 100644--- a/web/src/__tests__/server/unit/chbWebhookHandler.servertest.ts+++ b/web/src/__tests__/server/unit/chbWebhookHandler.servertest.ts@@ -21,6 +21,8 @@ const mocks = vi.hoisted(() => ({recordIncrement: vi.fn(),auditLog: vi.fn(),sendChbProjectEvent: vi.fn(),+ getChbApiClient: vi.fn(),+ getAttachedPlan: vi.fn(),}));vi.mock("@/src/env.mjs", () => ({ env: mocks.env }));@@ -54,6 +56,10 @@ vi.mock("@/src/ee/features/billing/server/chb/chbProjectEvents", () => ({sendChbProjectEvent: mocks.sendChbProjectEvent,}));+vi.mock("@/src/ee/features/billing/server/chb/chbApiClient", () => ({+ getChbApiClient: mocks.getChbApiClient,+}));+import {chbWebhookHandler,verifyChbSignature,@@ -89,7 +95,7 @@ describe("verifyChbSignature", () => {const timestamp = String(Math.floor(nowMs / 1000));const rawBody = JSON.stringify({eventId: "evt_1",- type: "attachedplan.created",+ type: "BILLING_ATTACHEDPLAN_CREATED",});const verify = (header: string | null, body: string = rawBody) =>@@ -201,17 +207,48 @@ describe("chbWebhookHandler", () => {});};- const attachedPlanCreated = (payment?: Record<string, unknown>) => ({+ // The body as the control-plane dispatcher delivers it: `data` is the+ // billing-api event bus record, and the organization lives in its payload.+ const attachedPlanCreated = (payload: Record<string, unknown> = {}) => ({eventId: "evt_1",- type: "attachedplan.created",+ type: "BILLING_ATTACHEDPLAN_CREATED",occurredAt: "2026-07-01T00:00:00Z",data: {- organizationId: CHB_ORG_ID,- id: "plan_1",- planCode: "LANGFUSE_PRO",+ id: "0de85656-bdc7-47e3-a656-a2ae2ec91b41",+ source: "billing-api",+ timestamp: 1_782_000_001_000,+ version: 1,+ payload: {+ createdAt: 1_782_000_000_000,+ eventType: "BILLING_ATTACHEDPLAN_CREATED",+ organizationId: CHB_ORG_ID,+ planCode: "LANGFUSE_CORE",+ ...payload,+ },+ },+ });++ // Any other attached-plan event: same envelope, different type.+ const chbEvent = (type: string, overrides: Record<string, unknown> = {}) => ({+ ...attachedPlanCreated({ eventType: type }),+ type,+ ...overrides,+ });++ // GET /attachedplan as CHB answers it for the org named in the event.+ const attachedPlan = (overrides: Record<string, unknown> = {}) => ({+ id: "plan_1",+ plan: { code: "LANGFUSE_CORE" },+ period: {startDate: "2026-07-01T00:00:00Z",- ...(payment ? { payment } : {}),+ endDate: "2026-08-01T00:00:00Z",+ },+ payment: {+ status: "active",+ provider: { name: "stripe", customerId: "cus_1" },},+ scheduled: null,+ ...overrides,});beforeEach(() => {@@ -229,15 +266,17 @@ describe("chbWebhookHandler", () => {// test rejects has to be pinned back here or it leaks into the next test.mocks.invalidateCachedOrgApiKeys.mockResolvedValue(undefined);mocks.auditLog.mockResolvedValue(undefined);+ mocks.getChbApiClient.mockReturnValue({+ getAttachedPlan: mocks.getAttachedPlan,+ });+ mocks.getAttachedPlan.mockResolvedValue(attachedPlan());});const orgColumnsOfUpdate = () =>mocks.updateOrg.mock.calls[0]?.[0]?.data ?? {};- it("un-suspends on attachedplan.created only once payment is active", async () => {- const response = await chbWebhookHandler(- post(attachedPlanCreated({ status: "active" })),- );+ it("un-suspends on BILLING_ATTACHEDPLAN_CREATED only once payment is active", async () => {+ const response = await chbWebhookHandler(post(attachedPlanCreated()));expect(response.status).toBe(200);expect(orgColumnsOfUpdate()).toMatchObject({@@ -247,16 +286,15 @@ describe("chbWebhookHandler", () => {it.each([["a pending initial payment", { status: "pending" }],- ["no payment block at all", undefined],+ ["no payment block at all", null],])(- "leaves the free-tier suspension in place on attachedplan.created with %s",+ "leaves the free-tier suspension in place on BILLING_ATTACHEDPLAN_CREATED with %s",async (_label, payment) => {// A failed or pending first payment must not un-block ingestion for an org- // suspended at the free-tier limit; attachedplan.updated lifts it when the- // payment goes active.- const response = await chbWebhookHandler(- post(attachedPlanCreated(payment)),- );+ // suspended at the free-tier limit.+ mocks.getAttachedPlan.mockResolvedValue(attachedPlan({ payment }));++ const response = await chbWebhookHandler(post(attachedPlanCreated()));expect(response.status).toBe(200);expect(orgColumnsOfUpdate()).not.toHaveProperty(@@ -270,9 +308,7 @@ describe("chbWebhookHandler", () => {it("releases the dedupe claim when applying the event fails", async () => {mocks.updateOrg.mockRejectedValue(new Error("could not serialize access"));- const response = await chbWebhookHandler(- post(attachedPlanCreated({ status: "active" })),- );+ const response = await chbWebhookHandler(post(attachedPlanCreated()));// Without the release, CHB's retry hits the claim this request took and is// dropped as a duplicate — the event is lost for the whole 24h TTL.@@ -284,7 +320,7 @@ describe("chbWebhookHandler", () => {});it("keeps the dedupe claim when the event applied cleanly", async () => {- await chbWebhookHandler(post(attachedPlanCreated({ status: "active" })));+ await chbWebhookHandler(post(attachedPlanCreated()));expect(mocks.redisDel).not.toHaveBeenCalled();});@@ -297,9 +333,7 @@ describe("chbWebhookHandler", () => {),);- const response = await chbWebhookHandler(- post(attachedPlanCreated({ status: "active" })),- );+ const response = await chbWebhookHandler(post(attachedPlanCreated()));// Double-billed org: applying more CHB state would bury the contradiction.expect(response.status).toBe(200);@@ -310,9 +344,7 @@ describe("chbWebhookHandler", () => {it("skips an event whose id was already claimed", async () => {mocks.redisSet.mockResolvedValue(null);- const response = await chbWebhookHandler(- post(attachedPlanCreated({ status: "active" })),- );+ const response = await chbWebhookHandler(post(attachedPlanCreated()));expect(response.status).toBe(200);expect(mocks.findOrg).not.toHaveBeenCalled();@@ -322,9 +354,7 @@ describe("chbWebhookHandler", () => {it("ignores an org that belongs to another region", async () => {mocks.findOrg.mockResolvedValue(null);- const response = await chbWebhookHandler(- post(attachedPlanCreated({ status: "active" })),- );+ const response = await chbWebhookHandler(post(attachedPlanCreated()));expect(response.status).toBe(200);expect(mocks.updateOrg).not.toHaveBeenCalled();@@ -334,7 +364,7 @@ describe("chbWebhookHandler", () => {const response = await chbWebhookHandler(new NextRequest("http://localhost/api/billing/clickhouse-webhook", {method: "POST",- body: JSON.stringify(attachedPlanCreated({ status: "active" })),+ body: JSON.stringify(attachedPlanCreated()),}),);@@ -346,10 +376,9 @@ describe("chbWebhookHandler", () => {expect(mocks.redisSet).not.toHaveBeenCalled();});- it("rejects an envelope without data.organizationId", async () => {- const event = attachedPlanCreated({ status: "active" });+ it("rejects an envelope without data.payload.organizationId", async () => {const response = await chbWebhookHandler(- post({ ...event, data: { ...event.data, organizationId: undefined } }),+ post(attachedPlanCreated({ organizationId: undefined })),);expect(response.status).toBe(400);@@ -359,17 +388,39 @@ describe("chbWebhookHandler", () => {expect(mocks.findOrg).not.toHaveBeenCalled();});- it("persists data.organizationId and occurredAt on the clickhouse block", async () => {- await chbWebhookHandler(post(attachedPlanCreated({ status: "active" })));+ it("reads the attached plan back from CHB and persists it with the event's organization and occurredAt", async () => {+ await chbWebhookHandler(post(attachedPlanCreated()));+ // The event names the organization but not the plan; the plan id and code+ // come from GET /attachedplan for that organization.+ expect(mocks.getAttachedPlan).toHaveBeenCalledWith({+ chOrganizationId: CHB_ORG_ID,+ });expect(orgColumnsOfUpdate().cloudConfig).toMatchObject({clickhouse: {organizationId: CHB_ORG_ID,attachedPlanId: "plan_1",- planCode: "LANGFUSE_PRO",+ planCode: "LANGFUSE_CORE",lastEventCreatedAt: "2026-07-01T00:00:00Z",},});+ expect(orgColumnsOfUpdate().cloudBillingCycleAnchor).toEqual(+ new Date("2026-07-01T00:00:00Z"),+ );+ });++ it("keeps the event retryable when the attached plan cannot be read", async () => {+ mocks.getAttachedPlan.mockRejectedValue(+ new Error("CHB API GET attachedplan failed with status 404"),+ );++ const response = await chbWebhookHandler(post(attachedPlanCreated()));++ // Nothing was persisted, so the claim has to go back for CHB's retry to+ // reach the read again once the plan is there.+ expect(response.status).toBe(500);+ expect(mocks.updateOrg).not.toHaveBeenCalled();+ expect(mocks.redisDel).toHaveBeenCalledWith("chb-webhook-event:evt_1");});it("drops an event that occurred at or before the last applied one", async () => {@@ -380,9 +431,7 @@ describe("chbWebhookHandler", () => {}),);- const response = await chbWebhookHandler(- post(attachedPlanCreated({ status: "active" })),- );+ const response = await chbWebhookHandler(post(attachedPlanCreated()));expect(response.status).toBe(200);expect(mocks.updateOrg).not.toHaveBeenCalled();@@ -391,9 +440,7 @@ describe("chbWebhookHandler", () => {it("keeps a failed project backfill out of the retry path", async () => {mocks.findProjects.mockRejectedValue(new Error("connection reset"));- const response = await chbWebhookHandler(- post(attachedPlanCreated({ status: "active" })),- );+ const response = await chbWebhookHandler(post(attachedPlanCreated()));// The plan is already persisted, so the ordering guard would drop a// retry as already applied; a 500 here only asks CHB for a retry that can@@ -409,9 +456,7 @@ describe("chbWebhookHandler", () => {new Error("connection reset"),);- const response = await chbWebhookHandler(- post(attachedPlanCreated({ status: "active" })),- );+ const response = await chbWebhookHandler(post(attachedPlanCreated()));// Same commit-point rule; the stale cache entry expires with its TTL.expect(response.status).toBe(200);@@ -422,9 +467,7 @@ describe("chbWebhookHandler", () => {it("keeps a failed audit log write out of the retry path", async () => {mocks.auditLog.mockRejectedValue(new Error("connection reset"));- const response = await chbWebhookHandler(- post(attachedPlanCreated({ status: "active" })),- );+ const response = await chbWebhookHandler(post(attachedPlanCreated()));// Post-commit like the cache invalidation: the state change is applied,// so the missing audit row is logged and paged rather than retried.@@ -436,16 +479,8 @@ describe("chbWebhookHandler", () => {);});- it("maps payment.dueDate and the provider customer onto the stored block", async () => {- await chbWebhookHandler(- post(- attachedPlanCreated({- status: "active",- dueDate: "2026-08-01T00:00:00Z",- provider: { name: "stripe", customerId: "cus_1" },- }),- ),- );+ it("maps the attached plan's payment, period end and provider customer onto the stored block", async () => {+ await chbWebhookHandler(post(attachedPlanCreated()));expect(orgColumnsOfUpdate().cloudConfig).toMatchObject({clickhouse: {@@ -456,57 +491,145 @@ describe("chbWebhookHandler", () => {});});- it("refuses an unknown plan code and keeps the event retryable", async () => {- const event = attachedPlanCreated({ status: "active" });+ it("refuses an unknown plan code on the event and keeps it retryable", async () => {const response = await chbWebhookHandler(- post({ ...event, data: { ...event.data, planCode: "LANGFUSE_ULTRA" } }),+ post(attachedPlanCreated({ planCode: "LANGFUSE_ULTRA" })),);// Storing the code would null the plan and drop a paying org to hobby.// The 500 keeps the event in the dispatcher's retry budget until the// mapping ships, so the dedupe claim has to go back.expect(response.status).toBe(500);+ expect(mocks.getAttachedPlan).not.toHaveBeenCalled();expect(mocks.updateOrg).not.toHaveBeenCalled();expect(mocks.redisDel).toHaveBeenCalledWith("chb-webhook-event:evt_1");expect(mocks.traceException).toHaveBeenCalled();});- const scheduledEvent = (scheduled: Record<string, unknown>) => ({- eventId: "evt_2",- type: "attachedplan.scheduled",- occurredAt: "2026-07-02T00:00:00Z",- data: { organizationId: CHB_ORG_ID, id: "plan_1", scheduled },+ it("refuses an unknown plan code reported by GET /attachedplan the same way", async () => {+ mocks.getAttachedPlan.mockResolvedValue(+ attachedPlan({ plan: { code: "LANGFUSE_ULTRA" } }),+ );++ const response = await chbWebhookHandler(post(attachedPlanCreated()));++ expect(response.status).toBe(500);+ expect(mocks.updateOrg).not.toHaveBeenCalled();+ expect(mocks.redisDel).toHaveBeenCalledWith("chb-webhook-event:evt_1");+ });++ it("rejects an event type outside the CHB enum", async () => {+ const response = await chbWebhookHandler(+ post(chbEvent("BILLING_ATTACHEDPLAN_DELETED")),+ );++ expect(response.status).toBe(400);+ expect(await response.json()).toEqual({+ message: "Webhook error: invalid payload",+ });+ expect(mocks.findOrg).not.toHaveBeenCalled();});- it("persists a scheduled cancellation with its end date", async () => {+ it("re-reads the plan on BILLING_ATTACHEDPLAN_UPDATED and overwrites the snapshot without moving the billing cycle anchor", async () => {mocks.findOrg.mockResolvedValue(- orgRow({ organizationId: CHB_ORG_ID, attachedPlanId: "plan_1" }),+ orgRow({+ organizationId: CHB_ORG_ID,+ attachedPlanId: "plan_1",+ planCode: "LANGFUSE_CORE",+ lastEventCreatedAt: "2026-07-01T00:00:00Z",+ }),+ );+ mocks.getAttachedPlan.mockResolvedValue(+ attachedPlan({ plan: { code: "LANGFUSE_PRO" } }),);const response = await chbWebhookHandler(post(- scheduledEvent({- type: "cancel",- when: "billing_cycle_end",- endDate: "2026-08-01T00:00:00Z",+ chbEvent("BILLING_ATTACHEDPLAN_UPDATED", {+ eventId: "evt_2",+ occurredAt: "2026-07-15T00:00:00Z",}),),);expect(response.status).toBe(200);- expect(orgColumnsOfUpdate().cloudConfig.clickhouse.scheduled).toEqual({- type: "cancel",- when: "billing_cycle_end",- endDate: "2026-08-01T00:00:00Z",+ expect(orgColumnsOfUpdate().cloudConfig.clickhouse).toMatchObject({+ attachedPlanId: "plan_1",+ planCode: "LANGFUSE_PRO",+ lastEventCreatedAt: "2026-07-15T00:00:00Z",+ });+ expect(orgColumnsOfUpdate()).not.toHaveProperty("cloudBillingCycleAnchor");+ // Only a created plan backfills projects.+ expect(mocks.findProjects).not.toHaveBeenCalled();+ });++ it("snapshots the pending change on BILLING_ATTACHEDPLAN_SCHEDULED and leaves the plan itself alone", async () => {+ mocks.findOrg.mockResolvedValue(+ orgRow({+ organizationId: CHB_ORG_ID,+ attachedPlanId: "plan_1",+ planCode: "LANGFUSE_PRO",+ lastEventCreatedAt: "2026-07-01T00:00:00Z",+ }),+ );+ const scheduled = {+ type: "downgrade",+ planCode: "LANGFUSE_CORE",+ startDate: "2026-08-01T00:00:00Z",+ };+ mocks.getAttachedPlan.mockResolvedValue(+ attachedPlan({ plan: { code: "LANGFUSE_PRO" }, scheduled }),+ );++ const response = await chbWebhookHandler(+ post(+ chbEvent("BILLING_ATTACHEDPLAN_SCHEDULED", {+ eventId: "evt_4",+ occurredAt: "2026-07-20T00:00:00Z",+ }),+ ),+ );++ expect(response.status).toBe(200);+ // The plan flips when the terminal updated/cancelled event lands, not here;+ // CHB's REST view wins over the plan code the event announced.+ expect(orgColumnsOfUpdate().cloudConfig.clickhouse).toMatchObject({+ planCode: "LANGFUSE_PRO",+ scheduled,+ lastEventCreatedAt: "2026-07-20T00:00:00Z",});});- it("rejects a scheduled change of an unknown type", async () => {+ it("drops the plan on BILLING_ATTACHEDPLAN_CANCELLED without reading it back", async () => {+ mocks.findOrg.mockResolvedValue(+ orgRow({+ organizationId: CHB_ORG_ID,+ attachedPlanId: "plan_1",+ planCode: "LANGFUSE_CORE",+ paymentStatus: "active",+ stripeCustomerId: "cus_1",+ lastEventCreatedAt: "2026-07-01T00:00:00Z",+ }),+ );+const response = await chbWebhookHandler(- post(scheduledEvent({ type: "pause", when: "immediate" })),+ post(+ chbEvent("BILLING_ATTACHEDPLAN_CANCELLED", {+ eventId: "evt_3",+ occurredAt: "2026-08-01T00:00:00Z",+ }),+ ),);- expect(response.status).toBe(400);- expect(mocks.updateOrg).not.toHaveBeenCalled();+ expect(response.status).toBe(200);+ // GET /attachedplan may already answer 404 for a cancelled organization.+ expect(mocks.getAttachedPlan).not.toHaveBeenCalled();+ // Only the customer identity survives; the org resolves back to hobby.+ expect(orgColumnsOfUpdate().cloudConfig.clickhouse).toEqual({+ organizationId: CHB_ORG_ID,+ stripeCustomerId: "cus_1",+ lastEventCreatedAt: "2026-08-01T00:00:00Z",+ });+ expect(orgColumnsOfUpdate()).toHaveProperty("cloudBillingCycleAnchor");});});diff --git a/web/src/ee/features/billing/server/chb/chbProjectEvents.ts b/web/src/ee/features/billing/server/chb/chbProjectEvents.tsindex a16e8bb826..05fa702bd1 100644--- a/web/src/ee/features/billing/server/chb/chbProjectEvents.ts+++ b/web/src/ee/features/billing/server/chb/chbProjectEvents.ts@@ -9,6 +9,7 @@ import { env } from "@/src/env.mjs";import { parseDbOrg } from "@langfuse/shared";import { prisma } from "@langfuse/shared/src/db";import { logger, recordIncrement } from "@langfuse/shared/src/server";+import { randomUUID } from "crypto";/*** Project lifecycle events for ClickHouse Billing (CHB).@@ -73,17 +74,20 @@ const buildChbProjectEventPayload = (params: {projectId: string;}) => {const cell = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;+ const timestamp = new Date();return {- type: params.type,- organizationId: params.chbOrganizationId,- projectId: params.projectId,- // CHB locates a deployment by provider region plus cell, not by our cell- // name alone: two cells can share one region (US and HIPAA both run in- // us-west-2), so the cell is what still tells them apart.- ...(cell ? CELL_LOCATION[cell] : null),- cell: cell?.toLowerCase(),- createdAt: new Date().toISOString(),+ id: randomUUID(),+ source: CHB_EVENT_SOURCE,+ timestamp: timestamp.getTime(),+ payload: {+ eventType: params.type,+ organizationId: params.chbOrganizationId,+ projectId: params.projectId,+ ...(cell ? CELL_LOCATION[cell] : null),+ cell: cell?.toLowerCase(),+ createdAt: timestamp.toISOString(),+ },};};@@ -127,21 +131,16 @@ const requireEventBusArn = (): string => {return eventBusArn;};-const buildChbEventEntry = (- eventBusArn: string,- params: {- type: ChbProjectEventType;- chbOrganizationId: string;- projectId: string;- },-) => ({+type ChbProjectEvent = ReturnType<typeof buildChbProjectEventPayload>;++const buildChbEventEntry = (eventBusArn: string, event: ChbProjectEvent) => ({EventBusName: eventBusArn,// CHB's bus policy whitelists the allowed event types by detail-type, so the// type has to travel here, not only in Detail — otherwise the bus rejects// the entry.- DetailType: params.type,+ DetailType: event.payload.eventType,Source: CHB_EVENT_SOURCE,- Detail: JSON.stringify(buildChbProjectEventPayload(params)),+ Detail: JSON.stringify(event),});/**@@ -212,7 +211,13 @@ export async function sendChbProjectEvent(params: {// segment is present.const region = eventBusArn.split(":")[3];- await putChbEventBatch(region, [buildChbEventEntry(eventBusArn, params)]);+ const event = buildChbProjectEventPayload(params);+ await putChbEventBatch(region, [buildChbEventEntry(eventBusArn, event)]);++ logger.info(+ `[CHB Project Events] Emitted ${params.type} for project ${params.projectId} (CHB org ${params.chbOrganizationId})`,+ { eventId: event.id },+ );}/**@@ -304,8 +309,8 @@ export async function backfillChbProjectEvents(params: {select: { id: true },});- const entries = projects.map((project) =>- buildChbEventEntry(eventBusArn, {+ const events = projects.map((project) =>+ buildChbProjectEventPayload({type: "LANGFUSE_PROJECT_CREATED",chbOrganizationId,projectId: project.id,@@ -314,17 +319,27 @@ export async function backfillChbProjectEvents(params: {for (let offset = 0;- offset < entries.length;+ offset < events.length;offset += EVENT_BUS_MAX_ENTRIES_PER_CALL) {- const batch = entries.slice(+ const batch = events.slice(offset,offset + EVENT_BUS_MAX_ENTRIES_PER_CALL,);try {- await putChbEventBatch(region, batch);+ await putChbEventBatch(+ region,+ batch.map((event) => buildChbEventEntry(eventBusArn, event)),+ );sent += batch.length;+ logger.info(+ `[CHB Project Events] Emitted ${batch.length} LANGFUSE_PROJECT_CREATED backfill events for org ${params.orgId} (CHB org ${chbOrganizationId})`,+ {+ projectIds: batch.map((event) => event.payload.projectId),+ eventIds: batch.map((event) => event.id),+ },+ );} catch (error) {// A rejection knows exactly how many entries the bus refused; anything// else (timeout, transport) means the whole batch is unaccounted for.diff --git a/web/src/ee/features/billing/server/chb/chbWebhookHandler.ts b/web/src/ee/features/billing/server/chb/chbWebhookHandler.tsindex a99afc7222..7810cd684f 100644--- a/web/src/ee/features/billing/server/chb/chbWebhookHandler.ts+++ b/web/src/ee/features/billing/server/chb/chbWebhookHandler.ts@@ -21,6 +21,7 @@ import {traceException,} from "@langfuse/shared/src/server";+import { type ChbAttachedPlan, getChbApiClient } from "./chbApiClient";import { sendChbProjectEvent } from "./chbProjectEvents";/**@@ -30,7 +31,8 @@ import { sendChbProjectEvent } from "./chbProjectEvents";* checkout-session write of `organizationId`).** Pipeline: verify HMAC → dedupe → resolve org (region fan-out) → ordering- * guard → per-event effect → invalidateCachedOrgApiKeys + auditLog.+ * guard → per-event effect (created/updated read the plan back from CHB) →+ * invalidateCachedOrgApiKeys + auditLog.*/const CHB_SIGNATURE_HEADER = "x-chb-signature";@@ -38,59 +40,47 @@ const MAX_CLOCK_SKEW_SECONDS = 5 * 60;const DEDUPE_TTL_SECONDS = 24 * 60 * 60;// Envelope as the ClickHouse control plane's signed-webhook dispatcher emits-// it: `{eventId, type, occurredAt, data}`. `data` is the attached-plan payload-// of the CHB webhook contract and names the owning ClickHouse organization.-// The body is signed as-is, so this schema only ever sees bytes that already-// passed verification. Objects stay permissive about unknown fields so CHB can-// extend the payload without a deploy here.--// A pending change on the attached plan. `when` is "immediate",-// "billing_cycle_end", or an ISO date. The plan itself flips only when the-// terminal attachedplan.updated / attachedplan.cancelled event lands.-const ChbWebhookScheduledChangeSchema = z.discriminatedUnion("type", [- z.object({- type: z.literal("cancel"),- when: z.string(),- endDate: z.string().nullish(),- }),- z.object({- type: z.literal("downgrade"),- when: z.string(),- planCode: z.string(),- startDate: z.string().nullish(),- }),- z.object({- type: z.literal("upgrade"),- when: z.string(),- planCode: z.string(),- startDate: z.string().nullish(),- }),+// it: `{eventId, type, occurredAt, data}`, where `data` is the billing-api+// event exactly as it travelled over CHB's event bus — a+// `{id, source, timestamp, version, payload}` record whose `payload` names the+// owning ClickHouse organization. `type` repeats the bus detail-type. The+// body is signed as-is, so this schema only ever sees bytes that already+// passed verification. Objects stay permissive about unknown fields so CHB+// can extend the payload without a deploy here.+//+// The event is a notification, not a state snapshot: it carries the plan code+// but neither the attached plan id nor payment or billing-period details, so+// the handler reads the attached plan back from CHB's REST API on receipt.++// The detail-types CHB's billing-api emits for an attached plan. The switch+// in the handler is exhaustive over this list; a type outside it fails the+// schema and is answered 400.+const ChbWebhookEventTypeSchema = z.enum([+ "BILLING_ATTACHEDPLAN_CREATED",+ "BILLING_ATTACHEDPLAN_UPDATED",+ "BILLING_ATTACHEDPLAN_SCHEDULED",+ "BILLING_ATTACHEDPLAN_CANCELLED",]);const ChbWebhookEventSchema = z.object({eventId: z.string().min(1),- type: z.string(),+ type: ChbWebhookEventTypeSchema,occurredAt: z.iso.datetime({ offset: true }),data: z.object({- // ClickHouse Organization ID owning the attached plan- organizationId: z.uuid(),- // Attached plan id, stored as `clickhouse.attachedPlanId`+ // Event-bus record id, not the attached plan idid: z.string().nullish(),- planCode: z.string().nullish(),- startDate: z.string().nullish(),- payment: z- .object({- status: z.string().nullish(),- dueDate: z.string().nullish(),- provider: z- .object({- name: z.string().nullish(),- customerId: z.string().nullish(),- })- .nullish(),- })- .nullish(),- scheduled: ChbWebhookScheduledChangeSchema.nullish(),+ source: z.string().nullish(),+ // Unix milliseconds+ timestamp: z.number().nullish(),+ version: z.number().nullish(),+ payload: z.object({+ eventType: ChbWebhookEventTypeSchema,+ // ClickHouse Organization ID owning the attached plan+ organizationId: z.uuid(),+ planCode: z.string().nullish(),+ // Unix milliseconds+ createdAt: z.number().nullish(),+ }),}),});export type ChbWebhookEvent = z.infer<typeof ChbWebhookEventSchema>;@@ -304,10 +294,11 @@ export async function chbWebhookHandler(req: NextRequest) {}// Region fan-out: CHB pings all Langfuse regions; exactly one owns the org.- const parsedOrg = await getOrgByChbOrganizationId(event.data.organizationId);+ const chOrganizationId = event.data.payload.organizationId;+ const parsedOrg = await getOrgByChbOrganizationId(chOrganizationId);if (!parsedOrg) {logger.info(- `[CHB Webhook] No org for CHB organization ${event.data.organizationId} in this region, ignoring`,+ `[CHB Webhook] No org for CHB organization ${chOrganizationId} in this region, ignoring`,);return NextResponse.json({ received: true }, { status: 200 });}@@ -345,12 +336,13 @@ export async function chbWebhookHandler(req: NextRequest) {// deployed here. Storing it would null the plan and resolve a paying org to// cloud:hobby, so refuse and page instead. The 500 keeps the event inside// the dispatcher's retry budget: once the mapping ships, the retry applies it.+ const announcedPlanCode = event.data.payload.planCode;if (- event.data.planCode != null &&- !(chbPlanCodes as readonly string[]).includes(event.data.planCode)+ announcedPlanCode != null &&+ !(chbPlanCodes as readonly string[]).includes(announcedPlanCode)) {await releaseEventClaim(event.eventId);- const message = `[CHB Webhook] Unknown plan code ${event.data.planCode} on ${event.type} for org ${parsedOrg.id}, refusing until a mapping ships`;+ const message = `[CHB Webhook] Unknown plan code ${announcedPlanCode} on ${event.type} for org ${parsedOrg.id}, refusing until a mapping ships`;logger.error(message);traceException(message);return NextResponse.json(@@ -361,20 +353,20 @@ export async function chbWebhookHandler(req: NextRequest) {try {switch (event.type) {- case "attachedplan.created":+ case "BILLING_ATTACHEDPLAN_CREATED":await handleAttachedPlanCreated(parsedOrg, event);break;- case "attachedplan.updated":- await handleAttachedPlanUpdated(parsedOrg, event);- break;- case "attachedplan.scheduled":- await handleAttachedPlanScheduled(parsedOrg, event);+ case "BILLING_ATTACHEDPLAN_UPDATED":+ case "BILLING_ATTACHEDPLAN_SCHEDULED":+ await handleAttachedPlanChanged(parsedOrg, event);break;- case "attachedplan.cancelled":+ case "BILLING_ATTACHEDPLAN_CANCELLED":await handleAttachedPlanCancelled(parsedOrg, event);break;- default:- logger.warn(`[CHB Webhook] Unhandled event type ${event.type}`);+ default: {+ const exhaustiveCheck: never = event.type;+ throw new Error(`Unhandled CHB event type ${exhaustiveCheck}`);+ }}} catch (error) {// The dedupe claim was taken before dispatch, so it has to go back before we@@ -471,49 +463,97 @@ async function persistAndPropagate(params: {return updatedCloudConfig;}-async function handleAttachedPlanCreated(+/**+ * The attached plan behind an event. The event only announces it: the id,+ * payment state and billing period live on CHB's REST side, so the handler+ * reads them back. Throws when CHB is not configured or the read fails; the+ * caller's catch releases the dedupe claim and answers 500, so CHB retries+ * once the plan is readable.+ */+async function fetchAttachedPlan(+ chOrganizationId: string,+): Promise<ChbAttachedPlan> {+ const client = getChbApiClient();+ if (!client) {+ throw new Error(+ "CHB REST client is not configured, cannot read the attached plan behind the event",+ );+ }+ return await client.getAttachedPlan({ chOrganizationId });+}++/**+ * What an event implies for the stored block, read back from CHB. Created and+ * updated both start here: the event only announces the attached plan, its+ * id, payment state and billing period live on CHB's REST side.+ */+async function readAttachedPlanState(parsedOrg: ParsedOrganization,event: ChbWebhookEvent,) {- const { data } = event;+ const chOrganizationId = event.data.payload.organizationId;const existing = parsedOrg.cloudConfig?.clickhouse;- if (!data.id) {- logger.error(- `[CHB Webhook] attachedplan.created without an attached plan id for org ${parsedOrg.id}, skipping`,- );- traceException(- `[CHB Webhook] attachedplan.created without an attached plan id for org ${parsedOrg.id}`,+ const attachedPlan = await fetchAttachedPlan(chOrganizationId);++ // CHB's REST view is authoritative; the event's plan code only fills a gap.+ // The pipeline already refused an unknown code on the event itself, so a+ // rejection here means CHB reports a code that has no mapping yet. Same+ // answer: the 500 keeps the event retryable until the mapping ships.+ const planCode =+ attachedPlan.plan?.code ?? event.data.payload.planCode ?? null;+ if (+ planCode != null &&+ !(chbPlanCodes as readonly string[]).includes(planCode)+ ) {+ throw new Error(+ `Attached plan ${attachedPlan.id} for org ${parsedOrg.id} carries unknown plan code ${planCode}`,);- return;}- // Same gate as attachedplan.updated and the Stripe path: only clear the- // free-tier suspension once payment is credibly current. A plan whose initial- // payment is pending or failed must not un-block ingestion for an org that- // was suspended at the free-tier limit -- attachedplan.updated lifts it when- // the payment goes active.- const isPaidAndCurrent = data.payment?.status === "active";-- await persistAndPropagate({- parsedOrg,- event,- // Fresh block: a re-subscription must not inherit scheduled state from a- // previously cancelled plan.+ return {+ attachedPlan,+ // Same gate as the Stripe path: only clear the free-tier suspension once+ // payment is credibly current. A plan whose payment is pending or failed+ // must not un-block ingestion for an org that was suspended at the+ // free-tier limit; BILLING_ATTACHEDPLAN_UPDATED lifts it once the payment+ // goes active.+ isPaidAndCurrent: attachedPlan.payment?.status === "active",+ // A full snapshot, not a merge: CHB is the source of truth, so a+ // re-subscription cannot inherit state from a previously cancelled plan.clickhouse: {- organizationId: event.data.organizationId,- attachedPlanId: data.id,- planCode: data.planCode,- paymentStatus: data.payment?.status,- nextPaymentDate: data.payment?.dueDate,+ organizationId: chOrganizationId,+ attachedPlanId: attachedPlan.id,+ planCode,+ paymentStatus: attachedPlan.payment?.status,+ // The current period ends at the renewal, which is the next charge.+ nextPaymentDate: attachedPlan.period?.endDate,stripeCustomerId:- data.payment?.provider?.customerId ?? existing?.stripeCustomerId,+ attachedPlan.payment?.provider?.customerId ??+ existing?.stripeCustomerId,+ // Pending upgrade/downgrade/cancel as CHB reports it; null clears one+ // that executed or was withdrawn.+ scheduled: attachedPlan.scheduled ?? null,lastEventCreatedAt: event.occurredAt,},+ };+}++async function handleAttachedPlanCreated(+ parsedOrg: ParsedOrganization,+ event: ChbWebhookEvent,+) {+ const { attachedPlan, isPaidAndCurrent, clickhouse } =+ await readAttachedPlanState(parsedOrg, event);++ await persistAndPropagate({+ parsedOrg,+ event,+ clickhouse,orgColumns: {// First paid subscription anchors the billing cycle on the plan start- cloudBillingCycleAnchor: data.startDate- ? new Date(data.startDate)+ cloudBillingCycleAnchor: attachedPlan.period?.startDate+ ? new Date(attachedPlan.period.startDate): startOfDayUTC(new Date()),...(isPaidAndCurrent ? { cloudFreeTierUsageThresholdState: null } : {}),},@@ -527,7 +567,7 @@ async function handleAttachedPlanCreated(// created before checkout would otherwise be invisible to CHB metering.// Post-commit and best-effort -- CHB's own backfill pipeline is the backstop.try {- await backfillProjectEvents(parsedOrg, event.data.organizationId);+ await backfillProjectEvents(parsedOrg, clickhouse.organizationId);} catch (error) {logger.error(`[CHB Webhook] Project backfill failed for org ${parsedOrg.id}`,@@ -537,101 +577,40 @@ async function handleAttachedPlanCreated(}}-async function backfillProjectEvents(- parsedOrg: ParsedOrganization,- chbOrganizationId: string,-) {- const projects = await prisma.project.findMany({- where: { orgId: parsedOrg.id, deletedAt: null },- select: { id: true },- });- const results = await Promise.allSettled(- projects.map((project) =>- sendChbProjectEvent({- type: "LANGFUSE_PROJECT_CREATED",- chbOrganizationId,- projectId: project.id,- }),- ),- );- const failed = results.filter((result) => result.status === "rejected");- if (failed.length > 0) {- recordIncrement("langfuse.billing_events.emit_failed", failed.length, {- unit: "events",- });- logger.error(- `[CHB Webhook] Project backfill emit failed for ${failed.length}/${projects.length} projects of org ${parsedOrg.id}`,- );- }-}--async function handleAttachedPlanUpdated(+/**+ * BILLING_ATTACHEDPLAN_UPDATED and BILLING_ATTACHEDPLAN_SCHEDULED: plan,+ * payment or scheduled-change state moved on CHB's side. Both re-read and+ * overwrite the snapshot. A scheduled change is only a snapshot as well: the+ * plan itself flips when the terminal updated or cancelled event lands, and+ * we never execute scheduled changes locally on a timer -- CHB owns that+ * moment. The billing cycle anchor stays where the created event put it.+ */+async function handleAttachedPlanChanged(parsedOrg: ParsedOrganization,event: ChbWebhookEvent,) {- const { data } = event;- const existing = parsedOrg.cloudConfig?.clickhouse;-- if (!existing?.attachedPlanId && !data.id) {- logger.error(- `[CHB Webhook] attachedplan.updated for org ${parsedOrg.id} without any attached plan id, skipping`,- );- return;- }-- // Mirror of the Stripe active|trialing logic: only clear the free-tier- // suspension when payment is credibly current.- const isPaidAndCurrent = data.payment?.status === "active";+ const { isPaidAndCurrent, clickhouse } = await readAttachedPlanState(+ parsedOrg,+ event,+ );await persistAndPropagate({parsedOrg,event,- // Merge semantics: absent fields keep their stored value; explicit null- // clears (e.g. scheduled: null after a scheduled change executed).- clickhouse: {- ...existing,- organizationId: event.data.organizationId,- ...(data.id !== undefined ? { attachedPlanId: data.id } : {}),- ...(data.planCode !== undefined ? { planCode: data.planCode } : {}),- ...(data.payment?.status !== undefined- ? { paymentStatus: data.payment.status }- : {}),- ...(data.payment?.dueDate !== undefined- ? { nextPaymentDate: data.payment.dueDate }- : {}),- ...(data.payment?.provider?.customerId- ? { stripeCustomerId: data.payment.provider.customerId }- : {}),- ...(data.scheduled !== undefined ? { scheduled: data.scheduled } : {}),- lastEventCreatedAt: event.occurredAt,- },+ clickhouse,orgColumns: isPaidAndCurrent? { cloudFreeTierUsageThresholdState: null }: undefined,});}-async function handleAttachedPlanScheduled(- parsedOrg: ParsedOrganization,- event: ChbWebhookEvent,-) {- const existing = parsedOrg.cloudConfig?.clickhouse;-- // Snapshot only — no plan change. The plan flips when attachedplan.updated or- // attachedplan.cancelled lands. We never execute scheduled changes locally on- // a timer: CHB owns the terminal event, and a local timer would race it.- await persistAndPropagate({- parsedOrg,- event,- clickhouse: {- ...existing,- organizationId: event.data.organizationId,- scheduled: event.data.scheduled ?? null,- lastEventCreatedAt: event.occurredAt,- },- });-}-+/**+ * Terminal: the plan is gone, so there is nothing to read back -- CHB may+ * already answer 404 for the organization. Keep organizationId (the customer+ * and CH org survive cancellation) and stripeCustomerId (support tooling);+ * drop the plan so the org resolves back to cloud:hobby -- same semantics as+ * Stripe subscription.deleted.+ */async function handleAttachedPlanCancelled(parsedOrg: ParsedOrganization,event: ChbWebhookEvent,@@ -641,17 +620,42 @@ async function handleAttachedPlanCancelled(await persistAndPropagate({parsedOrg,event,- // Keep organizationId (the customer and CH org survive cancellation) and- // stripeCustomerId (support tooling); drop the plan so the org resolves- // back to cloud:hobby — same semantics as Stripe subscription.deleted.clickhouse: {- organizationId: event.data.organizationId,+ organizationId: event.data.payload.organizationId,stripeCustomerId: existing?.stripeCustomerId,lastEventCreatedAt: event.occurredAt,},orgColumns: {- // Reset billing cycle anchor on downgrade to hobby to start of today+ // Back on hobby: the billing cycle restarts todaycloudBillingCycleAnchor: startOfDayUTC(new Date()),},});}++async function backfillProjectEvents(+ parsedOrg: ParsedOrganization,+ chbOrganizationId: string,+) {+ const projects = await prisma.project.findMany({+ where: { orgId: parsedOrg.id, deletedAt: null },+ select: { id: true },+ });+ const results = await Promise.allSettled(+ projects.map((project) =>+ sendChbProjectEvent({+ type: "LANGFUSE_PROJECT_CREATED",+ chbOrganizationId,+ projectId: project.id,+ }),+ ),+ );+ const failed = results.filter((result) => result.status === "rejected");+ if (failed.length > 0) {+ recordIncrement("langfuse.billing_events.emit_failed", failed.length, {+ unit: "events",+ });+ logger.error(+ `[CHB Webhook] Project backfill emit failed for ${failed.length}/${projects.length} projects of org ${parsedOrg.id}`,+ );+ }+}