🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import crypto from "crypto";
import { type NextRequest, NextResponse } from "next/server";
import { z } from "zod";
import { env } from "@/src/env.mjs";
import { auditLog } from "@/src/features/audit-logs/auditLog";
import { invalidateGatewayResolveCacheForOrganization } from "@/src/features/ai-gateway/server/resolve/gatewayResolveCache";
import {
isoDateTime,
type ChbPlanCode,
chbPlanCodes,
chbPlanCodeToPlan,
CloudConfigSchema,
parseDbOrg,
type ParsedOrganization,
} from "@langfuse/shared";
import { prisma } from "@langfuse/shared/src/db";
import {
invalidateCachedOrgApiKeys,
logger,
recordIncrement,
redis,
startOfDayUTC,
traceException,
} from "@langfuse/shared/src/server";
import { createDefaultSpendAlerts } from "../defaultSpendAlerts";
import { type ChbAttachedPlan, getChbApiClient } from "./chbApiClient";
import { sendChbProjectEvent } from "./chbProjectEvents";
/**
* ClickHouse Billing webhook handler — structural twin of
* stripeWebhookHandler. CHB is the source of truth for attached-plan state;
* this handler is the single writer of `cloudConfig.clickhouse` (plus the
* checkout-session write of `organizationId`).
*
* Pipeline: verify HMAC → dedupe → resolve org (region fan-out) → ordering
* guard → per-event effect (created/updated read the plan back from CHB) →
* invalidateCachedOrgApiKeys + auditLog.
*/
const CHB_SIGNATURE_HEADER = "x-chb-signature";
const MAX_CLOCK_SKEW_SECONDS = 5 * 60;
const DEDUPE_TTL_SECONDS = 24 * 60 * 60;
// Envelope as the ClickHouse control plane's signed-webhook dispatcher emits
// it: `{eventId, type, occurredAt, data}`, where `data` is the billing-api
// event exactly as it travelled over CHB's event bus — a
// `{id, source, timestamp, version, payload}` record whose `payload` names the
// owning ClickHouse organization. `type` repeats the bus detail-type. The
// body is signed as-is, so this schema only ever sees bytes that already
// passed verification. Objects stay permissive about unknown fields so CHB
// can extend the payload without a deploy here.
//
// The event is a notification, not a state snapshot: it carries the plan code
// but neither the attached plan id nor payment or billing-period details, so
// the handler reads the attached plan back from CHB's REST API on receipt.
// The detail-types CHB's billing-api emits for an attached plan. The switch
// in the handler is exhaustive over this list; a type outside it fails the
// schema and is answered 400.
const ChbWebhookEventTypeSchema = z.enum([
"BILLING_ATTACHEDPLAN_CREATED",
"BILLING_ATTACHEDPLAN_UPDATED",
"BILLING_ATTACHEDPLAN_SCHEDULED",
"BILLING_ATTACHEDPLAN_CANCELLED",
]);
// Event-bus timestamps arrive as Unix milliseconds or as a string (ISO-8601 or
// numeric). Nothing reads them -- ordering runs on `occurredAt` -- so any of
// these shapes is accepted rather than failing a signed event over a field the
// handler ignores.
const ChbEventBusTimestampSchema = z.union([z.number(), z.string()]).nullish();
const ChbWebhookEventSchema = z.object({
eventId: z.string().min(1),
type: ChbWebhookEventTypeSchema,
occurredAt: isoDateTime({ offset: true }),
data: z.object({
// Event-bus record id, not the attached plan id
id: z.string().nullish(),
source: z.string().nullish(),
timestamp: ChbEventBusTimestampSchema,
version: z.number().nullish(),
payload: z.object({
eventType: ChbWebhookEventTypeSchema,
// ClickHouse Organization ID owning the attached plan
organizationId: z.uuid(),
planCode: z.string().nullish(),
createdAt: ChbEventBusTimestampSchema,
}),
}),
});
export type ChbWebhookEvent = z.infer<typeof ChbWebhookEventSchema>;
/**
* `X-CHB-Signature: t=<unix seconds>,v1=<hex>[,v1=<hex>...]`, as emitted by
* the control plane's signed-webhook dispatcher. The timestamp stays a string
* because it is signed byte for byte. Several `v1` values appear while CHB
* rotates its signing key: the dispatcher signs with every key it holds, and a
* receiver accepts the request if any of them matches the key it holds.
*/
function parseChbSignatureHeader(
header: string,
): { timestamp: string; signatures: string[] } | null {
let timestamp: string | undefined;
const signatures: string[] = [];
for (const part of header.split(",")) {
const separator = part.indexOf("=");
if (separator === -1) continue;
const key = part.slice(0, separator);
const value = part.slice(separator + 1);
if (key === "t") timestamp = value;
else if (key === "v1") signatures.push(value);
}
if (timestamp === undefined || signatures.length === 0) return null;
return { timestamp, signatures };
}
/** Constant-time compare of two hex digests. `timingSafeEqual` throws on a
* length mismatch and the candidate is attacker-controlled; the length is not
* secret, so checking it first is fine. */
function digestsMatch(expectedHex: string, candidateHex: string): boolean {
const expected = Buffer.from(expectedHex, "hex");
const candidate = Buffer.from(candidateHex, "hex");
return (
expected.length === candidate.length &&
crypto.timingSafeEqual(expected, candidate)
);
}
/**
* Verify the CHB webhook signature: HMAC-SHA256 over `${t}.${rawBody}`, hex
* encoded, compared in constant time, with a ±5 minute skew window on `t`.
* The dispatcher re-signs on every delivery attempt, so the short window stays
* compatible with its multi-day retry budget. One local secret is enough for a
* rotation: while CHB signs with both keys, any `v1` matching ours passes.
*/
export function verifyChbSignature(params: {
rawBody: string | Buffer;
signatureHeader: string | null;
secret: string;
nowMs?: number;
}): { valid: boolean; reason?: string } {
const { rawBody, signatureHeader, secret } = params;
if (signatureHeader === null) {
return { valid: false, reason: "missing signature header" };
}
const parsed = parseChbSignatureHeader(signatureHeader);
if (!parsed) {
return { valid: false, reason: "malformed signature header" };
}
if (!/^\d+$/.test(parsed.timestamp)) {
return { valid: false, reason: "malformed timestamp" };
}
const nowSeconds = Math.floor((params.nowMs ?? Date.now()) / 1000);
if (
Math.abs(nowSeconds - Number(parsed.timestamp)) > MAX_CLOCK_SKEW_SECONDS
) {
return { valid: false, reason: "timestamp outside allowed clock skew" };
}
const expected = crypto
.createHmac("sha256", secret)
.update(`${parsed.timestamp}.`)
.update(rawBody)
.digest("hex");
const matches = parsed.signatures.some((candidate) =>
digestsMatch(expected, candidate),
);
return matches
? { valid: true }
: { valid: false, reason: "signature mismatch" };
}
async function getOrgByChbOrganizationId(
chbOrganizationId: string,
): Promise<ParsedOrganization | null> {
const org = await prisma.organization.findFirst({
where: {
cloudConfig: {
path: ["clickhouse", "organizationId"],
equals: chbOrganizationId,
},
},
});
return org ? parseDbOrg(org) : null;
}
const dedupeKey = (eventId: string) => `chb-webhook-event:${eventId}`;
/** Best-effort Redis dedupe; every handler is also idempotent, so a Redis
* flush cannot corrupt state. Returns true when the event was seen before.
*
* Claiming the id is the same operation as checking it, so a claim that is not
* followed by a successful apply has to be released — see releaseEventClaim. */
async function isDuplicateEvent(eventId: string): Promise<boolean> {
try {
const result = await redis?.set(
dedupeKey(eventId),
"1",
"EX",
DEDUPE_TTL_SECONDS,
"NX",
);
// ioredis returns null when NX prevented the write → key existed
return redis ? result === null : false;
} catch (error) {
logger.warn("[CHB Webhook] Redis dedupe check failed, processing event", {
eventId,
error,
});
return false;
}
}
/**
* Drop the dedupe claim for an event that failed to apply. Without this, the
* 500 we return sends CHB into a retry that its own claim now suppresses as a
* duplicate, and the event is lost for the whole 24h TTL — long after CHB has
* stopped retrying. Losing one means a missing cloudConfig.clickhouse block, a
* still-suspended paying org, or an unmetered project, all needing manual
* reconciliation.
*/
async function releaseEventClaim(eventId: string): Promise<void> {
try {
await redis?.del(dedupeKey(eventId));
} catch (error) {
// Nothing better to do than page: the event is now un-retryable.
logger.error(
`[CHB Webhook] Failed to release the dedupe claim for event ${eventId}; a CHB retry will be dropped as a duplicate`,
error,
);
traceException(error);
}
}
export async function chbWebhookHandler(req: NextRequest) {
if (req.method !== "POST") {
return NextResponse.json(
{ message: "Method not allowed" },
{ status: 405 },
);
}
if (!env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION) {
logger.error("[CHB Webhook] Endpoint only available in Langfuse Cloud");
return NextResponse.json(
{ message: "CHB webhook endpoint only available in Langfuse Cloud" },
{ status: 500 },
);
}
if (!env.CLICKHOUSE_BILLING_WEBHOOK_SIGNING_SECRET) {
logger.error(
"[CHB Webhook] CLICKHOUSE_BILLING_WEBHOOK_SIGNING_SECRET is not configured",
);
return NextResponse.json(
{ message: "CHB webhook signing secret not found" },
{ status: 500 },
);
}
// The bytes as received: the signature covers the body exactly as CHB
// serialized it, so nothing may decode or re-encode it before verification.
const rawBody = Buffer.from(await req.arrayBuffer());
const verification = verifyChbSignature({
rawBody,
signatureHeader: req.headers.get(CHB_SIGNATURE_HEADER),
secret: env.CLICKHOUSE_BILLING_WEBHOOK_SIGNING_SECRET,
});
if (!verification.valid) {
logger.error(
`[CHB Webhook] Signature verification failed: ${verification.reason}`,
);
return NextResponse.json(
{ message: `Webhook error: ${verification.reason}` },
{ status: 400 },
);
}
let event: ChbWebhookEvent;
try {
event = ChbWebhookEventSchema.parse(JSON.parse(rawBody.toString("utf8")));
} catch (error) {
logger.error("[CHB Webhook] Failed to parse event payload", error);
return NextResponse.json(
{ message: "Webhook error: invalid payload" },
{ status: 400 },
);
}
logger.info(`[CHB Webhook] Start ${event.type}`, { payload: event });
if (await isDuplicateEvent(event.eventId)) {
logger.info(
`[CHB Webhook] Duplicate event ${event.eventId} (${event.type}), skipping`,
);
return NextResponse.json({ received: true }, { status: 200 });
}
// Region fan-out: CHB pings all Langfuse regions; exactly one owns the org.
const chOrganizationId = event.data.payload.organizationId;
const parsedOrg = await getOrgByChbOrganizationId(chOrganizationId);
if (!parsedOrg) {
logger.info(
`[CHB Webhook] No org for CHB organization ${chOrganizationId} in this region, ignoring`,
);
return NextResponse.json({ received: true }, { status: 200 });
}
// Interlock (must never happen): an org bills through exactly one provider.
// Checking getBillingProvider here would be dead code -- it returns
// "clickhouse" for anything carrying a clickhouse.organizationId, which is
// precisely how this org was just resolved. The state that actually needs
// catching is the contradiction: a live Stripe subscription alongside CHB
// state means the customer is being billed twice, and applying more CHB state
// would bury that. Refuse and page instead; a retry cannot fix it.
if (parsedOrg.cloudConfig?.stripe?.activeSubscriptionId) {
const message = `[CHB Webhook] Org ${parsedOrg.id} carries an active Stripe subscription alongside CHB state, refusing to apply ${event.type}`;
logger.error(message);
traceException(message);
return NextResponse.json({ received: true }, { status: 200 });
}
// Ordering guard: drop events at or before the last applied one (protects
// against retries and out-of-order delivery).
const lastEventCreatedAt =
parsedOrg.cloudConfig?.clickhouse?.lastEventCreatedAt;
if (
lastEventCreatedAt &&
Date.parse(event.occurredAt) <= Date.parse(lastEventCreatedAt)
) {
logger.info(
`[CHB Webhook] Out-of-order event ${event.eventId} (${event.type}) for org ${parsedOrg.id}, skipping`,
{ occurredAt: event.occurredAt, lastEventCreatedAt },
);
return NextResponse.json({ received: true }, { status: 200 });
}
// An unknown plan code means CHB shipped a tier before its mapping is
// deployed here. Storing it would null the plan and resolve a paying org to
// cloud:hobby, so refuse and page instead. The 500 keeps the event inside
// the dispatcher's retry budget: once the mapping ships, the retry applies it.
const announcedPlanCode = event.data.payload.planCode;
if (
announcedPlanCode != null &&
!(chbPlanCodes as readonly string[]).includes(announcedPlanCode)
) {
await releaseEventClaim(event.eventId);
const message = `[CHB Webhook] Unknown plan code ${announcedPlanCode} on ${event.type} for org ${parsedOrg.id}, refusing until a mapping ships`;
logger.error(message);
traceException(message);
return NextResponse.json(
{ message: "Webhook error: unknown plan code" },
{ status: 500 },
);
}
try {
switch (event.type) {
case "BILLING_ATTACHEDPLAN_CREATED":
await handleAttachedPlanCreated(parsedOrg, event);
break;
case "BILLING_ATTACHEDPLAN_UPDATED":
case "BILLING_ATTACHEDPLAN_SCHEDULED":
await handleAttachedPlanChanged(parsedOrg, event);
break;
case "BILLING_ATTACHEDPLAN_CANCELLED":
await handleAttachedPlanCancelled(parsedOrg, event);
break;
default: {
const exhaustiveCheck: never = event.type;
throw new Error(`Unhandled CHB event type ${exhaustiveCheck}`);
}
}
} catch (error) {
// The dedupe claim was taken before dispatch, so it has to go back before we
// ask CHB to retry -- otherwise the retry is dropped as a duplicate and the
// event is lost for good.
await releaseEventClaim(event.eventId);
logger.error(
`[CHB Webhook] Failed to apply ${event.type} (${event.eventId}) for org ${parsedOrg.id}`,
error,
);
traceException(error);
return NextResponse.json(
{ message: "Failed to apply webhook event" },
{ status: 500 },
);
}
return NextResponse.json({ received: true }, { status: 200 });
}
/**
* Persist a new clickhouse block + org columns, then propagate: the resolved
* plan and suspension flag are baked into the Redis-cached API-key record, so
* every write ends with a cache invalidation.
*
* The organization update is the commit point. It persists this event's
* occurredAt as lastEventCreatedAt, after which the ordering guard drops any
* retry of the same event as already applied. Nothing that runs after it may
* fail the request: a 500 would ask CHB for a retry that can never reach that
* code again. Post-commit steps -- here and in the per-event handlers -- are
* best-effort: log, page, and let the request succeed.
*/
async function persistAndPropagate(params: {
parsedOrg: ParsedOrganization;
event: ChbWebhookEvent;
clickhouse: unknown;
orgColumns?: {
cloudBillingCycleAnchor?: Date;
cloudFreeTierUsageThresholdState?: null;
};
}) {
const { parsedOrg, event } = params;
// Validate through the stored schema so a bad write can never poison
// parseDbOrg for the whole cloudConfig.
const updatedCloudConfig = {
...parsedOrg.cloudConfig,
clickhouse: CloudConfigSchema.shape.clickhouse.parse(params.clickhouse),
};
await prisma.organization.update({
where: { id: parsedOrg.id },
data: {
cloudConfig: updatedCloudConfig,
...(params.orgColumns ?? {}),
},
});
// Post-commit. Bounded: a stale cached plan expires with the API-key cache
// TTL.
try {
await invalidateCachedOrgApiKeys(parsedOrg.id);
await invalidateGatewayResolveCacheForOrganization(parsedOrg.id);
} catch (error) {
logger.error(
`[CHB Webhook] Failed to invalidate cached API keys for org ${parsedOrg.id} after applying ${event.type}`,
error,
);
traceException(error);
}
// Post-commit as well: the state change stands, a missing audit row is
// surfaced rather than retried.
try {
await auditLog({
session: {
user: { id: "clickhouse-webhook" },
orgId: parsedOrg.id,
},
orgId: parsedOrg.id,
resourceType: "organization",
resourceId: parsedOrg.id,
action: `BillingService.chb.${event.type}`,
before: parsedOrg.cloudConfig,
after: updatedCloudConfig,
});
} catch (error) {
logger.error(
`[CHB Webhook] Failed to write the audit log for ${event.type} on org ${parsedOrg.id}`,
error,
);
traceException(error);
}
return updatedCloudConfig;
}
/**
* The attached plan behind an event. The event only announces it: the id,
* payment state and billing period live on CHB's REST side, so the handler
* reads them back. Throws when CHB is not configured or the read fails; the
* caller's catch releases the dedupe claim and answers 500, so CHB retries
* once the plan is readable.
*/
async function fetchAttachedPlan(
chOrganizationId: string,
): Promise<ChbAttachedPlan> {
const client = getChbApiClient();
if (!client) {
throw new Error(
"CHB REST client is not configured, cannot read the attached plan behind the event",
);
}
return await client.getAttachedPlan({ chOrganizationId });
}
/**
* What an event implies for the stored block, read back from CHB. Created and
* updated both start here: the event only announces the attached plan, its
* id, payment state and billing period live on CHB's REST side.
*/
async function readAttachedPlanState(
parsedOrg: ParsedOrganization,
event: ChbWebhookEvent,
) {
const chOrganizationId = event.data.payload.organizationId;
const existing = parsedOrg.cloudConfig?.clickhouse;
const attachedPlan = await fetchAttachedPlan(chOrganizationId);
// CHB's REST view is authoritative; the event's plan code only fills a gap.
// The pipeline already refused an unknown code on the event itself, so a
// rejection here means CHB reports a code that has no mapping yet. Same
// answer: the 500 keeps the event retryable until the mapping ships.
const reportedPlanCode =
attachedPlan.plan?.code ?? event.data.payload.planCode ?? null;
if (
reportedPlanCode != null &&
!(chbPlanCodes as readonly string[]).includes(reportedPlanCode)
) {
throw new Error(
`Attached plan ${attachedPlan.id} for org ${parsedOrg.id} carries unknown plan code ${reportedPlanCode}`,
);
}
// Narrowed against the membership check above, which is the runtime source
// of truth; callers need the code typed to map it onto a Langfuse plan.
const planCode = reportedPlanCode as ChbPlanCode | null;
return {
attachedPlan,
// Same gate as the Stripe path: only clear the free-tier suspension once
// payment is credibly current. A plan whose payment is pending or failed
// must not un-block ingestion for an org that was suspended at the
// free-tier limit; BILLING_ATTACHEDPLAN_UPDATED lifts it once the payment
// goes active.
isPaidAndCurrent: attachedPlan.payment?.status === "active",
// A full snapshot, not a merge: CHB is the source of truth, so a
// re-subscription cannot inherit state from a previously cancelled plan.
clickhouse: {
organizationId: chOrganizationId,
attachedPlanId: attachedPlan.id,
planCode,
paymentStatus: attachedPlan.payment?.status,
// The current period ends at the renewal, which is the next charge.
nextPaymentDate: attachedPlan.period?.endDate,
stripeCustomerId:
attachedPlan.payment?.provider?.customerId ??
existing?.stripeCustomerId,
// Pending upgrade/downgrade/cancel as CHB reports it; null clears one
// that executed or was withdrawn.
scheduled: attachedPlan.scheduled ?? null,
lastEventCreatedAt: event.occurredAt,
},
};
}
async function handleAttachedPlanCreated(
parsedOrg: ParsedOrganization,
event: ChbWebhookEvent,
) {
const { attachedPlan, isPaidAndCurrent, clickhouse } =
await readAttachedPlanState(parsedOrg, event);
await persistAndPropagate({
parsedOrg,
event,
clickhouse,
orgColumns: {
// First paid subscription anchors the billing cycle on the plan start
cloudBillingCycleAnchor: attachedPlan.period?.startDate
? new Date(attachedPlan.period.startDate)
: startOfDayUTC(new Date()),
...(isPaidAndCurrent ? { cloudFreeTierUsageThresholdState: null } : {}),
},
});
// Seed the plan's default spend alerts, as the Stripe path does on a first
// subscription. Post-commit and best-effort: the plan is already applied, and
// an org without seeded alerts can still configure its own.
if (clickhouse.planCode) {
try {
await createDefaultSpendAlerts({
orgId: parsedOrg.id,
plan: chbPlanCodeToPlan[clickhouse.planCode],
source: "clickhouse",
});
} catch (error) {
logger.error(
`[CHB Webhook] Failed to create default spend alerts for org ${parsedOrg.id}`,
error,
);
traceException(error);
}
}
// Backfill-emit LANGFUSE_PROJECT_CREATED for all existing projects: projects
// created before checkout would otherwise be invisible to CHB metering.
// Post-commit and best-effort -- CHB's own backfill pipeline is the backstop.
try {
await backfillProjectEvents(parsedOrg, clickhouse.organizationId);
} catch (error) {
logger.error(
`[CHB Webhook] Project backfill failed for org ${parsedOrg.id}`,
error,
);
traceException(error);
}
}
/**
* BILLING_ATTACHEDPLAN_UPDATED and BILLING_ATTACHEDPLAN_SCHEDULED: plan,
* payment or scheduled-change state moved on CHB's side. Both re-read and
* overwrite the snapshot. A scheduled change is only a snapshot as well: the
* plan itself flips when the terminal updated or cancelled event lands, and
* we never execute scheduled changes locally on a timer -- CHB owns that
* moment. The billing cycle anchor stays where the created event put it.
*/
async function handleAttachedPlanChanged(
parsedOrg: ParsedOrganization,
event: ChbWebhookEvent,
) {
const { isPaidAndCurrent, clickhouse } = await readAttachedPlanState(
parsedOrg,
event,
);
await persistAndPropagate({
parsedOrg,
event,
clickhouse,
orgColumns: isPaidAndCurrent
? { cloudFreeTierUsageThresholdState: null }
: undefined,
});
}
/**
* Terminal: the plan is gone, so there is nothing to read back -- CHB may
* already answer 404 for the organization. Keep organizationId (the customer
* and CH org survive cancellation) and stripeCustomerId (support tooling);
* drop the plan so the org resolves back to cloud:hobby -- same semantics as
* Stripe subscription.deleted.
*/
async function handleAttachedPlanCancelled(
parsedOrg: ParsedOrganization,
event: ChbWebhookEvent,
) {
const existing = parsedOrg.cloudConfig?.clickhouse;
await persistAndPropagate({
parsedOrg,
event,
clickhouse: {
organizationId: event.data.payload.organizationId,
stripeCustomerId: existing?.stripeCustomerId,
lastEventCreatedAt: event.occurredAt,
},
orgColumns: {
// Back on hobby: the billing cycle restarts today
cloudBillingCycleAnchor: startOfDayUTC(new Date()),
},
});
}
async function backfillProjectEvents(
parsedOrg: ParsedOrganization,
chbOrganizationId: string,
) {
const projects = await prisma.project.findMany({
where: { orgId: parsedOrg.id, deletedAt: null },
select: { id: true },
});
const results = await Promise.allSettled(
projects.map((project) =>
sendChbProjectEvent({
type: "LANGFUSE_PROJECT_CREATED",
chbOrganizationId,
projectId: project.id,
}),
),
);
const failed = results.filter((result) => result.status === "rejected");
if (failed.length > 0) {
recordIncrement("langfuse.billing_events.emit_failed", failed.length, {
unit: "events",
});
logger.error(
`[CHB Webhook] Project backfill emit failed for ${failed.length}/${projects.length} projects of org ${parsedOrg.id}`,
);
}
}