Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

24.2 KB
Raw Blame
import crypto from "crypto";

import { type NextRequest, NextResponse } from "next/server";
import { z } from "zod";

import { env } from "@/src/env.mjs";
import { auditLog } from "@/src/features/audit-logs/auditLog";
import { invalidateGatewayResolveCacheForOrganization } from "@/src/features/ai-gateway/server/resolve/gatewayResolveCache";
import {
  isoDateTime,
  type ChbPlanCode,
  chbPlanCodes,
  chbPlanCodeToPlan,
  CloudConfigSchema,
  parseDbOrg,
  type ParsedOrganization,
} from "@langfuse/shared";
import { prisma } from "@langfuse/shared/src/db";
import {
  invalidateCachedOrgApiKeys,
  logger,
  recordIncrement,
  redis,
  startOfDayUTC,
  traceException,
} from "@langfuse/shared/src/server";

import { createDefaultSpendAlerts } from "../defaultSpendAlerts";
import { type ChbAttachedPlan, getChbApiClient } from "./chbApiClient";
import { sendChbProjectEvent } from "./chbProjectEvents";

/**
 * ClickHouse Billing webhook handler — structural twin of
 * stripeWebhookHandler. CHB is the source of truth for attached-plan state;
 * this handler is the single writer of `cloudConfig.clickhouse` (plus the
 * checkout-session write of `organizationId`).
 *
 * Pipeline: verify HMAC → dedupe → resolve org (region fan-out) → ordering
 * guard → per-event effect (created/updated read the plan back from CHB) →
 * invalidateCachedOrgApiKeys + auditLog.
 */

const CHB_SIGNATURE_HEADER = "x-chb-signature";
const MAX_CLOCK_SKEW_SECONDS = 5 * 60;
const DEDUPE_TTL_SECONDS = 24 * 60 * 60;

// Envelope as the ClickHouse control plane's signed-webhook dispatcher emits
// it: `{eventId, type, occurredAt, data}`, where `data` is the billing-api
// event exactly as it travelled over CHB's event bus — a
// `{id, source, timestamp, version, payload}` record whose `payload` names the
// owning ClickHouse organization. `type` repeats the bus detail-type. The
// body is signed as-is, so this schema only ever sees bytes that already
// passed verification. Objects stay permissive about unknown fields so CHB
// can extend the payload without a deploy here.
//
// The event is a notification, not a state snapshot: it carries the plan code
// but neither the attached plan id nor payment or billing-period details, so
// the handler reads the attached plan back from CHB's REST API on receipt.

// The detail-types CHB's billing-api emits for an attached plan. The switch
// in the handler is exhaustive over this list; a type outside it fails the
// schema and is answered 400.
const ChbWebhookEventTypeSchema = z.enum([
  "BILLING_ATTACHEDPLAN_CREATED",
  "BILLING_ATTACHEDPLAN_UPDATED",
  "BILLING_ATTACHEDPLAN_SCHEDULED",
  "BILLING_ATTACHEDPLAN_CANCELLED",
]);

// Event-bus timestamps arrive as Unix milliseconds or as a string (ISO-8601 or
// numeric). Nothing reads them -- ordering runs on `occurredAt` -- so any of
// these shapes is accepted rather than failing a signed event over a field the
// handler ignores.
const ChbEventBusTimestampSchema = z.union([z.number(), z.string()]).nullish();

const ChbWebhookEventSchema = z.object({
  eventId: z.string().min(1),
  type: ChbWebhookEventTypeSchema,
  occurredAt: isoDateTime({ offset: true }),
  data: z.object({
    // Event-bus record id, not the attached plan id
    id: z.string().nullish(),
    source: z.string().nullish(),
    timestamp: ChbEventBusTimestampSchema,
    version: z.number().nullish(),
    payload: z.object({
      eventType: ChbWebhookEventTypeSchema,
      // ClickHouse Organization ID owning the attached plan
      organizationId: z.uuid(),
      planCode: z.string().nullish(),
      createdAt: ChbEventBusTimestampSchema,
    }),
  }),
});
export type ChbWebhookEvent = z.infer<typeof ChbWebhookEventSchema>;

/**
 * `X-CHB-Signature: t=<unix seconds>,v1=<hex>[,v1=<hex>...]`, as emitted by
 * the control plane's signed-webhook dispatcher. The timestamp stays a string
 * because it is signed byte for byte. Several `v1` values appear while CHB
 * rotates its signing key: the dispatcher signs with every key it holds, and a
 * receiver accepts the request if any of them matches the key it holds.
 */
function parseChbSignatureHeader(
  header: string,
): { timestamp: string; signatures: string[] } | null {
  let timestamp: string | undefined;
  const signatures: string[] = [];

  for (const part of header.split(",")) {
    const separator = part.indexOf("=");
    if (separator === -1) continue;
    const key = part.slice(0, separator);
    const value = part.slice(separator + 1);
    if (key === "t") timestamp = value;
    else if (key === "v1") signatures.push(value);
  }

  if (timestamp === undefined || signatures.length === 0) return null;
  return { timestamp, signatures };
}

/** Constant-time compare of two hex digests. `timingSafeEqual` throws on a
 * length mismatch and the candidate is attacker-controlled; the length is not
 * secret, so checking it first is fine. */
function digestsMatch(expectedHex: string, candidateHex: string): boolean {
  const expected = Buffer.from(expectedHex, "hex");
  const candidate = Buffer.from(candidateHex, "hex");
  return (
    expected.length === candidate.length &&
    crypto.timingSafeEqual(expected, candidate)
  );
}

/**
 * Verify the CHB webhook signature: HMAC-SHA256 over `${t}.${rawBody}`, hex
 * encoded, compared in constant time, with a ±5 minute skew window on `t`.
 * The dispatcher re-signs on every delivery attempt, so the short window stays
 * compatible with its multi-day retry budget. One local secret is enough for a
 * rotation: while CHB signs with both keys, any `v1` matching ours passes.
 */
export function verifyChbSignature(params: {
  rawBody: string | Buffer;
  signatureHeader: string | null;
  secret: string;
  nowMs?: number;
}): { valid: boolean; reason?: string } {
  const { rawBody, signatureHeader, secret } = params;
  if (signatureHeader === null) {
    return { valid: false, reason: "missing signature header" };
  }
  const parsed = parseChbSignatureHeader(signatureHeader);
  if (!parsed) {
    return { valid: false, reason: "malformed signature header" };
  }
  if (!/^\d+$/.test(parsed.timestamp)) {
    return { valid: false, reason: "malformed timestamp" };
  }

  const nowSeconds = Math.floor((params.nowMs ?? Date.now()) / 1000);
  if (
    Math.abs(nowSeconds - Number(parsed.timestamp)) > MAX_CLOCK_SKEW_SECONDS
  ) {
    return { valid: false, reason: "timestamp outside allowed clock skew" };
  }

  const expected = crypto
    .createHmac("sha256", secret)
    .update(`${parsed.timestamp}.`)
    .update(rawBody)
    .digest("hex");
  const matches = parsed.signatures.some((candidate) =>
    digestsMatch(expected, candidate),
  );
  return matches
    ? { valid: true }
    : { valid: false, reason: "signature mismatch" };
}

async function getOrgByChbOrganizationId(
  chbOrganizationId: string,
): Promise<ParsedOrganization | null> {
  const org = await prisma.organization.findFirst({
    where: {
      cloudConfig: {
        path: ["clickhouse", "organizationId"],
        equals: chbOrganizationId,
      },
    },
  });
  return org ? parseDbOrg(org) : null;
}

const dedupeKey = (eventId: string) => `chb-webhook-event:${eventId}`;

/** Best-effort Redis dedupe; every handler is also idempotent, so a Redis
 * flush cannot corrupt state. Returns true when the event was seen before.
 *
 * Claiming the id is the same operation as checking it, so a claim that is not
 * followed by a successful apply has to be released — see releaseEventClaim. */
async function isDuplicateEvent(eventId: string): Promise<boolean> {
  try {
    const result = await redis?.set(
      dedupeKey(eventId),
      "1",
      "EX",
      DEDUPE_TTL_SECONDS,
      "NX",
    );
    // ioredis returns null when NX prevented the write → key existed
    return redis ? result === null : false;
  } catch (error) {
    logger.warn("[CHB Webhook] Redis dedupe check failed, processing event", {
      eventId,
      error,
    });
    return false;
  }
}

/**
 * Drop the dedupe claim for an event that failed to apply. Without this, the
 * 500 we return sends CHB into a retry that its own claim now suppresses as a
 * duplicate, and the event is lost for the whole 24h TTL — long after CHB has
 * stopped retrying. Losing one means a missing cloudConfig.clickhouse block, a
 * still-suspended paying org, or an unmetered project, all needing manual
 * reconciliation.
 */
async function releaseEventClaim(eventId: string): Promise<void> {
  try {
    await redis?.del(dedupeKey(eventId));
  } catch (error) {
    // Nothing better to do than page: the event is now un-retryable.
    logger.error(
      `[CHB Webhook] Failed to release the dedupe claim for event ${eventId}; a CHB retry will be dropped as a duplicate`,
      error,
    );
    traceException(error);
  }
}

export async function chbWebhookHandler(req: NextRequest) {
  if (req.method !== "POST") {
    return NextResponse.json(
      { message: "Method not allowed" },
      { status: 405 },
    );
  }

  if (!env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION) {
    logger.error("[CHB Webhook] Endpoint only available in Langfuse Cloud");
    return NextResponse.json(
      { message: "CHB webhook endpoint only available in Langfuse Cloud" },
      { status: 500 },
    );
  }
  if (!env.CLICKHOUSE_BILLING_WEBHOOK_SIGNING_SECRET) {
    logger.error(
      "[CHB Webhook] CLICKHOUSE_BILLING_WEBHOOK_SIGNING_SECRET is not configured",
    );
    return NextResponse.json(
      { message: "CHB webhook signing secret not found" },
      { status: 500 },
    );
  }

  // The bytes as received: the signature covers the body exactly as CHB
  // serialized it, so nothing may decode or re-encode it before verification.
  const rawBody = Buffer.from(await req.arrayBuffer());
  const verification = verifyChbSignature({
    rawBody,
    signatureHeader: req.headers.get(CHB_SIGNATURE_HEADER),
    secret: env.CLICKHOUSE_BILLING_WEBHOOK_SIGNING_SECRET,
  });
  if (!verification.valid) {
    logger.error(
      `[CHB Webhook] Signature verification failed: ${verification.reason}`,
    );
    return NextResponse.json(
      { message: `Webhook error: ${verification.reason}` },
      { status: 400 },
    );
  }

  let event: ChbWebhookEvent;
  try {
    event = ChbWebhookEventSchema.parse(JSON.parse(rawBody.toString("utf8")));
  } catch (error) {
    logger.error("[CHB Webhook] Failed to parse event payload", error);
    return NextResponse.json(
      { message: "Webhook error: invalid payload" },
      { status: 400 },
    );
  }

  logger.info(`[CHB Webhook] Start ${event.type}`, { payload: event });

  if (await isDuplicateEvent(event.eventId)) {
    logger.info(
      `[CHB Webhook] Duplicate event ${event.eventId} (${event.type}), skipping`,
    );
    return NextResponse.json({ received: true }, { status: 200 });
  }

  // Region fan-out: CHB pings all Langfuse regions; exactly one owns the org.
  const chOrganizationId = event.data.payload.organizationId;
  const parsedOrg = await getOrgByChbOrganizationId(chOrganizationId);
  if (!parsedOrg) {
    logger.info(
      `[CHB Webhook] No org for CHB organization ${chOrganizationId} in this region, ignoring`,
    );
    return NextResponse.json({ received: true }, { status: 200 });
  }

  // Interlock (must never happen): an org bills through exactly one provider.
  // Checking getBillingProvider here would be dead code -- it returns
  // "clickhouse" for anything carrying a clickhouse.organizationId, which is
  // precisely how this org was just resolved. The state that actually needs
  // catching is the contradiction: a live Stripe subscription alongside CHB
  // state means the customer is being billed twice, and applying more CHB state
  // would bury that. Refuse and page instead; a retry cannot fix it.
  if (parsedOrg.cloudConfig?.stripe?.activeSubscriptionId) {
    const message = `[CHB Webhook] Org ${parsedOrg.id} carries an active Stripe subscription alongside CHB state, refusing to apply ${event.type}`;
    logger.error(message);
    traceException(message);
    return NextResponse.json({ received: true }, { status: 200 });
  }

  // Ordering guard: drop events at or before the last applied one (protects
  // against retries and out-of-order delivery).
  const lastEventCreatedAt =
    parsedOrg.cloudConfig?.clickhouse?.lastEventCreatedAt;
  if (
    lastEventCreatedAt &&
    Date.parse(event.occurredAt) <= Date.parse(lastEventCreatedAt)
  ) {
    logger.info(
      `[CHB Webhook] Out-of-order event ${event.eventId} (${event.type}) for org ${parsedOrg.id}, skipping`,
      { occurredAt: event.occurredAt, lastEventCreatedAt },
    );
    return NextResponse.json({ received: true }, { status: 200 });
  }

  // An unknown plan code means CHB shipped a tier before its mapping is
  // deployed here. Storing it would null the plan and resolve a paying org to
  // cloud:hobby, so refuse and page instead. The 500 keeps the event inside
  // the dispatcher's retry budget: once the mapping ships, the retry applies it.
  const announcedPlanCode = event.data.payload.planCode;
  if (
    announcedPlanCode != null &&
    !(chbPlanCodes as readonly string[]).includes(announcedPlanCode)
  ) {
    await releaseEventClaim(event.eventId);
    const message = `[CHB Webhook] Unknown plan code ${announcedPlanCode} on ${event.type} for org ${parsedOrg.id}, refusing until a mapping ships`;
    logger.error(message);
    traceException(message);
    return NextResponse.json(
      { message: "Webhook error: unknown plan code" },
      { status: 500 },
    );
  }

  try {
    switch (event.type) {
      case "BILLING_ATTACHEDPLAN_CREATED":
        await handleAttachedPlanCreated(parsedOrg, event);
        break;
      case "BILLING_ATTACHEDPLAN_UPDATED":
      case "BILLING_ATTACHEDPLAN_SCHEDULED":
        await handleAttachedPlanChanged(parsedOrg, event);
        break;
      case "BILLING_ATTACHEDPLAN_CANCELLED":
        await handleAttachedPlanCancelled(parsedOrg, event);
        break;
      default: {
        const exhaustiveCheck: never = event.type;
        throw new Error(`Unhandled CHB event type ${exhaustiveCheck}`);
      }
    }
  } catch (error) {
    // The dedupe claim was taken before dispatch, so it has to go back before we
    // ask CHB to retry -- otherwise the retry is dropped as a duplicate and the
    // event is lost for good.
    await releaseEventClaim(event.eventId);
    logger.error(
      `[CHB Webhook] Failed to apply ${event.type} (${event.eventId}) for org ${parsedOrg.id}`,
      error,
    );
    traceException(error);
    return NextResponse.json(
      { message: "Failed to apply webhook event" },
      { status: 500 },
    );
  }

  return NextResponse.json({ received: true }, { status: 200 });
}

/**
 * Persist a new clickhouse block + org columns, then propagate: the resolved
 * plan and suspension flag are baked into the Redis-cached API-key record, so
 * every write ends with a cache invalidation.
 *
 * The organization update is the commit point. It persists this event's
 * occurredAt as lastEventCreatedAt, after which the ordering guard drops any
 * retry of the same event as already applied. Nothing that runs after it may
 * fail the request: a 500 would ask CHB for a retry that can never reach that
 * code again. Post-commit steps -- here and in the per-event handlers -- are
 * best-effort: log, page, and let the request succeed.
 */
async function persistAndPropagate(params: {
  parsedOrg: ParsedOrganization;
  event: ChbWebhookEvent;
  clickhouse: unknown;
  orgColumns?: {
    cloudBillingCycleAnchor?: Date;
    cloudFreeTierUsageThresholdState?: null;
  };
}) {
  const { parsedOrg, event } = params;

  // Validate through the stored schema so a bad write can never poison
  // parseDbOrg for the whole cloudConfig.
  const updatedCloudConfig = {
    ...parsedOrg.cloudConfig,
    clickhouse: CloudConfigSchema.shape.clickhouse.parse(params.clickhouse),
  };

  await prisma.organization.update({
    where: { id: parsedOrg.id },
    data: {
      cloudConfig: updatedCloudConfig,
      ...(params.orgColumns ?? {}),
    },
  });

  // Post-commit. Bounded: a stale cached plan expires with the API-key cache
  // TTL.
  try {
    await invalidateCachedOrgApiKeys(parsedOrg.id);
    await invalidateGatewayResolveCacheForOrganization(parsedOrg.id);
  } catch (error) {
    logger.error(
      `[CHB Webhook] Failed to invalidate cached API keys for org ${parsedOrg.id} after applying ${event.type}`,
      error,
    );
    traceException(error);
  }

  // Post-commit as well: the state change stands, a missing audit row is
  // surfaced rather than retried.
  try {
    await auditLog({
      session: {
        user: { id: "clickhouse-webhook" },
        orgId: parsedOrg.id,
      },
      orgId: parsedOrg.id,
      resourceType: "organization",
      resourceId: parsedOrg.id,
      action: `BillingService.chb.${event.type}`,
      before: parsedOrg.cloudConfig,
      after: updatedCloudConfig,
    });
  } catch (error) {
    logger.error(
      `[CHB Webhook] Failed to write the audit log for ${event.type} on org ${parsedOrg.id}`,
      error,
    );
    traceException(error);
  }

  return updatedCloudConfig;
}

/**
 * The attached plan behind an event. The event only announces it: the id,
 * payment state and billing period live on CHB's REST side, so the handler
 * reads them back. Throws when CHB is not configured or the read fails; the
 * caller's catch releases the dedupe claim and answers 500, so CHB retries
 * once the plan is readable.
 */
async function fetchAttachedPlan(
  chOrganizationId: string,
): Promise<ChbAttachedPlan> {
  const client = getChbApiClient();
  if (!client) {
    throw new Error(
      "CHB REST client is not configured, cannot read the attached plan behind the event",
    );
  }
  return await client.getAttachedPlan({ chOrganizationId });
}

/**
 * What an event implies for the stored block, read back from CHB. Created and
 * updated both start here: the event only announces the attached plan, its
 * id, payment state and billing period live on CHB's REST side.
 */
async function readAttachedPlanState(
  parsedOrg: ParsedOrganization,
  event: ChbWebhookEvent,
) {
  const chOrganizationId = event.data.payload.organizationId;
  const existing = parsedOrg.cloudConfig?.clickhouse;

  const attachedPlan = await fetchAttachedPlan(chOrganizationId);

  // CHB's REST view is authoritative; the event's plan code only fills a gap.
  // The pipeline already refused an unknown code on the event itself, so a
  // rejection here means CHB reports a code that has no mapping yet. Same
  // answer: the 500 keeps the event retryable until the mapping ships.
  const reportedPlanCode =
    attachedPlan.plan?.code ?? event.data.payload.planCode ?? null;
  if (
    reportedPlanCode != null &&
    !(chbPlanCodes as readonly string[]).includes(reportedPlanCode)
  ) {
    throw new Error(
      `Attached plan ${attachedPlan.id} for org ${parsedOrg.id} carries unknown plan code ${reportedPlanCode}`,
    );
  }
  // Narrowed against the membership check above, which is the runtime source
  // of truth; callers need the code typed to map it onto a Langfuse plan.
  const planCode = reportedPlanCode as ChbPlanCode | null;

  return {
    attachedPlan,
    // Same gate as the Stripe path: only clear the free-tier suspension once
    // payment is credibly current. A plan whose payment is pending or failed
    // must not un-block ingestion for an org that was suspended at the
    // free-tier limit; BILLING_ATTACHEDPLAN_UPDATED lifts it once the payment
    // goes active.
    isPaidAndCurrent: attachedPlan.payment?.status === "active",
    // A full snapshot, not a merge: CHB is the source of truth, so a
    // re-subscription cannot inherit state from a previously cancelled plan.
    clickhouse: {
      organizationId: chOrganizationId,
      attachedPlanId: attachedPlan.id,
      planCode,
      paymentStatus: attachedPlan.payment?.status,
      // The current period ends at the renewal, which is the next charge.
      nextPaymentDate: attachedPlan.period?.endDate,
      stripeCustomerId:
        attachedPlan.payment?.provider?.customerId ??
        existing?.stripeCustomerId,
      // Pending upgrade/downgrade/cancel as CHB reports it; null clears one
      // that executed or was withdrawn.
      scheduled: attachedPlan.scheduled ?? null,
      lastEventCreatedAt: event.occurredAt,
    },
  };
}

async function handleAttachedPlanCreated(
  parsedOrg: ParsedOrganization,
  event: ChbWebhookEvent,
) {
  const { attachedPlan, isPaidAndCurrent, clickhouse } =
    await readAttachedPlanState(parsedOrg, event);

  await persistAndPropagate({
    parsedOrg,
    event,
    clickhouse,
    orgColumns: {
      // First paid subscription anchors the billing cycle on the plan start
      cloudBillingCycleAnchor: attachedPlan.period?.startDate
        ? new Date(attachedPlan.period.startDate)
        : startOfDayUTC(new Date()),
      ...(isPaidAndCurrent ? { cloudFreeTierUsageThresholdState: null } : {}),
    },
  });

  // Seed the plan's default spend alerts, as the Stripe path does on a first
  // subscription. Post-commit and best-effort: the plan is already applied, and
  // an org without seeded alerts can still configure its own.
  if (clickhouse.planCode) {
    try {
      await createDefaultSpendAlerts({
        orgId: parsedOrg.id,
        plan: chbPlanCodeToPlan[clickhouse.planCode],
        source: "clickhouse",
      });
    } catch (error) {
      logger.error(
        `[CHB Webhook] Failed to create default spend alerts for org ${parsedOrg.id}`,
        error,
      );
      traceException(error);
    }
  }

  // Backfill-emit LANGFUSE_PROJECT_CREATED for all existing projects: projects
  // created before checkout would otherwise be invisible to CHB metering.
  // Post-commit and best-effort -- CHB's own backfill pipeline is the backstop.
  try {
    await backfillProjectEvents(parsedOrg, clickhouse.organizationId);
  } catch (error) {
    logger.error(
      `[CHB Webhook] Project backfill failed for org ${parsedOrg.id}`,
      error,
    );
    traceException(error);
  }
}

/**
 * BILLING_ATTACHEDPLAN_UPDATED and BILLING_ATTACHEDPLAN_SCHEDULED: plan,
 * payment or scheduled-change state moved on CHB's side. Both re-read and
 * overwrite the snapshot. A scheduled change is only a snapshot as well: the
 * plan itself flips when the terminal updated or cancelled event lands, and
 * we never execute scheduled changes locally on a timer -- CHB owns that
 * moment. The billing cycle anchor stays where the created event put it.
 */
async function handleAttachedPlanChanged(
  parsedOrg: ParsedOrganization,
  event: ChbWebhookEvent,
) {
  const { isPaidAndCurrent, clickhouse } = await readAttachedPlanState(
    parsedOrg,
    event,
  );

  await persistAndPropagate({
    parsedOrg,
    event,
    clickhouse,
    orgColumns: isPaidAndCurrent
      ? { cloudFreeTierUsageThresholdState: null }
      : undefined,
  });
}

/**
 * Terminal: the plan is gone, so there is nothing to read back -- CHB may
 * already answer 404 for the organization. Keep organizationId (the customer
 * and CH org survive cancellation) and stripeCustomerId (support tooling);
 * drop the plan so the org resolves back to cloud:hobby -- same semantics as
 * Stripe subscription.deleted.
 */
async function handleAttachedPlanCancelled(
  parsedOrg: ParsedOrganization,
  event: ChbWebhookEvent,
) {
  const existing = parsedOrg.cloudConfig?.clickhouse;

  await persistAndPropagate({
    parsedOrg,
    event,
    clickhouse: {
      organizationId: event.data.payload.organizationId,
      stripeCustomerId: existing?.stripeCustomerId,
      lastEventCreatedAt: event.occurredAt,
    },
    orgColumns: {
      // Back on hobby: the billing cycle restarts today
      cloudBillingCycleAnchor: startOfDayUTC(new Date()),
    },
  });
}

async function backfillProjectEvents(
  parsedOrg: ParsedOrganization,
  chbOrganizationId: string,
) {
  const projects = await prisma.project.findMany({
    where: { orgId: parsedOrg.id, deletedAt: null },
    select: { id: true },
  });
  const results = await Promise.allSettled(
    projects.map((project) =>
      sendChbProjectEvent({
        type: "LANGFUSE_PROJECT_CREATED",
        chbOrganizationId,
        projectId: project.id,
      }),
    ),
  );
  const failed = results.filter((result) => result.status === "rejected");
  if (failed.length > 0) {
    recordIncrement("langfuse.billing_events.emit_failed", failed.length, {
      unit: "events",
    });
    logger.error(
      `[CHB Webhook] Project backfill emit failed for ${failed.length}/${projects.length} projects of org ${parsedOrg.id}`,
    );
  }
}