🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(billing): report Stripe webhook org lookup misses at the right severity (#16757)
Steffen Schmitze1ce41244255
diff --git a/web/src/__tests__/server/stripe-webhook-handler.servertest.ts b/web/src/__tests__/server/stripe-webhook-handler.servertest.tsindex f51fe7e8de..8b6609aaf5 100644--- a/web/src/__tests__/server/stripe-webhook-handler.servertest.ts+++ b/web/src/__tests__/server/stripe-webhook-handler.servertest.ts@@ -1,10 +1,30 @@-import { describe, it, expect, beforeEach, afterEach } from "vitest";+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";import { v4 } from "uuid";import type Stripe from "stripe";+import type * as SharedServer from "@langfuse/shared/src/server";import { prisma } from "@langfuse/shared/src/db";import { handleSubscriptionChanged } from "@/src/ee/features/billing/server/stripe/stripeWebhookHandler";import { env } from "@/src/env.mjs";+const mocks = vi.hoisted(() => ({+ traceException: vi.fn(),+ listCheckoutSessions: vi.fn(),+}));++// traceException is what marks the webhook span as an error in APM, so it is+// the assertion target for "does this miss deserve a human's attention?".+vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({+ ...(await importOriginal<typeof SharedServer>()),+ traceException: mocks.traceException,+}));++vi.mock("@/src/ee/features/billing/utils/stripe", () => ({+ stripeClient: {+ checkout: { sessions: { list: mocks.listCheckoutSessions } },+ subscriptions: { update: vi.fn(), retrieve: vi.fn() },+ },+}));+const buildSubscription = (args: {orgId: string;status?: Stripe.Subscription.Status;@@ -103,3 +123,41 @@ describe("stripeWebhookHandler.handleSubscriptionChanged", () => {},);});++describe("stripeWebhookHandler org lookup miss severity", () => {+ beforeEach(() => {+ mocks.traceException.mockClear();+ // No checkout session, which is the shape a subscription created in the+ // Stripe dashboard rather than through our pricing page leaves behind.+ mocks.listCheckoutSessions.mockReset();+ mocks.listCheckoutSessions.mockResolvedValue({ data: [] });+ });++ it("does not mark the span as an error when the subscription carries no region marker", async () => {+ const subscription = buildSubscription({ orgId: v4() });+ subscription.metadata = {};++ await handleSubscriptionChanged(subscription, "updated");++ expect(mocks.traceException).not.toHaveBeenCalled();+ });++ it("does not mark the span as an error when the org row is already gone on subscription.deleted", async () => {+ // Metadata names this region, but the org row is gone: organization+ // deletion cancels the subscription and then deletes the row, so Stripe+ // delivers customer.subscription.deleted after the row is already gone.+ const subscription = buildSubscription({ orgId: v4() });++ await handleSubscriptionChanged(subscription, "deleted");++ expect(mocks.traceException).not.toHaveBeenCalled();+ });++ it("marks the span as an error once when the owning region cannot resolve the org on subscription.updated", async () => {+ const subscription = buildSubscription({ orgId: v4() });++ await handleSubscriptionChanged(subscription, "updated");++ expect(mocks.traceException).toHaveBeenCalledTimes(1);+ });+});diff --git a/web/src/ee/features/billing/server/stripe/stripeWebhookHandler.ts b/web/src/ee/features/billing/server/stripe/stripeWebhookHandler.tsindex 835c36f120..6db7b7e0e9 100644--- a/web/src/ee/features/billing/server/stripe/stripeWebhookHandler.ts+++ b/web/src/ee/features/billing/server/stripe/stripeWebhookHandler.ts@@ -11,6 +11,7 @@ import {CloudConfigSchema,InternalServerError,type Organization,+ type ParsedOrganization,type Plan,parseDbOrg,} from "@langfuse/shared";@@ -168,6 +169,14 @@ async function getOrgBasedOnActiveSubscriptionId(return orgBasedOnSubscriptionId;}+/**+ * Whether a Stripe object names this cloud region, another one, or none at all.+ * Only Stripe itself is shared between regions, so these markers are the only+ * ownership information a region can read without access to another region's+ * database.+ */+type RegionMarker = "own-region" | "other-region" | "none";+/*** Fallback method to find an organization using the checkout session attached to a subscription.* Used primarily for new subscriptions where the subscription ID hasn't been saved to the org yet.@@ -179,11 +188,12 @@ async function getOrgBasedOnActiveSubscriptionId(* 4. Looks up the organization** @param subscriptionId - The Stripe subscription ID to look up- * @returns The organization associated with the checkout session, or null if not found+ * @returns The organization associated with the checkout session (null if not+ * found) plus the region marker the session carries*/async function getOrgBasedOnCheckoutSessionAttachedToSubscription(subscriptionId: string,-): Promise<Organization | null> {+): Promise<{ organization: Organization | null; marker: RegionMarker }> {// get the checkout session from the subscription to retrieve the client reference for this subscriptionconst checkoutSessionsResponse = await stripeClient?.checkout.sessions.list({subscription: subscriptionId,@@ -191,7 +201,7 @@ async function getOrgBasedOnCheckoutSessionAttachedToSubscription(});if (!checkoutSessionsResponse || checkoutSessionsResponse.data.length !== 1) {logger.warn("[Stripe Webhook] No checkout session found");- return null;+ return { organization: null, marker: "none" };}const checkoutSession = checkoutSessionsResponse.data[0];@@ -199,13 +209,13 @@ async function getOrgBasedOnCheckoutSessionAttachedToSubscription(const clientReference = checkoutSession.client_reference_id;if (!clientReference) {logger.warn("[Stripe Webhook] No client reference");- return null;+ return { organization: null, marker: "none" };}if (!isStripeClientReferenceFromCurrentCloudRegion(clientReference)) {logger.info("[Stripe Webhook] Client reference not from current cloud region",);- return null;+ return { organization: null, marker: "other-region" };}const orgId = getOrgIdFromStripeClientReference(clientReference);@@ -215,26 +225,46 @@ async function getOrgBasedOnCheckoutSessionAttachedToSubscription(id: orgId,},});- return organization;+ return { organization, marker: "own-region" };}+/**+ * Why the organization for a subscription could not be resolved. The+ * distinction decides whether a human has to look: Stripe delivers every event+ * to all cloud regions, so a miss in a region that does not own the+ * subscription is the normal case, not a fault.+ */+type OrgLookupMiss =+ /** A region marker exists and names a different region: expected fan-out. */+ | "other-region"+ /** A marker names this region, but the organization row is gone. */+ | "org-row-missing"+ /** No region marker at all, so no region can own this event. */+ | "unattributable";++type OrgLookupResult =+ | { found: true; org: ParsedOrganization }+ | { found: false; miss: OrgLookupMiss };+/*** Resolve the organization for a given subscription using layered fallbacks:* 1) by active subscription id* 2) by Stripe customer id* 3) by checkout session attached to the subscription* 4) by subscription.metadata.orgId (last, because there might be a mismatch)- * Returns parsed org or null if not found (caller should log/return).+ *+ * Never logs a miss itself: only the caller knows whether a missing+ * organization is expected in its context. Use `logOrgLookupMiss` for that.*/async function getOrgForSubscriptionWithFallbacks(subscription: Stripe.Subscription,-) {+): Promise<OrgLookupResult> {const subscriptionId = subscription.id;// 1) by active subscription idlet organization = await getOrgBasedOnActiveSubscriptionId(subscriptionId);if (organization) {- return parseDbOrg(organization);+ return { found: true, org: parseDbOrg(organization) };}// 2) by Stripe customer id@@ -245,15 +275,15 @@ async function getOrgForSubscriptionWithFallbacks(if (customerId) {organization = await getOrgBasedOnCustomerId(customerId);if (organization) {- return parseDbOrg(organization);+ return { found: true, org: parseDbOrg(organization) };}}// 3) by checkout session attached to the subscription- organization =+ const checkoutSession =await getOrgBasedOnCheckoutSessionAttachedToSubscription(subscriptionId);- if (organization) {- return parseDbOrg(organization);+ if (checkoutSession.organization) {+ return { found: true, org: parseDbOrg(checkoutSession.organization) };}// 4) by metadata.orgId@@ -261,17 +291,71 @@ async function getOrgForSubscriptionWithFallbacks(if (metadataOrgId) {organization = await getOrgById(metadataOrgId);if (organization) {- return parseDbOrg(organization);+ return { found: true, org: parseDbOrg(organization) };}}- logger.error(- `[Stripe Webhook] getOrgForSubscriptionWithFallbacks: Organization not found for subscription ${subscriptionId}`,- );- traceException(- `[Stripe Webhook] getOrgForSubscriptionWithFallbacks: Organization not found for subscription ${subscriptionId}`,- );- return null;+ // Subscription metadata is the stronger marker: it is written by the region+ // that owns the subscription, while a checkout session only exists for+ // subscriptions that went through our pricing page.+ const metadataRegion = subscription.metadata?.cloudRegion;+ const marker: RegionMarker = metadataRegion+ ? metadataRegion === env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION+ ? "own-region"+ : "other-region"+ : checkoutSession.marker;++ switch (marker) {+ case "own-region":+ return { found: false, miss: "org-row-missing" };+ case "other-region":+ return { found: false, miss: "other-region" };+ case "none":+ return { found: false, miss: "unattributable" };+ }+}++/**+ * Report an unresolved organization at a severity that matches whether anyone+ * has to act on it. Only `org-row-missing` raises a traced exception, so the+ * webhook endpoint's error rate stays a signal rather than a background hum of+ * expected cross-region misses.+ *+ * @param expected - Set by callers that know a missing organization needs no+ * action in their context, e.g. a deletion event that has nothing left to write+ */+function logOrgLookupMiss({+ miss,+ subscriptionId,+ context,+ expected = false,+}: {+ miss: OrgLookupMiss;+ subscriptionId: string;+ context: string;+ expected?: boolean;+}) {+ const message = `[Stripe Webhook] (${env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION}) ${context}: Organization not found for subscription ${subscriptionId} (${miss})`;++ switch (miss) {+ case "other-region":+ logger.info(message);+ return;+ case "unattributable":+ // Every region reports this same miss for the same subscription, so it is+ // a fleet-wide fact about the Stripe object rather than a regional fault.+ // Alerting belongs in billing reconciliation, not in one region's spans.+ logger.warn(message);+ return;+ case "org-row-missing":+ if (expected) {+ logger.info(message);+ return;+ }+ logger.error(message);+ traceException(message);+ return;+ }}/**@@ -310,15 +394,19 @@ async function ensureMetadataIsSetOnStripeSubscription(}try {- const parsedOrg = await getOrgForSubscriptionWithFallbacks(subscription);- if (!parsedOrg) {+ const lookup = await getOrgForSubscriptionWithFallbacks(subscription);+ if (!lookup.found) {// Note: all our production environments receive all webhooks from Stripe.- // Only one should handle the webhook; it is expected in 2/3 cases the organization is not found.- logger.info(- `[Stripe Webhook] (${currentEnvironment}) ensureMetadataIsSetOnStripeSubscription: Organization not found for subscription ${subscription.id} in Environment ${currentEnvironment}`,- );+ // Only one should handle the webhook, so all but one region are expected+ // not to find the organization here.+ logOrgLookupMiss({+ miss: lookup.miss,+ subscriptionId: subscription.id,+ context: "ensureMetadataIsSetOnStripeSubscription",+ });return;}+ const parsedOrg = lookup.org;logger.info(`[Stripe Webhook] (${currentEnvironment}) ensureMetadataIsSetOnStripeSubscription: Organization for subscription ${subscription.id} found in Environment ${currentEnvironment}`,);@@ -504,16 +592,20 @@ export async function handleSubscriptionChanged(const subscriptionId = subscription.id;- const parsedOrg = await getOrgForSubscriptionWithFallbacks(subscription);- if (!parsedOrg) {- logger.error(- `[Stripe Webhook] (${currentEnvironment}) Organization not found for subscription ${subscriptionId}`,- );- traceException(- `[Stripe Webhook] (${currentEnvironment}) Organization not found for subscription ${subscriptionId}`,- );+ const lookup = await getOrgForSubscriptionWithFallbacks(subscription);+ if (!lookup.found) {+ logOrgLookupMiss({+ miss: lookup.miss,+ subscriptionId,+ context: "handleSubscriptionChanged",+ // Organization deletion cancels the subscription before deleting the row,+ // so Stripe delivers customer.subscription.deleted once the row is+ // already gone. There is nothing left to write, and nothing to fix.+ expected: action === "deleted",+ });return;}+ const parsedOrg = lookup.org;if (parsedOrg.cloudConfig?.stripe?.activeSubscriptionId &&