🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(billing): trace ClickHouse Billing operations in APM and align plan codes (#16706)
Steffen Schmitz542261c30afe
diff --git a/packages/shared/src/features/entitlements/plans.ts b/packages/shared/src/features/entitlements/plans.tsindex 99d8dc6720..bba22f0a9e 100644--- a/packages/shared/src/features/entitlements/plans.ts+++ b/packages/shared/src/features/entitlements/plans.ts@@ -31,15 +31,19 @@ export const isPlan = (value: string): value is Plan =>plans.includes(value as Plan);/**- * ClickHouse Billing (CHB) identifies plans by code. These are the lowercased- * paid tiers of `cloudConfigPlans` — there is deliberately no code for the free- * tier, because a free org simply carries no CHB bundle.+ * ClickHouse Billing (CHB) identifies plans by code.** Lives here rather than next to the Stripe catalogue in web because* `cloudConfigSchema` validates stored plan codes against it, and shared cannot* import from web.*/-export const chbPlanCodes = ["core", "pro", "team", "enterprise"] as const;+export const chbPlanCodes = [+ "LANGFUSE_HOBBY",+ "LANGFUSE_CORE",+ "LANGFUSE_PRO",+ "LANGFUSE_PRO_TEAMS",+ "LANGFUSE_ENTERPRISE",+] as const;export type ChbPlanCode = (typeof chbPlanCodes)[number];@@ -49,8 +53,9 @@ export type ChbPlanCode = (typeof chbPlanCodes)[number];* code without a plan is a compile error.*/export const chbPlanCodeToPlan = {- core: "cloud:core",- pro: "cloud:pro",- team: "cloud:team",- enterprise: "cloud:enterprise",+ LANGFUSE_HOBBY: "cloud:hobby",+ LANGFUSE_CORE: "cloud:core",+ LANGFUSE_PRO: "cloud:pro",+ LANGFUSE_PRO_TEAMS: "cloud:team",+ LANGFUSE_ENTERPRISE: "cloud:enterprise",} as const satisfies Record<ChbPlanCode, Plan>;diff --git a/web/src/__tests__/server/chbCheckoutClaim.servertest.ts b/web/src/__tests__/server/chbCheckoutClaim.servertest.tsindex 696956ea4f..5ef0445d71 100644--- a/web/src/__tests__/server/chbCheckoutClaim.servertest.ts+++ b/web/src/__tests__/server/chbCheckoutClaim.servertest.ts@@ -27,7 +27,7 @@ import { ChbBillingService } from "@/src/ee/features/billing/server/chb/chbBilliimport { mapChbPlanCodeToStripeProductId } from "@/src/ee/features/billing/utils/chbCatalogue";import { type OrgAuthedContext } from "@/src/server/api/trpc";-const CORE_PRODUCT_ID = mapChbPlanCodeToStripeProductId("core")!;+const CORE_PRODUCT_ID = mapChbPlanCodeToStripeProductId("LANGFUSE_CORE")!;let orgId: string;let chOrganizationId: string;@@ -125,12 +125,17 @@ describe("chbBillingService checkout claim (postgres)", () => {it("merges into an existing clickhouse sub-object instead of replacing it", async () => {// A webhook may have written plan state before checkout returns; the claim// rebuilds the sub-object from the row, so that state must survive.- await setCloudConfig(JSON.stringify({ clickhouse: { planCode: "core" } }));+ await setCloudConfig(+ JSON.stringify({ clickhouse: { planCode: "LANGFUSE_CORE" } }),+ );await service().createCheckoutSession(orgId, CORE_PRODUCT_ID);expect(await parsedCloudConfig()).toMatchObject({- clickhouse: { organizationId: chOrganizationId, planCode: "core" },+ clickhouse: {+ organizationId: chOrganizationId,+ planCode: "LANGFUSE_CORE",+ },});});diff --git a/web/src/__tests__/server/unit/chbAccessToken.servertest.ts b/web/src/__tests__/server/unit/chbAccessToken.servertest.tsindex d1b39205a3..079dd455a4 100644--- a/web/src/__tests__/server/unit/chbAccessToken.servertest.ts+++ b/web/src/__tests__/server/unit/chbAccessToken.servertest.ts@@ -176,6 +176,14 @@ describe("chbAccessToken", () => {await expect(provider().getToken()).rejects.toBeInstanceOf(ChbAuthError);});+ it("hands back a token whose claims cannot be decoded", async () => {+ // The provider reads iss/aud off the token to tag the APM span. That is a+ // diagnostic, so a token it cannot parse must not fail the billing request.+ fetchMock.mockResolvedValue(grant("not.base64url-{{.sig"));++ await expect(provider().getToken()).resolves.toBe("not.base64url-{{.sig");+ });+it("retries the grant after a failure instead of latching onto it", async () => {fetchMock.mockResolvedValueOnce(jsonResponse(500, {}))diff --git a/web/src/__tests__/server/unit/chbApiClient.servertest.ts b/web/src/__tests__/server/unit/chbApiClient.servertest.tsindex a4bce494ad..fa98006040 100644--- a/web/src/__tests__/server/unit/chbApiClient.servertest.ts+++ b/web/src/__tests__/server/unit/chbApiClient.servertest.ts@@ -224,7 +224,7 @@ describe("chbApiClient", () => {);await client().createCheckoutSession({email: "[email protected]",- planCode: "pro",+ planCode: "LANGFUSE_PRO",returnUrl: "https://cloud.langfuse.com/back",});// Checkout creates the CH organization, so there is none to scope to yet.@@ -241,7 +241,7 @@ describe("chbApiClient", () => {change: {type: "downgrade",when: "billing_cycle_end",- planCode: "core",+ planCode: "LANGFUSE_CORE",},});const withBody = lastChbCall();@@ -249,7 +249,7 @@ describe("chbApiClient", () => {expect(JSON.parse(withBody.init.body as string)).toEqual({type: "downgrade",when: "billing_cycle_end",- planCode: "core",+ planCode: "LANGFUSE_CORE",});onChb(jsonResponse(202, {}));@@ -293,7 +293,11 @@ describe("chbApiClient", () => {.setScheduledChange({chOrganizationId: CH_ORG_ID,bundleId: "bundle_1",- change: { type: "upgrade", when: "immediate", planCode: "team" },+ change: {+ type: "upgrade",+ when: "immediate",+ planCode: "LANGFUSE_PRO_TEAMS",+ },}).catch((e) => e);@@ -344,7 +348,11 @@ describe("chbApiClient", () => {onChb(jsonResponse(200, {id: "bundle_1",- plan: { planCode: "team", tierName: "Team", extra: true },+ plan: {+ planCode: "LANGFUSE_PRO_TEAMS",+ tierName: "Team",+ extra: true,+ },period: { startDate: "2026-08-01T00:00:00Z" },payment: { status: "active", provider: { customerId: "cus_1" } },unknownTopLevel: { nested: 1 },@@ -357,7 +365,7 @@ describe("chbApiClient", () => {});expect(bundle.id).toBe("bundle_1");- expect(bundle.plan?.planCode).toBe("team");+ expect(bundle.plan?.planCode).toBe("LANGFUSE_PRO_TEAMS");expect(bundle.payment?.provider?.customerId).toBe("cus_1");// Nothing required beyond `id`, so a sparse bundle still parses.expect(bundle.scheduled).toBeUndefined();@@ -387,7 +395,7 @@ describe("chbApiClient", () => {await expect(client().createCheckoutSession({email: "[email protected]",- planCode: "pro",+ planCode: "LANGFUSE_PRO",returnUrl: "https://cloud.langfuse.com/back",}),).rejects.toThrow();diff --git a/web/src/__tests__/server/unit/chbBillingService.servertest.ts b/web/src/__tests__/server/unit/chbBillingService.servertest.tsindex 66f2240b4a..570bbfa42f 100644--- a/web/src/__tests__/server/unit/chbBillingService.servertest.ts+++ b/web/src/__tests__/server/unit/chbBillingService.servertest.ts@@ -91,7 +91,7 @@ const chbConfig = (clickhouse: {organizationId: CH_ORG_ID,bundleId: BUNDLE_ID,- planCode: "pro",+ planCode: "LANGFUSE_PRO",...overrides,},});@@ -194,7 +194,7 @@ describe("chbBillingService", () => {scheduled: {type: "downgrade",when: "billing_cycle_end",- planCode: "core",+ planCode: "LANGFUSE_CORE",startDate: "2026-09-05T00:00:00Z",},});@@ -206,7 +206,7 @@ describe("chbBillingService", () => {scheduleId: `chb:${BUNDLE_ID}`,// An explicit startDate wins over the period end.switchAt: Date.parse("2026-09-05T00:00:00Z") / 1000,- newProductId: productIdFor("core"),+ newProductId: productIdFor("LANGFUSE_CORE"),message: null,});});@@ -255,7 +255,7 @@ describe("chbBillingService", () => {const url = await service().createCheckoutSession(ORG_ID,- productIdFor("pro"),+ productIdFor("LANGFUSE_PRO"),"op-checkout",);@@ -263,9 +263,9 @@ describe("chbBillingService", () => {expect(clientMock.createCheckoutSession).toHaveBeenCalledWith({organizationId: undefined,email: "[email protected]",- planCode: "pro",+ planCode: "LANGFUSE_PRO",returnUrl: `https://cloud.langfuse.com/organization/${ORG_ID}/settings/billing`,- idempotencyKey: `chb.checkout.create:orgId=${ORG_ID}:planCode=pro:op=op-checkout`,+ idempotencyKey: `chb.checkout.create:orgId=${ORG_ID}:planCode=LANGFUSE_PRO:op=op-checkout`,});// Claimed with the guarded statement, never a blind read-then-write.expect(executeRaw).toHaveBeenCalledTimes(1);@@ -281,13 +281,13 @@ describe("chbBillingService", () => {withOrg(null);await service().createCheckoutSession(ORG_ID,- productIdFor("core"),+ productIdFor("LANGFUSE_CORE"),"op-1",);withOrg(null);await service().createCheckoutSession(ORG_ID,- productIdFor("team"),+ productIdFor("LANGFUSE_PRO_TEAMS"),"op-1",);@@ -301,7 +301,10 @@ describe("chbBillingService", () => {withOrg(null);clientMock.createCheckoutSession.mockResolvedValue(session);- await service().createCheckoutSession(ORG_ID, productIdFor("pro"));+ await service().createCheckoutSession(+ ORG_ID,+ productIdFor("LANGFUSE_PRO"),+ );// Matches makeIdempotencyKey's contract: no client opId, no claim.expect(clientMock.createCheckoutSession).toHaveBeenCalledWith(@@ -313,13 +316,16 @@ describe("chbBillingService", () => {withOrg({ clickhouse: { organizationId: CH_ORG_ID } });clientMock.createCheckoutSession.mockResolvedValue(session);- await service().createCheckoutSession(ORG_ID, productIdFor("team"));+ await service().createCheckoutSession(+ ORG_ID,+ productIdFor("LANGFUSE_PRO_TEAMS"),+ );// A retry must recover the same CH org instead of orphaning one.expect(clientMock.createCheckoutSession).toHaveBeenCalledWith(expect.objectContaining({organizationId: CH_ORG_ID,- planCode: "team",+ planCode: "LANGFUSE_PRO_TEAMS",}),);// Already stored, so the retry claims nothing.@@ -338,7 +344,10 @@ describe("chbBillingService", () => {// the org at a bundle it never bought.expect(await trpcCode(- service().createCheckoutSession(ORG_ID, productIdFor("team")),+ service().createCheckoutSession(+ ORG_ID,+ productIdFor("LANGFUSE_PRO_TEAMS"),+ ),),).toBe("INTERNAL_SERVER_ERROR");expect(executeRaw).not.toHaveBeenCalled();@@ -356,7 +365,11 @@ describe("chbBillingService", () => {// make parseDbOrg null the whole cloudConfig on every later read.expect(await trpcCode(- service().createCheckoutSession(ORG_ID, productIdFor("pro"), "op-1"),+ service().createCheckoutSession(+ ORG_ID,+ productIdFor("LANGFUSE_PRO"),+ "op-1",+ ),),).toBe("INTERNAL_SERVER_ERROR");expect(executeRaw).not.toHaveBeenCalled();@@ -371,7 +384,11 @@ describe("chbBillingService", () => {// resolves to a CH org the Langfuse row no longer points at.expect(await trpcCode(- service().createCheckoutSession(ORG_ID, productIdFor("pro"), "op-1"),+ service().createCheckoutSession(+ ORG_ID,+ productIdFor("LANGFUSE_PRO"),+ "op-1",+ ),),).toBe("CONFLICT");expect(mocks.auditLog).not.toHaveBeenCalled();@@ -388,7 +405,11 @@ describe("chbBillingService", () => {);await expect(- service().createCheckoutSession(ORG_ID, productIdFor("pro"), "op-1"),+ service().createCheckoutSession(+ ORG_ID,+ productIdFor("LANGFUSE_PRO"),+ "op-1",+ ),).rejects.toThrow();expect(executeRaw).not.toHaveBeenCalled();expect(update).not.toHaveBeenCalled();@@ -399,7 +420,7 @@ describe("chbBillingService", () => {expect(await trpcCode(- service().createCheckoutSession(ORG_ID, productIdFor("pro")),+ service().createCheckoutSession(ORG_ID, productIdFor("LANGFUSE_PRO")),),).toBe("INTERNAL_SERVER_ERROR");expect(clientMock.createCheckoutSession).not.toHaveBeenCalled();@@ -413,7 +434,7 @@ describe("chbBillingService", () => {expect(await trpcCode(- service().createCheckoutSession(ORG_ID, productIdFor("pro")),+ service().createCheckoutSession(ORG_ID, productIdFor("LANGFUSE_PRO")),),).toBe("INTERNAL_SERVER_ERROR");// An org bills through exactly one provider.@@ -432,30 +453,38 @@ describe("chbBillingService", () => {describe("changePlan", () => {it("applies an upgrade immediately", async () => {- withOrg(chbConfig({ planCode: "core" }));+ withOrg(chbConfig({ planCode: "LANGFUSE_CORE" }));- await service().changePlan(ORG_ID, productIdFor("team"), "op-1");+ await service().changePlan(+ ORG_ID,+ productIdFor("LANGFUSE_PRO_TEAMS"),+ "op-1",+ );expect(clientMock.setScheduledChange).toHaveBeenCalledWith({chOrganizationId: CH_ORG_ID,bundleId: BUNDLE_ID,- change: { type: "upgrade", when: "immediate", planCode: "team" },+ change: {+ type: "upgrade",+ when: "immediate",+ planCode: "LANGFUSE_PRO_TEAMS",+ },idempotencyKey:- "chb.bundle.scheduled.set:bundleId=bundle_1:to=team:op=op-1",+ "chb.bundle.scheduled.set:bundleId=bundle_1:to=LANGFUSE_PRO_TEAMS:op=op-1",});});it("defers a downgrade to the end of the billing cycle", async () => {- withOrg(chbConfig({ planCode: "team" }));+ withOrg(chbConfig({ planCode: "LANGFUSE_PRO_TEAMS" }));- await service().changePlan(ORG_ID, productIdFor("core"), "op-2");+ await service().changePlan(ORG_ID, productIdFor("LANGFUSE_CORE"), "op-2");expect(clientMock.setScheduledChange).toHaveBeenCalledWith(expect.objectContaining({change: {type: "downgrade",when: "billing_cycle_end",- planCode: "core",+ planCode: "LANGFUSE_CORE",},}),);@@ -464,13 +493,17 @@ describe("chbBillingService", () => {it("treats an org with no stored plan code as upgrading", async () => {withOrg(chbConfig({ planCode: null }));- await service().changePlan(ORG_ID, productIdFor("core"));+ await service().changePlan(ORG_ID, productIdFor("LANGFUSE_CORE"));// Nothing to compare against, so apply now rather than stranding the org// on its current tier until the cycle ends.expect(clientMock.setScheduledChange).toHaveBeenCalledWith(expect.objectContaining({- change: { type: "upgrade", when: "immediate", planCode: "core" },+ change: {+ type: "upgrade",+ when: "immediate",+ planCode: "LANGFUSE_CORE",+ },// No opId → no key, matching makeIdempotencyKey's contract.idempotencyKey: undefined,}),@@ -484,7 +517,9 @@ describe("chbBillingService", () => {);expect(- await trpcCode(service().changePlan(ORG_ID, productIdFor("team"))),+ await trpcCode(+ service().changePlan(ORG_ID, productIdFor("LANGFUSE_PRO_TEAMS")),+ ),).toBe("PRECONDITION_FAILED");expect(mocks.auditLog).not.toHaveBeenCalled();});@@ -495,7 +530,7 @@ describe("chbBillingService", () => {clientMock.setScheduledChange.mockRejectedValue(boom);await expect(- service().changePlan(ORG_ID, productIdFor("team")),+ service().changePlan(ORG_ID, productIdFor("LANGFUSE_PRO_TEAMS")),).rejects.toBe(boom);});@@ -507,7 +542,9 @@ describe("chbBillingService", () => {withOrg(cloudConfig);expect(- await trpcCode(service().changePlan(ORG_ID, productIdFor("team"))),+ await trpcCode(+ service().changePlan(ORG_ID, productIdFor("LANGFUSE_PRO_TEAMS")),+ ),).toBe("INTERNAL_SERVER_ERROR");expect(clientMock.setScheduledChange).not.toHaveBeenCalled();});diff --git a/web/src/__tests__/server/unit/chbCatalogue.servertest.ts b/web/src/__tests__/server/unit/chbCatalogue.servertest.tsindex c6e045f850..ee310fda9a 100644--- a/web/src/__tests__/server/unit/chbCatalogue.servertest.ts+++ b/web/src/__tests__/server/unit/chbCatalogue.servertest.ts@@ -31,22 +31,28 @@ describe("chbCatalogue", () => {it.each([["an unknown plan code", "platinum"],["an empty plan code", ""],+ // CHB lists a Hobby tier, but a free Langfuse org carries no CHB bundle, so+ // no Stripe product maps to it. Since the outbound plan code is only ever+ // produced by this lookup, that is what keeps us from ever sending it.+ ["CHB's Hobby tier", "LANGFUSE_HOBBY"],])("returns null when bridging %s to a product id", (_label, planCode) => {expect(mapChbPlanCodeToStripeProductId(planCode)).toBeNull();});it("classifies upgrades and downgrades by the shared Stripe order keys", () => {- expect(isChbUpgrade("core", "pro")).toBe(true);- expect(isChbUpgrade("pro", "team")).toBe(true);- expect(isChbUpgrade("team", "core")).toBe(false);- expect(isChbUpgrade("enterprise", "enterprise")).toBe(false);+ expect(isChbUpgrade("LANGFUSE_CORE", "LANGFUSE_PRO")).toBe(true);+ expect(isChbUpgrade("LANGFUSE_PRO", "LANGFUSE_PRO_TEAMS")).toBe(true);+ expect(isChbUpgrade("LANGFUSE_PRO_TEAMS", "LANGFUSE_CORE")).toBe(false);+ expect(isChbUpgrade("LANGFUSE_ENTERPRISE", "LANGFUSE_ENTERPRISE")).toBe(+ false,+ );});it("does not treat a drifted plan code as an upgrade", () => {// A code CHB ships before we deploy support for it resolves to order key 0.// It must not read as an upgrade from a real tier, which would let a// downgrade apply immediately instead of at cycle end.- expect(isChbUpgrade("team", "platinum")).toBe(false);- expect(isChbUpgrade("platinum", "team")).toBe(true);+ expect(isChbUpgrade("LANGFUSE_PRO_TEAMS", "platinum")).toBe(false);+ expect(isChbUpgrade("platinum", "LANGFUSE_PRO_TEAMS")).toBe(true);});});diff --git a/web/src/__tests__/server/unit/getPlan.servertest.ts b/web/src/__tests__/server/unit/getPlan.servertest.tsindex a74de94513..8557f35964 100644--- a/web/src/__tests__/server/unit/getPlan.servertest.ts+++ b/web/src/__tests__/server/unit/getPlan.servertest.ts@@ -26,10 +26,11 @@ describe("getOrganizationPlanServerSide (cloud)", () => {});it.each([- ["core", "cloud:core"],- ["pro", "cloud:pro"],- ["team", "cloud:team"],- ["enterprise", "cloud:enterprise"],+ ["LANGFUSE_HOBBY", "cloud:hobby"],+ ["LANGFUSE_CORE", "cloud:core"],+ ["LANGFUSE_PRO", "cloud:pro"],+ ["LANGFUSE_PRO_TEAMS", "cloud:team"],+ ["LANGFUSE_ENTERPRISE", "cloud:enterprise"],])("maps CHB plan code %s to %s", (planCode, expected) => {expect(getOrganizationPlanServerSide(@@ -64,7 +65,7 @@ describe("getOrganizationPlanServerSide (cloud)", () => {getOrganizationPlanServerSide(cloudConfig({plan: "Enterprise",- clickhouse: { organizationId: CHB_ORG_ID, planCode: "core" },+ clickhouse: { organizationId: CHB_ORG_ID, planCode: "LANGFUSE_CORE" },}),),).toBe("cloud:enterprise");diff --git a/web/src/ee/features/billing/server/chb/chbAccessToken.ts b/web/src/ee/features/billing/server/chb/chbAccessToken.tsindex c62521bfff..38a7c7d6d0 100644--- a/web/src/ee/features/billing/server/chb/chbAccessToken.ts+++ b/web/src/ee/features/billing/server/chb/chbAccessToken.ts@@ -1,6 +1,7 @@+import { SpanKind, type Span } from "@opentelemetry/api";import { z } from "zod";-import { logger } from "@langfuse/shared/src/server";+import { instrumentAsync, logger } from "@langfuse/shared/src/server";/*** Auth0 client-credentials (M2M) token source for the ClickHouse Billing (CHB)@@ -39,6 +40,45 @@ export type ChbAccessTokenConfig = {audience: string;};+/**+ * Tag the span with the issuer and audience of the token we just minted, so an+ * `Invalid or expired token` rejection from CHB can be attributed to a tenant+ * or audience mismatch without reproducing the request.+ *+ * Reads the claims without verifying the signature — this is a diagnostic on a+ * token we just received from Auth0 over TLS, never an authorization decision.+ * The token itself is never tagged or logged.+ */+const tagTokenClaims = (span: Span, accessToken: string): void => {+ const segments = accessToken.split(".");+ // An opaque token means the requested audience is not a registered Auth0+ // API, and CHB's JWT verifier will reject it on arrival.+ span.setAttribute(+ "chb.auth.token_format",+ segments.length === 3 ? "jwt" : "opaque",+ );+ if (segments.length !== 3) return;++ try {+ const claims = JSON.parse(+ Buffer.from(segments[1]!, "base64url").toString("utf8"),+ ) as { iss?: unknown; aud?: unknown };+ if (typeof claims.iss === "string") {+ span.setAttribute("chb.auth.token_issuer", claims.iss);+ }+ const audience = Array.isArray(claims.aud) ? claims.aud : [claims.aud];+ const audienceStrings = audience.filter(+ (entry): entry is string => typeof entry === "string",+ );+ if (audienceStrings.length > 0) {+ span.setAttribute("chb.auth.token_audience", audienceStrings);+ }+ } catch {+ // A payload we cannot read is itself the signal; the format tag already+ // carries it, and a diagnostic must never fail the request.+ }+};+export class ChbAccessTokenProvider {private cached: { token: string; expiresAtMs: number } | null = null;/** Single-flight: a burst of billing requests must not stampede Auth0. */@@ -66,6 +106,20 @@ export class ChbAccessTokenProvider {}private async mintToken(): Promise<string> {+ return await instrumentAsync(+ { name: "chb.auth.token.mint", spanKind: SpanKind.CLIENT },+ async (span) => {+ span.setAttributes({+ "chb.auth.domain": this.config.auth0Domain,+ "chb.auth.audience": this.config.audience,+ "chb.auth.client_id": this.config.clientId,+ });+ return await this.requestToken(span);+ },+ );+ }++ private async requestToken(span: Span): Promise<string> {const response = await fetch(`https://${this.config.auth0Domain}/oauth/token`,{@@ -85,6 +139,8 @@ export class ChbAccessTokenProvider {},);+ span.setAttribute("http.status_code", response.status);+if (!response.ok) {// Deliberately not logging the body: Auth0 error payloads echo request// parameters back.@@ -118,6 +174,10 @@ export class ChbAccessTokenProvider {token: parsed.data.access_token,expiresAtMs: Date.now() + lifetimeSeconds * 1000,};++ span.setAttribute("chb.auth.expires_in_seconds", parsed.data.expires_in);+ tagTokenClaims(span, parsed.data.access_token);+return parsed.data.access_token;}}diff --git a/web/src/ee/features/billing/server/chb/chbApiClient.ts b/web/src/ee/features/billing/server/chb/chbApiClient.tsindex f61d3e26e4..d07c9fe1c3 100644--- a/web/src/ee/features/billing/server/chb/chbApiClient.ts+++ b/web/src/ee/features/billing/server/chb/chbApiClient.ts@@ -1,7 +1,8 @@+import { SpanKind, type Span } from "@opentelemetry/api";import { z } from "zod";import { env } from "@/src/env.mjs";-import { logger } from "@langfuse/shared/src/server";+import { instrumentAsync, logger } from "@langfuse/shared/src/server";import {ChbAccessTokenProvider,@@ -106,6 +107,12 @@ const ChbPortalSessionSchema = z.object({const REQUEST_TIMEOUT_MS = 15_000;type ChbRequestOptions = {+ /**+ * APM span name for the CHB operation, e.g. `chb.checkout_session.create`.+ * Named after the operation rather than the route so a path carrying an id+ * does not fan one operation out into unrelated APM resources.+ */+ operation: string;method: "GET" | "POST" | "PUT" | "DELETE";path: string;/** CH-Organization-Id header — required for org-scoped endpoints */@@ -140,33 +147,102 @@ export class ChbApiClient {return url;}- private async send(url: URL, opts: ChbRequestOptions): Promise<Response> {- return await fetch(url, {- method: opts.method,- headers: {- authorization: `Bearer ${await this.tokenProvider.getToken()}`,- ...(opts.chOrganizationId- ? { "CH-Organization-Id": opts.chOrganizationId }- : {}),- ...(opts.body !== undefined- ? { "content-type": "application/json" }- : {}),- ...(opts.idempotencyKey- ? { "Idempotency-Key": opts.idempotencyKey }- : {}),+ /**+ * One span per HTTP attempt, so a token replay shows as two attempt spans —+ * each with the token mint it triggered nested underneath — rather than as+ * one opaque slow call.+ */+ private async send(+ url: URL,+ opts: ChbRequestOptions,+ attempt: number,+ ): Promise<Response> {+ return await instrumentAsync(+ { name: "chb.api.request", spanKind: SpanKind.CLIENT },+ async (span) => {+ span.setAttributes({+ "http.method": opts.method,+ "http.route": opts.path,+ "peer.hostname": url.hostname,+ "chb.operation": opts.operation,+ "chb.attempt": attempt,+ });++ const response = await fetch(url, {+ method: opts.method,+ headers: {+ authorization: `Bearer ${await this.tokenProvider.getToken()}`,+ ...(opts.chOrganizationId+ ? { "CH-Organization-Id": opts.chOrganizationId }+ : {}),+ ...(opts.body !== undefined+ ? { "content-type": "application/json" }+ : {}),+ ...(opts.idempotencyKey+ ? { "Idempotency-Key": opts.idempotencyKey }+ : {}),+ },+ body: opts.body !== undefined ? JSON.stringify(opts.body) : undefined,+ // The base URL is operator-configured; an unexpected redirect is an+ // error, not something to follow with a bearer token attached.+ redirect: "error",+ signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),+ });++ span.setAttribute("http.status_code", response.status);+ return response;},- body: opts.body !== undefined ? JSON.stringify(opts.body) : undefined,- // The base URL is operator-configured; an unexpected redirect is an- // error, not something to follow with a bearer token attached.- redirect: "error",- signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),- });+ );}private async request(opts: ChbRequestOptions): Promise<unknown> {+ const outcome = await instrumentAsync(+ { name: opts.operation, spanKind: SpanKind.CLIENT },+ async (span) => {+ span.setAttributes({+ "http.method": opts.method,+ "http.route": opts.path,+ ...(opts.chOrganizationId+ ? { "chb.ch_organization_id": opts.chOrganizationId }+ : {}),+ ...(opts.idempotencyKey+ ? { "chb.idempotency_key": opts.idempotencyKey }+ : {}),+ });++ try {+ return {+ thrown: undefined,+ body: await this.sendWithReplay(opts, span),+ };+ } catch (error) {+ // A 409 is the "organization needs a payment method" UX branch, not+ // an incident — which is why it is deliberately not logged as an+ // error either. Carrying it out of the span instead of throwing+ // through it keeps instrumentAsync from marking the operation as an+ // APM error, so a routine checkout prompt cannot inflate the error+ // rate this instrumentation exists to make readable.+ if (error instanceof ChbPaymentRequiredError) {+ span.setAttribute("chb.payment_required", true);+ return { thrown: error, body: undefined };+ }+ throw error;+ }+ },+ );++ // Rethrown outside the span so the caller contract is unchanged.+ if (outcome.thrown) throw outcome.thrown;+ return outcome.body;+ }++ private async sendWithReplay(+ opts: ChbRequestOptions,+ span: Span,+ ): Promise<unknown> {const url = this.requestUrl(opts);- let response = await this.send(url, opts);+ let response = await this.send(url, opts, 1);if (response.status === 401) {// CHB rejected a token we still considered fresh — revoked, or the Auth0// signing key rotated. A 401 means CHB did no work, and the retry carries@@ -175,10 +251,13 @@ export class ChbApiClient {method: opts.method,path: opts.path,});+ span.setAttribute("chb.token_replayed", true);this.tokenProvider.invalidate();- response = await this.send(url, opts);+ response = await this.send(url, opts, 2);}+ span.setAttribute("http.status_code", response.status);+const responseBody = await response.json().catch(() => undefined as unknown);@@ -211,6 +290,7 @@ export class ChbApiClient {idempotencyKey?: string;}): Promise<ChbCheckoutSession> {const body = await this.request({+ operation: "chb.checkout_session.create",method: "POST",path: "checkout-sessions",body: {@@ -234,6 +314,7 @@ export class ChbApiClient {bundleId: string;}): Promise<ChbBundle> {const body = await this.request({+ operation: "chb.bundle.get",method: "GET",path: `bundles/${encodeURIComponent(params.bundleId)}`,chOrganizationId: params.chOrganizationId,@@ -254,6 +335,7 @@ export class ChbApiClient {idempotencyKey?: string;}): Promise<void> {await this.request({+ operation: "chb.bundle.scheduled.set",method: "PUT",path: `bundles/${encodeURIComponent(params.bundleId)}/scheduled`,chOrganizationId: params.chOrganizationId,@@ -269,6 +351,7 @@ export class ChbApiClient {idempotencyKey?: string;}): Promise<void> {await this.request({+ operation: "chb.bundle.scheduled.clear",method: "DELETE",path: `bundles/${encodeURIComponent(params.bundleId)}/scheduled`,chOrganizationId: params.chOrganizationId,@@ -281,6 +364,7 @@ export class ChbApiClient {bundleId: string;}): Promise<ChbInvoice[]> {const body = await this.request({+ operation: "chb.invoices.list",method: "GET",path: "invoices",chOrganizationId: params.chOrganizationId,@@ -294,6 +378,7 @@ export class ChbApiClient {returnUrl: string;}): Promise<string> {const body = await this.request({+ operation: "chb.portal_session.create",method: "POST",path: "portal-sessions",chOrganizationId: params.chOrganizationId,