🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(billing): CHB project events to event bus (#15461)
Steffen Schmitzfa243fa4fba0
diff --git a/.env.dev.example b/.env.dev.exampleindex 69b6940ad0..595d8b1598 100644--- a/.env.dev.example+++ b/.env.dev.example@@ -263,3 +263,6 @@ CLICKHOUSE_DISABLE_TOP_K_THROUGH_JOIN=auto# LANGFUSE_CLOUD_BILLING_CHB_CUTOFF_DATE=# Bearer secret for GET /api/billing/metrics; unset = endpoint disabled.# CLICKHOUSE_BILLING_METRICS_API_KEY=+# CHB's EventBridge bus for project lifecycle events, authenticated by IAM from+# the task role; unset = no events emitted.+# CLICKHOUSE_BILLING_EVENT_BUS_ARN=diff --git a/.env.prod.example b/.env.prod.exampleindex 41f31fb6c1..57680fec41 100644--- a/.env.prod.example+++ b/.env.prod.example@@ -390,3 +390,6 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true# LANGFUSE_CLOUD_BILLING_CHB_CUTOFF_DATE=# Bearer secret for GET /api/billing/metrics; unset = endpoint disabled.# CLICKHOUSE_BILLING_METRICS_API_KEY=+# CHB's EventBridge bus for project lifecycle events, authenticated by IAM from+# the task role; unset = no events emitted.+# CLICKHOUSE_BILLING_EVENT_BUS_ARN=diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 494d4576ce..a7984a2081 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -9,6 +9,9 @@ catalogs:'@aws-sdk/client-cloudwatch':specifier: 3.1074.0version: 3.1074.0+ '@aws-sdk/client-eventbridge':+ specifier: 3.1074.0+ version: 3.1074.0'@aws-sdk/client-lambda':specifier: 3.1074.0version: 3.1074.0@@ -474,6 +477,9 @@ importers:'@aws-crypto/sha256-browser':specifier: ^5.2.0version: 5.2.0+ '@aws-sdk/client-eventbridge':+ specifier: 'catalog:'+ version: 3.1074.0'@baselime/trpc-opentelemetry-middleware':specifier: ^0.1.2@@ -723,6 +729,9 @@ importers:elkjs:specifier: ^0.11.1version: 0.11.1+ exponential-backoff:+ specifier: ^3.1.2+ version: 3.1.2fastest-levenshtein:specifier: ^1.0.16version: 1.0.16@@ -1459,6 +1468,14 @@ packages:resolution: {integrity: sha512-+Ty97ACrY0BEH/aWczd5Vs7VK/MAVpUGatxbv4Rkwkk5llLAHLvqJPwp8QUTQpa7DRuMuyUWbyMwIghLVf81MA==}engines: {node: '>=20.0.0'}+ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-6Fhu/JOC+zifM51+1Qx4N2mC/NHkulY87RFnnMcpgN89v3VgxVTVPvUcphcFIakEEiP7NtvkqSVicg5M8LsRMw==}+ engines: {node: '>=20.0.0'}++ '@aws-sdk/[email protected]':+ resolution: {integrity: sha512-KYd1SYF+WaZinfzg4rsfprZQ3X1qboaJnHtjppd82JG3YSO/LVkenOkFIIsREPnXMoy6Dy6Xx+BXh4Y9A+a1zQ==}+ engines: {node: '>=20.0.0'}+'@aws-sdk/[email protected]':resolution: {integrity: sha512-h4NyvOj6jzgcnsZPD+ycJxCMUTFqmY5IpDuBu+MWpNBKZdYxu0L7lUVF7tdI2HeibjVuN+xdtArgxRcovUcobQ==}engines: {node: '>=20.0.0'}@@ -11497,6 +11514,65 @@ snapshots:'@smithy/types': 4.16.1tslib: 2.8.1+ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-crypto/sha256-browser': 5.2.0+ '@aws-crypto/sha256-js': 5.2.0+ '@aws-sdk/core': 3.975.1+ '@aws-sdk/credential-provider-node': 3.972.66+ '@aws-sdk/signature-v4-multi-region': 3.996.39+ '@aws-sdk/types': 3.974.0+ '@smithy/core': 3.30.0+ '@smithy/fetch-http-handler': 5.6.5+ '@smithy/node-http-handler': 4.9.5+ '@smithy/types': 4.16.1+ tslib: 2.8.1++ '@aws-sdk/[email protected]':+ dependencies:+ '@aws-crypto/sha256-browser': 5.2.0+ '@aws-crypto/sha256-js': 5.2.0+ '@aws-sdk/core': 3.975.1+ '@aws-sdk/credential-provider-node': 3.972.66+ '@aws-sdk/middleware-host-header': 3.972.11+ '@aws-sdk/middleware-logger': 3.972.10+ '@aws-sdk/middleware-recursion-detection': 3.972.12+ '@aws-sdk/middleware-user-agent': 3.972.39+ '@aws-sdk/region-config-resolver': 3.972.14+ '@aws-sdk/types': 3.974.0+ '@aws-sdk/util-endpoints': 3.996.9+ '@aws-sdk/util-user-agent-browser': 3.972.11+ '@aws-sdk/util-user-agent-node': 3.973.25+ '@smithy/config-resolver': 4.5.0+ '@smithy/core': 3.30.0+ '@smithy/fetch-http-handler': 5.6.5+ '@smithy/hash-node': 4.3.0+ '@smithy/invalid-dependency': 4.3.0+ '@smithy/middleware-content-length': 4.3.0+ '@smithy/middleware-endpoint': 4.5.0+ '@smithy/middleware-retry': 4.6.0+ '@smithy/middleware-serde': 4.3.0+ '@smithy/middleware-stack': 4.3.0+ '@smithy/node-config-provider': 4.4.0+ '@smithy/node-http-handler': 4.5.1+ '@smithy/protocol-http': 5.4.0+ '@smithy/smithy-client': 4.13.0+ '@smithy/types': 4.16.1+ '@smithy/url-parser': 4.3.0+ '@smithy/util-base64': 4.3.2+ '@smithy/util-body-length-browser': 4.2.2+ '@smithy/util-body-length-node': 4.2.3+ '@smithy/util-defaults-mode-browser': 4.4.0+ '@smithy/util-defaults-mode-node': 4.3.0+ '@smithy/util-endpoints': 3.5.0+ '@smithy/util-middleware': 4.3.0+ '@smithy/util-retry': 4.4.0+ '@smithy/util-utf8': 4.4.14+ tslib: 2.8.1+ transitivePeerDependencies:+ - aws-crt+ optional: true+'@aws-sdk/[email protected]':dependencies:'@aws-crypto/sha256-browser': 5.2.0diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yamlindex b70538720a..79cd1024a7 100644--- a/pnpm-workspace.yaml+++ b/pnpm-workspace.yaml@@ -6,6 +6,7 @@ packages:- "ee"catalog:"@aws-sdk/client-cloudwatch": 3.1074.0+ "@aws-sdk/client-eventbridge": 3.1074.0"@aws-sdk/client-lambda": 3.1074.0"@aws-sdk/client-lambda-microvms": 3.1074.0"@aws-sdk/client-s3": 3.1074.0diff --git a/web/package.json b/web/package.jsonindex 3a32b26f2a..1701b47af2 100644--- a/web/package.json+++ b/web/package.json@@ -38,6 +38,7 @@"@appsignal/opentelemetry-instrumentation-bullmq": "^0.8.0","@astral-sh/ruff-wasm-web": "0.15.13","@aws-crypto/sha256-browser": "^5.2.0",+ "@aws-sdk/client-eventbridge": "catalog:","@baselime/trpc-opentelemetry-middleware": "^0.1.2","@codemirror/autocomplete": "^6.17.0","@codemirror/lang-javascript": "^6.2.5",@@ -121,6 +122,7 @@"diff": "^8.0.4","dompurify": "^3.4.12","elkjs": "^0.11.1",+ "exponential-backoff": "^3.1.2","fastest-levenshtein": "^1.0.16","https-proxy-agent": "^7.0.6","ioredis": "^5.10.1",diff --git a/web/src/__tests__/server/projectsRouter-chbProjectEvents.servertest.ts b/web/src/__tests__/server/projectsRouter-chbProjectEvents.servertest.tsnew file mode 100644index 0000000000..661fb40341--- /dev/null+++ b/web/src/__tests__/server/projectsRouter-chbProjectEvents.servertest.ts@@ -0,0 +1,183 @@+import { type Plan, Role } from "@langfuse/shared";+import { prisma } from "@langfuse/shared/src/db";+import type { Session } from "next-auth";+import { v4 as uuidv4 } from "uuid";+import { beforeEach, describe, expect, it, vi } from "vitest";++const mocks = vi.hoisted(() => ({ emit: vi.fn() }));++// The emit itself is fire-and-forget inside the helper; what this file pins is+// which lifecycle mutations call it, and with which org on each side.+vi.mock("@/src/ee/features/billing/server/chb/chbProjectEvents", () => ({+ emitChbProjectEvent: mocks.emit,+}));++import { appRouter } from "@/src/server/api/root";+import { createInnerTRPCContext } from "@/src/server/api/trpc";++async function createOrg(label: string) {+ const orgId = uuidv4();+ await prisma.organization.create({+ data: { id: orgId, name: `${label} ${orgId.slice(0, 8)}` },+ });+ return orgId;+}++async function createProject(orgId: string) {+ const projectId = uuidv4();+ await prisma.project.create({+ data: {+ id: projectId,+ name: `CHB Project ${projectId.slice(0, 8)}`,+ orgId,+ },+ });+ return projectId;+}++async function createUserInOrgs(orgIds: string[]) {+ const userId = uuidv4();+ const user = await prisma.user.create({+ data: {+ id: userId,+ email: `chb-events-${userId.slice(0, 8)}@test.com`,+ name: "Test User",+ },+ });+ for (const orgId of orgIds) {+ await prisma.organizationMembership.create({+ data: { userId: user.id, orgId, role: Role.OWNER },+ });+ }+ return user;+}++function makeCaller({+ userId,+ orgIds,+ projectId,+ plan = "cloud:pro",+}: {+ userId: string;+ orgIds: string[];+ projectId?: string;+ plan?: Plan;+}) {+ const session: Session = {+ expires: "1",+ user: {+ id: userId,+ canCreateOrganizations: true,+ name: "Test User",+ email: "[email protected]",+ organizations: orgIds.map((orgId) => ({+ id: orgId,+ name: "Test Organization",+ role: Role.OWNER,+ plan,+ cloudConfig: undefined,+ metadata: {},+ aiFeaturesEnabled: false,+ aiTelemetryEnabled: true,+ projects: projectId+ ? [+ {+ id: projectId,+ role: Role.OWNER,+ retentionDays: 0,+ deletedAt: null,+ hasTraces: false,+ name: "Test Project",+ metadata: {},+ createdAt: new Date().toISOString(),+ },+ ]+ : [],+ })),+ featureFlags: {+ searchBar: false,+ excludeClickhouseRead: false,+ templateFlag: true,+ v4BetaToggleVisible: false,+ observationEvals: false,+ experimentsV4Enabled: false,+ },+ admin: false,+ },+ environment: {} as any,+ };+ const ctx = createInnerTRPCContext({ session, headers: {} });+ return appRouter.createCaller({ ...ctx, prisma });+}++describe("projectsRouter CHB project lifecycle events", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ });++ it("emits a delete for the source org and a create for the destination on transfer", async () => {+ const sourceOrgId = await createOrg("CHB Source Org");+ const targetOrgId = await createOrg("CHB Target Org");+ const projectId = await createProject(sourceOrgId);+ const user = await createUserInOrgs([sourceOrgId, targetOrgId]);++ const caller = makeCaller({+ userId: user.id,+ orgIds: [sourceOrgId, targetOrgId],+ projectId,+ });++ await caller.projects.transfer({ projectId, targetOrgId });++ // CHB's registry is keyed by (organizationId, projectId): without both+ // sides the source org keeps being metered for a project it no longer owns.+ expect(mocks.emit).toHaveBeenCalledWith({+ type: "LANGFUSE_PROJECT_DELETED",+ orgId: sourceOrgId,+ projectId,+ });+ expect(mocks.emit).toHaveBeenCalledWith({+ type: "LANGFUSE_PROJECT_CREATED",+ orgId: targetOrgId,+ projectId,+ });+ expect(mocks.emit).toHaveBeenCalledTimes(2);++ const project = await prisma.project.findUnique({+ where: { id: projectId },+ });+ expect(project?.orgId).toBe(targetOrgId);+ });++ it("emits a create on project creation", async () => {+ const orgId = await createOrg("CHB Create Org");+ const user = await createUserInOrgs([orgId]);+ const caller = makeCaller({ userId: user.id, orgIds: [orgId] });++ const project = await caller.projects.create({+ name: "New CHB Project",+ orgId,+ });++ expect(mocks.emit).toHaveBeenCalledWith({+ type: "LANGFUSE_PROJECT_CREATED",+ orgId,+ projectId: project.id,+ });+ });++ it("emits a delete when the project is soft-deleted", async () => {+ const orgId = await createOrg("CHB Delete Org");+ const projectId = await createProject(orgId);+ const user = await createUserInOrgs([orgId]);+ const caller = makeCaller({ userId: user.id, orgIds: [orgId], projectId });++ await caller.projects.delete({ projectId });++ expect(mocks.emit).toHaveBeenCalledWith({+ type: "LANGFUSE_PROJECT_DELETED",+ orgId,+ projectId,+ });+ });+});diff --git a/web/src/__tests__/server/unit/chb-project-events.servertest.ts b/web/src/__tests__/server/unit/chb-project-events.servertest.tsnew file mode 100644index 0000000000..324b06fe5f--- /dev/null+++ b/web/src/__tests__/server/unit/chb-project-events.servertest.ts@@ -0,0 +1,335 @@+import type * as SharedServer from "@langfuse/shared/src/server";+import { beforeEach, describe, expect, it, vi } from "vitest";++const EVENT_BUS_ARN =+ "arn:aws:events:eu-central-1:720474339533:event-bus/control-plane-events";++const mocks = vi.hoisted(() => ({+ env: {+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "eu" as string | undefined,+ CLICKHOUSE_BILLING_EVENT_BUS_ARN:+ "arn:aws:events:eu-central-1:720474339533:event-bus/control-plane-events" as+ | string+ | undefined,+ },+ findOrg: vi.fn(),+ findProjects: vi.fn(),+ send: vi.fn(),+ recordIncrement: vi.fn(),+ // Plain array, deliberately not reset between tests: the client is a lazy+ // singleton, so only whichever test publishes first constructs one.+ clientRegions: [] as (string | undefined)[],+}));++vi.mock("@/src/env.mjs", () => ({ env: mocks.env }));++// Classes, not vi.fn(...): both of these are invoked with `new`, and an arrow+// implementation is not constructible.+vi.mock("@aws-sdk/client-eventbridge", () => ({+ EventBridgeClient: class {+ send = mocks.send;+ constructor(config: { region?: string }) {+ mocks.clientRegions.push(config?.region);+ }+ },+ PutEventsCommand: class {+ constructor(public input: unknown) {}+ },+}));++vi.mock("@langfuse/shared/src/db", () => ({+ prisma: {+ organization: { findUnique: mocks.findOrg },+ project: { findMany: mocks.findProjects },+ },+}));++vi.mock("@langfuse/shared/src/server", async (importOriginal) => {+ const actual = await importOriginal<typeof SharedServer>();++ return {+ ...actual,+ recordIncrement: mocks.recordIncrement,+ };+});++import {+ backfillChbProjectEvents,+ emitChbProjectEvent,+ sendChbProjectEvent,+} from "@/src/ee/features/billing/server/chb/chbProjectEvents";+import { logger } from "@langfuse/shared/src/server";++const loggerError = vi+ .spyOn(logger, "error")+ .mockImplementation((() => {}) as never);++const ORG_ID = "org-1";+const PROJECT_ID = "project-1";+// Must be a real v4 uuid: cloudConfigSchema uuid-validates the CHB org id and+// parseDbOrg nulls the whole cloudConfig on a parse failure.+const CHB_ORG_ID = "3f7c1b0a-2d5e-4c8b-9a1f-8e6d4c2b1a09";++const orgRow = (cloudConfig: unknown) => ({+ id: ORG_ID,+ name: "Org",+ cloudConfig,+});++// Every entry published, flattened across calls and across each call's batch.+const publishedEntries = () =>+ mocks.send.mock.calls.flatMap(([command]) => command.input.Entries);++const publishedDetails = () =>+ publishedEntries().map((entry) => JSON.parse(entry.Detail));++// emitChbProjectEvent is deliberately fire-and-forget, and the retry wrapper+// schedules its attempts on timers, so tests poll for the effect instead of+// awaiting the call.+const waitFor = (assertion: () => void) =>+ vi.waitFor(assertion, { timeout: 5_000, interval: 10 });++describe("chbProjectEvents", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ mocks.env.CLICKHOUSE_BILLING_EVENT_BUS_ARN = EVENT_BUS_ARN;+ mocks.send.mockResolvedValue({ FailedEntryCount: 0 });+ mocks.findProjects.mockResolvedValue([]);+ });++ it("publishes the event envelope for a CHB-billed org", async () => {+ mocks.findOrg.mockResolvedValue(+ orgRow({ clickhouse: { organizationId: CHB_ORG_ID } }),+ );++ emitChbProjectEvent({+ type: "LANGFUSE_PROJECT_CREATED",+ orgId: ORG_ID,+ projectId: PROJECT_ID,+ });+ await waitFor(() => expect(mocks.send).toHaveBeenCalledTimes(1));++ const [entry] = publishedEntries();+ expect(entry.EventBusName).toBe(EVENT_BUS_ARN);+ // CHB's bus policy whitelists event types by detail-type, so the type must+ // be here and not only inside Detail.+ expect(entry.DetailType).toBe("LANGFUSE_PROJECT_CREATED");+ expect(entry.Source).toBe("langfuse");+ // The event carries CHB's organization id, not ours -- CHB's registry is+ // keyed by its own org id.+ expect(JSON.parse(entry.Detail)).toMatchObject({+ type: "LANGFUSE_PROJECT_CREATED",+ organizationId: CHB_ORG_ID,+ projectId: PROJECT_ID,+ regionId: "eu",+ });+ });++ // CHB's buses sit in their own regions, which never match ours; the ARN is+ // the only place that region appears.+ it("targets the bus region parsed from the ARN", async () => {+ mocks.findOrg.mockResolvedValue(+ orgRow({ clickhouse: { organizationId: CHB_ORG_ID } }),+ );++ await sendChbProjectEvent({+ type: "LANGFUSE_PROJECT_CREATED",+ chbOrganizationId: CHB_ORG_ID,+ projectId: PROJECT_ID,+ });++ expect(mocks.clientRegions).toContain("eu-central-1");+ });++ it("does not emit for an org without CHB state", async () => {+ mocks.findOrg.mockResolvedValue(+ orgRow({ stripe: { customerId: "cus_1" } }),+ );++ emitChbProjectEvent({+ type: "LANGFUSE_PROJECT_CREATED",+ orgId: ORG_ID,+ projectId: PROJECT_ID,+ });+ await waitFor(() => expect(mocks.findOrg).toHaveBeenCalled());++ expect(mocks.send).not.toHaveBeenCalled();+ });++ it("swallows delivery failures and records them", async () => {+ mocks.findOrg.mockResolvedValue(+ orgRow({ clickhouse: { organizationId: CHB_ORG_ID } }),+ );+ mocks.send.mockRejectedValue(new Error("ECONNRESET"));++ // Must not reject: project create/delete latency and success are unaffected+ // by the billing signal.+ expect(() =>+ emitChbProjectEvent({+ type: "LANGFUSE_PROJECT_DELETED",+ orgId: ORG_ID,+ projectId: PROJECT_ID,+ }),+ ).not.toThrow();++ await waitFor(() =>+ expect(mocks.recordIncrement).toHaveBeenCalledWith(+ "langfuse.billing_events.emit_failed",+ 1,+ {+ unit: "events",+ source: "request",+ // A lost delete has to be separable from a lost create on the alert.+ event_type: "LANGFUSE_PROJECT_DELETED",+ },+ ),+ );+ expect(loggerError).toHaveBeenCalled();+ });++ // PutEvents answers 200 with a per-entry failure when the bus rejects an+ // event, so a naive "it resolved" check would treat a rejected event as sent.+ it("treats a failed entry in a 200 response as a failure", async () => {+ mocks.send.mockResolvedValue({+ FailedEntryCount: 1,+ Entries: [+ {+ ErrorCode: "NotAuthorizedForSourceException",+ ErrorMessage: "denied",+ },+ ],+ });++ await expect(+ sendChbProjectEvent({+ type: "LANGFUSE_PROJECT_CREATED",+ chbOrganizationId: CHB_ORG_ID,+ projectId: PROJECT_ID,+ }),+ ).rejects.toThrow("NotAuthorizedForSourceException");+ });++ it("throws from the awaited send path when the event bus is unconfigured", async () => {+ mocks.env.CLICKHOUSE_BILLING_EVENT_BUS_ARN = undefined;++ await expect(+ sendChbProjectEvent({+ type: "LANGFUSE_PROJECT_CREATED",+ chbOrganizationId: CHB_ORG_ID,+ projectId: PROJECT_ID,+ }),+ ).rejects.toThrow("CHB event bus is not configured");+ expect(mocks.send).not.toHaveBeenCalled();+ });++ describe("backfillChbProjectEvents", () => {+ it("sends CREATED for every existing project once the org has checked out", async () => {+ mocks.findOrg.mockResolvedValue(+ orgRow({ clickhouse: { organizationId: CHB_ORG_ID } }),+ );+ mocks.findProjects.mockResolvedValue([+ { id: "project-a" },+ { id: "project-b" },+ ]);++ await expect(+ backfillChbProjectEvents({ orgId: ORG_ID }),+ ).resolves.toEqual({ sent: 2, failed: 0 });++ // Soft-deleted projects must stay out of the backfill: CHB would start+ // metering a project the org can no longer see.+ expect(mocks.findProjects).toHaveBeenCalledWith({+ where: { orgId: ORG_ID, deletedAt: null },+ select: { id: true },+ });+ expect(publishedDetails().map((detail) => detail.projectId)).toEqual([+ "project-a",+ "project-b",+ ]);+ // Batched: both projects ride one PutEvents call, not one call each.+ expect(mocks.send).toHaveBeenCalledTimes(1);+ expect(publishedEntries()[0].DetailType).toBe("LANGFUSE_PROJECT_CREATED");+ expect(publishedDetails()[0]).toMatchObject({+ organizationId: CHB_ORG_ID,+ });+ });++ // The whole point of the checkout-time backfill: projects are not synced+ // before the org carries a CHB organization id.+ it("sends nothing for an org that has not checked out", async () => {+ mocks.findOrg.mockResolvedValue(+ orgRow({ stripe: { customerId: "cus_1" } }),+ );+ mocks.findProjects.mockResolvedValue([{ id: "project-a" }]);++ await expect(+ backfillChbProjectEvents({ orgId: ORG_ID }),+ ).resolves.toEqual({ sent: 0, failed: 0 });++ expect(mocks.findProjects).not.toHaveBeenCalled();+ expect(mocks.send).not.toHaveBeenCalled();+ });++ // PutEvents fails per entry, so a batch can come back half-rejected. The+ // rejected entry must be the only one counted lost, and the only one+ // retried -- resending the whole batch would duplicate what already landed.+ it("counts only the rejected entry of a partially failed batch", async () => {+ mocks.findOrg.mockResolvedValue(+ orgRow({ clickhouse: { organizationId: CHB_ORG_ID } }),+ );+ mocks.findProjects.mockResolvedValue([+ { id: "project-a" },+ { id: "project-b" },+ ]);+ mocks.send.mockImplementation((command) => {+ const results = command.input.Entries.map(+ (entry: { Detail: string }) =>+ JSON.parse(entry.Detail).projectId === "project-a"+ ? { ErrorCode: "InternalException" }+ : { EventId: "ok" },+ );+ return Promise.resolve({+ FailedEntryCount: results.filter(+ (result: { ErrorCode?: string }) => result.ErrorCode,+ ).length,+ Entries: results,+ });+ });++ await expect(+ backfillChbProjectEvents({ orgId: ORG_ID }),+ ).resolves.toEqual({ sent: 1, failed: 1 });++ // Retry #2 and #3 carry only project-a; project-b is not resent.+ expect(+ mocks.send.mock.calls+ .slice(1)+ .flatMap(([command]) => command.input.Entries)+ .map(+ (entry: { Detail: string }) => JSON.parse(entry.Detail).projectId,+ ),+ ).toEqual(["project-a", "project-a"]);++ expect(mocks.recordIncrement).toHaveBeenCalledWith(+ "langfuse.billing_events.emit_failed",+ 1,+ {+ unit: "events",+ source: "backfill",+ event_type: "LANGFUSE_PROJECT_CREATED",+ },+ );+ expect(loggerError).toHaveBeenCalled();+ });++ // A failed backfill must never fail checkout.+ it("does not throw when the org lookup itself fails", async () => {+ mocks.findOrg.mockRejectedValue(new Error("postgres down"));++ await expect(+ backfillChbProjectEvents({ orgId: ORG_ID }),+ ).resolves.toEqual({ sent: 0, failed: 0 });+ expect(loggerError).toHaveBeenCalled();+ });+ });+});diff --git a/web/src/ee/features/admin-api/server/projects/createProject.ts b/web/src/ee/features/admin-api/server/projects/createProject.tsindex 6deeee84ca..946c3d0c35 100644--- a/web/src/ee/features/admin-api/server/projects/createProject.ts+++ b/web/src/ee/features/admin-api/server/projects/createProject.ts@@ -5,6 +5,7 @@ import { projectNameSchema } from "@/src/features/auth/lib/projectNameSchema";import { projectRetentionSchema } from "@/src/features/auth/lib/projectRetentionSchema";import { hasEntitlementBasedOnPlan } from "@/src/features/entitlements/server/hasEntitlement";import { type ApiAccessScope } from "@langfuse/shared/src/server";+import { emitChbProjectEvent } from "@/src/ee/features/billing/server/chb/chbProjectEvents";export async function handleCreateProject(req: NextApiRequest,@@ -95,6 +96,13 @@ export async function handleCreateProject(},});+ // Best-effort CHB metering signal; no-op unless the org is CHB-billed+ emitChbProjectEvent({+ type: "LANGFUSE_PROJECT_CREATED",+ orgId: scope.orgId,+ projectId: project.id,+ });+return res.status(201).json({id: project.id,name: project.name,diff --git a/web/src/ee/features/admin-api/server/projects/projectById/index.ts b/web/src/ee/features/admin-api/server/projects/projectById/index.tsindex 8fd5735342..dc7bde095d 100644--- a/web/src/ee/features/admin-api/server/projects/projectById/index.ts+++ b/web/src/ee/features/admin-api/server/projects/projectById/index.ts@@ -13,6 +13,7 @@ import { hasEntitlementBasedOnPlan } from "@/src/features/entitlements/server/haimport { projectNameSchema } from "@/src/features/auth/lib/projectNameSchema";import { ApiAuthService } from "@/src/features/public-api/server/apiAuth";import { auditLog } from "@/src/features/audit-logs/auditLog";+import { emitChbProjectEvent } from "@/src/ee/features/billing/server/chb/chbProjectEvents";export async function handleUpdateProject(req: NextApiRequest,@@ -154,6 +155,14 @@ export async function handleDeleteProject(action: "delete",});+ // Soft-delete is the billing-relevant moment: the customer stops being+ // billable now, not when the async hard-delete worker finishes.+ emitChbProjectEvent({+ type: "LANGFUSE_PROJECT_DELETED",+ orgId: scope.orgId,+ projectId,+ });+// Queue project deletion jobconst projectDeleteQueue = ProjectDeleteQueue.getInstance();if (!projectDeleteQueue) {diff --git a/web/src/ee/features/billing/server/chb/chbProjectEvents.ts b/web/src/ee/features/billing/server/chb/chbProjectEvents.tsnew file mode 100644index 0000000000..a29b318419--- /dev/null+++ b/web/src/ee/features/billing/server/chb/chbProjectEvents.ts@@ -0,0 +1,341 @@+import {+ EventBridgeClient,+ PutEventsCommand,+} from "@aws-sdk/client-eventbridge";+import { backOff } from "exponential-backoff";++import { env } from "@/src/env.mjs";+import { parseDbOrg } from "@langfuse/shared";+import { prisma } from "@langfuse/shared/src/db";+import { logger, recordIncrement } from "@langfuse/shared/src/server";++/**+ * Project lifecycle events for ClickHouse Billing (CHB).+ *+ * CHB needs to know which projects exist per organization to poll our billing+ * metrics API. Delivery is a direct best-effort cross-account EventBridge+ * PutEvents from web, no queue: the bus is an always-on managed endpoint;+ * absorbing brief unavailability is CHB's responsibility. At-most-once is+ * acceptable by design — a lost PROJECT_DELETED is benign (the metrics API+ * returns zeros for deleted projects by contract), a lost PROJECT_CREATED+ * delays metering for one project until CHB's own backfill pipeline catches+ * it. If `langfuse.billing_events.emit_failed` ever shows real loss, swap this+ * helper's internals for a queue — call sites keep the same signature.+ *+ * Sync starts at checkout, not at signup. Nothing is mirrored to CHB until+ * checkout persists `cloudConfig.clickhouse.organizationId`: before that CHB+ * has no organization to attribute a project to, so an event would be+ * unroutable, and the vast majority of orgs never check out at all.+ * `resolveChbOrganizationId` is the single gate enforcing this, and checkout+ * closes the gap it creates by calling `backfillChbProjectEvents` once for the+ * projects the org already had.+ */++export type ChbProjectEventType =+ | "LANGFUSE_PROJECT_CREATED"+ | "LANGFUSE_PROJECT_DELETED";++// EventBridge `source` for everything we publish. Free-form on our side; CHB's+// bus policy filters on detail-type, not source.+const CHB_EVENT_SOURCE = "langfuse";++const EVENT_BUS_REQUEST_TIMEOUT_MS = 5_000;++// EventBridge caps PutEvents at 10 entries per call. Our entries are a few+// hundred bytes each, so the 256KB per-call limit is never the binding one.+const EVENT_BUS_MAX_ENTRIES_PER_CALL = 10;++// The envelope (type, CHB organizationId, Langfuse projectId, regionId) still+// needs a final confirmation from CHB before rollout, so it is isolated here:+// a contract change stays a one-function edit. `type` is repeated here and in+// detail-type so a consumer reading either one sees it.+const buildChbProjectEventPayload = (params: {+ type: ChbProjectEventType;+ chbOrganizationId: string;+ projectId: string;+}) => ({+ type: params.type,+ organizationId: params.chbOrganizationId,+ projectId: params.projectId,+ regionId: env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,+ createdAt: new Date().toISOString(),+});++// One bus per deployment, so one client: the ARN comes from env and cannot+// change at runtime. Lazy so a web instance that never touches CHB billing+// never constructs one, and shared so the rest reuse its connection pool.+let cachedClient: EventBridgeClient | undefined;++const getEventBridgeClient = (region: string): EventBridgeClient => {+ cachedClient ??= new EventBridgeClient({+ region,+ requestHandler: {+ requestTimeout: EVENT_BUS_REQUEST_TIMEOUT_MS,+ throwOnRequestTimeout: true,+ },+ });+ return cachedClient;+};++/**+ * The bus rejected some entries. Carries how many, so the caller can count the+ * events it actually lost instead of assuming the whole batch died.+ */+class ChbEventBusRejection extends Error {+ constructor(+ readonly failedEntryCount: number,+ message: string,+ ) {+ super(message);+ this.name = "ChbEventBusRejection";+ }+}++const requireEventBusArn = (): string => {+ const eventBusArn = env.CLICKHOUSE_BILLING_EVENT_BUS_ARN;+ if (!eventBusArn) {+ throw new Error(+ "CHB event bus is not configured (CLICKHOUSE_BILLING_EVENT_BUS_ARN)",+ );+ }+ return eventBusArn;+};++const buildChbEventEntry = (+ eventBusArn: string,+ params: {+ type: ChbProjectEventType;+ chbOrganizationId: string;+ projectId: string;+ },+) => ({+ EventBusName: eventBusArn,+ // CHB's bus policy whitelists the allowed event types by detail-type, so the+ // type has to travel here, not only in Detail — otherwise the bus rejects+ // the entry.+ DetailType: params.type,+ Source: CHB_EVENT_SOURCE,+ Detail: JSON.stringify(buildChbProjectEventPayload(params)),+});++/**+ * PutEvents one batch of at most `EVENT_BUS_MAX_ENTRIES_PER_CALL` entries,+ * retrying over a few seconds.+ *+ * Retries carry only the entries the bus is still rejecting: PutEvents can fail+ * per entry, and resending a whole batch because one entry was throttled would+ * duplicate the ones that already landed. Throws `ChbEventBusRejection` if+ * anything is still failing after the last attempt, and whatever the SDK threw+ * if the call never completed.+ */+async function putChbEventBatch(+ region: string,+ entries: ReturnType<typeof buildChbEventEntry>[],+): Promise<void> {+ let pending = entries;++ await backOff(+ async () => {+ const response = await getEventBridgeClient(region).send(+ new PutEventsCommand({ Entries: pending }),+ );++ // PutEvents answers 200 even when it rejected entries (bad policy,+ // throttling, malformed detail), so the failure count is the real status.+ if (!response.FailedEntryCount) {+ pending = [];+ return;+ }++ // The response entries are positionally aligned with the request, so an+ // entry with an ErrorCode identifies which input failed.+ const results = response.Entries ?? [];+ const firstError = results.find((result) => result.ErrorCode);+ pending = pending.filter((_, index) => results[index]?.ErrorCode);++ throw new ChbEventBusRejection(+ pending.length,+ `CHB event bus rejected ${pending.length} of ${entries.length} events: ${+ firstError?.ErrorCode ?? "unknown error"+ }${firstError?.ErrorMessage ? ` (${firstError.ErrorMessage})` : ""}`,+ );+ },+ { numOfAttempts: 3 },+ );+}++/**+ * Publish a single project event to CHB's event bus, retrying over a few+ * seconds. Throws on terminal failure — callers decide whether that is+ * fire-and-forget noise (request path) or worth counting (checkout backfill).+ *+ * Authentication is SigV4 from the task role: infrastructure grants+ * events:PutEvents on exactly this bus ARN, and CHB's side allows our account+ * on the bus resource policy. No shared secret is involved — the CHB service+ * token belongs to their REST API, not here.+ */+export async function sendChbProjectEvent(params: {+ type: ChbProjectEventType;+ chbOrganizationId: string;+ projectId: string;+}): Promise<void> {+ const eventBusArn = requireEventBusArn();++ // CHB's buses live in their own regions, which do not match ours, and the ARN+ // is the only place that region appears. env.mjs shape-checks the ARN, so the+ // segment is present.+ const region = eventBusArn.split(":")[3];++ await putChbEventBatch(region, [buildChbEventEntry(eventBusArn, params)]);+}++/**+ * The CHB organization id to attribute events to, or null when the org has not+ * checked out yet. Every outbound project event goes through this gate: an org+ * without `cloudConfig.clickhouse.organizationId` is not a CHB customer, so+ * there is nothing for CHB to route a project event to.+ */+async function resolveChbOrganizationId(orgId: string): Promise<string | null> {+ const org = await prisma.organization.findUnique({ where: { id: orgId } });+ if (!org) return null;+ return parseDbOrg(org).cloudConfig?.clickhouse?.organizationId ?? null;+}++/**+ * Fire-and-forget emit for the project create/delete request paths. No-ops+ * unless the org has checked out, and never throws into the caller — project+ * create/delete latency and success are unaffected.+ */+export function emitChbProjectEvent(params: {+ type: ChbProjectEventType;+ orgId: string;+ projectId: string;+}): void {+ (async () => {+ const chbOrganizationId = await resolveChbOrganizationId(params.orgId);+ if (!chbOrganizationId) return;++ await sendChbProjectEvent({+ type: params.type,+ chbOrganizationId,+ projectId: params.projectId,+ });+ })().catch((error) => {+ recordIncrement("langfuse.billing_events.emit_failed", 1, {+ unit: "events",+ source: "request",+ // A lost create and a lost delete fail differently — the first+ // under-meters an org, the second keeps billing a project that is gone —+ // so they have to be separable on the alert.+ event_type: params.type,+ });+ logger.error(+ `[CHB Project Events] Failed to emit ${params.type} for project ${params.projectId} (org ${params.orgId})`,+ error,+ );+ });+}++/**+ * Send LANGFUSE_PROJECT_CREATED for every project the org already has.+ *+ * Checkout calls this right after it persists+ * `cloudConfig.clickhouse.organizationId`, which is the first moment CHB can+ * attribute projects to an organization. Projects created before that point+ * were deliberately never synced, so without this backfill CHB would only ever+ * meter projects created after checkout.+ *+ * Best-effort and never throws: a failed backfill must not fail checkout. One+ * batch's delivery failure is isolated so the remaining batches still land, and+ * each lost event increments `langfuse.billing_events.emit_failed`+ * (`source:backfill`) so a systematically broken backfill is alertable —+ * silent failure here means CHB under-meters the org indefinitely.+ *+ * Sent in batches of at most ten, which is the PutEvents limit: an org with+ * fifty projects costs five sequential calls rather than fifty. The project+ * query is deliberately unpaginated — Prisma applies no default `take`, so it+ * returns every project, and a cap here would silently under-backfill. Only the+ * ids are selected, so the result stays small even for a very large org.+ *+ * A concurrent project create during the backfill can produce a duplicate+ * CREATED, which is harmless — CHB keys projects by id.+ */+export async function backfillChbProjectEvents(params: {+ orgId: string;+}): Promise<{ sent: number; failed: number }> {+ let sent = 0;+ let failed = 0;++ try {+ const chbOrganizationId = await resolveChbOrganizationId(params.orgId);+ if (!chbOrganizationId) return { sent, failed };++ const eventBusArn = requireEventBusArn();+ const region = eventBusArn.split(":")[3];++ const projects = await prisma.project.findMany({+ where: { orgId: params.orgId, deletedAt: null },+ select: { id: true },+ });++ const entries = projects.map((project) =>+ buildChbEventEntry(eventBusArn, {+ type: "LANGFUSE_PROJECT_CREATED",+ chbOrganizationId,+ projectId: project.id,+ }),+ );++ for (+ let offset = 0;+ offset < entries.length;+ offset += EVENT_BUS_MAX_ENTRIES_PER_CALL+ ) {+ const batch = entries.slice(+ offset,+ offset + EVENT_BUS_MAX_ENTRIES_PER_CALL,+ );++ try {+ await putChbEventBatch(region, batch);+ sent += batch.length;+ } catch (error) {+ // A rejection knows exactly how many entries the bus refused; anything+ // else (timeout, transport) means the whole batch is unaccounted for.+ const lost =+ error instanceof ChbEventBusRejection+ ? error.failedEntryCount+ : batch.length;+ sent += batch.length - lost;+ failed += lost;+ recordIncrement("langfuse.billing_events.emit_failed", lost, {+ unit: "events",+ source: "backfill",+ event_type: "LANGFUSE_PROJECT_CREATED",+ });+ logger.error(+ `[CHB Project Events] Failed to backfill ${lost} of ${batch.length} LANGFUSE_PROJECT_CREATED events (org ${params.orgId})`,+ error,+ );+ }+ }++ logger.info(+ `[CHB Project Events] Backfilled ${sent}/${projects.length} projects for org ${params.orgId} (${failed} failed)`,+ );+ } catch (error) {+ // Loading the org or its projects failed — nothing was sent, and checkout+ // still has to succeed.+ recordIncrement("langfuse.billing_events.emit_failed", 1, {+ unit: "events",+ source: "backfill",+ event_type: "LANGFUSE_PROJECT_CREATED",+ });+ logger.error(+ `[CHB Project Events] Failed to backfill projects for org ${params.orgId}`,+ error,+ );+ }++ return { sent, failed };+}diff --git a/web/src/env.mjs b/web/src/env.mjsindex 0d907139bf..2092b3254d 100644--- a/web/src/env.mjs+++ b/web/src/env.mjs@@ -415,6 +415,17 @@ export const env = createEnv({// Bearer secret CHB's metering pipeline presents to GET /api/billing/metrics.// Unset disables the endpoint (it 500s) rather than leaving it open.CLICKHOUSE_BILLING_METRICS_API_KEY: z.string().optional(),+ // ARN of CHB's cross-account EventBridge bus we publish project lifecycle+ // events to. Unset = no events are emitted. The ARN carries the bus region,+ // so it is the only input needed; authentication is SigV4 from the task+ // role, which infrastructure grants events:PutEvents on exactly this ARN.+ CLICKHOUSE_BILLING_EVENT_BUS_ARN: z+ .string()+ .regex(+ /^arn:aws:events:[a-z0-9-]+:\d{12}:event-bus\/.+$/,+ "must be an EventBridge bus ARN (arn:aws:events:<region>:<account-id>:event-bus/<name>)",+ )+ .optional(),SENTRY_AUTH_TOKEN: z.string().optional(),SENTRY_CSP_REPORT_URI: z.string().optional(),LANGFUSE_RATE_LIMITS_ENABLED: z.enum(["true", "false"]).default("true"),@@ -949,6 +960,8 @@ export const env = createEnv({process.env.LANGFUSE_CLOUD_BILLING_CHB_CUTOFF_DATE,CLICKHOUSE_BILLING_METRICS_API_KEY:process.env.CLICKHOUSE_BILLING_METRICS_API_KEY,+ CLICKHOUSE_BILLING_EVENT_BUS_ARN:+ process.env.CLICKHOUSE_BILLING_EVENT_BUS_ARN,SENTRY_AUTH_TOKEN: process.env.SENTRY_AUTH_TOKEN,SENTRY_CSP_REPORT_URI: process.env.SENTRY_CSP_REPORT_URI,LANGFUSE_RATE_LIMITS_ENABLED: process.env.LANGFUSE_RATE_LIMITS_ENABLED,diff --git a/web/src/features/projects/server/projectsRouter.ts b/web/src/features/projects/server/projectsRouter.tsindex 138197c059..b2c7b82fd7 100644--- a/web/src/features/projects/server/projectsRouter.ts+++ b/web/src/features/projects/server/projectsRouter.ts@@ -20,6 +20,7 @@ import {import { randomUUID } from "crypto";import { StringNoHTMLNonEmpty } from "@langfuse/shared";import { buildAdminOrgContext } from "@/src/features/organizations/server/adminOrgContext";+import { emitChbProjectEvent } from "@/src/ee/features/billing/server/chb/chbProjectEvents";export const projectsRouter = createTRPCRouter({create: protectedOrganizationProcedure@@ -66,6 +67,13 @@ export const projectsRouter = createTRPCRouter({after: project,});+ // Best-effort CHB metering signal; no-op unless the org is CHB-billed+ emitChbProjectEvent({+ type: "LANGFUSE_PROJECT_CREATED",+ orgId: input.orgId,+ projectId: project.id,+ });+return {id: project.id,name: project.name,@@ -210,6 +218,14 @@ export const projectsRouter = createTRPCRouter({action: "delete",});+ // Soft-delete is the billing-relevant moment: the customer stops being+ // billable now, not when the async hard-delete worker finishes.+ emitChbProjectEvent({+ type: "LANGFUSE_PROJECT_DELETED",+ orgId: ctx.session.orgId,+ projectId: input.projectId,+ });+const projectDeleteQueue = ProjectDeleteQueue.getInstance();if (!projectDeleteQueue) {throw new TRPCError({@@ -298,6 +314,23 @@ export const projectsRouter = createTRPCRouter({ctx.prisma,redis,).invalidateCachedProjectApiKeys(input.projectId);++ // A transfer is a delete for the source org and a create for the+ // destination one. CHB's registry is keyed by (organizationId,+ // projectId), so emitting only one side leaves the source org billed for+ // usage it no longer owns, or the destination org unmetered. Each emit+ // no-ops unless that org is CHB-billed, so a Stripe-to-Stripe transfer+ // sends nothing.+ emitChbProjectEvent({+ type: "LANGFUSE_PROJECT_DELETED",+ orgId: ctx.session.orgId,+ projectId: input.projectId,+ });+ emitChbProjectEvent({+ type: "LANGFUSE_PROJECT_CREATED",+ orgId: input.targetOrgId,+ projectId: input.projectId,+ });}),environmentFilterOptions: protectedProjectProcedure