🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(billing): count ClickHouse Billing usage over its own billing period (#18401)
Steffen Reuther646718f8195f
diff --git a/packages/shared/src/server/repositories/observations.ts b/packages/shared/src/server/repositories/observations.tsindex b2b4ed6c4e..163f2df4b0 100644--- a/packages/shared/src/server/repositories/observations.ts+++ b/packages/shared/src/server/repositories/observations.ts@@ -1635,10 +1635,12 @@ export const getObservationCountsByProjectInCreationInterval = async ({start,end,projectId,+ projectIds,}: {start: Date;end: Date;projectId?: string;+ projectIds?: string[];}) => {const query = `SELECT@@ -1648,6 +1650,7 @@ export const getObservationCountsByProjectInCreationInterval = async ({WHERE created_at >= {start: DateTime64(3)}AND created_at < {end: DateTime64(3)}${projectId ? "AND project_id = {projectId: String}" : ""}+ ${projectIds ? "AND project_id IN ({projectIds: Array(String)})" : ""}GROUP BY project_id`;@@ -1657,6 +1660,7 @@ export const getObservationCountsByProjectInCreationInterval = async ({start: convertDateToClickhouseDateTime(start),end: convertDateToClickhouseDateTime(end),...(projectId ? { projectId } : {}),+ ...(projectIds ? { projectIds } : {}),},clickhouseConfigs: {request_timeout: 300000, // 5 minutes timeoutdiff --git a/packages/shared/src/server/repositories/scores.ts b/packages/shared/src/server/repositories/scores.tsindex 7da2779c36..1e9dfcce8f 100644--- a/packages/shared/src/server/repositories/scores.ts+++ b/packages/shared/src/server/repositories/scores.ts@@ -2355,10 +2355,12 @@ export const getScoreCountsByProjectInCreationInterval = async ({start,end,projectId,+ projectIds,}: {start: Date;end: Date;projectId?: string;+ projectIds?: string[];}) => {const query = `SELECT@@ -2368,6 +2370,7 @@ export const getScoreCountsByProjectInCreationInterval = async ({WHERE created_at >= {start: DateTime64(3)}AND created_at < {end: DateTime64(3)}${projectId ? "AND project_id = {projectId: String}" : ""}+ ${projectIds ? "AND project_id IN ({projectIds: Array(String)})" : ""}AND data_type IN ({dataTypes: Array(String)})GROUP BY project_id`;@@ -2379,6 +2382,7 @@ export const getScoreCountsByProjectInCreationInterval = async ({end: convertDateToClickhouseDateTime(end),dataTypes: LISTABLE_SCORE_TYPES,...(projectId ? { projectId } : {}),+ ...(projectIds ? { projectIds } : {}),},clickhouseConfigs: {request_timeout: 300000, // 5 minutes timeoutdiff --git a/packages/shared/src/server/repositories/traces.ts b/packages/shared/src/server/repositories/traces.tsindex 0fae71f747..5a712573f1 100644--- a/packages/shared/src/server/repositories/traces.ts+++ b/packages/shared/src/server/repositories/traces.ts@@ -366,10 +366,12 @@ export const getTraceCountsByProjectInCreationInterval = async ({start,end,projectId,+ projectIds,}: {start: Date;end: Date;projectId?: string;+ projectIds?: string[];}) => {const query = `SELECT@@ -379,6 +381,7 @@ export const getTraceCountsByProjectInCreationInterval = async ({WHERE created_at >= {start: DateTime64(3)}AND created_at < {end: DateTime64(3)}${projectId ? "AND project_id = {projectId: String}" : ""}+ ${projectIds ? "AND project_id IN ({projectIds: Array(String)})" : ""}GROUP BY project_id`;@@ -388,6 +391,7 @@ export const getTraceCountsByProjectInCreationInterval = async ({start: convertDateToClickhouseDateTime(start),end: convertDateToClickhouseDateTime(end),...(projectId ? { projectId } : {}),+ ...(projectIds ? { projectIds } : {}),},clickhouseConfigs: {request_timeout: 300000, // 5 minutes timeoutdiff --git a/web/src/__tests__/server/unit/chbBillingService.servertest.ts b/web/src/__tests__/server/unit/chbBillingService.servertest.tsindex 5b86bddc8f..974794b23f 100644--- a/web/src/__tests__/server/unit/chbBillingService.servertest.ts+++ b/web/src/__tests__/server/unit/chbBillingService.servertest.ts@@ -1,12 +1,15 @@import type { CloudConfigSchema } from "@langfuse/shared";import type * as SharedServer from "@langfuse/shared/src/server";import { TRPCError } from "@trpc/server";-import { beforeEach, describe, expect, it, vi } from "vitest";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";const mocks = vi.hoisted(() => ({env: { NEXTAUTH_URL: "https://cloud.langfuse.com" },logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },auditLog: vi.fn(),+ traceCounts: vi.fn(),+ observationCounts: vi.fn(),+ scoreCounts: vi.fn(),}));vi.mock("@/src/env.mjs", () => ({ env: mocks.env }));@@ -15,11 +18,17 @@ vi.mock("@/src/features/audit-logs/server", () => ({auditLog: mocks.auditLog,}));-// Only the logger is stubbed: getBillingCycleStart/End are the real cached-cycle-// fallback this service shares with the Stripe path.+// The logger and the ClickHouse unit counts are stubbed: getBillingCycleStart/End+// are the real cached-cycle fallback this service shares with the Stripe path.vi.mock("@langfuse/shared/src/server", async (importOriginal) => {const actual = await importOriginal<typeof SharedServer>();- return { ...actual, logger: mocks.logger };+ return {+ ...actual,+ logger: mocks.logger,+ getTraceCountsByProjectInCreationInterval: mocks.traceCounts,+ getObservationCountsByProjectInCreationInterval: mocks.observationCounts,+ getScoreCountsByProjectInCreationInterval: mocks.scoreCounts,+ };});import {@@ -48,6 +57,7 @@ const update = vi.fn();// Checkout claims the CH organization id with a guarded UPDATE rather than a// read-then-write, so the claim is a raw statement returning a row count.const executeRaw = vi.fn();+const projectFindMany = vi.fn();const clientMock = {createCheckoutSession: vi.fn(),@@ -59,7 +69,11 @@ const clientMock = {};const ctx = {- prisma: { organization: { findUnique, update }, $executeRaw: executeRaw },+ prisma: {+ organization: { findUnique, update },+ project: { findMany: projectFindMany },+ $executeRaw: executeRaw,+ },session: {orgId: ORG_ID,orgRole: "OWNER",@@ -770,6 +784,75 @@ describe("chbBillingService", () => {});});+ describe("getUsage", () => {+ afterEach(() => {+ vi.useRealTimers();+ });++ it("counts live projects' ingested units over CHB's billing period", async () => {+ vi.useFakeTimers({ toFake: ["Date"] });+ vi.setSystemTime(new Date("2026-10-06T12:00:00Z"));+ withOrg(chbConfig());+ clientMock.getAttachedPlan.mockResolvedValue({+ id: ATTACHED_PLAN_ID,+ period: {+ startDate: "2026-10-01T00:00:00Z",+ endDate: "2026-11-01T00:00:00Z",+ },+ });+ projectFindMany.mockResolvedValue([+ { id: "project-1" },+ { id: "project-2" },+ ]);+ mocks.traceCounts.mockResolvedValue([+ { projectId: "project-1", count: 10 },+ ]);+ mocks.observationCounts.mockResolvedValue([+ { projectId: "project-1", count: 100 },+ { projectId: "project-2", count: 50 },+ ]);+ mocks.scoreCounts.mockResolvedValue([+ { projectId: "project-2", count: 5 },+ ]);++ const usage = await service().getUsage(ORG_ID);++ // CHB's calendar month, not the org's cycle anchored on the 15th or its+ // cached counter+ expect(usage).toEqual({+ usageCount: 165,+ usageType: "units",+ billingPeriod: {+ start: new Date("2026-10-01T00:00:00Z"),+ end: new Date("2026-11-01T00:00:00Z"),+ },+ });+ expect(projectFindMany).toHaveBeenCalledWith(+ expect.objectContaining({ where: { orgId: ORG_ID, deletedAt: null } }),+ );+ const interval = {+ start: new Date("2026-10-01T00:00:00Z"),+ end: new Date("2026-10-06T12:00:00Z"),+ projectIds: ["project-1", "project-2"],+ };+ expect(mocks.traceCounts).toHaveBeenCalledWith(interval);+ expect(mocks.observationCounts).toHaveBeenCalledWith(interval);+ expect(mocks.scoreCounts).toHaveBeenCalledWith(interval);+ });++ it("falls back to the cached cycle counter without an attached plan", async () => {+ withOrg({ clickhouse: { organizationId: CH_ORG_ID } });++ const usage = await service().getUsage(ORG_ID);++ expect(clientMock.getAttachedPlan).not.toHaveBeenCalled();+ expect(mocks.traceCounts).not.toHaveBeenCalled();+ expect(usage.usageCount).toBe(4_200);+ expect(usage.usageType).toBe("units");+ expect(usage.billingPeriod.start.getUTCDate()).toBe(15);+ });+ });+describe("remaining surface", () => {it("builds a portal session against the org's CH id", async () => {withOrg(chbConfig());@@ -794,16 +877,6 @@ describe("chbBillingService", () => {);});- it("reports usage from the cached cycle counter", async () => {- withOrg(chbConfig());-- const usage = await service().getUsage(ORG_ID);-- expect(usage.usageCount).toBe(4_200);- expect(usage.usageType).toBe("units");- expect(usage.billingPeriod.start.getUTCDate()).toBe(15);- });-it("rejects promotion codes, which CHB has no API for", async () => {withOrg(chbConfig());diff --git a/web/src/ee/features/billing/server/chb/chbBillingService.ts b/web/src/ee/features/billing/server/chb/chbBillingService.tsindex 324f77d42c..3353208a1b 100644--- a/web/src/ee/features/billing/server/chb/chbBillingService.ts+++ b/web/src/ee/features/billing/server/chb/chbBillingService.ts@@ -7,6 +7,9 @@ import { CloudConfigSchema, parseDbOrg } from "@langfuse/shared";import {getBillingCycleEnd,getBillingCycleStart,+ getObservationCountsByProjectInCreationInterval,+ getScoreCountsByProjectInCreationInterval,+ getTraceCountsByProjectInCreationInterval,logger,} from "@langfuse/shared/src/server";@@ -76,6 +79,16 @@ export class ChbBillingService {return `${env.NEXTAUTH_URL}/organization/${orgId}/settings/billing`;}+ private toBillingPeriod(+ attachedPlan: ChbAttachedPlan,+ ): { start: Date; end: Date } | null {+ const startDate = attachedPlan.period?.startDate;+ const endDate = attachedPlan.period?.endDate;+ return startDate && endDate+ ? { start: new Date(startDate), end: new Date(endDate) }+ : null;+ }+private toUnixSeconds(value: string | null | undefined): number | null {if (!value) return null;const ms = Date.parse(value);@@ -145,19 +158,9 @@ export class ChbBillingService {chOrganizationId: chb.organizationId,});- const periodStart = attachedPlan.period?.startDate- ? new Date(attachedPlan.period.startDate)- : null;- const periodEnd = attachedPlan.period?.endDate- ? new Date(attachedPlan.period.endDate)- : null;-return {...this.mapScheduled(attachedPlan),- billingPeriod:- periodStart && periodEnd- ? { start: periodStart, end: periodEnd }- : null,+ billingPeriod: this.toBillingPeriod(attachedPlan),// No promotion-code API on the CHB path yetdiscounts: [],hasValidPaymentMethod: attachedPlan.payment?.status === "active",@@ -629,23 +632,71 @@ export class ChbBillingService {}/**- * v1 usage source of truth for CHB orgs is the existing non-Stripe- * fallback: billing cycle from the org's anchor + the cached cycle usage- * the hourly job maintains. Spend-in-USD can later come from- * `GET /invoices?includePreview=true` (the open period's accrued usage).+ * With an attached plan, usage is counted live over CHB's own billing+ * period, by ingestion time (`created_at`) and with the same queries the+ * billing metrics API serves to CHB's metering, so the figure tracks what+ * CHB invoices. Without an attached plan (or a period on it), usage falls+ * back to the cached counter the hourly free-tier job maintains for the+ * org's anchored cycle.*/async getUsage(orgId: string) {- const { org } = await this.getParsedOrg(orgId);+ const { org, parsedOrg } = await this.getParsedOrg(orgId);+ const chb = parsedOrg.cloudConfig?.clickhouse;++ const billingPeriod = chb?.attachedPlanId+ ? this.toBillingPeriod(+ await this.client.getAttachedPlan({+ chOrganizationId: chb.organizationId,+ }),+ )+ : null;++ if (!billingPeriod) {+ const now = new Date();+ return {+ usageCount: org.cloudCurrentCycleUsage ?? 0,+ usageType: "units",+ billingPeriod: {+ start: getBillingCycleStart(org, now),+ end: getBillingCycleEnd(org, now),+ },+ };+ }- const now = new Date();return {- usageCount: org.cloudCurrentCycleUsage ?? 0,+ usageCount: await this.countBillableUnits(orgId, billingPeriod),usageType: "units",- billingPeriod: {- start: getBillingCycleStart(org, now),- end: getBillingCycleEnd(org, now),- },+ billingPeriod,+ };+ }++ /**+ * Billable units (traces + observations + scores) ingested into the org's+ * live projects since the period start. Soft-deleted projects are left out,+ * as they are when usage is metered.+ */+ private async countBillableUnits(+ orgId: string,+ period: { start: Date; end: Date },+ ): Promise<number> {+ const projects = await this.ctx.prisma.project.findMany({+ where: { orgId, deletedAt: null },+ select: { id: true },+ });+ const end = new Date(Math.min(Date.now(), period.end.getTime()));+ if (projects.length === 0 || period.start >= end) return 0;++ const interval = {+ start: period.start,+ end,+ projectIds: projects.map((project) => project.id),};+ const counts = await Promise.all([+ getTraceCountsByProjectInCreationInterval(interval),+ getObservationCountsByProjectInCreationInterval(interval),+ getScoreCountsByProjectInCreationInterval(interval),+ ]);+ return counts.flat().reduce((sum, row) => sum + row.count, 0);}async applyPromotionCode(