Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(billing): count ClickHouse Billing usage over its own billing period (#18401)

Steffen Reuther646718f8195f
diff --git a/packages/shared/src/server/repositories/observations.ts b/packages/shared/src/server/repositories/observations.ts
index b2b4ed6c4e..163f2df4b0 100644
--- a/packages/shared/src/server/repositories/observations.ts
+++ b/packages/shared/src/server/repositories/observations.ts
@@ -1635,10 +1635,12 @@ export const getObservationCountsByProjectInCreationInterval = async ({
start,
end,
projectId,
+ projectIds,
}: {
start: Date;
end: Date;
projectId?: string;
+ projectIds?: string[];
}) => {
const query = `
SELECT
@@ -1648,6 +1650,7 @@ export const getObservationCountsByProjectInCreationInterval = async ({
WHERE created_at >= {start: DateTime64(3)}
AND created_at < {end: DateTime64(3)}
${projectId ? "AND project_id = {projectId: String}" : ""}
+ ${projectIds ? "AND project_id IN ({projectIds: Array(String)})" : ""}
GROUP BY project_id
`;
@@ -1657,6 +1660,7 @@ export const getObservationCountsByProjectInCreationInterval = async ({
start: convertDateToClickhouseDateTime(start),
end: convertDateToClickhouseDateTime(end),
...(projectId ? { projectId } : {}),
+ ...(projectIds ? { projectIds } : {}),
},
clickhouseConfigs: {
request_timeout: 300000, // 5 minutes timeout
diff --git a/packages/shared/src/server/repositories/scores.ts b/packages/shared/src/server/repositories/scores.ts
index 7da2779c36..1e9dfcce8f 100644
--- a/packages/shared/src/server/repositories/scores.ts
+++ b/packages/shared/src/server/repositories/scores.ts
@@ -2355,10 +2355,12 @@ export const getScoreCountsByProjectInCreationInterval = async ({
start,
end,
projectId,
+ projectIds,
}: {
start: Date;
end: Date;
projectId?: string;
+ projectIds?: string[];
}) => {
const query = `
SELECT
@@ -2368,6 +2370,7 @@ export const getScoreCountsByProjectInCreationInterval = async ({
WHERE created_at >= {start: DateTime64(3)}
AND created_at < {end: DateTime64(3)}
${projectId ? "AND project_id = {projectId: String}" : ""}
+ ${projectIds ? "AND project_id IN ({projectIds: Array(String)})" : ""}
AND data_type IN ({dataTypes: Array(String)})
GROUP BY project_id
`;
@@ -2379,6 +2382,7 @@ export const getScoreCountsByProjectInCreationInterval = async ({
end: convertDateToClickhouseDateTime(end),
dataTypes: LISTABLE_SCORE_TYPES,
...(projectId ? { projectId } : {}),
+ ...(projectIds ? { projectIds } : {}),
},
clickhouseConfigs: {
request_timeout: 300000, // 5 minutes timeout
diff --git a/packages/shared/src/server/repositories/traces.ts b/packages/shared/src/server/repositories/traces.ts
index 0fae71f747..5a712573f1 100644
--- a/packages/shared/src/server/repositories/traces.ts
+++ b/packages/shared/src/server/repositories/traces.ts
@@ -366,10 +366,12 @@ export const getTraceCountsByProjectInCreationInterval = async ({
start,
end,
projectId,
+ projectIds,
}: {
start: Date;
end: Date;
projectId?: string;
+ projectIds?: string[];
}) => {
const query = `
SELECT
@@ -379,6 +381,7 @@ export const getTraceCountsByProjectInCreationInterval = async ({
WHERE created_at >= {start: DateTime64(3)}
AND created_at < {end: DateTime64(3)}
${projectId ? "AND project_id = {projectId: String}" : ""}
+ ${projectIds ? "AND project_id IN ({projectIds: Array(String)})" : ""}
GROUP BY project_id
`;
@@ -388,6 +391,7 @@ export const getTraceCountsByProjectInCreationInterval = async ({
start: convertDateToClickhouseDateTime(start),
end: convertDateToClickhouseDateTime(end),
...(projectId ? { projectId } : {}),
+ ...(projectIds ? { projectIds } : {}),
},
clickhouseConfigs: {
request_timeout: 300000, // 5 minutes timeout
diff --git a/web/src/__tests__/server/unit/chbBillingService.servertest.ts b/web/src/__tests__/server/unit/chbBillingService.servertest.ts
index 5b86bddc8f..974794b23f 100644
--- a/web/src/__tests__/server/unit/chbBillingService.servertest.ts
+++ b/web/src/__tests__/server/unit/chbBillingService.servertest.ts
@@ -1,12 +1,15 @@
import type { CloudConfigSchema } from "@langfuse/shared";
import type * as SharedServer from "@langfuse/shared/src/server";
import { TRPCError } from "@trpc/server";
-import { beforeEach, describe, expect, it, vi } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
env: { NEXTAUTH_URL: "https://cloud.langfuse.com" },
logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
auditLog: vi.fn(),
+ traceCounts: vi.fn(),
+ observationCounts: vi.fn(),
+ scoreCounts: vi.fn(),
}));
vi.mock("@/src/env.mjs", () => ({ env: mocks.env }));
@@ -15,11 +18,17 @@ vi.mock("@/src/features/audit-logs/server", () => ({
auditLog: mocks.auditLog,
}));
-// Only the logger is stubbed: getBillingCycleStart/End are the real cached-cycle
-// fallback this service shares with the Stripe path.
+// The logger and the ClickHouse unit counts are stubbed: getBillingCycleStart/End
+// are the real cached-cycle fallback this service shares with the Stripe path.
vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
const actual = await importOriginal<typeof SharedServer>();
- return { ...actual, logger: mocks.logger };
+ return {
+ ...actual,
+ logger: mocks.logger,
+ getTraceCountsByProjectInCreationInterval: mocks.traceCounts,
+ getObservationCountsByProjectInCreationInterval: mocks.observationCounts,
+ getScoreCountsByProjectInCreationInterval: mocks.scoreCounts,
+ };
});
import {
@@ -48,6 +57,7 @@ const update = vi.fn();
// Checkout claims the CH organization id with a guarded UPDATE rather than a
// read-then-write, so the claim is a raw statement returning a row count.
const executeRaw = vi.fn();
+const projectFindMany = vi.fn();
const clientMock = {
createCheckoutSession: vi.fn(),
@@ -59,7 +69,11 @@ const clientMock = {
};
const ctx = {
- prisma: { organization: { findUnique, update }, $executeRaw: executeRaw },
+ prisma: {
+ organization: { findUnique, update },
+ project: { findMany: projectFindMany },
+ $executeRaw: executeRaw,
+ },
session: {
orgId: ORG_ID,
orgRole: "OWNER",
@@ -770,6 +784,75 @@ describe("chbBillingService", () => {
});
});
+ describe("getUsage", () => {
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
+ it("counts live projects' ingested units over CHB's billing period", async () => {
+ vi.useFakeTimers({ toFake: ["Date"] });
+ vi.setSystemTime(new Date("2026-10-06T12:00:00Z"));
+ withOrg(chbConfig());
+ clientMock.getAttachedPlan.mockResolvedValue({
+ id: ATTACHED_PLAN_ID,
+ period: {
+ startDate: "2026-10-01T00:00:00Z",
+ endDate: "2026-11-01T00:00:00Z",
+ },
+ });
+ projectFindMany.mockResolvedValue([
+ { id: "project-1" },
+ { id: "project-2" },
+ ]);
+ mocks.traceCounts.mockResolvedValue([
+ { projectId: "project-1", count: 10 },
+ ]);
+ mocks.observationCounts.mockResolvedValue([
+ { projectId: "project-1", count: 100 },
+ { projectId: "project-2", count: 50 },
+ ]);
+ mocks.scoreCounts.mockResolvedValue([
+ { projectId: "project-2", count: 5 },
+ ]);
+
+ const usage = await service().getUsage(ORG_ID);
+
+ // CHB's calendar month, not the org's cycle anchored on the 15th or its
+ // cached counter
+ expect(usage).toEqual({
+ usageCount: 165,
+ usageType: "units",
+ billingPeriod: {
+ start: new Date("2026-10-01T00:00:00Z"),
+ end: new Date("2026-11-01T00:00:00Z"),
+ },
+ });
+ expect(projectFindMany).toHaveBeenCalledWith(
+ expect.objectContaining({ where: { orgId: ORG_ID, deletedAt: null } }),
+ );
+ const interval = {
+ start: new Date("2026-10-01T00:00:00Z"),
+ end: new Date("2026-10-06T12:00:00Z"),
+ projectIds: ["project-1", "project-2"],
+ };
+ expect(mocks.traceCounts).toHaveBeenCalledWith(interval);
+ expect(mocks.observationCounts).toHaveBeenCalledWith(interval);
+ expect(mocks.scoreCounts).toHaveBeenCalledWith(interval);
+ });
+
+ it("falls back to the cached cycle counter without an attached plan", async () => {
+ withOrg({ clickhouse: { organizationId: CH_ORG_ID } });
+
+ const usage = await service().getUsage(ORG_ID);
+
+ expect(clientMock.getAttachedPlan).not.toHaveBeenCalled();
+ expect(mocks.traceCounts).not.toHaveBeenCalled();
+ expect(usage.usageCount).toBe(4_200);
+ expect(usage.usageType).toBe("units");
+ expect(usage.billingPeriod.start.getUTCDate()).toBe(15);
+ });
+ });
+
describe("remaining surface", () => {
it("builds a portal session against the org's CH id", async () => {
withOrg(chbConfig());
@@ -794,16 +877,6 @@ describe("chbBillingService", () => {
);
});
- it("reports usage from the cached cycle counter", async () => {
- withOrg(chbConfig());
-
- const usage = await service().getUsage(ORG_ID);
-
- expect(usage.usageCount).toBe(4_200);
- expect(usage.usageType).toBe("units");
- expect(usage.billingPeriod.start.getUTCDate()).toBe(15);
- });
-
it("rejects promotion codes, which CHB has no API for", async () => {
withOrg(chbConfig());
diff --git a/web/src/ee/features/billing/server/chb/chbBillingService.ts b/web/src/ee/features/billing/server/chb/chbBillingService.ts
index 324f77d42c..3353208a1b 100644
--- a/web/src/ee/features/billing/server/chb/chbBillingService.ts
+++ b/web/src/ee/features/billing/server/chb/chbBillingService.ts
@@ -7,6 +7,9 @@ import { CloudConfigSchema, parseDbOrg } from "@langfuse/shared";
import {
getBillingCycleEnd,
getBillingCycleStart,
+ getObservationCountsByProjectInCreationInterval,
+ getScoreCountsByProjectInCreationInterval,
+ getTraceCountsByProjectInCreationInterval,
logger,
} from "@langfuse/shared/src/server";
@@ -76,6 +79,16 @@ export class ChbBillingService {
return `${env.NEXTAUTH_URL}/organization/${orgId}/settings/billing`;
}
+ private toBillingPeriod(
+ attachedPlan: ChbAttachedPlan,
+ ): { start: Date; end: Date } | null {
+ const startDate = attachedPlan.period?.startDate;
+ const endDate = attachedPlan.period?.endDate;
+ return startDate && endDate
+ ? { start: new Date(startDate), end: new Date(endDate) }
+ : null;
+ }
+
private toUnixSeconds(value: string | null | undefined): number | null {
if (!value) return null;
const ms = Date.parse(value);
@@ -145,19 +158,9 @@ export class ChbBillingService {
chOrganizationId: chb.organizationId,
});
- const periodStart = attachedPlan.period?.startDate
- ? new Date(attachedPlan.period.startDate)
- : null;
- const periodEnd = attachedPlan.period?.endDate
- ? new Date(attachedPlan.period.endDate)
- : null;
-
return {
...this.mapScheduled(attachedPlan),
- billingPeriod:
- periodStart && periodEnd
- ? { start: periodStart, end: periodEnd }
- : null,
+ billingPeriod: this.toBillingPeriod(attachedPlan),
// No promotion-code API on the CHB path yet
discounts: [],
hasValidPaymentMethod: attachedPlan.payment?.status === "active",
@@ -629,23 +632,71 @@ export class ChbBillingService {
}
/**
- * v1 usage source of truth for CHB orgs is the existing non-Stripe
- * fallback: billing cycle from the org's anchor + the cached cycle usage
- * the hourly job maintains. Spend-in-USD can later come from
- * `GET /invoices?includePreview=true` (the open period's accrued usage).
+ * With an attached plan, usage is counted live over CHB's own billing
+ * period, by ingestion time (`created_at`) and with the same queries the
+ * billing metrics API serves to CHB's metering, so the figure tracks what
+ * CHB invoices. Without an attached plan (or a period on it), usage falls
+ * back to the cached counter the hourly free-tier job maintains for the
+ * org's anchored cycle.
*/
async getUsage(orgId: string) {
- const { org } = await this.getParsedOrg(orgId);
+ const { org, parsedOrg } = await this.getParsedOrg(orgId);
+ const chb = parsedOrg.cloudConfig?.clickhouse;
+
+ const billingPeriod = chb?.attachedPlanId
+ ? this.toBillingPeriod(
+ await this.client.getAttachedPlan({
+ chOrganizationId: chb.organizationId,
+ }),
+ )
+ : null;
+
+ if (!billingPeriod) {
+ const now = new Date();
+ return {
+ usageCount: org.cloudCurrentCycleUsage ?? 0,
+ usageType: "units",
+ billingPeriod: {
+ start: getBillingCycleStart(org, now),
+ end: getBillingCycleEnd(org, now),
+ },
+ };
+ }
- const now = new Date();
return {
- usageCount: org.cloudCurrentCycleUsage ?? 0,
+ usageCount: await this.countBillableUnits(orgId, billingPeriod),
usageType: "units",
- billingPeriod: {
- start: getBillingCycleStart(org, now),
- end: getBillingCycleEnd(org, now),
- },
+ billingPeriod,
+ };
+ }
+
+ /**
+ * Billable units (traces + observations + scores) ingested into the org's
+ * live projects since the period start. Soft-deleted projects are left out,
+ * as they are when usage is metered.
+ */
+ private async countBillableUnits(
+ orgId: string,
+ period: { start: Date; end: Date },
+ ): Promise<number> {
+ const projects = await this.ctx.prisma.project.findMany({
+ where: { orgId, deletedAt: null },
+ select: { id: true },
+ });
+ const end = new Date(Math.min(Date.now(), period.end.getTime()));
+ if (projects.length === 0 || period.start >= end) return 0;
+
+ const interval = {
+ start: period.start,
+ end,
+ projectIds: projects.map((project) => project.id),
};
+ const counts = await Promise.all([
+ getTraceCountsByProjectInCreationInterval(interval),
+ getObservationCountsByProjectInCreationInterval(interval),
+ getScoreCountsByProjectInCreationInterval(interval),
+ ]);
+ return counts.flat().reduce((sum, row) => sum + row.count, 0);
}
async applyPromotionCode(