Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(billing): record billable units in an append-only ClickHouse ledger (#18722)

Steffen Reuthere6e7e35f800c
diff --git a/packages/shared/AGENTS.md b/packages/shared/AGENTS.md
index 4ea945c841..d03b0e40a0 100644
--- a/packages/shared/AGENTS.md
+++ b/packages/shared/AGENTS.md
@@ -28,6 +28,10 @@
generation in the full thread.
- Domain model types: `src/domain/*`
- Repository layer: `src/server/repositories/*`
+- Billing ledger: the append-only `billable_events` table (created by
+ `clickhouse/scripts/dev-tables.sh`, applied by hand on Cloud, not
+ migration-managed) read via `src/server/repositories/billable-events.ts`.
+ Keep it out of every delete, retention and cleaner path.
- Queue payload schemas: `src/server/queues.ts`
- Queue helpers: `src/server/redis/*`
`getShardIndex` from `src/server/redis/sharding.ts` is exported by the server
diff --git a/packages/shared/clickhouse/scripts/dev-tables.sh b/packages/shared/clickhouse/scripts/dev-tables.sh
index 1161b2a578..88ed4dfd08 100644
--- a/packages/shared/clickhouse/scripts/dev-tables.sh
+++ b/packages/shared/clickhouse/scripts/dev-tables.sh
@@ -123,6 +123,79 @@ SELECT
byteSize(*) AS total_size
FROM traces;
+-- Billing ledger: one row per billable unit (observation, trace, score) each
+-- time it is ingested. A unit is billed once in a billing window of
+-- ingested_at when its id appears there and not in the 24 hours before the
+-- window starts (getBillableUnitCountsByProjectInWindow).
+CREATE TABLE IF NOT EXISTS billable_events (
+ ingested_at DateTime('UTC') DEFAULT now(),
+ project_id String,
+ event_type LowCardinality(String),
+ event_id String,
+ event_time DateTime64(3),
+ event_bytes UInt64
+) ENGINE = MergeTree
+PARTITION BY toYYYYMM(ingested_at)
+ORDER BY (toStartOfFifteenMinutes(ingested_at), project_id, event_type, event_id);
+
+-- The billing MVs only take rows created in the last 48 hours, so re-inserts
+-- of historic rows that keep their created_at (v4 backfills, updates) are not
+-- new usage. Rows with is_deleted = 1 are tombstones, not usage.
+
+-- MV: events_full -> billable_events (observations). Dual-write trace spans
+-- (span_id 't-<trace_id>') only count as a trace. Span ids are only unique
+-- within their trace, and ingestion API clients choose them, so event_id is
+-- '<length(trace_id)>:<trace_id><span_id>'. The length prefix keeps distinct
+-- (trace_id, span_id) pairs from concatenating to the same id.
+CREATE MATERIALIZED VIEW IF NOT EXISTS billable_events_observations_mv
+TO billable_events AS
+SELECT
+ now() AS ingested_at,
+ project_id,
+ 'observation' AS event_type,
+ concat(toString(length(trace_id)), ':', trace_id, span_id) AS event_id,
+ start_time AS event_time,
+ event_bytes
+FROM events_full
+WHERE is_deleted = 0
+ AND (
+ span_id != concat('t-', trace_id)
+ AND source NOT LIKE '%-dual-write'
+ )
+ AND created_at >= now() - INTERVAL 48 HOUR;
+
+-- MV: events_full -> billable_events (traces), one row per trace per insert
+-- block. event_bytes carries the dual-write trace span, which has no
+-- observation row, so event_bytes over all rows adds up to events_full.
+CREATE MATERIALIZED VIEW IF NOT EXISTS billable_events_traces_mv
+TO billable_events AS
+SELECT
+ now() AS ingested_at,
+ project_id,
+ 'trace' AS event_type,
+ trace_id AS event_id,
+ min(start_time) AS event_time,
+ sumIf(event_bytes, span_id = concat('t-', trace_id)) AS event_bytes
+FROM events_full
+WHERE is_deleted = 0
+ AND trace_id != ''
+ AND created_at >= now() - INTERVAL 48 HOUR
+GROUP BY project_id, trace_id;
+
+-- MV: scores -> billable_events.
+CREATE MATERIALIZED VIEW IF NOT EXISTS billable_events_scores_mv
+TO billable_events AS
+SELECT
+ now() AS ingested_at,
+ project_id,
+ 'score' AS event_type,
+ id AS event_id,
+ timestamp AS event_time,
+ byteSize(*) AS event_bytes
+FROM scores
+WHERE is_deleted = 0
+ AND created_at >= now() - INTERVAL 48 HOUR;
+
-- PROD ROLLOUT: unlike analytics_scores (managed via numbered migrations),
-- this view is NOT migration-managed because it is a Cloud-only diagnostic
-- over events_core. Any change here must be applied manually as DROP VIEW IF
@@ -183,6 +256,8 @@ clickhouse client \
SET type_json_skip_duplicated_paths = 1;
TRUNCATE events_core;
TRUNCATE events_full;
+ -- The inserts below fire the billing MVs, so reset their target as well.
+ TRUNCATE billable_events;
-- Insert observations into events_full (experiment metadata included when dataset_run_items match)
INSERT INTO events_full (project_id, trace_id, span_id, parent_span_id, start_time, end_time, name, type,
diff --git a/packages/shared/src/server/repositories/billable-events.ts b/packages/shared/src/server/repositories/billable-events.ts
new file mode 100644
index 0000000000..48bcb99d38
--- /dev/null
+++ b/packages/shared/src/server/repositories/billable-events.ts
@@ -0,0 +1,94 @@
+import { queryClickhouse } from "./clickhouse";
+import { convertDateToClickhouseDateTime } from "../clickhouse/client";
+
+export type BillableEventType = "observation" | "trace" | "score";
+
+const BILLING_LOOKBACK_MS = 24 * 60 * 60 * 1000;
+
+/**
+ * Counts the billable units per project and event type ingested in
+ * `[windowFrom, windowTo)` of `billable_events.ingested_at`. Each unit counts
+ * once per window, and not at all if its id was also ingested in the 24 hours
+ * before `windowFrom`. A re-sent or long-running trace is therefore billed once
+ * as long as it never goes 24 hours without new data.
+ *
+ * `billable_events` is never deleted from, so a closed window always
+ * recomputes to the same counts. A window still receives inserts until a few
+ * minutes after `windowTo`; counting it earlier under-bills.
+ *
+ * The table is created by `clickhouse/scripts/dev-tables.sh` locally and by
+ * hand on Langfuse Cloud. Where it does not exist, the query fails.
+ */
+export const getBillableUnitCountsByProjectInWindow = async ({
+ windowFrom,
+ windowTo,
+ projectId,
+}: {
+ windowFrom: Date;
+ windowTo: Date;
+ projectId?: string;
+}): Promise<
+ Array<{ projectId: string; eventType: BillableEventType; count: number }>
+> => {
+ if (windowTo.getTime() <= windowFrom.getTime()) {
+ throw new Error(
+ `Billing window must end after it starts, got ${windowFrom.toISOString()} to ${windowTo.toISOString()}`,
+ );
+ }
+
+ const projectFilter = projectId ? "AND project_id = {projectId: String}" : "";
+
+ // The NOT IN set only holds lookback ids that also appear in the window, so
+ // memory grows with the window, not with the 24-hour lookback.
+ const query = `
+ WITH window_units AS (
+ SELECT project_id, event_type, event_id
+ FROM billable_events
+ WHERE ingested_at >= {windowFrom: DateTime64(3, 'UTC')}
+ AND ingested_at < {windowTo: DateTime64(3, 'UTC')}
+ ${projectFilter}
+ )
+ SELECT
+ project_id,
+ event_type,
+ uniqExact(event_id) AS count
+ FROM window_units
+ WHERE (project_id, event_type, event_id) NOT IN (
+ SELECT project_id, event_type, event_id
+ FROM billable_events
+ WHERE ingested_at >= {lookbackFrom: DateTime64(3, 'UTC')}
+ AND ingested_at < {windowFrom: DateTime64(3, 'UTC')}
+ ${projectFilter}
+ AND (project_id, event_type, event_id) IN (
+ SELECT project_id, event_type, event_id FROM window_units
+ )
+ )
+ GROUP BY project_id, event_type
+ `;
+
+ const rows = await queryClickhouse<{
+ project_id: string;
+ event_type: BillableEventType;
+ count: string;
+ }>({
+ query,
+ params: {
+ windowFrom: convertDateToClickhouseDateTime(windowFrom),
+ windowTo: convertDateToClickhouseDateTime(windowTo),
+ lookbackFrom: convertDateToClickhouseDateTime(
+ new Date(windowFrom.getTime() - BILLING_LOOKBACK_MS),
+ ),
+ ...(projectId ? { projectId } : {}),
+ },
+ clickhouseConfigs: {
+ request_timeout: 300000, // 5 minutes timeout
+ },
+ ...(projectId ? { tags: { projectId } } : {}),
+ });
+
+ return rows.map((row) => ({
+ projectId: row.project_id,
+ eventType: row.event_type,
+ count: Number(row.count),
+ }));
+};
diff --git a/packages/shared/src/server/repositories/index.ts b/packages/shared/src/server/repositories/index.ts
index b9c7897837..7dc92c8788 100644
--- a/packages/shared/src/server/repositories/index.ts
+++ b/packages/shared/src/server/repositories/index.ts
@@ -29,3 +29,4 @@ export * from "./daily-metrics";
export * from "./dataset-runs";
export * from "./score-configs";
export * from "./auditLogs";
+export * from "./billable-events";
diff --git a/worker/src/__tests__/billableEvents.test.ts b/worker/src/__tests__/billableEvents.test.ts
new file mode 100644
index 0000000000..f94950e7ee
--- /dev/null
+++ b/worker/src/__tests__/billableEvents.test.ts
@@ -0,0 +1,328 @@
+import { describe, expect, it, type TestContext } from "vitest";
+import { randomUUID } from "crypto";
+import {
+ type BillableEventType,
+ clickhouseClient,
+ createEvent,
+ createEventsCh,
+ createScoresCh,
+ createTraceScore,
+ deleteEventsByTraceIds,
+ deleteScoresByTraceIds,
+ getBillableUnitCountsByProjectInWindow,
+ queryClickhouse,
+} from "@langfuse/shared/src/server";
+import { skipUnlessClickhouseTablesExist } from "./helpers/clickhouseTables";
+
+// billable_events and its MVs are created only by the ClickHouse dev-tables
+// setup, which CI runs for the default deploy mode only.
+const skipUnlessBillableEvents = (ctx: TestContext) =>
+ skipUnlessClickhouseTablesExist(
+ ctx,
+ ["billable_events"],
+ "billable_events is created by ch:dev-tables only",
+ );
+
+const SECOND = 1000;
+const MINUTE = 60 * SECOND;
+const HOUR = 60 * MINUTE;
+const THREE_DAYS_AGO = Date.now() - 72 * HOUR;
+
+/** Ledger id of an observation: span ids are only unique within a trace. */
+const observationEventId = (traceId: string, spanId: string) =>
+ `${traceId.length}:${traceId}${spanId}`;
+
+const getBillableRows = (projectId: string) =>
+ queryClickhouse<{
+ event_type: BillableEventType;
+ event_id: string;
+ event_bytes: string;
+ }>({
+ query: `
+ SELECT event_type, event_id, event_bytes
+ FROM billable_events
+ WHERE project_id = {projectId: String}
+ ORDER BY event_type, event_id
+ `,
+ params: { projectId },
+ });
+
+const countsInWindow = async (
+ projectId: string,
+ windowFrom: Date,
+ windowTo: Date,
+) => {
+ const rows = await getBillableUnitCountsByProjectInWindow({
+ windowFrom,
+ windowTo,
+ projectId,
+ });
+ return Object.fromEntries(rows.map((row) => [row.eventType, row.count]));
+};
+
+/** Billed units of rows the MVs just wrote, whose ingested_at is now. */
+const billedUnitsAroundNow = (projectId: string) =>
+ countsInWindow(
+ projectId,
+ new Date(Date.now() - HOUR),
+ new Date(Date.now() + HOUR),
+ );
+
+/** Inserts ledger rows directly, to place them in chosen ingestion windows. */
+const insertBillableEvents = (
+ projectId: string,
+ rows: Array<{ eventType: BillableEventType; eventId: string; at: Date }>,
+) =>
+ clickhouseClient().insert({
+ table: "billable_events",
+ format: "JSONEachRow",
+ values: rows.map((row) => ({
+ ingested_at: row.at.toISOString().slice(0, 19).replace("T", " "),
+ project_id: projectId,
+ event_type: row.eventType,
+ event_id: row.eventId,
+ event_time: row.at.toISOString().replace("T", " ").replace("Z", ""),
+ event_bytes: 0,
+ })),
+ });
+
+describe("billable_events materialized views", () => {
+ it("records observations, one row per trace and every score type", async (ctx) => {
+ await skipUnlessBillableEvents(ctx);
+ const projectId = randomUUID();
+ const traceId = randomUUID();
+ const backfilledTraceId = randomUUID();
+
+ await createEventsCh([
+ createEvent({
+ project_id: projectId,
+ trace_id: traceId,
+ span_id: "span-1",
+ event_bytes: 10,
+ }),
+ createEvent({
+ project_id: projectId,
+ trace_id: traceId,
+ span_id: "span-2",
+ event_bytes: 20,
+ }),
+ // Dual-write trace span: only counts as the trace.
+ createEvent({
+ project_id: projectId,
+ trace_id: traceId,
+ span_id: `t-${traceId}`,
+ event_bytes: 7,
+ }),
+ // Tombstone: not usage.
+ createEvent({
+ project_id: projectId,
+ trace_id: traceId,
+ span_id: "span-deleted",
+ is_deleted: 1,
+ }),
+ ]);
+ // A backfill keeps the source created_at, so it is not new usage.
+ await createEventsCh([
+ createEvent({
+ project_id: projectId,
+ trace_id: backfilledTraceId,
+ span_id: "span-backfilled",
+ created_at: THREE_DAYS_AGO,
+ }),
+ ]);
+ await createScoresCh([
+ createTraceScore({
+ project_id: projectId,
+ trace_id: traceId,
+ id: "score-numeric",
+ data_type: "NUMERIC",
+ }),
+ createTraceScore({
+ project_id: projectId,
+ trace_id: traceId,
+ id: "score-correction",
+ data_type: "CORRECTION",
+ }),
+ createTraceScore({
+ project_id: projectId,
+ trace_id: traceId,
+ id: "score-backfilled",
+ created_at: THREE_DAYS_AGO,
+ }),
+ ]);
+
+ const rows = await getBillableRows(projectId);
+ expect(
+ rows.map((row) => ({
+ type: row.event_type,
+ id: row.event_id,
+ bytes: Number(row.event_bytes),
+ })),
+ ).toEqual([
+ {
+ type: "observation",
+ id: observationEventId(traceId, "span-1"),
+ bytes: 10,
+ },
+ {
+ type: "observation",
+ id: observationEventId(traceId, "span-2"),
+ bytes: 20,
+ },
+ { type: "score", id: "score-correction", bytes: expect.any(Number) },
+ { type: "score", id: "score-numeric", bytes: expect.any(Number) },
+ { type: "trace", id: traceId, bytes: 7 },
+ ]);
+ });
+
+ it("bills a span id once per trace it appears in", async (ctx) => {
+ await skipUnlessBillableEvents(ctx);
+ const projectId = randomUUID();
+
+ await createEventsCh([
+ // Ingestion API clients choose observation ids and may reuse them.
+ createEvent({
+ project_id: projectId,
+ trace_id: randomUUID(),
+ span_id: "llm-call",
+ }),
+ createEvent({
+ project_id: projectId,
+ trace_id: randomUUID(),
+ span_id: "llm-call",
+ }),
+ // Trace and span ids whose plain concatenation would collide.
+ createEvent({ project_id: projectId, trace_id: "t:x", span_id: "s" }),
+ createEvent({ project_id: projectId, trace_id: "t", span_id: "x:s" }),
+ ]);
+
+ expect(await billedUnitsAroundNow(projectId)).toEqual({
+ observation: 4,
+ trace: 4,
+ });
+ });
+
+ it("keeps billed units when the trace is deleted", async (ctx) => {
+ await skipUnlessBillableEvents(ctx);
+ const projectId = randomUUID();
+ const traceId = randomUUID();
+
+ await createEventsCh([
+ createEvent({ project_id: projectId, trace_id: traceId }),
+ createEvent({ project_id: projectId, trace_id: traceId }),
+ ]);
+ await createScoresCh([
+ createTraceScore({ project_id: projectId, trace_id: traceId }),
+ ]);
+ const billedBeforeDelete = await billedUnitsAroundNow(projectId);
+ expect(billedBeforeDelete).toEqual({ observation: 2, trace: 1, score: 1 });
+
+ await deleteEventsByTraceIds(projectId, [traceId]);
+ await deleteScoresByTraceIds(projectId, [traceId]);
+
+ const remainingEvents = await queryClickhouse<{ count: string }>({
+ query: `SELECT count() AS count FROM events_full WHERE project_id = {projectId: String}`,
+ params: { projectId },
+ });
+ expect(Number(remainingEvents[0].count)).toBe(0);
+ expect(await billedUnitsAroundNow(projectId)).toEqual(billedBeforeDelete);
+ });
+});
+
+describe("getBillableUnitCountsByProjectInWindow", () => {
+ // A fixed window start; each test uses its own project.
+ const T0 = new Date("2026-01-05T00:00:00.000Z");
+ const at = (offsetMs: number) => new Date(T0.getTime() + offsetMs);
+
+ it("deduplicates within the window and against the 24 hours before it", async (ctx) => {
+ await skipUnlessBillableEvents(ctx);
+ const projectId = randomUUID();
+ await insertBillableEvents(projectId, [
+ { eventType: "trace", eventId: "twice-in-window", at: at(0) },
+ { eventType: "trace", eventId: "twice-in-window", at: at(40 * MINUTE) },
+ // The lookback starts exactly 24 hours before windowFrom.
+ { eventType: "trace", eventId: "resent-within-24h", at: at(-24 * HOUR) },
+ { eventType: "trace", eventId: "resent-within-24h", at: at(5 * MINUTE) },
+ {
+ eventType: "trace",
+ eventId: "resent-after-24h",
+ at: at(-24 * HOUR - SECOND),
+ },
+ { eventType: "trace", eventId: "resent-after-24h", at: at(5 * MINUTE) },
+ // windowTo is exclusive, so this belongs to the next window.
+ { eventType: "trace", eventId: "at-window-end", at: at(HOUR) },
+ ]);
+
+ expect(await countsInWindow(projectId, T0, at(HOUR))).toEqual({
+ trace: 2,
+ });
+ expect(await countsInWindow(projectId, at(HOUR), at(2 * HOUR))).toEqual({
+ trace: 1,
+ });
+ });
+
+ it("bills a multi-day trace once while its gaps stay under 24 hours", async (ctx) => {
+ await skipUnlessBillableEvents(ctx);
+ const projectId = randomUUID();
+ const windowStarts = [0, 20 * HOUR, 40 * HOUR, 60 * HOUR].map(at);
+ await insertBillableEvents(
+ projectId,
+ windowStarts.map((windowFrom) => ({
+ eventType: "trace",
+ eventId: "long-running-trace",
+ at: windowFrom,
+ })),
+ );
+
+ const counts = await Promise.all(
+ windowStarts.map((windowFrom) =>
+ countsInWindow(
+ projectId,
+ windowFrom,
+ new Date(windowFrom.getTime() + 15 * MINUTE),
+ ),
+ ),
+ );
+ expect(counts).toEqual([{ trace: 1 }, {}, {}, {}]);
+ });
+
+ it("counts per project and event type", async (ctx) => {
+ await skipUnlessBillableEvents(ctx);
+ const projectA = randomUUID();
+ const projectB = randomUUID();
+ await insertBillableEvents(projectA, [
+ { eventType: "observation", eventId: "id-1", at: at(MINUTE) },
+ { eventType: "observation", eventId: "id-1", at: at(2 * MINUTE) },
+ { eventType: "observation", eventId: "id-2", at: at(14 * MINUTE) },
+ { eventType: "trace", eventId: "id-1", at: at(MINUTE) },
+ { eventType: "score", eventId: "id-3", at: at(MINUTE) },
+ ]);
+ // The same id in another project is a separate unit.
+ await insertBillableEvents(projectB, [
+ { eventType: "observation", eventId: "id-1", at: at(MINUTE) },
+ ]);
+
+ expect(await countsInWindow(projectA, T0, at(15 * MINUTE))).toEqual({
+ observation: 2,
+ trace: 1,
+ score: 1,
+ });
+
+ const allProjects = await getBillableUnitCountsByProjectInWindow({
+ windowFrom: T0,
+ windowTo: at(15 * MINUTE),
+ });
+ expect(
+ allProjects.filter((row) => row.projectId === projectA),
+ ).toHaveLength(3);
+ expect(allProjects.filter((row) => row.projectId === projectB)).toEqual([
+ { projectId: projectB, eventType: "observation", count: 1 },
+ ]);
+ });
+
+ it("rejects a window that does not end after it starts", async () => {
+ await expect(
+ getBillableUnitCountsByProjectInWindow({ windowFrom: T0, windowTo: T0 }),
+ ).rejects.toThrow("must end after it starts");
+ });
+});