🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(billing): record billable units in an append-only ClickHouse ledger (#18722)
Steffen Reuthere6e7e35f800c
diff --git a/packages/shared/AGENTS.md b/packages/shared/AGENTS.mdindex 4ea945c841..d03b0e40a0 100644--- a/packages/shared/AGENTS.md+++ b/packages/shared/AGENTS.md@@ -28,6 +28,10 @@generation in the full thread.- Domain model types: `src/domain/*`- Repository layer: `src/server/repositories/*`+- Billing ledger: the append-only `billable_events` table (created by+ `clickhouse/scripts/dev-tables.sh`, applied by hand on Cloud, not+ migration-managed) read via `src/server/repositories/billable-events.ts`.+ Keep it out of every delete, retention and cleaner path.- Queue payload schemas: `src/server/queues.ts`- Queue helpers: `src/server/redis/*``getShardIndex` from `src/server/redis/sharding.ts` is exported by the serverdiff --git a/packages/shared/clickhouse/scripts/dev-tables.sh b/packages/shared/clickhouse/scripts/dev-tables.shindex 1161b2a578..88ed4dfd08 100644--- a/packages/shared/clickhouse/scripts/dev-tables.sh+++ b/packages/shared/clickhouse/scripts/dev-tables.sh@@ -123,6 +123,79 @@ SELECTbyteSize(*) AS total_sizeFROM traces;+-- Billing ledger: one row per billable unit (observation, trace, score) each+-- time it is ingested. A unit is billed once in a billing window of+-- ingested_at when its id appears there and not in the 24 hours before the+-- window starts (getBillableUnitCountsByProjectInWindow).+CREATE TABLE IF NOT EXISTS billable_events (+ ingested_at DateTime('UTC') DEFAULT now(),+ project_id String,+ event_type LowCardinality(String),+ event_id String,+ event_time DateTime64(3),+ event_bytes UInt64+) ENGINE = MergeTree+PARTITION BY toYYYYMM(ingested_at)+ORDER BY (toStartOfFifteenMinutes(ingested_at), project_id, event_type, event_id);++-- The billing MVs only take rows created in the last 48 hours, so re-inserts+-- of historic rows that keep their created_at (v4 backfills, updates) are not+-- new usage. Rows with is_deleted = 1 are tombstones, not usage.++-- MV: events_full -> billable_events (observations). Dual-write trace spans+-- (span_id 't-<trace_id>') only count as a trace. Span ids are only unique+-- within their trace, and ingestion API clients choose them, so event_id is+-- '<length(trace_id)>:<trace_id><span_id>'. The length prefix keeps distinct+-- (trace_id, span_id) pairs from concatenating to the same id.+CREATE MATERIALIZED VIEW IF NOT EXISTS billable_events_observations_mv+TO billable_events AS+SELECT+ now() AS ingested_at,+ project_id,+ 'observation' AS event_type,+ concat(toString(length(trace_id)), ':', trace_id, span_id) AS event_id,+ start_time AS event_time,+ event_bytes+FROM events_full+WHERE is_deleted = 0+ AND (+ span_id != concat('t-', trace_id)+ AND source NOT LIKE '%-dual-write'+ )+ AND created_at >= now() - INTERVAL 48 HOUR;++-- MV: events_full -> billable_events (traces), one row per trace per insert+-- block. event_bytes carries the dual-write trace span, which has no+-- observation row, so event_bytes over all rows adds up to events_full.+CREATE MATERIALIZED VIEW IF NOT EXISTS billable_events_traces_mv+TO billable_events AS+SELECT+ now() AS ingested_at,+ project_id,+ 'trace' AS event_type,+ trace_id AS event_id,+ min(start_time) AS event_time,+ sumIf(event_bytes, span_id = concat('t-', trace_id)) AS event_bytes+FROM events_full+WHERE is_deleted = 0+ AND trace_id != ''+ AND created_at >= now() - INTERVAL 48 HOUR+GROUP BY project_id, trace_id;++-- MV: scores -> billable_events.+CREATE MATERIALIZED VIEW IF NOT EXISTS billable_events_scores_mv+TO billable_events AS+SELECT+ now() AS ingested_at,+ project_id,+ 'score' AS event_type,+ id AS event_id,+ timestamp AS event_time,+ byteSize(*) AS event_bytes+FROM scores+WHERE is_deleted = 0+ AND created_at >= now() - INTERVAL 48 HOUR;+-- PROD ROLLOUT: unlike analytics_scores (managed via numbered migrations),-- this view is NOT migration-managed because it is a Cloud-only diagnostic-- over events_core. Any change here must be applied manually as DROP VIEW IF@@ -183,6 +256,8 @@ clickhouse client \SET type_json_skip_duplicated_paths = 1;TRUNCATE events_core;TRUNCATE events_full;+ -- The inserts below fire the billing MVs, so reset their target as well.+ TRUNCATE billable_events;-- Insert observations into events_full (experiment metadata included when dataset_run_items match)INSERT INTO events_full (project_id, trace_id, span_id, parent_span_id, start_time, end_time, name, type,diff --git a/packages/shared/src/server/repositories/billable-events.ts b/packages/shared/src/server/repositories/billable-events.tsnew file mode 100644index 0000000000..48bcb99d38--- /dev/null+++ b/packages/shared/src/server/repositories/billable-events.ts@@ -0,0 +1,94 @@+import { queryClickhouse } from "./clickhouse";+import { convertDateToClickhouseDateTime } from "../clickhouse/client";++export type BillableEventType = "observation" | "trace" | "score";++const BILLING_LOOKBACK_MS = 24 * 60 * 60 * 1000;++/**+ * Counts the billable units per project and event type ingested in+ * `[windowFrom, windowTo)` of `billable_events.ingested_at`. Each unit counts+ * once per window, and not at all if its id was also ingested in the 24 hours+ * before `windowFrom`. A re-sent or long-running trace is therefore billed once+ * as long as it never goes 24 hours without new data.+ *+ * `billable_events` is never deleted from, so a closed window always+ * recomputes to the same counts. A window still receives inserts until a few+ * minutes after `windowTo`; counting it earlier under-bills.+ *+ * The table is created by `clickhouse/scripts/dev-tables.sh` locally and by+ * hand on Langfuse Cloud. Where it does not exist, the query fails.+ */+export const getBillableUnitCountsByProjectInWindow = async ({+ windowFrom,+ windowTo,+ projectId,+}: {+ windowFrom: Date;+ windowTo: Date;+ projectId?: string;+}): Promise<+ Array<{ projectId: string; eventType: BillableEventType; count: number }>+> => {+ if (windowTo.getTime() <= windowFrom.getTime()) {+ throw new Error(+ `Billing window must end after it starts, got ${windowFrom.toISOString()} to ${windowTo.toISOString()}`,+ );+ }++ const projectFilter = projectId ? "AND project_id = {projectId: String}" : "";++ // The NOT IN set only holds lookback ids that also appear in the window, so+ // memory grows with the window, not with the 24-hour lookback.+ const query = `+ WITH window_units AS (+ SELECT project_id, event_type, event_id+ FROM billable_events+ WHERE ingested_at >= {windowFrom: DateTime64(3, 'UTC')}+ AND ingested_at < {windowTo: DateTime64(3, 'UTC')}+ ${projectFilter}+ )+ SELECT+ project_id,+ event_type,+ uniqExact(event_id) AS count+ FROM window_units+ WHERE (project_id, event_type, event_id) NOT IN (+ SELECT project_id, event_type, event_id+ FROM billable_events+ WHERE ingested_at >= {lookbackFrom: DateTime64(3, 'UTC')}+ AND ingested_at < {windowFrom: DateTime64(3, 'UTC')}+ ${projectFilter}+ AND (project_id, event_type, event_id) IN (+ SELECT project_id, event_type, event_id FROM window_units+ )+ )+ GROUP BY project_id, event_type+ `;++ const rows = await queryClickhouse<{+ project_id: string;+ event_type: BillableEventType;+ count: string;+ }>({+ query,+ params: {+ windowFrom: convertDateToClickhouseDateTime(windowFrom),+ windowTo: convertDateToClickhouseDateTime(windowTo),+ lookbackFrom: convertDateToClickhouseDateTime(+ new Date(windowFrom.getTime() - BILLING_LOOKBACK_MS),+ ),+ ...(projectId ? { projectId } : {}),+ },+ clickhouseConfigs: {+ request_timeout: 300000, // 5 minutes timeout+ },+ ...(projectId ? { tags: { projectId } } : {}),+ });++ return rows.map((row) => ({+ projectId: row.project_id,+ eventType: row.event_type,+ count: Number(row.count),+ }));+};diff --git a/packages/shared/src/server/repositories/index.ts b/packages/shared/src/server/repositories/index.tsindex b9c7897837..7dc92c8788 100644--- a/packages/shared/src/server/repositories/index.ts+++ b/packages/shared/src/server/repositories/index.ts@@ -29,3 +29,4 @@ export * from "./daily-metrics";export * from "./dataset-runs";export * from "./score-configs";export * from "./auditLogs";+export * from "./billable-events";diff --git a/worker/src/__tests__/billableEvents.test.ts b/worker/src/__tests__/billableEvents.test.tsnew file mode 100644index 0000000000..f94950e7ee--- /dev/null+++ b/worker/src/__tests__/billableEvents.test.ts@@ -0,0 +1,328 @@+import { describe, expect, it, type TestContext } from "vitest";+import { randomUUID } from "crypto";+import {+ type BillableEventType,+ clickhouseClient,+ createEvent,+ createEventsCh,+ createScoresCh,+ createTraceScore,+ deleteEventsByTraceIds,+ deleteScoresByTraceIds,+ getBillableUnitCountsByProjectInWindow,+ queryClickhouse,+} from "@langfuse/shared/src/server";+import { skipUnlessClickhouseTablesExist } from "./helpers/clickhouseTables";++// billable_events and its MVs are created only by the ClickHouse dev-tables+// setup, which CI runs for the default deploy mode only.+const skipUnlessBillableEvents = (ctx: TestContext) =>+ skipUnlessClickhouseTablesExist(+ ctx,+ ["billable_events"],+ "billable_events is created by ch:dev-tables only",+ );++const SECOND = 1000;+const MINUTE = 60 * SECOND;+const HOUR = 60 * MINUTE;+const THREE_DAYS_AGO = Date.now() - 72 * HOUR;++/** Ledger id of an observation: span ids are only unique within a trace. */+const observationEventId = (traceId: string, spanId: string) =>+ `${traceId.length}:${traceId}${spanId}`;++const getBillableRows = (projectId: string) =>+ queryClickhouse<{+ event_type: BillableEventType;+ event_id: string;+ event_bytes: string;+ }>({+ query: `+ SELECT event_type, event_id, event_bytes+ FROM billable_events+ WHERE project_id = {projectId: String}+ ORDER BY event_type, event_id+ `,+ params: { projectId },+ });++const countsInWindow = async (+ projectId: string,+ windowFrom: Date,+ windowTo: Date,+) => {+ const rows = await getBillableUnitCountsByProjectInWindow({+ windowFrom,+ windowTo,+ projectId,+ });+ return Object.fromEntries(rows.map((row) => [row.eventType, row.count]));+};++/** Billed units of rows the MVs just wrote, whose ingested_at is now. */+const billedUnitsAroundNow = (projectId: string) =>+ countsInWindow(+ projectId,+ new Date(Date.now() - HOUR),+ new Date(Date.now() + HOUR),+ );++/** Inserts ledger rows directly, to place them in chosen ingestion windows. */+const insertBillableEvents = (+ projectId: string,+ rows: Array<{ eventType: BillableEventType; eventId: string; at: Date }>,+) =>+ clickhouseClient().insert({+ table: "billable_events",+ format: "JSONEachRow",+ values: rows.map((row) => ({+ ingested_at: row.at.toISOString().slice(0, 19).replace("T", " "),+ project_id: projectId,+ event_type: row.eventType,+ event_id: row.eventId,+ event_time: row.at.toISOString().replace("T", " ").replace("Z", ""),+ event_bytes: 0,+ })),+ });++describe("billable_events materialized views", () => {+ it("records observations, one row per trace and every score type", async (ctx) => {+ await skipUnlessBillableEvents(ctx);+ const projectId = randomUUID();+ const traceId = randomUUID();+ const backfilledTraceId = randomUUID();++ await createEventsCh([+ createEvent({+ project_id: projectId,+ trace_id: traceId,+ span_id: "span-1",+ event_bytes: 10,+ }),+ createEvent({+ project_id: projectId,+ trace_id: traceId,+ span_id: "span-2",+ event_bytes: 20,+ }),+ // Dual-write trace span: only counts as the trace.+ createEvent({+ project_id: projectId,+ trace_id: traceId,+ span_id: `t-${traceId}`,+ event_bytes: 7,+ }),+ // Tombstone: not usage.+ createEvent({+ project_id: projectId,+ trace_id: traceId,+ span_id: "span-deleted",+ is_deleted: 1,+ }),+ ]);+ // A backfill keeps the source created_at, so it is not new usage.+ await createEventsCh([+ createEvent({+ project_id: projectId,+ trace_id: backfilledTraceId,+ span_id: "span-backfilled",+ created_at: THREE_DAYS_AGO,+ }),+ ]);+ await createScoresCh([+ createTraceScore({+ project_id: projectId,+ trace_id: traceId,+ id: "score-numeric",+ data_type: "NUMERIC",+ }),+ createTraceScore({+ project_id: projectId,+ trace_id: traceId,+ id: "score-correction",+ data_type: "CORRECTION",+ }),+ createTraceScore({+ project_id: projectId,+ trace_id: traceId,+ id: "score-backfilled",+ created_at: THREE_DAYS_AGO,+ }),+ ]);++ const rows = await getBillableRows(projectId);+ expect(+ rows.map((row) => ({+ type: row.event_type,+ id: row.event_id,+ bytes: Number(row.event_bytes),+ })),+ ).toEqual([+ {+ type: "observation",+ id: observationEventId(traceId, "span-1"),+ bytes: 10,+ },+ {+ type: "observation",+ id: observationEventId(traceId, "span-2"),+ bytes: 20,+ },+ { type: "score", id: "score-correction", bytes: expect.any(Number) },+ { type: "score", id: "score-numeric", bytes: expect.any(Number) },+ { type: "trace", id: traceId, bytes: 7 },+ ]);+ });++ it("bills a span id once per trace it appears in", async (ctx) => {+ await skipUnlessBillableEvents(ctx);+ const projectId = randomUUID();++ await createEventsCh([+ // Ingestion API clients choose observation ids and may reuse them.+ createEvent({+ project_id: projectId,+ trace_id: randomUUID(),+ span_id: "llm-call",+ }),+ createEvent({+ project_id: projectId,+ trace_id: randomUUID(),+ span_id: "llm-call",+ }),+ // Trace and span ids whose plain concatenation would collide.+ createEvent({ project_id: projectId, trace_id: "t:x", span_id: "s" }),+ createEvent({ project_id: projectId, trace_id: "t", span_id: "x:s" }),+ ]);++ expect(await billedUnitsAroundNow(projectId)).toEqual({+ observation: 4,+ trace: 4,+ });+ });++ it("keeps billed units when the trace is deleted", async (ctx) => {+ await skipUnlessBillableEvents(ctx);+ const projectId = randomUUID();+ const traceId = randomUUID();++ await createEventsCh([+ createEvent({ project_id: projectId, trace_id: traceId }),+ createEvent({ project_id: projectId, trace_id: traceId }),+ ]);+ await createScoresCh([+ createTraceScore({ project_id: projectId, trace_id: traceId }),+ ]);+ const billedBeforeDelete = await billedUnitsAroundNow(projectId);+ expect(billedBeforeDelete).toEqual({ observation: 2, trace: 1, score: 1 });++ await deleteEventsByTraceIds(projectId, [traceId]);+ await deleteScoresByTraceIds(projectId, [traceId]);++ const remainingEvents = await queryClickhouse<{ count: string }>({+ query: `SELECT count() AS count FROM events_full WHERE project_id = {projectId: String}`,+ params: { projectId },+ });+ expect(Number(remainingEvents[0].count)).toBe(0);+ expect(await billedUnitsAroundNow(projectId)).toEqual(billedBeforeDelete);+ });+});++describe("getBillableUnitCountsByProjectInWindow", () => {+ // A fixed window start; each test uses its own project.+ const T0 = new Date("2026-01-05T00:00:00.000Z");+ const at = (offsetMs: number) => new Date(T0.getTime() + offsetMs);++ it("deduplicates within the window and against the 24 hours before it", async (ctx) => {+ await skipUnlessBillableEvents(ctx);+ const projectId = randomUUID();+ await insertBillableEvents(projectId, [+ { eventType: "trace", eventId: "twice-in-window", at: at(0) },+ { eventType: "trace", eventId: "twice-in-window", at: at(40 * MINUTE) },+ // The lookback starts exactly 24 hours before windowFrom.+ { eventType: "trace", eventId: "resent-within-24h", at: at(-24 * HOUR) },+ { eventType: "trace", eventId: "resent-within-24h", at: at(5 * MINUTE) },+ {+ eventType: "trace",+ eventId: "resent-after-24h",+ at: at(-24 * HOUR - SECOND),+ },+ { eventType: "trace", eventId: "resent-after-24h", at: at(5 * MINUTE) },+ // windowTo is exclusive, so this belongs to the next window.+ { eventType: "trace", eventId: "at-window-end", at: at(HOUR) },+ ]);++ expect(await countsInWindow(projectId, T0, at(HOUR))).toEqual({+ trace: 2,+ });+ expect(await countsInWindow(projectId, at(HOUR), at(2 * HOUR))).toEqual({+ trace: 1,+ });+ });++ it("bills a multi-day trace once while its gaps stay under 24 hours", async (ctx) => {+ await skipUnlessBillableEvents(ctx);+ const projectId = randomUUID();+ const windowStarts = [0, 20 * HOUR, 40 * HOUR, 60 * HOUR].map(at);+ await insertBillableEvents(+ projectId,+ windowStarts.map((windowFrom) => ({+ eventType: "trace",+ eventId: "long-running-trace",+ at: windowFrom,+ })),+ );++ const counts = await Promise.all(+ windowStarts.map((windowFrom) =>+ countsInWindow(+ projectId,+ windowFrom,+ new Date(windowFrom.getTime() + 15 * MINUTE),+ ),+ ),+ );+ expect(counts).toEqual([{ trace: 1 }, {}, {}, {}]);+ });++ it("counts per project and event type", async (ctx) => {+ await skipUnlessBillableEvents(ctx);+ const projectA = randomUUID();+ const projectB = randomUUID();+ await insertBillableEvents(projectA, [+ { eventType: "observation", eventId: "id-1", at: at(MINUTE) },+ { eventType: "observation", eventId: "id-1", at: at(2 * MINUTE) },+ { eventType: "observation", eventId: "id-2", at: at(14 * MINUTE) },+ { eventType: "trace", eventId: "id-1", at: at(MINUTE) },+ { eventType: "score", eventId: "id-3", at: at(MINUTE) },+ ]);+ // The same id in another project is a separate unit.+ await insertBillableEvents(projectB, [+ { eventType: "observation", eventId: "id-1", at: at(MINUTE) },+ ]);++ expect(await countsInWindow(projectA, T0, at(15 * MINUTE))).toEqual({+ observation: 2,+ trace: 1,+ score: 1,+ });++ const allProjects = await getBillableUnitCountsByProjectInWindow({+ windowFrom: T0,+ windowTo: at(15 * MINUTE),+ });+ expect(+ allProjects.filter((row) => row.projectId === projectA),+ ).toHaveLength(3);+ expect(allProjects.filter((row) => row.projectId === projectB)).toEqual([+ { projectId: projectB, eventType: "observation", count: 1 },+ ]);+ });++ it("rejects a window that does not end after it starts", async () => {+ await expect(+ getBillableUnitCountsByProjectInWindow({ windowFrom: T0, windowTo: T0 }),+ ).rejects.toThrow("must end after it starts");+ });+});