🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(billing): metrics api integration (#15460)
Steffen Schmitzef6750ca818f
diff --git a/.env.dev.example b/.env.dev.exampleindex 4e82fa7226..cf1eadefac 100644--- a/.env.dev.example+++ b/.env.dev.example@@ -247,3 +247,5 @@ CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION=auto# ClickHouse Billing (CHB) integration (Langfuse Cloud only). First-time# upgrades on/after this UTC date (YYYY-MM-DD) route to CHB; unset = off.# LANGFUSE_CLOUD_BILLING_CHB_CUTOFF_DATE=+# Bearer secret for GET /api/billing/metrics; unset = endpoint disabled.+# CLICKHOUSE_BILLING_METRICS_API_KEY=diff --git a/.env.prod.example b/.env.prod.exampleindex e5c033b4dc..846a7fd5c0 100644--- a/.env.prod.example+++ b/.env.prod.example@@ -384,3 +384,5 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true# ClickHouse Billing (CHB) integration (Langfuse Cloud only). First-time# upgrades on/after this UTC date (YYYY-MM-DD) route to CHB; unset = off.# LANGFUSE_CLOUD_BILLING_CHB_CUTOFF_DATE=+# Bearer secret for GET /api/billing/metrics; unset = endpoint disabled.+# CLICKHOUSE_BILLING_METRICS_API_KEY=diff --git a/packages/shared/src/server/repositories/observations.ts b/packages/shared/src/server/repositories/observations.tsindex b38d5ac761..9d945ec378 100644--- a/packages/shared/src/server/repositories/observations.ts+++ b/packages/shared/src/server/repositories/observations.ts@@ -1608,9 +1608,11 @@ export const getObservationsGroupedByTraceId = async (export const getObservationCountsByProjectInCreationInterval = async ({start,end,+ projectId,}: {start: Date;end: Date;+ projectId?: string;}) => {const query = `SELECT@@ -1619,6 +1621,7 @@ export const getObservationCountsByProjectInCreationInterval = async ({FROM observationsWHERE created_at >= {start: DateTime64(3)}AND created_at < {end: DateTime64(3)}+ ${projectId ? "AND project_id = {projectId: String}" : ""}GROUP BY project_id`;@@ -1627,6 +1630,7 @@ export const getObservationCountsByProjectInCreationInterval = async ({params: {start: convertDateToClickhouseDateTime(start),end: convertDateToClickhouseDateTime(end),+ ...(projectId ? { projectId } : {}),},clickhouseConfigs: {request_timeout: 300000, // 5 minutes timeoutdiff --git a/packages/shared/src/server/repositories/scores.ts b/packages/shared/src/server/repositories/scores.tsindex 7e7f24d5bf..31345e0cf6 100644--- a/packages/shared/src/server/repositories/scores.ts+++ b/packages/shared/src/server/repositories/scores.ts@@ -2143,9 +2143,11 @@ export const getAggregatedScoresForPromptsFromEvents = async (export const getScoreCountsByProjectInCreationInterval = async ({start,end,+ projectId,}: {start: Date;end: Date;+ projectId?: string;}) => {const query = `SELECT@@ -2154,6 +2156,7 @@ export const getScoreCountsByProjectInCreationInterval = async ({FROM scoresWHERE created_at >= {start: DateTime64(3)}AND created_at < {end: DateTime64(3)}+ ${projectId ? "AND project_id = {projectId: String}" : ""}AND data_type IN ({dataTypes: Array(String)})GROUP BY project_id`;@@ -2164,6 +2167,7 @@ export const getScoreCountsByProjectInCreationInterval = async ({start: convertDateToClickhouseDateTime(start),end: convertDateToClickhouseDateTime(end),dataTypes: LISTABLE_SCORE_TYPES,+ ...(projectId ? { projectId } : {}),},clickhouseConfigs: {request_timeout: 300000, // 5 minutes timeoutdiff --git a/packages/shared/src/server/repositories/traces.ts b/packages/shared/src/server/repositories/traces.tsindex 316ff49cf0..65c5c3bf19 100644--- a/packages/shared/src/server/repositories/traces.ts+++ b/packages/shared/src/server/repositories/traces.ts@@ -365,9 +365,11 @@ export const hasAnyTrace = async (projectId: string) => {export const getTraceCountsByProjectInCreationInterval = async ({start,end,+ projectId,}: {start: Date;end: Date;+ projectId?: string;}) => {const query = `SELECT@@ -376,6 +378,7 @@ export const getTraceCountsByProjectInCreationInterval = async ({FROM tracesWHERE created_at >= {start: DateTime64(3)}AND created_at < {end: DateTime64(3)}+ ${projectId ? "AND project_id = {projectId: String}" : ""}GROUP BY project_id`;@@ -384,6 +387,7 @@ export const getTraceCountsByProjectInCreationInterval = async ({params: {start: convertDateToClickhouseDateTime(start),end: convertDateToClickhouseDateTime(end),+ ...(projectId ? { projectId } : {}),},clickhouseConfigs: {request_timeout: 300000, // 5 minutes timeoutdiff --git a/web/src/__tests__/server/unit/chb-metrics-api-handler.servertest.ts b/web/src/__tests__/server/unit/chb-metrics-api-handler.servertest.tsnew file mode 100644index 0000000000..11ebde7908--- /dev/null+++ b/web/src/__tests__/server/unit/chb-metrics-api-handler.servertest.ts@@ -0,0 +1,215 @@+import { NextRequest } from "next/server";+import type * as SharedServer from "@langfuse/shared/src/server";+import { beforeEach, describe, expect, it, vi } from "vitest";++const mocks = vi.hoisted(() => ({+ env: {+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "eu" as string | undefined,+ CLICKHOUSE_BILLING_METRICS_API_KEY: "chb-secret" as string | undefined,+ },+ findProject: vi.fn(),+ traceCounts: vi.fn(),+ observationCounts: vi.fn(),+ scoreCounts: vi.fn(),+ recordIncrement: vi.fn(),+}));++vi.mock("@/src/env.mjs", () => ({ env: mocks.env }));++vi.mock("@langfuse/shared/src/db", () => ({+ prisma: { project: { findUnique: mocks.findProject } },+}));++// Partial mock: the shared teardown hook imports redis/logger/ClickHouse from+// this module, so the real exports have to stay reachable.+vi.mock("@langfuse/shared/src/server", async (importOriginal) => {+ const actual = await importOriginal<typeof SharedServer>();++ return {+ ...actual,+ getTraceCountsByProjectInCreationInterval: mocks.traceCounts,+ getObservationCountsByProjectInCreationInterval: mocks.observationCounts,+ getScoreCountsByProjectInCreationInterval: mocks.scoreCounts,+ recordIncrement: mocks.recordIncrement,+ };+});++import { chbMetricsApiHandler } from "@/src/ee/features/billing/server/chb/chbMetricsApiHandler";+import { logger } from "@langfuse/shared/src/server";++const loggerError = vi+ .spyOn(logger, "error")+ .mockImplementation((() => {}) as never);++const PROJECT_ID = "project-1";++function createRequest({+ startTime = "2026-07-01T00:00:00Z",+ endTime = "2026-07-02T00:00:00Z",+ resourceId = PROJECT_ID,+ token = "chb-secret",+}: {+ startTime?: string;+ endTime?: string;+ resourceId?: string;+ token?: string | null;+} = {}) {+ const url = new URL("http://localhost/api/billing/metrics");+ if (startTime) url.searchParams.set("startTime", startTime);+ if (endTime) url.searchParams.set("endTime", endTime);+ if (resourceId) url.searchParams.set("resourceId", resourceId);++ return new NextRequest(url, {+ method: "GET",+ headers: token ? { authorization: `Bearer ${token}` } : {},+ });+}++describe("chbMetricsApiHandler", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ mocks.env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "eu";+ mocks.env.CLICKHOUSE_BILLING_METRICS_API_KEY = "chb-secret";+ mocks.findProject.mockResolvedValue({ id: PROJECT_ID });+ mocks.traceCounts.mockResolvedValue([{ projectId: PROJECT_ID, count: 7 }]);+ mocks.observationCounts.mockResolvedValue([+ { projectId: PROJECT_ID, count: 11 },+ ]);+ mocks.scoreCounts.mockResolvedValue([{ projectId: PROJECT_ID, count: 3 }]);+ });++ it("returns per-resource usage counts for an authorized poll", async () => {+ const response = await chbMetricsApiHandler(createRequest());++ expect(response.status).toBe(200);+ expect(await response.json()).toEqual({+ metrics: {+ traces: { sum: 7 },+ observations: { sum: 11 },+ scores: { sum: 3 },+ },+ });+ // The window is passed through as instants and scoped to the resource, so a+ // poll can never bill one org for another org's project.+ expect(mocks.traceCounts).toHaveBeenCalledWith({+ start: new Date("2026-07-01T00:00:00Z"),+ end: new Date("2026-07-02T00:00:00Z"),+ projectId: PROJECT_ID,+ });+ });++ it("returns zeros and a telemetry signal for an unknown resource", async () => {+ mocks.findProject.mockResolvedValue(null);+ mocks.traceCounts.mockResolvedValue([]);+ mocks.observationCounts.mockResolvedValue([]);+ mocks.scoreCounts.mockResolvedValue([]);++ const response = await chbMetricsApiHandler(+ createRequest({ resourceId: "never-existed" }),+ );++ expect(response.status).toBe(200);+ expect(await response.json()).toEqual({+ metrics: {+ traces: { sum: 0 },+ observations: { sum: 0 },+ scores: { sum: 0 },+ },+ });+ expect(mocks.recordIncrement).toHaveBeenCalledWith(+ "langfuse.billing_metrics.unknown_resource",+ 1,+ { unit: "requests" },+ );+ });++ it("keeps the structured JSON contract when a count query fails", async () => {+ mocks.observationCounts.mockRejectedValue(new Error("Timeout error."));++ const response = await chbMetricsApiHandler(createRequest());++ // A transient ClickHouse failure must not surface as a zero-valued 200 --+ // CHB would meter the org for nothing -- nor as Next.js's default HTML 500.+ expect(response.status).toBe(500);+ expect(await response.json()).toEqual({+ message: "Failed to resolve usage metrics",+ });+ expect(loggerError).toHaveBeenCalled();+ });++ it.each([+ ["no authorization header", null],+ ["a wrong secret of equal length", "chb-secreT"],+ ["a wrong secret of different length", "nope"],+ ])("rejects %s with 401", async (_label, token) => {+ const response = await chbMetricsApiHandler(createRequest({ token }));++ expect(response.status).toBe(401);+ expect(await response.json()).toEqual({ message: "Unauthorized" });+ expect(mocks.traceCounts).not.toHaveBeenCalled();+ });++ it("refuses to run when the API key is not configured", async () => {+ mocks.env.CLICKHOUSE_BILLING_METRICS_API_KEY = undefined;++ const response = await chbMetricsApiHandler(createRequest());++ expect(response.status).toBe(500);+ expect(await response.json()).toEqual({+ message: "Billing metrics endpoint is not configured",+ });+ });++ it("refuses to run outside Langfuse Cloud", async () => {+ mocks.env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;++ const response = await chbMetricsApiHandler(createRequest());++ expect(response.status).toBe(500);+ expect(await response.json()).toEqual({+ message: "Billing metrics endpoint only available in Langfuse Cloud",+ });+ });++ it.each([+ ["a non-ISO startTime", { startTime: "not-a-date" }],+ ["a missing resourceId", { resourceId: "" }],+ ])("rejects %s with 400", async (_label, overrides) => {+ const response = await chbMetricsApiHandler(createRequest(overrides));++ expect(response.status).toBe(400);+ expect((await response.json()).message).toBe("Invalid query parameters");+ expect(mocks.traceCounts).not.toHaveBeenCalled();+ });++ it("rejects an inverted window with 400", async () => {+ const response = await chbMetricsApiHandler(+ createRequest({+ startTime: "2026-07-02T00:00:00Z",+ endTime: "2026-07-01T00:00:00Z",+ }),+ );++ expect(response.status).toBe(400);+ expect(await response.json()).toEqual({+ message: "endTime must be after startTime",+ });+ });++ it("rejects a window wider than 35 days with 400", async () => {+ // Guards against full-history created_at scans, which have hit the+ // ClickHouse request ceiling in production.+ const response = await chbMetricsApiHandler(+ createRequest({+ startTime: "2026-01-01T00:00:00Z",+ endTime: "2026-07-01T00:00:00Z",+ }),+ );++ expect(response.status).toBe(400);+ expect(await response.json()).toEqual({+ message: "Requested window exceeds the 35 day maximum",+ });+ expect(mocks.traceCounts).not.toHaveBeenCalled();+ });+});diff --git a/web/src/app/api/billing/metrics/route.ts b/web/src/app/api/billing/metrics/route.tsnew file mode 100644index 0000000000..cdf547902c--- /dev/null+++ b/web/src/app/api/billing/metrics/route.ts@@ -0,0 +1,5 @@+import { chbMetricsApiHandler } from "@/src/ee/features/billing/server/chb/chbMetricsApiHandler";++export const dynamic = "force-dynamic";++export const GET = chbMetricsApiHandler;diff --git a/web/src/ee/features/billing/server/chb/chbMetricsApiHandler.ts b/web/src/ee/features/billing/server/chb/chbMetricsApiHandler.tsnew file mode 100644index 0000000000..740f70bbd3--- /dev/null+++ b/web/src/ee/features/billing/server/chb/chbMetricsApiHandler.ts@@ -0,0 +1,170 @@+import crypto from "crypto";++import { type NextRequest, NextResponse } from "next/server";+import { z } from "zod";++import { env } from "@/src/env.mjs";+import { prisma } from "@langfuse/shared/src/db";+import {+ getObservationCountsByProjectInCreationInterval,+ getScoreCountsByProjectInCreationInterval,+ getTraceCountsByProjectInCreationInterval,+ logger,+ recordIncrement,+} from "@langfuse/shared/src/server";++/**+ * Billing metrics API for ClickHouse Billing (CHB).+ *+ * CHB's metering pipeline polls this endpoint per project (`resourceId`) and+ * short trailing time window to derive billable usage. Response contract:+ * unknown resources return zeros, never 404.+ *+ * Auth: dedicated bearer secret (`CLICKHOUSE_BILLING_METRICS_API_KEY`).+ *+ * Every exit path returns structured `{ message }` JSON, including unexpected+ * failures.+ */++// Reject windows above 35 days: keeps a misbehaving caller from issuing+// full-history created_at scans; CHB polls short trailing windows.+const MAX_WINDOW_MS = 35 * 24 * 60 * 60 * 1000;++const MetricsQuerySchema = z.object({+ startTime: z.iso.datetime({ offset: true }),+ endTime: z.iso.datetime({ offset: true }),+ resourceId: z.string().min(1),+});++const sumForProject = (+ rows: { projectId: string; count: number }[],+ projectId: string,+): number =>+ rows+ .filter((row) => row.projectId === projectId)+ .reduce((sum, row) => sum + row.count, 0);++const isAuthorized = (req: NextRequest, apiKey: string): boolean => {+ const authHeader = req.headers.get("authorization");+ if (!authHeader) return false;+ const [scheme, token] = authHeader.split(" ");+ if (scheme !== "Bearer" || !token) return false;+ try {+ // timingSafeEqual throws on different input lengths, handle accordingly+ return crypto.timingSafeEqual(Buffer.from(token), Buffer.from(apiKey));+ } catch {+ return false;+ }+};++export async function chbMetricsApiHandler(req: NextRequest) {+ if (!env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION) {+ logger.error("[CHB Metrics API] Endpoint only available in Langfuse Cloud");+ return NextResponse.json(+ { message: "Billing metrics endpoint only available in Langfuse Cloud" },+ { status: 500 },+ );+ }+ if (!env.CLICKHOUSE_BILLING_METRICS_API_KEY) {+ logger.error(+ "[CHB Metrics API] CLICKHOUSE_BILLING_METRICS_API_KEY is not configured",+ );+ return NextResponse.json(+ { message: "Billing metrics endpoint is not configured" },+ { status: 500 },+ );+ }++ if (!isAuthorized(req, env.CLICKHOUSE_BILLING_METRICS_API_KEY)) {+ return NextResponse.json({ message: "Unauthorized" }, { status: 401 });+ }++ const parsedQuery = MetricsQuerySchema.safeParse({+ startTime: req.nextUrl.searchParams.get("startTime") ?? undefined,+ endTime: req.nextUrl.searchParams.get("endTime") ?? undefined,+ resourceId: req.nextUrl.searchParams.get("resourceId") ?? undefined,+ });+ if (!parsedQuery.success) {+ return NextResponse.json(+ {+ message: "Invalid query parameters",+ issues: parsedQuery.error.issues,+ },+ { status: 400 },+ );+ }++ const { resourceId } = parsedQuery.data;+ const start = new Date(parsedQuery.data.startTime);+ const end = new Date(parsedQuery.data.endTime);++ if (end.getTime() <= start.getTime()) {+ return NextResponse.json(+ { message: "endTime must be after startTime" },+ { status: 400 },+ );+ }+ if (end.getTime() - start.getTime() > MAX_WINDOW_MS) {+ return NextResponse.json(+ { message: "Requested window exceeds the 35 day maximum" },+ { status: 400 },+ );+ }++ try {+ // Telemetry-only lookup: a resourceId unknown to this region still returns+ // zeros per contract, but zeros can mask a misconfigured/misrouted CHB+ // poller — surface a signal without changing the response. Soft-deleted+ // projects still have a Postgres row and are intentionally not counted as+ // unknown.+ const project = await prisma.project.findUnique({+ where: { id: resourceId },+ select: { id: true },+ });+ if (!project) {+ recordIncrement("langfuse.billing_metrics.unknown_resource", 1, {+ unit: "requests",+ });+ logger.info(+ `[CHB Metrics API] resourceId ${resourceId} is unknown to this region, returning zeros`,+ );+ }++ const [traceCounts, observationCounts, scoreCounts] = await Promise.all([+ getTraceCountsByProjectInCreationInterval({+ start,+ end,+ projectId: resourceId,+ }),+ getObservationCountsByProjectInCreationInterval({+ start,+ end,+ projectId: resourceId,+ }),+ getScoreCountsByProjectInCreationInterval({+ start,+ end,+ projectId: resourceId,+ }),+ ]);++ return NextResponse.json(+ {+ metrics: {+ traces: { sum: sumForProject(traceCounts, resourceId) },+ observations: { sum: sumForProject(observationCounts, resourceId) },+ scores: { sum: sumForProject(scoreCounts, resourceId) },+ },+ },+ { status: 200 },+ );+ } catch (err) {+ // A ClickHouse or Postgres blip must not turn into a zero-valued 200: CHB+ // would meter the org for nothing. Fail loud with a retryable 5xx.+ logger.error("[CHB Metrics API] Failed to resolve usage metrics", err);+ return NextResponse.json(+ { message: "Failed to resolve usage metrics" },+ { status: 500 },+ );+ }+}diff --git a/web/src/env.mjs b/web/src/env.mjsindex 38c6b50456..497d75fb39 100644--- a/web/src/env.mjs+++ b/web/src/env.mjs@@ -412,6 +412,9 @@ export const env = createEnv({.date().optional().transform((date) => (date ? new Date(date) : null)),+ // Bearer secret CHB's metering pipeline presents to GET /api/billing/metrics.+ // Unset disables the endpoint (it 500s) rather than leaving it open.+ CLICKHOUSE_BILLING_METRICS_API_KEY: z.string().optional(),SENTRY_AUTH_TOKEN: z.string().optional(),SENTRY_CSP_REPORT_URI: z.string().optional(),LANGFUSE_RATE_LIMITS_ENABLED: z.enum(["true", "false"]).default("true"),@@ -946,6 +949,8 @@ export const env = createEnv({STRIPE_WEBHOOK_SIGNING_SECRET: process.env.STRIPE_WEBHOOK_SIGNING_SECRET,LANGFUSE_CLOUD_BILLING_CHB_CUTOFF_DATE:process.env.LANGFUSE_CLOUD_BILLING_CHB_CUTOFF_DATE,+ CLICKHOUSE_BILLING_METRICS_API_KEY:+ process.env.CLICKHOUSE_BILLING_METRICS_API_KEY,SENTRY_AUTH_TOKEN: process.env.SENTRY_AUTH_TOKEN,SENTRY_CSP_REPORT_URI: process.env.SENTRY_CSP_REPORT_URI,LANGFUSE_RATE_LIMITS_ENABLED: process.env.LANGFUSE_RATE_LIMITS_ENABLED,