🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(telemetry): count self-hosted traces and observations from events table in events_only mode (#19430)
Steffen Reuther6c615cf9a0c8
diff --git a/packages/shared/src/server/clickhouse/compatibility.ts b/packages/shared/src/server/clickhouse/compatibility.tsindex 1131da7125..5581474329 100644--- a/packages/shared/src/server/clickhouse/compatibility.ts+++ b/packages/shared/src/server/clickhouse/compatibility.ts@@ -251,6 +251,13 @@ export const getClickHouseCompatibilitySettings = (): ClickHouseSettings =>resolveClickHouseCompatibility({ version: detectedClickHouseVersion }).settings;+/**+ * ClickHouse server version detected at startup, or null when detection+ * failed or has not run.+ */+export const getDetectedClickHouseVersion = (): string | null =>+ detectedClickHouseVersion;+export const initializeClickhouseCompatibility = async (): Promise<void> => {if (initializationPromise) return initializationPromise;diff --git a/packages/shared/src/server/index.ts b/packages/shared/src/server/index.tsindex 75d482ef2e..457c679e18 100644--- a/packages/shared/src/server/index.ts+++ b/packages/shared/src/server/index.ts@@ -83,6 +83,7 @@ export * from "./otel/utils";export * from "./clickhouse/client";export {getClickHouseCompatibilitySettings,+ getDetectedClickHouseVersion,initializeClickhouseCompatibility,} from "./clickhouse/compatibility";export * from "./clickhouse/schemaUtils";diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.tsindex 7f0a97dcc6..471b695d84 100644--- a/packages/shared/src/server/repositories/events.ts+++ b/packages/shared/src/server/repositories/events.ts@@ -1499,6 +1499,61 @@ export const getLastTraceTimestampsByProjects = async (return getLastTraceTimestampsByProjectsFromEventsTable(params);};+/**+ * Trace and observation counts per project for events_core rows created in+ * `[start, end)`. Ingestion-API traces carry a synthetic `t-<traceId>` root+ * row, which counts towards the trace but not as an observation. A trace with+ * rows created in several intervals counts once in each.+ */+export const getTracingCountsByProjectInCreationIntervalFromEventsTable =+ async ({+ start,+ end,+ projectIds,+ }: {+ start: Date;+ end: Date;+ projectIds?: string[];+ }) => {+ // Spans several projects, so it cannot go through compileClickhouseQuery,+ // whose tenancy scope is always a single project.+ const query = `+ SELECT+ project_id,+ uniq(trace_id) as traces,+ countIf(span_id != concat('t-', trace_id)) as observations+ FROM events_core+ WHERE created_at >= {start: DateTime64(3)}+ AND created_at < {end: DateTime64(3)}+ AND is_deleted = 0+ ${projectIds ? "AND project_id IN ({projectIds: Array(String)})" : ""}+ GROUP BY project_id+ `;++ const rows = await queryClickhouse<{+ project_id: string;+ traces: string;+ observations: string;+ }>({+ query,+ params: {+ start: convertDateToClickhouseDateTime(start),+ end: convertDateToClickhouseDateTime(end),+ ...(projectIds ? { projectIds } : {}),+ },+ clickhouseConfigs: {+ request_timeout: 300000, // 5 minutes timeout+ },+ preferredClickhouseService: "EventsReadOnly",+ });++ return rows.map((row) => ({+ projectId: row.project_id,+ traces: Number(row.traces),+ observations: Number(row.observations),+ }));+ };+type PublicApiObservationsQuery = {projectId: string;page: number;diff --git a/web/src/__tests__/server/repositories/event-repository.servertest.ts b/web/src/__tests__/server/repositories/event-repository.servertest.tsindex be62da2c11..03a85c3034 100644--- a/web/src/__tests__/server/repositories/event-repository.servertest.ts+++ b/web/src/__tests__/server/repositories/event-repository.servertest.ts@@ -28,6 +28,7 @@ import {createTraceScore,getUsersFromEventsTable,getUsersCountFromEventsTable,+ getTracingCountsByProjectInCreationIntervalFromEventsTable,type EventFilterOptionColumn,} from "@langfuse/shared/src/server";import {@@ -5999,6 +6000,72 @@ describe("Clickhouse Events Repository Test", () => {});});+describe("tracing counts by project on the events table", () => {+ it("counts traces and observations created in the interval, excluding synthetic trace rows", async () => {+ const projectA = randomUUID();+ const projectB = randomUUID();+ const legacyTraceId = randomUUID();+ const otelTraceId = randomUUID();+ const start = new Date("2026-01-10T00:00:00.000Z");+ const end = new Date("2026-01-10T12:00:00.000Z");+ const inWindow = new Date("2026-01-10T06:00:00.000Z");++ const event = (+ overrides: Partial<Parameters<typeof createEvent>[0]> & {+ project_id: string;+ trace_id: string;+ },+ ) => {+ const spanId = overrides.span_id ?? randomUUID();+ return createEvent({+ created_at: inWindow,+ ...overrides,+ span_id: spanId,+ id: spanId,+ });+ };++ await createEventsCh([+ // Ingestion-API trace: synthetic `t-` root row plus two observations.+ event({+ project_id: projectA,+ trace_id: legacyTraceId,+ span_id: `t-${legacyTraceId}`,+ type: "SPAN",+ }),+ event({ project_id: projectA, trace_id: legacyTraceId }),+ event({ project_id: projectA, trace_id: legacyTraceId }),+ // OTel-native trace: real root span plus one child, no synthetic row.+ event({ project_id: projectA, trace_id: otelTraceId }),+ event({ project_id: projectA, trace_id: otelTraceId }),+ // Created on the interval bounds: start is inclusive, end is exclusive.+ event({+ project_id: projectA,+ trace_id: randomUUID(),+ created_at: start,+ }),+ event({ project_id: projectA, trace_id: randomUUID(), created_at: end }),+ event({ project_id: projectB, trace_id: randomUUID() }),+ ]);++ const counts =+ await getTracingCountsByProjectInCreationIntervalFromEventsTable({+ start,+ end,+ projectIds: [projectA, projectB],+ });++ expect(+ counts.sort((a, b) => a.projectId.localeCompare(b.projectId)),+ ).toEqual(+ [+ { projectId: projectA, traces: 3, observations: 5 },+ { projectId: projectB, traces: 1, observations: 1 },+ ].sort((a, b) => a.projectId.localeCompare(b.projectId)),+ );+ });+});+describe("users search on the events table", () => {it("matches LIKE wildcard characters in the search query literally", async () => {const projectId = randomUUID();diff --git a/web/src/features/telemetry/index.ts b/web/src/features/telemetry/index.tsindex 9ed0a98f47..a6da7f2a58 100644--- a/web/src/features/telemetry/index.ts+++ b/web/src/features/telemetry/index.ts@@ -5,9 +5,11 @@ import { Prisma, prisma } from "@langfuse/shared/src/db";import { v4 as uuidv4 } from "uuid";import {getDatasetRunItemCountsByProjectInCreationInterval,+ getDetectedClickHouseVersion,getObservationCountsByProjectInCreationInterval,getScoreCountsByProjectInCreationInterval,getTraceCountsByProjectInCreationInterval,+ getTracingCountsByProjectInCreationIntervalFromEventsTable,logger,} from "@langfuse/shared/src/server";import { env } from "@/src/env.mjs";@@ -159,6 +161,37 @@ async function jobScheduler(): Promise<return { shouldRunJob: true, jobStartedAt, lastRun, clientId };}+/**+ * Counts traces and observations created in `[start, end)` from the tables the+ * worker writes: under `events_only` the legacy traces/observations tables are+ * empty, so the events table is the only source.+ */+async function countTracing({ start, end }: { start: Date; end: Date }) {+ if (env.LANGFUSE_MIGRATION_V4_WRITE_MODE === "events_only") {+ const counts =+ await getTracingCountsByProjectInCreationIntervalFromEventsTable({+ start,+ end,+ });+ return {+ countTraces: counts.reduce((acc, curr) => acc + curr.traces, 0),+ countObservations: counts.reduce(+ (acc, curr) => acc + curr.observations,+ 0,+ ),+ };+ }++ const [traces, observations] = await Promise.all([+ getTraceCountsByProjectInCreationInterval({ start, end }),+ getObservationCountsByProjectInCreationInterval({ start, end }),+ ]);+ return {+ countTraces: traces.reduce((acc, curr) => acc + curr.count, 0),+ countObservations: observations.reduce((acc, curr) => acc + curr.count, 0),+ };+}+async function posthogTelemetry({startTimeframe,endTimeframe,@@ -177,16 +210,11 @@ async function posthogTelemetry({},});- // Count traces- const countTracesClickhouse =- await getTraceCountsByProjectInCreationInterval({- start: startTimeframe ?? new Date(0),- end: endTimeframe,- });- const countTraces = countTracesClickhouse.reduce(- (acc, curr) => acc + curr.count,- 0,- );+ // Count traces and observations+ const { countTraces, countObservations } = await countTracing({+ start: startTimeframe ?? new Date(0),+ end: endTimeframe,+ });// Count scoresconst countScoresClickhouse =@@ -199,17 +227,6 @@ async function posthogTelemetry({0,);- // Count observations- const countObservationsClickhouse =- await getObservationCountsByProjectInCreationInterval({- start: startTimeframe ?? new Date(0),- end: endTimeframe,- });- const countObservations = countObservationsClickhouse.reduce(- (acc, curr) => acc + curr.count,- 0,- );-// Count datasetsconst countDatasets = await prisma.dataset.count({where: {@@ -292,11 +309,15 @@ async function posthogTelemetry({endTimeframe: endTimeframe.toISOString(),eeLicenseKey: env.LANGFUSE_EE_LICENSE_KEY,langfuseCloudRegion: env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,+ writeMode: env.LANGFUSE_MIGRATION_V4_WRITE_MODE,+ clickhouseVersion: getDetectedClickHouseVersion(),$set: {environment: process.env.NODE_ENV,userDomains: domains,docker: true,langfuseVersion: VERSION,+ writeMode: env.LANGFUSE_MIGRATION_V4_WRITE_MODE,+ clickhouseVersion: getDetectedClickHouseVersion(),},},});