Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(telemetry): count self-hosted traces and observations from events table in events_only mode (#19430)

Steffen Reuther6c615cf9a0c8
diff --git a/packages/shared/src/server/clickhouse/compatibility.ts b/packages/shared/src/server/clickhouse/compatibility.ts
index 1131da7125..5581474329 100644
--- a/packages/shared/src/server/clickhouse/compatibility.ts
+++ b/packages/shared/src/server/clickhouse/compatibility.ts
@@ -251,6 +251,13 @@ export const getClickHouseCompatibilitySettings = (): ClickHouseSettings =>
resolveClickHouseCompatibility({ version: detectedClickHouseVersion })
.settings;
+/**
+ * ClickHouse server version detected at startup, or null when detection
+ * failed or has not run.
+ */
+export const getDetectedClickHouseVersion = (): string | null =>
+ detectedClickHouseVersion;
+
export const initializeClickhouseCompatibility = async (): Promise<void> => {
if (initializationPromise) return initializationPromise;
diff --git a/packages/shared/src/server/index.ts b/packages/shared/src/server/index.ts
index 75d482ef2e..457c679e18 100644
--- a/packages/shared/src/server/index.ts
+++ b/packages/shared/src/server/index.ts
@@ -83,6 +83,7 @@ export * from "./otel/utils";
export * from "./clickhouse/client";
export {
getClickHouseCompatibilitySettings,
+ getDetectedClickHouseVersion,
initializeClickhouseCompatibility,
} from "./clickhouse/compatibility";
export * from "./clickhouse/schemaUtils";
diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.ts
index 7f0a97dcc6..471b695d84 100644
--- a/packages/shared/src/server/repositories/events.ts
+++ b/packages/shared/src/server/repositories/events.ts
@@ -1499,6 +1499,61 @@ export const getLastTraceTimestampsByProjects = async (
return getLastTraceTimestampsByProjectsFromEventsTable(params);
};
+/**
+ * Trace and observation counts per project for events_core rows created in
+ * `[start, end)`. Ingestion-API traces carry a synthetic `t-<traceId>` root
+ * row, which counts towards the trace but not as an observation. A trace with
+ * rows created in several intervals counts once in each.
+ */
+export const getTracingCountsByProjectInCreationIntervalFromEventsTable =
+ async ({
+ start,
+ end,
+ projectIds,
+ }: {
+ start: Date;
+ end: Date;
+ projectIds?: string[];
+ }) => {
+ // Spans several projects, so it cannot go through compileClickhouseQuery,
+ // whose tenancy scope is always a single project.
+ const query = `
+ SELECT
+ project_id,
+ uniq(trace_id) as traces,
+ countIf(span_id != concat('t-', trace_id)) as observations
+ FROM events_core
+ WHERE created_at >= {start: DateTime64(3)}
+ AND created_at < {end: DateTime64(3)}
+ AND is_deleted = 0
+ ${projectIds ? "AND project_id IN ({projectIds: Array(String)})" : ""}
+ GROUP BY project_id
+ `;
+
+ const rows = await queryClickhouse<{
+ project_id: string;
+ traces: string;
+ observations: string;
+ }>({
+ query,
+ params: {
+ start: convertDateToClickhouseDateTime(start),
+ end: convertDateToClickhouseDateTime(end),
+ ...(projectIds ? { projectIds } : {}),
+ },
+ clickhouseConfigs: {
+ request_timeout: 300000, // 5 minutes timeout
+ },
+ preferredClickhouseService: "EventsReadOnly",
+ });
+
+ return rows.map((row) => ({
+ projectId: row.project_id,
+ traces: Number(row.traces),
+ observations: Number(row.observations),
+ }));
+ };
+
type PublicApiObservationsQuery = {
projectId: string;
page: number;
diff --git a/web/src/__tests__/server/repositories/event-repository.servertest.ts b/web/src/__tests__/server/repositories/event-repository.servertest.ts
index be62da2c11..03a85c3034 100644
--- a/web/src/__tests__/server/repositories/event-repository.servertest.ts
+++ b/web/src/__tests__/server/repositories/event-repository.servertest.ts
@@ -28,6 +28,7 @@ import {
createTraceScore,
getUsersFromEventsTable,
getUsersCountFromEventsTable,
+ getTracingCountsByProjectInCreationIntervalFromEventsTable,
type EventFilterOptionColumn,
} from "@langfuse/shared/src/server";
import {
@@ -5999,6 +6000,72 @@ describe("Clickhouse Events Repository Test", () => {
});
});
+describe("tracing counts by project on the events table", () => {
+ it("counts traces and observations created in the interval, excluding synthetic trace rows", async () => {
+ const projectA = randomUUID();
+ const projectB = randomUUID();
+ const legacyTraceId = randomUUID();
+ const otelTraceId = randomUUID();
+ const start = new Date("2026-01-10T00:00:00.000Z");
+ const end = new Date("2026-01-10T12:00:00.000Z");
+ const inWindow = new Date("2026-01-10T06:00:00.000Z");
+
+ const event = (
+ overrides: Partial<Parameters<typeof createEvent>[0]> & {
+ project_id: string;
+ trace_id: string;
+ },
+ ) => {
+ const spanId = overrides.span_id ?? randomUUID();
+ return createEvent({
+ created_at: inWindow,
+ ...overrides,
+ span_id: spanId,
+ id: spanId,
+ });
+ };
+
+ await createEventsCh([
+ // Ingestion-API trace: synthetic `t-` root row plus two observations.
+ event({
+ project_id: projectA,
+ trace_id: legacyTraceId,
+ span_id: `t-${legacyTraceId}`,
+ type: "SPAN",
+ }),
+ event({ project_id: projectA, trace_id: legacyTraceId }),
+ event({ project_id: projectA, trace_id: legacyTraceId }),
+ // OTel-native trace: real root span plus one child, no synthetic row.
+ event({ project_id: projectA, trace_id: otelTraceId }),
+ event({ project_id: projectA, trace_id: otelTraceId }),
+ // Created on the interval bounds: start is inclusive, end is exclusive.
+ event({
+ project_id: projectA,
+ trace_id: randomUUID(),
+ created_at: start,
+ }),
+ event({ project_id: projectA, trace_id: randomUUID(), created_at: end }),
+ event({ project_id: projectB, trace_id: randomUUID() }),
+ ]);
+
+ const counts =
+ await getTracingCountsByProjectInCreationIntervalFromEventsTable({
+ start,
+ end,
+ projectIds: [projectA, projectB],
+ });
+
+ expect(
+ counts.sort((a, b) => a.projectId.localeCompare(b.projectId)),
+ ).toEqual(
+ [
+ { projectId: projectA, traces: 3, observations: 5 },
+ { projectId: projectB, traces: 1, observations: 1 },
+ ].sort((a, b) => a.projectId.localeCompare(b.projectId)),
+ );
+ });
+});
+
describe("users search on the events table", () => {
it("matches LIKE wildcard characters in the search query literally", async () => {
const projectId = randomUUID();
diff --git a/web/src/features/telemetry/index.ts b/web/src/features/telemetry/index.ts
index 9ed0a98f47..a6da7f2a58 100644
--- a/web/src/features/telemetry/index.ts
+++ b/web/src/features/telemetry/index.ts
@@ -5,9 +5,11 @@ import { Prisma, prisma } from "@langfuse/shared/src/db";
import { v4 as uuidv4 } from "uuid";
import {
getDatasetRunItemCountsByProjectInCreationInterval,
+ getDetectedClickHouseVersion,
getObservationCountsByProjectInCreationInterval,
getScoreCountsByProjectInCreationInterval,
getTraceCountsByProjectInCreationInterval,
+ getTracingCountsByProjectInCreationIntervalFromEventsTable,
logger,
} from "@langfuse/shared/src/server";
import { env } from "@/src/env.mjs";
@@ -159,6 +161,37 @@ async function jobScheduler(): Promise<
return { shouldRunJob: true, jobStartedAt, lastRun, clientId };
}
+/**
+ * Counts traces and observations created in `[start, end)` from the tables the
+ * worker writes: under `events_only` the legacy traces/observations tables are
+ * empty, so the events table is the only source.
+ */
+async function countTracing({ start, end }: { start: Date; end: Date }) {
+ if (env.LANGFUSE_MIGRATION_V4_WRITE_MODE === "events_only") {
+ const counts =
+ await getTracingCountsByProjectInCreationIntervalFromEventsTable({
+ start,
+ end,
+ });
+ return {
+ countTraces: counts.reduce((acc, curr) => acc + curr.traces, 0),
+ countObservations: counts.reduce(
+ (acc, curr) => acc + curr.observations,
+ 0,
+ ),
+ };
+ }
+
+ const [traces, observations] = await Promise.all([
+ getTraceCountsByProjectInCreationInterval({ start, end }),
+ getObservationCountsByProjectInCreationInterval({ start, end }),
+ ]);
+ return {
+ countTraces: traces.reduce((acc, curr) => acc + curr.count, 0),
+ countObservations: observations.reduce((acc, curr) => acc + curr.count, 0),
+ };
+}
+
async function posthogTelemetry({
startTimeframe,
endTimeframe,
@@ -177,16 +210,11 @@ async function posthogTelemetry({
},
});
- // Count traces
- const countTracesClickhouse =
- await getTraceCountsByProjectInCreationInterval({
- start: startTimeframe ?? new Date(0),
- end: endTimeframe,
- });
- const countTraces = countTracesClickhouse.reduce(
- (acc, curr) => acc + curr.count,
- 0,
- );
+ // Count traces and observations
+ const { countTraces, countObservations } = await countTracing({
+ start: startTimeframe ?? new Date(0),
+ end: endTimeframe,
+ });
// Count scores
const countScoresClickhouse =
@@ -199,17 +227,6 @@ async function posthogTelemetry({
0,
);
- // Count observations
- const countObservationsClickhouse =
- await getObservationCountsByProjectInCreationInterval({
- start: startTimeframe ?? new Date(0),
- end: endTimeframe,
- });
- const countObservations = countObservationsClickhouse.reduce(
- (acc, curr) => acc + curr.count,
- 0,
- );
-
// Count datasets
const countDatasets = await prisma.dataset.count({
where: {
@@ -292,11 +309,15 @@ async function posthogTelemetry({
endTimeframe: endTimeframe.toISOString(),
eeLicenseKey: env.LANGFUSE_EE_LICENSE_KEY,
langfuseCloudRegion: env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,
+ writeMode: env.LANGFUSE_MIGRATION_V4_WRITE_MODE,
+ clickhouseVersion: getDetectedClickHouseVersion(),
$set: {
environment: process.env.NODE_ENV,
userDomains: domains,
docker: true,
langfuseVersion: VERSION,
+ writeMode: env.LANGFUSE_MIGRATION_V4_WRITE_MODE,
+ clickhouseVersion: getDetectedClickHouseVersion(),
},
},
});