🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(api): accept epoch fromTimestamp on public metrics v2 (#16169)
Max Deichmannbfe1b57f707d
diff --git a/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts b/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.tsindex b29cc4160a..15cbe042af 100644--- a/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts+++ b/packages/shared/src/features/query/server/queryBuilder.filterValidation.test.ts@@ -313,3 +313,77 @@ describe("legacy traces compatibility when routed through v2", () => {},);});++describe("queryBuilder DateTime64 parameter encoding", () => {+ it("encodes epoch fromTimestamp without ClickHouse-rejected numeric 0", async () => {+ const query = {+ view: "traces",+ dimensions: [],+ metrics: [{ measure: "count", aggregation: "count" }],+ filters: [],+ timeDimension: null,+ fromTimestamp: "1970-01-01T00:00:00.000Z",+ toTimestamp: "2026-08-14T21:30:00.000Z",+ orderBy: null,+ } as QueryType;++ const queryBuilder = new QueryBuilder(undefined, "v2");+ queryBuilder.setRootEventConditionMaxWindowHours(0);++ const { query: compiledQuery, parameters } = await queryBuilder.build(+ query,+ "test-project",+ true,+ );++ expect(compiledQuery).toContain("DateTime64(3, 'UTC')");+ expect(compiledQuery).not.toContain(": DateTime64(3)}");++ const dateTimeParams = Object.entries(parameters).filter(+ ([key]) =>+ key.startsWith("dateTimeFilter") ||+ key.startsWith("subFrom") ||+ key.startsWith("subTo"),+ );++ expect(dateTimeParams.length).toBeGreaterThan(0);+ expect(dateTimeParams.some(([key]) => key.startsWith("subFrom"))).toBe(+ true,+ );+ expect(dateTimeParams.some(([key]) => key.startsWith("subTo"))).toBe(true);+ for (const [, value] of dateTimeParams) {+ // ClickHouse rejects numeric 0 for DateTime64(3) query parameters.+ expect(value).not.toBe(0);+ expect(typeof value).toBe("string");+ }+ expect(Object.values(parameters)).toContain("1970-01-01 00:00:00.000");+ expect(Object.values(parameters)).toContain("2026-08-14 21:30:00.000");+ });++ it("binds WITH FILL bounds as UTC DateTime64 parameters", async () => {+ const query = {+ view: "traces",+ dimensions: [],+ metrics: [{ measure: "count", aggregation: "count" }],+ filters: [],+ timeDimension: { granularity: "day" },+ fromTimestamp: "2025-01-01T00:00:00.000Z",+ toTimestamp: "2025-01-02T00:00:00.000Z",+ orderBy: null,+ } as QueryType;++ const { query: compiledQuery, parameters } = await new QueryBuilder(+ undefined,+ "v2",+ ).build(query, "test-project", true);++ expect(compiledQuery).toContain(+ "toDate({fillFromDate: DateTime64(3, 'UTC')})",+ );+ expect(compiledQuery).toContain(+ "toDate({fillToDate: DateTime64(3, 'UTC')})",+ );+ expect(parameters.fillFromDate).toBe("2025-01-01 00:00:00.000");+ expect(parameters.fillToDate).toBe("2025-01-02 00:00:00.000");+ });+});diff --git a/packages/shared/src/features/query/server/queryBuilder.ts b/packages/shared/src/features/query/server/queryBuilder.tsindex 7b0e0d1c6e..61b548833c 100644--- a/packages/shared/src/features/query/server/queryBuilder.ts+++ b/packages/shared/src/features/query/server/queryBuilder.ts@@ -1,8 +1,8 @@import { randomUUID } from "crypto";import { type z } from "zod";-import { convertDateToClickhouseDateTime } from "../../../server/clickhouse/client";import { shouldSkipObservationsFinal } from "../../../server/queries/clickhouse-sql/query-options";import {+ bindUtcDateTimeParam,FilterList,filtersRequireEventsFull,type Filter,@@ -1248,14 +1248,18 @@ export class QueryBuilder {step = "INTERVAL 1 DAY"; // Default to day if granularity is unknown}- parameters["fillFromDate"] = convertDateToClickhouseDateTime(+ const fillFromDateParam = bindUtcDateTimeParam(+ "fillFromDate",new Date(fromTimestamp),);- parameters["fillToDate"] = convertDateToClickhouseDateTime(+ const fillToDateParam = bindUtcDateTimeParam(+ "fillToDate",new Date(toTimestamp),);+ parameters["fillFromDate"] = fillFromDateParam.value;+ parameters["fillToDate"] = fillToDateParam.value;- return ` WITH FILL FROM ${this.getTimeDimensionSql("{fillFromDate: DateTime64(3)}", appliedBucketingDimension.granularity)} TO ${this.getTimeDimensionSql("{fillToDate: DateTime64(3)}", appliedBucketingDimension.granularity)} STEP ${step}`;+ return ` WITH FILL FROM ${this.getTimeDimensionSql(fillFromDateParam.placeholder, appliedBucketingDimension.granularity)} TO ${this.getTimeDimensionSql(fillToDateParam.placeholder, appliedBucketingDimension.granularity)} STEP ${step}`;}/**@@ -1672,17 +1676,22 @@ export class QueryBuilder {const baseTable = this.actualTableName(view);const tableAlias = this.tableAlias(view);const { column, condition } = view.rootEventCondition;+ const fromParam = bindUtcDateTimeParam(+ fromP,+ new Date(query.fromTimestamp),+ );+ const toParam = bindUtcDateTimeParam(toP, new Date(query.toTimestamp));const subquery =`SELECT ${tableAlias}.${column} FROM ${baseTable} ${tableAlias} ` +`WHERE ${tableAlias}.project_id = {${projP}: String} ` +`AND ${condition} ` +- `AND ${tableAlias}.${view.timeDimension} >= {${fromP}: DateTime64(3)} ` +- `AND ${tableAlias}.${view.timeDimension} <= {${toP}: DateTime64(3)}`;+ `AND ${tableAlias}.${view.timeDimension} >= ${fromParam.placeholder} ` ++ `AND ${tableAlias}.${view.timeDimension} <= ${toParam.placeholder}`;fromClause +=` AND (${baseTable}.${column} IN (${subquery})` +` OR NOT EXISTS (${subquery} LIMIT 1))`;- parameters[fromP] = new Date(query.fromTimestamp).getTime();- parameters[toP] = new Date(query.toTimestamp).getTime();+ parameters[fromP] = fromParam.value;+ parameters[toP] = toParam.value;parameters[projP] = projectId;}}diff --git a/packages/shared/src/server/clickhouse/client.ts b/packages/shared/src/server/clickhouse/client.tsindex bd9ce974a3..efc87278a6 100644--- a/packages/shared/src/server/clickhouse/client.ts+++ b/packages/shared/src/server/clickhouse/client.ts@@ -257,9 +257,10 @@ export const clickhouseClient = (};/**- * Accepts a JavaScript date and returns the DateTime in format YYYY-MM-DD HH:MM:SS+ * Accepts a JavaScript date and returns its UTC calendar time in ClickHouse's+ * YYYY-MM-DD HH:MM:SS.sss format.*/export const convertDateToClickhouseDateTime = (date: Date): string => {- // 2024-11-06T20:37:00.123Z -> 2024-11-06 21:37:00.123+ // 2024-11-06T20:37:00.123Z -> 2024-11-06 20:37:00.123return date.toISOString().replace("T", " ").replace("Z", "");};diff --git a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.datetime.test.ts b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.datetime.test.tsnew file mode 100644index 0000000000..0248ff418f--- /dev/null+++ b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.datetime.test.ts@@ -0,0 +1,43 @@+import { describe, expect, it } from "vitest";++import { convertDateToClickhouseDateTime } from "../../clickhouse/client";+import { DateTimeFilter } from "./clickhouse-filter";++describe("DateTimeFilter ClickHouse parameter encoding", () => {+ it("encodes Unix epoch as a ClickHouse DateTime64 string, not numeric 0", () => {+ const epoch = new Date("1970-01-01T00:00:00.000Z");+ const filter = new DateTimeFilter({+ clickhouseTable: "traces",+ field: "timestamp",+ operator: ">=",+ value: epoch,+ tablePrefix: "t",+ });++ const applied = filter.apply();++ expect(applied.query).toContain("DateTime64(3, 'UTC')");+ const paramValue = Object.values(applied.params)[0];+ // ClickHouse rejects query parameter value 0 for DateTime64(3):+ // "Cannot parse DateTime: value 0 cannot be parsed as DateTime64(3)"+ expect(paramValue).not.toBe(0);+ expect(paramValue).toBe(convertDateToClickhouseDateTime(epoch));+ });++ it("encodes ordinary timestamps as ClickHouse DateTime64 strings", () => {+ const value = new Date("2026-08-14T23:30:00.123+02:00");+ const filter = new DateTimeFilter({+ clickhouseTable: "traces",+ field: "timestamp",+ operator: "<=",+ value,+ });++ const applied = filter.apply();+ const paramValue = Object.values(applied.params)[0];++ expect(applied.query).toContain("DateTime64(3, 'UTC')");+ expect(paramValue).toBe(convertDateToClickhouseDateTime(value));+ expect(paramValue).toBe("2026-08-14 21:30:00.123");+ });+});diff --git a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.tsindex 73d95fa79c..0842311b47 100644--- a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts@@ -3,6 +3,7 @@ import {type FtsMatchOperator,filterOperators,} from "../../../interfaces/filters";+import { convertDateToClickhouseDateTime } from "../../clickhouse/client";import { clickhouseCompliantRandomCharacters } from "../../repositories";import { escapeSqlLikePattern } from "../../utils/sqlLike";import {@@ -172,6 +173,11 @@ export class NumberFilter implements Filter {}}+export const bindUtcDateTimeParam = (name: string, value: Date) => ({+ placeholder: `{${name}: DateTime64(3, 'UTC')}`,+ value: convertDateToClickhouseDateTime(value),+});+export class DateTimeFilter implements Filter {public clickhouseTable: string;public field: string;@@ -196,9 +202,17 @@ export class DateTimeFilter implements Filter {apply(): ClickhouseFilter {const uid = clickhouseCompliantRandomCharacters();const varName = `dateTimeFilter${uid}`;+ const dateTimeParam = bindUtcDateTimeParam(varName, new Date(this.value));+ // Use ClickHouse DateTime string encoding rather than epoch millis.+ // ClickHouse rejects query parameter value 0 for DateTime64(3), which is+ // exactly what Date#getTime() returns for 1970-01-01T00:00:00.000Z. The+ // converter emits UTC calendar time, so declare UTC explicitly rather than+ // relying on the ClickHouse server or session timezone.return {- query: `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} ${this.operator} {${varName}: DateTime64(3)}`,- params: { [varName]: new Date(this.value).getTime() },+ query: `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} ${this.operator} ${dateTimeParam.placeholder}`,+ params: {+ [varName]: dateTimeParam.value,+ },};}}diff --git a/packages/shared/src/server/queries/clickhouse-sql/events-stream-query.test.ts b/packages/shared/src/server/queries/clickhouse-sql/events-stream-query.test.tsindex 1d2ceeca90..279b4aa92e 100644--- a/packages/shared/src/server/queries/clickhouse-sql/events-stream-query.test.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/events-stream-query.test.ts@@ -2,6 +2,7 @@ import type { FilterCondition } from "../../../types";import type { OrderByState } from "../../../interfaces/orderBy";import { InvalidRequestError } from "../../../errors";import { describe, expect, it } from "vitest";+import { convertDateToClickhouseDateTime } from "../../clickhouse/client";import {buildEventsObservationRowSelection,groupEventsObservationFilters,@@ -81,7 +82,9 @@ describe("buildEventsStreamQuery", () => {limit: 17,});expect(Object.values(params)).toContainEqual(["GENERATION"]);- expect(Object.values(params)).toContain(cutoffCreatedAt.getTime());+ expect(Object.values(params)).toContain(+ convertDateToClickhouseDateTime(cutoffCreatedAt),+ );});it("omits the optional cutoff when it is absent", () => {diff --git a/web/src/__tests__/server/metrics-v2-api.servertest.ts b/web/src/__tests__/server/metrics-v2-api.servertest.tsindex 640dd83408..cd71c91d03 100644--- a/web/src/__tests__/server/metrics-v2-api.servertest.ts+++ b/web/src/__tests__/server/metrics-v2-api.servertest.ts@@ -9,6 +9,7 @@ import {createEventsCh,createScoresCh,createTraceScore,+ DateTimeFilter,queryClickhouse,} from "@langfuse/shared/src/server";import { env } from "@/src/env.mjs";@@ -119,6 +120,29 @@ describe("/api/public/v2/metrics API Endpoint", () => {});maybe("Basic Functionality", () => {+ it("preserves DateTime64 filter instants outside the ClickHouse session timezone", async () => {+ const instant = new Date("2026-08-14T23:30:00.123+02:00");+ const applied = new DateTimeFilter({+ clickhouseTable: "events",+ field: "start_time",+ operator: ">=",+ value: instant,+ }).apply();+ const placeholder = applied.query.match(/\{[^}]+\}/)?.[0];++ if (!placeholder) {+ throw new Error("DateTimeFilter did not produce a query parameter");+ }++ const result = await queryClickhouse<{ timestampMs: string }>({+ query: `SELECT toUnixTimestamp64Milli(${placeholder}) AS timestampMs`,+ params: applied.params,+ clickhouseSettings: { session_timezone: "Europe/Berlin" },+ });++ expect(Number(result[0]?.timestampMs)).toBe(instant.getTime());+ });+it("should apply default row_limit of 100 when not specified", async () => {// Create enough observations to exceed default limitconst rowLimitTraceId = randomUUID();@@ -231,6 +255,38 @@ describe("/api/public/v2/metrics API Endpoint", () => {expect(Array.isArray(response.body.data)).toBe(true);});+ it("should accept Unix epoch fromTimestamp without ClickHouse DateTime64 parse errors", async () => {+ // Regression: ClickHouse rejects DateTime64(3) query parameter value 0,+ // which is Date#getTime() for 1970-01-01T00:00:00.000Z. Clients often use+ // epoch as an open-ended lower bound; the API must return 200, not 500.+ const query = {+ view: "observations",+ metrics: [{ measure: "count", aggregation: "count" }],+ filters: [+ {+ column: "traceId",+ operator: "=",+ value: traceId,+ type: "string",+ },+ ],+ fromTimestamp: "1970-01-01T00:00:00.000Z",+ toTimestamp: new Date(timestamp.getTime() + 10_000).toISOString(),+ };++ const response = await makeZodVerifiedAPICall(+ GetMetricsV1Response,+ "GET",+ `/api/public/v2/metrics?query=${encodeURIComponent(JSON.stringify(query))}`,+ );++ expect(response.status).toBe(200);+ expect(response.body.data).toHaveLength(1);+ expect(Number(response.body.data[0]?.count_count)).toBe(+ observationIds.length,+ );+ });+it("should support latency metrics with microsecond to millisecond conversion", async () => {const query = {view: "observations",diff --git a/web/src/__tests__/server/orderByToPrisma.servertest.ts b/web/src/__tests__/server/orderByToPrisma.servertest.tsindex b625b71cb3..af624b383e 100644--- a/web/src/__tests__/server/orderByToPrisma.servertest.ts+++ b/web/src/__tests__/server/orderByToPrisma.servertest.ts@@ -98,7 +98,7 @@ describe("orderByToPrisma (Convert orderBy to Prisma.sql)", () => {);expect(filterList.apply().query).toMatch(- /^s\.timestamp >= \{dateTimeFilter[A-Za-z]{5}: DateTime64\(3\)\}$/,+ /^s\.timestamp >= \{dateTimeFilter[A-Za-z]{5}: DateTime64\(3, 'UTC'\)\}$/,);});});diff --git a/web/src/features/dashboard/server/dashboard-router.ts b/web/src/features/dashboard/server/dashboard-router.tsindex a95e094536..5a54ef6521 100644--- a/web/src/features/dashboard/server/dashboard-router.ts+++ b/web/src/features/dashboard/server/dashboard-router.ts@@ -129,9 +129,8 @@ const LEGACY_CAMEL_CASE_MAP: Record<string, string> = {*/function prepareScoresNumericV2Params(filter: FilterState) {const [from, to] = extractFromAndToTimestampsFromFilter(filter);- // Fallback to 2000-01-01 instead of epoch 0 — ClickHouse DateTimeFilter- // passes new Date(value).getTime() as the parameter, and the value 0- // (epoch) is rejected by ClickHouse's DateTime64(3) parameter parser.+ // Preserve the existing default range. DateTimeFilter now accepts epoch as a+ // lower bound, but widening dashboard queries is a separate behavior change.const fromIso = from?.value? new Date(from.value as Date).toISOString(): new Date("2000-01-01T00:00:00.000Z").toISOString();