🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(clickhouse): stringify DateTime64 timestamps on insert (#16892)
Steffen Schmitzd812e3afd34c
diff --git a/docker-compose.dev-redis-cluster.yml b/docker-compose.dev-redis-cluster.ymlindex c8561a6fbd..46335ee741 100644--- a/docker-compose.dev-redis-cluster.yml+++ b/docker-compose.dev-redis-cluster.yml@@ -12,7 +12,9 @@ x-redis-cluster-healthcheck: &redis-cluster-healthcheckservices:clickhouse:- image: docker.io/clickhouse/clickhouse-server:25.12+ # Latest ClickHouse in the CI matrix. Used to catch parser/settings+ # regressions such as DateTime64 unquoted-number scaling on 26.8+.+ image: docker.io/clickhouse/clickhouse-server:26.8user: "101:101"environment:CLICKHOUSE_DB: defaultdiff --git a/packages/shared/scripts/seeder/scenarios/event-mirror.ts b/packages/shared/scripts/seeder/scenarios/event-mirror.tsindex 43ed53f3a8..809f226ebe 100644--- a/packages/shared/scripts/seeder/scenarios/event-mirror.ts+++ b/packages/shared/scripts/seeder/scenarios/event-mirror.ts@@ -5,9 +5,6 @@ import {TraceRecordInsertType,} from "../../../src/server";-const toMicros = (ms: number | null | undefined): number | null =>- ms === null || ms === undefined ? null : ms * 1000;-const utf8Bytes = (value: unknown): number =>typeof value === "string" ? Buffer.byteLength(value, "utf8") : 0;@@ -66,12 +63,12 @@ export const traceToEvent = (prompt_version: null,metadata_names: names,metadata_values: values,- start_time: toMicros(trace.timestamp) ?? Date.now() * 1000,+ start_time: trace.timestamp,end_time: null,completion_start_time: null,- created_at: toMicros(trace.created_at) ?? Date.now() * 1000,- updated_at: toMicros(trace.updated_at) ?? Date.now() * 1000,- event_ts: toMicros(trace.event_ts) ?? Date.now() * 1000,+ created_at: trace.created_at,+ updated_at: trace.updated_at,+ event_ts: trace.event_ts,event_bytes: utf8Bytes(trace.input) + utf8Bytes(trace.output),source: "API",});@@ -131,12 +128,12 @@ export const observationToEvent = (tool_call_names: observation.tool_call_names ?? [],metadata_names: names,metadata_values: values,- start_time: toMicros(observation.start_time) ?? Date.now() * 1000,- end_time: toMicros(observation.end_time),- completion_start_time: toMicros(observation.completion_start_time),- created_at: toMicros(observation.created_at) ?? Date.now() * 1000,- updated_at: toMicros(observation.updated_at) ?? Date.now() * 1000,- event_ts: toMicros(observation.event_ts) ?? Date.now() * 1000,+ start_time: observation.start_time,+ end_time: observation.end_time,+ completion_start_time: observation.completion_start_time,+ created_at: observation.created_at,+ updated_at: observation.updated_at,+ event_ts: observation.event_ts,event_bytes: utf8Bytes(observation.input) + utf8Bytes(observation.output),source: "API",});diff --git a/packages/shared/scripts/seeder/scenarios/long-session.ts b/packages/shared/scripts/seeder/scenarios/long-session.tsindex ef5fcdf678..1c3e2a1959 100644--- a/packages/shared/scripts/seeder/scenarios/long-session.ts+++ b/packages/shared/scripts/seeder/scenarios/long-session.ts@@ -8,6 +8,7 @@ import {createObservationsCh,createScoresCh,createEventsCh,+ toClickhouseDateTime,EventRecordInsertType,ObservationRecordInsertType,ScoreRecordInsertType,@@ -258,7 +259,9 @@ const run = async (// The root "session-turn" span must cover its children, otherwise the// waterfall shows children running after their parent already ended.if (observationsPerTrace > 1) {- observations[rootObservationIndex].end_time = maxChildEndTime + 25;+ observations[rootObservationIndex].end_time = toClickhouseDateTime(+ maxChildEndTime + 25,+ );}if (t % 3 === 0) {@@ -335,7 +338,11 @@ const run = async (const links = [sessionLink(ctx, sessionId),- traceLink(ctx, traces[0].id, traces[0].timestamp as number),+ traceLink(+ ctx,+ traces[0].id,+ Date.parse(traces[0].timestamp.replace(" ", "T") + "Z"),+ ),];// The session detail page 404s without the Postgres trace_sessions row.diff --git a/packages/shared/scripts/seeder/utils/data-generators.ts b/packages/shared/scripts/seeder/utils/data-generators.tsindex 39e384f633..db7cadc2ea 100644--- a/packages/shared/scripts/seeder/utils/data-generators.ts+++ b/packages/shared/scripts/seeder/utils/data-generators.ts@@ -36,6 +36,7 @@ import {createDatasetRunItem,createDatasetRunScore,UNKNOWN_INGESTION_SDK_VALUE,+ toClickhouseDateTime,} from "../../../src/server";/**@@ -1106,7 +1107,7 @@ export class DataGenerator {const now = Date.now();const traces: TraceRecordInsertType[] = dialogues.map((d, index) => ({id: `support-chat-${index}-${projectId.slice(-8)}`,- timestamp: now + index * 1000,+ timestamp: toClickhouseDateTime(now + index * 1000),name: "SupportChatSession",user_id: null,metadata: this.buildNestedSeedMetadata("support-chat", index, {@@ -1137,9 +1138,9 @@ export class DataGenerator {),output: JSON.stringify({ role: "assistant", content: d.assistant }),session_id: "support-chat-session",- created_at: now + index * 1000,- updated_at: now + index * 1000 + 500,- event_ts: now + index * 1000,+ created_at: toClickhouseDateTime(now + index * 1000),+ updated_at: toClickhouseDateTime(now + index * 1000 + 500),+ event_ts: toClickhouseDateTime(now + index * 1000),is_deleted: 0,}));@@ -1159,8 +1160,8 @@ export class DataGenerator {type: "GENERATION",parent_observation_id: null,environment: "default",- start_time: start,- end_time: end,+ start_time: toClickhouseDateTime(start),+ end_time: toClickhouseDateTime(end),name: "llm-generation",metadata: {},level: "DEFAULT",@@ -1203,13 +1204,13 @@ export class DataGenerator {total: Math.round(totalTokens * 5) / 1_000_000,},total_cost: Math.round(totalTokens * 5) / 1_000_000,- completion_start_time: start + 120,+ completion_start_time: toClickhouseDateTime(start + 120),prompt_id: null,prompt_name: null,prompt_version: null,- created_at: start,- updated_at: end,- event_ts: start,+ created_at: toClickhouseDateTime(start),+ updated_at: toClickhouseDateTime(end),+ event_ts: toClickhouseDateTime(start),is_deleted: 0,tool_definitions: d.tool? {@@ -1232,8 +1233,8 @@ export class DataGenerator {type: "TOOL",parent_observation_id: null,environment: "default",- start_time: start - 40,- end_time: start - 5,+ start_time: toClickhouseDateTime(start - 40),+ end_time: toClickhouseDateTime(start - 5),name: d.tool.name,metadata: {},level: "DEFAULT",@@ -1253,9 +1254,9 @@ export class DataGenerator {prompt_id: null,prompt_name: null,prompt_version: null,- created_at: start - 40,- updated_at: start - 5,- event_ts: start - 40,+ created_at: toClickhouseDateTime(start - 40),+ updated_at: toClickhouseDateTime(start - 5),+ event_ts: toClickhouseDateTime(start - 40),is_deleted: 0,tool_definitions: undefined,tool_calls: undefined,@@ -1289,10 +1290,10 @@ export class DataGenerator {long_string_value: "",queue_id: null,...this.buildSeedIngestionAttribution(projectId, "javascript-sdk"),- created_at: baseTs,- updated_at: baseTs,- timestamp: baseTs,- event_ts: baseTs,+ created_at: toClickhouseDateTime(baseTs),+ updated_at: toClickhouseDateTime(baseTs),+ timestamp: toClickhouseDateTime(baseTs),+ event_ts: toClickhouseDateTime(baseTs),is_deleted: 0,};@@ -1317,10 +1318,10 @@ export class DataGenerator {string_value: safeVal === 1 ? "True" : "False",queue_id: null,...this.buildSeedIngestionAttribution(projectId, "python-sdk"),- created_at: baseTs + 10,- updated_at: baseTs + 10,- timestamp: baseTs + 10,- event_ts: baseTs + 10,+ created_at: toClickhouseDateTime(baseTs + 10),+ updated_at: toClickhouseDateTime(baseTs + 10),+ timestamp: toClickhouseDateTime(baseTs + 10),+ event_ts: toClickhouseDateTime(baseTs + 10),is_deleted: 0,};@@ -1347,10 +1348,10 @@ export class DataGenerator {string_value: "True",queue_id: null,...this.buildSeedIngestionAttribution(projectId, "raw-api"),- created_at: baseTs + 20,- updated_at: baseTs + 20,- timestamp: baseTs + 20,- event_ts: baseTs + 20,+ created_at: toClickhouseDateTime(baseTs + 20),+ updated_at: toClickhouseDateTime(baseTs + 20),+ timestamp: toClickhouseDateTime(baseTs + 20),+ event_ts: toClickhouseDateTime(baseTs + 20),is_deleted: 0,}: null;diff --git a/packages/shared/src/env.ts b/packages/shared/src/env.tsindex cf051dbd38..61f1b6bc67 100644--- a/packages/shared/src/env.ts+++ b/packages/shared/src/env.ts@@ -145,9 +145,9 @@ const EnvSchema = z.object({CLICKHOUSE_UPDATE_PARALLEL_MODE: z.enum(["sync", "async", "auto"]).default("auto"),- // Workaround for ClickHouse analyzer/lazy materialization bugs. In "auto",- // Langfuse detects the ClickHouse version on startup and applies known- // compatibility settings for affected version bands.+ // Workarounds for ClickHouse version-specific parser and analyzer bugs. In+ // "auto", Langfuse detects the ClickHouse version on startup and applies+ // known compatibility settings for affected version bands.CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION: z.enum(["auto", "true", "false"]).default("auto"),diff --git a/packages/shared/src/server/clickhouse/client.datetime.test.ts b/packages/shared/src/server/clickhouse/client.datetime.test.tsnew file mode 100644index 0000000000..77ce2dcce0--- /dev/null+++ b/packages/shared/src/server/clickhouse/client.datetime.test.ts@@ -0,0 +1,47 @@+import { describe, expect, it } from "vitest";+import {+ convertDateToClickhouseDateTime,+ toClickhouseDateTime,+} from "./client";+import { parseClickhouseUTCDateTimeFormat } from "../repositories/clickhouse";++describe("toClickhouseDateTime", () => {+ const ticks = Date.UTC(2024, 10, 6, 20, 37, 0, 123);+ const quoted = "2024-11-06 20:37:00.123";++ it("formats Date values", () => {+ expect(toClickhouseDateTime(new Date(ticks))).toBe(quoted);+ });++ it("formats unix millisecond timestamps", () => {+ expect(toClickhouseDateTime(ticks)).toBe(quoted);+ });++ it("formats ISO strings", () => {+ expect(toClickhouseDateTime("2024-11-06T20:37:00.123Z")).toBe(quoted);+ });++ it("passes through ClickHouse datetime strings", () => {+ expect(toClickhouseDateTime(quoted)).toBe(quoted);+ });++ it("matches convertDateToClickhouseDateTime for Date inputs", () => {+ const date = new Date(ticks);+ expect(toClickhouseDateTime(date)).toBe(+ convertDateToClickhouseDateTime(date),+ );+ });+});++describe("parseClickhouseUTCDateTimeFormat", () => {+ it("reads DateTime64 insert strings back as the same UTC instant", () => {+ const ticks = Date.UTC(2024, 10, 6, 20, 37, 0, 123);+ const quoted = toClickhouseDateTime(ticks);++ expect(quoted).toBe("2024-11-06 20:37:00.123");+ expect(parseClickhouseUTCDateTimeFormat(quoted).toISOString()).toBe(+ "2024-11-06T20:37:00.123Z",+ );+ expect(parseClickhouseUTCDateTimeFormat(quoted).getTime()).toBe(ticks);+ });+});diff --git a/packages/shared/src/server/clickhouse/client.ts b/packages/shared/src/server/clickhouse/client.tsindex ad81a02918..e3d00b3c35 100644--- a/packages/shared/src/server/clickhouse/client.ts+++ b/packages/shared/src/server/clickhouse/client.ts@@ -268,3 +268,27 @@ export const convertDateToClickhouseDateTime = (date: Date): string => {// 2024-11-06T20:37:00.123Z -> 2024-11-06 20:37:00.123return date.toISOString().replace("T", " ").replace("Z", "");};++const CLICKHOUSE_DATETIME_STRING = /^\d{4}-\d{2}-\d{2} /;++/**+ * Convert a Date, unix-millisecond timestamp, ISO string, or already-quoted+ * ClickHouse datetime string to YYYY-MM-DD HH:MM:SS.sss for JSONEachRow inserts.+ */+export const toClickhouseDateTime = (+ value?: Date | number | string | null,+): string => {+ if (value == null) {+ return convertDateToClickhouseDateTime(new Date());+ }+ if (value instanceof Date) {+ return convertDateToClickhouseDateTime(value);+ }+ if (typeof value === "number") {+ return convertDateToClickhouseDateTime(new Date(value));+ }+ if (CLICKHOUSE_DATETIME_STRING.test(value)) {+ return value;+ }+ return convertDateToClickhouseDateTime(new Date(value));+};diff --git a/packages/shared/src/server/clickhouse/compatibility.test.ts b/packages/shared/src/server/clickhouse/compatibility.test.tsindex b1a667b566..31744173d3 100644--- a/packages/shared/src/server/clickhouse/compatibility.test.ts+++ b/packages/shared/src/server/clickhouse/compatibility.test.ts@@ -70,6 +70,9 @@ describe("resolveClickHouseCompatibility", () => {["26.7.1.1334", noCompatibilitySettings],["26.7.2.0", disableTopKThroughJoin],["26.7.2.11", noCompatibilitySettings],+ ["26.7.99.9999", noCompatibilitySettings],+ ["26.8.0.0", noCompatibilitySettings],+ ["26.8.1.2041", noCompatibilitySettings],])("resolves automatic settings for %s", (version, expectedSettings) => {expect(resolveClickHouseCompatibility({ version }).settings).toEqual(expectedSettings,@@ -107,7 +110,9 @@ describe("resolveClickHouseCompatibility", () => {CLICKHOUSE_DISABLE_TOP_K_THROUGH_JOIN: "false",},}).settings,- ).toEqual(disableLazyMaterialization);+ ).toEqual({+ ...disableLazyMaterialization,+ });});});diff --git a/packages/shared/src/server/data-deletion/ingestionFileDeletion.ts b/packages/shared/src/server/data-deletion/ingestionFileDeletion.tsindex c4e346d76d..59cb5ff3f7 100644--- a/packages/shared/src/server/data-deletion/ingestionFileDeletion.ts+++ b/packages/shared/src/server/data-deletion/ingestionFileDeletion.ts@@ -8,7 +8,7 @@ import {import { BlobStorageFileRefRecordReadType } from "../repositories/definitions";import { logger } from "../logger";import { env } from "../../env";-import { clickhouseClient } from "../clickhouse/client";+import { clickhouseClient, toClickhouseDateTime } from "../clickhouse/client";import { buildClickHouseLogComment } from "../clickhouse/queryTags";import { getS3EventStorageClient, S3_DELETE_OBJECTS_CHUNK_SIZE } from "../s3";@@ -225,8 +225,8 @@ async function softDeleteInClickhouse(values: blobStorageRefs.map((e) => ({...e,is_deleted: "1",- event_ts: new Date().getTime(),- updated_at: new Date().getTime(),+ event_ts: toClickhouseDateTime(),+ updated_at: toClickhouseDateTime(),})),format: "JSONEachRow",clickhouse_settings: {diff --git a/packages/shared/src/server/repositories/definitions.ts b/packages/shared/src/server/repositories/definitions.tsindex 819b3ff2fa..04ba2995fa 100644--- a/packages/shared/src/server/repositories/definitions.ts+++ b/packages/shared/src/server/repositories/definitions.ts@@ -1,6 +1,7 @@import z from "zod";import type { IngestionAttribution } from "../ingestion/ingestionAttribution";import { DEFAULT_TRACE_ENVIRONMENT } from "../ingestion/types";+import { toClickhouseDateTime } from "../clickhouse/client";export const clickhouseStringDateSchema = z.string()@@ -79,12 +80,12 @@ export type ObservationRecordReadType = z.infer<export const observationRecordInsertSchema = observationRecordBaseSchema.extend({- created_at: z.number(),- updated_at: z.number(),- start_time: z.number(),- end_time: z.number().nullish(),- completion_start_time: z.number().nullish(),- event_ts: z.number(),+ created_at: z.string(),+ updated_at: z.string(),+ start_time: z.string(),+ end_time: z.string().nullish(),+ completion_start_time: z.string().nullish(),+ event_ts: z.string(),provided_usage_details: UsageCostSchema,provided_cost_details: UsageCostSchema,usage_details: UsageCostSchema,@@ -100,7 +101,7 @@ export const observationBatchStagingRecordInsertSchema =ingestion_api_key: z.string(),ingestion_sdk_name: z.string(),ingestion_sdk_version: z.string(),- s3_first_seen_timestamp: z.number(),+ s3_first_seen_timestamp: z.string(),});export type ObservationBatchStagingRecordInsertType = z.infer<typeof observationBatchStagingRecordInsertSchema@@ -166,10 +167,10 @@ export const traceRecordReadSchema = traceRecordBaseSchema.extend({export type TraceRecordReadType = z.infer<typeof traceRecordReadSchema>;export const traceRecordInsertSchema = traceRecordBaseSchema.extend({- timestamp: z.number(),- created_at: z.number(),- updated_at: z.number(),- event_ts: z.number(),+ timestamp: z.string(),+ created_at: z.string(),+ updated_at: z.string(),+ event_ts: z.string(),});export type TraceRecordInsertType = z.infer<typeof traceRecordInsertSchema>;@@ -177,8 +178,8 @@ export const traceNullRecordInsertSchema = z.object({// Identifiersproject_id: z.string(),id: z.string(),- start_time: z.number(),- end_time: z.number().nullish(),+ start_time: z.string(),+ end_time: z.string().nullish(),name: z.string().nullish(),// Metadata properties@@ -204,9 +205,9 @@ export const traceNullRecordInsertSchema = z.object({input: z.string(),output: z.string(),- created_at: z.number(),- updated_at: z.number(),- event_ts: z.number(),+ created_at: z.string(),+ updated_at: z.string(),+ event_ts: z.string(),});export type TraceNullRecordInsertType = z.infer<typeof traceNullRecordInsertSchema@@ -249,10 +250,10 @@ export const scoreRecordReadSchema = scoreRecordBaseSchema.extend({export type ScoreRecordReadType = z.infer<typeof scoreRecordReadSchema>;export const scoreRecordInsertSchema = scoreRecordBaseSchema.extend({- created_at: z.number(),- updated_at: z.number(),- timestamp: z.number(),- event_ts: z.number(),+ created_at: z.string(),+ updated_at: z.string(),+ timestamp: z.string(),+ event_ts: z.string(),});export type ScoreRecordInsertType = z.infer<typeof scoreRecordInsertSchema>;@@ -298,11 +299,11 @@ export type DatasetRunItemRecord<WithIO extends boolean = true> =export const datasetRunItemRecordInsertSchema =datasetRunItemRecordBaseSchema.extend({- created_at: z.number(),- updated_at: z.number(),- event_ts: z.number(),- dataset_run_created_at: z.number(),- dataset_item_version: z.number().nullish(),+ created_at: z.string(),+ updated_at: z.string(),+ event_ts: z.string(),+ dataset_run_created_at: z.string(),+ dataset_item_version: z.string().nullish(),});export type DatasetRunItemRecordInsertType = z.infer<typeof datasetRunItemRecordInsertSchema@@ -331,9 +332,9 @@ export type BlobStorageFileRefRecordReadType = z.infer<>;export const blobStorageFileLogRecordInsertSchema =blobStorageFileLogRecordBaseSchema.extend({- created_at: z.number(),- updated_at: z.number(),- event_ts: z.number(),+ created_at: z.string(),+ updated_at: z.string(),+ event_ts: z.string(),});export type BlobStorageFileLogInsertType = z.infer<typeof blobStorageFileLogRecordInsertSchema@@ -344,28 +345,28 @@ export const convertTraceReadToInsert = (): TraceRecordInsertType => {return {...record,- created_at: new Date(record.created_at).getTime(),- updated_at: new Date(record.updated_at).getTime(),- timestamp: new Date(record.timestamp).getTime(),- event_ts: new Date(record.event_ts).getTime(),+ created_at: toClickhouseDateTime(record.created_at),+ updated_at: toClickhouseDateTime(record.updated_at),+ timestamp: toClickhouseDateTime(record.timestamp),+ event_ts: toClickhouseDateTime(record.event_ts),};};export const convertObservationReadToInsert = (record: ObservationRecordReadType,): ObservationRecordInsertType => {- const convertDate = (date: string) => new Date(date).getTime();-return {...record,- created_at: convertDate(record.created_at),- updated_at: convertDate(record.updated_at),- start_time: convertDate(record.start_time),- end_time: record.end_time ? convertDate(record.end_time) : undefined,+ created_at: toClickhouseDateTime(record.created_at),+ updated_at: toClickhouseDateTime(record.updated_at),+ start_time: toClickhouseDateTime(record.start_time),+ end_time: record.end_time+ ? toClickhouseDateTime(record.end_time)+ : undefined,completion_start_time: record.completion_start_time- ? convertDate(record.completion_start_time)+ ? toClickhouseDateTime(record.completion_start_time): undefined,- event_ts: convertDate(record.event_ts),+ event_ts: toClickhouseDateTime(record.event_ts),provided_usage_details: record.provided_usage_details,provided_cost_details: record.provided_cost_details,usage_details: record.usage_details,@@ -378,10 +379,10 @@ export const convertScoreReadToInsert = (): ScoreRecordInsertType => {return {...record,- created_at: new Date(record.created_at).getTime(),- updated_at: new Date(record.updated_at).getTime(),- timestamp: new Date(record.timestamp).getTime(),- event_ts: new Date(record.event_ts).getTime(),+ created_at: toClickhouseDateTime(record.created_at),+ updated_at: toClickhouseDateTime(record.updated_at),+ timestamp: toClickhouseDateTime(record.timestamp),+ event_ts: toClickhouseDateTime(record.event_ts),};};@@ -392,7 +393,7 @@ export const convertScoreReadToInsert = (*/export const convertTraceToStagingObservation = (traceRecord: TraceRecordInsertType,- s3FirstSeenTimestamp: number,+ s3FirstSeenTimestamp: string,attribution: IngestionAttribution,): ObservationBatchStagingRecordInsertType => {return {@@ -572,11 +573,11 @@ export const eventRecordReadSchema = eventRecordBaseSchema.extend({export type EventRecordReadType = z.infer<typeof eventRecordReadSchema>;export const eventRecordInsertSchema = eventRecordBaseSchema.extend({- start_time: z.number(),- end_time: z.number().nullish(),- completion_start_time: z.number().nullish(),- created_at: z.number(),- updated_at: z.number(),- event_ts: z.number(),+ start_time: z.string(),+ end_time: z.string().nullish(),+ completion_start_time: z.string().nullish(),+ created_at: z.string(),+ updated_at: z.string(),+ event_ts: z.string(),});export type EventRecordInsertType = z.infer<typeof eventRecordInsertSchema>;diff --git a/packages/shared/src/server/test-utils/tracing-factory.test.ts b/packages/shared/src/server/test-utils/tracing-factory.test.tsnew file mode 100644index 0000000000..b200e97e3d--- /dev/null+++ b/packages/shared/src/server/test-utils/tracing-factory.test.ts@@ -0,0 +1,35 @@+import { describe, expect, it } from "vitest";+import { createEvent, createObservation } from "./tracing-factory";++describe("tracing factories DateTime64 nulls", () => {+ it("keeps explicit null end and completion times on observations", () => {+ const observation = createObservation({+ end_time: null,+ completion_start_time: null,+ });++ expect(observation.end_time).toBeNull();+ expect(observation.completion_start_time).toBeNull();+ });++ it("keeps explicit null end and completion times on events", () => {+ const event = createEvent({+ end_time: null,+ completion_start_time: null,+ });++ expect(event.end_time).toBeNull();+ expect(event.completion_start_time).toBeNull();+ });++ it("stringifies millisecond and microsecond event ticks to the same instant", () => {+ const millisecond = Date.UTC(2024, 0, 15, 12, 0, 0);++ expect(createEvent({ start_time: millisecond }).start_time).toBe(+ "2024-01-15 12:00:00.000",+ );+ expect(createEvent({ start_time: millisecond * 1000 }).start_time).toBe(+ "2024-01-15 12:00:00.000",+ );+ });+});diff --git a/packages/shared/src/server/test-utils/tracing-factory.ts b/packages/shared/src/server/test-utils/tracing-factory.tsindex 6410fae6a0..179db79145 100644--- a/packages/shared/src/server/test-utils/tracing-factory.ts+++ b/packages/shared/src/server/test-utils/tracing-factory.ts@@ -1,4 +1,5 @@import { v4 } from "uuid";+import { toClickhouseDateTime } from "../clickhouse/client";import {TraceRecordInsertType,ObservationRecordInsertType,@@ -8,14 +9,33 @@ import {} from "../repositories/definitions";import { UNKNOWN_INGESTION_SDK_VALUE } from "../ingestion/ingestionAttribution";+type DateTimeInput = Date | number | string;++type WithDateTimeInputs<T, K extends keyof T> = Omit<Partial<T>, K> & {+ [P in K]?: DateTimeInput | null;+};++// Test fixtures historically passed events_full ticks as microseconds.+// Numbers above 1e14 cannot be unix milliseconds in the DateTime64 range.+const toEventClickhouseDateTime = (value?: DateTimeInput | null): string => {+ if (typeof value === "number" && Math.abs(value) > 1e14) {+ return toClickhouseDateTime(Math.trunc(value / 1000));+ }+ return toClickhouseDateTime(value);+};+export const createTrace = (- trace: Partial<TraceRecordInsertType>,+ trace: WithDateTimeInputs<+ TraceRecordInsertType,+ "timestamp" | "created_at" | "updated_at" | "event_ts"+ >,): TraceRecordInsertType => {+ const { timestamp, created_at, updated_at, event_ts, ...rest } = trace;return {id: v4(),project_id: v4(),session_id: v4(),- timestamp: Date.now(),+ timestamp: toClickhouseDateTime(timestamp),environment: "default",metadata: {source: "API",@@ -28,17 +48,32 @@ export const createTrace = (release: "1.0.0",version: "2",user_id: v4(),- created_at: Date.now(),- updated_at: Date.now(),- event_ts: Date.now(),+ created_at: toClickhouseDateTime(created_at),+ updated_at: toClickhouseDateTime(updated_at),+ event_ts: toClickhouseDateTime(event_ts),is_deleted: 0,- ...trace,+ ...rest,};};export const createDatasetRunItem = (- datasetRunItem: Partial<DatasetRunItemRecordInsertType>,+ datasetRunItem: WithDateTimeInputs<+ DatasetRunItemRecordInsertType,+ | "created_at"+ | "updated_at"+ | "event_ts"+ | "dataset_run_created_at"+ | "dataset_item_version"+ >,): DatasetRunItemRecordInsertType => {+ const {+ created_at,+ updated_at,+ event_ts,+ dataset_run_created_at,+ dataset_item_version,+ ...rest+ } = datasetRunItem;return {id: v4(),project_id: v4(),@@ -52,19 +87,39 @@ export const createDatasetRunItem = (dataset_item_input: "{}",dataset_item_expected_output: "{}",dataset_item_metadata: { key: "value" },- dataset_run_created_at: Date.now(),- created_at: Date.now(),- updated_at: Date.now(),- event_ts: Date.now(),+ dataset_run_created_at: toClickhouseDateTime(dataset_run_created_at),+ created_at: toClickhouseDateTime(created_at),+ updated_at: toClickhouseDateTime(updated_at),+ event_ts: toClickhouseDateTime(event_ts),+ dataset_item_version: dataset_item_version+ ? toClickhouseDateTime(dataset_item_version)+ : undefined,is_deleted: 0,error: datasetRunItem.error ?? null,- ...datasetRunItem,+ ...rest,};};export const createObservation = (- observation: Partial<ObservationRecordInsertType>,+ observation: WithDateTimeInputs<+ ObservationRecordInsertType,+ | "created_at"+ | "updated_at"+ | "start_time"+ | "end_time"+ | "completion_start_time"+ | "event_ts"+ >,): ObservationRecordInsertType => {+ const {+ created_at,+ updated_at,+ start_time,+ end_time,+ completion_start_time,+ event_ts,+ ...rest+ } = observation;return {id: v4(),trace_id: v4(),@@ -80,10 +135,10 @@ export const createObservation = (usage_details: { input: 1234, output: 5678, total: 6912 },cost_details: { input: 100, output: 200, total: 300 },is_deleted: 0,- created_at: Date.now(),- updated_at: Date.now(),- start_time: Date.now(),- event_ts: Date.now(),+ created_at: toClickhouseDateTime(created_at),+ updated_at: toClickhouseDateTime(updated_at),+ start_time: toClickhouseDateTime(start_time),+ event_ts: toClickhouseDateTime(event_ts),name: "sample_name" + v4(),level: "DEFAULT",status_message: "status",@@ -97,18 +152,25 @@ export const createObservation = (prompt_id: v4(),prompt_name: "generation-prompt",prompt_version: 1,- end_time: Date.now(),- completion_start_time: Date.now(),+ end_time: end_time === null ? null : toClickhouseDateTime(end_time),+ completion_start_time:+ completion_start_time === null+ ? null+ : toClickhouseDateTime(completion_start_time),tool_definitions: {},tool_calls: [],tool_call_names: [],- ...observation,+ ...rest,};};export const createTraceScore = (- score: Partial<ScoreRecordInsertType>,+ score: WithDateTimeInputs<+ ScoreRecordInsertType,+ "timestamp" | "created_at" | "updated_at" | "event_ts"+ >,): ScoreRecordInsertType => {+ const { timestamp, created_at, updated_at, event_ts, ...rest } = score;return {id: v4(),project_id: v4(),@@ -116,7 +178,7 @@ export const createTraceScore = (observation_id: null, // Trace-level scores must have observation_id as null by defaultenvironment: "default",name: "test-score" + v4(),- timestamp: Date.now(),+ timestamp: toClickhouseDateTime(timestamp),value: 100.5,string_value: null,long_string_value: "",@@ -124,43 +186,47 @@ export const createTraceScore = (comment: "comment",metadata: { "test-key": "test-value" },data_type: "NUMERIC" as const,- created_at: Date.now(),- updated_at: Date.now(),- event_ts: Date.now(),+ created_at: toClickhouseDateTime(created_at),+ updated_at: toClickhouseDateTime(updated_at),+ event_ts: toClickhouseDateTime(event_ts),ingestion_api_key: "",ingestion_sdk_name: UNKNOWN_INGESTION_SDK_VALUE,ingestion_sdk_version: UNKNOWN_INGESTION_SDK_VALUE,is_deleted: 0,- ...score,+ ...rest,session_id: null,dataset_run_id: null,};};export const createSessionScore = (- score: Partial<ScoreRecordInsertType>,+ score: WithDateTimeInputs<+ ScoreRecordInsertType,+ "timestamp" | "created_at" | "updated_at" | "event_ts"+ >,): ScoreRecordInsertType => {+ const { timestamp, created_at, updated_at, event_ts, ...rest } = score;return {id: v4(),project_id: v4(),session_id: v4(),environment: "default",name: "test-session-score" + v4(),- timestamp: Date.now(),+ timestamp: toClickhouseDateTime(timestamp),value: 100.5,long_string_value: "",source: "API",comment: "comment",metadata: { "test-key": "test-value" },data_type: "NUMERIC" as const,- created_at: Date.now(),- updated_at: Date.now(),- event_ts: Date.now(),+ created_at: toClickhouseDateTime(created_at),+ updated_at: toClickhouseDateTime(updated_at),+ event_ts: toClickhouseDateTime(event_ts),ingestion_api_key: "",ingestion_sdk_name: UNKNOWN_INGESTION_SDK_VALUE,ingestion_sdk_version: UNKNOWN_INGESTION_SDK_VALUE,is_deleted: 0,- ...score,+ ...rest,observation_id: null,trace_id: null,dataset_run_id: null,@@ -168,29 +234,33 @@ export const createSessionScore = (};export const createDatasetRunScore = (- score: Partial<ScoreRecordInsertType>,+ score: WithDateTimeInputs<+ ScoreRecordInsertType,+ "timestamp" | "created_at" | "updated_at" | "event_ts"+ >,): ScoreRecordInsertType => {+ const { timestamp, created_at, updated_at, event_ts, ...rest } = score;return {id: v4(),project_id: v4(),dataset_run_id: v4(),environment: "default",name: "test-run-score" + v4(),- timestamp: Date.now(),+ timestamp: toClickhouseDateTime(timestamp),value: 100.5,long_string_value: "",source: "API",comment: "comment",metadata: { "test-key": "test-value" },data_type: "NUMERIC" as const,- created_at: Date.now(),- updated_at: Date.now(),- event_ts: Date.now(),+ created_at: toClickhouseDateTime(created_at),+ updated_at: toClickhouseDateTime(updated_at),+ event_ts: toClickhouseDateTime(event_ts),ingestion_api_key: "",ingestion_sdk_name: UNKNOWN_INGESTION_SDK_VALUE,ingestion_sdk_version: UNKNOWN_INGESTION_SDK_VALUE,is_deleted: 0,- ...score,+ ...rest,observation_id: null,trace_id: null,session_id: null,@@ -198,17 +268,30 @@ export const createDatasetRunScore = (};export const createEvent = (- event: Partial<EventRecordInsertType> & {+ event: WithDateTimeInputs<+ EventRecordInsertType,+ | "start_time"+ | "end_time"+ | "completion_start_time"+ | "created_at"+ | "updated_at"+ | "event_ts"+ > & {metadata_values?: (string | null | undefined)[];},): EventRecordInsertType => {const spanId = v4();- const now = Date.now() * 1000; // Convert to micro// Extract metadata array overrides before spreading to prevent undefined from clobbering defaultsconst {metadata_values: metadataValuesAlias,metadata_names: metadataNamesOverride,+ start_time,+ end_time,+ completion_start_time,+ created_at,+ updated_at,+ event_ts,...eventOverrides} = event;@@ -308,13 +391,16 @@ export const createEvent = (event_bytes: 2,is_deleted: 0,- // Timestamps- start_time: now,- end_time: now,- completion_start_time: null,- created_at: now,- updated_at: now,- event_ts: now,+ // Timestamps (ClickHouse DateTime64 strings)+ start_time: toEventClickhouseDateTime(start_time),+ end_time: end_time === null ? null : toEventClickhouseDateTime(end_time),+ completion_start_time:+ completion_start_time == null+ ? null+ : toEventClickhouseDateTime(completion_start_time),+ created_at: toEventClickhouseDateTime(created_at),+ updated_at: toEventClickhouseDateTime(updated_at),+ event_ts: toEventClickhouseDateTime(event_ts),...eventOverrides,};diff --git a/web/src/__tests__/server/api/tables/prompts-ui.servertest.ts b/web/src/__tests__/server/api/tables/prompts-ui.servertest.tsindex 9ce6e192c0..6f9f2cb1fe 100644--- a/web/src/__tests__/server/api/tables/prompts-ui.servertest.ts+++ b/web/src/__tests__/server/api/tables/prompts-ui.servertest.ts@@ -11,6 +11,7 @@ import {getObservationMetricsForPromptsFromEvents,getObservationsWithPromptName,getObservationsWithPromptNameFromEvents,+ toClickhouseDateTime,} from "@langfuse/shared/src/server";import { env } from "@/src/env.mjs";import { v4 } from "uuid";@@ -472,7 +473,7 @@ describe("UI Prompts Table", () => {const promptId = v4();const traceId = v4();const spanId = v4();- const startTime = Date.parse("2026-07-16T13:42:33.028Z") * 1000;+ const startTime = Date.parse("2026-07-16T13:42:33.028Z");const event = createEvent({project_id: projectId,trace_id: traceId,@@ -482,7 +483,7 @@ describe("UI Prompts Table", () => {prompt_name: "folder/test-prompt",prompt_version: 7,start_time: startTime,- end_time: startTime + 1_000_000,+ end_time: startTime + 1_000,event_ts: startTime,usage_details: { input: 100, output: 200, total: 300 },cost_details: { input: 1, output: 2, total: 3 },@@ -492,8 +493,8 @@ describe("UI Prompts Table", () => {await createEventsCh([{...event,- end_time: startTime + 4_000_000,- event_ts: startTime + 1_000_000,+ end_time: toClickhouseDateTime(startTime + 4_000),+ event_ts: toClickhouseDateTime(startTime + 1_000),usage_details: { input: 400, output: 500, total: 900 },cost_details: { input: 4, output: 5, total: 9 },},diff --git a/web/src/__tests__/server/dashboard-v1-v2-consistency.servertest.ts b/web/src/__tests__/server/dashboard-v1-v2-consistency.servertest.tsindex 36964aff78..573d38d677 100644--- a/web/src/__tests__/server/dashboard-v1-v2-consistency.servertest.ts+++ b/web/src/__tests__/server/dashboard-v1-v2-consistency.servertest.ts@@ -122,15 +122,15 @@ function buildMatchingEvents(input: t.input ?? null,output: t.output ?? null,metadata: t.metadata ?? {},- start_time: t.timestamp * 1000,+ start_time: t.timestamp,end_time: null,cost_details: {},provided_cost_details: {},usage_details: {},provided_usage_details: {},- created_at: t.created_at * 1000,- updated_at: t.updated_at * 1000,- event_ts: t.event_ts * 1000,+ created_at: t.created_at,+ updated_at: t.updated_at,+ event_ts: t.event_ts,}),),);@@ -176,14 +176,12 @@ function buildMatchingEvents(tool_definitions: o.tool_definitions ?? {},tool_calls: o.tool_calls ?? [],tool_call_names: o.tool_call_names ?? [],- start_time: o.start_time * 1000,- end_time: o.end_time ? o.end_time * 1000 : null,- completion_start_time: o.completion_start_time- ? o.completion_start_time * 1000- : null,- created_at: o.created_at * 1000,- updated_at: o.updated_at * 1000,- event_ts: o.event_ts * 1000,+ start_time: o.start_time,+ end_time: o.end_time ?? null,+ completion_start_time: o.completion_start_time ?? null,+ created_at: o.created_at,+ updated_at: o.updated_at,+ event_ts: o.event_ts,}),),);diff --git a/web/src/__tests__/server/mcp-tools-read.servertest.ts b/web/src/__tests__/server/mcp-tools-read.servertest.tsindex 6194525ba2..5233f40138 100644--- a/web/src/__tests__/server/mcp-tools-read.servertest.ts+++ b/web/src/__tests__/server/mcp-tools-read.servertest.ts@@ -2176,7 +2176,9 @@ describe("MCP Read Tools", () => {projectId,traceId,name: `mcp-metrics-custom-limit-${index}-${nanoid()}`,- startTime: new Date(`2026-01-01T00:00:${index}.000Z`),+ startTime: new Date(+ `2026-01-01T00:00:${String(index).padStart(2, "0")}.000Z`,+ ),}),),);diff --git a/web/src/__tests__/server/observations-api.servertest.ts b/web/src/__tests__/server/observations-api.servertest.tsindex 864e3344d7..5548ab9dc3 100644--- a/web/src/__tests__/server/observations-api.servertest.ts+++ b/web/src/__tests__/server/observations-api.servertest.ts@@ -819,17 +819,19 @@ describe("/api/public/observations API Endpoint", () => {: baseTimestamp;// Create two traces with different userIds+ // traces.timestamp is DateTime64(3) millisecond ticks. timeValue is+ // microseconds when the events table is enabled — do not reuse it here.const trace1 = createTrace({id: trace1Id,project_id: projectId,- timestamp: timeValue,+ timestamp: baseTimestamp,user_id: "user-A",});const trace2 = createTrace({id: trace2Id,project_id: projectId,- timestamp: timeValue,+ timestamp: baseTimestamp,user_id: "user-B",});diff --git a/web/src/__tests__/server/queryBuilder.servertest.ts b/web/src/__tests__/server/queryBuilder.servertest.tsindex 4ce7a97f9c..34a1a5f668 100644--- a/web/src/__tests__/server/queryBuilder.servertest.ts+++ b/web/src/__tests__/server/queryBuilder.servertest.ts@@ -3381,8 +3381,13 @@ describe("queryBuilder", () => {result.data = await executeQuery(projectId, query);expect(result.data).toHaveLength(2);- expect(result.data[0].name).toBe("observation-basic");- expect(Number(result.data[0].count_count)).toBe(1);+ expect(result.data.map((row) => row.name).toSorted()).toEqual([+ "observation-basic",+ "observation-premium",+ ]);+ expect(result.data.every((row) => Number(row.count_count) === 1)).toBe(+ true,+ );});it("should generate histogram with custom bin count for cost distribution", async () => {@@ -4089,13 +4094,13 @@ describe("queryBuilder", () => {project_id: projectId,type: "GENERATION",cost_details: { input: 10, output: 20, total: 30 },- start_time: Date.now() * 1000,+ start_time: Date.now(),}),createEvent({project_id: projectId,type: "GENERATION",cost_details: { input: 5, output: 15, total: 20 },- start_time: Date.now() * 1000,+ start_time: Date.now(),}),];await createEventsCh(events);@@ -4139,13 +4144,13 @@ describe("queryBuilder", () => {project_id: projectId,type: "GENERATION",usage_details: { input: 100, output: 200, total: 300 },- start_time: Date.now() * 1000,+ start_time: Date.now(),}),createEvent({project_id: projectId,type: "GENERATION",usage_details: { input: 50, output: 75, total: 125 },- start_time: Date.now() * 1000,+ start_time: Date.now(),}),];await createEventsCh(events);@@ -4247,13 +4252,13 @@ describe("queryBuilder", () => {project_id: projectId,type: "GENERATION",cost_details: { input: 10 },- start_time: Date.now() * 1000,+ start_time: Date.now(),}),createEvent({project_id: projectId,type: "SPAN",cost_details: { input: 999 },- start_time: Date.now() * 1000,+ start_time: Date.now(),}),];await createEventsCh(events);@@ -4300,13 +4305,13 @@ describe("queryBuilder", () => {project_id: projectId,type: "GENERATION",cost_details: { input: 10, output: 20, total: 30 },- start_time: Date.now() * 1000,+ start_time: Date.now(),}),createEvent({project_id: projectId,type: "GENERATION",cost_details: { input: 5, output: 15, total: 20 },- start_time: Date.now() * 1000,+ start_time: Date.now(),}),];await createEventsCh(events);@@ -4352,7 +4357,7 @@ describe("queryBuilder", () => {project_id: projectId,type: "GENERATION",cost_details: { input: 7, output: 3 },- start_time: Date.now() * 1000,+ start_time: Date.now(),}),];await createEventsCh(events);@@ -5044,7 +5049,7 @@ describe("query builder measure-aggregation validation", () => {trace_name: "",name: "my-trace",parent_span_id: "", // root event- start_time: Date.now() * 1000,+ start_time: Date.now(),}),createEvent({project_id: projectId,@@ -5052,7 +5057,7 @@ describe("query builder measure-aggregation validation", () => {trace_name: "",name: "child-observation",parent_span_id: "some-parent", // child event- start_time: Date.now() * 1000,+ start_time: Date.now(),}),// Trace 2: trace_name is "other-trace"createEvent({@@ -5061,7 +5066,7 @@ describe("query builder measure-aggregation validation", () => {trace_name: "other-trace",name: "root-event",parent_span_id: "", // root event- start_time: Date.now() * 1000,+ start_time: Date.now(),}),];await createEventsCh(events);@@ -5111,7 +5116,7 @@ describe("query builder measure-aggregation validation", () => {trace_name: "target-trace",name: "root-observation",parent_span_id: "",- start_time: Date.now() * 1000,+ start_time: Date.now(),}),createEvent({project_id: projectId,@@ -5119,7 +5124,7 @@ describe("query builder measure-aggregation validation", () => {trace_name: "other-trace",name: "root-observation",parent_span_id: "",- start_time: Date.now() * 1000,+ start_time: Date.now(),}),];await createEventsCh(events);@@ -5172,7 +5177,7 @@ describe("query builder measure-aggregation validation", () => {name: "root-op",parent_span_id: "",environment: "production",- start_time: Date.now() * 1000,+ start_time: Date.now(),}),// Trace 2: name="target-trace", environment="staging"createEvent({@@ -5182,7 +5187,7 @@ describe("query builder measure-aggregation validation", () => {name: "root-op",parent_span_id: "",environment: "staging",- start_time: Date.now() * 1000,+ start_time: Date.now(),}),// Trace 3: name="other-trace", environment="production"createEvent({@@ -5192,7 +5197,7 @@ describe("query builder measure-aggregation validation", () => {name: "root-op",parent_span_id: "",environment: "production",- start_time: Date.now() * 1000,+ start_time: Date.now(),}),];await createEventsCh(events);@@ -5321,7 +5326,7 @@ describe("query builder measure-aggregation validation", () => {parent_span_id: "",metadata_names: ["experiments"],metadata_values: [longValue],- start_time: Date.now() * 1000,+ start_time: Date.now(),}),]);diff --git a/web/src/__tests__/server/queryBuilderDashboards.servertest.ts b/web/src/__tests__/server/queryBuilderDashboards.servertest.tsindex eb55e0c2ca..07a8c5183f 100644--- a/web/src/__tests__/server/queryBuilderDashboards.servertest.ts+++ b/web/src/__tests__/server/queryBuilderDashboards.servertest.ts@@ -4,6 +4,7 @@ import {createTracesCh,createObservation,createObservationsCh,+ convertDateToClickhouseDateTime,} from "@langfuse/shared/src/server";import { executeQuery } from "@langfuse/shared/query/server";import { type QueryType } from "@langfuse/shared/query";@@ -220,7 +221,8 @@ describe("selfServeDashboards", () => {// Count recent production traces (within the last hour)stats.recentProductionTraces = traces.filter((t) =>- t.environment === "production" && t.timestamp >= oneHourAgo.getTime(),+ t.environment === "production" &&+ t.timestamp >= convertDateToClickhouseDateTime(oneHourAgo),).length;});diff --git a/web/src/__tests__/server/repositories/experiment-items-repository.servertest.ts b/web/src/__tests__/server/repositories/experiment-items-repository.servertest.tsindex 2ad8878819..de810dc94f 100644--- a/web/src/__tests__/server/repositories/experiment-items-repository.servertest.ts+++ b/web/src/__tests__/server/repositories/experiment-items-repository.servertest.ts@@ -18,7 +18,7 @@ import { type FilterCondition } from "@langfuse/shared";* Wraps createEvent with experiment defaults.*/function createExperimentEvent(- params: Partial<EventRecordInsertType> & {+ params: Parameters<typeof createEvent>[0] & {experimentId: string;experimentName: string;datasetId?: string;diff --git a/web/src/__tests__/server/repositories/experiments-public-api-repository.servertest.ts b/web/src/__tests__/server/repositories/experiments-public-api-repository.servertest.tsindex 962756a0eb..c465143b56 100644--- a/web/src/__tests__/server/repositories/experiments-public-api-repository.servertest.ts+++ b/web/src/__tests__/server/repositories/experiments-public-api-repository.servertest.ts@@ -926,8 +926,8 @@ describe("Public API experiments repository", () => {it("preserves microsecond precision in experiment item cursors", async () => {const { projectId } = await createOrgProjectAndApiKey();const startTimeMs = Date.UTC(2026, 0, 1, 0, 0, 0, 123);- const firstStartTimeUs = startTimeMs * 1_000 + 456;- const secondStartTimeUs = startTimeMs * 1_000;+ const firstStartTime = "2026-01-01 00:00:00.123456";+ const secondStartTime = "2026-01-01 00:00:00.123000";await createEventsCh([{@@ -937,8 +937,8 @@ describe("Public API experiments repository", () => {startTimeMs,spanId: "span-cursor-first",}),- start_time: firstStartTimeUs,- event_ts: firstStartTimeUs,+ start_time: firstStartTime,+ event_ts: firstStartTime,},{...createExperimentRootEvent({@@ -947,8 +947,8 @@ describe("Public API experiments repository", () => {startTimeMs,spanId: "span-cursor-second",}),- start_time: secondStartTimeUs,- event_ts: secondStartTimeUs,+ start_time: secondStartTime,+ event_ts: secondStartTime,},]);diff --git a/web/src/__tests__/server/sessions-ui-table.servertest.ts b/web/src/__tests__/server/sessions-ui-table.servertest.tsindex 9a57d9e8c7..cc767c0227 100644--- a/web/src/__tests__/server/sessions-ui-table.servertest.ts+++ b/web/src/__tests__/server/sessions-ui-table.servertest.ts@@ -68,7 +68,7 @@ function asEventInsert(** One root event per trace (parent_span_id = ''), plus one event per* observation with trace-level fields denormalized.- * Timestamps are converted from ms to µs (* 1000).+ * DateTime64 fields are already ClickHouse datetime strings on insert types.*/function buildMatchingEvents(traces: TraceRecordInsertType[],@@ -101,15 +101,15 @@ function buildMatchingEvents(input: t.input ?? null,output: t.output ?? null,metadata: t.metadata ?? {},- start_time: t.timestamp * 1000,+ start_time: t.timestamp,end_time: null,cost_details: {},provided_cost_details: {},usage_details: {},provided_usage_details: {},- created_at: t.created_at * 1000,- updated_at: t.updated_at * 1000,- event_ts: t.event_ts * 1000,+ created_at: t.created_at,+ updated_at: t.updated_at,+ event_ts: t.event_ts,}),),);@@ -153,14 +153,12 @@ function buildMatchingEvents(tool_definitions: o.tool_definitions ?? {},tool_calls: o.tool_calls ?? [],tool_call_names: o.tool_call_names ?? [],- start_time: o.start_time * 1000,- end_time: o.end_time ? o.end_time * 1000 : null,- completion_start_time: o.completion_start_time- ? o.completion_start_time * 1000- : null,- created_at: o.created_at * 1000,- updated_at: o.updated_at * 1000,- event_ts: o.event_ts * 1000,+ start_time: o.start_time,+ end_time: o.end_time ?? null,+ completion_start_time: o.completion_start_time ?? null,+ created_at: o.created_at,+ updated_at: o.updated_at,+ event_ts: o.event_ts,}),),);diff --git a/web/src/__tests__/server/traces-api.servertest.ts b/web/src/__tests__/server/traces-api.servertest.tsindex 109f8e9200..13830646d5 100644--- a/web/src/__tests__/server/traces-api.servertest.ts+++ b/web/src/__tests__/server/traces-api.servertest.ts@@ -7,7 +7,6 @@ import {getTraceById,createEvent,createOrgProjectAndApiKey,- type EventRecordInsertType,createObservationsCh,createTracesCh,createEventsCh,@@ -50,17 +49,18 @@ type ObservationEventData = {total_cost?: number;};-// Helper to create observation/event data in the appropriate format-// Handles time conversion internally: milliseconds -> microseconds for events tableconst createObservationOrEvent = (useEventsTable: boolean,- data: ObservationEventData & Partial<EventRecordInsertType>,+ data: ObservationEventData & {+ environment?: string;+ user_id?: string | null;+ session_id?: string | null;+ trace_name?: string;+ },) => {const id = data.id ?? randomUUID();- const timeMultiplier = useEventsTable ? 1000 : 1; // microseconds vs millisecondsif (useEventsTable) {- // For events table: microseconds, requires span_idreturn createEvent({...data,id,@@ -68,13 +68,8 @@ const createObservationOrEvent = (parent_span_id: data.trace_id, // Observations have trace as parenttype: data.type ?? "SPAN",level: data.level ?? "DEFAULT",- start_time: data.start_time * timeMultiplier, // Convert ms to microseconds- end_time:- data.end_time === null- ? null- : data.end_time- ? data.end_time * timeMultiplier- : null,+ start_time: data.start_time,+ end_time: data.end_time ?? null,});}// For observations table: milliseconds@@ -157,7 +152,7 @@ const createTraceWithObservations = async (name: trace.name ?? "trace",trace_name: trace.name ?? "trace",type: "GENERATION", // Trace events are typically GENERATION type- start_time: trace.timestamp * 1000, // Convert ms to microseconds+ start_time: trace.timestamp,end_time: null,environment: trace.environment ?? "default",version: trace.version ?? null,diff --git a/worker/src/__tests__/batchDataRetentionCleaner.test.ts b/worker/src/__tests__/batchDataRetentionCleaner.test.tsindex 125259a216..fe5d712641 100644--- a/worker/src/__tests__/batchDataRetentionCleaner.test.ts+++ b/worker/src/__tests__/batchDataRetentionCleaner.test.ts@@ -16,6 +16,7 @@ import {createScoresCh,createTraceScore,queryClickhouse,+ toClickhouseDateTime,traceException,} from "@langfuse/shared/src/server";import { prisma } from "@langfuse/shared/src/db";@@ -167,8 +168,8 @@ async function insertRetentionTestRows(format: "JSONEachRow",values: rows.map((row) => ({project_id: row.projectId,- start_time: row.startTime,- event_ts: Date.now(),+ start_time: toClickhouseDateTime(row.startTime),+ event_ts: toClickhouseDateTime(),})),});}diff --git a/worker/src/__tests__/batchProjectBlobCleaner.test.ts b/worker/src/__tests__/batchProjectBlobCleaner.test.tsindex c8eb42fb0f..4d35ff5b77 100644--- a/worker/src/__tests__/batchProjectBlobCleaner.test.ts+++ b/worker/src/__tests__/batchProjectBlobCleaner.test.ts@@ -4,6 +4,7 @@ import {clickhouseClient,createOrgProjectAndApiKey,getDeletedProjects,+ toClickhouseDateTime,} from "@langfuse/shared/src/server";import { prisma } from "@langfuse/shared/src/db";import { BatchProjectBlobCleaner } from "../features/batch-project-blob-cleaner";@@ -34,9 +35,9 @@ async function insertBlobRefs(projectId: string, count: number): Promise<void> {event_id: randomUUID(),bucket_name: "test-bucket",bucket_path: `${projectId}/traces/${randomUUID()}.json`,- created_at: new Date().getTime(),- updated_at: new Date().getTime(),- event_ts: new Date().getTime(),+ created_at: toClickhouseDateTime(),+ updated_at: toClickhouseDateTime(),+ event_ts: toClickhouseDateTime(),is_deleted: 0,}));@@ -59,9 +60,9 @@ async function softDeleteBlobRefs(event_id: randomUUID(),bucket_name: "test-bucket",bucket_path: `${projectId}/traces/${randomUUID()}.json`,- created_at: new Date().getTime(),- updated_at: new Date().getTime(),- event_ts: new Date().getTime(),+ created_at: toClickhouseDateTime(),+ updated_at: toClickhouseDateTime(),+ event_ts: toClickhouseDateTime(),is_deleted: 1,}));diff --git a/worker/src/__tests__/dataRetentionProcessing.test.ts b/worker/src/__tests__/dataRetentionProcessing.test.tsindex 88535e9e94..3d822a2fbd 100644--- a/worker/src/__tests__/dataRetentionProcessing.test.ts+++ b/worker/src/__tests__/dataRetentionProcessing.test.ts@@ -15,6 +15,7 @@ import {getTraceById,StorageService,StorageServiceFactory,+ toClickhouseDateTime,} from "@langfuse/shared/src/server";import { prisma } from "@langfuse/shared/src/db";import { handleDataRetentionProcessingJob } from "../ee/dataRetention/handleDataRetentionProcessingJob";@@ -82,8 +83,12 @@ describe("DataRetentionProcessingJob", () => {event_id: randomUUID(),bucket_name: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,bucket_path: fileName,- created_at: new Date(Date.now() - 1000 * 60 * 60 * 24 * 3).getTime(), // 3 days in the past- updated_at: new Date(Date.now() - 1000 * 60 * 60 * 24 * 3).getTime(), // 3 days in the past+ created_at: toClickhouseDateTime(+ Date.now() - 1000 * 60 * 60 * 24 * 3,+ ), // 3 days in the past+ updated_at: toClickhouseDateTime(+ Date.now() - 1000 * 60 * 60 * 24 * 3,+ ), // 3 days in the past},],});@@ -141,8 +146,12 @@ describe("DataRetentionProcessingJob", () => {event_id: randomUUID(),bucket_name: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,bucket_path: fileName,- created_at: new Date(Date.now() - 1000 * 60 * 60 * 24 * 30).getTime(), // 30 days in the past- updated_at: new Date(Date.now() - 1000 * 60 * 60 * 24 * 30).getTime(), // 30 days in the past+ created_at: toClickhouseDateTime(+ Date.now() - 1000 * 60 * 60 * 24 * 30,+ ), // 30 days in the past+ updated_at: toClickhouseDateTime(+ Date.now() - 1000 * 60 * 60 * 24 * 30,+ ), // 30 days in the past},],});diff --git a/worker/src/__tests__/datetime64Insert.integration.test.ts b/worker/src/__tests__/datetime64Insert.integration.test.tsnew file mode 100644index 0000000000..57c6e4dfbf--- /dev/null+++ b/worker/src/__tests__/datetime64Insert.integration.test.ts@@ -0,0 +1,109 @@+import { afterAll, beforeAll, describe, expect, it } from "vitest";+import {+ clickhouseClient,+ toClickhouseDateTime,+} from "@langfuse/shared/src/server";++const TABLE_MS = "datetime64_insert_probe_ms";+const TICKS_MS = Date.UTC(2024, 10, 6, 20, 37, 0, 123);+const QUOTED_MS = "2024-11-06 20:37:00.123";++const isClickHouseAtLeast26_8 = (version: string): boolean => {+ const match = version.trim().match(/^v?(\d+)\.(\d+)/);+ if (!match) return false;+ const major = Number(match[1]);+ const minor = Number(match[2]);+ return major > 26 || (major === 26 && minor >= 8);+};++describe("DateTime64 JSONEachRow inserts", () => {+ let clickHouseVersion = "";+ let clickHouseAvailable = false;++ beforeAll(async () => {+ try {+ const result = await clickhouseClient().query({+ query: "SELECT version() AS version",+ format: "JSONEachRow",+ });+ const rows = await result.json<{ version: string }>();+ clickHouseVersion = rows[0]?.version ?? "";+ clickHouseAvailable = Boolean(clickHouseVersion);++ await clickhouseClient().command({+ query: `+ CREATE TABLE IF NOT EXISTS ${TABLE_MS} (+ id String,+ t DateTime64(3, 'UTC')+ ) ENGINE = Memory+ `,+ });+ } catch {+ clickHouseAvailable = false;+ }+ });++ afterAll(async () => {+ if (!clickHouseAvailable) return;+ await clickhouseClient().command({+ query: `DROP TABLE IF EXISTS ${TABLE_MS}`,+ });+ });++ const insertAndRead = async (id: string, value: number | string) => {+ await clickhouseClient().insert({+ table: TABLE_MS,+ format: "JSONEachRow",+ values: [{ id, t: value }],+ });++ const result = await clickhouseClient().query({+ query: `+ SELECT+ toYear(t) AS year,+ formatDateTime(t, '%F %T.%f', 'UTC') AS formatted+ FROM ${TABLE_MS}+ WHERE id = {id: String}+ `,+ query_params: { id },+ format: "JSONEachRow",+ });+ const rows = await result.json<{+ year: number | string;+ formatted: string;+ }>();+ return {+ year: Number(rows[0]?.year),+ formatted: rows[0]?.formatted ?? "",+ };+ };++ it("stores toClickhouseDateTime strings as the real event time", async (ctx) => {+ if (!clickHouseAvailable) {+ ctx.skip("ClickHouse is not available");+ }++ const quoted = toClickhouseDateTime(TICKS_MS);+ expect(quoted).toBe(QUOTED_MS);++ const stored = await insertAndRead("quoted", quoted);++ expect(stored.year).toBe(2024);+ expect(stored.formatted.startsWith(QUOTED_MS)).toBe(true);+ });++ it("overflows unquoted millisecond ticks on ClickHouse 26.8+ default parsing", async (ctx) => {+ if (!clickHouseAvailable) {+ ctx.skip("ClickHouse is not available");+ }+ if (!isClickHouseAtLeast26_8(clickHouseVersion)) {+ ctx.skip(+ `ClickHouse ${clickHouseVersion || "unknown"} is older than 26.8`,+ );+ }++ const stored = await insertAndRead("unquoted-default", TICKS_MS);++ expect(stored.year).toBeGreaterThanOrEqual(9000);+ });+});diff --git a/worker/src/__tests__/deletedMaskCleaner.test.ts b/worker/src/__tests__/deletedMaskCleaner.test.tsindex f78b1dcdac..ee2b637c69 100644--- a/worker/src/__tests__/deletedMaskCleaner.test.ts+++ b/worker/src/__tests__/deletedMaskCleaner.test.ts@@ -22,8 +22,6 @@ import {import {buildApplyDeletedMaskQuery,buildMutationCountQuery,- DELETED_MASK_CLEANER_TABLES,- DELETED_MASK_CLEANER_WORK_QUERY,normalizeMutationCounts,selectCandidateToProcess,shouldUseDeletedMaskCleanerClusterMode,@@ -34,6 +32,8 @@ import { skipUnlessClickhouseTablesExist } from "./helpers/clickhouseTables";const TEST_TABLE = "events_full" as const;const PATCH_WAIT_TIMEOUT_MS = 30_000;+const PATCH_CLEAN_TIMEOUT_MS = 60_000;+const DRAIN_TIMEOUT_MS = 120_000;async function skipUnlessEventsTableEnabled(ctx: TestContext): Promise<void> {await skipUnlessClickhouseTablesExist(@@ -53,8 +53,9 @@ async function eventually<T>(getValue: () => Promise<T>,predicate: (value: T) => boolean,message: string,+ timeoutMs: number = PATCH_WAIT_TIMEOUT_MS,): Promise<T> {- const deadline = Date.now() + PATCH_WAIT_TIMEOUT_MS;+ const deadline = Date.now() + timeoutMs;let lastValue: T | undefined;while (Date.now() < deadline) {@@ -95,26 +96,53 @@ function workCandidate(};}-async function hasCleanerCandidate(partitionToClean: string): Promise<boolean> {- const rows = await queryClickhouse<WorkCandidateRow>({- query: DELETED_MASK_CLEANER_WORK_QUERY,+async function getLeftoverPatchPartitions(): Promise<string[]> {+ const rows = await queryClickhouse<{ partition_to_clean: string }>({+ query: `+ SELECT DISTINCT splitByString('-', partition)[3] AS partition_to_clean+ FROM system.parts+ WHERE table = {table: String}+ AND database = {database: String}+ AND startsWith(partition, 'patch-')+ AND partition_to_clean <> ''+ AND partition_to_clean <> toString(toYYYYMM(now()))+ AND active = 1+ `,params: {+ table: TEST_TABLE,database: env.CLICKHOUSE_DB,- tables: Array.from(DELETED_MASK_CLEANER_TABLES),},});- return rows.some(- (row) =>- row.table === TEST_TABLE && row.partition_to_clean === partitionToClean,- );+ return rows.map((row) => row.partition_to_clean);+}++async function hasActivePatchPart(partitionToClean: string): Promise<boolean> {+ const rows = await queryClickhouse<{ count: string }>({+ query: `+ SELECT count() AS count+ FROM system.parts+ WHERE table = {table: String}+ AND database = {database: String}+ AND startsWith(partition, 'patch-')+ AND splitByString('-', partition)[3] = {partitionToClean: String}+ AND active = 1+ `,+ params: {+ table: TEST_TABLE,+ database: env.CLICKHOUSE_DB,+ partitionToClean,+ },+ });++ return Number(rows[0]?.count ?? 0) > 0;}async function waitForCleanerCandidate(partitionToClean: string,): Promise<void> {await eventually(- () => hasCleanerCandidate(partitionToClean),+ () => hasActivePatchPart(partitionToClean),(hasCandidate) => hasCandidate,`Timed out waiting for ClickHouse patch part ${partitionToClean}`,);@@ -124,9 +152,10 @@ async function waitForCleanerCandidateGone(partitionToClean: string,): Promise<void> {await eventually(- () => hasCleanerCandidate(partitionToClean),+ () => hasActivePatchPart(partitionToClean),(hasCandidate) => !hasCandidate,`Timed out waiting for ClickHouse patch part ${partitionToClean} to be cleaned`,+ PATCH_CLEAN_TIMEOUT_MS,);}@@ -152,44 +181,78 @@ async function getActiveMutationCount(return Number(rows[0]?.mutation_count ?? 0);}-async function waitForNoActiveMutations(): Promise<void> {+async function waitForNoActiveMutations(+ table: string = TEST_TABLE,+): Promise<void> {await eventually(- () => getActiveMutationCount(TEST_TABLE),+ () => getActiveMutationCount(table),(mutationCount) => mutationCount === 0,- "Timed out waiting for ClickHouse mutations to finish",+ `Timed out waiting for ClickHouse mutations to finish on ${table}`,+ PATCH_CLEAN_TIMEOUT_MS,);}+function cleanerDdlConfig() {+ return {+ database: env.CLICKHOUSE_DB,+ clusterEnabled: shouldUseDeletedMaskCleanerClusterMode({+ clusterEnabled: env.CLICKHOUSE_CLUSTER_ENABLED === "true",+ cleanerClusterModeEnabled:+ env.LANGFUSE_CLICKHOUSE_DELETED_MASK_CLEANER_CLUSTER_MODE_ENABLED ===+ "true",+ }),+ clusterName: env.CLICKHOUSE_CLUSTER_NAME,+ };+}++async function applyDeletedMask(candidate: WorkCandidateRow): Promise<void> {+ await commandClickhouse({+ query: buildApplyDeletedMaskQuery(candidate, cleanerDdlConfig()),+ clickhouseSettings: {+ mutations_sync: "2",+ },+ });++ // Mutation count can still be 0 before the ALTER is visible. The patch+ // disappearing from system.parts is the completion signal.+ await waitForCleanerCandidateGone(candidate.partition_to_clean);+ await waitForNoActiveMutations(candidate.table);+}+async function applyDeletedMaskIfNeeded(partitionToClean: string,): Promise<void> {- if (!(await hasCleanerCandidate(partitionToClean))) {+ if (!(await hasActivePatchPart(partitionToClean))) {return;}- await commandClickhouse({- query: buildApplyDeletedMaskQuery(- {- partition: `patch-cleanup-${partitionToClean}`,- table: TEST_TABLE,- partition_to_clean: partitionToClean,- total_rows: 0,- },- {- database: env.CLICKHOUSE_DB,- clusterEnabled: shouldUseDeletedMaskCleanerClusterMode({- clusterEnabled: env.CLICKHOUSE_CLUSTER_ENABLED === "true",- cleanerClusterModeEnabled:- env.LANGFUSE_CLICKHOUSE_DELETED_MASK_CLEANER_CLUSTER_MODE_ENABLED ===- "true",- }),- clusterName: env.CLICKHOUSE_CLUSTER_NAME,- },- ),+ await applyDeletedMask({+ partition: `patch-cleanup-${partitionToClean}`,+ table: TEST_TABLE,+ partition_to_clean: partitionToClean,+ total_rows: 0,});+}++async function drainCleanerCandidates(): Promise<void> {+ const deadline = Date.now() + DRAIN_TIMEOUT_MS;- await waitForNoActiveMutations();- await waitForCleanerCandidateGone(partitionToClean);+ while (Date.now() < deadline) {+ const partitions = await getLeftoverPatchPartitions();+ if (partitions.length === 0) {+ return;+ }++ for (const partition of partitions) {+ await applyDeletedMaskIfNeeded(partition);+ }+ }++ throw new Error(+ `Timed out draining leftover deleted-mask candidates: ${JSON.stringify(+ await getLeftoverPatchPartitions(),+ )}`,+ );}async function createEventPatchParts(@@ -209,11 +272,11 @@ async function createEventPatchParts(trace_id: randomUUID(),span_id: randomUUID(),name: `deleted-mask-cleaner-${partition}`,- start_time: timestamp * 1000,- end_time: timestamp * 1000,- created_at: timestamp * 1000,- updated_at: timestamp * 1000,- event_ts: timestamp * 1000,+ start_time: timestamp,+ end_time: timestamp,+ created_at: timestamp,+ updated_at: timestamp,+ event_ts: timestamp,}),),),@@ -390,7 +453,7 @@ describe.sequential("DeletedMaskCleaner integration", () => {await applyDeletedMaskIfNeeded(partition);}cleanupPartitions.clear();- }, 90_000);+ }, 150_000);it("applies a deleted mask for a real events patch partition", async (ctx) => {await skipUnlessEventsTableEnabled(ctx);@@ -398,6 +461,8 @@ describe.sequential("DeletedMaskCleaner integration", () => {const partitionToClean = getRandomPastMonthPartition();cleanupPartitions.add(partitionToClean);+ await drainCleanerCandidates();+await createEventPatchParts([{partition: partitionToClean,@@ -406,13 +471,19 @@ describe.sequential("DeletedMaskCleaner integration", () => {},]);- await expect(hasCleanerCandidate(partitionToClean)).resolves.toBe(true);+ await expect(hasActivePatchPart(partitionToClean)).resolves.toBe(true);const cleaner = new DeletedMaskCleaner();- await cleaner.processBatch();-- await waitForNoActiveMutations();- await waitForCleanerCandidateGone(partitionToClean);+ await eventually(+ async () => {+ await cleaner.processBatch();+ await waitForNoActiveMutations();+ return hasActivePatchPart(partitionToClean);+ },+ (stillPresent) => !stillPresent,+ `Timed out waiting for ClickHouse patch part ${partitionToClean} to be cleaned`,+ PATCH_CLEAN_TIMEOUT_MS,+ );expect(await redis?.get(DELETED_MASK_CLEANER_LOCK_KEY)).toBeNull();- }, 90_000);+ }, 240_000);});diff --git a/worker/src/__tests__/eventPropagationJob.test.ts b/worker/src/__tests__/eventPropagationJob.test.tsindex 45e848ad6b..a28db10d58 100644--- a/worker/src/__tests__/eventPropagationJob.test.ts+++ b/worker/src/__tests__/eventPropagationJob.test.ts@@ -16,6 +16,7 @@ import {createTracesCh,queryClickhouse,redis,+ toClickhouseDateTime,} from "@langfuse/shared/src/server";import { handleEventPropagationJob } from "../features/eventPropagation/handleEventPropagationJob";import { TableName } from "../services/ClickhouseWriter";@@ -107,7 +108,9 @@ describe("handleEventPropagationJob", () => {ingestion_api_key: "pk-lf-propagation-test",ingestion_sdk_name: "langfuse-js",ingestion_sdk_version: "4.2.0",- s3_first_seen_timestamp: oldEnoughForPropagation,+ s3_first_seen_timestamp: toClickhouseDateTime(+ oldEnoughForPropagation,+ ),},],});diff --git a/worker/src/__tests__/ingestionFileDeletion.test.ts b/worker/src/__tests__/ingestionFileDeletion.test.tsindex af8c02e99e..590519bd47 100644--- a/worker/src/__tests__/ingestionFileDeletion.test.ts+++ b/worker/src/__tests__/ingestionFileDeletion.test.ts@@ -8,6 +8,7 @@ import {removeIngestionEventsFromS3AndDeleteClickhouseRefsForProject,StorageService,StorageServiceFactory,+ toClickhouseDateTime,} from "@langfuse/shared/src/server";import { randomUUID } from "crypto";import { env } from "../env";@@ -73,8 +74,8 @@ describe("ingestion file deletion pipeline", () => {event_id: randomUUID(),bucket_name: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,bucket_path: r.bucketPath,- created_at: new Date().getTime(),- updated_at: new Date().getTime(),+ created_at: toClickhouseDateTime(),+ updated_at: toClickhouseDateTime(),})),});diff --git a/worker/src/__tests__/scoreDeletion.test.ts b/worker/src/__tests__/scoreDeletion.test.tsindex f2313f3d1f..f61da42784 100644--- a/worker/src/__tests__/scoreDeletion.test.ts+++ b/worker/src/__tests__/scoreDeletion.test.ts@@ -8,6 +8,7 @@ import {getScoresByIds,StorageService,StorageServiceFactory,+ toClickhouseDateTime,} from "@langfuse/shared/src/server";import { randomUUID } from "crypto";import { env } from "../env";@@ -74,8 +75,8 @@ describe("score deletion", () => {event_id: randomUUID(),bucket_name: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,bucket_path: `${projectId}/score/${scoreId}-score.json`,- created_at: new Date().getTime(),- updated_at: new Date().getTime(),+ created_at: toClickhouseDateTime(),+ updated_at: toClickhouseDateTime(),},],});diff --git a/worker/src/__tests__/traceDeletion.test.ts b/worker/src/__tests__/traceDeletion.test.tsindex c9d7eae52c..ca2b96d67f 100644--- a/worker/src/__tests__/traceDeletion.test.ts+++ b/worker/src/__tests__/traceDeletion.test.ts@@ -19,6 +19,7 @@ import {removeIngestionEventsFromS3AndDeleteClickhouseRefsForTraces,StorageService,StorageServiceFactory,+ toClickhouseDateTime,} from "@langfuse/shared/src/server";import { randomUUID } from "crypto";import { processClickhouseTraceDelete } from "../features/traces/processClickhouseTraceDelete";@@ -439,8 +440,8 @@ describe("trace deletion", () => {event_id: randomUUID(),bucket_name: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,bucket_path: `${projectId}/traces/${traceId}-trace.json`,- created_at: new Date().getTime(),- updated_at: new Date().getTime(),+ created_at: toClickhouseDateTime(),+ updated_at: toClickhouseDateTime(),},{id: randomUUID(),@@ -450,8 +451,8 @@ describe("trace deletion", () => {event_id: randomUUID(),bucket_name: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,bucket_path: `${projectId}/observation/${traceId}-observation.json`,- created_at: new Date().getTime(),- updated_at: new Date().getTime(),+ created_at: toClickhouseDateTime(),+ updated_at: toClickhouseDateTime(),},{id: randomUUID(),@@ -461,8 +462,8 @@ describe("trace deletion", () => {event_id: randomUUID(),bucket_name: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,bucket_path: `${projectId}/score/${traceId}-score.json`,- created_at: new Date().getTime(),- updated_at: new Date().getTime(),+ created_at: toClickhouseDateTime(),+ updated_at: toClickhouseDateTime(),},],});@@ -509,8 +510,8 @@ describe("trace deletion", () => {event_id: randomUUID(),bucket_name: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,bucket_path: bucketPath,- created_at: new Date().getTime(),- updated_at: new Date().getTime(),+ created_at: toClickhouseDateTime(),+ updated_at: toClickhouseDateTime(),},],});@@ -594,8 +595,8 @@ describe("trace deletion", () => {event_id: randomUUID(),bucket_name: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,bucket_path: bucketPath,- created_at: new Date().getTime(),- updated_at: new Date().getTime(),+ created_at: toClickhouseDateTime(),+ updated_at: toClickhouseDateTime(),},],});@@ -675,8 +676,8 @@ describe("trace deletion", () => {event_id: randomUUID(),bucket_name: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,bucket_path: `${projectId}/traces/${traceId}-trace.json`,- created_at: new Date().getTime(),- updated_at: new Date().getTime(),+ created_at: toClickhouseDateTime(),+ updated_at: toClickhouseDateTime(),},{id: randomUUID(),@@ -686,8 +687,8 @@ describe("trace deletion", () => {event_id: randomUUID(),bucket_name: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,bucket_path: `${projectId}/observation/${observationId}-observation.json`,- created_at: new Date().getTime(),- updated_at: new Date().getTime(),+ created_at: toClickhouseDateTime(),+ updated_at: toClickhouseDateTime(),},{id: randomUUID(),@@ -697,8 +698,8 @@ describe("trace deletion", () => {event_id: randomUUID(),bucket_name: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,bucket_path: `${projectId}/score/${scoreId}-score.json`,- created_at: new Date().getTime(),- updated_at: new Date().getTime(),+ created_at: toClickhouseDateTime(),+ updated_at: toClickhouseDateTime(),},],});diff --git a/worker/src/features/eventPropagation/handleExperimentBackfill.ts b/worker/src/features/eventPropagation/handleExperimentBackfill.tsindex c197d70377..6c187c6147 100644--- a/worker/src/features/eventPropagation/handleExperimentBackfill.ts+++ b/worker/src/features/eventPropagation/handleExperimentBackfill.ts@@ -3,6 +3,7 @@ import {queryClickhouse,redis,convertDateToClickhouseDateTime,+ toClickhouseDateTime,flattenJsonToPathArrays,recordGauge,UNKNOWN_INGESTION_SDK_VALUE,@@ -536,7 +537,7 @@ export function convertEnrichedSpansToEventRecords(opts: { plainEventTsFromSource?: boolean } = {},): EventRecordInsertType[] {const records: EventRecordInsertType[] = [];- const now = Date.now() * 1000; // microseconds+ const now = toClickhouseDateTime();for (const span of spans) {// Flatten metadata for ClickHouse Array(String) columns@@ -557,7 +558,7 @@ export function convertEnrichedSpansToEventRecords(const isPlain = !span.experiment_id;const eventTs =opts.plainEventTsFromSource && isPlain && span.event_ts- ? new Date(span.event_ts).getTime() * 1000+ ? toClickhouseDateTime(span.event_ts): now;const eventRecord: EventRecordInsertType = {@@ -585,10 +586,10 @@ export function convertEnrichedSpansToEventRecords(level: span.level || "DEFAULT",status_message: span.status_message || undefined,- start_time: new Date(span.start_time).getTime() * 1000,- end_time: span.end_time ? new Date(span.end_time).getTime() * 1000 : null,+ start_time: toClickhouseDateTime(span.start_time),+ end_time: span.end_time ? toClickhouseDateTime(span.end_time) : null,completion_start_time: span.completion_start_time- ? new Date(span.completion_start_time).getTime() * 1000+ ? toClickhouseDateTime(span.completion_start_time): null,prompt_id: span.prompt_id || "",diff --git a/worker/src/queues/ingestionQueue.ts b/worker/src/queues/ingestionQueue.tsindex 5d010b21fa..d40d2193de 100644--- a/worker/src/queues/ingestionQueue.ts+++ b/worker/src/queues/ingestionQueue.ts@@ -21,6 +21,7 @@ import {traceException,type IngestionAttribution,UNKNOWN_INGESTION_SDK_VALUE,+ toClickhouseDateTime,} from "@langfuse/shared/src/server";import { prisma } from "@langfuse/shared/src/db";@@ -306,9 +307,9 @@ export const ingestionQueueProcessorBuilder = (event_id: job.data.payload.data.fileKey,bucket_name: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,bucket_path: `${bucketPrefix}${fileName}`,- created_at: new Date().getTime(),- updated_at: new Date().getTime(),- event_ts: new Date().getTime(),+ created_at: toClickhouseDateTime(),+ updated_at: toClickhouseDateTime(),+ event_ts: toClickhouseDateTime(),is_deleted: 0,});}diff --git a/worker/src/services/IngestionService/index.ts b/worker/src/services/IngestionService/index.tsindex 7900f8dab2..5b53adc40c 100644--- a/worker/src/services/IngestionService/index.ts+++ b/worker/src/services/IngestionService/index.ts@@ -14,6 +14,8 @@ import {import {ClickhouseClientType,convertDateToClickhouseDateTime,+ toClickhouseDateTime,+ parseClickhouseUTCDateTimeFormat,convertObservationReadToInsert,convertScoreReadToInsert,convertTraceReadToInsert,@@ -355,7 +357,7 @@ export class IngestionService {: null,]);- const now = this.getMicrosecondTimestamp();+ const now = toClickhouseDateTime();// Flatten raw metadata first (before stringification destroys nested structure)const flattened = flattenJsonToPathArrays(metadata);@@ -395,10 +397,10 @@ export class IngestionService {status_message: eventData.statusMessage,// Timestamps- start_time: this.getMicrosecondTimestamp(eventData.startTimeISO),- end_time: this.getMicrosecondTimestamp(eventData.endTimeISO),+ start_time: toClickhouseDateTime(eventData.startTimeISO),+ end_time: toClickhouseDateTime(eventData.endTimeISO),completion_start_time: eventData.completionStartTime- ? this.getMicrosecondTimestamp(eventData.completionStartTime)+ ? toClickhouseDateTime(eventData.completionStartTime): null,// Prompt@@ -552,12 +554,10 @@ export class IngestionService {if (!runData || !itemData) return [];- const timestamp = event.body.createdAt- ? new Date(event.body.createdAt).getTime()- : new Date().getTime();+ const timestamp = toClickhouseDateTime(event.body.createdAt);const datasetItemVersion = itemData.validFrom- ? itemData.validFrom.getTime()+ ? toClickhouseDateTime(itemData.validFrom): null;return [@@ -580,7 +580,7 @@ export class IngestionService {dataset_run_metadata: runData.metadata? convertPostgresJsonToMetadataRecord(runData.metadata): {},- dataset_run_created_at: runData.createdAt.getTime(),+ dataset_run_created_at: toClickhouseDateTime(runData.createdAt),// enriched with item datadataset_item_version: datasetItemVersion,dataset_item_input: JSON.stringify(itemData.input),@@ -666,7 +666,7 @@ export class IngestionService {id: entityId,project_id: projectId,environment: validatedScore.environment,- timestamp: this.getMillisecondTimestamp(scoreEvent.timestamp),+ timestamp: toClickhouseDateTime(scoreEvent.timestamp),name: validatedScore.name,value: validatedScore.value,source: validatedScore.source,@@ -687,9 +687,9 @@ export class IngestionService {ingestion_api_key: attribution.ingestionApiKey,ingestion_sdk_name: attribution.ingestionSdkName,ingestion_sdk_version: attribution.ingestionSdkVersion,- created_at: Date.now(),- updated_at: Date.now(),- event_ts: new Date(scoreEvent.timestamp).getTime(),+ created_at: toClickhouseDateTime(),+ updated_at: toClickhouseDateTime(),+ event_ts: toClickhouseDateTime(scoreEvent.timestamp),is_deleted: 0,};// Gracefully handle any score schema validation errors, skip the score insert and reject silently.@@ -768,7 +768,8 @@ export class IngestionService {scoreRecords,});finalScoreRecord.created_at =- clickhouseScoreRecord?.created_at ?? createdAtTimestamp.getTime();+ clickhouseScoreRecord?.created_at ??+ toClickhouseDateTime(createdAtTimestamp);this.clickHouseWriter.addToQueue(TableName.Scores, finalScoreRecord);}@@ -845,7 +846,8 @@ export class IngestionService {traceRecords,});finalTraceRecord.created_at =- clickhouseTraceRecord?.created_at ?? createdAtTimestamp.getTime();+ clickhouseTraceRecord?.created_at ??+ toClickhouseDateTime(createdAtTimestamp);finalTraceRecord.input = finalIO.input ?? clickhouseTraceRecord?.input;finalTraceRecord.output = finalIO.output ?? clickhouseTraceRecord?.output;@@ -912,7 +914,9 @@ export class IngestionService {payload: {projectId,traceId: entityId,- exactTimestamp: new Date(finalTraceRecord.timestamp),+ exactTimestamp: parseClickhouseUTCDateTimeFormat(+ finalTraceRecord.timestamp,+ ),traceEnvironment: finalTraceRecord.environment,},id: randomUUID(),@@ -989,7 +993,8 @@ export class IngestionService {clickhouseObservationRecord,});mergedObservationRecord.created_at =- clickhouseObservationRecord?.created_at ?? createdAtTimestamp.getTime();+ clickhouseObservationRecord?.created_at ??+ toClickhouseDateTime(createdAtTimestamp);mergedObservationRecord.level = mergedObservationRecord.level ?? "DEFAULT";// Search for the first non-null input and output in the observation events and set them on the merged result.@@ -1075,13 +1080,13 @@ export class IngestionService {timestamp: finalObservationRecord.start_time,project_id: projectId,environment: finalObservationRecord.environment,- created_at: Date.now(),- updated_at: Date.now(),+ created_at: toClickhouseDateTime(),+ updated_at: toClickhouseDateTime(),metadata: {},tags: [],bookmarked: false,public: false,- event_ts: Date.now(),+ event_ts: toClickhouseDateTime(),is_deleted: 0,};@@ -1220,7 +1225,7 @@ export class IngestionService {result = overwriteObject(result, record, immutableEntityKeys);}- result.event_ts = new Date().getTime();+ result.event_ts = toClickhouseDateTime();return result;}@@ -1842,7 +1847,7 @@ export class IngestionService {return traceEventList.map((trace) => {const traceRecord: TraceRecordInsertType = {id: entityId,- timestamp: this.getMillisecondTimestamp(+ timestamp: toClickhouseDateTime(trace.body.timestamp ?? trace.timestamp,),// timestamp: ("timestamp" in trace.body && trace.body.timestamp@@ -1865,9 +1870,9 @@ export class IngestionService {// input: this.stringify(trace.body.input),// output: this.stringify(trace.body.output), // convert even json to stringsession_id: trace.body.sessionId,- created_at: Date.now(),- updated_at: Date.now(),- event_ts: new Date(trace.timestamp).getTime(),+ created_at: toClickhouseDateTime(),+ updated_at: toClickhouseDateTime(),+ event_ts: toClickhouseDateTime(trace.timestamp),is_deleted: 0,};@@ -1990,19 +1995,17 @@ export class IngestionService {name: obs.body.name,environment:"environment" in obs.body ? obs.body.environment : "default",- start_time: this.getMillisecondTimestamp(- obs.body.startTime ?? obs.timestamp,- ),+ start_time: toClickhouseDateTime(obs.body.startTime ?? obs.timestamp),// start_time: ("startTime" in obs.body && obs.body.startTime// ? this.getMillisecondTimestamp(obs.body.startTime)// : undefined) as number, // Casting here is dirty, but our requirement is to have a start_time _after_ the mergeend_time:"endTime" in obs.body && obs.body.endTime- ? this.getMillisecondTimestamp(obs.body.endTime)+ ? toClickhouseDateTime(obs.body.endTime): undefined,completion_start_time:"completionStartTime" in obs.body && obs.body.completionStartTime- ? this.getMillisecondTimestamp(obs.body.completionStartTime)+ ? toClickhouseDateTime(obs.body.completionStartTime): undefined,metadata: obs.body.metadata? convertJsonSchemaToRecord(obs.body.metadata)@@ -2030,9 +2033,9 @@ export class IngestionService {prompt_id: prompt?.id,prompt_name: prompt?.name,prompt_version: prompt?.version,- created_at: Date.now(),- updated_at: Date.now(),- event_ts: new Date(obs.timestamp).getTime(),+ created_at: toClickhouseDateTime(),+ updated_at: toClickhouseDateTime(),+ event_ts: toClickhouseDateTime(obs.timestamp),is_deleted: 0,};@@ -2048,14 +2051,6 @@ export class IngestionService {return typeof obj === "string" ? obj : JSON.stringify(obj);}- private getMicrosecondTimestamp(timestamp?: string | null): number {- return timestamp ? new Date(timestamp).getTime() * 1000 : Date.now() * 1000;- }-- private getMillisecondTimestamp(timestamp?: string | null): number {- return timestamp ? new Date(timestamp).getTime() : Date.now();- }-/*** Returns a partition-aware timestamp for staging table writes.* If the createdAtTimestamp is within the last 2 minutes, returns it as-is.@@ -2069,7 +2064,7 @@ export class IngestionService {* that data is processed correctly. Worst case is slightly more duplication in the events table* which should resolve automatically using the ReplacingMergeTree.*/- private getPartitionAwareTimestamp(createdAtTimestamp: Date): number {+ private getPartitionAwareTimestamp(createdAtTimestamp: Date): string {const now = Date.now();const createdAt = createdAtTimestamp.getTime();const ageInMs = now - createdAt;@@ -2077,7 +2072,7 @@ export class IngestionService {// If the createdAtTimestamp is within the last 2 minutes, use it// Otherwise, use the current timestamp to avoid updating old partitions- return ageInMs < twoMinutesInMs ? createdAt : now;+ return toClickhouseDateTime(ageInMs < twoMinutesInMs ? createdAt : now);}}diff --git a/worker/src/services/IngestionService/tests/IngestionService.integration.test.ts b/worker/src/services/IngestionService/tests/IngestionService.integration.test.tsindex c11d261405..03e7c9ea2f 100644--- a/worker/src/services/IngestionService/tests/IngestionService.integration.test.ts+++ b/worker/src/services/IngestionService/tests/IngestionService.integration.test.ts@@ -139,7 +139,7 @@ describe("Ingestion end-to-end tests", () => {const secondSpanId = randomUUID();const wireStartTimeA = "2026-01-01T12:00:00.123456Z";const wireStartTimeB = "2026-01-01T12:00:00.123789Z";- const normalizedStartTime = Date.parse(wireStartTimeA) * 1000;+ const normalizedStartTime = "2026-01-01 12:00:00.123";const eventRecords = await Promise.all([@@ -169,9 +169,6 @@ describe("Ingestion end-to-end tests", () => {normalizedStartTime,normalizedStartTime,]);- expect(eventRecords.every((record) => record.start_time % 1000 === 0)).toBe(- true,- );await Promise.all(eventRecords.map((record) => ingestionService.writeEventRecord(record)),diff --git a/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts b/worker/src/services/IngestionService/tests/IngestionService.unit.test.tsindex 2b59a9512c..4e5811832e 100644--- a/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts+++ b/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts@@ -2,6 +2,9 @@ import { beforeEach, expect, describe, it, vi } from "vitest";const mocks = vi.hoisted(() => ({applyObservationFieldOverflow: vi.fn(),+ validateAndInflateScoreOverride: undefined as+ | ((...args: unknown[]) => unknown)+ | undefined,}));vi.mock(@@ -11,6 +14,20 @@ vi.mock(}),);+vi.mock("@langfuse/shared/src/server", async (importOriginal) => {+ const actual =+ await importOriginal<typeof import("@langfuse/shared/src/server")>();+ return {+ ...actual,+ validateAndInflateScore: (...args: unknown[]) =>+ mocks.validateAndInflateScoreOverride+ ? mocks.validateAndInflateScoreOverride(...args)+ : (actual.validateAndInflateScore as (...a: unknown[]) => unknown)(+ ...args,+ ),+ };+});+import { IngestionService } from "../../IngestionService";import {convertDateToClickhouseDateTime,@@ -26,6 +43,7 @@ describe("IngestionService unit tests", () => {mocks.applyObservationFieldOverflow.mockImplementation(async (eventRecord) => eventRecord,);+ mocks.validateAndInflateScoreOverride = undefined;});it("writes the final serialized event size instead of the raw OTEL span size", async () => {@@ -802,12 +820,9 @@ describe("IngestionService unit tests", () => {vi.spyOn(ingestionService as any, "getClickhouseRecord").mockResolvedValue(null,);- vi.spyOn(- ingestionService as any,- "getMillisecondTimestamp",- ).mockImplementation(() => {- throw new Error("unexpected timestamp failure");- });+ mocks.validateAndInflateScoreOverride = () => {+ throw new Error("unexpected score validation failure");+ };await expect((ingestionService as any).processScoreEventList({@@ -860,12 +875,9 @@ describe("IngestionService unit tests", () => {timestamp: new Date(timestamp).getTime(),}),);- vi.spyOn(- ingestionService as any,- "getMillisecondTimestamp",- ).mockImplementation(() => {- throw new Error("unexpected timestamp failure");- });+ mocks.validateAndInflateScoreOverride = () => {+ throw new Error("unexpected score validation failure");+ };await expect((ingestionService as any).processScoreEventList({diff --git a/worker/src/services/IngestionService/tests/scoreValidationDroppedMetric.unit.test.ts b/worker/src/services/IngestionService/tests/scoreValidationDroppedMetric.unit.test.tsindex 6bdce7a235..01732397ea 100644--- a/worker/src/services/IngestionService/tests/scoreValidationDroppedMetric.unit.test.ts+++ b/worker/src/services/IngestionService/tests/scoreValidationDroppedMetric.unit.test.ts@@ -198,12 +198,9 @@ describe("score validation drop metric", () => {it("does not emit for unexpected errors — those reject the batch instead of dropping it", async () => {const { ingestionService, addToQueue } = createService();- vi.spyOn(- ingestionService as any,- "getMillisecondTimestamp",- ).mockImplementation(() => {- throw new Error("unexpected timestamp failure");- });+ mocks.validateAndInflateScoreOverride = () => {+ throw new Error("unexpected score validation failure");+ };await expect(processScores(ingestionService, [validScoreEvent()]),