Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(clickhouse): stringify DateTime64 timestamps on insert (#16892)

Steffen Schmitzd812e3afd34c
diff --git a/docker-compose.dev-redis-cluster.yml b/docker-compose.dev-redis-cluster.yml
index c8561a6fbd..46335ee741 100644
--- a/docker-compose.dev-redis-cluster.yml
+++ b/docker-compose.dev-redis-cluster.yml
@@ -12,7 +12,9 @@ x-redis-cluster-healthcheck: &redis-cluster-healthcheck
services:
clickhouse:
- image: docker.io/clickhouse/clickhouse-server:25.12
+ # Latest ClickHouse in the CI matrix. Used to catch parser/settings
+ # regressions such as DateTime64 unquoted-number scaling on 26.8+.
+ image: docker.io/clickhouse/clickhouse-server:26.8
user: "101:101"
environment:
CLICKHOUSE_DB: default
diff --git a/packages/shared/scripts/seeder/scenarios/event-mirror.ts b/packages/shared/scripts/seeder/scenarios/event-mirror.ts
index 43ed53f3a8..809f226ebe 100644
--- a/packages/shared/scripts/seeder/scenarios/event-mirror.ts
+++ b/packages/shared/scripts/seeder/scenarios/event-mirror.ts
@@ -5,9 +5,6 @@ import {
TraceRecordInsertType,
} from "../../../src/server";
-const toMicros = (ms: number | null | undefined): number | null =>
- ms === null || ms === undefined ? null : ms * 1000;
-
const utf8Bytes = (value: unknown): number =>
typeof value === "string" ? Buffer.byteLength(value, "utf8") : 0;
@@ -66,12 +63,12 @@ export const traceToEvent = (
prompt_version: null,
metadata_names: names,
metadata_values: values,
- start_time: toMicros(trace.timestamp) ?? Date.now() * 1000,
+ start_time: trace.timestamp,
end_time: null,
completion_start_time: null,
- created_at: toMicros(trace.created_at) ?? Date.now() * 1000,
- updated_at: toMicros(trace.updated_at) ?? Date.now() * 1000,
- event_ts: toMicros(trace.event_ts) ?? Date.now() * 1000,
+ created_at: trace.created_at,
+ updated_at: trace.updated_at,
+ event_ts: trace.event_ts,
event_bytes: utf8Bytes(trace.input) + utf8Bytes(trace.output),
source: "API",
});
@@ -131,12 +128,12 @@ export const observationToEvent = (
tool_call_names: observation.tool_call_names ?? [],
metadata_names: names,
metadata_values: values,
- start_time: toMicros(observation.start_time) ?? Date.now() * 1000,
- end_time: toMicros(observation.end_time),
- completion_start_time: toMicros(observation.completion_start_time),
- created_at: toMicros(observation.created_at) ?? Date.now() * 1000,
- updated_at: toMicros(observation.updated_at) ?? Date.now() * 1000,
- event_ts: toMicros(observation.event_ts) ?? Date.now() * 1000,
+ start_time: observation.start_time,
+ end_time: observation.end_time,
+ completion_start_time: observation.completion_start_time,
+ created_at: observation.created_at,
+ updated_at: observation.updated_at,
+ event_ts: observation.event_ts,
event_bytes: utf8Bytes(observation.input) + utf8Bytes(observation.output),
source: "API",
});
diff --git a/packages/shared/scripts/seeder/scenarios/long-session.ts b/packages/shared/scripts/seeder/scenarios/long-session.ts
index ef5fcdf678..1c3e2a1959 100644
--- a/packages/shared/scripts/seeder/scenarios/long-session.ts
+++ b/packages/shared/scripts/seeder/scenarios/long-session.ts
@@ -8,6 +8,7 @@ import {
createObservationsCh,
createScoresCh,
createEventsCh,
+ toClickhouseDateTime,
EventRecordInsertType,
ObservationRecordInsertType,
ScoreRecordInsertType,
@@ -258,7 +259,9 @@ const run = async (
// The root "session-turn" span must cover its children, otherwise the
// waterfall shows children running after their parent already ended.
if (observationsPerTrace > 1) {
- observations[rootObservationIndex].end_time = maxChildEndTime + 25;
+ observations[rootObservationIndex].end_time = toClickhouseDateTime(
+ maxChildEndTime + 25,
+ );
}
if (t % 3 === 0) {
@@ -335,7 +338,11 @@ const run = async (
const links = [
sessionLink(ctx, sessionId),
- traceLink(ctx, traces[0].id, traces[0].timestamp as number),
+ traceLink(
+ ctx,
+ traces[0].id,
+ Date.parse(traces[0].timestamp.replace(" ", "T") + "Z"),
+ ),
];
// The session detail page 404s without the Postgres trace_sessions row.
diff --git a/packages/shared/scripts/seeder/utils/data-generators.ts b/packages/shared/scripts/seeder/utils/data-generators.ts
index 39e384f633..db7cadc2ea 100644
--- a/packages/shared/scripts/seeder/utils/data-generators.ts
+++ b/packages/shared/scripts/seeder/utils/data-generators.ts
@@ -36,6 +36,7 @@ import {
createDatasetRunItem,
createDatasetRunScore,
UNKNOWN_INGESTION_SDK_VALUE,
+ toClickhouseDateTime,
} from "../../../src/server";
/**
@@ -1106,7 +1107,7 @@ export class DataGenerator {
const now = Date.now();
const traces: TraceRecordInsertType[] = dialogues.map((d, index) => ({
id: `support-chat-${index}-${projectId.slice(-8)}`,
- timestamp: now + index * 1000,
+ timestamp: toClickhouseDateTime(now + index * 1000),
name: "SupportChatSession",
user_id: null,
metadata: this.buildNestedSeedMetadata("support-chat", index, {
@@ -1137,9 +1138,9 @@ export class DataGenerator {
),
output: JSON.stringify({ role: "assistant", content: d.assistant }),
session_id: "support-chat-session",
- created_at: now + index * 1000,
- updated_at: now + index * 1000 + 500,
- event_ts: now + index * 1000,
+ created_at: toClickhouseDateTime(now + index * 1000),
+ updated_at: toClickhouseDateTime(now + index * 1000 + 500),
+ event_ts: toClickhouseDateTime(now + index * 1000),
is_deleted: 0,
}));
@@ -1159,8 +1160,8 @@ export class DataGenerator {
type: "GENERATION",
parent_observation_id: null,
environment: "default",
- start_time: start,
- end_time: end,
+ start_time: toClickhouseDateTime(start),
+ end_time: toClickhouseDateTime(end),
name: "llm-generation",
metadata: {},
level: "DEFAULT",
@@ -1203,13 +1204,13 @@ export class DataGenerator {
total: Math.round(totalTokens * 5) / 1_000_000,
},
total_cost: Math.round(totalTokens * 5) / 1_000_000,
- completion_start_time: start + 120,
+ completion_start_time: toClickhouseDateTime(start + 120),
prompt_id: null,
prompt_name: null,
prompt_version: null,
- created_at: start,
- updated_at: end,
- event_ts: start,
+ created_at: toClickhouseDateTime(start),
+ updated_at: toClickhouseDateTime(end),
+ event_ts: toClickhouseDateTime(start),
is_deleted: 0,
tool_definitions: d.tool
? {
@@ -1232,8 +1233,8 @@ export class DataGenerator {
type: "TOOL",
parent_observation_id: null,
environment: "default",
- start_time: start - 40,
- end_time: start - 5,
+ start_time: toClickhouseDateTime(start - 40),
+ end_time: toClickhouseDateTime(start - 5),
name: d.tool.name,
metadata: {},
level: "DEFAULT",
@@ -1253,9 +1254,9 @@ export class DataGenerator {
prompt_id: null,
prompt_name: null,
prompt_version: null,
- created_at: start - 40,
- updated_at: start - 5,
- event_ts: start - 40,
+ created_at: toClickhouseDateTime(start - 40),
+ updated_at: toClickhouseDateTime(start - 5),
+ event_ts: toClickhouseDateTime(start - 40),
is_deleted: 0,
tool_definitions: undefined,
tool_calls: undefined,
@@ -1289,10 +1290,10 @@ export class DataGenerator {
long_string_value: "",
queue_id: null,
...this.buildSeedIngestionAttribution(projectId, "javascript-sdk"),
- created_at: baseTs,
- updated_at: baseTs,
- timestamp: baseTs,
- event_ts: baseTs,
+ created_at: toClickhouseDateTime(baseTs),
+ updated_at: toClickhouseDateTime(baseTs),
+ timestamp: toClickhouseDateTime(baseTs),
+ event_ts: toClickhouseDateTime(baseTs),
is_deleted: 0,
};
@@ -1317,10 +1318,10 @@ export class DataGenerator {
string_value: safeVal === 1 ? "True" : "False",
queue_id: null,
...this.buildSeedIngestionAttribution(projectId, "python-sdk"),
- created_at: baseTs + 10,
- updated_at: baseTs + 10,
- timestamp: baseTs + 10,
- event_ts: baseTs + 10,
+ created_at: toClickhouseDateTime(baseTs + 10),
+ updated_at: toClickhouseDateTime(baseTs + 10),
+ timestamp: toClickhouseDateTime(baseTs + 10),
+ event_ts: toClickhouseDateTime(baseTs + 10),
is_deleted: 0,
};
@@ -1347,10 +1348,10 @@ export class DataGenerator {
string_value: "True",
queue_id: null,
...this.buildSeedIngestionAttribution(projectId, "raw-api"),
- created_at: baseTs + 20,
- updated_at: baseTs + 20,
- timestamp: baseTs + 20,
- event_ts: baseTs + 20,
+ created_at: toClickhouseDateTime(baseTs + 20),
+ updated_at: toClickhouseDateTime(baseTs + 20),
+ timestamp: toClickhouseDateTime(baseTs + 20),
+ event_ts: toClickhouseDateTime(baseTs + 20),
is_deleted: 0,
}
: null;
diff --git a/packages/shared/src/env.ts b/packages/shared/src/env.ts
index cf051dbd38..61f1b6bc67 100644
--- a/packages/shared/src/env.ts
+++ b/packages/shared/src/env.ts
@@ -145,9 +145,9 @@ const EnvSchema = z.object({
CLICKHOUSE_UPDATE_PARALLEL_MODE: z
.enum(["sync", "async", "auto"])
.default("auto"),
- // Workaround for ClickHouse analyzer/lazy materialization bugs. In "auto",
- // Langfuse detects the ClickHouse version on startup and applies known
- // compatibility settings for affected version bands.
+ // Workarounds for ClickHouse version-specific parser and analyzer bugs. In
+ // "auto", Langfuse detects the ClickHouse version on startup and applies
+ // known compatibility settings for affected version bands.
CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION: z
.enum(["auto", "true", "false"])
.default("auto"),
diff --git a/packages/shared/src/server/clickhouse/client.datetime.test.ts b/packages/shared/src/server/clickhouse/client.datetime.test.ts
new file mode 100644
index 0000000000..77ce2dcce0
--- /dev/null
+++ b/packages/shared/src/server/clickhouse/client.datetime.test.ts
@@ -0,0 +1,47 @@
+import { describe, expect, it } from "vitest";
+import {
+ convertDateToClickhouseDateTime,
+ toClickhouseDateTime,
+} from "./client";
+import { parseClickhouseUTCDateTimeFormat } from "../repositories/clickhouse";
+
+describe("toClickhouseDateTime", () => {
+ const ticks = Date.UTC(2024, 10, 6, 20, 37, 0, 123);
+ const quoted = "2024-11-06 20:37:00.123";
+
+ it("formats Date values", () => {
+ expect(toClickhouseDateTime(new Date(ticks))).toBe(quoted);
+ });
+
+ it("formats unix millisecond timestamps", () => {
+ expect(toClickhouseDateTime(ticks)).toBe(quoted);
+ });
+
+ it("formats ISO strings", () => {
+ expect(toClickhouseDateTime("2024-11-06T20:37:00.123Z")).toBe(quoted);
+ });
+
+ it("passes through ClickHouse datetime strings", () => {
+ expect(toClickhouseDateTime(quoted)).toBe(quoted);
+ });
+
+ it("matches convertDateToClickhouseDateTime for Date inputs", () => {
+ const date = new Date(ticks);
+ expect(toClickhouseDateTime(date)).toBe(
+ convertDateToClickhouseDateTime(date),
+ );
+ });
+});
+
+describe("parseClickhouseUTCDateTimeFormat", () => {
+ it("reads DateTime64 insert strings back as the same UTC instant", () => {
+ const ticks = Date.UTC(2024, 10, 6, 20, 37, 0, 123);
+ const quoted = toClickhouseDateTime(ticks);
+
+ expect(quoted).toBe("2024-11-06 20:37:00.123");
+ expect(parseClickhouseUTCDateTimeFormat(quoted).toISOString()).toBe(
+ "2024-11-06T20:37:00.123Z",
+ );
+ expect(parseClickhouseUTCDateTimeFormat(quoted).getTime()).toBe(ticks);
+ });
+});
diff --git a/packages/shared/src/server/clickhouse/client.ts b/packages/shared/src/server/clickhouse/client.ts
index ad81a02918..e3d00b3c35 100644
--- a/packages/shared/src/server/clickhouse/client.ts
+++ b/packages/shared/src/server/clickhouse/client.ts
@@ -268,3 +268,27 @@ export const convertDateToClickhouseDateTime = (date: Date): string => {
// 2024-11-06T20:37:00.123Z -> 2024-11-06 20:37:00.123
return date.toISOString().replace("T", " ").replace("Z", "");
};
+
+const CLICKHOUSE_DATETIME_STRING = /^\d{4}-\d{2}-\d{2} /;
+
+/**
+ * Convert a Date, unix-millisecond timestamp, ISO string, or already-quoted
+ * ClickHouse datetime string to YYYY-MM-DD HH:MM:SS.sss for JSONEachRow inserts.
+ */
+export const toClickhouseDateTime = (
+ value?: Date | number | string | null,
+): string => {
+ if (value == null) {
+ return convertDateToClickhouseDateTime(new Date());
+ }
+ if (value instanceof Date) {
+ return convertDateToClickhouseDateTime(value);
+ }
+ if (typeof value === "number") {
+ return convertDateToClickhouseDateTime(new Date(value));
+ }
+ if (CLICKHOUSE_DATETIME_STRING.test(value)) {
+ return value;
+ }
+ return convertDateToClickhouseDateTime(new Date(value));
+};
diff --git a/packages/shared/src/server/clickhouse/compatibility.test.ts b/packages/shared/src/server/clickhouse/compatibility.test.ts
index b1a667b566..31744173d3 100644
--- a/packages/shared/src/server/clickhouse/compatibility.test.ts
+++ b/packages/shared/src/server/clickhouse/compatibility.test.ts
@@ -70,6 +70,9 @@ describe("resolveClickHouseCompatibility", () => {
["26.7.1.1334", noCompatibilitySettings],
["26.7.2.0", disableTopKThroughJoin],
["26.7.2.11", noCompatibilitySettings],
+ ["26.7.99.9999", noCompatibilitySettings],
+ ["26.8.0.0", noCompatibilitySettings],
+ ["26.8.1.2041", noCompatibilitySettings],
])("resolves automatic settings for %s", (version, expectedSettings) => {
expect(resolveClickHouseCompatibility({ version }).settings).toEqual(
expectedSettings,
@@ -107,7 +110,9 @@ describe("resolveClickHouseCompatibility", () => {
CLICKHOUSE_DISABLE_TOP_K_THROUGH_JOIN: "false",
},
}).settings,
- ).toEqual(disableLazyMaterialization);
+ ).toEqual({
+ ...disableLazyMaterialization,
+ });
});
});
diff --git a/packages/shared/src/server/data-deletion/ingestionFileDeletion.ts b/packages/shared/src/server/data-deletion/ingestionFileDeletion.ts
index c4e346d76d..59cb5ff3f7 100644
--- a/packages/shared/src/server/data-deletion/ingestionFileDeletion.ts
+++ b/packages/shared/src/server/data-deletion/ingestionFileDeletion.ts
@@ -8,7 +8,7 @@ import {
import { BlobStorageFileRefRecordReadType } from "../repositories/definitions";
import { logger } from "../logger";
import { env } from "../../env";
-import { clickhouseClient } from "../clickhouse/client";
+import { clickhouseClient, toClickhouseDateTime } from "../clickhouse/client";
import { buildClickHouseLogComment } from "../clickhouse/queryTags";
import { getS3EventStorageClient, S3_DELETE_OBJECTS_CHUNK_SIZE } from "../s3";
@@ -225,8 +225,8 @@ async function softDeleteInClickhouse(
values: blobStorageRefs.map((e) => ({
...e,
is_deleted: "1",
- event_ts: new Date().getTime(),
- updated_at: new Date().getTime(),
+ event_ts: toClickhouseDateTime(),
+ updated_at: toClickhouseDateTime(),
})),
format: "JSONEachRow",
clickhouse_settings: {
diff --git a/packages/shared/src/server/repositories/definitions.ts b/packages/shared/src/server/repositories/definitions.ts
index 819b3ff2fa..04ba2995fa 100644
--- a/packages/shared/src/server/repositories/definitions.ts
+++ b/packages/shared/src/server/repositories/definitions.ts
@@ -1,6 +1,7 @@
import z from "zod";
import type { IngestionAttribution } from "../ingestion/ingestionAttribution";
import { DEFAULT_TRACE_ENVIRONMENT } from "../ingestion/types";
+import { toClickhouseDateTime } from "../clickhouse/client";
export const clickhouseStringDateSchema = z
.string()
@@ -79,12 +80,12 @@ export type ObservationRecordReadType = z.infer<
export const observationRecordInsertSchema = observationRecordBaseSchema.extend(
{
- created_at: z.number(),
- updated_at: z.number(),
- start_time: z.number(),
- end_time: z.number().nullish(),
- completion_start_time: z.number().nullish(),
- event_ts: z.number(),
+ created_at: z.string(),
+ updated_at: z.string(),
+ start_time: z.string(),
+ end_time: z.string().nullish(),
+ completion_start_time: z.string().nullish(),
+ event_ts: z.string(),
provided_usage_details: UsageCostSchema,
provided_cost_details: UsageCostSchema,
usage_details: UsageCostSchema,
@@ -100,7 +101,7 @@ export const observationBatchStagingRecordInsertSchema =
ingestion_api_key: z.string(),
ingestion_sdk_name: z.string(),
ingestion_sdk_version: z.string(),
- s3_first_seen_timestamp: z.number(),
+ s3_first_seen_timestamp: z.string(),
});
export type ObservationBatchStagingRecordInsertType = z.infer<
typeof observationBatchStagingRecordInsertSchema
@@ -166,10 +167,10 @@ export const traceRecordReadSchema = traceRecordBaseSchema.extend({
export type TraceRecordReadType = z.infer<typeof traceRecordReadSchema>;
export const traceRecordInsertSchema = traceRecordBaseSchema.extend({
- timestamp: z.number(),
- created_at: z.number(),
- updated_at: z.number(),
- event_ts: z.number(),
+ timestamp: z.string(),
+ created_at: z.string(),
+ updated_at: z.string(),
+ event_ts: z.string(),
});
export type TraceRecordInsertType = z.infer<typeof traceRecordInsertSchema>;
@@ -177,8 +178,8 @@ export const traceNullRecordInsertSchema = z.object({
// Identifiers
project_id: z.string(),
id: z.string(),
- start_time: z.number(),
- end_time: z.number().nullish(),
+ start_time: z.string(),
+ end_time: z.string().nullish(),
name: z.string().nullish(),
// Metadata properties
@@ -204,9 +205,9 @@ export const traceNullRecordInsertSchema = z.object({
input: z.string(),
output: z.string(),
- created_at: z.number(),
- updated_at: z.number(),
- event_ts: z.number(),
+ created_at: z.string(),
+ updated_at: z.string(),
+ event_ts: z.string(),
});
export type TraceNullRecordInsertType = z.infer<
typeof traceNullRecordInsertSchema
@@ -249,10 +250,10 @@ export const scoreRecordReadSchema = scoreRecordBaseSchema.extend({
export type ScoreRecordReadType = z.infer<typeof scoreRecordReadSchema>;
export const scoreRecordInsertSchema = scoreRecordBaseSchema.extend({
- created_at: z.number(),
- updated_at: z.number(),
- timestamp: z.number(),
- event_ts: z.number(),
+ created_at: z.string(),
+ updated_at: z.string(),
+ timestamp: z.string(),
+ event_ts: z.string(),
});
export type ScoreRecordInsertType = z.infer<typeof scoreRecordInsertSchema>;
@@ -298,11 +299,11 @@ export type DatasetRunItemRecord<WithIO extends boolean = true> =
export const datasetRunItemRecordInsertSchema =
datasetRunItemRecordBaseSchema.extend({
- created_at: z.number(),
- updated_at: z.number(),
- event_ts: z.number(),
- dataset_run_created_at: z.number(),
- dataset_item_version: z.number().nullish(),
+ created_at: z.string(),
+ updated_at: z.string(),
+ event_ts: z.string(),
+ dataset_run_created_at: z.string(),
+ dataset_item_version: z.string().nullish(),
});
export type DatasetRunItemRecordInsertType = z.infer<
typeof datasetRunItemRecordInsertSchema
@@ -331,9 +332,9 @@ export type BlobStorageFileRefRecordReadType = z.infer<
>;
export const blobStorageFileLogRecordInsertSchema =
blobStorageFileLogRecordBaseSchema.extend({
- created_at: z.number(),
- updated_at: z.number(),
- event_ts: z.number(),
+ created_at: z.string(),
+ updated_at: z.string(),
+ event_ts: z.string(),
});
export type BlobStorageFileLogInsertType = z.infer<
typeof blobStorageFileLogRecordInsertSchema
@@ -344,28 +345,28 @@ export const convertTraceReadToInsert = (
): TraceRecordInsertType => {
return {
...record,
- created_at: new Date(record.created_at).getTime(),
- updated_at: new Date(record.updated_at).getTime(),
- timestamp: new Date(record.timestamp).getTime(),
- event_ts: new Date(record.event_ts).getTime(),
+ created_at: toClickhouseDateTime(record.created_at),
+ updated_at: toClickhouseDateTime(record.updated_at),
+ timestamp: toClickhouseDateTime(record.timestamp),
+ event_ts: toClickhouseDateTime(record.event_ts),
};
};
export const convertObservationReadToInsert = (
record: ObservationRecordReadType,
): ObservationRecordInsertType => {
- const convertDate = (date: string) => new Date(date).getTime();
-
return {
...record,
- created_at: convertDate(record.created_at),
- updated_at: convertDate(record.updated_at),
- start_time: convertDate(record.start_time),
- end_time: record.end_time ? convertDate(record.end_time) : undefined,
+ created_at: toClickhouseDateTime(record.created_at),
+ updated_at: toClickhouseDateTime(record.updated_at),
+ start_time: toClickhouseDateTime(record.start_time),
+ end_time: record.end_time
+ ? toClickhouseDateTime(record.end_time)
+ : undefined,
completion_start_time: record.completion_start_time
- ? convertDate(record.completion_start_time)
+ ? toClickhouseDateTime(record.completion_start_time)
: undefined,
- event_ts: convertDate(record.event_ts),
+ event_ts: toClickhouseDateTime(record.event_ts),
provided_usage_details: record.provided_usage_details,
provided_cost_details: record.provided_cost_details,
usage_details: record.usage_details,
@@ -378,10 +379,10 @@ export const convertScoreReadToInsert = (
): ScoreRecordInsertType => {
return {
...record,
- created_at: new Date(record.created_at).getTime(),
- updated_at: new Date(record.updated_at).getTime(),
- timestamp: new Date(record.timestamp).getTime(),
- event_ts: new Date(record.event_ts).getTime(),
+ created_at: toClickhouseDateTime(record.created_at),
+ updated_at: toClickhouseDateTime(record.updated_at),
+ timestamp: toClickhouseDateTime(record.timestamp),
+ event_ts: toClickhouseDateTime(record.event_ts),
};
};
@@ -392,7 +393,7 @@ export const convertScoreReadToInsert = (
*/
export const convertTraceToStagingObservation = (
traceRecord: TraceRecordInsertType,
- s3FirstSeenTimestamp: number,
+ s3FirstSeenTimestamp: string,
attribution: IngestionAttribution,
): ObservationBatchStagingRecordInsertType => {
return {
@@ -572,11 +573,11 @@ export const eventRecordReadSchema = eventRecordBaseSchema.extend({
export type EventRecordReadType = z.infer<typeof eventRecordReadSchema>;
export const eventRecordInsertSchema = eventRecordBaseSchema.extend({
- start_time: z.number(),
- end_time: z.number().nullish(),
- completion_start_time: z.number().nullish(),
- created_at: z.number(),
- updated_at: z.number(),
- event_ts: z.number(),
+ start_time: z.string(),
+ end_time: z.string().nullish(),
+ completion_start_time: z.string().nullish(),
+ created_at: z.string(),
+ updated_at: z.string(),
+ event_ts: z.string(),
});
export type EventRecordInsertType = z.infer<typeof eventRecordInsertSchema>;
diff --git a/packages/shared/src/server/test-utils/tracing-factory.test.ts b/packages/shared/src/server/test-utils/tracing-factory.test.ts
new file mode 100644
index 0000000000..b200e97e3d
--- /dev/null
+++ b/packages/shared/src/server/test-utils/tracing-factory.test.ts
@@ -0,0 +1,35 @@
+import { describe, expect, it } from "vitest";
+import { createEvent, createObservation } from "./tracing-factory";
+
+describe("tracing factories DateTime64 nulls", () => {
+ it("keeps explicit null end and completion times on observations", () => {
+ const observation = createObservation({
+ end_time: null,
+ completion_start_time: null,
+ });
+
+ expect(observation.end_time).toBeNull();
+ expect(observation.completion_start_time).toBeNull();
+ });
+
+ it("keeps explicit null end and completion times on events", () => {
+ const event = createEvent({
+ end_time: null,
+ completion_start_time: null,
+ });
+
+ expect(event.end_time).toBeNull();
+ expect(event.completion_start_time).toBeNull();
+ });
+
+ it("stringifies millisecond and microsecond event ticks to the same instant", () => {
+ const millisecond = Date.UTC(2024, 0, 15, 12, 0, 0);
+
+ expect(createEvent({ start_time: millisecond }).start_time).toBe(
+ "2024-01-15 12:00:00.000",
+ );
+ expect(createEvent({ start_time: millisecond * 1000 }).start_time).toBe(
+ "2024-01-15 12:00:00.000",
+ );
+ });
+});
diff --git a/packages/shared/src/server/test-utils/tracing-factory.ts b/packages/shared/src/server/test-utils/tracing-factory.ts
index 6410fae6a0..179db79145 100644
--- a/packages/shared/src/server/test-utils/tracing-factory.ts
+++ b/packages/shared/src/server/test-utils/tracing-factory.ts
@@ -1,4 +1,5 @@
import { v4 } from "uuid";
+import { toClickhouseDateTime } from "../clickhouse/client";
import {
TraceRecordInsertType,
ObservationRecordInsertType,
@@ -8,14 +9,33 @@ import {
} from "../repositories/definitions";
import { UNKNOWN_INGESTION_SDK_VALUE } from "../ingestion/ingestionAttribution";
+type DateTimeInput = Date | number | string;
+
+type WithDateTimeInputs<T, K extends keyof T> = Omit<Partial<T>, K> & {
+ [P in K]?: DateTimeInput | null;
+};
+
+// Test fixtures historically passed events_full ticks as microseconds.
+// Numbers above 1e14 cannot be unix milliseconds in the DateTime64 range.
+const toEventClickhouseDateTime = (value?: DateTimeInput | null): string => {
+ if (typeof value === "number" && Math.abs(value) > 1e14) {
+ return toClickhouseDateTime(Math.trunc(value / 1000));
+ }
+ return toClickhouseDateTime(value);
+};
+
export const createTrace = (
- trace: Partial<TraceRecordInsertType>,
+ trace: WithDateTimeInputs<
+ TraceRecordInsertType,
+ "timestamp" | "created_at" | "updated_at" | "event_ts"
+ >,
): TraceRecordInsertType => {
+ const { timestamp, created_at, updated_at, event_ts, ...rest } = trace;
return {
id: v4(),
project_id: v4(),
session_id: v4(),
- timestamp: Date.now(),
+ timestamp: toClickhouseDateTime(timestamp),
environment: "default",
metadata: {
source: "API",
@@ -28,17 +48,32 @@ export const createTrace = (
release: "1.0.0",
version: "2",
user_id: v4(),
- created_at: Date.now(),
- updated_at: Date.now(),
- event_ts: Date.now(),
+ created_at: toClickhouseDateTime(created_at),
+ updated_at: toClickhouseDateTime(updated_at),
+ event_ts: toClickhouseDateTime(event_ts),
is_deleted: 0,
- ...trace,
+ ...rest,
};
};
export const createDatasetRunItem = (
- datasetRunItem: Partial<DatasetRunItemRecordInsertType>,
+ datasetRunItem: WithDateTimeInputs<
+ DatasetRunItemRecordInsertType,
+ | "created_at"
+ | "updated_at"
+ | "event_ts"
+ | "dataset_run_created_at"
+ | "dataset_item_version"
+ >,
): DatasetRunItemRecordInsertType => {
+ const {
+ created_at,
+ updated_at,
+ event_ts,
+ dataset_run_created_at,
+ dataset_item_version,
+ ...rest
+ } = datasetRunItem;
return {
id: v4(),
project_id: v4(),
@@ -52,19 +87,39 @@ export const createDatasetRunItem = (
dataset_item_input: "{}",
dataset_item_expected_output: "{}",
dataset_item_metadata: { key: "value" },
- dataset_run_created_at: Date.now(),
- created_at: Date.now(),
- updated_at: Date.now(),
- event_ts: Date.now(),
+ dataset_run_created_at: toClickhouseDateTime(dataset_run_created_at),
+ created_at: toClickhouseDateTime(created_at),
+ updated_at: toClickhouseDateTime(updated_at),
+ event_ts: toClickhouseDateTime(event_ts),
+ dataset_item_version: dataset_item_version
+ ? toClickhouseDateTime(dataset_item_version)
+ : undefined,
is_deleted: 0,
error: datasetRunItem.error ?? null,
- ...datasetRunItem,
+ ...rest,
};
};
export const createObservation = (
- observation: Partial<ObservationRecordInsertType>,
+ observation: WithDateTimeInputs<
+ ObservationRecordInsertType,
+ | "created_at"
+ | "updated_at"
+ | "start_time"
+ | "end_time"
+ | "completion_start_time"
+ | "event_ts"
+ >,
): ObservationRecordInsertType => {
+ const {
+ created_at,
+ updated_at,
+ start_time,
+ end_time,
+ completion_start_time,
+ event_ts,
+ ...rest
+ } = observation;
return {
id: v4(),
trace_id: v4(),
@@ -80,10 +135,10 @@ export const createObservation = (
usage_details: { input: 1234, output: 5678, total: 6912 },
cost_details: { input: 100, output: 200, total: 300 },
is_deleted: 0,
- created_at: Date.now(),
- updated_at: Date.now(),
- start_time: Date.now(),
- event_ts: Date.now(),
+ created_at: toClickhouseDateTime(created_at),
+ updated_at: toClickhouseDateTime(updated_at),
+ start_time: toClickhouseDateTime(start_time),
+ event_ts: toClickhouseDateTime(event_ts),
name: "sample_name" + v4(),
level: "DEFAULT",
status_message: "status",
@@ -97,18 +152,25 @@ export const createObservation = (
prompt_id: v4(),
prompt_name: "generation-prompt",
prompt_version: 1,
- end_time: Date.now(),
- completion_start_time: Date.now(),
+ end_time: end_time === null ? null : toClickhouseDateTime(end_time),
+ completion_start_time:
+ completion_start_time === null
+ ? null
+ : toClickhouseDateTime(completion_start_time),
tool_definitions: {},
tool_calls: [],
tool_call_names: [],
- ...observation,
+ ...rest,
};
};
export const createTraceScore = (
- score: Partial<ScoreRecordInsertType>,
+ score: WithDateTimeInputs<
+ ScoreRecordInsertType,
+ "timestamp" | "created_at" | "updated_at" | "event_ts"
+ >,
): ScoreRecordInsertType => {
+ const { timestamp, created_at, updated_at, event_ts, ...rest } = score;
return {
id: v4(),
project_id: v4(),
@@ -116,7 +178,7 @@ export const createTraceScore = (
observation_id: null, // Trace-level scores must have observation_id as null by default
environment: "default",
name: "test-score" + v4(),
- timestamp: Date.now(),
+ timestamp: toClickhouseDateTime(timestamp),
value: 100.5,
string_value: null,
long_string_value: "",
@@ -124,43 +186,47 @@ export const createTraceScore = (
comment: "comment",
metadata: { "test-key": "test-value" },
data_type: "NUMERIC" as const,
- created_at: Date.now(),
- updated_at: Date.now(),
- event_ts: Date.now(),
+ created_at: toClickhouseDateTime(created_at),
+ updated_at: toClickhouseDateTime(updated_at),
+ event_ts: toClickhouseDateTime(event_ts),
ingestion_api_key: "",
ingestion_sdk_name: UNKNOWN_INGESTION_SDK_VALUE,
ingestion_sdk_version: UNKNOWN_INGESTION_SDK_VALUE,
is_deleted: 0,
- ...score,
+ ...rest,
session_id: null,
dataset_run_id: null,
};
};
export const createSessionScore = (
- score: Partial<ScoreRecordInsertType>,
+ score: WithDateTimeInputs<
+ ScoreRecordInsertType,
+ "timestamp" | "created_at" | "updated_at" | "event_ts"
+ >,
): ScoreRecordInsertType => {
+ const { timestamp, created_at, updated_at, event_ts, ...rest } = score;
return {
id: v4(),
project_id: v4(),
session_id: v4(),
environment: "default",
name: "test-session-score" + v4(),
- timestamp: Date.now(),
+ timestamp: toClickhouseDateTime(timestamp),
value: 100.5,
long_string_value: "",
source: "API",
comment: "comment",
metadata: { "test-key": "test-value" },
data_type: "NUMERIC" as const,
- created_at: Date.now(),
- updated_at: Date.now(),
- event_ts: Date.now(),
+ created_at: toClickhouseDateTime(created_at),
+ updated_at: toClickhouseDateTime(updated_at),
+ event_ts: toClickhouseDateTime(event_ts),
ingestion_api_key: "",
ingestion_sdk_name: UNKNOWN_INGESTION_SDK_VALUE,
ingestion_sdk_version: UNKNOWN_INGESTION_SDK_VALUE,
is_deleted: 0,
- ...score,
+ ...rest,
observation_id: null,
trace_id: null,
dataset_run_id: null,
@@ -168,29 +234,33 @@ export const createSessionScore = (
};
export const createDatasetRunScore = (
- score: Partial<ScoreRecordInsertType>,
+ score: WithDateTimeInputs<
+ ScoreRecordInsertType,
+ "timestamp" | "created_at" | "updated_at" | "event_ts"
+ >,
): ScoreRecordInsertType => {
+ const { timestamp, created_at, updated_at, event_ts, ...rest } = score;
return {
id: v4(),
project_id: v4(),
dataset_run_id: v4(),
environment: "default",
name: "test-run-score" + v4(),
- timestamp: Date.now(),
+ timestamp: toClickhouseDateTime(timestamp),
value: 100.5,
long_string_value: "",
source: "API",
comment: "comment",
metadata: { "test-key": "test-value" },
data_type: "NUMERIC" as const,
- created_at: Date.now(),
- updated_at: Date.now(),
- event_ts: Date.now(),
+ created_at: toClickhouseDateTime(created_at),
+ updated_at: toClickhouseDateTime(updated_at),
+ event_ts: toClickhouseDateTime(event_ts),
ingestion_api_key: "",
ingestion_sdk_name: UNKNOWN_INGESTION_SDK_VALUE,
ingestion_sdk_version: UNKNOWN_INGESTION_SDK_VALUE,
is_deleted: 0,
- ...score,
+ ...rest,
observation_id: null,
trace_id: null,
session_id: null,
@@ -198,17 +268,30 @@ export const createDatasetRunScore = (
};
export const createEvent = (
- event: Partial<EventRecordInsertType> & {
+ event: WithDateTimeInputs<
+ EventRecordInsertType,
+ | "start_time"
+ | "end_time"
+ | "completion_start_time"
+ | "created_at"
+ | "updated_at"
+ | "event_ts"
+ > & {
metadata_values?: (string | null | undefined)[];
},
): EventRecordInsertType => {
const spanId = v4();
- const now = Date.now() * 1000; // Convert to micro
// Extract metadata array overrides before spreading to prevent undefined from clobbering defaults
const {
metadata_values: metadataValuesAlias,
metadata_names: metadataNamesOverride,
+ start_time,
+ end_time,
+ completion_start_time,
+ created_at,
+ updated_at,
+ event_ts,
...eventOverrides
} = event;
@@ -308,13 +391,16 @@ export const createEvent = (
event_bytes: 2,
is_deleted: 0,
- // Timestamps
- start_time: now,
- end_time: now,
- completion_start_time: null,
- created_at: now,
- updated_at: now,
- event_ts: now,
+ // Timestamps (ClickHouse DateTime64 strings)
+ start_time: toEventClickhouseDateTime(start_time),
+ end_time: end_time === null ? null : toEventClickhouseDateTime(end_time),
+ completion_start_time:
+ completion_start_time == null
+ ? null
+ : toEventClickhouseDateTime(completion_start_time),
+ created_at: toEventClickhouseDateTime(created_at),
+ updated_at: toEventClickhouseDateTime(updated_at),
+ event_ts: toEventClickhouseDateTime(event_ts),
...eventOverrides,
};
diff --git a/web/src/__tests__/server/api/tables/prompts-ui.servertest.ts b/web/src/__tests__/server/api/tables/prompts-ui.servertest.ts
index 9ce6e192c0..6f9f2cb1fe 100644
--- a/web/src/__tests__/server/api/tables/prompts-ui.servertest.ts
+++ b/web/src/__tests__/server/api/tables/prompts-ui.servertest.ts
@@ -11,6 +11,7 @@ import {
getObservationMetricsForPromptsFromEvents,
getObservationsWithPromptName,
getObservationsWithPromptNameFromEvents,
+ toClickhouseDateTime,
} from "@langfuse/shared/src/server";
import { env } from "@/src/env.mjs";
import { v4 } from "uuid";
@@ -472,7 +473,7 @@ describe("UI Prompts Table", () => {
const promptId = v4();
const traceId = v4();
const spanId = v4();
- const startTime = Date.parse("2026-07-16T13:42:33.028Z") * 1000;
+ const startTime = Date.parse("2026-07-16T13:42:33.028Z");
const event = createEvent({
project_id: projectId,
trace_id: traceId,
@@ -482,7 +483,7 @@ describe("UI Prompts Table", () => {
prompt_name: "folder/test-prompt",
prompt_version: 7,
start_time: startTime,
- end_time: startTime + 1_000_000,
+ end_time: startTime + 1_000,
event_ts: startTime,
usage_details: { input: 100, output: 200, total: 300 },
cost_details: { input: 1, output: 2, total: 3 },
@@ -492,8 +493,8 @@ describe("UI Prompts Table", () => {
await createEventsCh([
{
...event,
- end_time: startTime + 4_000_000,
- event_ts: startTime + 1_000_000,
+ end_time: toClickhouseDateTime(startTime + 4_000),
+ event_ts: toClickhouseDateTime(startTime + 1_000),
usage_details: { input: 400, output: 500, total: 900 },
cost_details: { input: 4, output: 5, total: 9 },
},
diff --git a/web/src/__tests__/server/dashboard-v1-v2-consistency.servertest.ts b/web/src/__tests__/server/dashboard-v1-v2-consistency.servertest.ts
index 36964aff78..573d38d677 100644
--- a/web/src/__tests__/server/dashboard-v1-v2-consistency.servertest.ts
+++ b/web/src/__tests__/server/dashboard-v1-v2-consistency.servertest.ts
@@ -122,15 +122,15 @@ function buildMatchingEvents(
input: t.input ?? null,
output: t.output ?? null,
metadata: t.metadata ?? {},
- start_time: t.timestamp * 1000,
+ start_time: t.timestamp,
end_time: null,
cost_details: {},
provided_cost_details: {},
usage_details: {},
provided_usage_details: {},
- created_at: t.created_at * 1000,
- updated_at: t.updated_at * 1000,
- event_ts: t.event_ts * 1000,
+ created_at: t.created_at,
+ updated_at: t.updated_at,
+ event_ts: t.event_ts,
}),
),
);
@@ -176,14 +176,12 @@ function buildMatchingEvents(
tool_definitions: o.tool_definitions ?? {},
tool_calls: o.tool_calls ?? [],
tool_call_names: o.tool_call_names ?? [],
- start_time: o.start_time * 1000,
- end_time: o.end_time ? o.end_time * 1000 : null,
- completion_start_time: o.completion_start_time
- ? o.completion_start_time * 1000
- : null,
- created_at: o.created_at * 1000,
- updated_at: o.updated_at * 1000,
- event_ts: o.event_ts * 1000,
+ start_time: o.start_time,
+ end_time: o.end_time ?? null,
+ completion_start_time: o.completion_start_time ?? null,
+ created_at: o.created_at,
+ updated_at: o.updated_at,
+ event_ts: o.event_ts,
}),
),
);
diff --git a/web/src/__tests__/server/mcp-tools-read.servertest.ts b/web/src/__tests__/server/mcp-tools-read.servertest.ts
index 6194525ba2..5233f40138 100644
--- a/web/src/__tests__/server/mcp-tools-read.servertest.ts
+++ b/web/src/__tests__/server/mcp-tools-read.servertest.ts
@@ -2176,7 +2176,9 @@ describe("MCP Read Tools", () => {
projectId,
traceId,
name: `mcp-metrics-custom-limit-${index}-${nanoid()}`,
- startTime: new Date(`2026-01-01T00:00:${index}.000Z`),
+ startTime: new Date(
+ `2026-01-01T00:00:${String(index).padStart(2, "0")}.000Z`,
+ ),
}),
),
);
diff --git a/web/src/__tests__/server/observations-api.servertest.ts b/web/src/__tests__/server/observations-api.servertest.ts
index 864e3344d7..5548ab9dc3 100644
--- a/web/src/__tests__/server/observations-api.servertest.ts
+++ b/web/src/__tests__/server/observations-api.servertest.ts
@@ -819,17 +819,19 @@ describe("/api/public/observations API Endpoint", () => {
: baseTimestamp;
// Create two traces with different userIds
+ // traces.timestamp is DateTime64(3) millisecond ticks. timeValue is
+ // microseconds when the events table is enabled — do not reuse it here.
const trace1 = createTrace({
id: trace1Id,
project_id: projectId,
- timestamp: timeValue,
+ timestamp: baseTimestamp,
user_id: "user-A",
});
const trace2 = createTrace({
id: trace2Id,
project_id: projectId,
- timestamp: timeValue,
+ timestamp: baseTimestamp,
user_id: "user-B",
});
diff --git a/web/src/__tests__/server/queryBuilder.servertest.ts b/web/src/__tests__/server/queryBuilder.servertest.ts
index 4ce7a97f9c..34a1a5f668 100644
--- a/web/src/__tests__/server/queryBuilder.servertest.ts
+++ b/web/src/__tests__/server/queryBuilder.servertest.ts
@@ -3381,8 +3381,13 @@ describe("queryBuilder", () => {
result.data = await executeQuery(projectId, query);
expect(result.data).toHaveLength(2);
- expect(result.data[0].name).toBe("observation-basic");
- expect(Number(result.data[0].count_count)).toBe(1);
+ expect(result.data.map((row) => row.name).toSorted()).toEqual([
+ "observation-basic",
+ "observation-premium",
+ ]);
+ expect(result.data.every((row) => Number(row.count_count) === 1)).toBe(
+ true,
+ );
});
it("should generate histogram with custom bin count for cost distribution", async () => {
@@ -4089,13 +4094,13 @@ describe("queryBuilder", () => {
project_id: projectId,
type: "GENERATION",
cost_details: { input: 10, output: 20, total: 30 },
- start_time: Date.now() * 1000,
+ start_time: Date.now(),
}),
createEvent({
project_id: projectId,
type: "GENERATION",
cost_details: { input: 5, output: 15, total: 20 },
- start_time: Date.now() * 1000,
+ start_time: Date.now(),
}),
];
await createEventsCh(events);
@@ -4139,13 +4144,13 @@ describe("queryBuilder", () => {
project_id: projectId,
type: "GENERATION",
usage_details: { input: 100, output: 200, total: 300 },
- start_time: Date.now() * 1000,
+ start_time: Date.now(),
}),
createEvent({
project_id: projectId,
type: "GENERATION",
usage_details: { input: 50, output: 75, total: 125 },
- start_time: Date.now() * 1000,
+ start_time: Date.now(),
}),
];
await createEventsCh(events);
@@ -4247,13 +4252,13 @@ describe("queryBuilder", () => {
project_id: projectId,
type: "GENERATION",
cost_details: { input: 10 },
- start_time: Date.now() * 1000,
+ start_time: Date.now(),
}),
createEvent({
project_id: projectId,
type: "SPAN",
cost_details: { input: 999 },
- start_time: Date.now() * 1000,
+ start_time: Date.now(),
}),
];
await createEventsCh(events);
@@ -4300,13 +4305,13 @@ describe("queryBuilder", () => {
project_id: projectId,
type: "GENERATION",
cost_details: { input: 10, output: 20, total: 30 },
- start_time: Date.now() * 1000,
+ start_time: Date.now(),
}),
createEvent({
project_id: projectId,
type: "GENERATION",
cost_details: { input: 5, output: 15, total: 20 },
- start_time: Date.now() * 1000,
+ start_time: Date.now(),
}),
];
await createEventsCh(events);
@@ -4352,7 +4357,7 @@ describe("queryBuilder", () => {
project_id: projectId,
type: "GENERATION",
cost_details: { input: 7, output: 3 },
- start_time: Date.now() * 1000,
+ start_time: Date.now(),
}),
];
await createEventsCh(events);
@@ -5044,7 +5049,7 @@ describe("query builder measure-aggregation validation", () => {
trace_name: "",
name: "my-trace",
parent_span_id: "", // root event
- start_time: Date.now() * 1000,
+ start_time: Date.now(),
}),
createEvent({
project_id: projectId,
@@ -5052,7 +5057,7 @@ describe("query builder measure-aggregation validation", () => {
trace_name: "",
name: "child-observation",
parent_span_id: "some-parent", // child event
- start_time: Date.now() * 1000,
+ start_time: Date.now(),
}),
// Trace 2: trace_name is "other-trace"
createEvent({
@@ -5061,7 +5066,7 @@ describe("query builder measure-aggregation validation", () => {
trace_name: "other-trace",
name: "root-event",
parent_span_id: "", // root event
- start_time: Date.now() * 1000,
+ start_time: Date.now(),
}),
];
await createEventsCh(events);
@@ -5111,7 +5116,7 @@ describe("query builder measure-aggregation validation", () => {
trace_name: "target-trace",
name: "root-observation",
parent_span_id: "",
- start_time: Date.now() * 1000,
+ start_time: Date.now(),
}),
createEvent({
project_id: projectId,
@@ -5119,7 +5124,7 @@ describe("query builder measure-aggregation validation", () => {
trace_name: "other-trace",
name: "root-observation",
parent_span_id: "",
- start_time: Date.now() * 1000,
+ start_time: Date.now(),
}),
];
await createEventsCh(events);
@@ -5172,7 +5177,7 @@ describe("query builder measure-aggregation validation", () => {
name: "root-op",
parent_span_id: "",
environment: "production",
- start_time: Date.now() * 1000,
+ start_time: Date.now(),
}),
// Trace 2: name="target-trace", environment="staging"
createEvent({
@@ -5182,7 +5187,7 @@ describe("query builder measure-aggregation validation", () => {
name: "root-op",
parent_span_id: "",
environment: "staging",
- start_time: Date.now() * 1000,
+ start_time: Date.now(),
}),
// Trace 3: name="other-trace", environment="production"
createEvent({
@@ -5192,7 +5197,7 @@ describe("query builder measure-aggregation validation", () => {
name: "root-op",
parent_span_id: "",
environment: "production",
- start_time: Date.now() * 1000,
+ start_time: Date.now(),
}),
];
await createEventsCh(events);
@@ -5321,7 +5326,7 @@ describe("query builder measure-aggregation validation", () => {
parent_span_id: "",
metadata_names: ["experiments"],
metadata_values: [longValue],
- start_time: Date.now() * 1000,
+ start_time: Date.now(),
}),
]);
diff --git a/web/src/__tests__/server/queryBuilderDashboards.servertest.ts b/web/src/__tests__/server/queryBuilderDashboards.servertest.ts
index eb55e0c2ca..07a8c5183f 100644
--- a/web/src/__tests__/server/queryBuilderDashboards.servertest.ts
+++ b/web/src/__tests__/server/queryBuilderDashboards.servertest.ts
@@ -4,6 +4,7 @@ import {
createTracesCh,
createObservation,
createObservationsCh,
+ convertDateToClickhouseDateTime,
} from "@langfuse/shared/src/server";
import { executeQuery } from "@langfuse/shared/query/server";
import { type QueryType } from "@langfuse/shared/query";
@@ -220,7 +221,8 @@ describe("selfServeDashboards", () => {
// Count recent production traces (within the last hour)
stats.recentProductionTraces = traces.filter(
(t) =>
- t.environment === "production" && t.timestamp >= oneHourAgo.getTime(),
+ t.environment === "production" &&
+ t.timestamp >= convertDateToClickhouseDateTime(oneHourAgo),
).length;
});
diff --git a/web/src/__tests__/server/repositories/experiment-items-repository.servertest.ts b/web/src/__tests__/server/repositories/experiment-items-repository.servertest.ts
index 2ad8878819..de810dc94f 100644
--- a/web/src/__tests__/server/repositories/experiment-items-repository.servertest.ts
+++ b/web/src/__tests__/server/repositories/experiment-items-repository.servertest.ts
@@ -18,7 +18,7 @@ import { type FilterCondition } from "@langfuse/shared";
* Wraps createEvent with experiment defaults.
*/
function createExperimentEvent(
- params: Partial<EventRecordInsertType> & {
+ params: Parameters<typeof createEvent>[0] & {
experimentId: string;
experimentName: string;
datasetId?: string;
diff --git a/web/src/__tests__/server/repositories/experiments-public-api-repository.servertest.ts b/web/src/__tests__/server/repositories/experiments-public-api-repository.servertest.ts
index 962756a0eb..c465143b56 100644
--- a/web/src/__tests__/server/repositories/experiments-public-api-repository.servertest.ts
+++ b/web/src/__tests__/server/repositories/experiments-public-api-repository.servertest.ts
@@ -926,8 +926,8 @@ describe("Public API experiments repository", () => {
it("preserves microsecond precision in experiment item cursors", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const startTimeMs = Date.UTC(2026, 0, 1, 0, 0, 0, 123);
- const firstStartTimeUs = startTimeMs * 1_000 + 456;
- const secondStartTimeUs = startTimeMs * 1_000;
+ const firstStartTime = "2026-01-01 00:00:00.123456";
+ const secondStartTime = "2026-01-01 00:00:00.123000";
await createEventsCh([
{
@@ -937,8 +937,8 @@ describe("Public API experiments repository", () => {
startTimeMs,
spanId: "span-cursor-first",
}),
- start_time: firstStartTimeUs,
- event_ts: firstStartTimeUs,
+ start_time: firstStartTime,
+ event_ts: firstStartTime,
},
{
...createExperimentRootEvent({
@@ -947,8 +947,8 @@ describe("Public API experiments repository", () => {
startTimeMs,
spanId: "span-cursor-second",
}),
- start_time: secondStartTimeUs,
- event_ts: secondStartTimeUs,
+ start_time: secondStartTime,
+ event_ts: secondStartTime,
},
]);
diff --git a/web/src/__tests__/server/sessions-ui-table.servertest.ts b/web/src/__tests__/server/sessions-ui-table.servertest.ts
index 9a57d9e8c7..cc767c0227 100644
--- a/web/src/__tests__/server/sessions-ui-table.servertest.ts
+++ b/web/src/__tests__/server/sessions-ui-table.servertest.ts
@@ -68,7 +68,7 @@ function asEventInsert(
*
* One root event per trace (parent_span_id = ''), plus one event per
* observation with trace-level fields denormalized.
- * Timestamps are converted from ms to µs (* 1000).
+ * DateTime64 fields are already ClickHouse datetime strings on insert types.
*/
function buildMatchingEvents(
traces: TraceRecordInsertType[],
@@ -101,15 +101,15 @@ function buildMatchingEvents(
input: t.input ?? null,
output: t.output ?? null,
metadata: t.metadata ?? {},
- start_time: t.timestamp * 1000,
+ start_time: t.timestamp,
end_time: null,
cost_details: {},
provided_cost_details: {},
usage_details: {},
provided_usage_details: {},
- created_at: t.created_at * 1000,
- updated_at: t.updated_at * 1000,
- event_ts: t.event_ts * 1000,
+ created_at: t.created_at,
+ updated_at: t.updated_at,
+ event_ts: t.event_ts,
}),
),
);
@@ -153,14 +153,12 @@ function buildMatchingEvents(
tool_definitions: o.tool_definitions ?? {},
tool_calls: o.tool_calls ?? [],
tool_call_names: o.tool_call_names ?? [],
- start_time: o.start_time * 1000,
- end_time: o.end_time ? o.end_time * 1000 : null,
- completion_start_time: o.completion_start_time
- ? o.completion_start_time * 1000
- : null,
- created_at: o.created_at * 1000,
- updated_at: o.updated_at * 1000,
- event_ts: o.event_ts * 1000,
+ start_time: o.start_time,
+ end_time: o.end_time ?? null,
+ completion_start_time: o.completion_start_time ?? null,
+ created_at: o.created_at,
+ updated_at: o.updated_at,
+ event_ts: o.event_ts,
}),
),
);
diff --git a/web/src/__tests__/server/traces-api.servertest.ts b/web/src/__tests__/server/traces-api.servertest.ts
index 109f8e9200..13830646d5 100644
--- a/web/src/__tests__/server/traces-api.servertest.ts
+++ b/web/src/__tests__/server/traces-api.servertest.ts
@@ -7,7 +7,6 @@ import {
getTraceById,
createEvent,
createOrgProjectAndApiKey,
- type EventRecordInsertType,
createObservationsCh,
createTracesCh,
createEventsCh,
@@ -50,17 +49,18 @@ type ObservationEventData = {
total_cost?: number;
};
-// Helper to create observation/event data in the appropriate format
-// Handles time conversion internally: milliseconds -> microseconds for events table
const createObservationOrEvent = (
useEventsTable: boolean,
- data: ObservationEventData & Partial<EventRecordInsertType>,
+ data: ObservationEventData & {
+ environment?: string;
+ user_id?: string | null;
+ session_id?: string | null;
+ trace_name?: string;
+ },
) => {
const id = data.id ?? randomUUID();
- const timeMultiplier = useEventsTable ? 1000 : 1; // microseconds vs milliseconds
if (useEventsTable) {
- // For events table: microseconds, requires span_id
return createEvent({
...data,
id,
@@ -68,13 +68,8 @@ const createObservationOrEvent = (
parent_span_id: data.trace_id, // Observations have trace as parent
type: data.type ?? "SPAN",
level: data.level ?? "DEFAULT",
- start_time: data.start_time * timeMultiplier, // Convert ms to microseconds
- end_time:
- data.end_time === null
- ? null
- : data.end_time
- ? data.end_time * timeMultiplier
- : null,
+ start_time: data.start_time,
+ end_time: data.end_time ?? null,
});
}
// For observations table: milliseconds
@@ -157,7 +152,7 @@ const createTraceWithObservations = async (
name: trace.name ?? "trace",
trace_name: trace.name ?? "trace",
type: "GENERATION", // Trace events are typically GENERATION type
- start_time: trace.timestamp * 1000, // Convert ms to microseconds
+ start_time: trace.timestamp,
end_time: null,
environment: trace.environment ?? "default",
version: trace.version ?? null,
diff --git a/worker/src/__tests__/batchDataRetentionCleaner.test.ts b/worker/src/__tests__/batchDataRetentionCleaner.test.ts
index 125259a216..fe5d712641 100644
--- a/worker/src/__tests__/batchDataRetentionCleaner.test.ts
+++ b/worker/src/__tests__/batchDataRetentionCleaner.test.ts
@@ -16,6 +16,7 @@ import {
createScoresCh,
createTraceScore,
queryClickhouse,
+ toClickhouseDateTime,
traceException,
} from "@langfuse/shared/src/server";
import { prisma } from "@langfuse/shared/src/db";
@@ -167,8 +168,8 @@ async function insertRetentionTestRows(
format: "JSONEachRow",
values: rows.map((row) => ({
project_id: row.projectId,
- start_time: row.startTime,
- event_ts: Date.now(),
+ start_time: toClickhouseDateTime(row.startTime),
+ event_ts: toClickhouseDateTime(),
})),
});
}
diff --git a/worker/src/__tests__/batchProjectBlobCleaner.test.ts b/worker/src/__tests__/batchProjectBlobCleaner.test.ts
index c8eb42fb0f..4d35ff5b77 100644
--- a/worker/src/__tests__/batchProjectBlobCleaner.test.ts
+++ b/worker/src/__tests__/batchProjectBlobCleaner.test.ts
@@ -4,6 +4,7 @@ import {
clickhouseClient,
createOrgProjectAndApiKey,
getDeletedProjects,
+ toClickhouseDateTime,
} from "@langfuse/shared/src/server";
import { prisma } from "@langfuse/shared/src/db";
import { BatchProjectBlobCleaner } from "../features/batch-project-blob-cleaner";
@@ -34,9 +35,9 @@ async function insertBlobRefs(projectId: string, count: number): Promise<void> {
event_id: randomUUID(),
bucket_name: "test-bucket",
bucket_path: `${projectId}/traces/${randomUUID()}.json`,
- created_at: new Date().getTime(),
- updated_at: new Date().getTime(),
- event_ts: new Date().getTime(),
+ created_at: toClickhouseDateTime(),
+ updated_at: toClickhouseDateTime(),
+ event_ts: toClickhouseDateTime(),
is_deleted: 0,
}));
@@ -59,9 +60,9 @@ async function softDeleteBlobRefs(
event_id: randomUUID(),
bucket_name: "test-bucket",
bucket_path: `${projectId}/traces/${randomUUID()}.json`,
- created_at: new Date().getTime(),
- updated_at: new Date().getTime(),
- event_ts: new Date().getTime(),
+ created_at: toClickhouseDateTime(),
+ updated_at: toClickhouseDateTime(),
+ event_ts: toClickhouseDateTime(),
is_deleted: 1,
}));
diff --git a/worker/src/__tests__/dataRetentionProcessing.test.ts b/worker/src/__tests__/dataRetentionProcessing.test.ts
index 88535e9e94..3d822a2fbd 100644
--- a/worker/src/__tests__/dataRetentionProcessing.test.ts
+++ b/worker/src/__tests__/dataRetentionProcessing.test.ts
@@ -15,6 +15,7 @@ import {
getTraceById,
StorageService,
StorageServiceFactory,
+ toClickhouseDateTime,
} from "@langfuse/shared/src/server";
import { prisma } from "@langfuse/shared/src/db";
import { handleDataRetentionProcessingJob } from "../ee/dataRetention/handleDataRetentionProcessingJob";
@@ -82,8 +83,12 @@ describe("DataRetentionProcessingJob", () => {
event_id: randomUUID(),
bucket_name: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
bucket_path: fileName,
- created_at: new Date(Date.now() - 1000 * 60 * 60 * 24 * 3).getTime(), // 3 days in the past
- updated_at: new Date(Date.now() - 1000 * 60 * 60 * 24 * 3).getTime(), // 3 days in the past
+ created_at: toClickhouseDateTime(
+ Date.now() - 1000 * 60 * 60 * 24 * 3,
+ ), // 3 days in the past
+ updated_at: toClickhouseDateTime(
+ Date.now() - 1000 * 60 * 60 * 24 * 3,
+ ), // 3 days in the past
},
],
});
@@ -141,8 +146,12 @@ describe("DataRetentionProcessingJob", () => {
event_id: randomUUID(),
bucket_name: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
bucket_path: fileName,
- created_at: new Date(Date.now() - 1000 * 60 * 60 * 24 * 30).getTime(), // 30 days in the past
- updated_at: new Date(Date.now() - 1000 * 60 * 60 * 24 * 30).getTime(), // 30 days in the past
+ created_at: toClickhouseDateTime(
+ Date.now() - 1000 * 60 * 60 * 24 * 30,
+ ), // 30 days in the past
+ updated_at: toClickhouseDateTime(
+ Date.now() - 1000 * 60 * 60 * 24 * 30,
+ ), // 30 days in the past
},
],
});
diff --git a/worker/src/__tests__/datetime64Insert.integration.test.ts b/worker/src/__tests__/datetime64Insert.integration.test.ts
new file mode 100644
index 0000000000..57c6e4dfbf
--- /dev/null
+++ b/worker/src/__tests__/datetime64Insert.integration.test.ts
@@ -0,0 +1,109 @@
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import {
+ clickhouseClient,
+ toClickhouseDateTime,
+} from "@langfuse/shared/src/server";
+
+const TABLE_MS = "datetime64_insert_probe_ms";
+const TICKS_MS = Date.UTC(2024, 10, 6, 20, 37, 0, 123);
+const QUOTED_MS = "2024-11-06 20:37:00.123";
+
+const isClickHouseAtLeast26_8 = (version: string): boolean => {
+ const match = version.trim().match(/^v?(\d+)\.(\d+)/);
+ if (!match) return false;
+ const major = Number(match[1]);
+ const minor = Number(match[2]);
+ return major > 26 || (major === 26 && minor >= 8);
+};
+
+describe("DateTime64 JSONEachRow inserts", () => {
+ let clickHouseVersion = "";
+ let clickHouseAvailable = false;
+
+ beforeAll(async () => {
+ try {
+ const result = await clickhouseClient().query({
+ query: "SELECT version() AS version",
+ format: "JSONEachRow",
+ });
+ const rows = await result.json<{ version: string }>();
+ clickHouseVersion = rows[0]?.version ?? "";
+ clickHouseAvailable = Boolean(clickHouseVersion);
+
+ await clickhouseClient().command({
+ query: `
+ CREATE TABLE IF NOT EXISTS ${TABLE_MS} (
+ id String,
+ t DateTime64(3, 'UTC')
+ ) ENGINE = Memory
+ `,
+ });
+ } catch {
+ clickHouseAvailable = false;
+ }
+ });
+
+ afterAll(async () => {
+ if (!clickHouseAvailable) return;
+ await clickhouseClient().command({
+ query: `DROP TABLE IF EXISTS ${TABLE_MS}`,
+ });
+ });
+
+ const insertAndRead = async (id: string, value: number | string) => {
+ await clickhouseClient().insert({
+ table: TABLE_MS,
+ format: "JSONEachRow",
+ values: [{ id, t: value }],
+ });
+
+ const result = await clickhouseClient().query({
+ query: `
+ SELECT
+ toYear(t) AS year,
+ formatDateTime(t, '%F %T.%f', 'UTC') AS formatted
+ FROM ${TABLE_MS}
+ WHERE id = {id: String}
+ `,
+ query_params: { id },
+ format: "JSONEachRow",
+ });
+ const rows = await result.json<{
+ year: number | string;
+ formatted: string;
+ }>();
+ return {
+ year: Number(rows[0]?.year),
+ formatted: rows[0]?.formatted ?? "",
+ };
+ };
+
+ it("stores toClickhouseDateTime strings as the real event time", async (ctx) => {
+ if (!clickHouseAvailable) {
+ ctx.skip("ClickHouse is not available");
+ }
+
+ const quoted = toClickhouseDateTime(TICKS_MS);
+ expect(quoted).toBe(QUOTED_MS);
+
+ const stored = await insertAndRead("quoted", quoted);
+
+ expect(stored.year).toBe(2024);
+ expect(stored.formatted.startsWith(QUOTED_MS)).toBe(true);
+ });
+
+ it("overflows unquoted millisecond ticks on ClickHouse 26.8+ default parsing", async (ctx) => {
+ if (!clickHouseAvailable) {
+ ctx.skip("ClickHouse is not available");
+ }
+ if (!isClickHouseAtLeast26_8(clickHouseVersion)) {
+ ctx.skip(
+ `ClickHouse ${clickHouseVersion || "unknown"} is older than 26.8`,
+ );
+ }
+
+ const stored = await insertAndRead("unquoted-default", TICKS_MS);
+
+ expect(stored.year).toBeGreaterThanOrEqual(9000);
+ });
+});
diff --git a/worker/src/__tests__/deletedMaskCleaner.test.ts b/worker/src/__tests__/deletedMaskCleaner.test.ts
index f78b1dcdac..ee2b637c69 100644
--- a/worker/src/__tests__/deletedMaskCleaner.test.ts
+++ b/worker/src/__tests__/deletedMaskCleaner.test.ts
@@ -22,8 +22,6 @@ import {
import {
buildApplyDeletedMaskQuery,
buildMutationCountQuery,
- DELETED_MASK_CLEANER_TABLES,
- DELETED_MASK_CLEANER_WORK_QUERY,
normalizeMutationCounts,
selectCandidateToProcess,
shouldUseDeletedMaskCleanerClusterMode,
@@ -34,6 +32,8 @@ import { skipUnlessClickhouseTablesExist } from "./helpers/clickhouseTables";
const TEST_TABLE = "events_full" as const;
const PATCH_WAIT_TIMEOUT_MS = 30_000;
+const PATCH_CLEAN_TIMEOUT_MS = 60_000;
+const DRAIN_TIMEOUT_MS = 120_000;
async function skipUnlessEventsTableEnabled(ctx: TestContext): Promise<void> {
await skipUnlessClickhouseTablesExist(
@@ -53,8 +53,9 @@ async function eventually<T>(
getValue: () => Promise<T>,
predicate: (value: T) => boolean,
message: string,
+ timeoutMs: number = PATCH_WAIT_TIMEOUT_MS,
): Promise<T> {
- const deadline = Date.now() + PATCH_WAIT_TIMEOUT_MS;
+ const deadline = Date.now() + timeoutMs;
let lastValue: T | undefined;
while (Date.now() < deadline) {
@@ -95,26 +96,53 @@ function workCandidate(
};
}
-async function hasCleanerCandidate(partitionToClean: string): Promise<boolean> {
- const rows = await queryClickhouse<WorkCandidateRow>({
- query: DELETED_MASK_CLEANER_WORK_QUERY,
+async function getLeftoverPatchPartitions(): Promise<string[]> {
+ const rows = await queryClickhouse<{ partition_to_clean: string }>({
+ query: `
+ SELECT DISTINCT splitByString('-', partition)[3] AS partition_to_clean
+ FROM system.parts
+ WHERE table = {table: String}
+ AND database = {database: String}
+ AND startsWith(partition, 'patch-')
+ AND partition_to_clean <> ''
+ AND partition_to_clean <> toString(toYYYYMM(now()))
+ AND active = 1
+ `,
params: {
+ table: TEST_TABLE,
database: env.CLICKHOUSE_DB,
- tables: Array.from(DELETED_MASK_CLEANER_TABLES),
},
});
- return rows.some(
- (row) =>
- row.table === TEST_TABLE && row.partition_to_clean === partitionToClean,
- );
+ return rows.map((row) => row.partition_to_clean);
+}
+
+async function hasActivePatchPart(partitionToClean: string): Promise<boolean> {
+ const rows = await queryClickhouse<{ count: string }>({
+ query: `
+ SELECT count() AS count
+ FROM system.parts
+ WHERE table = {table: String}
+ AND database = {database: String}
+ AND startsWith(partition, 'patch-')
+ AND splitByString('-', partition)[3] = {partitionToClean: String}
+ AND active = 1
+ `,
+ params: {
+ table: TEST_TABLE,
+ database: env.CLICKHOUSE_DB,
+ partitionToClean,
+ },
+ });
+
+ return Number(rows[0]?.count ?? 0) > 0;
}
async function waitForCleanerCandidate(
partitionToClean: string,
): Promise<void> {
await eventually(
- () => hasCleanerCandidate(partitionToClean),
+ () => hasActivePatchPart(partitionToClean),
(hasCandidate) => hasCandidate,
`Timed out waiting for ClickHouse patch part ${partitionToClean}`,
);
@@ -124,9 +152,10 @@ async function waitForCleanerCandidateGone(
partitionToClean: string,
): Promise<void> {
await eventually(
- () => hasCleanerCandidate(partitionToClean),
+ () => hasActivePatchPart(partitionToClean),
(hasCandidate) => !hasCandidate,
`Timed out waiting for ClickHouse patch part ${partitionToClean} to be cleaned`,
+ PATCH_CLEAN_TIMEOUT_MS,
);
}
@@ -152,44 +181,78 @@ async function getActiveMutationCount(
return Number(rows[0]?.mutation_count ?? 0);
}
-async function waitForNoActiveMutations(): Promise<void> {
+async function waitForNoActiveMutations(
+ table: string = TEST_TABLE,
+): Promise<void> {
await eventually(
- () => getActiveMutationCount(TEST_TABLE),
+ () => getActiveMutationCount(table),
(mutationCount) => mutationCount === 0,
- "Timed out waiting for ClickHouse mutations to finish",
+ `Timed out waiting for ClickHouse mutations to finish on ${table}`,
+ PATCH_CLEAN_TIMEOUT_MS,
);
}
+function cleanerDdlConfig() {
+ return {
+ database: env.CLICKHOUSE_DB,
+ clusterEnabled: shouldUseDeletedMaskCleanerClusterMode({
+ clusterEnabled: env.CLICKHOUSE_CLUSTER_ENABLED === "true",
+ cleanerClusterModeEnabled:
+ env.LANGFUSE_CLICKHOUSE_DELETED_MASK_CLEANER_CLUSTER_MODE_ENABLED ===
+ "true",
+ }),
+ clusterName: env.CLICKHOUSE_CLUSTER_NAME,
+ };
+}
+
+async function applyDeletedMask(candidate: WorkCandidateRow): Promise<void> {
+ await commandClickhouse({
+ query: buildApplyDeletedMaskQuery(candidate, cleanerDdlConfig()),
+ clickhouseSettings: {
+ mutations_sync: "2",
+ },
+ });
+
+ // Mutation count can still be 0 before the ALTER is visible. The patch
+ // disappearing from system.parts is the completion signal.
+ await waitForCleanerCandidateGone(candidate.partition_to_clean);
+ await waitForNoActiveMutations(candidate.table);
+}
+
async function applyDeletedMaskIfNeeded(
partitionToClean: string,
): Promise<void> {
- if (!(await hasCleanerCandidate(partitionToClean))) {
+ if (!(await hasActivePatchPart(partitionToClean))) {
return;
}
- await commandClickhouse({
- query: buildApplyDeletedMaskQuery(
- {
- partition: `patch-cleanup-${partitionToClean}`,
- table: TEST_TABLE,
- partition_to_clean: partitionToClean,
- total_rows: 0,
- },
- {
- database: env.CLICKHOUSE_DB,
- clusterEnabled: shouldUseDeletedMaskCleanerClusterMode({
- clusterEnabled: env.CLICKHOUSE_CLUSTER_ENABLED === "true",
- cleanerClusterModeEnabled:
- env.LANGFUSE_CLICKHOUSE_DELETED_MASK_CLEANER_CLUSTER_MODE_ENABLED ===
- "true",
- }),
- clusterName: env.CLICKHOUSE_CLUSTER_NAME,
- },
- ),
+ await applyDeletedMask({
+ partition: `patch-cleanup-${partitionToClean}`,
+ table: TEST_TABLE,
+ partition_to_clean: partitionToClean,
+ total_rows: 0,
});
+}
+
+async function drainCleanerCandidates(): Promise<void> {
+ const deadline = Date.now() + DRAIN_TIMEOUT_MS;
- await waitForNoActiveMutations();
- await waitForCleanerCandidateGone(partitionToClean);
+ while (Date.now() < deadline) {
+ const partitions = await getLeftoverPatchPartitions();
+ if (partitions.length === 0) {
+ return;
+ }
+
+ for (const partition of partitions) {
+ await applyDeletedMaskIfNeeded(partition);
+ }
+ }
+
+ throw new Error(
+ `Timed out draining leftover deleted-mask candidates: ${JSON.stringify(
+ await getLeftoverPatchPartitions(),
+ )}`,
+ );
}
async function createEventPatchParts(
@@ -209,11 +272,11 @@ async function createEventPatchParts(
trace_id: randomUUID(),
span_id: randomUUID(),
name: `deleted-mask-cleaner-${partition}`,
- start_time: timestamp * 1000,
- end_time: timestamp * 1000,
- created_at: timestamp * 1000,
- updated_at: timestamp * 1000,
- event_ts: timestamp * 1000,
+ start_time: timestamp,
+ end_time: timestamp,
+ created_at: timestamp,
+ updated_at: timestamp,
+ event_ts: timestamp,
}),
),
),
@@ -390,7 +453,7 @@ describe.sequential("DeletedMaskCleaner integration", () => {
await applyDeletedMaskIfNeeded(partition);
}
cleanupPartitions.clear();
- }, 90_000);
+ }, 150_000);
it("applies a deleted mask for a real events patch partition", async (ctx) => {
await skipUnlessEventsTableEnabled(ctx);
@@ -398,6 +461,8 @@ describe.sequential("DeletedMaskCleaner integration", () => {
const partitionToClean = getRandomPastMonthPartition();
cleanupPartitions.add(partitionToClean);
+ await drainCleanerCandidates();
+
await createEventPatchParts([
{
partition: partitionToClean,
@@ -406,13 +471,19 @@ describe.sequential("DeletedMaskCleaner integration", () => {
},
]);
- await expect(hasCleanerCandidate(partitionToClean)).resolves.toBe(true);
+ await expect(hasActivePatchPart(partitionToClean)).resolves.toBe(true);
const cleaner = new DeletedMaskCleaner();
- await cleaner.processBatch();
-
- await waitForNoActiveMutations();
- await waitForCleanerCandidateGone(partitionToClean);
+ await eventually(
+ async () => {
+ await cleaner.processBatch();
+ await waitForNoActiveMutations();
+ return hasActivePatchPart(partitionToClean);
+ },
+ (stillPresent) => !stillPresent,
+ `Timed out waiting for ClickHouse patch part ${partitionToClean} to be cleaned`,
+ PATCH_CLEAN_TIMEOUT_MS,
+ );
expect(await redis?.get(DELETED_MASK_CLEANER_LOCK_KEY)).toBeNull();
- }, 90_000);
+ }, 240_000);
});
diff --git a/worker/src/__tests__/eventPropagationJob.test.ts b/worker/src/__tests__/eventPropagationJob.test.ts
index 45e848ad6b..a28db10d58 100644
--- a/worker/src/__tests__/eventPropagationJob.test.ts
+++ b/worker/src/__tests__/eventPropagationJob.test.ts
@@ -16,6 +16,7 @@ import {
createTracesCh,
queryClickhouse,
redis,
+ toClickhouseDateTime,
} from "@langfuse/shared/src/server";
import { handleEventPropagationJob } from "../features/eventPropagation/handleEventPropagationJob";
import { TableName } from "../services/ClickhouseWriter";
@@ -107,7 +108,9 @@ describe("handleEventPropagationJob", () => {
ingestion_api_key: "pk-lf-propagation-test",
ingestion_sdk_name: "langfuse-js",
ingestion_sdk_version: "4.2.0",
- s3_first_seen_timestamp: oldEnoughForPropagation,
+ s3_first_seen_timestamp: toClickhouseDateTime(
+ oldEnoughForPropagation,
+ ),
},
],
});
diff --git a/worker/src/__tests__/ingestionFileDeletion.test.ts b/worker/src/__tests__/ingestionFileDeletion.test.ts
index af8c02e99e..590519bd47 100644
--- a/worker/src/__tests__/ingestionFileDeletion.test.ts
+++ b/worker/src/__tests__/ingestionFileDeletion.test.ts
@@ -8,6 +8,7 @@ import {
removeIngestionEventsFromS3AndDeleteClickhouseRefsForProject,
StorageService,
StorageServiceFactory,
+ toClickhouseDateTime,
} from "@langfuse/shared/src/server";
import { randomUUID } from "crypto";
import { env } from "../env";
@@ -73,8 +74,8 @@ describe("ingestion file deletion pipeline", () => {
event_id: randomUUID(),
bucket_name: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
bucket_path: r.bucketPath,
- created_at: new Date().getTime(),
- updated_at: new Date().getTime(),
+ created_at: toClickhouseDateTime(),
+ updated_at: toClickhouseDateTime(),
})),
});
diff --git a/worker/src/__tests__/scoreDeletion.test.ts b/worker/src/__tests__/scoreDeletion.test.ts
index f2313f3d1f..f61da42784 100644
--- a/worker/src/__tests__/scoreDeletion.test.ts
+++ b/worker/src/__tests__/scoreDeletion.test.ts
@@ -8,6 +8,7 @@ import {
getScoresByIds,
StorageService,
StorageServiceFactory,
+ toClickhouseDateTime,
} from "@langfuse/shared/src/server";
import { randomUUID } from "crypto";
import { env } from "../env";
@@ -74,8 +75,8 @@ describe("score deletion", () => {
event_id: randomUUID(),
bucket_name: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
bucket_path: `${projectId}/score/${scoreId}-score.json`,
- created_at: new Date().getTime(),
- updated_at: new Date().getTime(),
+ created_at: toClickhouseDateTime(),
+ updated_at: toClickhouseDateTime(),
},
],
});
diff --git a/worker/src/__tests__/traceDeletion.test.ts b/worker/src/__tests__/traceDeletion.test.ts
index c9d7eae52c..ca2b96d67f 100644
--- a/worker/src/__tests__/traceDeletion.test.ts
+++ b/worker/src/__tests__/traceDeletion.test.ts
@@ -19,6 +19,7 @@ import {
removeIngestionEventsFromS3AndDeleteClickhouseRefsForTraces,
StorageService,
StorageServiceFactory,
+ toClickhouseDateTime,
} from "@langfuse/shared/src/server";
import { randomUUID } from "crypto";
import { processClickhouseTraceDelete } from "../features/traces/processClickhouseTraceDelete";
@@ -439,8 +440,8 @@ describe("trace deletion", () => {
event_id: randomUUID(),
bucket_name: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
bucket_path: `${projectId}/traces/${traceId}-trace.json`,
- created_at: new Date().getTime(),
- updated_at: new Date().getTime(),
+ created_at: toClickhouseDateTime(),
+ updated_at: toClickhouseDateTime(),
},
{
id: randomUUID(),
@@ -450,8 +451,8 @@ describe("trace deletion", () => {
event_id: randomUUID(),
bucket_name: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
bucket_path: `${projectId}/observation/${traceId}-observation.json`,
- created_at: new Date().getTime(),
- updated_at: new Date().getTime(),
+ created_at: toClickhouseDateTime(),
+ updated_at: toClickhouseDateTime(),
},
{
id: randomUUID(),
@@ -461,8 +462,8 @@ describe("trace deletion", () => {
event_id: randomUUID(),
bucket_name: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
bucket_path: `${projectId}/score/${traceId}-score.json`,
- created_at: new Date().getTime(),
- updated_at: new Date().getTime(),
+ created_at: toClickhouseDateTime(),
+ updated_at: toClickhouseDateTime(),
},
],
});
@@ -509,8 +510,8 @@ describe("trace deletion", () => {
event_id: randomUUID(),
bucket_name: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
bucket_path: bucketPath,
- created_at: new Date().getTime(),
- updated_at: new Date().getTime(),
+ created_at: toClickhouseDateTime(),
+ updated_at: toClickhouseDateTime(),
},
],
});
@@ -594,8 +595,8 @@ describe("trace deletion", () => {
event_id: randomUUID(),
bucket_name: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
bucket_path: bucketPath,
- created_at: new Date().getTime(),
- updated_at: new Date().getTime(),
+ created_at: toClickhouseDateTime(),
+ updated_at: toClickhouseDateTime(),
},
],
});
@@ -675,8 +676,8 @@ describe("trace deletion", () => {
event_id: randomUUID(),
bucket_name: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
bucket_path: `${projectId}/traces/${traceId}-trace.json`,
- created_at: new Date().getTime(),
- updated_at: new Date().getTime(),
+ created_at: toClickhouseDateTime(),
+ updated_at: toClickhouseDateTime(),
},
{
id: randomUUID(),
@@ -686,8 +687,8 @@ describe("trace deletion", () => {
event_id: randomUUID(),
bucket_name: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
bucket_path: `${projectId}/observation/${observationId}-observation.json`,
- created_at: new Date().getTime(),
- updated_at: new Date().getTime(),
+ created_at: toClickhouseDateTime(),
+ updated_at: toClickhouseDateTime(),
},
{
id: randomUUID(),
@@ -697,8 +698,8 @@ describe("trace deletion", () => {
event_id: randomUUID(),
bucket_name: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
bucket_path: `${projectId}/score/${scoreId}-score.json`,
- created_at: new Date().getTime(),
- updated_at: new Date().getTime(),
+ created_at: toClickhouseDateTime(),
+ updated_at: toClickhouseDateTime(),
},
],
});
diff --git a/worker/src/features/eventPropagation/handleExperimentBackfill.ts b/worker/src/features/eventPropagation/handleExperimentBackfill.ts
index c197d70377..6c187c6147 100644
--- a/worker/src/features/eventPropagation/handleExperimentBackfill.ts
+++ b/worker/src/features/eventPropagation/handleExperimentBackfill.ts
@@ -3,6 +3,7 @@ import {
queryClickhouse,
redis,
convertDateToClickhouseDateTime,
+ toClickhouseDateTime,
flattenJsonToPathArrays,
recordGauge,
UNKNOWN_INGESTION_SDK_VALUE,
@@ -536,7 +537,7 @@ export function convertEnrichedSpansToEventRecords(
opts: { plainEventTsFromSource?: boolean } = {},
): EventRecordInsertType[] {
const records: EventRecordInsertType[] = [];
- const now = Date.now() * 1000; // microseconds
+ const now = toClickhouseDateTime();
for (const span of spans) {
// Flatten metadata for ClickHouse Array(String) columns
@@ -557,7 +558,7 @@ export function convertEnrichedSpansToEventRecords(
const isPlain = !span.experiment_id;
const eventTs =
opts.plainEventTsFromSource && isPlain && span.event_ts
- ? new Date(span.event_ts).getTime() * 1000
+ ? toClickhouseDateTime(span.event_ts)
: now;
const eventRecord: EventRecordInsertType = {
@@ -585,10 +586,10 @@ export function convertEnrichedSpansToEventRecords(
level: span.level || "DEFAULT",
status_message: span.status_message || undefined,
- start_time: new Date(span.start_time).getTime() * 1000,
- end_time: span.end_time ? new Date(span.end_time).getTime() * 1000 : null,
+ start_time: toClickhouseDateTime(span.start_time),
+ end_time: span.end_time ? toClickhouseDateTime(span.end_time) : null,
completion_start_time: span.completion_start_time
- ? new Date(span.completion_start_time).getTime() * 1000
+ ? toClickhouseDateTime(span.completion_start_time)
: null,
prompt_id: span.prompt_id || "",
diff --git a/worker/src/queues/ingestionQueue.ts b/worker/src/queues/ingestionQueue.ts
index 5d010b21fa..d40d2193de 100644
--- a/worker/src/queues/ingestionQueue.ts
+++ b/worker/src/queues/ingestionQueue.ts
@@ -21,6 +21,7 @@ import {
traceException,
type IngestionAttribution,
UNKNOWN_INGESTION_SDK_VALUE,
+ toClickhouseDateTime,
} from "@langfuse/shared/src/server";
import { prisma } from "@langfuse/shared/src/db";
@@ -306,9 +307,9 @@ export const ingestionQueueProcessorBuilder = (
event_id: job.data.payload.data.fileKey,
bucket_name: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
bucket_path: `${bucketPrefix}${fileName}`,
- created_at: new Date().getTime(),
- updated_at: new Date().getTime(),
- event_ts: new Date().getTime(),
+ created_at: toClickhouseDateTime(),
+ updated_at: toClickhouseDateTime(),
+ event_ts: toClickhouseDateTime(),
is_deleted: 0,
});
}
diff --git a/worker/src/services/IngestionService/index.ts b/worker/src/services/IngestionService/index.ts
index 7900f8dab2..5b53adc40c 100644
--- a/worker/src/services/IngestionService/index.ts
+++ b/worker/src/services/IngestionService/index.ts
@@ -14,6 +14,8 @@ import {
import {
ClickhouseClientType,
convertDateToClickhouseDateTime,
+ toClickhouseDateTime,
+ parseClickhouseUTCDateTimeFormat,
convertObservationReadToInsert,
convertScoreReadToInsert,
convertTraceReadToInsert,
@@ -355,7 +357,7 @@ export class IngestionService {
: null,
]);
- const now = this.getMicrosecondTimestamp();
+ const now = toClickhouseDateTime();
// Flatten raw metadata first (before stringification destroys nested structure)
const flattened = flattenJsonToPathArrays(metadata);
@@ -395,10 +397,10 @@ export class IngestionService {
status_message: eventData.statusMessage,
// Timestamps
- start_time: this.getMicrosecondTimestamp(eventData.startTimeISO),
- end_time: this.getMicrosecondTimestamp(eventData.endTimeISO),
+ start_time: toClickhouseDateTime(eventData.startTimeISO),
+ end_time: toClickhouseDateTime(eventData.endTimeISO),
completion_start_time: eventData.completionStartTime
- ? this.getMicrosecondTimestamp(eventData.completionStartTime)
+ ? toClickhouseDateTime(eventData.completionStartTime)
: null,
// Prompt
@@ -552,12 +554,10 @@ export class IngestionService {
if (!runData || !itemData) return [];
- const timestamp = event.body.createdAt
- ? new Date(event.body.createdAt).getTime()
- : new Date().getTime();
+ const timestamp = toClickhouseDateTime(event.body.createdAt);
const datasetItemVersion = itemData.validFrom
- ? itemData.validFrom.getTime()
+ ? toClickhouseDateTime(itemData.validFrom)
: null;
return [
@@ -580,7 +580,7 @@ export class IngestionService {
dataset_run_metadata: runData.metadata
? convertPostgresJsonToMetadataRecord(runData.metadata)
: {},
- dataset_run_created_at: runData.createdAt.getTime(),
+ dataset_run_created_at: toClickhouseDateTime(runData.createdAt),
// enriched with item data
dataset_item_version: datasetItemVersion,
dataset_item_input: JSON.stringify(itemData.input),
@@ -666,7 +666,7 @@ export class IngestionService {
id: entityId,
project_id: projectId,
environment: validatedScore.environment,
- timestamp: this.getMillisecondTimestamp(scoreEvent.timestamp),
+ timestamp: toClickhouseDateTime(scoreEvent.timestamp),
name: validatedScore.name,
value: validatedScore.value,
source: validatedScore.source,
@@ -687,9 +687,9 @@ export class IngestionService {
ingestion_api_key: attribution.ingestionApiKey,
ingestion_sdk_name: attribution.ingestionSdkName,
ingestion_sdk_version: attribution.ingestionSdkVersion,
- created_at: Date.now(),
- updated_at: Date.now(),
- event_ts: new Date(scoreEvent.timestamp).getTime(),
+ created_at: toClickhouseDateTime(),
+ updated_at: toClickhouseDateTime(),
+ event_ts: toClickhouseDateTime(scoreEvent.timestamp),
is_deleted: 0,
};
// Gracefully handle any score schema validation errors, skip the score insert and reject silently.
@@ -768,7 +768,8 @@ export class IngestionService {
scoreRecords,
});
finalScoreRecord.created_at =
- clickhouseScoreRecord?.created_at ?? createdAtTimestamp.getTime();
+ clickhouseScoreRecord?.created_at ??
+ toClickhouseDateTime(createdAtTimestamp);
this.clickHouseWriter.addToQueue(TableName.Scores, finalScoreRecord);
}
@@ -845,7 +846,8 @@ export class IngestionService {
traceRecords,
});
finalTraceRecord.created_at =
- clickhouseTraceRecord?.created_at ?? createdAtTimestamp.getTime();
+ clickhouseTraceRecord?.created_at ??
+ toClickhouseDateTime(createdAtTimestamp);
finalTraceRecord.input = finalIO.input ?? clickhouseTraceRecord?.input;
finalTraceRecord.output = finalIO.output ?? clickhouseTraceRecord?.output;
@@ -912,7 +914,9 @@ export class IngestionService {
payload: {
projectId,
traceId: entityId,
- exactTimestamp: new Date(finalTraceRecord.timestamp),
+ exactTimestamp: parseClickhouseUTCDateTimeFormat(
+ finalTraceRecord.timestamp,
+ ),
traceEnvironment: finalTraceRecord.environment,
},
id: randomUUID(),
@@ -989,7 +993,8 @@ export class IngestionService {
clickhouseObservationRecord,
});
mergedObservationRecord.created_at =
- clickhouseObservationRecord?.created_at ?? createdAtTimestamp.getTime();
+ clickhouseObservationRecord?.created_at ??
+ toClickhouseDateTime(createdAtTimestamp);
mergedObservationRecord.level = mergedObservationRecord.level ?? "DEFAULT";
// Search for the first non-null input and output in the observation events and set them on the merged result.
@@ -1075,13 +1080,13 @@ export class IngestionService {
timestamp: finalObservationRecord.start_time,
project_id: projectId,
environment: finalObservationRecord.environment,
- created_at: Date.now(),
- updated_at: Date.now(),
+ created_at: toClickhouseDateTime(),
+ updated_at: toClickhouseDateTime(),
metadata: {},
tags: [],
bookmarked: false,
public: false,
- event_ts: Date.now(),
+ event_ts: toClickhouseDateTime(),
is_deleted: 0,
};
@@ -1220,7 +1225,7 @@ export class IngestionService {
result = overwriteObject(result, record, immutableEntityKeys);
}
- result.event_ts = new Date().getTime();
+ result.event_ts = toClickhouseDateTime();
return result;
}
@@ -1842,7 +1847,7 @@ export class IngestionService {
return traceEventList.map((trace) => {
const traceRecord: TraceRecordInsertType = {
id: entityId,
- timestamp: this.getMillisecondTimestamp(
+ timestamp: toClickhouseDateTime(
trace.body.timestamp ?? trace.timestamp,
),
// timestamp: ("timestamp" in trace.body && trace.body.timestamp
@@ -1865,9 +1870,9 @@ export class IngestionService {
// input: this.stringify(trace.body.input),
// output: this.stringify(trace.body.output), // convert even json to string
session_id: trace.body.sessionId,
- created_at: Date.now(),
- updated_at: Date.now(),
- event_ts: new Date(trace.timestamp).getTime(),
+ created_at: toClickhouseDateTime(),
+ updated_at: toClickhouseDateTime(),
+ event_ts: toClickhouseDateTime(trace.timestamp),
is_deleted: 0,
};
@@ -1990,19 +1995,17 @@ export class IngestionService {
name: obs.body.name,
environment:
"environment" in obs.body ? obs.body.environment : "default",
- start_time: this.getMillisecondTimestamp(
- obs.body.startTime ?? obs.timestamp,
- ),
+ start_time: toClickhouseDateTime(obs.body.startTime ?? obs.timestamp),
// start_time: ("startTime" in obs.body && obs.body.startTime
// ? this.getMillisecondTimestamp(obs.body.startTime)
// : undefined) as number, // Casting here is dirty, but our requirement is to have a start_time _after_ the merge
end_time:
"endTime" in obs.body && obs.body.endTime
- ? this.getMillisecondTimestamp(obs.body.endTime)
+ ? toClickhouseDateTime(obs.body.endTime)
: undefined,
completion_start_time:
"completionStartTime" in obs.body && obs.body.completionStartTime
- ? this.getMillisecondTimestamp(obs.body.completionStartTime)
+ ? toClickhouseDateTime(obs.body.completionStartTime)
: undefined,
metadata: obs.body.metadata
? convertJsonSchemaToRecord(obs.body.metadata)
@@ -2030,9 +2033,9 @@ export class IngestionService {
prompt_id: prompt?.id,
prompt_name: prompt?.name,
prompt_version: prompt?.version,
- created_at: Date.now(),
- updated_at: Date.now(),
- event_ts: new Date(obs.timestamp).getTime(),
+ created_at: toClickhouseDateTime(),
+ updated_at: toClickhouseDateTime(),
+ event_ts: toClickhouseDateTime(obs.timestamp),
is_deleted: 0,
};
@@ -2048,14 +2051,6 @@ export class IngestionService {
return typeof obj === "string" ? obj : JSON.stringify(obj);
}
- private getMicrosecondTimestamp(timestamp?: string | null): number {
- return timestamp ? new Date(timestamp).getTime() * 1000 : Date.now() * 1000;
- }
-
- private getMillisecondTimestamp(timestamp?: string | null): number {
- return timestamp ? new Date(timestamp).getTime() : Date.now();
- }
-
/**
* Returns a partition-aware timestamp for staging table writes.
* If the createdAtTimestamp is within the last 2 minutes, returns it as-is.
@@ -2069,7 +2064,7 @@ export class IngestionService {
* that data is processed correctly. Worst case is slightly more duplication in the events table
* which should resolve automatically using the ReplacingMergeTree.
*/
- private getPartitionAwareTimestamp(createdAtTimestamp: Date): number {
+ private getPartitionAwareTimestamp(createdAtTimestamp: Date): string {
const now = Date.now();
const createdAt = createdAtTimestamp.getTime();
const ageInMs = now - createdAt;
@@ -2077,7 +2072,7 @@ export class IngestionService {
// If the createdAtTimestamp is within the last 2 minutes, use it
// Otherwise, use the current timestamp to avoid updating old partitions
- return ageInMs < twoMinutesInMs ? createdAt : now;
+ return toClickhouseDateTime(ageInMs < twoMinutesInMs ? createdAt : now);
}
}
diff --git a/worker/src/services/IngestionService/tests/IngestionService.integration.test.ts b/worker/src/services/IngestionService/tests/IngestionService.integration.test.ts
index c11d261405..03e7c9ea2f 100644
--- a/worker/src/services/IngestionService/tests/IngestionService.integration.test.ts
+++ b/worker/src/services/IngestionService/tests/IngestionService.integration.test.ts
@@ -139,7 +139,7 @@ describe("Ingestion end-to-end tests", () => {
const secondSpanId = randomUUID();
const wireStartTimeA = "2026-01-01T12:00:00.123456Z";
const wireStartTimeB = "2026-01-01T12:00:00.123789Z";
- const normalizedStartTime = Date.parse(wireStartTimeA) * 1000;
+ const normalizedStartTime = "2026-01-01 12:00:00.123";
const eventRecords = await Promise.all(
[
@@ -169,9 +169,6 @@ describe("Ingestion end-to-end tests", () => {
normalizedStartTime,
normalizedStartTime,
]);
- expect(eventRecords.every((record) => record.start_time % 1000 === 0)).toBe(
- true,
- );
await Promise.all(
eventRecords.map((record) => ingestionService.writeEventRecord(record)),
diff --git a/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts b/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts
index 2b59a9512c..4e5811832e 100644
--- a/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts
+++ b/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts
@@ -2,6 +2,9 @@ import { beforeEach, expect, describe, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
applyObservationFieldOverflow: vi.fn(),
+ validateAndInflateScoreOverride: undefined as
+ | ((...args: unknown[]) => unknown)
+ | undefined,
}));
vi.mock(
@@ -11,6 +14,20 @@ vi.mock(
}),
);
+vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
+ const actual =
+ await importOriginal<typeof import("@langfuse/shared/src/server")>();
+ return {
+ ...actual,
+ validateAndInflateScore: (...args: unknown[]) =>
+ mocks.validateAndInflateScoreOverride
+ ? mocks.validateAndInflateScoreOverride(...args)
+ : (actual.validateAndInflateScore as (...a: unknown[]) => unknown)(
+ ...args,
+ ),
+ };
+});
+
import { IngestionService } from "../../IngestionService";
import {
convertDateToClickhouseDateTime,
@@ -26,6 +43,7 @@ describe("IngestionService unit tests", () => {
mocks.applyObservationFieldOverflow.mockImplementation(
async (eventRecord) => eventRecord,
);
+ mocks.validateAndInflateScoreOverride = undefined;
});
it("writes the final serialized event size instead of the raw OTEL span size", async () => {
@@ -802,12 +820,9 @@ describe("IngestionService unit tests", () => {
vi.spyOn(ingestionService as any, "getClickhouseRecord").mockResolvedValue(
null,
);
- vi.spyOn(
- ingestionService as any,
- "getMillisecondTimestamp",
- ).mockImplementation(() => {
- throw new Error("unexpected timestamp failure");
- });
+ mocks.validateAndInflateScoreOverride = () => {
+ throw new Error("unexpected score validation failure");
+ };
await expect(
(ingestionService as any).processScoreEventList({
@@ -860,12 +875,9 @@ describe("IngestionService unit tests", () => {
timestamp: new Date(timestamp).getTime(),
}),
);
- vi.spyOn(
- ingestionService as any,
- "getMillisecondTimestamp",
- ).mockImplementation(() => {
- throw new Error("unexpected timestamp failure");
- });
+ mocks.validateAndInflateScoreOverride = () => {
+ throw new Error("unexpected score validation failure");
+ };
await expect(
(ingestionService as any).processScoreEventList({
diff --git a/worker/src/services/IngestionService/tests/scoreValidationDroppedMetric.unit.test.ts b/worker/src/services/IngestionService/tests/scoreValidationDroppedMetric.unit.test.ts
index 6bdce7a235..01732397ea 100644
--- a/worker/src/services/IngestionService/tests/scoreValidationDroppedMetric.unit.test.ts
+++ b/worker/src/services/IngestionService/tests/scoreValidationDroppedMetric.unit.test.ts
@@ -198,12 +198,9 @@ describe("score validation drop metric", () => {
it("does not emit for unexpected errors — those reject the batch instead of dropping it", async () => {
const { ingestionService, addToQueue } = createService();
- vi.spyOn(
- ingestionService as any,
- "getMillisecondTimestamp",
- ).mockImplementation(() => {
- throw new Error("unexpected timestamp failure");
- });
+ mocks.validateAndInflateScoreOverride = () => {
+ throw new Error("unexpected score validation failure");
+ };
await expect(
processScores(ingestionService, [validScoreEvent()]),