🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(ingestion): merge same-timestamp events in arrival order (#16925)
David Trainaf030468973e6
diff --git a/packages/shared/src/server/services/StorageService.ts b/packages/shared/src/server/services/StorageService.tsindex f878e5a822..cd3da4d6c6 100644--- a/packages/shared/src/server/services/StorageService.ts+++ b/packages/shared/src/server/services/StorageService.ts@@ -1737,6 +1737,8 @@ class OCIObjectStorageService implements StorageService {namespaceName,bucketName: this.bucketName,prefix,+ // Object summaries only carry `name` unless asked for more.+ fields: "name,timeCreated",};const resp = await client.listObjects(req);const objects = ((resp as any).listObjects?.objects ?? []) as Array<{diff --git a/worker/src/queues/ingestionQueue.ts b/worker/src/queues/ingestionQueue.tsindex d40d2193de..186f54dfad 100644--- a/worker/src/queues/ingestionQueue.ts+++ b/worker/src/queues/ingestionQueue.ts@@ -183,6 +183,12 @@ export const ingestionQueueProcessorBuilder = (events.push(...(Array.isArray(parsedFile) ? parsedFile : [parsedFile]));} else {eventFiles = await s3Client.listFiles(s3Prefix);+ // Listings are ordered by key (client-supplied event ids), not by+ // upload time. Sort so arrival order is upload order; score merges+ // rely on that for same-timestamp re-sends.+ eventFiles.sort(+ (a, b) => a.createdAt.getTime() - b.createdAt.getTime(),+ );// Process files in batches// If a user has 5k events, this will likely take 100 seconds.@@ -226,11 +232,8 @@ export const ingestionQueueProcessorBuilder = (totalS3DownloadSizeBytes,);- const firstS3WriteTime =- eventFiles- .map((fileRef) => fileRef.createdAt)- .sort()- .shift() ?? new Date();+ // eventFiles is sorted by createdAt above.+ const firstS3WriteTime = eventFiles[0]?.createdAt ?? new Date();if (events.length === 0) {logger.warn(diff --git a/worker/src/services/IngestionService/index.ts b/worker/src/services/IngestionService/index.tsindex 5b53adc40c..48a4719850 100644--- a/worker/src/services/IngestionService/index.ts+++ b/worker/src/services/IngestionService/index.ts@@ -620,8 +620,15 @@ export class IngestionService {} = params;if (scoreEventList.length === 0) return;- const timeSortedEvents =- IngestionService.toTimeSortedEventList(scoreEventList);+ // Re-sending a score with the same id and timestamp is the documented way+ // to overwrite it, so ties must keep arrival order for the later one to+ // win. Score events are all creates, so no create-first rule is needed.+ const timeSortedEvents = scoreEventList+ .slice()+ .sort(+ (a, b) =>+ new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime(),+ );const minTimestamp = Math.min(...timeSortedEvents.flatMap((e) =>@@ -1231,14 +1238,20 @@ export class IngestionService {}private static toTimeSortedEventList<- T extends TraceEventType | ScoreEventType | ObservationEvent,+ T extends TraceEventType | ObservationEvent,>(eventList: T[]): T[] {return eventList.slice().sort((a, b) => {const aTimestamp = new Date(a.timestamp).getTime();const bTimestamp = new Date(b.timestamp).getTime();if (aTimestamp === bTimestamp) {- return a.type.includes("create") ? -1 : 1; // create events should come first+ // Creates sort before updates. For two tied creates this is not a+ // total order: V8 places the later-arriving one first, so the+ // first-arriving create wins the merge. OTel relies on that: a root+ // span's trace-create and a child span's trace update share the span+ // start time, and the child's explicit fields must beat the root's+ // derived ones. Score events use their own stable sort instead.+ return a.type.includes("create") ? -1 : 1;}return aTimestamp - bTimestamp;diff --git a/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts b/worker/src/services/IngestionService/tests/IngestionService.unit.test.tsindex 4e5811832e..d587983a7f 100644--- a/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts+++ b/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts@@ -495,6 +495,17 @@ describe("IngestionService unit tests", () => {expect(sortedEventList).not.toBe(records); // Ensure that the original array is not mutated});+ it("puts the first-arriving of tied create events last so it wins the merge", () => {+ // OTel stamps a root span's trace-create and a child span's trace update+ // with the same start time; the child arrives first and must win.+ const first = { timestamp: 1, type: "trace-create", id: "first" };+ const second = { timestamp: 1, type: "trace-create", id: "second" };++ expect(+ (IngestionService as any).toTimeSortedEventList([first, second]),+ ).toEqual([second, first]);+ });+it("correctly convert Date to Clickhouse DateTime", async () => {const date = new Date("2024-10-12T12:13:14.123Z");@@ -895,4 +906,71 @@ describe("IngestionService unit tests", () => {expect(addToQueue).not.toHaveBeenCalled();});++ it("keeps the last-arriving event when score events share a timestamp", async () => {+ // Re-sending the full score with the same id, name, and timestamp is the+ // documented way to overwrite it, so the events tie on timestamp and+ // arrival order has to decide the winner.+ const timestamp = "2026-08-31T12:00:00.000Z";+ const buildEvent = (id: string, value: string): ScoreEventType => ({+ id,+ timestamp,+ type: "score-create",+ body: {+ id: "score-id",+ name: "quality",+ dataType: "TEXT",+ source: "API",+ sessionId: "session-id",+ environment: "default",+ value,+ },+ });+ const first = buildEvent("event-first", "first");+ const second = buildEvent("event-second", "second");++ const mergeAndReadValue = async (events: ScoreEventType[]) => {+ const addToQueue = vi.fn();+ const ingestionService = new IngestionService(+ {} as any,+ {} as any,+ { addToQueue } as any,+ {} as any,+ );+ vi.spyOn(+ ingestionService as any,+ "getClickhouseRecord",+ ).mockResolvedValue(null);++ await ingestionService.mergeAndWrite({+ eventType: "score",+ projectId: "project-id",+ entityId: "score-id",+ createdAtTimestamp: new Date(timestamp),+ events,+ forwardToEventsTable: false,+ attribution: {+ ingestionApiKey: "pk-lf-unit-test",+ ingestionSdkName: "langfuse-test",+ ingestionSdkVersion: "0.0.0",+ },+ });++ expect(addToQueue).toHaveBeenCalledWith(+ TableName.Scores,+ expect.anything(),+ );+ return addToQueue.mock.calls[0]?.[1].string_value;+ };++ for (const events of [+ [first, second],+ [second, first],+ ]) {+ await expect(+ mergeAndReadValue(events),+ `arrival order ${events.map((e) => e.id).join(", ")}`,+ ).resolves.toBe(events.at(-1)?.body.value);+ }+ });});