Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(ingestion): merge same-timestamp events in arrival order (#16925)

David Trainaf030468973e6
diff --git a/packages/shared/src/server/services/StorageService.ts b/packages/shared/src/server/services/StorageService.ts
index f878e5a822..cd3da4d6c6 100644
--- a/packages/shared/src/server/services/StorageService.ts
+++ b/packages/shared/src/server/services/StorageService.ts
@@ -1737,6 +1737,8 @@ class OCIObjectStorageService implements StorageService {
namespaceName,
bucketName: this.bucketName,
prefix,
+ // Object summaries only carry `name` unless asked for more.
+ fields: "name,timeCreated",
};
const resp = await client.listObjects(req);
const objects = ((resp as any).listObjects?.objects ?? []) as Array<{
diff --git a/worker/src/queues/ingestionQueue.ts b/worker/src/queues/ingestionQueue.ts
index d40d2193de..186f54dfad 100644
--- a/worker/src/queues/ingestionQueue.ts
+++ b/worker/src/queues/ingestionQueue.ts
@@ -183,6 +183,12 @@ export const ingestionQueueProcessorBuilder = (
events.push(...(Array.isArray(parsedFile) ? parsedFile : [parsedFile]));
} else {
eventFiles = await s3Client.listFiles(s3Prefix);
+ // Listings are ordered by key (client-supplied event ids), not by
+ // upload time. Sort so arrival order is upload order; score merges
+ // rely on that for same-timestamp re-sends.
+ eventFiles.sort(
+ (a, b) => a.createdAt.getTime() - b.createdAt.getTime(),
+ );
// Process files in batches
// If a user has 5k events, this will likely take 100 seconds.
@@ -226,11 +232,8 @@ export const ingestionQueueProcessorBuilder = (
totalS3DownloadSizeBytes,
);
- const firstS3WriteTime =
- eventFiles
- .map((fileRef) => fileRef.createdAt)
- .sort()
- .shift() ?? new Date();
+ // eventFiles is sorted by createdAt above.
+ const firstS3WriteTime = eventFiles[0]?.createdAt ?? new Date();
if (events.length === 0) {
logger.warn(
diff --git a/worker/src/services/IngestionService/index.ts b/worker/src/services/IngestionService/index.ts
index 5b53adc40c..48a4719850 100644
--- a/worker/src/services/IngestionService/index.ts
+++ b/worker/src/services/IngestionService/index.ts
@@ -620,8 +620,15 @@ export class IngestionService {
} = params;
if (scoreEventList.length === 0) return;
- const timeSortedEvents =
- IngestionService.toTimeSortedEventList(scoreEventList);
+ // Re-sending a score with the same id and timestamp is the documented way
+ // to overwrite it, so ties must keep arrival order for the later one to
+ // win. Score events are all creates, so no create-first rule is needed.
+ const timeSortedEvents = scoreEventList
+ .slice()
+ .sort(
+ (a, b) =>
+ new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime(),
+ );
const minTimestamp = Math.min(
...timeSortedEvents.flatMap((e) =>
@@ -1231,14 +1238,20 @@ export class IngestionService {
}
private static toTimeSortedEventList<
- T extends TraceEventType | ScoreEventType | ObservationEvent,
+ T extends TraceEventType | ObservationEvent,
>(eventList: T[]): T[] {
return eventList.slice().sort((a, b) => {
const aTimestamp = new Date(a.timestamp).getTime();
const bTimestamp = new Date(b.timestamp).getTime();
if (aTimestamp === bTimestamp) {
- return a.type.includes("create") ? -1 : 1; // create events should come first
+ // Creates sort before updates. For two tied creates this is not a
+ // total order: V8 places the later-arriving one first, so the
+ // first-arriving create wins the merge. OTel relies on that: a root
+ // span's trace-create and a child span's trace update share the span
+ // start time, and the child's explicit fields must beat the root's
+ // derived ones. Score events use their own stable sort instead.
+ return a.type.includes("create") ? -1 : 1;
}
return aTimestamp - bTimestamp;
diff --git a/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts b/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts
index 4e5811832e..d587983a7f 100644
--- a/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts
+++ b/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts
@@ -495,6 +495,17 @@ describe("IngestionService unit tests", () => {
expect(sortedEventList).not.toBe(records); // Ensure that the original array is not mutated
});
+ it("puts the first-arriving of tied create events last so it wins the merge", () => {
+ // OTel stamps a root span's trace-create and a child span's trace update
+ // with the same start time; the child arrives first and must win.
+ const first = { timestamp: 1, type: "trace-create", id: "first" };
+ const second = { timestamp: 1, type: "trace-create", id: "second" };
+
+ expect(
+ (IngestionService as any).toTimeSortedEventList([first, second]),
+ ).toEqual([second, first]);
+ });
+
it("correctly convert Date to Clickhouse DateTime", async () => {
const date = new Date("2024-10-12T12:13:14.123Z");
@@ -895,4 +906,71 @@ describe("IngestionService unit tests", () => {
expect(addToQueue).not.toHaveBeenCalled();
});
+
+ it("keeps the last-arriving event when score events share a timestamp", async () => {
+ // Re-sending the full score with the same id, name, and timestamp is the
+ // documented way to overwrite it, so the events tie on timestamp and
+ // arrival order has to decide the winner.
+ const timestamp = "2026-08-31T12:00:00.000Z";
+ const buildEvent = (id: string, value: string): ScoreEventType => ({
+ id,
+ timestamp,
+ type: "score-create",
+ body: {
+ id: "score-id",
+ name: "quality",
+ dataType: "TEXT",
+ source: "API",
+ sessionId: "session-id",
+ environment: "default",
+ value,
+ },
+ });
+ const first = buildEvent("event-first", "first");
+ const second = buildEvent("event-second", "second");
+
+ const mergeAndReadValue = async (events: ScoreEventType[]) => {
+ const addToQueue = vi.fn();
+ const ingestionService = new IngestionService(
+ {} as any,
+ {} as any,
+ { addToQueue } as any,
+ {} as any,
+ );
+ vi.spyOn(
+ ingestionService as any,
+ "getClickhouseRecord",
+ ).mockResolvedValue(null);
+
+ await ingestionService.mergeAndWrite({
+ eventType: "score",
+ projectId: "project-id",
+ entityId: "score-id",
+ createdAtTimestamp: new Date(timestamp),
+ events,
+ forwardToEventsTable: false,
+ attribution: {
+ ingestionApiKey: "pk-lf-unit-test",
+ ingestionSdkName: "langfuse-test",
+ ingestionSdkVersion: "0.0.0",
+ },
+ });
+
+ expect(addToQueue).toHaveBeenCalledWith(
+ TableName.Scores,
+ expect.anything(),
+ );
+ return addToQueue.mock.calls[0]?.[1].string_value;
+ };
+
+ for (const events of [
+ [first, second],
+ [second, first],
+ ]) {
+ await expect(
+ mergeAndReadValue(events),
+ `arrival order ${events.map((e) => e.id).join(", ")}`,
+ ).resolves.toBe(events.at(-1)?.body.value);
+ }
+ });
});