Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

refactor(ingestion): total order for trace/observation same-timestamp merges (#16979)

Niklas Semmler43cd5bc0dd62
diff --git a/worker/src/services/IngestionService/index.ts b/worker/src/services/IngestionService/index.ts
index 48a4719850..00b3ab4484 100644
--- a/worker/src/services/IngestionService/index.ts
+++ b/worker/src/services/IngestionService/index.ts
@@ -1240,22 +1240,34 @@ export class IngestionService {
private static toTimeSortedEventList<
T extends TraceEventType | ObservationEvent,
>(eventList: T[]): T[] {
- return eventList.slice().sort((a, b) => {
- const aTimestamp = new Date(a.timestamp).getTime();
- const bTimestamp = new Date(b.timestamp).getTime();
-
- if (aTimestamp === bTimestamp) {
- // Creates sort before updates. For two tied creates this is not a
- // total order: V8 places the later-arriving one first, so the
- // first-arriving create wins the merge. OTel relies on that: a root
- // span's trace-create and a child span's trace update share the span
- // start time, and the child's explicit fields must beat the root's
- // derived ones. Score events use their own stable sort instead.
- return a.type.includes("create") ? -1 : 1;
- }
-
- return aTimestamp - bTimestamp;
- });
+ // The merge folds this list left to right with a last-wins overwrite, so
+ // the winner for any group is whichever event sorts last. Events arrive in
+ // upload order (the queue processor lists files by upload time), captured
+ // here as the arrival index and used as the tie-break so the ordering is a
+ // total order rather than relying on the sort engine's handling of ties.
+ return eventList
+ .map((event, index) => ({ event, index }))
+ .sort((a, b) => {
+ const aTimestamp = new Date(a.event.timestamp).getTime();
+ const bTimestamp = new Date(b.event.timestamp).getTime();
+ if (aTimestamp !== bTimestamp) return aTimestamp - bTimestamp;
+
+ const aIsCreate = a.event.type.includes("create");
+ const bIsCreate = b.event.type.includes("create");
+ // Creates sort before updates, so an update at the same timestamp wins.
+ if (aIsCreate !== bIsCreate) return aIsCreate ? -1 : 1;
+
+ // Tied creates: the first-arriving one wins. OTel emits a root span's
+ // trace-create and a child span's trace-create carrying explicit
+ // update_current_trace() values at the same span start time; the child
+ // arrives first and its explicit fields must beat the root's derived
+ // ones. Placing the earlier arrival last makes it win the merge.
+ if (aIsCreate) return b.index - a.index;
+
+ // Tied updates: the last-arriving one wins.
+ return a.index - b.index;
+ })
+ .map(({ event }) => event);
}
private async getPrompt(
diff --git a/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts b/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts
index d587983a7f..9a9b7ef67e 100644
--- a/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts
+++ b/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts
@@ -506,6 +506,31 @@ describe("IngestionService unit tests", () => {
).toEqual([second, first]);
});
+ it("keeps the last-arriving of tied update events last so it wins the merge", () => {
+ const first = { timestamp: 1, type: "trace-update", id: "first" };
+ const second = { timestamp: 1, type: "trace-update", id: "second" };
+
+ expect(
+ (IngestionService as any).toTimeSortedEventList([first, second]),
+ ).toEqual([first, second]);
+ });
+
+ it("orders a mixed run of tied creates and updates deterministically", () => {
+ // Creates sort before updates so the update wins the merge, and among the
+ // tied creates the first-arriving one sorts last.
+ const firstCreate = { timestamp: 1, type: "trace-create", id: "c0" };
+ const update = { timestamp: 1, type: "trace-update", id: "u1" };
+ const secondCreate = { timestamp: 1, type: "trace-create", id: "c2" };
+
+ expect(
+ (IngestionService as any).toTimeSortedEventList([
+ firstCreate,
+ update,
+ secondCreate,
+ ]),
+ ).toEqual([secondCreate, firstCreate, update]);
+ });
+
it("correctly convert Date to Clickhouse DateTime", async () => {
const date = new Date("2024-10-12T12:13:14.123Z");