🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(blob-export): half-open export time window to stop chunk-boundary double-count (#16613)
Niklas Semmlera53d296f2199
diff --git a/packages/shared/src/server/repositories/blobStorageExportWindow.test.ts b/packages/shared/src/server/repositories/blobStorageExportWindow.test.tsnew file mode 100644index 0000000000..2411ae67a7--- /dev/null+++ b/packages/shared/src/server/repositories/blobStorageExportWindow.test.ts@@ -0,0 +1,99 @@+import { describe, it, expect, beforeEach, vi } from "vitest";++const mockQueryClickhouseStream = vi.hoisted(() =>+ vi.fn(() => (async function* () {})()),+);++vi.mock("./clickhouse", () => ({+ queryClickhouse: vi.fn(),+ queryClickhouseStream: mockQueryClickhouseStream,+ queryClickhouseStreamRawText: mockQueryClickhouseStream,+ queryClickhouseExecRaw: vi.fn(),+ commandClickhouse: vi.fn(),+ upsertClickhouse: vi.fn(),+ parseClickhouseUTCDateTimeFormat: vi.fn(),+ clickhouseCompliantRandomCharacters: vi.fn(() => "x"),+ BLOB_EXPORT_PARQUET_CLICKHOUSE_SETTINGS: {},+}));++vi.mock("../queries/clickhouse-sql/query-options", () => ({+ shouldSkipObservationsFinal: vi.fn().mockResolvedValue(false),+}));++import { getTracesForBlobStorageExport } from "./traces";+import { getObservationsForBlobStorageExport } from "./observations";+import { getScoresForBlobStorageExport } from "./scores";+import { getEventsForBlobStorageExport } from "./events";++const PROJECT = "proj-1";+const MIN = new Date("2026-01-01T00:00:00.000Z");+const MAX = new Date("2026-01-01T01:00:00.000Z");++const capturedQuery = (): string => {+ expect(mockQueryClickhouseStream).toHaveBeenCalledTimes(1);+ const call = mockQueryClickhouseStream.mock.calls[0] as unknown as [+ { query: string },+ ];+ return call[0].query;+};++const builders = [+ {+ name: "getTracesForBlobStorageExport",+ run: async () => {+ getTracesForBlobStorageExport(PROJECT, MIN, MAX);+ },+ },+ {+ name: "getObservationsForBlobStorageExport",+ // async generator: iterate once to run the builder+ run: async () => {+ await getObservationsForBlobStorageExport(PROJECT, MIN, MAX).next();+ },+ },+ {+ name: "getScoresForBlobStorageExport",+ run: async () => {+ getScoresForBlobStorageExport(PROJECT, MIN, MAX);+ },+ },+ {+ name: "getEventsForBlobStorageExport",+ run: async () => {+ getEventsForBlobStorageExport(PROJECT, MIN, MAX);+ },+ },+];++describe("blob-storage export window is half-open", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ });++ it.each(builders)(+ "$name uses an exclusive upper bound and inclusive lower bound",+ async ({ run }) => {+ await run();+ const query = capturedQuery();++ expect(query).toContain(">= {minTimestamp");+ expect(query).toContain("< {maxTimestamp");+ expect(query).not.toContain("<= {maxTimestamp");+ },+ );++ it("boundary row belongs to the next window only (traces)", async () => {+ // Window 1 [MIN, MAX) excludes a row at exactly MAX...+ getTracesForBlobStorageExport(PROJECT, MIN, MAX);+ const window1 = capturedQuery();+ expect(window1).toContain("< {maxTimestamp");++ vi.clearAllMocks();++ // ...window 2's inclusive min = prior MAX includes it.+ const next = new Date("2026-01-01T02:00:00.000Z");+ getTracesForBlobStorageExport(PROJECT, MAX, next);+ const window2 = capturedQuery();+ expect(window2).toContain(">= {minTimestamp");+ });+});diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.tsindex aedc2b623e..941321e60a 100644--- a/packages/shared/src/server/repositories/events.ts+++ b/packages/shared/src/server/repositories/events.ts@@ -3171,7 +3171,7 @@ const buildEventsForBlobStorageExportQuery = (queryBuilder.whereRaw(- "e.start_time >= {minTimestamp: DateTime64(3)} AND e.start_time <= {maxTimestamp: DateTime64(3)}",+ "e.start_time >= {minTimestamp: DateTime64(3)} AND e.start_time < {maxTimestamp: DateTime64(3)}",{minTimestamp: convertDateToClickhouseDateTime(minTimestamp),maxTimestamp: convertDateToClickhouseDateTime(maxTimestamp),diff --git a/packages/shared/src/server/repositories/observations.ts b/packages/shared/src/server/repositories/observations.tsindex b2871cc487..6a255e86c2 100644--- a/packages/shared/src/server/repositories/observations.ts+++ b/packages/shared/src/server/repositories/observations.ts@@ -1755,7 +1755,7 @@ const buildObservationsForBlobStorageExportQuery = (FROM observationsWHERE project_id = {projectId: String}AND start_time >= {minTimestamp: DateTime64(3)}- AND start_time <= {maxTimestamp: DateTime64(3)}+ AND start_time < {maxTimestamp: DateTime64(3)}${skipDedup? ""diff --git a/packages/shared/src/server/repositories/scores.ts b/packages/shared/src/server/repositories/scores.tsindex 670871d375..82c68a56c8 100644--- a/packages/shared/src/server/repositories/scores.ts+++ b/packages/shared/src/server/repositories/scores.ts@@ -2323,7 +2323,7 @@ const buildScoresForBlobStorageExportQuery = (FROM scores FINALWHERE project_id = {projectId: String}AND timestamp >= {minTimestamp: DateTime64(3)}- AND timestamp <= {maxTimestamp: DateTime64(3)}+ AND timestamp < {maxTimestamp: DateTime64(3)}AND data_type IN ({dataTypes: Array(String)})`;diff --git a/packages/shared/src/server/repositories/traces.ts b/packages/shared/src/server/repositories/traces.tsindex 65c5c3bf19..e36a8a4e72 100644--- a/packages/shared/src/server/repositories/traces.ts+++ b/packages/shared/src/server/repositories/traces.ts@@ -1222,7 +1222,7 @@ const buildTracesForBlobStorageExportQuery = (FROM ${traceTable} FINALWHERE project_id = {projectId: String}AND timestamp >= {minTimestamp: DateTime64(3)}- AND timestamp <= {maxTimestamp: DateTime64(3)}+ AND timestamp < {maxTimestamp: DateTime64(3)}`;return {diff --git a/worker/src/features/blobstorage/manifest.ts b/worker/src/features/blobstorage/manifest.tsindex e89e680034..94149e8d36 100644--- a/worker/src/features/blobstorage/manifest.ts+++ b/worker/src/features/blobstorage/manifest.ts@@ -23,10 +23,7 @@ export type BlobExportManifest = {version: number;projectId: string;exportSource: string;- /**- * Export window, both bounds inclusive — a row at exactly maxTimestamp also- * falls into the next run's window.- */+ /** Half-open `[minTimestamp, maxTimestamp)`. */window: { minTimestamp: string; maxTimestamp: string };/** Equals window.maxTimestamp; also the manifest key stem. */maxTimestamp: string;