🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(blob-export): start full-history exports at the real data minimum (#17200)
Niklas Semmler6af4dd472e82
diff --git a/worker/src/features/blobstorage/firstExportStart.test.ts b/worker/src/features/blobstorage/firstExportStart.test.tsnew file mode 100644index 0000000000..a3d0e9cd15--- /dev/null+++ b/worker/src/features/blobstorage/firstExportStart.test.ts@@ -0,0 +1,76 @@+import { describe, expect, it } from "vitest";+import { BlobStorageExportMode } from "@langfuse/shared";+import { resolveFirstExportStart } from "./firstExportStart";++const NOW = new Date("2026-08-28T12:00:00.000Z");++describe("resolveFirstExportStart", () => {+ const base = {+ exportStartDate: null as Date | null,+ historicalMinTimestampMs: null as number | null,+ now: NOW,+ };++ it("uses the real ClickHouse min for FULL_HISTORY when data exists", () => {+ const min = new Date("2025-03-01T00:00:00.000Z");+ expect(+ resolveFirstExportStart({+ ...base,+ exportMode: BlobStorageExportMode.FULL_HISTORY,+ historicalMinTimestampMs: min.getTime(),+ }),+ ).toEqual(min);+ });++ it("does not floor a backfilled min that predates a much later project", () => {+ // The reported case: data backfilled years before the project row exists.+ // A full-history export must reach back to the real data minimum.+ const backfilledMin = new Date("2019-01-01T00:00:00.000Z");+ expect(+ resolveFirstExportStart({+ ...base,+ exportMode: BlobStorageExportMode.FULL_HISTORY,+ historicalMinTimestampMs: backfilledMin.getTime(),+ }),+ ).toEqual(backfilledMin);+ });++ it("starts a first FULL_HISTORY export at now when there are no rows", () => {+ expect(+ resolveFirstExportStart({+ ...base,+ exportMode: BlobStorageExportMode.FULL_HISTORY,+ }),+ ).toEqual(NOW);+ });++ it("treats a zero min timestamp as no data and starts at now", () => {+ expect(+ resolveFirstExportStart({+ ...base,+ exportMode: BlobStorageExportMode.FULL_HISTORY,+ historicalMinTimestampMs: 0,+ }),+ ).toEqual(NOW);+ });++ it("uses the custom start date for FROM_CUSTOM_DATE", () => {+ const custom = new Date("2026-01-01T00:00:00.000Z");+ expect(+ resolveFirstExportStart({+ ...base,+ exportMode: BlobStorageExportMode.FROM_CUSTOM_DATE,+ exportStartDate: custom,+ }),+ ).toEqual(custom);+ });++ it("falls back to now for FROM_TODAY without a start date", () => {+ expect(+ resolveFirstExportStart({+ ...base,+ exportMode: BlobStorageExportMode.FROM_TODAY,+ }),+ ).toEqual(NOW);+ });+});diff --git a/worker/src/features/blobstorage/firstExportStart.ts b/worker/src/features/blobstorage/firstExportStart.tsnew file mode 100644index 0000000000..9f0971e061--- /dev/null+++ b/worker/src/features/blobstorage/firstExportStart.ts@@ -0,0 +1,38 @@+import { BlobStorageExportMode } from "@langfuse/shared";++/**+ * Resolve the inclusive lower bound for the *first* blob-export run (no+ * `lastSyncAt` yet).+ *+ * The FULL_HISTORY start is the real data minimum from ClickHouse. It is used+ * as-is and never floored at `project.createdAt`: projects legitimately+ * backfill data that predates the project row, and a full-history export must+ * include it.+ *+ * When no rows exist yet, the start resolves to `now` so the caller's+ * empty-window check skips the run instead of advancing the cursor. Returning+ * the Unix epoch would make the worker walk tens of thousands of empty daily+ * windows from 1970, uploading empty files to the customer bucket.+ */+export function resolveFirstExportStart(params: {+ exportMode: BlobStorageExportMode;+ exportStartDate: Date | null;+ historicalMinTimestampMs: number | null;+ now?: Date;+}): Date {+ const now = params.now ?? new Date();++ switch (params.exportMode) {+ case BlobStorageExportMode.FULL_HISTORY: {+ const minMs = Number(params.historicalMinTimestampMs);+ return minMs && minMs > 0 ? new Date(minMs) : now;+ }+ case BlobStorageExportMode.FROM_TODAY:+ case BlobStorageExportMode.FROM_CUSTOM_DATE:+ return params.exportStartDate ?? now;+ default: {+ const _exhaustiveCheck: never = params.exportMode;+ throw new Error(`Invalid export mode: ${_exhaustiveCheck}`);+ }+ }+}diff --git a/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts b/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.tsindex d72d24727e..27e4ca28d6 100644--- a/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts+++ b/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts@@ -89,6 +89,7 @@ import {buildBlobExportDeprecationNotice,buildBlobExportDeprecationNoticeKey,} from "./deprecationNotice";+import { resolveFirstExportStart } from "./firstExportStart";const BlobExportFormat = {JSON_RAW: "json-raw",@@ -206,13 +207,15 @@ const getMinTimestampForExport = async (return lastSyncAt;}- // For first export, use the export mode to determine start date- switch (exportMode) {- case BlobStorageExportMode.FULL_HISTORY:- // Query ClickHouse for the actual minimum timestamp from traces, observations, and scores tables- try {- const result = await queryClickhouse<{ min_timestamp: number | null }>({- query: `+ // For a first FULL_HISTORY export, probe ClickHouse for the actual minimum+ // timestamp across every source table. This is the real data minimum, not+ // project createdAt: projects legitimately backfill data that predates the+ // project, and a full-history export must include it.+ let historicalMinTimestampMs: number | null = null;+ if (exportMode === BlobStorageExportMode.FULL_HISTORY) {+ try {+ const result = await queryClickhouse<{ min_timestamp: number | null }>({+ query: `SELECT min(toUnixTimestamp(ts)) * 1000 as min_timestampFROM (SELECT min(timestamp) as ts@@ -230,46 +233,40 @@ const getMinTimestampForExport = async (SELECT min(timestamp) as tsFROM scoresWHERE project_id = {projectId: String}++ UNION ALL++ SELECT min(start_time) as ts+ FROM events_core+ WHERE project_id = {projectId: String}+ AND is_deleted = 0 -- match the events export query's visibility)WHERE ts > 0 -- Ignore 0 results (usually empty tables)`,- params: { projectId },- });-- // Extract the minimum timestamp- logger.info(- `[BLOB INTEGRATION] ClickHouse min_timestamp for project ${projectId}: ${result[0]?.min_timestamp}, type: ${typeof result[0]?.min_timestamp}`,- );- const minTimestampValue = Number(result[0]?.min_timestamp);-- if (minTimestampValue && minTimestampValue > 0) {- const date = new Date(minTimestampValue);- logger.info(- `[BLOB INTEGRATION] Created Date from min_timestamp for project ${projectId}: ${date}, isValid: ${!isNaN(date.getTime())}, getTime: ${date.getTime()}`,- );- return date;- }+ params: { projectId },+ });- // If no data exists, use current time as a fallback- logger.info(- `[BLOB INTEGRATION] No historical data found for project ${projectId}, using current time`,- );- return new Date(0);- } catch (error) {- logger.error(- `[BLOB INTEGRATION] Error querying ClickHouse for minimum timestamp for project ${projectId}`,- error,- );- throw new Error(`Failed to fetch minimum timestamp: ${error}`);+ const minTimestampValue = Number(result[0]?.min_timestamp);+ if (minTimestampValue && minTimestampValue > 0) {+ historicalMinTimestampMs = minTimestampValue;}- case BlobStorageExportMode.FROM_TODAY:- case BlobStorageExportMode.FROM_CUSTOM_DATE:- return exportStartDate || new Date(); // Use export start date or current time as fallback- default:- // eslint-disable-next-line no-case-declarations- const _exhaustiveCheck: never = exportMode;- throw new Error(`Invalid export mode: ${exportMode}`);+ logger.info(+ `[BLOB INTEGRATION] ClickHouse min_timestamp for project ${projectId}: ${historicalMinTimestampMs}`,+ );+ } catch (error) {+ logger.error(+ `[BLOB INTEGRATION] Error querying ClickHouse for minimum timestamp for project ${projectId}`,+ error,+ );+ throw new Error(`Failed to fetch minimum timestamp: ${error}`);+ }}++ return resolveFirstExportStart({+ exportMode,+ exportStartDate,+ historicalMinTimestampMs,+ });};/**