Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

chore(worker): drop unused blob export metrics (#18770)

Niklas Semmlere1be1de9da60
diff --git a/worker/src/__tests__/blobStorageIntegrationProcessing.test.ts b/worker/src/__tests__/blobStorageIntegrationProcessing.test.ts
index 84fd7ab800..787351f716 100644
--- a/worker/src/__tests__/blobStorageIntegrationProcessing.test.ts
+++ b/worker/src/__tests__/blobStorageIntegrationProcessing.test.ts
@@ -15,10 +15,8 @@ const originalCloudRegion = vi.hoisted(() => {
return cloudRegion;
});
-// Override recordIncrement + recordHistogram so the attempt counter and
-// per-stage timing metrics are assertable.
+// Override recordIncrement so the attempt counter is assertable.
const mockRecordIncrement = vi.hoisted(() => vi.fn());
-const mockRecordHistogram = vi.hoisted(() => vi.fn());
// Stubbable endpoint preflight — the customer-fault tests reject it to drive an
// in-try failure without infra. Defaults to the real impl for other tests.
const mockValidateBlobStorageEndpoint = vi.hoisted(() => vi.fn());
@@ -40,7 +38,6 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
return {
...actual,
recordIncrement: mockRecordIncrement,
- recordHistogram: mockRecordHistogram,
validateBlobStorageEndpoint: mockValidateBlobStorageEndpoint,
dispatchProjectNotification: mockDispatchProjectNotification,
};
@@ -824,89 +821,6 @@ describe("BlobStorageIntegrationProcessingJob", () => {
expect(files.filter((f) => f.file.includes(projectId))).toHaveLength(0);
});
- // LFE-10441: per-stage timing histograms must also be emitted when an export
- // fails (originally gated on upload success), tagged outcome="failure" so the
- // happy-path percentiles stay clean.
- it("emits per-stage timing histograms tagged outcome=failure when the upload fails", async () => {
- const { projectId } = await createOrgProjectAndApiKey();
- s3Prefix = projectId;
-
- await prisma.blobStorageIntegration.create({
- data: {
- projectId,
- type: BlobStorageIntegrationType.S3,
- bucketName,
- prefix: s3Prefix,
- accessKeyId: minioAccessKeyId,
- secretAccessKey: encrypt(minioAccessKeySecret),
- region: region ? region : "auto",
- // endpoint null -> skip the persisted-endpoint preflight; the storage
- // service is mocked anyway, so no real connection is made.
- endpoint: null,
- forcePathStyle:
- env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
- enabled: true,
- exportFrequency: "daily",
- // Non-enriched source: sidesteps the enriched-export guard regardless of
- // V4 preview state, so this runs outside maybeDescribe on every CI leg.
- exportSource: "TRACES_OBSERVATIONS",
- // A past lastSyncAt yields a non-empty export window without requiring
- // the ClickHouse min-timestamp probe.
- lastSyncAt: new Date(Date.now() - 2 * 24 * 60 * 60 * 1000),
- },
- });
-
- // Force every upload to reject so the export takes the failure path.
- const getInstanceSpy = vi
- .spyOn(StorageServiceFactory, "getInstance")
- .mockReturnValue({
- uploadFileBuffered: vi
- .fn()
- .mockRejectedValue(new Error("simulated upload failure")),
- } as unknown as StorageService);
-
- mockRecordIncrement.mockClear();
- mockRecordHistogram.mockClear();
-
- try {
- await expect(
- handleBlobStorageIntegrationProjectJob({
- data: { payload: { projectId } },
- } as Job),
- ).rejects.toThrow(/simulated upload failure/i);
- } finally {
- getInstanceSpy.mockRestore();
- }
-
- // Failure attempt counter fired.
- expect(mockRecordIncrement).toHaveBeenCalledWith(
- "langfuse.blobstorage.table_export.count",
- 1,
- expect.objectContaining({ outcome: "failure" }),
- );
-
- // All four stage timers emitted with outcome=failure...
- for (const metric of [
- "langfuse.blob_export.ch_read_ms",
- "langfuse.blob_export.enrich_ms",
- "langfuse.blob_export.gzip_cpu_ms",
- "langfuse.blob_export.upload_wait_ms",
- ]) {
- expect(mockRecordHistogram).toHaveBeenCalledWith(
- metric,
- expect.any(Number),
- expect.objectContaining({ outcome: "failure" }),
- );
- }
-
- // ...and never with outcome=success, since no upload succeeded.
- expect(mockRecordHistogram).not.toHaveBeenCalledWith(
- "langfuse.blob_export.ch_read_ms",
- expect.any(Number),
- expect.objectContaining({ outcome: "success" }),
- );
- });
-
maybeDescribe("events table export tests", () => {
it("should export traces, generations, and scores to S3", async () => {
// Setup
diff --git a/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts b/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts
index 56066e09fc..63502e3c2b 100644
--- a/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts
+++ b/worker/src/features/blobstorage/handleBlobStorageIntegrationProjectJob.ts
@@ -21,8 +21,6 @@ import {
getEventsForBlobStorageExportParquet,
getCurrentSpan,
instrumentAsync,
- recordGauge,
- recordHistogram,
recordIncrement,
BlobStorageIntegrationProcessingQueue,
queryClickhouse,
@@ -497,7 +495,6 @@ const processBlobStorageExport = async (config: {
// Outside the try so the catch can distinguish a real upload success from
// a failure.
let uploadSucceeded = false;
- let heartbeat: ReturnType<typeof setInterval> | undefined;
// Per-stage errors so the catch can name the originating cause instead of
// the bare "aborted" the pipeline teardown propagates to every stage.
@@ -600,21 +597,6 @@ const processBlobStorageExport = async (config: {
}
: null;
- const HEARTBEAT_INTERVAL_MS = 5 * 60 * 1000;
- const heartbeatTags = {
- table: config.table,
- projectId: config.projectId,
- };
- heartbeat = setInterval(() => {
- recordGauge(
- "langfuse.blobstorage.export_heartbeat.serialized_bytes",
- compressedCounter
- ? compressedCounter.bytes
- : serializedCounter.bytes,
- heartbeatTags,
- );
- }, HEARTBEAT_INTERVAL_MS);
-
let fileStream: Readable;
if (parquetEligible) {
@@ -963,10 +945,7 @@ const processBlobStorageExport = async (config: {
uploadDurationMsFinal ??
Math.round(performance.now() - uploadStartMs);
span.setAttribute("blob.uploadDurationMs", totalUploadMs);
- // Emit stage timings on both success and failure. On failure the
- // values are partial (the upload aborted mid-stream), so an
- // `outcome` tag keeps them out of the happy-path percentiles while
- // still capturing where a failed export spent its time.
+ // On failure the values are partial (the upload aborted mid-stream).
const finalGzipCpuMs = gzipStats
? Math.max(
0,
@@ -984,37 +963,6 @@ const processBlobStorageExport = async (config: {
})();
span.setAttribute("blob.gzipCpuMs", finalGzipCpuMs);
span.setAttribute("blob.uploadWaitMs", finalUploadWaitMs);
- const finalExportFormat = parquetEligible
- ? BlobExportFormat.PARQUET
- : resolveBlobExportFormat(config.fileType, config.compressed);
- const stageTags = {
- table: config.table,
- path: exportPath,
- source: finalExportFormat,
- outcome: (uploadSucceeded
- ? "success"
- : "failure") satisfies BlobTableExportOutcome,
- };
- recordHistogram(
- "langfuse.blob_export.ch_read_ms",
- finalChReadMs,
- stageTags,
- );
- recordHistogram(
- "langfuse.blob_export.enrich_ms",
- finalEnrichMs,
- stageTags,
- );
- recordHistogram(
- "langfuse.blob_export.gzip_cpu_ms",
- finalGzipCpuMs,
- stageTags,
- );
- recordHistogram(
- "langfuse.blob_export.upload_wait_ms",
- finalUploadWaitMs,
- stageTags,
- );
}
if (producesUploadStats) {
span.setAttribute("blob.upload.parts", uploadStats.partsUploaded);
@@ -1046,27 +994,6 @@ const processBlobStorageExport = async (config: {
"blob.gzip.throughputMbPerSec",
Number(throughputMbPerSec.toFixed(2)),
);
-
- const metricTags = {
- table: config.table,
- path: exportPath,
- gzipLevel: gzipStats.level,
- };
- recordHistogram(
- "langfuse.blob_export.gzip.active_ms",
- activeMs,
- metricTags,
- );
- recordHistogram(
- "langfuse.blob_export.gzip.ratio",
- ratio,
- metricTags,
- );
- recordHistogram(
- "langfuse.blob_export.gzip.throughput_mb_per_sec",
- throughputMbPerSec,
- metricTags,
- );
}
}
} catch (error) {
@@ -1098,7 +1025,6 @@ const processBlobStorageExport = async (config: {
);
throw error;
} finally {
- clearInterval(heartbeat);
unregisterInFlightBlobExport(inFlightHandle);
// ns → ms; the histogram yields NaN/Infinity with zero samples.
@@ -1108,17 +1034,6 @@ const processBlobStorageExport = async (config: {
const maxMs = toFiniteMs(eventLoopDelay.max);
const p99Ms = toFiniteMs(eventLoopDelay.percentile(99));
const meanMs = toFiniteMs(eventLoopDelay.mean);
- const delayTags = { table: config.table, unit: "milliseconds" };
- recordGauge(
- "langfuse.blobstorage.event_loop_delay.max",
- maxMs,
- delayTags,
- );
- recordGauge(
- "langfuse.blobstorage.event_loop_delay.p99",
- p99Ms,
- delayTags,
- );
span.setAttribute("blob.eventLoopDelay.maxMs", Math.round(maxMs));
span.setAttribute("blob.eventLoopDelay.p99Ms", Math.round(p99Ms));
span.setAttribute("blob.eventLoopDelay.meanMs", Math.round(meanMs));