Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

chore(export): emit export staleness from the integration schedulers (#17995)

Niklas Semmler90399fd143e1
diff --git a/.agents/skills/datadog-query-recipes/SKILL.md b/.agents/skills/datadog-query-recipes/SKILL.md
index e03ec09f60..dcb907caa3 100644
--- a/.agents/skills/datadog-query-recipes/SKILL.md
+++ b/.agents/skills/datadog-query-recipes/SKILL.md
@@ -41,7 +41,9 @@ data domain you need: traces, logs, metrics, and visualizations.
[`references/public-api-tenant-usage.md`](references/public-api-tenant-usage.md)
- Queue inventory, queue consumers, and queue metrics:
[`references/queue-consumers.md`](references/queue-consumers.md)
- - Scheduled-export freshness lag (blob / PostHog / Mixpanel):
+ - Scheduled-export staleness (blob / PostHog / Mixpanel):
+ [`references/export-staleness.md`](references/export-staleness.md)
+ - Scheduled-export freshness lag (legacy per-run metric):
[`references/export-freshness-lag.md`](references/export-freshness-lag.md)
3. Start with aggregate queries, grouped by environment, service, route,
queue, project, org, status, or error facets as appropriate.
diff --git a/.agents/skills/datadog-query-recipes/references/export-staleness.md b/.agents/skills/datadog-query-recipes/references/export-staleness.md
new file mode 100644
index 0000000000..93131b5d70
--- /dev/null
+++ b/.agents/skills/datadog-query-recipes/references/export-staleness.md
@@ -0,0 +1,44 @@
+# Export staleness
+
+Scheduled integrations (blob, PostHog, Mixpanel) emit one sample per enabled
+integration on every scheduler tick. Provenance: LFE-16783, replacing the
+per-run freshness lag from LFE-15853.
+
+## Metric
+
+- Name: `langfuse.export.staleness_seconds`
+- Type: distribution
+- Value: `now − lastSyncAt` (seconds, clamped at 0), where `lastSyncAt` is the
+ export watermark (newest exported timestamp)
+- Emitted by the `handle*IntegrationSchedule` cron, not by the export job, so:
+ - every enabled integration contributes one sample per tick, which makes
+ percentiles time-weighted rather than run-weighted;
+ - an integration that stops running keeps emitting a rising value, so a
+ stall surfaces within one tick (blob: 20 min; PostHog/Mixpanel: 1 h).
+- Steady-state floor is the lag buffer (blob 20 min, PostHog/Mixpanel 30 min)
+ plus up to one export interval.
+- Integrations that have never synced (`lastSyncAt` null) are skipped.
+- Tags: `integration` (`blob_storage` / `posthog` / `mixpanel`), `window`
+ (`20m` / `1h` / `1d` / `1w`; `unknown` for an unrecognised blob frequency),
+ `unit:seconds`. `env` comes from the agent. **No `project_id`.**
+- PostHog and Mixpanel are tagged `window:1h` (hourly cron). Blob uses the
+ configured cadence.
+
+## Queries (both Datadog sites)
+
+EU (`prod-eu`) and US (`prod-us`, `prod-hipaa`, `prod-jp`):
+
+```text
+p95:langfuse.export.staleness_seconds{*} by {integration,window,env}
+max:langfuse.export.staleness_seconds{*} by {integration,window}
+```
+
+Normalise by cadence when comparing windows: a `1w` blob export is healthy at
+~1 week of staleness.
+
+## Emit sites
+
+- `worker/src/features/blobstorage/handleBlobStorageIntegrationSchedule.ts`
+- `worker/src/features/posthog/handlePostHogIntegrationSchedule.ts`
+- `worker/src/features/mixpanel/handleMixpanelIntegrationSchedule.ts`
+- Helper: `worker/src/services/exportStalenessMetric.ts`
diff --git a/worker/src/features/blobstorage/handleBlobStorageIntegrationSchedule.test.ts b/worker/src/features/blobstorage/handleBlobStorageIntegrationSchedule.test.ts
new file mode 100644
index 0000000000..a80601c1c9
--- /dev/null
+++ b/worker/src/features/blobstorage/handleBlobStorageIntegrationSchedule.test.ts
@@ -0,0 +1,96 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const { findMany, addBulk, recordDistribution } = vi.hoisted(() => ({
+ findMany: vi.fn(),
+ addBulk: vi.fn(),
+ recordDistribution: vi.fn(),
+}));
+
+vi.mock("@langfuse/shared/src/db", () => ({
+ prisma: { blobStorageIntegration: { findMany } },
+}));
+
+vi.mock("@langfuse/shared/src/server", () => ({
+ BlobStorageIntegrationProcessingQueue: {
+ getInstance: () => ({ addBulk, clean: vi.fn() }),
+ },
+ QueueJobs: { BlobStorageIntegrationProcessingJob: "job" },
+ logger: { info: vi.fn() },
+ recordDistribution,
+}));
+
+import { handleBlobStorageIntegrationSchedule } from "./handleBlobStorageIntegrationSchedule";
+import { EXPORT_STALENESS_METRIC } from "../../services/exportStalenessMetric";
+
+describe("handleBlobStorageIntegrationSchedule", () => {
+ const now = new Date("2026-09-28T12:00:00.000Z");
+
+ beforeEach(() => {
+ vi.clearAllMocks();
+ vi.useFakeTimers({ now });
+ });
+
+ it("emits staleness for every enabled integration but enqueues only due ones", async () => {
+ findMany.mockResolvedValue([
+ {
+ projectId: "due",
+ exportFrequency: "every_20_minutes",
+ lastSyncAt: new Date("2026-09-28T11:20:00.000Z"),
+ nextSyncAt: new Date("2026-09-28T11:59:00.000Z"),
+ },
+ {
+ // Stalled: watermark is a day old but nextSyncAt says not due.
+ projectId: "stalled",
+ exportFrequency: "hourly",
+ lastSyncAt: new Date("2026-09-27T12:00:00.000Z"),
+ nextSyncAt: new Date("2026-09-28T13:00:00.000Z"),
+ },
+ {
+ projectId: "never-synced",
+ exportFrequency: "daily",
+ lastSyncAt: null,
+ nextSyncAt: null,
+ },
+ ]);
+
+ await handleBlobStorageIntegrationSchedule();
+
+ expect(recordDistribution.mock.calls).toEqual([
+ [
+ EXPORT_STALENESS_METRIC,
+ 40 * 60,
+ { integration: "blob_storage", window: "20m", unit: "seconds" },
+ ],
+ [
+ EXPORT_STALENESS_METRIC,
+ 24 * 60 * 60,
+ { integration: "blob_storage", window: "1h", unit: "seconds" },
+ ],
+ ]);
+ const enqueued = addBulk.mock.calls[0][0].map(
+ (job: { data: { payload: { projectId: string } } }) =>
+ job.data.payload.projectId,
+ );
+ expect(enqueued).toEqual(["due", "never-synced"]);
+ });
+
+ it("emits staleness even when nothing is due", async () => {
+ findMany.mockResolvedValue([
+ {
+ projectId: "stalled",
+ exportFrequency: "monthly",
+ lastSyncAt: new Date("2026-09-28T10:00:00.000Z"),
+ nextSyncAt: new Date("2026-09-28T13:00:00.000Z"),
+ },
+ ]);
+
+ await handleBlobStorageIntegrationSchedule();
+
+ expect(recordDistribution).toHaveBeenCalledWith(
+ EXPORT_STALENESS_METRIC,
+ 2 * 60 * 60,
+ { integration: "blob_storage", window: "unknown", unit: "seconds" },
+ );
+ expect(addBulk).not.toHaveBeenCalled();
+ });
+});
diff --git a/worker/src/features/blobstorage/handleBlobStorageIntegrationSchedule.ts b/worker/src/features/blobstorage/handleBlobStorageIntegrationSchedule.ts
index 88b1ec1ea5..be1026015e 100644
--- a/worker/src/features/blobstorage/handleBlobStorageIntegrationSchedule.ts
+++ b/worker/src/features/blobstorage/handleBlobStorageIntegrationSchedule.ts
@@ -5,28 +5,42 @@ import {
logger,
} from "@langfuse/shared/src/server";
import { randomUUID } from "crypto";
+import {
+ recordExportStaleness,
+ windowClassFromBlobFrequency,
+} from "../../services/exportStalenessMetric";
let legacyJobsDrained = false;
export const handleBlobStorageIntegrationSchedule = async () => {
const now = new Date();
- const blobStorageIntegrationProjects =
- await prisma.blobStorageIntegration.findMany({
- select: {
- lastSyncAt: true,
- projectId: true,
- },
- where: {
- enabled: true,
- OR: [
- // Never synced before
- { lastSyncAt: null },
- // Next sync is due
- { nextSyncAt: { lte: now } },
- ],
- },
- });
+ const enabledIntegrations = await prisma.blobStorageIntegration.findMany({
+ select: {
+ lastSyncAt: true,
+ nextSyncAt: true,
+ exportFrequency: true,
+ projectId: true,
+ },
+ where: {
+ enabled: true,
+ },
+ });
+
+ recordExportStaleness({
+ integration: "blob_storage",
+ now,
+ integrations: enabledIntegrations.map((integration) => ({
+ lastSyncAt: integration.lastSyncAt,
+ window: windowClassFromBlobFrequency(integration.exportFrequency),
+ })),
+ });
+
+ const blobStorageIntegrationProjects = enabledIntegrations.filter(
+ (integration) =>
+ integration.lastSyncAt === null ||
+ (integration.nextSyncAt !== null && integration.nextSyncAt <= now),
+ );
if (blobStorageIntegrationProjects.length === 0) {
logger.info("No blob storage integrations ready for sync");
diff --git a/worker/src/features/mixpanel/handleMixpanelIntegrationSchedule.ts b/worker/src/features/mixpanel/handleMixpanelIntegrationSchedule.ts
index 8c5e875dd6..8eec1dd4e2 100644
--- a/worker/src/features/mixpanel/handleMixpanelIntegrationSchedule.ts
+++ b/worker/src/features/mixpanel/handleMixpanelIntegrationSchedule.ts
@@ -5,6 +5,7 @@ import {
logger,
} from "@langfuse/shared/src/server";
import { randomUUID } from "crypto";
+import { recordExportStaleness } from "../../services/exportStalenessMetric";
export const handleMixpanelIntegrationSchedule = async () => {
const mixpanelIntegrationProjects = await prisma.mixpanelIntegration.findMany(
@@ -19,6 +20,15 @@ export const handleMixpanelIntegrationSchedule = async () => {
},
);
+ recordExportStaleness({
+ integration: "mixpanel",
+ now: new Date(),
+ integrations: mixpanelIntegrationProjects.map((integration) => ({
+ lastSyncAt: integration.lastSyncAt,
+ window: "1h",
+ })),
+ });
+
if (mixpanelIntegrationProjects.length === 0) {
logger.info("[MIXPANEL] No Mixpanel integrations ready for sync");
return;
diff --git a/worker/src/features/posthog/handlePostHogIntegrationSchedule.ts b/worker/src/features/posthog/handlePostHogIntegrationSchedule.ts
index a014910e2c..c39174e716 100644
--- a/worker/src/features/posthog/handlePostHogIntegrationSchedule.ts
+++ b/worker/src/features/posthog/handlePostHogIntegrationSchedule.ts
@@ -5,6 +5,7 @@ import {
logger,
} from "@langfuse/shared/src/server";
import { randomUUID } from "crypto";
+import { recordExportStaleness } from "../../services/exportStalenessMetric";
export const handlePostHogIntegrationSchedule = async () => {
const postHogIntegrationProjects = await prisma.posthogIntegration.findMany({
@@ -17,6 +18,15 @@ export const handlePostHogIntegrationSchedule = async () => {
},
});
+ recordExportStaleness({
+ integration: "posthog",
+ now: new Date(),
+ integrations: postHogIntegrationProjects.map((integration) => ({
+ lastSyncAt: integration.lastSyncAt,
+ window: "1h",
+ })),
+ });
+
if (postHogIntegrationProjects.length === 0) {
logger.info("[POSTHOG] No PostHog integrations ready for sync");
return;
diff --git a/worker/src/services/exportStalenessMetric.ts b/worker/src/services/exportStalenessMetric.ts
new file mode 100644
index 0000000000..e6c8d64414
--- /dev/null
+++ b/worker/src/services/exportStalenessMetric.ts
@@ -0,0 +1,46 @@
+import { recordDistribution } from "@langfuse/shared/src/server";
+
+export const EXPORT_STALENESS_METRIC = "langfuse.export.staleness_seconds";
+
+type ExportStalenessIntegration = "blob_storage" | "posthog" | "mixpanel";
+
+type ExportStalenessWindow = "20m" | "1h" | "1d" | "1w" | "unknown";
+
+const BLOB_FREQUENCY_TO_WINDOW: Record<string, ExportStalenessWindow> = {
+ every_20_minutes: "20m",
+ hourly: "1h",
+ daily: "1d",
+ weekly: "1w",
+};
+
+// Non-throwing: runs inside the scheduler, where one bad row must not block
+// scheduling for every other integration.
+export const windowClassFromBlobFrequency = (
+ frequency: string,
+): ExportStalenessWindow => BLOB_FREQUENCY_TO_WINDOW[frequency] ?? "unknown";
+
+/**
+ * Emits `now − lastSyncAt` for every enabled integration on each scheduler
+ * tick. Sampling at a fixed cadence makes percentiles time-weighted, and an
+ * integration that stops running keeps emitting a rising value instead of
+ * going silent. Integrations without a watermark yet are skipped.
+ */
+export const recordExportStaleness = ({
+ integration,
+ now,
+ integrations,
+}: {
+ integration: ExportStalenessIntegration;
+ now: Date;
+ integrations: { lastSyncAt: Date | null; window: ExportStalenessWindow }[];
+}): void => {
+ for (const { lastSyncAt, window } of integrations) {
+ if (!lastSyncAt) continue;
+ const stalenessSeconds = (now.getTime() - lastSyncAt.getTime()) / 1000;
+ recordDistribution(EXPORT_STALENESS_METRIC, Math.max(0, stalenessSeconds), {
+ integration,
+ window,
+ unit: "seconds",
+ });
+ }
+};