Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(otel): share worker shadow across server bundles (#17652)

Valery Meleshkin36291fd88786
diff --git a/web/src/__tests__/server/unit/otel-ingestion-worker-pool.servertest.ts b/web/src/__tests__/server/unit/otel-ingestion-worker-pool.servertest.ts
index 997b09fb1d..0d225b29b5 100644
--- a/web/src/__tests__/server/unit/otel-ingestion-worker-pool.servertest.ts
+++ b/web/src/__tests__/server/unit/otel-ingestion-worker-pool.servertest.ts
@@ -1,3 +1,6 @@
+import { EventEmitter } from "node:events";
+
+import type { NextApiResponse } from "next";
import { describe, expect, it, vi } from "vitest";
import { tryScheduleOtelIngestionWorkerLifecycle } from "@/src/server/otel/otelIngestionWorkerPool";
@@ -62,4 +65,53 @@ describe("OTel ingestion worker admission", () => {
expect(abortedTaskStarted).toBe(false);
expect(nextTaskStarted).toBe(true);
});
+
+ it("keeps shadow admission enabled across server bundle instances", async () => {
+ vi.resetModules();
+ const scheduledTasks: Promise<void>[] = [];
+ const tryScheduleOtelIngestionWorkerLifecycle = vi.fn(
+ (task: () => Promise<void>) => {
+ const scheduledTask = task();
+ scheduledTasks.push(scheduledTask);
+ return scheduledTask;
+ },
+ );
+ vi.doMock("@/src/server/otel/otelIngestionWorkerPool", () => ({
+ dispatchOtelIngestionWorkerTask: vi
+ .fn()
+ .mockResolvedValue({ kind: "shadow" }),
+ preloadOtelIngestionWorker: vi.fn().mockResolvedValue(undefined),
+ tryScheduleOtelIngestionWorkerLifecycle,
+ }));
+
+ try {
+ const preloadBundle =
+ await import("@/src/server/otel/otelIngestionWorkerShadow");
+ await preloadBundle.preloadOtelIngestionWorkerShadow();
+
+ const firstResponse = new EventEmitter() as NextApiResponse;
+ preloadBundle.startOtelIngestionWorkerAdmissionShadow(
+ firstResponse,
+ "project-id",
+ );
+ firstResponse.emit("finish");
+
+ vi.resetModules();
+ const routeBundle =
+ await import("@/src/server/otel/otelIngestionWorkerShadow");
+ routeBundle.startOtelIngestionWorkerAdmissionShadow(
+ new EventEmitter() as NextApiResponse,
+ "project-id",
+ );
+
+ expect(tryScheduleOtelIngestionWorkerLifecycle).toHaveBeenCalledOnce();
+ await Promise.all(scheduledTasks);
+ } finally {
+ delete (globalThis as typeof globalThis & Record<symbol, unknown>)[
+ Symbol.for("langfuse.otelIngestionWorkerShadow.start")
+ ];
+ vi.doUnmock("@/src/server/otel/otelIngestionWorkerPool");
+ vi.resetModules();
+ }
+ });
});
diff --git a/web/src/server/otel/otelIngestionWorkerShadow.ts b/web/src/server/otel/otelIngestionWorkerShadow.ts
index 25555ca660..ef8a4ec4b6 100644
--- a/web/src/server/otel/otelIngestionWorkerShadow.ts
+++ b/web/src/server/otel/otelIngestionWorkerShadow.ts
@@ -13,7 +13,13 @@ import {
let lastProcessingDurationMs: number | undefined;
let workerPreloadPromise: Promise<void> | undefined;
-let workerShadowAvailable = false;
+
+type WorkerShadowStart = (res: NextApiResponse, projectId: string) => void;
+const workerShadowStartKey = Symbol.for(
+ "langfuse.otelIngestionWorkerShadow.start",
+);
+const workerShadowGlobal = globalThis as typeof globalThis &
+ Record<symbol, WorkerShadowStart | undefined>;
function scheduleAdmissionShadow(projectId: string, durationMs: number): void {
const task = tryScheduleOtelIngestionWorkerLifecycle(async (signal) => {
@@ -47,12 +53,10 @@ function scheduleAdmissionShadow(projectId: string, durationMs: number): void {
});
}
-export function startOtelIngestionWorkerAdmissionShadow(
+function runOtelIngestionWorkerAdmissionShadow(
res: NextApiResponse,
projectId: string,
): void {
- if (!workerShadowAvailable) return;
-
if (lastProcessingDurationMs !== undefined) {
scheduleAdmissionShadow(projectId, lastProcessingDurationMs);
}
@@ -83,9 +87,17 @@ export function startOtelIngestionWorkerAdmissionShadow(
res.once("close", stopTracking);
}
+export function startOtelIngestionWorkerAdmissionShadow(
+ res: NextApiResponse,
+ projectId: string,
+): void {
+ workerShadowGlobal[workerShadowStartKey]?.(res, projectId);
+}
+
async function runWorkerPreload(): Promise<void> {
await preloadOtelIngestionWorker();
- workerShadowAvailable = true;
+ workerShadowGlobal[workerShadowStartKey] =
+ runOtelIngestionWorkerAdmissionShadow;
logger.info("OTel ingestion worker shadow preloaded");
}