The Open Framework for Building Agents
fix(eve): drain and bound spans held for unfinished parents (#2817)
Chad Hietalaf2c96a136b81
diff --git a/.changeset/drain-pending-spans.md b/.changeset/drain-pending-spans.mdnew file mode 100644index 000000000..6ae062968--- /dev/null+++ b/.changeset/drain-pending-spans.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Spans held for a parent that never ends are no longer lost or buffered without bound: eve's span-filtering processor now drains them to destinations on `forceFlush` and `shutdown`, and caps how many a stuck parent can hold.diff --git a/packages/eve/src/tracing/otel-registration.test.ts b/packages/eve/src/tracing/otel-registration.test.tsindex 6df76c5d7..b786dfb74 100644--- a/packages/eve/src/tracing/otel-registration.test.ts+++ b/packages/eve/src/tracing/otel-registration.test.ts@@ -161,8 +161,101 @@ describe("registerOtelPipeline", () => {registerOtelPipeline({ pipeline: { spanProcessors: [] }, serviceName: "weather" }),).toThrow(/another runtime already owns/u);});++ it("drains held children on flush when their parent never ends", async () => {+ const { downstream, processor } = filteringProcessor();+ const parent = physicalSpan("2");+ const child = physicalSpan("3", "2");+ processor.onStart(parent, {});+ processor.onStart(child, {});+ processor.onEnd(child);+ expect(downstream.onEnd).not.toHaveBeenCalled();++ await processor.forceFlush();++ expect(downstream.onEnd).toHaveBeenCalledExactlyOnceWith(child);+ expect(downstream.forceFlush).toHaveBeenCalledOnce();+ expect(downstream.onEnd.mock.invocationCallOrder[0]).toBeLessThan(+ downstream.forceFlush.mock.invocationCallOrder[0]!,+ );+ });++ it("drains held children at shutdown", async () => {+ const { downstream, processor } = filteringProcessor();+ const parent = physicalSpan("2");+ const child = physicalSpan("3", "2");+ processor.onStart(parent, {});+ processor.onStart(child, {});+ processor.onEnd(child);++ await processor.shutdown();++ expect(downstream.onEnd).toHaveBeenCalledExactlyOnceWith(child);+ expect(downstream.shutdown).toHaveBeenCalledOnce();+ });++ it("does not re-forward drained children when the parent ends later", async () => {+ const { downstream, processor } = filteringProcessor();+ const parent = physicalSpan("2");+ const child = physicalSpan("3", "2");+ processor.onStart(parent, {});+ processor.onStart(child, {});+ processor.onEnd(child);+ await processor.forceFlush();++ processor.onEnd(parent);++ expect(downstream.onEnd.mock.calls).toEqual([[child], [parent]]);+ });++ it("releases held children once one stuck parent exceeds the pending cap", () => {+ const { downstream, processor } = filteringProcessor();+ const parent = physicalSpan("2");+ processor.onStart(parent, {});+ const overflow = 10_001;+ for (let index = 0; index < overflow; index += 1) {+ const child = {+ parentSpanContext: { spanId: "2".repeat(16), traceId: "1".repeat(32) },+ spanContext: () => ({+ spanId: index.toString(16).padStart(16, "0"),+ traceId: "1".repeat(32),+ }),+ };+ processor.onStart(child, {});+ processor.onEnd(child);+ }++ expect(downstream.onEnd).toHaveBeenCalledTimes(overflow);+ });});+function filteringProcessor() {+ const downstream = {+ forceFlush: vi.fn(async () => {}),+ onEnd: vi.fn(),+ onStart: vi.fn(),+ shutdown: vi.fn(async () => {}),+ };+ registerOTel.mockImplementation(() => undefined);+ expect(() =>+ registerOtelPipeline({+ pipeline: { spanProcessors: [downstream] },+ serviceName: "weather",+ }),+ ).toThrow();+ const processor = (+ registerOTel.mock.calls.at(-1)![0] as {+ spanProcessors: {+ forceFlush(): Promise<void>;+ onEnd(span: unknown): void;+ onStart(span: unknown, parentContext: unknown): void;+ shutdown(): Promise<void>;+ }[];+ }+ ).spanProcessors[0]!;+ return { downstream, processor };+}+function replaySpan(marker: string) {return {marker,diff --git a/packages/eve/src/tracing/otel-registration.ts b/packages/eve/src/tracing/otel-registration.tsindex 860e367f5..96e22835d 100644--- a/packages/eve/src/tracing/otel-registration.ts+++ b/packages/eve/src/tracing/otel-registration.ts@@ -13,6 +13,7 @@ import type { OtelPipeline } from "#tracing/otel-declaration.js";const REGISTRATION_SPAN_NAME = "eve.otel.registration";const REPLAY_DEDUPLICATION_LIMIT = 100_000;+const PENDING_CHILD_SPAN_LIMIT = 10_000;const require = createRequire(import.meta.url);class RegistrationMarkerPropagator {@@ -42,6 +43,7 @@ class PrivateSpanFilteringProcessor implements SpanProcessor {private readonly endedSpans = new Set<string>();private readonly forwardedSpans = new Set<string>();private readonly pendingByParent = new Map<string, unknown[]>();+ private pendingSpanCount = 0;private readonly processors: readonly SpanProcessor[];private readonly startedSpans = new Set<string>();@@ -50,6 +52,7 @@ class PrivateSpanFilteringProcessor implements SpanProcessor {}async forceFlush(): Promise<void> {+ this.drainPendingSpans();await Promise.all(this.processors.map((processor) => processor.forceFlush()));}@@ -69,6 +72,8 @@ class PrivateSpanFilteringProcessor implements SpanProcessor {const pending = this.pendingByParent.get(parent) ?? [];pending.push(span);this.pendingByParent.set(parent, pending);+ this.pendingSpanCount += 1;+ if (this.pendingSpanCount > PENDING_CHILD_SPAN_LIMIT) this.releaseOldestPendingParent();return;}this.forward(span, identity);@@ -82,17 +87,38 @@ class PrivateSpanFilteringProcessor implements SpanProcessor {}async shutdown(): Promise<void> {+ this.drainPendingSpans();await Promise.all(this.processors.map((processor) => processor.shutdown()));}+ // A parent that never ends (lost worker, abandoned attempt) must not hold+ // its ended children forever: flush and shutdown are the last chance to+ // export them, and the cap bounds what one stuck parent can buffer. Spans+ // released here precede their parent; an ordinary flush drains nothing+ // because parents end before the per-step flush runs.+ private drainPendingSpans(): void {+ while (this.pendingByParent.size > 0) this.releaseOldestPendingParent();+ }++ private releaseOldestPendingParent(): void {+ const parent = this.pendingByParent.keys().next().value;+ if (parent === undefined) return;+ this.releaseChildren(parent);+ }++ private releaseChildren(parent: string): void {+ const children = this.pendingByParent.get(parent);+ if (children === undefined) return;+ this.pendingByParent.delete(parent);+ this.pendingSpanCount -= children.length;+ for (const child of children) this.forward(child, spanIdentity(child));+ }+private forward(span: unknown, identity: string | undefined): void {for (const processor of this.processors) processor.onEnd(span);if (identity === undefined) return;addBounded(this.forwardedSpans, identity);- const children = this.pendingByParent.get(identity);- if (children === undefined) return;- this.pendingByParent.delete(identity);- for (const child of children) this.forward(child, spanIdentity(child));+ this.releaseChildren(identity);}}