Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): drain and bound spans held for unfinished parents (#2817)

Chad Hietalaf2c96a136b81
diff --git a/.changeset/drain-pending-spans.md b/.changeset/drain-pending-spans.md
new file mode 100644
index 000000000..6ae062968
--- /dev/null
+++ b/.changeset/drain-pending-spans.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Spans held for a parent that never ends are no longer lost or buffered without bound: eve's span-filtering processor now drains them to destinations on `forceFlush` and `shutdown`, and caps how many a stuck parent can hold.
diff --git a/packages/eve/src/tracing/otel-registration.test.ts b/packages/eve/src/tracing/otel-registration.test.ts
index 6df76c5d7..b786dfb74 100644
--- a/packages/eve/src/tracing/otel-registration.test.ts
+++ b/packages/eve/src/tracing/otel-registration.test.ts
@@ -161,8 +161,101 @@ describe("registerOtelPipeline", () => {
registerOtelPipeline({ pipeline: { spanProcessors: [] }, serviceName: "weather" }),
).toThrow(/another runtime already owns/u);
});
+
+ it("drains held children on flush when their parent never ends", async () => {
+ const { downstream, processor } = filteringProcessor();
+ const parent = physicalSpan("2");
+ const child = physicalSpan("3", "2");
+ processor.onStart(parent, {});
+ processor.onStart(child, {});
+ processor.onEnd(child);
+ expect(downstream.onEnd).not.toHaveBeenCalled();
+
+ await processor.forceFlush();
+
+ expect(downstream.onEnd).toHaveBeenCalledExactlyOnceWith(child);
+ expect(downstream.forceFlush).toHaveBeenCalledOnce();
+ expect(downstream.onEnd.mock.invocationCallOrder[0]).toBeLessThan(
+ downstream.forceFlush.mock.invocationCallOrder[0]!,
+ );
+ });
+
+ it("drains held children at shutdown", async () => {
+ const { downstream, processor } = filteringProcessor();
+ const parent = physicalSpan("2");
+ const child = physicalSpan("3", "2");
+ processor.onStart(parent, {});
+ processor.onStart(child, {});
+ processor.onEnd(child);
+
+ await processor.shutdown();
+
+ expect(downstream.onEnd).toHaveBeenCalledExactlyOnceWith(child);
+ expect(downstream.shutdown).toHaveBeenCalledOnce();
+ });
+
+ it("does not re-forward drained children when the parent ends later", async () => {
+ const { downstream, processor } = filteringProcessor();
+ const parent = physicalSpan("2");
+ const child = physicalSpan("3", "2");
+ processor.onStart(parent, {});
+ processor.onStart(child, {});
+ processor.onEnd(child);
+ await processor.forceFlush();
+
+ processor.onEnd(parent);
+
+ expect(downstream.onEnd.mock.calls).toEqual([[child], [parent]]);
+ });
+
+ it("releases held children once one stuck parent exceeds the pending cap", () => {
+ const { downstream, processor } = filteringProcessor();
+ const parent = physicalSpan("2");
+ processor.onStart(parent, {});
+ const overflow = 10_001;
+ for (let index = 0; index < overflow; index += 1) {
+ const child = {
+ parentSpanContext: { spanId: "2".repeat(16), traceId: "1".repeat(32) },
+ spanContext: () => ({
+ spanId: index.toString(16).padStart(16, "0"),
+ traceId: "1".repeat(32),
+ }),
+ };
+ processor.onStart(child, {});
+ processor.onEnd(child);
+ }
+
+ expect(downstream.onEnd).toHaveBeenCalledTimes(overflow);
+ });
});
+function filteringProcessor() {
+ const downstream = {
+ forceFlush: vi.fn(async () => {}),
+ onEnd: vi.fn(),
+ onStart: vi.fn(),
+ shutdown: vi.fn(async () => {}),
+ };
+ registerOTel.mockImplementation(() => undefined);
+ expect(() =>
+ registerOtelPipeline({
+ pipeline: { spanProcessors: [downstream] },
+ serviceName: "weather",
+ }),
+ ).toThrow();
+ const processor = (
+ registerOTel.mock.calls.at(-1)![0] as {
+ spanProcessors: {
+ forceFlush(): Promise<void>;
+ onEnd(span: unknown): void;
+ onStart(span: unknown, parentContext: unknown): void;
+ shutdown(): Promise<void>;
+ }[];
+ }
+ ).spanProcessors[0]!;
+ return { downstream, processor };
+}
+
function replaySpan(marker: string) {
return {
marker,
diff --git a/packages/eve/src/tracing/otel-registration.ts b/packages/eve/src/tracing/otel-registration.ts
index 860e367f5..96e22835d 100644
--- a/packages/eve/src/tracing/otel-registration.ts
+++ b/packages/eve/src/tracing/otel-registration.ts
@@ -13,6 +13,7 @@ import type { OtelPipeline } from "#tracing/otel-declaration.js";
const REGISTRATION_SPAN_NAME = "eve.otel.registration";
const REPLAY_DEDUPLICATION_LIMIT = 100_000;
+const PENDING_CHILD_SPAN_LIMIT = 10_000;
const require = createRequire(import.meta.url);
class RegistrationMarkerPropagator {
@@ -42,6 +43,7 @@ class PrivateSpanFilteringProcessor implements SpanProcessor {
private readonly endedSpans = new Set<string>();
private readonly forwardedSpans = new Set<string>();
private readonly pendingByParent = new Map<string, unknown[]>();
+ private pendingSpanCount = 0;
private readonly processors: readonly SpanProcessor[];
private readonly startedSpans = new Set<string>();
@@ -50,6 +52,7 @@ class PrivateSpanFilteringProcessor implements SpanProcessor {
}
async forceFlush(): Promise<void> {
+ this.drainPendingSpans();
await Promise.all(this.processors.map((processor) => processor.forceFlush()));
}
@@ -69,6 +72,8 @@ class PrivateSpanFilteringProcessor implements SpanProcessor {
const pending = this.pendingByParent.get(parent) ?? [];
pending.push(span);
this.pendingByParent.set(parent, pending);
+ this.pendingSpanCount += 1;
+ if (this.pendingSpanCount > PENDING_CHILD_SPAN_LIMIT) this.releaseOldestPendingParent();
return;
}
this.forward(span, identity);
@@ -82,17 +87,38 @@ class PrivateSpanFilteringProcessor implements SpanProcessor {
}
async shutdown(): Promise<void> {
+ this.drainPendingSpans();
await Promise.all(this.processors.map((processor) => processor.shutdown()));
}
+ // A parent that never ends (lost worker, abandoned attempt) must not hold
+ // its ended children forever: flush and shutdown are the last chance to
+ // export them, and the cap bounds what one stuck parent can buffer. Spans
+ // released here precede their parent; an ordinary flush drains nothing
+ // because parents end before the per-step flush runs.
+ private drainPendingSpans(): void {
+ while (this.pendingByParent.size > 0) this.releaseOldestPendingParent();
+ }
+
+ private releaseOldestPendingParent(): void {
+ const parent = this.pendingByParent.keys().next().value;
+ if (parent === undefined) return;
+ this.releaseChildren(parent);
+ }
+
+ private releaseChildren(parent: string): void {
+ const children = this.pendingByParent.get(parent);
+ if (children === undefined) return;
+ this.pendingByParent.delete(parent);
+ this.pendingSpanCount -= children.length;
+ for (const child of children) this.forward(child, spanIdentity(child));
+ }
+
private forward(span: unknown, identity: string | undefined): void {
for (const processor of this.processors) processor.onEnd(span);
if (identity === undefined) return;
addBounded(this.forwardedSpans, identity);
- const children = this.pendingByParent.get(identity);
- if (children === undefined) return;
- this.pendingByParent.delete(identity);
- for (const child of children) this.forward(child, spanIdentity(child));
+ this.releaseChildren(identity);
}
}