🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(otel): bound concurrent media uploads by decoded size (#18630)
Valery Meleshkin1be9751db27b
diff --git a/.env.dev.example b/.env.dev.exampleindex 1346352018..09af7f157f 100644--- a/.env.dev.example+++ b/.env.dev.example@@ -158,6 +158,9 @@ LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/# LANGFUSE_S3_MEDIA_UPLOAD_REQUEST_TIMEOUT_MS=30000# Opt in to extracting and uploading media embedded in OTEL span attributes.LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED=false+# Per-process OTEL uploads; one oversized item may exceed the byte budget alone.+# LANGFUSE_OTEL_MEDIA_UPLOAD_CONCURRENCY=100+# LANGFUSE_OTEL_MEDIA_UPLOAD_MAX_BYTES=134217728# Early extraction requires both the toggle and a project selection.LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED=false# Comma-separated project IDs; empty selects nobody, * selects everyone.diff --git a/.env.prod.example b/.env.prod.exampleindex ee29393cdf..e76247f613 100644--- a/.env.prod.example+++ b/.env.prod.example@@ -303,6 +303,9 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true# LANGFUSE_OTEL_INGESTION_MAX_BODY_BYTES=536870912# Simulate OTLP worker admission while keeping inline ingestion authoritative (default: false)# LANGFUSE_OTEL_INGESTION_WORKER_SHADOW_ENABLED=false+# Per-process OTEL uploads; one oversized item may exceed the byte budget alone.+# LANGFUSE_OTEL_MEDIA_UPLOAD_CONCURRENCY=100+# LANGFUSE_OTEL_MEDIA_UPLOAD_MAX_BYTES=134217728# Early extraction requires both the toggle and a project selection.# LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED=false# Comma-separated project IDs; empty selects nobody, * selects everyone.diff --git a/packages/shared/src/env.ts b/packages/shared/src/env.tsindex 41d73a2ba0..324f213738 100644--- a/packages/shared/src/env.ts+++ b/packages/shared/src/env.ts@@ -297,6 +297,16 @@ const EnvSchema = z.object({LANGFUSE_S3_EVENT_UPLOAD_SSE_KMS_KEY_ID: z.string().optional(),LANGFUSE_S3_EVENT_KEY_MAX_SEGMENT_BYTES:langfuseS3EventKeyMaxSegmentBytesSchema,+ LANGFUSE_OTEL_MEDIA_UPLOAD_CONCURRENCY: z.coerce+ .number()+ .int()+ .positive()+ .default(100),+ LANGFUSE_OTEL_MEDIA_UPLOAD_MAX_BYTES: z.coerce+ .number()+ .int()+ .positive()+ .default(128 * 1024 * 1024),LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: z.string().optional(),LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: z.string().default(""),LANGFUSE_S3_MEDIA_UPLOAD_REGION: z.string().optional(),diff --git a/packages/shared/src/server/media/MediaUploadPool.test.ts b/packages/shared/src/server/media/MediaUploadPool.test.tsindex 0bb32c4820..9f66503bdf 100644--- a/packages/shared/src/server/media/MediaUploadPool.test.ts+++ b/packages/shared/src/server/media/MediaUploadPool.test.ts@@ -226,6 +226,29 @@ describe("MediaUploadPool", () => {expect(nextStarted).toBe(true);});+ it("drains a large backlog within the concurrency limit", async () => {+ const pool = new MediaUploadPool(4, 4);+ const gate = deferred<void>();+ const first = pool.run(4, async () => gate.promise);+ let active = 0;+ let peakActive = 0;+ let completed = 0;+ const waiting = Array.from({ length: 2050 }, () =>+ pool.run(1, async () => {+ active++;+ peakActive = Math.max(peakActive, active);+ await Promise.resolve();+ active--;+ completed++;+ }),+ );++ gate.resolve();+ await Promise.all([first, ...waiting]);+ expect(completed).toBe(2050);+ expect(peakActive).toBeLessThanOrEqual(4);+ });+it("does not share reservations between pool calls", async () => {const firstPool = new MediaUploadPool(1, 10);const secondPool = new MediaUploadPool(1, 10);diff --git a/packages/shared/src/server/media/MediaUploadPool.ts b/packages/shared/src/server/media/MediaUploadPool.tsindex 7db1901648..136b76f66b 100644--- a/packages/shared/src/server/media/MediaUploadPool.ts+++ b/packages/shared/src/server/media/MediaUploadPool.ts@@ -1,11 +1,17 @@/**- * Bounds active uploads and decoded bytes. Fitting work may bypass the oldest- * waiter up to `concurrency` times; then new work waits until it starts.- * The upload must reject on timeout itself. This pool keeps its reservation- * until work actually settles; timing out only the waiter would free it too soon.+ * Bounds active uploads and their decoded bytes. Waiting `run()` calls pause+ * until admitted; admission reserves capacity before waking them. Each settled+ * upload admits more work, so progress requires active uploads to settle.+ * Oversized uploads run alone.+ *+ * A fitting upload may bypass the oldest waiter up to `concurrency` times.+ * Work must enforce its own timeout; its reservation lasts until it settles.*/export class MediaUploadPool {- private waiting: PendingUpload[] = [];+ private waiting: Array<PendingUpload | undefined> = [];+ private first = 0;+ private capacity = 1;+ private minBytes = [Infinity, Infinity];private active = 0;private reservedBytes = 0;private bypasses = 0;@@ -33,7 +39,23 @@ export class MediaUploadPool {await new Promise<void>((ready) => {const upload = { bytes, ready };- if (!this.startIfPossible(upload)) this.waiting.push(upload);+ if (+ this.waiting.length === 0 &&+ this.active < this.concurrency &&+ (this.active === 0 || bytes <= this.maxBytes - this.reservedBytes)+ ) {+ this.bypasses = 0;+ this.admit(upload);+ return;+ }+ this.waiting.push(upload);+ if (this.waiting.length > this.capacity) {+ this.capacity *= 2;+ this.rebuildIndex();+ } else {+ this.setSize(this.waiting.length - 1, bytes);+ }+ this.admitFittingWaiters();});try {@@ -41,41 +63,90 @@ export class MediaUploadPool {} finally {this.active -= 1;this.reservedBytes -= bytes;- this.startWaiting();+ this.admitFittingWaiters();}}- private startWaiting(): void {- const waiting = this.waiting;- this.waiting = [];- for (const upload of waiting) {- if (!this.startIfPossible(upload)) this.waiting.push(upload);+ // Each call wakes at most the number of slots free when it begins. Waitlist+ // compaction can still scan the full backlog.+ private admitFittingWaiters(): void {+ while (this.active < this.concurrency && this.waiting.length > 0) {+ const available =+ this.active === 0+ ? Number.MAX_SAFE_INTEGER+ : this.maxBytes - this.reservedBytes;+ const index = this.firstFitting(available);+ if (index === undefined) return;+ const bypass = index !== this.first;+ if (bypass && this.bypasses >= this.concurrency) return;++ const upload = this.waiting[index]!;+ this.removeWaiting(index);+ this.bypasses = bypass ? this.bypasses + 1 : 0;+ this.admit(upload);}}- private startIfPossible(upload: PendingUpload): boolean {- if (!this.canStart(upload)) return false;- const hasOlderWaiters = this.waiting.length > 0;- if (hasOlderWaiters && this.bypasses >= this.concurrency) return false;- this.bypasses = hasOlderWaiters ? this.bypasses + 1 : 0;- this.start(upload);- return true;+ // Each subtree stores its smallest reservation, so this finds the earliest fit.+ private firstFitting(bytes: number): number | undefined {+ if (this.minBytes[1]! > bytes) return undefined;+ let node = 1;+ while (node < this.capacity) {+ node = this.minBytes[node * 2]! <= bytes ? node * 2 : node * 2 + 1;+ }+ return node - this.capacity;}- private start(upload: PendingUpload): void {- this.active += 1;- this.reservedBytes += upload.bytes;- upload.ready();+ private removeWaiting(index: number): void {+ this.waiting[index] = undefined;+ this.setSize(index, Infinity);+ while (this.first < this.waiting.length && !this.waiting[this.first]) {+ this.first++;+ }+ if (this.first === this.waiting.length) {+ this.waiting = [];+ this.first = 0;+ this.capacity = 1;+ this.minBytes = [Infinity, Infinity];+ } else if (this.first >= 1024 && this.first * 2 >= this.waiting.length) {+ this.waiting = this.waiting.slice(this.first);+ this.first = 0;+ this.capacity = 1;+ while (this.capacity < this.waiting.length) this.capacity *= 2;+ this.rebuildIndex();+ }+ }++ private setSize(index: number, bytes: number): void {+ let node = this.capacity + index;+ this.minBytes[node] = bytes;+ while (node > 1) {+ node = Math.floor(node / 2);+ this.minBytes[node] = Math.min(+ this.minBytes[node * 2]!,+ this.minBytes[node * 2 + 1]!,+ );+ }}- private canStart(next: PendingUpload): boolean {- if (this.active >= this.concurrency) return false;- if (this.active === 0) return true;+ private rebuildIndex(): void {+ this.minBytes = new Array<number>(this.capacity * 2).fill(Infinity);+ for (let index = 0; index < this.waiting.length; index++) {+ const upload = this.waiting[index];+ if (upload) this.minBytes[this.capacity + index] = upload.bytes;+ }+ for (let node = this.capacity - 1; node > 0; node--) {+ this.minBytes[node] = Math.min(+ this.minBytes[node * 2]!,+ this.minBytes[node * 2 + 1]!,+ );+ }+ }- return (- this.reservedBytes <= this.maxBytes &&- next.bytes <= this.maxBytes - this.reservedBytes- );+ private admit(upload: PendingUpload): void {+ this.active += 1;+ this.reservedBytes += upload.bytes;+ upload.ready();}}diff --git a/packages/shared/src/server/media/index.ts b/packages/shared/src/server/media/index.tsindex ee664b5da8..43068d3383 100644--- a/packages/shared/src/server/media/index.ts+++ b/packages/shared/src/server/media/index.ts@@ -1,2 +1,7 @@export * from "./MediaPayloadProcessor";export * from "./mediaService";+export {+ otelMediaUploads,+ forEachMedia,+ mediaDispatchConcurrency,+} from "./otelMediaUploads";diff --git a/packages/shared/src/server/media/otelMediaUploads.ts b/packages/shared/src/server/media/otelMediaUploads.tsnew file mode 100644index 0000000000..bf21c4aa1b--- /dev/null+++ b/packages/shared/src/server/media/otelMediaUploads.ts@@ -0,0 +1,47 @@+import { env } from "../../env";+import { logger } from "../logger";+import { MediaUploadPool } from "./MediaUploadPool";++// Shared by early-media ingestion jobs in this worker process.+// Build-time imports skip env validation and therefore may not have defaults.+export const otelMediaUploads = new MediaUploadPool(+ Number(env.LANGFUSE_OTEL_MEDIA_UPLOAD_CONCURRENCY ?? 100),+ Number(env.LANGFUSE_OTEL_MEDIA_UPLOAD_MAX_BYTES ?? 128 * 1024 * 1024),+);++// Structured-value and URI traversals can nest; keep each dispatch level small+// even when many batches share a larger upload pool.+export function mediaDispatchConcurrency(): number {+ return Math.min(5, Number(env.LANGFUSE_OTEL_MEDIA_UPLOAD_CONCURRENCY ?? 100));+}++/** Iterate lazily; finish active work, log later failures, then rethrow the first. */+export async function forEachMedia<T>(+ values: Iterable<T>,+ process: (value: T) => Promise<unknown>,+): Promise<void> {+ const iterator = values[Symbol.iterator]();+ let failure: { error: unknown } | undefined;+ const concurrency = mediaDispatchConcurrency();+ const workers = Array.isArray(values)+ ? Math.min(values.length, concurrency)+ : concurrency;+ await Promise.all(+ Array.from({ length: workers }, async () => {+ try {+ while (!failure) {+ const next = iterator.next();+ if (next.done) return;+ await process(next.value);+ }+ } catch (error) {+ if (!failure) {+ failure = { error };+ } else {+ logger.error("Additional OTEL media processing failure", { error });+ }+ }+ }),+ );+ if (failure) throw failure.error;+}diff --git a/worker/src/features/otel-media/mediaReferenceText.ts b/worker/src/features/otel-media/mediaReferenceText.tsindex 1d4b267fd2..b46311363a 100644--- a/worker/src/features/otel-media/mediaReferenceText.ts+++ b/worker/src/features/otel-media/mediaReferenceText.ts@@ -4,6 +4,7 @@ import type {MediaRestoreRequest,} from "@langfuse/native";import { asRecord, defineOwnValue } from "@langfuse/shared";+import { forEachMedia } from "@langfuse/shared/src/server";export type MediaRegistry = ReadonlyMap<string, ExtractedOtelMedia>;@@ -80,11 +81,29 @@ export function createMediaReferenceRewriter(value: string,resolve?: (entry: ExtractedOtelMedia) => Promise<string | undefined>,): Promise<string> => {+ const matches = knownMediaReferenceMatches(value, media);+ let next = matches.next();+ if (next.done) return value;+ // Uploads share the decoded-byte pool; restoration read sizes must not+ // serialize successful uploads of large originals.+ const uploadResults: Array<string | undefined> = [];+ if (resolve) {+ let index = 0;+ // Scan references twice: this pass starts uploads, and the other replaces+ // them in order. Keeping every match saves little time below roughly 10k+ // references, but holds an extra object for each one.+ await forEachMedia(+ knownMediaReferenceMatches(value, media),+ async ({ entry }) => {+ const position = index++;+ uploadResults[position] = await resolve(entry);+ },+ );+ }+ let resultIndex = 0;let layers: Map<number, number> | undefined;let output = "";let end = 0;- const matches = knownMediaReferenceMatches(value, media);- let next = matches.next();while (!next.done) {const group: KnownMediaReferenceMatch[] = [];let sourceBytes = 0;@@ -103,7 +122,7 @@ export function createMediaReferenceRewriter(const requests = new Map<string, MediaRestoreRequest>();const replacements: Array<string | { key: string }> = [];for (const match of group) {- const replacement = await resolve?.(match.entry);+ const replacement = uploadResults[resultIndex++];if (replacement !== undefined) {replacements.push(replacement);continue;@@ -148,10 +167,12 @@ export function createMediaReferenceRewriter(* Rewrite actual object values in place. Actual object keys are not visited;* opaque string values are rewritten as source text, including JSON keys* contained inside those strings.+ * Concurrent rewriting requires an owned JSON tree without shared object nodes.*/export async function rewriteMediaValues(value: unknown,rewrite: (text: string) => Promise<string>,+ concurrent = false,): Promise<unknown> {if (typeof value === "string") return rewrite(value);if (typeof value !== "object" || value === null) return value;@@ -173,22 +194,37 @@ export async function rewriteMediaValues(});};push(value);- while (stack.length > 0) {- const frame = stack[stack.length - 1]!;- if (frame.next === frame.length) {- stack.pop();- continue;- }- const index = frame.next++;- const key = frame.keys ? frame.keys[index]! : index;- const current = frame.object[key];- if (typeof current === "string") {- const next = await rewrite(current);- if (current !== next) defineOwnValue(frame.object, key, next);- } else if (typeof current === "object" && current !== null) {- push(current);+ function* strings() {+ while (stack.length > 0) {+ const frame = stack[stack.length - 1]!;+ if (frame.next === frame.length) {+ stack.pop();+ continue;+ }+ const index = frame.next++;+ const key = frame.keys ? frame.keys[index]! : index;+ const current = frame.object[key];+ if (typeof current === "string") {+ yield { object: frame.object, key, current };+ } else if (typeof current === "object" && current !== null) {+ push(current);+ }}}+ const replace = async ({+ object,+ key,+ current,+ }: {+ object: Record<string, unknown>;+ key: string | number;+ current: string;+ }) => {+ const next = await rewrite(current);+ if (current !== next) defineOwnValue(object, key, next);+ };+ if (concurrent) await forEachMedia(strings(), replace);+ else for (const entry of strings()) await replace(entry);return value;}diff --git a/worker/src/features/otel-media/processOtelMedia.test.ts b/worker/src/features/otel-media/processOtelMedia.test.tsindex 9da2f31230..e397816ecc 100644--- a/worker/src/features/otel-media/processOtelMedia.test.ts+++ b/worker/src/features/otel-media/processOtelMedia.test.ts@@ -5,6 +5,7 @@ import {type ExtractedOtelMedia,type MediaRestoreRequest,} from "@langfuse/native";+import * as otelMediaUploads from "../../../../packages/shared/src/server/media/otelMediaUploads";const mocks = vi.hoisted(() => {const span = { setAttributes: vi.fn() };@@ -24,6 +25,7 @@ const mocks = vi.hoisted(() => {});vi.mock("@langfuse/shared/src/server", () => ({+ ...otelMediaUploads,getClickhouseEntityType: (eventType: string) =>eventType === "trace-create" ? "trace" : "observation",instrumentAsync: mocks.instrumentAsync,@@ -78,6 +80,14 @@ const MEDIA_DESTINATION = {mediaPrefix: "media/",} as const;+function deferred<T>() {+ let resolve!: (value: T) => void;+ const promise = new Promise<T>((promiseResolve) => {+ resolve = promiseResolve;+ });+ return { promise, resolve };+}+function createTestDescriptors(count: number,prefix: string,@@ -527,6 +537,112 @@ describe("processOtelEventMedia", () => {});});+ it.each([+ { name: "nested values", oneString: false },+ { name: "all references in one string", oneString: true },+ ])(+ "reserves upload capacity before decoding media: $name",+ async ({ oneString }) => {+ const reservedBytes = 64 * 1024 * 1024;+ const descriptors = createTestDescriptors(3, "weighted", reservedBytes);+ const { batch, mediaBody } = createFakeMediaBatch({+ entries: descriptors,+ });+ mediaBody.mockImplementation((index: number) =>+ Promise.resolve(Buffer.from([index])),+ );+ const gates = Array.from({ length: 3 }, () =>+ deferred<{ outcome: "uploaded"; mediaId: string }>(),+ );+ const uploadIndexes: number[] = [];+ mocks.uploadMediaForTrace.mockImplementation(+ async ({ contentBytes }: { contentBytes: Buffer }) => {+ const index = contentBytes[0]!;+ uploadIndexes.push(index);+ return gates[index]!.promise;+ },+ );+ const references = descriptors.map((entry) => entry.reference);+ const payload = {+ traceId: "trace-id",+ spanId: "observation-id",+ input: oneString+ ? references.join(" ")+ : [references[0], { nested: references.slice(1) }],+ };+ const resolving = resolveExtractedMedia({+ batch,+ targets: createDirectOtelMediaTargets([payload]),+ ...MEDIA_DESTINATION,+ writePath: "direct",+ });+ try {+ await vi.waitFor(() => expect(uploadIndexes).toHaveLength(2));+ expect(uploadIndexes).toEqual([0, 1]);+ expect(mediaBody.mock.calls.map(([index]) => index)).toEqual([0, 1]);++ gates[0]!.resolve({ outcome: "uploaded", mediaId: "uploaded-0" });+ await vi.waitFor(() => expect(mediaBody).toHaveBeenCalledTimes(3));+ expect(mediaBody.mock.calls.map(([index]) => index)).toEqual([0, 1, 2]);++ gates[2]!.resolve({ outcome: "uploaded", mediaId: "uploaded-2" });+ gates[1]!.resolve({ outcome: "uploaded", mediaId: "uploaded-1" });+ await expect(resolving).resolves.toMatchObject({+ uploaded: 3,+ reused: 0,+ failed: 0,+ });+ } finally {+ for (const gate of gates)+ gate.resolve({ outcome: "uploaded", mediaId: "cleanup" });+ await resolving.catch(() => undefined);+ }+ },+ );++ it.each([+ {+ name: "five strings",+ input: Array.from({ length: 5 }, () =>+ Array(5).fill(DEFAULT_MEDIA_REFERENCE).join(" "),+ ),+ },+ {+ name: "one string",+ input: Array(25).fill(DEFAULT_MEDIA_REFERENCE).join(" "),+ },+ ])(+ "bounds concurrent links for repeated early references: $name",+ async ({ input }) => {+ const { batch } = createFakeMediaBatch();+ const releaseLinks = deferred<void>();+ let activeLinks = 0;+ let peakLinks = 0;+ mocks.linkMediaToTraceOrObservation.mockImplementation(async () => {+ activeLinks++;+ peakLinks = Math.max(peakLinks, activeLinks);+ try {+ await releaseLinks.promise;+ } finally {+ activeLinks--;+ }+ });+ const payload = {+ traceId: "trace-id",+ spanId: "observation-id",+ input,+ };+ const resolving = resolveTarget(batch, payload);+ try {+ await vi.waitFor(() => expect(peakLinks).toBeGreaterThanOrEqual(5));+ expect(peakLinks).toBe(5);+ } finally {+ releaseLinks.resolve();+ await resolving;+ }+ },+ );+it("uploads a descriptor without reading the batch media registry", async () => {const { batch, mediaBody: loadBody, readMedia } = createFakeMediaBatch();readMedia.mockImplementation(() => {@@ -542,6 +658,7 @@ describe("processOtelEventMedia", () => {mediaPrefix: "media/",sha256Hash: "hash",contentType: "image/png",+ byteLengthUpperBound: 5,loadBody,});@@ -681,8 +798,8 @@ describe("processOtelEventMedia", () => {const result = await resolveTarget(batch, target);expect(result).toMatchObject({ failed: 2, uploaded: 0, reused: 0 });expect(target.input).toEqual(input);- // A failed upload is evicted, so the next occurrence can retry independently.- expect(mocks.uploadMediaForTrace).toHaveBeenCalledTimes(2);+ // Distinct spellings share one failed attempt, then restore independently.+ expect(mocks.uploadMediaForTrace).toHaveBeenCalledOnce();});});diff --git a/worker/src/features/otel-media/resolveExtractedMedia.ts b/worker/src/features/otel-media/resolveExtractedMedia.tsindex 0db671ede4..f6b37bab0f 100644--- a/worker/src/features/otel-media/resolveExtractedMedia.ts+++ b/worker/src/features/otel-media/resolveExtractedMedia.ts@@ -1,4 +1,5 @@import type { EarlyOtelBatch, ExtractedOtelMedia } from "@langfuse/native";+import pLimit from "p-limit";import {defineOwnValue,isMediaContentType,@@ -8,6 +9,8 @@ import {} from "@langfuse/shared";import {logger,+ mediaDispatchConcurrency,+ otelMediaUploads,recordDistribution,recordIncrement,type OtelMediaTarget,@@ -83,6 +86,7 @@ export async function uploadExtractedMediaOnce(params: {mediaPrefix: string;sha256Hash: string;contentType: MediaContentType;+ byteLengthUpperBound: number;loadBody: () => Promise<Buffer>;}): Promise<CachedMediaUpload> {const {@@ -123,30 +127,32 @@ export async function uploadExtractedMediaOnce(params: {return { ...result, outcome: "reused" };}- const uploadPromise = (async (): Promise<CachedMediaUpload> => {- const contentBytes = await loadBody();- return {- ...(await uploadMediaForTrace({- projectId,- traceId,- observationId,- field,- contentType,- contentBytes,- sha256Hash,- mediaBucket,- mediaPrefix,- origin: MediaAssociationOrigin.INGESTION_MEDIA_EXTRACTION,- })),- byteLength: contentBytes.length,- };- })();+ const uploadPromise = otelMediaUploads.run(+ params.byteLengthUpperBound,+ async (): Promise<CachedMediaUpload> => {+ const contentBytes = await loadBody();+ return {+ ...(await uploadMediaForTrace({+ projectId,+ traceId,+ observationId,+ field,+ contentType,+ contentBytes,+ sha256Hash,+ mediaBucket,+ mediaPrefix,+ origin: MediaAssociationOrigin.INGESTION_MEDIA_EXTRACTION,+ })),+ byteLength: contentBytes.length,+ };+ },+ );cache.set(key, uploadPromise);try {return await uploadPromise;} catch (error) {- // A failed upload must not poison another occurrence or a later path that- // can retry the same asset with a fresh decoded body.+ // Evict a failed attempt so later calls can retry.if (cache.get(key) === uploadPromise) cache.delete(key);throw error;}@@ -202,6 +208,8 @@ export async function resolveExtractedMedia(params: {const media = await batchMedia(batch);if (media.size === 0) return stats;const rewrite = createMediaReferenceRewriter(batch, media);+ // Share dispatch slots across strings so nested traversals cannot multiply links.+ const dispatch = pLimit(mediaDispatchConcurrency());async function resolveReference(entry: ExtractedOtelMedia,@@ -222,6 +230,8 @@ export async function resolveExtractedMedia(params: {traceId: target.traceId,observationId: target.observationId,field,+ // Native retains encoded source spelling, which bounds the decoded body.+ byteLengthUpperBound: entry.originalByteLength,loadBody: () => batch.mediaBody(entry.index),});const replacement = entry.reference.replace(@@ -266,7 +276,10 @@ export async function resolveExtractedMedia(params: {target.payload[field] = await rewriteMediaValues(target.payload[field],(text) =>- rewrite(text, (entry) => resolveReference(entry, target, field)),+ rewrite(text, (entry) =>+ dispatch(() => resolveReference(entry, target, field)),+ ),+ true,);}}