Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(otel): bound concurrent media uploads by decoded size (#18630)

Valery Meleshkin1be9751db27b
diff --git a/.env.dev.example b/.env.dev.example
index 1346352018..09af7f157f 100644
--- a/.env.dev.example
+++ b/.env.dev.example
@@ -158,6 +158,9 @@ LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
# LANGFUSE_S3_MEDIA_UPLOAD_REQUEST_TIMEOUT_MS=30000
# Opt in to extracting and uploading media embedded in OTEL span attributes.
LANGFUSE_OTEL_MEDIA_UPLOAD_ENABLED=false
+# Per-process OTEL uploads; one oversized item may exceed the byte budget alone.
+# LANGFUSE_OTEL_MEDIA_UPLOAD_CONCURRENCY=100
+# LANGFUSE_OTEL_MEDIA_UPLOAD_MAX_BYTES=134217728
# Early extraction requires both the toggle and a project selection.
LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED=false
# Comma-separated project IDs; empty selects nobody, * selects everyone.
diff --git a/.env.prod.example b/.env.prod.example
index ee29393cdf..e76247f613 100644
--- a/.env.prod.example
+++ b/.env.prod.example
@@ -303,6 +303,9 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true
# LANGFUSE_OTEL_INGESTION_MAX_BODY_BYTES=536870912
# Simulate OTLP worker admission while keeping inline ingestion authoritative (default: false)
# LANGFUSE_OTEL_INGESTION_WORKER_SHADOW_ENABLED=false
+# Per-process OTEL uploads; one oversized item may exceed the byte budget alone.
+# LANGFUSE_OTEL_MEDIA_UPLOAD_CONCURRENCY=100
+# LANGFUSE_OTEL_MEDIA_UPLOAD_MAX_BYTES=134217728
# Early extraction requires both the toggle and a project selection.
# LANGFUSE_OTEL_EARLY_MEDIA_EXTRACTION_ENABLED=false
# Comma-separated project IDs; empty selects nobody, * selects everyone.
diff --git a/packages/shared/src/env.ts b/packages/shared/src/env.ts
index 41d73a2ba0..324f213738 100644
--- a/packages/shared/src/env.ts
+++ b/packages/shared/src/env.ts
@@ -297,6 +297,16 @@ const EnvSchema = z.object({
LANGFUSE_S3_EVENT_UPLOAD_SSE_KMS_KEY_ID: z.string().optional(),
LANGFUSE_S3_EVENT_KEY_MAX_SEGMENT_BYTES:
langfuseS3EventKeyMaxSegmentBytesSchema,
+ LANGFUSE_OTEL_MEDIA_UPLOAD_CONCURRENCY: z.coerce
+ .number()
+ .int()
+ .positive()
+ .default(100),
+ LANGFUSE_OTEL_MEDIA_UPLOAD_MAX_BYTES: z.coerce
+ .number()
+ .int()
+ .positive()
+ .default(128 * 1024 * 1024),
LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: z.string().optional(),
LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: z.string().default(""),
LANGFUSE_S3_MEDIA_UPLOAD_REGION: z.string().optional(),
diff --git a/packages/shared/src/server/media/MediaUploadPool.test.ts b/packages/shared/src/server/media/MediaUploadPool.test.ts
index 0bb32c4820..9f66503bdf 100644
--- a/packages/shared/src/server/media/MediaUploadPool.test.ts
+++ b/packages/shared/src/server/media/MediaUploadPool.test.ts
@@ -226,6 +226,29 @@ describe("MediaUploadPool", () => {
expect(nextStarted).toBe(true);
});
+ it("drains a large backlog within the concurrency limit", async () => {
+ const pool = new MediaUploadPool(4, 4);
+ const gate = deferred<void>();
+ const first = pool.run(4, async () => gate.promise);
+ let active = 0;
+ let peakActive = 0;
+ let completed = 0;
+ const waiting = Array.from({ length: 2050 }, () =>
+ pool.run(1, async () => {
+ active++;
+ peakActive = Math.max(peakActive, active);
+ await Promise.resolve();
+ active--;
+ completed++;
+ }),
+ );
+
+ gate.resolve();
+ await Promise.all([first, ...waiting]);
+ expect(completed).toBe(2050);
+ expect(peakActive).toBeLessThanOrEqual(4);
+ });
+
it("does not share reservations between pool calls", async () => {
const firstPool = new MediaUploadPool(1, 10);
const secondPool = new MediaUploadPool(1, 10);
diff --git a/packages/shared/src/server/media/MediaUploadPool.ts b/packages/shared/src/server/media/MediaUploadPool.ts
index 7db1901648..136b76f66b 100644
--- a/packages/shared/src/server/media/MediaUploadPool.ts
+++ b/packages/shared/src/server/media/MediaUploadPool.ts
@@ -1,11 +1,17 @@
/**
- * Bounds active uploads and decoded bytes. Fitting work may bypass the oldest
- * waiter up to `concurrency` times; then new work waits until it starts.
- * The upload must reject on timeout itself. This pool keeps its reservation
- * until work actually settles; timing out only the waiter would free it too soon.
+ * Bounds active uploads and their decoded bytes. Waiting `run()` calls pause
+ * until admitted; admission reserves capacity before waking them. Each settled
+ * upload admits more work, so progress requires active uploads to settle.
+ * Oversized uploads run alone.
+ *
+ * A fitting upload may bypass the oldest waiter up to `concurrency` times.
+ * Work must enforce its own timeout; its reservation lasts until it settles.
*/
export class MediaUploadPool {
- private waiting: PendingUpload[] = [];
+ private waiting: Array<PendingUpload | undefined> = [];
+ private first = 0;
+ private capacity = 1;
+ private minBytes = [Infinity, Infinity];
private active = 0;
private reservedBytes = 0;
private bypasses = 0;
@@ -33,7 +39,23 @@ export class MediaUploadPool {
await new Promise<void>((ready) => {
const upload = { bytes, ready };
- if (!this.startIfPossible(upload)) this.waiting.push(upload);
+ if (
+ this.waiting.length === 0 &&
+ this.active < this.concurrency &&
+ (this.active === 0 || bytes <= this.maxBytes - this.reservedBytes)
+ ) {
+ this.bypasses = 0;
+ this.admit(upload);
+ return;
+ }
+ this.waiting.push(upload);
+ if (this.waiting.length > this.capacity) {
+ this.capacity *= 2;
+ this.rebuildIndex();
+ } else {
+ this.setSize(this.waiting.length - 1, bytes);
+ }
+ this.admitFittingWaiters();
});
try {
@@ -41,41 +63,90 @@ export class MediaUploadPool {
} finally {
this.active -= 1;
this.reservedBytes -= bytes;
- this.startWaiting();
+ this.admitFittingWaiters();
}
}
- private startWaiting(): void {
- const waiting = this.waiting;
- this.waiting = [];
- for (const upload of waiting) {
- if (!this.startIfPossible(upload)) this.waiting.push(upload);
+ // Each call wakes at most the number of slots free when it begins. Waitlist
+ // compaction can still scan the full backlog.
+ private admitFittingWaiters(): void {
+ while (this.active < this.concurrency && this.waiting.length > 0) {
+ const available =
+ this.active === 0
+ ? Number.MAX_SAFE_INTEGER
+ : this.maxBytes - this.reservedBytes;
+ const index = this.firstFitting(available);
+ if (index === undefined) return;
+ const bypass = index !== this.first;
+ if (bypass && this.bypasses >= this.concurrency) return;
+
+ const upload = this.waiting[index]!;
+ this.removeWaiting(index);
+ this.bypasses = bypass ? this.bypasses + 1 : 0;
+ this.admit(upload);
}
}
- private startIfPossible(upload: PendingUpload): boolean {
- if (!this.canStart(upload)) return false;
- const hasOlderWaiters = this.waiting.length > 0;
- if (hasOlderWaiters && this.bypasses >= this.concurrency) return false;
- this.bypasses = hasOlderWaiters ? this.bypasses + 1 : 0;
- this.start(upload);
- return true;
+ // Each subtree stores its smallest reservation, so this finds the earliest fit.
+ private firstFitting(bytes: number): number | undefined {
+ if (this.minBytes[1]! > bytes) return undefined;
+ let node = 1;
+ while (node < this.capacity) {
+ node = this.minBytes[node * 2]! <= bytes ? node * 2 : node * 2 + 1;
+ }
+ return node - this.capacity;
}
- private start(upload: PendingUpload): void {
- this.active += 1;
- this.reservedBytes += upload.bytes;
- upload.ready();
+ private removeWaiting(index: number): void {
+ this.waiting[index] = undefined;
+ this.setSize(index, Infinity);
+ while (this.first < this.waiting.length && !this.waiting[this.first]) {
+ this.first++;
+ }
+ if (this.first === this.waiting.length) {
+ this.waiting = [];
+ this.first = 0;
+ this.capacity = 1;
+ this.minBytes = [Infinity, Infinity];
+ } else if (this.first >= 1024 && this.first * 2 >= this.waiting.length) {
+ this.waiting = this.waiting.slice(this.first);
+ this.first = 0;
+ this.capacity = 1;
+ while (this.capacity < this.waiting.length) this.capacity *= 2;
+ this.rebuildIndex();
+ }
+ }
+
+ private setSize(index: number, bytes: number): void {
+ let node = this.capacity + index;
+ this.minBytes[node] = bytes;
+ while (node > 1) {
+ node = Math.floor(node / 2);
+ this.minBytes[node] = Math.min(
+ this.minBytes[node * 2]!,
+ this.minBytes[node * 2 + 1]!,
+ );
+ }
}
- private canStart(next: PendingUpload): boolean {
- if (this.active >= this.concurrency) return false;
- if (this.active === 0) return true;
+ private rebuildIndex(): void {
+ this.minBytes = new Array<number>(this.capacity * 2).fill(Infinity);
+ for (let index = 0; index < this.waiting.length; index++) {
+ const upload = this.waiting[index];
+ if (upload) this.minBytes[this.capacity + index] = upload.bytes;
+ }
+ for (let node = this.capacity - 1; node > 0; node--) {
+ this.minBytes[node] = Math.min(
+ this.minBytes[node * 2]!,
+ this.minBytes[node * 2 + 1]!,
+ );
+ }
+ }
- return (
- this.reservedBytes <= this.maxBytes &&
- next.bytes <= this.maxBytes - this.reservedBytes
- );
+ private admit(upload: PendingUpload): void {
+ this.active += 1;
+ this.reservedBytes += upload.bytes;
+ upload.ready();
}
}
diff --git a/packages/shared/src/server/media/index.ts b/packages/shared/src/server/media/index.ts
index ee664b5da8..43068d3383 100644
--- a/packages/shared/src/server/media/index.ts
+++ b/packages/shared/src/server/media/index.ts
@@ -1,2 +1,7 @@
export * from "./MediaPayloadProcessor";
export * from "./mediaService";
+export {
+ otelMediaUploads,
+ forEachMedia,
+ mediaDispatchConcurrency,
+} from "./otelMediaUploads";
diff --git a/packages/shared/src/server/media/otelMediaUploads.ts b/packages/shared/src/server/media/otelMediaUploads.ts
new file mode 100644
index 0000000000..bf21c4aa1b
--- /dev/null
+++ b/packages/shared/src/server/media/otelMediaUploads.ts
@@ -0,0 +1,47 @@
+import { env } from "../../env";
+import { logger } from "../logger";
+import { MediaUploadPool } from "./MediaUploadPool";
+
+// Shared by early-media ingestion jobs in this worker process.
+// Build-time imports skip env validation and therefore may not have defaults.
+export const otelMediaUploads = new MediaUploadPool(
+ Number(env.LANGFUSE_OTEL_MEDIA_UPLOAD_CONCURRENCY ?? 100),
+ Number(env.LANGFUSE_OTEL_MEDIA_UPLOAD_MAX_BYTES ?? 128 * 1024 * 1024),
+);
+
+// Structured-value and URI traversals can nest; keep each dispatch level small
+// even when many batches share a larger upload pool.
+export function mediaDispatchConcurrency(): number {
+ return Math.min(5, Number(env.LANGFUSE_OTEL_MEDIA_UPLOAD_CONCURRENCY ?? 100));
+}
+
+/** Iterate lazily; finish active work, log later failures, then rethrow the first. */
+export async function forEachMedia<T>(
+ values: Iterable<T>,
+ process: (value: T) => Promise<unknown>,
+): Promise<void> {
+ const iterator = values[Symbol.iterator]();
+ let failure: { error: unknown } | undefined;
+ const concurrency = mediaDispatchConcurrency();
+ const workers = Array.isArray(values)
+ ? Math.min(values.length, concurrency)
+ : concurrency;
+ await Promise.all(
+ Array.from({ length: workers }, async () => {
+ try {
+ while (!failure) {
+ const next = iterator.next();
+ if (next.done) return;
+ await process(next.value);
+ }
+ } catch (error) {
+ if (!failure) {
+ failure = { error };
+ } else {
+ logger.error("Additional OTEL media processing failure", { error });
+ }
+ }
+ }),
+ );
+ if (failure) throw failure.error;
+}
diff --git a/worker/src/features/otel-media/mediaReferenceText.ts b/worker/src/features/otel-media/mediaReferenceText.ts
index 1d4b267fd2..b46311363a 100644
--- a/worker/src/features/otel-media/mediaReferenceText.ts
+++ b/worker/src/features/otel-media/mediaReferenceText.ts
@@ -4,6 +4,7 @@ import type {
MediaRestoreRequest,
} from "@langfuse/native";
import { asRecord, defineOwnValue } from "@langfuse/shared";
+import { forEachMedia } from "@langfuse/shared/src/server";
export type MediaRegistry = ReadonlyMap<string, ExtractedOtelMedia>;
@@ -80,11 +81,29 @@ export function createMediaReferenceRewriter(
value: string,
resolve?: (entry: ExtractedOtelMedia) => Promise<string | undefined>,
): Promise<string> => {
+ const matches = knownMediaReferenceMatches(value, media);
+ let next = matches.next();
+ if (next.done) return value;
+ // Uploads share the decoded-byte pool; restoration read sizes must not
+ // serialize successful uploads of large originals.
+ const uploadResults: Array<string | undefined> = [];
+ if (resolve) {
+ let index = 0;
+ // Scan references twice: this pass starts uploads, and the other replaces
+ // them in order. Keeping every match saves little time below roughly 10k
+ // references, but holds an extra object for each one.
+ await forEachMedia(
+ knownMediaReferenceMatches(value, media),
+ async ({ entry }) => {
+ const position = index++;
+ uploadResults[position] = await resolve(entry);
+ },
+ );
+ }
+ let resultIndex = 0;
let layers: Map<number, number> | undefined;
let output = "";
let end = 0;
- const matches = knownMediaReferenceMatches(value, media);
- let next = matches.next();
while (!next.done) {
const group: KnownMediaReferenceMatch[] = [];
let sourceBytes = 0;
@@ -103,7 +122,7 @@ export function createMediaReferenceRewriter(
const requests = new Map<string, MediaRestoreRequest>();
const replacements: Array<string | { key: string }> = [];
for (const match of group) {
- const replacement = await resolve?.(match.entry);
+ const replacement = uploadResults[resultIndex++];
if (replacement !== undefined) {
replacements.push(replacement);
continue;
@@ -148,10 +167,12 @@ export function createMediaReferenceRewriter(
* Rewrite actual object values in place. Actual object keys are not visited;
* opaque string values are rewritten as source text, including JSON keys
* contained inside those strings.
+ * Concurrent rewriting requires an owned JSON tree without shared object nodes.
*/
export async function rewriteMediaValues(
value: unknown,
rewrite: (text: string) => Promise<string>,
+ concurrent = false,
): Promise<unknown> {
if (typeof value === "string") return rewrite(value);
if (typeof value !== "object" || value === null) return value;
@@ -173,22 +194,37 @@ export async function rewriteMediaValues(
});
};
push(value);
- while (stack.length > 0) {
- const frame = stack[stack.length - 1]!;
- if (frame.next === frame.length) {
- stack.pop();
- continue;
- }
- const index = frame.next++;
- const key = frame.keys ? frame.keys[index]! : index;
- const current = frame.object[key];
- if (typeof current === "string") {
- const next = await rewrite(current);
- if (current !== next) defineOwnValue(frame.object, key, next);
- } else if (typeof current === "object" && current !== null) {
- push(current);
+ function* strings() {
+ while (stack.length > 0) {
+ const frame = stack[stack.length - 1]!;
+ if (frame.next === frame.length) {
+ stack.pop();
+ continue;
+ }
+ const index = frame.next++;
+ const key = frame.keys ? frame.keys[index]! : index;
+ const current = frame.object[key];
+ if (typeof current === "string") {
+ yield { object: frame.object, key, current };
+ } else if (typeof current === "object" && current !== null) {
+ push(current);
+ }
}
}
+ const replace = async ({
+ object,
+ key,
+ current,
+ }: {
+ object: Record<string, unknown>;
+ key: string | number;
+ current: string;
+ }) => {
+ const next = await rewrite(current);
+ if (current !== next) defineOwnValue(object, key, next);
+ };
+ if (concurrent) await forEachMedia(strings(), replace);
+ else for (const entry of strings()) await replace(entry);
return value;
}
diff --git a/worker/src/features/otel-media/processOtelMedia.test.ts b/worker/src/features/otel-media/processOtelMedia.test.ts
index 9da2f31230..e397816ecc 100644
--- a/worker/src/features/otel-media/processOtelMedia.test.ts
+++ b/worker/src/features/otel-media/processOtelMedia.test.ts
@@ -5,6 +5,7 @@ import {
type ExtractedOtelMedia,
type MediaRestoreRequest,
} from "@langfuse/native";
+import * as otelMediaUploads from "../../../../packages/shared/src/server/media/otelMediaUploads";
const mocks = vi.hoisted(() => {
const span = { setAttributes: vi.fn() };
@@ -24,6 +25,7 @@ const mocks = vi.hoisted(() => {
});
vi.mock("@langfuse/shared/src/server", () => ({
+ ...otelMediaUploads,
getClickhouseEntityType: (eventType: string) =>
eventType === "trace-create" ? "trace" : "observation",
instrumentAsync: mocks.instrumentAsync,
@@ -78,6 +80,14 @@ const MEDIA_DESTINATION = {
mediaPrefix: "media/",
} as const;
+function deferred<T>() {
+ let resolve!: (value: T) => void;
+ const promise = new Promise<T>((promiseResolve) => {
+ resolve = promiseResolve;
+ });
+ return { promise, resolve };
+}
+
function createTestDescriptors(
count: number,
prefix: string,
@@ -527,6 +537,112 @@ describe("processOtelEventMedia", () => {
});
});
+ it.each([
+ { name: "nested values", oneString: false },
+ { name: "all references in one string", oneString: true },
+ ])(
+ "reserves upload capacity before decoding media: $name",
+ async ({ oneString }) => {
+ const reservedBytes = 64 * 1024 * 1024;
+ const descriptors = createTestDescriptors(3, "weighted", reservedBytes);
+ const { batch, mediaBody } = createFakeMediaBatch({
+ entries: descriptors,
+ });
+ mediaBody.mockImplementation((index: number) =>
+ Promise.resolve(Buffer.from([index])),
+ );
+ const gates = Array.from({ length: 3 }, () =>
+ deferred<{ outcome: "uploaded"; mediaId: string }>(),
+ );
+ const uploadIndexes: number[] = [];
+ mocks.uploadMediaForTrace.mockImplementation(
+ async ({ contentBytes }: { contentBytes: Buffer }) => {
+ const index = contentBytes[0]!;
+ uploadIndexes.push(index);
+ return gates[index]!.promise;
+ },
+ );
+ const references = descriptors.map((entry) => entry.reference);
+ const payload = {
+ traceId: "trace-id",
+ spanId: "observation-id",
+ input: oneString
+ ? references.join(" ")
+ : [references[0], { nested: references.slice(1) }],
+ };
+ const resolving = resolveExtractedMedia({
+ batch,
+ targets: createDirectOtelMediaTargets([payload]),
+ ...MEDIA_DESTINATION,
+ writePath: "direct",
+ });
+ try {
+ await vi.waitFor(() => expect(uploadIndexes).toHaveLength(2));
+ expect(uploadIndexes).toEqual([0, 1]);
+ expect(mediaBody.mock.calls.map(([index]) => index)).toEqual([0, 1]);
+
+ gates[0]!.resolve({ outcome: "uploaded", mediaId: "uploaded-0" });
+ await vi.waitFor(() => expect(mediaBody).toHaveBeenCalledTimes(3));
+ expect(mediaBody.mock.calls.map(([index]) => index)).toEqual([0, 1, 2]);
+
+ gates[2]!.resolve({ outcome: "uploaded", mediaId: "uploaded-2" });
+ gates[1]!.resolve({ outcome: "uploaded", mediaId: "uploaded-1" });
+ await expect(resolving).resolves.toMatchObject({
+ uploaded: 3,
+ reused: 0,
+ failed: 0,
+ });
+ } finally {
+ for (const gate of gates)
+ gate.resolve({ outcome: "uploaded", mediaId: "cleanup" });
+ await resolving.catch(() => undefined);
+ }
+ },
+ );
+
+ it.each([
+ {
+ name: "five strings",
+ input: Array.from({ length: 5 }, () =>
+ Array(5).fill(DEFAULT_MEDIA_REFERENCE).join(" "),
+ ),
+ },
+ {
+ name: "one string",
+ input: Array(25).fill(DEFAULT_MEDIA_REFERENCE).join(" "),
+ },
+ ])(
+ "bounds concurrent links for repeated early references: $name",
+ async ({ input }) => {
+ const { batch } = createFakeMediaBatch();
+ const releaseLinks = deferred<void>();
+ let activeLinks = 0;
+ let peakLinks = 0;
+ mocks.linkMediaToTraceOrObservation.mockImplementation(async () => {
+ activeLinks++;
+ peakLinks = Math.max(peakLinks, activeLinks);
+ try {
+ await releaseLinks.promise;
+ } finally {
+ activeLinks--;
+ }
+ });
+ const payload = {
+ traceId: "trace-id",
+ spanId: "observation-id",
+ input,
+ };
+ const resolving = resolveTarget(batch, payload);
+ try {
+ await vi.waitFor(() => expect(peakLinks).toBeGreaterThanOrEqual(5));
+ expect(peakLinks).toBe(5);
+ } finally {
+ releaseLinks.resolve();
+ await resolving;
+ }
+ },
+ );
+
it("uploads a descriptor without reading the batch media registry", async () => {
const { batch, mediaBody: loadBody, readMedia } = createFakeMediaBatch();
readMedia.mockImplementation(() => {
@@ -542,6 +658,7 @@ describe("processOtelEventMedia", () => {
mediaPrefix: "media/",
sha256Hash: "hash",
contentType: "image/png",
+ byteLengthUpperBound: 5,
loadBody,
});
@@ -681,8 +798,8 @@ describe("processOtelEventMedia", () => {
const result = await resolveTarget(batch, target);
expect(result).toMatchObject({ failed: 2, uploaded: 0, reused: 0 });
expect(target.input).toEqual(input);
- // A failed upload is evicted, so the next occurrence can retry independently.
- expect(mocks.uploadMediaForTrace).toHaveBeenCalledTimes(2);
+ // Distinct spellings share one failed attempt, then restore independently.
+ expect(mocks.uploadMediaForTrace).toHaveBeenCalledOnce();
});
});
diff --git a/worker/src/features/otel-media/resolveExtractedMedia.ts b/worker/src/features/otel-media/resolveExtractedMedia.ts
index 0db671ede4..f6b37bab0f 100644
--- a/worker/src/features/otel-media/resolveExtractedMedia.ts
+++ b/worker/src/features/otel-media/resolveExtractedMedia.ts
@@ -1,4 +1,5 @@
import type { EarlyOtelBatch, ExtractedOtelMedia } from "@langfuse/native";
+import pLimit from "p-limit";
import {
defineOwnValue,
isMediaContentType,
@@ -8,6 +9,8 @@ import {
} from "@langfuse/shared";
import {
logger,
+ mediaDispatchConcurrency,
+ otelMediaUploads,
recordDistribution,
recordIncrement,
type OtelMediaTarget,
@@ -83,6 +86,7 @@ export async function uploadExtractedMediaOnce(params: {
mediaPrefix: string;
sha256Hash: string;
contentType: MediaContentType;
+ byteLengthUpperBound: number;
loadBody: () => Promise<Buffer>;
}): Promise<CachedMediaUpload> {
const {
@@ -123,30 +127,32 @@ export async function uploadExtractedMediaOnce(params: {
return { ...result, outcome: "reused" };
}
- const uploadPromise = (async (): Promise<CachedMediaUpload> => {
- const contentBytes = await loadBody();
- return {
- ...(await uploadMediaForTrace({
- projectId,
- traceId,
- observationId,
- field,
- contentType,
- contentBytes,
- sha256Hash,
- mediaBucket,
- mediaPrefix,
- origin: MediaAssociationOrigin.INGESTION_MEDIA_EXTRACTION,
- })),
- byteLength: contentBytes.length,
- };
- })();
+ const uploadPromise = otelMediaUploads.run(
+ params.byteLengthUpperBound,
+ async (): Promise<CachedMediaUpload> => {
+ const contentBytes = await loadBody();
+ return {
+ ...(await uploadMediaForTrace({
+ projectId,
+ traceId,
+ observationId,
+ field,
+ contentType,
+ contentBytes,
+ sha256Hash,
+ mediaBucket,
+ mediaPrefix,
+ origin: MediaAssociationOrigin.INGESTION_MEDIA_EXTRACTION,
+ })),
+ byteLength: contentBytes.length,
+ };
+ },
+ );
cache.set(key, uploadPromise);
try {
return await uploadPromise;
} catch (error) {
- // A failed upload must not poison another occurrence or a later path that
- // can retry the same asset with a fresh decoded body.
+ // Evict a failed attempt so later calls can retry.
if (cache.get(key) === uploadPromise) cache.delete(key);
throw error;
}
@@ -202,6 +208,8 @@ export async function resolveExtractedMedia(params: {
const media = await batchMedia(batch);
if (media.size === 0) return stats;
const rewrite = createMediaReferenceRewriter(batch, media);
+ // Share dispatch slots across strings so nested traversals cannot multiply links.
+ const dispatch = pLimit(mediaDispatchConcurrency());
async function resolveReference(
entry: ExtractedOtelMedia,
@@ -222,6 +230,8 @@ export async function resolveExtractedMedia(params: {
traceId: target.traceId,
observationId: target.observationId,
field,
+ // Native retains encoded source spelling, which bounds the decoded body.
+ byteLengthUpperBound: entry.originalByteLength,
loadBody: () => batch.mediaBody(entry.index),
});
const replacement = entry.reference.replace(
@@ -266,7 +276,10 @@ export async function resolveExtractedMedia(params: {
target.payload[field] = await rewriteMediaValues(
target.payload[field],
(text) =>
- rewrite(text, (entry) => resolveReference(entry, target, field)),
+ rewrite(text, (entry) =>
+ dispatch(() => resolveReference(entry, target, field)),
+ ),
+ true,
);
}
}