Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(trace-batching): measure pending trace size estimates (#17586)

Lysander Kiesel536e38e5bcdc
diff --git a/worker/src/__tests__/traceBatching.test.ts b/worker/src/__tests__/traceBatching.test.ts
index 7c4f5a71df..6536c90455 100644
--- a/worker/src/__tests__/traceBatching.test.ts
+++ b/worker/src/__tests__/traceBatching.test.ts
@@ -430,9 +430,14 @@ describe("trace micro-batch scheduling with Redis", () => {
};
const member = (projectId: string, traceId: string) =>
JSON.stringify([projectId, traceId]);
- const event = (traceId: string, start = 1_000_000) => ({
+ const event = (
+ traceId: string,
+ start = 1_000_000,
+ serializedEventBytes = 100,
+ ) => ({
traceId,
startTimeISO: new Date(start).toISOString(),
+ serializedEventBytes,
});
const runner = () => {
const instance = new TraceBatchDispatcher();
@@ -822,24 +827,61 @@ describe("trace micro-batch scheduling with Redis", () => {
it("preserves min/max across concurrent out-of-order arrivals, resets readiness, and separates tenants", async () => {
await Promise.all(
[30, 0, 20, 10].map((start) =>
- trackTraceBatchActivity("project", [event("trace", start)]),
+ trackTraceBatchActivity("project", [
+ event("trace", start, 100 + start),
+ event("trace", start, 200 + start),
+ ]),
),
);
const state = JSON.parse(
(await client().hget(stateKey, member("project", "trace")))!,
);
- expect(state).toMatchObject({ minStart: 0, maxStart: 30 });
+ expect(state).toMatchObject({
+ minStart: 0,
+ maxStart: 30,
+ eventUpdateCount: 8,
+ serializedEventBytes: 1_320,
+ });
await makeDue("project", "trace");
- await trackTraceBatchActivity("project", [event("trace", 15)]);
+ await trackTraceBatchActivity("project", [
+ event("trace", 15, 25),
+ event("trace", 15, 25),
+ ]);
+ expect(
+ JSON.parse((await client().hget(stateKey, member("project", "trace")))!),
+ ).toMatchObject({ eventUpdateCount: 10, serializedEventBytes: 1_370 });
expect(
Number(await client().zscore(dueKey, member("project", "trace"))),
).toBeGreaterThan(Date.now() + 590_000);
- await trackTraceBatchActivity("other", [event("trace", 100)]);
+ await trackTraceBatchActivity("other", [event("trace", 100, 7)]);
expect(
JSON.parse((await client().hget(stateKey, member("other", "trace")))!),
- ).toMatchObject({ minStart: 100, maxStart: 100 });
+ ).toMatchObject({
+ minStart: 100,
+ maxStart: 100,
+ eventUpdateCount: 1,
+ serializedEventBytes: 7,
+ });
await runner().processBatch();
expect(await queue.getWaitingCount()).toBe(0);
+ await makeDue("project", "trace");
+ await makeDue("other", "trace");
+ await runner().processBatch();
+ for (const [count, bytes] of [
+ [10, 1_370],
+ [1, 7],
+ ]) {
+ expect(recordDistribution).toHaveBeenCalledWith(
+ "langfuse.trace_batch.estimated_event_update_count",
+ count,
+ { scope: "trace", strategy: "project" },
+ );
+ expect(recordDistribution).toHaveBeenCalledWith(
+ "langfuse.trace_batch.estimated_serialized_event_bytes",
+ bytes,
+ { scope: "trace", strategy: "project" },
+ );
+ }
});
it.each([
@@ -855,9 +897,14 @@ describe("trace micro-batch scheduling with Redis", () => {
);
await trackTraceBatchActivity(
"project",
- traces.map((traceId) => event(traceId)),
+ traces.map((traceId, index) => event(traceId, 1_000_000, index + 1)),
);
- await trackTraceBatchActivity("other", [event("trace-0")]);
+ await trackTraceBatchActivity("project", [
+ event("trace-0", 1_000_000, 5),
+ ]);
+ await trackTraceBatchActivity("other", [
+ event("trace-0", 1_000_000, 10_000),
+ ]);
await client().zadd(
dueKey,
...traces.flatMap((traceId, index) => [
@@ -917,11 +964,158 @@ describe("trace micro-batch scheduling with Redis", () => {
.map(([, value]) => value)
.sort((a, b) => Number(a) - Number(b)),
).toEqual(projectCounts);
+ for (const [metric, estimate] of [
+ ["estimated_event_update_count", "count"],
+ ["estimated_serialized_event_bytes", "bytes"],
+ ] as const) {
+ const expected = add.mock.calls.map(([, job]) =>
+ job.payload.traces.reduce((total, trace) => {
+ if (trace.projectId === "other")
+ return total + (estimate === "count" ? 1 : 10_000);
+ const index = Number(trace.traceId.replace("trace-", ""));
+ const estimates =
+ index === 0
+ ? { count: 2, bytes: 6 }
+ : { count: 1, bytes: index + 1 };
+ return total + estimates[estimate];
+ }, 0),
+ );
+ expect(
+ vi
+ .mocked(recordDistribution)
+ .mock.calls.filter(
+ ([name, , tags]) =>
+ name === `langfuse.trace_batch.${metric}` &&
+ tags?.scope === "batch",
+ )
+ .map(([, value, tags]) => ({ value, tags })),
+ ).toEqual(
+ expected.map((value) => ({
+ value,
+ tags: { scope: "batch", strategy: "project" },
+ })),
+ );
+ }
+ for (const [, job] of add.mock.calls) {
+ for (const trace of job.payload.traces) {
+ expect(Object.keys(trace).sort()).toEqual([
+ "maxStart",
+ "minStart",
+ "projectId",
+ "revision",
+ "traceId",
+ ]);
+ }
+ }
await runner().processBatch();
expect(await queue.getWaitingCount()).toBe(sizes.length);
},
);
+ it("keeps legacy estimates unknown through updates and restores them after dispatch", async () => {
+ const legacyStates = [
+ {},
+ { eventUpdateCount: 4 },
+ { serializedEventBytes: 400 },
+ ];
+ const legacyIds = legacyStates.map((_, index) => `legacy-${index}`);
+ for (const [index, estimates] of legacyStates.entries()) {
+ await client().hset(
+ stateKey,
+ member("project", legacyIds[index]),
+ JSON.stringify({
+ minStart: 1_000_000,
+ maxStart: 1_000_000,
+ revision: randomUUID(),
+ ...estimates,
+ }),
+ );
+ }
+ await trackTraceBatchActivity("project", [
+ ...legacyIds.map((traceId) => event(traceId, 1_000_001, 20)),
+ event("known", 1_000_000, 30),
+ ]);
+ await makeDue("project", ...legacyIds, "known");
+ const dispatcher = runner();
+ await dispatcher.processBatch();
+ const [job] = await queue.getJobs(["wait"]);
+ expect(
+ job.data.payload.traces.map(({ traceId }) => traceId).sort(),
+ ).toEqual(["known", ...legacyIds].sort());
+ const estimateSamples = vi
+ .mocked(recordDistribution)
+ .mock.calls.filter(([name]) =>
+ name.startsWith("langfuse.trace_batch.estimated_"),
+ );
+ expect(estimateSamples).toHaveLength(2);
+ expect(estimateSamples).toEqual(
+ expect.arrayContaining([
+ [
+ "langfuse.trace_batch.estimated_event_update_count",
+ 1,
+ { scope: "trace", strategy: "project" },
+ ],
+ [
+ "langfuse.trace_batch.estimated_serialized_event_bytes",
+ 30,
+ { scope: "trace", strategy: "project" },
+ ],
+ ]),
+ );
+ for (const [scope, count] of [
+ ["trace", 3],
+ ["batch", 1],
+ ] as const) {
+ expect(
+ vi
+ .mocked(recordIncrement)
+ .mock.calls.filter(
+ ([name, , tags]) =>
+ name === "langfuse.trace_batch.estimates_unavailable" &&
+ tags?.scope === scope &&
+ tags?.strategy === "project",
+ )
+ .reduce((total, [, value]) => total + Number(value), 0),
+ ).toBe(count);
+ }
+ expect(await client().hlen(stateKey)).toBe(0);
+
+ vi.clearAllMocks();
+ await trackTraceBatchActivity(
+ "project",
+ legacyIds.map((traceId) => event(traceId, 1_000_002, 20)),
+ );
+ for (const traceId of legacyIds) {
+ expect(
+ JSON.parse(
+ (await client().hget(stateKey, member("project", traceId)))!,
+ ),
+ ).toMatchObject({
+ eventUpdateCount: 1,
+ serializedEventBytes: 20,
+ });
+ }
+ await makeDue("project", ...legacyIds);
+ await dispatcher.processBatch();
+ expect(recordDistribution).toHaveBeenCalledWith(
+ "langfuse.trace_batch.estimated_event_update_count",
+ 3,
+ { scope: "batch", strategy: "project" },
+ );
+ expect(recordDistribution).toHaveBeenCalledWith(
+ "langfuse.trace_batch.estimated_serialized_event_bytes",
+ 60,
+ { scope: "batch", strategy: "project" },
+ );
+ expect(
+ vi
+ .mocked(recordIncrement)
+ .mock.calls.some(
+ ([name]) => name === "langfuse.trace_batch.estimates_unavailable",
+ ),
+ ).toBe(false);
+ });
+
it.each(["project", "locality"] as const)(
"dispatches the complete due cohort across hydration chunks with %s",
async (strategy) => {
diff --git a/worker/src/features/traceBatching/README.md b/worker/src/features/traceBatching/README.md
index 1f4f235732..6eb79a6354 100644
--- a/worker/src/features/traceBatching/README.md
+++ b/worker/src/features/traceBatching/README.md
@@ -35,12 +35,12 @@ processing, and the shared queue accessor, including admin queue inspection.
- `{trace-batch}:due`: sorted set, with JSON `[projectId, traceId]` members and
Redis server time plus idle delay as the score.
- `{trace-batch}:state`: hash with the same members and JSON
- `{minStart, maxStart, revision}` values. Bounds describe observed event start
+ `{minStart, maxStart, revision, eventUpdateCount, serializedEventBytes}` values. Bounds describe observed event start
times; revision protects newer ingestion from an older dispatch acknowledgment.
- `{trace-batch}:dispatcher`: renewable dispatcher lease.
-No event payloads, per-trace byte counts or observation counts are stored here.
-Atomic Lua updates refresh bounds, revision and readiness. Each admitted intake
+No event payloads are stored here. Atomic Lua updates accumulate estimates and
+refresh bounds, revision and readiness. Each admitted intake
sets a shared absolute expiry on both pending keys with `PEXPIREAT`, preserving
any later existing expiry using `PEXPIRETIME` (Redis 7+). With no further intake,
both keys expire after the retention period even if every application stops;
@@ -58,6 +58,31 @@ to the configured trace cap; project leftovers can share a job. The dispatcher
enqueues before acknowledging matching revisions, so concurrent intake stays
pending and failed enqueueing can be retried. Stable job IDs limit duplicates.
+### Pending-window size estimates
+
+Accepted direct-v4 writes contribute their final serialized UTF-8 event size after
+field overflow handling, reusing the writer's existing `event_bytes` calculation.
+The accounting field itself is excluded. Acceptance means admission to the writer,
+not confirmation of a ClickHouse flush. `eventUpdateCount` counts accepted updates,
+including retries and repeated observation versions; neither counter represents
+unique observations or a trace's lifetime total. Deletion after enqueue, readiness
+and two-hour retention are unchanged. Later arrivals start another pending window.
+Entries missing either counter remain unknown until dispatch or expiry removes them.
+
+The dispatcher emits `estimated_event_update_count` and
+`estimated_serialized_event_bytes` distributions under `langfuse.trace_batch`, tagged
+with `scope:trace|batch` and `strategy`. Batch totals require known estimates for
+every member; `estimates_unavailable` counts unknown traces and affected batches.
+Estimates stay out of queue payloads, job IDs and batch selection. They do not cap
+work or change oversized-trace handling. Dispatch retries can emit another sample.
+
+Serialized event bytes differ from the reader's `io_metadata_bytes` metric: they
+include the serialized event fields and JSON encoding, and do not represent RAM,
+network transfer or ClickHouse scan bytes. After rollout, collect a stable hour
+with unchanged sampling and batching settings; compare trace/batch distributions,
+unknown-estimate counts, ingestion latency and Redis command rate/CPU/memory with
+the baseline before choosing any size-aware batching policy.
+
The worker streams full `events_full` payloads and records batch observation,
trace, project and logical payload-byte metrics. The `input_bytes`, `output_bytes`
and `metadata_bytes` distributions under `langfuse.trace_batch` each report one
diff --git a/worker/src/features/traceBatching/traceBatching.ts b/worker/src/features/traceBatching/traceBatching.ts
index 7c3588cc21..20f83dfe58 100644
--- a/worker/src/features/traceBatching/traceBatching.ts
+++ b/worker/src/features/traceBatching/traceBatching.ts
@@ -33,6 +33,7 @@ export type TraceBatchStrategy = "project" | "locality";
export type PendingTrace = {
member: string;
due: number;
+ estimates?: { eventUpdateCount: number; serializedEventBytes: number };
trace: TQueueJobTypes[QueueName.TraceBatch]["payload"]["traces"][number];
};
@@ -451,18 +452,29 @@ const TRACK_SCRIPT = `
${EXPIRE_PENDING_SCRIPT}
local clock = redis.call('TIME')
local now = clock[1] * 1000 + math.floor(clock[2] / 1000)
- for i = 3, #ARGV, 4 do
+ for i = 3, #ARGV, 6 do
local member = ARGV[i]
local minStart = tonumber(ARGV[i + 1])
local maxStart = tonumber(ARGV[i + 2])
+ local eventUpdateCount = tonumber(ARGV[i + 4])
+ local serializedEventBytes = tonumber(ARGV[i + 5])
local previous = redis.call('HGET', KEYS[2], member)
if previous then
local state = cjson.decode(previous)
minStart = math.min(minStart, state.minStart)
maxStart = math.max(maxStart, state.maxStart)
+ if state.eventUpdateCount and state.serializedEventBytes then
+ eventUpdateCount = eventUpdateCount + state.eventUpdateCount
+ serializedEventBytes = serializedEventBytes + state.serializedEventBytes
+ else
+ -- Missing history cannot be reconstructed from a later update.
+ eventUpdateCount = nil
+ serializedEventBytes = nil
+ end
end
redis.call('HSET', KEYS[2], member, cjson.encode({
- minStart = minStart, maxStart = maxStart, revision = ARGV[i + 3]
+ minStart = minStart, maxStart = maxStart, revision = ARGV[i + 3],
+ eventUpdateCount = eventUpdateCount, serializedEventBytes = serializedEventBytes
}))
redis.call('ZADD', KEYS[1], now + tonumber(ARGV[1]), member)
end
@@ -531,7 +543,11 @@ const ACKNOWLEDGE_SCRIPT = `
export async function trackTraceBatchActivity(
projectId: string,
- events: { traceId: string; startTimeISO: string }[],
+ events: {
+ traceId: string;
+ startTimeISO: string;
+ serializedEventBytes: number;
+ }[],
): Promise<void> {
if (
!env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION ||
@@ -539,7 +555,15 @@ export async function trackTraceBatchActivity(
)
return;
- const bounds = new Map<string, { minStart: number; maxStart: number }>();
+ const bounds = new Map<
+ string,
+ {
+ minStart: number;
+ maxStart: number;
+ eventUpdateCount: number;
+ serializedEventBytes: number;
+ }
+ >();
for (const event of events) {
const start = Date.parse(event.startTimeISO);
if (!Number.isFinite(start)) continue;
@@ -547,6 +571,9 @@ export async function trackTraceBatchActivity(
bounds.set(event.traceId, {
minStart: Math.min(previous?.minStart ?? start, start),
maxStart: Math.max(previous?.maxStart ?? start, start),
+ eventUpdateCount: (previous?.eventUpdateCount ?? 0) + 1,
+ serializedEventBytes:
+ (previous?.serializedEventBytes ?? 0) + event.serializedEventBytes,
});
}
if (bounds.size === 0) return;
@@ -593,6 +620,8 @@ export async function trackTraceBatchActivity(
state.maxStart,
// Unique even after dispatch deletes and a later arrival recreates state.
randomUUID(),
+ state.eventUpdateCount,
+ state.serializedEventBytes,
]),
),
);
@@ -765,7 +794,26 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {
recordIncrement("langfuse.trace_batch.dispatched_traces", batch.length, {
batch_kind: batch.length === 1 ? "singleton" : "multi",
});
+ let estimatedEventUpdateCount = 0;
+ let estimatedSerializedEventBytes = 0;
+ let unavailableEstimates = 0;
for (const entry of batch) {
+ if (entry.estimates) {
+ estimatedEventUpdateCount += entry.estimates.eventUpdateCount;
+ estimatedSerializedEventBytes += entry.estimates.serializedEventBytes;
+ recordDistribution(
+ "langfuse.trace_batch.estimated_event_update_count",
+ entry.estimates.eventUpdateCount,
+ { scope: "trace", strategy },
+ );
+ recordDistribution(
+ "langfuse.trace_batch.estimated_serialized_event_bytes",
+ entry.estimates.serializedEventBytes,
+ { scope: "trace", strategy },
+ );
+ } else {
+ unavailableEstimates++;
+ }
recordDistribution(
"langfuse.trace_batch.observed_start_span_ms",
entry.trace.maxStart - entry.trace.minStart,
@@ -776,6 +824,31 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {
Math.max(0, Date.now() - entry.due),
);
}
+ if (unavailableEstimates === 0) {
+ recordDistribution(
+ "langfuse.trace_batch.estimated_event_update_count",
+ estimatedEventUpdateCount,
+ { scope: "batch", strategy },
+ );
+ recordDistribution(
+ "langfuse.trace_batch.estimated_serialized_event_bytes",
+ estimatedSerializedEventBytes,
+ { scope: "batch", strategy },
+ );
+ } else {
+ recordIncrement(
+ "langfuse.trace_batch.estimates_unavailable",
+ unavailableEstimates,
+ {
+ scope: "trace",
+ strategy,
+ },
+ );
+ recordIncrement("langfuse.trace_batch.estimates_unavailable", 1, {
+ scope: "batch",
+ strategy,
+ });
+ }
const removed = Number(
await redis!.eval(
ACKNOWLEDGE_SCRIPT,
@@ -818,11 +891,22 @@ export class TraceBatchDispatcher extends PeriodicExclusiveRunner {
if (this.stopping) return;
const member = hydrated[i];
const [projectId, traceId] = JSON.parse(member) as [string, string];
+ const state = JSON.parse(hydrated[i + 1]);
hydratedCandidates.push({
member,
due: Number(hydrated[i + 2]),
+ estimates:
+ Number.isSafeInteger(state.eventUpdateCount) &&
+ state.eventUpdateCount >= 0 &&
+ Number.isSafeInteger(state.serializedEventBytes) &&
+ state.serializedEventBytes >= 0
+ ? {
+ eventUpdateCount: state.eventUpdateCount,
+ serializedEventBytes: state.serializedEventBytes,
+ }
+ : undefined,
trace: TraceBatchTraceSchema.parse({
- ...JSON.parse(hydrated[i + 1]),
+ ...state,
projectId,
traceId,
}),
diff --git a/worker/src/queues/__tests__/otelDirectEventWrite.test.ts b/worker/src/queues/__tests__/otelDirectEventWrite.test.ts
index 1b65d2ce64..6a1e388ec4 100644
--- a/worker/src/queues/__tests__/otelDirectEventWrite.test.ts
+++ b/worker/src/queues/__tests__/otelDirectEventWrite.test.ts
@@ -92,7 +92,7 @@ describe("direct-v4 trace batch tracking", () => {
"createEventRecord",
).mockImplementation(async (input) => {
if (input.traceId === "conversion-failed") throw new Error("invalid");
- return { span_id: input.spanId } as Awaited<
+ return { span_id: input.spanId, event_bytes: 10_000 } as Awaited<
ReturnType<IngestionService["createEventRecord"]>
>;
});
@@ -100,6 +100,7 @@ describe("direct-v4 trace batch tracking", () => {
.spyOn(IngestionService.prototype, "writeEventRecord")
.mockImplementation(async (record) => {
if (record.span_id === "third") throw new Error("write rejected");
+ return record.span_id === "first" ? 101 : 202;
});
const processor = otelIngestionQueueProcessorBuilder(false);
const job = {
@@ -126,9 +127,11 @@ describe("direct-v4 trace batch tracking", () => {
expect(write).toHaveBeenCalledTimes(3);
expect(trackTraceBatchActivity).toHaveBeenCalledExactlyOnceWith(
"project",
- inputs
- .slice(0, 2)
- .map(({ traceId, startTimeISO }) => ({ traceId, startTimeISO })),
+ inputs.slice(0, 2).map(({ traceId, startTimeISO }, index) => ({
+ traceId,
+ startTimeISO,
+ serializedEventBytes: index === 0 ? 101 : 202,
+ })),
);
expect(recordDistribution).toHaveBeenCalledWith(
"langfuse.trace_batch.ingestion_trace_count",
diff --git a/worker/src/queues/otelIngestionQueue.ts b/worker/src/queues/otelIngestionQueue.ts
index 3b5045a629..876d442a83 100644
--- a/worker/src/queues/otelIngestionQueue.ts
+++ b/worker/src/queues/otelIngestionQueue.ts
@@ -832,7 +832,11 @@ export const otelIngestionQueueProcessorBuilder = (
? createObservationEvalSchedulerDeps()
: null;
- const traceBatchEvents: { traceId: string; startTimeISO: string }[] = [];
+ const traceBatchEvents: {
+ traceId: string;
+ startTimeISO: string;
+ serializedEventBytes: number;
+ }[] = [];
await Promise.all(
// Process each event independently
@@ -883,7 +887,8 @@ export const otelIngestionQueueProcessorBuilder = (
// Step 3: Write to events table (independent of eval scheduling)
if (shouldWriteToEventsTable) {
try {
- await ingestionService.writeEventRecord(eventRecord);
+ const serializedEventBytes =
+ await ingestionService.writeEventRecord(eventRecord);
if (
env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION &&
env.LANGFUSE_TRACE_BATCH_INGESTION_ENABLED === "true"
@@ -891,6 +896,7 @@ export const otelIngestionQueueProcessorBuilder = (
traceBatchEvents.push({
traceId: eventInput.traceId,
startTimeISO: eventInput.startTimeISO,
+ serializedEventBytes,
});
}
} catch (error) {
diff --git a/worker/src/services/IngestionService/index.ts b/worker/src/services/IngestionService/index.ts
index 01a44ecfdd..e4b82892f3 100644
--- a/worker/src/services/IngestionService/index.ts
+++ b/worker/src/services/IngestionService/index.ts
@@ -500,16 +500,17 @@ export class IngestionService {
* supplied record is not mutated.
*
* @param eventRecord - The event record to write
+ * @returns The serialized byte size of the accepted event record
*/
public async writeEventRecord(
eventRecord: EventRecordInsertType,
- ): Promise<void> {
- const persistedRecord = await applyObservationFieldOverflow(eventRecord);
-
- this.clickHouseWriter.addToQueue(
- TableName.EventsFull,
- withSerializedEventByteLength(persistedRecord),
+ ): Promise<number> {
+ const persistedRecord = withSerializedEventByteLength(
+ await applyObservationFieldOverflow(eventRecord),
);
+
+ this.clickHouseWriter.addToQueue(TableName.EventsFull, persistedRecord);
+ return persistedRecord.event_bytes;
}
private async processDatasetRunItemEventList(params: {
diff --git a/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts b/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts
index 9a9b7ef67e..af46f770ad 100644
--- a/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts
+++ b/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts
@@ -77,7 +77,7 @@ describe("IngestionService unit tests", () => {
expect(eventRecord.event_bytes).toBe(rawOtelSpanBytes);
- await ingestionService.writeEventRecord(eventRecord);
+ const acceptedBytes = await ingestionService.writeEventRecord(eventRecord);
expect(addToQueue).toHaveBeenCalledOnce();
const queuedRecord = addToQueue.mock.calls[0]?.[1];
@@ -89,6 +89,7 @@ describe("IngestionService unit tests", () => {
Buffer.byteLength(JSON.stringify(eventWithoutSize), "utf8"),
);
expect(eventBytes).toBeLessThan(rawOtelSpanBytes);
+ expect(acceptedBytes).toBe(eventBytes);
});
it("overflows only the direct events_full copy and preserves the enriched record", async () => {
@@ -128,7 +129,7 @@ describe("IngestionService unit tests", () => {
],
});
- await ingestionService.writeEventRecord(eventRecord);
+ const acceptedBytes = await ingestionService.writeEventRecord(eventRecord);
expect(mocks.applyObservationFieldOverflow).toHaveBeenCalledWith(
eventRecord,
@@ -141,6 +142,12 @@ describe("IngestionService unit tests", () => {
metadata_names: ["keep", "large.nested"],
metadata_values: ["small", expect.stringContaining("metadata-media")],
});
+ const { event_bytes: eventBytes, ...eventWithoutSize } =
+ addToQueue.mock.calls[0]![1];
+ expect(acceptedBytes).toBe(eventBytes);
+ expect(acceptedBytes).toBe(
+ Buffer.byteLength(JSON.stringify(eventWithoutSize), "utf8"),
+ );
});
it("promotes provided usage and cost for model-less direct events", async () => {