Skip to content

The open-source app everyone uses to manage agents at work

feat(duplex): bound aggregate duplex route resource consumption with a process-owned byte ledger (#12003)

Nicky Leach05b35d4669ce
diff --git a/packages/adapter-utils/src/duplex-aggregate-byte-ledger.test.ts b/packages/adapter-utils/src/duplex-aggregate-byte-ledger.test.ts
new file mode 100644
index 0000000000..790100509e
--- /dev/null
+++ b/packages/adapter-utils/src/duplex-aggregate-byte-ledger.test.ts
@@ -0,0 +1,221 @@
+import { describe, expect, it } from "vitest";
+import {
+ DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES,
+ DUPLEX_AGGREGATE_TOKEN_OWNERS,
+ DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED,
+ DuplexAggregateByteLedger,
+ MIN_HOST_PROCESS_MEMORY_BYTES_FOR_DUPLEX,
+ assertDuplexAggregateCeilingBytes,
+ resolveDuplexAggregateCeilingBytes,
+ type DuplexAggregateByteLedgerTelemetry,
+} from "./duplex-aggregate-byte-ledger.js";
+
+// A telemetry surface that counts each fixed record, so a test asserts the exact
+// gauge value and the exact counter counts.
+function createCountingTelemetry(): DuplexAggregateByteLedgerTelemetry & {
+ gauge: number;
+ rejections: number;
+ underflows: number;
+} {
+ const state = { gauge: 0, rejections: 0, underflows: 0 };
+ return {
+ ...state,
+ setBytesInUse(bytes: number) {
+ this.gauge = bytes;
+ },
+ recordReservationRejection() {
+ this.rejections += 1;
+ },
+ recordAccountingUnderflow() {
+ this.underflows += 1;
+ },
+ };
+}
+
+describe("DuplexAggregateByteLedger", () => {
+ it("reserves under the ceiling and tracks the gauge and the live-token count", () => {
+ const telemetry = createCountingTelemetry();
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1000, telemetry });
+
+ const a = ledger.reserve("buffered_chunk", 400);
+ const b = ledger.reserve("buffered_chunk", 500);
+
+ expect(a).not.toBeNull();
+ expect(b).not.toBeNull();
+ expect(ledger.bytesInUse).toBe(900);
+ expect(ledger.liveTokenCount).toBe(2);
+ expect(telemetry.gauge).toBe(900);
+ expect(a?.bytes).toBe(400);
+ expect(a?.owner).toBe("buffered_chunk");
+ expect(a?.state).toBe("held");
+ });
+
+ it("rejects a one-byte-over reservation, retains nothing, and increments the rejection counter", () => {
+ const telemetry = createCountingTelemetry();
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1000, telemetry });
+
+ ledger.reserve("buffered_chunk", 1000);
+ const over = ledger.reserve("buffered_chunk", 1);
+
+ expect(over).toBeNull();
+ expect(ledger.bytesInUse).toBe(1000);
+ expect(ledger.liveTokenCount).toBe(1);
+ expect(telemetry.rejections).toBe(1);
+ // The public rejection marker is the fixed string the caller reports.
+ expect(DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED).toBe("DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED");
+ });
+
+ it("transfers ownership without a decrement or a re-reserve, so no admission gap opens", () => {
+ const telemetry = createCountingTelemetry();
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 100, telemetry });
+
+ // Reserve the whole ceiling as a pre-bind event, then transfer to buffered.
+ const token = ledger.reserve("pre_bind_event", 100);
+ expect(token).not.toBeNull();
+ // A second reservation cannot fit while the token stays held.
+ expect(ledger.reserve("buffered_chunk", 1)).toBeNull();
+
+ ledger.transfer(token!, "buffered_chunk");
+
+ // The transfer changed only the owner. The gauge never moved, so no window
+ // opened where the bytes were released and re-admitted.
+ expect(ledger.bytesInUse).toBe(100);
+ expect(ledger.liveTokenCount).toBe(1);
+ expect(token?.owner).toBe("buffered_chunk");
+ expect(telemetry.gauge).toBe(100);
+ // No reservation fit during the transfer, so no rejection or underflow fired
+ // beyond the one rejection above.
+ expect(telemetry.rejections).toBe(1);
+ expect(telemetry.underflows).toBe(0);
+ });
+
+ it("releases a token one time and decrements the gauge", () => {
+ const telemetry = createCountingTelemetry();
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1000, telemetry });
+
+ const token = ledger.reserve("response_body", 250);
+ expect(token).not.toBeNull();
+ ledger.release(token!);
+
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ expect(token?.state).toBe("released");
+ expect(telemetry.gauge).toBe(0);
+ expect(telemetry.underflows).toBe(0);
+ });
+
+ it("records the underflow counter on a second release and does not clamp the gauge again", () => {
+ const telemetry = createCountingTelemetry();
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1000, telemetry });
+
+ const kept = ledger.reserve("request_frame", 300);
+ const doubled = ledger.reserve("request_frame", 200);
+ expect(kept).not.toBeNull();
+ expect(doubled).not.toBeNull();
+ ledger.release(doubled!);
+ expect(ledger.bytesInUse).toBe(300);
+
+ // A forced second release of the same token is an accounting defect.
+ ledger.release(doubled!);
+
+ expect(telemetry.underflows).toBe(1);
+ // The gauge did not drop a second time, so the defect stays visible.
+ expect(ledger.bytesInUse).toBe(300);
+ expect(ledger.liveTokenCount).toBe(1);
+ });
+
+ it("records the underflow counter on a transfer of a released token", () => {
+ const telemetry = createCountingTelemetry();
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1000, telemetry });
+ const token = ledger.reserve("pre_bind_event", 100);
+ ledger.release(token!);
+
+ ledger.transfer(token!, "buffered_chunk");
+
+ expect(telemetry.underflows).toBe(1);
+ expect(ledger.bytesInUse).toBe(0);
+ });
+
+ it("throws on a non-integer or negative reservation, so a programming error fails loud", () => {
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1000 });
+ expect(() => ledger.reserve("buffered_chunk", -1)).toThrow();
+ expect(() => ledger.reserve("buffered_chunk", 1.5)).toThrow();
+ expect(() => ledger.reserve("buffered_chunk", Number.NaN)).toThrow();
+ });
+
+ it("admits a zero-byte reservation as a valid held token", () => {
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1000 });
+ const token = ledger.reserve("seen_request_id", 0);
+ expect(token).not.toBeNull();
+ expect(token?.bytes).toBe(0);
+ expect(ledger.liveTokenCount).toBe(1);
+ });
+
+ it("exposes the closed owner-label set", () => {
+ expect([...DUPLEX_AGGREGATE_TOKEN_OWNERS]).toEqual([
+ "pre_bind_event",
+ "buffered_chunk",
+ "terminal_buffered",
+ "request_frame",
+ "request_payload",
+ "response_body",
+ "seen_request_id",
+ "decoder_buffer",
+ "readiness_buffer",
+ "readiness_replay",
+ "pending_write",
+ "stdin_write",
+ ]);
+ });
+});
+
+describe("aggregate ceiling configuration", () => {
+ it("uses the default when the override is absent", () => {
+ expect(resolveDuplexAggregateCeilingBytes()).toBe(DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES);
+ expect(resolveDuplexAggregateCeilingBytes(null)).toBe(DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES);
+ expect(DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES).toBe(256 * 1024 * 1024);
+ });
+
+ it("accepts a valid positive override under the process allocation minus the reserve", () => {
+ expect(resolveDuplexAggregateCeilingBytes(64 * 1024 * 1024)).toBe(64 * 1024 * 1024);
+ const maxCeiling =
+ MIN_HOST_PROCESS_MEMORY_BYTES_FOR_DUPLEX - DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES;
+ expect(resolveDuplexAggregateCeilingBytes(maxCeiling)).toBe(maxCeiling);
+ });
+
+ it("rejects an invalid override to the safe default and reports each rejection", () => {
+ const overMaximum = MIN_HOST_PROCESS_MEMORY_BYTES_FOR_DUPLEX;
+ const invalidOverrides = [0, -1, 1.5, Number.POSITIVE_INFINITY, Number.NaN, overMaximum];
+ for (const invalid of invalidOverrides) {
+ const rejected: number[] = [];
+ // An invalid present override returns the safe default. It never throws, so a
+ // single bad environment value never fails host startup.
+ expect(resolveDuplexAggregateCeilingBytes(invalid, (value) => rejected.push(value))).toBe(
+ DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES,
+ );
+ // The resolver reports the rejection one time with the rejected numeric value.
+ expect(rejected).toEqual([invalid]);
+ }
+ });
+
+ it("does not report a rejection for a valid or a missing override", () => {
+ const rejected: number[] = [];
+ const reporter = (value: number): void => {
+ rejected.push(value);
+ };
+ expect(resolveDuplexAggregateCeilingBytes(64 * 1024 * 1024, reporter)).toBe(64 * 1024 * 1024);
+ expect(resolveDuplexAggregateCeilingBytes(undefined, reporter)).toBe(
+ DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES,
+ );
+ expect(resolveDuplexAggregateCeilingBytes(null, reporter)).toBe(
+ DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES,
+ );
+ expect(rejected).toEqual([]);
+ });
+
+ it("still throws for a direct ceiling assertion on an unsafe integer", () => {
+ // The ledger constructor asserts its ceiling and must fail loud on a
+ // programming error, so `assertDuplexAggregateCeilingBytes` keeps throwing.
+ expect(() => assertDuplexAggregateCeilingBytes(Number.MAX_SAFE_INTEGER + 2)).toThrow();
+ });
+});
diff --git a/packages/adapter-utils/src/duplex-aggregate-byte-ledger.ts b/packages/adapter-utils/src/duplex-aggregate-byte-ledger.ts
new file mode 100644
index 0000000000..e7f7aec3c1
--- /dev/null
+++ b/packages/adapter-utils/src/duplex-aggregate-byte-ledger.ts
@@ -0,0 +1,400 @@
+/**
+ * The process-owned aggregate byte ledger for the sandbox duplex channel.
+ *
+ * The route-count controller bounds how many concurrent duplex routes stay live.
+ * It does not bound the aggregate bytes those routes retain. At the maximum route
+ * count the per-route byte bounds multiply to many gigabytes of retained bytes.
+ * This ledger closes that gap. One ledger per host server process owns a single
+ * byte ceiling. Every host-side retention site reserves its exact retained bytes
+ * against this ledger before it allocates, so the aggregate retained bytes across
+ * all live routes never pass the ceiling.
+ *
+ * The ledger is fail-closed. A reservation that would pass the ceiling returns no
+ * token. The caller retains nothing and reports the fixed rejection marker
+ * {@link DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED}.
+ *
+ * Ownership is transfer-only. A token holds an immutable byte amount, an owner
+ * label from a closed enum, and a one-way state: `held` then `released`. The
+ * ledger moves a token from one representation to the next with
+ * {@link DuplexAggregateByteLedger.transfer}, which changes only the owner label.
+ * It never decrements and re-reserves, so no admission gap opens between two
+ * representations of the same retained bytes.
+ *
+ * The ledger holds every live token in one owner registry. Registry removal and
+ * the release of a token run in one synchronous critical section, before any
+ * `await` or external callback. A second release of the same token is an
+ * accounting defect. The ledger records the fixed underflow counter and does not
+ * clamp the gauge down a second time, so a real defect stays visible.
+ *
+ * The ledger carries no payload data and no arbitrary labels. It exposes only the
+ * byte gauge, the live-token count, and the closed owner enum.
+ */
+
+import {
+ DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL,
+ DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL,
+ DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE,
+} from "./duplex-telemetry.js";
+
+/**
+ * The public rejection marker. A retention site that cannot reserve its bytes
+ * reports this fixed string. It carries no payload, no route, and no raw value.
+ */
+export const DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED = "DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED";
+
+/**
+ * The default aggregate ceiling, in bytes: 256 MiB. This is one eighth of the
+ * documented minimum host process memory allocation for the sandbox duplex
+ * transport ({@link MIN_HOST_PROCESS_MEMORY_BYTES_FOR_DUPLEX}, 2 GiB). The
+ * remaining seven eighths cover the Node runtime, ordinary server work, parser
+ * and transient headroom, and allocator or garbage-collector variance.
+ */
+export const DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES = 256 * 1024 * 1024;
+
+/**
+ * The documented minimum host process memory allocation to enable the sandbox
+ * duplex transport, in bytes: 2 GiB. An operator override of the aggregate
+ * ceiling must not pass this value minus the documented reserve.
+ */
+export const MIN_HOST_PROCESS_MEMORY_BYTES_FOR_DUPLEX = 2 * 1024 * 1024 * 1024;
+
+/**
+ * The closed set of owner labels a reservation token can carry. Each label names
+ * one host-side representation that retains bytes. The ledger accepts no label
+ * outside this set, so no arbitrary label ever reaches a token.
+ *
+ * - `pre_bind_event`: a held pre-bind duplex event, before route bind.
+ * - `buffered_chunk`: a buffered data chunk retained for a late listener.
+ * - `terminal_buffered`: a buffered chunk moved to the terminal registry at
+ * route terminalization, held for a late listener drain.
+ * - `request_frame`: a bounded raw request frame the broker retains at dispatch.
+ * - `request_payload`: a normalized request payload the broker retains at dispatch.
+ * - `response_body`: a response-body chunk or replacement buffer the reader retains.
+ * - `seen_request_id`: one entry of the broker no-replay request-id set.
+ * - `decoder_buffer`: the raw partial-frame bytes the host frame decoder retains
+ * between chunks, plus the peak replacement buffer it allocates on concat.
+ * - `readiness_buffer`: the raw untrusted bytes the readiness gate retains before
+ * the READY frame completes, before the broker binds and the decoder takes over.
+ * - `readiness_replay`: the post-READY suffix bytes the readiness gate holds in the
+ * pending replay buffer, from the READY accept until the broker binds and the
+ * frame decoder charges its own retention. This is a separate retention from
+ * `readiness_buffer`: the gate drops the whole pre-READY buffer on READY, then
+ * charges only the retained suffix and each later pre-bind chunk under this owner.
+ * - `pending_write`: the raw host-to-worker write payload a pending duplex write
+ * RPC retains, from the enqueue seam until the RPC settles.
+ * - `stdin_write`: the serialized host-to-worker frame the child-stdin transport
+ * buffer retains, from the write until the stream flushes the chunk, the stream
+ * errors, the stream closes, or the worker exits. This is a separate retention
+ * from `pending_write`: the RPC holds the raw payload while the transport buffer
+ * holds the larger serialized frame, so the two tokens cover the peak of both.
+ */
+export const DUPLEX_AGGREGATE_TOKEN_OWNERS = [
+ "pre_bind_event",
+ "buffered_chunk",
+ "terminal_buffered",
+ "request_frame",
+ "request_payload",
+ "response_body",
+ "seen_request_id",
+ "decoder_buffer",
+ "readiness_buffer",
+ "readiness_replay",
+ "pending_write",
+ "stdin_write",
+] as const;
+
+/** One owner label from the closed {@link DUPLEX_AGGREGATE_TOKEN_OWNERS} set. */
+export type DuplexAggregateTokenOwner = (typeof DUPLEX_AGGREGATE_TOKEN_OWNERS)[number];
+
+/** The one-way lifecycle state of a reservation token. */
+export type DuplexReservationTokenState = "held" | "released";
+
+/**
+ * An opaque reservation token. The byte amount is immutable. The owner label
+ * moves through the closed enum by transfer only. The state moves one way, from
+ * `held` to `released`. A caller reads these fields but never mutates them; only
+ * the owning ledger changes the owner and the state.
+ */
+export interface ReservationToken {
+ /** The immutable exact byte amount this token reserves. */
+ readonly bytes: number;
+ /** The current owner label. The ledger changes it only through `transfer`. */
+ readonly owner: DuplexAggregateTokenOwner;
+ /** The one-way lifecycle state. The ledger sets it to `released` one time. */
+ readonly state: DuplexReservationTokenState;
+}
+
+/**
+ * The internal mutable token. The ledger owns it and returns it typed as the
+ * read-only {@link ReservationToken}. Only code in this module changes the owner
+ * or the state.
+ */
+class ReservationTokenImpl implements ReservationToken {
+ readonly bytes: number;
+ owner: DuplexAggregateTokenOwner;
+ state: DuplexReservationTokenState = "held";
+
+ constructor(bytes: number, owner: DuplexAggregateTokenOwner) {
+ this.bytes = bytes;
+ this.owner = owner;
+ }
+}
+
+/**
+ * The fixed telemetry surface for the aggregate byte ledger. The host binds it to
+ * the closed duplex telemetry contract. Each record uses only fixed constant
+ * dimensions. No company, agent, run, route, URL, header, request id, token, byte
+ * payload, or raw provider value ever reaches a record. The default is a no-op.
+ */
+export interface DuplexAggregateByteLedgerTelemetry {
+ /** Set the aggregate-bytes-in-use gauge to the current value. */
+ setBytesInUse(bytes: number): void;
+ /** Increment the reservation-rejection counter one time. */
+ recordReservationRejection(): void;
+ /** Increment the accounting-underflow counter one time. */
+ recordAccountingUnderflow(): void;
+}
+
+/** A no-op telemetry surface. Every method does nothing, so the ledger stays inert. */
+export const NOOP_DUPLEX_AGGREGATE_BYTE_LEDGER_TELEMETRY: DuplexAggregateByteLedgerTelemetry = {
+ setBytesInUse() {},
+ recordReservationRejection() {},
+ recordAccountingUnderflow() {},
+};
+
+/**
+ * The low-level process metric sink the host binds. It carries the fixed metric
+ * name and the value only, so no dynamic dimension reaches a sink. The host maps
+ * the gauge and the two counters to the process metric pipeline and to a warn log
+ * for a defect.
+ */
+export interface DuplexAggregateByteLedgerMetricSink {
+ /** Set a named process gauge to a value. */
+ setGauge(name: string, value: number): void;
+ /** Increment a named process counter one time. */
+ incrementCounter(name: string): void;
+}
+
+/**
+ * Build a {@link DuplexAggregateByteLedgerTelemetry} that maps each ledger event
+ * to the fixed metric name on the process metric sink. The gauge uses
+ * {@link DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE}. The rejection counter uses
+ * {@link DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL}. The defect
+ * counter uses {@link DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL}.
+ * Each call sits in an error swallow, so a sink failure never breaks the reserve
+ * or the release path.
+ */
+export function createDuplexAggregateByteLedgerTelemetry(
+ sink: DuplexAggregateByteLedgerMetricSink,
+): DuplexAggregateByteLedgerTelemetry {
+ return {
+ setBytesInUse(bytes: number): void {
+ try {
+ sink.setGauge(DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE, bytes);
+ } catch {
+ // A telemetry failure never breaks the reserve or the release path.
+ }
+ },
+ recordReservationRejection(): void {
+ try {
+ sink.incrementCounter(DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL);
+ } catch {
+ // A telemetry failure never breaks the reserve path.
+ }
+ },
+ recordAccountingUnderflow(): void {
+ try {
+ sink.incrementCounter(DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL);
+ } catch {
+ // A telemetry failure never breaks the release path.
+ }
+ },
+ };
+}
+
+/** The options for {@link DuplexAggregateByteLedger}. */
+export interface DuplexAggregateByteLedgerOptions {
+ /** The aggregate ceiling, in bytes. It must be a positive safe integer. */
+ ceilingBytes: number;
+ /** The fixed telemetry surface. The default is the no-op surface. */
+ telemetry?: DuplexAggregateByteLedgerTelemetry;
+}
+
+/**
+ * Assert an aggregate ceiling byte value. The value must be a positive safe
+ * integer that is no larger than the documented process allocation minus the
+ * documented reserve. The function throws on any other value, so an invalid
+ * ceiling fails startup instead of silently disabling the ledger.
+ *
+ * The reserve equals the documented process allocation minus the default
+ * ceiling ({@link DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES}), so the maximum
+ * accepted ceiling is exactly the documented process allocation minus that
+ * reserve.
+ */
+export function assertDuplexAggregateCeilingBytes(value: number): void {
+ const maxCeiling = maxDuplexAggregateCeilingBytes();
+ if (!Number.isSafeInteger(value) || value < 1) {
+ throw new Error(
+ `The duplex aggregate byte ceiling must be a positive safe integer; got ${String(value)}.`,
+ );
+ }
+ if (value > maxCeiling) {
+ throw new Error(
+ `The duplex aggregate byte ceiling ${value} must not pass the documented process allocation minus the reserve (${maxCeiling}).`,
+ );
+ }
+}
+
+/**
+ * The maximum accepted aggregate ceiling, in bytes: the documented process
+ * allocation minus the documented reserve. Both {@link assertDuplexAggregateCeilingBytes}
+ * and {@link isValidDuplexAggregateCeilingBytes} read this one bound.
+ */
+function maxDuplexAggregateCeilingBytes(): number {
+ return MIN_HOST_PROCESS_MEMORY_BYTES_FOR_DUPLEX - DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES;
+}
+
+/**
+ * Report whether `value` is a valid aggregate ceiling. A valid ceiling is a
+ * positive safe integer that is no larger than {@link maxDuplexAggregateCeilingBytes}.
+ * This is the non-throwing form of {@link assertDuplexAggregateCeilingBytes}. The
+ * override resolver uses it to reject an invalid override without a throw.
+ */
+export function isValidDuplexAggregateCeilingBytes(value: number): boolean {
+ return Number.isSafeInteger(value) && value >= 1 && value <= maxDuplexAggregateCeilingBytes();
+}
+
+/**
+ * A reporter the host binds to receive one invalid-override rejection signal.
+ * {@link resolveDuplexAggregateCeilingBytes} calls it one time when it rejects a
+ * present invalid override and falls back to the safe default. The signal carries
+ * only the rejected numeric value. It carries no route, company, run, or payload
+ * value.
+ */
+export type DuplexAggregateCeilingOverrideRejectionReporter = (rejectedValue: number) => void;
+
+/**
+ * Resolve an aggregate ceiling from an optional operator override. A missing or
+ * `null` override uses {@link DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES}. A present
+ * valid value passes through unchanged.
+ *
+ * A present invalid override does not fail host startup. The host process is
+ * multi-tenant, so one invalid environment value must not brick the whole host.
+ * The resolver rejects the invalid override, reports it through the optional
+ * `onRejectedOverride` reporter, and returns the conservative safe default. This
+ * fails loud without failing closed on availability.
+ */
+export function resolveDuplexAggregateCeilingBytes(
+ override?: number | null,
+ onRejectedOverride?: DuplexAggregateCeilingOverrideRejectionReporter,
+): number {
+ if (override === undefined || override === null) {
+ return DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES;
+ }
+ if (!isValidDuplexAggregateCeilingBytes(override)) {
+ onRejectedOverride?.(override);
+ return DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES;
+ }
+ return override;
+}
+
+/**
+ * The process-owned aggregate byte ledger. Create one per host server process.
+ * Inject the same object into every host-side retention site, so one shared gauge
+ * bounds the aggregate retained bytes across all live duplex routes.
+ */
+export class DuplexAggregateByteLedger {
+ private readonly ceiling: number;
+ private readonly telemetry: DuplexAggregateByteLedgerTelemetry;
+ private used = 0;
+ /** The owner registry. It holds every live token. */
+ private readonly liveTokens = new Set<ReservationTokenImpl>();
+
+ constructor(options: DuplexAggregateByteLedgerOptions) {
+ assertDuplexAggregateCeilingBytes(options.ceilingBytes);
+ this.ceiling = options.ceilingBytes;
+ this.telemetry = options.telemetry ?? NOOP_DUPLEX_AGGREGATE_BYTE_LEDGER_TELEMETRY;
+ }
+
+ /** The configured aggregate ceiling, in bytes. */
+ get ceilingBytes(): number {
+ return this.ceiling;
+ }
+
+ /** The current aggregate bytes in use across every live token. */
+ get bytesInUse(): number {
+ return this.used;
+ }
+
+ /** The current number of live tokens in the owner registry. */
+ get liveTokenCount(): number {
+ return this.liveTokens.size;
+ }
+
+ /**
+ * Reserve `bytes` for `owner`. Check and increment the gauge synchronously
+ * before the caller allocates. Return a held token when the reservation fits
+ * under the ceiling. Return `null` when `used + bytes` would pass the ceiling;
+ * the caller retains nothing and reports
+ * {@link DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED}.
+ *
+ * The byte amount must be a non-negative safe integer. A caller that passes any
+ * other value made a programming error, so the ledger throws and fails loud.
+ */
+ reserve(owner: DuplexAggregateTokenOwner, bytes: number): ReservationToken | null {
+ if (!Number.isSafeInteger(bytes) || bytes < 0) {
+ throw new Error(
+ `A duplex byte reservation must be a non-negative safe integer; got ${String(bytes)}.`,
+ );
+ }
+ if (this.used + bytes > this.ceiling) {
+ this.telemetry.recordReservationRejection();
+ return null;
+ }
+ this.used += bytes;
+ const token = new ReservationTokenImpl(bytes, owner);
+ this.liveTokens.add(token);
+ this.telemetry.setBytesInUse(this.used);
+ return token;
+ }
+
+ /**
+ * Transfer a held token to a new owner label. Change only the owner. Never
+ * decrement and never re-reserve. This is the atomic pre-bind handoff: it moves
+ * the same reserved bytes from one representation to the next with no admission
+ * gap.
+ *
+ * A transfer of a token the ledger does not hold, or of an already-released
+ * token, is an accounting defect. The ledger records the fixed underflow
+ * counter and leaves the gauge unchanged.
+ */
+ transfer(token: ReservationToken, owner: DuplexAggregateTokenOwner): void {
+ const impl = token as ReservationTokenImpl;
+ if (impl.state !== "held" || !this.liveTokens.has(impl)) {
+ this.telemetry.recordAccountingUnderflow();
+ return;
+ }
+ impl.owner = owner;
+ }
+
+ /**
+ * Release a held token. Remove it from the owner registry, mark it `released`,
+ * and decrement the gauge, all in one synchronous critical section before any
+ * `await` or external callback. A token releases one time.
+ *
+ * A second release of the same token, or a release of a token the ledger does
+ * not hold, is an accounting defect. The ledger records the fixed underflow
+ * counter and does not clamp the gauge a second time, so a real defect stays
+ * visible on the counter.
+ */
+ release(token: ReservationToken): void {
+ const impl = token as ReservationTokenImpl;
+ if (impl.state === "released" || !this.liveTokens.delete(impl)) {
+ this.telemetry.recordAccountingUnderflow();
+ return;
+ }
+ impl.state = "released";
+ this.used -= impl.bytes;
+ this.telemetry.setBytesInUse(this.used);
+ }
+}
diff --git a/packages/adapter-utils/src/duplex-bridge-broker-byte-ledger.test.ts b/packages/adapter-utils/src/duplex-bridge-broker-byte-ledger.test.ts
new file mode 100644
index 0000000000..78780c9ff5
--- /dev/null
+++ b/packages/adapter-utils/src/duplex-bridge-broker-byte-ledger.test.ts
@@ -0,0 +1,317 @@
+import { afterEach, describe, expect, it } from "vitest";
+
+import {
+ createDuplexBridgeBroker,
+ type DuplexBridgeBroker,
+ type DuplexBrokerForwardResult,
+} from "./duplex-bridge-broker.js";
+import {
+ DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED,
+ DuplexAggregateByteLedger,
+} from "./duplex-aggregate-byte-ledger.js";
+import {
+ DUPLEX_FRAME_VERSION,
+ DuplexFrameDecoder,
+ encodeDuplexFrame,
+ type DuplexRequestFrame,
+ type DuplexResponseFrame,
+} from "./duplex-frame-codec.js";
+import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js";
+
+/**
+ * Regression harness for the broker aggregate byte ledger charging.
+ *
+ * The harness feeds request frames straight into the broker through an in-memory
+ * channel, the same as a provider that controls the transport. A test controls
+ * when each forward settles, so it can assert the ledger charge while the forward
+ * is still in flight and after it settles.
+ */
+
+/** One pending forward a test settles by hand. */
+interface PendingForward {
+ id: string;
+ resolve: (result: DuplexBrokerForwardResult) => void;
+ reject: (error: Error) => void;
+ settled: boolean;
+}
+
+/** The in-memory channel plus the levers a test uses to drive the broker. */
+interface FakeChannelHarness {
+ channel: CommandManagedDuplexChannel;
+ feed: (frame: DuplexRequestFrame) => void;
+ exit: () => void;
+ responses: DuplexResponseFrame[];
+ forwards: PendingForward[];
+ resolveForward: (id: string, body?: string) => void;
+}
+
+/** Build one valid request frame. */
+function requestFrame(id: string, method = "POST"): DuplexRequestFrame {
+ return {
+ version: DUPLEX_FRAME_VERSION,
+ type: "request",
+ id,
+ method,
+ path: `/api/issues/${id}`,
+ query: "",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({ id }),
+ };
+}
+
+function createFakeChannelHarness(): FakeChannelHarness {
+ const responses: DuplexResponseFrame[] = [];
+ const forwards: PendingForward[] = [];
+ const writtenDecoder = new DuplexFrameDecoder();
+ let dataListener: ((chunk: string) => void) | null = null;
+ let exitListener: ((exit: { exitCode: number | null }) => void) | null = null;
+
+ const channel: CommandManagedDuplexChannel = {
+ write: (data) => {
+ for (const result of writtenDecoder.push(data)) {
+ if (result.ok && result.frame.type === "response") responses.push(result.frame);
+ }
+ },
+ onData: (listener) => {
+ dataListener = listener;
+ },
+ onExit: (listener) => {
+ exitListener = listener;
+ },
+ stop: () => undefined,
+ close: () => Promise.resolve(),
+ };
+
+ return {
+ channel,
+ feed: (frame) => {
+ if (!dataListener) throw new Error("The broker did not bind the data listener.");
+ dataListener(encodeDuplexFrame(frame));
+ },
+ exit: () => {
+ if (!exitListener) throw new Error("The broker did not bind the exit listener.");
+ exitListener({ exitCode: 0 });
+ },
+ responses,
+ forwards,
+ resolveForward: (id, body = JSON.stringify({ ok: true })) => {
+ const forward = [...forwards].reverse().find((entry) => entry.id === id && !entry.settled);
+ if (!forward) throw new Error(`No unsettled forward for id ${id}.`);
+ forward.settled = true;
+ forward.resolve({ status: 200, headers: { "content-type": "application/json" }, body });
+ },
+ };
+}
+
+/** A forward handler that hands each call to the harness and never auto-resolves. */
+function controllableForward(harness: FakeChannelHarness) {
+ return (request: DuplexRequestFrame): Promise<DuplexBrokerForwardResult> =>
+ new Promise<DuplexBrokerForwardResult>((resolve, reject) => {
+ harness.forwards.push({ id: request.id, resolve, reject, settled: false });
+ });
+}
+
+/** Wait for the microtask queue to drain, so a settled promise runs its handlers. */
+async function flush(): Promise<void> {
+ await Promise.resolve();
+ await Promise.resolve();
+}
+
+describe("duplex bridge broker aggregate byte ledger", () => {
+ const brokers: DuplexBridgeBroker[] = [];
+
+ afterEach(async () => {
+ while (brokers.length > 0) {
+ const broker = brokers.pop();
+ if (broker) await broker.close();
+ }
+ });
+
+ it("charges request-frame, request-payload, and seen-id tokens, then releases the request tokens on forward settlement", async () => {
+ const harness = createFakeChannelHarness();
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 });
+ const broker = createDuplexBridgeBroker({
+ channel: harness.channel,
+ forwardRequest: controllableForward(harness),
+ duplexAggregateByteLedger: ledger,
+ });
+ brokers.push(broker);
+ broker.start();
+
+ harness.feed(requestFrame("req-1"));
+ // The dispatch retains three tokens: the raw frame, the normalized payload,
+ // and the no-replay set entry.
+ expect(ledger.liveTokenCount).toBe(3);
+ expect(ledger.bytesInUse).toBeGreaterThan(0);
+ const chargedInFlight = ledger.bytesInUse;
+
+ harness.resolveForward("req-1");
+ await flush();
+
+ // The forward settled. The finally owner released the request-frame and the
+ // request-payload tokens. The seen-id token stays charged for the channel
+ // lifetime, so exactly one token remains.
+ expect(ledger.liveTokenCount).toBe(1);
+ expect(ledger.bytesInUse).toBeGreaterThan(0);
+ expect(ledger.bytesInUse).toBeLessThan(chargedInFlight);
+
+ // A close releases the seen-id token, so the ledger returns to zero.
+ await broker.close();
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ });
+
+ it("keeps request tokens charged when the response timer answers before the forward settles, and releases them on settlement", async () => {
+ const harness = createFakeChannelHarness();
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 });
+ const broker = createDuplexBridgeBroker({
+ channel: harness.channel,
+ forwardRequest: controllableForward(harness),
+ // Squeeze the nested budgets so the response timer fires quickly while the
+ // forward stays unsettled.
+ budgets: { forwardTimeoutMs: 5, responseBudgetMs: 10, gatewayWaitMs: 20 },
+ duplexAggregateByteLedger: ledger,
+ });
+ brokers.push(broker);
+ broker.start();
+
+ harness.feed(requestFrame("req-1", "GET"));
+ expect(ledger.liveTokenCount).toBe(3);
+
+ // Wait past the response budget so the backstop answers the gateway while the
+ // forward is still in flight.
+ await new Promise((resolve) => setTimeout(resolve, 40));
+ expect(harness.responses.some((frame) => frame.id === "req-1")).toBe(true);
+ // The gateway got its answer, but the forward has not settled. The request
+ // tokens and the seen-id token stay charged: nothing released yet.
+ expect(ledger.liveTokenCount).toBe(3);
+
+ // Settle the orphaned forward. Its finally owner now releases the two request
+ // tokens; the seen-id token still stays for the channel lifetime.
+ harness.resolveForward("req-1");
+ await flush();
+ expect(ledger.liveTokenCount).toBe(1);
+
+ await broker.close();
+ expect(ledger.bytesInUse).toBe(0);
+ });
+
+ it("refuses a one-byte-over dispatch with the fixed marker and retains nothing", async () => {
+ const harness = createFakeChannelHarness();
+ // Measure the exact per-request charge with a throwaway broker, so the test
+ // can size a ceiling one byte below it and force the reservation to fail.
+ const probeHarness = createFakeChannelHarness();
+ const measured = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 });
+ const measuringBroker = createDuplexBridgeBroker({
+ channel: probeHarness.channel,
+ forwardRequest: controllableForward(probeHarness),
+ duplexAggregateByteLedger: measured,
+ });
+ measuringBroker.start();
+ probeHarness.feed(requestFrame("req-1"));
+ const perRequestBytes = measured.bytesInUse;
+ await measuringBroker.close();
+
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: perRequestBytes - 1 });
+ const broker = createDuplexBridgeBroker({
+ channel: harness.channel,
+ forwardRequest: controllableForward(harness),
+ duplexAggregateByteLedger: ledger,
+ });
+ brokers.push(broker);
+ broker.start();
+
+ harness.feed(requestFrame("req-1"));
+ await flush();
+
+ // The broker retained nothing: no token, no forward, no seen id.
+ expect(ledger.liveTokenCount).toBe(0);
+ expect(ledger.bytesInUse).toBe(0);
+ expect(harness.forwards.length).toBe(0);
+ // The refusal carries the fixed marker and is a bounded terminal response.
+ const refusal = harness.responses.find((frame) => frame.id === "req-1");
+ expect(refusal).toBeTruthy();
+ expect(refusal?.status).toBe(503);
+ expect(refusal?.body).toContain(DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED);
+
+ // The refusal did not retain the id, so a resend after pressure eases is
+ // admitted: raise the ceiling and re-feed.
+ const roomyLedger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 });
+ const roomyHarness = createFakeChannelHarness();
+ const roomyBroker = createDuplexBridgeBroker({
+ channel: roomyHarness.channel,
+ forwardRequest: controllableForward(roomyHarness),
+ duplexAggregateByteLedger: roomyLedger,
+ });
+ brokers.push(roomyBroker);
+ roomyBroker.start();
+ roomyHarness.feed(requestFrame("req-1"));
+ expect(roomyHarness.forwards.length).toBe(1);
+ });
+
+ it("releases every retained token on a terminal channel loss", async () => {
+ const harness = createFakeChannelHarness();
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 });
+ const broker = createDuplexBridgeBroker({
+ channel: harness.channel,
+ forwardRequest: controllableForward(harness),
+ duplexAggregateByteLedger: ledger,
+ });
+ brokers.push(broker);
+ broker.start();
+
+ harness.feed(requestFrame("req-1"));
+ harness.feed(requestFrame("req-2"));
+ expect(ledger.liveTokenCount).toBe(6);
+
+ // The channel exits mid-flight. The broker latches the loss, transfers the
+ // in-flight forwards to the orphan registry, and releases the seen-id tokens.
+ harness.exit();
+ expect(broker.runDisposition.failed).toBe(true);
+ // The seen-id tokens released at loss; the two request tokens per forward stay
+ // charged until each aborted forward settles.
+ expect(ledger.liveTokenCount).toBe(4);
+
+ // The aborted forwards settle. Each finally owner releases its two request
+ // tokens, so the ledger returns to zero.
+ harness.forwards[0]?.reject(new Error("aborted"));
+ harness.forwards[1]?.reject(new Error("aborted"));
+ await flush();
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ });
+
+ it("does not double-release a token when a forward settles after the channel closed", async () => {
+ const harness = createFakeChannelHarness();
+ let underflow = 0;
+ const ledger = new DuplexAggregateByteLedger({
+ ceilingBytes: 1_000_000,
+ telemetry: {
+ setBytesInUse() {},
+ recordReservationRejection() {},
+ recordAccountingUnderflow() {
+ underflow += 1;
+ },
+ },
+ });
+ const broker = createDuplexBridgeBroker({
+ channel: harness.channel,
+ forwardRequest: controllableForward(harness),
+ duplexAggregateByteLedger: ledger,
+ });
+ brokers.push(broker);
+ broker.start();
+
+ harness.feed(requestFrame("req-1"));
+ await broker.close();
+ // The close aborted the in-flight forward and orphaned it. The forward now
+ // settles after the close: its finally owner releases the request tokens once.
+ harness.forwards[0]?.reject(new Error("aborted"));
+ await flush();
+
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ // No accounting defect: every token released exactly one time.
+ expect(underflow).toBe(0);
+ });
+});
diff --git a/packages/adapter-utils/src/duplex-bridge-broker.ts b/packages/adapter-utils/src/duplex-bridge-broker.ts
index 1a902360a9..b7a04021a1 100644
--- a/packages/adapter-utils/src/duplex-bridge-broker.ts
+++ b/packages/adapter-utils/src/duplex-bridge-broker.ts
@@ -42,6 +42,11 @@
*/
import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js";
+import {
+ DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED,
+ type DuplexAggregateByteLedger,
+ type ReservationToken,
+} from "./duplex-aggregate-byte-ledger.js";
import {
DEFAULT_MAX_DUPLEX_FRAME_BYTES,
DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES,
@@ -169,6 +174,17 @@ export const DEFAULT_DUPLEX_BROKER_MAX_IN_FLIGHT_REQUESTS = 64;
*/
export const DEFAULT_DUPLEX_BROKER_MAX_LIFETIME_REQUESTS = 50_000;
+/**
+ * The fixed, documented per-entry allocation the broker charges the aggregate byte
+ * ledger for one no-replay request-id set entry. The type and the cardinality of
+ * `seenRequestIds` are bounded: the codec caps each id at
+ * {@link DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES}, and the lifetime limit caps the
+ * entry count. The broker charges the exact raw id bytes plus this fixed entry
+ * overhead, so the retained set never grows uncharged. This constant models the
+ * fixed per-entry cost of the string key and the Set slot, not the id bytes.
+ */
+export const DUPLEX_SEEN_REQUEST_ID_SET_ENTRY_BYTES = 64;
+
/** The result of one forward call. The broker turns it into one response frame. */
export interface DuplexBrokerForwardResult {
status: number;
@@ -250,6 +266,17 @@ export interface DuplexBrokerOptions {
* raw error rides a span or a counter. The default records nothing.
*/
telemetry?: DuplexTelemetry;
+ /**
+ * The process-owned aggregate byte ledger. The broker reserves the exact retained
+ * bytes of each dispatched request against it before it retains the frame: the
+ * raw request frame, the normalized request payload, and the no-replay set entry.
+ * A reservation that would pass the ceiling makes the broker retain nothing and
+ * refuse the request with the fixed marker
+ * {@link DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED}. When the ledger is absent the
+ * broker charges nothing and behaves as before, so a non-duplex or a legacy path
+ * stays unchanged.
+ */
+ duplexAggregateByteLedger?: DuplexAggregateByteLedger | null;
}
/** The broker handle the factory returns. */
@@ -358,6 +385,27 @@ interface PendingRequest {
responseTimer: ReturnType<typeof setTimeout>;
/** The point the broker started to dispatch the request. It sets the span latency. */
dispatchStartMs: number;
+ /** True once the forward promise settled. The finally owner sets it one time. */
+ forwardSettled: boolean;
+ /**
+ * Release the request-frame token and the request-payload token exactly one time.
+ * The single forward-promise finally owner calls it after the forward settles.
+ * A second call is a no-op, so no token releases twice.
+ */
+ releaseForwardTokens: () => void;
+}
+
+/**
+ * One orphaned forward. The broker answered the gateway and removed the request
+ * from `pending`, but the forward promise or its response-body reader had not
+ * settled. The orphan keeps the request tokens charged and the controller live
+ * until the forward finally releases them, so the ledger reports nonzero ownership
+ * until the async work settles.
+ */
+interface OrphanedForward {
+ controller: AbortController;
+ /** Release the request-frame and request-payload tokens exactly one time. */
+ releaseForwardTokens: () => void;
}
/**
@@ -382,12 +430,42 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr
const heartbeatIntervalMs = options.heartbeatIntervalMs ?? DEFAULT_DUPLEX_BROKER_HEARTBEAT_INTERVAL_MS;
const closeTimeoutMs = options.closeTimeoutMs ?? DEFAULT_DUPLEX_BROKER_CLOSE_TIMEOUT_MS;
const now = options.now ?? (() => Date.now());
+
+ // The process-owned aggregate byte ledger, or `null` when the caller injected
+ // none. When `null` the broker charges nothing and behaves as before.
+ const ledger = options.duplexAggregateByteLedger ?? null;
// The one frame size bound the broker enforces on both sides. The decoder
// rejects an inbound frame over this bound, and the encode guard refuses to
// write an outbound frame over it. Encode and decode share one value, so a
// frame the broker writes always decodes on the peer.
const maxFrameBytes = options.maxFrameBytes ?? DEFAULT_MAX_DUPLEX_FRAME_BYTES;
- const decoder = new DuplexFrameDecoder({ maxFrameBytes });
+ // The decoder charges its retained partial-frame bytes against the same host
+ // ledger. It receives the ledger object directly, so one gauge bounds the
+ // decoder buffer with every other host retention site.
+ const decoder = new DuplexFrameDecoder({
+ maxFrameBytes,
+ ...(ledger ? { aggregateByteLedger: ledger } : {}),
+ });
+ // Release one token exactly one time. A `null` token or an absent ledger is a
+ // no-op, so the broker never records a false accounting defect.
+ const releaseToken = (token: ReservationToken | null): void => {
+ if (ledger && token) ledger.release(token);
+ };
+ // The byte size of the normalized request payload the broker retains across the
+ // forward. It counts the exact retained scalar and header bytes, so the charge
+ // matches the retained bytes and never a parsed object graph.
+ const requestPayloadBytes = (frame: DuplexRequestFrame): number => {
+ let bytes =
+ Buffer.byteLength(frame.id, "utf8") +
+ Buffer.byteLength(frame.method, "utf8") +
+ Buffer.byteLength(frame.path, "utf8") +
+ Buffer.byteLength(frame.query, "utf8") +
+ Buffer.byteLength(frame.body, "utf8");
+ for (const [key, value] of Object.entries(frame.headers)) {
+ bytes += Buffer.byteLength(key, "utf8") + Buffer.byteLength(value, "utf8");
+ }
+ return bytes;
+ };
let state: DuplexBrokerState = "opening";
let stopped = false;
@@ -432,7 +510,24 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr
// The ids the broker already dispatched. The broker forwards one id one time,
// so a repeated frame never reaches the API twice.
const seenRequestIds = new Set<string>();
+ // The aggregate-ledger tokens for the no-replay set entries. The broker holds
+ // one token per live set entry and releases every token one time at terminal
+ // teardown, so the retained set never leaves bytes charged after the channel
+ // ends.
+ const seenRequestIdTokens = new Set<ReservationToken>();
const pending = new Map<string, PendingRequest>();
+ // The forwards that answered the gateway but whose async work has not settled.
+ // The broker keeps their tokens charged and their controller live until each
+ // forward finally releases its tokens, so the ledger reports the real ownership.
+ const orphanedForwards = new Map<string, OrphanedForward>();
+
+ // Release every no-replay set-entry token one time and clear the set. The
+ // ledger release is one way, and the cleared set stops any second release, so
+ // this helper is safe to call more than one time at terminal teardown.
+ const releaseSeenRequestIdTokens = (): void => {
+ for (const token of seenRequestIdTokens) releaseToken(token);
+ seenRequestIdTokens.clear();
+ };
const setState = (next: DuplexBrokerState): void => {
if (state === next) return;
@@ -448,12 +543,27 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr
};
const clearPending = (): void => {
- for (const entry of pending.values()) {
+ for (const [id, entry] of pending) {
clearTimeout(entry.forwardTimer);
clearTimeout(entry.responseTimer);
entry.controller.abort(new Error("Duplex broker stopped."));
+ // A forward that has not settled still owns its request tokens. Transfer it
+ // to the orphan registry, so its tokens stay charged until the forward
+ // finally releases them. The abort only asks the forward to stop; it never
+ // releases a token by itself.
+ if (!entry.forwardSettled) {
+ orphanedForwards.set(id, {
+ controller: entry.controller,
+ releaseForwardTokens: entry.releaseForwardTokens,
+ });
+ }
}
pending.clear();
+ // Ask every already-orphaned forward to stop as well. The forward-promise
+ // finally owner releases each orphan token when the forward settles.
+ for (const orphan of orphanedForwards.values()) {
+ orphan.controller.abort(new Error("Duplex broker stopped."));
+ }
};
const recordLoss = (reason: DuplexBrokerLossReason, message: string): void => {
@@ -470,6 +580,8 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr
stopped = true;
clearHeartbeat();
clearPending();
+ releaseSeenRequestIdTokens();
+ decoder.dispose();
if (state !== "closing") setState("closed");
return;
}
@@ -493,6 +605,8 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr
}
clearHeartbeat();
clearPending();
+ releaseSeenRequestIdTokens();
+ decoder.dispose();
lossRecord = { reason, message, atMs: now() };
setState("lost");
// Log the internal reason only. The broker never writes the raw provider
@@ -600,6 +714,17 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr
clearTimeout(entry.forwardTimer);
clearTimeout(entry.responseTimer);
pending.delete(id);
+ // The response timer, an abort, or a broker loss can answer the gateway before
+ // the forward promise settles. When the forward has not settled, transfer it to
+ // the orphan registry so its request tokens stay charged until the forward
+ // finally releases them. A settled forward already released or will release
+ // through its finally owner, so it needs no orphan record.
+ if (!entry.forwardSettled) {
+ orphanedForwards.set(id, {
+ controller: entry.controller,
+ releaseForwardTokens: entry.releaseForwardTokens,
+ });
+ }
// Do not write on a lost or closed channel. The gateway answers its own
// outstanding request on loss, so a late write would go to a dead channel.
if (state !== "open") return;
@@ -662,6 +787,34 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr
writeFrame(frame);
};
+ const respondAggregateExceeded = (id: string): void => {
+ // Answer a request the aggregate byte ledger refused with a bounded terminal
+ // response. The broker reserved nothing, added no id to the seen set, and made
+ // no controller, timer, or forward, so the host API stays untouched. The
+ // response carries the fixed marker only; it holds no route, query, body, or
+ // token. The `unavailable` outcome tells the gateway this is not a delivered
+ // host response. The refusal is retryable, because the broker did not retain
+ // the id: the aggregate pressure can ease, and a resend can then get through.
+ if (state !== "open") return;
+ const frame: DuplexResponseFrame = {
+ version: DUPLEX_FRAME_VERSION,
+ type: "response",
+ id,
+ status: 503,
+ headers: {
+ "content-type": "application/json",
+ "x-paperclip-bridge-outcome": "unavailable",
+ },
+ body: JSON.stringify({
+ error: DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED,
+ outcome: "unavailable",
+ retryable: true,
+ }),
+ outcome: "unavailable",
+ };
+ writeFrame(frame);
+ };
+
const dispatch = (frame: DuplexRequestFrame): void => {
// Dispatch only while open. After loss or close the broker forwards nothing.
if (state !== "open") return;
@@ -699,7 +852,54 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr
respondSaturated(frame.id, true);
return;
}
+ // Reserve the exact retained bytes against the aggregate ledger before the
+ // broker retains anything. It reserves three tokens in order: the raw request
+ // frame, the normalized request payload, and the no-replay set entry. A
+ // reservation that would pass the ceiling makes the broker retain nothing,
+ // release the tokens it already took, and refuse the request with the fixed
+ // marker. The broker adds no id to the seen set and makes no controller, timer,
+ // or forward, so the host API stays untouched. When the ledger is absent all
+ // tokens stay `null` and the broker behaves as before.
+ let requestFrameToken: ReservationToken | null = null;
+ let requestPayloadToken: ReservationToken | null = null;
+ let seenRequestIdToken: ReservationToken | null = null;
+ if (ledger) {
+ const rawFrameBytes = Buffer.byteLength(encodeDuplexFrame(frame), "utf8");
+ requestFrameToken = ledger.reserve("request_frame", rawFrameBytes);
+ if (!requestFrameToken) {
+ respondAggregateExceeded(frame.id);
+ return;
+ }
+ requestPayloadToken = ledger.reserve("request_payload", requestPayloadBytes(frame));
+ if (!requestPayloadToken) {
+ ledger.release(requestFrameToken);
+ respondAggregateExceeded(frame.id);
+ return;
+ }
+ const seenEntryBytes =
+ Buffer.byteLength(frame.id, "utf8") + DUPLEX_SEEN_REQUEST_ID_SET_ENTRY_BYTES;
+ seenRequestIdToken = ledger.reserve("seen_request_id", seenEntryBytes);
+ if (!seenRequestIdToken) {
+ ledger.release(requestFrameToken);
+ ledger.release(requestPayloadToken);
+ respondAggregateExceeded(frame.id);
+ return;
+ }
+ }
seenRequestIds.add(frame.id);
+ if (seenRequestIdToken) seenRequestIdTokens.add(seenRequestIdToken);
+
+ // Release the request-frame and the request-payload tokens exactly one time.
+ // The single forward-promise finally owner calls it after the forward settles.
+ // The seen-id token stays charged for the channel lifetime, so it is not part
+ // of this release; the terminal teardown releases it.
+ let forwardTokensReleased = false;
+ const releaseForwardTokens = (): void => {
+ if (forwardTokensReleased) return;
+ forwardTokensReleased = true;
+ releaseToken(requestFrameToken);
+ releaseToken(requestPayloadToken);
+ };
const record: DuplexBrokerRequestRecord = {
id: frame.id,
@@ -770,23 +970,53 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr
forwardTimer,
responseTimer,
dispatchStartMs: record.dispatchStartMs,
+ forwardSettled: false,
+ releaseForwardTokens,
});
- forwardRequest(frame, { signal: controller.signal }).then(
- (result) => {
- // Keep the outcome classification consistent with the file path. A
- // possibly-committed mutation carries the indeterminate marker header, so
- // map it to the indeterminate outcome. Any other result is completed.
- const outcome: DuplexResponseOutcome =
- result.headers?.["x-paperclip-bridge-outcome"] === "indeterminate"
- ? "indeterminate"
- : "completed";
- // The host delivered a real response, so the request span outcome is `ok`.
- // A host application status (200, a 4xx, a 5xx) is still a delivered
- // response; only a broker-synthesized failure below is `error`.
- respond(frame.id, result, outcome, "ok");
- },
- (error) => {
+ // The forward promise has one cleanup owner. The two settle handlers mark the
+ // forward settled and answer the gateway. The `finally` then releases the
+ // request tokens exactly one time and removes the request from both the pending
+ // map and the orphan map. The forward resolves only after the forward and its
+ // response-body reader both settle, so this owner waits for both.
+ const markForwardSettled = (): void => {
+ // Mark the pending entry settled before `respond` runs, so `respond` never
+ // moves a just-settled forward to the orphan registry. An already-orphaned
+ // forward needs no flag; its finally owner releases and drains it.
+ const entry = pending.get(frame.id);
+ if (entry) entry.forwardSettled = true;
+ };
+ forwardRequest(frame, { signal: controller.signal })
+ .then(
+ (result) => {
+ markForwardSettled();
+ // Keep the outcome classification consistent with the file path. A
+ // possibly-committed mutation carries the indeterminate marker header, so
+ // map it to the indeterminate outcome. Any other result is completed.
+ const outcome: DuplexResponseOutcome =
+ result.headers?.["x-paperclip-bridge-outcome"] === "indeterminate"
+ ? "indeterminate"
+ : "completed";
+ // The host delivered a real response, so the request span outcome is `ok`.
+ // A host application status (200, a 4xx, a 5xx) is still a delivered
+ // response; only a broker-synthesized failure below is `error`.
+ respond(frame.id, result, outcome, "ok");
+ },
+ (error) => {
+ markForwardSettled();
+ return handleForwardRejection(error);
+ },
+ )
+ .finally(() => {
+ pending.delete(frame.id);
+ orphanedForwards.delete(frame.id);
+ releaseForwardTokens();
+ });
+
+ // The forward rejection handler. It classifies the rejection by method safety
+ // and answers the gateway. The `markForwardSettled` call above runs first, so
+ // the response never re-orphans a settled forward.
+ function handleForwardRejection(error: unknown): void {
if (controller.signal.aborted) {
// The forward budget aborted the call. A safe method never changes
// host state, so a forward timeout stays retryable for it. Return a
@@ -873,8 +1103,7 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr
"indeterminate",
"error",
);
- },
- );
+ }
};
const handleFrame = (frame: DuplexFrame): void => {
@@ -947,6 +1176,8 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr
setState("closing");
clearHeartbeat();
clearPending();
+ releaseSeenRequestIdTokens();
+ decoder.dispose();
// Send an orderly close frame. Ignore a write failure here; the broker is
// already closing, so a dead channel needs no loss record.
try {
diff --git a/packages/adapter-utils/src/duplex-frame-codec.test.ts b/packages/adapter-utils/src/duplex-frame-codec.test.ts
index 42ec35a4db..c19008fb55 100644
--- a/packages/adapter-utils/src/duplex-frame-codec.test.ts
+++ b/packages/adapter-utils/src/duplex-frame-codec.test.ts
@@ -12,6 +12,7 @@ import {
type DuplexDecodeResult,
type DuplexFrame,
} from "./duplex-frame-codec.js";
+import { DuplexAggregateByteLedger } from "./duplex-aggregate-byte-ledger.js";
// One expected decode result in the fixture: either a decoded frame or the code
// of a protocol error. The codec must never throw on the read path.
@@ -322,6 +323,117 @@ describe("request id byte bound", () => {
});
});
+describe("decoder aggregate byte ledger charging", () => {
+ // The host injects the process-owned ledger into the decoder. The decoder
+ // charges the raw partial-frame bytes it retains between chunks, plus the peak
+ // replacement buffer it allocates on concat. It never retains an uncharged
+ // buffer, and it fails closed when a reservation would pass the ceiling.
+ function makeLedger(ceilingBytes = 1024): DuplexAggregateByteLedger {
+ return new DuplexAggregateByteLedger({ ceilingBytes });
+ }
+
+ it("charges the retained partial frame, then releases it when the frame completes", () => {
+ const ledger = makeLedger();
+ const decoder = new DuplexFrameDecoder({ aggregateByteLedger: ledger });
+ const line = encodeDuplexFrame({ version: DUPLEX_FRAME_VERSION, type: "heartbeat" });
+ const bytes = Buffer.from(line, "utf8");
+
+ // A partial frame with no newline stays retained and stays charged.
+ const first = decoder.push(bytes.subarray(0, 3));
+ expect(first).toHaveLength(0);
+ expect(ledger.bytesInUse).toBe(3);
+ expect(ledger.liveTokenCount).toBe(1);
+
+ // The completing chunk delivers the frame and releases the retained bytes.
+ const second = decoder.push(bytes.subarray(3));
+ expect(second).toHaveLength(1);
+ expect(second[0].ok).toBe(true);
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ });
+
+ it("covers the peak concat buffer and settles on the remaining bytes", () => {
+ const ledger = makeLedger();
+ const decoder = new DuplexFrameDecoder({ aggregateByteLedger: ledger });
+ const first = encodeDuplexFrame({ version: DUPLEX_FRAME_VERSION, type: "heartbeat" });
+ const partial = '{"version":1,"type":"hea';
+ // One full frame plus a partial second frame arrive in one chunk. The full
+ // frame decodes and releases; the partial tail stays charged at its exact size.
+ const results = decoder.push(Buffer.from(first + partial, "utf8"));
+ expect(results).toHaveLength(1);
+ expect(results[0].ok).toBe(true);
+ expect(ledger.bytesInUse).toBe(Buffer.byteLength(partial, "utf8"));
+ expect(ledger.liveTokenCount).toBe(1);
+ });
+
+ it("fails closed when a chunk would pass the ceiling and retains nothing", () => {
+ const ledger = makeLedger(16);
+ const decoder = new DuplexFrameDecoder({ aggregateByteLedger: ledger });
+ const results = decoder.push(Buffer.from("z".repeat(64), "utf8"));
+ expect(results).toHaveLength(1);
+ expect(results[0].ok).toBe(false);
+ if (!results[0].ok) {
+ expect(results[0].error.code).toBe("aggregate_bytes_exceeded");
+ }
+ // The decoder retained nothing, so the ledger stays at zero.
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ });
+
+ it("resynchronizes on the next newline after an aggregate rejection", () => {
+ const ledger = makeLedger(64);
+ const decoder = new DuplexFrameDecoder({ aggregateByteLedger: ledger });
+ // A first chunk over the ceiling fails closed.
+ const rejected = decoder.push(Buffer.from("z".repeat(128), "utf8"));
+ expect(rejected[0].ok).toBe(false);
+
+ // The next chunk starts with a newline, so the decoder drops the stale tail
+ // and decodes the good frame that follows.
+ const good = encodeDuplexFrame({ version: DUPLEX_FRAME_VERSION, type: "heartbeat" });
+ const results = decoder.push(Buffer.from(`\n${good}`, "utf8"));
+ expect(results).toHaveLength(1);
+ expect(results[0].ok).toBe(true);
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ });
+
+ it("releases the retained token when an oversized frame is discarded", () => {
+ const ledger = makeLedger();
+ const decoder = new DuplexFrameDecoder({ maxFrameBytes: 16, aggregateByteLedger: ledger });
+ // An oversized frame with no newline is discarded and its bytes are released.
+ const results = decoder.push(Buffer.from("z".repeat(64), "utf8"));
+ expect(results).toHaveLength(1);
+ expect(results[0].ok).toBe(false);
+ if (!results[0].ok) expect(results[0].error.code).toBe("frame_too_large");
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ });
+
+ it("releases the retained partial frame on dispose", () => {
+ const ledger = makeLedger();
+ const decoder = new DuplexFrameDecoder({ aggregateByteLedger: ledger });
+ decoder.push(Buffer.from('{"version":1', "utf8"));
+ expect(ledger.bytesInUse).toBeGreaterThan(0);
+ decoder.dispose();
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ // A second dispose is a no-op and records no accounting defect.
+ decoder.dispose();
+ expect(ledger.bytesInUse).toBe(0);
+ });
+
+ it("charges nothing when the caller injects no ledger", () => {
+ const decoder = new DuplexFrameDecoder();
+ const results = decoder.push(Buffer.from('{"version":1', "utf8"));
+ expect(results).toHaveLength(0);
+ // No ledger means no charge; the decoder still buffers and decodes as before.
+ const line = encodeDuplexFrame({ version: DUPLEX_FRAME_VERSION, type: "heartbeat" });
+ const done = decoder.push(Buffer.from(`}\n${line}`, "utf8"));
+ // The stitched first frame is malformed JSON; the second is a valid heartbeat.
+ expect(done.some((result) => result.ok)).toBe(true);
+ });
+});
+
describe("size-checked encode", () => {
it("runs every shared encode vector and matches the expected result", () => {
// Every codec copy runs the same encode vectors. This copy proves it enforces
diff --git a/packages/adapter-utils/src/duplex-frame-codec.ts b/packages/adapter-utils/src/duplex-frame-codec.ts
index 20cd814506..c1723ad763 100644
--- a/packages/adapter-utils/src/duplex-frame-codec.ts
+++ b/packages/adapter-utils/src/duplex-frame-codec.ts
@@ -17,6 +17,12 @@
* keeps one bad frame from crashing the read loop.
*/
+import {
+ DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED,
+ type DuplexAggregateByteLedger,
+ type ReservationToken,
+} from "./duplex-aggregate-byte-ledger.js";
+
/** The wire version this codec reads and writes. */
export const DUPLEX_FRAME_VERSION = 1;
@@ -134,7 +140,8 @@ export type DuplexProtocolErrorCode =
| "unknown_type"
| "version_mismatch"
| "frame_too_large"
- | "id_too_large";
+ | "id_too_large"
+ | "aggregate_bytes_exceeded";
/** A decode-time protocol error. The read path returns it; it never throws. */
export interface DuplexProtocolError {
@@ -344,6 +351,18 @@ function validateError(frame: Record<string, unknown>): DuplexDecodeResult {
export interface DuplexFrameDecoderOptions {
/** The maximum size of one frame, in bytes. Defaults to {@link DEFAULT_MAX_DUPLEX_FRAME_BYTES}. */
maxFrameBytes?: number;
+ /**
+ * The process-owned aggregate byte ledger. When present, the decoder reserves
+ * the exact bytes of the raw partial frame it retains between chunks, and the
+ * peak replacement buffer it allocates on concat, against this ledger under the
+ * `decoder_buffer` owner. The host injects the same ledger object it injects at
+ * every other retention site, so one gauge bounds the aggregate retained bytes.
+ * When absent the decoder charges nothing and behaves as before.
+ *
+ * The generated sandbox decoder runs in a separate operating-system process. It
+ * cannot share this host ledger. It receives its own separate cap instead.
+ */
+ aggregateByteLedger?: DuplexAggregateByteLedger;
}
/**
@@ -364,15 +383,41 @@ export class DuplexFrameDecoder {
private buffer: Buffer = EMPTY;
private discarding = false;
private readonly maxFrameBytes: number;
+ private readonly ledger: DuplexAggregateByteLedger | null;
+ /**
+ * The token for the bytes currently retained in `this.buffer`. Its byte amount
+ * equals `this.buffer.length` at the end of every `push`. It is `null` when the
+ * buffer is empty or the decoder has no ledger.
+ */
+ private retainedToken: ReservationToken | null = null;
constructor(options: DuplexFrameDecoderOptions = {}) {
this.maxFrameBytes = options.maxFrameBytes ?? DEFAULT_MAX_DUPLEX_FRAME_BYTES;
+ this.ledger = options.aggregateByteLedger ?? null;
}
/** Feed one chunk. Return the frames and protocol errors that complete on it. */
push(chunk: Buffer | Uint8Array | string): DuplexDecodeResult[] {
const incoming =
typeof chunk === "string" ? Buffer.from(chunk, "utf8") : Buffer.from(chunk);
+
+ // Reserve the incoming bytes before the concat allocates the replacement
+ // buffer. The retained token already covers the current buffer, so the two
+ // tokens together cover the peak `old + incoming` allocation. A rejected
+ // reservation fails closed: the decoder drops the incoming chunk, releases
+ // the retained buffer, resynchronizes at the next newline, and reports the
+ // fixed aggregate rejection marker. It retains nothing uncharged.
+ let incomingToken: ReservationToken | null = null;
+ if (this.ledger && incoming.length > 0) {
+ incomingToken = this.ledger.reserve("decoder_buffer", incoming.length);
+ if (incomingToken === null) {
+ this.releaseRetained();
+ this.buffer = EMPTY;
+ this.discarding = true;
+ return [fail("aggregate_bytes_exceeded", DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED)];
+ }
+ }
+
this.buffer =
this.buffer.length === 0 ? incoming : Buffer.concat([this.buffer, incoming]);
@@ -411,6 +456,54 @@ export class DuplexFrameDecoder {
}
results.push(decodeDuplexLine(line));
}
+
+ // Reconcile the ledger to the bytes still retained in `this.buffer`. Release
+ // the old retained token and the incoming token, then reserve one token for
+ // the remainder. The remainder is a subset of the just-released
+ // `old + incoming` bytes, so this reserve always fits.
+ this.reconcileRetained(incomingToken);
return results;
}
+
+ /**
+ * Release the retained-buffer token and drop the buffer. The host calls this at
+ * channel teardown, so the decoder never leaks a `decoder_buffer` token after
+ * the channel ends. A second call is a no-op, because the field is already
+ * `null`.
+ */
+ dispose(): void {
+ this.releaseRetained();
+ this.buffer = EMPTY;
+ this.discarding = false;
+ }
+
+ /** Release the retained-buffer token one time and clear the field. */
+ private releaseRetained(): void {
+ if (this.ledger && this.retainedToken) {
+ this.ledger.release(this.retainedToken);
+ }
+ this.retainedToken = null;
+ }
+
+ /**
+ * Move the ledger charge to the bytes still in `this.buffer`. Release the old
+ * retained token and the `incoming` token, then reserve one token for the
+ * remaining bytes. A `null` reserve here is an accounting defect, because the
+ * remainder never passes the just-released bytes; the decoder fails closed and
+ * drops the buffer so it retains nothing uncharged.
+ */
+ private reconcileRetained(incomingToken: ReservationToken | null): void {
+ if (!this.ledger) return;
+ this.releaseRetained();
+ if (incomingToken) {
+ this.ledger.release(incomingToken);
+ }
+ if (this.buffer.length > 0) {
+ this.retainedToken = this.ledger.reserve("decoder_buffer", this.buffer.length);
+ if (this.retainedToken === null) {
+ this.buffer = EMPTY;
+ this.discarding = true;
+ }
+ }
+ }
}
diff --git a/packages/adapter-utils/src/duplex-telemetry.ts b/packages/adapter-utils/src/duplex-telemetry.ts
index 2bcf8073cb..b31c655918 100644
--- a/packages/adapter-utils/src/duplex-telemetry.ts
+++ b/packages/adapter-utils/src/duplex-telemetry.ts
@@ -35,6 +35,40 @@ export const DUPLEX_COUNTER_LOSS_TOTAL = "sandbox_duplex_loss_total";
/** The guarded counter for one leaked provider session on teardown. */
export const DUPLEX_COUNTER_SESSION_LEAK_TOTAL = "sandbox_duplex_session_leak_total";
+/**
+ * The process-scoped gauge for the aggregate retained bytes across every live
+ * duplex route. The host aggregate byte ledger sets it on each reserve and each
+ * release. The record carries no dynamic dimension.
+ */
+export const DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE = "sandbox_duplex_aggregate_bytes_in_use";
+/**
+ * The counter for one rejected aggregate byte reservation. The host ledger
+ * increments it when a reservation would pass the aggregate ceiling. The record
+ * carries no dynamic dimension.
+ */
+export const DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL =
+ "sandbox_duplex_aggregate_byte_reservation_rejections_total";
+/**
+ * The counter for one aggregate byte accounting defect. The host ledger
+ * increments it on a double release or a transfer of a token it does not hold.
+ * The record carries no dynamic dimension.
+ */
+export const DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL =
+ "sandbox_duplex_aggregate_byte_accounting_underflow_total";
+
+/**
+ * The closed set of aggregate byte ledger metric names. A test pins this exact
+ * set, so a new ledger metric name needs an explicit review. Each record uses
+ * only closed constant dimensions and no dynamic label. The telemetry contract
+ * documents these metrics under "Aggregate byte ledger metrics" in
+ * `packages/shared/src/telemetry/README.md`.
+ */
+export const DUPLEX_AGGREGATE_BYTE_LEDGER_METRIC_NAMES = [
+ DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE,
+ DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL,
+ DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL,
+] as const;
+
/**
* The closed dimension-key set. Every span attribute, counter label, and event
* field uses only these keys. A test asserts the exact set, so a new key never
@@ -64,6 +98,9 @@ export type DuplexOutcomeValue = "ok" | "error";
* stage: the process-scoped route ceiling was full (`route_busy`), the entrypoint
* sync failed (`entrypoint_sync_failed`), the broker construction failed
* (`broker_construction_failed`), or the channel open failed (`channel_open_failed`).
+ * The `aggregate_bytes_exceeded` reason names a readiness handshake the host fell
+ * back because the process aggregate byte ceiling had no room for the readiness
+ * buffer.
*/
export type DuplexFallbackReason =
| "gate_off"
@@ -75,7 +112,8 @@ export type DuplexFallbackReason =
| "ready_invalid"
| "ready_nonce_mismatch"
| "ready_timeout"
- | "contaminated";
+ | "contaminated"
+ | "aggregate_bytes_exceeded";
/** The class of a terminal loss, relative to the first request dispatch. */
export type DuplexLossClass = "pre_dispatch" | "post_dispatch";
diff --git a/packages/adapter-utils/src/execution-target-sandbox.test.ts b/packages/adapter-utils/src/execution-target-sandbox.test.ts
index 73660142f9..0ecf167102 100644
--- a/packages/adapter-utils/src/execution-target-sandbox.test.ts
+++ b/packages/adapter-utils/src/execution-target-sandbox.test.ts
@@ -43,12 +43,18 @@ import {
type StartupTraceContext,
type StartupTracer,
} from "./acpx-engine/startup-timing.js";
+import {
+ DuplexAggregateByteLedger,
+ DUPLEX_AGGREGATE_TOKEN_OWNERS,
+ DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED,
+} from "./duplex-aggregate-byte-ledger.js";
import { createSandboxRunLogTailFactory, type SandboxRunLogTailFactory } from "./sandbox-run-log-stream.js";
import { runChildProcess } from "./server-utils.js";
import { shellQuote } from "./ssh.js";
import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js";
import {
DEFAULT_MAX_DUPLEX_FRAME_BYTES,
+ DuplexFrameDecoder,
DUPLEX_FRAME_VERSION,
decodeDuplexLine,
encodeDuplexFrame,
@@ -66,10 +72,14 @@ import {
} from "./duplex-bridge-broker.js";
import {
createDuplexTelemetry,
+ DUPLEX_AGGREGATE_BYTE_LEDGER_METRIC_NAMES,
+ DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL,
+ DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL,
DUPLEX_COUNTER_CHANNEL_OPEN_TOTAL,
DUPLEX_COUNTER_FALLBACK_TOTAL,
DUPLEX_COUNTER_LOSS_TOTAL,
DUPLEX_DIMENSION_KEYS,
+ DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE,
DUPLEX_SPAN_CHANNEL_OPEN,
DUPLEX_SPAN_REQUEST,
DUPLEX_TRANSPORT_EVENT,
@@ -2678,6 +2688,155 @@ describe("sandbox adapter execution targets", () => {
}
});
+ it("charges the response-body bytes against the host aggregate ledger and releases every token on success", async () => {
+ // The host stamps one process-owned aggregate byte ledger on the sandbox
+ // target. The forward response-body reader charges its retained bytes against
+ // that ledger. A successful read charges the chunk bytes and the
+ // concatenation buffer, then releases every token, so the ledger returns to
+ // zero after the forward completes.
+ const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-execution-target-bridge-ledger-ok-"));
+ cleanupDirs.push(rootDir);
+ const remoteCwd = path.join(rootDir, "workspace");
+ const runtimeRootDir = path.join(remoteCwd, ".paperclip-runtime", "codex");
+ await mkdir(runtimeRootDir, { recursive: true });
+
+ const responseBody = JSON.stringify({ id: "issue-1" });
+ const apiServer = createServer((_req, res) => {
+ res.writeHead(200, {
+ "content-type": "application/json",
+ "content-length": String(Buffer.byteLength(responseBody, "utf8")),
+ });
+ res.end(responseBody);
+ });
+ await new Promise<void>((resolve, reject) => {
+ apiServer.once("error", reject);
+ apiServer.listen(0, "127.0.0.1", () => resolve());
+ });
+ const address = apiServer.address();
+ if (!address || typeof address === "string") {
+ throw new Error("Expected the bridge test API server to listen on a TCP port.");
+ }
+
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1024 * 1024 });
+ const target: AdapterSandboxExecutionTarget = {
+ kind: "remote",
+ transport: "sandbox",
+ providerKey: "e2b",
+ environmentId: "env-1",
+ leaseId: "lease-1",
+ remoteCwd,
+ runner: createLocalSandboxRunner(),
+ timeoutMs: 30_000,
+ duplexAggregateByteLedger: ledger,
+ };
+
+ const bridge = await startAdapterExecutionTargetPaperclipBridge({
+ runId: "run-bridge-ledger-ok",
+ target,
+ runtimeRootDir,
+ adapterKey: "codex",
+ hostApiToken: "real-run-jwt",
+ hostApiUrl: `http://127.0.0.1:${address.port}`,
+ maxBodyBytes: 512,
+ });
+ try {
+ const response = await fetch(`${bridge!.env.PAPERCLIP_API_URL}/api/issues/issue-1`, {
+ method: "GET",
+ headers: {
+ authorization: `Bearer ${bridge!.env.PAPERCLIP_API_KEY}`,
+ },
+ });
+
+ expect(response.status).toBe(200);
+ await expect(response.json()).resolves.toEqual({ id: "issue-1" });
+ // The reader released every token, so the aggregate gauge and the live-token
+ // registry both return to zero.
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ } finally {
+ await bridge?.stop();
+ await new Promise<void>((resolve) => apiServer.close(() => resolve()));
+ }
+ });
+
+ it("fails a response-body read closed when the host aggregate ledger has no room and retains no bytes", async () => {
+ // The aggregate ledger sits at a tiny ceiling, so a response body larger than
+ // the ceiling cannot reserve its bytes. The reader fails closed: it cancels
+ // the stream reader, retains nothing, and reports the fixed marker. The safe
+ // GET maps the marker to a retryable 502. The ledger returns to zero, because
+ // the reader released the tokens it held before the rejection.
+ const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-execution-target-bridge-ledger-full-"));
+ cleanupDirs.push(rootDir);
+ const remoteCwd = path.join(rootDir, "workspace");
+ const runtimeRootDir = path.join(remoteCwd, ".paperclip-runtime", "codex");
+ await mkdir(runtimeRootDir, { recursive: true });
+
+ // The body sits under the per-request size limit but over the aggregate
+ // ceiling, so the aggregate ledger, not the per-request limit, rejects it.
+ const responseBody = "x".repeat(256);
+ const apiServer = createServer((_req, res) => {
+ res.writeHead(200, {
+ "content-type": "application/json",
+ "content-length": String(Buffer.byteLength(responseBody, "utf8")),
+ });
+ res.end(responseBody);
+ });
+ await new Promise<void>((resolve, reject) => {
+ apiServer.once("error", reject);
+ apiServer.listen(0, "127.0.0.1", () => resolve());
+ });
+ const address = apiServer.address();
+ if (!address || typeof address === "string") {
+ throw new Error("Expected the bridge test API server to listen on a TCP port.");
+ }
+
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 8 });
+ const target: AdapterSandboxExecutionTarget = {
+ kind: "remote",
+ transport: "sandbox",
+ providerKey: "e2b",
+ environmentId: "env-1",
+ leaseId: "lease-1",
+ remoteCwd,
+ runner: createLocalSandboxRunner(),
+ timeoutMs: 30_000,
+ duplexAggregateByteLedger: ledger,
+ };
+
+ const bridge = await startAdapterExecutionTargetPaperclipBridge({
+ runId: "run-bridge-ledger-full",
+ target,
+ runtimeRootDir,
+ adapterKey: "codex",
+ hostApiToken: "real-run-jwt",
+ hostApiUrl: `http://127.0.0.1:${address.port}`,
+ maxBodyBytes: 4096,
+ });
+ try {
+ const response = await fetch(`${bridge!.env.PAPERCLIP_API_URL}/api/issues/issue-1`, {
+ method: "GET",
+ headers: {
+ authorization: `Bearer ${bridge!.env.PAPERCLIP_API_KEY}`,
+ },
+ });
+
+ // The safe GET maps the aggregate rejection to a retryable 502 with no
+ // indeterminate marker. The body carries only the fixed rejection marker.
+ expect(response.status).toBe(502);
+ expect(response.headers.get("x-paperclip-bridge-outcome")).toBeNull();
+ await expect(response.json()).resolves.toEqual({
+ error: DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED,
+ });
+ // The reader released the tokens it held before the rejection, so the
+ // aggregate gauge and the live-token registry both return to zero.
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ } finally {
+ await bridge?.stop();
+ await new Promise<void>((resolve) => apiServer.close(() => resolve()));
+ }
+ });
+
it("forwards the host indeterminate-outcome header so the sandbox server maps the 504 to a non-retryable 409", async () => {
// The host marks a possibly-committed mutation with a 504 and the
// `x-paperclip-bridge-outcome: indeterminate` header. The forward must keep
@@ -3130,6 +3289,66 @@ describe("sandbox adapter execution targets", () => {
expect(control.stopCount).toBeGreaterThanOrEqual(1);
}, 20000);
+ it("falls back to the file bridge when a post-READY pre-bind flood exceeds the aggregate ceiling", async () => {
+ // The gateway sends a valid READY, then floods the channel before the broker
+ // binds. The pre-READY buffer cap does not bound the post-READY replay buffer,
+ // so the replay reservation must. The ceiling admits the small READY frame but
+ // rejects the flood. The host drops the buffer, stops the channel, and selects
+ // the file bridge with the aggregate marker. No request forwards, and the
+ // aggregate ledger returns to zero.
+ const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-replay-flood-"));
+ cleanupDirs.push(rootDir);
+ const remoteCwd = path.join(rootDir, "workspace");
+ await mkdir(remoteCwd, { recursive: true });
+ const api = await startRecordingApiServer();
+ // The flood is larger than the ceiling; the READY frame is far smaller.
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 4096 });
+ const { runner, control } = makeDuplexSelectionRunner((ctx) => {
+ ctx.emitFrame({ version: 1, type: "ready", nonce: ctx.nonce });
+ ctx.emitRaw("x".repeat(64 * 1024));
+ });
+ const { recorder, counters } = createRecordingDuplexRecorder();
+ const target: AdapterSandboxExecutionTarget = {
+ kind: "remote",
+ transport: "sandbox",
+ providerKey: "daytona",
+ remoteCwd,
+ timeoutMs: 30_000,
+ runner,
+ effectiveCapabilities: duplexCapabilities(true),
+ duplexAggregateByteLedger: ledger,
+ };
+
+ const bridge = await startAdapterExecutionTargetPaperclipBridge({
+ runId: "run-duplex-replay-flood",
+ target,
+ runtimeRootDir: path.join(remoteCwd, ".paperclip-runtime", "codex"),
+ adapterKey: "codex",
+ hostApiToken: "real-run-jwt",
+ hostApiUrl: api.origin,
+ enableSandboxDuplexBridge: true,
+ duplexTelemetryRecorder: recorder,
+ });
+ try {
+ // The file bridge serves, not the duplex transport.
+ expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("queue_v1");
+ // The fallback names the aggregate marker on the file transport.
+ const fallback = counters.find((c) => c.metric === DUPLEX_COUNTER_FALLBACK_TOTAL);
+ expect(fallback?.dimensions.fallback_reason).toBe("aggregate_bytes_exceeded");
+ expect(fallback?.dimensions.transport).toBe("file");
+ // The gate stopped the flooded channel.
+ expect(control.stopCount).toBeGreaterThanOrEqual(1);
+ // No request forwarded, because the broker never bound.
+ expect(api.requests).toHaveLength(0);
+ // The aggregate ledger returns to zero with no live token.
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ } finally {
+ await bridge?.stop();
+ await api.close();
+ }
+ }, 20000);
+
it("streams run logs on the duplex path under the same gate and log line as the file path", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-runlog-"));
cleanupDirs.push(rootDir);
@@ -3583,6 +3802,24 @@ describe("sandbox adapter execution targets", () => {
]);
});
+ it("pins the exact aggregate byte ledger metric names", () => {
+ // The aggregate byte ledger metric names are closed. This test locks the
+ // exact set, so a new gauge or counter name needs an explicit change here.
+ // Each record carries only closed constant dimensions and no dynamic label.
+ expect([...DUPLEX_AGGREGATE_BYTE_LEDGER_METRIC_NAMES]).toEqual([
+ "sandbox_duplex_aggregate_bytes_in_use",
+ "sandbox_duplex_aggregate_byte_reservation_rejections_total",
+ "sandbox_duplex_aggregate_byte_accounting_underflow_total",
+ ]);
+ expect(DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE).toBe("sandbox_duplex_aggregate_bytes_in_use");
+ expect(DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL).toBe(
+ "sandbox_duplex_aggregate_byte_reservation_rejections_total",
+ );
+ expect(DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL).toBe(
+ "sandbox_duplex_aggregate_byte_accounting_underflow_total",
+ );
+ });
+
it("records a duplex request span with latency and the fixed dimension keys", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-obs-"));
cleanupDirs.push(rootDir);
@@ -3701,6 +3938,7 @@ describe("sandbox adapter execution targets", () => {
"ready_nonce_mismatch",
"ready_timeout",
"contaminated",
+ "aggregate_bytes_exceeded",
];
expect(approvedReasons).toContain(fallback?.dimensions.fallback_reason);
expect(fallback?.dimensions).toMatchObject({ transport: "file", outcome: "error" });
@@ -4213,6 +4451,128 @@ describe("sandbox adapter execution targets", () => {
}
}, 20000);
+ it("fails the readiness handshake closed when the host aggregate ledger has no room for the pre-READY buffer", async () => {
+ const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-ready-ledger-full-"));
+ cleanupDirs.push(rootDir);
+ const remoteCwd = path.join(rootDir, "workspace");
+ await mkdir(remoteCwd, { recursive: true });
+ const api = await startRecordingApiServer();
+ // The host stamps one process-owned aggregate byte ledger on the sandbox
+ // target at a tiny ceiling. The fake gateway sends a pre-READY blob larger
+ // than the ceiling, so the gate cannot reserve the blob bytes. The gate fails
+ // closed: it retains nothing, records the aggregate fallback reason, and falls
+ // back to the file bridge. The blob is smaller than the readiness buffer cap,
+ // so the aggregate ledger, not the buffer cap, drives the failure.
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 256 });
+ const preReadyBlob = "x".repeat(4_096);
+ const { runner, control } = makeDuplexSelectionRunner((ctx) => {
+ ctx.emitRaw(preReadyBlob);
+ });
+ const { recorder, counters } = createRecordingDuplexRecorder();
+ const target: AdapterSandboxExecutionTarget = {
+ kind: "remote",
+ transport: "sandbox",
+ providerKey: "daytona",
+ remoteCwd,
+ timeoutMs: 30_000,
+ runner,
+ effectiveCapabilities: duplexCapabilities(true),
+ duplexAggregateByteLedger: ledger,
+ };
+
+ const bridge = await startAdapterExecutionTargetPaperclipBridge({
+ runId: "run-ready-ledger-full",
+ target,
+ runtimeRootDir: path.join(remoteCwd, ".paperclip-runtime", "codex"),
+ adapterKey: "codex",
+ hostApiToken: "real-run-jwt",
+ hostApiUrl: api.origin,
+ enableSandboxDuplexBridge: true,
+ // A long readiness timeout, so the aggregate ledger, not the timeout, drives
+ // the failure.
+ duplexReadinessTimeoutMs: 5_000,
+ duplexTelemetryRecorder: recorder,
+ });
+ try {
+ expect(bridge).not.toBeNull();
+ expect(control.openCount).toBe(1);
+ // The aggregate rejection drove the failure, so the file bridge serves.
+ expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("queue_v1");
+ const fallback = counters.find((c) => c.metric === DUPLEX_COUNTER_FALLBACK_TOTAL);
+ expect(fallback?.dimensions.fallback_reason).toBe("aggregate_bytes_exceeded");
+ // The gate retained nothing after the rejection, so the aggregate gauge and
+ // the live-token registry both return to zero.
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ expect(control.closeCount + control.stopCount).toBeGreaterThanOrEqual(1);
+ } finally {
+ await bridge?.stop();
+ await api.close();
+ }
+ }, 20000);
+
+ it("charges the pre-READY buffer against the injected host ledger and releases it when readiness passes", async () => {
+ const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-ready-ledger-ok-"));
+ cleanupDirs.push(rootDir);
+ const remoteCwd = path.join(rootDir, "workspace");
+ await mkdir(remoteCwd, { recursive: true });
+ const api = await startRecordingApiServer();
+ // The host stamps one process-owned aggregate byte ledger on the sandbox
+ // target at a generous ceiling. The fake gateway sends one pre-READY noise
+ // line, then the valid READY frame. The gate charges the noise bytes against
+ // the injected ledger, passes readiness, and releases the pre-READY tokens.
+ // The broker's frame decoder re-charges any post-READY bytes, so the ledger
+ // returns to zero after the handshake.
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1024 * 1024 });
+ const reserveSpy = vi.spyOn(ledger, "reserve");
+ const { runner, control } = makeDuplexSelectionRunner((ctx) => {
+ ctx.emitRaw("pty-echo-noise");
+ ctx.emitFrame({ version: 1, type: "ready", nonce: ctx.nonce });
+ });
+ const target: AdapterSandboxExecutionTarget = {
+ kind: "remote",
+ transport: "sandbox",
+ providerKey: "daytona",
+ remoteCwd,
+ timeoutMs: 30_000,
+ runner,
+ effectiveCapabilities: duplexCapabilities(true),
+ duplexAggregateByteLedger: ledger,
+ };
+
+ const bridge = await startAdapterExecutionTargetPaperclipBridge({
+ runId: "run-ready-ledger-ok",
+ target,
+ runtimeRootDir: path.join(remoteCwd, ".paperclip-runtime", "codex"),
+ adapterKey: "codex",
+ hostApiToken: "real-run-jwt",
+ hostApiUrl: api.origin,
+ enableSandboxDuplexBridge: true,
+ duplexReadinessTimeoutMs: 5_000,
+ });
+ try {
+ expect(bridge).not.toBeNull();
+ // Readiness passed, so the duplex transport serves.
+ expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("duplex_v1");
+ // The gate charged the pre-READY noise against the exact injected ledger, so
+ // the identity holds at this seam.
+ expect(reserveSpy).toHaveBeenCalledWith("readiness_buffer", expect.any(Number));
+ // The gate released every readiness-buffer token on settle, so the aggregate
+ // gauge and the live-token registry both return to zero.
+ await waitForCondition(
+ () => ledger.bytesInUse === 0 && ledger.liveTokenCount === 0,
+ "the readiness gate to release every pre-READY token",
+ 4000,
+ );
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ } finally {
+ reserveSpy.mockRestore();
+ await bridge?.stop();
+ await api.close();
+ }
+ }, 20000);
+
it("bounds the pre-READY newline-scan work by the bytes received", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-scan-"));
cleanupDirs.push(rootDir);
@@ -4850,11 +5210,16 @@ interface EmbeddedCodec {
encodeDuplexFrame: (frame: unknown) => string;
encodeDuplexFrameChecked: (frame: unknown, maxFrameBytes?: number) => EmbeddedEncodeResult;
decodeDuplexLine: (line: string | Buffer) => EmbeddedDecodeResult;
- DuplexFrameDecoder: new (options?: { maxFrameBytes?: number }) => {
+ DuplexFrameDecoder: new (options?: { maxFrameBytes?: number; maxAggregateBytes?: number }) => {
push: (chunk: Buffer) => EmbeddedDecodeResult[];
+ scope: string;
+ bytesInUse: number;
+ aggregateRejections: number;
};
DUPLEX_FRAME_VERSION: number;
DEFAULT_MAX_DUPLEX_FRAME_BYTES: number;
+ DUPLEX_DECODER_SCOPE: string;
+ DEFAULT_MAX_DUPLEX_DECODER_BYTES: number;
}
type ExpectedVectorResult = { frame: unknown } | { error: string };
@@ -5099,7 +5464,7 @@ describe("sandbox duplex gateway", () => {
it("embedded gateway codec passes every vector in the shared fixture", async () => {
const codecFactory = new Function(
- `${getSandboxDuplexGatewayCodecSource()}\nreturn { encodeDuplexFrame, encodeDuplexFrameChecked, decodeDuplexLine, DuplexFrameDecoder, DUPLEX_FRAME_VERSION, DEFAULT_MAX_DUPLEX_FRAME_BYTES };`,
+ `${getSandboxDuplexGatewayCodecSource()}\nreturn { encodeDuplexFrame, encodeDuplexFrameChecked, decodeDuplexLine, DuplexFrameDecoder, DUPLEX_FRAME_VERSION, DEFAULT_MAX_DUPLEX_FRAME_BYTES, DUPLEX_DECODER_SCOPE, DEFAULT_MAX_DUPLEX_DECODER_BYTES };`,
) as unknown as () => EmbeddedCodec;
const codec = codecFactory();
@@ -5199,6 +5564,61 @@ describe("sandbox duplex gateway", () => {
expect(encodeFailures).toEqual([]);
});
+ it("bounds the sandbox decoder with a separate sandbox_process scope, distinct from the host ledger scope", () => {
+ // The generated gateway runs in a separate operating-system process, so its
+ // decoder cannot share the host aggregate byte ledger. It enforces a separate
+ // local cap under the `sandbox_process` scope. This test wraps the embedded
+ // source and the host decoder, then proves the two scopes never overlap and the
+ // sandbox cap fails closed.
+ const codecFactory = new Function(
+ `${getSandboxDuplexGatewayCodecSource()}\nreturn { encodeDuplexFrame, decodeDuplexLine, DuplexFrameDecoder, DUPLEX_FRAME_VERSION, DEFAULT_MAX_DUPLEX_FRAME_BYTES, DUPLEX_DECODER_SCOPE, DEFAULT_MAX_DUPLEX_DECODER_BYTES };`,
+ ) as unknown as () => EmbeddedCodec;
+ const codec = codecFactory();
+
+ // The sandbox scope is the fixed `sandbox_process` label.
+ expect(codec.DUPLEX_DECODER_SCOPE).toBe("sandbox_process");
+ expect(codec.DEFAULT_MAX_DUPLEX_DECODER_BYTES).toBeGreaterThan(codec.DEFAULT_MAX_DUPLEX_FRAME_BYTES);
+ // The two scopes are distinct: the host owner set never carries the sandbox
+ // scope, so the sandbox counter can never map to a host aggregate token.
+ expect((DUPLEX_AGGREGATE_TOKEN_OWNERS as readonly string[]).includes("sandbox_process")).toBe(false);
+
+ // The sandbox decoder tracks its own `sandbox_process` counter. A frame under
+ // the cap charges the local counter, and the counter returns to zero once the
+ // frame drains.
+ const sandboxDecoder = new codec.DuplexFrameDecoder({ maxAggregateBytes: 64 });
+ expect(sandboxDecoder.scope).toBe("sandbox_process");
+ const partial = sandboxDecoder.push(Buffer.from('{"version":1,', "utf8"));
+ expect(partial).toEqual([]);
+ expect(sandboxDecoder.bytesInUse).toBe(Buffer.byteLength('{"version":1,', "utf8"));
+ sandboxDecoder.push(Buffer.from('"type":"heartbeat"}\n', "utf8"));
+ expect(sandboxDecoder.bytesInUse).toBe(0);
+
+ // A chunk over the local cap fails closed. The decoder retains nothing, reports
+ // the aggregate rejection, and increments only its local `sandbox_process`
+ // counter.
+ const cappedDecoder = new codec.DuplexFrameDecoder({ maxAggregateBytes: 8 });
+ const overCap = cappedDecoder.push(Buffer.from("x".repeat(64), "utf8"));
+ expect(overCap.length).toBe(1);
+ const rejection = overCap[0];
+ expect(rejection.ok).toBe(false);
+ expect(rejection.error?.code).toBe("aggregate_bytes_exceeded");
+ expect(cappedDecoder.bytesInUse).toBe(0);
+ expect(cappedDecoder.aggregateRejections).toBe(1);
+
+ // The host decoder charges the host aggregate byte ledger under the
+ // `decoder_buffer` owner. It never uses the sandbox scope. This proves the two
+ // implementations use distinct scopes: the host uses the injected ledger; the
+ // sandbox uses its local counter.
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1024 });
+ const hostDecoder = new DuplexFrameDecoder({ aggregateByteLedger: ledger });
+ hostDecoder.push(Buffer.from('{"version":1,', "utf8"));
+ expect(ledger.bytesInUse).toBe(Buffer.byteLength('{"version":1,', "utf8"));
+ expect(ledger.liveTokenCount).toBeGreaterThan(0);
+ hostDecoder.dispose();
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ });
+
it("returns the same HTTP response as the file gateway for a forwarded request", async () => {
const token = "duplex-token-forward";
const gateway = await startDuplexGateway({ PAPERCLIP_BRIDGE_TOKEN: token });
@@ -6172,3 +6592,213 @@ describe("CLI-lane run-disposition seam", () => {
expect(settleCalls).toBe(0);
});
});
+
+describe("duplex readiness gate replay-buffer reservation", () => {
+ const READY_NONCE = "0123456789abcdef0123456789abcdef";
+
+ // A fake duplex channel the test drives directly. `control.emitData` re-enters
+ // the data listener the gate bound at construction. `control.emitExit` re-enters
+ // the exit listener. The fake records the stop and the close calls.
+ function makeFakeReadinessChannel(): {
+ channel: CommandManagedDuplexChannel;
+ control: {
+ stopCount: number;
+ closeCount: number;
+ written: string[];
+ emitData: (chunk: string) => void;
+ emitExit: (exit: { exitCode: number | null }) => void;
+ };
+ } {
+ let dataListener: ((chunk: string) => void) | null = null;
+ let exitListener: ((exit: { exitCode: number | null }) => void) | null = null;
+ const control = {
+ stopCount: 0,
+ closeCount: 0,
+ written: [] as string[],
+ emitData: (chunk: string): void => dataListener?.(chunk),
+ emitExit: (exit: { exitCode: number | null }): void => exitListener?.(exit),
+ };
+ const channel: CommandManagedDuplexChannel = {
+ write(data: string): void {
+ control.written.push(data);
+ },
+ onData(listener: (chunk: string) => void): void {
+ dataListener = listener;
+ },
+ onExit(listener: (exit: { exitCode: number | null }) => void): void {
+ exitListener = listener;
+ },
+ stop(): void {
+ control.stopCount += 1;
+ },
+ close(): Promise<void> {
+ control.closeCount += 1;
+ return Promise.resolve();
+ },
+ };
+ return { channel, control };
+ }
+
+ // A ledger that counts the reservation-rejection and the accounting-underflow
+ // signals, so a test proves the one-owner-one-release invariant holds.
+ function makeCountingLedger(ceilingBytes: number): {
+ ledger: DuplexAggregateByteLedger;
+ counts: { rejections: number; underflows: number };
+ } {
+ const counts = { rejections: 0, underflows: 0 };
+ const ledger = new DuplexAggregateByteLedger({
+ ceilingBytes,
+ telemetry: {
+ setBytesInUse(): void {},
+ recordReservationRejection(): void {
+ counts.rejections += 1;
+ },
+ recordAccountingUnderflow(): void {
+ counts.underflows += 1;
+ },
+ },
+ });
+ return { ledger, counts };
+ }
+
+ function readyLine(): string {
+ return `${JSON.stringify({ version: 1, type: "ready", nonce: READY_NONCE })}\n`;
+ }
+
+ it("charges the post-READY suffix and releases it after the broker handoff", async () => {
+ const { channel, control } = makeFakeReadinessChannel();
+ const { ledger, counts } = makeCountingLedger(1024 * 1024);
+ const gate = __duplexReadinessTesting.createReadinessGate(channel, {
+ nonce: READY_NONCE,
+ timeoutMs: 5_000,
+ ledger,
+ });
+ const suffix = "hello-post-ready-suffix";
+ // The READY line and the suffix arrive in one chunk. The gate drops the whole
+ // pre-READY buffer charge, then charges only the retained suffix.
+ control.emitData(`${readyLine()}${suffix}`);
+ const readiness = await gate.ready;
+ expect(readiness.ok).toBe(true);
+ expect(gate.replayOverflowed()).toBe(false);
+ // The gate holds the suffix under one readiness_replay token before the bind.
+ expect(ledger.bytesInUse).toBe(Buffer.byteLength(suffix, "utf8"));
+ expect(ledger.liveTokenCount).toBe(1);
+ // The broker binds and replays the suffix; the gate releases the token after
+ // the synchronous handoff.
+ const replayed: string[] = [];
+ gate.brokerChannel.onData((chunk) => replayed.push(chunk));
+ expect(replayed).toEqual([suffix]);
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ expect(counts.underflows).toBe(0);
+ });
+
+ it("releases the suffix token on disposePendingReplay without a broker handoff", async () => {
+ const { channel, control } = makeFakeReadinessChannel();
+ const { ledger, counts } = makeCountingLedger(1024 * 1024);
+ const gate = __duplexReadinessTesting.createReadinessGate(channel, {
+ nonce: READY_NONCE,
+ timeoutMs: 5_000,
+ ledger,
+ });
+ const suffix = "abandoned-suffix";
+ control.emitData(`${readyLine()}${suffix}`);
+ expect((await gate.ready).ok).toBe(true);
+ expect(ledger.bytesInUse).toBe(Buffer.byteLength(suffix, "utf8"));
+ // A broker-construction failure abandons the buffer, so the caller disposes it.
+ gate.disposePendingReplay();
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ // A second dispose is a no-op and never underflows.
+ gate.disposePendingReplay();
+ expect(ledger.bytesInUse).toBe(0);
+ expect(counts.underflows).toBe(0);
+ });
+
+ it("releases the suffix token after a pre-bind exit then a broker handoff", async () => {
+ const { channel, control } = makeFakeReadinessChannel();
+ const { ledger, counts } = makeCountingLedger(1024 * 1024);
+ const gate = __duplexReadinessTesting.createReadinessGate(channel, {
+ nonce: READY_NONCE,
+ timeoutMs: 5_000,
+ ledger,
+ });
+ const suffix = "pre-bind-exit-suffix";
+ control.emitData(`${readyLine()}${suffix}`);
+ expect((await gate.ready).ok).toBe(true);
+ // The channel exits after READY but before the broker binds. The gate holds the
+ // exit and keeps the pending suffix charged.
+ control.emitExit({ exitCode: 0 });
+ expect(ledger.bytesInUse).toBe(Buffer.byteLength(suffix, "utf8"));
+ expect(ledger.liveTokenCount).toBe(1);
+ // The broker binds, replays the suffix and the exit, then the gate releases the
+ // reservation.
+ const replayed: string[] = [];
+ const exits: Array<{ exitCode: number | null }> = [];
+ gate.brokerChannel.onData((chunk) => replayed.push(chunk));
+ gate.brokerChannel.onExit((exit) => exits.push(exit));
+ expect(replayed).toEqual([suffix]);
+ expect(exits).toEqual([{ exitCode: 0 }]);
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ expect(counts.underflows).toBe(0);
+ });
+
+ it("fails closed when a post-READY pre-bind chunk floods past the ceiling", async () => {
+ const { channel, control } = makeFakeReadinessChannel();
+ // The ceiling admits the small READY line but not the flood chunk.
+ const { ledger, counts } = makeCountingLedger(256);
+ const gate = __duplexReadinessTesting.createReadinessGate(channel, {
+ nonce: READY_NONCE,
+ timeoutMs: 5_000,
+ ledger,
+ });
+ // The READY line arrives alone, so the pending suffix starts empty.
+ control.emitData(readyLine());
+ expect((await gate.ready).ok).toBe(true);
+ expect(ledger.bytesInUse).toBe(0);
+ // A post-READY chunk larger than the ceiling floods the replay buffer before
+ // the broker binds. The gate refuses the reservation and fails closed.
+ control.emitData("x".repeat(512));
+ expect(gate.replayOverflowed()).toBe(true);
+ expect(control.stopCount).toBe(1);
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ expect(counts.rejections).toBe(1);
+ expect(counts.underflows).toBe(0);
+ // Binding the broker replays nothing, because the gate dropped the buffer.
+ const replayed: string[] = [];
+ gate.brokerChannel.onData((chunk) => replayed.push(chunk));
+ expect(replayed).toEqual([]);
+ });
+
+ it("drops the retained pre-READY buffer on READY acceptance", async () => {
+ const { channel, control } = makeFakeReadinessChannel();
+ const { ledger, counts } = makeCountingLedger(1024 * 1024);
+ const gate = __duplexReadinessTesting.createReadinessGate(channel, {
+ nonce: READY_NONCE,
+ timeoutMs: 5_000,
+ ledger,
+ });
+ // A large pre-READY noise line and a large suffix arrive with the READY line
+ // in one chunk. A sandbox controls every byte here.
+ const noise = `${"n".repeat(4096)}\n`;
+ const suffix = "s".repeat(2048);
+ control.emitData(`${noise}${readyLine()}${suffix}`);
+ expect((await gate.ready).ok).toBe(true);
+ // The gate drops the pre-READY buffer, so the process no longer retains the
+ // noise prefix. Without this, the process holds the full sandbox string while
+ // the ledger charges only the suffix, so retention passes the ceiling.
+ expect(gate.retainedReadinessBufferLength()).toBe(0);
+ // The ledger charges only the retained suffix, not the dropped prefix.
+ expect(ledger.bytesInUse).toBe(Buffer.byteLength(suffix, "utf8"));
+ expect(ledger.liveTokenCount).toBe(1);
+ // The broker binds, replays the suffix, and the gate releases the token.
+ const replayed: string[] = [];
+ gate.brokerChannel.onData((chunk) => replayed.push(chunk));
+ expect(replayed).toEqual([suffix]);
+ expect(ledger.bytesInUse).toBe(0);
+ expect(gate.retainedReadinessBufferLength()).toBe(0);
+ expect(counts.underflows).toBe(0);
+ });
+});
diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.ts
index faece1d9bf..065a7f8c5b 100644
--- a/packages/adapter-utils/src/execution-target.ts
+++ b/packages/adapter-utils/src/execution-target.ts
@@ -29,6 +29,7 @@ import {
createSandboxCallbackBridgeAsset,
createSandboxCallbackBridgeToken,
DEFAULT_SANDBOX_CALLBACK_BRIDGE_MAX_BODY_BYTES,
+ DEFAULT_SANDBOX_DUPLEX_DECODER_MAX_BYTES,
SANDBOX_CALLBACK_BRIDGE_DUPLEX_MODE,
SANDBOX_CALLBACK_BRIDGE_ENTRYPOINT,
sandboxCallbackBridgeDirectories,
@@ -63,6 +64,11 @@ import {
type DuplexFallbackReason,
type DuplexTelemetryRecorder,
} from "./duplex-telemetry.js";
+import {
+ DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED,
+ type DuplexAggregateByteLedger,
+ type ReservationToken,
+} from "./duplex-aggregate-byte-ledger.js";
import { createSshCommandManagedRuntimeRunner, parseSshRemoteExecutionSpec, runSshCommand, shellQuote } from "./ssh.js";
import {
ensureCommandResolvable,
@@ -184,6 +190,16 @@ export interface AdapterSandboxExecutionTarget extends AdapterExecutionTargetWor
* observability surface. Absent means the safe no-op default.
*/
duplexTelemetryRecorder?: DuplexTelemetryRecorder | null;
+ /**
+ * The process-owned aggregate byte ledger for the sandbox duplex channel. The
+ * host stamps this same object on every sandbox target on the same seam as
+ * `runner`, so one shared gauge bounds the aggregate retained bytes across all
+ * live duplex routes. The live object stays on the host and never enters the
+ * sandbox environment. The bridge passes it to the broker, the decoder, and the
+ * response-body reader. Absent means no host ledger; a non-duplex run keeps the
+ * bridge inert for this seam.
+ */
+ duplexAggregateByteLedger?: DuplexAggregateByteLedger | null;
}
export type AdapterExecutionTarget =
@@ -428,6 +444,20 @@ export function adapterExecutionTargetDuplexTelemetryRecorder(
: null;
}
+/**
+ * Read the injected aggregate byte ledger off a target. Only a sandbox target
+ * with a ledger attached returns it. Every other target returns null, so the
+ * bridge stays inert for this seam. The reader never makes a fresh ledger, so a
+ * host duplex run always uses the one process-owned ledger the host stamped.
+ */
+export function adapterExecutionTargetDuplexAggregateByteLedger(
+ target: AdapterExecutionTarget | null | undefined,
+): DuplexAggregateByteLedger | null {
+ return target?.kind === "remote" && target.transport === "sandbox"
+ ? target.duplexAggregateByteLedger ?? null
+ : null;
+}
+
export function adapterExecutionTargetRemoteCwd(
target: AdapterExecutionTarget | null | undefined,
localCwd: string,
@@ -1468,7 +1498,27 @@ function bridgeResponseBodyLimitError(maxBodyBytes: number): Error {
return new Error(`Bridge response body exceeded the configured size limit of ${maxBodyBytes} bytes.`);
}
-async function readBridgeForwardResponseBody(response: Response, maxBodyBytes: number): Promise<string> {
+/**
+ * Read the forward response body into a string. The reader bounds the body with
+ * two controls. The per-request `maxBodyBytes` limit rejects a body larger than
+ * the configured per-request ceiling. The optional host aggregate byte ledger
+ * bounds the retained bytes across all live routes.
+ *
+ * The reader charges the ledger for every retained buffer before it allocates
+ * that buffer. It reserves the exact chunk bytes before it copies a chunk into a
+ * retained `Buffer`. It reserves the concatenation buffer before it allocates it.
+ * A reservation that would pass the aggregate ceiling returns no token; the
+ * reader retains nothing more, cancels the stream reader, and throws the fixed
+ * marker {@link DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED}. The `finally` releases
+ * every token exactly one time, so the reader charges the retained bytes only
+ * while the raw buffers live and never leaves a token held after it returns or
+ * throws.
+ */
+async function readBridgeForwardResponseBody(
+ response: Response,
+ maxBodyBytes: number,
+ ledger?: DuplexAggregateByteLedger | null,
+): Promise<string> {
const rawContentLength = response.headers.get("content-length");
if (rawContentLength) {
const contentLength = Number.parseInt(rawContentLength, 10);
@@ -1483,19 +1533,54 @@ async function readBridgeForwardResponseBody(response: Response, maxBodyBytes: n
const reader = response.body.getReader();
const chunks: Buffer[] = [];
+ // Every response-body reservation token the reader holds. The `finally` block
+ // releases each token one time, so a return, a size error, an aggregate
+ // rejection, and a read error all release every token.
+ const tokens: ReservationToken[] = [];
let totalBytes = 0;
- while (true) {
- const { done, value } = await reader.read();
- if (done) break;
- if (!value) continue;
- totalBytes += value.byteLength;
- if (totalBytes > maxBodyBytes) {
- await reader.cancel().catch(() => undefined);
- throw bridgeResponseBodyLimitError(maxBodyBytes);
+ try {
+ while (true) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ if (!value) continue;
+ const chunkBytes = value.byteLength;
+ totalBytes += chunkBytes;
+ if (totalBytes > maxBodyBytes) {
+ await reader.cancel().catch(() => undefined);
+ throw bridgeResponseBodyLimitError(maxBodyBytes);
+ }
+ // Reserve the exact chunk bytes before the host copies the chunk into a
+ // retained buffer. A rejection fails closed: cancel the stream reader and
+ // report the fixed marker; the reader retains nothing more.
+ if (ledger) {
+ const token = ledger.reserve("response_body", chunkBytes);
+ if (!token) {
+ await reader.cancel().catch(() => undefined);
+ throw new Error(DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED);
+ }
+ tokens.push(token);
+ }
+ chunks.push(Buffer.from(value));
+ }
+ // Reserve the concatenation buffer before the reader allocates it. The
+ // concatenation buffer is a second copy of the body bytes that lives next to
+ // the chunk buffers during the concatenation, so it is the peak retained
+ // allocation. A rejection fails closed with the fixed marker.
+ if (ledger && totalBytes > 0) {
+ const concatToken = ledger.reserve("response_body", totalBytes);
+ if (!concatToken) {
+ throw new Error(DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED);
+ }
+ tokens.push(concatToken);
+ }
+ return Buffer.concat(chunks, totalBytes).toString("utf8");
+ } finally {
+ if (ledger) {
+ for (const token of tokens) {
+ ledger.release(token);
+ }
}
- chunks.push(Buffer.from(value));
}
- return Buffer.concat(chunks, totalBytes).toString("utf8");
}
const PROCESS_SESSION_PROXY_SCRIPT = "paperclip-process-session-proxy.mjs";
@@ -2394,7 +2479,8 @@ type DuplexReadinessFailure =
| "protocol_contamination"
| "nonce_mismatch"
| "channel_exit"
- | "timeout";
+ | "timeout"
+ | "aggregate_bytes_exceeded";
/** The outcome of the duplex readiness handshake. */
type DuplexReadinessResult =
@@ -2416,6 +2502,8 @@ function duplexReadinessFallbackReason(reason: DuplexReadinessFailure): DuplexFa
return "ready_timeout";
case "channel_exit":
return "ready_invalid";
+ case "aggregate_bytes_exceeded":
+ return "aggregate_bytes_exceeded";
}
}
@@ -2542,6 +2630,13 @@ export const __duplexReadinessTesting = {
resetNewlineScanUnits: (): void => {
duplexReadinessNewlineScanUnits = 0;
},
+ // Build one readiness gate over a supplied channel, so a test can drive the
+ // readiness-replay reservation lifecycle across every terminal path without the
+ // whole bridge. Production code never reads this factory.
+ createReadinessGate: (
+ channel: CommandManagedDuplexChannel,
+ options: { nonce: string; timeoutMs: number; ledger?: DuplexAggregateByteLedger | null },
+ ) => createDuplexReadinessGate(channel, options),
};
interface DuplexReadinessGate {
@@ -2554,14 +2649,82 @@ interface DuplexReadinessGate {
* the channel.
*/
readonly brokerChannel: CommandManagedDuplexChannel;
+ /**
+ * Report whether a post-READY pre-bind chunk could not reserve its replay bytes
+ * against the aggregate ledger. On such a refusal the gate drops the pending
+ * replay buffer and stops the channel. The caller reads this after `ready`
+ * resolves `ok`, and before it binds the broker. A `true` result means the caller
+ * must abandon the broker and select the file bridge with the aggregate marker.
+ */
+ replayOverflowed(): boolean;
+ /**
+ * Release every held readiness-replay reservation exactly once and drop the
+ * pending replay buffer. The caller runs this on a terminal path that abandons
+ * the pending replay without a broker handoff: a readiness failure, a replay
+ * overflow, or a broker-construction failure. The normal handoff releases the
+ * reservation inside `brokerChannel.onData`, so a later call here is a no-op.
+ */
+ disposePendingReplay(): void;
+ /**
+ * Test-only. Report the length of the retained pre-READY buffer, in UTF-16 code
+ * units. A test reads this to prove the gate drops the pre-READY buffer on READY
+ * acceptance, so the process does not retain the sandbox-controlled prefix.
+ * Production code does not read this.
+ */
+ retainedReadinessBufferLength(): number;
}
function createDuplexReadinessGate(
channel: CommandManagedDuplexChannel,
- options: { nonce: string; timeoutMs: number },
+ options: {
+ nonce: string;
+ timeoutMs: number;
+ // The one host-process aggregate byte ledger. The gate charges the untrusted
+ // pre-READY buffer bytes against it, so a pre-READY flood counts toward the
+ // aggregate ceiling across all live routes. A gate with no ledger stays inert
+ // for this seam. The gate holds the same object every other host retention
+ // site holds, so the aggregate identity holds at this seam.
+ ledger?: DuplexAggregateByteLedger | null;
+ },
): DuplexReadinessGate {
+ const ledger = options.ledger ?? null;
let settled = false;
let readyOk = false;
+ // Every readiness-buffer reservation token the gate holds for the pre-READY
+ // bytes. The gate releases each token one time when it settles or when it
+ // accepts READY. On a failed handshake the gate drops the buffer, so the release
+ // frees the untrusted bytes. On READY the gate discards the whole pre-READY
+ // buffer, then re-charges only the retained suffix under `readiness_replay`.
+ const retainedTokens: ReservationToken[] = [];
+ // Every readiness-replay reservation token the gate holds for the post-READY
+ // suffix and each later pre-bind chunk. The gate releases each token one time
+ // after the synchronous handoff to the broker, or on a terminal path that
+ // abandons the pending replay without a broker handoff.
+ const replayTokens: ReservationToken[] = [];
+ // The gate sets this when a post-READY pre-bind chunk cannot reserve its replay
+ // bytes. On that refusal the gate drops the pending buffer and stops the channel.
+ // The caller reads it through `replayOverflowed` and selects the file bridge.
+ let replayOverflow = false;
+
+ // Release every readiness-buffer token exactly once and clear the registry. A
+ // second call is a no-op, because the array is empty.
+ function releaseReadinessBufferTokens(): void {
+ if (!ledger) return;
+ for (const token of retainedTokens) {
+ ledger.release(token);
+ }
+ retainedTokens.length = 0;
+ }
+
+ // Release every readiness-replay token exactly once and clear the registry. A
+ // second call is a no-op, because the array is empty.
+ function releaseReplayTokens(): void {
+ if (!ledger) return;
+ for (const token of replayTokens) {
+ ledger.release(token);
+ }
+ replayTokens.length = 0;
+ }
// The raw bytes the host reads before the READY frame completes. The buffer is
// append-only, so the O(1) cap check on `buffer.length` stays valid.
let buffer = "";
@@ -2590,6 +2753,11 @@ function createDuplexReadinessGate(
settled = true;
clearTimeout(timer);
if (result.ok) readyOk = true;
+ // Release every readiness-buffer token exactly once. The gate no longer owns
+ // the pre-READY bytes: a failed handshake drops the buffer. The READY-accept
+ // path already released these tokens and charged the retained suffix under
+ // `readiness_replay`, so this call is a no-op there.
+ releaseReadinessBufferTokens();
resolveReady(result);
}
@@ -2599,7 +2767,24 @@ function createDuplexReadinessGate(
return;
}
if (readyOk) {
- // READY already passed; hold the bytes until the broker binds.
+ // READY already passed; hold the bytes until the broker binds. Reserve the
+ // exact UTF-8 bytes under `readiness_replay` before the append, so the replay
+ // buffer counts toward the aggregate ceiling. A refusal fails closed: the gate
+ // drops the pending buffer, releases the replay tokens, stops the channel, and
+ // sets the overflow flag. The caller reads the flag and selects the file bridge
+ // with the aggregate marker, because `ready` already resolved before this
+ // synchronous post-READY chunk arrived.
+ if (ledger) {
+ const token = ledger.reserve("readiness_replay", Buffer.byteLength(chunk, "utf8"));
+ if (!token) {
+ replayOverflow = true;
+ pending = "";
+ releaseReplayTokens();
+ channel.stop();
+ return;
+ }
+ replayTokens.push(token);
+ }
pending += chunk;
return;
}
@@ -2609,6 +2794,18 @@ function createDuplexReadinessGate(
// never grows the buffer after the gate settles.
return;
}
+ // Reserve the exact UTF-8 bytes of this chunk against the aggregate ledger
+ // before the gate retains it. The pre-READY buffer holds untrusted bytes, so
+ // a flood counts toward the process aggregate ceiling. A rejection fails
+ // closed: the gate retains nothing more and falls back to the file bridge.
+ if (ledger) {
+ const token = ledger.reserve("readiness_buffer", Buffer.byteLength(chunk, "utf8"));
+ if (!token) {
+ finish({ ok: false, reason: "aggregate_bytes_exceeded" });
+ return;
+ }
+ retainedTokens.push(token);
+ }
// Append the new bytes and continue the newline search from `scanFrom`, the
// first index not yet examined. Each code unit is read at most one time for
// the search, so the total scan work stays linear in the bytes received.
@@ -2682,8 +2879,33 @@ function createDuplexReadinessGate(
finish({ ok: false, reason: "nonce_mismatch" });
return;
}
- // Hold the bytes that follow the READY line for the broker to replay.
- pending = buffer.slice(newlineIndex + 1);
+ // The bytes that follow the READY line become the replay buffer for the
+ // broker. Drop the whole pre-READY buffer charge first, then reserve the
+ // retained suffix under `readiness_replay`. The release-before-reserve order
+ // keeps the transient charge equal to the suffix, not the sum of the dropped
+ // prefix and the retained suffix. The two steps run in one synchronous
+ // section, so no other route can take the freed bytes in between.
+ const suffix = buffer.slice(newlineIndex + 1);
+ // Drop the original pre-READY buffer now. The gate keeps only the
+ // retained suffix as `pending`, and it charges that suffix under
+ // `readiness_replay` below. If the gate keeps the buffer, the process
+ // retains the full sandbox-controlled string while the ledger counts
+ // only the suffix, so aggregate retention passes the ceiling. This clear
+ // also covers the broker handoff and the replay disposal. Both run later
+ // and read no buffer bytes.
+ buffer = "";
+ releaseReadinessBufferTokens();
+ if (ledger && suffix.length > 0) {
+ const token = ledger.reserve("readiness_replay", Buffer.byteLength(suffix, "utf8"));
+ if (!token) {
+ // The retained suffix passes the aggregate ceiling. Fail closed: drop
+ // the suffix and fall back to the file bridge with the aggregate marker.
+ finish({ ok: false, reason: "aggregate_bytes_exceeded" });
+ return;
+ }
+ replayTokens.push(token);
+ }
+ pending = suffix;
finish({ ok: true });
return;
}
@@ -2723,6 +2945,11 @@ function createDuplexReadinessGate(
pending = "";
listener(replay);
}
+ // Release every readiness-replay token exactly once, after the synchronous
+ // handoff to the broker. The broker charges its own decode retention inside
+ // the `listener(replay)` call above, so the release here never opens an
+ // admission gap for the same retained bytes.
+ releaseReplayTokens();
},
onExit: (listener: (exit: { exitCode: number | null }) => void) => {
exitSink = listener;
@@ -2736,7 +2963,16 @@ function createDuplexReadinessGate(
close: () => channel.close(),
};
- return { ready, brokerChannel };
+ return {
+ ready,
+ brokerChannel,
+ replayOverflowed: () => replayOverflow,
+ disposePendingReplay: () => {
+ pending = "";
+ releaseReplayTokens();
+ },
+ retainedReadinessBufferLength: () => buffer.length,
+ };
}
/**
@@ -2817,6 +3053,12 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: {
}
const target = input.target;
+ // The process-owned aggregate byte ledger the host stamped on this sandbox
+ // target. The forward response-body reader charges its retained bytes against
+ // this one ledger, so the aggregate retained bytes across all live routes stay
+ // under the ceiling. A target with no ledger keeps the reader inert for this
+ // seam.
+ const duplexAggregateByteLedger = adapterExecutionTargetDuplexAggregateByteLedger(target);
const onLog = input.onLog ?? (async () => {});
const hostApiToken = input.hostApiToken?.trim() ?? "";
if (hostApiToken.length === 0) {
@@ -2943,7 +3185,11 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: {
// to a non-retryable 409 for both the file bridge and the duplex broker.
let responseBody: string;
try {
- responseBody = await readBridgeForwardResponseBody(response, maxBodyBytes);
+ responseBody = await readBridgeForwardResponseBody(
+ response,
+ maxBodyBytes,
+ duplexAggregateByteLedger,
+ );
} catch (error) {
if (isSafeBridgeMethod(method)) {
// The method is safe, so a retry cannot double-apply a mutation. Return a
@@ -3049,6 +3295,11 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: {
PAPERCLIP_BRIDGE_PORT: String(assignedPort),
PAPERCLIP_BRIDGE_NONCE: nonce,
PAPERCLIP_BRIDGE_MAX_BODY_BYTES: String(maxBodyBytes),
+ // The separate sandbox-process raw-decoder cap. The generated gateway runs
+ // in a different operating-system process, so it cannot share the host
+ // aggregate byte ledger. It enforces this cap locally under the
+ // `sandbox_process` scope, and the provider memory allocation bounds it.
+ PAPERCLIP_BRIDGE_MAX_DUPLEX_DECODER_BYTES: String(DEFAULT_SANDBOX_DUPLEX_DECODER_MAX_BYTES),
};
const command = buildDuplexGatewayLaunchArgv({
shellCommand,
@@ -3080,7 +3331,14 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: {
}
if (channel) {
- const gate = createDuplexReadinessGate(channel, { nonce, timeoutMs: readinessTimeoutMs });
+ const gate = createDuplexReadinessGate(channel, {
+ nonce,
+ timeoutMs: readinessTimeoutMs,
+ // Inject the one host-process aggregate byte ledger, the same object the
+ // broker and the response-body reader hold. The gate charges the untrusted
+ // pre-READY buffer against it, so the aggregate identity holds at this seam.
+ ledger: duplexAggregateByteLedger,
+ });
const readiness = await gate.ready;
if (!readiness.ok) {
// Fail closed. Close the partial channel inside a bounded budget, then
@@ -3088,12 +3346,26 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: {
// carries the bridge token reached the channel or any endpoint. The reason
// is a fixed enum, so it rides the log line and the fallback telemetry
// with no raw value.
+ gate.disposePendingReplay();
await closeDuplexChannelWithinBudget(channel, DEFAULT_DUPLEX_CLEANUP_BUDGET_MS);
duplexChannelOpen.fallback(duplexReadinessFallbackReason(readiness.reason));
await onLog(
"stderr",
`[paperclip] Sandbox duplex readiness failed (${readiness.reason}). Using the file bridge.\n`,
);
+ } else if (gate.replayOverflowed()) {
+ // Readiness passed, but a post-READY pre-bind chunk passed the aggregate
+ // byte ceiling. The gate dropped the replay buffer and stopped the channel.
+ // Release any held replay reservation, close the partial channel inside the
+ // cleanup budget, and select the file bridge with the aggregate marker. The
+ // broker never bound, so no request reached the channel or any endpoint.
+ gate.disposePendingReplay();
+ await closeDuplexChannelWithinBudget(channel, DEFAULT_DUPLEX_CLEANUP_BUDGET_MS);
+ duplexChannelOpen.fallback(duplexReadinessFallbackReason("aggregate_bytes_exceeded"));
+ await onLog(
+ "stderr",
+ "[paperclip] Sandbox duplex readiness replay exceeded the aggregate byte ceiling (aggregate_bytes_exceeded). Using the file bridge.\n",
+ );
} else {
// Readiness passed. Construct the broker inside the guarded region, so a
// construction throw closes the channel within the cleanup budget and
@@ -3149,12 +3421,20 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: {
`[paperclip] Sandbox duplex channel lost (${typedDuplexLossReason(record.reason)}). The run fails.\n`,
);
},
+ // Inject the one host-process aggregate byte ledger. The broker
+ // reserves the retained request-frame, request-payload, and no-replay
+ // set-entry bytes against it, so the aggregate retained bytes across all
+ // live routes stay under the ceiling. It is the same object the
+ // response-body reader reads off the stamped target above.
+ duplexAggregateByteLedger,
});
} catch {
- // The broker construction failed, so no broker owns the channel. Close
- // the channel within the cleanup budget, then select the file bridge.
- // The log line names no raw error, so no raw error rides a log line on
- // the duplex path.
+ // The broker construction failed, so no broker owns the channel. The
+ // broker never bound, so it never released the pending replay reservation;
+ // release it here. Then close the channel within the cleanup budget and
+ // select the file bridge. The log line names no raw error, so no raw error
+ // rides a log line on the duplex path.
+ gate.disposePendingReplay();
await closeDuplexChannelWithinBudget(channel, DEFAULT_DUPLEX_CLEANUP_BUDGET_MS);
duplexChannelOpen.fallback("broker_construction_failed");
await onLog(
diff --git a/packages/adapter-utils/src/sandbox-callback-bridge.ts b/packages/adapter-utils/src/sandbox-callback-bridge.ts
index 7c81cfde26..9c3d073919 100644
--- a/packages/adapter-utils/src/sandbox-callback-bridge.ts
+++ b/packages/adapter-utils/src/sandbox-callback-bridge.ts
@@ -19,6 +19,14 @@ const DEFAULT_BRIDGE_RESPONSE_TIMEOUT_MS = 30_000;
const DEFAULT_BRIDGE_STOP_TIMEOUT_MS = 2_000;
const DEFAULT_BRIDGE_MAX_QUEUE_DEPTH = 64;
const DEFAULT_BRIDGE_MAX_BODY_BYTES = 256 * 1024;
+// The default cap on the aggregate raw bytes the in-sandbox duplex frame decoder
+// retains between chunks. The generated gateway runs in a separate operating-system
+// process, so it cannot share the host aggregate byte ledger. It enforces this
+// separate cap locally under the `sandbox_process` scope, and the provider memory
+// allocation bounds it. The host passes the value through
+// PAPERCLIP_BRIDGE_MAX_DUPLEX_DECODER_BYTES. The default is well above one maximum
+// frame, so a legitimate single frame never trips it.
+const DEFAULT_BRIDGE_MAX_DUPLEX_DECODER_BYTES = 8 * 1024 * 1024;
// Per-iteration timeout for one poll-loop client call. A healthy control-plane
// round trip finishes in well under one second, so 10s is far above a normal
// iteration and never false-fires on a slow-but-live call. It is also well
@@ -89,6 +97,18 @@ const CALLBACK_BRIDGE_WORKER_FAILED_SPAN = "sandbox.callbackBridge.workerFailed"
export const DEFAULT_SANDBOX_CALLBACK_BRIDGE_MAX_BODY_BYTES = DEFAULT_BRIDGE_MAX_BODY_BYTES;
+/**
+ * The default cap on the aggregate raw bytes the generated in-sandbox duplex frame
+ * decoder retains. The host passes it through
+ * `PAPERCLIP_BRIDGE_MAX_DUPLEX_DECODER_BYTES`. The in-sandbox decoder enforces the
+ * cap locally under the `sandbox_process` scope; it never touches the host
+ * aggregate byte ledger.
+ */
+export const DEFAULT_SANDBOX_DUPLEX_DECODER_MAX_BYTES = DEFAULT_BRIDGE_MAX_DUPLEX_DECODER_BYTES;
+
+/** The scope label the in-sandbox duplex decoder reports for its local byte counter. */
+export const SANDBOX_DUPLEX_DECODER_SCOPE = "sandbox_process";
+
export interface SandboxCallbackBridgeRouteRule {
method: string;
path: RegExp;
@@ -1746,6 +1766,8 @@ export async function startSandboxCallbackBridgeServer(input: {
const DUPLEX_GATEWAY_CODEC_SOURCE = `const DUPLEX_FRAME_VERSION = 1;
const DEFAULT_MAX_DUPLEX_FRAME_BYTES = 1000000;
const DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES = 256;
+const DEFAULT_MAX_DUPLEX_DECODER_BYTES = ${DEFAULT_BRIDGE_MAX_DUPLEX_DECODER_BYTES};
+const DUPLEX_DECODER_SCOPE = "${SANDBOX_DUPLEX_DECODER_SCOPE}";
const DUPLEX_NEWLINE_BYTE = 0x0a;
const DUPLEX_EMPTY = Buffer.alloc(0);
const DUPLEX_RESPONSE_OUTCOMES = new Set(["completed", "indeterminate", "unavailable"]);
@@ -1881,10 +1903,32 @@ class DuplexFrameDecoder {
this.discarding = false;
this.maxFrameBytes =
options && options.maxFrameBytes != null ? options.maxFrameBytes : DEFAULT_MAX_DUPLEX_FRAME_BYTES;
+ // The in-sandbox decoder runs in a separate operating-system process, so it
+ // cannot share the host aggregate byte ledger. It bounds its own retained
+ // bytes with a local cap under the "sandbox_process" scope. The counters here
+ // never increment or release a host aggregate token.
+ this.scope = DUPLEX_DECODER_SCOPE;
+ this.maxAggregateBytes =
+ options && options.maxAggregateBytes != null
+ ? options.maxAggregateBytes
+ : DEFAULT_MAX_DUPLEX_DECODER_BYTES;
+ this.bytesInUse = 0;
+ this.aggregateRejections = 0;
}
push(chunk) {
const incoming = typeof chunk === "string" ? Buffer.from(chunk, "utf8") : Buffer.from(chunk);
+ // Enforce the local sandbox-process cap before the concat allocates the peak
+ // "old + incoming" buffer. A rejection fails closed: the decoder drops the
+ // retained buffer and the incoming chunk, resynchronizes at the next newline,
+ // and reports the aggregate rejection. It retains nothing over the cap.
+ if (this.buffer.length + incoming.length > this.maxAggregateBytes) {
+ this.buffer = DUPLEX_EMPTY;
+ this.discarding = true;
+ this.bytesInUse = 0;
+ this.aggregateRejections += 1;
+ return [duplexFail("aggregate_bytes_exceeded", "aggregate retained bytes exceeded the sandbox-process cap")];
+ }
this.buffer = this.buffer.length === 0 ? incoming : Buffer.concat([this.buffer, incoming]);
const results = [];
for (;;) {
@@ -1916,6 +1960,8 @@ class DuplexFrameDecoder {
}
results.push(decodeDuplexLine(line));
}
+ // Reconcile the local sandbox-process counter to the bytes still retained.
+ this.bytesInUse = this.buffer.length;
return results;
}
}`;
@@ -1959,6 +2005,12 @@ const responseTimeoutMs = Number(
);
const maxQueueDepth = Number(process.env.PAPERCLIP_BRIDGE_MAX_QUEUE_DEPTH || "${DEFAULT_BRIDGE_MAX_QUEUE_DEPTH}");
const maxBodyBytes = Number(process.env.PAPERCLIP_BRIDGE_MAX_BODY_BYTES || "${DEFAULT_BRIDGE_MAX_BODY_BYTES}");
+// The host passes the separate sandbox-process raw-decoder cap here. The in-sandbox
+// decoder enforces it locally under the "sandbox_process" scope; it never shares the
+// host aggregate byte ledger.
+const maxDuplexDecoderBytes = Number(
+ process.env.PAPERCLIP_BRIDGE_MAX_DUPLEX_DECODER_BYTES || "${DEFAULT_BRIDGE_MAX_DUPLEX_DECODER_BYTES}",
+);
const heartbeatIntervalMs = Number(
process.env.PAPERCLIP_BRIDGE_HEARTBEAT_INTERVAL_MS || "${DEFAULT_DUPLEX_GATEWAY_HEARTBEAT_INTERVAL_MS}",
);
@@ -2220,7 +2272,7 @@ function runDuplexGateway() {
// local action here.
}
- const decoder = new DuplexFrameDecoder();
+ const decoder = new DuplexFrameDecoder({ maxAggregateBytes: maxDuplexDecoderBytes });
process.stdin.on("data", (chunk) => {
lastInboundAt = Date.now();
for (const result of decoder.push(chunk)) {
diff --git a/packages/shared/src/telemetry/README.md b/packages/shared/src/telemetry/README.md
index bba168e5cb..9ff1448863 100644
--- a/packages/shared/src/telemetry/README.md
+++ b/packages/shared/src/telemetry/README.md
@@ -350,6 +350,24 @@ field.
| `sandbox_duplex_loss_total` | One terminal duplex channel loss. The `loss_class` dimension records the phase. |
| `sandbox_duplex_session_leak_total` | One leaked provider session at teardown. |
+### Aggregate byte ledger metrics
+
+The host aggregate byte ledger owns one process-scoped gauge and two
+process-scoped counters. The ledger bounds the retained bytes across every live
+duplex route in one process. It sets the gauge on each reserve and each release.
+It increments a counter on a rejected reservation and on an accounting defect.
+These records carry no dimension label. The guarded counter store keys each
+counter on `(companyId, metric)`, and the gauge reports one process value, so no
+dynamic dimension rides them. The code owner is
+`packages/adapter-utils/src/duplex-aggregate-byte-ledger.ts`, and the metric
+names are literal constants in `duplex-telemetry.ts`.
+
+| Metric | Type | Scope |
+| --- | --- | --- |
+| `sandbox_duplex_aggregate_bytes_in_use` | gauge | The aggregate retained bytes across every live duplex route. The ledger sets it on each reserve and each release. |
+| `sandbox_duplex_aggregate_byte_reservation_rejections_total` | counter | One rejected aggregate byte reservation. The ledger increments it when a reservation would pass the aggregate ceiling. |
+| `sandbox_duplex_aggregate_byte_accounting_underflow_total` | counter | One aggregate byte accounting defect. The ledger increments it on a double release or on a transfer of a token it does not hold. |
+
### Dimension keys
Counters carry no dimension labels. The guarded counter store keys each counter
diff --git a/server/src/__tests__/duplex-aggregate-ceiling-env.test.ts b/server/src/__tests__/duplex-aggregate-ceiling-env.test.ts
new file mode 100644
index 0000000000..a6f1a50348
--- /dev/null
+++ b/server/src/__tests__/duplex-aggregate-ceiling-env.test.ts
@@ -0,0 +1,70 @@
+import { describe, expect, it, vi } from "vitest";
+import { DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES } from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger";
+import { resolveDuplexAggregateCeilingBytesFromEnv } from "../duplex-aggregate-ceiling-env.js";
+
+describe("resolveDuplexAggregateCeilingBytesFromEnv", () => {
+ it("uses the safe default and reports nothing when the variable is absent", () => {
+ const onRejectedOverride = vi.fn();
+
+ const resolved = resolveDuplexAggregateCeilingBytesFromEnv(undefined, onRejectedOverride);
+
+ expect(resolved).toBe(DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES);
+ expect(onRejectedOverride).not.toHaveBeenCalled();
+ });
+
+ it("treats a whitespace-only value as a present invalid override, not as absent", () => {
+ const rawOverride = " ";
+ const onRejectedOverride = vi.fn();
+
+ // The resolver must continue with the safe default. Startup does not fail.
+ const resolved = resolveDuplexAggregateCeilingBytesFromEnv(rawOverride, onRejectedOverride);
+
+ expect(resolved).toBe(DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES);
+ // The rejection reporter fires one time. A blank value is now visible as
+ // invalid instead of silent as absent.
+ expect(onRejectedOverride).toHaveBeenCalledTimes(1);
+ // The reporter receives only the parsed number. The raw whitespace string
+ // never reaches the reporter, so no operator-supplied text can reach a log
+ // line.
+ const reportedValue = onRejectedOverride.mock.calls[0]?.[0];
+ expect(typeof reportedValue).toBe("number");
+ expect(reportedValue).not.toBe(rawOverride);
+ expect(Number.isFinite(reportedValue)).toBe(true);
+ });
+
+ it("treats an empty-string value as a present invalid override", () => {
+ const onRejectedOverride = vi.fn();
+
+ const resolved = resolveDuplexAggregateCeilingBytesFromEnv("", onRejectedOverride);
+
+ expect(resolved).toBe(DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES);
+ expect(onRejectedOverride).toHaveBeenCalledTimes(1);
+ });
+
+ it("rejects a present non-numeric value and uses the safe default", () => {
+ const onRejectedOverride = vi.fn();
+
+ const resolved = resolveDuplexAggregateCeilingBytesFromEnv("not-a-number", onRejectedOverride);
+
+ expect(resolved).toBe(DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES);
+ expect(onRejectedOverride).toHaveBeenCalledTimes(1);
+ });
+
+ it("passes a valid override through unchanged", () => {
+ const onRejectedOverride = vi.fn();
+
+ const resolved = resolveDuplexAggregateCeilingBytesFromEnv("1048576", onRejectedOverride);
+
+ expect(resolved).toBe(1048576);
+ expect(onRejectedOverride).not.toHaveBeenCalled();
+ });
+
+ it("accepts a valid override that carries surrounding whitespace", () => {
+ const onRejectedOverride = vi.fn();
+
+ const resolved = resolveDuplexAggregateCeilingBytesFromEnv(" 1048576 ", onRejectedOverride);
+
+ expect(resolved).toBe(1048576);
+ expect(onRejectedOverride).not.toHaveBeenCalled();
+ });
+});
diff --git a/server/src/__tests__/environment-execution-target-duplex.test.ts b/server/src/__tests__/environment-execution-target-duplex.test.ts
index 7af64eb431..663653c699 100644
--- a/server/src/__tests__/environment-execution-target-duplex.test.ts
+++ b/server/src/__tests__/environment-execution-target-duplex.test.ts
@@ -21,6 +21,11 @@ vi.mock("../services/plugin-environment-driver.js", async (importActual) => ({
}));
import type { EffectiveSandboxCapabilities } from "@paperclipai/adapter-utils/execution-target";
+import { adapterExecutionTargetDuplexAggregateByteLedger } from "@paperclipai/adapter-utils/execution-target";
+import {
+ DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES,
+ DuplexAggregateByteLedger,
+} from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger";
import { createSshCommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/ssh";
import type { Environment, EnvironmentLease } from "@paperclipai/shared";
import { resolveEnvironmentExecutionTarget } from "../services/environment-execution-target.js";
@@ -293,6 +298,50 @@ describe("EnvironmentRuntimeService.openDuplexChannel capability gate", () => {
});
});
+describe("sandbox execution target aggregate byte ledger stamp", () => {
+ it("stamps the injected ledger on the target with object identity", async () => {
+ const ledger = new DuplexAggregateByteLedger({
+ ceilingBytes: DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES,
+ });
+ const target = await resolveEnvironmentExecutionTarget({
+ db: {} as never,
+ companyId: "company-1",
+ adapterType: "codex_local",
+ environment: { id: "env-1", driver: "sandbox", config: { provider: "daytona" } },
+ leaseId: "lease-1",
+ leaseMetadata: { remoteCwd: "/work" },
+ lease: { id: "lease-1", leasePolicy: "reuse_by_environment" } as never,
+ duplexAggregateByteLedger: ledger,
+ });
+ if (!target || target.kind !== "remote" || target.transport !== "sandbox") {
+ throw new Error("expected a sandbox execution target");
+ }
+ // The stamped field and the accessor both return the one injected object, so
+ // one process-owned ledger reaches the host bridge seam.
+ expect(target.duplexAggregateByteLedger).toBe(ledger);
+ expect(adapterExecutionTargetDuplexAggregateByteLedger(target)).toBe(ledger);
+ });
+
+ it("leaves the ledger absent when the host injects none", async () => {
+ const target = await resolveEnvironmentExecutionTarget({
+ db: {} as never,
+ companyId: "company-1",
+ adapterType: "codex_local",
+ environment: { id: "env-1", driver: "sandbox", config: { provider: "daytona" } },
+ leaseId: "lease-1",
+ leaseMetadata: { remoteCwd: "/work" },
+ lease: { id: "lease-1", leasePolicy: "reuse_by_environment" } as never,
+ });
+ if (!target || target.kind !== "remote" || target.transport !== "sandbox") {
+ throw new Error("expected a sandbox execution target");
+ }
+ // A run with no injected ledger keeps the seam null; the accessor never makes
+ // a fresh ledger.
+ expect(target.duplexAggregateByteLedger).toBeNull();
+ expect(adapterExecutionTargetDuplexAggregateByteLedger(target)).toBeNull();
+ });
+});
+
// Build a sandbox execution target with a fixed capability snapshot and a fake
// environment runtime whose openDuplexChannel is a spy. The helper returns the
// runner and the spy so a test reads the capability-gated member.
diff --git a/server/src/__tests__/fixtures/plugin-worker-duplex-channel.cjs b/server/src/__tests__/fixtures/plugin-worker-duplex-channel.cjs
index 013ecb0362..69ac9e7e45 100644
--- a/server/src/__tests__/fixtures/plugin-worker-duplex-channel.cjs
+++ b/server/src/__tests__/fixtures/plugin-worker-duplex-channel.cjs
@@ -23,6 +23,18 @@
// and never lets the exit crowd a data frame out of the pre-open hold.
// - `echoInput`: when true, the fixture echoes each `duplexChannelWrite` back as
// one data notification for the bound session.
+// - `writeReplyDelayMs`: when a positive number, the fixture delays each
+// `duplexChannelWrite` reply by that many milliseconds, so a test proves the
+// host holds the pending-write reservation until the RPC settles.
+// - `stopReadingStdinAfterOpen`: when true, the fixture stops reading its stdin
+// right after it sends the open reply. The host writes then stay in the host
+// stdin write buffer, so a test proves the host meters the transport buffer and
+// never releases the transport token on the write-RPC timeout.
+// - `exitAfterStopMs`: when a positive number, and the fixture stopped reading
+// stdin, the fixture exits after that many milliseconds. The worker exit
+// discards the stdin buffer, so a test proves the host releases every held
+// transport token on the worker exit. It gives a fast exit for a worker that no
+// longer reads its stdin and never sees the shutdown request.
// - `closeMode`: "ack" | "bad-ack" | "no-ack" (default "ack"). It controls the
// close reply, so a test proves the host retires the worker on an unconfirmed
// close.
@@ -134,6 +146,8 @@ rl.on("line", (line) => {
closeMode,
echoInput: directive.echoInput === true,
noWriteReply: mode === "no-write-reply",
+ writeReplyDelayMs:
+ typeof directive.writeReplyDelayMs === "number" ? directive.writeReplyDelayMs : 0,
emitAfterCloseChunk:
typeof directive.emitAfterCloseChunk === "string" ? directive.emitAfterCloseChunk : null,
});
@@ -170,6 +184,20 @@ rl.on("line", (line) => {
reply();
}
+ if (directive.stopReadingStdinAfterOpen === true) {
+ // Stop reading stdin, so every later host write stays in the host stdin
+ // write buffer. The host meters the transport buffer and holds the transport
+ // token until the stream flush, the stream error, or the worker exit.
+ rl.pause();
+ process.stdin.pause();
+ if (typeof directive.exitAfterStopMs === "number" && directive.exitAfterStopMs >= 0) {
+ // Exit after the delay, so a test observes the worker exit release the held
+ // transport tokens without a slow shutdown drain on an unread stdin.
+ setTimeout(() => process.exit(0), directive.exitAfterStopMs);
+ }
+ return;
+ }
+
// Emit the scripted data and the exit after the open reply, so the host
// binds the route first. Each frame echoes the exact pair; a test overrides
// `sid` or `rid` to force a mismatch.
@@ -205,7 +233,14 @@ rl.on("line", (line) => {
},
});
}
- send({ jsonrpc: "2.0", id: message.id, result: null });
+ const replyWrite = () => send({ jsonrpc: "2.0", id: message.id, result: null });
+ if (entry.writeReplyDelayMs > 0) {
+ // Delay the write reply, so the host holds the pending-write reservation for
+ // a measurable time before the RPC settles.
+ setTimeout(replyWrite, entry.writeReplyDelayMs);
+ } else {
+ replyWrite();
+ }
return;
}
diff --git a/server/src/__tests__/plugin-worker-manager-duplex-byte-ledger.test.ts b/server/src/__tests__/plugin-worker-manager-duplex-byte-ledger.test.ts
new file mode 100644
index 0000000000..2b2215891a
--- /dev/null
+++ b/server/src/__tests__/plugin-worker-manager-duplex-byte-ledger.test.ts
@@ -0,0 +1,348 @@
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { describe, expect, it, vi } from "vitest";
+import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
+import {
+ DuplexAggregateByteLedger,
+ type DuplexAggregateByteLedgerTelemetry,
+} from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger";
+import {
+ createDuplexRouteSlotController,
+ createPluginWorkerHandle,
+} from "../services/plugin-worker-manager.js";
+
+// This suite proves the plugin worker manager charges every retained duplex route
+// representation against the injected aggregate byte ledger, and releases each
+// token exactly once through the one cleanup path. The tests drive a real worker
+// fixture process, so they exercise the true frame order across the bind boundary.
+
+const FIXTURES_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures");
+const DUPLEX_CHANNEL_WORKER_ENTRYPOINT = path.join(
+ FIXTURES_DIR,
+ "plugin-worker-duplex-channel.cjs",
+);
+
+const TEST_MANIFEST: PaperclipPluginManifestV1 = {
+ id: "test.plugin",
+ apiVersion: 1,
+ version: "1.0.0",
+ displayName: "Test plugin",
+ description: "Test plugin",
+ author: "Paperclip",
+ categories: ["automation"],
+ capabilities: [],
+ entrypoints: { worker: "dist/worker.js" },
+};
+
+function makeDuplexHandle(extra?: Record<string, unknown>) {
+ return createPluginWorkerHandle("test.plugin", {
+ entrypointPath: DUPLEX_CHANNEL_WORKER_ENTRYPOINT,
+ manifest: TEST_MANIFEST,
+ config: {},
+ instanceInfo: { instanceId: "instance-1", hostVersion: "1.0.0" },
+ apiVersion: 1,
+ hostHandlers: {},
+ ...extra,
+ });
+}
+
+// The test directive rides in `providerLeaseId`, an opaque field the manager
+// forwards to the worker unchanged.
+function duplexOpenInput(directive: unknown, companyId = "company-1") {
+ return {
+ driverKey: "daytona",
+ companyId,
+ environmentId: "env-1",
+ providerLeaseId: JSON.stringify(directive),
+ command: "bridge-callback",
+ };
+}
+
+// A counting telemetry surface. It records how many times the ledger rejected a
+// reservation and how many accounting defects it saw, so a test asserts a
+// fail-closed rejection and proves the cleanup never double-releases.
+function countingTelemetry(): DuplexAggregateByteLedgerTelemetry & {
+ rejections: number;
+ underflows: number;
+} {
+ const state = {
+ rejections: 0,
+ underflows: 0,
+ setBytesInUse() {},
+ recordReservationRejection() {
+ state.rejections += 1;
+ },
+ recordAccountingUnderflow() {
+ state.underflows += 1;
+ },
+ };
+ return state;
+}
+
+describe("plugin worker manager duplex aggregate byte ledger", () => {
+ it("keeps terminalized buffered bytes charged until a late listener drains them, then a worker exit is harmless", async () => {
+ const telemetry = countingTelemetry();
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1 << 20, telemetry });
+ // Lower the per-chunk char bound, so the second chunk ends the route while the
+ // first chunk stays buffered.
+ const handle = makeDuplexHandle({
+ duplexAggregateByteLedger: ledger,
+ duplexChannelLimits: { maxChunkChars: 4 },
+ });
+ try {
+ await handle.start();
+ const session = await handle.openDuplexChannel(
+ // No listener attaches, so "ok" buffers and charges its two raw bytes. The
+ // "toolong" chunk passes the per-chunk bound, so the route terminalizes and
+ // moves the still-buffered "ok" token to the terminal registry.
+ duplexOpenInput({ data: [{ chunk: "ok" }, { chunk: "toolong" }] }),
+ );
+ // The buffered "ok" stays charged after the route leaves the live map.
+ await vi.waitFor(() => {
+ expect(ledger.bytesInUse).toBe(2);
+ expect(ledger.liveTokenCount).toBe(1);
+ });
+ // A late listener drains the terminal buffered record and releases its token.
+ const chunks: string[] = [];
+ session.onData((chunk) => chunks.push(chunk));
+ await vi.waitFor(() => {
+ expect(chunks).toEqual(["ok"]);
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ });
+ // The over-bound chunk retained nothing, so no reservation ever rejected.
+ expect(telemetry.rejections).toBe(0);
+ } finally {
+ await handle.stop().catch(() => undefined);
+ }
+ // The worker exit sweep runs on stop. The ledger stays at zero, and the sweep
+ // records no accounting defect, so the drain and the exit never double-release.
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ expect(telemetry.underflows).toBe(0);
+ });
+
+ it("transfers pre-bind held bytes to the buffered representation across the bind, then releases them on drain", async () => {
+ const telemetry = countingTelemetry();
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1 << 20, telemetry });
+ const handle = makeDuplexHandle({ duplexAggregateByteLedger: ledger });
+ try {
+ await handle.start();
+ const session = await handle.openDuplexChannel(
+ // The worker writes the open reply and the two data frames in one stdout
+ // write, so both frames arrive before the route binds. The host holds them
+ // as pre-bind events, then the bind replays and transfers each token to the
+ // buffered representation.
+ duplexOpenInput({ batchWithOpenReply: true, data: [{ chunk: "aa" }, { chunk: "bb" }] }),
+ );
+ // The two held events keep their exact reserved bytes across the transfer, so
+ // the live-token count never grows and the gauge never re-admits.
+ await vi.waitFor(() => {
+ expect(ledger.bytesInUse).toBe(4);
+ expect(ledger.liveTokenCount).toBe(2);
+ });
+ const chunks: string[] = [];
+ session.onData((chunk) => chunks.push(chunk));
+ await vi.waitFor(() => {
+ expect(chunks).toEqual(["aa", "bb"]);
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ });
+ } finally {
+ await handle.stop().catch(() => undefined);
+ }
+ expect(ledger.bytesInUse).toBe(0);
+ expect(telemetry.underflows).toBe(0);
+ });
+
+ it("fails closed and retains nothing when a buffered reservation would pass the ceiling", async () => {
+ const telemetry = countingTelemetry();
+ // A four-byte ceiling. One five-byte chunk cannot fit.
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 4, telemetry });
+ const handle = makeDuplexHandle({ duplexAggregateByteLedger: ledger });
+ try {
+ await handle.start();
+ await handle.openDuplexChannel(
+ // No listener attaches, so "hello" tries to buffer. Its five raw bytes pass
+ // the four-byte ceiling, so the reservation rejects and the route ends.
+ duplexOpenInput({ data: [{ chunk: "hello" }] }),
+ );
+ await vi.waitFor(() => {
+ expect(telemetry.rejections).toBeGreaterThanOrEqual(1);
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ });
+ } finally {
+ await handle.stop().catch(() => undefined);
+ }
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ expect(telemetry.underflows).toBe(0);
+ });
+});
+
+// A telemetry surface that also tracks the peak gauge value. A test asserts the
+// peak never passes the ceiling, so it proves aggregate admission stops at the
+// ceiling instead of overshooting it.
+function peakTrackingTelemetry(): DuplexAggregateByteLedgerTelemetry & {
+ gauge: number;
+ peak: number;
+ rejections: number;
+ underflows: number;
+} {
+ const state = {
+ gauge: 0,
+ peak: 0,
+ rejections: 0,
+ underflows: 0,
+ setBytesInUse(bytes: number) {
+ state.gauge = bytes;
+ if (bytes > state.peak) state.peak = bytes;
+ },
+ recordReservationRejection() {
+ state.rejections += 1;
+ },
+ recordAccountingUnderflow() {
+ state.underflows += 1;
+ },
+ };
+ return state;
+}
+
+// The exact raw byte count of one buffered chunk each route retains.
+const LOAD_CHUNK_BYTES = 8;
+const LOAD_CHUNK = "x".repeat(LOAD_CHUNK_BYTES);
+
+describe("plugin worker manager duplex aggregate byte ledger load across workers", () => {
+ it("stops aggregate byte admission at the ceiling across many routes and workers while the route-count controller stays open", async () => {
+ const telemetry = peakTrackingTelemetry();
+ // The ceiling holds exactly six eight-byte buffered chunks.
+ const fittingRoutes = 6;
+ const ceilingBytes = fittingRoutes * LOAD_CHUNK_BYTES;
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes, telemetry });
+ // The route-count controller has ample room, so the byte ledger binds first.
+ const routeSlots = createDuplexRouteSlotController(100);
+ const handleCount = 3;
+ const routesPerHandle = 3;
+ const totalRoutes = handleCount * routesPerHandle;
+ const rejectedRoutes = totalRoutes - fittingRoutes;
+ const handles: Array<ReturnType<typeof makeDuplexHandle>> = [];
+ try {
+ for (let h = 0; h < handleCount; h += 1) {
+ const handle = makeDuplexHandle({
+ duplexAggregateByteLedger: ledger,
+ duplexRouteSlots: routeSlots,
+ });
+ await handle.start();
+ handles.push(handle);
+ }
+ // Open every route in order. Each worker batches one data chunk with the open
+ // reply, so the host holds the chunk as a pre-bind event and reserves its exact
+ // raw bytes before the route binds. No listener attaches, so a bound route holds
+ // the chunk. The first six chunks fit the ceiling. Each later reservation passes
+ // the ceiling, so the ledger rejects it and the route fails closed with the open
+ // marker (not the route-busy marker), because the route-count controller has room.
+ let opened = 0;
+ let failedClosed = 0;
+ for (let h = 0; h < handleCount; h += 1) {
+ for (let r = 0; r < routesPerHandle; r += 1) {
+ try {
+ await handles[h].openDuplexChannel(
+ duplexOpenInput({
+ workerSessionId: `ws-${h}-${r}`,
+ batchWithOpenReply: true,
+ data: [{ chunk: LOAD_CHUNK }],
+ }),
+ );
+ opened += 1;
+ } catch (err) {
+ // A byte-ceiling rejection fails the route closed with the open marker.
+ // The route-count controller had room, so this is never the route-busy
+ // marker.
+ expect(String(err)).toContain("DUPLEX_CHANNEL_OPEN_FAILED");
+ expect(String(err)).not.toContain("DUPLEX_CHANNEL_ROUTE_BUSY");
+ failedClosed += 1;
+ }
+ }
+ }
+ // Six routes bound and hold their bytes; three failed closed at the ceiling.
+ expect(opened).toBe(fittingRoutes);
+ expect(failedClosed).toBe(rejectedRoutes);
+ expect(opened + failedClosed).toBe(totalRoutes);
+ // The aggregate gauge holds at the ceiling and the ledger recorded one rejection
+ // per over-ceiling route.
+ await vi.waitFor(() => {
+ expect(telemetry.rejections).toBe(rejectedRoutes);
+ expect(ledger.bytesInUse).toBe(ceilingBytes);
+ expect(ledger.liveTokenCount).toBe(fittingRoutes);
+ });
+ // The gauge never passed the ceiling, so admission stopped at the ceiling and
+ // never overshot it.
+ expect(telemetry.peak).toBe(ceilingBytes);
+ } finally {
+ for (const handle of handles) await handle.stop().catch(() => undefined);
+ }
+ // Every worker exit sweep released its buffered tokens once, so the ledger ends
+ // at zero with no accounting defect.
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ expect(telemetry.underflows).toBe(0);
+ });
+
+ it("still bounds the route count for fixed per-route overhead when the byte ceiling has ample room", async () => {
+ const telemetry = peakTrackingTelemetry();
+ // A large byte ceiling, so retained bytes never bind admission.
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1 << 20, telemetry });
+ // The route-count controller holds two slots for the whole process.
+ const routeSlots = createDuplexRouteSlotController(2);
+ const handleA = makeDuplexHandle({
+ duplexAggregateByteLedger: ledger,
+ duplexRouteSlots: routeSlots,
+ });
+ const handleB = makeDuplexHandle({
+ duplexAggregateByteLedger: ledger,
+ duplexRouteSlots: routeSlots,
+ });
+ let opened = 0;
+ let busy = 0;
+ try {
+ await handleA.start();
+ await handleB.start();
+ // Attempt four routes across the two workers. The shared controller admits the
+ // first two and rejects the last two with the fixed route-busy error.
+ const attempts: Array<[ReturnType<typeof makeDuplexHandle>, string]> = [
+ [handleA, "ws-a1"],
+ [handleA, "ws-a2"],
+ [handleB, "ws-b1"],
+ [handleB, "ws-b2"],
+ ];
+ for (const [handle, ws] of attempts) {
+ try {
+ await handle.openDuplexChannel(
+ duplexOpenInput({ workerSessionId: ws, data: [{ chunk: "aa" }] }),
+ );
+ opened += 1;
+ } catch (err) {
+ expect(String(err)).toContain("DUPLEX_CHANNEL_ROUTE_BUSY");
+ busy += 1;
+ }
+ }
+ // The route-count controller admitted two routes and rejected the rest, so it
+ // still bounds the fixed per-route overhead on its own.
+ expect(opened).toBe(2);
+ expect(busy).toBe(2);
+ // The two open routes buffered four raw bytes, far under the byte ceiling, and
+ // the byte ledger rejected nothing.
+ await vi.waitFor(() => {
+ expect(ledger.bytesInUse).toBe(4);
+ expect(ledger.liveTokenCount).toBe(2);
+ });
+ expect(telemetry.rejections).toBe(0);
+ } finally {
+ await handleA.stop().catch(() => undefined);
+ await handleB.stop().catch(() => undefined);
+ }
+ expect(ledger.bytesInUse).toBe(0);
+ expect(telemetry.underflows).toBe(0);
+ });
+});
diff --git a/server/src/__tests__/plugin-worker-manager-duplex-pending-write-ledger.test.ts b/server/src/__tests__/plugin-worker-manager-duplex-pending-write-ledger.test.ts
new file mode 100644
index 0000000000..4c22219eba
--- /dev/null
+++ b/server/src/__tests__/plugin-worker-manager-duplex-pending-write-ledger.test.ts
@@ -0,0 +1,251 @@
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { describe, expect, it, vi } from "vitest";
+import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
+import {
+ DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED,
+ DuplexAggregateByteLedger,
+ type DuplexAggregateByteLedgerTelemetry,
+} from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger";
+
+// Mock the shared logger, so a test reads the fixed rejection marker the manager
+// logs when a pending-write reservation fails. The child logger returns the same
+// object, so `log.warn` is this mock's `warn`.
+vi.mock("../middleware/logger.js", () => {
+ const mockLogger: Record<string, unknown> = {
+ trace: vi.fn(),
+ debug: vi.fn(),
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ fatal: vi.fn(),
+ child: vi.fn(() => mockLogger),
+ };
+ return { logger: mockLogger, httpLogger: vi.fn() };
+});
+
+import { logger } from "../middleware/logger.js";
+import { createPluginWorkerHandle } from "../services/plugin-worker-manager.js";
+
+// This suite proves the plugin worker manager charges every host→worker write raw
+// payload against the injected aggregate byte ledger under the `pending_write`
+// owner, and releases each token one time when the write RPC settles. The child
+// reads its stdin here, so the separate `stdin_write` transport token flushes and
+// releases at once. Each test waits for that flush, so the assertions isolate the
+// raw-payload token. The transport token has its own suite.
+
+const FIXTURES_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures");
+const DUPLEX_CHANNEL_WORKER_ENTRYPOINT = path.join(
+ FIXTURES_DIR,
+ "plugin-worker-duplex-channel.cjs",
+);
+
+const TEST_MANIFEST: PaperclipPluginManifestV1 = {
+ id: "test.plugin",
+ apiVersion: 1,
+ version: "1.0.0",
+ displayName: "Test plugin",
+ description: "Test plugin",
+ author: "Paperclip",
+ categories: ["automation"],
+ capabilities: [],
+ entrypoints: { worker: "dist/worker.js" },
+};
+
+function makeDuplexHandle(extra?: Record<string, unknown>) {
+ return createPluginWorkerHandle("test.plugin", {
+ entrypointPath: DUPLEX_CHANNEL_WORKER_ENTRYPOINT,
+ manifest: TEST_MANIFEST,
+ config: {},
+ instanceInfo: { instanceId: "instance-1", hostVersion: "1.0.0" },
+ apiVersion: 1,
+ hostHandlers: {},
+ ...extra,
+ });
+}
+
+// The test directive rides in `providerLeaseId`, an opaque field the manager
+// forwards to the worker unchanged.
+function duplexOpenInput(directive: unknown, companyId = "company-1") {
+ return {
+ driverKey: "daytona",
+ companyId,
+ environmentId: "env-1",
+ providerLeaseId: JSON.stringify(directive),
+ command: "bridge-callback",
+ };
+}
+
+// A telemetry surface that tracks the peak gauge value and the two counters. A
+// test asserts the peak never passes the ceiling, so it proves aggregate admission
+// stops at the ceiling instead of overshooting it.
+function peakTrackingTelemetry(): DuplexAggregateByteLedgerTelemetry & {
+ gauge: number;
+ peak: number;
+ rejections: number;
+ underflows: number;
+} {
+ const state = {
+ gauge: 0,
+ peak: 0,
+ rejections: 0,
+ underflows: 0,
+ setBytesInUse(bytes: number) {
+ state.gauge = bytes;
+ if (bytes > state.peak) state.peak = bytes;
+ },
+ recordReservationRejection() {
+ state.rejections += 1;
+ },
+ recordAccountingUnderflow() {
+ state.underflows += 1;
+ },
+ };
+ return state;
+}
+
+// Return every reason string the manager logged through the mock `warn` sink.
+function loggedWarnReasons(): string[] {
+ const calls = (logger.warn as unknown as { mock: { calls: unknown[][] } }).mock.calls;
+ return calls
+ .map((call) => {
+ const first = call[0];
+ return first && typeof first === "object"
+ ? (first as { reason?: unknown }).reason
+ : undefined;
+ })
+ .filter((reason): reason is string => typeof reason === "string");
+}
+
+describe("plugin worker manager duplex pending-write byte ledger", () => {
+ it("charges each held raw payload and returns to zero after worker exit", async () => {
+ const telemetry = peakTrackingTelemetry();
+ // The ceiling has ample room, so a transient transport reservation never
+ // rejects. Each write holds its raw payload until the RPC settles.
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1 << 20, telemetry });
+ const handle = makeDuplexHandle({ duplexAggregateByteLedger: ledger });
+ const writeBytes = 1000;
+ const writeData = "x".repeat(writeBytes);
+ try {
+ await handle.start();
+ // The worker reads its stdin but never replies to a write, so each write RPC
+ // stays pending and holds its raw payload.
+ const route = await handle.openDuplexChannel(
+ duplexOpenInput({ workerSessionId: "ws-a", mode: "no-write-reply" }),
+ );
+ route.write(writeData);
+ route.write(writeData);
+ route.write(writeData);
+ // The worker reads its stdin, so each transport token flushes and releases.
+ // Only the three held raw payloads remain, so the gauge settles at three
+ // times the payload byte count with three live tokens.
+ await vi.waitFor(() => {
+ expect(ledger.liveTokenCount).toBe(3);
+ expect(ledger.bytesInUse).toBe(3 * writeBytes);
+ });
+ expect(telemetry.peak).toBeLessThanOrEqual(ledger.ceilingBytes);
+ } finally {
+ await handle.stop().catch(() => undefined);
+ }
+ // The worker exit settles each pending write, so each token releases one time
+ // and the ledger ends at zero with no accounting defect.
+ await vi.waitFor(() => {
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ });
+ expect(telemetry.underflows).toBe(0);
+ });
+
+ it("releases the raw-payload token after a delayed write RPC settles, with no worker exit", async () => {
+ const telemetry = peakTrackingTelemetry();
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1 << 20, telemetry });
+ const handle = makeDuplexHandle({ duplexAggregateByteLedger: ledger });
+ try {
+ await handle.start();
+ // The worker reads its stdin and delays its write reply, so the host holds the
+ // raw-payload reservation for a measurable time before the RPC settles.
+ const route = await handle.openDuplexChannel(
+ duplexOpenInput({ workerSessionId: "ws-d", writeReplyDelayMs: 150 }),
+ );
+ const writeBytes = 1000;
+ route.write("x".repeat(writeBytes));
+ // The transport token flushes at once, so one raw-payload token stays held for
+ // the full write byte count while the RPC is in flight.
+ await vi.waitFor(() => {
+ expect(ledger.liveTokenCount).toBe(1);
+ expect(ledger.bytesInUse).toBe(writeBytes);
+ });
+ // The delayed reply settles the RPC, and the `finally` releases the token.
+ await vi.waitFor(() => {
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ });
+ } finally {
+ await handle.stop().catch(() => undefined);
+ }
+ expect(ledger.bytesInUse).toBe(0);
+ expect(telemetry.underflows).toBe(0);
+ });
+
+ it("reserves the exact UTF-8 byte count of the raw payload, not the character length", async () => {
+ const telemetry = peakTrackingTelemetry();
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1 << 20, telemetry });
+ const handle = makeDuplexHandle({ duplexAggregateByteLedger: ledger });
+ try {
+ await handle.start();
+ const route = await handle.openDuplexChannel(
+ duplexOpenInput({ workerSessionId: "ws-u", mode: "no-write-reply" }),
+ );
+ // "a€" is two characters but four UTF-8 bytes (one plus three). The raw-payload
+ // reservation must charge four bytes, so it uses the UTF-8 byte count.
+ const data = "a€";
+ expect(data.length).toBe(2);
+ expect(Buffer.byteLength(data, "utf8")).toBe(4);
+ route.write(data);
+ // The transport token flushes, so one raw-payload token of four bytes remains.
+ await vi.waitFor(() => {
+ expect(ledger.liveTokenCount).toBe(1);
+ expect(ledger.bytesInUse).toBe(4);
+ });
+ } finally {
+ await handle.stop().catch(() => undefined);
+ }
+ await vi.waitFor(() => {
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ });
+ expect(telemetry.underflows).toBe(0);
+ });
+
+ it("ends an over-ceiling single write fail-closed with the aggregate marker, not the route-busy marker", async () => {
+ vi.mocked(logger.warn).mockClear();
+ const telemetry = peakTrackingTelemetry();
+ // The ceiling is smaller than one raw payload, so the raw-payload reservation
+ // fails before the frame reaches the transport. The manager retains nothing.
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 100, telemetry });
+ const handle = makeDuplexHandle({ duplexAggregateByteLedger: ledger });
+ try {
+ await handle.start();
+ const route = await handle.openDuplexChannel(
+ duplexOpenInput({ workerSessionId: "ws-o", mode: "no-write-reply" }),
+ );
+ // The single write is larger than the ceiling. The raw-payload reservation
+ // fails, so the manager ends the route with the aggregate marker, never the
+ // route-busy marker, and never writes the frame.
+ route.write("x".repeat(200));
+ expect(telemetry.rejections).toBeGreaterThanOrEqual(1);
+ expect(ledger.bytesInUse).toBe(0);
+ expect(telemetry.peak).toBeLessThanOrEqual(ledger.ceilingBytes);
+ const reasons = loggedWarnReasons();
+ expect(reasons).toContain(DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED);
+ expect(reasons).not.toContain("DUPLEX_CHANNEL_ROUTE_BUSY");
+ } finally {
+ await handle.stop().catch(() => undefined);
+ }
+ await vi.waitFor(() => {
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ });
+ expect(telemetry.underflows).toBe(0);
+ });
+});
diff --git a/server/src/__tests__/plugin-worker-manager-duplex-stdin-write-ledger.test.ts b/server/src/__tests__/plugin-worker-manager-duplex-stdin-write-ledger.test.ts
new file mode 100644
index 0000000000..27668aafd2
--- /dev/null
+++ b/server/src/__tests__/plugin-worker-manager-duplex-stdin-write-ledger.test.ts
@@ -0,0 +1,297 @@
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { describe, expect, it, vi } from "vitest";
+import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
+import {
+ DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED,
+ DuplexAggregateByteLedger,
+ type DuplexAggregateByteLedgerTelemetry,
+} from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger";
+
+// Mock the shared logger, so a test reads the fixed rejection marker the manager
+// logs when a transport reservation fails. The child logger returns the same
+// object, so `log.warn` is this mock's `warn`.
+vi.mock("../middleware/logger.js", () => {
+ const mockLogger: Record<string, unknown> = {
+ trace: vi.fn(),
+ debug: vi.fn(),
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ fatal: vi.fn(),
+ child: vi.fn(() => mockLogger),
+ };
+ return { logger: mockLogger, httpLogger: vi.fn() };
+});
+
+import { logger } from "../middleware/logger.js";
+import { createPluginWorkerHandle } from "../services/plugin-worker-manager.js";
+
+// This suite proves the plugin worker manager charges the child-stdin transport
+// buffer for every host→worker duplex write, under the `stdin_write` owner. The
+// worker completes the open, then stops reading its stdin. The host write then
+// stays in the host stdin write buffer. The tests prove the reservation covers the
+// serialized frame, holds until the flush or the worker exit, survives the write
+// RPC timeout and the route terminalization, and bounds the aggregate transport
+// bytes across several routes and workers.
+
+const FIXTURES_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures");
+const DUPLEX_CHANNEL_WORKER_ENTRYPOINT = path.join(
+ FIXTURES_DIR,
+ "plugin-worker-duplex-channel.cjs",
+);
+
+const TEST_MANIFEST: PaperclipPluginManifestV1 = {
+ id: "test.plugin",
+ apiVersion: 1,
+ version: "1.0.0",
+ displayName: "Test plugin",
+ description: "Test plugin",
+ author: "Paperclip",
+ categories: ["automation"],
+ capabilities: [],
+ entrypoints: { worker: "dist/worker.js" },
+};
+
+function makeDuplexHandle(extra?: Record<string, unknown>) {
+ return createPluginWorkerHandle("test.plugin", {
+ entrypointPath: DUPLEX_CHANNEL_WORKER_ENTRYPOINT,
+ manifest: TEST_MANIFEST,
+ config: {},
+ instanceInfo: { instanceId: "instance-1", hostVersion: "1.0.0" },
+ apiVersion: 1,
+ hostHandlers: {},
+ ...extra,
+ });
+}
+
+// The test directive rides in `providerLeaseId`, an opaque field the manager
+// forwards to the worker unchanged.
+function duplexOpenInput(directive: unknown, companyId = "company-1") {
+ return {
+ driverKey: "daytona",
+ companyId,
+ environmentId: "env-1",
+ providerLeaseId: JSON.stringify(directive),
+ command: "bridge-callback",
+ };
+}
+
+// A telemetry surface that tracks the peak gauge value and the two counters. A
+// test asserts the peak never passes the ceiling, so it proves aggregate admission
+// stops at the ceiling instead of overshooting it.
+function peakTrackingTelemetry(): DuplexAggregateByteLedgerTelemetry & {
+ gauge: number;
+ peak: number;
+ rejections: number;
+ underflows: number;
+} {
+ const state = {
+ gauge: 0,
+ peak: 0,
+ rejections: 0,
+ underflows: 0,
+ setBytesInUse(bytes: number) {
+ state.gauge = bytes;
+ if (bytes > state.peak) state.peak = bytes;
+ },
+ recordReservationRejection() {
+ state.rejections += 1;
+ },
+ recordAccountingUnderflow() {
+ state.underflows += 1;
+ },
+ };
+ return state;
+}
+
+// Return every reason string the manager logged through the mock `warn` sink.
+function loggedWarnReasons(): string[] {
+ const calls = (logger.warn as unknown as { mock: { calls: unknown[][] } }).mock.calls;
+ return calls
+ .map((call) => {
+ const first = call[0];
+ return first && typeof first === "object"
+ ? (first as { reason?: unknown }).reason
+ : undefined;
+ })
+ .filter((reason): reason is string => typeof reason === "string");
+}
+
+// A frame the host keeps in its stdin write buffer must pass the operating-system
+// pipe buffer (about 64 KiB), so the write callback stays pending while the child
+// does not read. Each write here is larger than that, so the host holds the
+// transport token until the worker exit.
+const HELD_WRITE_CHARS = 200_000;
+
+describe("plugin worker manager duplex stdin transport byte ledger", () => {
+ it("reserves the encoded serialized-frame size, larger than the raw payload, including JSON escaping", async () => {
+ const telemetry = peakTrackingTelemetry();
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 8 << 20, telemetry });
+ const handle = makeDuplexHandle({ duplexAggregateByteLedger: ledger });
+ try {
+ await handle.start();
+ const route = await handle.openDuplexChannel(
+ duplexOpenInput({
+ workerSessionId: "ws-a",
+ stopReadingStdinAfterOpen: true,
+ exitAfterStopMs: 500,
+ }),
+ );
+ // Every character is a quote. The JSON serialization escapes each quote to two
+ // bytes, so the serialized frame is at least twice the raw payload byte count.
+ const data = '"'.repeat(50_000);
+ const rawBytes = Buffer.byteLength(data, "utf8");
+ route.write(data);
+ // The raw-payload token and the transport token both hold at once. The worker
+ // does not read its stdin, so the transport token never flushes.
+ expect(ledger.liveTokenCount).toBe(2);
+ const transportBytes = ledger.bytesInUse - rawBytes;
+ // The transport reservation covers the serialized frame, so it is larger than
+ // the raw payload and it includes the doubled escaped quotes.
+ expect(transportBytes).toBeGreaterThan(rawBytes);
+ expect(transportBytes).toBeGreaterThanOrEqual(2 * rawBytes);
+ } finally {
+ await handle.stop().catch(() => undefined);
+ }
+ // The worker exit discards the stdin buffer, so every token releases one time.
+ await vi.waitFor(() => {
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ });
+ expect(telemetry.underflows).toBe(0);
+ });
+
+ it("holds the transport token across a write-RPC timeout and releases it on worker exit", async () => {
+ const telemetry = peakTrackingTelemetry();
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 8 << 20, telemetry });
+ // The write RPC times out fast; the worker exits later. The gap lets a test
+ // prove the timeout does not release the transport token.
+ const handle = makeDuplexHandle({
+ duplexAggregateByteLedger: ledger,
+ duplexChannelLimits: { openTimeoutMs: 150 },
+ });
+ try {
+ await handle.start();
+ const route = await handle.openDuplexChannel(
+ duplexOpenInput({
+ workerSessionId: "ws-t",
+ stopReadingStdinAfterOpen: true,
+ exitAfterStopMs: 900,
+ }),
+ );
+ const data = "x".repeat(HELD_WRITE_CHARS);
+ const rawBytes = Buffer.byteLength(data, "utf8");
+ route.write(data);
+ // Both tokens hold at first: the raw payload and the serialized frame.
+ expect(ledger.liveTokenCount).toBe(2);
+ const transportBytes = ledger.bytesInUse - rawBytes;
+ expect(transportBytes).toBeGreaterThan(0);
+ // The write RPC times out. The RPC settle releases the raw-payload token, so
+ // one token remains: the transport token. The timeout never releases it.
+ await vi.waitFor(() => {
+ expect(ledger.liveTokenCount).toBe(1);
+ });
+ expect(ledger.bytesInUse).toBe(transportBytes);
+ } finally {
+ await handle.stop().catch(() => undefined);
+ }
+ // The worker exit releases the still-held transport token.
+ await vi.waitFor(() => {
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ });
+ expect(telemetry.underflows).toBe(0);
+ });
+
+ it("does not release the transport token when the route terminalizes", async () => {
+ const telemetry = peakTrackingTelemetry();
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 8 << 20, telemetry });
+ const handle = makeDuplexHandle({ duplexAggregateByteLedger: ledger });
+ try {
+ await handle.start();
+ const route = await handle.openDuplexChannel(
+ duplexOpenInput({
+ workerSessionId: "ws-c",
+ stopReadingStdinAfterOpen: true,
+ exitAfterStopMs: 700,
+ }),
+ );
+ const data = "x".repeat(HELD_WRITE_CHARS);
+ const rawBytes = Buffer.byteLength(data, "utf8");
+ route.write(data);
+ expect(ledger.liveTokenCount).toBe(2);
+ const transportBytes = ledger.bytesInUse - rawBytes;
+ // End the route. Route terminalization releases the route-owned tokens, but it
+ // must never release the transport token, so at least the serialized frame
+ // stays charged.
+ void route.close();
+ expect(ledger.bytesInUse).toBeGreaterThanOrEqual(transportBytes);
+ } finally {
+ await handle.stop().catch(() => undefined);
+ }
+ await vi.waitFor(() => {
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ });
+ expect(telemetry.underflows).toBe(0);
+ });
+
+ it("bounds the aggregate transport bytes across several routes and workers, ending an over-ceiling near-limit write with the aggregate marker", async () => {
+ vi.mocked(logger.warn).mockClear();
+ const telemetry = peakTrackingTelemetry();
+ // The ceiling admits only a couple of held serialized frames, so a near-limit
+ // write across the routes passes it and rejects.
+ const ceilingBytes = 600_000;
+ const ledger = new DuplexAggregateByteLedger({ ceilingBytes, telemetry });
+ const handleA = makeDuplexHandle({ duplexAggregateByteLedger: ledger });
+ const handleB = makeDuplexHandle({ duplexAggregateByteLedger: ledger });
+ try {
+ await handleA.start();
+ await handleB.start();
+ const routeA = await handleA.openDuplexChannel(
+ duplexOpenInput({
+ workerSessionId: "ws-a",
+ stopReadingStdinAfterOpen: true,
+ exitAfterStopMs: 900,
+ }),
+ );
+ const routeB = await handleB.openDuplexChannel(
+ duplexOpenInput({
+ workerSessionId: "ws-b",
+ stopReadingStdinAfterOpen: true,
+ exitAfterStopMs: 900,
+ }),
+ );
+ // Each write mixes escaped quotes and multi-byte characters near the per-write
+ // size. The serialized frame escapes each quote and keeps each euro sign as
+ // three UTF-8 bytes, so the reservation must count the encoded size.
+ const escaped = '"'.repeat(40_000);
+ const multiByte = "€".repeat(20_000);
+ const nearLimit = escaped + multiByte;
+ const routes = [routeA, routeB, routeA, routeB];
+ for (const route of routes) {
+ route.write(nearLimit);
+ // The aggregate bytes never pass the ceiling. Admission stops at the ceiling.
+ expect(ledger.bytesInUse).toBeLessThanOrEqual(ceilingBytes);
+ }
+ // At least one near-limit write passed the ceiling and rejected. The manager
+ // ended that route with the aggregate marker, never the route-busy marker.
+ expect(telemetry.rejections).toBeGreaterThanOrEqual(1);
+ expect(telemetry.peak).toBeLessThanOrEqual(ceilingBytes);
+ const reasons = loggedWarnReasons();
+ expect(reasons).toContain(DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED);
+ expect(reasons).not.toContain("DUPLEX_CHANNEL_ROUTE_BUSY");
+ } finally {
+ await handleA.stop().catch(() => undefined);
+ await handleB.stop().catch(() => undefined);
+ }
+ // Every worker exit releases its held transport tokens, so the ledger ends at
+ // zero with no accounting defect.
+ await vi.waitFor(() => {
+ expect(ledger.bytesInUse).toBe(0);
+ expect(ledger.liveTokenCount).toBe(0);
+ });
+ expect(telemetry.underflows).toBe(0);
+ });
+});
diff --git a/server/src/duplex-aggregate-ceiling-env.ts b/server/src/duplex-aggregate-ceiling-env.ts
new file mode 100644
index 0000000000..a1381c910c
--- /dev/null
+++ b/server/src/duplex-aggregate-ceiling-env.ts
@@ -0,0 +1,32 @@
+import {
+ resolveDuplexAggregateCeilingBytes,
+ type DuplexAggregateCeilingOverrideRejectionReporter,
+} from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger";
+
+/**
+ * Resolve the aggregate duplex route byte ceiling from the raw operator override
+ * string. The host reads the override from
+ * PAPERCLIP_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES.
+ *
+ * This helper distinguishes an absent variable from a present blank value. An
+ * absent variable is `undefined`. The helper then uses the documented default and
+ * does not report a rejection. A present variable is a string, even when the
+ * string holds only whitespace. The helper sends every present string through
+ * `Number`, so a blank or whitespace-only value becomes `0`. The numeric
+ * validation in {@link resolveDuplexAggregateCeilingBytes} rejects `0`, reports it
+ * through `onRejectedOverride`, and returns the safe default. So a present blank
+ * value is visible as invalid instead of silent as absent.
+ *
+ * `Number` ignores surrounding whitespace, so a valid value with surrounding
+ * spaces still parses. The reporter receives only the parsed number, never the raw
+ * string, so no operator-supplied text reaches a log line.
+ */
+export function resolveDuplexAggregateCeilingBytesFromEnv(
+ rawOverride: string | undefined,
+ onRejectedOverride?: DuplexAggregateCeilingOverrideRejectionReporter,
+): number {
+ return resolveDuplexAggregateCeilingBytes(
+ rawOverride === undefined ? undefined : Number(rawOverride),
+ onRejectedOverride,
+ );
+}
diff --git a/server/src/index.ts b/server/src/index.ts
index 6786407a65..058348ddc3 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -83,6 +83,13 @@ import { createFeedbackTraceShareClientFromConfig } from "./services/feedback-sh
import { buildRuntimeApiCandidateUrls, choosePrimaryRuntimeApiUrl } from "./runtime-api.js";
import { isLoopbackHost, rewriteLoopbackUrlPort } from "./url-utils.js";
import { createPluginWorkerManager } from "./services/plugin-worker-manager.js";
+import {
+ createDuplexAggregateByteLedgerTelemetry,
+ DuplexAggregateByteLedger,
+ type DuplexAggregateByteLedgerMetricSink,
+} from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger";
+import { resolveDuplexAggregateCeilingBytesFromEnv } from "./duplex-aggregate-ceiling-env.js";
+import { DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL } from "@paperclipai/adapter-utils/duplex-telemetry";
import { createStorageServiceFromConfig } from "./storage/index.js";
import { printStartupBanner } from "./startup-banner.js";
import { getBoardClaimWarningUrl, initializeBoardClaimChallenge } from "./board-claim.js";
@@ -754,9 +761,57 @@ export async function startServer(): Promise<StartedServer> {
databaseBackupInFlight = false;
}
};
- const pluginWorkerManager = createPluginWorkerManager();
+ // The process-owned aggregate byte ledger for the sandbox duplex channel. One
+ // ledger per host process bounds the aggregate bytes that all live duplex routes
+ // retain. The route-count controller bounds only the route count, so without this
+ // ledger the per-route byte bounds multiply to many gigabytes at the maximum
+ // route count. The manager injects this same object into every worker handle, so
+ // one shared gauge bounds every host-side retention site.
+ //
+ // The optional operator override reads PAPERCLIP_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES.
+ // An absent variable uses the documented default. A present invalid, blank,
+ // whitespace-only, non-finite, zero, negative, non-integer, unsafe, or
+ // over-maximum value does not fail startup. The host process is multi-tenant, so
+ // one invalid environment value must not brick the whole host. The helper sends
+ // the raw string to the resolver, so a present blank value is invalid, not
+ // absent. The resolver rejects the invalid override and returns the safe default.
+ // The reporter logs the rejection loudly at error, so the misconfiguration stays
+ // visible while the host stays up. The log line carries only the rejected numeric
+ // value, never the raw string.
+ const duplexAggregateCeilingBytes = resolveDuplexAggregateCeilingBytesFromEnv(
+ process.env.PAPERCLIP_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES,
+ (rejectedValue) => {
+ logger.error(
+ { rejectedValue },
+ "duplex aggregate byte ceiling override rejected; using the safe default",
+ );
+ },
+ );
+ // The server has no process metric pipeline yet, so the ledger telemetry maps to
+ // the structured logger. The gauge logs at debug. A reservation rejection logs at
+ // warn, because it marks an availability limit hit. An accounting-underflow defect
+ // logs at error, because it marks a real cleanup bug. Each record carries only the
+ // fixed metric name and the numeric value; no route, company, run, or payload
+ // value reaches a log line.
+ const duplexAggregateByteLedgerMetricSink: DuplexAggregateByteLedgerMetricSink = {
+ setGauge(name, value) {
+ logger.debug({ metric: name, value }, "duplex aggregate byte ledger gauge");
+ },
+ incrementCounter(name) {
+ if (name === DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL) {
+ logger.error({ metric: name }, "duplex aggregate byte ledger accounting underflow");
+ return;
+ }
+ logger.warn({ metric: name }, "duplex aggregate byte ledger reservation rejected");
+ },
+ };
+ const duplexAggregateByteLedger = new DuplexAggregateByteLedger({
+ ceilingBytes: duplexAggregateCeilingBytes,
+ telemetry: createDuplexAggregateByteLedgerTelemetry(duplexAggregateByteLedgerMetricSink),
+ });
+ const pluginWorkerManager = createPluginWorkerManager({ duplexAggregateByteLedger });
const heartbeat = config.heartbeatSchedulerEnabled
- ? heartbeatService(db as any, { pluginWorkerManager })
+ ? heartbeatService(db as any, { pluginWorkerManager, duplexAggregateByteLedger })
: null;
const decisionServiceOptions = {
wakeOriginAgent: createDecisionWakeOriginAgent(heartbeat?.wakeup ?? null),
diff --git a/server/src/services/environment-execution-target.ts b/server/src/services/environment-execution-target.ts
index 90fc7ac3c9..c792929f50 100644
--- a/server/src/services/environment-execution-target.ts
+++ b/server/src/services/environment-execution-target.ts
@@ -6,6 +6,7 @@ import {
type AdapterExecutionTarget,
} from "@paperclipai/adapter-utils/execution-target";
import type { DuplexTelemetryRecorder } from "@paperclipai/adapter-utils/duplex-telemetry";
+import type { DuplexAggregateByteLedger } from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger";
import {
clampSpanLabel,
getActiveStepContext,
@@ -206,6 +207,10 @@ export async function resolveEnvironmentExecutionTarget(input: {
// enters the sandbox environment. Absent keeps the safe no-op default in the
// bridge, so the surface stays inert until the host injects a real recorder.
duplexTelemetryRecorder?: DuplexTelemetryRecorder | null;
+ // The process-owned aggregate byte ledger. The seam stamps it onto the sandbox
+ // target next to the runner, so the live object stays on the host and never
+ // enters the sandbox environment. Absent keeps the bridge inert for this seam.
+ duplexAggregateByteLedger?: DuplexAggregateByteLedger | null;
}): Promise<AdapterExecutionTarget | null> {
if (input.environment.driver === "local") {
return {
@@ -323,6 +328,10 @@ export async function resolveEnvironmentExecutionTarget(input: {
// binds it to the fixed observability surface. Absent keeps the no-op
// default, so the surface stays inert on a run with no injected recorder.
duplexTelemetryRecorder: input.duplexTelemetryRecorder ?? null,
+ // Attach the process-owned aggregate byte ledger next to the runner. The
+ // bridge passes it to the broker, the decoder, and the response-body reader.
+ // Absent keeps the bridge inert for this seam.
+ duplexAggregateByteLedger: input.duplexAggregateByteLedger ?? null,
...(effectiveCapabilities ? { effectiveCapabilities: Object.freeze({ ...effectiveCapabilities }) } : {}),
environmentId: input.environment.id ?? null,
leaseId: input.leaseId ?? null,
diff --git a/server/src/services/environment-run-orchestrator.ts b/server/src/services/environment-run-orchestrator.ts
index 3a76dcfec1..a6db690eee 100644
--- a/server/src/services/environment-run-orchestrator.ts
+++ b/server/src/services/environment-run-orchestrator.ts
@@ -43,6 +43,7 @@ import {
type AdapterWorkspaceRealization,
} from "@paperclipai/adapter-utils/execution-target";
import type { DuplexTelemetryRecorder } from "@paperclipai/adapter-utils/duplex-telemetry";
+import type { DuplexAggregateByteLedger } from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger";
import { buildWorkspaceRealizationRequest } from "./workspace-realization.js";
import { executionWorkspaceService } from "./execution-workspaces.js";
import { logActivity } from "./activity-log.js";
@@ -154,6 +155,14 @@ export function environmentRunOrchestrator(
options: {
pluginWorkerManager?: PluginWorkerManager;
environmentRuntime?: EnvironmentRuntimeService;
+ /**
+ * The process-owned aggregate byte ledger for the sandbox duplex channel.
+ * The server root creates one ledger per host process and injects the same
+ * object here. The orchestrator stamps it onto the sandbox execution target,
+ * so one shared gauge bounds the aggregate retained bytes across all live
+ * duplex routes. Absent keeps the bridge inert for this seam.
+ */
+ duplexAggregateByteLedger?: DuplexAggregateByteLedger | null;
} = {},
) {
const environmentsSvc = environmentService(db);
@@ -523,6 +532,7 @@ export function environmentRunOrchestrator(
lease,
environmentRuntime,
duplexTelemetryRecorder: input.duplexTelemetryRecorder ?? null,
+ duplexAggregateByteLedger: options.duplexAggregateByteLedger ?? null,
});
const realizationMode = workspaceRealization.mode === "in_place" ? "in_place" : "copy";
const authoritativeRoot =
diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts
index 611f310fbb..4a665fc0d1 100644
--- a/server/src/services/heartbeat.ts
+++ b/server/src/services/heartbeat.ts
@@ -72,6 +72,7 @@ import {
import { conflict, HttpError, notFound } from "../errors.js";
import { getStartupTraceContext, getStartupTracer } from "../instrumentation.js";
import { createHostDuplexTelemetryRecorder } from "./duplex-telemetry-recorder.js";
+import type { DuplexAggregateByteLedger } from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger";
import { incrementToolRuntimeMetricCounter } from "./tool-runtime-metrics.js";
import { logger } from "../middleware/logger.js";
import {
@@ -6692,6 +6693,14 @@ export interface HeartbeatServiceOptions {
pluginWorkerManager?: PluginWorkerManager;
environmentRuntime?: HeartbeatEnvironmentRuntime;
runtimeEnv?: Record<string, string | undefined>;
+ /**
+ * The process-owned aggregate byte ledger for the sandbox duplex channel. The
+ * server root creates one ledger per host process and injects the same object
+ * here. The heartbeat threads it into the environment run orchestrator, which
+ * stamps it onto the sandbox execution target. Absent keeps the bridge inert
+ * for this seam.
+ */
+ duplexAggregateByteLedger?: DuplexAggregateByteLedger | null;
}
type WorkspaceReadyCommentWriter = {
@@ -6815,6 +6824,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
const envOrchestrator = environmentRunOrchestrator(db, {
pluginWorkerManager: options.pluginWorkerManager,
environmentRuntime,
+ duplexAggregateByteLedger: options.duplexAggregateByteLedger,
});
const workspaceOperationsSvc = workspaceOperationService(db);
const liveRunExecutions = {
diff --git a/server/src/services/plugin-worker-manager.ts b/server/src/services/plugin-worker-manager.ts
index 77ac1ff63a..b755e8388c 100644
--- a/server/src/services/plugin-worker-manager.ts
+++ b/server/src/services/plugin-worker-manager.ts
@@ -57,10 +57,31 @@ import type {
InitializeParams,
} from "@paperclipai/plugin-sdk";
import { getActiveStepContext } from "@paperclipai/adapter-utils/acpx-engine/startup-timing";
+import {
+ DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED,
+ type DuplexAggregateByteLedger,
+ type DuplexAggregateTokenOwner,
+ type ReservationToken,
+} from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger";
import { CLAUDE_SETUP_TOKEN_COMMAND } from "@paperclipai/adapter-claude-local/server";
import { logger } from "../middleware/logger.js";
import { traceparentFromContextToken } from "../instrumentation.js";
+/**
+ * The host raises this error when the child-stdin transport reservation for a
+ * duplex write fails against the aggregate byte ledger. The host does not write
+ * the frame. The write path throws it, `callInternal` rejects the RPC with it
+ * unwrapped, and the duplex write caller ends the route fail-closed with the
+ * {@link DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED} marker. Only a duplex write
+ * meters the transport, so this error never reaches a non-duplex control message.
+ */
+class DuplexAggregateBytesExceededError extends Error {
+ constructor() {
+ super(DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED);
+ this.name = "DuplexAggregateBytesExceededError";
+ }
+}
+
// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------
@@ -397,6 +418,14 @@ export interface WorkerStartOptions {
* constructs a handle this way).
*/
duplexRouteSlots?: DuplexRouteSlotController | null;
+ /**
+ * The process-owned aggregate byte ledger for the duplex channel. The manager
+ * injects one shared instance into every worker handle, so one gauge bounds the
+ * aggregate retained bytes across every route in the process. When it is absent,
+ * the worker retains duplex bytes unbounded (a unit test constructs a handle
+ * this way). The manager never makes a fresh per-handle default.
+ */
+ duplexAggregateByteLedger?: DuplexAggregateByteLedger | null;
/**
* Companies this worker may act on from proactive (no-invocation) worker→host
* calls — the plugin's configured companies. Seeded onto the handle at
@@ -966,11 +995,42 @@ export function createPluginWorkerHandle(
// JSON-RPC message sending
// -----------------------------------------------------------------------
- function sendMessage(message: unknown): void {
+ function sendMessage(message: unknown, meterDuplexWrite = false): void {
if (!childProcess?.stdin?.writable) {
throw new Error(`Worker process for plugin "${pluginId}" is not writable`);
}
const serialized = serializeMessage(message as any);
+ const ledger = duplexAggregateByteLedger;
+ if (meterDuplexWrite && ledger) {
+ // Charge the child-stdin transport buffer for a duplex write. The host writes
+ // the serialized frame to the child stdin without backpressure. When the child
+ // stops reading its stdin, the frame stays in the host stdin write buffer.
+ // Reserve the exact serialized-frame byte count. That count includes the JSON
+ // escaping and the newline framing, so the ledger covers the retained transport
+ // bytes, not only the raw payload. The RPC separately holds the raw payload
+ // under a `pending_write` token, so the two tokens cover the peak of both
+ // retentions at the same time.
+ const bytes = Buffer.byteLength(serialized);
+ const token = ledger.reserve("stdin_write", bytes);
+ if (!token) {
+ // The reservation would pass the aggregate ceiling. Fail closed before the
+ // enqueue: do not write the frame. The duplex write caller ends the route.
+ throw new DuplexAggregateBytesExceededError();
+ }
+ // Hold the token until the stream flushes the chunk. The write callback fires
+ // when the stream hands the chunk to the operating system, so the bytes then
+ // leave the host stream buffer. The RPC settle and the RPC timeout never
+ // release this token. Only the flush, the stream error, the stream close, or
+ // the worker exit releases it. Release through the outstanding-token set, so a
+ // later stream-error or worker-exit sweep never double-releases the same token.
+ pendingStdinWriteTokens.add(token);
+ childProcess.stdin.write(serialized, () => {
+ if (pendingStdinWriteTokens.delete(token)) {
+ ledger.release(token);
+ }
+ });
+ return;
+ }
childProcess.stdin.write(serialized);
}
@@ -1550,12 +1610,37 @@ export function createPluginWorkerHandle(
// 6. total data bytes — the cumulative inbound bytes over the whole life,
// counted before and after a data listener attaches;
// 7. route lifetime — the milliseconds from the open to the terminal end.
+
+ // One buffered data chunk retained for a late listener drain. It carries the raw
+ // chunk string and the aggregate byte token that reserved its raw bytes. The
+ // token is `null` when no ledger is injected.
+ interface BufferedDuplexChunk {
+ chunk: string;
+ token: ReservationToken | null;
+ }
+ // One pre-bind data event, normalized to the narrow duplex-event schema. The host
+ // retains only these bounded scalar fields plus the reservation token, never the
+ // original arbitrary notification graph. The bind re-resolves the pair from
+ // `workerSessionId`, so a frame whose pair does not match the bound pair still
+ // fails closed.
+ interface HeldDuplexEvent {
+ workerSessionId: string;
+ chunk: string;
+ token: ReservationToken | null;
+ }
+ // One pre-bind exit event, normalized to the narrow duplex-event schema.
+ interface HeldDuplexExitEvent {
+ workerSessionId: string;
+ exitCode: number | null;
+ token: ReservationToken | null;
+ }
+
interface DuplexChannelRoute {
hostRouteId: string;
state: RouteState;
workerSessionId: string | null;
listener: ((chunk: string) => void) | null;
- buffered: string[];
+ buffered: BufferedDuplexChunk[];
bufferedChars: number;
pendingRequests: number;
protocolErrors: number;
@@ -1563,18 +1648,23 @@ export function createPluginWorkerHandle(
lifetimeTimer: ReturnType<typeof setTimeout> | null;
terminalized: boolean;
settleWait: (value: { exitCode: number | null; transportClosed?: boolean }) => void;
- // The data frames that arrived before the bind, held in order. The bind
- // replays them through the exact-pair routing, so an early frame is never
- // lost and a frame whose pair does not match the bound pair still fails
- // closed. The hold ceiling is `maxDuplexChannelPreBindHoldFrames`, one frame
- // above the buffered bound, so the replay's buffered-bound check ends the
- // route, not the hold.
- preBind: JsonRpcNotification[];
- // The single exit frame that arrived before the bind. An exit never consumes
- // a data hold slot, so a worker that batches an exit among enough data frames
- // to fill the hold cannot crowd out a data frame. The bind replays the held
- // data frames first, then this exit last.
- preBindExit: JsonRpcNotification | null;
+ // The bounded raw data events that arrived before the bind, held in order. The
+ // bind replays them through the exact-pair routing, so an early frame is never
+ // lost and a frame whose pair does not match the bound pair still fails closed.
+ // The hold ceiling is `maxDuplexChannelPreBindHoldFrames`, one frame above the
+ // buffered bound, so the replay's buffered-bound check ends the route, not the
+ // hold. Each held event carries the aggregate byte token that reserved its raw
+ // bytes; the host never retains the original arbitrary notification graph.
+ preBind: HeldDuplexEvent[];
+ // The single bounded exit event that arrived before the bind. An exit never
+ // consumes a data hold slot, so a worker that batches an exit among enough data
+ // frames to fill the hold cannot crowd out a data frame. The bind replays the
+ // held data events first, then this exit last.
+ preBindExit: HeldDuplexExitEvent | null;
+ // The authoritative registry of every aggregate byte token this route still
+ // holds, across the pre-bind, buffered, and terminal-buffered representations.
+ // The byte cleanup releases every token here exactly once.
+ retainedTokens: Set<ReservationToken>;
}
// The live duplex routes on this worker, keyed by the exact
// `{ hostRouteId, workerSessionId }` pair. The host binds one pair once, at
@@ -1600,6 +1690,89 @@ export function createPluginWorkerHandle(
// time per route, so a double terminalize never releases two slots.
const duplexRouteSlotHolders = new Set<DuplexChannelRoute>();
+ // The process-owned aggregate byte ledger. The manager injects it, so one gauge
+ // bounds the aggregate retained bytes across every route in the process. When it
+ // is absent, the worker retains duplex bytes unbounded (a unit test constructs it
+ // this way).
+ const duplexAggregateByteLedger = options.duplexAggregateByteLedger ?? null;
+ // The outstanding child-stdin transport tokens for duplex writes. Each token
+ // covers one serialized frame the host stdin write buffer still retains. The
+ // write callback releases a token on the flush; a stream error, a stream close,
+ // or a worker exit releases every remaining token, because each of those
+ // discards the stdin write buffer. The set is the release guard, so a token
+ // releases one time across the two paths.
+ const pendingStdinWriteTokens = new Set<ReservationToken>();
+ // Release every outstanding child-stdin transport token. A stream error, a
+ // stream close, or a worker exit calls this, because each discards the stdin
+ // write buffer. The `delete` guard drops each token one time, so a later flush
+ // callback or a second sweep releases nothing again.
+ function releaseAllPendingStdinWriteTokens(): void {
+ if (!duplexAggregateByteLedger) return;
+ for (const token of pendingStdinWriteTokens) {
+ duplexAggregateByteLedger.release(token);
+ }
+ pendingStdinWriteTokens.clear();
+ }
+ // The terminalized routes that still hold buffered bytes for a late listener
+ // drain. A terminalized route leaves the opening and live maps, so this registry
+ // keeps the worker-exit sweep able to release its still-charged buffered tokens.
+ const terminalDuplexRoutes = new Set<DuplexChannelRoute>();
+
+ // Reserve `bytes` for one route retention against the aggregate ledger. Return
+ // the held token, or `null` when the reservation would pass the ceiling. When no
+ // ledger is present, return `"no-ledger"`, so the caller admits the retention
+ // with no token. The helper adds a real token to `route.retainedTokens`.
+ function reserveRouteBytes(
+ route: DuplexChannelRoute,
+ owner: DuplexAggregateTokenOwner,
+ bytes: number,
+ ): ReservationToken | "no-ledger" | null {
+ if (!duplexAggregateByteLedger) return "no-ledger";
+ const token = duplexAggregateByteLedger.reserve(owner, bytes);
+ if (!token) return null;
+ route.retainedTokens.add(token);
+ return token;
+ }
+
+ // Transfer a held route token to a new owner label. The token identity and the
+ // reserved bytes stay the same, so no admission gap opens between two
+ // representations of the same retained bytes.
+ function transferRouteToken(
+ token: ReservationToken | null,
+ owner: DuplexAggregateTokenOwner,
+ ): void {
+ if (!token) return;
+ duplexAggregateByteLedger?.transfer(token, owner);
+ }
+
+ // Release one held route token and drop it from the route registry, in one
+ // synchronous step. A `null` token (no ledger) releases nothing.
+ function releaseRouteToken(route: DuplexChannelRoute, token: ReservationToken | null): void {
+ if (!token) return;
+ route.retainedTokens.delete(token);
+ duplexAggregateByteLedger?.release(token);
+ }
+
+ // Release every token a route still holds and clear its retained representations,
+ // exactly once. A later call finds an empty registry and releases nothing, so the
+ // helper is idempotent. Every terminal-discard, open-failure, bind-replay
+ // failure, close-acknowledgement failure, worker-exit, and shutdown path calls
+ // it. Terminal map deletion is never treated as proof that retained bytes are
+ // gone; this registry is authoritative for byte cleanup.
+ function discardRouteRetained(route: DuplexChannelRoute): void {
+ if (duplexAggregateByteLedger) {
+ for (const token of route.retainedTokens) {
+ duplexAggregateByteLedger.release(token);
+ }
+ }
+ route.retainedTokens.clear();
+ route.preBind = [];
+ route.preBindExit = null;
+ route.buffered = [];
+ route.bufferedChars = 0;
+ terminalDuplexRoutes.delete(route);
+ }
+
// Try to reserve one aggregate route slot for a route. Return true when the
// route holds a slot after the call. When no controller is present, the route
// always holds a slot.
@@ -1697,15 +1870,38 @@ export function createPluginWorkerHandle(
route.terminalized = true;
route.state = "closed";
route.listener = null;
- // Keep the buffered chunks the host accepted before the route ended, so a
- // listener that attaches after the end still drains them. A frame can end the
- // route during the pre-bind replay, before a listener attaches, and the
- // chunks the host accepted before that frame are valid data the listener must
- // still receive. The buffered bytes stay bounded by the pre-bind buffered
- // bound, and `onData` clears them once it drains them. Drop the held pre-bind
- // frames, which never bound to a listener.
+ // Stop admission first. Then move the byte cleanup. Keep the buffered chunks the
+ // host accepted before the route ended, so a listener that attaches after the
+ // end still drains them. A frame can end the route during the pre-bind replay,
+ // before a listener attaches, and the chunks the host accepted before that frame
+ // are valid data the listener must still receive. The buffered bytes stay
+ // bounded by the pre-bind buffered bound, and `onData` clears and releases them
+ // once it drains them.
+ //
+ // The buffered records keep their tokens; move each token to the terminal owner
+ // label and register the route in the terminal registry, so the worker-exit
+ // sweep can release the still-charged bytes later. Do not release a buffered
+ // token at map deletion; terminal map deletion is never proof the bytes are
+ // gone. Release every other token the route still holds (the pre-bind events,
+ // the held exit, and any stranded token), because they never reach a listener.
+ const bufferedTokens = new Set<ReservationToken>();
+ for (const record of route.buffered) {
+ if (record.token) {
+ bufferedTokens.add(record.token);
+ transferRouteToken(record.token, "terminal_buffered");
+ }
+ }
+ for (const token of [...route.retainedTokens]) {
+ if (!bufferedTokens.has(token)) {
+ route.retainedTokens.delete(token);
+ duplexAggregateByteLedger?.release(token);
+ }
+ }
route.preBind = [];
route.preBindExit = null;
+ if (route.buffered.length > 0) {
+ terminalDuplexRoutes.add(route);
+ }
clearDuplexChannelLifetimeTimer(route);
// Remove the live binding and install the tombstone in one synchronous step,
// before the worker close and before any reuse. A reserved route that never
@@ -1787,14 +1983,26 @@ export function createPluginWorkerHandle(
return "violation";
}
- function routeDuplexChannelData(notification: JsonRpcNotification): void {
+ // Route one duplex channel data frame. On a live route with a listener, deliver
+ // the chunk transiently. On a live route with no listener, buffer the chunk and
+ // charge its raw bytes against the aggregate ledger. The `carried` argument marks
+ // a bind-replay frame: it carries the pre-bind token the buffered record must
+ // reuse, so the replay transfers the token to the buffered representation with no
+ // decrement and no re-reserve. A live frame passes no `carried`, so it reserves a
+ // fresh buffered token. A `carried.token` of `null` means no ledger is present.
+ function routeDuplexChannelData(
+ notification: JsonRpcNotification,
+ carried?: { token: ReservationToken | null },
+ ): void {
const params = isRecord(notification.params) ? notification.params : {};
const hostRouteId = readNonEmptyString(params.hostRouteId);
const workerSessionId = readNonEmptyString(params.workerSessionId);
const resolved = resolveDuplexRouteByPair(hostRouteId, workerSessionId);
if (resolved === "tombstoned") {
// A late frame for a closed pair. It reaches no listener and changes no
- // state. Never log the raw frame content.
+ // state. A bind replay never lands here, because the replayed route is live;
+ // a carried token stays on its route and the terminal cleanup releases it.
+ // Never log the raw frame content.
return;
}
if (resolved === "violation") {
@@ -1804,8 +2012,8 @@ export function createPluginWorkerHandle(
return;
}
if (resolved === "opening") {
- // The frame arrived before the bind. Hold it; the bind replays it through the
- // exact-pair routing.
+ // The frame arrived before the bind. A replay never lands here, because the
+ // route is live during replay. Hold the live frame; the bind replays it.
bufferPreBindDuplexFrame(hostRouteId, notification);
return;
}
@@ -1813,13 +2021,15 @@ export function createPluginWorkerHandle(
const chunk = params.chunk;
if (typeof chunk !== "string" || chunk.length === 0) {
// The exact pair matches, but the chunk is malformed. Count one per-route
- // protocol error.
+ // protocol error. Release a carried replay token first.
+ releaseRouteToken(route, carried?.token ?? null);
recordDuplexChannelProtocolError(route);
return;
}
if (chunk.length > maxDuplexChannelChunkChars) {
// One inbound chunk is larger than the per-chunk limit. End the route at
// once. Do not count the chunk as a protocol error.
+ releaseRouteToken(route, carried?.token ?? null);
void terminalizeDuplexChannelRoute(route);
return;
}
@@ -1828,12 +2038,16 @@ export function createPluginWorkerHandle(
// bound listener cannot receive data past the cap.
const chunkBytes = Buffer.byteLength(chunk);
if (route.totalDataBytes + chunkBytes > maxDuplexChannelTotalDataBytes) {
+ releaseRouteToken(route, carried?.token ?? null);
void terminalizeDuplexChannelRoute(route);
return;
}
route.totalDataBytes += chunkBytes;
if (route.listener) {
+ // A listener is attached. Deliver the chunk transiently and release a carried
+ // replay token after the synchronous delivery boundary.
deliverDuplexChannelChunk(route.listener, chunk);
+ releaseRouteToken(route, carried?.token ?? null);
return;
}
// No listener attached yet. Buffer the frame under the pre-bind bounds. End
@@ -1842,10 +2056,28 @@ export function createPluginWorkerHandle(
route.buffered.length + 1 > maxDuplexChannelPreBindFrames ||
route.bufferedChars + chunk.length > maxDuplexChannelPreBindChars
) {
+ releaseRouteToken(route, carried?.token ?? null);
void terminalizeDuplexChannelRoute(route);
return;
}
- route.buffered.push(chunk);
+ let token: ReservationToken | null;
+ if (carried) {
+ // The bind replay transfers the pre-bind token to the buffered
+ // representation. No decrement and no re-reserve, so no admission gap opens.
+ token = carried.token;
+ transferRouteToken(token, "buffered_chunk");
+ } else {
+ // A live frame reserves a fresh buffered token for its exact raw bytes.
+ const reserved = reserveRouteBytes(route, "buffered_chunk", chunkBytes);
+ if (reserved === null) {
+ // The aggregate ceiling rejected the reservation. Retain nothing and fail
+ // closed with the fixed marker.
+ void terminalizeDuplexChannelRoute(route);
+ return;
+ }
+ token = reserved === "no-ledger" ? null : reserved;
+ }
+ route.buffered.push({ chunk, token });
route.bufferedChars += chunk.length;
}
@@ -1900,15 +2132,58 @@ export function createPluginWorkerHandle(
if (!hostRouteId) return;
const route = openingDuplexRoutes.get(hostRouteId);
if (!route) return;
+ const params = isRecord(notification.params) ? notification.params : {};
+ const workerSessionId = readNonEmptyString(params.workerSessionId);
+ if (!workerSessionId) {
+ // A malformed pair. Count one per-route protocol error and hold nothing.
+ recordDuplexChannelProtocolError(route);
+ return;
+ }
if (notification.method === DUPLEX_CHANNEL_EXIT_NOTIFICATION) {
- route.preBindExit = notification;
+ // Normalize the exit to the narrow duplex-event schema. An exit retains only
+ // a bounded scalar, so reserve a zero-byte token that carries the record
+ // through the one cleanup path. Release any earlier held exit token first, so
+ // a replaced exit never leaks its reservation.
+ const exitCode = typeof params.exitCode === "number" ? params.exitCode : null;
+ if (route.preBindExit) releaseRouteToken(route, route.preBindExit.token);
+ const reserved = reserveRouteBytes(route, "pre_bind_event", 0);
+ if (reserved === null) {
+ // The aggregate ceiling rejected the reservation. Fail closed.
+ void terminalizeDuplexChannelRoute(route);
+ return;
+ }
+ route.preBindExit = {
+ workerSessionId,
+ exitCode,
+ token: reserved === "no-ledger" ? null : reserved,
+ };
+ return;
+ }
+ // A data event. Validate and normalize it to the narrow duplex-event schema
+ // before any retention.
+ const chunk = params.chunk;
+ if (typeof chunk !== "string" || chunk.length === 0) {
+ recordDuplexChannelProtocolError(route);
return;
}
if (route.preBind.length >= maxDuplexChannelPreBindHoldFrames) {
recordDuplexChannelProtocolError(route);
return;
}
- route.preBind.push(notification);
+ // Reserve the exact retained raw byte count before the host holds the event.
+ const reserved = reserveRouteBytes(route, "pre_bind_event", Buffer.byteLength(chunk));
+ if (reserved === null) {
+ // The aggregate ceiling rejected the reservation. The caller retains nothing
+ // and the route fails closed with the fixed marker.
+ log.warn({ pluginId, reason: DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED }, "duplex pre-bind hold rejected");
+ void terminalizeDuplexChannelRoute(route);
+ return;
+ }
+ route.preBind.push({
+ workerSessionId,
+ chunk,
+ token: reserved === "no-ledger" ? null : reserved,
+ });
}
// Replay the frames a route held before it bound. The route is live now, so the
@@ -1921,14 +2196,47 @@ export function createPluginWorkerHandle(
function replayPreBindDuplexFrames(route: DuplexChannelRoute): void {
const held = route.preBind;
route.preBind = [];
- for (const notification of held) {
- if (route.terminalized) break;
- routeDuplexChannelData(notification);
+ for (const event of held) {
+ if (route.terminalized) {
+ // The route ended mid-replay. Release the remaining held tokens, so the
+ // unreplayed events retain nothing.
+ releaseRouteToken(route, event.token);
+ continue;
+ }
+ // Reconstruct a transient data notification for the exact-pair routing. The
+ // host holds only the bounded event, so it builds this notification for the
+ // routing step and discards it at once. The carried token moves to the
+ // buffered representation without a decrement or a re-reserve.
+ routeDuplexChannelData(
+ {
+ jsonrpc: "2.0",
+ method: DUPLEX_CHANNEL_DATA_NOTIFICATION,
+ params: {
+ hostRouteId: route.hostRouteId,
+ workerSessionId: event.workerSessionId,
+ chunk: event.chunk,
+ },
+ },
+ { token: event.token },
+ );
}
const heldExit = route.preBindExit;
route.preBindExit = null;
- if (heldExit && !route.terminalized) {
- routeDuplexChannelExit(heldExit);
+ if (heldExit) {
+ // The exit retains only a bounded scalar. Release its placeholder token, then
+ // resolve the wait through the exact-pair routing when the route still lives.
+ releaseRouteToken(route, heldExit.token);
+ if (!route.terminalized) {
+ routeDuplexChannelExit({
+ jsonrpc: "2.0",
+ method: DUPLEX_CHANNEL_EXIT_NOTIFICATION,
+ params: {
+ hostRouteId: route.hostRouteId,
+ workerSessionId: heldExit.workerSessionId,
+ exitCode: heldExit.exitCode,
+ },
+ });
+ }
}
}
@@ -1939,23 +2247,34 @@ export function createPluginWorkerHandle(
// id never returns, so no closed pair can revive on a restart. The pending
// channel calls reject through `rejectAllPending`.
function closeDuplexChannelRouteOnWorkerExit(): void {
- const routes = [...openingDuplexRoutes.values(), ...liveDuplexRoutes.values()];
+ // Enumerate the opening index, the live index, and the terminal registry. A
+ // terminalized route left the opening and live maps but may still hold buffered
+ // bytes for a late listener; the worker is gone, so the host releases them now.
+ const routes = [
+ ...openingDuplexRoutes.values(),
+ ...liveDuplexRoutes.values(),
+ ...terminalDuplexRoutes,
+ ];
openingDuplexRoutes.clear();
liveDuplexRoutes.clear();
duplexPairTombstones.clear();
for (const route of routes) {
- if (route.terminalized) continue;
- route.terminalized = true;
- route.state = "closed";
- route.listener = null;
- route.buffered = [];
- route.bufferedChars = 0;
- route.preBind = [];
- route.preBindExit = null;
- clearDuplexChannelLifetimeTimer(route);
- releaseDuplexRouteSlot(route);
- settleRouteWait(route, { exitCode: null });
+ if (!route.terminalized) {
+ route.terminalized = true;
+ route.state = "closed";
+ route.listener = null;
+ route.bufferedChars = 0;
+ clearDuplexChannelLifetimeTimer(route);
+ releaseDuplexRouteSlot(route);
+ settleRouteWait(route, { exitCode: null });
+ }
+ // Release every token the route still holds and clear its retained
+ // representations, exactly once. A route already drained by a late listener
+ // holds no token, so this is harmless and leaves the ledger at zero.
+ discardRouteRetained(route);
}
+ // Clear the terminal registry. Every route in it was just discarded above.
+ terminalDuplexRoutes.clear();
}
// Open one live generic duplex channel route. Reserve the route before the open
@@ -1988,6 +2307,7 @@ export function createPluginWorkerHandle(
settleWait,
preBind: [],
preBindExit: null,
+ retainedTokens: new Set<ReservationToken>(),
};
// Reserve one aggregate route slot before any work. When the process-scoped
// ceiling is full, reject with the fixed route-busy error and open nothing, so
@@ -2074,11 +2394,61 @@ export function createPluginWorkerHandle(
void terminalizeDuplexChannelRoute(route);
return;
}
+ // Reserve the exact UTF-8 byte count of a host→worker write against the
+ // aggregate ledger before `callInternal` retains the payload. A pending write
+ // RPC holds `params.data` until it settles, so this reservation bounds the
+ // aggregate host→worker pending-write bytes across every route. Compute the
+ // byte count with `Buffer.byteLength`, not `data.length`, because one
+ // character can encode as several UTF-8 bytes. A stop request carries no
+ // payload, so it reserves nothing. When no ledger is present, admit the write
+ // with no token (a unit test constructs the handle this way).
+ let pendingWriteToken: ReservationToken | null = null;
+ if (method === "duplexChannelWrite" && duplexAggregateByteLedger) {
+ const data = (params as HostToWorkerMethods["duplexChannelWrite"][0]).data;
+ const bytes = Buffer.byteLength(data, "utf8");
+ pendingWriteToken = duplexAggregateByteLedger.reserve("pending_write", bytes);
+ if (!pendingWriteToken) {
+ // The reservation would pass the aggregate ceiling. Retain nothing, do
+ // not enqueue the RPC, and end the route fail-closed with the aggregate
+ // marker, not the route-busy marker.
+ log.warn(
+ { pluginId, reason: DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED },
+ "duplex pending write reservation rejected",
+ );
+ void terminalizeDuplexChannelRoute(route);
+ return;
+ }
+ }
route.pendingRequests += 1;
- void callInternal(method, params, duplexChannelOpenTimeoutMs)
- .catch(() => {})
+ // Meter the child-stdin transport buffer only for a duplex write. A stop
+ // request carries a tiny fixed frame that the host never lets grow, so it
+ // does not meter or reject. The write path reserves the serialized frame in
+ // `sendMessage` right before the stdin write.
+ const meterDuplexWrite = method === "duplexChannelWrite";
+ void callInternal(method, params, duplexChannelOpenTimeoutMs, undefined, meterDuplexWrite)
+ .catch((err: unknown) => {
+ if (err instanceof DuplexAggregateBytesExceededError) {
+ // The transport reservation failed. The host did not write the frame.
+ // End the route fail-closed with the aggregate marker, not the
+ // route-busy marker.
+ log.warn(
+ { pluginId, reason: DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED },
+ "duplex stdin write reservation rejected",
+ );
+ void terminalizeDuplexChannelRoute(route);
+ }
+ })
.finally(() => {
route.pendingRequests -= 1;
+ // Release the pending-write token one time, after the RPC settles on any
+ // path: success, error, timeout, worker exit, or shutdown. The token is
+ // not in `route.retainedTokens`, so route terminalization never releases
+ // it; only this settlement releases it. The separate `stdin_write` token
+ // covers the serialized frame and releases on the stream flush, the
+ // stream error, the stream close, or the worker exit, never here.
+ if (pendingWriteToken) {
+ duplexAggregateByteLedger?.release(pendingWriteToken);
+ }
});
};
@@ -2089,7 +2459,16 @@ export function createPluginWorkerHandle(
const pending = route.buffered;
route.buffered = [];
route.bufferedChars = 0;
- for (const chunk of pending) deliverDuplexChannelChunk(listener, chunk);
+ for (const record of pending) {
+ // Deliver each buffered record, drop its retained buffer, and release
+ // its exact token before the drain proceeds to the next record. A
+ // terminal route drains through this same code.
+ deliverDuplexChannelChunk(listener, record.chunk);
+ releaseRouteToken(route, record.token);
+ }
+ // The buffered records are gone, so the route no longer holds terminal
+ // bytes for a late listener.
+ terminalDuplexRoutes.delete(route);
}
},
write(data: string): void {
@@ -2392,6 +2771,18 @@ export function createPluginWorkerHandle(
readline.on("line", handleLine);
}
+ // Release the outstanding child-stdin transport tokens when the stdin stream
+ // errors or closes, because each discards the stdin write buffer. The `error`
+ // listener also stops an unhandled EPIPE from a child that closed its stdin.
+ if (child.stdin) {
+ child.stdin.on("error", () => {
+ releaseAllPendingStdinWriteTokens();
+ });
+ child.stdin.on("close", () => {
+ releaseAllPendingStdinWriteTokens();
+ });
+ }
+
// Capture stderr for logging
if (child.stderr) {
stderrReadline = createInterface({ input: child.stderr });
@@ -2442,6 +2833,12 @@ export function createPluginWorkerHandle(
childProcess = null;
startedAt = null;
+ // The worker exit discards the child-stdin write buffer, so release every
+ // outstanding transport token. The RPC rejections below never release these
+ // tokens; only this sweep, a stream flush, a stream error, or a stream close
+ // releases them.
+ releaseAllPendingStdinWriteTokens();
+
// Reject all pending requests
rejectAllPending(
new Error(formatWorkerFailureMessage(
@@ -2777,6 +3174,7 @@ export function createPluginWorkerHandle(
params: HostToWorkerMethods[M][0],
timeoutMs?: number,
executeLogSink?: ExecuteLogSink,
+ meterDuplexWrite = false,
): Promise<HostToWorkerMethods[M][1]> {
const rpcPromise = new Promise<HostToWorkerMethods[M][1]>((resolve, reject) => {
if (!childProcess?.stdin?.writable) {
@@ -2850,19 +3248,26 @@ export function createPluginWorkerHandle(
...createRequest(method, params, id),
...(invocation ? { paperclipInvocation: invocation } : {}),
};
- sendMessage(request);
+ sendMessage(request, meterDuplexWrite);
} catch (err) {
clearTimeout(timer);
pendingRequests.delete(id);
clearInvocation(invocation);
clearExecuteRoute(invocation?.id);
- reject(
- new Error(
- `Failed to send "${method}" to worker: ${
- err instanceof Error ? err.message : String(err)
- }`,
- ),
- );
+ if (err instanceof DuplexAggregateBytesExceededError) {
+ // The transport reservation failed before the write. Reject with the
+ // typed error unwrapped, so the duplex write caller ends the route
+ // fail-closed with the aggregate marker.
+ reject(err);
+ } else {
+ reject(
+ new Error(
+ `Failed to send "${method}" to worker: ${
+ err instanceof Error ? err.message : String(err)
+ }`,
+ ),
+ );
+ }
}
});
@@ -3035,6 +3440,14 @@ export interface PluginWorkerManagerOptions {
* stays upstream admission only.
*/
maxConcurrentDuplexRoutes?: number | null;
+ /**
+ * The process-owned aggregate byte ledger. The process root creates one ledger
+ * from validated configuration and passes it here. The manager injects the same
+ * object into every worker handle, so one gauge bounds the aggregate retained
+ * bytes across every route in the process. When it is absent, the worker retains
+ * duplex bytes unbounded. The manager never makes a fresh default ledger.
+ */
+ duplexAggregateByteLedger?: DuplexAggregateByteLedger | null;
}
/**
@@ -3106,6 +3519,11 @@ export function createPluginWorkerManager(
const duplexRouteSlots = createDuplexRouteSlotController(
managerOptions?.maxConcurrentDuplexRoutes,
);
+ // The one shared, process-owned aggregate byte ledger. The manager injects the
+ // same object into every worker handle, so one gauge bounds the aggregate
+ // retained bytes across every route in the process. It is `null` when the
+ // process root injected no ledger.
+ const duplexAggregateByteLedger = managerOptions?.duplexAggregateByteLedger ?? null;
return {
async startWorker(
@@ -3127,9 +3545,11 @@ export function createPluginWorkerManager(
}
const handle = createPluginWorkerHandle(pluginId, {
- // Inject the shared process-scoped route-slot controller, unless the caller
- // already supplied one (a test may inject its own).
+ // Inject the shared process-scoped route-slot controller and the shared
+ // process-owned aggregate byte ledger, unless the caller already supplied
+ // its own (a test may inject its own).
duplexRouteSlots,
+ duplexAggregateByteLedger,
...options,
});
workers.set(pluginId, handle);