Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
perf(json): encode streamed JSON iteratively (#1322)
Kaoru082256598beb4e82
diff --git a/src/domain/jsonSerialization.test.ts b/src/domain/jsonSerialization.test.tsindex bca145a95..45caf7898 100644--- a/src/domain/jsonSerialization.test.ts+++ b/src/domain/jsonSerialization.test.ts@@ -7,6 +7,16 @@ import {jsonParts,} from "./jsonSerialization.js";+/** Compact and pretty streamed JSON must equal native JSON. */+const expectNativeJson = (value: unknown): void => {+ expect([...bufferedJsonParts(jsonParts(value))].join("")).toBe(+ JSON.stringify(value),+ );+ expect([...jsonParts(value, true)].join("")).toBe(+ JSON.stringify(value, null, 2),+ );+};+describe("streamed JSON representation", () => {it.each([null,@@ -15,14 +25,7 @@ describe("streamed JSON representation", () => {[1, false, null, 'quotes: " / slash: \\ / newline: \n'],{ nested: { value: "中文 😀 \ud800" }, items: [1, 2] },JSON.parse('{"__proto__":{"value":1},"constructor":2,"10":3,"2":4}'),- ])("matches native JSON for ordinary values: %j", (value: unknown) => {- expect([...bufferedJsonParts(jsonParts(value))].join("")).toBe(- JSON.stringify(value),- );- expect([...jsonParts(value, true)].join("")).toBe(- JSON.stringify(value, null, 2),- );- });+ ])("matches native JSON for ordinary values: %j", expectNativeJson);it("preserves surrogate pairs across string and buffer boundaries", () => {const value = { text: `${"x".repeat(8190)}😀${'中文\n"'.repeat(20000)}` };@@ -59,3 +62,55 @@ describe("streamed JSON representation", () => {);});});++describe("iterative JSON encoding", () => {+ it("matches native JSON for long keys, omitted members, and deep nesting", () => {+ let deep: unknown = { leaf: [1, "two", null] };+ for (let level = 0; level < 500; level += 1)+ deep = level % 2 === 0 ? [deep, level] : { [`k${String(level)}`]: deep };+ const value = {+ [`${"k".repeat(9000)}😀`]: { nested: ["x".repeat(20000), {}, []] },+ omitted: undefined,+ method: () => 1,+ symbol: Symbol("ignored"),+ deep,+ };+ expectNativeJson(value);+ });++ it("rejects values native JSON would not encode as ordinary JSON", () => {+ expect(() => [...jsonParts([1, undefined])]).toThrow(+ "ordinary JSON values",+ );+ expect(() => [...jsonParts({ value: Number.NaN })]).toThrow(+ "ordinary JSON values",+ );+ expect(() => [...jsonParts({ value: new Date(0) })]).toThrow(+ "ordinary JSON objects",+ );+ });++ it("yields bounded parts before reading later members", () => {+ const value: Record<string, unknown> = { first: "x".repeat(100_000) };+ Object.defineProperty(value, "later", {+ enumerable: true,+ get() {+ throw new Error("later members must not be read yet");+ },+ });+ const parts = jsonParts(value);+ const first = parts.next();+ expect(first.done).toBe(false);+ expect(first.value?.length).toBeLessThanOrEqual(64 * 1024);+ // Many small members are grouped into parts instead of one per token.+ const small = Object.fromEntries(+ Array.from({ length: 10_000 }, (_, index) => [+ `k${String(index)}`,+ index,+ ]),+ );+ const smallParts = [...jsonParts(small)];+ expect(smallParts.join("")).toBe(JSON.stringify(small));+ expect(smallParts.length).toBeLessThan(20);+ });+});diff --git a/src/domain/jsonSerialization.ts b/src/domain/jsonSerialization.tsindex 57462f254..65330950e 100644--- a/src/domain/jsonSerialization.ts+++ b/src/domain/jsonSerialization.ts@@ -1,11 +1,11 @@/** Encode ordinary JSON as bounded string parts without a document-sized allocation. */export function* jsonParts(value: unknown, pretty = false): Generator<string> {- yield* encodeValue(value, pretty, 0, new Set(), false);+ yield* encodeValue(value, pretty, false);}/** Encode validated JSON with canonical key ordering without allocating the whole document. */export function* canonicalJsonParts(value: unknown): Generator<string> {- yield* encodeValue(value, false, 0, new Set(), true);+ yield* encodeValue(value, false, true);}/** Buffer JSON parts while preserving complete UTF-16 surrogate pairs. */@@ -46,73 +46,143 @@ function* encodeString(value: string): Generator<string> {yield '"';}-function* encodeValue(+/** Encoded text accumulated before it is yielded as one part. */+const PART_CHARACTERS = 16 * 1024;++const indentation: string[] = [];+const indent = (depth: number): string =>+ (indentation[depth] ??= `\n${" ".repeat(depth)}`);++interface ContainerFrame {+ readonly value: object;+ readonly array: boolean;+ readonly keys: readonly string[] | undefined;+ readonly count: number;+ readonly depth: number;+ index: number;+ written: number;+}++/** Encoded text of a scalar, or undefined for a container or a long string. */+const scalarText = (value: unknown): string | undefined => {+ if (typeof value === "string")+ return value.length <= 8192 ? JSON.stringify(value) : undefined;+ if (value === null) return "null";+ if (typeof value === "boolean") return String(value);+ if (typeof value === "number" && Number.isFinite(value)) return String(value);+ return undefined;+};++const openContainer = (value: unknown,- pretty: boolean,depth: number,ancestors: Set<object>,canonical: boolean,-): Generator<string> {- if (typeof value === "string") {- yield* encodeString(value);- return;- }- if (value === null || typeof value === "boolean") {- yield value === null ? "null" : String(value);- return;- }- if (typeof value === "number" && Number.isFinite(value)) {- yield String(value);- return;- }+): ContainerFrame => {if (typeof value !== "object" || value === null)throw new TypeError("JSON output requires ordinary JSON values");if (ancestors.has(value))throw new TypeError("JSON output contains a circular reference");- if (- !Array.isArray(value) &&- Object.prototype.toString.call(value) !== "[object Object]"- )+ const array = Array.isArray(value);+ if (!array && Object.prototype.toString.call(value) !== "[object Object]")throw new TypeError("JSON output requires ordinary JSON objects");ancestors.add(value);- try {- const array = Array.isArray(value);- const keys = array ? undefined : Object.keys(value);- if (canonical) keys?.sort();- const count = array ? value.length : (keys?.length ?? 0);- let written = 0;- yield array ? "[" : "{";- for (let index = 0; index < count; index += 1) {- let item: unknown;- let key: string | undefined;- if (array) {- item = value[index];+ const keys = array ? undefined : Object.keys(value);+ if (canonical) keys?.sort();+ const count = array ? value.length : (keys?.length ?? 0);+ return { value, array, keys, count, depth, index: 0, written: 0 };+};++/** The next written member of a container, skipping values JSON omits. */+const nextMember = (+ frame: ContainerFrame,+): { readonly key: string | undefined; readonly item: unknown } | undefined => {+ while (frame.index < frame.count) {+ const index = frame.index;+ frame.index += 1;+ if (frame.array)+ return { key: undefined, item: Reflect.get(frame.value, index) };+ const key = frame.keys?.[index];+ if (key === undefined)+ throw new TypeError("JSON object changed during serialization");+ const item: unknown = Reflect.get(frame.value, key);+ // Incur's filter can select inherited methods. Match native JSON's+ // omission without eagerly reading the rest of a bounded result.+ if (+ item !== undefined &&+ typeof item !== "function" &&+ typeof item !== "symbol"+ )+ return { key, item };+ }+ return undefined;+};++/**+ * Encode iteratively into bounded parts. Recursive `yield*` delegation passed+ * every token through one generator frame per nesting level, which dominated+ * the cost of writing large nested results.+ */+function* encodeValue(+ root: unknown,+ pretty: boolean,+ canonical: boolean,+): Generator<string> {+ const ancestors = new Set<object>();+ const stack: ContainerFrame[] = [];+ let buffer = "";+ let pending: unknown = root;+ let hasPending = true;+ while (true) {+ if (hasPending) {+ hasPending = false;+ const text = scalarText(pending);+ if (text !== undefined) buffer += text;+ else if (typeof pending === "string") {+ // A long string is split into surrogate-safe parts of its own.+ if (buffer !== "") yield buffer;+ buffer = "";+ yield* encodeString(pending);} else {- key = keys?.[index];- if (key === undefined)- throw new TypeError("JSON object changed during serialization");- item = Reflect.get(value, key);- // Incur's filter can select inherited methods. Match native JSON's- // omission without eagerly reading the rest of a bounded result.- if (- item === undefined ||- typeof item === "function" ||- typeof item === "symbol"- )- continue;+ const frame = openContainer(+ pending,+ stack.length,+ ancestors,+ canonical,+ );+ stack.push(frame);+ buffer += frame.array ? "[" : "{";}- if (written > 0) yield ",";- if (pretty) yield `\n${" ".repeat(depth + 1)}`;- if (key !== undefined) {- yield* encodeString(key);- yield pretty ? ": " : ":";+ }+ if (buffer.length >= PART_CHARACTERS) {+ yield buffer;+ buffer = "";+ }+ const frame = stack.at(-1);+ if (frame === undefined) break;+ const member = nextMember(frame);+ if (member === undefined) {+ if (pretty && frame.written > 0) buffer += indent(frame.depth);+ buffer += frame.array ? "]" : "}";+ ancestors.delete(frame.value);+ stack.pop();+ continue;+ }+ if (frame.written > 0) buffer += ",";+ if (pretty) buffer += indent(frame.depth + 1);+ if (member.key !== undefined) {+ const key = scalarText(member.key);+ if (key !== undefined) buffer += key;+ else {+ if (buffer !== "") yield buffer;+ buffer = "";+ yield* encodeString(member.key);}- yield* encodeValue(item, pretty, depth + 1, ancestors, canonical);- written += 1;+ buffer += pretty ? ": " : ":";}- if (pretty && written > 0) yield `\n${" ".repeat(depth)}`;- yield array ? "]" : "}";- } finally {- ancestors.delete(value);+ frame.written += 1;+ pending = member.item;+ hasPending = true;}+ if (buffer !== "") yield buffer;}