Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
perf(environment): index observed-state write cutoffs (#5350)
gensokyo-devd148ce090512
diff --git a/docs/specs/agent-environments/state-view/verification.md b/docs/specs/agent-environments/state-view/verification.mdindex 26101f651..48db9935b 100644--- a/docs/specs/agent-environments/state-view/verification.md+++ b/docs/specs/agent-environments/state-view/verification.md@@ -91,7 +91,7 @@ workflow: 22.836 s / 19.880 CPU s / 1,072.43 MiB; actual browser-state workflow:exhausts an isolated decoder heap and verifies that MCP remains usable; noworkspace OOM occurred.-Before/after used immutable pre-fix `29e745923` and the final source, the same+The first follow-up benchmark used pre-fix `29e745923` and `6ad63bcc2`, the samecompleted Evidence files, Node and CPU affinity. One fresh process per revision,with a first call and two warm repeats; filesystem caches were not cleared.Complete normalized output digests matched for both synthetic projection tasks.@@ -116,3 +116,41 @@ No new macOS/Windows runtime, state serving, AgentEnv data-plane or write-effectclaim is made. A type first observed only after an unrelated-root write maystill lack the earlier observation needed to detect its effect. Item-shapedand delete-only document paths remain recorded, as specified.++## Post-merge write-cutoff follow-up++Base `3fca2e0ca` includes #5319 and the validated immutable-capture reuse from+#5349. The remaining review finding was repeated write-root parsing for each+read type and again during output assembly. One admitted prefix/wildcard index+now owns parsed write roots, with per-projection positive/negative cutoff+caching. Backed type membership is collected once, replacing another plan scan.+The old linear matcher is removed. Prefix, parameter, redacted-root and timeline+semantics are preserved; no cross-request cache or new authority is introduced.++- Source checks: 41 files, 508 passed, 3 existing skips, including independent+ matching-oracle and 12,000-segment iterative-lookup checks.+- Serial static/build/artifact gate: 11 tasks passed, 197.356 s wall /+ 197.195 CPU s / 4,744.92 MiB peak process-family RSS.+- Both selected compiled state CLI/MCP cases passed after rebuilding.+- Live Gitea HAR workflows and fresh actual Chromium capture passed again:+ initial values, recorded fallbacks, CLI/default stdio parity and six exact+ HTTP replays. HAR workflow: 22.834 s wall / 19.623 CPU s / 1,067.36 MiB;+ browser workflow: 22.858 s / 20.912 CPU s / 1,546.59 MiB. Owned processes stopped.++The same completed Evidence with 1,000 collection reads and 1,000 unrelated+writes was projected before and after on one CPU. Two larger full-HAR inputs+(3,000/4,000 total records) were refused by the existing 96 MiB admission budget;+the limit was retained. One fresh process per revision, first call plus one+warm repeat, filesystem caches not cleared:++| Complete file-to-state projection | Main before | Follow-up |+| -------------------------------------------------------- | ------------------- | ------------------- |+| First call wall / CPU (ms) | 1,727.89 / 1,727.36 | 1,244.82 / 1,244.05 |+| Warm call wall / CPU (ms) | 1,300.84 / 1,300.42 | 624.01 / 623.89 |+| Process wall / CPU (s), including imports and both calls | 4.690 / 4.363 | 3.370 / 3.062 |+| Sampled process-family peak RSS (MiB) | 365.84 | 360.03 |++Parent Evidence IDs and complete normalized output digests were identical.+RSS sampling is every 50 ms, includes descendants and can double-count shared+pages; these few runs do not establish a general memory reduction. Model token+usage remains unmeasured. No workspace OOM occurred.diff --git a/src/domain/analysisView/EnvironmentStateExtraction.ts b/src/domain/analysisView/EnvironmentStateExtraction.tsindex 7ed3ef79e..11f457edd 100644--- a/src/domain/analysisView/EnvironmentStateExtraction.ts+++ b/src/domain/analysisView/EnvironmentStateExtraction.ts@@ -20,7 +20,6 @@ import {locateItem,locateList,rootOf,- rootsRelated,scalarText,segmentsOf,apiBaseLengths,@@ -51,6 +50,7 @@ import {type StateGateContext,type StatePlan,} from "./environmentStateGate.js";+import { StateWriteCutoffs } from "./environmentStateWriteCutoffs.js";class StateBudgetExhausted extends Error {}@@ -78,6 +78,8 @@ export class EnvironmentStateExtraction {readonly #writes: readonly StateRecordFacts[];readonly #writeWithin: ReturnType<typeof writeSpanCheck>;readonly #bases: ReadonlyMap<string, number>;+ #writeCutoffs: StateWriteCutoffs | undefined;+ readonly #initLimits = new Map<string, StateRecordFacts | undefined>();readonly #identity = new Map<string,{ readonly field: string; readonly basis: StateType["id_basis"] }@@ -115,6 +117,15 @@ export class EnvironmentStateExtraction {/** The extracted state, or null when the representation budget is exhausted. */run(): ExtractedState | null {try {+ if (this.#writes.length > 0)+ this.#writeCutoffs = new StateWriteCutoffs(+ this.#writes,+ this.#bases,+ (left, right) =>+ this.#position.get(left.ordinal)! <+ this.#position.get(right.ordinal)!,+ (bytes) => this.#retain(bytes),+ );const candidates = this.#candidates();for (const candidate of candidates.filter((entry) => entry.item))this.#planItem(candidate);@@ -187,10 +198,12 @@ export class EnvironmentStateExtraction {/** The first write whose root relates to the template's bounds its init recordings. */#initLimit(template: string): StateRecordFacts | undefined {- const root = rootOf(template, this.#bases);- return this.#writes.find((write) =>- rootsRelated(root, rootOf(write.path, this.#bases)),- );+ if (this.#writeCutoffs === undefined) return undefined;+ if (this.#initLimits.has(template)) return this.#initLimits.get(template);+ const first = this.#writeCutoffs.first(rootOf(template, this.#bases));+ this.#retain(ENTRY_BYTES + textBytes(template.length));+ this.#initLimits.set(template, first);+ return first;}/** Successful recordings that finished before the first related write began. */@@ -545,17 +558,19 @@ export class EnvironmentStateExtraction {/** Types and entities that at least one state-backed view renders; view values only for those views. */#result(documents: StateDocument[]): ExtractedState {- const backed = new Set(- [...this.#views.values()].flatMap((view) =>- view.shape === "item" || view.shape === "list" ? [view.tool] : [],- ),- );+ const backed = new Set<string>();+ const backedTypes = new Set<string>();+ for (const view of this.#views.values())+ if (view.shape === "item" || view.shape === "list") {+ this.#retain(ENTRY_BYTES);+ backed.add(view.tool);+ if (!backedTypes.has(view.type)) {+ this.#retain(ENTRY_BYTES);+ backedTypes.add(view.type);+ }+ }const types = [...this.#identity.entries()]- .filter(([type]) =>- this.#plans.some(- (plan) => plan.type === type && backed.has(plan.tool.name),- ),- )+ .filter(([type]) => backedTypes.has(type)).map(([type, { field, basis }]): StateType => ({collection_template: type,id_field: field,diff --git a/src/domain/analysisView/environmentStateRules.ts b/src/domain/analysisView/environmentStateRules.tsindex ce7006be0..ab0fb9aed 100644--- a/src/domain/analysisView/environmentStateRules.ts+++ b/src/domain/analysisView/environmentStateRules.ts@@ -266,18 +266,6 @@ export const rootOf = (return segments.slice(0, end + 1);};-/** Roots relate when they agree on their common length; template parameters match any segment. */-export const rootsRelated = (- left: readonly string[],- right: readonly string[],-): boolean =>- left- .slice(0, right.length)- .every(- (segment, index) =>- segment === right[index] || isParam(segment) || isParam(right[index]!),- );-/** Value bytes as retained state, charged before a value is kept. */export const jsonBytes = (value: JsonValue): number => {if (typeof value === "string") return textBytes(value.length);diff --git a/src/domain/analysisView/environmentStateView.test.ts b/src/domain/analysisView/environmentStateView.test.tsindex 7a0f5cdec..872aba189 100644--- a/src/domain/analysisView/environmentStateView.test.ts+++ b/src/domain/analysisView/environmentStateView.test.ts@@ -590,6 +590,64 @@ it("proves the identity field against every item response", () => {expect(state.entities.map((entity) => entity.id)).toEqual(["1", "2"]);});+it.each(["/api", "/api/items/1/update"])(+ "bounds a type by a write at a related root (%s)",+ (path) => {+ const state = stateOf(+ captureOf([+ ["GET", "/api/items/1", { id: 1, title: "initial" }],+ ["POST", path, {}],+ ["GET", "/api/items/1", { id: 1, title: "after" }],+ ]),+ );+ expect(state.types).toMatchObject([{ init_before_ordinal: 1 }]);+ expect(state.entities).toMatchObject([{ fields: { title: "initial" } }]);+ },+);++it("chooses the earliest related write in timeline order, across ancestor and descendant roots", () => {+ const parent = captureOf([+ ["GET", "/api/items/1", { id: 1, title: "initial" }],+ ["POST", "/api/items/1/update", {}],+ ["POST", "/api", {}],+ ["GET", "/api/items/1", { id: 1, title: "after" }],+ ]);+ const data = structuredClone(parent.normalizedResult) as {+ records: { reported: { startedDateTime: string } }[];+ };+ const first = data.records[1]!.reported.startedDateTime;+ data.records[1]!.reported.startedDateTime =+ data.records[2]!.reported.startedDateTime;+ data.records[2]!.reported.startedDateTime = first;+ const state = stateOf({ ...parent, normalizedResult: data });+ expect(state.types).toMatchObject([{ init_before_ordinal: 2 }]);+ expect(state.entities).toMatchObject([{ fields: { title: "initial" } }]);+});++it("bounds a parameterized namespace by concrete writes before publishing new items", () => {+ const result = projectAnalysisView(+ captureOf([+ ["GET", "/api/v1/repos/alice/alpha/issues/1", { id: 1 }],+ ["GET", "/api/v1/repos/bob/beta/issues/2", { id: 2 }],+ ["GET", "/api/v1/repos/carol/gamma/issues/3", { id: 3 }],+ ["POST", "/api/v1/repos/dana/delta/issues", { id: 4 }],+ ["GET", "/api/v1/repos/dana/delta/issues/4", { id: 4 }],+ ]),+ {+ kind: "environment-state",+ template_rules: [...ALL_RULES, "sibling-pair-variance"],+ },+ );+ if (!result.ok || result.value.kind !== "environment-state")+ throw new Error("Expected state");+ expect(result.value.entities.map((entity) => entity.id)).toEqual([+ "1",+ "2",+ "3",+ ]);+ expect(result.value.types).toMatchObject([{ init_before_ordinal: 3 }]);+});+it.each(["totalcount", "itemcount"])("recognizes a compact compound item total (%s)",(name) => {diff --git a/src/domain/analysisView/environmentStateWriteCutoffs.test.ts b/src/domain/analysisView/environmentStateWriteCutoffs.test.tsnew file mode 100644index 000000000..664a90034--- /dev/null+++ b/src/domain/analysisView/environmentStateWriteCutoffs.test.ts@@ -0,0 +1,86 @@+import { expect, it } from "vitest";+import {+ isParam,+ rootOf,+ type StateRecordFacts,+} from "./environmentStateRules.js";+import { StateWriteCutoffs } from "./environmentStateWriteCutoffs.js";++const write = (path: string, ordinal: number): StateRecordFacts => ({+ path,+ ordinal,+ method: "POST",+ query: undefined,+ status: 200,+ start: ordinal,+ end: ordinal,+});++it("matches prefix and parameter semantics against an independent linear oracle", () => {+ const bases = new Map([+ ["api", 8],+ ["other", 8],+ ]);+ const paths = [+ "/api/a",+ "/api/b/c",+ "/api/{owner}/d",+ "/other/x",+ "/api/a/{item}/e",+ "",+ ];+ const writes = paths.map(write);+ const patterns: string[][] = [[]];+ let level: string[][] = [[]];+ for (let depth = 0; depth < 4; depth += 1) {+ level = level.flatMap((prefix) =>+ ["api", "a", "b", "d", "{key}"].map((segment) => [...prefix, segment]),+ );+ patterns.push(...level);+ }+ for (const selection of [+ writes,+ writes.slice(0, -1),+ [...writes].reverse(),+ ]) {+ const index = new StateWriteCutoffs(+ selection,+ bases,+ (a, b) => a.start < b.start,+ () => {},+ );+ for (const root of patterns) {+ const expected = [...selection]+ .sort((a, b) => a.start - b.start)+ .find((record) => {+ const candidate = rootOf(record.path, bases);+ const shared = Math.min(root.length, candidate.length);+ return Array.from({ length: shared }, (_, i) => i).every(+ (i) =>+ root[i] === candidate[i] ||+ isParam(root[i]!) ||+ isParam(candidate[i]!),+ );+ });+ expect(index.first(root), JSON.stringify(root)).toBe(expected);+ }+ }+});++it("handles deep roots iteratively within admitted storage", () => {+ const root = ["api", ...Array<string>(12_000).fill("a")];+ const record = write(`/${root.join("/")}`, 1);+ let admitted = 0;+ const index = new StateWriteCutoffs(+ [record],+ new Map([["api", root.length]]),+ () => false,+ (bytes) => {+ admitted += bytes;+ if (admitted > 16 * 1024 * 1024)+ throw new Error("Test storage budget exceeded");+ },+ );+ expect(index.first(root)).toBe(record);+ expect(index.first(["other"])).toBeUndefined();+});diff --git a/src/domain/analysisView/environmentStateWriteCutoffs.ts b/src/domain/analysisView/environmentStateWriteCutoffs.tsnew file mode 100644index 000000000..91f148c36--- /dev/null+++ b/src/domain/analysisView/environmentStateWriteCutoffs.ts@@ -0,0 +1,101 @@+import { ENTRY_BYTES, textBytes } from "./webNetworkRouteTemplates.js";+import {+ isParam,+ rootOf,+ type StateRecordFacts,+} from "./environmentStateRules.js";++interface WriteRootNode {+ readonly children: Map<string, WriteRootNode>;+ wildcard: WriteRootNode | undefined;+ exact: StateRecordFacts | undefined;+ first: StateRecordFacts | undefined;+}++const node = (): WriteRootNode => ({+ children: new Map(),+ wildcard: undefined,+ exact: undefined,+ first: undefined,+});++/** Parsed write roots with the earliest write at each prefix and subtree. */+export class StateWriteCutoffs {+ readonly #root: WriteRootNode;++ constructor(+ writes: readonly StateRecordFacts[],+ bases: ReadonlyMap<string, number>,+ private readonly before: (+ left: StateRecordFacts,+ right: StateRecordFacts,+ ) => boolean,+ retain: (bytes: number) => void,+ ) {+ // Admission covers node maps, references and the iterative lookup frontier.+ retain(ENTRY_BYTES * 4);+ this.#root = node();+ for (const write of writes) {+ let current = this.#root;+ current.first = this.#earlier(current.first, write);+ for (const segment of rootOf(write.path, bases)) {+ const wildcard = isParam(segment);+ let child = wildcard ? current.wildcard : current.children.get(segment);+ if (child === undefined) {+ retain(ENTRY_BYTES * 4 + textBytes(segment.length));+ child = node();+ if (wildcard) current.wildcard = child;+ else current.children.set(segment, child);+ }+ current = child;+ current.first = this.#earlier(current.first, write);+ }+ current.exact = this.#earlier(current.exact, write);+ }+ }++ #earlier(+ left: StateRecordFacts | undefined,+ right: StateRecordFacts | undefined,+ ): StateRecordFacts | undefined {+ if (right === undefined) return left;+ return left === undefined || this.before(right, left) ? right : left;+ }++ /** Earliest ancestor/descendant match; parameters on either side match any one segment. */+ first(root: readonly string[]): StateRecordFacts | undefined {+ const nodes = [this.#root];+ const depths = [0];+ let first: StateRecordFacts | undefined;+ while (nodes.length > 0) {+ const current = nodes.pop()!;+ const depth = depths.pop()!;+ first = this.#earlier(first, current.exact);+ if (+ depth === root.length ||+ (depth === root.length - 1 && isParam(root[depth]!))+ ) {+ first = this.#earlier(first, current.first);+ continue;+ }+ const segment = root[depth]!;+ if (isParam(segment)) {+ for (const child of current.children.values()) {+ nodes.push(child);+ depths.push(depth + 1);+ }+ } else {+ const child = current.children.get(segment);+ if (child !== undefined) {+ nodes.push(child);+ depths.push(depth + 1);+ }+ }+ if (current.wildcard !== undefined) {+ nodes.push(current.wildcard);+ depths.push(depth + 1);+ }+ }+ return first;+ }+}