Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

perf(environment): index observed-state write cutoffs (#5350)

gensokyo-devd148ce090512
diff --git a/docs/specs/agent-environments/state-view/verification.md b/docs/specs/agent-environments/state-view/verification.md
index 26101f651..48db9935b 100644
--- a/docs/specs/agent-environments/state-view/verification.md
+++ b/docs/specs/agent-environments/state-view/verification.md
@@ -91,7 +91,7 @@ workflow: 22.836 s / 19.880 CPU s / 1,072.43 MiB; actual browser-state workflow:
exhausts an isolated decoder heap and verifies that MCP remains usable; no
workspace OOM occurred.
-Before/after used immutable pre-fix `29e745923` and the final source, the same
+The first follow-up benchmark used pre-fix `29e745923` and `6ad63bcc2`, the same
completed Evidence files, Node and CPU affinity. One fresh process per revision,
with a first call and two warm repeats; filesystem caches were not cleared.
Complete normalized output digests matched for both synthetic projection tasks.
@@ -116,3 +116,41 @@ No new macOS/Windows runtime, state serving, AgentEnv data-plane or write-effect
claim is made. A type first observed only after an unrelated-root write may
still lack the earlier observation needed to detect its effect. Item-shaped
and delete-only document paths remain recorded, as specified.
+
+## Post-merge write-cutoff follow-up
+
+Base `3fca2e0ca` includes #5319 and the validated immutable-capture reuse from
+#5349. The remaining review finding was repeated write-root parsing for each
+read type and again during output assembly. One admitted prefix/wildcard index
+now owns parsed write roots, with per-projection positive/negative cutoff
+caching. Backed type membership is collected once, replacing another plan scan.
+The old linear matcher is removed. Prefix, parameter, redacted-root and timeline
+semantics are preserved; no cross-request cache or new authority is introduced.
+
+- Source checks: 41 files, 508 passed, 3 existing skips, including independent
+ matching-oracle and 12,000-segment iterative-lookup checks.
+- Serial static/build/artifact gate: 11 tasks passed, 197.356 s wall /
+ 197.195 CPU s / 4,744.92 MiB peak process-family RSS.
+- Both selected compiled state CLI/MCP cases passed after rebuilding.
+- Live Gitea HAR workflows and fresh actual Chromium capture passed again:
+ initial values, recorded fallbacks, CLI/default stdio parity and six exact
+ HTTP replays. HAR workflow: 22.834 s wall / 19.623 CPU s / 1,067.36 MiB;
+ browser workflow: 22.858 s / 20.912 CPU s / 1,546.59 MiB. Owned processes stopped.
+
+The same completed Evidence with 1,000 collection reads and 1,000 unrelated
+writes was projected before and after on one CPU. Two larger full-HAR inputs
+(3,000/4,000 total records) were refused by the existing 96 MiB admission budget;
+the limit was retained. One fresh process per revision, first call plus one
+warm repeat, filesystem caches not cleared:
+
+| Complete file-to-state projection | Main before | Follow-up |
+| -------------------------------------------------------- | ------------------- | ------------------- |
+| First call wall / CPU (ms) | 1,727.89 / 1,727.36 | 1,244.82 / 1,244.05 |
+| Warm call wall / CPU (ms) | 1,300.84 / 1,300.42 | 624.01 / 623.89 |
+| Process wall / CPU (s), including imports and both calls | 4.690 / 4.363 | 3.370 / 3.062 |
+| Sampled process-family peak RSS (MiB) | 365.84 | 360.03 |
+
+Parent Evidence IDs and complete normalized output digests were identical.
+RSS sampling is every 50 ms, includes descendants and can double-count shared
+pages; these few runs do not establish a general memory reduction. Model token
+usage remains unmeasured. No workspace OOM occurred.
diff --git a/src/domain/analysisView/EnvironmentStateExtraction.ts b/src/domain/analysisView/EnvironmentStateExtraction.ts
index 7ed3ef79e..11f457edd 100644
--- a/src/domain/analysisView/EnvironmentStateExtraction.ts
+++ b/src/domain/analysisView/EnvironmentStateExtraction.ts
@@ -20,7 +20,6 @@ import {
locateItem,
locateList,
rootOf,
- rootsRelated,
scalarText,
segmentsOf,
apiBaseLengths,
@@ -51,6 +50,7 @@ import {
type StateGateContext,
type StatePlan,
} from "./environmentStateGate.js";
+import { StateWriteCutoffs } from "./environmentStateWriteCutoffs.js";
class StateBudgetExhausted extends Error {}
@@ -78,6 +78,8 @@ export class EnvironmentStateExtraction {
readonly #writes: readonly StateRecordFacts[];
readonly #writeWithin: ReturnType<typeof writeSpanCheck>;
readonly #bases: ReadonlyMap<string, number>;
+ #writeCutoffs: StateWriteCutoffs | undefined;
+ readonly #initLimits = new Map<string, StateRecordFacts | undefined>();
readonly #identity = new Map<
string,
{ readonly field: string; readonly basis: StateType["id_basis"] }
@@ -115,6 +117,15 @@ export class EnvironmentStateExtraction {
/** The extracted state, or null when the representation budget is exhausted. */
run(): ExtractedState | null {
try {
+ if (this.#writes.length > 0)
+ this.#writeCutoffs = new StateWriteCutoffs(
+ this.#writes,
+ this.#bases,
+ (left, right) =>
+ this.#position.get(left.ordinal)! <
+ this.#position.get(right.ordinal)!,
+ (bytes) => this.#retain(bytes),
+ );
const candidates = this.#candidates();
for (const candidate of candidates.filter((entry) => entry.item))
this.#planItem(candidate);
@@ -187,10 +198,12 @@ export class EnvironmentStateExtraction {
/** The first write whose root relates to the template's bounds its init recordings. */
#initLimit(template: string): StateRecordFacts | undefined {
- const root = rootOf(template, this.#bases);
- return this.#writes.find((write) =>
- rootsRelated(root, rootOf(write.path, this.#bases)),
- );
+ if (this.#writeCutoffs === undefined) return undefined;
+ if (this.#initLimits.has(template)) return this.#initLimits.get(template);
+ const first = this.#writeCutoffs.first(rootOf(template, this.#bases));
+ this.#retain(ENTRY_BYTES + textBytes(template.length));
+ this.#initLimits.set(template, first);
+ return first;
}
/** Successful recordings that finished before the first related write began. */
@@ -545,17 +558,19 @@ export class EnvironmentStateExtraction {
/** Types and entities that at least one state-backed view renders; view values only for those views. */
#result(documents: StateDocument[]): ExtractedState {
- const backed = new Set(
- [...this.#views.values()].flatMap((view) =>
- view.shape === "item" || view.shape === "list" ? [view.tool] : [],
- ),
- );
+ const backed = new Set<string>();
+ const backedTypes = new Set<string>();
+ for (const view of this.#views.values())
+ if (view.shape === "item" || view.shape === "list") {
+ this.#retain(ENTRY_BYTES);
+ backed.add(view.tool);
+ if (!backedTypes.has(view.type)) {
+ this.#retain(ENTRY_BYTES);
+ backedTypes.add(view.type);
+ }
+ }
const types = [...this.#identity.entries()]
- .filter(([type]) =>
- this.#plans.some(
- (plan) => plan.type === type && backed.has(plan.tool.name),
- ),
- )
+ .filter(([type]) => backedTypes.has(type))
.map(([type, { field, basis }]): StateType => ({
collection_template: type,
id_field: field,
diff --git a/src/domain/analysisView/environmentStateRules.ts b/src/domain/analysisView/environmentStateRules.ts
index ce7006be0..ab0fb9aed 100644
--- a/src/domain/analysisView/environmentStateRules.ts
+++ b/src/domain/analysisView/environmentStateRules.ts
@@ -266,18 +266,6 @@ export const rootOf = (
return segments.slice(0, end + 1);
};
-/** Roots relate when they agree on their common length; template parameters match any segment. */
-export const rootsRelated = (
- left: readonly string[],
- right: readonly string[],
-): boolean =>
- left
- .slice(0, right.length)
- .every(
- (segment, index) =>
- segment === right[index] || isParam(segment) || isParam(right[index]!),
- );
-
/** Value bytes as retained state, charged before a value is kept. */
export const jsonBytes = (value: JsonValue): number => {
if (typeof value === "string") return textBytes(value.length);
diff --git a/src/domain/analysisView/environmentStateView.test.ts b/src/domain/analysisView/environmentStateView.test.ts
index 7a0f5cdec..872aba189 100644
--- a/src/domain/analysisView/environmentStateView.test.ts
+++ b/src/domain/analysisView/environmentStateView.test.ts
@@ -590,6 +590,64 @@ it("proves the identity field against every item response", () => {
expect(state.entities.map((entity) => entity.id)).toEqual(["1", "2"]);
});
+it.each(["/api", "/api/items/1/update"])(
+ "bounds a type by a write at a related root (%s)",
+ (path) => {
+ const state = stateOf(
+ captureOf([
+ ["GET", "/api/items/1", { id: 1, title: "initial" }],
+ ["POST", path, {}],
+ ["GET", "/api/items/1", { id: 1, title: "after" }],
+ ]),
+ );
+ expect(state.types).toMatchObject([{ init_before_ordinal: 1 }]);
+ expect(state.entities).toMatchObject([{ fields: { title: "initial" } }]);
+ },
+);
+
+it("chooses the earliest related write in timeline order, across ancestor and descendant roots", () => {
+ const parent = captureOf([
+ ["GET", "/api/items/1", { id: 1, title: "initial" }],
+ ["POST", "/api/items/1/update", {}],
+ ["POST", "/api", {}],
+ ["GET", "/api/items/1", { id: 1, title: "after" }],
+ ]);
+ const data = structuredClone(parent.normalizedResult) as {
+ records: { reported: { startedDateTime: string } }[];
+ };
+ const first = data.records[1]!.reported.startedDateTime;
+ data.records[1]!.reported.startedDateTime =
+ data.records[2]!.reported.startedDateTime;
+ data.records[2]!.reported.startedDateTime = first;
+ const state = stateOf({ ...parent, normalizedResult: data });
+ expect(state.types).toMatchObject([{ init_before_ordinal: 2 }]);
+ expect(state.entities).toMatchObject([{ fields: { title: "initial" } }]);
+});
+
+it("bounds a parameterized namespace by concrete writes before publishing new items", () => {
+ const result = projectAnalysisView(
+ captureOf([
+ ["GET", "/api/v1/repos/alice/alpha/issues/1", { id: 1 }],
+ ["GET", "/api/v1/repos/bob/beta/issues/2", { id: 2 }],
+ ["GET", "/api/v1/repos/carol/gamma/issues/3", { id: 3 }],
+ ["POST", "/api/v1/repos/dana/delta/issues", { id: 4 }],
+ ["GET", "/api/v1/repos/dana/delta/issues/4", { id: 4 }],
+ ]),
+ {
+ kind: "environment-state",
+ template_rules: [...ALL_RULES, "sibling-pair-variance"],
+ },
+ );
+ if (!result.ok || result.value.kind !== "environment-state")
+ throw new Error("Expected state");
+ expect(result.value.entities.map((entity) => entity.id)).toEqual([
+ "1",
+ "2",
+ "3",
+ ]);
+ expect(result.value.types).toMatchObject([{ init_before_ordinal: 3 }]);
+});
+
it.each(["totalcount", "itemcount"])(
"recognizes a compact compound item total (%s)",
(name) => {
diff --git a/src/domain/analysisView/environmentStateWriteCutoffs.test.ts b/src/domain/analysisView/environmentStateWriteCutoffs.test.ts
new file mode 100644
index 000000000..664a90034
--- /dev/null
+++ b/src/domain/analysisView/environmentStateWriteCutoffs.test.ts
@@ -0,0 +1,86 @@
+import { expect, it } from "vitest";
+import {
+ isParam,
+ rootOf,
+ type StateRecordFacts,
+} from "./environmentStateRules.js";
+import { StateWriteCutoffs } from "./environmentStateWriteCutoffs.js";
+
+const write = (path: string, ordinal: number): StateRecordFacts => ({
+ path,
+ ordinal,
+ method: "POST",
+ query: undefined,
+ status: 200,
+ start: ordinal,
+ end: ordinal,
+});
+
+it("matches prefix and parameter semantics against an independent linear oracle", () => {
+ const bases = new Map([
+ ["api", 8],
+ ["other", 8],
+ ]);
+ const paths = [
+ "/api/a",
+ "/api/b/c",
+ "/api/{owner}/d",
+ "/other/x",
+ "/api/a/{item}/e",
+ "",
+ ];
+ const writes = paths.map(write);
+ const patterns: string[][] = [[]];
+ let level: string[][] = [[]];
+ for (let depth = 0; depth < 4; depth += 1) {
+ level = level.flatMap((prefix) =>
+ ["api", "a", "b", "d", "{key}"].map((segment) => [...prefix, segment]),
+ );
+ patterns.push(...level);
+ }
+ for (const selection of [
+ writes,
+ writes.slice(0, -1),
+ [...writes].reverse(),
+ ]) {
+ const index = new StateWriteCutoffs(
+ selection,
+ bases,
+ (a, b) => a.start < b.start,
+ () => {},
+ );
+ for (const root of patterns) {
+ const expected = [...selection]
+ .sort((a, b) => a.start - b.start)
+ .find((record) => {
+ const candidate = rootOf(record.path, bases);
+ const shared = Math.min(root.length, candidate.length);
+ return Array.from({ length: shared }, (_, i) => i).every(
+ (i) =>
+ root[i] === candidate[i] ||
+ isParam(root[i]!) ||
+ isParam(candidate[i]!),
+ );
+ });
+ expect(index.first(root), JSON.stringify(root)).toBe(expected);
+ }
+ }
+});
+
+it("handles deep roots iteratively within admitted storage", () => {
+ const root = ["api", ...Array<string>(12_000).fill("a")];
+ const record = write(`/${root.join("/")}`, 1);
+ let admitted = 0;
+ const index = new StateWriteCutoffs(
+ [record],
+ new Map([["api", root.length]]),
+ () => false,
+ (bytes) => {
+ admitted += bytes;
+ if (admitted > 16 * 1024 * 1024)
+ throw new Error("Test storage budget exceeded");
+ },
+ );
+ expect(index.first(root)).toBe(record);
+ expect(index.first(["other"])).toBeUndefined();
+});
diff --git a/src/domain/analysisView/environmentStateWriteCutoffs.ts b/src/domain/analysisView/environmentStateWriteCutoffs.ts
new file mode 100644
index 000000000..91f148c36
--- /dev/null
+++ b/src/domain/analysisView/environmentStateWriteCutoffs.ts
@@ -0,0 +1,101 @@
+import { ENTRY_BYTES, textBytes } from "./webNetworkRouteTemplates.js";
+import {
+ isParam,
+ rootOf,
+ type StateRecordFacts,
+} from "./environmentStateRules.js";
+
+interface WriteRootNode {
+ readonly children: Map<string, WriteRootNode>;
+ wildcard: WriteRootNode | undefined;
+ exact: StateRecordFacts | undefined;
+ first: StateRecordFacts | undefined;
+}
+
+const node = (): WriteRootNode => ({
+ children: new Map(),
+ wildcard: undefined,
+ exact: undefined,
+ first: undefined,
+});
+
+/** Parsed write roots with the earliest write at each prefix and subtree. */
+export class StateWriteCutoffs {
+ readonly #root: WriteRootNode;
+
+ constructor(
+ writes: readonly StateRecordFacts[],
+ bases: ReadonlyMap<string, number>,
+ private readonly before: (
+ left: StateRecordFacts,
+ right: StateRecordFacts,
+ ) => boolean,
+ retain: (bytes: number) => void,
+ ) {
+ // Admission covers node maps, references and the iterative lookup frontier.
+ retain(ENTRY_BYTES * 4);
+ this.#root = node();
+ for (const write of writes) {
+ let current = this.#root;
+ current.first = this.#earlier(current.first, write);
+ for (const segment of rootOf(write.path, bases)) {
+ const wildcard = isParam(segment);
+ let child = wildcard ? current.wildcard : current.children.get(segment);
+ if (child === undefined) {
+ retain(ENTRY_BYTES * 4 + textBytes(segment.length));
+ child = node();
+ if (wildcard) current.wildcard = child;
+ else current.children.set(segment, child);
+ }
+ current = child;
+ current.first = this.#earlier(current.first, write);
+ }
+ current.exact = this.#earlier(current.exact, write);
+ }
+ }
+
+ #earlier(
+ left: StateRecordFacts | undefined,
+ right: StateRecordFacts | undefined,
+ ): StateRecordFacts | undefined {
+ if (right === undefined) return left;
+ return left === undefined || this.before(right, left) ? right : left;
+ }
+
+ /** Earliest ancestor/descendant match; parameters on either side match any one segment. */
+ first(root: readonly string[]): StateRecordFacts | undefined {
+ const nodes = [this.#root];
+ const depths = [0];
+ let first: StateRecordFacts | undefined;
+ while (nodes.length > 0) {
+ const current = nodes.pop()!;
+ const depth = depths.pop()!;
+ first = this.#earlier(first, current.exact);
+ if (
+ depth === root.length ||
+ (depth === root.length - 1 && isParam(root[depth]!))
+ ) {
+ first = this.#earlier(first, current.first);
+ continue;
+ }
+ const segment = root[depth]!;
+ if (isParam(segment)) {
+ for (const child of current.children.values()) {
+ nodes.push(child);
+ depths.push(depth + 1);
+ }
+ } else {
+ const child = current.children.get(segment);
+ if (child !== undefined) {
+ nodes.push(child);
+ depths.push(depth + 1);
+ }
+ }
+ if (current.wildcard !== undefined) {
+ nodes.push(current.wildcard);
+ depths.push(depth + 1);
+ }
+ }
+ return first;
+ }
+}