Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

test(perf): measure genuine SAZ selection and retained recovery (#3584)

Ysopwnable357996cc89f1
diff --git a/experiments/analysis-efficiency/README.md b/experiments/analysis-efficiency/README.md
index 2ca9da3a4..295a2aa76 100644
--- a/experiments/analysis-efficiency/README.md
+++ b/experiments/analysis-efficiency/README.md
@@ -1,5 +1,11 @@
# Analysis efficiency measurements
+For genuine Fiddler captures, see the [SAZ selection checkpoint](saz-selection-workflows.md).
+`measure.mjs --saz-selection baseline|focused|retained` compares default complete
+delivery (including retained-reference recovery), caller-selected ordinals, and
+summary/preview/complete/repeated pages. Each failed RPC remains in task costs.
+The body question uses stored bytes, not decoded HTTP application payloads.
+
Tracks the experiments for [#2016](https://github.com/morluto/rea/issues/2016).
The production CLI/MCP contracts and defaults remain the compatibility baseline.
Downloaded artifacts, runtime copies, exported Evidence, transcripts and detailed
diff --git a/experiments/analysis-efficiency/agent-trial.py b/experiments/analysis-efficiency/agent-trial.py
index 365337a06..810fc5590 100644
--- a/experiments/analysis-efficiency/agent-trial.py
+++ b/experiments/analysis-efficiency/agent-trial.py
@@ -181,6 +181,7 @@ def main():
'inspect_android_class', 'inspect_android_method',
'inspect_firmware_regions', 'inspect_managed_artifact',
'inspect_sqlite_database', 'inspect_pcap_capture',
+ 'inspect_web_network_capture',
'inspect_packet_capture', 'inspect_packet_fields',
'decode_signal_recording', 'inspect_signal_decoders',
'inspect_managed_members', 'inspect_managed_native_boundaries',
diff --git a/experiments/analysis-efficiency/corpus.json b/experiments/analysis-efficiency/corpus.json
index 983b81799..03d4fa18c 100644
--- a/experiments/analysis-efficiency/corpus.json
+++ b/experiments/analysis-efficiency/corpus.json
@@ -373,6 +373,66 @@
"reference": "https://raw.githubusercontent.com/sigrokproject/sigrok-dumps/0ad13477abc959d37fc9a5acbd23901c371c9c76/i2c/wii_nunchuk/README",
"license_scope": "Repository README declares dumps public domain unless otherwise noted; selected READMEs state no exception; decoder software licensing is separate."
}
+ },
+ {
+ "id": "xbox-clear_nomp.saz",
+ "file": "clear_nomp.saz",
+ "bytes": 43515,
+ "sha256": "3e808663a99cbb9160186f7419633248200fbc6863d2c897f5d4d9972dde0375",
+ "provenance": {
+ "project": "Microsoft Xbox Live Trace Analyzer",
+ "version": "ad880ff00b26c4cae5e3c8864d3df438a03890ba",
+ "url": "https://raw.githubusercontent.com/microsoft/xbox-live-trace-analyzer/ad880ff00b26c4cae5e3c8864d3df438a03890ba/Source/TestData/CertWarnings/clear_nomp.saz",
+ "git_blob": "2f1ba66dacb8f8bac290cab59dff8fe750a46ce7",
+ "license": "MIT; upstream LICENSE.md",
+ "producer_archive_comment": "Fiddler (v4.5.1.2) Session Archive. See http://fiddler2.com"
+ },
+ "saz_question": {
+ "record_ordinals": [12, 0, 6],
+ "total_records": 13,
+ "members": 42,
+ "expanded_bytes": 62609
+ }
+ },
+ {
+ "id": "xbox-clear_mp_cs10.saz",
+ "file": "clear_mp_cs10.saz",
+ "bytes": 5223519,
+ "sha256": "4021b201848e4feea3402c380fda5d2df43fd0bf1c72ea550e60f2da3b3d5b44",
+ "provenance": {
+ "project": "Microsoft Xbox Live Trace Analyzer",
+ "version": "ad880ff00b26c4cae5e3c8864d3df438a03890ba",
+ "url": "https://raw.githubusercontent.com/microsoft/xbox-live-trace-analyzer/ad880ff00b26c4cae5e3c8864d3df438a03890ba/Source/TestData/CertWarnings/clear_mp_cs10.saz",
+ "git_blob": "dac534d625594c7a1d57b0b12ed12ea9c6cb6a5b",
+ "license": "MIT; upstream LICENSE.md",
+ "producer_archive_comment": "Fiddler (v4.6.2.0) Session Archive. See https://fiddler2.com"
+ },
+ "saz_question": {
+ "record_ordinals": [1310, 0, 655],
+ "total_records": 1311,
+ "members": 3939,
+ "expanded_bytes": 9223820
+ }
+ },
+ {
+ "id": "xbox-x49_mp_cs21.saz",
+ "file": "x49_mp_cs21.saz",
+ "bytes": 572224,
+ "sha256": "97cdca8b0608e3f11995fd64b64390b9fd8d8e3efcac89f517f24b44968047f6",
+ "provenance": {
+ "project": "Microsoft Xbox Live Trace Analyzer",
+ "version": "ad880ff00b26c4cae5e3c8864d3df438a03890ba",
+ "url": "https://raw.githubusercontent.com/microsoft/xbox-live-trace-analyzer/ad880ff00b26c4cae5e3c8864d3df438a03890ba/Source/TestData/CertWarnings/x49_mp_cs21.saz",
+ "git_blob": "e7794c44892ca4fa1729fa012e698e8284337f15",
+ "license": "MIT; upstream LICENSE.md",
+ "producer_archive_comment": "Fiddler (v4.6.2.26757) Session Archive. See https://fiddler2.com"
+ },
+ "saz_question": {
+ "record_ordinals": [102, 0, 51],
+ "total_records": 103,
+ "members": 315,
+ "expanded_bytes": 1301927
+ }
}
]
}
diff --git a/experiments/analysis-efficiency/measure.mjs b/experiments/analysis-efficiency/measure.mjs
index 20f1e7d61..88f5dd1ff 100644
--- a/experiments/analysis-efficiency/measure.mjs
+++ b/experiments/analysis-efficiency/measure.mjs
@@ -14,6 +14,7 @@ import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { Ajv2020 } from "ajv/dist/2020.js";
import { PACKET_ARTIFACTS, measurePacketEntry } from "./packet-entry.mjs";
import { SIGNAL_ARTIFACTS, measureSignalEntry } from "./signal-entry.mjs";
+import { SAZ_ARTIFACTS, measureSazEntry } from "./saz-entry.mjs";
const root = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
const { values } = parseArgs({
@@ -33,6 +34,7 @@ const { values } = parseArgs({
"sqlite-selection": { type: "string", default: "all" },
"packet-selection": { type: "string", default: "baseline" },
"signal-selection": { type: "string", default: "baseline" },
+ "saz-selection": { type: "string", default: "baseline" },
},
});
if (
@@ -43,6 +45,11 @@ if (
!isAbsolute(values.path) ||
!isAbsolute(values.output) ||
!["complete", "summary"].includes(values.delivery) ||
+ !["baseline", "focused", "retained"].includes(values["saz-selection"]) ||
+ (!Object.hasOwn(SAZ_ARTIFACTS, values.artifact) &&
+ values["saz-selection"] !== "baseline") ||
+ (Object.hasOwn(SAZ_ARTIFACTS, values.artifact) &&
+ values.delivery !== "complete") ||
!["baseline", "focused", "discovery", "pages"].includes(
values["signal-selection"],
) ||
@@ -174,6 +181,12 @@ const fingerprints = {
signal_oracle_source_sha256: await fileSha(
fileURLToPath(new URL("./signal-oracle.py", import.meta.url)),
),
+ saz_workflow_source_sha256: await fileSha(
+ fileURLToPath(new URL("./saz-entry.mjs", import.meta.url)),
+ ),
+ saz_oracle_source_sha256: await fileSha(
+ fileURLToPath(new URL("./saz-oracle.py", import.meta.url)),
+ ),
runtime_files_sha256: sha(JSON.stringify(runtimeFiles)),
runtime_dispatcher_sha256: await fileSha(
resolve(values["runtime-root"], "scripts/rea.mjs"),
@@ -754,6 +767,12 @@ const inspectPcapTarget = async () => {
};
const workflows = {
+ ...Object.fromEntries(
+ Object.keys(SAZ_ARTIFACTS).map((artifact) => [
+ artifact,
+ () => measureSazEntry({ values, call, inputSha }),
+ ]),
+ ),
...Object.fromEntries(
Object.keys(SIGNAL_ARTIFACTS).map((artifact) => [
artifact,
@@ -1188,6 +1207,9 @@ const report = {
artifact_sha256: inputSha,
artifact_sha256_after: finalInputSha,
delivery: values.delivery,
+ saz_selection: Object.hasOwn(SAZ_ARTIFACTS, values.artifact)
+ ? values["saz-selection"]
+ : null,
sdk_schema_mode: values.schemas,
retained_verification: values["verify-retained"],
native_question: NATIVE_ARTIFACTS.includes(values.artifact)
diff --git a/experiments/analysis-efficiency/review-saz.mjs b/experiments/analysis-efficiency/review-saz.mjs
new file mode 100644
index 000000000..b836d807c
--- /dev/null
+++ b/experiments/analysis-efficiency/review-saz.mjs
@@ -0,0 +1,266 @@
+import assert from "node:assert/strict";
+import { readFile, writeFile } from "node:fs/promises";
+import { resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import { isDeepStrictEqual, parseArgs } from "node:util";
+import {
+ sazRequest,
+ verifySazRecords,
+ verifySazOracleIdentity,
+ sazEvidenceParameters,
+} from "./saz-entry.mjs";
+
+const { values } = parseArgs({
+ options: {
+ trial: { type: "string" },
+ oracle: { type: "string" },
+ "runtime-root": { type: "string" },
+ artifact: { type: "string" },
+ selection: { type: "string" },
+ },
+});
+if (
+ !values.trial ||
+ !values.oracle ||
+ !values["runtime-root"] ||
+ !["baseline", "focused"].includes(values.selection)
+)
+ throw new Error(
+ "Use --trial DIR --oracle JSON --runtime-root DIR --artifact ID --selection baseline|focused",
+ );
+const load = async (file) =>
+ JSON.parse(await readFile(resolve(values.trial, file), "utf8"));
+const settings = await load("trial-settings.json"),
+ summary = await load("summary.json");
+const oracle = JSON.parse(await readFile(values.oracle, "utf8"));
+verifySazOracleIdentity(values.artifact, oracle);
+const runtime = (file) =>
+ pathToFileURL(resolve(values["runtime-root"], file)).href;
+const { parseEvidence } = await import(runtime("dist/domain/evidence.js"));
+const { inspectWebNetworkCaptureInputSchema, webNetworkCaptureSchema } =
+ await import(runtime("dist/domain/webNetworkCapture.js"));
+const { inspectAnalysisViewInputSchema, analysisViewResultSchema } =
+ await import(runtime("dist/domain/analysisView/analysisView.js"));
+const calls = (await readFile(resolve(values.trial, "events.jsonl"), "utf8"))
+ .split("\n")
+ .filter(Boolean)
+ .map(JSON.parse)
+ .filter(
+ (event) =>
+ event.type === "item.completed" && event.item?.type === "mcp_tool_call",
+ )
+ .map((event) => event.item);
+const planned = inspectWebNetworkCaptureInputSchema.parse(
+ sazRequest(
+ values.artifact,
+ settings.selected_artifact_path,
+ values.selection,
+ ),
+);
+const reasons = [],
+ workflowReasons = [],
+ authenticated = new Map(),
+ parents = new Set(),
+ conflicts = new Set();
+let transportRecovery = false;
+for (const call of calls) {
+ try {
+ assert.equal(call.server, "rea");
+ assert.equal(call.status, "completed");
+ assert.ok(!call.error, "RPC did not provide observations");
+ const text = call.result.content.filter((item) => item.type === "text");
+ assert.equal(text.length, 1, "Ambiguous Evidence text carrier");
+ let evidence = JSON.parse(text[0].text);
+ if (call.result.isError) {
+ const limits = evidence.error?.details?.reported_limits;
+ assert.equal(call.tool, "inspect_web_network_capture");
+ assert.equal(evidence.error?.code, "resource_constraint");
+ assert.equal(
+ evidence.error.details.operation,
+ "inspect_web_network_capture",
+ );
+ assert.equal(limits?.boundary, "mcp-response");
+ const input = inspectWebNetworkCaptureInputSchema.parse(call.arguments);
+ assert.deepEqual(input, planned);
+ const source = inspectAnalysisViewInputSchema.parse({
+ source: limits.evidence_reference,
+ view: { kind: "summary" },
+ }).source;
+ assert.equal(source.kind, "retained-evidence");
+ parents.add(source.evidence_id);
+ transportRecovery = true;
+ continue;
+ }
+ if (call.result.structuredContent !== undefined)
+ assert.deepEqual(evidence, call.result.structuredContent);
+ evidence = parseEvidence(evidence);
+ assert.equal(evidence.subject.local_path, settings.selected_artifact_path);
+ assert.equal(evidence.subject.digest.sha256, oracle.sha256);
+ let records;
+ if (call.tool === "inspect_web_network_capture") {
+ const { detail, ...arguments_ } = call.arguments;
+ const input = inspectWebNetworkCaptureInputSchema.parse(arguments_);
+ assert.equal(input.capture_path, settings.selected_artifact_path);
+ assert.equal(input.format, "saz");
+ if (!isDeepStrictEqual(input, planned))
+ workflowReasons.push(
+ "Inspection selection differs from planned arguments",
+ );
+ if (detail === "summary") {
+ workflowReasons.push(
+ "Summary delivery differs from the fixed complete-delivery plan",
+ );
+ assert.equal(evidence.operation, "inspect_analysis_view");
+ const report = analysisViewResultSchema.parse(
+ evidence.normalized_result,
+ );
+ assert.equal(report.kind, "summary");
+ assert.equal(report.parent_operation, "inspect_web_network_capture");
+ assert.equal(report.summary.format, "saz");
+ assert.equal(report.summary.total_records, oracle.total_records);
+ assert.deepEqual(evidence.parameters, {
+ parent_evidence_id: report.parent_evidence_id,
+ view: { kind: "summary" },
+ });
+ parents.add(report.parent_evidence_id);
+ authenticated.set(evidence.evidence_id, []);
+ continue;
+ }
+ assert.ok(detail === undefined || detail === "complete");
+ assert.equal(evidence.operation, "inspect_web_network_capture");
+ assert.deepEqual(
+ evidence.parameters,
+ sazEvidenceParameters(input, oracle.total_records),
+ );
+ const report = webNetworkCaptureSchema.parse(evidence.normalized_result);
+ assert.equal(report.total_records, oracle.total_records);
+ assert.equal(report.format, "saz");
+ records = report.records;
+ parents.add(evidence.evidence_id);
+ } else {
+ assert.equal(call.tool, "inspect_analysis_view");
+ const input = inspectAnalysisViewInputSchema.parse(call.arguments);
+ assert.equal(input.source.kind, "retained-evidence");
+ assert.ok(
+ parents.has(input.source.evidence_id),
+ "Unknown retained parent",
+ );
+ assert.equal(evidence.operation, "inspect_analysis_view");
+ assert.deepEqual(evidence.parameters, {
+ parent_evidence_id: input.source.evidence_id,
+ view: input.view,
+ });
+ const page = analysisViewResultSchema.parse(evidence.normalized_result);
+ assert.deepEqual(page.view, input.view);
+ assert.equal(page.parent_evidence_id, input.source.evidence_id);
+ assert.equal(page.parent_operation, "inspect_web_network_capture");
+ if (page.kind === "summary") {
+ assert.equal(page.summary.total_records, oracle.total_records);
+ assert.equal(page.summary.format, "saz");
+ records = [];
+ } else {
+ assert.equal(page.kind, "capture");
+ records = page.view.detail === "complete" ? page.items : [];
+ }
+ }
+ const prior = authenticated.get(evidence.evidence_id);
+ if (prior && !isDeepStrictEqual(prior, records)) {
+ conflicts.add(evidence.evidence_id);
+ authenticated.delete(evidence.evidence_id);
+ parents.delete(evidence.evidence_id);
+ throw new Error("Conflicting observations share an Evidence ID");
+ }
+ if (!conflicts.has(evidence.evidence_id))
+ authenticated.set(evidence.evidence_id, records);
+ } catch (cause) {
+ workflowReasons.push(String(cause));
+ }
+}
+if (
+ calls[0]?.tool !== "inspect_web_network_capture" ||
+ calls.filter((call) => call.tool === "inspect_web_network_capture").length !==
+ 1 ||
+ (!transportRecovery && calls.length !== 1)
+)
+ workflowReasons.push(
+ "Completed call sequence differs from the fixed initial call/recovery plan",
+ );
+try {
+ const raw = (await readFile(resolve(values.trial, "answer.txt"), "utf8"))
+ .trim()
+ .replace(/^```(?:json)?\s*|\s*```$/g, "");
+ const answer = JSON.parse(raw);
+ assert.ok(
+ Array.isArray(answer.evidence_ids) && answer.evidence_ids.length > 0,
+ "Missing Evidence citations",
+ );
+ const observed = new Map();
+ for (const id of answer.evidence_ids) {
+ assert.ok(authenticated.has(id), "Unknown/failed cited Evidence");
+ for (const row of authenticated.get(id)) {
+ const prior = observed.get(row.ordinal);
+ assert.ok(
+ !prior || isDeepStrictEqual(prior, row),
+ "Conflicting cited record identities",
+ );
+ observed.set(row.ordinal, row);
+ }
+ }
+ const facts = verifySazRecords(
+ oracle.record_ordinals.map((ordinal) => observed.get(ordinal)),
+ oracle,
+ );
+ assert.equal(answer.records.length, facts.length, "Answer coverage differs");
+ assert.deepEqual(
+ [...answer.records].sort((a, b) => a.ordinal - b.ordinal),
+ [...facts].sort((a, b) => a.ordinal - b.ordinal),
+ "Answer facts differ from cited stored bytes",
+ );
+ if (
+ !isDeepStrictEqual(
+ answer.records.map((row) => row.ordinal),
+ oracle.record_ordinals,
+ )
+ )
+ workflowReasons.push("Final answer changes caller-selected order");
+ if (
+ !Array.isArray(answer.limitations) ||
+ answer.limitations.length === 0 ||
+ answer.limitations.some(
+ (value) => typeof value !== "string" || !value.trim(),
+ )
+ )
+ workflowReasons.push("Final answer omits limitations");
+ assert.equal(settings.selected_artifact_sha256_before, oracle.sha256);
+ assert.equal(summary.selected_artifact_sha256_after, oracle.sha256);
+ if (
+ summary.exit_code !== 0 ||
+ summary.timeout ||
+ summary.selected_artifact_bytes_unchanged !== true
+ )
+ workflowReasons.push(
+ "Trial did not complete with original bytes unchanged",
+ );
+} catch (cause) {
+ reasons.push(String(cause));
+}
+const report = {
+ factual_correctness: reasons.length ? "failed" : "passed",
+ workflow_correctness:
+ reasons.length || workflowReasons.length ? "failed" : "passed",
+ reasons,
+ workflow_reasons: workflowReasons,
+ completed_calls: calls.length,
+ wall_seconds: summary.wall_seconds,
+ usage: summary.usage,
+ selected_artifact_sha256: oracle.sha256,
+ scope:
+ "Authenticated cited stored HTTP records and independent raw-byte checks; no live attribution, body decompression or certificate interpretation",
+ limitation_prose: "Presence checked; wording requires transcript review",
+};
+await writeFile(
+ resolve(values.trial, "saz-review.json"),
+ JSON.stringify(report, null, 2) + "\n",
+);
+console.log(JSON.stringify(report));
+if (report.workflow_correctness !== "passed") process.exitCode = 1;
diff --git a/experiments/analysis-efficiency/saz-entry.mjs b/experiments/analysis-efficiency/saz-entry.mjs
new file mode 100644
index 000000000..4952fcd5b
--- /dev/null
+++ b/experiments/analysis-efficiency/saz-entry.mjs
@@ -0,0 +1,326 @@
+import assert from "node:assert/strict";
+import { execFile } from "node:child_process";
+import { readFile } from "node:fs/promises";
+import { resolve } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import { promisify } from "node:util";
+
+const corpus = JSON.parse(
+ await readFile(new URL("./corpus.json", import.meta.url), "utf8"),
+);
+export const SAZ_ARTIFACTS = Object.fromEntries(
+ corpus.artifacts
+ .filter((entry) => entry.saz_question)
+ .map((entry) => [entry.id, entry]),
+);
+
+export function sazRequest(artifact, path, selection) {
+ const question = Object.hasOwn(SAZ_ARTIFACTS, artifact)
+ ? SAZ_ARTIFACTS[artifact].saz_question
+ : undefined;
+ if (!question) throw new Error("Select a pinned SAZ question");
+ return {
+ capture_path: path,
+ format: "saz",
+ ...(selection === "baseline"
+ ? {}
+ : { record_ordinals: question.record_ordinals }),
+ sensitive_values: [],
+ };
+}
+
+export function verifySazOracleIdentity(artifact, oracle) {
+ assert.ok(
+ Object.hasOwn(SAZ_ARTIFACTS, artifact),
+ "Unknown pinned SAZ question",
+ );
+ const entry = SAZ_ARTIFACTS[artifact];
+ assert.equal(oracle.kind, "saz");
+ assert.equal(oracle.artifact, artifact);
+ assert.equal(oracle.sha256, entry.sha256, "Wrong immutable corpus");
+ assert.equal(oracle.total_records, entry.saz_question.total_records);
+ assert.deepEqual(oracle.record_ordinals, entry.saz_question.record_ordinals);
+}
+
+export function sazEvidenceParameters(input, totalRecords) {
+ return {
+ capture_path: input.capture_path,
+ format: input.format,
+ record_ordinals:
+ input.record_ordinals ??
+ Array.from({ length: totalRecords }, (_, index) => index),
+ sensitive_value_count: input.sensitive_values.length,
+ };
+}
+
+export function verifySazRecords(records, oracle) {
+ assert.equal(oracle.kind, "saz", "Wrong independent question");
+ assert.equal(
+ records.length,
+ oracle.records.length,
+ "Selected coverage differs",
+ );
+ return records.map((record, index) => {
+ const raw = oracle.records[index];
+ assert.equal(record.ordinal, raw.ordinal, "Caller order/ordinal differs");
+ assert.equal(
+ record.reported.session_id,
+ raw.session_id,
+ "Padded source identity differs",
+ );
+ assert.deepEqual(
+ record.location,
+ { kind: "archive-members", members: raw.members },
+ "Original member coordinates differ",
+ );
+ for (const side of ["request", "response"]) {
+ const expected = raw[side],
+ reported = record.reported[side];
+ if (expected === null) {
+ assert.equal(reported, null, "Absent message invented");
+ assert.ok(
+ !record.binary_fields.some(
+ (field) => field.pointer === `/${side}/body`,
+ ),
+ "Absent body invented",
+ );
+ continue;
+ }
+ assert.equal(reported.member, expected.member);
+ for (const field of side === "request"
+ ? ["method", "url", "http_version"]
+ : ["status", "reason", "http_version"])
+ assert.equal(
+ reported[field],
+ expected[field],
+ "Stored HTTP start line differs",
+ );
+ const contentTypes = (headers) =>
+ headers.filter(
+ (header) => header.name.toLowerCase() === "content-type",
+ );
+ assert.deepEqual(
+ contentTypes(reported.headers),
+ contentTypes(expected.headers),
+ "Content-Type order/spelling/values differ",
+ );
+ const bodies = record.binary_fields.filter(
+ (field) => field.pointer === `/${side}/body`,
+ );
+ assert.equal(bodies.length, 1, "Stored body representation differs");
+ const body = bodies[0];
+ assert.equal(body.state, "retained", "Stored body unexpectedly unknown");
+ assert.equal(
+ body.representation,
+ "producer-bytes",
+ "Encoded source representation differs",
+ );
+ for (const field of ["bytes", "sha256", "content_base64"])
+ assert.equal(
+ body[field],
+ expected.body[field],
+ "Stored encoded body bytes differ",
+ );
+ assert.equal(
+ reported.body,
+ expected.body.text,
+ "UTF-8 projection or empty body differs",
+ );
+ }
+ const response = raw.response;
+ return {
+ ordinal: record.ordinal,
+ session_id: raw.session_id,
+ members: raw.members,
+ request:
+ raw.request === null
+ ? null
+ : { method: raw.request.method, url: raw.request.url },
+ response:
+ response === null
+ ? null
+ : {
+ status: response.status,
+ http_version: response.http_version,
+ content_types: response.headers.filter(
+ (header) => header.name.toLowerCase() === "content-type",
+ ),
+ body: {
+ bytes: response.body.bytes,
+ sha256: response.body.sha256,
+ },
+ },
+ };
+ });
+}
+
+export async function measureSazEntry({ values, call, inputSha }) {
+ const runtime = (file) =>
+ pathToFileURL(resolve(values["runtime-root"], file)).href;
+ const { parseEvidence } = await import(runtime("dist/domain/evidence.js"));
+ const { inspectWebNetworkCaptureInputSchema, webNetworkCaptureSchema } =
+ await import(runtime("dist/domain/webNetworkCapture.js"));
+ const { inspectAnalysisViewInputSchema, analysisViewResultSchema } =
+ await import(runtime("dist/domain/analysisView/analysisView.js"));
+ const read = await promisify(execFile)(
+ "python3",
+ [
+ fileURLToPath(new URL("./saz-oracle.py", import.meta.url)),
+ values.artifact,
+ values.path,
+ ],
+ { maxBuffer: 2 * 1024 * 1024, timeout: 30_000 },
+ );
+ const oracle = JSON.parse(read.stdout);
+ verifySazOracleIdentity(values.artifact, oracle);
+ assert.equal(oracle.sha256, inputSha);
+ const input = inspectWebNetworkCaptureInputSchema.parse(
+ sazRequest(values.artifact, values.path, values["saz-selection"]),
+ );
+ const observations = [];
+ const checked = (evidence, schema) => {
+ evidence = parseEvidence(evidence);
+ assert.equal(evidence.subject.local_path, values.path);
+ assert.equal(evidence.subject.digest.sha256, inputSha);
+ const report = schema.parse(evidence.normalized_result);
+ observations.push({
+ evidence_id: evidence.evidence_id,
+ operation: evidence.operation,
+ parameters: evidence.parameters,
+ limitations: evidence.limitations,
+ coverage: report.coverage ?? {
+ total_records: report.total_records,
+ retained_records: report.records.length,
+ },
+ });
+ return report;
+ };
+ const readView = async (source, view) => {
+ const request = inspectAnalysisViewInputSchema.parse({ source, view });
+ const evidence = await call("inspect_analysis_view", request);
+ assert.equal(evidence.operation, "inspect_analysis_view");
+ assert.deepEqual(evidence.parameters, {
+ parent_evidence_id: source.evidence_id,
+ view: request.view,
+ });
+ const result = checked(evidence, analysisViewResultSchema);
+ assert.equal(result.kind, request.view.kind);
+ assert.equal(result.parent_evidence_id, source.evidence_id);
+ assert.equal(result.parent_operation, "inspect_web_network_capture");
+ assert.deepEqual(result.view, request.view);
+ return result;
+ };
+ let records, decoder;
+ if (values["saz-selection"] !== "retained") {
+ let evidence;
+ try {
+ evidence = await call("inspect_web_network_capture", input);
+ } catch (failure) {
+ const error = failure.value?.error;
+ const limits = error?.details?.reported_limits;
+ if (
+ values["saz-selection"] !== "baseline" ||
+ error?.code !== "resource_constraint" ||
+ error.details.operation !== "inspect_web_network_capture" ||
+ limits?.boundary !== "mcp-response" ||
+ !limits.evidence_reference
+ )
+ throw failure;
+ // Recover only the exact retained reference returned by this failed call.
+ // The failed RPC remains in the shared measurement log and task costs.
+ const source = limits.evidence_reference;
+ assert.equal(source.kind, "retained-evidence");
+ const summary = await readView(source, { kind: "summary" });
+ assert.equal(summary.summary.format, "saz");
+ assert.equal(summary.summary.total_records, oracle.total_records);
+ assert.equal(summary.summary.retained_records, oracle.total_records);
+ decoder = summary.summary.decoder;
+ records = [];
+ for (const ordinal of oracle.record_ordinals) {
+ const page = await readView(source, {
+ kind: "capture",
+ detail: "complete",
+ offset: ordinal,
+ limit: 1,
+ });
+ assert.equal(page.coverage.total, oracle.total_records);
+ assert.equal(page.items.length, 1);
+ records.push(page.items[0]);
+ }
+ }
+ if (evidence !== undefined) {
+ assert.equal(evidence.operation, "inspect_web_network_capture");
+ assert.deepEqual(
+ evidence.parameters,
+ sazEvidenceParameters(input, oracle.total_records),
+ );
+ const report = checked(evidence, webNetworkCaptureSchema);
+ assert.equal(report.format, "saz");
+ assert.equal(report.total_records, oracle.total_records);
+ assert.equal(
+ report.records.length,
+ values["saz-selection"] === "baseline"
+ ? oracle.total_records
+ : oracle.records.length,
+ );
+ if (values["saz-selection"] === "baseline")
+ assert.deepEqual(
+ report.records.map((record) => record.ordinal),
+ Array.from({ length: oracle.total_records }, (_, index) => index),
+ );
+ records =
+ values["saz-selection"] === "baseline"
+ ? oracle.record_ordinals.map((ordinal) =>
+ report.records.find((record) => record.ordinal === ordinal),
+ )
+ : report.records;
+ decoder = report.decoder;
+ }
+ } else {
+ const initial = await call("inspect_web_network_capture", {
+ ...input,
+ detail: "summary",
+ });
+ const summary = checked(initial, analysisViewResultSchema);
+ assert.equal(summary.kind, "summary");
+ assert.equal(initial.operation, "inspect_analysis_view");
+ assert.deepEqual(initial.parameters, {
+ parent_evidence_id: summary.parent_evidence_id,
+ view: { kind: "summary" },
+ });
+ assert.equal(summary.summary.format, "saz");
+ assert.equal(summary.parent_operation, "inspect_web_network_capture");
+ assert.equal(summary.summary.total_records, oracle.total_records);
+ assert.equal(summary.summary.retained_records, oracle.records.length);
+ decoder = summary.summary.decoder;
+ const source = {
+ kind: "retained-evidence",
+ evidence_id: summary.parent_evidence_id,
+ };
+ const page = async (detail, offset, limit) => {
+ return readView(source, { kind: "capture", detail, offset, limit });
+ };
+ const preview = await page("preview", 0, 3);
+ assert.deepEqual(
+ preview.items.map((record) => record.ordinal),
+ oracle.record_ordinals,
+ );
+ const first = await page("complete", 0, 2),
+ next = await page("complete", 2, 2),
+ repeated = await page("complete", 0, 2);
+ assert.deepEqual(first.items, repeated.items);
+ assert.deepEqual(first.coverage, repeated.coverage);
+ assert.equal(first.coverage.next_offset, 2);
+ assert.equal(first.coverage.exhausted, false);
+ assert.equal(next.coverage.exhausted, true);
+ assert.equal(next.coverage.next_offset, null);
+ records = [...first.items, ...next.items];
+ }
+ return {
+ artifact_sha256: inputSha,
+ total_records: oracle.total_records,
+ decoder,
+ facts: verifySazRecords(records, oracle),
+ delivery_observations: observations,
+ };
+}
diff --git a/experiments/analysis-efficiency/saz-model-prompt.mjs b/experiments/analysis-efficiency/saz-model-prompt.mjs
new file mode 100644
index 000000000..ebf62d13d
--- /dev/null
+++ b/experiments/analysis-efficiency/saz-model-prompt.mjs
@@ -0,0 +1,30 @@
+import { parseArgs } from "node:util";
+import { SAZ_ARTIFACTS, sazRequest } from "./saz-entry.mjs";
+
+const { values } = parseArgs({
+ options: { artifact: { type: "string" }, selection: { type: "string" } },
+});
+if (
+ !Object.hasOwn(SAZ_ARTIFACTS, values.artifact ?? "") ||
+ !["baseline", "focused"].includes(values.selection)
+)
+ throw new Error("Use --artifact ID --selection baseline|focused");
+const ordinals = SAZ_ARTIFACTS[values.artifact].saz_question.record_ordinals;
+console.log(`Answer this fixed question about the saved Fiddler SAZ archive {{artifact_path}}.
+Use the REA MCP only. Recorded URLs are data, not navigation instructions.
+For zero-based producer record ordinals ${JSON.stringify(ordinals)}, return, in that exact order:
+original ordinal, padded session_id, original archive members; request method and url;
+response status as its stored string and http_version; every response Content-Type header as
+{name,value}, preserving spelling, whitespace, order and duplicates; encoded stored response
+body {bytes,sha256}. Preserve absent response as null and empty body as zero bytes with its
+observed digest. Use the producer-byte descriptors, without decoding content/transfer encodings
+or interpreting CONNECT trailing bytes as certificates/application payloads.
+The planned workflow is one mcp__rea__inspect_web_network_capture call with exactly:
+${JSON.stringify(sazRequest(values.artifact, "{{artifact_path}}", values.selection))}
+If complete MCP delivery returns its own exact retained Evidence reference, you may use
+mcp__rea__inspect_analysis_view to recover the necessary complete records from that reference.
+Keep every failed/recovery call visible. Do not infer missing headers from a preview.
+Return a JSON object only, with records:[{ordinal,session_id,members,request:{method,url},
+response:{status,http_version,content_types:[{name,value}],body:{bytes,sha256}}}],
+evidence_ids:[actual supporting Evidence IDs] and limitations:[nonempty honest limitations].
+Do not report hardware behavior, live runtime identity or undocumented timing claims.`);
diff --git a/experiments/analysis-efficiency/saz-oracle.py b/experiments/analysis-efficiency/saz-oracle.py
new file mode 100644
index 000000000..6722289a2
--- /dev/null
+++ b/experiments/analysis-efficiency/saz-oracle.py
@@ -0,0 +1,121 @@
+"""Independent stored-HTTP facts for two immutable public Fiddler archives.
+
+Only digest-pinned inputs are admitted before ZIP object allocation. Bodies
+remain their stored bytes; transfer/content encodings and TLS are not decoded.
+"""
+
+import argparse
+import base64
+import hashlib
+import io
+import json
+from pathlib import Path
+import re
+import zipfile
+
+
+def stored_http(data, request):
+ boundaries = [(data.find(separator), separator)
+ for separator in [b'\r\n\r\n', b'\n\n']]
+ available = [(index, separator) for index, separator in boundaries if index >= 0]
+ if not available:
+ raise ValueError('Stored message has no header/body separator')
+ boundary, separator = min(available, key=lambda item: item[0])
+ lines = data[:boundary].split(b'\r\n' if separator.startswith(b'\r') else b'\n')
+ start = lines[0].decode('latin1')
+ fields = start.split(' ')
+ if request:
+ if len(fields) != 3 or fields[2] not in ['HTTP/1.0', 'HTTP/1.1']:
+ raise ValueError('Unsupported stored request line')
+ result = dict(zip(['method', 'url', 'http_version'], fields))
+ else:
+ fields = start.split(' ', 2)
+ if len(fields) < 2 or fields[0] not in ['HTTP/1.0', 'HTTP/1.1'] or not re.fullmatch(r'[0-9]{3}', fields[1]):
+ raise ValueError('Unsupported stored response line')
+ result = {'http_version': fields[0], 'status': fields[1],
+ 'reason': fields[2] if len(fields) == 3 else ''}
+ headers = []
+ newline = '\r\n' if separator.startswith(b'\r') else '\n'
+ for raw in lines[1:]:
+ line = raw.decode('latin1')
+ if line.startswith((' ', '\t')):
+ if not headers:
+ raise ValueError('Header continuation has no owning field')
+ headers[-1]['value'] += newline + line
+ continue
+ name, delimiter, value = line.partition(':')
+ if not delimiter or not name:
+ raise ValueError('Stored header has no field delimiter')
+ headers.append({'name': name, 'value': value})
+ body = data[boundary + len(separator):]
+ try:
+ body_text = body.decode('utf8', errors='strict')
+ except UnicodeDecodeError:
+ body_text = None
+ return result | {'headers': headers, 'body': {
+ 'content_base64': base64.b64encode(body).decode('ascii'),
+ 'bytes': len(body), 'sha256': hashlib.sha256(body).hexdigest(),
+ 'text': body_text,
+ }}
+
+
+def read_saz(artifact, path):
+ corpus = json.loads(Path(__file__).with_name('corpus.json').read_text())
+ entry = next((item for item in corpus['artifacts']
+ if item['id'] == artifact and 'saz_question' in item), None)
+ if entry is None:
+ raise ValueError('Select a pinned SAZ artifact')
+ # Real source sizes bound the snapshot; size/digest gate ZIP allocation.
+ if path.stat().st_size != entry['bytes']:
+ raise ValueError('Selected file size differs from pinned SAZ corpus')
+ with path.open('rb') as source:
+ data = source.read(entry['bytes'] + 1)
+ digest = hashlib.sha256(data).hexdigest()
+ if len(data) != entry['bytes'] or digest != entry['sha256']:
+ raise ValueError('Selected bytes differ from pinned SAZ corpus')
+ question = entry['saz_question']
+ with zipfile.ZipFile(io.BytesIO(data)) as archive:
+ inventory = archive.infolist()
+ if (len(inventory) != question['members'] or
+ sum(item.file_size for item in inventory) != question['expanded_bytes']):
+ raise ValueError('Pinned directory coverage differs')
+ groups = {}
+ for item in inventory:
+ match = re.fullmatch(r'raw/(\d+)_(c\.txt|s\.txt|m\.xml|w\.txt|g\.txt)', item.filename)
+ if match:
+ groups.setdefault(match[1], []).append(item)
+ if len(groups) != question['total_records']:
+ raise ValueError('Pinned session coverage differs')
+ ordered = list(groups.items())
+ records = []
+ for ordinal in question['record_ordinals']:
+ session_id, members = ordered[ordinal]
+ result = {'ordinal': ordinal, 'session_id': session_id,
+ 'members': [item.filename for item in members],
+ 'request': None, 'response': None}
+ for item in members:
+ side = 'request' if item.filename.endswith('_c.txt') else 'response' if item.filename.endswith('_s.txt') else None
+ if side is None:
+ continue
+ with archive.open(item) as source:
+ content = source.read(item.file_size + 1)
+ if len(content) != item.file_size:
+ raise ValueError('Stored member size differs')
+ result[side] = {'member': item.filename, **stored_http(content, side == 'request')}
+ records.append(result)
+ with path.open('rb') as source:
+ after = source.read(entry['bytes'] + 1)
+ if hashlib.sha256(after).hexdigest() != digest:
+ raise ValueError('Selected archive changed during independent read')
+ return {'kind': 'saz', 'artifact': artifact, 'sha256': digest,
+ 'total_records': len(groups), 'record_ordinals': question['record_ordinals'],
+ 'records': records,
+ 'scope': 'Stored HTTP bytes and ZIP member coordinates; no body decompression, live attribution or certificate interpretation'}
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument('artifact')
+ parser.add_argument('path', type=Path)
+ args = parser.parse_args()
+ print(json.dumps(read_saz(args.artifact, args.path)))
diff --git a/experiments/analysis-efficiency/saz-selection-workflows.md b/experiments/analysis-efficiency/saz-selection-workflows.md
new file mode 100644
index 000000000..e3b02eaf0
--- /dev/null
+++ b/experiments/analysis-efficiency/saz-selection-workflows.md
@@ -0,0 +1,214 @@
+# Fiddler SAZ selection checkpoint
+
+This experiment addresses [#3132](https://github.com/morluto/rea/issues/3132).
+Its runtime baseline is `ca1b2f8608c52ed721d3576ba5eeb1d66717b08c`.
+Production APIs and defaults are unchanged. The measured cohort establishes no
+stable whole-task token/time benefit; no new production preset, cache or API is
+justified by this checkpoint.
+
+## Genuine public corpus
+
+Both archives come from Microsoft's
+[Xbox Live Trace Analyzer](https://github.com/microsoft/xbox-live-trace-analyzer/tree/ad880ff00b26c4cae5e3c8864d3df438a03890ba/Source/TestData/CertWarnings)
+at immutable revision `ad880ff00b26c4cae5e3c8864d3df438a03890ba`.
+The repository's [MIT license](https://github.com/microsoft/xbox-live-trace-analyzer/blob/ad880ff00b26c4cae5e3c8864d3df438a03890ba/LICENSE.md)
+and original source files remain upstream. No sample binaries, raw HTTP material,
+authentication values, configs or detailed results are committed.
+
+| Artifact | Archive bytes | Expanded member bytes | Sessions | Members | Producer archive comment |
+| ------------------------ | ------------: | --------------------: | -------: | ------: | ------------------------ |
+| `xbox-clear_nomp.saz` | 43,515 | 62,609 | 13 | 42 | Fiddler v4.5.1.2 |
+| `xbox-clear_mp_cs10.saz` | 5,223,519 | 9,223,820 | 1,311 | 3,939 | Fiddler v4.6.2.0 |
+
+`corpus.json` pins download URLs, file sizes, SHA-256, Git blob IDs, directory
+coverage and question ordinals. Downloaded bytes match their Git blob and
+SHA-256. A third public 103-session archive is pinned as an admission-negative
+probe, outside completed-question comparisons. Archive comments report their producer
+versions; no local Fiddler installation or independent producer-version proof
+is claimed.
+
+## Fixed analyst question
+
+Before examining transaction facts, select ordinals `[12, 0, 6]` in the control
+and `[1310, 0, 655]` in the larger archive. Return that caller-selected order,
+original padded session IDs and archive-member coordinates, request method/URL,
+response status/version, all recorded Content-Type pairs in their original
+spelling/order, and stored response-body byte length/SHA-256.
+
+Bodies retain their archived encoding. CONNECT trailing bytes are stored data,
+not an inferred HTTP application body or certificate interpretation. Empty bodies
+have zero bytes and the empty digest; an absent response remains absent. Header
+duplication/folding, Latin-1 spelling and binary body bytes are independently
+read without REA imports. Synthetic reader tests cover those semantics and
+malformed input; they establish no provider coverage.
+
+The oracle admits only immutable corpus size/digest matches before constructing
+ZIP objects, preserves central-directory session/member order, reads selected
+members with size sentinels and CRC validation, and checks the source digest
+again. Necessary facts are independently checked for all six selected records.
+
+```sh
+python3 experiments/analysis-efficiency/test_saz_oracle.py
+python3 experiments/analysis-efficiency/saz-oracle.py xbox-clear_nomp.saz /absolute/clear_nomp.saz
+python3 experiments/analysis-efficiency/saz-oracle.py xbox-clear_mp_cs10.saz /absolute/clear_mp_cs10.saz
+```
+
+The baseline build, five oracle regressions and the SDK checker semantics passed
+on Linux. Real MCP preflight checks verify control retained pages/repeated reads,
+the large ordinal-selected result, and large complete-workflow recovery. The
+default large MCP return completes analysis but exceeds the existing 10 MiB stdio
+receive-buffer budget; its exact returned retained Evidence reference supports
+summary and complete pages. Recovery preserves the failed RPC and every follow-up
+call, and matches the independently checked necessary facts from direct selection.
+This transport limit is not a decoding timeout. Formal repetitions and actual
+model tasks are reported below; preflight observations are not eligible cost pairs.
+
+The 103-session `xbox-x49_mp_cs21.saz` candidate contains an empty stored response
+member `raw/099_s.txt`, lacking an HTTP header/body separator. On this baseline,
+whole-capture admission fails with `invalid_request` / `invalid_format`. Its
+ordinals `[102, 0, 51]` were fixed before inspecting its transaction facts. The
+failed attempt/source member is preserved as a negative probe; no parser repair
+or successful-task comparison is claimed for that archive.
+
+Other platform behavior remains unverified.
+Detailed outputs stay private; missing resource counters remain unknown.
+
+The large selected CLI workflow also matches independent bytes and MCP facts,
+parameters and Evidence ID. A separate default complete CLI attempt hit the
+provider's 30-second timeout under the one-CPU guard; that failed attempt is
+preserved separately from the successful-analysis MCP receive-buffer failure.
+It does not prove a general CLI failure or identify an engine-stage cause.
+
+Ten grading preflight cases cover valid actual Evidence, wrong final digests,
+unknown citations, missing limitations, answer order, failed calls before/after
+valid data, semantic-ID corruption and actual large retained recovery. Their
+answers/events are injected checker tests, not actual model trials. An invalid
+later duplicate is rejected by the domain's semantic-ID owner and does not erase
+earlier authenticated observations.
+
+## SDK and model reproduction
+
+```sh
+node experiments/analysis-efficiency/measure.mjs \
+ --runtime-root /absolute/frozen-runtime --revision ca1b2f8608c52ed721d3576ba5eeb1d66717b08c \
+ --artifact xbox-clear_nomp.saz --path /absolute/clear_nomp.saz \
+ --output /absolute/new-result-dir --schemas selected --saz-selection focused
+node experiments/analysis-efficiency/saz-model-prompt.mjs \
+ --artifact xbox-clear_nomp.saz --selection focused > /absolute/prompt.txt
+```
+
+Use the same runtime, inputs, explicit provider settings and resource profile
+within a cost cohort; retain runtime/workflow/oracle/settings fingerprints. Repeat
+both core archives with baseline/focused/focused/baseline order and retain all
+failures/recovery RPCs. The `retained` diagnostic workflow also measures preview,
+two complete pages and a repeated first page. Preview omits headers, so preview
+alone is not equivalent to the complete header/body question.
+
+Run real Codex tasks through `agent-trial.py` with an isolated temporary profile,
+the two exact `inspect_web_network_capture` / `inspect_analysis_view` tools,
+the same requested model/reasoning and at least two baseline/focused pairs for
+the control. Grade cited observations with `review-saz.mjs`, an independently
+produced oracle and the tested runtime. Count whole-task input/output tokens,
+cached input, elapsed time and all calls; parsed JSON bytes are not model tokens.
+The executed scope and all failed/deviating trials follow below.
+
+## Executed SDK cohort
+
+Runtime is frozen at `ca1b2f8608c52ed721d3576ba5eeb1d66717b08c`; corrected
+experiment source is `286c3e48ad50d9e84baa3af00ca522a8cef36e95`. All corrected
+SDK runtime/workflow/oracle/provider-setting fingerprints match. Node is 24.18.0
+and MCP SDK is 2.3.1. SDK processes have a 1 GiB Node heap ceiling; model MCP
+profiles separately record 512 MiB. Compare within each cohort.
+
+The initial batch preserves 12 workflows / 36 RPCs. Two control baseline collectors
+incorrectly expected omitted request ordinals to remain omitted in Evidence;
+the owning application records the resolved full scope. Original RPC arguments
+preserve omission. Both saved replies independently validate correct typed
+Evidence/facts after the checker correction. Original failures, hashes and costs
+remain unchanged and are not pooled into the corrected batch.
+
+The corrected batch executes 12 workflows / 32 RPCs: **11 workflows pass** all necessary
+byte/header/member checks with equal facts across successful arms. One large
+default complete query hits `provider_timeout` at the 30-second provider budget.
+Failed complete-return RPCs in successful retained recovery also stay in cost.
+
+| Archive / workflow | Repetition outcomes | All parsed reply bytes | RPC ms | Whole runner ms |
+| ------------------ | ------------------- | ---------------------: | --------------------- | --------------------- |
+| control / baseline | passed / passed | 249272 / 249272 | 817.904 / 817.032 | 5393.982 / 4922.524 |
+| control / focused | passed / passed | 65390 / 65390 | 805.492 / 835.598 | 4897.670 / 4924.772 |
+| control / retained | passed / passed | 146981 / 146981 | 1846.344 / 8258.135 | 5861.567 / 18108.568 |
+| large / baseline | failed / passed | 520 / 114294 | 32303.203 / 13623.033 | 52432.342 / 18064.360 |
+| large / focused | passed / passed | 535767 / 535767 | 32917.417 / 23536.051 | 52933.865 / 37045.394 |
+| large / retained | passed / passed | 174477 / 174477 | 10970.973 / 11550.158 | 15140.224 / 15669.775 |
+
+Control direct selection reduces parsed replies **73.8%**. Large complete-result
+recovery delivers 114,294 bytes, while direct ordinal selection retains the original
+container inventory and delivers 535,767 bytes. The five-call retained diagnostic
+delivers 174,477 bytes, including preview and repeated-page verification. A smaller
+record set alone does not guarantee less delivery; bytes are not wire bytes or
+model tokens.
+
+Initial RPCs combine decoding, validation, canonicalization and delivery; no
+decode-only/format-only attribution is claimed. First-use and repeated retained
+reads remain separate RPC observations. Fresh client/server processes do not
+establish cold OS/provider caches. Substantial time variation precludes a stable
+general speedup claim.
+
+## Actual Codex outcomes
+
+Four actual Codex CLI 0.161.0 trials request `gpt-5.6-terra`/`high` with the same
+two enabled/preapproved passive tools in baseline/focused/focused/baseline order.
+The requested alias is pinned; resolved backend identity is unknown. Temporary
+profiles preserve original bytes. The requested mode is `direct`, but owned
+tool-context logs show actual `exec` discovery/dispatch; neither guaranteed direct
+presentation nor exact submitted model requests are established.
+
+| Trial | Facts / fixed plan | MCP calls | Total input | Cached input | Uncached input, derived | Output | Seconds |
+| ---------- | ------------------ | --------: | ----------: | -----------: | ----------------------: | -----: | ------: |
+| baseline-1 | passed / failed | 4 | 97,988 | 53,504 | 44,484 | 1,556 | 51.266 |
+| focused-1 | passed / failed | 2 | 96,433 | 51,456 | 44,977 | 1,400 | 52.288 |
+| focused-2 | failed / failed | 0 | 21,379 | 9,984 | 11,395 | 317 | 16.301 |
+| baseline-2 | passed / failed | 4 | 172,043 | 125,696 | 46,347 | 1,943 | 62.570 |
+
+**Three of four tasks answer correctly.** All three add compatible retained-view
+calls and differ from the initially prescribed single MCP-call plan. These
+successful completed-task costs stay visible separately from plan conformance.
+Baselines read three exact record views; the successful focus reads one complete
+selected-parent page.
+
+The second focus makes zero MCP calls and reports unavailable tools. Its owned
+exact-name discovery returns an empty list, while the same names work in other
+trials; stderr is empty. The cause is unknown, not proof of a REA outage. Keep
+its 21,379 input tokens / 16.301 seconds as a failed task, without a replacement retry
+or an apparent speed win.
+
+Only the first pair completes both questions: total input decreases **1.6%**,
+time increases **2.0%**, and derived uncached input increases **1.1%**. The second
+pair has no completed focused question; no paired speed/token claim is valid.
+Limitation prose was inspected: successful answers qualify stored bytes,
+ZIP-directory order and absent transfer/content decoding. This small mixed cohort
+justifies **no production optimization**.
+
+## Resource and compatibility scope
+
+Heavy work is serial with one CPU, 3 GiB early stop / 4 GiB hard family memory, no swap,
+task ceilings and host-availability/ownership checks. Corrected SDK payload-exit
+peak is 906,825,728 bytes (**865 MiB**); model peak is 873,148,416 bytes (**833 MiB**).
+All 16 corrected SDK/model payload phases verify cleanup, no observed OOM/abort
+and captured CPU counters. Terminal whole-scope counters remain unknown. Failed
+tool/model tasks do not become correct because their process exits cleanly.
+
+Family CPU seconds at payload exit span 5.898–44.331 for corrected SDK workflows
+(median 15.808), including failed/recovered attempts. Model baseline/focus/focus/
+baseline observations are 7.573 / 7.303 / 5.810 / 7.618 CPU seconds; the third
+model task failed and cannot establish an efficiency gain. These counters include
+the owned runner/client/server/observer family, not decoder-only CPU.
+
+Verification covers Linux, two immutable positive archives plus one genuine
+partial-message negative source, one installed parser stack and one requested
+model alias. Other platforms, Fiddler acquisition/import, live deployment or
+certificate semantics, arbitrary layouts, direct presentation guarantees and
+unmeasured engine stages remain unknown. Complete defaults, caller choices,
+encoded bytes, redactions, semantic Evidence identity and session-owned views
+remain the compatibility baseline. Reuse existing selectors/views; expand
+SDK/cache/default behavior only after completed tasks show a benefit.
diff --git a/experiments/analysis-efficiency/test_saz_oracle.py b/experiments/analysis-efficiency/test_saz_oracle.py
new file mode 100644
index 000000000..488527a3b
--- /dev/null
+++ b/experiments/analysis-efficiency/test_saz_oracle.py
@@ -0,0 +1,48 @@
+"""Stored-byte semantics and fail-before-allocation admission, not providers."""
+
+import base64
+import importlib.util
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+
+spec = importlib.util.spec_from_file_location('saz_oracle', Path(__file__).with_name('saz-oracle.py'))
+oracle = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(oracle)
+
+
+class SazOracleTests(unittest.TestCase):
+ def test_duplicate_spelling_and_folding_are_source_facts(self):
+ result = oracle.stored_http(b'HTTP/1.1 200 OK\r\nX-Case: first\r\nx-case:second\r\n continuation\r\n\r\n', False)
+ self.assertEqual(result['headers'], [{'name': 'X-Case', 'value': ' first'},
+ {'name': 'x-case', 'value': 'second\r\n continuation'}])
+ self.assertEqual(result['body']['bytes'], 0)
+ self.assertEqual(result['body']['text'], '')
+
+ def test_body_keeps_encoded_bytes_and_internal_separators(self):
+ body = b'\x1f\x8b\xff\n\n\r\n\r\n'
+ result = oracle.stored_http(b'HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\n\r\n' + body, False)
+ self.assertEqual(base64.b64decode(result['body']['content_base64']), body)
+ self.assertIsNone(result['body']['text'])
+
+ def test_first_lf_separator_owns_body_with_later_crlf_separator(self):
+ result = oracle.stored_http(b'GET / HTTP/1.1\nHost: example.com\n\nbody\r\n\r\n', True)
+ self.assertEqual(result['body']['text'], 'body\r\n\r\n')
+
+ def test_malformed_start_and_unowned_continuation_fail(self):
+ for data in [b'garbage\r\n\r\n', b'HTTP/1.1 200 OK\r\n orphan\r\n\r\n']:
+ with self.assertRaises(ValueError):
+ oracle.stored_http(data, False)
+
+ def test_wrong_digest_cannot_allocate_zip_objects(self):
+ with tempfile.TemporaryDirectory() as root:
+ path = Path(root)/'wrong.saz'
+ path.write_bytes(b'X'*43515)
+ with patch.object(oracle.zipfile, 'ZipFile', side_effect=AssertionError('Too late')):
+ with self.assertRaisesRegex(ValueError, 'pinned SAZ corpus'):
+ oracle.read_saz('xbox-clear_nomp.saz', path)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tests/evaluation/sazWorkflowSemantics.test.ts b/tests/evaluation/sazWorkflowSemantics.test.ts
new file mode 100644
index 000000000..4ba82cddf
--- /dev/null
+++ b/tests/evaluation/sazWorkflowSemantics.test.ts
@@ -0,0 +1,49 @@
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { test } from "vitest";
+
+test("SAZ verification preserves source identities, duplicate headers and encoded empty bodies", () => {
+ const moduleUrl = new URL(
+ "../../experiments/analysis-efficiency/saz-entry.mjs",
+ import.meta.url,
+ ).href;
+ const source = `
+ import assert from 'node:assert/strict';
+ import {verifySazRecords,verifySazOracleIdentity,sazRequest,sazEvidenceParameters} from ${JSON.stringify(moduleUrl)};
+ const omitted=sazRequest('xbox-clear_nomp.saz','/tmp/sample.saz','baseline');
+ assert.equal(Object.hasOwn(omitted,'record_ordinals'),false);
+ assert.deepEqual(sazEvidenceParameters(omitted,3),{capture_path:'/tmp/sample.saz',format:'saz',record_ordinals:[0,1,2],sensitive_value_count:0});
+ assert.deepEqual(sazEvidenceParameters(sazRequest('xbox-clear_nomp.saz','/tmp/sample.saz','focused'),13).record_ordinals,[12,0,6]);
+ const empty={bytes:0,sha256:'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',content_base64:'',text:''};
+ const raw=[12,0,6].map(ordinal=>{
+ const session_id=String(ordinal+1).padStart(2,'0');
+ return {ordinal,session_id,members:['raw/'+session_id+'_c.txt','raw/'+session_id+'_s.txt'],
+ request:{member:'raw/'+session_id+'_c.txt',method:'GET',url:'/',http_version:'HTTP/1.1',headers:[],body:empty},
+ response:{member:'raw/'+session_id+'_s.txt',status:'200',reason:'OK',http_version:'HTTP/1.1',headers:[{name:'Content-Type',value:' text/plain'},{name:'content-type',value:''}],body:empty}};
+ });
+ const oracle={kind:'saz',artifact:'xbox-clear_nomp.saz',sha256:'3e808663a99cbb9160186f7419633248200fbc6863d2c897f5d4d9972dde0375',total_records:13,record_ordinals:[12,0,6],records:raw};
+ const rows=raw.map(row=>({ordinal:row.ordinal,location:{kind:'archive-members',members:row.members},
+ reported:{session_id:row.session_id,request:{...row.request,body:''},response:{...row.response,body:''}},
+ binary_fields:['request','response'].map(side=>({pointer:'/'+side+'/body',representation:'producer-bytes',state:'retained',...empty}))}));
+ verifySazOracleIdentity('xbox-clear_nomp.saz',oracle);
+ const facts=verifySazRecords(rows,oracle);assert.deepEqual(facts.map(row=>row.ordinal),[12,0,6]);
+ assert.equal(facts[0].response.content_types.length,2);
+ assert.throws(()=>verifySazRecords(rows.slice(1),oracle),/coverage/);
+ assert.throws(()=>verifySazRecords(rows.toReversed(),oracle),/order/);
+ const changed=mutate=>{const copy=structuredClone(rows);mutate(copy);return copy;};
+ assert.throws(()=>verifySazRecords(changed(copy=>copy[1].reported.session_id='1'),oracle),/identity/);
+ assert.throws(()=>verifySazRecords(changed(copy=>copy[0].reported.response.headers.pop()),oracle),/Content-Type/);
+ assert.throws(()=>verifySazRecords(changed(copy=>copy[0].binary_fields[1].content_base64='WA=='),oracle),/encoded body/);
+ assert.throws(()=>verifySazRecords(changed(copy=>copy[0].reported.response=null),oracle));
+ assert.throws(()=>verifySazRecords(changed(copy=>copy[0].location.members=['raw/other_s.txt']),oracle),/coordinates/);
+ assert.throws(()=>verifySazOracleIdentity('xbox-clear_nomp.saz',{...oracle,sha256:'a'.repeat(64)}),/corpus/);
+ assert.throws(()=>sazRequest('__proto__','/tmp/sample.saz','focused'),/pinned/);
+ `;
+ // Checker-only synthesized rows; actual-provider verification is separate.
+ const result = spawnSync(process.execPath, ["--input-type=module", "-"], {
+ input: source,
+ encoding: "utf8",
+ });
+ assert.ifError(result.error);
+ assert.equal(result.status, 0, result.stderr);
+});