Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
test(perf): evaluate larger and non-HTTP packet workflows (#3232)
Ysopwnableca7450beffff
diff --git a/docs/packet-captures.md b/docs/packet-captures.mdindex 32cedb11b..8db097386 100644--- a/docs/packet-captures.md+++ b/docs/packet-captures.md@@ -54,6 +54,22 @@ resource limit; use selected fields, a display filter or an input-packet limitwhen dissection output is too large. Each call checks the recording's currentdigest, so pages from different source digests must not be combined.+For a question about a known input prefix, select `packet_limit` for that+prefix, the fields needed for the answer, and a matching output window. For+example, a question about the first twelve PostgreSQL TCP packets within+original frames 1–2,000 can use fields `tcp.srcport`, `tcp.dstport`, `tcp.flags`,+filter `tcp.port == 5432`, `packet_limit: 2000` and `match_limit: 12`.+The matching count then describes that input prefix. A filter containing+`frame.number <= 2000` without an input limit still reads the entire capture;+returning twelve rows does not limit producer output. These controls answer a+bounded question and must not be used to infer whole-capture absence.++Discover unfamiliar field names with `inspect_packet_fields`, using an exact+protocol and a narrow query. If the installed names are already established,+direct field selection avoids an additional glossary probe. Count discovery+and any retries when comparing investigation cost; a small page alone does+not establish a faster workflow.+Additional selections:- `display_filter`: Wireshark display-filter syntax; omitted means no filter.diff --git a/experiments/analysis-efficiency/README.md b/experiments/analysis-efficiency/README.mdindex cdb6c3c9b..9a14b764a 100644--- a/experiments/analysis-efficiency/README.md+++ b/experiments/analysis-efficiency/README.md@@ -72,6 +72,12 @@ checkpoint commits retain their original receipts; no history was rewritten.## Optimization priorities and gradual migration+The [larger/non-HTTP packet checkpoint](packet-field-workflows.md) records+same-question SDK and actual Codex comparisons using existing input limits,+field/filter selection, discovery and output windows. Lower delivered JSON did+not establish stable total-token or speed gains; failed plans and negative+controls remain visible.+Agent token consumption and end-to-end task time are the primary objectives.Memory and CPU consumption are also measured and controlled. Correctness,Evidence provenance, caller choices and existing-user compatibility arediff --git a/experiments/analysis-efficiency/agent-trial.py b/experiments/analysis-efficiency/agent-trial.pyindex edb848a28..db994cea1 100644--- a/experiments/analysis-efficiency/agent-trial.py+++ b/experiments/analysis-efficiency/agent-trial.py@@ -181,6 +181,7 @@ def main():'inspect_android_class', 'inspect_android_method','inspect_firmware_regions', 'inspect_managed_artifact','inspect_sqlite_database', 'inspect_pcap_capture',+ 'inspect_packet_capture', 'inspect_packet_fields','inspect_managed_members', 'inspect_managed_native_boundaries','inspect_artifact', 'inspect_wasm_artifact','inspect_evm_interface', 'get_evidence_bundle', 'export_evidence_bundle',diff --git a/experiments/analysis-efficiency/compare.mjs b/experiments/analysis-efficiency/compare.mjsindex 4e33e5ead..01b9f97af 100644--- a/experiments/analysis-efficiency/compare.mjs+++ b/experiments/analysis-efficiency/compare.mjs@@ -2,6 +2,7 @@ import { isDeepStrictEqual } from "node:util";import { readFile, writeFile } from "node:fs/promises";import { resolve } from "node:path";import { parseArgs } from "node:util";+import { PACKET_ARTIFACTS } from "./packet-entry.mjs";const { values } = parseArgs({options: {@@ -33,6 +34,8 @@ const sameSettings = ["provider_environment_policy_sha256","measure_source_sha256","native_workflow_source_sha256",+ "packet_workflow_source_sha256",+ "packet_oracle_source_sha256","package_lock_sha256","node_version","sdk_client_version",@@ -76,6 +79,7 @@ for (const [side, answer] of [// coverage. Repeated equivalent deliveries must also match every observation// field except the receipt and parent IDs, which identify their own invocations.if (+ Object.hasOwn(PACKET_ARTIFACTS, left.artifact) ||["wireshark-http.cap", "wireshark-http_redirects.pcapng"].includes(left.artifact,)@@ -94,7 +98,11 @@ if (path: ["delivery_observations"],value: answer.delivery_observations,});- if (left.delivery === right.delivery) {+ if (+ left.delivery === right.delivery &&+ (!Object.hasOwn(PACKET_ARTIFACTS, left.artifact) ||+ left.packet_selection === right.packet_selection)+ ) {answer.delivery_observations = answer.delivery_observations.map((observation) =>Object.fromEntries(diff --git a/experiments/analysis-efficiency/corpus.json b/experiments/analysis-efficiency/corpus.jsonindex 7896c558d..e595dde34 100644--- a/experiments/analysis-efficiency/corpus.json+++ b/experiments/analysis-efficiency/corpus.json@@ -291,6 +291,58 @@"upstream_expected_digest": null,"derivation": "Unmodified capture published by official Wireshark wiki. Local digest pins downloaded bytes; no independently published upstream digest is claimed."}+ },+ {+ "id": "wireshark-pgsql-jdbc.pcap",+ "file": "pgsql-jdbc.pcap",+ "bytes": 4906316,+ "sha256": "6e68b8b792a0f4336092f98856abc7de8c54b55a29112854d28d676f9093c16b",+ "provenance": {+ "project": "Wireshark public sample captures",+ "source_page": "https://wiki.wireshark.org/SampleCaptures",+ "url": "https://wiki.wireshark.org/uploads/__moin_import__/attachments/SampleCaptures/pgsql-jdbc.pcap.gz",+ "resolved_url": "https://gitlab.com/wireshark/wireshark/-/wikis/uploads/__moin_import__/attachments/SampleCaptures/pgsql-jdbc.pcap.gz",+ "upstream_expected_digest": null,+ "license": "not stated in inspected upstream catalogue",+ "derivation": "Unmodified official saved sample; decompressed gzip where indicated. Local digest pins acquired bytes, not an upstream-published digest.",+ "compressed_source": {+ "name": "pgsql-jdbc.pcap.gz",+ "bytes": 590093,+ "sha256": "15b51bb4cc197753c1ac6b7dc3cf5697f22a81e3bb5d2e1d24d15de469c3a06b"+ }+ }+ },+ {+ "id": "wireshark-dns.cap",+ "file": "dns.cap",+ "bytes": 4338,+ "sha256": "041eeb6f98bb398f1ee8b09651b5b5a84f6a62639f95bf226f9e7b77355d9f28",+ "provenance": {+ "project": "Wireshark public sample captures",+ "source_page": "https://wiki.wireshark.org/SampleCaptures",+ "url": "https://wiki.wireshark.org/uploads/__moin_import__/attachments/SampleCaptures/dns.cap",+ "resolved_url": "https://gitlab.com/wireshark/wireshark/-/wikis/uploads/__moin_import__/attachments/SampleCaptures/dns.cap",+ "upstream_expected_digest": null,+ "license": "not stated in inspected upstream catalogue",+ "derivation": "Unmodified official saved sample; decompressed gzip where indicated. Local digest pins acquired bytes, not an upstream-published digest.",+ "compressed_source": null+ }+ },+ {+ "id": "wireshark-dhcp.pcap",+ "file": "dhcp.pcap",+ "bytes": 1400,+ "sha256": "2471b5420bdac826eecf8f61a2bbb4a3eb20dbfab7c02ff2be502f349f368214",+ "provenance": {+ "project": "Wireshark public sample captures",+ "source_page": "https://wiki.wireshark.org/SampleCaptures",+ "url": "https://wiki.wireshark.org/uploads/__moin_import__/attachments/SampleCaptures/dhcp.pcap",+ "resolved_url": "https://gitlab.com/wireshark/wireshark/-/wikis/uploads/__moin_import__/attachments/SampleCaptures/dhcp.pcap",+ "upstream_expected_digest": null,+ "license": "not stated in inspected upstream catalogue",+ "derivation": "Unmodified official saved sample; decompressed gzip where indicated. Local digest pins acquired bytes, not an upstream-published digest.",+ "compressed_source": null+ }}]}diff --git a/experiments/analysis-efficiency/measure.mjs b/experiments/analysis-efficiency/measure.mjsindex 60dce4f80..07a39e3c1 100644--- a/experiments/analysis-efficiency/measure.mjs+++ b/experiments/analysis-efficiency/measure.mjs@@ -12,6 +12,7 @@ import {import { Client } from "@modelcontextprotocol/client";import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";import { Ajv2020 } from "ajv/dist/2020.js";+import { PACKET_ARTIFACTS, measurePacketEntry } from "./packet-entry.mjs";const root = resolve(dirname(fileURLToPath(import.meta.url)), "../..");const { values } = parseArgs({@@ -29,6 +30,7 @@ const { values } = parseArgs({"managed-question": { type: "string", default: "inventory" },schemas: { type: "string", default: "default" },"sqlite-selection": { type: "string", default: "all" },+ "packet-selection": { type: "string", default: "baseline" },},});if (@@ -45,6 +47,15 @@ if (!["inventory", "selected"].includes(values["javascript-question"]) ||!["default", "selected"].includes(values.schemas) ||!["all", "columns"].includes(values["sqlite-selection"]) ||+ !["baseline", "focused", "discovery", "pages"].includes(+ values["packet-selection"],+ ) ||+ (!Object.hasOwn(PACKET_ARTIFACTS, values.artifact) &&+ values["packet-selection"] !== "baseline") ||+ (Object.hasOwn(PACKET_ARTIFACTS, values.artifact) &&+ values.delivery !== "complete") ||+ (values["packet-selection"] === "pages" &&+ values.artifact !== "wireshark-pgsql-jdbc.pcap") ||(values.artifact !== "qgis-world-map.gpkg" &&values["sqlite-selection"] !== "all") ||(values.artifact === "qgis-world-map.gpkg" &&@@ -142,6 +153,12 @@ const fingerprints = {native_workflow_source_sha256: await fileSha(fileURLToPath(new URL("./native-entry.mjs", import.meta.url)),),+ packet_workflow_source_sha256: await fileSha(+ fileURLToPath(new URL("./packet-entry.mjs", import.meta.url)),+ ),+ packet_oracle_source_sha256: await fileSha(+ fileURLToPath(new URL("./packet-oracle.py", import.meta.url)),+ ),runtime_files_sha256: sha(JSON.stringify(runtimeFiles)),runtime_dispatcher_sha256: await fileSha(resolve(values["runtime-root"], "scripts/rea.mjs"),@@ -722,6 +739,12 @@ const inspectPcapTarget = async () => {};const workflows = {+ ...Object.fromEntries(+ Object.keys(PACKET_ARTIFACTS).map((artifact) => [+ artifact,+ () => measurePacketEntry({ values, call, inputSha, catalogDefinitions }),+ ]),+ ),"wireshark-http.cap": inspectPcapTarget,"wireshark-http_redirects.pcapng": inspectPcapTarget,"qgis-world-map.gpkg": async () => {@@ -1160,6 +1183,9 @@ const report = {values.artifact === "qgis-world-map.gpkg"? values["sqlite-selection"]: null,+ packet_selection: Object.hasOwn(PACKET_ARTIFACTS, values.artifact)+ ? values["packet-selection"]+ : null,provider_settings_sha256: sha(JSON.stringify(selectedProviders)),provider_search_path_sha256: sha(selectedProviders.PATH ?? process.env.PATH ?? "",diff --git a/experiments/analysis-efficiency/packet-entry.mjs b/experiments/analysis-efficiency/packet-entry.mjsnew file mode 100644index 000000000..5c6722a74--- /dev/null+++ b/experiments/analysis-efficiency/packet-entry.mjs@@ -0,0 +1,316 @@+import { isDeepStrictEqual } from "node:util";+import { resolve } from "node:path";+import { pathToFileURL } from "node:url";+import { fileURLToPath } from "node:url";+import { execFile } from "node:child_process";+import { promisify } from "node:util";++export const verifyPacketPage = (first, next, oracle) => {+ if (+ first.artifact.sha256 !== oracle.sha256 ||+ next.artifact.sha256 !== oracle.sha256 ||+ first.output_window.matching_packets !== oracle.matching_packets ||+ next.output_window.matching_packets !== oracle.matching_packets ||+ !isDeepStrictEqual(+ packetFacts("postgres", first).rows,+ oracle.rows.slice(0, 12),+ ) ||+ !isDeepStrictEqual(+ packetFacts("postgres", next).rows,+ oracle.rows.slice(12, 24),+ )+ )+ throw new Error(+ "Packet pages differ from independently parsed original matches",+ );+};++export const PACKET_ARTIFACTS = {+ "wireshark-pgsql-jdbc.pcap": {+ kind: "postgres",+ fields: ["tcp.srcport", "tcp.dstport", "tcp.flags"],+ extra: ["tcp.len"],+ protocol: "tcp",+ query: "tcp.flags",+ discovered: "tcp.flags",+ filter: "tcp.port == 5432",+ },+ "wireshark-dns.cap": {+ kind: "dns",+ fields: [+ "dns.id",+ "dns.flags.response",+ "dns.flags.rcode",+ "dns.qry.name",+ "dns.qry.type",+ ],+ extra: ["dns.resp.name", "dns.resp.ttl", "udp.srcport", "udp.dstport"],+ protocol: "dns",+ query: "dns.flags.rcode",+ discovered: "dns.flags.rcode",+ filter: "dns.flags.response == 1 && dns.flags.rcode != 0",+ },+ "wireshark-dhcp.pcap": {+ kind: "dhcp",+ fields: [+ "dhcp.id",+ "dhcp.option.dhcp",+ "dhcp.ip.your",+ "dhcp.option.dhcp_server_id",+ ],+ extra: [+ "dhcp.type",+ "dhcp.ip.client",+ "dhcp.hw.mac_addr",+ "udp.srcport",+ "udp.dstport",+ ],+ protocol: "dhcp",+ query: "Option 53:",+ discovered: "dhcp.option.dhcp",+ filter: "dhcp",+ },+};++export const packetRequest = (artifact, path, selection) => {+ const descriptor = PACKET_ARTIFACTS[artifact];+ if (!descriptor) throw new Error("Unknown pinned packet question");+ const baseline = selection === "baseline";+ return {+ path,+ fields: baseline+ ? [...descriptor.fields, ...descriptor.extra]+ : descriptor.fields,+ display_filter:+ descriptor.kind === "postgres" && baseline+ ? `${descriptor.filter} && frame.number <= 2000`+ : descriptor.kind === "dns" && baseline+ ? "dns"+ : descriptor.filter,+ ...(descriptor.kind === "postgres" && !baseline+ ? { packet_limit: 2000, match_limit: 12 }+ : {}),+ };+};++const number = (row, field) => {+ const value = row[field];+ if (+ !Array.isArray(value) ||+ value.length !== 1 ||+ !/^(?:[0-9]+|0x[0-9a-f]+)$/iu.test(value[0])+ )+ throw new Error(`Expected one recorded integer for ${field}`);+ const parsed = Number(value[0]);+ if (!Number.isSafeInteger(parsed))+ throw new Error(`Unsafe recorded integer for ${field}`);+ return parsed;+};++export const packetFacts = (kind, report) => {+ let packets = report.packets;+ if (kind === "postgres") packets = packets.slice(0, 12);+ if (kind === "dns")+ packets = packets.filter(+ (row) =>+ number(row, "dns.flags.response") === 1 &&+ number(row, "dns.flags.rcode") !== 0,+ );+ return {+ kind,+ matching_packets:+ kind === "dns" ? packets.length : report.output_window.matching_packets,+ rows: packets.map((row) => {+ const frame = number(row, "frame.number");+ if (kind === "postgres")+ return {+ frame,+ source_port: number(row, "tcp.srcport"),+ destination_port: number(row, "tcp.dstport"),+ flags: number(row, "tcp.flags"),+ };+ if (kind === "dns") {+ const names = row["dns.qry.name"];+ const types = row["dns.qry.type"];+ if (+ !Array.isArray(names) ||+ names.length === 0 ||+ !Array.isArray(types) ||+ types.length !== names.length+ )+ throw new Error(+ "The fixed DNS question requires recorded names and types; missing fields are unknown",+ );+ return {+ frame,+ id: number(row, "dns.id"),+ rcode: number(row, "dns.flags.rcode"),+ names,+ types: types.map((value) => number({ type: [value] }, "type")),+ };+ }+ const server = row["dhcp.option.dhcp_server_id"];+ const address = row["dhcp.ip.your"];+ if (+ !Array.isArray(address) ||+ address.length !== 1 ||+ (server !== undefined &&+ (!Array.isArray(server) || server.length !== 1))+ )+ throw new Error("Expected recorded DHCP address scalar");+ return {+ frame,+ id: number(row, "dhcp.id"),+ message_type: number(row, "dhcp.option.dhcp"),+ your_address: address[0],+ ...(server === undefined ? {} : { server_identifier: server[0] }),+ };+ }),+ };+};++export const measurePacketEntry = async ({+ values,+ call,+ inputSha,+ catalogDefinitions,+}) => {+ const descriptor = PACKET_ARTIFACTS[values.artifact];+ const { parseEvidence } = await import(+ pathToFileURL(resolve(values["runtime-root"], "dist/domain/evidence.js"))+ .href+ );+ const { inspectPacketCaptureInputSchema, packetCaptureSchema } = await import(+ pathToFileURL(+ resolve(values["runtime-root"], "dist/domain/captures/packetCapture.js"),+ ).href+ );+ const { inspectPacketFieldsInputSchema, packetFieldsCatalogSchema } =+ await import(+ pathToFileURL(+ resolve(values["runtime-root"], "dist/domain/captures/packetFields.js"),+ ).href+ );+ const observations = [];+ const selection = values["packet-selection"];+ const checked = (evidence, operation, input, schema) => {+ parseEvidence(evidence);+ const report = schema.parse(evidence.normalized_result);+ if (+ evidence.operation !== operation ||+ !isDeepStrictEqual(evidence.parameters, input) ||+ !isDeepStrictEqual(report.selection, input)+ )+ throw new Error(+ "Packet Evidence operation or choices differs from the request",+ );+ if (+ operation === "inspect_packet_capture" &&+ (evidence.subject?.local_path !== values.path ||+ evidence.subject?.digest?.sha256 !== inputSha ||+ report.artifact.path !== values.path ||+ report.artifact.sha256 !== inputSha)+ )+ throw new Error("Packet Evidence identifies different capture bytes");+ if (+ operation === "inspect_packet_fields" &&+ (evidence.subject?.local_path !== report.tool.path ||+ evidence.subject?.digest?.sha256 !== report.tool.sha256)+ )+ throw new Error("Field discovery identifies a different executable");+ observations.push({+ evidence_id: evidence.evidence_id,+ operation,+ selection: report.selection,+ output_window: report.output_window,+ read_scope: report.read_scope ?? null,+ limitations: evidence.limitations,+ });+ return report;+ };+ if (selection === "discovery") {+ const input = inspectPacketFieldsInputSchema.parse({+ kind: "fields",+ protocol: descriptor.protocol,+ query: descriptor.query,+ limit: 1,+ });+ const found = checked(+ await call("inspect_packet_fields", input),+ "inspect_packet_fields",+ input,+ packetFieldsCatalogSchema,+ );+ if (+ !found.records.some((row) => row.abbreviation === descriptor.discovered)+ )+ throw new Error(+ "Selected installation does not advertise the required protocol field",+ );+ }+ const input = inspectPacketCaptureInputSchema.parse(+ packetRequest(values.artifact, values.path, selection),+ );+ for (const name of [+ "inspect_packet_capture",+ ...(selection === "discovery" ? ["inspect_packet_fields"] : []),+ ])+ if (!catalogDefinitions.has(name))+ throw new Error(`Missing live packet contract ${name}`);+ const report = checked(+ await call("inspect_packet_capture", input),+ "inspect_packet_capture",+ input,+ packetCaptureSchema,+ );+ const facts = packetFacts(descriptor.kind, report);+ if (selection === "pages") {+ const { stdout } = await promisify(execFile)(+ "python3",+ [+ fileURLToPath(new URL("./packet-oracle.py", import.meta.url)),+ "postgres",+ values.path,+ "--row-limit",+ "24",+ ],+ { timeout: 30_000, maxBuffer: 128 * 1024 },+ );+ const oracle = JSON.parse(stdout);+ const nextInput = inspectPacketCaptureInputSchema.parse({+ ...input,+ match_offset: 12,+ });+ const next = checked(+ await call("inspect_packet_capture", nextInput),+ "inspect_packet_capture",+ nextInput,+ packetCaptureSchema,+ );+ const repeated = checked(+ await call("inspect_packet_capture", input),+ "inspect_packet_capture",+ input,+ packetCaptureSchema,+ );+ verifyPacketPage(report, next, oracle);+ if (+ !isDeepStrictEqual(report.packets, repeated.packets) ||+ !isDeepStrictEqual(report.output_window, repeated.output_window) ||+ next.artifact.sha256 !== report.artifact.sha256 ||+ next.output_window.matching_packets !==+ report.output_window.matching_packets ||+ Number(next.packets[0]?.["frame.number"]?.[0]) <=+ Number(report.packets.at(-1)?.["frame.number"]?.[0])+ )+ throw new Error(+ "Packet page order, digest or repeated observations changed",+ );+ }+ return {+ artifact_sha256: inputSha,+ facts,+ delivery_observations: observations,+ evidence_ids: [...new Set(observations.map((row) => row.evidence_id))],+ };+};diff --git a/experiments/analysis-efficiency/packet-field-workflows.md b/experiments/analysis-efficiency/packet-field-workflows.mdnew file mode 100644index 000000000..8f893c30c--- /dev/null+++ b/experiments/analysis-efficiency/packet-field-workflows.md@@ -0,0 +1,226 @@+# Larger and non-HTTP packet investigation costs++This closes the bounded experiment in [#3130](https://github.com/morluto/rea/issues/3130).+Existing field, filter and input/output-window controls substantially reduce+delivered JSON for the selected questions. Actual Codex total input and elapsed+time are less consistent. Requiring field discovery for a known field adds cost.+These results support documenting caller-selected controls, while retaining the+current API, defaults and uncached provider behavior.++## Runtime and public corpus++Measurements use immutable main+`ec67aebed02dbd20fa5c7703fcd183ad0b1c343c`, Node 24.18.0, the locked MCP client and+server, Linux, TShark/capinfos 4.0.17 and util-linux prlimit. The SDK captures live+advertised contracts and passes selected definitions to actual public MCP calls.+Runtime-file, dispatcher, lockfile, experiment-source and explicit provider+configuration digests remain with the private trial records. Subsequent main+integration checks do not relabel these measurements.++Three additional saved recordings come from the+[official Wireshark catalogue](https://wiki.wireshark.org/SampleCaptures).+`corpus.json` pins their acquisition URLs, exact bytes and SHA-256 values.+The PostgreSQL recording is described upstream as a JDBC test session; it is+not a representative production workload. Capture-specific licenses were not+stated in the inspected catalogue. No captures or raw trial records are+redistributed in Git.++| Recording | Acquired/analyzed size | Recorded frames | Fixed question |+| -------------------- | --------------------------------------------: | --------------: | ------------------------------------------------------------------------------------------------------------------------------------------------------ |+| `pgsql-jdbc.pcap.gz` | 590,093 compressed / 4,906,316 expanded bytes | 18,472 | Within original frames 1–2,000, report the first twelve TCP packets using port 5432, their source/destination ports and flags, and the matching count. |+| `dns.cap` | 4,338 bytes | 38 | Report nonzero-RCODE DNS responses, transaction IDs and question names/types. |+| `dhcp.pcap` | 1,400 bytes | 4 | Report message types, transaction IDs, offered address and present server identifiers in frame order. |++The larger application recording and two small non-HTTP controls expand the+question/protocol coverage. Large DNS/DHCP, PCAPNG non-HTTP, USB, Bluetooth,+Windows and macOS provider performance remain unverified.++`packet-oracle.py` independently reads classic PCAP, Ethernet/IPv4 and selected+TCP/UDP headers. It does not consume TShark output. Its narrow supported scope+fails on truncation, IP fragmentation, DNS compression cycles, duplicate DHCP+options or DHCP overload. TCP header facts follow+[RFC 9293](https://www.rfc-editor.org/rfc/rfc9293.html), DNS names/header fields+[RFC 1035](https://www.rfc-editor.org/rfc/rfc1035.html), and DHCP options+[RFC 2132](https://www.rfc-editor.org/rfc/rfc2132.html).+SQL interpretation, TCP reassembly and network completeness are outside the+oracle. The question and oracle were fixed before measured cohorts.++The oracle establishes 1,337 selected PostgreSQL TCP packets in the input prefix+and six DNS error responses. DHCP records contain Discover/Offer with transaction+ID 15645 and Request/ACK with ID 15646. An absent server identifier stays omitted;+the recording does not establish a single transaction containing all four steps.++## Equivalent questions and SDK measurements++Each arm uses fresh actual MCP sessions; heavy trials run sequentially.+Two interleaved repetitions cover each arm and each recording. Discovery, catalog,+RPC, schema checking, encoding and cleanup costs are retained. JSON bytes describe+serialized SDK replies, not raw transport bytes or model tokens.++The PostgreSQL baseline selects `tcp.srcport`, `tcp.dstport`, `tcp.flags` and+`tcp.len`, retains the default fifty-row output window and reads to EOF with+`tcp.port == 5432 && frame.number <= 2000`. The focused arm selects the three+required fields, uses `packet_limit: 2000`, filter `tcp.port == 5432` and+`match_limit: 12`. Both establish the same requested prefix facts and count.+Their source read scopes differ explicitly: to EOF versus an input limit.+Neither scope establishes whole-network completeness.++DNS focuses the nine-field baseline on five requested fields and the error-response+filter. DHCP selects four required fields from the nine-field baseline; all four+records are still read. The discovery arm adds one narrow installed-field probe+before the focused request. It tests the cost of validating an already specified+field, rather than proving a benefit for a previously unknown field or guessing+that an installed registration appears in traffic.++| Question | Baseline reply bytes | Focused reply bytes | Focused change | Discovery + focused bytes | Discovery change |+| ----------------- | -------------------: | ------------------: | -------------: | ------------------------: | ---------------: |+| PostgreSQL prefix | 23,054 | 13,304 | −42.29% | 21,096 | −8.49% |+| DNS errors | 29,002 | 13,424 | −53.71% | 21,226 | −26.81% |+| DHCP sequence | 13,770 | 12,020 | −12.71% | 19,840 | +44.08% |++Both repetitions produce the same reply size in each cell and pass the independent+fact comparison. Authentication uses the selected runtime's `parseEvidence` and+named input/output schemas, checking operation, normalized caller choices,+artifact or executable identity, original frame order and omission semantics.++| Question | Baseline tool-call time, ms | Focused tool-call time, ms | Discovery + focused tool-call time, ms |+| ----------------- | --------------------------- | -------------------------- | -------------------------------------- |+| PostgreSQL prefix | 1,670 / 2,255 | 1,708 / 933 | 3,549 / 2,674 |+| DNS errors | 892 / 786 | 889 / 778 | 2,506 / 2,608 |+| DHCP sequence | 763 / 759 | 755 / 731 | 2,502 / 2,502 |++Whole SDK workflow times, including connection/catalog, vary from 4.23–8.79 seconds+for these arms. The PostgreSQL focused tool time improves in one repetition and+regresses slightly in the other. Discovery consistently adds tool work. Two+repetitions are descriptive observations, not a statistical speed guarantee.++A separate first-page, next-page and repeated-first-page sequence costs+922 / 804 / 1,244 ms and returns 39,918 reply bytes in total. Each call repeats+dissection, reports the same 1,337 matching count and capture digest, and preserves+the original frame order. Identical repeated observations retain the same+content-addressed Evidence ID; distinct pages have distinct IDs. Page delivery+does not establish capture or network completeness. This narrow repeat does not+justify introducing a generic cache or weakening provider identity/cleanup rules.++The two original HTTP controls were also replayed with two SDK pairs each on the+new pinned runtime. Useful summary/prefix facts remain equivalent. For `http.cap`,+complete delivery returns 7,601 bytes and summary plus a two-URI retained page+17,540 bytes (+130.76%); for `http_redirects.pcapng`, 7,535 versus 17,240 bytes+(+128.80%). These remain negative controls. The original model observations in+[the earlier checkpoint](pcap-small-captures.md) keep their original runtime pin.++## Actual Codex CLI observations++Six actual Codex CLI 0.161.0 runs use requested alias `gpt-5.6-terra`, reasoning+`high`, a disposable configuration and the pinned REA MCP runtime. Resolved+server-side model identity is not inferred. Both tool names are enabled in every+arm, exact-name discovery guidance is identical, shell/browser/other-agent tools+are disabled and copied authentication is removed after each run.+The shared question and generated plans contain no oracle answers.++Order is baseline 1, focused 1, discovery 1, focused 2, baseline 2, discovery 2.+The rubric checks authenticated actual tool results, exact header-derived facts,+citations and the planned calls. Answer correctness and adherence to the plan+are reported separately. Limitation wording is reviewed in transcripts; a string+or string array is accepted because the prompt does not require one representation.++| Trial | Total input tokens | Cached input tokens | Output tokens | Elapsed seconds | Actual MCP calls | Facts / plan |+| ----------- | -----------------: | ------------------: | ------------: | --------------: | ---------------: | ---------------------------------- |+| Baseline 1 | 40,649 | 27,904 | 1,574 | 27.989 | 1 | pass / pass |+| Focused 1 | 36,642 | 19,968 | 943 | 38.542 | 1 | pass / pass |+| Discovery 1 | 52,802 | 33,024 | 1,089 | 44.398 | 2 | pass / pass |+| Focused 2 | 40,912 | 22,016 | 726 | 32.520 | 1 | pass / pass |+| Baseline 2 | 40,181 | 19,968 | 896 | 40.299 | 1 | pass / pass |+| Discovery 2 | 48,754 | 30,976 | 1,780 | 76.081 | 1 | pass / **fail: discovery omitted** |++The two eligible baseline/focused pairs show total-input changes of −9.86% and++1.82%, and elapsed changes of +37.70% and −19.30%. Thus reduced SDK reply size+does not establish a stable total-token or speed improvement in these runs.+The first complete discovery comparison adds 29.90% input and 58.63% elapsed time.+The second discovery attempt is retained as a workflow failure and cannot serve+as a successful two-call discovery comparison. Its cost is not discarded or+counted as a passing trial. No model-side recovery or retries occurred in these+six attempts.++An earlier configuration preflight rejected the packet tool under Codex's+`approval_policy: never`: 36,651 input tokens, 549 output tokens, 21.603 seconds,+one rejected call and no packet observations. The experiment runner now explicitly+preapproves the two authorized passive packet tools, consistently across all arms.+That failed preflight remains separate from the corrected cohort and its cost is+reported here; CLI exit zero alone is never a factual or workflow pass.++## Resource and compatibility scope++All engine/model workloads use an owned Linux resource scope with one CPU,+3 GiB early-stop threshold, 4 GiB hard memory ceiling, no swap and a 192-task cap.+Node's heap is additionally limited to 512 MiB. The nineteen non-HTTP SDK trials+have a maximum observed payload-exit memory peak of 860,909,568 bytes; the six model+attempts peak at 940,806,144 bytes. Payload-exit counters and cleanup were verified+with the observer still alive and report no memory-limit/OOM events in that phase.+Final whole-scope counters remain unavailable, so no stronger whole-scope claim+is made. CPU usage and observed process ownership remain in each private receipt.++A full PostgreSQL probe with four selected fields and no input prefix exceeds the+existing 4 MiB producer-output ceiling even with the default fifty-row window.+It correctly fails instead of delivering partial success. The SDK error retains+large truncated diagnostics (5,790,979 serialized bytes); this separate product+concern is recorded privately for later repair, outside #3130. Paging alone cannot+prevent producer expansion. The successful prefix experiment does not prove that+the full recording can be inspected with those selections.++Only experiment sources, corpus provenance, a question template and guidance+change. Production schemas, provider limits, CLI/MCP defaults, Evidence identity+and session/resource ownership stay as before. Eight parser regressions retain+malformed/truncated-input and omission checks. Existing clients need no migration.++## Reproduction++Acquire the exact corpus outside the repository and verify the pinned hashes.+Use the pinned Node/npm toolchain, locked dependencies, `npm run build:cached` and+an external resource guard before invoking actual engines/models. Supply explicit+provider settings in a private JSON file. An example SDK arm is:++```sh+node experiments/analysis-efficiency/measure.mjs \+ --runtime-root /absolute/pinned/runtime --revision PINNED_SHA \+ --artifact wireshark-pgsql-jdbc.pcap --path /private/pgsql-jdbc.pcap \+ --providers /private/providers.json --schemas selected \+ --packet-selection focused --output /private/new-trial+```++Use `baseline`, `focused`, `discovery`, or the PostgreSQL-only `pages` diagnostic.+Run `packet-oracle.py postgres PATH` independently and compare `answer.json` facts.+Generate each model prompt with `packet-model-prompt.mjs --selection ARM`, then use+the existing `agent-trial.py` with those two enabled tools and context capture.+Run `review-packet.mjs` against the actual trial, oracle, runtime and selected arm.+Its nonzero exit includes a plan failure even when all requested facts are correct.+Retain failed attempts, actual usage, digests, coverage, cleanup and resource+receipts alongside passing observations. No raw results need to be committed.++## Review verification++The comparator now recognizes the three non-HTTP questions: observations remain+attached to each run, while different plans compare requested facts. Repeated+identical plans still compare stable observation scope. Capture grading authenticates+Evidence against the actual call and assesses plan differences separately; an+actual 24-row SDK window replay retains a factual pass and a plan failure. This+is a grading replay, not an additional model trial. The stricter header oracle+preserves all original sample facts and rejects truncated IPv4/TCP options and+unsupported Ethernet protocols. The page diagnostic now derives matches 13–24+independently before accepting the second page; skipped, reversed and changed-byte+pages are rejected. Original cohort measurements retain their recorded source+digests and runtime pin; review checks do not relabel them.++Capture grading keeps authenticated facts by Evidence ID and evaluates the cited+observations. Four replays using actual first-page and second-page SDK Evidence+verify that later or reordered pages and repeated reads preserve a correct cited+answer, while citing only the wrong page fails factual grading. Extra calls still+fail the fixed workflow plan. Regrading all six original model records preserves+their outcomes and recorded costs; these replays are not new model trials.++Failed or malformed calls remain workflow failures and never become authenticated+Evidence. Nine failure-injection and answer-shape replays using an actual successful+capture verify that later recovery preserves supported facts, while failed-only+attempts and missing or unobserved citations fail factual grading. Missing+limitations and extra answer keys fail the fixed plan without changing correct+recorded facts. Successful original-model outcomes remain unchanged.diff --git a/experiments/analysis-efficiency/packet-model-prompt.mjs b/experiments/analysis-efficiency/packet-model-prompt.mjsnew file mode 100644index 000000000..f49348fbd--- /dev/null+++ b/experiments/analysis-efficiency/packet-model-prompt.mjs@@ -0,0 +1,40 @@+import { readFile } from "node:fs/promises";+import { parseArgs } from "node:util";+import { packetRequest } from "./packet-entry.mjs";++const { values } = parseArgs({ options: { selection: { type: "string" } } });+if (!["baseline", "focused", "discovery"].includes(values.selection))+ throw new Error("Select baseline, focused or discovery");+const question = await readFile(+ new URL("./questions/packet-postgres.txt", import.meta.url),+ "utf8",+);+const plan = [+ ...(values.selection === "discovery"+ ? [+ {+ tool: "inspect_packet_fields",+ arguments: {+ kind: "fields",+ protocol: "tcp",+ query: "tcp.flags",+ limit: 1,+ },+ },+ ]+ : []),+ {+ tool: "inspect_packet_capture",+ arguments: packetRequest(+ "wireshark-pgsql-jdbc.pcap",+ "{{artifact_path}}",+ values.selection,+ ),+ },+];+process.stdout.write(+ question ++ "\nExecute this fixed public REA MCP plan in order. These are input selections, not expected answers. Preserve omitted defaults. All calls count toward the comparison.\n" ++ JSON.stringify(plan, null, 2) ++ "\n",+);diff --git a/experiments/analysis-efficiency/packet-oracle.py b/experiments/analysis-efficiency/packet-oracle.pynew file mode 100644index 000000000..db5797b5f--- /dev/null+++ b/experiments/analysis-efficiency/packet-oracle.py@@ -0,0 +1,180 @@+"""Independent, narrow oracle for the three pinned Ethernet/IPv4 PCAPs.++Reads header bytes, not TShark output. No PCAPNG, IP fragmentation or TCP+reassembly support is claimed. DNS compression follows RFC 1035; DHCP options+53/54 follow RFC 2132. Unsupported selected traffic fails rather than guessing.+"""+import argparse+import hashlib+import json+import struct+from pathlib import Path+++def require(condition, reason):+ if not condition:+ raise ValueError(reason)+++def packets(data):+ require(len(data) >= 24, 'truncated PCAP header')+ magic = data[:4]+ require(magic in (b'\xd4\xc3\xb2\xa1', b'\xa1\xb2\xc3\xd4'),+ 'oracle requires microsecond classic PCAP')+ endian = '<' if magic == b'\xd4\xc3\xb2\xa1' else '>'+ require(struct.unpack_from(endian + 'HH', data, 4) == (2, 4),+ 'unsupported PCAP version')+ require(struct.unpack_from(endian + 'I', data, 20)[0] == 1,+ 'oracle requires Ethernet')+ offset, frame = 24, 0+ while offset < len(data):+ require(offset + 16 <= len(data), 'truncated PCAP record header')+ captured, original = struct.unpack_from(endian + 'II', data, offset + 8)+ offset += 16+ require(captured <= original and offset + captured <= len(data),+ 'invalid captured packet length')+ frame += 1+ yield frame, data[offset:offset + captured]+ offset += captured+++def transport(packet):+ require(len(packet) >= 14, 'truncated Ethernet header')+ if packet[12:14] == b'\x08\x06':+ return None # ARP cannot contain the selected TCP/UDP messages.+ require(packet[12:14] == b'\x08\x00', 'unsupported Ethernet protocol')+ ip = packet[14:]+ require(len(ip) >= 20 and ip[0] >> 4 == 4, 'invalid IPv4 header')+ header = (ip[0] & 15) * 4+ total = struct.unpack_from('!H', ip, 2)[0]+ require(20 <= header <= len(ip) and total >= header, 'invalid IPv4 lengths')+ require(total <= len(ip), 'truncated IPv4 packet')+ require(struct.unpack_from('!H', ip, 6)[0] & 0x3fff == 0,+ 'fragmentation is outside oracle scope')+ return ip[9], ip[header:min(total, len(ip))]+++def dns_name(message, offset):+ labels, visited, end = [], set(), None+ while True:+ require(offset < len(message) and offset not in visited,+ 'invalid DNS name or compression cycle')+ visited.add(offset)+ size = message[offset]+ if size & 0xc0 == 0xc0:+ require(offset + 1 < len(message), 'truncated DNS compression pointer')+ if end is None:+ end = offset + 2+ offset = ((size & 0x3f) << 8) | message[offset + 1]+ continue+ require(size & 0xc0 == 0 and offset + 1 + size <= len(message),+ 'invalid DNS label')+ offset += 1+ if size == 0:+ return '.'.join(labels), end if end is not None else offset+ labels.append(message[offset:offset + size].decode('ascii'))+ require(sum(len(label) + 1 for label in labels) <= 255,+ 'DNS name exceeds protocol length')+ offset += size+++def dhcp_options(payload):+ require(len(payload) >= 240 and payload[236:240] == b'\x63\x82\x53\x63',+ 'invalid DHCP fixed header or cookie')+ options, offset = {}, 240+ while offset < len(payload):+ kind = payload[offset]+ offset += 1+ if kind == 255:+ return options+ if kind == 0:+ continue+ require(offset < len(payload), 'missing DHCP option length')+ size = payload[offset]+ offset += 1+ require(offset + size <= len(payload), 'truncated DHCP option')+ require(kind not in options, 'duplicate DHCP options outside oracle scope')+ options[kind] = payload[offset:offset + size]+ offset += size+ raise ValueError('DHCP end option missing')+++def analyze(data, kind, row_limit=12):+ rows, matching, count = [], 0, 0+ for frame, packet in packets(data):+ count = frame+ decoded = transport(packet)+ if decoded is None:+ continue+ protocol, segment = decoded+ if kind == 'postgres' and protocol == 6 and frame <= 2000:+ require(len(segment) >= 20, 'truncated TCP header')+ tcp_header = (segment[12] >> 4) * 4+ require(20 <= tcp_header <= len(segment), 'invalid or truncated TCP header/options')+ source, destination = struct.unpack_from('!HH', segment)+ if 5432 not in (source, destination):+ continue+ matching += 1+ if len(rows) < row_limit:+ rows.append({'frame': frame, 'source_port': source,+ 'destination_port': destination,+ 'flags': struct.unpack_from('!H', segment, 12)[0] & 0xfff})+ elif kind in ('dns', 'dhcp') and protocol == 17:+ require(len(segment) >= 8, 'truncated UDP header')+ source, destination, length = struct.unpack_from('!HHH', segment)+ selected = 53 in (source, destination) if kind == 'dns' else (+ source, destination) in ((67, 68), (68, 67))+ if not selected:+ continue+ require(8 <= length <= len(segment), 'truncated selected UDP message')+ payload = segment[8:length]+ if kind == 'dns':+ require(len(payload) >= 12, 'truncated DNS header')+ identifier, flags, questions = struct.unpack_from('!HHH', payload)+ if not flags & 0x8000 or flags & 15 == 0:+ continue+ matching += 1+ offset, names, types = 12, [], []+ for _ in range(questions):+ name, offset = dns_name(payload, offset)+ require(offset + 4 <= len(payload), 'truncated DNS question')+ types.append(struct.unpack_from('!H', payload, offset)[0])+ names.append(name)+ offset += 4+ rows.append({'frame': frame, 'id': identifier, 'rcode': flags & 15,+ 'names': names, 'types': types})+ else:+ options = dhcp_options(payload)+ require(53 in options and len(options[53]) == 1,+ 'missing DHCP message type')+ server = options.get(54)+ require(server is None or len(server) == 4, 'invalid DHCP server ID')+ require(52 not in options, 'DHCP overload outside oracle scope')+ matching += 1+ row = {'frame': frame, 'id': struct.unpack_from('!I', payload, 4)[0],+ 'message_type': options[53][0],+ 'your_address': '.'.join(map(str, payload[16:20]))}+ if server is not None:+ row['server_identifier'] = '.'.join(map(str, server))+ rows.append(row)+ return {'kind': kind, 'capture_frames': count, 'matching_packets': matching,+ 'question_input_frames': min(2000, count) if kind == 'postgres' else count,+ 'rows': rows, 'scope': 'selected recorded headers; no network completeness claim'}+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument('kind', choices=['postgres', 'dns', 'dhcp'])+ parser.add_argument('path', type=Path)+ parser.add_argument('--row-limit', type=int, choices=[12, 24], default=12,+ help='PostgreSQL rows; 24 independently checks the second page')+ args = parser.parse_args()+ with args.path.open('rb') as source:+ data = source.read(32 * 1024 * 1024 + 1)+ require(len(data) <= 32 * 1024 * 1024, 'oracle input budget exceeded')+ print(json.dumps({'sha256': hashlib.sha256(data).hexdigest(),+ **analyze(data, args.kind, args.row_limit)}, indent=2))+++if __name__ == '__main__':+ main()diff --git a/experiments/analysis-efficiency/questions/packet-postgres.txt b/experiments/analysis-efficiency/questions/packet-postgres.txtnew file mode 100644index 000000000..82914ba3e--- /dev/null+++ b/experiments/analysis-efficiency/questions/packet-postgres.txt@@ -0,0 +1,3 @@+Inspect the selected saved PostgreSQL capture {{artifact_path}} with REA. Within original capture frames 1 through 2000, return the first 12 TCP packets with source or destination port 5432 in original frame order. For each return integer frame, source_port, destination_port and flags (the numeric TCP flags word, not prose). Also report the matching packet count within this 2000-frame question scope.++Do not claim whole-capture or network completeness. Return a single JSON object with exactly kind ("postgres"), matching_packets, rows, evidence_ids and limitations. Use actual successful REA Evidence IDs. Include a concise limitation explaining the recorded input scope; no extra claims or tool calls are needed.diff --git a/experiments/analysis-efficiency/review-packet.mjs b/experiments/analysis-efficiency/review-packet.mjsnew file mode 100644index 000000000..e7e3b0def--- /dev/null+++ b/experiments/analysis-efficiency/review-packet.mjs@@ -0,0 +1,247 @@+import { readFile, writeFile } from "node:fs/promises";+import { resolve } from "node:path";+import { pathToFileURL } from "node:url";+import { isDeepStrictEqual, parseArgs } from "node:util";+import { packetFacts, packetRequest } from "./packet-entry.mjs";++const { values } = parseArgs({+ options: {+ trial: { type: "string" },+ oracle: { type: "string" },+ "runtime-root": { type: "string" },+ selection: { type: "string" },+ },+});+if (+ !values.trial ||+ !values.oracle ||+ !values["runtime-root"] ||+ !["baseline", "focused", "discovery"].includes(values.selection)+)+ throw new Error(+ "Use --trial DIR --oracle JSON --runtime-root DIR --selection baseline|focused|discovery",+ );++const root = values["runtime-root"];+const { parseEvidence } = await import(+ pathToFileURL(resolve(root, "dist/domain/evidence.js")).href+);+const { inspectPacketCaptureInputSchema, packetCaptureSchema } = await import(+ pathToFileURL(resolve(root, "dist/domain/captures/packetCapture.js")).href+);+const { inspectPacketFieldsInputSchema, packetFieldsCatalogSchema } =+ await import(+ pathToFileURL(resolve(root, "dist/domain/captures/packetFields.js")).href+ );+const oracle = JSON.parse(await readFile(values.oracle, "utf8"));+const settings = JSON.parse(+ await readFile(resolve(values.trial, "trial-settings.json"), "utf8"),+);+const summary = JSON.parse(+ await readFile(resolve(values.trial, "summary.json"), "utf8"),+);+const expected = Object.fromEntries(+ ["kind", "matching_packets", "rows"].map((key) => [key, oracle[key]]),+);+const calls = (await readFile(resolve(values.trial, "events.jsonl"), "utf8"))+ .split("\n")+ .filter(Boolean)+ .map(JSON.parse)+ .filter(+ (event) =>+ event.type === "item.completed" && event.item?.type === "mcp_tool_call",+ )+ .map((event) => event.item);+const reasons = [];+const workflowReasons = [];+let answer;+const captures = new Map();+try {+ if (oracle.kind !== "postgres")+ throw new Error("This model rubric is the fixed PostgreSQL question only");+ const expectedCalls =+ values.selection === "discovery"+ ? ["inspect_packet_fields", "inspect_packet_capture"]+ : ["inspect_packet_capture"];+ if (+ !isDeepStrictEqual(+ calls.map((call) => call.tool),+ expectedCalls,+ ) ||+ calls.some(+ (call) =>+ call.server !== "rea" || call.status !== "completed" || call.error,+ )+ )+ workflowReasons.push(+ "Completed public REA call sequence differs from the fixed plan",+ );+ const ids = new Set();+ const conflictingIds = new Set();+ for (const call of calls) {+ if (call.server !== "rea" || call.status !== "completed" || call.error)+ continue;+ try {+ const result = call.result;+ if (result?.isError)+ throw new Error("Tool failure is not observed packet evidence");+ const text =+ result?.content?.filter((item) => item.type === "text") ?? [];+ if (!result?.structuredContent && text.length !== 1)+ throw new Error("No unambiguous captured Evidence envelope");+ const evidence = result.structuredContent ?? JSON.parse(text[0].text);+ parseEvidence(evidence);+ if (conflictingIds.has(evidence.evidence_id))+ throw new Error("Previously conflicting Evidence identity");+ if (call.tool === "inspect_packet_fields") {+ const input = inspectPacketFieldsInputSchema.parse({+ kind: "fields",+ protocol: "tcp",+ query: "tcp.flags",+ limit: 1,+ });+ const report = packetFieldsCatalogSchema.parse(+ evidence.normalized_result,+ );+ if (+ !isDeepStrictEqual(+ inspectPacketFieldsInputSchema.parse(call.arguments),+ input,+ ) ||+ !isDeepStrictEqual(evidence.parameters, input) ||+ !isDeepStrictEqual(report.selection, input) ||+ evidence.operation !== call.tool ||+ !report.records.some((row) => row.abbreviation === "tcp.flags") ||+ evidence.subject?.local_path !== report.tool.path ||+ evidence.subject?.digest?.sha256 !== report.tool.sha256+ )+ throw new Error(+ "Discovery Evidence does not establish the requested installed field",+ );+ } else if (call.tool === "inspect_packet_capture") {+ const planned = inspectPacketCaptureInputSchema.parse(+ packetRequest(+ "wireshark-pgsql-jdbc.pcap",+ settings.selected_artifact_path,+ values.selection,+ ),+ );+ const input = inspectPacketCaptureInputSchema.parse(call.arguments);+ if (!isDeepStrictEqual(input, planned))+ workflowReasons.push("Capture selection differs from the fixed plan");+ const report = packetCaptureSchema.parse(evidence.normalized_result);+ if (+ !isDeepStrictEqual(evidence.parameters, input) ||+ !isDeepStrictEqual(report.selection, input) ||+ evidence.operation !== call.tool ||+ evidence.subject?.local_path !== input.path ||+ evidence.subject?.digest?.sha256 !== oracle.sha256 ||+ report.artifact.path !== input.path ||+ report.artifact.sha256 !== oracle.sha256+ )+ throw new Error(+ "Capture Evidence differs from the fixed input/byte identity",+ );+ const observed = packetFacts("postgres", report);+ if (+ captures.has(evidence.evidence_id) &&+ !isDeepStrictEqual(captures.get(evidence.evidence_id), observed)+ ) {+ captures.delete(evidence.evidence_id);+ ids.delete(evidence.evidence_id);+ conflictingIds.add(evidence.evidence_id);+ throw new Error("Repeated Evidence identity has inconsistent facts");+ }+ captures.set(evidence.evidence_id, observed);+ } else {+ throw new Error("Call is not a packet observation tool");+ }+ ids.add(evidence.evidence_id);+ } catch (error) {+ workflowReasons.push(+ `Call ${call.tool} did not provide authenticated packet observations: ${String(error)}`,+ );+ }+ }+ answer = JSON.parse(+ (await readFile(resolve(values.trial, "answer.txt"), "utf8")).trim(),+ );+ if (+ !isDeepStrictEqual(+ Object.keys(answer).sort(),+ [+ "kind",+ "matching_packets",+ "rows",+ "evidence_ids",+ "limitations",+ ].sort(),+ )+ )+ workflowReasons.push("Final answer shape differs from the fixed plan");+ if (+ !isDeepStrictEqual(+ Object.fromEntries(+ Object.keys(expected).map((key) => [key, answer[key]]),+ ),+ expected,+ )+ )+ throw new Error("Final answer is not the exact requested recorded facts");+ if (+ !Array.isArray(answer.evidence_ids) ||+ answer.evidence_ids.some((id) => !ids.has(id)) ||+ new Set(answer.evidence_ids).size !== answer.evidence_ids.length+ )+ throw new Error("Final citations lack the authenticated packet Evidence");+ const supportingFacts = answer.evidence_ids+ .map((id) => captures.get(id))+ .find((observed) => isDeepStrictEqual(observed, expected));+ if (!supportingFacts)+ throw new Error(+ "Cited observations differ from independent raw header oracle",+ );+ const limitations =+ typeof answer.limitations === "string"+ ? [answer.limitations]+ : answer.limitations;+ if (+ !Array.isArray(limitations) ||+ limitations.length === 0 ||+ limitations.some((item) => typeof item !== "string" || !item.trim())+ )+ workflowReasons.push("Final answer omits investigation limitations");+ if (+ summary.exit_code !== 0 ||+ summary.timeout ||+ summary.selected_artifact_bytes_unchanged !== true ||+ settings.selected_artifact_sha256_before !== oracle.sha256 ||+ summary.selected_artifact_sha256_after !== oracle.sha256+ )+ workflowReasons.push(+ "Trial did not complete with the selected artifact unchanged",+ );+} catch (error) {+ reasons.push(String(error));+}+const review = {+ factual_correctness: reasons.length ? "failed" : "passed",+ scope:+ "Fixed first-2000-frame PostgreSQL question; authenticated tool observations and independent raw header oracle",+ workflow_correctness:+ reasons.length || workflowReasons.length ? "failed" : "passed",+ reasons,+ workflow_reasons: workflowReasons,+ completed_calls: calls.length,+ wall_seconds: summary.wall_seconds,+ usage: summary.usage,+ selected_artifact_sha256: oracle.sha256,+ limitation_prose:+ "Presence checked; wording requires transcript review, not heuristic factual certification",+};+await writeFile(+ resolve(values.trial, "packet-review.json"),+ JSON.stringify(review, null, 2) + "\n",+);+console.log(JSON.stringify(review));+if (reasons.length || workflowReasons.length) process.exitCode = 1;diff --git a/experiments/analysis-efficiency/test_packet_oracle.py b/experiments/analysis-efficiency/test_packet_oracle.pynew file mode 100644index 000000000..6727b3b7e--- /dev/null+++ b/experiments/analysis-efficiency/test_packet_oracle.py@@ -0,0 +1,67 @@+import importlib.util+import struct+import unittest+from pathlib import Path++spec = importlib.util.spec_from_file_location('packet_oracle', Path(__file__).with_name('packet-oracle.py'))+oracle = importlib.util.module_from_spec(spec)+spec.loader.exec_module(oracle)+++class PacketOracleTest(unittest.TestCase):+ def test_compression_pointer_preserves_question_cursor(self):+ message = b'\x03www\x07example\x03com\0' + b'\xc0\0' + b'\0\x01\0\x01'+ self.assertEqual(oracle.dns_name(message, 17), ('www.example.com', 19))++ def test_compression_cycle_and_missing_pointer_byte_fail(self):+ for message in (b'\xc0\0', b'\xc0', b'\xc0\x02\xc0\0'):+ with self.subTest(message=message), self.assertRaises(ValueError):+ oracle.dns_name(message, 0)++ def test_truncated_pcap_record_is_not_exhaustion(self):+ header = struct.pack('<IHHIIII', 0xa1b2c3d4, 2, 4, 0, 0, 65535, 1)+ with self.assertRaises(ValueError):+ list(oracle.packets(header + b'\0'))+ record = struct.pack('<IIII', 0, 0, 20, 20)+ with self.assertRaises(ValueError):+ list(oracle.packets(header + record + b'\0' * 19))++ def test_missing_dhcp_server_id_stays_omitted(self):+ payload = b'\0' * 236 + b'\x63\x82\x53\x63\x35\x01\x01\xff'+ self.assertEqual(oracle.dhcp_options(payload), {53: b'\x01'})++ def test_dhcp_duplicates_truncation_and_missing_end_fail(self):+ prefix = b'\0' * 236 + b'\x63\x82\x53\x63'+ for suffix in (b'\x35\x01\x01\x35\x01\x02\xff', b'\x36\x04\0', b'\x35\x01\x01'):+ with self.subTest(suffix=suffix), self.assertRaises(ValueError):+ oracle.dhcp_options(prefix + suffix)++ def test_fragmented_selected_traffic_cannot_pass_header_oracle(self):+ ip = bytearray(20)+ ip[0], ip[9] = 0x45, 17+ struct.pack_into('!H', ip, 2, 20)+ struct.pack_into('!H', ip, 6, 0x2000)+ with self.assertRaises(ValueError):+ oracle.transport(b'\0' * 12 + b'\x08\0' + ip)++ def test_declared_ip_length_and_tcp_options_cannot_be_truncated(self):+ ip = bytearray(40)+ ip[0], ip[9], ip[32] = 0x45, 6, 0xf0+ struct.pack_into('!H', ip, 2, 40)+ struct.pack_into('!HH', ip, 20, 12345, 5432)+ packet = b'\0' * 12 + b'\x08\0' + ip+ header = struct.pack('<IHHIIII', 0xa1b2c3d4, 2, 4, 0, 0, 65535, 1)+ record = struct.pack('<IIII', 0, 0, len(packet), len(packet))+ with self.assertRaisesRegex(ValueError, 'TCP header'):+ oracle.analyze(header + record + packet, 'postgres')+ struct.pack_into('!H', ip, 2, 60)+ with self.assertRaisesRegex(ValueError, 'truncated IPv4'):+ oracle.transport(b'\0' * 12 + b'\x08\0' + ip)++ def test_unknown_link_protocol_does_not_prove_tcp_absence(self):+ with self.assertRaisesRegex(ValueError, 'unsupported Ethernet'):+ oracle.transport(b'\0' * 12 + b'\x86\xdd' + b'\0' * 40)+++if __name__ == '__main__':+ unittest.main()diff --git a/tests/evaluation/analysisEfficiencyComparison.test.ts b/tests/evaluation/analysisEfficiencyComparison.test.tsindex 57a1ffb8c..924ba1612 100644--- a/tests/evaluation/analysisEfficiencyComparison.test.ts+++ b/tests/evaluation/analysisEfficiencyComparison.test.ts@@ -30,11 +30,17 @@ type Fixture = {delivery: Delivery;provider_search_path_sha256: string;calls: unknown[];+ packet_selection?: string;};answer: {capture: { coverage: string; capture_start: null };http_request_uris_first2: string[];delivery_observations: Observation[];+ facts?: {+ kind: string;+ matching_packets: number;+ rows: { frame: number }[];+ };};};const reportSchema = z.object({@@ -123,6 +129,39 @@ function compare(left: Fixture, right: Fixture) {}}+for (const artifact of [+ "wireshark-pgsql-jdbc.pcap",+ "wireshark-dns.cap",+ "wireshark-dhcp.pcap",+]) {+ test(`${artifact}: compare requested facts across packet plans and preserve observations`, () => {+ const left = fixture("complete", artifact);+ left.summary.packet_selection = "baseline";+ left.answer.facts = {+ kind: "packet",+ matching_packets: 1337,+ rows: [{ frame: 9 }],+ };+ const right = structuredClone(left);+ right.summary.packet_selection = "focused";+ right.answer.delivery_observations[0]!.evidence_id = "ev_focused";+ right.answer.delivery_observations[0]!.coverage =+ "up-to-input-packet-limit";+ const equivalent = compare(left, right);+ assert.equal(equivalent.status, 0);+ assert.equal(equivalent.report.facts_match, true);+ assert.deepEqual(+ equivalent.report.preserved_run_specific_values.map((row) => row.value),+ [left.answer.delivery_observations, right.answer.delivery_observations],+ );+ right.answer.facts!.matching_packets++;+ assert.equal(compare(left, right).report.facts_match, false);+ right.answer.facts!.matching_packets--;+ right.summary.packet_selection = "baseline";+ assert.equal(compare(left, right).report.facts_match, false);+ });+}+for (const artifact of ["wireshark-http.cap","wireshark-http_redirects.pcapng",diff --git a/tests/evaluation/packetWorkflowSemantics.test.ts b/tests/evaluation/packetWorkflowSemantics.test.tsnew file mode 100644index 000000000..aac1f5976--- /dev/null+++ b/tests/evaluation/packetWorkflowSemantics.test.ts@@ -0,0 +1,32 @@+import assert from "node:assert/strict";+import { spawnSync } from "node:child_process";+import { test } from "vitest";++test("packet page verification rejects skipped matches with unchanged counts and monotone frames", () => {+ const moduleUrl = new URL(+ "../../experiments/analysis-efficiency/packet-entry.mjs",+ import.meta.url,+ ).href;+ const source = `+ import assert from 'node:assert/strict';+ import {verifyPacketPage} from ${JSON.stringify(moduleUrl)};+ const rows = Array.from({length:36}, (_,i)=>({frame:i+1, source_port:1000,destination_port:5432,flags:16}));+ const sha256 = 'a'.repeat(64);+ const report = (selected)=>({artifact:{sha256},output_window:{matching_packets:36},packets:selected.map(row=>({+ 'frame.number':[String(row.frame)],'tcp.srcport':['1000'],'tcp.dstport':['5432'],'tcp.flags':['0x0010']+ }))});+ const oracle={sha256,matching_packets:36,rows:rows.slice(0,24)};+ const first=report(rows.slice(0,12));+ verifyPacketPage(first,report(rows.slice(12,24)),oracle);+ assert.throws(()=>verifyPacketPage(first,report(rows.slice(24,36)),oracle),/independently parsed/);+ assert.throws(()=>verifyPacketPage(first,report(rows.slice(12,24).reverse()),oracle),/independently parsed/);+ assert.throws(()=>verifyPacketPage(first,{...report(rows.slice(12,24)),artifact:{sha256:'b'.repeat(64)}},oracle),/independently parsed/);+ `;+ // Semantic checker only; synthetic rows do not establish provider behavior.+ const result = spawnSync(process.execPath, ["--input-type=module", "-"], {+ input: source,+ encoding: "utf8",+ });+ assert.ifError(result.error);+ assert.equal(result.status, 0, result.stderr);+});