Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

test(perf): evaluate larger and non-HTTP packet workflows (#3232)

Ysopwnableca7450beffff
diff --git a/docs/packet-captures.md b/docs/packet-captures.md
index 32cedb11b..8db097386 100644
--- a/docs/packet-captures.md
+++ b/docs/packet-captures.md
@@ -54,6 +54,22 @@ resource limit; use selected fields, a display filter or an input-packet limit
when dissection output is too large. Each call checks the recording's current
digest, so pages from different source digests must not be combined.
+For a question about a known input prefix, select `packet_limit` for that
+prefix, the fields needed for the answer, and a matching output window. For
+example, a question about the first twelve PostgreSQL TCP packets within
+original frames 1–2,000 can use fields `tcp.srcport`, `tcp.dstport`, `tcp.flags`,
+filter `tcp.port == 5432`, `packet_limit: 2000` and `match_limit: 12`.
+The matching count then describes that input prefix. A filter containing
+`frame.number <= 2000` without an input limit still reads the entire capture;
+returning twelve rows does not limit producer output. These controls answer a
+bounded question and must not be used to infer whole-capture absence.
+
+Discover unfamiliar field names with `inspect_packet_fields`, using an exact
+protocol and a narrow query. If the installed names are already established,
+direct field selection avoids an additional glossary probe. Count discovery
+and any retries when comparing investigation cost; a small page alone does
+not establish a faster workflow.
+
Additional selections:
- `display_filter`: Wireshark display-filter syntax; omitted means no filter.
diff --git a/experiments/analysis-efficiency/README.md b/experiments/analysis-efficiency/README.md
index cdb6c3c9b..9a14b764a 100644
--- a/experiments/analysis-efficiency/README.md
+++ b/experiments/analysis-efficiency/README.md
@@ -72,6 +72,12 @@ checkpoint commits retain their original receipts; no history was rewritten.
## Optimization priorities and gradual migration
+The [larger/non-HTTP packet checkpoint](packet-field-workflows.md) records
+same-question SDK and actual Codex comparisons using existing input limits,
+field/filter selection, discovery and output windows. Lower delivered JSON did
+not establish stable total-token or speed gains; failed plans and negative
+controls remain visible.
+
Agent token consumption and end-to-end task time are the primary objectives.
Memory and CPU consumption are also measured and controlled. Correctness,
Evidence provenance, caller choices and existing-user compatibility are
diff --git a/experiments/analysis-efficiency/agent-trial.py b/experiments/analysis-efficiency/agent-trial.py
index edb848a28..db994cea1 100644
--- a/experiments/analysis-efficiency/agent-trial.py
+++ b/experiments/analysis-efficiency/agent-trial.py
@@ -181,6 +181,7 @@ def main():
'inspect_android_class', 'inspect_android_method',
'inspect_firmware_regions', 'inspect_managed_artifact',
'inspect_sqlite_database', 'inspect_pcap_capture',
+ 'inspect_packet_capture', 'inspect_packet_fields',
'inspect_managed_members', 'inspect_managed_native_boundaries',
'inspect_artifact', 'inspect_wasm_artifact',
'inspect_evm_interface', 'get_evidence_bundle', 'export_evidence_bundle',
diff --git a/experiments/analysis-efficiency/compare.mjs b/experiments/analysis-efficiency/compare.mjs
index 4e33e5ead..01b9f97af 100644
--- a/experiments/analysis-efficiency/compare.mjs
+++ b/experiments/analysis-efficiency/compare.mjs
@@ -2,6 +2,7 @@ import { isDeepStrictEqual } from "node:util";
import { readFile, writeFile } from "node:fs/promises";
import { resolve } from "node:path";
import { parseArgs } from "node:util";
+import { PACKET_ARTIFACTS } from "./packet-entry.mjs";
const { values } = parseArgs({
options: {
@@ -33,6 +34,8 @@ const sameSettings = [
"provider_environment_policy_sha256",
"measure_source_sha256",
"native_workflow_source_sha256",
+ "packet_workflow_source_sha256",
+ "packet_oracle_source_sha256",
"package_lock_sha256",
"node_version",
"sdk_client_version",
@@ -76,6 +79,7 @@ for (const [side, answer] of [
// coverage. Repeated equivalent deliveries must also match every observation
// field except the receipt and parent IDs, which identify their own invocations.
if (
+ Object.hasOwn(PACKET_ARTIFACTS, left.artifact) ||
["wireshark-http.cap", "wireshark-http_redirects.pcapng"].includes(
left.artifact,
)
@@ -94,7 +98,11 @@ if (
path: ["delivery_observations"],
value: answer.delivery_observations,
});
- if (left.delivery === right.delivery) {
+ if (
+ left.delivery === right.delivery &&
+ (!Object.hasOwn(PACKET_ARTIFACTS, left.artifact) ||
+ left.packet_selection === right.packet_selection)
+ ) {
answer.delivery_observations = answer.delivery_observations.map(
(observation) =>
Object.fromEntries(
diff --git a/experiments/analysis-efficiency/corpus.json b/experiments/analysis-efficiency/corpus.json
index 7896c558d..e595dde34 100644
--- a/experiments/analysis-efficiency/corpus.json
+++ b/experiments/analysis-efficiency/corpus.json
@@ -291,6 +291,58 @@
"upstream_expected_digest": null,
"derivation": "Unmodified capture published by official Wireshark wiki. Local digest pins downloaded bytes; no independently published upstream digest is claimed."
}
+ },
+ {
+ "id": "wireshark-pgsql-jdbc.pcap",
+ "file": "pgsql-jdbc.pcap",
+ "bytes": 4906316,
+ "sha256": "6e68b8b792a0f4336092f98856abc7de8c54b55a29112854d28d676f9093c16b",
+ "provenance": {
+ "project": "Wireshark public sample captures",
+ "source_page": "https://wiki.wireshark.org/SampleCaptures",
+ "url": "https://wiki.wireshark.org/uploads/__moin_import__/attachments/SampleCaptures/pgsql-jdbc.pcap.gz",
+ "resolved_url": "https://gitlab.com/wireshark/wireshark/-/wikis/uploads/__moin_import__/attachments/SampleCaptures/pgsql-jdbc.pcap.gz",
+ "upstream_expected_digest": null,
+ "license": "not stated in inspected upstream catalogue",
+ "derivation": "Unmodified official saved sample; decompressed gzip where indicated. Local digest pins acquired bytes, not an upstream-published digest.",
+ "compressed_source": {
+ "name": "pgsql-jdbc.pcap.gz",
+ "bytes": 590093,
+ "sha256": "15b51bb4cc197753c1ac6b7dc3cf5697f22a81e3bb5d2e1d24d15de469c3a06b"
+ }
+ }
+ },
+ {
+ "id": "wireshark-dns.cap",
+ "file": "dns.cap",
+ "bytes": 4338,
+ "sha256": "041eeb6f98bb398f1ee8b09651b5b5a84f6a62639f95bf226f9e7b77355d9f28",
+ "provenance": {
+ "project": "Wireshark public sample captures",
+ "source_page": "https://wiki.wireshark.org/SampleCaptures",
+ "url": "https://wiki.wireshark.org/uploads/__moin_import__/attachments/SampleCaptures/dns.cap",
+ "resolved_url": "https://gitlab.com/wireshark/wireshark/-/wikis/uploads/__moin_import__/attachments/SampleCaptures/dns.cap",
+ "upstream_expected_digest": null,
+ "license": "not stated in inspected upstream catalogue",
+ "derivation": "Unmodified official saved sample; decompressed gzip where indicated. Local digest pins acquired bytes, not an upstream-published digest.",
+ "compressed_source": null
+ }
+ },
+ {
+ "id": "wireshark-dhcp.pcap",
+ "file": "dhcp.pcap",
+ "bytes": 1400,
+ "sha256": "2471b5420bdac826eecf8f61a2bbb4a3eb20dbfab7c02ff2be502f349f368214",
+ "provenance": {
+ "project": "Wireshark public sample captures",
+ "source_page": "https://wiki.wireshark.org/SampleCaptures",
+ "url": "https://wiki.wireshark.org/uploads/__moin_import__/attachments/SampleCaptures/dhcp.pcap",
+ "resolved_url": "https://gitlab.com/wireshark/wireshark/-/wikis/uploads/__moin_import__/attachments/SampleCaptures/dhcp.pcap",
+ "upstream_expected_digest": null,
+ "license": "not stated in inspected upstream catalogue",
+ "derivation": "Unmodified official saved sample; decompressed gzip where indicated. Local digest pins acquired bytes, not an upstream-published digest.",
+ "compressed_source": null
+ }
}
]
}
diff --git a/experiments/analysis-efficiency/measure.mjs b/experiments/analysis-efficiency/measure.mjs
index 60dce4f80..07a39e3c1 100644
--- a/experiments/analysis-efficiency/measure.mjs
+++ b/experiments/analysis-efficiency/measure.mjs
@@ -12,6 +12,7 @@ import {
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { Ajv2020 } from "ajv/dist/2020.js";
+import { PACKET_ARTIFACTS, measurePacketEntry } from "./packet-entry.mjs";
const root = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
const { values } = parseArgs({
@@ -29,6 +30,7 @@ const { values } = parseArgs({
"managed-question": { type: "string", default: "inventory" },
schemas: { type: "string", default: "default" },
"sqlite-selection": { type: "string", default: "all" },
+ "packet-selection": { type: "string", default: "baseline" },
},
});
if (
@@ -45,6 +47,15 @@ if (
!["inventory", "selected"].includes(values["javascript-question"]) ||
!["default", "selected"].includes(values.schemas) ||
!["all", "columns"].includes(values["sqlite-selection"]) ||
+ !["baseline", "focused", "discovery", "pages"].includes(
+ values["packet-selection"],
+ ) ||
+ (!Object.hasOwn(PACKET_ARTIFACTS, values.artifact) &&
+ values["packet-selection"] !== "baseline") ||
+ (Object.hasOwn(PACKET_ARTIFACTS, values.artifact) &&
+ values.delivery !== "complete") ||
+ (values["packet-selection"] === "pages" &&
+ values.artifact !== "wireshark-pgsql-jdbc.pcap") ||
(values.artifact !== "qgis-world-map.gpkg" &&
values["sqlite-selection"] !== "all") ||
(values.artifact === "qgis-world-map.gpkg" &&
@@ -142,6 +153,12 @@ const fingerprints = {
native_workflow_source_sha256: await fileSha(
fileURLToPath(new URL("./native-entry.mjs", import.meta.url)),
),
+ packet_workflow_source_sha256: await fileSha(
+ fileURLToPath(new URL("./packet-entry.mjs", import.meta.url)),
+ ),
+ packet_oracle_source_sha256: await fileSha(
+ fileURLToPath(new URL("./packet-oracle.py", import.meta.url)),
+ ),
runtime_files_sha256: sha(JSON.stringify(runtimeFiles)),
runtime_dispatcher_sha256: await fileSha(
resolve(values["runtime-root"], "scripts/rea.mjs"),
@@ -722,6 +739,12 @@ const inspectPcapTarget = async () => {
};
const workflows = {
+ ...Object.fromEntries(
+ Object.keys(PACKET_ARTIFACTS).map((artifact) => [
+ artifact,
+ () => measurePacketEntry({ values, call, inputSha, catalogDefinitions }),
+ ]),
+ ),
"wireshark-http.cap": inspectPcapTarget,
"wireshark-http_redirects.pcapng": inspectPcapTarget,
"qgis-world-map.gpkg": async () => {
@@ -1160,6 +1183,9 @@ const report = {
values.artifact === "qgis-world-map.gpkg"
? values["sqlite-selection"]
: null,
+ packet_selection: Object.hasOwn(PACKET_ARTIFACTS, values.artifact)
+ ? values["packet-selection"]
+ : null,
provider_settings_sha256: sha(JSON.stringify(selectedProviders)),
provider_search_path_sha256: sha(
selectedProviders.PATH ?? process.env.PATH ?? "",
diff --git a/experiments/analysis-efficiency/packet-entry.mjs b/experiments/analysis-efficiency/packet-entry.mjs
new file mode 100644
index 000000000..5c6722a74
--- /dev/null
+++ b/experiments/analysis-efficiency/packet-entry.mjs
@@ -0,0 +1,316 @@
+import { isDeepStrictEqual } from "node:util";
+import { resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import { fileURLToPath } from "node:url";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+
+export const verifyPacketPage = (first, next, oracle) => {
+ if (
+ first.artifact.sha256 !== oracle.sha256 ||
+ next.artifact.sha256 !== oracle.sha256 ||
+ first.output_window.matching_packets !== oracle.matching_packets ||
+ next.output_window.matching_packets !== oracle.matching_packets ||
+ !isDeepStrictEqual(
+ packetFacts("postgres", first).rows,
+ oracle.rows.slice(0, 12),
+ ) ||
+ !isDeepStrictEqual(
+ packetFacts("postgres", next).rows,
+ oracle.rows.slice(12, 24),
+ )
+ )
+ throw new Error(
+ "Packet pages differ from independently parsed original matches",
+ );
+};
+
+export const PACKET_ARTIFACTS = {
+ "wireshark-pgsql-jdbc.pcap": {
+ kind: "postgres",
+ fields: ["tcp.srcport", "tcp.dstport", "tcp.flags"],
+ extra: ["tcp.len"],
+ protocol: "tcp",
+ query: "tcp.flags",
+ discovered: "tcp.flags",
+ filter: "tcp.port == 5432",
+ },
+ "wireshark-dns.cap": {
+ kind: "dns",
+ fields: [
+ "dns.id",
+ "dns.flags.response",
+ "dns.flags.rcode",
+ "dns.qry.name",
+ "dns.qry.type",
+ ],
+ extra: ["dns.resp.name", "dns.resp.ttl", "udp.srcport", "udp.dstport"],
+ protocol: "dns",
+ query: "dns.flags.rcode",
+ discovered: "dns.flags.rcode",
+ filter: "dns.flags.response == 1 && dns.flags.rcode != 0",
+ },
+ "wireshark-dhcp.pcap": {
+ kind: "dhcp",
+ fields: [
+ "dhcp.id",
+ "dhcp.option.dhcp",
+ "dhcp.ip.your",
+ "dhcp.option.dhcp_server_id",
+ ],
+ extra: [
+ "dhcp.type",
+ "dhcp.ip.client",
+ "dhcp.hw.mac_addr",
+ "udp.srcport",
+ "udp.dstport",
+ ],
+ protocol: "dhcp",
+ query: "Option 53:",
+ discovered: "dhcp.option.dhcp",
+ filter: "dhcp",
+ },
+};
+
+export const packetRequest = (artifact, path, selection) => {
+ const descriptor = PACKET_ARTIFACTS[artifact];
+ if (!descriptor) throw new Error("Unknown pinned packet question");
+ const baseline = selection === "baseline";
+ return {
+ path,
+ fields: baseline
+ ? [...descriptor.fields, ...descriptor.extra]
+ : descriptor.fields,
+ display_filter:
+ descriptor.kind === "postgres" && baseline
+ ? `${descriptor.filter} && frame.number <= 2000`
+ : descriptor.kind === "dns" && baseline
+ ? "dns"
+ : descriptor.filter,
+ ...(descriptor.kind === "postgres" && !baseline
+ ? { packet_limit: 2000, match_limit: 12 }
+ : {}),
+ };
+};
+
+const number = (row, field) => {
+ const value = row[field];
+ if (
+ !Array.isArray(value) ||
+ value.length !== 1 ||
+ !/^(?:[0-9]+|0x[0-9a-f]+)$/iu.test(value[0])
+ )
+ throw new Error(`Expected one recorded integer for ${field}`);
+ const parsed = Number(value[0]);
+ if (!Number.isSafeInteger(parsed))
+ throw new Error(`Unsafe recorded integer for ${field}`);
+ return parsed;
+};
+
+export const packetFacts = (kind, report) => {
+ let packets = report.packets;
+ if (kind === "postgres") packets = packets.slice(0, 12);
+ if (kind === "dns")
+ packets = packets.filter(
+ (row) =>
+ number(row, "dns.flags.response") === 1 &&
+ number(row, "dns.flags.rcode") !== 0,
+ );
+ return {
+ kind,
+ matching_packets:
+ kind === "dns" ? packets.length : report.output_window.matching_packets,
+ rows: packets.map((row) => {
+ const frame = number(row, "frame.number");
+ if (kind === "postgres")
+ return {
+ frame,
+ source_port: number(row, "tcp.srcport"),
+ destination_port: number(row, "tcp.dstport"),
+ flags: number(row, "tcp.flags"),
+ };
+ if (kind === "dns") {
+ const names = row["dns.qry.name"];
+ const types = row["dns.qry.type"];
+ if (
+ !Array.isArray(names) ||
+ names.length === 0 ||
+ !Array.isArray(types) ||
+ types.length !== names.length
+ )
+ throw new Error(
+ "The fixed DNS question requires recorded names and types; missing fields are unknown",
+ );
+ return {
+ frame,
+ id: number(row, "dns.id"),
+ rcode: number(row, "dns.flags.rcode"),
+ names,
+ types: types.map((value) => number({ type: [value] }, "type")),
+ };
+ }
+ const server = row["dhcp.option.dhcp_server_id"];
+ const address = row["dhcp.ip.your"];
+ if (
+ !Array.isArray(address) ||
+ address.length !== 1 ||
+ (server !== undefined &&
+ (!Array.isArray(server) || server.length !== 1))
+ )
+ throw new Error("Expected recorded DHCP address scalar");
+ return {
+ frame,
+ id: number(row, "dhcp.id"),
+ message_type: number(row, "dhcp.option.dhcp"),
+ your_address: address[0],
+ ...(server === undefined ? {} : { server_identifier: server[0] }),
+ };
+ }),
+ };
+};
+
+export const measurePacketEntry = async ({
+ values,
+ call,
+ inputSha,
+ catalogDefinitions,
+}) => {
+ const descriptor = PACKET_ARTIFACTS[values.artifact];
+ const { parseEvidence } = await import(
+ pathToFileURL(resolve(values["runtime-root"], "dist/domain/evidence.js"))
+ .href
+ );
+ const { inspectPacketCaptureInputSchema, packetCaptureSchema } = await import(
+ pathToFileURL(
+ resolve(values["runtime-root"], "dist/domain/captures/packetCapture.js"),
+ ).href
+ );
+ const { inspectPacketFieldsInputSchema, packetFieldsCatalogSchema } =
+ await import(
+ pathToFileURL(
+ resolve(values["runtime-root"], "dist/domain/captures/packetFields.js"),
+ ).href
+ );
+ const observations = [];
+ const selection = values["packet-selection"];
+ const checked = (evidence, operation, input, schema) => {
+ parseEvidence(evidence);
+ const report = schema.parse(evidence.normalized_result);
+ if (
+ evidence.operation !== operation ||
+ !isDeepStrictEqual(evidence.parameters, input) ||
+ !isDeepStrictEqual(report.selection, input)
+ )
+ throw new Error(
+ "Packet Evidence operation or choices differs from the request",
+ );
+ if (
+ operation === "inspect_packet_capture" &&
+ (evidence.subject?.local_path !== values.path ||
+ evidence.subject?.digest?.sha256 !== inputSha ||
+ report.artifact.path !== values.path ||
+ report.artifact.sha256 !== inputSha)
+ )
+ throw new Error("Packet Evidence identifies different capture bytes");
+ if (
+ operation === "inspect_packet_fields" &&
+ (evidence.subject?.local_path !== report.tool.path ||
+ evidence.subject?.digest?.sha256 !== report.tool.sha256)
+ )
+ throw new Error("Field discovery identifies a different executable");
+ observations.push({
+ evidence_id: evidence.evidence_id,
+ operation,
+ selection: report.selection,
+ output_window: report.output_window,
+ read_scope: report.read_scope ?? null,
+ limitations: evidence.limitations,
+ });
+ return report;
+ };
+ if (selection === "discovery") {
+ const input = inspectPacketFieldsInputSchema.parse({
+ kind: "fields",
+ protocol: descriptor.protocol,
+ query: descriptor.query,
+ limit: 1,
+ });
+ const found = checked(
+ await call("inspect_packet_fields", input),
+ "inspect_packet_fields",
+ input,
+ packetFieldsCatalogSchema,
+ );
+ if (
+ !found.records.some((row) => row.abbreviation === descriptor.discovered)
+ )
+ throw new Error(
+ "Selected installation does not advertise the required protocol field",
+ );
+ }
+ const input = inspectPacketCaptureInputSchema.parse(
+ packetRequest(values.artifact, values.path, selection),
+ );
+ for (const name of [
+ "inspect_packet_capture",
+ ...(selection === "discovery" ? ["inspect_packet_fields"] : []),
+ ])
+ if (!catalogDefinitions.has(name))
+ throw new Error(`Missing live packet contract ${name}`);
+ const report = checked(
+ await call("inspect_packet_capture", input),
+ "inspect_packet_capture",
+ input,
+ packetCaptureSchema,
+ );
+ const facts = packetFacts(descriptor.kind, report);
+ if (selection === "pages") {
+ const { stdout } = await promisify(execFile)(
+ "python3",
+ [
+ fileURLToPath(new URL("./packet-oracle.py", import.meta.url)),
+ "postgres",
+ values.path,
+ "--row-limit",
+ "24",
+ ],
+ { timeout: 30_000, maxBuffer: 128 * 1024 },
+ );
+ const oracle = JSON.parse(stdout);
+ const nextInput = inspectPacketCaptureInputSchema.parse({
+ ...input,
+ match_offset: 12,
+ });
+ const next = checked(
+ await call("inspect_packet_capture", nextInput),
+ "inspect_packet_capture",
+ nextInput,
+ packetCaptureSchema,
+ );
+ const repeated = checked(
+ await call("inspect_packet_capture", input),
+ "inspect_packet_capture",
+ input,
+ packetCaptureSchema,
+ );
+ verifyPacketPage(report, next, oracle);
+ if (
+ !isDeepStrictEqual(report.packets, repeated.packets) ||
+ !isDeepStrictEqual(report.output_window, repeated.output_window) ||
+ next.artifact.sha256 !== report.artifact.sha256 ||
+ next.output_window.matching_packets !==
+ report.output_window.matching_packets ||
+ Number(next.packets[0]?.["frame.number"]?.[0]) <=
+ Number(report.packets.at(-1)?.["frame.number"]?.[0])
+ )
+ throw new Error(
+ "Packet page order, digest or repeated observations changed",
+ );
+ }
+ return {
+ artifact_sha256: inputSha,
+ facts,
+ delivery_observations: observations,
+ evidence_ids: [...new Set(observations.map((row) => row.evidence_id))],
+ };
+};
diff --git a/experiments/analysis-efficiency/packet-field-workflows.md b/experiments/analysis-efficiency/packet-field-workflows.md
new file mode 100644
index 000000000..8f893c30c
--- /dev/null
+++ b/experiments/analysis-efficiency/packet-field-workflows.md
@@ -0,0 +1,226 @@
+# Larger and non-HTTP packet investigation costs
+
+This closes the bounded experiment in [#3130](https://github.com/morluto/rea/issues/3130).
+Existing field, filter and input/output-window controls substantially reduce
+delivered JSON for the selected questions. Actual Codex total input and elapsed
+time are less consistent. Requiring field discovery for a known field adds cost.
+These results support documenting caller-selected controls, while retaining the
+current API, defaults and uncached provider behavior.
+
+## Runtime and public corpus
+
+Measurements use immutable main
+`ec67aebed02dbd20fa5c7703fcd183ad0b1c343c`, Node 24.18.0, the locked MCP client and
+server, Linux, TShark/capinfos 4.0.17 and util-linux prlimit. The SDK captures live
+advertised contracts and passes selected definitions to actual public MCP calls.
+Runtime-file, dispatcher, lockfile, experiment-source and explicit provider
+configuration digests remain with the private trial records. Subsequent main
+integration checks do not relabel these measurements.
+
+Three additional saved recordings come from the
+[official Wireshark catalogue](https://wiki.wireshark.org/SampleCaptures).
+`corpus.json` pins their acquisition URLs, exact bytes and SHA-256 values.
+The PostgreSQL recording is described upstream as a JDBC test session; it is
+not a representative production workload. Capture-specific licenses were not
+stated in the inspected catalogue. No captures or raw trial records are
+redistributed in Git.
+
+| Recording | Acquired/analyzed size | Recorded frames | Fixed question |
+| -------------------- | --------------------------------------------: | --------------: | ------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `pgsql-jdbc.pcap.gz` | 590,093 compressed / 4,906,316 expanded bytes | 18,472 | Within original frames 1–2,000, report the first twelve TCP packets using port 5432, their source/destination ports and flags, and the matching count. |
+| `dns.cap` | 4,338 bytes | 38 | Report nonzero-RCODE DNS responses, transaction IDs and question names/types. |
+| `dhcp.pcap` | 1,400 bytes | 4 | Report message types, transaction IDs, offered address and present server identifiers in frame order. |
+
+The larger application recording and two small non-HTTP controls expand the
+question/protocol coverage. Large DNS/DHCP, PCAPNG non-HTTP, USB, Bluetooth,
+Windows and macOS provider performance remain unverified.
+
+`packet-oracle.py` independently reads classic PCAP, Ethernet/IPv4 and selected
+TCP/UDP headers. It does not consume TShark output. Its narrow supported scope
+fails on truncation, IP fragmentation, DNS compression cycles, duplicate DHCP
+options or DHCP overload. TCP header facts follow
+[RFC 9293](https://www.rfc-editor.org/rfc/rfc9293.html), DNS names/header fields
+[RFC 1035](https://www.rfc-editor.org/rfc/rfc1035.html), and DHCP options
+[RFC 2132](https://www.rfc-editor.org/rfc/rfc2132.html).
+SQL interpretation, TCP reassembly and network completeness are outside the
+oracle. The question and oracle were fixed before measured cohorts.
+
+The oracle establishes 1,337 selected PostgreSQL TCP packets in the input prefix
+and six DNS error responses. DHCP records contain Discover/Offer with transaction
+ID 15645 and Request/ACK with ID 15646. An absent server identifier stays omitted;
+the recording does not establish a single transaction containing all four steps.
+
+## Equivalent questions and SDK measurements
+
+Each arm uses fresh actual MCP sessions; heavy trials run sequentially.
+Two interleaved repetitions cover each arm and each recording. Discovery, catalog,
+RPC, schema checking, encoding and cleanup costs are retained. JSON bytes describe
+serialized SDK replies, not raw transport bytes or model tokens.
+
+The PostgreSQL baseline selects `tcp.srcport`, `tcp.dstport`, `tcp.flags` and
+`tcp.len`, retains the default fifty-row output window and reads to EOF with
+`tcp.port == 5432 && frame.number <= 2000`. The focused arm selects the three
+required fields, uses `packet_limit: 2000`, filter `tcp.port == 5432` and
+`match_limit: 12`. Both establish the same requested prefix facts and count.
+Their source read scopes differ explicitly: to EOF versus an input limit.
+Neither scope establishes whole-network completeness.
+
+DNS focuses the nine-field baseline on five requested fields and the error-response
+filter. DHCP selects four required fields from the nine-field baseline; all four
+records are still read. The discovery arm adds one narrow installed-field probe
+before the focused request. It tests the cost of validating an already specified
+field, rather than proving a benefit for a previously unknown field or guessing
+that an installed registration appears in traffic.
+
+| Question | Baseline reply bytes | Focused reply bytes | Focused change | Discovery + focused bytes | Discovery change |
+| ----------------- | -------------------: | ------------------: | -------------: | ------------------------: | ---------------: |
+| PostgreSQL prefix | 23,054 | 13,304 | −42.29% | 21,096 | −8.49% |
+| DNS errors | 29,002 | 13,424 | −53.71% | 21,226 | −26.81% |
+| DHCP sequence | 13,770 | 12,020 | −12.71% | 19,840 | +44.08% |
+
+Both repetitions produce the same reply size in each cell and pass the independent
+fact comparison. Authentication uses the selected runtime's `parseEvidence` and
+named input/output schemas, checking operation, normalized caller choices,
+artifact or executable identity, original frame order and omission semantics.
+
+| Question | Baseline tool-call time, ms | Focused tool-call time, ms | Discovery + focused tool-call time, ms |
+| ----------------- | --------------------------- | -------------------------- | -------------------------------------- |
+| PostgreSQL prefix | 1,670 / 2,255 | 1,708 / 933 | 3,549 / 2,674 |
+| DNS errors | 892 / 786 | 889 / 778 | 2,506 / 2,608 |
+| DHCP sequence | 763 / 759 | 755 / 731 | 2,502 / 2,502 |
+
+Whole SDK workflow times, including connection/catalog, vary from 4.23–8.79 seconds
+for these arms. The PostgreSQL focused tool time improves in one repetition and
+regresses slightly in the other. Discovery consistently adds tool work. Two
+repetitions are descriptive observations, not a statistical speed guarantee.
+
+A separate first-page, next-page and repeated-first-page sequence costs
+922 / 804 / 1,244 ms and returns 39,918 reply bytes in total. Each call repeats
+dissection, reports the same 1,337 matching count and capture digest, and preserves
+the original frame order. Identical repeated observations retain the same
+content-addressed Evidence ID; distinct pages have distinct IDs. Page delivery
+does not establish capture or network completeness. This narrow repeat does not
+justify introducing a generic cache or weakening provider identity/cleanup rules.
+
+The two original HTTP controls were also replayed with two SDK pairs each on the
+new pinned runtime. Useful summary/prefix facts remain equivalent. For `http.cap`,
+complete delivery returns 7,601 bytes and summary plus a two-URI retained page
+17,540 bytes (+130.76%); for `http_redirects.pcapng`, 7,535 versus 17,240 bytes
+(+128.80%). These remain negative controls. The original model observations in
+[the earlier checkpoint](pcap-small-captures.md) keep their original runtime pin.
+
+## Actual Codex CLI observations
+
+Six actual Codex CLI 0.161.0 runs use requested alias `gpt-5.6-terra`, reasoning
+`high`, a disposable configuration and the pinned REA MCP runtime. Resolved
+server-side model identity is not inferred. Both tool names are enabled in every
+arm, exact-name discovery guidance is identical, shell/browser/other-agent tools
+are disabled and copied authentication is removed after each run.
+The shared question and generated plans contain no oracle answers.
+
+Order is baseline 1, focused 1, discovery 1, focused 2, baseline 2, discovery 2.
+The rubric checks authenticated actual tool results, exact header-derived facts,
+citations and the planned calls. Answer correctness and adherence to the plan
+are reported separately. Limitation wording is reviewed in transcripts; a string
+or string array is accepted because the prompt does not require one representation.
+
+| Trial | Total input tokens | Cached input tokens | Output tokens | Elapsed seconds | Actual MCP calls | Facts / plan |
+| ----------- | -----------------: | ------------------: | ------------: | --------------: | ---------------: | ---------------------------------- |
+| Baseline 1 | 40,649 | 27,904 | 1,574 | 27.989 | 1 | pass / pass |
+| Focused 1 | 36,642 | 19,968 | 943 | 38.542 | 1 | pass / pass |
+| Discovery 1 | 52,802 | 33,024 | 1,089 | 44.398 | 2 | pass / pass |
+| Focused 2 | 40,912 | 22,016 | 726 | 32.520 | 1 | pass / pass |
+| Baseline 2 | 40,181 | 19,968 | 896 | 40.299 | 1 | pass / pass |
+| Discovery 2 | 48,754 | 30,976 | 1,780 | 76.081 | 1 | pass / **fail: discovery omitted** |
+
+The two eligible baseline/focused pairs show total-input changes of −9.86% and
++1.82%, and elapsed changes of +37.70% and −19.30%. Thus reduced SDK reply size
+does not establish a stable total-token or speed improvement in these runs.
+The first complete discovery comparison adds 29.90% input and 58.63% elapsed time.
+The second discovery attempt is retained as a workflow failure and cannot serve
+as a successful two-call discovery comparison. Its cost is not discarded or
+counted as a passing trial. No model-side recovery or retries occurred in these
+six attempts.
+
+An earlier configuration preflight rejected the packet tool under Codex's
+`approval_policy: never`: 36,651 input tokens, 549 output tokens, 21.603 seconds,
+one rejected call and no packet observations. The experiment runner now explicitly
+preapproves the two authorized passive packet tools, consistently across all arms.
+That failed preflight remains separate from the corrected cohort and its cost is
+reported here; CLI exit zero alone is never a factual or workflow pass.
+
+## Resource and compatibility scope
+
+All engine/model workloads use an owned Linux resource scope with one CPU,
+3 GiB early-stop threshold, 4 GiB hard memory ceiling, no swap and a 192-task cap.
+Node's heap is additionally limited to 512 MiB. The nineteen non-HTTP SDK trials
+have a maximum observed payload-exit memory peak of 860,909,568 bytes; the six model
+attempts peak at 940,806,144 bytes. Payload-exit counters and cleanup were verified
+with the observer still alive and report no memory-limit/OOM events in that phase.
+Final whole-scope counters remain unavailable, so no stronger whole-scope claim
+is made. CPU usage and observed process ownership remain in each private receipt.
+
+A full PostgreSQL probe with four selected fields and no input prefix exceeds the
+existing 4 MiB producer-output ceiling even with the default fifty-row window.
+It correctly fails instead of delivering partial success. The SDK error retains
+large truncated diagnostics (5,790,979 serialized bytes); this separate product
+concern is recorded privately for later repair, outside #3130. Paging alone cannot
+prevent producer expansion. The successful prefix experiment does not prove that
+the full recording can be inspected with those selections.
+
+Only experiment sources, corpus provenance, a question template and guidance
+change. Production schemas, provider limits, CLI/MCP defaults, Evidence identity
+and session/resource ownership stay as before. Eight parser regressions retain
+malformed/truncated-input and omission checks. Existing clients need no migration.
+
+## Reproduction
+
+Acquire the exact corpus outside the repository and verify the pinned hashes.
+Use the pinned Node/npm toolchain, locked dependencies, `npm run build:cached` and
+an external resource guard before invoking actual engines/models. Supply explicit
+provider settings in a private JSON file. An example SDK arm is:
+
+```sh
+node experiments/analysis-efficiency/measure.mjs \
+ --runtime-root /absolute/pinned/runtime --revision PINNED_SHA \
+ --artifact wireshark-pgsql-jdbc.pcap --path /private/pgsql-jdbc.pcap \
+ --providers /private/providers.json --schemas selected \
+ --packet-selection focused --output /private/new-trial
+```
+
+Use `baseline`, `focused`, `discovery`, or the PostgreSQL-only `pages` diagnostic.
+Run `packet-oracle.py postgres PATH` independently and compare `answer.json` facts.
+Generate each model prompt with `packet-model-prompt.mjs --selection ARM`, then use
+the existing `agent-trial.py` with those two enabled tools and context capture.
+Run `review-packet.mjs` against the actual trial, oracle, runtime and selected arm.
+Its nonzero exit includes a plan failure even when all requested facts are correct.
+Retain failed attempts, actual usage, digests, coverage, cleanup and resource
+receipts alongside passing observations. No raw results need to be committed.
+
+## Review verification
+
+The comparator now recognizes the three non-HTTP questions: observations remain
+attached to each run, while different plans compare requested facts. Repeated
+identical plans still compare stable observation scope. Capture grading authenticates
+Evidence against the actual call and assesses plan differences separately; an
+actual 24-row SDK window replay retains a factual pass and a plan failure. This
+is a grading replay, not an additional model trial. The stricter header oracle
+preserves all original sample facts and rejects truncated IPv4/TCP options and
+unsupported Ethernet protocols. The page diagnostic now derives matches 13–24
+independently before accepting the second page; skipped, reversed and changed-byte
+pages are rejected. Original cohort measurements retain their recorded source
+digests and runtime pin; review checks do not relabel them.
+
+Capture grading keeps authenticated facts by Evidence ID and evaluates the cited
+observations. Four replays using actual first-page and second-page SDK Evidence
+verify that later or reordered pages and repeated reads preserve a correct cited
+answer, while citing only the wrong page fails factual grading. Extra calls still
+fail the fixed workflow plan. Regrading all six original model records preserves
+their outcomes and recorded costs; these replays are not new model trials.
+
+Failed or malformed calls remain workflow failures and never become authenticated
+Evidence. Nine failure-injection and answer-shape replays using an actual successful
+capture verify that later recovery preserves supported facts, while failed-only
+attempts and missing or unobserved citations fail factual grading. Missing
+limitations and extra answer keys fail the fixed plan without changing correct
+recorded facts. Successful original-model outcomes remain unchanged.
diff --git a/experiments/analysis-efficiency/packet-model-prompt.mjs b/experiments/analysis-efficiency/packet-model-prompt.mjs
new file mode 100644
index 000000000..f49348fbd
--- /dev/null
+++ b/experiments/analysis-efficiency/packet-model-prompt.mjs
@@ -0,0 +1,40 @@
+import { readFile } from "node:fs/promises";
+import { parseArgs } from "node:util";
+import { packetRequest } from "./packet-entry.mjs";
+
+const { values } = parseArgs({ options: { selection: { type: "string" } } });
+if (!["baseline", "focused", "discovery"].includes(values.selection))
+ throw new Error("Select baseline, focused or discovery");
+const question = await readFile(
+ new URL("./questions/packet-postgres.txt", import.meta.url),
+ "utf8",
+);
+const plan = [
+ ...(values.selection === "discovery"
+ ? [
+ {
+ tool: "inspect_packet_fields",
+ arguments: {
+ kind: "fields",
+ protocol: "tcp",
+ query: "tcp.flags",
+ limit: 1,
+ },
+ },
+ ]
+ : []),
+ {
+ tool: "inspect_packet_capture",
+ arguments: packetRequest(
+ "wireshark-pgsql-jdbc.pcap",
+ "{{artifact_path}}",
+ values.selection,
+ ),
+ },
+];
+process.stdout.write(
+ question +
+ "\nExecute this fixed public REA MCP plan in order. These are input selections, not expected answers. Preserve omitted defaults. All calls count toward the comparison.\n" +
+ JSON.stringify(plan, null, 2) +
+ "\n",
+);
diff --git a/experiments/analysis-efficiency/packet-oracle.py b/experiments/analysis-efficiency/packet-oracle.py
new file mode 100644
index 000000000..db5797b5f
--- /dev/null
+++ b/experiments/analysis-efficiency/packet-oracle.py
@@ -0,0 +1,180 @@
+"""Independent, narrow oracle for the three pinned Ethernet/IPv4 PCAPs.
+
+Reads header bytes, not TShark output. No PCAPNG, IP fragmentation or TCP
+reassembly support is claimed. DNS compression follows RFC 1035; DHCP options
+53/54 follow RFC 2132. Unsupported selected traffic fails rather than guessing.
+"""
+import argparse
+import hashlib
+import json
+import struct
+from pathlib import Path
+
+
+def require(condition, reason):
+ if not condition:
+ raise ValueError(reason)
+
+
+def packets(data):
+ require(len(data) >= 24, 'truncated PCAP header')
+ magic = data[:4]
+ require(magic in (b'\xd4\xc3\xb2\xa1', b'\xa1\xb2\xc3\xd4'),
+ 'oracle requires microsecond classic PCAP')
+ endian = '<' if magic == b'\xd4\xc3\xb2\xa1' else '>'
+ require(struct.unpack_from(endian + 'HH', data, 4) == (2, 4),
+ 'unsupported PCAP version')
+ require(struct.unpack_from(endian + 'I', data, 20)[0] == 1,
+ 'oracle requires Ethernet')
+ offset, frame = 24, 0
+ while offset < len(data):
+ require(offset + 16 <= len(data), 'truncated PCAP record header')
+ captured, original = struct.unpack_from(endian + 'II', data, offset + 8)
+ offset += 16
+ require(captured <= original and offset + captured <= len(data),
+ 'invalid captured packet length')
+ frame += 1
+ yield frame, data[offset:offset + captured]
+ offset += captured
+
+
+def transport(packet):
+ require(len(packet) >= 14, 'truncated Ethernet header')
+ if packet[12:14] == b'\x08\x06':
+ return None # ARP cannot contain the selected TCP/UDP messages.
+ require(packet[12:14] == b'\x08\x00', 'unsupported Ethernet protocol')
+ ip = packet[14:]
+ require(len(ip) >= 20 and ip[0] >> 4 == 4, 'invalid IPv4 header')
+ header = (ip[0] & 15) * 4
+ total = struct.unpack_from('!H', ip, 2)[0]
+ require(20 <= header <= len(ip) and total >= header, 'invalid IPv4 lengths')
+ require(total <= len(ip), 'truncated IPv4 packet')
+ require(struct.unpack_from('!H', ip, 6)[0] & 0x3fff == 0,
+ 'fragmentation is outside oracle scope')
+ return ip[9], ip[header:min(total, len(ip))]
+
+
+def dns_name(message, offset):
+ labels, visited, end = [], set(), None
+ while True:
+ require(offset < len(message) and offset not in visited,
+ 'invalid DNS name or compression cycle')
+ visited.add(offset)
+ size = message[offset]
+ if size & 0xc0 == 0xc0:
+ require(offset + 1 < len(message), 'truncated DNS compression pointer')
+ if end is None:
+ end = offset + 2
+ offset = ((size & 0x3f) << 8) | message[offset + 1]
+ continue
+ require(size & 0xc0 == 0 and offset + 1 + size <= len(message),
+ 'invalid DNS label')
+ offset += 1
+ if size == 0:
+ return '.'.join(labels), end if end is not None else offset
+ labels.append(message[offset:offset + size].decode('ascii'))
+ require(sum(len(label) + 1 for label in labels) <= 255,
+ 'DNS name exceeds protocol length')
+ offset += size
+
+
+def dhcp_options(payload):
+ require(len(payload) >= 240 and payload[236:240] == b'\x63\x82\x53\x63',
+ 'invalid DHCP fixed header or cookie')
+ options, offset = {}, 240
+ while offset < len(payload):
+ kind = payload[offset]
+ offset += 1
+ if kind == 255:
+ return options
+ if kind == 0:
+ continue
+ require(offset < len(payload), 'missing DHCP option length')
+ size = payload[offset]
+ offset += 1
+ require(offset + size <= len(payload), 'truncated DHCP option')
+ require(kind not in options, 'duplicate DHCP options outside oracle scope')
+ options[kind] = payload[offset:offset + size]
+ offset += size
+ raise ValueError('DHCP end option missing')
+
+
+def analyze(data, kind, row_limit=12):
+ rows, matching, count = [], 0, 0
+ for frame, packet in packets(data):
+ count = frame
+ decoded = transport(packet)
+ if decoded is None:
+ continue
+ protocol, segment = decoded
+ if kind == 'postgres' and protocol == 6 and frame <= 2000:
+ require(len(segment) >= 20, 'truncated TCP header')
+ tcp_header = (segment[12] >> 4) * 4
+ require(20 <= tcp_header <= len(segment), 'invalid or truncated TCP header/options')
+ source, destination = struct.unpack_from('!HH', segment)
+ if 5432 not in (source, destination):
+ continue
+ matching += 1
+ if len(rows) < row_limit:
+ rows.append({'frame': frame, 'source_port': source,
+ 'destination_port': destination,
+ 'flags': struct.unpack_from('!H', segment, 12)[0] & 0xfff})
+ elif kind in ('dns', 'dhcp') and protocol == 17:
+ require(len(segment) >= 8, 'truncated UDP header')
+ source, destination, length = struct.unpack_from('!HHH', segment)
+ selected = 53 in (source, destination) if kind == 'dns' else (
+ source, destination) in ((67, 68), (68, 67))
+ if not selected:
+ continue
+ require(8 <= length <= len(segment), 'truncated selected UDP message')
+ payload = segment[8:length]
+ if kind == 'dns':
+ require(len(payload) >= 12, 'truncated DNS header')
+ identifier, flags, questions = struct.unpack_from('!HHH', payload)
+ if not flags & 0x8000 or flags & 15 == 0:
+ continue
+ matching += 1
+ offset, names, types = 12, [], []
+ for _ in range(questions):
+ name, offset = dns_name(payload, offset)
+ require(offset + 4 <= len(payload), 'truncated DNS question')
+ types.append(struct.unpack_from('!H', payload, offset)[0])
+ names.append(name)
+ offset += 4
+ rows.append({'frame': frame, 'id': identifier, 'rcode': flags & 15,
+ 'names': names, 'types': types})
+ else:
+ options = dhcp_options(payload)
+ require(53 in options and len(options[53]) == 1,
+ 'missing DHCP message type')
+ server = options.get(54)
+ require(server is None or len(server) == 4, 'invalid DHCP server ID')
+ require(52 not in options, 'DHCP overload outside oracle scope')
+ matching += 1
+ row = {'frame': frame, 'id': struct.unpack_from('!I', payload, 4)[0],
+ 'message_type': options[53][0],
+ 'your_address': '.'.join(map(str, payload[16:20]))}
+ if server is not None:
+ row['server_identifier'] = '.'.join(map(str, server))
+ rows.append(row)
+ return {'kind': kind, 'capture_frames': count, 'matching_packets': matching,
+ 'question_input_frames': min(2000, count) if kind == 'postgres' else count,
+ 'rows': rows, 'scope': 'selected recorded headers; no network completeness claim'}
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('kind', choices=['postgres', 'dns', 'dhcp'])
+ parser.add_argument('path', type=Path)
+ parser.add_argument('--row-limit', type=int, choices=[12, 24], default=12,
+ help='PostgreSQL rows; 24 independently checks the second page')
+ args = parser.parse_args()
+ with args.path.open('rb') as source:
+ data = source.read(32 * 1024 * 1024 + 1)
+ require(len(data) <= 32 * 1024 * 1024, 'oracle input budget exceeded')
+ print(json.dumps({'sha256': hashlib.sha256(data).hexdigest(),
+ **analyze(data, args.kind, args.row_limit)}, indent=2))
+
+
+if __name__ == '__main__':
+ main()
diff --git a/experiments/analysis-efficiency/questions/packet-postgres.txt b/experiments/analysis-efficiency/questions/packet-postgres.txt
new file mode 100644
index 000000000..82914ba3e
--- /dev/null
+++ b/experiments/analysis-efficiency/questions/packet-postgres.txt
@@ -0,0 +1,3 @@
+Inspect the selected saved PostgreSQL capture {{artifact_path}} with REA. Within original capture frames 1 through 2000, return the first 12 TCP packets with source or destination port 5432 in original frame order. For each return integer frame, source_port, destination_port and flags (the numeric TCP flags word, not prose). Also report the matching packet count within this 2000-frame question scope.
+
+Do not claim whole-capture or network completeness. Return a single JSON object with exactly kind ("postgres"), matching_packets, rows, evidence_ids and limitations. Use actual successful REA Evidence IDs. Include a concise limitation explaining the recorded input scope; no extra claims or tool calls are needed.
diff --git a/experiments/analysis-efficiency/review-packet.mjs b/experiments/analysis-efficiency/review-packet.mjs
new file mode 100644
index 000000000..e7e3b0def
--- /dev/null
+++ b/experiments/analysis-efficiency/review-packet.mjs
@@ -0,0 +1,247 @@
+import { readFile, writeFile } from "node:fs/promises";
+import { resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import { isDeepStrictEqual, parseArgs } from "node:util";
+import { packetFacts, packetRequest } from "./packet-entry.mjs";
+
+const { values } = parseArgs({
+ options: {
+ trial: { type: "string" },
+ oracle: { type: "string" },
+ "runtime-root": { type: "string" },
+ selection: { type: "string" },
+ },
+});
+if (
+ !values.trial ||
+ !values.oracle ||
+ !values["runtime-root"] ||
+ !["baseline", "focused", "discovery"].includes(values.selection)
+)
+ throw new Error(
+ "Use --trial DIR --oracle JSON --runtime-root DIR --selection baseline|focused|discovery",
+ );
+
+const root = values["runtime-root"];
+const { parseEvidence } = await import(
+ pathToFileURL(resolve(root, "dist/domain/evidence.js")).href
+);
+const { inspectPacketCaptureInputSchema, packetCaptureSchema } = await import(
+ pathToFileURL(resolve(root, "dist/domain/captures/packetCapture.js")).href
+);
+const { inspectPacketFieldsInputSchema, packetFieldsCatalogSchema } =
+ await import(
+ pathToFileURL(resolve(root, "dist/domain/captures/packetFields.js")).href
+ );
+const oracle = JSON.parse(await readFile(values.oracle, "utf8"));
+const settings = JSON.parse(
+ await readFile(resolve(values.trial, "trial-settings.json"), "utf8"),
+);
+const summary = JSON.parse(
+ await readFile(resolve(values.trial, "summary.json"), "utf8"),
+);
+const expected = Object.fromEntries(
+ ["kind", "matching_packets", "rows"].map((key) => [key, oracle[key]]),
+);
+const calls = (await readFile(resolve(values.trial, "events.jsonl"), "utf8"))
+ .split("\n")
+ .filter(Boolean)
+ .map(JSON.parse)
+ .filter(
+ (event) =>
+ event.type === "item.completed" && event.item?.type === "mcp_tool_call",
+ )
+ .map((event) => event.item);
+const reasons = [];
+const workflowReasons = [];
+let answer;
+const captures = new Map();
+try {
+ if (oracle.kind !== "postgres")
+ throw new Error("This model rubric is the fixed PostgreSQL question only");
+ const expectedCalls =
+ values.selection === "discovery"
+ ? ["inspect_packet_fields", "inspect_packet_capture"]
+ : ["inspect_packet_capture"];
+ if (
+ !isDeepStrictEqual(
+ calls.map((call) => call.tool),
+ expectedCalls,
+ ) ||
+ calls.some(
+ (call) =>
+ call.server !== "rea" || call.status !== "completed" || call.error,
+ )
+ )
+ workflowReasons.push(
+ "Completed public REA call sequence differs from the fixed plan",
+ );
+ const ids = new Set();
+ const conflictingIds = new Set();
+ for (const call of calls) {
+ if (call.server !== "rea" || call.status !== "completed" || call.error)
+ continue;
+ try {
+ const result = call.result;
+ if (result?.isError)
+ throw new Error("Tool failure is not observed packet evidence");
+ const text =
+ result?.content?.filter((item) => item.type === "text") ?? [];
+ if (!result?.structuredContent && text.length !== 1)
+ throw new Error("No unambiguous captured Evidence envelope");
+ const evidence = result.structuredContent ?? JSON.parse(text[0].text);
+ parseEvidence(evidence);
+ if (conflictingIds.has(evidence.evidence_id))
+ throw new Error("Previously conflicting Evidence identity");
+ if (call.tool === "inspect_packet_fields") {
+ const input = inspectPacketFieldsInputSchema.parse({
+ kind: "fields",
+ protocol: "tcp",
+ query: "tcp.flags",
+ limit: 1,
+ });
+ const report = packetFieldsCatalogSchema.parse(
+ evidence.normalized_result,
+ );
+ if (
+ !isDeepStrictEqual(
+ inspectPacketFieldsInputSchema.parse(call.arguments),
+ input,
+ ) ||
+ !isDeepStrictEqual(evidence.parameters, input) ||
+ !isDeepStrictEqual(report.selection, input) ||
+ evidence.operation !== call.tool ||
+ !report.records.some((row) => row.abbreviation === "tcp.flags") ||
+ evidence.subject?.local_path !== report.tool.path ||
+ evidence.subject?.digest?.sha256 !== report.tool.sha256
+ )
+ throw new Error(
+ "Discovery Evidence does not establish the requested installed field",
+ );
+ } else if (call.tool === "inspect_packet_capture") {
+ const planned = inspectPacketCaptureInputSchema.parse(
+ packetRequest(
+ "wireshark-pgsql-jdbc.pcap",
+ settings.selected_artifact_path,
+ values.selection,
+ ),
+ );
+ const input = inspectPacketCaptureInputSchema.parse(call.arguments);
+ if (!isDeepStrictEqual(input, planned))
+ workflowReasons.push("Capture selection differs from the fixed plan");
+ const report = packetCaptureSchema.parse(evidence.normalized_result);
+ if (
+ !isDeepStrictEqual(evidence.parameters, input) ||
+ !isDeepStrictEqual(report.selection, input) ||
+ evidence.operation !== call.tool ||
+ evidence.subject?.local_path !== input.path ||
+ evidence.subject?.digest?.sha256 !== oracle.sha256 ||
+ report.artifact.path !== input.path ||
+ report.artifact.sha256 !== oracle.sha256
+ )
+ throw new Error(
+ "Capture Evidence differs from the fixed input/byte identity",
+ );
+ const observed = packetFacts("postgres", report);
+ if (
+ captures.has(evidence.evidence_id) &&
+ !isDeepStrictEqual(captures.get(evidence.evidence_id), observed)
+ ) {
+ captures.delete(evidence.evidence_id);
+ ids.delete(evidence.evidence_id);
+ conflictingIds.add(evidence.evidence_id);
+ throw new Error("Repeated Evidence identity has inconsistent facts");
+ }
+ captures.set(evidence.evidence_id, observed);
+ } else {
+ throw new Error("Call is not a packet observation tool");
+ }
+ ids.add(evidence.evidence_id);
+ } catch (error) {
+ workflowReasons.push(
+ `Call ${call.tool} did not provide authenticated packet observations: ${String(error)}`,
+ );
+ }
+ }
+ answer = JSON.parse(
+ (await readFile(resolve(values.trial, "answer.txt"), "utf8")).trim(),
+ );
+ if (
+ !isDeepStrictEqual(
+ Object.keys(answer).sort(),
+ [
+ "kind",
+ "matching_packets",
+ "rows",
+ "evidence_ids",
+ "limitations",
+ ].sort(),
+ )
+ )
+ workflowReasons.push("Final answer shape differs from the fixed plan");
+ if (
+ !isDeepStrictEqual(
+ Object.fromEntries(
+ Object.keys(expected).map((key) => [key, answer[key]]),
+ ),
+ expected,
+ )
+ )
+ throw new Error("Final answer is not the exact requested recorded facts");
+ if (
+ !Array.isArray(answer.evidence_ids) ||
+ answer.evidence_ids.some((id) => !ids.has(id)) ||
+ new Set(answer.evidence_ids).size !== answer.evidence_ids.length
+ )
+ throw new Error("Final citations lack the authenticated packet Evidence");
+ const supportingFacts = answer.evidence_ids
+ .map((id) => captures.get(id))
+ .find((observed) => isDeepStrictEqual(observed, expected));
+ if (!supportingFacts)
+ throw new Error(
+ "Cited observations differ from independent raw header oracle",
+ );
+ const limitations =
+ typeof answer.limitations === "string"
+ ? [answer.limitations]
+ : answer.limitations;
+ if (
+ !Array.isArray(limitations) ||
+ limitations.length === 0 ||
+ limitations.some((item) => typeof item !== "string" || !item.trim())
+ )
+ workflowReasons.push("Final answer omits investigation limitations");
+ if (
+ summary.exit_code !== 0 ||
+ summary.timeout ||
+ summary.selected_artifact_bytes_unchanged !== true ||
+ settings.selected_artifact_sha256_before !== oracle.sha256 ||
+ summary.selected_artifact_sha256_after !== oracle.sha256
+ )
+ workflowReasons.push(
+ "Trial did not complete with the selected artifact unchanged",
+ );
+} catch (error) {
+ reasons.push(String(error));
+}
+const review = {
+ factual_correctness: reasons.length ? "failed" : "passed",
+ scope:
+ "Fixed first-2000-frame PostgreSQL question; authenticated tool observations and independent raw header oracle",
+ workflow_correctness:
+ reasons.length || workflowReasons.length ? "failed" : "passed",
+ reasons,
+ workflow_reasons: workflowReasons,
+ completed_calls: calls.length,
+ wall_seconds: summary.wall_seconds,
+ usage: summary.usage,
+ selected_artifact_sha256: oracle.sha256,
+ limitation_prose:
+ "Presence checked; wording requires transcript review, not heuristic factual certification",
+};
+await writeFile(
+ resolve(values.trial, "packet-review.json"),
+ JSON.stringify(review, null, 2) + "\n",
+);
+console.log(JSON.stringify(review));
+if (reasons.length || workflowReasons.length) process.exitCode = 1;
diff --git a/experiments/analysis-efficiency/test_packet_oracle.py b/experiments/analysis-efficiency/test_packet_oracle.py
new file mode 100644
index 000000000..6727b3b7e
--- /dev/null
+++ b/experiments/analysis-efficiency/test_packet_oracle.py
@@ -0,0 +1,67 @@
+import importlib.util
+import struct
+import unittest
+from pathlib import Path
+
+spec = importlib.util.spec_from_file_location('packet_oracle', Path(__file__).with_name('packet-oracle.py'))
+oracle = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(oracle)
+
+
+class PacketOracleTest(unittest.TestCase):
+ def test_compression_pointer_preserves_question_cursor(self):
+ message = b'\x03www\x07example\x03com\0' + b'\xc0\0' + b'\0\x01\0\x01'
+ self.assertEqual(oracle.dns_name(message, 17), ('www.example.com', 19))
+
+ def test_compression_cycle_and_missing_pointer_byte_fail(self):
+ for message in (b'\xc0\0', b'\xc0', b'\xc0\x02\xc0\0'):
+ with self.subTest(message=message), self.assertRaises(ValueError):
+ oracle.dns_name(message, 0)
+
+ def test_truncated_pcap_record_is_not_exhaustion(self):
+ header = struct.pack('<IHHIIII', 0xa1b2c3d4, 2, 4, 0, 0, 65535, 1)
+ with self.assertRaises(ValueError):
+ list(oracle.packets(header + b'\0'))
+ record = struct.pack('<IIII', 0, 0, 20, 20)
+ with self.assertRaises(ValueError):
+ list(oracle.packets(header + record + b'\0' * 19))
+
+ def test_missing_dhcp_server_id_stays_omitted(self):
+ payload = b'\0' * 236 + b'\x63\x82\x53\x63\x35\x01\x01\xff'
+ self.assertEqual(oracle.dhcp_options(payload), {53: b'\x01'})
+
+ def test_dhcp_duplicates_truncation_and_missing_end_fail(self):
+ prefix = b'\0' * 236 + b'\x63\x82\x53\x63'
+ for suffix in (b'\x35\x01\x01\x35\x01\x02\xff', b'\x36\x04\0', b'\x35\x01\x01'):
+ with self.subTest(suffix=suffix), self.assertRaises(ValueError):
+ oracle.dhcp_options(prefix + suffix)
+
+ def test_fragmented_selected_traffic_cannot_pass_header_oracle(self):
+ ip = bytearray(20)
+ ip[0], ip[9] = 0x45, 17
+ struct.pack_into('!H', ip, 2, 20)
+ struct.pack_into('!H', ip, 6, 0x2000)
+ with self.assertRaises(ValueError):
+ oracle.transport(b'\0' * 12 + b'\x08\0' + ip)
+
+ def test_declared_ip_length_and_tcp_options_cannot_be_truncated(self):
+ ip = bytearray(40)
+ ip[0], ip[9], ip[32] = 0x45, 6, 0xf0
+ struct.pack_into('!H', ip, 2, 40)
+ struct.pack_into('!HH', ip, 20, 12345, 5432)
+ packet = b'\0' * 12 + b'\x08\0' + ip
+ header = struct.pack('<IHHIIII', 0xa1b2c3d4, 2, 4, 0, 0, 65535, 1)
+ record = struct.pack('<IIII', 0, 0, len(packet), len(packet))
+ with self.assertRaisesRegex(ValueError, 'TCP header'):
+ oracle.analyze(header + record + packet, 'postgres')
+ struct.pack_into('!H', ip, 2, 60)
+ with self.assertRaisesRegex(ValueError, 'truncated IPv4'):
+ oracle.transport(b'\0' * 12 + b'\x08\0' + ip)
+
+ def test_unknown_link_protocol_does_not_prove_tcp_absence(self):
+ with self.assertRaisesRegex(ValueError, 'unsupported Ethernet'):
+ oracle.transport(b'\0' * 12 + b'\x86\xdd' + b'\0' * 40)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tests/evaluation/analysisEfficiencyComparison.test.ts b/tests/evaluation/analysisEfficiencyComparison.test.ts
index 57a1ffb8c..924ba1612 100644
--- a/tests/evaluation/analysisEfficiencyComparison.test.ts
+++ b/tests/evaluation/analysisEfficiencyComparison.test.ts
@@ -30,11 +30,17 @@ type Fixture = {
delivery: Delivery;
provider_search_path_sha256: string;
calls: unknown[];
+ packet_selection?: string;
};
answer: {
capture: { coverage: string; capture_start: null };
http_request_uris_first2: string[];
delivery_observations: Observation[];
+ facts?: {
+ kind: string;
+ matching_packets: number;
+ rows: { frame: number }[];
+ };
};
};
const reportSchema = z.object({
@@ -123,6 +129,39 @@ function compare(left: Fixture, right: Fixture) {
}
}
+for (const artifact of [
+ "wireshark-pgsql-jdbc.pcap",
+ "wireshark-dns.cap",
+ "wireshark-dhcp.pcap",
+]) {
+ test(`${artifact}: compare requested facts across packet plans and preserve observations`, () => {
+ const left = fixture("complete", artifact);
+ left.summary.packet_selection = "baseline";
+ left.answer.facts = {
+ kind: "packet",
+ matching_packets: 1337,
+ rows: [{ frame: 9 }],
+ };
+ const right = structuredClone(left);
+ right.summary.packet_selection = "focused";
+ right.answer.delivery_observations[0]!.evidence_id = "ev_focused";
+ right.answer.delivery_observations[0]!.coverage =
+ "up-to-input-packet-limit";
+ const equivalent = compare(left, right);
+ assert.equal(equivalent.status, 0);
+ assert.equal(equivalent.report.facts_match, true);
+ assert.deepEqual(
+ equivalent.report.preserved_run_specific_values.map((row) => row.value),
+ [left.answer.delivery_observations, right.answer.delivery_observations],
+ );
+ right.answer.facts!.matching_packets++;
+ assert.equal(compare(left, right).report.facts_match, false);
+ right.answer.facts!.matching_packets--;
+ right.summary.packet_selection = "baseline";
+ assert.equal(compare(left, right).report.facts_match, false);
+ });
+}
+
for (const artifact of [
"wireshark-http.cap",
"wireshark-http_redirects.pcapng",
diff --git a/tests/evaluation/packetWorkflowSemantics.test.ts b/tests/evaluation/packetWorkflowSemantics.test.ts
new file mode 100644
index 000000000..aac1f5976
--- /dev/null
+++ b/tests/evaluation/packetWorkflowSemantics.test.ts
@@ -0,0 +1,32 @@
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { test } from "vitest";
+
+test("packet page verification rejects skipped matches with unchanged counts and monotone frames", () => {
+ const moduleUrl = new URL(
+ "../../experiments/analysis-efficiency/packet-entry.mjs",
+ import.meta.url,
+ ).href;
+ const source = `
+ import assert from 'node:assert/strict';
+ import {verifyPacketPage} from ${JSON.stringify(moduleUrl)};
+ const rows = Array.from({length:36}, (_,i)=>({frame:i+1, source_port:1000,destination_port:5432,flags:16}));
+ const sha256 = 'a'.repeat(64);
+ const report = (selected)=>({artifact:{sha256},output_window:{matching_packets:36},packets:selected.map(row=>({
+ 'frame.number':[String(row.frame)],'tcp.srcport':['1000'],'tcp.dstport':['5432'],'tcp.flags':['0x0010']
+ }))});
+ const oracle={sha256,matching_packets:36,rows:rows.slice(0,24)};
+ const first=report(rows.slice(0,12));
+ verifyPacketPage(first,report(rows.slice(12,24)),oracle);
+ assert.throws(()=>verifyPacketPage(first,report(rows.slice(24,36)),oracle),/independently parsed/);
+ assert.throws(()=>verifyPacketPage(first,report(rows.slice(12,24).reverse()),oracle),/independently parsed/);
+ assert.throws(()=>verifyPacketPage(first,{...report(rows.slice(12,24)),artifact:{sha256:'b'.repeat(64)}},oracle),/independently parsed/);
+ `;
+ // Semantic checker only; synthetic rows do not establish provider behavior.
+ const result = spawnSync(process.execPath, ["--input-type=module", "-"], {
+ input: source,
+ encoding: "utf8",
+ });
+ assert.ifError(result.error);
+ assert.equal(result.status, 0, result.stderr);
+});