Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

test(perf): measure real sigrok annotation selection workflows

Ysopwnableca1b2f8608c5
diff --git a/experiments/analysis-efficiency/README.md b/experiments/analysis-efficiency/README.md
index 9a14b764a..2ca9da3a4 100644
--- a/experiments/analysis-efficiency/README.md
+++ b/experiments/analysis-efficiency/README.md
@@ -78,6 +78,12 @@ field/filter selection, discovery and output windows. Lower delivered JSON did
not establish stable total-token or speed gains; failed plans and negative
controls remain visible.
+The [sigrok selection checkpoint](signal-selection-workflows.md) records two
+real hardware recordings, independently verified bytes, repeated decoder/pages,
+and an eight-task Codex cohort with mixed token/time results. Existing annotation
+controls reduce replies; these measurements introduce no production cache,
+default or API changes.
+
Agent token consumption and end-to-end task time are the primary objectives.
Memory and CPU consumption are also measured and controlled. Correctness,
Evidence provenance, caller choices and existing-user compatibility are
diff --git a/experiments/analysis-efficiency/agent-trial.py b/experiments/analysis-efficiency/agent-trial.py
index db994cea1..365337a06 100644
--- a/experiments/analysis-efficiency/agent-trial.py
+++ b/experiments/analysis-efficiency/agent-trial.py
@@ -182,6 +182,7 @@ def main():
'inspect_firmware_regions', 'inspect_managed_artifact',
'inspect_sqlite_database', 'inspect_pcap_capture',
'inspect_packet_capture', 'inspect_packet_fields',
+ 'decode_signal_recording', 'inspect_signal_decoders',
'inspect_managed_members', 'inspect_managed_native_boundaries',
'inspect_artifact', 'inspect_wasm_artifact',
'inspect_evm_interface', 'get_evidence_bundle', 'export_evidence_bundle',
diff --git a/experiments/analysis-efficiency/compare.mjs b/experiments/analysis-efficiency/compare.mjs
index 01b9f97af..e1e9f4cda 100644
--- a/experiments/analysis-efficiency/compare.mjs
+++ b/experiments/analysis-efficiency/compare.mjs
@@ -3,6 +3,7 @@ import { readFile, writeFile } from "node:fs/promises";
import { resolve } from "node:path";
import { parseArgs } from "node:util";
import { PACKET_ARTIFACTS } from "./packet-entry.mjs";
+import { SIGNAL_ARTIFACTS } from "./signal-entry.mjs";
const { values } = parseArgs({
options: {
@@ -36,6 +37,8 @@ const sameSettings = [
"native_workflow_source_sha256",
"packet_workflow_source_sha256",
"packet_oracle_source_sha256",
+ "signal_workflow_source_sha256",
+ "signal_oracle_source_sha256",
"package_lock_sha256",
"node_version",
"sdk_client_version",
@@ -73,13 +76,14 @@ for (const [side, answer] of [
value: answer.evidence_ids,
});
}
-// PCAP delivery observations preserve each actual receipt's identity, limitations
+// Recording delivery observations preserve each actual receipt's identity, limitations
// and view coverage. Summary/page scopes differ from complete source scope;
// retain them separately rather than equating page exhaustion with capture
// coverage. Repeated equivalent deliveries must also match every observation
// field except the receipt and parent IDs, which identify their own invocations.
if (
Object.hasOwn(PACKET_ARTIFACTS, left.artifact) ||
+ Object.hasOwn(SIGNAL_ARTIFACTS, left.artifact) ||
["wireshark-http.cap", "wireshark-http_redirects.pcapng"].includes(
left.artifact,
)
@@ -92,7 +96,7 @@ if (
!Array.isArray(answer.delivery_observations) ||
!answer.delivery_observations.length
)
- throw new Error("Missing original PCAP delivery observations");
+ throw new Error("Missing original recording delivery observations");
runSpecific.push({
side,
path: ["delivery_observations"],
@@ -101,7 +105,9 @@ if (
if (
left.delivery === right.delivery &&
(!Object.hasOwn(PACKET_ARTIFACTS, left.artifact) ||
- left.packet_selection === right.packet_selection)
+ left.packet_selection === right.packet_selection) &&
+ (!Object.hasOwn(SIGNAL_ARTIFACTS, left.artifact) ||
+ left.signal_selection === right.signal_selection)
) {
answer.delivery_observations = answer.delivery_observations.map(
(observation) =>
diff --git a/experiments/analysis-efficiency/corpus.json b/experiments/analysis-efficiency/corpus.json
index e595dde34..983b81799 100644
--- a/experiments/analysis-efficiency/corpus.json
+++ b/experiments/analysis-efficiency/corpus.json
@@ -343,6 +343,36 @@
"derivation": "Unmodified official saved sample; decompressed gzip where indicated. Local digest pins acquired bytes, not an upstream-published digest.",
"compressed_source": null
}
+ },
+ {
+ "id": "sigrok-uart-max3232e.sr",
+ "file": "uart-max3232e.sr",
+ "bytes": 12742,
+ "sha256": "f797d60d6b8e12d2a1549b9a5a499ef389f44cb9e870e8610af26b57c3b2be76",
+ "provenance": {
+ "project": "sigrok-dumps",
+ "version": "0ad13477abc959d37fc9a5acbd23901c371c9c76",
+ "upstreamFile": "uart/maxim_max3232e/max3232e_hello_world_57600_8n1.sr",
+ "url": "https://raw.githubusercontent.com/sigrokproject/sigrok-dumps/0ad13477abc959d37fc9a5acbd23901c371c9c76/uart/maxim_max3232e/max3232e_hello_world_57600_8n1.sr",
+ "git_blob_sha1": "52e47b3014819471a2f348a0d8c7694fe304e052",
+ "reference": "https://raw.githubusercontent.com/sigrokproject/sigrok-dumps/0ad13477abc959d37fc9a5acbd23901c371c9c76/uart/maxim_max3232e/README",
+ "license_scope": "Repository README declares dumps public domain unless otherwise noted; selected READMEs state no exception; decoder software licensing is separate."
+ }
+ },
+ {
+ "id": "sigrok-i2c-nunchuk.sr",
+ "file": "i2c-nunchuk.sr",
+ "bytes": 20373,
+ "sha256": "0fa6adb8297d7f3fe5ba19c94ee655c9b0f952a8b5e48a17aaa499a2f6d10d70",
+ "provenance": {
+ "project": "sigrok-dumps",
+ "version": "0ad13477abc959d37fc9a5acbd23901c371c9c76",
+ "upstreamFile": "i2c/wii_nunchuk/wii_nunchuk_init_reg_3xdata.sr",
+ "url": "https://raw.githubusercontent.com/sigrokproject/sigrok-dumps/0ad13477abc959d37fc9a5acbd23901c371c9c76/i2c/wii_nunchuk/wii_nunchuk_init_reg_3xdata.sr",
+ "git_blob_sha1": "a7232fce7b54e68e2be44e15946c90a3c756e8e9",
+ "reference": "https://raw.githubusercontent.com/sigrokproject/sigrok-dumps/0ad13477abc959d37fc9a5acbd23901c371c9c76/i2c/wii_nunchuk/README",
+ "license_scope": "Repository README declares dumps public domain unless otherwise noted; selected READMEs state no exception; decoder software licensing is separate."
+ }
}
]
}
diff --git a/experiments/analysis-efficiency/measure.mjs b/experiments/analysis-efficiency/measure.mjs
index 07a39e3c1..20f1e7d61 100644
--- a/experiments/analysis-efficiency/measure.mjs
+++ b/experiments/analysis-efficiency/measure.mjs
@@ -13,6 +13,7 @@ import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { Ajv2020 } from "ajv/dist/2020.js";
import { PACKET_ARTIFACTS, measurePacketEntry } from "./packet-entry.mjs";
+import { SIGNAL_ARTIFACTS, measureSignalEntry } from "./signal-entry.mjs";
const root = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
const { values } = parseArgs({
@@ -31,6 +32,7 @@ const { values } = parseArgs({
schemas: { type: "string", default: "default" },
"sqlite-selection": { type: "string", default: "all" },
"packet-selection": { type: "string", default: "baseline" },
+ "signal-selection": { type: "string", default: "baseline" },
},
});
if (
@@ -41,6 +43,13 @@ if (
!isAbsolute(values.path) ||
!isAbsolute(values.output) ||
!["complete", "summary"].includes(values.delivery) ||
+ !["baseline", "focused", "discovery", "pages"].includes(
+ values["signal-selection"],
+ ) ||
+ (!Object.hasOwn(SIGNAL_ARTIFACTS, values.artifact) &&
+ values["signal-selection"] !== "baseline") ||
+ (Object.hasOwn(SIGNAL_ARTIFACTS, values.artifact) &&
+ values.delivery !== "complete") ||
!["entry", "entry-and-main", "entry-views"].includes(
values["native-question"],
) ||
@@ -159,6 +168,12 @@ const fingerprints = {
packet_oracle_source_sha256: await fileSha(
fileURLToPath(new URL("./packet-oracle.py", import.meta.url)),
),
+ signal_workflow_source_sha256: await fileSha(
+ fileURLToPath(new URL("./signal-entry.mjs", import.meta.url)),
+ ),
+ signal_oracle_source_sha256: await fileSha(
+ fileURLToPath(new URL("./signal-oracle.py", import.meta.url)),
+ ),
runtime_files_sha256: sha(JSON.stringify(runtimeFiles)),
runtime_dispatcher_sha256: await fileSha(
resolve(values["runtime-root"], "scripts/rea.mjs"),
@@ -739,6 +754,12 @@ const inspectPcapTarget = async () => {
};
const workflows = {
+ ...Object.fromEntries(
+ Object.keys(SIGNAL_ARTIFACTS).map((artifact) => [
+ artifact,
+ () => measureSignalEntry({ values, call, inputSha, catalogDefinitions }),
+ ]),
+ ),
...Object.fromEntries(
Object.keys(PACKET_ARTIFACTS).map((artifact) => [
artifact,
@@ -1186,6 +1207,9 @@ const report = {
packet_selection: Object.hasOwn(PACKET_ARTIFACTS, values.artifact)
? values["packet-selection"]
: null,
+ signal_selection: Object.hasOwn(SIGNAL_ARTIFACTS, values.artifact)
+ ? values["signal-selection"]
+ : null,
provider_settings_sha256: sha(JSON.stringify(selectedProviders)),
provider_search_path_sha256: sha(
selectedProviders.PATH ?? process.env.PATH ?? "",
diff --git a/experiments/analysis-efficiency/review-signal.mjs b/experiments/analysis-efficiency/review-signal.mjs
new file mode 100644
index 000000000..9ae454207
--- /dev/null
+++ b/experiments/analysis-efficiency/review-signal.mjs
@@ -0,0 +1,237 @@
+import { readFile, writeFile } from "node:fs/promises";
+import { resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import { isDeepStrictEqual, parseArgs } from "node:util";
+import {
+ SIGNAL_ARTIFACTS,
+ signalRequest,
+ signalDataRows,
+ signalFacts,
+ verifySignalRows,
+ verifySignalOracleIdentity,
+} from "./signal-entry.mjs";
+
+const { values } = parseArgs({
+ options: {
+ trial: { type: "string" },
+ oracle: { type: "string" },
+ "runtime-root": { type: "string" },
+ artifact: { type: "string" },
+ selection: { type: "string" },
+ },
+});
+if (
+ !values.trial ||
+ !values.oracle ||
+ !values["runtime-root"] ||
+ !Object.hasOwn(SIGNAL_ARTIFACTS, values.artifact ?? "") ||
+ !["baseline", "focused"].includes(values.selection)
+)
+ throw new Error(
+ "Use --trial DIR --oracle JSON --runtime-root DIR --artifact ID --selection baseline|focused",
+ );
+const json = async (name) =>
+ JSON.parse(await readFile(resolve(values.trial, name), "utf8"));
+const settings = await json("trial-settings.json"),
+ summary = await json("summary.json");
+const oracle = JSON.parse(await readFile(values.oracle, "utf8"));
+const kind = SIGNAL_ARTIFACTS[values.artifact].kind;
+const expectedRows =
+ kind === "uart"
+ ? oracle.rows
+ : oracle.reads.flatMap((read) => read.bytes.slice(1));
+const expected = signalFacts(kind, expectedRows);
+const runtime = (file) =>
+ pathToFileURL(resolve(values["runtime-root"], file)).href;
+const { parseEvidence } = await import(runtime("dist/domain/evidence.js"));
+const { decodeSignalRecordingInputSchema, signalRecordingSchema } =
+ await import(runtime("dist/domain/signals/signalRecording.js"));
+const { inspectSignalDecodersInputSchema, signalDecodersSchema } = await import(
+ runtime("dist/domain/signals/signalDecoders.js")
+);
+const calls = (await readFile(resolve(values.trial, "events.jsonl"), "utf8"))
+ .split("\n")
+ .filter(Boolean)
+ .map(JSON.parse)
+ .filter(
+ (event) =>
+ event.type === "item.completed" && event.item?.type === "mcp_tool_call",
+ )
+ .map((event) => event.item);
+const reasons = [],
+ workflowReasons = [],
+ authenticated = new Map(),
+ conflicting = new Set();
+try {
+ verifySignalOracleIdentity(values.artifact, oracle);
+ if (oracle.kind !== kind)
+ throw new Error("Wrong independent signal question oracle");
+ if (calls.length !== 1 || calls[0]?.tool !== "decode_signal_recording")
+ workflowReasons.push(
+ "Completed public call sequence differs from the fixed single-call plan",
+ );
+ const planned = decodeSignalRecordingInputSchema.parse(
+ signalRequest(
+ values.artifact,
+ settings.selected_artifact_path,
+ values.selection,
+ ),
+ );
+ for (const call of calls) {
+ try {
+ if (
+ call.server !== "rea" ||
+ call.status !== "completed" ||
+ call.error ||
+ call.result?.isError
+ )
+ throw new Error("Call did not return successful REA observations");
+ const text =
+ call.result?.content?.filter((item) => item.type === "text") ?? [];
+ if (!call.result?.structuredContent && text.length !== 1)
+ throw new Error("No unambiguous Evidence envelope");
+ const evidence =
+ call.result.structuredContent ?? JSON.parse(text[0].text);
+ parseEvidence(evidence);
+ const capture = call.tool === "decode_signal_recording";
+ if (!capture && call.tool !== "inspect_signal_decoders")
+ throw new Error("Not a signal observation tool");
+ const input = (
+ capture
+ ? decodeSignalRecordingInputSchema
+ : inspectSignalDecodersInputSchema
+ ).parse(call.arguments);
+ const report = (
+ capture ? signalRecordingSchema : signalDecodersSchema
+ ).parse(evidence.normalized_result);
+ if (
+ evidence.operation !== call.tool ||
+ !isDeepStrictEqual(evidence.parameters, input) ||
+ !isDeepStrictEqual(report.selection, input)
+ )
+ throw new Error(
+ "Evidence differs from actual operation or caller choices",
+ );
+ if (
+ capture
+ ? evidence.subject?.local_path !== input.path ||
+ evidence.subject?.digest?.sha256 !== oracle.sha256 ||
+ report.artifact.path !== input.path ||
+ report.artifact.sha256 !== oracle.sha256
+ : evidence.subject?.local_path !== report.tool.path ||
+ evidence.subject?.digest?.sha256 !== report.tool.sha256
+ )
+ throw new Error("Evidence identifies different recording/tool bytes");
+ if (capture && !isDeepStrictEqual(input, planned))
+ workflowReasons.push("Capture selection differs from the fixed plan");
+ const observation = {
+ operation: call.tool,
+ input,
+ report,
+ rows: capture ? signalDataRows(kind, report.annotations_text) : null,
+ };
+ const id = evidence.evidence_id;
+ if (conflicting.has(id))
+ throw new Error("Previously conflicting Evidence identity");
+ if (
+ authenticated.has(id) &&
+ !isDeepStrictEqual(authenticated.get(id), observation)
+ ) {
+ authenticated.delete(id);
+ conflicting.add(id);
+ throw new Error(
+ "Repeated Evidence identity has inconsistent observations",
+ );
+ }
+ authenticated.set(id, observation);
+ } catch (error) {
+ workflowReasons.push(
+ `Call ${call.tool} did not provide authenticated observations: ${String(error)}`,
+ );
+ }
+ }
+ const answer = JSON.parse(
+ (await readFile(resolve(values.trial, "answer.txt"), "utf8")).trim(),
+ );
+ if (
+ !isDeepStrictEqual(
+ Object.keys(answer).sort(),
+ [...Object.keys(expected), "evidence_ids", "limitations"].sort(),
+ )
+ )
+ workflowReasons.push("Final answer shape differs from the fixed plan");
+ if (
+ !isDeepStrictEqual(
+ Object.fromEntries(
+ Object.keys(expected).map((key) => [key, answer[key]]),
+ ),
+ expected,
+ )
+ )
+ throw new Error("Final answer differs from independent sample facts");
+ if (
+ !Array.isArray(answer.evidence_ids) ||
+ new Set(answer.evidence_ids).size !== answer.evidence_ids.length ||
+ answer.evidence_ids.some((id) => !authenticated.has(id))
+ )
+ throw new Error(
+ "Final citations are not authenticated signal observations",
+ );
+ const rows = new Map();
+ for (const id of answer.evidence_ids) {
+ for (const row of authenticated.get(id).rows ?? []) {
+ const old = rows.get(row.start);
+ if (old && !isDeepStrictEqual(old, row))
+ throw new Error("Cited sample ranges have conflicting decoded data");
+ rows.set(row.start, row);
+ }
+ }
+ verifySignalRows(
+ kind,
+ [...rows.values()].sort((a, b) => a.start - b.start),
+ oracle,
+ oracle.sha256,
+ );
+ const limitations =
+ typeof answer.limitations === "string"
+ ? [answer.limitations]
+ : answer.limitations;
+ if (
+ !Array.isArray(limitations) ||
+ limitations.length === 0 ||
+ limitations.some((value) => typeof value !== "string" || !value.trim())
+ )
+ workflowReasons.push("Final answer omits investigation limitations");
+ if (
+ summary.exit_code !== 0 ||
+ summary.timeout ||
+ summary.selected_artifact_bytes_unchanged !== true ||
+ settings.selected_artifact_sha256_before !== oracle.sha256 ||
+ summary.selected_artifact_sha256_after !== oracle.sha256
+ )
+ workflowReasons.push(
+ "Trial did not complete with original selected bytes unchanged",
+ );
+} catch (error) {
+ reasons.push(String(error));
+}
+const report = {
+ factual_correctness: reasons.length ? "failed" : "passed",
+ workflow_correctness:
+ reasons.length || workflowReasons.length ? "failed" : "passed",
+ reasons,
+ workflow_reasons: workflowReasons,
+ completed_calls: calls.length,
+ wall_seconds: summary.wall_seconds,
+ usage: summary.usage,
+ selected_artifact_sha256: oracle.sha256,
+ scope:
+ "Two pinned single-device recordings; independently sampled bytes and authenticated cited decoder observations; hardware behavior is not inferred",
+ limitation_prose: "Presence checked; wording requires transcript review",
+};
+await writeFile(
+ resolve(values.trial, "signal-review.json"),
+ JSON.stringify(report, null, 2) + "\n",
+);
+console.log(JSON.stringify(report));
+if (reasons.length || workflowReasons.length) process.exitCode = 1;
diff --git a/experiments/analysis-efficiency/signal-entry.mjs b/experiments/analysis-efficiency/signal-entry.mjs
new file mode 100644
index 000000000..b44d4d66a
--- /dev/null
+++ b/experiments/analysis-efficiency/signal-entry.mjs
@@ -0,0 +1,264 @@
+import { execFile } from "node:child_process";
+import { readFile } from "node:fs/promises";
+import { resolve } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import { isDeepStrictEqual, promisify } from "node:util";
+
+const corpus = JSON.parse(
+ await readFile(new URL("./corpus.json", import.meta.url), "utf8"),
+);
+
+export const SIGNAL_ARTIFACTS = {
+ "sigrok-uart-max3232e.sr": {
+ kind: "uart",
+ decoders: "uart:rx=MAX3232E DOUT1:baudrate=57600:format=hex:invert_rx=yes",
+ annotations: "uart=rx-data",
+ query: "rx-data",
+ firstPage: 32,
+ },
+ "sigrok-i2c-nunchuk.sr": {
+ kind: "i2c",
+ decoders: "i2c:scl=SCL:sda=SDA",
+ annotations: "i2c=data-read",
+ query: "data-read",
+ firstPage: 9,
+ },
+};
+
+export function signalRequest(artifact, path, selection) {
+ const descriptor = SIGNAL_ARTIFACTS[artifact];
+ if (!descriptor) throw new Error("Unknown fixed signal question");
+ return {
+ path,
+ decoders: descriptor.decoders,
+ input_format: null,
+ annotations: selection === "baseline" ? null : descriptor.annotations,
+ line_offset: 0,
+ line_limit: selection === "pages" ? descriptor.firstPage : null,
+ };
+}
+
+export function signalDataRows(kind, text) {
+ const pattern =
+ kind === "uart"
+ ? /^(\d+)-(\d+) uart-1: ([0-9A-F]{2})$/
+ : /^(\d+)-(\d+) i2c-1: Data read: ([0-9A-F]{2})$/;
+ const rows = [];
+ for (const line of text.split("\n")) {
+ const match = pattern.exec(line);
+ if (!match) continue;
+ const start = Number(match[1]),
+ end = Number(match[2]);
+ if (
+ !Number.isSafeInteger(start) ||
+ !Number.isSafeInteger(end) ||
+ start > end
+ )
+ throw new Error("Unsupported decoded sample interval");
+ rows.push({ start, end, byte: Number.parseInt(match[3], 16) });
+ }
+ return rows;
+}
+
+export function signalFacts(kind, rows) {
+ if (kind === "uart") return { kind, bytes: rows.map((row) => row.byte) };
+ if (kind !== "i2c" || rows.length !== 18)
+ throw new Error(
+ "The fixed I2C question requires three complete six-byte reads",
+ );
+ return {
+ kind,
+ reads: [0, 6, 12].map((offset) =>
+ rows.slice(offset, offset + 6).map((row) => row.byte),
+ ),
+ };
+}
+
+export function verifySignalOracleIdentity(artifact, oracle) {
+ const pinned = corpus.artifacts.find((entry) => entry.id === artifact);
+ if (
+ !pinned ||
+ oracle.kind !== SIGNAL_ARTIFACTS[artifact]?.kind ||
+ oracle.sha256 !== pinned.sha256
+ )
+ throw new Error(
+ "Independent oracle does not identify the pinned signal corpus",
+ );
+}
+
+export function verifySignalRows(kind, rows, oracle, digest) {
+ const expected =
+ kind === "uart"
+ ? oracle.rows
+ : oracle.reads.flatMap((read) =>
+ read.bytes.slice(1).map((byte, index) => ({
+ ...byte,
+ after_previous: read.bytes[index].last_clock,
+ before_next: read.bytes[index + 2]?.first_clock ?? read.stop,
+ })),
+ );
+ if (
+ oracle.kind !== kind ||
+ oracle.sha256 !== digest ||
+ rows.length !== expected.length
+ )
+ throw new Error(
+ "Signal observations differ from independent sample identity/coverage",
+ );
+ for (let index = 0; index < rows.length; index++) {
+ const observed = rows[index],
+ raw = expected[index];
+ if (
+ observed.byte !== raw.byte ||
+ observed.start < 0 ||
+ observed.end >= oracle.total_samples
+ )
+ throw new Error("Decoded data differs from independent sample bits");
+ if (
+ kind === "uart"
+ ? !Number.isSafeInteger(raw.first_data_sample) ||
+ !Number.isSafeInteger(raw.last_data_sample) ||
+ observed.start < raw.start ||
+ observed.end >= raw.end_exclusive ||
+ observed.start > raw.first_data_sample ||
+ observed.end < raw.last_data_sample
+ : !Number.isSafeInteger(raw.after_previous) ||
+ !Number.isSafeInteger(raw.before_next) ||
+ observed.start <= raw.after_previous ||
+ observed.end >= raw.before_next ||
+ observed.start > raw.first_clock ||
+ observed.end < raw.last_clock
+ )
+ throw new Error(
+ "Decoded interval does not identify the independently read byte",
+ );
+ }
+}
+
+export async function measureSignalEntry({
+ values,
+ call,
+ inputSha,
+ catalogDefinitions,
+}) {
+ const descriptor = SIGNAL_ARTIFACTS[values.artifact];
+ const runtime = (file) =>
+ pathToFileURL(resolve(values["runtime-root"], file)).href;
+ const { parseEvidence } = await import(runtime("dist/domain/evidence.js"));
+ const { decodeSignalRecordingInputSchema, signalRecordingSchema } =
+ await import(runtime("dist/domain/signals/signalRecording.js"));
+ const { inspectSignalDecodersInputSchema, signalDecodersSchema } =
+ await import(runtime("dist/domain/signals/signalDecoders.js"));
+ const observations = [];
+ const checked = (evidence, operation, input, schema) => {
+ parseEvidence(evidence);
+ const report = schema.parse(evidence.normalized_result);
+ if (
+ evidence.operation !== operation ||
+ !isDeepStrictEqual(evidence.parameters, input) ||
+ !isDeepStrictEqual(report.selection, input)
+ )
+ throw new Error(
+ "Signal Evidence differs from the actual requested operation or choices",
+ );
+ if (
+ operation === "decode_signal_recording"
+ ? evidence.subject?.local_path !== values.path ||
+ evidence.subject?.digest?.sha256 !== inputSha ||
+ report.artifact.path !== values.path ||
+ report.artifact.sha256 !== inputSha
+ : evidence.subject?.local_path !== report.tool.path ||
+ evidence.subject?.digest?.sha256 !== report.tool.sha256
+ )
+ throw new Error(
+ "Signal Evidence identifies different recording/tool bytes",
+ );
+ observations.push({
+ evidence_id: evidence.evidence_id,
+ operation,
+ selection: report.selection,
+ output_window: report.output_window,
+ limitations: evidence.limitations,
+ });
+ return report;
+ };
+ const selection = values["signal-selection"];
+ for (const name of [
+ "decode_signal_recording",
+ ...(selection === "discovery" ? ["inspect_signal_decoders"] : []),
+ ])
+ if (!catalogDefinitions.has(name))
+ throw new Error(`Missing live signal contract ${name}`);
+ if (selection === "discovery") {
+ const input = inspectSignalDecodersInputSchema.parse({
+ decoder: descriptor.kind,
+ query: descriptor.query,
+ limit: 5,
+ });
+ const report = checked(
+ await call("inspect_signal_decoders", input),
+ "inspect_signal_decoders",
+ input,
+ signalDecodersSchema,
+ );
+ if (!report.help_lines.some((line) => line.text.includes(descriptor.query)))
+ throw new Error(
+ "Installed decoder help does not establish the selected annotation class",
+ );
+ }
+ const input = decodeSignalRecordingInputSchema.parse(
+ signalRequest(values.artifact, values.path, selection),
+ );
+ const first = checked(
+ await call("decode_signal_recording", input),
+ "decode_signal_recording",
+ input,
+ signalRecordingSchema,
+ );
+ let rows = signalDataRows(descriptor.kind, first.annotations_text);
+ if (selection === "pages") {
+ const nextInput = decodeSignalRecordingInputSchema.parse({
+ ...input,
+ line_offset: descriptor.firstPage,
+ line_limit: null,
+ });
+ const next = checked(
+ await call("decode_signal_recording", nextInput),
+ "decode_signal_recording",
+ nextInput,
+ signalRecordingSchema,
+ );
+ const repeated = checked(
+ await call("decode_signal_recording", input),
+ "decode_signal_recording",
+ input,
+ signalRecordingSchema,
+ );
+ if (
+ first.annotations_text !== repeated.annotations_text ||
+ !isDeepStrictEqual(first.output_window, repeated.output_window) ||
+ first.output_window.total_lines !== next.output_window.total_lines
+ )
+ throw new Error("Repeated/next display pages changed observation scope");
+ rows = [...rows, ...signalDataRows(descriptor.kind, next.annotations_text)];
+ }
+ const { stdout } = await promisify(execFile)(
+ "python3",
+ [
+ fileURLToPath(new URL("./signal-oracle.py", import.meta.url)),
+ descriptor.kind,
+ values.path,
+ ...(descriptor.kind === "uart" ? ["--invert"] : []),
+ ],
+ { timeout: 30_000, maxBuffer: 256 * 1024 },
+ );
+ const oracle = JSON.parse(stdout);
+ verifySignalOracleIdentity(values.artifact, oracle);
+ verifySignalRows(descriptor.kind, rows, oracle, inputSha);
+ return {
+ artifact_sha256: inputSha,
+ facts: signalFacts(descriptor.kind, rows),
+ delivery_observations: observations,
+ evidence_ids: [...new Set(observations.map((item) => item.evidence_id))],
+ };
+}
diff --git a/experiments/analysis-efficiency/signal-model-prompt.mjs b/experiments/analysis-efficiency/signal-model-prompt.mjs
new file mode 100644
index 000000000..64f87fee1
--- /dev/null
+++ b/experiments/analysis-efficiency/signal-model-prompt.mjs
@@ -0,0 +1,22 @@
+import { parseArgs } from "node:util";
+import { signalRequest, SIGNAL_ARTIFACTS } from "./signal-entry.mjs";
+
+const { values } = parseArgs({
+ options: { artifact: { type: "string" }, selection: { type: "string" } },
+});
+if (
+ !Object.hasOwn(SIGNAL_ARTIFACTS, values.artifact ?? "") ||
+ !["baseline", "focused"].includes(values.selection)
+)
+ throw new Error(
+ "Use --artifact fixed-signal-ID --selection baseline|focused",
+ );
+const kind = SIGNAL_ARTIFACTS[values.artifact].kind;
+const question =
+ kind === "uart"
+ ? "Report every decoded RX data octet on the specified DOUT1 channel, in recording order, as decimal integer bytes. Include all repetitions; do not deduplicate data."
+ : "Report every I2C READ data payload in recording order, grouped into consecutive six-byte payloads. Exclude address bytes and write data. Do not infer button or hardware behavior.";
+console.log(`${question}
+Use the configured REA MCP and make exactly one decode_signal_recording call with this selection:
+${JSON.stringify(signalRequest(values.artifact, "{{artifact_path}}", values.selection))}
+Return one JSON object with exactly these keys: kind (${JSON.stringify(kind)}), ${kind === "uart" ? "bytes (an array of decimal octets)" : "reads (an array of six-octet decimal arrays)"}, evidence_ids (unique actual returned Evidence IDs), limitations (nonempty investigation limitations). Preserve source identity and recorded order. Output lines are not sample or annotation counts; selected decoder observations do not prove complete hardware traffic or original application behavior. Do not run shell commands, browse, acquire hardware samples, invent missing data, or fabricate citations.`);
diff --git a/experiments/analysis-efficiency/signal-oracle.py b/experiments/analysis-efficiency/signal-oracle.py
new file mode 100644
index 000000000..dbccf01a2
--- /dev/null
+++ b/experiments/analysis-efficiency/signal-oracle.py
@@ -0,0 +1,211 @@
+"""Independent readers for the two pinned, single-device digital SR recordings."""
+
+import argparse
+from bisect import bisect_right
+import configparser
+import hashlib
+import io
+import json
+from pathlib import Path
+import re
+import struct
+import zipfile
+
+
+def read_edges(path, channels):
+ size = path.stat().st_size
+ if size > 32 * 1024 * 1024:
+ raise ValueError('Compressed SR input exceeds the experiment budget')
+ with path.open('rb') as source:
+ admitted = source.read(size + 1)
+ if len(admitted) != size:
+ raise ValueError('SR input size changed during admission')
+ marker = admitted.rfind(b'PK\x05\x06', max(0, len(admitted) - 65557))
+ if marker < 0 or len(admitted) - marker < 22:
+ raise ValueError('Missing classic ZIP directory receipt')
+ header = struct.unpack_from('<4s4H2IH', admitted, marker)
+ if (marker >= 20 and admitted[marker-20:marker-16] == b'PK\x06\x07'):
+ raise ValueError('ZIP64 directory is unsupported')
+ if header[1] or header[2] or header[3] != header[4] or not 1 <= header[4] <= 64 or header[5] > 65536 or marker + 22 + header[7] != len(admitted) or header[6] + header[5] != marker:
+ raise ValueError('Unsupported or excessive ZIP directory representation')
+ directory = memoryview(admitted)[header[6]:marker]
+ position = count = 0
+ while position < len(directory):
+ if count >= 64 or len(directory) - position < 46 or directory[position:position+4] != b'PK\x01\x02':
+ raise ValueError('Unsupported or excessive actual ZIP directory records')
+ name_size, extra_size, comment_size = struct.unpack_from('<3H', directory, position + 28)
+ end = position + 46 + name_size + extra_size + comment_size
+ if end > len(directory):
+ raise ValueError('Incomplete actual ZIP directory record')
+ extra = position + 46 + name_size
+ extra_end = extra + extra_size
+ while extra < extra_end:
+ if extra_end - extra < 4:
+ raise ValueError('Incomplete ZIP extra field')
+ tag, length = struct.unpack_from('<2H', directory, extra)
+ if tag == 1:
+ raise ValueError('ZIP64 member is unsupported')
+ extra += 4 + length
+ if extra > extra_end:
+ raise ValueError('Incomplete ZIP extra field')
+ count += 1
+ position = end
+ if count != header[4]:
+ raise ValueError('Declared and actual ZIP directory counts differ')
+ before = hashlib.sha256(admitted).hexdigest()
+ # Admission and ZipFile parse the same bounded snapshot, not a reopened path.
+ with zipfile.ZipFile(io.BytesIO(admitted), allowZip64=False) as archive:
+ entries = archive.infolist()
+ names = [item.filename for item in entries]
+ if len(names) != len(set(names)) or len(names) > 64:
+ raise ValueError("Ambiguous or excessive SR members")
+ if sum(item.file_size for item in entries) > 32 * 1024 * 1024:
+ raise ValueError("Expanded SR input exceeds the experiment budget")
+ if archive.getinfo('metadata').file_size > 8192:
+ raise ValueError("Excessive SR metadata")
+ # Declared sizes are untrusted. A bounded read also caps each inflater
+ # step; logic members below likewise use bounded block reads.
+ with archive.open('metadata') as source:
+ metadata_bytes = source.read(8193)
+ if len(metadata_bytes) > 8192:
+ raise ValueError("Expanded SR metadata exceeds the experiment budget")
+ metadata = metadata_bytes.decode('utf-8')
+ config = configparser.ConfigParser(interpolation=None)
+ config.read_string(metadata)
+ if set(config.sections()) != {'global', 'device 1'}:
+ raise ValueError("Only the pinned single-device profile is supported")
+ device = config['device 1']
+ unit = int(device['unitsize'])
+ probes = int(device['total probes'])
+ if unit not in [1, 2] or not 1 <= probes <= unit * 8:
+ raise ValueError("Unsupported digital sample geometry")
+ rate = re.fullmatch(r'(\d+)\s+(Hz|kHz|MHz)', device['samplerate'])
+ if rate is None or int(rate[1]) <= 0:
+ raise ValueError("Unsupported sample-rate representation")
+ samplerate = int(rate[1]) * {'Hz': 1, 'kHz': 1000, 'MHz': 1000000}[rate[2]]
+ bits = []
+ for channel in channels:
+ matches = [index - 1 for index in range(1, probes + 1)
+ if device.get(f'probe{index}') == channel]
+ if len(matches) != 1:
+ raise ValueError("Channel identity is missing or ambiguous")
+ bits.append(matches[0])
+ prefix = device['capturefile']
+ chunks = [name for name in names if re.fullmatch(re.escape(prefix) + r'-\d+', name)]
+ if prefix in names and chunks:
+ raise ValueError("Ambiguous direct and chunked logic data")
+ if prefix in names:
+ chunks = [prefix]
+ else:
+ chunks.sort(key=lambda name: int(name.rsplit('-', 1)[1]))
+ if [int(name.rsplit('-', 1)[1]) for name in chunks] != list(range(1, len(chunks) + 1)):
+ raise ValueError("Missing or ambiguous logic chunk order")
+ if not chunks or any(archive.getinfo(name).file_size % unit for name in chunks):
+ raise ValueError("Missing or incomplete digital samples")
+ edges = [[] for _ in channels]
+ previous = None
+ sample = 0
+ mask = sum(1 << bit for bit in bits)
+ for name in chunks:
+ with archive.open(name) as source:
+ while block := source.read(65536):
+ values = block if unit == 1 else (item[0] for item in struct.iter_unpack('<H', block))
+ for value in values:
+ selected = value & mask
+ if selected != previous:
+ for index, bit in enumerate(bits):
+ level = (selected >> bit) & 1
+ if not edges[index] or edges[index][-1][1] != level:
+ if sum(map(len, edges)) >= 65536:
+ raise ValueError('Selected transitions exceed the experiment representation budget')
+ edges[index].append((sample, level))
+ previous = selected
+ sample += 1
+ with path.open('rb') as source:
+ after = source.read(size + 1)
+ if len(after) != size or hashlib.sha256(after).hexdigest() != before:
+ raise ValueError("Recording bytes changed during the independent read")
+ return {'sha256': before, 'metadata': metadata, 'sample_rate_hz': samplerate,
+ 'total_samples': sample, 'channels': dict(zip(channels, bits))}, edges
+
+
+def uart(path, invert):
+ metadata, traces = read_edges(path, ['MAX3232E DOUT1'])
+ edges = [(sample, level ^ invert) for sample, level in traces[0]]
+ positions = [sample for sample, _ in edges]
+ rate, baud = metadata['sample_rate_hz'], 57600
+ def level(sample):
+ return edges[bisect_right(positions, sample) - 1][1]
+ rows = []
+ busy_until = 0
+ for index, (start, value) in enumerate(edges):
+ if index == 0 or value != 0 or start < busy_until:
+ continue
+ center = lambda halfbits: start + (halfbits * rate) // (2 * baud)
+ if center(19) >= metadata['total_samples']:
+ raise ValueError("Incomplete UART frame at the capture boundary")
+ if level(center(1)) != 0 or level(center(19)) != 1:
+ raise ValueError(f"UART framing differs from selected polarity at sample {start}")
+ byte = sum(level(center(3 + 2 * bit)) << bit for bit in range(8))
+ # A UART receiver becomes ready after sampling the stop bit. Waiting
+ # a full nominal ten bits would miss a valid slightly faster sender.
+ rows.append({'start': start, 'end_exclusive': center(19) + 1, 'byte': byte,
+ 'first_data_sample': center(3), 'last_data_sample': center(17)})
+ busy_until = center(19) + 1
+ return metadata | {'kind': 'uart', 'baud_rate': baud, 'invert': bool(invert),
+ 'complete_frames': len(rows), 'rows': rows}
+
+
+def i2c(path):
+ metadata, traces = read_edges(path, ['SCL', 'SDA'])
+ transitions = {}
+ for channel, edges in enumerate(traces):
+ for sample, value in edges:
+ transitions.setdefault(sample, {})[channel] = value
+ clock = data = None
+ active = None
+ bits = []
+ transactions = []
+ for sample, changes in sorted(transitions.items()):
+ next_clock, next_data = changes.get(0, clock), changes.get(1, data)
+ if clock is not None and data is not None:
+ if clock == next_clock == 1 and next_data != data:
+ if next_data == 0:
+ if active is not None:
+ raise ValueError("The pinned profile does not contain repeated START")
+ active = {'start': sample, 'bytes': []}
+ bits = []
+ elif active is not None:
+ # SCL may rise with SDA low to prepare STOP; that final
+ # rise is not the first bit of a new data byte.
+ if len(bits) == 1 and bits[0][1] == 0:
+ bits = []
+ if bits:
+ raise ValueError("Incomplete I2C byte before STOP")
+ active['stop'] = sample
+ transactions.append(active)
+ active = None
+ elif active is not None and clock == 0 and next_clock == 1:
+ if next_data != data:
+ raise ValueError("Ambiguous simultaneous clock/data transition")
+ bits.append((sample, next_data))
+ if len(bits) == 9:
+ byte = sum(bit << (7-index) for index, (_, bit) in enumerate(bits[:8]))
+ active['bytes'].append({'byte': byte, 'first_clock': bits[0][0],
+ 'last_clock': bits[7][0], 'ack': bits[8][1] == 0})
+ bits = []
+ clock, data = next_clock, next_data
+ if active is not None or bits:
+ raise ValueError("Incomplete I2C transaction at capture end")
+ reads = [transaction for transaction in transactions
+ if transaction['bytes'] and transaction['bytes'][0]['byte'] & 1]
+ return metadata | {'kind': 'i2c', 'transactions': transactions, 'reads': reads}
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument('kind', choices=['uart', 'i2c'])
+ parser.add_argument('path', type=Path)
+ parser.add_argument('--invert', action='store_true')
+ args = parser.parse_args()
+ print(json.dumps(uart(args.path, args.invert) if args.kind == 'uart' else i2c(args.path)))
diff --git a/experiments/analysis-efficiency/signal-selection-workflows.md b/experiments/analysis-efficiency/signal-selection-workflows.md
new file mode 100644
index 000000000..4cc216827
--- /dev/null
+++ b/experiments/analysis-efficiency/signal-selection-workflows.md
@@ -0,0 +1,188 @@
+# Sigrok selection and repeated-page checkpoint (#3131)
+
+This experiment preserves production APIs, defaults, caller selections and Evidence
+semantics. Existing annotation selectors substantially reduce replies, but this
+small real-model cohort does **not** establish a consistent reduction in total
+input tokens or elapsed task time. No decoder cache or new production preset is
+introduced. Complete necessary output stays available through existing explicit
+null limits; original defaults remain 50 display lines.
+
+## Immutable corpus and independent questions
+
+The runtime is pinned to `ca7450beffff59325ce3bddc78e181693ce77177`, with Node
+24.18.0, SDK 2.3.1 and Linux x86-64. Experiment driver code for model trials is
+`23317a58d`; each SDK result separately records workflow/oracle source hashes,
+runtime-file and lockfile digests, explicit provider settings and original bytes.
+Later integration checks must remain separate from this cohort.
+
+Two official sigrok-dumps recordings are pinned to upstream commit
+`0ad13477abc959d37fc9a5acbd23901c371c9c76` in [corpus.json](corpus.json).
+Original Git blob IDs, SHA-256, sizes and immutable reference/download URLs are
+recorded there. No sample binaries or raw output are committed. The upstream
+repository declares dumps public domain unless otherwise noted; the inspected
+sample READMEs state no exception. Decoder software licensing is separate.
+
+- MAX3232E UART: 12,742 compressed bytes, 10,000,000 logic bytes, 5,000,000 samples
+ at the recorded nominal 100 MHz. Stored DOUT1 is bit 1; DIN1 is bit 2. The exact
+ question requests all RX octets in order, including repetitions. Independent
+ 57,600-baud 8N1 stop-bit sampling with DOUT1 inversion yields 65 bytes: five
+ copies of `Hello world\r\n`. The source metadata and physical probe numbers
+ remain distinct from stored bit positions.
+- Wii Nunchuk I2C: 20,373 compressed bytes, 20,000,000 samples at nominal 1 MHz.
+ The source README's physical probes 3/7 are repacked as stored named channels
+ SCL/SDA at bits 0/1. The question asks every read payload as consecutive
+ six-byte arrays, excluding address/write bytes and hardware/button inference.
+ Independent START/STOP, rising-clock, byte and ACK/NACK reading identifies seven
+ transactions and three read payloads, matching the upstream reference:
+ `75 7F 77 4F 82 3B`, `75 7F 75 44 82 34`, `75 7F 77 43 83 5D`.
+
+The independent reader accepts the scoped single-device classic-ZIP profile,
+checks directory representation before object allocation, bounds compressed and
+expanded input to 32 MiB, and limits selected transitions to 65,536. It preserves
+source metadata/digest, chunk ordering and channel identity; incomplete samples,
+ambiguous names, partial bytes and unsupported transactions fail explicitly.
+Receiver rearming follows the verified UART stop sample, accommodating the
+recording's slightly faster sender. I2C STOP preparation is distinct from data.
+These checks do not establish other formats, decoders, OSes or physical acquisition.
+
+An initial DOUT1 decode without inversion exited successfully but produced wrong
+bytes. Its cost/output is retained privately as configuration exploration, outside
+eligible measurements. An independent sample oracle prevents equal-but-wrong
+producer results from passing. Correct inversion was established before the cohort.
+
+## Actual CLI and SDK evidence
+
+sigrok-cli 0.7.2 reports libsigrok 0.5.2 and libsigrokdecode 0.5.3. Its executable
+SHA-256 is `e1b1eb2f25edc17d24a78acbab16ff7e1dfdee74bb2cccb7ad00497ccc7a0c44`.
+Selected installed UART/I2C decoder module bytes are fingerprinted privately.
+The banner and executable digest do not fingerprint the complete installation;
+modules and libraries remain trusted. Both actual CLI and MCP results match raw
+sample bytes; data intervals correlate with the independently examined samples.
+
+Fourteen actual SDK workflows comprise two interleaved repetitions per recording
+and arm (baseline/focused/discovery), plus first/next/repeated pages. All 22 calls
+preserve source bytes and necessary decoded facts. Baseline returns all necessary
+annotation output. Focused selects `uart=rx-data` or `i2c=data-read`, with explicit
+null line limits because the task needs the entire selected payload. Discovery
+first queries the installed decoder help for the already specified class; it does
+not establish discovery benefits for unknown protocols.
+
+| Recording | Baseline bytes | Focused bytes | Discovery + focused bytes | First + next + repeat bytes |
+| --------- | -------------: | ------------: | ------------------------: | --------------------------: |
+| UART | 49,417 | 12,441 | 18,839 | 31,779 |
+| I2C | 26,829 | 9,928 | 16,274 | 27,623 |
+
+Focused reply reductions are 74.8% and 63.0%. These are parsed SDK JSON bytes,
+not wire bytes or model tokens. Baseline/focused RPC times (ms, repetitions 1/2)
+are UART 2010/582 versus 1459/911, and I2C 2851/676 versus 2330/685; n=2,
+with order/cold/warm variation, does not establish a general speedup. Pages retain
+input/class/window coverage separately, rather than equating display exhaustion
+with a complete protocol decode. Repeated first pages match their original text
+and scope; reused content-based Evidence identities are valid.
+
+## Complete real-model task costs
+
+Eight actual Codex CLI 0.161.0 tasks requested `gpt-5.6-terra` / `high`, in isolated
+profiles using this runtime's REA MCP. Both tools were enabled and preapproved
+identically in all arms; shell, browser and extra agents were disabled. The model
+alias is a requested identifier, not independently resolved backend identity.
+Questions contain no expected payloads. Trial order is baseline 1, focused 1,
+focused 2, baseline 2, with UART then I2C in each step.
+
+| Trial | Total input | Cached input | Uncached input (derived) | Output | Seconds | Completed tool calls | Facts / fixed plan |
+| --------------- | ----------: | -----------: | -----------------------: | -----: | ------: | -------------------: | ------------------ |
+| uart baseline 1 | 92,399 | 49,408 | 42,991 | 1,295 | 86.411 | 1 | passed / passed |
+| uart focused 1 | 130,029 | 98,048 | 31,981 | 780 | 49.623 | 1 | passed / passed |
+| uart focused 2 | 98,894 | 78,592 | 20,302 | 878 | 73.861 | 1 | passed / passed |
+| uart baseline 2 | 90,395 | 48,384 | 42,011 | 926 | 43.974 | 1 | passed / passed |
+| i2c baseline 1 | 172,162 | 121,344 | 50,818 | 1,194 | 36.330 | 2 | passed / failed |
+| i2c focused 1 | 74,815 | 49,408 | 25,407 | 937 | 114.515 | 1 | passed / passed |
+| i2c focused 2 | 74,684 | 48,384 | 26,300 | 668 | 73.877 | 1 | passed / passed |
+| i2c baseline 2 | 86,241 | 49,408 | 36,833 | 778 | 41.084 | 1 | passed / passed |
+
+All eight answers match independent raw samples and cite authenticated decoder
+observations. I2C baseline 1 also queried generic MCP resources before its single
+required decode. The predeclared strict single-call grader marks its plan failed;
+its actual 172,162 input tokens and 36.330 seconds remain recorded, separately
+from the eligible fixed-plan pairs. It is not retried into a better result.
+
+UART eligible pairs increase total input by 40.7% and 9.4%; elapsed changes are
+-42.6% and +68.0%. The one eligible I2C pair reduces total input by 13.4% but
+increases elapsed time by 79.8%. Reduced uncached input does not replace total
+input as the primary token measure. No reliable whole-task token/time benefit is
+established across these two questions; reply projection alone is insufficient.
+
+## Producer, window and resource scope
+
+Eight separately bounded direct producer runs preserve source bytes and render
+correct stdout: UART baseline 159/131 ms versus focused 163/130 ms; I2C baseline
+355/298 ms versus focused 317/290 ms. This isolates the producer process from REA
+RPC, version probing and snapshot work, but startup/decoding/annotation formatting
+remain combined. Decoding-only versus formatting-only time is unmeasured.
+
+REA literal-window extraction and JSON encoding were measured separately from
+retained producer text (one cold observation and 200 warm iterations per arm).
+Warm mean extraction ranges from 0.0040 to 0.1794 ms; JSON encoding from 0.0021 to 0.0121 ms. These exclude file reads, RPC and decoding.
+
+Every page still invokes decoding. Retained output might help repeated inspection,
+but this fixed cohort does not demonstrate a completed-task improvement warranting
+a new cache. Any future work should own bounded output explicitly by original
+Evidence identity and lifetime, with cancellation/cleanup, rather than use a
+CLI-digest-only generic cache of an incompletely fingerprinted installation.
+
+Heavy work ran serially under one CPU, 3 GiB early stop / 4 GiB hard memory,
+no swap and host-availability checks. Verified payload-exit peak bytes were
+SDK 587,513,856; model 889,323,520; direct producer 26,288,128. No OOM/OOM-kill or resource abort occurred in that verified phase; task descendants
+were cleaned. Terminal whole-scope counters are unavailable and remain unknown.
+
+## Verification and replay
+
+Eleven independent-oracle regressions and 12 comparator/sample-interval tests pass.
+Twenty-four private replays using actual SDK Evidence cover valid answers, failed
+calls before/after recovery, wrong bytes, unknown citations and missing limitations.
+These are grading checks with injected state, not additional model/provider trials.
+The grader retains authenticated observations by identity, rejects conflicts,
+combines cited pages in sample order and keeps failed calls/plan deviations separate
+from supported facts. Relevant typecheck/lint and pre-push checks pass.
+
+Review hardening ties the oracle to the named corpus digest, bounds each I2C
+interval against neighboring byte clocks/transaction STOP, and counts actual
+classic-ZIP directory records on the same bounded snapshot before allocating
+`ZipFile` objects; ZIP64 is rejected. Forged-count and ZIP64 admission regressions
+fail on the original implementation. Replaying all eight original model trials
+and 24 injected grading cases with these checks preserves their factual/plan
+grades and recorded usage/time; these replays are not new model measurements.
+
+Metadata reads cap inflater output even when ZIP sizes are forged. A streamed
+8 MiB malformed metadata member declaring 8 KiB is rejected with traced Python
+allocation below 1 MiB; the same regression fails on the preceding implementation.
+Compressed snapshots use actual file size plus one growth-detection byte rather
+than preallocating the entire admission budget. Both genuine waveform oracles
+retain identical bytes, metadata, sample intervals and source digests.
+
+UART interval checks also require coverage of the first and last independently
+sampled data-bit centers within the owning frame. Shortened, shifted and
+single-sample intervals are rejected even when the decoded byte is correct.
+The added oracle coordinates retain the original byte/frame facts; replaying
+saved real Evidence and all eight model transcripts preserves the original
+grades and costs. This is stronger retrospective verification, not a new cohort.
+
+After rebasing onto main `44d98de2367b99808d78db732ec7d72533266233`,
+`npm run build:cached` and the 16 focused comparator/sample-interval/MCP-boundary
+tests passed. Two additional real focused MCP workflows independently verified
+UART/I2C bytes and intervals on that base. These integration checks are separate
+from the original cost cohort and do not replace or extend its measurements.
+
+A subsequent rebase onto main `3b79af28e80785afce9f5998d8f03289402a1847`
+passed the 17 focused tests, including the upstream-fixed Hopper doctor case,
+and two additional real UART/I2C MCP workflows. Original cohort costs and grades
+remain unchanged; these checks verify integration on the updated main only.
+
+Use `measure.mjs --signal-selection baseline|focused|discovery|pages`, preserving
+runtime/source fingerprints and explicit provider JSON. Generate the exact task
+with `signal-model-prompt.mjs --artifact ID --selection baseline|focused`, run the
+isolated `agent-trial.py --mode direct --capture-tool-context`, and grade with
+`review-signal.mjs` and the independently produced oracle. Keep original failures,
+resources and raw observations private. For migration, existing optional annotation
+selection is sufficient for scoped reply reduction; no default/API change follows
+from this mixed whole-task result.
diff --git a/experiments/analysis-efficiency/test_signal_oracle.py b/experiments/analysis-efficiency/test_signal_oracle.py
new file mode 100644
index 000000000..9decd9a18
--- /dev/null
+++ b/experiments/analysis-efficiency/test_signal_oracle.py
@@ -0,0 +1,149 @@
+"""Distinct framing, identity and admission failures in the sample-bit oracle."""
+
+import importlib.util
+from pathlib import Path
+import struct
+import tempfile
+import tracemalloc
+import unittest
+from unittest.mock import patch
+import zipfile
+
+spec = importlib.util.spec_from_file_location(
+ 'signal_oracle', Path(__file__).with_name('signal-oracle.py'))
+oracle = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(oracle)
+
+
+class SignalOracleTests(unittest.TestCase):
+ def setUp(self):
+ self.root = tempfile.TemporaryDirectory()
+ self.addCleanup(self.root.cleanup)
+ self.path = Path(self.root.name) / 'sample.sr'
+
+ def recording(self, data, unit, channels, chunk='logic-1', extra=0):
+ metadata = '[global]\nsigrok version=0.3.0\n[device 1]\ncapturefile=logic-1\n'
+ metadata += f'total probes={len(channels)}\nsamplerate=1 MHz\nunitsize={unit}\n'
+ metadata += ''.join(f'probe{index+1}={name}\n' for index, name in enumerate(channels))
+ with zipfile.ZipFile(self.path, 'w') as archive:
+ archive.writestr('version', '2')
+ archive.writestr('metadata', metadata)
+ archive.writestr(chunk, data)
+ for index in range(extra):
+ archive.writestr(f'extra-{index}', '')
+
+ def test_uart_rearms_after_stop_sample_for_a_slightly_faster_sender(self):
+ levels = [1] * 100
+ for byte in [0x48, 0x65]:
+ for bit in [0, *[(byte >> index) & 1 for index in range(8)], 1]:
+ levels.extend([bit] * 17)
+ levels.extend([1] * 100)
+ data = b''.join(struct.pack('<H', (1-level) << 1) for level in levels)
+ self.recording(data, 2, ['unused', 'MAX3232E DOUT1'])
+ result = oracle.uart(self.path, True)
+ self.assertEqual([row['byte'] for row in result['rows']], [0x48, 0x65])
+ self.assertEqual(result['channels']['MAX3232E DOUT1'], 1)
+ for row, start in zip(result['rows'], [100, 270]):
+ self.assertEqual(row['first_data_sample'], start + 26)
+ self.assertEqual(row['last_data_sample'], start + 147)
+
+ def i2c_levels(self, incomplete=False):
+ states = [(1, 1), (1, 0)]
+ for byte, ack in [(0xA5, 0), (0x75, 1)]:
+ values = [(byte >> index) & 1 for index in reversed(range(8))] + [ack]
+ if incomplete and byte == 0x75:
+ values = values[:4]
+ for bit in values:
+ states.extend([(0, bit), (1, bit)])
+ states.extend([(0, 0), (1, 0), (1, 1)])
+ return bytes(clock | data << 1 for clock, data in states)
+
+ def test_i2c_stop_setup_is_distinct_from_another_data_byte(self):
+ self.recording(self.i2c_levels(), 1, ['SCL', 'SDA'])
+ result = oracle.i2c(self.path)
+ self.assertEqual(len(result['reads']), 1)
+ self.assertEqual([row['byte'] for row in result['reads'][0]['bytes']], [0xA5, 0x75])
+ self.assertFalse(result['reads'][0]['bytes'][-1]['ack'])
+
+ def test_incomplete_i2c_byte_is_not_invented_at_stop(self):
+ self.recording(self.i2c_levels(incomplete=True), 1, ['SCL', 'SDA'])
+ with self.assertRaisesRegex(ValueError, 'Incomplete I2C byte'):
+ oracle.i2c(self.path)
+
+ def test_channel_names_must_identify_one_stored_bit(self):
+ self.recording(b'\x00', 1, ['SCL', 'SCL'])
+ with self.assertRaisesRegex(ValueError, 'ambiguous'):
+ oracle.read_edges(self.path, ['SCL'])
+
+ def test_missing_chunk_is_not_renumbered(self):
+ self.recording(b'\x00', 1, ['SCL', 'SDA'], chunk='logic-1-2')
+ with self.assertRaisesRegex(ValueError, 'chunk order'):
+ oracle.read_edges(self.path, ['SCL', 'SDA'])
+
+ def test_partial_sample_word_is_rejected(self):
+ self.recording(b'\x00', 2, ['unused', 'MAX3232E DOUT1'])
+ with self.assertRaisesRegex(ValueError, 'incomplete digital samples'):
+ oracle.read_edges(self.path, ['MAX3232E DOUT1'])
+
+ def test_directory_budget_precedes_zip_object_allocation(self):
+ self.recording(b'\x00', 1, ['SCL', 'SDA'], extra=62)
+ with patch.object(oracle.zipfile, 'ZipFile', side_effect=AssertionError('Too late')):
+ with self.assertRaisesRegex(ValueError, 'excessive ZIP directory'):
+ oracle.read_edges(self.path, ['SCL', 'SDA'])
+
+ def test_declared_expansion_is_rejected_before_decompression(self):
+ self.recording(b'\x00', 1, ['SCL', 'SDA'])
+ data = bytearray(self.path.read_bytes())
+ directory = data.index(b'PK\x01\x02')
+ struct.pack_into('<I', data, directory + 24, 32 * 1024 * 1024 + 1)
+ self.path.write_bytes(data)
+ with self.assertRaisesRegex(ValueError, 'Expanded SR input'):
+ oracle.read_edges(self.path, ['SCL', 'SDA'])
+
+ def test_forged_member_count_cannot_allocate_extra_zip_objects(self):
+ self.recording(b'\x00', 1, ['SCL', 'SDA'], extra=62)
+ data = bytearray(self.path.read_bytes())
+ end = data.rfind(b'PK\x05\x06')
+ struct.pack_into('<2H', data, end + 8, 1, 1)
+ self.path.write_bytes(data)
+ with patch.object(oracle.zipfile, 'ZipFile', side_effect=AssertionError('Too late')):
+ with self.assertRaisesRegex(ValueError, 'actual ZIP directory'):
+ oracle.read_edges(self.path, ['SCL', 'SDA'])
+
+ def test_zip64_locator_cannot_override_the_classic_budget(self):
+ self.recording(b'\x00', 1, ['SCL', 'SDA'])
+ data = bytearray(self.path.read_bytes())
+ end = data.rfind(b'PK\x05\x06')
+ # Syntactically complete ZIP64 directory/locator before a classic EOCD.
+ header = struct.unpack_from('<4s4H2IH', data, end)
+ record = struct.pack('<4sQ2H2I4Q', b'PK\x06\x06', 44, 45, 45, 0, 0,
+ header[4], header[4], 100000000, header[6])
+ locator = struct.pack('<4sIQI', b'PK\x06\x07', 0, end, 1)
+ data[end:end] = record + locator
+ self.path.write_bytes(data)
+ with patch.object(oracle.zipfile, 'ZipFile', side_effect=AssertionError('Too late')):
+ with self.assertRaisesRegex(ValueError, 'ZIP64 directory'):
+ oracle.read_edges(self.path, ['SCL', 'SDA'])
+
+ def test_forged_metadata_size_cannot_expand_past_its_output_budget(self):
+ # Stream the malformed member so constructing the fixture is bounded.
+ with zipfile.ZipFile(self.path, 'w', compression=zipfile.ZIP_DEFLATED) as archive:
+ with archive.open('metadata', 'w') as member:
+ for _ in range(128):
+ member.write(b'X' * 65536)
+ data = bytearray(self.path.read_bytes())
+ directory = data.index(b'PK\x01\x02')
+ struct.pack_into('<I', data, directory + 24, 8192)
+ self.path.write_bytes(data)
+ tracemalloc.start()
+ try:
+ with self.assertRaises(zipfile.BadZipFile):
+ oracle.read_edges(self.path, ['SCL', 'SDA'])
+ _, peak = tracemalloc.get_traced_memory()
+ finally:
+ tracemalloc.stop()
+ self.assertLess(peak, 1024 * 1024, 'Metadata allocation followed forged ZIP expansion')
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tests/evaluation/analysisEfficiencyComparison.test.ts b/tests/evaluation/analysisEfficiencyComparison.test.ts
index 924ba1612..94af61d3e 100644
--- a/tests/evaluation/analysisEfficiencyComparison.test.ts
+++ b/tests/evaluation/analysisEfficiencyComparison.test.ts
@@ -31,6 +31,7 @@ type Fixture = {
provider_search_path_sha256: string;
calls: unknown[];
packet_selection?: string;
+ signal_selection?: string;
};
answer: {
capture: { coverage: string; capture_start: null };
@@ -38,8 +39,9 @@ type Fixture = {
delivery_observations: Observation[];
facts?: {
kind: string;
- matching_packets: number;
- rows: { frame: number }[];
+ matching_packets?: number;
+ rows?: { frame: number }[];
+ bytes?: number[];
};
};
};
@@ -154,14 +156,38 @@ for (const artifact of [
equivalent.report.preserved_run_specific_values.map((row) => row.value),
[left.answer.delivery_observations, right.answer.delivery_observations],
);
- right.answer.facts!.matching_packets++;
+ right.answer.facts!.matching_packets = 1338;
assert.equal(compare(left, right).report.facts_match, false);
- right.answer.facts!.matching_packets--;
+ right.answer.facts!.matching_packets = 1337;
right.summary.packet_selection = "baseline";
assert.equal(compare(left, right).report.facts_match, false);
});
}
+for (const artifact of ["sigrok-uart-max3232e.sr", "sigrok-i2c-nunchuk.sr"]) {
+ test(`${artifact}: compare decoded facts across signal plans and preserve display scope`, () => {
+ const left = fixture("complete", artifact);
+ left.summary.signal_selection = "baseline";
+ left.answer.facts = { kind: "signal", bytes: [72, 101] };
+ const right = structuredClone(left);
+ right.summary.signal_selection = "focused";
+ right.answer.delivery_observations[0]!.coverage =
+ "selected annotation lines";
+ const equivalent = compare(left, right);
+ assert.equal(equivalent.status, 0);
+ assert.equal(equivalent.report.facts_match, true);
+ assert.deepEqual(
+ equivalent.report.preserved_run_specific_values.map((row) => row.value),
+ [left.answer.delivery_observations, right.answer.delivery_observations],
+ );
+ right.answer.facts!.bytes![0] = 73;
+ assert.equal(compare(left, right).report.facts_match, false);
+ right.answer.facts!.bytes![0] = 72;
+ right.summary.signal_selection = "baseline";
+ assert.equal(compare(left, right).report.facts_match, false);
+ });
+}
+
for (const artifact of [
"wireshark-http.cap",
"wireshark-http_redirects.pcapng",
diff --git a/tests/evaluation/signalWorkflowSemantics.test.ts b/tests/evaluation/signalWorkflowSemantics.test.ts
new file mode 100644
index 000000000..7877c494e
--- /dev/null
+++ b/tests/evaluation/signalWorkflowSemantics.test.ts
@@ -0,0 +1,46 @@
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { test } from "vitest";
+
+test("signal verification rejects skipped identical UART bytes and reordered I2C samples", () => {
+ const moduleUrl = new URL(
+ "../../experiments/analysis-efficiency/signal-entry.mjs",
+ import.meta.url,
+ ).href;
+ const source = `
+ import assert from 'node:assert/strict';
+ import {verifySignalRows,signalDataRows,verifySignalOracleIdentity} from ${JSON.stringify(moduleUrl)};
+ const sha256='a'.repeat(64);
+ const rows=Array.from({length:8},(_,i)=>({start:i*100+10,end:i*100+80,byte:72}));
+ const uart={kind:'uart',sha256,total_samples:2000,rows:rows.map((_,i)=>({start:i*100,end_exclusive:i*100+90,first_data_sample:i*100+15,last_data_sample:i*100+75,byte:72}))};
+ verifySignalRows('uart',rows,uart,sha256);
+ verifySignalRows('uart',rows.map((row,i)=>({...row,start:i*100+15,end:i*100+75})),uart,sha256);
+ assert.throws(()=>verifySignalRows('uart',[{...rows[0],start:0,end:0},...rows.slice(1)],uart,sha256),/interval/);
+ assert.throws(()=>verifySignalRows('uart',[{...rows[0],end:74},...rows.slice(1)],uart,sha256),/interval/);
+ assert.throws(()=>verifySignalRows('uart',[{...rows[0],start:16},...rows.slice(1)],uart,sha256),/interval/);
+ assert.throws(()=>verifySignalRows('uart',rows,{...uart,rows:uart.rows.map(({first_data_sample,...row})=>row)},sha256),/interval/);
+ assert.throws(()=>verifySignalRows('uart',rows.slice(1).concat({start:810,end:880,byte:72}),uart,sha256),/interval/);
+ assert.throws(()=>verifySignalRows('uart',rows.slice(1),uart,sha256),/coverage/);
+ assert.throws(()=>verifySignalRows('uart',rows,uart,'b'.repeat(64)),/identity/);
+ const data=Array.from({length:18},(_,i)=>({start:i*100+10,end:i*100+90,byte:117}));
+ const raw=data.map((_,i)=>({byte:117,first_clock:i*100+20,last_clock:i*100+80}));
+ const i2c={kind:'i2c',sha256,total_samples:2000,reads:[{bytes:[{byte:165,last_clock:0},...raw],stop:1900}]};
+ verifySignalRows('i2c',data,i2c,sha256);
+ assert.throws(()=>verifySignalRows('i2c',data.toReversed(),i2c,sha256),/interval/);
+ assert.throws(()=>verifySignalRows('i2c',[{...data[0],byte:118},...data.slice(1)],i2c,sha256),/sample bits/);
+ assert.throws(()=>verifySignalRows('i2c',data.map((row,i)=>({...row,start:i,end:1999})),i2c,sha256),/interval/);
+ assert.throws(()=>verifySignalRows('i2c',[{...data[0],start:0},...data.slice(1)],i2c,sha256),/interval/);
+ assert.throws(()=>verifySignalRows('i2c',[...data.slice(0,-1),{...data.at(-1),end:1900}],i2c,sha256),/interval/);
+ verifySignalOracleIdentity('sigrok-uart-max3232e.sr',{kind:'uart',sha256:'f797d60d6b8e12d2a1549b9a5a499ef389f44cb9e870e8610af26b57c3b2be76'});
+ assert.throws(()=>verifySignalOracleIdentity('sigrok-uart-max3232e.sr',uart),/pinned signal corpus/);
+ assert.throws(()=>verifySignalOracleIdentity('sigrok-i2c-nunchuk.sr',{kind:'uart',sha256:'0fa6adb8297d7f3fe5ba19c94ee655c9b0f952a8b5e48a17aaa499a2f6d10d70'}),/pinned signal corpus/);
+ assert.throws(()=>signalDataRows('uart','9007199254740993-9007199254740994 uart-1: 48'),/sample interval/);
+ `;
+ // Checker semantics only; synthesized rows do not establish provider behavior.
+ const result = spawnSync(process.execPath, ["--input-type=module", "-"], {
+ input: source,
+ encoding: "utf8",
+ });
+ assert.ifError(result.error);
+ assert.equal(result.status, 0, result.stderr);
+});