Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
test(perf): measure real sigrok annotation selection workflows
Ysopwnableca1b2f8608c5
diff --git a/experiments/analysis-efficiency/README.md b/experiments/analysis-efficiency/README.mdindex 9a14b764a..2ca9da3a4 100644--- a/experiments/analysis-efficiency/README.md+++ b/experiments/analysis-efficiency/README.md@@ -78,6 +78,12 @@ field/filter selection, discovery and output windows. Lower delivered JSON didnot establish stable total-token or speed gains; failed plans and negativecontrols remain visible.+The [sigrok selection checkpoint](signal-selection-workflows.md) records two+real hardware recordings, independently verified bytes, repeated decoder/pages,+and an eight-task Codex cohort with mixed token/time results. Existing annotation+controls reduce replies; these measurements introduce no production cache,+default or API changes.+Agent token consumption and end-to-end task time are the primary objectives.Memory and CPU consumption are also measured and controlled. Correctness,Evidence provenance, caller choices and existing-user compatibility arediff --git a/experiments/analysis-efficiency/agent-trial.py b/experiments/analysis-efficiency/agent-trial.pyindex db994cea1..365337a06 100644--- a/experiments/analysis-efficiency/agent-trial.py+++ b/experiments/analysis-efficiency/agent-trial.py@@ -182,6 +182,7 @@ def main():'inspect_firmware_regions', 'inspect_managed_artifact','inspect_sqlite_database', 'inspect_pcap_capture','inspect_packet_capture', 'inspect_packet_fields',+ 'decode_signal_recording', 'inspect_signal_decoders','inspect_managed_members', 'inspect_managed_native_boundaries','inspect_artifact', 'inspect_wasm_artifact','inspect_evm_interface', 'get_evidence_bundle', 'export_evidence_bundle',diff --git a/experiments/analysis-efficiency/compare.mjs b/experiments/analysis-efficiency/compare.mjsindex 01b9f97af..e1e9f4cda 100644--- a/experiments/analysis-efficiency/compare.mjs+++ b/experiments/analysis-efficiency/compare.mjs@@ -3,6 +3,7 @@ import { readFile, writeFile } from "node:fs/promises";import { resolve } from "node:path";import { parseArgs } from "node:util";import { PACKET_ARTIFACTS } from "./packet-entry.mjs";+import { SIGNAL_ARTIFACTS } from "./signal-entry.mjs";const { values } = parseArgs({options: {@@ -36,6 +37,8 @@ const sameSettings = ["native_workflow_source_sha256","packet_workflow_source_sha256","packet_oracle_source_sha256",+ "signal_workflow_source_sha256",+ "signal_oracle_source_sha256","package_lock_sha256","node_version","sdk_client_version",@@ -73,13 +76,14 @@ for (const [side, answer] of [value: answer.evidence_ids,});}-// PCAP delivery observations preserve each actual receipt's identity, limitations+// Recording delivery observations preserve each actual receipt's identity, limitations// and view coverage. Summary/page scopes differ from complete source scope;// retain them separately rather than equating page exhaustion with capture// coverage. Repeated equivalent deliveries must also match every observation// field except the receipt and parent IDs, which identify their own invocations.if (Object.hasOwn(PACKET_ARTIFACTS, left.artifact) ||+ Object.hasOwn(SIGNAL_ARTIFACTS, left.artifact) ||["wireshark-http.cap", "wireshark-http_redirects.pcapng"].includes(left.artifact,)@@ -92,7 +96,7 @@ if (!Array.isArray(answer.delivery_observations) ||!answer.delivery_observations.length)- throw new Error("Missing original PCAP delivery observations");+ throw new Error("Missing original recording delivery observations");runSpecific.push({side,path: ["delivery_observations"],@@ -101,7 +105,9 @@ if (if (left.delivery === right.delivery &&(!Object.hasOwn(PACKET_ARTIFACTS, left.artifact) ||- left.packet_selection === right.packet_selection)+ left.packet_selection === right.packet_selection) &&+ (!Object.hasOwn(SIGNAL_ARTIFACTS, left.artifact) ||+ left.signal_selection === right.signal_selection)) {answer.delivery_observations = answer.delivery_observations.map((observation) =>diff --git a/experiments/analysis-efficiency/corpus.json b/experiments/analysis-efficiency/corpus.jsonindex e595dde34..983b81799 100644--- a/experiments/analysis-efficiency/corpus.json+++ b/experiments/analysis-efficiency/corpus.json@@ -343,6 +343,36 @@"derivation": "Unmodified official saved sample; decompressed gzip where indicated. Local digest pins acquired bytes, not an upstream-published digest.","compressed_source": null}+ },+ {+ "id": "sigrok-uart-max3232e.sr",+ "file": "uart-max3232e.sr",+ "bytes": 12742,+ "sha256": "f797d60d6b8e12d2a1549b9a5a499ef389f44cb9e870e8610af26b57c3b2be76",+ "provenance": {+ "project": "sigrok-dumps",+ "version": "0ad13477abc959d37fc9a5acbd23901c371c9c76",+ "upstreamFile": "uart/maxim_max3232e/max3232e_hello_world_57600_8n1.sr",+ "url": "https://raw.githubusercontent.com/sigrokproject/sigrok-dumps/0ad13477abc959d37fc9a5acbd23901c371c9c76/uart/maxim_max3232e/max3232e_hello_world_57600_8n1.sr",+ "git_blob_sha1": "52e47b3014819471a2f348a0d8c7694fe304e052",+ "reference": "https://raw.githubusercontent.com/sigrokproject/sigrok-dumps/0ad13477abc959d37fc9a5acbd23901c371c9c76/uart/maxim_max3232e/README",+ "license_scope": "Repository README declares dumps public domain unless otherwise noted; selected READMEs state no exception; decoder software licensing is separate."+ }+ },+ {+ "id": "sigrok-i2c-nunchuk.sr",+ "file": "i2c-nunchuk.sr",+ "bytes": 20373,+ "sha256": "0fa6adb8297d7f3fe5ba19c94ee655c9b0f952a8b5e48a17aaa499a2f6d10d70",+ "provenance": {+ "project": "sigrok-dumps",+ "version": "0ad13477abc959d37fc9a5acbd23901c371c9c76",+ "upstreamFile": "i2c/wii_nunchuk/wii_nunchuk_init_reg_3xdata.sr",+ "url": "https://raw.githubusercontent.com/sigrokproject/sigrok-dumps/0ad13477abc959d37fc9a5acbd23901c371c9c76/i2c/wii_nunchuk/wii_nunchuk_init_reg_3xdata.sr",+ "git_blob_sha1": "a7232fce7b54e68e2be44e15946c90a3c756e8e9",+ "reference": "https://raw.githubusercontent.com/sigrokproject/sigrok-dumps/0ad13477abc959d37fc9a5acbd23901c371c9c76/i2c/wii_nunchuk/README",+ "license_scope": "Repository README declares dumps public domain unless otherwise noted; selected READMEs state no exception; decoder software licensing is separate."+ }}]}diff --git a/experiments/analysis-efficiency/measure.mjs b/experiments/analysis-efficiency/measure.mjsindex 07a39e3c1..20f1e7d61 100644--- a/experiments/analysis-efficiency/measure.mjs+++ b/experiments/analysis-efficiency/measure.mjs@@ -13,6 +13,7 @@ import { Client } from "@modelcontextprotocol/client";import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";import { Ajv2020 } from "ajv/dist/2020.js";import { PACKET_ARTIFACTS, measurePacketEntry } from "./packet-entry.mjs";+import { SIGNAL_ARTIFACTS, measureSignalEntry } from "./signal-entry.mjs";const root = resolve(dirname(fileURLToPath(import.meta.url)), "../..");const { values } = parseArgs({@@ -31,6 +32,7 @@ const { values } = parseArgs({schemas: { type: "string", default: "default" },"sqlite-selection": { type: "string", default: "all" },"packet-selection": { type: "string", default: "baseline" },+ "signal-selection": { type: "string", default: "baseline" },},});if (@@ -41,6 +43,13 @@ if (!isAbsolute(values.path) ||!isAbsolute(values.output) ||!["complete", "summary"].includes(values.delivery) ||+ !["baseline", "focused", "discovery", "pages"].includes(+ values["signal-selection"],+ ) ||+ (!Object.hasOwn(SIGNAL_ARTIFACTS, values.artifact) &&+ values["signal-selection"] !== "baseline") ||+ (Object.hasOwn(SIGNAL_ARTIFACTS, values.artifact) &&+ values.delivery !== "complete") ||!["entry", "entry-and-main", "entry-views"].includes(values["native-question"],) ||@@ -159,6 +168,12 @@ const fingerprints = {packet_oracle_source_sha256: await fileSha(fileURLToPath(new URL("./packet-oracle.py", import.meta.url)),),+ signal_workflow_source_sha256: await fileSha(+ fileURLToPath(new URL("./signal-entry.mjs", import.meta.url)),+ ),+ signal_oracle_source_sha256: await fileSha(+ fileURLToPath(new URL("./signal-oracle.py", import.meta.url)),+ ),runtime_files_sha256: sha(JSON.stringify(runtimeFiles)),runtime_dispatcher_sha256: await fileSha(resolve(values["runtime-root"], "scripts/rea.mjs"),@@ -739,6 +754,12 @@ const inspectPcapTarget = async () => {};const workflows = {+ ...Object.fromEntries(+ Object.keys(SIGNAL_ARTIFACTS).map((artifact) => [+ artifact,+ () => measureSignalEntry({ values, call, inputSha, catalogDefinitions }),+ ]),+ ),...Object.fromEntries(Object.keys(PACKET_ARTIFACTS).map((artifact) => [artifact,@@ -1186,6 +1207,9 @@ const report = {packet_selection: Object.hasOwn(PACKET_ARTIFACTS, values.artifact)? values["packet-selection"]: null,+ signal_selection: Object.hasOwn(SIGNAL_ARTIFACTS, values.artifact)+ ? values["signal-selection"]+ : null,provider_settings_sha256: sha(JSON.stringify(selectedProviders)),provider_search_path_sha256: sha(selectedProviders.PATH ?? process.env.PATH ?? "",diff --git a/experiments/analysis-efficiency/review-signal.mjs b/experiments/analysis-efficiency/review-signal.mjsnew file mode 100644index 000000000..9ae454207--- /dev/null+++ b/experiments/analysis-efficiency/review-signal.mjs@@ -0,0 +1,237 @@+import { readFile, writeFile } from "node:fs/promises";+import { resolve } from "node:path";+import { pathToFileURL } from "node:url";+import { isDeepStrictEqual, parseArgs } from "node:util";+import {+ SIGNAL_ARTIFACTS,+ signalRequest,+ signalDataRows,+ signalFacts,+ verifySignalRows,+ verifySignalOracleIdentity,+} from "./signal-entry.mjs";++const { values } = parseArgs({+ options: {+ trial: { type: "string" },+ oracle: { type: "string" },+ "runtime-root": { type: "string" },+ artifact: { type: "string" },+ selection: { type: "string" },+ },+});+if (+ !values.trial ||+ !values.oracle ||+ !values["runtime-root"] ||+ !Object.hasOwn(SIGNAL_ARTIFACTS, values.artifact ?? "") ||+ !["baseline", "focused"].includes(values.selection)+)+ throw new Error(+ "Use --trial DIR --oracle JSON --runtime-root DIR --artifact ID --selection baseline|focused",+ );+const json = async (name) =>+ JSON.parse(await readFile(resolve(values.trial, name), "utf8"));+const settings = await json("trial-settings.json"),+ summary = await json("summary.json");+const oracle = JSON.parse(await readFile(values.oracle, "utf8"));+const kind = SIGNAL_ARTIFACTS[values.artifact].kind;+const expectedRows =+ kind === "uart"+ ? oracle.rows+ : oracle.reads.flatMap((read) => read.bytes.slice(1));+const expected = signalFacts(kind, expectedRows);+const runtime = (file) =>+ pathToFileURL(resolve(values["runtime-root"], file)).href;+const { parseEvidence } = await import(runtime("dist/domain/evidence.js"));+const { decodeSignalRecordingInputSchema, signalRecordingSchema } =+ await import(runtime("dist/domain/signals/signalRecording.js"));+const { inspectSignalDecodersInputSchema, signalDecodersSchema } = await import(+ runtime("dist/domain/signals/signalDecoders.js")+);+const calls = (await readFile(resolve(values.trial, "events.jsonl"), "utf8"))+ .split("\n")+ .filter(Boolean)+ .map(JSON.parse)+ .filter(+ (event) =>+ event.type === "item.completed" && event.item?.type === "mcp_tool_call",+ )+ .map((event) => event.item);+const reasons = [],+ workflowReasons = [],+ authenticated = new Map(),+ conflicting = new Set();+try {+ verifySignalOracleIdentity(values.artifact, oracle);+ if (oracle.kind !== kind)+ throw new Error("Wrong independent signal question oracle");+ if (calls.length !== 1 || calls[0]?.tool !== "decode_signal_recording")+ workflowReasons.push(+ "Completed public call sequence differs from the fixed single-call plan",+ );+ const planned = decodeSignalRecordingInputSchema.parse(+ signalRequest(+ values.artifact,+ settings.selected_artifact_path,+ values.selection,+ ),+ );+ for (const call of calls) {+ try {+ if (+ call.server !== "rea" ||+ call.status !== "completed" ||+ call.error ||+ call.result?.isError+ )+ throw new Error("Call did not return successful REA observations");+ const text =+ call.result?.content?.filter((item) => item.type === "text") ?? [];+ if (!call.result?.structuredContent && text.length !== 1)+ throw new Error("No unambiguous Evidence envelope");+ const evidence =+ call.result.structuredContent ?? JSON.parse(text[0].text);+ parseEvidence(evidence);+ const capture = call.tool === "decode_signal_recording";+ if (!capture && call.tool !== "inspect_signal_decoders")+ throw new Error("Not a signal observation tool");+ const input = (+ capture+ ? decodeSignalRecordingInputSchema+ : inspectSignalDecodersInputSchema+ ).parse(call.arguments);+ const report = (+ capture ? signalRecordingSchema : signalDecodersSchema+ ).parse(evidence.normalized_result);+ if (+ evidence.operation !== call.tool ||+ !isDeepStrictEqual(evidence.parameters, input) ||+ !isDeepStrictEqual(report.selection, input)+ )+ throw new Error(+ "Evidence differs from actual operation or caller choices",+ );+ if (+ capture+ ? evidence.subject?.local_path !== input.path ||+ evidence.subject?.digest?.sha256 !== oracle.sha256 ||+ report.artifact.path !== input.path ||+ report.artifact.sha256 !== oracle.sha256+ : evidence.subject?.local_path !== report.tool.path ||+ evidence.subject?.digest?.sha256 !== report.tool.sha256+ )+ throw new Error("Evidence identifies different recording/tool bytes");+ if (capture && !isDeepStrictEqual(input, planned))+ workflowReasons.push("Capture selection differs from the fixed plan");+ const observation = {+ operation: call.tool,+ input,+ report,+ rows: capture ? signalDataRows(kind, report.annotations_text) : null,+ };+ const id = evidence.evidence_id;+ if (conflicting.has(id))+ throw new Error("Previously conflicting Evidence identity");+ if (+ authenticated.has(id) &&+ !isDeepStrictEqual(authenticated.get(id), observation)+ ) {+ authenticated.delete(id);+ conflicting.add(id);+ throw new Error(+ "Repeated Evidence identity has inconsistent observations",+ );+ }+ authenticated.set(id, observation);+ } catch (error) {+ workflowReasons.push(+ `Call ${call.tool} did not provide authenticated observations: ${String(error)}`,+ );+ }+ }+ const answer = JSON.parse(+ (await readFile(resolve(values.trial, "answer.txt"), "utf8")).trim(),+ );+ if (+ !isDeepStrictEqual(+ Object.keys(answer).sort(),+ [...Object.keys(expected), "evidence_ids", "limitations"].sort(),+ )+ )+ workflowReasons.push("Final answer shape differs from the fixed plan");+ if (+ !isDeepStrictEqual(+ Object.fromEntries(+ Object.keys(expected).map((key) => [key, answer[key]]),+ ),+ expected,+ )+ )+ throw new Error("Final answer differs from independent sample facts");+ if (+ !Array.isArray(answer.evidence_ids) ||+ new Set(answer.evidence_ids).size !== answer.evidence_ids.length ||+ answer.evidence_ids.some((id) => !authenticated.has(id))+ )+ throw new Error(+ "Final citations are not authenticated signal observations",+ );+ const rows = new Map();+ for (const id of answer.evidence_ids) {+ for (const row of authenticated.get(id).rows ?? []) {+ const old = rows.get(row.start);+ if (old && !isDeepStrictEqual(old, row))+ throw new Error("Cited sample ranges have conflicting decoded data");+ rows.set(row.start, row);+ }+ }+ verifySignalRows(+ kind,+ [...rows.values()].sort((a, b) => a.start - b.start),+ oracle,+ oracle.sha256,+ );+ const limitations =+ typeof answer.limitations === "string"+ ? [answer.limitations]+ : answer.limitations;+ if (+ !Array.isArray(limitations) ||+ limitations.length === 0 ||+ limitations.some((value) => typeof value !== "string" || !value.trim())+ )+ workflowReasons.push("Final answer omits investigation limitations");+ if (+ summary.exit_code !== 0 ||+ summary.timeout ||+ summary.selected_artifact_bytes_unchanged !== true ||+ settings.selected_artifact_sha256_before !== oracle.sha256 ||+ summary.selected_artifact_sha256_after !== oracle.sha256+ )+ workflowReasons.push(+ "Trial did not complete with original selected bytes unchanged",+ );+} catch (error) {+ reasons.push(String(error));+}+const report = {+ factual_correctness: reasons.length ? "failed" : "passed",+ workflow_correctness:+ reasons.length || workflowReasons.length ? "failed" : "passed",+ reasons,+ workflow_reasons: workflowReasons,+ completed_calls: calls.length,+ wall_seconds: summary.wall_seconds,+ usage: summary.usage,+ selected_artifact_sha256: oracle.sha256,+ scope:+ "Two pinned single-device recordings; independently sampled bytes and authenticated cited decoder observations; hardware behavior is not inferred",+ limitation_prose: "Presence checked; wording requires transcript review",+};+await writeFile(+ resolve(values.trial, "signal-review.json"),+ JSON.stringify(report, null, 2) + "\n",+);+console.log(JSON.stringify(report));+if (reasons.length || workflowReasons.length) process.exitCode = 1;diff --git a/experiments/analysis-efficiency/signal-entry.mjs b/experiments/analysis-efficiency/signal-entry.mjsnew file mode 100644index 000000000..b44d4d66a--- /dev/null+++ b/experiments/analysis-efficiency/signal-entry.mjs@@ -0,0 +1,264 @@+import { execFile } from "node:child_process";+import { readFile } from "node:fs/promises";+import { resolve } from "node:path";+import { fileURLToPath, pathToFileURL } from "node:url";+import { isDeepStrictEqual, promisify } from "node:util";++const corpus = JSON.parse(+ await readFile(new URL("./corpus.json", import.meta.url), "utf8"),+);++export const SIGNAL_ARTIFACTS = {+ "sigrok-uart-max3232e.sr": {+ kind: "uart",+ decoders: "uart:rx=MAX3232E DOUT1:baudrate=57600:format=hex:invert_rx=yes",+ annotations: "uart=rx-data",+ query: "rx-data",+ firstPage: 32,+ },+ "sigrok-i2c-nunchuk.sr": {+ kind: "i2c",+ decoders: "i2c:scl=SCL:sda=SDA",+ annotations: "i2c=data-read",+ query: "data-read",+ firstPage: 9,+ },+};++export function signalRequest(artifact, path, selection) {+ const descriptor = SIGNAL_ARTIFACTS[artifact];+ if (!descriptor) throw new Error("Unknown fixed signal question");+ return {+ path,+ decoders: descriptor.decoders,+ input_format: null,+ annotations: selection === "baseline" ? null : descriptor.annotations,+ line_offset: 0,+ line_limit: selection === "pages" ? descriptor.firstPage : null,+ };+}++export function signalDataRows(kind, text) {+ const pattern =+ kind === "uart"+ ? /^(\d+)-(\d+) uart-1: ([0-9A-F]{2})$/+ : /^(\d+)-(\d+) i2c-1: Data read: ([0-9A-F]{2})$/;+ const rows = [];+ for (const line of text.split("\n")) {+ const match = pattern.exec(line);+ if (!match) continue;+ const start = Number(match[1]),+ end = Number(match[2]);+ if (+ !Number.isSafeInteger(start) ||+ !Number.isSafeInteger(end) ||+ start > end+ )+ throw new Error("Unsupported decoded sample interval");+ rows.push({ start, end, byte: Number.parseInt(match[3], 16) });+ }+ return rows;+}++export function signalFacts(kind, rows) {+ if (kind === "uart") return { kind, bytes: rows.map((row) => row.byte) };+ if (kind !== "i2c" || rows.length !== 18)+ throw new Error(+ "The fixed I2C question requires three complete six-byte reads",+ );+ return {+ kind,+ reads: [0, 6, 12].map((offset) =>+ rows.slice(offset, offset + 6).map((row) => row.byte),+ ),+ };+}++export function verifySignalOracleIdentity(artifact, oracle) {+ const pinned = corpus.artifacts.find((entry) => entry.id === artifact);+ if (+ !pinned ||+ oracle.kind !== SIGNAL_ARTIFACTS[artifact]?.kind ||+ oracle.sha256 !== pinned.sha256+ )+ throw new Error(+ "Independent oracle does not identify the pinned signal corpus",+ );+}++export function verifySignalRows(kind, rows, oracle, digest) {+ const expected =+ kind === "uart"+ ? oracle.rows+ : oracle.reads.flatMap((read) =>+ read.bytes.slice(1).map((byte, index) => ({+ ...byte,+ after_previous: read.bytes[index].last_clock,+ before_next: read.bytes[index + 2]?.first_clock ?? read.stop,+ })),+ );+ if (+ oracle.kind !== kind ||+ oracle.sha256 !== digest ||+ rows.length !== expected.length+ )+ throw new Error(+ "Signal observations differ from independent sample identity/coverage",+ );+ for (let index = 0; index < rows.length; index++) {+ const observed = rows[index],+ raw = expected[index];+ if (+ observed.byte !== raw.byte ||+ observed.start < 0 ||+ observed.end >= oracle.total_samples+ )+ throw new Error("Decoded data differs from independent sample bits");+ if (+ kind === "uart"+ ? !Number.isSafeInteger(raw.first_data_sample) ||+ !Number.isSafeInteger(raw.last_data_sample) ||+ observed.start < raw.start ||+ observed.end >= raw.end_exclusive ||+ observed.start > raw.first_data_sample ||+ observed.end < raw.last_data_sample+ : !Number.isSafeInteger(raw.after_previous) ||+ !Number.isSafeInteger(raw.before_next) ||+ observed.start <= raw.after_previous ||+ observed.end >= raw.before_next ||+ observed.start > raw.first_clock ||+ observed.end < raw.last_clock+ )+ throw new Error(+ "Decoded interval does not identify the independently read byte",+ );+ }+}++export async function measureSignalEntry({+ values,+ call,+ inputSha,+ catalogDefinitions,+}) {+ const descriptor = SIGNAL_ARTIFACTS[values.artifact];+ const runtime = (file) =>+ pathToFileURL(resolve(values["runtime-root"], file)).href;+ const { parseEvidence } = await import(runtime("dist/domain/evidence.js"));+ const { decodeSignalRecordingInputSchema, signalRecordingSchema } =+ await import(runtime("dist/domain/signals/signalRecording.js"));+ const { inspectSignalDecodersInputSchema, signalDecodersSchema } =+ await import(runtime("dist/domain/signals/signalDecoders.js"));+ const observations = [];+ const checked = (evidence, operation, input, schema) => {+ parseEvidence(evidence);+ const report = schema.parse(evidence.normalized_result);+ if (+ evidence.operation !== operation ||+ !isDeepStrictEqual(evidence.parameters, input) ||+ !isDeepStrictEqual(report.selection, input)+ )+ throw new Error(+ "Signal Evidence differs from the actual requested operation or choices",+ );+ if (+ operation === "decode_signal_recording"+ ? evidence.subject?.local_path !== values.path ||+ evidence.subject?.digest?.sha256 !== inputSha ||+ report.artifact.path !== values.path ||+ report.artifact.sha256 !== inputSha+ : evidence.subject?.local_path !== report.tool.path ||+ evidence.subject?.digest?.sha256 !== report.tool.sha256+ )+ throw new Error(+ "Signal Evidence identifies different recording/tool bytes",+ );+ observations.push({+ evidence_id: evidence.evidence_id,+ operation,+ selection: report.selection,+ output_window: report.output_window,+ limitations: evidence.limitations,+ });+ return report;+ };+ const selection = values["signal-selection"];+ for (const name of [+ "decode_signal_recording",+ ...(selection === "discovery" ? ["inspect_signal_decoders"] : []),+ ])+ if (!catalogDefinitions.has(name))+ throw new Error(`Missing live signal contract ${name}`);+ if (selection === "discovery") {+ const input = inspectSignalDecodersInputSchema.parse({+ decoder: descriptor.kind,+ query: descriptor.query,+ limit: 5,+ });+ const report = checked(+ await call("inspect_signal_decoders", input),+ "inspect_signal_decoders",+ input,+ signalDecodersSchema,+ );+ if (!report.help_lines.some((line) => line.text.includes(descriptor.query)))+ throw new Error(+ "Installed decoder help does not establish the selected annotation class",+ );+ }+ const input = decodeSignalRecordingInputSchema.parse(+ signalRequest(values.artifact, values.path, selection),+ );+ const first = checked(+ await call("decode_signal_recording", input),+ "decode_signal_recording",+ input,+ signalRecordingSchema,+ );+ let rows = signalDataRows(descriptor.kind, first.annotations_text);+ if (selection === "pages") {+ const nextInput = decodeSignalRecordingInputSchema.parse({+ ...input,+ line_offset: descriptor.firstPage,+ line_limit: null,+ });+ const next = checked(+ await call("decode_signal_recording", nextInput),+ "decode_signal_recording",+ nextInput,+ signalRecordingSchema,+ );+ const repeated = checked(+ await call("decode_signal_recording", input),+ "decode_signal_recording",+ input,+ signalRecordingSchema,+ );+ if (+ first.annotations_text !== repeated.annotations_text ||+ !isDeepStrictEqual(first.output_window, repeated.output_window) ||+ first.output_window.total_lines !== next.output_window.total_lines+ )+ throw new Error("Repeated/next display pages changed observation scope");+ rows = [...rows, ...signalDataRows(descriptor.kind, next.annotations_text)];+ }+ const { stdout } = await promisify(execFile)(+ "python3",+ [+ fileURLToPath(new URL("./signal-oracle.py", import.meta.url)),+ descriptor.kind,+ values.path,+ ...(descriptor.kind === "uart" ? ["--invert"] : []),+ ],+ { timeout: 30_000, maxBuffer: 256 * 1024 },+ );+ const oracle = JSON.parse(stdout);+ verifySignalOracleIdentity(values.artifact, oracle);+ verifySignalRows(descriptor.kind, rows, oracle, inputSha);+ return {+ artifact_sha256: inputSha,+ facts: signalFacts(descriptor.kind, rows),+ delivery_observations: observations,+ evidence_ids: [...new Set(observations.map((item) => item.evidence_id))],+ };+}diff --git a/experiments/analysis-efficiency/signal-model-prompt.mjs b/experiments/analysis-efficiency/signal-model-prompt.mjsnew file mode 100644index 000000000..64f87fee1--- /dev/null+++ b/experiments/analysis-efficiency/signal-model-prompt.mjs@@ -0,0 +1,22 @@+import { parseArgs } from "node:util";+import { signalRequest, SIGNAL_ARTIFACTS } from "./signal-entry.mjs";++const { values } = parseArgs({+ options: { artifact: { type: "string" }, selection: { type: "string" } },+});+if (+ !Object.hasOwn(SIGNAL_ARTIFACTS, values.artifact ?? "") ||+ !["baseline", "focused"].includes(values.selection)+)+ throw new Error(+ "Use --artifact fixed-signal-ID --selection baseline|focused",+ );+const kind = SIGNAL_ARTIFACTS[values.artifact].kind;+const question =+ kind === "uart"+ ? "Report every decoded RX data octet on the specified DOUT1 channel, in recording order, as decimal integer bytes. Include all repetitions; do not deduplicate data."+ : "Report every I2C READ data payload in recording order, grouped into consecutive six-byte payloads. Exclude address bytes and write data. Do not infer button or hardware behavior.";+console.log(`${question}+Use the configured REA MCP and make exactly one decode_signal_recording call with this selection:+${JSON.stringify(signalRequest(values.artifact, "{{artifact_path}}", values.selection))}+Return one JSON object with exactly these keys: kind (${JSON.stringify(kind)}), ${kind === "uart" ? "bytes (an array of decimal octets)" : "reads (an array of six-octet decimal arrays)"}, evidence_ids (unique actual returned Evidence IDs), limitations (nonempty investigation limitations). Preserve source identity and recorded order. Output lines are not sample or annotation counts; selected decoder observations do not prove complete hardware traffic or original application behavior. Do not run shell commands, browse, acquire hardware samples, invent missing data, or fabricate citations.`);diff --git a/experiments/analysis-efficiency/signal-oracle.py b/experiments/analysis-efficiency/signal-oracle.pynew file mode 100644index 000000000..dbccf01a2--- /dev/null+++ b/experiments/analysis-efficiency/signal-oracle.py@@ -0,0 +1,211 @@+"""Independent readers for the two pinned, single-device digital SR recordings."""++import argparse+from bisect import bisect_right+import configparser+import hashlib+import io+import json+from pathlib import Path+import re+import struct+import zipfile+++def read_edges(path, channels):+ size = path.stat().st_size+ if size > 32 * 1024 * 1024:+ raise ValueError('Compressed SR input exceeds the experiment budget')+ with path.open('rb') as source:+ admitted = source.read(size + 1)+ if len(admitted) != size:+ raise ValueError('SR input size changed during admission')+ marker = admitted.rfind(b'PK\x05\x06', max(0, len(admitted) - 65557))+ if marker < 0 or len(admitted) - marker < 22:+ raise ValueError('Missing classic ZIP directory receipt')+ header = struct.unpack_from('<4s4H2IH', admitted, marker)+ if (marker >= 20 and admitted[marker-20:marker-16] == b'PK\x06\x07'):+ raise ValueError('ZIP64 directory is unsupported')+ if header[1] or header[2] or header[3] != header[4] or not 1 <= header[4] <= 64 or header[5] > 65536 or marker + 22 + header[7] != len(admitted) or header[6] + header[5] != marker:+ raise ValueError('Unsupported or excessive ZIP directory representation')+ directory = memoryview(admitted)[header[6]:marker]+ position = count = 0+ while position < len(directory):+ if count >= 64 or len(directory) - position < 46 or directory[position:position+4] != b'PK\x01\x02':+ raise ValueError('Unsupported or excessive actual ZIP directory records')+ name_size, extra_size, comment_size = struct.unpack_from('<3H', directory, position + 28)+ end = position + 46 + name_size + extra_size + comment_size+ if end > len(directory):+ raise ValueError('Incomplete actual ZIP directory record')+ extra = position + 46 + name_size+ extra_end = extra + extra_size+ while extra < extra_end:+ if extra_end - extra < 4:+ raise ValueError('Incomplete ZIP extra field')+ tag, length = struct.unpack_from('<2H', directory, extra)+ if tag == 1:+ raise ValueError('ZIP64 member is unsupported')+ extra += 4 + length+ if extra > extra_end:+ raise ValueError('Incomplete ZIP extra field')+ count += 1+ position = end+ if count != header[4]:+ raise ValueError('Declared and actual ZIP directory counts differ')+ before = hashlib.sha256(admitted).hexdigest()+ # Admission and ZipFile parse the same bounded snapshot, not a reopened path.+ with zipfile.ZipFile(io.BytesIO(admitted), allowZip64=False) as archive:+ entries = archive.infolist()+ names = [item.filename for item in entries]+ if len(names) != len(set(names)) or len(names) > 64:+ raise ValueError("Ambiguous or excessive SR members")+ if sum(item.file_size for item in entries) > 32 * 1024 * 1024:+ raise ValueError("Expanded SR input exceeds the experiment budget")+ if archive.getinfo('metadata').file_size > 8192:+ raise ValueError("Excessive SR metadata")+ # Declared sizes are untrusted. A bounded read also caps each inflater+ # step; logic members below likewise use bounded block reads.+ with archive.open('metadata') as source:+ metadata_bytes = source.read(8193)+ if len(metadata_bytes) > 8192:+ raise ValueError("Expanded SR metadata exceeds the experiment budget")+ metadata = metadata_bytes.decode('utf-8')+ config = configparser.ConfigParser(interpolation=None)+ config.read_string(metadata)+ if set(config.sections()) != {'global', 'device 1'}:+ raise ValueError("Only the pinned single-device profile is supported")+ device = config['device 1']+ unit = int(device['unitsize'])+ probes = int(device['total probes'])+ if unit not in [1, 2] or not 1 <= probes <= unit * 8:+ raise ValueError("Unsupported digital sample geometry")+ rate = re.fullmatch(r'(\d+)\s+(Hz|kHz|MHz)', device['samplerate'])+ if rate is None or int(rate[1]) <= 0:+ raise ValueError("Unsupported sample-rate representation")+ samplerate = int(rate[1]) * {'Hz': 1, 'kHz': 1000, 'MHz': 1000000}[rate[2]]+ bits = []+ for channel in channels:+ matches = [index - 1 for index in range(1, probes + 1)+ if device.get(f'probe{index}') == channel]+ if len(matches) != 1:+ raise ValueError("Channel identity is missing or ambiguous")+ bits.append(matches[0])+ prefix = device['capturefile']+ chunks = [name for name in names if re.fullmatch(re.escape(prefix) + r'-\d+', name)]+ if prefix in names and chunks:+ raise ValueError("Ambiguous direct and chunked logic data")+ if prefix in names:+ chunks = [prefix]+ else:+ chunks.sort(key=lambda name: int(name.rsplit('-', 1)[1]))+ if [int(name.rsplit('-', 1)[1]) for name in chunks] != list(range(1, len(chunks) + 1)):+ raise ValueError("Missing or ambiguous logic chunk order")+ if not chunks or any(archive.getinfo(name).file_size % unit for name in chunks):+ raise ValueError("Missing or incomplete digital samples")+ edges = [[] for _ in channels]+ previous = None+ sample = 0+ mask = sum(1 << bit for bit in bits)+ for name in chunks:+ with archive.open(name) as source:+ while block := source.read(65536):+ values = block if unit == 1 else (item[0] for item in struct.iter_unpack('<H', block))+ for value in values:+ selected = value & mask+ if selected != previous:+ for index, bit in enumerate(bits):+ level = (selected >> bit) & 1+ if not edges[index] or edges[index][-1][1] != level:+ if sum(map(len, edges)) >= 65536:+ raise ValueError('Selected transitions exceed the experiment representation budget')+ edges[index].append((sample, level))+ previous = selected+ sample += 1+ with path.open('rb') as source:+ after = source.read(size + 1)+ if len(after) != size or hashlib.sha256(after).hexdigest() != before:+ raise ValueError("Recording bytes changed during the independent read")+ return {'sha256': before, 'metadata': metadata, 'sample_rate_hz': samplerate,+ 'total_samples': sample, 'channels': dict(zip(channels, bits))}, edges+++def uart(path, invert):+ metadata, traces = read_edges(path, ['MAX3232E DOUT1'])+ edges = [(sample, level ^ invert) for sample, level in traces[0]]+ positions = [sample for sample, _ in edges]+ rate, baud = metadata['sample_rate_hz'], 57600+ def level(sample):+ return edges[bisect_right(positions, sample) - 1][1]+ rows = []+ busy_until = 0+ for index, (start, value) in enumerate(edges):+ if index == 0 or value != 0 or start < busy_until:+ continue+ center = lambda halfbits: start + (halfbits * rate) // (2 * baud)+ if center(19) >= metadata['total_samples']:+ raise ValueError("Incomplete UART frame at the capture boundary")+ if level(center(1)) != 0 or level(center(19)) != 1:+ raise ValueError(f"UART framing differs from selected polarity at sample {start}")+ byte = sum(level(center(3 + 2 * bit)) << bit for bit in range(8))+ # A UART receiver becomes ready after sampling the stop bit. Waiting+ # a full nominal ten bits would miss a valid slightly faster sender.+ rows.append({'start': start, 'end_exclusive': center(19) + 1, 'byte': byte,+ 'first_data_sample': center(3), 'last_data_sample': center(17)})+ busy_until = center(19) + 1+ return metadata | {'kind': 'uart', 'baud_rate': baud, 'invert': bool(invert),+ 'complete_frames': len(rows), 'rows': rows}+++def i2c(path):+ metadata, traces = read_edges(path, ['SCL', 'SDA'])+ transitions = {}+ for channel, edges in enumerate(traces):+ for sample, value in edges:+ transitions.setdefault(sample, {})[channel] = value+ clock = data = None+ active = None+ bits = []+ transactions = []+ for sample, changes in sorted(transitions.items()):+ next_clock, next_data = changes.get(0, clock), changes.get(1, data)+ if clock is not None and data is not None:+ if clock == next_clock == 1 and next_data != data:+ if next_data == 0:+ if active is not None:+ raise ValueError("The pinned profile does not contain repeated START")+ active = {'start': sample, 'bytes': []}+ bits = []+ elif active is not None:+ # SCL may rise with SDA low to prepare STOP; that final+ # rise is not the first bit of a new data byte.+ if len(bits) == 1 and bits[0][1] == 0:+ bits = []+ if bits:+ raise ValueError("Incomplete I2C byte before STOP")+ active['stop'] = sample+ transactions.append(active)+ active = None+ elif active is not None and clock == 0 and next_clock == 1:+ if next_data != data:+ raise ValueError("Ambiguous simultaneous clock/data transition")+ bits.append((sample, next_data))+ if len(bits) == 9:+ byte = sum(bit << (7-index) for index, (_, bit) in enumerate(bits[:8]))+ active['bytes'].append({'byte': byte, 'first_clock': bits[0][0],+ 'last_clock': bits[7][0], 'ack': bits[8][1] == 0})+ bits = []+ clock, data = next_clock, next_data+ if active is not None or bits:+ raise ValueError("Incomplete I2C transaction at capture end")+ reads = [transaction for transaction in transactions+ if transaction['bytes'] and transaction['bytes'][0]['byte'] & 1]+ return metadata | {'kind': 'i2c', 'transactions': transactions, 'reads': reads}+++if __name__ == '__main__':+ parser = argparse.ArgumentParser()+ parser.add_argument('kind', choices=['uart', 'i2c'])+ parser.add_argument('path', type=Path)+ parser.add_argument('--invert', action='store_true')+ args = parser.parse_args()+ print(json.dumps(uart(args.path, args.invert) if args.kind == 'uart' else i2c(args.path)))diff --git a/experiments/analysis-efficiency/signal-selection-workflows.md b/experiments/analysis-efficiency/signal-selection-workflows.mdnew file mode 100644index 000000000..4cc216827--- /dev/null+++ b/experiments/analysis-efficiency/signal-selection-workflows.md@@ -0,0 +1,188 @@+# Sigrok selection and repeated-page checkpoint (#3131)++This experiment preserves production APIs, defaults, caller selections and Evidence+semantics. Existing annotation selectors substantially reduce replies, but this+small real-model cohort does **not** establish a consistent reduction in total+input tokens or elapsed task time. No decoder cache or new production preset is+introduced. Complete necessary output stays available through existing explicit+null limits; original defaults remain 50 display lines.++## Immutable corpus and independent questions++The runtime is pinned to `ca7450beffff59325ce3bddc78e181693ce77177`, with Node+24.18.0, SDK 2.3.1 and Linux x86-64. Experiment driver code for model trials is+`23317a58d`; each SDK result separately records workflow/oracle source hashes,+runtime-file and lockfile digests, explicit provider settings and original bytes.+Later integration checks must remain separate from this cohort.++Two official sigrok-dumps recordings are pinned to upstream commit+`0ad13477abc959d37fc9a5acbd23901c371c9c76` in [corpus.json](corpus.json).+Original Git blob IDs, SHA-256, sizes and immutable reference/download URLs are+recorded there. No sample binaries or raw output are committed. The upstream+repository declares dumps public domain unless otherwise noted; the inspected+sample READMEs state no exception. Decoder software licensing is separate.++- MAX3232E UART: 12,742 compressed bytes, 10,000,000 logic bytes, 5,000,000 samples+ at the recorded nominal 100 MHz. Stored DOUT1 is bit 1; DIN1 is bit 2. The exact+ question requests all RX octets in order, including repetitions. Independent+ 57,600-baud 8N1 stop-bit sampling with DOUT1 inversion yields 65 bytes: five+ copies of `Hello world\r\n`. The source metadata and physical probe numbers+ remain distinct from stored bit positions.+- Wii Nunchuk I2C: 20,373 compressed bytes, 20,000,000 samples at nominal 1 MHz.+ The source README's physical probes 3/7 are repacked as stored named channels+ SCL/SDA at bits 0/1. The question asks every read payload as consecutive+ six-byte arrays, excluding address/write bytes and hardware/button inference.+ Independent START/STOP, rising-clock, byte and ACK/NACK reading identifies seven+ transactions and three read payloads, matching the upstream reference:+ `75 7F 77 4F 82 3B`, `75 7F 75 44 82 34`, `75 7F 77 43 83 5D`.++The independent reader accepts the scoped single-device classic-ZIP profile,+checks directory representation before object allocation, bounds compressed and+expanded input to 32 MiB, and limits selected transitions to 65,536. It preserves+source metadata/digest, chunk ordering and channel identity; incomplete samples,+ambiguous names, partial bytes and unsupported transactions fail explicitly.+Receiver rearming follows the verified UART stop sample, accommodating the+recording's slightly faster sender. I2C STOP preparation is distinct from data.+These checks do not establish other formats, decoders, OSes or physical acquisition.++An initial DOUT1 decode without inversion exited successfully but produced wrong+bytes. Its cost/output is retained privately as configuration exploration, outside+eligible measurements. An independent sample oracle prevents equal-but-wrong+producer results from passing. Correct inversion was established before the cohort.++## Actual CLI and SDK evidence++sigrok-cli 0.7.2 reports libsigrok 0.5.2 and libsigrokdecode 0.5.3. Its executable+SHA-256 is `e1b1eb2f25edc17d24a78acbab16ff7e1dfdee74bb2cccb7ad00497ccc7a0c44`.+Selected installed UART/I2C decoder module bytes are fingerprinted privately.+The banner and executable digest do not fingerprint the complete installation;+modules and libraries remain trusted. Both actual CLI and MCP results match raw+sample bytes; data intervals correlate with the independently examined samples.++Fourteen actual SDK workflows comprise two interleaved repetitions per recording+and arm (baseline/focused/discovery), plus first/next/repeated pages. All 22 calls+preserve source bytes and necessary decoded facts. Baseline returns all necessary+annotation output. Focused selects `uart=rx-data` or `i2c=data-read`, with explicit+null line limits because the task needs the entire selected payload. Discovery+first queries the installed decoder help for the already specified class; it does+not establish discovery benefits for unknown protocols.++| Recording | Baseline bytes | Focused bytes | Discovery + focused bytes | First + next + repeat bytes |+| --------- | -------------: | ------------: | ------------------------: | --------------------------: |+| UART | 49,417 | 12,441 | 18,839 | 31,779 |+| I2C | 26,829 | 9,928 | 16,274 | 27,623 |++Focused reply reductions are 74.8% and 63.0%. These are parsed SDK JSON bytes,+not wire bytes or model tokens. Baseline/focused RPC times (ms, repetitions 1/2)+are UART 2010/582 versus 1459/911, and I2C 2851/676 versus 2330/685; n=2,+with order/cold/warm variation, does not establish a general speedup. Pages retain+input/class/window coverage separately, rather than equating display exhaustion+with a complete protocol decode. Repeated first pages match their original text+and scope; reused content-based Evidence identities are valid.++## Complete real-model task costs++Eight actual Codex CLI 0.161.0 tasks requested `gpt-5.6-terra` / `high`, in isolated+profiles using this runtime's REA MCP. Both tools were enabled and preapproved+identically in all arms; shell, browser and extra agents were disabled. The model+alias is a requested identifier, not independently resolved backend identity.+Questions contain no expected payloads. Trial order is baseline 1, focused 1,+focused 2, baseline 2, with UART then I2C in each step.++| Trial | Total input | Cached input | Uncached input (derived) | Output | Seconds | Completed tool calls | Facts / fixed plan |+| --------------- | ----------: | -----------: | -----------------------: | -----: | ------: | -------------------: | ------------------ |+| uart baseline 1 | 92,399 | 49,408 | 42,991 | 1,295 | 86.411 | 1 | passed / passed |+| uart focused 1 | 130,029 | 98,048 | 31,981 | 780 | 49.623 | 1 | passed / passed |+| uart focused 2 | 98,894 | 78,592 | 20,302 | 878 | 73.861 | 1 | passed / passed |+| uart baseline 2 | 90,395 | 48,384 | 42,011 | 926 | 43.974 | 1 | passed / passed |+| i2c baseline 1 | 172,162 | 121,344 | 50,818 | 1,194 | 36.330 | 2 | passed / failed |+| i2c focused 1 | 74,815 | 49,408 | 25,407 | 937 | 114.515 | 1 | passed / passed |+| i2c focused 2 | 74,684 | 48,384 | 26,300 | 668 | 73.877 | 1 | passed / passed |+| i2c baseline 2 | 86,241 | 49,408 | 36,833 | 778 | 41.084 | 1 | passed / passed |++All eight answers match independent raw samples and cite authenticated decoder+observations. I2C baseline 1 also queried generic MCP resources before its single+required decode. The predeclared strict single-call grader marks its plan failed;+its actual 172,162 input tokens and 36.330 seconds remain recorded, separately+from the eligible fixed-plan pairs. It is not retried into a better result.++UART eligible pairs increase total input by 40.7% and 9.4%; elapsed changes are+-42.6% and +68.0%. The one eligible I2C pair reduces total input by 13.4% but+increases elapsed time by 79.8%. Reduced uncached input does not replace total+input as the primary token measure. No reliable whole-task token/time benefit is+established across these two questions; reply projection alone is insufficient.++## Producer, window and resource scope++Eight separately bounded direct producer runs preserve source bytes and render+correct stdout: UART baseline 159/131 ms versus focused 163/130 ms; I2C baseline+355/298 ms versus focused 317/290 ms. This isolates the producer process from REA+RPC, version probing and snapshot work, but startup/decoding/annotation formatting+remain combined. Decoding-only versus formatting-only time is unmeasured.++REA literal-window extraction and JSON encoding were measured separately from+retained producer text (one cold observation and 200 warm iterations per arm).+Warm mean extraction ranges from 0.0040 to 0.1794 ms; JSON encoding from 0.0021 to 0.0121 ms. These exclude file reads, RPC and decoding.++Every page still invokes decoding. Retained output might help repeated inspection,+but this fixed cohort does not demonstrate a completed-task improvement warranting+a new cache. Any future work should own bounded output explicitly by original+Evidence identity and lifetime, with cancellation/cleanup, rather than use a+CLI-digest-only generic cache of an incompletely fingerprinted installation.++Heavy work ran serially under one CPU, 3 GiB early stop / 4 GiB hard memory,+no swap and host-availability checks. Verified payload-exit peak bytes were+SDK 587,513,856; model 889,323,520; direct producer 26,288,128. No OOM/OOM-kill or resource abort occurred in that verified phase; task descendants+were cleaned. Terminal whole-scope counters are unavailable and remain unknown.++## Verification and replay++Eleven independent-oracle regressions and 12 comparator/sample-interval tests pass.+Twenty-four private replays using actual SDK Evidence cover valid answers, failed+calls before/after recovery, wrong bytes, unknown citations and missing limitations.+These are grading checks with injected state, not additional model/provider trials.+The grader retains authenticated observations by identity, rejects conflicts,+combines cited pages in sample order and keeps failed calls/plan deviations separate+from supported facts. Relevant typecheck/lint and pre-push checks pass.++Review hardening ties the oracle to the named corpus digest, bounds each I2C+interval against neighboring byte clocks/transaction STOP, and counts actual+classic-ZIP directory records on the same bounded snapshot before allocating+`ZipFile` objects; ZIP64 is rejected. Forged-count and ZIP64 admission regressions+fail on the original implementation. Replaying all eight original model trials+and 24 injected grading cases with these checks preserves their factual/plan+grades and recorded usage/time; these replays are not new model measurements.++Metadata reads cap inflater output even when ZIP sizes are forged. A streamed+8 MiB malformed metadata member declaring 8 KiB is rejected with traced Python+allocation below 1 MiB; the same regression fails on the preceding implementation.+Compressed snapshots use actual file size plus one growth-detection byte rather+than preallocating the entire admission budget. Both genuine waveform oracles+retain identical bytes, metadata, sample intervals and source digests.++UART interval checks also require coverage of the first and last independently+sampled data-bit centers within the owning frame. Shortened, shifted and+single-sample intervals are rejected even when the decoded byte is correct.+The added oracle coordinates retain the original byte/frame facts; replaying+saved real Evidence and all eight model transcripts preserves the original+grades and costs. This is stronger retrospective verification, not a new cohort.++After rebasing onto main `44d98de2367b99808d78db732ec7d72533266233`,+`npm run build:cached` and the 16 focused comparator/sample-interval/MCP-boundary+tests passed. Two additional real focused MCP workflows independently verified+UART/I2C bytes and intervals on that base. These integration checks are separate+from the original cost cohort and do not replace or extend its measurements.++A subsequent rebase onto main `3b79af28e80785afce9f5998d8f03289402a1847`+passed the 17 focused tests, including the upstream-fixed Hopper doctor case,+and two additional real UART/I2C MCP workflows. Original cohort costs and grades+remain unchanged; these checks verify integration on the updated main only.++Use `measure.mjs --signal-selection baseline|focused|discovery|pages`, preserving+runtime/source fingerprints and explicit provider JSON. Generate the exact task+with `signal-model-prompt.mjs --artifact ID --selection baseline|focused`, run the+isolated `agent-trial.py --mode direct --capture-tool-context`, and grade with+`review-signal.mjs` and the independently produced oracle. Keep original failures,+resources and raw observations private. For migration, existing optional annotation+selection is sufficient for scoped reply reduction; no default/API change follows+from this mixed whole-task result.diff --git a/experiments/analysis-efficiency/test_signal_oracle.py b/experiments/analysis-efficiency/test_signal_oracle.pynew file mode 100644index 000000000..9decd9a18--- /dev/null+++ b/experiments/analysis-efficiency/test_signal_oracle.py@@ -0,0 +1,149 @@+"""Distinct framing, identity and admission failures in the sample-bit oracle."""++import importlib.util+from pathlib import Path+import struct+import tempfile+import tracemalloc+import unittest+from unittest.mock import patch+import zipfile++spec = importlib.util.spec_from_file_location(+ 'signal_oracle', Path(__file__).with_name('signal-oracle.py'))+oracle = importlib.util.module_from_spec(spec)+spec.loader.exec_module(oracle)+++class SignalOracleTests(unittest.TestCase):+ def setUp(self):+ self.root = tempfile.TemporaryDirectory()+ self.addCleanup(self.root.cleanup)+ self.path = Path(self.root.name) / 'sample.sr'++ def recording(self, data, unit, channels, chunk='logic-1', extra=0):+ metadata = '[global]\nsigrok version=0.3.0\n[device 1]\ncapturefile=logic-1\n'+ metadata += f'total probes={len(channels)}\nsamplerate=1 MHz\nunitsize={unit}\n'+ metadata += ''.join(f'probe{index+1}={name}\n' for index, name in enumerate(channels))+ with zipfile.ZipFile(self.path, 'w') as archive:+ archive.writestr('version', '2')+ archive.writestr('metadata', metadata)+ archive.writestr(chunk, data)+ for index in range(extra):+ archive.writestr(f'extra-{index}', '')++ def test_uart_rearms_after_stop_sample_for_a_slightly_faster_sender(self):+ levels = [1] * 100+ for byte in [0x48, 0x65]:+ for bit in [0, *[(byte >> index) & 1 for index in range(8)], 1]:+ levels.extend([bit] * 17)+ levels.extend([1] * 100)+ data = b''.join(struct.pack('<H', (1-level) << 1) for level in levels)+ self.recording(data, 2, ['unused', 'MAX3232E DOUT1'])+ result = oracle.uart(self.path, True)+ self.assertEqual([row['byte'] for row in result['rows']], [0x48, 0x65])+ self.assertEqual(result['channels']['MAX3232E DOUT1'], 1)+ for row, start in zip(result['rows'], [100, 270]):+ self.assertEqual(row['first_data_sample'], start + 26)+ self.assertEqual(row['last_data_sample'], start + 147)++ def i2c_levels(self, incomplete=False):+ states = [(1, 1), (1, 0)]+ for byte, ack in [(0xA5, 0), (0x75, 1)]:+ values = [(byte >> index) & 1 for index in reversed(range(8))] + [ack]+ if incomplete and byte == 0x75:+ values = values[:4]+ for bit in values:+ states.extend([(0, bit), (1, bit)])+ states.extend([(0, 0), (1, 0), (1, 1)])+ return bytes(clock | data << 1 for clock, data in states)++ def test_i2c_stop_setup_is_distinct_from_another_data_byte(self):+ self.recording(self.i2c_levels(), 1, ['SCL', 'SDA'])+ result = oracle.i2c(self.path)+ self.assertEqual(len(result['reads']), 1)+ self.assertEqual([row['byte'] for row in result['reads'][0]['bytes']], [0xA5, 0x75])+ self.assertFalse(result['reads'][0]['bytes'][-1]['ack'])++ def test_incomplete_i2c_byte_is_not_invented_at_stop(self):+ self.recording(self.i2c_levels(incomplete=True), 1, ['SCL', 'SDA'])+ with self.assertRaisesRegex(ValueError, 'Incomplete I2C byte'):+ oracle.i2c(self.path)++ def test_channel_names_must_identify_one_stored_bit(self):+ self.recording(b'\x00', 1, ['SCL', 'SCL'])+ with self.assertRaisesRegex(ValueError, 'ambiguous'):+ oracle.read_edges(self.path, ['SCL'])++ def test_missing_chunk_is_not_renumbered(self):+ self.recording(b'\x00', 1, ['SCL', 'SDA'], chunk='logic-1-2')+ with self.assertRaisesRegex(ValueError, 'chunk order'):+ oracle.read_edges(self.path, ['SCL', 'SDA'])++ def test_partial_sample_word_is_rejected(self):+ self.recording(b'\x00', 2, ['unused', 'MAX3232E DOUT1'])+ with self.assertRaisesRegex(ValueError, 'incomplete digital samples'):+ oracle.read_edges(self.path, ['MAX3232E DOUT1'])++ def test_directory_budget_precedes_zip_object_allocation(self):+ self.recording(b'\x00', 1, ['SCL', 'SDA'], extra=62)+ with patch.object(oracle.zipfile, 'ZipFile', side_effect=AssertionError('Too late')):+ with self.assertRaisesRegex(ValueError, 'excessive ZIP directory'):+ oracle.read_edges(self.path, ['SCL', 'SDA'])++ def test_declared_expansion_is_rejected_before_decompression(self):+ self.recording(b'\x00', 1, ['SCL', 'SDA'])+ data = bytearray(self.path.read_bytes())+ directory = data.index(b'PK\x01\x02')+ struct.pack_into('<I', data, directory + 24, 32 * 1024 * 1024 + 1)+ self.path.write_bytes(data)+ with self.assertRaisesRegex(ValueError, 'Expanded SR input'):+ oracle.read_edges(self.path, ['SCL', 'SDA'])++ def test_forged_member_count_cannot_allocate_extra_zip_objects(self):+ self.recording(b'\x00', 1, ['SCL', 'SDA'], extra=62)+ data = bytearray(self.path.read_bytes())+ end = data.rfind(b'PK\x05\x06')+ struct.pack_into('<2H', data, end + 8, 1, 1)+ self.path.write_bytes(data)+ with patch.object(oracle.zipfile, 'ZipFile', side_effect=AssertionError('Too late')):+ with self.assertRaisesRegex(ValueError, 'actual ZIP directory'):+ oracle.read_edges(self.path, ['SCL', 'SDA'])++ def test_zip64_locator_cannot_override_the_classic_budget(self):+ self.recording(b'\x00', 1, ['SCL', 'SDA'])+ data = bytearray(self.path.read_bytes())+ end = data.rfind(b'PK\x05\x06')+ # Syntactically complete ZIP64 directory/locator before a classic EOCD.+ header = struct.unpack_from('<4s4H2IH', data, end)+ record = struct.pack('<4sQ2H2I4Q', b'PK\x06\x06', 44, 45, 45, 0, 0,+ header[4], header[4], 100000000, header[6])+ locator = struct.pack('<4sIQI', b'PK\x06\x07', 0, end, 1)+ data[end:end] = record + locator+ self.path.write_bytes(data)+ with patch.object(oracle.zipfile, 'ZipFile', side_effect=AssertionError('Too late')):+ with self.assertRaisesRegex(ValueError, 'ZIP64 directory'):+ oracle.read_edges(self.path, ['SCL', 'SDA'])++ def test_forged_metadata_size_cannot_expand_past_its_output_budget(self):+ # Stream the malformed member so constructing the fixture is bounded.+ with zipfile.ZipFile(self.path, 'w', compression=zipfile.ZIP_DEFLATED) as archive:+ with archive.open('metadata', 'w') as member:+ for _ in range(128):+ member.write(b'X' * 65536)+ data = bytearray(self.path.read_bytes())+ directory = data.index(b'PK\x01\x02')+ struct.pack_into('<I', data, directory + 24, 8192)+ self.path.write_bytes(data)+ tracemalloc.start()+ try:+ with self.assertRaises(zipfile.BadZipFile):+ oracle.read_edges(self.path, ['SCL', 'SDA'])+ _, peak = tracemalloc.get_traced_memory()+ finally:+ tracemalloc.stop()+ self.assertLess(peak, 1024 * 1024, 'Metadata allocation followed forged ZIP expansion')+++if __name__ == '__main__':+ unittest.main()diff --git a/tests/evaluation/analysisEfficiencyComparison.test.ts b/tests/evaluation/analysisEfficiencyComparison.test.tsindex 924ba1612..94af61d3e 100644--- a/tests/evaluation/analysisEfficiencyComparison.test.ts+++ b/tests/evaluation/analysisEfficiencyComparison.test.ts@@ -31,6 +31,7 @@ type Fixture = {provider_search_path_sha256: string;calls: unknown[];packet_selection?: string;+ signal_selection?: string;};answer: {capture: { coverage: string; capture_start: null };@@ -38,8 +39,9 @@ type Fixture = {delivery_observations: Observation[];facts?: {kind: string;- matching_packets: number;- rows: { frame: number }[];+ matching_packets?: number;+ rows?: { frame: number }[];+ bytes?: number[];};};};@@ -154,14 +156,38 @@ for (const artifact of [equivalent.report.preserved_run_specific_values.map((row) => row.value),[left.answer.delivery_observations, right.answer.delivery_observations],);- right.answer.facts!.matching_packets++;+ right.answer.facts!.matching_packets = 1338;assert.equal(compare(left, right).report.facts_match, false);- right.answer.facts!.matching_packets--;+ right.answer.facts!.matching_packets = 1337;right.summary.packet_selection = "baseline";assert.equal(compare(left, right).report.facts_match, false);});}+for (const artifact of ["sigrok-uart-max3232e.sr", "sigrok-i2c-nunchuk.sr"]) {+ test(`${artifact}: compare decoded facts across signal plans and preserve display scope`, () => {+ const left = fixture("complete", artifact);+ left.summary.signal_selection = "baseline";+ left.answer.facts = { kind: "signal", bytes: [72, 101] };+ const right = structuredClone(left);+ right.summary.signal_selection = "focused";+ right.answer.delivery_observations[0]!.coverage =+ "selected annotation lines";+ const equivalent = compare(left, right);+ assert.equal(equivalent.status, 0);+ assert.equal(equivalent.report.facts_match, true);+ assert.deepEqual(+ equivalent.report.preserved_run_specific_values.map((row) => row.value),+ [left.answer.delivery_observations, right.answer.delivery_observations],+ );+ right.answer.facts!.bytes![0] = 73;+ assert.equal(compare(left, right).report.facts_match, false);+ right.answer.facts!.bytes![0] = 72;+ right.summary.signal_selection = "baseline";+ assert.equal(compare(left, right).report.facts_match, false);+ });+}+for (const artifact of ["wireshark-http.cap","wireshark-http_redirects.pcapng",diff --git a/tests/evaluation/signalWorkflowSemantics.test.ts b/tests/evaluation/signalWorkflowSemantics.test.tsnew file mode 100644index 000000000..7877c494e--- /dev/null+++ b/tests/evaluation/signalWorkflowSemantics.test.ts@@ -0,0 +1,46 @@+import assert from "node:assert/strict";+import { spawnSync } from "node:child_process";+import { test } from "vitest";++test("signal verification rejects skipped identical UART bytes and reordered I2C samples", () => {+ const moduleUrl = new URL(+ "../../experiments/analysis-efficiency/signal-entry.mjs",+ import.meta.url,+ ).href;+ const source = `+ import assert from 'node:assert/strict';+ import {verifySignalRows,signalDataRows,verifySignalOracleIdentity} from ${JSON.stringify(moduleUrl)};+ const sha256='a'.repeat(64);+ const rows=Array.from({length:8},(_,i)=>({start:i*100+10,end:i*100+80,byte:72}));+ const uart={kind:'uart',sha256,total_samples:2000,rows:rows.map((_,i)=>({start:i*100,end_exclusive:i*100+90,first_data_sample:i*100+15,last_data_sample:i*100+75,byte:72}))};+ verifySignalRows('uart',rows,uart,sha256);+ verifySignalRows('uart',rows.map((row,i)=>({...row,start:i*100+15,end:i*100+75})),uart,sha256);+ assert.throws(()=>verifySignalRows('uart',[{...rows[0],start:0,end:0},...rows.slice(1)],uart,sha256),/interval/);+ assert.throws(()=>verifySignalRows('uart',[{...rows[0],end:74},...rows.slice(1)],uart,sha256),/interval/);+ assert.throws(()=>verifySignalRows('uart',[{...rows[0],start:16},...rows.slice(1)],uart,sha256),/interval/);+ assert.throws(()=>verifySignalRows('uart',rows,{...uart,rows:uart.rows.map(({first_data_sample,...row})=>row)},sha256),/interval/);+ assert.throws(()=>verifySignalRows('uart',rows.slice(1).concat({start:810,end:880,byte:72}),uart,sha256),/interval/);+ assert.throws(()=>verifySignalRows('uart',rows.slice(1),uart,sha256),/coverage/);+ assert.throws(()=>verifySignalRows('uart',rows,uart,'b'.repeat(64)),/identity/);+ const data=Array.from({length:18},(_,i)=>({start:i*100+10,end:i*100+90,byte:117}));+ const raw=data.map((_,i)=>({byte:117,first_clock:i*100+20,last_clock:i*100+80}));+ const i2c={kind:'i2c',sha256,total_samples:2000,reads:[{bytes:[{byte:165,last_clock:0},...raw],stop:1900}]};+ verifySignalRows('i2c',data,i2c,sha256);+ assert.throws(()=>verifySignalRows('i2c',data.toReversed(),i2c,sha256),/interval/);+ assert.throws(()=>verifySignalRows('i2c',[{...data[0],byte:118},...data.slice(1)],i2c,sha256),/sample bits/);+ assert.throws(()=>verifySignalRows('i2c',data.map((row,i)=>({...row,start:i,end:1999})),i2c,sha256),/interval/);+ assert.throws(()=>verifySignalRows('i2c',[{...data[0],start:0},...data.slice(1)],i2c,sha256),/interval/);+ assert.throws(()=>verifySignalRows('i2c',[...data.slice(0,-1),{...data.at(-1),end:1900}],i2c,sha256),/interval/);+ verifySignalOracleIdentity('sigrok-uart-max3232e.sr',{kind:'uart',sha256:'f797d60d6b8e12d2a1549b9a5a499ef389f44cb9e870e8610af26b57c3b2be76'});+ assert.throws(()=>verifySignalOracleIdentity('sigrok-uart-max3232e.sr',uart),/pinned signal corpus/);+ assert.throws(()=>verifySignalOracleIdentity('sigrok-i2c-nunchuk.sr',{kind:'uart',sha256:'0fa6adb8297d7f3fe5ba19c94ee655c9b0f952a8b5e48a17aaa499a2f6d10d70'}),/pinned signal corpus/);+ assert.throws(()=>signalDataRows('uart','9007199254740993-9007199254740994 uart-1: 48'),/sample interval/);+ `;+ // Checker semantics only; synthesized rows do not establish provider behavior.+ const result = spawnSync(process.execPath, ["--input-type=module", "-"], {+ input: source,+ encoding: "utf8",+ });+ assert.ifError(result.error);+ assert.equal(result.status, 0, result.stderr);+});