Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
test(perf): distinguish Codex tool context and token accounting (#2416)
Ysopwnableee87111009ac
diff --git a/experiments/analysis-efficiency/README.md b/experiments/analysis-efficiency/README.mdindex e41e6b7df..fb645b1e0 100644--- a/experiments/analysis-efficiency/README.md+++ b/experiments/analysis-efficiency/README.md@@ -32,6 +32,11 @@ records two shipped assembly checks, previous-runtime portable Evidence, and fouactual Codex trials. Selected replies shrink substantially; observed model inputtokens increase and latency is mixed, so model-token savings are not claimed.+Follow-up: [model tool context and per-response usage](model-context.md) separates+raw MCP replies from observed client execution/discovery and usage. Optional+trial capture retains tool/usage items while excluding account and ambient state;+existing ephemeral trials keep their default behavior.+The 12 detailed checkpoint receipts were archived byte-for-byte before removalfrom the final PR tree. The archive `review-receipts-00d51e0fc-20261012` is in theprivate experiment lab outside the checkout; its `archive-index.json` recordsdiff --git a/experiments/analysis-efficiency/agent-trial.py b/experiments/analysis-efficiency/agent-trial.pyindex 8392000ca..682be27ec 100644--- a/experiments/analysis-efficiency/agent-trial.py+++ b/experiments/analysis-efficiency/agent-trial.py@@ -11,6 +11,8 @@ import subprocessimport tempfileimport time+from codex_context import capture_tool_context+def file_digest(path):digest = hashlib.sha256()@@ -48,7 +50,20 @@ def main():parser.add_argument('--providers', type=Path)parser.add_argument('--enabled-tools', type=Path)parser.add_argument('--authority', choices=['static', 'browser'], default='static')+ parser.add_argument('--capture-tool-context', action='store_true',+ help='Retain owned tool/usage items; excludes account/ambient state')+ parser.add_argument('--guide-enabled-tools', action='store_true',+ help='Opt-in exact-name discovery guidance for selected REA tools')args = parser.parse_args()+ if args.guide_enabled_tools and args.enabled_tools is None:+ parser.error('--guide-enabled-tools requires --enabled-tools')+ enabled_tools = None+ if args.enabled_tools:+ enabled_tools = json.loads(args.enabled_tools.read_text())+ if not isinstance(enabled_tools, list) or not all(isinstance(name, str) and name for name in enabled_tools) or len(enabled_tools) != len(set(enabled_tools)):+ raise ValueError('enabled-tools must be a unique list of exact tool names')+ if args.guide_enabled_tools and not enabled_tools:+ parser.error('--guide-enabled-tools requires a nonempty selected tool list')def terminate(signum, _frame):# The outer resource scope terminates descendants together. Ensure the# runner's copied credentials are also removed on a resource abort.@@ -130,11 +145,14 @@ def main():'possible and select the requested normalized_result fields before returning ''intermediate output to model context. Keep real Evidence IDs and limitations.')- enabled_tools = None- if args.enabled_tools:- enabled_tools = json.loads(args.enabled_tools.read_text())- if not isinstance(enabled_tools, list) or not all(isinstance(name, str) and name for name in enabled_tools) or len(enabled_tools) != len(set(enabled_tools)):- raise ValueError('enabled-tools must be a unique list of exact tool names')+ if args.guide_enabled_tools:+ instruction += (+ ' Configured REA tool function names for this trial: '+ + json.dumps(['mcp__rea__' + name for name in enabled_tools])+ + '. For discovery select only these exact names from ALL_TOOLS and '+ 'return only their metadata. Preserve the original analyst question, '+ 'dependency order and source Evidence.'+ )quote = json.dumpsconfig = '\n'.join(['approval_policy = "never"', 'sandbox_mode = "read-only"',@@ -183,6 +201,8 @@ def main():(args.output / 'prompt.txt').write_text(prompt)(args.output / 'trial-settings.json').write_text(json.dumps({'mode': args.mode, 'model': args.model, 'reasoning': args.reasoning,+ 'capture_tool_context': args.capture_tool_context,+ 'guide_enabled_tools': args.guide_enabled_tools,'driver_commit': subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=driver_repository, text=True).strip(),'runtime_revision': args.revision,@@ -196,7 +216,7 @@ def main():'package_lock_sha256': file_digest(repository / 'package-lock.json'),'experiment_sources_sha256': {name: hashlib.sha256((experiment / name).read_bytes()).hexdigest()- for name in ['agent-trial.py', 'provider-environment.json']+ for name in ['agent-trial.py', 'codex_context.py', 'provider-environment.json']},'developer_instructions': instruction,'developer_instructions_sha256': hashlib.sha256(instruction.encode()).hexdigest(),@@ -212,7 +232,8 @@ def main():'multi_agent': False, 'code_mode': args.mode == 'client-code','code_mode_only': args.mode == 'client-code'},}, indent=2) + '\n')- invocation = [args.codex, '--no-daemon', 'exec', '--strict-config', '--ephemeral',+ invocation = [args.codex, '--no-daemon', 'exec', '--strict-config',+ *([] if args.capture_tool_context else ['--ephemeral']),'--skip-git-repo-check', '--ignore-rules', '--json','--color', 'never', '-C', str(workspace),'--output-last-message', str(args.output.resolve() / 'answer.txt'), '-']@@ -234,26 +255,28 @@ def main():usage = []tools = []event_types = {}- for line in (args.output / 'events.jsonl').open():- if not line.strip():- continue- event = json.loads(line)- kind = event.get('type', 'unknown')- event_types[kind] = event_types.get(kind, 0) + 1- if 'usage' in event:- usage.append(event['usage'])- item = event.get('item', {})- if item.get('type') == 'mcp_tool_call' and kind == 'item.completed':- tools.append({- **{k: item.get(k) for k in ['server', 'tool', 'arguments', 'status', 'error']},- 'codex_event_result_json_utf8_bytes': len(json.dumps(- item.get('result'), separators=(',', ':'), ensure_ascii=False- ).encode()),- })+ with (args.output / 'events.jsonl').open() as event_file:+ for line in event_file:+ if not line.strip():+ continue+ event = json.loads(line)+ kind = event.get('type', 'unknown')+ event_types[kind] = event_types.get(kind, 0) + 1+ if 'usage' in event:+ usage.append(event['usage'])+ item = event.get('item', {})+ if item.get('type') == 'mcp_tool_call' and kind == 'item.completed':+ tools.append({+ **{k: item.get(k) for k in ['server', 'tool', 'arguments', 'status', 'error']},+ 'codex_event_result_json_utf8_bytes': len(json.dumps(+ item.get('result'), separators=(',', ':'), ensure_ascii=False+ ).encode()),+ })final_target_digest = file_digest(Path(args.artifact_path)) if args.artifact_path else Nonesummary = {'mode': args.mode, 'requested_model': args.model, 'reasoning': args.reasoning,'runtime_revision': args.revision,+ 'capture_tool_context_requested': args.capture_tool_context,'selected_artifact_sha256_before': target_digest,'selected_artifact_sha256_after': final_target_digest,'selected_artifact_bytes_unchanged': target_digest == final_target_digest if target_digest else None,@@ -268,9 +291,13 @@ def main():(args.output / 'summary.json').write_text(json.dumps(summary, indent=2) + '\n')print(json.dumps(summary))finally:- # Always remove copied credentials, config, caches and profile state.- shutil.rmtree(profile)- (args.output / 'cleanup.json').write_text(json.dumps({'profile_removed': not profile.exists()}) + '\n')+ try:+ if args.capture_tool_context:+ capture_tool_context(trial_codex_home, args.output)+ finally:+ # Capture failure must never prevent copied-credential cleanup.+ shutil.rmtree(profile)+ (args.output / 'cleanup.json').write_text(json.dumps({'profile_removed': not profile.exists()}) + '\n')raise SystemExit(124 if timeout else exit_code)diff --git a/experiments/analysis-efficiency/codex_context.py b/experiments/analysis-efficiency/codex_context.pynew file mode 100644index 000000000..cce8bf9cf--- /dev/null+++ b/experiments/analysis-efficiency/codex_context.py@@ -0,0 +1,117 @@+"""Retain owned Codex tool/usage observations without account or ambient state."""++import hashlib+import json+import os+++def token_counts(value):+ if not isinstance(value, dict):+ raise ValueError('Expected a token-count object')+ result = {}+ for key in ['input_tokens', 'cached_input_tokens', 'cache_write_input_tokens',+ 'output_tokens', 'reasoning_output_tokens', 'total_tokens']:+ if key in value:+ count = value[key]+ if not isinstance(count, int) or isinstance(count, bool) or count < 0:+ raise ValueError('Expected nonnegative integer token counts')+ result[key] = count+ return result+++def selected_event(event):+ """Select observed items; these are not the exact submitted model request."""+ if not isinstance(event, dict) or not isinstance(event.get('payload'), dict):+ raise ValueError('Expected an event object with an object payload')+ if not isinstance(event.get('type'), str):+ raise ValueError('Expected an event type string')+ if event.get('timestamp') is not None and not isinstance(event['timestamp'], str):+ raise ValueError('Expected a timestamp string or omission')+ payload = event['payload']+ kind = event.get('type')+ if kind == 'response_item' and payload.get('type') in [+ 'custom_tool_call', 'custom_tool_call_output',+ 'function_call', 'function_call_output',+ ]:+ keys = ['type', 'call_id', 'name', 'input', 'arguments', 'output']+ elif kind == 'token_usage_record':+ keys = ['response_id', 'usage', 'turn_token_usage', 'thread_token_usage']+ payload = {key: token_counts(payload[key]) if key != 'response_id'+ else payload[key] for key in keys if key in payload}+ elif kind == 'event_msg' and payload.get('type') == 'token_count':+ keys = ['type', 'info']+ info = payload.get('info')+ if info is not None:+ if not isinstance(info, dict):+ raise ValueError('Expected token-count info')+ selected_info = {key: token_counts(info[key]) for key in+ ['total_token_usage', 'last_token_usage'] if key in info}+ window = info.get('model_context_window')+ if 'model_context_window' in info:+ if window is not None and (not isinstance(window, int) or isinstance(window, bool) or window <= 0):+ raise ValueError('Expected a positive model context window')+ selected_info['model_context_window'] = window+ payload = {'type': 'token_count', 'info': selected_info}+ else:+ payload = {'type': 'token_count', 'info': None}+ else:+ return None+ return {+ 'timestamp': event.get('timestamp'),+ 'type': kind,+ 'payload': {key: payload[key] for key in keys if key in payload},+ }+++def capture_tool_context(codex_home, output):+ """Capture only an explicitly selected trial's rollouts; never its auth/config."""+ sessions = codex_home / 'sessions'+ blocked = sessions.is_symlink()+ sources = [] if blocked else sorted(sessions.rglob('*.jsonl'))+ target = output / 'codex-tool-context.jsonl'+ source_receipts = []+ errors = [{'source': 'sessions', 'reason': 'not_an_owned_directory'}] if blocked else []+ retained = 0+ with os.fdopen(os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600),+ 'w', encoding='utf-8') as result:+ for source in sources:+ name = source.relative_to(codex_home).as_posix()+ if source.is_symlink() or not source.is_file():+ errors.append({'source': name, 'reason': 'not_an_owned_regular_file'})+ continue+ digest = hashlib.sha256()+ counts = {}+ with source.open('rb') as lines:+ for ordinal, line in enumerate(lines, 1):+ digest.update(line)+ try:+ event = json.loads(line)+ selected = selected_event(event)+ except ValueError as error:+ errors.append({'source': name, 'line': ordinal,+ 'reason': type(error).__name__})+ continue+ kind = event.get('type')+ counts[str(kind)] = counts.get(str(kind), 0) + 1+ if selected is not None:+ result.write(json.dumps(selected, ensure_ascii=False) + '\n')+ retained += 1+ source_receipts.append({'source': name, 'sha256': digest.hexdigest(),+ 'event_counts': counts})+ retained_digest = hashlib.sha256()+ with target.open('rb') as captured:+ for chunk in iter(lambda: captured.read(1024 * 1024), b''):+ retained_digest.update(chunk)+ receipt = {+ 'status': 'partial' if errors else 'captured' if source_receipts else 'unavailable',+ 'retained_file': target.name,+ 'retained_sha256': retained_digest.hexdigest(),+ 'retained_events': retained, 'sources': source_receipts, 'errors': errors,+ 'scope': 'Owned client tool-call/reply and per-response usage items, not '+ 'the exact submitted model request. Session/account metadata, '+ 'world state, user/developer messages, encrypted reasoning and '+ 'internal passthrough are omitted. Authentication, configuration '+ 'and ambient environment are never copied by this capture.',+ }+ (output / 'context-receipt.json').write_text(json.dumps(receipt, indent=2) + '\n')+ return receiptdiff --git a/experiments/analysis-efficiency/model-context.md b/experiments/analysis-efficiency/model-context.mdnew file mode 100644index 000000000..3d878beed--- /dev/null+++ b/experiments/analysis-efficiency/model-context.md@@ -0,0 +1,209 @@+# Observe the client boundary before claiming token savings++Continues [#2016](https://github.com/morluto/rea/issues/2016) after the managed+summary [PR #2413](https://github.com/morluto/rea/pull/2413) merged as+`82cfcee19f18b0eeae9a330ba04e0411cdf85225`. That opt-in view has verified smaller+delivery and complete-default migration, while its original real-model cohort+does not establish token savings. This follow-up measures why raw reply bytes+and aggregate model usage cannot be treated as interchangeable observations.++## Concrete diagnostic observation++An individual real Codex CLI 0.161.0 control uses frozen runtime+`ae8051302b5834322fa00cfa31c968ae74ab6ed9`, the unchanged ShareX identity question,+requested `gpt-5.6-terra` high and the same two enabled REA tools. A private+diagnostic copy removes the original runner's explicit `--ephemeral` flag.+Authentication/profile cleanup still runs; only selected tool/usage items are+retained after filtering the owned rollout. Its source rollout SHA-256 is+`d4df41d6187880d177f00b7533d85ad032c9c4ec502a39e7d5a566180e10461b` and the retained+filtered-item SHA-256 is+`0d5b4fb67c5674d1ba14ac32ae37e9a72b0368f4d9f5f9f81eb4d13193cda260`.+The original whole-copy is removed; account metadata, world state, messages and+internal passthrough are not retained in the filtered receipt.++Despite the requested `direct` configuration label, the observed client uses+three JavaScript `exec` calls. First it filters `ALL_TOOLS` with a broad+`/REA|reverse|artifact|managed|assembly|metadata/i` expression and prints matching+metadata. The client reports truncation of a 164,005-token original output.+This is the client's output measurement, not billed input or a complete inventory.+Next it invokes complete managed inspection but selects requested normalized+identity/classification fields before printing. Finally it invokes native-boundary+inspection and prints selected fields. The actual original complete MCP replies+remain available in the separate public-event transcript.++The persisted custom-tool output values are 41,704 / 2,154 / 13,489 UTF-8 bytes.+These are the serialized recorded output **values**, not raw wire bytes or exact+submitted model requests. The trace reports four per-response usage observations:++| Response | Input | Cached input | Output |+| -------- | -----: | -----------: | -----: |+| 1 | 10,400 | 0 | 132 |+| 2 | 19,992 | 9,984 | 456 |+| 3 | 21,242 | 19,200 | 251 |+| 4 | 24,799 | 20,224 | 1,535 |++Their totals match public turn usage: 76,433 input, 49,408 cached, 27,025 uncached+and 2,374 output, with 61.436 seconds observed wall time. Two MCP calls alone do+not describe the client's discovery/execution rounds. This proves the observed+control can project a complete result before printing it; it does not establish+that every earlier control/candidate did the same, or attribute every token+difference to discovery. The earlier `direct` label specifies requested client+configuration, not a verified absence of an executor.++Initial transcript-capture attempts retained empty source lists because the+original runner intentionally passed `--ephemeral`. A first private flag-removal+edit missed its newline and failed; that candidate remains an unsuccessful+capture attempt. The later corrected control is an individual diagnostic,+not a comparable pair with that candidate. Other exploratory aggregates remain+private and are not silently folded into the original four-trial cohort.++## Reproducible optional capture++`agent-trial.py --capture-tool-context` omits `--ephemeral` only for that explicitly+selected temporary trial. Its default is unchanged. The profile still disappears+in `finally`, including when capture fails. The capture reads only its owned+`sessions/*.jsonl` records, rejects symlink sources, and retains:++- Custom/function tool calls and replies, preserving call IDs and exact values.+- Per-response token counts and cumulative counts, with original response IDs.+- Client token-count events, retaining explicit unknown `info: null`.++It excludes session/account metadata, world state, user/developer messages,+encrypted reasoning, internal passthrough and unknown nested usage fields. It+does not copy authentication, configuration or the ambient environment. Receipts+record source/retained digests, event counts, malformed-line failures and missing+rollouts as unavailable. Capture is an observation of persisted client items,+not proof of the exact request accepted by a remote model service.++Run the same frozen runtime, question, tool selection and capture mode on both+sides. Preserve source/settings hashes and separate requested versus observed+behavior. Use the existing resource wrapper with unchanged one CPU, 4 GiB hard+memory, 3 GiB early-stop, no swap and task/host safeguards. Final physical counters+remain unknown when transient cgroups disappear; this instrumentation does not+repair that measurement limitation.++The focused Python tests run through the existing CI `test:python` gate.+They cover account/ambient exclusion while preserving caller data, malformed and+missing records, foreign symlinks, and capture failure without copied-credential+leakage. The cleanup test simulates a CLI and establishes only cleanup semantics;+it is not real-model/provider evidence.++## An explicit discovery-guidance experiment++`--guide-enabled-tools` is another opt-in experiment flag. It requires a nonempty+`--enabled-tools` list and adds the exact corresponding `mcp__rea__...` function+names to the temporary developer instructions, requesting only exact-name+metadata discovery. It preserves the analyst question, operation order, tool+selection and provider inputs. The default instruction is unchanged. Missing or+invalid selected tools fail before creating an output/profile. The test gate+also verifies the new prerequisite without acquiring a profile.++Compare guided and unguided runs on the **same** frozen REA runtime with matching+capture mode and requested facts. A smaller discovery response or additional+hint is not itself a task/token success: grade answers, count all response usage,+preserve failed/truncated runs and repeat both orders before recommending this+guidance. This is a caller-side experiment, not a new REA tool or provider preset.++## First committed capture pair++Checkpoint `9ebdcc458c399b9bdecaaca2ef47422888231181` runs the unchanged question+against frozen control `ae8051302b5834322fa00cfa31c968ae74ab6ed9` and candidate+`8bc1601ca51e417ca472d90ea0367e9418a1a0f2`, both with capture enabled and no tool+guidance. Both captures succeed with 10 retained events, unchanged source bytes,+removed profiles, two successful MCP calls and passing selected factual checks.+Full grader gates remain false because final physical resource counters are+unavailable; factual subsets do not override that limitation.++| Side | Input | Cached | Uncached | Output | Wall seconds |+| ----------------- | -----: | -----: | -------: | -----: | -----------: |+| Complete control | 52,656 | 27,136 | 25,520 | 2,015 | 57.736 |+| Summary candidate | 56,655 | 29,184 | 27,471 | 1,534 | 55.055 |++Both clients make two executor calls and three response-usage observations.+First they print broad discovered metadata, with truncation. Then each executor+sequentially performs both MCP operations, preserving selected-target dependency+order. The control prints a 2,086-byte output value; the candidate prints 22,770+bytes. Their discovery output values are 41,723 and 41,675 bytes. These are+recorded serialized values and remain distinct from raw MCP envelopes, billed+tokens and exact submitted requests. Both requested modes are `direct`; the+actual executor use is separately observed.++The candidate's smaller raw metadata reply does not result in a smaller printed+client value in this pair. Candidate input/uncached tokens remain higher. The+observed projection and round counts are useful constraints for the next+experiment; this one pair does not attribute all usage differences to either+formatting or discovery, or establish a reliable speedup.++## Same-runtime guided discovery, repeated in both orders++Checkpoint `e20bd0e2a905f06bc8c56fe04b0e61f3da88636a` runs four actual Codex trials+in unguided/guided/guided/unguided order. All use the **same** frozen candidate+runtime, artifact, original question, requested model/reasoning, two selected+tools, provider settings and capture mode. Only the explicit discovery-guidance+treatment changes. Every trial selects artifact summary and makes the same two+successful MCP calls in dependency order; source bytes stay unchanged, captures+succeed and profiles are removed.++| Trial | Input | Cached | Uncached | Output | Seconds | Observed responses |+| ---------- | ------: | -----: | -------: | -----: | ------: | -----------------: |+| Unguided 1 | 77,251 | 50,432 | 26,819 | 1,931 | 62.066 | 4 |+| Guided 1 | 39,175 | 23,040 | 16,135 | 1,763 | 55.658 | 3 |+| Guided 2 | 58,336 | 38,144 | 20,192 | 1,626 | 50.685 | 4 |+| Unguided 2 | 103,301 | 72,704 | 30,597 | 1,983 | 60.336 | 5 |++Both guided discovery output values repeat at 12,336 bytes without truncation,+versus 41,709 bytes with truncation on both unguided trials. The recorded+executor/output usage explains observable discovery and round differences; it+does not certify exact remote request contents or attribute every token to the+hint. CPU samples are 6.303–6.554 seconds and peak family memory 684,519,424–+712,028,160 bytes, with no guard abort. Final resource counters remain unknown.+These samples do not establish a memory or CPU improvement.++Unguided 1 omits `classification.container` and fails the selected factual gate;+it is not a fully acceptable comparison answer. Guided 1 and Unguided 2 pass+that gate. Guided 2 initially fails exact coverage-object equality because it+adds `module_ref_count: 0` and `pinvoke_import_count: 0` to the explicitly named+native scope. Both numbers exactly match the producer's returned arrays. The+grader now explicitly verifies only these two known additional counts, including+integer type, while still requiring all original coverage facts. Unknown extra+claims, conflicting aliases, wrong counts, boolean/string counts and omitted+issues continue to fail. Source coverage remains unchanged; returned-row counts+do not establish global absence. Original failure receipts remain preserved;+the regrade separately records its new source hash. Guided 2 then passes selected+facts, but every **full** gate still fails unknown final physical resource checks.++The second comparison has two factually acceptable answers: observed guided input+falls about 43.5%, uncached input about 34.0%, and wall time about 16.0%. The first+comparison has an incomplete unguided answer and is reported without counting it+as equivalent completed work. Two trials per treatment on one question/application+remain a preliminary signal, not a general default recommendation. The hint stays+opt-in; wider questions/domains, count/coverage representation and client round+behavior need further verification before promoting it into general guidance.++## Real Android follow-up: the primary outcomes are mixed++Checkpoint `22834c7d73056e2480b426848f58c31af0c307bc` checks the pinned real+NewPipe 0.29.1 APK using actual configured JADX 0.7.1/Java, the original Android+question and the same frozen candidate runtime. First a fresh SDK workflow+validates three live advertised operations and produces the comparison facts:+package/version identity, every case-sensitive MainActivity match and all+reported onCreate entries, preserving signatures, overload indices and null DEX+descriptors. It takes 26.961 seconds; the owned scope samples 1,234,735,104 bytes.++Both subsequent actual Codex trials use the same three selected tool names,+capture mode, model/reasoning, question and provider settings. Only discovery+guidance changes. Each makes three successful MCP calls and five observed model+responses, passes selected facts against the fresh actual producer, preserves+source bytes and removes its temporary profile. No APK execution is requested.++| Trial | Input | Cached | Uncached | Output | Wall seconds | Sampled peak bytes |+| -------- | -----: | -----: | -------: | -----: | -----------: | -----------------: |+| Unguided | 99,913 | 80,640 | 19,273 | 1,527 | 66.797 | 1,405,829,120 |+| Guided | 89,220 | 62,464 | 26,756 | 2,142 | 76.936 | 1,345,576,960 |++Total input falls about 10.7%, but uncached input increases and wall time is+longer. One pair does not demonstrate improved Android performance or memory.+No guard abort occurs; final counters remain unknown and full resource gates+remain false. This second domain is a concrete counterexample to generalizing+the favorable managed comparison. Keep both flags optional and continue+task-specific experiments before changing installed/default agent guidance.diff --git a/experiments/analysis-efficiency/review-model.py b/experiments/analysis-efficiency/review-model.pyindex 3c56aabb2..a61bb0984 100644--- a/experiments/analysis-efficiency/review-model.py+++ b/experiments/analysis-efficiency/review-model.py@@ -52,6 +52,21 @@ def named_scope(value, names):return {'conflicting_reported_scopes': selected}+def scope_with_verified_counts(value, expected_scope, returned_counts):+ """Validate explicit extra row counts; never erase unknown or conflicting facts."""+ if not isinstance(value, dict):+ return value, {}+ scope = dict(value)+ checks = {}+ for name, expected in returned_counts.items():+ if name in scope:+ reported = scope[name]+ checks[name] = type(reported) is int and reported == expected+ if name not in expected_scope:+ del scope[name]+ return scope, checks++def main():parser = argparse.ArgumentParser()parser.add_argument('--domain', choices=['wasm', 'evm', 'android', 'managed', 'managed-nonempty', 'firmware', 'inventory', 'javascript', 'native', 'browser'], required=True)@@ -134,7 +149,13 @@ def main():for key in ['module_refs', 'pinvoke_imports']:check(key, answer.get('native_boundaries', {}).get(key), reference['boundaries'][key])check('artifact_coverage', named_scope(answer.get('coverage'), ['artifact', 'managed_artifact', 'managed_metadata']), n['coverage'])- check('native_boundaries_coverage', named_scope(answer.get('coverage'), ['native_boundaries', 'native_boundary_declarations']), reference['boundaries']['coverage'])+ boundary_scope, counts = scope_with_verified_counts(+ named_scope(answer.get('coverage'), ['native_boundaries', 'native_boundary_declarations']),+ reference['boundaries']['coverage'],+ {'module_ref_count': len(reference['boundaries']['module_refs']),+ 'pinvoke_import_count': len(reference['boundaries']['pinvoke_imports'])})+ check('native_boundaries_coverage', boundary_scope, reference['boundaries']['coverage'])+ checks.update({'native_coverage_' + name: valid for name, valid in counts.items()})elif domain == 'managed-nonempty':n = reference['boundaries']check('module_mvid', answer.get('module_mvid'), n['module']['mvid'])diff --git a/experiments/analysis-efficiency/test_codex_context.py b/experiments/analysis-efficiency/test_codex_context.pynew file mode 100644index 000000000..37df83a7c--- /dev/null+++ b/experiments/analysis-efficiency/test_codex_context.py@@ -0,0 +1,175 @@+"""Distinct capture/cleanup failures; simulated CLI is not real-model evidence."""++import importlib.util+import io+import json+import os+import runpy+import sys+from pathlib import Path+import tempfile+import unittest+from unittest.mock import Mock, patch++from codex_context import capture_tool_context, selected_event+++class ManagedCoverageTests(unittest.TestCase):+ def test_verifies_extra_counts_and_preserves_unknown_fields(self):+ review = runpy.run_path(str(Path(__file__).with_name('review-model.py')))+ project = review['scope_with_verified_counts']+ expected = {'state': 'partial', 'issues': ['unreturned rows remain unknown']}+ original = {**expected, 'module_ref_count': 0, 'pinvoke_import_count': 2}+ scope, checks = project(original, expected, {'module_ref_count': 0, 'pinvoke_import_count': 2})+ self.assertEqual(scope, expected)+ self.assertTrue(all(checks.values()))+ self.assertEqual(original['pinvoke_import_count'], 2)+ for count in [3, '2', True]:+ scope, checks = project({**expected, 'pinvoke_import_count': count}, expected, {'pinvoke_import_count': 2})+ self.assertFalse(checks['pinvoke_import_count'])+ _, checks = project({**expected, 'module_ref_count': False}, {**expected, 'module_ref_count': 0}, {'module_ref_count': 0})+ self.assertFalse(checks['module_ref_count'])+ scope, _ = project({**expected, 'unsupported_claim': True}, expected, {})+ self.assertNotEqual(scope, expected)+ scope, _ = project({'state': 'partial', 'module_ref_count': 0}, expected, {'module_ref_count': 0})+ self.assertNotEqual(scope, expected)+ ambiguous = review['named_scope']({'native_boundaries': expected, 'native_boundary_declarations': {'state': 'complete'}}, ['native_boundaries', 'native_boundary_declarations'])+ scope, _ = project(ambiguous, expected, {'module_ref_count': 0})+ self.assertNotEqual(scope, expected)+++class ContextCaptureTests(unittest.TestCase):+ def test_filters_ambient_account_and_nested_usage_state(self):+ with tempfile.TemporaryDirectory() as directory:+ root = Path(directory)+ home = root / 'home'+ sessions = home / 'sessions'+ sessions.mkdir(parents=True)+ output = root / 'output'+ output.mkdir()+ events = [+ {'type': 'world_state', 'payload': {'state': {'env': 'ambient-marker'}}},+ {'type': 'session_meta', 'payload': {'creator_account_id': 'account-marker'}},+ {'type': 'response_item', 'payload': {+ 'type': 'custom_tool_call_output', 'call_id': 'original-call',+ 'output': 'caller-selected evidence and paths',+ 'internal_chat_message_metadata_passthrough': 'private-marker'}},+ {'type': 'token_usage_record', 'payload': {+ 'response_id': 'original-response', 'usage': {+ 'input_tokens': 12, 'cached_input_tokens': 4,+ 'environment': 'nested-ambient-marker'},+ 'transport_auth': 'auth-marker'}},+ ]+ (sessions / 'rollout.jsonl').write_text(''.join(json.dumps(x) + '\n' for x in events))+ (home / 'auth.json').write_text('copied-credential-marker')+ receipt = capture_tool_context(home, output)+ text = (output / 'codex-tool-context.jsonl').read_text()+ self.assertEqual(receipt['status'], 'captured')+ self.assertEqual(receipt['retained_events'], 2)+ for marker in ['ambient-marker', 'account-marker', 'private-marker',+ 'auth-marker', 'copied-credential-marker']:+ self.assertNotIn(marker, text)+ self.assertIn('caller-selected evidence and paths', text)+ self.assertIn('original-response', text)+ self.assertEqual((output / 'codex-tool-context.jsonl').stat().st_mode & 0o777, 0o600)++ def test_missing_rollouts_are_unknown_and_malformed_lines_remain_partial(self):+ with tempfile.TemporaryDirectory() as directory:+ root = Path(directory)+ output = root / 'missing'+ output.mkdir()+ self.assertEqual(capture_tool_context(root / 'home', output)['status'], 'unavailable')+ sessions = root / 'home' / 'sessions'+ sessions.mkdir(parents=True)+ (sessions / 'rollout.jsonl').write_text('not-json\n' + json.dumps({+ 'type': 'event_msg', 'payload': {'type': 'token_count', 'info': None}}) + '\n')+ partial = root / 'partial'+ partial.mkdir()+ receipt = capture_tool_context(root / 'home', partial)+ self.assertEqual(receipt['status'], 'partial')+ self.assertEqual(receipt['errors'][0]['line'], 1)+ self.assertEqual(receipt['retained_events'], 1)+ with self.assertRaises(ValueError):+ selected_event({'type': 'token_usage_record', 'payload': {'usage': {'input_tokens': True}}})+ with self.assertRaises(ValueError):+ selected_event({'type': {'environment': 'unexpected'}, 'payload': {}})+ with self.assertRaises(ValueError):+ selected_event({'type': 'response_item', 'timestamp': {'environment': 'unexpected'}, 'payload': {}})+ unknown = selected_event({'type': 'event_msg', 'payload': {'type': 'token_count', 'info': {'model_context_window': None}}})+ self.assertIsNone(unknown['payload']['info']['model_context_window'])++ def test_capture_failure_still_removes_trial_credentials_and_profile(self):+ spec = importlib.util.spec_from_file_location('agent_trial', Path(__file__).with_name('agent-trial.py'))+ agent = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(agent)+ with tempfile.TemporaryDirectory() as directory:+ root = Path(directory)+ runtime = root / 'runtime'+ (runtime / 'scripts').mkdir(parents=True)+ (runtime / 'scripts' / 'rea.mjs').write_text('')+ (runtime / 'package-lock.json').write_text('{}')+ auth_home = root / 'source-auth'+ auth_home.mkdir()+ (auth_home / 'auth.json').write_text('synthetic-credential')+ question = root / 'question.txt'+ question.write_text('Observe the declared fixture.')+ output = root / 'trial'+ child = Mock(stdin=io.StringIO())+ child.wait.return_value = 0+ arguments = ['agent-trial.py', '--runtime-root', str(runtime), '--revision', 'fixture',+ '--prompt', str(question), '--output', str(output), '--capture-tool-context']+ with patch.dict(os.environ, {'CODEX_HOME': str(auth_home)}, clear=True), \+ patch.object(sys, 'argv', arguments), \+ patch.object(agent.shutil, 'which', return_value='/fixture/node'), \+ patch.object(agent.subprocess, 'check_output', return_value='v24.18.0\n'), \+ patch.object(agent.subprocess, 'Popen', return_value=child), \+ patch.object(agent, 'capture_tool_context', side_effect=OSError('fixture capture failure')), \+ patch('builtins.print'):+ with self.assertRaisesRegex(OSError, 'fixture capture failure'):+ agent.main()+ self.assertFalse(list(output.glob('profile-*')))+ self.assertTrue(json.loads((output / 'cleanup.json').read_text())['profile_removed'])+ self.assertEqual((auth_home / 'auth.json').read_text(), 'synthetic-credential')++ def test_refuses_foreign_session_directory_and_symlink_records(self):+ with tempfile.TemporaryDirectory() as directory:+ root = Path(directory)+ home = root / 'home'+ home.mkdir()+ foreign = root / 'foreign'+ foreign.mkdir()+ (foreign / 'rollout.jsonl').write_text('private marker')+ (home / 'sessions').symlink_to(foreign, target_is_directory=True)+ output = root / 'directory-result'+ output.mkdir()+ receipt = capture_tool_context(home, output)+ self.assertEqual(receipt['status'], 'partial')+ self.assertEqual(receipt['retained_events'], 0)+ (home / 'sessions').unlink()+ (home / 'sessions').mkdir()+ (home / 'sessions' / 'linked.jsonl').symlink_to(foreign / 'rollout.jsonl')+ output = root / 'file-result'+ output.mkdir()+ receipt = capture_tool_context(home, output)+ self.assertEqual(receipt['status'], 'partial')+ self.assertEqual(receipt['errors'][0]['reason'], 'not_an_owned_regular_file')+ self.assertEqual((foreign / 'rollout.jsonl').read_text(), 'private marker')++ def test_guidance_requires_selected_tools_before_acquiring_a_profile(self):+ spec = importlib.util.spec_from_file_location('agent_trial', Path(__file__).with_name('agent-trial.py'))+ agent = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(agent)+ with tempfile.TemporaryDirectory() as directory:+ root = Path(directory)+ output = root / 'trial'+ arguments = ['agent-trial.py', '--runtime-root', str(root), '--revision', 'fixture',+ '--prompt', str(root / 'question'), '--output', str(output), '--guide-enabled-tools']+ with patch.object(sys, 'argv', arguments), patch.object(sys, 'stderr', io.StringIO()):+ with self.assertRaises(SystemExit) as raised:+ agent.main()+ self.assertEqual(raised.exception.code, 2)+ self.assertFalse(output.exists())+++if __name__ == '__main__':+ unittest.main()diff --git a/package.json b/package.jsonindex 5635c6d3c..327c14315 100644--- a/package.json+++ b/package.json@@ -130,7 +130,7 @@"test": "npm run test:with-build","test:with-build": "npm run test:prepare && vitest run","test:run": "vitest run",- "test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py -v",+ "test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py -v && python3 -m unittest discover -s experiments/analysis-efficiency -p test_codex_context.py -v","test:local": "node scripts/run-development-tests.mjs local","test:fast": "vitest run --project domain --project services --project adapters --project composition --project conformance --project evaluation","pretest:boundary": "npm run test:prepare",