Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

test(perf): distinguish Codex tool context and token accounting (#2416)

Ysopwnableee87111009ac
diff --git a/experiments/analysis-efficiency/README.md b/experiments/analysis-efficiency/README.md
index e41e6b7df..fb645b1e0 100644
--- a/experiments/analysis-efficiency/README.md
+++ b/experiments/analysis-efficiency/README.md
@@ -32,6 +32,11 @@ records two shipped assembly checks, previous-runtime portable Evidence, and fou
actual Codex trials. Selected replies shrink substantially; observed model input
tokens increase and latency is mixed, so model-token savings are not claimed.
+Follow-up: [model tool context and per-response usage](model-context.md) separates
+raw MCP replies from observed client execution/discovery and usage. Optional
+trial capture retains tool/usage items while excluding account and ambient state;
+existing ephemeral trials keep their default behavior.
+
The 12 detailed checkpoint receipts were archived byte-for-byte before removal
from the final PR tree. The archive `review-receipts-00d51e0fc-20261012` is in the
private experiment lab outside the checkout; its `archive-index.json` records
diff --git a/experiments/analysis-efficiency/agent-trial.py b/experiments/analysis-efficiency/agent-trial.py
index 8392000ca..682be27ec 100644
--- a/experiments/analysis-efficiency/agent-trial.py
+++ b/experiments/analysis-efficiency/agent-trial.py
@@ -11,6 +11,8 @@ import subprocess
import tempfile
import time
+from codex_context import capture_tool_context
+
def file_digest(path):
digest = hashlib.sha256()
@@ -48,7 +50,20 @@ def main():
parser.add_argument('--providers', type=Path)
parser.add_argument('--enabled-tools', type=Path)
parser.add_argument('--authority', choices=['static', 'browser'], default='static')
+ parser.add_argument('--capture-tool-context', action='store_true',
+ help='Retain owned tool/usage items; excludes account/ambient state')
+ parser.add_argument('--guide-enabled-tools', action='store_true',
+ help='Opt-in exact-name discovery guidance for selected REA tools')
args = parser.parse_args()
+ if args.guide_enabled_tools and args.enabled_tools is None:
+ parser.error('--guide-enabled-tools requires --enabled-tools')
+ enabled_tools = None
+ if args.enabled_tools:
+ enabled_tools = json.loads(args.enabled_tools.read_text())
+ if not isinstance(enabled_tools, list) or not all(isinstance(name, str) and name for name in enabled_tools) or len(enabled_tools) != len(set(enabled_tools)):
+ raise ValueError('enabled-tools must be a unique list of exact tool names')
+ if args.guide_enabled_tools and not enabled_tools:
+ parser.error('--guide-enabled-tools requires a nonempty selected tool list')
def terminate(signum, _frame):
# The outer resource scope terminates descendants together. Ensure the
# runner's copied credentials are also removed on a resource abort.
@@ -130,11 +145,14 @@ def main():
'possible and select the requested normalized_result fields before returning '
'intermediate output to model context. Keep real Evidence IDs and limitations.'
)
- enabled_tools = None
- if args.enabled_tools:
- enabled_tools = json.loads(args.enabled_tools.read_text())
- if not isinstance(enabled_tools, list) or not all(isinstance(name, str) and name for name in enabled_tools) or len(enabled_tools) != len(set(enabled_tools)):
- raise ValueError('enabled-tools must be a unique list of exact tool names')
+ if args.guide_enabled_tools:
+ instruction += (
+ ' Configured REA tool function names for this trial: '
+ + json.dumps(['mcp__rea__' + name for name in enabled_tools])
+ + '. For discovery select only these exact names from ALL_TOOLS and '
+ 'return only their metadata. Preserve the original analyst question, '
+ 'dependency order and source Evidence.'
+ )
quote = json.dumps
config = '\n'.join([
'approval_policy = "never"', 'sandbox_mode = "read-only"',
@@ -183,6 +201,8 @@ def main():
(args.output / 'prompt.txt').write_text(prompt)
(args.output / 'trial-settings.json').write_text(json.dumps({
'mode': args.mode, 'model': args.model, 'reasoning': args.reasoning,
+ 'capture_tool_context': args.capture_tool_context,
+ 'guide_enabled_tools': args.guide_enabled_tools,
'driver_commit': subprocess.check_output(
['git', 'rev-parse', 'HEAD'], cwd=driver_repository, text=True).strip(),
'runtime_revision': args.revision,
@@ -196,7 +216,7 @@ def main():
'package_lock_sha256': file_digest(repository / 'package-lock.json'),
'experiment_sources_sha256': {
name: hashlib.sha256((experiment / name).read_bytes()).hexdigest()
- for name in ['agent-trial.py', 'provider-environment.json']
+ for name in ['agent-trial.py', 'codex_context.py', 'provider-environment.json']
},
'developer_instructions': instruction,
'developer_instructions_sha256': hashlib.sha256(instruction.encode()).hexdigest(),
@@ -212,7 +232,8 @@ def main():
'multi_agent': False, 'code_mode': args.mode == 'client-code',
'code_mode_only': args.mode == 'client-code'},
}, indent=2) + '\n')
- invocation = [args.codex, '--no-daemon', 'exec', '--strict-config', '--ephemeral',
+ invocation = [args.codex, '--no-daemon', 'exec', '--strict-config',
+ *([] if args.capture_tool_context else ['--ephemeral']),
'--skip-git-repo-check', '--ignore-rules', '--json',
'--color', 'never', '-C', str(workspace),
'--output-last-message', str(args.output.resolve() / 'answer.txt'), '-']
@@ -234,26 +255,28 @@ def main():
usage = []
tools = []
event_types = {}
- for line in (args.output / 'events.jsonl').open():
- if not line.strip():
- continue
- event = json.loads(line)
- kind = event.get('type', 'unknown')
- event_types[kind] = event_types.get(kind, 0) + 1
- if 'usage' in event:
- usage.append(event['usage'])
- item = event.get('item', {})
- if item.get('type') == 'mcp_tool_call' and kind == 'item.completed':
- tools.append({
- **{k: item.get(k) for k in ['server', 'tool', 'arguments', 'status', 'error']},
- 'codex_event_result_json_utf8_bytes': len(json.dumps(
- item.get('result'), separators=(',', ':'), ensure_ascii=False
- ).encode()),
- })
+ with (args.output / 'events.jsonl').open() as event_file:
+ for line in event_file:
+ if not line.strip():
+ continue
+ event = json.loads(line)
+ kind = event.get('type', 'unknown')
+ event_types[kind] = event_types.get(kind, 0) + 1
+ if 'usage' in event:
+ usage.append(event['usage'])
+ item = event.get('item', {})
+ if item.get('type') == 'mcp_tool_call' and kind == 'item.completed':
+ tools.append({
+ **{k: item.get(k) for k in ['server', 'tool', 'arguments', 'status', 'error']},
+ 'codex_event_result_json_utf8_bytes': len(json.dumps(
+ item.get('result'), separators=(',', ':'), ensure_ascii=False
+ ).encode()),
+ })
final_target_digest = file_digest(Path(args.artifact_path)) if args.artifact_path else None
summary = {
'mode': args.mode, 'requested_model': args.model, 'reasoning': args.reasoning,
'runtime_revision': args.revision,
+ 'capture_tool_context_requested': args.capture_tool_context,
'selected_artifact_sha256_before': target_digest,
'selected_artifact_sha256_after': final_target_digest,
'selected_artifact_bytes_unchanged': target_digest == final_target_digest if target_digest else None,
@@ -268,9 +291,13 @@ def main():
(args.output / 'summary.json').write_text(json.dumps(summary, indent=2) + '\n')
print(json.dumps(summary))
finally:
- # Always remove copied credentials, config, caches and profile state.
- shutil.rmtree(profile)
- (args.output / 'cleanup.json').write_text(json.dumps({'profile_removed': not profile.exists()}) + '\n')
+ try:
+ if args.capture_tool_context:
+ capture_tool_context(trial_codex_home, args.output)
+ finally:
+ # Capture failure must never prevent copied-credential cleanup.
+ shutil.rmtree(profile)
+ (args.output / 'cleanup.json').write_text(json.dumps({'profile_removed': not profile.exists()}) + '\n')
raise SystemExit(124 if timeout else exit_code)
diff --git a/experiments/analysis-efficiency/codex_context.py b/experiments/analysis-efficiency/codex_context.py
new file mode 100644
index 000000000..cce8bf9cf
--- /dev/null
+++ b/experiments/analysis-efficiency/codex_context.py
@@ -0,0 +1,117 @@
+"""Retain owned Codex tool/usage observations without account or ambient state."""
+
+import hashlib
+import json
+import os
+
+
+def token_counts(value):
+ if not isinstance(value, dict):
+ raise ValueError('Expected a token-count object')
+ result = {}
+ for key in ['input_tokens', 'cached_input_tokens', 'cache_write_input_tokens',
+ 'output_tokens', 'reasoning_output_tokens', 'total_tokens']:
+ if key in value:
+ count = value[key]
+ if not isinstance(count, int) or isinstance(count, bool) or count < 0:
+ raise ValueError('Expected nonnegative integer token counts')
+ result[key] = count
+ return result
+
+
+def selected_event(event):
+ """Select observed items; these are not the exact submitted model request."""
+ if not isinstance(event, dict) or not isinstance(event.get('payload'), dict):
+ raise ValueError('Expected an event object with an object payload')
+ if not isinstance(event.get('type'), str):
+ raise ValueError('Expected an event type string')
+ if event.get('timestamp') is not None and not isinstance(event['timestamp'], str):
+ raise ValueError('Expected a timestamp string or omission')
+ payload = event['payload']
+ kind = event.get('type')
+ if kind == 'response_item' and payload.get('type') in [
+ 'custom_tool_call', 'custom_tool_call_output',
+ 'function_call', 'function_call_output',
+ ]:
+ keys = ['type', 'call_id', 'name', 'input', 'arguments', 'output']
+ elif kind == 'token_usage_record':
+ keys = ['response_id', 'usage', 'turn_token_usage', 'thread_token_usage']
+ payload = {key: token_counts(payload[key]) if key != 'response_id'
+ else payload[key] for key in keys if key in payload}
+ elif kind == 'event_msg' and payload.get('type') == 'token_count':
+ keys = ['type', 'info']
+ info = payload.get('info')
+ if info is not None:
+ if not isinstance(info, dict):
+ raise ValueError('Expected token-count info')
+ selected_info = {key: token_counts(info[key]) for key in
+ ['total_token_usage', 'last_token_usage'] if key in info}
+ window = info.get('model_context_window')
+ if 'model_context_window' in info:
+ if window is not None and (not isinstance(window, int) or isinstance(window, bool) or window <= 0):
+ raise ValueError('Expected a positive model context window')
+ selected_info['model_context_window'] = window
+ payload = {'type': 'token_count', 'info': selected_info}
+ else:
+ payload = {'type': 'token_count', 'info': None}
+ else:
+ return None
+ return {
+ 'timestamp': event.get('timestamp'),
+ 'type': kind,
+ 'payload': {key: payload[key] for key in keys if key in payload},
+ }
+
+
+def capture_tool_context(codex_home, output):
+ """Capture only an explicitly selected trial's rollouts; never its auth/config."""
+ sessions = codex_home / 'sessions'
+ blocked = sessions.is_symlink()
+ sources = [] if blocked else sorted(sessions.rglob('*.jsonl'))
+ target = output / 'codex-tool-context.jsonl'
+ source_receipts = []
+ errors = [{'source': 'sessions', 'reason': 'not_an_owned_directory'}] if blocked else []
+ retained = 0
+ with os.fdopen(os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600),
+ 'w', encoding='utf-8') as result:
+ for source in sources:
+ name = source.relative_to(codex_home).as_posix()
+ if source.is_symlink() or not source.is_file():
+ errors.append({'source': name, 'reason': 'not_an_owned_regular_file'})
+ continue
+ digest = hashlib.sha256()
+ counts = {}
+ with source.open('rb') as lines:
+ for ordinal, line in enumerate(lines, 1):
+ digest.update(line)
+ try:
+ event = json.loads(line)
+ selected = selected_event(event)
+ except ValueError as error:
+ errors.append({'source': name, 'line': ordinal,
+ 'reason': type(error).__name__})
+ continue
+ kind = event.get('type')
+ counts[str(kind)] = counts.get(str(kind), 0) + 1
+ if selected is not None:
+ result.write(json.dumps(selected, ensure_ascii=False) + '\n')
+ retained += 1
+ source_receipts.append({'source': name, 'sha256': digest.hexdigest(),
+ 'event_counts': counts})
+ retained_digest = hashlib.sha256()
+ with target.open('rb') as captured:
+ for chunk in iter(lambda: captured.read(1024 * 1024), b''):
+ retained_digest.update(chunk)
+ receipt = {
+ 'status': 'partial' if errors else 'captured' if source_receipts else 'unavailable',
+ 'retained_file': target.name,
+ 'retained_sha256': retained_digest.hexdigest(),
+ 'retained_events': retained, 'sources': source_receipts, 'errors': errors,
+ 'scope': 'Owned client tool-call/reply and per-response usage items, not '
+ 'the exact submitted model request. Session/account metadata, '
+ 'world state, user/developer messages, encrypted reasoning and '
+ 'internal passthrough are omitted. Authentication, configuration '
+ 'and ambient environment are never copied by this capture.',
+ }
+ (output / 'context-receipt.json').write_text(json.dumps(receipt, indent=2) + '\n')
+ return receipt
diff --git a/experiments/analysis-efficiency/model-context.md b/experiments/analysis-efficiency/model-context.md
new file mode 100644
index 000000000..3d878beed
--- /dev/null
+++ b/experiments/analysis-efficiency/model-context.md
@@ -0,0 +1,209 @@
+# Observe the client boundary before claiming token savings
+
+Continues [#2016](https://github.com/morluto/rea/issues/2016) after the managed
+summary [PR #2413](https://github.com/morluto/rea/pull/2413) merged as
+`82cfcee19f18b0eeae9a330ba04e0411cdf85225`. That opt-in view has verified smaller
+delivery and complete-default migration, while its original real-model cohort
+does not establish token savings. This follow-up measures why raw reply bytes
+and aggregate model usage cannot be treated as interchangeable observations.
+
+## Concrete diagnostic observation
+
+An individual real Codex CLI 0.161.0 control uses frozen runtime
+`ae8051302b5834322fa00cfa31c968ae74ab6ed9`, the unchanged ShareX identity question,
+requested `gpt-5.6-terra` high and the same two enabled REA tools. A private
+diagnostic copy removes the original runner's explicit `--ephemeral` flag.
+Authentication/profile cleanup still runs; only selected tool/usage items are
+retained after filtering the owned rollout. Its source rollout SHA-256 is
+`d4df41d6187880d177f00b7533d85ad032c9c4ec502a39e7d5a566180e10461b` and the retained
+filtered-item SHA-256 is
+`0d5b4fb67c5674d1ba14ac32ae37e9a72b0368f4d9f5f9f81eb4d13193cda260`.
+The original whole-copy is removed; account metadata, world state, messages and
+internal passthrough are not retained in the filtered receipt.
+
+Despite the requested `direct` configuration label, the observed client uses
+three JavaScript `exec` calls. First it filters `ALL_TOOLS` with a broad
+`/REA|reverse|artifact|managed|assembly|metadata/i` expression and prints matching
+metadata. The client reports truncation of a 164,005-token original output.
+This is the client's output measurement, not billed input or a complete inventory.
+Next it invokes complete managed inspection but selects requested normalized
+identity/classification fields before printing. Finally it invokes native-boundary
+inspection and prints selected fields. The actual original complete MCP replies
+remain available in the separate public-event transcript.
+
+The persisted custom-tool output values are 41,704 / 2,154 / 13,489 UTF-8 bytes.
+These are the serialized recorded output **values**, not raw wire bytes or exact
+submitted model requests. The trace reports four per-response usage observations:
+
+| Response | Input | Cached input | Output |
+| -------- | -----: | -----------: | -----: |
+| 1 | 10,400 | 0 | 132 |
+| 2 | 19,992 | 9,984 | 456 |
+| 3 | 21,242 | 19,200 | 251 |
+| 4 | 24,799 | 20,224 | 1,535 |
+
+Their totals match public turn usage: 76,433 input, 49,408 cached, 27,025 uncached
+and 2,374 output, with 61.436 seconds observed wall time. Two MCP calls alone do
+not describe the client's discovery/execution rounds. This proves the observed
+control can project a complete result before printing it; it does not establish
+that every earlier control/candidate did the same, or attribute every token
+difference to discovery. The earlier `direct` label specifies requested client
+configuration, not a verified absence of an executor.
+
+Initial transcript-capture attempts retained empty source lists because the
+original runner intentionally passed `--ephemeral`. A first private flag-removal
+edit missed its newline and failed; that candidate remains an unsuccessful
+capture attempt. The later corrected control is an individual diagnostic,
+not a comparable pair with that candidate. Other exploratory aggregates remain
+private and are not silently folded into the original four-trial cohort.
+
+## Reproducible optional capture
+
+`agent-trial.py --capture-tool-context` omits `--ephemeral` only for that explicitly
+selected temporary trial. Its default is unchanged. The profile still disappears
+in `finally`, including when capture fails. The capture reads only its owned
+`sessions/*.jsonl` records, rejects symlink sources, and retains:
+
+- Custom/function tool calls and replies, preserving call IDs and exact values.
+- Per-response token counts and cumulative counts, with original response IDs.
+- Client token-count events, retaining explicit unknown `info: null`.
+
+It excludes session/account metadata, world state, user/developer messages,
+encrypted reasoning, internal passthrough and unknown nested usage fields. It
+does not copy authentication, configuration or the ambient environment. Receipts
+record source/retained digests, event counts, malformed-line failures and missing
+rollouts as unavailable. Capture is an observation of persisted client items,
+not proof of the exact request accepted by a remote model service.
+
+Run the same frozen runtime, question, tool selection and capture mode on both
+sides. Preserve source/settings hashes and separate requested versus observed
+behavior. Use the existing resource wrapper with unchanged one CPU, 4 GiB hard
+memory, 3 GiB early-stop, no swap and task/host safeguards. Final physical counters
+remain unknown when transient cgroups disappear; this instrumentation does not
+repair that measurement limitation.
+
+The focused Python tests run through the existing CI `test:python` gate.
+They cover account/ambient exclusion while preserving caller data, malformed and
+missing records, foreign symlinks, and capture failure without copied-credential
+leakage. The cleanup test simulates a CLI and establishes only cleanup semantics;
+it is not real-model/provider evidence.
+
+## An explicit discovery-guidance experiment
+
+`--guide-enabled-tools` is another opt-in experiment flag. It requires a nonempty
+`--enabled-tools` list and adds the exact corresponding `mcp__rea__...` function
+names to the temporary developer instructions, requesting only exact-name
+metadata discovery. It preserves the analyst question, operation order, tool
+selection and provider inputs. The default instruction is unchanged. Missing or
+invalid selected tools fail before creating an output/profile. The test gate
+also verifies the new prerequisite without acquiring a profile.
+
+Compare guided and unguided runs on the **same** frozen REA runtime with matching
+capture mode and requested facts. A smaller discovery response or additional
+hint is not itself a task/token success: grade answers, count all response usage,
+preserve failed/truncated runs and repeat both orders before recommending this
+guidance. This is a caller-side experiment, not a new REA tool or provider preset.
+
+## First committed capture pair
+
+Checkpoint `9ebdcc458c399b9bdecaaca2ef47422888231181` runs the unchanged question
+against frozen control `ae8051302b5834322fa00cfa31c968ae74ab6ed9` and candidate
+`8bc1601ca51e417ca472d90ea0367e9418a1a0f2`, both with capture enabled and no tool
+guidance. Both captures succeed with 10 retained events, unchanged source bytes,
+removed profiles, two successful MCP calls and passing selected factual checks.
+Full grader gates remain false because final physical resource counters are
+unavailable; factual subsets do not override that limitation.
+
+| Side | Input | Cached | Uncached | Output | Wall seconds |
+| ----------------- | -----: | -----: | -------: | -----: | -----------: |
+| Complete control | 52,656 | 27,136 | 25,520 | 2,015 | 57.736 |
+| Summary candidate | 56,655 | 29,184 | 27,471 | 1,534 | 55.055 |
+
+Both clients make two executor calls and three response-usage observations.
+First they print broad discovered metadata, with truncation. Then each executor
+sequentially performs both MCP operations, preserving selected-target dependency
+order. The control prints a 2,086-byte output value; the candidate prints 22,770
+bytes. Their discovery output values are 41,723 and 41,675 bytes. These are
+recorded serialized values and remain distinct from raw MCP envelopes, billed
+tokens and exact submitted requests. Both requested modes are `direct`; the
+actual executor use is separately observed.
+
+The candidate's smaller raw metadata reply does not result in a smaller printed
+client value in this pair. Candidate input/uncached tokens remain higher. The
+observed projection and round counts are useful constraints for the next
+experiment; this one pair does not attribute all usage differences to either
+formatting or discovery, or establish a reliable speedup.
+
+## Same-runtime guided discovery, repeated in both orders
+
+Checkpoint `e20bd0e2a905f06bc8c56fe04b0e61f3da88636a` runs four actual Codex trials
+in unguided/guided/guided/unguided order. All use the **same** frozen candidate
+runtime, artifact, original question, requested model/reasoning, two selected
+tools, provider settings and capture mode. Only the explicit discovery-guidance
+treatment changes. Every trial selects artifact summary and makes the same two
+successful MCP calls in dependency order; source bytes stay unchanged, captures
+succeed and profiles are removed.
+
+| Trial | Input | Cached | Uncached | Output | Seconds | Observed responses |
+| ---------- | ------: | -----: | -------: | -----: | ------: | -----------------: |
+| Unguided 1 | 77,251 | 50,432 | 26,819 | 1,931 | 62.066 | 4 |
+| Guided 1 | 39,175 | 23,040 | 16,135 | 1,763 | 55.658 | 3 |
+| Guided 2 | 58,336 | 38,144 | 20,192 | 1,626 | 50.685 | 4 |
+| Unguided 2 | 103,301 | 72,704 | 30,597 | 1,983 | 60.336 | 5 |
+
+Both guided discovery output values repeat at 12,336 bytes without truncation,
+versus 41,709 bytes with truncation on both unguided trials. The recorded
+executor/output usage explains observable discovery and round differences; it
+does not certify exact remote request contents or attribute every token to the
+hint. CPU samples are 6.303–6.554 seconds and peak family memory 684,519,424–
+712,028,160 bytes, with no guard abort. Final resource counters remain unknown.
+These samples do not establish a memory or CPU improvement.
+
+Unguided 1 omits `classification.container` and fails the selected factual gate;
+it is not a fully acceptable comparison answer. Guided 1 and Unguided 2 pass
+that gate. Guided 2 initially fails exact coverage-object equality because it
+adds `module_ref_count: 0` and `pinvoke_import_count: 0` to the explicitly named
+native scope. Both numbers exactly match the producer's returned arrays. The
+grader now explicitly verifies only these two known additional counts, including
+integer type, while still requiring all original coverage facts. Unknown extra
+claims, conflicting aliases, wrong counts, boolean/string counts and omitted
+issues continue to fail. Source coverage remains unchanged; returned-row counts
+do not establish global absence. Original failure receipts remain preserved;
+the regrade separately records its new source hash. Guided 2 then passes selected
+facts, but every **full** gate still fails unknown final physical resource checks.
+
+The second comparison has two factually acceptable answers: observed guided input
+falls about 43.5%, uncached input about 34.0%, and wall time about 16.0%. The first
+comparison has an incomplete unguided answer and is reported without counting it
+as equivalent completed work. Two trials per treatment on one question/application
+remain a preliminary signal, not a general default recommendation. The hint stays
+opt-in; wider questions/domains, count/coverage representation and client round
+behavior need further verification before promoting it into general guidance.
+
+## Real Android follow-up: the primary outcomes are mixed
+
+Checkpoint `22834c7d73056e2480b426848f58c31af0c307bc` checks the pinned real
+NewPipe 0.29.1 APK using actual configured JADX 0.7.1/Java, the original Android
+question and the same frozen candidate runtime. First a fresh SDK workflow
+validates three live advertised operations and produces the comparison facts:
+package/version identity, every case-sensitive MainActivity match and all
+reported onCreate entries, preserving signatures, overload indices and null DEX
+descriptors. It takes 26.961 seconds; the owned scope samples 1,234,735,104 bytes.
+
+Both subsequent actual Codex trials use the same three selected tool names,
+capture mode, model/reasoning, question and provider settings. Only discovery
+guidance changes. Each makes three successful MCP calls and five observed model
+responses, passes selected facts against the fresh actual producer, preserves
+source bytes and removes its temporary profile. No APK execution is requested.
+
+| Trial | Input | Cached | Uncached | Output | Wall seconds | Sampled peak bytes |
+| -------- | -----: | -----: | -------: | -----: | -----------: | -----------------: |
+| Unguided | 99,913 | 80,640 | 19,273 | 1,527 | 66.797 | 1,405,829,120 |
+| Guided | 89,220 | 62,464 | 26,756 | 2,142 | 76.936 | 1,345,576,960 |
+
+Total input falls about 10.7%, but uncached input increases and wall time is
+longer. One pair does not demonstrate improved Android performance or memory.
+No guard abort occurs; final counters remain unknown and full resource gates
+remain false. This second domain is a concrete counterexample to generalizing
+the favorable managed comparison. Keep both flags optional and continue
+task-specific experiments before changing installed/default agent guidance.
diff --git a/experiments/analysis-efficiency/review-model.py b/experiments/analysis-efficiency/review-model.py
index 3c56aabb2..a61bb0984 100644
--- a/experiments/analysis-efficiency/review-model.py
+++ b/experiments/analysis-efficiency/review-model.py
@@ -52,6 +52,21 @@ def named_scope(value, names):
return {'conflicting_reported_scopes': selected}
+def scope_with_verified_counts(value, expected_scope, returned_counts):
+ """Validate explicit extra row counts; never erase unknown or conflicting facts."""
+ if not isinstance(value, dict):
+ return value, {}
+ scope = dict(value)
+ checks = {}
+ for name, expected in returned_counts.items():
+ if name in scope:
+ reported = scope[name]
+ checks[name] = type(reported) is int and reported == expected
+ if name not in expected_scope:
+ del scope[name]
+ return scope, checks
+
+
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--domain', choices=['wasm', 'evm', 'android', 'managed', 'managed-nonempty', 'firmware', 'inventory', 'javascript', 'native', 'browser'], required=True)
@@ -134,7 +149,13 @@ def main():
for key in ['module_refs', 'pinvoke_imports']:
check(key, answer.get('native_boundaries', {}).get(key), reference['boundaries'][key])
check('artifact_coverage', named_scope(answer.get('coverage'), ['artifact', 'managed_artifact', 'managed_metadata']), n['coverage'])
- check('native_boundaries_coverage', named_scope(answer.get('coverage'), ['native_boundaries', 'native_boundary_declarations']), reference['boundaries']['coverage'])
+ boundary_scope, counts = scope_with_verified_counts(
+ named_scope(answer.get('coverage'), ['native_boundaries', 'native_boundary_declarations']),
+ reference['boundaries']['coverage'],
+ {'module_ref_count': len(reference['boundaries']['module_refs']),
+ 'pinvoke_import_count': len(reference['boundaries']['pinvoke_imports'])})
+ check('native_boundaries_coverage', boundary_scope, reference['boundaries']['coverage'])
+ checks.update({'native_coverage_' + name: valid for name, valid in counts.items()})
elif domain == 'managed-nonempty':
n = reference['boundaries']
check('module_mvid', answer.get('module_mvid'), n['module']['mvid'])
diff --git a/experiments/analysis-efficiency/test_codex_context.py b/experiments/analysis-efficiency/test_codex_context.py
new file mode 100644
index 000000000..37df83a7c
--- /dev/null
+++ b/experiments/analysis-efficiency/test_codex_context.py
@@ -0,0 +1,175 @@
+"""Distinct capture/cleanup failures; simulated CLI is not real-model evidence."""
+
+import importlib.util
+import io
+import json
+import os
+import runpy
+import sys
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import Mock, patch
+
+from codex_context import capture_tool_context, selected_event
+
+
+class ManagedCoverageTests(unittest.TestCase):
+ def test_verifies_extra_counts_and_preserves_unknown_fields(self):
+ review = runpy.run_path(str(Path(__file__).with_name('review-model.py')))
+ project = review['scope_with_verified_counts']
+ expected = {'state': 'partial', 'issues': ['unreturned rows remain unknown']}
+ original = {**expected, 'module_ref_count': 0, 'pinvoke_import_count': 2}
+ scope, checks = project(original, expected, {'module_ref_count': 0, 'pinvoke_import_count': 2})
+ self.assertEqual(scope, expected)
+ self.assertTrue(all(checks.values()))
+ self.assertEqual(original['pinvoke_import_count'], 2)
+ for count in [3, '2', True]:
+ scope, checks = project({**expected, 'pinvoke_import_count': count}, expected, {'pinvoke_import_count': 2})
+ self.assertFalse(checks['pinvoke_import_count'])
+ _, checks = project({**expected, 'module_ref_count': False}, {**expected, 'module_ref_count': 0}, {'module_ref_count': 0})
+ self.assertFalse(checks['module_ref_count'])
+ scope, _ = project({**expected, 'unsupported_claim': True}, expected, {})
+ self.assertNotEqual(scope, expected)
+ scope, _ = project({'state': 'partial', 'module_ref_count': 0}, expected, {'module_ref_count': 0})
+ self.assertNotEqual(scope, expected)
+ ambiguous = review['named_scope']({'native_boundaries': expected, 'native_boundary_declarations': {'state': 'complete'}}, ['native_boundaries', 'native_boundary_declarations'])
+ scope, _ = project(ambiguous, expected, {'module_ref_count': 0})
+ self.assertNotEqual(scope, expected)
+
+
+class ContextCaptureTests(unittest.TestCase):
+ def test_filters_ambient_account_and_nested_usage_state(self):
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ home = root / 'home'
+ sessions = home / 'sessions'
+ sessions.mkdir(parents=True)
+ output = root / 'output'
+ output.mkdir()
+ events = [
+ {'type': 'world_state', 'payload': {'state': {'env': 'ambient-marker'}}},
+ {'type': 'session_meta', 'payload': {'creator_account_id': 'account-marker'}},
+ {'type': 'response_item', 'payload': {
+ 'type': 'custom_tool_call_output', 'call_id': 'original-call',
+ 'output': 'caller-selected evidence and paths',
+ 'internal_chat_message_metadata_passthrough': 'private-marker'}},
+ {'type': 'token_usage_record', 'payload': {
+ 'response_id': 'original-response', 'usage': {
+ 'input_tokens': 12, 'cached_input_tokens': 4,
+ 'environment': 'nested-ambient-marker'},
+ 'transport_auth': 'auth-marker'}},
+ ]
+ (sessions / 'rollout.jsonl').write_text(''.join(json.dumps(x) + '\n' for x in events))
+ (home / 'auth.json').write_text('copied-credential-marker')
+ receipt = capture_tool_context(home, output)
+ text = (output / 'codex-tool-context.jsonl').read_text()
+ self.assertEqual(receipt['status'], 'captured')
+ self.assertEqual(receipt['retained_events'], 2)
+ for marker in ['ambient-marker', 'account-marker', 'private-marker',
+ 'auth-marker', 'copied-credential-marker']:
+ self.assertNotIn(marker, text)
+ self.assertIn('caller-selected evidence and paths', text)
+ self.assertIn('original-response', text)
+ self.assertEqual((output / 'codex-tool-context.jsonl').stat().st_mode & 0o777, 0o600)
+
+ def test_missing_rollouts_are_unknown_and_malformed_lines_remain_partial(self):
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ output = root / 'missing'
+ output.mkdir()
+ self.assertEqual(capture_tool_context(root / 'home', output)['status'], 'unavailable')
+ sessions = root / 'home' / 'sessions'
+ sessions.mkdir(parents=True)
+ (sessions / 'rollout.jsonl').write_text('not-json\n' + json.dumps({
+ 'type': 'event_msg', 'payload': {'type': 'token_count', 'info': None}}) + '\n')
+ partial = root / 'partial'
+ partial.mkdir()
+ receipt = capture_tool_context(root / 'home', partial)
+ self.assertEqual(receipt['status'], 'partial')
+ self.assertEqual(receipt['errors'][0]['line'], 1)
+ self.assertEqual(receipt['retained_events'], 1)
+ with self.assertRaises(ValueError):
+ selected_event({'type': 'token_usage_record', 'payload': {'usage': {'input_tokens': True}}})
+ with self.assertRaises(ValueError):
+ selected_event({'type': {'environment': 'unexpected'}, 'payload': {}})
+ with self.assertRaises(ValueError):
+ selected_event({'type': 'response_item', 'timestamp': {'environment': 'unexpected'}, 'payload': {}})
+ unknown = selected_event({'type': 'event_msg', 'payload': {'type': 'token_count', 'info': {'model_context_window': None}}})
+ self.assertIsNone(unknown['payload']['info']['model_context_window'])
+
+ def test_capture_failure_still_removes_trial_credentials_and_profile(self):
+ spec = importlib.util.spec_from_file_location('agent_trial', Path(__file__).with_name('agent-trial.py'))
+ agent = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(agent)
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ runtime = root / 'runtime'
+ (runtime / 'scripts').mkdir(parents=True)
+ (runtime / 'scripts' / 'rea.mjs').write_text('')
+ (runtime / 'package-lock.json').write_text('{}')
+ auth_home = root / 'source-auth'
+ auth_home.mkdir()
+ (auth_home / 'auth.json').write_text('synthetic-credential')
+ question = root / 'question.txt'
+ question.write_text('Observe the declared fixture.')
+ output = root / 'trial'
+ child = Mock(stdin=io.StringIO())
+ child.wait.return_value = 0
+ arguments = ['agent-trial.py', '--runtime-root', str(runtime), '--revision', 'fixture',
+ '--prompt', str(question), '--output', str(output), '--capture-tool-context']
+ with patch.dict(os.environ, {'CODEX_HOME': str(auth_home)}, clear=True), \
+ patch.object(sys, 'argv', arguments), \
+ patch.object(agent.shutil, 'which', return_value='/fixture/node'), \
+ patch.object(agent.subprocess, 'check_output', return_value='v24.18.0\n'), \
+ patch.object(agent.subprocess, 'Popen', return_value=child), \
+ patch.object(agent, 'capture_tool_context', side_effect=OSError('fixture capture failure')), \
+ patch('builtins.print'):
+ with self.assertRaisesRegex(OSError, 'fixture capture failure'):
+ agent.main()
+ self.assertFalse(list(output.glob('profile-*')))
+ self.assertTrue(json.loads((output / 'cleanup.json').read_text())['profile_removed'])
+ self.assertEqual((auth_home / 'auth.json').read_text(), 'synthetic-credential')
+
+ def test_refuses_foreign_session_directory_and_symlink_records(self):
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ home = root / 'home'
+ home.mkdir()
+ foreign = root / 'foreign'
+ foreign.mkdir()
+ (foreign / 'rollout.jsonl').write_text('private marker')
+ (home / 'sessions').symlink_to(foreign, target_is_directory=True)
+ output = root / 'directory-result'
+ output.mkdir()
+ receipt = capture_tool_context(home, output)
+ self.assertEqual(receipt['status'], 'partial')
+ self.assertEqual(receipt['retained_events'], 0)
+ (home / 'sessions').unlink()
+ (home / 'sessions').mkdir()
+ (home / 'sessions' / 'linked.jsonl').symlink_to(foreign / 'rollout.jsonl')
+ output = root / 'file-result'
+ output.mkdir()
+ receipt = capture_tool_context(home, output)
+ self.assertEqual(receipt['status'], 'partial')
+ self.assertEqual(receipt['errors'][0]['reason'], 'not_an_owned_regular_file')
+ self.assertEqual((foreign / 'rollout.jsonl').read_text(), 'private marker')
+
+ def test_guidance_requires_selected_tools_before_acquiring_a_profile(self):
+ spec = importlib.util.spec_from_file_location('agent_trial', Path(__file__).with_name('agent-trial.py'))
+ agent = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(agent)
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ output = root / 'trial'
+ arguments = ['agent-trial.py', '--runtime-root', str(root), '--revision', 'fixture',
+ '--prompt', str(root / 'question'), '--output', str(output), '--guide-enabled-tools']
+ with patch.object(sys, 'argv', arguments), patch.object(sys, 'stderr', io.StringIO()):
+ with self.assertRaises(SystemExit) as raised:
+ agent.main()
+ self.assertEqual(raised.exception.code, 2)
+ self.assertFalse(output.exists())
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/package.json b/package.json
index 5635c6d3c..327c14315 100644
--- a/package.json
+++ b/package.json
@@ -130,7 +130,7 @@
"test": "npm run test:with-build",
"test:with-build": "npm run test:prepare && vitest run",
"test:run": "vitest run",
- "test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py -v",
+ "test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py -v && python3 -m unittest discover -s experiments/analysis-efficiency -p test_codex_context.py -v",
"test:local": "node scripts/run-development-tests.mjs local",
"test:fast": "vitest run --project domain --project services --project adapters --project composition --project conformance --project evaluation",
"pretest:boundary": "npm run test:prepare",