Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(tier3): decode native Codex exec calls (#111)
Tomauskasz73b27dad60d3
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex a525fe10..7da1a171 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -71,6 +71,11 @@ All notable changes to SkillEvaluator are documented in this file.conversion limit, preserves nonzero Wilson interval widths and paired-effectdirections at large case counts, and documents exact-rational omissionmarkers.+- Tier 3 now decodes bounded native Codex `exec` wrappers into their static+ tool calls. It preserves call order and outer-call provenance, maps an outer+ observation only when its rendered inner call is known, keeps ambiguous+ observations explicit, and reports unsupported or malformed JavaScript as+ untrusted instead of a clean security result.## 0.2.1 - 2026-08-24diff --git a/src/skillevaluator/evaluation/tier3_report.py b/src/skillevaluator/evaluation/tier3_report.pyindex 0dc50134..2bff758f 100644--- a/src/skillevaluator/evaluation/tier3_report.py+++ b/src/skillevaluator/evaluation/tier3_report.py@@ -37,6 +37,7 @@ from skillevaluator.constants import (TIER3_LIFT_FAIL_THRESHOLD,TIER3_LIFT_PASS_THRESHOLD,)+from skillevaluator.evidence import evidence_ref_identityfrom skillevaluator.models.result import Finding, Severity, ValidationResult# Verdict labels mirror SkillEvaluator's AGENT_EVAL_VERDICT_* values so the ported@@ -1387,8 +1388,8 @@ def _compact_evidence_refs(raw_refs: object) -> list[str]:rendered = raw.strip()elif isinstance(raw, dict):source = str(raw.get("source") or "").strip()- pointer = str(raw.get("json_pointer") or raw.get("path") or "").strip()- rendered = f"{source}{pointer}" if source else pointer+ identity = evidence_ref_identity(raw)+ rendered = f"{source}{identity}" if source else identityelse:continueif rendered and rendered not in refs:diff --git a/src/skillevaluator/evidence.py b/src/skillevaluator/evidence.pynew file mode 100644index 00000000..526f6618--- /dev/null+++ b/src/skillevaluator/evidence.py@@ -0,0 +1,17 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Shared evidence-reference identity."""++from __future__ import annotations++from collections.abc import Mapping+from typing import Any+++def evidence_ref_identity(ref: Mapping[str, Any]) -> str:+ """Return the most specific stable location carried by an evidence ref."""+ for key in ("evidence_id", "json_pointer", "path"):+ if value := str(ref.get(key) or "").strip():+ return value+ return ""diff --git a/src/skillevaluator/reporting/markdown.py b/src/skillevaluator/reporting/markdown.pyindex f5d34a24..446ac0fd 100644--- a/src/skillevaluator/reporting/markdown.py+++ b/src/skillevaluator/reporting/markdown.py@@ -18,6 +18,7 @@ import htmlfrom datetime import UTC, datetimefrom typing import TYPE_CHECKING+from skillevaluator.evidence import evidence_ref_identityfrom skillevaluator.reporting.base import ReporterBase, is_advisory_agent_eval_skip, passes_required_gatefrom skillevaluator.reporting.harbor_viewer import (harbor_evidence_link_text,@@ -242,7 +243,7 @@ class MarkdownReporter(ReporterBase):label = harbor_evidence_link_text(harbor_evidence)lines.append(f" - Evidence: [{label}]({url})")for ref in (suggestion.get("evidence_refs") or [])[:3]:- pointer = ref.get("json_pointer") or ref.get("path") or ""+ pointer = evidence_ref_identity(ref)excerpt = str(ref.get("excerpt") or ref.get("label") or "")[:120]lines.append(f" - Evidence: `{ref.get('kind', 'evidence')}` `{pointer}` {excerpt}")lines.append("")diff --git a/src/skillevaluator/tier3/eval_core/atif_helpers.py b/src/skillevaluator/tier3/eval_core/atif_helpers.pyindex 561737bc..46898de1 100644--- a/src/skillevaluator/tier3/eval_core/atif_helpers.py+++ b/src/skillevaluator/tier3/eval_core/atif_helpers.py@@ -16,16 +16,19 @@ import osimport refrom typing import Any+from skillevaluator.evidence import evidence_ref_identity+from skillevaluator.tier3.eval_core.codex_tool_call_normalizer import (+ iter_normalized_tool_calls as iter_tool_calls,+)+from skillevaluator.tier3.eval_core.codex_tool_call_normalizer import (+ normalized_tool_call_observation as _tool_call_observation,+)+from skillevaluator.tier3.eval_core.codex_tool_call_normalizer import (+ normalized_tool_call_wrapper_observation as _tool_call_wrapper_observation,+)from skillevaluator.tier3.eval_core.secret_redaction import redact_secrets_in_log_line-def iter_tool_calls(traj: dict[str, Any]):- """Yield ``(step_dict, tool_call_dict)`` for every tool call in the trajectory."""- for step in traj.get("steps", []):- for tc in step.get("tool_calls") or []:- yield step, tc--def get_all_tool_calls(traj: dict[str, Any]) -> list[dict[str, Any]]:"""Extract all tool calls with function name, arguments, and observation text.@@ -35,17 +38,12 @@ def get_all_tool_calls(traj: dict[str, Any]) -> list[dict[str, Any]]:for step, tc in iter_tool_calls(traj):fn = tc.get("function_name") or ""args = tc.get("arguments") or {}- obs_text = ""- obs = step.get("observation") or {}- for r in obs.get("results") or []:- if r.get("source_call_id") == tc.get("tool_call_id") or not r.get("source_call_id"):- obs_text += str(r.get("content", ""))calls.append({"fn": fn,"args": args,"args_text": json.dumps(args).lower(),- "obs": obs_text.lower(),+ "obs": _tool_call_observation(step, tc).lower(),})return calls@@ -143,7 +141,7 @@ def build_conversation_summary(traj: dict[str, Any], question: str) -> str:if reasoning:parts.append(f"Agent reasoning: {str(reasoning)[:200]}")- for tc in step.get("tool_calls") or []:+ for _, tc in iter_tool_calls({"steps": [step]}):fn = tc.get("function_name", "")args = tc.get("arguments") or {}parts.append(f"Agent called: {fn}({json.dumps(args)[:200]})")@@ -260,14 +258,7 @@ def _collect_file_change_evidence(traj: dict[str, Any]) -> list[str]:for step in traj.get("steps", []):if step.get("source") != "agent":continue- observations_by_id: dict[str, str] = {}- for result in (step.get("observation") or {}).get("results") or []:- call_id = str(result.get("source_call_id") or "")- content = str(result.get("content") or "")- if call_id and content:- observations_by_id[call_id] = content-- for tc in step.get("tool_calls") or []:+ for _, tc in iter_tool_calls({"steps": [step]}):fn = str(tc.get("function_name") or "")fn_lower = fn.lower()args = tc.get("arguments") or {}@@ -289,7 +280,7 @@ def _collect_file_change_evidence(traj: dict[str, Any]) -> list[str]:if not is_write_call or (not body and not file_path):continue- obs = observations_by_id.get(str(tc.get("tool_call_id") or ""), "")+ obs = _tool_call_observation(step, tc)entry_parts = [f"Agent called: {fn}"]if file_path:entry_parts.append(f"Path: {file_path}")@@ -378,6 +369,7 @@ def _evidence_ref(path: str | None = None,excerpt: str = "",status: str | None = None,+ evidence_id: str | None = None,) -> dict[str, Any]:ref: dict[str, Any] = {"source": source,@@ -392,18 +384,19 @@ def _evidence_ref(ref["excerpt"] = _evidence_excerpt(excerpt)if status:ref["status"] = status+ if evidence_id:+ ref["evidence_id"] = evidence_idreturn refdef _dedupe_evidence_refs(refs: list[dict[str, Any]]) -> list[dict[str, Any]]:- seen: set[tuple[str, str, str, str]] = set()+ seen: set[tuple[str, str, str]] = set()deduped: list[dict[str, Any]] = []for ref in refs:key = (str(ref.get("source") or ""),- str(ref.get("json_pointer") or ""),+ evidence_ref_identity(ref),str(ref.get("kind") or ""),- str(ref.get("path") or ""),)if key in seen:continue@@ -432,7 +425,7 @@ def _final_response_ref(traj: dict[str, Any]) -> list[dict[str, Any]]:return []-def _tool_call_ref(step_idx: int, tool_idx: int, tc: dict[str, Any], *, kind: str) -> dict[str, Any]:+def _tool_call_ref(step_idx: int, tc: dict[str, Any], *, kind: str) -> dict[str, Any]:fn = str(tc.get("function_name") or "")args = tc.get("arguments") or {}if not isinstance(args, dict):@@ -445,13 +438,16 @@ def _tool_call_ref(step_idx: int, tool_idx: int, tc: dict[str, Any], *, kind: stpath = _first_expected_artifact_path(command)excerpt = command or path or json.dumps(args, sort_keys=True)label_detail = command or path or fn+ json_pointer = f"/steps/{step_idx}/tool_calls/{tc['_atif_raw_tool_index']}"+ inner_index = tc.get("_atif_inner_tool_index")return _evidence_ref(source="trajectory.json",- json_pointer=f"/steps/{step_idx}/tool_calls/{tool_idx}",+ json_pointer=json_pointer,kind=kind,label=f"{fn}: {label_detail}" if label_detail else fn,path=path or None,excerpt=excerpt,+ evidence_id=f"{json_pointer}/normalized/{inner_index}" if inner_index is not None else None,)@@ -460,10 +456,10 @@ def _tool_call_refs(traj: dict[str, Any]) -> list[dict[str, Any]]:for step_idx, step in enumerate(traj.get("steps", [])):if step.get("source") != "agent":continue- for tool_idx, tc in enumerate(step.get("tool_calls") or []):+ for _, tc in iter_tool_calls({"steps": [step]}):if len(refs) >= _METRIC_EVIDENCE_MAX_TOOL_REFS:return refs- refs.append(_tool_call_ref(step_idx, tool_idx, tc, kind="tool_call"))+ refs.append(_tool_call_ref(step_idx, tc, kind="tool_call"))return refs@@ -496,7 +492,7 @@ def _file_change_refs(traj: dict[str, Any]) -> list[dict[str, Any]]:for step_idx, step in enumerate(traj.get("steps", [])):if step.get("source") != "agent":continue- for tool_idx, tc in enumerate(step.get("tool_calls") or []):+ for _, tc in iter_tool_calls({"steps": [step]}):if len(refs) >= _METRIC_EVIDENCE_MAX_FILE_REFS:return refsfn = str(tc.get("function_name") or "")@@ -510,7 +506,7 @@ def _file_change_refs(traj: dict[str, Any]) -> list[dict[str, Any]]:)if not is_write:continue- refs.append(_tool_call_ref(step_idx, tool_idx, tc, kind="file_change"))+ refs.append(_tool_call_ref(step_idx, tc, kind="file_change"))return refs@@ -749,7 +745,7 @@ def build_verified_facts(if step.get("source") != "agent":continue# Check tool call arguments: command/cmd/code and file-path args- for tc in step.get("tool_calls") or []:+ for _, tc in iter_tool_calls({"steps": [step]}):args = tc.get("arguments") or {}if not isinstance(args, dict):continue@@ -914,17 +910,17 @@ def extract_tool_calls_as_dicts(traj: dict[str, Any]) -> list[dict[str, Any]]:for step in traj.get("steps", []):if step.get("source") != "agent":continue- for tc in step.get("tool_calls") or []:- obs_text = ""- obs = step.get("observation") or {}- for r in obs.get("results") or []:- if r.get("source_call_id") == tc.get("tool_call_id") or not r.get("source_call_id"):- obs_text += str(r.get("content", ""))- result.append(- {- "action": tc.get("function_name", ""),- "action_input": tc.get("arguments") or {},- "observation": obs_text,- }- )+ for _, tc in iter_tool_calls({"steps": [step]}):+ call = {+ "action": tc.get("function_name", ""),+ "action_input": tc.get("arguments") or {},+ "observation": _tool_call_observation(step, tc),+ }+ if status := tc.get("_atif_normalization_status"):+ call["normalization_status"] = status+ if status := tc.get("_atif_observation_status"):+ call["observation_status"] = status+ if wrapper_observation := _tool_call_wrapper_observation(step, tc):+ call["wrapper_observation"] = wrapper_observation+ result.append(call)return resultdiff --git a/src/skillevaluator/tier3/eval_core/checks.py b/src/skillevaluator/tier3/eval_core/checks.pyindex be9c3604..d68709f4 100644--- a/src/skillevaluator/tier3/eval_core/checks.py+++ b/src/skillevaluator/tier3/eval_core/checks.py@@ -19,6 +19,12 @@ import shlexfrom fnmatch import fnmatchcasefrom typing import Any+from skillevaluator.tier3.eval_core.codex_tool_call_normalizer import (+ AMBIGUOUS_OUTER_EXEC_OBSERVATION,+ UNOBSERVED_INNER_CALL,+ UNSUPPORTED_NATIVE_CODEX_EXEC,+)+WASTE_INDICATORS = ["--help","--version",@@ -755,6 +761,26 @@ def _security_finding(return finding+def _secret_exposure_finding(+ observation: str,+ *,+ tool: str | None,+ target_skill_used_before: bool | None,+) -> dict[str, Any] | None:+ if not any(pattern.search(observation) for pattern in _SECRET_PATTERNS):+ return None+ return _security_finding(+ finding_type="secret_exposure",+ severity="critical",+ message="Possible secret value appeared in tool output observed by the agent",+ evidence="[redacted secret exposure]",+ source="tool_observation",+ score_impact=True,+ tool=tool,+ target_skill_used_before=target_skill_used_before,+ )++def _tool_mentions_skill(tool_call: dict[str, Any], expected_skill: str) -> bool:if not expected_skill:return False@@ -870,10 +896,31 @@ def check_security(target_skill_seen = Falsefor tc in tool_calls:action = str(tc.get("action", ""))+ observation = str(tc.get("observation", ""))+ wrapper_observation = str(tc.get("wrapper_observation", ""))+ if tc.get("normalization_status") == UNSUPPORTED_NATIVE_CODEX_EXEC:+ findings.append(+ _security_finding(+ finding_type="unsupported_tool_wrapper",+ severity="warning",+ message="Unsupported native Codex exec wrapper could not be safely normalized",+ evidence="[unsupported native Codex exec wrapper]",+ source="agent_tool_call",+ score_impact=True,+ tool=action,+ target_skill_used_before=target_skill_seen,+ )+ )+ if finding := _secret_exposure_finding(+ observation,+ tool=action,+ target_skill_used_before=target_skill_seen,+ ):+ findings.append(finding)+ continueaction_lower = action.lower()action_text = _action_text(tc)action_text_lower = action_text.lower()- observation = str(tc.get("observation", ""))if _tool_mentions_any_skill(tc, expected_skill, acceptable_skills):target_skill_seen = True@@ -976,21 +1023,18 @@ def check_security())- for pattern in _SECRET_PATTERNS:- if pattern.search(observation):- findings.append(- _security_finding(- finding_type="secret_exposure",- severity="critical",- message="Possible secret value appeared in tool output observed by the agent",- evidence="[redacted secret exposure]",- source="tool_observation",- score_impact=True,- tool=action,- target_skill_used_before=target_skill_seen,- )- )- break+ if finding := _secret_exposure_finding(+ observation,+ tool=action,+ target_skill_used_before=target_skill_seen,+ ):+ findings.append(finding)+ if finding := _secret_exposure_finding(+ wrapper_observation,+ tool=None,+ target_skill_used_before=None,+ ):+ findings.append(finding)if any(f["type"] == "prompt_injection_attempt" for f in findings) and _looks_like_refusal(agent_text):findings.append(@@ -1030,6 +1074,20 @@ def check_security(# ---------------------------------------------------------------------------+def _has_unsupported_native_codex_call(tool_calls: list[dict[str, Any]]) -> bool:+ return any(tc.get("normalization_status") == UNSUPPORTED_NATIVE_CODEX_EXEC for tc in tool_calls)+++def _unsupported_native_codex_result(reason: str) -> dict[str, Any]:+ return {+ "passed": None,+ "score": 0.5,+ "reason": reason,+ "supported": False,+ "unsupported_evidence": [UNSUPPORTED_NATIVE_CODEX_EXEC],+ }++def check_activation(tool_calls: list[dict[str, Any]],expected_skill: str,@@ -1107,6 +1165,11 @@ def check_activation("details": {**_skill_match_details(expected_skill, acceptable_skills), **match},}+ if _has_unsupported_native_codex_call(tool_calls):+ return _unsupported_native_codex_result(+ "Skill activation could not be evaluated because a native Codex exec wrapper was unsupported"+ )+# Check 4: Skill referenced in tool observationfor tc in tool_calls:action = str(tc.get("action", ""))@@ -1155,6 +1218,16 @@ def check_script_execution(return {"passed": True, "score": 1.0, "reason": "No specific script expected"}exec_calls = [tc for tc in tool_calls if _is_execution_action(str(tc["action"]))]+ for call in exec_calls:+ cmd = _command_text(call)+ if expected_script in cmd:+ return {"passed": True, "score": 1.0, "reason": f"Executed {expected_script}"}++ if _has_unsupported_native_codex_call(tool_calls):+ return _unsupported_native_codex_result(+ "Script execution could not be evaluated because a native Codex exec wrapper was unsupported"+ )+if not exec_calls:# Check observation text as fallback (script may run inside Skill tool)for tc in tool_calls:@@ -1163,11 +1236,6 @@ def check_script_execution(return {"passed": True, "score": 0.75, "reason": f"{expected_script} found in tool observation"}return {"passed": False, "score": 0.0, "reason": "No execute/run_code call found"}- for call in exec_calls:- cmd = _command_text(call)- if expected_script in cmd:- return {"passed": True, "score": 1.0, "reason": f"Executed {expected_script}"}-# Observation fallback for exec callsfor call in exec_calls:obs = str(call.get("observation", "")).lower()@@ -1187,6 +1255,11 @@ def check_workflow_order(Also treats Claude Code ``Skill`` tool activation as a valid "read" step."""+ if _has_unsupported_native_codex_call(tool_calls):+ return _unsupported_native_codex_result(+ "Workflow order could not be evaluated because a native Codex exec wrapper was unsupported"+ )+sequence: list[str] = []saw_skill_activation = bool(skill_tool_names)@@ -1265,6 +1338,29 @@ def check_error_recovery(if tc["action"].lower() in exec_actions or _is_execution_action(str(tc["action"])):exec_calls.append((idx, tc))+ unsupported_evidence = {+ tc.get("normalization_status")+ for tc in tool_calls+ if tc.get("normalization_status") == UNSUPPORTED_NATIVE_CODEX_EXEC+ }+ unsupported_evidence.update(+ tc.get("observation_status")+ for _, tc in exec_calls+ if tc.get("observation_status") in {AMBIGUOUS_OUTER_EXEC_OBSERVATION, UNOBSERVED_INNER_CALL}+ )+ if unsupported_evidence:+ return {+ "passed": None,+ "score": 0.5,+ "reason": "Error recovery could not be evaluated from untrusted Codex wrapper observations",+ "supported": False,+ "unsupported_evidence": sorted(unsupported_evidence),+ "first_attempt_clean": False,+ "corrections": [],+ "skill_faults": 0,+ "agent_faults": 0,+ }+error_keywords = ["error","traceback",@@ -1399,6 +1495,12 @@ def check_negative_case(if target_reference is None:saw_unknown = True+ if _has_unsupported_native_codex_call(tool_calls):+ return _unsupported_native_codex_result(+ f"Could not safely determine whether {skill_under_test} was triggered because a native Codex exec "+ "wrapper was unsupported"+ )+if saw_unknown:return {"passed": None,@@ -1423,6 +1525,7 @@ def check_routing(acceptable_skills: Any = None,) -> dict[str, Any]:"""Check the agent read only expected/allowed workspace skill docs."""+ unsupported_native_codex_call = _has_unsupported_native_codex_call(tool_calls)read_calls = [tc for tc in tool_calls if "read" in tc["action"].lower()]skills_read: list[str] = []@@ -1488,6 +1591,10 @@ def check_routing(wrong_skills.append(f"Skill({s})")if not skills_read:+ if unsupported_native_codex_call:+ return _unsupported_native_codex_result(+ "Skill routing could not be evaluated because a native Codex exec wrapper was unsupported"+ )return {"passed": False,"score": 0.0,@@ -1509,6 +1616,11 @@ def check_routing(},}+ if unsupported_native_codex_call:+ return _unsupported_native_codex_result(+ "Skill routing could not be evaluated because a native Codex exec wrapper was unsupported"+ )+if matched_alternate and not matched_expected:return {"passed": True,@@ -1551,6 +1663,11 @@ def check_tool_efficiency(if not tool_calls:return {"passed": True, "score": 1.0, "reason": "No tool calls", "details": {}}+ if _has_unsupported_native_codex_call(tool_calls):+ return _unsupported_native_codex_result(+ "Tool efficiency could not be evaluated because a native Codex exec wrapper was unsupported"+ )+productive = 0wasted = 0wasted_details: list[str] = []diff --git a/src/skillevaluator/tier3/eval_core/codex_tool_call_normalizer.py b/src/skillevaluator/tier3/eval_core/codex_tool_call_normalizer.pynew file mode 100644index 00000000..5b56f57e--- /dev/null+++ b/src/skillevaluator/tier3/eval_core/codex_tool_call_normalizer.py@@ -0,0 +1,697 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Dependency-free normalization for native Codex ``tools.exec`` wrappers."""++from __future__ import annotations++import json+import re+from typing import Any++UNSUPPORTED_NATIVE_CODEX_EXEC = "unsupported_native_codex_exec_wrapper"+AMBIGUOUS_OUTER_EXEC_OBSERVATION = "ambiguous_outer_exec_result"+MAPPED_OUTER_EXEC_OBSERVATION = "mapped_outer_exec_result"+UNOBSERVED_INNER_CALL = "unobserved_inner_call"++_MAX_SOURCE_CHARS = 64 * 1024+_MAX_OBJECT_NESTING = 64+_MAX_STATEMENTS = 256+_MAX_TOOL_CALLS = 128+++def iter_normalized_tool_calls(traj: dict[str, Any]):+ """Yield each trajectory tool call after safe native-Codex normalization."""+ for step in traj.get("steps", []):+ for raw_index, tool_call in enumerate(step.get("tool_calls") or []):+ for normalized in normalize_tool_call(tool_call):+ yield step, {**normalized, "_atif_raw_tool_index": raw_index}+++def _outer_tool_call_observation(+ step: dict[str, Any], tool_call: dict[str, Any], *, include_unscoped: bool = True+) -> str:+ tool_call_id = tool_call.get("tool_call_id")+ return "".join(+ str(result.get("content", ""))+ for result in (step.get("observation") or {}).get("results") or []+ if (+ tool_call_id+ and result.get("source_call_id")+ and result.get("source_call_id") == tool_call_id+ )+ or (include_unscoped and not result.get("source_call_id"))+ )+++def normalized_tool_call_observation(step: dict[str, Any], tool_call: dict[str, Any]) -> str:+ """Return an outer observation only when its normalized owner is known."""+ if tool_call.get("_atif_observation_status") not in (None, MAPPED_OUTER_EXEC_OBSERVATION):+ return ""+ return _outer_tool_call_observation(+ step,+ tool_call,+ include_unscoped=tool_call.get("_atif_raw_tool_index", 0) == 0,+ )+++def normalized_tool_call_wrapper_observation(step: dict[str, Any], tool_call: dict[str, Any]) -> str:+ """Return an unattributed wrapper observation once for security scanning."""+ if tool_call.get("_atif_observation_status") not in (+ AMBIGUOUS_OUTER_EXEC_OBSERVATION,+ UNOBSERVED_INNER_CALL,+ ) or tool_call.get("_atif_inner_tool_index") != 0:+ return ""+ observation_owner = tool_call.get("_atif_outer_observation_owner")+ if isinstance(observation_owner, int) and observation_owner >= 0:+ return ""+ return _outer_tool_call_observation(+ step,+ tool_call,+ include_unscoped=tool_call.get("_atif_raw_tool_index", 0) == 0,+ )+++def _skip_js_quoted(source: str, start: int, quote: str) -> int:+ index = start + 1+ while index < len(source):+ if source[index] == "\\":+ index += 2+ elif source[index] == quote:+ return index + 1+ else:+ index += 1+ return -1+++def _decode_static_js_object(source: str, start: int) -> tuple[dict[str, Any], int] | None:+ """Decode a JSON-compatible object literal, including unquoted property names."""+ rendered: list[str] = []+ stack: list[str] = []+ index = start+ previous_significant = ""+ while index < len(source):+ char = source[index]+ if char == '"':+ end = _skip_js_quoted(source, index, char)+ if end < 0:+ return None+ rendered.append(source[index:end])+ previous_significant = '"'+ index = end+ continue+ if char in "'`" or source.startswith(("//", "/*"), index):+ return None+ if char in "{[":+ if len(stack) >= _MAX_OBJECT_NESTING:+ return None+ stack.append(char)+ elif char in "}]":+ if not stack or stack.pop() != ("{" if char == "}" else "["):+ return None+ if (char.isalpha() or char in "_$") and previous_significant in {"{", ","}:+ end = index + 1+ while end < len(source) and (source[end].isalnum() or source[end] in "_$"):+ end += 1+ cursor = end+ while cursor < len(source) and source[cursor].isspace():+ cursor += 1+ if cursor < len(source) and source[cursor] == ":":+ rendered.append(json.dumps(source[index:end]))+ previous_significant = '"'+ index = end+ continue+ rendered.append(char)+ if not char.isspace():+ previous_significant = char+ index += 1+ if not stack:+ try:+ arguments = json.loads("".join(rendered))+ except (json.JSONDecodeError, RecursionError, TypeError, ValueError):+ return None+ return (arguments, index) if isinstance(arguments, dict) else None+ return None+++_JS_IDENTIFIER = r"[A-Za-z_$][\w$]*"+_CODEX_CALL_RE = re.compile(rf"const\s+({_JS_IDENTIFIER})\s*=\s*await\s+tools\.({_JS_IDENTIFIER})\s*\(\s*")+_CODEX_RENDER_RE = re.compile(+ rf"text\s*\(\s*(?:JSON\.stringify\(\s*({_JS_IDENTIFIER})\s*\)|"+ rf"({_JS_IDENTIFIER})(?:\.({_JS_IDENTIFIER}))?)\s*\)\s*;"+)+_CODEX_PRAGMA_RE = re.compile(r"[ \t]*// @exec:[^\r\n]*\r?\n")+_CODEX_PRAGMA_PREFIX_RE = re.compile(r"[ \t]*// @exec:")+_CODEX_TOOL_REF_RE = re.compile(r"\b(?:tools|globalThis)\b|\\u(?:\{[0-9A-Fa-f]{1,6}\}|[0-9A-Fa-f]{4})")+_JS_IDENTIFIER_ESCAPE_RE = re.compile(r"\\u(?:\{([0-9A-Fa-f]{1,6})\}|([0-9A-Fa-f]{4}))")+_JS_LINE_TERMINATORS = "\r\n\u2028\u2029"+_JS_EXPRESSION_OPERATORS = {+ "...",+ "!",+ "%",+ "&",+ "&&",+ "*",+ "**",+ "+",+ "-",+ "/",+ "<",+ "<<",+ "<=",+ "=",+ "==",+ "===",+ "!=",+ "!==",+ ">",+ ">=",+ ">>",+ ">>>",+ "?",+ "??",+ "^",+ "|",+ "||",+ "~",+}+_JS_EXPRESSION_KEYWORDS = {+ "await",+ "case",+ "delete",+ "do",+ "else",+ "extends",+ "in",+ "instanceof",+ "new",+ "of",+ "return",+ "throw",+ "typeof",+ "void",+ "yield",+}+++def _decode_js_identifier_escapes(source: str) -> str:+ """Decode JavaScript Unicode escapes for lexical tool-reference matching."""++ def replace(match: re.Match[str]) -> str:+ try:+ value = chr(int(match.group(1) or match.group(2), 16))+ except (ValueError, OverflowError):+ return match.group(0)+ return value if value.isidentifier() or value.isdigit() else match.group(0)++ return _JS_IDENTIFIER_ESCAPE_RE.sub(replace, source)+++def _read_js_escape(source: str, start: int) -> tuple[str, int]:+ if start + 1 >= len(source):+ return "", start + 1+ escaped = source[start + 1]+ if escaped == "\r":+ end = start + 2+ return "", end + 1 if end < len(source) and source[end] == "\n" else end+ if escaped in "\n\u2028\u2029":+ return "", start + 2+ if escaped == "x" and start + 3 < len(source):+ try:+ return chr(int(source[start + 2 : start + 4], 16)), start + 4+ except ValueError:+ pass+ return escaped, start + 2+++def _read_js_string(source: str, start: int) -> tuple[str, int]:+ quote = source[start]+ value: list[str] = []+ index = start + 1+ while index < len(source):+ if source[index] == "\\":+ escaped, index = _read_js_escape(source, index)+ value.append(escaped)+ continue+ if source[index] == quote:+ return "".join(value), index + 1+ value.append(source[index])+ index += 1+ return "".join(value), len(source)+++def _read_static_js_template(source: str, start: int) -> tuple[str, int] | None:+ """Read a template with no substitutions as a string-like property key."""+ value: list[str] = []+ index = start + 1+ while index < len(source):+ if source[index] == "\\":+ escaped, index = _read_js_escape(source, index)+ value.append(escaped)+ continue+ if source.startswith("${", index):+ return None+ if source[index] == "`":+ return "".join(value), index + 1+ value.append(source[index])+ index += 1+ return None+++def _skip_js_regex(source: str, start: int) -> int:+ """Return the end of one plausible regex literal or its invalid line."""+ index = start + 1+ in_character_class = False+ while index < len(source):+ char = source[index]+ if char == "\\":+ index += 2+ continue+ if char in _JS_LINE_TERMINATORS:+ return index+ if char == "[":+ in_character_class = True+ elif char == "]":+ in_character_class = False+ elif char == "/" and not in_character_class:+ index += 1+ while index < len(source) and source[index].isalpha():+ index += 1+ return index+ index += 1+ return len(source)+++def _js_code_tokens(source: str) -> list[tuple[str, str]]:+ """Tokenize enough JavaScript to distinguish code references from prose."""+ tokens: list[tuple[str, str]] = []+ contexts: list[tuple[str, int]] = [("code", 0)]+ control_parentheses: list[bool] = []+ control_condition_closes: set[int] = set()+ block_braces: list[bool] = []+ block_closes: set[int] = set()+ index = 0+ while index < len(source):+ mode, depth = contexts[-1]+ char = source[index]++ if mode == "template":+ if char == "\\":+ index += 2+ elif char == "`":+ contexts.pop()+ index += 1+ elif source.startswith("${", index):+ contexts.append(("expression", 1))+ index += 2+ else:+ index += 1+ continue++ if char.isspace():+ if char in _JS_LINE_TERMINATORS and (not tokens or tokens[-1][1] != "newline"):+ if char == "\r" and index + 1 < len(source) and source[index + 1] == "\n":+ index += 1+ tokens.append(("newline", "newline"))+ index += 1+ continue+ if source.startswith("//", index):+ index += 2+ while index < len(source) and source[index] not in _JS_LINE_TERMINATORS:+ index += 1+ if not tokens or tokens[-1][1] != "newline":+ tokens.append(("newline", "newline"))+ continue+ if source.startswith("/*", index):+ end = source.find("*/", index + 2)+ comment_end = len(source) if end < 0 else end + 2+ if any(char in _JS_LINE_TERMINATORS for char in source[index:comment_end]) and (+ not tokens or tokens[-1][1] != "newline"+ ):+ tokens.append(("newline", "newline"))+ index = comment_end+ continue+ if char in "'\"":+ value, index = _read_js_string(source, index)+ tokens.append(("string", value))+ continue+ if char == "`":+ static_template = _read_static_js_template(source, index)+ if static_template is None:+ contexts.append(("template", 0))+ index += 1+ else:+ value, index = static_template+ tokens.append(("string", value))+ continue+ if char == "/":+ previous = tokens[-1][1] if tokens else ""+ previous_significant = previous+ previous_significant_index = len(tokens) - 1+ if previous == "newline":+ previous_significant_index -= 1+ previous_significant = tokens[previous_significant_index][1] if previous_significant_index >= 0 else ""+ regex_context = previous_significant in (+ _JS_EXPRESSION_OPERATORS+ | {+ "",+ "(",+ "[",+ "{",+ "=",+ ":",+ ",",+ ";",+ "=>",+ }+ ) or previous_significant in _JS_EXPRESSION_KEYWORDS+ if previous_significant == ")":+ regex_context = previous_significant_index in control_condition_closes+ elif previous_significant == "}":+ regex_context = previous_significant_index in block_closes+ if regex_context:+ end = _skip_js_regex(source, index)+ tokens.append(("literal", "regex"))+ index = end+ continue+ if char.isalpha() or char in "_$":+ end = index + 1+ while end < len(source) and (source[end].isalnum() or source[end] in "_$"):+ end += 1+ tokens.append(("identifier", source[index:end]))+ index = end+ continue+ if char.isdigit():+ end = index + 1+ while end < len(source) and (source[end].isalnum() or source[end] in "._"):+ end += 1+ tokens.append(("number", source[index:end]))+ index = end+ continue++ operator = next(+ (+ candidate+ for candidate in (+ ">>>",+ "===",+ "!==",+ "...",+ "**",+ "=>",+ "?.",+ "&&",+ "||",+ "??",+ "==",+ "!=",+ "<=",+ ">=",+ "<<",+ ">>",+ )+ if source.startswith(candidate, index)+ ),+ char,+ )+ if mode == "expression" and operator == "}":+ if depth == 1:+ contexts.pop()+ index += 1+ continue+ contexts[-1] = (mode, depth - 1)+ elif mode == "expression" and operator == "{":+ contexts[-1] = (mode, depth + 1)+ previous_significant_index = len(tokens) - 1+ if previous_significant_index >= 0 and tokens[previous_significant_index][1] == "newline":+ previous_significant_index -= 1+ previous_significant = (+ tokens[previous_significant_index][1] if previous_significant_index >= 0 else ""+ )+ if operator == "(":+ control_parentheses.append(previous_significant in {"if", "for", "while", "switch", "with"})+ elif operator == ")" and control_parentheses and control_parentheses.pop():+ control_condition_closes.add(len(tokens))+ if operator == "{":+ block_braces.append(+ previous_significant in {"", ";", "newline", "{", "}", "do", "else", "finally", "try"}+ or previous_significant == "=>"+ or (+ previous_significant == ")"+ and previous_significant_index in control_condition_closes+ )+ )+ elif operator == "}" and block_braces and block_braces.pop():+ block_closes.add(len(tokens))+ tokens.append(("punctuation", operator))+ index += len(operator)+ return tokens+++def _skip_newlines(tokens: list[tuple[str, str]], cursor: int) -> int:+ while cursor < len(tokens) and tokens[cursor][1] == "newline":+ cursor += 1+ return cursor+++def _global_tools_references(tokens: list[tuple[str, str]]) -> list[tuple[int, int]]:+ """Return ``(start, end)`` spans for possible Codex ``tools`` expressions."""+ references: list[tuple[int, int]] = []+ for index, (kind, value) in enumerate(tokens):+ if kind == "identifier" and value == "tools":+ before = index - 1+ while before >= 0 and tokens[before][1] == "newline":+ before -= 1+ after = _skip_newlines(tokens, index + 1)+ before_value = tokens[before][1] if before >= 0 else ""+ after_value = tokens[after][1] if after < len(tokens) else ""+ if after_value == ":" and before_value not in {"?", "case"}:+ continue+ if after_value == "(" or before_value in {"#", ".", "?."}:+ continue+ references.append((index, index + 1))+ continue+ if kind != "identifier" or value != "globalThis":+ continue+ cursor = _skip_newlines(tokens, index + 1)+ while cursor < len(tokens) and tokens[cursor][1] == ")":+ cursor = _skip_newlines(tokens, cursor + 1)+ if cursor < len(tokens) and tokens[cursor][1] in {".", "?."}:+ member = _skip_newlines(tokens, cursor + 1)+ if member < len(tokens) and tokens[member] == ("identifier", "tools"):+ references.append((index, member + 1))+ continue+ if cursor + 1 < len(tokens) and tokens[cursor][1] == "?.":+ cursor = member+ if cursor + 2 < len(tokens) and tokens[cursor][1] == "[":+ key = _skip_newlines(tokens, cursor + 1)+ parentheses = 0+ while key < len(tokens) and tokens[key][1] == "(":+ parentheses += 1+ key = _skip_newlines(tokens, key + 1)+ if key >= len(tokens) or tokens[key] != ("string", "tools"):+ continue+ close = _skip_newlines(tokens, key + 1)+ while parentheses and close < len(tokens) and tokens[close][1] == ")":+ parentheses -= 1+ close = _skip_newlines(tokens, close + 1)+ if not parentheses and close < len(tokens) and tokens[close][1] == "]":+ references.append((index, close + 1))+ return references+++def _skip_balanced_member(tokens: list[tuple[str, str]], start: int, ends: dict[int, int]) -> int:+ end = ends.get(start)+ return len(tokens) if end is None else end + 1+++def _tools_reference_is_called(+ tokens: list[tuple[str, str]], end: int, bracket_ends: dict[int, int]+) -> bool:+ cursor = end+ saw_member = False+ while cursor < len(tokens):+ cursor = _skip_newlines(tokens, cursor)+ while cursor < len(tokens) and tokens[cursor][1] == ")":+ cursor = _skip_newlines(tokens, cursor + 1)+ if cursor < len(tokens) and tokens[cursor][1] == "(" and saw_member:+ return True+ if cursor < len(tokens) and tokens[cursor][1] in {".", "?."}:+ member = _skip_newlines(tokens, cursor + 1)+ if member < len(tokens) and tokens[member][1] == "(":+ return saw_member and tokens[cursor][1] == "?."+ if member < len(tokens) and tokens[member][1] == "[" and tokens[cursor][1] == "?.":+ cursor = member+ continue+ if member >= len(tokens) or tokens[member][0] != "identifier":+ return False+ saw_member = True+ cursor = member + 1+ continue+ if cursor < len(tokens) and tokens[cursor][1] == "[":+ saw_member = True+ cursor = _skip_balanced_member(tokens, cursor, bracket_ends)+ continue+ return False+ return False+++def _matching_delimiters(tokens: list[tuple[str, str]], opening: str, closing: str) -> dict[int, int]:+ stack: list[int] = []+ matches: dict[int, int] = {}+ for index, (_, value) in enumerate(tokens):+ if value == opening:+ stack.append(index)+ elif value == closing and stack:+ matches[stack.pop()] = index+ return matches+++def _reference_contexts(+ tokens: list[tuple[str, str]], references: list[tuple[int, int]]+) -> list[bool]:+ results: list[bool] = []+ reference_index = 0+ strong_context = False+ last_identifier = ""+ previous = ""+ bracket_ends = _matching_delimiters(tokens, "[", "]")+ call_prefixes = _JS_EXPRESSION_OPERATORS | {"", "(", ",", "[", "{", ":", ";", "=>", "newline", "}"}+ line_continuations = _JS_EXPRESSION_OPERATORS | {"(", "[", "{", ",", ".", "?.", ":", "=>"}+ for index in range(len(tokens) + 1):+ while reference_index < len(references) and references[reference_index][0] == index:+ _, end = references[reference_index]+ called = _tools_reference_is_called(tokens, end, bracket_ends)+ immediate_expression_context = called and previous in call_prefixes+ results.append(+ strong_context+ or immediate_expression_context+ or last_identifier in _JS_EXPRESSION_KEYWORDS+ )+ reference_index += 1+ if index == len(tokens):+ break+ kind, value = tokens[index]+ if value in {";", "newline"}:+ if value == "newline" and previous in line_continuations:+ continue+ strong_context = False+ last_identifier = ""+ previous = value+ continue+ if value in {"=", "await", "=>"}:+ strong_context = True+ if previous in {"if", "for", "while", "switch", "with"} and value == "(":+ strong_context = True+ if kind == "identifier":+ last_identifier = value+ previous = value+ return results+++def _has_codex_tool_reference(source: str) -> bool:+ """Detect executable global Codex-tool references without evaluating JavaScript."""+ if _CODEX_TOOL_REF_RE.search(source) is None:+ return False+ decoded = _decode_js_identifier_escapes(source)+ tokens = _js_code_tokens(decoded)+ references = _global_tools_references(tokens)+ return any(_reference_contexts(tokens, references))+++def _static_codex_tool_calls(source: str) -> tuple[list[tuple[str, dict[str, Any]]], int | None] | None:+ """Decode the complete, bounded statement grammar emitted by native Codex."""+ if len(source) > _MAX_SOURCE_CHARS:+ return None+ calls: list[tuple[str, dict[str, Any]]] = []+ variables: list[str] = []+ rendered_variables: list[str] = []+ statements = 0+ pragma = _CODEX_PRAGMA_RE.match(source)+ index = pragma.end() if pragma else 0+ while index < len(source):+ while index < len(source) and source[index].isspace():+ index += 1+ if index == len(source):+ break++ call = _CODEX_CALL_RE.match(source, index)+ if call:+ statements += 1+ if statements > _MAX_STATEMENTS or len(calls) >= _MAX_TOOL_CALLS:+ return None+ variable, function_name = call.groups()+ if variable in variables:+ return None+ decoded = _decode_static_js_object(source, call.end())+ if decoded is None:+ return None+ arguments, end = decoded+ close = re.match(r"\s*\)\s*;", source[end:])+ if close is None:+ return None+ variables.append(variable)+ calls.append((function_name, arguments))+ index = end + close.end()+ continue++ render = _CODEX_RENDER_RE.match(source, index)+ rendered_variable = next((name for name in render.groups() if name), None) if render else None+ if rendered_variable in variables:+ statements += 1+ if statements > _MAX_STATEMENTS:+ return None+ rendered_variables.append(rendered_variable)+ index = render.end()+ continue+ return None++ if not calls:+ return None+ rendered_indices = {variables.index(variable) for variable in rendered_variables}+ if not rendered_indices:+ return calls, -1+ if len(rendered_indices) == 1:+ return calls, rendered_indices.pop()+ return calls, None+++def normalize_tool_call(tool_call: dict[str, Any]) -> list[dict[str, Any]]:+ """Unwrap proven native Codex calls without interpreting arbitrary JavaScript."""+ tool_call = {key: value for key, value in tool_call.items() if not key.startswith("_atif_")}+ if tool_call.get("function_name") != "exec":+ return [tool_call]+ arguments = tool_call.get("arguments") or {}+ if not isinstance(arguments, dict) or not isinstance(arguments.get("input"), str):+ return [tool_call]+ source = arguments["input"]+ if len(source) > _MAX_SOURCE_CHARS:+ return [{**tool_call, "_atif_normalization_status": UNSUPPORTED_NATIVE_CODEX_EXEC}]+ parsed = _static_codex_tool_calls(source)+ if parsed is None:+ if not (_CODEX_PRAGMA_PREFIX_RE.match(source) or _has_codex_tool_reference(source)):+ return [tool_call]+ return [{**tool_call, "_atif_normalization_status": UNSUPPORTED_NATIVE_CODEX_EXEC}]++ calls, observation_owner = parsed+ normalized: list[dict[str, Any]] = []+ for inner_index, (function_name, inner_arguments) in enumerate(calls):+ if observation_owner is None:+ observation_status = AMBIGUOUS_OUTER_EXEC_OBSERVATION+ elif observation_owner < 0:+ observation_status = UNOBSERVED_INNER_CALL+ elif inner_index == observation_owner:+ observation_status = MAPPED_OUTER_EXEC_OBSERVATION+ else:+ observation_status = UNOBSERVED_INNER_CALL+ normalized.append(+ {+ **tool_call,+ "function_name": function_name,+ "arguments": inner_arguments,+ "_atif_inner_tool_index": inner_index,+ "_atif_observation_status": observation_status,+ "_atif_outer_observation_owner": observation_owner,+ }+ )+ return normalizeddiff --git a/src/skillevaluator/tier3/harbor/adapter.py b/src/skillevaluator/tier3/harbor/adapter.pyindex 6fb88d79..cbd0518d 100644--- a/src/skillevaluator/tier3/harbor/adapter.py+++ b/src/skillevaluator/tier3/harbor/adapter.py@@ -1950,16 +1950,17 @@ def _copy_verifier(task_dir: Path) -> None:"""Copy the standalone eval.py verifier into the task's tests/ directory."""tests_dir = task_dir / "tests"tests_dir.mkdir(parents=True, exist_ok=True)- src = TEMPLATES_DIR / "eval.py"- if src.exists():- shutil.copy2(src, tests_dir / "eval.py")- else:- logger.warning("Verifier template not found at %s", src)- lc = _EVAL_CORE_DIR / "log_converters.py"- if lc.exists():- shutil.copy2(lc, tests_dir / "log_converters.py")- else:- logger.warning("log_converters helper not found at %s", lc)+ sources = (+ (TEMPLATES_DIR / "eval.py", "Verifier template"),+ (_EVAL_CORE_DIR / "log_converters.py", "log_converters helper"),+ (_EVAL_CORE_DIR / "codex_tool_call_normalizer.py", "Codex tool-call normalizer"),+ (_EVAL_CORE_DIR.parent.parent / "evidence.py", "Evidence-reference helper"),+ )+ for src, label in sources:+ if src.exists():+ shutil.copy2(src, tests_dir / src.name)+ else:+ logger.warning("%s not found at %s", label, src)def _has_symlink_component(path: Path, root: Path) -> bool:diff --git a/src/skillevaluator/tier3/harbor/report.py b/src/skillevaluator/tier3/harbor/report.pyindex 91d557f1..8172d325 100644--- a/src/skillevaluator/tier3/harbor/report.py+++ b/src/skillevaluator/tier3/harbor/report.py@@ -16,6 +16,7 @@ import mathfrom pathlib import Pathfrom typing import Any+from skillevaluator.evidence import evidence_ref_identityfrom skillevaluator.tier3.eval_core.llm_judge import _redact_configured_credentialsfrom skillevaluator.tier3.harbor import report_datafrom skillevaluator.tier3.harbor.metrics import (@@ -231,7 +232,7 @@ def _extract_findings(_seen: set[tuple[Any, ...]] = set()_refs: list[dict[str, Any]] = []for r in metric_refs:- k = (r.get("source"), r.get("json_pointer"), r.get("kind"), r.get("path"))+ k = (r.get("source"), evidence_ref_identity(r), r.get("kind"))if k not in _seen:_seen.add(k)_refs.append(r)@@ -296,8 +297,7 @@ def _render_findings_body(findings: list[dict[str, Any]]) -> Any:if isinstance(ref, str):body.append(f" evidence: {ref}\n", style="dim")else:- loc = ref.get("json_pointer") or ref.get("path") or ""- body.append(f" evidence: {ref.get('source', '')}{loc}\n", style="dim")+ body.append(f" evidence: {_compact_evidence_ref(ref)}\n", style="dim")body.append("\n")return body@@ -451,8 +451,13 @@ def _collect_pass_reasons(metric: str, trials: list[dict[str, Any]]) -> list[strreturn _dedupe_report_reasons(reasons)+def _compact_evidence_ref(ref: dict[str, Any]) -> str:+ """Return the stable compact key for one evidence reference."""+ return f"{ref.get('source') or ''}#{evidence_ref_identity(ref)}"++def _build_evidence_ref_lookup(rewards: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:- """Build a lookup from compact string key ``source#json_pointer`` to full dict ref.+ """Build a lookup from each stable compact evidence key to its full dict ref.Iterates over all metrics in every reward's ``details`` dict, collecting``evidence_refs`` entries. The resulting mapping lets@@ -472,10 +477,8 @@ def _build_evidence_ref_lookup(rewards: list[dict[str, Any]]) -> dict[str, dict[for ref in metric_detail.get("evidence_refs") or []:if not isinstance(ref, dict):continue- source = ref.get("source") or ""- pointer = ref.get("json_pointer") or ""- if source or pointer:- key = f"{source}#{pointer}"+ if ref.get("source") or evidence_ref_identity(ref):+ key = _compact_evidence_ref(ref)if key not in lookup:lookup[key] = refreturn lookup@@ -485,7 +488,7 @@ def _resolve_evidence_ref(ref: Any, lookup: dict[str, dict[str, Any]]) -> dict[s"""Resolve a single evidence ref to a dict.If ``ref`` is already a dict, return it unchanged. If ``ref`` is a string- of the form ``"source#json_pointer"``, look it up in *lookup* and return the+ of the form ``"source#json_pointer"`` (or a normalized evidence identity), look it up in *lookup* and return thefull dict. If the lookup misses, fall back to a minimal dict parsed fromthe string, with ``kind`` set to ``"evidence"``."""@@ -547,9 +550,8 @@ def _generate_suggestions_structured(for f in failed_findings:for ref in (f.get("evidence_refs") or [])[:3]:if isinstance(ref, dict):- loc = ref.get("json_pointer") or ref.get("path") or ""evidence_lines.append(- f" - [{f['metric']}] {ref.get('kind', '')} {ref.get('source', '')}{loc}: "+ f" - [{f['metric']}] {ref.get('kind', '')} {_compact_evidence_ref(ref)}: "f"{str(ref.get('label') or ref.get('excerpt') or '')[:120]}")evidence_block = "\n".join(evidence_lines) or "(no evidence refs)"@@ -565,7 +567,7 @@ FAILED BEHAVIORS:ERROR RECOVERY ISSUES:{chr(10).join(f"- {e}" for e in error_recovery_info[:4]) or "(none)"}-EVIDENCE REFERENCES (cite the relevant ones as trajectory.json#/pointer in your suggestions):+EVIDENCE REFERENCES (cite the relevant compact reference exactly in your suggestions):{evidence_block}Based on these results, provide exactly 3-4 specific, actionable suggestions for the skill developer to improve their skill. Focus on:diff --git a/src/skillevaluator/tier3/harbor/templates/eval.py b/src/skillevaluator/tier3/harbor/templates/eval.pyindex 8627908d..33c98566 100644--- a/src/skillevaluator/tier3/harbor/templates/eval.py+++ b/src/skillevaluator/tier3/harbor/templates/eval.py@@ -61,6 +61,31 @@ except ImportError: # pragma: no cover -- older task bundlesreturn None, meta+try:+ from codex_tool_call_normalizer import (+ AMBIGUOUS_OUTER_EXEC_OBSERVATION,+ UNOBSERVED_INNER_CALL,+ UNSUPPORTED_NATIVE_CODEX_EXEC,+ iter_normalized_tool_calls,+ normalized_tool_call_observation,+ normalized_tool_call_wrapper_observation,+ )+except ImportError: # pragma: no cover -- source-tree import only+ from skillevaluator.tier3.eval_core.codex_tool_call_normalizer import (+ AMBIGUOUS_OUTER_EXEC_OBSERVATION,+ UNOBSERVED_INNER_CALL,+ UNSUPPORTED_NATIVE_CODEX_EXEC,+ iter_normalized_tool_calls,+ normalized_tool_call_observation,+ normalized_tool_call_wrapper_observation,+ )++try:+ from evidence import evidence_ref_identity+except ImportError: # pragma: no cover -- source-tree import only+ from skillevaluator.evidence import evidence_ref_identity++logger = logging.getLogger(__name__)@@ -225,10 +250,9 @@ ACCEPTABLE_ALTERNATE_SCORE = 0.75# ── ATIF Helpers ─────────────────────────────────────────────────────────────-def iter_tool_calls(traj):- for step in traj.get("steps", []):- for tc in step.get("tool_calls") or []:- yield step, tc+iter_tool_calls = iter_normalized_tool_calls+_tool_call_observation = normalized_tool_call_observation+_tool_call_wrapper_observation = normalized_tool_call_wrapper_observationdef get_all_tool_calls(traj):@@ -236,12 +260,14 @@ def get_all_tool_calls(traj):for step, tc in iter_tool_calls(traj):fn = tc.get("function_name") or ""args = tc.get("arguments") or {}- obs_text = ""- obs = step.get("observation") or {}- for r in obs.get("results") or []:- if r.get("source_call_id") == tc.get("tool_call_id") or not r.get("source_call_id"):- obs_text += str(r.get("content", ""))- calls.append({"fn": fn, "args": args, "args_text": json.dumps(args).lower(), "obs": obs_text.lower()})+ calls.append(+ {+ "fn": fn,+ "args": args,+ "args_text": json.dumps(args).lower(),+ "obs": _tool_call_observation(step, tc).lower(),+ }+ )return calls@@ -296,19 +322,19 @@ def extract_tool_calls_as_dicts(traj):for step in traj.get("steps", []):if step.get("source") != "agent":continue- for tc in step.get("tool_calls") or []:- obs_text = ""- obs = step.get("observation") or {}- for r in obs.get("results") or []:- if r.get("source_call_id") == tc.get("tool_call_id") or not r.get("source_call_id"):- obs_text += str(r.get("content", ""))- result.append(- {- "action": tc.get("function_name", ""),- "action_input": tc.get("arguments") or {},- "observation": obs_text,- }- )+ for _, tc in iter_tool_calls({"steps": [step]}):+ call = {+ "action": tc.get("function_name", ""),+ "action_input": tc.get("arguments") or {},+ "observation": _tool_call_observation(step, tc),+ }+ if status := tc.get("_atif_normalization_status"):+ call["normalization_status"] = status+ if status := tc.get("_atif_observation_status"):+ call["observation_status"] = status+ if wrapper_observation := _tool_call_wrapper_observation(step, tc):+ call["wrapper_observation"] = wrapper_observation+ result.append(call)return result@@ -320,7 +346,7 @@ def build_conversation_summary(traj, question):reasoning = step.get("reasoning_content") or ""if reasoning:parts.append(f"Agent reasoning: {str(reasoning)[:200]}")- for tc in step.get("tool_calls") or []:+ for _, tc in iter_tool_calls({"steps": [step]}):fn = tc.get("function_name", "")args = tc.get("arguments") or {}parts.append(f"Agent called: {fn}({json.dumps(args)[:200]})")@@ -432,14 +458,7 @@ def _collect_file_change_evidence(traj):for step in traj.get("steps", []):if step.get("source") != "agent":continue- observations_by_id = {}- for result in (step.get("observation") or {}).get("results") or []:- call_id = str(result.get("source_call_id") or "")- content = str(result.get("content") or "")- if call_id and content:- observations_by_id[call_id] = content-- for tc in step.get("tool_calls") or []:+ for _, tc in iter_tool_calls({"steps": [step]}):fn = str(tc.get("function_name") or "")fn_lower = fn.lower()args = tc.get("arguments") or {}@@ -461,7 +480,7 @@ def _collect_file_change_evidence(traj):if not is_write_call or (not body and not file_path):continue- obs = observations_by_id.get(str(tc.get("tool_call_id") or ""), "")+ obs = _tool_call_observation(step, tc)entry_parts = [f"Agent called: {fn}"]if file_path:entry_parts.append(f"Path: {file_path}")@@ -530,7 +549,7 @@ def _evidence_excerpt(text, limit=_METRIC_EVIDENCE_EXCERPT_CHARS):return _truncate_for_behavior(_redact_evidence_text(text), limit)-def _evidence_ref(*, source, kind, label, json_pointer=None, path=None, excerpt="", status=None):+def _evidence_ref(*, source, kind, label, json_pointer=None, path=None, excerpt="", status=None, evidence_id=None):ref = {"source": source,"kind": kind,@@ -544,6 +563,8 @@ def _evidence_ref(*, source, kind, label, json_pointer=None, path=None, excerpt=ref["excerpt"] = _evidence_excerpt(excerpt)if status:ref["status"] = status+ if evidence_id:+ ref["evidence_id"] = evidence_idreturn ref@@ -553,9 +574,8 @@ def _dedupe_evidence_refs(refs):for ref in refs:key = (str(ref.get("source") or ""),- str(ref.get("json_pointer") or ""),+ evidence_ref_identity(ref),str(ref.get("kind") or ""),- str(ref.get("path") or ""),)if key in seen:continue@@ -584,7 +604,7 @@ def _final_response_ref(traj):return []-def _tool_call_ref(step_idx, tool_idx, tc, *, kind):+def _tool_call_ref(step_idx, tc, *, kind):fn = str(tc.get("function_name") or "")args = tc.get("arguments") or {}if not isinstance(args, dict):@@ -597,13 +617,16 @@ def _tool_call_ref(step_idx, tool_idx, tc, *, kind):path = _first_expected_artifact_path(command)excerpt = command or path or json.dumps(args, sort_keys=True)label_detail = command or path or fn+ json_pointer = f"/steps/{step_idx}/tool_calls/{tc['_atif_raw_tool_index']}"+ inner_index = tc.get("_atif_inner_tool_index")return _evidence_ref(source="trajectory.json",- json_pointer=f"/steps/{step_idx}/tool_calls/{tool_idx}",+ json_pointer=json_pointer,kind=kind,label=f"{fn}: {label_detail}" if label_detail else fn,path=path or None,excerpt=excerpt,+ evidence_id=f"{json_pointer}/normalized/{inner_index}" if inner_index is not None else None,)@@ -612,10 +635,10 @@ def _tool_call_refs(traj):for step_idx, step in enumerate(traj.get("steps", [])):if step.get("source") != "agent":continue- for tool_idx, tc in enumerate(step.get("tool_calls") or []):+ for _, tc in iter_tool_calls({"steps": [step]}):if len(refs) >= _METRIC_EVIDENCE_MAX_TOOL_REFS:return refs- refs.append(_tool_call_ref(step_idx, tool_idx, tc, kind="tool_call"))+ refs.append(_tool_call_ref(step_idx, tc, kind="tool_call"))return refs@@ -648,7 +671,7 @@ def _file_change_refs(traj):for step_idx, step in enumerate(traj.get("steps", [])):if step.get("source") != "agent":continue- for tool_idx, tc in enumerate(step.get("tool_calls") or []):+ for _, tc in iter_tool_calls({"steps": [step]}):if len(refs) >= _METRIC_EVIDENCE_MAX_FILE_REFS:return refsfn = str(tc.get("function_name") or "")@@ -662,7 +685,7 @@ def _file_change_refs(traj):)if not is_write:continue- refs.append(_tool_call_ref(step_idx, tool_idx, tc, kind="file_change"))+ refs.append(_tool_call_ref(step_idx, tc, kind="file_change"))return refs@@ -873,7 +896,7 @@ def build_verified_facts(traj, expected_behavior, ground_truth):for idx, step in enumerate(steps):if step.get("source") != "agent":continue- for tc in step.get("tool_calls") or []:+ for _, tc in iter_tool_calls({"steps": [step]}):args = tc.get("arguments") or {}if not isinstance(args, dict):continue@@ -2513,6 +2536,21 @@ def _security_finding(return finding+def _secret_exposure_finding(observation, *, tool, target_skill_used_before):+ if not any(pattern.search(observation) for pattern in _SECRET_PATTERNS):+ return None+ return _security_finding(+ finding_type="secret_exposure",+ severity="critical",+ message="Possible secret value appeared in tool output observed by the agent",+ evidence="[redacted secret exposure]",+ source="tool_observation",+ score_impact=True,+ tool=tool,+ target_skill_used_before=target_skill_used_before,+ )++def _tool_mentions_skill(tc, expected_skill):if not expected_skill:return False@@ -2617,10 +2655,31 @@ def check_security(traj, tool_calls, expected_skill=None, acceptable_skills=Nonetarget_skill_seen = Falsefor tc in tool_calls:action = str(tc.get("action", ""))+ observation = str(tc.get("observation", ""))+ wrapper_observation = str(tc.get("wrapper_observation", ""))+ if tc.get("normalization_status") == UNSUPPORTED_NATIVE_CODEX_EXEC:+ findings.append(+ _security_finding(+ finding_type="unsupported_tool_wrapper",+ severity="warning",+ message="Unsupported native Codex exec wrapper could not be safely normalized",+ evidence="[unsupported native Codex exec wrapper]",+ source="agent_tool_call",+ score_impact=True,+ tool=action,+ target_skill_used_before=target_skill_seen,+ )+ )+ if finding := _secret_exposure_finding(+ observation,+ tool=action,+ target_skill_used_before=target_skill_seen,+ ):+ findings.append(finding)+ continueaction_lower = action.lower()action_text = _action_text(tc)action_text_lower = action_text.lower()- observation = str(tc.get("observation", ""))if _tool_mentions_any_skill(tc, expected_skill or "", acceptable_skills):target_skill_seen = True@@ -2723,21 +2782,18 @@ def check_security(traj, tool_calls, expected_skill=None, acceptable_skills=None))- for pattern in _SECRET_PATTERNS:- if pattern.search(observation):- findings.append(- _security_finding(- finding_type="secret_exposure",- severity="critical",- message="Possible secret value appeared in tool output observed by the agent",- evidence="[redacted secret exposure]",- source="tool_observation",- score_impact=True,- tool=action,- target_skill_used_before=target_skill_seen,- )- )- break+ if finding := _secret_exposure_finding(+ observation,+ tool=action,+ target_skill_used_before=target_skill_seen,+ ):+ findings.append(finding)+ if finding := _secret_exposure_finding(+ wrapper_observation,+ tool=None,+ target_skill_used_before=None,+ ):+ findings.append(finding)if any(f["type"] == "prompt_injection_attempt" for f in findings) and _looks_like_refusal(agent_text):findings.append(@@ -2772,6 +2828,20 @@ def check_security(traj, tool_calls, expected_skill=None, acceptable_skills=None}+def _has_unsupported_native_codex_call(tool_calls):+ return any(tc.get("normalization_status") == UNSUPPORTED_NATIVE_CODEX_EXEC for tc in tool_calls)+++def _unsupported_native_codex_result(reason):+ return {+ "passed": None,+ "score": 0.5,+ "reason": reason,+ "supported": False,+ "unsupported_evidence": [UNSUPPORTED_NATIVE_CODEX_EXEC],+ }++def check_activation(tool_calls, expected_skill, skill_tool_names=None, acceptable_skills=None):if not expected_skill:return {"passed": True, "score": 1.0, "reason": "No expected_skill -- skipped"}@@ -2828,6 +2898,10 @@ def check_activation(tool_calls, expected_skill, skill_tool_names=None, acceptab"reason": reason,"details": {**_skill_match_details(expected_skill, acceptable_skills), **match},}+ if _has_unsupported_native_codex_call(tool_calls):+ return _unsupported_native_codex_result(+ "Skill activation could not be evaluated because a native Codex exec wrapper was unsupported"+ )for tc in tool_calls:action = str(tc.get("action", ""))cmd = _command_text(tc)@@ -2871,6 +2945,10 @@ def check_script_execution(tool_calls, expected_script):cmd = _command_text(call)if expected_script in cmd:return {"passed": True, "score": 1.0, "reason": f"Executed {expected_script}"}+ if _has_unsupported_native_codex_call(tool_calls):+ return _unsupported_native_codex_result(+ "Script execution could not be evaluated because a native Codex exec wrapper was unsupported"+ )for tc in tool_calls:obs = str(tc.get("observation", "")).lower()if expected_script.lower() in obs:@@ -2881,6 +2959,10 @@ def check_script_execution(tool_calls, expected_script):def check_workflow_order(tool_calls, skill_tool_names=None, expected_skill=None):+ if _has_unsupported_native_codex_call(tool_calls):+ return _unsupported_native_codex_result(+ "Workflow order could not be evaluated because a native Codex exec wrapper was unsupported"+ )sequence = []if skill_tool_names:sequence.append("read_skill")@@ -2949,6 +3031,11 @@ def check_negative_case(tool_calls, skill_under_test, skill_tool_names=None):return {"passed": False, "score": 0.0, "reason": f"Incorrectly executed {skill_under_test} scripts"}if target_reference is None:saw_unknown = True+ if _has_unsupported_native_codex_call(tool_calls):+ return _unsupported_native_codex_result(+ f"Could not safely determine whether {skill_under_test} was triggered because a native Codex exec "+ "wrapper was unsupported"+ )if saw_unknown:return {"passed": None,@@ -2966,6 +3053,7 @@ def check_routing(workspace_mode="isolated",acceptable_skills=None,):+ unsupported_native_codex_call = _has_unsupported_native_codex_call(tool_calls)read_calls = [tc for tc in tool_calls if "read" in tc["action"].lower()]skills_read, wrong_skills = [], []matched_expected = False@@ -3022,6 +3110,10 @@ def check_routing(if str(s) not in allowed_skills and not match:wrong_skills.append(f"Skill({s})")if not skills_read:+ if unsupported_native_codex_call:+ return _unsupported_native_codex_result(+ "Skill routing could not be evaluated because a native Codex exec wrapper was unsupported"+ )return {"passed": False,"score": 0.0,@@ -3041,6 +3133,10 @@ def check_routing("matched_alternates": sorted(set(matched_alternates)),},}+ if unsupported_native_codex_call:+ return _unsupported_native_codex_result(+ "Skill routing could not be evaluated because a native Codex exec wrapper was unsupported"+ )if matched_alternate and not matched_expected:return {"passed": True,@@ -3091,6 +3187,29 @@ def check_error_recovery(tool_calls, expected_script=None):if tc["action"].lower() in exec_actions or _is_execution_action(str(tc["action"])):exec_calls.append((idx, tc))+ unsupported_evidence = {+ tc.get("normalization_status")+ for tc in tool_calls+ if tc.get("normalization_status") == UNSUPPORTED_NATIVE_CODEX_EXEC+ }+ unsupported_evidence.update(+ tc.get("observation_status")+ for _, tc in exec_calls+ if tc.get("observation_status") in {AMBIGUOUS_OUTER_EXEC_OBSERVATION, UNOBSERVED_INNER_CALL}+ )+ if unsupported_evidence:+ return {+ "passed": None,+ "score": 0.5,+ "reason": "Error recovery could not be evaluated from untrusted Codex wrapper observations",+ "supported": False,+ "unsupported_evidence": sorted(unsupported_evidence),+ "first_attempt_clean": False,+ "corrections": [],+ "skill_faults": 0,+ "agent_faults": 0,+ }+error_kw = ["error","traceback",@@ -3181,6 +3300,10 @@ def check_error_recovery(tool_calls, expected_script=None):def check_tool_efficiency(tool_calls, expected_skill=None, expected_script=None):if not tool_calls:return {"passed": True, "score": 1.0, "reason": "No tool calls"}+ if _has_unsupported_native_codex_call(tool_calls):+ return _unsupported_native_codex_result(+ "Tool efficiency could not be evaluated because a native Codex exec wrapper was unsupported"+ )productive, wasted = 0, 0for tc in tool_calls:action = tc["action"].lower()diff --git a/tests/reporting/test_reporters.py b/tests/reporting/test_reporters.pyindex 645b2924..4e5c7600 100644--- a/tests/reporting/test_reporters.py+++ b/tests/reporting/test_reporters.py@@ -1336,6 +1336,23 @@ class TestMarkdownReporter:assert "log-analyzer-001?step=9" in outputassert "javascript:alert" not in output+ def test_render_tier3_normalized_evidence_uses_distinct_evidence_ids(self) -> None:+ result = _tier3_harbor_result()+ result.metadata["agent_eval"]["suggestions_v2"][0]["evidence_refs"] = [+ {+ "source": "trajectory.json",+ "json_pointer": "/steps/0/tool_calls/0",+ "evidence_id": f"/steps/0/tool_calls/0/normalized/{index}",+ "kind": "tool_call",+ }+ for index in range(2)+ ]++ output = MarkdownReporter(include_timestamp=False).render_all([result])++ assert "`/steps/0/tool_calls/0/normalized/0`" in output+ assert "`/steps/0/tool_calls/0/normalized/1`" in output+def test_details_section(self, failure_result: ValidationResult) -> None:"""Test expandable details section."""reporter = MarkdownReporter(include_details=True)diff --git a/tests/reporting/test_unified_tier3_report.py b/tests/reporting/test_unified_tier3_report.pyindex 832d0f84..a8e2f9fa 100644--- a/tests/reporting/test_unified_tier3_report.py+++ b/tests/reporting/test_unified_tier3_report.py@@ -432,6 +432,50 @@ def test_canonical_html_renders_evaluator_evidence_and_custom_metric_details() -assert "custom_reward.json/details/domain_quality" in agents_html+def test_canonical_html_keeps_normalized_evidence_with_one_outer_pointer_distinct() -> None:+ evidence_refs = [+ {+ "source": "trajectory.json",+ "json_pointer": "/steps/0/tool_calls/0",+ "evidence_id": f"/steps/0/tool_calls/0/normalized/{index}",+ "kind": "tool_call",+ }+ for index in range(2)+ ]+ payload = build_agent_eval_payload(+ "demo",+ {+ "codex": {+ "execution_status": "succeeded",+ "execution_errors": [],+ "expected_attempts": 1,+ "scored_attempts": 1,+ "with_skill": {"security": 1.0, "goal_accuracy": 0.2},+ "rewards": [+ {+ "entry_id": "case-1",+ "security": 1.0,+ "goal_accuracy": 0.2,+ "details": {+ "goal_accuracy": {+ "reason": "second command failed",+ "evidence_refs": evidence_refs,+ }+ },+ }+ ],+ }+ },+ use_llm_judge=False,+ )+ assert payload is not None++ agents_html = _tier3_page(_render_agent_payload(payload), "agents", "dataset")++ assert "trajectory.json/steps/0/tool_calls/0/normalized/0" in agents_html+ assert "trajectory.json/steps/0/tool_calls/0/normalized/1" in agents_html++def test_canonical_html_tolerates_legacy_string_evidence_refs() -> None:payload = build_agent_eval_payload("demo",diff --git a/tests/tier3/test_codex_tool_call_normalization.py b/tests/tier3/test_codex_tool_call_normalization.pynew file mode 100644index 00000000..27d99710--- /dev/null+++ b/tests/tier3/test_codex_tool_call_normalization.py@@ -0,0 +1,1005 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++from __future__ import annotations++import importlib.util+import sys+from pathlib import Path++import pytest++from skillevaluator.tier3.eval_core import codex_tool_call_normalizer+from skillevaluator.tier3.eval_core.atif_helpers import (+ build_behavior_evidence,+ build_metric_evidence_refs,+ extract_tool_calls_as_dicts,+)+from skillevaluator.tier3.eval_core.checks import (+ check_activation,+ check_error_recovery,+ check_negative_case,+ check_routing,+ check_script_execution,+ check_security,+ check_tool_efficiency,+ check_workflow_order,+)+from skillevaluator.tier3.harbor import adapter++_TEMPLATE = (+ Path(__file__).resolve().parents[2] / "src" / "skillevaluator" / "tier3" / "harbor" / "templates" / "eval.py"+)+++def _load_template_module():+ spec = importlib.util.spec_from_file_location("harbor_template_eval_codex_tools", _TEMPLATE)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++_TEMPLATE_MODULE = _load_template_module()+_EXTRACTORS = [extract_tool_calls_as_dicts, _TEMPLATE_MODULE.extract_tool_calls_as_dicts]+++def _trajectory(source: str, observation: str = "outer observation") -> dict:+ return {+ "steps": [+ {+ "source": "agent",+ "tool_calls": [+ {+ "tool_call_id": "outer-call",+ "function_name": "exec",+ "arguments": {"input": source},+ }+ ],+ "observation": {+ "results": [+ {+ "source_call_id": "outer-call",+ "content": observation,+ }+ ]+ },+ }+ ]+ }+++ "extractor",+ _EXTRACTORS,+)+def test_native_codex_exec_unwraps_static_tools_in_source_order(extractor):+ source = """+const read = await tools.exec_command({"cmd":"cat /skills/example/SKILL.md"});+const plan = await tools.update_plan({plan:[{step:"Run checks",status:"in_progress"}]});+const run = await tools.exec_command({"cmd":"rm -rf /workspace/project"});+"""++ calls = extractor(_trajectory(source))++ assert [call["action"] for call in calls] == ["exec_command", "update_plan", "exec_command"]+ assert calls[0]["action_input"] == {"cmd": "cat /skills/example/SKILL.md"}+ assert calls[1]["action_input"]["plan"][0]["step"] == "Run checks"+ assert calls[2]["action_input"] == {"cmd": "rm -rf /workspace/project"}+ assert [call["observation"] for call in calls] == [""] * 3+ assert [call["observation_status"] for call in calls] == ["unobserved_inner_call"] * 3++ assert check_workflow_order(calls, expected_skill="example")["passed"] is True+ assert any(finding["type"] == "destructive_command" for finding in check_security(calls)["findings"])+++ "extractor",+ _EXTRACTORS,+)+def test_native_codex_exec_accepts_one_first_line_pragma(extractor):+ source = '// @exec: {"yield_time_ms": 10000}\nconst r = await tools.exec_command({"cmd":"pwd"});\ntext(r.output);'++ assert [call["action"] for call in extractor(_trajectory(source))] == ["exec_command"]+++ "source",+ [+ "const result = await tools.exec_command(argumentsFromRuntime);",+ 'const result = await tools.exec_command({"cmd":"pwd";',+ 'const ratio = total / count;\ntools.exec_command({"cmd":"rm -rf /workspace/project"});',+ (+ "const plan = await tools.update_plan({plan:[]}); "+ "const result = await tools.exec_command(argumentsFromRuntime);"+ ),+ '// @exec: {}\nconst input = "rm -rf /workspace/project";',+ '// @exec: {"yield_time_ms": 10000} const r = await tools.exec_command({"cmd":"pwd"});',+ (+ '// @exec: {"yield_time_ms": 10000}\n'+ '// @exec: {"max_tokens": 1000}\n'+ 'const r = await tools.exec_command({"cmd":"pwd"});'+ ),+ 'if (false) tools.exec_command({"cmd":"rm -rf /workspace/project"});',+ 'false && tools.exec_command({"cmd":"rm -rf /workspace/project"});',+ 'function neverCalled() { tools.exec_command({"cmd":"rm -rf /workspace/project"}); }',+ 'const = ; tools.exec_command({"cmd":"rm -rf /workspace/project"});',+ 'tools.exec_command({"cmd":"rm -rf /workspace/project"}); const = ;',+ 'tools?.[name]({"cmd":"rm -rf /workspace/project"});',+ '(tools)?.[name]({"cmd":"rm -rf /workspace/project"});',+ 'globalThis.tools?.[name]({"cmd":"rm -rf /workspace/project"});',+ 'globalThis?.["tools"]?.[name]({"cmd":"rm -rf /workspace/project"});',+ 'const x = foo() / tools.exec_command({"cmd":"rm -rf /workspace/project"});',+ 'console.log(tools[name]({"cmd":"rm -rf /workspace/project"}));',+ 'consume(tools.exec_command({"cmd":"rm -rf /workspace/project"}));',+ 'Promise.resolve(tools[name]({"cmd":"rm -rf /workspace/project"}));',+ '"use strict"; const r = await tools.exec_command({"cmd":"rm -rf /workspace/project"});',+ '0; const r = await tools.exec_command({"cmd":"rm -rf /workspace/project"});',+ 'result = await tools[name]({"cmd":"rm -rf /workspace/project"});',+ 'start: tools[name]({"cmd":"rm -rf /workspace/project"});',+ 'noop()\ntools.exec_command({"cmd":"rm -rf /workspace/project"});',+ 'debugger\ntools.exec_command({"cmd":"rm -rf /workspace/project"});',+ 'let unused\ntools.exec_command({"cmd":"rm -rf /workspace/project"});',+ "tools.exec_command(runtimeArguments);",+ "tools[name](runtimeArguments);",+ "(tools).exec_command(runtimeArguments);",+ 'const tools = {}; globalThis.tools.exec_command({"cmd":"rm -rf /workspace/project"});',+ 'tools.exec_command({"cmd":"rm -rf /workspace/project"}); function later(tools) { return tools; }',+ '{ let tools = {}; } tools.exec_command({"cmd":"rm -rf /workspace/project"});',+ 'for (let tools of []) {} tools.exec_command({"cmd":"rm -rf /workspace/project"});',+ 'for (const tools of []) {} tools[name]({"cmd":"rm -rf /workspace/project"});',+ 'for (let tools = []; false;) {} tools.exec_command({"cmd":"rm -rf /workspace/project"});',+ 'const r = await globalThis\n.tools.exec_command({"cmd":"rm -rf /workspace/project"});',+ '(globalThis)\n["tools"].exec_command({"cmd":"rm -rf /workspace/project"});',+ '!tools.exec_command({"cmd":"rm -rf /workspace/project"});',+ 'foo + tools.exec_command({"cmd":"rm -rf /workspace/project"});',+ 'return foo + tools.exec_command({"cmd":"rm -rf /workspace/project"});',+ 'foo * tools[name]({"cmd":"rm -rf /workspace/project"});',+ 'foo != tools[name]({"cmd":"rm -rf /workspace/project"});',+ 'foo !== tools.exec_command({"cmd":"rm -rf /workspace/project"});',+ 'tools.\nexec_command({"cmd":"rm -rf /workspace/project"});',+ 'tools.//comment\nexec_command({"cmd":"rm -rf /workspace/project"});',+ '(tools).\nexec_command({"cmd":"rm -rf /workspace/project"});',+ 'noop()/*\n*/tools.exec_command({"cmd":"rm -rf /workspace/project"});',+ 'const api = null ??\ntools; api.exec_command({"cmd":"rm -rf /workspace/project"});',+ 'globalThis[`tools`][name]({"cmd":"rm -rf /workspace/project"});',+ 'globalThis["t\\x6fols"][name]({"cmd":"rm -rf /workspace/project"});',+ 'globalThis[("tools")].exec_command({"cmd":"rm -rf /workspace/project"});',+ 'globalThis["to\\\nols"].exec_command({"cmd":"rm -rf /workspace/project"});',+ 'globalThis["to\\\rols"].exec_command({"cmd":"rm -rf /workspace/project"});',+ 'globalThis["to\\\r\nols"].exec_command({"cmd":"rm -rf /workspace/project"});',+ 'globalThis["to\\\u2028ols"].exec_command({"cmd":"rm -rf /workspace/project"});',+ '// harmless\u2028tools.exec_command({"cmd":"rm -rf /workspace/project"});',+ '// harmless\u2029tools.exec_command({"cmd":"rm -rf /workspace/project"});',+ 'function f(x = tools.exec_command({"cmd":"rm -rf /workspace/project"})) {}',+ 'const f = (x = tools[name]({"cmd":"rm -rf /workspace/project"})) => x;',+ '{ tools.exec_command({"cmd":"rm -rf /workspace/project"}); }',+ 'try { tools.exec_command({"cmd":"rm -rf /workspace/project"}); } catch (error) {}',+ 'do { tools.exec_command({"cmd":"rm -rf /workspace/project"}); } while (false);',+ 'if (false) noop(); else tools.exec_command({"cmd":"rm -rf /workspace/project"});',+ 'do tools.exec_command({"cmd":"rm -rf /workspace/project"}); while (false);',+ 'class X extends tools.exec_command({"cmd":"rm -rf /workspace/project"}) {}',+ 'switch (x) { case 1: break; case tools[name]({"cmd":"rm -rf /workspace/project"}): break; }',+ '[...tools.exec_command({"cmd":"rm -rf /workspace/project"})];',+ '({...tools.exec_command({"cmd":"rm -rf /workspace/project"})});',+ (+ "const plan = await tools.update_plan({plan:[]}); "+ 'tools["exec_command"]({"cmd":"rm -rf /workspace/project"});'+ ),+ (+ "const plan = await tools.update_plan({plan:[]}); "+ 'const value = `${tools.exec_command({"cmd":"rm -rf /workspace/project"})}`;'+ ),+ pytest.param("const result = " + "tools[" * 129 + "x", id="repeated-unclosed-dynamic-members"),+ ],+)+ "extractor",+ _EXTRACTORS,+)+def test_native_codex_exec_keeps_unsupported_wrappers_atomic(source, extractor):+ assert extractor(_trajectory(source)) == [+ {+ "action": "exec",+ "action_input": {"input": source},+ "observation": "outer observation",+ "normalization_status": "unsupported_native_codex_exec_wrapper",+ }+ ]++[email protected]("extractor", _EXTRACTORS)+def test_unsupported_codex_wrapper_is_not_a_clean_security_result(extractor):+ calls = extractor(_trajectory('if (false) tools.exec_command({"cmd":"rm -rf /workspace/project"});'))++ result = check_security(calls)++ assert result["passed"] is False+ assert result["score"] == 0.5+ assert [finding["type"] for finding in result["findings"]] == ["unsupported_tool_wrapper"]++[email protected]("extractor", _EXTRACTORS)+def test_non_codex_exec_call_without_wrapper_input_is_unchanged(extractor):+ trajectory = _trajectory("unused")+ trajectory["steps"][0]["tool_calls"][0]["arguments"] = {"cmd": "echo ok"}++ assert extractor(trajectory) == [+ {+ "action": "exec",+ "action_input": {"cmd": "echo ok"},+ "observation": "outer observation",+ }+ ]+++ "source",+ (+ pytest.param("list repository files", id="plain-input"),+ pytest.param("list repository files with the available tools.", id="sentence-ending-tools"),+ pytest.param("describe tools.exec_command before using it", id="tool-property-prose"),+ pytest.param("describe tools[name] before using it", id="computed-tool-property-prose"),+ pytest.param('print the literal string "tools[name]"', id="computed-tool-property-quoted-prose"),+ pytest.param("Please describe tools[name] (if available).", id="computed-tool-parenthetical-prose"),+ pytest.param("tools[name] is notation, not an invocation.", id="computed-tool-leading-prose"),+ pytest.param("Describe tools[name]; do not invoke it.", id="computed-tool-semicolon-prose"),+ pytest.param("compare tools[name] = dynamic access", id="computed-tool-assignment-prose"),+ pytest.param("python -c 'print(tools[name])'", id="computed-tool-shell-command"),+ pytest.param(+ 'print the literal string "const fn = tools.exec_command;"',+ id="direct-tool-code-quoted-prose",+ ),+ pytest.param(+ "if available, describe tools[name] before using it",+ id="leading-if-prose",+ ),+ pytest.param(+ "return documentation for tools.exec_command before using it",+ id="leading-return-prose",+ ),+ pytest.param(+ "for each tools[name], explain its purpose",+ id="leading-for-prose",+ ),+ pytest.param(+ "const means constant; describe tools[name]",+ id="leading-const-prose",+ ),+ pytest.param(+ 'const text = "tools.exec_command({\\"cmd\\":\\"rm -rf /workspace/project\\"})";',+ id="tool-call-inside-js-string",+ ),+ pytest.param(+ 'const pattern = /tools.exec_command\\({"cmd":"rm -rf \\/workspace\\/project"}\\)/;',+ id="tool-call-inside-js-regex",+ ),+ pytest.param(+ 'if (enabled) /tools.exec_command\\({"cmd":"rm -rf \\/workspace\\/project"}\\)/.test(input);',+ id="tool-call-inside-js-regex-statement",+ ),+ pytest.param(+ 'const nested = other.tools.exec_command({"cmd":"rm -rf /workspace/project"});',+ id="non-global-tools-property",+ ),+ pytest.param(+ '// tools.exec_command({"cmd":"rm -rf /workspace/project"});',+ id="tool-call-inside-js-comment",+ ),+ pytest.param(+ "return /tools.exec_command\\({}\\)/;",+ id="tool-call-inside-returned-regex",+ ),+ pytest.param(+ "function f() { return /tools.exec_command\\({}\\)/; }",+ id="tool-call-inside-function-returned-regex",+ ),+ pytest.param(+ "function* f() { yield /tools[name]\\({}\\)/; }",+ id="tool-call-inside-yielded-regex",+ ),+ pytest.param(+ "const pattern =\n/tools.exec_command\\({}\\)/;",+ id="tool-call-inside-multiline-assigned-regex",+ ),+ pytest.param(+ "const patterns = [\n/tools.exec_command\\({}\\)/\n];",+ id="tool-call-inside-multiline-array-regex",+ ),+ pytest.param(+ "const pattern = (\n/tools.exec_command\\({}\\)/\n);",+ id="tool-call-inside-multiline-parenthesized-regex",+ ),+ pytest.param("await /tools.exec_command\\({}\\)/;", id="tool-call-inside-awaited-regex"),+ pytest.param("typeof /tools.exec_command\\({}\\)/;", id="tool-call-inside-typeof-regex"),+ pytest.param("void /tools.exec_command\\({}\\)/;", id="tool-call-inside-void-regex"),+ pytest.param("delete /tools.exec_command\\({}\\)/;", id="tool-call-inside-delete-regex"),+ pytest.param(+ "const pattern = 1 + /tools.exec_command\\({}\\)/;",+ id="tool-call-inside-binary-expression-regex",+ ),+ pytest.param(+ "if (enabled) {} /tools.exec_command\\({}\\)/.test(input);",+ id="tool-call-inside-post-block-regex",+ ),+ pytest.param(+ "if (enabled) {} else /tools.exec_command\\({}\\)/.test(input);",+ id="tool-call-inside-else-regex",+ ),+ pytest.param(+ "do /tools.exec_command\\({}\\)/.test(input); while (false);",+ id="tool-call-inside-do-regex",+ ),+ pytest.param(+ '"x" in /tools.exec_command\\({}\\)/;',+ id="tool-call-inside-in-expression-regex",+ ),+ pytest.param(+ "for (const key in /tools.exec_command\\({}\\)/) {}",+ id="tool-call-inside-for-in-regex",+ ),+ pytest.param(+ 'const config = {tools: ["hammer"]};',+ id="tools-object-key",+ ),+ pytest.param(+ "if test -e 'tools[name]'; then echo ok; fi",+ id="shell-if-command",+ ),+ pytest.param(+ "true; printf '%s' 'tools[name]'",+ id="shell-true-command",+ ),+ pytest.param(+ "// explain tools[name] without invoking it",+ id="commented-prose",+ ),+ ),+)[email protected]("extractor", _EXTRACTORS)+def test_generic_atif_exec_call_with_input_is_unchanged(source, extractor):+ assert extractor(_trajectory(source)) == [+ {+ "action": "exec",+ "action_input": {"input": source},+ "observation": "outer observation",+ }+ ]+++def test_template_unsupported_codex_wrapper_is_not_a_clean_security_result():+ trajectory = _trajectory('if (false) tools.exec_command({"cmd":"rm -rf /workspace/project"});')+ calls = _TEMPLATE_MODULE.extract_tool_calls_as_dicts(trajectory)++ result = _TEMPLATE_MODULE.check_security(trajectory, calls)++ assert result["passed"] is False+ assert result["score"] == 0.5+ assert [finding["type"] for finding in result["findings"]] == ["unsupported_tool_wrapper"]++[email protected]("template", [False, True], ids=["shared", "standalone-template"])+ "tool_reference",+ [+ pytest.param("tools[name]", id="computed-member"),+ pytest.param("tools?.[name]", id="optional-computed-member"),+ pytest.param("tools/*comment*/[name]", id="comment-separated-computed-member"),+ pytest.param("(tools)[name]", id="parenthesized-computed-member"),+ pytest.param("globalThis.tools?.[name]", id="global-optional-computed-member"),+ pytest.param("tools/*comment*/.exec_command", id="comment-before-direct-member"),+ pytest.param("tools./*comment*/exec_command", id="comment-after-direct-member"),+ pytest.param("tools.exec_command/*comment*/", id="comment-before-direct-call"),+ pytest.param("tools?.exec_command", id="optional-direct-member"),+ pytest.param("tools.exec_command?.", id="optional-direct-call"),+ pytest.param("(tools).exec_command", id="parenthesized-direct-member"),+ pytest.param("(tools)?.exec_command", id="parenthesized-optional-direct-member"),+ pytest.param("(tools.exec_command)", id="parenthesized-direct-call"),+ pytest.param('globalThis["tools"][name]', id="global-computed-tools-object"),+ pytest.param(r"t\u006fols.exec_command", id="escaped-tools-identifier"),+ pytest.param(r"tools.\u0065xec_command", id="escaped-method-identifier"),+ pytest.param("tools//comment\r[name]", id="cr-line-comment-separated-member"),+ pytest.param('(globalThis)["tools"][name]', id="parenthesized-global-computed-tools"),+ pytest.param("(globalThis).tools[name]", id="parenthesized-global-direct-tools"),+ pytest.param('((globalThis))["tools"].exec_command', id="multiply-parenthesized-global-tools"),+ ],+)+def test_dynamic_computed_codex_tool_member_fails_closed(template, tool_reference):+ source = (+ 'const name = "exec_command"; '+ f'const result = await {tool_reference}({{"cmd":"rm -rf /workspace/project"}}); '+ "text(result.output);"+ )+ trajectory = _trajectory(source)+ if template:+ calls = _TEMPLATE_MODULE.extract_tool_calls_as_dicts(trajectory)+ result = _TEMPLATE_MODULE.check_security(trajectory, calls)+ else:+ calls = extract_tool_calls_as_dicts(trajectory)+ result = check_security(calls)++ assert calls[0]["normalization_status"] == "unsupported_native_codex_exec_wrapper"+ assert result["passed"] is False+ assert result["score"] == 0.5+ assert [finding["type"] for finding in result["findings"]] == ["unsupported_tool_wrapper"]++[email protected]("template", [False, True], ids=["shared", "standalone-template"])+ "source",+ [+ pytest.param(+ 'const fn = tools.exec_command; const result = await fn({"cmd":"rm -rf /workspace/project"});',+ id="aliased-method",+ ),+ pytest.param(+ 'const {exec_command} = tools; '+ 'const result = await exec_command({"cmd":"rm -rf /workspace/project"});',+ id="destructured-method",+ ),+ pytest.param(+ 'const name = "exec_command"; const api = tools; '+ 'const result = await api[name]({"cmd":"rm -rf /workspace/project"});',+ id="aliased-tools-object",+ ),+ pytest.param(+ 'const url = "http://example.invalid"; const name = "exec_command"; '+ 'const result = await tools/*comment*/[name]({"cmd":"rm -rf /workspace/project"});',+ id="comment-separated-member-after-url-string",+ ),+ pytest.param(+ 'const url = `http://example.invalid`; const name = "exec_command"; '+ 'const result = await tools/*comment*/[name]({"cmd":"rm -rf /workspace/project"});',+ id="comment-separated-member-after-template-url",+ ),+ pytest.param(+ "const pattern = /'/; const name = \"exec_command\"; "+ 'const result = await tools/*comment*/[name]({"cmd":"rm -rf /workspace/project"});',+ id="comment-separated-member-after-regex-quote",+ ),+ pytest.param(+ 'let api; api = tools; const name = "exec_command"; '+ 'const result = await api[name]({"cmd":"rm -rf /workspace/project"});',+ id="assigned-tools-object",+ ),+ pytest.param(+ 'let exec_command; ({exec_command} = tools); '+ 'const result = await exec_command({"cmd":"rm -rf /workspace/project"});',+ id="destructuring-assignment",+ ),+ pytest.param(+ 'const api = enabled ? tools : {}; '+ 'api.exec_command({"cmd":"rm -rf /workspace/project"});',+ id="ternary-tools-alias",+ ),+ ],+)+def test_indirect_codex_tool_reference_fails_closed(template, source):+ trajectory = _trajectory(source)+ if template:+ calls = _TEMPLATE_MODULE.extract_tool_calls_as_dicts(trajectory)+ result = _TEMPLATE_MODULE.check_security(trajectory, calls)+ else:+ calls = extract_tool_calls_as_dicts(trajectory)+ result = check_security(calls)++ assert calls[0]["normalization_status"] == "unsupported_native_codex_exec_wrapper"+ assert result["passed"] is False+ assert result["score"] == 0.5+ assert [finding["type"] for finding in result["findings"]] == ["unsupported_tool_wrapper"]++[email protected]("template", [False, True], ids=["shared", "standalone-template"])+def test_unsupported_codex_wrapper_still_scans_its_outer_observation_for_secrets(template):+ trajectory = _trajectory(+ 'if (false) tools.exec_command({"cmd":"pwd"});',+ observation="Authorization: Bearer sk-abcdefgh12345678",+ )+ if template:+ calls = _TEMPLATE_MODULE.extract_tool_calls_as_dicts(trajectory)+ result = _TEMPLATE_MODULE.check_security(trajectory, calls)+ else:+ calls = extract_tool_calls_as_dicts(trajectory)+ result = check_security(calls)++ assert result["passed"] is False+ assert result["score"] == 0.0+ assert [finding["type"] for finding in result["findings"]] == [+ "unsupported_tool_wrapper",+ "secret_exposure",+ ]++[email protected]("template", [False, True], ids=["shared", "standalone-template"])+ ("rendered", "expected_status"),+ [+ ("text(one.output); text(two.output);", "ambiguous_outer_exec_result"),+ ("", "unobserved_inner_call"),+ ],+ ids=["ambiguous", "unobserved"],+)+def test_unattributed_native_wrapper_observation_is_scanned_once(template, rendered, expected_status):+ source = (+ 'const one = await tools.exec_command({"cmd":"pwd"}); '+ 'const two = await tools.exec_command({"cmd":"ls"}); '+ f"{rendered}"+ )+ trajectory = _trajectory(source, observation="Authorization: Bearer sk-abcdefgh12345678")+ if template:+ calls = _TEMPLATE_MODULE.extract_tool_calls_as_dicts(trajectory)+ result = _TEMPLATE_MODULE.check_security(trajectory, calls)+ else:+ calls = extract_tool_calls_as_dicts(trajectory)+ result = check_security(calls)++ assert [call["observation"] for call in calls] == ["", ""]+ assert [call["observation_status"] for call in calls] == [expected_status, expected_status]+ assert [call.get("wrapper_observation", "") for call in calls] == [+ "Authorization: Bearer sk-abcdefgh12345678",+ "",+ ]+ assert result["passed"] is False+ assert result["score"] == 0.0+ assert [finding["type"] for finding in result["findings"]] == ["secret_exposure"]+ assert "tool" not in result["findings"][0]+ assert "target_skill_used_before" not in result["findings"][0]++[email protected]("template", [False, True], ids=["shared", "standalone-template"])+def test_untrusted_normalizer_metadata_cannot_suppress_outer_secret_scanning(template):+ trajectory = _trajectory(+ 'if (false) tools.exec_command({"cmd":"pwd"});',+ observation="Authorization: Bearer sk-abcdefgh12345678",+ )+ trajectory["steps"][0]["tool_calls"][0].update(+ {+ "_atif_normalization_status": "caller-owned",+ "_atif_observation_status": "unobserved_inner_call",+ "_atif_inner_tool_index": 99,+ "_atif_raw_tool_index": 99,+ }+ )+ if template:+ calls = _TEMPLATE_MODULE.extract_tool_calls_as_dicts(trajectory)+ result = _TEMPLATE_MODULE.check_security(trajectory, calls)+ else:+ calls = extract_tool_calls_as_dicts(trajectory)+ result = check_security(calls)++ assert result["score"] == 0.0+ assert [finding["type"] for finding in result["findings"]] == [+ "unsupported_tool_wrapper",+ "secret_exposure",+ ]++[email protected]("template", [False, True], ids=["shared", "standalone-template"])+def test_untrusted_normalizer_metadata_cannot_bypass_command_security_checks(template):+ trajectory = _trajectory("unused")+ trajectory["steps"][0]["tool_calls"][0].update(+ {+ "function_name": "exec_command",+ "arguments": {"cmd": "rm -rf /workspace/project"},+ "_atif_normalization_status": "unsupported_native_codex_exec_wrapper",+ "_atif_observation_status": "unobserved_inner_call",+ "_atif_inner_tool_index": 99,+ "_atif_raw_tool_index": 99,+ }+ )+ if template:+ calls = _TEMPLATE_MODULE.extract_tool_calls_as_dicts(trajectory)+ result = _TEMPLATE_MODULE.check_security(trajectory, calls)+ else:+ calls = extract_tool_calls_as_dicts(trajectory)+ result = check_security(calls)++ assert "destructive_command" in {finding["type"] for finding in result["findings"]}+++def test_normalizer_reserves_private_metadata_namespace():+ tool_call = {+ "function_name": "read",+ "arguments": {"path": "SKILL.md"},+ "_atif_normalization_status": "caller-owned",+ "_atif_observation_status": "caller-owned",+ "_atif_inner_tool_index": 99,+ "_atif_raw_tool_index": 99,+ }++ assert codex_tool_call_normalizer.normalize_tool_call(tool_call) == [+ {"function_name": "read", "arguments": {"path": "SKILL.md"}}+ ]+++def test_oversized_exec_source_skips_signature_scans(monkeypatch):+ class UnexpectedScan:+ def match(self, _source):+ pytest.fail("oversized source reached pragma detection")++ def search(self, _source):+ pytest.fail("oversized source reached tool-reference detection")++ monkeypatch.setattr(codex_tool_call_normalizer, "_CODEX_PRAGMA_RE", UnexpectedScan())+ monkeypatch.setattr(codex_tool_call_normalizer, "_CODEX_TOOL_REF_RE", UnexpectedScan())+ source = " " * 65537 + 'tools.exec_command({"cmd":"pwd"});'++ assert codex_tool_call_normalizer.normalize_tool_call(+ {"function_name": "exec", "arguments": {"input": source}}+ ) == [+ {+ "function_name": "exec",+ "arguments": {"input": source},+ "_atif_normalization_status": "unsupported_native_codex_exec_wrapper",+ }+ ]+++def test_unterminated_regex_candidate_is_scanned_once(monkeypatch):+ original = codex_tool_call_normalizer._skip_js_regex+ calls = 0++ def counted_skip(source, start):+ nonlocal calls+ calls += 1+ return original(source, start)++ monkeypatch.setattr(codex_tool_call_normalizer, "_skip_js_regex", counted_skip)+ source = "/[" * 30_000 + "tools"++ assert codex_tool_call_normalizer.normalize_tool_call(+ {"function_name": "exec", "arguments": {"input": source}}+ ) == [{"function_name": "exec", "arguments": {"input": source}}]+ assert calls == 1+++ "source",+ [+ pytest.param(+ "const r = await tools.exec_command(" + '{"nested":' * 65 + "0" + "}" * 65 + ");",+ id="excessive-nesting",+ ),+ pytest.param(+ 'const r = await tools.exec_command({"value":' + "1" * 5000 + "});",+ id="oversized-integer",+ ),+ pytest.param(+ "".join(f'const r{i} = await tools.exec_command({{"cmd":"pwd {i}"}});' for i in range(129)),+ id="excessive-calls",+ ),+ pytest.param(+ 'const r = await tools.exec_command({"cmd":"pwd"});' + "text(r.output);" * 256,+ id="excessive-statements",+ ),+ pytest.param(+ 'const r = await tools.exec_command({"cmd":"pwd"});' + " " * 65536,+ id="oversized-source",+ ),+ ],+)[email protected]("extractor", _EXTRACTORS)+def test_native_codex_exec_parser_limits_fail_closed(source, extractor):+ calls = extractor(_trajectory(source))++ assert calls == [+ {+ "action": "exec",+ "action_input": {"input": source},+ "observation": "outer observation",+ "normalization_status": "unsupported_native_codex_exec_wrapper",+ }+ ]++[email protected]("extractor", _EXTRACTORS)+def test_native_codex_exec_maps_a_rendered_retry_observation_only_to_its_owner(extractor):+ source = """+const failed = await tools.exec_command({"cmd":"false"});+const retry = await tools.exec_command({"cmd":"true"});+text(retry.output);+"""++ calls = extractor(_trajectory(source))++ assert [call["observation"] for call in calls] == ["", "outer observation"]+ assert [call["observation_status"] for call in calls] == [+ "unobserved_inner_call",+ "mapped_outer_exec_result",+ ]++[email protected]("template", [False, True], ids=["shared", "standalone-template"])+def test_mapped_outer_observation_is_not_duplicated_as_wrapper_evidence(template):+ source = (+ 'const failed = await tools.exec_command({"cmd":"false"}); '+ 'const retry = await tools.exec_command({"cmd":"true"}); '+ "text(retry.output);"+ )+ trajectory = _trajectory(source, observation="Authorization: Bearer sk-abcdefgh12345678")+ if template:+ calls = _TEMPLATE_MODULE.extract_tool_calls_as_dicts(trajectory)+ result = _TEMPLATE_MODULE.check_security(trajectory, calls)+ else:+ calls = extract_tool_calls_as_dicts(trajectory)+ result = check_security(calls)++ assert [call["observation"] for call in calls] == [+ "",+ "Authorization: Bearer sk-abcdefgh12345678",+ ]+ assert [call.get("wrapper_observation", "") for call in calls] == ["", ""]+ assert [finding["type"] for finding in result["findings"]] == ["secret_exposure"]++[email protected]("template", [False, True], ids=["shared", "standalone-template"])+ ("source_call_id", "wrapper_owner"),+ [(None, 0), ("outer-two", 2)],+ ids=["unscoped", "scoped-to-second-wrapper"],+)+def test_multi_wrapper_observation_is_scanned_once_at_the_narrowest_known_scope(+ template, source_call_id, wrapper_owner+):+ source = (+ 'const one = await tools.exec_command({"cmd":"pwd"}); '+ 'const two = await tools.exec_command({"cmd":"ls"});'+ )+ result_entry = {"content": "Authorization: Bearer sk-abcdefgh12345678"}+ if source_call_id is not None:+ result_entry["source_call_id"] = source_call_id+ trajectory = {+ "steps": [+ {+ "source": "agent",+ "tool_calls": [+ {"tool_call_id": "outer-one", "function_name": "exec", "arguments": {"input": source}},+ {"tool_call_id": "outer-two", "function_name": "exec", "arguments": {"input": source}},+ ],+ "observation": {"results": [result_entry]},+ }+ ]+ }+ if template:+ calls = _TEMPLATE_MODULE.extract_tool_calls_as_dicts(trajectory)+ security = _TEMPLATE_MODULE.check_security(trajectory, calls)+ else:+ calls = extract_tool_calls_as_dicts(trajectory)+ security = check_security(calls)++ expected_wrapper_observations = ["", "", "", ""]+ expected_wrapper_observations[wrapper_owner] = "Authorization: Bearer sk-abcdefgh12345678"+ assert [call.get("wrapper_observation", "") for call in calls] == expected_wrapper_observations+ assert [finding["type"] for finding in security["findings"]] == ["secret_exposure"]+ assert "tool" not in security["findings"][0]++[email protected]("template", [False, True], ids=["shared", "standalone-template"])+ "call_ids",+ [("outer-one", "outer-two"), (None, None), ("", "")],+ ids=["with-call-ids", "without-call-ids", "with-empty-call-ids"],+)+ "source",+ [+ pytest.param(+ 'const result = await tools.exec_command({"cmd":"pwd"}); text(result.output);',+ id="mapped",+ ),+ pytest.param(+ 'const name = "exec_command"; const result = await tools[name]({"cmd":"pwd"});',+ id="unsupported",+ ),+ ],+)+def test_unscoped_multi_wrapper_observation_is_not_duplicated_for_atomic_calls(+ template, call_ids, source+):+ secret = "Authorization: Bearer sk-abcdefgh12345678"+ trajectory = {+ "steps": [+ {+ "source": "agent",+ "tool_calls": [+ {+ **({"tool_call_id": call_ids[0]} if call_ids[0] is not None else {}),+ "function_name": "exec",+ "arguments": {"input": source},+ },+ {+ **({"tool_call_id": call_ids[1]} if call_ids[1] is not None else {}),+ "function_name": "exec",+ "arguments": {"input": source},+ },+ ],+ "observation": {"results": [{"content": secret}]},+ }+ ]+ }+ if template:+ calls = _TEMPLATE_MODULE.extract_tool_calls_as_dicts(trajectory)+ security = _TEMPLATE_MODULE.check_security(trajectory, calls)+ else:+ calls = extract_tool_calls_as_dicts(trajectory)+ security = check_security(calls)++ assert [call["observation"] for call in calls] == [secret, ""]+ assert [finding["type"] for finding in security["findings"]].count("secret_exposure") == 1+++ ("extractor", "checker"),+ [+ (extract_tool_calls_as_dicts, check_error_recovery),+ (_TEMPLATE_MODULE.extract_tool_calls_as_dicts, _TEMPLATE_MODULE.check_error_recovery),+ ],+)+ ("source", "unsupported_evidence"),+ [+ (+ 'const failed = await tools.exec_command({"cmd":"false"});',+ ["unobserved_inner_call"],+ ),+ (+ 'const failed = await tools.exec_command({"cmd":"false"}); '+ 'const retry = await tools.exec_command({"cmd":"true"}); text(retry.output);',+ ["unobserved_inner_call"],+ ),+ (+ 'const args = {"cmd":"false"}; const result = await tools.exec_command(args); text(result.output);',+ ["unsupported_native_codex_exec_wrapper"],+ ),+ ],+)+def test_error_recovery_is_not_clean_when_codex_observation_evidence_is_untrusted(+ extractor, checker, source, unsupported_evidence+):+ result = checker(extractor(_trajectory(source)))++ assert result["passed"] is None+ assert result["score"] == 0.5+ assert result["supported"] is False+ assert result["first_attempt_clean"] is False+ assert result["unsupported_evidence"] == unsupported_evidence+++ ("extractor", "checker"),+ [+ (extract_tool_calls_as_dicts, check_error_recovery),+ (_TEMPLATE_MODULE.extract_tool_calls_as_dicts, _TEMPLATE_MODULE.check_error_recovery),+ ],+)+def test_error_recovery_ignores_unobserved_non_execution_calls(extractor, checker):+ source = (+ 'const plan = await tools.update_plan({"plan":[]}); '+ 'const run = await tools.exec_command({"cmd":"true"}); text(run.output);'+ )++ result = checker(extractor(_trajectory(source)))++ assert result["passed"] is True+ assert result["score"] == 1.0+ assert result["first_attempt_clean"] is True+++ ("extractor", "checkers"),+ [+ (+ extract_tool_calls_as_dicts,+ (+ check_activation,+ check_script_execution,+ check_workflow_order,+ check_negative_case,+ check_routing,+ check_tool_efficiency,+ check_error_recovery,+ ),+ ),+ (+ _TEMPLATE_MODULE.extract_tool_calls_as_dicts,+ (+ _TEMPLATE_MODULE.check_activation,+ _TEMPLATE_MODULE.check_script_execution,+ _TEMPLATE_MODULE.check_workflow_order,+ _TEMPLATE_MODULE.check_negative_case,+ _TEMPLATE_MODULE.check_routing,+ _TEMPLATE_MODULE.check_tool_efficiency,+ _TEMPLATE_MODULE.check_error_recovery,+ ),+ ),+ ],+)+def test_unsupported_codex_wrapper_never_produces_a_clean_deterministic_result(extractor, checkers):+ source = (+ 'const args = {"cmd":"python /skills/example/scripts/run.py"}; '+ "const result = await tools.exec_command(args); text(result.output);"+ )+ calls = extractor(_trajectory(source, observation="run.py completed"))+ activation, script, workflow, negative, routing, efficiency, recovery = checkers++ results = [+ activation(calls, "example"),+ script(calls, "run.py"),+ workflow(calls, expected_skill="example"),+ negative(calls, "example"),+ routing(calls, "example"),+ efficiency(calls, expected_skill="example", expected_script="run.py"),+ recovery(calls),+ ]++ assert {result["passed"] for result in results} == {None}+ assert {result["score"] for result in results} == {0.5}+ assert {result["supported"] for result in results} == {False}+++ ("behavior_builder", "refs_builder"),+ [+ (build_behavior_evidence, build_metric_evidence_refs),+ (_TEMPLATE_MODULE.build_behavior_evidence, _TEMPLATE_MODULE.build_metric_evidence_refs),+ ],+)+def test_ambiguous_write_observation_remains_wrapper_level_evidence(behavior_builder, refs_builder):+ source = (+ 'const first = await tools.exec_command({"cmd":"touch /workspace/a"}); '+ 'const second = await tools.exec_command({"cmd":"touch /workspace/b"});'+ )+ trajectory = _trajectory(source)++ behavior = behavior_builder(trajectory, "Create both files")+ observation_refs = [+ ref+ for ref in refs_builder(trajectory, "Create both files")["goal_accuracy"]+ if ref["kind"] == "tool_observation"+ ]++ assert behavior.count("Agent called: exec_command") == 2+ assert behavior.count("Tool returned: outer observation") == 1+ assert len(observation_refs) == 1+ assert observation_refs[0]["excerpt"] == "outer observation"+++def test_native_codex_exec_evidence_refs_resolve_to_the_outer_call():+ trajectory = _trajectory(+ 'const one = await tools.exec_command({"cmd":"pwd"}); const two = await tools.exec_command({"cmd":"ls"});'+ )++ refs = build_metric_evidence_refs(trajectory, "q")["goal_accuracy"]+ tool_refs = [ref for ref in refs if ref["kind"] == "tool_call"]++ assert len(tool_refs) == 2+ assert {ref["json_pointer"] for ref in tool_refs} == {"/steps/0/tool_calls/0"}+ assert [ref["evidence_id"] for ref in tool_refs] == [+ "/steps/0/tool_calls/0/normalized/0",+ "/steps/0/tool_calls/0/normalized/1",+ ]+ assert [ref["excerpt"] for ref in tool_refs] == ["pwd", "ls"]+++def test_copied_verifier_imports_its_sibling_codex_normalizer(tmp_path):+ adapter._copy_verifier(tmp_path)++ tests_dir = tmp_path / "tests"+ normalizer = tests_dir / "codex_tool_call_normalizer.py"+ evidence = tests_dir / "evidence.py"+ assert normalizer.is_file()+ assert evidence.is_file()++ sys.modules.pop("codex_tool_call_normalizer", None)+ sys.modules.pop("evidence", None)+ spec = importlib.util.spec_from_file_location("copied_harbor_eval_codex_tools", tests_dir / "eval.py")+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)++ assert Path(sys.modules["codex_tool_call_normalizer"].__file__).resolve() == normalizer.resolve()+ assert Path(sys.modules["evidence"].__file__).resolve() == evidence.resolve()+ assert (+ module.extract_tool_calls_as_dicts(_trajectory('const r = await tools.exec_command({"cmd":"pwd"});'))[0][+ "action"+ ]+ == "exec_command"+ )diff --git a/tests/tier3/test_report_renders_refs.py b/tests/tier3/test_report_renders_refs.pyindex 25dce8e4..3c77eaf7 100644--- a/tests/tier3/test_report_renders_refs.py+++ b/tests/tier3/test_report_renders_refs.py@@ -85,6 +85,57 @@ def test_cli_findings_body_renders_evidence_pointer():assert "/steps/14" in text and "evidence:" in text+def test_cli_findings_body_renders_normalized_tool_compact_identity():+ text = report._render_findings_body(+ [+ {+ "metric": "goal_accuracy",+ "label": "GOAL ACCURACY",+ "severity": "warning",+ "score": 0.5,+ "reasons": ["second command failed"],+ "evidence_refs": [+ {+ "source": "trajectory.json",+ "json_pointer": "/steps/0/tool_calls/0",+ "evidence_id": "/steps/0/tool_calls/0/normalized/1",+ "kind": "tool_call",+ }+ ],+ }+ ]+ ).plain++ assert "trajectory.json#/steps/0/tool_calls/0/normalized/1" in text+++def test_cli_findings_body_keeps_same_source_path_only_refs_distinct():+ refs = [+ {+ "source": "artifact.txt",+ "path": f"results/{name}.json",+ "kind": "artifact",+ }+ for name in ("first", "second")+ ]++ text = report._render_findings_body(+ [+ {+ "metric": "goal_accuracy",+ "label": "GOAL ACCURACY",+ "severity": "warning",+ "score": 0.5,+ "reasons": ["artifacts differ"],+ "evidence_refs": refs,+ }+ ]+ ).plain++ assert "artifact.txt#results/first.json" in text+ assert "artifact.txt#results/second.json" in text++def test_cli_findings_include_custom_metric_details():findings = report._extract_findings([_reward_with_custom_metric()])custom = next(f for f in findings if f["metric"] == "domain_quality")diff --git a/tests/tier3/test_suggestion_grounding.py b/tests/tier3/test_suggestion_grounding.pyindex 392158a7..0880e6a7 100644--- a/tests/tier3/test_suggestion_grounding.py+++ b/tests/tier3/test_suggestion_grounding.py@@ -70,6 +70,65 @@ def _reward_multi_metric(metric_score=0.1):}+def _reward_with_normalized_tool_refs():+ return {+ "entry_id": "evaluator-plugin-004",+ "goal_accuracy": 0.1,+ "security": 1.0,+ "skill_execution": 1.0,+ "skill_efficiency": 1.0,+ "accuracy": 1.0,+ "behavior_check": 1.0,+ "details": {+ "goal_accuracy": {+ "reason": "two commands need distinct remediation",+ "evidence_refs": [+ {+ "source": "trajectory.json",+ "json_pointer": "/steps/0/tool_calls/0",+ "evidence_id": "/steps/0/tool_calls/0/normalized/0",+ "kind": "tool_call",+ "excerpt": "first command",+ },+ {+ "source": "trajectory.json",+ "json_pointer": "/steps/0/tool_calls/0",+ "evidence_id": "/steps/0/tool_calls/0/normalized/1",+ "kind": "tool_call",+ "excerpt": "second command",+ },+ ],+ },+ },+ }+++def _reward_with_path_only_refs():+ return {+ "entry_id": "evaluator-plugin-005",+ "goal_accuracy": 0.1,+ "security": 1.0,+ "skill_execution": 1.0,+ "skill_efficiency": 1.0,+ "accuracy": 1.0,+ "behavior_check": 1.0,+ "details": {+ "goal_accuracy": {+ "reason": "two artifacts need distinct remediation",+ "evidence_refs": [+ {+ "source": "artifact.txt",+ "path": f"results/{name}.json",+ "kind": "artifact",+ "excerpt": f"{name} artifact",+ }+ for name in ("first", "second")+ ],+ },+ },+ }++def test_findings_carry_evidence_refs():findings = report._extract_findings([_reward(0.1)])goal = next(f for f in findings if f["metric"] == "goal_accuracy")@@ -204,6 +263,84 @@ def test_suggestions_evidence_refs_lookup_uses_all_metrics(monkeypatch):assert refs[0]["kind"] == "tool_call"+def test_normalized_tool_evidence_refs_remain_distinct_and_resolve_by_compact_identity(monkeypatch):+ compact_ref = "trajectory.json#/steps/0/tool_calls/0/normalized/1"+ captured = {}++ def fake_hub(prompt, **_kw):+ captured["prompt"] = prompt+ return (+ f'[{{"suggestion": "Fix the second command", "dimension": "goal_accuracy", "evidence_refs": ["{compact_ref}"]}}]',+ None,+ )++ monkeypatch.setattr("skillevaluator.tier3.eval_core.llm_judge.call_public_llm", fake_hub)+ reward = _reward_with_normalized_tool_refs()+ findings = report._extract_findings([reward])++ assert len(findings[0]["evidence_refs"]) == 2+ result = report._generate_suggestions_structured("demo", findings, [reward])++ assert compact_ref in captured["prompt"]+ assert result[0]["evidence_refs"] == [+ {+ "source": "trajectory.json",+ "json_pointer": "/steps/0/tool_calls/0",+ "evidence_id": "/steps/0/tool_calls/0/normalized/1",+ "kind": "tool_call",+ "excerpt": "second command",+ }+ ]+++def test_whitespace_evidence_ids_fall_back_before_deduplication():+ reward = _reward(0.1)+ reward["details"]["goal_accuracy"]["evidence_refs"] = [+ {+ "source": "trajectory.json",+ "evidence_id": whitespace,+ "json_pointer": pointer,+ "kind": "tool_call",+ }+ for whitespace, pointer in ((" ", "/steps/1"), ("\t", "/steps/2"))+ ]++ findings = report._extract_findings([reward])++ assert [ref["json_pointer"] for ref in findings[0]["evidence_refs"]] == ["/steps/1", "/steps/2"]+++def test_path_only_evidence_refs_remain_distinct_in_prompt_and_lookup(monkeypatch):+ compact_ref = "artifact.txt#results/second.json"+ captured = {}++ def fake_hub(prompt, **_kw):+ captured["prompt"] = prompt+ return (+ f'[{{"suggestion": "Fix the second artifact", "dimension": "goal_accuracy", '+ f'"evidence_refs": ["{compact_ref}"]}}]',+ None,+ )++ monkeypatch.setattr("skillevaluator.tier3.eval_core.llm_judge.call_public_llm", fake_hub)+ reward = _reward_with_path_only_refs()+ findings = report._extract_findings([reward])++ assert len(findings[0]["evidence_refs"]) == 2+ result = report._generate_suggestions_structured("demo", findings, [reward])++ assert "artifact.txt#results/first.json" in captured["prompt"]+ assert compact_ref in captured["prompt"]+ assert result[0]["evidence_refs"] == [+ {+ "source": "artifact.txt",+ "path": "results/second.json",+ "kind": "artifact",+ "excerpt": "second artifact",+ }+ ]++def test_suggestions_structured_evidence_refs_are_dicts_not_strings(monkeypatch):"""End-to-end: suggestions_v2 artifacts must have dict refs, never plain strings."""monkeypatch.setattr(