Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(tier3): decode freeform Codex apply_patch literals and trust an explicit exit 0 (#530)
Narendran Raghavan73db5cd95121
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex cef3ee5d..d0bb49c6 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -462,6 +462,28 @@ All notable changes to SkillEvaluator are documented in this file.refused as `no_progress_evidence`, where result collection used to stopwith `browser-unsafe JSON integer` and write no `result.json` or report.+- A native Codex `exec` wrapper that calls the freeform `apply_patch` tool with+ the patch as a bare string is decoded instead of being discarded. The bounded+ statement grammar read every argument as an object literal, so+ `const r1 = await tools.apply_patch("*** Begin Patch\n...");` followed by+ another call made the whole wrapper+ `unsupported_native_codex_exec_wrapper`: the patch and its sibling calls went+ ungraded, although the same wrapper written as+ `tools.apply_patch({input: "..."})` decoded. A single- or double-quoted+ string is now read as that tool's `input`. Nothing is interpreted: a template+ literal, a concatenation, an extra argument, an unterminated string and a+ bare string passed to any other tool are still rejected, and the object path+ keeps its character gates unchanged.+- A tool call the harness reports as exiting 0 is no longer recorded as failed+ because its output holds error words. An agent that read a document *about*+ errors, such as a `SKILL.md` headed `# Troubleshooting: command not found`,+ lost that call's component activation in the plugin component signals. An+ explicit zero exit status in every correlated result's header now passes the+ call, as a harness success flag already did. Only the harness's own header+ lines are read, so a body that spells `exit_code=0` still reports nothing,+ and a result with no exit status, or a non-zero one, is judged by its text+ exactly as before.+## 0.9.0 - 2026-11-13### Fixeddiff --git a/src/skillevaluator/tier3/eval_core/codex_tool_call_normalizer.py b/src/skillevaluator/tier3/eval_core/codex_tool_call_normalizer.pyindex ec2af945..d92da78d 100644--- a/src/skillevaluator/tier3/eval_core/codex_tool_call_normalizer.py+++ b/src/skillevaluator/tier3/eval_core/codex_tool_call_normalizer.py@@ -174,6 +174,31 @@ def _reject_constant(name: str) -> float:raise ValueError(name)+def _decode_js_string_literal(source: str, start: int) -> tuple[str, int] | None:+ """Decode one plain single- or double-quoted string that uses JSON escapes only.++ Returns ``(value, end)``, or None when ``start`` does not open such a+ string: no template literal, no concatenation and no escape JSON rejects,+ so the decoded text is the one the wrapper passed and nothing is evaluated.+ """+ quote = source[start : start + 1]+ if quote not in ("'", '"'):+ return None+ end = start + 1+ while end < len(source) and source[end] != quote:+ end += 2 if source[end] == "\\" else 1+ if end >= len(source):+ return None+ body = source[start + 1 : end]+ if quote == "'":+ body = re.sub(r"\\'|\\.|\"", lambda m: {"\\'": "'", '"': '\\"'}.get(m.group(0), m.group(0)), body)+ try:+ value = json.loads(f'"{body}"')+ except (json.JSONDecodeError, ValueError):+ return None+ return value, end + 1++def _decode_static_js_object(source: str, start: int) -> tuple[dict[str, Any], int] | None:"""Decode a JSON-compatible object literal, including unquoted property names."""rendered: list[str] = []@@ -739,10 +764,18 @@ def _static_codex_tool_calls(source: str) -> tuple[list[tuple[str, dict[str, Anyvariable, function_name = call.groups()if variable in variables:return None- decoded = _decode_static_js_object(source, call.end())- if decoded is None:- return None- arguments, end = decoded+ if function_name == "apply_patch" and source[call.end() : call.end() + 1] in ("'", '"'):+ # Codex records its freeform apply_patch tool with the patch as a bare string+ # literal rather than an object; its text is patch data, never code to run.+ literal = _decode_js_string_literal(source, call.end())+ if literal is None:+ return None+ arguments, end = {"input": literal[0]}, literal[1]+ else:+ decoded = _decode_static_js_object(source, call.end())+ if decoded is None:+ return None+ arguments, end = decodedclose = re.match(r"\s*\)\s*;", source[end:])if close is None:return None@@ -909,20 +942,10 @@ class _StaticExecSource:def string(self) -> str:"""Decode one plain single- or double-quoted string that uses JSON escapes only."""- quote = self.source[self.index]- end = self.index + 1- while end < len(self.source) and self.source[end] != quote:- end += 2 if self.source[end] == "\\" else 1- if end >= len(self.source):+ decoded = _decode_js_string_literal(self.source, self.index)+ if decoded is None:raise _NotStatic("string")- body = self.source[self.index + 1 : end]- if quote == "'":- body = re.sub(r"\\'|\\.|\"", lambda m: {"\\'": "'", '"': '\\"'}.get(m.group(0), m.group(0)), body)- try:- value = json.loads(f'"{body}"')- except (json.JSONDecodeError, ValueError) as exc:- raise _NotStatic("string") from exc- self.index = end + 1+ value, self.index = decodedreturn valuedef literal(self, depth: int = 0) -> Any:diff --git a/src/skillevaluator/tier3/eval_core/plugin_signals.py b/src/skillevaluator/tier3/eval_core/plugin_signals.pyindex 646dae94..0fcd7ee9 100644--- a/src/skillevaluator/tier3/eval_core/plugin_signals.py+++ b/src/skillevaluator/tier3/eval_core/plugin_signals.py@@ -51,8 +51,11 @@ owns that observation; otherwise its outcome is unknown.``succeeded`` is tri-state. ``True``: a correlated result came back without astructured error flag or a failure/unavailable marker in the head of any of its-content blocks or in its tail. ``False``: the correlated result is flagged or-carries such a marker ("no such file" only counts for non-shell calls, whose+content blocks or in its tail, or every correlated result's own status header+states that the command exited 0, which outranks the text (a document *about*+errors is the agent's answer, not this call's diagnostic). ``False``: the+correlated result is flagged or carries such a marker without an explicit+successful status ("no such file" only counts for non-shell calls, whoseoutput is not a mix of several commands). A file read or skill load returnsfile text, so only its first line counts (past a harness's shell status lines),plus, for a shell read, any shell error line that names a ``SKILL.md``. ``None``:@@ -572,6 +575,18 @@ _SHELL_STATUS_LINE_RE = re.compile(r"|original token count:.*|total output lines:.*|output:",re.IGNORECASE,)+# The exit status inside such a header line, in the spellings the converters emit+# (``exit_code=0``, ``Process exited with code 0``).+_EXIT_STATUS_LINE_RE = re.compile(+ r"\[?(?:exit[ _]code|return[ _]code|process[ \t]+exited[ \t]+with[ \t]+code)\]?[ \t]*[=:]?[ \t]*(-?\d+)\]?",+ re.IGNORECASE,+)+# Every line a harness may put in that header, including the converter's+# ``status=`` line, which ``_SHELL_STATUS_LINE_RE`` does not cover.+_STATUS_HEADER_LINE_RE = re.compile(+ rf"{_SHELL_STATUS_LINE_RE.pattern}|status[ \t]*=[ \t]*\w+|{_EXIT_STATUS_LINE_RE.pattern}",+ re.IGNORECASE,+)# A shell tool's error line about a file (``cat: <dir>/SKILL.md: Permission denied``)._SHELL_ERROR_LINE_RE = re.compile(r"[\w./+-]+:\s")# The MCP content wrapper a harness keeps around a tool's text: Codex prints@@ -2012,6 +2027,32 @@ def _content_read_scan(text: str, *, shell: bool) -> str:return "\n".join(lines)+def _header_exit_status(text: str) -> int | None:+ """The exit status a harness states in its own header above a result's body.++ Only complete status lines before the body are read, so a document whose+ own text spells ``exit_code=0`` never reports an outcome. ``None`` when the+ header states no status, or when two of its lines disagree.+ """+ status: int | None = None+ for line in text[:_FAILURE_SCAN_CHARS].splitlines():+ stripped = line.strip()+ if not stripped:+ continue+ if stripped.casefold() == "output:":+ break # the body starts on the next line+ if not _STATUS_HEADER_LINE_RE.fullmatch(stripped):+ break+ match = _EXIT_STATUS_LINE_RE.fullmatch(stripped)+ if match is None:+ continue+ code = int(match.group(1))+ if status is not None and status != code:+ return None+ status = code+ return status++def _outcome(correlated: Sequence[_Result], *, shell: bool, content_read: bool = False) -> tuple[str | None, bool | None]:@@ -2019,7 +2060,8 @@ def _outcome(A structured outcome wins: any failure flag fails the call, and theharness's own success report (Claude Code ``tool_result_is_error: false``,- a Codex ``completed`` status) passes it whatever the text says. Without+ a Codex ``completed`` status, or an explicit zero exit status in every+ correlated result's header) passes it whatever the text says. Withoutone, failure text is read on the result's lead lines only. For a``content_read`` (a file read or a skill load) the body is file or skilltext, so only the first line of a result (past a harness's shell status@@ -2035,6 +2077,12 @@ def _outcome(return text, Noneif any(flag is False for flag in flags):return text, True+ # A command the harness reports as exiting 0 succeeded, however its output reads: a+ # document *about* errors is the agent's answer, not this call's diagnostic. Only an+ # explicit success counts, so a result that states no status, or a non-zero one, is+ # still judged by its text.+ if all(_header_exit_status(result.text) == 0 for result in correlated):+ return text, Trueif not text.strip():return text, Noneif content_read:diff --git a/tests/tier3/test_codex_tool_call_normalization.py b/tests/tier3/test_codex_tool_call_normalization.pyindex e3accf6f..1bd06e6c 100644--- a/tests/tier3/test_codex_tool_call_normalization.py+++ b/tests/tier3/test_codex_tool_call_normalization.py@@ -1218,6 +1218,59 @@ def test_extended_codex_apply_patch_string_literal_normalizes(extractor, source)assert calls[0]["action_input"] == {"input": "*** Begin Patch\n*** End Patch"}+# The bounded statement grammar answers wrappers the extended one declines, such as two+# bound calls of which only one is rendered. It read every argument as an object literal,+# so a freeform apply_patch string made the whole wrapper unsupported and its calls went+# ungraded, although the same patch passed as ``{input: "..."}`` decoded (#236).+_BOUNDED_PATCH_TAIL = 'const r2=await tools.exec_command({cmd:"cat a.txt"});text(r2);'++[email protected]("extractor", _EXTRACTORS)+ "argument",+ [+ pytest.param(r"'*** Begin Patch\n*** End Patch'", id="single-quoted"),+ pytest.param(r'"*** Begin Patch\n*** End Patch"', id="double-quoted"),+ pytest.param(r'"*** Begin Patch\n+echo $HOME `date` a\\b\n*** End Patch"', id="shell-characters"),+ ],+)+def test_bounded_codex_apply_patch_string_literal_normalizes(extractor, argument):+ calls = extractor(_trajectory(f"const r1=await tools.apply_patch({argument});{_BOUNDED_PATCH_TAIL}"))++ assert [call["action"] for call in calls] == ["apply_patch", "exec_command"]+ assert calls[0]["action_input"]["input"].startswith("*** Begin Patch\n")+ assert calls[0]["action_input"]["input"].endswith("*** End Patch")+ assert "normalization_status" not in calls[0]++[email protected]("extractor", _EXTRACTORS)+ "argument",+ [+ pytest.param(r'"*** Begin Patch\n*** End Patch', id="unterminated-string"),+ pytest.param("`*** Begin Patch`", id="template-literal"),+ pytest.param("`${patch}`", id="interpolated-template"),+ pytest.param('"*** Begin "+patch', id="concatenation"),+ pytest.param('"*** Begin Patch",options', id="extra-argument"),+ ],+)+def test_bounded_codex_malformed_apply_patch_literal_stays_atomic(extractor, argument):+ calls = extractor(_trajectory(f"const r1=await tools.apply_patch({argument});{_BOUNDED_PATCH_TAIL}"))++ assert len(calls) == 1+ assert calls[0]["action"] == "exec"+ assert calls[0]["normalization_status"] == "unsupported_native_codex_exec_wrapper"++[email protected]("extractor", _EXTRACTORS)+def test_bounded_codex_string_literal_is_only_for_apply_patch(extractor):+ calls = extractor(_trajectory(f'const r1=await tools.exec_command("pwd");{_BOUNDED_PATCH_TAIL}'))++ assert len(calls) == 1+ assert calls[0]["action"] == "exec"+ assert calls[0]["normalization_status"] == "unsupported_native_codex_exec_wrapper"++@pytest.mark.parametrize("extractor", _EXTRACTORS)@pytest.mark.parametrize("source",diff --git a/tests/tier3/test_plugin_signals.py b/tests/tier3/test_plugin_signals.pyindex 030528f2..7988eec8 100644--- a/tests/tier3/test_plugin_signals.py+++ b/tests/tier3/test_plugin_signals.py@@ -64,6 +64,11 @@ def _one(fn: str, args: dict[str, Any] | None = None, content: Any = "ok", call_return _step([_tc(fn, args, call_id)], [_res(call_id, content, **extra)])+def _exec_result(exit_code: int, body: str) -> str:+ """A shell result inside the status header Harbor puts above Codex ``exec`` output."""+ return f"Wall time: 0.2 seconds\nProcess exited with code {exit_code}\nOutput:\n\n{body}"++def _signals(traj: dict[str, Any], case: dict[str, Any] | None = None, **kwargs: Any) -> dict[str, Any]:kwargs.setdefault("declared", DECLARED)signals = compute_plugin_signals(traj, case or {}, **kwargs)@@ -474,6 +479,41 @@ class TestOutcomeTriState:traj = _traj(_one("mcp__github__x", content=content))assert _activations(traj)[0]["succeeded"] is False+ def test_explicit_zero_exit_status_outranks_error_words_a_read_returned(self) -> None:+ # The agent read a manifest *about* errors; the harness says the command exited 0 (#236).+ body = "# Troubleshooting: command not found\n\nInstall the CLI when you see this.\n"+ traj = _traj(_one("exec_command", {"cmd": "cat skills/alpha/SKILL.md"}, _exec_result(0, body)))+ assert _activations(traj)[0]["succeeded"] is True++ @pytest.mark.parametrize(+ "observation",+ [+ # A non-zero status never overrides the text, so these stay judged exactly as before.+ pytest.param(_exec_result(127, "bash: cat: command not found\n"), id="exit-127"),+ pytest.param(+ _exec_result(1, "cat: skills/alpha/SKILL.md: Permission denied\n"),+ id="exit-1",+ ),+ # Only the harness's own header counts: a body spelling a status proves nothing.+ pytest.param(+ _exec_result(1, "cat: skills/alpha/SKILL.md: Permission denied\nexit_code=0\n"),+ id="zero-status-in-the-body",+ ),+ # An absent status keeps today's behaviour.+ pytest.param("cat: skills/alpha/SKILL.md: Permission denied\n", id="no-status"),+ ],+ )+ def test_a_failed_read_stays_failed_without_an_explicit_success(self, observation: str) -> None:+ traj = _traj(_one("exec_command", {"cmd": "cat skills/alpha/SKILL.md"}, observation))+ assert _activations(traj)[0]["succeeded"] is False++ def test_a_failure_flag_still_beats_a_zero_exit_status(self) -> None:+ body = "# Troubleshooting: command not found\n"+ traj = _traj(+ _one("exec_command", {"cmd": "cat skills/alpha/SKILL.md"}, _exec_result(0, body), is_error=True)+ )+ assert _activations(traj)[0]["succeeded"] is False+def test_missing_sibling_file_does_not_fail_a_shell_manifest_read(self) -> None:output = "cat: skills/alpha/REFERENCE.md: No such file or directory\n---\nname: alpha\n---\n"traj = _traj(_one("exec_command", {"cmd": "cat skills/alpha/SKILL.md skills/alpha/REFERENCE.md"}, output))