Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier3): decode inline and Promise.allSettled Codex wrappers (#235) (#325)

Amine Marzoukiaa09db231bbf
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 20385a5c..57720591 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -639,6 +639,16 @@ All notable changes to SkillEvaluator are documented in this file.
- Tier 2 includes qualifying shell comment blocks at end of file in context
deduplication. Previously those blocks disappeared unless a blank line or
code followed them, which could hide repeated guidance across shell files.
+- Tier 3 decodes the native Codex `exec` wrappers that Codex 0.161.0 emits:
+ inline `text(await tools.NAME(...))`, single-quoted string literals, and
+ literal `Promise.allSettled` batches rendered by an indexed `for`, indexed
+ `forEach`, or `forEach(text)`. Multi-call wrappers decode only when each call
+ owns its own JSON record of the outer output; dynamic arguments, shadowed
+ bindings, unsupported renderers, and missing, extra, reordered, or truncated
+ records, including duplicate JSON keys, remain unsupported. A nonzero exit
+ code or rejected promise in a record is graded as a failure. This restores
+ security and skill-execution scoring for those wrappers instead of the
+ neutral fallback.
- All-tier CI retains Tier 2's native reports as advisory while preserving
failed and incomplete evaluation outcomes. GitHub `ci status` validates the
diff --git a/src/skillevaluator/tier3/eval_core/checks.py b/src/skillevaluator/tier3/eval_core/checks.py
index 85562ecb..29d89368 100644
--- a/src/skillevaluator/tier3/eval_core/checks.py
+++ b/src/skillevaluator/tier3/eval_core/checks.py
@@ -16,6 +16,7 @@ from __future__ import annotations
import ast
import bisect
+import json
import posixpath
import re
import shlex
@@ -4942,6 +4943,21 @@ def check_error_recovery(
return program
return None
+ def _record_output(observation: str) -> str | None:
+ # A Codex `exec` record prints the command's output inside its JSON:
+ # `{"exit_code":1,"output":""}`, wrapped for settled promises in an object under a field of
+ # any name. Follow a lone object-valued field; two of them make the record ambiguous.
+ try:
+ record, end = json.JSONDecoder().raw_decode(observation)
+ except (ValueError, RecursionError):
+ return None
+ if _EXIT_STATUS_RE.sub("", observation[end:]).strip():
+ return None # text after the record may be a diagnostic
+ while isinstance(record, dict) and not isinstance(record.get("output"), str):
+ nested = [item for item in record.values() if isinstance(item, dict)]
+ record = nested[0] if len(nested) == 1 else None
+ return record["output"] if isinstance(record, dict) and isinstance(record["output"], str) else None
+
def _is_quiet_negative_probe(tc: dict[str, Any], observation: str) -> bool:
# Exit 1 is a normal negative answer for these probes. Be conservative:
# keep diagnostics and other compound shell commands as failures.
@@ -4998,6 +5014,8 @@ def check_error_recovery(
)
if unified_output is not None:
output = unified_output.group(1)
+ elif (record_output := _record_output(observation)) is not None:
+ output = record_output
else:
# Other converters emit status/exit_code lines without an envelope.
output = _EXIT_STATUS_RE.sub("", observation)
diff --git a/src/skillevaluator/tier3/eval_core/codex_tool_call_normalizer.py b/src/skillevaluator/tier3/eval_core/codex_tool_call_normalizer.py
index d406f431..634012b6 100644
--- a/src/skillevaluator/tier3/eval_core/codex_tool_call_normalizer.py
+++ b/src/skillevaluator/tier3/eval_core/codex_tool_call_normalizer.py
@@ -6,6 +6,7 @@
from __future__ import annotations
import json
+import math
import re
from typing import Any
@@ -160,6 +161,19 @@ def _skip_js_quoted(source: str, start: int, quote: str) -> int:
return -1
+def _finite_float(text: str) -> float:
+ """Parse a JSON float; JavaScript serializes an overflowing one as null, so inf is not its value."""
+ value = float(text)
+ if not math.isfinite(value):
+ raise ValueError(text)
+ return value
+
+
+def _reject_constant(name: str) -> float:
+ """Reject NaN and Infinity, which `json.loads` takes but JavaScript serializes as null."""
+ raise ValueError(name)
+
+
def _decode_static_js_object(source: str, start: int) -> tuple[dict[str, Any], int] | None:
"""Decode a JSON-compatible object literal, including unquoted property names."""
rendered: list[str] = []
@@ -203,7 +217,7 @@ def _decode_static_js_object(source: str, start: int) -> tuple[dict[str, Any], i
index += 1
if not stack:
try:
- arguments = json.loads("".join(rendered))
+ arguments = json.loads("".join(rendered), parse_float=_finite_float, parse_constant=_reject_constant)
except (json.JSONDecodeError, RecursionError, TypeError, ValueError):
return None
return (arguments, index) if isinstance(arguments, dict) else None
@@ -216,9 +230,44 @@ _CODEX_RENDER_RE = re.compile(
rf"text\s*\(\s*(?:JSON\.stringify\(\s*({_JS_IDENTIFIER})\s*\)|"
rf"({_JS_IDENTIFIER})(?:\.({_JS_IDENTIFIER}))?)\s*\)\s*;"
)
-_CODEX_PRAGMA_RE = re.compile(r"[ \t]*// @exec:[^\r\n]*\r?\n")
+# JavaScript ends a `//` comment at U+2028 and U+2029 too, so a pragma line may not contain them.
+_CODEX_PRAGMA_RE = re.compile(r"[ \t]*// @exec:[^\r\n\u2028\u2029]*\r?\n")
_CODEX_PRAGMA_PREFIX_RE = re.compile(r"[ \t]*// @exec:")
_CODEX_TOOL_REF_RE = re.compile(r"\b(?:tools|globalThis)\b|\\u(?:\{[0-9A-Fa-f]{1,6}\}|[0-9A-Fa-f]{4})")
+
+
+def _js_name(name: str) -> bool:
+ """Whether JavaScript accepts ``name``: `\\w` also takes U+00B2, which it rejects."""
+ return name.replace("$", "_").isidentifier()
+
+
+def _js_match(pattern: re.Pattern[str], source: str, index: int) -> re.Match[str] | None:
+ """Match ``pattern`` at ``index``, keeping only a match whose names JavaScript accepts."""
+ match = pattern.match(source, index)
+ return match if match and all(_js_name(name) for name in match.groups() if name) else None
+
+
+# `isspace()` and `\\s` take these, but JavaScript does not, so a script using one never runs.
+_NON_JS_SPACE = frozenset("\x1c\x1d\x1e\x1f\x85")
+
+
+def _has_non_js_space(source: str) -> bool:
+ """Whether the code, outside string literals and the pragma line, holds a `_NON_JS_SPACE` character."""
+ pragma = _CODEX_PRAGMA_RE.match(source)
+ index = pragma.end() if pragma else 0
+ while index < len(source):
+ char = source[index]
+ if char in "\"'":
+ index = _skip_js_quoted(source, index, char)
+ if index < 0:
+ return False
+ else:
+ if char in _NON_JS_SPACE:
+ return True
+ index += 1
+ return False
+
+
_JS_IDENTIFIER_ESCAPE_RE = re.compile(r"\\u(?:\{([0-9A-Fa-f]{1,6})\}|([0-9A-Fa-f]{4}))")
_JS_LINE_TERMINATORS = "\r\n\u2028\u2029"
_JS_EXPRESSION_OPERATORS = {
@@ -691,7 +740,7 @@ def _static_codex_tool_calls(source: str) -> tuple[list[tuple[str, dict[str, Any
if index == len(source):
break
- call = _CODEX_CALL_RE.match(source, index)
+ call = _js_match(_CODEX_CALL_RE, source, index)
if call:
statements += 1
if statements > _MAX_STATEMENTS or len(calls) >= _MAX_TOOL_CALLS:
@@ -711,7 +760,7 @@ def _static_codex_tool_calls(source: str) -> tuple[list[tuple[str, dict[str, Any
index = end + close.end()
continue
- render = _CODEX_RENDER_RE.match(source, index)
+ render = _js_match(_CODEX_RENDER_RE, source, index)
rendered_variable = next((name for name in render.groups() if name), None) if render else None
if rendered_variable in variables:
statements += 1
@@ -771,3 +820,513 @@ def normalize_tool_call(tool_call: dict[str, Any]) -> list[dict[str, Any]]:
}
)
return normalized
+
+
+# ── Extended static grammar for inline and Promise.allSettled wrappers (#235) ──
+# Codex 0.161.0 also emits `text(await tools.NAME(...))`, single-quoted strings and literal
+# `Promise.allSettled` batches. This grammar decodes only those static forms, without evaluating
+# JavaScript. It runs first for every `exec` wrapper, so a call is graded the same whatever its
+# spelling, and the grammar above answers only where this one declines. A wrapper with more
+# than one call is decoded only when each call renders one JSON record of the outer output: the
+# call owns that record, and a missing, extra, reordered, changed, truncated or duplicate-key
+# record leaves the whole wrapper unsupported. A record's own exit code or rejection decides
+# whether its call failed; a record that states neither leaves the outcome unknown.
+
+_RECORD_FORMAT = "_atif_record_format"
+_RECORD_INDEX = "_atif_record_index"
+_RECORD_OBSERVATION = "_atif_record_observation"
+_RECORD_IS_ERROR = "_atif_record_is_error"
+_EXEC_OUTPUT_RE = re.compile(r"Script completed\r?\nWall time [^\r\n]*\r?\nOutput:\r?\n\r?\n")
+# JavaScript's whitespace and line terminators; `str.isspace()` also takes U+001C-U+001F and U+0085.
+_JS_SPACE = frozenset("\t\n\v\f\r \xa0\u1680\u2028\u2029\u202f\u205f\u3000\ufeff") | {
+ chr(code) for code in range(0x2000, 0x200B)
+}
+_JS_LINE_BREAK_RE = re.compile("[\n\r\u2028\u2029]")
+_JS_NUMBER_RE = re.compile(r"-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?(?![\w$.])")
+_JS_NAME_RE = re.compile(rf"{_JS_IDENTIFIER}(?![\w$\\])")
+# Names a binding may not take: the globals the grammar relies on, and reserved words.
+_RESERVED_WORDS = (
+ "Infinity JSON NaN Promise arguments async await break case catch class const continue debugger "
+ "default delete do else enum eval export extends false finally for function globalThis if implements "
+ "import in instanceof interface let new null of package private protected public return static super "
+ "switch text this throw tools true try typeof undefined var void while with yield"
+)
+_RESERVED_BINDINGS = frozenset(_RESERVED_WORDS.split())
+
+
+class _NotStatic(Exception):
+ """The wrapper source is outside the extended static grammar."""
+
+
+class _StaticExecSource:
+ """A cursor over one native Codex wrapper that accepts literal tokens only."""
+
+ def __init__(self, source: str) -> None:
+ self.source = source
+ pragma = _CODEX_PRAGMA_RE.match(source)
+ self.index = pragma.end() if pragma else 0
+
+ def at_end(self) -> bool:
+ while self.index < len(self.source) and self.source[self.index] in _JS_SPACE:
+ self.index += 1
+ return self.index == len(self.source)
+
+ def same_line(self) -> None:
+ """Reject a line break before the next token: `=>` and a postfix `++` may not follow one."""
+ start = self.index
+ self.at_end()
+ if _JS_LINE_BREAK_RE.search(self.source, start, self.index):
+ raise _NotStatic("line break")
+
+ def peek(self) -> str:
+ self.at_end()
+ return self.source[self.index : self.index + 1]
+
+ def take(self, token: str) -> bool:
+ if self.at_end() or not self.source.startswith(token, self.index):
+ return False
+ end = self.index + len(token)
+ if (
+ token[-1].isalnum()
+ and end < len(self.source)
+ and (self.source[end].isalnum() or self.source[end] in "_$\\")
+ ):
+ return False
+ self.index = end
+ return True
+
+ def expect(self, *tokens: str) -> None:
+ for token in tokens:
+ if not self.take(token):
+ raise _NotStatic(token)
+
+ def name(self) -> str:
+ self.at_end()
+ match = _JS_NAME_RE.match(self.source, self.index)
+ if match is None or not _js_name(match.group(0)):
+ raise _NotStatic("identifier")
+ self.index = match.end()
+ return match.group(0)
+
+ def binding(self, taken: set[str]) -> str:
+ """Read a new binding; reusing or shadowing a name is rejected."""
+ name = self.name()
+ if name in _RESERVED_BINDINGS or name in taken:
+ raise _NotStatic(name)
+ taken.add(name)
+ return name
+
+ def string(self) -> str:
+ """Decode one plain single- or double-quoted string that uses JSON escapes only."""
+ quote = self.source[self.index]
+ end = self.index + 1
+ while end < len(self.source) and self.source[end] != quote:
+ end += 2 if self.source[end] == "\\" else 1
+ if end >= len(self.source):
+ raise _NotStatic("string")
+ body = self.source[self.index + 1 : end]
+ if quote == "'":
+ body = re.sub(r"\\'|\\.|\"", lambda m: {"\\'": "'", '"': '\\"'}.get(m.group(0), m.group(0)), body)
+ try:
+ value = json.loads(f'"{body}"')
+ except (json.JSONDecodeError, ValueError) as exc:
+ raise _NotStatic("string") from exc
+ self.index = end + 1
+ return value
+
+ def literal(self, depth: int = 0) -> Any:
+ """Decode a JSON-compatible literal: no templates, spreads, computed keys or comments."""
+ char = self.peek()
+ if depth > _MAX_OBJECT_NESTING or not char:
+ raise _NotStatic("literal")
+ if char in "\"'":
+ return self.string()
+ if char in "{[":
+ self.index += 1
+ items: list[Any] = []
+ value: dict[str, Any] = {}
+ while not self.take("}" if char == "{" else "]"):
+ if items or value:
+ self.expect(",")
+ if char == "[":
+ items.append(self.literal(depth + 1))
+ continue
+ key = self.string() if self.peek() in ("'", '"') else self.name()
+ if key in value or key == "__proto__":
+ raise _NotStatic(key)
+ self.expect(":")
+ value[key] = self.literal(depth + 1)
+ return value if char == "{" else items
+ for word, word_value in (("true", True), ("false", False), ("null", None)):
+ if self.take(word):
+ return word_value
+ number = _JS_NUMBER_RE.match(self.source, self.index)
+ if number is None:
+ raise _NotStatic("literal")
+ try:
+ number_value = json.loads(number.group(0))
+ except ValueError as exc: # an integer beyond Python's digit limit
+ raise _NotStatic("number") from exc
+ if isinstance(number_value, float) and not math.isfinite(number_value):
+ raise _NotStatic("number")
+ self.index = number.end()
+ return number_value
+
+ def tool_call(self) -> tuple[str, dict[str, Any]]:
+ """Read `tools.NAME(<object literal>)` or `tools.apply_patch(<string literal>)`."""
+ self.expect("tools", ".")
+ function_name = self.name()
+ self.expect("(")
+ if function_name == "apply_patch" and self.peek() in ("'", '"'):
+ # Codex records its freeform apply_patch tool with the patch as `input`.
+ arguments: Any = {"input": self.string()}
+ else:
+ arguments = self.literal()
+ if not isinstance(arguments, dict):
+ raise _NotStatic("arguments")
+ self.expect(")")
+ return function_name, arguments
+
+ def end_statement(self) -> None:
+ if not self.take(";") and not self.at_end():
+ raise _NotStatic(";")
+
+
+def _settled_renderer(cursor: _StaticExecSource, results: str, taken: set[str]) -> tuple[str, ...]:
+ """Read the one renderer of a settled batch and return the format of the records it prints.
+
+ Accepted: `for (let i = 0; i < results.length; i++) text({i, ...results[i]})`,
+ `results.forEach((r, i) => text({i, ...r}))` (both also with `{i, result: ...}` and a braced
+ body), and `results.forEach(text)` or `results.forEach(r => text(r))`.
+ """
+
+ def payload(index: str, element) -> tuple[str, ...]:
+ if index in ("status", "value", "reason"):
+ raise _NotStatic(index)
+ cursor.expect("text", "(", "{", index, ",")
+ if cursor.take("..."):
+ element()
+ record_format: tuple[str, ...] = ("settled_spread", index)
+ else:
+ key = cursor.name()
+ if key == index:
+ raise _NotStatic(key)
+ cursor.expect(":")
+ element()
+ record_format = ("settled_field", index, key)
+ cursor.expect("}", ")")
+ return record_format
+
+ def body(render) -> tuple[str, ...]:
+ braced = cursor.take("{")
+ record_format = render()
+ if braced:
+ cursor.take(";")
+ cursor.expect("}")
+ return record_format
+
+ def render_element(element: str) -> tuple[str, ...]:
+ cursor.expect("text", "(", element, ")")
+ return ("settled",)
+
+ if cursor.take("for"):
+ cursor.expect("(", "let")
+ index = cursor.binding(taken)
+ cursor.expect("=", "0", ";", index, "<", results, ".", "length", ";", index)
+ cursor.same_line()
+ cursor.expect("++", ")")
+ return body(lambda: payload(index, lambda: cursor.expect(results, "[", index, "]")))
+ cursor.expect(results, ".", "forEach", "(")
+ if cursor.take("text"):
+ cursor.expect(")")
+ return ("settled",)
+ parenthesized = cursor.take("(")
+ element = cursor.binding(taken)
+ index = cursor.binding(taken) if parenthesized and cursor.take(",") else None
+ if parenthesized:
+ cursor.expect(")")
+ cursor.same_line()
+ cursor.expect("=>")
+ if index is None:
+ record_format = body(lambda: render_element(element))
+ else:
+ record_format = body(lambda: payload(index, lambda: cursor.expect(element)))
+ cursor.expect(")")
+ return record_format
+
+
+def _extended_codex_program(source: str) -> tuple[list[tuple[str, dict[str, Any]]], int | None, tuple] | None:
+ """Decode the extended grammar into ``(calls, owner, record_format)``, or None outside it.
+
+ A single call keeps the upstream ownership: ``owner`` 0 owns the whole outer observation,
+ -1 renders nothing. A lone call that prints its whole result has the ``("single",)`` format,
+ so its own exit code can mark it failed. With records, ``owner`` is None and ``record_format``
+ says how to split the outer observation; for inline renders it lists the call of each record
+ in output order.
+ """
+ cursor = _StaticExecSource(source)
+ calls: list[tuple[str, dict[str, Any]]] = []
+ bindings: dict[str, int] = {}
+ renders: list[tuple[int, bool]] = []
+ taken: set[str] = set()
+ statements = 0
+ try:
+ while not cursor.at_end():
+ statements += 1
+ if statements > _MAX_STATEMENTS or len(calls) >= _MAX_TOOL_CALLS:
+ return None
+ if cursor.take("const"):
+ variable = cursor.binding(taken)
+ cursor.expect("=", "await")
+ if cursor.take("Promise"):
+ if calls:
+ return None
+ cursor.expect(".", "allSettled", "(", "[")
+ while not cursor.take("]"):
+ if calls:
+ cursor.expect(",")
+ if cursor.take("]"):
+ break
+ if len(calls) >= _MAX_TOOL_CALLS:
+ return None
+ calls.append(cursor.tool_call())
+ cursor.expect(")")
+ cursor.end_statement()
+ record_format = _settled_renderer(cursor, variable, taken)
+ cursor.end_statement()
+ return (calls, None, record_format) if calls and cursor.at_end() else None
+ bindings[variable] = len(calls)
+ calls.append(cursor.tool_call())
+ else:
+ cursor.expect("text", "(")
+ if cursor.take("("):
+ cursor.expect("await")
+ renders.append((len(calls), False))
+ calls.append(cursor.tool_call())
+ cursor.expect(")", ".")
+ cursor.name()
+ elif cursor.take("await"):
+ renders.append((len(calls), True))
+ calls.append(cursor.tool_call())
+ elif cursor.take("JSON"):
+ cursor.expect(".", "stringify", "(")
+ renders.append((bindings[cursor.name()], True))
+ cursor.expect(")")
+ else:
+ call_index = bindings[cursor.name()]
+ whole = not cursor.take(".")
+ if not whole:
+ cursor.name()
+ renders.append((call_index, whole))
+ cursor.expect(")")
+ cursor.end_statement()
+ except (_NotStatic, KeyError, RecursionError):
+ return None
+ if len(calls) == 1:
+ # A lone call that prints its whole result owns the outer output, which is its record.
+ return calls, 0 if renders else -1, ("single",) if renders and renders[0][1] else ()
+ owners = [call_index for call_index, _ in renders]
+ if not calls or not all(whole for _, whole in renders) or sorted(owners) != list(range(len(calls))):
+ return None
+ return calls, None, ("result", *owners)
+
+
+def _settled_result(value: object) -> dict[str, Any] | None:
+ """Return a `Promise.allSettled` result with exactly its own keys, else None."""
+ if not isinstance(value, dict):
+ return None
+ if value.get("status") == "fulfilled" and set(value) == {"status", "value"}:
+ return value
+ if value.get("status") == "rejected" and set(value) == {"status", "reason"}:
+ return value
+ return None
+
+
+def _returned_outcome(value: object) -> tuple[bool | None, str]:
+ """Return the outcome of a tool that returned: only its own exit code says whether it succeeded."""
+ code = value.get("exit_code") if isinstance(value, dict) else None
+ return (code != 0, f"exit_code={code}" if code else "") if type(code) is int else (None, "")
+
+
+def _exec_record(value: object, record_format: tuple, record_index: int) -> tuple[bool | None, str] | None:
+ """Check one record against its format; return its ``(is_error, evidence)``, or None if malformed.
+
+ ``is_error`` is None while the record states no outcome. ``evidence`` is the status line, in the
+ form the scorers read, that makes an explicit outcome visible in the record's observation.
+ """
+ kind = record_format[0]
+ if kind in ("result", "single"):
+ return _returned_outcome(value) if isinstance(value, dict) else None
+ if kind == "settled":
+ settled = _settled_result(value)
+ elif (
+ not isinstance(value, dict)
+ or type(value.get(record_format[1])) is not int
+ or value[record_format[1]] != record_index
+ ):
+ return None
+ elif kind == "settled_spread":
+ settled = _settled_result({key: item for key, item in value.items() if key != record_format[1]})
+ else:
+ settled = _settled_result(value.get(record_format[2])) if len(value) == 2 else None
+ if settled is None:
+ return None
+ return (True, "status=failed") if settled["status"] == "rejected" else _returned_outcome(settled["value"])
+
+
+class _DuplicateKey(ValueError):
+ """A record repeats a key, so which value counts would be ambiguous."""
+
+
+def _unique_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
+ """Reject a duplicate key, which would let the last value silently win."""
+ if len({key for key, _ in pairs}) != len(pairs):
+ raise _DuplicateKey("duplicate key")
+ return dict(pairs)
+
+
+def _codex_exec_records(observation: str, record_format: tuple, count: int) -> list[tuple[str, bool | None]] | None:
+ """Split a completed outer observation into exactly ``count`` checked records, in output order."""
+ header = _EXEC_OUTPUT_RE.match(observation)
+ if header is None:
+ return None
+ body = observation[header.end() :]
+ decoder = json.JSONDecoder(object_pairs_hook=_unique_keys)
+ position = len(body) - len(body.lstrip()) # `raw_decode` does not skip the whitespace before a record
+ records: list[tuple[str, bool | None]] = []
+ for record_index in range(count):
+ if record_index:
+ if body.startswith("\r\n", position):
+ position += 1
+ if not body.startswith("\n", position):
+ return None
+ position += 1
+ try:
+ value, end = decoder.raw_decode(body, position)
+ except _DuplicateKey:
+ raise
+ except (json.JSONDecodeError, RecursionError, ValueError):
+ return None
+ outcome = _exec_record(value, record_format, record_index)
+ if outcome is None:
+ return None
+ is_error, evidence = outcome
+ records.append((f"{body[position:end]}\n{evidence}" if evidence else body[position:end], is_error))
+ position = end
+ return records if not body[position:].strip() else None
+
+
+_upstream_normalize_tool_call = normalize_tool_call
+_upstream_normalized_tool_call_observation = normalized_tool_call_observation
+_upstream_normalized_tool_call_error_flag = normalized_tool_call_error_flag
+
+
+def _normalize_tool_call_with_records(tool_call: dict[str, Any]) -> list[dict[str, Any]]:
+ """Unwrap proven native Codex calls, adding the inline and allSettled forms of #235."""
+ normalized = _upstream_normalize_tool_call(tool_call)
+ arguments = tool_call.get("arguments")
+ source = arguments.get("input") if tool_call.get("function_name") == "exec" and isinstance(arguments, dict) else None
+ parsed = _extended_codex_program(source) if isinstance(source, str) and len(source) <= _MAX_SOURCE_CHARS else None
+ if parsed is None:
+ if "_atif_inner_tool_index" in normalized[0] and _has_non_js_space(source):
+ # The upstream lexer accepted a character JavaScript rejects, so nothing in it ran.
+ outer = {key: value for key, value in tool_call.items() if not key.startswith("_atif_")}
+ return [{**outer, "_atif_normalization_status": UNSUPPORTED_NATIVE_CODEX_EXEC}]
+ return normalized
+ calls, observation_owner, record_format = parsed
+ outer = {key: value for key, value in tool_call.items() if not key.startswith("_atif_")}
+ result: list[dict[str, Any]] = []
+ for inner_index, (function_name, inner_arguments) in enumerate(calls):
+ call = {
+ **outer,
+ "function_name": function_name,
+ "arguments": inner_arguments,
+ "_atif_inner_tool_index": inner_index,
+ "_atif_outer_observation_owner": observation_owner,
+ }
+ if observation_owner is not None:
+ call["_atif_observation_status"] = (
+ MAPPED_OUTER_EXEC_OBSERVATION if observation_owner == inner_index else UNOBSERVED_INNER_CALL
+ )
+ if record_format:
+ call[_RECORD_FORMAT], call[_RECORD_INDEX] = record_format, 0
+ else:
+ # Ambiguous until the outer observation proves which record belongs to this call.
+ call["_atif_observation_status"] = AMBIGUOUS_OUTER_EXEC_OBSERVATION
+ call[_RECORD_FORMAT] = record_format
+ call[_RECORD_INDEX] = record_format[1:].index(inner_index) if record_format[0] == "result" else inner_index
+ result.append(call)
+ return result
+
+
+def _resolve_exec_records(
+ step: dict[str, Any], tool_call: dict[str, Any], raw_index: int, normalized: list[dict[str, Any]]
+) -> list[dict[str, Any]]:
+ """Give each call of a record wrapper its own record, or leave the wrapper unsupported."""
+ observation = _outer_tool_call_observation(step, tool_call, include_unscoped=raw_index == 0)
+ record_format = normalized[0][_RECORD_FORMAT]
+ try:
+ records = _codex_exec_records(observation, record_format, len(normalized))
+ duplicate = False
+ except _DuplicateKey:
+ records, duplicate = None, True
+ if record_format == ("single",):
+ header = _EXEC_OUTPUT_RE.match(observation)
+ # A lone call may print plain text, but a body that opens like a record and does not parse
+ # is incomplete evidence: a cut-off failure must not score as a clean run.
+ incomplete = duplicate or (
+ records is None and header is not None and observation[header.end() :].lstrip()[:1] == "{"
+ )
+ if incomplete:
+ outer = {key: value for key, value in tool_call.items() if not key.startswith("_atif_")}
+ return [{**outer, "_atif_normalization_status": UNSUPPORTED_NATIVE_CODEX_EXEC}]
+ if not (records and records[0][1] is not None):
+ return normalized # a lone call keeps its whole output unless its record states an outcome
+ if records is None:
+ return _upstream_normalize_tool_call(tool_call)
+ return [
+ {
+ **call,
+ "_atif_observation_status": MAPPED_OUTER_EXEC_OBSERVATION,
+ "_atif_outer_observation_owner": call["_atif_inner_tool_index"],
+ _RECORD_OBSERVATION: records[call[_RECORD_INDEX]][0],
+ _RECORD_IS_ERROR: records[call[_RECORD_INDEX]][1],
+ }
+ for call in normalized
+ ]
+
+
+def _iter_normalized_tool_calls_with_records(traj: dict[str, Any]):
+ """Yield each trajectory tool call after safe native-Codex normalization."""
+ for step in traj.get("steps", []):
+ for raw_index, tool_call in enumerate(step.get("tool_calls") or []):
+ normalized = normalize_tool_call(tool_call)
+ if normalized and _RECORD_FORMAT in normalized[0]:
+ normalized = _resolve_exec_records(step, tool_call, raw_index, normalized)
+ for call in normalized:
+ yield step, {**call, "_atif_raw_tool_index": raw_index}
+
+
+def _normalized_tool_call_observation_with_records(step: dict[str, Any], tool_call: dict[str, Any]) -> str:
+ """Return a call's own record, else the outer observation when its owner is known."""
+ if tool_call.get("_atif_observation_status") == MAPPED_OUTER_EXEC_OBSERVATION and _RECORD_OBSERVATION in tool_call:
+ return tool_call[_RECORD_OBSERVATION]
+ return _upstream_normalized_tool_call_observation(step, tool_call)
+
+
+def _normalized_tool_call_error_flag_with_records(
+ step: dict[str, Any], tool_call: dict[str, Any]
+) -> bool | None:
+ """Report a call's own record outcome, which may be unknown, before the upstream flag."""
+ if (
+ tool_call.get("_atif_observation_status") == MAPPED_OUTER_EXEC_OBSERVATION
+ and _RECORD_IS_ERROR in tool_call
+ ):
+ return tool_call[_RECORD_IS_ERROR]
+ return _upstream_normalized_tool_call_error_flag(step, tool_call)
+
+
+normalize_tool_call = _normalize_tool_call_with_records
+iter_normalized_tool_calls = _iter_normalized_tool_calls_with_records # noqa: F811
+normalized_tool_call_observation = _normalized_tool_call_observation_with_records
+normalized_tool_call_error_flag = _normalized_tool_call_error_flag_with_records
diff --git a/tests/tier3/test_codex_tool_call_normalization.py b/tests/tier3/test_codex_tool_call_normalization.py
index 036a775c..bb31dfc1 100644
--- a/tests/tier3/test_codex_tool_call_normalization.py
+++ b/tests/tier3/test_codex_tool_call_normalization.py
@@ -4,12 +4,13 @@
from __future__ import annotations
import importlib.util
+import json
import sys
from pathlib import Path
import pytest
-from skillevaluator.tier3.eval_core import codex_tool_call_normalizer
+from skillevaluator.tier3.eval_core import checks, codex_tool_call_normalizer
from skillevaluator.tier3.eval_core.atif_helpers import (
build_behavior_evidence,
build_metric_evidence_refs,
@@ -1058,3 +1059,813 @@ def test_host_and_template_summaries_agree_on_list_observations() -> None:
assert atif_helpers._late_observation_excerpts(trajectory, 200) == _TEMPLATE_MODULE._late_observation_excerpts(
trajectory, 200
)
+
+
+# ── Extended static grammar for inline and Promise.allSettled wrappers (#235) ──
+
+_ALL_SETTLED_SOURCE = (
+ "const results=await Promise.allSettled(["
+ 'tools.exec_command({cmd:"pwd"}),'
+ 'tools.write_stdin({session_id:1,chars:""})'
+ "]);"
+)
+_ALL_SETTLED_SPREAD_RECORDS = [
+ {"i": 0, "status": "fulfilled", "value": {"exit_code": 0, "output": "/workspace\n"}},
+ {"i": 1, "status": "rejected", "reason": {"error": "boom"}},
+]
+_ALL_SETTLED_FIELD_RECORDS = [
+ {"i": index, "result": {key: value for key, value in record.items() if key != "i"}}
+ for index, record in enumerate(_ALL_SETTLED_SPREAD_RECORDS)
+]
+_ALL_SETTLED_RAW_RECORDS = [
+ {"status": "fulfilled", "value": {"exit_code": 0, "output": "/workspace\n"}},
+ {"status": "rejected", "reason": {"error": "boom"}},
+]
+
+
+def _expected_records(records: list[dict]) -> list[str]:
+ """Each record's JSON text, plus the failed-status line a rejected promise gains."""
+ return [
+ json.dumps(record, separators=(",", ":")) + ("\nstatus=failed" if "rejected" in str(record) else "")
+ for record in records
+ ]
+
+
+_RECORD_FRAME = ("Script completed\nWall time 0.1 seconds\nOutput:\n\n", "")
+
+
+def _record_observation(records: list[dict]) -> str:
+ body = "\n".join(json.dumps(record, separators=(",", ":")) for record in records)
+ return "Script completed\nWall time 0.1 seconds\nOutput:\n\n" + body
+
+
+def _security_result(source: str, extractor):
+ trajectory = _trajectory(source)
+ calls = extractor(trajectory)
+ if extractor is _TEMPLATE_MODULE.extract_tool_calls_as_dicts:
+ return _TEMPLATE_MODULE.check_security(trajectory, calls)
+ return check_security(calls)
+
+
[email protected]("extractor", _EXTRACTORS)
+ "renderer, records",
+ [
+ pytest.param(
+ "results.forEach((r,i)=>text({i,...r}));",
+ _ALL_SETTLED_SPREAD_RECORDS,
+ id="forEach-spread",
+ ),
+ pytest.param(
+ "results.forEach((r, i) => { text({i, ...r}); });",
+ _ALL_SETTLED_SPREAD_RECORDS,
+ id="forEach-spread-braced",
+ ),
+ pytest.param(
+ "for (let i = 0; i < results.length; i++) text({i, ...results[i]});",
+ _ALL_SETTLED_SPREAD_RECORDS,
+ id="for-spread",
+ ),
+ pytest.param(
+ "for (let i = 0; i < results.length; i++) { text({i, ...results[i]}); }",
+ _ALL_SETTLED_SPREAD_RECORDS,
+ id="for-spread-braced",
+ ),
+ pytest.param("results.forEach(text);", _ALL_SETTLED_RAW_RECORDS, id="forEach-text"),
+ pytest.param("results.forEach(r => text(r));", _ALL_SETTLED_RAW_RECORDS, id="forEach-arrow-text"),
+ pytest.param(
+ "results.forEach((r,i)=>text({i,result:r}));",
+ _ALL_SETTLED_FIELD_RECORDS,
+ id="forEach-field",
+ ),
+ ],
+)
+def test_extended_codex_allsettled_maps_each_call_to_its_record(extractor, renderer, records):
+ calls = extractor(_trajectory(_ALL_SETTLED_SOURCE + renderer, _record_observation(records)))
+
+ assert [call["action"] for call in calls] == ["exec_command", "write_stdin"]
+ assert calls[0]["action_input"] == {"cmd": "pwd"}
+ assert calls[1]["action_input"] == {"session_id": 1, "chars": ""}
+ assert [call["observation_status"] for call in calls] == ["mapped_outer_exec_result"] * 2
+ assert [call["observation"] for call in calls] == _expected_records(records)
+
+
[email protected]("extractor", _EXTRACTORS)
+ "source",
+ [
+ pytest.param('text(await tools.exec_command({cmd:"pwd"}));', id="inline-await"),
+ pytest.param("const r=await tools.exec_command({cmd:'pwd'});text(r);", id="single-quoted-string"),
+ pytest.param("const r=await tools.exec_command({'cmd':'pwd'});text(r);", id="single-quoted-key"),
+ pytest.param('text((await tools.exec_command({cmd:"pwd"})).output);', id="inline-output"),
+ pytest.param("const r=await tools.exec_command({cmd:'pwd'});text(r.output);", id="bound-output"),
+ pytest.param('text(await tools.exec_command({cmd:"pwd"}))', id="inline-without-final-semicolon"),
+ pytest.param('const r=await tools.exec_command({cmd:"pwd"});text(r)', id="bound-without-final-semicolon"),
+ ],
+)
+def test_extended_codex_inline_and_single_quoted_wrappers_normalize(extractor, source):
+ calls = extractor(_trajectory(source))
+
+ assert [call["action"] for call in calls] == ["exec_command"]
+ assert calls[0]["action_input"] == {"cmd": "pwd"}
+ assert calls[0]["observation_status"] == "mapped_outer_exec_result"
+ assert "normalization_status" not in calls[0]
+
+
+ "source, expected",
+ [
+ pytest.param(
+ r"const r=await tools.exec_command({cmd:'echo \'hi\''});text(r);",
+ "echo 'hi'",
+ id="escaped-single-quote",
+ ),
+ pytest.param(
+ r"""const r=await tools.exec_command({cmd:'say "hi"'});text(r);""",
+ 'say "hi"',
+ id="double-quote-inside-single",
+ ),
+ ],
+)
+def test_extended_codex_single_quoted_string_escapes(source, expected):
+ calls = extract_tool_calls_as_dicts(_trajectory(source))
+
+ assert calls[0]["action_input"] == {"cmd": expected}
+
+
[email protected]("extractor", _EXTRACTORS)
+ "source",
+ [
+ pytest.param(
+ r"const p=await tools.apply_patch('*** Begin Patch\n*** End Patch');text(p);",
+ id="single-quoted",
+ ),
+ pytest.param(
+ r'const p=await tools.apply_patch("*** Begin Patch\n*** End Patch");text(p);',
+ id="double-quoted",
+ ),
+ pytest.param(
+ r"text(await tools.apply_patch('*** Begin Patch\n*** End Patch'));",
+ id="inline",
+ ),
+ ],
+)
+def test_extended_codex_apply_patch_string_literal_normalizes(extractor, source):
+ calls = extractor(_trajectory(source))
+
+ assert [call["action"] for call in calls] == ["apply_patch"]
+ assert calls[0]["action_input"] == {"input": "*** Begin Patch\n*** End Patch"}
+
+
[email protected]("extractor", _EXTRACTORS)
+ "source",
+ [
+ pytest.param('const cmd="pwd";text(await tools.exec_command({cmd}));', id="dynamic-argument"),
+ pytest.param('text(await tools.exec_command({["cmd"]:"pwd"}));', id="computed-key"),
+ pytest.param("text(await tools.exec_command({...args}));", id="spread-arguments"),
+ pytest.param("text(await tools.exec_command({cmd:`pwd`}));", id="template-literal"),
+ pytest.param('text(await tools.exec_command({cmd:"pwd"})); // comment', id="trailing-comment"),
+ pytest.param('const text=1;text(await tools.exec_command({cmd:"pwd"}));', id="shadowed-global"),
+ pytest.param(
+ 'const r=await tools.exec_command({cmd:"pwd"});const r=await tools.exec_command({cmd:"ls"});text(r);',
+ id="duplicate-binding",
+ ),
+ pytest.param(
+ 'text(await tools.exec_command({cmd:"pwd"}));text(await tools.exec_command({cmd:"ls"}));',
+ id="multi-inline-without-records",
+ ),
+ pytest.param(_ALL_SETTLED_SOURCE + "results.map((r,i)=>text({i,...r}));", id="unsupported-renderer-map"),
+ pytest.param(_ALL_SETTLED_SOURCE + "for (const r of results) text(r);", id="unsupported-renderer-for-of"),
+ ],
+)
+def test_extended_codex_dynamic_and_ambiguous_forms_stay_atomic(extractor, source):
+ calls = extractor(_trajectory(source))
+
+ assert len(calls) == 1
+ assert calls[0]["action"] == "exec"
+ assert calls[0]["normalization_status"] == "unsupported_native_codex_exec_wrapper"
+
+
[email protected]("extractor", _EXTRACTORS)
+ "records",
+ [
+ pytest.param(_ALL_SETTLED_SPREAD_RECORDS[:1], id="missing-record"),
+ pytest.param(
+ [*_ALL_SETTLED_SPREAD_RECORDS, {"i": 2, "status": "fulfilled", "value": {}}],
+ id="extra-record",
+ ),
+ pytest.param(list(reversed(_ALL_SETTLED_SPREAD_RECORDS)), id="reordered-records"),
+ pytest.param([_ALL_SETTLED_SPREAD_RECORDS[0], _ALL_SETTLED_SPREAD_RECORDS[0]], id="duplicate-index"),
+ pytest.param([{"i": 0, "status": "fulfilled"}, _ALL_SETTLED_SPREAD_RECORDS[1]], id="malformed-record"),
+ pytest.param(
+ [
+ {**_ALL_SETTLED_SPREAD_RECORDS[0], "extra": True},
+ _ALL_SETTLED_SPREAD_RECORDS[1],
+ ],
+ id="extra-key",
+ ),
+ ],
+)
+def test_extended_codex_untrusted_allsettled_records_fall_back_to_atomic(extractor, records):
+ source = _ALL_SETTLED_SOURCE + "results.forEach((r,i)=>text({i,...r}));"
+ calls = extractor(_trajectory(source, _record_observation(records)))
+
+ assert len(calls) == 1
+ assert calls[0]["action"] == "exec"
+ assert calls[0]["normalization_status"] == "unsupported_native_codex_exec_wrapper"
+
+
+ "observation",
+ [
+ pytest.param("", id="empty"),
+ pytest.param("partial", id="without-output-header"),
+ pytest.param(_record_observation(_ALL_SETTLED_SPREAD_RECORDS).rsplit("\n", 1)[0], id="truncated"),
+ pytest.param(_record_observation(_ALL_SETTLED_SPREAD_RECORDS) + "\nextra", id="trailing-text"),
+ ],
+)
+def test_extended_codex_incomplete_allsettled_observation_falls_back_to_atomic(observation):
+ source = _ALL_SETTLED_SOURCE + "results.forEach((r,i)=>text({i,...r}));"
+ calls = extract_tool_calls_as_dicts(_trajectory(source, observation))
+
+ assert len(calls) == 1
+ assert calls[0]["action"] == "exec"
+ assert calls[0]["normalization_status"] == "unsupported_native_codex_exec_wrapper"
+
+
+def test_extended_codex_rejected_allsettled_record_is_an_error():
+ source = _ALL_SETTLED_SOURCE + "results.forEach((r,i)=>text({i,...r}));"
+ trajectory = _trajectory(source, _record_observation(_ALL_SETTLED_SPREAD_RECORDS))
+ yielded = list(codex_tool_call_normalizer.iter_normalized_tool_calls(trajectory))
+
+ assert [codex_tool_call_normalizer.normalized_tool_call_is_error(step, call) for step, call in yielded] == [
+ False,
+ True,
+ ]
+ assert [codex_tool_call_normalizer.normalized_tool_call_error_flag(step, call) for step, call in yielded] == [
+ False,
+ True,
+ ]
+ assert [
+ codex_tool_call_normalizer.normalized_tool_call_observation(step, call) for step, call in yielded
+ ] == _expected_records(_ALL_SETTLED_SPREAD_RECORDS)
+
+
[email protected]("extractor", _EXTRACTORS)
+def test_extended_codex_extracted_records_carry_is_error(extractor):
+ source = _ALL_SETTLED_SOURCE + "results.forEach((r,i)=>text({i,...r}));"
+ calls = extractor(_trajectory(source, _record_observation(_ALL_SETTLED_SPREAD_RECORDS)))
+
+ assert [call.get("is_error") for call in calls] == [False, True]
+
+
+def test_extended_codex_record_status_wins_over_outer_failure():
+ source = _ALL_SETTLED_SOURCE + "results.forEach((r,i)=>text({i,...r}));"
+ trajectory = {
+ "steps": [
+ {
+ "source": "agent",
+ "tool_calls": [
+ {
+ "tool_call_id": "outer-call",
+ "function_name": "exec",
+ "arguments": {"input": source},
+ }
+ ],
+ "observation": {
+ "results": [
+ {
+ "source_call_id": "outer-call",
+ "content": _record_observation(_ALL_SETTLED_SPREAD_RECORDS),
+ "extra": {"tool_result_is_error": True},
+ }
+ ]
+ },
+ }
+ ]
+ }
+ calls = extract_tool_calls_as_dicts(trajectory)
+
+ assert [call.get("is_error") for call in calls] == [False, True]
+
+
[email protected]("extractor", _EXTRACTORS)
+def test_extended_codex_inline_multi_call_records_are_split_in_order(extractor):
+ source = 'text(await tools.exec_command({cmd:"pwd"}));text(await tools.exec_command({cmd:"ls"}));'
+ records = [{"stdout": "/workspace\n"}, {"stdout": "file\n"}]
+ calls = extractor(_trajectory(source, _record_observation(records)))
+
+ assert [call["action"] for call in calls] == ["exec_command", "exec_command"]
+ assert [call["action_input"] for call in calls] == [{"cmd": "pwd"}, {"cmd": "ls"}]
+ assert [call["observation"] for call in calls] == [json.dumps(record, separators=(",", ":")) for record in records]
+ assert all("is_error" not in call for call in calls)
+
+
[email protected]("extractor", _EXTRACTORS)
+def test_extended_codex_single_allsettled_record_normalizes(extractor):
+ source = (
+ 'const results=await Promise.allSettled([tools.exec_command({cmd:"pwd"})]);'
+ "results.forEach((r,i)=>text({i,...r}));"
+ )
+ records = [{"i": 0, "status": "fulfilled", "value": {"exit_code": 0}}]
+ calls = extractor(_trajectory(source, _record_observation(records)))
+
+ assert len(calls) == 1
+ assert calls[0]["action"] == "exec_command"
+ assert calls[0]["observation"] == json.dumps(records[0], separators=(",", ":"))
+
+
[email protected]("extractor", _EXTRACTORS)
+def test_extended_codex_inline_destructive_command_is_still_flagged(extractor):
+ source = 'text(await tools.exec_command({cmd:"rm -rf /workspace/project"}));'
+ result = _security_result(source, extractor)
+
+ assert result["passed"] is False
+ assert result["score"] == 0.0
+ assert [finding["type"] for finding in result["findings"]] == ["destructive_command"]
+
+
[email protected]("extractor", _EXTRACTORS)
+def test_extended_codex_dynamic_inline_is_not_a_clean_security_result(extractor):
+ source = 'const cmd="rm -rf /workspace/project";text(await tools.exec_command({cmd}));'
+ result = _security_result(source, extractor)
+
+ assert result["passed"] is False
+ assert result["score"] == 0.5
+ assert [finding["type"] for finding in result["findings"]] == ["unsupported_tool_wrapper"]
+
+
+def _scorer(extractor, name: str):
+ """The host or bundled-verifier scorer that pairs with ``extractor``."""
+ return getattr(_TEMPLATE_MODULE if extractor is _TEMPLATE_MODULE.extract_tool_calls_as_dicts else checks, name)
+
+
+def _error_recovery(trajectory: dict, extractor) -> dict:
+ return _scorer(extractor, "check_error_recovery")(extractor(trajectory))
+
+
+def _failed_exit_records(shape: str, code: int) -> list[dict]:
+ """The `pwd` record fails with ``code`` in the given record shape; the second record is clean."""
+ records = [
+ {"i": 0, "status": "fulfilled", "value": {"exit_code": code, "output": "boom\n"}},
+ {"i": 1, "status": "fulfilled", "value": {"exit_code": 0}},
+ ]
+ if shape == "field":
+ return [{"i": r["i"], "result": {k: v for k, v in r.items() if k != "i"}} for r in records]
+ return [{k: v for k, v in r.items() if k != "i"} for r in records] if shape == "raw" else records
+
+
+_RENDERERS = [
+ ("results.forEach((r,i)=>text({i,...r}));", "spread"),
+ ("for (let i = 0; i < results.length; i++) text({i, ...results[i]});", "spread"),
+ ("results.forEach(text);", "raw"),
+ ("results.forEach((r,i)=>text({i,result:r}));", "field"),
+]
+
+
[email protected]("extractor", _EXTRACTORS)
[email protected](("renderer", "shape"), _RENDERERS)
+def test_extended_codex_fulfilled_record_keeps_its_exit_code(extractor, renderer, shape):
+ records = _failed_exit_records(shape, 7)
+ calls = extractor(_trajectory(_ALL_SETTLED_SOURCE + renderer, _record_observation(records)))
+
+ assert [call["is_error"] for call in calls] == [True, False]
+ assert calls[0]["observation"].endswith("\nexit_code=7")
+ assert "exit_code=" not in calls[1]["observation"]
+
+
[email protected]("extractor", _EXTRACTORS)
+def test_extended_codex_inline_result_keeps_its_exit_code(extractor):
+ source = 'text(await tools.exec_command({cmd:"false"}));text(await tools.exec_command({cmd:"pwd"}));'
+ records = [{"exit_code": 3, "output": "boom\n"}, {"exit_code": 0, "output": "/workspace\n"}]
+ trajectory = _trajectory(source, _record_observation(records))
+
+ assert [call["is_error"] for call in extractor(trajectory)] == [True, False]
+ assert _error_recovery(trajectory, extractor)["first_attempt_clean"] is False
+
+
[email protected]("extractor", _EXTRACTORS)
+def test_extended_codex_fulfilled_record_without_exit_code_stays_unknown(extractor):
+ records = [
+ {"i": 0, "status": "fulfilled", "value": {"session_id": 4, "output": "running\n"}},
+ {"i": 1, "status": "fulfilled", "value": "done"},
+ ]
+ source = _ALL_SETTLED_SOURCE + "results.forEach((r,i)=>text({i,...r}));"
+ calls = extractor(_trajectory(source, _record_observation(records)))
+
+ assert [call["observation"] for call in calls] == _expected_records(records)
+ assert all("is_error" not in call for call in calls)
+
+
[email protected]("extractor", _EXTRACTORS)
[email protected]("reason", [{"error": "boom"}, {}, "boom"])
+def test_extended_codex_rejected_record_scores_as_a_failed_command(extractor, reason):
+ records = [{"status": "rejected", "reason": reason}, {"status": "fulfilled", "value": {"exit_code": 0}}]
+ source = _ALL_SETTLED_SOURCE + "results.forEach(text);"
+ trajectory = _trajectory(source, _record_observation(records))
+ result = _error_recovery(trajectory, extractor)
+
+ assert result["first_attempt_clean"] is False
+
+
[email protected]("extractor", _EXTRACTORS)
+def test_extended_codex_fulfilled_nonzero_exit_scores_as_a_failed_command(extractor):
+ records = _failed_exit_records("raw", 7)
+ trajectory = _trajectory(_ALL_SETTLED_SOURCE + "results.forEach(text);", _record_observation(records))
+ result = _error_recovery(trajectory, extractor)
+
+ assert result["first_attempt_clean"] is False
+ assert result["score"] < 1.0
+
+
[email protected]("extractor", _EXTRACTORS)
+def test_extended_codex_clean_records_still_score_clean(extractor):
+ records = _failed_exit_records("raw", 0)
+ trajectory = _trajectory(_ALL_SETTLED_SOURCE + "results.forEach(text);", _record_observation(records))
+
+ assert _error_recovery(trajectory, extractor)["first_attempt_clean"] is True
+
+
[email protected]("extractor", _EXTRACTORS)
+def test_extended_codex_failed_skill_read_gets_no_activation_credit(extractor):
+ source = (
+ 'const results=await Promise.allSettled([tools.exec_command({cmd:"cat /skills/example/SKILL.md"})]);'
+ "results.forEach(text);"
+ )
+ record = {"status": "fulfilled", "value": {"exit_code": 1, "output": "cat: No such file\n"}}
+ calls = extractor(_trajectory(source, _record_observation([record])))
+ result = _scorer(extractor, "check_activation")(calls, "example")
+
+ assert calls[0]["is_error"] is True
+ assert result["score"] == 0.0
+
+
[email protected]("extractor", _EXTRACTORS)
+ "records",
+ [
+ '{"i":1,"i":0,"status":"fulfilled","value":{}}\n{"i":1,"status":"fulfilled","value":{}}',
+ '{"i":0,"status":"rejected","status":"fulfilled","value":{}}\n{"i":1,"status":"fulfilled","value":{}}',
+ '{"i":0,"status":"fulfilled","value":{},"value":{"exit_code":0}}\n{"i":1,"status":"fulfilled","value":{}}',
+ '{"i":0,"status":"fulfilled","value":{"exit_code":1,"exit_code":0}}\n{"i":1,"status":"fulfilled","value":{}}',
+ ],
+ ids=["index", "status", "value", "nested"],
+)
+def test_extended_codex_duplicate_record_keys_stay_unsupported(extractor, records):
+ source = _ALL_SETTLED_SOURCE + "results.forEach((r,i)=>text({i,...r}));"
+ calls = extractor(_trajectory(source, "Script completed\nWall time 0.1 seconds\nOutput:\n\n" + records))
+
+ assert len(calls) == 1
+ assert calls[0]["normalization_status"] == "unsupported_native_codex_exec_wrapper"
+
+
[email protected]("extractor", _EXTRACTORS)
+ ("command", "output", "clean"),
+ [
+ ("grep foo file.txt", "", True),
+ ("grep foo file.txt", "grep: file.txt: Permission denied\n", False),
+ ("false", "", False),
+ ],
+ ids=["quiet-probe-no-match", "probe-with-diagnostic", "failed-command"],
+)
[email protected]("settled", [True, False], ids=["allSettled", "inline"])
+def test_extended_codex_exit_one_record_keeps_the_quiet_probe_exemption(extractor, command, output, clean, settled):
+ # A second call keeps the inline form on the record path; a lone inline call owns the whole output.
+ records = [{"exit_code": 1, "output": output}, {"exit_code": 0, "output": "/workspace\n"}]
+ calls = f'tools.exec_command({{cmd:"{command}"}}),tools.exec_command({{cmd:"pwd"}})'
+ if settled:
+ source = f"const results=await Promise.allSettled([{calls}]);results.forEach(text);"
+ records = [{"status": "fulfilled", "value": record} for record in records]
+ else:
+ source = "".join(f"text(await {call});" for call in calls.split(","))
+ result = _error_recovery(_trajectory(source, _record_observation(records)), extractor)
+
+ assert result["first_attempt_clean"] is clean
+
+
[email protected]("extractor", _EXTRACTORS)
+def test_extended_codex_lone_inline_call_keeps_its_exit_code(extractor):
+ source = 'text(await tools.exec_command({cmd:"exit 7"}));'
+ trajectory = _trajectory(source, _record_observation([{"exit_code": 7, "output": "boom\n"}]))
+ calls = extractor(trajectory)
+
+ assert [call["action"] for call in calls] == ["exec_command"]
+ assert calls[0]["is_error"] is True
+ assert _error_recovery(trajectory, extractor)["first_attempt_clean"] is False
+
+
[email protected]("extractor", _EXTRACTORS)
+def test_extended_codex_lone_inline_call_without_a_record_keeps_the_whole_output(extractor):
+ source = 'text(await tools.exec_command({cmd:"pwd"}));'
+ calls = extractor(_trajectory(source, "run.py completed"))
+
+ assert calls[0]["observation"] == "run.py completed"
+ assert "is_error" not in calls[0]
+
+
[email protected]("checker", [check_error_recovery, _TEMPLATE_MODULE.check_error_recovery])
+ ("observation", "clean"),
+ [
+ ('{"exit_code":1,"output":""}\nexit_code=1', True),
+ ('{"exit_code":1,"output":""}\nexit_code=1\ngrep: f: Permission denied', False),
+ ("[" * 100_000 + "\nexit_code=1", False),
+ ('{"a":{"output":""},"b":{"output":"x"}}\nexit_code=1', False),
+ ],
+ ids=["record", "text-after-record", "deeply-nested-is-not-a-record", "two-object-fields-are-ambiguous"],
+)
+def test_quiet_probe_reads_only_a_whole_record(checker, observation, clean):
+ call = {"action": "bash", "action_input": {"command": "grep foo f"}, "observation": observation}
+
+ assert checker([call])["first_attempt_clean"] is clean
+
+
[email protected]("extractor", _EXTRACTORS)
+ "source",
+ [
+ 'const r=await tools.exec_command({cmd:"cat /skills/example/SKILL.md"});text(r);',
+ 'const r=await tools.exec_command({cmd:"cat /skills/example/SKILL.md"});text(r)',
+ "const r=await tools.exec_command({cmd:'cat /skills/example/SKILL.md'});text(r);",
+ 'text(await tools.exec_command({cmd:"cat /skills/example/SKILL.md"}));',
+ ],
+ ids=["double-quoted", "no-final-semicolon", "single-quoted", "inline"],
+)
+def test_extended_codex_failed_call_grades_the_same_in_every_spelling(extractor, source):
+ record = {"exit_code": 1, "output": "cat: /skills/example/SKILL.md: No such file or directory\n"}
+ trajectory = _trajectory(source, _record_observation([record]))
+ calls = extractor(trajectory)
+
+ assert calls[0]["is_error"] is True
+ assert _scorer(extractor, "check_activation")(calls, "example")["score"] == 0.0
+ assert _error_recovery(trajectory, extractor)["first_attempt_clean"] is False
+
+
[email protected]("extractor", _EXTRACTORS)
+def test_extended_codex_two_bound_calls_each_get_their_own_record(extractor):
+ source = (
+ 'const a=await tools.exec_command({cmd:"false"});const b=await tools.exec_command({cmd:"pwd"});'
+ "text(a);text(b);"
+ )
+ records = [{"exit_code": 3, "output": "boom\n"}, {"exit_code": 0, "output": "/workspace\n"}]
+ calls = extractor(_trajectory(source, _record_observation(records)))
+
+ assert [call["observation_status"] for call in calls] == ["mapped_outer_exec_result"] * 2
+ assert [call["is_error"] for call in calls] == [True, False]
+
+
[email protected]("extractor", _EXTRACTORS)
+ "source",
+ [
+ 'const a=await tools.exec_command({cmd:"pwd"});const b=await tools.exec_command({cmd:"ls"});text(a);',
+ 'const a=await tools.exec_command({cmd:"pwd"});const b=await tools.exec_command({cmd:"ls"});'
+ "text(a.output);text(b.output);",
+ ],
+ ids=["one-rendered", "output-field"],
+)
+def test_extended_codex_bound_forms_it_cannot_split_keep_the_upstream_result(extractor, source):
+ calls = extractor(_trajectory(source, "outer observation"))
+
+ assert [call["action"] for call in calls] == ["exec_command", "exec_command"]
+ assert "is_error" not in calls[0]
+
+
[email protected]("extractor", _EXTRACTORS)
[email protected]("terminator", ["\u2028", "\u2029"], ids=["U+2028", "U+2029"])
+ "hidden",
+ [
+ "text(await tools.exec_command({cmd:'rm -rf /workspace/project'}));",
+ "const r=await tools.exec_command({cmd:'rm -rf /workspace/project'});text(r);",
+ ],
+ ids=["inline", "const"],
+)
+def test_pragma_line_ended_by_a_javascript_line_terminator_fails_closed(extractor, terminator, hidden):
+ # JavaScript ends a `//` comment at U+2028 and U+2029, so the code after one still runs.
+ source = f"// @exec: {{}}{terminator}{hidden}\ntext(await tools.exec_command({{cmd:'pwd'}}));"
+ result = _security_result(source, extractor)
+
+ assert result["passed"] is False
+ assert result["score"] == 0.5
+
+
[email protected]("extractor", _EXTRACTORS)
[email protected]("outer_error", [None, True], ids=["no-outer-flag", "outer-error-flag"])
+def test_extended_codex_lone_successful_call_uses_its_own_record(extractor, outer_error):
+ # The record says exit 0, so its traceback text and the outer flag must not decide the grade.
+ source = "text(await tools.exec_command({cmd:'cat tests/fixture.log'}));"
+ record = {"exit_code": 0, "output": "Traceback (most recent call last):\nValueError: bad\n"}
+ trajectory = _trajectory(source, _record_observation([record]))
+ if outer_error is not None:
+ trajectory["steps"][0]["observation"]["results"][0]["extra"] = {"tool_result_is_error": outer_error}
+ calls = extractor(trajectory)
+
+ assert calls[0]["is_error"] is False
+ assert _error_recovery(trajectory, extractor)["first_attempt_clean"] is True
+
+
[email protected]("extractor", _EXTRACTORS)
+ "source",
+ [
+ pytest.param("text(await\x1ctools.exec_command({cmd:'cat /skills/example/SKILL.md'}));", id="fs-whitespace"),
+ pytest.param("text(await\x85tools.exec_command({cmd:'cat /skills/example/SKILL.md'}));", id="nel-whitespace"),
+ pytest.param("const r²=await tools.exec_command({cmd:'cat /skills/example/SKILL.md'});text(r²);", id="name"),
+ pytest.param(_ALL_SETTLED_SOURCE + "results.forEach((r,i)\n=>text({i,...r}));", id="newline-before-arrow"),
+ pytest.param(
+ _ALL_SETTLED_SOURCE + "for (let i = 0; i < results.length; i\n++) text({i, ...results[i]});",
+ id="newline-before-increment",
+ ),
+ pytest.param("text(await tools.exec_command({cmd:'x',n:1e400}));", id="non-finite-number"),
+ ],
+)
+def test_wrapper_that_javascript_rejects_stays_unsupported(extractor, source):
+ # JavaScript throws a SyntaxError before any call runs, so no call may be credited. A batch gets
+ # valid records, so only its lexing can leave it unsupported.
+ observation = _record_observation(_ALL_SETTLED_SPREAD_RECORDS) if "allSettled" in source else "SyntaxError"
+ calls = extractor(_trajectory(source, observation))
+
+ assert len(calls) == 1
+ assert calls[0]["normalization_status"] == "unsupported_native_codex_exec_wrapper"
+
+
[email protected]("extractor", _EXTRACTORS)
[email protected]("field", ["result", "outcome", "settled", "output"])
+def test_quiet_probe_exemption_does_not_depend_on_the_settled_field_name(extractor, field):
+ calls = 'tools.exec_command({cmd:"grep -q foo README.md"}),tools.exec_command({cmd:"pwd"})'
+ source = f"const results=await Promise.allSettled([{calls}]);results.forEach((r,i)=>text({{i,{field}:r}}));"
+ values = [{"exit_code": 1, "output": ""}, {"exit_code": 0, "output": "/workspace\n"}]
+ records = [{"i": i, field: {"status": "fulfilled", "value": value}} for i, value in enumerate(values)]
+ result = _error_recovery(_trajectory(source, _record_observation(records)), extractor)
+
+ assert result["first_attempt_clean"] is True
+
+
[email protected]("extractor", _EXTRACTORS)
+def test_valid_non_ascii_javascript_name_still_decodes(extractor):
+ # U+00E9 is a valid JavaScript identifier character, unlike U+00B2.
+ source = "const résultat=await tools.exec_command({cmd:'pwd'});text(résultat);"
+ calls = extractor(_trajectory(source, "/workspace"))
+
+ assert [call["action"] for call in calls] == ["exec_command"]
+ assert calls[0]["action_input"] == {"cmd": "pwd"}
+
+
[email protected]("extractor", _EXTRACTORS)
+ "source",
+ [
+ 'text(await tools.exec_command({cmd:"false"}));',
+ 'const r=await tools.exec_command({cmd:"false"});text(r);',
+ ],
+ ids=["inline", "bound"],
+)
+def test_lone_call_with_a_duplicate_key_record_stays_unsupported(extractor, source):
+ calls = extractor(_trajectory(source, '{"exit_code":1,"exit_code":0,"output":""}'.join(_RECORD_FRAME)))
+
+ assert len(calls) == 1
+ assert calls[0]["normalization_status"] == "unsupported_native_codex_exec_wrapper"
+
+
+_LONE_SOURCES = [
+ pytest.param('text(await tools.exec_command({cmd:"make"}));', id="inline"),
+ pytest.param('const r=await tools.exec_command({cmd:"make"});text(r);', id="bound"),
+]
+
+
[email protected]("extractor", _EXTRACTORS)
[email protected]("source", _LONE_SOURCES)
+ "body",
+ [
+ '{"exit_code":1,"output":"' + "x" * 300,
+ '{"exit_code":1,"output":"x"}\nextra text',
+ '{"exit_code":1,"outp',
+ ],
+ ids=["truncated-mid-string", "trailing-text", "truncated-mid-key"],
+)
+def test_lone_call_with_an_incomplete_record_stays_unsupported(extractor, source, body):
+ # Incomplete evidence must stop grading: a cut-off failure would otherwise score as a clean run.
+ trajectory = _trajectory(source, "".join(_RECORD_FRAME[:1]) + body)
+ calls = extractor(trajectory)
+
+ assert len(calls) == 1
+ assert calls[0]["normalization_status"] == "unsupported_native_codex_exec_wrapper"
+ assert _error_recovery(trajectory, extractor)["first_attempt_clean"] is False
+
+
[email protected]("extractor", _EXTRACTORS)
+ ("source", "body"),
+ [
+ ('const r=await tools.exec_command({cmd:"pwd"});text(r);', "/workspace\n"),
+ ('const r=await tools.exec_command({cmd:"pwd"});text(r);', ""),
+ ("text(await tools.apply_patch('*** Begin Patch\\n*** End Patch\\n'));", "Success. Updated the files:\nA a.txt\n"),
+ ],
+ ids=["plain-text", "empty-body", "apply-patch-text"],
+)
+def test_lone_call_with_a_plain_text_result_keeps_the_whole_output(extractor, source, body):
+ observation = _RECORD_FRAME[0] + body
+ calls = extractor(_trajectory(source, observation))
+
+ assert len(calls) == 1
+ assert "normalization_status" not in calls[0]
+ assert calls[0]["observation"] == observation
+
+
+_SKILL_CAT = "cat /skills/example/SKILL.md"
+
+
[email protected]("extractor", _EXTRACTORS)
[email protected]("space", ["\x1c", "\x1f", "\x85"], ids=["U+001C", "U+001F", "U+0085"])
+ "source",
+ [
+ 'const r=await{s}tools.exec_command({{cmd:"{c}"}});text(r);',
+ 'const r=await tools.exec_command({{cmd:"{c}"}});{s}text(r);',
+ 'const r=await tools.exec_command({{cmd:"{c}"}});text(r);{s}',
+ 'const r={s}await tools.exec_command({{cmd:"{c}"}});text(r);',
+ ],
+ ids=["after-await", "before-text", "trailing", "after-equals"],
+)
+def test_upstream_grammar_form_with_a_non_javascript_space_stays_unsupported(extractor, space, source):
+ # Python's isspace() and \s take these characters, JavaScript does not, so the script never runs.
+ calls = extractor(_trajectory(source.format(s=space, c=_SKILL_CAT), "SyntaxError"))
+
+ assert len(calls) == 1
+ assert calls[0]["normalization_status"] == "unsupported_native_codex_exec_wrapper"
+ assert _scorer(extractor, "check_activation")(calls, "example")["score"] != 0.75
+
+
[email protected]("extractor", _EXTRACTORS)
+ "source",
+ [
+ 'const r=await tools.exec_command({cmd:"echo a\x85b"});text(r);',
+ '// @exec: {"note":"a\x1cb"}\nconst r=await tools.exec_command({cmd:"pwd"});text(r);',
+ 'const r=await\u00a0tools.exec_command({cmd:"pwd"});text(r);',
+ ],
+ ids=["inside-a-string", "on-the-pragma-line", "valid-javascript-space"],
+)
+def test_wrapper_whose_only_odd_characters_are_valid_javascript_still_decodes(extractor, source):
+ calls = extractor(_trajectory(source, "/workspace"))
+
+ assert [call["action"] for call in calls] == ["exec_command"]
+ assert "normalization_status" not in calls[0]
+
+
[email protected]("extractor", _EXTRACTORS)
+def test_exec_call_that_is_not_a_codex_wrapper_is_left_alone_by_the_space_check(extractor):
+ source = "echo\x1chello"
+ trajectory = _trajectory(source, "hello")
+ calls = extractor(trajectory)
+
+ assert [call["action"] for call in calls] == ["exec"]
+ assert "normalization_status" not in calls[0]
+
+
[email protected]("extractor", _EXTRACTORS)
[email protected]("number", ["1e400", "-1e400", "NaN", "Infinity", "-Infinity"])
+ "source",
+ [
+ 'const r=await tools.exec_command({{cmd:"x",n:{n}}});text(r);',
+ 'const r=await tools.exec_command({{cmd:"x",n:[{n}]}});text(r);',
+ ],
+ ids=["bound", "bound-nested"],
+)
+def test_upstream_grammar_form_with_a_non_finite_number_stays_unsupported(extractor, number, source):
+ # JavaScript serializes these as null, so the decoded inf or nan would not be the call's real argument.
+ calls = extractor(_trajectory(source.format(n=number), "outer observation"))
+
+ assert len(calls) == 1
+ assert calls[0]["normalization_status"] == "unsupported_native_codex_exec_wrapper"
+
+
[email protected]("extractor", _EXTRACTORS)
[email protected]("number", ["1e3", "-0", "1.5", "2.5e-3"])
+def test_upstream_grammar_form_with_a_finite_number_still_decodes(extractor, number):
+ source = f'const r=await tools.exec_command({{cmd:"x",n:{number}}});text(r);'
+ calls = extractor(_trajectory(source, "outer observation"))
+
+ assert [call["action"] for call in calls] == ["exec_command"]
+
+
[email protected]("extractor", _EXTRACTORS)
[email protected]("source", _LONE_SOURCES)
[email protected]("prefix", ["\n", "\n\n", " ", "\t", "\n "], ids=["blank-line", "two-blank-lines", "space", "tab", "blank-and-indent"])
+def test_lone_call_record_after_leading_whitespace_is_graded(extractor, source, prefix):
+ # The header check and the record decoder must agree: whitespace before a complete record is not
+ # an incomplete record.
+ record = '{"exit_code":2,"output":"make: *** Error 2\\n"}'
+ trajectory = _trajectory(source, _RECORD_FRAME[0] + prefix + record)
+ calls = extractor(trajectory)
+
+ assert "normalization_status" not in calls[0]
+ assert calls[0]["is_error"] is True
+ assert _error_recovery(trajectory, extractor)["score"] == 0.0