Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
feat(tier3): rebuild ATIF from OpenCode and Codex agent logs (#119)
Mohammed Imran Khan7d49311db8f2
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 9f9c3ddc..59adb941 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -24,6 +24,17 @@ All notable changes to SkillEvaluator are documented in this file.missing `message` fields on reasoning token exhaustion. The canonical OpenAIRAGAS goal scorer retains its separate scoring path.- Configurable evidence bundle budgets (`SKILL_EVAL_ACCURACY_BUDGET`, `SKILL_EVAL_GOAL_ACCURACY_BUDGET`, `SKILL_EVAL_BEHAVIOR_CHECK_BUDGET`) and final response limit (`SKILL_EVAL_BEHAVIOR_FINAL_RESPONSE_LIMIT`).+- Tier 3 log converters now rebuild ATIF trajectories from OpenCode JSON streams+ (`opencode.txt`) and structured Codex tee logs (`codex.txt`) when+ `trajectory.json` is missing or empty.++### Fixed++- Harbor ``result.json`` case ids now prefer canonical ``task_id.path`` metadata+ over repository-prefixed ``task_name`` values when resolving eval entries.+- Codex log synthesis maps ``web_search`` action payloads and ``collab_tool_call``+ thread items into ATIF, and error-recovery checks recognize ``status=failed`` /+ ``exit_code=`` terminal evidence emitted by Codex converters.## 0.4.0 - 2026-09-30diff --git a/src/skillevaluator/tier3/eval_core/checks.py b/src/skillevaluator/tier3/eval_core/checks.pyindex bdf84a84..4ad180fc 100644--- a/src/skillevaluator/tier3/eval_core/checks.py+++ b/src/skillevaluator/tier3/eval_core/checks.py@@ -5200,8 +5200,8 @@ def check_error_recovery("error","traceback","exception",- "exit code 1",- "exit code 2",+ "status=failed",+ "status=error","not found","command not found","permission denied",@@ -5211,6 +5211,8 @@ def check_error_recovery("connectionrefused","timeout",]+ # Match exit_code=N / "exit code N" for any nonzero N (not just 1/2).+ nonzero_exit_re = re.compile(r"(?:exit_code|exit\s+code)\s*[=:]?\s*(?!0\b)(\d+)", re.IGNORECASE)skill_fault_keywords = ["no such file","filenotfounderror",@@ -5224,6 +5226,8 @@ def check_error_recovery(def _is_failure(tc: dict[str, Any]) -> bool:obs = str(tc.get("observation", "")).lower()+ if nonzero_exit_re.search(obs):+ return Truereturn any(kw in obs for kw in error_keywords)def _cmd_text(tc: dict[str, Any]) -> str:diff --git a/src/skillevaluator/tier3/eval_core/log_converters.py b/src/skillevaluator/tier3/eval_core/log_converters.pyindex 5e29f2e7..4f4812d4 100644--- a/src/skillevaluator/tier3/eval_core/log_converters.py+++ b/src/skillevaluator/tier3/eval_core/log_converters.py@@ -19,6 +19,22 @@ import refrom pathlib import Pathfrom typing import Any+_OBSERVATION_CHAR_LIMIT = 8000+++def _bounded_tool_observation(*, prefix: str, body: str, suffix: str, limit: int = _OBSERVATION_CHAR_LIMIT) -> str:+ """Keep status/error suffixes when tool output fills the observation budget."""+ prefix = prefix.strip()+ suffix = suffix.strip()+ body = body or ""+ fixed_parts = [part for part in (prefix, suffix) if part]+ fixed_len = sum(len(part) for part in fixed_parts) + max(0, len(fixed_parts) - 1)+ body_sep = 1 if body and fixed_parts else 0+ body_limit = max(0, limit - fixed_len - body_sep)+ trimmed_body = body[:body_limit]+ segments = [segment for segment in (prefix, trimmed_body, suffix) if segment]+ return "\n".join(segments)[:limit]+def _stringify_tool_result_content(content: Any) -> str:if content is None:@@ -385,6 +401,607 @@ def synthetic_trajectory_from_cline_cli(text: str) -> dict[str, Any] | None:}+def _normalize_opencode_tool(tool_name: str, raw_input: Any) -> tuple[str, dict[str, Any]]:+ """Map OpenCode tool names and inputs to ATIF tool-call fields."""+ if not isinstance(raw_input, dict):+ return tool_name, {}+ name = str(tool_name or "").strip()+ lowered = name.lower()+ arguments = dict(raw_input)+ if lowered in {"read", "read_file"}:+ function_name = "read"+ if "filePath" in arguments and "path" not in arguments:+ arguments["path"] = arguments["filePath"]+ if "file_path" in arguments and "path" not in arguments:+ arguments["path"] = arguments["file_path"]+ elif lowered in {"bash", "shell"}:+ function_name = "bash"+ elif lowered in {"write", "edit"}:+ function_name = lowered+ if "filePath" in arguments and "path" not in arguments:+ arguments["path"] = arguments["filePath"]+ else:+ function_name = name or lowered or "tool"+ return function_name, arguments+++def _opencode_output_payload(state: dict[str, Any]) -> str:+ output = state.get("output")+ if output is None:+ return ""+ if isinstance(output, str):+ return output+ if isinstance(output, dict):+ message = output.get("message") or output.get("text")+ if message is not None:+ return str(message)+ return json.dumps(output, ensure_ascii=False)+ return str(output)+++def _opencode_error_text(state: dict[str, Any]) -> str:+ error = state.get("error")+ if error is None:+ return ""+ if isinstance(error, str):+ return error+ if isinstance(error, dict):+ data = error.get("data")+ if isinstance(data, dict):+ message = data.get("message")+ if message is not None:+ return str(message)+ message = error.get("message")+ if message is not None:+ return str(message)+ return json.dumps(error, ensure_ascii=False)+ return str(error)+++def _opencode_exit_code(state: dict[str, Any]) -> int | None:+ """Return OpenCode bash ``metadata.exit`` when present."""+ metadata = state.get("metadata")+ if not isinstance(metadata, dict):+ return None+ exit_code = metadata.get("exit")+ if exit_code is None:+ return None+ try:+ return int(exit_code)+ except (TypeError, ValueError):+ return None+++def _opencode_tool_observation(state: dict[str, Any]) -> str:+ """Build a bounded observation that retains status, exit code, and errors."""+ prefix_parts: list[str] = []+ status = state.get("status")+ if status is not None and str(status).strip():+ prefix_parts.append(f"status={status}")+ exit_code = _opencode_exit_code(state)+ if exit_code is not None:+ # Emit exit_code= so Harbor error-recovery matches Codex-shaped evidence.+ prefix_parts.append(f"exit_code={exit_code}")+ return _bounded_tool_observation(+ prefix="\n".join(prefix_parts),+ body=_opencode_output_payload(state),+ suffix=_opencode_error_text(state),+ )+++def synthetic_trajectory_from_opencode_json(text: str) -> dict[str, Any] | None:+ """Parse OpenCode ``run --format=json`` JSONL (tee'd to ``opencode.txt``)."""+ if not text or not text.strip():+ return None++ steps: list[dict[str, Any]] = []+ saw_content = False++ for raw_line in text.splitlines():+ line = raw_line.strip()+ if not line:+ continue+ try:+ evt = json.loads(line)+ except json.JSONDecodeError:+ continue+ if not isinstance(evt, dict):+ continue++ et = str(evt.get("type") or "")+ if et == "text":+ part = evt.get("part")+ if not isinstance(part, dict):+ continue+ msg = str(part.get("text") or "").strip()+ if not msg:+ continue+ saw_content = True+ if steps and not steps[-1].get("tool_calls"):+ prev = (steps[-1].get("message") or "").strip()+ steps[-1]["message"] = (prev + "\n" + msg).strip() if prev else msg+ else:+ steps.append(+ {+ "source": "agent",+ "message": msg,+ "tool_calls": [],+ "observation": {"results": []},+ }+ )+ continue++ if et != "tool_use":+ continue++ part = evt.get("part")+ if not isinstance(part, dict):+ continue+ state = part.get("state")+ if not isinstance(state, dict):+ continue+ status = str(state.get("status") or "").lower()+ if status in {"pending", "running"}:+ continue++ tool_name = str(part.get("tool") or part.get("name") or "")+ call_id = str(part.get("callID") or part.get("call_id") or part.get("id") or "")+ function_name, arguments = _normalize_opencode_tool(tool_name, state.get("input"))+ if not call_id:+ call_id = f"opencode-{len(steps) + 1}"++ saw_content = True+ step = {+ "source": "agent",+ "message": "",+ "tool_calls": [+ {+ "tool_call_id": call_id,+ "function_name": function_name,+ "arguments": arguments,+ }+ ],+ "observation": {"results": []},+ }+ observation_text = _opencode_tool_observation(state)+ if observation_text:+ step["observation"]["results"].append(+ {+ "source_call_id": call_id,+ "content": observation_text,+ }+ )+ steps.append(step)++ if not saw_content or not steps:+ return None+ return {+ "steps": steps,+ "schema_version": "ATIF-v1.2-synthetic-opencode-log",+ "final_metrics": {},+ }+++def _iter_jsonl_dicts(text: str) -> list[dict[str, Any]]:+ """Parse JSONL lines, skipping non-JSON noise (e.g. stderr mixed into tee logs)."""+ events: list[dict[str, Any]] = []+ for raw_line in text.splitlines():+ line = raw_line.strip()+ if not line.startswith("{") or not line.endswith("}"):+ continue+ try:+ evt = json.loads(line)+ except json.JSONDecodeError:+ continue+ if isinstance(evt, dict):+ events.append(evt)+ return events+++def _codex_item_text(item: dict[str, Any]) -> str | None:+ text = item.get("text")+ if isinstance(text, str) and text.strip():+ return text.strip()+ item_type = str(item.get("type") or "")+ if item_type not in {"message", "agent_message"}:+ return None+ content = item.get("content")+ if not isinstance(content, list):+ return None+ parts: list[str] = []+ for block in content:+ if isinstance(block, dict) and block.get("type") == "text":+ parts.append(str(block.get("text") or ""))+ joined = "\n".join(parts).strip()+ return joined or None+++def _codex_function_arguments(function_call: dict[str, Any]) -> dict[str, Any]:+ args = function_call.get("arguments")+ if isinstance(args, dict):+ return dict(args)+ return {}+++def _codex_thread_item(evt: dict[str, Any]) -> dict[str, Any] | None:+ """Return the Codex thread item payload from a completed event."""+ if str(evt.get("type") or "") == "item.completed":+ item = evt.get("item")+ return item if isinstance(item, dict) else None+ if str(evt.get("type") or "") == "item" and evt.get("item.completed", True):+ item = evt.get("item")+ return item if isinstance(item, dict) else None+ return None+++def _codex_status_error_prefix(item: dict[str, Any]) -> str:+ lines: list[str] = []+ status = item.get("status")+ if status is not None and str(status).strip():+ lines.append(f"status={status}")+ exit_code = item.get("exit_code")+ if exit_code is not None:+ lines.append(f"exit_code={exit_code}")+ error = item.get("error")+ if error is not None:+ if isinstance(error, dict):+ message = error.get("message")+ lines.append(str(message) if message is not None else json.dumps(error, ensure_ascii=False))+ else:+ lines.append(str(error))+ return "\n".join(lines)+++def _codex_observation_body(item: dict[str, Any], *, include_result: bool = False) -> str:+ parts: list[str] = []+ for key in ("aggregated_output", "output"):+ output = item.get(key)+ if output is not None and str(output).strip():+ parts.append(str(output))+ break+ if include_result:+ result_text = _codex_mcp_result_text(item)+ if result_text:+ parts.append(result_text)+ return "\n".join(parts)+++def _codex_observation_content(item: dict[str, Any], *, include_result: bool = False) -> str:+ return _bounded_tool_observation(+ prefix=_codex_status_error_prefix(item),+ body=_codex_observation_body(item, include_result=include_result),+ suffix="",+ )+++def _codex_mcp_result_text(item: dict[str, Any]) -> str:+ result = item.get("result")+ if not isinstance(result, dict):+ return ""+ content = result.get("content")+ if isinstance(content, list):+ blocks: list[str] = []+ for block in content:+ if isinstance(block, dict):+ text = block.get("text")+ blocks.append(str(text) if text is not None else json.dumps(block, ensure_ascii=False))+ else:+ blocks.append(str(block))+ joined = "\n".join(part for part in blocks if part).strip()+ if joined:+ return joined+ structured = result.get("structured_content")+ if structured is not None:+ return json.dumps(structured, ensure_ascii=False)+ return json.dumps(result, ensure_ascii=False)+++def _codex_mcp_observation(item: dict[str, Any]) -> str:+ return _codex_observation_content(item, include_result=True)+++def _codex_web_search_arguments(item: dict[str, Any]) -> dict[str, Any]:+ """Map Codex ``WebSearchItem`` wire fields into ATIF tool arguments.++ Schema shape (``codex exec --json``): required top-level ``query`` plus an+ ``action`` object (``search`` with optional ``query``/``queries``,+ ``open_page``, or ``find_in_page``). Preserve the full action payload so+ multi-query and navigation evidence is not collapsed to a summary string.+ """+ arguments: dict[str, Any] = {}+ if "query" in item and item.get("query") is not None:+ arguments["query"] = str(item["query"])+ action = item.get("action")+ if isinstance(action, dict):+ arguments["action"] = dict(action)+ return arguments+ legacy = str(item.get("input") or "").strip()+ if legacy and "query" not in arguments:+ arguments["query"] = legacy+ return arguments+++def _codex_collab_tool_arguments(item: dict[str, Any]) -> dict[str, Any]:+ """Map Codex ``CollabToolCallItem`` wire fields into ATIF tool arguments.++ Schema fields: ``sender_thread_id``, ``receiver_thread_ids``, ``prompt``,+ ``agents_states``, and ``status`` (``tool`` becomes the function name).+ """+ arguments: dict[str, Any] = {}+ for key in (+ "sender_thread_id",+ "receiver_thread_ids",+ "prompt",+ "agents_states",+ "status",+ ):+ if key in item and item[key] is not None:+ arguments[key] = item[key]+ return arguments+++def _codex_tool_step(+ *,+ item: dict[str, Any],+ evt: dict[str, Any],+ tool_index: int,+ function_name: str,+ arguments: dict[str, Any],+ observation: str,+) -> dict[str, Any]:+ call_id = str(item.get("id") or evt.get("item_id") or f"codex-{tool_index + 1}")+ step: dict[str, Any] = {+ "source": "agent",+ "message": "",+ "tool_calls": [+ {+ "tool_call_id": call_id,+ "function_name": function_name,+ "arguments": arguments,+ }+ ],+ "observation": {"results": []},+ }+ if observation:+ step["observation"]["results"].append(+ {+ "source_call_id": call_id,+ "content": observation,+ }+ )+ return step+++def _codex_file_change_step(item: dict[str, Any], evt: dict[str, Any], tool_index: int) -> dict[str, Any] | None:+ changes = item.get("changes")+ if not isinstance(changes, list):+ return None+ tool_calls: list[dict[str, Any]] = []+ base_id = str(item.get("id") or evt.get("item_id") or f"codex-{tool_index + 1}")+ for idx, change in enumerate(changes):+ if not isinstance(change, dict):+ continue+ path = str(change.get("path") or "").strip()+ if not path:+ continue+ tool_calls.append(+ {+ "tool_call_id": f"{base_id}-{idx}",+ "function_name": "write",+ "arguments": {"path": path, "kind": str(change.get("kind") or "update")},+ }+ )+ if not tool_calls:+ return None+ step: dict[str, Any] = {+ "source": "agent",+ "message": "",+ "tool_calls": tool_calls,+ "observation": {"results": []},+ }+ evidence = _codex_observation_content(item)+ if evidence:+ for tool_call in tool_calls:+ step["observation"]["results"].append(+ {+ "source_call_id": tool_call["tool_call_id"],+ "content": evidence,+ }+ )+ return step+++def synthetic_trajectory_from_codex_json(text: str) -> dict[str, Any] | None:+ """Parse Codex ``exec --json`` ThreadEvent JSONL."""+ if not text or not text.strip():+ return None++ steps: list[dict[str, Any]] = []+ saw_content = False+ tool_index = 0++ for evt in _iter_jsonl_dicts(text):+ item = _codex_thread_item(evt)+ if item is None:+ continue++ item_type = str(item.get("type") or "")++ if item_type in {"agent_message", "message"}:+ message = _codex_item_text(item)+ function_call = item.get("function_call")+ if not message and not isinstance(function_call, dict):+ continue+ saw_content = True+ step: dict[str, Any] = {+ "source": "agent",+ "message": message or "",+ "tool_calls": [],+ "observation": {"results": []},+ }+ if isinstance(function_call, dict):+ call_id = str(item.get("id") or evt.get("item_id") or f"codex-{tool_index + 1}")+ function_name = str(function_call.get("name") or "tool")+ if function_name.lower() in {"bash", "shell"}:+ function_name = "bash"+ arguments = _codex_function_arguments(function_call)+ step["tool_calls"] = [+ {+ "tool_call_id": call_id,+ "function_name": function_name,+ "arguments": arguments,+ }+ ]+ output = item.get("output")+ if output is not None:+ step["observation"]["results"].append(+ {+ "source_call_id": call_id,+ "content": str(output)[:8000],+ }+ )+ tool_index += 1+ steps.append(step)+ continue++ if item_type == "command_execution":+ command_raw = item.get("command")+ command = "" if command_raw is None else str(command_raw).strip()+ evidence = _codex_observation_content(item)+ if not command and not evidence:+ continue+ saw_content = True+ call_id = str(item.get("id") or evt.get("item_id") or f"codex-{tool_index + 1}")+ tool_index += 1+ step = {+ "source": "agent",+ "message": "",+ "tool_calls": [+ {+ "tool_call_id": call_id,+ "function_name": "bash",+ "arguments": {"command": command},+ }+ ],+ "observation": {"results": []},+ }+ if evidence:+ step["observation"]["results"].append(+ {+ "source_call_id": call_id,+ "content": evidence,+ }+ )+ steps.append(step)+ continue++ if item_type == "file_change":+ step = _codex_file_change_step(item, evt, tool_index)+ if step is None:+ continue+ saw_content = True+ tool_index += 1+ steps.append(step)+ continue++ if item_type == "mcp_tool_call":+ tool = str(item.get("tool") or "mcp_tool")+ arguments = item.get("arguments")+ if not isinstance(arguments, dict):+ arguments = {}+ call_id = str(item.get("id") or evt.get("item_id") or f"codex-{tool_index + 1}")+ saw_content = True+ tool_index += 1+ step = {+ "source": "agent",+ "message": "",+ "tool_calls": [+ {+ "tool_call_id": call_id,+ "function_name": tool,+ "arguments": dict(arguments),+ }+ ],+ "observation": {"results": []},+ }+ observation = _codex_mcp_observation(item)+ if observation:+ step["observation"]["results"].append(+ {+ "source_call_id": call_id,+ "content": observation,+ }+ )+ steps.append(step)+ continue++ if item_type in {"web_search", "web_search_call"}:+ arguments = _codex_web_search_arguments(item)+ observation = _codex_observation_content(item, include_result=True)+ action = item.get("action")+ if isinstance(action, dict) and not observation:+ observation = json.dumps(action, ensure_ascii=False)+ saw_content = True+ tool_index += 1+ steps.append(+ _codex_tool_step(+ item=item,+ evt=evt,+ tool_index=tool_index,+ function_name="web_search",+ arguments=arguments,+ observation=observation,+ )+ )+ continue++ if item_type == "collab_tool_call":+ tool = str(item.get("tool") or "collab_tool_call")+ arguments = _codex_collab_tool_arguments(item)+ observation = _codex_observation_content(item, include_result=True)+ saw_content = True+ tool_index += 1+ steps.append(+ _codex_tool_step(+ item=item,+ evt=evt,+ tool_index=tool_index,+ function_name=tool,+ arguments=arguments,+ observation=observation,+ )+ )+ continue++ if not saw_content or not steps:+ return None+ return {+ "steps": steps,+ "schema_version": "ATIF-v1.2-synthetic-codex-log",+ "final_metrics": {},+ }+++def synthetic_trajectory_from_codex_txt(text: str) -> dict[str, Any] | None:+ """Reconstruct ATIF from Codex tee logs (``exec --json`` JSONL or OpenCode-shaped JSONL)."""+ if not text or not text.strip():+ return None++ synth = synthetic_trajectory_from_codex_json(text)+ if synth and synth.get("steps"):+ return synth++ lines = [line.strip() for line in text.splitlines() if line.strip()]+ if not lines:+ return None+ if all(line.startswith("{") and line.endswith("}") for line in lines):+ synth = synthetic_trajectory_from_opencode_json(text)+ if synth and synth.get("steps"):+ synth["schema_version"] = "ATIF-v1.2-synthetic-codex-log"+ return synth+ return None++def load_trajectory_with_fallback(trajectory_path: Path,logs_dir: Path | None = None,@@ -436,4 +1053,22 @@ def load_trajectory_with_fallback(meta["note"] = "Synthetic ATIF from Cline CLI JSONL log"return synth, meta+ opencode_path = logs / "opencode.txt"+ if opencode_path.exists():+ raw = opencode_path.read_text(encoding="utf-8", errors="replace")+ synth = synthetic_trajectory_from_opencode_json(raw)+ if synth and synth.get("steps"):+ meta["source"] = "opencode.txt"+ meta["note"] = "Synthetic ATIF from OpenCode JSON stream"+ return synth, meta++ codex_path = logs / "codex.txt"+ if codex_path.exists():+ raw = codex_path.read_text(encoding="utf-8", errors="replace")+ synth = synthetic_trajectory_from_codex_txt(raw)+ if synth and synth.get("steps"):+ meta["source"] = "codex.txt"+ meta["note"] = "Synthetic ATIF from Codex structured log"+ return synth, meta+return None, metadiff --git a/src/skillevaluator/tier3/generate_dataset.py b/src/skillevaluator/tier3/generate_dataset.pyindex 809d81a9..10992ec9 100644--- a/src/skillevaluator/tier3/generate_dataset.py+++ b/src/skillevaluator/tier3/generate_dataset.py@@ -647,7 +647,16 @@ def _discover_trajectories(if not results_dir.exists():return {}- agent_priority = ["claude-code", "cursor-cli", "codex", "openhands", "mini-swe-agent", "aider", "gemini-cli"]+ agent_priority = [+ "claude-code",+ "cursor-cli",+ "opencode",+ "codex",+ "openhands",+ "mini-swe-agent",+ "aider",+ "gemini-cli",+ ]for agent_name in agent_priority:trials_dir = results_dir / agent_name / "with-skill" / "trials"if not trials_dir.exists():diff --git a/src/skillevaluator/tier3/harbor/collector.py b/src/skillevaluator/tier3/harbor/collector.pyindex a5a6985e..0a4b9b63 100644--- a/src/skillevaluator/tier3/harbor/collector.py+++ b/src/skillevaluator/tier3/harbor/collector.py@@ -1766,10 +1766,6 @@ def _harbor_result_rewards(result: dict[str, Any]) -> dict[str, Any] | None:def _entry_id_from_harbor_result(result: dict[str, Any]) -> str:- task_name = result.get("task_name")- if isinstance(task_name, str) and task_name.strip():- return task_name.strip().rsplit("/", 1)[-1]-task_id = result.get("task_id")if isinstance(task_id, dict):task_path = task_id.get("path")@@ -1784,6 +1780,10 @@ def _entry_id_from_harbor_result(result: dict[str, Any]) -> str:if isinstance(task_path, str) and task_path.strip():return Path(task_path).name+ task_name = result.get("task_name")+ if isinstance(task_name, str) and task_name.strip():+ return task_name.strip().rsplit("/", 1)[-1]+return ""diff --git a/src/skillevaluator/tier3/harbor/templates/eval.py b/src/skillevaluator/tier3/harbor/templates/eval.pyindex 9128cfdf..295e4ba9 100644--- a/src/skillevaluator/tier3/harbor/templates/eval.py+++ b/src/skillevaluator/tier3/harbor/templates/eval.py@@ -7765,8 +7765,8 @@ def check_error_recovery(tool_calls, expected_script=None):"error","traceback","exception",- "exit code 1",- "exit code 2",+ "status=failed",+ "status=error","not found","command not found","permission denied",@@ -7774,6 +7774,8 @@ def check_error_recovery(tool_calls, expected_script=None):"filenotfounderror","modulenotfounderror",]+ # Match exit_code=N / "exit code N" for any nonzero N (not just 1/2).+ nonzero_exit_re = re.compile(r"(?:exit_code|exit\s+code)\s*[=:]?\s*(?!0\b)(\d+)", re.IGNORECASE)skill_fault_kw = ["no such file","filenotfounderror",@@ -7787,6 +7789,8 @@ def check_error_recovery(tool_calls, expected_script=None):def _is_failure(tc):obs = str(tc.get("observation", "")).lower()+ if nonzero_exit_re.search(obs):+ return Truereturn any(kw in obs for kw in error_kw)def _cmd_text(tc):diff --git a/tests/tier3/test_generate_dataset_results.py b/tests/tier3/test_generate_dataset_results.pyindex 283aad82..2e21d77b 100644--- a/tests/tier3/test_generate_dataset_results.py+++ b/tests/tier3/test_generate_dataset_results.py@@ -40,6 +40,58 @@ def test_discover_trajectories_uses_env_results_root(tmp_path, monkeypatch):assert _discover_trajectories(skill) == {"case-001": trajectory}+def test_discover_trajectories_opencode_txt_fallback(tmp_path):+ skill = tmp_path / "my-skill"+ skill.mkdir()+ results_root = tmp_path / "results"+ skill_results = results_root / "my-skill"+ run_id = "20260709_120000"+ run_dir = skill_results / run_id+ trial = run_dir / "opencode" / "with-skill" / "trials" / "case-001"+ trial.mkdir(parents=True)+ (trial / "opencode.txt").write_text(+ '{"type":"tool_use","part":{"type":"tool","tool":"bash","callID":"c1",'+ '"state":{"status":"completed","input":{"command":"echo hi"},"output":"hi"}}}\n',+ encoding="utf-8",+ )+ (run_dir / "run_config.json").write_text("{}", encoding="utf-8")+ (run_dir / "result.json").write_text(json.dumps({"run_id": run_id}), encoding="utf-8")+ (skill_results / "latest").symlink_to(run_id)++ trajectories = _discover_trajectories(skill, results_dir=results_root)+ assert "case-001" in trajectories+ assert trajectories["case-001"].get("steps")+++def test_discover_trajectories_opencode_maps_harbor_folder_via_result_metadata(tmp_path):+ skill = tmp_path / "demo"+ skill.mkdir()+ results_root = tmp_path / "results"+ run_id = "20260709_120000"+ run_dir = results_root / "demo" / run_id+ trial_folder = "case-001__LRZctSP"+ trial = run_dir / "opencode" / "with-skill" / "trials" / trial_folder+ trial.mkdir(parents=True)+ trajectory = {"steps": [{"tool_calls": [{"tool": "Read"}]}]}+ trial.joinpath("trajectory.json").write_text(json.dumps(trajectory), encoding="utf-8")+ trial.joinpath("result.json").write_text(+ json.dumps(+ {+ "trial_name": trial_folder,+ "config": {"task": {"path": "tasks/case-001"}},+ }+ ),+ encoding="utf-8",+ )+ (run_dir / "run_config.json").write_text("{}", encoding="utf-8")+ (run_dir / "result.json").write_text(json.dumps({"run_id": run_id}), encoding="utf-8")+ (results_root / "demo" / "latest").symlink_to(run_id)++ found = _discover_trajectories(skill, results_dir=results_root)+ assert list(found) == ["case-001"]+ assert found["case-001"] == trajectory++def test_discover_trajectories_maps_harbor_trial_folder_to_case_id(tmp_path, monkeypatch):"""Harbor persists trials as ``{case_id}__{suffix}``; refine looks up by case id."""skill = tmp_path / "demo"diff --git a/tests/tier3/test_harbor_result_entry_id.py b/tests/tier3/test_harbor_result_entry_id.pynew file mode 100644index 00000000..f3abb251--- /dev/null+++ b/tests/tier3/test_harbor_result_entry_id.py@@ -0,0 +1,14 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++from __future__ import annotations++from skillevaluator.tier3.harbor.collector import _entry_id_from_harbor_result+++def test_entry_id_prefers_task_id_path_over_prefixed_task_name() -> None:+ result = {+ "task_name": "nvidia/skillevaluator-apt-pinning",+ "task_id": {"path": "evals/cases/case-014-apt-pinning"},+ }+ assert _entry_id_from_harbor_result(result) == "case-014-apt-pinning"diff --git a/tests/tier3/test_log_converters.py b/tests/tier3/test_log_converters.pyindex b4d0c943..843ecdb3 100644--- a/tests/tier3/test_log_converters.py+++ b/tests/tier3/test_log_converters.py@@ -9,10 +9,13 @@ import jsonfrom pathlib import Pathfrom skillevaluator.tier3.eval_core.atif_helpers import extract_tool_calls_as_dicts+from skillevaluator.tier3.eval_core.checks import check_error_recoveryfrom skillevaluator.tier3.eval_core.log_converters import (load_trajectory_with_fallback,synthetic_trajectory_from_claude_stream_jsonl,+ synthetic_trajectory_from_codex_txt,synthetic_trajectory_from_cursor_cli,+ synthetic_trajectory_from_opencode_json,)@@ -89,3 +92,353 @@ def test_load_prefers_claude_log_over_cursor_when_both(tmp_path: Path):assert meta["source"] == "claude-code.txt"tcs = extract_tool_calls_as_dicts(data)assert any(tc["action"] == "Bash" for tc in tcs)+++def test_opencode_json_tool_use_and_read():+ log = (+ '{"type":"text","part":{"type":"text","text":"Reading skill"}}\n'+ '{"type":"tool_use","part":{"type":"tool","tool":"read","callID":"call-1",'+ '"state":{"status":"completed","input":{"filePath":"/workspace/skills/calculator/SKILL.md"},'+ '"output":"# Calculator skill"}}}\n'+ '{"type":"tool_use","part":{"type":"tool","tool":"bash","callID":"call-2",'+ '"state":{"status":"completed","input":{"command":"cat skills/foo/SKILL.md"},'+ '"output":"ok","metadata":{"exit":0}}}}\n'+ )+ traj = synthetic_trajectory_from_opencode_json(log)+ assert traj is not None+ tcs = extract_tool_calls_as_dicts(traj)+ assert len(tcs) == 2+ read_call = next(tc for tc in tcs if tc["action"] == "read")+ assert "/calculator/" in json.dumps(read_call["action_input"])+ assert "Calculator" in read_call["observation"]+ bash_call = next(tc for tc in tcs if tc["action"] == "bash")+ assert "cat" in json.dumps(bash_call["action_input"]).lower()+ assert "exit_code=0" in bash_call["observation"]+++def test_opencode_error_only_log_returns_none():+ log = json.dumps(+ {+ "type": "error",+ "error": {+ "name": "UnknownError",+ "data": {"message": "ResourceExhausted: Worker local total request limit reached"},+ },+ }+ )+ assert synthetic_trajectory_from_opencode_json(log + "\n") is None+++def test_opencode_tool_error_uses_state_error_observation():+ log = (+ '{"type":"tool_use","part":{"type":"tool","tool":"bash","callID":"call-err",'+ '"state":{"status":"error","input":{"command":"missing-cmd"},'+ '"error":"command not found: missing-cmd"}}}\n'+ )+ traj = synthetic_trajectory_from_opencode_json(log)+ assert traj is not None+ tcs = extract_tool_calls_as_dicts(traj)+ assert len(tcs) == 1+ assert "missing-cmd" in tcs[0]["observation"]+++def test_codex_thread_event_agent_message_and_command_execution():+ log = (+ '{"type":"item","item":{"type":"agent_message","id":"msg-1",'+ '"content":[{"type":"text","text":"Reading skill file"}],'+ '"function_call":{"name":"shell","arguments":{"command":"ls"}},'+ '"output":"demo"},"item.completed":true}\n'+ '{"type":"item","item":{"type":"agent_message","id":"msg-2",'+ '"content":[{"type":"text","text":"Done"}]},'+ '"item.completed":true}\n'+ )+ traj = synthetic_trajectory_from_codex_txt(log)+ assert traj is not None+ assert traj["schema_version"] == "ATIF-v1.2-synthetic-codex-log"+ tcs = extract_tool_calls_as_dicts(traj)+ assert len(tcs) == 1+ assert tcs[0]["action"] == "bash"+++def test_codex_exec_json_tolerates_stderr_noise():+ log = (+ "ERROR responses_websocket: HTTP error: 405 Method Not Allowed\n"+ '{"type":"item","item":{"type":"agent_message","id":"msg-1",'+ '"content":[{"type":"text","text":"hello"}]},'+ '"item.completed":true}\n'+ )+ traj = synthetic_trajectory_from_codex_txt(log)+ assert traj is not None+ assert len(traj["steps"]) == 1+++def test_codex_opencode_shaped_jsonl_fallback():+ log = (+ '{"type":"tool_use","part":{"type":"tool","tool":"read","callID":"c1",'+ '"state":{"status":"completed","input":{"path":"/workspace/skills/demo/SKILL.md"},'+ '"output":"# Demo"}}}\n'+ )+ traj = synthetic_trajectory_from_codex_txt(log)+ assert traj is not None+ assert traj["schema_version"] == "ATIF-v1.2-synthetic-codex-log"+ tcs = extract_tool_calls_as_dicts(traj)+ assert len(tcs) == 1+ assert tcs[0]["action"] == "read"+++def test_codex_file_change_emits_write_calls():+ log = (+ '{"type":"item.completed","item":{"type":"file_change","id":"fc-1",'+ '"changes":[{"path":"/workspace/output/real.py","kind":"add"}],'+ '"status":"completed"}}\n'+ )+ traj = synthetic_trajectory_from_codex_txt(log)+ assert traj is not None+ tcs = extract_tool_calls_as_dicts(traj)+ assert len(tcs) == 1+ assert tcs[0]["action"] == "write"+ assert tcs[0]["action_input"]["path"] == "/workspace/output/real.py"+++def test_codex_web_search_multi_query_preserves_action_payload():+ # Schema-valid WebSearchItem: required top-level query, action.search with queries.+ log = (+ '{"type":"item.completed","item":{"type":"web_search","id":"ws-1",'+ '"query":"parallel query batch",'+ '"action":{"type":"search","queries":["alpha","beta"]}}}\n'+ )+ traj = synthetic_trajectory_from_codex_txt(log)+ assert traj is not None+ tcs = extract_tool_calls_as_dicts(traj)+ assert len(tcs) == 1+ assert tcs[0]["action"] == "web_search"+ assert tcs[0]["action_input"] == {+ "query": "parallel query batch",+ "action": {"type": "search", "queries": ["alpha", "beta"]},+ }+++def test_codex_web_search_open_page_action_preserves_url():+ log = (+ '{"type":"item.completed","item":{"type":"web_search","id":"ws-2",'+ '"query":"https://example.com/docs",'+ '"action":{"type":"open_page","url":"https://example.com/docs"}}}\n'+ )+ traj = synthetic_trajectory_from_codex_txt(log)+ assert traj is not None+ tcs = extract_tool_calls_as_dicts(traj)+ assert tcs[0]["action_input"] == {+ "query": "https://example.com/docs",+ "action": {"type": "open_page", "url": "https://example.com/docs"},+ }+++def test_codex_web_search_find_in_page_preserves_pattern():+ log = (+ '{"type":"item.completed","item":{"type":"web_search","id":"ws-3",'+ '"query":"\'install\' in https://example.com/docs",'+ '"action":{"type":"find_in_page","url":"https://example.com/docs","pattern":"install"}}}\n'+ )+ traj = synthetic_trajectory_from_codex_txt(log)+ assert traj is not None+ tcs = extract_tool_calls_as_dicts(traj)+ assert tcs[0]["action_input"]["action"] == {+ "type": "find_in_page",+ "url": "https://example.com/docs",+ "pattern": "install",+ }+++def test_codex_collab_tool_call_spawn_agent_preserves_wire_fields():+ log = (+ '{"type":"item.completed","item":{"type":"collab_tool_call","id":"col-1",'+ '"tool":"spawn_agent","sender_thread_id":"thread-parent",'+ '"receiver_thread_ids":["thread-child"],"prompt":"review logs",'+ '"agents_states":{"thread-child":{"status":"completed","message":null}},'+ '"status":"completed"}}\n'+ )+ traj = synthetic_trajectory_from_codex_txt(log)+ assert traj is not None+ tcs = extract_tool_calls_as_dicts(traj)+ assert len(tcs) == 1+ assert tcs[0]["action"] == "spawn_agent"+ assert tcs[0]["action_input"] == {+ "sender_thread_id": "thread-parent",+ "receiver_thread_ids": ["thread-child"],+ "prompt": "review logs",+ "agents_states": {"thread-child": {"status": "completed", "message": None}},+ "status": "completed",+ }+ assert "status=completed" in tcs[0]["observation"]+++def test_codex_mcp_tool_call_preserves_result():+ log = (+ '{"type":"item.completed","item":{"type":"mcp_tool_call","id":"mcp-1",'+ '"server":"demo","tool":"lookup","arguments":{"query":"skill"},'+ '"result":{"content":[{"type":"text","text":"found skill"}]},'+ '"status":"completed"}}\n'+ )+ traj = synthetic_trajectory_from_codex_txt(log)+ assert traj is not None+ tcs = extract_tool_calls_as_dicts(traj)+ assert len(tcs) == 1+ assert tcs[0]["action"] == "lookup"+ assert "found skill" in tcs[0]["observation"]+++def test_error_recovery_detects_codex_status_failed_observations():+ log = (+ '{"type":"item.completed","item":{"type":"command_execution","id":"cmd-1",'+ '"command":"python run.py","exit_code":1,"status":"failed","aggregated_output":""}}\n'+ '{"type":"item.completed","item":{"type":"command_execution","id":"cmd-2",'+ '"command":"python run.py","exit_code":0,"status":"completed","aggregated_output":"ok"}}\n'+ )+ traj = synthetic_trajectory_from_codex_txt(log)+ assert traj is not None+ result = check_error_recovery(extract_tool_calls_as_dicts(traj))+ assert result["first_attempt_clean"] is False+ assert result["score"] < 1.0+++def test_error_recovery_detects_opencode_completed_nonzero_exit():+ # OpenCode can mark bash as status=completed while metadata.exit is nonzero.+ log = (+ '{"type":"tool_use","part":{"type":"tool","tool":"bash","callID":"call-fail",'+ '"state":{"status":"completed","input":{"command":"python run.py"},'+ '"output":"","metadata":{"exit":1}}}}\n'+ '{"type":"tool_use","part":{"type":"tool","tool":"bash","callID":"call-ok",'+ '"state":{"status":"completed","input":{"command":"python run.py"},'+ '"output":"ok","metadata":{"exit":0}}}}\n'+ )+ traj = synthetic_trajectory_from_opencode_json(log)+ assert traj is not None+ tcs = extract_tool_calls_as_dicts(traj)+ assert "exit_code=1" in tcs[0]["observation"]+ assert "status=completed" in tcs[0]["observation"]+ assert "exit_code=0" in tcs[1]["observation"]+ result = check_error_recovery(tcs)+ assert result["first_attempt_clean"] is False+ assert result["score"] < 1.0+ assert "All commands succeeded on first attempt" not in result["reason"]+++def test_codex_command_execution_preserves_failed_terminal_evidence():+ log = (+ '{"type":"item.completed","item":{"type":"command_execution","id":"cmd-1",'+ '"command":false,"exit_code":1,"status":"failed","aggregated_output":""}}\n'+ )+ traj = synthetic_trajectory_from_codex_txt(log)+ assert traj is not None+ tcs = extract_tool_calls_as_dicts(traj)+ assert len(tcs) == 1+ obs = tcs[0]["observation"]+ assert "status=failed" in obs+ assert "exit_code=1" in obs+++def test_codex_file_change_preserves_status_and_error():+ log = (+ '{"type":"item.completed","item":{"type":"file_change","id":"fc-1",'+ '"changes":[{"path":"/workspace/output/real.py","kind":"add"}],'+ '"status":"failed","error":{"message":"disk full"}}}\n'+ )+ traj = synthetic_trajectory_from_codex_txt(log)+ assert traj is not None+ tcs = extract_tool_calls_as_dicts(traj)+ assert len(tcs) == 1+ obs = tcs[0]["observation"]+ assert "status=failed" in obs+ assert "disk full" in obs+++def test_codex_mcp_tool_call_string_error():+ log = (+ '{"type":"item.completed","item":{"type":"mcp_tool_call","id":"mcp-2",'+ '"tool":"lookup","arguments":{},"error":"server unavailable","status":"failed"}}\n'+ )+ traj = synthetic_trajectory_from_codex_txt(log)+ assert traj is not None+ tcs = extract_tool_calls_as_dicts(traj)+ assert len(tcs) == 1+ obs = tcs[0]["observation"]+ assert "server unavailable" in obs+ assert "status=failed" in obs+++def test_opencode_tool_error_retained_when_output_fills_budget():+ long_output = "o" * 9000+ log = (+ '{"type":"tool_use","part":{"type":"tool","tool":"bash","callID":"c-long",'+ f'"state":{{"status":"error","output":"{long_output}",'+ '"error":{"message":"command aborted"}}}}\n'+ )+ traj = synthetic_trajectory_from_opencode_json(log)+ assert traj is not None+ tcs = extract_tool_calls_as_dicts(traj)+ obs = tcs[0]["observation"]+ assert "status=error" in obs+ assert "command aborted" in obs+ assert len(obs) <= 8000+++def test_codex_multi_file_change_terminal_evidence_on_each_write():+ log = (+ '{"type":"item.completed","item":{"type":"file_change","id":"fc-2",'+ '"changes":[{"path":"/workspace/a.py","kind":"add"},'+ '{"path":"/workspace/b.py","kind":"add"}],'+ '"status":"failed","error":{"message":"disk full"}}}\n'+ )+ traj = synthetic_trajectory_from_codex_txt(log)+ assert traj is not None+ tcs = extract_tool_calls_as_dicts(traj)+ assert len(tcs) == 2+ for call in tcs:+ assert "status=failed" in call["observation"]+ assert "disk full" in call["observation"]+++def test_opencode_tool_error_retains_status_with_partial_output():+ log = (+ '{"type":"tool_use","part":{"type":"tool","tool":"bash","callID":"c-err",'+ '"state":{"status":"error","output":"partial log line",'+ '"error":{"message":"command aborted"}}}}\n'+ )+ traj = synthetic_trajectory_from_opencode_json(log)+ assert traj is not None+ tcs = extract_tool_calls_as_dicts(traj)+ assert len(tcs) == 1+ obs = tcs[0]["observation"]+ assert "status=error" in obs+ assert "partial log line" in obs+ assert "command aborted" in obs+++def test_codex_plain_text_errors_return_none():+ text = "ERROR responses_websocket: HTTP error: 405 Method Not Allowed\n"+ assert synthetic_trajectory_from_codex_txt(text) is None+++def test_load_falls_back_to_opencode_txt(tmp_path: Path):+ logs = tmp_path / "agent"+ logs.mkdir()+ (logs / "opencode.txt").write_text(+ '{"type":"tool_use","part":{"type":"tool","tool":"bash","callID":"c1",'+ '"state":{"status":"completed","input":{"command":"echo hi"},"output":"hi"}}}\n',+ encoding="utf-8",+ )+ traj_path = logs / "trajectory.json"+ data, meta = load_trajectory_with_fallback(traj_path, logs)+ assert data is not None+ assert meta["source"] == "opencode.txt"+++def test_opencode_non_dict_tool_input_is_not_treated_as_shell_command():+ log = (+ '{"type":"tool_use","part":{"type":"tool","tool":"bash","callID":"c1",'+ '"state":{"status":"completed","input":"cat /etc/passwd","output":"blocked"}}}\n'+ )+ traj = synthetic_trajectory_from_opencode_json(log)+ assert traj is not None+ tcs = extract_tool_calls_as_dicts(traj)+ assert tcs[0]["action_input"] == {}