Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Bounded harness evidence readers for the private runtime observer."""
from __future__ import annotations
import json
import os
import re
import stat
from collections.abc import Mapping
from dataclasses import dataclass
from itertools import islice
from pathlib import Path
from typing import Any
from skillevaluator.tier3.harbor.collector import (
_materialized_trial_trajectory,
_read_bounded_text,
_safe_trial_path_component,
)
from skillevaluator.utils.secure_fs import stat_is_link_or_reparse
def _plugin_signal_trajectory(trial_root: Path) -> dict[str, Any] | None:
"""Reuse the collector's bounded, topology-authorized continuation/child graph."""
trajectory, _reason = _materialized_trial_trajectory(trial_root, None)
return trajectory
_CODEX_LOG_MAX_BYTES = 16 * 1024 * 1024
_CODEX_SESSION_FILES = 8
_CODEX_SESSION_WALK_ENTRIES = 4_096
_CODEX_MCP_CALLS = 4_000
_CODEX_SERVER_NAME_RE = re.compile(r"[^\x00-\x1f\x7f-\x9f]{1,128}")
def _codex_server(value: Any) -> str | None:
if not isinstance(value, str):
return None
server = value.removeprefix("mcp__").removesuffix("__")
return server if _CODEX_SERVER_NAME_RE.fullmatch(server) else None
def _codex_json_lines(path: Path) -> list[Any]:
"""Parsed JSON lines that mention MCP (bounded, no-follow read; unparsable lines are skipped)."""
text = _read_bounded_text(path, max_bytes=_CODEX_LOG_MAX_BYTES)
if not text or "mcp" not in text.casefold():
return []
events: list[Any] = []
for line in text.splitlines():
if "mcp" not in line.casefold():
continue
try:
events.append(json.loads(line))
except (ValueError, RecursionError):
continue
return events
def _is_real_directory(path: Path) -> bool:
try:
metadata = path.lstat()
except OSError:
return False
return not stat_is_link_or_reparse(metadata) and stat.S_ISDIR(metadata.st_mode)
def _codex_session_files(sessions: Path) -> list[Path]:
"""Return the first Codex session logs (``*.jsonl``) under ``<trial>/agent/sessions``.
The tree is agent-writable, so the walk never follows a link or reparse
point and lists at most ``_CODEX_SESSION_WALK_ENTRIES`` entries. It visits
names in sorted order, a directory's files before its subdirectories. A
directory with more entries than the budget has left ends the walk, so the
logs found never depend on the order the filesystem lists names in.
"""
if not (_is_real_directory(sessions.parent) and _is_real_directory(sessions)):
return []
found: list[Path] = []
budget = _CODEX_SESSION_WALK_ENTRIES
pending = [sessions]
while pending and budget > 0 and len(found) < _CODEX_SESSION_FILES:
directory = pending.pop()
try:
with os.scandir(directory) as iterator:
# One entry past the budget shows that the directory does not fit in it.
entries = list(islice(iterator, budget + 1))
except OSError:
continue
if len(entries) > budget:
break
budget -= len(entries)
subdirectories: list[Path] = []
for entry in sorted(entries, key=lambda entry: entry.name):
try:
metadata = entry.stat(follow_symlinks=False)
except OSError:
continue
if stat_is_link_or_reparse(metadata):
continue
if stat.S_ISDIR(metadata.st_mode):
subdirectories.append(Path(entry.path))
elif stat.S_ISREG(metadata.st_mode) and entry.name.endswith(".jsonl"):
found.append(Path(entry.path))
pending.extend(reversed(subdirectories))
return found[:_CODEX_SESSION_FILES]
_CODEX_ARGS_KEY_CHARS = 4_096
def _codex_args_key(value: Any) -> str | None:
"""Canonical JSON text of a Codex call's arguments (a JSON string or an object), bounded."""
if isinstance(value, str):
try:
value = json.loads(value)
except (ValueError, RecursionError):
return value if len(value) <= _CODEX_ARGS_KEY_CHARS else None
try:
text = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
except (TypeError, ValueError, RecursionError):
return None
return text if len(text) <= _CODEX_ARGS_KEY_CHARS else None
def _codex_item_status(item: Mapping[str, Any]) -> str | None:
"""``failed`` or ``completed`` for a finished ``mcp_tool_call`` item, else ``None``."""
error = item.get("error")
result = item.get("result")
flagged = isinstance(result, Mapping) and any(result.get(key) is True for key in ("isError", "is_error"))
if error not in (None, "", {}, []) or flagged:
return "failed"
status = item.get("status")
if isinstance(status, str) and status.strip().casefold() in {"failed", "completed"}:
return status.strip().casefold()
return None
def _codex_txt_items(agent_dir: Path) -> list[dict[str, Any]]:
"""``mcp_tool_call`` items of ``codex.txt`` in start order: server, tool, arguments key, status.
``codex.txt`` item ids (``item_N``) are not tool call ids, so callers pair
an item with a call by server, tool, and arguments.
"""
items: dict[str, dict[str, Any]] = {}
for event in _codex_json_lines(agent_dir / "codex.txt"):
event_type = event.get("type") if isinstance(event, Mapping) else None
if not isinstance(event_type, str) or event_type not in {"item.started", "item.completed"}:
continue
item = event.get("item")
if not isinstance(item, Mapping) or item.get("type") != "mcp_tool_call":
continue
server, tool, item_id = _codex_server(item.get("server")), item.get("tool"), item.get("id")
if not server or not isinstance(tool, str) or not tool or not isinstance(item_id, str):
continue
entry = items.get(item_id)
if entry is None:
if len(items) >= _CODEX_MCP_CALLS:
continue
entry = items[item_id] = {
"server": server,
"tool": tool,
"args": _codex_args_key(item.get("arguments")),
"status": None,
}
if event.get("type") == "item.completed":
entry["status"] = _codex_item_status(item)
entry["text"] = _codex_item_text(item)
return list(items.values())
_CODEX_TEXT_CHARS = 512
def _codex_item_text(item: Mapping[str, Any]) -> str:
"""The start of a finished item's first text block (the tool's own answer), or ``""``."""
result = item.get("result")
content = result.get("content") if isinstance(result, Mapping) else None
for block in content[:8] if isinstance(content, list) else ():
if isinstance(block, Mapping) and isinstance(block.get("text"), str) and block["text"].strip():
return block["text"][:_CODEX_TEXT_CHARS]
return ""
def _codex_call_outputs(trajectory: Mapping[str, Any] | None) -> dict[str, str]:
"""Tool call id -> the start of its result text in the trajectory (bounded)."""
outputs: dict[str, str] = {}
steps = trajectory.get("steps") if isinstance(trajectory, Mapping) else None
for step in steps[:_CODEX_MCP_CALLS] if isinstance(steps, list) else ():
observation = step.get("observation") if isinstance(step, Mapping) else None
results = observation.get("results") if isinstance(observation, Mapping) else None
for result in results[:64] if isinstance(results, list) else ():
if not isinstance(result, Mapping) or not isinstance(result.get("source_call_id"), str):
continue
content = result.get("content")
if isinstance(content, list):
content = "\n".join(
block.get("text", "") if isinstance(block, Mapping) else str(block)
for block in content[:8]
if isinstance(block, Mapping | str)
)
if isinstance(content, str):
outputs.setdefault(result["source_call_id"], content[: _CODEX_TEXT_CHARS * 16])
return outputs
def _codex_server_key(server: str) -> str:
# The session log spells a server the way Codex sanitizes it (``deepwiki_bad``);
# codex.txt keeps the configured name (``deepwiki-bad``).
return re.sub(r"[^a-z0-9]", "_", server.casefold())
def _text_in_output(text: str, output: str) -> bool:
# The trajectory keeps the tool text raw, or JSON-escaped inside Codex's content wrapper.
return bool(text) and (text in output or json.dumps(text, ensure_ascii=False)[1:-1] in output)
def _take_codex_item(
items: list[dict[str, Any]], *, tool: str, args: str | None, server: str | None, output: str = ""
) -> Any:
"""Pop the unpaired item with this tool and arguments (and server, when known).
An answer can disambiguate candidates. Conflicting identities/outcomes
remain unknown rather than inheriting the first log item's success.
"""
key = _codex_server_key(server) if server is not None else None
fits = [
index
for index, item in enumerate(items)
if item["tool"] == tool and item["args"] == args and (key is None or _codex_server_key(item["server"]) == key)
]
if not fits or args is None:
return None
matching_output = [index for index in fits if _text_in_output(items[index].get("text") or "", output)]
candidates = matching_output or fits
outcomes = {(_codex_server_key(items[index]["server"]), items[index].get("status")) for index in candidates}
if len(outcomes) != 1:
return None
return items.pop(candidates[0])
def _codex_mcp_calls(trial_root: Path, trajectory: Mapping[str, Any] | None) -> tuple[dict[str, str], dict[str, str]]:
"""Codex MCP calls read from the raw Codex logs: ``(call id -> server, call id -> status)``.
Harbor's Codex trajectory keeps only the bare MCP tool name (for example
``lookup``) and no error flag. Codex's own session log records the server
(``namespace: "mcp__<server>"`` on the call, and an ``McpToolCall`` item
with the call id); the ``codex.txt`` JSON stream records each call's
outcome (``status: "failed"`` or ``"completed"``) but under its own item
id. So each session-log call is paired with the first unpaired
``codex.txt`` item of the same server, tool, and arguments. Without a
readable session log, the trajectory's calls are paired with ``codex.txt``
items the same way (tool and arguments), never by name and order alone; a
tool name served by one server only still maps without arguments. When
several items fit, the one whose answer is in the call's trajectory result
wins. Empty for other harnesses.
"""
servers: dict[str, str] = {}
session_calls: list[tuple[str, str, str, str | None]] = []
agent_dir = trial_root / "agent"
for path in _codex_session_files(agent_dir / "sessions"):
for event in _codex_json_lines(path):
payload = event.get("payload") if isinstance(event, Mapping) else None
if not isinstance(payload, Mapping):
continue
item = payload.get("item")
tool: Any = None
args: str | None = None
item_type = item.get("type") if isinstance(item, Mapping) else None
payload_type = payload.get("type")
if isinstance(item_type, str) and item_type in {"McpToolCall", "mcp_tool_call"}:
call_id, server = item.get("id"), _codex_server(item.get("server"))
tool, args = item.get("tool"), _codex_args_key(item.get("arguments"))
elif isinstance(payload_type, str) and payload_type in {"function_call", "custom_tool_call"}:
namespace = payload.get("namespace")
call_id = payload.get("call_id")
server = _codex_server(namespace) if isinstance(namespace, str) and namespace[:5] == "mcp__" else None
tool = payload.get("name")
args = _codex_args_key(payload.get("arguments", payload.get("input")))
else:
continue
if isinstance(call_id, str) and call_id and server and len(servers) < _CODEX_MCP_CALLS:
if call_id not in servers and isinstance(tool, str) and tool:
session_calls.append((call_id, server, tool, args))
servers.setdefault(call_id, server)
items = _codex_txt_items(agent_dir)
outputs = _codex_call_outputs(trajectory) if items else {}
statuses: dict[str, str] = {}
if servers:
for call_id, server, tool, args in session_calls:
item = _take_codex_item(items, tool=tool, args=args, server=server, output=outputs.get(call_id, ""))
if item is not None and item["status"]:
statuses[call_id] = item["status"]
elif item is None and any(
entry["tool"] == tool and _codex_server_key(entry["server"]) == _codex_server_key(server)
for entry in items
):
statuses[call_id] = "unknown"
return servers, statuses
if not items or not isinstance(trajectory, Mapping):
return servers, statuses
servers_by_tool: dict[str, set[str]] = {}
for item in items:
servers_by_tool.setdefault(item["tool"], set()).add(item["server"])
for step in (trajectory.get("steps") or ())[:_CODEX_MCP_CALLS]:
for call in (step.get("tool_calls") if isinstance(step, Mapping) else None) or ():
if not isinstance(call, Mapping):
continue
name, call_id = call.get("function_name"), call.get("tool_call_id")
if not isinstance(name, str) or not isinstance(call_id, str) or not call_id or call_id in servers:
continue
item = _take_codex_item(
items,
tool=name,
args=_codex_args_key(call.get("arguments")),
server=None,
output=outputs.get(call_id, ""),
)
if item is not None:
servers[call_id] = item["server"]
if item["status"]:
statuses[call_id] = item["status"]
elif len(servers_by_tool.get(name, ())) == 1:
servers[call_id] = next(iter(servers_by_tool[name]))
statuses[call_id] = "unknown"
if len(servers) >= _CODEX_MCP_CALLS:
return servers, statuses
return servers, statuses
_OPENCODE_LOG_MAX_BYTES = 16 * 1024 * 1024
_OPENCODE_FAILED_CALLS = 4_000
_OPENCODE_ERROR_CHARS = 2_048
def _opencode_call_errors(trial_root: Path) -> dict[str, str]:
"""OpenCode tool calls that failed, read from the raw ``opencode.txt``: ``call id -> error text``.
OpenCode's JSON stream ends each tool part with ``state.status``:
``completed`` with an ``output``, or ``error`` with an ``error``. Harbor's
OpenCode trajectory keeps a call's result only when it has an output, so a
failed call (a ``task`` to an agent that does not exist, an MCP call whose
connection closed) is left with no result and no outcome. The part's
``callID`` is the trajectory's tool call id. Empty for other harnesses.
"""
from skillevaluator.tier3.eval_core.log_converters import _opencode_error_text
text = _read_bounded_text(trial_root / "agent" / "opencode.txt", max_bytes=_OPENCODE_LOG_MAX_BYTES)
errors: dict[str, str] = {}
for line in text.splitlines() if text and "tool_use" in text else ():
if "tool_use" not in line or "error" not in line:
continue
try:
event = json.loads(line)
except (ValueError, RecursionError):
continue
part = event.get("part") if isinstance(event, Mapping) and event.get("type") == "tool_use" else None
state = part.get("state") if isinstance(part, Mapping) else None
if not isinstance(state, Mapping) or state.get("status") != "error":
continue
call_id = part.get("callID", part.get("id"))
if isinstance(call_id, str) and call_id and call_id not in errors:
errors[call_id] = _opencode_error_text(dict(state))[:_OPENCODE_ERROR_CHARS]
if len(errors) >= _OPENCODE_FAILED_CALLS:
break
return errors
def _call_status(result: Any, statuses: Mapping[str, str]) -> str | None:
call_id = result.get("source_call_id") if isinstance(result, Mapping) else None
return statuses.get(call_id) if isinstance(call_id, str) else None
def _step_results(step: Any) -> list[Any]:
observation = step.get("observation") if isinstance(step, Mapping) else None
results = observation.get("results") if isinstance(observation, Mapping) else None
return results if isinstance(results, list) else []
def _with_harness_statuses(
trajectory: dict[str, Any] | None, statuses: Mapping[str, str], errors: Mapping[str, str] | None = None
) -> dict[str, Any] | None:
"""A copy of ``trajectory`` whose results carry the harness's own per-call status.
The status goes to ``extra.harness_status`` of every result whose
``source_call_id`` names the call, where plugin signals read it as the
call's structured outcome. A call in ``errors`` (call id -> error text)
that no result names gets one in its step: the error text, with an
``error`` status. The trajectory read from disk is not changed.
"""
from skillevaluator.tier3.eval_core.plugin_signals import HARNESS_STATUS_KEY
steps = trajectory.get("steps") if isinstance(trajectory, dict) else None
if not (statuses or errors) or not isinstance(steps, list):
return trajectory
errors = errors or {}
# Only a call that no result names had its result dropped.
answered = {
result["source_call_id"]
for step in steps
for result in _step_results(step)
if errors and isinstance(result, Mapping) and isinstance(result.get("source_call_id"), str)
}
updated: list[Any] = []
for step in steps:
results = _step_results(step)
calls = step.get("tool_calls") if errors and isinstance(step, Mapping) else None
dropped = {
call_id: None
for call in (calls if isinstance(calls, list) else ())
if isinstance(call, Mapping)
and isinstance(call_id := call.get("tool_call_id"), str)
and call_id in errors
and call_id not in answered
}
if not dropped and not any(_call_status(result, statuses) for result in results):
updated.append(step)
continue
marked = []
for result in results:
status = _call_status(result, statuses)
if status is None:
marked.append(result)
continue
extra = result.get("extra") if isinstance(result.get("extra"), Mapping) else {}
marked.append({**result, "extra": {**extra, HARNESS_STATUS_KEY: status}})
# The harness dropped these results (OpenCode keeps none for a failed call).
marked.extend(
{"source_call_id": call_id, "content": errors[call_id], "extra": {HARNESS_STATUS_KEY: "error"}}
for call_id in dropped
)
answered.update(dropped)
observation = step.get("observation") if isinstance(step.get("observation"), Mapping) else {}
updated.append({**step, "observation": {**observation, "results": marked}})
return {**trajectory, "steps": updated}
@dataclass(frozen=True)
class TrialTrace:
trajectory: dict[str, Any] | None
call_servers: Mapping[str, str]
def _harness_sources(trial_root: Path, trajectory: dict[str, Any] | None) -> list[tuple[Path, dict[str, Any] | None]]:
"""Partition a materialized native multi-step trajectory by its authorized sources.
A physical steps layout makes the materializer require authoritative
``step_results`` and reject a root trajectory. Only after that succeeds
are its generated step names trusted as sibling harness-log locations.
Single-step trajectories cannot select log paths through authored extras.
"""
if trajectory is None or not _is_real_directory(trial_root / "steps"):
return [(trial_root, trajectory)]
extra = trajectory.get("extra")
multi = extra.get("harbor_multi_step") if isinstance(extra, Mapping) else None
names = multi.get("step_names") if isinstance(multi, Mapping) else None
if not isinstance(names, list) or any(_safe_trial_path_component(name) != name for name in names):
return [(trial_root, None)]
groups: dict[str, list[Any]] = {name: [] for name in names}
for step in trajectory.get("steps", []):
extra = step.get("extra") if isinstance(step, Mapping) else None
name = extra.get("harbor_step_name") if isinstance(extra, Mapping) else None
if name not in groups:
return [(trial_root, None)]
groups[name].append(step)
return [(trial_root / "steps" / name, {**trajectory, "steps": groups[name]}) for name in names]
def _scope_harness_call_ids(trajectory: dict[str, Any], prefix: str) -> dict[str, Any]:
"""Keep independent step sessions' reused call IDs distinct in the merged document."""
updated = []
for step in trajectory["steps"]:
if not isinstance(step, Mapping):
updated.append(step)
continue
scoped = dict(step)
calls = step.get("tool_calls")
if isinstance(calls, list):
scoped["tool_calls"] = [
{**call, "tool_call_id": prefix + call["tool_call_id"]}
if isinstance(call, Mapping) and isinstance(call.get("tool_call_id"), str)
else call
for call in calls
]
observation = step.get("observation")
results = observation.get("results") if isinstance(observation, Mapping) else None
if isinstance(results, list):
scoped["observation"] = {
**observation,
"results": [
{**result, "source_call_id": prefix + result["source_call_id"]}
if isinstance(result, Mapping) and isinstance(result.get("source_call_id"), str)
else result
for result in results
],
}
updated.append(scoped)
return {**trajectory, "steps": updated}
class TrialTraceArtifacts:
"""Read one whole-trial trajectory and harness logs once per logical root."""
def __init__(self) -> None:
self._traces: dict[Path, TrialTrace] = {}
def trace(self, trial_root: Path) -> TrialTrace:
if trial_root not in self._traces:
trajectory = _plugin_signal_trajectory(trial_root)
sources = _harness_sources(trial_root, trajectory)
servers, steps = {}, []
for index, (source_root, source) in enumerate(sources):
if source is None:
trajectory = None
break
try:
local_servers, statuses = _codex_mcp_calls(source_root, source)
errors = _opencode_call_errors(source_root)
statuses.update(dict.fromkeys(errors, "error"))
marked = _with_harness_statuses(source, statuses, errors)
except Exception:
# Advisory harness logs cannot discard an authorized trajectory.
local_servers, marked = {}, source
if source_root != trial_root:
prefix = f"harbor-step:{index}:"
marked = _scope_harness_call_ids(marked, prefix)
local_servers = {prefix + call_id: server for call_id, server in local_servers.items()}
servers.update(local_servers)
steps.extend(marked["steps"])
if trajectory is not None:
trajectory = {**trajectory, "steps": steps}
self._traces[trial_root] = TrialTrace(trajectory, servers)
return self._traces[trial_root]