Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(tier3): keep per-trial activation diagnostics for failed arms (#664)
Christopher Kevin940ed63300f7
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 142abf61..7c9532a2 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -330,6 +330,7 @@ All notable changes to SkillEvaluator are documented in this file.### Fixed+- Failed Tier 3 arms retain scored and errored trial activation diagnostics without restoring aggregate quality or lift, or double-counting token usage.- Budget-limited Tier 3 coverage checks retain admitted attempt ordinals, avoiding false missing/excess errors when a denied slot leaves a gap.- HTML reports retain budget context for fully budget-denied runs withoutrestoring untrusted score evidence.diff --git a/src/skillevaluator/evaluation/tier3_report.py b/src/skillevaluator/evaluation/tier3_report.pyindex db5fe1a1..7f6bfbc3 100644--- a/src/skillevaluator/evaluation/tier3_report.py+++ b/src/skillevaluator/evaluation/tier3_report.py@@ -978,7 +978,10 @@ def build_agent_eval_payload(dataset_summary = (dict(persisted_dataset_summary)if isinstance(persisted_dataset_summary, dict)- else _dataset_summary(public_dataset, canonical_trials)+ else _dataset_summary(+ public_dataset,+ [trial for trial in canonical_trials if not trial.get("diagnostic_only")],+ ))from skillevaluator.tier3.dataset_sufficiency import MIN_NEGATIVE_CASES, assess_dataset_sufficiency@@ -2529,19 +2532,25 @@ def _build_agent(pass_with_skill = info.get("pass_with_skill")pass_threshold = _finite_float(pass_with_skill.get("pass_threshold")) if isinstance(pass_with_skill, dict) else Nonepass_threshold = pass_threshold if pass_threshold is not None else 0.5- trials = _normalize_trials(info.get("rewards") or [], with_metrics, pass_threshold=pass_threshold)+ trials = _normalize_trials(+ info.get("diagnostic_rewards", info.get("rewards")) or [], with_metrics, pass_threshold=pass_threshold+ )baseline_trials = _normalize_trials(- info.get("rewards_baseline") or [],+ info.get("diagnostic_rewards_baseline", info.get("rewards_baseline")) or [],baseline_metrics,pass_threshold=pass_threshold,)+ for key, rows in (("diagnostic_rewards", trials), ("diagnostic_rewards_baseline", baseline_trials)):+ if key in info:+ for trial in rows:+ trial["diagnostic_only"] = Trueif not never_activated:_attach_baseline_pairs(trials, baseline_trials, with_metrics, comparable_overall=comparable_scores)from skillevaluator.tier3.harbor.resource_usage import resource_comparisonresources = resource_comparison(- trials,- baseline_trials,+ trials if with_quality_available else [],+ baseline_trials if baseline_quality_available else [],with_complete=with_quality_available and info.get("rewards_complete") is True,baseline_complete=baseline_quality_available and info.get("rewards_baseline_complete") is True,)@@ -3575,6 +3584,8 @@ def _normalize_trials("scores": scores,"overall": _complete_mean([overall_score(item) for item in group]) if group_is_consistent else None,}+ if reward.get("_unscored") is True:+ trial["scored"] = Falseverdict = attempt_verdict(trial["overall"], pass_threshold, reward=reward)trial["attempt_verdict"] = {"passed": verdict.passed,@@ -3645,6 +3656,10 @@ def _normalize_trials(if harbor_viewer:trial["harbor_viewer"] = harbor_viewertrial["activation_validation"] = _trial_activation_validation(group, trial)+ if trial.get("scored") is False:+ trial["activation_validation"]["diagnostics"] = [+ {"code": "attempt_unscored", "message": "Attempt errored or was unscored; activation is unknown."}+ ]out.append(trial)return outdiff --git a/src/skillevaluator/tier3/harbor/report_data.py b/src/skillevaluator/tier3/harbor/report_data.pyindex 5dbae24d..2ae54091 100644--- a/src/skillevaluator/tier3/harbor/report_data.py+++ b/src/skillevaluator/tier3/harbor/report_data.py@@ -18,7 +18,7 @@ import osimport statfrom collections.abc import Callable, Iterablefrom itertools import islice-from pathlib import Path+from pathlib import Path, PurePosixPathfrom typing import Anyfrom skillevaluator.tier3.declarative_coverage import agent_declarative_checks@@ -589,12 +589,16 @@ def logical_trial_reward_groups(rewards: list[dict[str, Any]]) -> list[list[dictgroups: dict[str, list[dict[str, Any]]] = {}for index, reward in enumerate(rewards):identity = reward.get("trial_id")- valid_identity = isinstance(identity, str) and 0 < len(identity) <= 512- key = f"trial:{identity}" if valid_identity else f"row:{index}"+ key = f"trial:{identity}" if _is_trial_identity(identity) else f"row:{index}"groups.setdefault(key, []).append(reward)return list(groups.values())+def _is_trial_identity(value: object) -> bool:+ """Whether ``value`` is a usable trial identity: a non-empty string of at most 512 characters."""+ return isinstance(value, str) and 0 < len(value) <= 512++def _nonnegative_counter(value: Any) -> int:return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else 0@@ -812,6 +816,31 @@ def _unscored_attempt_usage(trial_dir: Path) -> dict[str, Any]:return attempt+def _with_trial_identity(rewards: list[dict[str, Any]]) -> list[dict[str, Any]]:+ """Return ``rewards`` with the trial directory as the identity of every row that has no usable one.++ A failed arm keeps each scored reward twice: as a diagnostic report trial+ and as a usage-only attempt. Token usage counts the attempt once by finding+ both copies under the same ``trial_id``, but older artifacts carry none.+ Nothing else tells the copies apart from two attempts: retries share a case+ ID, and two attempts can use the same tokens. A trial directory holds one+ attempt, so both copies name it and no two attempts share it. A reward with+ a usable ``trial_id`` keeps it, so ``logical_trial_reward_groups`` groups it+ as before. Only withheld arms need this, so the rewards of an arm that+ succeeded are loaded exactly as they were read.+ """+ identified = []+ for reward in rewards:+ artifacts = reward.get("_activation_artifacts")+ reward_path = artifacts.get("reward") if isinstance(artifacts, dict) else None+ trial_dir = PurePosixPath(reward_path).parent.name if isinstance(reward_path, str) else ""+ if _is_trial_identity(reward.get("trial_id")) or not _is_trial_identity(trial_dir):+ identified.append(reward)+ else:+ identified.append({**reward, "trial_id": trial_dir})+ return identified++def _withheld_attempt_usage(rewards: list[dict[str, Any]]) -> list[dict[str, Any]]:"""Return the token usage of scored attempts whose arm's scores are not reported.@@ -1006,6 +1035,7 @@ def load_agent_data()for variant_key, variant_dir_name in (("rewards", "with-skill"), ("rewards_baseline", "without-skill")):+ condition = "with_skill" if variant_key == "rewards" else "without_skill"trial_list: list[dict[str, Any]] = []unscored_attempts: list[dict[str, Any]] = []count_key = "num_trials" if variant_key == "rewards" else "num_trials_baseline"@@ -1049,7 +1079,10 @@ def load_agent_data(unscored_attempts.append(_unscored_attempt_usage(trial_dir))continuereward = _load_bounded_json(reward_file, agent_diagnostics, artifact="reward")- if not isinstance(reward, dict):+ if not isinstance(reward, dict) or (+ condition_execution.get(condition, {}).get("execution_status") != "succeeded"+ and str(reward.get("evaluation_status") or "").casefold() in {"error", "failed"}+ ):rewards_complete = Falseunscored_attempts.append(_unscored_attempt_usage(trial_dir))continue@@ -1183,9 +1216,19 @@ def load_agent_data(condition_status = _condition_status(agent_info, condition)if condition_status == "succeeded":continue+ # Both copies of a scored attempt, the diagnostic trial and the usage-only+ # attempt below, come from the same rows, so they share one identity.+ scored_rewards = _with_trial_identity(agent_info.get(fields[-1]) or [])+ agent_info[f"diagnostic_{fields[-1]}"] = [+ *scored_rewards,+ *[+ {"entry_id": row["entry_id"], "trial_id": row["trial_id"], "_unscored": True}+ for row in agent_info.get(_USAGE_ONLY_KEYS[condition], [])+ ],+ ]# The arm's scores are withheld below, but its scored attempts used tokens.agent_info[_USAGE_ONLY_KEYS[condition]] = [- *_withheld_attempt_usage(agent_info.get(fields[-1]) or []),+ *_withheld_attempt_usage(scored_rewards),*agent_info.get(_USAGE_ONLY_KEYS[condition], []),]condition_info = condition_execution.get(condition, {})diff --git a/src/skillevaluator/tier3/token_usage.py b/src/skillevaluator/tier3/token_usage.pyindex e85459bd..c3123594 100644--- a/src/skillevaluator/tier3/token_usage.py+++ b/src/skillevaluator/tier3/token_usage.py@@ -223,16 +223,25 @@ def _agent_attempts(agent: Mapping[str, Any], known_cases: set[str]) -> dict[strattempts: dict[str, list[_Attempt]] = {}for arm, _condition, key, usage_key in _ARMS:attempts[arm] = []+ usage_rows = _rows(agent.get(usage_key))+ # A failed arm keeps a scored attempt as a diagnostic report trial and as+ # a usage-only copy. The loader gives both the same ``trial_id`` (the+ # trial directory, for rewards that carry none), and the usage-only copy+ # is the one counted. Nothing else tells one attempt from two: retries+ # share a case ID, and two attempts can use the same tokens.+ usage_trial_ids = {row["trial_id"] for row in usage_rows if isinstance(row.get("trial_id"), str)}for trial in _rows(agent.get(key)):trial_id = trial.get("trial_id")+ if isinstance(trial_id, str) and trial_id in usage_trial_ids:+ continueattempts[arm].append(_Attempt(_matching_case(_case_label(trial), trial_id if isinstance(trial_id, str) else "", known_cases),_attempt_tokens(trial),- True,+ trial.get("scored") is not False,))- for row in _rows(agent.get(usage_key)):+ for row in usage_rows:scored = row.get("scored") is True# Scored or not, the name an attempt carries is its case directory,# its recorded entry_id or its trial name, and any of the three maydiff --git a/tests/reporting/test_failed_arm_trials.py b/tests/reporting/test_failed_arm_trials.pynew file mode 100644index 00000000..d2d9af0a--- /dev/null+++ b/tests/reporting/test_failed_arm_trials.py@@ -0,0 +1,312 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Failed arms retain trial diagnostics without restoring aggregate scores."""++from __future__ import annotations++import json+from pathlib import Path++import pytest++from skillevaluator.evaluation.tier3_report import (+ _validation_result_from_payload,+ agent_eval_result_from_directory,+)+from skillevaluator.reporting.benchmark import BenchmarkReporter+from skillevaluator.tier3.harbor import report_data+from skillevaluator.tier3.harbor.metrics import DEFAULT_METRIC_SET, DEFAULT_METRICS+from skillevaluator.tier3.token_usage import build_token_usage+++def _write(path: Path, value: object) -> None:+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(json.dumps(value))+++def _failed_run(+ tmp_path: Path,+ arm: str,+ *,+ malformed: bool = False,+ empty: bool = False,+ failed_status: str | None = None,+ scored_trials: tuple[str, ...] = ("case__scored",),+ legacy_ids: bool = False,+) -> dict:+ """Write a run whose ``arm`` failed; ``legacy_ids`` leaves ``trial_id`` out of the failed arm's rewards.++ ``scored_trials`` are the failed arm's scored attempts. The other arm keeps one.+ """+ skill = tmp_path / "demo"+ skill.mkdir()+ run = tmp_path / "results"+ for variant in ("with-skill", "without-skill"):+ failed = variant == arm+ _write(+ run / "codex" / variant / "summary.json",+ {+ "scores": dict.fromkeys(DEFAULT_METRICS, 1.0),+ "metrics": list(DEFAULT_METRICS),+ "num_trials": 1,+ "execution_status": "failed" if failed else "succeeded",+ "execution_errors": ["AgentTimeoutError: timed out"] if failed else [],+ "expected_attempts": 2 if failed else 1,+ "scored_attempts": 1,+ },+ )+ if failed and empty:+ continue+ trials_dir = run / "codex" / variant / "trials"+ for name in scored_trials if failed else ("case__scored",):+ trial = trials_dir / name+ reward = {+ **dict.fromkeys(DEFAULT_METRICS, 1.0),+ "metric_set": DEFAULT_METRIC_SET,+ "entry_id": "case",+ "trial_id": name,+ "skill_invoked": variant == "with-skill",+ "invocation_evidence_source": "trajectory",+ "invocation_evidence_scope": "logical_trial",+ }+ if failed and legacy_ids:+ # Older artifacts carry no collector-issued trial identity.+ del reward["trial_id"]+ _write(trial / "reward.json", reward)+ _write(+ trial / "trajectory.json", {"final_metrics": {"total_prompt_tokens": 10, "total_completion_tokens": 2}}+ )+ if failed:+ error = trials_dir / "case__errored"+ _write(error / "failure.json", {"entry_id": "case", "reason": "AgentTimeoutError: timed out"})+ _write(+ error / "trajectory.json", {"final_metrics": {"total_prompt_tokens": 20, "total_completion_tokens": 3}}+ )+ if malformed:+ (error / "reward.json").write_text("{")+ if failed_status:+ _write(+ error / "reward.json",+ {+ **dict.fromkeys(DEFAULT_METRICS, 1.0),+ "metric_set": DEFAULT_METRIC_SET,+ "entry_id": "case",+ "trial_id": "case__errored",+ "evaluation_status": failed_status,+ "skill_invoked": False,+ "invocation_evidence_source": "trajectory",+ "invocation_evidence_scope": "logical_trial",+ },+ )+ result = agent_eval_result_from_directory(skill, run, use_llm_judge=False)+ assert result is not None+ return result.metadata["agent_eval"]++[email protected]("arm,key", [("with-skill", "trials"), ("without-skill", "trials_baseline")])+def test_failed_arm_keeps_scored_trial_activation(tmp_path: Path, arm: str, key: str) -> None:+ payload = _failed_run(tmp_path, arm)+ agent = payload["agents"]["codex"]+ scored = next((row for row in agent[key] if row["trial_id"] == "case__scored"), None)+ assert scored is not None+ assert scored["scores"]["goal_accuracy"] == 1.0+ assert scored["activation_validation"]["observed_activation"] == (+ "invoked" if arm == "with-skill" else "not_invoked"+ )+ assert agent["with_skill" if arm == "with-skill" else "baseline"] is None+ assert agent["lift"] is None+ usage = payload["token_usage"]["agents"]["codex"]["with_skill" if arm == "with-skill" else "baseline"]+ assert usage["coverage"]["observed_attempts"] == 2+ assert usage["coverage"]["unscored_attempts"] == 1+ assert usage["totals"]["total_tokens"] == 35++[email protected]("arm,key", [("with-skill", "trials"), ("without-skill", "trials_baseline")])+def test_failed_arm_rewards_without_trial_id_are_counted_once(tmp_path: Path, arm: str, key: str) -> None:+ """Older rewards carry no ``trial_id``: the diagnostic trial and the usage-only copy are one attempt."""+ (tmp_path / "with-ids").mkdir()+ (tmp_path / "legacy").mkdir()+ with_ids = _failed_run(tmp_path / "with-ids", arm)+ legacy = _failed_run(tmp_path / "legacy", arm, legacy_ids=True)+ scored = [row for row in legacy["agents"]["codex"][key] if row.get("scored") is not False]+ assert len(scored) == 1+ assert scored[0]["scores"]["goal_accuracy"] == 1.0+ usage = legacy["token_usage"]["agents"]["codex"]["with_skill" if arm == "with-skill" else "baseline"]+ assert usage["coverage"]["observed_attempts"] == 2+ assert usage["coverage"]["measured_attempts"] == 2+ assert usage["coverage"]["unscored_attempts"] == 1+ totals = usage["totals"]+ assert (totals["prompt_tokens"], totals["completion_tokens"], totals["total_tokens"]) == (30, 5, 35)+ # The same run with trial IDs reports the same usage.+ assert legacy["token_usage"] == with_ids["token_usage"]++[email protected]("legacy_ids", [True, False])+def test_failed_arm_distinct_attempts_with_identical_usage_each_count(tmp_path: Path, legacy_ids: bool) -> None:+ """Retries share a case ID, and these two also share their token counts. Each is still one attempt.++ Attempts are told apart by the trial directory they ran in, never by case ID or usage.+ """+ payload = _failed_run(tmp_path, "with-skill", legacy_ids=legacy_ids, scored_trials=("case__first", "case__second"))+ agent = payload["agents"]["codex"]+ scored = [row for row in agent["trials"] if row.get("scored") is not False]+ assert [row["entry_id"] for row in scored] == ["case", "case"]+ usage = payload["token_usage"]["agents"]["codex"]["with_skill"]+ assert usage["coverage"]["observed_attempts"] == 3+ assert usage["coverage"]["unscored_attempts"] == 1+ # Two scored attempts of 12 tokens each, and the errored attempt's 23.+ assert usage["totals"]["total_tokens"] == 47++[email protected]("identity", ["absent", "", 7, "x" * 513])+def test_withheld_rewards_without_a_valid_trial_id_share_their_directory_identity(+ tmp_path: Path, identity: object+) -> None:+ _failed_run(tmp_path, "with-skill", legacy_ids=True, scored_trials=("case__a", "case__b"))+ reward_path = tmp_path / "results" / "codex" / "with-skill" / "trials" / "case__a" / "reward.json"+ reward = json.loads(reward_path.read_text())+ if identity != "absent":+ reward["trial_id"] = identity+ _write(reward_path, reward)+ # A reward that does carry a collector-issued identity keeps it.+ other_path = reward_path.parents[1] / "case__b" / "reward.json"+ _write(other_path, {**json.loads(other_path.read_text()), "trial_id": "collector-issued"})++ agent = report_data.load_agent_data(tmp_path / "results")["codex"]++ diagnostic = [row.get("trial_id") for row in agent["diagnostic_rewards"] if row.get("_unscored") is not True]+ usage = [row["trial_id"] for row in agent["usage_only_attempts"] if row["scored"] is True]+ assert diagnostic == usage == ["case__a", "collector-issued"]+ assert agent["rewards"] == []+++def test_succeeded_arm_rewards_without_trial_id_load_unchanged(tmp_path: Path) -> None:+ """Only a withheld arm needs a stand-in identity; a succeeded arm's legacy rewards stay identity-less."""+ _failed_run(tmp_path, "")+ trials = tmp_path / "results" / "codex" / "with-skill" / "trials"+ for name in ("case__first", "case__second"):+ reward = {**dict.fromkeys(DEFAULT_METRICS, 1.0), "metric_set": DEFAULT_METRIC_SET, "entry_id": "case"}+ _write(trials / name / "reward.json", reward)+ agent = report_data.load_agent_data(tmp_path / "results")["codex"]+ assert [row.get("trial_id") for row in agent["rewards"]] == [None, "case__scored", None]+ assert len(report_data.logical_trial_reward_groups(agent["rewards"])) == 3++[email protected]("artifact", ["missing-reward", "malformed-reward", "failed", "error"])+def test_errored_trial_has_unknown_activation(tmp_path: Path, artifact: str) -> None:+ agent = _failed_run(+ tmp_path,+ "with-skill",+ malformed=artifact == "malformed-reward",+ failed_status=artifact if artifact in {"failed", "error"} else None,+ )["agents"]["codex"]+ error = next((row for row in agent["trials"] if row["trial_id"] == "case__errored"), None)+ assert error is not None+ assert error["activation_validation"]["observed_activation"] == "unknown"+ assert error["activation_validation"]["expectation_status"] == "unknown"+ assert error["activation_validation"]["diagnostics"][0]["code"] == "attempt_unscored"+ assert error["scored"] is False+ assert error["scores"] == {}+ assert error["overall"] is None+++def test_empty_failed_arm_does_not_invent_trials(tmp_path: Path) -> None:+ assert _failed_run(tmp_path, "with-skill", empty=True)["agents"]["codex"]["trials"] == []+++def test_diagnostic_trials_do_not_double_count_usage() -> None:+ agent = {+ "conditions": {"with_skill": {"execution_status": "failed", "expected_attempts": 2, "scored_attempts": 1}},+ "trials": [+ {"entry_id": "case", "trial_id": "case__scored", "tokens": {"prompt": 10, "completion": 2}},+ {"entry_id": "case", "trial_id": "case__errored", "scored": False},+ ],+ "usage_only_attempts": [+ {+ "entry_id": "case",+ "trial_id": "case__scored",+ "scored": True,+ "tokens": {"prompt_tokens": 10, "completion_tokens": 2},+ },+ {+ "entry_id": "case",+ "trial_id": "case__errored",+ "scored": False,+ "tokens": {"prompt_tokens": 20, "completion_tokens": 3},+ },+ ],+ }+ usage = build_token_usage({"codex": agent}, mode="with_skill_only", dataset_case_ids=["case"])+ arm = usage["agents"]["codex"]["with_skill"]+ assert arm["coverage"]["observed_attempts"] == 2+ assert arm["coverage"]["unscored_attempts"] == 1+ assert arm["totals"]["total_tokens"] == 35++[email protected]("arm,key", [("with-skill", "trials"), ("without-skill", "trials_baseline")])+def test_failed_arm_score_bearing_outputs_are_unchanged(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, arm: str, key: str+) -> None:+ payload = _failed_run(tmp_path, arm)+ load = report_data.load_agent_data++ def without_diagnostics(*args, **kwargs):+ agents = load(*args, **kwargs)+ for info in agents.values():+ info.pop("diagnostic_rewards", None)+ info.pop("diagnostic_rewards_baseline", None)+ return agents++ monkeypatch.setattr(report_data, "load_agent_data", without_diagnostics)+ reference = agent_eval_result_from_directory(tmp_path / "demo", tmp_path / "results", use_llm_judge=False)+ assert reference is not None+ expected = reference.metadata["agent_eval"]+ agent, before = payload["agents"]["codex"], expected["agents"]["codex"]+ assert agent["with_skill" if arm == "with-skill" else "baseline"] is None+ assert agent["lift"] is None+ for field in ("with_skill", "baseline", "lift", "dimensions", "evaluators", "cases", "resources"):+ assert agent[field] == before[field]+ for field in (+ "cases",+ "verdict",+ "recommendations",+ "suggestions",+ "suggestions_v2",+ "conclusions",+ "case_diagnostics",+ "dataset_summary",+ "dataset_sufficiency",+ "overall_score",+ "overall_lift",+ ):+ assert payload[field] == expected[field]+ assert payload["dataset_summary"]["total_tasks"] == (0 if arm == "with-skill" else 1)+ other_key = "trials_baseline" if key == "trials" else "trials"+ assert all(row.get("diagnostic_only") is True for row in agent[key])+ assert all("diagnostic_only" not in row for row in agent[other_key])+ reporter = BenchmarkReporter(include_timestamp=False, report_scope="tier3")+ result = _validation_result_from_payload(payload)+ assert result is not None+ benchmark = reporter.render_all([result])+ assert benchmark == reporter.render_all([reference])+ overall = next(line for line in benchmark.splitlines() if line.startswith("| Overall |"))+ assert "uplift unavailable" in overall+ assert ("Not scored with the skill" if arm == "with-skill" else "baseline incomplete") in overall+ assert len(payload["agents"]["codex"][key]) == 2++[email protected]("status", ["failed", "error"])+def test_succeeded_arm_aggregates_are_unchanged(tmp_path: Path, status: str) -> None:+ expected = _failed_run(tmp_path, "")+ reward_path = tmp_path / "results" / "codex" / "with-skill" / "trials" / "case__scored" / "reward.json"+ reward = json.loads(reward_path.read_text())+ reward["evaluation_status"] = status+ _write(reward_path, reward)+ result = agent_eval_result_from_directory(tmp_path / "demo", tmp_path / "results", use_llm_judge=False)+ assert result is not None+ agent = result.metadata["agent_eval"]["agents"]["codex"]+ assert agent["conditions"]["with_skill"]["execution_status"] == "succeeded"+ assert len(agent["trials"]) == 1+ for field in ("with_skill", "baseline", "lift", "dimensions", "evaluators", "cases", "resources"):+ assert agent[field] == expected["agents"]["codex"][field]