Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Token usage per case, agent and run: measured, never estimated and never scored."""
from __future__ import annotations
import copy
import json
from typing import Any
import pytest
from skillevaluator.tier3.harbor.collector import _pass_summary
from skillevaluator.tier3.harbor.metrics import DEFAULT_METRICS
from skillevaluator.tier3.token_usage import build_token_usage, normalize_token_counts
def _trial(case: str, prompt: int, completion: int, cached: int | None = None) -> dict[str, Any]:
tokens = {"prompt": prompt, "completion": completion}
if cached is not None:
tokens["cached"] = cached
return {"entry_id": case, "trial_id": f"{case}__a", "tokens": tokens}
def _agent(
with_skill: list[dict[str, Any]],
baseline: list[dict[str, Any]],
*,
with_status: str = "succeeded",
baseline_status: str = "succeeded",
unscored: list[dict[str, Any]] | None = None,
unscored_baseline: list[dict[str, Any]] | None = None,
) -> dict[str, Any]:
unscored = unscored or []
unscored_baseline = unscored_baseline or []
return {
"trials": with_skill,
"trials_baseline": baseline,
"usage_only_attempts": [{**row, "scored": False} for row in unscored],
"usage_only_attempts_baseline": [{**row, "scored": False} for row in unscored_baseline],
"conditions": {
"with_skill": {
"execution_status": with_status,
"expected_attempts": len(with_skill) + len(unscored),
"scored_attempts": len(with_skill),
},
"without_skill": {
"execution_status": baseline_status,
"expected_attempts": len(baseline) + len(unscored_baseline),
"scored_attempts": len(baseline),
},
},
"execution_status": "failed" if "failed" in (with_status, baseline_status) else "succeeded",
}
def _collector_pass_at_k(scored: dict[str, int], *, k: int, planned: dict[str, int] | None = None) -> dict[str, Any]:
"""One arm's pass@k as the collector records it: ``scored`` attempts per case, ``planned`` under a run budget."""
scores = dict.fromkeys(DEFAULT_METRICS, 0.9)
rewards = [
{"entry_id": case, "_trial_name": f"{case}__{index}", **scores}
for case, count in scored.items()
for index in range(count)
]
return _pass_summary(
rewards,
n_attempts=k,
pass_threshold=0.5,
expected_cases=len(scored),
expected_case_ids=list(scored),
expected_attempts_by_case=planned,
)
def _scopes(usage: dict[str, Any], agent: str = "codex") -> list[dict[str, Any]]:
return [usage["dataset"], usage["agents"][agent], *usage["agents"][agent]["cases"]]
# -- one attempt's counters --------------------------------------------------------
def test_cached_tokens_are_part_of_the_prompt_and_never_added_again() -> None:
assert normalize_token_counts({"prompt": 100, "completion": 20, "cached": 40}) == {
"prompt_tokens": 100,
"cached_tokens": 40,
"uncached_prompt_tokens": 60,
"completion_tokens": 20,
"total_tokens": 120,
"complete": True,
}
def test_an_omitted_cache_counter_is_zero_and_atif_names_are_read() -> None:
normalized = normalize_token_counts({"prompt_tokens": 100, "completion_tokens": 20})
assert normalized is not None
assert (normalized["cached_tokens"], normalized["total_tokens"], normalized["complete"]) == (0, 120, True)
assert normalize_token_counts({}) is None
assert normalize_token_counts(None) is None
@pytest.mark.parametrize("field", ["prompt", "completion", "cached"])
@pytest.mark.parametrize("value", [-1, True, "100", float("nan"), 1.5, 2**60])
def test_invalid_counters_are_rejected_and_named_without_their_values(field: str, value: object) -> None:
raw: dict[str, Any] = {"prompt": 100, "completion": 20, "cached": 10, field: value}
normalized = normalize_token_counts(raw)
assert normalized is not None
assert normalized["complete"] is False
assert normalized["invalid_counters"] == [f"{field}_tokens"]
assert "100" not in json.dumps(normalized["invalid_counters"])
def test_a_counter_the_trajectory_recorded_invalid_is_rejected_although_tokens_leave_it_out() -> None:
# A report trial's ``tokens`` leave out an invalid counter; ``invalid_token_counters`` names it.
normalized = normalize_token_counts({"prompt": 100, "completion": 20}, invalid_counters=["cached_tokens"])
assert normalized is not None
assert (normalized["complete"], normalized["cached_tokens"], normalized["invalid_counters"]) == (
False,
None,
["cached_tokens"],
)
without_tokens = normalize_token_counts(None, invalid_counters=["prompt_tokens"])
assert without_tokens is not None
assert (without_tokens["complete"], without_tokens["invalid_counters"]) == (False, ["prompt_tokens"])
# Anything but known counter names is ignored.
assert normalize_token_counts(None, invalid_counters=["steps", {"x": 1}]) is None
trial = {**_trial("case-1", 100, 20), "invalid_token_counters": ["cached_tokens"]}
usage = build_token_usage({"codex": _agent([trial, _trial("case-2", 50, 5)], [_trial("case-1", 90, 10)])})
assert usage is not None
with_skill = usage["agents"]["codex"]["with_skill"]
assert (with_skill["coverage"]["measured_attempts"], with_skill["coverage"]["reason_codes"]) == (
1,
["invalid_token_counters"],
)
assert with_skill["totals"]["total_tokens"] == 55
def test_cached_above_prompt_is_invalid() -> None:
normalized = normalize_token_counts({"prompt": 10, "completion": 5, "cached": 11})
assert normalized is not None
assert (normalized["complete"], normalized["invalid_counters"]) == (False, ["cached_tokens"])
# -- sums per case, agent and run -----------------------------------------------------
def test_sums_each_case_each_agent_and_all_agents() -> None:
usage = build_token_usage(
{
"codex": _agent(
[_trial("case-1", 100, 20, 40), _trial("case-2", 200, 30, 50)],
[_trial("case-1", 150, 25, 30), _trial("case-2", 250, 35, 40)],
),
"opencode": _agent([_trial("case-1", 80, 10, 20)], [_trial("case-1", 90, 15, 10)]),
}
)
assert usage is not None
assert usage["scored"] is False
codex = usage["agents"]["codex"]
assert codex["with_skill"]["totals"] == {
"prompt_tokens": 300,
"cached_tokens": 90,
"uncached_prompt_tokens": 210,
"completion_tokens": 50,
"total_tokens": 350,
}
cases = {row["entry_id"]: row for row in codex["cases"]}
assert cases["case-1"]["with_skill"]["totals"]["total_tokens"] == 120
assert cases["case-1"]["baseline"]["totals"]["total_tokens"] == 175
assert (cases["case-1"]["delta"]["total_tokens"], cases["case-1"]["delta"]["total_tokens_change_percent"]) == (
-55,
-31.43,
)
assert codex["delta"]["total_tokens"] == 350 - 460
assert usage["dataset"]["with_skill"]["totals"]["total_tokens"] == 440
assert usage["dataset"]["baseline"]["totals"]["total_tokens"] == 565
assert usage["dataset"]["delta"]["total_tokens"] == -125
assert usage["dataset"]["with_skill"]["coverage"]["measured_attempts"] == 3
def test_a_failed_attempt_is_counted_for_usage_but_not_as_a_scored_attempt() -> None:
failed = {"entry_id": "case-1", "trial_id": "case-1__b", "tokens": {"prompt": 400, "completion": 4000}}
usage = build_token_usage(
{
"codex": _agent(
[_trial("case-1", 100, 20)],
[_trial("case-1", 90, 10)],
unscored=[failed],
)
}
)
assert usage is not None
with_skill = usage["agents"]["codex"]["with_skill"]
assert with_skill["totals"]["total_tokens"] == 120 + 4400
assert with_skill["coverage"] == {
"status": "complete",
"expected_attempts": 2,
"observed_attempts": 2,
"measured_attempts": 2,
"unscored_attempts": 1,
"reason_codes": [],
}
def test_a_failed_attempt_with_a_cut_harbor_trial_name_is_matched_to_its_case() -> None:
case = "a-very-long-case-identifier-that-harbor-cuts-short"
trial_prefix = case[:32].rstrip("_-")
failed = {"entry_id": trial_prefix, "trial_id": f"{trial_prefix}__b", "tokens": {"prompt": 50, "completion": 5}}
usage = build_token_usage({"codex": _agent([_trial(case, 100, 20)], [_trial(case, 90, 10)], unscored=[failed])})
assert usage is not None
[row] = usage["agents"]["codex"]["cases"]
assert row["entry_id"] == case
assert row["with_skill"]["totals"]["total_tokens"] == 175
def test_a_stop_on_pass_trial_name_is_matched_to_its_case() -> None:
def timed_out(arm: str) -> dict[str, Any]:
# Without a Harbor result, the loader keeps the trial name before Harbor's ``__<id>``.
name = f"demo-codex-{arm}-case-1-attempt001__case-1"
return {"entry_id": name, "trial_id": f"{name}__a", "tokens": {"prompt": 40, "completion": 4}}
usage = build_token_usage(
{
"codex": _agent(
[_trial("case-1", 100, 20)],
[_trial("case-1", 90, 10)],
unscored=[timed_out("with")],
unscored_baseline=[timed_out("without")],
)
}
)
assert usage is not None
codex = usage["agents"]["codex"]
assert [row["entry_id"] for row in codex["cases"]] == ["case-1"]
assert codex["delta"]["status"] == "available"
def test_partial_coverage_sums_only_measured_attempts_and_never_estimates() -> None:
missing = {"entry_id": "case-2", "trial_id": "case-2__a"}
invalid = {"entry_id": "case-3", "trial_id": "case-3__a", "tokens": {"prompt": -5, "completion": 9}}
usage = build_token_usage(
{
"codex": _agent(
[_trial("case-1", 100, 20), missing, invalid],
[_trial("case-1", 90, 10), _trial("case-2", 90, 10), _trial("case-3", 90, 10)],
)
}
)
assert usage is not None
with_skill = usage["agents"]["codex"]["with_skill"]
assert with_skill["totals"]["total_tokens"] == 120
assert with_skill["totals"]["prompt_tokens"] == 100
assert with_skill["coverage"]["status"] == "partial"
assert (with_skill["coverage"]["measured_attempts"], with_skill["coverage"]["expected_attempts"]) == (1, 3)
assert with_skill["coverage"]["reason_codes"] == ["invalid_token_counters", "missing_token_counters"]
for scope in (usage["dataset"], usage["agents"]["codex"]):
assert scope["with_skill"]["coverage"]["status"] == "partial"
assert scope["delta"]["status"] == "unavailable"
assert scope["delta"]["total_tokens"] is None
assert "with_skill_tokens_incomplete" in scope["delta"]["reason_codes"]
assert "With-skill run token counts cover 1 of 3 attempts." in usage["agents"]["codex"]["delta"]["reason"]
cases = {row["entry_id"]: row for row in usage["agents"]["codex"]["cases"]}
assert cases["case-1"]["delta"]["total_tokens"] == 20
assert cases["case-2"]["with_skill"]["totals"] is None
assert cases["case-2"]["with_skill"]["coverage"]["status"] == "unavailable"
@pytest.mark.parametrize(
"pass_at_k",
[
_collector_pass_at_k({"case-1": 1}, k=2),
# A result file from before the collector counted each case's attempts.
{"k": 2, "cases": {"case-1": {"attempts_skipped": 0}}},
],
ids=["collector", "k-only"],
)
def test_an_attempt_that_never_ran_leaves_the_arm_partial(pass_at_k: dict[str, Any]) -> None:
agent = _agent([_trial("case-1", 100, 20)], [_trial("case-1", 90, 10)])
agent["conditions"]["with_skill"]["expected_attempts"] = 2
agent["pass_at_k"] = {"with_skill": pass_at_k}
usage = build_token_usage({"codex": agent})
assert usage is not None
coverage = usage["agents"]["codex"]["with_skill"]["coverage"]
assert (coverage["status"], coverage["observed_attempts"], coverage["reason_codes"]) == (
"partial",
1,
["missing_attempts"],
)
[case] = usage["agents"]["codex"]["cases"]
assert case["with_skill"]["coverage"]["expected_attempts"] == 2
assert case["delta"]["total_tokens"] is None
def test_a_planned_case_without_attempts_gets_a_row_with_no_tokens() -> None:
agent = _agent([_trial("case-1", 100, 20)], [])
agent["pass_at_k"] = {
"with_skill": {"k": 1, "cases": {"case-1": {"attempts_skipped": 0}, "case-2": {"attempts_skipped": 0}}}
}
agent["conditions"]["without_skill"] = {"execution_status": "skipped", "expected_attempts": 0}
usage = build_token_usage({"codex": agent})
assert usage is not None
cases = {row["entry_id"]: row for row in usage["agents"]["codex"]["cases"]}
assert cases["case-2"]["with_skill"] == {
"totals": None,
"coverage": {
"status": "unavailable",
"expected_attempts": 1,
"observed_attempts": 0,
"measured_attempts": 0,
"unscored_attempts": 0,
"reason_codes": ["missing_attempts"],
},
}
assert cases["case-2"]["baseline"]["coverage"]["status"] == "not_run"
def test_attempts_skipped_after_a_pass_are_not_missing() -> None:
agent = _agent([_trial("case-1", 100, 20)], [_trial("case-1", 90, 10)])
agent["pass_at_k"] = {"with_skill": {"k": 3, "cases": {"case-1": {"attempts_skipped": 2}}}}
usage = build_token_usage({"codex": agent})
assert usage is not None
[case] = usage["agents"]["codex"]["cases"]
assert case["with_skill"]["coverage"]["status"] == "complete"
assert case["delta"]["total_tokens"] == 20
def test_a_case_that_a_run_budget_cut_short_expects_only_the_attempts_it_admitted() -> None:
# The budget admitted all 3 attempts of case-1 but only 1 of case-2, in both arms.
trials = [
{**_trial(case, 100, 10), "trial_id": f"{case}__{index}"}
for case, count in (("case-1", 3), ("case-2", 1))
for index in range(count)
]
agent = _agent(trials, copy.deepcopy(trials))
pass_at_k = _collector_pass_at_k({"case-1": 3, "case-2": 1}, k=3, planned={"case-1": 3, "case-2": 1})
agent["pass_at_k"] = {"with_skill": pass_at_k, "without_skill": copy.deepcopy(pass_at_k)}
usage = build_token_usage({"codex": agent})
assert usage is not None
cases = {row["entry_id"]: row for row in usage["agents"]["codex"]["cases"]}
for arm in ("with_skill", "baseline"):
coverage = cases["case-2"][arm]["coverage"]
assert (coverage["status"], coverage["expected_attempts"], coverage["reason_codes"]) == ("complete", 1, [])
assert cases["case-2"]["delta"]["total_tokens"] == 0
assert usage["agents"]["codex"]["delta"]["status"] == "available"
def test_a_recovered_timeout_attempt_is_expected_usage() -> None:
history = {"entry_id": "case-1", "trial_id": "case-1__x", "tokens": {"prompt": 10, "completion": 1}}
agent = _agent([_trial("case-1", 100, 20)], [_trial("case-1", 90, 10)])
agent["usage_only_attempts"] = [{**history, "scored": False}]
agent["conditions"]["with_skill"]["recovered_attempts_total"] = 1
usage = build_token_usage({"codex": agent})
assert usage is not None
with_skill = usage["agents"]["codex"]["with_skill"]
assert (with_skill["coverage"]["status"], with_skill["coverage"]["expected_attempts"]) == ("complete", 2)
assert with_skill["totals"]["total_tokens"] == 131
# The arms ran different attempts, so they are not compared.
assert usage["agents"]["codex"]["delta"]["reason_codes"] == ["case_scope_mismatch"]
# -- when a change is given ---------------------------------------------------------------
@pytest.mark.parametrize(("with_total", "percent"), [(590, -41.0), (1000, 0.0), (1250, 25.0)])
def test_percentage_change_is_given_at_every_scope(with_total: int, percent: float) -> None:
usage = build_token_usage({"codex": _agent([_trial("case-1", with_total, 0)], [_trial("case-1", 1000, 0)])})
assert usage is not None
for scope in _scopes(usage):
assert scope["delta"] == {
"status": "available",
"total_tokens": with_total - 1000,
"total_tokens_change_percent": percent,
"reason_codes": [],
"reason": None,
}
def test_a_zero_token_baseline_keeps_the_difference_without_a_percentage() -> None:
usage = build_token_usage({"codex": _agent([_trial("case-1", 10, 0)], [_trial("case-1", 0, 0)])})
assert usage is not None
delta = usage["dataset"]["delta"]
assert (delta["total_tokens"], delta["total_tokens_change_percent"], delta["reason_codes"]) == (
10,
None,
["zero_baseline"],
)
def test_a_reused_baseline_is_labelled_and_otherwise_unchanged() -> None:
agents = {"codex": _agent([_trial("case-1", 100, 10)], [_trial("case-1", 200, 20)])}
plain = build_token_usage(agents)
reused = build_token_usage(agents, baseline_reused=True)
assert plain is not None and reused is not None
assert "baseline_reused" not in plain
assert reused["baseline_reused"] is True
# The reused rows are measurements like any other, so the sums and the change stay.
assert {key: value for key, value in reused.items() if key != "baseline_reused"} == plain
assert reused["dataset"]["delta"]["total_tokens"] == -110
def test_a_skipped_baseline_is_not_run_and_has_no_change() -> None:
agent = _agent([_trial("case-1", 100, 20)], [])
agent["conditions"]["without_skill"] = {"execution_status": "skipped", "expected_attempts": 0}
usage = build_token_usage({"codex": agent})
assert usage is not None
for scope in _scopes(usage):
assert scope["baseline"]["coverage"]["status"] == "not_run"
assert scope["baseline"]["totals"] is None
assert scope["delta"]["total_tokens"] is None
assert scope["delta"]["reason_codes"][0] == "baseline_not_run"
# The skip is the one reason: the arms did not run different cases.
codex = usage["agents"]["codex"]
assert (codex["delta"]["reason_codes"], codex["delta"]["reason"]) == (
["baseline_not_run"],
"The baseline did not run.",
)
@pytest.mark.parametrize("mode", ["both", "with_skill_only"])
def test_a_baseline_without_a_record_or_attempts_did_not_run(mode: str) -> None:
agent = _agent([_trial("case-1", 100, 20)], [])
del agent["conditions"]["without_skill"]
usage = build_token_usage({"codex": agent}, mode=mode)
assert usage is not None
for scope in _scopes(usage)[1:]:
assert scope["baseline"]["coverage"] == {
"status": "not_run",
"expected_attempts": 0,
"observed_attempts": 0,
"measured_attempts": 0,
"unscored_attempts": 0,
"reason_codes": [],
}
assert (scope["delta"]["reason_codes"], scope["delta"]["reason"]) == (
["baseline_not_run"],
"The baseline did not run.",
)
assert usage["dataset"]["baseline"]["coverage"]["status"] == "not_run"
def test_an_arm_the_run_mode_leaves_out_did_not_run() -> None:
agent = _agent([_trial("case-1", 100, 20)], [])
agent["conditions"]["without_skill"] = {"expected_attempts": 1}
usage = build_token_usage({"codex": agent}, mode="with_skill_only")
assert usage is not None
assert usage["agents"]["codex"]["baseline"]["coverage"]["status"] == "not_run"
assert usage["agents"]["codex"]["delta"]["reason_codes"] == ["baseline_not_run"]
# In a run of both arms the same arm planned an attempt and recorded none.
usage = build_token_usage({"codex": agent})
assert usage is not None
codex = usage["agents"]["codex"]
assert codex["baseline"]["coverage"]["status"] == "unavailable"
assert (codex["delta"]["reason_codes"], codex["delta"]["reason"]) == (
["baseline_tokens_incomplete"],
"No baseline attempts were recorded.",
)
with_skill = _agent([], [_trial("case-1", 100, 20)])
del with_skill["conditions"]["with_skill"]
usage = build_token_usage({"codex": with_skill}, mode="baseline_only")
assert usage is not None
assert usage["agents"]["codex"]["with_skill"]["coverage"]["status"] == "not_run"
assert usage["agents"]["codex"]["delta"]["reason_codes"] == ["with_skill_not_run"]
def test_absent_baseline_evidence_does_not_claim_the_baseline_never_ran() -> None:
agent = _agent([_trial("case-1", 100, 20)], [])
for record in ({}, {"execution_status": "succeeded"}, {"scored_attempts": 1}):
agent["conditions"]["without_skill"] = record
usage = build_token_usage({"codex": agent})
assert usage is not None
baseline = usage["agents"]["codex"]["baseline"]
assert baseline["coverage"]["status"] == "unavailable"
assert "no_attempts_recorded" in baseline["coverage"]["reason_codes"]
# One reason, not a status or case mismatch on top of it.
assert (usage["agents"]["codex"]["delta"]["reason_codes"], usage["agents"]["codex"]["delta"]["reason"]) == (
["baseline_tokens_incomplete"],
"No baseline attempts were recorded.",
)
def test_arms_with_different_attempt_counts_or_cases_are_not_compared() -> None:
usage = build_token_usage(
{"codex": _agent([_trial("case-1", 590, 0)], [_trial("case-1", 1000, 0), _trial("case-2", 1000, 0)])}
)
assert usage is not None
assert usage["agents"]["codex"]["delta"]["reason_codes"] == ["case_scope_mismatch"]
assert usage["dataset"]["delta"]["reason_codes"] == ["agent_scope_mismatch"]
usage = build_token_usage({"codex": _agent([_trial("case-1", 50, 0)], [_trial("case-2", 100, 0)])})
assert usage is not None
assert usage["agents"]["codex"]["delta"]["reason_codes"] == ["case_scope_mismatch"]
assert usage["dataset"]["delta"]["reason_codes"] == ["agent_scope_mismatch"]
def test_one_agent_without_a_change_blocks_the_change_over_all_agents() -> None:
usage = build_token_usage(
{
"codex": _agent([_trial("case-1", 100, 0)], [_trial("case-1", 200, 0)]),
"opencode": _agent([_trial("case-1", 100, 0)], [_trial("case-1", 200, 0)], baseline_status="failed"),
}
)
assert usage is not None
assert usage["agents"]["codex"]["delta"]["total_tokens"] == -100
assert usage["dataset"]["baseline"]["totals"]["total_tokens"] == 400
assert usage["dataset"]["delta"]["reason_codes"] == ["agent_scope_mismatch"]
def test_a_failed_baseline_arm_keeps_its_usage_but_gets_no_change() -> None:
timed_out = {"entry_id": "case-1", "trial_id": "case-1__b", "tokens": {"prompt": 400, "completion": 47522}}
agent = _agent(
[_trial("case-1", 100, 20), _trial("case-1", 110, 20)],
[_trial("case-1", 90, 10)],
baseline_status="failed",
unscored_baseline=[timed_out],
)
usage = build_token_usage({"claude-code": agent})
assert usage is not None
agent_usage = usage["agents"]["claude-code"]
assert agent_usage["baseline"]["totals"]["total_tokens"] == 100 + 47922
assert agent_usage["baseline"]["coverage"]["status"] == "complete"
assert agent_usage["baseline"]["coverage"]["unscored_attempts"] == 1
for delta in (agent_usage["delta"], *(case["delta"] for case in agent_usage["cases"])):
assert delta["status"] == "unavailable"
assert delta["reason_codes"] == ["baseline_failed"]
assert delta["reason"] == "No change: baseline incomplete (1 of 2 attempts scored)."
@pytest.mark.parametrize("status", ["unknown", "interrupted", None])
def test_an_arm_without_a_known_run_status_gets_no_change(status: str | None) -> None:
agent = _agent([_trial("case-1", 100, 20)], [_trial("case-1", 500, 20)], baseline_status="succeeded")
agent["conditions"]["without_skill"]["execution_status"] = status
usage = build_token_usage({"codex": agent})
assert usage is not None
codex = usage["agents"]["codex"]
# The usage is still measured, but the report withholds this arm's scores.
assert codex["baseline"]["coverage"]["status"] == "complete"
for delta in (codex["delta"], *(case["delta"] for case in codex["cases"]), usage["dataset"]["delta"]):
assert delta["status"] == "unavailable"
assert delta["total_tokens"] is None
assert codex["delta"]["reason_codes"] == ["baseline_status_unknown"]
assert codex["delta"]["reason"] == "No change: baseline status unknown."
def test_a_sum_above_the_exact_json_range_is_not_reported() -> None:
# Each counter is valid, but each arm's sum is above 2**53 - 1.
half = 2**52
usage = build_token_usage(
{
"codex": _agent(
[_trial("case-1", half + 2, 0), _trial("case-1", half + 2, 0)],
[_trial("case-1", half, 0), _trial("case-1", half, 0)],
)
}
)
assert usage is not None
codex = usage["agents"]["codex"]
for scope in (usage["dataset"], codex, codex["cases"][0]):
for arm in ("with_skill", "baseline"):
assert scope[arm]["totals"] is None
assert scope[arm]["coverage"]["status"] == "unavailable"
assert "total_out_of_range" in scope[arm]["coverage"]["reason_codes"]
assert scope["delta"]["status"] == "unavailable"
assert scope["delta"]["total_tokens"] is None
assert codex["delta"]["reason_codes"] == ["with_skill_total_out_of_range", "baseline_total_out_of_range"]
assert "largest integer JSON keeps exact" in codex["delta"]["reason"]
def test_agent_sums_within_range_can_still_pass_it_over_all_agents() -> None:
half = 2**52
usage = build_token_usage(
{
"codex": _agent([_trial("case-1", half, 0)], [_trial("case-1", half, 0)]),
"opencode": _agent([_trial("case-1", half, 0)], [_trial("case-1", half, 0)]),
}
)
assert usage is not None
assert usage["agents"]["codex"]["with_skill"]["totals"]["total_tokens"] == half
dataset = usage["dataset"]
assert dataset["with_skill"]["totals"] is None
assert dataset["with_skill"]["coverage"]["reason_codes"] == ["total_out_of_range"]
assert "with_skill_total_out_of_range" in dataset["delta"]["reason_codes"]
def test_no_measured_attempt_gives_no_token_usage() -> None:
bare = {"entry_id": "case-1", "trial_id": "case-1__a"}
assert build_token_usage({"codex": _agent([bare], [bare])}) is None
assert build_token_usage({}) is None
# -- case rows are validated against the dataset's canonical IDs (issue #192) ----------
# A real dataset ID that Harbor's 32-character trial name cuts short.
_LONG_CASE = "huggingface-factory-peft-sft-local-return"
_CUT_CASE = _LONG_CASE[:32].rstrip("_-")
def test_a_cut_case_name_never_becomes_a_second_row_beside_its_dataset_case() -> None:
# The scored arm recorded Harbor's cut trial name; the other arm the full one.
agent = _agent([_trial(_CUT_CASE, 100, 20)], [_trial(_LONG_CASE, 90, 10)])
usage = build_token_usage({"codex": agent}, dataset_case_ids=[_LONG_CASE])
assert usage is not None
[row] = usage["agents"]["codex"]["cases"]
assert row["entry_id"] == _LONG_CASE
assert row["with_skill"]["totals"]["total_tokens"] == 120
assert row["baseline"]["totals"]["total_tokens"] == 100
assert usage["case_id_validation"] == {
"status": "validated",
"dataset_case_ids": 1,
"unmatched_case_ids": [],
}
def test_an_arm_that_died_keeps_its_tokens_on_the_canonical_case() -> None:
# Both arms were killed before scoring, so only the cut trial name exists.
def killed(suffix: str) -> dict[str, Any]:
return {
"entry_id": _CUT_CASE,
"trial_id": f"{_CUT_CASE}__{suffix}",
"tokens": {"prompt": 40, "completion": 4},
}
agent = _agent(
[_trial("case-1", 100, 20)],
[_trial("case-1", 90, 10)],
unscored=[killed("a")],
unscored_baseline=[killed("b")],
)
usage = build_token_usage({"codex": agent}, dataset_case_ids=["case-1", _LONG_CASE])
assert usage is not None
rows = {row["entry_id"]: row for row in usage["agents"]["codex"]["cases"]}
assert sorted(rows) == ["case-1", _LONG_CASE]
assert rows[_LONG_CASE]["with_skill"]["totals"]["total_tokens"] == 44
assert rows[_LONG_CASE]["baseline"]["totals"]["total_tokens"] == 44
assert usage["case_id_validation"]["unmatched_case_ids"] == []
def test_every_case_row_uses_a_dataset_id_at_most_once() -> None:
other = "huggingface-global-negative-serving"
agent = _agent(
[_trial(_LONG_CASE, 100, 20), _trial(other[:32].rstrip("_-"), 50, 5)],
[_trial(_CUT_CASE, 90, 10), _trial(other, 40, 4)],
)
usage = build_token_usage({"codex": agent}, dataset_case_ids=[_LONG_CASE, other])
assert usage is not None
ids = [row["entry_id"] for row in usage["agents"]["codex"]["cases"]]
assert ids == sorted({_LONG_CASE, other})
assert len(ids) == len(set(ids))
def test_a_row_that_no_dataset_id_explains_is_marked_not_rendered_as_a_case() -> None:
agent = _agent([_trial("case-1", 100, 20)], [_trial("ghost-case", 90, 10)])
usage = build_token_usage({"codex": agent}, dataset_case_ids=["case-1"])
assert usage is not None
rows = {row["entry_id"]: row for row in usage["agents"]["codex"]["cases"]}
assert rows["case-1"].get("case_id_in_dataset") is None
assert rows["ghost-case"]["case_id_in_dataset"] is False
assert usage["case_id_validation"] == {
"status": "unverified_case_ids",
"dataset_case_ids": 1,
"unmatched_case_ids": ["ghost-case"],
}
def test_two_dataset_ids_with_the_same_cut_name_are_never_guessed() -> None:
first = "huggingface-factory-peft-sft-local-return"
second = "huggingface-factory-peft-sft-local-detach"
assert first[:32] == second[:32]
agent = _agent([_trial("case-1", 100, 20)], [_trial("case-1", 90, 10)], unscored=[_trial(_CUT_CASE, 40, 4)])
usage = build_token_usage({"codex": agent}, dataset_case_ids=["case-1", first, second])
assert usage is not None
rows = {row["entry_id"]: row for row in usage["agents"]["codex"]["cases"]}
assert rows[_CUT_CASE]["case_id_in_dataset"] is False
assert usage["case_id_validation"]["unmatched_case_ids"] == [_CUT_CASE]
def test_an_agent_without_scores_still_gets_canonical_case_rows() -> None:
# An optional agent the deadline killed: no scored attempt, usage only.
killed = {"entry_id": _CUT_CASE, "trial_id": f"{_CUT_CASE}__a", "tokens": {"prompt": 40, "completion": 4}}
claude = {
"trials": [],
"trials_baseline": [],
"usage_only_attempts": [{**killed, "scored": False}],
"usage_only_attempts_baseline": [],
"conditions": {
"with_skill": {"execution_status": "failed", "expected_attempts": 1, "scored_attempts": 0},
"without_skill": {"execution_status": "failed", "expected_attempts": 1, "scored_attempts": 0},
},
"execution_status": "failed",
}
usage = build_token_usage(
{"claude": claude, "codex": _agent([_trial(_LONG_CASE, 100, 20)], [_trial(_LONG_CASE, 90, 10)])},
dataset_case_ids=[_LONG_CASE],
)
assert usage is not None
assert [row["entry_id"] for row in usage["agents"]["claude"]["cases"]] == [_LONG_CASE]
assert usage["agents"]["claude"]["baseline"]["coverage"]["status"] == "unavailable"
assert usage["agents"]["claude"]["delta"]["status"] == "unavailable"
assert usage["case_id_validation"]["unmatched_case_ids"] == []
def test_without_a_dataset_nothing_is_claimed_about_case_ids() -> None:
usage = build_token_usage({"codex": _agent([_trial("case-1", 100, 20)], [_trial("case-1", 90, 10)])})
assert usage is not None
assert "case_id_validation" not in usage
assert all("case_id_in_dataset" not in row for row in usage["agents"]["codex"]["cases"])
def test_a_pass_at_k_case_named_by_its_cut_name_is_one_canonical_row() -> None:
agent = _agent([_trial(_LONG_CASE, 100, 20)], [_trial(_LONG_CASE, 90, 10)])
agent["pass_at_k"] = {"with_skill": {"k": 2, "cases": {_CUT_CASE: {"attempts_skipped": 0}}}}
usage = build_token_usage({"codex": agent}, dataset_case_ids=[_LONG_CASE])
assert usage is not None
[row] = usage["agents"]["codex"]["cases"]
assert row["entry_id"] == _LONG_CASE
assert row["with_skill"]["coverage"]["expected_attempts"] == 2
@pytest.mark.parametrize("cut_first", [True, False], ids=["cut-first", "dataset-id-first"])
def test_two_pass_at_k_keys_for_one_case_merge_the_same_way_in_either_order(cut_first: bool) -> None:
# Harbor's cut name and the dataset ID canonicalize onto one case. Which
# record survives must not depend on the order the payload lists them in:
# the key that is already a dataset ID wins, whichever comes first.
agent = _agent([_trial(_LONG_CASE, 100, 20)], [_trial(_LONG_CASE, 90, 10)])
records = [(_CUT_CASE, {"attempts_skipped": 2}), (_LONG_CASE, {"attempts_skipped": 0})]
agent["pass_at_k"] = {
"with_skill": {"k": 3, "cases": dict(records if cut_first else reversed(records))},
}
usage = build_token_usage({"codex": agent}, dataset_case_ids=[_LONG_CASE])
assert usage is not None
[row] = usage["agents"]["codex"]["cases"]
assert row["entry_id"] == _LONG_CASE
assert row["with_skill"]["coverage"]["expected_attempts"] == 3