Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier3): reject ambiguous or invalid price-file rates (#749)

Afloat47a6738a91d7
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e6095a56..89d3cbc7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,10 @@ All notable changes to SkillEvaluator are documented in this file.
## Unreleased
+### Fixed
+
+- Reject duplicate price-file keys and boolean rates before budget accounting, and report unrepresentable integer rates as configuration errors.
+
## 1.0.0 - 2026-11-17
### Added
diff --git a/docs/tier3-run-reference.mdx b/docs/tier3-run-reference.mdx
index d03aa387..a38040f8 100644
--- a/docs/tier3-run-reference.mdx
+++ b/docs/tier3-run-reference.mdx
@@ -565,10 +565,11 @@ Runs can be bounded with `--max-agent-tokens`, `--max-judge-calls`,
`--max-attempts-total`, `--max-cost-usd`, and `--price-file`. The price file is
JSON keyed by model identifier; each model entry requires `input_per_mtok` and
`output_per_mtok`, and may set `cached_input_per_mtok` (defaulting to the
-input rate). Cached prompt tokens use the cached rate and remaining prompt
-tokens use the input rate. A model key must exactly match the resolved agent
-model. Relative `harbor.price_file` paths resolve from the source skill
-directory.
+input rate). Duplicate JSON object keys and boolean rates are rejected; rates
+must be finite and non-negative. Cached prompt tokens use the cached rate and
+remaining prompt tokens use the input rate. A model key must exactly match the
+resolved agent model. Relative `harbor.price_file` paths resolve from the source
+skill directory.
The price file is user-supplied, not a verified billing rate card. Its currency
is as supplied by the file, labelled USD by `--max-cost-usd`; no currency
diff --git a/src/skillevaluator/tier3/harbor/run_budget.py b/src/skillevaluator/tier3/harbor/run_budget.py
index 7c30235c..0359c174 100644
--- a/src/skillevaluator/tier3/harbor/run_budget.py
+++ b/src/skillevaluator/tier3/harbor/run_budget.py
@@ -29,6 +29,15 @@ class Price:
cached_input_per_mtok: float | None = None
+def _unique_price_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
+ result: dict[str, Any] = {}
+ for key, value in pairs:
+ if key in result:
+ raise ValueError(f"duplicate JSON object key {key!r} in price file")
+ result[key] = value
+ return result
+
+
@dataclass(frozen=True)
class PriceBook:
prices: dict[str, Price]
@@ -39,8 +48,8 @@ class PriceBook:
def load(cls, path: Path) -> PriceBook:
try:
contents = path.read_bytes()
- raw = json.loads(contents.decode("utf-8"))
- except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
+ raw = json.loads(contents.decode("utf-8"), object_pairs_hook=_unique_price_object)
+ except (OSError, ValueError) as exc:
raise BudgetConfigurationError(f"{path}: invalid price file: {exc}") from exc
if not isinstance(raw, dict) or not raw:
raise BudgetConfigurationError(f"{path}: price file must be a non-empty object")
@@ -48,10 +57,17 @@ class PriceBook:
for model, value in raw.items():
if not isinstance(model, str) or not isinstance(value, dict):
raise BudgetConfigurationError(f"{path}: each model price must be an object")
+ if any(
+ isinstance(value.get(field), bool)
+ for field in ("input_per_mtok", "output_per_mtok", "cached_input_per_mtok")
+ ):
+ raise BudgetConfigurationError(f"{path}: {model} prices must not be booleans")
try:
input_price = float(value["input_per_mtok"])
output_price = float(value["output_per_mtok"])
cached_input_price = float(value.get("cached_input_per_mtok", input_price))
+ except OverflowError as exc:
+ raise BudgetConfigurationError(f"{path}: {model} prices must be finite and non-negative") from exc
except (KeyError, TypeError, ValueError) as exc:
raise BudgetConfigurationError(
f"{path}: {model} must provide input_per_mtok and output_per_mtok, with optional cached_input_per_mtok"
diff --git a/tests/tier3/test_price_file_validation.py b/tests/tier3/test_price_file_validation.py
new file mode 100644
index 00000000..fcd89198
--- /dev/null
+++ b/tests/tier3/test_price_file_validation.py
@@ -0,0 +1,64 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Price-file ambiguity must fail before it can affect budget accounting."""
+
+import json
+from pathlib import Path
+
+import pytest
+
+from skillevaluator.tier3.harbor.run_budget import BudgetConfigurationError, BudgetController, PriceBook
+
+_RATE_FIELDS = ("input_per_mtok", "output_per_mtok", "cached_input_per_mtok")
+
+
+ ("contents", "duplicate_key"),
+ [
+ (
+ '{"model":{"input_per_mtok":2,"output_per_mtok":4},"model":{"input_per_mtok":0,"output_per_mtok":0}}',
+ "model",
+ ),
+ ('{"model":{"input_per_mtok":2,"input_per_mtok":0,"output_per_mtok":4}}', "input_per_mtok"),
+ ('{"model":{"input_per_mtok":2,"output_per_mtok":4,"output_per_mtok":0}}', "output_per_mtok"),
+ (
+ '{"model":{"input_per_mtok":2,"output_per_mtok":4,"cached_input_per_mtok":1,"cached_input_per_mtok":0}}',
+ "cached_input_per_mtok",
+ ),
+ ('{"model":{"input_per_mtok":2,"input_\\u0070er_mtok":0,"output_per_mtok":4}}', "input_per_mtok"),
+ ],
+ ids=["model", "input", "output", "cached", "escaped-key"],
+)
+def test_duplicate_price_keys_are_rejected(tmp_path: Path, contents: str, duplicate_key: str) -> None:
+ path = tmp_path / "prices.json"
+ path.write_text(contents, encoding="utf-8")
+ with pytest.raises(BudgetConfigurationError, match=f"duplicate JSON object key {duplicate_key!r}"):
+ PriceBook.load(path)
+
+
[email protected]("field", _RATE_FIELDS)
[email protected]("value", [True, False, 10**1000], ids=["true", "false", "overflowing-integer"])
+def test_invalid_price_scalars_have_configuration_errors(tmp_path: Path, field: str, value: object) -> None:
+ rates = {"input_per_mtok": 2, "output_per_mtok": 4, "cached_input_per_mtok": 1, field: value}
+ path = tmp_path / "prices.json"
+ path.write_text(json.dumps({"model": rates}), encoding="utf-8")
+ message = "must not be booleans" if isinstance(value, bool) else "prices must be finite and non-negative"
+ with pytest.raises(BudgetConfigurationError, match=message):
+ PriceBook.load(path)
+
+
[email protected]("as_strings", [False, True])
+def test_valid_zero_rates_and_repeated_field_names_in_different_models_remain_usable(
+ tmp_path: Path, as_strings: bool
+) -> None:
+ rates = {"input_per_mtok": 0, "output_per_mtok": 4, "cached_input_per_mtok": 0}
+ if as_strings:
+ rates = {field: str(value) for field, value in rates.items()}
+ path = tmp_path / "prices.json"
+ path.write_text(json.dumps({"model": rates, "other": rates}), encoding="utf-8")
+ prices = PriceBook.load(path)
+ assert prices.price_for("model") == prices.price_for("other")
+ budget = BudgetController(price_book=prices)
+ budget.record_usage({"input_tokens": 1000, "output_tokens": 500, "cached_input_tokens": 200}, model="model")
+ assert budget.metadata()["used"]["cost_usd"] == pytest.approx(0.002)