Autonomous research system for measurable, computer-executable research.
Python
fix(budget): enforce token grants and record provider cost in the ledger (#199)
Glen-SP9d6932d6b415
diff --git a/praxist/core/budget.py b/praxist/core/budget.pyindex d2b76a2..5071fca 100644--- a/praxist/core/budget.py+++ b/praxist/core/budget.py@@ -14,7 +14,10 @@ from praxist.core.registry import (require_execution_plugin,)-ALLOWED_BUDGET_UNITS = {"tokens", "wall_clock_seconds", "gpu_hours"}+INFORMATIONAL_USAGE_UNITS = frozenset({"cost_usd", "neurons"})+ALLOWED_BUDGET_UNITS = {"tokens", "wall_clock_seconds", "gpu_hours"} | set(+ INFORMATIONAL_USAGE_UNITS+)class BudgetPolicy(Protocol):diff --git a/praxist/core/execution_guards.py b/praxist/core/execution_guards.pyindex 3492ad8..0b84430 100644--- a/praxist/core/execution_guards.py+++ b/praxist/core/execution_guards.py@@ -10,6 +10,7 @@ from datetime import UTC, datetimefrom pathlib import Pathfrom typing import Any+from praxist.core.budget import INFORMATIONAL_USAGE_UNITSfrom praxist.core.ledgers import BudgetLedgerfrom praxist.core.run_config import RunConfigfrom praxist.core.trajectory import TrajectoryWriter@@ -129,16 +130,27 @@ class BudgetedActionGuard:raise ResourceBudgetError(f"{self.action_type} requires an approved budget grant")if self.budget_grant_id and self.run_dir is not None:try:- BudgetLedger(self.run_dir, self.run_id).require_active_grant(self.budget_grant_id)+ ledger = BudgetLedger(self.run_dir, self.run_id)+ ledger.require_active_grant(self.budget_grant_id)+ exhausted = ledger.get_exhausted_units(self.budget_grant_id)+ if exhausted:+ raise ResourceBudgetError(+ f"Budget exhausted for units: {', '.join(sorted(exhausted))}"+ )except Exception as exc: # noqa: BLE001 - preflight should report the real reason.+ is_overrun = isinstance(exc, ResourceBudgetError)self._emit_event("resource.action_denied"- if self.require_budget_grant+ if self.require_budget_grant or is_overrunelse "resource.action_budget_warning",- severity="error" if self.require_budget_grant else "warning",- payload={"reason": "invalid_budget_grant", "error": str(exc), **self.metadata},+ severity="error" if self.require_budget_grant or is_overrun else "warning",+ payload={+ "reason": "budget_overrun" if is_overrun else "invalid_budget_grant",+ "error": str(exc),+ **self.metadata,+ },)- if self.require_budget_grant:+ if self.require_budget_grant or is_overrun:raise ResourceBudgetError(str(exc)) from excself._emit_event("resource.action_started", payload=self.metadata)@@ -173,7 +185,11 @@ class BudgetedActionGuard:approved = grant.get("granted_budget") or {}if not isinstance(approved, dict):raise ValueError(f"invalid granted budget for {self.budget_grant_id}")- usage = {unit: value for unit, value in usage.items() if unit in approved}+ usage = {+ unit: value+ for unit, value in usage.items()+ if unit in approved or unit in INFORMATIONAL_USAGE_UNITS+ }if "wall_clock_seconds" in expected_units and "wall_clock_seconds" in approved:usage.setdefault("wall_clock_seconds", elapsed)unknown_units = [diff --git a/praxist/core/ledgers.py b/praxist/core/ledgers.pyindex 821acd9..7600bdb 100644--- a/praxist/core/ledgers.py+++ b/praxist/core/ledgers.py@@ -8,7 +8,7 @@ from dataclasses import asdictfrom pathlib import Pathfrom typing import Any-from praxist.core.budget import ALLOWED_BUDGET_UNITS+from praxist.core.budget import ALLOWED_BUDGET_UNITS, INFORMATIONAL_USAGE_UNITSfrom praxist.core.protocol import BudgetDecision, BudgetRequestfrom praxist.core.storage import append_jsonl, read_jsonl, utc_now@@ -213,6 +213,26 @@ class BudgetLedger:raise ValueError(f"Budget grant not found: {grant_id}")return grants[grant_id]+ def get_exhausted_units(self, grant_id: str) -> list[str]:+ grant = self.require_active_grant(grant_id)+ approved = grant.get("granted_budget") or {}+ if not isinstance(approved, dict):+ raise ValueError(f"Budget grant has invalid approved budget: {grant_id}")++ totals = self._usage_totals_by_grant().get(grant_id, {})+ exhausted_units = []+ for unit, raw_amount in approved.items():+ if unit in INFORMATIONAL_USAGE_UNITS:+ continue # recorded for reporting; never gates execution+ try:+ allowed = float(raw_amount)+ used = float(totals.get(unit, 0.0))+ if used >= allowed and allowed > 0:+ exhausted_units.append(str(unit))+ except (TypeError, ValueError):+ continue+ return exhausted_units+def _require_usage_within_grant(self, grant_id: str, actual_usage: dict[str, float]) -> list[dict[str, Any]]:@@ -230,6 +250,8 @@ class BudgetLedger:f"Budget usage must be finite and non-negative for {unit}: {raw_amount}")if unit not in approved:+ if unit in INFORMATIONAL_USAGE_UNITS:+ continueraise ValueError(f"Budget usage unit not approved by grant {grant_id}: {unit}")approved_amount = float(approved[unit])new_total = float(totals.get(unit, 0.0)) + amountdiff --git a/praxist/core/modeling.py b/praxist/core/modeling.pyindex 991c4e2..dedbb1e 100644--- a/praxist/core/modeling.py+++ b/praxist/core/modeling.py@@ -103,16 +103,19 @@ class ModelProviderAdapter:model=str(raw_redacted.get("model", "")),text=None,usage={},+ cost={},error=str(raw_redacted.get("error")),failover_reason=self.classify_error(raw_redacted),)usage = raw_redacted.get("usage")+ cost = raw_redacted.get("cost")return ModelResult(success=True,provider_ref=self.provider_ref,model=str(raw_redacted.get("model", "")),text=str(raw_redacted.get("text", "")),usage={str(key): float(value) for key, value in dict(usage or {}).items()},+ cost={str(key): float(value) for key, value in dict(cost or {}).items()},error=None,failover_reason="none",)diff --git a/praxist/core/protocol.py b/praxist/core/protocol.pyindex 8e25c6a..fbd9783 100644--- a/praxist/core/protocol.py+++ b/praxist/core/protocol.py@@ -68,6 +68,7 @@ class ModelResult:usage: dict[str, float]error: str | Nonefailover_reason: str | None+ cost: dict[str, float] = field(default_factory=dict)def to_dict(self) -> dict[str, Any]:return asdict(self)@@ -264,6 +265,7 @@ class AgentRunResult:terminal_status: str | None = Nonetimed_out: bool = Falsecancelled: bool = False+ cost: dict[str, float] = field(default_factory=dict)def to_dict(self) -> dict[str, Any]:return {@@ -275,6 +277,7 @@ class AgentRunResult:"failover_reason": self.failover_reason,"credential_ref": self.credential_ref.to_dict() if self.credential_ref else None,"usage": self.usage,+ "cost": self.cost,"terminal_status": self.terminal_status,"timed_out": self.timed_out,"cancelled": self.cancelled,diff --git a/praxist/core/replay.py b/praxist/core/replay.pyindex d5dbfb3..254e49f 100644--- a/praxist/core/replay.py+++ b/praxist/core/replay.py@@ -16,7 +16,11 @@ from typing import Anyimport yaml-from praxist.core.budget import ALLOWED_BUDGET_UNITS, policy_for_ref+from praxist.core.budget import (+ ALLOWED_BUDGET_UNITS,+ INFORMATIONAL_USAGE_UNITS,+ policy_for_ref,+)from praxist.core.prompt_layout import sha256_json, sha256_textfrom praxist.core.protocol import BudgetRequestfrom praxist.core.redaction import scan_file@@ -615,6 +619,8 @@ def _verify_budget_ledger()continueif unit not in approved:+ if unit in INFORMATIONAL_USAGE_UNITS:+ continueerrors.append(f"budget usage {record.get('record_id')} uses unapproved budget unit: {unit}")diff --git a/praxist/core/runtimes.py b/praxist/core/runtimes.pyindex a606d7c..7f1d9e3 100644--- a/praxist/core/runtimes.py+++ b/praxist/core/runtimes.py@@ -227,6 +227,8 @@ def _collect_result_usage(result: AgentRunResult) -> None:collector = _runtime_usage_collector.get()if collector is not None:collector.add(result.usage)+ if hasattr(result, "cost") and result.cost:+ collector.add(result.cost)def prompt_text_for_request(request: AgentRunRequest) -> str:diff --git a/praxist/plugins/agent_runtimes/codex_sdk/_events.py b/praxist/plugins/agent_runtimes/codex_sdk/_events.pyindex 8c8ab13..48a5322 100644--- a/praxist/plugins/agent_runtimes/codex_sdk/_events.py+++ b/praxist/plugins/agent_runtimes/codex_sdk/_events.py@@ -42,6 +42,7 @@ class CodexEventCollector:self.tool_uses: list[ToolCallRecord] = []self.legacy_tool_uses: list[dict[str, Any]] = []self.usage: dict[str, float] = {}+ self.cost: dict[str, float] = {}self.terminal: TerminalState | None = Noneself._event_index = 0self._tool_started_at: dict[str, int] = {}@@ -64,7 +65,8 @@ class CodexEventCollector:self._item_completed(_item_dict(payload), payload_data)elif method == "thread/tokenUsage/updated":self.usage = _usage_dict(payload_data.get("tokenUsage"))- self.emit("usage", {"usage": self.usage})+ self.cost = _cost_dict(payload_data.get("cost"))+ self.emit("usage", {"usage": self.usage, "cost": self.cost})elif method == "turn/started":self.emit("turn_started", {"turn_id": _turn_id(payload_data)})elif method == "turn/completed":@@ -148,6 +150,7 @@ class CodexEventCollector:failover_reason=failover_reason,credential_ref=self.request.credential_ref,usage=dict(self.usage),+ cost=dict(self.cost),terminal_status=terminal_status,timed_out=timed_out,cancelled=interrupted_by_stop,@@ -330,6 +333,16 @@ def _usage_dict(value: Any) -> dict[str, float]:}+def _cost_dict(value: Any) -> dict[str, float]:+ if not isinstance(value, dict):+ return {}+ return {+ str(key): float(val)+ for key, val in value.items()+ if isinstance(val, (int, float)) and not isinstance(val, bool)+ }++def _error_text(value: Any) -> str | None:if value in (None, "", {}):return Nonediff --git a/praxist/plugins/workflow_stages/research_loop/stage.py b/praxist/plugins/workflow_stages/research_loop/stage.pyindex 774cd9a..4a8dc49 100644--- a/praxist/plugins/workflow_stages/research_loop/stage.py+++ b/praxist/plugins/workflow_stages/research_loop/stage.py@@ -10,6 +10,7 @@ from dataclasses import dataclassfrom pathlib import Pathfrom typing import Any+from praxist.core.budget import INFORMATIONAL_USAGE_UNITSfrom praxist.core.ledgers import BudgetLedgerfrom praxist.core.role_skills import RoleSkill, load_role_skillfrom praxist.core.runtimes import close_runtime_for_ref, collect_runtime_usage@@ -303,8 +304,11 @@ def _usage_for_grant(result: dict[str, Any] | None = None,allow_zero_for_unmeasured: bool = False,) -> dict[str, float]:+approved = grant_record.get("granted_budget") or {}usage: dict[str, float] = {}++ # Collect approved unitsif isinstance(approved, dict):for unit in approved:unit_name = str(unit)@@ -313,6 +317,13 @@ def _usage_for_grant(usage[unit_name] = measuredelif allow_zero_for_unmeasured:usage[unit_name] = 0.0++ # Keep provider-reported billing units even when the grant did not set a cost ceiling.+ if isinstance(result, dict) and isinstance(result.get("runtime_usage"), dict):+ for unit_name, value in result["runtime_usage"].items():+ if unit_name in INFORMATIONAL_USAGE_UNITS and unit_name not in usage:+ usage[unit_name] = float(value)+if ("wall_clock_seconds" in usageor (isinstance(approved, dict) and "wall_clock_seconds" in approved)diff --git a/tests/core/test_resource_guards.py b/tests/core/test_resource_guards.pyindex b1431db..755a175 100644--- a/tests/core/test_resource_guards.py+++ b/tests/core/test_resource_guards.py@@ -81,6 +81,128 @@ class Step16ResourceGuardMigrationTest(unittest.TestCase):unknown = [record for record in records if record["kind"] == "usage_unknown"][-1]self.assertEqual(unknown["unknown_units"], ["gpu_hours"])+ def test_budgeted_action_guard_blocks_exhausted_budget(self) -> None:+ with tempfile.TemporaryDirectory() as tmp:+ run_dir = Path(tmp) / "run_guard_block"+ run_dir.mkdir()+ grant_id, request_id = _write_budget_grant(+ run_dir,+ {"wall_clock_seconds": 10.0},+ )++ guard1 = BudgetedActionGuard(+ run_dir=run_dir,+ run_id=run_dir.name,+ stage_id="research_loop",+ actor_ref="resource_guard:test",+ action_type="eval_runner",+ budget_grant_id=grant_id,+ request_id=request_id,+ require_budget_grant=True,+ )+ guard1.start()+ guard1.finish(+ actual_usage={"wall_clock_seconds": 15.0},+ expected_units=("wall_clock_seconds",),+ reason="test_eval_runner_usage",+ )++ guard2 = BudgetedActionGuard(+ run_dir=run_dir,+ run_id=run_dir.name,+ stage_id="research_loop",+ actor_ref="resource_guard:test",+ action_type="eval_runner",+ budget_grant_id=grant_id,+ request_id=request_id,+ require_budget_grant=True,+ )+ with self.assertRaises(ResourceBudgetError) as ctx:+ guard2.start()+ self.assertIn("Budget exhausted for units: wall_clock_seconds", str(ctx.exception))++ def test_exhaustion_probe_rejects_malformed_grant_budget(self) -> None:+ with tempfile.TemporaryDirectory() as tmp:+ run_dir = Path(tmp) / "run_malformed_grant"+ run_dir.mkdir()+ ledger = BudgetLedger(run_dir, run_dir.name)++ with (+ patch.object(+ ledger,+ "require_active_grant",+ return_value={"granted_budget": ["tokens"]},+ ),+ self.assertRaisesRegex(ValueError, "invalid approved budget"),+ ):+ ledger.get_exhausted_units("grant-malformed")++ def test_exhaustion_probe_ignores_informational_units(self) -> None:+ with tempfile.TemporaryDirectory() as tmp:+ run_dir = Path(tmp) / "run_informational_grant"+ run_dir.mkdir()+ grant_id, request_id = _write_budget_grant(+ run_dir,+ {"tokens": 1000.0, "cost_usd": 1.0},+ )+ ledger = BudgetLedger(run_dir, run_dir.name)+ ledger.append_usage(+ request_id=request_id,+ grant_id=grant_id,+ actor_ref="resource_guard:test",+ stage_id="research_loop",+ action_type="eval_runner",+ actual_usage={"tokens": 10.0, "cost_usd": 1.0},+ reason="informational_units_do_not_exhaust",+ )++ # cost_usd has reached its granted amount but must not gate execution+ self.assertEqual(ledger.get_exhausted_units(grant_id), [])++ ledger.append_usage(+ request_id=request_id,+ grant_id=grant_id,+ actor_ref="resource_guard:test",+ stage_id="research_loop",+ action_type="eval_runner",+ actual_usage={"tokens": 990.0},+ reason="enforced_unit_still_exhausts",+ )+ self.assertEqual(ledger.get_exhausted_units(grant_id), ["tokens"])++ def test_budgeted_action_allows_informational_cost_metrics(self) -> None:+ with tempfile.TemporaryDirectory() as tmp:+ run_dir = Path(tmp) / "run_cost"+ run_dir.mkdir()+ grant_id, request_id = _write_budget_grant(+ run_dir,+ {"tokens": 1000.0},+ )++ guard = BudgetedActionGuard(+ run_dir=run_dir,+ run_id=run_dir.name,+ stage_id="research_loop",+ actor_ref="resource_guard:test",+ action_type="eval_runner",+ budget_grant_id=grant_id,+ request_id=request_id,+ require_budget_grant=True,+ )+ guard.start()+ report = guard.finish(+ actual_usage={"tokens": 500.0, "cost_usd": 0.05, "neurons": 200.0},+ expected_units=("tokens", "cost_usd", "neurons"),+ reason="test_cost_usage",+ )++ self.assertTrue(report.recorded)+ records = BudgetLedger(run_dir, run_dir.name).records()+ usage = [record for record in records if record["kind"] == "usage"][-1]+ self.assertEqual(usage["actual_usage"]["tokens"], 500.0)+ self.assertEqual(usage["actual_usage"]["cost_usd"], 0.05)+ self.assertEqual(usage["actual_usage"]["neurons"], 200.0)+def test_wait_for_file_records_tool_wall_clock_usage_from_env(self) -> None:with tempfile.TemporaryDirectory() as tmp:run_dir = Path(tmp) / "run_wait"diff --git a/tests/unit/test_codex_sdk_events.py b/tests/unit/test_codex_sdk_events.pyindex 2629781..796e620 100644--- a/tests/unit/test_codex_sdk_events.py+++ b/tests/unit/test_codex_sdk_events.py@@ -228,7 +228,8 @@ class CodexEventCollectorTest(unittest.TestCase):"reasoningOutputTokens": 2,"totalTokens": 18,}- }+ },+ "cost": {"cost_usd": 0.125},},))@@ -240,6 +241,7 @@ class CodexEventCollectorTest(unittest.TestCase):self.assertEqual(result.terminal_status, "completed")self.assertEqual(result.usage["cached_input_tokens"], 7.0)self.assertEqual(result.usage["total_tokens"], 18.0)+ self.assertEqual(result.cost, {"cost_usd": 0.125})self.assertEqual([event.type for event in result.events][-1], "final_result")self.assertTrue(result.events[-1].payload["relay_used"])diff --git a/tests/unit/test_coverage_95_contracts.py b/tests/unit/test_coverage_95_contracts.pyindex e40c313..3568811 100644--- a/tests/unit/test_coverage_95_contracts.py+++ b/tests/unit/test_coverage_95_contracts.py@@ -1919,9 +1919,17 @@ class CoreContractCoverage95Test(unittest.TestCase):failed = adapter.normalize_result({"error": "bad key", "model": "m"})self.assertFalse(failed.success)self.assertEqual(failed.failover_reason, "runtime_error")- ok = adapter.normalize_result({"text": "hello", "model": "m", "usage": {"tokens": 2}})+ ok = adapter.normalize_result(+ {+ "text": "hello",+ "model": "m",+ "usage": {"tokens": 2},+ "cost": {"cost_usd": 0.5},+ }+ )self.assertTrue(ok.success)self.assertEqual(ok.usage["tokens"], 2.0)+ self.assertEqual(ok.cost["cost_usd"], 0.5)profile = ModelProfile(profile_id="p",diff --git a/tests/unit/test_coverage_contracts.py b/tests/unit/test_coverage_contracts.pyindex ac84433..bfaa0ab 100644--- a/tests/unit/test_coverage_contracts.py+++ b/tests/unit/test_coverage_contracts.py@@ -158,6 +158,21 @@ class ProtocolCoverageContractsTest(unittest.TestCase):],1,)+ legacy_result = AgentRunResult(+ True,+ [],+ [],+ [],+ None,+ None,+ None,+ {"tokens": 1},+ "completed",+ False,+ False,+ )+ self.assertEqual(legacy_result.terminal_status, "completed")+ self.assertEqual(legacy_result.cost, {})self.assertEqual(ToolCallResult("s", "t", True, {"ok": True}).to_dict()["output"]["ok"], True)diff --git a/tests/unit/test_replay_helper_contracts.py b/tests/unit/test_replay_helper_contracts.pyindex a947357..32b5dd6 100644--- a/tests/unit/test_replay_helper_contracts.py+++ b/tests/unit/test_replay_helper_contracts.py@@ -316,7 +316,12 @@ class ReplayHelperContractsTest(unittest.TestCase):"kind": "usage","record_id": "u1","grant_id": "g1",- "actual_usage": {"tokens": 3, "wall_clock_seconds": "oops", "gpu_hours": 1},+ "actual_usage": {+ "tokens": 3,+ "wall_clock_seconds": "oops",+ "gpu_hours": 1,+ "cost_usd": 0.25,+ },},{"kind": "usage",@@ -348,6 +353,7 @@ class ReplayHelperContractsTest(unittest.TestCase):self.assertIn("references unknown grant_id", joined_errors)self.assertIn("unknown_units is not a list", joined_errors)self.assertIn("uses unapproved budget unit: gpu_hours", joined_errors)+ self.assertNotIn("uses unapproved budget unit: cost_usd", joined_errors)self.assertIn("non-numeric usage", joined_errors)self.assertIn("uses unapproved budget unit: gpu_hours", joined_errors)self.assertIn("non-numeric approval", joined_errors)diff --git a/tests/unit/test_research_loop_usage_aggregation.py b/tests/unit/test_research_loop_usage_aggregation.pyindex 5f2ad8a..11b3576 100644--- a/tests/unit/test_research_loop_usage_aggregation.py+++ b/tests/unit/test_research_loop_usage_aggregation.py@@ -20,6 +20,7 @@ from praxist.plugins.workflow_stages.research_loop.backend import agentdef _result(*,usage: dict[str, float],+ cost: dict[str, float] | None = None,success: bool = True,error: str | None = None,) -> runtime_core.AgentRunResult:@@ -32,6 +33,7 @@ def _result(failover_reason=None,credential_ref=None,usage=usage,+ cost=dict(cost or {}),)@@ -86,13 +88,18 @@ class ResearchLoopUsageAggregationTest(unittest.TestCase):{"total_tokens": 3.0},{"input_tokens": 7.0, "output_tokens": 1.0},]+ per_call_cost = [0.01, 0.02, 0.03, 0.04]observed_agent_usage: list[dict[str, float] | None] = []class FakeRuntime:async def execute(self, _request: object, _context: object) -> runtime_core.AgentRunResult:- return _result(usage=dict(per_call_usage[len(observed_agent_usage)]))+ index = len(observed_agent_usage)+ return _result(+ usage=dict(per_call_usage[index]),+ cost={"cost_usd": per_call_cost[index]},+ )class FakeGenerationLoop:def __init__(self, **kwargs: object) -> None:@@ -131,11 +138,16 @@ class ResearchLoopUsageAggregationTest(unittest.TestCase):"input_tokens": 22.0,"output_tokens": 8.0,"total_tokens": 33.0,+ "cost_usd": 0.1,},)self.assertEqual(stage_result.summary["total_tokens"], 33.0)actual_usage = _Ledger.instances[0].usage_records[0]["actual_usage"]self.assertEqual(actual_usage["tokens"], 33.0)+ self.assertEqual(actual_usage["cost_usd"], 0.1)+ self.assertNotIn("input_tokens", actual_usage)+ self.assertNotIn("output_tokens", actual_usage)+ self.assertNotIn("total_tokens", actual_usage)self.assertIn("wall_clock_seconds", actual_usage)self.assertEqual(_Ledger.instances[0].unknown_records, [])