Skip to content

Autonomous research system for measurable, computer-executable research.

Python

fix(budget): enforce token grants and record provider cost in the ledger (#199)

Glen-SP9d6932d6b415
diff --git a/praxist/core/budget.py b/praxist/core/budget.py
index d2b76a2..5071fca 100644
--- a/praxist/core/budget.py
+++ b/praxist/core/budget.py
@@ -14,7 +14,10 @@ from praxist.core.registry import (
require_execution_plugin,
)
-ALLOWED_BUDGET_UNITS = {"tokens", "wall_clock_seconds", "gpu_hours"}
+INFORMATIONAL_USAGE_UNITS = frozenset({"cost_usd", "neurons"})
+ALLOWED_BUDGET_UNITS = {"tokens", "wall_clock_seconds", "gpu_hours"} | set(
+ INFORMATIONAL_USAGE_UNITS
+)
class BudgetPolicy(Protocol):
diff --git a/praxist/core/execution_guards.py b/praxist/core/execution_guards.py
index 3492ad8..0b84430 100644
--- a/praxist/core/execution_guards.py
+++ b/praxist/core/execution_guards.py
@@ -10,6 +10,7 @@ from datetime import UTC, datetime
from pathlib import Path
from typing import Any
+from praxist.core.budget import INFORMATIONAL_USAGE_UNITS
from praxist.core.ledgers import BudgetLedger
from praxist.core.run_config import RunConfig
from praxist.core.trajectory import TrajectoryWriter
@@ -129,16 +130,27 @@ class BudgetedActionGuard:
raise ResourceBudgetError(f"{self.action_type} requires an approved budget grant")
if self.budget_grant_id and self.run_dir is not None:
try:
- BudgetLedger(self.run_dir, self.run_id).require_active_grant(self.budget_grant_id)
+ ledger = BudgetLedger(self.run_dir, self.run_id)
+ ledger.require_active_grant(self.budget_grant_id)
+ exhausted = ledger.get_exhausted_units(self.budget_grant_id)
+ if exhausted:
+ raise ResourceBudgetError(
+ f"Budget exhausted for units: {', '.join(sorted(exhausted))}"
+ )
except Exception as exc: # noqa: BLE001 - preflight should report the real reason.
+ is_overrun = isinstance(exc, ResourceBudgetError)
self._emit_event(
"resource.action_denied"
- if self.require_budget_grant
+ if self.require_budget_grant or is_overrun
else "resource.action_budget_warning",
- severity="error" if self.require_budget_grant else "warning",
- payload={"reason": "invalid_budget_grant", "error": str(exc), **self.metadata},
+ severity="error" if self.require_budget_grant or is_overrun else "warning",
+ payload={
+ "reason": "budget_overrun" if is_overrun else "invalid_budget_grant",
+ "error": str(exc),
+ **self.metadata,
+ },
)
- if self.require_budget_grant:
+ if self.require_budget_grant or is_overrun:
raise ResourceBudgetError(str(exc)) from exc
self._emit_event("resource.action_started", payload=self.metadata)
@@ -173,7 +185,11 @@ class BudgetedActionGuard:
approved = grant.get("granted_budget") or {}
if not isinstance(approved, dict):
raise ValueError(f"invalid granted budget for {self.budget_grant_id}")
- usage = {unit: value for unit, value in usage.items() if unit in approved}
+ usage = {
+ unit: value
+ for unit, value in usage.items()
+ if unit in approved or unit in INFORMATIONAL_USAGE_UNITS
+ }
if "wall_clock_seconds" in expected_units and "wall_clock_seconds" in approved:
usage.setdefault("wall_clock_seconds", elapsed)
unknown_units = [
diff --git a/praxist/core/ledgers.py b/praxist/core/ledgers.py
index 821acd9..7600bdb 100644
--- a/praxist/core/ledgers.py
+++ b/praxist/core/ledgers.py
@@ -8,7 +8,7 @@ from dataclasses import asdict
from pathlib import Path
from typing import Any
-from praxist.core.budget import ALLOWED_BUDGET_UNITS
+from praxist.core.budget import ALLOWED_BUDGET_UNITS, INFORMATIONAL_USAGE_UNITS
from praxist.core.protocol import BudgetDecision, BudgetRequest
from praxist.core.storage import append_jsonl, read_jsonl, utc_now
@@ -213,6 +213,26 @@ class BudgetLedger:
raise ValueError(f"Budget grant not found: {grant_id}")
return grants[grant_id]
+ def get_exhausted_units(self, grant_id: str) -> list[str]:
+ grant = self.require_active_grant(grant_id)
+ approved = grant.get("granted_budget") or {}
+ if not isinstance(approved, dict):
+ raise ValueError(f"Budget grant has invalid approved budget: {grant_id}")
+
+ totals = self._usage_totals_by_grant().get(grant_id, {})
+ exhausted_units = []
+ for unit, raw_amount in approved.items():
+ if unit in INFORMATIONAL_USAGE_UNITS:
+ continue # recorded for reporting; never gates execution
+ try:
+ allowed = float(raw_amount)
+ used = float(totals.get(unit, 0.0))
+ if used >= allowed and allowed > 0:
+ exhausted_units.append(str(unit))
+ except (TypeError, ValueError):
+ continue
+ return exhausted_units
+
def _require_usage_within_grant(
self, grant_id: str, actual_usage: dict[str, float]
) -> list[dict[str, Any]]:
@@ -230,6 +250,8 @@ class BudgetLedger:
f"Budget usage must be finite and non-negative for {unit}: {raw_amount}"
)
if unit not in approved:
+ if unit in INFORMATIONAL_USAGE_UNITS:
+ continue
raise ValueError(f"Budget usage unit not approved by grant {grant_id}: {unit}")
approved_amount = float(approved[unit])
new_total = float(totals.get(unit, 0.0)) + amount
diff --git a/praxist/core/modeling.py b/praxist/core/modeling.py
index 991c4e2..dedbb1e 100644
--- a/praxist/core/modeling.py
+++ b/praxist/core/modeling.py
@@ -103,16 +103,19 @@ class ModelProviderAdapter:
model=str(raw_redacted.get("model", "")),
text=None,
usage={},
+ cost={},
error=str(raw_redacted.get("error")),
failover_reason=self.classify_error(raw_redacted),
)
usage = raw_redacted.get("usage")
+ cost = raw_redacted.get("cost")
return ModelResult(
success=True,
provider_ref=self.provider_ref,
model=str(raw_redacted.get("model", "")),
text=str(raw_redacted.get("text", "")),
usage={str(key): float(value) for key, value in dict(usage or {}).items()},
+ cost={str(key): float(value) for key, value in dict(cost or {}).items()},
error=None,
failover_reason="none",
)
diff --git a/praxist/core/protocol.py b/praxist/core/protocol.py
index 8e25c6a..fbd9783 100644
--- a/praxist/core/protocol.py
+++ b/praxist/core/protocol.py
@@ -68,6 +68,7 @@ class ModelResult:
usage: dict[str, float]
error: str | None
failover_reason: str | None
+ cost: dict[str, float] = field(default_factory=dict)
def to_dict(self) -> dict[str, Any]:
return asdict(self)
@@ -264,6 +265,7 @@ class AgentRunResult:
terminal_status: str | None = None
timed_out: bool = False
cancelled: bool = False
+ cost: dict[str, float] = field(default_factory=dict)
def to_dict(self) -> dict[str, Any]:
return {
@@ -275,6 +277,7 @@ class AgentRunResult:
"failover_reason": self.failover_reason,
"credential_ref": self.credential_ref.to_dict() if self.credential_ref else None,
"usage": self.usage,
+ "cost": self.cost,
"terminal_status": self.terminal_status,
"timed_out": self.timed_out,
"cancelled": self.cancelled,
diff --git a/praxist/core/replay.py b/praxist/core/replay.py
index d5dbfb3..254e49f 100644
--- a/praxist/core/replay.py
+++ b/praxist/core/replay.py
@@ -16,7 +16,11 @@ from typing import Any
import yaml
-from praxist.core.budget import ALLOWED_BUDGET_UNITS, policy_for_ref
+from praxist.core.budget import (
+ ALLOWED_BUDGET_UNITS,
+ INFORMATIONAL_USAGE_UNITS,
+ policy_for_ref,
+)
from praxist.core.prompt_layout import sha256_json, sha256_text
from praxist.core.protocol import BudgetRequest
from praxist.core.redaction import scan_file
@@ -615,6 +619,8 @@ def _verify_budget_ledger(
)
continue
if unit not in approved:
+ if unit in INFORMATIONAL_USAGE_UNITS:
+ continue
errors.append(
f"budget usage {record.get('record_id')} uses unapproved budget unit: {unit}"
)
diff --git a/praxist/core/runtimes.py b/praxist/core/runtimes.py
index a606d7c..7f1d9e3 100644
--- a/praxist/core/runtimes.py
+++ b/praxist/core/runtimes.py
@@ -227,6 +227,8 @@ def _collect_result_usage(result: AgentRunResult) -> None:
collector = _runtime_usage_collector.get()
if collector is not None:
collector.add(result.usage)
+ if hasattr(result, "cost") and result.cost:
+ collector.add(result.cost)
def prompt_text_for_request(request: AgentRunRequest) -> str:
diff --git a/praxist/plugins/agent_runtimes/codex_sdk/_events.py b/praxist/plugins/agent_runtimes/codex_sdk/_events.py
index 8c8ab13..48a5322 100644
--- a/praxist/plugins/agent_runtimes/codex_sdk/_events.py
+++ b/praxist/plugins/agent_runtimes/codex_sdk/_events.py
@@ -42,6 +42,7 @@ class CodexEventCollector:
self.tool_uses: list[ToolCallRecord] = []
self.legacy_tool_uses: list[dict[str, Any]] = []
self.usage: dict[str, float] = {}
+ self.cost: dict[str, float] = {}
self.terminal: TerminalState | None = None
self._event_index = 0
self._tool_started_at: dict[str, int] = {}
@@ -64,7 +65,8 @@ class CodexEventCollector:
self._item_completed(_item_dict(payload), payload_data)
elif method == "thread/tokenUsage/updated":
self.usage = _usage_dict(payload_data.get("tokenUsage"))
- self.emit("usage", {"usage": self.usage})
+ self.cost = _cost_dict(payload_data.get("cost"))
+ self.emit("usage", {"usage": self.usage, "cost": self.cost})
elif method == "turn/started":
self.emit("turn_started", {"turn_id": _turn_id(payload_data)})
elif method == "turn/completed":
@@ -148,6 +150,7 @@ class CodexEventCollector:
failover_reason=failover_reason,
credential_ref=self.request.credential_ref,
usage=dict(self.usage),
+ cost=dict(self.cost),
terminal_status=terminal_status,
timed_out=timed_out,
cancelled=interrupted_by_stop,
@@ -330,6 +333,16 @@ def _usage_dict(value: Any) -> dict[str, float]:
}
+def _cost_dict(value: Any) -> dict[str, float]:
+ if not isinstance(value, dict):
+ return {}
+ return {
+ str(key): float(val)
+ for key, val in value.items()
+ if isinstance(val, (int, float)) and not isinstance(val, bool)
+ }
+
+
def _error_text(value: Any) -> str | None:
if value in (None, "", {}):
return None
diff --git a/praxist/plugins/workflow_stages/research_loop/stage.py b/praxist/plugins/workflow_stages/research_loop/stage.py
index 774cd9a..4a8dc49 100644
--- a/praxist/plugins/workflow_stages/research_loop/stage.py
+++ b/praxist/plugins/workflow_stages/research_loop/stage.py
@@ -10,6 +10,7 @@ from dataclasses import dataclass
from pathlib import Path
from typing import Any
+from praxist.core.budget import INFORMATIONAL_USAGE_UNITS
from praxist.core.ledgers import BudgetLedger
from praxist.core.role_skills import RoleSkill, load_role_skill
from praxist.core.runtimes import close_runtime_for_ref, collect_runtime_usage
@@ -303,8 +304,11 @@ def _usage_for_grant(
result: dict[str, Any] | None = None,
allow_zero_for_unmeasured: bool = False,
) -> dict[str, float]:
+
approved = grant_record.get("granted_budget") or {}
usage: dict[str, float] = {}
+
+ # Collect approved units
if isinstance(approved, dict):
for unit in approved:
unit_name = str(unit)
@@ -313,6 +317,13 @@ def _usage_for_grant(
usage[unit_name] = measured
elif allow_zero_for_unmeasured:
usage[unit_name] = 0.0
+
+ # Keep provider-reported billing units even when the grant did not set a cost ceiling.
+ if isinstance(result, dict) and isinstance(result.get("runtime_usage"), dict):
+ for unit_name, value in result["runtime_usage"].items():
+ if unit_name in INFORMATIONAL_USAGE_UNITS and unit_name not in usage:
+ usage[unit_name] = float(value)
+
if (
"wall_clock_seconds" in usage
or (isinstance(approved, dict) and "wall_clock_seconds" in approved)
diff --git a/tests/core/test_resource_guards.py b/tests/core/test_resource_guards.py
index b1431db..755a175 100644
--- a/tests/core/test_resource_guards.py
+++ b/tests/core/test_resource_guards.py
@@ -81,6 +81,128 @@ class Step16ResourceGuardMigrationTest(unittest.TestCase):
unknown = [record for record in records if record["kind"] == "usage_unknown"][-1]
self.assertEqual(unknown["unknown_units"], ["gpu_hours"])
+ def test_budgeted_action_guard_blocks_exhausted_budget(self) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ run_dir = Path(tmp) / "run_guard_block"
+ run_dir.mkdir()
+ grant_id, request_id = _write_budget_grant(
+ run_dir,
+ {"wall_clock_seconds": 10.0},
+ )
+
+ guard1 = BudgetedActionGuard(
+ run_dir=run_dir,
+ run_id=run_dir.name,
+ stage_id="research_loop",
+ actor_ref="resource_guard:test",
+ action_type="eval_runner",
+ budget_grant_id=grant_id,
+ request_id=request_id,
+ require_budget_grant=True,
+ )
+ guard1.start()
+ guard1.finish(
+ actual_usage={"wall_clock_seconds": 15.0},
+ expected_units=("wall_clock_seconds",),
+ reason="test_eval_runner_usage",
+ )
+
+ guard2 = BudgetedActionGuard(
+ run_dir=run_dir,
+ run_id=run_dir.name,
+ stage_id="research_loop",
+ actor_ref="resource_guard:test",
+ action_type="eval_runner",
+ budget_grant_id=grant_id,
+ request_id=request_id,
+ require_budget_grant=True,
+ )
+ with self.assertRaises(ResourceBudgetError) as ctx:
+ guard2.start()
+ self.assertIn("Budget exhausted for units: wall_clock_seconds", str(ctx.exception))
+
+ def test_exhaustion_probe_rejects_malformed_grant_budget(self) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ run_dir = Path(tmp) / "run_malformed_grant"
+ run_dir.mkdir()
+ ledger = BudgetLedger(run_dir, run_dir.name)
+
+ with (
+ patch.object(
+ ledger,
+ "require_active_grant",
+ return_value={"granted_budget": ["tokens"]},
+ ),
+ self.assertRaisesRegex(ValueError, "invalid approved budget"),
+ ):
+ ledger.get_exhausted_units("grant-malformed")
+
+ def test_exhaustion_probe_ignores_informational_units(self) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ run_dir = Path(tmp) / "run_informational_grant"
+ run_dir.mkdir()
+ grant_id, request_id = _write_budget_grant(
+ run_dir,
+ {"tokens": 1000.0, "cost_usd": 1.0},
+ )
+ ledger = BudgetLedger(run_dir, run_dir.name)
+ ledger.append_usage(
+ request_id=request_id,
+ grant_id=grant_id,
+ actor_ref="resource_guard:test",
+ stage_id="research_loop",
+ action_type="eval_runner",
+ actual_usage={"tokens": 10.0, "cost_usd": 1.0},
+ reason="informational_units_do_not_exhaust",
+ )
+
+ # cost_usd has reached its granted amount but must not gate execution
+ self.assertEqual(ledger.get_exhausted_units(grant_id), [])
+
+ ledger.append_usage(
+ request_id=request_id,
+ grant_id=grant_id,
+ actor_ref="resource_guard:test",
+ stage_id="research_loop",
+ action_type="eval_runner",
+ actual_usage={"tokens": 990.0},
+ reason="enforced_unit_still_exhausts",
+ )
+ self.assertEqual(ledger.get_exhausted_units(grant_id), ["tokens"])
+
+ def test_budgeted_action_allows_informational_cost_metrics(self) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ run_dir = Path(tmp) / "run_cost"
+ run_dir.mkdir()
+ grant_id, request_id = _write_budget_grant(
+ run_dir,
+ {"tokens": 1000.0},
+ )
+
+ guard = BudgetedActionGuard(
+ run_dir=run_dir,
+ run_id=run_dir.name,
+ stage_id="research_loop",
+ actor_ref="resource_guard:test",
+ action_type="eval_runner",
+ budget_grant_id=grant_id,
+ request_id=request_id,
+ require_budget_grant=True,
+ )
+ guard.start()
+ report = guard.finish(
+ actual_usage={"tokens": 500.0, "cost_usd": 0.05, "neurons": 200.0},
+ expected_units=("tokens", "cost_usd", "neurons"),
+ reason="test_cost_usage",
+ )
+
+ self.assertTrue(report.recorded)
+ records = BudgetLedger(run_dir, run_dir.name).records()
+ usage = [record for record in records if record["kind"] == "usage"][-1]
+ self.assertEqual(usage["actual_usage"]["tokens"], 500.0)
+ self.assertEqual(usage["actual_usage"]["cost_usd"], 0.05)
+ self.assertEqual(usage["actual_usage"]["neurons"], 200.0)
+
def test_wait_for_file_records_tool_wall_clock_usage_from_env(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
run_dir = Path(tmp) / "run_wait"
diff --git a/tests/unit/test_codex_sdk_events.py b/tests/unit/test_codex_sdk_events.py
index 2629781..796e620 100644
--- a/tests/unit/test_codex_sdk_events.py
+++ b/tests/unit/test_codex_sdk_events.py
@@ -228,7 +228,8 @@ class CodexEventCollectorTest(unittest.TestCase):
"reasoningOutputTokens": 2,
"totalTokens": 18,
}
- }
+ },
+ "cost": {"cost_usd": 0.125},
},
)
)
@@ -240,6 +241,7 @@ class CodexEventCollectorTest(unittest.TestCase):
self.assertEqual(result.terminal_status, "completed")
self.assertEqual(result.usage["cached_input_tokens"], 7.0)
self.assertEqual(result.usage["total_tokens"], 18.0)
+ self.assertEqual(result.cost, {"cost_usd": 0.125})
self.assertEqual([event.type for event in result.events][-1], "final_result")
self.assertTrue(result.events[-1].payload["relay_used"])
diff --git a/tests/unit/test_coverage_95_contracts.py b/tests/unit/test_coverage_95_contracts.py
index e40c313..3568811 100644
--- a/tests/unit/test_coverage_95_contracts.py
+++ b/tests/unit/test_coverage_95_contracts.py
@@ -1919,9 +1919,17 @@ class CoreContractCoverage95Test(unittest.TestCase):
failed = adapter.normalize_result({"error": "bad key", "model": "m"})
self.assertFalse(failed.success)
self.assertEqual(failed.failover_reason, "runtime_error")
- ok = adapter.normalize_result({"text": "hello", "model": "m", "usage": {"tokens": 2}})
+ ok = adapter.normalize_result(
+ {
+ "text": "hello",
+ "model": "m",
+ "usage": {"tokens": 2},
+ "cost": {"cost_usd": 0.5},
+ }
+ )
self.assertTrue(ok.success)
self.assertEqual(ok.usage["tokens"], 2.0)
+ self.assertEqual(ok.cost["cost_usd"], 0.5)
profile = ModelProfile(
profile_id="p",
diff --git a/tests/unit/test_coverage_contracts.py b/tests/unit/test_coverage_contracts.py
index ac84433..bfaa0ab 100644
--- a/tests/unit/test_coverage_contracts.py
+++ b/tests/unit/test_coverage_contracts.py
@@ -158,6 +158,21 @@ class ProtocolCoverageContractsTest(unittest.TestCase):
],
1,
)
+ legacy_result = AgentRunResult(
+ True,
+ [],
+ [],
+ [],
+ None,
+ None,
+ None,
+ {"tokens": 1},
+ "completed",
+ False,
+ False,
+ )
+ self.assertEqual(legacy_result.terminal_status, "completed")
+ self.assertEqual(legacy_result.cost, {})
self.assertEqual(
ToolCallResult("s", "t", True, {"ok": True}).to_dict()["output"]["ok"], True
)
diff --git a/tests/unit/test_replay_helper_contracts.py b/tests/unit/test_replay_helper_contracts.py
index a947357..32b5dd6 100644
--- a/tests/unit/test_replay_helper_contracts.py
+++ b/tests/unit/test_replay_helper_contracts.py
@@ -316,7 +316,12 @@ class ReplayHelperContractsTest(unittest.TestCase):
"kind": "usage",
"record_id": "u1",
"grant_id": "g1",
- "actual_usage": {"tokens": 3, "wall_clock_seconds": "oops", "gpu_hours": 1},
+ "actual_usage": {
+ "tokens": 3,
+ "wall_clock_seconds": "oops",
+ "gpu_hours": 1,
+ "cost_usd": 0.25,
+ },
},
{
"kind": "usage",
@@ -348,6 +353,7 @@ class ReplayHelperContractsTest(unittest.TestCase):
self.assertIn("references unknown grant_id", joined_errors)
self.assertIn("unknown_units is not a list", joined_errors)
self.assertIn("uses unapproved budget unit: gpu_hours", joined_errors)
+ self.assertNotIn("uses unapproved budget unit: cost_usd", joined_errors)
self.assertIn("non-numeric usage", joined_errors)
self.assertIn("uses unapproved budget unit: gpu_hours", joined_errors)
self.assertIn("non-numeric approval", joined_errors)
diff --git a/tests/unit/test_research_loop_usage_aggregation.py b/tests/unit/test_research_loop_usage_aggregation.py
index 5f2ad8a..11b3576 100644
--- a/tests/unit/test_research_loop_usage_aggregation.py
+++ b/tests/unit/test_research_loop_usage_aggregation.py
@@ -20,6 +20,7 @@ from praxist.plugins.workflow_stages.research_loop.backend import agent
def _result(
*,
usage: dict[str, float],
+ cost: dict[str, float] | None = None,
success: bool = True,
error: str | None = None,
) -> runtime_core.AgentRunResult:
@@ -32,6 +33,7 @@ def _result(
failover_reason=None,
credential_ref=None,
usage=usage,
+ cost=dict(cost or {}),
)
@@ -86,13 +88,18 @@ class ResearchLoopUsageAggregationTest(unittest.TestCase):
{"total_tokens": 3.0},
{"input_tokens": 7.0, "output_tokens": 1.0},
]
+ per_call_cost = [0.01, 0.02, 0.03, 0.04]
observed_agent_usage: list[dict[str, float] | None] = []
class FakeRuntime:
async def execute(
self, _request: object, _context: object
) -> runtime_core.AgentRunResult:
- return _result(usage=dict(per_call_usage[len(observed_agent_usage)]))
+ index = len(observed_agent_usage)
+ return _result(
+ usage=dict(per_call_usage[index]),
+ cost={"cost_usd": per_call_cost[index]},
+ )
class FakeGenerationLoop:
def __init__(self, **kwargs: object) -> None:
@@ -131,11 +138,16 @@ class ResearchLoopUsageAggregationTest(unittest.TestCase):
"input_tokens": 22.0,
"output_tokens": 8.0,
"total_tokens": 33.0,
+ "cost_usd": 0.1,
},
)
self.assertEqual(stage_result.summary["total_tokens"], 33.0)
actual_usage = _Ledger.instances[0].usage_records[0]["actual_usage"]
self.assertEqual(actual_usage["tokens"], 33.0)
+ self.assertEqual(actual_usage["cost_usd"], 0.1)
+ self.assertNotIn("input_tokens", actual_usage)
+ self.assertNotIn("output_tokens", actual_usage)
+ self.assertNotIn("total_tokens", actual_usage)
self.assertIn("wall_clock_seconds", actual_usage)
self.assertEqual(_Ledger.instances[0].unknown_records, [])