Open-source AI penetration testing tool to find and fix your app’s vulnerabilities.
Python agentsai-hackingai-penetration-testingai-pentestingai-securityartificial-intelligencebug-bountycode-qualityctf-toolscybersecuritycybersecurity-toolsethical-hackinghackingllm-securityoffensive-securitypenetration-testingpentesting-toolsred-teamingsecuritysecurity-automation
Use observed LiteLLM cost for LiteLLM-routed calls (#529)
Ahmed Allam1c9ab993bbed
diff --git a/pyproject.toml b/pyproject.tomlindex ce97bbc0..d12c3b38 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -219,6 +219,9 @@ ignore = [# ReportState carries scan artifact/report fields and# a runtime ``Callable`` annotation on ``vulnerability_found_callback``."strix/report/state.py" = ["TC003", "PLR0912", "PLR0915", "E501", "PERF401"]+"strix/report/usage.py" = ["PLC0415"]+"strix/report/cost_capture.py" = ["PLC0415"]+"strix/config/models.py" = ["PLC0415"]# Interface utility branches per scope-mode / target-type combination;# splitting would obscure the decision tree without simplifying it."strix/interface/utils.py" = ["PLR0912", "BLE001", "PLC0415"]diff --git a/strix/config/models.py b/strix/config/models.pyindex 4d2e072e..6c050fc4 100644--- a/strix/config/models.py+++ b/strix/config/models.py@@ -104,6 +104,22 @@ def _configure_litellm_compatibility() -> None:litellm.disable_streaming_logging = Truelitellm.suppress_debug_info = True+ _register_litellm_cost_callback()+++def _register_litellm_cost_callback() -> None:+ import litellm++ from strix.report.cost_capture import litellm_cost_callback++ for bucket_name in ("success_callback", "_async_success_callback"):+ bucket = getattr(litellm, bucket_name, None)+ if not isinstance(bucket, list):+ continue+ if litellm_cost_callback in bucket:+ continue+ bucket.append(litellm_cost_callback)+def _configure_litellm_default(name: str, value: str) -> None:"""Set LiteLLM's module-level defaults without adding a provider wrapper."""diff --git a/strix/report/cost_capture.py b/strix/report/cost_capture.pynew file mode 100644index 00000000..78c05bb5--- /dev/null+++ b/strix/report/cost_capture.py@@ -0,0 +1,53 @@+"""LiteLLM success-callback that feeds observed cost into the report ledger."""++from __future__ import annotations++import logging+from typing import Any+++logger = logging.getLogger(__name__)+++def litellm_cost_callback(+ kwargs: dict[str, Any],+ completion_response: Any,+ _start_time: Any = None,+ _end_time: Any = None,+) -> None:+ cost = _extract_cost(kwargs, completion_response)+ if cost is None or cost <= 0:+ return++ from strix.report.state import get_global_report_state++ report_state = get_global_report_state()+ if report_state is None:+ return++ try:+ report_state._llm_usage.record_observed_cost(cost)+ except Exception:+ logger.exception("Failed to record observed LiteLLM cost")+++def _extract_cost(kwargs: dict[str, Any], completion_response: Any) -> float | None:+ cost = kwargs.get("response_cost") if isinstance(kwargs, dict) else None+ if isinstance(cost, int | float) and cost > 0:+ return float(cost)++ hidden = getattr(completion_response, "_hidden_params", None)+ if isinstance(hidden, dict):+ candidate = hidden.get("response_cost")+ if isinstance(candidate, int | float) and candidate > 0:+ return float(candidate)+ headers = hidden.get("additional_headers") or {}+ if isinstance(headers, dict):+ from_header = headers.get("llm_provider-x-litellm-response-cost")+ try:+ value = float(from_header) if from_header is not None else None+ except (TypeError, ValueError):+ value = None+ if value is not None and value > 0:+ return value+ return Nonediff --git a/strix/report/usage.py b/strix/report/usage.pyindex 69f6f285..eaf7aada 100644--- a/strix/report/usage.py+++ b/strix/report/usage.py@@ -18,8 +18,9 @@ class LLMUsageLedger:self._total_usage = Usage()self._agent_usage: dict[str, Usage] = {}self._agent_metadata: dict[str, dict[str, str]] = {}- self._total_cost = 0.0- self._agent_cost: dict[str, float] = {}+ self._estimated_cost = 0.0+ self._agent_estimated_cost: dict[str, float] = {}+ self._observed_cost = 0.0def record(self,@@ -33,8 +34,6 @@ class LLMUsageLedger:return Falsenormalized_agent_id = str(agent_id or "unknown")- estimated_cost = _estimate_litellm_cost(usage, model)-self._total_usage.add(usage)self._agent_usage.setdefault(normalized_agent_id, Usage()).add(usage)@@ -44,31 +43,49 @@ class LLMUsageLedger:if model:metadata["model"] = model- if estimated_cost is not None:- self._total_cost += estimated_cost- self._agent_cost[normalized_agent_id] = (- self._agent_cost.get(normalized_agent_id, 0.0) + estimated_cost- )+ if not _is_litellm_routed(model):+ estimated_cost = _estimate_litellm_cost(usage, model)+ if estimated_cost is not None:+ self._estimated_cost += estimated_cost+ self._agent_estimated_cost[normalized_agent_id] = (+ self._agent_estimated_cost.get(normalized_agent_id, 0.0) + estimated_cost+ )return True+ def record_observed_cost(self, cost: float) -> None:+ if isinstance(cost, int | float) and cost > 0:+ self._observed_cost += float(cost)+def to_record(self) -> dict[str, Any]:record = serialize_usage(self._total_usage)- record["cost"] = _round_cost(self._total_cost)- record["cost_source"] = "litellm_estimate"+ grand_total = self._estimated_cost + self._observed_cost+ record["cost"] = _round_cost(grand_total)+ record["cost_source"] = _cost_source_label(self._estimated_cost, self._observed_cost)record["agents"] = []+ total_tokens = max(0, int(self._total_usage.total_tokens or 0))+for agent_id in sorted(self._agent_usage):usage = self._agent_usage[agent_id]metadata = self._agent_metadata.get(agent_id, {})+ agent_tokens = max(0, int(usage.total_tokens or 0))+ observed_share = (+ self._observed_cost * (agent_tokens / total_tokens) if total_tokens else 0.0+ )+ agent_total = self._agent_estimated_cost.get(agent_id, 0.0) + observed_share+agent_record = serialize_usage(usage)agent_record.update({"agent_id": agent_id,"agent_name": metadata.get("agent_name") or agent_id,"model": metadata.get("model"),- "cost": _round_cost(self._agent_cost.get(agent_id, 0.0)),- "cost_source": "litellm_estimate",+ "cost": _round_cost(agent_total),+ "cost_source": _cost_source_label(+ self._agent_estimated_cost.get(agent_id, 0.0),+ observed_share,+ ),})record["agents"].append(agent_record)@@ -79,8 +96,9 @@ class LLMUsageLedger:self._total_usage = Usage()self._agent_usage.clear()self._agent_metadata.clear()- self._total_cost = 0.0- self._agent_cost.clear()+ self._estimated_cost = 0.0+ self._agent_estimated_cost.clear()+ self._observed_cost = 0.0if not isinstance(raw_usage, dict):return@@ -91,7 +109,9 @@ class LLMUsageLedger:logger.exception("Failed to hydrate aggregate llm_usage from run.json")self._total_usage = Usage()- self._total_cost = _float_or_zero(raw_usage.get("cost"))+ # Resumed runs have already-aggregated cost; treat as estimated. New calls+ # in this resume add observed cost on top.+ self._estimated_cost = _float_or_zero(raw_usage.get("cost"))agents = raw_usage.get("agents") or []if not isinstance(agents, list):return@@ -116,7 +136,24 @@ class LLMUsageLedger:if isinstance(model, str) and model:metadata["model"] = modelself._agent_metadata[agent_id] = metadata- self._agent_cost[agent_id] = _float_or_zero(raw_agent.get("cost"))+ self._agent_estimated_cost[agent_id] = _float_or_zero(raw_agent.get("cost"))+++def _is_litellm_routed(model: str | None) -> bool:+ if not model:+ return False+ name = model.strip().lower()+ if "/" not in name:+ return False+ return not name.startswith("openai/")+++def _cost_source_label(estimated: float, observed: float) -> str:+ if observed > 0 and estimated > 0:+ return "mixed"+ if observed > 0:+ return "litellm_observed"+ return "litellm_estimate"def _usage_has_activity(usage: Usage) -> bool:@@ -171,18 +208,25 @@ def _estimate_litellm_entry_cost(entry: Any, model: str) -> float | None:if completion_details:usage_payload["completion_tokens_details"] = completion_details- try:- from litellm import completion_cost-- cost = completion_cost(- completion_response={- "model": model.split("/", 1)[-1],- "usage": usage_payload,- },- model=model,- )- except Exception: # noqa: BLE001 - LiteLLM raises plain Exception for unknown model prices.- logger.debug("LiteLLM cost estimate unavailable for model %s", model, exc_info=True)+ from litellm import completion_cost++ candidates = [model]+ if "/" in model:+ candidates.append(model.split("/", 1)[-1])++ cost: Any = None+ for candidate in candidates:+ try:+ cost = completion_cost(+ completion_response={"model": candidate, "usage": usage_payload},+ model=model,+ )+ break+ except Exception: # nosec B112 # noqa: BLE001, S112+ continue++ if cost is None:+ logger.debug("LiteLLM cost estimate unavailable for model %s", model)return Nonereturn cost if isinstance(cost, int | float) and cost >= 0 else None