Open-source AI penetration testing tool to find and fix your app’s vulnerabilities.
Python agentsai-hackingai-penetration-testingai-pentestingai-securityartificial-intelligencebug-bountycode-qualityctf-toolscybersecuritycybersecurity-toolsethical-hackinghackingllm-securityoffensive-securitypenetration-testingpentesting-toolsred-teamingsecuritysecurity-automation
fix(llm): keep request_log.install(); record tool call timings; both only with STRIX_RUN_RECORD_TIMINGS
Ianaa2036f22942
diff --git a/docs/advanced/configuration.mdx b/docs/advanced/configuration.mdxindex 29b003ec..f14b110f 100644--- a/docs/advanced/configuration.mdx+++ b/docs/advanced/configuration.mdx@@ -141,6 +141,10 @@ affecting the agents that do the actual testing.Telemetry toggle. Set to `0`, `false`, `no`, or `off` to disable telemetry (PostHog, Scarf, OTEL).</ParamField>+<ParamField path="STRIX_RUN_RECORD_TIMINGS" default="false" type="boolean">+ Record every LLM request's and tool call's timing in `run.json`, under `llm_requests` and `tool_calls`. Adds about 0.5 MB to a typical scan's `run.json`.+</ParamField>+<ParamField path="TRACELOOP_BASE_URL" type="string">OTLP/Traceloop base URL for remote OpenTelemetry export. If unset, Strix keeps traces local only.</ParamField>diff --git a/strix/config/models.py b/strix/config/models.pyindex 8a11695f..d9127ee8 100644--- a/strix/config/models.py+++ b/strix/config/models.py@@ -707,7 +707,9 @@ def configure_sdk_model_defaults(settings: Settings) -> None:set_tracing_disabled(True)from strix.report.state import record_llm_request- request_log.register_sink(record_llm_request)+ request_log.install()+ if settings.telemetry.run_record_timings:+ request_log.register_sink(record_llm_request)if codex.subscription_model(llm.model):return_configure_litellm_compatibility()diff --git a/strix/config/settings.py b/strix/config/settings.pyindex b965281a..22e6c596 100644--- a/strix/config/settings.py+++ b/strix/config/settings.py@@ -163,6 +163,7 @@ class TelemetrySettings(BaseSettings):model_config = _BASE_CONFIGenabled: bool = Field(default=True, alias="STRIX_TELEMETRY")+ run_record_timings: bool = Field(default=False, alias="STRIX_RUN_RECORD_TIMINGS")WebSearchProvider = Literal["auto", "perplexity", "exa"]diff --git a/strix/core/hooks.py b/strix/core/hooks.pyindex 13fc98f9..daddda1b 100644--- a/strix/core/hooks.py+++ b/strix/core/hooks.py@@ -4,18 +4,21 @@ from __future__ import annotationsimport loggingimport math+import time+from datetime import UTC, datetimefrom typing import TYPE_CHECKING, Anyfrom agents.lifecycle import RunHooksfrom strix.core.agents import BudgetPolicy, coordinator_from_context-from strix.report.state import get_global_report_state+from strix.report.state import get_global_report_state, record_tool_callif TYPE_CHECKING:from agents import RunContextWrapperfrom agents.agent import Agentfrom agents.items import ModelResponse, TResponseInputItem+ from agents.tool import Toollogger = logging.getLogger(__name__)@@ -163,6 +166,7 @@ class ReportUsageHooks(RunHooks[dict[str, Any]]):max_turns: int | None = None,interactive: bool = False,budget_policy: BudgetPolicy = "stop",+ record_tool_timings: bool = False,) -> None:_validate_budget(max_budget_usd)if max_turns is not None and max_turns <= 0:@@ -175,6 +179,8 @@ class ReportUsageHooks(RunHooks[dict[str, Any]]):self._max_turns = max_turnsself._interactive = interactiveself._budget_policy: BudgetPolicy = budget_policy+ self._record_tool_timings = record_tool_timings+ self._tool_starts: dict[str, tuple[datetime, float]] = {}@propertydef max_budget_usd(self) -> float | None:@@ -315,6 +321,37 @@ class ReportUsageHooks(RunHooks[dict[str, Any]]):)input_items.append({"role": "user", "content": content})+ async def on_tool_start(+ self,+ context: RunContextWrapper[dict[str, Any]],+ agent: Agent[dict[str, Any]], # noqa: ARG002+ tool: Tool, # noqa: ARG002+ ) -> None:+ call_id = getattr(context, "tool_call_id", None)+ if self._record_tool_timings and isinstance(call_id, str):+ self._tool_starts[call_id] = (datetime.now(UTC), time.monotonic())++ async def on_tool_end(+ self,+ context: RunContextWrapper[dict[str, Any]],+ agent: Agent[dict[str, Any]], # noqa: ARG002+ tool: Tool,+ result: object, # noqa: ARG002+ ) -> None:+ call_id = getattr(context, "tool_call_id", None)+ start = self._tool_starts.pop(call_id, None) if isinstance(call_id, str) else None+ if start is None:+ return+ started_at, started = start+ ctx = context.context if isinstance(context.context, dict) else {}+ agent_id = ctx.get("agent_id")+ record_tool_call(+ started_at=started_at,+ agent_id=agent_id if isinstance(agent_id, str) else None,+ tool=tool.name,+ duration_ms=round((time.monotonic() - started) * 1000),+ )+async def on_llm_end(self,context: RunContextWrapper[dict[str, Any]],diff --git a/strix/core/runner.py b/strix/core/runner.pyindex 7c9c1d98..ae6dc08b 100644--- a/strix/core/runner.py+++ b/strix/core/runner.py@@ -419,6 +419,7 @@ async def run_strix_scan(max_turns=max_turns,interactive=interactive,budget_policy=budget_policy,+ record_tool_timings=settings.telemetry.run_record_timings,)coordinator.set_budget_limit_setter(hooks.set_max_budget_usd)if interactive and budget_policy != "pause":diff --git a/strix/report/state.py b/strix/report/state.pyindex 5daf2849..104e4af4 100644--- a/strix/report/state.py+++ b/strix/report/state.py@@ -1045,6 +1045,23 @@ def record_llm_request(event: "LlmRequestEvent") -> None:)+def record_tool_call(+ *, started_at: datetime, agent_id: str | None, tool: str, duration_ms: int+) -> None:+ """Each tool call's timing, kept in run.json's ``tool_calls``."""+ report_state = get_global_report_state()+ if report_state is None:+ return+ report_state.run_record.setdefault("tool_calls", []).append(+ {+ "started_at": started_at.isoformat(),+ "agent_id": agent_id,+ "tool": tool,+ "duration_ms": duration_ms,+ }+ )++def record_openrouter_provider(provider: Any, usage: Any) -> None:"""Tally which upstream provider served a stream, from its final usage chunk.diff --git a/tests/test_cost_tracking.py b/tests/test_cost_tracking.pyindex 66eb040d..4f12bc0e 100644--- a/tests/test_cost_tracking.py+++ b/tests/test_cost_tracking.py@@ -19,6 +19,7 @@ from strix.config.models import (_configure_litellm_compatibility,_install_openrouter_stream_cost_capture,)+from strix.core.hooks import ReportUsageHooksfrom strix.llm import request_logfrom strix.report.state import (ReportState,@@ -508,3 +509,22 @@ def test_request_timings_land_in_run_record() -> None:"retry_attempt": 0,}]+++async def test_tool_call_timings_land_in_run_record() -> None:+ hooks = ReportUsageHooks(model="m", record_tool_timings=True)+ context = SimpleNamespace(tool_call_id="call-1", context={"agent_id": "agent-1"})+ tool = SimpleNamespace(name="terminal_execute")+ state = ReportState.__new__(ReportState)+ state.run_record = {}+ try:+ set_global_report_state(state)+ await hooks.on_tool_start(context, MagicMock(), tool) # type: ignore[arg-type]+ await hooks.on_tool_end(context, MagicMock(), tool, "ok") # type: ignore[arg-type]+ finally:+ set_global_report_state(None)++ [entry] = state.run_record["tool_calls"]+ assert entry["agent_id"] == "agent-1"+ assert entry["tool"] == "terminal_execute"+ assert entry["duration_ms"] >= 0