Open-source AI penetration testing tool to find and fix your app’s vulnerabilities.
Python agentsai-hackingai-penetration-testingai-pentestingai-securityartificial-intelligencebug-bountycode-qualityctf-toolscybersecuritycybersecurity-toolsethical-hackinghackingllm-securityoffensive-securitypenetration-testingpentesting-toolsred-teamingsecuritysecurity-automation
feat(telemetry): classify error beacons by phase and exception class
Ahmed Allam2e1db25786a6
diff --git a/strix/core/runner.py b/strix/core/runner.pyindex 71c08742..b40a36b0 100644--- a/strix/core/runner.py+++ b/strix/core/runner.py@@ -45,6 +45,7 @@ from strix.core.paths import run_dir_for, runtime_state_dirfrom strix.core.sessions import open_agent_sessionfrom strix.report.state import get_global_report_statefrom strix.runtime import session_manager+from strix.telemetry import set_scan_phasefrom strix.telemetry.logging import set_scan_id, setup_scan_loggingfrom strix.tools.output_store import (WORKSPACE_SPILL_DIR,@@ -116,6 +117,13 @@ def _record_mcp_connections(connections: list[ConnectedMcpServer]) -> None:report_state.record_mcp_connections([connection.name for connection in connections])+def _note_exit_reason(reason: str) -> None:+ """Record why the scan stopped so the end-of-scan beacon reports it."""+ report_state = get_global_report_state()+ if report_state is not None and report_state.scan_ended_exit_reason is None:+ report_state.scan_ended_exit_reason = reason++def _persist_mcp_status(roster: list[dict[str, Any]]) -> None:"""Write the run's non-secret MCP connection status roster to run.json.@@ -313,6 +321,7 @@ async def run_strix_scan(root_id = uuid.uuid4().hex[:8]logger.info("Bringing up sandbox session for scan %s", scan_id)+ set_scan_phase("sandbox_init")bundle = await session_manager.create_or_reuse(scan_id,image=image,@@ -322,6 +331,7 @@ async def run_strix_scan()report("Waiting for the first model response")logger.info("Sandbox ready for scan %s", scan_id)+ set_scan_phase("agent_setup")sandbox_session = bundle["session"]@@ -573,6 +583,7 @@ async def run_strix_scan(async with coordinator._lock:root_status = coordinator.statuses.get(root_id)+ set_scan_phase("agent_loop")result = await run_agent_loop(agent=root_agent,initial_input=initial_input,@@ -610,6 +621,7 @@ async def run_strix_scan(return result # noqa: TRY300except BudgetExceededError as exc:logger.info("Scan %s stopped: %s", scan_id, exc)+ _note_exit_reason("budget_exceeded")if root_id is not None:with contextlib.suppress(Exception):await coordinator.set_status(root_id, "stopped")@@ -622,6 +634,7 @@ async def run_strix_scan(exc,scan_id,)+ _note_exit_reason("rate_limited")if root_id is not None:with contextlib.suppress(Exception):await coordinator.set_status(root_id, "stopped")diff --git a/strix/interface/environment.py b/strix/interface/environment.pyindex 49bdd82b..522067df 100644--- a/strix/interface/environment.py+++ b/strix/interface/environment.py@@ -14,6 +14,7 @@ from strix.interface.utils import (image_exists,process_pull_line,)+from strix.telemetry import report_errorlogger = logging.getLogger(__name__)@@ -44,6 +45,7 @@ def validate_environment() -> None:f"[red]STRIX_LLM={settings.llm.model} uses your ChatGPT subscription, ""but you're not signed in.[/] Run [cyan]strix auth login chatgpt[/] first.")+ report_error("subscription_not_signed_in")sys.exit(1)logger.info("Environment OK (ChatGPT subscription)")return@@ -153,6 +155,7 @@ def validate_environment() -> None:console.print("\n")console.print(panel)console.print()+ report_error("missing_required_config")sys.exit(1)logger.info("Environment OK (optional missing: %s)",@@ -180,6 +183,7 @@ def check_docker_installed() -> None:padding=(1, 2),)console.print("\n", panel, "\n")+ report_error("docker_not_installed")sys.exit(1)logger.debug("Docker CLI present")@@ -227,6 +231,7 @@ def pull_docker_image() -> None:padding=(1, 2),)console.print(panel, "\n")+ report_error("image_pull_failed", e)sys.exit(1)logger.info("Docker image %s ready", image)diff --git a/strix/interface/main.py b/strix/interface/main.pyindex 5723ef43..c9bd5596 100644--- a/strix/interface/main.py+++ b/strix/interface/main.py@@ -42,7 +42,7 @@ from strix.interface.utils import (build_final_stats_text,)from strix.llm.warmup import start_import_warmup, wait_for_import_warmup-from strix.telemetry import posthog, scarf+from strix.telemetry import posthog, report_error, scarf, set_scan_phasefrom strix.telemetry.logging import configure_dependency_logging@@ -401,15 +401,18 @@ def _bootstrap_scan(args: argparse.Namespace) -> None:happen inside the TUI so the interface paints immediately instead ofwaiting on a model round trip."""+ set_scan_phase("preflight")try:asyncio.run(warm_up_llm(show_model_warning=True))except ModelConnectionError as exc:+ report_error("model_connection_failed", exc)_print_model_connection_error(exc, exc.model_name)sys.exit(1)persist_current()try:prepare_run(args)except ValueError as e:+ report_error("scan_preparation_failed", e)_print_error_panel("SCAN PREPARATION FAILED", str(e))sys.exit(1)telemetry_start(args)@@ -484,18 +487,21 @@ def main() -> None:from strix.interface.cli import run_cliasyncio.run(run_cli(args))+ # Headless runs have no user to quit: the agent either finished+ # (already beaconed as finished_by_tool) or stopped on its own.+ exit_reason = "agent_stopped"else:asyncio.run(run_tui(args))except InteractiveSetupUnavailableError as exc:exit_reason = "error"+ report_error("interactive_setup_unavailable", exc)_print_error_panel("INTERACTIVE SETUP UNAVAILABLE", str(exc))sys.exit(1)except KeyboardInterrupt:exit_reason = "interrupted"- except Exception:+ except Exception as exc:exit_reason = "error"- posthog.error("unhandled_exception")- scarf.error("unhandled_exception")+ report_error("unhandled_exception", exc)raisefinally:report_state = get_global_report_state()diff --git a/strix/interface/tui/runtime.py b/strix/interface/tui/runtime.pyindex 728e7023..08fa0522 100644--- a/strix/interface/tui/runtime.py+++ b/strix/interface/tui/runtime.py@@ -37,6 +37,7 @@ from strix.interface.tui.sidecar import ()from strix.interface.utils import read_workspace_filesfrom strix.report.state import ReportState, set_global_report_state+from strix.telemetry import report_error, set_scan_phasefrom strix.utils.resource_paths import get_strix_resource_path@@ -138,6 +139,7 @@ class GoTuiRuntime:await self._preflight_model()except Exception as exc:logger.exception("Go TUI setup model preflight failed")+ report_error("model_connection_failed", exc)raise RuntimeError(f"Model connection failed: {exc}") from excasync def _preflight_model(self) -> None:@@ -195,13 +197,21 @@ class GoTuiRuntime:launch so the interface appears immediately."""model = (load_settings().llm.model or "").strip()+ set_scan_phase("preflight")try:await preflight_model_connection(model)+ except Exception as exc:+ logger.exception("Go TUI scan preparation failed")+ report_error("model_connection_failed", exc)+ self.controller.fail_preparation(str(exc))+ return+ try:persist_current()prepare_run(self.args)telemetry_start(self.args)except Exception as exc:logger.exception("Go TUI scan preparation failed")+ report_error("scan_preparation_failed", exc)self.controller.fail_preparation(str(exc))returnself.controller.scan_state = "running"@@ -240,6 +250,9 @@ class GoTuiRuntime:self.controller.scan_state = "completed" if report_status == "completed" else "stopped"except Exception as exc:logger.exception("Go TUI scan failed")+ report_error("unhandled_exception", exc)+ if self.report_state is not None and self.report_state.scan_ended_exit_reason is None:+ self.report_state.scan_ended_exit_reason = "error"self.scan_error = excself.controller.error = str(exc)self.controller.scan_state = "failed"diff --git a/strix/interface/utils.py b/strix/interface/utils.pyindex faab1772..e051a185 100644--- a/strix/interface/utils.py+++ b/strix/interface/utils.py@@ -19,6 +19,7 @@ from rich.panel import Panelfrom rich.text import Textfrom strix.config import load_settings+from strix.telemetry import report_errorfrom strix.utils.api_spec import detect_spec_format@@ -1602,7 +1603,8 @@ def check_docker_connection() -> Any:try:return docker.from_env()- except DockerException:+ except DockerException as exc:+ report_error("docker_unavailable", exc)console = Console()error_text = Text()error_text.append("DOCKER NOT AVAILABLE", style="bold red")diff --git a/strix/telemetry/README.md b/strix/telemetry/README.mdindex 023eb932..3985093e 100644--- a/strix/telemetry/README.md+++ b/strix/telemetry/README.md@@ -12,7 +12,7 @@ Privacy is our priority. All collected data is anonymized by default. Each sessiWe collect only very **basic** usage data including:-**Session Errors:** Duration and error types (not messages or stack traces)\+**Session Errors:** Duration, the failure category, the scan phase, and the exception class name (not messages or stack traces)\**System Context:** OS type, architecture, Strix version\**Scan Context:** Scan mode (quick/standard/deep), scan type (whitebox/blackbox)\**Model Usage:** Which LLM model is being used and whether it runs via an API key or a model subscription (not prompts or responses)\diff --git a/strix/telemetry/__init__.py b/strix/telemetry/__init__.pyindex b9ca597e..30cc3b21 100644--- a/strix/telemetry/__init__.py+++ b/strix/telemetry/__init__.py@@ -1,7 +1,19 @@from . import posthog, scarf+from ._common import set_scan_phase+++def report_error(error_type: str, exc: BaseException | None = None) -> None:+ """Beacon a failure category, plus the exception class when one is given.++ Only class names travel: never the message, arguments, or traceback.+ """+ posthog.error(error_type, exc)+ scarf.error(error_type, exc)__all__ = ["posthog",+ "report_error","scarf",+ "set_scan_phase",]diff --git a/strix/telemetry/_common.py b/strix/telemetry/_common.pyindex 7923a506..f7f2866c 100644--- a/strix/telemetry/_common.py+++ b/strix/telemetry/_common.py@@ -5,7 +5,7 @@ import platformimport sysfrom importlib.metadata import PackageNotFoundError, versionfrom pathlib import Path-from typing import Any+from typing import Any, castfrom uuid import uuid4@@ -54,3 +54,43 @@ def base_props() -> dict[str, Any]:"python": f"{sys.version_info.major}.{sys.version_info.minor}","strix_version": get_version(),}+++# Coarse stage of the current run, attached to ``error`` beacons so a failure+# can be placed without a message or trace. Process-local, like the rest of the+# CLI telemetry: one process runs one scan.+_scan_phase = "startup"+++def set_scan_phase(phase: str) -> None:+ global _scan_phase # noqa: PLW0603+ _scan_phase = phase+++def get_scan_phase() -> str:+ return _scan_phase+++def _exception_name(exc: BaseException) -> str:+ cls = type(exc)+ package = cls.__module__.split(".")[0]+ return cls.__name__ if package == "builtins" else f"{package}.{cls.__name__}"+++def _unwrap_group(exc: BaseException) -> BaseException:+ if not isinstance(exc, BaseExceptionGroup):+ return exc+ group = cast("BaseExceptionGroup[BaseException]", exc)+ return group.exceptions[0] if group.exceptions else group+++def exception_props(exc: BaseException) -> dict[str, str]:+ """Class names only. Messages, arguments, and tracebacks never leave the machine."""+ exc = _unwrap_group(exc)+ props = {"exception_type": _exception_name(exc)}+ cause = exc.__cause__+ if cause is None and not exc.__suppress_context__:+ cause = exc.__context__+ if cause is not None:+ props["exception_cause"] = _exception_name(cause)+ return propsdiff --git a/strix/telemetry/posthog.py b/strix/telemetry/posthog.pyindex a85a843b..72927fbc 100644--- a/strix/telemetry/posthog.py+++ b/strix/telemetry/posthog.py@@ -9,6 +9,8 @@ from strix.telemetry._common import (SEND_TIMEOUT,SESSION_ID,base_props,+ exception_props,+ get_scan_phase,get_version,is_first_run,)@@ -178,6 +180,12 @@ def viewer_agent_steered() -> None:_send("viewer_agent_steered", {**base_props()})-def error(error_type: str) -> None:- props = {**base_props(), "error_type": error_type}+def error(error_type: str, exc: BaseException | None = None) -> None:+ props: dict[str, Any] = {+ **base_props(),+ "error_type": error_type,+ "phase": get_scan_phase(),+ }+ if exc is not None:+ props.update(exception_props(exc))_send("error", props)diff --git a/strix/telemetry/scarf.py b/strix/telemetry/scarf.pyindex fbaea998..c8161642 100644--- a/strix/telemetry/scarf.py+++ b/strix/telemetry/scarf.py@@ -12,6 +12,8 @@ from strix.telemetry._common import (SEND_TIMEOUT,SESSION_ID,base_props,+ exception_props,+ get_scan_phase,get_version,is_first_run,)@@ -135,10 +137,13 @@ def end(report_state: ReportState, exit_reason: str = "completed") -> None:)-def error(error_type: str) -> None:+def error(error_type: str, exc: BaseException | None = None) -> None:props: dict[str, Any] = {**base_props(),"session": SESSION_ID,"error_type": error_type,+ "phase": get_scan_phase(),}+ if exc is not None:+ props.update(exception_props(exc))_send("error", props)diff --git a/tests/test_telemetry_errors.py b/tests/test_telemetry_errors.pynew file mode 100644index 00000000..5e3b7c14--- /dev/null+++ b/tests/test_telemetry_errors.py@@ -0,0 +1,125 @@+"""Error beacons carry a category, phase, and exception class — never a message."""++from __future__ import annotations++from typing import Any++import pytest+import requests++from strix.report.state import ReportState+from strix.telemetry import posthog, report_error, scarf, set_scan_phase+from strix.telemetry._common import exception_props+++PRIVATE_MESSAGE = "private message that must stay on the machine"+++def _capture(sent: list[dict[str, Any]], event: str, props: dict[str, Any]) -> bool:+ sent.append({"event": event, **props})+ return True+++def test_exception_props_uses_bare_name_for_builtins() -> None:+ assert exception_props(ValueError(PRIVATE_MESSAGE)) == {"exception_type": "ValueError"}+++def test_exception_props_prefixes_third_party_top_level_package() -> None:+ props = exception_props(requests.exceptions.ConnectTimeout(PRIVATE_MESSAGE))+ assert props == {"exception_type": "requests.ConnectTimeout"}+++def _chained(cause: BaseException | None, *, explicit: bool) -> RuntimeError:+ exc = RuntimeError("wrapped")+ if explicit:+ exc.__cause__ = cause+ exc.__suppress_context__ = True+ else:+ exc.__context__ = cause+ return exc+++def test_exception_props_reports_explicit_cause() -> None:+ props = exception_props(_chained(ConnectionError(PRIVATE_MESSAGE), explicit=True))+ assert props == {"exception_type": "RuntimeError", "exception_cause": "ConnectionError"}+++def test_exception_props_reports_implicit_context() -> None:+ props = exception_props(_chained(KeyError("k"), explicit=False))+ assert props["exception_cause"] == "KeyError"+++def test_exception_props_ignores_suppressed_context() -> None:+ exc = _chained(None, explicit=True)+ exc.__context__ = KeyError("k")+ assert exception_props(exc) == {"exception_type": "RuntimeError"}+++def test_exception_props_unwraps_exception_group() -> None:+ group = ExceptionGroup("tasks", [TimeoutError("t"), ValueError("v")])+ assert exception_props(group) == {"exception_type": "TimeoutError"}++[email protected]("telemetry", [posthog, scarf])+def test_error_event_carries_phase_and_class_but_no_message(+ telemetry: Any,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ sent: list[dict[str, Any]] = []+ monkeypatch.setattr(telemetry, "_send", lambda event, props: _capture(sent, event, props))+ set_scan_phase("sandbox_init")++ telemetry.error("scan_failed", RuntimeError(PRIVATE_MESSAGE))++ assert len(sent) == 1+ event = sent[0]+ assert event["event"] == "error"+ assert event["error_type"] == "scan_failed"+ assert event["phase"] == "sandbox_init"+ assert event["exception_type"] == "RuntimeError"+ assert PRIVATE_MESSAGE not in repr(event)++[email protected]("telemetry", [posthog, scarf])+def test_error_event_without_exception_omits_exception_fields(+ telemetry: Any,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ sent: list[dict[str, Any]] = []+ monkeypatch.setattr(telemetry, "_send", lambda event, props: _capture(sent, event, props))+ set_scan_phase("startup")++ telemetry.error("docker_not_installed")++ assert sent[0]["error_type"] == "docker_not_installed"+ assert sent[0]["phase"] == "startup"+ assert "exception_type" not in sent[0]+ assert "exception_cause" not in sent[0]+++def test_report_error_fans_out_to_both_backends(monkeypatch: pytest.MonkeyPatch) -> None:+ sent: list[dict[str, Any]] = []+ monkeypatch.setattr(posthog, "_send", lambda event, props: _capture(sent, event, props))+ monkeypatch.setattr(scarf, "_send", lambda event, props: _capture(sent, event, props))++ report_error("model_connection_failed", TimeoutError(PRIVATE_MESSAGE))++ assert len(sent) == 2+ assert {e["error_type"] for e in sent} == {"model_connection_failed"}+ assert {e["exception_type"] for e in sent} == {"TimeoutError"}++[email protected]("telemetry", [posthog, scarf])+def test_scan_ended_prefers_recorded_exit_reason(+ telemetry: Any,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ state = ReportState()+ state.scan_ended_exit_reason = "budget_exceeded"+ sent: list[dict[str, Any]] = []+ monkeypatch.setattr(telemetry, "_send", lambda event, props: _capture(sent, event, props))++ telemetry.end(state, exit_reason="user_exit")++ assert sent[0]["event"] == "scan_ended"+ assert sent[0]["exit_reason"] == "budget_exceeded"