Open-source AI penetration testing tool to find and fix your app’s vulnerabilities.
Python agentsai-hackingai-penetration-testingai-pentestingai-securityartificial-intelligencebug-bountycode-qualityctf-toolscybersecuritycybersecurity-toolsethical-hackinghackingllm-securityoffensive-securitypenetration-testingpentesting-toolsred-teamingsecuritysecurity-automation
Add Scarf telemetry alongside PostHog
0xallam393548c6e8d7
diff --git a/docs/advanced/configuration.mdx b/docs/advanced/configuration.mdxindex a40b9fcf..27fba22a 100644--- a/docs/advanced/configuration.mdx+++ b/docs/advanced/configuration.mdx@@ -42,15 +42,7 @@ Configure Strix using environment variables or a config file.</ParamField><ParamField path="STRIX_TELEMETRY" default="1" type="string">- Global telemetry default toggle. Set to `0`, `false`, `no`, or `off` to disable both PostHog and OTEL unless overridden by per-channel flags below.-</ParamField>--<ParamField path="STRIX_OTEL_TELEMETRY" type="string">- Enable/disable OpenTelemetry run observability independently. When unset, falls back to `STRIX_TELEMETRY`.-</ParamField>--<ParamField path="STRIX_POSTHOG_TELEMETRY" type="string">- Enable/disable PostHog product telemetry independently. When unset, falls back to `STRIX_TELEMETRY`.+ Telemetry toggle. Set to `0`, `false`, `no`, or `off` to disable telemetry (PostHog, Scarf, OTEL).</ParamField><ParamField path="TRACELOOP_BASE_URL" type="string">diff --git a/strix/config/settings.py b/strix/config/settings.pyindex 6e696b19..bdd96c20 100644--- a/strix/config/settings.py+++ b/strix/config/settings.py@@ -72,17 +72,11 @@ class RuntimeSettings(BaseSettings):class TelemetrySettings(BaseSettings):- """Telemetry toggles. ``posthog`` is None → inherit ``master``."""+ """Telemetry toggle."""model_config = _BASE_CONFIG- master: bool = Field(default=True, alias="STRIX_TELEMETRY")- posthog: bool | None = Field(default=None, alias="STRIX_POSTHOG_TELEMETRY")-- @property- def posthog_enabled(self) -> bool:- """Effective PostHog toggle: explicit value if set, else ``master``."""- return self.master if self.posthog is None else self.posthog+ enabled: bool = Field(default=True, alias="STRIX_TELEMETRY")class IntegrationSettings(BaseSettings):diff --git a/strix/interface/main.py b/strix/interface/main.pyindex 6fafab1e..b9c01452 100644--- a/strix/interface/main.py+++ b/strix/interface/main.py@@ -44,7 +44,7 @@ from strix.interface.utils import ()from strix.report.state import get_global_report_statefrom strix.report.writer import read_run_record, write_run_record-from strix.telemetry import posthog+from strix.telemetry import posthog, scarffrom strix.telemetry.logging import configure_dependency_logging@@ -735,13 +735,15 @@ def main() -> None:# re-supplying targets / instructions / scope._persist_run_record(args)- posthog.start(- model=load_settings().llm.model,- scan_mode=args.scan_mode,- is_whitebox=is_whitebox_scan(args.targets_info),- interactive=not args.non_interactive,- has_instructions=bool(args.instruction),- )+ _telemetry_start_kwargs = {+ "model": load_settings().llm.model,+ "scan_mode": args.scan_mode,+ "is_whitebox": is_whitebox_scan(args.targets_info),+ "interactive": not args.non_interactive,+ "has_instructions": bool(args.instruction),+ }+ posthog.start(**_telemetry_start_kwargs)+ scarf.start(**_telemetry_start_kwargs)exit_reason = "user_exit"try:@@ -754,6 +756,7 @@ def main() -> None:except Exception as e:exit_reason = "error"posthog.error("unhandled_exception", str(e))+ scarf.error("unhandled_exception", str(e))raisefinally:report_state = get_global_report_state()@@ -764,6 +767,7 @@ def main() -> None:)report_state.cleanup(status=status)posthog.end(report_state, exit_reason=exit_reason)+ scarf.end(report_state, exit_reason=exit_reason)results_path = run_dir_for(args.run_name)display_completion_message(args, results_path)diff --git a/strix/report/state.py b/strix/report/state.pyindex c9541d01..a69ced7f 100644--- a/strix/report/state.py+++ b/strix/report/state.py@@ -16,7 +16,7 @@ from strix.report.writer import (write_run_record,write_vulnerabilities,)-from strix.telemetry import posthog+from strix.telemetry import posthog, scarflogger = logging.getLogger(__name__)@@ -202,6 +202,7 @@ class ReportState:self.vulnerability_reports.append(report)logger.info(f"Added vulnerability report: {report_id} - {title}")posthog.finding(severity)+ scarf.finding(severity)if self.vulnerability_found_callback:self.vulnerability_found_callback(report)@@ -254,6 +255,7 @@ class ReportState:logger.info("Updated scan final fields")self.save_run_data(mark_complete=True)posthog.end(self, exit_reason="finished_by_tool")+ scarf.end(self, exit_reason="finished_by_tool")def set_scan_config(self, config: dict[str, Any]) -> None:self.scan_config = configdiff --git a/strix/telemetry/README.md b/strix/telemetry/README.mdindex bb90fe1a..ba9fc1ef 100644--- a/strix/telemetry/README.md+++ b/strix/telemetry/README.md@@ -18,11 +18,9 @@ We collect only very **basic** usage data including:**Model Usage:** Which LLM model is being used (not prompts or responses)\**Aggregate Metrics:** Vulnerability counts by severity-For complete transparency, you can inspect our [telemetry implementation](https://github.com/usestrix/strix/blob/main/strix/telemetry/posthog.py) to see the exact events we track.-### What We **Never** Collect-- IP addresses, usernames, or any identifying information+- Usernames, or any identifying information- Scan targets, file paths, target URLs, or domains- Vulnerability details, descriptions, or code- LLM requests and responsesdiff --git a/strix/telemetry/__init__.py b/strix/telemetry/__init__.pyindex 8421eb73..b9ca597e 100644--- a/strix/telemetry/__init__.py+++ b/strix/telemetry/__init__.py@@ -1,6 +1,7 @@-from . import posthog+from . import posthog, scarf__all__ = ["posthog",+ "scarf",]diff --git a/strix/telemetry/_common.py b/strix/telemetry/_common.pynew file mode 100644index 00000000..ff53ceef--- /dev/null+++ b/strix/telemetry/_common.py@@ -0,0 +1,50 @@+from __future__ import annotations++import logging+import platform+import sys+from importlib.metadata import PackageNotFoundError, version+from pathlib import Path+from typing import Any+from uuid import uuid4+++logger = logging.getLogger(__name__)++SESSION_ID: str = uuid4().hex[:16]++_FIRST_RUN_CACHED: bool | None = None+++def get_version() -> str:+ try:+ return version("strix-agent")+ except PackageNotFoundError:+ logger.debug("strix-agent version lookup failed", exc_info=True)+ return "unknown"+++def is_first_run() -> bool:+ global _FIRST_RUN_CACHED # noqa: PLW0603+ if _FIRST_RUN_CACHED is not None:+ return _FIRST_RUN_CACHED+ marker = Path.home() / ".strix" / ".seen"+ if marker.exists():+ _FIRST_RUN_CACHED = False+ return False+ try:+ marker.parent.mkdir(parents=True, exist_ok=True)+ marker.touch()+ except Exception: # noqa: BLE001, S110+ pass # nosec B110+ _FIRST_RUN_CACHED = True+ return True+++def base_props() -> dict[str, Any]:+ return {+ "os": platform.system().lower(),+ "arch": platform.machine(),+ "python": f"{sys.version_info.major}.{sys.version_info.minor}",+ "strix_version": get_version(),+ }diff --git a/strix/telemetry/posthog.py b/strix/telemetry/posthog.pyindex 3a2a6696..3db7c0f9 100644--- a/strix/telemetry/posthog.py+++ b/strix/telemetry/posthog.py@@ -1,13 +1,15 @@import jsonimport logging-import platform-import sysimport urllib.request-from pathlib import Path+from datetime import datetimefrom typing import TYPE_CHECKING, Any-from uuid import uuid4from strix.config import load_settings+from strix.telemetry._common import (+ SESSION_ID,+ base_props,+ is_first_run,+)if TYPE_CHECKING:@@ -19,34 +21,9 @@ logger = logging.getLogger(__name__)_POSTHOG_PUBLIC_API_KEY = "phc_7rO3XRuNT5sgSKAl6HDIrWdSGh1COzxw0vxVIAR6vVZ"_POSTHOG_HOST = "https://us.i.posthog.com"-_SESSION_ID = uuid4().hex[:16]-def _is_enabled() -> bool:- """Master telemetry gate. ``STRIX_POSTHOG_TELEMETRY`` overrides ``STRIX_TELEMETRY``."""- return load_settings().telemetry.posthog_enabled---def _is_first_run() -> bool:- marker = Path.home() / ".strix" / ".seen"- if marker.exists():- return False- try:- marker.parent.mkdir(parents=True, exist_ok=True)- marker.touch()- except Exception: # noqa: BLE001, S110- pass # nosec B110- return True---def _get_version() -> str:- try:- from importlib.metadata import version-- return version("strix-agent")- except Exception: # noqa: BLE001- logger.debug("strix-agent version lookup failed", exc_info=True)- return "unknown"+ return load_settings().telemetry.enableddef _send(event: str, properties: dict[str, Any]) -> None:@@ -57,7 +34,7 @@ def _send(event: str, properties: dict[str, Any]) -> None:payload = {"api_key": _POSTHOG_PUBLIC_API_KEY,"event": event,- "distinct_id": _SESSION_ID,+ "distinct_id": SESSION_ID,"properties": properties,}req = urllib.request.Request( # noqa: S310@@ -74,15 +51,6 @@ def _send(event: str, properties: dict[str, Any]) -> None:logger.debug("posthog event sent: %s", event)-def _base_props() -> dict[str, Any]:- return {- "os": platform.system().lower(),- "arch": platform.machine(),- "python": f"{sys.version_info.major}.{sys.version_info.minor}",- "strix_version": _get_version(),- }--def start(model: str | None,scan_mode: str | None,@@ -93,13 +61,13 @@ def start(_send("scan_started",{- **_base_props(),+ **base_props(),"model": model or "unknown","scan_mode": scan_mode or "unknown","scan_type": "whitebox" if is_whitebox else "blackbox","interactive": interactive,"has_instructions": has_instructions,- "first_run": _is_first_run(),+ "first_run": is_first_run(),},)@@ -108,7 +76,7 @@ def finding(severity: str) -> None:_send("finding_reported",{- **_base_props(),+ **base_props(),"severity": severity.lower(),},)@@ -123,8 +91,6 @@ def end(report_state: "ReportState", exit_reason: str = "completed") -> None:duration = 0.0try:- from datetime import datetime-start = datetime.fromisoformat(report_state.start_time.replace("Z", "+00:00"))end_iso = report_state.end_time or datetime.now(start.tzinfo).isoformat()duration = (datetime.fromisoformat(end_iso.replace("Z", "+00:00")) - start).total_seconds()@@ -148,7 +114,7 @@ def end(report_state: "ReportState", exit_reason: str = "completed") -> None:_send("scan_ended",{- **_base_props(),+ **base_props(),"exit_reason": exit_reason,"duration_seconds": round(duration),"vulnerabilities_total": len(report_state.vulnerability_reports),@@ -159,7 +125,7 @@ def end(report_state: "ReportState", exit_reason: str = "completed") -> None:def error(error_type: str, error_msg: str | None = None) -> None:- props = {**_base_props(), "error_type": error_type}+ props = {**base_props(), "error_type": error_type}if error_msg:props["error_msg"] = error_msg_send("error", props)diff --git a/strix/telemetry/scarf.py b/strix/telemetry/scarf.pynew file mode 100644index 00000000..25e38c49--- /dev/null+++ b/strix/telemetry/scarf.py@@ -0,0 +1,140 @@+from __future__ import annotations++import logging+import urllib.parse+import urllib.request+from datetime import datetime+from typing import TYPE_CHECKING, Any++from strix.config import load_settings+from strix.telemetry._common import (+ SESSION_ID,+ base_props,+ get_version,+ is_first_run,+)+++if TYPE_CHECKING:+ from strix.report.state import ReportState+++logger = logging.getLogger(__name__)++_SCARF_ENDPOINT = "https://strix.gateway.scarf.sh"+++def _is_enabled() -> bool:+ return load_settings().telemetry.enabled+++def _send(event: str, properties: dict[str, Any]) -> None:+ if not _is_enabled():+ logger.debug("scarf disabled; skipping event %s", event)+ return+ try:+ props = dict(properties)+ version = str(props.pop("strix_version", get_version()) or "unknown")+ path = f"/{urllib.parse.quote(event, safe='')}/{urllib.parse.quote(version, safe='')}"+ query = urllib.parse.urlencode(+ {k: ("" if v is None else str(v)) for k, v in props.items()},+ )+ url = f"{_SCARF_ENDPOINT}{path}"+ if query:+ url = f"{url}?{query}"+ req = urllib.request.Request(url, method="POST") # noqa: S310+ with urllib.request.urlopen(req, timeout=10): # noqa: S310 # nosec B310+ pass+ except Exception: # noqa: BLE001+ logger.debug("scarf send failed for event %s", event, exc_info=True)+ else:+ logger.debug("scarf event sent: %s", event)+++def start(+ model: str | None,+ scan_mode: str | None,+ is_whitebox: bool,+ interactive: bool,+ has_instructions: bool,+) -> None:+ _send(+ "scan_started",+ {+ **base_props(),+ "session": SESSION_ID,+ "model": model or "unknown",+ "scan_mode": scan_mode or "unknown",+ "scan_type": "whitebox" if is_whitebox else "blackbox",+ "interactive": interactive,+ "has_instructions": has_instructions,+ "first_run": is_first_run(),+ },+ )+++def finding(severity: str) -> None:+ _send(+ "finding_reported",+ {+ **base_props(),+ "session": SESSION_ID,+ "severity": severity.lower(),+ },+ )+++def end(report_state: ReportState, exit_reason: str = "completed") -> None:+ vulnerabilities_counts = {"critical": 0, "high": 0, "medium": 0, "low": 0, "info": 0}+ for v in report_state.vulnerability_reports:+ sev = v.get("severity", "info").lower()+ if sev in vulnerabilities_counts:+ vulnerabilities_counts[sev] += 1++ duration = 0.0+ try:+ scan_start = datetime.fromisoformat(report_state.start_time.replace("Z", "+00:00"))+ end_iso = report_state.end_time or datetime.now(scan_start.tzinfo).isoformat()+ duration = (+ datetime.fromisoformat(end_iso.replace("Z", "+00:00")) - scan_start+ ).total_seconds()+ except (ValueError, TypeError, AttributeError):+ pass++ llm_props: dict[str, int | float] = {}+ try:+ usage = report_state.get_total_llm_usage()+ if isinstance(usage, dict):+ llm_props = {+ "llm_requests": int(usage.get("requests") or 0),+ "llm_input_tokens": int(usage.get("input_tokens") or 0),+ "llm_output_tokens": int(usage.get("output_tokens") or 0),+ "llm_tokens": int(usage.get("total_tokens") or 0),+ "llm_cost": float(usage.get("cost") or 0.0),+ }+ except (TypeError, ValueError, AttributeError):+ pass++ _send(+ "scan_ended",+ {+ **base_props(),+ "session": SESSION_ID,+ "exit_reason": exit_reason,+ "duration_seconds": round(duration),+ "vulnerabilities_total": len(report_state.vulnerability_reports),+ **{f"vulnerabilities_{k}": v for k, v in vulnerabilities_counts.items()},+ **llm_props,+ },+ )+++def error(error_type: str, error_msg: str | None = None) -> None:+ props: dict[str, Any] = {+ **base_props(),+ "session": SESSION_ID,+ "error_type": error_type,+ }+ if error_msg:+ props["error_msg"] = error_msg+ _send("error", props)