Skip to content

armgit / ARES

Public Imported from GitHub
0

ARES - authorized red-team engagement automation with dashboard, campaign scope, module orchestration, OPSEC controls, encrypted vault, and reporting.

14.9 KB
Raw Blame
"""
ARES OpenTelemetry Distributed Tracing - v3.1.0 (canonical)

Single source of truth for all tracing in ARES.
Replaces ares/telemetry/tracing.py (deleted - this file is the merge target).

Provides trace ID correlation from:
    HTTP request → engine → module execution → outgoing HTTP

Features:
    - OTLP gRPC exporter (Jaeger / Grafana Tempo / OTLP-compatible)
    - Console exporter for local dev (ARES_OTEL_CONSOLE=true)
    - FastAPI auto-instrumentation (per-request spans with route + status)
    - @trace_module() decorator - wraps module run() in a child span
    - span() / async_span() context managers for inline spans
    - inject_trace_context() - W3C traceparent header for outgoing calls
    - TraceIDLogFilter - injects trace_id + span_id into every loguru record
    - X-Trace-Id response header set by server middleware
    - Full NoOp fallback - zero overhead when opentelemetry not installed

Environment variables:
    ARES_OTEL_ENDPOINT    - OTLP gRPC endpoint (e.g. http://jaeger:4317)
    ARES_OTEL_SERVICE     - Service name shown in trace UI (default: ares-api)
    ARES_OTEL_SAMPLE_RATE - 0.0–1.0 sampling rate (default: 1.0 = 100%)
    ARES_OTEL_CONSOLE     - also print spans to stdout (dev mode, default: false)

Usage:

    # Decorator (preferred for module run() methods):
    @trace_module("ad.kerberoast")
    async def run(self, dc, domain, ...):
        ...

    # Inline span:
    async with async_span("ad.kerberoast.tgs_request", {"dc": dc}):
        tickets = await request_tgs(...)

    # Inject into outgoing request headers:
    headers = inject_trace_context({})
    await httpx.get(url, headers=headers)
"""
from __future__ import annotations

import functools
from collections.abc import AsyncGenerator, Callable, Generator
from contextlib import asynccontextmanager, contextmanager
from typing import Any

from ares.core.logger import get_logger

logger = get_logger("ares.core.tracing")


# ── Optional dependency guard ─────────────────────────────────────────────────

try:
    from opentelemetry import trace
    from opentelemetry.sdk.resources import SERVICE_NAME, Resource
    from opentelemetry.sdk.trace import TracerProvider
    from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter
    from opentelemetry.trace import StatusCode
    _OTEL_AVAILABLE = True
except ImportError:
    _OTEL_AVAILABLE = False


# ── NoOp implementations (used when OTel not installed or not configured) ─────

class _NoOpSpan:
    """
    Context-manager no-op span - identical public API to real OTel span.
    Used when opentelemetry-sdk is not installed. All methods are intentional
    no-ops (pass) that satisfy the interface contract without side effects.
    """
    def set_attribute(self, key: str, value: Any) -> None:    pass  # no-op by design
    def set_status(self, *a: Any, **kw: Any) -> None:         pass  # no-op by design
    def record_exception(self, exc: Exception, **kw: Any) -> None: pass  # no-op by design
    def add_event(self, name: str, attributes: dict | None = None) -> None: pass  # no-op by design
    def get_span_context(self) -> None:                       return None
    def __enter__(self) -> "_NoOpSpan":                       return self
    def __exit__(self, *a: Any) -> None:                      pass  # no-op by design


class _NoOpTracer:
    def start_as_current_span(self, name: str, **kw: Any) -> _NoOpSpan:
        return _NoOpSpan()
    def start_span(self, name: str, **kw: Any) -> _NoOpSpan:
        return _NoOpSpan()


_NOOP_TRACER = _NoOpTracer()
_provider: Any = None
_tracer:   Any = None


# ── Setup ─────────────────────────────────────────────────────────────────────

def setup_tracing(
    service_name:  str   = "ares-api",
    otel_endpoint: str   = "",
    sample_rate:   float = 1.0,
    console:       bool  = False,
) -> bool:
    """
    Initialize OTel tracing. Returns True if successfully configured.
    Safe to call multiple times - subsequent calls are no-ops.
    Called automatically from server.py lifespan.
    """
    global _provider, _tracer

    if not _OTEL_AVAILABLE:
        logger.info("otel_disabled", reason="opentelemetry-sdk not installed")
        return False

    if _provider is not None:
        return True  # Already configured

    if not otel_endpoint and not console:
        logger.info("otel_disabled",
                    reason="Set ARES_OTEL_ENDPOINT or ARES_OTEL_CONSOLE=true to enable")
        return False

    try:
        from opentelemetry.sdk.trace.sampling import TraceIdRatioBased
        resource  = Resource(attributes={SERVICE_NAME: service_name})
        sampler   = TraceIdRatioBased(sample_rate)
        _provider = TracerProvider(resource=resource, sampler=sampler)

        if console:
            _provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter()))
            logger.info("otel_console_exporter_active")

        if otel_endpoint:
            try:
                from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
                exporter = OTLPSpanExporter(endpoint=otel_endpoint, insecure=True)
                _provider.add_span_processor(BatchSpanProcessor(exporter))
                logger.info("otel_otlp_exporter_active", endpoint=otel_endpoint)
            except ImportError:
                logger.warning("otel_grpc_exporter_missing",
                               hint="pip install opentelemetry-exporter-otlp-proto-grpc")

        trace.set_tracer_provider(_provider)
        _tracer = trace.get_tracer(
            "ares",
            schema_url="https://opentelemetry.io/schemas/1.11.0",
        )
        logger.info("otel_configured", service=service_name, sample_rate=sample_rate)
        return True

    except Exception as exc:
        logger.warning("otel_setup_failed", error=str(exc)[:200])
        return False


def instrument_fastapi(app: Any) -> None:
    """Auto-instrument FastAPI app: one span per request with route + status code."""
    if not _OTEL_AVAILABLE:
        return
    try:
        from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
        FastAPIInstrumentor.instrument_app(
            app,
            excluded_urls="/health,/metrics",
        )
        logger.debug("otel_fastapi_instrumented")
    except ImportError:
        logger.debug("otel_fastapi_instrumentor_missing",
                     hint="pip install opentelemetry-instrumentation-fastapi")
    except Exception as exc:
        logger.warning("otel_fastapi_instrument_failed", error=str(exc)[:100])


# ── Accessors ─────────────────────────────────────────────────────────────────

def get_tracer() -> Any:
    """Return the configured tracer (or NoOp if tracing not active)."""
    if _tracer is not None:
        return _tracer
    if _OTEL_AVAILABLE:
        return trace.get_tracer("ares")
    return _NOOP_TRACER


def get_current_trace_id() -> str | None:
    """Return 32-char hex trace ID of the current span, or None."""
    if not _OTEL_AVAILABLE:
        return None
    try:
        ctx = trace.get_current_span().get_span_context()
        if ctx and ctx.trace_id and ctx.trace_id != 0:
            return format(ctx.trace_id, "032x")
    except (AttributeError, TypeError):
        pass
    return None


# Alias used by legacy telemetry/tracing.py callers
current_trace_id = get_current_trace_id


def get_current_span_id() -> str | None:
    """Return 16-char hex span ID of the current span, or None."""
    if not _OTEL_AVAILABLE:
        return None
    try:
        ctx = trace.get_current_span().get_span_context()
        if ctx and ctx.span_id:
            return format(ctx.span_id, "016x")
    except (AttributeError, TypeError):
        pass
    return None


# Alias
current_span_id = get_current_span_id


def inject_trace_context(headers: dict[str, str]) -> dict[str, str]:
    """
    Inject W3C traceparent header into outgoing HTTP request headers.
    Use this when making HTTP calls from within a traced span so
    downstream services can continue the trace.
    """
    if not _OTEL_AVAILABLE:
        return headers
    try:
        ctx = trace.get_current_span().get_span_context()
        if ctx and ctx.is_valid:
            headers["traceparent"] = (
                f"00-{format(ctx.trace_id, '032x')}"
                f"-{format(ctx.span_id, '016x')}"
                f"-{'01' if ctx.trace_flags else '00'}"
            )
    except (AttributeError, TypeError, ValueError):
        pass
    return headers


# ── Span context managers (inline use) ───────────────────────────────────────

@contextmanager
def span(
    name:       str,
    attributes: dict[str, Any] | None = None,
) -> Generator[Any, None, None]:
    """
    Sync context manager for a child span.

    Usage:
        with tracing.span("ad.ldap_query", {"dc": dc, "domain": domain}):
            results = ldap_search(...)
    """
    tracer = get_tracer()
    with tracer.start_as_current_span(name) as s:
        if attributes:
            _set_span_attrs(s, attributes)
        try:
            yield s
        except Exception as exc:
            _mark_span_error(s, exc)
            raise


@asynccontextmanager
async def async_span(
    name:       str,
    attributes: dict[str, Any] | None = None,
) -> AsyncGenerator[Any, None]:
    """
    Async context manager for a child span.

    Usage:
        async with tracing.async_span("engine.run_module", {"module_id": mid}):
            result = await engine.run_module(...)
    """
    tracer = get_tracer()
    with tracer.start_as_current_span(name) as s:
        if attributes:
            _set_span_attrs(s, attributes)
        try:
            yield s
        except Exception as exc:
            _mark_span_error(s, exc)
            raise


def _set_span_attrs(s: Any, attrs: dict[str, Any]) -> None:
    for k, v in attrs.items():
        try:
            v = v if isinstance(v, (bool, int, float, str)) else str(v)
            s.set_attribute(k, v)
        except (TypeError, AttributeError):
            pass


def _mark_span_error(s: Any, exc: Exception) -> None:
    try:
        if _OTEL_AVAILABLE:
            s.set_status(StatusCode.ERROR, str(exc))
            s.record_exception(exc)
    except (AttributeError, TypeError):
        pass


# ── @trace_module decorator ───────────────────────────────────────────────────

def trace_module(module_id: str) -> Callable:
    """
    Decorator that wraps a module's run() method in a named OTel span.

    Apply to the run() method of every BaseModule subclass:

        class KerberoastModule(BaseModule):
            MODULE_ID = "ad.kerberoast"

            @trace_module("ad.kerberoast")
            async def run(self, dc, domain, ...):
                ...

    The span automatically records:
        ares.module_id      - module identifier
        ares.campaign_id    - campaign ID (from self.campaign.id if available)
        ares.findings_count - count of findings returned
        ares.status         - "success" or "error"
        error details       - exception type + message on failure
    """
    def decorator(fn: Callable) -> Callable:
        @functools.wraps(fn)
        async def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any:
            tracer    = get_tracer()
            span_name = f"ares.module.{module_id}"
            with tracer.start_as_current_span(span_name) as s:
                s.set_attribute("ares.module_id", module_id)
                campaign_id = getattr(getattr(self, "campaign", None), "id", "")
                if campaign_id:
                    s.set_attribute("ares.campaign_id", str(campaign_id))

                try:
                    result = await fn(self, *args, **kwargs)
                    findings, raw = result
                    s.set_attribute("ares.findings_count", len(findings))
                    s.set_attribute("ares.status", "success")
                    if _OTEL_AVAILABLE:
                        s.set_status(StatusCode.OK)
                    return result

                except Exception as exc:
                    s.set_attribute("ares.status", "error")
                    s.set_attribute("ares.error_type", type(exc).__name__)
                    s.set_attribute("ares.error_msg", str(exc)[:300])
                    _mark_span_error(s, exc)
                    raise

        return wrapper
    return decorator


def record_module_event(
    span: Any,
    module_id: str,
    event: str,
    attrs: dict[str, Any] | None = None,
) -> None:
    """Record a named event on the current span (milestones within a module run)."""
    try:
        all_attrs = {"module_id": module_id, **(attrs or {})}
        span.add_event(event, attributes={str(k): str(v) for k, v in all_attrs.items()})
    except (AttributeError, TypeError, ValueError):
        pass


# ── Loguru integration ────────────────────────────────────────────────────────

class TraceIDLogFilter:
    """
    Loguru filter that injects trace_id + span_id into every log record.
    Add to your loguru sink so every log line carries trace context.

    Usage in logger setup (e.g. ares/core/logger.py):
        logger.add(sys.stderr, filter=TraceIDLogFilter(), format=TRACE_LOG_FORMAT)
    """
    def __call__(self, record: dict) -> bool:
        record["extra"].setdefault("trace_id", get_current_trace_id() or " - ")
        record["extra"].setdefault("span_id",  get_current_span_id()  or " - ")
        return True


TRACE_LOG_FORMAT = (
    "<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> | "
    "<level>{level: <8}</level> | "
    "<cyan>{name}</cyan>:<cyan>{line}</cyan> | "
    "trace={extra[trace_id]} span={extra[span_id]} | "
    "<level>{message}</level>"
)


# ── Convenience: init from AresSettings ──────────────────────────────────────

def init_from_settings() -> None:
    """Read tracing settings from AresSettings and call setup_tracing()."""
    try:
        from ares.core.config import get_settings
        s = get_settings()
        setup_tracing(
            service_name  = getattr(s, "ares_otel_service",     "ares-api"),
            otel_endpoint = getattr(s, "ares_otel_endpoint",    ""),
            sample_rate   = getattr(s, "ares_otel_sample_rate", 1.0),
            console       = getattr(s, "ares_otel_console",     False),
        )
    except Exception as exc:
        logger.warning("otel_init_from_settings_failed", error=str(exc)[:100])