Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
import asyncio
import os
from collections.abc import AsyncGenerator
from contextlib import asynccontextmanager
import structlog
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from app._health import install_health_routes, register_dependency, register_subscription
from app.api.contextual import router as contextual_router
from app.api.copilot import router as copilot_router
from app.api.explain import router as explain_router
from app.api.hunt_search import router as hunt_search_router
from app.api.hunts import router as hunts_router
from app.api.investigate import router as investigate_router
from app.api.metrics import router as metrics_router
from app.api.playbooks import router as playbook_router
from app.api.playbooks import verifications_router as playbook_verifications_router
from app.api.playbooks import waits_router as playbook_waits_router
from app.api.replay_router import router as replay_router
from app.api.router import router
from app.api.triage import router as triage_router
from app.core.telemetry import instrument_app
# Wave 3. Imported so the quality metrics have a production importer:
# tool selection, evidence completeness and time to verdict had zero
# hits anywhere in the tree, and a metric nothing imports is a module.
from app.eval import score_agent_quality
# Re-exported rather than left as a bare import: an import with a
# `noqa` and no reader is indistinguishable from a stale one, and
# CodeQL reads it as unused. `__all__` states the intent.
__all__ = ["score_agent_quality"]
from app.hunt import scheduler as hunt_scheduler
from app.hunt import store as hunt_store
from app.investigator import ledger as investigation_ledger
from app.investigator import run_reaper as investigation_run_reaper
from app.llm.factory import preflight_llm
from app.playbook import PlaybookStore
from app.playbook import sweeper as playbook_sweeper
from app.tools.mitre_full import embed_techniques_into_qdrant, load_attck_corpus
from app.workers.business_context import BusinessContextApplier
from app.workers.business_context import is_enabled as business_context_enabled
from app.workers.fused_alert_consumer import FusedAlertTriageWorker, worker_enabled
logger = structlog.get_logger()
def _log_triage_worker_exit(task: asyncio.Task) -> None:
"""Report the auto-triage worker's exit instead of losing it in the task."""
if task.cancelled():
logger.info("auto_triage_worker.cancelled")
return
exc = task.exception()
if exc is None:
logger.error("auto_triage_worker.exited", detail="consume loop returned; fused alerts are no longer triaged")
return
logger.error("auto_triage_worker.died", error=str(exc), error_type=type(exc).__name__, exc_info=exc)
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
"""Load resources on startup."""
# Seed playbook store with default templates
try:
store = PlaybookStore.default()
n = store.seed_defaults()
if n:
logger.info("playbook_store.seeded", count=n)
except Exception as exc:
logger.warning("Playbook store seed failed", error=str(exc))
# Wave 1 — surface an unroutable LLM gateway/alias config at boot instead of
# silently degrading to heuristics on the first (400ing) live call.
for warning in preflight_llm():
logger.warning("llm_preflight", detail=warning)
# Load full MITRE ATT&CK corpus
try:
await load_attck_corpus()
except Exception as exc:
logger.warning("MITRE ATT&CK corpus load failed at startup", error=str(exc))
# Embed into Qdrant for RAG (only if configured)
qdrant_url = os.getenv("QDRANT_URL", "")
openai_key = os.getenv("OPENAI_API_KEY", "")
if qdrant_url and openai_key:
try:
await embed_techniques_into_qdrant(
qdrant_url=qdrant_url,
openai_api_key=openai_key,
)
except Exception as exc:
logger.warning("ATT&CK Qdrant embedding skipped", error=str(exc))
# Warm up the investigation-ledger pool. This is best-effort: if the DB is
# unreachable we keep running, ledger writes just become no-ops.
try:
await investigation_ledger.get_pool()
except Exception as exc: # noqa: BLE001
logger.warning("investigation_ledger.warmup_failed", error=str(exc))
# Start the continuous hunt scheduler (Wave 2 — w2-hac). Gated by env
# so dev/CI runs that don't want background jobs can opt out via
# AISOC_HUNT_SCHEDULER_DISABLE=1. Best-effort: a corpus load failure
# or DB outage must not block API startup.
if os.getenv("AISOC_HUNT_SCHEDULER_DISABLE", "").strip() not in ("1", "true", "yes"):
try:
await hunt_scheduler.start_scheduler()
except Exception as exc: # noqa: BLE001
logger.warning("hunt.scheduler.start_failed", error=str(exc))
# Phase B1 — auto-triage every fused alert off the Kafka stream (copilot:
# read-only triage, no response dispatch). Off unless KAFKA_BOOTSTRAP_SERVERS
# is set; degrades to deterministic triage without an LLM key.
app.state.triage_worker = None
app.state.triage_worker_task = None
if worker_enabled():
try:
# Phase B4 — environment-specific noise reduction (post-fusion →
# pre-triage). Enabled by default; needs a rules file to do anything.
applier = BusinessContextApplier() if business_context_enabled() else None
triage_worker = FusedAlertTriageWorker(
bootstrap_servers=os.environ["KAFKA_BOOTSTRAP_SERVERS"],
topic=os.getenv("KAFKA_TOPIC_ALERTS_FUSED", "aisoc.alerts.fused"),
business_context=applier,
)
app.state.triage_worker = triage_worker
task = asyncio.create_task(triage_worker.start())
# Same pair as fusion and ueba: retrieve the task's outcome so a
# dead worker is in the log, and make readiness depend on the
# subscription rather than on this block having run. Without the
# probe, /readyz answered 200 for a worker whose loop had exited —
# and nothing auto-triages a fused alert after that.
task.add_done_callback(_log_triage_worker_exit)
app.state.triage_worker_task = task
register_subscription(app, triage_worker.topic, lambda: triage_worker.attached)
logger.info("auto_triage_worker.enabled")
except Exception as exc: # noqa: BLE001 — never block API startup
logger.warning("auto_triage_worker.start_failed", error=str(exc))
# Depth 5.3 — wake timer waits and expire pauses nobody decided.
# `pause.expire_due` had been written and never called from production,
# so an undecided approval sat `waiting` forever; a `wait` step inherits
# that and turns it into a run that stops and never continues.
app.state.playbook_sweeper_task = None
try:
playbook_sweeper.start(app)
except Exception as exc: # noqa: BLE001 — never block API startup
logger.warning("playbook_sweeper.start_failed", error=str(exc))
# Issue #1242 — close investigation runs orphaned by a process that died.
# The wall-clock deadline on the manual path only ends a run whose process
# is still alive; the run store is a module dict, so a restart leaves the
# `investigation_runs` row at 'running' with nothing left to finish it.
app.state.investigation_reaper_task = None
try:
investigation_run_reaper.start(app)
except Exception as exc: # noqa: BLE001 — never block API startup
logger.warning("investigation_run_reaper.start_failed", error=str(exc))
# Phase 2.6 — flip /readyz to 200 once startup work is done.
app.state.mark_ready()
yield
# Phase 2.6 — flip /readyz to 503 the moment we start draining.
app.state.mark_not_ready()
# Stop the auto-triage worker first so in-flight triage can finish.
if getattr(app.state, "triage_worker", None) is not None:
try:
await app.state.triage_worker.stop()
if app.state.triage_worker_task is not None:
app.state.triage_worker_task.cancel()
except Exception as exc: # noqa: BLE001
logger.warning("auto_triage_worker.stop_failed", error=str(exc))
if getattr(app.state, "playbook_sweeper_task", None) is not None:
app.state.playbook_sweeper_task.cancel()
if getattr(app.state, "investigation_reaper_task", None) is not None:
app.state.investigation_reaper_task.cancel()
# Stop the hunt scheduler before draining DB pools so in-flight runs
# can flush their writes.
try:
await hunt_scheduler.stop_scheduler()
except Exception as exc: # noqa: BLE001
logger.warning("hunt.scheduler.stop_failed", error=str(exc))
try:
await hunt_store.close_pool()
except Exception as exc: # noqa: BLE001
logger.warning("hunt.store.close_failed", error=str(exc))
# Drain the pool so the container exits cleanly on shutdown.
try:
await investigation_ledger.close_pool()
except Exception as exc: # noqa: BLE001
logger.warning("investigation_ledger.close_failed", error=str(exc))
app = FastAPI(
title="AiSOC Agent Orchestrator",
description="LangGraph-based autonomous investigation and response agents",
version="0.1.0",
lifespan=lifespan,
)
# Phase 2.6 — k8s liveness + readiness probes (see app/_health.py).
# /readyz returns 503 until the lifespan startup finishes touching
# MITRE, Qdrant, ledger pool, and the hunt scheduler.
_mark_ready, _mark_not_ready = install_health_routes(app, service_name="aisoc-agents")
app.state.mark_ready = _mark_ready
app.state.mark_not_ready = _mark_not_ready
async def _database_ready() -> bool:
import asyncpg
dsn = os.getenv("DATABASE_URL", "").strip()
if not dsn:
return False
for prefix in ("postgresql+asyncpg://", "postgres+asyncpg://"):
if dsn.startswith(prefix):
dsn = "postgresql://" + dsn[len(prefix) :]
break
conn = await asyncpg.connect(dsn, timeout=1.4)
await conn.close()
return True
register_dependency(app, "database", _database_ready)
# CORS — the web console talks to this service directly (it does not go through
# the Next.js rewrite layer for agent endpoints because we want to stream
# NDJSON without buffering through the proxy). Origins are resolved via the
# shared ``build_cors_kwargs`` helper which reads AISOC_CORS_ORIGINS (canonical)
# / CORS_ORIGINS (legacy) and enforces the "no wildcard with credentials in
# production" invariant so a careless ``export CORS_ORIGINS=*`` can't ship
# CSRF to prod.
from app.core.cors import build_cors_kwargs # noqa: E402
app.add_middleware(
CORSMiddleware,
**build_cors_kwargs(service_name="agents", allow_credentials=True),
)
# OpenTelemetry auto-instrumentation (FastAPI + httpx)
instrument_app(app)
# Unprefixed: Prometheus scrapes /metrics, not /api/v1/metrics, and the
# scrape config is shared shape across services.
app.include_router(metrics_router)
app.include_router(router, prefix="/api/v1")
app.include_router(investigate_router) # prefix already set in investigate.py
app.include_router(triage_router) # prefix: /api/v1 (POST /cases/{id}/triage — router topology, T2.2)
app.include_router(playbook_router)
app.include_router(playbook_waits_router) # prefix: /api/v1/playbook-waits
app.include_router(playbook_verifications_router) # prefix: /api/v1/playbook-verifications
app.include_router(contextual_router) # prefix: /api/v1/contextual
app.include_router(hunts_router) # prefix: /api/v1/hunts
app.include_router(hunt_search_router) # prefix: /api/v1/hunt (search + saved)
app.include_router(copilot_router) # prefix: /api/v1/copilot
app.include_router(explain_router) # prefix: /api/v1 (POST /explain — NDJSON stream)
# Gap-closure Phase 1.4: POST /api/v1/replay/run. The runner shipped in 1.2
# with no caller; this is what the API's evaluation job drives.
app.include_router(replay_router, prefix="/api/v1")
@app.get("/health")
async def health():
from app.tools.mitre_full import get_coverage_summary
summary = get_coverage_summary()
return {
"status": "healthy",
"service": "aisoc-agents",
"attck_corpus": summary,
}