Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
"""
AiSOC Threat Intelligence Feed Service
Polls multiple threat intelligence feeds on configurable intervals,
deduplicates via Redis Bloom filter, and writes normalized IOCs/actors
into OpenSearch, Qdrant, and Neo4j.
AiSOC — open-source AI Security Operations Center (MIT License)
"""
from __future__ import annotations
import asyncio
from collections.abc import AsyncGenerator
from contextlib import asynccontextmanager
from functools import partial
from typing import Annotated
import redis.asyncio as aioredis
import structlog
from aiokafka import AIOKafkaProducer
from fastapi import Depends, FastAPI
from neo4j import AsyncGraphDatabase
from opensearchpy import AsyncOpenSearch
from prometheus_client import Counter, make_asgi_app
from qdrant_client import AsyncQdrantClient
from app._health import install_health_routes
from app.actors.attribution import ThreatActorAttributionEngine
from app.airgap import airgap_status, is_host_allowed_for_airgap
from app.api.actor_attribution import router as actor_attribution_router
from app.api.indicators import router as indicators_router
from app.clients.cisa_kev import CisaKevClient
from app.clients.misp import MispClient
from app.clients.opencti import OpenCtiClient
from app.clients.otx import OtxClient
from app.clients.taxii import TaxiiClient
from app.config import settings
from app.core.kafka_security import kafka_client_kwargs
from app.feeds.handlers import (
handle_cisa_kev_feed,
handle_misp_feed,
handle_opencti_feed,
handle_otx_feed,
handle_taxii_feed,
)
from app.feeds.pipeline import ThreatIntelPipeline
from app.feeds.scheduler import FeedScheduler
from app.security.tenant_scope import TenantPrincipal, require_console_or_service_auth
from app.storage.bloom import RedisBloomFilter
from app.storage.neo4j import Neo4jStore
from app.storage.opensearch import OpenSearchStore
from app.storage.qdrant import QdrantStore
structlog.configure(
processors=[
structlog.contextvars.merge_contextvars,
structlog.processors.add_log_level,
structlog.processors.TimeStamper(fmt="iso"),
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
structlog.processors.UnicodeDecoder(),
structlog.processors.JSONRenderer(),
],
context_class=dict,
logger_factory=structlog.PrintLoggerFactory(),
cache_logger_on_first_use=True,
)
logger = structlog.get_logger(__name__)
# ─── Prometheus metrics ───────────────────────────────────────────────────────
iocs_ingested = Counter("threatintel_iocs_ingested_total", "Total IOCs ingested", ["source"])
actors_ingested = Counter("threatintel_actors_ingested_total", "Total actors ingested", ["source"])
# ─── Application lifespan ─────────────────────────────────────────────────────
def _airgap_check_feed_url(feed_name: str, url: str | None) -> bool:
"""Return True if a feed at ``url`` may be polled under current air-gap policy.
Logs and refuses registration when air-gap mode is on and the feed's URL
points at a public host that is not on the allowlist. Internal mirrors
(RFC1918, ``.local``, ``.internal``, single-label hosts, allowlisted
suffixes) are always permitted so customers running their own MISP or
TAXII server inside the perimeter can keep polling normally.
"""
if not settings.AISOC_AIRGAPPED:
return True
if not url:
# Feeds without a configured URL (e.g. CISA KEV uses a hardcoded
# public URL inside the client) are evaluated by their default URL
# at the call site below; the bare check here only short-circuits
# operator-supplied URLs.
return True
from urllib.parse import urlparse
host = urlparse(url).hostname or ""
allowed = is_host_allowed_for_airgap(host)
if not allowed:
logger.warning(
"airgap.feed_blocked",
feed=feed_name,
host=host,
reason="public host blocked by AISOC_AIRGAPPED policy",
)
return allowed
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
logger.info(
"ThreatIntel service starting…",
airgap=airgap_status(),
)
# ── Redis ──────────────────────────────────────────────────────────────────
redis = aioredis.from_url(settings.REDIS_URL, decode_responses=False)
# ── OpenSearch ────────────────────────────────────────────────────────────
os_client = AsyncOpenSearch(
hosts=[{"host": settings.OPENSEARCH_HOST, "port": settings.OPENSEARCH_PORT}],
http_auth=(settings.OPENSEARCH_USER, settings.OPENSEARCH_PASSWORD) if settings.OPENSEARCH_USER else None,
use_ssl=False,
)
# ── Qdrant ────────────────────────────────────────────────────────────────
qdrant_client = AsyncQdrantClient(host=settings.QDRANT_HOST, port=settings.QDRANT_PORT)
# ── Neo4j ─────────────────────────────────────────────────────────────────
neo4j_driver = AsyncGraphDatabase.driver(
settings.NEO4J_URI,
auth=(settings.NEO4J_USER, settings.NEO4J_PASSWORD),
)
# ── Kafka (optional) ──────────────────────────────────────────────────────
kafka_producer = None
if settings.KAFKA_BOOTSTRAP_SERVERS:
try:
kafka_producer = AIOKafkaProducer(
bootstrap_servers=settings.KAFKA_BOOTSTRAP_SERVERS,
**kafka_client_kwargs(),
)
await kafka_producer.start()
except Exception as exc:
logger.warning("Kafka producer unavailable", error=str(exc))
kafka_producer = None
# ── Storage layer ─────────────────────────────────────────────────────────
bloom = RedisBloomFilter(redis)
os_store = OpenSearchStore(os_client)
qdrant_store = QdrantStore(qdrant_client)
neo4j_store = Neo4jStore(neo4j_driver)
# Both stores are best-effort. OpenSearch used to be the one unguarded
# call in this lifespan, which made it a hard dependency of the whole
# service: it ships in the `full` profile, this service now ships in CORE,
# and an unreachable OpenSearch killed the container before a single feed
# was registered. Qdrant is the CORE store and is what the console reads
# through; OpenSearch adds full-text search on top when it is there.
try:
await os_store.initialize()
except Exception as exc:
logger.warning(
"OpenSearch init failed — full-text IOC search is unavailable; feeds still write to Qdrant",
error=str(exc),
)
try:
await qdrant_store.initialize()
except Exception as exc:
logger.warning("Qdrant init failed", error=str(exc))
# ── Pipeline ──────────────────────────────────────────────────────────────
pipeline = ThreatIntelPipeline(
bloom=bloom,
os_store=os_store,
qdrant_store=qdrant_store,
neo4j_store=neo4j_store,
kafka_producer=kafka_producer,
kafka_topic=settings.KAFKA_TOPIC_THREAT_INTEL,
)
# ── Clients ───────────────────────────────────────────────────────────────
taxii_client = TaxiiClient(
base_url=settings.TAXII_URL,
username=settings.TAXII_USERNAME,
password=settings.TAXII_PASSWORD,
)
misp_client = (
MispClient(
url=settings.MISP_URL,
api_key=settings.MISP_API_KEY,
)
if settings.MISP_URL
else None
)
otx_client = (
OtxClient(
api_key=settings.OTX_API_KEY,
)
if settings.OTX_API_KEY
else None
)
opencti_client = (
OpenCtiClient(
url=settings.OPENCTI_URL,
token=settings.OPENCTI_TOKEN,
verify_ssl=settings.OPENCTI_VERIFY_SSL,
)
if settings.OPENCTI_URL and settings.OPENCTI_TOKEN
else None
)
kev_client = CisaKevClient()
# ── Scheduler ─────────────────────────────────────────────────────────────
scheduler = FeedScheduler(pipeline)
# Register TAXII feeds (skipped entirely if the configured TAXII server
# is on the public Internet and AISOC_AIRGAPPED=1 — this prevents even
# a single boot-time DNS lookup from leaking the deployment).
if settings.TAXII_URL and settings.TAXII_COLLECTION_IDS and _airgap_check_feed_url("taxii", settings.TAXII_URL):
for col_id in settings.TAXII_COLLECTION_IDS.split(","):
col_id = col_id.strip()
if col_id:
scheduler.register(
feed_name=f"taxii:{col_id}",
handler=partial(
handle_taxii_feed,
client=taxii_client,
pipeline=pipeline,
api_root=settings.TAXII_API_ROOT,
collection_id=col_id,
),
interval_seconds=settings.TAXII_POLL_INTERVAL,
)
# Register MISP feed
if misp_client and _airgap_check_feed_url("misp", settings.MISP_URL):
scheduler.register(
feed_name="misp",
handler=partial(handle_misp_feed, client=misp_client, pipeline=pipeline),
interval_seconds=settings.MISP_POLL_INTERVAL,
)
# Register OTX feed (always public, so air-gapped mode unconditionally
# disables it — there is no internal mirror of AlienVault OTX).
if otx_client and _airgap_check_feed_url("otx", "https://otx.alienvault.com"):
scheduler.register(
feed_name="otx",
handler=partial(handle_otx_feed, client=otx_client, pipeline=pipeline),
interval_seconds=settings.OTX_POLL_INTERVAL,
)
# OpenCTI (typically an internal deployment — the air-gap helper
# permits RFC1918/internal hosts and skips public URLs unless
# explicitly allowlisted, same contract as the other feeds).
if opencti_client and _airgap_check_feed_url("opencti", settings.OPENCTI_URL):
scheduler.register(
feed_name="opencti",
handler=partial(handle_opencti_feed, client=opencti_client, pipeline=pipeline),
interval_seconds=settings.OPENCTI_POLL_INTERVAL,
)
# CISA KEV (always enabled — no key required) — public CISA URL, so
# an air-gapped deployment must mirror the JSON internally and
# override via an internal allowlist entry, otherwise we skip.
if _airgap_check_feed_url("cisa-kev", "https://www.cisa.gov"):
scheduler.register(
feed_name="cisa-kev",
handler=partial(handle_cisa_kev_feed, client=kev_client, pipeline=pipeline),
interval_seconds=settings.CISA_KEV_POLL_INTERVAL,
)
scheduler.start()
# Store refs for health endpoint
app.state.scheduler = scheduler
app.state.pipeline = pipeline
app.state.redis = redis
app.state.os_store = os_store
# Read path for GET /api/v1/threat-intel/indicators. Qdrant is the sink
# CORE has, so it is the one the console's IOC list is served from.
app.state.qdrant_client = qdrant_client
# Threat actor attribution engine — shares the os_store so the IOC
# component of the score can match against collected threat intel.
app.state.attribution_engine = ThreatActorAttributionEngine(os_store=os_store)
# Phase 2.6 — flip /readyz to 200 once feed scheduler + storage clients
# are wired. Failures in any individual feed degrade gracefully (the
# handler logs and the scheduler retries), so we don't tie readiness
# to a specific feed being reachable.
app.state.mark_ready()
yield
# Phase 2.6 — drain readiness before tearing down feeds + storage.
app.state.mark_not_ready()
# ── Shutdown ──────────────────────────────────────────────────────────────
scheduler.stop()
if kafka_producer:
await kafka_producer.stop()
await neo4j_driver.close()
await os_client.close()
await redis.close()
logger.info("ThreatIntel service stopped")
# ─── FastAPI app ──────────────────────────────────────────────────────────────
app = FastAPI(
title="AiSOC Threat Intelligence Service",
version="0.1.0",
lifespan=lifespan,
)
# Phase 2.6 — k8s liveness + readiness probes (see app/_health.py).
_mark_ready, _mark_not_ready = install_health_routes(app, service_name="aisoc-threatintel")
app.state.mark_ready = _mark_ready
app.state.mark_not_ready = _mark_not_ready
# Mount Prometheus metrics
metrics_app = make_asgi_app()
app.mount("/metrics", metrics_app)
# Threat actor attribution router (v0)
app.include_router(actor_attribution_router)
# What the feeds have collected. The API proxies this to serve the console's
# /threat-intel page, which had no backend at all in CORE.
app.include_router(indicators_router)
@app.get("/health")
async def health() -> dict:
"""Health and status endpoint, naming every dependency it checked.
``vector_store`` is the Qdrant similarity leg (#1446): a **degrade**
dependency — IOC ingestion continues without it and enrichment responses
say the similarity leg was skipped — so its loss flips the key while the
status stays ``ok``. The chaos store-outage grader reads exactly this
key; before it existed, Qdrant being down was invisible here, which is
indistinguishable from a healthy idle store. Each probe is hard-capped
well inside the grader's 4-second read budget, because a probe that
waits out a full connect timeout turns "down and we say so" into "down
and we say nothing".
"""
try:
async with asyncio.timeout(1.5):
redis: aioredis.Redis = app.state.redis
await redis.ping()
redis_ok = True
except Exception:
redis_ok = False
try:
async with asyncio.timeout(1.5):
await app.state.qdrant_client.get_collections()
vector_ok = True
except Exception:
vector_ok = False
return {
"status": "ok",
"redis": redis_ok,
"vector_store": vector_ok,
"scheduler": app.state.scheduler._scheduler.running if hasattr(app.state, "scheduler") else False,
"airgap": airgap_status(),
}
@app.get("/api/v1/iocs/search")
async def search_iocs(
principal: Annotated[TenantPrincipal, Depends(require_console_or_service_auth)],
value: str | None = None,
ioc_type: str | None = None,
source: str | None = None,
limit: int = 20,
) -> dict:
"""Search stored IOCs."""
pipeline: ThreatIntelPipeline = app.state.pipeline
iocs = await pipeline._os.search_iocs(value=value, ioc_type=ioc_type, source=source, limit=limit)
return {"total": len(iocs), "iocs": iocs}