Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

16.0 KB
Raw Blame
"""
AiSOC Threat Intelligence Feed Service

Polls multiple threat intelligence feeds on configurable intervals,
deduplicates via Redis Bloom filter, and writes normalized IOCs/actors
into OpenSearch, Qdrant, and Neo4j.

AiSOC — open-source AI Security Operations Center (MIT License)
"""

from __future__ import annotations

import asyncio
from collections.abc import AsyncGenerator
from contextlib import asynccontextmanager
from functools import partial
from typing import Annotated

import redis.asyncio as aioredis
import structlog
from aiokafka import AIOKafkaProducer
from fastapi import Depends, FastAPI
from neo4j import AsyncGraphDatabase
from opensearchpy import AsyncOpenSearch
from prometheus_client import Counter, make_asgi_app
from qdrant_client import AsyncQdrantClient

from app._health import install_health_routes
from app.actors.attribution import ThreatActorAttributionEngine
from app.airgap import airgap_status, is_host_allowed_for_airgap
from app.api.actor_attribution import router as actor_attribution_router
from app.api.indicators import router as indicators_router
from app.clients.cisa_kev import CisaKevClient
from app.clients.misp import MispClient
from app.clients.opencti import OpenCtiClient
from app.clients.otx import OtxClient
from app.clients.taxii import TaxiiClient
from app.config import settings
from app.core.kafka_security import kafka_client_kwargs
from app.feeds.handlers import (
    handle_cisa_kev_feed,
    handle_misp_feed,
    handle_opencti_feed,
    handle_otx_feed,
    handle_taxii_feed,
)
from app.feeds.pipeline import ThreatIntelPipeline
from app.feeds.scheduler import FeedScheduler
from app.security.tenant_scope import TenantPrincipal, require_console_or_service_auth
from app.storage.bloom import RedisBloomFilter
from app.storage.neo4j import Neo4jStore
from app.storage.opensearch import OpenSearchStore
from app.storage.qdrant import QdrantStore

structlog.configure(
    processors=[
        structlog.contextvars.merge_contextvars,
        structlog.processors.add_log_level,
        structlog.processors.TimeStamper(fmt="iso"),
        structlog.processors.StackInfoRenderer(),
        structlog.processors.format_exc_info,
        structlog.processors.UnicodeDecoder(),
        structlog.processors.JSONRenderer(),
    ],
    context_class=dict,
    logger_factory=structlog.PrintLoggerFactory(),
    cache_logger_on_first_use=True,
)
logger = structlog.get_logger(__name__)

# ─── Prometheus metrics ───────────────────────────────────────────────────────
iocs_ingested = Counter("threatintel_iocs_ingested_total", "Total IOCs ingested", ["source"])
actors_ingested = Counter("threatintel_actors_ingested_total", "Total actors ingested", ["source"])


# ─── Application lifespan ─────────────────────────────────────────────────────


def _airgap_check_feed_url(feed_name: str, url: str | None) -> bool:
    """Return True if a feed at ``url`` may be polled under current air-gap policy.

    Logs and refuses registration when air-gap mode is on and the feed's URL
    points at a public host that is not on the allowlist. Internal mirrors
    (RFC1918, ``.local``, ``.internal``, single-label hosts, allowlisted
    suffixes) are always permitted so customers running their own MISP or
    TAXII server inside the perimeter can keep polling normally.
    """

    if not settings.AISOC_AIRGAPPED:
        return True

    if not url:
        # Feeds without a configured URL (e.g. CISA KEV uses a hardcoded
        # public URL inside the client) are evaluated by their default URL
        # at the call site below; the bare check here only short-circuits
        # operator-supplied URLs.
        return True

    from urllib.parse import urlparse

    host = urlparse(url).hostname or ""
    allowed = is_host_allowed_for_airgap(host)
    if not allowed:
        logger.warning(
            "airgap.feed_blocked",
            feed=feed_name,
            host=host,
            reason="public host blocked by AISOC_AIRGAPPED policy",
        )
    return allowed


@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
    logger.info(
        "ThreatIntel service starting…",
        airgap=airgap_status(),
    )

    # ── Redis ──────────────────────────────────────────────────────────────────
    redis = aioredis.from_url(settings.REDIS_URL, decode_responses=False)

    # ── OpenSearch ────────────────────────────────────────────────────────────
    os_client = AsyncOpenSearch(
        hosts=[{"host": settings.OPENSEARCH_HOST, "port": settings.OPENSEARCH_PORT}],
        http_auth=(settings.OPENSEARCH_USER, settings.OPENSEARCH_PASSWORD) if settings.OPENSEARCH_USER else None,
        use_ssl=False,
    )

    # ── Qdrant ────────────────────────────────────────────────────────────────
    qdrant_client = AsyncQdrantClient(host=settings.QDRANT_HOST, port=settings.QDRANT_PORT)

    # ── Neo4j ─────────────────────────────────────────────────────────────────
    neo4j_driver = AsyncGraphDatabase.driver(
        settings.NEO4J_URI,
        auth=(settings.NEO4J_USER, settings.NEO4J_PASSWORD),
    )

    # ── Kafka (optional) ──────────────────────────────────────────────────────
    kafka_producer = None
    if settings.KAFKA_BOOTSTRAP_SERVERS:
        try:
            kafka_producer = AIOKafkaProducer(
                bootstrap_servers=settings.KAFKA_BOOTSTRAP_SERVERS,
                **kafka_client_kwargs(),
            )
            await kafka_producer.start()
        except Exception as exc:
            logger.warning("Kafka producer unavailable", error=str(exc))
            kafka_producer = None

    # ── Storage layer ─────────────────────────────────────────────────────────
    bloom = RedisBloomFilter(redis)
    os_store = OpenSearchStore(os_client)
    qdrant_store = QdrantStore(qdrant_client)
    neo4j_store = Neo4jStore(neo4j_driver)

    # Both stores are best-effort. OpenSearch used to be the one unguarded
    # call in this lifespan, which made it a hard dependency of the whole
    # service: it ships in the `full` profile, this service now ships in CORE,
    # and an unreachable OpenSearch killed the container before a single feed
    # was registered. Qdrant is the CORE store and is what the console reads
    # through; OpenSearch adds full-text search on top when it is there.
    try:
        await os_store.initialize()
    except Exception as exc:
        logger.warning(
            "OpenSearch init failed — full-text IOC search is unavailable; feeds still write to Qdrant",
            error=str(exc),
        )
    try:
        await qdrant_store.initialize()
    except Exception as exc:
        logger.warning("Qdrant init failed", error=str(exc))

    # ── Pipeline ──────────────────────────────────────────────────────────────
    pipeline = ThreatIntelPipeline(
        bloom=bloom,
        os_store=os_store,
        qdrant_store=qdrant_store,
        neo4j_store=neo4j_store,
        kafka_producer=kafka_producer,
        kafka_topic=settings.KAFKA_TOPIC_THREAT_INTEL,
    )

    # ── Clients ───────────────────────────────────────────────────────────────
    taxii_client = TaxiiClient(
        base_url=settings.TAXII_URL,
        username=settings.TAXII_USERNAME,
        password=settings.TAXII_PASSWORD,
    )

    misp_client = (
        MispClient(
            url=settings.MISP_URL,
            api_key=settings.MISP_API_KEY,
        )
        if settings.MISP_URL
        else None
    )

    otx_client = (
        OtxClient(
            api_key=settings.OTX_API_KEY,
        )
        if settings.OTX_API_KEY
        else None
    )

    opencti_client = (
        OpenCtiClient(
            url=settings.OPENCTI_URL,
            token=settings.OPENCTI_TOKEN,
            verify_ssl=settings.OPENCTI_VERIFY_SSL,
        )
        if settings.OPENCTI_URL and settings.OPENCTI_TOKEN
        else None
    )

    kev_client = CisaKevClient()

    # ── Scheduler ─────────────────────────────────────────────────────────────
    scheduler = FeedScheduler(pipeline)

    # Register TAXII feeds (skipped entirely if the configured TAXII server
    # is on the public Internet and AISOC_AIRGAPPED=1 — this prevents even
    # a single boot-time DNS lookup from leaking the deployment).
    if settings.TAXII_URL and settings.TAXII_COLLECTION_IDS and _airgap_check_feed_url("taxii", settings.TAXII_URL):
        for col_id in settings.TAXII_COLLECTION_IDS.split(","):
            col_id = col_id.strip()
            if col_id:
                scheduler.register(
                    feed_name=f"taxii:{col_id}",
                    handler=partial(
                        handle_taxii_feed,
                        client=taxii_client,
                        pipeline=pipeline,
                        api_root=settings.TAXII_API_ROOT,
                        collection_id=col_id,
                    ),
                    interval_seconds=settings.TAXII_POLL_INTERVAL,
                )

    # Register MISP feed
    if misp_client and _airgap_check_feed_url("misp", settings.MISP_URL):
        scheduler.register(
            feed_name="misp",
            handler=partial(handle_misp_feed, client=misp_client, pipeline=pipeline),
            interval_seconds=settings.MISP_POLL_INTERVAL,
        )

    # Register OTX feed (always public, so air-gapped mode unconditionally
    # disables it — there is no internal mirror of AlienVault OTX).
    if otx_client and _airgap_check_feed_url("otx", "https://otx.alienvault.com"):
        scheduler.register(
            feed_name="otx",
            handler=partial(handle_otx_feed, client=otx_client, pipeline=pipeline),
            interval_seconds=settings.OTX_POLL_INTERVAL,
        )

    # OpenCTI (typically an internal deployment — the air-gap helper
    # permits RFC1918/internal hosts and skips public URLs unless
    # explicitly allowlisted, same contract as the other feeds).
    if opencti_client and _airgap_check_feed_url("opencti", settings.OPENCTI_URL):
        scheduler.register(
            feed_name="opencti",
            handler=partial(handle_opencti_feed, client=opencti_client, pipeline=pipeline),
            interval_seconds=settings.OPENCTI_POLL_INTERVAL,
        )

    # CISA KEV (always enabled — no key required) — public CISA URL, so
    # an air-gapped deployment must mirror the JSON internally and
    # override via an internal allowlist entry, otherwise we skip.
    if _airgap_check_feed_url("cisa-kev", "https://www.cisa.gov"):
        scheduler.register(
            feed_name="cisa-kev",
            handler=partial(handle_cisa_kev_feed, client=kev_client, pipeline=pipeline),
            interval_seconds=settings.CISA_KEV_POLL_INTERVAL,
        )

    scheduler.start()

    # Store refs for health endpoint
    app.state.scheduler = scheduler
    app.state.pipeline = pipeline
    app.state.redis = redis
    app.state.os_store = os_store
    # Read path for GET /api/v1/threat-intel/indicators. Qdrant is the sink
    # CORE has, so it is the one the console's IOC list is served from.
    app.state.qdrant_client = qdrant_client

    # Threat actor attribution engine — shares the os_store so the IOC
    # component of the score can match against collected threat intel.
    app.state.attribution_engine = ThreatActorAttributionEngine(os_store=os_store)

    # Phase 2.6 — flip /readyz to 200 once feed scheduler + storage clients
    # are wired. Failures in any individual feed degrade gracefully (the
    # handler logs and the scheduler retries), so we don't tie readiness
    # to a specific feed being reachable.
    app.state.mark_ready()

    yield

    # Phase 2.6 — drain readiness before tearing down feeds + storage.
    app.state.mark_not_ready()

    # ── Shutdown ──────────────────────────────────────────────────────────────
    scheduler.stop()
    if kafka_producer:
        await kafka_producer.stop()
    await neo4j_driver.close()
    await os_client.close()
    await redis.close()
    logger.info("ThreatIntel service stopped")


# ─── FastAPI app ──────────────────────────────────────────────────────────────

app = FastAPI(
    title="AiSOC Threat Intelligence Service",
    version="0.1.0",
    lifespan=lifespan,
)

# Phase 2.6 — k8s liveness + readiness probes (see app/_health.py).
_mark_ready, _mark_not_ready = install_health_routes(app, service_name="aisoc-threatintel")
app.state.mark_ready = _mark_ready
app.state.mark_not_ready = _mark_not_ready

# Mount Prometheus metrics
metrics_app = make_asgi_app()
app.mount("/metrics", metrics_app)

# Threat actor attribution router (v0)
app.include_router(actor_attribution_router)

# What the feeds have collected. The API proxies this to serve the console's
# /threat-intel page, which had no backend at all in CORE.
app.include_router(indicators_router)


@app.get("/health")
async def health() -> dict:
    """Health and status endpoint, naming every dependency it checked.

    ``vector_store`` is the Qdrant similarity leg (#1446): a **degrade**
    dependency — IOC ingestion continues without it and enrichment responses
    say the similarity leg was skipped — so its loss flips the key while the
    status stays ``ok``. The chaos store-outage grader reads exactly this
    key; before it existed, Qdrant being down was invisible here, which is
    indistinguishable from a healthy idle store. Each probe is hard-capped
    well inside the grader's 4-second read budget, because a probe that
    waits out a full connect timeout turns "down and we say so" into "down
    and we say nothing".
    """
    try:
        async with asyncio.timeout(1.5):
            redis: aioredis.Redis = app.state.redis
            await redis.ping()
        redis_ok = True
    except Exception:
        redis_ok = False

    try:
        async with asyncio.timeout(1.5):
            await app.state.qdrant_client.get_collections()
        vector_ok = True
    except Exception:
        vector_ok = False

    return {
        "status": "ok",
        "redis": redis_ok,
        "vector_store": vector_ok,
        "scheduler": app.state.scheduler._scheduler.running if hasattr(app.state, "scheduler") else False,
        "airgap": airgap_status(),
    }


@app.get("/api/v1/iocs/search")
async def search_iocs(
    principal: Annotated[TenantPrincipal, Depends(require_console_or_service_auth)],
    value: str | None = None,
    ioc_type: str | None = None,
    source: str | None = None,
    limit: int = 20,
) -> dict:
    """Search stored IOCs."""
    pipeline: ThreatIntelPipeline = app.state.pipeline
    iocs = await pipeline._os.search_iocs(value=value, ioc_type=ioc_type, source=source, limit=limit)
    return {"total": len(iocs), "iocs": iocs}