Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

9.5 KB
Raw Blame
"""Read the indicators this service has collected.

Why this route exists
---------------------
The console ships a `/threat-intel` page that calls
``GET /api/v1/threat-intel/indicators``. Nothing implemented it. The page was
recently caught rendering five invented IOCs and a hard-coded "3 Added Today"
counter on tenants that had never ingested one; removing the fabrication left
the page correct and empty, because the real feed was in the `full` profile and
the endpoint it wanted did not exist in any profile. The missing feed and the
fabrication were one hole.

``services/threatintel`` and Qdrant now ship in CORE, and the CISA Known
Exploited Vulnerabilities catalog is authoritative, public and needs no API
key — so a default ``make up`` collects real indicators. This route serves
them, and the API proxies it.

Why Qdrant and not OpenSearch
-----------------------------
``ThreatIntelPipeline`` writes to three sinks. OpenSearch is the full-text one
and stays in the `full` profile; Qdrant is the one CORE has, it already carries
the complete IOC payload on every point, and ``tenant_scope_filter`` already
expresses the read scope this route needs — public feed intel under the
``shared`` sentinel, private intel under its owning tenant. Serving from the
store that is actually present is what makes the page work in CORE.

Scrolling a vector store is an unusual list path, and it is bounded on purpose:
``limit`` is capped, and this is a browse surface, not an export.
"""

from __future__ import annotations

from typing import Annotated, Any

import structlog
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from pydantic import BaseModel, Field
from qdrant_client import AsyncQdrantClient

from app.security.tenant_scope import TenantPrincipal, require_console_or_service_auth
from app.storage.qdrant import IOC_COLLECTION, tenant_scope_filter

logger = structlog.get_logger(__name__)

router = APIRouter(
    prefix="/api/v1/threat-intel",
    tags=["threat-intel"],
    dependencies=[Depends(require_console_or_service_auth)],
)

#: Ceiling on one page. The console renders a browse list; anything wanting
#: the whole catalog should read the feed at source.
MAX_LIMIT = 500


class Indicator(BaseModel):
    """One indicator, in the shape the console's IOC inbox renders.

    Deliberately permissive about the source payload: feeds disagree about
    which fields they carry, and a missing `description` is not a reason to
    drop a real IOC from the list.
    """

    type: str
    value: str
    description: str | None = None
    confidence: int = 50
    malicious: bool = True
    sources: list[str] = Field(default_factory=list)
    tags: list[str] = Field(default_factory=list)
    first_seen: str | None = None
    last_seen: str | None = None
    tlp: str | None = None


class IndicatorListResponse(BaseModel):
    indicators: list[Indicator]
    #: How many indicators are in the caller's scope, from the store's own
    #: count — not how many this response carries, and not how many the scan
    #: window happened to reach.
    #:
    #: It used to be ``len(indicators)`` after post-filtering, i.e. the size of
    #: the bounded scan window. With 1,725 KEV entries collected, the store
    #: held 1,725, this field said 400 and the console's headline card said
    #: 100 — three different answers to one question, the two published ones
    #: both wrong, and the page presenting the smallest as the corpus.
    total: int
    #: How many indicators this response carries. Distinct from ``total`` on
    #: purpose: the console needs to say "100 of 1,725" rather than implying
    #: a page is the catalogue.
    shown: int = 0
    #: True when ``type`` / ``tag`` / ``q`` narrowed the result, in which case
    #: the match was made inside the bounded scan window rather than across
    #: the whole collection — so ``shown`` is a lower bound on the matches
    #: that exist. Published rather than hidden because a search that silently
    #: reads part of the corpus is the kind of thing that gets believed.
    bounded: bool = False
    #: Which store answered. The console has no other way to tell "no
    #: indicators collected yet" from "the store is not there", and the two
    #: call for different things from the reader.
    source: str


def _to_indicator(payload: dict[str, Any]) -> Indicator:
    raw_tags = payload.get("tags")
    tags = [str(t) for t in raw_tags] if isinstance(raw_tags, list) else []
    source = payload.get("source")
    return Indicator(
        type=str(payload.get("type") or "unknown"),
        value=str(payload.get("value") or ""),
        description=payload.get("description") or payload.get("vulnerability_name") or None,
        confidence=int(payload.get("confidence") or 50),
        # A feed entry is an indicator of compromise; CISA KEV in particular
        # lists vulnerabilities under active exploitation. `false` here would
        # be a claim the feed does not make.
        malicious=bool(payload.get("malicious", True)),
        sources=[str(source)] if source else [],
        tags=tags,
        first_seen=payload.get("first_seen") or payload.get("date_added"),
        last_seen=payload.get("last_seen") or payload.get("date_added"),
        tlp=payload.get("tlp"),
    )


def _qdrant(request: Request) -> AsyncQdrantClient:
    client = getattr(request.app.state, "qdrant_client", None)
    if client is None:
        raise HTTPException(status_code=503, detail="vector store is not configured on this deployment")
    return client


async def _scope_total(client: AsyncQdrantClient, scope: Any, *, fallback: int) -> int:
    """How many indicators are in the caller's scope, asked of the store.

    The scan above is deliberately bounded, so counting its results answers
    "how many did I look at", which is not the question the console is asking.
    Qdrant can answer the real one directly.

    A count failure falls back to the window size rather than raising: the
    indicators themselves were read successfully, and losing the whole list
    over a secondary count would turn a wrong number into no page at all.
    """
    try:
        result = await client.count(collection_name=IOC_COLLECTION, count_filter=scope, exact=True)
    except Exception as exc:  # noqa: BLE001 — any count failure degrades to the window size
        logger.warning("threatintel.indicators.count_failed", error=type(exc).__name__)
        return fallback
    count = getattr(result, "count", None)
    return int(count) if isinstance(count, int) else fallback


@router.get("/indicators", response_model=IndicatorListResponse)
async def list_indicators(
    request: Request,
    principal: Annotated[TenantPrincipal, Depends(require_console_or_service_auth)],
    # `alias` keeps the query string `?type=` the console already sends while
    # the parameter itself is named something that does not shadow `type()` —
    # which this function calls, in the except arm below.
    ioc_type: str | None = Query(default=None, alias="type", description="filter to one indicator type"),
    tag: str | None = Query(default=None),
    q: str | None = Query(default=None, description="substring match on value or description"),
    limit: int = Query(default=100, ge=1, le=MAX_LIMIT),
) -> IndicatorListResponse:
    """Indicators collected by the feed scheduler, scoped to the caller.

    ``tenant_scope_filter`` is what enforces the scope: a caller sees their own
    tenant's points plus the ``shared`` sentinel that public feed intel is
    written under. A principal with no tenants is an empty scope, never a
    global one.
    """
    client = _qdrant(request)

    tenant_ids = principal.ordered_ids()
    scope = tenant_scope_filter(str(tenant_ids[0]) if tenant_ids else None)

    try:
        points, _next_page = await client.scroll(
            collection_name=IOC_COLLECTION,
            scroll_filter=scope,
            # Over-read so post-filtering (`q`, `tag`, `type`) still fills a
            # page. Bounded at 4x so a wide filter cannot walk the collection.
            limit=min(limit * 4, MAX_LIMIT * 4),
            with_payload=True,
            with_vectors=False,
        )
    except Exception as exc:
        # An absent collection is "nothing collected yet", not an error; the
        # scheduler creates it on the first successful poll. Anything else is
        # reported rather than rendered as an empty list, because an empty list
        # is indistinguishable from a working store with no data.
        message = str(exc)
        if "not found" in message.lower() or "doesn't exist" in message.lower():
            logger.info("threatintel.indicators.collection_absent", collection=IOC_COLLECTION)
            return IndicatorListResponse(indicators=[], total=0, shown=0, source="qdrant")
        logger.warning("threatintel.indicators.read_failed", error=type(exc).__name__)
        raise HTTPException(
            status_code=503,
            detail="the threat-intel vector store did not answer; no indicators can be listed",
        ) from exc

    indicators = [_to_indicator(p.payload or {}) for p in points if p.payload]

    bounded = bool(ioc_type or tag or q)
    if ioc_type:
        indicators = [i for i in indicators if i.type == ioc_type]
    if tag:
        indicators = [i for i in indicators if tag in i.tags]
    if q:
        needle = q.lower()
        indicators = [i for i in indicators if needle in i.value.lower() or needle in (i.description or "").lower()]

    page = indicators[:limit]
    total = await _scope_total(client, scope, fallback=len(indicators))
    return IndicatorListResponse(indicators=page, total=total, shown=len(page), bounded=bounded, source="qdrant")