Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
fix(threatintel): embedding ran on the event loop, so the service answered nothing while a feed ingested (#1446) (#1495)
Beenu Arora93030bfda588
diff --git a/services/threatintel/app/storage/qdrant.py b/services/threatintel/app/storage/qdrant.pyindex d189abbb2..d84abd354 100644--- a/services/threatintel/app/storage/qdrant.py+++ b/services/threatintel/app/storage/qdrant.py@@ -6,7 +6,9 @@ AiSOC — open-source AI Security Operations Center (MIT License)from __future__ import annotations+import asyncioimport hashlib+import threadingfrom typing import Anyimport structlog@@ -56,6 +58,40 @@ def tenant_scope_filter(tenant_id: str | None, *, include_shared: bool = True) -return Filter(must=[FieldCondition(key="tenant_id", match=MatchAny(any=allowed))])+#: Loaded once, on first use, and shared. `TextEmbedding()` reads an ONNX+#: model off disk; the old code constructed one per `_embed` call, so every+#: batch, search and query paid that again. Guarded by a lock because+#: `_embed_sync` runs in a worker thread and several may arrive together.+_EMBEDDING_MODEL: Any = None+_EMBEDDING_MODEL_LOCK = threading.Lock()+++def _embedding_model() -> Any:+ global _EMBEDDING_MODEL+ if _EMBEDDING_MODEL is None:+ with _EMBEDDING_MODEL_LOCK:+ if _EMBEDDING_MODEL is None:+ from fastembed import TextEmbedding++ _EMBEDDING_MODEL = TextEmbedding(model_name="BAAI/bge-small-en-v1.5")+ return _EMBEDDING_MODEL+++def _embed_sync(texts: list[str]) -> list[list[float]]:+ """The blocking half, kept separate so it is obvious what runs in a thread."""+ embeddings = list(_embedding_model().embed(texts))+ result = []+ for emb in embeddings:+ vec = list(emb)+ # Pad/truncate to expected dimension+ if len(vec) < _EMBEDDING_DIM:+ vec = vec + [0.0] * (_EMBEDDING_DIM - len(vec))+ elif len(vec) > _EMBEDDING_DIM:+ vec = vec[:_EMBEDDING_DIM]+ result.append(vec)+ return result++class QdrantStore:"""Async Qdrant writer/searcher for threat intel embeddings.@@ -210,22 +246,31 @@ class QdrantStore:# ─── Private helpers ──────────────────────────────────────────────────────async def _embed(self, texts: list[str]) -> list[list[float]]:- """Generate embeddings using fastembed or a stub."""+ """Generate embeddings using fastembed or a stub.++ Off the event loop, and with the model loaded once.++ This was ``async def`` and awaited nothing: ``model.embed(texts)`` is+ synchronous ONNX inference, so it ran on the event loop and the+ service answered *nothing at all* for its duration — not a request,+ not a health probe. The weekly chaos ``qdrant`` leg measured the+ consequence twice in a row: threatintel's first CISA KEV poll embeds+ 1,756 indicators about 25 s after boot, and a readiness read landing+ in that window got no status line and no body, which the grader+ correctly scored as "threatintel stopped serving" while the store it+ was grading was not the cause.++ `asyncio.timeout` cannot save a caller from this, which is the part+ worth remembering: `/health` caps its own Qdrant probe at 1.5 s, but+ a blocked loop cannot run the timer either, so every in-process+ deadline in the service is suspended at once.++ The model is also built once rather than per call — `TextEmbedding()`+ loads an ONNX model from disk, which the old code paid on every+ batch, every search and every query.+ """try:- from fastembed import TextEmbedding-- model = TextEmbedding(model_name="BAAI/bge-small-en-v1.5")- embeddings = list(model.embed(texts))- # Pad/truncate to expected dimension- result = []- for emb in embeddings:- vec = list(emb)- if len(vec) < _EMBEDDING_DIM:- vec = vec + [0.0] * (_EMBEDDING_DIM - len(vec))- elif len(vec) > _EMBEDDING_DIM:- vec = vec[:_EMBEDDING_DIM]- result.append(vec)- return result+ return await asyncio.to_thread(_embed_sync, texts)except Exception:# Fallback: zero vectors (embeddings will be non-functional but won't crash)return [[0.0] * _EMBEDDING_DIM for _ in texts]diff --git a/services/threatintel/tests/test_embedding_does_not_block.py b/services/threatintel/tests/test_embedding_does_not_block.pynew file mode 100644index 000000000..dc8ef8740--- /dev/null+++ b/services/threatintel/tests/test_embedding_does_not_block.py@@ -0,0 +1,119 @@+"""The embedding path must not stop the service answering.++Found by the weekly chaos `qdrant` leg, which failed twice in a row on+"threatintel stopped serving" while the store it was grading was not the+cause. `QdrantStore._embed` was `async def` and awaited nothing:+`model.embed(texts)` is synchronous ONNX inference, so it ran on the event+loop and the process answered nothing at all for its duration -- including+`/health`. threatintel's first CISA KEV poll embeds 1,756 indicators about+25 s after boot, which is exactly where a readiness read was landing.++What is asserted here is the property, not the implementation: while an+embed is in flight, another coroutine must still get scheduled. A test that+only checked for `asyncio.to_thread` in the source would pass for a future+edit that reintroduces the block by another route.+"""++from __future__ import annotations++import asyncio+import time++import pytest+from app.storage import qdrant as qdrant_module+from app.storage.qdrant import QdrantStore+++class _SlowModel:+ """Stands in for `TextEmbedding` at the one thing that matters: it blocks.++ `time.sleep` rather than `asyncio.sleep` on purpose -- the defect is CPU+ work that never yields, and an awaitable double could not reproduce it.+ """++ def __init__(self, seconds: float = 0.5) -> None:+ self.seconds = seconds+ self.calls = 0++ def embed(self, texts: list[str]):+ self.calls += 1+ time.sleep(self.seconds)+ return [[0.1] * 384 for _ in texts]+++async def test_the_event_loop_keeps_running_while_an_embed_is_in_flight(monkeypatch) -> None:+ model = _SlowModel(seconds=0.5)+ monkeypatch.setattr(qdrant_module, "_EMBEDDING_MODEL", model)++ store = QdrantStore(client=None) # type: ignore[arg-type]++ # The deadline is absolute and fixed BEFORE the embed starts, which is+ # the whole load-bearing detail. A relative window computed inside the+ # heartbeat is useless here: a blocking embed simply runs first and the+ # heartbeat then gets its full window afterwards, so the test passes+ # against the defect. That version was written, run against the pre-fix+ # implementation, and did not catch it.+ deadline = time.monotonic() + 0.4++ async def _heartbeat() -> int:+ # Stands in for anything the service still owes an answer to while a+ # feed is ingesting: a health probe, a search, a push.+ beats = 0+ while time.monotonic() < deadline:+ await asyncio.sleep(0.02)+ beats += 1+ return beats++ heartbeat = asyncio.create_task(_heartbeat())+ vectors = await store._embed(["a", "b", "c"])+ heartbeats = await heartbeat++ assert model.calls == 1, "the embed never ran, so this proves nothing"+ assert len(vectors) == 3+ # Negative control: with the inference back on the event loop the+ # heartbeat cannot be scheduled until the embed returns, by which time+ # the deadline has passed, and this lands at 0.+ assert heartbeats > 5, f"the event loop was blocked during the embed ({heartbeats} heartbeats)"+++async def test_the_model_is_loaded_once_and_reused(monkeypatch) -> None:+ """`TextEmbedding()` reads an ONNX model off disk; the old code paid that per call."""+ loads = 0+ model = _SlowModel(seconds=0.0)++ def _fake_text_embedding(**_kwargs):+ nonlocal loads+ loads += 1+ return model++ monkeypatch.setattr(qdrant_module, "_EMBEDDING_MODEL", None)+ import fastembed # noqa: PLC0415 - patched per test, not at import time++ monkeypatch.setattr(fastembed, "TextEmbedding", _fake_text_embedding)++ store = QdrantStore(client=None) # type: ignore[arg-type]+ await store._embed(["a"])+ await store._embed(["b"])+ await store._embed(["c"])++ assert loads == 1, f"the embedding model was constructed {loads} times"+ assert model.calls == 3+++async def test_an_unavailable_model_still_returns_vectors_rather_than_raising(monkeypatch) -> None:+ """Unchanged behaviour: embeddings degrade to zeros, ingestion continues."""++ class _Broken:+ def embed(self, _texts):+ raise RuntimeError("onnxruntime is not installed")++ monkeypatch.setattr(qdrant_module, "_EMBEDDING_MODEL", _Broken())++ store = QdrantStore(client=None) # type: ignore[arg-type]+ vectors = await store._embed(["a", "b"])++ assert len(vectors) == 2+ assert all(all(component == 0.0 for component in vector) for vector in vectors)