Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
"""
Memory Recall Eval — Three-Tier Fidelity Gate
==============================================
Cross-cutting eval for the AiSOC capability roadmap (2026 H2): the agent's
three-tier memory (session → working → institutional) must store values
faithfully, return them on recall, respect tier priority ordering, and
maintain cross-tenant isolation.
The suite is intentionally deterministic — no LLM calls, no external
services. Session uses in-process LRU, working falls back to its
in-process dict (no Redis), and institutional falls back to its in-process
dict (no PostgreSQL). This keeps CI fast and dependency-free.
Gates (all must pass):
1. Write-then-recall round-trip per tier
2. Recall priority: session > working > institutional
3. Cross-tenant isolation: tenant A cannot read tenant B data
4. Session eviction after clear()
5. Structured-value fidelity (dict/list round-trip)
6. Override ingestion end-to-end
7. institutional search by tag
Metrics:
* recall_accuracy — fraction of test cases where recalled value matches
the written value exactly. Floor: 1.0 (all cases must pass).
* isolation_score — fraction of cross-tenant probes that return None.
Floor: 1.0 (perfect isolation required).
"""
from __future__ import annotations
import asyncio
import os
import sys
from pathlib import Path
from typing import Any
import pytest
_AGENTS_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(_AGENTS_ROOT))
# `DATABASE_URL` is handled by the `_no_ambient_database` fixture in
# tests/conftest.py. The `setdefault` that used to stand here was a no-op in
# the only case that mattered — a developer who already had one exported.
os.environ.setdefault("REDIS_URL", "")
from app.memory import MemoryManager # noqa: E402
from app.memory.institutional import _FALLBACK as _INSTITUTIONAL_FALLBACK # noqa: E402
from app.memory.models import OverrideFeedback # noqa: E402
from app.memory.session import _session_caches # noqa: E402
from app.memory.working import _FALLBACK as _WORKING_FALLBACK # noqa: E402
RECALL_ACCURACY_FLOOR = 1.0
ISOLATION_FLOOR = 1.0
def _clear_all_fallbacks() -> None:
_session_caches.clear()
_WORKING_FALLBACK.clear()
_INSTITUTIONAL_FALLBACK.clear()
@pytest.fixture(autouse=True)
def _clean_state():
_clear_all_fallbacks()
yield
_clear_all_fallbacks()
# ------------------------------------------------------------------
# 1. Write-then-recall per tier
# ------------------------------------------------------------------
@pytest.mark.asyncio
async def test_session_round_trip():
mgr = await MemoryManager.create(tenant_id="t-recall", run_id="run-1")
await mgr.write_session("key_a", "value_a")
assert await mgr.recall("key_a", tiers=("session",)) == "value_a"
@pytest.mark.asyncio
async def test_working_round_trip():
mgr = await MemoryManager.create(tenant_id="t-recall", run_id="run-1")
await mgr.write_working("key_b", {"nested": True})
result = await mgr.recall("key_b", tiers=("working",))
assert result == {"nested": True}
@pytest.mark.asyncio
async def test_institutional_round_trip():
mgr = await MemoryManager.create(tenant_id="t-recall", run_id="run-1")
await mgr.write_institutional("key_c", [1, 2, 3], tags=["test"])
result = await mgr.recall("key_c", tiers=("institutional",))
assert result == [1, 2, 3]
# ------------------------------------------------------------------
# 2. Recall priority: session > working > institutional
# ------------------------------------------------------------------
@pytest.mark.asyncio
async def test_recall_priority_session_wins():
mgr = await MemoryManager.create(tenant_id="t-prio", run_id="run-p")
await mgr.write_session("shared_key", "from_session")
await mgr.write_working("shared_key", "from_working")
await mgr.write_institutional("shared_key", "from_institutional")
result = await mgr.recall("shared_key")
assert result == "from_session"
@pytest.mark.asyncio
async def test_recall_priority_working_wins_without_session():
mgr = await MemoryManager.create(tenant_id="t-prio2", run_id="run-p2")
await mgr.write_working("shared_key", "from_working")
await mgr.write_institutional("shared_key", "from_institutional")
result = await mgr.recall("shared_key")
assert result == "from_working"
@pytest.mark.asyncio
async def test_recall_priority_institutional_fallback():
mgr = await MemoryManager.create(tenant_id="t-prio3", run_id="run-p3")
await mgr.write_institutional("only_inst", "from_institutional")
result = await mgr.recall("only_inst")
assert result == "from_institutional"
# ------------------------------------------------------------------
# 3. Cross-tenant isolation
# ------------------------------------------------------------------
@pytest.mark.asyncio
async def test_cross_tenant_isolation_working():
mgr_a = await MemoryManager.create(tenant_id="tenant-A", run_id="r1")
mgr_b = await MemoryManager.create(tenant_id="tenant-B", run_id="r2")
await mgr_a.write_working("secret", "tenant-A-data")
result = await mgr_b.recall("secret", tiers=("working",))
assert result is None
@pytest.mark.asyncio
async def test_cross_tenant_isolation_institutional():
mgr_a = await MemoryManager.create(tenant_id="tenant-C", run_id="r3")
mgr_b = await MemoryManager.create(tenant_id="tenant-D", run_id="r4")
await mgr_a.write_institutional("classified", "tenant-C-only")
result = await mgr_b.recall("classified", tiers=("institutional",))
assert result is None
# ------------------------------------------------------------------
# 4. Session eviction after clear()
# ------------------------------------------------------------------
@pytest.mark.asyncio
async def test_session_clear_evicts():
mgr = await MemoryManager.create(tenant_id="t-clear", run_id="run-clear")
await mgr.write_session("ephemeral", "gone-soon")
assert await mgr.recall("ephemeral", tiers=("session",)) == "gone-soon"
await mgr.clear_session()
assert await mgr.recall("ephemeral", tiers=("session",)) is None
# ------------------------------------------------------------------
# 5. Structured-value fidelity
# ------------------------------------------------------------------
@pytest.mark.asyncio
async def test_structured_value_fidelity():
mgr = await MemoryManager.create(tenant_id="t-struct", run_id="run-struct")
complex_val = {
"alert_id": "ALT-001",
"iocs": ["10.0.0.1", "evil.com"],
"mitre": {"tactic": "TA0001", "technique": "T1566.001"},
"score": 0.87,
}
await mgr.write_working("case_context", complex_val)
recalled = await mgr.recall("case_context", tiers=("working",))
assert recalled == complex_val
# ------------------------------------------------------------------
# 6. Override ingestion end-to-end
# ------------------------------------------------------------------
@pytest.mark.asyncio
async def test_override_ingestion():
mgr = await MemoryManager.create(tenant_id="t-override", run_id="run-ov")
feedback = OverrideFeedback(
tenant_id="t-override",
run_id="run-ov",
alert_id="ALT-999",
original_verdict="benign",
corrected_verdict="true_positive",
analyst_id="analyst-42",
reason="Confirmed C2 callback after manual PCAP review",
)
await mgr.ingest_override(feedback)
result = await mgr.recall(f"analyst_override:{feedback.alert_id}", tiers=("institutional",))
assert result is not None
assert result["corrected_verdict"] == "true_positive"
assert result["analyst_id"] == "analyst-42"
# ------------------------------------------------------------------
# 7. Institutional search by tag
# ------------------------------------------------------------------
@pytest.mark.asyncio
async def test_institutional_search_by_tag():
mgr = await MemoryManager.create(tenant_id="t-search", run_id="run-s")
await mgr.write_institutional("fp-1", {"rule": "sigma-001"}, tags=["known_fp"])
await mgr.write_institutional("fp-2", {"rule": "sigma-002"}, tags=["known_fp"])
await mgr.write_institutional("other", {"rule": "sigma-003"}, tags=["playbook"])
results = await mgr.search_institutional(tags=["known_fp"])
keys = [r["key"] for r in results]
assert "fp-1" in keys
assert "fp-2" in keys
# ------------------------------------------------------------------
# 8. Missing key returns None
# ------------------------------------------------------------------
@pytest.mark.asyncio
async def test_missing_key_returns_none():
mgr = await MemoryManager.create(tenant_id="t-miss", run_id="run-miss")
assert await mgr.recall("nonexistent") is None
# ------------------------------------------------------------------
# 9. Delete removes from tier
# ------------------------------------------------------------------
@pytest.mark.asyncio
async def test_delete_removes_from_working():
mgr = await MemoryManager.create(tenant_id="t-del", run_id="run-del")
await mgr.write_working("to_delete", "temp")
assert await mgr.recall("to_delete", tiers=("working",)) == "temp"
await mgr.delete("to_delete", "working")
assert await mgr.recall("to_delete", tiers=("working",)) is None
# ------------------------------------------------------------------
# Aggregate runner for the eval harness
# ------------------------------------------------------------------
def run_evaluation() -> dict[str, Any]:
"""Execute all memory-recall checks and return a JSON-serializable report.
Called by ``scripts/run_evals.py`` for the unified evaluation runner.
"""
_clear_all_fallbacks()
cases: list[dict] = []
async def _run_case(name: str, coro):
try:
await coro
cases.append({"name": name, "passed": True})
except AssertionError as exc:
cases.append({"name": name, "passed": False, "error": str(exc)})
async def _run_all():
await _run_case("session_round_trip", test_session_round_trip())
await _run_case("working_round_trip", test_working_round_trip())
await _run_case("institutional_round_trip", test_institutional_round_trip())
await _run_case("recall_priority_session_wins", test_recall_priority_session_wins())
await _run_case(
"recall_priority_working_wins",
test_recall_priority_working_wins_without_session(),
)
await _run_case(
"recall_priority_institutional_fallback",
test_recall_priority_institutional_fallback(),
)
await _run_case("cross_tenant_working", test_cross_tenant_isolation_working())
await _run_case(
"cross_tenant_institutional",
test_cross_tenant_isolation_institutional(),
)
await _run_case("session_clear", test_session_clear_evicts())
await _run_case("structured_fidelity", test_structured_value_fidelity())
await _run_case("override_ingestion", test_override_ingestion())
await _run_case("search_by_tag", test_institutional_search_by_tag())
await _run_case("missing_key_none", test_missing_key_returns_none())
await _run_case("delete_from_working", test_delete_removes_from_working())
asyncio.run(_run_all())
passed = sum(1 for c in cases if c["passed"])
total = len(cases)
accuracy = passed / total if total else 0.0
isolation_cases = [c for c in cases if "cross_tenant" in c["name"]]
isolation_passed = sum(1 for c in isolation_cases if c["passed"])
isolation_total = len(isolation_cases)
isolation_score = isolation_passed / isolation_total if isolation_total else 1.0
_clear_all_fallbacks()
return {
"passed": accuracy >= RECALL_ACCURACY_FLOOR and isolation_score >= ISOLATION_FLOOR,
"recall_accuracy": round(accuracy, 4),
"recall_accuracy_floor": RECALL_ACCURACY_FLOOR,
"isolation_score": round(isolation_score, 4),
"isolation_floor": ISOLATION_FLOOR,
"total_cases": total,
"passed_cases": passed,
"failed_cases": [c for c in cases if not c["passed"]],
}