Graph-Native Infrastructure for Context and Accountable AI Systems
Python agent-memoryaiai-governanceai-infrastructureartificial-intelligencecontext-engineeringcontext-graphsdata-engineeringdecision-intelligencedeveloper-toolsexplainable-aigenerative-aigraph-ragknowledge-graphllmontologyprovenancepythonreasoningsemantic-search
"""
Tests for the Decision Evaluation Hook
This module tests the opt-in evaluators/eval_config wiring between
semantica.evals and DecisionRecorder / AgentContext.
"""
import logging
from datetime import datetime
from unittest.mock import Mock
import pytest
from semantica.context.agent_context import AgentContext
from semantica.context.context_graph import ContextGraph
from semantica.context.decision_models import Decision
from semantica.context.decision_recorder import DecisionRecorder
from semantica.evals.types import CaseResult, EvalMetric, EvalSummary
def _decision(**overrides):
"""Build a Decision that passes every decision_scores check by default."""
base = dict(
decision_id="eval-hook-d1",
category="loan",
scenario="mortgage application",
reasoning="strong credit history",
outcome="approved",
confidence=0.95,
timestamp=datetime(2026, 1, 1),
decision_maker="loan_officer",
metadata={"provenance": {"prov_record": "rid-1"}},
)
base.update(overrides)
return Decision(**base)
class TestDecisionRecorderEvalHook:
"""Test the opt-in evaluation hook on DecisionRecorder.record_decision()."""
@pytest.fixture
def mock_graph_store(self):
"""Mock graph store for testing."""
mock_store = Mock()
mock_store.execute_query = Mock()
return mock_store
def test_no_evaluators_configured_is_unchanged(self, mock_graph_store):
"""Backward compat: omitting evaluators= adds no eval_* metadata."""
recorder = DecisionRecorder(graph_store=mock_graph_store)
decision = _decision()
decision_id = recorder.record_decision(decision, [], [])
assert decision_id == decision.decision_id
assert "eval_score" not in decision.metadata
assert "eval_passed" not in decision.metadata
assert "eval_details" not in decision.metadata
def test_full_pass_enriches_metadata(self, mock_graph_store):
"""All decision_scores checks pass -> eval_score 1.0, eval_passed True."""
recorder = DecisionRecorder(
graph_store=mock_graph_store,
evaluators=["decision_scores"],
eval_config={"decision_scores": {"expected_outcome": "approved"}},
)
decision = _decision()
recorder.record_decision(decision, [], [])
assert decision.metadata["eval_score"] == 1.0
assert decision.metadata["eval_passed"] is True
assert list(decision.metadata["eval_details"].keys()) == ["decision_scores"]
def test_failing_check_surfaces_in_metadata(self, mock_graph_store):
"""Empty reasoning fails a field check -> eval_passed False, named in details."""
recorder = DecisionRecorder(
graph_store=mock_graph_store,
evaluators=["decision_scores"],
)
decision = _decision(reasoning="")
recorder.record_decision(decision, [], [])
assert decision.metadata["eval_passed"] is False
assert decision.metadata["eval_score"] < 1.0
assert (
decision.metadata["eval_details"]["decision_scores"]["meta"]["reasoning"]
is False
)
def test_none_metadata_does_not_crash_recording(self, mock_graph_store):
"""Decision(metadata=None) -> hook normalizes it instead of crashing."""
recorder = DecisionRecorder(
graph_store=mock_graph_store,
evaluators=["decision_scores"],
)
decision = _decision(metadata=None)
decision_id = recorder.record_decision(decision, [], [])
assert decision_id == decision.decision_id
assert decision.metadata["eval_passed"] is False # provenance check fails
assert mock_graph_store.execute_query.called
def test_policy_mismatch_surfaces_in_metadata(self, mock_graph_store):
"""PolicyEngine.check_compliance() False -> failing 'policy' check surfaced."""
class FakePolicyEngine:
def check_compliance(self, decision, policy_id):
return False
recorder = DecisionRecorder(
graph_store=mock_graph_store,
evaluators=["decision_scores"],
eval_config={
"decision_scores": {
"policy_engine": FakePolicyEngine(),
"policy_id": "p1",
}
},
)
decision = _decision()
recorder.record_decision(decision, [], [])
assert decision.metadata["eval_passed"] is False
assert (
decision.metadata["eval_details"]["decision_scores"]["meta"]["policy"]
is False
)
def test_evaluator_failure_does_not_block_recording(
self, mock_graph_store, monkeypatch, caplog
):
"""evaluate() itself raising -> decision still recorded, no eval_* keys."""
monkeypatch.setattr(
"semantica.evals.evaluate",
Mock(side_effect=RuntimeError("boom")),
)
recorder = DecisionRecorder(
graph_store=mock_graph_store,
evaluators=["decision_scores"],
)
decision = _decision()
with caplog.at_level(logging.WARNING):
decision_id = recorder.record_decision(decision, [], [])
assert decision_id == decision.decision_id
assert "eval_score" not in decision.metadata
assert "eval_passed" not in decision.metadata
assert mock_graph_store.execute_query.called
# The traceback must survive the log call, not just the message string.
record = next(r for r in caplog.records if "Decision evaluation failed" in r.message)
assert record.exc_info is not None
def test_evaluator_error_status_is_logged(
self, mock_graph_store, monkeypatch, caplog
):
"""A named evaluator erroring (unknown name, internal raise) -> logged,
not just buried in eval_details on the stored decision."""
error_summary = EvalSummary(
total=1,
passed=0,
failed=0,
errors=1,
pass_rate=0.0,
cases=[
CaseResult(
case_id="eval-hook-d1",
status="error",
metrics={"decision_scores": EvalMetric(0.0, False, {"error": "boom"})},
details={"decision_scores": {"error": "boom"}},
)
],
)
monkeypatch.setattr(
"semantica.evals.evaluate", Mock(return_value=error_summary)
)
recorder = DecisionRecorder(
graph_store=mock_graph_store,
evaluators=["decision_scores"],
)
decision = _decision()
with caplog.at_level(logging.WARNING):
recorder.record_decision(decision, [], [])
assert decision.metadata["eval_passed"] is False
assert any(
"evaluator(s) errored" in r.message for r in caplog.records
)
class TestAgentContextEvalHookWiring:
"""Test that AgentContext threads evaluators/eval_config into DecisionRecorder."""
@pytest.fixture(autouse=True)
def _stub_node_embedder(self, monkeypatch):
"""AgentContext(vector_store=..., knowledge_graph=...) constructs a real
ContextRetriever, which eagerly builds a DecisionEmbeddingPipeline and a
DecisionQuery, both of which construct NodeEmbedder(method="node2vec").
NodeEmbedder raises ImportError when the optional `gensim` dependency
(semantica[graph-embeddings]) isn't installed. These tests only check
that evaluators/eval_config land on the DecisionRecorder, so stub
NodeEmbedder out at both import sites rather than requiring gensim.
"""
monkeypatch.setattr(
"semantica.vector_store.decision_embedding_pipeline.NodeEmbedder", Mock()
)
monkeypatch.setattr("semantica.context.decision_query.NodeEmbedder", Mock())
@pytest.fixture
def mock_vector_store(self):
"""Mock vector store for testing."""
mock_store = Mock()
mock_store.add = Mock()
mock_store.search = Mock(return_value=[])
return mock_store
@pytest.fixture
def mock_knowledge_graph(self):
"""Mock knowledge graph for testing."""
mock_graph = Mock()
mock_graph.execute_query = Mock(return_value=[])
return mock_graph
def test_evaluators_threaded_into_decision_recorder(
self, mock_vector_store, mock_knowledge_graph
):
"""Constructor-level evaluators/eval_config land on the recorder."""
context = AgentContext(
vector_store=mock_vector_store,
knowledge_graph=mock_knowledge_graph,
decision_tracking=True,
evaluators=["decision_scores"],
eval_config={"decision_scores": {"expected_outcome": "approved"}},
)
assert context._decision_recorder.evaluators == ["decision_scores"]
assert context._decision_recorder.eval_config == {
"decision_scores": {"expected_outcome": "approved"}
}
def test_evaluators_threaded_on_exception_fallback_path(
self, mock_vector_store, mock_knowledge_graph, monkeypatch
):
"""Same wiring survives the DecisionQuery/-Analyzer init failure fallback."""
# The success path constructs DecisionQuery once; on failure the
# except block constructs it again as part of the fallback. Only
# the first call should fail, so the fallback branch itself succeeds.
monkeypatch.setattr(
"semantica.context.agent_context.DecisionQuery",
Mock(side_effect=[RuntimeError("boom"), Mock()]),
)
context = AgentContext(
vector_store=mock_vector_store,
knowledge_graph=mock_knowledge_graph,
decision_tracking=True,
evaluators=["decision_scores"],
eval_config={"x": 1},
)
assert context._decision_recorder.evaluators == ["decision_scores"]
assert context._decision_recorder.eval_config == {"x": 1}
def test_no_evaluators_default_is_none(
self, mock_vector_store, mock_knowledge_graph
):
"""Backward compat at the AgentContext layer: default is untouched."""
context = AgentContext(
vector_store=mock_vector_store,
knowledge_graph=mock_knowledge_graph,
decision_tracking=True,
)
assert context._decision_recorder.evaluators is None
assert context._decision_recorder.eval_config == {}
class TestAgentContextContextGraphEvalHook:
"""Evaluators must run on the ContextGraph backend too.
``AgentContext`` documents ``evaluators``/``eval_config`` as applying to
the graph_store decision backend in 0.8.0, leaving the ContextGraph
backend as a known follow-up. The constructor accepts the options either
way, so the ContextGraph path must not silently ignore them.
"""
@pytest.fixture(autouse=True)
def _stub_node_embedder(self, monkeypatch):
monkeypatch.setattr(
"semantica.vector_store.decision_embedding_pipeline.NodeEmbedder", Mock()
)
monkeypatch.setattr("semantica.context.decision_query.NodeEmbedder", Mock())
@pytest.fixture
def vector_store(self):
store = Mock()
store.add = Mock()
store.search = Mock(return_value=[])
return store
def _record(self, vector_store, cross_system_context=None, **context_kwargs):
graph = ContextGraph()
context = AgentContext(
vector_store=vector_store,
knowledge_graph=graph,
decision_tracking=True,
**context_kwargs,
)
decision_id = context.record_decision(
category="loan",
scenario="mortgage application",
reasoning="strong credit history",
outcome="approved",
confidence=0.95,
decision_maker="loan_officer",
cross_system_context=cross_system_context,
)
return graph, decision_id
def test_evaluators_run_for_context_graph_backend(self, vector_store):
graph, decision_id = self._record(
vector_store,
evaluators=["decision_scores"],
eval_config={"decision_scores": {"expected_outcome": "approved"}},
)
props = graph.nodes[decision_id].properties
assert props["eval_score"] == pytest.approx(5 / 6)
assert props["eval_passed"] is False # no provenance in this path
assert props["eval_details"]["decision_scores"]["meta"]["provenance"] is False
def test_no_evaluators_leaves_context_graph_decision_unchanged(self, vector_store):
graph, decision_id = self._record(vector_store)
props = graph.nodes[decision_id].properties
assert "eval_score" not in props
assert "eval_passed" not in props
assert "eval_details" not in props
def test_cross_system_context_and_eval_metadata_are_merged(self, vector_store):
graph, decision_id = self._record(
vector_store,
cross_system_context={"ticket": "T-1"},
evaluators=["decision_scores"],
)
props = graph.nodes[decision_id].properties
assert props["cross_system_context"] == {"ticket": "T-1"}
assert props["eval_passed"] is False
assert "eval_details" in props
def test_evaluator_failure_does_not_block_context_graph_recording(
self, vector_store, monkeypatch, caplog
):
monkeypatch.setattr(
"semantica.evals.evaluate",
Mock(side_effect=RuntimeError("boom")),
)
with caplog.at_level(logging.WARNING):
graph, decision_id = self._record(
vector_store, evaluators=["decision_scores"]
)
assert decision_id in graph.nodes
props = graph.nodes[decision_id].properties
assert "eval_score" not in props
assert "eval_passed" not in props
assert any(
"Decision evaluation failed" in record.message for record in caplog.records
)
def test_evaluator_sees_the_persisted_decision_id(self, vector_store, monkeypatch):
seen = {}
def fake_evaluate(cases, evaluators, config):
seen["decision_id"] = cases[0]["actual"].decision_id
return EvalSummary(
total=1,
passed=1,
failed=0,
errors=0,
pass_rate=1.0,
cases=[
CaseResult(
case_id=seen["decision_id"],
status="pass",
metrics={"decision_scores": EvalMetric(1.0, True, {})},
details={},
)
],
)
monkeypatch.setattr("semantica.evals.evaluate", fake_evaluate)
graph, decision_id = self._record(vector_store, evaluators=["decision_scores"])
assert seen["decision_id"] == decision_id
assert decision_id in graph.nodes
def test_long_eval_details_are_omitted_without_blocking_recording(
self, vector_store, caplog
):
# ContextGraph caps metadata values at 1,000 characters; the evaluator
# echoes a valid but long outcome into its mismatch reason.
long_outcome = "approved" + "x" * 990
graph = ContextGraph()
context = AgentContext(
vector_store=vector_store,
knowledge_graph=graph,
decision_tracking=True,
evaluators=["decision_scores"],
eval_config={"decision_scores": {"expected_outcome": "approved"}},
)
with caplog.at_level(logging.WARNING):
decision_id = context.record_decision(
category="loan",
scenario="mortgage application",
reasoning="strong credit history",
outcome=long_outcome,
confidence=0.95,
decision_maker="loan_officer",
)
assert decision_id in graph.nodes
props = graph.nodes[decision_id].properties
assert props["eval_score"] < 1.0
assert props["eval_passed"] is False
assert "eval_details" not in props
assert any(
"metadata value limit" in record.message for record in caplog.records
)