Skip to content

AI-powered virtual executive team — a single coherent executive persona backed by 8 specialist agents (FastAPI + Next.js).

16.1 KB
Raw Blame
"""Tests for the short-message bypass and distance-threshold gating in retrieve()."""
from __future__ import annotations

from types import SimpleNamespace
from typing import Any
from unittest.mock import MagicMock

import pytest

from openexecutive.knowledge.retriever import (
    _DISTANCE_THRESHOLD,
    retrieve,
)


@pytest.fixture(autouse=True)
def _isolate_audit(monkeypatch: pytest.MonkeyPatch) -> None:
    """Keep `_emit_retrieval_audit` out of the default ./episodic_memory.db.

    retrieve() and retrieve_failures() both emit an audit row. Unpatched they
    write to the module default, and the leaked rows only break *other*
    modules' assertions in a full-suite run.
    """
    import openexecutive.knowledge.retriever as retriever_mod

    monkeypatch.setattr(retriever_mod, "_audit_log", lambda *a, **k: None)


@pytest.fixture
def fake_review_store() -> SimpleNamespace:
    """ReviewStore stub that returns empty sets — keep retrieval focused on the gate."""
    return SimpleNamespace(
        get_withheld_keys=lambda _ct: set(),
        get_withheld_source_ids=lambda: set(),
        get_priority_map=lambda _ct: {},
        list_annotations=lambda domains=None, active_only=True: [],
    )


def _make_store(builtin_hits: list[dict[str, Any]], company_hits: list[dict[str, Any]]) -> MagicMock:
    """ChromaDBStore stub. query() dispatches on collection name."""
    store = MagicMock()
    from openexecutive.knowledge.store import ChromaDBStore

    def fake_query(*, query_text: str, collection: str, domain_filter: Any, n_results: int) -> list[dict[str, Any]]:
        if collection == ChromaDBStore.BUILTIN_COLLECTION:
            return builtin_hits
        if collection == ChromaDBStore.COMPANY_COLLECTION:
            return company_hits
        return []

    store.query.side_effect = fake_query
    return store


@pytest.mark.parametrize("greeting", ["Hi", "ok", "  Hi  ", "?", ""])
def test_short_message_bypasses_retrieval(
    fake_review_store: SimpleNamespace, greeting: str
) -> None:
    """Very short greetings must not trigger a vector-store query."""
    store = _make_store(builtin_hits=[{"text": "x", "metadata": {}, "distance": 0.1}], company_hits=[])

    out = retrieve(query=greeting, store=store, review_store=fake_review_store)

    assert out == ""
    store.query.assert_not_called()


@pytest.mark.parametrize(
    "query",
    [
        "What now?",  # English follow-up — 9 chars
        "How come?",  # English follow-up — 9 chars
        "我们的营销策略",  # Chinese — 7 chars, would be 1 token under \w+
        "How should I price the assessment?",  # full question
        "ROI",  # 3-letter business acronym — must fire
        "CFO",
        "P&L",
    ],
)
def test_meaningful_query_fires_retrieval(
    fake_review_store: SimpleNamespace, query: str
) -> None:
    """Real questions — including non-whitespace-tokenized scripts and 3-letter
    business acronyms (ROI, CFO, P&L) — must fire RAG."""
    store = _make_store(
        builtin_hits=[{"text": "useful", "metadata": {"filename": "f.md"}, "distance": 0.2}],
        company_hits=[],
    )
    out = retrieve(query=query, store=store, review_store=fake_review_store)
    assert "useful" in out
    store.query.assert_called()


def test_distance_threshold_drops_weak_builtin_hits(fake_review_store: SimpleNamespace) -> None:
    """Only hits with distance <= _DISTANCE_THRESHOLD survive in builtin."""
    threshold = _DISTANCE_THRESHOLD
    store = _make_store(
        builtin_hits=[
            {"text": "strong match", "metadata": {"filename": "good.md"}, "distance": threshold - 0.1},
            {"text": "weak match", "metadata": {"filename": "bad.md"}, "distance": threshold + 0.1},
        ],
        company_hits=[],
    )
    out = retrieve(query="a real question about strategy", store=store, review_store=fake_review_store)
    assert "strong match" in out
    assert "weak match" not in out


def test_distance_threshold_drops_weak_company_hits(fake_review_store: SimpleNamespace) -> None:
    """Same threshold applies to the company-docs collection."""
    threshold = _DISTANCE_THRESHOLD
    store = _make_store(
        builtin_hits=[],
        company_hits=[
            {"text": "strong company doc", "metadata": {"filename": "good.md"}, "distance": threshold - 0.1},
            {"text": "weak company doc", "metadata": {"filename": "bad.md"}, "distance": threshold + 0.1},
        ],
    )
    out = retrieve(query="a real question about strategy", store=store, review_store=fake_review_store)
    assert "strong company doc" in out
    assert "weak company doc" not in out


def test_all_weak_hits_yields_empty_string(fake_review_store: SimpleNamespace) -> None:
    """When everything is below threshold, return empty rather than poison the prompt."""
    threshold = _DISTANCE_THRESHOLD
    store = _make_store(
        builtin_hits=[{"text": "noise", "metadata": {"filename": "n.md"}, "distance": threshold + 0.2}],
        company_hits=[{"text": "more noise", "metadata": {"filename": "m.md"}, "distance": threshold + 0.2}],
    )
    out = retrieve(query="a real question about strategy", store=store, review_store=fake_review_store)
    assert out == ""


def test_missing_or_none_distance_is_treated_as_far(
    fake_review_store: SimpleNamespace,
) -> None:
    """`distance` of None or missing must NOT crash the filter and must be
    treated as out-of-bounds so we never silently surface chunks of unknown
    relevance."""
    store = _make_store(
        builtin_hits=[
            {"text": "no distance", "metadata": {"filename": "x.md"}, "distance": None},
            {"text": "absent key", "metadata": {"filename": "y.md"}},
        ],
        company_hits=[],
    )
    out = retrieve(query="a real question about strategy", store=store, review_store=fake_review_store)
    assert out == ""


def test_perfect_match_distance_zero_is_kept(
    fake_review_store: SimpleNamespace,
) -> None:
    """A verbatim hit (distance=0.0) is the STRONGEST possible match. Naive
    `r.get('distance') or 1.0` would falsy-coerce 0.0 to 1.0 and silently
    drop it — this test guards against that regression."""
    store = _make_store(
        builtin_hits=[
            {"text": "perfect", "metadata": {"filename": "f.md"}, "distance": 0.0},
        ],
        company_hits=[
            {"text": "also perfect", "metadata": {"filename": "g.md"}, "distance": 0.0},
        ],
    )
    out = retrieve(query="a real question about strategy", store=store, review_store=fake_review_store)
    assert "perfect" in out
    assert "also perfect" in out


# ---------------------------------------------------------------------------
# The pending gate: withheld items never reach a specialist
# ---------------------------------------------------------------------------


def _hit(filename: str, text: str, *, distance: float = 0.1) -> dict[str, Any]:
    return {
        "text": text,
        "distance": distance,
        "metadata": {"filename": filename, "domain": "finance", "type": "builtin"},
    }


def _review_store(withheld: set[tuple[str, str]]) -> SimpleNamespace:
    return SimpleNamespace(
        get_withheld_keys=lambda _ct: withheld,
        get_withheld_source_ids=lambda: set(),
        get_priority_map=lambda _ct: {},
        list_annotations=lambda domains=None, active_only=True: [],
    )


def test_withheld_builtin_chunk_is_not_retrieved() -> None:
    """A doc queued for curation must not reach the Executive."""
    store = _make_store(
        [_hit("queued.md", "WITHHELD CONTENT"), _hit("trusted.md", "TRUSTED CONTENT")],
        [],
    )
    out = retrieve(
        query="a real question about capital structure",
        store=store,
        review_store=_review_store({("finance", "queued.md")}),
    )
    assert "TRUSTED CONTENT" in out
    assert "WITHHELD CONTENT" not in out


def test_nothing_withheld_returns_everything() -> None:
    store = _make_store([_hit("a.md", "ALPHA"), _hit("b.md", "BETA")], [])
    out = retrieve(
        query="a real question about capital structure",
        store=store,
        review_store=_review_store(set()),
    )
    assert "ALPHA" in out
    assert "BETA" in out


def test_withheld_external_source_is_not_retrieved() -> None:
    hit = _hit("oer.md", "EXTERNAL CONTENT")
    hit["metadata"]["source_id"] = "openstax-finance"
    store = _make_store([hit, _hit("trusted.md", "TRUSTED CONTENT")], [])
    rs = _review_store(set())
    rs.get_withheld_source_ids = lambda: {"openstax-finance"}

    out = retrieve(query="a real question about capital structure", store=store, review_store=rs)

    assert "TRUSTED CONTENT" in out
    assert "EXTERNAL CONTENT" not in out


def test_retrieve_failures_honors_withheld_items() -> None:
    """Failure case studies live in their own collection and used to ignore
    review state entirely — rejecting one did nothing."""
    from openexecutive.knowledge.retriever import retrieve_failures
    from openexecutive.knowledge.store import ChromaDBStore

    store = MagicMock()

    def fake_query(*, query_text: str, collection: str, domain_filter: Any, n_results: int):
        if collection == ChromaDBStore.FAILURES_COLLECTION:
            return [
                _hit("bad_case.md", "REJECTED FAILURE STORY"),
                _hit("good_case.md", "USEFUL FAILURE STORY"),
            ]
        return []

    store.query.side_effect = fake_query

    out = retrieve_failures(
        query="what went wrong with this rollout",
        store=store,
        review_store=_review_store({("finance", "bad_case.md")}),
    )

    assert "USEFUL FAILURE STORY" in out
    assert "REJECTED FAILURE STORY" not in out


def test_builtin_threshold_setting_tightens_builtin_only(
    fake_review_store: SimpleNamespace, monkeypatch: pytest.MonkeyPatch
) -> None:
    """KNOWLEDGE_BUILTIN_DISTANCE_THRESHOLD gates builtin without touching company.

    Both collections return a hit at 0.62. With the shared gate at 0.65 and the
    builtin-only gate at 0.60, the company doc survives and the builtin chunk
    does not -- the whole point of the split.
    """
    import openexecutive.config as config_mod

    monkeypatch.setattr(
        config_mod,
        "get_settings",
        lambda: SimpleNamespace(
            knowledge_builtin_n_results=5,
            knowledge_company_n_results=3,
            knowledge_distance_threshold=0.65,
            knowledge_builtin_distance_threshold=0.60,
            vector_store_path="/unused",
        ),
    )
    store = _make_store(
        builtin_hits=[{"text": "generic handbook", "metadata": {"filename": "b.md"}, "distance": 0.62}],
        company_hits=[{"text": "our own doc", "metadata": {"filename": "c.md"}, "distance": 0.62}],
    )
    out = retrieve(query="a real question about strategy", store=store, review_store=fake_review_store)
    assert "our own doc" in out
    assert "generic handbook" not in out


def test_builtin_threshold_unset_falls_back_to_shared(
    fake_review_store: SimpleNamespace, monkeypatch: pytest.MonkeyPatch
) -> None:
    """Unset (None) preserves the historical single-threshold behaviour."""
    import openexecutive.config as config_mod

    monkeypatch.setattr(
        config_mod,
        "get_settings",
        lambda: SimpleNamespace(
            knowledge_builtin_n_results=5,
            knowledge_company_n_results=3,
            knowledge_distance_threshold=0.65,
            knowledge_builtin_distance_threshold=None,
            vector_store_path="/unused",
        ),
    )
    store = _make_store(
        builtin_hits=[{"text": "generic handbook", "metadata": {"filename": "b.md"}, "distance": 0.62}],
        company_hits=[],
    )
    out = retrieve(query="a real question about strategy", store=store, review_store=fake_review_store)
    assert "generic handbook" in out


def test_builtin_setting_survives_a_caller_pinned_shared_threshold(
    fake_review_store: SimpleNamespace, monkeypatch: pytest.MonkeyPatch
) -> None:
    """A pinned shared threshold must not revoke the operator's builtin gate.

    A caller loosening the shared gate for an unrelated reason (wider company
    recall for one report) would otherwise silently re-open builtin to the
    generic-chunk flood the setting exists to prevent.
    """
    import openexecutive.config as config_mod

    monkeypatch.setattr(
        config_mod,
        "get_settings",
        lambda: SimpleNamespace(
            knowledge_builtin_n_results=5,
            knowledge_company_n_results=3,
            knowledge_distance_threshold=0.55,
            knowledge_builtin_distance_threshold=0.60,
            vector_store_path="/unused",
        ),
    )
    store = _make_store(
        builtin_hits=[{"text": "generic handbook", "metadata": {"filename": "b.md"}, "distance": 0.70}],
        company_hits=[{"text": "our own doc", "metadata": {"filename": "c.md"}, "distance": 0.70}],
    )
    out = retrieve(
        query="a real question about strategy",
        store=store,
        review_store=fake_review_store,
        distance_threshold=0.80,
    )
    # The pinned 0.80 reaches company...
    assert "our own doc" in out
    # ...but builtin stays on its own 0.60 gate.
    assert "generic handbook" not in out


def test_builtin_gate_can_only_tighten_never_loosen(
    fake_review_store: SimpleNamespace, monkeypatch: pytest.MonkeyPatch
) -> None:
    """A builtin value looser than the shared gate is clamped to the shared gate.

    Guards the transposed-env-var mistake: pasting the company number into the
    builtin variable would otherwise make builtin LOOSER than company, admitting
    handbook prose at a distance where a company doc is still dropped.
    """
    import openexecutive.config as config_mod

    monkeypatch.setattr(
        config_mod,
        "get_settings",
        lambda: SimpleNamespace(
            knowledge_builtin_n_results=5,
            knowledge_company_n_results=3,
            knowledge_distance_threshold=0.55,
            knowledge_builtin_distance_threshold=0.65,
            vector_store_path="/unused",
        ),
    )
    store = _make_store(
        builtin_hits=[{"text": "generic handbook", "metadata": {"filename": "b.md"}, "distance": 0.62}],
        company_hits=[{"text": "our own doc", "metadata": {"filename": "c.md"}, "distance": 0.62}],
    )
    out = retrieve(query="a real question about strategy", store=store, review_store=fake_review_store)
    assert "generic handbook" not in out
    assert "our own doc" not in out


def test_builtin_gate_is_inclusive_at_the_boundary(
    fake_review_store: SimpleNamespace, monkeypatch: pytest.MonkeyPatch
) -> None:
    """distance == threshold is kept (_passes_threshold uses <=)."""
    import openexecutive.config as config_mod

    monkeypatch.setattr(
        config_mod,
        "get_settings",
        lambda: SimpleNamespace(
            knowledge_builtin_n_results=5,
            knowledge_company_n_results=3,
            knowledge_distance_threshold=0.65,
            knowledge_builtin_distance_threshold=0.60,
            vector_store_path="/unused",
        ),
    )
    store = _make_store(
        builtin_hits=[{"text": "exactly at the gate", "metadata": {"filename": "b.md"}, "distance": 0.60}],
        company_hits=[],
    )
    out = retrieve(query="a real question about strategy", store=store, review_store=fake_review_store)
    assert "exactly at the gate" in out


def test_explicit_builtin_arg_wins_over_everything(
    fake_review_store: SimpleNamespace, monkeypatch: pytest.MonkeyPatch
) -> None:
    """builtin_distance_threshold outranks both the pinned shared arg and the setting."""
    import openexecutive.config as config_mod

    monkeypatch.setattr(
        config_mod,
        "get_settings",
        lambda: SimpleNamespace(
            knowledge_builtin_n_results=5,
            knowledge_company_n_results=3,
            knowledge_distance_threshold=0.65,
            knowledge_builtin_distance_threshold=0.99,
            vector_store_path="/unused",
        ),
    )
    store = _make_store(
        builtin_hits=[{"text": "generic handbook", "metadata": {"filename": "b.md"}, "distance": 0.62}],
        company_hits=[],
    )
    out = retrieve(
        query="a real question about strategy",
        store=store,
        review_store=fake_review_store,
        distance_threshold=0.90,
        builtin_distance_threshold=0.50,
    )
    assert "generic handbook" not in out