AI-powered virtual executive team — a single coherent executive persona backed by 8 specialist agents (FastAPI + Next.js).
"""Deterministic grounding of unattended prose (briefing/grounding.py).
The checker must catch a fabricated colleague and a wrong figure, and must
not trip on the brief's own section headers, dates, times, quarters or small
counts — a false positive drops a real line from the principal's brief.
"""
from __future__ import annotations
import json
from typing import Any
import pytest
from openexecutive.briefing import grounding as g
from openexecutive.briefing.grounding import (
HELD_BACK_ALL,
SOURCES_HEADER,
GroundingScope,
GroundingSource,
check_text,
extract_figures,
extract_names,
ground_alert_text,
ground_brief,
sources_from_context,
sources_from_text,
)
CONTEXT = """PERIOD: 2026-09-29
INBOUND SINCE THE LAST BRIEF (3 messages, newest first, local times; quoted):
- 08:14 email from Dana Whitfield <[email protected]>: Lease renewal for 48 units due Friday
- 07:50 email from [email protected]: Budget $1,234,567 approved
- 07:10 slack from Tom Ruiz: can we sync?
DEPARTMENTS WITH RISK:
- Sales: at_risk=2 off_track=1 awaiting=0
• Q3 pipeline (at_risk: 1.2M of 2M)
CARRIED OVER: 12 older item(s) still open (oldest 9d) — see /today
"""
@pytest.fixture
def sources() -> list[GroundingSource]:
return sources_from_context(CONTEXT)
@pytest.fixture(autouse=True)
def _no_company_data(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str, str, dict[str, Any]]]:
"""Grounding reads the profile, departments and roster; keep them empty
and capture audit rows instead of writing them."""
monkeypatch.setattr(g, "profile_sources", lambda: [])
monkeypatch.setattr(g, "org_sources", lambda: [])
monkeypatch.setattr(g, "roster", lambda: ["Dana Whitfield", "Tom Ruiz"])
monkeypatch.setattr(g, "grounding_mode", lambda: "enforce")
monkeypatch.setattr(g, "citations_enabled", lambda: True)
rows: list[tuple[str, str, dict[str, Any]]] = []
def _log(event_type: str, summary: str, **kw: Any) -> None:
rows.append((event_type, summary, kw))
monkeypatch.setattr("openexecutive.audit.log_event", _log)
return rows
def _bad(text: str, sources: list[GroundingSource], roster: list[str] | None = None) -> list[str]:
report = check_text(text, sources, roster or [])
return report.names_bad + report.figures_bad
# --------------------------------------------------------------------------- #
# Figures
# --------------------------------------------------------------------------- #
@pytest.mark.parametrize(
"text",
[
"Board meets 2026-10-02 at 9:30",
"Call at 10am, then 3pm",
"Due Sep 29 or 29th September",
"Q3 plan, FY26 budget, H1 review",
"3 of 5 goals on track; top 3 today",
"1. First item",
"Waiting 14 days, 3 weeks overdue",
"Alert #42 is open",
"Plan for 2027",
"See [1] above",
],
)
def test_figure_exemptions(text: str) -> None:
assert extract_figures(text) == []
@pytest.mark.parametrize(
("text", "value", "kind"),
[
("Burn is $1.4M", 1_400_000, "money"),
("Budget of $1,234,567", 1_234_567, "money"),
("Churn is 12%", 12, "pct"),
("12.5 percent of revenue", 12.5, "pct"),
("48 units renewing", 48, "count"),
("About 10k signups", 10_000, "count"),
("£3 billion market", 3e9, "money"),
],
)
def test_figures_extracted(text: str, value: float, kind: str) -> None:
[fig] = extract_figures(text)
assert fig.value == pytest.approx(value)
assert fig.kind == kind
def test_figure_matches_within_shown_precision(sources: list[GroundingSource]) -> None:
assert _bad("Budget of $1.2M approved", sources) == []
assert _bad("Budget of $1.3M approved", sources) == ["$1.3M"]
assert _bad("Budget of $1,234,567 approved", sources) == []
assert _bad("Budget of $1,234,000 approved", sources) == ["$1,234,000"]
def test_wrong_unit_count_is_caught(sources: list[GroundingSource]) -> None:
assert _bad("Lease renewal covers 48 units", sources) == []
assert _bad("Lease renewal covers 47 units", sources) == ["47"]
def test_percent_from_two_numbers_on_one_line(sources: list[GroundingSource]) -> None:
# 1.2M of 2M is 60%; 18% is nowhere.
assert _bad("Pipeline at 60% of target", sources) == []
assert _bad("Revenue up 18% this quarter", sources) == ["18%"]
def test_section_bullet_count_grounds_a_count() -> None:
ctx = "PEOPLE WAITING ON YOU:\n" + "\n".join(f"- Person {i} (x)" for i in range(12))
assert _bad("12 people are waiting on you", sources_from_context(ctx)) == []
assert _bad("13 people are waiting on you", sources_from_context(ctx)) == ["13"]
def test_profile_figures_ground() -> None:
profile = sources_from_text("**ARR**: $90,000,000\n Runway: 14.0 months", "Company profile", "p")
assert _bad("ARR is $90M", profile) == []
assert _bad("ARR is $95M", profile) == ["$95M"]
# --------------------------------------------------------------------------- #
# Names
# --------------------------------------------------------------------------- #
@pytest.mark.parametrize(
"text",
[
"**Top call** — Approve the lease renewal.",
"**Top Three Today**",
"**Needs you**",
"**What changed**",
"Monday is packed; OKR review in Q3.",
"Take it to the Board via Slack.",
"Burn trending high.",
"**Sleep on this** — is the Acme deal worth it?",
"Everyone needs the deck by Friday.",
],
)
def test_no_name_candidates_in_brief_furniture(text: str) -> None:
assert extract_names(text) == []
@pytest.mark.parametrize(
("text", "names"),
[
("Marcus Lee flagged the lease", ["Marcus Lee"]),
("ask Marcus to sign", ["Marcus"]),
("Marcus's reply is overdue", ["Marcus"]),
("Marcus wrote about the lease", ["Marcus"]),
("Loop in Sarah before noon", ["Sarah"]),
("Mary-Jane O'Brien signed", ["Mary-Jane O'Brien"]),
("Ask Ludwig van Beethoven", ["Ludwig van Beethoven"]),
],
)
def test_person_names_found(text: str, names: list[str]) -> None:
assert extract_names(text) == names
def test_fabricated_colleague_is_caught(sources: list[GroundingSource]) -> None:
assert _bad("Marcus Lee flagged the lease — ask Marcus to sign.", sources) == [
"Marcus Lee", "Marcus",
]
def test_real_people_pass_in_any_form(sources: list[GroundingSource]) -> None:
assert _bad("Sign Dana's lease renewal", sources) == []
assert _bad("Dana Whitfield wrote about the lease", sources) == []
assert _bad("ask Tom to sync", sources) == []
def test_names_cannot_be_spliced_across_lines(sources: list[GroundingSource]) -> None:
# "Dana" and "Ruiz" both appear — on different lines.
assert _bad("Dana Ruiz wrote about the lease", sources) == ["Dana Ruiz"]
def test_email_local_part_grounds_a_name() -> None:
src = sources_from_text("email from [email protected]: lease", "Inbound", "i")
assert _bad("Dana Whitfield wrote", src) == []
def test_roster_person_mentioned_by_first_name_grounds_full_name() -> None:
src = sources_from_text("slack from Dana: lease signed", "Inbound", "i")
assert _bad("Dana Whitfield signed the lease", src, ["Dana Whitfield"]) == []
# Being on the roster alone is not enough.
assert _bad("Tom Ruiz signed the lease", src, ["Dana Whitfield", "Tom Ruiz"]) == ["Tom Ruiz"]
# --------------------------------------------------------------------------- #
# Sources
# --------------------------------------------------------------------------- #
def test_sources_are_labelled_by_section(sources: list[GroundingSource]) -> None:
labels = {s.label for s in sources}
assert "Inbound since the last brief" in labels
assert "Departments with risk" in labels
assert "Carried over" in labels
lease = next(s for s in sources if "Lease" in s.text)
assert lease.label == "Inbound since the last brief"
assert not lease.text.startswith("-")
# --------------------------------------------------------------------------- #
# ground_brief
# --------------------------------------------------------------------------- #
BRIEF = """**Top call** — Sign Dana's lease renewal for 48 units by Friday.
**What changed**
- Budget of $1.2M approved.
- Marcus Lee flagged the lease.
**At risk**
- Revenue up 18% this quarter"""
@pytest.mark.asyncio
async def test_ground_brief_drops_cites_and_notes(
_no_company_data: list[tuple[str, str, dict[str, Any]]],
) -> None:
out, result = await ground_brief(BRIEF, context=CONTEXT, kind="k", private=True)
assert result is not None
assert "Marcus" not in out and "18%" not in out
# The emptied "At risk" header goes too.
assert "**At risk**" not in out
assert "48 [1] units" in out
assert "$1.2M [2] approved" in out
assert "_I left out 2 lines" in out and "Left out of your brief on Home" in out
body, footer = out.split(SOURCES_HEADER)
assert "I left out" in body
assert "- [1] Inbound since the last brief — " in footer
assert "- [2] Inbound since the last brief — " in footer and "Budget $1,234,567" in footer
# Never "[n]:" at a line start (a Markdown link reference definition).
assert not any(ln.lstrip().startswith("[") for ln in out.splitlines())
[(event_type, _summary, kw)] = _no_company_data
assert event_type == "grounding"
assert kw["private"] is True
assert kw["details"]["names_bad"] == ["Marcus Lee"]
assert kw["details"]["figures_bad"] == ["18%"]
assert len(kw["details"]["held_back"]) == 2
# Every held line rides in the untruncated payload Home reads back.
assert len(kw["full"]["held_back"]) == 2
@pytest.mark.asyncio
async def test_clean_text_without_figures_is_byte_identical(
_no_company_data: list[tuple[str, str, dict[str, Any]]],
) -> None:
text = "**Top call** — Sign Dana's lease renewal by Friday."
out, _ = await ground_brief(text, context=CONTEXT, kind="k")
assert out == text
assert _no_company_data == []
@pytest.mark.asyncio
async def test_everything_ungrounded_is_held_back_whole() -> None:
out, result = await ground_brief(
"**Needs you**\n- Marcus Lee wants $9M", context=CONTEXT, kind="k",
)
assert out == HELD_BACK_ALL
assert result is not None and len(result.held) == 1
@pytest.mark.asyncio
async def test_report_mode_leaves_text_and_audits(
monkeypatch: pytest.MonkeyPatch,
_no_company_data: list[tuple[str, str, dict[str, Any]]],
) -> None:
monkeypatch.setattr(g, "grounding_mode", lambda: "report")
out, result = await ground_brief(BRIEF, context=CONTEXT, kind="k")
assert out == BRIEF
assert result is not None and len(result.held) == 2
assert _no_company_data and _no_company_data[0][2]["details"]["mode"] == "report"
assert _no_company_data[0][2]["full"] is None # nothing was left out
@pytest.mark.asyncio
async def test_off_mode_does_nothing(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(g, "grounding_mode", lambda: "off")
assert await ground_brief(BRIEF, context=CONTEXT, kind="k") == (BRIEF, None)
@pytest.mark.asyncio
async def test_citations_can_be_turned_off(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(g, "citations_enabled", lambda: False)
text = "**Top call** — Sign the renewal for 48 units."
out, _ = await ground_brief(text, context=CONTEXT, kind="k")
assert out == text
@pytest.mark.asyncio
async def test_citations_cap_and_sanitise() -> None:
lines = [f"- ITEM{i}: [x](javascript:y) <b>{100 + i}</b> things" for i in range(12)]
ctx = "STUFF:\n" + "\n".join(lines)
brief = "\n".join(f"- {100 + i} things" for i in range(12))
out, _ = await ground_brief(brief, context=ctx, kind="k")
footer = out.split(SOURCES_HEADER)[1]
assert footer.count("\n- [") == 9
assert "[x]" not in footer and "<b>" not in footer and "(javascript" not in footer
assert "- 111 things" in out # the 12th figure: over the cap, left unmarked
@pytest.mark.asyncio
async def test_repair_call_used_when_it_grounds(monkeypatch: pytest.MonkeyPatch) -> None:
prompts: list[str] = []
async def _repair(draft: str, context: str, system: str, report: Any) -> str:
prompts.append(report.summary())
return draft.replace("- Marcus Lee flagged the lease.\n", "").replace(
"- Revenue up 18% this quarter", "- Sales pipeline at risk"
)
monkeypatch.setattr(g, "_repair", _repair)
out, result = await ground_brief(BRIEF, context=CONTEXT, kind="k", system="SYS")
assert prompts == ["'Marcus Lee', '18%'"]
assert result is not None and result.repaired and result.held == []
assert "I left out" not in out
assert "Sales pipeline at risk" in out
@pytest.mark.asyncio
async def test_repair_failure_falls_back_to_dropping(monkeypatch: pytest.MonkeyPatch) -> None:
async def _boom(*_a: Any, **_k: Any) -> str:
raise RuntimeError("provider down")
monkeypatch.setattr(g, "_repair", _boom)
out, result = await ground_brief(BRIEF, context=CONTEXT, kind="k", system="SYS")
assert result is not None and not result.repaired and len(result.held) == 2
assert "Marcus" not in out
@pytest.mark.asyncio
async def test_internal_error_fails_open(monkeypatch: pytest.MonkeyPatch) -> None:
def _boom(*_a: Any, **_k: Any) -> Any:
raise RuntimeError("bug")
monkeypatch.setattr(g, "check_text", _boom)
assert await ground_brief(BRIEF, context=CONTEXT, kind="k") == (BRIEF, None)
# --------------------------------------------------------------------------- #
# GroundingScope (unattended tool loops)
# --------------------------------------------------------------------------- #
@pytest.mark.asyncio
async def test_scope_refuses_ungrounded_outward_prose(
sources: list[GroundingSource],
_no_company_data: list[tuple[str, str, dict[str, Any]]],
) -> None:
calls: list[dict[str, Any]] = []
async def _send(tool_input: dict[str, Any]) -> str:
calls.append(tool_input)
return json.dumps({"ok": True})
scope = GroundingScope(sources, [], surface="test")
handlers = scope.guard({"create_alert": _send, "send_company_broadcast": _send})
refused = json.loads(await handlers["create_alert"](
{"subject": "Lease", "body": "Marcus Lee says the renewal is 52 units."}
))
assert "error" in refused and "'Marcus Lee'" in refused["error"] and "'52'" in refused["error"]
assert calls == []
ok = await handlers["send_company_broadcast"]({"text": "Dana's renewal covers 48 units."})
assert json.loads(ok) == {"ok": True}
assert len(calls) == 1
assert _no_company_data[0][2]["details"]["tool"] == "create_alert"
@pytest.mark.asyncio
async def test_tool_results_ground_later_prose(sources: list[GroundingSource]) -> None:
async def _lookup(_tool_input: dict[str, Any]) -> str:
return json.dumps({"person_id": 7, "full_name": "Marcus Lee", "role": "Property manager"})
async def _send(_tool_input: dict[str, Any]) -> str:
return json.dumps({"ok": True})
scope = GroundingScope(sources, [], surface="test")
handlers = scope.guard({"lookup_person": _lookup, "message_person": _send})
msg = {"person_id": 7, "text": "Marcus Lee: Dana needs the lease checked."}
assert "error" in json.loads(await handlers["message_person"](msg))
await handlers["lookup_person"]({"query": "property manager"})
assert json.loads(await handlers["message_person"](msg)) == {"ok": True}
@pytest.mark.asyncio
async def test_non_outward_tools_and_report_mode_are_not_refused(
monkeypatch: pytest.MonkeyPatch, sources: list[GroundingSource],
) -> None:
async def _ok(_tool_input: dict[str, Any]) -> str:
return "{}"
scope = GroundingScope(sources, [], surface="test")
assert await scope.guard({"lookup_person": _ok})["lookup_person"]({"query": "Marcus Lee"}) == "{}"
monkeypatch.setattr(g, "grounding_mode", lambda: "report")
scope = GroundingScope(sources, [], surface="test")
assert await scope.guard({"create_alert": _ok})["create_alert"](
{"subject": "x", "body": "Marcus Lee"}
) == "{}"
def test_filter_section_drops_ungrounded_flags(sources: list[GroundingSource]) -> None:
text = (
"**Acted on:**\n- Marcus Lee pinged\n\n"
"**Flagged for the brief:**\n- Dana's renewal: 48 units\n- Marcus Lee wants a call\n\n"
"**Quiet:** nothing else.\n\n---\n_Tool calls this run:_"
)
scope = GroundingScope(sources, [], surface="test")
out, held = scope.filter_section(text, "Flagged for the brief")
assert held == ["- Marcus Lee wants a call"]
assert "Dana's renewal" in out
# Only the flags section is filtered.
assert "- Marcus Lee pinged" in out
assert "**Quiet:**" in out
def test_filter_section_numbers_lines_like_check_text(sources: list[GroundingSource]) -> None:
"""A lone \\r inside a flag must not shift which line is dropped."""
text = (
"**Flagged for the brief:**\n- Dana's renewal:\r48 units\n"
"- Marcus Lee wants a call\n- Nothing else pressing\n\n**Quiet:** done."
)
scope = GroundingScope(sources, [], surface="test")
out, held = scope.filter_section(text, "Flagged for the brief")
assert held == ["- Marcus Lee wants a call"]
assert "Marcus" not in out
assert "- Dana's renewal:\r48 units\n" in out and "- Nothing else pressing" in out
# --------------------------------------------------------------------------- #
# Alert text
# --------------------------------------------------------------------------- #
def test_alert_text_falls_back_to_the_event() -> None:
src = sources_from_text("Lease renewal\nDana Whitfield: renewal for 48 units due Friday", "Event", "e")
headline, body, action, changed = ground_alert_text(
"Marcus Lee: lease renewal", "Dana asks to renew 48 units. Marcus Lee manages it.",
"Call Marcus Lee",
sources=src, fallback_headline="Lease renewal",
fallback_body="Dana Whitfield: renewal for 48 units due Friday", surface="t",
)
assert changed
assert headline == "Lease renewal"
assert body == "Dana asks to renew 48 units."
assert action == ""
def test_alert_text_unchanged_when_grounded() -> None:
src = sources_from_text("Dana Whitfield: renewal for 48 units", "Event", "e")
assert ground_alert_text(
"Dana: renewal", "Renewal for 48 units.", "Reply to Dana",
sources=src, fallback_headline="x", fallback_body="y", surface="t",
) == ("Dana: renewal", "Renewal for 48 units.", "Reply to Dana", False)
def test_grounding_is_a_declared_audit_event_type() -> None:
from openexecutive.audit.logger import EVENT_TYPES
assert "grounding" in EVENT_TYPES
def test_raw_event_fallbacks_are_made_inert() -> None:
"""The fallback is attacker-written event text a broadcast posts as is:
no live link, no Slack <!channel>, no @everyone."""
src = sources_from_text("Revenue fell by forty-two million", "Event", "e")
headline, body, _action, changed = ground_alert_text(
"Revenue fell $42M", "Revenue fell $42M.", "",
sources=src,
fallback_headline="Revenue fell <!channel> see https://evil.example/x @everyone",
fallback_body="<https://evil.example/pay|Reset password> now @here www.evil.example",
surface="t",
)
assert changed
for text in (headline, body):
assert "evil.example" not in text and "<" not in text and "|" not in text
assert "@everyone" not in text and "@here" not in text
assert "(link removed)" in headline
@pytest.mark.asyncio
async def test_citation_snippets_mask_links_and_mentions() -> None:
ctx = (
"INBOUND SINCE THE LAST BRIEF (1 message):\n"
"- 08:14 email from Al <[email protected]>: 48 units, pay at https://evil.example/pay @everyone"
)
out, _ = await ground_brief("- Renewal covers 48 units", context=ctx, kind="k")
footer = out.split(SOURCES_HEADER)[1]
assert "evil.example" not in footer and "@everyone" not in footer
assert "(link removed)" in footer and "[email protected]" in footer
def test_recent_held_back_lists_enforced_brief_lines_newest_first(
tmp_path: Any, monkeypatch: pytest.MonkeyPatch
) -> None:
from openexecutive.audit import logger as audit_logger_module
audit = audit_logger_module.AuditLogger(tmp_path / "audit.db")
monkeypatch.setattr(audit_logger_module, "get_audit_logger", lambda: audit)
lines = [f"- line {i} naming Marcus Lee " + "x" * 300 for i in range(12)]
audit.log(
"grounding", "old", actor="executive",
details={"surface": "end-of-day digest", "mode": "enforce", "held_back": ["- yesterday's line"]},
full={"held_back": ["- yesterday's line"]}, private=True,
)
audit.log(
"grounding", "new", actor="executive",
details={"surface": "morning brief", "mode": "enforce", "held_back": lines[:5]},
full={"held_back": lines},
)
audit.log( # report mode left nothing out
"grounding", "r", actor="executive",
details={"surface": "morning brief", "mode": "report"}, full=None,
)
audit.log( # not a brief
"grounding", "t", actor="executive",
details={"surface": "unattended turn", "mode": "enforce"}, full={"held_back": ["x"]},
)
audit.log( # repair-only pass: nothing held, so the payload is never read
"grounding", "fix", actor="executive",
details={"surface": "morning brief", "mode": "enforce", "held_back": []},
full={"held_back": ["- not shown"]},
)
found = g.recent_held_back(since="2000-01-01T00:00:00+00:00")
assert [h.surface for h in found] == ["morning brief", "end-of-day digest"]
assert found[0].lines == lines # all of them, past the 4 KB details cap
assert g.recent_held_back(since="2999-01-01T00:00:00+00:00") == []