AI-powered virtual executive team — a single coherent executive persona backed by 8 specialist agents (FastAPI + Next.js).
"""Deterministic grounding for prose nobody reads before it ships.
Structured outputs are already grounded: the alert review closes only against
evidence ids the server issued, memory extraction drops anything not quoted
verbatim, and a research watch must name an entity in company data. This
module does the same for the prose written by unattended passes — the
morning brief, the end-of-day digest, the executive reflection's flags and
outward tool calls, and the text triage writes onto an alert.
The rule is narrow and mechanical, with no model in the loop:
- **People.** A person-shaped name in the prose (a run of two or more
capitalised words, or a single capitalised word in a person slot such as
"ask Marcus" / "Marcus wrote") must appear in ONE source line the pass was
given — every token of it, so a first name from one line and a surname
from another cannot be spliced together — or belong to a roster person
who is mentioned in some source. Being on the roster alone is not enough:
the model only knows what it read.
- **Figures.** Money, percentages, numbers with a k/M/B multiplier and any
bare number of 11 or more must equal (within the precision the prose
shows: ``$1.2M`` matches ``$1,234,567``) a number in some source or the
company profile. Dates, times, years, durations, ids, list markers,
numbers glued to letters (``Q3``, ``FY26``) and small bare counts are not
checked.
Every entry point fails open, like ``orchestrator.outbound_guard``: an
internal error is logged and the input comes back unchanged, because a
checker must never turn a working brief into a crash. ``GROUNDING_CHECKS``
selects ``enforce`` (hold back / refuse), ``report`` (audit only, text
unchanged) or ``off``.
"""
from __future__ import annotations
import json
import logging
import re
import unicodedata
from collections.abc import Awaitable, Callable, Iterable
from dataclasses import dataclass, field
from typing import Any
logger = logging.getLogger(__name__)
MODES = ("enforce", "report", "off")
# Both point at the "Left out of your brief" block on Home, which lists the
# held-back lines (``recent_held_back`` below, served by GET /today).
HELD_BACK_NOTE = (
"_I left out {n} line{s} that named people or figures I couldn't match to "
"your data. They're under Left out of your brief on Home._"
)
HELD_BACK_ALL = (
"Everything I drafted for this brief named people or figures I couldn't "
"match to your data, so I've held it back. The lines are under Left out "
"of your brief on Home."
)
# The briefs whose held-back lines Home lists (``surface`` on the audit row).
BRIEF_SURFACES = ("morning brief", "end-of-day digest")
# Lines kept for Home per brief, and characters per line.
_HELD_KEEP = 30
_HELD_LINE_CHARS = 400
SOURCES_HEADER = "**Sources**"
_MAX_CITED_SOURCES = 9
_SNIPPET_CHARS = 90
_MAX_RESULT_CHARS = 4000
# --------------------------------------------------------------------------- #
# Data
# --------------------------------------------------------------------------- #
@dataclass(frozen=True)
class GroundingSource:
"""One line of input a pass was given. ``extra_values`` are numbers the
line implies without spelling them (a section header's bullet count)."""
ref: str
label: str
text: str
extra_values: tuple[float, ...] = ()
@dataclass(frozen=True)
class Figure:
start: int
end: int
raw: str
value: float
tol: float
kind: str # money | pct | count
@dataclass
class LineCheck:
index: int
text: str
names_bad: list[str] = field(default_factory=list)
figures_bad: list[str] = field(default_factory=list)
grounded: list[tuple[Figure, GroundingSource]] = field(default_factory=list)
@property
def ok(self) -> bool:
return not self.names_bad and not self.figures_bad
@dataclass
class GroundingReport:
lines: list[LineCheck]
@property
def bad_lines(self) -> list[LineCheck]:
return [c for c in self.lines if not c.ok]
@property
def ok(self) -> bool:
return not self.bad_lines
@property
def names_bad(self) -> list[str]:
return _dedupe(n for c in self.lines for n in c.names_bad)
@property
def figures_bad(self) -> list[str]:
return _dedupe(f for c in self.lines for f in c.figures_bad)
def summary(self) -> str:
items = [f"'{x}'" for x in (self.names_bad + self.figures_bad)[:6]]
return ", ".join(items)
def _dedupe(items: Iterable[str]) -> list[str]:
seen: dict[str, None] = {}
for item in items:
seen.setdefault(item, None)
return list(seen)
# --------------------------------------------------------------------------- #
# Normalisation
# --------------------------------------------------------------------------- #
_FOLDS = str.maketrans({"‘": "'", "’": "'", "“": '"', "”": '"', "–": "-", "—": "-"})
_TOKEN_RE = re.compile(r"[^\W_]+", re.UNICODE)
def _fold(text: str) -> str:
return unicodedata.normalize("NFKC", text or "").translate(_FOLDS)
def _tokens(text: str) -> set[str]:
return set(_TOKEN_RE.findall(_fold(text).casefold()))
def _mask(text: str, pattern: re.Pattern[str]) -> str:
"""Blank each match with spaces so positions in ``text`` stay valid."""
return pattern.sub(lambda m: " " * len(m.group(0)), text)
# --------------------------------------------------------------------------- #
# Figures
# --------------------------------------------------------------------------- #
_NUM_RE = re.compile(
r"(?P<cur>[$€£])?\s?"
r"(?P<num>\d{1,3}(?:,\d{3})+(?:\.\d+)?|\d+(?:\.\d+)?)"
r"(?:\s?(?P<mult>bn|mm|thousand|million|billion|[kmb])(?![a-z]))?"
r"(?:\s?(?P<pct>%|percent\b))?",
re.IGNORECASE,
)
_MULTIPLIERS = {
"k": 1e3, "thousand": 1e3,
"m": 1e6, "mm": 1e6, "million": 1e6,
"b": 1e9, "bn": 1e9, "billion": 1e9,
}
_MONTHS = (
r"(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*\.?"
)
# Masked before any number is read, in prose and sources alike.
_ALWAYS_MASK = re.compile(
r"https?://\S+"
r"|(?<![\w.+-])[\w.+-]+@[\w-]+(?:\.[\w-]+)+"
r"|\b\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2})?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?)?"
r"|\b\d{1,2}:\d{2}(?::\d{2})?(?:\s?[ap]\.?m\.?)?"
r"|\b\d{1,2}/\d{1,2}(?:/\d{2,4})?\b",
re.IGNORECASE,
)
# Masked in prose only: things that look like figures but never are.
_PROSE_MASK = re.compile(
r"\[\d{1,2}\]"
r"|^\s*\d+[.)]\s"
r"|\b\d{1,2}\s?[ap]\.?m\b\.?"
rf"|\b{_MONTHS}\s+\d{{1,2}}(?:st|nd|rd|th)?\b"
rf"|\b\d{{1,2}}(?:st|nd|rd|th)?\s+{_MONTHS}(?![a-z])"
r"|\b\d+(?:\.\d+)?\s?(?:minutes?|mins?|hours?|hrs?|days?|weeks?|wks?|"
r"months?|mos?|years?|yrs?|quarters?|business days?)\b",
re.IGNORECASE | re.MULTILINE,
)
def _figure_from(m: re.Match[str]) -> tuple[float, float, str]:
num = m.group("num")
mult_key = (m.group("mult") or "").lower()
mult = _MULTIPLIERS.get(mult_key, 1.0)
decimals = len(num.split(".", 1)[1]) if "." in num else 0
value = float(num.replace(",", "")) * mult
tol = (10.0 ** -decimals) / 2 * mult
kind = "money" if m.group("cur") else "pct" if m.group("pct") else "count"
return value, tol, kind
def extract_figures(text: str) -> list[Figure]:
"""The figures in ``text`` that must be grounded (see module doc)."""
masked = _mask(_mask(text, _ALWAYS_MASK), _PROSE_MASK)
out: list[Figure] = []
for m in _NUM_RE.finditer(masked):
s = m.start("num")
before = masked[s - 1] if s > 0 else " "
if m.group("cur"):
before = " "
num_end = m.end("num")
after = masked[num_end] if num_end < len(masked) else " "
if before.isalpha() or before in "#_.-/=" or before.isdigit():
continue # Q3, FY26, #42, v1.2, 2026-…, id=7
if not (m.group("mult") or m.group("pct")) and (after.isalpha() or after == "_"):
continue # 3d, 5th, 2x
value, tol, kind = _figure_from(m)
plain = kind == "count" and not m.group("mult")
if plain and value <= 10:
continue
if plain and "," not in m.group("num") and "." not in m.group("num") and 1990 <= value <= 2100:
continue # a year
start = m.start("cur") if m.group("cur") else s
out.append(Figure(start, m.end(), text[start:m.end()].strip(), value, tol, kind))
return out
def _values_in(text: str) -> tuple[float, ...]:
"""Every number a source line holds, read generously: glued forms
(``14d``, ``at_risk=2``) count, and a scaled figure also offers its
unscaled digits."""
values: list[float] = []
for m in _NUM_RE.finditer(_mask(_fold(text), _ALWAYS_MASK)):
value, _tol, _kind = _figure_from(m)
values.append(value)
if m.group("mult"):
values.append(float(m.group("num").replace(",", "")))
return tuple(values)
# --------------------------------------------------------------------------- #
# Names
# --------------------------------------------------------------------------- #
def _wordset(*chunks: str) -> frozenset[str]:
return frozenset(w.casefold() for chunk in chunks for w in chunk.split())
_STOP = _wordset(
# calendar
"Monday Tuesday Wednesday Thursday Friday Saturday Sunday Mon Tue Tues Wed "
"Thu Thur Thurs Fri Sat Sun January February March April May June July "
"August September October November December Jan Feb Mar Apr Jun Jul Aug "
"Sep Sept Oct Nov Dec Today Tonight Tomorrow Yesterday Morning Afternoon "
"Evening Weekend Week Month Quarter Year Overnight EOD"
# the brief / digest / reflection section headers
" Top Call Three What Changed Handled Needs Need Waiting On At Risk Due "
"This Goals Goal Still Pending Sleep Open Decisions Decision Acted Flagged "
"For The Brief Quiet Move Bottom Line Did Carried Over Sources Source Held "
"Back Grounding Tool Calls Run Day Digest Update Status Summary Next Steps "
"Action Actions Items Item Note Notes Heads Up Reminder FYI"
# determiners, pronouns, function words
" A An And Or But Nor So Yet If Then Than That These Those This There "
"Their They Them We Us Our You Your Yours He She His Her Hers It Its I Me "
"My Mine Who Whom Whose Which What When Where Why How All Any Each Every "
"Everyone Everybody Everything Someone Somebody Something Anyone Anybody "
"Anything Nobody Nothing None No Not Yes Both Either Neither Few Many Most "
"Some Several Other Another Such Of To In Into Onto Off Out Up Down With "
"Without Within From By As About Above Below After Before Between Per Via "
"Re Fwd Hi Hey Hello Dear Thanks Thank Please Also Just Only Still New Old "
"Here Now Soon Later Again Once"
# imperative openers
" Ask Approve Tell Ping Nudge Chase Reply Review Sign Send Book Decide "
"Confirm Follow Check Loop Hold Push Clear Close Escalate Share Draft Focus "
"Protect Delegate Prep Prepare Read Skim Block Schedule Reschedule Cancel "
"Accept Decline Answer Respond Consider Keep Watch Expect Note Plan Start "
"Finish Wrap Get Give Make Take Let Go Set Put Use Try Call Meet Talk "
"Discuss Sync Align Update Fix Resolve Unblock Flag Raise Move Pick Grab"
# product and org nouns
" Open Executive OE Slack Discord Telegram Gmail Google Calendar Drive Docs "
"Sheets Notion Zoom Meet Teams Email Inbox Chat Web Watchlist Board Finance "
"Sales Marketing Engineering Product Operations Ops Legal People HR Support "
"Team Company Department Departments Area Areas Customer Customers Revenue "
"Pipeline Budget Hiring Security Design Data Research Strategy CEO CFO COO "
"CTO CMO VP Head Director Manager Lead Principal Owner Founder"
)
# Words inside a title-cased run that split it (real names never hold them).
_SPLITTERS = _wordset("the and of for on at to with in your you this that a an or from by")
_PARTICLES = _wordset("van von de da di del della la le du al bin ibn st")
_BEFORE_SLOT = _wordset(
"ask asked asking tell told ping pinged nudge nudged chase chased remind "
"reminded email emailed message messaged dm call called cc cc'd meet met "
"with from by to and for thank thanks per"
)
_AFTER_SLOT = _wordset(
"wrote writes asked asks said says replied replies sent sends emailed "
"messaged pinged called texted confirmed approved agreed declined promised "
"mentioned requested signed owes flagged hasn't hasnt isn't"
)
_WORD_RE = re.compile(r"[^\W\d_]+(?:['\-][^\W\d_]+)*'?", re.UNICODE)
_SEGMENT_SPLIT = re.compile(r"(?<=[.!?;:])\s+|\s[-]\s|[()\[\]\"•|,]|\s-{2,}\s")
_MD_MASK = re.compile(
r"\*\*|__|`|^\s*#{1,6}\s|^\s*>+\s?|^\s*[-*+]\s|\]\([^)]*\)|https?://\S+"
r"|(?<![\w.+-])[\w.+-]+@[\w-]+(?:\.[\w-]+)+",
re.MULTILINE,
)
def _is_name_token(word: str) -> bool:
letters = [c for c in word if c.isalpha()]
if len(letters) < 2 or not word[0].isupper():
return False
if not any(c.islower() for c in word):
return False
for i, part in enumerate(re.split(r"['\-]", word)):
if not part:
continue
tail = part[1:]
if tail and not tail.islower():
return False # OKR, iPhone, McKinsey — never a name token
if i == 0 and not part[0].isupper():
return False
return True
@dataclass(frozen=True)
class _Word:
text: str # possessive stripped
start: int
end: int
possessive: bool
def _segment_words(segment: str) -> list[_Word]:
words: list[_Word] = []
for m in _WORD_RE.finditer(segment):
raw = m.group(0)
possessive = False
if raw.lower().endswith("'s"):
raw, possessive = raw[:-2], True
elif raw.endswith("'"):
raw = raw[:-1]
possessive = raw.lower().endswith("s")
if raw:
words.append(_Word(raw, m.start(), m.start() + len(raw), possessive))
return words
def extract_names(text: str) -> list[str]:
"""Person-shaped names in ``text`` that must be grounded."""
cleaned = _mask(_fold(text), _MD_MASK)
out: list[str] = []
for line in cleaned.splitlines():
for segment in _SEGMENT_SPLIT.split(line):
if segment and segment.strip():
out.extend(_segment_names(segment))
return _dedupe(out)
def _segment_names(segment: str) -> list[str]:
words = _segment_words(segment)
names: list[str] = []
i = 0
while i < len(words):
if not _is_name_token(words[i].text):
i += 1
continue
run = [i]
j = i + 1
while j < len(words):
if words[j - 1].possessive or segment[words[j - 1].end:words[j].start] != " ":
break
w = words[j].text
particle = (
w.casefold() in _PARTICLES
and j + 1 < len(words)
and _is_name_token(words[j + 1].text)
)
if not (_is_name_token(w) or particle):
break
run.append(j)
j += 1
names.extend(_run_candidates(words, run))
i = j
return names
def _run_candidates(words: list[_Word], run: list[int]) -> list[str]:
"""Split a title-cased run at function words, trim STOP words from each
piece's edges, and keep what is person-shaped."""
pieces: list[list[int]] = [[]]
for idx in run:
if words[idx].text.casefold() in _SPLITTERS:
pieces.append([])
else:
pieces[-1].append(idx)
out: list[str] = []
for piece in pieces:
while piece and (words[piece[0]].text.casefold() in _STOP or words[piece[0]].text.casefold() in _PARTICLES):
piece = piece[1:]
while piece and (words[piece[-1]].text.casefold() in _STOP or words[piece[-1]].text.casefold() in _PARTICLES):
piece = piece[:-1]
if not piece:
continue
if len(piece) >= 2:
out.append(" ".join(words[k].text for k in piece))
continue
k = piece[0]
prev_word = words[k - 1].text.casefold() if k > 0 else ""
prev2 = words[k - 2].text.casefold() if k > 1 else ""
next_word = words[k + 1].text.casefold() if k + 1 < len(words) else ""
in_slot = (
(k > 0 and prev_word in _BEFORE_SLOT)
or (prev2 == "loop" and prev_word == "in")
or next_word in _AFTER_SLOT
or words[k].possessive
)
if in_slot:
out.append(words[k].text)
return out
# --------------------------------------------------------------------------- #
# Checking
# --------------------------------------------------------------------------- #
class _Index:
"""Sources pre-digested for lookups: token sets, numbers, and which
roster people some source mentions."""
def __init__(self, sources: list[GroundingSource], roster: list[str]) -> None:
self.sources = sources
self.tokens = [_tokens(s.text) | _tokens(s.label) for s in sources]
self.values = [_values_in(s.text) + s.extra_values for s in sources]
people = [(_tokens(n), n) for n in roster if n and n.strip()]
firsts: dict[str, int] = {}
for _toks, name in people:
first = _fold(name).split()[0].casefold() if name.split() else ""
if first:
firsts[first] = firsts.get(first, 0) + 1
self.present: list[set[str]] = []
for toks, name in people:
if not toks:
continue
first = _fold(name).split()[0].casefold()
for src in self.tokens:
if toks <= src or (firsts.get(first) == 1 and first in src):
self.present.append(toks)
break
def name_ok(self, name: str) -> bool:
toks = _tokens(name)
if not toks:
return True
if any(toks <= src for src in self.tokens):
return True
return any(toks <= person for person in self.present)
def figure_sources(self, fig: Figure) -> list[int]:
target = abs(fig.value)
hits = [
i for i, values in enumerate(self.values)
if any(abs(abs(v) - target) <= fig.tol + 1e-9 * max(1.0, target) for v in values)
]
if hits or fig.kind != "pct":
return hits
# A percentage the model worked out from two numbers on one line
# ("40 of 100" grounds 40%).
for i, values in enumerate(self.values):
small = values[:12]
if any(
b and abs(100.0 * a / b - target) <= fig.tol + 1e-9
for a in small for b in small if a is not b
):
hits.append(i)
return hits
_CLAUSE_BREAK = re.compile(r"[;,.!?()]\s|\s[-–—]\s|[()]")
def _clause(line: str, fig: Figure) -> str:
"""The clause of ``line`` a figure sits in — what its citation is about."""
start = 0
end = len(line)
for m in _CLAUSE_BREAK.finditer(line):
if m.end() <= fig.start:
start = m.end()
elif m.start() >= fig.end:
end = m.start()
break
return line[start:end]
def _content_tokens(text: str) -> set[str]:
return {t for t in _tokens(text) if len(t) >= 3 and not t.isdigit() and t not in _SPLITTERS}
def check_text(
text: str, sources: list[GroundingSource], roster: list[str] | None = None,
*, index: _Index | None = None,
) -> GroundingReport:
"""Check every line of ``text``. Pure: no I/O."""
idx = index or _Index(sources, roster or [])
lines: list[LineCheck] = []
for n, line in enumerate(text.splitlines()):
check = LineCheck(index=n, text=line)
for name in extract_names(line):
if not idx.name_ok(name):
check.names_bad.append(name)
for fig in extract_figures(line):
hits = idx.figure_sources(fig)
if not hits:
check.figures_bad.append(fig.raw)
continue
near = _content_tokens(_clause(line, fig))
whole = _content_tokens(line)
best = max(
hits,
key=lambda i: (len(near & idx.tokens[i]), len(whole & idx.tokens[i]), -i),
)
check.grounded.append((fig, idx.sources[best]))
lines.append(check)
return GroundingReport(lines)
# --------------------------------------------------------------------------- #
# Sources
# --------------------------------------------------------------------------- #
_HEADER_HEAD = re.compile(r"^([A-Z][A-Z0-9 '’&/\-—]*[A-Z])\b")
_BULLET_PREFIX = re.compile(r"^\s*(?:[-*•]|>|\d+[.)])\s*")
def _header_label(line: str) -> str | None:
"""The label of a rendered-context section header (``INBOUND SINCE THE
LAST BRIEF (3 messages…):`` → "Inbound since the last brief"), or None."""
if line.startswith((" ", "-", "\t")):
return None
m = _HEADER_HEAD.match(line)
if not m:
return None
head = m.group(1).split(" — ")[0].strip(" -—")
if sum(c.isalpha() for c in head) < 3:
return None
rest = line[m.end():].lstrip()
if not (rest.startswith(("(", ":", "—", "-")) or not rest):
return None
return head[:1] + head[1:].lower()
def sources_from_context(rendered: str, prefix: str = "ctx") -> list[GroundingSource]:
"""One source per non-empty line of a rendered context, labelled with the
section it sits under. A header also carries its bullet count, so "three
new messages" can be grounded by the list itself."""
label = "Context"
out: list[GroundingSource] = []
header_at: int | None = None
count = 0
def close_header() -> None:
if header_at is not None and count:
h = out[header_at]
out[header_at] = GroundingSource(h.ref, h.label, h.text, h.extra_values + (float(count),))
for raw in (rendered or "").splitlines():
if not raw.strip():
continue
head = _header_label(raw)
if head is not None:
close_header()
label, count = head, 0
header_at = len(out)
out.append(GroundingSource(f"{prefix}{len(out)}", label, raw.strip()))
continue
if _BULLET_PREFIX.match(raw):
count += 1
text = _BULLET_PREFIX.sub("", raw).strip()
if text:
out.append(GroundingSource(f"{prefix}{len(out)}", label, text))
close_header()
return out
def sources_from_text(text: str, label: str, prefix: str) -> list[GroundingSource]:
return [
GroundingSource(f"{prefix}{i}", label, line.strip())
for i, line in enumerate((text or "").splitlines())
if line.strip()
]
def profile_sources() -> list[GroundingSource]:
"""The company profile's lines (name, ARR, burn, runway, key metrics,
leadership, vendors…), labelled "Company profile". [] when unavailable."""
try:
from openexecutive.onboarding.profile_builder import load_or_create_profile
block = load_or_create_profile().to_prompt_block()
except Exception:
logger.debug("grounding: profile unavailable", exc_info=True)
return []
lines = [ln for ln in block.splitlines() if ln.strip() and not ln.startswith("## ")]
return sources_from_text("\n".join(lines), "Company profile", "profile")
def org_sources() -> list[GroundingSource]:
"""Department titles and slugs — names the prose may use for an area
without the context spelling them. [] when unavailable."""
try:
from openexecutive.departments.registry import list_states
titles = [f"{d.config.title} ({d.config.slug})" for d in list_states()]
except Exception:
logger.debug("grounding: departments unavailable", exc_info=True)
return []
return sources_from_text("\n".join(titles), "Departments", "dept")
def roster() -> list[str]:
"""Full names of everyone on the roster, contacts included."""
try:
from openexecutive.people.store import list_people
return [p.full_name for p in list_people(include_contacts=True) if p.full_name]
except Exception:
logger.debug("grounding: roster unavailable", exc_info=True)
return []
def grounding_mode() -> str:
try:
from openexecutive.config import get_settings
mode = str(get_settings().grounding_checks)
except Exception:
return "enforce"
return mode if mode in MODES else "enforce"
def citations_enabled() -> bool:
try:
from openexecutive.config import get_settings
return bool(get_settings().grounding_citations)
except Exception:
return True
def _audit(
summary: str,
details: dict[str, Any],
*,
private: bool,
held: list[str] | None = None,
) -> None:
"""Audit a grounding pass. ``held`` (every held-back line) goes in the
row's untruncated payload, which Home reads back; ``details`` keeps a
short preview for the audit list."""
try:
from openexecutive.audit import log_event
full = {"held_back": [h[:_HELD_LINE_CHARS] for h in held[:_HELD_KEEP]]} if held else None
log_event(
"grounding", summary, actor="executive", details=details, full=full, private=private
)
except Exception:
logger.debug("grounding: audit failed", exc_info=True)
@dataclass
class HeldBack:
"""One brief's held-back lines, for Home."""
event_id: int
surface: str
at: str
lines: list[str]
def recent_held_back(*, since: str, limit: int = 3) -> list[HeldBack]:
"""The lines grounding held back from the principal's briefs since
``since`` (ISO), newest brief first. Only ``enforce`` passes: in
``report`` mode nothing was left out. Never raises.
Private rows included: callers serve this to the principal alone."""
try:
from openexecutive.audit.logger import get_audit_logger
audit = get_audit_logger()
out: list[HeldBack] = []
for ev in audit.query(event_type="grounding", since=since, limit=50):
details = ev.details or {}
if details.get("surface") not in BRIEF_SURFACES or details.get("mode") != "enforce":
continue
if not details.get("held_back"):
continue # nothing held (a repair-only pass): skip the payload read
full = audit.get(ev.id)
lines = ((full.full if full else None) or {}).get("held_back") or []
lines = [str(ln) for ln in lines if str(ln).strip()]
if not lines:
continue
out.append(HeldBack(event_id=ev.id, surface=str(details["surface"]), at=ev.ts, lines=lines))
if len(out) >= limit:
break
return out
except Exception:
logger.debug("grounding: held-back lines unavailable", exc_info=True)
return []
# --------------------------------------------------------------------------- #
# Briefs: drop, note, cite
# --------------------------------------------------------------------------- #
_HEADER_LINE = re.compile(r"^\s*(?:#{1,6}\s+\S.*|\*\*[^*]+\*\*:?|__[^_]+__:?)\s*$")
def _is_header(line: str) -> bool:
return bool(_HEADER_LINE.match(line))
def _drop_lines(text: str, bad: set[int]) -> tuple[str, list[str]]:
"""Remove the ``bad`` line indexes, then any header left with nothing
under it, then runs of blank lines."""
lines = text.splitlines()
held = [lines[i] for i in sorted(bad) if i < len(lines)]
kept = [ln for i, ln in enumerate(lines) if i not in bad]
out: list[str] = []
for i, ln in enumerate(kept):
if _is_header(ln):
rest = kept[i + 1:]
has_body = False
for nxt in rest:
if _is_header(nxt):
break
if nxt.strip():
has_body = True
break
if not has_body:
continue
if not ln.strip() and (not out or not out[-1].strip()):
continue
out.append(ln)
while out and not out[-1].strip():
out.pop()
return "\n".join(out), held
_SNIPPET_STRIP = re.compile(r"[\[\]()<>*_`|\\]")
_LINKISH = re.compile(r"(?:https?://|www\.)\S+", re.IGNORECASE)
_MENTION = re.compile(r"(?<![\w.])@(?=\w)")
def _plain(text: str) -> str:
"""Quoted text made inert for every channel the prose reaches: links
masked (the Sources list and a fallback quote attacker-written mail, and
Gmail / Slack / Discord linkify bare URLs), Markdown and Slack's
``<!channel>`` / ``<url|text>`` syntax stripped, @-mentions broken."""
text = _SNIPPET_STRIP.sub(" ", _LINKISH.sub("\x00", text))
text = _MENTION.sub("", text).replace("\x00", "(link removed)")
return " ".join(text.split())
def _snippet(text: str, limit: int = _SNIPPET_CHARS) -> str:
clean = _plain(text)
return clean if len(clean) <= limit else clean[: limit - 1].rstrip() + "…"
def _cite(text: str, report: GroundingReport) -> tuple[str, int]:
"""Put ``[n]`` after each grounded figure and a Sources list at the end.
Numbered by first appearance; at most ``_MAX_CITED_SOURCES`` sources."""
numbers: dict[str, int] = {}
listed: list[GroundingSource] = []
lines = text.splitlines()
by_index = {c.index: c for c in report.lines}
for i, line in enumerate(lines):
check = by_index.get(i)
if check is None or not check.grounded or _is_header(line):
continue
inserts: list[tuple[int, str]] = []
for fig, src in check.grounded:
n = numbers.get(src.ref)
if n is None:
if len(listed) >= _MAX_CITED_SOURCES:
continue
listed.append(src)
n = numbers[src.ref] = len(listed)
if line[fig.end:fig.end + 1] == "[" or line[fig.end:fig.end + 2] == " [":
continue
tail = " " if line[fig.end:fig.end + 1] == "(" else ""
inserts.append((fig.end, f" [{n}]{tail}"))
for pos, mark in sorted(inserts, reverse=True):
line = line[:pos] + mark + line[pos:]
lines[i] = line
if not listed:
return text, 0
footer = [
"", SOURCES_HEADER,
*(f"- [{n}] {_snippet(src.label, 40)} — {_snippet(src.text)}" for n, src in enumerate(listed, 1)),
]
return "\n".join(lines + footer), len(listed)
_REPAIR_INSTRUCTION = (
"Below is the context you were given and a draft you wrote from it. The "
"draft names people or figures that do not appear anywhere in the "
"context: {items}. Rewrite the draft in the same format, removing each "
"of those or correcting it to exactly what the context says. Change "
"nothing else. Output only the rewritten message."
)
async def _repair(draft: str, context: str, system: str, report: GroundingReport) -> str:
from openexecutive.agents.utility_fast import get_fast_model
from openexecutive.audit.usage import log_model_usage
from openexecutive.providers import get_provider
model = get_fast_model()
content = (
f"{context}\n\n---\nDRAFT:\n{draft}\n\n"
+ _REPAIR_INSTRUCTION.format(items=report.summary())
)
response = await get_provider(model).messages_create(
model=model,
max_tokens=700,
system=system,
messages=[{"role": "user", "content": content}],
)
log_model_usage(response, model=model, actor="briefing_grounding")
blocks = [b for b in response.content if getattr(b, "type", "") == "text"]
return blocks[0].text.strip() if blocks else ""
def context_sources(context: str) -> list[GroundingSource]:
return sources_from_context(context) + profile_sources() + org_sources()
@dataclass
class BriefGrounding:
"""What the pass did to one brief."""
mode: str
held: list[str] = field(default_factory=list)
names_bad: list[str] = field(default_factory=list)
figures_bad: list[str] = field(default_factory=list)
cited: int = 0
repaired: bool = False
async def ground_brief(
text: str,
*,
context: str,
kind: str,
private: bool = False,
system: str | None = None,
surface: str = "brief",
) -> tuple[str, BriefGrounding | None]:
"""Ground a brief or digest against the context it was written from.
``enforce``: one repair call when something is ungrounded (only when
``system`` is given), then drop the lines still ungrounded, drop headers
left empty, add the held-back note and the citations. ``report``: audit
only, text unchanged. ``off``: nothing. Never raises; on an internal
error the text comes back unchanged."""
mode = grounding_mode()
if mode == "off" or not text.strip():
return text, None
try:
sources = context_sources(context)
idx = _Index(sources, roster())
report = check_text(text, sources, index=idx)
result = BriefGrounding(mode=mode)
if not report.ok and mode == "enforce" and system:
try:
fixed = await _repair(text, context, system, report)
except Exception:
logger.warning("grounding: repair call failed; dropping lines instead", exc_info=True)
fixed = ""
if fixed:
fixed_report = check_text(fixed, sources, index=idx)
if len(fixed_report.bad_lines) <= len(report.bad_lines):
text, report, result.repaired = fixed, fixed_report, True
result.names_bad = report.names_bad[:10]
result.figures_bad = report.figures_bad[:10]
out = text
if mode == "enforce":
if not report.ok:
out, result.held = _drop_lines(text, {c.index for c in report.bad_lines})
if any(ln.strip() and not _is_header(ln) for ln in out.splitlines()):
report = check_text(out, sources, index=idx)
else:
out = HELD_BACK_ALL
if citations_enabled() and out != HELD_BACK_ALL:
out, result.cited = _cite(out, report)
if result.held and out != HELD_BACK_ALL:
n = len(result.held)
note = HELD_BACK_NOTE.format(n=n, s="" if n == 1 else "s")
body, sep, footer = out.partition("\n\n" + SOURCES_HEADER)
out = f"{body}\n\n{note}" + (sep + footer if sep else "")
else:
result.held = [c.text for c in report.bad_lines]
if result.held or result.repaired:
verb = "held back" if mode == "enforce" else "found (report only)"
_audit(
f"Grounding: {verb} {len(result.held)} line(s) of the {surface}",
{
"surface": surface, "kind": kind, "mode": mode,
"repaired": result.repaired, "cited": result.cited,
"names_bad": result.names_bad, "figures_bad": result.figures_bad,
"held_back": [h[:200] for h in result.held[:5]],
},
private=private,
held=result.held if mode == "enforce" else None,
)
return out, result
except Exception:
logger.exception("grounding: brief check failed; delivering unchanged")
return text, None
# --------------------------------------------------------------------------- #
# Unattended tool loops: refuse ungrounded outward prose
# --------------------------------------------------------------------------- #
OUTWARD_FIELDS: dict[str, tuple[str, ...]] = {
"create_alert": ("subject", "body"),
"message_person": ("text",),
"send_department_message": ("text",),
"send_company_broadcast": ("text",),
"schedule_followup": ("intent",),
}
Handler = Callable[[dict[str, Any]], Awaitable[Any]]
class GroundingScope:
"""The inputs of one unattended tool loop. Outward tools refuse prose
that names people or figures not in them; every tool result joins them,
so a ``lookup_person`` answer grounds the DM that follows it."""
def __init__(
self,
sources: list[GroundingSource],
roster_names: list[str],
*,
surface: str,
private: bool = False,
) -> None:
self.sources = list(sources)
self.roster = list(roster_names)
self.surface = surface
self.private = private
self.mode = grounding_mode()
self._results = 0
self._index: _Index | None = None
def _idx(self) -> _Index:
if self._index is None:
self._index = _Index(self.sources, self.roster)
return self._index
def add_text(self, label: str, text: str) -> None:
self._results += 1
self.sources.extend(
sources_from_text(text[:_MAX_RESULT_CHARS], label, f"r{self._results}.")
)
self._index = None
def check(self, text: str) -> GroundingReport:
return check_text(text, self.sources, index=self._idx())
def refusal(self, tool: str, tool_input: dict[str, Any]) -> str | None:
"""The reason ``tool`` must not run, or None. Audits every finding;
only ``enforce`` refuses."""
if self.mode == "off" or tool not in OUTWARD_FIELDS:
return None
try:
prose = "\n".join(
str(tool_input.get(f) or "") for f in OUTWARD_FIELDS[tool]
).strip()
if not prose:
return None
report = self.check(prose)
if report.ok and tool == "message_person":
return None
if not report.ok and tool == "message_person":
# The recipient's own name is grounded by the call itself.
recipient = _person_name(tool_input.get("person_id"))
if recipient:
extra = self.sources + [GroundingSource("recipient", "Recipient", recipient)]
report = check_text(prose, extra, self.roster)
if report.ok:
return None
reason = (
f"Not sent — couldn't find {report.summary()} in your inputs. Use "
"only people and figures from the context or a tool result, or "
"flag it for the brief instead."
)
_audit(
f"Grounding: {'refused' if self.mode == 'enforce' else 'would refuse'} {tool}",
{
"surface": self.surface, "tool": tool, "mode": self.mode,
"names_bad": report.names_bad[:10],
"figures_bad": report.figures_bad[:10],
"text_preview": prose[:200],
},
private=self.private,
)
return reason if self.mode == "enforce" else None
except Exception:
logger.exception("grounding: tool check failed; allowing %s", tool)
return None
def guard(self, handlers: dict[str, Handler]) -> dict[str, Handler]:
return {name: self._wrap(name, fn) for name, fn in handlers.items()}
def _wrap(self, name: str, fn: Handler) -> Handler:
async def guarded(tool_input: dict[str, Any]) -> Any:
reason = self.refusal(name, tool_input or {})
if reason:
return json.dumps({"error": reason})
result = await fn(tool_input)
try:
self.add_text(f"Tool result — {name}", str(result))
except Exception:
logger.debug("grounding: could not record %s result", name, exc_info=True)
return result
return guarded
def filter_section(self, text: str, heading: str) -> tuple[str, list[str]]:
"""Drop the ungrounded lines of the ``**heading:**`` section (through
the next bold header or ``---``). Returns (text, held-back lines)."""
if self.mode != "enforce":
return text, []
try:
pattern = re.compile(
rf"(\*\*{re.escape(heading)}:?\*\*:?)(.*?)(?=\n\s*\*\*[^*\n]+:?\*\*|\n---|\Z)",
re.DOTALL | re.IGNORECASE,
)
m = pattern.search(text)
if m is None:
return text, []
body = m.group(2)
report = self.check(body)
if report.ok:
return text, []
bad = {c.index for c in report.bad_lines}
# Same numbering as check_text (splitlines), so a lone \r or
#
in the flags can't shift which line gets dropped.
lines = body.splitlines(keepends=True)
held = [lines[i].strip() for i in sorted(bad) if lines[i].strip()]
kept = "".join(ln for i, ln in enumerate(lines) if i not in bad)
if not kept.strip():
kept = " (nothing I could ground)"
_audit(
f"Grounding: held back {len(held)} line(s) of the {self.surface}",
{
"surface": self.surface, "section": heading, "mode": self.mode,
"names_bad": report.names_bad[:10],
"figures_bad": report.figures_bad[:10],
"held_back": [h[:200] for h in held[:5]],
},
private=self.private,
held=held,
)
return text[: m.start(2)] + kept + text[m.end(2):], held
except Exception:
logger.exception("grounding: section filter failed; keeping it")
return text, []
def _person_name(person_id: Any) -> str:
try:
from openexecutive.people.store import get_person
person = get_person(int(person_id))
except Exception:
return ""
return person.full_name if person is not None else ""
# --------------------------------------------------------------------------- #
# Alert text
# --------------------------------------------------------------------------- #
def ungrounded(text: str, sources: list[GroundingSource]) -> list[str]:
"""The names and figures in ``text`` that no source holds ([] when all
are grounded, and on any error)."""
try:
report = check_text(text or "", sources)
return report.names_bad + report.figures_bad
except Exception:
logger.exception("grounding: check failed; treating as grounded")
return []
_SENTENCE_SPLIT = re.compile(r"(?<=[.!?])\s+")
def ground_alert_text(
headline: str,
body: str,
suggested_action: str,
*,
sources: list[GroundingSource],
fallback_headline: str,
fallback_body: str,
surface: str,
private: bool = False,
) -> tuple[str, str, str, bool]:
"""Ground model-written alert text against what it was written from.
An ungrounded headline becomes ``fallback_headline``; ungrounded body
sentences are dropped (all of them → ``fallback_body`` cut to 280); both
fallbacks are made inert first (links masked, markup and mentions
stripped), because they are raw event text; an
ungrounded suggested action is dropped. Returns ``(headline, body,
suggested_action, changed)``. Fails open."""
mode = grounding_mode()
if mode == "off":
return headline, body, suggested_action, False
try:
idx = _Index(sources, [])
new_headline, new_body, new_action = headline, body, suggested_action
bad: list[str] = []
head_report = check_text(headline, sources, index=idx)
if not head_report.ok and fallback_headline.strip():
bad += head_report.names_bad + head_report.figures_bad
# The fallback is raw event text (inbound mail, a feed): quote it
# inert, never verbatim, since broadcasts post it as written.
new_headline = _plain(fallback_headline)[:100]
sentences = [s for s in _SENTENCE_SPLIT.split(body.strip()) if s]
kept: list[str] = []
for sentence in sentences:
r = check_text(sentence, sources, index=idx)
if r.ok:
kept.append(sentence)
else:
bad += r.names_bad + r.figures_bad
if len(kept) != len(sentences):
new_body = " ".join(kept) if kept else _plain(fallback_body)[:280]
action_report = check_text(suggested_action, sources, index=idx)
if not action_report.ok:
bad += action_report.names_bad + action_report.figures_bad
new_action = ""
changed = (new_headline, new_body, new_action) != (headline, body, suggested_action)
if bad:
_audit(
f"Grounding: {'rewrote' if mode == 'enforce' else 'would rewrite'} {surface} text",
{
"surface": surface, "mode": mode, "items": _dedupe(bad)[:10],
"headline_preview": headline[:160],
},
private=private,
)
if mode != "enforce":
return headline, body, suggested_action, False
return new_headline, new_body, new_action, changed
except Exception:
logger.exception("grounding: alert text check failed; keeping it")
return headline, body, suggested_action, False
__all__ = [
"HELD_BACK_ALL",
"HELD_BACK_NOTE",
"OUTWARD_FIELDS",
"SOURCES_HEADER",
"BriefGrounding",
"GroundingReport",
"GroundingScope",
"GroundingSource",
"check_text",
"context_sources",
"extract_figures",
"extract_names",
"ground_alert_text",
"ground_brief",
"org_sources",
"profile_sources",
"roster",
"sources_from_context",
"sources_from_text",
"ungrounded",
]