AI-powered virtual executive team — a single coherent executive persona backed by 8 specialist agents (FastAPI + Next.js).
from __future__ import annotations
import logging
import re
from collections import defaultdict
from pathlib import Path
from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel
from openexecutive.knowledge.loader import (
BUILTIN_KNOWLEDGE_PATH,
DOMAIN_MAP,
FAILURES_KNOWLEDGE_PATH,
UPLOAD_DOMAINS,
)
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/knowledge")
_VALID_FILENAME = re.compile(r"^[a-zA-Z0-9_\-]+\.md$")
_VALID_SOURCE_ID = re.compile(r"^[a-zA-Z0-9_\-]+$")
class BuiltinFileMeta(BaseModel):
domain: str
filename: str
size_bytes: int
class BuiltinFileContent(BaseModel):
domain: str
filename: str
content: str
class BuiltinFileWrite(BaseModel):
domain: str
filename: str
content: str
class BuiltinListResponse(BaseModel):
files: list[BuiltinFileMeta]
class BuiltinWriteResponse(BaseModel):
domain: str
filename: str
chunks_indexed: int
def _validate_domain(domain: str) -> None:
if domain not in DOMAIN_MAP:
raise HTTPException(status_code=400, detail=f"Unknown domain: {domain}")
def _validate_filename(filename: str) -> None:
if not _VALID_FILENAME.match(filename):
raise HTTPException(
status_code=400,
detail="Filename must be alphanumeric with dashes or underscores and end in .md",
)
def _resolve_path(domain: str, filename: str) -> Path:
return BUILTIN_KNOWLEDGE_PATH / domain / filename
def _get_store(request: Request): # type: ignore[return]
if hasattr(request.app.state, "store"):
return request.app.state.store
from openexecutive.config import get_settings
from openexecutive.knowledge.store import ChromaDBStore
return ChromaDBStore(persist_directory=get_settings().vector_store_path)
def _review_store(): # type: ignore[no-untyped-def]
"""ReviewStore bound to the DB every other consumer reads.
`ReviewStore()`'s default is captured at import, so a runtime override of
`EPISODIC_DB_PATH` (client slots, tests) left this module writing review
rows into one database while `routes/review._store` and
`retriever._default_review_store` read another — the gate would silently
consult state this module never wrote.
"""
from openexecutive.knowledge.review_store import ReviewStore
from openexecutive.memory.episodic import DB_PATH
return ReviewStore(db_path=DB_PATH)
def _existing_review_item(item_id: str): # type: ignore[no-untyped-def]
"""Look up a review item without 500ing or materialising the database.
`sqlite3.connect` creates the file, and a DB with no `review_items` table
raises — neither should turn a legitimate upload into a server error.
Absent review state simply means nothing is known to be trusted.
"""
from openexecutive.memory.episodic import DB_PATH
if not DB_PATH.exists():
return None
try:
return _review_store().get_item(item_id)
except Exception:
logger.warning("review state unreadable during upload check", exc_info=True)
return None
@router.get("/builtin", response_model=BuiltinListResponse)
async def list_builtin_files() -> BuiltinListResponse:
files: list[BuiltinFileMeta] = []
for domain in sorted(DOMAIN_MAP.keys()):
domain_dir = BUILTIN_KNOWLEDGE_PATH / domain
if not domain_dir.exists():
continue
for f in sorted(domain_dir.glob("*.md")):
files.append(
BuiltinFileMeta(domain=domain, filename=f.name, size_bytes=f.stat().st_size)
)
return BuiltinListResponse(files=files)
@router.get("/builtin/{domain}/{filename}", response_model=BuiltinFileContent)
async def get_builtin_file(domain: str, filename: str) -> BuiltinFileContent:
_validate_domain(domain)
_validate_filename(filename)
path = _resolve_path(domain, filename)
if not path.exists():
raise HTTPException(status_code=404, detail="File not found")
return BuiltinFileContent(domain=domain, filename=filename, content=path.read_text(encoding="utf-8"))
@router.post("/builtin", response_model=BuiltinWriteResponse)
async def create_builtin_file(body: BuiltinFileWrite, request: Request) -> BuiltinWriteResponse:
_validate_domain(body.domain)
_validate_filename(body.filename)
path = _resolve_path(body.domain, body.filename)
if path.exists():
raise HTTPException(status_code=409, detail="File already exists. Use PUT to update.")
from openexecutive.knowledge.loader import ingest_builtin_file
from openexecutive.knowledge.review_store import ContentType, build_item_id
# Defence in depth. Shipped failure docs used to derive ids in this very
# namespace, so an upload of a name like `board/theranos.md` landed on a
# row already marked `approved` + `trusted_default = 1` and inherited it —
# trusted, unqueued, and labelled "ships with the product" in the UI. The
# FAILURE namespace fixes that at the source; this makes sure no future
# id-space change can quietly re-open it.
item_id = build_item_id(ContentType.BUILTIN, body.domain, body.filename)
existing = _existing_review_item(item_id)
if existing is not None and existing.trusted_default:
raise HTTPException(
status_code=409,
detail="That name belongs to content shipped with Open Executive.",
)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(body.content, encoding="utf-8")
chunks = await ingest_builtin_file(path, _get_store(request))
_review_store().register(
item_id=item_id,
content_type=ContentType.BUILTIN,
domain=body.domain,
filename=body.filename,
)
return BuiltinWriteResponse(domain=body.domain, filename=body.filename, chunks_indexed=chunks)
@router.put("/builtin/{domain}/{filename}", response_model=BuiltinWriteResponse)
async def update_builtin_file(
domain: str, filename: str, body: BuiltinFileWrite, request: Request
) -> BuiltinWriteResponse:
_validate_domain(domain)
_validate_filename(filename)
path = _resolve_path(domain, filename)
if not path.exists():
raise HTTPException(status_code=404, detail="File not found. Use POST to create.")
from openexecutive.knowledge.loader import ingest_builtin_file
from openexecutive.knowledge.review_store import ContentType, build_item_id
from openexecutive.knowledge.store import ChromaDBStore
store = _get_store(request)
store.delete_documents(
collection=ChromaDBStore.BUILTIN_COLLECTION,
where={"source": str(path)},
)
path.write_text(body.content, encoding="utf-8")
chunks = await ingest_builtin_file(path, store)
rs = _review_store()
item_id = build_item_id(ContentType.BUILTIN, domain, filename)
rs.register(item_id=item_id, content_type=ContentType.BUILTIN, domain=domain, filename=filename)
rs.touch_modified(item_id)
return BuiltinWriteResponse(domain=domain, filename=filename, chunks_indexed=chunks)
# ---------------------------------------------------------------------------
# External / OER reference library (read-only)
# ---------------------------------------------------------------------------
class ExternalSourceInfo(BaseModel):
id: str
title: str
publisher: str
license: str
phase: int
domains: list[str]
type: str
url: str
slug: str | None = None
chunks: int
files: int
is_ingested: bool
last_fetched_at: float | None = None
class ExternalSourcesResponse(BaseModel):
sources: list[ExternalSourceInfo]
total_chunks: int
class ExternalPeekChunk(BaseModel):
domain: str
filename: str
chunk_index: int
text: str
class ExternalPeekResponse(BaseModel):
source_id: str
chunks: list[ExternalPeekChunk]
def _validate_source_id(source_id: str) -> None:
if not _VALID_SOURCE_ID.match(source_id):
raise HTTPException(status_code=400, detail="Invalid source id")
def _cache_mtime(cache_dir: Path) -> float | None:
"""Newest mtime of any non-hidden file in the source's local cache.
Used as a "last fetched at" indicator — that's when the on-disk artifact
was last written, which is also when the most recent ingest read it.
"""
if not cache_dir.exists():
return None
newest: float | None = None
for p in cache_dir.rglob("*"):
if not p.is_file():
continue
if any(part.startswith(".") for part in p.relative_to(cache_dir).parts):
continue
m = p.stat().st_mtime
if newest is None or m > newest:
newest = m
return newest
@router.get("/external", response_model=ExternalSourcesResponse)
async def list_external_sources(request: Request) -> ExternalSourcesResponse:
"""List every source declared in sources.yaml with live ingest stats."""
from openexecutive.knowledge.external_sources import load_manifest
from openexecutive.knowledge.store import ChromaDBStore
manifest = load_manifest()
store = _get_store(request)
# Single bulk fetch is faster than per-source queries when there are 10+ sources.
try:
col = store._client.get_collection(ChromaDBStore.BUILTIN_COLLECTION)
rows = col.get(include=["metadatas"])
except Exception:
rows = {"metadatas": []}
chunk_counts: dict[str, int] = defaultdict(int)
files_per_source: dict[str, set[str]] = defaultdict(set)
for md in rows["metadatas"] or []:
sid = md.get("source_id")
if not sid:
continue
chunk_counts[sid] += 1
if fn := md.get("filename"):
files_per_source[sid].add(fn)
sources: list[ExternalSourceInfo] = []
for src in manifest:
chunks = chunk_counts.get(src.id, 0)
sources.append(
ExternalSourceInfo(
id=src.id,
title=src.title,
publisher=src.publisher,
license=src.license,
phase=src.phase,
domains=src.domains,
type=src.type,
url=src.url,
slug=src.slug,
chunks=chunks,
files=len(files_per_source.get(src.id, set())),
is_ingested=chunks > 0,
last_fetched_at=_cache_mtime(src.cache_dir),
)
)
from openexecutive.knowledge.review_store import ReviewStore
ingested = [
{"id": s.id, "domains": s.domains}
for s in sources
if s.is_ingested
]
if ingested:
from openexecutive.memory.episodic import DB_PATH as _REVIEW_DB
ReviewStore.sync_external_registrations(ingested, _REVIEW_DB)
return ExternalSourcesResponse(sources=sources, total_chunks=sum(chunk_counts.values()))
@router.get("/external/{source_id}/peek", response_model=ExternalPeekResponse)
async def peek_external_source(
source_id: str, request: Request, limit: int = 5
) -> ExternalPeekResponse:
"""Return the first N indexed chunks of a source so a human can spot-check them."""
from openexecutive.knowledge.external_sources import load_manifest
from openexecutive.knowledge.store import ChromaDBStore
_validate_source_id(source_id)
if not any(src.id == source_id for src in load_manifest()):
raise HTTPException(status_code=404, detail=f"Unknown source: {source_id}")
limit = max(1, min(limit, 25))
store = _get_store(request)
try:
col = store._client.get_collection(ChromaDBStore.BUILTIN_COLLECTION)
rows = col.get(
where={"source_id": source_id},
limit=limit,
include=["documents", "metadatas"],
)
except Exception:
rows = {"documents": [], "metadatas": []}
chunks = [
ExternalPeekChunk(
domain=md.get("domain", "?"),
filename=md.get("filename", "?"),
chunk_index=md.get("chunk_index", 0),
text=doc,
)
for doc, md in zip(rows["documents"] or [], rows["metadatas"] or [], strict=False)
]
return ExternalPeekResponse(source_id=source_id, chunks=chunks)
@router.delete("/builtin/{domain}/{filename}")
async def delete_builtin_file(domain: str, filename: str, request: Request) -> dict:
_validate_domain(domain)
_validate_filename(filename)
path = _resolve_path(domain, filename)
if not path.exists():
raise HTTPException(status_code=404, detail="File not found")
from openexecutive.knowledge.review_store import ContentType, build_item_id
from openexecutive.knowledge.store import ChromaDBStore
store = _get_store(request)
store.delete_documents(
collection=ChromaDBStore.BUILTIN_COLLECTION,
where={"source": str(path)},
)
path.unlink()
_review_store().delete_item(
build_item_id(ContentType.BUILTIN, domain, filename)
)
return {"deleted": filename}
# ---------------------------------------------------------------------------
# Failures (negative learnings) — separate ChromaDB collection (failure_cases)
# but managed via the same CRUD shape as builtin playbooks. Chunked smaller
# (400 words, 40 overlap) to match seed_failures so a re-index produces the
# same artifacts whether done at seed-time or via PUT.
# ---------------------------------------------------------------------------
def _resolve_failure_path(domain: str, filename: str) -> Path:
return FAILURES_KNOWLEDGE_PATH / domain / filename
@router.get("/failures", response_model=BuiltinListResponse)
async def list_failure_files() -> BuiltinListResponse:
files: list[BuiltinFileMeta] = []
for domain in sorted(DOMAIN_MAP.keys()):
domain_dir = FAILURES_KNOWLEDGE_PATH / domain
if not domain_dir.exists():
continue
for f in sorted(domain_dir.glob("*.md")):
files.append(
BuiltinFileMeta(domain=domain, filename=f.name, size_bytes=f.stat().st_size)
)
return BuiltinListResponse(files=files)
@router.get("/failures/{domain}/{filename}", response_model=BuiltinFileContent)
async def get_failure_file(domain: str, filename: str) -> BuiltinFileContent:
_validate_domain(domain)
_validate_filename(filename)
path = _resolve_failure_path(domain, filename)
if not path.exists():
raise HTTPException(status_code=404, detail="File not found")
return BuiltinFileContent(
domain=domain, filename=filename, content=path.read_text(encoding="utf-8")
)
@router.post("/failures", response_model=BuiltinWriteResponse)
async def create_failure_file(body: BuiltinFileWrite, request: Request) -> BuiltinWriteResponse:
_validate_domain(body.domain)
_validate_filename(body.filename)
path = _resolve_failure_path(body.domain, body.filename)
if path.exists():
raise HTTPException(status_code=409, detail="File already exists. Use PUT to update.")
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(body.content, encoding="utf-8")
from openexecutive.knowledge.loader import ingest_builtin_file
from openexecutive.knowledge.store import ChromaDBStore
chunks = await ingest_builtin_file(
path,
_get_store(request),
collection=ChromaDBStore.FAILURES_COLLECTION,
chunk_type="failure_case",
chunk_size=400,
overlap=40,
)
# Register, exactly as create_builtin_file does. Without this a
# user-authored failure case study had no review row at all, so it could
# never enter the withheld set — `retrieve_failures` applied a gate that
# could not reach it, and no SME decision could block it.
from openexecutive.knowledge.review_store import ContentType, build_item_id
_review_store().register(
item_id=build_item_id(ContentType.FAILURE, body.domain, body.filename),
content_type=ContentType.FAILURE,
domain=body.domain,
filename=body.filename,
)
return BuiltinWriteResponse(domain=body.domain, filename=body.filename, chunks_indexed=chunks)
@router.put("/failures/{domain}/{filename}", response_model=BuiltinWriteResponse)
async def update_failure_file(
domain: str, filename: str, body: BuiltinFileWrite, request: Request
) -> BuiltinWriteResponse:
_validate_domain(domain)
_validate_filename(filename)
path = _resolve_failure_path(domain, filename)
if not path.exists():
raise HTTPException(status_code=404, detail="File not found. Use POST to create.")
from openexecutive.knowledge.loader import ingest_builtin_file
from openexecutive.knowledge.store import ChromaDBStore
store = _get_store(request)
store.delete_documents(
collection=ChromaDBStore.FAILURES_COLLECTION,
where={"source": str(path)},
)
path.write_text(body.content, encoding="utf-8")
chunks = await ingest_builtin_file(
path,
store,
collection=ChromaDBStore.FAILURES_COLLECTION,
chunk_type="failure_case",
chunk_size=400,
overlap=40,
)
# Mirror update_builtin_file: register (a doc created before failure docs
# were registered still has no row, and editing must create one) and mark
# it needs_revision so an edit behaves the same for both content types.
from openexecutive.knowledge.review_store import ContentType, build_item_id
rs = _review_store()
item_id = build_item_id(ContentType.FAILURE, domain, filename)
rs.register(
item_id=item_id,
content_type=ContentType.FAILURE,
domain=domain,
filename=filename,
)
rs.touch_modified(item_id)
return BuiltinWriteResponse(domain=domain, filename=filename, chunks_indexed=chunks)
@router.delete("/failures/{domain}/{filename}")
async def delete_failure_file(domain: str, filename: str, request: Request) -> dict:
_validate_domain(domain)
_validate_filename(filename)
path = _resolve_failure_path(domain, filename)
if not path.exists():
raise HTTPException(status_code=404, detail="File not found")
from openexecutive.knowledge.store import ChromaDBStore
store = _get_store(request)
store.delete_documents(
collection=ChromaDBStore.FAILURES_COLLECTION,
where={"source": str(path)},
)
path.unlink()
from openexecutive.knowledge.review_store import ContentType, build_item_id
_review_store().delete_item(build_item_id(ContentType.FAILURE, domain, filename))
return {"deleted": filename}
# ---------------------------------------------------------------------------
# Search — diagnostic "what would RAG retrieve for this question" endpoint.
# Returns raw structured chunks (text/metadata/distance) per collection so
# the UI can show exactly what each specialist would see. This is a read-only
# parallel of `retrieve()` and `retrieve_failures()` — it does NOT replace
# them or change the chat path.
# ---------------------------------------------------------------------------
_VALID_SOURCE_TYPES = {"builtin", "company", "failures", "external"}
# Max characters of chunk text returned per search hit. UI shows ~600 chars,
# leaving headroom for "…" truncation indicator and tail context.
_SEARCH_SNIPPET_LIMIT = 800
# When both 'builtin' (no source_id) and 'external' (with source_id) are requested
# from BUILTIN_COLLECTION, we over-fetch from a single query and partition by
# source_id. The window must be large enough that a dominant category doesn't
# starve the other. We pull (n_builtin + n_external) * OVERFETCH_MULTIPLIER rows.
_BUILTIN_OVERFETCH_MULTIPLIER = 5
# Per-bucket result count ceiling (caller-supplied n_* values are clamped here).
_MAX_RESULTS_PER_BUCKET = 25
class KnowledgeSearchRequest(BaseModel):
query: str
domain_filter: list[str] | None = None
specialist: str | None = None
n_builtin: int = 5
n_company: int = 3
n_failures: int = 3
n_external: int = 5
include: list[str] | None = None # subset of _VALID_SOURCE_TYPES
class SearchHit(BaseModel):
filename: str
domain: str
source: str | None = None
source_id: str | None = None
source_url: str | None = None
license: str | None = None
publisher: str | None = None
chunk_index: int | None = None
distance: float
text: str
class KnowledgeSearchResponse(BaseModel):
query: str
effective_domains: list[str] | None
specialists_that_would_see_this: list[str]
builtin: list[SearchHit]
company: list[SearchHit]
failures: list[SearchHit]
external: list[SearchHit]
def _hits_from_chroma(
rows: list[dict[str, object]], snippet_limit: int = _SEARCH_SNIPPET_LIMIT
) -> list[SearchHit]:
hits: list[SearchHit] = []
for r in rows:
md = r.get("metadata") or {}
if not isinstance(md, dict):
md = {}
text = str(r.get("text") or "")
if len(text) > snippet_limit:
text = text[:snippet_limit] + "…"
chunk_index_raw = md.get("chunk_index")
chunk_index = chunk_index_raw if isinstance(chunk_index_raw, int) else None
hits.append(
SearchHit(
filename=str(md.get("filename", "unknown")),
domain=str(md.get("domain", "?")),
source=md.get("source") if isinstance(md.get("source"), str) else None,
source_id=md.get("source_id") if isinstance(md.get("source_id"), str) else None,
source_url=md.get("source_url") if isinstance(md.get("source_url"), str) else None,
license=md.get("license") if isinstance(md.get("license"), str) else None,
publisher=md.get("publisher") if isinstance(md.get("publisher"), str) else None,
chunk_index=chunk_index,
distance=float(r.get("distance", 0.0) or 0.0), # type: ignore[arg-type]
text=text,
)
)
return hits
@router.post("/search", response_model=KnowledgeSearchResponse)
async def search_knowledge(
body: KnowledgeSearchRequest, request: Request
) -> KnowledgeSearchResponse:
"""Diagnostic: show what RAG would surface for a given query.
Mirrors the per-collection queries the chat path uses but returns raw
chunks (text/distance/metadata) instead of the formatted markdown blob
`retrieve()` produces. Intended for the Knowledge UI's Query mode and
for tuning the knowledge base offline.
"""
from openexecutive.knowledge.retriever import (
DOMAIN_ALIASES,
_default_review_store,
_with_general,
)
from openexecutive.knowledge.review_store import ContentType
from openexecutive.knowledge.store import ChromaDBStore
if not body.query.strip():
raise HTTPException(status_code=400, detail="query must be non-empty")
include = set(body.include) if body.include else set(_VALID_SOURCE_TYPES)
bad = include - _VALID_SOURCE_TYPES
if bad:
raise HTTPException(status_code=400, detail=f"Invalid include values: {sorted(bad)}")
# UPLOAD_DOMAINS, not DOMAIN_MAP: `general` is a real, uploadable company
# domain, and rejecting it here made this endpoint unable to introspect the
# documents most likely to need it — the unclassified ones.
if body.domain_filter:
for d in body.domain_filter:
if d not in UPLOAD_DOMAINS:
raise HTTPException(status_code=400, detail=f"Unknown domain: {d}")
if body.specialist and body.specialist not in DOMAIN_ALIASES:
raise HTTPException(status_code=400, detail=f"Unknown specialist: {body.specialist}")
effective_domains: list[str] | None = body.domain_filter
if effective_domains is None and body.specialist:
effective_domains = DOMAIN_ALIASES.get(body.specialist)
# Reverse-map: which specialists would see at least one of these domains?
if effective_domains:
specialists_seeing = sorted(
name
for name, doms in DOMAIN_ALIASES.items()
if any(d in doms for d in effective_domains)
)
else:
specialists_seeing = sorted(DOMAIN_ALIASES.keys())
store = _get_store(request)
n_builtin = max(1, min(body.n_builtin, _MAX_RESULTS_PER_BUCKET))
n_company = max(1, min(body.n_company, _MAX_RESULTS_PER_BUCKET))
n_failures = max(1, min(body.n_failures, _MAX_RESULTS_PER_BUCKET))
n_external = max(1, min(body.n_external, _MAX_RESULTS_PER_BUCKET))
def _query_collection(
collection: str, n: int, domains: list[str] | None = None
) -> list[dict[str, object]]:
try:
return store.query(
query_text=body.query,
collection=collection,
domain_filter=effective_domains if domains is None else domains,
n_results=n,
)
except Exception:
return []
# Mirror `retrieve()`'s review gate. Withheld = pending or rejected; the
# chat path drops those before a specialist ever sees them, so a panel
# that advertises itself as showing "what RAG would surface" must drop
# them too — otherwise it reports knowledge the Executive cannot use.
# Resolved once per request, via the retriever's own resolver so both
# paths read the same database. Company docs are never registered in
# `review_items`, so the company bucket is deliberately not filtered.
#
# Degrades to "nothing withheld" rather than failing, matching the
# per-collection `except` above: this is a read-only diagnostic, and a
# broken panel helps nobody. `retrieve()` is the enforcing gate and does
# NOT degrade. The exists() check matters on its own — `sqlite3.connect`
# creates the file, and a diagnostic read must not materialise a database.
def _withheld_sets() -> tuple[set[tuple[str, str]], set[str]]:
from openexecutive.memory.episodic import DB_PATH
if not DB_PATH.exists():
return set(), set()
try:
rs = _default_review_store()
return (
rs.get_withheld_keys(ContentType.BUILTIN)
| rs.get_withheld_keys(ContentType.FAILURE),
rs.get_withheld_source_ids(),
)
except Exception:
# Say so. A silently degraded mirror is worse than a loud one: an
# SME who rejects a doc, still sees it listed here, and has no
# signal will conclude rejection is broken — while chat is in fact
# withholding it correctly.
logger.warning(
"review state unreadable; /knowledge/search is showing "
"unfiltered results this request",
exc_info=True,
)
return set(), set()
_withheld_builtin, _withheld_external = _withheld_sets()
def _is_withheld(row: dict[str, object]) -> bool:
md = row.get("metadata") or {}
md_dict = md if isinstance(md, dict) else {}
source_id = md_dict.get("source_id")
if source_id:
return source_id in _withheld_external
return (md_dict.get("domain"), md_dict.get("filename")) in _withheld_builtin
builtin_hits: list[SearchHit] = []
external_hits: list[SearchHit] = []
want_builtin = "builtin" in include
want_external = "external" in include
if want_builtin or want_external:
# builtin + external share BUILTIN_COLLECTION — only metadata source_id
# distinguishes them. Single over-fetched query, then partition.
# The window must be wide enough that a dominant category (e.g. heavy
# OER ingest) doesn't crowd out the requested count in the other.
wanted_builtin = n_builtin if want_builtin else 0
wanted_external = n_external if want_external else 0
window = (wanted_builtin + wanted_external) * _BUILTIN_OVERFETCH_MULTIPLIER
raw = [
r
for r in _query_collection(ChromaDBStore.BUILTIN_COLLECTION, window)
if not _is_withheld(r)
]
for r in raw:
md = r.get("metadata") or {}
md_dict = md if isinstance(md, dict) else {}
if md_dict.get("source_id"):
if want_external and len(external_hits) < wanted_external:
external_hits.extend(_hits_from_chroma([r]))
else:
if want_builtin and len(builtin_hits) < wanted_builtin:
builtin_hits.extend(_hits_from_chroma([r]))
if (
len(builtin_hits) >= wanted_builtin
and len(external_hits) >= wanted_external
):
break
company_hits: list[SearchHit] = []
if "company" in include:
# Mirror `retrieve()` exactly: it widens the COMPANY filter with the
# `general` catch-all. Without this, the panel an operator uses to
# debug retrieval would report zero company hits for a specialist that
# does retrieve them in chat — showing the very #114 symptom the
# catch-all fixes.
company_hits = _hits_from_chroma(
_query_collection(
ChromaDBStore.COMPANY_COLLECTION,
n_company,
domains=_with_general(effective_domains),
)
)
failure_hits: list[SearchHit] = []
if "failures" in include:
failure_hits = _hits_from_chroma(
[
r
for r in _query_collection(
ChromaDBStore.FAILURES_COLLECTION, n_failures
)
if not _is_withheld(r)
]
)
return KnowledgeSearchResponse(
query=body.query,
effective_domains=effective_domains,
specialists_that_would_see_this=specialists_seeing,
builtin=builtin_hits,
company=company_hits,
failures=failure_hits,
external=external_hits,
)