Skip to content

AI-powered virtual executive team — a single coherent executive persona backed by 8 specialist agents (FastAPI + Next.js).

fix(documents): close logic-review findings on the #113/#114 fix

Rufus074679e6f0d5
diff --git a/packages/core/openexecutive/api/main.py b/packages/core/openexecutive/api/main.py
index 47cab157..c9838501 100644
--- a/packages/core/openexecutive/api/main.py
+++ b/packages/core/openexecutive/api/main.py
@@ -182,20 +182,34 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
# Repair company_docs rows left by the old upload path, which indexed
# documents under their random staging filename: those chunks match no
# DELETE and are displaced by no re-upload, so nothing else can reach
- # them. Idempotent — chunk ids are filename-derived, so a clean store
- # makes this a no-op. Best-effort, like the seeding above.
- try:
- swept, indexed = await reconcile_company_docs(
- store, settings.company_profile_path.parent / "docs"
- )
- if swept or indexed:
- logging.getLogger("openexecutive").info(
- "company_docs reconcile: dropped %d orphaned chunk(s), indexed %d document(s)",
- swept,
- indexed,
+ # them. Converges on a stable store, so it is safe to run every boot.
+ #
+ # Deliberately NOT awaited here. `ingest_file` does synchronous embedding
+ # work, and the boot that matters most — the first one after this deploy —
+ # is exactly the one where every pre-fix document needs re-embedding at
+ # once. Awaiting that would stall the lifespan before the app serves
+ # anything, failing the container healthcheck and crash-looping on
+ # precisely the installs with the most to repair. A degraded search index
+ # for a few seconds after boot is the cheaper failure.
+ #
+ # Strong ref, same reason as `_thread_rename_tasks` in discord_bot: a bare
+ # create_task is only weakly held and can be GC'd mid-flight.
+ async def _reconcile_company_docs() -> None:
+ try:
+ swept, indexed = await reconcile_company_docs(
+ store, settings.company_profile_path.parent / "docs"
)
- except Exception:
- logging.getLogger("openexecutive").exception("company_docs reconcile failed")
+ if swept or indexed:
+ logging.getLogger("openexecutive").info(
+ "company_docs reconcile: dropped %d orphaned chunk(s), indexed %d document(s)",
+ swept,
+ indexed,
+ )
+ except Exception:
+ logging.getLogger("openexecutive").exception("company_docs reconcile failed")
+
+ _reconcile_task = asyncio.create_task(_reconcile_company_docs())
+ app.state.reconcile_task = _reconcile_task
initialize_db()
initialize_alerts_db()
diff --git a/packages/core/openexecutive/api/routes/knowledge.py b/packages/core/openexecutive/api/routes/knowledge.py
index dd00766e..43c8581e 100644
--- a/packages/core/openexecutive/api/routes/knowledge.py
+++ b/packages/core/openexecutive/api/routes/knowledge.py
@@ -11,6 +11,7 @@ from openexecutive.knowledge.loader import (
BUILTIN_KNOWLEDGE_PATH,
DOMAIN_MAP,
FAILURES_KNOWLEDGE_PATH,
+ UPLOAD_DOMAINS,
)
router = APIRouter(prefix="/knowledge")
@@ -544,7 +545,7 @@ async def search_knowledge(
`retrieve()` produces. Intended for the Knowledge UI's Query mode and
for tuning the knowledge base offline.
"""
- from openexecutive.knowledge.retriever import DOMAIN_ALIASES
+ from openexecutive.knowledge.retriever import DOMAIN_ALIASES, _with_general
from openexecutive.knowledge.store import ChromaDBStore
if not body.query.strip():
@@ -555,9 +556,12 @@ async def search_knowledge(
if bad:
raise HTTPException(status_code=400, detail=f"Invalid include values: {sorted(bad)}")
+ # UPLOAD_DOMAINS, not DOMAIN_MAP: `general` is a real, uploadable company
+ # domain, and rejecting it here made this endpoint unable to introspect the
+ # documents most likely to need it — the unclassified ones.
if body.domain_filter:
for d in body.domain_filter:
- if d not in DOMAIN_MAP:
+ if d not in UPLOAD_DOMAINS:
raise HTTPException(status_code=400, detail=f"Unknown domain: {d}")
if body.specialist and body.specialist not in DOMAIN_ALIASES:
@@ -584,12 +588,14 @@ async def search_knowledge(
n_failures = max(1, min(body.n_failures, _MAX_RESULTS_PER_BUCKET))
n_external = max(1, min(body.n_external, _MAX_RESULTS_PER_BUCKET))
- def _query_collection(collection: str, n: int) -> list[dict[str, object]]:
+ def _query_collection(
+ collection: str, n: int, domains: list[str] | None = None
+ ) -> list[dict[str, object]]:
try:
return store.query(
query_text=body.query,
collection=collection,
- domain_filter=effective_domains,
+ domain_filter=effective_domains if domains is None else domains,
n_results=n,
)
except Exception:
@@ -625,8 +631,17 @@ async def search_knowledge(
company_hits: list[SearchHit] = []
if "company" in include:
+ # Mirror `retrieve()` exactly: it widens the COMPANY filter with the
+ # `general` catch-all. Without this, the panel an operator uses to
+ # debug retrieval would report zero company hits for a specialist that
+ # does retrieve them in chat — showing the very #114 symptom the
+ # catch-all fixes.
company_hits = _hits_from_chroma(
- _query_collection(ChromaDBStore.COMPANY_COLLECTION, n_company)
+ _query_collection(
+ ChromaDBStore.COMPANY_COLLECTION,
+ n_company,
+ domains=_with_general(effective_domains),
+ )
)
failure_hits: list[SearchHit] = []
diff --git a/packages/core/openexecutive/architecture/architecture-facts.yaml b/packages/core/openexecutive/architecture/architecture-facts.yaml
index 81c1d074..92d26c94 100644
--- a/packages/core/openexecutive/architecture/architecture-facts.yaml
+++ b/packages/core/openexecutive/architecture/architecture-facts.yaml
@@ -2895,13 +2895,21 @@ integrations:
upsert straight over the curated company document of that name, so
inbound content stays in its own namespace and its provenance is
visible in the ``[filename]`` citation. (``Path(...).name`` strips
- directory components first.) The domain was ``company_docs`` until
- #113/#114, which is not one of the specialist domains — no specialist
- could retrieve an attachment at all. Passing the staging tempfile path
- as the identity was the #113 bug on this path too: duplicate chunks on
- every re-send, and no filename any API call could delete (attachments
- are never written to docs/, so the boot reconcile drops their orphans
- rather than recovering them).
+ directory components first.)
+
+ The domain stays ``company_docs``, which is deliberately NOT one of the
+ specialist domains: these chunks match no domain filter and so do not
+ reach the Executive. Leaving them inert is a decision, not an oversight —
+ anyone who can attach a file in an integration channel would otherwise be
+ writing into every specialist's RAG context, and these rows have no
+ removal path (never written to docs/, so GET /documents does not list them
+ and DELETE /documents/{filename} 404s before reaching the store). Making
+ them retrievable needs its own change, with a delete path alongside it.
+
+ Passing the staging tempfile path as the identity was the #113 bug on this
+ path too: duplicate chunks on every re-send, and no filename any API call
+ could delete. The boot reconcile drops their orphans rather than
+ recovering them, there being no file on disk to recover from.
broadcast_tools: |
openexecutive.orchestrator.broadcast_tools — two new tools exposed
to the Executive alongside the existing per-Person DM tools:
diff --git a/packages/core/openexecutive/architecture/prebuilt/api.json b/packages/core/openexecutive/architecture/prebuilt/api.json
index 76dcc58d..7371b91c 100644
--- a/packages/core/openexecutive/architecture/prebuilt/api.json
+++ b/packages/core/openexecutive/architecture/prebuilt/api.json
@@ -1,7 +1,7 @@
{
"section_id": "api",
"title": "API Reference",
- "markdown": "The FastAPI surface lives in `openexecutive.api.main`, which mounts one router per topic. A shared-secret middleware gates every request via an `x-api-key` header, except `/health` and the `/webhook/*` callbacks (which carry their own verification). CORS is restricted to configured UI origins. The gate is off when `BACKEND_SHARED_SECRET` is unset — intended for local development only — so any internet-reachable deployment sets `OE_PUBLIC_DEPLOYMENT`, and `create_app()` then refuses to boot without the secret rather than serving an unauthenticated API.\n\nEndpoints grouped by router:\n\n| Router | Key endpoints | Purpose |\n|---|---|---|\n| `chat` | `POST /chat` | SSE stream of a live Executive turn. The request body optionally carries `page_context` (route, page title, guide section id, and a form descriptor with current values) — sent only by the Ask OE side panel so the Executive can explain the current page or propose form values. When the Executive calls its `propose_form_values` tool, the stream emits a `form_patch` event (`{type, form_id, fields, rationale, iteration}`) that the panel applies into the on-screen form as highlighted suggestions; the user reviews and saves through the form's normal submit path. |\n| `sessions` | `GET /sessions`, `GET /sessions/{id}/messages` | List and read saved chat sessions. |\n| `today` | `GET /today`, `GET /today/activity`, `GET /today/activity/daily` | Morning-brief aggregation, recent-action rail, and Pulse heartbeat heatmap (per-day activity counts). |\n| `people` | `GET\\|POST\\|PATCH /people`, `POST /people/{id}/archive` | Roster CRUD and scope lookups. |\n| `departments` | `GET\\|POST\\|PATCH\\|DELETE /departments`, `.../goals` | Department state and per-department goals. `PATCH /departments/{slug}` also takes `watched_entities` (≤50 names, `[]` clears) — the entities the research watch policy may add watches for and route to this department. |\n| `talent` | `GET|POST|PATCH /engagements`, `GET|POST|PATCH /candidates`, `POST /candidates/{id}/stage`, `GET /engagements/{id}/matches`, `GET /candidates/{id}/similar`, `POST /talent/reindex`, `GET|POST|PATCH /offers`, `POST /offers/{id}/{extend,decision,archive}` | Executive-search pipeline: searches, candidates, semantic matching, and the offer lifecycle (extend/decision return the offer plus approval state, scheduled/cancelled expiry nudges, side effects, and warnings; lifecycle conflicts are 409s). |\n| `agents` | `GET /agents`, `PATCH /agents/{id}`, `POST /agents/{id}/test` | Specialist metadata and Council overrides. |\n| `personas` | `GET\\|PUT\\|POST\\|DELETE /personas` | Executive persona definitions. |\n| `alerts` | `POST /alerts/{id}/ack`, `POST /alerts/bulk-ack`, `POST /alerts/{id}/reopen`, `POST /alerts/review` | Groom the briefing queue. `ack` (`read`/`ack`/`dismissed`, optional `mute_topic`) also teaches a watch-sourced alert's watch (`trust_score`); `bulk-ack` takes explicit `alert_ids` and/or `older_than_days` + `category` (exempt sources skipped); `reopen` is the Undo for a `resolved` / `expired` / `dismissed` alert (never `ack` or an artifact / decision row; back to `unread` with every review stamp cleared and its TTL anchor refreshed, so it is live again even if older than its TTL); `review` runs the Executive's relevance review on demand (ignores the interval; still honours `ALERT_REVIEW_ENABLED`, the per-pass caps, the single-flight lock and per-alert idempotency) and returns the move counts. `bulk-ack` caps ids at 500, needs `older_than_days >= 1`, and applies watch feedback per row; `mute_topic` must be one of the alert's tags or a scoped `external:` / `department:` pattern. Alerts still surface only through the briefing (`/today` reads the live view) — there is no list, prefs, mutes, or live-stream endpoint. |\n| `artifacts` | `GET /artifacts`, `POST /artifacts/{id}/archive` | Browsable research/document artifacts. |\n| `decisions` | `GET /decisions`, `POST /decisions/{id}/{approve,reject,cancel}`, `GET /audit/reliability` | Approval-gated decision instances (e.g. calendar bookings). A proposed booking is surfaced on the briefing as a companion alert; approve/reject/cancel here also clear that alert. Reviewed in the briefing — there is no separate proposals page. |\n| `audit` | `GET /audit/logs`, `GET /audit/logs/{id}`, `GET /audit/sessions/{id}`, `GET /audit/usage`, `POST /audit/log` | Durable audit trail of side effects. `/audit/sessions/{id}` adds a per-session token cost summary; `/audit/usage` aggregates token usage and OpenRouter cost across all sessions (totals, by day, by model, and by source — the part of the system that made the calls — including server-side web-search counts). |\n| `architecture` | `GET /architecture/sections`, `.../sections/{id}` | This documentation surface. |\n| `guide` | `GET /guide/sections`, `.../sections/{id}` | Plain-language user-guide content surface — mirrors `architecture` (static, version-controlled files; no LLM on the serving path). |\n| `health` | `GET /health`, `GET /health/honcho` | Liveness probes (unauthenticated). |\n| `documents` | `POST\\|GET\\|DELETE /documents`, `GET /documents/{filename}` | Upload, list, view extracted text, and delete indexed company docs. Upload is keyed on the filename, so re-uploading a document replaces its chunks rather than duplicating them, and `DELETE` removes the rows as well as the file. `domain` must be one of the eight specialist domains or `general`; anything else is a 400 rather than a 200 that indexes where no specialist can retrieve. |\n| `scheduled` | `GET\\|DELETE` scheduled actions (`?status=…&order=asc\\|desc`) | Pending and fired scheduled actions; `order` flips the `run_at` sort (asc=soonest-first for the upcoming queue, desc=most-recent-first for terminal-status history). |\n| `watchlist` | `GET\\|POST\\|PATCH\\|DELETE /watchlist`, `POST /watchlist/{slug}/approve`, `POST /watchlist/{slug}/decline` | External-monitoring watchlist + signals. Rows carry `origin` (`manual` \\| `executive` \\| `research` \\| `research_proposed`); `approve` turns a research suggestion (`research_proposed`, `dry_run`) into a live watch, `decline` `{reason: not_relevant \\| too_noisy \\| wrong_source}` removes it and records the decline (or, for `too_noisy`, keeps it live at a high floor), both 409 unless the row is a pending suggestion; `DELETE ?reason=` on a research-origin row records the decline too. |\n| `workflows` | `GET /workflows`, `POST /workflows/{name}/runs`, `GET\\|POST\\|PUT\\|DELETE /workflows/custom` | Workflow catalog, run management, and user-created (dynamic) workflow CRUD. |\n| `knowledge` | `/knowledge/builtin`, `/external`, `/search` | MBA knowledge base and external sources. |\n| `skills` | `/skills`, `/skills/search` | Skill registry CRUD and search. |\n| `evals` | `/evals/scenarios`, `/evals/runs` | Eval scenarios and run results. |\n| `memories` | `/memories/{decisions,initiatives,advice}` | Episodic-memory record management. |\n| `review` | `/review/items`, `/review/stats` | Knowledge review queue and annotations. |\n| `onboarding` | `/onboard/interview/start`, `/onboard/interview/message`, `/onboard/interview/draft`, `/onboard/interview/{id}`, `/onboard/interview/commit` | Conversational company setup. The `onboarding_interviewer` agent asks clarifying questions, then emits a draft profile + people + departments. **Nothing is persisted until `POST /onboard/interview/commit`** — the interview itself never writes. `start` is multipart (description + optional deck/one-pager uploads). |\n| `onboarding` (fallback) | `/onboard/start`, `/onboard/answer`, `/onboard/status/{id}` | The original 12-step form wizard, kept as a fallback (UI at `/onboard?mode=form`) and used by the `openexecutive onboard` CLI. |\n| `company-profile` | `GET\\|PATCH /company-profile` | Structured company profile. |\n| `auth` | `GET /auth/allowed-emails` | Allowed sign-in emails. |\n| `fixtures` | `/fixtures`, `/fixtures/{name}/load` | Demo-company fixture load/reset/snapshot. |\n| `clients` | `/clients`, `/clients/generate`, `/clients/cockpit`, `/clients/save`, `/clients/{slug}/activate` | Client-company slots — multi-client mode for fractional use; create from current/blank/AI-generated intake draft (pasted notes and/or file attachments, multipart on `/clients/generate`), save-back switching, engagement metadata (PATCH), cross-client cockpit, delete. See the Client Companies section. |\n| webhooks | `POST /webhook/telegram`, `POST /webhook/google-chat` | Inbound integration callbacks (self-verified). |\n| MCP server | `POST\\|GET /mcp` | Open Executive exposed as an MCP server (Streamable-HTTP). Not a router — a mounted FastMCP sub-app. Same shared-secret gate (`x-api-key`); exposes company context as resources and the specialist council as tools. See the MCP Server section. |",
+ "markdown": "The FastAPI surface lives in `openexecutive.api.main`, which mounts one router per topic. A shared-secret middleware gates every request via an `x-api-key` header, except `/health` and the `/webhook/*` callbacks (which carry their own verification). CORS is restricted to configured UI origins. The gate is off when `BACKEND_SHARED_SECRET` is unset — intended for local development only — so any internet-reachable deployment sets `OE_PUBLIC_DEPLOYMENT`, and `create_app()` then refuses to boot without the secret rather than serving an unauthenticated API.\n\nEndpoints grouped by router:\n\n| Router | Key endpoints | Purpose |\n|---|---|---|\n| `chat` | `POST /chat` | SSE stream of a live Executive turn. The request body optionally carries `page_context` (route, page title, guide section id, and a form descriptor with current values) — sent only by the Ask OE side panel so the Executive can explain the current page or propose form values. When the Executive calls its `propose_form_values` tool, the stream emits a `form_patch` event (`{type, form_id, fields, rationale, iteration}`) that the panel applies into the on-screen form as highlighted suggestions; the user reviews and saves through the form's normal submit path. |\n| `sessions` | `GET /sessions`, `GET /sessions/{id}/messages` | List and read saved chat sessions. |\n| `today` | `GET /today`, `GET /today/activity`, `GET /today/activity/daily` | Morning-brief aggregation, recent-action rail, and Pulse heartbeat heatmap (per-day activity counts). |\n| `people` | `GET\\|POST\\|PATCH /people`, `POST /people/{id}/archive` | Roster CRUD and scope lookups. |\n| `departments` | `GET\\|POST\\|PATCH\\|DELETE /departments`, `.../goals` | Department state and per-department goals. `PATCH /departments/{slug}` also takes `watched_entities` (≤50 names, `[]` clears) — the entities the research watch policy may add watches for and route to this department. |\n| `talent` | `GET|POST|PATCH /engagements`, `GET|POST|PATCH /candidates`, `POST /candidates/{id}/stage`, `GET /engagements/{id}/matches`, `GET /candidates/{id}/similar`, `POST /talent/reindex`, `GET|POST|PATCH /offers`, `POST /offers/{id}/{extend,decision,archive}` | Executive-search pipeline: searches, candidates, semantic matching, and the offer lifecycle (extend/decision return the offer plus approval state, scheduled/cancelled expiry nudges, side effects, and warnings; lifecycle conflicts are 409s). |\n| `agents` | `GET /agents`, `PATCH /agents/{id}`, `POST /agents/{id}/test` | Specialist metadata and Council overrides. |\n| `personas` | `GET\\|PUT\\|POST\\|DELETE /personas` | Executive persona definitions. |\n| `alerts` | `POST /alerts/{id}/ack`, `POST /alerts/bulk-ack`, `POST /alerts/{id}/reopen`, `POST /alerts/review` | Groom the briefing queue. `ack` (`read`/`ack`/`dismissed`, optional `mute_topic`) also teaches a watch-sourced alert's watch (`trust_score`); `bulk-ack` takes explicit `alert_ids` and/or `older_than_days` + `category` (exempt sources skipped); `reopen` is the Undo for a `resolved` / `expired` / `dismissed` alert (never `ack` or an artifact / decision row; back to `unread` with every review stamp cleared and its TTL anchor refreshed, so it is live again even if older than its TTL); `review` runs the Executive's relevance review on demand (ignores the interval; still honours `ALERT_REVIEW_ENABLED`, the per-pass caps, the single-flight lock and per-alert idempotency) and returns the move counts. `bulk-ack` caps ids at 500, needs `older_than_days >= 1`, and applies watch feedback per row; `mute_topic` must be one of the alert's tags or a scoped `external:` / `department:` pattern. Alerts still surface only through the briefing (`/today` reads the live view) — there is no list, prefs, mutes, or live-stream endpoint. |\n| `artifacts` | `GET /artifacts`, `POST /artifacts/{id}/archive` | Browsable research/document artifacts. |\n| `decisions` | `GET /decisions`, `POST /decisions/{id}/{approve,reject,cancel}`, `GET /audit/reliability` | Approval-gated decision instances (e.g. calendar bookings). A proposed booking is surfaced on the briefing as a companion alert; approve/reject/cancel here also clear that alert. Reviewed in the briefing — there is no separate proposals page. |\n| `audit` | `GET /audit/logs`, `GET /audit/logs/{id}`, `GET /audit/sessions/{id}`, `GET /audit/usage`, `POST /audit/log` | Durable audit trail of side effects. `/audit/sessions/{id}` adds a per-session token cost summary; `/audit/usage` aggregates token usage and OpenRouter cost across all sessions (totals, by day, by model, and by source — the part of the system that made the calls — including server-side web-search counts). |\n| `architecture` | `GET /architecture/sections`, `.../sections/{id}` | This documentation surface. |\n| `guide` | `GET /guide/sections`, `.../sections/{id}` | Plain-language user-guide content surface — mirrors `architecture` (static, version-controlled files; no LLM on the serving path). |\n| `health` | `GET /health`, `GET /health/honcho` | Liveness probes (unauthenticated). |\n| `documents` | `POST\\|GET\\|DELETE /documents`, `GET /documents/{filename}` | Upload, list, view extracted text, and delete indexed company docs. Upload is keyed on the filename, so re-uploading a document replaces its chunks rather than duplicating them, and `DELETE` removes the rows as well as the file. `domain` must be one of the eight specialist domains or `general`; anything else is a 400 rather than a 200 that indexes where no specialist can retrieve. |\n| `scheduled` | `GET\\|DELETE` scheduled actions (`?status=…&order=asc\\|desc`) | Pending and fired scheduled actions; `order` flips the `run_at` sort (asc=soonest-first for the upcoming queue, desc=most-recent-first for terminal-status history). |\n| `watchlist` | `GET\\|POST\\|PATCH\\|DELETE /watchlist`, `POST /watchlist/{slug}/approve`, `POST /watchlist/{slug}/decline` | External-monitoring watchlist + signals. Rows carry `origin` (`manual` \\| `executive` \\| `research` \\| `research_proposed`); `approve` turns a research suggestion (`research_proposed`, `dry_run`) into a live watch, `decline` `{reason: not_relevant \\| too_noisy \\| wrong_source}` removes it and records the decline (or, for `too_noisy`, keeps it live at a high floor), both 409 unless the row is a pending suggestion; `DELETE ?reason=` on a research-origin row records the decline too. |\n| `workflows` | `GET /workflows`, `POST /workflows/{name}/runs`, `GET\\|POST\\|PUT\\|DELETE /workflows/custom` | Workflow catalog, run management, and user-created (dynamic) workflow CRUD. |\n| `knowledge` | `/knowledge/builtin`, `/external`, `/search` | MBA knowledge base and external sources. `/search` is the introspection parallel of `retriever.retrieve` and mirrors its filters — including the `general` catch-all on the company collection — so the panel cannot report a specialist seeing nothing that it does retrieve in chat. It accepts `general` in `domain_filter` for the same reason. |\n| `skills` | `/skills`, `/skills/search` | Skill registry CRUD and search. |\n| `evals` | `/evals/scenarios`, `/evals/runs` | Eval scenarios and run results. |\n| `memories` | `/memories/{decisions,initiatives,advice}` | Episodic-memory record management. |\n| `review` | `/review/items`, `/review/stats` | Knowledge review queue and annotations. |\n| `onboarding` | `/onboard/interview/start`, `/onboard/interview/message`, `/onboard/interview/draft`, `/onboard/interview/{id}`, `/onboard/interview/commit` | Conversational company setup. The `onboarding_interviewer` agent asks clarifying questions, then emits a draft profile + people + departments. **Nothing is persisted until `POST /onboard/interview/commit`** — the interview itself never writes. `start` is multipart (description + optional deck/one-pager uploads). |\n| `onboarding` (fallback) | `/onboard/start`, `/onboard/answer`, `/onboard/status/{id}` | The original 12-step form wizard, kept as a fallback (UI at `/onboard?mode=form`) and used by the `openexecutive onboard` CLI. |\n| `company-profile` | `GET\\|PATCH /company-profile` | Structured company profile. |\n| `auth` | `GET /auth/allowed-emails` | Allowed sign-in emails. |\n| `fixtures` | `/fixtures`, `/fixtures/{name}/load` | Demo-company fixture load/reset/snapshot. |\n| `clients` | `/clients`, `/clients/generate`, `/clients/cockpit`, `/clients/save`, `/clients/{slug}/activate` | Client-company slots — multi-client mode for fractional use; create from current/blank/AI-generated intake draft (pasted notes and/or file attachments, multipart on `/clients/generate`), save-back switching, engagement metadata (PATCH), cross-client cockpit, delete. See the Client Companies section. |\n| webhooks | `POST /webhook/telegram`, `POST /webhook/google-chat` | Inbound integration callbacks (self-verified). |\n| MCP server | `POST\\|GET /mcp` | Open Executive exposed as an MCP server (Streamable-HTTP). Not a router — a mounted FastMCP sub-app. Same shared-secret gate (`x-api-key`); exposes company context as resources and the specialist council as tools. See the MCP Server section. |",
"mermaid": null,
"generated_at": "2026-09-16T10:00:04.693907+00:00"
}
diff --git a/packages/core/openexecutive/architecture/prebuilt/integrations.json b/packages/core/openexecutive/architecture/prebuilt/integrations.json
index e43621d3..eebe4281 100644
--- a/packages/core/openexecutive/architecture/prebuilt/integrations.json
+++ b/packages/core/openexecutive/architecture/prebuilt/integrations.json
@@ -1,7 +1,7 @@
{
"section_id": "integrations",
"title": "Integrations",
- "markdown": "External channels connect callers to the Executive. Each inbound adapter resolves the sender to a Person via `find_person_by_{slack,discord,telegram,email}_id` and threads `person_id` into `Executive.chat()`.\n\n### Inbound channels\n\n- **Slack** (`integrations.slack_bot`) — Bolt socket-mode; responds to `@mention`s, DMs, and thread continuations.\n- **Discord** (`integrations.discord_bot`) — discord.py gateway embedded in the lifespan; per-thread auto-titling, session-scoped memory, author attribution, configurable `@mention` thread promotion.\n- **Telegram** — webhook gating + per-chat history.\n- **Google Chat** — webhook + service-account auth.\n- **Email** (`integrations.email_poller`) — IMAP poll loop; strips Reply-To, filters auto-replies.\n\nSlack, Discord, and Telegram are roster-gated on inbound: an unrostered sender's message is silently dropped (`details.outcome=\"rejected_unknown_sender\"`). Email is not inbound-gated — non-roster mail flows through with a `[POLICY]` notice so triage can classify it. Slack/Discord thread continuations pass the shared `integrations.response_gate` (utility-fast model, biased toward replying).\n\n### Attachments\n\n`integrations.attachments` handles web chat, Discord, and Telegram. Images become Anthropic vision blocks; documents (PDF/DOCX/TXT/MD/CSV) are inlined and indexed into the ChromaDB `company_docs` collection for future RAG. 20 MB/file cap; web chat uses `POST /chat/upload`.\n\nIndexed chunks are keyed on `attachment:<bare filename>`, under the `general` domain. Both parts matter. The key is the chunk-id namespace and the retrieval citation, and an attachment's name is chosen by whoever sent the message — unprefixed, an emailed `strategy-2026.md` would upsert straight over the curated company document of that name, so inbound content stays in its own namespace and its provenance shows up in the `[filename]` citation. The domain was previously `company_docs`, which is not one of the specialist domains, so no specialist could retrieve an attachment at all; `general` is the catch-all every specialist's company filter includes.\n\n### MCP gateway & outbound\n\nMCP runs in **both directions**. As a *client*, `orchestrator.mcp_gateway.MCPGateway` proxies remote MCP servers (Gmail, Calendar, GitHub) as Executive tools, with egress gating on Gmail recipients, Calendar attendees, and Drive share grantees against the roster (and a refusal of public / whole-domain Drive sharing). The Google Workspace server (`workspace-mcp`) behind those Gmail/Calendar/Drive tools runs **co-located inside the API** as a stdio child of the gateway — fitting the single-tenant model (one install = one company = one exec Google account), so there's no separate service to operate. It's baked into the API image and launched by `docker/workspace-mcp-launch.sh`, which selects single-user OAuth (token persisted at `/data/google_credentials`) or service-account domain-wide delegation from `GWORKSPACE_AUTH_MODE`. The gateway forwards the Google secrets to the child via `_FORWARDED_ENV_VARS`. As a *server*, `openexecutive.mcp_server` exposes OE's own company context and specialist council to external MCP clients at `/mcp` — the inverse direction; see the dedicated **MCP Server** section. The gateway launches extensible-mcp as a `uvx` subprocess whose fastembed tool-search index uses an embedding model baked into the API image and loaded offline, so it no longer re-downloads that model from the Hugging Face Hub on every cold start. Because the MCP stdio client does not inherit the parent environment, `MCPGateway.start()` explicitly forwards the embedding-cache/offline env vars (`FASTEMBED_CACHE_PATH`, `HF_HUB_OFFLINE`, …) into the child. Outbound tools: per-Person DMs (`send_slack_dm`, `send_discord_dm`, `send_telegram_message`) plus room-targeted `send_department_message` and `send_company_broadcast`. The per-Person DMs are additionally rate-limited and de-duplicated by the outbound anti-spam guard (below).\n\n### Outbound anti-spam guard\n\nEvery per-Person DM passes a deterministic guard (`orchestrator.outbound_guard.check_outbound_allowed`) after the roster gate and recipient-id recovery but before the network send — the single chokepoint **all** proactive DM paths reach (ad-hoc chat sends, the scheduler's followup/nudge dispatch, principal-brief delivery, the reflection turn, and research-finding routing). On a violation it returns a human-readable reason and the handler **suppresses** the send (`{\"status\":\"suppressed\",\"reason\":…}`), writes a `tool_invocation` audit row with `suppressed=true`, and writes **no** `done` activity row — so a blocked attempt never counts toward its own rate cap, and the Executive can read the reason and hold or `schedule_followup` for the recipient's next window. Three controls: **content dedup** (a near-identical message — whitespace/case-normalized, first 160 chars — already sent to the same recipient within `OUTBOUND_DEDUP_WINDOW_MINUTES`, default 360); a **per-recipient rate cap** (`OUTBOUND_MAX_PER_RECIPIENT_PER_WINDOW`, default 5, over `OUTBOUND_RATE_WINDOW_MINUTES`, default 60); and **quiet hours / availability** (`OUTBOUND_RESPECT_QUIET_HOURS`, default on — refuse while the recipient is on leave or outside their `people.channel` availability windows, surfacing the next open window). Delivery history is read from the existing `status='done'` `scheduled_actions` rows (`episodic.recent_sends_for_channel_ref`), so there is no new table. The guard is **fail-open**: any internal lookup error returns \"allow\", so it can never turn a working send into a crash or a silently-dropped message. Principal briefs and the internal heartbeat kinds use the `__internal__` channel and never reach these handlers, so they are unaffected.\n\n### Calendar & Google Meet\n\nThe Executive books meetings through two typed tools over the Calendar MCP. `create_calendar_event` schedules a **future** meeting (propose→approve by default, or auto-execute once that decision class is promoted) and may be used proactively when a stalled goal warrants a sync. `create_instant_meeting` spins up an **impromptu call starting now**, auto-executing without the approval step and skipping the business-hours/horizon caps, so the Executive can hand back a join link immediately. Both attach a **Google Meet** link by default (`add_google_meet`; opt out for in-person), parsed from the created event's `conferenceData` and surfaced to the caller plus persisted on the decision ledger. Either way the caps that matter still hold — roster membership, principal-protection, max attendees, and a daily booking ceiling.\n\n**Around-the-meeting participation.** The Executive doesn't join the call as a live voice; instead, on every successful booking it schedules a best-effort post-meeting follow-up (~5 min after the end) that DMs a human attendee asking for a quick recap — key decisions and action items. The reply flows back through the same cross-session DM context linkage described below, so the Executive can act on it (summarize, follow up) in a later turn. No meeting-transcript or Drive integration is required.\n\n### Cross-session DM context\n\nWhen the principal says \"DM Alex about X\" in one session, OE sends the DM and — if a live conversation is active — records an `outbound_context` linkage row (recipient, full outbound text, originating session id). Email is covered too but recorded one layer deeper — there is no `send_email` tool, so the linkage is written in `orchestrator.mcp_gateway` after a successful `google_workspace__send_gmail_message`, one row per `to`/`cc` recipient keyed by the bare lowercased address (OE's own address skipped). When Alex later replies on the 1:1 DM path — or by email — the shared helper `integrations.inbound_hydration.hydrate_user_message` (called by the Discord, Telegram, and Slack handlers, and by the email poller keyed on the sender address) looks up the most-recent `open` linkage for that sender within 72h and prepends an `<outbound_reply_context>` block — the outbound text plus a backstory excerpt from the originating session — to the message handed to `Executive.chat()`. So OE knows what the reply is about instead of starting from an empty per-person session. The block lands in the user turn (caching rule) and is injected into a copy only — the pristine reply is persisted to history. The linkage is one-shot (consumed on first matched reply, race-safe) so an unrelated later DM isn't re-hydrated. This is deterministic and independent of the per-person `peer_memory` (Honcho) layer.",
+ "markdown": "External channels connect callers to the Executive. Each inbound adapter resolves the sender to a Person via `find_person_by_{slack,discord,telegram,email}_id` and threads `person_id` into `Executive.chat()`.\n\n### Inbound channels\n\n- **Slack** (`integrations.slack_bot`) — Bolt socket-mode; responds to `@mention`s, DMs, and thread continuations.\n- **Discord** (`integrations.discord_bot`) — discord.py gateway embedded in the lifespan; per-thread auto-titling, session-scoped memory, author attribution, configurable `@mention` thread promotion.\n- **Telegram** — webhook gating + per-chat history.\n- **Google Chat** — webhook + service-account auth.\n- **Email** (`integrations.email_poller`) — IMAP poll loop; strips Reply-To, filters auto-replies.\n\nSlack, Discord, and Telegram are roster-gated on inbound: an unrostered sender's message is silently dropped (`details.outcome=\"rejected_unknown_sender\"`). Email is not inbound-gated — non-roster mail flows through with a `[POLICY]` notice so triage can classify it. Slack/Discord thread continuations pass the shared `integrations.response_gate` (utility-fast model, biased toward replying).\n\n### Attachments\n\n`integrations.attachments` handles web chat, Discord, and Telegram. Images become Anthropic vision blocks; documents (PDF/DOCX/TXT/MD/CSV) are inlined and indexed into the ChromaDB `company_docs` collection for future RAG. 20 MB/file cap; web chat uses `POST /chat/upload`.\n\nIndexed chunks are keyed on `attachment:<bare filename>`. An attachment's name is chosen by whoever sent the message, and the name is the chunk-id namespace — unprefixed, an emailed `strategy-2026.md` would upsert straight over the curated company document of that name. The prefix keeps inbound content in its own namespace and makes its provenance visible in the `[filename]` retrieval citation.\n\nThey carry the domain `company_docs`, which is deliberately **not** one of the specialist domains: the chunks match no domain filter, so they do not currently reach the Executive. Anyone who can attach a file in an integration channel would otherwise be writing into every specialist's context, and these rows have no removal path — they are never written to `company/docs/`, so `GET /documents` does not list them and `DELETE /documents/{filename}` 404s before reaching the store. Making them retrievable is a trust-boundary change that needs a delete path alongside it.\n\n### MCP gateway & outbound\n\nMCP runs in **both directions**. As a *client*, `orchestrator.mcp_gateway.MCPGateway` proxies remote MCP servers (Gmail, Calendar, GitHub) as Executive tools, with egress gating on Gmail recipients, Calendar attendees, and Drive share grantees against the roster (and a refusal of public / whole-domain Drive sharing). The Google Workspace server (`workspace-mcp`) behind those Gmail/Calendar/Drive tools runs **co-located inside the API** as a stdio child of the gateway — fitting the single-tenant model (one install = one company = one exec Google account), so there's no separate service to operate. It's baked into the API image and launched by `docker/workspace-mcp-launch.sh`, which selects single-user OAuth (token persisted at `/data/google_credentials`) or service-account domain-wide delegation from `GWORKSPACE_AUTH_MODE`. The gateway forwards the Google secrets to the child via `_FORWARDED_ENV_VARS`. As a *server*, `openexecutive.mcp_server` exposes OE's own company context and specialist council to external MCP clients at `/mcp` — the inverse direction; see the dedicated **MCP Server** section. The gateway launches extensible-mcp as a `uvx` subprocess whose fastembed tool-search index uses an embedding model baked into the API image and loaded offline, so it no longer re-downloads that model from the Hugging Face Hub on every cold start. Because the MCP stdio client does not inherit the parent environment, `MCPGateway.start()` explicitly forwards the embedding-cache/offline env vars (`FASTEMBED_CACHE_PATH`, `HF_HUB_OFFLINE`, …) into the child. Outbound tools: per-Person DMs (`send_slack_dm`, `send_discord_dm`, `send_telegram_message`) plus room-targeted `send_department_message` and `send_company_broadcast`. The per-Person DMs are additionally rate-limited and de-duplicated by the outbound anti-spam guard (below).\n\n### Outbound anti-spam guard\n\nEvery per-Person DM passes a deterministic guard (`orchestrator.outbound_guard.check_outbound_allowed`) after the roster gate and recipient-id recovery but before the network send — the single chokepoint **all** proactive DM paths reach (ad-hoc chat sends, the scheduler's followup/nudge dispatch, principal-brief delivery, the reflection turn, and research-finding routing). On a violation it returns a human-readable reason and the handler **suppresses** the send (`{\"status\":\"suppressed\",\"reason\":…}`), writes a `tool_invocation` audit row with `suppressed=true`, and writes **no** `done` activity row — so a blocked attempt never counts toward its own rate cap, and the Executive can read the reason and hold or `schedule_followup` for the recipient's next window. Three controls: **content dedup** (a near-identical message — whitespace/case-normalized, first 160 chars — already sent to the same recipient within `OUTBOUND_DEDUP_WINDOW_MINUTES`, default 360); a **per-recipient rate cap** (`OUTBOUND_MAX_PER_RECIPIENT_PER_WINDOW`, default 5, over `OUTBOUND_RATE_WINDOW_MINUTES`, default 60); and **quiet hours / availability** (`OUTBOUND_RESPECT_QUIET_HOURS`, default on — refuse while the recipient is on leave or outside their `people.channel` availability windows, surfacing the next open window). Delivery history is read from the existing `status='done'` `scheduled_actions` rows (`episodic.recent_sends_for_channel_ref`), so there is no new table. The guard is **fail-open**: any internal lookup error returns \"allow\", so it can never turn a working send into a crash or a silently-dropped message. Principal briefs and the internal heartbeat kinds use the `__internal__` channel and never reach these handlers, so they are unaffected.\n\n### Calendar & Google Meet\n\nThe Executive books meetings through two typed tools over the Calendar MCP. `create_calendar_event` schedules a **future** meeting (propose→approve by default, or auto-execute once that decision class is promoted) and may be used proactively when a stalled goal warrants a sync. `create_instant_meeting` spins up an **impromptu call starting now**, auto-executing without the approval step and skipping the business-hours/horizon caps, so the Executive can hand back a join link immediately. Both attach a **Google Meet** link by default (`add_google_meet`; opt out for in-person), parsed from the created event's `conferenceData` and surfaced to the caller plus persisted on the decision ledger. Either way the caps that matter still hold — roster membership, principal-protection, max attendees, and a daily booking ceiling.\n\n**Around-the-meeting participation.** The Executive doesn't join the call as a live voice; instead, on every successful booking it schedules a best-effort post-meeting follow-up (~5 min after the end) that DMs a human attendee asking for a quick recap — key decisions and action items. The reply flows back through the same cross-session DM context linkage described below, so the Executive can act on it (summarize, follow up) in a later turn. No meeting-transcript or Drive integration is required.\n\n### Cross-session DM context\n\nWhen the principal says \"DM Alex about X\" in one session, OE sends the DM and — if a live conversation is active — records an `outbound_context` linkage row (recipient, full outbound text, originating session id). Email is covered too but recorded one layer deeper — there is no `send_email` tool, so the linkage is written in `orchestrator.mcp_gateway` after a successful `google_workspace__send_gmail_message`, one row per `to`/`cc` recipient keyed by the bare lowercased address (OE's own address skipped). When Alex later replies on the 1:1 DM path — or by email — the shared helper `integrations.inbound_hydration.hydrate_user_message` (called by the Discord, Telegram, and Slack handlers, and by the email poller keyed on the sender address) looks up the most-recent `open` linkage for that sender within 72h and prepends an `<outbound_reply_context>` block — the outbound text plus a backstory excerpt from the originating session — to the message handed to `Executive.chat()`. So OE knows what the reply is about instead of starting from an empty per-person session. The block lands in the user turn (caching rule) and is injected into a copy only — the pristine reply is persisted to history. The linkage is one-shot (consumed on first matched reply, race-safe) so an unrelated later DM isn't re-hydrated. This is deterministic and independent of the per-person `peer_memory` (Honcho) layer.",
"mermaid": "flowchart TD\n Channels[\"Slack / Discord<br/>Telegram / GChat / Email\"]\n Resolve[\"find_person_by_*_id\"]\n Gate[\"roster gate\"]\n Drop[\"drop unrostered\"]\n Exec[\"Executive.chat()\"]\n Attach[\"attachments<br/>(vision / company_docs)\"]\n MCP[\"MCP gateway\"]\n Guard[\"outbound anti-spam guard<br/>(dedup / rate cap / quiet hours)\"]\n Out[\"DM / department<br/>/ broadcast tools\"]\n Channels --> Resolve\n Resolve --> Gate\n Gate -- \"unknown sender\" --> Drop\n Gate -- \"Person\" --> Exec\n Channels --> Attach\n Attach --> Exec\n Exec --> MCP\n Exec --> Guard\n Guard -- \"suppress\" --> Drop\n Guard -- \"allow\" --> Out\n Out --> Channels\n class Channels entry\n class Resolve compute\n class Gate compute\n class Drop external\n class Exec hot\n class Attach compute\n class MCP external\n class Guard compute\n class Out compute",
"generated_at": "2026-09-16T10:00:04.693907+00:00"
}
diff --git a/packages/core/openexecutive/architecture/prebuilt/rag.json b/packages/core/openexecutive/architecture/prebuilt/rag.json
index 48f5b108..b23e9b3b 100644
--- a/packages/core/openexecutive/architecture/prebuilt/rag.json
+++ b/packages/core/openexecutive/architecture/prebuilt/rag.json
@@ -1,7 +1,7 @@
{
"section_id": "rag",
"title": "Knowledge & RAG",
- "markdown": "Retrieval is handled by `openexecutive.knowledge.retriever.retrieve`, querying ChromaDB collections through `openexecutive.knowledge.store.ChromaDBStore`. Five collections back the system:\n\n- **BUILTIN** — curated MBA-style domain knowledge shipped with the repo.\n- **COMPANY** — customer documents ingested via the `/documents` endpoint. Curated uploads only — synced Notion pages do not land here. A document's identity is its **filename**: chunk metadata and the chunk id both derive from it, so re-uploading upserts in place rather than duplicating, and `DELETE /documents/{filename}` matches the rows it claims to remove. Chat/email attachments share the collection but are namespaced `attachment:<name>`, since the sender picks that name and an unprefixed one would overwrite the curated document it collides with.\n- **NOTION_WIKI** — optional Notion sync (`NOTION_SYNC_ENABLED`) incrementally re-indexes pages shared with a Notion internal integration into a separate `notion_wiki` collection (`type=notion`). A Notion share is inherited and multi-writer, so this text is unreviewed relative to curated company docs; the retriever labels it as such and ranks it below COMPANY. Un-shared pages are purged on the next tick.\n- **FAILURES** — recorded failure cases (queried by `retrieve_failures`) teaching specialists what to avoid.\n- **RECENT_RESEARCH** — web-research artifacts persisted from `executive_research` runs, kept separate from COMPANY so unvetted machine-generated text never blends into curated knowledge, and ranked below company documents.\n\nEach parallel specialist call runs its own domain-filtered query. When no explicit filter is passed, `DOMAIN_ALIASES` maps the specialist key to its domains — e.g. `cso`→`strategy`, `cpo`→`product`,`strategy`, `board_comms`→`board`,`finance`. Hits beyond a cosine distance threshold are dropped, and very short messages (greetings) bypass retrieval entirely.\n\n`POST /documents` rejects any domain outside that set with a 400 — an unvalidated string would index the document under a domain no specialist filters on, failing silently and only at retrieval time. The ninth accepted value, `general`, is the catch-all for an upload nobody classified (it is the default on the endpoint and in the UI's picker). It maps to no specialist, so every specialist's **company** filter is widened to include it — an unclassified document is visible to all rather than to none. That widening is deliberately not applied to BUILTIN, whose rows are shared with external OER sources that fall back to `general` when they declare no domains; fanning those into every specialist would blend unvetted third-party material into every answer.\n\nMulti-domain chunks fan out to one row per declared domain, so `_dedupe_by_text` drops duplicate-text hits while preserving relevance order before results are returned.\n\nThe formatted context is injected into the **user turn** (wrapped in `<relevant_knowledge>` by `BaseAgent.analyze`), never into a cached system block — per-query hits would otherwise poison the prompt cache.",
+ "markdown": "Retrieval is handled by `openexecutive.knowledge.retriever.retrieve`, querying ChromaDB collections through `openexecutive.knowledge.store.ChromaDBStore`. Five collections back the system:\n\n- **BUILTIN** — curated MBA-style domain knowledge shipped with the repo.\n- **COMPANY** — customer documents ingested via the `/documents` endpoint. Curated uploads only — synced Notion pages do not land here. A document's identity is its **filename**: chunk metadata and the chunk id both derive from it, so re-uploading upserts in place rather than duplicating, and `DELETE /documents/{filename}` matches the rows it claims to remove. Chat/email attachments share the collection but are namespaced `attachment:<name>`, since the sender picks that name and an unprefixed one would overwrite the curated document it collides with; they also carry a non-specialist domain, so they are not retrieved (see Integrations).\n- **NOTION_WIKI** — optional Notion sync (`NOTION_SYNC_ENABLED`) incrementally re-indexes pages shared with a Notion internal integration into a separate `notion_wiki` collection (`type=notion`). A Notion share is inherited and multi-writer, so this text is unreviewed relative to curated company docs; the retriever labels it as such and ranks it below COMPANY. Un-shared pages are purged on the next tick.\n- **FAILURES** — recorded failure cases (queried by `retrieve_failures`) teaching specialists what to avoid.\n- **RECENT_RESEARCH** — web-research artifacts persisted from `executive_research` runs, kept separate from COMPANY so unvetted machine-generated text never blends into curated knowledge, and ranked below company documents.\n\nEach parallel specialist call runs its own domain-filtered query. When no explicit filter is passed, `DOMAIN_ALIASES` maps the specialist key to its domains — e.g. `cso`→`strategy`, `cpo`→`product`,`strategy`, `board_comms`→`board`,`finance`. Hits beyond a cosine distance threshold are dropped, and very short messages (greetings) bypass retrieval entirely.\n\n`POST /documents` rejects any domain outside that set with a 400 — an unvalidated string would index the document under a domain no specialist filters on, failing silently and only at retrieval time. The ninth accepted value, `general`, is the catch-all for an upload nobody classified (it is the default on the endpoint and in the UI's picker). It maps to no specialist, so every specialist's **company** filter is widened to include it — an unclassified document is visible to all rather than to none. That widening is deliberately not applied to BUILTIN, whose rows are shared with external OER sources that fall back to `general` when they declare no domains; fanning those into every specialist would blend unvetted third-party material into every answer.\n\nMulti-domain chunks fan out to one row per declared domain, so `_dedupe_by_text` drops duplicate-text hits while preserving relevance order before results are returned.\n\nThe formatted context is injected into the **user turn** (wrapped in `<relevant_knowledge>` by `BaseAgent.analyze`), never into a cached system block — per-query hits would otherwise poison the prompt cache.",
"mermaid": "flowchart TD\n Call[\"Specialist call<br/>(domain-filtered)\"] --> Q[\"retrieve(query, domain)\"]\n Q --> Builtin[(BUILTIN)]\n Q --> Company[(COMPANY)]\n Q --> NotionWiki[(NOTION_WIKI)]\n Q --> Failures[(FAILURES)]\n Q --> Research[(RECENT_RESEARCH)]\n Builtin --> Dedupe[\"dedupe by text<br/>+ distance gate\"]\n Company --> Dedupe\n NotionWiki --> Dedupe\n Failures --> Dedupe\n Research --> Dedupe\n Dedupe --> Inject[\"Inject into USER turn<br/>(not cached)\"]\n class Call entry\n class Q compute\n class Dedupe compute\n class Builtin storage\n class Company storage\n class NotionWiki storage\n class Failures storage\n class Research storage\n class Inject hot",
"generated_at": "2026-09-16T10:00:04.693907+00:00"
}
diff --git a/packages/core/openexecutive/integrations/attachments.py b/packages/core/openexecutive/integrations/attachments.py
index f939b2e8..8be9a443 100644
--- a/packages/core/openexecutive/integrations/attachments.py
+++ b/packages/core/openexecutive/integrations/attachments.py
@@ -158,7 +158,7 @@ def _schedule_ingest(data: bytes, filename: str) -> None:
async def _run() -> None:
suffix = _suffix_from_filename(filename)
try:
- from openexecutive.knowledge.loader import GENERAL_DOMAIN, ingest_file
+ from openexecutive.knowledge.loader import ingest_file
from openexecutive.knowledge.store import ChromaDBStore
store = ChromaDBStore()
@@ -180,13 +180,22 @@ def _schedule_ingest(data: bytes, filename: str) -> None:
# provenance visible in the `[filename]` retrieval citation —
# the same reason Notion and research artifacts are isolated.
#
- # `domain` is the catch-all rather than the old "company_docs"
- # — that was not one of the specialist domains, so every
- # attachment landed where no specialist could retrieve it.
+ # `domain` stays "company_docs", which is NOT one of the
+ # specialist domains — so these chunks match no specialist's
+ # domain filter and never reach the Executive's context. That
+ # is a known gap, left deliberately: anyone who can attach a
+ # file in an integration channel would otherwise be writing
+ # into every specialist's RAG context, and these rows have no
+ # removal path at all (they are never written to
+ # `company/docs/`, so `GET /documents` does not list them and
+ # `DELETE /documents/{filename}` 404s before reaching the
+ # store). Making them retrievable is a trust-boundary decision
+ # that needs its own change, with a delete path alongside it —
+ # not a side effect of fixing the chunk-id bug below.
count = await ingest_file(
tmp_path,
store,
- domain=GENERAL_DOMAIN,
+ domain="company_docs",
source_name=f"attachment:{Path(filename).name}",
)
logger.info(
diff --git a/packages/core/openexecutive/knowledge/loader.py b/packages/core/openexecutive/knowledge/loader.py
index 37e2746a..426c4728 100644
--- a/packages/core/openexecutive/knowledge/loader.py
+++ b/packages/core/openexecutive/knowledge/loader.py
@@ -332,20 +332,42 @@ async def reconcile_company_docs(
) -> tuple[int, int]:
"""Sweep temp-named orphans, then index documents that have no chunks.
- Returns ``(orphans_deleted, files_indexed)``. Safe to run on every boot.
+ Returns ``(orphans_deleted, files_indexed)``. Converges on a stable store,
+ so it is safe to run on every boot.
Only files with *no* rows in the collection are indexed. A document that is
already indexed under its real filename is left untouched — its ``domain``
came from whatever the uploader chose, and the on-disk copy carries no
record of that, so re-ingesting would silently retag a ``finance`` document
- as ``general``. Recovered documents do land under ``general``, which is the
- honest answer (their domain died with the orphaned rows) and is retrievable
- by every specialist rather than by none.
+ as ``general``. Recovered documents land under ``general`` explicitly (NOT
+ via ``infer_domain_from_path``, which scans every component of the absolute
+ path and would tag them ``finance`` on an install rooted under, say,
+ ``/srv/finance/``). That is the honest answer — their domain died with the
+ orphaned rows — and ``general`` is retrievable by every specialist rather
+ than by none.
Documents whose only copy was the orphaned index — chat/email attachments,
which are never written to ``docs_dir`` — cannot be recovered and are
dropped by the sweep.
+
+ A document that extracts to no text writes no chunks, so it is retried on
+ every boot. That is wasted work rather than churn (the store still
+ converges), and it is bounded by however many unreadable files are sitting
+ in ``docs_dir``.
+
+ Does nothing at all when ``docs_dir`` does not exist. An absent directory is
+ ambiguous — "no documents" and "the volume is not mounted yet" look
+ identical — and in the second case every recoverable document would be
+ swept precisely because the file that would have spared it is invisible.
"""
+ if not docs_dir.is_dir():
+ logger.warning(
+ "reconcile_company_docs: %s does not exist — skipping "
+ "(cannot distinguish an empty docs dir from an unmounted one)",
+ docs_dir,
+ )
+ return 0, 0
+
live = _company_doc_names(docs_dir)
orphans: list[str] = []
@@ -359,19 +381,24 @@ async def reconcile_company_docs(
else:
indexed_names.add(name)
- store.delete_by_ids(collection, orphans)
+ deleted = store.delete_by_ids(collection, orphans)
indexed = 0
for doc_name in sorted(live - indexed_names):
try:
- if await ingest_file(docs_dir / doc_name, store, collection=collection):
+ if await ingest_file(
+ docs_dir / doc_name,
+ store,
+ domain=GENERAL_DOMAIN,
+ collection=collection,
+ ):
indexed += 1
except Exception:
# One unreadable document must not abort the reconcile —
# the rest of the index is still worth repairing.
logger.exception("reconcile_company_docs: index failed: %s", doc_name)
- return len(orphans), indexed
+ return deleted, indexed
async def seed_builtin_knowledge(
diff --git a/packages/core/openexecutive/knowledge/retriever.py b/packages/core/openexecutive/knowledge/retriever.py
index 97810fa8..a36c3c16 100644
--- a/packages/core/openexecutive/knowledge/retriever.py
+++ b/packages/core/openexecutive/knowledge/retriever.py
@@ -173,10 +173,15 @@ def _with_general(domains: list[str] | None) -> list[str] | None:
shares its rows with external OER sources, which fall back to ``general``
when a source declares no domains (``external_sources``); fanning those
into every specialist would blend unvetted third-party material into every
- answer. ``None`` means "no domain filter" and already matches everything.
+ answer.
+
+ An absent filter is returned unchanged. ``store.query`` treats both ``None``
+ and ``[]`` as "no domain filter", so both already match every domain —
+ widening ``[]`` to ``["general"]`` would *narrow* it to general-only, the
+ exact inversion of this function's purpose.
"""
- if domains is None:
- return None
+ if not domains:
+ return domains
return domains if GENERAL_DOMAIN in domains else [*domains, GENERAL_DOMAIN]
diff --git a/packages/core/openexecutive/knowledge/store.py b/packages/core/openexecutive/knowledge/store.py
index 96def478..fdf16c45 100644
--- a/packages/core/openexecutive/knowledge/store.py
+++ b/packages/core/openexecutive/knowledge/store.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import logging
from abc import ABC, abstractmethod
from pathlib import Path
from typing import Any
@@ -161,16 +162,28 @@ class ChromaDBStore(KnowledgeStore):
for cid, md in zip(ids, metas, strict=False)
]
- def delete_by_ids(self, collection: str, ids: list[str]) -> None:
- """Delete specific chunk ids. No-op on an empty list (Chroma treats a
- delete with neither ids nor where as 'delete everything')."""
+ def delete_by_ids(self, collection: str, ids: list[str]) -> int:
+ """Delete specific chunk ids; returns how many were actually deleted.
+
+ No-op on an empty list — Chroma treats a delete with neither ids nor
+ where as 'delete everything', so the guard must come before the call,
+ not inside it.
+
+ Returns 0 rather than ``len(ids)`` when the delete raises, so a caller
+ reporting the count cannot claim to have removed rows that are still
+ there.
+ """
if not ids:
- return
+ return 0
try:
col = self._get_or_create_collection(collection)
col.delete(ids=ids)
+ return len(ids)
except Exception:
- pass
+ logging.getLogger(__name__).exception(
+ "delete_by_ids failed for %d id(s) in %s", len(ids), collection
+ )
+ return 0
def delete_company_docs(self) -> None:
"""Delete and recreate the company_docs collection, clearing all indexed documents."""
diff --git a/packages/core/tests/unit/_fake_store.py b/packages/core/tests/unit/_fake_store.py
index 8fba5ada..389c1799 100644
--- a/packages/core/tests/unit/_fake_store.py
+++ b/packages/core/tests/unit/_fake_store.py
@@ -37,9 +37,10 @@ class FakeStore:
def iter_chunk_metadata(self, collection: str) -> list[tuple[str, dict[str, Any]]]:
return list(self.rows.items())
- def delete_by_ids(self, collection: str, ids: list[str]) -> None:
- for chunk_id in ids:
- self.rows.pop(chunk_id, None)
+ def delete_by_ids(self, collection: str, ids: list[str]) -> int:
+ # Returns the count actually removed, like ChromaDBStore.delete_by_ids —
+ # callers report that number, so the fake must not overstate it either.
+ return sum(self.rows.pop(chunk_id, None) is not None for chunk_id in ids)
# -- assertion helpers ------------------------------------------------
@property
diff --git a/packages/core/tests/unit/test_attachments.py b/packages/core/tests/unit/test_attachments.py
index b8c77b41..26d26bc2 100644
--- a/packages/core/tests/unit/test_attachments.py
+++ b/packages/core/tests/unit/test_attachments.py
@@ -292,11 +292,17 @@ def test_attachment_name_is_stripped_to_a_bare_name():
assert kwargs["source_name"] == "attachment:passwd.md"
-def test_attachment_lands_in_a_retrievable_domain():
- """It used to be tagged "company_docs", which is not one of the specialist
- domains, so no specialist could ever retrieve an attachment."""
+def test_attachment_domain_stays_outside_the_specialist_domains():
+ """Pins a deliberate gap, so a later change has to be deliberate too.
+
+ "company_docs" is not one of the specialist domains, so these chunks match
+ no domain filter and never reach the Executive. Anyone who can attach a
+ file in an integration channel would otherwise be writing into every
+ specialist's RAG context, and these rows have no removal path — they are
+ never written to company/docs/, so GET /documents does not list them and
+ DELETE /documents/{filename} 404s before reaching the store."""
from openexecutive.knowledge.loader import UPLOAD_DOMAINS
_, kwargs = _ingest_call("board-deck.md")
- assert kwargs["domain"] in UPLOAD_DOMAINS
+ assert kwargs["domain"] not in UPLOAD_DOMAINS
diff --git a/packages/core/tests/unit/test_company_doc_identity.py b/packages/core/tests/unit/test_company_doc_identity.py
index e5f0cce5..adc36c60 100644
--- a/packages/core/tests/unit/test_company_doc_identity.py
+++ b/packages/core/tests/unit/test_company_doc_identity.py
@@ -141,12 +141,17 @@ async def test_sweep_spares_a_real_document_named_like_a_temp_file(tmp_path: Pat
@pytest.mark.asyncio
-async def test_sweep_tolerates_a_missing_docs_dir(tmp_path: Path) -> None:
+async def test_sweep_skips_entirely_when_the_docs_dir_is_missing(tmp_path: Path) -> None:
+ """An absent docs dir is ambiguous: "no documents" and "the volume is not
+ mounted yet" look identical. Sweeping in the second case destroys exactly
+ the documents that were recoverable, because the file that would have
+ spared each one is invisible."""
store = _FakeStore(dict([_orphan("tmp1du9epr4.md")]))
swept, indexed = await reconcile_company_docs(store, tmp_path / "does-not-exist")
- assert (swept, indexed) == (1, 0)
+ assert (swept, indexed) == (0, 0)
+ assert store.rows, "an unmounted docs dir must not be read as 'delete everything'"
@pytest.mark.asyncio
@@ -216,3 +221,92 @@ def test_every_specialist_retrieves_general_company_docs(specialist: str) -> Non
"""The whole point of #114: an unclassified upload must be visible to all
specialists rather than to none."""
assert GENERAL_DOMAIN in (_with_general(DOMAIN_ALIASES[specialist]) or [])
+
+
+def test_with_general_leaves_an_empty_filter_alone() -> None:
+ """`store.query` treats both None and [] as "no filter", so [] already
+ matches every domain — widening it to ["general"] would *narrow* it to
+ general-only, inverting this function's purpose."""
+ assert _with_general([]) == []
+
+
+async def test_recovered_document_lands_under_general_despite_the_install_path(
+ tmp_path: Path,
+) -> None:
+ """`infer_domain_from_path` scans every component of the absolute path, so
+ an install rooted under e.g. /srv/finance/ would tag recovered documents
+ `finance`. The reconcile must pass the domain explicitly."""
+ docs = tmp_path / "finance" / "company" / "docs"
+ docs.mkdir(parents=True)
+ (docs / "plan.md").write_text("# Plan\n" + "grow revenue thirty percent. " * 50)
+ store = _FakeStore()
+
+ await reconcile_company_docs(store, docs)
+
+ assert {m["domain"] for m in store.rows.values()} == {GENERAL_DOMAIN}
+
+
+async def test_sweep_reports_zero_when_the_delete_fails(tmp_path: Path) -> None:
+ """The boot log must not claim to have dropped rows that are still there."""
+ docs = _docs_dir(tmp_path)
+ store = _FakeStore(dict([_orphan("tmp1du9epr4.md")]))
+ store.delete_by_ids = lambda collection, ids: 0 # type: ignore[assignment]
+
+ swept, _ = await reconcile_company_docs(store, docs)
+
+ assert swept == 0
+
+
+class _RecordingStore(_FakeStore):
+ """Captures the domain filter each collection is queried with."""
+
+ def __init__(self) -> None:
+ super().__init__()
+ self.queries: dict[str, list[str] | None] = {}
+
+ def query(
+ self,
+ query_text: str,
+ collection: str,
+ domain_filter: list[str] | None = None,
+ n_results: int = 5,
+ ) -> list[dict[str, Any]]:
+ self.queries[collection] = domain_filter
+ return []
+
+
+def test_retrieve_widens_only_the_company_filter(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Guards the wiring, not just the helper: reverting the `_with_general`
+ call in `retrieve()` would otherwise leave the whole suite green while
+ #114 silently regresses."""
+ from openexecutive.knowledge import retriever
+ from openexecutive.knowledge.store import ChromaDBStore
+
+ store = _RecordingStore()
+ monkeypatch.setattr(retriever, "_default_review_store", lambda: _NullReviewStore())
+
+ retriever.retrieve(
+ query="what is our runway and hiring plan for next quarter",
+ specialist_name="cfo",
+ store=store, # type: ignore[arg-type]
+ )
+
+ assert store.queries[ChromaDBStore.COMPANY_COLLECTION] == ["finance", GENERAL_DOMAIN]
+ assert store.queries[ChromaDBStore.BUILTIN_COLLECTION] == ["finance"]
+ assert store.queries[ChromaDBStore.NOTION_COLLECTION] == ["finance"]
+
+
+class _NullReviewStore:
+ def get_rejected_filenames(self, *a: Any, **k: Any) -> set[str]:
+ return set()
+
+ def get_rejected_source_ids(self, *a: Any, **k: Any) -> set[str]:
+ return set()
+
+ def get_priority_map(self, *a: Any, **k: Any) -> dict[str, str]:
+ return {}
+
+ def list_annotations(self, *a: Any, **k: Any) -> list[Any]:
+ return []
diff --git a/packages/core/tests/unit/test_knowledge_search_and_failures.py b/packages/core/tests/unit/test_knowledge_search_and_failures.py
index 269fe716..fb96671e 100644
--- a/packages/core/tests/unit/test_knowledge_search_and_failures.py
+++ b/packages/core/tests/unit/test_knowledge_search_and_failures.py
@@ -233,3 +233,57 @@ def test_failures_rejects_path_traversal(client: TestClient) -> None:
# The path-segment regex requires a *.md and no slashes/dots-as-traversal,
# so the encoded "../evil.md" must be rejected as a bad filename.
assert res.status_code in (400, 404)
+
+
+# ── #114: the Query panel must mirror what retrieve() actually does ────────
+
+
+def test_search_accepts_the_general_domain(client: TestClient) -> None:
+ """`general` is a real, uploadable company domain. Rejecting it left the
+ operator unable to introspect the documents most likely to need it — the
+ unclassified ones."""
+ res = client.post(
+ "/knowledge/search", json={"query": "pricing", "domain_filter": ["general"]}
+ )
+
+ assert res.status_code == 200, res.text
+
+
+def test_search_still_rejects_an_unknown_domain(client: TestClient) -> None:
+ res = client.post(
+ "/knowledge/search", json={"query": "pricing", "domain_filter": ["finanace"]}
+ )
+
+ assert res.status_code == 400
+
+
+def test_search_widens_the_company_filter_with_general(
+ monkeypatch: pytest.MonkeyPatch, client: TestClient
+) -> None:
+ """This endpoint is documented as the parallel of `retrieve()`. If it does
+ not widen COMPANY the same way, the panel reports zero company hits for a
+ specialist that does retrieve them in chat — the exact #114 symptom, shown
+ by the tool an operator would use to diagnose it."""
+ seen: dict[str, list[str] | None] = {}
+
+ def recording_query(
+ query_text: str,
+ collection: str,
+ domain_filter: list[str] | None = None,
+ n_results: int = 5,
+ ) -> list[dict[str, Any]]:
+ seen[collection] = domain_filter
+ return []
+
+ recording_store = MagicMock()
+ recording_store.query.side_effect = recording_query
+ monkeypatch.setattr(
+ "openexecutive.api.routes.knowledge._get_store",
+ lambda _request: recording_store,
+ )
+
+ res = client.post("/knowledge/search", json={"query": "pricing", "specialist": "cfo"})
+
+ assert res.status_code == 200, res.text
+ assert seen["company_docs"] == ["finance", "general"]
+ assert seen["builtin_knowledge"] == ["finance"]