Skip to content

AI-powered virtual executive team — a single coherent executive persona backed by 8 specialist agents (FastAPI + Next.js).

fix(documents): key company docs on their filename, not the temp path

Rufusffae3a0edb83
diff --git a/packages/core/openexecutive/api/main.py b/packages/core/openexecutive/api/main.py
index 52345fad..47cab157 100644
--- a/packages/core/openexecutive/api/main.py
+++ b/packages/core/openexecutive/api/main.py
@@ -151,7 +151,11 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
from openexecutive.alerts.store import initialize_db as initialize_alerts_db
from openexecutive.audit import AuditLogger, set_audit_logger
from openexecutive.config import get_settings
- from openexecutive.knowledge.loader import seed_builtin_knowledge, seed_failures
+ from openexecutive.knowledge.loader import (
+ reconcile_company_docs,
+ seed_builtin_knowledge,
+ seed_failures,
+ )
from openexecutive.knowledge.skills_index import seed_builtin_skills
from openexecutive.knowledge.store import ChromaDBStore
from openexecutive.memory.episodic import initialize_db
@@ -175,6 +179,24 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
await seed_builtin_skills(store=store)
await seed_failures(store=store)
+ # Repair company_docs rows left by the old upload path, which indexed
+ # documents under their random staging filename: those chunks match no
+ # DELETE and are displaced by no re-upload, so nothing else can reach
+ # them. Idempotent — chunk ids are filename-derived, so a clean store
+ # makes this a no-op. Best-effort, like the seeding above.
+ try:
+ swept, indexed = await reconcile_company_docs(
+ store, settings.company_profile_path.parent / "docs"
+ )
+ if swept or indexed:
+ logging.getLogger("openexecutive").info(
+ "company_docs reconcile: dropped %d orphaned chunk(s), indexed %d document(s)",
+ swept,
+ indexed,
+ )
+ except Exception:
+ logging.getLogger("openexecutive").exception("company_docs reconcile failed")
+
initialize_db()
initialize_alerts_db()
diff --git a/packages/core/openexecutive/api/routes/documents.py b/packages/core/openexecutive/api/routes/documents.py
index 10d331f2..4faf6f16 100644
--- a/packages/core/openexecutive/api/routes/documents.py
+++ b/packages/core/openexecutive/api/routes/documents.py
@@ -37,6 +37,18 @@ async def upload_document(
detail=f"Unsupported file type: {ext}. Allowed: {', '.join(ALLOWED_EXTENSIONS)}",
)
+ # Reject an unknown domain rather than indexing under it. Specialist
+ # retrieval filters on these exact values, so a typo ("finanace") would
+ # return 200 and then make the document permanently unretrievable — a
+ # silent, delayed failure with no way to notice it from the API.
+ from openexecutive.knowledge.loader import UPLOAD_DOMAINS
+
+ if domain not in UPLOAD_DOMAINS:
+ raise HTTPException(
+ status_code=400,
+ detail=f"Unknown domain: {domain}. Allowed: {', '.join(sorted(UPLOAD_DOMAINS))}",
+ )
+
content = await file.read()
if len(content) > 50 * 1024 * 1024:
raise HTTPException(status_code=413, detail="File too large (max 50MB)")
@@ -57,11 +69,19 @@ async def upload_document(
else ChromaDBStore(persist_directory=settings.vector_store_path)
)
+ # `source_name` — NOT tmp_path.name. The temp file is only a staging
+ # buffer so the PDF/DOCX extractors have a real path to open; the
+ # document's identity is the name the user uploaded it under. Deriving
+ # identity from the random temp name instead gives every re-upload
+ # fresh chunk ids (so the id-keyed upsert never collides and the
+ # collection grows without bound) and stores a `filename` that the
+ # DELETE endpoint below can never match.
chunks_indexed = await ingest_file(
path=tmp_path,
store=store,
domain=domain,
collection=ChromaDBStore.COMPANY_COLLECTION,
+ source_name=safe_filename,
)
company_docs_dir = settings.company_profile_path.parent / "docs"
diff --git a/packages/core/openexecutive/architecture/architecture-facts.yaml b/packages/core/openexecutive/architecture/architecture-facts.yaml
index c91adb57..81c1d074 100644
--- a/packages/core/openexecutive/architecture/architecture-facts.yaml
+++ b/packages/core/openexecutive/architecture/architecture-facts.yaml
@@ -677,6 +677,36 @@ knowledge:
for read-only viewing (GET /documents/{filename}), and delete
(DELETE /documents/{filename}) individual docs. Synced Notion pages
do NOT land here — see NOTION_WIKI.
+
+ IDENTITY IS THE FILENAME. ingest_file takes an explicit `source_name`
+ that becomes the filename/source metadata AND the chunk-id namespace
+ (_make_chunk_id is an MD5 of it). The upload endpoint stages the body
+ in a tempfile so the PDF/DOCX extractors have a real path to open, and
+ MUST pass the real name — passing the staging path was issue #113:
+ every re-upload got fresh ids so the id-keyed upsert never collided
+ and company_docs grew without bound, and the stored filename was
+ `tmpXXXXXXXX.md`, which the DELETE `where={"filename": …}` could never
+ match (it unlinked the file, returned 200, and stranded every chunk).
+ Because the id namespace is the bare filename and not the absolute
+ path, an uploaded plan.md and a fixture-loaded plan.md are the same
+ document rather than two.
+
+ Chat/email attachments (integrations.attachments) share this
+ collection but are namespaced `attachment:<bare filename>`. The
+ sender chooses that filename and the filename is the id namespace, so
+ an unprefixed one would upsert straight over the curated company
+ document it collides with — same "unvetted text must not blend into
+ curated uploads" reasoning that isolates NOTION_WIKI and
+ RECENT_RESEARCH, applied within a collection.
+
+ RECONCILE ON BOOT. The lifespan calls
+ loader.reconcile_company_docs(store, docs_dir), which drops rows whose
+ filename still matches the temp pattern and is absent from docs_dir,
+ then indexes any docs_dir file with no rows at all. It deliberately
+ does NOT re-ingest documents that are already indexed: the on-disk
+ copy carries no record of the domain the uploader chose, so
+ re-ingesting would retag a `finance` document as `general`. Idempotent
+ (filename-derived ids), so it is a no-op on a clean store.
- name: NOTION_WIKI
what: |
Synced Notion wiki pages (ChromaDB collection `notion_wiki`). Kept
@@ -715,6 +745,26 @@ knowledge:
BELOW the curated company-documents section.
domain_aliases_source: openexecutive.knowledge.retriever.DOMAIN_ALIASES
dedupe_note: Multi-domain chunks can fan out to multiple rows; the retriever dedupes by text before returning.
+ upload_domains: |
+ POST /documents accepts the eight DOMAIN_MAP domains plus `general`
+ (loader.UPLOAD_DOMAINS) and returns 400 for anything else. Unvalidated, a
+ typo like `domain=finanace` returned 200 and indexed the document under a
+ value no specialist filters on — issue #114, silent and only observable by
+ asking a specialist a question it should be able to answer and getting
+ nothing back.
+ general_catch_all: |
+ `general` is the domain an upload gets when nobody classified it: the
+ default on POST /documents, the default option in the UI's picker, and
+ what reconcile_company_docs assigns to a recovered document. It maps to no
+ specialist in DOMAIN_ALIASES, so retriever._with_general widens each
+ specialist's COMPANY filter to include it — otherwise an unclassified
+ document is retrievable by NO specialist rather than by all of them.
+
+ Scoped to the COMPANY collection on purpose. BUILTIN shares its rows with
+ external OER sources, and external_sources falls back to ["general"] for a
+ source that declares no domains; widening BUILTIN too would fan unvetted
+ third-party material into every specialist's context. NOTION_WIKI and
+ RECENT_RESEARCH keep their existing filters.
memory:
episodic_db: ./episodic_memory.db
@@ -2837,6 +2887,21 @@ integrations:
surface in future RAG retrieval. Per-file cap 20 MB; extracted text
truncated to 40 000 chars. Web chat exposes this through
``POST /chat/upload`` (multipart) — capped at 5 files per turn.
+
+ Indexed under ``source_name="attachment:<bare filename>"`` and domain
+ ``general``. Both matter. The name is the chunk-id namespace and the
+ retrieval citation, and an attachment's filename is chosen by whoever
+ sent the message — unprefixed, an emailed ``strategy-2026.md`` would
+ upsert straight over the curated company document of that name, so
+ inbound content stays in its own namespace and its provenance is
+ visible in the ``[filename]`` citation. (``Path(...).name`` strips
+ directory components first.) The domain was ``company_docs`` until
+ #113/#114, which is not one of the specialist domains — no specialist
+ could retrieve an attachment at all. Passing the staging tempfile path
+ as the identity was the #113 bug on this path too: duplicate chunks on
+ every re-send, and no filename any API call could delete (attachments
+ are never written to docs/, so the boot reconcile drops their orphans
+ rather than recovering them).
broadcast_tools: |
openexecutive.orchestrator.broadcast_tools — two new tools exposed
to the Executive alongside the existing per-Person DM tools:
diff --git a/packages/core/openexecutive/architecture/prebuilt/api.json b/packages/core/openexecutive/architecture/prebuilt/api.json
index 137a3f95..be657b60 100644
--- a/packages/core/openexecutive/architecture/prebuilt/api.json
+++ b/packages/core/openexecutive/architecture/prebuilt/api.json
@@ -1,7 +1,7 @@
{
"section_id": "api",
"title": "API Reference",
- "markdown": "The FastAPI surface lives in `openexecutive.api.main`, which mounts one router per topic. A shared-secret middleware gates every request via an `x-api-key` header, except `/health` and the `/webhook/*` callbacks (which carry their own verification). CORS is restricted to configured UI origins. The gate is off when `BACKEND_SHARED_SECRET` is unset — intended for local development only — so any internet-reachable deployment sets `OE_PUBLIC_DEPLOYMENT`, and `create_app()` then refuses to boot without the secret rather than serving an unauthenticated API.\n\nEndpoints grouped by router:\n\n| Router | Key endpoints | Purpose |\n|---|---|---|\n| `chat` | `POST /chat` | SSE stream of a live Executive turn. The request body optionally carries `page_context` (route, page title, guide section id, and a form descriptor with current values) — sent only by the Ask OE side panel so the Executive can explain the current page or propose form values. When the Executive calls its `propose_form_values` tool, the stream emits a `form_patch` event (`{type, form_id, fields, rationale, iteration}`) that the panel applies into the on-screen form as highlighted suggestions; the user reviews and saves through the form's normal submit path. |\n| `sessions` | `GET /sessions`, `GET /sessions/{id}/messages` | List and read saved chat sessions. |\n| `today` | `GET /today`, `GET /today/activity`, `GET /today/activity/daily` | Morning-brief aggregation, recent-action rail, and Pulse heartbeat heatmap (per-day activity counts). |\n| `people` | `GET\\|POST\\|PATCH /people`, `POST /people/{id}/archive` | Roster CRUD and scope lookups. |\n| `departments` | `GET\\|POST\\|PATCH\\|DELETE /departments`, `.../goals` | Department state and per-department goals. `PATCH /departments/{slug}` also takes `watched_entities` (≤50 names, `[]` clears) — the entities the research watch policy may add watches for and route to this department. |\n| `talent` | `GET|POST|PATCH /engagements`, `GET|POST|PATCH /candidates`, `POST /candidates/{id}/stage`, `GET /engagements/{id}/matches`, `GET /candidates/{id}/similar`, `POST /talent/reindex`, `GET|POST|PATCH /offers`, `POST /offers/{id}/{extend,decision,archive}` | Executive-search pipeline: searches, candidates, semantic matching, and the offer lifecycle (extend/decision return the offer plus approval state, scheduled/cancelled expiry nudges, side effects, and warnings; lifecycle conflicts are 409s). |\n| `agents` | `GET /agents`, `PATCH /agents/{id}`, `POST /agents/{id}/test` | Specialist metadata and Council overrides. |\n| `personas` | `GET\\|PUT\\|POST\\|DELETE /personas` | Executive persona definitions. |\n| `alerts` | `POST /alerts/{id}/ack`, `POST /alerts/bulk-ack`, `POST /alerts/{id}/reopen`, `POST /alerts/review` | Groom the briefing queue. `ack` (`read`/`ack`/`dismissed`, optional `mute_topic`) also teaches a watch-sourced alert's watch (`trust_score`); `bulk-ack` takes explicit `alert_ids` and/or `older_than_days` + `category` (exempt sources skipped); `reopen` is the Undo for a `resolved` / `expired` / `dismissed` alert (never `ack` or an artifact / decision row; back to `unread` with every review stamp cleared and its TTL anchor refreshed, so it is live again even if older than its TTL); `review` runs the Executive's relevance review on demand (ignores the interval; still honours `ALERT_REVIEW_ENABLED`, the per-pass caps, the single-flight lock and per-alert idempotency) and returns the move counts. `bulk-ack` caps ids at 500, needs `older_than_days >= 1`, and applies watch feedback per row; `mute_topic` must be one of the alert's tags or a scoped `external:` / `department:` pattern. Alerts still surface only through the briefing (`/today` reads the live view) — there is no list, prefs, mutes, or live-stream endpoint. |\n| `artifacts` | `GET /artifacts`, `POST /artifacts/{id}/archive` | Browsable research/document artifacts. |\n| `decisions` | `GET /decisions`, `POST /decisions/{id}/{approve,reject,cancel}`, `GET /audit/reliability` | Approval-gated decision instances (e.g. calendar bookings). A proposed booking is surfaced on the briefing as a companion alert; approve/reject/cancel here also clear that alert. Reviewed in the briefing — there is no separate proposals page. |\n| `audit` | `GET /audit/logs`, `GET /audit/logs/{id}`, `GET /audit/sessions/{id}`, `GET /audit/usage`, `POST /audit/log` | Durable audit trail of side effects. `/audit/sessions/{id}` adds a per-session token cost summary; `/audit/usage` aggregates token usage and OpenRouter cost across all sessions (totals, by day, by model, and by source — the part of the system that made the calls — including server-side web-search counts). |\n| `architecture` | `GET /architecture/sections`, `.../sections/{id}` | This documentation surface. |\n| `guide` | `GET /guide/sections`, `.../sections/{id}` | Plain-language user-guide content surface — mirrors `architecture` (static, version-controlled files; no LLM on the serving path). |\n| `health` | `GET /health`, `GET /health/honcho` | Liveness probes (unauthenticated). |\n| `documents` | `POST\\|GET\\|DELETE /documents`, `GET /documents/{filename}` | Upload, list, view extracted text, and delete indexed company docs. |\n| `scheduled` | `GET\\|DELETE` scheduled actions (`?status=…&order=asc\\|desc`) | Pending and fired scheduled actions; `order` flips the `run_at` sort (asc=soonest-first for the upcoming queue, desc=most-recent-first for terminal-status history). |\n| `watchlist` | `GET\\|POST\\|PATCH\\|DELETE /watchlist`, `POST /watchlist/{slug}/approve`, `POST /watchlist/{slug}/decline` | External-monitoring watchlist + signals. Rows carry `origin` (`manual` \\| `executive` \\| `research` \\| `research_proposed`); `approve` turns a research suggestion (`research_proposed`, `dry_run`) into a live watch, `decline` `{reason: not_relevant \\| too_noisy \\| wrong_source}` removes it and records the decline (or, for `too_noisy`, keeps it live at a high floor), both 409 unless the row is a pending suggestion; `DELETE ?reason=` on a research-origin row records the decline too. |\n| `workflows` | `GET /workflows`, `POST /workflows/{name}/runs`, `GET\\|POST\\|PUT\\|DELETE /workflows/custom` | Workflow catalog, run management, and user-created (dynamic) workflow CRUD. |\n| `knowledge` | `/knowledge/builtin`, `/external`, `/search` | MBA knowledge base and external sources. |\n| `skills` | `/skills`, `/skills/search` | Skill registry CRUD and search. |\n| `evals` | `/evals/scenarios`, `/evals/runs` | Eval scenarios and run results. |\n| `memories` | `/memories/{decisions,initiatives,advice}` | Episodic-memory record management. |\n| `review` | `/review/items`, `/review/stats` | Knowledge review queue and annotations. |\n| `onboarding` | `/onboard/interview/start`, `/onboard/interview/message`, `/onboard/interview/draft`, `/onboard/interview/{id}`, `/onboard/interview/commit` | Conversational company setup. The `onboarding_interviewer` agent asks clarifying questions, then emits a draft profile + people + departments. **Nothing is persisted until `POST /onboard/interview/commit`** — the interview itself never writes. `start` is multipart (description + optional deck/one-pager uploads). |\n| `onboarding` (fallback) | `/onboard/start`, `/onboard/answer`, `/onboard/status/{id}` | The original 12-step form wizard, kept as a fallback (UI at `/onboard?mode=form`) and used by the `openexecutive onboard` CLI. |\n| `company-profile` | `GET\\|PATCH /company-profile` | Structured company profile. |\n| `auth` | `GET /auth/allowed-emails` | Allowed sign-in emails. |\n| `fixtures` | `/fixtures`, `/fixtures/{name}/load` | Demo-company fixture load/reset/snapshot. |\n| `clients` | `/clients`, `/clients/generate`, `/clients/cockpit`, `/clients/save`, `/clients/{slug}/activate` | Client-company slots — multi-client mode for fractional use; create from current/blank/AI-generated intake draft (pasted notes and/or file attachments, multipart on `/clients/generate`), save-back switching, engagement metadata (PATCH), cross-client cockpit, delete. See the Client Companies section. |\n| webhooks | `POST /webhook/telegram`, `POST /webhook/google-chat` | Inbound integration callbacks (self-verified). |\n| MCP server | `POST\\|GET /mcp` | Open Executive exposed as an MCP server (Streamable-HTTP). Not a router — a mounted FastMCP sub-app. Same shared-secret gate (`x-api-key`); exposes company context as resources and the specialist council as tools. See the MCP Server section. |",
+ "markdown": "The FastAPI surface lives in `openexecutive.api.main`, which mounts one router per topic. A shared-secret middleware gates every request via an `x-api-key` header, except `/health` and the `/webhook/*` callbacks (which carry their own verification). CORS is restricted to configured UI origins. The gate is off when `BACKEND_SHARED_SECRET` is unset — intended for local development only — so any internet-reachable deployment sets `OE_PUBLIC_DEPLOYMENT`, and `create_app()` then refuses to boot without the secret rather than serving an unauthenticated API.\n\nEndpoints grouped by router:\n\n| Router | Key endpoints | Purpose |\n|---|---|---|\n| `chat` | `POST /chat` | SSE stream of a live Executive turn. The request body optionally carries `page_context` (route, page title, guide section id, and a form descriptor with current values) — sent only by the Ask OE side panel so the Executive can explain the current page or propose form values. When the Executive calls its `propose_form_values` tool, the stream emits a `form_patch` event (`{type, form_id, fields, rationale, iteration}`) that the panel applies into the on-screen form as highlighted suggestions; the user reviews and saves through the form's normal submit path. |\n| `sessions` | `GET /sessions`, `GET /sessions/{id}/messages` | List and read saved chat sessions. |\n| `today` | `GET /today`, `GET /today/activity`, `GET /today/activity/daily` | Morning-brief aggregation, recent-action rail, and Pulse heartbeat heatmap (per-day activity counts). |\n| `people` | `GET\\|POST\\|PATCH /people`, `POST /people/{id}/archive` | Roster CRUD and scope lookups. |\n| `departments` | `GET\\|POST\\|PATCH\\|DELETE /departments`, `.../goals` | Department state and per-department goals. `PATCH /departments/{slug}` also takes `watched_entities` (≤50 names, `[]` clears) — the entities the research watch policy may add watches for and route to this department. |\n| `talent` | `GET|POST|PATCH /engagements`, `GET|POST|PATCH /candidates`, `POST /candidates/{id}/stage`, `GET /engagements/{id}/matches`, `GET /candidates/{id}/similar`, `POST /talent/reindex`, `GET|POST|PATCH /offers`, `POST /offers/{id}/{extend,decision,archive}` | Executive-search pipeline: searches, candidates, semantic matching, and the offer lifecycle (extend/decision return the offer plus approval state, scheduled/cancelled expiry nudges, side effects, and warnings; lifecycle conflicts are 409s). |\n| `agents` | `GET /agents`, `PATCH /agents/{id}`, `POST /agents/{id}/test` | Specialist metadata and Council overrides. |\n| `personas` | `GET\\|PUT\\|POST\\|DELETE /personas` | Executive persona definitions. |\n| `alerts` | `POST /alerts/{id}/ack`, `POST /alerts/bulk-ack`, `POST /alerts/{id}/reopen`, `POST /alerts/review` | Groom the briefing queue. `ack` (`read`/`ack`/`dismissed`, optional `mute_topic`) also teaches a watch-sourced alert's watch (`trust_score`); `bulk-ack` takes explicit `alert_ids` and/or `older_than_days` + `category` (exempt sources skipped); `reopen` is the Undo for a `resolved` / `expired` / `dismissed` alert (never `ack` or an artifact / decision row; back to `unread` with every review stamp cleared and its TTL anchor refreshed, so it is live again even if older than its TTL); `review` runs the Executive's relevance review on demand (ignores the interval; still honours `ALERT_REVIEW_ENABLED`, the per-pass caps, the single-flight lock and per-alert idempotency) and returns the move counts. `bulk-ack` caps ids at 500, needs `older_than_days >= 1`, and applies watch feedback per row; `mute_topic` must be one of the alert's tags or a scoped `external:` / `department:` pattern. Alerts still surface only through the briefing (`/today` reads the live view) — there is no list, prefs, mutes, or live-stream endpoint. |\n| `artifacts` | `GET /artifacts`, `POST /artifacts/{id}/archive` | Browsable research/document artifacts. |\n| `decisions` | `GET /decisions`, `POST /decisions/{id}/{approve,reject,cancel}`, `GET /audit/reliability` | Approval-gated decision instances (e.g. calendar bookings). A proposed booking is surfaced on the briefing as a companion alert; approve/reject/cancel here also clear that alert. Reviewed in the briefing — there is no separate proposals page. |\n| `audit` | `GET /audit/logs`, `GET /audit/logs/{id}`, `GET /audit/sessions/{id}`, `GET /audit/usage`, `POST /audit/log` | Durable audit trail of side effects. `/audit/sessions/{id}` adds a per-session token cost summary; `/audit/usage` aggregates token usage and OpenRouter cost across all sessions (totals, by day, by model, and by source — the part of the system that made the calls — including server-side web-search counts). |\n| `architecture` | `GET /architecture/sections`, `.../sections/{id}` | This documentation surface. |\n| `guide` | `GET /guide/sections`, `.../sections/{id}` | Plain-language user-guide content surface — mirrors `architecture` (static, version-controlled files; no LLM on the serving path). |\n| `health` | `GET /health`, `GET /health/honcho` | Liveness probes (unauthenticated). |\n| `documents` | `POST\\|GET\\|DELETE /documents`, `GET /documents/{filename}` | Upload, list, view extracted text, and delete indexed company docs. Upload is keyed on the filename, so re-uploading a document replaces its chunks rather than duplicating them, and `DELETE` removes the rows as well as the file. `domain` must be one of the eight specialist domains or `general`; anything else is a 400 rather than a 200 that indexes where no specialist can retrieve. |\n| `scheduled` | `GET\\|DELETE` scheduled actions (`?status=…&order=asc\\|desc`) | Pending and fired scheduled actions; `order` flips the `run_at` sort (asc=soonest-first for the upcoming queue, desc=most-recent-first for terminal-status history). |\n| `watchlist` | `GET\\|POST\\|PATCH\\|DELETE /watchlist`, `POST /watchlist/{slug}/approve`, `POST /watchlist/{slug}/decline` | External-monitoring watchlist + signals. Rows carry `origin` (`manual` \\| `executive` \\| `research` \\| `research_proposed`); `approve` turns a research suggestion (`research_proposed`, `dry_run`) into a live watch, `decline` `{reason: not_relevant \\| too_noisy \\| wrong_source}` removes it and records the decline (or, for `too_noisy`, keeps it live at a high floor), both 409 unless the row is a pending suggestion; `DELETE ?reason=` on a research-origin row records the decline too. |\n| `workflows` | `GET /workflows`, `POST /workflows/{name}/runs`, `GET\\|POST\\|PUT\\|DELETE /workflows/custom` | Workflow catalog, run management, and user-created (dynamic) workflow CRUD. |\n| `knowledge` | `/knowledge/builtin`, `/external`, `/search` | MBA knowledge base and external sources. |\n| `skills` | `/skills`, `/skills/search` | Skill registry CRUD and search. |\n| `evals` | `/evals/scenarios`, `/evals/runs` | Eval scenarios and run results. |\n| `memories` | `/memories/{decisions,initiatives,advice}` | Episodic-memory record management. |\n| `review` | `/review/items`, `/review/stats` | Knowledge review queue and annotations. |\n| `onboarding` | `/onboard/interview/start`, `/onboard/interview/message`, `/onboard/interview/draft`, `/onboard/interview/{id}`, `/onboard/interview/commit` | Conversational company setup. The `onboarding_interviewer` agent asks clarifying questions, then emits a draft profile + people + departments. **Nothing is persisted until `POST /onboard/interview/commit`** — the interview itself never writes. `start` is multipart (description + optional deck/one-pager uploads). |\n| `onboarding` (fallback) | `/onboard/start`, `/onboard/answer`, `/onboard/status/{id}` | The original 12-step form wizard, kept as a fallback (UI at `/onboard?mode=form`) and used by the `openexecutive onboard` CLI. |\n| `company-profile` | `GET\\|PATCH /company-profile` | Structured company profile. |\n| `auth` | `GET /auth/allowed-emails` | Allowed sign-in emails. |\n| `fixtures` | `/fixtures`, `/fixtures/{name}/load` | Demo-company fixture load/reset/snapshot. |\n| `clients` | `/clients`, `/clients/generate`, `/clients/cockpit`, `/clients/save`, `/clients/{slug}/activate` | Client-company slots — multi-client mode for fractional use; create from current/blank/AI-generated intake draft (pasted notes and/or file attachments, multipart on `/clients/generate`), save-back switching, engagement metadata (PATCH), cross-client cockpit, delete. See the Client Companies section. |\n| webhooks | `POST /webhook/telegram`, `POST /webhook/google-chat` | Inbound integration callbacks (self-verified). |\n| MCP server | `POST\\|GET /mcp` | Open Executive exposed as an MCP server (Streamable-HTTP). Not a router — a mounted FastMCP sub-app. Same shared-secret gate (`x-api-key`); exposes company context as resources and the specialist council as tools. See the MCP Server section. |",
"mermaid": null,
"generated_at": "2026-09-12T12:03:38.574978+00:00"
}
diff --git a/packages/core/openexecutive/architecture/prebuilt/integrations.json b/packages/core/openexecutive/architecture/prebuilt/integrations.json
index dae618fb..1f2d7363 100644
--- a/packages/core/openexecutive/architecture/prebuilt/integrations.json
+++ b/packages/core/openexecutive/architecture/prebuilt/integrations.json
@@ -1,7 +1,7 @@
{
"section_id": "integrations",
"title": "Integrations",
- "markdown": "External channels connect callers to the Executive. Each inbound adapter resolves the sender to a Person via `find_person_by_{slack,discord,telegram,email}_id` and threads `person_id` into `Executive.chat()`.\n\n### Inbound channels\n\n- **Slack** (`integrations.slack_bot`) — Bolt socket-mode; responds to `@mention`s, DMs, and thread continuations.\n- **Discord** (`integrations.discord_bot`) — discord.py gateway embedded in the lifespan; per-thread auto-titling, session-scoped memory, author attribution, configurable `@mention` thread promotion.\n- **Telegram** — webhook gating + per-chat history.\n- **Google Chat** — webhook + service-account auth.\n- **Email** (`integrations.email_poller`) — IMAP poll loop; strips Reply-To, filters auto-replies.\n\nSlack, Discord, and Telegram are roster-gated on inbound: an unrostered sender's message is silently dropped (`details.outcome=\"rejected_unknown_sender\"`). Email is not inbound-gated — non-roster mail flows through with a `[POLICY]` notice so triage can classify it. Slack/Discord thread continuations pass the shared `integrations.response_gate` (utility-fast model, biased toward replying).\n\n### Attachments\n\n`integrations.attachments` handles web chat, Discord, and Telegram. Images become Anthropic vision blocks; documents (PDF/DOCX/TXT/MD/CSV) are inlined and indexed into the ChromaDB `company_docs` collection for future RAG. 20 MB/file cap; web chat uses `POST /chat/upload`.\n\n### MCP gateway & outbound\n\nMCP runs in **both directions**. As a *client*, `orchestrator.mcp_gateway.MCPGateway` proxies remote MCP servers (Gmail, Calendar, GitHub) as Executive tools, with egress gating on Gmail recipients, Calendar attendees, and Drive share grantees against the roster (and a refusal of public / whole-domain Drive sharing). The Google Workspace server (`workspace-mcp`) behind those Gmail/Calendar/Drive tools runs **co-located inside the API** as a stdio child of the gateway — fitting the single-tenant model (one install = one company = one exec Google account), so there's no separate service to operate. It's baked into the API image and launched by `docker/workspace-mcp-launch.sh`, which selects single-user OAuth (token persisted at `/data/google_credentials`) or service-account domain-wide delegation from `GWORKSPACE_AUTH_MODE`. The gateway forwards the Google secrets to the child via `_FORWARDED_ENV_VARS`. As a *server*, `openexecutive.mcp_server` exposes OE's own company context and specialist council to external MCP clients at `/mcp` — the inverse direction; see the dedicated **MCP Server** section. The gateway launches extensible-mcp as a `uvx` subprocess whose fastembed tool-search index uses an embedding model baked into the API image and loaded offline, so it no longer re-downloads that model from the Hugging Face Hub on every cold start. Because the MCP stdio client does not inherit the parent environment, `MCPGateway.start()` explicitly forwards the embedding-cache/offline env vars (`FASTEMBED_CACHE_PATH`, `HF_HUB_OFFLINE`, …) into the child. Outbound tools: per-Person DMs (`send_slack_dm`, `send_discord_dm`, `send_telegram_message`) plus room-targeted `send_department_message` and `send_company_broadcast`. The per-Person DMs are additionally rate-limited and de-duplicated by the outbound anti-spam guard (below).\n\n### Outbound anti-spam guard\n\nEvery per-Person DM passes a deterministic guard (`orchestrator.outbound_guard.check_outbound_allowed`) after the roster gate and recipient-id recovery but before the network send — the single chokepoint **all** proactive DM paths reach (ad-hoc chat sends, the scheduler's followup/nudge dispatch, principal-brief delivery, the reflection turn, and research-finding routing). On a violation it returns a human-readable reason and the handler **suppresses** the send (`{\"status\":\"suppressed\",\"reason\":…}`), writes a `tool_invocation` audit row with `suppressed=true`, and writes **no** `done` activity row — so a blocked attempt never counts toward its own rate cap, and the Executive can read the reason and hold or `schedule_followup` for the recipient's next window. Three controls: **content dedup** (a near-identical message — whitespace/case-normalized, first 160 chars — already sent to the same recipient within `OUTBOUND_DEDUP_WINDOW_MINUTES`, default 360); a **per-recipient rate cap** (`OUTBOUND_MAX_PER_RECIPIENT_PER_WINDOW`, default 5, over `OUTBOUND_RATE_WINDOW_MINUTES`, default 60); and **quiet hours / availability** (`OUTBOUND_RESPECT_QUIET_HOURS`, default on — refuse while the recipient is on leave or outside their `people.channel` availability windows, surfacing the next open window). Delivery history is read from the existing `status='done'` `scheduled_actions` rows (`episodic.recent_sends_for_channel_ref`), so there is no new table. The guard is **fail-open**: any internal lookup error returns \"allow\", so it can never turn a working send into a crash or a silently-dropped message. Principal briefs and the internal heartbeat kinds use the `__internal__` channel and never reach these handlers, so they are unaffected.\n\n### Calendar & Google Meet\n\nThe Executive books meetings through two typed tools over the Calendar MCP. `create_calendar_event` schedules a **future** meeting (propose→approve by default, or auto-execute once that decision class is promoted) and may be used proactively when a stalled goal warrants a sync. `create_instant_meeting` spins up an **impromptu call starting now**, auto-executing without the approval step and skipping the business-hours/horizon caps, so the Executive can hand back a join link immediately. Both attach a **Google Meet** link by default (`add_google_meet`; opt out for in-person), parsed from the created event's `conferenceData` and surfaced to the caller plus persisted on the decision ledger. Either way the caps that matter still hold — roster membership, principal-protection, max attendees, and a daily booking ceiling.\n\n**Around-the-meeting participation.** The Executive doesn't join the call as a live voice; instead, on every successful booking it schedules a best-effort post-meeting follow-up (~5 min after the end) that DMs a human attendee asking for a quick recap — key decisions and action items. The reply flows back through the same cross-session DM context linkage described below, so the Executive can act on it (summarize, follow up) in a later turn. No meeting-transcript or Drive integration is required.\n\n### Cross-session DM context\n\nWhen the principal says \"DM Alex about X\" in one session, OE sends the DM and — if a live conversation is active — records an `outbound_context` linkage row (recipient, full outbound text, originating session id). Email is covered too but recorded one layer deeper — there is no `send_email` tool, so the linkage is written in `orchestrator.mcp_gateway` after a successful `google_workspace__send_gmail_message`, one row per `to`/`cc` recipient keyed by the bare lowercased address (OE's own address skipped). When Alex later replies on the 1:1 DM path — or by email — the shared helper `integrations.inbound_hydration.hydrate_user_message` (called by the Discord, Telegram, and Slack handlers, and by the email poller keyed on the sender address) looks up the most-recent `open` linkage for that sender within 72h and prepends an `<outbound_reply_context>` block — the outbound text plus a backstory excerpt from the originating session — to the message handed to `Executive.chat()`. So OE knows what the reply is about instead of starting from an empty per-person session. The block lands in the user turn (caching rule) and is injected into a copy only — the pristine reply is persisted to history. The linkage is one-shot (consumed on first matched reply, race-safe) so an unrelated later DM isn't re-hydrated. This is deterministic and independent of the per-person `peer_memory` (Honcho) layer.",
+ "markdown": "External channels connect callers to the Executive. Each inbound adapter resolves the sender to a Person via `find_person_by_{slack,discord,telegram,email}_id` and threads `person_id` into `Executive.chat()`.\n\n### Inbound channels\n\n- **Slack** (`integrations.slack_bot`) — Bolt socket-mode; responds to `@mention`s, DMs, and thread continuations.\n- **Discord** (`integrations.discord_bot`) — discord.py gateway embedded in the lifespan; per-thread auto-titling, session-scoped memory, author attribution, configurable `@mention` thread promotion.\n- **Telegram** — webhook gating + per-chat history.\n- **Google Chat** — webhook + service-account auth.\n- **Email** (`integrations.email_poller`) — IMAP poll loop; strips Reply-To, filters auto-replies.\n\nSlack, Discord, and Telegram are roster-gated on inbound: an unrostered sender's message is silently dropped (`details.outcome=\"rejected_unknown_sender\"`). Email is not inbound-gated — non-roster mail flows through with a `[POLICY]` notice so triage can classify it. Slack/Discord thread continuations pass the shared `integrations.response_gate` (utility-fast model, biased toward replying).\n\n### Attachments\n\n`integrations.attachments` handles web chat, Discord, and Telegram. Images become Anthropic vision blocks; documents (PDF/DOCX/TXT/MD/CSV) are inlined and indexed into the ChromaDB `company_docs` collection for future RAG. 20 MB/file cap; web chat uses `POST /chat/upload`.\n\nIndexed chunks are keyed on `attachment:<bare filename>`, under the `general` domain. Both parts matter. The key is the chunk-id namespace and the retrieval citation, and an attachment's name is chosen by whoever sent the message — unprefixed, an emailed `strategy-2026.md` would upsert straight over the curated company document of that name, so inbound content stays in its own namespace and its provenance shows up in the `[filename]` citation. The domain was previously `company_docs`, which is not one of the specialist domains, so no specialist could retrieve an attachment at all; `general` is the catch-all every specialist's company filter includes.\n\n### MCP gateway & outbound\n\nMCP runs in **both directions**. As a *client*, `orchestrator.mcp_gateway.MCPGateway` proxies remote MCP servers (Gmail, Calendar, GitHub) as Executive tools, with egress gating on Gmail recipients, Calendar attendees, and Drive share grantees against the roster (and a refusal of public / whole-domain Drive sharing). The Google Workspace server (`workspace-mcp`) behind those Gmail/Calendar/Drive tools runs **co-located inside the API** as a stdio child of the gateway — fitting the single-tenant model (one install = one company = one exec Google account), so there's no separate service to operate. It's baked into the API image and launched by `docker/workspace-mcp-launch.sh`, which selects single-user OAuth (token persisted at `/data/google_credentials`) or service-account domain-wide delegation from `GWORKSPACE_AUTH_MODE`. The gateway forwards the Google secrets to the child via `_FORWARDED_ENV_VARS`. As a *server*, `openexecutive.mcp_server` exposes OE's own company context and specialist council to external MCP clients at `/mcp` — the inverse direction; see the dedicated **MCP Server** section. The gateway launches extensible-mcp as a `uvx` subprocess whose fastembed tool-search index uses an embedding model baked into the API image and loaded offline, so it no longer re-downloads that model from the Hugging Face Hub on every cold start. Because the MCP stdio client does not inherit the parent environment, `MCPGateway.start()` explicitly forwards the embedding-cache/offline env vars (`FASTEMBED_CACHE_PATH`, `HF_HUB_OFFLINE`, …) into the child. Outbound tools: per-Person DMs (`send_slack_dm`, `send_discord_dm`, `send_telegram_message`) plus room-targeted `send_department_message` and `send_company_broadcast`. The per-Person DMs are additionally rate-limited and de-duplicated by the outbound anti-spam guard (below).\n\n### Outbound anti-spam guard\n\nEvery per-Person DM passes a deterministic guard (`orchestrator.outbound_guard.check_outbound_allowed`) after the roster gate and recipient-id recovery but before the network send — the single chokepoint **all** proactive DM paths reach (ad-hoc chat sends, the scheduler's followup/nudge dispatch, principal-brief delivery, the reflection turn, and research-finding routing). On a violation it returns a human-readable reason and the handler **suppresses** the send (`{\"status\":\"suppressed\",\"reason\":…}`), writes a `tool_invocation` audit row with `suppressed=true`, and writes **no** `done` activity row — so a blocked attempt never counts toward its own rate cap, and the Executive can read the reason and hold or `schedule_followup` for the recipient's next window. Three controls: **content dedup** (a near-identical message — whitespace/case-normalized, first 160 chars — already sent to the same recipient within `OUTBOUND_DEDUP_WINDOW_MINUTES`, default 360); a **per-recipient rate cap** (`OUTBOUND_MAX_PER_RECIPIENT_PER_WINDOW`, default 5, over `OUTBOUND_RATE_WINDOW_MINUTES`, default 60); and **quiet hours / availability** (`OUTBOUND_RESPECT_QUIET_HOURS`, default on — refuse while the recipient is on leave or outside their `people.channel` availability windows, surfacing the next open window). Delivery history is read from the existing `status='done'` `scheduled_actions` rows (`episodic.recent_sends_for_channel_ref`), so there is no new table. The guard is **fail-open**: any internal lookup error returns \"allow\", so it can never turn a working send into a crash or a silently-dropped message. Principal briefs and the internal heartbeat kinds use the `__internal__` channel and never reach these handlers, so they are unaffected.\n\n### Calendar & Google Meet\n\nThe Executive books meetings through two typed tools over the Calendar MCP. `create_calendar_event` schedules a **future** meeting (propose→approve by default, or auto-execute once that decision class is promoted) and may be used proactively when a stalled goal warrants a sync. `create_instant_meeting` spins up an **impromptu call starting now**, auto-executing without the approval step and skipping the business-hours/horizon caps, so the Executive can hand back a join link immediately. Both attach a **Google Meet** link by default (`add_google_meet`; opt out for in-person), parsed from the created event's `conferenceData` and surfaced to the caller plus persisted on the decision ledger. Either way the caps that matter still hold — roster membership, principal-protection, max attendees, and a daily booking ceiling.\n\n**Around-the-meeting participation.** The Executive doesn't join the call as a live voice; instead, on every successful booking it schedules a best-effort post-meeting follow-up (~5 min after the end) that DMs a human attendee asking for a quick recap — key decisions and action items. The reply flows back through the same cross-session DM context linkage described below, so the Executive can act on it (summarize, follow up) in a later turn. No meeting-transcript or Drive integration is required.\n\n### Cross-session DM context\n\nWhen the principal says \"DM Alex about X\" in one session, OE sends the DM and — if a live conversation is active — records an `outbound_context` linkage row (recipient, full outbound text, originating session id). Email is covered too but recorded one layer deeper — there is no `send_email` tool, so the linkage is written in `orchestrator.mcp_gateway` after a successful `google_workspace__send_gmail_message`, one row per `to`/`cc` recipient keyed by the bare lowercased address (OE's own address skipped). When Alex later replies on the 1:1 DM path — or by email — the shared helper `integrations.inbound_hydration.hydrate_user_message` (called by the Discord, Telegram, and Slack handlers, and by the email poller keyed on the sender address) looks up the most-recent `open` linkage for that sender within 72h and prepends an `<outbound_reply_context>` block — the outbound text plus a backstory excerpt from the originating session — to the message handed to `Executive.chat()`. So OE knows what the reply is about instead of starting from an empty per-person session. The block lands in the user turn (caching rule) and is injected into a copy only — the pristine reply is persisted to history. The linkage is one-shot (consumed on first matched reply, race-safe) so an unrelated later DM isn't re-hydrated. This is deterministic and independent of the per-person `peer_memory` (Honcho) layer.",
"mermaid": "flowchart TD\n Channels[\"Slack / Discord<br/>Telegram / GChat / Email\"]\n Resolve[\"find_person_by_*_id\"]\n Gate[\"roster gate\"]\n Drop[\"drop unrostered\"]\n Exec[\"Executive.chat()\"]\n Attach[\"attachments<br/>(vision / company_docs)\"]\n MCP[\"MCP gateway\"]\n Guard[\"outbound anti-spam guard<br/>(dedup / rate cap / quiet hours)\"]\n Out[\"DM / department<br/>/ broadcast tools\"]\n Channels --> Resolve\n Resolve --> Gate\n Gate -- \"unknown sender\" --> Drop\n Gate -- \"Person\" --> Exec\n Channels --> Attach\n Attach --> Exec\n Exec --> MCP\n Exec --> Guard\n Guard -- \"suppress\" --> Drop\n Guard -- \"allow\" --> Out\n Out --> Channels\n class Channels entry\n class Resolve compute\n class Gate compute\n class Drop external\n class Exec hot\n class Attach compute\n class MCP external\n class Guard compute\n class Out compute",
"generated_at": "2026-06-03T12:30:00+00:00"
}
diff --git a/packages/core/openexecutive/architecture/prebuilt/rag.json b/packages/core/openexecutive/architecture/prebuilt/rag.json
index a09bb6a4..a8c2da11 100644
--- a/packages/core/openexecutive/architecture/prebuilt/rag.json
+++ b/packages/core/openexecutive/architecture/prebuilt/rag.json
@@ -1,7 +1,7 @@
{
"section_id": "rag",
"title": "Knowledge & RAG",
- "markdown": "Retrieval is handled by `openexecutive.knowledge.retriever.retrieve`, querying ChromaDB collections through `openexecutive.knowledge.store.ChromaDBStore`. Five collections back the system:\n\n- **BUILTIN** \u2014 curated MBA-style domain knowledge shipped with the repo.\n- **COMPANY** \u2014 customer documents ingested via the `/documents` endpoint. Curated uploads only \u2014 synced Notion pages do not land here.\n- **NOTION_WIKI** \u2014 optional Notion sync (`NOTION_SYNC_ENABLED`) incrementally re-indexes pages shared with a Notion internal integration into a separate `notion_wiki` collection (`type=notion`). A Notion share is inherited and multi-writer, so this text is unreviewed relative to curated company docs; the retriever labels it as such and ranks it below COMPANY. Un-shared pages are purged on the next tick.\n- **FAILURES** \u2014 recorded failure cases (queried by `retrieve_failures`) teaching specialists what to avoid.\n- **RECENT_RESEARCH** \u2014 web-research artifacts persisted from `executive_research` runs, kept separate from COMPANY so unvetted machine-generated text never blends into curated knowledge, and ranked below company documents.\n\nEach parallel specialist call runs its own domain-filtered query. When no explicit filter is passed, `DOMAIN_ALIASES` maps the specialist key to its domains \u2014 e.g. `cso`\u2192`strategy`, `cpo`\u2192`product`,`strategy`, `board_comms`\u2192`board`,`finance`. Hits beyond a cosine distance threshold are dropped, and very short messages (greetings) bypass retrieval entirely.\n\nMulti-domain chunks fan out to one row per declared domain, so `_dedupe_by_text` drops duplicate-text hits while preserving relevance order before results are returned.\n\nThe formatted context is injected into the **user turn** (wrapped in `<relevant_knowledge>` by `BaseAgent.analyze`), never into a cached system block \u2014 per-query hits would otherwise poison the prompt cache.",
+ "markdown": "Retrieval is handled by `openexecutive.knowledge.retriever.retrieve`, querying ChromaDB collections through `openexecutive.knowledge.store.ChromaDBStore`. Five collections back the system:\n\n- **BUILTIN** — curated MBA-style domain knowledge shipped with the repo.\n- **COMPANY** — customer documents ingested via the `/documents` endpoint. Curated uploads only — synced Notion pages do not land here. A document's identity is its **filename**: chunk metadata and the chunk id both derive from it, so re-uploading upserts in place rather than duplicating, and `DELETE /documents/{filename}` matches the rows it claims to remove. Chat/email attachments share the collection but are namespaced `attachment:<name>`, since the sender picks that name and an unprefixed one would overwrite the curated document it collides with.\n- **NOTION_WIKI** — optional Notion sync (`NOTION_SYNC_ENABLED`) incrementally re-indexes pages shared with a Notion internal integration into a separate `notion_wiki` collection (`type=notion`). A Notion share is inherited and multi-writer, so this text is unreviewed relative to curated company docs; the retriever labels it as such and ranks it below COMPANY. Un-shared pages are purged on the next tick.\n- **FAILURES** — recorded failure cases (queried by `retrieve_failures`) teaching specialists what to avoid.\n- **RECENT_RESEARCH** — web-research artifacts persisted from `executive_research` runs, kept separate from COMPANY so unvetted machine-generated text never blends into curated knowledge, and ranked below company documents.\n\nEach parallel specialist call runs its own domain-filtered query. When no explicit filter is passed, `DOMAIN_ALIASES` maps the specialist key to its domains — e.g. `cso`→`strategy`, `cpo`→`product`,`strategy`, `board_comms`→`board`,`finance`. Hits beyond a cosine distance threshold are dropped, and very short messages (greetings) bypass retrieval entirely.\n\n`POST /documents` rejects any domain outside that set with a 400 — an unvalidated string would index the document under a domain no specialist filters on, failing silently and only at retrieval time. The ninth accepted value, `general`, is the catch-all for an upload nobody classified (it is the default on the endpoint and in the UI's picker). It maps to no specialist, so every specialist's **company** filter is widened to include it — an unclassified document is visible to all rather than to none. That widening is deliberately not applied to BUILTIN, whose rows are shared with external OER sources that fall back to `general` when they declare no domains; fanning those into every specialist would blend unvetted third-party material into every answer.\n\nMulti-domain chunks fan out to one row per declared domain, so `_dedupe_by_text` drops duplicate-text hits while preserving relevance order before results are returned.\n\nThe formatted context is injected into the **user turn** (wrapped in `<relevant_knowledge>` by `BaseAgent.analyze`), never into a cached system block — per-query hits would otherwise poison the prompt cache.",
"mermaid": "flowchart TD\n Call[\"Specialist call<br/>(domain-filtered)\"] --> Q[\"retrieve(query, domain)\"]\n Q --> Builtin[(BUILTIN)]\n Q --> Company[(COMPANY)]\n Q --> NotionWiki[(NOTION_WIKI)]\n Q --> Failures[(FAILURES)]\n Q --> Research[(RECENT_RESEARCH)]\n Builtin --> Dedupe[\"dedupe by text<br/>+ distance gate\"]\n Company --> Dedupe\n NotionWiki --> Dedupe\n Failures --> Dedupe\n Research --> Dedupe\n Dedupe --> Inject[\"Inject into USER turn<br/>(not cached)\"]\n class Call entry\n class Q compute\n class Dedupe compute\n class Builtin storage\n class Company storage\n class NotionWiki storage\n class Failures storage\n class Research storage\n class Inject hot",
"generated_at": "2026-08-31T00:00:00+00:00"
}
diff --git a/packages/core/openexecutive/integrations/attachments.py b/packages/core/openexecutive/integrations/attachments.py
index cd2ea50b..f939b2e8 100644
--- a/packages/core/openexecutive/integrations/attachments.py
+++ b/packages/core/openexecutive/integrations/attachments.py
@@ -158,7 +158,7 @@ def _schedule_ingest(data: bytes, filename: str) -> None:
async def _run() -> None:
suffix = _suffix_from_filename(filename)
try:
- from openexecutive.knowledge.loader import ingest_file
+ from openexecutive.knowledge.loader import GENERAL_DOMAIN, ingest_file
from openexecutive.knowledge.store import ChromaDBStore
store = ChromaDBStore()
@@ -167,7 +167,28 @@ def _schedule_ingest(data: bytes, filename: str) -> None:
tmp_path = Path(tmp.name)
try:
- count = await ingest_file(tmp_path, store, domain="company_docs")
+ # `source_name` is the real attachment name: `tmp_path` is a
+ # random staging name, and indexing under it both duplicates
+ # on every re-send and leaves chunks no API call can delete.
+ #
+ # It is PREFIXED, and stripped to a bare name, because an
+ # attachment name is chosen by whoever sent the message. The
+ # name is the chunk-id namespace, so an unprefixed
+ # "strategy-2026.md" arriving by email would upsert straight
+ # over the curated company document of that name. The prefix
+ # keeps inbound content in its own namespace and makes the
+ # provenance visible in the `[filename]` retrieval citation —
+ # the same reason Notion and research artifacts are isolated.
+ #
+ # `domain` is the catch-all rather than the old "company_docs"
+ # — that was not one of the specialist domains, so every
+ # attachment landed where no specialist could retrieve it.
+ count = await ingest_file(
+ tmp_path,
+ store,
+ domain=GENERAL_DOMAIN,
+ source_name=f"attachment:{Path(filename).name}",
+ )
logger.info(
"attachments: indexed %d chunks from %s into ChromaDB",
count,
diff --git a/packages/core/openexecutive/knowledge/loader.py b/packages/core/openexecutive/knowledge/loader.py
index 8cef28df..00fb9a58 100644
--- a/packages/core/openexecutive/knowledge/loader.py
+++ b/packages/core/openexecutive/knowledge/loader.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import hashlib
import logging
+import re
from pathlib import Path
from typing import Any
@@ -23,6 +24,16 @@ DOMAIN_MAP: dict[str, str] = {
"product": "product",
}
+# The catch-all domain for company documents the uploader didn't classify.
+# Unlike the eight specialist domains it maps to no single specialist — every
+# specialist retrieves it (see ``retriever.retrieve``), so an unclassified
+# upload is visible to all rather than to none.
+GENERAL_DOMAIN = "general"
+
+# Domains ``POST /documents`` accepts. Anything else is a typo that would
+# silently index the document where no specialist can ever retrieve it.
+UPLOAD_DOMAINS: frozenset[str] = frozenset(DOMAIN_MAP) | {GENERAL_DOMAIN}
+
def chunk_text(text: str, chunk_size: int = 512, overlap: int = 50) -> list[str]:
words = text.split()
@@ -130,11 +141,30 @@ async def ingest_file(
store: ChromaDBStore,
domain: str | None = None,
collection: str = ChromaDBStore.COMPANY_COLLECTION,
+ *,
+ source_name: str | None = None,
) -> int:
+ """Index a file on disk into a knowledge collection.
+
+ ``source_name`` is the *logical* identity of the document — it becomes the
+ ``filename``/``source`` metadata and the chunk-id namespace, exactly as in
+ ``ingest_text_sync``. Callers that stage an upload through a temp file MUST
+ pass the real filename: ``path`` is then a random ``tmpXXXXXXXX`` name, and
+ deriving identity from it gives every re-upload fresh chunk ids (so the
+ id-keyed upsert never collides and duplicates without bound) and stores a
+ ``filename`` that ``DELETE /documents/{filename}`` can never match.
+
+ Defaults to ``path.name`` rather than ``str(path)`` so that identity is the
+ bare filename for every caller. That keeps an uploaded ``plan.md`` and a
+ fixture-loaded ``plan.md`` on the same ids instead of duplicating each
+ other, and matches how ``DELETE`` and ``list_company_docs`` already treat
+ filename as the document's identity.
+ """
text = extract_text_from_file(path)
if not text.strip():
return 0
+ name = source_name or path.name
inferred_domain = domain or infer_domain_from_path(path)
chunks = chunk_text(text, chunk_size=512, overlap=50)
@@ -142,13 +172,13 @@ async def ingest_file(
metadatas: list[dict[str, Any]] = [
{
"domain": inferred_domain,
- "filename": path.name,
- "source": str(path),
+ "filename": name,
+ "source": name,
"chunk_index": i,
}
for i in range(len(chunks))
]
- ids = [_make_chunk_id(str(path), i) for i in range(len(chunks))]
+ ids = [_make_chunk_id(name, i) for i in range(len(chunks))]
store.add_documents(texts=texts, metadatas=metadatas, ids=ids, collection=collection)
return len(chunks)
@@ -276,6 +306,64 @@ def list_company_docs(docs_dir: Path) -> list[dict[str, Any]]:
]
+# Names produced by ``tempfile.NamedTemporaryFile(suffix=ext)`` — "tmp" plus a
+# random stem, e.g. "tmp1du9epr4.md". Uploads used to be indexed under these
+# (the staging path was passed to ``ingest_file`` instead of the real name), so
+# deployed stores hold chunks no filename can ever match.
+_TEMP_CHUNK_NAME = re.compile(r"^tmp[A-Za-z0-9_]{6,12}\.[A-Za-z0-9]{1,5}$")
+
+
+async def reconcile_company_docs(
+ store: ChromaDBStore,
+ docs_dir: Path,
+ collection: str = ChromaDBStore.COMPANY_COLLECTION,
+) -> tuple[int, int]:
+ """Sweep temp-named orphans, then index documents that have no chunks.
+
+ Returns ``(orphans_deleted, files_indexed)``. Safe to run on every boot.
+
+ Only files with *no* rows in the collection are indexed. A document that is
+ already indexed under its real filename is left untouched — its ``domain``
+ came from whatever the uploader chose, and the on-disk copy carries no
+ record of that, so re-ingesting would silently retag a ``finance`` document
+ as ``general``. Recovered documents do land under ``general``, which is the
+ honest answer (their domain died with the orphaned rows) and is retrievable
+ by every specialist rather than by none.
+
+ Documents whose only copy was the orphaned index — chat/email attachments,
+ which are never written to ``docs_dir`` — cannot be recovered and are
+ dropped by the sweep.
+ """
+ live = {p.name for p in docs_dir.iterdir() if p.is_file()} if docs_dir.is_dir() else set()
+
+ orphans: list[str] = []
+ indexed_names: set[str] = set()
+ for cid, md in store.iter_chunk_metadata(collection):
+ name = md.get("filename")
+ if not isinstance(name, str):
+ continue
+ if _TEMP_CHUNK_NAME.match(name) and name not in live:
+ orphans.append(cid)
+ else:
+ indexed_names.add(name)
+
+ store.delete_by_ids(collection, orphans)
+
+ indexed = 0
+ for doc_name in sorted(live - indexed_names):
+ if doc_name.startswith("."):
+ continue
+ try:
+ if await ingest_file(docs_dir / doc_name, store, collection=collection):
+ indexed += 1
+ except Exception:
+ # One unreadable document must not abort the reconcile —
+ # the rest of the index is still worth repairing.
+ logger.exception("reconcile_company_docs: index failed: %s", doc_name)
+
+ return len(orphans), indexed
+
+
async def seed_builtin_knowledge(
store: ChromaDBStore | None = None,
force: bool = False,
diff --git a/packages/core/openexecutive/knowledge/retriever.py b/packages/core/openexecutive/knowledge/retriever.py
index 99493152..97810fa8 100644
--- a/packages/core/openexecutive/knowledge/retriever.py
+++ b/packages/core/openexecutive/knowledge/retriever.py
@@ -5,6 +5,7 @@ from typing import Any
from openexecutive.audit import get_active_ids
from openexecutive.audit import log_event as _audit_log
+from openexecutive.knowledge.loader import GENERAL_DOMAIN
from openexecutive.knowledge.review_store import (
PRIORITY_ORDER,
ContentType,
@@ -159,6 +160,26 @@ DOMAIN_ALIASES: dict[str, list[str]] = {
}
+def _with_general(domains: list[str] | None) -> list[str] | None:
+ """Widen a company-document domain filter to include the catch-all.
+
+ ``general`` is the domain an upload gets when the uploader did not classify
+ it — it is the default in the UI's picker and the default on
+ ``POST /documents``. It maps to no specialist, so without this an
+ unclassified company document is retrievable by *no* specialist rather than
+ by all of them, silently, with nothing in the API to reveal it.
+
+ Deliberately applied to the COMPANY collection only. The builtin collection
+ shares its rows with external OER sources, which fall back to ``general``
+ when a source declares no domains (``external_sources``); fanning those
+ into every specialist would blend unvetted third-party material into every
+ answer. ``None`` means "no domain filter" and already matches everything.
+ """
+ if domains is None:
+ return None
+ return domains if GENERAL_DOMAIN in domains else [*domains, GENERAL_DOMAIN]
+
+
def retrieve(
query: str,
domain_filter: list[str] | None = None,
@@ -252,7 +273,7 @@ def retrieve(
raw_company = store.query(
query_text=query,
collection=ChromaDBStore.COMPANY_COLLECTION,
- domain_filter=effective_domains,
+ domain_filter=_with_general(effective_domains),
n_results=n_company,
)
company_results = [
diff --git a/packages/core/openexecutive/knowledge/store.py b/packages/core/openexecutive/knowledge/store.py
index 1d691670..96def478 100644
--- a/packages/core/openexecutive/knowledge/store.py
+++ b/packages/core/openexecutive/knowledge/store.py
@@ -142,6 +142,36 @@ class ChromaDBStore(KnowledgeStore):
except Exception:
pass
+ def iter_chunk_metadata(self, collection: str) -> list[tuple[str, dict[str, Any]]]:
+ """Return every ``(chunk_id, metadata)`` pair in *collection*.
+
+ Chroma's ``where`` has no prefix/substring operator, so metadata
+ patterns (rather than exact matches) have to be filtered in Python.
+ Only used against the small ``company_docs`` collection.
+ """
+ try:
+ col = self._get_or_create_collection(collection)
+ rows = col.get(include=["metadatas"])
+ except Exception:
+ return []
+ ids = rows.get("ids") or []
+ metas = rows.get("metadatas") or []
+ return [
+ (str(cid), dict(md) if isinstance(md, dict) else {})
+ for cid, md in zip(ids, metas, strict=False)
+ ]
+
+ def delete_by_ids(self, collection: str, ids: list[str]) -> None:
+ """Delete specific chunk ids. No-op on an empty list (Chroma treats a
+ delete with neither ids nor where as 'delete everything')."""
+ if not ids:
+ return
+ try:
+ col = self._get_or_create_collection(collection)
+ col.delete(ids=ids)
+ except Exception:
+ pass
+
def delete_company_docs(self) -> None:
"""Delete and recreate the company_docs collection, clearing all indexed documents."""
import contextlib
diff --git a/packages/core/tests/unit/test_attachments.py b/packages/core/tests/unit/test_attachments.py
index 4ff7635c..b8c77b41 100644
--- a/packages/core/tests/unit/test_attachments.py
+++ b/packages/core/tests/unit/test_attachments.py
@@ -241,3 +241,62 @@ async def test_process_attachments_concatenates_multiple_texts():
assert "a.txt" in extra_text
assert "b.txt" in extra_text
assert image_blocks == []
+
+
+# ── #113/#114 at the attachment ingest path ───────────────────────────────
+
+
+def _ingest_call(filename: str, data: bytes = b"# Notes\nRevenue grew."):
+ """Run `_schedule_ingest` to completion and return the `ingest_file` call."""
+ from openexecutive.integrations import attachments as att
+
+ mock_ingest = AsyncMock(return_value=3)
+ with (
+ patch("openexecutive.knowledge.loader.ingest_file", mock_ingest),
+ patch("openexecutive.knowledge.store.ChromaDBStore", MagicMock()),
+ ):
+
+ async def _drive() -> None:
+ att._schedule_ingest(data, filename)
+ # `_schedule_ingest` fires a background task; let it run.
+ await asyncio.sleep(0)
+ await asyncio.sleep(0)
+
+ asyncio.run(_drive())
+
+ mock_ingest.assert_called_once()
+ return mock_ingest.call_args
+
+
+def test_attachment_is_indexed_under_its_real_name_not_the_temp_path():
+ """The bug: the staging temp path was passed straight to `ingest_file`, so
+ every re-send duplicated and no chunk could ever be deleted."""
+ _, kwargs = _ingest_call("board-deck.md")
+
+ assert kwargs["source_name"] == "attachment:board-deck.md"
+
+
+def test_attachment_name_cannot_collide_with_a_company_document():
+ """`source_name` is the chunk-id namespace, and an attachment name is
+ chosen by whoever sent the message — without the prefix an emailed
+ `strategy-2026.md` would upsert over the curated document of that name."""
+ _, kwargs = _ingest_call("strategy-2026.md")
+
+ assert kwargs["source_name"] != "strategy-2026.md"
+ assert kwargs["source_name"].startswith("attachment:")
+
+
+def test_attachment_name_is_stripped_to_a_bare_name():
+ _, kwargs = _ingest_call("../../etc/passwd.md")
+
+ assert kwargs["source_name"] == "attachment:passwd.md"
+
+
+def test_attachment_lands_in_a_retrievable_domain():
+ """It used to be tagged "company_docs", which is not one of the specialist
+ domains, so no specialist could ever retrieve an attachment."""
+ from openexecutive.knowledge.loader import UPLOAD_DOMAINS
+
+ _, kwargs = _ingest_call("board-deck.md")
+
+ assert kwargs["domain"] in UPLOAD_DOMAINS
diff --git a/packages/core/tests/unit/test_company_doc_identity.py b/packages/core/tests/unit/test_company_doc_identity.py
new file mode 100644
index 00000000..d2c1efca
--- /dev/null
+++ b/packages/core/tests/unit/test_company_doc_identity.py
@@ -0,0 +1,243 @@
+"""Company-document identity: the orphan sweep and the `general` catch-all.
+
+Companions to `test_documents_upload.py`, covering the two halves of the fix
+that live below the API layer:
+
+* `reconcile_company_docs` — deployed stores still hold chunks the old upload
+ path indexed under a random `tmpXXXXXXXX` staging name (#113). Nothing can
+ reach them: DELETE matches on `filename`, and a re-upload lands on different
+ ids, so they are neither replaceable nor removable.
+* `retriever._with_general` — `general` is the default on `POST /documents`
+ and the default option in the UI's picker, but it maps to no specialist
+ (#114), so an unclassified document was retrievable by none of them.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from openexecutive.knowledge.loader import (
+ GENERAL_DOMAIN,
+ UPLOAD_DOMAINS,
+ _make_chunk_id,
+ reconcile_company_docs,
+)
+from openexecutive.knowledge.retriever import DOMAIN_ALIASES, _with_general
+
+
+class _FakeStore:
+ """Id-keyed store exposing the surface `reconcile_company_docs` uses."""
+
+ def __init__(self, rows: dict[str, dict[str, Any]] | None = None) -> None:
+ self.rows: dict[str, dict[str, Any]] = dict(rows or {})
+
+ def add_documents(
+ self,
+ texts: list[str],
+ metadatas: list[dict[str, Any]],
+ ids: list[str],
+ collection: str,
+ ) -> None:
+ for chunk_id, meta in zip(ids, metadatas, strict=True):
+ self.rows[chunk_id] = meta
+
+ def iter_chunk_metadata(self, collection: str) -> list[tuple[str, dict[str, Any]]]:
+ return list(self.rows.items())
+
+ def delete_by_ids(self, collection: str, ids: list[str]) -> None:
+ for chunk_id in ids:
+ self.rows.pop(chunk_id, None)
+
+ def filenames(self) -> set[str]:
+ return {m["filename"] for m in self.rows.values()}
+
+
+def _orphan(name: str, index: int = 0) -> tuple[str, dict[str, Any]]:
+ """A row as the pre-fix upload path wrote it: temp name, temp-path source."""
+ return (
+ _make_chunk_id(f"/tmp/{name}", index),
+ {"domain": "finance", "filename": name, "source": f"/tmp/{name}", "chunk_index": index},
+ )
+
+
+def _docs_dir(tmp_path: Path) -> Path:
+ d = tmp_path / "docs"
+ d.mkdir()
+ return d
+
+
+# ── the sweep ─────────────────────────────────────────────────────────────
+
+
+async def test_sweep_drops_orphaned_temp_chunks(tmp_path: Path) -> None:
+ store = _FakeStore(dict([_orphan("tmp1du9epr4.md"), _orphan("tmp8qd9liu4.md", 1)]))
+
+ swept, _ = await reconcile_company_docs(store, _docs_dir(tmp_path))
+
+ assert swept == 2
+ assert store.rows == {}
+
+
+async def test_sweep_spares_real_documents(tmp_path: Path) -> None:
+ docs = _docs_dir(tmp_path)
+ (docs / "plan.md").write_text("# Plan\nGrow revenue.")
+ store = _FakeStore(
+ {
+ _make_chunk_id("plan.md", 0): {
+ "domain": "strategy",
+ "filename": "plan.md",
+ "source": "plan.md",
+ "chunk_index": 0,
+ },
+ **dict([_orphan("tmp1du9epr4.md")]),
+ }
+ )
+
+ swept, indexed = await reconcile_company_docs(store, docs)
+
+ assert swept == 1
+ assert indexed == 0, "a document already indexed under its real name is left alone"
+ assert store.filenames() == {"plan.md"}
+
+
+async def test_sweep_preserves_the_domain_of_indexed_documents(tmp_path: Path) -> None:
+ """Re-ingesting from disk would retag everything `general` — the on-disk
+ copy carries no record of the domain the uploader chose."""
+ docs = _docs_dir(tmp_path)
+ (docs / "plan.md").write_text("# Plan\nGrow revenue.")
+ store = _FakeStore(
+ {
+ _make_chunk_id("plan.md", 0): {
+ "domain": "finance",
+ "filename": "plan.md",
+ "source": "plan.md",
+ "chunk_index": 0,
+ }
+ }
+ )
+
+ await reconcile_company_docs(store, docs)
+
+ assert [m["domain"] for m in store.rows.values()] == ["finance"]
+
+
+async def test_sweep_reindexes_a_document_with_no_chunks(tmp_path: Path) -> None:
+ """The recovery path: the file survived on disk, its chunks were orphans."""
+ docs = _docs_dir(tmp_path)
+ (docs / "plan.md").write_text("# Plan\n" + "grow revenue thirty percent. " * 50)
+ store = _FakeStore(dict([_orphan("tmp1du9epr4.md")]))
+
+ swept, indexed = await reconcile_company_docs(store, docs)
+
+ assert (swept, indexed) == (1, 1)
+ assert store.filenames() == {"plan.md"}
+ assert {m["domain"] for m in store.rows.values()} == {GENERAL_DOMAIN}
+
+
+async def test_sweep_is_idempotent(tmp_path: Path) -> None:
+ docs = _docs_dir(tmp_path)
+ (docs / "plan.md").write_text("# Plan\n" + "grow revenue thirty percent. " * 50)
+ store = _FakeStore(dict([_orphan("tmp1du9epr4.md")]))
+
+ await reconcile_company_docs(store, docs)
+ snapshot = dict(store.rows)
+ second = await reconcile_company_docs(store, docs)
+
+ assert second == (0, 0)
+ assert store.rows == snapshot
+
+
+async def test_sweep_spares_a_real_document_named_like_a_temp_file(tmp_path: Path) -> None:
+ docs = _docs_dir(tmp_path)
+ (docs / "tmp1du9epr4.md").write_text("# Notes\nA real file, awkwardly named.")
+ store = _FakeStore(dict([_orphan("tmp1du9epr4.md")]))
+
+ swept, _ = await reconcile_company_docs(store, docs)
+
+ assert swept == 0, "a document still present on disk is never swept"
+
+
+async def test_sweep_tolerates_a_missing_docs_dir(tmp_path: Path) -> None:
+ store = _FakeStore(dict([_orphan("tmp1du9epr4.md")]))
+
+ swept, indexed = await reconcile_company_docs(store, tmp_path / "does-not-exist")
+
+ assert (swept, indexed) == (1, 0)
+
+
+async def test_sweep_continues_past_an_unreadable_document(tmp_path: Path) -> None:
+ docs = _docs_dir(tmp_path)
+ (docs / "broken.pdf").write_bytes(b"not a real pdf")
+ (docs / "plan.md").write_text("# Plan\n" + "grow revenue thirty percent. " * 50)
+ store = _FakeStore()
+
+ _, indexed = await reconcile_company_docs(store, docs)
+
+ assert indexed == 1
+ assert store.filenames() == {"plan.md"}
+
+
+async def test_sweep_ignores_subdirectories(tmp_path: Path) -> None:
+ """Notion sync writes Markdown into docs/notion/, which belongs to the
+ notion_wiki collection — it must not be pulled into company_docs."""
+ docs = _docs_dir(tmp_path)
+ (docs / "notion").mkdir()
+ (docs / "notion" / "page.md").write_text("# Wiki\n" + "wiki content. " * 50)
+ store = _FakeStore()
+
+ swept, indexed = await reconcile_company_docs(store, docs)
+
+ assert (swept, indexed) == (0, 0)
+ assert store.rows == {}
+
+
+def test_delete_by_ids_ignores_an_empty_list() -> None:
+ """Chroma's `col.delete()` with neither `ids` nor `where` deletes the
+ ENTIRE collection, and the sweep passes an empty list whenever there is
+ nothing orphaned — i.e. on every boot of a healthy install."""
+ from openexecutive.knowledge.store import ChromaDBStore
+
+ calls: list[Any] = []
+ store = ChromaDBStore.__new__(ChromaDBStore) # no ChromaDB client needed
+ store._get_or_create_collection = lambda name: calls.append(name) # type: ignore[method-assign]
+
+ store.delete_by_ids("company_docs", [])
+
+ assert calls == [], "an empty id list must not reach Chroma at all"
+
+
+# ── the `general` catch-all ───────────────────────────────────────────────
+
+
+def test_general_is_an_accepted_upload_domain() -> None:
+ assert GENERAL_DOMAIN in UPLOAD_DOMAINS
+
+
+def test_with_general_widens_a_specialist_filter() -> None:
+ assert _with_general(["finance"]) == ["finance", GENERAL_DOMAIN]
+
+
+def test_with_general_does_not_duplicate() -> None:
+ assert _with_general([GENERAL_DOMAIN]) == [GENERAL_DOMAIN]
+
+
+def test_with_general_leaves_an_absent_filter_alone() -> None:
+ assert _with_general(None) is None, "None already matches every domain"
+
+
[email protected]("specialist", sorted(DOMAIN_ALIASES))
+def test_every_specialist_retrieves_general_company_docs(specialist: str) -> None:
+ """The whole point of #114: an unclassified upload must be visible to all
+ specialists rather than to none."""
+ assert GENERAL_DOMAIN in (_with_general(DOMAIN_ALIASES[specialist]) or [])
diff --git a/packages/core/tests/unit/test_documents_upload.py b/packages/core/tests/unit/test_documents_upload.py
index 8d712ae0..11a2504d 100644
--- a/packages/core/tests/unit/test_documents_upload.py
+++ b/packages/core/tests/unit/test_documents_upload.py
@@ -1,9 +1,22 @@
-"""Unit tests for POST /documents — domain must come from the multipart form.
+"""Unit tests for POST/DELETE /documents.
-Regression guard: `domain` was declared as a bare default (`domain: str =
-"general"`), which FastAPI parses as a query parameter. The UI sends it as a
-form field, so it was silently dropped and every upload landed under
-"general" — invisible to domain-filtered specialist retrieval.
+Regression guards, in the order they were found:
+
+1. `domain` was declared as a bare default (`domain: str = "general"`), which
+ FastAPI parses as a query parameter. The UI sends it as a form field, so it
+ was silently dropped and every upload landed under "general".
+
+2. (#113) The handler staged the upload in a `tempfile.NamedTemporaryFile` and
+ passed *that* path to `ingest_file`, which derives chunk metadata AND the
+ chunk id from it. The id is an MD5 of a freshly random path, so the id-keyed
+ upsert never collided: re-uploads appended a whole duplicate chunk set, and
+ the stored `filename` was `tmpXXXXXXXX.md`, which
+ `DELETE /documents/{filename}` could never match — it unlinked the on-disk
+ copy, returned 200, and left every chunk in the store forever.
+
+3. (#114) `domain` was an unvalidated free string, so `domain=finanace`
+ returned 200 and indexed the document where no specialist filters, making it
+ permanently unretrievable with no error and no way to notice.
"""
from __future__ import annotations
@@ -19,11 +32,15 @@ from openexecutive.api.routes import documents
class _CapturingStore:
- """Records the metadata passed to add_documents so the test can assert
- the domain tag without standing up a real ChromaDB / embedding model."""
+ """Models the parts of ChromaDBStore the document routes rely on.
+
+ Critically it upserts by id and filters deletes by the `where` clause, so
+ the duplicate-chunk and failed-delete bugs are reproducible here rather
+ than needing a real ChromaDB and embedding model.
+ """
def __init__(self) -> None:
- self.added: list[dict[str, Any]] = []
+ self.rows: dict[str, dict[str, Any]] = {}
def add_documents(
self,
@@ -32,7 +49,27 @@ class _CapturingStore:
ids: list[str],
collection: str,
) -> None:
- self.added.extend(metadatas)
+ for chunk_id, meta in zip(ids, metadatas, strict=True):
+ self.rows[chunk_id] = meta
+
+ def delete_documents(self, collection: str, where: dict[str, Any]) -> None:
+ key, value = next(iter(where.items()))
+ self.rows = {i: m for i, m in self.rows.items() if m.get(key) != value}
+
+ def iter_chunk_metadata(self, collection: str) -> list[tuple[str, dict[str, Any]]]:
+ return list(self.rows.items())
+
+ def delete_by_ids(self, collection: str, ids: list[str]) -> None:
+ for chunk_id in ids:
+ self.rows.pop(chunk_id, None)
+
+ # -- assertion helpers ------------------------------------------------
+ @property
+ def added(self) -> list[dict[str, Any]]:
+ return list(self.rows.values())
+
+ def filenames(self) -> set[str]:
+ return {m["filename"] for m in self.rows.values()}
@pytest.fixture
@@ -100,3 +137,87 @@ def test_get_document_rejects_dotfile(client: TestClient) -> None:
# collapsed by path normalization before it ever reaches the handler.)
resp = client.get("/documents/.env")
assert resp.status_code == 400
+
+
+# ── #113: stable, filename-derived document identity ──────────────────────
+
+
+def test_reupload_upserts_instead_of_duplicating(client: TestClient) -> None:
+ """The bug: each upload staged to a fresh temp path, so chunk ids (an MD5
+ of that path) never collided and the upsert always inserted."""
+ store: _CapturingStore = client.app.state.store # type: ignore[attr-defined]
+
+ assert _upload(client).status_code == 200
+ after_first = set(store.rows)
+ assert after_first, "expected at least one indexed chunk"
+
+ assert _upload(client).status_code == 200
+
+ assert set(store.rows) == after_first, "re-upload must reuse the same chunk ids"
+ assert len(store.rows) == len(after_first), "re-upload must not grow the collection"
+
+
+def test_indexed_under_real_filename_not_temp_path(client: TestClient) -> None:
+ assert _upload(client).status_code == 200
+ store: _CapturingStore = client.app.state.store # type: ignore[attr-defined]
+
+ assert store.filenames() == {"plan.md"}
+ assert {m["source"] for m in store.added} == {"plan.md"}
+ # The specific failure mode: identity taken from the staging file.
+ assert not any(m["filename"].startswith("tmp") for m in store.added)
+
+
+def test_delete_removes_chunks_from_the_index(client: TestClient) -> None:
+ """The bug: DELETE matched on `filename`, which held the temp name, so it
+ removed nothing, unlinked the on-disk copy and still returned 200 —
+ leaving the chunks permanently unreachable."""
+ assert _upload(client).status_code == 200
+ store: _CapturingStore = client.app.state.store # type: ignore[attr-defined]
+ assert store.rows
+
+ resp = client.delete("/documents/plan.md")
+
+ assert resp.status_code == 200, resp.text
+ assert resp.json() == {"deleted": "plan.md"}
+ assert store.rows == {}, "DELETE must drop the document's chunks, not just the file"
+
+
+def test_delete_leaves_other_documents_indexed(client: TestClient) -> None:
+ files = {"file": ("other.md", io.BytesIO(b"# Other\nUnrelated content."), "text/markdown")}
+ assert client.post("/documents", files=files, data={"domain": "hr"}).status_code == 200
+ assert _upload(client).status_code == 200
+
+ assert client.delete("/documents/plan.md").status_code == 200
+
+ store: _CapturingStore = client.app.state.store # type: ignore[attr-defined]
+ assert store.filenames() == {"other.md"}
+
+
+# ── #114: domain validation ───────────────────────────────────────────────
+
+
+def test_unknown_domain_is_rejected(client: TestClient) -> None:
+ """A typo used to return 200 and index the document where no specialist
+ filters — silently unretrievable, with nothing to reveal it."""
+ resp = _upload(client, domain="finanace")
+
+ assert resp.status_code == 400
+ assert "finanace" in resp.json()["detail"]
+ store: _CapturingStore = client.app.state.store # type: ignore[attr-defined]
+ assert store.rows == {}, "a rejected upload must not be indexed"
+
+
+def test_rejected_domain_does_not_write_the_file(client: TestClient, tmp_path: Path) -> None:
+ assert _upload(client, domain="nonsense").status_code == 400
+ assert not (tmp_path / "company" / "docs" / "plan.md").exists()
+
+
+ "domain",
+ ["strategy", "finance", "hr", "legal", "operations", "marketing", "board", "product", "general"],
+)
+def test_every_known_domain_is_accepted(client: TestClient, domain: str) -> None:
+ resp = _upload(client, domain=domain)
+
+ assert resp.status_code == 200, resp.text
+ assert resp.json()["domain"] == domain