Skip to content

AI-powered virtual executive team — a single coherent executive persona backed by 8 specialist agents (FastAPI + Next.js).

19.2 KB
Raw Blame
from __future__ import annotations

import asyncio
import logging
import shutil
import tempfile
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, Literal

from fastapi import APIRouter, Form, HTTPException, Request, UploadFile

from openexecutive.api.models import (
    CompanyDocContent,
    DocumentUploadResponse,
    SyncedDocContent,
)

logger = logging.getLogger(__name__)

router = APIRouter()

# Every type ``knowledge.loader.extract_text_from_file`` reads (legacy binary
# .xls is left out: openpyxl reads only .xlsx/.xlsm).
ALLOWED_EXTENSIONS = {".pdf", ".docx", ".doc", ".xlsx", ".xlsm", ".csv", ".md", ".txt"}
MAX_UPLOAD_BYTES = 50 * 1024 * 1024
# The upload is copied to its staging file this many bytes at a time, so the
# process never holds a whole (up to 50 MB) document in memory.
_COPY_CHUNK_BYTES = 1024 * 1024
_ALERT_EXCERPT_BYTES = 8000


async def _stage_upload(file: UploadFile, suffix: str) -> Path:
    """Copy the upload to a temp file in pieces and return its path.

    Raises 413 as soon as it passes ``MAX_UPLOAD_BYTES``, removing the
    partial copy."""
    with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp:
        tmp_path = Path(tmp.name)
        try:
            size = 0
            while chunk := await file.read(_COPY_CHUNK_BYTES):
                size += len(chunk)
                if size > MAX_UPLOAD_BYTES:
                    raise HTTPException(status_code=413, detail="File too large (max 50MB)")
                tmp.write(chunk)
        except BaseException:
            tmp.close()
            tmp_path.unlink(missing_ok=True)
            raise
    return tmp_path


def unreadable_document_error(name: str, exc: Exception) -> HTTPException:
    """422 for a document its parser could not read, in words a person can
    act on; 503 when every parser was busy and the file was never tried.
    Only the exception's type is logged: its message can quote the file."""
    from openexecutive.knowledge.isolated import IsolatedError, ParserBusy, WorkerStopped

    kind = exc.kind if isinstance(exc, IsolatedError) else type(exc).__name__
    logger.warning("documents: could not read %r (%s)", name, kind)
    if isinstance(exc, ParserBusy):
        return HTTPException(
            status_code=503,
            detail="Documents are being read right now. Try again in a minute.",
            headers={"Retry-After": "30"},
        )
    if isinstance(exc, WorkerStopped):
        reason = "it is too large or took too long to read"
    else:
        reason = "the file may be damaged, or not the type its name says"
    return HTTPException(status_code=422, detail=f"Could not read {name}: {reason}.")


def _read_head(path: Path, size: int) -> bytes:
    with path.open("rb") as f:
        return f.read(size)


@router.post("/documents", response_model=DocumentUploadResponse)
async def upload_document(
    file: UploadFile,
    # `Form(...)` (not a bare default) so FastAPI reads `domain` from the
    # multipart body the UI sends. A bare `domain: str = "general"` is parsed
    # as a query param, so the form field is dropped and every upload lands
    # under "general" — invisible to domain-filtered specialist retrieval.
    domain: str = Form("general"),
    request: Request = None,  # type: ignore[assignment]
) -> DocumentUploadResponse:
    if not file.filename:
        raise HTTPException(status_code=400, detail="No filename provided")

    # Strip directory components to prevent path traversal (e.g. "../../etc/passwd.md")
    safe_filename = Path(file.filename).name
    if not safe_filename or safe_filename.startswith("."):
        raise HTTPException(status_code=400, detail="Invalid filename")

    ext = Path(safe_filename).suffix.lower()
    if ext not in ALLOWED_EXTENSIONS:
        raise HTTPException(
            status_code=400,
            detail=f"Unsupported file type: {ext}. Allowed: {', '.join(sorted(ALLOWED_EXTENSIONS))}",
        )

    # Reject an unknown domain rather than indexing under it. Specialist
    # retrieval filters on these exact values, so a typo ("finanace") would
    # return 200 and then make the document permanently unretrievable — a
    # silent, delayed failure with no way to notice it from the API.
    from openexecutive.knowledge.loader import UPLOAD_DOMAINS

    if domain not in UPLOAD_DOMAINS:
        raise HTTPException(
            status_code=400,
            detail=f"Unknown domain: {domain}. Allowed: {', '.join(sorted(UPLOAD_DOMAINS))}",
        )

    tmp_path = await _stage_upload(file, ext)

    try:
        from openexecutive.config import get_settings
        from openexecutive.knowledge.isolated import IsolatedError
        from openexecutive.knowledge.loader import ingest_file
        from openexecutive.knowledge.store import ChromaDBStore

        settings = get_settings()
        store = (
            request.app.state.store
            if request and hasattr(request.app.state, "store")
            else ChromaDBStore(persist_directory=settings.vector_store_path)
        )

        # `source_name` — NOT tmp_path.name. The temp file is only a staging
        # buffer so the PDF/DOCX extractors have a real path to open; the
        # document's identity is the name the user uploaded it under. Deriving
        # identity from the random temp name instead gives every re-upload
        # fresh chunk ids (so the id-keyed upsert never collides and the
        # collection grows without bound) and stores a `filename` that the
        # DELETE endpoint below can never match.
        try:
            chunks_indexed = await ingest_file(
                path=tmp_path,
                store=store,
                domain=domain,
                collection=ChromaDBStore.COMPANY_COLLECTION,
                source_name=safe_filename,
                busy_raises=True,
            )
        except IsolatedError as exc:
            # A Word/Excel parse that failed, died or ran out of time, or any
            # file every parser was too busy to try (a PDF otherwise never
            # raises: it comes back empty, with a note).
            raise unreadable_document_error(safe_filename, exc) from exc

        company_docs_dir = settings.company_profile_path.parent / "docs"
        company_docs_dir.mkdir(parents=True, exist_ok=True)
        dest = company_docs_dir / safe_filename
        await asyncio.to_thread(shutil.copyfile, tmp_path, dest)

        # Fire the proactive-alerts pipeline. Body is a best-effort excerpt
        # for triage context; PDFs/docx won't decode cleanly and that's fine —
        # the triage prompt still sees source, title, and domain.
        try:
            from openexecutive.alerts.models import AlertEvent
            from openexecutive.alerts.pipeline import schedule_evaluation

            excerpt = ""
            if ext in {".md", ".txt", ".csv"}:
                head = await asyncio.to_thread(_read_head, tmp_path, _ALERT_EXCERPT_BYTES)
                excerpt = head.decode("utf-8", errors="replace")
            else:
                excerpt = f"Newly ingested {ext} document: {safe_filename} (domain: {domain})"

            schedule_evaluation(
                AlertEvent(
                    source="document",
                    external_id=safe_filename,
                    title=safe_filename,
                    body=excerpt,
                )
            )
        except Exception:
            # Never let an alert failure 500 the upload.
            import logging

            logging.getLogger(__name__).exception(
                "Failed to schedule alert evaluation for document upload"
            )

    finally:
        tmp_path.unlink(missing_ok=True)

    return DocumentUploadResponse(
        filename=safe_filename,
        chunks_indexed=chunks_indexed,
        domain=domain,
        status="indexed",
    )


def _store(request: Request | None) -> Any:
    from openexecutive.config import get_settings
    from openexecutive.knowledge.store import ChromaDBStore

    if request is not None and hasattr(request.app.state, "store"):
        return request.app.state.store
    return ChromaDBStore(persist_directory=get_settings().vector_store_path)


def _upload_domains(store: Any) -> dict[str, str]:
    """filename → domain for uploaded documents, from their chunks' metadata.

    The domain is only a label on the list; a store that cannot be read
    leaves the list untagged rather than failing it.
    """
    from openexecutive.knowledge.store import ChromaDBStore

    domains: dict[str, str] = {}
    try:
        rows = store.iter_chunk_metadata(ChromaDBStore.COMPANY_COLLECTION)
    except Exception:
        logger.exception("documents: could not read upload domains")
        return domains
    for _cid, md in rows:
        filename, domain = md.get("filename"), md.get("domain")
        if isinstance(filename, str) and isinstance(domain, str):
            domains.setdefault(filename, domain)
    return domains


@router.get("/documents")
async def list_documents(request: Request = None) -> dict:  # type: ignore[assignment]
    from openexecutive.config import get_settings
    from openexecutive.knowledge.loader import list_company_docs

    settings = get_settings()
    docs_dir = settings.company_profile_path.parent / "docs"
    docs = list_company_docs(docs_dir)
    domains = await asyncio.to_thread(_upload_domains, _store(request)) if docs else {}
    return {
        "documents": [
            {**doc, "source": "upload", "domain": domains.get(doc["filename"])} for doc in docs
        ]
    }


# ---------------------------------------------------------------------------
# Connected sources (Google Drive, OneDrive, Notion): read-only lists, a viewer, and a
# "Sync now" that runs one tick of the same sync the scheduler runs. Files
# from these sources are managed where they live — removing one from the
# shared folder / Notion integration is what drops it from the knowledge base.
#
# Declared before ``/documents/{filename}`` so "sources", "drive",
# "onedrive" and "notion" are never read as an uploaded file's name.
# ---------------------------------------------------------------------------

SourceId = Literal["drive", "onedrive", "notion"]
_SOURCE_LABELS: dict[str, str] = {"drive": "Google Drive", "onedrive": "OneDrive", "notion": "Notion"}
# A manual sync this soon after the last tick is refused: the sync is
# incremental, so a second run finds nothing new and only spends API quota.
SYNC_COOLDOWN_S = 60
# The running manual sync per source. It doubles as the strong reference the
# event loop doesn't keep, and it marks a sync as started before its task has
# taken the module's run lock, so two "Sync now" requests can't both start one.
_SYNC_TASKS: dict[str, asyncio.Task[Any]] = {}


def _source_module(source: str) -> Any:
    if source == "drive":
        from openexecutive.knowledge import drive_sync

        return drive_sync
    if source == "onedrive":
        from openexecutive.knowledge import onedrive_sync

        return onedrive_sync
    if source == "notion":
        from openexecutive.knowledge import notion_sync

        return notion_sync
    raise HTTPException(status_code=404, detail="Unknown source")


def _source_enabled(source: str) -> bool:
    from openexecutive.config import get_settings

    settings = get_settings()
    if source == "drive":
        return bool(settings.drive_sync_enabled)
    if source == "onedrive":
        return bool(settings.onedrive_sync_enabled)
    return bool(settings.notion_sync_enabled and settings.notion_api_key)


def _source_interval(source: str) -> int:
    from openexecutive.config import get_settings

    settings = get_settings()
    if source == "drive":
        return int(settings.drive_sync_interval_minutes)
    if source == "onedrive":
        return int(settings.onedrive_sync_interval_minutes)
    return int(settings.notion_sync_interval_minutes)


def _list_source(source: str) -> dict[str, Any]:
    module = _source_module(source)
    listing: dict[str, Any] = (
        module.list_synced_pages() if source == "notion" else module.list_synced_files()
    )
    return listing


def _source_status(source: str) -> dict[str, Any]:
    listing = _list_source(source)
    return {
        "id": source,
        "label": _SOURCE_LABELS[source],
        "enabled": _source_enabled(source),
        "syncing": bool(_source_module(source).is_syncing()),
        "last_run": listing["last_run"],
        "last_error": listing["last_error"],
        "interval_minutes": _source_interval(source),
        "file_count": sum(1 for f in listing["files"] if f["indexed"]),
    }


@router.get("/documents/sources")
async def list_sources() -> dict:
    return {
        "sources": [
            await asyncio.to_thread(_source_status, source) for source in _SOURCE_LABELS
        ]
    }


@router.post("/documents/sources/{source}/sync", status_code=202)
async def sync_source(source: SourceId) -> dict:
    module = _source_module(source)
    if not _source_enabled(source):
        raise HTTPException(status_code=409, detail=f"{_SOURCE_LABELS[source]} is not connected")
    if module.is_syncing() or source in _SYNC_TASKS:
        raise HTTPException(status_code=409, detail="A sync is already running")
    # The cooldown runs from the latest of the last successful tick's start
    # and the end of the last tick in this process (failed ones included).
    last_run = (await asyncio.to_thread(_list_source, source))["last_run"]
    marks: list[datetime] = []
    if last_run:
        try:
            parsed = datetime.fromisoformat(last_run)
            marks.append(parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC))
        except (TypeError, ValueError):
            pass
    if (finished := module.last_finished_at()) is not None:
        marks.append(finished)
    if marks:
        elapsed = (datetime.now(UTC) - max(marks)).total_seconds()
        if 0 <= elapsed < SYNC_COOLDOWN_S:
            raise HTTPException(
                status_code=429,
                detail="Synced less than a minute ago. Try again shortly.",
                headers={"Retry-After": str(int(SYNC_COOLDOWN_S - elapsed) + 1)},
            )
    # Check again after the await above: another request may have started a
    # sync meanwhile. Nothing awaits between this check and registering the
    # task, so the two can't interleave.
    if module.is_syncing() or source in _SYNC_TASKS:
        raise HTTPException(status_code=409, detail="A sync is already running")
    runners = {
        "drive": "run_drive_sync",
        "onedrive": "run_onedrive_sync",
        "notion": "run_notion_sync",
    }
    runner = getattr(module, runners[source])
    task = asyncio.create_task(_run_manual_sync(source, runner))
    _SYNC_TASKS[source] = task
    task.add_done_callback(lambda _t: _SYNC_TASKS.pop(source, None))
    return {"source": source, "status": "started"}


async def _run_manual_sync(source: str, runner: Any) -> None:
    try:
        stats = await runner()
        logger.info("documents: manual %s sync %s", source, stats)
    except Exception:
        # The sync module has already recorded a user-facing last_error.
        logger.exception("documents: manual %s sync failed", source)


@router.get("/documents/drive")
async def list_drive_documents() -> dict:
    listing = await asyncio.to_thread(_list_source, "drive")
    return {"enabled": _source_enabled("drive"), **listing}


@router.get("/documents/drive/{file_id}", response_model=SyncedDocContent)
async def get_drive_document(file_id: str) -> SyncedDocContent:
    from openexecutive.knowledge.drive_sync import read_synced_file

    doc = await asyncio.to_thread(read_synced_file, file_id)
    if doc is None:
        raise HTTPException(status_code=404, detail="Document not found")
    return SyncedDocContent(**doc)


@router.get("/documents/onedrive")
async def list_onedrive_documents() -> dict:
    listing = await asyncio.to_thread(_list_source, "onedrive")
    return {"enabled": _source_enabled("onedrive"), **listing}


@router.get("/documents/onedrive/{file_key}", response_model=SyncedDocContent)
async def get_onedrive_document(file_key: str) -> SyncedDocContent:
    from openexecutive.knowledge.onedrive_sync import read_synced_file

    doc = await asyncio.to_thread(read_synced_file, file_key)
    if doc is None:
        raise HTTPException(status_code=404, detail="Document not found")
    return SyncedDocContent(**doc)


@router.get("/documents/notion")
async def list_notion_documents() -> dict:
    listing = await asyncio.to_thread(_list_source, "notion")
    return {"enabled": _source_enabled("notion"), **listing}


@router.get("/documents/notion/{page_id}", response_model=SyncedDocContent)
async def get_notion_document(page_id: str) -> SyncedDocContent:
    from openexecutive.knowledge.notion_sync import read_synced_page

    doc = await asyncio.to_thread(read_synced_page, page_id)
    if doc is None:
        raise HTTPException(status_code=404, detail="Document not found")
    return SyncedDocContent(**doc)


@router.get("/documents/{filename}", response_model=CompanyDocContent)
async def get_document(
    filename: str,
    request: Request = None,  # type: ignore[assignment]
) -> CompanyDocContent:
    # Same sanitization as delete: reject anything that isn't a bare filename
    # so a crafted path can't escape the docs directory.
    safe = Path(filename).name
    if not safe or safe.startswith(".") or safe != filename:
        raise HTTPException(status_code=400, detail="Invalid filename")

    from openexecutive.config import get_settings
    from openexecutive.knowledge.isolated import IsolatedError
    from openexecutive.knowledge.loader import extract_text_from_file_async

    settings = get_settings()
    docs_dir = settings.company_profile_path.parent / "docs"
    path = docs_dir / safe
    if not path.exists():
        raise HTTPException(status_code=404, detail="Document not found")

    # Show the extracted text — exactly what gets chunked into the vector store
    # and retrieved by the Executive. Works uniformly across PDF/DOCX/MD/TXT; a
    # scanned PDF shows its converted text (knowledge/pdf_reader.py).
    try:
        content = await extract_text_from_file_async(path, busy_raises=True)
    except IsolatedError as exc:
        raise unreadable_document_error(safe, exc) from exc
    if not content.strip():
        content = "_No text could be read from this document._"
    return CompanyDocContent(filename=safe, content=content)


@router.delete("/documents/{filename}")
async def delete_document(
    filename: str,
    request: Request = None,  # type: ignore[assignment]
) -> dict:
    safe = Path(filename).name
    if not safe or safe.startswith(".") or safe != filename:
        raise HTTPException(status_code=400, detail="Invalid filename")

    from openexecutive.config import get_settings
    from openexecutive.knowledge.store import ChromaDBStore

    settings = get_settings()
    docs_dir = settings.company_profile_path.parent / "docs"
    path = docs_dir / safe
    if not path.exists():
        raise HTTPException(status_code=404, detail="Document not found")

    store = (
        request.app.state.store
        if request and hasattr(request.app.state, "store")
        else ChromaDBStore(persist_directory=settings.vector_store_path)
    )
    store.delete_documents(
        collection=ChromaDBStore.COMPANY_COLLECTION,
        where={"filename": safe},
    )
    path.unlink()
    return {"deleted": safe}