Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

67.2 KB
Raw Blame
"""
Modal app for CUA documentation crawling and MCP server

This app provides:
1. Scheduled daily crawling of cua.ai/docs stored in a Modal volume
2. MCP server that serves documentation search over the crawled data

Usage:
    modal deploy docs/scripts/modal_app.py
"""

import asyncio
import html
from html.parser import HTMLParser
import json
import re
import sqlite3
from pathlib import Path
from typing import Optional

import modal
from markdown_it import MarkdownIt

# Define the Modal app
app = modal.App("cua-docs-mcp")

# Create persistent volumes for storing data
docs_volume = modal.Volume.from_name("cua-docs-data", create_if_missing=True)
code_volume = modal.Volume.from_name("cua-code-index", create_if_missing=True)

# GitHub token secret for cloning
github_secret = modal.Secret.from_name("github-secret", required_keys=["GITHUB_TOKEN"])

# AWS IAM role ARN for OIDC-based S3 write access (no static keys needed)
# Role created in cloud repo: terraform/aws/docs-mcp-storage/main.tf
S3_WRITE_ROLE_ARN = "arn:aws:iam::296062593712:role/modal-docs-mcp-write-role"
S3_BUCKET_NAME = "trycua-docs-mcp-data"
S3_BUCKET_REGION = "us-west-2"

# Define the container image with all dependencies
image = (
    modal.Image.debian_slim(python_version="3.12")
    .apt_install("git")
    .pip_install(
        "playwright>=1.40.0",
        # Pinned exactly, and kept in step with docs-mcp-server/uv.lock.
        #
        # This job writes docs.lance; docs-mcp-server reads it. Both also build
        # embeddings through lancedb.embeddings.get_registry(). These specs used
        # to be ">=" on both sides, which looks symmetric but is not: this image
        # re-resolves on every run while the server's versions froze at whenever
        # its Docker image was last built. The two drifted apart, the writer
        # started emitting newer-format Lance manifests, and the server could no
        # longer open the table -- reporting it as a missing database.
        #
        # If you change these, relock docs-mcp-server and rebuild its image in
        # the same change, so reader and writer move together.
        "lancedb==0.37.1",
        "sentence-transformers==5.7.0",
        "pyarrow==25.0.1",
        "fastapi>=0.100.0",
        "fastmcp>=2.14.0,<3",
        "pydantic>=2.0.0",
        "pandas>=2.0.0",
        "markdown-it-py>=3.0.0",
        "markitdown>=0.0.1",
        "boto3>=1.34.0",
    )
    .run_commands("playwright install --with-deps chromium")
    .add_local_file(Path(__file__).parent / "docs-mcp-server" / "docs_index.py", "/root/docs_index.py")
)

# Volume mount paths
VOLUME_PATH = "/data"
CRAWLED_DATA_PATH = f"{VOLUME_PATH}/crawled_data"
DB_PATH = f"{VOLUME_PATH}/docs_db"

# Code index volume mount path
CODE_VOLUME_PATH = "/code_data"
CODE_REPO_PATH = f"{CODE_VOLUME_PATH}/repo"
CODE_DB_PATH = f"{CODE_VOLUME_PATH}/code_db"


# =============================================================================
# Helper Functions
# =============================================================================


async def load_docs_page(page, url: str) -> str:
    """Wait for the docs body, not unrelated background network activity."""
    response = await page.goto(url, wait_until="domcontentloaded", timeout=30_000)
    if response is None or not response.ok:
        status = response.status if response is not None else "no response"
        raise RuntimeError(f"HTTP {status}")
    await page.wait_for_selector("article, main", state="attached", timeout=30_000)
    return await page.content()


class HTMLToMarkdown(HTMLParser):
    """Small dependency-free HTML-to-Markdown converter for crawled docs pages.

    Extraction is scoped to the page's main content container (``<article>``,
    falling back to ``<main>``) and site chrome (``nav``/``aside``/``footer``) is
    dropped, so the crawled corpus is the documentation body rather than the
    navigation tree that repeats identically on every page.
    """

    block_tags = {
        "blockquote",
        "br",
        "div",
        "h1",
        "h2",
        "h3",
        "h4",
        "h5",
        "h6",
        "header",
        "li",
        "main",
        "ol",
        "p",
        "pre",
        "section",
        "table",
        "tr",
        "ul",
    }
    # Content of these tags is dropped entirely: non-text assets and the site
    # chrome (sidebar/nav tree, "on this page" aside, footer) that is identical
    # on every page and would otherwise dominate the embedded corpus.
    skip_tags = {"script", "style", "svg", "nav", "aside", "footer"}

    def __init__(self, scope_tag: str | None = None) -> None:
        super().__init__(convert_charrefs=True)
        self.parts: list[str] = []
        self.skip_depth = 0
        self.in_pre = False
        # When set, only emit text while inside this container; None = emit all.
        self.scope_tag = scope_tag
        self.scope_depth = 0

    @property
    def _capturing(self) -> bool:
        return self.scope_tag is None or self.scope_depth > 0

    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
        if tag in self.skip_tags:
            self.skip_depth += 1
            return
        if tag == self.scope_tag:
            self.scope_depth += 1
        if self.skip_depth or not self._capturing:
            return
        if tag in self.block_tags:
            self.parts.append("\n")
        if tag == "li":
            self.parts.append("- ")
        elif tag == "pre":
            self.in_pre = True
            self.parts.append("\n```\n")
        elif tag == "code" and not self.in_pre:
            self.parts.append("`")

    def handle_endtag(self, tag: str) -> None:
        if tag in self.skip_tags and self.skip_depth:
            self.skip_depth -= 1
            return
        if self.skip_depth:
            return
        if self._capturing:
            if tag == "pre":
                self.in_pre = False
                self.parts.append("\n```\n")
            elif tag == "code" and not self.in_pre:
                self.parts.append("`")
            if tag in self.block_tags:
                self.parts.append("\n")
        if tag == self.scope_tag and self.scope_depth:
            self.scope_depth -= 1

    def handle_data(self, data: str) -> None:
        if self.skip_depth or not self._capturing:
            return
        text = data if self.in_pre else re.sub(r"\s+", " ", data)
        if text.strip():
            self.parts.append(text)

    def markdown(self) -> str:
        text = html.unescape("".join(self.parts))
        text = re.sub(r"[ \t]+\n", "\n", text)
        text = re.sub(r"\n{3,}", "\n\n", text)
        return text.strip()


def html_to_markdown(page_html: str) -> str:
    # Prefer the main content container so the navigation/sidebar chrome that
    # repeats on every page does not pollute the crawled corpus; fall back to
    # the whole document when neither container is present.
    scope_tag = None
    for tag in ("article", "main"):
        if re.search(rf"<{tag}[\s>]", page_html, re.IGNORECASE):
            scope_tag = tag
            break
    parser = HTMLToMarkdown(scope_tag)
    parser.feed(page_html)
    return parser.markdown()


def extract_metadata(page_html: str, title: str) -> dict[str, str]:
    description = ""
    match = re.search(
        r'<meta[^>]+name=["\']description["\'][^>]+content=["\']([^"\']*)["\']',
        page_html,
        re.IGNORECASE,
    )
    if match:
        description = html.unescape(match.group(1))
    return {"title": title, "description": description}


def clean_markdown(markdown: str) -> str:
    """Extract plain text content from markdown using markdown-it-py parser"""
    md_parser = MarkdownIt()
    tokens = md_parser.parse(markdown)

    text_parts = []

    def extract_text(token_list):
        for token in token_list:
            if token.type == "inline" and token.children:
                for child in token.children:
                    if child.type == "text":
                        text_parts.append(child.content)
                    elif child.type == "code_inline":
                        text_parts.append(child.content)
                    elif child.type == "softbreak":
                        text_parts.append(" ")
                    elif child.type == "hardbreak":
                        text_parts.append("\n")
            elif token.type == "fence" or token.type == "code_block":
                text_parts.append(token.content)
                text_parts.append("\n")

            if token.children:
                extract_text(token.children)

            if token.type in [
                "heading_close",
                "paragraph_close",
                "list_item_close",
                "blockquote_close",
            ]:
                text_parts.append("\n")

    extract_text(tokens)
    text = "".join(text_parts)
    text = re.sub(r"\n{3,}", "\n\n", text)
    text = re.sub(r" {2,}", " ", text)
    return text.strip()


# =============================================================================
# Crawling Functions
# =============================================================================


@app.function(
    image=image,
    volumes={VOLUME_PATH: docs_volume},
    timeout=3600,  # 1 hour timeout
    cpu=2.0,
    memory=4096,
)
async def crawl_docs():
    """Crawl CUA documentation and save to volume"""
    import re
    from urllib.parse import urljoin, urlparse

    from playwright.async_api import async_playwright

    print("Starting documentation crawl...")

    BASE_URL = "https://cua.ai"
    DOCS_URL = f"{BASE_URL}/docs"
    OUTPUT_DIR = Path(CRAWLED_DATA_PATH)

    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)

    visited_urls = set()
    to_visit = set()
    failed_urls = set()
    all_data = []

    def normalize_url(url: str) -> str:
        """Normalize URL to avoid duplicates"""
        parsed = urlparse(url)
        path = parsed.path.rstrip("/")
        if not path:
            path = ""
        return f"{parsed.scheme}://{parsed.netloc}{path}"

    def is_valid_url(url: str) -> bool:
        """Check if URL should be crawled (only /docs pages)"""
        parsed = urlparse(url)
        if parsed.netloc and parsed.netloc not in ["cua.ai", "www.cua.ai"]:
            return False
        if parsed.path != "/docs" and not parsed.path.startswith("/docs/"):
            return False
        # Skip non-page resources
        excluded_extensions = [
            ".pdf",
            ".zip",
            ".png",
            ".jpg",
            ".jpeg",
            ".gif",
            ".svg",
            ".ico",
            ".css",
            ".js",
        ]
        if any(parsed.path.lower().endswith(ext) for ext in excluded_extensions):
            return False
        return True

    def extract_links(html: str, base_url: str) -> set[str]:
        """Extract all valid links from HTML"""
        links = set()
        # Find all href attributes
        href_pattern = r'href=["\']([^"\']+)["\']'
        matches = re.findall(href_pattern, html)

        for match in matches:
            full_url = urljoin(base_url, match)
            normalized = normalize_url(full_url)
            if is_valid_url(normalized):
                links.add(normalized)

        return links

    def extract_path_info(url: str) -> dict:
        """Extract meaningful path information from URL"""
        parsed = urlparse(url)
        path = parsed.path.replace("/docs/", "").strip("/")
        parts = path.split("/") if path else []

        return {
            "path": path,
            "category": parts[0] if parts else "root",
            "subcategory": parts[1] if len(parts) > 1 else None,
            "page": parts[-1] if parts else "index",
            "depth": len(parts),
        }

    def save_page(url: str, data: dict):
        """Save page data to a JSON file"""
        parsed = urlparse(url)
        path = parsed.path.strip("/") or "index"
        filename = path.replace("/", "_") + ".json"

        filepath = OUTPUT_DIR / filename
        with open(filepath, "w", encoding="utf-8") as f:
            json.dump(data, f, indent=2, ensure_ascii=False)

    # Discover the public corpus from the canonical docs root.
    seed_urls = [DOCS_URL]

    for url in seed_urls:
        normalized = normalize_url(url)
        if is_valid_url(normalized):
            to_visit.add(normalized)

    async with async_playwright() as playwright:
        browser = await playwright.chromium.launch(headless=True)
        try:
            while to_visit:
                # Get batch of URLs to crawl
                batch = []
                MAX_CONCURRENT = 5
                while to_visit and len(batch) < MAX_CONCURRENT:
                    url = to_visit.pop()
                    if url not in visited_urls:
                        batch.append(url)
                        visited_urls.add(url)

                if not batch:
                    break

                # Crawl each URL in batch
                for url in batch:
                    page = None
                    try:
                        print(f"Crawling: {url}")

                        page = await browser.new_page()
                        page_html = await load_docs_page(page, url)
                        metadata = extract_metadata(page_html, await page.title())

                        # Extract new links from the page
                        new_links = extract_links(page_html, url)
                        for link in new_links:
                            if link not in visited_urls and link not in to_visit:
                                to_visit.add(link)

                        path_info = extract_path_info(url)

                        page_data = {
                            "url": url,
                            "title": metadata["title"],
                            "description": metadata["description"],
                            "markdown": html_to_markdown(page_html),
                            "path_info": path_info,
                            "links_found": list(new_links),
                        }

                        # Save individual page
                        save_page(url, page_data)
                        all_data.append(page_data)

                        await asyncio.sleep(0.5)

                    except Exception as e:
                        print(f"Error crawling {url}: {e}")
                        failed_urls.add(url)
                    finally:
                        if page is not None:
                            await page.close()

                print(f"Progress: {len(visited_urls)} crawled, {len(to_visit)} remaining")
        finally:
            await browser.close()

    from docs_index import capture_corpus, write_json

    corpus = capture_corpus(all_data, visited_urls, failed_urls)
    # A single snapshot is the only input to both index writers.
    write_json(OUTPUT_DIR / "_corpus.json", corpus)

    # Save summary
    summary = {
        "total_pages": len(all_data),
        "failed_urls": list(failed_urls),
        "all_urls": list(visited_urls),
    }

    with open(OUTPUT_DIR / "_summary.json", "w", encoding="utf-8") as f:
        json.dump(summary, f, indent=2)

    # Save all data in one file too
    with open(OUTPUT_DIR / "_all_pages.json", "w", encoding="utf-8") as f:
        json.dump(all_data, f, indent=2, ensure_ascii=False)

    # Commit changes to volume
    docs_volume.commit()

    print(f"Crawl complete! Crawled {len(all_data)} pages")
    return summary


def _publish_dir(s3, bucket: str, local_dir, prefix: str) -> int:
    """Publish local_dir to s3://bucket/prefix, removing anything stale.

    Uploads every file, then deletes keys under the prefix that the current
    build did not write. Plain upload-only publishing is what broke docs
    vector search: the indexer rebuilds docs.lance from scratch, but old
    files were never removed from S3, so manifests from a previous Lance
    format lingered next to the new ones. Lance then refused to open the
    table ("Found multiple manifest naming schemes in the same directory:
    V1 and V2") and the server reported it as a missing database.

    This legacy code-index publisher is not atomic: a consumer can observe
    partially uploaded files. Docs use immutable generations and a commit marker.
    """
    uploaded_keys = set()
    count = 0
    for fpath in sorted(local_dir.rglob("*")):
        if fpath.is_file():
            key = f"{prefix}{fpath.relative_to(local_dir).as_posix()}"
            s3.upload_file(str(fpath), bucket, key)
            uploaded_keys.add(key)
            count += 1

    stale = []
    for page in s3.get_paginator("list_objects_v2").paginate(
        Bucket=bucket, Prefix=prefix
    ):
        for obj in page.get("Contents", []):
            if obj["Key"] not in uploaded_keys:
                stale.append({"Key": obj["Key"]})

    for batch_start in range(0, len(stale), 1000):
        s3.delete_objects(
            Bucket=bucket,
            Delete={"Objects": stale[batch_start : batch_start + 1000]},
        )
    if stale:
        print(f"  Removed {len(stale)} stale object(s) under {prefix}")

    return count


@app.function(
    image=image,
    volumes={VOLUME_PATH: docs_volume, CODE_VOLUME_PATH: code_volume},
    timeout=1800,  # 30 minutes
    cpu=1.0,
    memory=2048,
)
def sync_to_s3(
    bucket: str = S3_BUCKET_NAME, publish_docs: bool = True, publish_code: bool = True
):
    """Sync generated databases from Modal volumes to S3.

    Uses Modal OIDC federation to assume an AWS IAM role for write access.
    No static AWS credentials needed — the IAM role trust policy is scoped
    to this Modal workspace and app (terraform/aws/docs-mcp-storage/main.tf).

    Args:
        bucket: S3 bucket name to upload to
        publish_docs: Publish the verified docs generation and its pointer.
        publish_code: Publish the legacy code index independently of docs.
    """
    import os

    import boto3

    print(f"Syncing databases to s3://{bucket}/ ...")

    # Use Modal's OIDC token (auto-injected env var) to assume AWS IAM role
    oidc_token = os.environ["MODAL_IDENTITY_TOKEN"]

    sts = boto3.client("sts", region_name=S3_BUCKET_REGION)
    creds = sts.assume_role_with_web_identity(
        RoleArn=S3_WRITE_ROLE_ARN,
        RoleSessionName="modal-docs-mcp-s3-sync",
        WebIdentityToken=oidc_token,
        DurationSeconds=3600,
    )["Credentials"]

    s3 = boto3.client(
        "s3",
        region_name=S3_BUCKET_REGION,
        aws_access_key_id=creds["AccessKeyId"],
        aws_secret_access_key=creds["SecretAccessKey"],
        aws_session_token=creds["SessionToken"],
    )

    uploaded = 0
    docs_volume.reload()

    # Publish only a complete pair. The consumer must activate current.json last.
    from docs_index import publish_generation

    docs_db_dir = Path(DB_PATH)
    pointer_path = docs_db_dir / "current.json"
    if publish_docs:
        if not pointer_path.exists():
            raise RuntimeError("No verified docs generation is available to publish")
        pointer = json.loads(pointer_path.read_text())
        generation = docs_db_dir / "generations" / pointer["build_id"]
        manifest = json.loads((generation / "manifest.json").read_text())
        from docs_index import pointer_for
        if pointer != pointer_for(manifest):
            raise ValueError("Docs publication pointer mismatch")
        uploaded += publish_generation(s3, bucket, generation, manifest)

    # --- code databases ---
    code_db_dir = Path(CODE_DB_PATH)
    if publish_code and code_db_dir.exists():
        # Aggregated SQLite
        code_sqlite = code_db_dir / "code_index.sqlite"
        if code_sqlite.exists():
            key = "code_db/code_index.sqlite"
            print(f"  Uploading {code_sqlite} -> {key}")
            s3.upload_file(str(code_sqlite), bucket, key)
            uploaded += 1

        # Aggregated LanceDB directory
        code_lance = code_db_dir / "code_index.lancedb"
        if code_lance.exists():
            uploaded += _publish_dir(
                s3, bucket, code_lance, "code_db/code_index.lancedb/"
            )
            print("  Uploaded code LanceDB directory")

    print(f"S3 sync complete: {uploaded} files uploaded to s3://{bucket}/")
    return {"bucket": bucket, "files_uploaded": uploaded}


@app.function(
    image=image,
    volumes={VOLUME_PATH: docs_volume},
    schedule=modal.Cron("0 6 * * *"),  # Daily at 6 AM UTC
    timeout=3600,
)
async def scheduled_crawl():
    """Scheduled daily crawl of documentation"""
    print("Running scheduled crawl...")
    summary = await crawl_docs.remote.aio()

    # Regenerate databases after crawl
    print("Generating databases...")
    await generate_docs_indexes.remote.aio()

    # Sync docs databases to S3
    print("Syncing docs databases to S3...")
    sync_result = sync_to_s3.remote(publish_code=False)
    print(f"S3 sync result: {sync_result}")

    print(f"Scheduled crawl complete: {summary}")
    return summary


# =============================================================================
# Database Generation Functions
# =============================================================================
#
# The MCP server provides access to two types of query databases:
#
# 1. DOCUMENTATION DATABASES (from cua.ai/docs crawl):
#    - SQLite FTS5 Database (docs.sqlite):
#      * `pages` table: stores URL, title, category, and plain-text content
#      * `pages_fts` virtual table: FTS5 full-text search index
#      * FTS is rebuilt once for the immutable generation
#      * Built by: generate_docs_indexes()
#
#    - LanceDB Vector Database (docs.lance/):
#      * DocsChunk schema: text, vector (384-dim), url, title, category, chunk_index
#      * Uses sentence-transformers/all-MiniLM-L6-v2 for embeddings
#      * Chunks searchable text into overlapping windows without dropping short pages
#      * Built by: generate_docs_indexes()
#
# 2. CODE INDEX DATABASES (from git tags):
#    Per-component databases (built in parallel):
#    - SQLite FTS5 Database (code_index_<component>.sqlite per component):
#      * `code_files` table: component, version, file_path, content, language
#      * `code_files_fts` virtual table: FTS5 full-text search index
#      * Indexes source files (.py, .ts, .js, .tsx) from all git tags
#      * Built by: index_component() called from generate_code_index_parallel()
#
#    - LanceDB Vector Database (code_index_<component>.lancedb/ per component):
#      * CodeFile schema: text, vector (384-dim), component, version, file_path, language
#      * Only embeds files under 100KB to avoid memory issues
#      * Built by: index_component() called from generate_code_index_parallel()
#
#    Aggregated databases (for MCP server queries):
#    - code_index.sqlite: Unified SQLite with all components' data + FTS5 index
#    - code_index.lancedb: Unified LanceDB with all components' vectors
#    - Built by: aggregate_code_databases() after parallel indexing completes
#
# Database Build Process:
# 1. scheduled_crawl() runs daily at 6 AM UTC:
#    - Calls crawl_docs() to crawl cua.ai/docs
#    - Calls generate_docs_indexes() to build and validate the pair from one snapshot
#
# 2. scheduled_code_index() runs daily at 5 AM UTC (before docs):
#    - Calls generate_code_index_parallel() which:
#      a. Clones/updates the git repository (bare clone)
#      b. Groups all git tags by component (agent, computer, etc.)
#      c. Dispatches parallel index_component() workers per component
#      d. Each worker builds its own SQLite + LanceDB
#    - Calls aggregate_code_databases() to merge per-component DBs into unified DBs
#
# Note: Modal volumes don't support atomic rename operations, so LanceDB is
# built in a temp directory first, then copied to the volume.
# =============================================================================


@app.function(
    image=image,
    volumes={VOLUME_PATH: docs_volume},
    timeout=1800,
    cpu=2.0,
    memory=8192,
)
async def generate_docs_indexes():
    """Build and validate both docs indexes before committing a generation."""
    import shutil
    import tempfile
    from docs_index import build_generation, load_corpus, pointer_for, verify_files, write_json

    docs_volume.reload()
    corpus = load_corpus(Path(CRAWLED_DATA_PATH) / "_corpus.json")
    staging = Path(tempfile.mkdtemp(prefix="docs-generation-"))
    path, manifest = build_generation(corpus, staging)
    root = Path(DB_PATH)
    destination = root / "generations" / manifest["build_id"]
    shutil.copytree(path, destination)
    verify_files(destination, manifest)
    # Modal commits make the complete generation visible before its pointer.
    docs_volume.commit()
    write_json(root / "current.json", pointer_for(manifest))
    docs_volume.commit()
    return {"build_id": manifest["build_id"], "pages": len(manifest["included_urls"]),
            "chunks": manifest["chunk_count"], "exclusions": manifest["exclusions"]}


@app.function(image=image, volumes={VOLUME_PATH: docs_volume}, timeout=1800)
async def generate_vector_db():
    """Compatibility entry point: rebuild the paired docs indexes."""
    return await generate_docs_indexes.remote.aio()


@app.function(image=image, volumes={VOLUME_PATH: docs_volume}, timeout=1800)
async def generate_sqlite_db():
    """Compatibility entry point: rebuild the paired docs indexes."""
    return await generate_docs_indexes.remote.aio()


# =============================================================================
# Code Index Generation Functions
# =============================================================================

# Source file extensions to index
SOURCE_EXTENSIONS = {".py", ".ts", ".js", ".tsx"}
MAX_FILE_SIZE_SQLITE = 1_000_000  # 1MB for SQLite
MAX_FILE_SIZE_EMBEDDINGS = 100_000  # 100KB for embeddings


def parse_tag(tag: str) -> tuple[str, str]:
    """Parse a git tag into component and version."""
    if tag.startswith("v") and len(tag) > 1 and tag[1].isdigit():
        return ("cua", tag[1:])
    match = re.match(r"^(.+)-v(\d+\.\d+\.\d+.*)$", tag)
    if match:
        return (match.group(1), match.group(2))
    raise ValueError(f"Cannot parse tag: {tag}")


def detect_language(file_path: str) -> str:
    """Detect programming language from file extension."""
    ext = Path(file_path).suffix.lower()
    return {".py": "python", ".ts": "typescript", ".tsx": "typescript", ".js": "javascript"}.get(
        ext, "unknown"
    )


def group_tags_by_component(tags: list[str]) -> dict[str, list[str]]:
    """Group git tags by their component."""
    grouped: dict[str, list[str]] = {}
    for tag in tags:
        try:
            component, _ = parse_tag(tag)
            if component not in grouped:
                grouped[component] = []
            grouped[component].append(tag)
        except ValueError:
            continue
    return grouped


@app.function(
    image=image,
    volumes={CODE_VOLUME_PATH: code_volume},
    secrets=[github_secret],
    timeout=3600,  # 1 hour per component
    cpu=2.0,
    memory=8192,
)
def index_component(component: str, tags: list[str], repo_path: str) -> dict:
    """Index a single component's tags into its own SQLite and LanceDB.

    Each component gets its own databases to enable parallel processing.

    Args:
        component: The component name (e.g., "agent", "computer")
        tags: List of git tags for this component
        repo_path: Path to the bare git repository

    Returns:
        Dict with indexing statistics
    """
    import shutil
    import subprocess
    import tempfile

    import lancedb
    from lancedb.embeddings import get_registry
    from lancedb.pydantic import LanceModel, Vector

    print(f"[{component}] Starting indexing of {len(tags)} tags...")

    DB_DIR = Path(CODE_DB_PATH)
    DB_DIR.mkdir(parents=True, exist_ok=True)

    # Component-specific database paths
    SQLITE_PATH = DB_DIR / f"code_index_{component}.sqlite"
    TMP_LANCE_DIR = Path(tempfile.mkdtemp(prefix=f"code_lancedb_{component}_"))

    # Initialize SQLite for this component
    if SQLITE_PATH.exists():
        SQLITE_PATH.unlink()

    conn = sqlite3.connect(SQLITE_PATH)
    cursor = conn.cursor()

    cursor.execute("""
        CREATE TABLE code_files (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            component TEXT NOT NULL,
            version TEXT NOT NULL,
            file_path TEXT NOT NULL,
            content TEXT NOT NULL,
            language TEXT NOT NULL,
            UNIQUE(component, version, file_path)
        )
    """)
    cursor.execute("CREATE INDEX idx_component ON code_files(component)")
    cursor.execute("CREATE INDEX idx_version ON code_files(component, version)")

    cursor.execute("""
        CREATE VIRTUAL TABLE code_files_fts USING fts5(
            content,
            component UNINDEXED,
            version UNINDEXED,
            file_path UNINDEXED,
            content='code_files',
            content_rowid='id'
        )
    """)

    cursor.execute("""
        CREATE TRIGGER code_files_ai AFTER INSERT ON code_files BEGIN
            INSERT INTO code_files_fts(rowid, content, component, version, file_path)
            VALUES (new.id, new.content, new.component, new.version, new.file_path);
        END;
    """)
    conn.commit()

    # Initialize LanceDB
    model = get_registry().get("sentence-transformers").create(name="all-MiniLM-L6-v2")

    class CodeFile(LanceModel):
        text: str = model.SourceField()
        vector: Vector(model.ndims()) = model.VectorField()
        component: str
        version: str
        file_path: str
        language: str

    lance_db = lancedb.connect(TMP_LANCE_DIR)
    lance_table = lance_db.create_table("code", schema=CodeFile, mode="overwrite")

    # Process tags for this component
    total_files = 0
    total_embedded = 0
    failed_tags = []
    COMMIT_BATCH_SIZE = 10

    for i, tag in enumerate(tags):
        print(f"[{component}] [{i + 1}/{len(tags)}] Processing {tag}")

        try:
            _, version = parse_tag(tag)
        except ValueError as e:
            print(f"[{component}]   Skipping: {e}")
            failed_tags.append(tag)
            continue

        # Get files at this tag
        try:
            result = subprocess.run(
                ["git", "ls-tree", "-r", "--name-only", tag],
                cwd=repo_path,
                check=True,
                capture_output=True,
                text=True,
            )
            files = [
                f.strip()
                for f in result.stdout.strip().split("\n")
                if f.strip() and Path(f.strip()).suffix.lower() in SOURCE_EXTENSIONS
            ]
        except subprocess.CalledProcessError:
            failed_tags.append(tag)
            continue

        lance_batch = []

        for file_path in files:
            try:
                result = subprocess.run(
                    ["git", "show", f"{tag}:{file_path}"],
                    cwd=repo_path,
                    check=True,
                    capture_output=True,
                )
                content = result.stdout.decode("utf-8", errors="replace")
                if "\x00" in content[:1024]:
                    continue  # Skip binary
            except (subprocess.CalledProcessError, UnicodeDecodeError):
                continue

            language = detect_language(file_path)
            content_size = len(content)

            # Add to SQLite
            if content_size <= MAX_FILE_SIZE_SQLITE:
                cursor.execute(
                    "INSERT OR REPLACE INTO code_files (component, version, file_path, content, language) VALUES (?, ?, ?, ?, ?)",
                    (component, version, file_path, content, language),
                )
                total_files += 1

            # Queue for LanceDB
            if content_size <= MAX_FILE_SIZE_EMBEDDINGS:
                lance_batch.append(
                    {
                        "text": content,
                        "component": component,
                        "version": version,
                        "file_path": file_path,
                        "language": language,
                    }
                )

        # Batch commits
        if (i + 1) % COMMIT_BATCH_SIZE == 0:
            conn.commit()
            print(f"[{component}]   Committed batch at tag {i + 1}/{len(tags)}")

        # Add to LanceDB
        if lance_batch:
            lance_table.add(lance_batch)
            total_embedded += len(lance_batch)

    # Final commit
    conn.commit()
    conn.close()

    # Copy LanceDB to volume
    lance_dest = DB_DIR / f"code_index_{component}.lancedb"
    if lance_dest.exists():
        shutil.rmtree(lance_dest)
    shutil.copytree(TMP_LANCE_DIR, lance_dest)
    shutil.rmtree(TMP_LANCE_DIR)

    # Clean up LanceDB resources
    del lance_table
    del lance_db

    print(f"[{component}] Complete: {total_files} files, {total_embedded} embedded")
    return {
        "component": component,
        "files": total_files,
        "embedded": total_embedded,
        "failed_tags": len(failed_tags),
        "tags_processed": len(tags),
    }


@app.function(
    image=image,
    volumes={CODE_VOLUME_PATH: code_volume},
    secrets=[github_secret],
    timeout=3600,  # 1 hour
    cpu=1.0,
    memory=4096,
)
def generate_code_index_parallel(max_concurrent: int = 4) -> dict:
    """Generate code search index with parallel component processing.

    This function:
    1. Clones/updates the git repository
    2. Groups tags by component
    3. Dispatches parallel workers to index each component
    4. Each component gets its own SQLite and LanceDB

    Args:
        max_concurrent: Maximum number of concurrent component indexing jobs

    Returns:
        Aggregated statistics from all component workers
    """
    import os
    import subprocess

    print(f"Starting parallel code indexing (max {max_concurrent} concurrent)...")

    # Build authenticated URL
    github_token = os.environ.get("GITHUB_TOKEN", "")
    if github_token:
        REPO_URL = f"https://{github_token}@github.com/trycua/cua.git"
        print("Using authenticated GitHub URL")
    else:
        REPO_URL = "https://github.com/trycua/cua.git"
        print("Warning: No GITHUB_TOKEN found")

    REPO_PATH = Path(CODE_REPO_PATH)

    # Clone or update repo
    if REPO_PATH.exists():
        print("Fetching latest tags...")
        subprocess.run(["git", "fetch", "--all", "--tags"], cwd=REPO_PATH, check=True)
    else:
        print("Cloning repository...")
        REPO_PATH.parent.mkdir(parents=True, exist_ok=True)
        subprocess.run(["git", "clone", "--bare", REPO_URL, str(REPO_PATH)], check=True)

    # Get all tags
    result = subprocess.run(
        ["git", "tag"], cwd=REPO_PATH, check=True, capture_output=True, text=True
    )
    all_tags = [t.strip() for t in result.stdout.strip().split("\n") if t.strip()]
    print(f"Found {len(all_tags)} tags")

    # Group tags by component
    component_tags = group_tags_by_component(all_tags)
    print(f"Components found: {list(component_tags.keys())}")
    for comp, tags in component_tags.items():
        print(f"  {comp}: {len(tags)} tags")

    # Dispatch parallel workers using Modal's map
    repo_path_str = str(REPO_PATH)
    args = [(comp, tags, repo_path_str) for comp, tags in component_tags.items()]

    print(f"Dispatching {len(args)} parallel indexing jobs...")

    # Process results with error handling for individual component failures
    results = []
    failed_components = []

    try:
        # Use return_exceptions=True to get results even when some workers fail
        for i, result in enumerate(
            index_component.starmap(args, order_outputs=False, return_exceptions=True)
        ):
            comp_name = args[i][0] if i < len(args) else f"component_{i}"
            if isinstance(result, Exception):
                # Handle individual component failures
                error_msg = str(result)
                print(f"[{comp_name}] Component indexing failed: {error_msg}")
                failed_components.append(
                    {
                        "component": comp_name,
                        "error": error_msg,
                        "files": 0,
                        "embedded": 0,
                        "failed_tags": len(args[i][1]) if i < len(args) else 0,
                    }
                )
            else:
                results.append(result)
                print(f"[{comp_name}] Component indexing succeeded")
    except Exception as e:
        # Handle catastrophic failures (e.g., all workers failed)
        print(f"Error during parallel indexing: {e}")
        # Still try to commit any partial results
        pass

    # Commit all changes to volume (including partial results)
    code_volume.commit()

    # Aggregate results from successful components
    total_files = sum(r["files"] for r in results)
    total_embedded = sum(r["embedded"] for r in results)
    total_failed = sum(r["failed_tags"] for r in results)

    # Add failed component stats
    total_failed += sum(f["failed_tags"] for f in failed_components)

    summary = {
        "total_files": total_files,
        "total_embedded": total_embedded,
        "total_failed_tags": total_failed,
        "components": results,
        "failed_components": failed_components,
        "success_count": len(results),
        "failure_count": len(failed_components),
    }

    print("\nParallel indexing complete:")
    print(f"  Total files: {total_files}")
    print(f"  Total embedded: {total_embedded}")
    print(f"  Components indexed: {len(results)}")
    if failed_components:
        print(f"  Components failed: {len(failed_components)}")
        for fc in failed_components:
            print(f"    - {fc['component']}: {fc['error'][:100]}")

    return summary


@app.function(
    image=image,
    volumes={CODE_VOLUME_PATH: code_volume},
    timeout=1800,  # 30 minutes
    cpu=2.0,
    memory=8192,
)
def aggregate_code_databases() -> dict:
    """Aggregate per-component databases into unified SQLite and LanceDB.

    This function runs after parallel indexing to create single aggregated
    databases that the MCP server can query directly, avoiding runtime
    aggregation overhead.

    Creates:
    - code_index.sqlite: Unified SQLite with FTS5 from all components
    - code_index.lancedb: Unified LanceDB with vectors from all components

    Returns:
        Dict with aggregation statistics
    """
    import shutil
    import tempfile

    import lancedb
    from lancedb.embeddings import get_registry
    from lancedb.pydantic import LanceModel, Vector

    print("Aggregating component databases...")

    DB_DIR = Path(CODE_DB_PATH)
    if not DB_DIR.exists():
        print("No database directory found")
        return {"error": "No database directory"}

    # Find all component SQLite databases
    component_dbs = list(DB_DIR.glob("code_index_*.sqlite"))
    if not component_dbs:
        print("No component databases found to aggregate")
        return {"error": "No component databases found"}

    print(f"Found {len(component_dbs)} component databases to aggregate")

    # === Aggregate SQLite databases ===
    AGGREGATED_SQLITE = DB_DIR / "code_index.sqlite"
    if AGGREGATED_SQLITE.exists():
        AGGREGATED_SQLITE.unlink()
        print("Removed existing aggregated SQLite database")

    conn = sqlite3.connect(AGGREGATED_SQLITE)
    cursor = conn.cursor()

    # Create main table
    cursor.execute("""
        CREATE TABLE code_files (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            component TEXT NOT NULL,
            version TEXT NOT NULL,
            file_path TEXT NOT NULL,
            content TEXT NOT NULL,
            language TEXT NOT NULL,
            UNIQUE(component, version, file_path)
        )
    """)
    cursor.execute("CREATE INDEX idx_component ON code_files(component)")
    cursor.execute("CREATE INDEX idx_version ON code_files(component, version)")

    # Create FTS5 virtual table
    cursor.execute("""
        CREATE VIRTUAL TABLE code_files_fts USING fts5(
            content,
            component UNINDEXED,
            version UNINDEXED,
            file_path UNINDEXED,
            content='code_files',
            content_rowid='id'
        )
    """)

    # Create FTS triggers BEFORE inserting data
    cursor.execute("""
        CREATE TRIGGER code_files_ai AFTER INSERT ON code_files BEGIN
            INSERT INTO code_files_fts(rowid, content, component, version, file_path)
            VALUES (new.id, new.content, new.component, new.version, new.file_path);
        END;
    """)
    conn.commit()

    # Copy data from each component database
    total_rows = 0
    for db_path in component_dbs:
        component_name = db_path.stem.replace("code_index_", "")
        print(f"  Aggregating {component_name}...")

        # Attach component database
        cursor.execute(f"ATTACH DATABASE 'file:{db_path}?mode=ro' AS comp")

        # Copy data (triggers will populate FTS automatically)
        cursor.execute("""
            INSERT INTO code_files (component, version, file_path, content, language)
            SELECT component, version, file_path, content, language FROM comp.code_files
        """)
        rows_copied = cursor.rowcount
        total_rows += rows_copied
        print(f"    Copied {rows_copied} rows from {component_name}")

        # Commit before detaching to release locks on the attached database
        conn.commit()
        cursor.execute("DETACH DATABASE comp")

    conn.close()
    print(f"SQLite aggregation complete: {total_rows} total rows")

    # === Aggregate LanceDB databases ===
    TMP_LANCE_DIR = Path(tempfile.mkdtemp(prefix="code_lancedb_agg_"))

    # Initialize embedding model and schema
    model = get_registry().get("sentence-transformers").create(name="all-MiniLM-L6-v2")

    class CodeFile(LanceModel):
        text: str = model.SourceField()
        vector: Vector(model.ndims()) = model.VectorField()
        component: str
        version: str
        file_path: str
        language: str

    lance_db = lancedb.connect(TMP_LANCE_DIR)
    lance_table = lance_db.create_table("code", schema=CodeFile, mode="overwrite")

    # Find and aggregate all component LanceDBs
    component_lance_dirs = list(DB_DIR.glob("code_index_*.lancedb"))
    total_vectors = 0

    for lance_dir in component_lance_dirs:
        component_name = lance_dir.stem.replace("code_index_", "").replace(".lancedb", "")
        print(f"  Aggregating vectors from {component_name}...")

        try:
            comp_db = lancedb.connect(lance_dir)
            comp_table = comp_db.open_table("code")

            # Read all data from component table (excluding vector column for re-embedding)
            # Actually, we want to preserve the vectors, so read everything
            data = comp_table.to_pandas()

            if len(data) > 0:
                # Convert to list of dicts, preserving vectors
                records = data.to_dict("records")
                lance_table.add(records)
                total_vectors += len(records)
                print(f"    Added {len(records)} vectors from {component_name}")

            del comp_table
            del comp_db
        except Exception as e:
            print(f"    Error aggregating {component_name}: {e}")
            continue

    # Close and copy to volume
    del lance_table
    del lance_db

    AGGREGATED_LANCE = DB_DIR / "code_index.lancedb"
    if AGGREGATED_LANCE.exists():
        shutil.rmtree(AGGREGATED_LANCE)
    shutil.copytree(TMP_LANCE_DIR, AGGREGATED_LANCE)
    shutil.rmtree(TMP_LANCE_DIR)

    # Commit changes to volume
    code_volume.commit()

    print(f"LanceDB aggregation complete: {total_vectors} total vectors")
    print("Aggregation complete!")

    return {
        "sqlite_rows": total_rows,
        "lance_vectors": total_vectors,
        "components_aggregated": len(component_dbs),
    }


@app.function(
    image=image,
    volumes={CODE_VOLUME_PATH: code_volume},
    secrets=[github_secret],
    timeout=3600,  # 1 hour
    cpu=2.0,
    memory=8192,
)
async def generate_code_index():
    """Generate code search index from all git tags"""
    import os
    import shutil
    import subprocess
    import tempfile

    import lancedb
    from lancedb.embeddings import get_registry
    from lancedb.pydantic import LanceModel, Vector

    print("Generating code search index...")

    # Build authenticated URL using GitHub token
    github_token = os.environ.get("GITHUB_TOKEN", "")
    if github_token:
        REPO_URL = f"https://{github_token}@github.com/trycua/cua.git"
        print("Using authenticated GitHub URL")
    else:
        REPO_URL = "https://github.com/trycua/cua.git"
        print("Warning: No GITHUB_TOKEN found, using unauthenticated URL")

    REPO_PATH = Path(CODE_REPO_PATH)
    DB_DIR = Path(CODE_DB_PATH)
    SQLITE_PATH = DB_DIR / "code_index.sqlite"

    DB_DIR.mkdir(parents=True, exist_ok=True)

    # Clone or update repo (bare clone for efficiency)
    if REPO_PATH.exists():
        print("Fetching latest tags...")
        subprocess.run(["git", "fetch", "--all", "--tags"], cwd=REPO_PATH, check=True)
    else:
        print("Cloning repository...")
        REPO_PATH.parent.mkdir(parents=True, exist_ok=True)
        subprocess.run(["git", "clone", "--bare", REPO_URL, str(REPO_PATH)], check=True)

    # Get all tags
    result = subprocess.run(
        ["git", "tag"], cwd=REPO_PATH, check=True, capture_output=True, text=True
    )
    all_tags = [t.strip() for t in result.stdout.strip().split("\n") if t.strip()]
    print(f"Found {len(all_tags)} tags")

    # Initialize SQLite
    if SQLITE_PATH.exists():
        SQLITE_PATH.unlink()

    conn = sqlite3.connect(SQLITE_PATH)
    cursor = conn.cursor()

    cursor.execute("""
        CREATE TABLE code_files (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            component TEXT NOT NULL,
            version TEXT NOT NULL,
            file_path TEXT NOT NULL,
            content TEXT NOT NULL,
            language TEXT NOT NULL,
            UNIQUE(component, version, file_path)
        )
    """)
    cursor.execute("CREATE INDEX idx_component ON code_files(component)")
    cursor.execute("CREATE INDEX idx_version ON code_files(component, version)")

    cursor.execute("""
        CREATE VIRTUAL TABLE code_files_fts USING fts5(
            content,
            component UNINDEXED,
            version UNINDEXED,
            file_path UNINDEXED,
            content='code_files',
            content_rowid='id'
        )
    """)

    # FTS triggers
    cursor.execute("""
        CREATE TRIGGER code_files_ai AFTER INSERT ON code_files BEGIN
            INSERT INTO code_files_fts(rowid, content, component, version, file_path)
            VALUES (new.id, new.content, new.component, new.version, new.file_path);
        END;
    """)
    conn.commit()

    # Initialize LanceDB in temp directory
    TMP_LANCE_DIR = Path(tempfile.mkdtemp(prefix="code_lancedb_"))
    model = get_registry().get("sentence-transformers").create(name="all-MiniLM-L6-v2")

    class CodeFile(LanceModel):
        text: str = model.SourceField()
        vector: Vector(model.ndims()) = model.VectorField()
        component: str
        version: str
        file_path: str
        language: str

    lance_db = lancedb.connect(TMP_LANCE_DIR)
    lance_table = lance_db.create_table("code", schema=CodeFile, mode="overwrite")

    # Process each tag
    total_files = 0
    total_embedded = 0
    failed_tags = []
    COMMIT_BATCH_SIZE = 10  # Commit every 10 tags for better performance

    for i, tag in enumerate(all_tags):
        print(f"[{i + 1}/{len(all_tags)}] Processing {tag}")

        try:
            component, version = parse_tag(tag)
        except ValueError as e:
            print(f"  Skipping: {e}")
            failed_tags.append(tag)
            continue

        # Get files at this tag
        try:
            result = subprocess.run(
                ["git", "ls-tree", "-r", "--name-only", tag],
                cwd=REPO_PATH,
                check=True,
                capture_output=True,
                text=True,
            )
            files = [
                f.strip()
                for f in result.stdout.strip().split("\n")
                if f.strip() and Path(f.strip()).suffix.lower() in SOURCE_EXTENSIONS
            ]
        except subprocess.CalledProcessError:
            failed_tags.append(tag)
            continue

        lance_batch = []

        for file_path in files:
            try:
                result = subprocess.run(
                    ["git", "show", f"{tag}:{file_path}"],
                    cwd=REPO_PATH,
                    check=True,
                    capture_output=True,
                )
                content = result.stdout.decode("utf-8", errors="replace")
                if "\x00" in content[:1024]:
                    continue  # Skip binary
            except (subprocess.CalledProcessError, UnicodeDecodeError):
                continue

            language = detect_language(file_path)
            content_size = len(content)

            # Add to SQLite
            if content_size <= MAX_FILE_SIZE_SQLITE:
                cursor.execute(
                    "INSERT OR REPLACE INTO code_files (component, version, file_path, content, language) VALUES (?, ?, ?, ?, ?)",
                    (component, version, file_path, content, language),
                )
                total_files += 1

            # Queue for LanceDB
            if content_size <= MAX_FILE_SIZE_EMBEDDINGS:
                lance_batch.append(
                    {
                        "text": content,
                        "component": component,
                        "version": version,
                        "file_path": file_path,
                        "language": language,
                    }
                )

        # Batch commits: commit every COMMIT_BATCH_SIZE tags
        if (i + 1) % COMMIT_BATCH_SIZE == 0:
            conn.commit()
            print(f"  Committed batch at tag {i + 1}/{len(all_tags)}")

        # Add to LanceDB
        if lance_batch:
            lance_table.add(lance_batch)
            total_embedded += len(lance_batch)

    # Final commit for any remaining tags
    conn.commit()
    conn.close()

    # Copy LanceDB to volume
    try:
        lance_dest = DB_DIR / "code_index.lancedb"
        if lance_dest.exists():
            shutil.rmtree(lance_dest)
        shutil.copytree(TMP_LANCE_DIR, lance_dest)
        shutil.rmtree(TMP_LANCE_DIR)
    finally:
        # Ensure LanceDB resources are released even if an exception occurs
        del lance_table
        del lance_db
    code_volume.commit()

    print(f"Code index complete: {total_files} files in SQLite, {total_embedded} embedded")
    return {"files": total_files, "embedded": total_embedded, "failed_tags": len(failed_tags)}


@app.function(
    image=image,
    volumes={CODE_VOLUME_PATH: code_volume},
    secrets=[github_secret],
    schedule=modal.Cron("0 5 * * *"),  # Daily at 5 AM UTC (before docs crawl)
    timeout=7200,  # 2 hours (includes aggregation time)
)
async def scheduled_code_index():
    """Scheduled daily code index generation (uses parallel processing)"""
    import modal.exception

    print("Running scheduled code indexing (parallel)...")
    try:
        result = await generate_code_index_parallel.remote.aio()
        print(f"Code indexing complete: {result}")

        # Log summary of any failed components
        if result.get("failed_components"):
            print(
                f"Warning: {len(result['failed_components'])} component(s) failed during indexing"
            )
            for fc in result["failed_components"]:
                print(f"  - {fc['component']}: {fc['error'][:200]}")

        # Aggregate component databases into unified DBs for the MCP server
        print("Aggregating component databases...")
        agg_result = aggregate_code_databases.remote()
        print(f"Aggregation complete: {agg_result}")
        result["aggregation"] = agg_result

        # Sync code databases to S3
        print("Syncing code databases to S3...")
        sync_result = sync_to_s3.remote(publish_docs=False)
        print(f"S3 sync result: {sync_result}")
        result["s3_sync"] = sync_result

        return result
    except modal.exception.FunctionTimeoutError as e:
        print(f"Code indexing timed out: {e}")
        # Return a partial result indicating the timeout
        return {
            "total_files": 0,
            "total_embedded": 0,
            "total_failed_tags": 0,
            "components": [],
            "failed_components": [],
            "error": f"Function timed out: {str(e)}",
            "success_count": 0,
            "failure_count": 0,
        }
    except Exception as e:
        print(f"Code indexing failed with error: {e}")
        # Return error information instead of crashing
        return {
            "total_files": 0,
            "total_embedded": 0,
            "total_failed_tags": 0,
            "components": [],
            "failed_components": [],
            "error": str(e),
            "success_count": 0,
            "failure_count": 0,
        }


# =============================================================================
# MCP Server
# =============================================================================


@app.function(
    image=image,
    volumes={VOLUME_PATH: docs_volume, CODE_VOLUME_PATH: code_volume},
    cpu=1.0,
    memory=2048,
    min_containers=1,  # Keep one container warm to avoid cold start latency
)
@modal.concurrent(max_inputs=10)
@modal.asgi_app(custom_domains=["docs-mcp.cua.ai"])
def web():
    """ASGI web endpoint for the MCP server"""
    import lancedb
    from fastmcp import FastMCP
    from lancedb.embeddings import get_registry
    from starlette.middleware.cors import CORSMiddleware

    # Initialize the MCP server
    mcp = FastMCP(
        name="CUA Docs & Code",
        instructions="""CUA Documentation and Code Server - provides direct read-only query access to Computer Use Agent (CUA) documentation and versioned source code.

=== AVAILABLE TOOLS ===

Documentation:
- query_docs_db: Execute SQL queries against the documentation SQLite database
- query_docs_vectors: Execute vector similarity searches against the documentation LanceDB

Code:
- query_code_db: Execute SQL queries against the code search SQLite database
- query_code_vectors: Execute vector similarity searches against the code LanceDB

All tools are READ-ONLY. Only SELECT queries are allowed for SQL databases.

=== DOCUMENTATION DATABASE ===

The documentation database contains crawled pages from cua.ai/docs covering:
- CUA SDK: Python library for building computer-use agents
- CUA Bench: Benchmarking framework for evaluating computer-use agents
- Agent Loop: Core execution loop for autonomous agent operation
- Sandboxes: Docker and cloud VM environments for safe agent execution
- Computer interfaces: Screen, mouse, keyboard, and bash interaction APIs

=== CODE DATABASE ===

The code database contains Python, TypeScript, and JavaScript files from indexed git tags.
Enumerate code_files for serving component, version, and language coverage.
Component names identify tag families, not complete standalone product coverage.

=== WORKFLOW EXAMPLES ===

1. Find documentation about a topic:
   - Use query_docs_vectors with a natural language query for semantic search
   - Use query_docs_db with FTS5 MATCH for keyword search

2. Explore code across versions:
   - List components: SELECT component, COUNT(DISTINCT version) FROM code_files GROUP BY component
   - Search code: Use query_code_db with FTS5 on code_files_fts
   - Get file content: SELECT content FROM code_files WHERE component='agent' AND version='0.7.3' AND file_path='...'

3. Semantic code search:
   - Use query_code_vectors with natural language queries like "screenshot capture implementation"

IMPORTANT: Always cite sources - URLs for docs, component@version:path for code.""",
    )

    # Initialize embedding model - load eagerly to avoid cold start on first search
    print("Initializing embedding model...")
    model = get_registry().get("sentence-transformers").create(name="all-MiniLM-L6-v2")

    # Eagerly initialize database connections at startup to reduce first-request latency
    print("Initializing database connections...")

    import tempfile
    from docs_index import GenerationReader, copy_generation

    # Keep open handles off the Modal volume so reload can see new commits.
    local_docs = Path(tempfile.mkdtemp(prefix="docs-reader-"))

    def refresh_docs():
        docs_volume.reload()
        copy_generation(Path(DB_PATH), local_docs)

    docs_reader = GenerationReader(local_docs, refresh=refresh_docs)

    # Code LanceDB
    _code_lance_db = None
    _code_lance_table = None
    code_lance_path = Path(CODE_DB_PATH) / "code_index.lancedb"
    if code_lance_path.exists():
        try:
            _code_lance_db = lancedb.connect(code_lance_path)
            _code_lance_table = _code_lance_db.open_table("code")
            print(f"  Code LanceDB loaded from {code_lance_path}")
        except Exception as e:
            print(f"  Warning: Could not load code LanceDB: {e}")

    # Code SQLite
    _code_sqlite_conn = None
    code_sqlite_path = Path(CODE_DB_PATH) / "code_index.sqlite"
    if code_sqlite_path.exists():
        try:
            _code_sqlite_conn = sqlite3.connect(f"file:{code_sqlite_path}?mode=ro", uri=True)
            _code_sqlite_conn.row_factory = sqlite3.Row
            print(f"  Code SQLite loaded from {code_sqlite_path}")
        except Exception as e:
            print(f"  Warning: Could not load code SQLite: {e}")

    print("Database initialization complete.")

    def get_lance_table():
        return docs_reader.get()[1]

    def get_sqlite_conn():
        return docs_reader.get()[0]

    def get_code_lance_table():
        """Get LanceDB connection for the aggregated code database (eagerly loaded)."""
        if _code_lance_table is None:
            raise RuntimeError(
                "Code LanceDB not found. Run generate_code_index_parallel and aggregate_code_databases first."
            )
        return _code_lance_table

    def get_code_sqlite_conn():
        """Get read-only SQLite connection for the aggregated code database (eagerly loaded)."""
        if _code_sqlite_conn is None:
            raise RuntimeError(
                "Code SQLite database not found. Run generate_code_index_parallel and aggregate_code_databases first."
            )
        return _code_sqlite_conn

    # =================== DOCUMENTATION QUERY TOOLS (READ-ONLY) ===================

    @mcp.tool()
    def query_docs_db(sql: str) -> list[dict]:
        """
        Execute a SQL query against the documentation database.
        The database is READ-ONLY.

        Database Schema:

        Table: pages
        - id INTEGER PRIMARY KEY AUTOINCREMENT
        - url TEXT NOT NULL UNIQUE         -- Full URL of the documentation page
        - title TEXT NOT NULL              -- Page title
        - category TEXT NOT NULL           -- URL category (e.g., 'reference', 'tutorials')
        - build_id TEXT NOT NULL           -- Verified paired-generation identifier
        - content TEXT NOT NULL            -- Plain text content (markdown stripped)

        Virtual Table: pages_fts (FTS5 full-text search)
        - content TEXT                     -- Full-text indexed content
        - url TEXT UNINDEXED
        - title TEXT UNINDEXED
        - category TEXT UNINDEXED

        Example queries:

        1. List all pages: SELECT url, title, category FROM pages ORDER BY category, title

        2. Full-text search with snippets:
           SELECT p.url, p.title, snippet(pages_fts, 0, '>>>', '<<<', '...', 64) as snippet
           FROM pages_fts JOIN pages p ON pages_fts.rowid = p.id
           WHERE pages_fts MATCH 'agent loop' ORDER BY rank LIMIT 10

        3. Get page content: SELECT url, title, content FROM pages WHERE url LIKE '%quickstart%'

        Args:
            sql: SQL query to execute

        Returns:
            List of dictionaries, one per row, with column names as keys
        """
        conn = get_sqlite_conn()
        cursor = conn.cursor()
        cursor.execute(sql)
        return [dict(row) for row in cursor.fetchall()]

    @mcp.tool()
    def query_docs_vectors(
        query: str,
        limit: int = 10,
        where: Optional[str] = None,
        select: Optional[list[str]] = None,
    ) -> list[dict]:
        """
        Execute a vector similarity search against the documentation LanceDB (read-only).

        Schema:
        - text TEXT           -- The document chunk text
        - vector VECTOR       -- Embedding vector (all-MiniLM-L6-v2, 384 dimensions)
        - url TEXT            -- Source URL
        - title TEXT          -- Document title
        - category TEXT       -- URL category (e.g., 'reference', 'tutorials')
        - build_id TEXT       -- Verified paired-generation identifier
        - chunk_index INT     -- Index of chunk within document

        Args:
            query: Natural language query to embed and search for
            limit: Maximum number of results (default: 10, max: 100)
            where: Optional SQL-like filter (e.g., "category = 'reference'")
            select: Optional list of columns to return (default: all except vector)

        Returns:
            List of matching documents with similarity scores (_distance field)
        """
        limit = min(max(1, limit), 100)
        table = get_lance_table()

        search = table.search(query).limit(limit)

        if where:
            search = search.where(where)
        if select:
            search = search.select(select)

        results = search.to_list()

        formatted = []
        for r in results:
            result = {}
            for key, value in r.items():
                if key == "vector":
                    continue
                result[key] = value
            formatted.append(result)

        return formatted

    # =================== CODE QUERY TOOLS (READ-ONLY) ===================

    @mcp.tool()
    def query_code_db(sql: str) -> list[dict]:
        """
        Execute a SQL query against the code search database.
        The database is READ-ONLY.

        Database Schema:

        Table: code_files
        - id INTEGER PRIMARY KEY AUTOINCREMENT
        - component TEXT NOT NULL          -- Component name (e.g., "agent", "computer")
        - version TEXT NOT NULL            -- Version string (e.g., "0.7.3")
        - file_path TEXT NOT NULL          -- Path to file
        - content TEXT NOT NULL            -- Full source code content
        - language TEXT NOT NULL           -- Programming language
        - UNIQUE(component, version, file_path)

        Virtual Table: code_files_fts (FTS5 full-text search)
        - content TEXT                     -- Full-text indexed content
        - component TEXT UNINDEXED
        - version TEXT UNINDEXED
        - file_path TEXT UNINDEXED

        Example queries:

        1. List components: SELECT component, COUNT(DISTINCT version) as version_count
           FROM code_files GROUP BY component ORDER BY component

        2. List versions: SELECT DISTINCT version FROM code_files
           WHERE component = 'agent' ORDER BY version DESC

        3. Full-text search:
           SELECT f.component, f.version, f.file_path,
                  snippet(code_files_fts, 0, '>>>', '<<<', '...', 64) as snippet
           FROM code_files_fts JOIN code_files f ON code_files_fts.rowid = f.id
           WHERE code_files_fts MATCH 'ComputerAgent' ORDER BY rank LIMIT 10

        4. Get file content: SELECT content, language FROM code_files
           WHERE component = 'agent' AND version = '0.7.3' AND file_path = 'agent/core.py'

        Args:
            sql: SQL query to execute

        Returns:
            List of dictionaries, one per row, with column names as keys
        """
        conn = get_code_sqlite_conn()
        cursor = conn.cursor()
        cursor.execute(sql)
        return [dict(row) for row in cursor.fetchall()]

    @mcp.tool()
    def query_code_vectors(
        query: str,
        limit: int = 10,
        where: Optional[str] = None,
        select: Optional[list[str]] = None,
        component: Optional[str] = None,
    ) -> list[dict]:
        """
        Execute a vector similarity search against the code LanceDB (read-only).

        Schema:
        - text TEXT           -- The source code content
        - vector VECTOR       -- Embedding vector (all-MiniLM-L6-v2, 384 dimensions)
        - component TEXT      -- Component name (e.g., "agent", "computer")
        - version TEXT        -- Version string (e.g., "0.7.3")
        - file_path TEXT      -- Path to file within the component
        - language TEXT       -- Programming language

        Args:
            query: Natural language query to embed and search for
            limit: Maximum number of results (default: 10, max: 100)
            where: Optional SQL-like filter (e.g., "version = '0.7.3'")
            select: Optional list of columns to return (default: all except vector)
            component: Optional component to filter by (if not specified, searches all)

        Returns:
            List of matching code files with similarity scores (_distance field)
        """
        limit = min(max(1, limit), 100)
        table = get_code_lance_table()

        search = table.search(query).limit(limit)

        # Build where clause, adding component filter if specified
        where_clauses = []
        if component:
            where_clauses.append(f"component = '{component}'")
        if where:
            where_clauses.append(where)

        if where_clauses:
            search = search.where(" AND ".join(where_clauses))
        if select:
            search = search.select(select)

        results = search.to_list()

        formatted = []
        for r in results:
            result = {}
            for key, value in r.items():
                if key == "vector":
                    continue
                result[key] = value
            formatted.append(result)

        return formatted

    # Create SSE app directly - endpoints at /sse (GET) and /messages (POST)
    from starlette.middleware import Middleware

    mcp_app = mcp.http_app(
        transport="sse",
        middleware=[
            Middleware(
                CORSMiddleware,
                allow_origins=["*"],
                allow_credentials=True,
                allow_methods=["*"],
                allow_headers=["*"],
            )
        ],
    )

    return mcp_app


# =============================================================================
# Local testing functions
# =============================================================================


@app.local_entrypoint()
def main(
    skip_docs: bool = False,
    skip_code: bool = False,
    parallel: bool = True,
    code_only: bool = False,
):
    """Run initial crawl and database generation

    Args:
        skip_docs: Skip documentation crawl and indexing
        skip_code: Skip code indexing
        parallel: Use parallel code indexing (default: True)
        code_only: Only run code indexing (shortcut for --skip-docs)
    """
    if code_only:
        skip_docs = True

    if not skip_docs:
        print("Running initial crawl...")
        summary = crawl_docs.remote()
        print(f"Crawl summary: {summary}")

        print("Generating paired docs indexes...")
        result = generate_docs_indexes.remote()
        print(f"Docs indexes: {result}")

    if not skip_code:
        if parallel:
            print("Generating code index (parallel)...")
            code_result = generate_code_index_parallel.remote()
        else:
            print("Generating code index (sequential)...")
            code_result = generate_code_index.remote()
        print(f"Code index: {code_result}")

        # Aggregate component databases for the MCP server
        print("Aggregating code databases...")
        agg_result = aggregate_code_databases.remote()
        print(f"Aggregation: {agg_result}")

    print("Done! Deploy with: modal deploy docs/scripts/modal_app.py")