Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
stabilizes the database architecture for the MCP server and fix broken tools (#805)
r33drichardsd663cfa8c554
diff --git a/docs/scripts/modal_app.py b/docs/scripts/modal_app.pyindex 5cc8346543..ab3b1edd1e 100644--- a/docs/scripts/modal_app.py+++ b/docs/scripts/modal_app.py@@ -343,6 +343,57 @@ async def scheduled_crawl():# =============================================================================# Database Generation Functions# =============================================================================+#+# The MCP server provides access to two types of query databases:+#+# 1. DOCUMENTATION DATABASES (from cua.ai/docs crawl):+# - SQLite FTS5 Database (docs.sqlite):+# * `pages` table: stores URL, title, category, and plain-text content+# * `pages_fts` virtual table: FTS5 full-text search index+# * Triggers keep FTS index synchronized with pages table+# * Built by: generate_sqlite_db()+#+# - LanceDB Vector Database (docs.lance/):+# * DocsChunk schema: text, vector (384-dim), url, title, category, chunk_index+# * Uses sentence-transformers/all-MiniLM-L6-v2 for embeddings+# * Chunks documents by paragraph for semantic search+# * Built by: generate_vector_db()+#+# 2. CODE INDEX DATABASES (from git tags):+# Per-component databases (built in parallel):+# - SQLite FTS5 Database (code_index_<component>.sqlite per component):+# * `code_files` table: component, version, file_path, content, language+# * `code_files_fts` virtual table: FTS5 full-text search index+# * Indexes source files (.py, .ts, .js, .tsx) from all git tags+# * Built by: index_component() called from generate_code_index_parallel()+#+# - LanceDB Vector Database (code_index_<component>.lancedb/ per component):+# * CodeFile schema: text, vector (384-dim), component, version, file_path, language+# * Only embeds files under 100KB to avoid memory issues+# * Built by: index_component() called from generate_code_index_parallel()+#+# Aggregated databases (for MCP server queries):+# - code_index.sqlite: Unified SQLite with all components' data + FTS5 index+# - code_index.lancedb: Unified LanceDB with all components' vectors+# - Built by: aggregate_code_databases() after parallel indexing completes+#+# Database Build Process:+# 1. scheduled_crawl() runs daily at 6 AM UTC:+# - Calls crawl_docs() to crawl cua.ai/docs+# - Calls generate_vector_db() to build LanceDB from crawled markdown+# - Calls generate_sqlite_db() to build SQLite FTS from crawled content+#+# 2. scheduled_code_index() runs daily at 5 AM UTC (before docs):+# - Calls generate_code_index_parallel() which:+# a. Clones/updates the git repository (bare clone)+# b. Groups all git tags by component (agent, computer, etc.)+# c. Dispatches parallel index_component() workers per component+# d. Each worker builds its own SQLite + LanceDB+# - Calls aggregate_code_databases() to merge per-component DBs into unified DBs+#+# Note: Modal volumes don't support atomic rename operations, so LanceDB is+# built in a temp directory first, then copied to the volume.+# =============================================================================@app.function(@@ -524,34 +575,10 @@ async def generate_sqlite_db():""")- # Load and insert data- json_files = list(CRAWLED_DIR.glob("*.json"))- json_files = [f for f in json_files if not f.name.startswith("_")]-- inserted = 0-- for json_file in json_files:- with open(json_file, "r", encoding="utf-8") as f:- page_data = json.load(f)-- url = page_data.get("url", "")- title = page_data.get("title", "")- markdown = page_data.get("markdown", "")- category = page_data.get("path_info", {}).get("category", "unknown")-- if not markdown:- continue-- # Convert markdown to plain text- text = clean_markdown(markdown)-- cursor.execute(- "INSERT OR REPLACE INTO pages (url, title, category, content) VALUES (?, ?, ?, ?)",- (url, title, category, text),- )- inserted += 1-- # Create FTS triggers+ # Create FTS triggers BEFORE inserting data+ # This is critical: since pages_fts uses external content (content='pages'),+ # the FTS index is only populated via these triggers. If triggers are created+ # after data insertion, the FTS table will be empty.cursor.execute("""CREATE TRIGGER pages_ai AFTER INSERT ON pages BEGIN@@ -579,6 +606,35 @@ async def generate_sqlite_db():""")+ conn.commit()++ # Load and insert data (triggers will populate FTS automatically)+ json_files = list(CRAWLED_DIR.glob("*.json"))+ json_files = [f for f in json_files if not f.name.startswith("_")]++ inserted = 0++ for json_file in json_files:+ with open(json_file, "r", encoding="utf-8") as f:+ page_data = json.load(f)++ url = page_data.get("url", "")+ title = page_data.get("title", "")+ markdown = page_data.get("markdown", "")+ category = page_data.get("path_info", {}).get("category", "unknown")++ if not markdown:+ continue++ # Convert markdown to plain text+ text = clean_markdown(markdown)++ cursor.execute(+ "INSERT OR REPLACE INTO pages (url, title, category, content) VALUES (?, ?, ?, ?)",+ (url, title, category, text),+ )+ inserted += 1+conn.commit()conn.close()@@ -965,6 +1021,196 @@ def generate_code_index_parallel(max_concurrent: int = 4) -> dict:return summary+ image=image,+ volumes={CODE_VOLUME_PATH: code_volume},+ timeout=1800, # 30 minutes+ cpu=2.0,+ memory=8192,+)+def aggregate_code_databases() -> dict:+ """Aggregate per-component databases into unified SQLite and LanceDB.++ This function runs after parallel indexing to create single aggregated+ databases that the MCP server can query directly, avoiding runtime+ aggregation overhead.++ Creates:+ - code_index.sqlite: Unified SQLite with FTS5 from all components+ - code_index.lancedb: Unified LanceDB with vectors from all components++ Returns:+ Dict with aggregation statistics+ """+ import shutil+ import tempfile++ import lancedb+ from lancedb.embeddings import get_registry+ from lancedb.pydantic import LanceModel, Vector++ print("Aggregating component databases...")++ DB_DIR = Path(CODE_DB_PATH)+ if not DB_DIR.exists():+ print("No database directory found")+ return {"error": "No database directory"}++ # Find all component SQLite databases+ component_dbs = list(DB_DIR.glob("code_index_*.sqlite"))+ if not component_dbs:+ print("No component databases found to aggregate")+ return {"error": "No component databases found"}++ print(f"Found {len(component_dbs)} component databases to aggregate")++ # === Aggregate SQLite databases ===+ AGGREGATED_SQLITE = DB_DIR / "code_index.sqlite"+ if AGGREGATED_SQLITE.exists():+ AGGREGATED_SQLITE.unlink()+ print("Removed existing aggregated SQLite database")++ conn = sqlite3.connect(AGGREGATED_SQLITE)+ cursor = conn.cursor()++ # Create main table+ cursor.execute(+ """+ CREATE TABLE code_files (+ id INTEGER PRIMARY KEY AUTOINCREMENT,+ component TEXT NOT NULL,+ version TEXT NOT NULL,+ file_path TEXT NOT NULL,+ content TEXT NOT NULL,+ language TEXT NOT NULL,+ UNIQUE(component, version, file_path)+ )+ """+ )+ cursor.execute("CREATE INDEX idx_component ON code_files(component)")+ cursor.execute("CREATE INDEX idx_version ON code_files(component, version)")++ # Create FTS5 virtual table+ cursor.execute(+ """+ CREATE VIRTUAL TABLE code_files_fts USING fts5(+ content,+ component UNINDEXED,+ version UNINDEXED,+ file_path UNINDEXED,+ content='code_files',+ content_rowid='id'+ )+ """+ )++ # Create FTS triggers BEFORE inserting data+ cursor.execute(+ """+ CREATE TRIGGER code_files_ai AFTER INSERT ON code_files BEGIN+ INSERT INTO code_files_fts(rowid, content, component, version, file_path)+ VALUES (new.id, new.content, new.component, new.version, new.file_path);+ END;+ """+ )+ conn.commit()++ # Copy data from each component database+ total_rows = 0+ for db_path in component_dbs:+ component_name = db_path.stem.replace("code_index_", "")+ print(f" Aggregating {component_name}...")++ # Attach component database+ cursor.execute(f"ATTACH DATABASE 'file:{db_path}?mode=ro' AS comp")++ # Copy data (triggers will populate FTS automatically)+ cursor.execute(+ """+ INSERT INTO code_files (component, version, file_path, content, language)+ SELECT component, version, file_path, content, language FROM comp.code_files+ """+ )+ rows_copied = cursor.rowcount+ total_rows += rows_copied+ print(f" Copied {rows_copied} rows from {component_name}")++ # Commit before detaching to release locks on the attached database+ conn.commit()+ cursor.execute("DETACH DATABASE comp")++ conn.close()+ print(f"SQLite aggregation complete: {total_rows} total rows")++ # === Aggregate LanceDB databases ===+ TMP_LANCE_DIR = Path(tempfile.mkdtemp(prefix="code_lancedb_agg_"))++ # Initialize embedding model and schema+ model = get_registry().get("sentence-transformers").create(name="all-MiniLM-L6-v2")++ class CodeFile(LanceModel):+ text: str = model.SourceField()+ vector: Vector(model.ndims()) = model.VectorField()+ component: str+ version: str+ file_path: str+ language: str++ lance_db = lancedb.connect(TMP_LANCE_DIR)+ lance_table = lance_db.create_table("code", schema=CodeFile, mode="overwrite")++ # Find and aggregate all component LanceDBs+ component_lance_dirs = list(DB_DIR.glob("code_index_*.lancedb"))+ total_vectors = 0++ for lance_dir in component_lance_dirs:+ component_name = lance_dir.stem.replace("code_index_", "").replace(".lancedb", "")+ print(f" Aggregating vectors from {component_name}...")++ try:+ comp_db = lancedb.connect(lance_dir)+ comp_table = comp_db.open_table("code")++ # Read all data from component table (excluding vector column for re-embedding)+ # Actually, we want to preserve the vectors, so read everything+ data = comp_table.to_pandas()++ if len(data) > 0:+ # Convert to list of dicts, preserving vectors+ records = data.to_dict("records")+ lance_table.add(records)+ total_vectors += len(records)+ print(f" Added {len(records)} vectors from {component_name}")++ del comp_table+ del comp_db+ except Exception as e:+ print(f" Error aggregating {component_name}: {e}")+ continue++ # Close and copy to volume+ del lance_table+ del lance_db++ AGGREGATED_LANCE = DB_DIR / "code_index.lancedb"+ if AGGREGATED_LANCE.exists():+ shutil.rmtree(AGGREGATED_LANCE)+ shutil.copytree(TMP_LANCE_DIR, AGGREGATED_LANCE)+ shutil.rmtree(TMP_LANCE_DIR)++ # Commit changes to volume+ code_volume.commit()++ print(f"LanceDB aggregation complete: {total_vectors} total vectors")+ print("Aggregation complete!")++ return {+ "sqlite_rows": total_rows,+ "lance_vectors": total_vectors,+ "components_aggregated": len(component_dbs),+ }++@app.function(image=image,volumes={CODE_VOLUME_PATH: code_volume},@@ -1188,7 +1434,7 @@ async def generate_code_index():volumes={CODE_VOLUME_PATH: code_volume},secrets=[github_secret],schedule=modal.Cron("0 5 * * *"), # Daily at 5 AM UTC (before docs crawl)- timeout=3600,+ timeout=7200, # 2 hours (includes aggregation time))async def scheduled_code_index():"""Scheduled daily code index generation (uses parallel processing)"""@@ -1205,6 +1451,12 @@ async def scheduled_code_index():for fc in result["failed_components"]:print(f" - {fc['component']}: {fc['error'][:200]}")+ # Aggregate component databases into unified DBs for the MCP server+ print("Aggregating component databases...")+ agg_result = aggregate_code_databases.remote()+ print(f"Aggregation complete: {agg_result}")+ result["aggregation"] = agg_result+return resultexcept modal.exception.FunctionTimeoutError as e:print(f"Code indexing timed out: {e}")@@ -1302,129 +1554,63 @@ Components include: agent, computer, mcp-server, som, etc.IMPORTANT: Always cite sources - URLs for docs, component@version:path for code.""",)- # Initialize embedding model+ # Initialize embedding model (unused but kept for potential future use)model = get_registry().get("sentence-transformers").create(name="all-MiniLM-L6-v2")- # Global database connections- _lance_db = None- _lance_table = None- _sqlite_conn = None- _code_lance_tables: dict = {} # component -> (db, table)+ # Global database connections (lazy-initialized, cached for reuse)+ _docs_lance_db = None+ _docs_lance_table = None+ _docs_sqlite_conn = None+ _code_lance_db = None+ _code_lance_table = None_code_sqlite_conn = None- _code_components: list[str] = []-- def discover_code_components() -> list[str]:- """Discover available component databases."""- nonlocal _code_components- if not _code_components:- db_dir = Path(CODE_DB_PATH)- if db_dir.exists():- # Find all component SQLite databases- for db_file in db_dir.glob("code_index_*.sqlite"):- component = db_file.stem.replace("code_index_", "")- _code_components.append(component)- # Fallback to legacy single database- if not _code_components and (db_dir / "code_index.sqlite").exists():- _code_components.append("_legacy")- return _code_componentsdef get_lance_table():"""Get or create LanceDB connection for docs"""- nonlocal _lance_db, _lance_table- if _lance_table is None:+ nonlocal _docs_lance_db, _docs_lance_table+ if _docs_lance_table is None:db_path = Path(DB_PATH)if not db_path.exists():raise RuntimeError("Database not found. Run crawl and generation functions first.")- _lance_db = lancedb.connect(db_path)- _lance_table = _lance_db.open_table("docs")- return _lance_table+ _docs_lance_db = lancedb.connect(db_path)+ _docs_lance_table = _docs_lance_db.open_table("docs")+ return _docs_lance_tabledef get_sqlite_conn():"""Get or create read-only SQLite connection for docs"""- nonlocal _sqlite_conn- if _sqlite_conn is None:+ nonlocal _docs_sqlite_conn+ if _docs_sqlite_conn is None:sqlite_path = Path(DB_PATH) / "docs.sqlite"if not sqlite_path.exists():raise RuntimeError("SQLite database not found.")- _sqlite_conn = sqlite3.connect(f"file:{sqlite_path}?mode=ro", uri=True)- _sqlite_conn.row_factory = sqlite3.Row- return _sqlite_conn-- def get_code_lance_tables() -> list[tuple[str, any]]:- """Get LanceDB connections for all code components."""- nonlocal _code_lance_tables- components = discover_code_components()-- for component in components:- if component not in _code_lance_tables:- if component == "_legacy":- db_path = Path(CODE_DB_PATH) / "code_index.lancedb"- else:- db_path = Path(CODE_DB_PATH) / f"code_index_{component}.lancedb"-- if db_path.exists():- db = lancedb.connect(db_path)- table = db.open_table("code")- _code_lance_tables[component] = (db, table)-- return [(comp, tbl) for comp, (_, tbl) in _code_lance_tables.items()]+ _docs_sqlite_conn = sqlite3.connect(f"file:{sqlite_path}?mode=ro", uri=True)+ _docs_sqlite_conn.row_factory = sqlite3.Row+ return _docs_sqlite_conn++ def get_code_lance_table():+ """Get LanceDB connection for the aggregated code database."""+ nonlocal _code_lance_db, _code_lance_table+ if _code_lance_table is None:+ db_path = Path(CODE_DB_PATH) / "code_index.lancedb"+ if not db_path.exists():+ raise RuntimeError(+ "Code LanceDB not found. Run generate_code_index_parallel and aggregate_code_databases first."+ )+ _code_lance_db = lancedb.connect(db_path)+ _code_lance_table = _code_lance_db.open_table("code")+ return _code_lance_tabledef get_code_sqlite_conn():- """Get SQLite connection with all component databases attached."""+ """Get read-only SQLite connection for the aggregated code database."""nonlocal _code_sqlite_connif _code_sqlite_conn is None:- components = discover_code_components()- if not components:+ sqlite_path = Path(CODE_DB_PATH) / "code_index.sqlite"+ if not sqlite_path.exists():raise RuntimeError(- "No code databases found. Run generate_code_index_parallel first."+ "Code SQLite database not found. Run generate_code_index_parallel and aggregate_code_databases first.")-- # Create in-memory database and attach all component databases- _code_sqlite_conn = sqlite3.connect(":memory:")+ _code_sqlite_conn = sqlite3.connect(f"file:{sqlite_path}?mode=ro", uri=True)_code_sqlite_conn.row_factory = sqlite3.Row-- db_dir = Path(CODE_DB_PATH)-- # Check for legacy single database first- legacy_path = db_dir / "code_index.sqlite"- if legacy_path.exists() and "_legacy" in components:- _code_sqlite_conn.execute(- f"ATTACH DATABASE 'file:{legacy_path}?mode=ro' AS code_legacy"- )-- # Attach each component database- attached = []- for component in components:- if component == "_legacy":- continue- db_path = db_dir / f"code_index_{component}.sqlite"- if db_path.exists():- # SQLite schema names can't have hyphens- safe_name = component.replace("-", "_")- _code_sqlite_conn.execute(- f"ATTACH DATABASE 'file:{db_path}?mode=ro' AS code_{safe_name}"- )- attached.append(safe_name)-- # Create unified view across all component databases- if attached:- union_parts = []- for safe_name in attached:- union_parts.append(- f"SELECT id, component, version, file_path, content, language FROM code_{safe_name}.code_files"- )- union_sql = " UNION ALL ".join(union_parts)- _code_sqlite_conn.execute(f"CREATE VIEW code_files AS {union_sql}")-- # Create unified FTS view (queries individual FTS tables)- fts_union_parts = []- for safe_name in attached:- fts_union_parts.append(- f"SELECT rowid, content, component, version, file_path FROM code_{safe_name}.code_files_fts"- )- fts_union_sql = " UNION ALL ".join(fts_union_parts)- _code_sqlite_conn.execute(f"CREATE VIEW code_files_fts_union AS {fts_union_sql}")-return _code_sqlite_conn# =================== DOCUMENTATION QUERY TOOLS (READ-ONLY) ===================@@ -1584,9 +1770,7 @@ IMPORTANT: Always cite sources - URLs for docs, component@version:path for code.component: Optional[str] = None,) -> list[dict]:"""- Execute a vector similarity search against code LanceDBs (read-only).-- Searches across all component databases and returns merged results sorted by similarity.+ Execute a vector similarity search against the code LanceDB (read-only).Schema:- text TEXT -- The source code content@@ -1601,37 +1785,32 @@ IMPORTANT: Always cite sources - URLs for docs, component@version:path for code.limit: Maximum number of results (default: 10, max: 100)where: Optional SQL-like filter (e.g., "version = '0.7.3'")select: Optional list of columns to return (default: all except vector)- component: Optional component to search (if not specified, searches all)+ component: Optional component to filter by (if not specified, searches all)Returns:List of matching code files with similarity scores (_distance field)"""limit = min(max(1, limit), 100)- tables = get_code_lance_tables()-- all_results = []+ table = get_code_lance_table()- for comp_name, table in tables:- # Skip if component filter specified and doesn't match- if component and comp_name != component and comp_name != "_legacy":- continue-- search = table.search(query).limit(limit)+ search = table.search(query).limit(limit)- if where:- search = search.where(where)- if select:- search = search.select(select)+ # Build where clause, adding component filter if specified+ where_clauses = []+ if component:+ where_clauses.append(f"component = '{component}'")+ if where:+ where_clauses.append(where)- results = search.to_list()- all_results.extend(results)+ if where_clauses:+ search = search.where(" AND ".join(where_clauses))+ if select:+ search = search.select(select)- # Sort by distance and take top results- all_results.sort(key=lambda x: x.get("_distance", float("inf")))- all_results = all_results[:limit]+ results = search.to_list()formatted = []- for r in all_results:+ for r in results:result = {}for key, value in r.items():if key == "vector":@@ -1705,4 +1884,9 @@ def main(code_result = generate_code_index.remote()print(f"Code index: {code_result}")+ # Aggregate component databases for the MCP server+ print("Aggregating code databases...")+ agg_result = aggregate_code_databases.remote()+ print(f"Aggregation: {agg_result}")+print("Done! Deploy with: modal deploy docs/scripts/modal_app.py")