Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
refactor(docs): remove redundant standalone scripts in favor of modal_app.py (#956)
r33drichards18bf6ad19781
diff --git a/docs/scripts/README.md b/docs/scripts/README.mdindex e08e3080e8..fac104e73d 100644--- a/docs/scripts/README.md+++ b/docs/scripts/README.md@@ -4,15 +4,7 @@ This directory contains scripts for crawling, indexing, and serving CUA document## Scripts-### Local Scripts--- **crawl_docs.py**: Crawls cua.ai/docs using crawl4ai-- **generate_db.py**: Creates LanceDB vector database for semantic search-- **generate_sqlite.py**: Creates SQLite FTS5 database for full-text search--### Modal Deployment--- **modal_app.py**: Complete Modal app with scheduled crawling and MCP server deployment+- **modal_app.py**: Complete Modal app with scheduled crawling, database generation, and MCP server deployment## Installation@@ -25,25 +17,7 @@ uv sync --group docs-scripts## Usage-### Option 1: Local Development--#### 1. Crawl Documentation--```bash-uv run docs/scripts/crawl_docs.py-```--#### 2. Generate Databases--```bash-# Generate vector database for semantic search-uv run docs/scripts/generate_db.py--# Generate SQLite FTS5 database for full-text search-uv run docs/scripts/generate_sqlite.py-```--### Option 2: Modal Deployment (Production)+### Modal DeploymentThe Modal app provides a production-ready deployment with:diff --git a/docs/scripts/crawl_docs.py b/docs/scripts/crawl_docs.pydeleted file mode 100644index 59d3a85881..0000000000--- a/docs/scripts/crawl_docs.py+++ /dev/null@@ -1,261 +0,0 @@-"""-Comprehensive crawler for cua.ai/docs using crawl4ai-Recursively crawls all documentation pages and saves content to JSON files-"""--import asyncio-import json-import re-from pathlib import Path-from urllib.parse import urljoin, urlparse--from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig--# Configuration-BASE_URL = "https://cua.ai"-DOCS_URL = f"{BASE_URL}/docs"-OUTPUT_DIR = Path(__file__).parent.parent / "crawled_data"-MAX_CONCURRENT = 5 # Limit concurrent requests to be polite-DELAY_BETWEEN_REQUESTS = 0.5 # seconds---class CuaDocsCrawler:- def __init__(self):- self.visited_urls: set[str] = set()- self.to_visit: set[str] = set()- self.failed_urls: set[str] = set()- self.all_data: list[dict] = []- self.semaphore = asyncio.Semaphore(MAX_CONCURRENT)-- def normalize_url(self, url: str) -> str:- """Normalize URL to avoid duplicates"""- parsed = urlparse(url)- # Remove trailing slashes and fragments- path = parsed.path.rstrip("/")- if not path:- path = ""- return f"{parsed.scheme}://{parsed.netloc}{path}"-- def is_valid_url(self, url: str) -> bool:- """Check if URL should be crawled (only /docs pages)"""- parsed = urlparse(url)-- # Only crawl cua.ai pages- if parsed.netloc and parsed.netloc not in ["cua.ai", "www.cua.ai"]:- return False-- # Only crawl /docs paths- if not parsed.path.startswith("/docs"):- return False-- # Skip non-page resources- skip_extensions = [- ".pdf",- ".png",- ".jpg",- ".jpeg",- ".gif",- ".svg",- ".css",- ".js",- ".ico",- ".woff",- ".woff2",- ".ttf",- ".zip",- ".tar",- ".gz",- ]- if any(parsed.path.lower().endswith(ext) for ext in skip_extensions):- return False-- # Skip external links and anchors- if url.startswith("#") or url.startswith("mailto:") or url.startswith("javascript:"):- return False-- return True-- def extract_links(self, html: str, current_url: str) -> set[str]:- """Extract all internal links from HTML content"""- links = set()-- # Find all href attributes- href_pattern = r'href=["\']([^"\']+)["\']'- matches = re.findall(href_pattern, html, re.IGNORECASE)-- for href in matches:- # Convert relative URLs to absolute- if href.startswith("/"):- full_url = urljoin(BASE_URL, href)- elif href.startswith("http"):- full_url = href- elif not href.startswith("#") and not href.startswith("mailto:"):- full_url = urljoin(current_url, href)- else:- continue-- normalized = self.normalize_url(full_url)- if self.is_valid_url(normalized):- links.add(normalized)-- return links-- def extract_path_info(self, url: str) -> dict:- """Extract meaningful path information from URL"""- parsed = urlparse(url)- path = parsed.path.replace("/docs/", "").strip("/")- parts = path.split("/") if path else []-- return {- "path": path,- "category": parts[0] if parts else "root",- "subcategory": parts[1] if len(parts) > 1 else None,- "page": parts[-1] if parts else "index",- "depth": len(parts),- }-- async def crawl_page(self, crawler: AsyncWebCrawler, url: str) -> dict | None:- """Crawl a single page"""- async with self.semaphore:- try:- print(f"Crawling: {url}")-- config = CrawlerRunConfig(- word_count_threshold=10,- exclude_external_links=True,- )-- result = await crawler.arun(url=url, config=config)-- if result.success:- # Extract new links from the page- new_links = self.extract_links(result.html, url)- for link in new_links:- if link not in self.visited_urls and link not in self.to_visit:- self.to_visit.add(link)-- path_info = self.extract_path_info(url)-- page_data = {- "url": url,- "title": result.metadata.get("title", "") if result.metadata else "",- "description": (- result.metadata.get("description", "") if result.metadata else ""- ),- "markdown": result.markdown,- "path_info": path_info,- "links_found": list(new_links),- }-- # Save individual page- self.save_page(url, page_data)-- await asyncio.sleep(DELAY_BETWEEN_REQUESTS)- return page_data- else:- print(f"Failed to crawl {url}: {result.error_message}")- self.failed_urls.add(url)- return None-- except Exception as e:- print(f"Error crawling {url}: {e}")- self.failed_urls.add(url)- return None-- def save_page(self, url: str, data: dict):- """Save page data to a JSON file"""- # Create filename from URL path- parsed = urlparse(url)- path = parsed.path.strip("/") or "index"- filename = path.replace("/", "_") + ".json"-- filepath = OUTPUT_DIR / filename- with open(filepath, "w", encoding="utf-8") as f:- json.dump(data, f, indent=2, ensure_ascii=False)-- async def crawl_all(self):- """Main crawl loop"""- OUTPUT_DIR.mkdir(exist_ok=True)-- # Start with the docs URL and key sections based on typical CUA docs structure- seed_urls = [- DOCS_URL,- f"{DOCS_URL}/cua",- f"{DOCS_URL}/cua/guide",- f"{DOCS_URL}/cua/guide/get-started",- f"{DOCS_URL}/cua/reference",- f"{DOCS_URL}/cua/reference/computer-sdk",- f"{DOCS_URL}/cua-bench",- f"{BASE_URL}/llms.txt", # LLM-optimized content if available- ]-- for url in seed_urls:- normalized = self.normalize_url(url)- if self.is_valid_url(normalized) or url.endswith("llms.txt"):- self.to_visit.add(normalized)-- browser_config = BrowserConfig(- headless=True,- verbose=False,- )-- async with AsyncWebCrawler(config=browser_config) as crawler:- while self.to_visit:- # Get batch of URLs to crawl- batch = []- while self.to_visit and len(batch) < MAX_CONCURRENT:- url = self.to_visit.pop()- if url not in self.visited_urls:- batch.append(url)- self.visited_urls.add(url)-- if not batch:- break-- # Crawl batch concurrently- tasks = [self.crawl_page(crawler, url) for url in batch]- results = await asyncio.gather(*tasks)-- # Collect successful results- for result in results:- if result:- self.all_data.append(result)-- print(f"Progress: {len(self.visited_urls)} crawled, {len(self.to_visit)} remaining")-- # Save summary- summary = {- "total_pages": len(self.all_data),- "failed_urls": list(self.failed_urls),- "all_urls": list(self.visited_urls),- "categories": self._get_categories(),- }-- with open(OUTPUT_DIR / "_summary.json", "w", encoding="utf-8") as f:- json.dump(summary, f, indent=2)-- # Save all data in one file too- with open(OUTPUT_DIR / "_all_pages.json", "w", encoding="utf-8") as f:- json.dump(self.all_data, f, indent=2, ensure_ascii=False)-- print("\nCrawl complete!")- print(f"Total pages crawled: {len(self.all_data)}")- print(f"Failed URLs: {len(self.failed_urls)}")- print(f"Output saved to: {OUTPUT_DIR.absolute()}")-- def _get_categories(self) -> dict:- """Get summary of categories crawled"""- categories = {}- for page in self.all_data:- cat = page.get("path_info", {}).get("category", "unknown")- categories[cat] = categories.get(cat, 0) + 1- return categories---async def main():- crawler = CuaDocsCrawler()- await crawler.crawl_all()---if __name__ == "__main__":- asyncio.run(main())diff --git a/docs/scripts/generate_db.py b/docs/scripts/generate_db.pydeleted file mode 100644index c0418e21cb..0000000000--- a/docs/scripts/generate_db.py+++ /dev/null@@ -1,261 +0,0 @@-"""-Database generator for CUA documentation-Parses crawled JSON data and creates a LanceDB vector database for RAG-"""--import json-import re-from pathlib import Path-from typing import Optional--import lancedb-from lancedb.embeddings import get_registry-from lancedb.pydantic import LanceModel, Vector--# Configuration-CRAWLED_DATA_DIR = Path(__file__).parent.parent / "crawled_data"-DB_PATH = Path(__file__).parent.parent / "docs_db"-CHUNK_SIZE = 1000 # Characters per chunk-CHUNK_OVERLAP = 200 # Overlap between chunks--# Use sentence-transformers for embeddings-model = get_registry().get("sentence-transformers").create(name="all-MiniLM-L6-v2")---class DocChunk(LanceModel):- """Schema for document chunks in the database"""-- text: str = model.SourceField()- vector: Vector(model.ndims()) = model.VectorField()- url: str- title: str- category: str- subcategory: Optional[str]- page: str- chunk_index: int---def clean_markdown(markdown: str) -> str:- """Clean markdown content for better chunking"""- # Remove excessive whitespace- text = re.sub(r"\n{3,}", "\n\n", markdown)- # Remove image markdown- text = re.sub(r"!\[.*?\]\(.*?\)", "", text)- # Remove link URLs but keep text- text = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", text)- # Remove HTML tags- text = re.sub(r"<[^>]+>", "", text)- # Clean up whitespace- text = re.sub(r" {2,}", " ", text)- return text.strip()---def chunk_text(text: str, chunk_size: int = CHUNK_SIZE, overlap: int = CHUNK_OVERLAP) -> list[str]:- """Split text into overlapping chunks, respecting sentence boundaries"""- if not text:- return []-- # Split by paragraphs first- paragraphs = text.split("\n\n")- chunks = []- current_chunk = ""-- for para in paragraphs:- para = para.strip()- if not para:- continue-- # If adding this paragraph exceeds chunk size, save current and start new- if len(current_chunk) + len(para) + 2 > chunk_size:- if current_chunk:- chunks.append(current_chunk.strip())- # Start new chunk with overlap from previous- if overlap > 0 and len(current_chunk) > overlap:- # Try to find a sentence boundary for overlap- overlap_text = current_chunk[-overlap:]- sentence_end = overlap_text.rfind(". ")- if sentence_end > 0:- overlap_text = overlap_text[sentence_end + 2 :]- current_chunk = overlap_text + "\n\n" + para- else:- current_chunk = para- else:- # Single paragraph exceeds chunk size, split by sentences- sentences = re.split(r"(?<=[.!?])\s+", para)- for sentence in sentences:- if len(current_chunk) + len(sentence) + 1 > chunk_size:- if current_chunk:- chunks.append(current_chunk.strip())- # Start new chunk with overlap from previous, similar to paragraph logic- if overlap > 0 and len(current_chunk) > overlap:- overlap_text = current_chunk[-overlap:]- sentence_end = overlap_text.rfind(". ")- if sentence_end > 0:- overlap_text = overlap_text[sentence_end + 2 :]- current_chunk = (overlap_text + " " + sentence).strip()- else:- current_chunk = sentence.strip()- else:- # No existing chunk; start with this sentence- current_chunk = sentence.strip()- else:- current_chunk = (current_chunk + " " + sentence).strip()- else:- current_chunk = (current_chunk + "\n\n" + para).strip()-- # Don't forget the last chunk- if current_chunk:- chunks.append(current_chunk.strip())-- return chunks---def load_crawled_data() -> list[dict]:- """Load all crawled page data"""- all_pages_file = CRAWLED_DATA_DIR / "_all_pages.json"-- if all_pages_file.exists():- with open(all_pages_file, "r", encoding="utf-8") as f:- return json.load(f)-- # Fallback: load individual files- pages = []- for json_file in CRAWLED_DATA_DIR.glob("*.json"):- if json_file.name.startswith("_"):- continue- with open(json_file, "r", encoding="utf-8") as f:- pages.append(json.load(f))-- return pages---def process_pages(pages: list[dict]) -> list[dict]:- """Process pages into document chunks"""- all_chunks = []-- for page in pages:- markdown = page.get("markdown", "")- if not markdown:- continue-- # Clean the markdown- cleaned_text = clean_markdown(markdown)- if not cleaned_text or len(cleaned_text) < 50:- continue-- # Get path info- path_info = page.get("path_info", {})-- # Chunk the text- text_chunks = chunk_text(cleaned_text)-- # Ensure non-null values for required fields- url = page.get("url", "")- title = page.get("title") or path_info.get("page", "") or "Untitled"- category = path_info.get("category") or "unknown"- page_name = path_info.get("page") or ""-- for i, chunk_text_content in enumerate(text_chunks):- chunk = {- "text": chunk_text_content,- "url": url,- "title": title,- "category": category,- "subcategory": path_info.get("subcategory"),- "page": page_name,- "chunk_index": i,- }- all_chunks.append(chunk)-- return all_chunks---def create_database(chunks: list[dict]):- """Create LanceDB database from chunks"""- # Remove existing database- if DB_PATH.exists():- import shutil-- shutil.rmtree(DB_PATH)-- # Create database- db = lancedb.connect(DB_PATH)-- # Create table with schema- table = db.create_table(- "docs",- schema=DocChunk,- mode="overwrite",- )-- # Add data in batches- batch_size = 100- for i in range(0, len(chunks), batch_size):- batch = chunks[i : i + batch_size]- print(f"Adding batch {i // batch_size + 1}/{(len(chunks) + batch_size - 1) // batch_size}")- table.add(batch)-- print(f"Database created at: {DB_PATH}")- print(f"Total chunks: {len(chunks)}")-- return db---def test_search(db: lancedb.DBConnection, query: str, limit: int = 5):- """Test search functionality"""- table = db.open_table("docs")-- print(f"\nSearching for: '{query}'")- print("-" * 50)-- results = table.search(query).limit(limit).to_list()-- for i, result in enumerate(results):- print(f"\n{i + 1}. [{result['category']}] {result['title']}")- print(f" URL: {result['url']}")- print(f" Score: {result.get('_distance', 'N/A'):.4f}")- print(f" Preview: {result['text'][:150]}...")---def main():- print("Loading crawled data...")- pages = load_crawled_data()- print(f"Loaded {len(pages)} pages")-- if not pages:- print("No crawled data found. Run crawl_docs.py first.")- return-- print("\nProcessing pages into chunks...")- chunks = process_pages(pages)- print(f"Created {len(chunks)} chunks")-- if not chunks:- print("No chunks created. Check your crawled data.")- return-- print("\nCreating database...")- db = create_database(chunks)-- # Test with sample queries- print("\n" + "=" * 50)- print("Testing search functionality")- print("=" * 50)-- test_queries = [- "how to install CUA",- "computer use agent",- "benchmark evaluation",- "API reference",- ]-- for query in test_queries:- test_search(db, query)-- print("\n" + "=" * 50)- print("Database generation complete!")- print(f"Database location: {DB_PATH}")---if __name__ == "__main__":- main()diff --git a/docs/scripts/generate_sqlite.py b/docs/scripts/generate_sqlite.pydeleted file mode 100644index 8aac1932b6..0000000000--- a/docs/scripts/generate_sqlite.py+++ /dev/null@@ -1,278 +0,0 @@-"""-SQLite database generator for CUA documentation-Creates a full-text search enabled SQLite database from crawled data-"""--import json-import re-import sqlite3-from pathlib import Path--from markdown_it import MarkdownIt--# Configuration-CRAWLED_DATA_DIR = Path(__file__).parent.parent / "crawled_data"-SQLITE_PATH = Path(__file__).parent.parent / "docs_db" / "docs.sqlite"---def clean_markdown(markdown: str) -> str:- """- Extract plain text content from markdown using a proper markdown parser.-- This function uses markdown-it-py to parse the markdown into a token tree- and then extracts only the text content, removing:- - Markdown formatting (bold, italic, headers, etc.)- - Links (keeping only the link text)- - Images (alt text is discarded)- - HTML tags- - Code block language identifiers-- Args:- markdown: Raw markdown content-- Returns:- Plain text content suitable for full-text search- """- md_parser = MarkdownIt()- tokens = md_parser.parse(markdown)-- text_parts = []-- def extract_text(token_list):- """Recursively extract text from token tree"""- for token in token_list:- if token.type == "inline" and token.children:- # Process inline content (text, links, formatting, etc.)- for child in token.children:- if child.type == "text":- text_parts.append(child.content)- elif child.type == "code_inline":- text_parts.append(child.content)- elif child.type == "softbreak":- text_parts.append(" ")- elif child.type == "hardbreak":- text_parts.append("\n")- # Skip link markup, images, and formatting tokens- # (link_open, link_close, image, strong_open, strong_close, em_open, em_close, etc.)- elif token.type == "fence" or token.type == "code_block":- # Include code content and add newline after- text_parts.append(token.content)- text_parts.append("\n")- elif token.type == "html_block" or token.type == "html_inline":- # Skip HTML blocks and inline HTML- pass-- # Recursively process nested children- if token.children:- extract_text(token.children)-- # Add spacing after block elements- if token.type in [- "heading_close",- "paragraph_close",- "list_item_close",- "blockquote_close",- ]:- text_parts.append("\n")-- extract_text(tokens)-- # Join and clean up whitespace- text = "".join(text_parts)- # Normalize multiple newlines to at most double newlines- text = re.sub(r"\n{3,}", "\n\n", text)- # Normalize multiple spaces to single space within lines- text = re.sub(r" {2,}", " ", text)-- return text.strip()---def load_crawled_data() -> list[dict]:- """Load all crawled page data"""- all_pages_file = CRAWLED_DATA_DIR / "_all_pages.json"-- if all_pages_file.exists():- with open(all_pages_file, "r", encoding="utf-8") as f:- return json.load(f)-- pages = []- for json_file in CRAWLED_DATA_DIR.glob("*.json"):- if json_file.name.startswith("_"):- continue- with open(json_file, "r", encoding="utf-8") as f:- pages.append(json.load(f))-- return pages---def create_database(pages: list[dict]):- """Create SQLite database with FTS5 full-text search"""- # Ensure parent directory exists- SQLITE_PATH.parent.mkdir(parents=True, exist_ok=True)-- # Remove existing database- if SQLITE_PATH.exists():- SQLITE_PATH.unlink()-- conn = sqlite3.connect(SQLITE_PATH)- cursor = conn.cursor()-- # Create main pages table- cursor.execute(- """- CREATE TABLE pages (- id INTEGER PRIMARY KEY AUTOINCREMENT,- url TEXT UNIQUE NOT NULL,- title TEXT,- description TEXT,- category TEXT,- subcategory TEXT,- page_name TEXT,- content TEXT,- raw_markdown TEXT- )- """- )-- # Create FTS5 virtual table for full-text search- cursor.execute(- """- CREATE VIRTUAL TABLE pages_fts USING fts5(- content,- url UNINDEXED,- title UNINDEXED,- category UNINDEXED,- content='pages',- content_rowid='id'- )- """- )-- # Create triggers to keep FTS in sync- cursor.execute(- """- CREATE TRIGGER pages_ai AFTER INSERT ON pages BEGIN- INSERT INTO pages_fts(rowid, content, url, title, category)- VALUES (new.id, new.content, new.url, new.title, new.category);- END;- """- )-- cursor.execute(- """- CREATE TRIGGER pages_ad AFTER DELETE ON pages BEGIN- DELETE FROM pages_fts WHERE rowid = old.id;- END;- """- )-- cursor.execute(- """- CREATE TRIGGER pages_au AFTER UPDATE ON pages BEGIN- DELETE FROM pages_fts WHERE rowid = old.id;- INSERT INTO pages_fts(rowid, content, url, title, category)- VALUES (new.id, new.content, new.url, new.title, new.category);- END;- """- )-- # Insert pages- for page in pages:- markdown = page.get("markdown", "")- if not markdown:- continue-- content = clean_markdown(markdown)- if not content or len(content) < 50:- continue-- path_info = page.get("path_info", {})-- cursor.execute(- """- INSERT OR REPLACE INTO pages- (url, title, description, category, subcategory, page_name, content, raw_markdown)- VALUES (?, ?, ?, ?, ?, ?, ?, ?)- """,- (- page.get("url", ""),- page.get("title") or path_info.get("page", "") or "Untitled",- page.get("description", ""),- path_info.get("category", "unknown"),- path_info.get("subcategory"),- path_info.get("page", ""),- content,- markdown,- ),- )-- conn.commit()-- # Get stats- cursor.execute("SELECT COUNT(*) FROM pages")- page_count = cursor.fetchone()[0]-- cursor.execute("SELECT category, COUNT(*) FROM pages GROUP BY category")- categories = cursor.fetchall()-- conn.close()-- print(f"SQLite database created at: {SQLITE_PATH}")- print(f"Total pages: {page_count}")- print("Pages by category:")- for cat, count in categories:- print(f" - {cat}: {count}")---def test_search(query: str):- """Test full-text search"""- conn = sqlite3.connect(SQLITE_PATH)- cursor = conn.cursor()-- print(f"\nFTS5 search for: '{query}'")- print("-" * 50)-- cursor.execute(- """- SELECT url, title, snippet(pages_fts, 0, '>>>', '<<<', '...', 50) as snippet- FROM pages_fts- WHERE pages_fts MATCH ?- ORDER BY rank- LIMIT 5- """,- (query,),- )-- results = cursor.fetchall()- for url, title, snippet in results:- print(f"\n{title}")- print(f" URL: {url}")- print(f" Snippet: {snippet}")-- conn.close()---def main():- print("Loading crawled data...")- pages = load_crawled_data()- print(f"Loaded {len(pages)} pages")-- if not pages:- print("No crawled data found. Run crawl_docs.py first.")- return-- print("\nCreating SQLite database...")- create_database(pages)-- # Test searches- print("\n" + "=" * 50)- print("Testing FTS5 search")- print("=" * 50)-- test_search("install")- test_search("computer use agent")- test_search("benchmark")---if __name__ == "__main__":- main()