Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

refactor(docs): remove redundant standalone scripts in favor of modal_app.py (#956)

r33drichards18bf6ad19781
diff --git a/docs/scripts/README.md b/docs/scripts/README.md
index e08e3080e8..fac104e73d 100644
--- a/docs/scripts/README.md
+++ b/docs/scripts/README.md
@@ -4,15 +4,7 @@ This directory contains scripts for crawling, indexing, and serving CUA document
## Scripts
-### Local Scripts
-
-- **crawl_docs.py**: Crawls cua.ai/docs using crawl4ai
-- **generate_db.py**: Creates LanceDB vector database for semantic search
-- **generate_sqlite.py**: Creates SQLite FTS5 database for full-text search
-
-### Modal Deployment
-
-- **modal_app.py**: Complete Modal app with scheduled crawling and MCP server deployment
+- **modal_app.py**: Complete Modal app with scheduled crawling, database generation, and MCP server deployment
## Installation
@@ -25,25 +17,7 @@ uv sync --group docs-scripts
## Usage
-### Option 1: Local Development
-
-#### 1. Crawl Documentation
-
-```bash
-uv run docs/scripts/crawl_docs.py
-```
-
-#### 2. Generate Databases
-
-```bash
-# Generate vector database for semantic search
-uv run docs/scripts/generate_db.py
-
-# Generate SQLite FTS5 database for full-text search
-uv run docs/scripts/generate_sqlite.py
-```
-
-### Option 2: Modal Deployment (Production)
+### Modal Deployment
The Modal app provides a production-ready deployment with:
diff --git a/docs/scripts/crawl_docs.py b/docs/scripts/crawl_docs.py
deleted file mode 100644
index 59d3a85881..0000000000
--- a/docs/scripts/crawl_docs.py
+++ /dev/null
@@ -1,261 +0,0 @@
-"""
-Comprehensive crawler for cua.ai/docs using crawl4ai
-Recursively crawls all documentation pages and saves content to JSON files
-"""
-
-import asyncio
-import json
-import re
-from pathlib import Path
-from urllib.parse import urljoin, urlparse
-
-from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
-
-# Configuration
-BASE_URL = "https://cua.ai"
-DOCS_URL = f"{BASE_URL}/docs"
-OUTPUT_DIR = Path(__file__).parent.parent / "crawled_data"
-MAX_CONCURRENT = 5 # Limit concurrent requests to be polite
-DELAY_BETWEEN_REQUESTS = 0.5 # seconds
-
-
-class CuaDocsCrawler:
- def __init__(self):
- self.visited_urls: set[str] = set()
- self.to_visit: set[str] = set()
- self.failed_urls: set[str] = set()
- self.all_data: list[dict] = []
- self.semaphore = asyncio.Semaphore(MAX_CONCURRENT)
-
- def normalize_url(self, url: str) -> str:
- """Normalize URL to avoid duplicates"""
- parsed = urlparse(url)
- # Remove trailing slashes and fragments
- path = parsed.path.rstrip("/")
- if not path:
- path = ""
- return f"{parsed.scheme}://{parsed.netloc}{path}"
-
- def is_valid_url(self, url: str) -> bool:
- """Check if URL should be crawled (only /docs pages)"""
- parsed = urlparse(url)
-
- # Only crawl cua.ai pages
- if parsed.netloc and parsed.netloc not in ["cua.ai", "www.cua.ai"]:
- return False
-
- # Only crawl /docs paths
- if not parsed.path.startswith("/docs"):
- return False
-
- # Skip non-page resources
- skip_extensions = [
- ".pdf",
- ".png",
- ".jpg",
- ".jpeg",
- ".gif",
- ".svg",
- ".css",
- ".js",
- ".ico",
- ".woff",
- ".woff2",
- ".ttf",
- ".zip",
- ".tar",
- ".gz",
- ]
- if any(parsed.path.lower().endswith(ext) for ext in skip_extensions):
- return False
-
- # Skip external links and anchors
- if url.startswith("#") or url.startswith("mailto:") or url.startswith("javascript:"):
- return False
-
- return True
-
- def extract_links(self, html: str, current_url: str) -> set[str]:
- """Extract all internal links from HTML content"""
- links = set()
-
- # Find all href attributes
- href_pattern = r'href=["\']([^"\']+)["\']'
- matches = re.findall(href_pattern, html, re.IGNORECASE)
-
- for href in matches:
- # Convert relative URLs to absolute
- if href.startswith("/"):
- full_url = urljoin(BASE_URL, href)
- elif href.startswith("http"):
- full_url = href
- elif not href.startswith("#") and not href.startswith("mailto:"):
- full_url = urljoin(current_url, href)
- else:
- continue
-
- normalized = self.normalize_url(full_url)
- if self.is_valid_url(normalized):
- links.add(normalized)
-
- return links
-
- def extract_path_info(self, url: str) -> dict:
- """Extract meaningful path information from URL"""
- parsed = urlparse(url)
- path = parsed.path.replace("/docs/", "").strip("/")
- parts = path.split("/") if path else []
-
- return {
- "path": path,
- "category": parts[0] if parts else "root",
- "subcategory": parts[1] if len(parts) > 1 else None,
- "page": parts[-1] if parts else "index",
- "depth": len(parts),
- }
-
- async def crawl_page(self, crawler: AsyncWebCrawler, url: str) -> dict | None:
- """Crawl a single page"""
- async with self.semaphore:
- try:
- print(f"Crawling: {url}")
-
- config = CrawlerRunConfig(
- word_count_threshold=10,
- exclude_external_links=True,
- )
-
- result = await crawler.arun(url=url, config=config)
-
- if result.success:
- # Extract new links from the page
- new_links = self.extract_links(result.html, url)
- for link in new_links:
- if link not in self.visited_urls and link not in self.to_visit:
- self.to_visit.add(link)
-
- path_info = self.extract_path_info(url)
-
- page_data = {
- "url": url,
- "title": result.metadata.get("title", "") if result.metadata else "",
- "description": (
- result.metadata.get("description", "") if result.metadata else ""
- ),
- "markdown": result.markdown,
- "path_info": path_info,
- "links_found": list(new_links),
- }
-
- # Save individual page
- self.save_page(url, page_data)
-
- await asyncio.sleep(DELAY_BETWEEN_REQUESTS)
- return page_data
- else:
- print(f"Failed to crawl {url}: {result.error_message}")
- self.failed_urls.add(url)
- return None
-
- except Exception as e:
- print(f"Error crawling {url}: {e}")
- self.failed_urls.add(url)
- return None
-
- def save_page(self, url: str, data: dict):
- """Save page data to a JSON file"""
- # Create filename from URL path
- parsed = urlparse(url)
- path = parsed.path.strip("/") or "index"
- filename = path.replace("/", "_") + ".json"
-
- filepath = OUTPUT_DIR / filename
- with open(filepath, "w", encoding="utf-8") as f:
- json.dump(data, f, indent=2, ensure_ascii=False)
-
- async def crawl_all(self):
- """Main crawl loop"""
- OUTPUT_DIR.mkdir(exist_ok=True)
-
- # Start with the docs URL and key sections based on typical CUA docs structure
- seed_urls = [
- DOCS_URL,
- f"{DOCS_URL}/cua",
- f"{DOCS_URL}/cua/guide",
- f"{DOCS_URL}/cua/guide/get-started",
- f"{DOCS_URL}/cua/reference",
- f"{DOCS_URL}/cua/reference/computer-sdk",
- f"{DOCS_URL}/cua-bench",
- f"{BASE_URL}/llms.txt", # LLM-optimized content if available
- ]
-
- for url in seed_urls:
- normalized = self.normalize_url(url)
- if self.is_valid_url(normalized) or url.endswith("llms.txt"):
- self.to_visit.add(normalized)
-
- browser_config = BrowserConfig(
- headless=True,
- verbose=False,
- )
-
- async with AsyncWebCrawler(config=browser_config) as crawler:
- while self.to_visit:
- # Get batch of URLs to crawl
- batch = []
- while self.to_visit and len(batch) < MAX_CONCURRENT:
- url = self.to_visit.pop()
- if url not in self.visited_urls:
- batch.append(url)
- self.visited_urls.add(url)
-
- if not batch:
- break
-
- # Crawl batch concurrently
- tasks = [self.crawl_page(crawler, url) for url in batch]
- results = await asyncio.gather(*tasks)
-
- # Collect successful results
- for result in results:
- if result:
- self.all_data.append(result)
-
- print(f"Progress: {len(self.visited_urls)} crawled, {len(self.to_visit)} remaining")
-
- # Save summary
- summary = {
- "total_pages": len(self.all_data),
- "failed_urls": list(self.failed_urls),
- "all_urls": list(self.visited_urls),
- "categories": self._get_categories(),
- }
-
- with open(OUTPUT_DIR / "_summary.json", "w", encoding="utf-8") as f:
- json.dump(summary, f, indent=2)
-
- # Save all data in one file too
- with open(OUTPUT_DIR / "_all_pages.json", "w", encoding="utf-8") as f:
- json.dump(self.all_data, f, indent=2, ensure_ascii=False)
-
- print("\nCrawl complete!")
- print(f"Total pages crawled: {len(self.all_data)}")
- print(f"Failed URLs: {len(self.failed_urls)}")
- print(f"Output saved to: {OUTPUT_DIR.absolute()}")
-
- def _get_categories(self) -> dict:
- """Get summary of categories crawled"""
- categories = {}
- for page in self.all_data:
- cat = page.get("path_info", {}).get("category", "unknown")
- categories[cat] = categories.get(cat, 0) + 1
- return categories
-
-
-async def main():
- crawler = CuaDocsCrawler()
- await crawler.crawl_all()
-
-
-if __name__ == "__main__":
- asyncio.run(main())
diff --git a/docs/scripts/generate_db.py b/docs/scripts/generate_db.py
deleted file mode 100644
index c0418e21cb..0000000000
--- a/docs/scripts/generate_db.py
+++ /dev/null
@@ -1,261 +0,0 @@
-"""
-Database generator for CUA documentation
-Parses crawled JSON data and creates a LanceDB vector database for RAG
-"""
-
-import json
-import re
-from pathlib import Path
-from typing import Optional
-
-import lancedb
-from lancedb.embeddings import get_registry
-from lancedb.pydantic import LanceModel, Vector
-
-# Configuration
-CRAWLED_DATA_DIR = Path(__file__).parent.parent / "crawled_data"
-DB_PATH = Path(__file__).parent.parent / "docs_db"
-CHUNK_SIZE = 1000 # Characters per chunk
-CHUNK_OVERLAP = 200 # Overlap between chunks
-
-# Use sentence-transformers for embeddings
-model = get_registry().get("sentence-transformers").create(name="all-MiniLM-L6-v2")
-
-
-class DocChunk(LanceModel):
- """Schema for document chunks in the database"""
-
- text: str = model.SourceField()
- vector: Vector(model.ndims()) = model.VectorField()
- url: str
- title: str
- category: str
- subcategory: Optional[str]
- page: str
- chunk_index: int
-
-
-def clean_markdown(markdown: str) -> str:
- """Clean markdown content for better chunking"""
- # Remove excessive whitespace
- text = re.sub(r"\n{3,}", "\n\n", markdown)
- # Remove image markdown
- text = re.sub(r"!\[.*?\]\(.*?\)", "", text)
- # Remove link URLs but keep text
- text = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", text)
- # Remove HTML tags
- text = re.sub(r"<[^>]+>", "", text)
- # Clean up whitespace
- text = re.sub(r" {2,}", " ", text)
- return text.strip()
-
-
-def chunk_text(text: str, chunk_size: int = CHUNK_SIZE, overlap: int = CHUNK_OVERLAP) -> list[str]:
- """Split text into overlapping chunks, respecting sentence boundaries"""
- if not text:
- return []
-
- # Split by paragraphs first
- paragraphs = text.split("\n\n")
- chunks = []
- current_chunk = ""
-
- for para in paragraphs:
- para = para.strip()
- if not para:
- continue
-
- # If adding this paragraph exceeds chunk size, save current and start new
- if len(current_chunk) + len(para) + 2 > chunk_size:
- if current_chunk:
- chunks.append(current_chunk.strip())
- # Start new chunk with overlap from previous
- if overlap > 0 and len(current_chunk) > overlap:
- # Try to find a sentence boundary for overlap
- overlap_text = current_chunk[-overlap:]
- sentence_end = overlap_text.rfind(". ")
- if sentence_end > 0:
- overlap_text = overlap_text[sentence_end + 2 :]
- current_chunk = overlap_text + "\n\n" + para
- else:
- current_chunk = para
- else:
- # Single paragraph exceeds chunk size, split by sentences
- sentences = re.split(r"(?<=[.!?])\s+", para)
- for sentence in sentences:
- if len(current_chunk) + len(sentence) + 1 > chunk_size:
- if current_chunk:
- chunks.append(current_chunk.strip())
- # Start new chunk with overlap from previous, similar to paragraph logic
- if overlap > 0 and len(current_chunk) > overlap:
- overlap_text = current_chunk[-overlap:]
- sentence_end = overlap_text.rfind(". ")
- if sentence_end > 0:
- overlap_text = overlap_text[sentence_end + 2 :]
- current_chunk = (overlap_text + " " + sentence).strip()
- else:
- current_chunk = sentence.strip()
- else:
- # No existing chunk; start with this sentence
- current_chunk = sentence.strip()
- else:
- current_chunk = (current_chunk + " " + sentence).strip()
- else:
- current_chunk = (current_chunk + "\n\n" + para).strip()
-
- # Don't forget the last chunk
- if current_chunk:
- chunks.append(current_chunk.strip())
-
- return chunks
-
-
-def load_crawled_data() -> list[dict]:
- """Load all crawled page data"""
- all_pages_file = CRAWLED_DATA_DIR / "_all_pages.json"
-
- if all_pages_file.exists():
- with open(all_pages_file, "r", encoding="utf-8") as f:
- return json.load(f)
-
- # Fallback: load individual files
- pages = []
- for json_file in CRAWLED_DATA_DIR.glob("*.json"):
- if json_file.name.startswith("_"):
- continue
- with open(json_file, "r", encoding="utf-8") as f:
- pages.append(json.load(f))
-
- return pages
-
-
-def process_pages(pages: list[dict]) -> list[dict]:
- """Process pages into document chunks"""
- all_chunks = []
-
- for page in pages:
- markdown = page.get("markdown", "")
- if not markdown:
- continue
-
- # Clean the markdown
- cleaned_text = clean_markdown(markdown)
- if not cleaned_text or len(cleaned_text) < 50:
- continue
-
- # Get path info
- path_info = page.get("path_info", {})
-
- # Chunk the text
- text_chunks = chunk_text(cleaned_text)
-
- # Ensure non-null values for required fields
- url = page.get("url", "")
- title = page.get("title") or path_info.get("page", "") or "Untitled"
- category = path_info.get("category") or "unknown"
- page_name = path_info.get("page") or ""
-
- for i, chunk_text_content in enumerate(text_chunks):
- chunk = {
- "text": chunk_text_content,
- "url": url,
- "title": title,
- "category": category,
- "subcategory": path_info.get("subcategory"),
- "page": page_name,
- "chunk_index": i,
- }
- all_chunks.append(chunk)
-
- return all_chunks
-
-
-def create_database(chunks: list[dict]):
- """Create LanceDB database from chunks"""
- # Remove existing database
- if DB_PATH.exists():
- import shutil
-
- shutil.rmtree(DB_PATH)
-
- # Create database
- db = lancedb.connect(DB_PATH)
-
- # Create table with schema
- table = db.create_table(
- "docs",
- schema=DocChunk,
- mode="overwrite",
- )
-
- # Add data in batches
- batch_size = 100
- for i in range(0, len(chunks), batch_size):
- batch = chunks[i : i + batch_size]
- print(f"Adding batch {i // batch_size + 1}/{(len(chunks) + batch_size - 1) // batch_size}")
- table.add(batch)
-
- print(f"Database created at: {DB_PATH}")
- print(f"Total chunks: {len(chunks)}")
-
- return db
-
-
-def test_search(db: lancedb.DBConnection, query: str, limit: int = 5):
- """Test search functionality"""
- table = db.open_table("docs")
-
- print(f"\nSearching for: '{query}'")
- print("-" * 50)
-
- results = table.search(query).limit(limit).to_list()
-
- for i, result in enumerate(results):
- print(f"\n{i + 1}. [{result['category']}] {result['title']}")
- print(f" URL: {result['url']}")
- print(f" Score: {result.get('_distance', 'N/A'):.4f}")
- print(f" Preview: {result['text'][:150]}...")
-
-
-def main():
- print("Loading crawled data...")
- pages = load_crawled_data()
- print(f"Loaded {len(pages)} pages")
-
- if not pages:
- print("No crawled data found. Run crawl_docs.py first.")
- return
-
- print("\nProcessing pages into chunks...")
- chunks = process_pages(pages)
- print(f"Created {len(chunks)} chunks")
-
- if not chunks:
- print("No chunks created. Check your crawled data.")
- return
-
- print("\nCreating database...")
- db = create_database(chunks)
-
- # Test with sample queries
- print("\n" + "=" * 50)
- print("Testing search functionality")
- print("=" * 50)
-
- test_queries = [
- "how to install CUA",
- "computer use agent",
- "benchmark evaluation",
- "API reference",
- ]
-
- for query in test_queries:
- test_search(db, query)
-
- print("\n" + "=" * 50)
- print("Database generation complete!")
- print(f"Database location: {DB_PATH}")
-
-
-if __name__ == "__main__":
- main()
diff --git a/docs/scripts/generate_sqlite.py b/docs/scripts/generate_sqlite.py
deleted file mode 100644
index 8aac1932b6..0000000000
--- a/docs/scripts/generate_sqlite.py
+++ /dev/null
@@ -1,278 +0,0 @@
-"""
-SQLite database generator for CUA documentation
-Creates a full-text search enabled SQLite database from crawled data
-"""
-
-import json
-import re
-import sqlite3
-from pathlib import Path
-
-from markdown_it import MarkdownIt
-
-# Configuration
-CRAWLED_DATA_DIR = Path(__file__).parent.parent / "crawled_data"
-SQLITE_PATH = Path(__file__).parent.parent / "docs_db" / "docs.sqlite"
-
-
-def clean_markdown(markdown: str) -> str:
- """
- Extract plain text content from markdown using a proper markdown parser.
-
- This function uses markdown-it-py to parse the markdown into a token tree
- and then extracts only the text content, removing:
- - Markdown formatting (bold, italic, headers, etc.)
- - Links (keeping only the link text)
- - Images (alt text is discarded)
- - HTML tags
- - Code block language identifiers
-
- Args:
- markdown: Raw markdown content
-
- Returns:
- Plain text content suitable for full-text search
- """
- md_parser = MarkdownIt()
- tokens = md_parser.parse(markdown)
-
- text_parts = []
-
- def extract_text(token_list):
- """Recursively extract text from token tree"""
- for token in token_list:
- if token.type == "inline" and token.children:
- # Process inline content (text, links, formatting, etc.)
- for child in token.children:
- if child.type == "text":
- text_parts.append(child.content)
- elif child.type == "code_inline":
- text_parts.append(child.content)
- elif child.type == "softbreak":
- text_parts.append(" ")
- elif child.type == "hardbreak":
- text_parts.append("\n")
- # Skip link markup, images, and formatting tokens
- # (link_open, link_close, image, strong_open, strong_close, em_open, em_close, etc.)
- elif token.type == "fence" or token.type == "code_block":
- # Include code content and add newline after
- text_parts.append(token.content)
- text_parts.append("\n")
- elif token.type == "html_block" or token.type == "html_inline":
- # Skip HTML blocks and inline HTML
- pass
-
- # Recursively process nested children
- if token.children:
- extract_text(token.children)
-
- # Add spacing after block elements
- if token.type in [
- "heading_close",
- "paragraph_close",
- "list_item_close",
- "blockquote_close",
- ]:
- text_parts.append("\n")
-
- extract_text(tokens)
-
- # Join and clean up whitespace
- text = "".join(text_parts)
- # Normalize multiple newlines to at most double newlines
- text = re.sub(r"\n{3,}", "\n\n", text)
- # Normalize multiple spaces to single space within lines
- text = re.sub(r" {2,}", " ", text)
-
- return text.strip()
-
-
-def load_crawled_data() -> list[dict]:
- """Load all crawled page data"""
- all_pages_file = CRAWLED_DATA_DIR / "_all_pages.json"
-
- if all_pages_file.exists():
- with open(all_pages_file, "r", encoding="utf-8") as f:
- return json.load(f)
-
- pages = []
- for json_file in CRAWLED_DATA_DIR.glob("*.json"):
- if json_file.name.startswith("_"):
- continue
- with open(json_file, "r", encoding="utf-8") as f:
- pages.append(json.load(f))
-
- return pages
-
-
-def create_database(pages: list[dict]):
- """Create SQLite database with FTS5 full-text search"""
- # Ensure parent directory exists
- SQLITE_PATH.parent.mkdir(parents=True, exist_ok=True)
-
- # Remove existing database
- if SQLITE_PATH.exists():
- SQLITE_PATH.unlink()
-
- conn = sqlite3.connect(SQLITE_PATH)
- cursor = conn.cursor()
-
- # Create main pages table
- cursor.execute(
- """
- CREATE TABLE pages (
- id INTEGER PRIMARY KEY AUTOINCREMENT,
- url TEXT UNIQUE NOT NULL,
- title TEXT,
- description TEXT,
- category TEXT,
- subcategory TEXT,
- page_name TEXT,
- content TEXT,
- raw_markdown TEXT
- )
- """
- )
-
- # Create FTS5 virtual table for full-text search
- cursor.execute(
- """
- CREATE VIRTUAL TABLE pages_fts USING fts5(
- content,
- url UNINDEXED,
- title UNINDEXED,
- category UNINDEXED,
- content='pages',
- content_rowid='id'
- )
- """
- )
-
- # Create triggers to keep FTS in sync
- cursor.execute(
- """
- CREATE TRIGGER pages_ai AFTER INSERT ON pages BEGIN
- INSERT INTO pages_fts(rowid, content, url, title, category)
- VALUES (new.id, new.content, new.url, new.title, new.category);
- END;
- """
- )
-
- cursor.execute(
- """
- CREATE TRIGGER pages_ad AFTER DELETE ON pages BEGIN
- DELETE FROM pages_fts WHERE rowid = old.id;
- END;
- """
- )
-
- cursor.execute(
- """
- CREATE TRIGGER pages_au AFTER UPDATE ON pages BEGIN
- DELETE FROM pages_fts WHERE rowid = old.id;
- INSERT INTO pages_fts(rowid, content, url, title, category)
- VALUES (new.id, new.content, new.url, new.title, new.category);
- END;
- """
- )
-
- # Insert pages
- for page in pages:
- markdown = page.get("markdown", "")
- if not markdown:
- continue
-
- content = clean_markdown(markdown)
- if not content or len(content) < 50:
- continue
-
- path_info = page.get("path_info", {})
-
- cursor.execute(
- """
- INSERT OR REPLACE INTO pages
- (url, title, description, category, subcategory, page_name, content, raw_markdown)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?)
- """,
- (
- page.get("url", ""),
- page.get("title") or path_info.get("page", "") or "Untitled",
- page.get("description", ""),
- path_info.get("category", "unknown"),
- path_info.get("subcategory"),
- path_info.get("page", ""),
- content,
- markdown,
- ),
- )
-
- conn.commit()
-
- # Get stats
- cursor.execute("SELECT COUNT(*) FROM pages")
- page_count = cursor.fetchone()[0]
-
- cursor.execute("SELECT category, COUNT(*) FROM pages GROUP BY category")
- categories = cursor.fetchall()
-
- conn.close()
-
- print(f"SQLite database created at: {SQLITE_PATH}")
- print(f"Total pages: {page_count}")
- print("Pages by category:")
- for cat, count in categories:
- print(f" - {cat}: {count}")
-
-
-def test_search(query: str):
- """Test full-text search"""
- conn = sqlite3.connect(SQLITE_PATH)
- cursor = conn.cursor()
-
- print(f"\nFTS5 search for: '{query}'")
- print("-" * 50)
-
- cursor.execute(
- """
- SELECT url, title, snippet(pages_fts, 0, '>>>', '<<<', '...', 50) as snippet
- FROM pages_fts
- WHERE pages_fts MATCH ?
- ORDER BY rank
- LIMIT 5
- """,
- (query,),
- )
-
- results = cursor.fetchall()
- for url, title, snippet in results:
- print(f"\n{title}")
- print(f" URL: {url}")
- print(f" Snippet: {snippet}")
-
- conn.close()
-
-
-def main():
- print("Loading crawled data...")
- pages = load_crawled_data()
- print(f"Loaded {len(pages)} pages")
-
- if not pages:
- print("No crawled data found. Run crawl_docs.py first.")
- return
-
- print("\nCreating SQLite database...")
- create_database(pages)
-
- # Test searches
- print("\n" + "=" * 50)
- print("Testing FTS5 search")
- print("=" * 50)
-
- test_search("install")
- test_search("computer use agent")
- test_search("benchmark")
-
-
-if __name__ == "__main__":
- main()