The agent that turns websites into APIs!
feat: collector beta
kalil03210fdd4495faa5
diff --git a/.gitignore b/.gitignoreindex 22da872..aa58099 100644--- a/.gitignore+++ b/.gitignore@@ -91,4 +91,6 @@ prompt*.mdtest*FEATURES.md-docs/\ No newline at end of file+docs/+collected/+collector_mode_plan.md\ No newline at end of filediff --git a/pyproject.toml b/pyproject.tomlindex 1302c3d..d8341c5 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -43,6 +43,10 @@ agent = [pricing = ["litellm>=1.0.0",]+collector = [+ "markdownify>=0.12.0",+ "beautifulsoup4>=4.12.0",+][project.urls]Homepage = "https://github.com/kalil0321/reverse-api-engineer"diff --git a/src/reverse_api/cli.py b/src/reverse_api/cli.pyindex 8da6210..38a713a 100644--- a/src/reverse_api/cli.py+++ b/src/reverse_api/cli.py@@ -49,11 +49,12 @@ config_manager = ConfigManager(get_config_path())session_manager = SessionManager(get_history_path())# Mode definitions-MODES = ["manual", "engineer", "agent"]+MODES = ["manual", "engineer", "agent", "collector"]MODE_DESCRIPTIONS = {"manual": "full pipeline","engineer": "reverse engineer only","agent": "autonomous agent + capture",+ "collector": "ai-powered data collection",}@@ -292,6 +293,17 @@ def prompt_interactive_options("model": model,}+ # Collector mode: just needs prompt+ if result_mode == "collector":+ if model is None:+ model = config_manager.get("collector_model", "claude-sonnet-4-5")++ return {+ "mode": result_mode,+ "prompt": prompt,+ "model": model,+ }+# Manual mode: need URLif url is None:try:@@ -346,7 +358,7 @@ def repl_loop():model = config_manager.get("claude_code_model", "claude-sonnet-4-5")display_banner(console, sdk=sdk, model=model)- console.print(" [dim]shift+tab to cycle modes: manual | engineer | agent[/dim]")+ console.print(" [dim]shift+tab to cycle modes: manual | engineer | agent | collector[/dim]")display_footer(console)current_mode = "manual"@@ -377,6 +389,8 @@ def repl_loop():handle_engineer_help(mode_color)elif current_mode == "agent":handle_agent_help(mode_color)+ elif current_mode == "collector":+ handle_collector_help(mode_color)elif current_mode == "manual":handle_manual_help(mode_color)elif cmd.startswith("/messages"):@@ -479,6 +493,14 @@ def repl_loop():)continue+ if mode == "collector":+ # Collector mode: AI-powered data collection+ run_collector(+ prompt=options["prompt"],+ model=options.get("model"),+ )+ continue+# Manual mode: run browser capturerun_manual_capture(prompt=options["prompt"],@@ -924,6 +946,31 @@ def handle_agent_help(mode_color=THEME_PRIMARY):console.print()+def handle_collector_help(mode_color=THEME_PRIMARY):+ """Show help specific to collector mode."""+ from rich.table import Table++ console.print()+ console.print(" [bold white]Collector Mode Help[/bold white]")+ console.print(" [dim]AI-powered web data collection. Describe what data you want, get JSON/CSV output.[/dim]")+ console.print()++ table = Table(show_header=False, box=None, padding=(0, 1))+ table.add_column(style=f"{mode_color} bold", justify="left", width=30)+ table.add_column(style="white", justify="left")++ table.add_row("<prompt>", "Describe data to collect in natural language.\n[dim]Example: Find 10 YC W24 AI startups with name, website, funding[/dim]")+ table.add_row("", "")++ table.add_row("Output", "JSON + CSV saved to ./collected/<folder>/")+ table.add_row("", "")++ table.add_row("Shift+Tab", "Cycle to other modes.")++ console.print(table)+ console.print()++def handle_engineer_help(mode_color=THEME_PRIMARY):"""Show help specific to engineer mode."""from rich.table import Table@@ -1313,6 +1360,55 @@ def run_agent_capture(prompt=None, url=None, reverse_engineer=False, model=None,traceback.print_exc()+def run_collector(prompt=None, model=None, output_dir=None):+ """Run AI-powered data collection with Collector class."""+ import asyncio++ from .collector import Collector++ # Generate run ID+ run_id = generate_run_id()+ output_dir = output_dir or config_manager.get("output_dir")++ # Use collector model from config if not specified+ if model is None:+ model = config_manager.get("collector_model", "claude-sonnet-4-5")++ # Initialize session+ session_manager.add_run(+ run_id=run_id,+ prompt=prompt or "",+ mode="collector",+ timestamp=get_timestamp(),+ )++ try:+ # Run collector+ collector = Collector(+ run_id=run_id,+ prompt=prompt or "",+ model=model,+ output_dir=output_dir,+ )++ result = asyncio.run(collector.run())++ if result and not result.get("error"):+ # Update session with results+ session_manager.update_run(+ run_id=run_id,+ sdk="claude",+ usage=result.get("usage", {}),+ paths={"output_path": result.get("output_path")},+ )+ except Exception as e:+ console.print(f" [red]collector error: {e}[/red]")+ console.print(f" [dim]{ERROR_CTA}[/dim]")+ import traceback++ traceback.print_exc()++def run_auto_capture(prompt=None, url=None, model=None, output_dir=None):"""Auto mode: LLM-driven browser automation + real-time reverse engineering."""output_dir = output_dir or config_manager.get("output_dir")diff --git a/src/reverse_api/collector.py b/src/reverse_api/collector.pynew file mode 100644index 0000000..91347b4--- /dev/null+++ b/src/reverse_api/collector.py@@ -0,0 +1,318 @@+import csv+import json+import logging+from datetime import datetime+from pathlib import Path+from typing import Any++from claude_agent_sdk import (+ AssistantMessage,+ ClaudeAgentOptions,+ ClaudeSDKClient,+ ResultMessage,+ TextBlock,+ ToolResultBlock,+ ToolUseBlock,+)++from .collector_ui import CollectorUI+from .messages import MessageStore+from .pricing import calculate_cost+from .utils import generate_folder_name, get_collected_dir++# Suppress claude_agent_sdk logs+logging.getLogger("claude_agent_sdk").setLevel(logging.WARNING)+logging.getLogger("claude_agent_sdk._internal.transport.subprocess_cli").setLevel(logging.WARNING)+++class Collector:+ """AI-powered web data collector using Claude Agent SDK.++ Uses Claude's built-in tools (WebFetch, Write) to collect data.+ The agent writes collected items to a JSONL file which is then+ processed into JSON + CSV output.+ """++ def __init__(+ self,+ run_id: str,+ prompt: str,+ model: str,+ output_dir: str | None = None,+ ):+ """Initialize collector.++ Args:+ run_id: Unique run identifier+ prompt: Natural language prompt describing data to collect+ model: Claude model to use+ output_dir: Optional custom output directory+ """+ self.run_id = run_id+ self.prompt = prompt+ self.model = model+ self.output_dir = output_dir++ self.ui = CollectorUI()+ self.message_store = MessageStore(run_id, output_dir)++ self._folder_name: str | None = None+ self._collected_dir: Path | None = None+ self.usage_metadata: dict[str, Any] = {}++ async def run(self) -> dict[str, Any] | None:+ """Run the collector agent loop.++ Returns:+ Result dict with output_path and collected_items, or None on error+ """+ self.ui.header(self.run_id, self.prompt, self.model)+ self.ui.start_collecting()++ self._folder_name = generate_folder_name(self.prompt)+ self._collected_dir = get_collected_dir(self._folder_name)++ self.items_path = self._collected_dir / "items.jsonl"++ self.message_store.save_prompt(self._build_collector_prompt())++ result = await self._agent_loop()++ if result and not result.get("error"):+ return self._finalize_collection()++ return result++ def _build_collector_prompt(self) -> str:+ """Build system prompt for collector agent."""+ return f"""You are a web data collection agent.++<mission>+{self.prompt}+</mission>++<output>+Save collected items to: {self.items_path}+Format: JSONL (one JSON object per line, append mode)+</output>++## Guidelines++- Use WebFetch or WebSearch to find relevant sources+- Extract structured data with consistent field names across items+- Include source_url in each item when possible+- Save items incrementally as you find them+- Aim for complete data, but partial items are still valuable++## Output Format++Each item should be a single-line JSON object:+```+{{"name": "...", "website": "...", "description": "...", "source_url": "..."}}+```++When complete, briefly summarize what was collected.+"""++ async def _agent_loop(self) -> dict[str, Any] | None:+ """Run Claude Agent SDK loop."""+ options = ClaudeAgentOptions(+ permission_mode="bypassPermissions",+ allowed_tools=[+ "Read",+ "Write",+ "Bash",+ "WebFetch",+ "WebSearch",+ ],+ cwd=str(self._collected_dir),+ model=self.model,+ )++ try:+ async with ClaudeSDKClient(options=options) as client:+ await client.query(self._build_collector_prompt())++ async for message in client.receive_response():+ # Track usage+ if hasattr(message, "usage"):+ usage = message.usage+ if isinstance(usage, dict):+ for key, value in usage.items():+ if isinstance(key, str) and isinstance(value, (int, float)):+ current = self.usage_metadata.get(key, 0)+ if isinstance(current, (int, float)):+ self.usage_metadata[key] = current + value++ if isinstance(message, AssistantMessage):+ last_tool_name = None+ for block in message.content:+ if isinstance(block, ToolUseBlock):+ last_tool_name = block.name+ tool_input = block.input if isinstance(block.input, dict) else {}+ self.ui.tool_start(block.name, tool_input)+ self.message_store.save_tool_start(block.name, tool_input)++ # Track item saves+ if block.name == "Write":+ file_path = tool_input.get("file_path", "")+ if "items.jsonl" in file_path:+ content = tool_input.get("content", "")+ self.ui.item_saved(content[:50])++ elif isinstance(block, ToolResultBlock):+ is_error = block.is_error if block.is_error else False+ tool_name = last_tool_name or "Tool"+ self.ui.tool_result(tool_name, is_error)+ self.message_store.save_tool_result(tool_name, is_error)+ elif isinstance(block, TextBlock):+ self.ui.thinking(block.text)+ self.message_store.save_thinking(block.text)++ elif isinstance(message, ResultMessage):+ if message.is_error:+ self.ui.error(message.result or "Unknown error")+ self.message_store.save_error(message.result or "Unknown error")+ return {"error": message.result}+ else:+ return {"success": True}++ except Exception as e:+ error_msg = str(e)+ self.ui.error(error_msg)+ self.message_store.save_error(error_msg)+ return {"error": error_msg}++ return {"success": True}++ def _finalize_collection(self) -> dict[str, Any]:+ """Post-process collected items and export to JSON/CSV."""+ if not self._collected_dir:+ return {"error": "No collection directory"}++ items_path = self._collected_dir / "items.jsonl"++ # Parse collected items from JSONL+ collected_items: list[dict[str, Any]] = []+ sources: set[str] = set()++ if items_path.exists():+ with open(items_path) as f:+ for line in f:+ line = line.strip()+ if line:+ try:+ item = json.loads(line)+ if isinstance(item, dict):+ collected_items.append(item)+ # Track sources+ if "source_url" in item:+ sources.add(item["source_url"])+ elif "url" in item:+ sources.add(item["url"])+ except json.JSONDecodeError:+ continue++ if not collected_items:+ self.ui.error("No items were collected. Check if the agent saved data correctly.")+ return {"error": "No items collected", "items": []}++ # Export JSON+ json_path = self._collected_dir / "data.json"+ with open(json_path, "w") as f:+ json.dump(collected_items, f, indent=2)++ # Export CSV+ csv_path = self._collected_dir / "data.csv"+ self._export_csv(csv_path, collected_items)++ # Update README with final stats+ readme_path = self._collected_dir / "README.md"+ self._export_readme(readme_path, collected_items, sources)++ # Calculate cost+ if self.usage_metadata:+ input_tokens = self.usage_metadata.get("input_tokens", 0)+ output_tokens = self.usage_metadata.get("output_tokens", 0)+ cache_creation = self.usage_metadata.get("cache_creation_input_tokens", 0)+ cache_read = self.usage_metadata.get("cache_read_input_tokens", 0)++ cost = calculate_cost(+ model_id=self.model,+ input_tokens=input_tokens,+ output_tokens=output_tokens,+ cache_creation_tokens=cache_creation,+ cache_read_tokens=cache_read,+ )+ self.usage_metadata["estimated_cost_usd"] = cost++ # Show completion+ self.ui.collection_complete(len(collected_items), str(self._collected_dir))+ if self.usage_metadata:+ self.ui.usage_summary(self.usage_metadata)++ result = {+ "output_path": str(self._collected_dir),+ "items_collected": len(collected_items),+ "files": {+ "json": str(json_path),+ "csv": str(csv_path),+ "readme": str(readme_path),+ },+ "usage": self.usage_metadata,+ }+ self.message_store.save_result(result)+ return result++ def _export_csv(self, csv_path: Path, items: list[dict[str, Any]]) -> None:+ """Export items to CSV with auto-flattened columns."""+ if not items:+ return++ # Collect all unique keys+ all_keys: set[str] = set()+ for item in items:+ all_keys.update(item.keys())++ # Sort keys for consistent column order+ fieldnames = sorted(all_keys)++ with open(csv_path, "w", newline="") as f:+ writer = csv.DictWriter(f, fieldnames=fieldnames)+ writer.writeheader()+ for item in items:+ # Convert non-string values to strings+ row = {k: str(v) if v is not None else "" for k, v in item.items()}+ writer.writerow(row)++ def _export_readme(self, readme_path: Path, items: list[dict[str, Any]], sources: set[str]) -> None:+ """Generate README with collection metadata."""+ folder_name = self._folder_name or "collection"+ readme_content = f"""# {folder_name.replace('_', ' ').title()}++## Query+{self.prompt}++## Metadata+- **Run ID**: {self.run_id}+- **Items Collected**: {len(items)}+- **Sources**: {len(sources)}+- **Collected At**: {datetime.now().isoformat()}++## Files+- `data.json` - All collected items in JSON format+- `data.csv` - All collected items in CSV format++## Schema+"""+ # Add field schema+ if items:+ all_keys: set[str] = set()+ for item in items:+ all_keys.update(item.keys())++ for key in sorted(all_keys):+ readme_content += f"- `{key}`\n"++ with open(readme_path, "w") as f:+ f.write(readme_content)diff --git a/src/reverse_api/collector_ui.py b/src/reverse_api/collector_ui.pynew file mode 100644index 0000000..ec10239--- /dev/null+++ b/src/reverse_api/collector_ui.py@@ -0,0 +1,106 @@+"""Terminal UI for collector mode with streaming progress updates."""++from rich.console import Console++from .tui import THEME_DIM++COLLECTOR_COLOR = "#ffd700" # Gold+++class CollectorUI:+ """Terminal UI with real-time collection progress."""++ def __init__(self, verbose: bool = True):+ self.console = Console()+ self.verbose = verbose+ self._items_collected = 0++ def header(self, run_id: str, prompt: str, model: str | None = None) -> None:+ """Display the collector session header."""+ from . import __version__++ self.console.print()+ self.console.print(f" [white]reverse-api[/white] [dim]v{__version__}[/dim]")+ self.console.print(f" [dim]━[/dim] [white]{run_id}[/white]")+ self.console.print(f" [{COLLECTOR_COLOR}]collector[/{COLLECTOR_COLOR}] [dim]|[/dim] [dim]model[/dim] [white]{model or '---'}[/white]")+ self.console.print(f" [{COLLECTOR_COLOR}]task[/{COLLECTOR_COLOR}] [white]{prompt[:80]}{'...' if len(prompt) > 80 else ''}[/white]")+ self.console.print()++ def start_collecting(self) -> None:+ """Display collection start message."""+ self.console.print(" [dim]planning collection strategy...[/dim]")+ self.console.print()++ def item_saved(self, preview: str) -> None:+ """Display when an item is saved."""+ self._items_collected += 1+ display_preview = preview[:50] + "..." if len(preview) > 50 else preview+ self.console.print(f" [dim]>[/dim] ++ [white]item_saved[/white] [dim]{display_preview}[/dim]")++ def thinking(self, text: str, max_length: int = 80) -> None:+ """Display agent thinking/response text."""+ if not self.verbose:+ return+ if len(text) < 20:+ return+ display_text = text[:max_length].replace("\n", " ").strip()+ if len(text) > max_length:+ display_text += "..."+ self.console.print(f" [dim].. {display_text}[/dim]")++ def tool_start(self, tool_name: str, tool_input: dict) -> None:+ """Display when a tool starts execution."""+ icon_map = {+ "Write": "wr",+ "Read": "rd",+ "WebFetch": "wf",+ "WebSearch": "ws",+ "Bash": "sh",+ }+ icon = icon_map.get(tool_name, "> ")+ summary = self._summarize_input(tool_name, tool_input)+ self.console.print(f" [dim]>[/dim] {icon} [white]{tool_name[:12]:12}[/white] {summary}")++ def tool_result(self, tool_name: str, is_error: bool = False, output: str | None = None) -> None:+ """Display when a tool completes."""+ if is_error:+ self.console.print(f" [dim]![/dim] [red]{tool_name} failed[/red]")++ def _summarize_input(self, tool_name: str, tool_input: dict) -> str:+ """Create a brief summary of tool input."""+ if tool_name == "WebFetch":+ url = tool_input.get("url", "")+ return f"[dim]{url[:50]}{'...' if len(url) > 50 else ''}[/dim]"+ elif tool_name == "Write":+ path = tool_input.get("file_path", "")+ return f"[dim]{path}[/dim]"+ return ""++ def collection_complete(self, total_items: int, output_path: str) -> None:+ """Display collection complete message."""+ self.console.print()+ self.console.print(f" [{COLLECTOR_COLOR}]collection complete[/{COLLECTOR_COLOR}] [white]({total_items} items)[/white]")+ self.console.print(f" [dim]output:[/dim] [white]{output_path}[/white]")+ self.console.print()++ def error(self, message: str) -> None:+ """Display error message."""+ from .tui import ERROR_CTA+ self.console.print()+ self.console.print(f" [dim]![/dim] [red]error:[/red] {message}")+ self.console.print(f" [dim]{ERROR_CTA}[/dim]")++ def usage_summary(self, usage: dict) -> None:+ """Display usage/cost summary."""+ input_tokens = usage.get("input_tokens", 0)+ output_tokens = usage.get("output_tokens", 0)+ cost = usage.get("estimated_cost_usd", 0)++ if input_tokens > 0 or output_tokens > 0:+ self.console.print(f" [dim]usage:[/dim]")+ if input_tokens > 0:+ self.console.print(f" [dim] input: {input_tokens:,} tokens[/dim]")+ if output_tokens > 0:+ self.console.print(f" [dim] output: {output_tokens:,} tokens[/dim]")+ if cost > 0:+ self.console.print(f" [dim] cost: ${cost:.4f}[/dim]")diff --git a/src/reverse_api/config.py b/src/reverse_api/config.pyindex fff483f..b5d5772 100644--- a/src/reverse_api/config.py+++ b/src/reverse_api/config.py@@ -8,6 +8,7 @@ DEFAULT_CONFIG = {"agent_provider": "auto", # "auto" (default, MCP-based browser + engineering), "browser-use", or "stagehand""browser_use_model": "bu-llm", # "bu-llm" or "{provider}/{model_name}" (e.g. "openai/gpt-5-mini")"claude_code_model": "claude-sonnet-4-5",+ "collector_model": "claude-sonnet-4-5", # Model for collector mode"opencode_model": "claude-sonnet-4-5","opencode_provider": "anthropic","output_dir": None, # None means use ~/.reverse-api/runsdiff --git a/src/reverse_api/tui.py b/src/reverse_api/tui.pyindex f7fbc55..c0a792b 100644--- a/src/reverse_api/tui.py+++ b/src/reverse_api/tui.py@@ -17,6 +17,7 @@ MODE_COLORS = {"manual": "#ff5f50", # Coral Red"engineer": "#5f9fff", # Blue"agent": "#50ff9f", # Green/Cyan+ "collector": "#ffd700", # Gold}diff --git a/src/reverse_api/utils.py b/src/reverse_api/utils.pyindex 9683f59..5ed0508 100644--- a/src/reverse_api/utils.py+++ b/src/reverse_api/utils.py@@ -29,6 +29,16 @@ def generate_folder_name(prompt: str, sdk: str = None, session_id: str = None) -sdk = "claude"try:+ # Check if event loop is already running (e.g., called from async context)+ try:+ loop = asyncio.get_running_loop()+ except RuntimeError:+ loop = None++ if loop is not None:+ # Already in async context, use fallback to avoid nested asyncio.run+ return _slugify(prompt)+if sdk == "opencode":return asyncio.run(_generate_folder_name_opencode_async(prompt, session_id))else:@@ -420,3 +430,17 @@ def get_messages_path(run_id: str, output_dir: str | None = None) -> Path:def get_timestamp() -> str:"""Get current timestamp in ISO format."""return datetime.now().isoformat()+++def get_collected_dir(folder_name: str) -> Path:+ """Get ./collected/{folder_name}/ directory for collector mode output.++ Args:+ folder_name: Name of the collection folder++ Returns:+ Path to the collection output directory+ """+ path = Path.cwd() / "collected" / folder_name+ path.mkdir(parents=True, exist_ok=True)+ return pathdiff --git a/uv.lock b/uv.lockindex 33eba12..56543de 100644--- a/uv.lock+++ b/uv.lock@@ -146,6 +146,19 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/3a/2a/7cc015f5b9f5db42b7d48157e23356022889fc354a2813c15934b7cb5c0e/attrs-25.4.0-py3-none-any.whl", hash = "sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373", size = 67615, upload-time = "2025-10-06T13:54:43.17Z" },]+[[package]]+name = "beautifulsoup4"+version = "4.14.3"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "soupsieve" },+ { name = "typing-extensions" },+]+sdist = { url = "https://files.pythonhosted.org/packages/c3/b0/1c6a16426d389813b48d95e26898aff79abbde42ad353958ad95cc8c9b21/beautifulsoup4-4.14.3.tar.gz", hash = "sha256:6292b1c5186d356bba669ef9f7f051757099565ad9ada5dd630bd9de5fa7fb86", size = 627737, upload-time = "2025-11-30T15:08:26.084Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/1a/39/47f9197bdd44df24d67ac8893641e16f386c984a0619ef2ee4c51fbbc019/beautifulsoup4-4.14.3-py3-none-any.whl", hash = "sha256:0918bfe44902e6ad8d57732ba310582e98da931428d231a5ecb9e7c703a735bb", size = 107721, upload-time = "2025-11-30T15:08:24.087Z" },+]+[[package]]name = "brotli"version = "1.2.0"@@ -1090,6 +1103,19 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" },]+[[package]]+name = "markdownify"+version = "1.2.2"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "beautifulsoup4" },+ { name = "six" },+]+sdist = { url = "https://files.pythonhosted.org/packages/3f/bc/c8c8eea5335341306b0fa7e1cb33c5e1c8d24ef70ddd684da65f41c49c92/markdownify-1.2.2.tar.gz", hash = "sha256:b274f1b5943180b031b699b199cbaeb1e2ac938b75851849a31fd0c3d6603d09", size = 18816, upload-time = "2025-11-16T19:21:18.565Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/43/ce/f1e3e9d959db134cedf06825fae8d5b294bd368aacdd0831a3975b7c4d55/markdownify-1.2.2-py3-none-any.whl", hash = "sha256:3f02d3cc52714084d6e589f70397b6fc9f2f3a8531481bf35e8cc39f975e186a", size = 15724, upload-time = "2025-11-16T19:21:17.622Z" },+]+[[package]]name = "markupsafe"version = "3.0.3"@@ -1989,6 +2015,10 @@ dependencies = [agent = [{ name = "stagehand" },]+collector = [+ { name = "beautifulsoup4" },+ { name = "markdownify" },+]pricing = [{ name = "litellm" },]@@ -2003,10 +2033,12 @@ dev = [requires-dist = [{ name = "aiohttp", specifier = ">=3.12.15" },{ name = "anthropic", specifier = ">=0.40.0" },+ { name = "beautifulsoup4", marker = "extra == 'collector'", specifier = ">=4.12.0" },{ name = "brotli", specifier = ">=1.2.0" },{ name = "claude-agent-sdk", specifier = ">=0.1.0" },{ name = "click", specifier = ">=8.1.0" },{ name = "litellm", marker = "extra == 'pricing'", specifier = ">=1.0.0" },+ { name = "markdownify", marker = "extra == 'collector'", specifier = ">=0.12.0" },{ name = "playwright", specifier = ">=1.40.0" },{ name = "playwright-stealth", specifier = ">=1.0.0" },{ name = "questionary", specifier = ">=2.0.0" },@@ -2015,7 +2047,7 @@ requires-dist = [{ name = "stagehand", marker = "extra == 'agent'" },{ name = "watchdog", specifier = ">=3.0.0" },]-provides-extras = ["agent", "pricing"]+provides-extras = ["agent", "pricing", "collector"][package.metadata.requires-dev]dev = [@@ -2191,6 +2223,15 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" },]+[[package]]+name = "six"+version = "1.17.0"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" },+]+[[package]]name = "sniffio"version = "1.3.1"@@ -2200,6 +2241,15 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" },]+[[package]]+name = "soupsieve"+version = "2.8.1"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/89/23/adf3796d740536d63a6fbda113d07e60c734b6ed5d3058d1e47fc0495e47/soupsieve-2.8.1.tar.gz", hash = "sha256:4cf733bc50fa805f5df4b8ef4740fc0e0fa6218cf3006269afd3f9d6d80fd350", size = 117856, upload-time = "2025-12-18T13:50:34.655Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/48/f3/b67d6ea49ca9154453b6d70b34ea22f3996b9fa55da105a79d8732227adc/soupsieve-2.8.1-py3-none-any.whl", hash = "sha256:a11fe2a6f3d76ab3cf2de04eb339c1be5b506a8a47f2ceb6d139803177f85434", size = 36710, upload-time = "2025-12-18T13:50:33.267Z" },+]+[[package]]name = "sse-starlette"version = "3.0.4"