Skip to content

The agent that turns websites into APIs!

feat: collector beta

kalil03210fdd4495faa5
diff --git a/.gitignore b/.gitignore
index 22da872..aa58099 100644
--- a/.gitignore
+++ b/.gitignore
@@ -91,4 +91,6 @@ prompt*.md
test*
FEATURES.md
-docs/
\ No newline at end of file
+docs/
+collected/
+collector_mode_plan.md
\ No newline at end of file
diff --git a/pyproject.toml b/pyproject.toml
index 1302c3d..d8341c5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -43,6 +43,10 @@ agent = [
pricing = [
"litellm>=1.0.0",
]
+collector = [
+ "markdownify>=0.12.0",
+ "beautifulsoup4>=4.12.0",
+]
[project.urls]
Homepage = "https://github.com/kalil0321/reverse-api-engineer"
diff --git a/src/reverse_api/cli.py b/src/reverse_api/cli.py
index 8da6210..38a713a 100644
--- a/src/reverse_api/cli.py
+++ b/src/reverse_api/cli.py
@@ -49,11 +49,12 @@ config_manager = ConfigManager(get_config_path())
session_manager = SessionManager(get_history_path())
# Mode definitions
-MODES = ["manual", "engineer", "agent"]
+MODES = ["manual", "engineer", "agent", "collector"]
MODE_DESCRIPTIONS = {
"manual": "full pipeline",
"engineer": "reverse engineer only",
"agent": "autonomous agent + capture",
+ "collector": "ai-powered data collection",
}
@@ -292,6 +293,17 @@ def prompt_interactive_options(
"model": model,
}
+ # Collector mode: just needs prompt
+ if result_mode == "collector":
+ if model is None:
+ model = config_manager.get("collector_model", "claude-sonnet-4-5")
+
+ return {
+ "mode": result_mode,
+ "prompt": prompt,
+ "model": model,
+ }
+
# Manual mode: need URL
if url is None:
try:
@@ -346,7 +358,7 @@ def repl_loop():
model = config_manager.get("claude_code_model", "claude-sonnet-4-5")
display_banner(console, sdk=sdk, model=model)
- console.print(" [dim]shift+tab to cycle modes: manual | engineer | agent[/dim]")
+ console.print(" [dim]shift+tab to cycle modes: manual | engineer | agent | collector[/dim]")
display_footer(console)
current_mode = "manual"
@@ -377,6 +389,8 @@ def repl_loop():
handle_engineer_help(mode_color)
elif current_mode == "agent":
handle_agent_help(mode_color)
+ elif current_mode == "collector":
+ handle_collector_help(mode_color)
elif current_mode == "manual":
handle_manual_help(mode_color)
elif cmd.startswith("/messages"):
@@ -479,6 +493,14 @@ def repl_loop():
)
continue
+ if mode == "collector":
+ # Collector mode: AI-powered data collection
+ run_collector(
+ prompt=options["prompt"],
+ model=options.get("model"),
+ )
+ continue
+
# Manual mode: run browser capture
run_manual_capture(
prompt=options["prompt"],
@@ -924,6 +946,31 @@ def handle_agent_help(mode_color=THEME_PRIMARY):
console.print()
+def handle_collector_help(mode_color=THEME_PRIMARY):
+ """Show help specific to collector mode."""
+ from rich.table import Table
+
+ console.print()
+ console.print(" [bold white]Collector Mode Help[/bold white]")
+ console.print(" [dim]AI-powered web data collection. Describe what data you want, get JSON/CSV output.[/dim]")
+ console.print()
+
+ table = Table(show_header=False, box=None, padding=(0, 1))
+ table.add_column(style=f"{mode_color} bold", justify="left", width=30)
+ table.add_column(style="white", justify="left")
+
+ table.add_row("<prompt>", "Describe data to collect in natural language.\n[dim]Example: Find 10 YC W24 AI startups with name, website, funding[/dim]")
+ table.add_row("", "")
+
+ table.add_row("Output", "JSON + CSV saved to ./collected/<folder>/")
+ table.add_row("", "")
+
+ table.add_row("Shift+Tab", "Cycle to other modes.")
+
+ console.print(table)
+ console.print()
+
+
def handle_engineer_help(mode_color=THEME_PRIMARY):
"""Show help specific to engineer mode."""
from rich.table import Table
@@ -1313,6 +1360,55 @@ def run_agent_capture(prompt=None, url=None, reverse_engineer=False, model=None,
traceback.print_exc()
+def run_collector(prompt=None, model=None, output_dir=None):
+ """Run AI-powered data collection with Collector class."""
+ import asyncio
+
+ from .collector import Collector
+
+ # Generate run ID
+ run_id = generate_run_id()
+ output_dir = output_dir or config_manager.get("output_dir")
+
+ # Use collector model from config if not specified
+ if model is None:
+ model = config_manager.get("collector_model", "claude-sonnet-4-5")
+
+ # Initialize session
+ session_manager.add_run(
+ run_id=run_id,
+ prompt=prompt or "",
+ mode="collector",
+ timestamp=get_timestamp(),
+ )
+
+ try:
+ # Run collector
+ collector = Collector(
+ run_id=run_id,
+ prompt=prompt or "",
+ model=model,
+ output_dir=output_dir,
+ )
+
+ result = asyncio.run(collector.run())
+
+ if result and not result.get("error"):
+ # Update session with results
+ session_manager.update_run(
+ run_id=run_id,
+ sdk="claude",
+ usage=result.get("usage", {}),
+ paths={"output_path": result.get("output_path")},
+ )
+ except Exception as e:
+ console.print(f" [red]collector error: {e}[/red]")
+ console.print(f" [dim]{ERROR_CTA}[/dim]")
+ import traceback
+
+ traceback.print_exc()
+
+
def run_auto_capture(prompt=None, url=None, model=None, output_dir=None):
"""Auto mode: LLM-driven browser automation + real-time reverse engineering."""
output_dir = output_dir or config_manager.get("output_dir")
diff --git a/src/reverse_api/collector.py b/src/reverse_api/collector.py
new file mode 100644
index 0000000..91347b4
--- /dev/null
+++ b/src/reverse_api/collector.py
@@ -0,0 +1,318 @@
+import csv
+import json
+import logging
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from claude_agent_sdk import (
+ AssistantMessage,
+ ClaudeAgentOptions,
+ ClaudeSDKClient,
+ ResultMessage,
+ TextBlock,
+ ToolResultBlock,
+ ToolUseBlock,
+)
+
+from .collector_ui import CollectorUI
+from .messages import MessageStore
+from .pricing import calculate_cost
+from .utils import generate_folder_name, get_collected_dir
+
+# Suppress claude_agent_sdk logs
+logging.getLogger("claude_agent_sdk").setLevel(logging.WARNING)
+logging.getLogger("claude_agent_sdk._internal.transport.subprocess_cli").setLevel(logging.WARNING)
+
+
+class Collector:
+ """AI-powered web data collector using Claude Agent SDK.
+
+ Uses Claude's built-in tools (WebFetch, Write) to collect data.
+ The agent writes collected items to a JSONL file which is then
+ processed into JSON + CSV output.
+ """
+
+ def __init__(
+ self,
+ run_id: str,
+ prompt: str,
+ model: str,
+ output_dir: str | None = None,
+ ):
+ """Initialize collector.
+
+ Args:
+ run_id: Unique run identifier
+ prompt: Natural language prompt describing data to collect
+ model: Claude model to use
+ output_dir: Optional custom output directory
+ """
+ self.run_id = run_id
+ self.prompt = prompt
+ self.model = model
+ self.output_dir = output_dir
+
+ self.ui = CollectorUI()
+ self.message_store = MessageStore(run_id, output_dir)
+
+ self._folder_name: str | None = None
+ self._collected_dir: Path | None = None
+ self.usage_metadata: dict[str, Any] = {}
+
+ async def run(self) -> dict[str, Any] | None:
+ """Run the collector agent loop.
+
+ Returns:
+ Result dict with output_path and collected_items, or None on error
+ """
+ self.ui.header(self.run_id, self.prompt, self.model)
+ self.ui.start_collecting()
+
+ self._folder_name = generate_folder_name(self.prompt)
+ self._collected_dir = get_collected_dir(self._folder_name)
+
+ self.items_path = self._collected_dir / "items.jsonl"
+
+ self.message_store.save_prompt(self._build_collector_prompt())
+
+ result = await self._agent_loop()
+
+ if result and not result.get("error"):
+ return self._finalize_collection()
+
+ return result
+
+ def _build_collector_prompt(self) -> str:
+ """Build system prompt for collector agent."""
+ return f"""You are a web data collection agent.
+
+<mission>
+{self.prompt}
+</mission>
+
+<output>
+Save collected items to: {self.items_path}
+Format: JSONL (one JSON object per line, append mode)
+</output>
+
+## Guidelines
+
+- Use WebFetch or WebSearch to find relevant sources
+- Extract structured data with consistent field names across items
+- Include source_url in each item when possible
+- Save items incrementally as you find them
+- Aim for complete data, but partial items are still valuable
+
+## Output Format
+
+Each item should be a single-line JSON object:
+```
+{{"name": "...", "website": "...", "description": "...", "source_url": "..."}}
+```
+
+When complete, briefly summarize what was collected.
+"""
+
+ async def _agent_loop(self) -> dict[str, Any] | None:
+ """Run Claude Agent SDK loop."""
+ options = ClaudeAgentOptions(
+ permission_mode="bypassPermissions",
+ allowed_tools=[
+ "Read",
+ "Write",
+ "Bash",
+ "WebFetch",
+ "WebSearch",
+ ],
+ cwd=str(self._collected_dir),
+ model=self.model,
+ )
+
+ try:
+ async with ClaudeSDKClient(options=options) as client:
+ await client.query(self._build_collector_prompt())
+
+ async for message in client.receive_response():
+ # Track usage
+ if hasattr(message, "usage"):
+ usage = message.usage
+ if isinstance(usage, dict):
+ for key, value in usage.items():
+ if isinstance(key, str) and isinstance(value, (int, float)):
+ current = self.usage_metadata.get(key, 0)
+ if isinstance(current, (int, float)):
+ self.usage_metadata[key] = current + value
+
+ if isinstance(message, AssistantMessage):
+ last_tool_name = None
+ for block in message.content:
+ if isinstance(block, ToolUseBlock):
+ last_tool_name = block.name
+ tool_input = block.input if isinstance(block.input, dict) else {}
+ self.ui.tool_start(block.name, tool_input)
+ self.message_store.save_tool_start(block.name, tool_input)
+
+ # Track item saves
+ if block.name == "Write":
+ file_path = tool_input.get("file_path", "")
+ if "items.jsonl" in file_path:
+ content = tool_input.get("content", "")
+ self.ui.item_saved(content[:50])
+
+ elif isinstance(block, ToolResultBlock):
+ is_error = block.is_error if block.is_error else False
+ tool_name = last_tool_name or "Tool"
+ self.ui.tool_result(tool_name, is_error)
+ self.message_store.save_tool_result(tool_name, is_error)
+ elif isinstance(block, TextBlock):
+ self.ui.thinking(block.text)
+ self.message_store.save_thinking(block.text)
+
+ elif isinstance(message, ResultMessage):
+ if message.is_error:
+ self.ui.error(message.result or "Unknown error")
+ self.message_store.save_error(message.result or "Unknown error")
+ return {"error": message.result}
+ else:
+ return {"success": True}
+
+ except Exception as e:
+ error_msg = str(e)
+ self.ui.error(error_msg)
+ self.message_store.save_error(error_msg)
+ return {"error": error_msg}
+
+ return {"success": True}
+
+ def _finalize_collection(self) -> dict[str, Any]:
+ """Post-process collected items and export to JSON/CSV."""
+ if not self._collected_dir:
+ return {"error": "No collection directory"}
+
+ items_path = self._collected_dir / "items.jsonl"
+
+ # Parse collected items from JSONL
+ collected_items: list[dict[str, Any]] = []
+ sources: set[str] = set()
+
+ if items_path.exists():
+ with open(items_path) as f:
+ for line in f:
+ line = line.strip()
+ if line:
+ try:
+ item = json.loads(line)
+ if isinstance(item, dict):
+ collected_items.append(item)
+ # Track sources
+ if "source_url" in item:
+ sources.add(item["source_url"])
+ elif "url" in item:
+ sources.add(item["url"])
+ except json.JSONDecodeError:
+ continue
+
+ if not collected_items:
+ self.ui.error("No items were collected. Check if the agent saved data correctly.")
+ return {"error": "No items collected", "items": []}
+
+ # Export JSON
+ json_path = self._collected_dir / "data.json"
+ with open(json_path, "w") as f:
+ json.dump(collected_items, f, indent=2)
+
+ # Export CSV
+ csv_path = self._collected_dir / "data.csv"
+ self._export_csv(csv_path, collected_items)
+
+ # Update README with final stats
+ readme_path = self._collected_dir / "README.md"
+ self._export_readme(readme_path, collected_items, sources)
+
+ # Calculate cost
+ if self.usage_metadata:
+ input_tokens = self.usage_metadata.get("input_tokens", 0)
+ output_tokens = self.usage_metadata.get("output_tokens", 0)
+ cache_creation = self.usage_metadata.get("cache_creation_input_tokens", 0)
+ cache_read = self.usage_metadata.get("cache_read_input_tokens", 0)
+
+ cost = calculate_cost(
+ model_id=self.model,
+ input_tokens=input_tokens,
+ output_tokens=output_tokens,
+ cache_creation_tokens=cache_creation,
+ cache_read_tokens=cache_read,
+ )
+ self.usage_metadata["estimated_cost_usd"] = cost
+
+ # Show completion
+ self.ui.collection_complete(len(collected_items), str(self._collected_dir))
+ if self.usage_metadata:
+ self.ui.usage_summary(self.usage_metadata)
+
+ result = {
+ "output_path": str(self._collected_dir),
+ "items_collected": len(collected_items),
+ "files": {
+ "json": str(json_path),
+ "csv": str(csv_path),
+ "readme": str(readme_path),
+ },
+ "usage": self.usage_metadata,
+ }
+ self.message_store.save_result(result)
+ return result
+
+ def _export_csv(self, csv_path: Path, items: list[dict[str, Any]]) -> None:
+ """Export items to CSV with auto-flattened columns."""
+ if not items:
+ return
+
+ # Collect all unique keys
+ all_keys: set[str] = set()
+ for item in items:
+ all_keys.update(item.keys())
+
+ # Sort keys for consistent column order
+ fieldnames = sorted(all_keys)
+
+ with open(csv_path, "w", newline="") as f:
+ writer = csv.DictWriter(f, fieldnames=fieldnames)
+ writer.writeheader()
+ for item in items:
+ # Convert non-string values to strings
+ row = {k: str(v) if v is not None else "" for k, v in item.items()}
+ writer.writerow(row)
+
+ def _export_readme(self, readme_path: Path, items: list[dict[str, Any]], sources: set[str]) -> None:
+ """Generate README with collection metadata."""
+ folder_name = self._folder_name or "collection"
+ readme_content = f"""# {folder_name.replace('_', ' ').title()}
+
+## Query
+{self.prompt}
+
+## Metadata
+- **Run ID**: {self.run_id}
+- **Items Collected**: {len(items)}
+- **Sources**: {len(sources)}
+- **Collected At**: {datetime.now().isoformat()}
+
+## Files
+- `data.json` - All collected items in JSON format
+- `data.csv` - All collected items in CSV format
+
+## Schema
+"""
+ # Add field schema
+ if items:
+ all_keys: set[str] = set()
+ for item in items:
+ all_keys.update(item.keys())
+
+ for key in sorted(all_keys):
+ readme_content += f"- `{key}`\n"
+
+ with open(readme_path, "w") as f:
+ f.write(readme_content)
diff --git a/src/reverse_api/collector_ui.py b/src/reverse_api/collector_ui.py
new file mode 100644
index 0000000..ec10239
--- /dev/null
+++ b/src/reverse_api/collector_ui.py
@@ -0,0 +1,106 @@
+"""Terminal UI for collector mode with streaming progress updates."""
+
+from rich.console import Console
+
+from .tui import THEME_DIM
+
+COLLECTOR_COLOR = "#ffd700" # Gold
+
+
+class CollectorUI:
+ """Terminal UI with real-time collection progress."""
+
+ def __init__(self, verbose: bool = True):
+ self.console = Console()
+ self.verbose = verbose
+ self._items_collected = 0
+
+ def header(self, run_id: str, prompt: str, model: str | None = None) -> None:
+ """Display the collector session header."""
+ from . import __version__
+
+ self.console.print()
+ self.console.print(f" [white]reverse-api[/white] [dim]v{__version__}[/dim]")
+ self.console.print(f" [dim]━[/dim] [white]{run_id}[/white]")
+ self.console.print(f" [{COLLECTOR_COLOR}]collector[/{COLLECTOR_COLOR}] [dim]|[/dim] [dim]model[/dim] [white]{model or '---'}[/white]")
+ self.console.print(f" [{COLLECTOR_COLOR}]task[/{COLLECTOR_COLOR}] [white]{prompt[:80]}{'...' if len(prompt) > 80 else ''}[/white]")
+ self.console.print()
+
+ def start_collecting(self) -> None:
+ """Display collection start message."""
+ self.console.print(" [dim]planning collection strategy...[/dim]")
+ self.console.print()
+
+ def item_saved(self, preview: str) -> None:
+ """Display when an item is saved."""
+ self._items_collected += 1
+ display_preview = preview[:50] + "..." if len(preview) > 50 else preview
+ self.console.print(f" [dim]>[/dim] ++ [white]item_saved[/white] [dim]{display_preview}[/dim]")
+
+ def thinking(self, text: str, max_length: int = 80) -> None:
+ """Display agent thinking/response text."""
+ if not self.verbose:
+ return
+ if len(text) < 20:
+ return
+ display_text = text[:max_length].replace("\n", " ").strip()
+ if len(text) > max_length:
+ display_text += "..."
+ self.console.print(f" [dim].. {display_text}[/dim]")
+
+ def tool_start(self, tool_name: str, tool_input: dict) -> None:
+ """Display when a tool starts execution."""
+ icon_map = {
+ "Write": "wr",
+ "Read": "rd",
+ "WebFetch": "wf",
+ "WebSearch": "ws",
+ "Bash": "sh",
+ }
+ icon = icon_map.get(tool_name, "> ")
+ summary = self._summarize_input(tool_name, tool_input)
+ self.console.print(f" [dim]>[/dim] {icon} [white]{tool_name[:12]:12}[/white] {summary}")
+
+ def tool_result(self, tool_name: str, is_error: bool = False, output: str | None = None) -> None:
+ """Display when a tool completes."""
+ if is_error:
+ self.console.print(f" [dim]![/dim] [red]{tool_name} failed[/red]")
+
+ def _summarize_input(self, tool_name: str, tool_input: dict) -> str:
+ """Create a brief summary of tool input."""
+ if tool_name == "WebFetch":
+ url = tool_input.get("url", "")
+ return f"[dim]{url[:50]}{'...' if len(url) > 50 else ''}[/dim]"
+ elif tool_name == "Write":
+ path = tool_input.get("file_path", "")
+ return f"[dim]{path}[/dim]"
+ return ""
+
+ def collection_complete(self, total_items: int, output_path: str) -> None:
+ """Display collection complete message."""
+ self.console.print()
+ self.console.print(f" [{COLLECTOR_COLOR}]collection complete[/{COLLECTOR_COLOR}] [white]({total_items} items)[/white]")
+ self.console.print(f" [dim]output:[/dim] [white]{output_path}[/white]")
+ self.console.print()
+
+ def error(self, message: str) -> None:
+ """Display error message."""
+ from .tui import ERROR_CTA
+ self.console.print()
+ self.console.print(f" [dim]![/dim] [red]error:[/red] {message}")
+ self.console.print(f" [dim]{ERROR_CTA}[/dim]")
+
+ def usage_summary(self, usage: dict) -> None:
+ """Display usage/cost summary."""
+ input_tokens = usage.get("input_tokens", 0)
+ output_tokens = usage.get("output_tokens", 0)
+ cost = usage.get("estimated_cost_usd", 0)
+
+ if input_tokens > 0 or output_tokens > 0:
+ self.console.print(f" [dim]usage:[/dim]")
+ if input_tokens > 0:
+ self.console.print(f" [dim] input: {input_tokens:,} tokens[/dim]")
+ if output_tokens > 0:
+ self.console.print(f" [dim] output: {output_tokens:,} tokens[/dim]")
+ if cost > 0:
+ self.console.print(f" [dim] cost: ${cost:.4f}[/dim]")
diff --git a/src/reverse_api/config.py b/src/reverse_api/config.py
index fff483f..b5d5772 100644
--- a/src/reverse_api/config.py
+++ b/src/reverse_api/config.py
@@ -8,6 +8,7 @@ DEFAULT_CONFIG = {
"agent_provider": "auto", # "auto" (default, MCP-based browser + engineering), "browser-use", or "stagehand"
"browser_use_model": "bu-llm", # "bu-llm" or "{provider}/{model_name}" (e.g. "openai/gpt-5-mini")
"claude_code_model": "claude-sonnet-4-5",
+ "collector_model": "claude-sonnet-4-5", # Model for collector mode
"opencode_model": "claude-sonnet-4-5",
"opencode_provider": "anthropic",
"output_dir": None, # None means use ~/.reverse-api/runs
diff --git a/src/reverse_api/tui.py b/src/reverse_api/tui.py
index f7fbc55..c0a792b 100644
--- a/src/reverse_api/tui.py
+++ b/src/reverse_api/tui.py
@@ -17,6 +17,7 @@ MODE_COLORS = {
"manual": "#ff5f50", # Coral Red
"engineer": "#5f9fff", # Blue
"agent": "#50ff9f", # Green/Cyan
+ "collector": "#ffd700", # Gold
}
diff --git a/src/reverse_api/utils.py b/src/reverse_api/utils.py
index 9683f59..5ed0508 100644
--- a/src/reverse_api/utils.py
+++ b/src/reverse_api/utils.py
@@ -29,6 +29,16 @@ def generate_folder_name(prompt: str, sdk: str = None, session_id: str = None) -
sdk = "claude"
try:
+ # Check if event loop is already running (e.g., called from async context)
+ try:
+ loop = asyncio.get_running_loop()
+ except RuntimeError:
+ loop = None
+
+ if loop is not None:
+ # Already in async context, use fallback to avoid nested asyncio.run
+ return _slugify(prompt)
+
if sdk == "opencode":
return asyncio.run(_generate_folder_name_opencode_async(prompt, session_id))
else:
@@ -420,3 +430,17 @@ def get_messages_path(run_id: str, output_dir: str | None = None) -> Path:
def get_timestamp() -> str:
"""Get current timestamp in ISO format."""
return datetime.now().isoformat()
+
+
+def get_collected_dir(folder_name: str) -> Path:
+ """Get ./collected/{folder_name}/ directory for collector mode output.
+
+ Args:
+ folder_name: Name of the collection folder
+
+ Returns:
+ Path to the collection output directory
+ """
+ path = Path.cwd() / "collected" / folder_name
+ path.mkdir(parents=True, exist_ok=True)
+ return path
diff --git a/uv.lock b/uv.lock
index 33eba12..56543de 100644
--- a/uv.lock
+++ b/uv.lock
@@ -146,6 +146,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/3a/2a/7cc015f5b9f5db42b7d48157e23356022889fc354a2813c15934b7cb5c0e/attrs-25.4.0-py3-none-any.whl", hash = "sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373", size = 67615, upload-time = "2025-10-06T13:54:43.17Z" },
]
+[[package]]
+name = "beautifulsoup4"
+version = "4.14.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "soupsieve" },
+ { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c3/b0/1c6a16426d389813b48d95e26898aff79abbde42ad353958ad95cc8c9b21/beautifulsoup4-4.14.3.tar.gz", hash = "sha256:6292b1c5186d356bba669ef9f7f051757099565ad9ada5dd630bd9de5fa7fb86", size = 627737, upload-time = "2025-11-30T15:08:26.084Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/1a/39/47f9197bdd44df24d67ac8893641e16f386c984a0619ef2ee4c51fbbc019/beautifulsoup4-4.14.3-py3-none-any.whl", hash = "sha256:0918bfe44902e6ad8d57732ba310582e98da931428d231a5ecb9e7c703a735bb", size = 107721, upload-time = "2025-11-30T15:08:24.087Z" },
+]
+
[[package]]
name = "brotli"
version = "1.2.0"
@@ -1090,6 +1103,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" },
]
+[[package]]
+name = "markdownify"
+version = "1.2.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "beautifulsoup4" },
+ { name = "six" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/3f/bc/c8c8eea5335341306b0fa7e1cb33c5e1c8d24ef70ddd684da65f41c49c92/markdownify-1.2.2.tar.gz", hash = "sha256:b274f1b5943180b031b699b199cbaeb1e2ac938b75851849a31fd0c3d6603d09", size = 18816, upload-time = "2025-11-16T19:21:18.565Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/43/ce/f1e3e9d959db134cedf06825fae8d5b294bd368aacdd0831a3975b7c4d55/markdownify-1.2.2-py3-none-any.whl", hash = "sha256:3f02d3cc52714084d6e589f70397b6fc9f2f3a8531481bf35e8cc39f975e186a", size = 15724, upload-time = "2025-11-16T19:21:17.622Z" },
+]
+
[[package]]
name = "markupsafe"
version = "3.0.3"
@@ -1989,6 +2015,10 @@ dependencies = [
agent = [
{ name = "stagehand" },
]
+collector = [
+ { name = "beautifulsoup4" },
+ { name = "markdownify" },
+]
pricing = [
{ name = "litellm" },
]
@@ -2003,10 +2033,12 @@ dev = [
requires-dist = [
{ name = "aiohttp", specifier = ">=3.12.15" },
{ name = "anthropic", specifier = ">=0.40.0" },
+ { name = "beautifulsoup4", marker = "extra == 'collector'", specifier = ">=4.12.0" },
{ name = "brotli", specifier = ">=1.2.0" },
{ name = "claude-agent-sdk", specifier = ">=0.1.0" },
{ name = "click", specifier = ">=8.1.0" },
{ name = "litellm", marker = "extra == 'pricing'", specifier = ">=1.0.0" },
+ { name = "markdownify", marker = "extra == 'collector'", specifier = ">=0.12.0" },
{ name = "playwright", specifier = ">=1.40.0" },
{ name = "playwright-stealth", specifier = ">=1.0.0" },
{ name = "questionary", specifier = ">=2.0.0" },
@@ -2015,7 +2047,7 @@ requires-dist = [
{ name = "stagehand", marker = "extra == 'agent'" },
{ name = "watchdog", specifier = ">=3.0.0" },
]
-provides-extras = ["agent", "pricing"]
+provides-extras = ["agent", "pricing", "collector"]
[package.metadata.requires-dev]
dev = [
@@ -2191,6 +2223,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" },
]
+[[package]]
+name = "six"
+version = "1.17.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" },
+]
+
[[package]]
name = "sniffio"
version = "1.3.1"
@@ -2200,6 +2241,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" },
]
+[[package]]
+name = "soupsieve"
+version = "2.8.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/89/23/adf3796d740536d63a6fbda113d07e60c734b6ed5d3058d1e47fc0495e47/soupsieve-2.8.1.tar.gz", hash = "sha256:4cf733bc50fa805f5df4b8ef4740fc0e0fa6218cf3006269afd3f9d6d80fd350", size = 117856, upload-time = "2025-12-18T13:50:34.655Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/48/f3/b67d6ea49ca9154453b6d70b34ea22f3996b9fa55da105a79d8732227adc/soupsieve-2.8.1-py3-none-any.whl", hash = "sha256:a11fe2a6f3d76ab3cf2de04eb339c1be5b506a8a47f2ceb6d139803177f85434", size = 36710, upload-time = "2025-12-18T13:50:33.267Z" },
+]
+
[[package]]
name = "sse-starlette"
version = "3.0.4"