The agent that turns websites into APIs!
import csv
import json
import logging
from datetime import datetime
from pathlib import Path
from typing import Any
from claude_agent_sdk import (
AssistantMessage,
ClaudeAgentOptions,
ClaudeSDKClient,
ResultMessage,
TextBlock,
ToolResultBlock,
ToolUseBlock,
)
from .collector_ui import CollectorUI
from .messages import MessageStore
from .pricing import calculate_cost
from .utils import build_sdk_env, generate_folder_name, get_collected_dir
# Suppress claude_agent_sdk logs
logging.getLogger("claude_agent_sdk").setLevel(logging.WARNING)
logging.getLogger("claude_agent_sdk._internal.transport.subprocess_cli").setLevel(logging.WARNING)
class Collector:
"""AI-powered web data collector using Claude Agent SDK.
Uses Claude's built-in tools (WebFetch, Write) to collect data.
The agent writes collected items to a JSONL file which is then
processed into JSON + CSV output.
"""
def __init__(
self,
run_id: str,
prompt: str,
model: str,
output_dir: str | None = None,
) -> None:
"""Initialize collector.
Args:
run_id: Unique run identifier
prompt: Natural language prompt describing data to collect
model: Claude model to use
output_dir: Optional custom output directory
"""
self.run_id = run_id
self.prompt = prompt
self.model = model
self.output_dir = output_dir
self.ui = CollectorUI()
self.message_store = MessageStore(run_id, output_dir)
self._folder_name: str | None = None
self._collected_dir: Path | None = None
self.usage_metadata: dict[str, Any] = {}
async def run(self) -> dict[str, Any] | None:
"""Run the collector agent loop.
Returns:
Result dict with output_path and collected_items, or None on error
"""
self.ui.header(self.run_id, self.prompt, self.model, mode="collector")
self.ui.start_collecting()
# A punctuation-only prompt can slugify to an empty string, which
# get_collected_dir rejects — fall back to the (validated) run_id
self._folder_name = generate_folder_name(self.prompt) or self.run_id
self._collected_dir = get_collected_dir(self._folder_name)
self.items_path = self._collected_dir / "items.jsonl"
self._system_prompt, self._user_message = self._build_prompts()
self.message_store.save_prompt(self._user_message)
result = await self._agent_loop()
if result and not result.get("error"):
return self._finalize_collection()
return result
def _build_prompts(self) -> tuple[str, str]:
"""Build (system_prompt, user_message) for the collector agent."""
from .prompts import load
system_prompt = load("collector/system")
user_message = load(
"collector/user",
prompt=self.prompt,
items_path=str(self.items_path),
)
return system_prompt, user_message
async def _agent_loop(self) -> dict[str, Any] | None:
"""Run Claude Agent SDK loop."""
options = ClaudeAgentOptions(
system_prompt=self._system_prompt,
permission_mode="bypassPermissions",
allowed_tools=[
"Read",
"Write",
"Bash",
"WebFetch",
"WebSearch",
],
cwd=str(self._collected_dir),
model=self.model,
env=build_sdk_env(),
)
try:
async with ClaudeSDKClient(options=options) as client:
await client.query(self._user_message)
async for message in client.receive_response():
# Track usage
if hasattr(message, "usage"):
usage = message.usage
if isinstance(usage, dict):
for key, value in usage.items():
if isinstance(key, str) and isinstance(value, (int, float)):
current = self.usage_metadata.get(key, 0)
if isinstance(current, (int, float)):
self.usage_metadata[key] = current + value
if isinstance(message, AssistantMessage):
last_tool_name = None
for block in message.content:
if isinstance(block, ToolUseBlock):
last_tool_name = block.name
tool_input = block.input if isinstance(block.input, dict) else {}
self.ui.tool_start(block.name, tool_input)
self.message_store.save_tool_start(block.name, tool_input)
# Track item saves
if block.name == "Write":
file_path = tool_input.get("file_path", "")
if "items.jsonl" in file_path:
content = tool_input.get("content", "")
self.ui.item_saved(content[:50])
elif isinstance(block, ToolResultBlock):
is_error = block.is_error if block.is_error else False
tool_name = last_tool_name or "Tool"
self.ui.tool_result(tool_name, is_error)
self.message_store.save_tool_result(tool_name, is_error)
elif isinstance(block, TextBlock):
self.ui.thinking(block.text)
self.message_store.save_thinking(block.text)
elif isinstance(message, ResultMessage):
if message.is_error:
self.ui.error(message.result or "Unknown error")
self.message_store.save_error(message.result or "Unknown error")
return {"error": message.result}
else:
return {"success": True}
except Exception as e:
error_msg = str(e)
self.ui.error(error_msg)
self.message_store.save_error(error_msg)
return {"error": error_msg}
return {"success": True}
def _finalize_collection(self) -> dict[str, Any]:
"""Post-process collected items and export to JSON/CSV."""
if not self._collected_dir:
return {"error": "No collection directory"}
items_path = self._collected_dir / "items.jsonl"
# Parse collected items from JSONL
collected_items: list[dict[str, Any]] = []
sources: set[str] = set()
if items_path.exists():
with open(items_path, encoding="utf-8") as f:
for line in f:
line = line.strip()
if line:
try:
item = json.loads(line)
if isinstance(item, dict):
collected_items.append(item)
# Track sources
if "source_url" in item:
sources.add(item["source_url"])
elif "url" in item:
sources.add(item["url"])
except json.JSONDecodeError:
continue
if not collected_items:
self.ui.error("No items were collected. Check if the agent saved data correctly.")
return {"error": "No items collected", "items": []}
# Export JSON
json_path = self._collected_dir / "data.json"
with open(json_path, "w", encoding="utf-8") as f:
json.dump(collected_items, f, indent=2)
# Export CSV
csv_path = self._collected_dir / "data.csv"
self._export_csv(csv_path, collected_items)
# Update README with final stats
readme_path = self._collected_dir / "README.md"
self._export_readme(readme_path, collected_items, sources)
# Calculate cost
if self.usage_metadata:
input_tokens = self.usage_metadata.get("input_tokens", 0)
output_tokens = self.usage_metadata.get("output_tokens", 0)
cache_creation = self.usage_metadata.get("cache_creation_input_tokens", 0)
cache_read = self.usage_metadata.get("cache_read_input_tokens", 0)
cost = calculate_cost(
model_id=self.model,
input_tokens=input_tokens,
output_tokens=output_tokens,
cache_creation_tokens=cache_creation,
cache_read_tokens=cache_read,
)
self.usage_metadata["estimated_cost_usd"] = cost
# Show completion
self.ui.collection_complete(len(collected_items), str(self._collected_dir))
if self.usage_metadata:
self.ui.usage_summary(self.usage_metadata)
result = {
"output_path": str(self._collected_dir),
"items_collected": len(collected_items),
"files": {
"json": str(json_path),
"csv": str(csv_path),
"readme": str(readme_path),
},
"usage": self.usage_metadata,
}
self.message_store.save_result(result)
return result
def _export_csv(self, csv_path: Path, items: list[dict[str, Any]]) -> None:
"""Export items to CSV with auto-flattened columns."""
if not items:
return
# Collect all unique keys
all_keys: set[str] = set()
for item in items:
all_keys.update(item.keys())
# Sort keys for consistent column order
fieldnames = sorted(all_keys)
with open(csv_path, "w", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=fieldnames)
writer.writeheader()
for item in items:
# Convert non-string values to strings, filter out keys not in fieldnames
row = {k: str(v) if v is not None else "" for k, v in item.items() if k in fieldnames}
writer.writerow(row)
def _export_readme(self, readme_path: Path, items: list[dict[str, Any]], sources: set[str]) -> None:
"""Generate README with collection metadata."""
folder_name = self._folder_name or "collection"
readme_content = f"""# {folder_name.replace("_", " ").title()}
## Query
{self.prompt}
## Metadata
- **Run ID**: {self.run_id}
- **Items Collected**: {len(items)}
- **Sources**: {len(sources)}
- **Collected At**: {datetime.now().isoformat()}
## Files
- `data.json` - All collected items in JSON format
- `data.csv` - All collected items in CSV format
## Schema
"""
# Add field schema
if items:
all_keys: set[str] = set()
for item in items:
all_keys.update(item.keys())
for key in sorted(all_keys):
readme_content += f"- `{key}`\n"
with open(readme_path, "w", encoding="utf-8") as f:
f.write(readme_content)