The agent that turns websites into APIs!
feat: add GitHub Copilot SDK as third SDK option (#42)
Claudec4266a9539af
diff --git a/pyproject.toml b/pyproject.tomlindex 04541e0..3705578 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -48,6 +48,9 @@ collector = ["markdownify>=0.12.0","beautifulsoup4>=4.12.0",]+copilot = [+ "github-copilot-sdk>=0.1.0",+][project.urls]Homepage = "https://github.com/kalil0321/reverse-api-engineer"diff --git a/src/reverse_api/auto_engineer.py b/src/reverse_api/auto_engineer.pyindex 179f69f..66eb770 100644--- a/src/reverse_api/auto_engineer.py+++ b/src/reverse_api/auto_engineer.py@@ -577,3 +577,137 @@ class OpenCodeAutoEngineer(OpenCodeEngineer):debug_log(f"Cleaned up MCP server: {self.mcp_name}")except Exception:pass # Ignore cleanup errors+++class CopilotAutoEngineer:+ """Auto mode using Copilot SDK: LLM controls browser via MCP while reverse engineering.++ Uses composition rather than inheritance since CopilotEngineer requires lazy imports.+ Delegates to CopilotEngineer for the core logic and adds MCP browser integration.+ """++ def __init__(+ self,+ run_id: str,+ prompt: str,+ copilot_model: str | None = None,+ output_dir: str | None = None,+ **kwargs: Any,+ ):+ from .copilot_engineer import CopilotEngineer++ har_dir = get_har_dir(run_id, output_dir)+ har_path = har_dir / "recording.har"++ self._engineer = CopilotEngineer(+ run_id=run_id,+ har_path=har_path,+ prompt=prompt,+ copilot_model=copilot_model,+ output_dir=output_dir,+ **kwargs,+ )+ self.mcp_run_id = run_id++ def start_sync(self) -> None:+ self._engineer.start_sync()++ def stop_sync(self) -> None:+ self._engineer.stop_sync()++ async def analyze_and_generate(self) -> dict[str, Any] | None:+ """Run auto mode with Copilot SDK and MCP browser integration."""+ try:+ from copilot import CopilotClient+ except ImportError:+ self._engineer.ui.error("GitHub Copilot SDK not installed. Install with: uv pip install 'reverse-api-engineer[copilot]'")+ return None++ eng = self._engineer+ eng.ui.header(eng.run_id, eng.prompt, eng.copilot_model, eng.sdk)+ eng.ui.start_analysis()++ auto_prompt = ClaudeAutoEngineer._build_auto_prompt(eng)+ eng.message_store.save_prompt(auto_prompt)++ done_event = asyncio.Event()+ accumulated_text: list[str] = []++ def on_event(event: Any) -> None:+ event_type = event.type.value if hasattr(event.type, "value") else str(event.type)++ if event_type == "assistant.message_delta":+ delta = ""+ if hasattr(event, "data") and hasattr(event.data, "delta_content"):+ delta = event.data.delta_content or ""+ if delta:+ accumulated_text.append(delta)+ eng.ui.thinking(delta)+ elif event_type == "assistant.message":+ if hasattr(event, "data") and hasattr(event.data, "usage"):+ usage = event.data.usage+ if isinstance(usage, dict):+ eng.usage_metadata["input_tokens"] = usage.get("prompt_tokens", 0)+ eng.usage_metadata["output_tokens"] = usage.get("completion_tokens", 0)+ elif event_type == "session.idle":+ done_event.set()++ try:+ client = CopilotClient(+ {+ "auto_start": True,+ "use_logged_in_user": True,+ }+ )+ await client.start()++ mcp_config = {+ "type": "stdio",+ "command": "npx",+ "args": [+ "rae-playwright-mcp@latest",+ "run-mcp-server",+ "--run-id",+ self.mcp_run_id,+ ],+ }++ session = await client.create_session(+ {+ "model": eng.copilot_model,+ "streaming": True,+ "infinite_sessions": {"enabled": True},+ "mcp_servers": {"playwright": mcp_config},+ }+ )++ session.on(on_event)+ await session.send({"prompt": auto_prompt})+ await done_event.wait()++ if accumulated_text:+ eng.message_store.save_thinking("".join(accumulated_text))++ script_path = str(eng.scripts_dir / eng._get_client_filename())+ local_path = str(eng.local_scripts_dir / eng._get_client_filename()) if eng.local_scripts_dir else None+ eng.ui.success(script_path, local_path)+ eng.usage_metadata["estimated_cost_usd"] = 0.0++ result: dict[str, Any] = {+ "script_path": script_path,+ "usage": eng.usage_metadata,+ }+ eng.message_store.save_result(result)+ return result++ except Exception as e:+ error_msg = str(e)+ eng.ui.error(error_msg)+ eng.message_store.save_error(error_msg)++ if "buffer size" in error_msg.lower() or "1048576" in error_msg or "exceeded maximum buffer" in error_msg.lower():+ eng.ui.console.print("\n[yellow]Screenshot too large (exceeds 1MB limit)[/yellow]")+ eng.ui.console.print("[dim]Tip: The AI should take element-specific screenshots instead of full-page screenshots.[/dim]")+ else:+ eng.ui.console.print("\n[dim]Make sure GitHub Copilot CLI is installed and you are logged in: gh auth login[/dim]")+ return Nonediff --git a/src/reverse_api/cli.py b/src/reverse_api/cli.pyindex 688cf06..67be552 100644--- a/src/reverse_api/cli.py+++ b/src/reverse_api/cli.py@@ -357,6 +357,8 @@ def repl_loop():sdk = config_manager.get("sdk", "claude")if sdk == "opencode":model = config_manager.get("opencode_model", "claude-opus-4-6")+ elif sdk == "copilot":+ model = config_manager.get("copilot_model", "gpt-5")else:model = config_manager.get("claude_code_model", "claude-sonnet-4-6")@@ -553,6 +555,7 @@ def handle_settings(mode_color=THEME_PRIMARY):Choice(title="Agent Provider", value="agent_provider"),Choice(title="Browser-Use Model", value="browser_use_model"),Choice(title="Claude Code Model", value="claude_code_model"),+ Choice(title="Copilot Model", value="copilot_model"),Choice(title="OpenCode Model", value="opencode_model"),Choice(title="OpenCode Provider", value="opencode_provider"),Choice(title="Output Directory", value="output_dir"),@@ -602,8 +605,9 @@ def handle_settings(mode_color=THEME_PRIMARY):elif action == "sdk":sdk_choices = [- Choice(title="opencode", value="opencode"),Choice(title="claude", value="claude"),+ Choice(title="copilot", value="copilot"),+ Choice(title="opencode", value="opencode"),Choice(title="back", value="back"),]sdk = questionary.select(@@ -690,6 +694,28 @@ def handle_settings(mode_color=THEME_PRIMARY):config_manager.set("opencode_provider", new_provider)console.print(f" [dim]updated[/dim] opencode provider: {new_provider}\n")+ elif action == "copilot_model":+ current = config_manager.get("copilot_model", "gpt-5")+ new_model = questionary.text(+ " > copilot model",+ default=current or "gpt-5",+ instruction="(e.g., 'gpt-5', 'gpt-4.1')",+ qmark="",+ style=questionary.Style(+ [+ ("question", f"fg:{THEME_SECONDARY}"),+ ("instruction", f"fg:{THEME_DIM} italic"),+ ]+ ),+ ).ask()+ if new_model is not None:+ new_model = new_model.strip()+ if not new_model:+ console.print(" [yellow]error:[/yellow] copilot model cannot be empty\n")+ else:+ config_manager.set("copilot_model", new_model)+ console.print(f" [dim]updated[/dim] copilot model: {new_model}\n")+elif action == "opencode_model":current = config_manager.get("opencode_model", "claude-opus-4-6")new_model = questionary.text(@@ -1464,6 +1490,18 @@ def run_auto_capture(prompt=None, url=None, model=None, output_dir=None):sdk=sdk,output_language=output_language,)+ elif sdk == "copilot":+ from .auto_engineer import CopilotAutoEngineer++ engineer = CopilotAutoEngineer(+ run_id=run_id,+ prompt=prompt,+ copilot_model=config_manager.get("copilot_model", "gpt-5"),+ output_dir=output_dir,+ enable_sync=config_manager.get("real_time_sync", False),+ sdk=sdk,+ output_language=output_language,+ )else:from .auto_engineer import ClaudeAutoEngineer@@ -1610,6 +1648,21 @@ def run_engineer(output_language=output_language,output_mode=output_mode,)+ elif sdk == "copilot":+ result = run_reverse_engineering(+ run_id=run_id,+ har_path=har_path,+ prompt=prompt,+ model=model,+ output_dir=output_dir,+ sdk=sdk,+ copilot_model=config_manager.get("copilot_model", "gpt-5"),+ enable_sync=enable_sync,+ additional_instructions=additional_instructions,+ is_fresh=is_fresh,+ output_language=output_language,+ output_mode=output_mode,+ )else:result = run_reverse_engineering(run_id=run_id,diff --git a/src/reverse_api/config.py b/src/reverse_api/config.pyindex 92387a8..3d6cbe0 100644--- a/src/reverse_api/config.py+++ b/src/reverse_api/config.py@@ -9,12 +9,13 @@ DEFAULT_CONFIG = {"browser_use_model": "bu-llm", # "bu-llm" or "{provider}/{model_name}" (e.g. "openai/gpt-5-mini")"claude_code_model": "claude-sonnet-4-6","collector_model": "claude-sonnet-4-6", # Model for collector mode+ "copilot_model": "gpt-5", # Model for Copilot SDK sessions"opencode_model": "claude-opus-4-6","opencode_provider": "anthropic","output_dir": None, # None means use ~/.reverse-api/runs"output_language": "python", # "python", "javascript", or "typescript""real_time_sync": True, # Enable real-time file sync during engineering- "sdk": "claude", # "opencode" or "claude"+ "sdk": "claude", # "claude", "opencode", or "copilot""stagehand_model": "openai/computer-use-preview-2025-03-11", # "{provider}/{model_name}" format}diff --git a/src/reverse_api/copilot_engineer.py b/src/reverse_api/copilot_engineer.pynew file mode 100644index 0000000..7c1e546--- /dev/null+++ b/src/reverse_api/copilot_engineer.py@@ -0,0 +1,245 @@+"""GitHub Copilot SDK implementation for reverse engineering."""++import asyncio+from typing import Any++import questionary++from .base_engineer import BaseEngineer+from .tui import THEME_PRIMARY, THEME_SECONDARY+++class CopilotEngineer(BaseEngineer):+ """Uses GitHub Copilot SDK to analyze HAR files and generate API scripts."""++ def __init__(+ self,+ run_id: str,+ har_path: Any,+ prompt: str,+ copilot_model: str | None = None,+ **kwargs: Any,+ ):+ super().__init__(run_id=run_id, har_path=har_path, prompt=prompt, **kwargs)+ self.copilot_model = copilot_model or "gpt-5"++ def _build_ask_user_tool(self) -> Any:+ """Build the ask_user_question custom tool for Copilot sessions."""+ try:+ from copilot import define_tool+ except ImportError:+ raise ImportError("GitHub Copilot SDK not installed. Install with: uv pip install 'reverse-api-engineer[copilot]'") from None++ from pydantic import BaseModel, Field++ class QuestionOption(BaseModel):+ label: str = Field(description="Display text for this option")+ description: str = Field(default="", description="Explanation of what this option means")++ class Question(BaseModel):+ question: str = Field(description="The question to ask the user")+ header: str = Field(default="", description="Short label for context")+ options: list[QuestionOption] = Field(default_factory=list, description="Available choices")+ multiSelect: bool = Field(default=False, description="Allow multiple selections")++ class AskUserParams(BaseModel):+ questions: list[Question] = Field(description="Questions to ask the user")++ ui = self.ui++ @define_tool(description="Ask the user a clarifying question. Use when you need user input to proceed.") # type: ignore[misc]+ async def ask_user_question(params: AskUserParams) -> str:+ answers: dict[str, str] = {}++ ui.console.print()+ ui.console.print(f" [{THEME_PRIMARY}]?[/{THEME_PRIMARY}] [bold white]Agent Question[/bold white]")+ ui.console.print()++ for q in params.questions:+ if not q.question:+ continue++ if q.header:+ ui.console.print(f" [dim]{q.header}[/dim]")++ try:+ if q.multiSelect:+ choices = [f"{opt.label} - {opt.description}" if opt.description else opt.label for opt in q.options]+ if choices:+ selected = await questionary.checkbox(+ f" > {q.question}",+ choices=choices,+ qmark="",+ style=questionary.Style(+ [+ ("pointer", f"fg:{THEME_PRIMARY} bold"),+ ("highlighted", f"fg:{THEME_PRIMARY} bold"),+ ("selected", f"fg:{THEME_PRIMARY}"),+ ]+ ),+ ).ask_async()++ if selected is None:+ raise KeyboardInterrupt++ labels = [s.split(" - ")[0] if " - " in s else s for s in selected]+ answers[q.question] = ", ".join(labels)+ else:+ answer = await questionary.text(+ f" > {q.question}",+ qmark="",+ style=questionary.Style([("question", f"fg:{THEME_SECONDARY}")]),+ ).ask_async()+ if answer is None:+ raise KeyboardInterrupt+ answers[q.question] = answer.strip()+ else:+ choices = [f"{opt.label} - {opt.description}" if opt.description else opt.label for opt in q.options]+ if choices:+ answer = await questionary.select(+ f" > {q.question}",+ choices=choices,+ qmark="",+ style=questionary.Style(+ [+ ("pointer", f"fg:{THEME_PRIMARY} bold"),+ ("highlighted", f"fg:{THEME_PRIMARY} bold"),+ ]+ ),+ ).ask_async()++ if answer is None:+ raise KeyboardInterrupt++ label = answer.split(" - ")[0] if " - " in answer else answer+ answers[q.question] = label+ else:+ answer = await questionary.text(+ f" > {q.question}",+ qmark="",+ style=questionary.Style([("question", f"fg:{THEME_SECONDARY}")]),+ ).ask_async()+ if answer is None:+ raise KeyboardInterrupt+ answers[q.question] = answer.strip()++ ui.console.print(f" [dim]→ {answers[q.question]}[/dim]")++ except KeyboardInterrupt:+ ui.console.print(" [dim]User cancelled question[/dim]")+ answers[q.question] = ""++ ui.console.print()++ # Return answers as formatted string+ return "\n".join(f"{k}: {v}" for k, v in answers.items())++ return ask_user_question++ async def analyze_and_generate(self) -> dict[str, Any] | None:+ """Run the reverse engineering analysis with GitHub Copilot."""+ try:+ from copilot import CopilotClient+ except ImportError:+ self.ui.error("GitHub Copilot SDK not installed. Install with: uv pip install 'reverse-api-engineer[copilot]'")+ return None++ self.ui.header(self.run_id, self.prompt, self.copilot_model, self.sdk)+ self.ui.start_analysis()++ prompt = self._build_analysis_prompt()+ self.message_store.save_prompt(prompt)++ done_event = asyncio.Event()+ accumulated_text: list[str] = []++ def on_event(event: Any) -> None:+ event_type = event.type.value if hasattr(event.type, "value") else str(event.type)++ if event_type == "assistant.message_delta":+ delta = ""+ if hasattr(event, "data") and hasattr(event.data, "delta_content"):+ delta = event.data.delta_content or ""+ if delta:+ accumulated_text.append(delta)+ self.ui.thinking(delta)++ elif event_type == "assistant.message":+ # Extract usage if available+ if hasattr(event, "data") and hasattr(event.data, "usage"):+ usage = event.data.usage+ if isinstance(usage, dict):+ self.usage_metadata["input_tokens"] = usage.get("prompt_tokens", 0)+ self.usage_metadata["output_tokens"] = usage.get("completion_tokens", 0)++ elif event_type == "session.idle":+ done_event.set()++ elif event_type == "session.compaction_start":+ self.ui.console.print(" [dim]session compacting...[/dim]")++ elif event_type == "session.compaction_complete":+ self.ui.console.print(" [dim]session compaction complete[/dim]")++ try:+ ask_user_tool = self._build_ask_user_tool()++ client = CopilotClient(+ {+ "auto_start": True,+ "use_logged_in_user": True,+ }+ )+ await client.start()++ session = await client.create_session(+ {+ "model": self.copilot_model,+ "streaming": True,+ "infinite_sessions": {"enabled": True},+ "tools": [ask_user_tool],+ }+ )++ session.on(on_event)++ await session.send({"prompt": prompt})++ # Wait for session to complete+ await done_event.wait()++ # Save accumulated thinking text+ if accumulated_text:+ self.message_store.save_thinking("".join(accumulated_text))++ # Build result+ script_path = str(self.scripts_dir / self._get_client_filename())+ local_path = str(self.local_scripts_dir / self._get_client_filename()) if self.local_scripts_dir else None+ self.ui.success(script_path, local_path)++ # GitHub subscription: cost is 0 (included in Copilot subscription)+ self.usage_metadata["estimated_cost_usd"] = 0.0++ # Display usage if available+ input_tokens = self.usage_metadata.get("input_tokens", 0)+ output_tokens = self.usage_metadata.get("output_tokens", 0)+ if input_tokens > 0 or output_tokens > 0:+ self.ui.console.print(" [dim]Usage:[/dim]")+ if input_tokens > 0:+ self.ui.console.print(f" [dim] input: {input_tokens:,} tokens[/dim]")+ if output_tokens > 0:+ self.ui.console.print(f" [dim] output: {output_tokens:,} tokens[/dim]")+ self.ui.console.print(" [dim] cost: $0.00 (Copilot subscription)[/dim]")++ result: dict[str, Any] = {+ "script_path": script_path,+ "usage": self.usage_metadata,+ }+ self.message_store.save_result(result)+ return result++ except Exception as e:+ self.ui.error(str(e))+ self.message_store.save_error(str(e))+ self.ui.console.print("\n[dim]Make sure GitHub Copilot CLI is installed and you are logged in: gh auth login[/dim]")+ return Nonediff --git a/src/reverse_api/engineer.py b/src/reverse_api/engineer.pyindex 60ecc03..6ab8374 100644--- a/src/reverse_api/engineer.py+++ b/src/reverse_api/engineer.py@@ -28,9 +28,7 @@ logging.getLogger("claude_agent_sdk._internal.transport.subprocess_cli").setLeveclass ClaudeEngineer(BaseEngineer):"""Uses Claude Agent SDK to analyze HAR files and generate Python API scripts."""- async def _handle_ask_user_question(- self, tool_name: str, input_params: dict[str, Any]- ) -> dict[str, Any]:+ async def _handle_ask_user_question(self, tool_name: str, input_params: dict[str, Any]) -> dict[str, Any]:"""Handle AskUserQuestion tool by prompting the user interactively."""if tool_name != "AskUserQuestion":# Default allow for other tools in acceptEdits mode@@ -41,9 +39,7 @@ class ClaudeEngineer(BaseEngineer):# Display headerself.ui.console.print()- self.ui.console.print(- f" [{THEME_PRIMARY}]?[/{THEME_PRIMARY}] [bold white]Agent Question[/bold white]"- )+ self.ui.console.print(f" [{THEME_PRIMARY}]?[/{THEME_PRIMARY}] [bold white]Agent Question[/bold white]")self.ui.console.print()for q in questions:@@ -63,9 +59,7 @@ class ClaudeEngineer(BaseEngineer):if multi_select:# Multi-select questionchoices = [- f"{opt.get('label', '')} - {opt.get('description', '')}"- if opt.get("description")- else opt.get("label", "")+ f"{opt.get('label', '')} - {opt.get('description', '')}" if opt.get("description") else opt.get("label", "")for opt in options]if choices:@@ -86,10 +80,7 @@ class ClaudeEngineer(BaseEngineer):raise KeyboardInterrupt# Extract just the labels (before the " - " separator)- labels = [- s.split(" - ")[0] if " - " in s else s- for s in selected- ]+ labels = [s.split(" - ")[0] if " - " in s else s for s in selected]answers[question_text] = ", ".join(labels)else:# Text input fallback@@ -105,9 +96,7 @@ class ClaudeEngineer(BaseEngineer):else:# Single select questionchoices = [- f"{opt.get('label', '')} - {opt.get('description', '')}"- if opt.get("description")- else opt.get("label", "")+ f"{opt.get('label', '')} - {opt.get('description', '')}" if opt.get("description") else opt.get("label", "")for opt in options]if choices:@@ -284,6 +273,7 @@ def run_reverse_engineering(sdk: str = "claude",opencode_provider: str | None = None,opencode_model: str | None = None,+ copilot_model: str | None = None,enable_sync: bool = False,is_fresh: bool = False,output_language: str = "python",@@ -292,9 +282,10 @@ def run_reverse_engineering("""Run reverse engineering with the specified SDK.Args:- sdk: "opencode" or "claude" - determines which SDK to use+ sdk: "claude", "opencode", or "copilot" - determines which SDK to useopencode_provider: Provider ID for OpenCode (e.g., "anthropic")opencode_model: Model ID for OpenCode (e.g., "claude-sonnet-4-6")+ copilot_model: Model ID for Copilot (e.g., "gpt-5")enable_sync: Enable real-time file syncing during engineeringis_fresh: Whether to start fresh (ignore previous scripts)output_language: Target language - "python", "javascript", or "typescript"@@ -319,6 +310,24 @@ def run_reverse_engineering(output_language=output_language,output_mode=output_mode,)+ elif sdk == "copilot":+ from .copilot_engineer import CopilotEngineer++ engineer = CopilotEngineer(+ run_id=run_id,+ har_path=har_path,+ prompt=prompt,+ model=model,+ additional_instructions=additional_instructions,+ output_dir=output_dir,+ verbose=verbose,+ enable_sync=enable_sync,+ sdk=sdk,+ is_fresh=is_fresh,+ output_language=output_language,+ output_mode=output_mode,+ copilot_model=copilot_model,+ )else:engineer = ClaudeEngineer(run_id=run_id,diff --git a/src/reverse_api/pricing.py b/src/reverse_api/pricing.pyindex 00ee661..6c05219 100644--- a/src/reverse_api/pricing.py+++ b/src/reverse_api/pricing.py@@ -92,6 +92,28 @@ MODEL_PRICING = {"cache_read": 0.50,"reasoning": 25.00,},+ # GPT models (for Copilot SDK - cost is $0 with GitHub subscription)+ "gpt-5": {+ "input": 2.00,+ "output": 8.00,+ "cache_creation": 0,+ "cache_read": 0,+ "reasoning": 8.00,+ },+ "gpt-4.1": {+ "input": 2.00,+ "output": 8.00,+ "cache_creation": 0,+ "cache_read": 0,+ "reasoning": 8.00,+ },+ "gpt-4.1-mini": {+ "input": 0.40,+ "output": 1.60,+ "cache_creation": 0,+ "cache_read": 0,+ "reasoning": 1.60,+ },}