Skip to content

The agent that turns websites into APIs!

feat: add GitHub Copilot SDK as third SDK option (#42)

Claudec4266a9539af
diff --git a/pyproject.toml b/pyproject.toml
index 04541e0..3705578 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -48,6 +48,9 @@ collector = [
"markdownify>=0.12.0",
"beautifulsoup4>=4.12.0",
]
+copilot = [
+ "github-copilot-sdk>=0.1.0",
+]
[project.urls]
Homepage = "https://github.com/kalil0321/reverse-api-engineer"
diff --git a/src/reverse_api/auto_engineer.py b/src/reverse_api/auto_engineer.py
index 179f69f..66eb770 100644
--- a/src/reverse_api/auto_engineer.py
+++ b/src/reverse_api/auto_engineer.py
@@ -577,3 +577,137 @@ class OpenCodeAutoEngineer(OpenCodeEngineer):
debug_log(f"Cleaned up MCP server: {self.mcp_name}")
except Exception:
pass # Ignore cleanup errors
+
+
+class CopilotAutoEngineer:
+ """Auto mode using Copilot SDK: LLM controls browser via MCP while reverse engineering.
+
+ Uses composition rather than inheritance since CopilotEngineer requires lazy imports.
+ Delegates to CopilotEngineer for the core logic and adds MCP browser integration.
+ """
+
+ def __init__(
+ self,
+ run_id: str,
+ prompt: str,
+ copilot_model: str | None = None,
+ output_dir: str | None = None,
+ **kwargs: Any,
+ ):
+ from .copilot_engineer import CopilotEngineer
+
+ har_dir = get_har_dir(run_id, output_dir)
+ har_path = har_dir / "recording.har"
+
+ self._engineer = CopilotEngineer(
+ run_id=run_id,
+ har_path=har_path,
+ prompt=prompt,
+ copilot_model=copilot_model,
+ output_dir=output_dir,
+ **kwargs,
+ )
+ self.mcp_run_id = run_id
+
+ def start_sync(self) -> None:
+ self._engineer.start_sync()
+
+ def stop_sync(self) -> None:
+ self._engineer.stop_sync()
+
+ async def analyze_and_generate(self) -> dict[str, Any] | None:
+ """Run auto mode with Copilot SDK and MCP browser integration."""
+ try:
+ from copilot import CopilotClient
+ except ImportError:
+ self._engineer.ui.error("GitHub Copilot SDK not installed. Install with: uv pip install 'reverse-api-engineer[copilot]'")
+ return None
+
+ eng = self._engineer
+ eng.ui.header(eng.run_id, eng.prompt, eng.copilot_model, eng.sdk)
+ eng.ui.start_analysis()
+
+ auto_prompt = ClaudeAutoEngineer._build_auto_prompt(eng)
+ eng.message_store.save_prompt(auto_prompt)
+
+ done_event = asyncio.Event()
+ accumulated_text: list[str] = []
+
+ def on_event(event: Any) -> None:
+ event_type = event.type.value if hasattr(event.type, "value") else str(event.type)
+
+ if event_type == "assistant.message_delta":
+ delta = ""
+ if hasattr(event, "data") and hasattr(event.data, "delta_content"):
+ delta = event.data.delta_content or ""
+ if delta:
+ accumulated_text.append(delta)
+ eng.ui.thinking(delta)
+ elif event_type == "assistant.message":
+ if hasattr(event, "data") and hasattr(event.data, "usage"):
+ usage = event.data.usage
+ if isinstance(usage, dict):
+ eng.usage_metadata["input_tokens"] = usage.get("prompt_tokens", 0)
+ eng.usage_metadata["output_tokens"] = usage.get("completion_tokens", 0)
+ elif event_type == "session.idle":
+ done_event.set()
+
+ try:
+ client = CopilotClient(
+ {
+ "auto_start": True,
+ "use_logged_in_user": True,
+ }
+ )
+ await client.start()
+
+ mcp_config = {
+ "type": "stdio",
+ "command": "npx",
+ "args": [
+ "rae-playwright-mcp@latest",
+ "run-mcp-server",
+ "--run-id",
+ self.mcp_run_id,
+ ],
+ }
+
+ session = await client.create_session(
+ {
+ "model": eng.copilot_model,
+ "streaming": True,
+ "infinite_sessions": {"enabled": True},
+ "mcp_servers": {"playwright": mcp_config},
+ }
+ )
+
+ session.on(on_event)
+ await session.send({"prompt": auto_prompt})
+ await done_event.wait()
+
+ if accumulated_text:
+ eng.message_store.save_thinking("".join(accumulated_text))
+
+ script_path = str(eng.scripts_dir / eng._get_client_filename())
+ local_path = str(eng.local_scripts_dir / eng._get_client_filename()) if eng.local_scripts_dir else None
+ eng.ui.success(script_path, local_path)
+ eng.usage_metadata["estimated_cost_usd"] = 0.0
+
+ result: dict[str, Any] = {
+ "script_path": script_path,
+ "usage": eng.usage_metadata,
+ }
+ eng.message_store.save_result(result)
+ return result
+
+ except Exception as e:
+ error_msg = str(e)
+ eng.ui.error(error_msg)
+ eng.message_store.save_error(error_msg)
+
+ if "buffer size" in error_msg.lower() or "1048576" in error_msg or "exceeded maximum buffer" in error_msg.lower():
+ eng.ui.console.print("\n[yellow]Screenshot too large (exceeds 1MB limit)[/yellow]")
+ eng.ui.console.print("[dim]Tip: The AI should take element-specific screenshots instead of full-page screenshots.[/dim]")
+ else:
+ eng.ui.console.print("\n[dim]Make sure GitHub Copilot CLI is installed and you are logged in: gh auth login[/dim]")
+ return None
diff --git a/src/reverse_api/cli.py b/src/reverse_api/cli.py
index 688cf06..67be552 100644
--- a/src/reverse_api/cli.py
+++ b/src/reverse_api/cli.py
@@ -357,6 +357,8 @@ def repl_loop():
sdk = config_manager.get("sdk", "claude")
if sdk == "opencode":
model = config_manager.get("opencode_model", "claude-opus-4-6")
+ elif sdk == "copilot":
+ model = config_manager.get("copilot_model", "gpt-5")
else:
model = config_manager.get("claude_code_model", "claude-sonnet-4-6")
@@ -553,6 +555,7 @@ def handle_settings(mode_color=THEME_PRIMARY):
Choice(title="Agent Provider", value="agent_provider"),
Choice(title="Browser-Use Model", value="browser_use_model"),
Choice(title="Claude Code Model", value="claude_code_model"),
+ Choice(title="Copilot Model", value="copilot_model"),
Choice(title="OpenCode Model", value="opencode_model"),
Choice(title="OpenCode Provider", value="opencode_provider"),
Choice(title="Output Directory", value="output_dir"),
@@ -602,8 +605,9 @@ def handle_settings(mode_color=THEME_PRIMARY):
elif action == "sdk":
sdk_choices = [
- Choice(title="opencode", value="opencode"),
Choice(title="claude", value="claude"),
+ Choice(title="copilot", value="copilot"),
+ Choice(title="opencode", value="opencode"),
Choice(title="back", value="back"),
]
sdk = questionary.select(
@@ -690,6 +694,28 @@ def handle_settings(mode_color=THEME_PRIMARY):
config_manager.set("opencode_provider", new_provider)
console.print(f" [dim]updated[/dim] opencode provider: {new_provider}\n")
+ elif action == "copilot_model":
+ current = config_manager.get("copilot_model", "gpt-5")
+ new_model = questionary.text(
+ " > copilot model",
+ default=current or "gpt-5",
+ instruction="(e.g., 'gpt-5', 'gpt-4.1')",
+ qmark="",
+ style=questionary.Style(
+ [
+ ("question", f"fg:{THEME_SECONDARY}"),
+ ("instruction", f"fg:{THEME_DIM} italic"),
+ ]
+ ),
+ ).ask()
+ if new_model is not None:
+ new_model = new_model.strip()
+ if not new_model:
+ console.print(" [yellow]error:[/yellow] copilot model cannot be empty\n")
+ else:
+ config_manager.set("copilot_model", new_model)
+ console.print(f" [dim]updated[/dim] copilot model: {new_model}\n")
+
elif action == "opencode_model":
current = config_manager.get("opencode_model", "claude-opus-4-6")
new_model = questionary.text(
@@ -1464,6 +1490,18 @@ def run_auto_capture(prompt=None, url=None, model=None, output_dir=None):
sdk=sdk,
output_language=output_language,
)
+ elif sdk == "copilot":
+ from .auto_engineer import CopilotAutoEngineer
+
+ engineer = CopilotAutoEngineer(
+ run_id=run_id,
+ prompt=prompt,
+ copilot_model=config_manager.get("copilot_model", "gpt-5"),
+ output_dir=output_dir,
+ enable_sync=config_manager.get("real_time_sync", False),
+ sdk=sdk,
+ output_language=output_language,
+ )
else:
from .auto_engineer import ClaudeAutoEngineer
@@ -1610,6 +1648,21 @@ def run_engineer(
output_language=output_language,
output_mode=output_mode,
)
+ elif sdk == "copilot":
+ result = run_reverse_engineering(
+ run_id=run_id,
+ har_path=har_path,
+ prompt=prompt,
+ model=model,
+ output_dir=output_dir,
+ sdk=sdk,
+ copilot_model=config_manager.get("copilot_model", "gpt-5"),
+ enable_sync=enable_sync,
+ additional_instructions=additional_instructions,
+ is_fresh=is_fresh,
+ output_language=output_language,
+ output_mode=output_mode,
+ )
else:
result = run_reverse_engineering(
run_id=run_id,
diff --git a/src/reverse_api/config.py b/src/reverse_api/config.py
index 92387a8..3d6cbe0 100644
--- a/src/reverse_api/config.py
+++ b/src/reverse_api/config.py
@@ -9,12 +9,13 @@ DEFAULT_CONFIG = {
"browser_use_model": "bu-llm", # "bu-llm" or "{provider}/{model_name}" (e.g. "openai/gpt-5-mini")
"claude_code_model": "claude-sonnet-4-6",
"collector_model": "claude-sonnet-4-6", # Model for collector mode
+ "copilot_model": "gpt-5", # Model for Copilot SDK sessions
"opencode_model": "claude-opus-4-6",
"opencode_provider": "anthropic",
"output_dir": None, # None means use ~/.reverse-api/runs
"output_language": "python", # "python", "javascript", or "typescript"
"real_time_sync": True, # Enable real-time file sync during engineering
- "sdk": "claude", # "opencode" or "claude"
+ "sdk": "claude", # "claude", "opencode", or "copilot"
"stagehand_model": "openai/computer-use-preview-2025-03-11", # "{provider}/{model_name}" format
}
diff --git a/src/reverse_api/copilot_engineer.py b/src/reverse_api/copilot_engineer.py
new file mode 100644
index 0000000..7c1e546
--- /dev/null
+++ b/src/reverse_api/copilot_engineer.py
@@ -0,0 +1,245 @@
+"""GitHub Copilot SDK implementation for reverse engineering."""
+
+import asyncio
+from typing import Any
+
+import questionary
+
+from .base_engineer import BaseEngineer
+from .tui import THEME_PRIMARY, THEME_SECONDARY
+
+
+class CopilotEngineer(BaseEngineer):
+ """Uses GitHub Copilot SDK to analyze HAR files and generate API scripts."""
+
+ def __init__(
+ self,
+ run_id: str,
+ har_path: Any,
+ prompt: str,
+ copilot_model: str | None = None,
+ **kwargs: Any,
+ ):
+ super().__init__(run_id=run_id, har_path=har_path, prompt=prompt, **kwargs)
+ self.copilot_model = copilot_model or "gpt-5"
+
+ def _build_ask_user_tool(self) -> Any:
+ """Build the ask_user_question custom tool for Copilot sessions."""
+ try:
+ from copilot import define_tool
+ except ImportError:
+ raise ImportError("GitHub Copilot SDK not installed. Install with: uv pip install 'reverse-api-engineer[copilot]'") from None
+
+ from pydantic import BaseModel, Field
+
+ class QuestionOption(BaseModel):
+ label: str = Field(description="Display text for this option")
+ description: str = Field(default="", description="Explanation of what this option means")
+
+ class Question(BaseModel):
+ question: str = Field(description="The question to ask the user")
+ header: str = Field(default="", description="Short label for context")
+ options: list[QuestionOption] = Field(default_factory=list, description="Available choices")
+ multiSelect: bool = Field(default=False, description="Allow multiple selections")
+
+ class AskUserParams(BaseModel):
+ questions: list[Question] = Field(description="Questions to ask the user")
+
+ ui = self.ui
+
+ @define_tool(description="Ask the user a clarifying question. Use when you need user input to proceed.") # type: ignore[misc]
+ async def ask_user_question(params: AskUserParams) -> str:
+ answers: dict[str, str] = {}
+
+ ui.console.print()
+ ui.console.print(f" [{THEME_PRIMARY}]?[/{THEME_PRIMARY}] [bold white]Agent Question[/bold white]")
+ ui.console.print()
+
+ for q in params.questions:
+ if not q.question:
+ continue
+
+ if q.header:
+ ui.console.print(f" [dim]{q.header}[/dim]")
+
+ try:
+ if q.multiSelect:
+ choices = [f"{opt.label} - {opt.description}" if opt.description else opt.label for opt in q.options]
+ if choices:
+ selected = await questionary.checkbox(
+ f" > {q.question}",
+ choices=choices,
+ qmark="",
+ style=questionary.Style(
+ [
+ ("pointer", f"fg:{THEME_PRIMARY} bold"),
+ ("highlighted", f"fg:{THEME_PRIMARY} bold"),
+ ("selected", f"fg:{THEME_PRIMARY}"),
+ ]
+ ),
+ ).ask_async()
+
+ if selected is None:
+ raise KeyboardInterrupt
+
+ labels = [s.split(" - ")[0] if " - " in s else s for s in selected]
+ answers[q.question] = ", ".join(labels)
+ else:
+ answer = await questionary.text(
+ f" > {q.question}",
+ qmark="",
+ style=questionary.Style([("question", f"fg:{THEME_SECONDARY}")]),
+ ).ask_async()
+ if answer is None:
+ raise KeyboardInterrupt
+ answers[q.question] = answer.strip()
+ else:
+ choices = [f"{opt.label} - {opt.description}" if opt.description else opt.label for opt in q.options]
+ if choices:
+ answer = await questionary.select(
+ f" > {q.question}",
+ choices=choices,
+ qmark="",
+ style=questionary.Style(
+ [
+ ("pointer", f"fg:{THEME_PRIMARY} bold"),
+ ("highlighted", f"fg:{THEME_PRIMARY} bold"),
+ ]
+ ),
+ ).ask_async()
+
+ if answer is None:
+ raise KeyboardInterrupt
+
+ label = answer.split(" - ")[0] if " - " in answer else answer
+ answers[q.question] = label
+ else:
+ answer = await questionary.text(
+ f" > {q.question}",
+ qmark="",
+ style=questionary.Style([("question", f"fg:{THEME_SECONDARY}")]),
+ ).ask_async()
+ if answer is None:
+ raise KeyboardInterrupt
+ answers[q.question] = answer.strip()
+
+ ui.console.print(f" [dim]→ {answers[q.question]}[/dim]")
+
+ except KeyboardInterrupt:
+ ui.console.print(" [dim]User cancelled question[/dim]")
+ answers[q.question] = ""
+
+ ui.console.print()
+
+ # Return answers as formatted string
+ return "\n".join(f"{k}: {v}" for k, v in answers.items())
+
+ return ask_user_question
+
+ async def analyze_and_generate(self) -> dict[str, Any] | None:
+ """Run the reverse engineering analysis with GitHub Copilot."""
+ try:
+ from copilot import CopilotClient
+ except ImportError:
+ self.ui.error("GitHub Copilot SDK not installed. Install with: uv pip install 'reverse-api-engineer[copilot]'")
+ return None
+
+ self.ui.header(self.run_id, self.prompt, self.copilot_model, self.sdk)
+ self.ui.start_analysis()
+
+ prompt = self._build_analysis_prompt()
+ self.message_store.save_prompt(prompt)
+
+ done_event = asyncio.Event()
+ accumulated_text: list[str] = []
+
+ def on_event(event: Any) -> None:
+ event_type = event.type.value if hasattr(event.type, "value") else str(event.type)
+
+ if event_type == "assistant.message_delta":
+ delta = ""
+ if hasattr(event, "data") and hasattr(event.data, "delta_content"):
+ delta = event.data.delta_content or ""
+ if delta:
+ accumulated_text.append(delta)
+ self.ui.thinking(delta)
+
+ elif event_type == "assistant.message":
+ # Extract usage if available
+ if hasattr(event, "data") and hasattr(event.data, "usage"):
+ usage = event.data.usage
+ if isinstance(usage, dict):
+ self.usage_metadata["input_tokens"] = usage.get("prompt_tokens", 0)
+ self.usage_metadata["output_tokens"] = usage.get("completion_tokens", 0)
+
+ elif event_type == "session.idle":
+ done_event.set()
+
+ elif event_type == "session.compaction_start":
+ self.ui.console.print(" [dim]session compacting...[/dim]")
+
+ elif event_type == "session.compaction_complete":
+ self.ui.console.print(" [dim]session compaction complete[/dim]")
+
+ try:
+ ask_user_tool = self._build_ask_user_tool()
+
+ client = CopilotClient(
+ {
+ "auto_start": True,
+ "use_logged_in_user": True,
+ }
+ )
+ await client.start()
+
+ session = await client.create_session(
+ {
+ "model": self.copilot_model,
+ "streaming": True,
+ "infinite_sessions": {"enabled": True},
+ "tools": [ask_user_tool],
+ }
+ )
+
+ session.on(on_event)
+
+ await session.send({"prompt": prompt})
+
+ # Wait for session to complete
+ await done_event.wait()
+
+ # Save accumulated thinking text
+ if accumulated_text:
+ self.message_store.save_thinking("".join(accumulated_text))
+
+ # Build result
+ script_path = str(self.scripts_dir / self._get_client_filename())
+ local_path = str(self.local_scripts_dir / self._get_client_filename()) if self.local_scripts_dir else None
+ self.ui.success(script_path, local_path)
+
+ # GitHub subscription: cost is 0 (included in Copilot subscription)
+ self.usage_metadata["estimated_cost_usd"] = 0.0
+
+ # Display usage if available
+ input_tokens = self.usage_metadata.get("input_tokens", 0)
+ output_tokens = self.usage_metadata.get("output_tokens", 0)
+ if input_tokens > 0 or output_tokens > 0:
+ self.ui.console.print(" [dim]Usage:[/dim]")
+ if input_tokens > 0:
+ self.ui.console.print(f" [dim] input: {input_tokens:,} tokens[/dim]")
+ if output_tokens > 0:
+ self.ui.console.print(f" [dim] output: {output_tokens:,} tokens[/dim]")
+ self.ui.console.print(" [dim] cost: $0.00 (Copilot subscription)[/dim]")
+
+ result: dict[str, Any] = {
+ "script_path": script_path,
+ "usage": self.usage_metadata,
+ }
+ self.message_store.save_result(result)
+ return result
+
+ except Exception as e:
+ self.ui.error(str(e))
+ self.message_store.save_error(str(e))
+ self.ui.console.print("\n[dim]Make sure GitHub Copilot CLI is installed and you are logged in: gh auth login[/dim]")
+ return None
diff --git a/src/reverse_api/engineer.py b/src/reverse_api/engineer.py
index 60ecc03..6ab8374 100644
--- a/src/reverse_api/engineer.py
+++ b/src/reverse_api/engineer.py
@@ -28,9 +28,7 @@ logging.getLogger("claude_agent_sdk._internal.transport.subprocess_cli").setLeve
class ClaudeEngineer(BaseEngineer):
"""Uses Claude Agent SDK to analyze HAR files and generate Python API scripts."""
- async def _handle_ask_user_question(
- self, tool_name: str, input_params: dict[str, Any]
- ) -> dict[str, Any]:
+ async def _handle_ask_user_question(self, tool_name: str, input_params: dict[str, Any]) -> dict[str, Any]:
"""Handle AskUserQuestion tool by prompting the user interactively."""
if tool_name != "AskUserQuestion":
# Default allow for other tools in acceptEdits mode
@@ -41,9 +39,7 @@ class ClaudeEngineer(BaseEngineer):
# Display header
self.ui.console.print()
- self.ui.console.print(
- f" [{THEME_PRIMARY}]?[/{THEME_PRIMARY}] [bold white]Agent Question[/bold white]"
- )
+ self.ui.console.print(f" [{THEME_PRIMARY}]?[/{THEME_PRIMARY}] [bold white]Agent Question[/bold white]")
self.ui.console.print()
for q in questions:
@@ -63,9 +59,7 @@ class ClaudeEngineer(BaseEngineer):
if multi_select:
# Multi-select question
choices = [
- f"{opt.get('label', '')} - {opt.get('description', '')}"
- if opt.get("description")
- else opt.get("label", "")
+ f"{opt.get('label', '')} - {opt.get('description', '')}" if opt.get("description") else opt.get("label", "")
for opt in options
]
if choices:
@@ -86,10 +80,7 @@ class ClaudeEngineer(BaseEngineer):
raise KeyboardInterrupt
# Extract just the labels (before the " - " separator)
- labels = [
- s.split(" - ")[0] if " - " in s else s
- for s in selected
- ]
+ labels = [s.split(" - ")[0] if " - " in s else s for s in selected]
answers[question_text] = ", ".join(labels)
else:
# Text input fallback
@@ -105,9 +96,7 @@ class ClaudeEngineer(BaseEngineer):
else:
# Single select question
choices = [
- f"{opt.get('label', '')} - {opt.get('description', '')}"
- if opt.get("description")
- else opt.get("label", "")
+ f"{opt.get('label', '')} - {opt.get('description', '')}" if opt.get("description") else opt.get("label", "")
for opt in options
]
if choices:
@@ -284,6 +273,7 @@ def run_reverse_engineering(
sdk: str = "claude",
opencode_provider: str | None = None,
opencode_model: str | None = None,
+ copilot_model: str | None = None,
enable_sync: bool = False,
is_fresh: bool = False,
output_language: str = "python",
@@ -292,9 +282,10 @@ def run_reverse_engineering(
"""Run reverse engineering with the specified SDK.
Args:
- sdk: "opencode" or "claude" - determines which SDK to use
+ sdk: "claude", "opencode", or "copilot" - determines which SDK to use
opencode_provider: Provider ID for OpenCode (e.g., "anthropic")
opencode_model: Model ID for OpenCode (e.g., "claude-sonnet-4-6")
+ copilot_model: Model ID for Copilot (e.g., "gpt-5")
enable_sync: Enable real-time file syncing during engineering
is_fresh: Whether to start fresh (ignore previous scripts)
output_language: Target language - "python", "javascript", or "typescript"
@@ -319,6 +310,24 @@ def run_reverse_engineering(
output_language=output_language,
output_mode=output_mode,
)
+ elif sdk == "copilot":
+ from .copilot_engineer import CopilotEngineer
+
+ engineer = CopilotEngineer(
+ run_id=run_id,
+ har_path=har_path,
+ prompt=prompt,
+ model=model,
+ additional_instructions=additional_instructions,
+ output_dir=output_dir,
+ verbose=verbose,
+ enable_sync=enable_sync,
+ sdk=sdk,
+ is_fresh=is_fresh,
+ output_language=output_language,
+ output_mode=output_mode,
+ copilot_model=copilot_model,
+ )
else:
engineer = ClaudeEngineer(
run_id=run_id,
diff --git a/src/reverse_api/pricing.py b/src/reverse_api/pricing.py
index 00ee661..6c05219 100644
--- a/src/reverse_api/pricing.py
+++ b/src/reverse_api/pricing.py
@@ -92,6 +92,28 @@ MODEL_PRICING = {
"cache_read": 0.50,
"reasoning": 25.00,
},
+ # GPT models (for Copilot SDK - cost is $0 with GitHub subscription)
+ "gpt-5": {
+ "input": 2.00,
+ "output": 8.00,
+ "cache_creation": 0,
+ "cache_read": 0,
+ "reasoning": 8.00,
+ },
+ "gpt-4.1": {
+ "input": 2.00,
+ "output": 8.00,
+ "cache_creation": 0,
+ "cache_read": 0,
+ "reasoning": 8.00,
+ },
+ "gpt-4.1-mini": {
+ "input": 0.40,
+ "output": 1.60,
+ "cache_creation": 0,
+ "cache_read": 0,
+ "reasoning": 1.60,
+ },
}