The agent that turns websites into APIs!
feat(cli): add --headless flag for manual and agent
Kalil Bouzigues3edf1519d8a2
diff --git a/src/reverse_api/auto_engineer.py b/src/reverse_api/auto_engineer.pyindex 71311d1..05191f8 100644--- a/src/reverse_api/auto_engineer.py+++ b/src/reverse_api/auto_engineer.py@@ -37,6 +37,9 @@ class ClaudeAutoEngineer(ClaudeEngineer):**kwargs,):"""Initialize auto engineer with expected HAR path (created by MCP)."""+ # `headless` is auto-engineer specific (controls the MCP-spawned browser),+ # not BaseEngineer concept; pop before super() to avoid an unknown kwarg.+ headless = kwargs.pop("headless", False)har_dir = get_har_dir(run_id, output_dir)har_path = har_dir / "recording.har"@@ -50,6 +53,7 @@ class ClaudeAutoEngineer(ClaudeEngineer):)self.mcp_run_id = run_idself.agent_provider = agent_provider+ self.headless = headlessdef _build_auto_prompts(self) -> tuple[str, str]:"""Build (system_prompt, user_message) for auto mode.@@ -112,22 +116,35 @@ class ClaudeAutoEngineer(ClaudeEngineer):return PermissionResultAllow(updated_input=input_data)def _get_mcp_config(self) -> tuple[str, dict]:- """Return (server_name, mcp_config) based on agent_provider."""+ """Return (server_name, mcp_config) based on agent_provider.++ Auto-connect requires a real headed Chrome instance with a remote+ debugging server, so it is dropped in headless mode and the MCP+ spawns its own headless Chromium instead.+ """if self.agent_provider == "chrome-mcp":+ args = ["chrome-devtools-mcp@latest", "--no-usage-statistics"]+ if self.headless:+ args.append("--headless")+ else:+ args.append("--autoConnect")return "chrome-devtools", {"type": "stdio","command": "npx",- "args": ["chrome-devtools-mcp@latest", "--autoConnect", "--no-usage-statistics"],+ "args": args,}+ playwright_args = [+ "rae-playwright-mcp@latest",+ "run-mcp-server",+ "--run-id",+ self.mcp_run_id,+ ]+ if self.headless:+ playwright_args.append("--headless")return "playwright", {"type": "stdio","command": "npx",- "args": [- "rae-playwright-mcp@latest",- "run-mcp-server",- "--run-id",- self.mcp_run_id,- ],+ "args": playwright_args,}async def analyze_and_generate(self) -> dict[str, Any] | None:@@ -210,6 +227,7 @@ class OpenCodeAutoEngineer(OpenCodeEngineer):def __init__(self, run_id: str, prompt: str, output_dir: str | None = None, agent_provider: str = "auto", **kwargs):"""Initialize auto engineer with expected HAR path (created by MCP)."""+ headless = kwargs.pop("headless", False)har_dir = get_har_dir(run_id, output_dir)har_path = har_dir / "recording.har"@@ -223,36 +241,50 @@ class OpenCodeAutoEngineer(OpenCodeEngineer):self.mcp_run_id = run_idself.agent_provider = agent_providerself.mcp_name = None+ self.headless = headlessdef _get_active_prompts(self) -> tuple[str, str]:return ClaudeAutoEngineer._build_auto_prompts(self)def _get_opencode_mcp_config(self) -> dict:- """Return OpenCode MCP registration payload based on agent_provider."""+ """Return OpenCode MCP registration payload based on agent_provider.++ Auto-connect requires a headed Chrome with a remote debugging server,+ so it is dropped in headless mode in favor of an MCP-spawned headless+ Chromium.+ """if self.agent_provider == "chrome-mcp":self.mcp_name = f"chrome-devtools-{self._session_id}"+ cmd = ["npx", "-y", "chrome-devtools-mcp@latest", "--no-usage-statistics"]+ if self.headless:+ cmd.append("--headless")+ else:+ cmd.append("--autoConnect")return {"name": self.mcp_name,"config": {"type": "local",- "command": ["npx", "-y", "chrome-devtools-mcp@latest", "--autoConnect", "--no-usage-statistics"],+ "command": cmd,"enabled": True,"timeout": 30000,},}self.mcp_name = f"playwright-{self._session_id}"+ cmd = [+ "npx",+ "-y",+ "rae-playwright-mcp@latest",+ "run-mcp-server",+ "--run-id",+ self.mcp_run_id,+ ]+ if self.headless:+ cmd.append("--headless")return {"name": self.mcp_name,"config": {"type": "local",- "command": [- "npx",- "-y",- "rae-playwright-mcp@latest",- "run-mcp-server",- "--run-id",- self.mcp_run_id,- ],+ "command": cmd,"enabled": True,"timeout": 30000,},@@ -450,6 +482,7 @@ class CopilotAutoEngineer:):from .copilot_engineer import CopilotEngineer+ headless = kwargs.pop("headless", False)har_dir = get_har_dir(run_id, output_dir)har_path = har_dir / "recording.har"@@ -463,6 +496,7 @@ class CopilotAutoEngineer:)self.mcp_run_id = run_idself.agent_provider = agent_provider+ self.headless = headlessdef start_sync(self) -> None:self._engineer.start_sync()@@ -526,25 +560,33 @@ class CopilotAutoEngineer:if self.agent_provider == "chrome-mcp":mcp_server_name = "chrome-devtools"+ chrome_args = ["-y", "chrome-devtools-mcp@latest", "--no-usage-statistics"]+ if self.headless:+ chrome_args.append("--headless")+ else:+ chrome_args.append("--autoConnect")mcp_config = {"type": "local","command": "npx",- "args": ["-y", "chrome-devtools-mcp@latest", "--autoConnect", "--no-usage-statistics"],+ "args": chrome_args,"tools": ["*"],"timeout": 30000,}else:mcp_server_name = "playwright"+ pw_args = [+ "-y",+ "rae-playwright-mcp@latest",+ "run-mcp-server",+ "--run-id",+ self.mcp_run_id,+ ]+ if self.headless:+ pw_args.append("--headless")mcp_config = {"type": "local","command": "npx",- "args": [- "-y",- "rae-playwright-mcp@latest",- "run-mcp-server",- "--run-id",- self.mcp_run_id,- ],+ "args": pw_args,"tools": ["*"],"timeout": 30000,}diff --git a/src/reverse_api/browser.py b/src/reverse_api/browser.pyindex 8986147..eae7771 100644--- a/src/reverse_api/browser.py+++ b/src/reverse_api/browser.py@@ -187,11 +187,13 @@ class ManualBrowser:prompt: str,output_dir: str | None = None,use_real_chrome: bool = True, # New option to use real Chrome+ headless: bool = False,):self.run_id = run_idself.prompt = promptself.output_dir = output_dirself.use_real_chrome = use_real_chrome+ self.headless = headlessself.har_dir = get_har_dir(run_id, output_dir)self.har_path = self.har_dir / "recording.har"@@ -249,7 +251,7 @@ class ManualBrowser:self._context = self._playwright.chromium.launch_persistent_context(user_data_dir=str(temp_profile_dir),channel="chrome", # Use real Chrome binary- headless=False,+ headless=self.headless,record_har_path=str(self.har_path),record_har_content="embed",no_viewport=True,@@ -326,10 +328,15 @@ class ManualBrowser:"--use-mock-keychain",]+ # In headless mode (scripted / CI / VPS), keep Playwright's default+ # `--no-sandbox` so Chromium can launch on hosts without unprivileged+ # user namespaces. In headed mode we still strip it for a more+ # realistic browser fingerprint.+ ignore = ["--enable-automation"] if self.headless else ["--enable-automation", "--no-sandbox"]self._browser = self._playwright.chromium.launch(- headless=False,+ headless=self.headless,args=chrome_args,- ignore_default_args=["--enable-automation", "--no-sandbox"],+ ignore_default_args=ignore,)# Create context with HAR recording and realistic settingsdiff --git a/src/reverse_api/cli.py b/src/reverse_api/cli.pyindex ee2e093..6c73cfd 100644--- a/src/reverse_api/cli.py+++ b/src/reverse_api/cli.py@@ -1035,9 +1035,14 @@ def handle_messages(run_id: str, mode_color=THEME_PRIMARY):default=None,)@click.option("--output-dir", "-o", default=None, help="Custom output directory.")-def manual(prompt, url, reverse_engineer, model, output_dir):+ "--headless",+ is_flag=True,+ help="Launch the browser in headless mode (no UI). Required on machines without an X server (CI / VPS).",+)+def manual(prompt, url, reverse_engineer, model, output_dir, headless):"""Start a manual browser session."""- run_manual_capture(prompt, url, reverse_engineer, model, output_dir)+ run_manual_capture(prompt, url, reverse_engineer, model, output_dir, headless=headless)@main.command(@@ -1089,7 +1094,12 @@ Exit codes:is_flag=True,help="Emit a single JSON result on stdout (logs go to stderr). Implies --no-interactive.",)-def agent(prompt, url, model, output_dir, no_interactive, as_json):+ "--headless",+ is_flag=True,+ help="Launch the MCP-controlled browser in headless mode (required on machines without an X server). For chrome-mcp this drops --autoConnect since auto-connect requires a headed Chrome instance.",+)+def agent(prompt, url, model, output_dir, no_interactive, as_json, headless):"""Run autonomous agent browser session.Agent mode runs an integrated capture + reverse-engineering pipeline; if you@@ -1117,7 +1127,12 @@ def agent(prompt, url, model, output_dir, no_interactive, as_json):if not as_json:run_agent_capture(- prompt=prompt, url=url, model=model, output_dir=output_dir, interactive=interactive+ prompt=prompt,+ url=url,+ model=model,+ output_dir=output_dir,+ interactive=interactive,+ headless=headless,)return@@ -1130,6 +1145,7 @@ def agent(prompt, url, model, output_dir, no_interactive, as_json):model=model,output_dir=output_dir,interactive=interactive,+ headless=headless,)payload = _build_agent_payload(result, prompt=prompt, url=url, output_dir=output_dir)except KeyboardInterrupt:@@ -1146,7 +1162,7 @@ def agent(prompt, url, model, output_dir, no_interactive, as_json):sys.exit(0 if payload["status"] == "ok" else 1)-def run_manual_capture(prompt=None, url=None, reverse_engineer=True, model=None, output_dir=None):+def run_manual_capture(prompt=None, url=None, reverse_engineer=True, model=None, output_dir=None, headless=False):"""Shared logic for manual capture."""output_dir = output_dir or config_manager.get("output_dir")@@ -1180,7 +1196,7 @@ def run_manual_capture(prompt=None, url=None, reverse_engineer=True, model=None,paths={"har_dir": str(get_har_dir(run_id, output_dir))},)- browser = ManualBrowser(run_id=run_id, prompt=prompt, output_dir=output_dir)+ browser = ManualBrowser(run_id=run_id, prompt=prompt, output_dir=output_dir, headless=headless)har_path = browser.start(start_url=url)if reverse_engineer:@@ -1205,7 +1221,7 @@ def run_manual_capture(prompt=None, url=None, reverse_engineer=True, model=None,console.print(f" [dim]>[/dim] [dim]use 'reverse-api-engineer engineer {run_id}' to engineer later[/dim]\n")-def run_agent_capture(prompt=None, url=None, model=None, output_dir=None, interactive=True):+def run_agent_capture(prompt=None, url=None, model=None, output_dir=None, interactive=True, headless=False):"""Shared logic for agent capture mode."""output_dir = output_dir or config_manager.get("output_dir")@@ -1230,6 +1246,7 @@ def run_agent_capture(prompt=None, url=None, model=None, output_dir=None, interaoutput_dir=output_dir,agent_provider=agent_provider,interactive=interactive,+ headless=headless,)@@ -1283,7 +1300,13 @@ def run_collector(prompt=None, model=None, output_dir=None):def run_auto_capture(- prompt=None, url=None, model=None, output_dir=None, agent_provider="auto", interactive=True+ prompt=None,+ url=None,+ model=None,+ output_dir=None,+ agent_provider="auto",+ interactive=True,+ headless=False,):"""Auto mode: LLM-driven browser automation + real-time reverse engineering."""output_dir = output_dir or config_manager.get("output_dir")@@ -1301,7 +1324,7 @@ def run_auto_capture(url = options.get("url")model = options["model"]- if agent_provider == "chrome-mcp":+ if agent_provider == "chrome-mcp" and not headless:console.print()console.print(" [dim]chrome devtools mcp (auto-connect)[/dim]")console.print(" [dim]controlling your real chrome browser[/dim]")@@ -1316,6 +1339,11 @@ def run_auto_capture(console.print(" [dim]warning: the agent will execute actions on your browser[/dim]")console.print(" [dim]avoid browsing sensitive sites during the session[/dim]")console.print()+ elif agent_provider == "chrome-mcp" and headless:+ console.print()+ console.print(" [dim]chrome devtools mcp (headless)[/dim]")+ console.print(" [dim]auto-connect disabled — mcp will spawn its own headless chrome[/dim]")+ console.print()run_id = generate_run_id()timestamp = get_timestamp()@@ -1349,6 +1377,7 @@ def run_auto_capture(sdk=sdk,output_language=output_language,interactive=interactive,+ headless=headless,)elif sdk == "copilot":from .auto_engineer import CopilotAutoEngineer@@ -1363,6 +1392,7 @@ def run_auto_capture(sdk=sdk,output_language=output_language,interactive=interactive,+ headless=headless,)else:from .auto_engineer import ClaudeAutoEngineer@@ -1377,6 +1407,7 @@ def run_auto_capture(sdk=sdk,output_language=output_language,interactive=interactive,+ headless=headless,)# Start sync before analysisdiff --git a/tests/test_cli_followups.py b/tests/test_cli_followups.pyindex 92d18b9..1c8b920 100644--- a/tests/test_cli_followups.py+++ b/tests/test_cli_followups.py@@ -316,3 +316,93 @@ class TestFollowUpPromptSuppressed:result = runner.invoke(agent_cmd, ["--json", "-p", "x"])assert result.exit_code == 0, result.outputassert mock_run.call_args.kwargs["interactive"] is False+++# ---------------------------------------------------------------------------+# --headless flag (CI / VPS / scripted)+# ---------------------------------------------------------------------------+++class TestHeadlessFlag:+ """`--headless` wires through to ManualBrowser / auto-engineer constructors+ and adjusts MCP args (drops --autoConnect for chrome-mcp, adds --headless+ for both chrome-mcp and playwright)."""++ def test_manual_command_threads_headless(self):+ from reverse_api.cli import manual as manual_cmd++ runner = CliRunner()+ with patch("reverse_api.cli.run_manual_capture") as mock_run:+ result = runner.invoke(manual_cmd, ["-p", "x", "-u", "https://example.com", "--headless"])+ assert result.exit_code == 0, result.output+ assert mock_run.call_args.kwargs["headless"] is True++ def test_manual_default_headless_false(self):+ from reverse_api.cli import manual as manual_cmd++ runner = CliRunner()+ with patch("reverse_api.cli.run_manual_capture") as mock_run:+ result = runner.invoke(manual_cmd, ["-p", "x", "-u", "https://example.com"])+ assert result.exit_code == 0, result.output+ assert mock_run.call_args.kwargs["headless"] is False++ def test_agent_command_threads_headless(self):+ from reverse_api.cli import agent as agent_cmd++ runner = CliRunner()+ with patch("reverse_api.cli.run_agent_capture", return_value={"run_id": "x", "mode": "auto"}) as mock_run:+ result = runner.invoke(agent_cmd, ["-p", "x", "--json", "--headless"])+ assert result.exit_code == 0, result.output+ assert mock_run.call_args.kwargs["headless"] is True++ def test_agent_default_headless_false(self):+ from reverse_api.cli import agent as agent_cmd++ runner = CliRunner()+ with patch("reverse_api.cli.run_agent_capture", return_value={"run_id": "x", "mode": "auto"}) as mock_run:+ result = runner.invoke(agent_cmd, ["-p", "x", "--json"])+ assert result.exit_code == 0, result.output+ assert mock_run.call_args.kwargs["headless"] is False+++class TestHeadlessMcpConfig:+ """ClaudeAutoEngineer._get_mcp_config arg construction for headed vs headless."""++ def _make(self, agent_provider: str, headless: bool, tmp_path):+ """Build a ClaudeAutoEngineer without invoking heavy init paths."""+ from reverse_api.auto_engineer import ClaudeAutoEngineer++ with patch("reverse_api.base_engineer.get_scripts_dir", return_value=tmp_path):+ with patch("reverse_api.base_engineer.MessageStore"):+ with patch("reverse_api.base_engineer.SessionManager"):+ with patch("reverse_api.auto_engineer.get_har_dir", return_value=tmp_path):+ eng = ClaudeAutoEngineer(+ run_id="r",+ prompt="x",+ model="claude-sonnet-4-6",+ agent_provider=agent_provider,+ headless=headless,+ )+ return eng++ def test_chrome_mcp_headed_uses_autoconnect(self, tmp_path):+ eng = self._make("chrome-mcp", headless=False, tmp_path=tmp_path)+ _, cfg = eng._get_mcp_config()+ assert "--autoConnect" in cfg["args"]+ assert "--headless" not in cfg["args"]++ def test_chrome_mcp_headless_drops_autoconnect_adds_headless(self, tmp_path):+ eng = self._make("chrome-mcp", headless=True, tmp_path=tmp_path)+ _, cfg = eng._get_mcp_config()+ assert "--autoConnect" not in cfg["args"], "auto-connect cannot work without a real headed Chrome"+ assert "--headless" in cfg["args"]++ def test_playwright_headless_adds_flag(self, tmp_path):+ eng = self._make("auto", headless=True, tmp_path=tmp_path)+ _, cfg = eng._get_mcp_config()+ assert "--headless" in cfg["args"]++ def test_playwright_headed_no_headless_flag(self, tmp_path):+ eng = self._make("auto", headless=False, tmp_path=tmp_path)+ _, cfg = eng._get_mcp_config()+ assert "--headless" not in cfg["args"]