Skip to content

The agent that turns websites into APIs!

Emit a client_executed json-stream event on successful verification (#114)

jarsh921ef09b290ab43
diff --git a/src/reverse_api/auto_engineer.py b/src/reverse_api/auto_engineer.py
index 563b829..c82d504 100644
--- a/src/reverse_api/auto_engineer.py
+++ b/src/reverse_api/auto_engineer.py
@@ -207,11 +207,26 @@ class ClaudeAutoEngineer(ClaudeEngineer):
system_prompt, user_message = self._get_active_prompts()
self.message_store.save_prompt(user_message)
+ # Every provider branch below needs the report_client_verified tool
+ # available (see engineer.py's _build_verification_mcp_server) —
+ # agent-drive sessions live-verify a generated client exactly the
+ # same way engineer-mode ones do.
+ verification_server = self._build_verification_mcp_server()
+
if self.agent_provider == "agent-browser":
options = ClaudeAgentOptions(
system_prompt=system_prompt,
- mcp_servers={},
- allowed_tools=allowed_tools_agent_browser_agent_mode(),
+ mcp_servers={"verification": verification_server},
+ # allowed_tools is an explicit allow-list here (unlike the
+ # else branch below), so the verification tool needs its
+ # SDK-qualified name added alongside it — confirmed live
+ # this is how the SDK namespaces an in-process MCP tool
+ # (matches real chrome-devtools-mcp tool-use blocks, e.g.
+ # "mcp__chrome-devtools__navigate_page").
+ allowed_tools=[
+ *allowed_tools_agent_browser_agent_mode(),
+ "mcp__verification__report_client_verified",
+ ],
permission_mode="bypassPermissions",
can_use_tool=self._handle_tool_permission,
cwd=str(self.scripts_dir.parent.parent),
@@ -223,7 +238,7 @@ class ClaudeAutoEngineer(ClaudeEngineer):
mcp_name, mcp_config = self._get_mcp_config()
options = ClaudeAgentOptions(
system_prompt=system_prompt,
- mcp_servers={mcp_name: mcp_config},
+ mcp_servers={mcp_name: mcp_config, "verification": verification_server},
permission_mode="bypassPermissions",
can_use_tool=self._handle_tool_permission,
cwd=str(self.scripts_dir.parent.parent),
diff --git a/src/reverse_api/base_engineer.py b/src/reverse_api/base_engineer.py
index 2d32c6f..ecbda95 100644
--- a/src/reverse_api/base_engineer.py
+++ b/src/reverse_api/base_engineer.py
@@ -34,6 +34,36 @@ NON_INTERACTIVE_ASK_USER_MESSAGE = (
"to start a new session with a clearer, more specific prompt."
)
+# Appended to every non-docs language's own codegen instructions, but only
+# for the Claude Agent SDK backend (see ClaudeEngineer._get_codegen_
+# instructions in engineer.py, the only place this is actually appended —
+# scoped there rather than in this shared BaseEngineer method specifically
+# because the report_client_verified tool it references only exists for
+# that one backend; OpenCode/Copilot/Cursor sessions would otherwise be
+# told to call a tool that was never registered in their environment, a
+# real regression flagged by automated review). Kept as one shared string
+# here since the wording applies identically regardless of language, so it
+# lives here once rather than being repeated in every partials/
+# _language_*.md file. Replaces the old approach entirely: previously,
+# --json-stream's real-time "client_executed" signal was inferred after
+# the fact by pattern-matching the agent's own Bash tool calls
+# (_is_client_verification_command, removed) — eight rounds of automated
+# PR review kept finding new ways a generated shell command could *look*
+# like a real execution without being one (quoted mentions, "||"/"&&"
+# masking, control-flow bodies, quoted keywords, index-misaligned
+# quoting from the fix for the previous one...). Suggested by the
+# upstream maintainer directly on the PR: have the agent report
+# verification itself via a dedicated tool call instead of inferring it
+# from shell syntax after the fact — see engineer.py's
+# report_client_verified tool. This closes the whole bug class by
+# construction rather than risking a ninth edge case.
+REPORT_CLIENT_VERIFIED_INSTRUCTION = (
+ "\n\nOnce you have confirmed the generated client actually works via a "
+ "real live execution against the target, call the `report_client_verified` "
+ "tool exactly once to report that. Call it only after a genuine successful "
+ "run you have actually observed — never before, and never speculatively."
+)
+
class BaseEngineer(ABC):
"""Abstract base class for API reverse engineering implementations."""
@@ -591,7 +621,17 @@ class BaseEngineer(ABC):
}.get(self.output_language, "python api_client.py")
def _get_codegen_instructions(self) -> str:
- """Return codegen instructions from the appropriate template partial."""
+ """Return codegen instructions from the appropriate template partial.
+
+ Base version, shared by every SDK backend (OpenCode, Copilot,
+ Cursor, Claude) — deliberately does NOT append
+ REPORT_CLIENT_VERIFIED_INSTRUCTION here. That tool only exists for
+ the Claude Agent SDK path (see engineer.py's
+ _build_verification_mcp_server); telling every other backend's
+ agent to call it too would just be an instruction for a tool that
+ was never registered in its environment. See ClaudeEngineer's own
+ override, the one place this actually applies.
+ """
from .prompts import load
if self.output_mode == "docs":
diff --git a/src/reverse_api/engineer.py b/src/reverse_api/engineer.py
index b5515a0..60c9c02 100644
--- a/src/reverse_api/engineer.py
+++ b/src/reverse_api/engineer.py
@@ -9,15 +9,18 @@ from claude_agent_sdk import (
AssistantMessage,
ClaudeAgentOptions,
ClaudeSDKClient,
+ McpSdkServerConfig,
PermissionResultAllow,
ResultMessage,
TextBlock,
ToolPermissionContext,
ToolResultBlock,
ToolUseBlock,
+ create_sdk_mcp_server,
+ tool,
)
-from .base_engineer import BaseEngineer
+from .base_engineer import REPORT_CLIENT_VERIFIED_INSTRUCTION, BaseEngineer
from .utils import build_sdk_env, is_context_overflow_error
# Suppress claude_agent_sdk logs
@@ -56,6 +59,96 @@ class ClaudeEngineer(BaseEngineer):
# Auto-approve all other tools
return PermissionResultAllow(updated_input=input_data)
+ def _build_verification_tool(self):
+ """The report_client_verified SdkMcpTool itself (handler + schema),
+ separate from _build_verification_mcp_server's server-wrapping step
+ purely so tests can call `.handler(args)` directly — create_sdk_mcp_
+ server hands back an opaque MCP `Server` instance with no easy way
+ to reach back into an individual tool's handler for a unit test.
+
+ So the agent can explicitly report a real, observed live-
+ verification success, instead of the caller trying to infer that
+ after the fact from the agent's own Bash tool-call text.
+
+ Replaces the previous approach entirely (see base_engineer.py's
+ REPORT_CLIENT_VERIFIED_INSTRUCTION for the full history: eight
+ rounds of automated review kept finding new ways a Bash command
+ could *look* like a real client execution without being one).
+ Suggested directly by the upstream maintainer on the PR this
+ shipped in — a deliberate tool call closes that whole bug class by
+ construction rather than risking a ninth parsing edge case.
+
+ Built fresh per call (not a module-level singleton): the inner
+ tool function is a closure over `self`, so it needs this specific
+ engineer instance's `_emit_json_event`/`scripts_dir`/
+ `_get_client_filename`, not a shared one. The same closure also
+ holds `already_reported`, a per-session (not per-instance) guard —
+ flagged by automated review: the tool's own description already
+ tells the agent to call it exactly once, but that's prompt text
+ only, not enforced, so a misbehaving or confused agent calling it
+ twice would otherwise emit two client_executed records for one
+ job. A closure-local flag is enough here (no need to touch
+ __init__/self for state that only needs to last one session) —
+ rebuilt fresh alongside the rest of this tool every time
+ analyze_and_generate starts one.
+ """
+ already_reported = False
+
+ @tool(
+ "report_client_verified",
+ "Call this exactly once, after you have actually run the generated "
+ "client live against the target and personally confirmed it works. "
+ "Do not call this speculatively, before a real run, or more than "
+ "once per session.",
+ {"summary": str},
+ )
+ async def report_client_verified(args: dict[str, Any]) -> dict[str, Any]:
+ nonlocal already_reported
+ if already_reported:
+ return {
+ "content": [
+ {"type": "text", "text": "Already recorded earlier in this session — no need to call this again."}
+ ]
+ }
+ already_reported = True
+ # Deliberately no manual message_store/UI logging here — the
+ # generic ToolUseBlock/ToolResultBlock handling in
+ # _process_streaming_response already logs every tool call,
+ # this one included, the same way it does for Bash/Read/etc.
+ self._emit_json_event(
+ {
+ "event": "client_executed",
+ "script_path": str(self.scripts_dir / self._get_client_filename()),
+ }
+ )
+ return {"content": [{"type": "text", "text": "Recorded — thanks for confirming."}]}
+
+ return report_client_verified
+
+ def _build_verification_mcp_server(self) -> McpSdkServerConfig:
+ """In-process MCP server exposing report_client_verified — see
+ _build_verification_tool's own docstring for the full reasoning."""
+ return create_sdk_mcp_server(
+ name="verification", tools=[self._build_verification_tool()]
+ )
+
+ def _get_codegen_instructions(self) -> str:
+ """BaseEngineer's own codegen instructions, plus the
+ report_client_verified tool-call instruction — scoped to this
+ Claude-SDK-specific override (inherited by ClaudeAutoEngineer too)
+ rather than living in the shared base method, since that tool only
+ exists for this backend. Flagged by automated review: appending it
+ in BaseEngineer directly meant OpenCode/Copilot/Cursor sessions
+ were also being told to call a tool that was never registered in
+ their environment. Docs mode still gets no suffix — BaseEngineer's
+ own version already returns before any run_command-related content
+ would apply, so there's nothing to append here either.
+ """
+ instructions = super()._get_codegen_instructions()
+ if self.output_mode == "docs":
+ return instructions
+ return instructions + REPORT_CLIENT_VERIFIED_INSTRUCTION
+
_USAGE_ACCUMULATE_KEYS = {
"input_tokens",
"output_tokens",
@@ -101,6 +194,12 @@ class ClaudeEngineer(BaseEngineer):
tool_name = last_tool_name or "Tool"
self.ui.tool_result(tool_name, is_error, output)
self.message_store.save_tool_result(tool_name, is_error, str(output) if output else None)
+ # The real-time "client_executed" --json-stream event
+ # (once inferred here from Bash tool-call text) now
+ # comes from the report_client_verified tool itself —
+ # see _build_verification_mcp_server — which fires it
+ # directly when called, so there's nothing left to do
+ # in this generic per-tool-result branch.
elif isinstance(block, TextBlock):
self.ui.thinking(block.text)
self.message_store.save_thinking(block.text)
@@ -181,6 +280,7 @@ class ClaudeEngineer(BaseEngineer):
model=self.model,
env=build_sdk_env(),
stderr=self._handle_cli_stderr,
+ mcp_servers={"verification": self._build_verification_mcp_server()},
)
last_result: dict[str, Any] | None = None
diff --git a/src/reverse_api/prompts/partials/_language_c.md b/src/reverse_api/prompts/partials/_language_c.md
index 3856599..c3b4d0f 100644
--- a/src/reverse_api/prompts/partials/_language_c.md
+++ b/src/reverse_api/prompts/partials/_language_c.md
@@ -4,6 +4,7 @@
- Don't hand-write or reconstruct `cJSON.c`/`cJSON.h` from memory — fetch the exact, pinned upstream source instead, so the vendored copy is the real, complete library rather than a possibly incomplete or subtly wrong approximation:
- `curl -fsSL -o {scripts_dir}/cJSON.c https://raw.githubusercontent.com/DaveGamble/cJSON/v1.7.18/cJSON.c`
- `curl -fsSL -o {scripts_dir}/cJSON.h https://raw.githubusercontent.com/DaveGamble/cJSON/v1.7.18/cJSON.h`
+- **cJSON ownership: only ever delete a full parsed tree once, from its root — never delete individual fields you pulled out of it.** `cJSON_GetObjectItem`/`cJSON_GetObjectItemCaseSensitive` return a pointer *into* the existing tree, not an independent copy, and `cJSON_Delete` on any node walks and frees that node *and every sibling after it* in the same object/array (it follows the node's `next` pointer in a loop) — not just the one node you passed it. Pulling two sibling fields out of the same parsed object (e.g. `args` and `headers` from one response) and then calling `cJSON_Delete` on each separately frees the first field's siblings too, so the second `cJSON_Delete` call double-frees an already-freed pointer. This is a real, confirmed-live bug, not a theoretical one — it reliably produces `STATUS_HEAP_CORRUPTION` on Windows, and the equivalent memory corruption on Linux/macOS, which can silently "work" for a while before crashing somewhere unrelated instead of failing immediately the way Windows does. If you need to keep using individual fields after you're done with the parsed response, either delete the whole root exactly once when it's truly no longer needed (`cJSON_Delete(json)` — never its individual children), or call `cJSON_DetachItemFromObject`/`cJSON_DetachItemFromObjectCaseSensitive` first to properly unlink a field from its parent before deleting it standalone.
- `libcurl` itself is a system library this project can't fetch or vendor — if compilation fails because `curl/curl.h` isn't found, stop and report a clear error naming the missing prerequisite (e.g. "libcurl development headers not found — install libcurl4-openssl-dev (Debian/Ubuntu) or curl (Homebrew) and retry") rather than running a package manager yourself; installing system packages is a host change the user should make and confirm, not something to do silently inside an auto-authorized session
- Reuse one `CURL` handle across requests rather than creating a new one per call
- Create a separate function for each distinct API endpoint, with a small struct for its response shape
diff --git a/tests/test_auto_engineer.py b/tests/test_auto_engineer.py
index 57d94a9..0dd4bfd 100644
--- a/tests/test_auto_engineer.py
+++ b/tests/test_auto_engineer.py
@@ -529,6 +529,72 @@ class TestClaudeAutoEngineerAnalyze:
assert result is not None
assert "script_path" in result
+ @pytest.mark.asyncio
+ async def test_registers_verification_mcp_server_alongside_browser_mcp(self, tmp_path):
+ """The default/chrome-mcp path (what route-reveal's agent-drive
+ actually uses via ExternalChromeAutoEngineer) must register the
+ report_client_verified tool's server *alongside* the browser MCP
+ server, not instead of it — see engineer.py's
+ _build_verification_mcp_server."""
+ eng = self._make_engineer(tmp_path)
+
+ mock_result = _sdk_result_message(result="Success")
+
+ mock_client = AsyncMock()
+ mock_client.query = AsyncMock()
+
+ async def mock_receive():
+ yield mock_result
+
+ mock_client.receive_response = mock_receive
+
+ with patch.object(eng, "_prompt_follow_up", new=AsyncMock(return_value=None)):
+ with patch("reverse_api.auto_engineer.ClaudeSDKClient") as mock_sdk:
+ mock_sdk.return_value.__aenter__ = AsyncMock(return_value=mock_client)
+ mock_sdk.return_value.__aexit__ = AsyncMock(return_value=False)
+
+ await eng.analyze_and_generate()
+
+ _, kwargs = mock_sdk.call_args
+ options = kwargs["options"]
+ assert "verification" in options.mcp_servers
+ assert options.mcp_servers["verification"]["name"] == "verification"
+ # The browser MCP server (playwright, for the default "auto"
+ # provider) must still be present too — this isn't a replacement.
+ assert len(options.mcp_servers) == 2
+
+ @pytest.mark.asyncio
+ async def test_agent_browser_path_allows_the_verification_tool(self, tmp_path):
+ """agent-browser mode uses an explicit allowed_tools list (unlike
+ the chrome-mcp/default path above), so the verification tool needs
+ its SDK-qualified name added there too, or Claude could never
+ actually call it even though the server is registered."""
+ eng = self._make_engineer(tmp_path, agent_provider="agent-browser")
+
+ with patch("reverse_api.auto_engineer.ensure_agent_browser_runtime") as mock_ensure:
+ mock_ensure.return_value = MagicMock(ok=True, error=None, notices=())
+
+ mock_result = _sdk_result_message(result="Success")
+ mock_client = AsyncMock()
+ mock_client.query = AsyncMock()
+
+ async def mock_receive():
+ yield mock_result
+
+ mock_client.receive_response = mock_receive
+
+ with patch.object(eng, "_prompt_follow_up", new=AsyncMock(return_value=None)):
+ with patch("reverse_api.auto_engineer.ClaudeSDKClient") as mock_sdk:
+ mock_sdk.return_value.__aenter__ = AsyncMock(return_value=mock_client)
+ mock_sdk.return_value.__aexit__ = AsyncMock(return_value=False)
+
+ await eng.analyze_and_generate()
+
+ _, kwargs = mock_sdk.call_args
+ options = kwargs["options"]
+ assert "verification" in options.mcp_servers
+ assert "mcp__verification__report_client_verified" in options.allowed_tools
+
@pytest.mark.asyncio
async def test_result_with_usage(self, tmp_path):
"""Result with usage metadata calculates cost."""
diff --git a/tests/test_base_engineer.py b/tests/test_base_engineer.py
index 3f7bc44..648348c 100644
--- a/tests/test_base_engineer.py
+++ b/tests/test_base_engineer.py
@@ -7,7 +7,7 @@ from unittest.mock import MagicMock, patch
import pytest
-from reverse_api.base_engineer import BaseEngineer
+from reverse_api.base_engineer import REPORT_CLIENT_VERIFIED_INSTRUCTION, BaseEngineer
class ConcreteEngineer(BaseEngineer):
@@ -432,6 +432,20 @@ class TestBaseEngineerHelpers:
eng = self._make_engineer(tmp_path, output_language="rust")
assert eng._get_run_command() == "python api_client.py"
+ def test_get_codegen_instructions_base_version_has_no_verification_instruction(self, tmp_path):
+ """BaseEngineer's own version (shared by OpenCode/Copilot/Cursor,
+ which don't subclass ClaudeEngineer) must NOT append
+ REPORT_CLIENT_VERIFIED_INSTRUCTION — that tool only exists for the
+ Claude Agent SDK path (see engineer.py's ClaudeEngineer override,
+ the one place this actually gets appended, and its own test
+ coverage in test_engineer.py). Regression flagged by automated
+ review: an earlier version of this appended it here directly,
+ which would have told every other backend's agent to call a tool
+ that was never registered in its environment."""
+ for language in ("python", "javascript", "typescript", "go", "java", "csharp", "php", "ruby", "c"):
+ eng = self._make_engineer(tmp_path, output_language=language, output_mode="client")
+ assert REPORT_CLIENT_VERIFIED_INSTRUCTION not in eng._get_codegen_instructions(), language
+
def test_quote_path_posix(self, monkeypatch):
"""POSIX platforms use shlex.quote (single quotes for spaces)."""
from reverse_api import base_engineer as be
diff --git a/tests/test_engineer.py b/tests/test_engineer.py
index afec783..b3759b2 100644
--- a/tests/test_engineer.py
+++ b/tests/test_engineer.py
@@ -823,6 +823,229 @@ class TestContextOverflowHandling:
assert follow_ups.await_count == 1
+class TestReportClientVerifiedTool:
+ """Test the report_client_verified MCP tool and the client_executed
+ json-stream event it now emits directly. Replaces the old approach of
+ inferring a real client execution from the agent's own Bash tool-call
+ text (_is_client_verification_command, removed) — eight rounds of
+ automated PR review kept finding new ways a shell command could *look*
+ like a real execution without being one (quoted mentions, "||"/"&&"
+ masking, control-flow bodies, quoted keywords, index-misaligned
+ quoting...). Suggested directly by the upstream maintainer on the PR:
+ have the agent report verification itself via a dedicated tool call
+ instead of inferring it from shell syntax after the fact — closes the
+ whole bug class by construction.
+ """
+
+ def _make_engineer(self, tmp_path, **kwargs):
+ har_path = tmp_path / "test.har"
+ har_path.touch()
+ defaults = {
+ "run_id": "test123",
+ "har_path": har_path,
+ "prompt": "test prompt",
+ "output_dir": str(tmp_path),
+ }
+ defaults.update(kwargs)
+ with patch("reverse_api.base_engineer.get_scripts_dir", return_value=tmp_path / "scripts"):
+ with patch("reverse_api.base_engineer.MessageStore"):
+ eng = ClaudeEngineer(**defaults)
+ eng.scripts_dir = tmp_path / "scripts"
+ eng.scripts_dir.mkdir(parents=True, exist_ok=True)
+ return eng
+
+ @pytest.mark.asyncio
+ async def test_tool_emits_client_executed_event_when_called(self, tmp_path):
+ eng = self._make_engineer(tmp_path, output_language="python")
+ events = []
+ eng._json_event_sink = events.append
+
+ tool = eng._build_verification_tool()
+ await tool.handler({"summary": "ran api_client.py, got a real 200 response"})
+
+ client_executed = [e for e in events if e["event"] == "client_executed"]
+ assert len(client_executed) == 1
+ assert client_executed[0]["script_path"] == str(eng.scripts_dir / "api_client.py")
+
+ @pytest.mark.asyncio
+ async def test_tool_second_call_in_same_session_does_not_re_emit(self, tmp_path):
+ """The tool's own description tells the agent to call it exactly
+ once, but that's prompt text, not enforced — flagged by automated
+ review: a misbehaving or confused agent calling it twice must not
+ emit two client_executed records for one job. Same `tool` object
+ both times (matching how one _build_verification_tool() call is
+ reused for a whole session, not rebuilt per tool-call)."""
+ eng = self._make_engineer(tmp_path, output_language="python")
+ events = []
+ eng._json_event_sink = events.append
+
+ tool = eng._build_verification_tool()
+ await tool.handler({"summary": "first confirmation"})
+ second_result = await tool.handler({"summary": "second confirmation, just to be sure"})
+
+ client_executed = [e for e in events if e["event"] == "client_executed"]
+ assert len(client_executed) == 1
+ assert "already recorded" in second_result["content"][0]["text"].lower()
+
+ @pytest.mark.asyncio
+ async def test_tool_two_separate_sessions_each_get_their_own_event(self, tmp_path):
+ """The de-dup guard is per-session (per _build_verification_tool()
+ call), not per-engine-instance or global — a fresh session (a new
+ call to analyze_and_generate, hence a fresh _build_verification_
+ tool()) must still be able to report verification normally."""
+ eng = self._make_engineer(tmp_path, output_language="python")
+ events = []
+ eng._json_event_sink = events.append
+
+ first_tool = eng._build_verification_tool()
+ await first_tool.handler({"summary": "session one"})
+
+ second_tool = eng._build_verification_tool()
+ await second_tool.handler({"summary": "session two"})
+
+ assert len([e for e in events if e["event"] == "client_executed"]) == 2
+
+ @pytest.mark.asyncio
+ async def test_tool_returns_confirmation_content(self, tmp_path):
+ """The SDK requires a dict with a "content" key back from every
+ tool call — confirms the handler's return value satisfies that,
+ not just that it has the side effect."""
+ eng = self._make_engineer(tmp_path, output_language="python")
+ eng._json_event_sink = lambda e: None
+
+ tool = eng._build_verification_tool()
+ result = await tool.handler({"summary": "verified"})
+
+ assert "content" in result
+ assert isinstance(result["content"], list)
+
+ @pytest.mark.asyncio
+ async def test_tool_no_event_when_no_sink_attached(self, tmp_path):
+ """No sink attached (plain --json, not --json-stream) — must not
+ raise, same posture as _emit_json_event's own no-op-when-unset
+ design."""
+ eng = self._make_engineer(tmp_path, output_language="python")
+ assert eng._json_event_sink is None
+
+ tool = eng._build_verification_tool()
+ result = await tool.handler({"summary": "verified"})
+ assert "content" in result # didn't raise
+
+ def test_tool_name_matches_the_documented_instruction(self, tmp_path):
+ """Must match REPORT_CLIENT_VERIFIED_INSTRUCTION's own reference to
+ this exact tool name — the two are only connected by string
+ agreement, nothing enforces it structurally."""
+ from reverse_api.base_engineer import REPORT_CLIENT_VERIFIED_INSTRUCTION
+
+ eng = self._make_engineer(tmp_path, output_language="python")
+ tool = eng._build_verification_tool()
+ assert tool.name == "report_client_verified"
+ assert f"`{tool.name}`" in REPORT_CLIENT_VERIFIED_INSTRUCTION
+
+ def test_build_verification_mcp_server_returns_sdk_server_config(self, tmp_path):
+ eng = self._make_engineer(tmp_path, output_language="python")
+ server = eng._build_verification_mcp_server()
+ assert server["type"] == "sdk"
+ assert server["name"] == "verification"
+
+ def test_get_codegen_instructions_appends_report_client_verified(self, tmp_path):
+ """ClaudeEngineer's own override (the one place this actually
+ applies — see engineer.py's own comment on why it's not in the
+ shared BaseEngineer method) appends the tool-call instruction for
+ every non-docs language."""
+ from reverse_api.base_engineer import REPORT_CLIENT_VERIFIED_INSTRUCTION
+
+ eng = self._make_engineer(tmp_path, output_language="python", output_mode="client")
+ assert eng._get_codegen_instructions().endswith(REPORT_CLIENT_VERIFIED_INSTRUCTION)
+
+ def test_get_codegen_instructions_docs_mode_has_no_verification_instruction(self, tmp_path):
+ """Docs mode generates an OpenAPI spec document, not runnable code —
+ nothing to live-verify, so it must not get the instruction."""
+ from reverse_api.base_engineer import REPORT_CLIENT_VERIFIED_INSTRUCTION
+
+ eng = self._make_engineer(tmp_path, output_language="python", output_mode="docs")
+ assert REPORT_CLIENT_VERIFIED_INSTRUCTION not in eng._get_codegen_instructions()
+
+ def test_get_codegen_instructions_appends_for_every_language(self, tmp_path):
+ """Confirms this isn't accidentally scoped to one language's own
+ partial template — every partials/_language_*.md gets it the same
+ way, since it's appended in Python after loading whichever one."""
+ from reverse_api.base_engineer import REPORT_CLIENT_VERIFIED_INSTRUCTION
+
+ for language in ("python", "javascript", "typescript", "go", "java", "csharp", "php", "ruby", "c"):
+ eng = self._make_engineer(tmp_path, output_language=language, output_mode="client")
+ assert eng._get_codegen_instructions().endswith(REPORT_CLIENT_VERIFIED_INSTRUCTION), language
+
+ @pytest.mark.asyncio
+ async def test_process_streaming_response_no_longer_infers_from_bash(self, tmp_path):
+ """The old mechanism (removed) would have emitted client_executed
+ just from seeing a successful `python api_client.py` Bash call.
+ That inference is gone entirely now — _process_streaming_response
+ must not emit the event no matter what the Bash command looks
+ like; only the tool itself does that."""
+ eng = self._make_engineer(tmp_path, output_language="python")
+ events = []
+ eng._json_event_sink = events.append
+
+ from claude_agent_sdk import AssistantMessage, ResultMessage, ToolResultBlock, ToolUseBlock
+
+ tool_use = MagicMock(spec=ToolUseBlock)
+ tool_use.name = "Bash"
+ tool_use.input = {"command": "python api_client.py"}
+ tool_result = MagicMock(spec=ToolResultBlock)
+ tool_result.is_error = False
+ tool_result.content = "output"
+ assistant = MagicMock(spec=AssistantMessage)
+ assistant.content = [tool_use, tool_result]
+ assistant.usage = None
+ result = MagicMock(spec=ResultMessage)
+ result.is_error = False
+ result.result = "Success"
+
+ async def receive():
+ yield assistant
+ yield result
+
+ client = MagicMock()
+ client.receive_response = receive
+
+ await eng._process_streaming_response(client)
+
+ assert [e for e in events if e["event"] == "client_executed"] == []
+
+ @pytest.mark.asyncio
+ async def test_analyze_and_generate_registers_verification_mcp_server(self, tmp_path):
+ """Integration-level: the actual ClaudeAgentOptions built by
+ analyze_and_generate must include the verification server, not
+ just that _build_verification_mcp_server works correctly in
+ isolation."""
+ eng = self._make_engineer(tmp_path, output_language="python")
+
+ mock_result = MagicMock()
+ mock_result.is_error = False
+ mock_result.result = "Success"
+
+ mock_client = AsyncMock()
+ mock_client.query = AsyncMock()
+
+ async def mock_receive():
+ yield mock_result
+
+ mock_client.receive_response = mock_receive
+
+ with patch("reverse_api.engineer.ClaudeSDKClient") as mock_sdk:
+ mock_sdk.return_value.__aenter__ = AsyncMock(return_value=mock_client)
+ mock_sdk.return_value.__aexit__ = AsyncMock(return_value=False)
+
+ with patch.object(eng, "_prompt_follow_up", new_callable=AsyncMock, return_value=None):
+ await eng.analyze_and_generate()
+
+ _, kwargs = mock_sdk.call_args
+ options = kwargs["options"]
+ assert "verification" in options.mcp_servers
+ assert options.mcp_servers["verification"]["name"] == "verification"
+
+
class TestSdkEnvWiring:
"""Test that SDK sessions get the auto-compact env override."""
diff --git a/tests/test_prompts.py b/tests/test_prompts.py
index 8f2f745..efa9f22 100644
--- a/tests/test_prompts.py
+++ b/tests/test_prompts.py
@@ -132,6 +132,32 @@ class TestLoadLanguagePartial:
assert "libcurl" in text
assert "/tmp/scripts/api_client.c" in text
+ def test_c_partial_warns_about_cjson_child_delete_double_free(self):
+ """Confirmed live: job dce90858fe54 / run 32c6d8533cb2
+ (httpbingo.org) generated a client that called cJSON_Delete on two
+ sibling fields (args, headers) pulled from the same parsed object —
+ cJSON_Delete walks and frees a node's siblings too, so the second
+ call double-freed an already-freed pointer, reliably producing
+ STATUS_HEAP_CORRUPTION on Windows. This guidance is what should
+ prevent that pattern from being generated again."""
+ text = load_language_partial(
+ "c",
+ scripts_dir="/tmp/scripts",
+ client_filename="api_client.c",
+ run_command="cc ...",
+ )
+ assert "cJSON_Delete" in text
+ assert "STATUS_HEAP_CORRUPTION" in text
+ assert "cJSON_DetachItemFromObject" in text
+ # Anchored to the actual prohibition and consequence wording, not
+ # just incidental mentions of the three terms above — a future edit
+ # that quietly dropped the real warning while still name-dropping
+ # cJSON_Delete/STATUS_HEAP_CORRUPTION/cJSON_DetachItemFromObject
+ # elsewhere in the partial would still pass the three bare `in`
+ # checks above; these phrases are unique to the warning itself.
+ assert "never delete individual fields you pulled out of it" in text
+ assert "double-frees an already-freed pointer" in text
+
class TestEngineerTemplates:
"""Test engineer system/user templates."""
diff --git a/website/content/docs/cli/scripted-usage.mdx b/website/content/docs/cli/scripted-usage.mdx
index 2a8f113..12a6f9a 100644
--- a/website/content/docs/cli/scripted-usage.mdx
+++ b/website/content/docs/cli/scripted-usage.mdx
@@ -43,7 +43,17 @@ reverse-api-engineer agent \
--json-stream
```
-Common event types for `agent` and `engineer`: `run_started` (includes `run_id`, `sdk`; agent runs also include `mode`, `url`, `command`), `tool_start`, `tool_end`, `thinking`, `ask_user_skipped`, `result`.
+Common event types for `agent` and `engineer`: `run_started` (includes `run_id`, `sdk`; agent runs also include `mode`, `url`, `command`), `tool_start`, `tool_end`, `thinking`, `ask_user_skipped`, `client_executed`, `result`.
+
+`client_executed` (`{"event": "client_executed", "script_path": "..."}`) fires
+when Claude calls the dedicated `report_client_verified` tool after reporting
+that it ran the generated client live and confirmed it works. The tool emits
+the event at most once per SDK session; it records Claude's verification
+report rather than running the client itself. Wrapper scripts can use this
+early signal to react before the run otherwise ends, since a long session can
+continue with unrelated cleanup or documentation work. Claude SDK only
+(`agent`/`engineer` with the default `sdk`); not emitted by the OpenCode,
+Copilot, or Cursor backends.
### Validate an agent run without launching a browser