The agent that turns websites into APIs!
feat(cli): stable usage normalization, error_kind enum, --json-schema-version
Kalil Bouziguesac5282fce78e
diff --git a/src/reverse_api/cli.py b/src/reverse_api/cli.pyindex be94944..5b59910 100644--- a/src/reverse_api/cli.py+++ b/src/reverse_api/cli.py@@ -54,9 +54,99 @@ console = Console()config_manager = ConfigManager(get_config_path())session_manager = SessionManager(get_history_path())-# Stable schema version for --json outputs consumed by other agents/scripts.+# Schema version for --json outputs. Wrappers can query it via+# `reverse-api-engineer --json-schema-version`.AGENT_JSON_SCHEMA_VERSION = 1+# Map of stable usage keys → SDK-specific candidates (first match wins).+# Lets `agent --json` and `engineer --json` emit a stable cost/token shape+# regardless of which SDK (Claude / OpenCode / Copilot) ran underneath.+_STABLE_USAGE_KEYS: dict[str, tuple[str, ...]] = {+ "input_tokens": ("input_tokens",),+ "output_tokens": ("output_tokens",),+ "cache_read_tokens": ("cache_read_input_tokens", "cache_read_tokens"),+ "cache_write_tokens": ("cache_creation_input_tokens", "cache_write_tokens"),+ "total_cost_usd": ("estimated_cost_usd", "total_cost_usd", "total_cost"),+}+++def _normalize_usage(raw: dict | None) -> dict:+ """Return a stable subset of usage fields, keeping the SDK-native dict under .raw.++ Wrappers can rely on the top-level keys; per-SDK extras stay under raw.+ """+ if not raw or not isinstance(raw, dict):+ return {}+ out: dict = {}+ for stable_key, candidates in _STABLE_USAGE_KEYS.items():+ for c in candidates:+ if c in raw:+ out[stable_key] = raw[c]+ break+ out["raw"] = raw+ return out+++# Machine-readable error categories. Wrappers can react differently to each+# without pattern-matching on the human-readable `error` string.+ERROR_KINDS = (+ "misuse", # user input invalid / missing required arg+ "config_invalid", # config file or env var malformed+ "permission_denied", # filesystem / API perm denied+ "network", # DNS / TCP / TLS / timeout+ "engine_failure", # SDK or capture engine crashed mid-run+ "interrupted", # KeyboardInterrupt / SIGINT+ "unknown", # default fallback+)+++def _format_error_message(error: str | BaseException | None) -> str | None:+ """Render an exception or string into a human-readable error message.++ KeyboardInterrupt has no useful str() — we return the conventional+ "interrupted" so downstream wrappers can match on a stable message.+ Empty exception messages fall back to the class name.+ """+ if error is None:+ return None+ if isinstance(error, KeyboardInterrupt):+ return "interrupted"+ if isinstance(error, BaseException):+ return str(error) or type(error).__name__+ return error+++def _classify_error(error: str | BaseException | None, *, default: str = "unknown") -> str | None:+ """Map an Exception or human error message to one of the ERROR_KINDS.++ Callers may pass a stronger hint by setting `default` (e.g. `misuse` from+ a CLI argument check, before any exception).+ """+ if error is None:+ return None+ if isinstance(error, BaseException):+ if isinstance(error, KeyboardInterrupt):+ return "interrupted"+ if isinstance(error, PermissionError):+ return "permission_denied"+ if isinstance(error, (ConnectionError, TimeoutError)):+ return "network"+ msg = str(error)+ else:+ msg = error+ low = msg.lower()+ if "permission denied" in low or "errno 13" in low:+ return "permission_denied"+ if "interrupted" in low:+ return "interrupted"+ if "is required" in low or "missing required" in low or "no such option" in low or "in non-interactive" in low:+ return "misuse"+ if any(kw in low for kw in ("connection refused", "timeout", "timed out", "dns", "network", "unreachable", "ssl")):+ return "network"+ if "not found" in low or "no run" in low or "produced no run" in low or "produced no result" in low:+ return "engine_failure"+ return default+@contextmanagerdef _quiet_consoles_for_json():@@ -91,7 +181,8 @@ def _build_agent_payload(prompt: str | None,url: str | None,output_dir: str | None = None,- error: str | None = None,+ error: str | BaseException | None = None,+ error_kind_hint: str = "unknown",) -> dict:"""Normalize an agent capture result into a stable JSON shape.@@ -101,22 +192,27 @@ def _build_agent_payload(result = result or {}run_id = result.get("run_id")inner_error = result.get("error")- final_error = error or inner_error or (None if run_id else "agent capture produced no run")+ final_error_obj = error if error is not None else inner_error+ if final_error_obj is None and not run_id:+ final_error_obj = "agent capture produced no run"+ error_str = _format_error_message(final_error_obj)+ error_kind = _classify_error(final_error_obj, default=error_kind_hint) if final_error_obj else Nonehar_path = Noneif run_id:candidate = get_har_dir(run_id, output_dir) / "recording.har"har_path = str(candidate) if candidate.exists() else Nonereturn {"schema_version": AGENT_JSON_SCHEMA_VERSION,- "status": "error" if final_error else "ok",+ "status": "error" if error_str else "ok","run_id": run_id,"prompt": prompt,"url": url,"mode": result.get("mode"),"har_path": har_path,"script_path": result.get("script_path"),- "usage": result.get("usage") or {},- "error": final_error,+ "usage": _normalize_usage(result.get("usage")),+ "error": error_str,+ "error_kind": error_kind,}@@ -126,7 +222,8 @@ def _build_engineer_payload(run_id: str,prompt: str | None,fresh: bool,- error: str | None = None,+ error: str | BaseException | None = None,+ error_kind_hint: str = "unknown",) -> dict:"""Normalize an engineer-mode result into a stable JSON shape.@@ -135,18 +232,23 @@ def _build_engineer_payload(to --prompt (which may have been used as either a full replacement or asadditional instructions depending on --fresh)."""- result = result if result is not None else {}- inner_error = result.get("error") if isinstance(result, dict) else None- final_error = error or inner_error or (None if result else "engineering produced no result")+ result = result if isinstance(result, dict) else {}+ inner_error = result.get("error")+ final_error_obj = error if error is not None else inner_error+ if final_error_obj is None and not result:+ final_error_obj = "engineering produced no result"+ error_str = _format_error_message(final_error_obj)+ error_kind = _classify_error(final_error_obj, default=error_kind_hint) if final_error_obj else Nonereturn {"schema_version": AGENT_JSON_SCHEMA_VERSION,- "status": "error" if final_error else "ok",+ "status": "error" if error_str else "ok","run_id": run_id,"prompt": prompt,"fresh": fresh,- "script_path": result.get("script_path") if isinstance(result, dict) else None,- "usage": (result.get("usage") if isinstance(result, dict) else None) or {},- "error": final_error,+ "script_path": result.get("script_path"),+ "usage": _normalize_usage(result.get("usage")),+ "error": error_str,+ "error_kind": error_kind,}# Mode definitions@@ -416,15 +518,26 @@ CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]}@click.group(invoke_without_command=True, context_settings=CONTEXT_SETTINGS)@click.pass_context@click.version_option(version=__version__)-def main(ctx: click.Context):+ "--json-schema-version",+ "show_schema_version",+ is_flag=True,+ help="Print the agent/engineer JSON schema_version this binary emits and exit.",+)+def main(ctx: click.Context, show_schema_version: bool):"""reverse-api-engineer: reverse engineer apis.Run without a subcommand to start the interactive REPL; use agent, manual,or engineer for CLI mode.Most subcommands accept --json and --no-interactive for scripted use; see- `<cmd> --help` for per-command details.+ `<cmd> --help` for per-command details. Wrappers that need to gate on the+ payload schema can call `reverse-api-engineer --json-schema-version`."""+ if show_schema_version:+ click.echo(str(AGENT_JSON_SCHEMA_VERSION))+ ctx.exit(0)+if ctx.invoked_subcommand is None:# Refuse to drop into the prompt_toolkit REPL when stdin is not a TTY:# without an interactive terminal the REPL would block on stdin@@ -1121,6 +1234,7 @@ def agent(prompt, url, model, output_dir, no_interactive, as_json, headless):url=url,output_dir=output_dir,error="--prompt is required in non-interactive/--json mode",+ error_kind_hint="misuse",)click.echo(json.dumps(misuse))else:@@ -1154,13 +1268,15 @@ def agent(prompt, url, model, output_dir, no_interactive, as_json, headless):headless=headless,)payload = _build_agent_payload(result, prompt=prompt, url=url, output_dir=output_dir)- except KeyboardInterrupt:+ except KeyboardInterrupt as e:payload = _build_agent_payload(- {}, prompt=prompt, url=url, output_dir=output_dir, error="interrupted"+ {}, prompt=prompt, url=url, output_dir=output_dir, error=e)except Exception as e:+ # Pass the exception object so _classify_error can use isinstance+ # (PermissionError → permission_denied, ConnectionError → network, ...)payload = _build_agent_payload(- {}, prompt=prompt, url=url, output_dir=output_dir, error=str(e)+ {}, prompt=prompt, url=url, output_dir=output_dir, error=e)real_stdout.write(json.dumps(payload) + "\n")@@ -1557,13 +1673,14 @@ def engineer(run_id, prompt, fresh, model, output_dir, no_interactive, as_json):interactive=interactive,)payload = _build_engineer_payload(result, run_id=run_id, prompt=prompt, fresh=fresh)- except KeyboardInterrupt:+ except KeyboardInterrupt as e:payload = _build_engineer_payload(- None, run_id=run_id, prompt=prompt, fresh=fresh, error="interrupted"+ None, run_id=run_id, prompt=prompt, fresh=fresh, error=e)except Exception as e:+ # Pass the exception object so _classify_error can use isinstance.payload = _build_engineer_payload(- None, run_id=run_id, prompt=prompt, fresh=fresh, error=str(e)+ None, run_id=run_id, prompt=prompt, fresh=fresh, error=e)real_stdout.write(json.dumps(payload) + "\n")diff --git a/tests/test_cli_agent_json.py b/tests/test_cli_agent_json.pyindex f15d27d..4bd7569 100644--- a/tests/test_cli_agent_json.py+++ b/tests/test_cli_agent_json.py@@ -28,6 +28,7 @@ EXPECTED_PAYLOAD_KEYS = {"script_path","usage","error",+ "error_kind",}@@ -40,20 +41,37 @@ class TestBuildAgentPayload:"run_id": "abc123","mode": "auto","script_path": str(tmp_path / "scripts" / "api_client.py"),- "usage": {"input_tokens": 1, "output_tokens": 2, "total_cost": 0.001},+ # Mix of Claude SDK keys to exercise normalization (cache_creation_input_tokens+ # → cache_write_tokens, estimated_cost_usd → total_cost_usd).+ "usage": {+ "input_tokens": 1,+ "output_tokens": 2,+ "cache_creation_input_tokens": 100,+ "cache_read_input_tokens": 50,+ "estimated_cost_usd": 0.001,+ },},prompt="capture the X api",url="https://example.com",)assert payload["schema_version"] == AGENT_JSON_SCHEMA_VERSION+ assert set(payload.keys()) == EXPECTED_PAYLOAD_KEYS# No run produced a HAR yet on disk in this test, so har_path is Noneassert payload["status"] == "ok"assert payload["run_id"] == "abc123"assert payload["mode"] == "auto"assert payload["prompt"] == "capture the X api"assert payload["url"] == "https://example.com"- assert payload["usage"]["total_cost"] == 0.001+ # Stable normalized usage subset+ assert payload["usage"]["input_tokens"] == 1+ assert payload["usage"]["output_tokens"] == 2+ assert payload["usage"]["cache_write_tokens"] == 100+ assert payload["usage"]["cache_read_tokens"] == 50+ assert payload["usage"]["total_cost_usd"] == 0.001+ # Raw SDK shape still available for power users+ assert payload["usage"]["raw"]["cache_creation_input_tokens"] == 100assert payload["error"] is None+ assert payload["error_kind"] is None# Must be JSON-serializable (no Path objects sneaking through)json.dumps(payload)@@ -62,6 +80,7 @@ class TestBuildAgentPayload:assert payload["status"] == "error"assert payload["error"]assert payload["run_id"] is None+ assert payload["error_kind"] == "engine_failure"def test_explicit_error_overrides(self):payload = _build_agent_payload(diff --git a/tests/test_cli_followups.py b/tests/test_cli_followups.pyindex c4d99af..e359da5 100644--- a/tests/test_cli_followups.py+++ b/tests/test_cli_followups.py@@ -27,6 +27,7 @@ EXPECTED_ENGINEER_KEYS = {"script_path","usage","error",+ "error_kind",}@@ -78,7 +79,10 @@ class TestBuildEngineerPayload:def test_success_shape(self):payload = _build_engineer_payload(- {"script_path": "/abs/api_client.py", "usage": {"total_cost": 0.001}},+ {+ "script_path": "/abs/api_client.py",+ "usage": {"input_tokens": 1, "output_tokens": 2, "total_cost": 0.001},+ },run_id="abc123",prompt="add pagination",fresh=False,@@ -89,14 +93,18 @@ class TestBuildEngineerPayload:assert payload["prompt"] == "add pagination"assert payload["fresh"] is Falseassert payload["script_path"] == "/abs/api_client.py"- assert payload["usage"]["total_cost"] == 0.001+ # `total_cost` (legacy/Copilot key) is normalized to `total_cost_usd`+ assert payload["usage"]["total_cost_usd"] == 0.001+ assert payload["usage"]["raw"]["total_cost"] == 0.001assert payload["error"] is None+ assert payload["error_kind"] is Noneassert set(payload.keys()) == EXPECTED_ENGINEER_KEYSdef test_none_result_is_error(self):payload = _build_engineer_payload(None, run_id="abc", prompt=None, fresh=False)assert payload["status"] == "error"assert payload["error"]+ assert payload["error_kind"] == "engine_failure"assert set(payload.keys()) == EXPECTED_ENGINEER_KEYSdef test_explicit_error_overrides(self):@@ -191,6 +199,115 @@ class TestEngineerCommandJson:assert result.exit_code == 0, result.output+class TestSchemaV2Normalization:+ """v2 helpers: _normalize_usage, _classify_error, --json-schema-version."""++ def test_normalize_usage_picks_stable_keys(self):+ """Claude SDK emits cache_creation_input_tokens / estimated_cost_usd;+ Copilot/OpenCode use different keys. Normalization gives a stable+ subset and parks everything under .raw."""+ from reverse_api.cli import _normalize_usage++ out = _normalize_usage({+ "input_tokens": 10,+ "output_tokens": 20,+ "cache_creation_input_tokens": 100,+ "cache_read_input_tokens": 200,+ "estimated_cost_usd": 0.05,+ "service_tier": "standard",+ "iterations": [],+ })+ assert out["input_tokens"] == 10+ assert out["output_tokens"] == 20+ assert out["cache_write_tokens"] == 100+ assert out["cache_read_tokens"] == 200+ assert out["total_cost_usd"] == 0.05+ # SDK extras are preserved under raw, not promoted to top level+ assert out["raw"]["service_tier"] == "standard"+ assert out["raw"]["iterations"] == []+ assert "service_tier" not in out # stable subset only+ assert "iterations" not in out++ def test_normalize_usage_alt_legacy_keys(self):+ """`total_cost` (Copilot) and direct `cache_read_tokens` (already-normalized+ input) map cleanly through the same pipeline."""+ from reverse_api.cli import _normalize_usage++ out = _normalize_usage({"total_cost": 0.42, "cache_read_tokens": 5})+ assert out["total_cost_usd"] == 0.42+ assert out["cache_read_tokens"] == 5++ def test_normalize_usage_empty(self):+ from reverse_api.cli import _normalize_usage++ assert _normalize_usage(None) == {}+ assert _normalize_usage({}) == {}+ assert _normalize_usage("not a dict") == {}++ def test_classify_error_kinds(self):+ from reverse_api.cli import _classify_error++ assert _classify_error(None) is None+ assert _classify_error(KeyboardInterrupt()) == "interrupted"+ assert _classify_error(PermissionError("nope")) == "permission_denied"+ assert _classify_error(ConnectionError("DNS failed")) == "network"+ assert _classify_error(TimeoutError("timed out")) == "network"+ assert _classify_error("[Errno 13] Permission denied: '/x'") == "permission_denied"+ assert _classify_error("--prompt is required in non-interactive/--json mode") == "misuse"+ assert _classify_error("agent capture produced no run") == "engine_failure"+ assert _classify_error("connection refused") == "network"+ assert _classify_error("totally unrecognized message") == "unknown"+ # Caller can override the default+ assert _classify_error("totally unrecognized message", default="engine_failure") == "engine_failure"++ def test_misuse_payload_has_misuse_kind(self):+ """`agent --json` without --prompt → error_kind=misuse (not unknown)."""+ from reverse_api.cli import agent++ runner = CliRunner()+ result = runner.invoke(agent, ["--json"])+ payload = json.loads(result.stdout.strip())+ assert payload["error_kind"] == "misuse"++ def test_keyboard_interrupt_payload_has_interrupted_kind(self):+ from reverse_api.cli import agent++ runner = CliRunner()+ with patch("reverse_api.cli.run_agent_capture", side_effect=KeyboardInterrupt):+ result = runner.invoke(agent, ["--json", "-p", "x"])+ payload = json.loads(result.stdout.strip().splitlines()[-1])+ assert payload["error_kind"] == "interrupted"+ assert payload["error"] == "interrupted"++ def test_permission_error_payload_has_permission_denied_kind(self):+ from reverse_api.cli import agent++ runner = CliRunner()+ with patch(+ "reverse_api.cli.run_agent_capture",+ side_effect=PermissionError(13, "Permission denied", "/forbidden"),+ ):+ result = runner.invoke(agent, ["--json", "-p", "x"])+ payload = json.loads(result.stdout.strip().splitlines()[-1])+ assert payload["error_kind"] == "permission_denied"+++class TestJsonSchemaVersionFlag:+ """`--json-schema-version` exposes the version a wrapper can gate on."""++ def test_root_flag_emits_version(self):+ runner = CliRunner()+ result = runner.invoke(main, ["--json-schema-version"])+ assert result.exit_code == 0+ from reverse_api.cli import AGENT_JSON_SCHEMA_VERSION as v+ assert result.stdout.strip() == str(v)++ def test_root_flag_advertised_in_help(self):+ runner = CliRunner()+ result = runner.invoke(main, ["--help"])+ assert "--json-schema-version" in result.output++class TestRootHelpMentionsScripted:"""Item #6 partial: root --help should advertise scripted features."""