Skip to content

The agent that turns websites into APIs!

feat(cli): stable usage normalization, error_kind enum, --json-schema-version

Kalil Bouziguesac5282fce78e
diff --git a/src/reverse_api/cli.py b/src/reverse_api/cli.py
index be94944..5b59910 100644
--- a/src/reverse_api/cli.py
+++ b/src/reverse_api/cli.py
@@ -54,9 +54,99 @@ console = Console()
config_manager = ConfigManager(get_config_path())
session_manager = SessionManager(get_history_path())
-# Stable schema version for --json outputs consumed by other agents/scripts.
+# Schema version for --json outputs. Wrappers can query it via
+# `reverse-api-engineer --json-schema-version`.
AGENT_JSON_SCHEMA_VERSION = 1
+# Map of stable usage keys → SDK-specific candidates (first match wins).
+# Lets `agent --json` and `engineer --json` emit a stable cost/token shape
+# regardless of which SDK (Claude / OpenCode / Copilot) ran underneath.
+_STABLE_USAGE_KEYS: dict[str, tuple[str, ...]] = {
+ "input_tokens": ("input_tokens",),
+ "output_tokens": ("output_tokens",),
+ "cache_read_tokens": ("cache_read_input_tokens", "cache_read_tokens"),
+ "cache_write_tokens": ("cache_creation_input_tokens", "cache_write_tokens"),
+ "total_cost_usd": ("estimated_cost_usd", "total_cost_usd", "total_cost"),
+}
+
+
+def _normalize_usage(raw: dict | None) -> dict:
+ """Return a stable subset of usage fields, keeping the SDK-native dict under .raw.
+
+ Wrappers can rely on the top-level keys; per-SDK extras stay under raw.
+ """
+ if not raw or not isinstance(raw, dict):
+ return {}
+ out: dict = {}
+ for stable_key, candidates in _STABLE_USAGE_KEYS.items():
+ for c in candidates:
+ if c in raw:
+ out[stable_key] = raw[c]
+ break
+ out["raw"] = raw
+ return out
+
+
+# Machine-readable error categories. Wrappers can react differently to each
+# without pattern-matching on the human-readable `error` string.
+ERROR_KINDS = (
+ "misuse", # user input invalid / missing required arg
+ "config_invalid", # config file or env var malformed
+ "permission_denied", # filesystem / API perm denied
+ "network", # DNS / TCP / TLS / timeout
+ "engine_failure", # SDK or capture engine crashed mid-run
+ "interrupted", # KeyboardInterrupt / SIGINT
+ "unknown", # default fallback
+)
+
+
+def _format_error_message(error: str | BaseException | None) -> str | None:
+ """Render an exception or string into a human-readable error message.
+
+ KeyboardInterrupt has no useful str() — we return the conventional
+ "interrupted" so downstream wrappers can match on a stable message.
+ Empty exception messages fall back to the class name.
+ """
+ if error is None:
+ return None
+ if isinstance(error, KeyboardInterrupt):
+ return "interrupted"
+ if isinstance(error, BaseException):
+ return str(error) or type(error).__name__
+ return error
+
+
+def _classify_error(error: str | BaseException | None, *, default: str = "unknown") -> str | None:
+ """Map an Exception or human error message to one of the ERROR_KINDS.
+
+ Callers may pass a stronger hint by setting `default` (e.g. `misuse` from
+ a CLI argument check, before any exception).
+ """
+ if error is None:
+ return None
+ if isinstance(error, BaseException):
+ if isinstance(error, KeyboardInterrupt):
+ return "interrupted"
+ if isinstance(error, PermissionError):
+ return "permission_denied"
+ if isinstance(error, (ConnectionError, TimeoutError)):
+ return "network"
+ msg = str(error)
+ else:
+ msg = error
+ low = msg.lower()
+ if "permission denied" in low or "errno 13" in low:
+ return "permission_denied"
+ if "interrupted" in low:
+ return "interrupted"
+ if "is required" in low or "missing required" in low or "no such option" in low or "in non-interactive" in low:
+ return "misuse"
+ if any(kw in low for kw in ("connection refused", "timeout", "timed out", "dns", "network", "unreachable", "ssl")):
+ return "network"
+ if "not found" in low or "no run" in low or "produced no run" in low or "produced no result" in low:
+ return "engine_failure"
+ return default
+
@contextmanager
def _quiet_consoles_for_json():
@@ -91,7 +181,8 @@ def _build_agent_payload(
prompt: str | None,
url: str | None,
output_dir: str | None = None,
- error: str | None = None,
+ error: str | BaseException | None = None,
+ error_kind_hint: str = "unknown",
) -> dict:
"""Normalize an agent capture result into a stable JSON shape.
@@ -101,22 +192,27 @@ def _build_agent_payload(
result = result or {}
run_id = result.get("run_id")
inner_error = result.get("error")
- final_error = error or inner_error or (None if run_id else "agent capture produced no run")
+ final_error_obj = error if error is not None else inner_error
+ if final_error_obj is None and not run_id:
+ final_error_obj = "agent capture produced no run"
+ error_str = _format_error_message(final_error_obj)
+ error_kind = _classify_error(final_error_obj, default=error_kind_hint) if final_error_obj else None
har_path = None
if run_id:
candidate = get_har_dir(run_id, output_dir) / "recording.har"
har_path = str(candidate) if candidate.exists() else None
return {
"schema_version": AGENT_JSON_SCHEMA_VERSION,
- "status": "error" if final_error else "ok",
+ "status": "error" if error_str else "ok",
"run_id": run_id,
"prompt": prompt,
"url": url,
"mode": result.get("mode"),
"har_path": har_path,
"script_path": result.get("script_path"),
- "usage": result.get("usage") or {},
- "error": final_error,
+ "usage": _normalize_usage(result.get("usage")),
+ "error": error_str,
+ "error_kind": error_kind,
}
@@ -126,7 +222,8 @@ def _build_engineer_payload(
run_id: str,
prompt: str | None,
fresh: bool,
- error: str | None = None,
+ error: str | BaseException | None = None,
+ error_kind_hint: str = "unknown",
) -> dict:
"""Normalize an engineer-mode result into a stable JSON shape.
@@ -135,18 +232,23 @@ def _build_engineer_payload(
to --prompt (which may have been used as either a full replacement or as
additional instructions depending on --fresh).
"""
- result = result if result is not None else {}
- inner_error = result.get("error") if isinstance(result, dict) else None
- final_error = error or inner_error or (None if result else "engineering produced no result")
+ result = result if isinstance(result, dict) else {}
+ inner_error = result.get("error")
+ final_error_obj = error if error is not None else inner_error
+ if final_error_obj is None and not result:
+ final_error_obj = "engineering produced no result"
+ error_str = _format_error_message(final_error_obj)
+ error_kind = _classify_error(final_error_obj, default=error_kind_hint) if final_error_obj else None
return {
"schema_version": AGENT_JSON_SCHEMA_VERSION,
- "status": "error" if final_error else "ok",
+ "status": "error" if error_str else "ok",
"run_id": run_id,
"prompt": prompt,
"fresh": fresh,
- "script_path": result.get("script_path") if isinstance(result, dict) else None,
- "usage": (result.get("usage") if isinstance(result, dict) else None) or {},
- "error": final_error,
+ "script_path": result.get("script_path"),
+ "usage": _normalize_usage(result.get("usage")),
+ "error": error_str,
+ "error_kind": error_kind,
}
# Mode definitions
@@ -416,15 +518,26 @@ CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]}
@click.group(invoke_without_command=True, context_settings=CONTEXT_SETTINGS)
@click.pass_context
@click.version_option(version=__version__)
-def main(ctx: click.Context):
+ "--json-schema-version",
+ "show_schema_version",
+ is_flag=True,
+ help="Print the agent/engineer JSON schema_version this binary emits and exit.",
+)
+def main(ctx: click.Context, show_schema_version: bool):
"""reverse-api-engineer: reverse engineer apis.
Run without a subcommand to start the interactive REPL; use agent, manual,
or engineer for CLI mode.
Most subcommands accept --json and --no-interactive for scripted use; see
- `<cmd> --help` for per-command details.
+ `<cmd> --help` for per-command details. Wrappers that need to gate on the
+ payload schema can call `reverse-api-engineer --json-schema-version`.
"""
+ if show_schema_version:
+ click.echo(str(AGENT_JSON_SCHEMA_VERSION))
+ ctx.exit(0)
+
if ctx.invoked_subcommand is None:
# Refuse to drop into the prompt_toolkit REPL when stdin is not a TTY:
# without an interactive terminal the REPL would block on stdin
@@ -1121,6 +1234,7 @@ def agent(prompt, url, model, output_dir, no_interactive, as_json, headless):
url=url,
output_dir=output_dir,
error="--prompt is required in non-interactive/--json mode",
+ error_kind_hint="misuse",
)
click.echo(json.dumps(misuse))
else:
@@ -1154,13 +1268,15 @@ def agent(prompt, url, model, output_dir, no_interactive, as_json, headless):
headless=headless,
)
payload = _build_agent_payload(result, prompt=prompt, url=url, output_dir=output_dir)
- except KeyboardInterrupt:
+ except KeyboardInterrupt as e:
payload = _build_agent_payload(
- {}, prompt=prompt, url=url, output_dir=output_dir, error="interrupted"
+ {}, prompt=prompt, url=url, output_dir=output_dir, error=e
)
except Exception as e:
+ # Pass the exception object so _classify_error can use isinstance
+ # (PermissionError → permission_denied, ConnectionError → network, ...)
payload = _build_agent_payload(
- {}, prompt=prompt, url=url, output_dir=output_dir, error=str(e)
+ {}, prompt=prompt, url=url, output_dir=output_dir, error=e
)
real_stdout.write(json.dumps(payload) + "\n")
@@ -1557,13 +1673,14 @@ def engineer(run_id, prompt, fresh, model, output_dir, no_interactive, as_json):
interactive=interactive,
)
payload = _build_engineer_payload(result, run_id=run_id, prompt=prompt, fresh=fresh)
- except KeyboardInterrupt:
+ except KeyboardInterrupt as e:
payload = _build_engineer_payload(
- None, run_id=run_id, prompt=prompt, fresh=fresh, error="interrupted"
+ None, run_id=run_id, prompt=prompt, fresh=fresh, error=e
)
except Exception as e:
+ # Pass the exception object so _classify_error can use isinstance.
payload = _build_engineer_payload(
- None, run_id=run_id, prompt=prompt, fresh=fresh, error=str(e)
+ None, run_id=run_id, prompt=prompt, fresh=fresh, error=e
)
real_stdout.write(json.dumps(payload) + "\n")
diff --git a/tests/test_cli_agent_json.py b/tests/test_cli_agent_json.py
index f15d27d..4bd7569 100644
--- a/tests/test_cli_agent_json.py
+++ b/tests/test_cli_agent_json.py
@@ -28,6 +28,7 @@ EXPECTED_PAYLOAD_KEYS = {
"script_path",
"usage",
"error",
+ "error_kind",
}
@@ -40,20 +41,37 @@ class TestBuildAgentPayload:
"run_id": "abc123",
"mode": "auto",
"script_path": str(tmp_path / "scripts" / "api_client.py"),
- "usage": {"input_tokens": 1, "output_tokens": 2, "total_cost": 0.001},
+ # Mix of Claude SDK keys to exercise normalization (cache_creation_input_tokens
+ # → cache_write_tokens, estimated_cost_usd → total_cost_usd).
+ "usage": {
+ "input_tokens": 1,
+ "output_tokens": 2,
+ "cache_creation_input_tokens": 100,
+ "cache_read_input_tokens": 50,
+ "estimated_cost_usd": 0.001,
+ },
},
prompt="capture the X api",
url="https://example.com",
)
assert payload["schema_version"] == AGENT_JSON_SCHEMA_VERSION
+ assert set(payload.keys()) == EXPECTED_PAYLOAD_KEYS
# No run produced a HAR yet on disk in this test, so har_path is None
assert payload["status"] == "ok"
assert payload["run_id"] == "abc123"
assert payload["mode"] == "auto"
assert payload["prompt"] == "capture the X api"
assert payload["url"] == "https://example.com"
- assert payload["usage"]["total_cost"] == 0.001
+ # Stable normalized usage subset
+ assert payload["usage"]["input_tokens"] == 1
+ assert payload["usage"]["output_tokens"] == 2
+ assert payload["usage"]["cache_write_tokens"] == 100
+ assert payload["usage"]["cache_read_tokens"] == 50
+ assert payload["usage"]["total_cost_usd"] == 0.001
+ # Raw SDK shape still available for power users
+ assert payload["usage"]["raw"]["cache_creation_input_tokens"] == 100
assert payload["error"] is None
+ assert payload["error_kind"] is None
# Must be JSON-serializable (no Path objects sneaking through)
json.dumps(payload)
@@ -62,6 +80,7 @@ class TestBuildAgentPayload:
assert payload["status"] == "error"
assert payload["error"]
assert payload["run_id"] is None
+ assert payload["error_kind"] == "engine_failure"
def test_explicit_error_overrides(self):
payload = _build_agent_payload(
diff --git a/tests/test_cli_followups.py b/tests/test_cli_followups.py
index c4d99af..e359da5 100644
--- a/tests/test_cli_followups.py
+++ b/tests/test_cli_followups.py
@@ -27,6 +27,7 @@ EXPECTED_ENGINEER_KEYS = {
"script_path",
"usage",
"error",
+ "error_kind",
}
@@ -78,7 +79,10 @@ class TestBuildEngineerPayload:
def test_success_shape(self):
payload = _build_engineer_payload(
- {"script_path": "/abs/api_client.py", "usage": {"total_cost": 0.001}},
+ {
+ "script_path": "/abs/api_client.py",
+ "usage": {"input_tokens": 1, "output_tokens": 2, "total_cost": 0.001},
+ },
run_id="abc123",
prompt="add pagination",
fresh=False,
@@ -89,14 +93,18 @@ class TestBuildEngineerPayload:
assert payload["prompt"] == "add pagination"
assert payload["fresh"] is False
assert payload["script_path"] == "/abs/api_client.py"
- assert payload["usage"]["total_cost"] == 0.001
+ # `total_cost` (legacy/Copilot key) is normalized to `total_cost_usd`
+ assert payload["usage"]["total_cost_usd"] == 0.001
+ assert payload["usage"]["raw"]["total_cost"] == 0.001
assert payload["error"] is None
+ assert payload["error_kind"] is None
assert set(payload.keys()) == EXPECTED_ENGINEER_KEYS
def test_none_result_is_error(self):
payload = _build_engineer_payload(None, run_id="abc", prompt=None, fresh=False)
assert payload["status"] == "error"
assert payload["error"]
+ assert payload["error_kind"] == "engine_failure"
assert set(payload.keys()) == EXPECTED_ENGINEER_KEYS
def test_explicit_error_overrides(self):
@@ -191,6 +199,115 @@ class TestEngineerCommandJson:
assert result.exit_code == 0, result.output
+class TestSchemaV2Normalization:
+ """v2 helpers: _normalize_usage, _classify_error, --json-schema-version."""
+
+ def test_normalize_usage_picks_stable_keys(self):
+ """Claude SDK emits cache_creation_input_tokens / estimated_cost_usd;
+ Copilot/OpenCode use different keys. Normalization gives a stable
+ subset and parks everything under .raw."""
+ from reverse_api.cli import _normalize_usage
+
+ out = _normalize_usage({
+ "input_tokens": 10,
+ "output_tokens": 20,
+ "cache_creation_input_tokens": 100,
+ "cache_read_input_tokens": 200,
+ "estimated_cost_usd": 0.05,
+ "service_tier": "standard",
+ "iterations": [],
+ })
+ assert out["input_tokens"] == 10
+ assert out["output_tokens"] == 20
+ assert out["cache_write_tokens"] == 100
+ assert out["cache_read_tokens"] == 200
+ assert out["total_cost_usd"] == 0.05
+ # SDK extras are preserved under raw, not promoted to top level
+ assert out["raw"]["service_tier"] == "standard"
+ assert out["raw"]["iterations"] == []
+ assert "service_tier" not in out # stable subset only
+ assert "iterations" not in out
+
+ def test_normalize_usage_alt_legacy_keys(self):
+ """`total_cost` (Copilot) and direct `cache_read_tokens` (already-normalized
+ input) map cleanly through the same pipeline."""
+ from reverse_api.cli import _normalize_usage
+
+ out = _normalize_usage({"total_cost": 0.42, "cache_read_tokens": 5})
+ assert out["total_cost_usd"] == 0.42
+ assert out["cache_read_tokens"] == 5
+
+ def test_normalize_usage_empty(self):
+ from reverse_api.cli import _normalize_usage
+
+ assert _normalize_usage(None) == {}
+ assert _normalize_usage({}) == {}
+ assert _normalize_usage("not a dict") == {}
+
+ def test_classify_error_kinds(self):
+ from reverse_api.cli import _classify_error
+
+ assert _classify_error(None) is None
+ assert _classify_error(KeyboardInterrupt()) == "interrupted"
+ assert _classify_error(PermissionError("nope")) == "permission_denied"
+ assert _classify_error(ConnectionError("DNS failed")) == "network"
+ assert _classify_error(TimeoutError("timed out")) == "network"
+ assert _classify_error("[Errno 13] Permission denied: '/x'") == "permission_denied"
+ assert _classify_error("--prompt is required in non-interactive/--json mode") == "misuse"
+ assert _classify_error("agent capture produced no run") == "engine_failure"
+ assert _classify_error("connection refused") == "network"
+ assert _classify_error("totally unrecognized message") == "unknown"
+ # Caller can override the default
+ assert _classify_error("totally unrecognized message", default="engine_failure") == "engine_failure"
+
+ def test_misuse_payload_has_misuse_kind(self):
+ """`agent --json` without --prompt → error_kind=misuse (not unknown)."""
+ from reverse_api.cli import agent
+
+ runner = CliRunner()
+ result = runner.invoke(agent, ["--json"])
+ payload = json.loads(result.stdout.strip())
+ assert payload["error_kind"] == "misuse"
+
+ def test_keyboard_interrupt_payload_has_interrupted_kind(self):
+ from reverse_api.cli import agent
+
+ runner = CliRunner()
+ with patch("reverse_api.cli.run_agent_capture", side_effect=KeyboardInterrupt):
+ result = runner.invoke(agent, ["--json", "-p", "x"])
+ payload = json.loads(result.stdout.strip().splitlines()[-1])
+ assert payload["error_kind"] == "interrupted"
+ assert payload["error"] == "interrupted"
+
+ def test_permission_error_payload_has_permission_denied_kind(self):
+ from reverse_api.cli import agent
+
+ runner = CliRunner()
+ with patch(
+ "reverse_api.cli.run_agent_capture",
+ side_effect=PermissionError(13, "Permission denied", "/forbidden"),
+ ):
+ result = runner.invoke(agent, ["--json", "-p", "x"])
+ payload = json.loads(result.stdout.strip().splitlines()[-1])
+ assert payload["error_kind"] == "permission_denied"
+
+
+class TestJsonSchemaVersionFlag:
+ """`--json-schema-version` exposes the version a wrapper can gate on."""
+
+ def test_root_flag_emits_version(self):
+ runner = CliRunner()
+ result = runner.invoke(main, ["--json-schema-version"])
+ assert result.exit_code == 0
+ from reverse_api.cli import AGENT_JSON_SCHEMA_VERSION as v
+ assert result.stdout.strip() == str(v)
+
+ def test_root_flag_advertised_in_help(self):
+ runner = CliRunner()
+ result = runner.invoke(main, ["--help"])
+ assert "--json-schema-version" in result.output
+
+
class TestRootHelpMentionsScripted:
"""Item #6 partial: root --help should advertise scripted features."""