The agent that turns websites into APIs!
Refactor path validation and add defense-in-depth checks (#98)
kalil032102fc798318b6
diff --git a/src/reverse_api/cli.py b/src/reverse_api/cli.pyindex 08a5c21..5e152ce 100644--- a/src/reverse_api/cli.py+++ b/src/reverse_api/cli.py@@ -2637,6 +2637,26 @@ def show_run(run_id, as_json):console.print(table)+def _extract_missing_module(stderr: str) -> str | None:+ """Extract a missing top-level module name from a ModuleNotFoundError message.++ Returns None if no module name is found or the name is not a plain Python+ identifier — a genuine ModuleNotFoundError always names one, and rejecting+ anything else prevents a crafted error message from smuggling pip options+ or arbitrary install targets.+ """+ import re as _re++ match = _re.search(r"No module named ['\"]([^'\"]+)['\"]", stderr)+ if not match:+ return None+ missing = match.group(1).split(".")[0]+ # fullmatch, since $ in re.match would accept a trailing newline+ if not _re.fullmatch(r"[a-zA-Z0-9_]+", missing) or len(missing) > 64:+ return None+ return missing++def _run_script_machine_payload(*,identifier: str,@@ -2647,7 +2667,6 @@ def _run_script_machine_payload(emit_event,) -> dict:"""Run or list generated scripts without prompts and with JSON-safe stdout."""- import re as _reimport subprocessrun_id: str | None = None@@ -2730,9 +2749,8 @@ def _run_script_machine_payload(stderr = result.stderr or ""if result.returncode != 0 and "ModuleNotFoundError: No module named" in stderr:- match = _re.search(r"No module named ['\"]([^'\"]+)['\"]", stderr)- if match:- missing = match.group(1).split(".")[0]+ missing = _extract_missing_module(stderr)+ if missing:emit_event("dependency_missing", run_id=run_id, package=missing)if auto_install:emit_event("dependency_install_started", run_id=run_id, package=missing)@@ -2753,6 +2771,21 @@ def _run_script_machine_payload(error=f"missing dependency: {missing}",error_kind_hint="engine_failure",)+ elif auto_install:+ # Keep the refusal observable instead of falling through to a+ # generic script-exit error+ return _build_run_payload(+ identifier=identifier,+ run_id=run_id,+ script_path=str(script),+ script_args=script_args,+ returncode=result.returncode,+ stdout=result.stdout or "",+ stderr=stderr,+ scripts=scripts,+ error="refusing to auto-install: no safe package name could be extracted from the ModuleNotFoundError output",+ error_kind_hint="engine_failure",+ )return _build_run_payload(identifier=identifier,@@ -2967,11 +3000,8 @@ def run_script(ctx, identifier, script_args, file_name, list_scripts, no_interacif result.returncode != 0:if "ModuleNotFoundError: No module named" in (result.stderr or ""):- import re as _re-- match = _re.search(r"No module named ['\"]([^'\"]+)['\"]", result.stderr)- if match:- missing = match.group(1).split(".")[0]+ missing = _extract_missing_module(result.stderr)+ if missing:console.print(f"[yellow]Missing dependency: {missing}[/yellow]")if auto_install:diff --git a/src/reverse_api/collector.py b/src/reverse_api/collector.pyindex d061525..8e9372a 100644--- a/src/reverse_api/collector.py+++ b/src/reverse_api/collector.py@@ -69,7 +69,9 @@ class Collector:self.ui.header(self.run_id, self.prompt, self.model, mode="collector")self.ui.start_collecting()- self._folder_name = generate_folder_name(self.prompt)+ # A punctuation-only prompt can slugify to an empty string, which+ # get_collected_dir rejects — fall back to the (validated) run_id+ self._folder_name = generate_folder_name(self.prompt) or self.run_idself._collected_dir = get_collected_dir(self._folder_name)self.items_path = self._collected_dir / "items.jsonl"diff --git a/src/reverse_api/utils.py b/src/reverse_api/utils.pyindex 3cb0dc9..a5f85f5 100644--- a/src/reverse_api/utils.py+++ b/src/reverse_api/utils.py@@ -289,6 +289,30 @@ def get_base_output_dir(output_dir: str | None = None) -> Path:return get_app_dir() / "runs"+def _validate_path_component(value: str, label: str = "run_id") -> None:+ """Validate a user-supplied path component to prevent path traversal attacks.++ Args:+ value: The path component to validate+ label: Name of the parameter, used in error messages++ Raises:+ ValueError: If the value is empty, contains invalid characters, or is too long+ """+ if not value:+ raise ValueError(f"{label} cannot be empty")++ # Allow alphanumeric characters, hyphens, and underscores+ # Chrome extension IDs start with crx- followed by UUID-style format+ # (fullmatch, since $ in re.match would accept a trailing newline)+ if not re.fullmatch(r"[a-zA-Z0-9_-]+", value):+ raise ValueError(f"Invalid {label}: {value}. Only alphanumeric characters, hyphens, and underscores are allowed")++ # Limit length to prevent extremely long paths+ if len(value) > 64:+ raise ValueError(f"{label} too long: {len(value)} characters (max 64)")++def get_har_dir(run_id: str, output_dir: str | None = None) -> Path:"""Get the HAR directory for a specific run.@@ -302,18 +326,7 @@ def get_har_dir(run_id: str, output_dir: str | None = None) -> Path:Raises:ValueError: If run_id contains invalid characters or attempts path traversal"""- # Validate run_id to prevent path traversal attacks- if not run_id:- raise ValueError("run_id cannot be empty")-- # Allow alphanumeric characters, hyphens, and underscores- # Chrome extension IDs start with crx- followed by UUID-style format- if not re.match(r"^[a-zA-Z0-9_-]+$", run_id):- raise ValueError(f"Invalid run_id: {run_id}. Only alphanumeric characters, hyphens, and underscores are allowed")-- # Limit length to prevent extremely long paths- if len(run_id) > 64:- raise ValueError(f"run_id too long: {len(run_id)} characters (max 64)")+ _validate_path_component(run_id)base_dir = get_base_output_dir(output_dir)har_dir = base_dir / "har" / run_id@@ -344,18 +357,7 @@ def get_scripts_dir(run_id: str, output_dir: str | None = None) -> Path:Raises:ValueError: If run_id contains invalid characters or attempts path traversal"""- # Validate run_id to prevent path traversal attacks- if not run_id:- raise ValueError("run_id cannot be empty")-- # Allow alphanumeric characters, hyphens, and underscores- # Chrome extension IDs start with crx- followed by UUID-style format- if not re.match(r"^[a-zA-Z0-9_-]+$", run_id):- raise ValueError(f"Invalid run_id: {run_id}. Only alphanumeric characters, hyphens, and underscores are allowed")-- # Limit length to prevent extremely long paths- if len(run_id) > 64:- raise ValueError(f"run_id too long: {len(run_id)} characters (max 64)")+ _validate_path_component(run_id)base_dir = get_base_output_dir(output_dir)scripts_dir = base_dir / "scripts" / run_id@@ -386,18 +388,7 @@ def get_docs_dir(run_id: str, output_dir: str | None = None) -> Path:Raises:ValueError: If run_id contains invalid characters or attempts path traversal"""- # Validate run_id to prevent path traversal attacks- if not run_id:- raise ValueError("run_id cannot be empty")-- # Allow alphanumeric characters, hyphens, and underscores- # Chrome extension IDs start with crx- followed by UUID-style format- if not re.match(r"^[a-zA-Z0-9_-]+$", run_id):- raise ValueError(f"Invalid run_id: {run_id}. Only alphanumeric characters, hyphens, and underscores are allowed")-- # Limit length to prevent extremely long paths- if len(run_id) > 64:- raise ValueError(f"run_id too long: {len(run_id)} characters (max 64)")+ _validate_path_component(run_id)base_dir = get_base_output_dir(output_dir)docs_dir = base_dir / "docs" / run_id@@ -416,11 +407,33 @@ def get_docs_dir(run_id: str, output_dir: str | None = None) -> Path:def get_messages_path(run_id: str, output_dir: str | None = None) -> Path:- """Get the messages file path for a specific run."""+ """Get the messages file path for a specific run.++ Args:+ run_id: Run identifier (must be alphanumeric with hyphens/underscores only)+ output_dir: Optional custom output directory++ Returns:+ Path to the messages file++ Raises:+ ValueError: If run_id contains invalid characters or attempts path traversal+ """+ _validate_path_component(run_id)+base_dir = get_base_output_dir(output_dir)messages_dir = base_dir / "messages"+ messages_path = messages_dir / f"{run_id}.jsonl"++ # Verify the resolved path is within the base directory (defense in depth)+ try:+ if not messages_path.resolve().is_relative_to(base_dir.resolve()):+ raise ValueError(f"Path traversal detected: {run_id}")+ except (OSError, RuntimeError) as e:+ raise ValueError(f"Invalid path for run_id {run_id}: {e}") from e+messages_dir.mkdir(parents=True, exist_ok=True)- return messages_dir / f"{run_id}.jsonl"+ return messages_pathdef get_timestamp() -> str:@@ -432,12 +445,27 @@ def get_collected_dir(folder_name: str) -> Path:"""Get ./collected/{folder_name}/ directory for collector mode output.Args:- folder_name: Name of the collection folder+ folder_name: Name of the collection folder (must be alphanumeric with+ hyphens/underscores only)Returns:Path to the collection output directory++ Raises:+ ValueError: If folder_name contains invalid characters or attempts path traversal"""- path = Path.cwd() / "collected" / folder_name+ _validate_path_component(folder_name, "folder_name")++ base_dir = Path.cwd() / "collected"+ path = base_dir / folder_name++ # Verify the resolved path is within the base directory (defense in depth)+ try:+ if not path.resolve().is_relative_to(base_dir.resolve()):+ raise ValueError(f"Path traversal detected: {folder_name}")+ except (OSError, RuntimeError) as e:+ raise ValueError(f"Invalid path for folder_name {folder_name}: {e}") from e+path.mkdir(parents=True, exist_ok=True)return pathdiff --git a/tests/test_cli_agent_json.py b/tests/test_cli_agent_json.pyindex 88799cb..3881e4f 100644--- a/tests/test_cli_agent_json.py+++ b/tests/test_cli_agent_json.py@@ -392,6 +392,31 @@ class TestRunCommandJson:assert {s["name"] for s in payload["scripts"]} == {"api_client.py", "example_usage.py"}mock_select.assert_not_called()+ def test_run_json_auto_install_refuses_invalid_package_name(self, multi_script_env):+ """A crafted ModuleNotFoundError must not smuggle pip options into auto-install."""+ tmp_path, _ = multi_script_env+ self._precreate_venv(tmp_path)++ runner = CliRunner()+ crafted = MagicMock(+ returncode=1,+ stdout="",+ stderr="ModuleNotFoundError: No module named '--index-url=https://evil.example'",+ )+ with patch("subprocess.run", return_value=crafted) as mock_sub:+ result = runner.invoke(+ main,+ ["run", "abc123def456", "--file", "api_client.py", "--json", "--auto-install"],+ )++ assert result.exit_code == 1+ payload = json.loads(result.stdout.strip())+ assert payload["status"] == "error"+ assert "refusing to auto-install" in payload["error"]+ # The crafted token must never appear in any subprocess invocation+ for call in mock_sub.call_args_list:+ assert "--index-url=https://evil.example" not in [str(a) for a in call.args[0]]+def test_run_json_stream_emits_events_and_result(self, multi_script_env):tmp_path, _ = multi_script_envself._precreate_venv(tmp_path)diff --git a/tests/test_run_command.py b/tests/test_run_command.pyindex 72fbce4..bc6ad1b 100644--- a/tests/test_run_command.py+++ b/tests/test_run_command.py@@ -637,3 +637,50 @@ class TestRunCommandOutputMessages:result = cli_runner.invoke(main, ["run", "def789ghi012"])assert "Setting up shared venv" in result.outputassert "Shared venv ready" in result.output+++# ---------------------------------------------------------------------------+# _extract_missing_module+# ---------------------------------------------------------------------------++class TestExtractMissingModule:+ """Test _extract_missing_module validation against crafted error messages."""++ def test_plain_module_name(self):+ from reverse_api.cli import _extract_missing_module+ stderr = "ModuleNotFoundError: No module named 'requests'"+ assert _extract_missing_module(stderr) == "requests"++ def test_dotted_module_returns_top_level(self):+ from reverse_api.cli import _extract_missing_module+ stderr = "ModuleNotFoundError: No module named 'bs4.element'"+ assert _extract_missing_module(stderr) == "bs4"++ def test_no_match_returns_none(self):+ from reverse_api.cli import _extract_missing_module+ assert _extract_missing_module("SyntaxError: invalid syntax") is None++ def test_pip_option_injection_rejected(self):+ from reverse_api.cli import _extract_missing_module+ stderr = "ModuleNotFoundError: No module named '--index-url=https://evil.example'"+ assert _extract_missing_module(stderr) is None++ def test_spaces_rejected(self):+ from reverse_api.cli import _extract_missing_module+ stderr = "ModuleNotFoundError: No module named '-q evil-package'"+ assert _extract_missing_module(stderr) is None++ def test_hyphen_rejected(self):+ from reverse_api.cli import _extract_missing_module+ stderr = "ModuleNotFoundError: No module named 'evil-package'"+ assert _extract_missing_module(stderr) is None++ def test_too_long_rejected(self):+ from reverse_api.cli import _extract_missing_module+ stderr = f"ModuleNotFoundError: No module named '{'a' * 65}'"+ assert _extract_missing_module(stderr) is None++ def test_trailing_newline_rejected(self):+ from reverse_api.cli import _extract_missing_module+ stderr = "ModuleNotFoundError: No module named 'requests\n'"+ assert _extract_missing_module(stderr) is Nonediff --git a/tests/test_utils.py b/tests/test_utils.pyindex b26ae63..55a5fa2 100644--- a/tests/test_utils.py+++ b/tests/test_utils.py@@ -133,6 +133,22 @@ class TestPathHelpers:path = get_messages_path("run123")assert path == tmp_path / "messages" / "run123.jsonl"+ def test_get_messages_path_empty_run_id_raises(self):+ """Empty run_id raises ValueError."""+ with pytest.raises(ValueError, match="cannot be empty"):+ get_messages_path("")++ def test_get_messages_path_traversal_raises(self, tmp_path):+ """Path traversal in run_id raises ValueError."""+ with patch("reverse_api.utils.get_base_output_dir", return_value=tmp_path):+ with pytest.raises(ValueError, match="Invalid run_id"):+ get_messages_path("../../../etc/passwd")++ def test_get_messages_path_too_long_raises(self):+ """Too long run_id raises ValueError."""+ with pytest.raises(ValueError, match="too long"):+ get_messages_path("x" * 65)+class TestGetHarDir:"""Test get_har_dir with validation."""@@ -181,6 +197,11 @@ class TestGetHarDir:with pytest.raises(ValueError, match="too long"):get_har_dir("a" * 65)+ def test_trailing_newline_raises(self):+ """A trailing newline in run_id raises ValueError."""+ with pytest.raises(ValueError, match="Invalid run_id"):+ get_har_dir("run123\n")+def test_custom_output_dir(self, tmp_path):"""Custom output_dir is used."""har_dir = get_har_dir("run123", output_dir=str(tmp_path))@@ -259,6 +280,22 @@ class TestGetCollectedDir:assert path.exists()assert path.name == "test_collection"+ def test_empty_folder_name_raises(self):+ """Empty folder_name raises ValueError."""+ with pytest.raises(ValueError, match="cannot be empty"):+ get_collected_dir("")++ def test_traversal_raises(self, tmp_path):+ """Path traversal in folder_name raises ValueError."""+ with patch("reverse_api.utils.Path.cwd", return_value=tmp_path):+ with pytest.raises(ValueError, match="Invalid folder_name"):+ get_collected_dir("../../etc")++ def test_too_long_raises(self):+ """Too long folder_name raises ValueError."""+ with pytest.raises(ValueError, match="too long"):+ get_collected_dir("x" * 65)+class TestCheckForUpdates:"""Test check_for_updates function."""diff --git a/uv.lock b/uv.lockindex 0425af3..e0faf81 100644--- a/uv.lock+++ b/uv.lock@@ -1558,7 +1558,7 @@ wheels = [[[package]]name = "reverse-api-engineer"-version = "0.9.0"+version = "0.10.0"source = { editable = "." }dependencies = [{ name = "aiohttp" },