Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat: Add option to set executable path in the CLI

Karim shoairbea8c71fc355
diff --git a/scrapling/cli.py b/scrapling/cli.py
index 942848f..0721d21 100644
--- a/scrapling/cli.py
+++ b/scrapling/cli.py
@@ -1,3 +1,4 @@
+from os import environ
from pathlib import Path
from subprocess import check_output
from sys import executable as python_executable
@@ -272,6 +273,12 @@ def _common_browser_options(f):
default=False,
help="Extract only main content and sanitize hidden elements for AI consumption (default: False)",
),
+ option(
+ "--executable-path",
+ type=str,
+ default=None,
+ help="Path to a custom Chromium-compatible browser executable. Falls back to the SCRAPLING_EXECUTABLE_PATH environment variable when not set.",
+ ),
option(
"--extra-headers",
"-H",
@@ -519,6 +526,7 @@ def __build_browser_kwargs(
parsed_headers,
dns_over_https,
block_ads,
+ executable_path,
) -> Dict[str, Any]:
"""Build shared kwargs dict for browser-based commands."""
kwargs: Dict[str, Any] = {
@@ -539,6 +547,9 @@ def __build_browser_kwargs(
kwargs["proxy"] = proxy
if parsed_headers:
kwargs["extra_headers"] = parsed_headers
+ executable_path = executable_path or environ.get("SCRAPLING_EXECUTABLE_PATH") or None
+ if executable_path:
+ kwargs["executable_path"] = executable_path
return kwargs
@@ -561,6 +572,7 @@ def fetch(
proxy,
extra_headers,
ai_targeted,
+ executable_path,
dns_over_https,
block_ads,
):
@@ -579,6 +591,7 @@ def fetch(
parsed_headers,
dns_over_https,
block_ads,
+ executable_path,
)
from scrapling.fetchers import DynamicFetcher
@@ -624,6 +637,7 @@ def stealthy_fetch(
allow_webgl,
hide_canvas,
ai_targeted,
+ executable_path,
dns_over_https,
block_ads,
):
@@ -642,6 +656,7 @@ def stealthy_fetch(
parsed_headers,
dns_over_https,
block_ads,
+ executable_path,
)
kwargs.update(
{
diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py
index 77dc81d..76ce660 100644
--- a/scrapling/engines/_browsers/_validators.py
+++ b/scrapling/engines/_browsers/_validators.py
@@ -26,15 +26,15 @@ from scrapling.engines._browsers._types import PlaywrightFetchParams, StealthFet
# Custom validators for msgspec
@lru_cache(8)
-def _is_invalid_file_path(value: str) -> bool | str: # pragma: no cover
+def _is_invalid_file_path(value: str, label: str = "Init script") -> bool | str: # pragma: no cover
"""Fast file path validation"""
path = Path(value)
if not path.exists():
- return f"Init script path not found: {value}"
+ return f"{label} path not found: {value}"
if not path.is_file():
- return f"Init script is not a file: {value}"
+ return f"{label} is not a file: {value}"
if not path.is_absolute():
- return f"Init script is not a absolute path: {value}"
+ return f"{label} is not an absolute path: {value}"
return False
@@ -128,7 +128,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
raise ValueError(validation_msg)
if self.executable_path is not None:
- validation_msg = _is_invalid_file_path(self.executable_path)
+ validation_msg = _is_invalid_file_path(self.executable_path, "Browser executable")
if validation_msg:
raise ValueError(validation_msg)
diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py
index ae60e53..42d0d4d 100644
--- a/tests/cli/test_cli.py
+++ b/tests/cli/test_cli.py
@@ -5,9 +5,7 @@ import pytest_httpbin
from scrapling.parser import Selector
from scrapling import __version__
-from scrapling.cli import (
- main, shell, mcp, get, post, put, delete, fetch, stealthy_fetch
-)
+from scrapling.cli import main, shell, mcp, get, post, put, delete, fetch, stealthy_fetch
@pytest_httpbin.use_class_based_httpbin
@@ -35,13 +33,13 @@ class TestCLI:
def test_version_flag(self, runner):
"""Test that the --version flag prints the Scrapling version and exits"""
- result = runner.invoke(main, ['--version'])
+ result = runner.invoke(main, ["--version"])
assert result.exit_code == 0
- assert result.output.strip() == f'Scrapling, version {__version__}'
+ assert result.output.strip() == f"Scrapling, version {__version__}"
def test_shell_command(self, runner):
"""Test shell command"""
- with patch('scrapling.core.shell.CustomShell') as mock_shell:
+ with patch("scrapling.core.shell.CustomShell") as mock_shell:
mock_instance = MagicMock()
mock_shell.return_value = mock_instance
@@ -51,7 +49,7 @@ class TestCLI:
def test_mcp_command(self, runner):
"""Test MCP command"""
- with patch('scrapling.core.ai.ScraplingMCPServer') as mock_server:
+ with patch("scrapling.core.ai.ScraplingMCPServer") as mock_server:
mock_instance = MagicMock()
mock_server.return_value = mock_instance
@@ -62,32 +60,29 @@ class TestCLI:
def test_mcp_command_with_executable_path(self, runner):
"""Test MCP command with a custom browser executable"""
- with patch('scrapling.core.ai.ScraplingMCPServer') as mock_server:
+ with patch("scrapling.core.ai.ScraplingMCPServer") as mock_server:
mock_instance = MagicMock()
mock_server.return_value = mock_instance
- result = runner.invoke(mcp, ['--executable-path', '/opt/custom-chromium'])
+ result = runner.invoke(mcp, ["--executable-path", "/opt/custom-chromium"])
assert result.exit_code == 0
- mock_server.assert_called_once_with(executable_path='/opt/custom-chromium')
+ mock_server.assert_called_once_with(executable_path="/opt/custom-chromium")
mock_instance.serve.assert_called_once_with(False, "0.0.0.0", 8000)
def test_extract_get_command(self, runner, tmp_path, html_url):
"""Test extract `get` command"""
output_file = tmp_path / "output.md"
- with patch('scrapling.fetchers.Fetcher.get') as mock_get:
+ with patch("scrapling.fetchers.Fetcher.get") as mock_get:
mock_response = configure_selector_mock()
mock_response.status = 200
mock_get.return_value = mock_response
- result = runner.invoke(
- get,
- [html_url, str(output_file)]
- )
+ result = runner.invoke(get, [html_url, str(output_file)])
assert result.exit_code == 0
# Test with various options
- with patch('scrapling.fetchers.Fetcher.get') as mock_get:
+ with patch("scrapling.fetchers.Fetcher.get") as mock_get:
mock_get.return_value = mock_response
result = runner.invoke(
@@ -95,13 +90,19 @@ class TestCLI:
[
html_url,
str(output_file),
- '-H', 'User-Agent: Test',
- '--cookies', 'session=abc123',
- '--timeout', '60',
- '--proxy', 'http://proxy:8080',
- '-s', '.content',
- '-p', 'page=1'
- ]
+ "-H",
+ "User-Agent: Test",
+ "--cookies",
+ "session=abc123",
+ "--timeout",
+ "60",
+ "--proxy",
+ "http://proxy:8080",
+ "-s",
+ ".content",
+ "-p",
+ "page=1",
+ ],
)
assert result.exit_code == 0
@@ -109,95 +110,117 @@ class TestCLI:
"""Test extract `post` command"""
output_file = tmp_path / "output.html"
- with patch('scrapling.fetchers.Fetcher.post') as mock_post:
+ with patch("scrapling.fetchers.Fetcher.post") as mock_post:
mock_response = configure_selector_mock()
mock_post.return_value = mock_response
- result = runner.invoke(
- post,
- [
- html_url,
- str(output_file),
- '-d', 'key=value',
- '-j', '{"data": "test"}'
- ]
- )
+ result = runner.invoke(post, [html_url, str(output_file), "-d", "key=value", "-j", '{"data": "test"}'])
assert result.exit_code == 0
def test_extract_put_command(self, runner, tmp_path, html_url):
"""Test extract `put` command"""
output_file = tmp_path / "output.html"
- with patch('scrapling.fetchers.Fetcher.put') as mock_put:
+ with patch("scrapling.fetchers.Fetcher.put") as mock_put:
mock_response = configure_selector_mock()
mock_put.return_value = mock_response
- result = runner.invoke(
- put,
- [
- html_url,
- str(output_file),
- '-d', 'key=value',
- '-j', '{"data": "test"}'
- ]
- )
+ result = runner.invoke(put, [html_url, str(output_file), "-d", "key=value", "-j", '{"data": "test"}'])
assert result.exit_code == 0
def test_extract_delete_command(self, runner, tmp_path, html_url):
"""Test extract `delete` command"""
output_file = tmp_path / "output.html"
- with patch('scrapling.fetchers.Fetcher.delete') as mock_delete:
+ with patch("scrapling.fetchers.Fetcher.delete") as mock_delete:
mock_response = configure_selector_mock()
mock_delete.return_value = mock_response
- result = runner.invoke(
- delete,
- [
- html_url,
- str(output_file)
- ]
- )
+ result = runner.invoke(delete, [html_url, str(output_file)])
assert result.exit_code == 0
def test_extract_fetch_command(self, runner, tmp_path, html_url):
"""Test extract fetch command"""
output_file = tmp_path / "output.txt"
- with patch('scrapling.fetchers.DynamicFetcher.fetch') as mock_fetch:
+ with patch("scrapling.fetchers.DynamicFetcher.fetch") as mock_fetch:
mock_response = configure_selector_mock()
mock_fetch.return_value = mock_response
- result = runner.invoke(
- fetch,
- [
- html_url,
- str(output_file),
- '--headless',
- '--timeout', '60000'
- ]
- )
+ result = runner.invoke(fetch, [html_url, str(output_file), "--headless", "--timeout", "60000"])
assert result.exit_code == 0
def test_extract_stealthy_fetch_command(self, runner, tmp_path, html_url):
"""Test extract fetch command"""
output_file = tmp_path / "output.md"
- with patch('scrapling.fetchers.StealthyFetcher.fetch') as mock_fetch:
+ with patch("scrapling.fetchers.StealthyFetcher.fetch") as mock_fetch:
mock_response = configure_selector_mock()
mock_fetch.return_value = mock_response
result = runner.invoke(
stealthy_fetch,
- [
- html_url,
- str(output_file),
- '--headless',
- '--css-selector', 'body',
- '--timeout', '60000'
- ]
+ [html_url, str(output_file), "--headless", "--css-selector", "body", "--timeout", "60000"],
+ )
+ assert result.exit_code == 0
+
+ def test_extract_fetch_with_executable_path(self, runner, tmp_path, html_url, monkeypatch):
+ """Test that --executable-path is passed through to DynamicFetcher and wins over the environment variable"""
+ output_file = tmp_path / "output.html"
+ monkeypatch.setenv("SCRAPLING_EXECUTABLE_PATH", "/opt/env-chromium")
+
+ with patch("scrapling.fetchers.DynamicFetcher.fetch") as mock_fetch:
+ mock_fetch.return_value = configure_selector_mock()
+
+ result = runner.invoke(fetch, [html_url, str(output_file), "--executable-path", "/opt/custom-chromium"])
+ assert result.exit_code == 0
+ assert mock_fetch.call_args.kwargs["executable_path"] == "/opt/custom-chromium"
+
+ def test_extract_fetch_executable_path_env_fallback(self, runner, tmp_path, html_url, monkeypatch):
+ """Test that SCRAPLING_EXECUTABLE_PATH is used when --executable-path is not passed"""
+ output_file = tmp_path / "output.html"
+ monkeypatch.setenv("SCRAPLING_EXECUTABLE_PATH", "/opt/env-chromium")
+
+ with patch("scrapling.fetchers.DynamicFetcher.fetch") as mock_fetch:
+ mock_fetch.return_value = configure_selector_mock()
+
+ result = runner.invoke(fetch, [html_url, str(output_file)])
+ assert result.exit_code == 0
+ assert mock_fetch.call_args.kwargs["executable_path"] == "/opt/env-chromium"
+
+ def test_extract_fetch_without_executable_path(self, runner, tmp_path, html_url, monkeypatch):
+ """Test that executable_path is not passed to the fetcher when neither source is set"""
+ output_file = tmp_path / "output.html"
+ monkeypatch.delenv("SCRAPLING_EXECUTABLE_PATH", raising=False)
+
+ with patch("scrapling.fetchers.DynamicFetcher.fetch") as mock_fetch:
+ mock_fetch.return_value = configure_selector_mock()
+
+ result = runner.invoke(fetch, [html_url, str(output_file)])
+ assert result.exit_code == 0
+ assert "executable_path" not in mock_fetch.call_args.kwargs
+
+ def test_extract_stealthy_fetch_with_executable_path(self, runner, tmp_path, html_url, monkeypatch):
+ """Test that --executable-path and the environment fallback work for stealthy_fetch too"""
+ output_file = tmp_path / "output.html"
+ monkeypatch.delenv("SCRAPLING_EXECUTABLE_PATH", raising=False)
+
+ with patch("scrapling.fetchers.StealthyFetcher.fetch") as mock_fetch:
+ mock_fetch.return_value = configure_selector_mock()
+
+ result = runner.invoke(
+ stealthy_fetch, [html_url, str(output_file), "--executable-path", "/opt/custom-chromium"]
)
assert result.exit_code == 0
+ assert mock_fetch.call_args.kwargs["executable_path"] == "/opt/custom-chromium"
+
+ monkeypatch.setenv("SCRAPLING_EXECUTABLE_PATH", "/opt/env-chromium")
+ with patch("scrapling.fetchers.StealthyFetcher.fetch") as mock_fetch:
+ mock_fetch.return_value = configure_selector_mock()
+
+ result = runner.invoke(stealthy_fetch, [html_url, str(output_file)])
+ assert result.exit_code == 0
+ assert mock_fetch.call_args.kwargs["executable_path"] == "/opt/env-chromium"
def test_invalid_arguments(self, runner, html_url):
"""Test invalid arguments handling"""
@@ -205,56 +228,39 @@ class TestCLI:
result = runner.invoke(get)
assert result.exit_code != 0
- _ = runner.invoke(
- get,
- [html_url, 'output.invalid']
- )
+ _ = runner.invoke(get, [html_url, "output.invalid"])
# Should handle the error gracefully
def test_impersonate_comma_separated(self, runner, tmp_path, html_url):
"""Test that comma-separated impersonate values are parsed correctly"""
output_file = tmp_path / "output.md"
- with patch('scrapling.fetchers.Fetcher.get') as mock_get:
+ with patch("scrapling.fetchers.Fetcher.get") as mock_get:
mock_response = configure_selector_mock()
mock_response.status = 200
mock_get.return_value = mock_response
- result = runner.invoke(
- get,
- [
- html_url,
- str(output_file),
- '--impersonate', 'chrome,firefox,safari'
- ]
- )
+ result = runner.invoke(get, [html_url, str(output_file), "--impersonate", "chrome,firefox,safari"])
assert result.exit_code == 0
# Verify that the impersonate argument was converted to a list
call_kwargs = mock_get.call_args[1]
- assert isinstance(call_kwargs['impersonate'], list)
- assert call_kwargs['impersonate'] == ['chrome', 'firefox', 'safari']
+ assert isinstance(call_kwargs["impersonate"], list)
+ assert call_kwargs["impersonate"] == ["chrome", "firefox", "safari"]
def test_impersonate_single_browser(self, runner, tmp_path, html_url):
"""Test that single impersonate value remains as string"""
output_file = tmp_path / "output.md"
- with patch('scrapling.fetchers.Fetcher.get') as mock_get:
+ with patch("scrapling.fetchers.Fetcher.get") as mock_get:
mock_response = configure_selector_mock()
mock_response.status = 200
mock_get.return_value = mock_response
- result = runner.invoke(
- get,
- [
- html_url,
- str(output_file),
- '--impersonate', 'chrome'
- ]
- )
+ result = runner.invoke(get, [html_url, str(output_file), "--impersonate", "chrome"])
assert result.exit_code == 0
# Verify that the impersonate argument remains a string
call_kwargs = mock_get.call_args[1]
- assert isinstance(call_kwargs['impersonate'], str)
- assert call_kwargs['impersonate'] == 'chrome'
+ assert isinstance(call_kwargs["impersonate"], str)
+ assert call_kwargs["impersonate"] == "chrome"