Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

14.8 KB
Raw Blame
import pytest
from click.testing import CliRunner
from unittest.mock import patch, MagicMock
import pytest_httpbin

from scrapling.parser import Selector
from scrapling import __version__
from scrapling.cli import main, shell, mcp, get, post, put, delete, fetch


@pytest_httpbin.use_class_based_httpbin
def configure_selector_mock():
    """Helper function to create a properly configured Selector mock"""
    mock_response = MagicMock(spec=Selector)
    mock_response.body = "<html><body>Test content</body></html>"
    mock_response.html_content = "<html><body>Test content</body></html>"
    mock_response.encoding = "utf-8"
    mock_response.get_all_text.return_value = "Test content"
    mock_response.css.return_value = [mock_response]
    return mock_response


class TestCLI:
    """Test CLI functionality"""

    @pytest.fixture
    def html_url(self, httpbin):
        return f"{httpbin.url}/html"

    @pytest.fixture
    def runner(self):
        return CliRunner()

    def test_version_flag(self, runner):
        """Test that the --version flag prints the Scrapling version and exits"""
        result = runner.invoke(main, ["--version"])
        assert result.exit_code == 0
        assert result.output.strip() == f"Scrapling, version {__version__}"

    def test_shell_command(self, runner):
        """Test shell command"""
        with patch("scrapling.core.shell.CustomShell") as mock_shell:
            mock_instance = MagicMock()
            mock_shell.return_value = mock_instance

            result = runner.invoke(shell)
            assert result.exit_code == 0
            mock_instance.start.assert_called_once()

    def test_mcp_command(self, runner):
        """Test MCP command"""
        with patch("scrapling.core.ai.ScraplingMCPServer") as mock_server:
            mock_instance = MagicMock()
            mock_server.return_value = mock_instance

            result = runner.invoke(mcp)
            assert result.exit_code == 0
            mock_server.assert_called_once_with(executable_path=None, auth_token=None)
            mock_instance.serve.assert_called_once_with(
                False, "127.0.0.1", 8000, allowed_hosts=(), allow_unauthenticated=False
            )

    def test_mcp_command_with_executable_path(self, runner):
        """Test MCP command with a custom browser executable"""
        with patch("scrapling.core.ai.ScraplingMCPServer") as mock_server:
            mock_instance = MagicMock()
            mock_server.return_value = mock_instance

            result = runner.invoke(mcp, ["--executable-path", "/opt/custom-chromium"])
            assert result.exit_code == 0
            mock_server.assert_called_once_with(executable_path="/opt/custom-chromium", auth_token=None)
            mock_instance.serve.assert_called_once_with(
                False, "127.0.0.1", 8000, allowed_hosts=(), allow_unauthenticated=False
            )

    def test_mcp_command_with_auth_token(self, runner):
        """Test MCP command with a shared authentication token"""
        with patch("scrapling.core.ai.ScraplingMCPServer") as mock_server:
            mock_instance = MagicMock()
            mock_server.return_value = mock_instance
            shared_key = "s3cret"

            result = runner.invoke(mcp, ["--http", "--auth-token", shared_key])
            assert result.exit_code == 0
            mock_server.assert_called_once_with(executable_path=None, auth_token=shared_key)
            mock_instance.serve.assert_called_once_with(
                True, "127.0.0.1", 8000, allowed_hosts=(), allow_unauthenticated=False
            )

    def test_mcp_command_with_no_auth(self, runner):
        """Test MCP command opting out of the streamable-http authentication"""
        with patch("scrapling.core.ai.ScraplingMCPServer") as mock_server:
            mock_instance = MagicMock()
            mock_server.return_value = mock_instance

            result = runner.invoke(mcp, ["--http", "--no-auth"])
            assert result.exit_code == 0
            mock_instance.serve.assert_called_once_with(
                True, "127.0.0.1", 8000, allowed_hosts=(), allow_unauthenticated=True
            )

    def test_mcp_command_binds_loopback_unless_asked_otherwise(self, runner):
        """`--http` stays off the network by default, so `--no-auth` can't expose the tools by accident"""
        with patch("scrapling.core.ai.ScraplingMCPServer") as mock_server:
            mock_instance = MagicMock()
            mock_server.return_value = mock_instance

            runner.invoke(mcp, ["--http", "--no-auth"])
            assert mock_instance.serve.call_args.args[1] == "127.0.0.1"

            mock_instance.serve.reset_mock()
            runner.invoke(mcp, ["--http", "--no-auth", "--host", "0.0.0.0"])
            assert mock_instance.serve.call_args.args[1] == "0.0.0.0"

    def test_mcp_command_reports_the_unauthenticated_refusal(self, runner):
        """The refusal raised by `serve` is shown as a CLI usage error instead of a traceback"""
        with patch("scrapling.core.ai.ScraplingMCPServer") as mock_server:
            mock_instance = MagicMock()
            mock_instance.serve.side_effect = ValueError("Refusing to serve without authentication")
            mock_server.return_value = mock_instance

            result = runner.invoke(mcp, ["--http"])
            assert result.exit_code == 2
            assert "Refusing to serve without authentication" in result.output

    def test_mcp_command_with_allowed_hosts(self, runner):
        """Test MCP command with repeated allowed hosts"""
        with patch("scrapling.core.ai.ScraplingMCPServer") as mock_server:
            mock_instance = MagicMock()
            mock_server.return_value = mock_instance

            result = runner.invoke(
                mcp, ["--http", "--allowed-host", "mcp.example.com:8000", "--allowed-host", "127.0.0.1:8000"]
            )
            assert result.exit_code == 0
            mock_instance.serve.assert_called_once_with(
                True,
                "127.0.0.1",
                8000,
                allowed_hosts=("mcp.example.com:8000", "127.0.0.1:8000"),
                allow_unauthenticated=False,
            )

    def test_extract_get_command(self, runner, tmp_path, html_url):
        """Test extract `get` command"""
        output_file = tmp_path / "output.md"

        with patch("scrapling.fetchers.Fetcher.get") as mock_get:
            mock_response = configure_selector_mock()
            mock_response.status = 200
            mock_get.return_value = mock_response

            result = runner.invoke(get, [html_url, str(output_file)])
            assert result.exit_code == 0

        # Test with various options
        with patch("scrapling.fetchers.Fetcher.get") as mock_get:
            mock_get.return_value = mock_response

            result = runner.invoke(
                get,
                [
                    html_url,
                    str(output_file),
                    "-H",
                    "User-Agent: Test",
                    "--cookies",
                    "session=abc123",
                    "--timeout",
                    "60",
                    "--proxy",
                    "http://proxy:8080",
                    "-s",
                    ".content",
                    "-p",
                    "page=1",
                ],
            )
            assert result.exit_code == 0

    def test_extract_post_command(self, runner, tmp_path, html_url):
        """Test extract `post` command"""
        output_file = tmp_path / "output.html"

        with patch("scrapling.fetchers.Fetcher.post") as mock_post:
            mock_response = configure_selector_mock()
            mock_post.return_value = mock_response

            result = runner.invoke(post, [html_url, str(output_file), "-d", "key=value", "-j", '{"data": "test"}'])
            assert result.exit_code == 0

    @pytest.mark.parametrize(
        "raw, expected",
        [("{}", {}), ("[]", []), ("0", 0), ("false", False)],
    )
    def test_extract_keeps_falsy_json_body(self, runner, tmp_path, html_url, raw, expected):
        """Test that a falsy JSON body given with `--json` still reaches the request"""
        output_file = tmp_path / "output.html"

        with patch("scrapling.fetchers.Fetcher.post") as mock_post:
            mock_post.return_value = configure_selector_mock()

            result = runner.invoke(post, [html_url, str(output_file), "-j", raw])
            assert result.exit_code == 0

            body = mock_post.call_args.kwargs["json"]
            assert body == expected
            assert type(body) is type(expected)

    def test_extract_keeps_non_empty_json_body(self, runner, tmp_path, html_url):
        """Test that a non-empty JSON body is passed on unchanged"""
        output_file = tmp_path / "output.html"

        with patch("scrapling.fetchers.Fetcher.post") as mock_post:
            mock_post.return_value = configure_selector_mock()

            result = runner.invoke(post, [html_url, str(output_file), "-j", '{"k": 1}'])
            assert result.exit_code == 0
            assert mock_post.call_args.kwargs["json"] == {"k": 1}

    def test_extract_without_json_body_sends_no_json(self, runner, tmp_path, html_url):
        """Test that no `--json` option leaves the `json` key out of the request"""
        output_file = tmp_path / "output.html"

        with patch("scrapling.fetchers.Fetcher.post") as mock_post:
            mock_post.return_value = configure_selector_mock()

            result = runner.invoke(post, [html_url, str(output_file)])
            assert result.exit_code == 0
            assert "json" not in mock_post.call_args.kwargs

    def test_extract_put_command(self, runner, tmp_path, html_url):
        """Test extract `put` command"""
        output_file = tmp_path / "output.html"

        with patch("scrapling.fetchers.Fetcher.put") as mock_put:
            mock_response = configure_selector_mock()
            mock_put.return_value = mock_response

            result = runner.invoke(put, [html_url, str(output_file), "-d", "key=value", "-j", '{"data": "test"}'])
            assert result.exit_code == 0

    def test_extract_delete_command(self, runner, tmp_path, html_url):
        """Test extract `delete` command"""
        output_file = tmp_path / "output.html"

        with patch("scrapling.fetchers.Fetcher.delete") as mock_delete:
            mock_response = configure_selector_mock()
            mock_delete.return_value = mock_response

            result = runner.invoke(delete, [html_url, str(output_file)])
            assert result.exit_code == 0

    def test_extract_fetch_command(self, runner, tmp_path, html_url):
        """Test extract fetch command"""
        output_file = tmp_path / "output.txt"

        with patch("scrapling.fetchers.StealthyFetcher.fetch") as mock_fetch:
            mock_response = configure_selector_mock()
            mock_fetch.return_value = mock_response

            result = runner.invoke(fetch, [html_url, str(output_file), "--headless", "--timeout", "60000"])
            assert result.exit_code == 0

    def test_extract_fetch_with_executable_path(self, runner, tmp_path, html_url, monkeypatch):
        """Test that --executable-path is passed through to StealthyFetcher and wins over the environment variable"""
        output_file = tmp_path / "output.html"
        monkeypatch.setenv("SCRAPLING_EXECUTABLE_PATH", "/opt/env-chromium")

        with patch("scrapling.fetchers.StealthyFetcher.fetch") as mock_fetch:
            mock_fetch.return_value = configure_selector_mock()

            result = runner.invoke(fetch, [html_url, str(output_file), "--executable-path", "/opt/custom-chromium"])
            assert result.exit_code == 0
            assert mock_fetch.call_args.kwargs["executable_path"] == "/opt/custom-chromium"

    def test_extract_fetch_executable_path_env_fallback(self, runner, tmp_path, html_url, monkeypatch):
        """Test that SCRAPLING_EXECUTABLE_PATH is used when --executable-path is not passed"""
        output_file = tmp_path / "output.html"
        monkeypatch.setenv("SCRAPLING_EXECUTABLE_PATH", "/opt/env-chromium")

        with patch("scrapling.fetchers.StealthyFetcher.fetch") as mock_fetch:
            mock_fetch.return_value = configure_selector_mock()

            result = runner.invoke(fetch, [html_url, str(output_file)])
            assert result.exit_code == 0
            assert mock_fetch.call_args.kwargs["executable_path"] == "/opt/env-chromium"

    def test_extract_fetch_without_executable_path(self, runner, tmp_path, html_url, monkeypatch):
        """Test that executable_path is not passed to the fetcher when neither source is set"""
        output_file = tmp_path / "output.html"
        monkeypatch.delenv("SCRAPLING_EXECUTABLE_PATH", raising=False)

        with patch("scrapling.fetchers.StealthyFetcher.fetch") as mock_fetch:
            mock_fetch.return_value = configure_selector_mock()

            result = runner.invoke(fetch, [html_url, str(output_file)])
            assert result.exit_code == 0
            assert "executable_path" not in mock_fetch.call_args.kwargs

    def test_invalid_arguments(self, runner, html_url):
        """Test invalid arguments handling"""
        # Missing required arguments
        result = runner.invoke(get)
        assert result.exit_code != 0

        _ = runner.invoke(get, [html_url, "output.invalid"])
        # Should handle the error gracefully

    def test_impersonate_comma_separated(self, runner, tmp_path, html_url):
        """Test that comma-separated impersonate values are parsed correctly"""
        output_file = tmp_path / "output.md"

        with patch("scrapling.fetchers.Fetcher.get") as mock_get:
            mock_response = configure_selector_mock()
            mock_response.status = 200
            mock_get.return_value = mock_response

            result = runner.invoke(get, [html_url, str(output_file), "--impersonate", "chrome,firefox,safari"])
            assert result.exit_code == 0

            # Verify that the impersonate argument was converted to a list
            call_kwargs = mock_get.call_args[1]
            assert isinstance(call_kwargs["impersonate"], list)
            assert call_kwargs["impersonate"] == ["chrome", "firefox", "safari"]

    def test_impersonate_single_browser(self, runner, tmp_path, html_url):
        """Test that single impersonate value remains as string"""
        output_file = tmp_path / "output.md"

        with patch("scrapling.fetchers.Fetcher.get") as mock_get:
            mock_response = configure_selector_mock()
            mock_response.status = 200
            mock_get.return_value = mock_response

            result = runner.invoke(get, [html_url, str(output_file), "--impersonate", "chrome"])
            assert result.exit_code == 0

            # Verify that the impersonate argument remains a string
            call_kwargs = mock_get.call_args[1]
            assert isinstance(call_kwargs["impersonate"], str)
            assert call_kwargs["impersonate"] == "chrome"