Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

1.3 KB
Raw Blame
from pathlib import Path
from unittest.mock import patch

import pytest
from click.testing import CliRunner

from scrapling.cli import main
from scrapling.parser import Selector


@pytest.mark.parametrize(
    ("flags", "enabled"), [([], False), (["--pierce-shadow"], True), (["--no-pierce-shadow"], False)]
)
def test_shadow_option_reaches_fetcher(tmp_path: Path, flags: list[str], enabled: bool) -> None:
    output = tmp_path / "page.txt"
    with patch("scrapling.fetchers.StealthyFetcher.fetch", return_value=Selector("<p>Content</p>")) as fetch:
        result = CliRunner().invoke(main, ["extract", "fetch", "https://example.com", str(output), *flags])
    assert result.exit_code == 0, result.output
    assert fetch.call_args.kwargs["pierce_shadow"] is enabled
    assert output.read_text() == "Content"


def test_shadow_option_in_browser_help() -> None:
    result = CliRunner().invoke(main, ["extract", "fetch", "--help"])
    assert result.exit_code == 0
    assert "--pierce-shadow / --no-pierce-shadow" in result.output


@pytest.mark.parametrize("command", ["get", "post", "put", "delete"])
def test_shadow_option_not_in_http_help(command: str) -> None:
    result = CliRunner().invoke(main, ["extract", command, "--help"])
    assert result.exit_code == 0
    assert "pierce-shadow" not in result.output