🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
from pathlib import Path
from unittest.mock import patch
import pytest
from click.testing import CliRunner
from scrapling.cli import main
from scrapling.parser import Selector
@pytest.mark.parametrize(
("flags", "enabled"), [([], False), (["--pierce-shadow"], True), (["--no-pierce-shadow"], False)]
)
def test_shadow_option_reaches_fetcher(tmp_path: Path, flags: list[str], enabled: bool) -> None:
output = tmp_path / "page.txt"
with patch("scrapling.fetchers.StealthyFetcher.fetch", return_value=Selector("<p>Content</p>")) as fetch:
result = CliRunner().invoke(main, ["extract", "fetch", "https://example.com", str(output), *flags])
assert result.exit_code == 0, result.output
assert fetch.call_args.kwargs["pierce_shadow"] is enabled
assert output.read_text() == "Content"
def test_shadow_option_in_browser_help() -> None:
result = CliRunner().invoke(main, ["extract", "fetch", "--help"])
assert result.exit_code == 0
assert "--pierce-shadow / --no-pierce-shadow" in result.output
@pytest.mark.parametrize("command", ["get", "post", "put", "delete"])
def test_shadow_option_not_in_http_help(command: str) -> None:
result = CliRunner().invoke(main, ["extract", command, "--help"])
assert result.exit_code == 0
assert "pierce-shadow" not in result.output