Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

5.3 KB
Raw Blame
import pytest
from types import SimpleNamespace

from scrapling.fetchers import AsyncStealthySession, StealthySession
from scrapling.engines._browsers._validators import (
    validate,
    validate_fetch,
    StealthConfig,
    PlaywrightConfig,
)


class TestValidators:
    """Test configuration validators"""

    @pytest.mark.parametrize("model", [PlaywrightConfig, StealthConfig])
    def test_request_recording_replaces_xhr_config(self, model):
        config = validate({"record_requests": True, "max_recorded_requests": 50}, model)
        assert config.record_requests and config.max_recorded_requests == 50
        assert "capture_xhr" not in model.__struct_fields__

    @pytest.mark.parametrize("model", [PlaywrightConfig, StealthConfig])
    @pytest.mark.parametrize("value", [None, ".*"])
    def test_removed_xhr_option_is_rejected_by_config_and_fetch(self, model, value):
        kwargs = {"capture_xhr": value}
        message = "capture_xhr was removed; use record_requests=True and session.network."
        with pytest.raises(TypeError, match=message):
            validate(kwargs, model)
        with pytest.raises(TypeError, match=message):
            validate_fetch(kwargs, SimpleNamespace(_config=model()), model)

    @pytest.mark.parametrize("model", [PlaywrightConfig, StealthConfig])
    def test_other_unknown_options_keep_existing_behavior(self, model):
        config = validate({"unknown_option": True}, model)
        assert not hasattr(config, "unknown_option")
        params = validate_fetch({"unknown_option": True}, SimpleNamespace(_config=config), model)
        assert params.timeout == config.timeout
        assert not hasattr(params, "unknown_option")

    @pytest.mark.parametrize("session_type", [StealthySession, AsyncStealthySession])
    def test_removed_xhr_option_is_rejected_by_sessions(self, session_type):
        with pytest.raises(TypeError, match="capture_xhr was removed"):
            session_type(capture_xhr=".*")

    def test_playwright_config_valid(self):
        """Test valid PlaywrightConfig"""
        params = {"max_pages": 2, "headless": True, "timeout": 30000, "proxy": "http://proxy.example.com:8080"}

        config = validate(params, PlaywrightConfig)

        assert config.max_pages == 2
        assert config.headless is True
        assert config.timeout == 30000
        assert isinstance(config.proxy, dict)

    def test_playwright_config_invalid_max_pages(self):
        """Test PlaywrightConfig with invalid max_pages"""
        params = {"max_pages": 0}

        with pytest.raises(TypeError):
            validate(params, PlaywrightConfig)

        params = {"max_pages": 51}

        with pytest.raises(TypeError):
            validate(params, PlaywrightConfig)

    def test_playwright_config_invalid_timeout(self):
        """Test PlaywrightConfig with an invalid timeout"""
        params = {"timeout": -1}

        with pytest.raises(TypeError):
            validate(params, PlaywrightConfig)

    def test_playwright_config_invalid_cdp_url(self):
        """Test PlaywrightConfig with invalid CDP URL"""
        params = {"cdp_url": "invalid-url"}

        with pytest.raises(TypeError):
            validate(params, PlaywrightConfig)

        for cdp_url in ("ftp://localhost:9222", "localhost:9222", "http://"):
            with pytest.raises(TypeError):
                validate({"cdp_url": cdp_url}, PlaywrightConfig)

    def test_playwright_config_cdp_url_schemes(self):
        """Test PlaywrightConfig accepts both WebSocket and HTTP CDP endpoints"""
        for cdp_url in (
            "ws://localhost:9222",
            "wss://cdp.example.com/session",
            "http://localhost:9222",
            "https://cdp.example.com",
        ):
            assert validate({"cdp_url": cdp_url}, PlaywrightConfig).cdp_url == cdp_url

    def test_stealth_config_valid(self):
        """Test valid StealthConfig"""
        params = {"max_pages": 1, "headless": True, "solve_cloudflare": False, "timeout": 30000}

        config = validate(params, StealthConfig)

        assert config.max_pages == 1
        assert config.headless is True
        assert config.solve_cloudflare is False
        assert config.timeout == 30000

    def test_stealth_config_cloudflare_timeout(self):
        """Test StealthConfig timeout adjustment for Cloudflare"""
        params = {
            "solve_cloudflare": True,
            "timeout": 10000,  # Less than the required 60,000
        }

        config = validate(params, StealthConfig)

        assert config.timeout == 60000  # Should be increased

    def test_playwright_config_blocked_domains(self):
        """Test PlaywrightConfig with blocked_domains"""
        params = {"blocked_domains": {"ads.example.com", "tracker.io"}}

        config = validate(params, PlaywrightConfig)

        assert config.blocked_domains == {"ads.example.com", "tracker.io"}

    def test_playwright_config_blocked_domains_default_none(self):
        """Test PlaywrightConfig blocked_domains defaults to None"""
        config = validate({}, PlaywrightConfig)

        assert config.blocked_domains is None

    def test_stealth_config_blocked_domains(self):
        """Test StealthConfig inherits blocked_domains"""
        params = {"blocked_domains": {"ads.example.com"}}

        config = validate(params, StealthConfig)

        assert config.blocked_domains == {"ads.example.com"}