Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

5.2 KB
Raw Blame
import re
import pytest
import pytest_httpbin

from scrapling.engines.constants import DEFAULT_ARGS, STEALTH_ARGS
from scrapling.engines._browsers._stealth import StealthySession, __CF_PATTERN__


class TestStealthConstants:
    """Test Stealth constants and patterns"""

    def test_cf_pattern_regex(self):
        """Test __CF_PATTERN__ regex compilation"""

        assert isinstance(__CF_PATTERN__, re.Pattern)

        # Test matching URLs
        test_urls = [
            "https://challenges.cloudflare.com/cdn-cgi/challenge-platform/h/123456",
            "https://challenges.cloudflare.com/cdn-cgi/challenge-platform/orchestrate/jsch/v1",
            "http://challenges.cloudflare.com/cdn-cgi/challenge-platform/scripts/abc",
        ]

        for url in test_urls:
            assert __CF_PATTERN__.search(url) is not None

        # Test non-matching URLs
        non_matching_urls = [
            "https://example.com/challenge",
            "https://cloudflare.com/something",
            "https://challenges.cloudflare.com/other-path",
        ]

        for url in non_matching_urls:
            assert __CF_PATTERN__.search(url) is None


class TestLaunchFlags:
    """The launch flags a session ends up handing to Chromium. No browser is started."""

    # in neither DEFAULT_ARGS nor STEALTH_ARGS, so its presence and the bundle's
    # presence are independent observations
    USER_FLAG = "--window-size=1280,720"

    @staticmethod
    def _args(**kwargs):
        session = StealthySession(headless=True, block_webrtc=True, allow_webgl=False, hide_canvas=True, **kwargs)
        return list(session._browser_options["args"])

    def test_extra_flags_extend_the_stealth_bundle(self):
        """`config.extra_flags or extra_flags` returned whichever came first, so passing
        one flag of your own dropped all 53 STEALTH_ARGS and every conditional flag."""
        without = self._args()
        with_extra = self._args(extra_flags=[self.USER_FLAG])

        assert self.USER_FLAG not in without, "test flag must not already ship"
        assert self.USER_FLAG in with_extra, "the user's own flag was dropped"

        missing = [flag for flag in STEALTH_ARGS if flag not in with_extra]
        assert missing == [], f"extra_flags displaced {len(missing)} stealth flags"
        assert [flag for flag in DEFAULT_ARGS if flag not in with_extra] == []

        # the conditional flags are built in the same place and were lost the same way
        for flag in (
            "--webrtc-ip-handling-policy=disable_non_proxied_udp",
            "--disable-webgl",
            "--fingerprinting-canvas-image-data-noise",
        ):
            assert flag in with_extra, f"{flag} lost to extra_flags"


@pytest.mark.browser
@pytest_httpbin.use_class_based_httpbin
class TestStealthySession:
    """All the code is tested in the async version tests, so no need to repeat it here. The async class inherits from this one."""

    @pytest.fixture(autouse=True)
    def setup_urls(self, httpbin):
        """Fixture to set up URLs for testing"""
        self.status_200 = f"{httpbin.url}/status/200"
        self.status_404 = f"{httpbin.url}/status/404"
        self.status_501 = f"{httpbin.url}/status/501"
        self.basic_url = f"{httpbin.url}/get"
        self.html_url = f"{httpbin.url}/html"
        self.delayed_url = f"{httpbin.url}/delay/10"  # 10 Seconds delay response
        self.cookies_url = f"{httpbin.url}/cookies/set/test/value"

    def test_session_creation(self):
        """Test if the session is created correctly"""

        with StealthySession(
            headless=True,
            disable_resources=True,
            solve_cloudflare=True,
            wait=1000,
            timeout=60000,
            cookies=[{"name": "test", "value": "123", "domain": "example.com", "path": "/"}],
        ) as session:
            assert session.max_pages == 1
            assert session._config.headless is True
            assert session._config.disable_resources is True
            assert session._config.solve_cloudflare is True
            assert session._config.wait == 1000
            assert session._config.timeout == 60000
            assert session.context is not None

            # Test Cloudflare detection
            for cloudflare_type in ("managed", "interactive", "non-interactive"):
                page_content = f"""
                <html>
                    <script>
                        cType: '{cloudflare_type}'
                    </script>
                </html>
                """
                result = session._detect_cloudflare(page_content)
                assert result == cloudflare_type

            page_content = """
            <html>
                <body>
                    <p>Regular page content</p>
                </body>
            </html>
            """

            result = StealthySession._detect_cloudflare(page_content)
            assert result is None
            assert session.fetch(self.status_200).status == 200
            assert session.page_pool.pages_count == 1
            page = session.page_pool.pages[0].page
            assert session.fetch(self.status_200).status == 200
            assert session.page_pool.pages[0].page is page, "sync sessions reuse their single tab"
            session.close_pages()
            assert session.page_pool.pages_count == 0