Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

Adding playwright/camoufox tests for cookies and improve the timeout test

Karim shoairde2d65856c20
diff --git a/tests/fetchers/test_camoufox.py b/tests/fetchers/test_camoufox.py
index 0249651..0a75c09 100644
--- a/tests/fetchers/test_camoufox.py
+++ b/tests/fetchers/test_camoufox.py
@@ -15,6 +15,8 @@ class TestStealthyFetcher(unittest.TestCase):
self.status_501 = f'{url}/status/501'
self.basic_url = f'{url}/get'
self.html_url = f'{url}/html'
+ self.delayed_url = f'{url}/delay/10' # 10 Seconds delay response
+ self.cookies_url = f"{url}/cookies/set/test/value"
def test_basic_fetch(self):
"""Test doing basic fetch request with multiple statuses"""
@@ -35,6 +37,10 @@ class TestStealthyFetcher(unittest.TestCase):
"""Test if waiting for a selector make page does not finish loading or not"""
self.assertEqual(self.fetcher.fetch(self.html_url, wait_selector='h1').status, 200)
+ def test_cookies_loading(self):
+ """Test if cookies are set after the request"""
+ self.assertEqual(self.fetcher.fetch(self.cookies_url).cookies, {'test': 'value'})
+
def test_automation(self):
"""Test if automation break the code or not"""
def scroll_page(page):
@@ -53,4 +59,4 @@ class TestStealthyFetcher(unittest.TestCase):
def test_infinite_timeout(self):
"""Test if infinite timeout breaks the code or not"""
- self.assertEqual(self.fetcher.fetch(self.html_url, timeout=None).status, 200)
+ self.assertEqual(self.fetcher.fetch(self.delayed_url, timeout=None).status, 200)
diff --git a/tests/fetchers/test_playwright.py b/tests/fetchers/test_playwright.py
index f0ba143..138fbbe 100644
--- a/tests/fetchers/test_playwright.py
+++ b/tests/fetchers/test_playwright.py
@@ -15,6 +15,8 @@ class TestPlayWrightFetcher(unittest.TestCase):
self.status_501 = f'{url}/status/501'
self.basic_url = f'{url}/get'
self.html_url = f'{url}/html'
+ self.delayed_url = f'{url}/delay/10' # 10 Seconds delay response
+ self.cookies_url = f"{url}/cookies/set/test/value"
def test_basic_fetch(self):
"""Test doing basic fetch request with multiple statuses"""
@@ -34,6 +36,10 @@ class TestPlayWrightFetcher(unittest.TestCase):
"""Test if waiting for a selector make page does not finish loading or not"""
self.assertEqual(self.fetcher.fetch(self.html_url, wait_selector='h1').status, 200)
+ def test_cookies_loading(self):
+ """Test if cookies are set after the request"""
+ self.assertEqual(self.fetcher.fetch(self.cookies_url).cookies, {'test': 'value'})
+
def test_automation(self):
"""Test if automation break the code or not"""
def scroll_page(page):
@@ -65,4 +71,4 @@ class TestPlayWrightFetcher(unittest.TestCase):
def test_infinite_timeout(self):
"""Test if infinite timeout breaks the code or not"""
- self.assertEqual(self.fetcher.fetch(self.html_url, timeout=None).status, 200)
+ self.assertEqual(self.fetcher.fetch(self.delayed_url, timeout=None).status, 200)