Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat: Validate all fetch-level parameters

Karim shoair60be9dc816b5
diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.py
index 3412fd6..fcb77ac 100644
--- a/scrapling/engines/_browsers/_camoufox.py
+++ b/scrapling/engines/_browsers/_camoufox.py
@@ -15,6 +15,7 @@ from playwright.async_api import (
Page as async_Page,
)
+from ._validators import validate, CamoufoxConfig
from ._base import SyncSession, AsyncSession, StealthySessionMixin
from scrapling.core.utils import log
from scrapling.core._types import (
@@ -164,10 +165,8 @@ class StealthySession(StealthySessionMixin, SyncSession):
def __create__(self):
"""Create a browser for this instance and context."""
self.playwright = sync_playwright().start()
- self.context = (
- self.playwright.firefox.launch_persistent_context( # pragma: no cover
- **self.launch_options
- )
+ self.context = self.playwright.firefox.launch_persistent_context( # pragma: no cover
+ **self.launch_options
)
# Get the default page and close it
@@ -281,32 +280,22 @@ class StealthySession(StealthySessionMixin, SyncSession):
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object.
"""
- google_search = self._get_with_precedence(
- google_search, self.google_search, _UNSET
- )
- timeout = self._get_with_precedence(timeout, self.timeout, _UNSET)
- wait = self._get_with_precedence(wait, self.wait, _UNSET)
- page_action = self._get_with_precedence(page_action, self.page_action, _UNSET)
- extra_headers = self._get_with_precedence(
- extra_headers, self.extra_headers, _UNSET
- )
- disable_resources = self._get_with_precedence(
- disable_resources, self.disable_resources, _UNSET
- )
- wait_selector = self._get_with_precedence(
- wait_selector, self.wait_selector, _UNSET
- )
- wait_selector_state = self._get_with_precedence(
- wait_selector_state, self.wait_selector_state, _UNSET
- )
- network_idle = self._get_with_precedence(
- network_idle, self.network_idle, _UNSET
- )
- solve_cloudflare = self._get_with_precedence(
- solve_cloudflare, self.solve_cloudflare, _UNSET
- )
- selector_config = self._get_with_precedence(
- selector_config, self.selector_config, _UNSET
+ # Validate all resolved parameters
+ params = validate(
+ dict(
+ google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
+ timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
+ wait=self._get_with_precedence(wait, self.wait, _UNSET),
+ page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
+ extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
+ disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
+ wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
+ wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
+ network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
+ solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET),
+ selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
+ ),
+ CamoufoxConfig,
)
if self._closed: # pragma: no cover
@@ -314,9 +303,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
final_response = None
referer = (
- generate_convincing_referer(url)
- if (google_search and "referer" not in self._headers_keys)
- else None
+ generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None
)
def handle_response(finished_response: SyncPlaywrightResponse):
@@ -327,7 +314,7 @@ class StealthySession(StealthySessionMixin, SyncSession):
):
final_response = finished_response
- page_info = self._get_page(timeout, extra_headers, disable_resources)
+ page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)
page_info.mark_busy(url=url)
try: # pragma: no cover
@@ -336,41 +323,41 @@ class StealthySession(StealthySessionMixin, SyncSession):
first_response = page_info.page.goto(url, referer=referer)
page_info.page.wait_for_load_state(state="domcontentloaded")
- if network_idle:
+ if params.network_idle:
page_info.page.wait_for_load_state("networkidle")
if not first_response:
raise RuntimeError(f"Failed to get response for {url}")
- if solve_cloudflare:
+ if params.solve_cloudflare:
self._solve_cloudflare(page_info.page)
# Make sure the page is fully loaded after the captcha
page_info.page.wait_for_load_state(state="load")
page_info.page.wait_for_load_state(state="domcontentloaded")
- if network_idle:
+ if params.network_idle:
page_info.page.wait_for_load_state("networkidle")
- if page_action is not None:
+ if params.page_action:
try:
- _ = page_action(page_info.page)
+ _ = params.page_action(page_info.page)
except Exception as e:
log.error(f"Error executing page_action: {e}")
- if wait_selector:
+ if params.wait_selector:
try:
- waiter: Locator = page_info.page.locator(wait_selector)
- waiter.first.wait_for(state=wait_selector_state)
+ waiter: Locator = page_info.page.locator(params.wait_selector)
+ waiter.first.wait_for(state=params.wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare
page_info.page.wait_for_load_state(state="load")
page_info.page.wait_for_load_state(state="domcontentloaded")
- if network_idle:
+ if params.network_idle:
page_info.page.wait_for_load_state("networkidle")
except Exception as e:
- log.error(f"Error waiting for selector {wait_selector}: {e}")
+ log.error(f"Error waiting for selector {params.wait_selector}: {e}")
- page_info.page.wait_for_timeout(wait)
+ page_info.page.wait_for_timeout(params.wait)
response = ResponseFactory.from_playwright_response(
- page_info.page, first_response, final_response, selector_config
+ page_info.page, first_response, final_response, params.selector_config
)
# Mark the page as finished for next use
@@ -478,10 +465,8 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
async def __create__(self):
"""Create a browser for this instance and context."""
self.playwright: AsyncPlaywright = await async_playwright().start()
- self.context: AsyncBrowserContext = (
- await self.playwright.firefox.launch_persistent_context(
- **self.launch_options
- )
+ self.context: AsyncBrowserContext = await self.playwright.firefox.launch_persistent_context(
+ **self.launch_options
)
# Get the default page and close it
@@ -551,9 +536,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
await page.wait_for_timeout(500)
# Calculate the Captcha coordinates for any viewport
- outer_box = await page.locator(
- ".main-content p+div>div>div"
- ).bounding_box()
+ outer_box = await page.locator(".main-content p+div>div>div").bounding_box()
captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25
# Move the mouse to the center of the window, then press and hold the left mouse button
@@ -597,32 +580,21 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object.
"""
- google_search = self._get_with_precedence(
- google_search, self.google_search, _UNSET
- )
- timeout = self._get_with_precedence(timeout, self.timeout, _UNSET)
- wait = self._get_with_precedence(wait, self.wait, _UNSET)
- page_action = self._get_with_precedence(page_action, self.page_action, _UNSET)
- extra_headers = self._get_with_precedence(
- extra_headers, self.extra_headers, _UNSET
- )
- disable_resources = self._get_with_precedence(
- disable_resources, self.disable_resources, _UNSET
- )
- wait_selector = self._get_with_precedence(
- wait_selector, self.wait_selector, _UNSET
- )
- wait_selector_state = self._get_with_precedence(
- wait_selector_state, self.wait_selector_state, _UNSET
- )
- network_idle = self._get_with_precedence(
- network_idle, self.network_idle, _UNSET
- )
- solve_cloudflare = self._get_with_precedence(
- solve_cloudflare, self.solve_cloudflare, _UNSET
- )
- selector_config = self._get_with_precedence(
- selector_config, self.selector_config, _UNSET
+ params = validate(
+ dict(
+ google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
+ timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
+ wait=self._get_with_precedence(wait, self.wait, _UNSET),
+ page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
+ extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
+ disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
+ wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
+ wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
+ network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
+ solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET),
+ selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
+ ),
+ CamoufoxConfig,
)
if self._closed: # pragma: no cover
@@ -630,9 +602,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
final_response = None
referer = (
- generate_convincing_referer(url)
- if (google_search and "referer" not in self._headers_keys)
- else None
+ generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None
)
async def handle_response(finished_response: AsyncPlaywrightResponse):
@@ -643,7 +613,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
):
final_response = finished_response
- page_info = await self._get_page(timeout, extra_headers, disable_resources)
+ page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)
page_info.mark_busy(url=url)
try:
@@ -652,43 +622,43 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):
first_response = await page_info.page.goto(url, referer=referer)
await page_info.page.wait_for_load_state(state="domcontentloaded")
- if network_idle:
+ if params.network_idle:
await page_info.page.wait_for_load_state("networkidle")
if not first_response:
raise RuntimeError(f"Failed to get response for {url}")
- if solve_cloudflare:
+ if params.solve_cloudflare:
await self._solve_cloudflare(page_info.page)
# Make sure the page is fully loaded after the captcha
await page_info.page.wait_for_load_state(state="load")
await page_info.page.wait_for_load_state(state="domcontentloaded")
- if network_idle:
+ if params.network_idle:
await page_info.page.wait_for_load_state("networkidle")
- if page_action is not None:
+ if params.page_action:
try:
- _ = await page_action(page_info.page)
+ _ = await params.page_action(page_info.page)
except Exception as e:
log.error(f"Error executing page_action: {e}")
- if wait_selector:
+ if params.wait_selector:
try:
- waiter: AsyncLocator = page_info.page.locator(wait_selector)
- await waiter.first.wait_for(state=wait_selector_state)
+ waiter: AsyncLocator = page_info.page.locator(params.wait_selector)
+ await waiter.first.wait_for(state=params.wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare
await page_info.page.wait_for_load_state(state="load")
await page_info.page.wait_for_load_state(state="domcontentloaded")
- if network_idle:
+ if params.network_idle:
await page_info.page.wait_for_load_state("networkidle")
except Exception as e:
- log.error(f"Error waiting for selector {wait_selector}: {e}")
+ log.error(f"Error waiting for selector {params.wait_selector}: {e}")
- await page_info.page.wait_for_timeout(wait)
+ await page_info.page.wait_for_timeout(params.wait)
# Create response object
response = await ResponseFactory.from_async_playwright_response(
- page_info.page, first_response, final_response, selector_config
+ page_info.page, first_response, final_response, params.selector_config
)
# Mark the page as finished for next use
diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py
index 49f6c9d..a53fa62 100644
--- a/scrapling/engines/_browsers/_controllers.py
+++ b/scrapling/engines/_browsers/_controllers.py
@@ -18,6 +18,7 @@ from rebrowser_playwright.async_api import (
from scrapling.core.utils import log
from ._base import SyncSession, AsyncSession, DynamicSessionMixin
+from ._validators import validate, PlaywrightConfig
from scrapling.core._types import (
Dict,
List,
@@ -157,13 +158,11 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
self.playwright: Playwright = sync_context().start()
if self.cdp_url: # pragma: no cover
- self.context = self.playwright.chromium.connect_over_cdp(
- endpoint_url=self.cdp_url
- ).new_context(**self.context_options)
- else:
- self.context = self.playwright.chromium.launch_persistent_context(
- user_data_dir="", **self.launch_options
+ self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self.cdp_url).new_context(
+ **self.context_options
)
+ else:
+ self.context = self.playwright.chromium.launch_persistent_context(user_data_dir="", **self.launch_options)
# Get the default page and close it
default_page = self.context.pages[0]
@@ -228,29 +227,21 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object.
"""
- google_search = self._get_with_precedence(
- google_search, self.google_search, _UNSET
- )
- timeout = self._get_with_precedence(timeout, self.timeout, _UNSET)
- wait = self._get_with_precedence(wait, self.wait, _UNSET)
- page_action = self._get_with_precedence(page_action, self.page_action, _UNSET)
- extra_headers = self._get_with_precedence(
- extra_headers, self.extra_headers, _UNSET
- )
- disable_resources = self._get_with_precedence(
- disable_resources, self.disable_resources, _UNSET
- )
- wait_selector = self._get_with_precedence(
- wait_selector, self.wait_selector, _UNSET
- )
- wait_selector_state = self._get_with_precedence(
- wait_selector_state, self.wait_selector_state, _UNSET
- )
- network_idle = self._get_with_precedence(
- network_idle, self.network_idle, _UNSET
- )
- selector_config = self._get_with_precedence(
- selector_config, self.selector_config, _UNSET
+ # Validate all resolved parameters
+ params = validate(
+ dict(
+ google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
+ timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
+ wait=self._get_with_precedence(wait, self.wait, _UNSET),
+ page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
+ extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
+ disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
+ wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
+ wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
+ network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
+ selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
+ ),
+ PlaywrightConfig,
)
if self._closed: # pragma: no cover
@@ -258,9 +249,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
final_response = None
referer = (
- generate_convincing_referer(url)
- if (google_search and "referer" not in self._headers_keys)
- else None
+ generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None
)
def handle_response(finished_response: SyncPlaywrightResponse):
@@ -271,7 +260,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
):
final_response = finished_response
- page_info = self._get_page(timeout, extra_headers, disable_resources)
+ page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)
page_info.mark_busy(url=url)
try: # pragma: no cover
@@ -280,35 +269,35 @@ class DynamicSession(DynamicSessionMixin, SyncSession):
first_response = page_info.page.goto(url, referer=referer)
page_info.page.wait_for_load_state(state="domcontentloaded")
- if network_idle:
+ if params.network_idle:
page_info.page.wait_for_load_state("networkidle")
if not first_response:
raise RuntimeError(f"Failed to get response for {url}")
- if page_action is not None:
+ if params.page_action:
try:
- _ = page_action(page_info.page)
+ _ = params.page_action(page_info.page)
except Exception as e: # pragma: no cover
log.error(f"Error executing page_action: {e}")
- if wait_selector:
+ if params.wait_selector:
try:
- waiter: Locator = page_info.page.locator(wait_selector)
- waiter.first.wait_for(state=wait_selector_state)
+ waiter: Locator = page_info.page.locator(params.wait_selector)
+ waiter.first.wait_for(state=params.wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare
page_info.page.wait_for_load_state(state="load")
page_info.page.wait_for_load_state(state="domcontentloaded")
- if network_idle:
+ if params.network_idle:
page_info.page.wait_for_load_state("networkidle")
except Exception as e: # pragma: no cover
- log.error(f"Error waiting for selector {wait_selector}: {e}")
+ log.error(f"Error waiting for selector {params.wait_selector}: {e}")
- page_info.page.wait_for_timeout(wait)
+ page_info.page.wait_for_timeout(params.wait)
# Create response object
response = ResponseFactory.from_playwright_response(
- page_info.page, first_response, final_response, selector_config
+ page_info.page, first_response, final_response, params.selector_config
)
# Mark the page as finished for next use
@@ -413,17 +402,11 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
self.playwright: AsyncPlaywright = await async_context().start()
if self.cdp_url:
- browser = await self.playwright.chromium.connect_over_cdp(
- endpoint_url=self.cdp_url
- )
- self.context: AsyncBrowserContext = await browser.new_context(
- **self.context_options
- )
+ browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self.cdp_url)
+ self.context: AsyncBrowserContext = await browser.new_context(**self.context_options)
else:
- self.context: AsyncBrowserContext = (
- await self.playwright.chromium.launch_persistent_context(
- user_data_dir="", **self.launch_options
- )
+ self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(
+ user_data_dir="", **self.launch_options
)
# Get the default page and close it
@@ -489,29 +472,21 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
:param selector_config: The arguments that will be passed in the end while creating the final Selector's class.
:return: A `Response` object.
"""
- google_search = self._get_with_precedence(
- google_search, self.google_search, _UNSET
- )
- timeout = self._get_with_precedence(timeout, self.timeout, _UNSET)
- wait = self._get_with_precedence(wait, self.wait, _UNSET)
- page_action = self._get_with_precedence(page_action, self.page_action, _UNSET)
- extra_headers = self._get_with_precedence(
- extra_headers, self.extra_headers, _UNSET
- )
- disable_resources = self._get_with_precedence(
- disable_resources, self.disable_resources, _UNSET
- )
- wait_selector = self._get_with_precedence(
- wait_selector, self.wait_selector, _UNSET
- )
- wait_selector_state = self._get_with_precedence(
- wait_selector_state, self.wait_selector_state, _UNSET
- )
- network_idle = self._get_with_precedence(
- network_idle, self.network_idle, _UNSET
- )
- selector_config = self._get_with_precedence(
- selector_config, self.selector_config, _UNSET
+ # Validate all resolved parameters
+ params = validate(
+ dict(
+ google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),
+ timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),
+ wait=self._get_with_precedence(wait, self.wait, _UNSET),
+ page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),
+ extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),
+ disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),
+ wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),
+ wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),
+ network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),
+ selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),
+ ),
+ PlaywrightConfig,
)
if self._closed: # pragma: no cover
@@ -519,9 +494,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
final_response = None
referer = (
- generate_convincing_referer(url)
- if (google_search and "referer" not in self._headers_keys)
- else None
+ generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None
)
async def handle_response(finished_response: AsyncPlaywrightResponse):
@@ -532,7 +505,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
):
final_response = finished_response
- page_info = await self._get_page(timeout, extra_headers, disable_resources)
+ page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)
page_info.mark_busy(url=url)
try:
@@ -541,35 +514,35 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):
first_response = await page_info.page.goto(url, referer=referer)
await page_info.page.wait_for_load_state(state="domcontentloaded")
- if network_idle:
+ if params.network_idle:
await page_info.page.wait_for_load_state("networkidle")
if not first_response:
raise RuntimeError(f"Failed to get response for {url}")
- if page_action is not None:
+ if params.page_action:
try:
- _ = await page_action(page_info.page)
+ _ = await params.page_action(page_info.page)
except Exception as e:
log.error(f"Error executing page_action: {e}")
- if wait_selector:
+ if params.wait_selector:
try:
- waiter: AsyncLocator = page_info.page.locator(wait_selector)
- await waiter.first.wait_for(state=wait_selector_state)
+ waiter: AsyncLocator = page_info.page.locator(params.wait_selector)
+ await waiter.first.wait_for(state=params.wait_selector_state)
# Wait again after waiting for the selector, helpful with protections like Cloudflare
await page_info.page.wait_for_load_state(state="load")
await page_info.page.wait_for_load_state(state="domcontentloaded")
- if network_idle:
+ if params.network_idle:
await page_info.page.wait_for_load_state("networkidle")
except Exception as e:
- log.error(f"Error waiting for selector {wait_selector}: {e}")
+ log.error(f"Error waiting for selector {params.wait_selector}: {e}")
- await page_info.page.wait_for_timeout(wait)
+ await page_info.page.wait_for_timeout(params.wait)
# Create response object
response = await ResponseFactory.from_async_playwright_response(
- page_info.page, first_response, final_response, selector_config
+ page_info.page, first_response, final_response, params.selector_config
)
# Mark the page as finished for next use