🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat: Validate all fetch-level parameters
Karim shoair60be9dc816b5
diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.pyindex 3412fd6..fcb77ac 100644--- a/scrapling/engines/_browsers/_camoufox.py+++ b/scrapling/engines/_browsers/_camoufox.py@@ -15,6 +15,7 @@ from playwright.async_api import (Page as async_Page,)+from ._validators import validate, CamoufoxConfigfrom ._base import SyncSession, AsyncSession, StealthySessionMixinfrom scrapling.core.utils import logfrom scrapling.core._types import (@@ -164,10 +165,8 @@ class StealthySession(StealthySessionMixin, SyncSession):def __create__(self):"""Create a browser for this instance and context."""self.playwright = sync_playwright().start()- self.context = (- self.playwright.firefox.launch_persistent_context( # pragma: no cover- **self.launch_options- )+ self.context = self.playwright.firefox.launch_persistent_context( # pragma: no cover+ **self.launch_options)# Get the default page and close it@@ -281,32 +280,22 @@ class StealthySession(StealthySessionMixin, SyncSession)::param selector_config: The arguments that will be passed in the end while creating the final Selector's class.:return: A `Response` object."""- google_search = self._get_with_precedence(- google_search, self.google_search, _UNSET- )- timeout = self._get_with_precedence(timeout, self.timeout, _UNSET)- wait = self._get_with_precedence(wait, self.wait, _UNSET)- page_action = self._get_with_precedence(page_action, self.page_action, _UNSET)- extra_headers = self._get_with_precedence(- extra_headers, self.extra_headers, _UNSET- )- disable_resources = self._get_with_precedence(- disable_resources, self.disable_resources, _UNSET- )- wait_selector = self._get_with_precedence(- wait_selector, self.wait_selector, _UNSET- )- wait_selector_state = self._get_with_precedence(- wait_selector_state, self.wait_selector_state, _UNSET- )- network_idle = self._get_with_precedence(- network_idle, self.network_idle, _UNSET- )- solve_cloudflare = self._get_with_precedence(- solve_cloudflare, self.solve_cloudflare, _UNSET- )- selector_config = self._get_with_precedence(- selector_config, self.selector_config, _UNSET+ # Validate all resolved parameters+ params = validate(+ dict(+ google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),+ timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),+ wait=self._get_with_precedence(wait, self.wait, _UNSET),+ page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),+ extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),+ disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),+ wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),+ wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),+ network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),+ solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET),+ selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),+ ),+ CamoufoxConfig,)if self._closed: # pragma: no cover@@ -314,9 +303,7 @@ class StealthySession(StealthySessionMixin, SyncSession):final_response = Nonereferer = (- generate_convincing_referer(url)- if (google_search and "referer" not in self._headers_keys)- else None+ generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None)def handle_response(finished_response: SyncPlaywrightResponse):@@ -327,7 +314,7 @@ class StealthySession(StealthySessionMixin, SyncSession):):final_response = finished_response- page_info = self._get_page(timeout, extra_headers, disable_resources)+ page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)page_info.mark_busy(url=url)try: # pragma: no cover@@ -336,41 +323,41 @@ class StealthySession(StealthySessionMixin, SyncSession):first_response = page_info.page.goto(url, referer=referer)page_info.page.wait_for_load_state(state="domcontentloaded")- if network_idle:+ if params.network_idle:page_info.page.wait_for_load_state("networkidle")if not first_response:raise RuntimeError(f"Failed to get response for {url}")- if solve_cloudflare:+ if params.solve_cloudflare:self._solve_cloudflare(page_info.page)# Make sure the page is fully loaded after the captchapage_info.page.wait_for_load_state(state="load")page_info.page.wait_for_load_state(state="domcontentloaded")- if network_idle:+ if params.network_idle:page_info.page.wait_for_load_state("networkidle")- if page_action is not None:+ if params.page_action:try:- _ = page_action(page_info.page)+ _ = params.page_action(page_info.page)except Exception as e:log.error(f"Error executing page_action: {e}")- if wait_selector:+ if params.wait_selector:try:- waiter: Locator = page_info.page.locator(wait_selector)- waiter.first.wait_for(state=wait_selector_state)+ waiter: Locator = page_info.page.locator(params.wait_selector)+ waiter.first.wait_for(state=params.wait_selector_state)# Wait again after waiting for the selector, helpful with protections like Cloudflarepage_info.page.wait_for_load_state(state="load")page_info.page.wait_for_load_state(state="domcontentloaded")- if network_idle:+ if params.network_idle:page_info.page.wait_for_load_state("networkidle")except Exception as e:- log.error(f"Error waiting for selector {wait_selector}: {e}")+ log.error(f"Error waiting for selector {params.wait_selector}: {e}")- page_info.page.wait_for_timeout(wait)+ page_info.page.wait_for_timeout(params.wait)response = ResponseFactory.from_playwright_response(- page_info.page, first_response, final_response, selector_config+ page_info.page, first_response, final_response, params.selector_config)# Mark the page as finished for next use@@ -478,10 +465,8 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):async def __create__(self):"""Create a browser for this instance and context."""self.playwright: AsyncPlaywright = await async_playwright().start()- self.context: AsyncBrowserContext = (- await self.playwright.firefox.launch_persistent_context(- **self.launch_options- )+ self.context: AsyncBrowserContext = await self.playwright.firefox.launch_persistent_context(+ **self.launch_options)# Get the default page and close it@@ -551,9 +536,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):await page.wait_for_timeout(500)# Calculate the Captcha coordinates for any viewport- outer_box = await page.locator(- ".main-content p+div>div>div"- ).bounding_box()+ outer_box = await page.locator(".main-content p+div>div>div").bounding_box()captcha_x, captcha_y = outer_box["x"] + 26, outer_box["y"] + 25# Move the mouse to the center of the window, then press and hold the left mouse button@@ -597,32 +580,21 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession)::param selector_config: The arguments that will be passed in the end while creating the final Selector's class.:return: A `Response` object."""- google_search = self._get_with_precedence(- google_search, self.google_search, _UNSET- )- timeout = self._get_with_precedence(timeout, self.timeout, _UNSET)- wait = self._get_with_precedence(wait, self.wait, _UNSET)- page_action = self._get_with_precedence(page_action, self.page_action, _UNSET)- extra_headers = self._get_with_precedence(- extra_headers, self.extra_headers, _UNSET- )- disable_resources = self._get_with_precedence(- disable_resources, self.disable_resources, _UNSET- )- wait_selector = self._get_with_precedence(- wait_selector, self.wait_selector, _UNSET- )- wait_selector_state = self._get_with_precedence(- wait_selector_state, self.wait_selector_state, _UNSET- )- network_idle = self._get_with_precedence(- network_idle, self.network_idle, _UNSET- )- solve_cloudflare = self._get_with_precedence(- solve_cloudflare, self.solve_cloudflare, _UNSET- )- selector_config = self._get_with_precedence(- selector_config, self.selector_config, _UNSET+ params = validate(+ dict(+ google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),+ timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),+ wait=self._get_with_precedence(wait, self.wait, _UNSET),+ page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),+ extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),+ disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),+ wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),+ wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),+ network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),+ solve_cloudflare=self._get_with_precedence(solve_cloudflare, self.solve_cloudflare, _UNSET),+ selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),+ ),+ CamoufoxConfig,)if self._closed: # pragma: no cover@@ -630,9 +602,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):final_response = Nonereferer = (- generate_convincing_referer(url)- if (google_search and "referer" not in self._headers_keys)- else None+ generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None)async def handle_response(finished_response: AsyncPlaywrightResponse):@@ -643,7 +613,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):):final_response = finished_response- page_info = await self._get_page(timeout, extra_headers, disable_resources)+ page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)page_info.mark_busy(url=url)try:@@ -652,43 +622,43 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):first_response = await page_info.page.goto(url, referer=referer)await page_info.page.wait_for_load_state(state="domcontentloaded")- if network_idle:+ if params.network_idle:await page_info.page.wait_for_load_state("networkidle")if not first_response:raise RuntimeError(f"Failed to get response for {url}")- if solve_cloudflare:+ if params.solve_cloudflare:await self._solve_cloudflare(page_info.page)# Make sure the page is fully loaded after the captchaawait page_info.page.wait_for_load_state(state="load")await page_info.page.wait_for_load_state(state="domcontentloaded")- if network_idle:+ if params.network_idle:await page_info.page.wait_for_load_state("networkidle")- if page_action is not None:+ if params.page_action:try:- _ = await page_action(page_info.page)+ _ = await params.page_action(page_info.page)except Exception as e:log.error(f"Error executing page_action: {e}")- if wait_selector:+ if params.wait_selector:try:- waiter: AsyncLocator = page_info.page.locator(wait_selector)- await waiter.first.wait_for(state=wait_selector_state)+ waiter: AsyncLocator = page_info.page.locator(params.wait_selector)+ await waiter.first.wait_for(state=params.wait_selector_state)# Wait again after waiting for the selector, helpful with protections like Cloudflareawait page_info.page.wait_for_load_state(state="load")await page_info.page.wait_for_load_state(state="domcontentloaded")- if network_idle:+ if params.network_idle:await page_info.page.wait_for_load_state("networkidle")except Exception as e:- log.error(f"Error waiting for selector {wait_selector}: {e}")+ log.error(f"Error waiting for selector {params.wait_selector}: {e}")- await page_info.page.wait_for_timeout(wait)+ await page_info.page.wait_for_timeout(params.wait)# Create response objectresponse = await ResponseFactory.from_async_playwright_response(- page_info.page, first_response, final_response, selector_config+ page_info.page, first_response, final_response, params.selector_config)# Mark the page as finished for next usediff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.pyindex 49f6c9d..a53fa62 100644--- a/scrapling/engines/_browsers/_controllers.py+++ b/scrapling/engines/_browsers/_controllers.py@@ -18,6 +18,7 @@ from rebrowser_playwright.async_api import (from scrapling.core.utils import logfrom ._base import SyncSession, AsyncSession, DynamicSessionMixin+from ._validators import validate, PlaywrightConfigfrom scrapling.core._types import (Dict,List,@@ -157,13 +158,11 @@ class DynamicSession(DynamicSessionMixin, SyncSession):self.playwright: Playwright = sync_context().start()if self.cdp_url: # pragma: no cover- self.context = self.playwright.chromium.connect_over_cdp(- endpoint_url=self.cdp_url- ).new_context(**self.context_options)- else:- self.context = self.playwright.chromium.launch_persistent_context(- user_data_dir="", **self.launch_options+ self.context = self.playwright.chromium.connect_over_cdp(endpoint_url=self.cdp_url).new_context(+ **self.context_options)+ else:+ self.context = self.playwright.chromium.launch_persistent_context(user_data_dir="", **self.launch_options)# Get the default page and close itdefault_page = self.context.pages[0]@@ -228,29 +227,21 @@ class DynamicSession(DynamicSessionMixin, SyncSession)::param selector_config: The arguments that will be passed in the end while creating the final Selector's class.:return: A `Response` object."""- google_search = self._get_with_precedence(- google_search, self.google_search, _UNSET- )- timeout = self._get_with_precedence(timeout, self.timeout, _UNSET)- wait = self._get_with_precedence(wait, self.wait, _UNSET)- page_action = self._get_with_precedence(page_action, self.page_action, _UNSET)- extra_headers = self._get_with_precedence(- extra_headers, self.extra_headers, _UNSET- )- disable_resources = self._get_with_precedence(- disable_resources, self.disable_resources, _UNSET- )- wait_selector = self._get_with_precedence(- wait_selector, self.wait_selector, _UNSET- )- wait_selector_state = self._get_with_precedence(- wait_selector_state, self.wait_selector_state, _UNSET- )- network_idle = self._get_with_precedence(- network_idle, self.network_idle, _UNSET- )- selector_config = self._get_with_precedence(- selector_config, self.selector_config, _UNSET+ # Validate all resolved parameters+ params = validate(+ dict(+ google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),+ timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),+ wait=self._get_with_precedence(wait, self.wait, _UNSET),+ page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),+ extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),+ disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),+ wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),+ wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),+ network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),+ selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),+ ),+ PlaywrightConfig,)if self._closed: # pragma: no cover@@ -258,9 +249,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):final_response = Nonereferer = (- generate_convincing_referer(url)- if (google_search and "referer" not in self._headers_keys)- else None+ generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None)def handle_response(finished_response: SyncPlaywrightResponse):@@ -271,7 +260,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):):final_response = finished_response- page_info = self._get_page(timeout, extra_headers, disable_resources)+ page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)page_info.mark_busy(url=url)try: # pragma: no cover@@ -280,35 +269,35 @@ class DynamicSession(DynamicSessionMixin, SyncSession):first_response = page_info.page.goto(url, referer=referer)page_info.page.wait_for_load_state(state="domcontentloaded")- if network_idle:+ if params.network_idle:page_info.page.wait_for_load_state("networkidle")if not first_response:raise RuntimeError(f"Failed to get response for {url}")- if page_action is not None:+ if params.page_action:try:- _ = page_action(page_info.page)+ _ = params.page_action(page_info.page)except Exception as e: # pragma: no coverlog.error(f"Error executing page_action: {e}")- if wait_selector:+ if params.wait_selector:try:- waiter: Locator = page_info.page.locator(wait_selector)- waiter.first.wait_for(state=wait_selector_state)+ waiter: Locator = page_info.page.locator(params.wait_selector)+ waiter.first.wait_for(state=params.wait_selector_state)# Wait again after waiting for the selector, helpful with protections like Cloudflarepage_info.page.wait_for_load_state(state="load")page_info.page.wait_for_load_state(state="domcontentloaded")- if network_idle:+ if params.network_idle:page_info.page.wait_for_load_state("networkidle")except Exception as e: # pragma: no cover- log.error(f"Error waiting for selector {wait_selector}: {e}")+ log.error(f"Error waiting for selector {params.wait_selector}: {e}")- page_info.page.wait_for_timeout(wait)+ page_info.page.wait_for_timeout(params.wait)# Create response objectresponse = ResponseFactory.from_playwright_response(- page_info.page, first_response, final_response, selector_config+ page_info.page, first_response, final_response, params.selector_config)# Mark the page as finished for next use@@ -413,17 +402,11 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):self.playwright: AsyncPlaywright = await async_context().start()if self.cdp_url:- browser = await self.playwright.chromium.connect_over_cdp(- endpoint_url=self.cdp_url- )- self.context: AsyncBrowserContext = await browser.new_context(- **self.context_options- )+ browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self.cdp_url)+ self.context: AsyncBrowserContext = await browser.new_context(**self.context_options)else:- self.context: AsyncBrowserContext = (- await self.playwright.chromium.launch_persistent_context(- user_data_dir="", **self.launch_options- )+ self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(+ user_data_dir="", **self.launch_options)# Get the default page and close it@@ -489,29 +472,21 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession)::param selector_config: The arguments that will be passed in the end while creating the final Selector's class.:return: A `Response` object."""- google_search = self._get_with_precedence(- google_search, self.google_search, _UNSET- )- timeout = self._get_with_precedence(timeout, self.timeout, _UNSET)- wait = self._get_with_precedence(wait, self.wait, _UNSET)- page_action = self._get_with_precedence(page_action, self.page_action, _UNSET)- extra_headers = self._get_with_precedence(- extra_headers, self.extra_headers, _UNSET- )- disable_resources = self._get_with_precedence(- disable_resources, self.disable_resources, _UNSET- )- wait_selector = self._get_with_precedence(- wait_selector, self.wait_selector, _UNSET- )- wait_selector_state = self._get_with_precedence(- wait_selector_state, self.wait_selector_state, _UNSET- )- network_idle = self._get_with_precedence(- network_idle, self.network_idle, _UNSET- )- selector_config = self._get_with_precedence(- selector_config, self.selector_config, _UNSET+ # Validate all resolved parameters+ params = validate(+ dict(+ google_search=self._get_with_precedence(google_search, self.google_search, _UNSET),+ timeout=self._get_with_precedence(timeout, self.timeout, _UNSET),+ wait=self._get_with_precedence(wait, self.wait, _UNSET),+ page_action=self._get_with_precedence(page_action, self.page_action, _UNSET),+ extra_headers=self._get_with_precedence(extra_headers, self.extra_headers, _UNSET),+ disable_resources=self._get_with_precedence(disable_resources, self.disable_resources, _UNSET),+ wait_selector=self._get_with_precedence(wait_selector, self.wait_selector, _UNSET),+ wait_selector_state=self._get_with_precedence(wait_selector_state, self.wait_selector_state, _UNSET),+ network_idle=self._get_with_precedence(network_idle, self.network_idle, _UNSET),+ selector_config=self._get_with_precedence(selector_config, self.selector_config, _UNSET),+ ),+ PlaywrightConfig,)if self._closed: # pragma: no cover@@ -519,9 +494,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):final_response = Nonereferer = (- generate_convincing_referer(url)- if (google_search and "referer" not in self._headers_keys)- else None+ generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None)async def handle_response(finished_response: AsyncPlaywrightResponse):@@ -532,7 +505,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):):final_response = finished_response- page_info = await self._get_page(timeout, extra_headers, disable_resources)+ page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)page_info.mark_busy(url=url)try:@@ -541,35 +514,35 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):first_response = await page_info.page.goto(url, referer=referer)await page_info.page.wait_for_load_state(state="domcontentloaded")- if network_idle:+ if params.network_idle:await page_info.page.wait_for_load_state("networkidle")if not first_response:raise RuntimeError(f"Failed to get response for {url}")- if page_action is not None:+ if params.page_action:try:- _ = await page_action(page_info.page)+ _ = await params.page_action(page_info.page)except Exception as e:log.error(f"Error executing page_action: {e}")- if wait_selector:+ if params.wait_selector:try:- waiter: AsyncLocator = page_info.page.locator(wait_selector)- await waiter.first.wait_for(state=wait_selector_state)+ waiter: AsyncLocator = page_info.page.locator(params.wait_selector)+ await waiter.first.wait_for(state=params.wait_selector_state)# Wait again after waiting for the selector, helpful with protections like Cloudflareawait page_info.page.wait_for_load_state(state="load")await page_info.page.wait_for_load_state(state="domcontentloaded")- if network_idle:+ if params.network_idle:await page_info.page.wait_for_load_state("networkidle")except Exception as e:- log.error(f"Error waiting for selector {wait_selector}: {e}")+ log.error(f"Error waiting for selector {params.wait_selector}: {e}")- await page_info.page.wait_for_timeout(wait)+ await page_info.page.wait_for_timeout(params.wait)# Create response objectresponse = await ResponseFactory.from_async_playwright_response(- page_info.page, first_response, final_response, selector_config+ page_info.page, first_response, final_response, params.selector_config)# Mark the page as finished for next use