🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
refactor(fetchers): Less duplicated code and better handling for unstable websites
Karim shoair59ec6be20108
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.pyindex e01a820..4157494 100644--- a/scrapling/engines/_browsers/_base.py+++ b/scrapling/engines/_browsers/_base.py@@ -2,17 +2,27 @@ from time import timefrom asyncio import sleep as asyncio_sleep, Lockfrom camoufox import DefaultAddons-from playwright.sync_api import BrowserContext, Playwright+from playwright.sync_api import (+ Page,+ Frame,+ BrowserContext,+ Playwright,+ Response as SyncPlaywrightResponse,+)from playwright.async_api import (- BrowserContext as AsyncBrowserContext,+ Page as AsyncPage,+ Frame as AsyncFrame,Playwright as AsyncPlaywright,+ Response as AsyncPlaywrightResponse,+ BrowserContext as AsyncBrowserContext,)+from playwright._impl._errors import Error as PlaywrightErrorfrom camoufox.pkgman import installed_verstr as camoufox_versionfrom camoufox.utils import launch_options as generate_launch_optionsfrom ._page import PageInfo, PagePoolfrom scrapling.parser import Selector-from scrapling.core._types import Any, cast, Dict, Optional, TYPE_CHECKING+from scrapling.core._types import Any, cast, Dict, List, Optional, Callable, TYPE_CHECKINGfrom scrapling.engines.toolbelt.fingerprints import get_os_namefrom ._validators import validate, PlaywrightConfig, CamoufoxConfigfrom ._config_tools import _compiled_stealth_scripts, _launch_kwargs, _context_kwargs@@ -26,10 +36,35 @@ class SyncSession:self.max_pages = max_pagesself.page_pool = PagePool(max_pages)self._max_wait_for_page = 60- self.playwright: Optional[Playwright] = None- self.context: Optional[BrowserContext] = None+ self.playwright: Playwright | Any = None+ self.context: BrowserContext | Any = Noneself._closed = False+ def __create__(self):+ pass++ def close(self): # pragma: no cover+ """Close all resources"""+ if self._closed:+ return++ if self.context:+ self.context.close()+ self.context = None++ if self.playwright:+ self.playwright.stop()+ self.playwright = None # pyright: ignore++ self._closed = True++ def __enter__(self):+ self.__create__()+ return self++ def __exit__(self, exc_type, exc_val, exc_tb):+ self.close()+def _get_page(self,timeout: int | float,@@ -65,17 +100,76 @@ class SyncSession:"max_pages": self.max_pages,}+ @staticmethod+ def _wait_for_networkidle(page: Page | Frame, timeout: Optional[int] = None):+ """Wait for the page to become idle (no network activity) even if there are never-ending requests."""+ try:+ page.wait_for_load_state("networkidle", timeout=timeout)+ except PlaywrightError:+ pass++ def _wait_for_page_stability(self, page: Page | Frame, load_dom: bool, network_idle: bool):+ page.wait_for_load_state(state="load")+ if load_dom:+ page.wait_for_load_state(state="domcontentloaded")+ if network_idle:+ self._wait_for_networkidle(page)++ @staticmethod+ def _create_response_handler(page_info: PageInfo, response_container: List) -> Callable:+ """Create a response handler that captures the final navigation response.++ :param page_info: The PageInfo object containing the page+ :param response_container: A list to store the final response (mutable container)+ :return: A callback function for page.on("response", ...)+ """++ def handle_response(finished_response: SyncPlaywrightResponse):+ if (+ finished_response.request.resource_type == "document"+ and finished_response.request.is_navigation_request()+ and finished_response.request.frame == page_info.page.main_frame+ ):+ response_container[0] = finished_response++ return handle_response+class AsyncSession:def __init__(self, max_pages: int = 1):self.max_pages = max_pagesself.page_pool = PagePool(max_pages)self._max_wait_for_page = 60- self.playwright: Optional[AsyncPlaywright] = None- self.context: Optional[AsyncBrowserContext] = None+ self.playwright: AsyncPlaywright | Any = None+ self.context: AsyncBrowserContext | Any = Noneself._closed = Falseself._lock = Lock()+ async def __create__(self):+ pass++ async def close(self):+ """Close all resources"""+ if self._closed: # pragma: no cover+ return++ if self.context:+ await self.context.close()+ self.context = None # pyright: ignore++ if self.playwright:+ await self.playwright.stop()+ self.playwright = None # pyright: ignore++ self._closed = True++ async def __aenter__(self):+ await self.__create__()+ return self++ async def __aexit__(self, exc_type, exc_val, exc_tb):+ await self.close()+async def _get_page(self,timeout: int | float,@@ -122,6 +216,40 @@ class AsyncSession:"max_pages": self.max_pages,}+ @staticmethod+ async def _wait_for_networkidle(page: AsyncPage | AsyncFrame, timeout: Optional[int] = None):+ """Wait for the page to become idle (no network activity) even if there are never-ending requests."""+ try:+ await page.wait_for_load_state("networkidle", timeout=timeout)+ except PlaywrightError:+ pass++ async def _wait_for_page_stability(self, page: AsyncPage | AsyncFrame, load_dom: bool, network_idle: bool):+ await page.wait_for_load_state(state="load")+ if load_dom:+ await page.wait_for_load_state(state="domcontentloaded")+ if network_idle:+ await self._wait_for_networkidle(page)++ @staticmethod+ def _create_response_handler(page_info: PageInfo, response_container: List) -> Callable:+ """Create an async response handler that captures the final navigation response.++ :param page_info: The PageInfo object containing the page+ :param response_container: A list to store the final response (mutable container)+ :return: A callback function for page.on("response", ...)+ """++ async def handle_response(finished_response: AsyncPlaywrightResponse):+ if (+ finished_response.request.resource_type == "document"+ and finished_response.request.is_navigation_request()+ and finished_response.request.frame == page_info.page.main_frame+ ):+ response_container[0] = finished_response++ return handle_response+class DynamicSessionMixin:def __validate__(self, **params):diff --git a/scrapling/engines/_browsers/_camoufox.py b/scrapling/engines/_browsers/_camoufox.pyindex e66a826..ac66870 100644--- a/scrapling/engines/_browsers/_camoufox.py+++ b/scrapling/engines/_browsers/_camoufox.py@@ -2,18 +2,16 @@ from random import randintfrom re import compile as re_compilefrom playwright.sync_api import (- Response as SyncPlaywrightResponse,- sync_playwright,- Locator,Page,+ Locator,+ sync_playwright,)from playwright.async_api import (async_playwright,- Response as AsyncPlaywrightResponse,- BrowserContext as AsyncBrowserContext,- Playwright as AsyncPlaywright,- Locator as AsyncLocator,Page as async_Page,+ Locator as AsyncLocator,+ Playwright as AsyncPlaywright,+ BrowserContext as AsyncBrowserContext,)from playwright._impl._errors import Error as PlaywrightError@@ -184,38 +182,13 @@ class StealthySession(StealthySessionMixin, SyncSession):if self.cookies: # pragma: no coverself.context.add_cookies(self.cookies)- def __enter__(self): # pragma: no cover- self.__create__()- return self-- def __exit__(self, exc_type, exc_val, exc_tb):- self.close()-- def close(self): # pragma: no cover- """Close all resources"""- if self._closed: # pragma: no cover- return-- if self.context:- self.context.close()- self.context = None-- if self.playwright:- self.playwright.stop()- self.playwright = None-- self._closed = True-def _solve_cloudflare(self, page: Page) -> None: # pragma: no cover"""Solve the cloudflare challenge displayed on the playwright page passed:param page: The targeted page:return:"""- try:- page.wait_for_load_state("networkidle", timeout=5000)- except PlaywrightError:- pass+ self._wait_for_networkidle(page, timeout=5000)challenge_type = self._detect_cloudflare(ResponseFactory._get_page_content(page))if not challenge_type:log.error("No Cloudflare challenge found.")@@ -241,8 +214,7 @@ class StealthySession(StealthySessionMixin, SyncSession):outer_box = {}iframe = page.frame(url=__CF_PATTERN__)if iframe is not None:- iframe.wait_for_load_state(state="domcontentloaded")- iframe.wait_for_load_state("networkidle")+ self._wait_for_page_stability(iframe, True, True)if challenge_type != "embedded":while not iframe.frame_element().is_visible():@@ -258,7 +230,7 @@ class StealthySession(StealthySessionMixin, SyncSession):# Move the mouse to the center of the window, then press and hold the left mouse buttonpage.mouse.click(captcha_x, captcha_y, delay=60, button="left")- page.wait_for_load_state("networkidle")+ self._wait_for_networkidle(page)if iframe is not None:# Wait for the frame to be removed from the pagewhile iframe in page.frames:@@ -266,8 +238,7 @@ class StealthySession(StealthySessionMixin, SyncSession):if challenge_type != "embedded":page.locator(box_selector).last.wait_for(state="detached")page.locator(".zone-name-title").wait_for(state="hidden")- page.wait_for_load_state(state="load")- page.wait_for_load_state(state="domcontentloaded")+ self._wait_for_page_stability(page, True, False)log.info("Cloudflare captcha is solved")return@@ -328,32 +299,19 @@ class StealthySession(StealthySessionMixin, SyncSession):if self._closed: # pragma: no coverraise RuntimeError("Context manager has been closed")- final_response = Nonereferer = (generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None)- def handle_response(finished_response: SyncPlaywrightResponse):- nonlocal final_response- if (- finished_response.request.resource_type == "document"- and finished_response.request.is_navigation_request()- and finished_response.request.frame == page_info.page.main_frame- ):- final_response = finished_response-page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)- page_info.mark_busy(url=url)+ final_response = [None]+ handle_response = self._create_response_handler(page_info, final_response)try: # pragma: no cover# Navigate to URL and wait for a specified statepage_info.page.on("response", handle_response)first_response = page_info.page.goto(url, referer=referer)- if params.load_dom:- page_info.page.wait_for_load_state(state="domcontentloaded")-- if params.network_idle:- page_info.page.wait_for_load_state("networkidle")+ self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)if not first_response:raise RuntimeError(f"Failed to get response for {url}")@@ -361,11 +319,7 @@ class StealthySession(StealthySessionMixin, SyncSession):if params.solve_cloudflare:self._solve_cloudflare(page_info.page)# Make sure the page is fully loaded after the captcha- page_info.page.wait_for_load_state(state="load")- if params.load_dom:- page_info.page.wait_for_load_state(state="domcontentloaded")- if params.network_idle:- page_info.page.wait_for_load_state("networkidle")+ self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)if params.page_action:try:@@ -378,17 +332,13 @@ class StealthySession(StealthySessionMixin, SyncSession):waiter: Locator = page_info.page.locator(params.wait_selector)waiter.first.wait_for(state=params.wait_selector_state)# Wait again after waiting for the selector, helpful with protections like Cloudflare- page_info.page.wait_for_load_state(state="load")- if params.load_dom:- page_info.page.wait_for_load_state(state="domcontentloaded")- if params.network_idle:- page_info.page.wait_for_load_state("networkidle")+ self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)except Exception as e:log.error(f"Error waiting for selector {params.wait_selector}: {e}")page_info.page.wait_for_timeout(params.wait)response = ResponseFactory.from_playwright_response(- page_info.page, first_response, final_response, params.selector_config, bool(params.page_action)+ page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action))# Close the page to free up resources@@ -513,38 +463,13 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):if self.cookies:await self.context.add_cookies(self.cookies) # pyright: ignore [reportArgumentType]- async def __aenter__(self):- await self.__create__()- return self-- async def __aexit__(self, exc_type, exc_val, exc_tb):- await self.close()-- async def close(self):- """Close all resources"""- if self._closed: # pragma: no cover- return-- if self.context:- await self.context.close()- self.context = None # pyright: ignore-- if self.playwright:- await self.playwright.stop()- self.playwright = None # pyright: ignore-- self._closed = True-- async def _solve_cloudflare(self, page: async_Page):+ async def _solve_cloudflare(self, page: async_Page): # pragma: no cover"""Solve the cloudflare challenge displayed on the playwright page passed. The async version:param page: The async targeted page:return:"""- try:- await page.wait_for_load_state("networkidle", timeout=5000)- except PlaywrightError:- pass+ await self._wait_for_networkidle(page, timeout=5000)challenge_type = self._detect_cloudflare(await ResponseFactory._get_async_page_content(page))if not challenge_type:log.error("No Cloudflare challenge found.")@@ -570,8 +495,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):outer_box = {}iframe = page.frame(url=__CF_PATTERN__)if iframe is not None:- await iframe.wait_for_load_state(state="domcontentloaded")- await iframe.wait_for_load_state("networkidle")+ await self._wait_for_page_stability(iframe, True, True)if challenge_type != "embedded":while not await (await iframe.frame_element()).is_visible():@@ -587,7 +511,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):# Move the mouse to the center of the window, then press and hold the left mouse buttonawait page.mouse.click(captcha_x, captcha_y, delay=60, button="left")- await page.wait_for_load_state("networkidle")+ await self._wait_for_networkidle(page)if iframe is not None:# Wait for the frame to be removed from the pagewhile iframe in page.frames:@@ -595,8 +519,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):if challenge_type != "embedded":await page.locator(box_selector).wait_for(state="detached")await page.locator(".zone-name-title").wait_for(state="hidden")- await page.wait_for_load_state(state="load")- await page.wait_for_load_state(state="domcontentloaded")+ await self._wait_for_page_stability(page, True, False)log.info("Cloudflare captcha is solved")return@@ -657,22 +580,13 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):if self._closed: # pragma: no coverraise RuntimeError("Context manager has been closed")- final_response = Nonereferer = (generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None)- async def handle_response(finished_response: AsyncPlaywrightResponse):- nonlocal final_response- if (- finished_response.request.resource_type == "document"- and finished_response.request.is_navigation_request()- and finished_response.request.frame == page_info.page.main_frame- ):- final_response = finished_response-page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)- page_info.mark_busy(url=url)+ final_response = [None]+ handle_response = self._create_response_handler(page_info, final_response)if TYPE_CHECKING:if not isinstance(page_info.page, async_Page):@@ -682,11 +596,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):# Navigate to URL and wait for a specified statepage_info.page.on("response", handle_response)first_response = await page_info.page.goto(url, referer=referer)- if params.load_dom:- await page_info.page.wait_for_load_state(state="domcontentloaded")-- if params.network_idle:- await page_info.page.wait_for_load_state("networkidle")+ await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)if not first_response:raise RuntimeError(f"Failed to get response for {url}")@@ -694,11 +604,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):if params.solve_cloudflare:await self._solve_cloudflare(page_info.page)# Make sure the page is fully loaded after the captcha- await page_info.page.wait_for_load_state(state="load")- if params.load_dom:- await page_info.page.wait_for_load_state(state="domcontentloaded")- if params.network_idle:- await page_info.page.wait_for_load_state("networkidle")+ await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)if params.page_action:try:@@ -711,11 +617,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):waiter: AsyncLocator = page_info.page.locator(params.wait_selector)await waiter.first.wait_for(state=params.wait_selector_state)# Wait again after waiting for the selector, helpful with protections like Cloudflare- await page_info.page.wait_for_load_state(state="load")- if params.load_dom:- await page_info.page.wait_for_load_state(state="domcontentloaded")- if params.network_idle:- await page_info.page.wait_for_load_state("networkidle")+ await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)except Exception as e:log.error(f"Error waiting for selector {params.wait_selector}: {e}")@@ -723,7 +625,7 @@ class AsyncStealthySession(StealthySessionMixin, AsyncSession):# Create response objectresponse = await ResponseFactory.from_async_playwright_response(- page_info.page, first_response, final_response, params.selector_config, bool(params.page_action)+ page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action))# Close the page to free up resourcesdiff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.pyindex 45a6938..155789a 100644--- a/scrapling/engines/_browsers/_controllers.py+++ b/scrapling/engines/_browsers/_controllers.py@@ -1,16 +1,13 @@from playwright.sync_api import (- Response as SyncPlaywrightResponse,- sync_playwright,- Playwright,Locator,+ Playwright,+ sync_playwright,)from playwright.async_api import (async_playwright,- Response as AsyncPlaywrightResponse,- BrowserContext as AsyncBrowserContext,- Playwright as AsyncPlaywright,Locator as AsyncLocator,- Page as async_Page,+ Playwright as AsyncPlaywright,+ BrowserContext as AsyncBrowserContext,)from patchright.sync_api import sync_playwright as sync_patchrightfrom patchright.async_api import async_playwright as async_patchright@@ -178,28 +175,6 @@ class DynamicSession(DynamicSessionMixin, SyncSession):if self.cookies: # pragma: no coverself.context.add_cookies(self.cookies)- def __enter__(self):- self.__create__()- return self-- def __exit__(self, exc_type, exc_val, exc_tb):- self.close()-- def close(self): # pragma: no cover- """Close all resources"""- if self._closed:- return-- if self.context:- self.context.close()- self.context = None-- if self.playwright:- self.playwright.stop()- self.playwright = None # pyright: ignore-- self._closed = True-def fetch(self,url: str,@@ -253,32 +228,19 @@ class DynamicSession(DynamicSessionMixin, SyncSession):if self._closed: # pragma: no coverraise RuntimeError("Context manager has been closed")- final_response = Nonereferer = (generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None)- def handle_response(finished_response: SyncPlaywrightResponse):- nonlocal final_response- if (- finished_response.request.resource_type == "document"- and finished_response.request.is_navigation_request()- and finished_response.request.frame == page_info.page.main_frame- ):- final_response = finished_response-page_info = self._get_page(params.timeout, params.extra_headers, params.disable_resources)- page_info.mark_busy(url=url)+ final_response = [None]+ handle_response = self._create_response_handler(page_info, final_response)try: # pragma: no cover# Navigate to URL and wait for a specified statepage_info.page.on("response", handle_response)first_response = page_info.page.goto(url, referer=referer)- if params.load_dom:- page_info.page.wait_for_load_state(state="domcontentloaded")-- if params.network_idle:- page_info.page.wait_for_load_state("networkidle")+ self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)if not first_response:raise RuntimeError(f"Failed to get response for {url}")@@ -294,11 +256,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):waiter: Locator = page_info.page.locator(params.wait_selector)waiter.first.wait_for(state=params.wait_selector_state)# Wait again after waiting for the selector, helpful with protections like Cloudflare- page_info.page.wait_for_load_state(state="load")- if params.load_dom:- page_info.page.wait_for_load_state(state="domcontentloaded")- if params.network_idle:- page_info.page.wait_for_load_state("networkidle")+ self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)except Exception as e: # pragma: no coverlog.error(f"Error waiting for selector {params.wait_selector}: {e}")@@ -306,7 +264,7 @@ class DynamicSession(DynamicSessionMixin, SyncSession):# Create response objectresponse = ResponseFactory.from_playwright_response(- page_info.page, first_response, final_response, params.selector_config, bool(params.page_action)+ page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action))# Close the page to free up resources@@ -431,28 +389,6 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):if self.cookies:await self.context.add_cookies(self.cookies) # pyright: ignore- async def __aenter__(self):- await self.__create__()- return self-- async def __aexit__(self, exc_type, exc_val, exc_tb):- await self.close()-- async def close(self):- """Close all resources"""- if self._closed: # pragma: no cover- return-- if self.context:- await self.context.close()- self.context = None # pyright: ignore-- if self.playwright:- await self.playwright.stop()- self.playwright = None # pyright: ignore-- self._closed = True-async def fetch(self,url: str,@@ -506,24 +442,17 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):if self._closed: # pragma: no coverraise RuntimeError("Context manager has been closed")- final_response = Nonereferer = (generate_convincing_referer(url) if (params.google_search and "referer" not in self._headers_keys) else None)- async def handle_response(finished_response: AsyncPlaywrightResponse):- nonlocal final_response- if (- finished_response.request.resource_type == "document"- and finished_response.request.is_navigation_request()- and finished_response.request.frame == page_info.page.main_frame- ):- final_response = finished_response-page_info = await self._get_page(params.timeout, params.extra_headers, params.disable_resources)- page_info.mark_busy(url=url)+ final_response = [None]+ handle_response = self._create_response_handler(page_info, final_response)if TYPE_CHECKING:+ from playwright.async_api import Page as async_Page+if not isinstance(page_info.page, async_Page):raise TypeError@@ -531,11 +460,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):# Navigate to URL and wait for a specified statepage_info.page.on("response", handle_response)first_response = await page_info.page.goto(url, referer=referer)- if self.load_dom:- await page_info.page.wait_for_load_state(state="domcontentloaded")-- if params.network_idle:- await page_info.page.wait_for_load_state("networkidle")+ await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)if not first_response:raise RuntimeError(f"Failed to get response for {url}")@@ -551,11 +476,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):waiter: AsyncLocator = page_info.page.locator(params.wait_selector)await waiter.first.wait_for(state=params.wait_selector_state)# Wait again after waiting for the selector, helpful with protections like Cloudflare- await page_info.page.wait_for_load_state(state="load")- if self.load_dom:- await page_info.page.wait_for_load_state(state="domcontentloaded")- if params.network_idle:- await page_info.page.wait_for_load_state("networkidle")+ await self._wait_for_page_stability(page_info.page, params.load_dom, params.network_idle)except Exception as e:log.error(f"Error waiting for selector {params.wait_selector}: {e}")@@ -563,7 +484,7 @@ class AsyncDynamicSession(DynamicSessionMixin, AsyncSession):# Create response objectresponse = await ResponseFactory.from_async_playwright_response(- page_info.page, first_response, final_response, params.selector_config, bool(params.page_action)+ page_info.page, first_response, final_response[0], params.selector_config, bool(params.page_action))# Close the page to free up resourcesdiff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.pyindex 43ef61c..1f20038 100644--- a/scrapling/engines/toolbelt/custom.py+++ b/scrapling/engines/toolbelt/custom.py@@ -209,15 +209,3 @@ class StatusText:def get(cls, status_code: int) -> str:"""Get the phrase for a given HTTP status code."""return cls._phrases.get(status_code, "Unknown Status Code")---def get_variable_name(var: Any) -> Optional[str]:- """Get the name of a variable using global and local scopes.- :param var: The variable to find the name for- :return: The name of the variable if found, None otherwise- """- for scope in [globals(), locals()]:- for name, value in scope.items():- if value is var:- return name- return Nonediff --git a/scrapling/engines/toolbelt/fingerprints.py b/scrapling/engines/toolbelt/fingerprints.pyindex 7bcad8a..c006ec2 100644--- a/scrapling/engines/toolbelt/fingerprints.py+++ b/scrapling/engines/toolbelt/fingerprints.py@@ -7,8 +7,9 @@ from platform import system as platform_systemfrom tldextract import extractfrom browserforge.headers import Browser, HeaderGenerator+from browserforge.headers.generator import SUPPORTED_OPERATING_SYSTEMS-from scrapling.core._types import Dict, Literal+from scrapling.core._types import Dict, Literal, Tuple__OS_NAME__ = platform_system()OSName = Literal["linux", "macos", "windows"]@@ -29,12 +30,12 @@ def generate_convincing_referer(url: str) -> str:@lru_cache(1, typed=True)-def get_os_name() -> OSName | None:+def get_os_name() -> OSName | Tuple:"""Get the current OS name in the same format needed for browserforge, if the OS is Unknown, return None so browserforge uses all.:return: Current OS name or `None` otherwise"""- match __OS_NAME__:+ match __OS_NAME__: # pragma: no covercase "Linux":return "linux"case "Darwin":@@ -42,7 +43,7 @@ def get_os_name() -> OSName | None:case "Windows":return "windows"case _:- return None+ return SUPPORTED_OPERATING_SYSTEMSdef generate_headers(browser_mode: bool = False) -> Dict:@@ -63,10 +64,7 @@ def generate_headers(browser_mode: bool = False) -> Dict:Browser(name="edge", min_version=130),])- if os_name:- return HeaderGenerator(browser=browsers, os=os_name, device="desktop").generate()- else:- return HeaderGenerator(browser=browsers, device="desktop").generate()+ return HeaderGenerator(browser=browsers, os=os_name, device="desktop").generate()__default_useragent__ = generate_headers(browser_mode=False).get("User-Agent")