🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
style: Fix all mypy errors and add type hints to untyped function bodies
Karim shoair5ec929435be2
diff --git a/scrapling/core/_types.py b/scrapling/core/_types.pyindex f19e706..f2fc097 100644--- a/scrapling/core/_types.py+++ b/scrapling/core/_types.py@@ -32,6 +32,7 @@ from typing import (Coroutine,SupportsIndex,)+from typing_extensions import Self, Unpack# Proxy can be a string URL or a dict (Playwright format: {"server": "...", "username": "...", "password": "..."})ProxyType = Union[str, Dict[str, str]]@@ -41,27 +42,6 @@ PageLoadStates = Literal["commit", "domcontentloaded", "load", "networkidle"]extraction_types = Literal["text", "html", "markdown"]StrOrBytes = Union[str, bytes]-if TYPE_CHECKING: # pragma: no cover- from typing_extensions import Unpack-else: # pragma: no cover-- class _Unpack:- @staticmethod- def __getitem__(*args, **kwargs):- pass-- Unpack = _Unpack()---try:- # Python 3.11+- from typing import Self # novermin-except ImportError: # pragma: no cover- try:- from typing_extensions import Self # Backport- except ImportError:- Self = object-# Copied from `playwright._impl._api_structures.SetCookieParam`class SetCookieParam(TypedDict, total=False):diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.pyindex 45ab4d5..d06ac29 100644--- a/scrapling/core/custom_types.py+++ b/scrapling/core/custom_types.py@@ -35,9 +35,7 @@ class TextHandler(str):lst = super().__getitem__(key)return TextHandler(lst)- def split(- self, sep: str | None = None, maxsplit: SupportsIndex = -1- ) -> Union[List, "TextHandlers"]: # pragma: no cover+ def split(self, sep: str | None = None, maxsplit: SupportsIndex = -1) -> list[Any]: # pragma: no coverreturn TextHandlers([TextHandler(s) for s in super().split(sep, maxsplit)])def strip(self, chars: str | None = None) -> Union[str, "TextHandler"]: # pragma: no cover@@ -61,7 +59,7 @@ class TextHandler(str):def expandtabs(self, tabsize: SupportsIndex = 8) -> Union[str, "TextHandler"]: # pragma: no coverreturn TextHandler(super().expandtabs(tabsize))- def format(self, *args: object, **kwargs: str) -> Union[str, "TextHandler"]: # pragma: no cover+ def format(self, *args: object, **kwargs: object) -> Union[str, "TextHandler"]: # pragma: no coverreturn TextHandler(super().format(*args, **kwargs))def format_map(self, mapping) -> Union[str, "TextHandler"]: # pragma: no cover@@ -291,7 +289,7 @@ class AttributesHandler(Mapping[str, _TextHandlerType]):__slots__ = ("_data",)- def __init__(self, mapping=None, **kwargs):+ def __init__(self, mapping: Any = None, **kwargs: Any) -> None:mapping = ({key: TextHandler(value) if isinstance(value, str) else value for key, value in mapping.items()}if mapping is not None@@ -324,8 +322,8 @@ class AttributesHandler(Mapping[str, _TextHandlerType]):yield AttributesHandler({key: value})@property- def json_string(self):- """Convert current attributes to JSON string if the attributes are JSON serializable otherwise throws error"""+ def json_string(self) -> bytes:+ """Convert current attributes to JSON bytes if the attributes are JSON serializable otherwise throws error"""return dumps(dict(self._data))def __getitem__(self, key: str) -> _TextHandlerType:diff --git a/scrapling/core/mixins.py b/scrapling/core/mixins.pyindex 0b66c07..c2e7420 100644--- a/scrapling/core/mixins.py+++ b/scrapling/core/mixins.py@@ -1,7 +1,4 @@-from scrapling.core._types import TYPE_CHECKING--if TYPE_CHECKING:- from scrapling.parser import Selector+from scrapling.core._types import Any, Dictclass SelectorsGeneration:@@ -11,7 +8,11 @@ class SelectorsGeneration:Inspiration: https://searchfox.org/mozilla-central/source/devtools/shared/inspector/css-logic.js#591"""- def _general_selection(self: "Selector", selection: str = "css", full_path: bool = False) -> str: # type: ignore[name-defined]+ # Note: This is a mixin class meant to be used with Selector.+ # The methods access Selector attributes (._root, .parent, .attrib, .tag, etc.)+ # through self, which will be a Selector instance at runtime.++ def _general_selection(self: Any, selection: str = "css", full_path: bool = False) -> str:"""Generate a selector for the current element.:return: A string of the generated selector."""@@ -36,7 +37,7 @@ class SelectorsGeneration:# if classes and css:# part += f".{'.'.join(classes)}"# else:- counter = {}+ counter: Dict[str, int] = {}for child in target.parent.children:counter.setdefault(child.tag, 0)counter[child.tag] += 1@@ -56,28 +57,28 @@ class SelectorsGeneration:return " > ".join(reversed(selectorPath)) if css else "//" + "/".join(reversed(selectorPath))@property- def generate_css_selector(self: "Selector") -> str: # type: ignore[name-defined]+ def generate_css_selector(self: Any) -> str:"""Generate a CSS selector for the current element:return: A string of the generated selector."""return self._general_selection()@property- def generate_full_css_selector(self: "Selector") -> str: # type: ignore[name-defined]+ def generate_full_css_selector(self: Any) -> str:"""Generate a complete CSS selector for the current element:return: A string of the generated selector."""return self._general_selection(full_path=True)@property- def generate_xpath_selector(self: "Selector") -> str: # type: ignore[name-defined]+ def generate_xpath_selector(self: Any) -> str:"""Generate an XPath selector for the current element:return: A string of the generated selector."""return self._general_selection("xpath")@property- def generate_full_xpath_selector(self: "Selector") -> str: # type: ignore[name-defined]+ def generate_full_xpath_selector(self: Any) -> str:"""Generate a complete XPath selector for the current element:return: A string of the generated selector."""diff --git a/scrapling/core/shell.py b/scrapling/core/shell.pyindex 3a1511e..0bd6efc 100644--- a/scrapling/core/shell.py+++ b/scrapling/core/shell.py@@ -30,6 +30,7 @@ from scrapling.core.custom_types import TextHandlerfrom scrapling.engines.toolbelt.custom import Responsefrom scrapling.core.utils._shell import _ParseHeaders, _CookieParserfrom scrapling.core._types import (+ Callable,Dict,Any,cast,@@ -82,7 +83,7 @@ class NoExitArgumentParser(ArgumentParser): # pragma: no coverclass CurlParser:"""Builds the argument parser for relevant curl flags from DevTools."""- def __init__(self):+ def __init__(self) -> None:from scrapling.fetchers import Fetcher as __Fetcherself.__fetcher = __Fetcher@@ -467,19 +468,21 @@ Type 'exit' or press Ctrl+D to exit.return result- def create_wrapper(self, func, get_signature=True, signature_name=None):+ def create_wrapper(+ self, func: Callable, get_signature: bool = True, signature_name: Optional[str] = None+ ) -> Callable:"""Create a wrapper that preserves function signature but updates page"""@wraps(func)- def wrapper(*args, **kwargs):+ def wrapper(*args: Any, **kwargs: Any) -> Any:result = func(*args, **kwargs)return self.update_page(result)if get_signature:# Explicitly preserve and unpack signature for IPython introspection and autocompletion- wrapper.__signature__ = _unpack_signature(func, signature_name) # pyright: ignore+ setattr(wrapper, "__signature__", _unpack_signature(func, signature_name))else:- wrapper.__signature__ = signature(func) # pyright: ignore+ setattr(wrapper, "__signature__", signature(func))return wrapper@@ -601,7 +604,7 @@ class Convertor:" ",):# Remove consecutive white-spaces- txt_content = re_sub(f"[{s}]+", s, txt_content)+ txt_content = TextHandler(re_sub(f"[{s}]+", s, txt_content))yield txt_contentyield ""diff --git a/scrapling/core/storage.py b/scrapling/core/storage.pyindex 8e04599..d93a320 100644--- a/scrapling/core/storage.py+++ b/scrapling/core/storage.py@@ -63,12 +63,11 @@ class StorageSystemMixin(ABC): # pragma: no coverdef _get_hash(identifier: str) -> str:"""If you want to hash identifier in your storage system, use this safer"""_identifier = identifier.lower().strip()- if isinstance(_identifier, str):- # Hash functions have to take bytes- _identifier = _identifier.encode("utf-8")+ # Hash functions have to take bytes+ _identifier_bytes = _identifier.encode("utf-8")- hash_value = sha256(_identifier).hexdigest()- return f"{hash_value}_{len(_identifier)}" # Length to reduce collision chance+ hash_value = sha256(_identifier_bytes).hexdigest()+ return f"{hash_value}_{len(_identifier_bytes)}" # Length to reduce collision chance@lru_cache(1, typed=True)diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.pyindex 8bd442d..e379882 100644--- a/scrapling/engines/_browsers/_base.py+++ b/scrapling/engines/_browsers/_base.py@@ -5,16 +5,12 @@ from contextlib import contextmanager, asynccontextmanagerfrom playwright.sync_api._generated import Pagefrom playwright.sync_api import (Frame,- Browser,BrowserContext,- Playwright,Response as SyncPlaywrightResponse,)from playwright.async_api._generated import Page as AsyncPagefrom playwright.async_api import (Frame as AsyncFrame,- Browser as AsyncBrowser,- Playwright as AsyncPlaywright,Response as AsyncPlaywrightResponse,BrowserContext as AsyncBrowserContext,)@@ -37,6 +33,7 @@ from scrapling.core._types import (Optional,Callable,TYPE_CHECKING,+ cast,overload,Tuple,ProxyType,@@ -61,12 +58,12 @@ class SyncSession:self.max_pages = max_pagesself.page_pool = PagePool(max_pages)self._max_wait_for_page = 60- self.playwright: Playwright | Any = None- self.context: BrowserContext | Any = None- self.browser: Optional[Browser] = None+ self.playwright: Any = None+ self.context: Any = None+ self.browser: Any = Noneself._is_alive = False- def start(self):+ def start(self) -> None:passdef close(self): # pragma: no cover@@ -215,13 +212,13 @@ class AsyncSession:self.max_pages = max_pagesself.page_pool = PagePool(max_pages)self._max_wait_for_page = 60- self.playwright: AsyncPlaywright | Any = None- self.context: AsyncBrowserContext | Any = None- self.browser: Optional[AsyncBrowser] = None+ self.playwright: Any = None+ self.context: Any = None+ self.browser: Any = Noneself._is_alive = Falseself._lock = Lock()- async def start(self):+ async def start(self) -> None:passasync def close(self):@@ -378,6 +375,8 @@ class AsyncSession:class BaseSessionMixin:+ _config: "PlaywrightConfig | StealthConfig"+@overloaddef __validate_routine__(self, params: Dict, model: type[StealthConfig]) -> StealthConfig: ...@@ -404,7 +403,7 @@ class BaseSessionMixin:return configdef __generate_options__(self, extra_flags: Tuple | None = None) -> None:- config: PlaywrightConfig | StealthConfig = self._config # type: ignore[has-type]+ config: PlaywrightConfig | StealthConfig = self._configself._context_options.update({"proxy": config.proxy,@@ -466,7 +465,7 @@ class DynamicSessionMixin(BaseSessionMixin):class StealthySessionMixin(BaseSessionMixin):def __validate__(self, **params):- self._config: StealthConfig = self.__validate_routine__(params, model=StealthConfig)+ self._config = self.__validate_routine__(params, model=StealthConfig)self._context_options.update({"is_mobile": False,@@ -482,22 +481,23 @@ class StealthySessionMixin(BaseSessionMixin):self.__generate_stealth_options()def __generate_stealth_options(self) -> None:- flags = tuple()- if not self._config.cdp_url:+ config = cast(StealthConfig, self._config)+ flags: Tuple[str, ...] = tuple()+ if not config.cdp_url:flags = DEFAULT_FLAGS + DEFAULT_STEALTH_FLAGS- if self._config.block_webrtc:+ if config.block_webrtc:flags += ("--webrtc-ip-handling-policy=disable_non_proxied_udp","--force-webrtc-ip-handling-policy", # Ensures the policy is enforced)- if not self._config.allow_webgl:+ if not config.allow_webgl:flags += ("--disable-webgl","--disable-webgl-image-chromium","--disable-webgl2",)- if self._config.hide_canvas:+ if config.hide_canvas:flags += ("--fingerprinting-canvas-image-data-noise",)super(StealthySessionMixin, self).__generate_options__(flags)diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.pyindex 5f0b247..5d9e801 100644--- a/scrapling/engines/_browsers/_controllers.py+++ b/scrapling/engines/_browsers/_controllers.py@@ -8,11 +8,10 @@ from playwright.sync_api import (from playwright.async_api import (async_playwright,Locator as AsyncLocator,- BrowserContext as AsyncBrowserContext,)from scrapling.core.utils import log-from scrapling.core._types import Unpack+from scrapling.core._types import Optional, ProxyType, Unpackfrom scrapling.engines.toolbelt.proxy_rotation import is_proxy_errorfrom scrapling.engines.toolbelt.convertor import Response, ResponseFactoryfrom scrapling.engines.toolbelt.fingerprints import generate_convincing_referer@@ -134,6 +133,7 @@ class DynamicSession(SyncSession, DynamicSessionMixin):)for attempt in range(self._config.retries):+ proxy: Optional[ProxyType] = Noneif self._config.proxy_rotator and static_proxy is None:proxy = self._config.proxy_rotator.get_proxy()else:@@ -238,7 +238,7 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):self.__validate__(**kwargs)super().__init__(max_pages=self._config.max_pages)- async def start(self):+ async def start(self) -> None:"""Create a browser for this instance and context."""if not self.playwright:self.playwright = await async_playwright().start()@@ -246,16 +246,14 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):if self._config.cdp_url:self.browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url)if not self._config.proxy_rotator and self.browser:- self.context: AsyncBrowserContext = await self.browser.new_context(**self._context_options)+ self.context = await self.browser.new_context(**self._context_options)elif self._config.proxy_rotator:self.browser = await self.playwright.chromium.launch(**self._browser_options)else:persistent_options = (self._browser_options | self._context_options | {"user_data_dir": self._user_data_dir})- self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(- **persistent_options- )+ self.context = await self.playwright.chromium.launch_persistent_context(**persistent_options)if self.context:self.context = await self._initialize_context(self._config, self.context)@@ -304,6 +302,7 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):)for attempt in range(self._config.retries):+ proxy: Optional[ProxyType] = Noneif self._config.proxy_rotator and static_proxy is None:proxy = self._config.proxy_rotator.get_proxy()else:diff --git a/scrapling/engines/_browsers/_page.py b/scrapling/engines/_browsers/_page.pyindex 655d3d1..481016e 100644--- a/scrapling/engines/_browsers/_page.py+++ b/scrapling/engines/_browsers/_page.py@@ -61,7 +61,9 @@ class PagePool:raise RuntimeError(f"Maximum page limit ({self.max_pages}) reached")if isinstance(page, AsyncPage):- page_info = cast(PageInfo[AsyncPage], PageInfo(page, "ready", ""))+ page_info: PageInfo[SyncPage] | PageInfo[AsyncPage] = cast(+ PageInfo[AsyncPage], PageInfo(page, "ready", "")+ )else:page_info = cast(PageInfo[SyncPage], PageInfo(page, "ready", ""))diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.pyindex 730cd7e..3c9ea58 100644--- a/scrapling/engines/_browsers/_stealth.py+++ b/scrapling/engines/_browsers/_stealth.py@@ -13,7 +13,7 @@ from patchright.sync_api import sync_playwrightfrom patchright.async_api import async_playwrightfrom scrapling.core.utils import log-from scrapling.core._types import Any, Unpack+from scrapling.core._types import Any, Optional, ProxyType, Unpackfrom scrapling.engines.toolbelt.proxy_rotation import is_proxy_errorfrom scrapling.engines.toolbelt.convertor import Response, ResponseFactoryfrom scrapling.engines.toolbelt.fingerprints import generate_convincing_referer@@ -78,7 +78,7 @@ class StealthySession(SyncSession, StealthySessionMixin):self.__validate__(**kwargs)super().__init__()- def start(self):+ def start(self) -> None:"""Create a browser for this instance and context."""if not self.playwright:self.playwright = sync_playwright().start()@@ -146,7 +146,7 @@ class StealthySession(SyncSession, StealthySessionMixin):# Waiting for the verify spinner to disappear, checking every 1s if it disappearedpage.wait_for_timeout(500)- outer_box = {}+ outer_box: Any = {}iframe = page.frame(url=__CF_PATTERN__)if iframe is not None:self._wait_for_page_stability(iframe, True, False)@@ -156,14 +156,14 @@ class StealthySession(SyncSession, StealthySessionMixin):# Double-checking that the iframe is loadedpage.wait_for_timeout(500)- outer_box: Any = iframe.frame_element().bounding_box()+ outer_box = iframe.frame_element().bounding_box()if not iframe or not outer_box:if "<title>Just a moment...</title>" not in (ResponseFactory._get_page_content(page)):log.info("Cloudflare captcha is solved")return- outer_box: Any = page.locator(box_selector).last.bounding_box()+ outer_box = page.locator(box_selector).last.bounding_box()# Calculate the Captcha coordinates for any viewportcaptcha_x, captcha_y = outer_box["x"] + randint(26, 28), outer_box["y"] + randint(25, 27)@@ -223,6 +223,7 @@ class StealthySession(SyncSession, StealthySessionMixin):)for attempt in range(self._config.retries):+ proxy: Optional[ProxyType] = Noneif self._config.proxy_rotator and static_proxy is None:proxy = self._config.proxy_rotator.get_proxy()else:@@ -335,7 +336,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):self.__validate__(**kwargs)super().__init__(max_pages=self._config.max_pages)- async def start(self):+ async def start(self) -> None:"""Create a browser for this instance and context."""if not self.playwright:self.playwright = await async_playwright().start()@@ -344,16 +345,14 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):self.browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url)if not self._config.proxy_rotator:assert self.browser is not None- self.context: AsyncBrowserContext = await self.browser.new_context(**self._context_options)+ self.context = await self.browser.new_context(**self._context_options)elif self._config.proxy_rotator:self.browser = await self.playwright.chromium.launch(**self._browser_options)else:persistent_options = (self._browser_options | self._context_options | {"user_data_dir": self._user_data_dir})- self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(- **persistent_options- )+ self.context = await self.playwright.chromium.launch_persistent_context(**persistent_options)if self.context:self.context = await self._initialize_context(self._config, self.context)@@ -367,7 +366,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):else:raise RuntimeError("Session has been already started")- async def _initialize_context(self, config, ctx: AsyncBrowserContext) -> AsyncBrowserContext:+ async def _initialize_context(self, config: Any, ctx: AsyncBrowserContext) -> AsyncBrowserContext:"""Initialize the browser context."""for script in _compiled_stealth_scripts():await ctx.add_init_script(script=script)@@ -404,7 +403,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):# Waiting for the verify spinner to disappear, checking every 1s if it disappearedawait page.wait_for_timeout(500)- outer_box = {}+ outer_box: Any = {}iframe = page.frame(url=__CF_PATTERN__)if iframe is not None:await self._wait_for_page_stability(iframe, True, False)@@ -414,14 +413,14 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):# Double-checking that the iframe is loadedawait page.wait_for_timeout(500)- outer_box: Any = await (await iframe.frame_element()).bounding_box()+ outer_box = await (await iframe.frame_element()).bounding_box()if not iframe or not outer_box:if "<title>Just a moment...</title>" not in (await ResponseFactory._get_async_page_content(page)):log.info("Cloudflare captcha is solved")return- outer_box: Any = await page.locator(box_selector).last.bounding_box()+ outer_box = await page.locator(box_selector).last.bounding_box()# Calculate the Captcha coordinates for any viewportcaptcha_x, captcha_y = outer_box["x"] + randint(26, 28), outer_box["y"] + randint(25, 27)@@ -482,6 +481,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):)for attempt in range(self._config.retries):+ proxy: Optional[ProxyType] = Noneif self._config.proxy_rotator and static_proxy is None:proxy = self._config.proxy_rotator.get_proxy()else:diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.pyindex af2baef..e2424f4 100644--- a/scrapling/engines/_browsers/_validators.py+++ b/scrapling/engines/_browsers/_validators.py@@ -157,15 +157,16 @@ def validate_fetch(session: Any,model: type[PlaywrightConfig] | type[StealthConfig],) -> _fetch_params: # pragma: no cover- result = {}- overrides = {}+ result: Dict[str, Any] = {}+ overrides: Dict[str, Any] = {}+ kwargs_dict: Dict[str, Any] = dict(method_kwargs)# Get all field names that _fetch_params needsfetch_param_fields = {f.name for f in fields(_fetch_params)}for key in fetch_param_fields:- if key in method_kwargs:- overrides[key] = method_kwargs[key]+ if key in kwargs_dict:+ overrides[key] = kwargs_dict[key]elif hasattr(session, "_config") and hasattr(session._config, key):result[key] = getattr(session._config, key)diff --git a/scrapling/engines/static.py b/scrapling/engines/static.pyindex 884fe0c..204c105 100644--- a/scrapling/engines/static.py+++ b/scrapling/engines/static.py@@ -753,7 +753,7 @@ class FetcherSession:class FetcherClient(_SyncSessionLogic):__slots__ = ("__enter__", "__exit__")- def __init__(self, **kwargs):+ def __init__(self, **kwargs: Any) -> None:super().__init__(**kwargs)self.__enter__: Any = Noneself.__exit__: Any = None@@ -763,7 +763,7 @@ class FetcherClient(_SyncSessionLogic):class AsyncFetcherClient(_ASyncSessionLogic):__slots__ = ("__aenter__", "__aexit__")- def __init__(self, **kwargs):+ def __init__(self, **kwargs: Any) -> None:super().__init__(**kwargs)self.__aenter__: Any = Noneself.__aexit__: Any = Nonediff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.pyindex 4b4caa2..1a8d799 100644--- a/scrapling/engines/toolbelt/convertor.py+++ b/scrapling/engines/toolbelt/convertor.py@@ -38,7 +38,7 @@ class ResponseFactory:@classmethoddef _process_response_history(cls, first_response: SyncResponse, parser_arguments: Dict) -> list[Response]:"""Process response history to build a list of `Response` objects"""- history = []+ history: list[Response] = []current_request = first_response.request.redirected_fromtry:@@ -101,6 +101,7 @@ class ResponseFactory::param first_response: An earlier or initial Playwright `Response` object that may serve as a fallback response in the absence of the final one.:param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked intothe `Response` object.+ :param meta: Additional meta data to be saved with the response.:return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata.:rtype: Response@@ -145,7 +146,7 @@ class ResponseFactory:cls, first_response: AsyncResponse, parser_arguments: Dict) -> list[Response]:"""Process response history to build a list of `Response` objects"""- history = []+ history: list[Response] = []current_request = first_response.request.redirected_fromtry:@@ -238,6 +239,7 @@ class ResponseFactory::param first_response: An earlier or initial Playwright `Response` object that may serve as a fallback response in the absence of the final one.:param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked intothe `Response` object.+ :param meta: Additional meta data to be saved with the response.:return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata.:rtype: Responsediff --git a/scrapling/parser.py b/scrapling/parser.pyindex 1ae6db1..820bc71 100644--- a/scrapling/parser.py+++ b/scrapling/parser.py@@ -118,8 +118,11 @@ class Selector(SelectorsGeneration):if root is None and content is None:raise ValueError("Selector class needs HTML content, or root arguments to work")- self.__text = None+ self.__text: Optional[TextHandler] = None+ self.__tag: Optional[str] = None+ self.__attributes: Optional[AttributesHandler] = Noneif root is None:+ body: str | bytesif isinstance(content, str):body = content.strip().replace("\x00", "") or "<html/>"elif isinstance(content, bytes):@@ -128,17 +131,18 @@ class Selector(SelectorsGeneration):raise TypeError(f"content argument must be str or bytes, got {type(content)}")# https://lxml.de/api/lxml.etree.HTMLParser-class.html- parser = HTMLParser(+ _parser_kwargs: Dict[str, Any] = dict(recover=True,remove_blank_text=True,remove_comments=(not keep_comments),encoding=encoding,compact=True,huge_tree=huge_tree,- default_doctype=True,+ default_doctype=True, # Supported by lxml but missing from stubsstrip_cdata=(not keep_cdata),)- self._root = cast(HtmlElement, fromstring(body or "<html/>", parser=parser, base_url=url or None))+ parser = HTMLParser(**_parser_kwargs)+ self._root = cast(HtmlElement, fromstring(body or "<html/>", parser=parser, base_url=url or ""))self._raw_body = contentelse:@@ -164,7 +168,7 @@ class Selector(SelectorsGeneration):self._root = cast(HtmlElement, root)self._raw_body = ""- self.__adaptive_enabled = adaptive+ self.__adaptive_enabled = bool(adaptive)if self.__adaptive_enabled:if _storage is not None:@@ -277,8 +281,8 @@ class Selector(SelectorsGeneration):if self._is_text_node(self._root):return "#text"if not self.__tag:- self.__tag = self._root.tag- return self.__tag+ self.__tag = str(self._root.tag)+ return self.__tag or ""@propertydef text(self) -> TextHandler:@@ -313,11 +317,11 @@ class Selector(SelectorsGeneration):if self._is_text_node(self._root):return TextHandler(str(self._root))- ignored_elements = set()+ ignored_elements: set[Any] = set()if ignore_tags:for element in self._root.iter(*ignore_tags):ignored_elements.add(element)- ignored_elements.update(set(_find_all_elements(element)))+ ignored_elements.update(cast(list, _find_all_elements(element)))_all_strings = []for node in self._root.iter():@@ -395,7 +399,7 @@ class Selector(SelectorsGeneration):"""Return all elements under the current element in the DOM tree"""if self._is_text_node(self._root):return Selectors()- below = _find_all_elements(self._root)+ below = cast(List, _find_all_elements(self._root))return self.__elements_convertor(below) if below is not None else Selectors()@property@@ -533,7 +537,7 @@ class Selector(SelectorsGeneration)::param selector_type: If True, the return result will be converted to `Selectors` object:return: List of pure HTML elements that got the highest matching score or 'Selectors' object"""- score_table = {}+ score_table: Dict[float, List[Any]] = {}# Note: `element` will most likely always be a dictionary at this point.if isinstance(element, self.__class__):element = element._root@@ -541,11 +545,11 @@ class Selector(SelectorsGeneration):if issubclass(type(element), HtmlElement):element = _StorageTools.element_to_dict(element)- for node in _find_all_elements(self._root):+ for node in cast(List, _find_all_elements(self._root)):# Collect all elements in the page, then for each element get the matching score of it against the node.# Hence: the code doesn't stop even if the score was 100%# because there might be another element(s) left in page with the same score- score = self.__calculate_similarity_score(element, node)+ score = self.__calculate_similarity_score(cast(Dict, element), node)score_table.setdefault(score, []).append(node)if score_table:@@ -710,7 +714,7 @@ class Selector(SelectorsGeneration):if not args and not kwargs:raise TypeError("You have to pass something to search with, like tag name(s), tag attributes, or both.")- attributes = dict()+ attributes: Dict[str, Any] = dict()tags: Set[str] = set()patterns: Set[Pattern] = set()results, functions, selectors = Selectors(), [], []@@ -809,21 +813,19 @@ class Selector(SelectorsGeneration)::param candidate: The element to compare with the original element.:return: A percentage score of how similar is the candidate to the original element"""- score, checks = 0, 0+ score: float = 0+ checks: int = 0data = _StorageTools.element_to_dict(candidate)- # Possible TODO:- # Study the idea of giving weight to each test below so some are more important than others- # Current results: With weights some websites had better score while it was worse for others- score += 1 if original["tag"] == data["tag"] else 0 # * 0.3 # 30%+ score += 1 if original["tag"] == data["tag"] else 0checks += 1if original["text"]:- score += SequenceMatcher(None, original["text"], data.get("text") or "").ratio() # * 0.3 # 30%+ score += SequenceMatcher(None, original["text"], data.get("text") or "").ratio()checks += 1# if both don't have attributes, it still counts for something!- score += self.__calculate_dict_diff(original["attributes"], data["attributes"]) # * 0.3 # 30%+ score += self.__calculate_dict_diff(original["attributes"], data["attributes"])checks += 1# Separate similarity test for class, id, href,... this will help in full structural changes@@ -838,23 +840,19 @@ class Selector(SelectorsGeneration):None,original["attributes"][attrib],data["attributes"].get(attrib) or "",- ).ratio() # * 0.3 # 30%+ ).ratio()checks += 1- score += SequenceMatcher(None, original["path"], data["path"]).ratio() # * 0.1 # 10%+ score += SequenceMatcher(None, original["path"], data["path"]).ratio()checks += 1if original.get("parent_name"):# Then we start comparing parents' dataif data.get("parent_name"):- score += SequenceMatcher(- None, original["parent_name"], data.get("parent_name") or ""- ).ratio() # * 0.2 # 20%+ score += SequenceMatcher(None, original["parent_name"], data.get("parent_name") or "").ratio()checks += 1- score += self.__calculate_dict_diff(- original["parent_attribs"], data.get("parent_attribs") or {}- ) # * 0.2 # 20%+ score += self.__calculate_dict_diff(original["parent_attribs"], data.get("parent_attribs") or {})checks += 1if original["parent_text"]:@@ -862,14 +860,14 @@ class Selector(SelectorsGeneration):None,original["parent_text"],data.get("parent_text") or "",- ).ratio() # * 0.1 # 10%+ ).ratio()checks += 1# else:# # The original element has a parent and this one not, this is not a good sign# score -= 0.1if original.get("siblings"):- score += SequenceMatcher(None, original["siblings"], data.get("siblings") or []).ratio() # * 0.1 # 10%+ score += SequenceMatcher(None, original["siblings"], data.get("siblings") or []).ratio()checks += 1# How % sure? let's see@@ -890,14 +888,14 @@ class Selector(SelectorsGeneration):the docs for more info."""if self.__adaptive_enabled:- target = element- if isinstance(target, self.__class__):- target: HtmlElement = target._root+ target_element: Any = element+ if isinstance(target_element, self.__class__):+ target_element = target_element._root- if self._is_text_node(target):- target: HtmlElement = target.getparent()+ if self._is_text_node(target_element):+ target_element = target_element.getparent()- self._storage.save(target, identifier)+ self._storage.save(target_element, identifier)else:raise RuntimeError("Can't use `adaptive` features while it's disabled globally, you have to start a new class instance."@@ -987,7 +985,8 @@ class Selector(SelectorsGeneration):candidate_attributes = (self.__get_attributes(candidate, ignore_attributes) if ignore_attributes else candidate.attrib)- score, checks = 0, 0+ score: float = 0+ checks: int = 0if original_attributes:score += sum(@@ -1116,16 +1115,16 @@ class Selector(SelectorsGeneration):if not case_sensitive:text = text.lower()- possible_targets = _find_all_elements_with_spaces(self._root)+ possible_targets = cast(List, _find_all_elements_with_spaces(self._root))if possible_targets:for node in self.__elements_convertor(possible_targets):"""Check if element matches given text otherwise, traverse the children tree and iterate"""- node_text = node.text+ node_text: TextHandler = node.textif clean_match:- node_text = node_text.clean()+ node_text = TextHandler(node_text.clean())if not case_sensitive:- node_text = node_text.lower()+ node_text = TextHandler(node_text.lower())if partial:if text in node_text:@@ -1178,7 +1177,7 @@ class Selector(SelectorsGeneration):results = Selectors()- possible_targets = _find_all_elements_with_spaces(self._root)+ possible_targets = cast(List, _find_all_elements_with_spaces(self._root))if possible_targets:for node in self.__elements_convertor(possible_targets):"""Check if element matches given regex otherwise, traverse the children tree and iterate"""diff --git a/scrapling/spiders/request.py b/scrapling/spiders/request.pyindex 3351100..edfc387 100644--- a/scrapling/spiders/request.py+++ b/scrapling/spiders/request.py@@ -7,7 +7,7 @@ import orjsonfrom w3lib.url import canonicalize_urlfrom scrapling.engines.toolbelt.custom import Response-from scrapling.core._types import Any, AsyncGenerator, Callable, Dict, Union, Tuple, TYPE_CHECKING+from scrapling.core._types import Any, AsyncGenerator, Callable, Dict, Optional, Union, Tuple, TYPE_CHECKINGif TYPE_CHECKING:from scrapling.spiders.spider import Spider@@ -42,7 +42,7 @@ class Request:self.meta: dict[str, Any] = meta if meta else {}self._retry_count: int = _retry_countself._session_kwargs = kwargs if kwargs else {}- self._fp = None+ self._fp: Optional[bytes] = Nonedef copy(self) -> "Request":"""Create a copy of this request."""@@ -89,7 +89,7 @@ class Request:body = b""else:post_data = self._session_kwargs.get("json", {})- body: bytes = orjson.dumps(post_data) if post_data else b""+ body = orjson.dumps(post_data) if post_data else b""data: Dict[str, str | Tuple] = {"sid": self.sid,diff --git a/scrapling/spiders/session.py b/scrapling/spiders/session.pyindex 8f5e7af..cc07042 100644--- a/scrapling/spiders/session.py+++ b/scrapling/spiders/session.py@@ -12,7 +12,7 @@ Session = FetcherSession | AsyncDynamicSession | AsyncStealthySessionclass SessionManager:"""Manages pre-configured session instances."""- def __init__(self):+ def __init__(self) -> None:self._sessions: dict[str, Session] = {}self._default_session_id: str | None = Noneself._started: bool = False@@ -109,17 +109,17 @@ class SessionManager:await session.__aenter__()if isinstance(session, FetcherSession):- session = session._client+ client = session._client- if isinstance(session, _ASyncSessionLogic):- response = await session._make_request(+ if isinstance(client, _ASyncSessionLogic):+ response = await client._make_request(method=cast(SUPPORTED_HTTP_METHODS, request._session_kwargs.pop("method", "GET")),url=request.url,**request._session_kwargs,)else:# Sync session or other types - shouldn't happen in async context- raise TypeError(f"Session type {type(session)} not supported for async fetch")+ raise TypeError(f"Session type {type(client)} not supported for async fetch")else:response = await session.fetch(url=request.url, **request._session_kwargs)