Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

style: Fix all mypy errors and add type hints to untyped function bodies

Karim shoair5ec929435be2
diff --git a/scrapling/core/_types.py b/scrapling/core/_types.py
index f19e706..f2fc097 100644
--- a/scrapling/core/_types.py
+++ b/scrapling/core/_types.py
@@ -32,6 +32,7 @@ from typing import (
Coroutine,
SupportsIndex,
)
+from typing_extensions import Self, Unpack
# Proxy can be a string URL or a dict (Playwright format: {"server": "...", "username": "...", "password": "..."})
ProxyType = Union[str, Dict[str, str]]
@@ -41,27 +42,6 @@ PageLoadStates = Literal["commit", "domcontentloaded", "load", "networkidle"]
extraction_types = Literal["text", "html", "markdown"]
StrOrBytes = Union[str, bytes]
-if TYPE_CHECKING: # pragma: no cover
- from typing_extensions import Unpack
-else: # pragma: no cover
-
- class _Unpack:
- @staticmethod
- def __getitem__(*args, **kwargs):
- pass
-
- Unpack = _Unpack()
-
-
-try:
- # Python 3.11+
- from typing import Self # novermin
-except ImportError: # pragma: no cover
- try:
- from typing_extensions import Self # Backport
- except ImportError:
- Self = object
-
# Copied from `playwright._impl._api_structures.SetCookieParam`
class SetCookieParam(TypedDict, total=False):
diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.py
index 45ab4d5..d06ac29 100644
--- a/scrapling/core/custom_types.py
+++ b/scrapling/core/custom_types.py
@@ -35,9 +35,7 @@ class TextHandler(str):
lst = super().__getitem__(key)
return TextHandler(lst)
- def split(
- self, sep: str | None = None, maxsplit: SupportsIndex = -1
- ) -> Union[List, "TextHandlers"]: # pragma: no cover
+ def split(self, sep: str | None = None, maxsplit: SupportsIndex = -1) -> list[Any]: # pragma: no cover
return TextHandlers([TextHandler(s) for s in super().split(sep, maxsplit)])
def strip(self, chars: str | None = None) -> Union[str, "TextHandler"]: # pragma: no cover
@@ -61,7 +59,7 @@ class TextHandler(str):
def expandtabs(self, tabsize: SupportsIndex = 8) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().expandtabs(tabsize))
- def format(self, *args: object, **kwargs: str) -> Union[str, "TextHandler"]: # pragma: no cover
+ def format(self, *args: object, **kwargs: object) -> Union[str, "TextHandler"]: # pragma: no cover
return TextHandler(super().format(*args, **kwargs))
def format_map(self, mapping) -> Union[str, "TextHandler"]: # pragma: no cover
@@ -291,7 +289,7 @@ class AttributesHandler(Mapping[str, _TextHandlerType]):
__slots__ = ("_data",)
- def __init__(self, mapping=None, **kwargs):
+ def __init__(self, mapping: Any = None, **kwargs: Any) -> None:
mapping = (
{key: TextHandler(value) if isinstance(value, str) else value for key, value in mapping.items()}
if mapping is not None
@@ -324,8 +322,8 @@ class AttributesHandler(Mapping[str, _TextHandlerType]):
yield AttributesHandler({key: value})
@property
- def json_string(self):
- """Convert current attributes to JSON string if the attributes are JSON serializable otherwise throws error"""
+ def json_string(self) -> bytes:
+ """Convert current attributes to JSON bytes if the attributes are JSON serializable otherwise throws error"""
return dumps(dict(self._data))
def __getitem__(self, key: str) -> _TextHandlerType:
diff --git a/scrapling/core/mixins.py b/scrapling/core/mixins.py
index 0b66c07..c2e7420 100644
--- a/scrapling/core/mixins.py
+++ b/scrapling/core/mixins.py
@@ -1,7 +1,4 @@
-from scrapling.core._types import TYPE_CHECKING
-
-if TYPE_CHECKING:
- from scrapling.parser import Selector
+from scrapling.core._types import Any, Dict
class SelectorsGeneration:
@@ -11,7 +8,11 @@ class SelectorsGeneration:
Inspiration: https://searchfox.org/mozilla-central/source/devtools/shared/inspector/css-logic.js#591
"""
- def _general_selection(self: "Selector", selection: str = "css", full_path: bool = False) -> str: # type: ignore[name-defined]
+ # Note: This is a mixin class meant to be used with Selector.
+ # The methods access Selector attributes (._root, .parent, .attrib, .tag, etc.)
+ # through self, which will be a Selector instance at runtime.
+
+ def _general_selection(self: Any, selection: str = "css", full_path: bool = False) -> str:
"""Generate a selector for the current element.
:return: A string of the generated selector.
"""
@@ -36,7 +37,7 @@ class SelectorsGeneration:
# if classes and css:
# part += f".{'.'.join(classes)}"
# else:
- counter = {}
+ counter: Dict[str, int] = {}
for child in target.parent.children:
counter.setdefault(child.tag, 0)
counter[child.tag] += 1
@@ -56,28 +57,28 @@ class SelectorsGeneration:
return " > ".join(reversed(selectorPath)) if css else "//" + "/".join(reversed(selectorPath))
@property
- def generate_css_selector(self: "Selector") -> str: # type: ignore[name-defined]
+ def generate_css_selector(self: Any) -> str:
"""Generate a CSS selector for the current element
:return: A string of the generated selector.
"""
return self._general_selection()
@property
- def generate_full_css_selector(self: "Selector") -> str: # type: ignore[name-defined]
+ def generate_full_css_selector(self: Any) -> str:
"""Generate a complete CSS selector for the current element
:return: A string of the generated selector.
"""
return self._general_selection(full_path=True)
@property
- def generate_xpath_selector(self: "Selector") -> str: # type: ignore[name-defined]
+ def generate_xpath_selector(self: Any) -> str:
"""Generate an XPath selector for the current element
:return: A string of the generated selector.
"""
return self._general_selection("xpath")
@property
- def generate_full_xpath_selector(self: "Selector") -> str: # type: ignore[name-defined]
+ def generate_full_xpath_selector(self: Any) -> str:
"""Generate a complete XPath selector for the current element
:return: A string of the generated selector.
"""
diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py
index 3a1511e..0bd6efc 100644
--- a/scrapling/core/shell.py
+++ b/scrapling/core/shell.py
@@ -30,6 +30,7 @@ from scrapling.core.custom_types import TextHandler
from scrapling.engines.toolbelt.custom import Response
from scrapling.core.utils._shell import _ParseHeaders, _CookieParser
from scrapling.core._types import (
+ Callable,
Dict,
Any,
cast,
@@ -82,7 +83,7 @@ class NoExitArgumentParser(ArgumentParser): # pragma: no cover
class CurlParser:
"""Builds the argument parser for relevant curl flags from DevTools."""
- def __init__(self):
+ def __init__(self) -> None:
from scrapling.fetchers import Fetcher as __Fetcher
self.__fetcher = __Fetcher
@@ -467,19 +468,21 @@ Type 'exit' or press Ctrl+D to exit.
return result
- def create_wrapper(self, func, get_signature=True, signature_name=None):
+ def create_wrapper(
+ self, func: Callable, get_signature: bool = True, signature_name: Optional[str] = None
+ ) -> Callable:
"""Create a wrapper that preserves function signature but updates page"""
@wraps(func)
- def wrapper(*args, **kwargs):
+ def wrapper(*args: Any, **kwargs: Any) -> Any:
result = func(*args, **kwargs)
return self.update_page(result)
if get_signature:
# Explicitly preserve and unpack signature for IPython introspection and autocompletion
- wrapper.__signature__ = _unpack_signature(func, signature_name) # pyright: ignore
+ setattr(wrapper, "__signature__", _unpack_signature(func, signature_name))
else:
- wrapper.__signature__ = signature(func) # pyright: ignore
+ setattr(wrapper, "__signature__", signature(func))
return wrapper
@@ -601,7 +604,7 @@ class Convertor:
" ",
):
# Remove consecutive white-spaces
- txt_content = re_sub(f"[{s}]+", s, txt_content)
+ txt_content = TextHandler(re_sub(f"[{s}]+", s, txt_content))
yield txt_content
yield ""
diff --git a/scrapling/core/storage.py b/scrapling/core/storage.py
index 8e04599..d93a320 100644
--- a/scrapling/core/storage.py
+++ b/scrapling/core/storage.py
@@ -63,12 +63,11 @@ class StorageSystemMixin(ABC): # pragma: no cover
def _get_hash(identifier: str) -> str:
"""If you want to hash identifier in your storage system, use this safer"""
_identifier = identifier.lower().strip()
- if isinstance(_identifier, str):
- # Hash functions have to take bytes
- _identifier = _identifier.encode("utf-8")
+ # Hash functions have to take bytes
+ _identifier_bytes = _identifier.encode("utf-8")
- hash_value = sha256(_identifier).hexdigest()
- return f"{hash_value}_{len(_identifier)}" # Length to reduce collision chance
+ hash_value = sha256(_identifier_bytes).hexdigest()
+ return f"{hash_value}_{len(_identifier_bytes)}" # Length to reduce collision chance
@lru_cache(1, typed=True)
diff --git a/scrapling/engines/_browsers/_base.py b/scrapling/engines/_browsers/_base.py
index 8bd442d..e379882 100644
--- a/scrapling/engines/_browsers/_base.py
+++ b/scrapling/engines/_browsers/_base.py
@@ -5,16 +5,12 @@ from contextlib import contextmanager, asynccontextmanager
from playwright.sync_api._generated import Page
from playwright.sync_api import (
Frame,
- Browser,
BrowserContext,
- Playwright,
Response as SyncPlaywrightResponse,
)
from playwright.async_api._generated import Page as AsyncPage
from playwright.async_api import (
Frame as AsyncFrame,
- Browser as AsyncBrowser,
- Playwright as AsyncPlaywright,
Response as AsyncPlaywrightResponse,
BrowserContext as AsyncBrowserContext,
)
@@ -37,6 +33,7 @@ from scrapling.core._types import (
Optional,
Callable,
TYPE_CHECKING,
+ cast,
overload,
Tuple,
ProxyType,
@@ -61,12 +58,12 @@ class SyncSession:
self.max_pages = max_pages
self.page_pool = PagePool(max_pages)
self._max_wait_for_page = 60
- self.playwright: Playwright | Any = None
- self.context: BrowserContext | Any = None
- self.browser: Optional[Browser] = None
+ self.playwright: Any = None
+ self.context: Any = None
+ self.browser: Any = None
self._is_alive = False
- def start(self):
+ def start(self) -> None:
pass
def close(self): # pragma: no cover
@@ -215,13 +212,13 @@ class AsyncSession:
self.max_pages = max_pages
self.page_pool = PagePool(max_pages)
self._max_wait_for_page = 60
- self.playwright: AsyncPlaywright | Any = None
- self.context: AsyncBrowserContext | Any = None
- self.browser: Optional[AsyncBrowser] = None
+ self.playwright: Any = None
+ self.context: Any = None
+ self.browser: Any = None
self._is_alive = False
self._lock = Lock()
- async def start(self):
+ async def start(self) -> None:
pass
async def close(self):
@@ -378,6 +375,8 @@ class AsyncSession:
class BaseSessionMixin:
+ _config: "PlaywrightConfig | StealthConfig"
+
@overload
def __validate_routine__(self, params: Dict, model: type[StealthConfig]) -> StealthConfig: ...
@@ -404,7 +403,7 @@ class BaseSessionMixin:
return config
def __generate_options__(self, extra_flags: Tuple | None = None) -> None:
- config: PlaywrightConfig | StealthConfig = self._config # type: ignore[has-type]
+ config: PlaywrightConfig | StealthConfig = self._config
self._context_options.update(
{
"proxy": config.proxy,
@@ -466,7 +465,7 @@ class DynamicSessionMixin(BaseSessionMixin):
class StealthySessionMixin(BaseSessionMixin):
def __validate__(self, **params):
- self._config: StealthConfig = self.__validate_routine__(params, model=StealthConfig)
+ self._config = self.__validate_routine__(params, model=StealthConfig)
self._context_options.update(
{
"is_mobile": False,
@@ -482,22 +481,23 @@ class StealthySessionMixin(BaseSessionMixin):
self.__generate_stealth_options()
def __generate_stealth_options(self) -> None:
- flags = tuple()
- if not self._config.cdp_url:
+ config = cast(StealthConfig, self._config)
+ flags: Tuple[str, ...] = tuple()
+ if not config.cdp_url:
flags = DEFAULT_FLAGS + DEFAULT_STEALTH_FLAGS
- if self._config.block_webrtc:
+ if config.block_webrtc:
flags += (
"--webrtc-ip-handling-policy=disable_non_proxied_udp",
"--force-webrtc-ip-handling-policy", # Ensures the policy is enforced
)
- if not self._config.allow_webgl:
+ if not config.allow_webgl:
flags += (
"--disable-webgl",
"--disable-webgl-image-chromium",
"--disable-webgl2",
)
- if self._config.hide_canvas:
+ if config.hide_canvas:
flags += ("--fingerprinting-canvas-image-data-noise",)
super(StealthySessionMixin, self).__generate_options__(flags)
diff --git a/scrapling/engines/_browsers/_controllers.py b/scrapling/engines/_browsers/_controllers.py
index 5f0b247..5d9e801 100644
--- a/scrapling/engines/_browsers/_controllers.py
+++ b/scrapling/engines/_browsers/_controllers.py
@@ -8,11 +8,10 @@ from playwright.sync_api import (
from playwright.async_api import (
async_playwright,
Locator as AsyncLocator,
- BrowserContext as AsyncBrowserContext,
)
from scrapling.core.utils import log
-from scrapling.core._types import Unpack
+from scrapling.core._types import Optional, ProxyType, Unpack
from scrapling.engines.toolbelt.proxy_rotation import is_proxy_error
from scrapling.engines.toolbelt.convertor import Response, ResponseFactory
from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer
@@ -134,6 +133,7 @@ class DynamicSession(SyncSession, DynamicSessionMixin):
)
for attempt in range(self._config.retries):
+ proxy: Optional[ProxyType] = None
if self._config.proxy_rotator and static_proxy is None:
proxy = self._config.proxy_rotator.get_proxy()
else:
@@ -238,7 +238,7 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
self.__validate__(**kwargs)
super().__init__(max_pages=self._config.max_pages)
- async def start(self):
+ async def start(self) -> None:
"""Create a browser for this instance and context."""
if not self.playwright:
self.playwright = await async_playwright().start()
@@ -246,16 +246,14 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
if self._config.cdp_url:
self.browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url)
if not self._config.proxy_rotator and self.browser:
- self.context: AsyncBrowserContext = await self.browser.new_context(**self._context_options)
+ self.context = await self.browser.new_context(**self._context_options)
elif self._config.proxy_rotator:
self.browser = await self.playwright.chromium.launch(**self._browser_options)
else:
persistent_options = (
self._browser_options | self._context_options | {"user_data_dir": self._user_data_dir}
)
- self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(
- **persistent_options
- )
+ self.context = await self.playwright.chromium.launch_persistent_context(**persistent_options)
if self.context:
self.context = await self._initialize_context(self._config, self.context)
@@ -304,6 +302,7 @@ class AsyncDynamicSession(AsyncSession, DynamicSessionMixin):
)
for attempt in range(self._config.retries):
+ proxy: Optional[ProxyType] = None
if self._config.proxy_rotator and static_proxy is None:
proxy = self._config.proxy_rotator.get_proxy()
else:
diff --git a/scrapling/engines/_browsers/_page.py b/scrapling/engines/_browsers/_page.py
index 655d3d1..481016e 100644
--- a/scrapling/engines/_browsers/_page.py
+++ b/scrapling/engines/_browsers/_page.py
@@ -61,7 +61,9 @@ class PagePool:
raise RuntimeError(f"Maximum page limit ({self.max_pages}) reached")
if isinstance(page, AsyncPage):
- page_info = cast(PageInfo[AsyncPage], PageInfo(page, "ready", ""))
+ page_info: PageInfo[SyncPage] | PageInfo[AsyncPage] = cast(
+ PageInfo[AsyncPage], PageInfo(page, "ready", "")
+ )
else:
page_info = cast(PageInfo[SyncPage], PageInfo(page, "ready", ""))
diff --git a/scrapling/engines/_browsers/_stealth.py b/scrapling/engines/_browsers/_stealth.py
index 730cd7e..3c9ea58 100644
--- a/scrapling/engines/_browsers/_stealth.py
+++ b/scrapling/engines/_browsers/_stealth.py
@@ -13,7 +13,7 @@ from patchright.sync_api import sync_playwright
from patchright.async_api import async_playwright
from scrapling.core.utils import log
-from scrapling.core._types import Any, Unpack
+from scrapling.core._types import Any, Optional, ProxyType, Unpack
from scrapling.engines.toolbelt.proxy_rotation import is_proxy_error
from scrapling.engines.toolbelt.convertor import Response, ResponseFactory
from scrapling.engines.toolbelt.fingerprints import generate_convincing_referer
@@ -78,7 +78,7 @@ class StealthySession(SyncSession, StealthySessionMixin):
self.__validate__(**kwargs)
super().__init__()
- def start(self):
+ def start(self) -> None:
"""Create a browser for this instance and context."""
if not self.playwright:
self.playwright = sync_playwright().start()
@@ -146,7 +146,7 @@ class StealthySession(SyncSession, StealthySessionMixin):
# Waiting for the verify spinner to disappear, checking every 1s if it disappeared
page.wait_for_timeout(500)
- outer_box = {}
+ outer_box: Any = {}
iframe = page.frame(url=__CF_PATTERN__)
if iframe is not None:
self._wait_for_page_stability(iframe, True, False)
@@ -156,14 +156,14 @@ class StealthySession(SyncSession, StealthySessionMixin):
# Double-checking that the iframe is loaded
page.wait_for_timeout(500)
- outer_box: Any = iframe.frame_element().bounding_box()
+ outer_box = iframe.frame_element().bounding_box()
if not iframe or not outer_box:
if "<title>Just a moment...</title>" not in (ResponseFactory._get_page_content(page)):
log.info("Cloudflare captcha is solved")
return
- outer_box: Any = page.locator(box_selector).last.bounding_box()
+ outer_box = page.locator(box_selector).last.bounding_box()
# Calculate the Captcha coordinates for any viewport
captcha_x, captcha_y = outer_box["x"] + randint(26, 28), outer_box["y"] + randint(25, 27)
@@ -223,6 +223,7 @@ class StealthySession(SyncSession, StealthySessionMixin):
)
for attempt in range(self._config.retries):
+ proxy: Optional[ProxyType] = None
if self._config.proxy_rotator and static_proxy is None:
proxy = self._config.proxy_rotator.get_proxy()
else:
@@ -335,7 +336,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
self.__validate__(**kwargs)
super().__init__(max_pages=self._config.max_pages)
- async def start(self):
+ async def start(self) -> None:
"""Create a browser for this instance and context."""
if not self.playwright:
self.playwright = await async_playwright().start()
@@ -344,16 +345,14 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
self.browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url)
if not self._config.proxy_rotator:
assert self.browser is not None
- self.context: AsyncBrowserContext = await self.browser.new_context(**self._context_options)
+ self.context = await self.browser.new_context(**self._context_options)
elif self._config.proxy_rotator:
self.browser = await self.playwright.chromium.launch(**self._browser_options)
else:
persistent_options = (
self._browser_options | self._context_options | {"user_data_dir": self._user_data_dir}
)
- self.context: AsyncBrowserContext = await self.playwright.chromium.launch_persistent_context(
- **persistent_options
- )
+ self.context = await self.playwright.chromium.launch_persistent_context(**persistent_options)
if self.context:
self.context = await self._initialize_context(self._config, self.context)
@@ -367,7 +366,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
else:
raise RuntimeError("Session has been already started")
- async def _initialize_context(self, config, ctx: AsyncBrowserContext) -> AsyncBrowserContext:
+ async def _initialize_context(self, config: Any, ctx: AsyncBrowserContext) -> AsyncBrowserContext:
"""Initialize the browser context."""
for script in _compiled_stealth_scripts():
await ctx.add_init_script(script=script)
@@ -404,7 +403,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
# Waiting for the verify spinner to disappear, checking every 1s if it disappeared
await page.wait_for_timeout(500)
- outer_box = {}
+ outer_box: Any = {}
iframe = page.frame(url=__CF_PATTERN__)
if iframe is not None:
await self._wait_for_page_stability(iframe, True, False)
@@ -414,14 +413,14 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
# Double-checking that the iframe is loaded
await page.wait_for_timeout(500)
- outer_box: Any = await (await iframe.frame_element()).bounding_box()
+ outer_box = await (await iframe.frame_element()).bounding_box()
if not iframe or not outer_box:
if "<title>Just a moment...</title>" not in (await ResponseFactory._get_async_page_content(page)):
log.info("Cloudflare captcha is solved")
return
- outer_box: Any = await page.locator(box_selector).last.bounding_box()
+ outer_box = await page.locator(box_selector).last.bounding_box()
# Calculate the Captcha coordinates for any viewport
captcha_x, captcha_y = outer_box["x"] + randint(26, 28), outer_box["y"] + randint(25, 27)
@@ -482,6 +481,7 @@ class AsyncStealthySession(AsyncSession, StealthySessionMixin):
)
for attempt in range(self._config.retries):
+ proxy: Optional[ProxyType] = None
if self._config.proxy_rotator and static_proxy is None:
proxy = self._config.proxy_rotator.get_proxy()
else:
diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py
index af2baef..e2424f4 100644
--- a/scrapling/engines/_browsers/_validators.py
+++ b/scrapling/engines/_browsers/_validators.py
@@ -157,15 +157,16 @@ def validate_fetch(
session: Any,
model: type[PlaywrightConfig] | type[StealthConfig],
) -> _fetch_params: # pragma: no cover
- result = {}
- overrides = {}
+ result: Dict[str, Any] = {}
+ overrides: Dict[str, Any] = {}
+ kwargs_dict: Dict[str, Any] = dict(method_kwargs)
# Get all field names that _fetch_params needs
fetch_param_fields = {f.name for f in fields(_fetch_params)}
for key in fetch_param_fields:
- if key in method_kwargs:
- overrides[key] = method_kwargs[key]
+ if key in kwargs_dict:
+ overrides[key] = kwargs_dict[key]
elif hasattr(session, "_config") and hasattr(session._config, key):
result[key] = getattr(session._config, key)
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py
index 884fe0c..204c105 100644
--- a/scrapling/engines/static.py
+++ b/scrapling/engines/static.py
@@ -753,7 +753,7 @@ class FetcherSession:
class FetcherClient(_SyncSessionLogic):
__slots__ = ("__enter__", "__exit__")
- def __init__(self, **kwargs):
+ def __init__(self, **kwargs: Any) -> None:
super().__init__(**kwargs)
self.__enter__: Any = None
self.__exit__: Any = None
@@ -763,7 +763,7 @@ class FetcherClient(_SyncSessionLogic):
class AsyncFetcherClient(_ASyncSessionLogic):
__slots__ = ("__aenter__", "__aexit__")
- def __init__(self, **kwargs):
+ def __init__(self, **kwargs: Any) -> None:
super().__init__(**kwargs)
self.__aenter__: Any = None
self.__aexit__: Any = None
diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.py
index 4b4caa2..1a8d799 100644
--- a/scrapling/engines/toolbelt/convertor.py
+++ b/scrapling/engines/toolbelt/convertor.py
@@ -38,7 +38,7 @@ class ResponseFactory:
@classmethod
def _process_response_history(cls, first_response: SyncResponse, parser_arguments: Dict) -> list[Response]:
"""Process response history to build a list of `Response` objects"""
- history = []
+ history: list[Response] = []
current_request = first_response.request.redirected_from
try:
@@ -101,6 +101,7 @@ class ResponseFactory:
:param first_response: An earlier or initial Playwright `Response` object that may serve as a fallback response in the absence of the final one.
:param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into
the `Response` object.
+ :param meta: Additional meta data to be saved with the response.
:return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata.
:rtype: Response
@@ -145,7 +146,7 @@ class ResponseFactory:
cls, first_response: AsyncResponse, parser_arguments: Dict
) -> list[Response]:
"""Process response history to build a list of `Response` objects"""
- history = []
+ history: list[Response] = []
current_request = first_response.request.redirected_from
try:
@@ -238,6 +239,7 @@ class ResponseFactory:
:param first_response: An earlier or initial Playwright `Response` object that may serve as a fallback response in the absence of the final one.
:param parser_arguments: A dictionary containing additional arguments needed for parsing or further customization of the returned `Response`. These arguments are dynamically unpacked into
the `Response` object.
+ :param meta: Additional meta data to be saved with the response.
:return: A fully populated `Response` object containing the page's URL, content, status, headers, cookies, and other derived metadata.
:rtype: Response
diff --git a/scrapling/parser.py b/scrapling/parser.py
index 1ae6db1..820bc71 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -118,8 +118,11 @@ class Selector(SelectorsGeneration):
if root is None and content is None:
raise ValueError("Selector class needs HTML content, or root arguments to work")
- self.__text = None
+ self.__text: Optional[TextHandler] = None
+ self.__tag: Optional[str] = None
+ self.__attributes: Optional[AttributesHandler] = None
if root is None:
+ body: str | bytes
if isinstance(content, str):
body = content.strip().replace("\x00", "") or "<html/>"
elif isinstance(content, bytes):
@@ -128,17 +131,18 @@ class Selector(SelectorsGeneration):
raise TypeError(f"content argument must be str or bytes, got {type(content)}")
# https://lxml.de/api/lxml.etree.HTMLParser-class.html
- parser = HTMLParser(
+ _parser_kwargs: Dict[str, Any] = dict(
recover=True,
remove_blank_text=True,
remove_comments=(not keep_comments),
encoding=encoding,
compact=True,
huge_tree=huge_tree,
- default_doctype=True,
+ default_doctype=True, # Supported by lxml but missing from stubs
strip_cdata=(not keep_cdata),
)
- self._root = cast(HtmlElement, fromstring(body or "<html/>", parser=parser, base_url=url or None))
+ parser = HTMLParser(**_parser_kwargs)
+ self._root = cast(HtmlElement, fromstring(body or "<html/>", parser=parser, base_url=url or ""))
self._raw_body = content
else:
@@ -164,7 +168,7 @@ class Selector(SelectorsGeneration):
self._root = cast(HtmlElement, root)
self._raw_body = ""
- self.__adaptive_enabled = adaptive
+ self.__adaptive_enabled = bool(adaptive)
if self.__adaptive_enabled:
if _storage is not None:
@@ -277,8 +281,8 @@ class Selector(SelectorsGeneration):
if self._is_text_node(self._root):
return "#text"
if not self.__tag:
- self.__tag = self._root.tag
- return self.__tag
+ self.__tag = str(self._root.tag)
+ return self.__tag or ""
@property
def text(self) -> TextHandler:
@@ -313,11 +317,11 @@ class Selector(SelectorsGeneration):
if self._is_text_node(self._root):
return TextHandler(str(self._root))
- ignored_elements = set()
+ ignored_elements: set[Any] = set()
if ignore_tags:
for element in self._root.iter(*ignore_tags):
ignored_elements.add(element)
- ignored_elements.update(set(_find_all_elements(element)))
+ ignored_elements.update(cast(list, _find_all_elements(element)))
_all_strings = []
for node in self._root.iter():
@@ -395,7 +399,7 @@ class Selector(SelectorsGeneration):
"""Return all elements under the current element in the DOM tree"""
if self._is_text_node(self._root):
return Selectors()
- below = _find_all_elements(self._root)
+ below = cast(List, _find_all_elements(self._root))
return self.__elements_convertor(below) if below is not None else Selectors()
@property
@@ -533,7 +537,7 @@ class Selector(SelectorsGeneration):
:param selector_type: If True, the return result will be converted to `Selectors` object
:return: List of pure HTML elements that got the highest matching score or 'Selectors' object
"""
- score_table = {}
+ score_table: Dict[float, List[Any]] = {}
# Note: `element` will most likely always be a dictionary at this point.
if isinstance(element, self.__class__):
element = element._root
@@ -541,11 +545,11 @@ class Selector(SelectorsGeneration):
if issubclass(type(element), HtmlElement):
element = _StorageTools.element_to_dict(element)
- for node in _find_all_elements(self._root):
+ for node in cast(List, _find_all_elements(self._root)):
# Collect all elements in the page, then for each element get the matching score of it against the node.
# Hence: the code doesn't stop even if the score was 100%
# because there might be another element(s) left in page with the same score
- score = self.__calculate_similarity_score(element, node)
+ score = self.__calculate_similarity_score(cast(Dict, element), node)
score_table.setdefault(score, []).append(node)
if score_table:
@@ -710,7 +714,7 @@ class Selector(SelectorsGeneration):
if not args and not kwargs:
raise TypeError("You have to pass something to search with, like tag name(s), tag attributes, or both.")
- attributes = dict()
+ attributes: Dict[str, Any] = dict()
tags: Set[str] = set()
patterns: Set[Pattern] = set()
results, functions, selectors = Selectors(), [], []
@@ -809,21 +813,19 @@ class Selector(SelectorsGeneration):
:param candidate: The element to compare with the original element.
:return: A percentage score of how similar is the candidate to the original element
"""
- score, checks = 0, 0
+ score: float = 0
+ checks: int = 0
data = _StorageTools.element_to_dict(candidate)
- # Possible TODO:
- # Study the idea of giving weight to each test below so some are more important than others
- # Current results: With weights some websites had better score while it was worse for others
- score += 1 if original["tag"] == data["tag"] else 0 # * 0.3 # 30%
+ score += 1 if original["tag"] == data["tag"] else 0
checks += 1
if original["text"]:
- score += SequenceMatcher(None, original["text"], data.get("text") or "").ratio() # * 0.3 # 30%
+ score += SequenceMatcher(None, original["text"], data.get("text") or "").ratio()
checks += 1
# if both don't have attributes, it still counts for something!
- score += self.__calculate_dict_diff(original["attributes"], data["attributes"]) # * 0.3 # 30%
+ score += self.__calculate_dict_diff(original["attributes"], data["attributes"])
checks += 1
# Separate similarity test for class, id, href,... this will help in full structural changes
@@ -838,23 +840,19 @@ class Selector(SelectorsGeneration):
None,
original["attributes"][attrib],
data["attributes"].get(attrib) or "",
- ).ratio() # * 0.3 # 30%
+ ).ratio()
checks += 1
- score += SequenceMatcher(None, original["path"], data["path"]).ratio() # * 0.1 # 10%
+ score += SequenceMatcher(None, original["path"], data["path"]).ratio()
checks += 1
if original.get("parent_name"):
# Then we start comparing parents' data
if data.get("parent_name"):
- score += SequenceMatcher(
- None, original["parent_name"], data.get("parent_name") or ""
- ).ratio() # * 0.2 # 20%
+ score += SequenceMatcher(None, original["parent_name"], data.get("parent_name") or "").ratio()
checks += 1
- score += self.__calculate_dict_diff(
- original["parent_attribs"], data.get("parent_attribs") or {}
- ) # * 0.2 # 20%
+ score += self.__calculate_dict_diff(original["parent_attribs"], data.get("parent_attribs") or {})
checks += 1
if original["parent_text"]:
@@ -862,14 +860,14 @@ class Selector(SelectorsGeneration):
None,
original["parent_text"],
data.get("parent_text") or "",
- ).ratio() # * 0.1 # 10%
+ ).ratio()
checks += 1
# else:
# # The original element has a parent and this one not, this is not a good sign
# score -= 0.1
if original.get("siblings"):
- score += SequenceMatcher(None, original["siblings"], data.get("siblings") or []).ratio() # * 0.1 # 10%
+ score += SequenceMatcher(None, original["siblings"], data.get("siblings") or []).ratio()
checks += 1
# How % sure? let's see
@@ -890,14 +888,14 @@ class Selector(SelectorsGeneration):
the docs for more info.
"""
if self.__adaptive_enabled:
- target = element
- if isinstance(target, self.__class__):
- target: HtmlElement = target._root
+ target_element: Any = element
+ if isinstance(target_element, self.__class__):
+ target_element = target_element._root
- if self._is_text_node(target):
- target: HtmlElement = target.getparent()
+ if self._is_text_node(target_element):
+ target_element = target_element.getparent()
- self._storage.save(target, identifier)
+ self._storage.save(target_element, identifier)
else:
raise RuntimeError(
"Can't use `adaptive` features while it's disabled globally, you have to start a new class instance."
@@ -987,7 +985,8 @@ class Selector(SelectorsGeneration):
candidate_attributes = (
self.__get_attributes(candidate, ignore_attributes) if ignore_attributes else candidate.attrib
)
- score, checks = 0, 0
+ score: float = 0
+ checks: int = 0
if original_attributes:
score += sum(
@@ -1116,16 +1115,16 @@ class Selector(SelectorsGeneration):
if not case_sensitive:
text = text.lower()
- possible_targets = _find_all_elements_with_spaces(self._root)
+ possible_targets = cast(List, _find_all_elements_with_spaces(self._root))
if possible_targets:
for node in self.__elements_convertor(possible_targets):
"""Check if element matches given text otherwise, traverse the children tree and iterate"""
- node_text = node.text
+ node_text: TextHandler = node.text
if clean_match:
- node_text = node_text.clean()
+ node_text = TextHandler(node_text.clean())
if not case_sensitive:
- node_text = node_text.lower()
+ node_text = TextHandler(node_text.lower())
if partial:
if text in node_text:
@@ -1178,7 +1177,7 @@ class Selector(SelectorsGeneration):
results = Selectors()
- possible_targets = _find_all_elements_with_spaces(self._root)
+ possible_targets = cast(List, _find_all_elements_with_spaces(self._root))
if possible_targets:
for node in self.__elements_convertor(possible_targets):
"""Check if element matches given regex otherwise, traverse the children tree and iterate"""
diff --git a/scrapling/spiders/request.py b/scrapling/spiders/request.py
index 3351100..edfc387 100644
--- a/scrapling/spiders/request.py
+++ b/scrapling/spiders/request.py
@@ -7,7 +7,7 @@ import orjson
from w3lib.url import canonicalize_url
from scrapling.engines.toolbelt.custom import Response
-from scrapling.core._types import Any, AsyncGenerator, Callable, Dict, Union, Tuple, TYPE_CHECKING
+from scrapling.core._types import Any, AsyncGenerator, Callable, Dict, Optional, Union, Tuple, TYPE_CHECKING
if TYPE_CHECKING:
from scrapling.spiders.spider import Spider
@@ -42,7 +42,7 @@ class Request:
self.meta: dict[str, Any] = meta if meta else {}
self._retry_count: int = _retry_count
self._session_kwargs = kwargs if kwargs else {}
- self._fp = None
+ self._fp: Optional[bytes] = None
def copy(self) -> "Request":
"""Create a copy of this request."""
@@ -89,7 +89,7 @@ class Request:
body = b""
else:
post_data = self._session_kwargs.get("json", {})
- body: bytes = orjson.dumps(post_data) if post_data else b""
+ body = orjson.dumps(post_data) if post_data else b""
data: Dict[str, str | Tuple] = {
"sid": self.sid,
diff --git a/scrapling/spiders/session.py b/scrapling/spiders/session.py
index 8f5e7af..cc07042 100644
--- a/scrapling/spiders/session.py
+++ b/scrapling/spiders/session.py
@@ -12,7 +12,7 @@ Session = FetcherSession | AsyncDynamicSession | AsyncStealthySession
class SessionManager:
"""Manages pre-configured session instances."""
- def __init__(self):
+ def __init__(self) -> None:
self._sessions: dict[str, Session] = {}
self._default_session_id: str | None = None
self._started: bool = False
@@ -109,17 +109,17 @@ class SessionManager:
await session.__aenter__()
if isinstance(session, FetcherSession):
- session = session._client
+ client = session._client
- if isinstance(session, _ASyncSessionLogic):
- response = await session._make_request(
+ if isinstance(client, _ASyncSessionLogic):
+ response = await client._make_request(
method=cast(SUPPORTED_HTTP_METHODS, request._session_kwargs.pop("method", "GET")),
url=request.url,
**request._session_kwargs,
)
else:
# Sync session or other types - shouldn't happen in async context
- raise TypeError(f"Session type {type(session)} not supported for async fetch")
+ raise TypeError(f"Session type {type(client)} not supported for async fetch")
else:
response = await session.fetch(url=request.url, **request._session_kwargs)