Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

style: type hints corrections and docstrings

Karim shoair18660f8132f1
diff --git a/README.md b/README.md
index 3838b1f..8d1c778 100644
--- a/README.md
+++ b/README.md
@@ -150,7 +150,7 @@ Tired of your PC slowing you down? Can’t keep your machine on 24/7 for scrapin
```python
from scrapling.fetchers import Fetcher
-# Do HTTP GET request to a web page and create an Selector instance
+# Do HTTP GET request to a web page and create a Selector instance
page = Fetcher.get('https://quotes.toscrape.com/', stealthy_headers=True)
# Get all text content from all HTML tags in the page except the `script` and `style` tags
page.get_all_text(ignore_tags=('script', 'style'))
diff --git a/scrapling/cli.py b/scrapling/cli.py
index e5e91d5..940c27d 100644
--- a/scrapling/cli.py
+++ b/scrapling/cli.py
@@ -3,6 +3,7 @@ from subprocess import check_output
from sys import executable as python_executable
from scrapling.core.utils import log
+from scrapling.engines.toolbelt import Response
from scrapling.core._types import List, Optional, Dict, Tuple, Any, Callable
from scrapling.fetchers import Fetcher, DynamicFetcher, StealthyFetcher
from scrapling.core.shell import Convertor, _CookieParser, _ParseHeaders
@@ -32,12 +33,12 @@ def __ParseJSONData(json_string: Optional[str] = None) -> Optional[Dict[str, Any
def __Request_and_Save(
- fetcher_func: Callable,
+ fetcher_func: Callable[..., Response],
url: str,
output_file: str,
css_selector: Optional[str] = None,
**kwargs,
-):
+) -> None:
"""Make a request using the specified fetcher function and save the result"""
# Handle relative paths - convert to an absolute path based on the current working directory
output_path = Path(output_file)
diff --git a/scrapling/core/_types.py b/scrapling/core/_types.py
index a41e077..85e7013 100644
--- a/scrapling/core/_types.py
+++ b/scrapling/core/_types.py
@@ -4,6 +4,7 @@ Type definitions for type checking purposes.
from typing import (
TYPE_CHECKING,
+ cast,
overload,
Any,
Callable,
@@ -32,8 +33,13 @@ extraction_types = Literal["text", "html", "markdown"]
StrOrBytes = Union[str, bytes]
-if TYPE_CHECKING:
- # typing.Self requires Python 3.11
- from typing_extensions import Self
-else:
- Self = object
+try:
+ # Python 3.11+
+ from typing import Self # novermin
+except ImportError:
+ try:
+ from typing_extensions import Self # Backport
+ except ImportError:
+ from typing import TypeVar
+
+ Self = object
diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py
index 3d523a3..ccbe536 100644
--- a/scrapling/core/ai.py
+++ b/scrapling/core/ai.py
@@ -63,7 +63,7 @@ class ScraplingMCPServer:
main_content_only: bool = True,
params: Optional[Union[Dict, List, Tuple]] = None,
headers: Optional[Mapping[str, Optional[str]]] = None,
- cookies: Optional[Union[dict[str, str], list[tuple[str, str]]]] = None,
+ cookies: Optional[Union[Dict[str, str], list[tuple[str, str]]]] = None,
timeout: Optional[Union[int, float]] = 30,
follow_redirects: bool = True,
max_redirects: int = 30,
@@ -142,7 +142,7 @@ class ScraplingMCPServer:
main_content_only: bool = True,
params: Optional[Union[Dict, List, Tuple]] = None,
headers: Optional[Mapping[str, Optional[str]]] = None,
- cookies: Optional[Union[dict[str, str], list[tuple[str, str]]]] = None,
+ cookies: Optional[Union[Dict[str, str], list[tuple[str, str]]]] = None,
timeout: Optional[Union[int, float]] = 30,
follow_redirects: bool = True,
max_redirects: int = 30,
diff --git a/scrapling/core/mixins.py b/scrapling/core/mixins.py
index 22859b9..afad094 100644
--- a/scrapling/core/mixins.py
+++ b/scrapling/core/mixins.py
@@ -1,9 +1,13 @@
class SelectorsGeneration:
- """Selectors generation functions
+ """
+ Functions for generating selectors
Trying to generate selectors like Firefox or maybe cleaner ones!? Ehm
- Inspiration: https://searchfox.org/mozilla-central/source/devtools/shared/inspector/css-logic.js#591"""
+ Inspiration: https://searchfox.org/mozilla-central/source/devtools/shared/inspector/css-logic.js#591
+ """
- def __general_selection(self, selection: str = "css", full_path=False) -> str:
+ def __general_selection(
+ self, selection: str = "css", full_path: bool = False
+ ) -> str:
"""Generate a selector for the current element.
:return: A string of the generated selector.
"""
@@ -80,7 +84,7 @@ class SelectorsGeneration:
@property
def generate_xpath_selector(self) -> str:
- """Generate a XPath selector for the current element
+ """Generate an XPath selector for the current element
:return: A string of the generated selector.
"""
return self.__general_selection("xpath")
diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py
index b100ad3..f11f386 100644
--- a/scrapling/core/shell.py
+++ b/scrapling/core/shell.py
@@ -570,7 +570,7 @@ Type 'exit' or press Ctrl+D to exit.
class Convertor:
"""Utils for the extract shell command"""
- _extension_map: dict[str, extraction_types] = {
+ _extension_map: Dict[str, extraction_types] = {
"md": "markdown",
"html": "html",
"txt": "text",
@@ -591,7 +591,7 @@ class Convertor:
css_selector: Optional[str] = None,
main_content_only: bool = False,
) -> Generator[str, None, None]:
- """Extract the content of an Selector"""
+ """Extract the content of a Selector"""
if not page or not isinstance(page, Selector):
raise TypeError("Input must be of type `Selector`")
elif not extraction_type or extraction_type not in cls._extension_map.values():
@@ -624,7 +624,7 @@ class Convertor:
def write_content_to_file(
cls, page: Selector, filename: str, css_selector: Optional[str] = None
) -> None:
- """Write an Selector's content to a file"""
+ """Write a Selector's content to a file"""
if not page or not isinstance(page, Selector):
raise TypeError("Input must be of type `Selector`")
elif not filename or not isinstance(filename, str) or not filename.strip():
diff --git a/scrapling/core/translator.py b/scrapling/core/translator.py
index 9250ab6..9c987ff 100644
--- a/scrapling/core/translator.py
+++ b/scrapling/core/translator.py
@@ -1,11 +1,11 @@
"""
-Most of this file is adapted version of the translator of parsel library with some modifications simply for 1 important reason...
+Most of this file is an adapted version of the parsel library's translator with some modifications simply for 1 important reason...
-To add pseudo-elements ``::text`` and ``::attr(ATTR_NAME)`` so we match Parsel/Scrapy selectors format which will be important in future releases but most importantly...
+To add pseudo-elements ``::text`` and ``::attr(ATTR_NAME)`` so we match the Parsel/Scrapy selectors format which will be important in future releases but most importantly...
So you don't have to learn a new selectors/api method like what bs4 done with soupsieve :)
- if you want to learn about this, head to https://cssselect.readthedocs.io/en/latest/#cssselect.FunctionalPseudoElement
+ If you want to learn about this, head to https://cssselect.readthedocs.io/en/latest/#cssselect.FunctionalPseudoElement
"""
import re