🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
style: type hints corrections and docstrings
Karim shoair18660f8132f1
diff --git a/README.md b/README.mdindex 3838b1f..8d1c778 100644--- a/README.md+++ b/README.md@@ -150,7 +150,7 @@ Tired of your PC slowing you down? Can’t keep your machine on 24/7 for scrapin```pythonfrom scrapling.fetchers import Fetcher-# Do HTTP GET request to a web page and create an Selector instance+# Do HTTP GET request to a web page and create a Selector instancepage = Fetcher.get('https://quotes.toscrape.com/', stealthy_headers=True)# Get all text content from all HTML tags in the page except the `script` and `style` tagspage.get_all_text(ignore_tags=('script', 'style'))diff --git a/scrapling/cli.py b/scrapling/cli.pyindex e5e91d5..940c27d 100644--- a/scrapling/cli.py+++ b/scrapling/cli.py@@ -3,6 +3,7 @@ from subprocess import check_outputfrom sys import executable as python_executablefrom scrapling.core.utils import log+from scrapling.engines.toolbelt import Responsefrom scrapling.core._types import List, Optional, Dict, Tuple, Any, Callablefrom scrapling.fetchers import Fetcher, DynamicFetcher, StealthyFetcherfrom scrapling.core.shell import Convertor, _CookieParser, _ParseHeaders@@ -32,12 +33,12 @@ def __ParseJSONData(json_string: Optional[str] = None) -> Optional[Dict[str, Anydef __Request_and_Save(- fetcher_func: Callable,+ fetcher_func: Callable[..., Response],url: str,output_file: str,css_selector: Optional[str] = None,**kwargs,-):+) -> None:"""Make a request using the specified fetcher function and save the result"""# Handle relative paths - convert to an absolute path based on the current working directoryoutput_path = Path(output_file)diff --git a/scrapling/core/_types.py b/scrapling/core/_types.pyindex a41e077..85e7013 100644--- a/scrapling/core/_types.py+++ b/scrapling/core/_types.py@@ -4,6 +4,7 @@ Type definitions for type checking purposes.from typing import (TYPE_CHECKING,+ cast,overload,Any,Callable,@@ -32,8 +33,13 @@ extraction_types = Literal["text", "html", "markdown"]StrOrBytes = Union[str, bytes]-if TYPE_CHECKING:- # typing.Self requires Python 3.11- from typing_extensions import Self-else:- Self = object+try:+ # Python 3.11++ from typing import Self # novermin+except ImportError:+ try:+ from typing_extensions import Self # Backport+ except ImportError:+ from typing import TypeVar++ Self = objectdiff --git a/scrapling/core/ai.py b/scrapling/core/ai.pyindex 3d523a3..ccbe536 100644--- a/scrapling/core/ai.py+++ b/scrapling/core/ai.py@@ -63,7 +63,7 @@ class ScraplingMCPServer:main_content_only: bool = True,params: Optional[Union[Dict, List, Tuple]] = None,headers: Optional[Mapping[str, Optional[str]]] = None,- cookies: Optional[Union[dict[str, str], list[tuple[str, str]]]] = None,+ cookies: Optional[Union[Dict[str, str], list[tuple[str, str]]]] = None,timeout: Optional[Union[int, float]] = 30,follow_redirects: bool = True,max_redirects: int = 30,@@ -142,7 +142,7 @@ class ScraplingMCPServer:main_content_only: bool = True,params: Optional[Union[Dict, List, Tuple]] = None,headers: Optional[Mapping[str, Optional[str]]] = None,- cookies: Optional[Union[dict[str, str], list[tuple[str, str]]]] = None,+ cookies: Optional[Union[Dict[str, str], list[tuple[str, str]]]] = None,timeout: Optional[Union[int, float]] = 30,follow_redirects: bool = True,max_redirects: int = 30,diff --git a/scrapling/core/mixins.py b/scrapling/core/mixins.pyindex 22859b9..afad094 100644--- a/scrapling/core/mixins.py+++ b/scrapling/core/mixins.py@@ -1,9 +1,13 @@class SelectorsGeneration:- """Selectors generation functions+ """+ Functions for generating selectorsTrying to generate selectors like Firefox or maybe cleaner ones!? Ehm- Inspiration: https://searchfox.org/mozilla-central/source/devtools/shared/inspector/css-logic.js#591"""+ Inspiration: https://searchfox.org/mozilla-central/source/devtools/shared/inspector/css-logic.js#591+ """- def __general_selection(self, selection: str = "css", full_path=False) -> str:+ def __general_selection(+ self, selection: str = "css", full_path: bool = False+ ) -> str:"""Generate a selector for the current element.:return: A string of the generated selector."""@@ -80,7 +84,7 @@ class SelectorsGeneration:@propertydef generate_xpath_selector(self) -> str:- """Generate a XPath selector for the current element+ """Generate an XPath selector for the current element:return: A string of the generated selector."""return self.__general_selection("xpath")diff --git a/scrapling/core/shell.py b/scrapling/core/shell.pyindex b100ad3..f11f386 100644--- a/scrapling/core/shell.py+++ b/scrapling/core/shell.py@@ -570,7 +570,7 @@ Type 'exit' or press Ctrl+D to exit.class Convertor:"""Utils for the extract shell command"""- _extension_map: dict[str, extraction_types] = {+ _extension_map: Dict[str, extraction_types] = {"md": "markdown","html": "html","txt": "text",@@ -591,7 +591,7 @@ class Convertor:css_selector: Optional[str] = None,main_content_only: bool = False,) -> Generator[str, None, None]:- """Extract the content of an Selector"""+ """Extract the content of a Selector"""if not page or not isinstance(page, Selector):raise TypeError("Input must be of type `Selector`")elif not extraction_type or extraction_type not in cls._extension_map.values():@@ -624,7 +624,7 @@ class Convertor:def write_content_to_file(cls, page: Selector, filename: str, css_selector: Optional[str] = None) -> None:- """Write an Selector's content to a file"""+ """Write a Selector's content to a file"""if not page or not isinstance(page, Selector):raise TypeError("Input must be of type `Selector`")elif not filename or not isinstance(filename, str) or not filename.strip():diff --git a/scrapling/core/translator.py b/scrapling/core/translator.pyindex 9250ab6..9c987ff 100644--- a/scrapling/core/translator.py+++ b/scrapling/core/translator.py@@ -1,11 +1,11 @@"""-Most of this file is adapted version of the translator of parsel library with some modifications simply for 1 important reason...+Most of this file is an adapted version of the parsel library's translator with some modifications simply for 1 important reason...-To add pseudo-elements ``::text`` and ``::attr(ATTR_NAME)`` so we match Parsel/Scrapy selectors format which will be important in future releases but most importantly...+To add pseudo-elements ``::text`` and ``::attr(ATTR_NAME)`` so we match the Parsel/Scrapy selectors format which will be important in future releases but most importantly...So you don't have to learn a new selectors/api method like what bs4 done with soupsieve :)- if you want to learn about this, head to https://cssselect.readthedocs.io/en/latest/#cssselect.FunctionalPseudoElement+ If you want to learn about this, head to https://cssselect.readthedocs.io/en/latest/#cssselect.FunctionalPseudoElement"""import re