🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
Using the new types file
Karim shoair23c4181ff239
diff --git a/scrapling/custom_types.py b/scrapling/custom_types.pyindex 0c5fb67..5ff3b5f 100644--- a/scrapling/custom_types.py+++ b/scrapling/custom_types.py@@ -1,9 +1,9 @@import refrom types import MappingProxyTypefrom collections.abc import Mapping-from typing import Dict, List, Union, Patternfrom scrapling.utils import _is_iterable, flatten+from scrapling._types import Dict, List, Union, Patternfrom orjson import loads, dumpsfrom w3lib.html import replace_entities as _replace_entitiesdiff --git a/scrapling/parser.py b/scrapling/parser.pyindex a517112..771d08a 100644--- a/scrapling/parser.py+++ b/scrapling/parser.py@@ -1,17 +1,12 @@import osfrom difflib import SequenceMatcher-from typing import Any, Dict, List, Tuple, Optional, Pattern, Union, Callable, Generator-try:- from typing import SupportsIndex-except ImportError:- # 'SupportsIndex' got added in Python 3.8- SupportsIndex = Nonefrom scrapling.translator import HTMLTranslatorfrom scrapling.mixins import SelectorsGenerationfrom scrapling.custom_types import TextHandler, AttributesHandlerfrom scrapling.storage_adaptors import SQLiteStorageSystem, StorageSystemMixin, _StorageToolsfrom scrapling.utils import setup_basic_logging, logging, clean_spaces, flatten, html_forbidden+from scrapling._types import Any, Dict, List, Tuple, Optional, Pattern, Union, Callable, Generator, SupportsIndexfrom lxml import etree, htmlfrom cssselect import SelectorError, SelectorSyntaxError, parse as split_selectorsdiff --git a/scrapling/storage_adaptors.py b/scrapling/storage_adaptors.pyindex ec88925..d9018a3 100644--- a/scrapling/storage_adaptors.py+++ b/scrapling/storage_adaptors.py@@ -4,8 +4,8 @@ import loggingimport threadingfrom hashlib import sha256from abc import ABC, abstractmethod-from typing import Dict, Optional, Union+from scrapling._types import Dict, Optional, Unionfrom scrapling.utils import _StorageTools, cachefrom lxml import htmldiff --git a/scrapling/translator.py b/scrapling/translator.pyindex ec6db86..77679a6 100644--- a/scrapling/translator.py+++ b/scrapling/translator.py@@ -9,24 +9,14 @@ which will be important in future releases but most importantly...import refrom w3lib.html import HTML5_WHITESPACE-from typing import TYPE_CHECKING, Any, Optional-try:- from typing import Protocol-except ImportError:- # Added in Python 3.8- Protocol = object-from scrapling.utils import cache+from scrapling._types import Any, Optional, Protocol, Selffrom cssselect.xpath import ExpressionErrorfrom cssselect.xpath import XPathExpr as OriginalXPathExprfrom cssselect import HTMLTranslator as OriginalHTMLTranslatorfrom cssselect.parser import Element, FunctionalPseudoElement, PseudoElement-if TYPE_CHECKING:- # typing.Self requires Python 3.11- from typing_extensions import Self-regex = f"[{HTML5_WHITESPACE}]+"replace_html5_whitespaces = re.compile(regex).sub