Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

Using the new types file

Karim shoair23c4181ff239
diff --git a/scrapling/custom_types.py b/scrapling/custom_types.py
index 0c5fb67..5ff3b5f 100644
--- a/scrapling/custom_types.py
+++ b/scrapling/custom_types.py
@@ -1,9 +1,9 @@
import re
from types import MappingProxyType
from collections.abc import Mapping
-from typing import Dict, List, Union, Pattern
from scrapling.utils import _is_iterable, flatten
+from scrapling._types import Dict, List, Union, Pattern
from orjson import loads, dumps
from w3lib.html import replace_entities as _replace_entities
diff --git a/scrapling/parser.py b/scrapling/parser.py
index a517112..771d08a 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -1,17 +1,12 @@
import os
from difflib import SequenceMatcher
-from typing import Any, Dict, List, Tuple, Optional, Pattern, Union, Callable, Generator
-try:
- from typing import SupportsIndex
-except ImportError:
- # 'SupportsIndex' got added in Python 3.8
- SupportsIndex = None
from scrapling.translator import HTMLTranslator
from scrapling.mixins import SelectorsGeneration
from scrapling.custom_types import TextHandler, AttributesHandler
from scrapling.storage_adaptors import SQLiteStorageSystem, StorageSystemMixin, _StorageTools
from scrapling.utils import setup_basic_logging, logging, clean_spaces, flatten, html_forbidden
+from scrapling._types import Any, Dict, List, Tuple, Optional, Pattern, Union, Callable, Generator, SupportsIndex
from lxml import etree, html
from cssselect import SelectorError, SelectorSyntaxError, parse as split_selectors
diff --git a/scrapling/storage_adaptors.py b/scrapling/storage_adaptors.py
index ec88925..d9018a3 100644
--- a/scrapling/storage_adaptors.py
+++ b/scrapling/storage_adaptors.py
@@ -4,8 +4,8 @@ import logging
import threading
from hashlib import sha256
from abc import ABC, abstractmethod
-from typing import Dict, Optional, Union
+from scrapling._types import Dict, Optional, Union
from scrapling.utils import _StorageTools, cache
from lxml import html
diff --git a/scrapling/translator.py b/scrapling/translator.py
index ec6db86..77679a6 100644
--- a/scrapling/translator.py
+++ b/scrapling/translator.py
@@ -9,24 +9,14 @@ which will be important in future releases but most importantly...
import re
from w3lib.html import HTML5_WHITESPACE
-from typing import TYPE_CHECKING, Any, Optional
-try:
- from typing import Protocol
-except ImportError:
- # Added in Python 3.8
- Protocol = object
-
from scrapling.utils import cache
+from scrapling._types import Any, Optional, Protocol, Self
from cssselect.xpath import ExpressionError
from cssselect.xpath import XPathExpr as OriginalXPathExpr
from cssselect import HTMLTranslator as OriginalHTMLTranslator
from cssselect.parser import Element, FunctionalPseudoElement, PseudoElement
-if TYPE_CHECKING:
- # typing.Self requires Python 3.11
- from typing_extensions import Self
-
regex = f"[{HTML5_WHITESPACE}]+"
replace_html5_whitespaces = re.compile(regex).sub