Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

perf: General code restructure to not use more than needed memory

Karim shoair3da806210b81
diff --git a/scrapling/cli.py b/scrapling/cli.py
index 12b96c6..6f9487d 100644
--- a/scrapling/cli.py
+++ b/scrapling/cli.py
@@ -2,8 +2,9 @@ from pathlib import Path
from subprocess import check_output
from sys import executable as python_executable
+from scrapling.core.utils import log
from scrapling.engines.toolbelt.custom import Response
-from scrapling.core.utils import log, _CookieParser, _ParseHeaders
+from scrapling.core.utils._shell import _CookieParser, _ParseHeaders
from scrapling.core._types import List, Optional, Dict, Tuple, Any, Callable
from orjson import loads as json_loads, JSONDecodeError
diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py
index f9f790f..5ef4458 100644
--- a/scrapling/core/shell.py
+++ b/scrapling/core/shell.py
@@ -22,10 +22,11 @@ from logging import (
from orjson import loads as json_loads, JSONDecodeError
from scrapling import __version__
+from scrapling.core.utils import log
from scrapling.parser import Selector, Selectors
from scrapling.core.custom_types import TextHandler
from scrapling.engines.toolbelt.custom import Response
-from scrapling.core.utils import log, _ParseHeaders, _CookieParser
+from scrapling.core.utils._shell import _ParseHeaders, _CookieParser
from scrapling.core._types import (
Optional,
Dict,
diff --git a/scrapling/core/storage.py b/scrapling/core/storage.py
index 03c05a2..e832cf0 100644
--- a/scrapling/core/storage.py
+++ b/scrapling/core/storage.py
@@ -6,7 +6,6 @@ from sqlite3 import connect as db_connect
from orjson import dumps, loads
from lxml.html import HtmlElement
-from tldextract import extract as tld
from scrapling.core.utils import _StorageTools, log
from scrapling.core._types import Dict, Optional, Any
@@ -26,6 +25,8 @@ class StorageSystemMixin(ABC): # pragma: no cover
return default_value
try:
+ from tldextract import extract as tld
+
extracted = tld(self.url)
return extracted.top_domain_under_public_suffix or extracted.domain or default_value
except AttributeError:
diff --git a/scrapling/core/utils/__init__.py b/scrapling/core/utils/__init__.py
index dc95705..6ae80fe 100644
--- a/scrapling/core/utils/__init__.py
+++ b/scrapling/core/utils/__init__.py
@@ -7,4 +7,3 @@ from ._utils import (
clean_spaces,
html_forbidden,
)
-from ._shell import _CookieParser, _ParseHeaders
diff --git a/scrapling/parser.py b/scrapling/parser.py
index 776dd55..6d0a575 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -1,8 +1,8 @@
-import re
from pathlib import Path
from inspect import signature
from urllib.parse import urljoin
from difflib import SequenceMatcher
+from re import Pattern as re_Pattern
from lxml.html import HtmlElement, HtmlMixin, HTMLParser
from cssselect import SelectorError, SelectorSyntaxError, parse as split_selectors
@@ -751,7 +751,7 @@ class Selector(SelectorsGeneration):
)
attributes.update(arg)
- elif isinstance(arg, re.Pattern):
+ elif isinstance(arg, re_Pattern):
patterns.add(arg)
elif callable(arg):