🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
perf: General code restructure to not use more than needed memory
Karim shoair3da806210b81
diff --git a/scrapling/cli.py b/scrapling/cli.pyindex 12b96c6..6f9487d 100644--- a/scrapling/cli.py+++ b/scrapling/cli.py@@ -2,8 +2,9 @@ from pathlib import Pathfrom subprocess import check_outputfrom sys import executable as python_executable+from scrapling.core.utils import logfrom scrapling.engines.toolbelt.custom import Response-from scrapling.core.utils import log, _CookieParser, _ParseHeaders+from scrapling.core.utils._shell import _CookieParser, _ParseHeadersfrom scrapling.core._types import List, Optional, Dict, Tuple, Any, Callablefrom orjson import loads as json_loads, JSONDecodeErrordiff --git a/scrapling/core/shell.py b/scrapling/core/shell.pyindex f9f790f..5ef4458 100644--- a/scrapling/core/shell.py+++ b/scrapling/core/shell.py@@ -22,10 +22,11 @@ from logging import (from orjson import loads as json_loads, JSONDecodeErrorfrom scrapling import __version__+from scrapling.core.utils import logfrom scrapling.parser import Selector, Selectorsfrom scrapling.core.custom_types import TextHandlerfrom scrapling.engines.toolbelt.custom import Response-from scrapling.core.utils import log, _ParseHeaders, _CookieParser+from scrapling.core.utils._shell import _ParseHeaders, _CookieParserfrom scrapling.core._types import (Optional,Dict,diff --git a/scrapling/core/storage.py b/scrapling/core/storage.pyindex 03c05a2..e832cf0 100644--- a/scrapling/core/storage.py+++ b/scrapling/core/storage.py@@ -6,7 +6,6 @@ from sqlite3 import connect as db_connectfrom orjson import dumps, loadsfrom lxml.html import HtmlElement-from tldextract import extract as tldfrom scrapling.core.utils import _StorageTools, logfrom scrapling.core._types import Dict, Optional, Any@@ -26,6 +25,8 @@ class StorageSystemMixin(ABC): # pragma: no coverreturn default_valuetry:+ from tldextract import extract as tld+extracted = tld(self.url)return extracted.top_domain_under_public_suffix or extracted.domain or default_valueexcept AttributeError:diff --git a/scrapling/core/utils/__init__.py b/scrapling/core/utils/__init__.pyindex dc95705..6ae80fe 100644--- a/scrapling/core/utils/__init__.py+++ b/scrapling/core/utils/__init__.py@@ -7,4 +7,3 @@ from ._utils import (clean_spaces,html_forbidden,)-from ._shell import _CookieParser, _ParseHeadersdiff --git a/scrapling/parser.py b/scrapling/parser.pyindex 776dd55..6d0a575 100644--- a/scrapling/parser.py+++ b/scrapling/parser.py@@ -1,8 +1,8 @@-import refrom pathlib import Pathfrom inspect import signaturefrom urllib.parse import urljoinfrom difflib import SequenceMatcher+from re import Pattern as re_Patternfrom lxml.html import HtmlElement, HtmlMixin, HTMLParserfrom cssselect import SelectorError, SelectorSyntaxError, parse as split_selectors@@ -751,7 +751,7 @@ class Selector(SelectorsGeneration):)attributes.update(arg)- elif isinstance(arg, re.Pattern):+ elif isinstance(arg, re_Pattern):patterns.add(arg)elif callable(arg):