🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
Better logic to handle json responses
Karim shoaireaa7da27c6f2
diff --git a/scrapling/core/utils.py b/scrapling/core/utils.pyindex db5ef15..748020b 100644--- a/scrapling/core/utils.py+++ b/scrapling/core/utils.py@@ -4,8 +4,9 @@ from itertools import chain# Using cache on top of a class is brilliant way to achieve Singleton design pattern without much codefrom functools import lru_cache as cache # functools.cache is available on Python 3.9+ only so let's keep lru_cache-from scrapling.core._types import Dict, Iterable, Any+from scrapling.core._types import Dict, Iterable, Any, Union+import orjsonfrom lxml import htmlhtml_forbidden = {html.HtmlComment, }@@ -18,6 +19,17 @@ logging.basicConfig()+def is_jsonable(content: Union[bytes, str]) -> bool:+ if type(content) is bytes:+ content = content.decode()++ try:+ _ = orjson.loads(content)+ return True+ except orjson.JSONDecodeError:+ return False++@cache(None, typed=True)def setup_basic_logging(level: str = 'debug'):levels = {diff --git a/scrapling/parser.py b/scrapling/parser.pyindex a9945d0..e74e5cb 100644--- a/scrapling/parser.py+++ b/scrapling/parser.py@@ -7,10 +7,9 @@ from scrapling.core.translator import HTMLTranslatorfrom scrapling.core.mixins import SelectorsGenerationfrom scrapling.core.custom_types import TextHandler, TextHandlers, AttributesHandlerfrom scrapling.core.storage_adaptors import SQLiteStorageSystem, StorageSystemMixin, _StorageTools-from scrapling.core.utils import setup_basic_logging, logging, clean_spaces, flatten, html_forbidden+from scrapling.core.utils import setup_basic_logging, logging, clean_spaces, flatten, html_forbidden, is_jsonablefrom scrapling.core._types import Any, Dict, List, Tuple, Optional, Pattern, Union, Callable, Generator, SupportsIndex, Iterablefrom lxml import etree, html-from lxml.etree import XMLSyntaxErrorfrom cssselect import SelectorError, SelectorSyntaxError, parse as split_selectors@@ -75,19 +74,12 @@ class Adaptor(SelectorsGeneration):body = text.strip().replace("\x00", "").encode(encoding) or b"<html/>"# https://lxml.de/api/lxml.etree.HTMLParser-class.html- try:- # Test with recover set to False first so if this is a text body like a json response, we get error- parser = html.HTMLParser(- recover=False, remove_blank_text=True, remove_comments=(keep_comments is False), encoding=encoding,- compact=True, huge_tree=huge_tree, default_doctype=True- )- self._root = etree.fromstring(body, parser=parser, base_url=url)- except XMLSyntaxError:- parser = html.HTMLParser(- recover=True, remove_blank_text=True, remove_comments=(keep_comments is False), encoding=encoding,- compact=True, huge_tree=huge_tree, default_doctype=True- )- self._root = etree.fromstring(body, parser=parser, base_url=url)+ parser = html.HTMLParser(+ recover=True, remove_blank_text=True, remove_comments=(keep_comments is False), encoding=encoding,+ compact=True, huge_tree=huge_tree, default_doctype=True+ )+ self._root = etree.fromstring(body, parser=parser, base_url=url)+ if is_jsonable(text or body.decode()):self.__text = TextHandler(text or body.decode())else: