Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

Better logic to handle json responses

Karim shoaireaa7da27c6f2
diff --git a/scrapling/core/utils.py b/scrapling/core/utils.py
index db5ef15..748020b 100644
--- a/scrapling/core/utils.py
+++ b/scrapling/core/utils.py
@@ -4,8 +4,9 @@ from itertools import chain
# Using cache on top of a class is brilliant way to achieve Singleton design pattern without much code
from functools import lru_cache as cache # functools.cache is available on Python 3.9+ only so let's keep lru_cache
-from scrapling.core._types import Dict, Iterable, Any
+from scrapling.core._types import Dict, Iterable, Any, Union
+import orjson
from lxml import html
html_forbidden = {html.HtmlComment, }
@@ -18,6 +19,17 @@ logging.basicConfig(
)
+def is_jsonable(content: Union[bytes, str]) -> bool:
+ if type(content) is bytes:
+ content = content.decode()
+
+ try:
+ _ = orjson.loads(content)
+ return True
+ except orjson.JSONDecodeError:
+ return False
+
+
@cache(None, typed=True)
def setup_basic_logging(level: str = 'debug'):
levels = {
diff --git a/scrapling/parser.py b/scrapling/parser.py
index a9945d0..e74e5cb 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -7,10 +7,9 @@ from scrapling.core.translator import HTMLTranslator
from scrapling.core.mixins import SelectorsGeneration
from scrapling.core.custom_types import TextHandler, TextHandlers, AttributesHandler
from scrapling.core.storage_adaptors import SQLiteStorageSystem, StorageSystemMixin, _StorageTools
-from scrapling.core.utils import setup_basic_logging, logging, clean_spaces, flatten, html_forbidden
+from scrapling.core.utils import setup_basic_logging, logging, clean_spaces, flatten, html_forbidden, is_jsonable
from scrapling.core._types import Any, Dict, List, Tuple, Optional, Pattern, Union, Callable, Generator, SupportsIndex, Iterable
from lxml import etree, html
-from lxml.etree import XMLSyntaxError
from cssselect import SelectorError, SelectorSyntaxError, parse as split_selectors
@@ -75,19 +74,12 @@ class Adaptor(SelectorsGeneration):
body = text.strip().replace("\x00", "").encode(encoding) or b"<html/>"
# https://lxml.de/api/lxml.etree.HTMLParser-class.html
- try:
- # Test with recover set to False first so if this is a text body like a json response, we get error
- parser = html.HTMLParser(
- recover=False, remove_blank_text=True, remove_comments=(keep_comments is False), encoding=encoding,
- compact=True, huge_tree=huge_tree, default_doctype=True
- )
- self._root = etree.fromstring(body, parser=parser, base_url=url)
- except XMLSyntaxError:
- parser = html.HTMLParser(
- recover=True, remove_blank_text=True, remove_comments=(keep_comments is False), encoding=encoding,
- compact=True, huge_tree=huge_tree, default_doctype=True
- )
- self._root = etree.fromstring(body, parser=parser, base_url=url)
+ parser = html.HTMLParser(
+ recover=True, remove_blank_text=True, remove_comments=(keep_comments is False), encoding=encoding,
+ compact=True, huge_tree=huge_tree, default_doctype=True
+ )
+ self._root = etree.fromstring(body, parser=parser, base_url=url)
+ if is_jsonable(text or body.decode()):
self.__text = TextHandler(text or body.decode())
else: