Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

Parser - New logic to handle JSON responses passed from Fetchers

Karim shoaird4b896b4e964
diff --git a/scrapling/parser.py b/scrapling/parser.py
index cac7fef..a9945d0 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -10,6 +10,7 @@ from scrapling.core.storage_adaptors import SQLiteStorageSystem, StorageSystemMi
from scrapling.core.utils import setup_basic_logging, logging, clean_spaces, flatten, html_forbidden
from scrapling.core._types import Any, Dict, List, Tuple, Optional, Pattern, Union, Callable, Generator, SupportsIndex, Iterable
from lxml import etree, html
+from lxml.etree import XMLSyntaxError
from cssselect import SelectorError, SelectorSyntaxError, parse as split_selectors
@@ -60,6 +61,7 @@ class Adaptor(SelectorsGeneration):
if root is None and not body and text is None:
raise ValueError("Adaptor class needs text, body, or root arguments to work")
+ self.__text = None
if root is None:
if text is None:
if not body or not isinstance(body, bytes):
@@ -72,12 +74,21 @@ class Adaptor(SelectorsGeneration):
body = text.strip().replace("\x00", "").encode(encoding) or b"<html/>"
- parser = html.HTMLParser(
- # https://lxml.de/api/lxml.etree.HTMLParser-class.html
- recover=True, remove_blank_text=True, remove_comments=(keep_comments is False), encoding=encoding,
- compact=True, huge_tree=huge_tree, default_doctype=True
- )
- self._root = etree.fromstring(body, parser=parser, base_url=url)
+ # https://lxml.de/api/lxml.etree.HTMLParser-class.html
+ try:
+ # Test with recover set to False first so if this is a text body like a json response, we get error
+ parser = html.HTMLParser(
+ recover=False, remove_blank_text=True, remove_comments=(keep_comments is False), encoding=encoding,
+ compact=True, huge_tree=huge_tree, default_doctype=True
+ )
+ self._root = etree.fromstring(body, parser=parser, base_url=url)
+ except XMLSyntaxError:
+ parser = html.HTMLParser(
+ recover=True, remove_blank_text=True, remove_comments=(keep_comments is False), encoding=encoding,
+ compact=True, huge_tree=huge_tree, default_doctype=True
+ )
+ self._root = etree.fromstring(body, parser=parser, base_url=url)
+ self.__text = TextHandler(text or body.decode())
else:
# All html types inherits from HtmlMixin so this to check for all at once
@@ -112,7 +123,6 @@ class Adaptor(SelectorsGeneration):
self.url = url
# For selector stuff
self.__attributes = None
- self.__text = None
self.__tag = None
self.__debug = debug