🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
Parser - New logic to handle JSON responses passed from Fetchers
Karim shoaird4b896b4e964
diff --git a/scrapling/parser.py b/scrapling/parser.pyindex cac7fef..a9945d0 100644--- a/scrapling/parser.py+++ b/scrapling/parser.py@@ -10,6 +10,7 @@ from scrapling.core.storage_adaptors import SQLiteStorageSystem, StorageSystemMifrom scrapling.core.utils import setup_basic_logging, logging, clean_spaces, flatten, html_forbiddenfrom scrapling.core._types import Any, Dict, List, Tuple, Optional, Pattern, Union, Callable, Generator, SupportsIndex, Iterablefrom lxml import etree, html+from lxml.etree import XMLSyntaxErrorfrom cssselect import SelectorError, SelectorSyntaxError, parse as split_selectors@@ -60,6 +61,7 @@ class Adaptor(SelectorsGeneration):if root is None and not body and text is None:raise ValueError("Adaptor class needs text, body, or root arguments to work")+ self.__text = Noneif root is None:if text is None:if not body or not isinstance(body, bytes):@@ -72,12 +74,21 @@ class Adaptor(SelectorsGeneration):body = text.strip().replace("\x00", "").encode(encoding) or b"<html/>"- parser = html.HTMLParser(- # https://lxml.de/api/lxml.etree.HTMLParser-class.html- recover=True, remove_blank_text=True, remove_comments=(keep_comments is False), encoding=encoding,- compact=True, huge_tree=huge_tree, default_doctype=True- )- self._root = etree.fromstring(body, parser=parser, base_url=url)+ # https://lxml.de/api/lxml.etree.HTMLParser-class.html+ try:+ # Test with recover set to False first so if this is a text body like a json response, we get error+ parser = html.HTMLParser(+ recover=False, remove_blank_text=True, remove_comments=(keep_comments is False), encoding=encoding,+ compact=True, huge_tree=huge_tree, default_doctype=True+ )+ self._root = etree.fromstring(body, parser=parser, base_url=url)+ except XMLSyntaxError:+ parser = html.HTMLParser(+ recover=True, remove_blank_text=True, remove_comments=(keep_comments is False), encoding=encoding,+ compact=True, huge_tree=huge_tree, default_doctype=True+ )+ self._root = etree.fromstring(body, parser=parser, base_url=url)+ self.__text = TextHandler(text or body.decode())else:# All html types inherits from HtmlMixin so this to check for all at once@@ -112,7 +123,6 @@ class Adaptor(SelectorsGeneration):self.url = url# For selector stuffself.__attributes = None- self.__text = Noneself.__tag = Noneself.__debug = debug