🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
Handling an edge case and adding more commentary
Karim shoaird546b3499b0d
diff --git a/scrapling/parser.py b/scrapling/parser.pyindex d117b23..a517112 100644--- a/scrapling/parser.py+++ b/scrapling/parser.py@@ -188,17 +188,21 @@ class Adaptor(SelectorsGeneration):"""Get text content of the element"""if not self.__text:if self.__keep_comments:- # If use chose to keep comments, remove comments from text- # Escape lxml default behaviour and remove comments like this `<span>CONDITION: <!-- -->Excellent</span>`- # This issue is present in parsel/scrapy as well so no need to repeat it here so the user can run regex on the full text.- code = self.html_content- parser = html.HTMLParser(- recover=True, remove_blank_text=True, remove_comments=True, encoding=self.encoding,- compact=True, huge_tree=self.__huge_tree_enabled, default_doctype=True- )- fragment_root = html.fragment_fromstring(code, parser=parser)- self.__text = TextHandler(fragment_root.text)+ if not self.children:+ # If use chose to keep comments, remove comments from text+ # Escape lxml default behaviour and remove comments like this `<span>CONDITION: <!-- -->Excellent</span>`+ # This issue is present in parsel/scrapy as well so no need to repeat it here so the user can run regex on the full text.+ code = self.html_content+ parser = html.HTMLParser(+ recover=True, remove_blank_text=True, remove_comments=True, encoding=self.encoding,+ compact=True, huge_tree=self.__huge_tree_enabled, default_doctype=True+ )+ fragment_root = html.fragment_fromstring(code, parser=parser)+ self.__text = TextHandler(fragment_root.text)+ else:+ self.__text = TextHandler(self._root.text)else:+ # If user already chose to not keep comments then all is goodself.__text = TextHandler(self._root.text)return self.__text