🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
Version 0.1.2
Karim shoaira01cfe66dbd0
diff --git a/MANIFEST.in b/MANIFEST.inindex 98d462a..c69a5e6 100644--- a/MANIFEST.in+++ b/MANIFEST.in@@ -1,4 +1,6 @@include LICENSE+include *.db+include scrapling/*.dbinclude scrapling/py.typedrecursive-exclude * __pycache__diff --git a/README.md b/README.mdindex 02df917..eeff0cc 100644--- a/README.md+++ b/README.md@@ -1,5 +1,5 @@# 🕷️ Scrapling: Lightning-Fast, Adaptive Web Scraping for Python-[](https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml) [](https://badge.fury.io/py/Scrapling) [](https://pypi.org/project/scrapling/) [](https://opensource.org/licenses/BSD-3-Clause)+[](https://github.com/D4Vinci/Scrapling/actions/workflows/tests.yml) [](https://badge.fury.io/py/Scrapling) [](https://pypi.org/project/scrapling/) [](https://pepy.tech/project/scrapling)Dealing with failing web scrapers due to website changes? Meet Scrapling.@@ -415,6 +415,9 @@ Of course, you can find elements by text/regex, find similar elements in a more### Is Scrapling thread-safe?Yes, Scrapling instances are thread-safe. Each Adaptor instance maintains its own state.+## Sponsors+[](https://www.capsolver.com/?utm_source=github&utm_medium=repo&utm_campaign=scraping&utm_term=Scrapling)+## ContributingEverybody is invited and welcome to contribute to Scrapling. There is a lot to do!diff --git a/images/CapSolver.png b/images/CapSolver.pngnew file mode 100644index 0000000..6f5b3bdBinary files /dev/null and b/images/CapSolver.png differdiff --git a/scrapling/__init__.py b/scrapling/__init__.pyindex 7985ef9..64a8d26 100644--- a/scrapling/__init__.py+++ b/scrapling/__init__.py@@ -3,7 +3,7 @@ from scrapling.parser import Adaptor, Adaptorsfrom scrapling.custom_types import TextHandler, AttributesHandler__author__ = "Karim Shoair ([email protected])"-__version__ = "0.1.1"+__version__ = "0.1.2"__copyright__ = "Copyright (c) 2024 Karim Shoair"diff --git a/scrapling/parser.py b/scrapling/parser.pyindex 375d384..d117b23 100644--- a/scrapling/parser.py+++ b/scrapling/parser.py@@ -78,7 +78,7 @@ class Adaptor(SelectorsGeneration):parser = html.HTMLParser(# https://lxml.de/api/lxml.etree.HTMLParser-class.html- recover=True, remove_blank_text=True, remove_comments=(keep_comments is True), encoding=encoding,+ recover=True, remove_blank_text=True, remove_comments=(keep_comments is False), encoding=encoding,compact=True, huge_tree=huge_tree, default_doctype=True)self._root = etree.fromstring(body, parser=parser, base_url=url)@@ -142,7 +142,8 @@ class Adaptor(SelectorsGeneration):if issubclass(type(element), html.HtmlMixin):return self.__class__(root=element, url=self.url, encoding=self.encoding, auto_match=self.__auto_match_enabled,- keep_comments=self.__keep_comments, huge_tree=self.__huge_tree_enabled, debug=self.__debug+ keep_comments=True, # if the comments are already removed in initialization, no need to try to delete them in sub-elements+ huge_tree=self.__huge_tree_enabled, debug=self.__debug)return element@@ -186,7 +187,19 @@ class Adaptor(SelectorsGeneration):def text(self) -> TextHandler:"""Get text content of the element"""if not self.__text:- self.__text = TextHandler(self._root.text)+ if self.__keep_comments:+ # If use chose to keep comments, remove comments from text+ # Escape lxml default behaviour and remove comments like this `<span>CONDITION: <!-- -->Excellent</span>`+ # This issue is present in parsel/scrapy as well so no need to repeat it here so the user can run regex on the full text.+ code = self.html_content+ parser = html.HTMLParser(+ recover=True, remove_blank_text=True, remove_comments=True, encoding=self.encoding,+ compact=True, huge_tree=self.__huge_tree_enabled, default_doctype=True+ )+ fragment_root = html.fragment_fromstring(code, parser=parser)+ self.__text = TextHandler(fragment_root.text)+ else:+ self.__text = TextHandler(self._root.text)return self.__textdef get_all_text(self, separator: str = "\n", strip: bool = False, ignore_tags: Tuple = ('script', 'style',), valid_values: bool = True) -> TextHandler:diff --git a/setup.cfg b/setup.cfgindex aaa936c..4102c89 100644--- a/setup.cfg+++ b/setup.cfg@@ -1,6 +1,6 @@[metadata]name = scrapling-version = 0.1.1+version = 0.1.2author = Karim Shoairauthor_email = [email protected]description = Scrapling is a powerful, flexible, adaptive, and high-performance web scraping library for Python.diff --git a/setup.py b/setup.pyindex 4b0beea..52e42c3 100644--- a/setup.py+++ b/setup.py@@ -6,7 +6,7 @@ with open("README.md", "r", encoding="utf-8") as fh:setup(name="scrapling",- version="0.1.1",+ version="0.1.2",description="""Scrapling is a powerful, flexible, and high-performance web scraping library for Python. Itsimplifies the process of extracting data from websites, even when they undergo structural changes, and offersimpressive speed improvements over many popular scraping tools.""",