🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(deps): add protego for robots.txt parsing and fix pyright type error in static.py
Abdullah1d15349e07d9
diff --git a/pyproject.toml b/pyproject.tomlindex bb69194..3aee4f1 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -67,6 +67,7 @@ dependencies = ["tld>=0.13.2","w3lib>=2.4.1","typing_extensions",+ "protego>=0.4.0",][project.optional-dependencies]diff --git a/scrapling/engines/static.py b/scrapling/engines/static.pyindex 1f4b09b..a962ccf 100644--- a/scrapling/engines/static.py+++ b/scrapling/engines/static.py@@ -250,6 +250,7 @@ class _SyncSessionLogic(_ConfigurationLogic):request_args = self._merge_request_args(stealth=stealth, proxy=proxy, **kwargs)try:response = session.request(method, **request_args)+ assert response is not Noneresult = ResponseFactory.from_http_request(response, selector_config, meta={"proxy": proxy})return resultexcept CurlError as e: # pragma: no cover