🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
test: align robots request-rate expectation
Disaster-Terminator139ba13af68c
diff --git a/pyproject.toml b/pyproject.tomlindex 9c390c0..256b16a 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -79,7 +79,7 @@ fetchers = ["apify-fingerprint-datapoints>=0.13.0","msgspec>=0.21.1","anyio>=4.13.0",- "protego>=0.6.0",+ "protego>=0.6.1",]ai = ["mcp>=1.27.0",@@ -125,4 +125,4 @@ check_untyped_defs = truepythonVersion = "3.10"typeCheckingMode = "basic"include = ["scrapling"]-ignore = ["tests", "benchmarks.py"]\ No newline at end of file+ignore = ["tests", "benchmarks.py"]diff --git a/tests/spiders/test_robotstxt.py b/tests/spiders/test_robotstxt.pyindex c4050c9..6a14ce7 100644--- a/tests/spiders/test_robotstxt.py+++ b/tests/spiders/test_robotstxt.py@@ -172,7 +172,7 @@ class TestGetDelayDirectives:assert c_delay is Noneassert r_rate is not None- assert r_rate == (1, 1)+ assert r_rate == (1, 10)@pytest.mark.asyncioasync def test_returns_both_none_when_not_set(self):