Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

test: align robots request-rate expectation

Disaster-Terminator139ba13af68c
diff --git a/pyproject.toml b/pyproject.toml
index 9c390c0..256b16a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -79,7 +79,7 @@ fetchers = [
"apify-fingerprint-datapoints>=0.13.0",
"msgspec>=0.21.1",
"anyio>=4.13.0",
- "protego>=0.6.0",
+ "protego>=0.6.1",
]
ai = [
"mcp>=1.27.0",
@@ -125,4 +125,4 @@ check_untyped_defs = true
pythonVersion = "3.10"
typeCheckingMode = "basic"
include = ["scrapling"]
-ignore = ["tests", "benchmarks.py"]
\ No newline at end of file
+ignore = ["tests", "benchmarks.py"]
diff --git a/tests/spiders/test_robotstxt.py b/tests/spiders/test_robotstxt.py
index c4050c9..6a14ce7 100644
--- a/tests/spiders/test_robotstxt.py
+++ b/tests/spiders/test_robotstxt.py
@@ -172,7 +172,7 @@ class TestGetDelayDirectives:
assert c_delay is None
assert r_rate is not None
- assert r_rate == (1, 1)
+ assert r_rate == (1, 10)
@pytest.mark.asyncio
async def test_returns_both_none_when_not_set(self):