Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(spider robots): removing dead code

Karim shoairafaf68e7d59e
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py
index 7729d62..1924298 100644
--- a/scrapling/spiders/engine.py
+++ b/scrapling/spiders/engine.py
@@ -96,7 +96,7 @@ class CrawlerEngine:
return self._domain_delays[domain]
# For domains covered by _prefetch_robots_txt this is a local parser read.
- # Domains discovered mid-crawl (not in start_urls/allowed_domains) will fetch here.
+ # Domains discovered mid-crawl (not in start_urls) will fetch here.
c_delay, r_rate = await robots_manager.get_delay_directives(request.url, request.sid)
delay = self.spider.download_delay
diff --git a/scrapling/spiders/robotstxt.py b/scrapling/spiders/robotstxt.py
index 867698b..10dc7ab 100644
--- a/scrapling/spiders/robotstxt.py
+++ b/scrapling/spiders/robotstxt.py
@@ -61,7 +61,6 @@ class RobotsTxtManager:
"""Check if a URL can be fetched according to the domain's robots.txt.
Handles:
- - User-agent specific rules (e.g., User-agent: SpinarakBot)
- Wildcard user-agent rules (User-agent: *)
- Allow/Disallow directives with wildcards (e.g., /*.pdf$)
- Allow directives that override Disallow (e.g., Allow: /admin/public-docs/)
@@ -80,42 +79,6 @@ class RobotsTxtManager:
parser = await self._get_parser(url, sid)
return parser.can_fetch(url, "*")
- async def get_crawl_delay(self, url: str, sid: str) -> Optional[float]:
- """Get the crawl delay for this crawler.
-
- Uses the wildcard user-agent (*) to get the general crawl delay
- that applies to all bots.
-
- Args:
- url: Any URL on the domain to check
- sid: Session ID for fetching robots.txt if not yet cached
-
- Returns:
- The crawl delay in seconds, or None if not specified
- """
- parser = await self._get_parser(url, sid)
- delay = parser.crawl_delay("*")
- return float(delay) if delay is not None else None
-
- async def get_request_rate(self, url: str, sid: str) -> Optional[tuple[int, int]]:
- """Get the request rate for this crawler.
-
- Uses the wildcard user-agent (*) to get the general request rate
- that applies to all bots.
-
- Args:
- url: Any URL on the domain to check
- sid: Session ID for fetching robots.txt if not yet cached
-
- Returns:
- A tuple of (requests, seconds) if specified, or None if not specified
- """
- parser = await self._get_parser(url, sid)
- rate = parser.request_rate("*")
- if rate is not None:
- return (rate.requests, rate.seconds)
- return None
-
async def get_delay_directives(self, url: str, sid: str) -> tuple[Optional[float], Optional[tuple[int, int]]]:
"""Return both crawl-delay and request-rate in a single parser lookup.
@@ -152,18 +115,3 @@ class RobotsTxtManager:
async with create_task_group() as tg:
for url in urls:
tg.start_soon(self._get_parser, url, sid)
-
- def clear_cache(self, domain: Optional[str] = None) -> None:
- """Clear the robots.txt cache.
-
- Note: the ``sid`` parameter was removed — the cache is now keyed by
- domain only, so clearing a domain evicts all sessions at once.
-
- Args:
- domain: If specified, only clear cache for this domain.
- If None, clears the entire cache.
- """
- if domain is None:
- self._cache.clear()
- else:
- self._cache.pop(domain, None)
diff --git a/tests/spiders/test_engine.py b/tests/spiders/test_engine.py
index e7382f8..74cf875 100644
--- a/tests/spiders/test_engine.py
+++ b/tests/spiders/test_engine.py
@@ -940,19 +940,7 @@ class TestPrefetchRobotsTxt:
return _fetch, calls
@pytest.mark.asyncio
- async def test_prefetch_uses_allowed_domains_when_set(self):
- fetch_fn, calls = self._make_counting_fetch()
- spider = MockSpider(allowed_domains={"a.com", "b.com"}, robots_txt_obey=True)
- engine = _make_engine(spider=spider)
- engine._robots_manager = RobotsTxtManager(fetch_fn)
-
- await engine._prefetch_robots_txt()
-
- fetched_domains = {Request(url).domain for url, _ in calls}
- assert fetched_domains == {"a.com", "b.com"}
-
- @pytest.mark.asyncio
- async def test_prefetch_falls_back_to_start_urls_when_no_allowed_domains(self):
+ async def test_prefetch_uses_start_urls(self):
fetch_fn, calls = self._make_counting_fetch()
spider = MockSpider(robots_txt_obey=True, start_urls=["https://example.com/page1"])
engine = _make_engine(spider=spider)
diff --git a/tests/spiders/test_robotstxt.py b/tests/spiders/test_robotstxt.py
index efb8328..c4050c9 100644
--- a/tests/spiders/test_robotstxt.py
+++ b/tests/spiders/test_robotstxt.py
@@ -53,14 +53,6 @@ Request-rate: 1/10
Disallow: /private/
"""
-ROBOTS_WITH_SITEMAP = """\
-User-agent: *
-Disallow:
-
-Sitemap: https://example.com/sitemap.xml
-Sitemap: https://example.com/sitemap2.xml
-"""
-
ROBOTS_ALLOW_OVERRIDE = """\
User-agent: *
Disallow: /secret/
@@ -157,121 +149,79 @@ class TestCanFetch:
# ---------------------------------------------------------------------------
-# Tests: get_crawl_delay
+# Tests: get_delay_directives
# ---------------------------------------------------------------------------
-class TestGetCrawlDelay:
+class TestGetDelayDirectives:
@pytest.mark.asyncio
- async def test_returns_float_when_set(self):
+ async def test_returns_crawl_delay_when_set(self):
mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC))
- delay = await mgr.get_crawl_delay("https://example.com/", "s1")
+ c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1")
+
+ assert c_delay == 2.0
+ assert isinstance(c_delay, float)
+ assert r_rate is None
+
+ @pytest.mark.asyncio
+ async def test_returns_request_rate_when_set(self):
+ mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_WITH_RATE))
+
+ c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1")
- assert delay == 2.0
- assert isinstance(delay, float)
+ assert c_delay is None
+ assert r_rate is not None
+ assert r_rate == (1, 1)
@pytest.mark.asyncio
- async def test_returns_none_when_not_set(self):
+ async def test_returns_both_none_when_not_set(self):
content = "User-agent: *\nDisallow: /admin/"
mgr = RobotsTxtManager(make_fetch_fn(content=content))
- assert await mgr.get_crawl_delay("https://example.com/", "s1") is None
+ c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1")
+
+ assert c_delay is None
+ assert r_rate is None
@pytest.mark.asyncio
- async def test_returns_none_for_empty_robots(self):
+ async def test_returns_both_none_for_empty_robots(self):
mgr = RobotsTxtManager(make_fetch_fn(content=""))
- assert await mgr.get_crawl_delay("https://example.com/", "s1") is None
+ c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1")
+
+ assert c_delay is None
+ assert r_rate is None
@pytest.mark.asyncio
- async def test_returns_none_on_fetch_error(self):
+ async def test_returns_both_none_on_fetch_error(self):
async def failing_fetch(url: str, sid: str) -> MockResponse:
raise ConnectionError("network failure")
mgr = RobotsTxtManager(failing_fetch)
- assert await mgr.get_crawl_delay("https://example.com/", "s1") is None
-
- @pytest.mark.asyncio
- async def test_returns_none_for_non_200_response(self):
- mgr = RobotsTxtManager(make_fetch_fn(status=404))
+ c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1")
- assert await mgr.get_crawl_delay("https://example.com/", "s1") is None
+ assert c_delay is None
+ assert r_rate is None
@pytest.mark.asyncio
- async def test_fractional_delay(self):
+ async def test_fractional_crawl_delay(self):
content = "User-agent: *\nCrawl-delay: 0.5"
mgr = RobotsTxtManager(make_fetch_fn(content=content))
- delay = await mgr.get_crawl_delay("https://example.com/", "s1")
+ c_delay, _ = await mgr.get_delay_directives("https://example.com/", "s1")
- assert delay == 0.5
+ assert c_delay == 0.5
@pytest.mark.asyncio
async def test_url_path_does_not_affect_result(self):
- """Any URL on the same domain should return the same delay."""
mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC))
- d1 = await mgr.get_crawl_delay("https://example.com/", "s1")
- d2 = await mgr.get_crawl_delay("https://example.com/deep/path/page.html", "s1")
+ r1 = await mgr.get_delay_directives("https://example.com/", "s1")
+ r2 = await mgr.get_delay_directives("https://example.com/deep/path/page.html", "s1")
- assert d1 == d2
-
-
-# ---------------------------------------------------------------------------
-# Tests: get_request_rate
-# ---------------------------------------------------------------------------
-
-
-class TestGetRequestRate:
- @pytest.mark.asyncio
- async def test_returns_tuple_when_set(self):
- mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_WITH_RATE))
-
- rate = await mgr.get_request_rate("https://example.com/", "s1")
-
- assert rate is not None
- assert isinstance(rate, tuple)
- assert len(rate) == 2
-
- @pytest.mark.asyncio
- async def test_tuple_contains_integers(self):
- mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_WITH_RATE))
-
- rate = await mgr.get_request_rate("https://example.com/", "s1")
-
- assert rate is not None
- requests, seconds = rate
- assert isinstance(requests, int)
- assert isinstance(seconds, int)
-
- @pytest.mark.asyncio
- async def test_returns_none_when_not_set(self):
- mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC))
-
- assert await mgr.get_request_rate("https://example.com/", "s1") is None
-
- @pytest.mark.asyncio
- async def test_returns_none_for_empty_robots(self):
- mgr = RobotsTxtManager(make_fetch_fn(content=""))
-
- assert await mgr.get_request_rate("https://example.com/", "s1") is None
-
- @pytest.mark.asyncio
- async def test_returns_none_on_fetch_error(self):
- async def failing_fetch(url: str, sid: str) -> MockResponse:
- raise ConnectionError("network failure")
-
- mgr = RobotsTxtManager(failing_fetch)
-
- assert await mgr.get_request_rate("https://example.com/", "s1") is None
-
- @pytest.mark.asyncio
- async def test_returns_none_for_non_200_response(self):
- mgr = RobotsTxtManager(make_fetch_fn(status=404))
-
- assert await mgr.get_request_rate("https://example.com/", "s1") is None
+ assert r1 == r2
# ---------------------------------------------------------------------------
@@ -296,8 +246,7 @@ class TestCachingBehaviour:
mgr = RobotsTxtManager(fetch_fn)
await mgr.can_fetch("https://example.com/", "s1")
- await mgr.get_crawl_delay("https://example.com/", "s1")
- await mgr.get_request_rate("https://example.com/", "s1")
+ await mgr.get_delay_directives("https://example.com/", "s1")
assert len(fetch_fn.calls) == 1
@@ -419,9 +368,9 @@ class TestEncoding:
return MockResponse(status=200, body=body, encoding="latin-1")
mgr = RobotsTxtManager(fetch_fn)
- delay = await mgr.get_crawl_delay("https://example.com/", "s1")
+ c_delay, _ = await mgr.get_delay_directives("https://example.com/", "s1")
- assert delay == 3.0
+ assert c_delay == 3.0
@pytest.mark.asyncio
async def test_bytes_body_decoded_correctly(self):
@@ -437,81 +386,6 @@ class TestEncoding:
assert await mgr.can_fetch("https://example.com/public/", "s1") is True
-# ---------------------------------------------------------------------------
-# Tests: clear_cache
-# ---------------------------------------------------------------------------
-
-
-class TestClearCache:
- @pytest.mark.asyncio
- async def test_clear_all_forces_refetch(self):
- fetch_fn = make_fetch_fn(content=ROBOTS_BASIC)
- mgr = RobotsTxtManager(fetch_fn)
-
- await mgr.can_fetch("https://example.com/", "s1")
- mgr.clear_cache()
- await mgr.can_fetch("https://example.com/", "s1")
-
- assert len(fetch_fn.calls) == 2
-
- @pytest.mark.asyncio
- async def test_clear_by_domain_only_invalidates_that_domain(self):
- fetch_fn = make_fetch_fn(content=ROBOTS_BASIC)
- mgr = RobotsTxtManager(fetch_fn)
-
- await mgr.can_fetch("https://example.com/", "s1")
- await mgr.can_fetch("https://other.com/", "s1")
- assert len(fetch_fn.calls) == 2
-
- mgr.clear_cache(domain="example.com")
-
- await mgr.can_fetch("https://example.com/", "s1") # refetched
- await mgr.can_fetch("https://other.com/", "s1") # still cached
-
- assert len(fetch_fn.calls) == 3
-
- @pytest.mark.asyncio
- async def test_clear_by_domain_invalidates_all_sessions(self):
- """Clearing a domain evicts the single shared cache entry for all sessions."""
- fetch_fn = make_fetch_fn(content=ROBOTS_BASIC)
- mgr = RobotsTxtManager(fetch_fn)
-
- await mgr.can_fetch("https://example.com/", "s1")
- assert len(fetch_fn.calls) == 1
-
- mgr.clear_cache(domain="example.com")
-
- await mgr.can_fetch("https://example.com/", "s1") # refetched — cache was cleared
- await mgr.can_fetch("https://example.com/", "s2") # hits the newly warm cache, no fetch
-
- assert len(fetch_fn.calls) == 2
-
- def test_clear_nonexistent_domain_does_not_raise(self):
- mgr = RobotsTxtManager(make_fetch_fn())
- mgr.clear_cache(domain="nevervisited.com") # should not raise
-
- def test_clear_empty_cache_does_not_raise(self):
- mgr = RobotsTxtManager(make_fetch_fn())
- mgr.clear_cache() # should not raise
-
- @pytest.mark.asyncio
- async def test_clear_all_empties_cache_completely(self):
- fetch_fn = make_fetch_fn(content=ROBOTS_BASIC)
- mgr = RobotsTxtManager(fetch_fn)
-
- await mgr.can_fetch("https://a.com/", "s1")
- await mgr.can_fetch("https://b.com/", "s1")
- await mgr.can_fetch("https://c.com/", "s1")
- assert len(fetch_fn.calls) == 3
-
- mgr.clear_cache()
-
- await mgr.can_fetch("https://a.com/", "s1")
- await mgr.can_fetch("https://b.com/", "s1")
- await mgr.can_fetch("https://c.com/", "s1")
-
- assert len(fetch_fn.calls) == 6
-
# ---------------------------------------------------------------------------
# Tests: concurrent access