🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(spider robots): removing dead code
Karim shoairafaf68e7d59e
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.pyindex 7729d62..1924298 100644--- a/scrapling/spiders/engine.py+++ b/scrapling/spiders/engine.py@@ -96,7 +96,7 @@ class CrawlerEngine:return self._domain_delays[domain]# For domains covered by _prefetch_robots_txt this is a local parser read.- # Domains discovered mid-crawl (not in start_urls/allowed_domains) will fetch here.+ # Domains discovered mid-crawl (not in start_urls) will fetch here.c_delay, r_rate = await robots_manager.get_delay_directives(request.url, request.sid)delay = self.spider.download_delaydiff --git a/scrapling/spiders/robotstxt.py b/scrapling/spiders/robotstxt.pyindex 867698b..10dc7ab 100644--- a/scrapling/spiders/robotstxt.py+++ b/scrapling/spiders/robotstxt.py@@ -61,7 +61,6 @@ class RobotsTxtManager:"""Check if a URL can be fetched according to the domain's robots.txt.Handles:- - User-agent specific rules (e.g., User-agent: SpinarakBot)- Wildcard user-agent rules (User-agent: *)- Allow/Disallow directives with wildcards (e.g., /*.pdf$)- Allow directives that override Disallow (e.g., Allow: /admin/public-docs/)@@ -80,42 +79,6 @@ class RobotsTxtManager:parser = await self._get_parser(url, sid)return parser.can_fetch(url, "*")- async def get_crawl_delay(self, url: str, sid: str) -> Optional[float]:- """Get the crawl delay for this crawler.-- Uses the wildcard user-agent (*) to get the general crawl delay- that applies to all bots.-- Args:- url: Any URL on the domain to check- sid: Session ID for fetching robots.txt if not yet cached-- Returns:- The crawl delay in seconds, or None if not specified- """- parser = await self._get_parser(url, sid)- delay = parser.crawl_delay("*")- return float(delay) if delay is not None else None-- async def get_request_rate(self, url: str, sid: str) -> Optional[tuple[int, int]]:- """Get the request rate for this crawler.-- Uses the wildcard user-agent (*) to get the general request rate- that applies to all bots.-- Args:- url: Any URL on the domain to check- sid: Session ID for fetching robots.txt if not yet cached-- Returns:- A tuple of (requests, seconds) if specified, or None if not specified- """- parser = await self._get_parser(url, sid)- rate = parser.request_rate("*")- if rate is not None:- return (rate.requests, rate.seconds)- return None-async def get_delay_directives(self, url: str, sid: str) -> tuple[Optional[float], Optional[tuple[int, int]]]:"""Return both crawl-delay and request-rate in a single parser lookup.@@ -152,18 +115,3 @@ class RobotsTxtManager:async with create_task_group() as tg:for url in urls:tg.start_soon(self._get_parser, url, sid)-- def clear_cache(self, domain: Optional[str] = None) -> None:- """Clear the robots.txt cache.-- Note: the ``sid`` parameter was removed — the cache is now keyed by- domain only, so clearing a domain evicts all sessions at once.-- Args:- domain: If specified, only clear cache for this domain.- If None, clears the entire cache.- """- if domain is None:- self._cache.clear()- else:- self._cache.pop(domain, None)diff --git a/tests/spiders/test_engine.py b/tests/spiders/test_engine.pyindex e7382f8..74cf875 100644--- a/tests/spiders/test_engine.py+++ b/tests/spiders/test_engine.py@@ -940,19 +940,7 @@ class TestPrefetchRobotsTxt:return _fetch, calls@pytest.mark.asyncio- async def test_prefetch_uses_allowed_domains_when_set(self):- fetch_fn, calls = self._make_counting_fetch()- spider = MockSpider(allowed_domains={"a.com", "b.com"}, robots_txt_obey=True)- engine = _make_engine(spider=spider)- engine._robots_manager = RobotsTxtManager(fetch_fn)-- await engine._prefetch_robots_txt()-- fetched_domains = {Request(url).domain for url, _ in calls}- assert fetched_domains == {"a.com", "b.com"}-- @pytest.mark.asyncio- async def test_prefetch_falls_back_to_start_urls_when_no_allowed_domains(self):+ async def test_prefetch_uses_start_urls(self):fetch_fn, calls = self._make_counting_fetch()spider = MockSpider(robots_txt_obey=True, start_urls=["https://example.com/page1"])engine = _make_engine(spider=spider)diff --git a/tests/spiders/test_robotstxt.py b/tests/spiders/test_robotstxt.pyindex efb8328..c4050c9 100644--- a/tests/spiders/test_robotstxt.py+++ b/tests/spiders/test_robotstxt.py@@ -53,14 +53,6 @@ Request-rate: 1/10Disallow: /private/"""-ROBOTS_WITH_SITEMAP = """\-User-agent: *-Disallow:--Sitemap: https://example.com/sitemap.xml-Sitemap: https://example.com/sitemap2.xml-"""-ROBOTS_ALLOW_OVERRIDE = """\User-agent: *Disallow: /secret/@@ -157,121 +149,79 @@ class TestCanFetch:# ----------------------------------------------------------------------------# Tests: get_crawl_delay+# Tests: get_delay_directives# ----------------------------------------------------------------------------class TestGetCrawlDelay:+class TestGetDelayDirectives:@pytest.mark.asyncio- async def test_returns_float_when_set(self):+ async def test_returns_crawl_delay_when_set(self):mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC))- delay = await mgr.get_crawl_delay("https://example.com/", "s1")+ c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1")++ assert c_delay == 2.0+ assert isinstance(c_delay, float)+ assert r_rate is None++ @pytest.mark.asyncio+ async def test_returns_request_rate_when_set(self):+ mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_WITH_RATE))++ c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1")- assert delay == 2.0- assert isinstance(delay, float)+ assert c_delay is None+ assert r_rate is not None+ assert r_rate == (1, 1)@pytest.mark.asyncio- async def test_returns_none_when_not_set(self):+ async def test_returns_both_none_when_not_set(self):content = "User-agent: *\nDisallow: /admin/"mgr = RobotsTxtManager(make_fetch_fn(content=content))- assert await mgr.get_crawl_delay("https://example.com/", "s1") is None+ c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1")++ assert c_delay is None+ assert r_rate is None@pytest.mark.asyncio- async def test_returns_none_for_empty_robots(self):+ async def test_returns_both_none_for_empty_robots(self):mgr = RobotsTxtManager(make_fetch_fn(content=""))- assert await mgr.get_crawl_delay("https://example.com/", "s1") is None+ c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1")++ assert c_delay is None+ assert r_rate is None@pytest.mark.asyncio- async def test_returns_none_on_fetch_error(self):+ async def test_returns_both_none_on_fetch_error(self):async def failing_fetch(url: str, sid: str) -> MockResponse:raise ConnectionError("network failure")mgr = RobotsTxtManager(failing_fetch)- assert await mgr.get_crawl_delay("https://example.com/", "s1") is None-- @pytest.mark.asyncio- async def test_returns_none_for_non_200_response(self):- mgr = RobotsTxtManager(make_fetch_fn(status=404))+ c_delay, r_rate = await mgr.get_delay_directives("https://example.com/", "s1")- assert await mgr.get_crawl_delay("https://example.com/", "s1") is None+ assert c_delay is None+ assert r_rate is None@pytest.mark.asyncio- async def test_fractional_delay(self):+ async def test_fractional_crawl_delay(self):content = "User-agent: *\nCrawl-delay: 0.5"mgr = RobotsTxtManager(make_fetch_fn(content=content))- delay = await mgr.get_crawl_delay("https://example.com/", "s1")+ c_delay, _ = await mgr.get_delay_directives("https://example.com/", "s1")- assert delay == 0.5+ assert c_delay == 0.5@pytest.mark.asyncioasync def test_url_path_does_not_affect_result(self):- """Any URL on the same domain should return the same delay."""mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC))- d1 = await mgr.get_crawl_delay("https://example.com/", "s1")- d2 = await mgr.get_crawl_delay("https://example.com/deep/path/page.html", "s1")+ r1 = await mgr.get_delay_directives("https://example.com/", "s1")+ r2 = await mgr.get_delay_directives("https://example.com/deep/path/page.html", "s1")- assert d1 == d2---# ----------------------------------------------------------------------------# Tests: get_request_rate-# ------------------------------------------------------------------------------class TestGetRequestRate:- @pytest.mark.asyncio- async def test_returns_tuple_when_set(self):- mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_WITH_RATE))-- rate = await mgr.get_request_rate("https://example.com/", "s1")-- assert rate is not None- assert isinstance(rate, tuple)- assert len(rate) == 2-- @pytest.mark.asyncio- async def test_tuple_contains_integers(self):- mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_WITH_RATE))-- rate = await mgr.get_request_rate("https://example.com/", "s1")-- assert rate is not None- requests, seconds = rate- assert isinstance(requests, int)- assert isinstance(seconds, int)-- @pytest.mark.asyncio- async def test_returns_none_when_not_set(self):- mgr = RobotsTxtManager(make_fetch_fn(content=ROBOTS_BASIC))-- assert await mgr.get_request_rate("https://example.com/", "s1") is None-- @pytest.mark.asyncio- async def test_returns_none_for_empty_robots(self):- mgr = RobotsTxtManager(make_fetch_fn(content=""))-- assert await mgr.get_request_rate("https://example.com/", "s1") is None-- @pytest.mark.asyncio- async def test_returns_none_on_fetch_error(self):- async def failing_fetch(url: str, sid: str) -> MockResponse:- raise ConnectionError("network failure")-- mgr = RobotsTxtManager(failing_fetch)-- assert await mgr.get_request_rate("https://example.com/", "s1") is None-- @pytest.mark.asyncio- async def test_returns_none_for_non_200_response(self):- mgr = RobotsTxtManager(make_fetch_fn(status=404))-- assert await mgr.get_request_rate("https://example.com/", "s1") is None+ assert r1 == r2# ---------------------------------------------------------------------------@@ -296,8 +246,7 @@ class TestCachingBehaviour:mgr = RobotsTxtManager(fetch_fn)await mgr.can_fetch("https://example.com/", "s1")- await mgr.get_crawl_delay("https://example.com/", "s1")- await mgr.get_request_rate("https://example.com/", "s1")+ await mgr.get_delay_directives("https://example.com/", "s1")assert len(fetch_fn.calls) == 1@@ -419,9 +368,9 @@ class TestEncoding:return MockResponse(status=200, body=body, encoding="latin-1")mgr = RobotsTxtManager(fetch_fn)- delay = await mgr.get_crawl_delay("https://example.com/", "s1")+ c_delay, _ = await mgr.get_delay_directives("https://example.com/", "s1")- assert delay == 3.0+ assert c_delay == 3.0@pytest.mark.asyncioasync def test_bytes_body_decoded_correctly(self):@@ -437,81 +386,6 @@ class TestEncoding:assert await mgr.can_fetch("https://example.com/public/", "s1") is True-# ----------------------------------------------------------------------------# Tests: clear_cache-# ------------------------------------------------------------------------------class TestClearCache:- @pytest.mark.asyncio- async def test_clear_all_forces_refetch(self):- fetch_fn = make_fetch_fn(content=ROBOTS_BASIC)- mgr = RobotsTxtManager(fetch_fn)-- await mgr.can_fetch("https://example.com/", "s1")- mgr.clear_cache()- await mgr.can_fetch("https://example.com/", "s1")-- assert len(fetch_fn.calls) == 2-- @pytest.mark.asyncio- async def test_clear_by_domain_only_invalidates_that_domain(self):- fetch_fn = make_fetch_fn(content=ROBOTS_BASIC)- mgr = RobotsTxtManager(fetch_fn)-- await mgr.can_fetch("https://example.com/", "s1")- await mgr.can_fetch("https://other.com/", "s1")- assert len(fetch_fn.calls) == 2-- mgr.clear_cache(domain="example.com")-- await mgr.can_fetch("https://example.com/", "s1") # refetched- await mgr.can_fetch("https://other.com/", "s1") # still cached-- assert len(fetch_fn.calls) == 3-- @pytest.mark.asyncio- async def test_clear_by_domain_invalidates_all_sessions(self):- """Clearing a domain evicts the single shared cache entry for all sessions."""- fetch_fn = make_fetch_fn(content=ROBOTS_BASIC)- mgr = RobotsTxtManager(fetch_fn)-- await mgr.can_fetch("https://example.com/", "s1")- assert len(fetch_fn.calls) == 1-- mgr.clear_cache(domain="example.com")-- await mgr.can_fetch("https://example.com/", "s1") # refetched — cache was cleared- await mgr.can_fetch("https://example.com/", "s2") # hits the newly warm cache, no fetch-- assert len(fetch_fn.calls) == 2-- def test_clear_nonexistent_domain_does_not_raise(self):- mgr = RobotsTxtManager(make_fetch_fn())- mgr.clear_cache(domain="nevervisited.com") # should not raise-- def test_clear_empty_cache_does_not_raise(self):- mgr = RobotsTxtManager(make_fetch_fn())- mgr.clear_cache() # should not raise-- @pytest.mark.asyncio- async def test_clear_all_empties_cache_completely(self):- fetch_fn = make_fetch_fn(content=ROBOTS_BASIC)- mgr = RobotsTxtManager(fetch_fn)-- await mgr.can_fetch("https://a.com/", "s1")- await mgr.can_fetch("https://b.com/", "s1")- await mgr.can_fetch("https://c.com/", "s1")- assert len(fetch_fn.calls) == 3-- mgr.clear_cache()-- await mgr.can_fetch("https://a.com/", "s1")- await mgr.can_fetch("https://b.com/", "s1")- await mgr.can_fetch("https://c.com/", "s1")-- assert len(fetch_fn.calls) == 6-# ---------------------------------------------------------------------------# Tests: concurrent access