Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(crawler): ignore absurd robots.txt crawl-delay values (#1787)

Jinguang Gufaf0096c4b13
diff --git a/deeptutor/services/web_source/robots.py b/deeptutor/services/web_source/robots.py
index 2eed6d8dc..650b9ccc4 100644
--- a/deeptutor/services/web_source/robots.py
+++ b/deeptutor/services/web_source/robots.py
@@ -20,6 +20,10 @@ from deeptutor.tools.web_fetch import (
)
DEFAULT_REQUEST_INTERVAL_S = 0.125
+#: A hostile robots.txt can otherwise park a crawl (and the shared per-host
+#: pace lock) on an arbitrarily long sleep; delays beyond a minute are treated
+#: as "no directive" and the default interval applies instead.
+MAX_HONORED_CRAWL_DELAY_S = 60.0
_UNRESERVED = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~")
@@ -90,7 +94,11 @@ def parse_robots_txt(raw: str) -> RobotsPolicy:
delay = float(value)
except ValueError:
continue
- if math.isfinite(delay) and delay >= 0:
+ # Ignore absurd delays instead of honoring them: a hostile
+ # robots.txt could otherwise park every request to this host
+ # on one unbounded sleep (RFC 9309 lets crawlers cap
+ # unreasonable values).
+ if math.isfinite(delay) and 0 <= delay <= MAX_HONORED_CRAWL_DELAY_S:
delays.append(delay)
groups.append((agents, rules, delays))
product = DEFAULT_USER_AGENT.split("/", 1)[0].lower()
diff --git a/tests/services/test_crawler_robots.py b/tests/services/test_crawler_robots.py
index b38b1cfb8..24903fe93 100644
--- a/tests/services/test_crawler_robots.py
+++ b/tests/services/test_crawler_robots.py
@@ -158,3 +158,19 @@ async def test_concurrent_requests_fetch_robots_once_and_keep_minimum_pacing(cra
assert all(
b[1] - a[1] >= robots.DEFAULT_REQUEST_INTERVAL_S for a, b in zip(requested, requested[1:])
)
+
+
+def test_absurd_crawl_delay_is_ignored_not_honored() -> None:
+ """A hostile robots.txt can otherwise park every request to this host on
+ one unbounded sleep; RFC 9309 lets crawlers cap unreasonable values."""
+ policy = robots.parse_robots_txt(
+ """
+User-agent: *
+Disallow: /private
+Crawl-delay: 1000000
+User-agent: deeptutor
+Crawl-delay: 45
+"""
+ )
+ assert policy.crawl_delay_s == 45
+ assert policy.permits("https://example.com/docs/open")