DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(crawler): ignore absurd robots.txt crawl-delay values (#1787)
Jinguang Gufaf0096c4b13
diff --git a/deeptutor/services/web_source/robots.py b/deeptutor/services/web_source/robots.pyindex 2eed6d8dc..650b9ccc4 100644--- a/deeptutor/services/web_source/robots.py+++ b/deeptutor/services/web_source/robots.py@@ -20,6 +20,10 @@ from deeptutor.tools.web_fetch import ()DEFAULT_REQUEST_INTERVAL_S = 0.125+#: A hostile robots.txt can otherwise park a crawl (and the shared per-host+#: pace lock) on an arbitrarily long sleep; delays beyond a minute are treated+#: as "no directive" and the default interval applies instead.+MAX_HONORED_CRAWL_DELAY_S = 60.0_UNRESERVED = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~")@@ -90,7 +94,11 @@ def parse_robots_txt(raw: str) -> RobotsPolicy:delay = float(value)except ValueError:continue- if math.isfinite(delay) and delay >= 0:+ # Ignore absurd delays instead of honoring them: a hostile+ # robots.txt could otherwise park every request to this host+ # on one unbounded sleep (RFC 9309 lets crawlers cap+ # unreasonable values).+ if math.isfinite(delay) and 0 <= delay <= MAX_HONORED_CRAWL_DELAY_S:delays.append(delay)groups.append((agents, rules, delays))product = DEFAULT_USER_AGENT.split("/", 1)[0].lower()diff --git a/tests/services/test_crawler_robots.py b/tests/services/test_crawler_robots.pyindex b38b1cfb8..24903fe93 100644--- a/tests/services/test_crawler_robots.py+++ b/tests/services/test_crawler_robots.py@@ -158,3 +158,19 @@ async def test_concurrent_requests_fetch_robots_once_and_keep_minimum_pacing(craassert all(b[1] - a[1] >= robots.DEFAULT_REQUEST_INTERVAL_S for a, b in zip(requested, requested[1:]))+++def test_absurd_crawl_delay_is_ignored_not_honored() -> None:+ """A hostile robots.txt can otherwise park every request to this host on+ one unbounded sleep; RFC 9309 lets crawlers cap unreasonable values."""+ policy = robots.parse_robots_txt(+ """+User-agent: *+Disallow: /private+Crawl-delay: 1000000+User-agent: deeptutor+Crawl-delay: 45+"""+ )+ assert policy.crawl_delay_s == 45+ assert policy.permits("https://example.com/docs/open")