Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
feat: robots parser test dynamic port
Krzysztof Oliprabe00ff2db649
diff --git a/tests/general/test_robot_parser.py b/tests/general/test_robot_parser.pyindex 6ed20fbc..6690339b 100644--- a/tests/general/test_robot_parser.py+++ b/tests/general/test_robot_parser.py@@ -134,14 +134,15 @@ Allow: /public/query_app.router.add_get('/robots.txt', query_robots)query_runner = web.AppRunner(query_app)await query_runner.setup()- query_site = web.TCPSite(query_runner, 'localhost', 8081)+ query_site = web.TCPSite(query_runner, '127.0.0.1', 0)await query_site.start()- return query_runner+ query_port = query_runner.addresses[0][1]+ return query_runner, query_port- query_runner = await start_query_server()+ query_runner, query_port = await start_query_server()try:print("\n4b. Testing query-string robots.txt rules...")- query_base = "http://localhost:8081"+ query_base = f"http://127.0.0.1:{query_port}"result = await parser.can_fetch(f"{query_base}/", "bot")print(f"Plain root (/): {'allowed' if result else 'denied'}")diff --git a/tests/unit/test_robots_query_rules.py b/tests/unit/test_robots_query_rules.pyindex 90b1c61b..e88e57bb 100644--- a/tests/unit/test_robots_query_rules.py+++ b/tests/unit/test_robots_query_rules.py@@ -43,14 +43,10 @@ def test_preserve_bare_query_line_rewriting(line, expected):def test_preserve_bare_query_keeps_document_structure():"""Untouched lines, blank lines and ordering survive verbatim.- The rewrite is splitlines()-based, so a trailing newline is not preserved.- That is harmless: the only caller re-splits the result immediately.+ Compared line by line, since the only caller re-splits the result+ immediately; whether a trailing newline survives is deliberately unpinned."""source = "User-agent: *\nDisallow: /private/\n\nDisallow: /*?\nAllow: /public/\n"- assert _preserve_bare_query(source) == (- "User-agent: *\nDisallow: /private/\n\nDisallow: /*?*\nAllow: /public/"- )- # What the caller actually consumes is unaffected by the missing newline.assert _preserve_bare_query(source).splitlines() == ["User-agent: *", "Disallow: /private/", "", "Disallow: /*?*", "Allow: /public/",]