Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

feat: robots parser test dynamic port

Krzysztof Oliprabe00ff2db649
diff --git a/tests/general/test_robot_parser.py b/tests/general/test_robot_parser.py
index 6ed20fbc..6690339b 100644
--- a/tests/general/test_robot_parser.py
+++ b/tests/general/test_robot_parser.py
@@ -134,14 +134,15 @@ Allow: /public/
query_app.router.add_get('/robots.txt', query_robots)
query_runner = web.AppRunner(query_app)
await query_runner.setup()
- query_site = web.TCPSite(query_runner, 'localhost', 8081)
+ query_site = web.TCPSite(query_runner, '127.0.0.1', 0)
await query_site.start()
- return query_runner
+ query_port = query_runner.addresses[0][1]
+ return query_runner, query_port
- query_runner = await start_query_server()
+ query_runner, query_port = await start_query_server()
try:
print("\n4b. Testing query-string robots.txt rules...")
- query_base = "http://localhost:8081"
+ query_base = f"http://127.0.0.1:{query_port}"
result = await parser.can_fetch(f"{query_base}/", "bot")
print(f"Plain root (/): {'allowed' if result else 'denied'}")
diff --git a/tests/unit/test_robots_query_rules.py b/tests/unit/test_robots_query_rules.py
index 90b1c61b..e88e57bb 100644
--- a/tests/unit/test_robots_query_rules.py
+++ b/tests/unit/test_robots_query_rules.py
@@ -43,14 +43,10 @@ def test_preserve_bare_query_line_rewriting(line, expected):
def test_preserve_bare_query_keeps_document_structure():
"""Untouched lines, blank lines and ordering survive verbatim.
- The rewrite is splitlines()-based, so a trailing newline is not preserved.
- That is harmless: the only caller re-splits the result immediately.
+ Compared line by line, since the only caller re-splits the result
+ immediately; whether a trailing newline survives is deliberately unpinned.
"""
source = "User-agent: *\nDisallow: /private/\n\nDisallow: /*?\nAllow: /public/\n"
- assert _preserve_bare_query(source) == (
- "User-agent: *\nDisallow: /private/\n\nDisallow: /*?*\nAllow: /public/"
- )
- # What the caller actually consumes is unaffected by the missing newline.
assert _preserve_bare_query(source).splitlines() == [
"User-agent: *", "Disallow: /private/", "", "Disallow: /*?*", "Allow: /public/",
]