Scrapy, a fast high-level web crawling & scraping framework for Python.
fix open_in_browser() logic issue plus add new tests (#7506)
Ethan Kuodf2f3d708e1c
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.pyindex abb5f6a70..c068d9b1e 100644--- a/scrapy/utils/response.py+++ b/scrapy/utils/response.py@@ -97,8 +97,7 @@ def open_in_browser(# XXX: this implementation is a bit dirty and could be improvedbody = response.bodyif isinstance(response, HtmlResponse):- if b"<base" not in body:- _remove_html_comments(body)+ if b"<base" not in _remove_html_comments(body):repl = rf'\g<0><base href="{response.url}">'body = re.sub(rb"<head(?:[^<>]*?>)", to_bytes(repl), body, count=1)ext = ".html"diff --git a/tests/test_utils_response.py b/tests/test_utils_response.pyindex 381a9c3ff..e02bdfb69 100644--- a/tests/test_utils_response.py+++ b/tests/test_utils_response.py@@ -4,7 +4,7 @@ from urllib.parse import urlparseimport pytest-from scrapy.http import HtmlResponse, Response+from scrapy.http import HtmlResponse, Response, TextResponsefrom scrapy.utils.python import to_bytesfrom scrapy.utils.response import (_remove_html_comments,@@ -15,6 +15,13 @@ from scrapy.utils.response import ()+def _read_browser_output(burl: str):+ path = urlparse(burl).path+ if not path or not Path(path).exists():+ path = burl.replace("file://", "")+ return Path(path).read_bytes()++def test_open_in_browser():url = "http:///www.example.com/some/page.html"body = (@@ -22,10 +29,7 @@ def test_open_in_browser():)def browser_open(burl: str) -> bool:- path = urlparse(burl).path- if not path or not Path(path).exists():- path = burl.replace("file://", "")- bbody = Path(path).read_bytes()+ bbody = _read_browser_output(burl)assert b'<base href="' + to_bytes(url) + b'">' in bbodyreturn True@@ -169,10 +173,7 @@ def test_inject_base_url(body: bytes) -> None:url = "http://www.example.com"def check_base_url(burl):- path = urlparse(burl).path- if not path or not Path(path).exists():- path = burl.replace("file://", "")- bbody = Path(path).read_bytes()+ bbody = _read_browser_output(burl)assert bbody.count(b'><base href="' + to_bytes(url) + b'">') == 1assert b"<head" in bbodyreturn True@@ -219,7 +220,104 @@ def test_open_in_browser_redos_head():(b"a<!--b-->c<!--d", b"ac"),(b"a<!--b-->c<!---->d", b"acd"),(b"a<!--b--><!--c-->d", b"ad"),+ (b"a<!-- <!-- inner --> -->b", b"a -->b"),+ (b"<!-- <head>fake</head> --><head>real</head>", b"<head>real</head>"),],)def test_remove_html_comments(input_body, output_body):assert _remove_html_comments(input_body) == output_body+++def test_open_in_browser_preserves_html_comments():+ url = "http://www.example.com"+ body = (+ b"<html>"+ b"<!-- preserved comment -->"+ b"<head><title>Real</title></head>"+ b"<body>content</body>"+ b"</html>"+ )++ def check(burl):+ bbody = _read_browser_output(burl)+ assert b"<!-- preserved comment -->" in bbody+ return True++ response = HtmlResponse(url, body=body)+ assert open_in_browser(response, _openfunc=check)+++def test_open_in_browser_does_not_inject_base_when_present():+ url = "http://www.example.com"+ body = (+ b"<html>"+ b'<head><base href="http://real.com"><title>T</title></head>'+ b"<body>hi</body>"+ b"</html>"+ )++ def check(burl):+ bbody = _read_browser_output(burl)+ assert b'<base href="' + to_bytes(url) + b'">' not in bbody+ assert b'<base href="http://real.com">' in bbody+ return True++ response = HtmlResponse(url, body=body)+ assert open_in_browser(response, _openfunc=check)+++def test_open_in_browser_injects_base_when_only_in_comment():+ url = "http://www.example.com"+ body = (+ b"<html>"+ b"<!-- <base href='http://other.com'> -->"+ b"<head><title>Real</title></head>"+ b"<body>content</body>"+ b"</html>"+ )++ def check(burl):+ bbody = _read_browser_output(burl)+ assert b'<base href="' + to_bytes(url) + b'">' in bbody+ return True++ response = HtmlResponse(url, body=body)+ assert open_in_browser(response, _openfunc=check)+++def test_open_in_browser_injects_base_at_real_head_not_commented_head():+ url = "http://www.example.com"+ body = (+ b"<html>"+ b"<!--<head>comment head</head>-->"+ b"<head><title>Actual</title></head>"+ b"<body>hello</body>"+ b"</html>"+ )++ def check(burl):+ bbody = _read_browser_output(burl)+ assert bbody.count(b'<base href="' + to_bytes(url) + b'">') == 1+ base_pos = bbody.find(b'<base href="' + to_bytes(url) + b'">')+ title_pos = bbody.find(b"<title>Actual</title>")+ assert base_pos < title_pos+ return True++ response = HtmlResponse(url, body=body)+ assert open_in_browser(response, _openfunc=check)+++def test_open_in_browser_text_response_uses_txt_extension():+ response = TextResponse("http://www.example.com", body=b"plain text content")++ def check(burl):+ assert burl.endswith(".txt")+ return True++ assert open_in_browser(response, _openfunc=check)+++def test_open_in_browser_raises_for_unsupported_response_type():+ response = Response("http://www.example.com", body=b"binary")+ with pytest.raises(TypeError):+ open_in_browser(response, _openfunc=lambda _: True)