Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix open_in_browser not adding a base tag without a head element (#7879)
Adrian23abf7e18f80
diff --git a/pyproject.toml b/pyproject.tomlindex 13aae01ac..81f3c062b 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -21,7 +21,7 @@ dependencies = ["queuelib>=1.6.1","service_identity>=24.2.0","tldextract",- "w3lib>=1.17.0",+ "w3lib>=2.1.0","zope.interface>=5.1.0",# Platform-specific dependencies'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"',diff --git a/scrapy/utils/response.py b/scrapy/utils/response.pyindex b3622c159..41bdac163 100644--- a/scrapy/utils/response.py+++ b/scrapy/utils/response.py@@ -60,15 +60,7 @@ def response_status_message(status: bytes | float | str) -> str:return f"{status_int} {to_unicode(message)}"-def _remove_html_comments(body: bytes) -> bytes:- start = body.find(b"<!--")- while start != -1:- end = body.find(b"-->", start + 1)- if end == -1:- return body[:start]- body = body[:start] + body[end + 3 :]- start = body.find(b"<!--")- return body+_DOCTYPE_RE = re.compile(rb"\s*<!doctype[^<>]*>", re.IGNORECASE)def open_in_browser(@@ -98,12 +90,17 @@ def open_in_browser(# circular importsfrom scrapy.http import HtmlResponse, TextResponse # noqa: PLC0415- # XXX: this implementation is a bit dirty and could be improvedbody = response.bodyif isinstance(response, HtmlResponse):- if b"<base" not in _remove_html_comments(body):- repl = rf'\g<0><base href="{response.url}">'- body = re.sub(rb"<head(?:[^<>]*?>)", to_bytes(repl), body, count=1)+ # Web browsers move a base tag that precedes the head element into it,+ # so the head element does not need to be found, which is not always+ # possible. The base tag must come after the doctype declaration, if+ # any, to keep the browser out of quirks mode. It takes precedence over+ # any base tag of the response, and matches it when there is one.+ doctype = _DOCTYPE_RE.match(body)+ index = doctype.end() if doctype else 0+ base_tag = to_bytes(f'<base href="{get_base_url(response)}">')+ body = body[:index] + base_tag + body[index:]ext = ".html"elif isinstance(response, TextResponse):ext = ".txt"diff --git a/tests/test_utils_response.py b/tests/test_utils_response.pyindex 3638c52b7..8269f0ad0 100644--- a/tests/test_utils_response.py+++ b/tests/test_utils_response.py@@ -9,7 +9,6 @@ import pytestfrom scrapy.http import HtmlResponse, Response, TextResponsefrom scrapy.utils.python import to_bytesfrom scrapy.utils.response import (- _remove_html_comments,get_base_url,get_meta_refresh,open_in_browser,@@ -169,6 +168,31 @@ def test_response_status_message():</html>""",id="Conditional comment",),+ pytest.param(+ b"""+ <html>+ <body><p>Hello world.</p></body>+ </html>""",+ id="No <head>",+ ),+ pytest.param(+ b"<p>Hello world.</p>",+ id="No <html>",+ ),+ pytest.param(+ b"""<!DOCTYPE html>+ <html>+ <head><title>Dummy</title></head>+ <body><p>Hello world.</p></body>+ </html>""",+ id="Doctype",+ ),+ pytest.param(+ b"""+ <!-- <head><base href="http://example.org"></head> -->+ <p>Hello world.</p>""",+ id="Only commented-out <head> and <base>",+ ),],)def test_inject_base_url(body: bytes) -> None:@@ -176,8 +200,13 @@ def test_inject_base_url(body: bytes) -> None:def check_base_url(burl):bbody = _read_browser_output(burl)- assert bbody.count(b'><base href="' + to_bytes(url) + b'">') == 1- assert b"<head" in bbody+ base_tag = b'<base href="' + to_bytes(url) + b'">'+ assert bbody.count(base_tag) == 1+ index = bbody.index(base_tag)+ # The base tag is not commented out.+ assert bbody.rfind(b"<!--", 0, index) <= bbody.rfind(b"-->", 0, index)+ # The base tag comes after the doctype declaration, if any.+ assert b"<!DOCTYPE" not in bbody[index:]return Trueresp = HtmlResponse(url, body=body)@@ -211,24 +240,6 @@ def test_open_in_browser_redos_head():_assert_open_in_browser_is_fast(b"<head\t" * 80_000)- ("input_body", "output_body"),- [- (b"a<!--", b"a"),- (b"a<!---->b", b"ab"),- (b"a<!--b-->c", b"ac"),- (b"a<!--b-->c<!--", b"ac"),- (b"a<!--b-->c<!--d", b"ac"),- (b"a<!--b-->c<!---->d", b"acd"),- (b"a<!--b--><!--c-->d", b"ad"),- (b"a<!-- <!-- inner --> -->b", b"a -->b"),- (b"<!-- <head>fake</head> --><head>real</head>", b"<head>real</head>"),- ],-)-def test_remove_html_comments(input_body: bytes, output_body: bytes) -> None:- assert _remove_html_comments(input_body) == output_body--def test_open_in_browser_preserves_html_comments():url = "http://www.example.com"body = (@@ -248,19 +259,24 @@ def test_open_in_browser_preserves_html_comments():assert open_in_browser(response, _openfunc=check)-def test_open_in_browser_does_not_inject_base_when_present():- url = "http://www.example.com"- body = (- b"<html>"- b'<head><base href="http://real.com"><title>T</title></head>'- b"<body>hi</body>"- b"</html>"- )+ ("base_tag", "expected_base_url"),+ [+ (b'<base href="http://real.com/">', b"http://real.com/"),+ (b'<BASE HREF="http://real.com/">', b"http://real.com/"),+ (b'<base href="/img/">', b"http://www.example.com/img/"),+ (b'<base target="_blank">', b"http://www.example.com/page.html"),+ ],+)+def test_open_in_browser_keeps_base_url_of_response(+ base_tag: bytes, expected_base_url: bytes+):+ url = "http://www.example.com/page.html"+ body = b"<html><head>" + base_tag + b"<title>T</title></head><body>hi</body></html>"def check(burl):bbody = _read_browser_output(burl)- assert b'<base href="' + to_bytes(url) + b'">' not in bbody- assert b'<base href="http://real.com">' in bbody+ assert bbody.startswith(b'<base href="' + expected_base_url + b'">')return Trueresponse = HtmlResponse(url, body=body)@@ -286,7 +302,7 @@ def test_open_in_browser_injects_base_when_only_in_comment():assert open_in_browser(response, _openfunc=check)-def test_open_in_browser_injects_base_at_real_head_not_commented_head():+def test_open_in_browser_injects_base_before_head_contents():url = "http://www.example.com"body = (b"<html>"diff --git a/tox.ini b/tox.iniindex e807317b3..d2a682415 100644--- a/tox.ini+++ b/tox.ini@@ -149,7 +149,7 @@ deps =pyOpenSSL==24.3.0queuelib==1.6.1service_identity==24.2.0- w3lib==1.17.0+ w3lib==2.1.0zope.interface==5.1.0{[test-requirements]deps}setenv =@@ -307,11 +307,6 @@ deps =pyOpenSSL==24.3.0queuelib==1.6.1service_identity==24.2.0- # w3lib 1.17 fails to import on PyPy 3.11 because its encoding regex uses- # an inline flag placement that Python 3.11 treats as an error: global- # flags not at the start of the expression. w3lib 1.18 stopped encoding []- # in URLs until 2.1.0 brought that behavior back. Tests for verbatim_url- # rely on that encoding.w3lib==2.1.0zope.interface==5.1.0commands =