Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix open_in_browser not adding a base tag without a head element (#7879)

Adrian23abf7e18f80
diff --git a/pyproject.toml b/pyproject.toml
index 13aae01ac..81f3c062b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -21,7 +21,7 @@ dependencies = [
"queuelib>=1.6.1",
"service_identity>=24.2.0",
"tldextract",
- "w3lib>=1.17.0",
+ "w3lib>=2.1.0",
"zope.interface>=5.1.0",
# Platform-specific dependencies
'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"',
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index b3622c159..41bdac163 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -60,15 +60,7 @@ def response_status_message(status: bytes | float | str) -> str:
return f"{status_int} {to_unicode(message)}"
-def _remove_html_comments(body: bytes) -> bytes:
- start = body.find(b"<!--")
- while start != -1:
- end = body.find(b"-->", start + 1)
- if end == -1:
- return body[:start]
- body = body[:start] + body[end + 3 :]
- start = body.find(b"<!--")
- return body
+_DOCTYPE_RE = re.compile(rb"\s*<!doctype[^<>]*>", re.IGNORECASE)
def open_in_browser(
@@ -98,12 +90,17 @@ def open_in_browser(
# circular imports
from scrapy.http import HtmlResponse, TextResponse # noqa: PLC0415
- # XXX: this implementation is a bit dirty and could be improved
body = response.body
if isinstance(response, HtmlResponse):
- if b"<base" not in _remove_html_comments(body):
- repl = rf'\g<0><base href="{response.url}">'
- body = re.sub(rb"<head(?:[^<>]*?>)", to_bytes(repl), body, count=1)
+ # Web browsers move a base tag that precedes the head element into it,
+ # so the head element does not need to be found, which is not always
+ # possible. The base tag must come after the doctype declaration, if
+ # any, to keep the browser out of quirks mode. It takes precedence over
+ # any base tag of the response, and matches it when there is one.
+ doctype = _DOCTYPE_RE.match(body)
+ index = doctype.end() if doctype else 0
+ base_tag = to_bytes(f'<base href="{get_base_url(response)}">')
+ body = body[:index] + base_tag + body[index:]
ext = ".html"
elif isinstance(response, TextResponse):
ext = ".txt"
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index 3638c52b7..8269f0ad0 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -9,7 +9,6 @@ import pytest
from scrapy.http import HtmlResponse, Response, TextResponse
from scrapy.utils.python import to_bytes
from scrapy.utils.response import (
- _remove_html_comments,
get_base_url,
get_meta_refresh,
open_in_browser,
@@ -169,6 +168,31 @@ def test_response_status_message():
</html>""",
id="Conditional comment",
),
+ pytest.param(
+ b"""
+ <html>
+ <body><p>Hello world.</p></body>
+ </html>""",
+ id="No <head>",
+ ),
+ pytest.param(
+ b"<p>Hello world.</p>",
+ id="No <html>",
+ ),
+ pytest.param(
+ b"""<!DOCTYPE html>
+ <html>
+ <head><title>Dummy</title></head>
+ <body><p>Hello world.</p></body>
+ </html>""",
+ id="Doctype",
+ ),
+ pytest.param(
+ b"""
+ <!-- <head><base href="http://example.org"></head> -->
+ <p>Hello world.</p>""",
+ id="Only commented-out <head> and <base>",
+ ),
],
)
def test_inject_base_url(body: bytes) -> None:
@@ -176,8 +200,13 @@ def test_inject_base_url(body: bytes) -> None:
def check_base_url(burl):
bbody = _read_browser_output(burl)
- assert bbody.count(b'><base href="' + to_bytes(url) + b'">') == 1
- assert b"<head" in bbody
+ base_tag = b'<base href="' + to_bytes(url) + b'">'
+ assert bbody.count(base_tag) == 1
+ index = bbody.index(base_tag)
+ # The base tag is not commented out.
+ assert bbody.rfind(b"<!--", 0, index) <= bbody.rfind(b"-->", 0, index)
+ # The base tag comes after the doctype declaration, if any.
+ assert b"<!DOCTYPE" not in bbody[index:]
return True
resp = HtmlResponse(url, body=body)
@@ -211,24 +240,6 @@ def test_open_in_browser_redos_head():
_assert_open_in_browser_is_fast(b"<head\t" * 80_000)
- ("input_body", "output_body"),
- [
- (b"a<!--", b"a"),
- (b"a<!---->b", b"ab"),
- (b"a<!--b-->c", b"ac"),
- (b"a<!--b-->c<!--", b"ac"),
- (b"a<!--b-->c<!--d", b"ac"),
- (b"a<!--b-->c<!---->d", b"acd"),
- (b"a<!--b--><!--c-->d", b"ad"),
- (b"a<!-- <!-- inner --> -->b", b"a -->b"),
- (b"<!-- <head>fake</head> --><head>real</head>", b"<head>real</head>"),
- ],
-)
-def test_remove_html_comments(input_body: bytes, output_body: bytes) -> None:
- assert _remove_html_comments(input_body) == output_body
-
-
def test_open_in_browser_preserves_html_comments():
url = "http://www.example.com"
body = (
@@ -248,19 +259,24 @@ def test_open_in_browser_preserves_html_comments():
assert open_in_browser(response, _openfunc=check)
-def test_open_in_browser_does_not_inject_base_when_present():
- url = "http://www.example.com"
- body = (
- b"<html>"
- b'<head><base href="http://real.com"><title>T</title></head>'
- b"<body>hi</body>"
- b"</html>"
- )
+ ("base_tag", "expected_base_url"),
+ [
+ (b'<base href="http://real.com/">', b"http://real.com/"),
+ (b'<BASE HREF="http://real.com/">', b"http://real.com/"),
+ (b'<base href="/img/">', b"http://www.example.com/img/"),
+ (b'<base target="_blank">', b"http://www.example.com/page.html"),
+ ],
+)
+def test_open_in_browser_keeps_base_url_of_response(
+ base_tag: bytes, expected_base_url: bytes
+):
+ url = "http://www.example.com/page.html"
+ body = b"<html><head>" + base_tag + b"<title>T</title></head><body>hi</body></html>"
def check(burl):
bbody = _read_browser_output(burl)
- assert b'<base href="' + to_bytes(url) + b'">' not in bbody
- assert b'<base href="http://real.com">' in bbody
+ assert bbody.startswith(b'<base href="' + expected_base_url + b'">')
return True
response = HtmlResponse(url, body=body)
@@ -286,7 +302,7 @@ def test_open_in_browser_injects_base_when_only_in_comment():
assert open_in_browser(response, _openfunc=check)
-def test_open_in_browser_injects_base_at_real_head_not_commented_head():
+def test_open_in_browser_injects_base_before_head_contents():
url = "http://www.example.com"
body = (
b"<html>"
diff --git a/tox.ini b/tox.ini
index e807317b3..d2a682415 100644
--- a/tox.ini
+++ b/tox.ini
@@ -149,7 +149,7 @@ deps =
pyOpenSSL==24.3.0
queuelib==1.6.1
service_identity==24.2.0
- w3lib==1.17.0
+ w3lib==2.1.0
zope.interface==5.1.0
{[test-requirements]deps}
setenv =
@@ -307,11 +307,6 @@ deps =
pyOpenSSL==24.3.0
queuelib==1.6.1
service_identity==24.2.0
- # w3lib 1.17 fails to import on PyPy 3.11 because its encoding regex uses
- # an inline flag placement that Python 3.11 treats as an error: global
- # flags not at the start of the expression. w3lib 1.18 stopped encoding []
- # in URLs until 2.1.0 brought that behavior back. Tests for verbatim_url
- # rely on that encoding.
w3lib==2.1.0
zope.interface==5.1.0
commands =