Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix get_meta_refresh to correctly resolve relative URLs using <base> tag (#7047)

Ashar Khand239fcf936bd
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index b6550432c..8b2417bfb 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -48,7 +48,7 @@ def get_meta_refresh(
if response not in _metaref_cache:
text = response.text[0:4096]
_metaref_cache[response] = html.get_meta_refresh(
- text, response.url, response.encoding, ignore_tags=ignore_tags
+ text, get_base_url(response), response.encoding, ignore_tags=ignore_tags
)
return _metaref_cache[response]
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index 179ca49e4..0aeb5594f 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -67,9 +67,23 @@ if(!checkCookies()){
</script>
""",
)
+ r4 = HtmlResponse(
+ "http://www.example.com",
+ body=b"""
+ <html>
+ <head><title>Dummy</title>
+ <base href="http://www.another-domain.com/base/path/">
+ <meta http-equiv="refresh" content="5;url=target.html"</head>
+ <body>blahablsdfsal&amp;</body>
+ </html>""",
+ )
assert get_meta_refresh(r1) == (5.0, "http://example.org/newpage")
assert get_meta_refresh(r2) == (None, None)
assert get_meta_refresh(r3) == (None, None)
+ assert get_meta_refresh(r4) == (
+ 5.0,
+ "http://www.another-domain.com/base/path/target.html",
+ )
def test_get_base_url():