Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix get_meta_refresh to correctly resolve relative URLs using <base> tag (#7047)
Ashar Khand239fcf936bd
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.pyindex b6550432c..8b2417bfb 100644--- a/scrapy/utils/response.py+++ b/scrapy/utils/response.py@@ -48,7 +48,7 @@ def get_meta_refresh(if response not in _metaref_cache:text = response.text[0:4096]_metaref_cache[response] = html.get_meta_refresh(- text, response.url, response.encoding, ignore_tags=ignore_tags+ text, get_base_url(response), response.encoding, ignore_tags=ignore_tags)return _metaref_cache[response]diff --git a/tests/test_utils_response.py b/tests/test_utils_response.pyindex 179ca49e4..0aeb5594f 100644--- a/tests/test_utils_response.py+++ b/tests/test_utils_response.py@@ -67,9 +67,23 @@ if(!checkCookies()){</script>""",)+ r4 = HtmlResponse(+ "http://www.example.com",+ body=b"""+ <html>+ <head><title>Dummy</title>+ <base href="http://www.another-domain.com/base/path/">+ <meta http-equiv="refresh" content="5;url=target.html"</head>+ <body>blahablsdfsal&</body>+ </html>""",+ )assert get_meta_refresh(r1) == (5.0, "http://example.org/newpage")assert get_meta_refresh(r2) == (None, None)assert get_meta_refresh(r3) == (None, None)+ assert get_meta_refresh(r4) == (+ 5.0,+ "http://www.another-domain.com/base/path/target.html",+ )def test_get_base_url():