Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Closes #6342. Setting METAREFRESH_IGNORE_TAGS to [‘noscript’] by default

aisha-partha180bc9bad7ac
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 1abbc4968..3f90cf2ed 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -876,13 +876,13 @@ Whether the Meta Refresh middleware will be enabled.
METAREFRESH_IGNORE_TAGS
^^^^^^^^^^^^^^^^^^^^^^^
-Default: ``[]``
+Default: ``['noscript']``
Meta tags within these tags are ignored.
.. versionchanged:: 2.0
The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from
- ``['script', 'noscript']`` to ``[]``.
+ ``[]`` to ``['noscript']``.
.. setting:: METAREFRESH_MAXDELAY
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 2b3d95a0e..d7ac7ec35 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -239,7 +239,7 @@ MEMUSAGE_NOTIFY_MAIL = []
MEMUSAGE_WARNING_MB = 0
METAREFRESH_ENABLED = True
-METAREFRESH_IGNORE_TAGS = []
+METAREFRESH_IGNORE_TAGS = ["noscript"]
METAREFRESH_MAXDELAY = 100
NEWSPIDER_MODULE = ""
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 10b8ca9af..83ff25982 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -395,9 +395,8 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
"""content="0;URL='http://example.org/newpage'"></noscript>"""
)
rsp = HtmlResponse(req.url, body=body.encode())
- req2 = self.mw.process_response(req, rsp, self.spider)
- assert isinstance(req2, Request)
- self.assertEqual(req2.url, "http://example.org/newpage")
+ response = self.mw.process_response(req, rsp, self.spider)
+ assert isinstance(response, Response)
def test_ignore_tags_1_x_list(self):
"""Test that Scrapy 1.x behavior remains possible"""