Scrapy, a fast high-level web crawling & scraping framework for Python.
Closes #6342. Setting METAREFRESH_IGNORE_TAGS to [‘noscript’] by default
aisha-partha180bc9bad7ac
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rstindex 1abbc4968..3f90cf2ed 100644--- a/docs/topics/downloader-middleware.rst+++ b/docs/topics/downloader-middleware.rst@@ -876,13 +876,13 @@ Whether the Meta Refresh middleware will be enabled.METAREFRESH_IGNORE_TAGS^^^^^^^^^^^^^^^^^^^^^^^-Default: ``[]``+Default: ``['noscript']``Meta tags within these tags are ignored... versionchanged:: 2.0The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from- ``['script', 'noscript']`` to ``[]``.+ ``[]`` to ``['noscript']``... setting:: METAREFRESH_MAXDELAYdiff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 2b3d95a0e..d7ac7ec35 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -239,7 +239,7 @@ MEMUSAGE_NOTIFY_MAIL = []MEMUSAGE_WARNING_MB = 0METAREFRESH_ENABLED = True-METAREFRESH_IGNORE_TAGS = []+METAREFRESH_IGNORE_TAGS = ["noscript"]METAREFRESH_MAXDELAY = 100NEWSPIDER_MODULE = ""diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.pyindex 10b8ca9af..83ff25982 100644--- a/tests/test_downloadermiddleware_redirect.py+++ b/tests/test_downloadermiddleware_redirect.py@@ -395,9 +395,8 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):"""content="0;URL='http://example.org/newpage'"></noscript>""")rsp = HtmlResponse(req.url, body=body.encode())- req2 = self.mw.process_response(req, rsp, self.spider)- assert isinstance(req2, Request)- self.assertEqual(req2.url, "http://example.org/newpage")+ response = self.mw.process_response(req, rsp, self.spider)+ assert isinstance(response, Response)def test_ignore_tags_1_x_list(self):"""Test that Scrapy 1.x behavior remains possible"""