Scrapy, a fast high-level web crawling & scraping framework for Python.
Remove duplicate code now handled by newer w3lib
nyov3787fec460f1
diff --git a/requirements.txt b/requirements.txtindex 368e9340b..db042cfcf 100644--- a/requirements.txt+++ b/requirements.txt@@ -2,7 +2,7 @@ Twisted>=10.0.0lxmlpyOpenSSLcssselect>=0.9-w3lib>=1.8.0+w3lib>=1.13.0queuelibsix>=1.5.2PyDispatcher>=2.0.5diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.pyindex da373eca2..72715dba7 100644--- a/scrapy/downloadermiddlewares/ajaxcrawl.py+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py@@ -8,7 +8,6 @@ from w3lib import htmlfrom scrapy.exceptions import NotConfiguredfrom scrapy.http import HtmlResponse-from scrapy.utils.response import _noscript_re, _script_relogger = logging.getLogger(__name__)@@ -89,8 +88,7 @@ def _has_ajaxcrawlable_meta(text):if 'content' not in text:return False- text = _script_re.sub(u'', text)- text = _noscript_re.sub(u'', text)- text = html.remove_comments(html.replace_entities(text))+ text = html.remove_tags_with_content(text, ('script', 'noscript'))+ text = html.replace_entities(text)+ text = html.remove_comments(text)return _ajax_crawlable_re.search(text) is not None-diff --git a/scrapy/utils/response.py b/scrapy/utils/response.pyindex 73db2641e..27ea5593e 100644--- a/scrapy/utils/response.py+++ b/scrapy/utils/response.py@@ -3,7 +3,6 @@ This module provides some useful functions for working withscrapy.http.Response objects"""import os-import reimport weakrefimport webbrowserimport tempfile@@ -31,17 +30,13 @@ def get_base_url(response):return _baseurl_cache[response]-_noscript_re = re.compile(u'<noscript>.*?</noscript>', re.IGNORECASE | re.DOTALL)-_script_re = re.compile(u'<script.*?>.*?</script>', re.IGNORECASE | re.DOTALL)_metaref_cache = weakref.WeakKeyDictionary()def get_meta_refresh(response):"""Parse the http-equiv refrsh parameter from the given response"""if response not in _metaref_cache:text = response.text[0:4096]- text = _noscript_re.sub(u'', text)- text = _script_re.sub(u'', text)_metaref_cache[response] = html.get_meta_refresh(text, response.url,- response.encoding)+ response.encoding, ignore_tags=('script', 'noscript'))return _metaref_cache[response]diff --git a/setup.py b/setup.pyindex d91bb6328..8387f81f4 100644--- a/setup.py+++ b/setup.py@@ -42,7 +42,7 @@ setup(],install_requires=['Twisted>=10.0.0',- 'w3lib>=1.8.0',+ 'w3lib>=1.13.0','queuelib','lxml','pyOpenSSL',