Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Remove monkeypatches module, E402 flake8 code

Eugenio Lacuestaa915af2e4592
diff --git a/pytest.ini b/pytest.ini
index 0e289866c..0797e8ac8 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -27,18 +27,16 @@ flake8-ignore =
# Exclude files that are meant to provide top-level imports
# E402: Module level import not at top of file
# F401: Module imported but unused
+ scrapy/__init__.py E402
scrapy/core/downloader/handlers/http.py F401
scrapy/http/__init__.py F401
scrapy/linkextractors/__init__.py E402 F401
scrapy/spiders/__init__.py E402 F401
# Issues pending a review:
- scrapy/__init__.py E402
scrapy/selector/__init__.py F403
- scrapy/spiders/__init__.py E402
scrapy/utils/http.py F403
scrapy/utils/markup.py F403
scrapy/utils/multipart.py F403
scrapy/utils/url.py F403 F405
tests/test_loader.py E741
- tests/test_webclient.py E402
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index fb8357f3c..e791deaa6 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -2,33 +2,11 @@
Scrapy - a web crawling and web scraping framework written for Python
"""
-__all__ = ['__version__', 'version_info', 'twisted_version',
- 'Spider', 'Request', 'FormRequest', 'Selector', 'Item', 'Field']
-
-# Scrapy version
import pkgutil
-__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip()
-version_info = tuple(int(v) if v.isdigit() else v
- for v in __version__.split('.'))
-del pkgutil
-
-# Check minimum required Python version
import sys
-if sys.version_info < (3, 5):
- print("Scrapy %s requires Python 3.5" % __version__)
- sys.exit(1)
-
-# Ignore noisy twisted deprecation warnings
import warnings
-warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
-del warnings
-
-# Apply monkey patches to fix issues in external libraries
-from scrapy import _monkeypatches
-del _monkeypatches
from twisted import version as _txv
-twisted_version = (_txv.major, _txv.minor, _txv.micro)
# Declare top-level shortcuts
from scrapy.spiders import Spider
@@ -36,4 +14,29 @@ from scrapy.http import Request, FormRequest
from scrapy.selector import Selector
from scrapy.item import Item, Field
+
+__all__ = [
+ '__version__', 'version_info', 'twisted_version', 'Spider',
+ 'Request', 'FormRequest', 'Selector', 'Item', 'Field',
+]
+
+
+# Scrapy and Twisted versions
+__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip()
+version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split('.'))
+twisted_version = (_txv.major, _txv.minor, _txv.micro)
+
+
+# Check minimum required Python version
+if sys.version_info < (3, 5):
+ print("Scrapy %s requires Python 3.5" % __version__)
+ sys.exit(1)
+
+
+# Ignore noisy twisted deprecation warnings
+warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
+
+
+del pkgutil
del sys
+del warnings
diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py
deleted file mode 100644
index f74f89bda..000000000
--- a/scrapy/_monkeypatches.py
+++ /dev/null
@@ -1,11 +0,0 @@
-import copyreg
-
-
-# Undo what Twisted's perspective broker adds to pickle register
-# to prevent bugs like Twisted#7989 while serializing requests
-import twisted.persisted.styles # NOQA
-# Remove only entries with twisted serializers for non-twisted types.
-for k, v in frozenset(copyreg.dispatch_table.items()):
- if not str(getattr(k, '__module__', '')).startswith('twisted') \
- and str(getattr(v, '__module__', '')).startswith('twisted'):
- copyreg.dispatch_table.pop(k)
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index ae019c70f..984a5c4e1 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -133,4 +133,4 @@ class FilteringLinkExtractor:
# Top-level imports
-from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor # noqa: F401
+from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index ba1c866f8..02f87f8f5 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -110,6 +110,6 @@ class Spider(object_ref):
# Top-level imports
-from scrapy.spiders.crawl import CrawlSpider, Rule # noqa: F401
-from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider # noqa: F401
-from scrapy.spiders.sitemap import SitemapSpider # noqa: F401
+from scrapy.spiders.crawl import CrawlSpider, Rule
+from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider
+from scrapy.spiders.sitemap import SitemapSpider