Scrapy, a fast high-level web crawling & scraping framework for Python.
Simplify some more imports.
Andrey Rakhmatullin92ffd2f249ec
diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.pyindex 2c35d1b90..2212d9688 100644--- a/scrapy/downloadermiddlewares/httpproxy.py+++ b/scrapy/downloadermiddlewares/httpproxy.py@@ -1,10 +1,7 @@import base64from six.moves.urllib.parse import unquote, urlunparsefrom six.moves.urllib.request import getproxies, proxy_bypass-try:- from urllib2 import _parse_proxy-except ImportError:- from urllib.request import _parse_proxy+from urllib.request import _parse_proxyfrom scrapy.exceptions import NotConfiguredfrom scrapy.utils.httpobj import urlparse_cacheddiff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.pyindex 2acdc8093..02c625acc 100644--- a/scrapy/loader/processors.py+++ b/scrapy/loader/processors.py@@ -3,10 +3,7 @@ This module provides some commonly used processors for Item Loaders.See documentation in docs/topics/loaders.rst"""-try:- from collections import ChainMap-except ImportError:- from scrapy.utils.datatypes import MergeDict as ChainMap+from collections import ChainMapfrom scrapy.utils.misc import arg_to_iterfrom scrapy.loader.common import wrap_loader_contextdiff --git a/tests/__init__.py b/tests/__init__.pyindex a54367f8c..12ce79fa9 100644--- a/tests/__init__.py+++ b/tests/__init__.py@@ -21,11 +21,6 @@ if 'COV_CORE_CONFIG' in os.environ:os.environ['COV_CORE_CONFIG'] = os.path.join(_sourceroot,os.environ['COV_CORE_CONFIG'])-try:- import unittest.mock as mock-except ImportError:- import mock-tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)),'sample_data')diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 109469503..6090998d4 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -2,11 +2,8 @@ import osimport siximport shutilimport tempfile+from unittest import mockimport contextlib-try:- from unittest import mock-except ImportError:- import mockfrom testfixtures import LogCapturefrom twisted.trial import unittestdiff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.pyindex 03564e748..6b9a5bee8 100644--- a/tests/test_downloadermiddleware.py+++ b/tests/test_downloadermiddleware.py@@ -1,3 +1,5 @@+from unittest import mock+from twisted.trial.unittest import TestCasefrom twisted.python.failure import Failure@@ -7,7 +9,6 @@ from scrapy.exceptions import _InvalidOutputfrom scrapy.core.downloader.middleware import DownloaderMiddlewareManagerfrom scrapy.utils.test import get_crawlerfrom scrapy.utils.python import to_bytes-from tests import mockclass ManagerTestCase(TestCase):diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.pyindex fbc46cba4..8266bf35f 100644--- a/tests/test_downloadermiddleware_robotstxt.py+++ b/tests/test_downloadermiddleware_robotstxt.py@@ -1,5 +1,8 @@# -*- coding: utf-8 -*-from __future__ import absolute_import++from unittest import mock+from twisted.internet import reactor, errorfrom twisted.internet.defer import Deferred, DeferredList, maybeDeferredfrom twisted.python import failure@@ -9,7 +12,6 @@ from scrapy.downloadermiddlewares.robotstxt import (RobotsTxtMiddleware,from scrapy.exceptions import IgnoreRequest, NotConfiguredfrom scrapy.http import Request, Response, TextResponsefrom scrapy.settings import Settings-from tests import mockfrom tests.test_robotstxt_interface import rerp_available, reppy_availablediff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.pyindex 4f389e5cb..875ceb83c 100644--- a/tests/test_extension_telnet.py+++ b/tests/test_extension_telnet.py@@ -1,8 +1,3 @@-try:- import unittest.mock as mock-except ImportError:- import mock-from twisted.trial import unittestfrom twisted.conch.telnet import ITelnetProtocolfrom twisted.cred import credentialsdiff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 11a5a8279..0c70bf80e 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -7,6 +7,7 @@ from io import BytesIOimport tempfileimport shutilimport string+from unittest import mockfrom six.moves.urllib.parse import urljoin, urlparse, quotefrom six.moves.urllib.request import pathname2url@@ -15,7 +16,6 @@ from twisted.trial import unittestfrom twisted.internet import deferfrom scrapy.crawler import CrawlerRunnerfrom scrapy.settings import Settings-from tests import mockfrom tests.mockserver import MockServerfrom w3lib.url import path_to_file_uridiff --git a/tests/test_http_request.py b/tests/test_http_request.pyindex bb451b5f4..9fe201579 100644--- a/tests/test_http_request.py+++ b/tests/test_http_request.py@@ -3,6 +3,7 @@ import cgiimport unittestimport reimport json+from unittest import mockfrom urllib.parse import unquote_to_bytesimport warnings@@ -13,8 +14,6 @@ from six.moves.urllib.parse import urlparse, parse_qs, unquotefrom scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponsefrom scrapy.utils.python import to_bytes, to_native_str-from tests import mock-class RequestTest(unittest.TestCase):diff --git a/tests/test_item.py b/tests/test_item.pyindex 0ad278701..d98c63ddd 100644--- a/tests/test_item.py+++ b/tests/test_item.py@@ -1,12 +1,12 @@import sysimport unittest+from unittest import mockfrom warnings import catch_warningsimport sixfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta-from tests import mockPY36_PLUS = (sys.version_info.major >= 3) and (sys.version_info.minor >= 6)diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex cb8f8da18..bd40e4103 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -1,10 +1,9 @@import osimport randomimport time-import hashlib-import warningsfrom tempfile import mkdtempfrom shutil import rmtree+from unittest import mockfrom six.moves.urllib.parse import urlparsefrom six import BytesIO@@ -15,13 +14,10 @@ from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore, GCfrom scrapy.item import Item, Fieldfrom scrapy.http import Request, Responsefrom scrapy.settings import Settings-from scrapy.utils.python import to_bytesfrom scrapy.utils.test import assert_aws_environ, get_s3_content_and_deletefrom scrapy.utils.test import assert_gcs_environ, get_gcs_content_and_deletefrom scrapy.utils.boto import is_botocore-from tests import mock-def _mocked_download_func(request, info):response = request.meta.get('response')diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.pyindex 08286ff02..32e65bed5 100644--- a/tests/test_settings/__init__.py+++ b/tests/test_settings/__init__.py@@ -1,10 +1,9 @@import siximport unittest-import warnings+from unittest import mockfrom scrapy.settings import (BaseSettings, Settings, SettingsAttribute,SETTINGS_PRIORITIES, get_settings_priority)-from tests import mockfrom . import default_settingsdiff --git a/tests/test_spider.py b/tests/test_spider.pyindex 6f6cdb8ff..c0fccfdd6 100644--- a/tests/test_spider.py+++ b/tests/test_spider.py@@ -1,5 +1,6 @@import gzipimport inspect+from unittest import mockimport warningsfrom io import BytesIO@@ -17,8 +18,6 @@ from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.trackref import object_reffrom scrapy.utils.test import get_crawler-from tests import mock-class SpiderTest(unittest.TestCase):diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.pyindex 832fd3330..55d665e79 100644--- a/tests/test_spidermiddleware.py+++ b/tests/test_spidermiddleware.py@@ -1,3 +1,5 @@+from unittest import mock+from twisted.trial.unittest import TestCasefrom twisted.python.failure import Failure@@ -6,7 +8,6 @@ from scrapy.http import Request, Responsefrom scrapy.exceptions import _InvalidOutputfrom scrapy.utils.test import get_crawlerfrom scrapy.core.spidermw import SpiderMiddlewareManager-from tests import mockclass SpiderMiddlewareTestCase(TestCase):diff --git a/tests/test_stats.py b/tests/test_stats.pyindex 2033dbe07..2bbbb9e2c 100644--- a/tests/test_stats.py+++ b/tests/test_stats.py@@ -1,10 +1,6 @@from datetime import datetimeimport unittest--try:- from unittest import mock-except ImportError:- import mock+from unittest import mockfrom scrapy.extensions.corestats import CoreStatsfrom scrapy.spiders import Spiderdiff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.pyindex 3e7236fb1..ce04e7f29 100644--- a/tests/test_utils_deprecate.py+++ b/tests/test_utils_deprecate.py@@ -2,12 +2,11 @@from __future__ import absolute_importimport inspectimport unittest+from unittest import mockimport warningsfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.deprecate import create_deprecated_class, update_classpath-from tests import mock-class MyWarning(UserWarning):passdiff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.pyindex e109d5343..de9da9104 100644--- a/tests/test_utils_misc/__init__.py+++ b/tests/test_utils_misc/__init__.py@@ -1,11 +1,11 @@import sysimport osimport unittest+from unittest import mockfrom scrapy.item import Item, Fieldfrom scrapy.utils.misc import arg_to_iter, create_instance, load_object, set_environ, walk_modules-from tests import mock__doctests__ = ['scrapy.utils.misc']diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.pyindex c6072fc0d..480a717e7 100644--- a/tests/test_utils_trackref.py+++ b/tests/test_utils_trackref.py@@ -1,7 +1,7 @@import siximport unittest+from unittest import mockfrom scrapy.utils import trackref-from tests import mockclass Foo(trackref.object_ref):