Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Simplify some more imports.

Andrey Rakhmatullin92ffd2f249ec
diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index 2c35d1b90..2212d9688 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -1,10 +1,7 @@
import base64
from six.moves.urllib.parse import unquote, urlunparse
from six.moves.urllib.request import getproxies, proxy_bypass
-try:
- from urllib2 import _parse_proxy
-except ImportError:
- from urllib.request import _parse_proxy
+from urllib.request import _parse_proxy
from scrapy.exceptions import NotConfigured
from scrapy.utils.httpobj import urlparse_cached
diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py
index 2acdc8093..02c625acc 100644
--- a/scrapy/loader/processors.py
+++ b/scrapy/loader/processors.py
@@ -3,10 +3,7 @@ This module provides some commonly used processors for Item Loaders.
See documentation in docs/topics/loaders.rst
"""
-try:
- from collections import ChainMap
-except ImportError:
- from scrapy.utils.datatypes import MergeDict as ChainMap
+from collections import ChainMap
from scrapy.utils.misc import arg_to_iter
from scrapy.loader.common import wrap_loader_context
diff --git a/tests/__init__.py b/tests/__init__.py
index a54367f8c..12ce79fa9 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -21,11 +21,6 @@ if 'COV_CORE_CONFIG' in os.environ:
os.environ['COV_CORE_CONFIG'] = os.path.join(_sourceroot,
os.environ['COV_CORE_CONFIG'])
-try:
- import unittest.mock as mock
-except ImportError:
- import mock
-
tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'sample_data')
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 109469503..6090998d4 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -2,11 +2,8 @@ import os
import six
import shutil
import tempfile
+from unittest import mock
import contextlib
-try:
- from unittest import mock
-except ImportError:
- import mock
from testfixtures import LogCapture
from twisted.trial import unittest
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index 03564e748..6b9a5bee8 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -1,3 +1,5 @@
+from unittest import mock
+
from twisted.trial.unittest import TestCase
from twisted.python.failure import Failure
@@ -7,7 +9,6 @@ from scrapy.exceptions import _InvalidOutput
from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
from scrapy.utils.test import get_crawler
from scrapy.utils.python import to_bytes
-from tests import mock
class ManagerTestCase(TestCase):
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index fbc46cba4..8266bf35f 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -1,5 +1,8 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import
+
+from unittest import mock
+
from twisted.internet import reactor, error
from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
from twisted.python import failure
@@ -9,7 +12,6 @@ from scrapy.downloadermiddlewares.robotstxt import (RobotsTxtMiddleware,
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Request, Response, TextResponse
from scrapy.settings import Settings
-from tests import mock
from tests.test_robotstxt_interface import rerp_available, reppy_available
diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py
index 4f389e5cb..875ceb83c 100644
--- a/tests/test_extension_telnet.py
+++ b/tests/test_extension_telnet.py
@@ -1,8 +1,3 @@
-try:
- import unittest.mock as mock
-except ImportError:
- import mock
-
from twisted.trial import unittest
from twisted.conch.telnet import ITelnetProtocol
from twisted.cred import credentials
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 11a5a8279..0c70bf80e 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -7,6 +7,7 @@ from io import BytesIO
import tempfile
import shutil
import string
+from unittest import mock
from six.moves.urllib.parse import urljoin, urlparse, quote
from six.moves.urllib.request import pathname2url
@@ -15,7 +16,6 @@ from twisted.trial import unittest
from twisted.internet import defer
from scrapy.crawler import CrawlerRunner
from scrapy.settings import Settings
-from tests import mock
from tests.mockserver import MockServer
from w3lib.url import path_to_file_uri
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index bb451b5f4..9fe201579 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -3,6 +3,7 @@ import cgi
import unittest
import re
import json
+from unittest import mock
from urllib.parse import unquote_to_bytes
import warnings
@@ -13,8 +14,6 @@ from six.moves.urllib.parse import urlparse, parse_qs, unquote
from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse
from scrapy.utils.python import to_bytes, to_native_str
-from tests import mock
-
class RequestTest(unittest.TestCase):
diff --git a/tests/test_item.py b/tests/test_item.py
index 0ad278701..d98c63ddd 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -1,12 +1,12 @@
import sys
import unittest
+from unittest import mock
from warnings import catch_warnings
import six
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta
-from tests import mock
PY36_PLUS = (sys.version_info.major >= 3) and (sys.version_info.minor >= 6)
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index cb8f8da18..bd40e4103 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -1,10 +1,9 @@
import os
import random
import time
-import hashlib
-import warnings
from tempfile import mkdtemp
from shutil import rmtree
+from unittest import mock
from six.moves.urllib.parse import urlparse
from six import BytesIO
@@ -15,13 +14,10 @@ from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore, GC
from scrapy.item import Item, Field
from scrapy.http import Request, Response
from scrapy.settings import Settings
-from scrapy.utils.python import to_bytes
from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
from scrapy.utils.test import assert_gcs_environ, get_gcs_content_and_delete
from scrapy.utils.boto import is_botocore
-from tests import mock
-
def _mocked_download_func(request, info):
response = request.meta.get('response')
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 08286ff02..32e65bed5 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -1,10 +1,9 @@
import six
import unittest
-import warnings
+from unittest import mock
from scrapy.settings import (BaseSettings, Settings, SettingsAttribute,
SETTINGS_PRIORITIES, get_settings_priority)
-from tests import mock
from . import default_settings
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 6f6cdb8ff..c0fccfdd6 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -1,5 +1,6 @@
import gzip
import inspect
+from unittest import mock
import warnings
from io import BytesIO
@@ -17,8 +18,6 @@ from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.trackref import object_ref
from scrapy.utils.test import get_crawler
-from tests import mock
-
class SpiderTest(unittest.TestCase):
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 832fd3330..55d665e79 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -1,3 +1,5 @@
+from unittest import mock
+
from twisted.trial.unittest import TestCase
from twisted.python.failure import Failure
@@ -6,7 +8,6 @@ from scrapy.http import Request, Response
from scrapy.exceptions import _InvalidOutput
from scrapy.utils.test import get_crawler
from scrapy.core.spidermw import SpiderMiddlewareManager
-from tests import mock
class SpiderMiddlewareTestCase(TestCase):
diff --git a/tests/test_stats.py b/tests/test_stats.py
index 2033dbe07..2bbbb9e2c 100644
--- a/tests/test_stats.py
+++ b/tests/test_stats.py
@@ -1,10 +1,6 @@
from datetime import datetime
import unittest
-
-try:
- from unittest import mock
-except ImportError:
- import mock
+from unittest import mock
from scrapy.extensions.corestats import CoreStats
from scrapy.spiders import Spider
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index 3e7236fb1..ce04e7f29 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -2,12 +2,11 @@
from __future__ import absolute_import
import inspect
import unittest
+from unittest import mock
import warnings
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.deprecate import create_deprecated_class, update_classpath
-from tests import mock
-
class MyWarning(UserWarning):
pass
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index e109d5343..de9da9104 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -1,11 +1,11 @@
import sys
import os
import unittest
+from unittest import mock
from scrapy.item import Item, Field
from scrapy.utils.misc import arg_to_iter, create_instance, load_object, set_environ, walk_modules
-from tests import mock
__doctests__ = ['scrapy.utils.misc']
diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py
index c6072fc0d..480a717e7 100644
--- a/tests/test_utils_trackref.py
+++ b/tests/test_utils_trackref.py
@@ -1,7 +1,7 @@
import six
import unittest
+from unittest import mock
from scrapy.utils import trackref
-from tests import mock
class Foo(trackref.object_ref):