Scrapy, a fast high-level web crawling & scraping framework for Python.
Add missing blank lines between functions and classes
Adrián Chavesf261cf65e999
diff --git a/pytest.ini b/pytest.iniindex 529ad5d27..a3693a778 100644--- a/pytest.ini+++ b/pytest.ini@@ -30,16 +30,15 @@ flake8-ignore =scrapy/commands/check.py F401 E501scrapy/commands/crawl.py E501scrapy/commands/edit.py E501- scrapy/commands/fetch.py E401 E302 E501 E128 E502 E731+ scrapy/commands/fetch.py E401 E501 E128 E502 E731scrapy/commands/genspider.py E128 E501 E502- scrapy/commands/list.py E302scrapy/commands/parse.py E128 E501 E731 E226scrapy/commands/runspider.py E501- scrapy/commands/settings.py E302 E128+ scrapy/commands/settings.py E128scrapy/commands/shell.py E128 E501 E502scrapy/commands/startproject.py E502 E127 E501 E128scrapy/commands/version.py E501 E128- scrapy/commands/view.py F401 E302+ scrapy/commands/view.py F401# scrapy/contractsscrapy/contracts/__init__.py E501 W504scrapy/contracts/default.py E502 E128@@ -60,7 +59,7 @@ flake8-ignore =scrapy/core/downloader/handlers/http11.py E501scrapy/core/downloader/handlers/s3.py E501 F401 E502 E128 E126# scrapy/downloadermiddlewares- scrapy/downloadermiddlewares/ajaxcrawl.py E302 E501 E226+ scrapy/downloadermiddlewares/ajaxcrawl.py E501 E226scrapy/downloadermiddlewares/decompression.py E501scrapy/downloadermiddlewares/defaultheaders.py E501scrapy/downloadermiddlewares/httpcache.py E501 E126@@ -72,11 +71,11 @@ flake8-ignore =scrapy/downloadermiddlewares/stats.py E501# scrapy/extensionsscrapy/extensions/closespider.py E501 E502 E128 E123- scrapy/extensions/corestats.py E302 E501+ scrapy/extensions/corestats.py E501scrapy/extensions/feedexport.py E128 E501scrapy/extensions/httpcache.py E128 E501 E303 F401scrapy/extensions/memdebug.py E501- scrapy/extensions/spiderstate.py E302 E501+ scrapy/extensions/spiderstate.py E501scrapy/extensions/telnet.py E501 W504scrapy/extensions/throttle.py E501# scrapy/http@@ -87,18 +86,14 @@ flake8-ignore =scrapy/http/request/form.py E501 E123scrapy/http/request/json_request.py E501scrapy/http/response/__init__.py E501 E128 W293 W291- scrapy/http/response/html.py E302scrapy/http/response/text.py E501 W293 E128 E124- scrapy/http/response/xml.py E302# scrapy/linkextractorsscrapy/linkextractors/__init__.py E731 E502 E501 E402 F401scrapy/linkextractors/lxmlhtml.py E501 E731 E226# scrapy/loaderscrapy/loader/__init__.py E501 E502 E128- scrapy/loader/common.py E302scrapy/loader/processors.py E501# scrapy/pipelines- scrapy/pipelines/__init__.py E302scrapy/pipelines/files.py E116 E501 E266scrapy/pipelines/images.py E265 E501scrapy/pipelines/media.py E125 E501 E266@@ -123,56 +118,50 @@ flake8-ignore =scrapy/utils/benchserver.py E501scrapy/utils/boto.py F401scrapy/utils/conf.py E402 E502 E501- scrapy/utils/console.py E302 E261 F401 E306 E305+ scrapy/utils/console.py E261 F401 E306 E305scrapy/utils/curl.py F401scrapy/utils/datatypes.py E501 E226- scrapy/utils/decorators.py E501 E302- scrapy/utils/defer.py E501 E302 E128+ scrapy/utils/decorators.py E501+ scrapy/utils/defer.py E501 E128scrapy/utils/deprecate.py E128 E501 E127 E502- scrapy/utils/display.py E302- scrapy/utils/engine.py F401 E261 E302- scrapy/utils/ftp.py E302- scrapy/utils/gz.py E305 E501 E302 W504+ scrapy/utils/engine.py F401 E261+ scrapy/utils/gz.py E305 E501 W504scrapy/utils/http.py F403 F401 E226- scrapy/utils/httpobj.py E302 E501+ scrapy/utils/httpobj.py E501scrapy/utils/iterators.py E501 E701- scrapy/utils/job.py E302scrapy/utils/log.py E128 W503scrapy/utils/markup.py F403 F401 W292scrapy/utils/misc.py E501 E226scrapy/utils/multipart.py F403 F401 W292scrapy/utils/project.py E501- scrapy/utils/python.py E501 E302- scrapy/utils/reactor.py E302 E226+ scrapy/utils/python.py E501+ scrapy/utils/reactor.py E226scrapy/utils/reqser.py E501- scrapy/utils/request.py E302 E127 E501- scrapy/utils/response.py E501 E302 E128+ scrapy/utils/request.py E127 E501+ scrapy/utils/response.py E501 E128scrapy/utils/signal.py E501 E128scrapy/utils/sitemap.py E501- scrapy/utils/spider.py E271 E302 E501+ scrapy/utils/spider.py E271 E501scrapy/utils/ssl.py E501- scrapy/utils/template.py E302- scrapy/utils/test.py E302 E501+ scrapy/utils/test.py E501scrapy/utils/url.py E501 F403 F401 E128 F405# scrapyscrapy/__init__.py E402 E501scrapy/_monkeypatches.py W293scrapy/cmdline.py E502 E501scrapy/crawler.py E501- scrapy/dupefilters.py E302 E501 E202- scrapy/exceptions.py E302 E501+ scrapy/dupefilters.py E501 E202+ scrapy/exceptions.py E501scrapy/exporters.py E501 E261 E226- scrapy/extension.py E302- scrapy/interfaces.py E302 E501+ scrapy/interfaces.py E501scrapy/item.py E501 E128scrapy/link.py E501scrapy/logformatter.py E501 W293scrapy/mail.py E402 E128 E501 E502scrapy/middleware.py E502 E128 E501scrapy/pqueues.py E501- scrapy/resolver.py E302scrapy/responsetypes.py E128 E501 E305- scrapy/robotstxt.py E302 E501+ scrapy/robotstxt.py E501scrapy/shell.py E501scrapy/signalmanager.py E501scrapy/spiderloader.py E225 F841 E501 E126@@ -181,91 +170,82 @@ flake8-ignore =# teststests/__init__.py F401 E402 E501tests/mockserver.py E401 E501 E126 E123 F401- tests/pipelines.py E302 F841 E226- tests/spiders.py E302 E501 E127+ tests/pipelines.py F841 E226+ tests/spiders.py E501 E127tests/test_closespider.py E501 E127tests/test_command_fetch.py E501 E261- tests/test_command_parse.py F401 E302 E501 E128 E303 E226+ tests/test_command_parse.py F401 E501 E128 E303 E226tests/test_command_shell.py E501 E128tests/test_commands.py F401 E128 E501tests/test_contracts.py E501 E128 W293tests/test_crawl.py E501 E741 E265tests/test_crawler.py F841 E306 E501- tests/test_dependencies.py E302 F841 E501 E305+ tests/test_dependencies.py F841 E501 E305tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 F401 E501 E502 E701 E126 E226 E123tests/test_downloadermiddleware.py E501- tests/test_downloadermiddleware_ajaxcrawlable.py E302 E501+ tests/test_downloadermiddleware_ajaxcrawlable.py E501tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126tests/test_downloadermiddleware_decompression.py E127tests/test_downloadermiddleware_defaultheaders.py E501tests/test_downloadermiddleware_downloadtimeout.py E501- tests/test_downloadermiddleware_httpcache.py E501 E302 E305 F401+ tests/test_downloadermiddleware_httpcache.py E501 E305 F401tests/test_downloadermiddleware_httpcompression.py E501 F401 E251 E126 E123tests/test_downloadermiddleware_httpproxy.py F401 E501 E128tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E502 E303 E126tests/test_downloadermiddleware_robotstxt.py E501tests/test_downloadermiddleware_stats.py E501- tests/test_dupefilters.py E302 E221 E501 E741 W293 W291 E128 E124+ tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124tests/test_engine.py E401 E501 E502 E128 E261tests/test_exporters.py E501 E731 E306 E128 E124tests/test_extension_telnet.py F401 F841tests/test_feedexport.py E501 F401 F841 E241tests/test_http_cookies.py E501- tests/test_http_headers.py E302 E501+ tests/test_http_headers.py E501tests/test_http_request.py F401 E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123tests/test_http_response.py E501 E301 E502 E128 E265tests/test_item.py E701 E128 F841 E306tests/test_link.py E501tests/test_linkextractors.py E501 E128 E124- tests/test_loader.py E302 E501 E731 E303 E741 E128 E117 E241- tests/test_logformatter.py E128 E501 E122 E302- tests/test_mail.py E302 E128 E501 E305- tests/test_middleware.py E302 E501 E128+ tests/test_loader.py E501 E731 E303 E741 E128 E117 E241+ tests/test_logformatter.py E128 E501 E122+ tests/test_mail.py E128 E501 E305+ tests/test_middleware.py E501 E128tests/test_pipeline_crawl.py E131 E501 E128 E126tests/test_pipeline_files.py F401 E501 W293 E303 E272 E226tests/test_pipeline_images.py F401 F841 E501 E303tests/test_pipeline_media.py E501 E741 E731 E128 E261 E306 E502tests/test_request_cb_kwargs.py E501- tests/test_responsetypes.py E501 E302 E305- tests/test_robotstxt_interface.py F401 E302 E501 W291 E501+ tests/test_responsetypes.py E501 E305+ tests/test_robotstxt_interface.py F401 E501 W291 E501tests/test_scheduler.py E501 E126 E123tests/test_selector.py F401 E501 E127tests/test_spider.py E501 F401tests/test_spidermiddleware.py E501 E226tests/test_spidermiddleware_httperror.py E128 E501 E127 E121- tests/test_spidermiddleware_offsite.py E302 E501 E128 E111 W293- tests/test_spidermiddleware_output_chain.py F401 E501 E302 W293 E226- tests/test_spidermiddleware_referer.py F401 E501 E302 F841 E125 E201 E261 E124 E501 E241 E121- tests/test_squeues.py E501 E302 E701 E741+ tests/test_spidermiddleware_offsite.py E501 E128 E111 W293+ tests/test_spidermiddleware_output_chain.py F401 E501 W293 E226+ tests/test_spidermiddleware_referer.py F401 E501 F841 E125 E201 E261 E124 E501 E241 E121+ tests/test_squeues.py E501 E701 E741tests/test_utils_conf.py E501 E303 E128- tests/test_utils_console.py E302tests/test_utils_curl.py E501tests/test_utils_datatypes.py E402 E501 E305- tests/test_utils_defer.py E306 E261 E501 E302 F841 E226+ tests/test_utils_defer.py E306 E261 E501 F841 E226tests/test_utils_deprecate.py F841 E306 E501- tests/test_utils_http.py E302 E501 E502 E128 W504- tests/test_utils_httpobj.py E302- tests/test_utils_iterators.py E501 E128 E129 E302 E303 E241+ tests/test_utils_http.py E501 E502 E128 W504+ tests/test_utils_iterators.py E501 E128 E129 E303 E241tests/test_utils_log.py E741 E226tests/test_utils_python.py E501 E303 E731 E701 E305tests/test_utils_reqser.py F401 E501 E128- tests/test_utils_request.py E302 E501 E128 E305+ tests/test_utils_request.py E501 E128 E305tests/test_utils_response.py E501- tests/test_utils_signal.py E741 F841 E302 E731 E226- tests/test_utils_sitemap.py E302 E128 E501 E124- tests/test_utils_spider.py E261 E302 E305+ tests/test_utils_signal.py E741 F841 E731 E226+ tests/test_utils_sitemap.py E128 E501 E124+ tests/test_utils_spider.py E261 E305tests/test_utils_template.py E305- tests/test_utils_url.py F401 E501 E127 E302 E305 E211 E125 E501 E226 E241 E126 E123+ tests/test_utils_url.py F401 E501 E127 E305 E211 E125 E501 E226 E241 E126 E123tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126- tests/mocks/dummydbm.py E302tests/test_cmdline/__init__.py E502 E501- tests/test_cmdline/extensions.py E302tests/test_settings/__init__.py F401 E501 E128- tests/test_spiderloader/__init__.py E128 E501 E302- tests/test_spiderloader/test_spiders/spider0.py E302- tests/test_spiderloader/test_spiders/spider1.py E302- tests/test_spiderloader/test_spiders/spider2.py E302- tests/test_spiderloader/test_spiders/spider3.py E302- tests/test_spiderloader/test_spiders/nested/spider4.py E302+ tests/test_spiderloader/__init__.py E128 E501tests/test_utils_misc/__init__.py E501diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.pyindex d45133e0e..724b4a1c4 100644--- a/scrapy/commands/fetch.py+++ b/scrapy/commands/fetch.py@@ -8,6 +8,7 @@ from scrapy.exceptions import UsageErrorfrom scrapy.utils.datatypes import SequenceExcludefrom scrapy.utils.spider import spidercls_for_request, DefaultSpider+class Command(ScrapyCommand):requires_project = Falsediff --git a/scrapy/commands/list.py b/scrapy/commands/list.pyindex a255b3b94..422183ac1 100644--- a/scrapy/commands/list.py+++ b/scrapy/commands/list.py@@ -1,6 +1,7 @@from __future__ import print_functionfrom scrapy.commands import ScrapyCommand+class Command(ScrapyCommand):requires_project = Truediff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.pyindex bee52f06a..ffe3aa2eb 100644--- a/scrapy/commands/settings.py+++ b/scrapy/commands/settings.py@@ -4,6 +4,7 @@ import jsonfrom scrapy.commands import ScrapyCommandfrom scrapy.settings import BaseSettings+class Command(ScrapyCommand):requires_project = Falsediff --git a/scrapy/commands/view.py b/scrapy/commands/view.pyindex 59e665016..31c17c0ab 100644--- a/scrapy/commands/view.py+++ b/scrapy/commands/view.py@@ -1,6 +1,7 @@from scrapy.commands import fetch, ScrapyCommandfrom scrapy.utils.response import open_in_browser+class Command(fetch.Command):def short_desc(self):diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.pyindex ad25beb3b..9e5020753 100644--- a/scrapy/core/downloader/handlers/datauri.py+++ b/scrapy/core/downloader/handlers/datauri.py@@ -17,8 +17,8 @@ class DataURIDownloadHandler(object):respcls = responsetypes.from_mimetype(uri.media_type)resp_kwargs = {}- if (issubclass(respcls, TextResponse) and- uri.media_type.split('/')[0] == 'text'):+ if (issubclass(respcls, TextResponse)+ and uri.media_type.split('/')[0] == 'text'):charset = uri.media_type_parameters.get('charset')resp_kwargs['encoding'] = charsetdiff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.pyindex 72715dba7..ba50793bb 100644--- a/scrapy/downloadermiddlewares/ajaxcrawl.py+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py@@ -68,6 +68,8 @@ class AjaxCrawlMiddleware(object):# XXX: move it to w3lib?_ajax_crawlable_re = re.compile(six.u(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>'))++def _has_ajaxcrawlable_meta(text):""">>> _has_ajaxcrawlable_meta('<html><head><meta name="fragment" content="!"/></head><body></body></html>')diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.pyindex 0bcdd3495..4d95eb847 100644--- a/scrapy/dupefilters.py+++ b/scrapy/dupefilters.py@@ -5,6 +5,7 @@ import loggingfrom scrapy.utils.job import job_dirfrom scrapy.utils.request import referer_str, request_fingerprint+class BaseDupeFilter(object):@classmethoddiff --git a/scrapy/exceptions.py b/scrapy/exceptions.pyindex 96949bdd9..7c4bb3d00 100644--- a/scrapy/exceptions.py+++ b/scrapy/exceptions.py@@ -7,10 +7,12 @@ new exceptions here without documenting them there.# Internal+class NotConfigured(Exception):"""Indicates a missing configuration situation"""pass+class _InvalidOutput(TypeError):"""Indicates an invalid value has been returned by a middleware's processing method.@@ -18,15 +20,19 @@ class _InvalidOutput(TypeError):"""pass+# HTTP and crawling+class IgnoreRequest(Exception):"""Indicates a decision was made not to process a request"""+class DontCloseSpider(Exception):"""Request the spider not to be closed yet"""pass+class CloseSpider(Exception):"""Raise this from callbacks to request the spider to be closed"""@@ -34,30 +40,37 @@ class CloseSpider(Exception):super(CloseSpider, self).__init__()self.reason = reason+# Items+class DropItem(Exception):"""Drop item from the item pipeline"""pass+class NotSupported(Exception):"""Indicates a feature or method is not supported"""pass+# Commands+class UsageError(Exception):"""To indicate a command-line usage error"""def __init__(self, *a, **kw):self.print_help = kw.pop('print_help', True)super(UsageError, self).__init__(*a, **kw)+class ScrapyDeprecationWarning(Warning):"""Warning category for deprecated features, since the defaultDeprecationWarning is silenced on Python 2.7+"""pass+class ContractFail(AssertionError):"""Error raised in case of a failing contract"""passdiff --git a/scrapy/extension.py b/scrapy/extension.pyindex e39e456fa..050b87e5f 100644--- a/scrapy/extension.py+++ b/scrapy/extension.py@@ -6,6 +6,7 @@ See documentation in docs/topics/extensions.rstfrom scrapy.middleware import MiddlewareManagerfrom scrapy.utils.conf import build_component_list+class ExtensionManager(MiddlewareManager):component_name = 'extension'diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.pyindex 2220cbd8f..8ba770ec0 100644--- a/scrapy/extensions/spiderstate.py+++ b/scrapy/extensions/spiderstate.py@@ -5,6 +5,7 @@ from scrapy import signalsfrom scrapy.exceptions import NotConfiguredfrom scrapy.utils.job import job_dir+class SpiderState(object):"""Store and load spider state during a scraping job"""diff --git a/scrapy/http/response/html.py b/scrapy/http/response/html.pyindex bd3559fbb..7eed052c2 100644--- a/scrapy/http/response/html.py+++ b/scrapy/http/response/html.py@@ -7,5 +7,6 @@ See documentation in docs/topics/request-response.rstfrom scrapy.http.response.text import TextResponse+class HtmlResponse(TextResponse):passdiff --git a/scrapy/http/response/xml.py b/scrapy/http/response/xml.pyindex 1df33fee5..abf474a2f 100644--- a/scrapy/http/response/xml.py+++ b/scrapy/http/response/xml.py@@ -7,5 +7,6 @@ See documentation in docs/topics/request-response.rstfrom scrapy.http.response.text import TextResponse+class XmlResponse(TextResponse):passdiff --git a/scrapy/interfaces.py b/scrapy/interfaces.pyindex d48babc3c..1896ec31e 100644--- a/scrapy/interfaces.py+++ b/scrapy/interfaces.py@@ -1,5 +1,6 @@from zope.interface import Interface+class ISpiderLoader(Interface):def from_settings(settings):diff --git a/scrapy/loader/common.py b/scrapy/loader/common.pyindex 916524947..42f8de636 100644--- a/scrapy/loader/common.py+++ b/scrapy/loader/common.py@@ -3,6 +3,7 @@from functools import partialfrom scrapy.utils.python import get_func_args+def wrap_loader_context(function, context):"""Wrap functions that receive loader_context to contain the context"pre-loaded" and expose a interface that receives only one argumentdiff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.pyindex 2ef8786d0..aa1bfb77f 100644--- a/scrapy/pipelines/__init__.py+++ b/scrapy/pipelines/__init__.py@@ -7,6 +7,7 @@ See documentation in docs/item-pipeline.rstfrom scrapy.middleware import MiddlewareManagerfrom scrapy.utils.conf import build_component_list+class ItemPipelineManager(MiddlewareManager):component_name = 'item pipeline'diff --git a/scrapy/resolver.py b/scrapy/resolver.pyindex 0aaced7e4..4df949015 100644--- a/scrapy/resolver.py+++ b/scrapy/resolver.py@@ -7,6 +7,7 @@ from scrapy.utils.datatypes import LocalCachednscache = LocalCache(10000)+class CachingThreadedResolver(ThreadedResolver):def __init__(self, reactor, cache_size, timeout):super(CachingThreadedResolver, self).__init__(reactor)diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.pyindex 95a8c09b8..f0f9c59dc 100644--- a/scrapy/robotstxt.py+++ b/scrapy/robotstxt.py@@ -5,8 +5,10 @@ from six import with_metaclassfrom scrapy.utils.python import to_unicode+logger = logging.getLogger(__name__)+def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):try:if to_native_str_type:@@ -23,6 +25,7 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):robotstxt_body = ''return robotstxt_body+class RobotParser(with_metaclass(ABCMeta)):@classmethod@abstractmethoddiff --git a/scrapy/utils/console.py b/scrapy/utils/console.pyindex 2e9981556..a26e84d38 100644--- a/scrapy/utils/console.py+++ b/scrapy/utils/console.py@@ -1,6 +1,7 @@from functools import wrapsfrom collections import OrderedDict+def _embed_ipython_shell(namespace={}, banner=''):"""Start an IPython Shell"""try:@@ -23,6 +24,7 @@ def _embed_ipython_shell(namespace={}, banner=''):shell()return wrapper+def _embed_bpython_shell(namespace={}, banner=''):"""Start a bpython shell"""import bpython@@ -31,6 +33,7 @@ def _embed_bpython_shell(namespace={}, banner=''):bpython.embed(locals_=namespace, banner=banner)return wrapper+def _embed_ptpython_shell(namespace={}, banner=''):"""Start a ptpython shell"""import ptpython.repl@@ -40,6 +43,7 @@ def _embed_ptpython_shell(namespace={}, banner=''):ptpython.repl.embed(locals=namespace)return wrapper+def _embed_standard_shell(namespace={}, banner=''):"""Start a standard python shell"""import code@@ -55,6 +59,7 @@ def _embed_standard_shell(namespace={}, banner=''):code.interact(banner=banner, local=namespace)return wrapper+DEFAULT_PYTHON_SHELLS = OrderedDict([('ptpython', _embed_ptpython_shell),('ipython', _embed_ipython_shell),@@ -62,6 +67,7 @@ DEFAULT_PYTHON_SHELLS = OrderedDict([('python', _embed_standard_shell),])+def get_shell_embed_func(shells=None, known_shells=None):"""Return the first acceptable shell-embed functionfrom a given list of shell names.@@ -79,6 +85,7 @@ def get_shell_embed_func(shells=None, known_shells=None):except ImportError:continue+def start_python_console(namespace=None, banner='', shells=None):"""Start Python console bound to the given namespace.Readline support and tab completion will be used on Unix, if available.diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.pyindex 38bee1a6c..2e2c7adc1 100644--- a/scrapy/utils/decorators.py+++ b/scrapy/utils/decorators.py@@ -34,6 +34,7 @@ def defers(func):return defer.maybeDeferred(func, *a, **kw)return wrapped+def inthread(func):"""Decorator to call a function in a thread and return a deferred with theresultdiff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.pyindex 69d621830..c5916c21c 100644--- a/scrapy/utils/defer.py+++ b/scrapy/utils/defer.py@@ -7,6 +7,7 @@ from twisted.python import failurefrom scrapy.exceptions import IgnoreRequest+def defer_fail(_failure):"""Same as twisted.internet.defer.fail but delay calling errback untilnext reactor loop@@ -18,6 +19,7 @@ def defer_fail(_failure):reactor.callLater(0.1, d.errback, _failure)return d+def defer_succeed(result):"""Same as twisted.internet.defer.succeed but delay calling callback untilnext reactor loop@@ -29,6 +31,7 @@ def defer_succeed(result):reactor.callLater(0.1, d.callback, result)return d+def defer_result(result):if isinstance(result, defer.Deferred):return result@@ -37,6 +40,7 @@ def defer_result(result):else:return defer_succeed(result)+def mustbe_deferred(f, *args, **kw):"""Same as twisted.internet.defer.maybeDeferred, but delay callingcallback/errback to next reactor loop@@ -53,6 +57,7 @@ def mustbe_deferred(f, *args, **kw):else:return defer_result(result)+def parallel(iterable, count, callable, *args, **named):"""Execute a callable over the objects in the given iterable, in parallel,using no more than ``count`` concurrent calls.@@ -63,6 +68,7 @@ def parallel(iterable, count, callable, *args, **named):work = (callable(elem, *args, **named) for elem in iterable)return defer.DeferredList([coop.coiterate(work) for _ in range(count)])+def process_chain(callbacks, input, *a, **kw):"""Return a Deferred built by chaining the given callbacks"""d = defer.Deferred()@@ -71,6 +77,7 @@ def process_chain(callbacks, input, *a, **kw):d.callback(input)return d+def process_chain_both(callbacks, errbacks, input, *a, **kw):"""Return a Deferred built by chaining the given callbacks and errbacks"""d = defer.Deferred()@@ -83,6 +90,7 @@ def process_chain_both(callbacks, errbacks, input, *a, **kw):d.callback(input)return d+def process_parallel(callbacks, input, *a, **kw):"""Return a Deferred with the output of all successful calls to the givencallbacks@@ -92,6 +100,7 @@ def process_parallel(callbacks, input, *a, **kw):d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure)return d+def iter_errback(iterable, errback, *a, **kw):"""Wraps an iterable calling an errback if an error is caught whileiterating it.diff --git a/scrapy/utils/display.py b/scrapy/utils/display.pyindex f6a6c4645..91ebdae11 100644--- a/scrapy/utils/display.py+++ b/scrapy/utils/display.py@@ -6,6 +6,7 @@ from __future__ import print_functionimport sysfrom pprint import pformat as pformat_+def _colorize(text, colorize=True):if not colorize or not sys.stdout.isatty():return text@@ -17,8 +18,10 @@ def _colorize(text, colorize=True):except ImportError:return text+def pformat(obj, *args, **kwargs):return _colorize(pformat_(obj), kwargs.pop('colorize', True))+def pprint(obj, *args, **kwargs):print(pformat(obj, *args, **kwargs))diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.pyindex 11dd36d91..2c20b5c88 100644--- a/scrapy/utils/engine.py+++ b/scrapy/utils/engine.py@@ -3,6 +3,7 @@from __future__ import print_functionfrom time import time # used in global tests code+def get_engine_status(engine):"""Return a report of the current engine status"""tests = [@@ -32,6 +33,7 @@ def get_engine_status(engine):return checks+def format_engine_status(engine=None):checks = get_engine_status(engine)s = "Execution engine status\n\n"@@ -41,5 +43,6 @@ def format_engine_status(engine=None):return s+def print_engine_status(engine):print(format_engine_status(engine))diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.pyindex 9eca6a4da..91d2439a9 100644--- a/scrapy/utils/ftp.py+++ b/scrapy/utils/ftp.py@@ -1,6 +1,7 @@from ftplib import error_permfrom posixpath import dirname+def ftp_makedirs_cwd(ftp, path, first_call=True):"""Set the current directory of the FTP connection given in the ``ftp``argument (as a ftplib.FTP object), creating all parent directories if theydiff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.pyindex f41e62fe3..9672e28da 100644--- a/scrapy/utils/gz.py+++ b/scrapy/utils/gz.py@@ -45,6 +45,7 @@ def gunzip(data):_is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search_is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search+@deprecateddef is_gzipped(response):"""Return True if the response is gzipped, or False otherwise"""diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.pyindex b4c929b0e..b2be0a901 100644--- a/scrapy/utils/httpobj.py+++ b/scrapy/utils/httpobj.py@@ -4,7 +4,10 @@ import weakreffrom six.moves.urllib.parse import urlparse+_urlparse_cache = weakref.WeakKeyDictionary()++def urlparse_cached(request_or_response):"""Return urlparse.urlparse caching the result, where the argument can be aRequest or Response objectdiff --git a/scrapy/utils/job.py b/scrapy/utils/job.pyindex 389fde73a..4f1e601fc 100644--- a/scrapy/utils/job.py+++ b/scrapy/utils/job.py@@ -1,5 +1,6 @@import os+def job_dir(settings):path = settings['JOBDIR']if path and not os.path.exists(path):diff --git a/scrapy/utils/python.py b/scrapy/utils/python.pyindex 663a8ebaa..a4201bb04 100644--- a/scrapy/utils/python.py+++ b/scrapy/utils/python.py@@ -165,6 +165,7 @@ def memoizemethod_noargs(method):_BINARYCHARS = {six.b(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"}_BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS}+@deprecated("scrapy.utils.python.binary_is_text")def isbinarytext(text):""" This function is deprecated.diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.pyindex 83186a372..b4b5f0645 100644--- a/scrapy/utils/reactor.py+++ b/scrapy/utils/reactor.py@@ -1,5 +1,6 @@from twisted.internet import reactor, error+def listen_tcp(portrange, host, factory):"""Like reactor.listenTCP but tries different ports in a range."""assert len(portrange) <= 2, "invalid portrange: %s" % portrangediff --git a/scrapy/utils/request.py b/scrapy/utils/request.pyindex 63d0ae772..0fce5a2e1 100644--- a/scrapy/utils/request.py+++ b/scrapy/utils/request.py@@ -16,6 +16,8 @@ from scrapy.utils.httpobj import urlparse_cached_fingerprint_cache = weakref.WeakKeyDictionary()++def request_fingerprint(request, include_headers=None, keep_fragments=False):"""Return the request fingerprint.diff --git a/scrapy/utils/response.py b/scrapy/utils/response.pyindex feab07431..29fdaaf2c 100644--- a/scrapy/utils/response.py+++ b/scrapy/utils/response.py@@ -13,6 +13,8 @@ from w3lib import html_baseurl_cache = weakref.WeakKeyDictionary()++def get_base_url(response):"""Return the base url of the given response, joined with the response url"""if response not in _baseurl_cache:@@ -23,6 +25,8 @@ def get_base_url(response):_metaref_cache = weakref.WeakKeyDictionary()++def get_meta_refresh(response, ignore_tags=('script', 'noscript')):"""Parse the http-equiv refrsh parameter from the given response"""if response not in _metaref_cache:diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.pyindex 94b24f67e..bf4973fbf 100644--- a/scrapy/utils/spider.py+++ b/scrapy/utils/spider.py@@ -28,6 +28,7 @@ def iter_spider_classes(module):getattr(obj, 'name', None):yield obj+def spidercls_for_request(spider_loader, request, default_spidercls=None,log_none=False, log_multiple=False):"""Return a spider class that handles the given Request.diff --git a/scrapy/utils/template.py b/scrapy/utils/template.pyindex 615372fc8..96ff4b09b 100644--- a/scrapy/utils/template.py+++ b/scrapy/utils/template.py@@ -19,6 +19,8 @@ def render_templatefile(path, **kwargs):CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]')++def string_camelcase(string):""" Convert a word to its CamelCase version and remove invalid charsdiff --git a/scrapy/utils/test.py b/scrapy/utils/test.pyindex 4b935c51b..9754366df 100644--- a/scrapy/utils/test.py+++ b/scrapy/utils/test.py@@ -32,6 +32,7 @@ def skip_if_no_boto():except NotConfigured as e:raise SkipTest(e)+def get_s3_content_and_delete(bucket, path, with_key=False):""" Get content from s3 key, and delete key afterwards."""@@ -51,6 +52,7 @@ def get_s3_content_and_delete(bucket, path, with_key=False):bucket.delete_key(path)return (content, key) if with_key else content+def get_gcs_content_and_delete(bucket, path):from google.cloud import storageclient = storage.Client(project=os.environ.get('GCS_PROJECT_ID'))@@ -61,6 +63,7 @@ def get_gcs_content_and_delete(bucket, path):bucket.delete_blob(path)return content, acl, blob+def get_crawler(spidercls=None, settings_dict=None):"""Return an unconfigured Crawler object. If settings_dict is given, itwill be used to populate the crawler settings with a project level@@ -72,12 +75,14 @@ def get_crawler(spidercls=None, settings_dict=None):runner = CrawlerRunner(settings_dict)return runner.create_crawler(spidercls or Spider)+def get_pythonpath():"""Return a PYTHONPATH suitable to use in processes so that they find thisinstallation of Scrapy"""scrapy_path = import_module('scrapy').__path__[0]return os.path.dirname(scrapy_path) + os.pathsep + os.environ.get('PYTHONPATH', '')+def get_testenv():"""Return a OS environment dict suitable to fork processes that need to importthis installation of Scrapy, instead of a system installed one.@@ -86,6 +91,7 @@ def get_testenv():env['PYTHONPATH'] = get_pythonpath()return env+def assert_samelines(testcase, text1, text2, msg=None):"""Asserts text1 and text2 have the same lines, ignoring differences inline endings between platformsdiff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.pyindex 48484b303..b0737d3d5 100644--- a/scrapy/utils/versions.py+++ b/scrapy/utils/versions.py@@ -27,5 +27,5 @@ def scrapy_components_versions():("Python", sys.version.replace("\n", "- ")),("pyOpenSSL", get_openssl_version()),("cryptography", cryptography.__version__),- ("Platform", platform.platform()),+ ("Platform", platform.platform()),]diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.pyindex 431428331..75c74daf5 100644--- a/tests/mocks/dummydbm.py+++ b/tests/mocks/dummydbm.py@@ -13,6 +13,7 @@ error = KeyError_DATABASES = collections.defaultdict(DummyDB)+def open(file, flag='r', mode=0o666):"""Open or create a dummy database compatible.diff --git a/tests/pipelines.py b/tests/pipelines.pyindex 7e2895a5c..d7d3b5259 100644--- a/tests/pipelines.py+++ b/tests/pipelines.py@@ -2,6 +2,7 @@Some pipelines used for testing"""+class ZeroDivisionErrorPipeline(object):def open_spider(self, spider):diff --git a/tests/spiders.py b/tests/spiders.pyindex 7816bf7c7..2487ecc22 100644--- a/tests/spiders.py+++ b/tests/spiders.py@@ -16,6 +16,7 @@ class MockServerSpider(Spider):super(MockServerSpider, self).__init__(*args, **kwargs)self.mockserver = mockserver+class MetaSpider(MockServerSpider):name = 'meta'diff --git a/tests/test_cmdline/extensions.py b/tests/test_cmdline/extensions.pyindex 28456b55d..c64e87d81 100644--- a/tests/test_cmdline/extensions.py+++ b/tests/test_cmdline/extensions.py@@ -1,5 +1,6 @@"""A test extension used to check the settings loading order"""+class TestExtension(object):def __init__(self, settings):diff --git a/tests/test_command_parse.py b/tests/test_command_parse.pyindex b134beb88..b7035fdff 100644--- a/tests/test_command_parse.py+++ b/tests/test_command_parse.py@@ -12,6 +12,7 @@ def _textmode(bstr):and reading from it in text mode"""return to_unicode(bstr).replace(os.linesep, '\n')+class ParseCommandTest(ProcessTest, SiteTest, CommandTest):command = 'parse'diff --git a/tests/test_dependencies.py b/tests/test_dependencies.pyindex 03bf2ffcf..e31ccd9b5 100644--- a/tests/test_dependencies.py+++ b/tests/test_dependencies.py@@ -1,6 +1,7 @@from importlib import import_modulefrom twisted.trial import unittest+class ScrapyUtilsTest(unittest.TestCase):def test_required_openssl_version(self):try:diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.pyindex 493691ea4..5a56c9db2 100644--- a/tests/test_downloadermiddleware_ajaxcrawlable.py+++ b/tests/test_downloadermiddleware_ajaxcrawlable.py@@ -5,8 +5,10 @@ from scrapy.spiders import Spiderfrom scrapy.http import Request, HtmlResponse, Responsefrom scrapy.utils.test import get_crawler+__doctests__ = ['scrapy.downloadermiddlewares.ajaxcrawl']+class AjaxCrawlMiddlewareTest(unittest.TestCase):def setUp(self):crawler = get_crawler(Spider, {'AJAXCRAWL_ENABLED': True})diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.pyindex 9d863b6e3..00e6c685e 100644--- a/tests/test_downloadermiddleware_httpcache.py+++ b/tests/test_downloadermiddleware_httpcache.py@@ -149,6 +149,7 @@ class FilesystemStorageTest(DefaultStorageTest):storage_class = 'scrapy.extensions.httpcache.FilesystemCacheStorage'+class FilesystemStorageGzipTest(FilesystemStorageTest):def _get_settings(self, **new_settings):diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.pyindex d7eb98c97..e4b0bdf83 100644--- a/tests/test_dupefilters.py+++ b/tests/test_dupefilters.py@@ -12,6 +12,7 @@ from scrapy.utils.job import job_dirfrom scrapy.utils.test import get_crawlerfrom tests.spiders import SimpleSpider+class FromCrawlerRFPDupeFilter(RFPDupeFilter):@classmethoddiff --git a/tests/test_http_headers.py b/tests/test_http_headers.pyindex 69d906fbf..c83cf3b66 100644--- a/tests/test_http_headers.py+++ b/tests/test_http_headers.py@@ -3,6 +3,7 @@ import copyfrom scrapy.http import Headers+class HeadersTest(unittest.TestCase):def assertSortedEqual(self, first, second, msg=None):diff --git a/tests/test_logformatter.py b/tests/test_logformatter.pyindex b4ea30bb7..0724d1807 100644--- a/tests/test_logformatter.py+++ b/tests/test_logformatter.py@@ -118,6 +118,7 @@ class DropSomeItemsPipeline(object):else:self.drop = True+class ShowOrSkipMessagesTestCase(TwistedTestCase):def setUp(self):self.mockserver = MockServer()diff --git a/tests/test_mail.py b/tests/test_mail.pyindex b139e98d8..ddb0f1e70 100644--- a/tests/test_mail.py+++ b/tests/test_mail.py@@ -6,6 +6,7 @@ from email.charset import Charsetfrom scrapy.mail import MailSender+class MailSenderTest(unittest.TestCase):def test_send(self):diff --git a/tests/test_middleware.py b/tests/test_middleware.pyindex af9b43d61..ebf817c7e 100644--- a/tests/test_middleware.py+++ b/tests/test_middleware.py@@ -4,6 +4,7 @@ from scrapy.settings import Settingsfrom scrapy.exceptions import NotConfiguredfrom scrapy.middleware import MiddlewareManager+class M1(object):def open_spider(self, spider):@@ -15,6 +16,7 @@ class M1(object):def process(self, response, request, spider):pass+class M2(object):def open_spider(self, spider):@@ -25,6 +27,7 @@ class M2(object):pass+class M3(object):def process(self, response, request, spider):@@ -54,6 +57,7 @@ class TestMiddlewareManager(MiddlewareManager):if hasattr(mw, 'process'):self.methods['process'].append(mw.process)+class MiddlewareManagerTest(unittest.TestCase):def test_init(self):diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.pyindex f89042b3d..d5a3371ab 100644--- a/tests/test_responsetypes.py+++ b/tests/test_responsetypes.py@@ -4,6 +4,7 @@ from scrapy.responsetypes import responsetypesfrom scrapy.http import Response, TextResponse, XmlResponse, HtmlResponse, Headers+class ResponseTypesTest(unittest.TestCase):def test_from_filename(self):diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.pyindex cd7480e33..080507276 100644--- a/tests/test_robotstxt_interface.py+++ b/tests/test_robotstxt_interface.py@@ -19,6 +19,7 @@ def rerp_available():return Falsereturn True+def protego_available():# check if protego parser is installedtry:@@ -27,6 +28,7 @@ def protego_available():return Falsereturn True+class BaseRobotParserTest:def _setUp(self, parser_cls):self.parser_cls = parser_clsdiff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.pyindex 106da798c..d8be6e277 100644--- a/tests/test_spiderloader/__init__.py+++ b/tests/test_spiderloader/__init__.py@@ -109,6 +109,7 @@ class SpiderLoaderTest(unittest.TestCase):spiders = spider_loader.list()self.assertEqual(spiders, [])+class DuplicateSpiderNameLoaderTest(unittest.TestCase):def setUp(self):diff --git a/tests/test_spiderloader/test_spiders/nested/spider4.py b/tests/test_spiderloader/test_spiders/nested/spider4.pyindex 35b71870a..dbd1fb123 100644--- a/tests/test_spiderloader/test_spiders/nested/spider4.py+++ b/tests/test_spiderloader/test_spiders/nested/spider4.py@@ -1,5 +1,6 @@from scrapy.spiders import Spider+class Spider4(Spider):name = "spider4"allowed_domains = ['spider4.com']diff --git a/tests/test_spiderloader/test_spiders/spider0.py b/tests/test_spiderloader/test_spiders/spider0.pyindex 75a90794e..af679dbd6 100644--- a/tests/test_spiderloader/test_spiders/spider0.py+++ b/tests/test_spiderloader/test_spiders/spider0.py@@ -1,4 +1,5 @@from scrapy.spiders import Spider+class Spider0(Spider):allowed_domains = ["scrapy1.org", "scrapy3.org"]diff --git a/tests/test_spiderloader/test_spiders/spider1.py b/tests/test_spiderloader/test_spiders/spider1.pyindex 76efddc7f..6b4317a90 100644--- a/tests/test_spiderloader/test_spiders/spider1.py+++ b/tests/test_spiderloader/test_spiders/spider1.py@@ -1,5 +1,6 @@from scrapy.spiders import Spider+class Spider1(Spider):name = "spider1"allowed_domains = ["scrapy1.org", "scrapy3.org"]diff --git a/tests/test_spiderloader/test_spiders/spider2.py b/tests/test_spiderloader/test_spiders/spider2.pyindex 0badd8437..352601863 100644--- a/tests/test_spiderloader/test_spiders/spider2.py+++ b/tests/test_spiderloader/test_spiders/spider2.py@@ -1,5 +1,6 @@from scrapy.spiders import Spider+class Spider2(Spider):name = "spider2"allowed_domains = ["scrapy2.org", "scrapy3.org"]diff --git a/tests/test_spiderloader/test_spiders/spider3.py b/tests/test_spiderloader/test_spiders/spider3.pyindex d406f2d4f..84998ba35 100644--- a/tests/test_spiderloader/test_spiders/spider3.py+++ b/tests/test_spiderloader/test_spiders/spider3.py@@ -1,5 +1,6 @@from scrapy.spiders import Spider+class Spider3(Spider):name = "spider3"allowed_domains = ['spider3.com']diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.pyindex 7e4af0d4c..b97d9b675 100644--- a/tests/test_spidermiddleware_offsite.py+++ b/tests/test_spidermiddleware_offsite.py@@ -9,6 +9,7 @@ from scrapy.spidermiddlewares.offsite import URLWarningfrom scrapy.utils.test import get_crawlerimport warnings+class TestOffsiteMiddleware(TestCase):def setUp(self):@@ -53,6 +54,7 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware):out = list(self.mw.process_spider_output(res, reqs, self.spider))self.assertEqual(out, reqs)+class TestOffsiteMiddleware3(TestOffsiteMiddleware2):def _get_spider(self):diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.pyindex 6f8727a15..940e31070 100644--- a/tests/test_spidermiddleware_output_chain.py+++ b/tests/test_spidermiddleware_output_chain.py@@ -34,6 +34,7 @@ class RecoverySpider(Spider):if not response.meta.get('dont_fail'):raise TabError()+class RecoveryMiddleware:def process_spider_exception(self, response, exception, spider):spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__)@@ -50,6 +51,7 @@ class FailProcessSpiderInputMiddleware:spider.logger.info('Middleware: will raise IndexError')raise IndexError()+class ProcessSpiderInputSpiderWithoutErrback(Spider):name = 'ProcessSpiderInputSpiderWithoutErrback'custom_settings = {@@ -177,6 +179,7 @@ class GeneratorRecoverMiddleware:spider.logger.info('%s: %s caught', method, exception.__class__.__name__)yield {'processed': [method]}+class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware):pass@@ -247,6 +250,7 @@ class NotGeneratorRecoverMiddleware:spider.logger.info('%s: %s caught', method, exception.__class__.__name__)return [{'processed': [method]}]+class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddleware):passdiff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.pyindex 21439c20e..a9c31a983 100644--- a/tests/test_spidermiddleware_referer.py+++ b/tests/test_spidermiddleware_referer.py@@ -349,6 +349,7 @@ class TestSettingsCustomPolicy(TestRefererMiddleware):]+# --- Tests using Request meta dict to set policyclass TestRequestMetaDefault(MixinDefault, TestRefererMiddleware):req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT}@@ -518,14 +519,17 @@ class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware):settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()}+class TestPolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware):settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()}+class TestPolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}+class TestPolicyHeaderPredecence004(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):"""The empty string means "no-referrer-when-downgrade"diff --git a/tests/test_squeues.py b/tests/test_squeues.pyindex 3ded5c027..d5fcf2f7f 100644--- a/tests/test_squeues.py+++ b/tests/test_squeues.py@@ -7,16 +7,20 @@ from scrapy.http import Requestfrom scrapy.loader import ItemLoaderfrom scrapy.selector import Selector+class TestItem(Item):name = Field()+def _test_procesor(x):return x + x+class TestLoader(ItemLoader):default_item_class = TestItemname_out = staticmethod(_test_procesor)+def nonserializable_object_test(self):q = self.queue()try:@@ -35,6 +39,7 @@ def nonserializable_object_test(self):sel = Selector(text='<html><body><p>some text</p></body></html>')self.assertRaises(ValueError, q.push, sel)+class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):chunksize = 100000@@ -53,15 +58,19 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):test_nonserializable_object = nonserializable_object_test+class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):chunksize = 1+class ChunkSize2MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):chunksize = 2+class ChunkSize3MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):chunksize = 3+class ChunkSize4MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):chunksize = 4@@ -100,15 +109,19 @@ class PickleFifoDiskQueueTest(MarshalFifoDiskQueueTest):self.assertEqual(r.url, r2.url)assert r2.meta['request'] is r2+class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):chunksize = 1+class ChunkSize2PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):chunksize = 2+class ChunkSize3PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):chunksize = 3+class ChunkSize4PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):chunksize = 4diff --git a/tests/test_utils_console.py b/tests/test_utils_console.pyindex c2211848c..380c41367 100644--- a/tests/test_utils_console.py+++ b/tests/test_utils_console.py@@ -14,6 +14,7 @@ try:except ImportError:ipy = False+class UtilsConsoleTestCase(unittest.TestCase):def test_get_shell_embed_func(self):diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.pyindex 003bb9b02..d642ed3ed 100644--- a/tests/test_utils_defer.py+++ b/tests/test_utils_defer.py@@ -33,14 +33,23 @@ class MustbeDeferredTest(unittest.TestCase):steps.append(2) # add another value, that should be catched by assertEqualreturn dfd+def cb1(value, arg1, arg2):return "(cb1 %s %s %s)" % (value, arg1, arg2)++def cb2(value, arg1, arg2):return defer.succeed("(cb2 %s %s %s)" % (value, arg1, arg2))++def cb3(value, arg1, arg2):return "(cb3 %s %s %s)" % (value, arg1, arg2)++def cb_fail(value, arg1, arg2):return Failure(TypeError())++def eb1(failure, arg1, arg2):return "(eb1 %s %s %s)" % (failure.value.__class__.__name__, arg1, arg2)diff --git a/tests/test_utils_http.py b/tests/test_utils_http.pyindex 2524153ea..f9af4bf87 100644--- a/tests/test_utils_http.py+++ b/tests/test_utils_http.py@@ -2,6 +2,7 @@ import unittestfrom scrapy.utils.http import decode_chunked_transfer+class ChunkedTest(unittest.TestCase):def test_decode_chunked_transfer(self):diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.pyindex 4f9f7a370..2c3965bbc 100644--- a/tests/test_utils_httpobj.py+++ b/tests/test_utils_httpobj.py@@ -4,6 +4,7 @@ from six.moves.urllib.parse import urlparsefrom scrapy.http import Requestfrom scrapy.utils.httpobj import urlparse_cached+class HttpobjUtilsTest(unittest.TestCase):def test_urlparse_cached(self):diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.pyindex 2d845697e..f16ef8110 100644--- a/tests/test_utils_iterators.py+++ b/tests/test_utils_iterators.py@@ -235,6 +235,7 @@ class LxmlXmliterTestCase(XmliterTestCase):i = self.xmliter(42, 'product')self.assertRaises(TypeError, next, i)+class UtilsCsvTestCase(unittest.TestCase):sample_feeds_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'feeds')sample_feed_path = os.path.join(sample_feeds_dir, 'feed-sample3.csv')diff --git a/tests/test_utils_request.py b/tests/test_utils_request.pyindex 625a32048..3da95b95a 100644--- a/tests/test_utils_request.py+++ b/tests/test_utils_request.py@@ -4,6 +4,7 @@ from scrapy.http import Requestfrom scrapy.utils.request import request_fingerprint, _fingerprint_cache, \request_authenticate, request_httprepr+class UtilsRequestTest(unittest.TestCase):def test_request_fingerprint(self):diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.pyindex 62edd420d..16b7c5c68 100644--- a/tests/test_utils_signal.py+++ b/tests/test_utils_signal.py@@ -66,6 +66,7 @@ class SendCatchLogDeferredTest2(SendCatchLogTest):def _get_result(self, signal, *a, **kw):return send_catch_log_deferred(signal, *a, **kw)+class SendCatchLogTest2(unittest.TestCase):def test_error_logged_if_deferred_not_supported(self):diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.pyindex 716bb44eb..db323ab31 100644--- a/tests/test_utils_sitemap.py+++ b/tests/test_utils_sitemap.py@@ -2,6 +2,7 @@ import unittestfrom scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots+class SitemapTest(unittest.TestCase):def test_sitemap(self):diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.pyindex d9de1ce77..edeeacc80 100644--- a/tests/test_utils_spider.py+++ b/tests/test_utils_spider.py@@ -9,12 +9,15 @@ from scrapy.spiders import CrawlSpiderclass MyBaseSpider(CrawlSpider):pass # abstract spider+class MySpider1(MyBaseSpider):name = 'myspider1'+class MySpider2(MyBaseSpider):name = 'myspider2'+class UtilsSpidersTestCase(unittest.TestCase):def test_iterate_spider_output(self):diff --git a/tests/test_utils_url.py b/tests/test_utils_url.pyindex e6588055c..93addc082 100644--- a/tests/test_utils_url.py+++ b/tests/test_utils_url.py@@ -187,6 +187,7 @@ class AddHttpIfNoScheme(unittest.TestCase):class GuessSchemeTest(unittest.TestCase):pass+def create_guess_scheme_t(args):def do_expected(self):url = guess_scheme(args[0])@@ -195,6 +196,7 @@ def create_guess_scheme_t(args):args[0], url, args[1])return do_expected+def create_skipped_scheme_t(args):def do_expected(self):raise unittest.SkipTest(args[2])