Scrapy, a fast high-level web crawling & scraping framework for Python.
Tests: use classes instead of paths in settings (#4817)
Eugenio Lacuesta797a6690c07a
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.pyindex 95a4fca0d..680bb6dc8 100644--- a/tests/test_dupefilters.py+++ b/tests/test_dupefilters.py@@ -43,7 +43,7 @@ class RFPDupeFilterTest(unittest.TestCase):def test_df_from_crawler_scheduler(self):settings = {'DUPEFILTER_DEBUG': True,- 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}+ 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}crawler = get_crawler(settings_dict=settings)scheduler = Scheduler.from_crawler(crawler)self.assertTrue(scheduler.df.debug)@@ -51,14 +51,14 @@ class RFPDupeFilterTest(unittest.TestCase):def test_df_from_settings_scheduler(self):settings = {'DUPEFILTER_DEBUG': True,- 'DUPEFILTER_CLASS': __name__ + '.FromSettingsRFPDupeFilter'}+ 'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter}crawler = get_crawler(settings_dict=settings)scheduler = Scheduler.from_crawler(crawler)self.assertTrue(scheduler.df.debug)self.assertEqual(scheduler.df.method, 'from_settings')def test_df_direct_scheduler(self):- settings = {'DUPEFILTER_CLASS': __name__ + '.DirectDupeFilter'}+ settings = {'DUPEFILTER_CLASS': DirectDupeFilter}crawler = get_crawler(settings_dict=settings)scheduler = Scheduler.from_crawler(crawler)self.assertEqual(scheduler.df.method, 'n/a')@@ -162,7 +162,7 @@ class RFPDupeFilterTest(unittest.TestCase):def test_log(self):with LogCapture() as log:settings = {'DUPEFILTER_DEBUG': False,- 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}+ 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}crawler = get_crawler(SimpleSpider, settings_dict=settings)scheduler = Scheduler.from_crawler(crawler)spider = SimpleSpider.from_crawler(crawler)@@ -191,7 +191,7 @@ class RFPDupeFilterTest(unittest.TestCase):def test_log_debug(self):with LogCapture() as log:settings = {'DUPEFILTER_DEBUG': True,- 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}+ 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}crawler = get_crawler(SimpleSpider, settings_dict=settings)scheduler = Scheduler.from_crawler(crawler)spider = SimpleSpider.from_crawler(crawler)diff --git a/tests/test_logformatter.py b/tests/test_logformatter.pyindex dc5be398f..6381f895b 100644--- a/tests/test_logformatter.py+++ b/tests/test_logformatter.py@@ -193,7 +193,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase):self.base_settings = {'LOG_LEVEL': 'DEBUG','ITEM_PIPELINES': {- __name__ + '.DropSomeItemsPipeline': 300,+ DropSomeItemsPipeline: 300,},}@@ -212,7 +212,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase):@defer.inlineCallbacksdef test_skip_messages(self):settings = self.base_settings.copy()- settings['LOG_FORMATTER'] = __name__ + '.SkipMessagesLogFormatter'+ settings['LOG_FORMATTER'] = SkipMessagesLogFormattercrawler = CrawlerRunner(settings).create_crawler(ItemSpider)with LogCapture() as lc:yield crawler.crawl(mockserver=self.mockserver)diff --git a/tests/test_pipelines.py b/tests/test_pipelines.pyindex c72f1a338..ff3af9a74 100644--- a/tests/test_pipelines.py+++ b/tests/test_pipelines.py@@ -68,7 +68,7 @@ class PipelineTestCase(unittest.TestCase):def _create_crawler(self, pipeline_class):settings = {- 'ITEM_PIPELINES': {__name__ + '.' + pipeline_class.__name__: 1},+ 'ITEM_PIPELINES': {pipeline_class: 1},}crawler = get_crawler(ItemSpider, settings)crawler.signals.connect(self._on_item_scraped, signals.item_scraped)diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.pyindex 907117468..00c532c41 100644--- a/tests/test_request_attribute_binding.py+++ b/tests/test_request_attribute_binding.py@@ -92,7 +92,7 @@ class CrawlTestCase(TestCase):url = self.mockserver.url("/status?n=200")runner = CrawlerRunner(settings={"DOWNLOADER_MIDDLEWARES": {- __name__ + ".RaiseExceptionRequestMiddleware": 590,+ RaiseExceptionRequestMiddleware: 590,},})crawler = runner.create_crawler(SingleRequestSpider)@@ -119,7 +119,7 @@ class CrawlTestCase(TestCase):url = self.mockserver.url("/status?n=200")runner = CrawlerRunner(settings={"DOWNLOADER_MIDDLEWARES": {- __name__ + ".ProcessResponseMiddleware": 595,+ ProcessResponseMiddleware: 595,}})crawler = runner.create_crawler(SingleRequestSpider)@@ -149,8 +149,8 @@ class CrawlTestCase(TestCase):url = self.mockserver.url("/status?n=200")runner = CrawlerRunner(settings={"DOWNLOADER_MIDDLEWARES": {- __name__ + ".RaiseExceptionRequestMiddleware": 590,- __name__ + ".CatchExceptionOverrideRequestMiddleware": 595,+ RaiseExceptionRequestMiddleware: 590,+ CatchExceptionOverrideRequestMiddleware: 595,},})crawler = runner.create_crawler(SingleRequestSpider)@@ -170,8 +170,8 @@ class CrawlTestCase(TestCase):url = self.mockserver.url("/status?n=200")runner = CrawlerRunner(settings={"DOWNLOADER_MIDDLEWARES": {- __name__ + ".RaiseExceptionRequestMiddleware": 590,- __name__ + ".CatchExceptionDoNotOverrideRequestMiddleware": 595,+ RaiseExceptionRequestMiddleware: 590,+ CatchExceptionDoNotOverrideRequestMiddleware: 595,},})crawler = runner.create_crawler(SingleRequestSpider)@@ -188,7 +188,7 @@ class CrawlTestCase(TestCase):"""runner = CrawlerRunner(settings={"DOWNLOADER_MIDDLEWARES": {- __name__ + ".AlternativeCallbacksMiddleware": 595,+ AlternativeCallbacksMiddleware: 595,}})crawler = runner.create_crawler(AlternativeCallbacksSpider)diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.pyindex bd49179aa..145a4e9b2 100644--- a/tests/test_request_cb_kwargs.py+++ b/tests/test_request_cb_kwargs.py@@ -50,10 +50,10 @@ class KeywordArgumentsSpider(MockServerSpider):name = 'kwargs'custom_settings = {'DOWNLOADER_MIDDLEWARES': {- __name__ + '.InjectArgumentsDownloaderMiddleware': 750,+ InjectArgumentsDownloaderMiddleware: 750,},'SPIDER_MIDDLEWARES': {- __name__ + '.InjectArgumentsSpiderMiddleware': 750,+ InjectArgumentsSpiderMiddleware: 750,},}diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.pyindex 2f454addc..029bf8bd6 100644--- a/tests/test_spidermiddleware_output_chain.py+++ b/tests/test_spidermiddleware_output_chain.py@@ -16,11 +16,20 @@ class LogExceptionMiddleware:# ================================================================================# (0) recover from an exception on a spider callback+class RecoveryMiddleware:+ def process_spider_exception(self, response, exception, spider):+ spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__)+ return [+ {'from': 'process_spider_exception'},+ Request(response.url, meta={'dont_fail': True}, dont_filter=True),+ ]++class RecoverySpider(Spider):name = 'RecoverySpider'custom_settings = {'SPIDER_MIDDLEWARES': {- __name__ + '.RecoveryMiddleware': 10,+ RecoveryMiddleware: 10,},}@@ -34,15 +43,6 @@ class RecoverySpider(Spider):raise TabError()-class RecoveryMiddleware:- def process_spider_exception(self, response, exception, spider):- spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__)- return [- {'from': 'process_spider_exception'},- Request(response.url, meta={'dont_fail': True}, dont_filter=True),- ]--# ================================================================================# (1) exceptions from a spider middleware's process_spider_input methodclass FailProcessSpiderInputMiddleware:@@ -56,9 +56,8 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider):custom_settings = {'SPIDER_MIDDLEWARES': {# spider- __name__ + '.LogExceptionMiddleware': 10,- __name__ + '.FailProcessSpiderInputMiddleware': 8,- __name__ + '.LogExceptionMiddleware': 6,+ FailProcessSpiderInputMiddleware: 8,+ LogExceptionMiddleware: 6,# engine}}@@ -87,7 +86,7 @@ class GeneratorCallbackSpider(Spider):name = 'GeneratorCallbackSpider'custom_settings = {'SPIDER_MIDDLEWARES': {- __name__ + '.LogExceptionMiddleware': 10,+ LogExceptionMiddleware: 10,},}@@ -106,7 +105,7 @@ class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider)name = 'GeneratorCallbackSpiderMiddlewareRightAfterSpider'custom_settings = {'SPIDER_MIDDLEWARES': {- __name__ + '.LogExceptionMiddleware': 100000,+ LogExceptionMiddleware: 100000,},}@@ -117,7 +116,7 @@ class NotGeneratorCallbackSpider(Spider):name = 'NotGeneratorCallbackSpider'custom_settings = {'SPIDER_MIDDLEWARES': {- __name__ + '.LogExceptionMiddleware': 10,+ LogExceptionMiddleware: 10,},}@@ -134,32 +133,13 @@ class NotGeneratorCallbackSpiderMiddlewareRightAfterSpider(NotGeneratorCallbackSname = 'NotGeneratorCallbackSpiderMiddlewareRightAfterSpider'custom_settings = {'SPIDER_MIDDLEWARES': {- __name__ + '.LogExceptionMiddleware': 100000,+ LogExceptionMiddleware: 100000,},}# ================================================================================# (4) exceptions from a middleware process_spider_output method (generator)-class GeneratorOutputChainSpider(Spider):- name = 'GeneratorOutputChainSpider'- custom_settings = {- 'SPIDER_MIDDLEWARES': {- __name__ + '.GeneratorFailMiddleware': 10,- __name__ + '.GeneratorDoNothingAfterFailureMiddleware': 8,- __name__ + '.GeneratorRecoverMiddleware': 5,- __name__ + '.GeneratorDoNothingAfterRecoveryMiddleware': 3,- },- }-- def start_requests(self):- yield Request(self.mockserver.url('/status?n=200'))-- def parse(self, response):- yield {'processed': ['parse-first-item']}- yield {'processed': ['parse-second-item']}--class _GeneratorDoNothingMiddleware:def process_spider_output(self, response, result, spider):for r in result:@@ -205,25 +185,27 @@ class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware):pass-# ================================================================================-# (5) exceptions from a middleware process_spider_output method (not generator)-class NotGeneratorOutputChainSpider(Spider):- name = 'NotGeneratorOutputChainSpider'+class GeneratorOutputChainSpider(Spider):+ name = 'GeneratorOutputChainSpider'custom_settings = {'SPIDER_MIDDLEWARES': {- __name__ + '.NotGeneratorFailMiddleware': 10,- __name__ + '.NotGeneratorDoNothingAfterFailureMiddleware': 8,- __name__ + '.NotGeneratorRecoverMiddleware': 5,- __name__ + '.NotGeneratorDoNothingAfterRecoveryMiddleware': 3,+ GeneratorFailMiddleware: 10,+ GeneratorDoNothingAfterFailureMiddleware: 8,+ GeneratorRecoverMiddleware: 5,+ GeneratorDoNothingAfterRecoveryMiddleware: 3,},}def start_requests(self):- return [Request(self.mockserver.url('/status?n=200'))]+ yield Request(self.mockserver.url('/status?n=200'))def parse(self, response):- return [{'processed': ['parse-first-item']}, {'processed': ['parse-second-item']}]+ yield {'processed': ['parse-first-item']}+ yield {'processed': ['parse-second-item']}++# ================================================================================+# (5) exceptions from a middleware process_spider_output method (not generator)class _NotGeneratorDoNothingMiddleware:def process_spider_output(self, response, result, spider):@@ -276,6 +258,24 @@ class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddlewpass+class NotGeneratorOutputChainSpider(Spider):+ name = 'NotGeneratorOutputChainSpider'+ custom_settings = {+ 'SPIDER_MIDDLEWARES': {+ NotGeneratorFailMiddleware: 10,+ NotGeneratorDoNothingAfterFailureMiddleware: 8,+ NotGeneratorRecoverMiddleware: 5,+ NotGeneratorDoNothingAfterRecoveryMiddleware: 3,+ },+ }++ def start_requests(self):+ return [Request(self.mockserver.url('/status?n=200'))]++ def parse(self, response):+ return [{'processed': ['parse-first-item']}, {'processed': ['parse-second-item']}]++# ================================================================================class TestSpiderMiddleware(TestCase):@classmethod