Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge branch 'master' into spider.parse
Adrián Chaves24bb9fd5f7ae
diff --cc tests/test_crawl.pyindex 6247ced35,85005eba4..f1d502ff7--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@@ -9,9 -11,9 +11,19 @@@ from scrapy.crawler import CrawlerRunnefrom scrapy.http import Requestfrom scrapy.utils.python import to_unicodefrom tests.mockserver import MockServer- from tests.spiders import (BrokenStartRequestsSpider, CrawlSpiderWithErrback,- CrawlSpiderWithParseMethod, DelaySpider, SimpleSpider,- DuplicateStartRequestsSpider, FollowAllSpider, SingleRequestSpider)-from tests.spiders import (FollowAllSpider, DelaySpider, SimpleSpider, BrokenStartRequestsSpider,- SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback,- AsyncDefSpider, AsyncDefAsyncioSpider, AsyncDefAsyncioReturnSpider)++from tests.spiders import (++ AsyncDefAsyncioReturnSpider,++ AsyncDefAsyncioSpider,++ AsyncDefSpider,++ BrokenStartRequestsSpider,++ CrawlSpiderWithErrback,++ CrawlSpiderWithParseMethod,++ DelaySpider,++ DuplicateStartRequestsSpider,++ FollowAllSpider,++ SimpleSpider,++ SingleRequestSpider,++)class CrawlTestCase(TestCase):@@@ -327,9 -307,39 +339,41 @@@ class CrawlSpiderTestCase(TestCase)with LogCapture() as log:yield self.runner.join()- self.assertIn("[callback] status 200", str(log))- self.assertIn("[callback] status 201", str(log))+ self.assertIn("[parse] status 200 (foo: None)", str(log))+ self.assertIn("[parse] status 201 (foo: None)", str(log))+ self.assertIn("[parse] status 202 (foo: bar)", str(log))self.assertIn("[errback] status 404", str(log))self.assertIn("[errback] status 500", str(log))+ self.assertIn("[errback] status 501", str(log))++ @defer.inlineCallbacks+ def test_async_def_parse(self):+ self.runner.crawl(AsyncDefSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)+ with LogCapture() as log:+ yield self.runner.join()+ self.assertIn("Got response 200", str(log))++ @mark.only_asyncio()+ @defer.inlineCallbacks+ def test_async_def_asyncio_parse(self):+ runner = CrawlerRunner({"ASYNCIO_REACTOR": True})+ runner.crawl(AsyncDefAsyncioSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)+ with LogCapture() as log:+ yield runner.join()+ self.assertIn("Got response 200", str(log))++ @mark.only_asyncio()+ @defer.inlineCallbacks+ def test_async_def_asyncio_parse_list(self):+ items = []++ def _on_item_scraped(item):+ items.append(item)++ crawler = self.runner.create_crawler(AsyncDefAsyncioReturnSpider)+ crawler.signals.connect(_on_item_scraped, signals.item_scraped)+ with LogCapture() as log:+ yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)+ self.assertIn("Got response 200", str(log))+ self.assertIn({'id': 1}, items)+ self.assertIn({'id': 2}, items)