Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge branch 'master' into spider.parse

Adrián Chaves24bb9fd5f7ae
diff --cc tests/test_crawl.py
index 6247ced35,85005eba4..f1d502ff7
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@@ -9,9 -11,9 +11,19 @@@ from scrapy.crawler import CrawlerRunne
from scrapy.http import Request
from scrapy.utils.python import to_unicode
from tests.mockserver import MockServer
- from tests.spiders import (BrokenStartRequestsSpider, CrawlSpiderWithErrback,
- CrawlSpiderWithParseMethod, DelaySpider, SimpleSpider,
- DuplicateStartRequestsSpider, FollowAllSpider, SingleRequestSpider)
-from tests.spiders import (FollowAllSpider, DelaySpider, SimpleSpider, BrokenStartRequestsSpider,
- SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback,
- AsyncDefSpider, AsyncDefAsyncioSpider, AsyncDefAsyncioReturnSpider)
++from tests.spiders import (
++ AsyncDefAsyncioReturnSpider,
++ AsyncDefAsyncioSpider,
++ AsyncDefSpider,
++ BrokenStartRequestsSpider,
++ CrawlSpiderWithErrback,
++ CrawlSpiderWithParseMethod,
++ DelaySpider,
++ DuplicateStartRequestsSpider,
++ FollowAllSpider,
++ SimpleSpider,
++ SingleRequestSpider,
++)
class CrawlTestCase(TestCase):
@@@ -327,9 -307,39 +339,41 @@@ class CrawlSpiderTestCase(TestCase)
with LogCapture() as log:
yield self.runner.join()
- self.assertIn("[callback] status 200", str(log))
- self.assertIn("[callback] status 201", str(log))
+ self.assertIn("[parse] status 200 (foo: None)", str(log))
+ self.assertIn("[parse] status 201 (foo: None)", str(log))
+ self.assertIn("[parse] status 202 (foo: bar)", str(log))
self.assertIn("[errback] status 404", str(log))
self.assertIn("[errback] status 500", str(log))
+ self.assertIn("[errback] status 501", str(log))
+
+ @defer.inlineCallbacks
+ def test_async_def_parse(self):
+ self.runner.crawl(AsyncDefSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+ with LogCapture() as log:
+ yield self.runner.join()
+ self.assertIn("Got response 200", str(log))
+
+ @mark.only_asyncio()
+ @defer.inlineCallbacks
+ def test_async_def_asyncio_parse(self):
+ runner = CrawlerRunner({"ASYNCIO_REACTOR": True})
+ runner.crawl(AsyncDefAsyncioSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+ with LogCapture() as log:
+ yield runner.join()
+ self.assertIn("Got response 200", str(log))
+
+ @mark.only_asyncio()
+ @defer.inlineCallbacks
+ def test_async_def_asyncio_parse_list(self):
+ items = []
+
+ def _on_item_scraped(item):
+ items.append(item)
+
+ crawler = self.runner.create_crawler(AsyncDefAsyncioReturnSpider)
+ crawler.signals.connect(_on_item_scraped, signals.item_scraped)
+ with LogCapture() as log:
+ yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+ self.assertIn("Got response 200", str(log))
+ self.assertIn({'id': 1}, items)
+ self.assertIn({'id': 2}, items)