Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

New signal update signature documentation for new signal utilize new signal correct signal handler signature emit new signal test another signal new signal rename test file faster test rename test case tests for signal emitting in bad cases

Vostretsov Nikita3b1db71dac87
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index ff07b9d55..f13e8270c 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -295,6 +295,23 @@ request_reached_downloader
:param spider: the spider that yielded the request
:type spider: :class:`~scrapy.spiders.Spider` object
+request_left_downloader
+---------------------------
+
+.. signal:: request_left_downloader
+.. function:: request_left_downloader(request, spider)
+
+ Sent when a :class:`~scrapy.http.Request` left downloader even in case of
+ failure.
+
+ The signal does not support returning deferreds from their handlers.
+
+ :param request: the request that reached downloader
+ :type request: :class:`~scrapy.http.Request` object
+
+ :param spider: the spider that yielded the request
+ :type spider: :class:`~scrapy.spiders.Spider` object
+
response_received
-----------------
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 4695d75f4..d856a2f37 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -188,6 +188,9 @@ class Downloader(object):
def finish_transferring(_):
slot.transferring.remove(request)
self._process_queue(spider, slot)
+ self.signals.send_catch_log(signal=signals.request_left_downloader,
+ request=request,
+ spider=spider)
return _
return dfd.addBoth(finish_transferring)
diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index d8eed010f..6a9feb599 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -7,7 +7,7 @@ from queuelib import PriorityQueue
from scrapy.core.downloader import Downloader
from scrapy.http import Request
-from scrapy.signals import request_reached_downloader, response_downloaded
+from scrapy.signals import request_reached_downloader, request_left_downloader
from scrapy.utils.httpobj import urlparse_cached
@@ -163,7 +163,7 @@ class DownloaderAwarePriorityQueue(object):
self._active_downloads = {slot: 0 for slot in self._slot_pqueues.pqueues}
crawler.signals.connect(self.on_response_download,
- signal=response_downloaded)
+ signal=request_left_downloader)
crawler.signals.connect(self.on_request_reached_downloader,
signal=request_reached_downloader)
@@ -199,7 +199,7 @@ class DownloaderAwarePriorityQueue(object):
if slot not in self._active_downloads:
self._active_downloads[slot] = 0
- def on_response_download(self, response, request, spider):
+ def on_response_download(self, request, spider):
if not self.check_mark(request):
return
self.unmark(request)
diff --git a/scrapy/signals.py b/scrapy/signals.py
index c0e4bb74e..2ea986b8c 100644
--- a/scrapy/signals.py
+++ b/scrapy/signals.py
@@ -14,6 +14,7 @@ spider_error = object()
request_scheduled = object()
request_dropped = object()
request_reached_downloader = object()
+request_left_downloader = object()
response_received = object()
response_downloaded = object()
item_scraped = object()
diff --git a/tests/test_request_left.py b/tests/test_request_left.py
new file mode 100644
index 000000000..ddeca0499
--- /dev/null
+++ b/tests/test_request_left.py
@@ -0,0 +1,59 @@
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase
+from scrapy.signals import request_left_downloader
+from scrapy.spiders import Spider
+from scrapy.utils.test import get_crawler
+from tests.mockserver import MockServer
+
+class SignalCatcherSpider(Spider):
+ name = 'signal_catcher'
+
+ def __init__(self, crawler, url, *args, **kwargs):
+ super(SignalCatcherSpider, self).__init__(*args, **kwargs)
+ crawler.signals.connect(self.on_response_download,
+ signal=request_left_downloader)
+ self.catched_times = 0
+ self.start_urls = [url]
+
+ @classmethod
+ def from_crawler(cls, crawler, *args, **kwargs):
+ spider = cls(crawler, *args, **kwargs)
+ return spider
+
+ def on_response_download(self, request, spider):
+ self.catched_times = self.catched_times + 1
+
+
+class TestCatching(TestCase):
+
+ def setUp(self):
+ self.mockserver = MockServer()
+ self.mockserver.__enter__()
+
+ def tearDown(self):
+ self.mockserver.__exit__(None, None, None)
+
+ @defer.inlineCallbacks
+ def test_success(self):
+ crawler = get_crawler(SignalCatcherSpider)
+ yield crawler.crawl(self.mockserver.url("/status?n=200"))
+ self.assertEqual(crawler.spider.catched_times, 1)
+
+ @defer.inlineCallbacks
+ def test_timeout(self):
+ crawler = get_crawler(SignalCatcherSpider,
+ {'DOWNLOAD_TIMEOUT': 0.1})
+ yield crawler.crawl(self.mockserver.url("/delay?n=0.2"))
+ self.assertEqual(crawler.spider.catched_times, 1)
+
+ @defer.inlineCallbacks
+ def test_disconnect(self):
+ crawler = get_crawler(SignalCatcherSpider)
+ yield crawler.crawl(self.mockserver.url("/drop"))
+ self.assertEqual(crawler.spider.catched_times, 1)
+
+ @defer.inlineCallbacks
+ def test_noconnect(self):
+ crawler = get_crawler(SignalCatcherSpider)
+ yield crawler.crawl('http://thereisdefinetelynosuchdomain.com')
+ self.assertEqual(crawler.spider.catched_times, 1)
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 3fb70a110..1bcc1e5a8 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -9,7 +9,7 @@ from scrapy.crawler import Crawler
from scrapy.core.scheduler import Scheduler
from scrapy.http import Request
from scrapy.pqueues import _scheduler_slot_read, _scheduler_slot_write
-from scrapy.signals import request_reached_downloader, response_downloaded
+from scrapy.signals import request_reached_downloader, request_left_downloader
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
from tests.mockserver import MockServer
@@ -216,9 +216,8 @@ class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester,
for request in requests:
self.mock_crawler.signals.send_catch_log(
- signal=response_downloaded,
+ signal=request_left_downloader,
request=request,
- response=None,
spider=self.spider
)
@@ -265,9 +264,8 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester,
for request in requests:
self.mock_crawler.signals.send_catch_log(
- signal=response_downloaded,
+ signal=request_left_downloader,
request=request,
- response=None,
spider=self.spider
)