Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Check for async callbacks in contracts.

Andrey Rakhmatullin518e56046e45
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 86098edca..1ec2a0234 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -2,7 +2,8 @@ import re
import sys
from functools import wraps
from inspect import getmembers
-from typing import Dict
+from types import CoroutineType
+from typing import AsyncGenerator, Dict
from unittest import TestCase
from scrapy.http import Request
@@ -37,7 +38,10 @@ class Contract:
else:
results.addSuccess(self.testcase_pre)
finally:
- return list(iterate_spider_output(cb(response, **cb_kwargs)))
+ cb_result = cb(response, **cb_kwargs)
+ if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
+ raise TypeError("Contracts don't support async callbacks")
+ return list(iterate_spider_output(cb_result))
request.callback = wrapper
@@ -49,7 +53,10 @@ class Contract:
@wraps(cb)
def wrapper(response, **cb_kwargs):
- output = list(iterate_spider_output(cb(response, **cb_kwargs)))
+ cb_result = cb(response, **cb_kwargs)
+ if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
+ raise TypeError("Contracts don't support async callbacks")
+ output = list(iterate_spider_output(cb_result))
try:
results.startTest(self.testcase_post)
self.post_process(output)
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index 813927fc5..1459e0b5f 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -63,6 +63,13 @@ class TestSpider(Spider):
"""
return Request("http://scrapy.org", callback=self.returns_item)
+ async def returns_request_async(self, response):
+ """async method which returns request
+ @url http://scrapy.org
+ @returns requests 1
+ """
+ return Request("http://scrapy.org", callback=self.returns_item)
+
def returns_item(self, response):
"""method which returns item
@url http://scrapy.org
@@ -337,6 +344,14 @@ class ContractsManagerTest(unittest.TestCase):
request.callback(response)
self.should_fail()
+ def test_returns_async(self):
+ spider = TestSpider()
+ response = ResponseMock()
+
+ request = self.conman.from_method(spider.returns_request_async, self.results)
+ request.callback(response)
+ self.should_error()
+
def test_scrapes(self):
spider = TestSpider()
response = ResponseMock()