Scrapy, a fast high-level web crawling & scraping framework for Python.
Improve test coverage for contracts (#7677)
Adrian0ccddb4f61b2
diff --git a/tests/test_contracts.py b/tests/test_contracts.pyindex e80945b93..698044de7 100644--- a/tests/test_contracts.py+++ b/tests/test_contracts.py@@ -133,6 +133,29 @@ class DemoSpider(Spider):"""return DemoItem(url=response.url)+ def returns_request_range_fail(self, response):+ """method which returns fewer requests than the expected range+ @url http://scrapy.org+ @returns requests 2 3+ """+ return Request("http://scrapy.org", callback=self.returns_item)++ def yields_item_and_request(self, response):+ """yields one item and one request+ @url http://scrapy.org+ @returns items 1 1+ @scrapes name url+ """+ yield DemoItem(name="test", url=response.url)+ yield Request("http://scrapy.org", callback=self.returns_item)++ async def returns_async_gen(self, response):+ """async generator callback+ @url http://scrapy.org+ @returns items 1 1+ """+ yield DemoItem(url=response.url)+def returns_dict_fail(self, response):"""method which returns item@url http://scrapy.org@@ -437,6 +460,48 @@ class TestContractsManager:request.callback(response)self.should_error()+ def test_returns_invalid_argument_count(self):+ spider = DemoSpider()+ with pytest.raises(ValueError, match="expected 1, 2 or 3, got 0"):+ ReturnsContract(spider.returns_item)+ with pytest.raises(ValueError, match="expected 1, 2 or 3, got 4"):+ ReturnsContract(spider.returns_item, "items", "1", "2", "3")++ def test_returns_default_bounds(self):+ spider = DemoSpider()+ contract = ReturnsContract(spider.returns_item, "items")+ assert contract.min_bound == 1+ assert contract.max_bound == float("inf")++ def test_returns_range_fail(self):+ spider = DemoSpider()+ response = ResponseMock()++ request = self.conman.from_method(+ spider.returns_request_range_fail, self.results+ )+ request.callback(response)+ self.should_fail()+ assert "expected 2..3" in self.results.failures[-1][-1]++ def test_returns_and_scrapes_ignore_other_types(self):+ spider = DemoSpider()+ response = ResponseMock()++ # @returns and @scrapes only count matching output objects and skip+ # the request that is also yielded.+ request = self.conman.from_method(spider.yields_item_and_request, self.results)+ request.callback(response)+ self.should_succeed()++ def test_testcase_str(self):+ spider = DemoSpider()+ contract = UrlContract(spider.returns_request, "http://scrapy.org")+ assert (+ str(contract.testcase_pre)+ == "[demo_spider] returns_request (@url pre-hook)"+ )+def test_scrapes(self):spider = DemoSpider()response = ResponseMock()@@ -570,6 +635,41 @@ class CustomFailContractPostProcess(Contract):raise KeyboardInterrupt("Post-process exception")+class PreProcessSuccessContract(Contract):+ name = "pre_success"++ def pre_process(self, response):+ return+++class PreProcessAssertionFailContract(Contract):+ name = "pre_assertion_fail"++ def pre_process(self, response):+ raise AssertionError("pre-process assertion")+++class PreProcessErrorContract(Contract):+ name = "pre_error"++ def pre_process(self, response):+ raise ValueError("pre-process error")+++class PostProcessSuccessContract(Contract):+ name = "post_success"++ def post_process(self, output):+ return+++class PostProcessErrorContract(Contract):+ name = "post_error"++ def post_process(self, output):+ raise ValueError("post-process error")++class TestCustomContractPrePostProcess:def setup_method(self):self.results = TextTestResult(stream=None, descriptions=False, verbosity=0)@@ -601,3 +701,83 @@ class TestCustomContractPrePostProcess:assert not self.results.failuresassert not self.results.errors++ def test_pre_hook_success(self):+ spider = DemoSpider()+ response = ResponseMock()+ contract = PreProcessSuccessContract(spider.returns_request)+ conman = ContractsManager([UrlContract, ReturnsContract, contract])++ request = conman.from_method(spider.returns_request, self.results)+ contract.add_pre_hook(request, self.results)+ request.callback(response, **request.cb_kwargs)++ assert not self.results.failures+ assert not self.results.errors++ def test_pre_hook_assertion_failure(self):+ spider = DemoSpider()+ response = ResponseMock()+ contract = PreProcessAssertionFailContract(spider.returns_request)+ conman = ContractsManager([UrlContract, ReturnsContract, contract])++ request = conman.from_method(spider.returns_request, self.results)+ contract.add_pre_hook(request, self.results)+ request.callback(response, **request.cb_kwargs)++ assert self.results.failures+ assert not self.results.errors++ def test_pre_hook_error(self):+ spider = DemoSpider()+ response = ResponseMock()+ contract = PreProcessErrorContract(spider.returns_request)+ conman = ContractsManager([UrlContract, ReturnsContract, contract])++ request = conman.from_method(spider.returns_request, self.results)+ contract.add_pre_hook(request, self.results)+ request.callback(response, **request.cb_kwargs)++ assert self.results.errors++ def test_pre_hook_async_callback(self):+ spider = DemoSpider()+ response = ResponseMock()+ contract = PreProcessSuccessContract(spider.returns_request_async)+ request = Request("http://scrapy.org", callback=spider.returns_request_async)+ contract.add_pre_hook(request, self.results)++ with pytest.raises(TypeError, match="async callbacks"):+ request.callback(response)++ def test_pre_hook_async_generator(self):+ spider = DemoSpider()+ response = ResponseMock()+ contract = PreProcessSuccessContract(spider.returns_async_gen)+ request = Request("http://scrapy.org", callback=spider.returns_async_gen)+ contract.add_pre_hook(request, self.results)++ with pytest.raises(TypeError, match="async callbacks"):+ request.callback(response)++ def test_post_hook_async_generator(self):+ spider = DemoSpider()+ response = ResponseMock()+ contract = PostProcessSuccessContract(spider.returns_async_gen)+ request = Request("http://scrapy.org", callback=spider.returns_async_gen)+ contract.add_post_hook(request, self.results)++ with pytest.raises(TypeError, match="async callbacks"):+ request.callback(response)++ def test_post_hook_error(self):+ spider = DemoSpider()+ response = ResponseMock()+ contract = PostProcessErrorContract(spider.returns_request)+ conman = ContractsManager([UrlContract, ReturnsContract, contract])++ request = conman.from_method(spider.returns_request, self.results)+ contract.add_post_hook(request, self.results)+ request.callback(response, **request.cb_kwargs)++ assert self.results.errors