Scrapy, a fast high-level web crawling & scraping framework for Python.
Rule.process_request: deprecate the use of functions taking only one argument
Eugenio Lacuesta56929e77d983
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.pyindex f474b0a18..f469891d0 100644--- a/scrapy/spiders/crawl.py+++ b/scrapy/spiders/crawl.py@@ -6,16 +6,19 @@ See documentation in docs/topics/spiders.rst"""import copy+import warnings+import six+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Request, HtmlResponsefrom scrapy.utils.spider import iterate_spider_outputfrom scrapy.utils.python import get_func_argsfrom scrapy.spiders import Spider-def identity(x):- return x+def _identity(request, response):+ return requestclass Rule(object):@@ -25,19 +28,21 @@ class Rule(object):self.callback = callbackself.cb_kwargs = cb_kwargs or {}self.process_links = process_links- self.process_request = process_request or identity- if follow is None:- self.follow = False if callback else True- else:- self.follow = follow+ self.process_request = process_request or _identity+ self.follow = follow if follow is not None else not callbackdef _process_request(self, request, response):"""- Wrapper around the request processing function to maintain backward compatibility- with functions that do not take a Response object as parameter.+ Wrapper around the request processing function to maintain backward+ compatibility with functions that do not take a Response object"""arg_count = len(get_func_args(self.process_request))- args = [request] if arg_count == 1 else [request, response]+ if arg_count == 1:+ args = [request]+ msg = 'Rule.process_request should accept two arguments (request, response), accepting only one is deprecated'+ warnings.warn(msg, category=ScrapyDeprecationWarning, stacklevel=2)+ else:+ args = [request, response]return self.process_request(*args)