Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Rule.process_request: deprecate the use of functions taking only one argument

Eugenio Lacuesta56929e77d983
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index f474b0a18..f469891d0 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -6,16 +6,19 @@ See documentation in docs/topics/spiders.rst
"""
import copy
+import warnings
+
import six
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, HtmlResponse
from scrapy.utils.spider import iterate_spider_output
from scrapy.utils.python import get_func_args
from scrapy.spiders import Spider
-def identity(x):
- return x
+def _identity(request, response):
+ return request
class Rule(object):
@@ -25,19 +28,21 @@ class Rule(object):
self.callback = callback
self.cb_kwargs = cb_kwargs or {}
self.process_links = process_links
- self.process_request = process_request or identity
- if follow is None:
- self.follow = False if callback else True
- else:
- self.follow = follow
+ self.process_request = process_request or _identity
+ self.follow = follow if follow is not None else not callback
def _process_request(self, request, response):
"""
- Wrapper around the request processing function to maintain backward compatibility
- with functions that do not take a Response object as parameter.
+ Wrapper around the request processing function to maintain backward
+ compatibility with functions that do not take a Response object
"""
arg_count = len(get_func_args(self.process_request))
- args = [request] if arg_count == 1 else [request, response]
+ if arg_count == 1:
+ args = [request]
+ msg = 'Rule.process_request should accept two arguments (request, response), accepting only one is deprecated'
+ warnings.warn(msg, category=ScrapyDeprecationWarning, stacklevel=2)
+ else:
+ args = [request, response]
return self.process_request(*args)