Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Rename Request.kwargs to Request.cb_kwargs

Eugenio Lacuesta6760bca74b1f
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index d12766676..b3f849540 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -24,7 +24,7 @@ below in :ref:`topics-request-response-ref-request-subclasses` and
Request objects
===============
-.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags, kwargs])
+.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags, cb_kwargs])
A :class:`Request` object represents an HTTP request, which is usually
generated in the Spider and executed by the Downloader, and thus generating
@@ -126,8 +126,8 @@ Request objects
:param flags: Flags sent to the request, can be used for logging or similar purposes.
:type flags: list
- :param kwargs: A dict with arbitrary data that will be passed as keyword arguments to the Request's callback.
- :type kwargs: dict
+ :param cb_kwargs: A dict with arbitrary data that will be passed as keyword arguments to the Request's callback.
+ :type cb_kwargs: dict
.. attribute:: Request.url
@@ -168,7 +168,7 @@ Request objects
``copy()`` or ``replace()`` methods, and can also be accessed, in your
spider, from the ``response.meta`` attribute.
- .. attribute:: Request.kwargs
+ .. attribute:: Request.cb_kwargs
A dictionary that contains arbitrary metadata for this request. Its contents
will be passed to the Request's callback as keyword arguments. It is empty
@@ -177,7 +177,7 @@ Request objects
This dict is `shallow copied`_ when the request is cloned using the
``copy()`` or ``replace()`` methods, and can also be accessed, in your
- spider, from the ``response.kwargs`` attribute.
+ spider, from the ``response.cb_kwargs`` attribute.
.. _shallow copied: https://docs.python.org/2/library/copy.html
@@ -216,7 +216,7 @@ Example::
In some cases you may be interested in passing arguments to those callback
functions so you can receive the arguments later, in the second callback.
The following two examples show how to achieve this by using the
-:attr:`Request.meta` and :attr:`Request.kwargs` attributes respectively::
+:attr:`Request.meta` and :attr:`Request.cb_kwargs` attributes respectively::
def parse_page1(self, response):
item = MyItem()
@@ -238,8 +238,8 @@ The following two examples show how to achieve this by using the
item['main_url'] = response.url
request = scrapy.Request("http://www.example.com/some_page.html",
callback=self.parse_page2)
- request.kwargs['item'] = item
- request.kwargs['foo'] = 'bar'
+ request.cb_kwargs['item'] = item
+ request.cb_kwargs['foo'] = 'bar'
yield request
def parse_page2(self, response, item, foo):
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 7981ce231..08dd1acc5 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -143,7 +143,9 @@ class Scraper(object):
def call_spider(self, result, request, spider):
result.request = request
dfd = defer_result(result)
- dfd.addCallbacks(request.callback or spider.parse, request.errback, callbackKeywords=request.kwargs)
+ dfd.addCallbacks(callback=request.callback or spider.parse,
+ errback=request.errback,
+ callbackKeywords=request.cb_kwargs)
return dfd.addCallback(iterate_spider_output)
def handle_spider_error(self, _failure, request, response, spider):
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index c016eb727..f5935c4ef 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -18,7 +18,7 @@ class Request(object_ref):
def __init__(self, url, callback=None, method='GET', headers=None, body=None,
cookies=None, meta=None, encoding='utf-8', priority=0,
- dont_filter=False, errback=None, flags=None, kwargs=None):
+ dont_filter=False, errback=None, flags=None, cb_kwargs=None):
self._encoding = encoding # this one has to be set first
self.method = str(method).upper()
@@ -40,14 +40,14 @@ class Request(object_ref):
self.dont_filter = dont_filter
self._meta = dict(meta) if meta else None
- self._kwargs = dict(kwargs) if kwargs else None
+ self._cb_kwargs = dict(cb_kwargs) if cb_kwargs else None
self.flags = [] if flags is None else list(flags)
@property
- def kwargs(self):
- if self._kwargs is None:
- self._kwargs = {}
- return self._kwargs
+ def cb_kwargs(self):
+ if self._cb_kwargs is None:
+ self._cb_kwargs = {}
+ return self._cb_kwargs
@property
def meta(self):
@@ -99,7 +99,7 @@ class Request(object_ref):
given new values.
"""
for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags',
- 'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'kwargs']:
+ 'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'cb_kwargs']:
kwargs.setdefault(x, getattr(self, x))
cls = kwargs.pop('cls', self.__class__)
return cls(*args, **kwargs)
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 99b04a26e..b0a526b72 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -106,7 +106,7 @@ class Response(object_ref):
def follow(self, url, callback=None, method='GET', headers=None, body=None,
cookies=None, meta=None, encoding='utf-8', priority=0,
- dont_filter=False, errback=None, kwargs=None):
+ dont_filter=False, errback=None, cb_kwargs=None):
# type: (...) -> Request
"""
Return a :class:`~.Request` instance to follow a link ``url``.
@@ -133,4 +133,4 @@ class Response(object_ref):
priority=priority,
dont_filter=dont_filter,
errback=errback,
- kwargs=kwargs)
+ cb_kwargs=cb_kwargs)
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 2039621b3..339913d4e 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -123,7 +123,7 @@ class TextResponse(Response):
def follow(self, url, callback=None, method='GET', headers=None, body=None,
cookies=None, meta=None, encoding=None, priority=0,
- dont_filter=False, errback=None, kwargs=None):
+ dont_filter=False, errback=None, cb_kwargs=None):
# type: (...) -> Request
"""
Return a :class:`~.Request` instance to follow a link ``url``.
@@ -155,7 +155,7 @@ class TextResponse(Response):
priority=priority,
dont_filter=dont_filter,
errback=errback,
- kwargs=kwargs,
+ cb_kwargs=cb_kwargs,
)
diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index d537057b1..e7016b92a 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -33,7 +33,7 @@ def request_to_dict(request, spider=None):
'priority': request.priority,
'dont_filter': request.dont_filter,
'flags': request.flags,
- 'kwargs': request.kwargs,
+ 'cb_kwargs': request.cb_kwargs,
}
if type(request) is not Request:
d['_class'] = request.__module__ + '.' + request.__class__.__name__
@@ -66,7 +66,7 @@ def request_from_dict(d, spider=None):
priority=d['priority'],
dont_filter=d['dont_filter'],
flags=d.get('flags'),
- kwargs=d.get('kwargs'),
+ cb_kwargs=d.get('cb_kwargs'),
)
diff --git a/tests/spiders.py b/tests/spiders.py
index 7b4707f62..a06985837 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -35,8 +35,8 @@ class KeywordArgumentsSpider(MockServerSpider):
def start_requests(self):
data = {'key': 'value', 'number': 123}
- yield Request(self.mockserver.url('/first'), self.parse_first, kwargs=data)
- yield Request(self.mockserver.url('/general_with'), self.parse_general, kwargs=data)
+ yield Request(self.mockserver.url('/first'), self.parse_first, cb_kwargs=data)
+ yield Request(self.mockserver.url('/general_with'), self.parse_general, cb_kwargs=data)
yield Request(self.mockserver.url('/general_without'), self.parse_general)
yield Request(self.mockserver.url('/no_kwargs'), self.parse_no_kwargs)
@@ -47,7 +47,7 @@ class KeywordArgumentsSpider(MockServerSpider):
yield response.follow(
self.mockserver.url('/two'),
self.parse_second,
- kwargs={'new_key': 'new_value'})
+ cb_kwargs={'new_key': 'new_value'})
def parse_second(self, response, new_key):
self.checks.append(new_key == 'new_value')
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 610893d8a..c1949a28c 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -177,7 +177,7 @@ class RequestTest(unittest.TestCase):
r1 = self.request_class("http://www.example.com", flags=['f1', 'f2'],
callback=somecallback, errback=somecallback)
r1.meta['foo'] = 'bar'
- r1.kwargs['key'] = 'value'
+ r1.cb_kwargs['key'] = 'value'
r2 = r1.copy()
# make sure copy does not propagate callbacks
@@ -190,9 +190,9 @@ class RequestTest(unittest.TestCase):
assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical"
self.assertEqual(r1.flags, r2.flags)
- # make sure kwargs dict is shallow copied
- assert r1.kwargs is not r2.kwargs, "kwargs must be a shallow copy, not identical"
- self.assertEqual(r1.kwargs, r2.kwargs)
+ # make sure cb_kwargs dict is shallow copied
+ assert r1.cb_kwargs is not r2.cb_kwargs, "cb_kwargs must be a shallow copy, not identical"
+ self.assertEqual(r1.cb_kwargs, r2.cb_kwargs)
# make sure meta dict is shallow copied
assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical"
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index 76de20f22..e1601b76b 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -26,7 +26,7 @@ class RequestSerializationTest(unittest.TestCase):
encoding='latin-1',
priority=20,
meta={'a': 'b'},
- kwargs={'k': 'v'},
+ cb_kwargs={'k': 'v'},
flags=['testFlag'])
self._assert_serializes_ok(r, spider=self.spider)
@@ -53,7 +53,7 @@ class RequestSerializationTest(unittest.TestCase):
self.assertEqual(r1.headers, r2.headers)
self.assertEqual(r1.cookies, r2.cookies)
self.assertEqual(r1.meta, r2.meta)
- self.assertEqual(r1.kwargs, r2.kwargs)
+ self.assertEqual(r1.cb_kwargs, r2.cb_kwargs)
self.assertEqual(r1._encoding, r2._encoding)
self.assertEqual(r1.priority, r2.priority)
self.assertEqual(r1.dont_filter, r2.dont_filter)