Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Make the regex align to the spec better; add unit tests for name variations

Andrew Baxter72b7d3e90ac2
diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index 07c51aaff..04665a2d4 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -9,7 +9,7 @@ from scrapy.utils.python import to_unicode, to_native_str
from scrapy.utils.misc import load_object
-private_name_regex = re.compile('^__[^_](.*[^_])?_?$')
+private_name_regex = re.compile('^__.*[^_]_?$')
def request_to_dict(request, spider=None):
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index f7191fcef..b49450ac5 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -3,7 +3,7 @@ import unittest
from scrapy.http import Request, FormRequest
from scrapy.spiders import Spider
-from scrapy.utils.reqser import request_to_dict, request_from_dict
+from scrapy.utils.reqser import request_to_dict, request_from_dict, private_name_regex
class RequestSerializationTest(unittest.TestCase):
@@ -74,6 +74,28 @@ class RequestSerializationTest(unittest.TestCase):
errback=self.spider.handle_error)
self._assert_serializes_ok(r, spider=self.spider)
+ def test_private_callback_name_matching(self):
+ self.assertTrue(private_name_regex.search('__a'))
+ self.assertTrue(private_name_regex.search('__a_'))
+ self.assertTrue(private_name_regex.search('__a_a'))
+ self.assertTrue(private_name_regex.search('__a_a_'))
+ self.assertTrue(private_name_regex.search('__a__a'))
+ self.assertTrue(private_name_regex.search('__a__a_'))
+ self.assertTrue(private_name_regex.search('__a___a'))
+ self.assertTrue(private_name_regex.search('__a___a_'))
+ self.assertTrue(private_name_regex.search('___a'))
+ self.assertTrue(private_name_regex.search('___a_'))
+ self.assertTrue(private_name_regex.search('___a_a'))
+ self.assertTrue(private_name_regex.search('___a_a_'))
+ self.assertTrue(private_name_regex.search('____a_a_'))
+
+ self.assertFalse(private_name_regex.search('_a'))
+ self.assertFalse(private_name_regex.search('_a_'))
+ self.assertFalse(private_name_regex.search('__a__'))
+ self.assertFalse(private_name_regex.search('__'))
+ self.assertFalse(private_name_regex.search('___'))
+ self.assertFalse(private_name_regex.search('____'))
+
def test_unserializable_callback1(self):
r = Request("http://www.example.com", callback=lambda x: x)
self.assertRaises(ValueError, request_to_dict, r)