Scrapy, a fast high-level web crawling & scraping framework for Python.
Validate values for components order
Eugenio Lacuesta24e82bfe75ad
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.pyindex e8af90f11..435e9a6b3 100644--- a/scrapy/utils/conf.py+++ b/scrapy/utils/conf.py@@ -1,5 +1,6 @@import osimport sys+import numbersfrom operator import itemgetterimport six@@ -34,6 +35,13 @@ def build_component_list(compdict, custom=None, convert=update_classpath):_check_components(compdict)return {convert(k): v for k, v in six.iteritems(compdict)}+ def _validate_values(compdict):+ """Fail if a value in the components dict is not a real number or None."""+ for name, value in six.iteritems(compdict):+ if value is not None and not isinstance(value, numbers.Real):+ raise ValueError('Invalid value {} for component {}, please provide ' \+ 'a real number or None instead'.format(value, name))+# BEGIN Backwards compatibility for old (base, custom) call signatureif isinstance(custom, (list, tuple)):_check_components(custom)@@ -43,6 +51,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):compdict.update(custom)# END Backwards compatibility+ _validate_values(compdict)compdict = without_none_values(_map_keys(compdict))return [k for k, v in sorted(six.iteritems(compdict), key=itemgetter(1))]diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.pyindex dab41ac8d..f203c32ef 100644--- a/tests/test_utils_conf.py+++ b/tests/test_utils_conf.py@@ -62,6 +62,27 @@ class BuildComponentListTest(unittest.TestCase):self.assertRaises(ValueError, build_component_list, duplicate_bs,convert=lambda x: x.lower())+ def test_valid_numbers(self):+ # work well with None and numeric values+ d = {'a': 10, 'b': None, 'c': 15, 'd': 5.0}+ self.assertEqual(build_component_list(d, convert=lambda x: x),+ ['d', 'a', 'c'])+ d = {'a': 33333333333333333333, 'b': 11111111111111111111, 'c': 22222222222222222222}+ self.assertEqual(build_component_list(d, convert=lambda x: x),+ ['b', 'c', 'a'])+ # raise exception for invalid values+ d = {'one': '5'}+ self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)+ d = {'one': '1.0'}+ self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)+ d = {'one': [1, 2, 3]}+ self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)+ d = {'one': {'a': 'a', 'b': 2}}+ self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)+ d = {'one': 'lorem ipsum',}+ self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)++class UtilsConfTestCase(unittest.TestCase):