Scrapy, a fast high-level web crawling & scraping framework for Python.
Deprecate the custom attribute of build_component_list(). (#5993)
Andrey Rakhmatullin9f9a2292e08c
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.pyindex 43a8b65a5..1889f7571 100644--- a/scrapy/utils/conf.py+++ b/scrapy/utils/conf.py@@ -50,11 +50,17 @@ def build_component_list(compdict, custom=None, convert=update_classpath):"please provide a real number or None instead")- if isinstance(custom, (list, tuple)):- _check_components(custom)- return type(custom)(convert(c) for c in custom)-if custom is not None:+ warnings.warn(+ "The 'custom' attribute of build_component_list() is deprecated. "+ "Please merge its value into 'compdict' manually or change your "+ "code to use Settings.getwithbase().",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )+ if isinstance(custom, (list, tuple)):+ _check_components(custom)+ return type(custom)(convert(c) for c in custom)compdict.update(custom)_validate_values(compdict)diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.pyindex 78ed9a7c9..dc3f01d57 100644--- a/tests/test_utils_conf.py+++ b/tests/test_utils_conf.py@@ -1,6 +1,8 @@import unittestimport warnings+import pytest+from scrapy.exceptions import ScrapyDeprecationWarning, UsageErrorfrom scrapy.settings import BaseSettings, Settingsfrom scrapy.utils.conf import (@@ -21,44 +23,45 @@ class BuildComponentListTest(unittest.TestCase):def test_backward_compatible_build_dict(self):base = {"one": 1, "two": 2, "three": 3, "five": 5, "six": None}custom = {"two": None, "three": 8, "four": 4}- self.assertEqual(- build_component_list(base, custom, convert=lambda x: x),- ["one", "four", "five", "three"],- )+ with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):+ self.assertEqual(+ build_component_list(base, custom, convert=lambda x: x),+ ["one", "four", "five", "three"],+ )def test_return_list(self):custom = ["a", "b", "c"]- self.assertEqual(- build_component_list(None, custom, convert=lambda x: x), custom- )+ with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):+ self.assertEqual(+ build_component_list(None, custom, convert=lambda x: x), custom+ )def test_map_dict(self):custom = {"one": 1, "two": 2, "three": 3}- self.assertEqual(- build_component_list({}, custom, convert=lambda x: x.upper()),- ["ONE", "TWO", "THREE"],- )+ with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):+ self.assertEqual(+ build_component_list({}, custom, convert=lambda x: x.upper()),+ ["ONE", "TWO", "THREE"],+ )def test_map_list(self):custom = ["a", "b", "c"]- self.assertEqual(- build_component_list(None, custom, lambda x: x.upper()), ["A", "B", "C"]- )+ with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):+ self.assertEqual(+ build_component_list(None, custom, lambda x: x.upper()), ["A", "B", "C"]+ )def test_duplicate_components_in_dict(self):duplicate_dict = {"one": 1, "two": 2, "ONE": 4}- self.assertRaises(- ValueError,- build_component_list,- {},- duplicate_dict,- convert=lambda x: x.lower(),- )+ with self.assertRaises(ValueError):+ with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):+ build_component_list({}, duplicate_dict, convert=lambda x: x.lower())def test_duplicate_components_in_list(self):duplicate_list = ["a", "b", "a"]with self.assertRaises(ValueError) as cm:- build_component_list(None, duplicate_list, convert=lambda x: x)+ with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):+ build_component_list(None, duplicate_list, convert=lambda x: x)self.assertIn(str(duplicate_list), str(cm.exception))def test_duplicate_components_in_basesettings(self):@@ -76,9 +79,8 @@ class BuildComponentListTest(unittest.TestCase):)# Same priority raises ValueErrorduplicate_bs.set("ONE", duplicate_bs["ONE"], priority=20)- self.assertRaises(- ValueError, build_component_list, duplicate_bs, convert=lambda x: x.lower()- )+ with self.assertRaises(ValueError):+ build_component_list(duplicate_bs, convert=lambda x: x.lower())def test_valid_numbers(self):# work well with None and numeric values@@ -92,15 +94,9 @@ class BuildComponentListTest(unittest.TestCase):self.assertEqual(build_component_list(d, convert=lambda x: x), ["b", "c", "a"])# raise exception for invalid valuesd = {"one": "5"}- self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)- d = {"one": "1.0"}- self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)- d = {"one": [1, 2, 3]}- self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)- d = {"one": {"a": "a", "b": 2}}- self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)- d = {"one": "lorem ipsum"}- self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)+ with self.assertRaises(ValueError):+ with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):+ build_component_list({}, d, convert=lambda x: x)class UtilsConfTestCase(unittest.TestCase):