Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

[itemloader-errors] updated Compose and MapCompose messages and added tests

Mabel Villalbab1e348b28138
diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py
index 56f2bc564..4b6d8624c 100644
--- a/scrapy/loader/processors.py
+++ b/scrapy/loader/processors.py
@@ -28,10 +28,12 @@ class MapCompose(object):
try:
next_values += arg_to_iter(func(v))
except Exception as e:
+ type_name = type(func).__name__
+ _name = (func.__name__ if type_name == 'function'
+ else type_name)
raise ValueError("Error in MapCompose with "
"function %s value=%r error='%s: %s'" %
- (func.__name__, value,
- type(e).__name__, str(e)))
+ (_name, value, type(e).__name__, str(e)))
values = next_values
return values
@@ -55,10 +57,12 @@ class Compose(object):
try:
value = func(value)
except Exception as e:
+ type_name = type(func).__name__
+ _name = (func.__name__ if type_name == 'function'
+ else type_name)
raise ValueError("Error in Compose with "
"function %s value=%r error='%s: %s'" %
- (func.__name__, value,
- type(e).__name__, str(e)))
+ (_name, value, type(e).__name__, str(e)))
return value
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 0f9070b8c..ce0fa0701 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -456,6 +456,42 @@ class BasicItemLoaderTest(unittest.TestCase):
'title': [u'Test item title 3', u'Test item 4'],
})
+ def test_error_input_processor(self):
+ class TestItem(Item):
+ name = Field()
+
+ class TestItemLoader(ItemLoader):
+ default_item_class = TestItem
+ name_in = MapCompose(float)
+
+ il = TestItemLoader()
+ self.assertRaises(ValueError, il.add_value, 'name',
+ [u'marta', u'other'])
+
+ def test_error_output_processor(self):
+ class TestItem(Item):
+ name = Field()
+
+ class TestItemLoader(ItemLoader):
+ default_item_class = TestItem
+ name_out = Compose(Join(), float)
+
+ il = TestItemLoader()
+ il.add_value('name', u'marta')
+ with self.assertRaises(ValueError):
+ il.load_item()
+
+ def test_error_processor_as_argument(self):
+ class TestItem(Item):
+ name = Field()
+
+ class TestItemLoader(ItemLoader):
+ default_item_class = TestItem
+
+ il = TestItemLoader()
+ self.assertRaises(ValueError, il.add_value, 'name',
+ [u'marta', u'other'], Compose(float))
+
class ProcessorsTest(unittest.TestCase):