Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix csviter call, add parse_rows test (#5394)

Laerte4bda0976b289
diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py
index bef2d6b24..79e12e030 100644
--- a/scrapy/spiders/feed.py
+++ b/scrapy/spiders/feed.py
@@ -123,7 +123,7 @@ class CSVFeedSpider(Spider):
process_results methods for pre and post-processing purposes.
"""
- for row in csviter(response, self.delimiter, self.headers, self.quotechar):
+ for row in csviter(response, self.delimiter, self.headers, quotechar=self.quotechar):
ret = iterate_spider_output(self.parse_row(response, row))
for result_item in self.process_results(response, ret):
yield result_item
diff --git a/tests/test_spider.py b/tests/test_spider.py
index a7c3ee048..689349999 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -21,6 +21,7 @@ from scrapy.spiders import (
)
from scrapy.linkextractors import LinkExtractor
from scrapy.utils.test import get_crawler
+from tests import get_testdata
class SpiderTest(unittest.TestCase):
@@ -167,6 +168,23 @@ class CSVFeedSpiderTest(SpiderTest):
spider_class = CSVFeedSpider
+ def test_parse_rows(self):
+ body = get_testdata('feeds', 'feed-sample6.csv')
+ response = Response("http://example.org/dummy.csv", body=body)
+
+ class _CrawlSpider(self.spider_class):
+ name = "test"
+ delimiter = ","
+ quotechar = "'"
+
+ def parse_row(self, response, row):
+ return row
+
+ spider = _CrawlSpider()
+ rows = list(spider.parse_rows(response))
+ assert rows[0] == {'id': '1', 'name': 'alpha', 'value': 'foobar'}
+ assert len(rows) == 4
+
class CrawlSpiderTest(SpiderTest):