Scrapy, a fast high-level web crawling & scraping framework for Python.
Make csviter work on python 3.7
Konstantin Lopuhinf4f39057cbbf
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.pyindex 73857b410..a12e14005 100644--- a/scrapy/utils/iterators.py+++ b/scrapy/utils/iterators.py@@ -98,8 +98,9 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):"""encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or 'utf-8'- def _getrow(csv_r):- return [to_unicode(field, encoding) for field in next(csv_r)]++ def row_to_unicode(row_):+ return [to_unicode(field, encoding) for field in row_]# Python 3 csv reader input object needs to return stringsif six.PY3:@@ -113,10 +114,14 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):csv_r = csv.reader(lines, **kwargs)if not headers:- headers = _getrow(csv_r)-- while True:- row = _getrow(csv_r)+ try:+ row = next(csv_r)+ except StopIteration:+ return+ headers = row_to_unicode(row)++ for row in csv_r:+ row = row_to_unicode(row)if len(row) != len(headers):logger.warning("ignoring row %(csvlnum)d (length: %(csvrow)d, ""should be: %(csvheader)d)",