Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge remote-tracking branch 'upstream/master' into fix_iternodes

Adrián Chavesd711eca4d937
diff --cc scrapy/utils/iterators.py
index 11b873f2e,5e15bf0c8..c356ad7f8
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@@ -20,30 -20,64 +20,72 @@@ def xmliter(obj, nodename)
- a unicode string
- a string encoded as utf-8
"""
+ nodename_patt = re.escape(nodename)
+
- HEADER_START_RE = re.compile(r'^(.*?)<\s*%s(?:\s|>)' % nodename_patt, re.S)
+ DOCUMENT_HEADER_RE = re.compile(r'<\?xml[^>]+>\s*', re.S)
- HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename, re.S)
+ HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename_patt, re.S)
+ END_TAG_RE = re.compile(r'<\s*/([^\s>]+)\s*>', re.S)
+ NAMESPACE_RE = re.compile(r'((xmlns[:A-Za-z]*)=[^>\s]+)', re.S)
text = _body_or_str(obj)
- header_start = re.search(HEADER_START_RE, text)
- header_start = header_start.group(1).strip() if header_start else ''
- header_end = re_rsearch(HEADER_END_RE, text)
- header_end = text[header_end[1]:].strip() if header_end else ''
+ document_header = re.search(DOCUMENT_HEADER_RE, text)
+ document_header = document_header.group().strip() if document_header else ''
+ header_end_idx = re_rsearch(HEADER_END_RE, text)
+ header_end = text[header_end_idx[1]:].strip() if header_end_idx else ''
+ namespaces = {}
+ if header_end:
+ for tagname in reversed(re.findall(END_TAG_RE, header_end)):
+ tag = re.search(r'<\s*%s.*?xmlns[:=][^>]*>' % tagname, text[:header_end_idx[1]], re.S)
+ if tag:
+ namespaces.update(reversed(x) for x in re.findall(NAMESPACE_RE, tag.group()))
- r = re.compile(r"<%s[\s>].*?</%s>" % (nodename, nodename), re.DOTALL)
+ r = re.compile(r'<%(np)s[\s>].*?</%(np)s>' % {'np': nodename_patt}, re.DOTALL)
for match in r.finditer(text):
- nodetext = header_start + match.group() + header_end
- yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0]
+ nodetext = document_header + match.group().replace(nodename, '%s %s' % (nodename, ' '.join(namespaces.values())), 1) + header_end
+ yield Selector(text=nodetext, type='xml')
- def csviter(obj, delimiter=None, headers=None, encoding=None):
+ def xmliter_lxml(obj, nodename, namespace=None, prefix='x'):
+ from lxml import etree
+ reader = _StreamReader(obj)
+ tag = '{%s}%s' % (namespace, nodename) if namespace else nodename
+ iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding)
+ selxpath = '//' + ('%s:%s' % (prefix, nodename) if namespace else nodename)
+ for _, node in iterable:
+ nodetext = etree.tostring(node, encoding='unicode')
+ node.clear()
+ xs = Selector(text=nodetext, type='xml')
+ if namespace:
+ xs.register_namespace(prefix, namespace)
+ yield xs.xpath(selxpath)[0]
+
+
+ class _StreamReader:
+
+ def __init__(self, obj):
+ self._ptr = 0
+ if isinstance(obj, Response):
+ self._text, self.encoding = obj.body, obj.encoding
+ else:
+ self._text, self.encoding = obj, 'utf-8'
+ self._is_unicode = isinstance(self._text, str)
+
+ def read(self, n=65535):
+ self.read = self._read_unicode if self._is_unicode else self._read_string
+ return self.read(n).lstrip()
+
+ def _read_string(self, n=65535):
+ s, e = self._ptr, self._ptr + n
+ self._ptr = e
+ return self._text[s:e]
+
+ def _read_unicode(self, n=65535):
+ s, e = self._ptr, self._ptr + n
+ self._ptr = e
+ return self._text[s:e].encode('utf-8')
+
+
+ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
""" Returns an iterator of dictionaries from the given csv object
obj can be:
diff --cc tests/test_utils_iterators.py
index 8b5941605,50190d4d1..bbdc88dd1
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@@ -61,25 -134,27 +134,31 @@@ class XmliterTestCase(unittest.TestCase
"""
response = XmlResponse(url='http://mydummycompany.com', body=body)
my_iter = self.xmliter(response, 'item')
-
node = next(my_iter)
node.register_namespace('g', 'http://base.google.com/ns/1.0')
- self.assertEqual(node.xpath('title/text()').extract(), ['Item 1'])
- self.assertEqual(node.xpath('description/text()').extract(), ['This is item 1'])
- self.assertEqual(node.xpath('link/text()').extract(), ['http://www.mydummycompany.com/items/1'])
- self.assertEqual(node.xpath('g:image_link/text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
- self.assertEqual(node.xpath('g:id/text()').extract(), ['ITEM_1'])
- self.assertEqual(node.xpath('g:price/text()').extract(), ['400'])
- self.assertEqual(node.xpath('image_link/text()').extract(), [])
- self.assertEqual(node.xpath('id/text()').extract(), [])
- self.assertEqual(node.xpath('price/text()').extract(), [])
+ self.assertEqual(node.xpath('title/text()').getall(), ['Item 1'])
+ self.assertEqual(node.xpath('description/text()').getall(), ['This is item 1'])
+ self.assertEqual(node.xpath('link/text()').getall(), ['http://www.mydummycompany.com/items/1'])
+ self.assertEqual(
+ node.xpath('g:image_link/text()').getall(),
+ ['http://www.mydummycompany.com/images/item1.jpg']
+ )
+ self.assertEqual(node.xpath('g:id/text()').getall(), ['ITEM_1'])
+ self.assertEqual(node.xpath('g:price/text()').getall(), ['400'])
+ self.assertEqual(node.xpath('image_link/text()').getall(), [])
+ self.assertEqual(node.xpath('id/text()').getall(), [])
+ self.assertEqual(node.xpath('price/text()').getall(), [])
+ my_iter = self.xmliter(response, 'g:image_link')
+ node = next(my_iter)
+ node.register_namespace('g', 'http://base.google.com/ns/1.0')
+ self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
+
def test_xmliter_exception(self):
- body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
+ body = (
+ '<?xml version="1.0" encoding="UTF-8"?>'
+ '<products><product>one</product><product>two</product></products>'
+ )
iter = self.xmliter(body, 'product')
next(iter)