Scrapy, a fast high-level web crawling & scraping framework for Python.
TST update tests to use get/getall/attrib instead of extract
Mikhail Korobov53da56c8dcd6
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.pyindex 3e27d6abd..36baacfbd 100644--- a/tests/test_command_shell.py+++ b/tests/test_command_shell.py@@ -35,7 +35,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):@defer.inlineCallbacksdef test_response_selector_html(self):- xpath = 'response.xpath("//p[@class=\'one\']/text()").extract()[0]'+ xpath = 'response.xpath("//p[@class=\'one\']/text()").get()'_, out, _ = yield self.execute([self.url('/html'), '-c', xpath])self.assertEqual(out.strip(), b'Works')diff --git a/tests/test_http_response.py b/tests/test_http_response.pyindex 820758dc9..3b90e3dac 100644--- a/tests/test_http_response.py+++ b/tests/test_http_response.py@@ -336,11 +336,11 @@ class TextResponseTest(BaseResponseTest):self.assertIs(response.selector.response, response)self.assertEqual(- response.selector.xpath("//title/text()").extract(),+ response.selector.xpath("//title/text()").getall(),[u'Some page'])self.assertEqual(- response.selector.css("title::text").extract(),+ response.selector.css("title::text").getall(),[u'Some page'])self.assertEqual(@@ -353,12 +353,12 @@ class TextResponseTest(BaseResponseTest):response = self.response_class("http://www.example.com", body=body)self.assertEqual(- response.xpath("//title/text()").extract(),- response.selector.xpath("//title/text()").extract(),+ response.xpath("//title/text()").getall(),+ response.selector.xpath("//title/text()").getall(),)self.assertEqual(- response.css("title::text").extract(),- response.selector.css("title::text").extract(),+ response.css("title::text").getall(),+ response.selector.css("title::text").getall(),)def test_selector_shortcuts_kwargs(self):@@ -366,13 +366,13 @@ class TextResponseTest(BaseResponseTest):response = self.response_class("http://www.example.com", body=body)self.assertEqual(- response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").extract(),- response.xpath("normalize-space(//p[@class=\"content\"])").extract(),+ response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").getall(),+ response.xpath("normalize-space(//p[@class=\"content\"])").getall(),)self.assertEqual(response.xpath("//title[count(following::p[@class=$pclass])=$pcount]/text()",- pclass="content", pcount=1).extract(),- response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").extract(),+ pclass="content", pcount=1).getall(),+ response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").getall(),)def test_urljoin_with_base_url(self):@@ -562,7 +562,7 @@ class XmlResponseTest(TextResponseTest):self.assertIs(response.selector.response, response)self.assertEqual(- response.selector.xpath("//elem/text()").extract(),+ response.selector.xpath("//elem/text()").getall(),[u'value'])@@ -571,8 +571,8 @@ class XmlResponseTest(TextResponseTest):response = self.response_class("http://www.example.com", body=body)self.assertEqual(- response.xpath("//elem/text()").extract(),- response.selector.xpath("//elem/text()").extract(),+ response.xpath("//elem/text()").getall(),+ response.selector.xpath("//elem/text()").getall(),)def test_selector_shortcuts_kwargs(self):@@ -583,12 +583,12 @@ class XmlResponseTest(TextResponseTest):response = self.response_class("http://www.example.com", body=body)self.assertEqual(- response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(),- response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(),+ response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).getall(),+ response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).getall(),)response.selector.register_namespace('s2', 'http://scrapy.org')self.assertEqual(- response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).extract(),- response.selector.xpath("//s2:elem/text()").extract(),+ response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).getall(),+ response.selector.xpath("//s2:elem/text()").getall(),)diff --git a/tests/test_loader.py b/tests/test_loader.pyindex 3b5714058..8b58e4dbd 100644--- a/tests/test_loader.py+++ b/tests/test_loader.py@@ -634,7 +634,7 @@ class SubselectorLoaderTest(unittest.TestCase):nl = l.nested_xpath("//header")nl.add_xpath('name', 'div/text()')nl.add_css('name_div', '#id')- nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract())+ nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())self.assertEqual(l.get_output_value('name'), [u'marta'])self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])@@ -649,7 +649,7 @@ class SubselectorLoaderTest(unittest.TestCase):nl = l.nested_css("header")nl.add_xpath('name', 'div/text()')nl.add_css('name_div', '#id')- nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract())+ nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())self.assertEqual(l.get_output_value('name'), [u'marta'])self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.pyindex 5985a6f3e..fb72c9d6d 100644--- a/tests/test_pipeline_crawl.py+++ b/tests/test_pipeline_crawl.py@@ -29,7 +29,7 @@ class MediaDownloadSpider(SimpleSpider):for href in response.xpath('''//table[thead/tr/th="Filename"]/tbody//a/@href- ''').extract()],+ ''').getall()],}yield itemdiff --git a/tests/test_selector.py b/tests/test_selector.pyindex 526660cc8..bc4baf7ea 100644--- a/tests/test_selector.py+++ b/tests/test_selector.py@@ -20,17 +20,17 @@ class SelectorTestCase(unittest.TestCase):for x in xl:assert isinstance(x, Selector)- self.assertEqual(sel.xpath('//input').extract(),- [x.extract() for x in sel.xpath('//input')])+ self.assertEqual(sel.xpath('//input').getall(),+ [x.get() for x in sel.xpath('//input')])- self.assertEqual([x.extract() for x in sel.xpath("//input[@name='a']/@name")],+ self.assertEqual([x.get() for x in sel.xpath("//input[@name='a']/@name")],[u'a'])- self.assertEqual([x.extract() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],+ self.assertEqual([x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],[u'12.0'])- self.assertEqual(sel.xpath("concat('xpath', 'rules')").extract(),+ self.assertEqual(sel.xpath("concat('xpath', 'rules')").getall(),[u'xpathrules'])- self.assertEqual([x.extract() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],+ self.assertEqual([x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],[u'12'])def test_root_base_url(self):@@ -60,12 +60,12 @@ class SelectorTestCase(unittest.TestCase):text = b'<div><img src="a.jpg"><p>Hello</div>'sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8'))self.assertEqual(sel.type, 'xml')- self.assertEqual(sel.xpath("//div").extract(),+ self.assertEqual(sel.xpath("//div").getall(),[u'<div><img src="a.jpg"><p>Hello</p></img></div>'])sel = Selector(HtmlResponse('http://example.com', body=text, encoding='utf-8'))self.assertEqual(sel.type, 'html')- self.assertEqual(sel.xpath("//div").extract(),+ self.assertEqual(sel.xpath("//div").getall(),[u'<div><img src="a.jpg"><p>Hello</p></div>'])def test_http_header_encoding_precedence(self):@@ -84,15 +84,15 @@ class SelectorTestCase(unittest.TestCase):headers = {'Content-Type': ['text/html; charset=utf-8']}response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)x = Selector(response)- self.assertEqual(x.xpath("//span[@id='blank']/text()").extract(),+ self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(),[u'\xa3'])def test_badly_encoded_body(self):# \xe9 alone isn't valid utf8 sequence- r1 = TextResponse('http://www.example.com', \- body=b'<html><p>an Jos\xe9 de</p><html>', \+ r1 = TextResponse('http://www.example.com',+ body=b'<html><p>an Jos\xe9 de</p><html>',encoding='utf-8')- Selector(r1).xpath('//text()').extract()+ Selector(r1).xpath('//text()').getall()def test_weakref_slots(self):"""Check that classes are using slots and are weak-referenceable"""diff --git a/tests/test_spider.py b/tests/test_spider.pyindex 929e0fea8..f26da2334 100644--- a/tests/test_spider.py+++ b/tests/test_spider.py@@ -147,10 +147,10 @@ class XMLFeedSpiderTest(SpiderTest):def parse_node(self, response, selector):yield {- 'loc': selector.xpath('a:loc/text()').extract(),- 'updated': selector.xpath('b:updated/text()').extract(),- 'other': selector.xpath('other/@value').extract(),- 'custom': selector.xpath('other/@b:custom').extract(),+ 'loc': selector.xpath('a:loc/text()').getall(),+ 'updated': selector.xpath('b:updated/text()').getall(),+ 'other': selector.xpath('other/@value').getall(),+ 'custom': selector.xpath('other/@b:custom').getall(),}for iterator in ('iternodes', 'xml'):diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.pyindex f953076b8..00eb78068 100644--- a/tests/test_utils_iterators.py+++ b/tests/test_utils_iterators.py@@ -30,10 +30,13 @@ class XmliterTestCase(unittest.TestCase):response = XmlResponse(url="http://example.com", body=body)attrs = []for x in self.xmliter(response, 'product'):- attrs.append((x.xpath("@id").extract(), x.xpath("name/text()").extract(), x.xpath("./type/text()").extract()))+ attrs.append((+ x.attrib['id'],+ x.xpath("name/text()").extract(),+ x.xpath("./type/text()").extract()))self.assertEqual(attrs,- [(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])])+ [('001', ['Name 1'], ['Type 1']), ('002', ['Name 2'], ['Type 2'])])def test_xmliter_unusual_node(self):body = b"""<?xml version="1.0" encoding="UTF-8"?>@@ -43,7 +46,7 @@ class XmliterTestCase(unittest.TestCase):</root>"""response = XmlResponse(url="http://example.com", body=body)- nodenames = [e.xpath('name()').extract()+ nodenames = [e.xpath('name()').getall()for e in self.xmliter(response, 'matchme...')]self.assertEqual(nodenames, [['matchme...']])@@ -93,19 +96,19 @@ class XmliterTestCase(unittest.TestCase):attrs = []for x in self.xmliter(r, u'þingflokkur'):- attrs.append((x.xpath('@id').extract(),- x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(),- x.xpath(u'./tímabil/fyrstaþing/text()').extract()))+ attrs.append((x.attrib['id'],+ x.xpath(u'./skammstafanir/stuttskammstöfun/text()').getall(),+ x.xpath(u'./tímabil/fyrstaþing/text()').getall()))self.assertEqual(attrs,- [([u'26'], [u'-'], [u'80']),- ([u'21'], [u'Ab'], [u'76']),- ([u'27'], [u'A'], [u'27'])])+ [(u'26', [u'-'], [u'80']),+ (u'21', [u'Ab'], [u'76']),+ (u'27', [u'A'], [u'27'])])def test_xmliter_text(self):body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""- self.assertEqual([x.xpath("text()").extract() for x in self.xmliter(body, 'product')],+ self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')],[[u'one'], [u'two']])def test_xmliter_namespaces(self):@@ -132,15 +135,15 @@ class XmliterTestCase(unittest.TestCase):node = next(my_iter)node.register_namespace('g', 'http://base.google.com/ns/1.0')- self.assertEqual(node.xpath('title/text()').extract(), ['Item 1'])- self.assertEqual(node.xpath('description/text()').extract(), ['This is item 1'])- self.assertEqual(node.xpath('link/text()').extract(), ['http://www.mydummycompany.com/items/1'])- self.assertEqual(node.xpath('g:image_link/text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])- self.assertEqual(node.xpath('g:id/text()').extract(), ['ITEM_1'])- self.assertEqual(node.xpath('g:price/text()').extract(), ['400'])- self.assertEqual(node.xpath('image_link/text()').extract(), [])- self.assertEqual(node.xpath('id/text()').extract(), [])- self.assertEqual(node.xpath('price/text()').extract(), [])+ self.assertEqual(node.xpath('title/text()').getall(), ['Item 1'])+ self.assertEqual(node.xpath('description/text()').getall(), ['This is item 1'])+ self.assertEqual(node.xpath('link/text()').getall(), ['http://www.mydummycompany.com/items/1'])+ self.assertEqual(node.xpath('g:image_link/text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg'])+ self.assertEqual(node.xpath('g:id/text()').getall(), ['ITEM_1'])+ self.assertEqual(node.xpath('g:price/text()').getall(), ['400'])+ self.assertEqual(node.xpath('image_link/text()').getall(), [])+ self.assertEqual(node.xpath('id/text()').getall(), [])+ self.assertEqual(node.xpath('price/text()').getall(), [])def test_xmliter_exception(self):body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""@@ -159,7 +162,7 @@ class XmliterTestCase(unittest.TestCase):body = b'<?xml version="1.0" encoding="ISO-8859-9"?>\n<xml>\n <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n</xml>\n\n'response = XmlResponse('http://www.example.com', body=body)self.assertEqual(- next(self.xmliter(response, 'item')).extract(),+ next(self.xmliter(response, 'item')).get(),u'<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>')@@ -192,9 +195,9 @@ class LxmlXmliterTestCase(XmliterTestCase):namespace_iter = self.xmliter(response, 'image_link', 'http://base.google.com/ns/1.0')node = next(namespace_iter)- self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])+ self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg'])node = next(namespace_iter)- self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item2.jpg'])+ self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item2.jpg'])def test_xmliter_namespaces_prefix(self):body = b"""\@@ -219,14 +222,14 @@ class LxmlXmliterTestCase(XmliterTestCase):my_iter = self.xmliter(response, 'table', 'http://www.w3.org/TR/html4/', 'h')node = next(my_iter)- self.assertEqual(len(node.xpath('h:tr/h:td').extract()), 2)- self.assertEqual(node.xpath('h:tr/h:td[1]/text()').extract(), ['Apples'])- self.assertEqual(node.xpath('h:tr/h:td[2]/text()').extract(), ['Bananas'])+ self.assertEqual(len(node.xpath('h:tr/h:td').getall()), 2)+ self.assertEqual(node.xpath('h:tr/h:td[1]/text()').getall(), ['Apples'])+ self.assertEqual(node.xpath('h:tr/h:td[2]/text()').getall(), ['Bananas'])my_iter = self.xmliter(response, 'table', 'http://www.w3schools.com/furniture', 'f')node = next(my_iter)- self.assertEqual(node.xpath('f:name/text()').extract(), ['African Coffee Table'])+ self.assertEqual(node.xpath('f:name/text()').getall(), ['African Coffee Table'])def test_xmliter_objtype_exception(self):i = self.xmliter(42, 'product')