Scrapy, a fast high-level web crawling & scraping framework for Python.
DOC switch from .extract to get/getall API in docs
Mikhail Korobovbdcc045f62a3
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rstindex 07fb4807f..46e84b21c 100644--- a/docs/intro/tutorial.rst+++ b/docs/intro/tutorial.rst@@ -254,7 +254,7 @@ data.To extract the text from the title above, you can do::- >>> response.css('title::text').extract()+ >>> response.css('title::text').getall()['Quotes to Scrape']There are two things to note here: one is that we've added ``::text`` to the@@ -262,12 +262,12 @@ CSS query, to mean we want to select only the text elements directly inside``<title>`` element. If we don't specify ``::text``, we'd get the full titleelement, including its tags::- >>> response.css('title').extract()+ >>> response.css('title').getall()['<title>Quotes to Scrape</title>']-The other thing is that the result of calling ``.extract()`` is a list, because-we're dealing with an instance of :class:`~scrapy.selector.SelectorList`. When-you know you just want the first result, as in this case, you can do::+The other thing is that the result of calling ``.getall()`` is a list: it is+possible that a selector returns more than one result, so we extract them all.+When you know you just want the first result, as in this case, you can do::>>> response.css('title::text').get()'Quotes to Scrape'@@ -392,10 +392,10 @@ using the ``quote`` object we just created::>>> author'Albert Einstein'-Given that the tags are a list of strings, we can use the ``.extract()`` method+Given that the tags are a list of strings, we can use the ``.getall()`` methodto get all of them::- >>> tags = quote.css("div.tags a.tag::text").extract()+ >>> tags = quote.css("div.tags a.tag::text").getall()>>> tags['change', 'deep-thoughts', 'thinking', 'world']@@ -405,7 +405,7 @@ quotes elements and put them together into a Python dictionary::>>> for quote in response.css("div.quote"):... text = quote.css("span.text::text").get()... author = quote.css("small.author::text").get()- ... tags = quote.css("div.tags a.tag::text").extract()+ ... tags = quote.css("div.tags a.tag::text").getall()... print(dict(text=text, author=author, tags=tags)){'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'}{'tags': ['abilities', 'choices'], 'author': 'J.K. Rowling', 'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”'}@@ -438,7 +438,7 @@ in the callback, as you can see below::yield {'text': quote.css('span.text::text').get(),'author': quote.css('small.author::text').get(),- 'tags': quote.css('div.tags a.tag::text').extract(),+ 'tags': quote.css('div.tags a.tag::text').getall(),}If you run this spider, it will output the extracted data with the log::@@ -543,7 +543,7 @@ page, extracting data from it::yield {'text': quote.css('span.text::text').get(),'author': quote.css('small.author::text').get(),- 'tags': quote.css('div.tags a.tag::text').extract(),+ 'tags': quote.css('div.tags a.tag::text').getall(),}next_page = response.css('li.next a::attr(href)').get()@@ -594,7 +594,7 @@ As a shortcut for creating Request objects you can useyield {'text': quote.css('span.text::text').get(),'author': quote.css('span small::text').get(),- 'tags': quote.css('div.tags a.tag::text').extract(),+ 'tags': quote.css('div.tags a.tag::text').getall(),}next_page = response.css('li.next a::attr(href)').get()diff --git a/docs/topics/shell.rst b/docs/topics/shell.rstindex 9de6abef7..68a0b19b5 100644--- a/docs/topics/shell.rst+++ b/docs/topics/shell.rst@@ -184,8 +184,8 @@ After that, we can start playing with the objects::>>> fetch("https://reddit.com")- >>> response.xpath('//title/text()').extract()- ['reddit: the front page of the internet']+ >>> response.xpath('//title/text()').get()+ 'reddit: the front page of the internet'>>> request = request.replace(method="POST")diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rstindex 4505b7315..a08dc30f2 100644--- a/docs/topics/spiders.rst+++ b/docs/topics/spiders.rst@@ -229,11 +229,11 @@ Return multiple Requests and items from a single callback::]def parse(self, response):- for h3 in response.xpath('//h3').extract():+ for h3 in response.xpath('//h3').getall():yield {"title": h3}- for url in response.xpath('//a/@href').extract():- yield scrapy.Request(url, callback=self.parse)+ for href in response.xpath('//a/@href').getall():+ yield scrapy.Request(response.urljoin(href), self.parse)Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly;to give data more structure you can use :ref:`topics-items`::@@ -251,11 +251,11 @@ to give data more structure you can use :ref:`topics-items`::yield scrapy.Request('http://www.example.com/3.html', self.parse)def parse(self, response):- for h3 in response.xpath('//h3').extract():+ for h3 in response.xpath('//h3').getall():yield MyItem(title=h3)- for url in response.xpath('//a/@href').extract():- yield scrapy.Request(url, callback=self.parse)+ for href in response.xpath('//a/@href').getall():+ yield scrapy.Request(response.urljoin(href), self.parse).. _spiderargs:@@ -545,7 +545,7 @@ These spiders are pretty easy to use, let's have a look at one example::itertag = 'item'def parse_node(self, response, node):- self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.extract()))+ self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.getall()))item = TestItem()item['id'] = node.xpath('@id').get()