Scrapy, a fast high-level web crawling & scraping framework for Python.
Expanded documentation for issue #6676 (#6957)
Michaela5bbeb2586c9
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rstindex 390a5f541..0a83b248e 100644--- a/docs/topics/spiders.rst+++ b/docs/topics/spiders.rst@@ -364,6 +364,52 @@ used by :class:`~scrapy.downloadermiddlewares.useragent.UserAgentMiddleware`::Spider arguments can also be passed through the Scrapyd ``schedule.json`` API.See `Scrapyd documentation`_.+scrapy-spider-metadata parameters+---------------------------------++Another alternative to pass spider arguments is the library `scrapy-spider-metadata`_.++This allows for Scrapy spiders to define, validate, document and pre-process+their arguments as Pydantic models.++The example shows how to define typed parameters where a string argument+is automatically converted to an integer:++.. code-block:: python++ import scrapy+ from pydantic import BaseModel+ from scrapy_spider_metadata import Args+++ class MyParams(BaseModel):+ pages: int+++ class BookSpider(Args[MyParams], scrapy.Spider):+ name = "bookspider"+ start_urls = ["http://books.toscrape.com/catalogue"]++ async def start(self):+ for start_url in self.start_urls:+ for index in range(1, self.args.pages + 1):+ yield scrapy.Request(f"{start_url}/page-{index}.html")++ def parse(self, response):+ book_links = response.css("article.product_pod h3 a::attr(href)").getall()+ for book_link in book_links:+ yield response.follow(book_link, self.parse_book)++ def parse_book(self, response):+ yield {+ "title": response.css("h1::text").get(),+ "price": response.css("p.price_color::text").get(),+ }++This spider can be called from the command line::++ scrapy crawl bookspider -a pages=2+.. _start-requests:Start requests@@ -938,6 +984,7 @@ Combine SitemapSpider with other sources of urls:def parse_other(self, response):pass # ... scrape other here ...+.. _scrapy-spider-metadata: https://scrapy-spider-metadata.readthedocs.io/en/latest/params.html.. _Sitemaps: https://www.sitemaps.org/index.html.. _Sitemap index files: https://www.sitemaps.org/protocol.html#index.. _robots.txt: https://www.robotstxt.org/