Scrapy, a fast high-level web crawling & scraping framework for Python.
Apply black formatting to templates (#5814)
Alex8c8894f4bec6
diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmplindex bd09890fe..8c9a86dce 100644--- a/scrapy/templates/project/module/middlewares.py.tmpl+++ b/scrapy/templates/project/module/middlewares.py.tmpl@@ -53,7 +53,7 @@ class ${ProjectName}SpiderMiddleware:yield rdef spider_opened(self, spider):- spider.logger.info('Spider opened: %s' % spider.name)+ spider.logger.info("Spider opened: %s" % spider.name)class ${ProjectName}DownloaderMiddleware:@@ -100,4 +100,4 @@ class ${ProjectName}DownloaderMiddleware:passdef spider_opened(self, spider):- spider.logger.info('Spider opened: %s' % spider.name)+ spider.logger.info("Spider opened: %s" % spider.name)diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmplindex 2f6df5abc..ecb1e5e5c 100644--- a/scrapy/templates/project/module/settings.py.tmpl+++ b/scrapy/templates/project/module/settings.py.tmpl@@ -7,14 +7,14 @@# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html# https://docs.scrapy.org/en/latest/topics/spider-middleware.html-BOT_NAME = '$project_name'+BOT_NAME = "$project_name"-SPIDER_MODULES = ['$project_name.spiders']-NEWSPIDER_MODULE = '$project_name.spiders'+SPIDER_MODULES = ["$project_name.spiders"]+NEWSPIDER_MODULE = "$project_name.spiders"# Crawl responsibly by identifying yourself (and your website) on the user-agent-#USER_AGENT = '$project_name (+http://www.yourdomain.com)'+#USER_AGENT = "$project_name (+http://www.yourdomain.com)"# Obey robots.txt rulesROBOTSTXT_OBEY = True@@ -38,32 +38,32 @@ ROBOTSTXT_OBEY = True# Override the default request headers:#DEFAULT_REQUEST_HEADERS = {-# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',-# 'Accept-Language': 'en',+# "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",+# "Accept-Language": "en",#}# Enable or disable spider middlewares# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html#SPIDER_MIDDLEWARES = {-# '$project_name.middlewares.${ProjectName}SpiderMiddleware': 543,+# "$project_name.middlewares.${ProjectName}SpiderMiddleware": 543,#}# Enable or disable downloader middlewares# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#DOWNLOADER_MIDDLEWARES = {-# '$project_name.middlewares.${ProjectName}DownloaderMiddleware': 543,+# "$project_name.middlewares.${ProjectName}DownloaderMiddleware": 543,#}# Enable or disable extensions# See https://docs.scrapy.org/en/latest/topics/extensions.html#EXTENSIONS = {-# 'scrapy.extensions.telnet.TelnetConsole': None,+# "scrapy.extensions.telnet.TelnetConsole": None,#}# Configure item pipelines# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html#ITEM_PIPELINES = {-# '$project_name.pipelines.${ProjectName}Pipeline': 300,+# "$project_name.pipelines.${ProjectName}Pipeline": 300,#}# Enable and configure the AutoThrottle extension (disabled by default)@@ -83,11 +83,11 @@ ROBOTSTXT_OBEY = True# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings#HTTPCACHE_ENABLED = True#HTTPCACHE_EXPIRATION_SECS = 0-#HTTPCACHE_DIR = 'httpcache'+#HTTPCACHE_DIR = "httpcache"#HTTPCACHE_IGNORE_HTTP_CODES = []-#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'+#HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"# Set settings whose default value is deprecated to a future-proof value-REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7'-TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor'-FEED_EXPORT_ENCODING = 'utf-8'+REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7"+TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"+FEED_EXPORT_ENCODING = "utf-8"diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmplindex e9112bc95..d3ba19553 100644--- a/scrapy/templates/spiders/basic.tmpl+++ b/scrapy/templates/spiders/basic.tmpl@@ -2,9 +2,9 @@ import scrapyclass $classname(scrapy.Spider):- name = '$name'- allowed_domains = ['$domain']- start_urls = ['http://$domain/']+ name = "$name"+ allowed_domains = ["$domain"]+ start_urls = ["http://$domain/"]def parse(self, response):passdiff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmplindex 356496487..2e467e632 100644--- a/scrapy/templates/spiders/crawl.tmpl+++ b/scrapy/templates/spiders/crawl.tmpl@@ -4,17 +4,15 @@ from scrapy.spiders import CrawlSpider, Ruleclass $classname(CrawlSpider):- name = '$name'- allowed_domains = ['$domain']- start_urls = ['http://$domain/']+ name = "$name"+ allowed_domains = ["$domain"]+ start_urls = ["http://$domain/"]- rules = (- Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True),- )+ rules = (Rule(LinkExtractor(allow=r"Items/"), callback="parse_item", follow=True),)def parse_item(self, response):item = {}- #item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get()- #item['name'] = response.xpath('//div[@id="name"]').get()- #item['description'] = response.xpath('//div[@id="description"]').get()+ #item["domain_id"] = response.xpath('//input[@id="sid"]/@value').get()+ #item["name"] = response.xpath('//div[@id="name"]').get()+ #item["description"] = response.xpath('//div[@id="description"]').get()return itemdiff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmplindex cbcbe9e2c..ce9c1dd20 100644--- a/scrapy/templates/spiders/csvfeed.tmpl+++ b/scrapy/templates/spiders/csvfeed.tmpl@@ -2,11 +2,11 @@ from scrapy.spiders import CSVFeedSpiderclass $classname(CSVFeedSpider):- name = '$name'- allowed_domains = ['$domain']- start_urls = ['http://$domain/feed.csv']- # headers = ['id', 'name', 'description', 'image_link']- # delimiter = '\t'+ name = "$name"+ allowed_domains = ["$domain"]+ start_urls = ["http://$domain/feed.csv"]+ #headers = ["id", "name", "description", "image_link"]+ #delimiter = "\t"# Do any adaptations you need here#def adapt_response(self, response):@@ -14,7 +14,7 @@ class $classname(CSVFeedSpider):def parse_row(self, response, row):i = {}- #i['url'] = row['url']- #i['name'] = row['name']- #i['description'] = row['description']+ #i["url"] = row["url"]+ #i["name"] = row["name"]+ #i["description"] = row["description"]return idiff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmplindex 5aa2aa8b0..6b50e4cf4 100644--- a/scrapy/templates/spiders/xmlfeed.tmpl+++ b/scrapy/templates/spiders/xmlfeed.tmpl@@ -2,15 +2,15 @@ from scrapy.spiders import XMLFeedSpiderclass $classname(XMLFeedSpider):- name = '$name'- allowed_domains = ['$domain']- start_urls = ['http://$domain/feed.xml']- iterator = 'iternodes' # you can change this; see the docs- itertag = 'item' # change it accordingly+ name = "$name"+ allowed_domains = ["$domain"]+ start_urls = ["http://$domain/feed.xml"]+ iterator = "iternodes" # you can change this; see the docs+ itertag = "item" # change it accordinglydef parse_node(self, response, selector):item = {}- #item['url'] = selector.select('url').get()- #item['name'] = selector.select('name').get()- #item['description'] = selector.select('description').get()+ #item["url"] = selector.select("url").get()+ #item["name"] = selector.select("name").get()+ #item["description"] = selector.select("description").get()return itemdiff --git a/tests/test_commands.py b/tests/test_commands.pyindex 5ff2dd482..d7ee75559 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -505,7 +505,7 @@ class GenspiderCommandTest(CommandTest):# change name of spider but not its file namewith file_path.open("r+", encoding="utf-8") as spider_file:file_data = spider_file.read()- file_data = file_data.replace("name = 'example'", "name = 'renamed'")+ file_data = file_data.replace('name = "example"', 'name = "renamed"')spider_file.seek(0)spider_file.write(file_data)spider_file.truncate()@@ -538,14 +538,14 @@ class GenspiderCommandTest(CommandTest):domain,self.find_in_file(Path(self.proj_mod_path, "spiders", "test_name.py"),- r"allowed_domains\s*=\s*\[\'(.+)\'\]",+ r"allowed_domains\s*=\s*\[['\"](.+)['\"]\]",).group(1),)self.assertEqual(f"http://{domain}/",self.find_in_file(Path(self.proj_mod_path, "spiders", "test_name.py"),- r"start_urls\s*=\s*\[\'(.+)\'\]",+ r"start_urls\s*=\s*\[['\"](.+)['\"]\]",).group(1),)