Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Apply black formatting to templates (#5814)

Alex8c8894f4bec6
diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl
index bd09890fe..8c9a86dce 100644
--- a/scrapy/templates/project/module/middlewares.py.tmpl
+++ b/scrapy/templates/project/module/middlewares.py.tmpl
@@ -53,7 +53,7 @@ class ${ProjectName}SpiderMiddleware:
yield r
def spider_opened(self, spider):
- spider.logger.info('Spider opened: %s' % spider.name)
+ spider.logger.info("Spider opened: %s" % spider.name)
class ${ProjectName}DownloaderMiddleware:
@@ -100,4 +100,4 @@ class ${ProjectName}DownloaderMiddleware:
pass
def spider_opened(self, spider):
- spider.logger.info('Spider opened: %s' % spider.name)
+ spider.logger.info("Spider opened: %s" % spider.name)
diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index 2f6df5abc..ecb1e5e5c 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -7,14 +7,14 @@
# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
-BOT_NAME = '$project_name'
+BOT_NAME = "$project_name"
-SPIDER_MODULES = ['$project_name.spiders']
-NEWSPIDER_MODULE = '$project_name.spiders'
+SPIDER_MODULES = ["$project_name.spiders"]
+NEWSPIDER_MODULE = "$project_name.spiders"
# Crawl responsibly by identifying yourself (and your website) on the user-agent
-#USER_AGENT = '$project_name (+http://www.yourdomain.com)'
+#USER_AGENT = "$project_name (+http://www.yourdomain.com)"
# Obey robots.txt rules
ROBOTSTXT_OBEY = True
@@ -38,32 +38,32 @@ ROBOTSTXT_OBEY = True
# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
-# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-# 'Accept-Language': 'en',
+# "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+# "Accept-Language": "en",
#}
# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
-# '$project_name.middlewares.${ProjectName}SpiderMiddleware': 543,
+# "$project_name.middlewares.${ProjectName}SpiderMiddleware": 543,
#}
# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
-# '$project_name.middlewares.${ProjectName}DownloaderMiddleware': 543,
+# "$project_name.middlewares.${ProjectName}DownloaderMiddleware": 543,
#}
# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
-# 'scrapy.extensions.telnet.TelnetConsole': None,
+# "scrapy.extensions.telnet.TelnetConsole": None,
#}
# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
#ITEM_PIPELINES = {
-# '$project_name.pipelines.${ProjectName}Pipeline': 300,
+# "$project_name.pipelines.${ProjectName}Pipeline": 300,
#}
# Enable and configure the AutoThrottle extension (disabled by default)
@@ -83,11 +83,11 @@ ROBOTSTXT_OBEY = True
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
-#HTTPCACHE_DIR = 'httpcache'
+#HTTPCACHE_DIR = "httpcache"
#HTTPCACHE_IGNORE_HTTP_CODES = []
-#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
+#HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
# Set settings whose default value is deprecated to a future-proof value
-REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7'
-TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor'
-FEED_EXPORT_ENCODING = 'utf-8'
+REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7"
+TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
+FEED_EXPORT_ENCODING = "utf-8"
diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl
index e9112bc95..d3ba19553 100644
--- a/scrapy/templates/spiders/basic.tmpl
+++ b/scrapy/templates/spiders/basic.tmpl
@@ -2,9 +2,9 @@ import scrapy
class $classname(scrapy.Spider):
- name = '$name'
- allowed_domains = ['$domain']
- start_urls = ['http://$domain/']
+ name = "$name"
+ allowed_domains = ["$domain"]
+ start_urls = ["http://$domain/"]
def parse(self, response):
pass
diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl
index 356496487..2e467e632 100644
--- a/scrapy/templates/spiders/crawl.tmpl
+++ b/scrapy/templates/spiders/crawl.tmpl
@@ -4,17 +4,15 @@ from scrapy.spiders import CrawlSpider, Rule
class $classname(CrawlSpider):
- name = '$name'
- allowed_domains = ['$domain']
- start_urls = ['http://$domain/']
+ name = "$name"
+ allowed_domains = ["$domain"]
+ start_urls = ["http://$domain/"]
- rules = (
- Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True),
- )
+ rules = (Rule(LinkExtractor(allow=r"Items/"), callback="parse_item", follow=True),)
def parse_item(self, response):
item = {}
- #item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get()
- #item['name'] = response.xpath('//div[@id="name"]').get()
- #item['description'] = response.xpath('//div[@id="description"]').get()
+ #item["domain_id"] = response.xpath('//input[@id="sid"]/@value').get()
+ #item["name"] = response.xpath('//div[@id="name"]').get()
+ #item["description"] = response.xpath('//div[@id="description"]').get()
return item
diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl
index cbcbe9e2c..ce9c1dd20 100644
--- a/scrapy/templates/spiders/csvfeed.tmpl
+++ b/scrapy/templates/spiders/csvfeed.tmpl
@@ -2,11 +2,11 @@ from scrapy.spiders import CSVFeedSpider
class $classname(CSVFeedSpider):
- name = '$name'
- allowed_domains = ['$domain']
- start_urls = ['http://$domain/feed.csv']
- # headers = ['id', 'name', 'description', 'image_link']
- # delimiter = '\t'
+ name = "$name"
+ allowed_domains = ["$domain"]
+ start_urls = ["http://$domain/feed.csv"]
+ #headers = ["id", "name", "description", "image_link"]
+ #delimiter = "\t"
# Do any adaptations you need here
#def adapt_response(self, response):
@@ -14,7 +14,7 @@ class $classname(CSVFeedSpider):
def parse_row(self, response, row):
i = {}
- #i['url'] = row['url']
- #i['name'] = row['name']
- #i['description'] = row['description']
+ #i["url"] = row["url"]
+ #i["name"] = row["name"]
+ #i["description"] = row["description"]
return i
diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl
index 5aa2aa8b0..6b50e4cf4 100644
--- a/scrapy/templates/spiders/xmlfeed.tmpl
+++ b/scrapy/templates/spiders/xmlfeed.tmpl
@@ -2,15 +2,15 @@ from scrapy.spiders import XMLFeedSpider
class $classname(XMLFeedSpider):
- name = '$name'
- allowed_domains = ['$domain']
- start_urls = ['http://$domain/feed.xml']
- iterator = 'iternodes' # you can change this; see the docs
- itertag = 'item' # change it accordingly
+ name = "$name"
+ allowed_domains = ["$domain"]
+ start_urls = ["http://$domain/feed.xml"]
+ iterator = "iternodes" # you can change this; see the docs
+ itertag = "item" # change it accordingly
def parse_node(self, response, selector):
item = {}
- #item['url'] = selector.select('url').get()
- #item['name'] = selector.select('name').get()
- #item['description'] = selector.select('description').get()
+ #item["url"] = selector.select("url").get()
+ #item["name"] = selector.select("name").get()
+ #item["description"] = selector.select("description").get()
return item
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 5ff2dd482..d7ee75559 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -505,7 +505,7 @@ class GenspiderCommandTest(CommandTest):
# change name of spider but not its file name
with file_path.open("r+", encoding="utf-8") as spider_file:
file_data = spider_file.read()
- file_data = file_data.replace("name = 'example'", "name = 'renamed'")
+ file_data = file_data.replace('name = "example"', 'name = "renamed"')
spider_file.seek(0)
spider_file.write(file_data)
spider_file.truncate()
@@ -538,14 +538,14 @@ class GenspiderCommandTest(CommandTest):
domain,
self.find_in_file(
Path(self.proj_mod_path, "spiders", "test_name.py"),
- r"allowed_domains\s*=\s*\[\'(.+)\'\]",
+ r"allowed_domains\s*=\s*\[['\"](.+)['\"]\]",
).group(1),
)
self.assertEqual(
f"http://{domain}/",
self.find_in_file(
Path(self.proj_mod_path, "spiders", "test_name.py"),
- r"start_urls\s*=\s*\[\'(.+)\'\]",
+ r"start_urls\s*=\s*\[['\"](.+)['\"]\]",
).group(1),
)