Scrapy, a fast high-level web crawling & scraping framework for Python.
Generate read-only files on the fly
Adrián Chavesca77ca1f751a
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.pyindex e702d7cdc..eccc2a3e1 100644--- a/scrapy/commands/startproject.py+++ b/scrapy/commands/startproject.py@@ -20,7 +20,12 @@ TEMPLATES_TO_RENDER = (('${project_name}', 'middlewares.py.tmpl'),)-IGNORE = ignore_patterns('*.pyc', '.svn')+IGNORE = ignore_patterns('*.pyc', '__pycache__', '.svn')+++def _make_writable(path):+ current_permissions = os.stat(path).st_mode+ os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION)class Command(ScrapyCommand):@@ -78,12 +83,10 @@ class Command(ScrapyCommand):self._copytree(srcname, dstname)else:copy2(srcname, dstname)- current_permissions = os.stat(dstname).st_mode- os.chmod(dstname, current_permissions | OWNER_WRITE_PERMISSION)+ _make_writable(dstname)copystat(src, dst)- current_permissions = os.stat(dst).st_mode- os.chmod(dst, current_permissions | OWNER_WRITE_PERMISSION)+ _make_writable(dst)def run(self, args, opts):if len(args) not in (1, 2):diff --git a/tests/sample_data/read_only_templates/project/module/__init__.py b/tests/sample_data/read_only_templates/project/module/__init__.pydeleted file mode 100644index e69de29bb..000000000diff --git a/tests/sample_data/read_only_templates/project/module/items.py.tmpl b/tests/sample_data/read_only_templates/project/module/items.py.tmpldeleted file mode 100644index 88a18331c..000000000--- a/tests/sample_data/read_only_templates/project/module/items.py.tmpl+++ /dev/null@@ -1,12 +0,0 @@-# Define here the models for your scraped items-#-# See documentation in:-# https://docs.scrapy.org/en/latest/topics/items.html--import scrapy---class ${ProjectName}Item(scrapy.Item):- # define the fields for your item here like:- # name = scrapy.Field()- passdiff --git a/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl b/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpldeleted file mode 100644index bd09890fe..000000000--- a/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl+++ /dev/null@@ -1,103 +0,0 @@-# Define here the models for your spider middleware-#-# See documentation in:-# https://docs.scrapy.org/en/latest/topics/spider-middleware.html--from scrapy import signals--# useful for handling different item types with a single interface-from itemadapter import is_item, ItemAdapter---class ${ProjectName}SpiderMiddleware:- # Not all methods need to be defined. If a method is not defined,- # scrapy acts as if the spider middleware does not modify the- # passed objects.-- @classmethod- def from_crawler(cls, crawler):- # This method is used by Scrapy to create your spiders.- s = cls()- crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)- return s-- def process_spider_input(self, response, spider):- # Called for each response that goes through the spider- # middleware and into the spider.-- # Should return None or raise an exception.- return None-- def process_spider_output(self, response, result, spider):- # Called with the results returned from the Spider, after- # it has processed the response.-- # Must return an iterable of Request, or item objects.- for i in result:- yield i-- def process_spider_exception(self, response, exception, spider):- # Called when a spider or process_spider_input() method- # (from other spider middleware) raises an exception.-- # Should return either None or an iterable of Request or item objects.- pass-- def process_start_requests(self, start_requests, spider):- # Called with the start requests of the spider, and works- # similarly to the process_spider_output() method, except- # that it doesn’t have a response associated.-- # Must return only requests (not items).- for r in start_requests:- yield r-- def spider_opened(self, spider):- spider.logger.info('Spider opened: %s' % spider.name)---class ${ProjectName}DownloaderMiddleware:- # Not all methods need to be defined. If a method is not defined,- # scrapy acts as if the downloader middleware does not modify the- # passed objects.-- @classmethod- def from_crawler(cls, crawler):- # This method is used by Scrapy to create your spiders.- s = cls()- crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)- return s-- def process_request(self, request, spider):- # Called for each request that goes through the downloader- # middleware.-- # Must either:- # - return None: continue processing this request- # - or return a Response object- # - or return a Request object- # - or raise IgnoreRequest: process_exception() methods of- # installed downloader middleware will be called- return None-- def process_response(self, request, response, spider):- # Called with the response returned from the downloader.-- # Must either;- # - return a Response object- # - return a Request object- # - or raise IgnoreRequest- return response-- def process_exception(self, request, exception, spider):- # Called when a download handler or a process_request()- # (from other downloader middleware) raises an exception.-- # Must either:- # - return None: continue processing this exception- # - return a Response object: stops process_exception() chain- # - return a Request object: stops process_exception() chain- pass-- def spider_opened(self, spider):- spider.logger.info('Spider opened: %s' % spider.name)diff --git a/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl b/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpldeleted file mode 100644index e845f43e9..000000000--- a/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl+++ /dev/null@@ -1,13 +0,0 @@-# Define your item pipelines here-#-# Don't forget to add your pipeline to the ITEM_PIPELINES setting-# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html---# useful for handling different item types with a single interface-from itemadapter import ItemAdapter---class ${ProjectName}Pipeline:- def process_item(self, item, spider):- return itemdiff --git a/tests/sample_data/read_only_templates/project/module/settings.py.tmpl b/tests/sample_data/read_only_templates/project/module/settings.py.tmpldeleted file mode 100644index a414b5fde..000000000--- a/tests/sample_data/read_only_templates/project/module/settings.py.tmpl+++ /dev/null@@ -1,88 +0,0 @@-# Scrapy settings for $project_name project-#-# For simplicity, this file contains only settings considered important or-# commonly used. You can find more settings consulting the documentation:-#-# https://docs.scrapy.org/en/latest/topics/settings.html-# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html-# https://docs.scrapy.org/en/latest/topics/spider-middleware.html--BOT_NAME = '$project_name'--SPIDER_MODULES = ['$project_name.spiders']-NEWSPIDER_MODULE = '$project_name.spiders'---# Crawl responsibly by identifying yourself (and your website) on the user-agent-#USER_AGENT = '$project_name (+http://www.yourdomain.com)'--# Obey robots.txt rules-ROBOTSTXT_OBEY = True--# Configure maximum concurrent requests performed by Scrapy (default: 16)-#CONCURRENT_REQUESTS = 32--# Configure a delay for requests for the same website (default: 0)-# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay-# See also autothrottle settings and docs-#DOWNLOAD_DELAY = 3-# The download delay setting will honor only one of:-#CONCURRENT_REQUESTS_PER_DOMAIN = 16-#CONCURRENT_REQUESTS_PER_IP = 16--# Disable cookies (enabled by default)-#COOKIES_ENABLED = False--# Disable Telnet Console (enabled by default)-#TELNETCONSOLE_ENABLED = False--# Override the default request headers:-#DEFAULT_REQUEST_HEADERS = {-# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',-# 'Accept-Language': 'en',-#}--# Enable or disable spider middlewares-# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html-#SPIDER_MIDDLEWARES = {-# '$project_name.middlewares.${ProjectName}SpiderMiddleware': 543,-#}--# Enable or disable downloader middlewares-# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html-#DOWNLOADER_MIDDLEWARES = {-# '$project_name.middlewares.${ProjectName}DownloaderMiddleware': 543,-#}--# Enable or disable extensions-# See https://docs.scrapy.org/en/latest/topics/extensions.html-#EXTENSIONS = {-# 'scrapy.extensions.telnet.TelnetConsole': None,-#}--# Configure item pipelines-# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html-#ITEM_PIPELINES = {-# '$project_name.pipelines.${ProjectName}Pipeline': 300,-#}--# Enable and configure the AutoThrottle extension (disabled by default)-# See https://docs.scrapy.org/en/latest/topics/autothrottle.html-#AUTOTHROTTLE_ENABLED = True-# The initial download delay-#AUTOTHROTTLE_START_DELAY = 5-# The maximum download delay to be set in case of high latencies-#AUTOTHROTTLE_MAX_DELAY = 60-# The average number of requests Scrapy should be sending in parallel to-# each remote server-#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0-# Enable showing throttling stats for every response received:-#AUTOTHROTTLE_DEBUG = False--# Enable and configure HTTP caching (disabled by default)-# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings-#HTTPCACHE_ENABLED = True-#HTTPCACHE_EXPIRATION_SECS = 0-#HTTPCACHE_DIR = 'httpcache'-#HTTPCACHE_IGNORE_HTTP_CODES = []-#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'diff --git a/tests/sample_data/read_only_templates/project/module/spiders/__init__.py b/tests/sample_data/read_only_templates/project/module/spiders/__init__.pydeleted file mode 100644index ebd689ac5..000000000--- a/tests/sample_data/read_only_templates/project/module/spiders/__init__.py+++ /dev/null@@ -1,4 +0,0 @@-# This package will contain the spiders of your Scrapy project-#-# Please refer to the documentation for information on how to create and manage-# your spiders.diff --git a/tests/sample_data/read_only_templates/project/scrapy.cfg b/tests/sample_data/read_only_templates/project/scrapy.cfgdeleted file mode 100644index 1daeaa541..000000000--- a/tests/sample_data/read_only_templates/project/scrapy.cfg+++ /dev/null@@ -1,11 +0,0 @@-# Automatically created by: scrapy startproject-#-# For more information about the [deploy] section see:-# https://scrapyd.readthedocs.io/en/latest/deploy.html--[settings]-default = ${project_name}.settings--[deploy]-#url = http://localhost:6800/-project = ${project_name}diff --git a/tests/sample_data/read_only_templates/spiders/basic.tmpl b/tests/sample_data/read_only_templates/spiders/basic.tmpldeleted file mode 100644index e9112bc95..000000000--- a/tests/sample_data/read_only_templates/spiders/basic.tmpl+++ /dev/null@@ -1,10 +0,0 @@-import scrapy---class $classname(scrapy.Spider):- name = '$name'- allowed_domains = ['$domain']- start_urls = ['http://$domain/']-- def parse(self, response):- passdiff --git a/tests/sample_data/read_only_templates/spiders/crawl.tmpl b/tests/sample_data/read_only_templates/spiders/crawl.tmpldeleted file mode 100644index 356496487..000000000--- a/tests/sample_data/read_only_templates/spiders/crawl.tmpl+++ /dev/null@@ -1,20 +0,0 @@-import scrapy-from scrapy.linkextractors import LinkExtractor-from scrapy.spiders import CrawlSpider, Rule---class $classname(CrawlSpider):- name = '$name'- allowed_domains = ['$domain']- start_urls = ['http://$domain/']-- rules = (- Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True),- )-- def parse_item(self, response):- item = {}- #item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get()- #item['name'] = response.xpath('//div[@id="name"]').get()- #item['description'] = response.xpath('//div[@id="description"]').get()- return itemdiff --git a/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl b/tests/sample_data/read_only_templates/spiders/csvfeed.tmpldeleted file mode 100644index cbcbe9e2c..000000000--- a/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl+++ /dev/null@@ -1,20 +0,0 @@-from scrapy.spiders import CSVFeedSpider---class $classname(CSVFeedSpider):- name = '$name'- allowed_domains = ['$domain']- start_urls = ['http://$domain/feed.csv']- # headers = ['id', 'name', 'description', 'image_link']- # delimiter = '\t'-- # Do any adaptations you need here- #def adapt_response(self, response):- # return response-- def parse_row(self, response, row):- i = {}- #i['url'] = row['url']- #i['name'] = row['name']- #i['description'] = row['description']- return idiff --git a/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl b/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpldeleted file mode 100644index 5aa2aa8b0..000000000--- a/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl+++ /dev/null@@ -1,16 +0,0 @@-from scrapy.spiders import XMLFeedSpider---class $classname(XMLFeedSpider):- name = '$name'- allowed_domains = ['$domain']- start_urls = ['http://$domain/feed.xml']- iterator = 'iternodes' # you can change this; see the docs- itertag = 'item' # change it accordingly-- def parse_node(self, response, selector):- item = {}- #item['url'] = selector.select('url').get()- #item['name'] = selector.select('name').get()- #item['description'] = selector.select('description').get()- return itemdiff --git a/tests/test_commands.py b/tests/test_commands.pyindex c25495d16..f3fe45139 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -2,6 +2,7 @@ import inspectimport jsonimport optparseimport os+from stat import S_IWRITE as ANYONE_WRITE_PERMISSIONimport subprocessimport sysimport tempfile@@ -17,7 +18,7 @@ from twisted.trial import unittestimport scrapyfrom scrapy.commands import ScrapyCommand-from scrapy.commands.startproject import IGNORE+from scrapy.commands.startproject import IGNORE, _make_writablefrom scrapy.settings import Settingsfrom scrapy.utils.python import to_unicodefrom scrapy.utils.test import get_testenv@@ -170,14 +171,14 @@ class StartprojectTemplatesTest(ProjectTest):template folder has the same permissions as in the project, i.e.everything is writable."""scrapy_path = scrapy.__path__[0]- templates_dir = os.path.join(scrapy_path, 'templates', 'project')+ project_template = os.path.join(scrapy_path, 'templates', 'project')project_name = 'startproject1'renamings = (('module', project_name),('.tmpl', ''),)expected_permissions = get_permissions_dict(- templates_dir,+ project_template,renamings,IGNORE,)@@ -209,22 +210,30 @@ class StartprojectTemplatesTest(ProjectTest):See https://github.com/scrapy/scrapy/pull/4604"""scrapy_path = scrapy.__path__[0]- templates_dir = os.path.join(scrapy_path, 'templates', 'project')+ templates_dir = os.path.join(scrapy_path, 'templates')+ project_template = os.path.join(templates_dir, 'project')project_name = 'startproject2'renamings = (('module', project_name),('.tmpl', ''),)expected_permissions = get_permissions_dict(- templates_dir,+ project_template,renamings,IGNORE,)- tests_path = os.path.dirname(__file__)- read_only_templates_dir = os.path.join(- tests_path, 'sample_data', 'read_only_templates'- )+ def _make_read_only(path):+ current_permissions = os.stat(path).st_mode+ os.chmod(path, current_permissions & ~ANYONE_WRITE_PERMISSION)++ read_only_templates_dir = str(Path(mkdtemp()) / 'templates')+ copytree(templates_dir, read_only_templates_dir)++ for root, dirs, files in os.walk(read_only_templates_dir):+ for node in chain(dirs, files):+ _make_read_only(os.path.join(root, node))+destination = mkdtemp()process = subprocess.Popen((@@ -250,14 +259,14 @@ class StartprojectTemplatesTest(ProjectTest):"""Check that pre-existing folders and files in the destination folderdo not see their permissions modified."""scrapy_path = scrapy.__path__[0]- templates_dir = os.path.join(scrapy_path, 'templates', 'project')+ project_template = os.path.join(scrapy_path, 'templates', 'project')project_name = 'startproject3'renamings = (('module', project_name),('.tmpl', ''),)expected_permissions = get_permissions_dict(- templates_dir,+ project_template,renamings,IGNORE,)