Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Generate read-only files on the fly

Adrián Chavesca77ca1f751a
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index e702d7cdc..eccc2a3e1 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -20,7 +20,12 @@ TEMPLATES_TO_RENDER = (
('${project_name}', 'middlewares.py.tmpl'),
)
-IGNORE = ignore_patterns('*.pyc', '.svn')
+IGNORE = ignore_patterns('*.pyc', '__pycache__', '.svn')
+
+
+def _make_writable(path):
+ current_permissions = os.stat(path).st_mode
+ os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION)
class Command(ScrapyCommand):
@@ -78,12 +83,10 @@ class Command(ScrapyCommand):
self._copytree(srcname, dstname)
else:
copy2(srcname, dstname)
- current_permissions = os.stat(dstname).st_mode
- os.chmod(dstname, current_permissions | OWNER_WRITE_PERMISSION)
+ _make_writable(dstname)
copystat(src, dst)
- current_permissions = os.stat(dst).st_mode
- os.chmod(dst, current_permissions | OWNER_WRITE_PERMISSION)
+ _make_writable(dst)
def run(self, args, opts):
if len(args) not in (1, 2):
diff --git a/tests/sample_data/read_only_templates/project/module/__init__.py b/tests/sample_data/read_only_templates/project/module/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/tests/sample_data/read_only_templates/project/module/items.py.tmpl b/tests/sample_data/read_only_templates/project/module/items.py.tmpl
deleted file mode 100644
index 88a18331c..000000000
--- a/tests/sample_data/read_only_templates/project/module/items.py.tmpl
+++ /dev/null
@@ -1,12 +0,0 @@
-# Define here the models for your scraped items
-#
-# See documentation in:
-# https://docs.scrapy.org/en/latest/topics/items.html
-
-import scrapy
-
-
-class ${ProjectName}Item(scrapy.Item):
- # define the fields for your item here like:
- # name = scrapy.Field()
- pass
diff --git a/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl b/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl
deleted file mode 100644
index bd09890fe..000000000
--- a/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl
+++ /dev/null
@@ -1,103 +0,0 @@
-# Define here the models for your spider middleware
-#
-# See documentation in:
-# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
-
-from scrapy import signals
-
-# useful for handling different item types with a single interface
-from itemadapter import is_item, ItemAdapter
-
-
-class ${ProjectName}SpiderMiddleware:
- # Not all methods need to be defined. If a method is not defined,
- # scrapy acts as if the spider middleware does not modify the
- # passed objects.
-
- @classmethod
- def from_crawler(cls, crawler):
- # This method is used by Scrapy to create your spiders.
- s = cls()
- crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
- return s
-
- def process_spider_input(self, response, spider):
- # Called for each response that goes through the spider
- # middleware and into the spider.
-
- # Should return None or raise an exception.
- return None
-
- def process_spider_output(self, response, result, spider):
- # Called with the results returned from the Spider, after
- # it has processed the response.
-
- # Must return an iterable of Request, or item objects.
- for i in result:
- yield i
-
- def process_spider_exception(self, response, exception, spider):
- # Called when a spider or process_spider_input() method
- # (from other spider middleware) raises an exception.
-
- # Should return either None or an iterable of Request or item objects.
- pass
-
- def process_start_requests(self, start_requests, spider):
- # Called with the start requests of the spider, and works
- # similarly to the process_spider_output() method, except
- # that it doesn’t have a response associated.
-
- # Must return only requests (not items).
- for r in start_requests:
- yield r
-
- def spider_opened(self, spider):
- spider.logger.info('Spider opened: %s' % spider.name)
-
-
-class ${ProjectName}DownloaderMiddleware:
- # Not all methods need to be defined. If a method is not defined,
- # scrapy acts as if the downloader middleware does not modify the
- # passed objects.
-
- @classmethod
- def from_crawler(cls, crawler):
- # This method is used by Scrapy to create your spiders.
- s = cls()
- crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
- return s
-
- def process_request(self, request, spider):
- # Called for each request that goes through the downloader
- # middleware.
-
- # Must either:
- # - return None: continue processing this request
- # - or return a Response object
- # - or return a Request object
- # - or raise IgnoreRequest: process_exception() methods of
- # installed downloader middleware will be called
- return None
-
- def process_response(self, request, response, spider):
- # Called with the response returned from the downloader.
-
- # Must either;
- # - return a Response object
- # - return a Request object
- # - or raise IgnoreRequest
- return response
-
- def process_exception(self, request, exception, spider):
- # Called when a download handler or a process_request()
- # (from other downloader middleware) raises an exception.
-
- # Must either:
- # - return None: continue processing this exception
- # - return a Response object: stops process_exception() chain
- # - return a Request object: stops process_exception() chain
- pass
-
- def spider_opened(self, spider):
- spider.logger.info('Spider opened: %s' % spider.name)
diff --git a/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl b/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl
deleted file mode 100644
index e845f43e9..000000000
--- a/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl
+++ /dev/null
@@ -1,13 +0,0 @@
-# Define your item pipelines here
-#
-# Don't forget to add your pipeline to the ITEM_PIPELINES setting
-# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
-
-
-# useful for handling different item types with a single interface
-from itemadapter import ItemAdapter
-
-
-class ${ProjectName}Pipeline:
- def process_item(self, item, spider):
- return item
diff --git a/tests/sample_data/read_only_templates/project/module/settings.py.tmpl b/tests/sample_data/read_only_templates/project/module/settings.py.tmpl
deleted file mode 100644
index a414b5fde..000000000
--- a/tests/sample_data/read_only_templates/project/module/settings.py.tmpl
+++ /dev/null
@@ -1,88 +0,0 @@
-# Scrapy settings for $project_name project
-#
-# For simplicity, this file contains only settings considered important or
-# commonly used. You can find more settings consulting the documentation:
-#
-# https://docs.scrapy.org/en/latest/topics/settings.html
-# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
-# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
-
-BOT_NAME = '$project_name'
-
-SPIDER_MODULES = ['$project_name.spiders']
-NEWSPIDER_MODULE = '$project_name.spiders'
-
-
-# Crawl responsibly by identifying yourself (and your website) on the user-agent
-#USER_AGENT = '$project_name (+http://www.yourdomain.com)'
-
-# Obey robots.txt rules
-ROBOTSTXT_OBEY = True
-
-# Configure maximum concurrent requests performed by Scrapy (default: 16)
-#CONCURRENT_REQUESTS = 32
-
-# Configure a delay for requests for the same website (default: 0)
-# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
-# See also autothrottle settings and docs
-#DOWNLOAD_DELAY = 3
-# The download delay setting will honor only one of:
-#CONCURRENT_REQUESTS_PER_DOMAIN = 16
-#CONCURRENT_REQUESTS_PER_IP = 16
-
-# Disable cookies (enabled by default)
-#COOKIES_ENABLED = False
-
-# Disable Telnet Console (enabled by default)
-#TELNETCONSOLE_ENABLED = False
-
-# Override the default request headers:
-#DEFAULT_REQUEST_HEADERS = {
-# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-# 'Accept-Language': 'en',
-#}
-
-# Enable or disable spider middlewares
-# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
-#SPIDER_MIDDLEWARES = {
-# '$project_name.middlewares.${ProjectName}SpiderMiddleware': 543,
-#}
-
-# Enable or disable downloader middlewares
-# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
-#DOWNLOADER_MIDDLEWARES = {
-# '$project_name.middlewares.${ProjectName}DownloaderMiddleware': 543,
-#}
-
-# Enable or disable extensions
-# See https://docs.scrapy.org/en/latest/topics/extensions.html
-#EXTENSIONS = {
-# 'scrapy.extensions.telnet.TelnetConsole': None,
-#}
-
-# Configure item pipelines
-# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
-#ITEM_PIPELINES = {
-# '$project_name.pipelines.${ProjectName}Pipeline': 300,
-#}
-
-# Enable and configure the AutoThrottle extension (disabled by default)
-# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
-#AUTOTHROTTLE_ENABLED = True
-# The initial download delay
-#AUTOTHROTTLE_START_DELAY = 5
-# The maximum download delay to be set in case of high latencies
-#AUTOTHROTTLE_MAX_DELAY = 60
-# The average number of requests Scrapy should be sending in parallel to
-# each remote server
-#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
-# Enable showing throttling stats for every response received:
-#AUTOTHROTTLE_DEBUG = False
-
-# Enable and configure HTTP caching (disabled by default)
-# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
-#HTTPCACHE_ENABLED = True
-#HTTPCACHE_EXPIRATION_SECS = 0
-#HTTPCACHE_DIR = 'httpcache'
-#HTTPCACHE_IGNORE_HTTP_CODES = []
-#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
diff --git a/tests/sample_data/read_only_templates/project/module/spiders/__init__.py b/tests/sample_data/read_only_templates/project/module/spiders/__init__.py
deleted file mode 100644
index ebd689ac5..000000000
--- a/tests/sample_data/read_only_templates/project/module/spiders/__init__.py
+++ /dev/null
@@ -1,4 +0,0 @@
-# This package will contain the spiders of your Scrapy project
-#
-# Please refer to the documentation for information on how to create and manage
-# your spiders.
diff --git a/tests/sample_data/read_only_templates/project/scrapy.cfg b/tests/sample_data/read_only_templates/project/scrapy.cfg
deleted file mode 100644
index 1daeaa541..000000000
--- a/tests/sample_data/read_only_templates/project/scrapy.cfg
+++ /dev/null
@@ -1,11 +0,0 @@
-# Automatically created by: scrapy startproject
-#
-# For more information about the [deploy] section see:
-# https://scrapyd.readthedocs.io/en/latest/deploy.html
-
-[settings]
-default = ${project_name}.settings
-
-[deploy]
-#url = http://localhost:6800/
-project = ${project_name}
diff --git a/tests/sample_data/read_only_templates/spiders/basic.tmpl b/tests/sample_data/read_only_templates/spiders/basic.tmpl
deleted file mode 100644
index e9112bc95..000000000
--- a/tests/sample_data/read_only_templates/spiders/basic.tmpl
+++ /dev/null
@@ -1,10 +0,0 @@
-import scrapy
-
-
-class $classname(scrapy.Spider):
- name = '$name'
- allowed_domains = ['$domain']
- start_urls = ['http://$domain/']
-
- def parse(self, response):
- pass
diff --git a/tests/sample_data/read_only_templates/spiders/crawl.tmpl b/tests/sample_data/read_only_templates/spiders/crawl.tmpl
deleted file mode 100644
index 356496487..000000000
--- a/tests/sample_data/read_only_templates/spiders/crawl.tmpl
+++ /dev/null
@@ -1,20 +0,0 @@
-import scrapy
-from scrapy.linkextractors import LinkExtractor
-from scrapy.spiders import CrawlSpider, Rule
-
-
-class $classname(CrawlSpider):
- name = '$name'
- allowed_domains = ['$domain']
- start_urls = ['http://$domain/']
-
- rules = (
- Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True),
- )
-
- def parse_item(self, response):
- item = {}
- #item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get()
- #item['name'] = response.xpath('//div[@id="name"]').get()
- #item['description'] = response.xpath('//div[@id="description"]').get()
- return item
diff --git a/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl b/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl
deleted file mode 100644
index cbcbe9e2c..000000000
--- a/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl
+++ /dev/null
@@ -1,20 +0,0 @@
-from scrapy.spiders import CSVFeedSpider
-
-
-class $classname(CSVFeedSpider):
- name = '$name'
- allowed_domains = ['$domain']
- start_urls = ['http://$domain/feed.csv']
- # headers = ['id', 'name', 'description', 'image_link']
- # delimiter = '\t'
-
- # Do any adaptations you need here
- #def adapt_response(self, response):
- # return response
-
- def parse_row(self, response, row):
- i = {}
- #i['url'] = row['url']
- #i['name'] = row['name']
- #i['description'] = row['description']
- return i
diff --git a/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl b/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl
deleted file mode 100644
index 5aa2aa8b0..000000000
--- a/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl
+++ /dev/null
@@ -1,16 +0,0 @@
-from scrapy.spiders import XMLFeedSpider
-
-
-class $classname(XMLFeedSpider):
- name = '$name'
- allowed_domains = ['$domain']
- start_urls = ['http://$domain/feed.xml']
- iterator = 'iternodes' # you can change this; see the docs
- itertag = 'item' # change it accordingly
-
- def parse_node(self, response, selector):
- item = {}
- #item['url'] = selector.select('url').get()
- #item['name'] = selector.select('name').get()
- #item['description'] = selector.select('description').get()
- return item
diff --git a/tests/test_commands.py b/tests/test_commands.py
index c25495d16..f3fe45139 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -2,6 +2,7 @@ import inspect
import json
import optparse
import os
+from stat import S_IWRITE as ANYONE_WRITE_PERMISSION
import subprocess
import sys
import tempfile
@@ -17,7 +18,7 @@ from twisted.trial import unittest
import scrapy
from scrapy.commands import ScrapyCommand
-from scrapy.commands.startproject import IGNORE
+from scrapy.commands.startproject import IGNORE, _make_writable
from scrapy.settings import Settings
from scrapy.utils.python import to_unicode
from scrapy.utils.test import get_testenv
@@ -170,14 +171,14 @@ class StartprojectTemplatesTest(ProjectTest):
template folder has the same permissions as in the project, i.e.
everything is writable."""
scrapy_path = scrapy.__path__[0]
- templates_dir = os.path.join(scrapy_path, 'templates', 'project')
+ project_template = os.path.join(scrapy_path, 'templates', 'project')
project_name = 'startproject1'
renamings = (
('module', project_name),
('.tmpl', ''),
)
expected_permissions = get_permissions_dict(
- templates_dir,
+ project_template,
renamings,
IGNORE,
)
@@ -209,22 +210,30 @@ class StartprojectTemplatesTest(ProjectTest):
See https://github.com/scrapy/scrapy/pull/4604
"""
scrapy_path = scrapy.__path__[0]
- templates_dir = os.path.join(scrapy_path, 'templates', 'project')
+ templates_dir = os.path.join(scrapy_path, 'templates')
+ project_template = os.path.join(templates_dir, 'project')
project_name = 'startproject2'
renamings = (
('module', project_name),
('.tmpl', ''),
)
expected_permissions = get_permissions_dict(
- templates_dir,
+ project_template,
renamings,
IGNORE,
)
- tests_path = os.path.dirname(__file__)
- read_only_templates_dir = os.path.join(
- tests_path, 'sample_data', 'read_only_templates'
- )
+ def _make_read_only(path):
+ current_permissions = os.stat(path).st_mode
+ os.chmod(path, current_permissions & ~ANYONE_WRITE_PERMISSION)
+
+ read_only_templates_dir = str(Path(mkdtemp()) / 'templates')
+ copytree(templates_dir, read_only_templates_dir)
+
+ for root, dirs, files in os.walk(read_only_templates_dir):
+ for node in chain(dirs, files):
+ _make_read_only(os.path.join(root, node))
+
destination = mkdtemp()
process = subprocess.Popen(
(
@@ -250,14 +259,14 @@ class StartprojectTemplatesTest(ProjectTest):
"""Check that pre-existing folders and files in the destination folder
do not see their permissions modified."""
scrapy_path = scrapy.__path__[0]
- templates_dir = os.path.join(scrapy_path, 'templates', 'project')
+ project_template = os.path.join(scrapy_path, 'templates', 'project')
project_name = 'startproject3'
renamings = (
('module', project_name),
('.tmpl', ''),
)
expected_permissions = get_permissions_dict(
- templates_dir,
+ project_template,
renamings,
IGNORE,
)