Scrapy, a fast high-level web crawling & scraping framework for Python.
Support genspider with HTTPS (#5808)
Alex101a0c32d710
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rstindex 54fd5d663..106045fc0 100644--- a/docs/topics/commands.rst+++ b/docs/topics/commands.rst@@ -238,9 +238,6 @@ genspiderCreate a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain or URL>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes.-.. note:: Even if an HTTPS URL is specified, the protocol used in- ``start_urls`` is always HTTP. This is a known issue: :issue:`3553`.-Usage example::$ scrapy genspider -ldiff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.pyindex c1565a138..68cbe8ff6 100644--- a/scrapy/commands/genspider.py+++ b/scrapy/commands/genspider.py@@ -31,6 +31,14 @@ def extract_domain(url):return o.netloc+def verify_url_scheme(url):+ """Check url for scheme and insert https if none found."""+ parsed = urlparse(url)+ if parsed.scheme == "" and parsed.netloc == "":+ parsed = urlparse("//" + url)._replace(scheme="https")+ return parsed.geturl()++class Command(ScrapyCommand):requires_project = Falsedefault_settings = {"LOG_ENABLED": False}@@ -91,7 +99,7 @@ class Command(ScrapyCommand):raise UsageError()name, url = args[0:2]- domain = extract_domain(url)+ url = verify_url_scheme(url)module = sanitize_module_name(name)if self.settings.get("BOT_NAME") == module:@@ -103,18 +111,20 @@ class Command(ScrapyCommand):template_file = self._find_template(opts.template)if template_file:- self._genspider(module, name, domain, opts.template, template_file)+ self._genspider(module, name, url, opts.template, template_file)if opts.edit:self.exitcode = os.system(f'scrapy edit "{name}"')- def _genspider(self, module, name, domain, template_name, template_file):+ def _genspider(self, module, name, url, template_name, template_file):"""Generate the spider module, based on the given template"""capitalized_module = "".join(s.capitalize() for s in module.split("_"))+ domain = extract_domain(url)tvars = {"project_name": self.settings.get("BOT_NAME"),"ProjectName": string_camelcase(self.settings.get("BOT_NAME")),"module": module,"name": name,+ "url": url,"domain": domain,"classname": f"{capitalized_module}Spider",}diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmplindex d3ba19553..20e777271 100644--- a/scrapy/templates/spiders/basic.tmpl+++ b/scrapy/templates/spiders/basic.tmpl@@ -4,7 +4,7 @@ import scrapyclass $classname(scrapy.Spider):name = "$name"allowed_domains = ["$domain"]- start_urls = ["http://$domain/"]+ start_urls = ["$url"]def parse(self, response):passdiff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmplindex 2e467e632..36d05e43a 100644--- a/scrapy/templates/spiders/crawl.tmpl+++ b/scrapy/templates/spiders/crawl.tmpl@@ -6,7 +6,7 @@ from scrapy.spiders import CrawlSpider, Ruleclass $classname(CrawlSpider):name = "$name"allowed_domains = ["$domain"]- start_urls = ["http://$domain/"]+ start_urls = ["$url"]rules = (Rule(LinkExtractor(allow=r"Items/"), callback="parse_item", follow=True),)diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmplindex ce9c1dd20..fe96878dc 100644--- a/scrapy/templates/spiders/csvfeed.tmpl+++ b/scrapy/templates/spiders/csvfeed.tmpl@@ -4,7 +4,7 @@ from scrapy.spiders import CSVFeedSpiderclass $classname(CSVFeedSpider):name = "$name"allowed_domains = ["$domain"]- start_urls = ["http://$domain/feed.csv"]+ start_urls = ["$url"]#headers = ["id", "name", "description", "image_link"]#delimiter = "\t"diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmplindex 6b50e4cf4..ac62d78d1 100644--- a/scrapy/templates/spiders/xmlfeed.tmpl+++ b/scrapy/templates/spiders/xmlfeed.tmpl@@ -4,7 +4,7 @@ from scrapy.spiders import XMLFeedSpiderclass $classname(XMLFeedSpider):name = "$name"allowed_domains = ["$domain"]- start_urls = ["http://$domain/feed.xml"]+ start_urls = ["$url"]iterator = "iternodes" # you can change this; see the docsitertag = "item" # change it accordinglydiff --git a/tests/test_commands.py b/tests/test_commands.pyindex 00ddcdd3e..014f50e92 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -541,7 +541,7 @@ class GenspiderCommandTest(CommandTest):).group(1),)self.assertEqual(- f"http://{domain}/",+ f"https://{domain}",self.find_in_file(Path(self.proj_mod_path, "spiders", "test_name.py"),r"start_urls\s*=\s*\[['\"](.+)['\"]\]",@@ -549,13 +549,64 @@ class GenspiderCommandTest(CommandTest):)def test_url_schema(self):- self.test_url("http://test.com", "test.com")+ self.test_url("https://test.com", "test.com")- def test_url_path(self):- self.test_url("test.com/some/other/page", "test.com")+ def test_template_start_urls(+ self, url="test.com", expected="https://test.com", template="basic"+ ):+ self.assertEqual(+ 0, self.call("genspider", "-t", template, "--force", "test_name", url)+ )+ self.assertEqual(+ expected,+ self.find_in_file(+ Path(self.proj_mod_path, "spiders", "test_name.py"),+ r"start_urls\s*=\s*\[['\"](.+)['\"]\]",+ ).group(1),+ )++ def test_genspider_basic_start_urls(self):+ self.test_template_start_urls("https://test.com", "https://test.com", "basic")+ self.test_template_start_urls("http://test.com", "http://test.com", "basic")+ self.test_template_start_urls(+ "http://test.com/other/path", "http://test.com/other/path", "basic"+ )+ self.test_template_start_urls(+ "test.com/other/path", "https://test.com/other/path", "basic"+ )- def test_url_schema_path(self):- self.test_url("https://test.com/some/other/page", "test.com")+ def test_genspider_crawl_start_urls(self):+ self.test_template_start_urls("https://test.com", "https://test.com", "crawl")+ self.test_template_start_urls("http://test.com", "http://test.com", "crawl")+ self.test_template_start_urls(+ "http://test.com/other/path", "http://test.com/other/path", "crawl"+ )+ self.test_template_start_urls(+ "test.com/other/path", "https://test.com/other/path", "crawl"+ )+ self.test_template_start_urls("test.com", "https://test.com", "crawl")++ def test_genspider_xmlfeed_start_urls(self):+ self.test_template_start_urls(+ "https://test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"+ )+ self.test_template_start_urls(+ "http://test.com/feed.xml", "http://test.com/feed.xml", "xmlfeed"+ )+ self.test_template_start_urls(+ "test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"+ )++ def test_genspider_csvfeed_start_urls(self):+ self.test_template_start_urls(+ "https://test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"+ )+ self.test_template_start_urls(+ "http://test.com/feed.xml", "http://test.com/feed.xml", "csvfeed"+ )+ self.test_template_start_urls(+ "test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"+ )class GenspiderStandaloneCommandTest(ProjectTest):