Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Support genspider with HTTPS (#5808)

Alex101a0c32d710
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 54fd5d663..106045fc0 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -238,9 +238,6 @@ genspider
Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain or URL>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes.
-.. note:: Even if an HTTPS URL is specified, the protocol used in
- ``start_urls`` is always HTTP. This is a known issue: :issue:`3553`.
-
Usage example::
$ scrapy genspider -l
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index c1565a138..68cbe8ff6 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -31,6 +31,14 @@ def extract_domain(url):
return o.netloc
+def verify_url_scheme(url):
+ """Check url for scheme and insert https if none found."""
+ parsed = urlparse(url)
+ if parsed.scheme == "" and parsed.netloc == "":
+ parsed = urlparse("//" + url)._replace(scheme="https")
+ return parsed.geturl()
+
+
class Command(ScrapyCommand):
requires_project = False
default_settings = {"LOG_ENABLED": False}
@@ -91,7 +99,7 @@ class Command(ScrapyCommand):
raise UsageError()
name, url = args[0:2]
- domain = extract_domain(url)
+ url = verify_url_scheme(url)
module = sanitize_module_name(name)
if self.settings.get("BOT_NAME") == module:
@@ -103,18 +111,20 @@ class Command(ScrapyCommand):
template_file = self._find_template(opts.template)
if template_file:
- self._genspider(module, name, domain, opts.template, template_file)
+ self._genspider(module, name, url, opts.template, template_file)
if opts.edit:
self.exitcode = os.system(f'scrapy edit "{name}"')
- def _genspider(self, module, name, domain, template_name, template_file):
+ def _genspider(self, module, name, url, template_name, template_file):
"""Generate the spider module, based on the given template"""
capitalized_module = "".join(s.capitalize() for s in module.split("_"))
+ domain = extract_domain(url)
tvars = {
"project_name": self.settings.get("BOT_NAME"),
"ProjectName": string_camelcase(self.settings.get("BOT_NAME")),
"module": module,
"name": name,
+ "url": url,
"domain": domain,
"classname": f"{capitalized_module}Spider",
}
diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl
index d3ba19553..20e777271 100644
--- a/scrapy/templates/spiders/basic.tmpl
+++ b/scrapy/templates/spiders/basic.tmpl
@@ -4,7 +4,7 @@ import scrapy
class $classname(scrapy.Spider):
name = "$name"
allowed_domains = ["$domain"]
- start_urls = ["http://$domain/"]
+ start_urls = ["$url"]
def parse(self, response):
pass
diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl
index 2e467e632..36d05e43a 100644
--- a/scrapy/templates/spiders/crawl.tmpl
+++ b/scrapy/templates/spiders/crawl.tmpl
@@ -6,7 +6,7 @@ from scrapy.spiders import CrawlSpider, Rule
class $classname(CrawlSpider):
name = "$name"
allowed_domains = ["$domain"]
- start_urls = ["http://$domain/"]
+ start_urls = ["$url"]
rules = (Rule(LinkExtractor(allow=r"Items/"), callback="parse_item", follow=True),)
diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl
index ce9c1dd20..fe96878dc 100644
--- a/scrapy/templates/spiders/csvfeed.tmpl
+++ b/scrapy/templates/spiders/csvfeed.tmpl
@@ -4,7 +4,7 @@ from scrapy.spiders import CSVFeedSpider
class $classname(CSVFeedSpider):
name = "$name"
allowed_domains = ["$domain"]
- start_urls = ["http://$domain/feed.csv"]
+ start_urls = ["$url"]
#headers = ["id", "name", "description", "image_link"]
#delimiter = "\t"
diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl
index 6b50e4cf4..ac62d78d1 100644
--- a/scrapy/templates/spiders/xmlfeed.tmpl
+++ b/scrapy/templates/spiders/xmlfeed.tmpl
@@ -4,7 +4,7 @@ from scrapy.spiders import XMLFeedSpider
class $classname(XMLFeedSpider):
name = "$name"
allowed_domains = ["$domain"]
- start_urls = ["http://$domain/feed.xml"]
+ start_urls = ["$url"]
iterator = "iternodes" # you can change this; see the docs
itertag = "item" # change it accordingly
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 00ddcdd3e..014f50e92 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -541,7 +541,7 @@ class GenspiderCommandTest(CommandTest):
).group(1),
)
self.assertEqual(
- f"http://{domain}/",
+ f"https://{domain}",
self.find_in_file(
Path(self.proj_mod_path, "spiders", "test_name.py"),
r"start_urls\s*=\s*\[['\"](.+)['\"]\]",
@@ -549,13 +549,64 @@ class GenspiderCommandTest(CommandTest):
)
def test_url_schema(self):
- self.test_url("http://test.com", "test.com")
+ self.test_url("https://test.com", "test.com")
- def test_url_path(self):
- self.test_url("test.com/some/other/page", "test.com")
+ def test_template_start_urls(
+ self, url="test.com", expected="https://test.com", template="basic"
+ ):
+ self.assertEqual(
+ 0, self.call("genspider", "-t", template, "--force", "test_name", url)
+ )
+ self.assertEqual(
+ expected,
+ self.find_in_file(
+ Path(self.proj_mod_path, "spiders", "test_name.py"),
+ r"start_urls\s*=\s*\[['\"](.+)['\"]\]",
+ ).group(1),
+ )
+
+ def test_genspider_basic_start_urls(self):
+ self.test_template_start_urls("https://test.com", "https://test.com", "basic")
+ self.test_template_start_urls("http://test.com", "http://test.com", "basic")
+ self.test_template_start_urls(
+ "http://test.com/other/path", "http://test.com/other/path", "basic"
+ )
+ self.test_template_start_urls(
+ "test.com/other/path", "https://test.com/other/path", "basic"
+ )
- def test_url_schema_path(self):
- self.test_url("https://test.com/some/other/page", "test.com")
+ def test_genspider_crawl_start_urls(self):
+ self.test_template_start_urls("https://test.com", "https://test.com", "crawl")
+ self.test_template_start_urls("http://test.com", "http://test.com", "crawl")
+ self.test_template_start_urls(
+ "http://test.com/other/path", "http://test.com/other/path", "crawl"
+ )
+ self.test_template_start_urls(
+ "test.com/other/path", "https://test.com/other/path", "crawl"
+ )
+ self.test_template_start_urls("test.com", "https://test.com", "crawl")
+
+ def test_genspider_xmlfeed_start_urls(self):
+ self.test_template_start_urls(
+ "https://test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"
+ )
+ self.test_template_start_urls(
+ "http://test.com/feed.xml", "http://test.com/feed.xml", "xmlfeed"
+ )
+ self.test_template_start_urls(
+ "test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"
+ )
+
+ def test_genspider_csvfeed_start_urls(self):
+ self.test_template_start_urls(
+ "https://test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"
+ )
+ self.test_template_start_urls(
+ "http://test.com/feed.xml", "http://test.com/feed.xml", "csvfeed"
+ )
+ self.test_template_start_urls(
+ "test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"
+ )
class GenspiderStandaloneCommandTest(ProjectTest):