Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

genspider: removing www. from starturl templates

Raul Gallegosdd778892d09f
diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl
index 2d3b96e38..abaa34ef5 100644
--- a/scrapy/templates/spiders/basic.tmpl
+++ b/scrapy/templates/spiders/basic.tmpl
@@ -6,7 +6,7 @@ class $classname(scrapy.Spider):
name = "$name"
allowed_domains = ["$domain"]
start_urls = (
- 'http://www.$domain/',
+ 'http://$domain/',
)
def parse(self, response):
diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl
index 154237d9c..802cb88a1 100644
--- a/scrapy/templates/spiders/crawl.tmpl
+++ b/scrapy/templates/spiders/crawl.tmpl
@@ -7,7 +7,7 @@ from scrapy.spiders import CrawlSpider, Rule
class $classname(CrawlSpider):
name = '$name'
allowed_domains = ['$domain']
- start_urls = ['http://www.$domain/']
+ start_urls = ['http://$domain/']
rules = (
Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True),
diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl
index 0544e0ae7..c2e4bacfe 100644
--- a/scrapy/templates/spiders/csvfeed.tmpl
+++ b/scrapy/templates/spiders/csvfeed.tmpl
@@ -5,7 +5,7 @@ from scrapy.spiders import CSVFeedSpider
class $classname(CSVFeedSpider):
name = '$name'
allowed_domains = ['$domain']
- start_urls = ['http://www.$domain/feed.csv']
+ start_urls = ['http://$domain/feed.csv']
# headers = ['id', 'name', 'description', 'image_link']
# delimiter = '\t'
diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl
index d8ff61f6e..7c2ff8850 100644
--- a/scrapy/templates/spiders/xmlfeed.tmpl
+++ b/scrapy/templates/spiders/xmlfeed.tmpl
@@ -5,7 +5,7 @@ from scrapy.spiders import XMLFeedSpider
class $classname(XMLFeedSpider):
name = '$name'
allowed_domains = ['$domain']
- start_urls = ['http://www.$domain/feed.xml']
+ start_urls = ['http://$domain/feed.xml']
iterator = 'iternodes' # you can change this; see the docs
itertag = 'item' # change it accordingly