Scrapy, a fast high-level web crawling & scraping framework for Python.
genspider: removing www. from starturl templates
Raul Gallegosdd778892d09f
diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmplindex 2d3b96e38..abaa34ef5 100644--- a/scrapy/templates/spiders/basic.tmpl+++ b/scrapy/templates/spiders/basic.tmpl@@ -6,7 +6,7 @@ class $classname(scrapy.Spider):name = "$name"allowed_domains = ["$domain"]start_urls = (- 'http://www.$domain/',+ 'http://$domain/',)def parse(self, response):diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmplindex 154237d9c..802cb88a1 100644--- a/scrapy/templates/spiders/crawl.tmpl+++ b/scrapy/templates/spiders/crawl.tmpl@@ -7,7 +7,7 @@ from scrapy.spiders import CrawlSpider, Ruleclass $classname(CrawlSpider):name = '$name'allowed_domains = ['$domain']- start_urls = ['http://www.$domain/']+ start_urls = ['http://$domain/']rules = (Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True),diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmplindex 0544e0ae7..c2e4bacfe 100644--- a/scrapy/templates/spiders/csvfeed.tmpl+++ b/scrapy/templates/spiders/csvfeed.tmpl@@ -5,7 +5,7 @@ from scrapy.spiders import CSVFeedSpiderclass $classname(CSVFeedSpider):name = '$name'allowed_domains = ['$domain']- start_urls = ['http://www.$domain/feed.csv']+ start_urls = ['http://$domain/feed.csv']# headers = ['id', 'name', 'description', 'image_link']# delimiter = '\t'diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmplindex d8ff61f6e..7c2ff8850 100644--- a/scrapy/templates/spiders/xmlfeed.tmpl+++ b/scrapy/templates/spiders/xmlfeed.tmpl@@ -5,7 +5,7 @@ from scrapy.spiders import XMLFeedSpiderclass $classname(XMLFeedSpider):name = '$name'allowed_domains = ['$domain']- start_urls = ['http://www.$domain/feed.xml']+ start_urls = ['http://$domain/feed.xml']iterator = 'iternodes' # you can change this; see the docsitertag = 'item' # change it accordingly