Scrapy, a fast high-level web crawling & scraping framework for Python.
Two fixes for when using the parse command and the '-r' flag (rules). 1. Use default "parse" as callback when the matching rule has no callback. 2. Log error and return when no rule matches the parsed url.
Joakim Uddholm743a0aa422ae
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.pyindex 6a8978415..5264982b6 100644--- a/scrapy/commands/parse.py+++ b/scrapy/commands/parse.py@@ -121,8 +121,8 @@ class Command(ScrapyCommand):def get_callback_from_rules(self, spider, response):if getattr(spider, 'rules', None):for rule in spider.rules:- if rule.link_extractor.matches(response.url) and rule.callback:- return rule.callback+ if rule.link_extractor.matches(response.url):+ return rule.callback or "parse"else:logger.error('No CrawlSpider rules found in spider %(spider)r, ''please specify a callback to use for parsing',@@ -166,6 +166,11 @@ class Command(ScrapyCommand):if not cb:if opts.rules and self.first_response == response:cb = self.get_callback_from_rules(spider, response)++ if not cb:+ logger.error('Cannot find a rule that matches %(url)r in spider: %(spider)s',+ {'url': response.url, 'spider': spider.name})+ returnelse:cb = 'parse'