Scrapy, a fast high-level web crawling & scraping framework for Python.
Add tests for crawl command non-default cases
Paul Tremberthfbb555929977
diff --git a/tests/test_commands.py b/tests/test_commands.pyindex d25045cb9..d13024922 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -257,6 +257,9 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest):with open(fname, 'w') as f:f.write("""import scrapy+from scrapy.linkextractors import LinkExtractor+from scrapy.spiders import CrawlSpider, Rule+class MySpider(scrapy.Spider):name = '{0}'@@ -265,6 +268,33 @@ class MySpider(scrapy.Spider):if getattr(self, 'test_arg', None):self.logger.debug('It Works!')return [scrapy.Item(), dict(foo='bar')]+++class MyGoodCrawlSpider(CrawlSpider):+ name = 'goodcrawl{0}'++ rules = (+ Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True),+ Rule(LinkExtractor(allow=r'/text'), follow=True),+ )++ def parse_item(self, response):+ return [scrapy.Item(), dict(foo='bar')]++ def parse(self, response):+ return [scrapy.Item(), dict(nomatch='default')]+++class MyBadCrawlSpider(CrawlSpider):+ '''Spider which doesn't define a parse_item callback while using it in a rule.'''+ name = 'badcrawl{0}'++ rules = (+ Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True),+ )++ def parse(self, response):+ return [scrapy.Item(), dict(foo='bar')]""".format(self.spider_name))fname = abspath(join(self.proj_mod_path, 'pipelines.py'))@@ -309,6 +339,58 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1})self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))+ @defer.inlineCallbacks+ def test_parse_items_no_callback_passed(self):+ status, out, stderr = yield self.execute(+ ['--spider', self.spider_name, self.url('/html')]+ )+ self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))++ @defer.inlineCallbacks+ def test_wrong_callback_passed(self):+ status, out, stderr = yield self.execute(+ ['--spider', self.spider_name, '-c', 'dummy', self.url('/html')]+ )+ self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""")++ @defer.inlineCallbacks+ def test_crawlspider_matching_rule_callback_set(self):+ """If a rule matches the URL, use it's defined callback."""+ status, out, stderr = yield self.execute(+ ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/html')]+ )+ self.assertIn("""[{}, {'foo': 'bar'}]""", to_native_str(out))++ @defer.inlineCallbacks+ def test_crawlspider_matching_rule_default_callback(self):+ """If a rule match but it has no callback set, use the 'parse' callback."""+ status, out, stderr = yield self.execute(+ ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/text')]+ )+ self.assertIn("""[{}, {'nomatch': 'default'}]""", to_native_str(out))++ @defer.inlineCallbacks+ def test_spider_with_no_rules_attribute(self):+ """Using -r with a spider with no rule should not produce items."""+ status, out, stderr = yield self.execute(+ ['--spider', self.spider_name, '-r', self.url('/html')]+ )+ self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""")++ @defer.inlineCallbacks+ def test_crawlspider_missing_callback(self):+ status, out, stderr = yield self.execute(+ ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/html')]+ )+ self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""")++ @defer.inlineCallbacks+ def test_crawlspider_no_matching_rule(self):+ """The requested URL has no matching rule, so no items should be scraped"""+ status, out, stderr = yield self.execute(+ ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/enc-gb18030')]+ )+ self.assertRegexpMatches(to_native_str(out), """# Scraped Items -+\n\[\]""")class BenchCommandTest(CommandTest):