Scrapy, a fast high-level web crawling & scraping framework for Python.
Add integration tests for MEDIA_ALLOW_REDIRECTS
Paul Tremberth708f1b009b9b
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.pynew file mode 100644index 000000000..1f5b80954--- /dev/null+++ b/tests/test_pipeline_crawl.py@@ -0,0 +1,163 @@+# -*- coding: utf-8 -*-+import os+import shutil++from testfixtures import LogCapture+from twisted.internet import defer+from twisted.trial.unittest import TestCase+from w3lib.url import add_or_replace_parameter++from scrapy.crawler import CrawlerRunner+from scrapy import signals+from tests.mockserver import MockServer+from tests.spiders import SimpleSpider+++class MediaDownloadSpider(SimpleSpider):+ name = 'mediadownload'++ def _process_url(self, url):+ return url++ def parse(self, response):+ self.logger.info(response.headers)+ self.logger.info(response.text)+ item = {+ 'images': [],+ 'image_urls': [+ self._process_url(response.urljoin(href))+ for href in response.xpath('''+ //table[thead/tr/th="Filename"]+ /tbody//a/@href+ ''').extract()],+ }+ yield item+++class BrokenLinksMediaDownloadSpider(MediaDownloadSpider):+ name = 'brokenmedia'++ def _process_url(self, url):+ return url + '.foo'+++class RedirectedMediaDownloadSpider(MediaDownloadSpider):+ name = 'redirectedmedia'++ def _process_url(self, url):+ return add_or_replace_parameter(+ 'http://localhost:8998/redirect-to',+ 'goto', url)+++class MediaDownloadCrawlTestCase(TestCase):++ def setUp(self):+ self.mockserver = MockServer()+ self.mockserver.__enter__()++ # prepare a directory for storing files+ self.tmpmediastore = self.mktemp()+ os.mkdir(self.tmpmediastore)+ self.settings = {+ 'ITEM_PIPELINES': {'scrapy.pipelines.images.ImagesPipeline': 1},+ 'IMAGES_STORE': self.tmpmediastore,+ }+ self.runner = CrawlerRunner(self.settings)+ self.items = []+ # these are the checksums for images in test_site/files/images+ # - scrapy.png+ # - python-powered-h-50x65.png+ # - python-logo-master-v3-TM-flattened.png+ self.expected_checksums = set([+ 'a7020c30837f971084834e603625af58',+ 'acac52d42b63cf2c3b05832641f3a53c',+ '195672ac5888feb400fbf7b352553afe'])++ def tearDown(self):+ shutil.rmtree(self.tmpmediastore)+ self.items = []+ self.mockserver.__exit__(None, None, None)++ def _on_item_scraped(self, item):+ self.items.append(item)++ def _create_crawler(self, spider_class):+ crawler = self.runner.create_crawler(spider_class)+ crawler.signals.connect(self._on_item_scraped, signals.item_scraped)+ return crawler++ def _assert_files_downloaded(self, items, logs):+ self.assertEqual(len(items), 1)+ self.assertIn('images', items[0])++ # check that logs show the expected number of successful file downloads+ file_dl_success = 'File (downloaded): Downloaded file from'+ self.assertEqual(logs.count(file_dl_success), 3)++ # check that the images checksums are what we know they should be+ checksums = set(+ i['checksum']+ for item in items+ for i in item['images'])+ self.assertEqual(checksums, self.expected_checksums)++ # check that the image files where actually written to the media store+ for item in items:+ for i in item['images']:+ self.assertTrue(+ os.path.exists(+ os.path.join(self.tmpmediastore, i['path'])))++ def _assert_files_download_failure(self, crawler, items, code, logs):++ # check that the item does NOT have the "images" field populated+ self.assertEqual(len(items), 1)+ self.assertIn('images', items[0])+ self.assertFalse(items[0]['images'])++ # check that there was 1 successful fetch and 3 other responses with non-200 code+ self.assertEqual(crawler.stats.get_value('downloader/request_method_count/GET'), 4)+ self.assertEqual(crawler.stats.get_value('downloader/response_count'), 4)+ self.assertEqual(crawler.stats.get_value('downloader/response_status_count/200'), 1)+ self.assertEqual(crawler.stats.get_value('downloader/response_status_count/%d' % code), 3)++ # check that logs do show the failure on the file downloads+ file_dl_failure = 'File (code: %d): Error downloading file from' % code+ self.assertEqual(logs.count(file_dl_failure), 3)++ # check that no files were written to the media store+ self.assertEqual(os.listdir(self.tmpmediastore), [])++ @defer.inlineCallbacks+ def test_download_media(self):+ crawler = self._create_crawler(MediaDownloadSpider)+ with LogCapture() as log:+ yield crawler.crawl("http://localhost:8998/files/images/")+ self._assert_files_downloaded(self.items, str(log))++ @defer.inlineCallbacks+ def test_download_media_wrong_urls(self):+ crawler = self._create_crawler(BrokenLinksMediaDownloadSpider)+ with LogCapture() as log:+ yield crawler.crawl("http://localhost:8998/files/images/")+ self._assert_files_download_failure(crawler, self.items, 404, str(log))++ @defer.inlineCallbacks+ def test_download_media_redirected_default_failure(self):+ crawler = self._create_crawler(RedirectedMediaDownloadSpider)+ with LogCapture() as log:+ yield crawler.crawl("http://localhost:8998/files/images/")+ self._assert_files_download_failure(crawler, self.items, 302, str(log))++ @defer.inlineCallbacks+ def test_download_media_redirected_allowed(self):+ settings = dict(self.settings)+ settings.update({'MEDIA_ALLOW_REDIRECTS': True})+ self.runner = CrawlerRunner(settings)++ crawler = self._create_crawler(RedirectedMediaDownloadSpider)+ with LogCapture() as log:+ yield crawler.crawl("http://localhost:8998/files/images/")+ self._assert_files_downloaded(self.items, str(log))+ self.assertEqual(crawler.stats.get_value('downloader/response_status_count/302'), 3)