Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add integration tests for MEDIA_ALLOW_REDIRECTS

Paul Tremberth708f1b009b9b
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
new file mode 100644
index 000000000..1f5b80954
--- /dev/null
+++ b/tests/test_pipeline_crawl.py
@@ -0,0 +1,163 @@
+# -*- coding: utf-8 -*-
+import os
+import shutil
+
+from testfixtures import LogCapture
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase
+from w3lib.url import add_or_replace_parameter
+
+from scrapy.crawler import CrawlerRunner
+from scrapy import signals
+from tests.mockserver import MockServer
+from tests.spiders import SimpleSpider
+
+
+class MediaDownloadSpider(SimpleSpider):
+ name = 'mediadownload'
+
+ def _process_url(self, url):
+ return url
+
+ def parse(self, response):
+ self.logger.info(response.headers)
+ self.logger.info(response.text)
+ item = {
+ 'images': [],
+ 'image_urls': [
+ self._process_url(response.urljoin(href))
+ for href in response.xpath('''
+ //table[thead/tr/th="Filename"]
+ /tbody//a/@href
+ ''').extract()],
+ }
+ yield item
+
+
+class BrokenLinksMediaDownloadSpider(MediaDownloadSpider):
+ name = 'brokenmedia'
+
+ def _process_url(self, url):
+ return url + '.foo'
+
+
+class RedirectedMediaDownloadSpider(MediaDownloadSpider):
+ name = 'redirectedmedia'
+
+ def _process_url(self, url):
+ return add_or_replace_parameter(
+ 'http://localhost:8998/redirect-to',
+ 'goto', url)
+
+
+class MediaDownloadCrawlTestCase(TestCase):
+
+ def setUp(self):
+ self.mockserver = MockServer()
+ self.mockserver.__enter__()
+
+ # prepare a directory for storing files
+ self.tmpmediastore = self.mktemp()
+ os.mkdir(self.tmpmediastore)
+ self.settings = {
+ 'ITEM_PIPELINES': {'scrapy.pipelines.images.ImagesPipeline': 1},
+ 'IMAGES_STORE': self.tmpmediastore,
+ }
+ self.runner = CrawlerRunner(self.settings)
+ self.items = []
+ # these are the checksums for images in test_site/files/images
+ # - scrapy.png
+ # - python-powered-h-50x65.png
+ # - python-logo-master-v3-TM-flattened.png
+ self.expected_checksums = set([
+ 'a7020c30837f971084834e603625af58',
+ 'acac52d42b63cf2c3b05832641f3a53c',
+ '195672ac5888feb400fbf7b352553afe'])
+
+ def tearDown(self):
+ shutil.rmtree(self.tmpmediastore)
+ self.items = []
+ self.mockserver.__exit__(None, None, None)
+
+ def _on_item_scraped(self, item):
+ self.items.append(item)
+
+ def _create_crawler(self, spider_class):
+ crawler = self.runner.create_crawler(spider_class)
+ crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
+ return crawler
+
+ def _assert_files_downloaded(self, items, logs):
+ self.assertEqual(len(items), 1)
+ self.assertIn('images', items[0])
+
+ # check that logs show the expected number of successful file downloads
+ file_dl_success = 'File (downloaded): Downloaded file from'
+ self.assertEqual(logs.count(file_dl_success), 3)
+
+ # check that the images checksums are what we know they should be
+ checksums = set(
+ i['checksum']
+ for item in items
+ for i in item['images'])
+ self.assertEqual(checksums, self.expected_checksums)
+
+ # check that the image files where actually written to the media store
+ for item in items:
+ for i in item['images']:
+ self.assertTrue(
+ os.path.exists(
+ os.path.join(self.tmpmediastore, i['path'])))
+
+ def _assert_files_download_failure(self, crawler, items, code, logs):
+
+ # check that the item does NOT have the "images" field populated
+ self.assertEqual(len(items), 1)
+ self.assertIn('images', items[0])
+ self.assertFalse(items[0]['images'])
+
+ # check that there was 1 successful fetch and 3 other responses with non-200 code
+ self.assertEqual(crawler.stats.get_value('downloader/request_method_count/GET'), 4)
+ self.assertEqual(crawler.stats.get_value('downloader/response_count'), 4)
+ self.assertEqual(crawler.stats.get_value('downloader/response_status_count/200'), 1)
+ self.assertEqual(crawler.stats.get_value('downloader/response_status_count/%d' % code), 3)
+
+ # check that logs do show the failure on the file downloads
+ file_dl_failure = 'File (code: %d): Error downloading file from' % code
+ self.assertEqual(logs.count(file_dl_failure), 3)
+
+ # check that no files were written to the media store
+ self.assertEqual(os.listdir(self.tmpmediastore), [])
+
+ @defer.inlineCallbacks
+ def test_download_media(self):
+ crawler = self._create_crawler(MediaDownloadSpider)
+ with LogCapture() as log:
+ yield crawler.crawl("http://localhost:8998/files/images/")
+ self._assert_files_downloaded(self.items, str(log))
+
+ @defer.inlineCallbacks
+ def test_download_media_wrong_urls(self):
+ crawler = self._create_crawler(BrokenLinksMediaDownloadSpider)
+ with LogCapture() as log:
+ yield crawler.crawl("http://localhost:8998/files/images/")
+ self._assert_files_download_failure(crawler, self.items, 404, str(log))
+
+ @defer.inlineCallbacks
+ def test_download_media_redirected_default_failure(self):
+ crawler = self._create_crawler(RedirectedMediaDownloadSpider)
+ with LogCapture() as log:
+ yield crawler.crawl("http://localhost:8998/files/images/")
+ self._assert_files_download_failure(crawler, self.items, 302, str(log))
+
+ @defer.inlineCallbacks
+ def test_download_media_redirected_allowed(self):
+ settings = dict(self.settings)
+ settings.update({'MEDIA_ALLOW_REDIRECTS': True})
+ self.runner = CrawlerRunner(settings)
+
+ crawler = self._create_crawler(RedirectedMediaDownloadSpider)
+ with LogCapture() as log:
+ yield crawler.crawl("http://localhost:8998/files/images/")
+ self._assert_files_downloaded(self.items, str(log))
+ self.assertEqual(crawler.stats.get_value('downloader/response_status_count/302'), 3)