Scrapy, a fast high-level web crawling & scraping framework for Python.
Add status to files information
ilias-ant83a0cc6cdf4b
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rstindex cd84905c5..86550d7a4 100644--- a/docs/topics/media-pipeline.rst+++ b/docs/topics/media-pipeline.rst@@ -50,7 +50,7 @@ this:4. When the files are downloaded, another field (``files``) will be populatedwith the results. This field will contain a list of dicts with informationabout the downloaded files, such as the downloaded path, the original- scraped url (taken from the ``file_urls`` field) , and the file checksum.+ scraped url (taken from the ``file_urls`` field), the file checksum and the file status.The files in the list of the ``files`` field will retain the same order ofthe original ``file_urls`` field. If some file failed downloading, anerror will be logged and the file won't be present in the ``files`` field.@@ -470,6 +470,14 @@ See here the methods that you can override in your custom Files Pipeline:* ``checksum`` - a `MD5 hash`_ of the image contents+ * ``status`` - the file status indication. It can be one of the following:++ * ``downloaded`` - file was downloaded.+ * ``uptodate`` - file was not downloaded, as it was downloaded recently,+ according to the file expiration policy.+ * ``cached`` - file was already scheduled for download, by another item+ sharing the same file.+The list of tuples received by :meth:`~item_completed` isguaranteed to retain the same order of the requests returned from the:meth:`~get_media_requests` method.@@ -479,7 +487,8 @@ See here the methods that you can override in your custom Files Pipeline:[(True,{'checksum': '2b00042f7481c7b056c4b410d28f33cf','path': 'full/0a79c461a4062ac383dc4fade7bc09f1384a3910.jpg',- 'url': 'http://www.example.com/files/product1.pdf'}),+ 'url': 'http://www.example.com/files/product1.pdf',+ 'status': 'downloaded'}),(False,Failure(...))]diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 101bf5fbc..747dfa065 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -417,7 +417,7 @@ class FilesPipeline(MediaPipeline):self.inc_stats(info.spider, 'uptodate')checksum = result.get('checksum', None)- return {'url': request.url, 'path': path, 'checksum': checksum}+ return {'url': request.url, 'path': path, 'checksum': checksum, 'status': 'uptodate'}path = self.file_path(request, info=info)dfd = defer.maybeDeferred(self.store.stat_file, path, info)@@ -494,7 +494,7 @@ class FilesPipeline(MediaPipeline):)raise FileException(str(exc))- return {'url': request.url, 'path': path, 'checksum': checksum}+ return {'url': request.url, 'path': path, 'checksum': checksum, 'status': status}def inc_stats(self, spider, status):spider.crawler.stats.inc_value('file_count', spider=spider)diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.pyindex 962c33144..73b03e9d2 100644--- a/tests/test_pipeline_crawl.py+++ b/tests/test_pipeline_crawl.py@@ -94,6 +94,11 @@ class FileDownloadCrawlTestCase(TestCase):file_dl_success = 'File (downloaded): Downloaded file from'self.assertEqual(logs.count(file_dl_success), 3)+ # check that the images/files status is `downloaded`+ for item in items:+ for i in item[self.media_key]:+ self.assertEqual(i['status'], 'downloaded')+# check that the images/files checksums are what we know they should beif self.expected_checksums is not None:checksums = set(