Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add status to files information

ilias-ant83a0cc6cdf4b
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index cd84905c5..86550d7a4 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -50,7 +50,7 @@ this:
4. When the files are downloaded, another field (``files``) will be populated
with the results. This field will contain a list of dicts with information
about the downloaded files, such as the downloaded path, the original
- scraped url (taken from the ``file_urls`` field) , and the file checksum.
+ scraped url (taken from the ``file_urls`` field), the file checksum and the file status.
The files in the list of the ``files`` field will retain the same order of
the original ``file_urls`` field. If some file failed downloading, an
error will be logged and the file won't be present in the ``files`` field.
@@ -470,6 +470,14 @@ See here the methods that you can override in your custom Files Pipeline:
* ``checksum`` - a `MD5 hash`_ of the image contents
+ * ``status`` - the file status indication. It can be one of the following:
+
+ * ``downloaded`` - file was downloaded.
+ * ``uptodate`` - file was not downloaded, as it was downloaded recently,
+ according to the file expiration policy.
+ * ``cached`` - file was already scheduled for download, by another item
+ sharing the same file.
+
The list of tuples received by :meth:`~item_completed` is
guaranteed to retain the same order of the requests returned from the
:meth:`~get_media_requests` method.
@@ -479,7 +487,8 @@ See here the methods that you can override in your custom Files Pipeline:
[(True,
{'checksum': '2b00042f7481c7b056c4b410d28f33cf',
'path': 'full/0a79c461a4062ac383dc4fade7bc09f1384a3910.jpg',
- 'url': 'http://www.example.com/files/product1.pdf'}),
+ 'url': 'http://www.example.com/files/product1.pdf',
+ 'status': 'downloaded'}),
(False,
Failure(...))]
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 101bf5fbc..747dfa065 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -417,7 +417,7 @@ class FilesPipeline(MediaPipeline):
self.inc_stats(info.spider, 'uptodate')
checksum = result.get('checksum', None)
- return {'url': request.url, 'path': path, 'checksum': checksum}
+ return {'url': request.url, 'path': path, 'checksum': checksum, 'status': 'uptodate'}
path = self.file_path(request, info=info)
dfd = defer.maybeDeferred(self.store.stat_file, path, info)
@@ -494,7 +494,7 @@ class FilesPipeline(MediaPipeline):
)
raise FileException(str(exc))
- return {'url': request.url, 'path': path, 'checksum': checksum}
+ return {'url': request.url, 'path': path, 'checksum': checksum, 'status': status}
def inc_stats(self, spider, status):
spider.crawler.stats.inc_value('file_count', spider=spider)
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 962c33144..73b03e9d2 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -94,6 +94,11 @@ class FileDownloadCrawlTestCase(TestCase):
file_dl_success = 'File (downloaded): Downloaded file from'
self.assertEqual(logs.count(file_dl_success), 3)
+ # check that the images/files status is `downloaded`
+ for item in items:
+ for i in item[self.media_key]:
+ self.assertEqual(i['status'], 'downloaded')
+
# check that the images/files checksums are what we know they should be
if self.expected_checksums is not None:
checksums = set(