Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix a race condition in the FilesPipeline
Konstantin Lopuhinfc8cd45a48fb
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex c5dbd307b..b1b8404bb 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -364,8 +364,9 @@ class FilesPipeline(MediaPipeline):def file_downloaded(self, response, request, info):path = self.file_path(request, response=response, info=info)buf = BytesIO(response.body)- self.store.persist_file(path, buf, info)checksum = md5sum(buf)+ buf.seek(0)+ self.store.persist_file(path, buf, info)return checksumdef item_completed(self, results, item, info):