Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix a race condition in the FilesPipeline

Konstantin Lopuhinfc8cd45a48fb
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index c5dbd307b..b1b8404bb 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -364,8 +364,9 @@ class FilesPipeline(MediaPipeline):
def file_downloaded(self, response, request, info):
path = self.file_path(request, response=response, info=info)
buf = BytesIO(response.body)
- self.store.persist_file(path, buf, info)
checksum = md5sum(buf)
+ buf.seek(0)
+ self.store.persist_file(path, buf, info)
return checksum
def item_completed(self, results, item, info):