Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Line buffering (#7094)

Vasiliy Kiryanov8c5fa6e6aea3
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index 2e264ade3..f0a6988c8 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -73,7 +73,11 @@ class RFPDupeFilter(BaseDupeFilter):
self.debug = debug
self.logger = logging.getLogger(__name__)
if path:
- self.file = Path(path, "requests.seen").open("a+", encoding="utf-8")
+ # line-by-line writing, see: https://github.com/scrapy/scrapy/issues/6019
+ self.file = Path(path, "requests.seen").open(
+ "a+", buffering=1, encoding="utf-8"
+ )
+ self.file.reconfigure(write_through=True)
self.file.seek(0)
self.fingerprints.update(x.rstrip() for x in self.file)