Scrapy, a fast high-level web crawling & scraping framework for Python.
Line buffering (#7094)
Vasiliy Kiryanov8c5fa6e6aea3
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.pyindex 2e264ade3..f0a6988c8 100644--- a/scrapy/dupefilters.py+++ b/scrapy/dupefilters.py@@ -73,7 +73,11 @@ class RFPDupeFilter(BaseDupeFilter):self.debug = debugself.logger = logging.getLogger(__name__)if path:- self.file = Path(path, "requests.seen").open("a+", encoding="utf-8")+ # line-by-line writing, see: https://github.com/scrapy/scrapy/issues/6019+ self.file = Path(path, "requests.seen").open(+ "a+", buffering=1, encoding="utf-8"+ )+ self.file.reconfigure(write_through=True)self.file.seek(0)self.fingerprints.update(x.rstrip() for x in self.file)