Scrapy, a fast high-level web crawling & scraping framework for Python.
Update RFPDupeFilter line separator for correct universal newlines mode usage (#4283)
Lane Shaw3263441fbcec
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.pyindex ea6a4cfc3..a36c8304f 100644--- a/scrapy/dupefilters.py+++ b/scrapy/dupefilters.py@@ -49,7 +49,7 @@ class RFPDupeFilter(BaseDupeFilter):return Trueself.fingerprints.add(fp)if self.file:- self.file.write(fp + os.linesep)+ self.file.write(fp + '\n')def request_fingerprint(self, request):return request_fingerprint(request)diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.pyindex 0546558bc..88ce9627f 100644--- a/tests/test_dupefilters.py+++ b/tests/test_dupefilters.py@@ -2,6 +2,8 @@ import hashlibimport tempfileimport unittestimport shutil+import os+import sysfrom testfixtures import LogCapturefrom scrapy.dupefilters import RFPDupeFilter@@ -84,17 +86,21 @@ class RFPDupeFilterTest(unittest.TestCase):path = tempfile.mkdtemp()try:df = RFPDupeFilter(path)- df.open()- assert not df.request_seen(r1)- assert df.request_seen(r1)- df.close('finished')+ try:+ df.open()+ assert not df.request_seen(r1)+ assert df.request_seen(r1)+ finally:+ df.close('finished')df2 = RFPDupeFilter(path)- df2.open()- assert df2.request_seen(r1)- assert not df2.request_seen(r2)- assert df2.request_seen(r2)- df2.close('finished')+ try:+ df2.open()+ assert df2.request_seen(r1)+ assert not df2.request_seen(r2)+ assert df2.request_seen(r2)+ finally:+ df2.close('finished')finally:shutil.rmtree(path)@@ -129,6 +135,30 @@ class RFPDupeFilterTest(unittest.TestCase):case_insensitive_dupefilter.close('finished')+ def test_seenreq_newlines(self):+ """ Checks against adding duplicate \r to+ line endings on Windows platforms. """++ r1 = Request('http://scrapytest.org/1')++ path = tempfile.mkdtemp()+ try:+ df = RFPDupeFilter(path)+ df.open()+ df.request_seen(r1)+ df.close('finished')++ with open(os.path.join(path, 'requests.seen'), 'rb') as seen_file:+ line = next(seen_file).decode()+ assert not line.endswith('\r\r\n')+ if sys.platform == 'win32':+ assert line.endswith('\r\n')+ else:+ assert line.endswith('\n')++ finally:+ shutil.rmtree(path)+def test_log(self):with LogCapture() as l:settings = {'DUPEFILTER_DEBUG': False,