Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Update RFPDupeFilter line separator for correct universal newlines mode usage (#4283)

Lane Shaw3263441fbcec
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index ea6a4cfc3..a36c8304f 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -49,7 +49,7 @@ class RFPDupeFilter(BaseDupeFilter):
return True
self.fingerprints.add(fp)
if self.file:
- self.file.write(fp + os.linesep)
+ self.file.write(fp + '\n')
def request_fingerprint(self, request):
return request_fingerprint(request)
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index 0546558bc..88ce9627f 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -2,6 +2,8 @@ import hashlib
import tempfile
import unittest
import shutil
+import os
+import sys
from testfixtures import LogCapture
from scrapy.dupefilters import RFPDupeFilter
@@ -84,17 +86,21 @@ class RFPDupeFilterTest(unittest.TestCase):
path = tempfile.mkdtemp()
try:
df = RFPDupeFilter(path)
- df.open()
- assert not df.request_seen(r1)
- assert df.request_seen(r1)
- df.close('finished')
+ try:
+ df.open()
+ assert not df.request_seen(r1)
+ assert df.request_seen(r1)
+ finally:
+ df.close('finished')
df2 = RFPDupeFilter(path)
- df2.open()
- assert df2.request_seen(r1)
- assert not df2.request_seen(r2)
- assert df2.request_seen(r2)
- df2.close('finished')
+ try:
+ df2.open()
+ assert df2.request_seen(r1)
+ assert not df2.request_seen(r2)
+ assert df2.request_seen(r2)
+ finally:
+ df2.close('finished')
finally:
shutil.rmtree(path)
@@ -129,6 +135,30 @@ class RFPDupeFilterTest(unittest.TestCase):
case_insensitive_dupefilter.close('finished')
+ def test_seenreq_newlines(self):
+ """ Checks against adding duplicate \r to
+ line endings on Windows platforms. """
+
+ r1 = Request('http://scrapytest.org/1')
+
+ path = tempfile.mkdtemp()
+ try:
+ df = RFPDupeFilter(path)
+ df.open()
+ df.request_seen(r1)
+ df.close('finished')
+
+ with open(os.path.join(path, 'requests.seen'), 'rb') as seen_file:
+ line = next(seen_file).decode()
+ assert not line.endswith('\r\r\n')
+ if sys.platform == 'win32':
+ assert line.endswith('\r\n')
+ else:
+ assert line.endswith('\n')
+
+ finally:
+ shutil.rmtree(path)
+
def test_log(self):
with LogCapture() as l:
settings = {'DUPEFILTER_DEBUG': False,