Scrapy, a fast high-level web crawling & scraping framework for Python.
Add DiskDupeFilter, a disk-backed duplicate request filter (#7952)
Adrian7b8eec6e654e
diff --git a/docs/topics/optimize.rst b/docs/topics/optimize.rstindex e82d2f258..f3741b48c 100644--- a/docs/topics/optimize.rst+++ b/docs/topics/optimize.rst@@ -295,6 +295,10 @@ Lowering memory usage- Set :setting:`JOBDIR` to offload all scheduled requests to disk.+- Set :setting:`DUPEFILTER_CLASS` to+ :class:`~scrapy.dupefilters.DiskDupeFilter` to keep the fingerprint of+ every sent request on disk instead of in memory.+- Set :setting:`MEDIA_CACHE_SIZE` if you use :ref:`media pipelines<topics-media-pipeline>`.diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 2aba03db5..637ef9621 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -1323,6 +1323,10 @@ The class used to detect and filter duplicate requests.The default, :class:`~scrapy.dupefilters.RFPDupeFilter`, filters based on the:setting:`REQUEST_FINGERPRINTER_CLASS` setting.+On crawls large enough for the memory that+:class:`~scrapy.dupefilters.RFPDupeFilter` uses to become a problem, set this+setting to :class:`~scrapy.dupefilters.DiskDupeFilter` instead.+To change how duplicates are checked, you can point :setting:`DUPEFILTER_CLASS`to a custom subclass of :class:`~scrapy.dupefilters.RFPDupeFilter` thatoverrides its ``__init__`` method to use a :ref:`different request@@ -1395,6 +1399,8 @@ interface:.. autoclass:: scrapy.dupefilters.RFPDupeFilter+.. autoclass:: scrapy.dupefilters.DiskDupeFilter+.. setting:: DUPEFILTER_DEBUGdiff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.pyindex 65c69b8a9..098394b9e 100644--- a/scrapy/dupefilters.py+++ b/scrapy/dupefilters.py@@ -1,7 +1,10 @@from __future__ import annotationsimport logging+import sqlite3from pathlib import Path+from shutil import rmtree+from tempfile import mkdtempfrom typing import TYPE_CHECKINGfrom warnings import warn@@ -174,3 +177,56 @@ class RFPDupeFilter(BaseDupeFilter):self.logdupes = Falsespider.crawler.stats.inc_value("dupefilter/filtered")+++class DiskDupeFilter(RFPDupeFilter):+ """Duplicate request filtering class (:setting:`DUPEFILTER_CLASS`) that+ keeps seen request fingerprints on disk, in an SQLite database, instead of+ in memory, trading some speed for a much lower memory usage on crawls that+ send a large number of requests.++ .. versionadded:: VERSION++ Job directory contents+ ======================++ .. warning:: The files that this class generates in the :ref:`job directory+ <job-dir>` are an implementation detail, and may change without a+ warning in a future version of Scrapy. Do not rely on the following+ information for anything other than debugging purposes.++ When using :setting:`JOBDIR`, seen fingerprints are tracked in an SQLite+ database named :file:`requests.seen.db` in the :ref:`job directory+ <job-dir>`.+ """++ def __init__(+ self,+ path: str | None = None,+ debug: bool = False,+ *,+ fingerprinter: RequestFingerprinterProtocol | None = None,+ ) -> None:+ super().__init__(debug=debug, fingerprinter=fingerprinter)+ if path:+ self._tempdir: str | None = None+ else:+ self._tempdir = path = mkdtemp()+ self._db = sqlite3.connect(Path(path, "requests.seen.db"))+ self._db.execute("PRAGMA journal_mode=WAL").close()+ self._db.execute("PRAGMA synchronous=NORMAL")+ self._db.execute(+ "CREATE TABLE IF NOT EXISTS seen (fingerprint BLOB PRIMARY KEY)"+ " WITHOUT ROWID"+ )++ def request_seen(self, request: Request) -> bool:+ fp = self._fingerprint(request)+ cursor = self._db.execute("INSERT OR IGNORE INTO seen VALUES (?)", (fp,))+ return not cursor.rowcount++ def close(self, reason: str) -> None:+ self._db.commit()+ self._db.close()+ if self._tempdir:+ rmtree(self._tempdir)diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.pyindex cd68bb8e0..98dfa62cc 100644--- a/tests/test_dupefilters.py+++ b/tests/test_dupefilters.py@@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Any, castimport pytestfrom scrapy.core.scheduler import Scheduler-from scrapy.dupefilters import BaseDupeFilter, RFPDupeFilter+from scrapy.dupefilters import BaseDupeFilter, DiskDupeFilter, RFPDupeFilterfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Requestfrom scrapy.utils.misc import build_from_crawler@@ -51,31 +51,22 @@ class DirectDupeFilter:method = "n/a"-class TestRFPDupeFilter:- def test_df_from_crawler_scheduler(self):- settings = {- "DUPEFILTER_DEBUG": True,- "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter,- }- crawler = get_crawler(settings_dict=settings)- scheduler = build_from_crawler(Scheduler, crawler)- dupefilter = cast("FromCrawlerRFPDupeFilter", scheduler.df)- assert dupefilter.debug- assert dupefilter.method == "from_crawler"+class DupeFilterTestMixin:+ """Tests that every :setting:`DUPEFILTER_CLASS` based on request+ fingerprints must pass, regardless of where it stores them."""- def test_df_direct_scheduler(self):- settings = {- "DUPEFILTER_CLASS": DirectDupeFilter,- }- crawler = get_crawler(settings_dict=settings)- scheduler = build_from_crawler(Scheduler, crawler)- assert cast("DirectDupeFilter", scheduler.df).method == "n/a"+ dupefilter_class: type[BaseDupeFilter]- def test_filter(self):- dupefilter = _get_dupefilter()- r1 = Request("http://scrapytest.org/1")- r2 = Request("http://scrapytest.org/2")- r3 = Request("http://scrapytest.org/2")+ def _get_dupefilter(self, **settings: Any) -> BaseDupeFilter:+ return _get_dupefilter(+ settings={"DUPEFILTER_CLASS": self.dupefilter_class, **settings}+ )++ def test_filter(self) -> None:+ dupefilter = self._get_dupefilter()+ r1 = Request("https://example.com/1")+ r2 = Request("https://example.com/2")+ r3 = Request("https://example.com/2")assert not dupefilter.request_seen(r1)assert dupefilter.request_seen(r1)@@ -85,60 +76,62 @@ class TestRFPDupeFilter:dupefilter.close("finished")- def test_dupefilter_path(self):- r1 = Request("http://scrapytest.org/1")- r2 = Request("http://scrapytest.org/2")-- path = tempfile.mkdtemp()- try:- df = _get_dupefilter(settings={"JOBDIR": path}, open_=False)- try:- df.open()- assert not df.request_seen(r1)- assert df.request_seen(r1)- finally:- df.close("finished")+ def test_jobdir(self, tmp_path: Path) -> None:+ r1 = Request("https://example.com/1")+ r2 = Request("https://example.com/2")- df2 = _get_dupefilter(settings={"JOBDIR": path}, open_=False)- assert df is not df2- try:- df2.open()- assert df2.request_seen(r1)- assert not df2.request_seen(r2)- assert df2.request_seen(r2)- finally:- df2.close("finished")- finally:- shutil.rmtree(path)+ dupefilter = self._get_dupefilter(JOBDIR=str(tmp_path))+ assert not dupefilter.request_seen(r1)+ dupefilter.close("finished")- def test_request_fingerprint(self):- """Test if customization of request_fingerprint method will change- output of request_seen.+ resumed = self._get_dupefilter(JOBDIR=str(tmp_path))+ assert resumed.request_seen(r1)+ assert not resumed.request_seen(r2)+ resumed.close("finished")- """- dupefilter = _get_dupefilter()- r1 = Request("http://scrapytest.org/index.html")- r2 = Request("http://scrapytest.org/INDEX.html")+ def test_fingerprinter(self) -> None:+ r1 = Request("https://example.com/index.html")+ r2 = Request("https://example.com/INDEX.html")+ dupefilter = self._get_dupefilter()assert not dupefilter.request_seen(r1)assert not dupefilter.request_seen(r2)-dupefilter.close("finished")- class RequestFingerprinter:- def fingerprint(self, request):- fp = hashlib.sha1()- fp.update(to_bytes(request.url.lower()))- return fp.digest()-- settings = {"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter}- case_insensitive_dupefilter = _get_dupefilter(settings=settings)+ class CaseInsensitiveRequestFingerprinter:+ def fingerprint(self, request: Request) -> bytes:+ return hashlib.sha1(to_bytes(request.url.lower())).digest()+ case_insensitive_dupefilter = self._get_dupefilter(+ REQUEST_FINGERPRINTER_CLASS=CaseInsensitiveRequestFingerprinter+ )assert not case_insensitive_dupefilter.request_seen(r1)assert case_insensitive_dupefilter.request_seen(r2)-case_insensitive_dupefilter.close("finished")++class TestRFPDupeFilter(DupeFilterTestMixin):+ dupefilter_class = RFPDupeFilter++ def test_df_from_crawler_scheduler(self):+ settings = {+ "DUPEFILTER_DEBUG": True,+ "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter,+ }+ crawler = get_crawler(settings_dict=settings)+ scheduler = build_from_crawler(Scheduler, crawler)+ dupefilter = cast("FromCrawlerRFPDupeFilter", scheduler.df)+ assert dupefilter.debug+ assert dupefilter.method == "from_crawler"++ def test_df_direct_scheduler(self):+ settings = {+ "DUPEFILTER_CLASS": DirectDupeFilter,+ }+ crawler = get_crawler(settings_dict=settings)+ scheduler = build_from_crawler(Scheduler, crawler)+ assert cast("DirectDupeFilter", scheduler.df).method == "n/a"+def test_seenreq_truncated(self):r1 = Request("http://scrapytest.org/1")r2 = Request("http://scrapytest.org/2")@@ -264,6 +257,20 @@ class TestRFPDupeFilter:dupefilter.close("finished")+class TestDiskDupeFilter(DupeFilterTestMixin):+ dupefilter_class = DiskDupeFilter++ def test_temporary_database_removed(self) -> None:+ dupefilter = self._get_dupefilter()+ assert isinstance(dupefilter, DiskDupeFilter)+ assert dupefilter._tempdir+ tempdir = Path(dupefilter._tempdir)+ assert tempdir.exists()++ dupefilter.close("finished")+ assert not tempdir.exists()++class TestBaseDupeFilter:def test_log_deprecation(self):dupefilter = _get_dupefilter(