Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add DiskDupeFilter, a disk-backed duplicate request filter (#7952)

Adrian7b8eec6e654e
diff --git a/docs/topics/optimize.rst b/docs/topics/optimize.rst
index e82d2f258..f3741b48c 100644
--- a/docs/topics/optimize.rst
+++ b/docs/topics/optimize.rst
@@ -295,6 +295,10 @@ Lowering memory usage
- Set :setting:`JOBDIR` to offload all scheduled requests to disk.
+- Set :setting:`DUPEFILTER_CLASS` to
+ :class:`~scrapy.dupefilters.DiskDupeFilter` to keep the fingerprint of
+ every sent request on disk instead of in memory.
+
- Set :setting:`MEDIA_CACHE_SIZE` if you use :ref:`media pipelines
<topics-media-pipeline>`.
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 2aba03db5..637ef9621 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1323,6 +1323,10 @@ The class used to detect and filter duplicate requests.
The default, :class:`~scrapy.dupefilters.RFPDupeFilter`, filters based on the
:setting:`REQUEST_FINGERPRINTER_CLASS` setting.
+On crawls large enough for the memory that
+:class:`~scrapy.dupefilters.RFPDupeFilter` uses to become a problem, set this
+setting to :class:`~scrapy.dupefilters.DiskDupeFilter` instead.
+
To change how duplicates are checked, you can point :setting:`DUPEFILTER_CLASS`
to a custom subclass of :class:`~scrapy.dupefilters.RFPDupeFilter` that
overrides its ``__init__`` method to use a :ref:`different request
@@ -1395,6 +1399,8 @@ interface:
.. autoclass:: scrapy.dupefilters.RFPDupeFilter
+.. autoclass:: scrapy.dupefilters.DiskDupeFilter
+
.. setting:: DUPEFILTER_DEBUG
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index 65c69b8a9..098394b9e 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -1,7 +1,10 @@
from __future__ import annotations
import logging
+import sqlite3
from pathlib import Path
+from shutil import rmtree
+from tempfile import mkdtemp
from typing import TYPE_CHECKING
from warnings import warn
@@ -174,3 +177,56 @@ class RFPDupeFilter(BaseDupeFilter):
self.logdupes = False
spider.crawler.stats.inc_value("dupefilter/filtered")
+
+
+class DiskDupeFilter(RFPDupeFilter):
+ """Duplicate request filtering class (:setting:`DUPEFILTER_CLASS`) that
+ keeps seen request fingerprints on disk, in an SQLite database, instead of
+ in memory, trading some speed for a much lower memory usage on crawls that
+ send a large number of requests.
+
+ .. versionadded:: VERSION
+
+ Job directory contents
+ ======================
+
+ .. warning:: The files that this class generates in the :ref:`job directory
+ <job-dir>` are an implementation detail, and may change without a
+ warning in a future version of Scrapy. Do not rely on the following
+ information for anything other than debugging purposes.
+
+ When using :setting:`JOBDIR`, seen fingerprints are tracked in an SQLite
+ database named :file:`requests.seen.db` in the :ref:`job directory
+ <job-dir>`.
+ """
+
+ def __init__(
+ self,
+ path: str | None = None,
+ debug: bool = False,
+ *,
+ fingerprinter: RequestFingerprinterProtocol | None = None,
+ ) -> None:
+ super().__init__(debug=debug, fingerprinter=fingerprinter)
+ if path:
+ self._tempdir: str | None = None
+ else:
+ self._tempdir = path = mkdtemp()
+ self._db = sqlite3.connect(Path(path, "requests.seen.db"))
+ self._db.execute("PRAGMA journal_mode=WAL").close()
+ self._db.execute("PRAGMA synchronous=NORMAL")
+ self._db.execute(
+ "CREATE TABLE IF NOT EXISTS seen (fingerprint BLOB PRIMARY KEY)"
+ " WITHOUT ROWID"
+ )
+
+ def request_seen(self, request: Request) -> bool:
+ fp = self._fingerprint(request)
+ cursor = self._db.execute("INSERT OR IGNORE INTO seen VALUES (?)", (fp,))
+ return not cursor.rowcount
+
+ def close(self, reason: str) -> None:
+ self._db.commit()
+ self._db.close()
+ if self._tempdir:
+ rmtree(self._tempdir)
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index cd68bb8e0..98dfa62cc 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Any, cast
import pytest
from scrapy.core.scheduler import Scheduler
-from scrapy.dupefilters import BaseDupeFilter, RFPDupeFilter
+from scrapy.dupefilters import BaseDupeFilter, DiskDupeFilter, RFPDupeFilter
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request
from scrapy.utils.misc import build_from_crawler
@@ -51,31 +51,22 @@ class DirectDupeFilter:
method = "n/a"
-class TestRFPDupeFilter:
- def test_df_from_crawler_scheduler(self):
- settings = {
- "DUPEFILTER_DEBUG": True,
- "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter,
- }
- crawler = get_crawler(settings_dict=settings)
- scheduler = build_from_crawler(Scheduler, crawler)
- dupefilter = cast("FromCrawlerRFPDupeFilter", scheduler.df)
- assert dupefilter.debug
- assert dupefilter.method == "from_crawler"
+class DupeFilterTestMixin:
+ """Tests that every :setting:`DUPEFILTER_CLASS` based on request
+ fingerprints must pass, regardless of where it stores them."""
- def test_df_direct_scheduler(self):
- settings = {
- "DUPEFILTER_CLASS": DirectDupeFilter,
- }
- crawler = get_crawler(settings_dict=settings)
- scheduler = build_from_crawler(Scheduler, crawler)
- assert cast("DirectDupeFilter", scheduler.df).method == "n/a"
+ dupefilter_class: type[BaseDupeFilter]
- def test_filter(self):
- dupefilter = _get_dupefilter()
- r1 = Request("http://scrapytest.org/1")
- r2 = Request("http://scrapytest.org/2")
- r3 = Request("http://scrapytest.org/2")
+ def _get_dupefilter(self, **settings: Any) -> BaseDupeFilter:
+ return _get_dupefilter(
+ settings={"DUPEFILTER_CLASS": self.dupefilter_class, **settings}
+ )
+
+ def test_filter(self) -> None:
+ dupefilter = self._get_dupefilter()
+ r1 = Request("https://example.com/1")
+ r2 = Request("https://example.com/2")
+ r3 = Request("https://example.com/2")
assert not dupefilter.request_seen(r1)
assert dupefilter.request_seen(r1)
@@ -85,60 +76,62 @@ class TestRFPDupeFilter:
dupefilter.close("finished")
- def test_dupefilter_path(self):
- r1 = Request("http://scrapytest.org/1")
- r2 = Request("http://scrapytest.org/2")
-
- path = tempfile.mkdtemp()
- try:
- df = _get_dupefilter(settings={"JOBDIR": path}, open_=False)
- try:
- df.open()
- assert not df.request_seen(r1)
- assert df.request_seen(r1)
- finally:
- df.close("finished")
+ def test_jobdir(self, tmp_path: Path) -> None:
+ r1 = Request("https://example.com/1")
+ r2 = Request("https://example.com/2")
- df2 = _get_dupefilter(settings={"JOBDIR": path}, open_=False)
- assert df is not df2
- try:
- df2.open()
- assert df2.request_seen(r1)
- assert not df2.request_seen(r2)
- assert df2.request_seen(r2)
- finally:
- df2.close("finished")
- finally:
- shutil.rmtree(path)
+ dupefilter = self._get_dupefilter(JOBDIR=str(tmp_path))
+ assert not dupefilter.request_seen(r1)
+ dupefilter.close("finished")
- def test_request_fingerprint(self):
- """Test if customization of request_fingerprint method will change
- output of request_seen.
+ resumed = self._get_dupefilter(JOBDIR=str(tmp_path))
+ assert resumed.request_seen(r1)
+ assert not resumed.request_seen(r2)
+ resumed.close("finished")
- """
- dupefilter = _get_dupefilter()
- r1 = Request("http://scrapytest.org/index.html")
- r2 = Request("http://scrapytest.org/INDEX.html")
+ def test_fingerprinter(self) -> None:
+ r1 = Request("https://example.com/index.html")
+ r2 = Request("https://example.com/INDEX.html")
+ dupefilter = self._get_dupefilter()
assert not dupefilter.request_seen(r1)
assert not dupefilter.request_seen(r2)
-
dupefilter.close("finished")
- class RequestFingerprinter:
- def fingerprint(self, request):
- fp = hashlib.sha1()
- fp.update(to_bytes(request.url.lower()))
- return fp.digest()
-
- settings = {"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter}
- case_insensitive_dupefilter = _get_dupefilter(settings=settings)
+ class CaseInsensitiveRequestFingerprinter:
+ def fingerprint(self, request: Request) -> bytes:
+ return hashlib.sha1(to_bytes(request.url.lower())).digest()
+ case_insensitive_dupefilter = self._get_dupefilter(
+ REQUEST_FINGERPRINTER_CLASS=CaseInsensitiveRequestFingerprinter
+ )
assert not case_insensitive_dupefilter.request_seen(r1)
assert case_insensitive_dupefilter.request_seen(r2)
-
case_insensitive_dupefilter.close("finished")
+
+class TestRFPDupeFilter(DupeFilterTestMixin):
+ dupefilter_class = RFPDupeFilter
+
+ def test_df_from_crawler_scheduler(self):
+ settings = {
+ "DUPEFILTER_DEBUG": True,
+ "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter,
+ }
+ crawler = get_crawler(settings_dict=settings)
+ scheduler = build_from_crawler(Scheduler, crawler)
+ dupefilter = cast("FromCrawlerRFPDupeFilter", scheduler.df)
+ assert dupefilter.debug
+ assert dupefilter.method == "from_crawler"
+
+ def test_df_direct_scheduler(self):
+ settings = {
+ "DUPEFILTER_CLASS": DirectDupeFilter,
+ }
+ crawler = get_crawler(settings_dict=settings)
+ scheduler = build_from_crawler(Scheduler, crawler)
+ assert cast("DirectDupeFilter", scheduler.df).method == "n/a"
+
def test_seenreq_truncated(self):
r1 = Request("http://scrapytest.org/1")
r2 = Request("http://scrapytest.org/2")
@@ -264,6 +257,20 @@ class TestRFPDupeFilter:
dupefilter.close("finished")
+class TestDiskDupeFilter(DupeFilterTestMixin):
+ dupefilter_class = DiskDupeFilter
+
+ def test_temporary_database_removed(self) -> None:
+ dupefilter = self._get_dupefilter()
+ assert isinstance(dupefilter, DiskDupeFilter)
+ assert dupefilter._tempdir
+ tempdir = Path(dupefilter._tempdir)
+ assert tempdir.exists()
+
+ dupefilter.close("finished")
+ assert not tempdir.exists()
+
+
class TestBaseDupeFilter:
def test_log_deprecation(self):
dupefilter = _get_dupefilter(