Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Make DiskDupeFilter commit within JOBDIR_SYNC_EVERY (#8322)

Hygor luzbdc269dc9e17
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 773f58367..3f5c189af 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1698,9 +1698,10 @@ so that a crawl killed before it can stop cleanly resumes from its latest
state. Higher values trade some of that safety for fewer writes, which matters
in broad crawls, where the state grows with the number of active domains.
-Applies to the ``active.json`` file of the :ref:`scheduler <topics-scheduler>`.
-For a killed crawl to resume, the scheduler queues must survive the kill too,
-which requires the SQLite types of :setting:`SCHEDULER_DISK_QUEUE` and
+Applies to the ``active.json`` file of the :ref:`scheduler <topics-scheduler>`
+and to the database of :class:`~scrapy.dupefilters.DiskDupeFilter`. For a
+killed crawl to resume, the scheduler queues must survive the kill too, which
+requires the SQLite types of :setting:`SCHEDULER_DISK_QUEUE` and
:setting:`SCHEDULER_START_DISK_QUEUE`.
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index 53e557846..847ea77c5 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -216,11 +216,26 @@ class DiskDupeFilter(RFPDupeFilter):
"CREATE TABLE IF NOT EXISTS seen (fingerprint BLOB PRIMARY KEY)"
" WITHOUT ROWID"
)
+ self._sync_every = 0
+ self._changes = 0
+
+ @classmethod
+ def from_crawler(cls, crawler: Crawler) -> Self:
+ dupefilter = super().from_crawler(crawler)
+ dupefilter._sync_every = crawler.settings.getint("JOBDIR_SYNC_EVERY")
+ return dupefilter
def request_seen(self, request: Request) -> bool:
fp = self._fingerprint(request)
cursor = self._db.execute("INSERT OR IGNORE INTO seen VALUES (?)", (fp,))
- return not cursor.rowcount
+ if not cursor.rowcount:
+ return True
+ if self._sync_every:
+ self._changes += 1
+ if self._changes >= self._sync_every:
+ self._changes = 0
+ self._db.commit()
+ return False
def close(self, reason: str) -> None:
self._db.commit()
diff --git a/tests/test_crash.py b/tests/test_crash.py
index d16af7400..299bb9f60 100644
--- a/tests/test_crash.py
+++ b/tests/test_crash.py
@@ -83,3 +83,15 @@ def test_resume_after_kill(tmp_path: Path) -> None:
assert log.count("Parsed data:,") == pending
assert "Spider closed (finished)" in log
assert _active_json(tmp_path) == {}
+
+
+def test_seen_requests_survive_kill(tmp_path: Path) -> None:
+ dupefilter = "DUPEFILTER_CLASS=scrapy.dupefilters.DiskDupeFilter"
+ log = run(tmp_path, "-a", "kill_at=10", "-s", "DOWNLOAD_DELAY=5", "-s", dupefilter)
+ # Without resume, the spider sends all its start requests again, and only
+ # those that did not reach the scheduler before the kill must get through.
+ log += run(tmp_path, "-s", dupefilter)
+ parsed = re.findall(r"Parsed (data:,\d+)", log)
+ assert len(parsed) == len(set(parsed))
+ # The request being downloaded when the spider dies is lost.
+ assert len(parsed) >= 19
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index 38ad4b689..da7dfdd88 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -3,6 +3,7 @@ from __future__ import annotations
import hashlib
import logging
import shutil
+import sqlite3
import tempfile
from pathlib import Path
from typing import TYPE_CHECKING, Any, cast
@@ -269,6 +270,30 @@ class TestDiskDupeFilter(DupeFilterTestMixin):
dupefilter.close("finished")
assert not tempdir.exists()
+ @pytest.mark.parametrize(
+ ("sync_every", "committed"),
+ [
+ (0, [0, 0, 0, 0]),
+ (1, [1, 1, 2, 3]),
+ (2, [0, 0, 2, 2]),
+ ],
+ )
+ def test_sync_every(
+ self, tmp_path: Path, sync_every: int, committed: list[int]
+ ) -> None:
+ dupefilter = self._get_dupefilter(
+ JOBDIR=str(tmp_path), JOBDIR_SYNC_EVERY=sync_every
+ )
+ # A second connection only sees committed rows.
+ reader = sqlite3.connect(tmp_path / "requests.seen.db")
+ urls = [f"https://example.com/{i}" for i in (1, 1, 2, 3)]
+ for url, expected in zip(urls, committed, strict=True):
+ dupefilter.request_seen(Request(url))
+ (count,) = reader.execute("SELECT COUNT(*) FROM seen").fetchone()
+ assert count == expected
+ reader.close()
+ dupefilter.close("finished")
+
class TestBaseDupeFilter:
def test_log_deprecation(self):