Scrapy, a fast high-level web crawling & scraping framework for Python.
Make DiskDupeFilter commit within JOBDIR_SYNC_EVERY (#8322)
Hygor luzbdc269dc9e17
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 773f58367..3f5c189af 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -1698,9 +1698,10 @@ so that a crawl killed before it can stop cleanly resumes from its lateststate. Higher values trade some of that safety for fewer writes, which mattersin broad crawls, where the state grows with the number of active domains.-Applies to the ``active.json`` file of the :ref:`scheduler <topics-scheduler>`.-For a killed crawl to resume, the scheduler queues must survive the kill too,-which requires the SQLite types of :setting:`SCHEDULER_DISK_QUEUE` and+Applies to the ``active.json`` file of the :ref:`scheduler <topics-scheduler>`+and to the database of :class:`~scrapy.dupefilters.DiskDupeFilter`. For a+killed crawl to resume, the scheduler queues must survive the kill too, which+requires the SQLite types of :setting:`SCHEDULER_DISK_QUEUE` and:setting:`SCHEDULER_START_DISK_QUEUE`.diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.pyindex 53e557846..847ea77c5 100644--- a/scrapy/dupefilters.py+++ b/scrapy/dupefilters.py@@ -216,11 +216,26 @@ class DiskDupeFilter(RFPDupeFilter):"CREATE TABLE IF NOT EXISTS seen (fingerprint BLOB PRIMARY KEY)"" WITHOUT ROWID")+ self._sync_every = 0+ self._changes = 0++ @classmethod+ def from_crawler(cls, crawler: Crawler) -> Self:+ dupefilter = super().from_crawler(crawler)+ dupefilter._sync_every = crawler.settings.getint("JOBDIR_SYNC_EVERY")+ return dupefilterdef request_seen(self, request: Request) -> bool:fp = self._fingerprint(request)cursor = self._db.execute("INSERT OR IGNORE INTO seen VALUES (?)", (fp,))- return not cursor.rowcount+ if not cursor.rowcount:+ return True+ if self._sync_every:+ self._changes += 1+ if self._changes >= self._sync_every:+ self._changes = 0+ self._db.commit()+ return Falsedef close(self, reason: str) -> None:self._db.commit()diff --git a/tests/test_crash.py b/tests/test_crash.pyindex d16af7400..299bb9f60 100644--- a/tests/test_crash.py+++ b/tests/test_crash.py@@ -83,3 +83,15 @@ def test_resume_after_kill(tmp_path: Path) -> None:assert log.count("Parsed data:,") == pendingassert "Spider closed (finished)" in logassert _active_json(tmp_path) == {}+++def test_seen_requests_survive_kill(tmp_path: Path) -> None:+ dupefilter = "DUPEFILTER_CLASS=scrapy.dupefilters.DiskDupeFilter"+ log = run(tmp_path, "-a", "kill_at=10", "-s", "DOWNLOAD_DELAY=5", "-s", dupefilter)+ # Without resume, the spider sends all its start requests again, and only+ # those that did not reach the scheduler before the kill must get through.+ log += run(tmp_path, "-s", dupefilter)+ parsed = re.findall(r"Parsed (data:,\d+)", log)+ assert len(parsed) == len(set(parsed))+ # The request being downloaded when the spider dies is lost.+ assert len(parsed) >= 19diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.pyindex 38ad4b689..da7dfdd88 100644--- a/tests/test_dupefilters.py+++ b/tests/test_dupefilters.py@@ -3,6 +3,7 @@ from __future__ import annotationsimport hashlibimport loggingimport shutil+import sqlite3import tempfilefrom pathlib import Pathfrom typing import TYPE_CHECKING, Any, cast@@ -269,6 +270,30 @@ class TestDiskDupeFilter(DupeFilterTestMixin):dupefilter.close("finished")assert not tempdir.exists()+ @pytest.mark.parametrize(+ ("sync_every", "committed"),+ [+ (0, [0, 0, 0, 0]),+ (1, [1, 1, 2, 3]),+ (2, [0, 0, 2, 2]),+ ],+ )+ def test_sync_every(+ self, tmp_path: Path, sync_every: int, committed: list[int]+ ) -> None:+ dupefilter = self._get_dupefilter(+ JOBDIR=str(tmp_path), JOBDIR_SYNC_EVERY=sync_every+ )+ # A second connection only sees committed rows.+ reader = sqlite3.connect(tmp_path / "requests.seen.db")+ urls = [f"https://example.com/{i}" for i in (1, 1, 2, 3)]+ for url, expected in zip(urls, committed, strict=True):+ dupefilter.request_seen(Request(url))+ (count,) = reader.execute("SELECT COUNT(*) FROM seen").fetchone()+ assert count == expected+ reader.close()+ dupefilter.close("finished")+class TestBaseDupeFilter:def test_log_deprecation(self):