Scrapy, a fast high-level web crawling & scraping framework for Python.
Add JOBDIR_SYNC_EVERY to keep active.json current, written atomically (#8152)
Adriandb5bad1dddb3
diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rstindex ee82f5b36..0496a987a 100644--- a/docs/topics/jobs.rst+++ b/docs/topics/jobs.rst@@ -176,9 +176,8 @@ Where:- :class:`~scrapy.core.scheduler.Scheduler` creates the ``requests.queue/``directory and the ``active.json`` file, the latter containing the state- data returned by :meth:`DownloaderAwarePriorityQueue.close()- <scrapy.pqueues.DownloaderAwarePriorityQueue.close>` the last time the job- was paused.+ data of :class:`~scrapy.pqueues.DownloaderAwarePriorityQueue`, written+ when the job stops and, with :setting:`JOBDIR_SYNC_EVERY`, as it runs.- :class:`~scrapy.pqueues.DownloaderAwarePriorityQueue` creates the``{hostname}-{hash}`` directories.diff --git a/docs/topics/scheduler.rst b/docs/topics/scheduler.rstindex 91e13a0ba..0181f6f11 100644--- a/docs/topics/scheduler.rst+++ b/docs/topics/scheduler.rst@@ -38,4 +38,7 @@ Priority queues===============.. autoclass:: scrapy.pqueues.DownloaderAwarePriorityQueue+ :members: changed, state+.. autoclass:: scrapy.pqueues.ScrapyPriorityQueue+ :members: changed, statediff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 2f7da6224..c8f4da941 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -1651,6 +1651,28 @@ A string indicating the directory for storing the state of a crawl when:ref:`pausing and resuming crawls <topics-jobs>`.+.. setting:: JOBDIR_SYNC_EVERY++JOBDIR_SYNC_EVERY+-----------------++.. versionadded:: VERSION++Default: ``0``++Number of changes to the crawl state kept in :setting:`JOBDIR` after which+that state is written to disk while the crawl runs, in addition to when it+stops. ``0`` writes it only when the crawl stops. ``1`` writes every change,+so that a crawl killed before it can stop cleanly resumes from its latest+state. Higher values trade some of that safety for fewer writes, which matters+in broad crawls, where the state grows with the number of active domains.++Applies to the ``active.json`` file of the :ref:`scheduler <topics-scheduler>`.+For a killed crawl to resume, the scheduler queues must survive the kill too,+which requires the SQLite types of :setting:`SCHEDULER_DISK_QUEUE` and+:setting:`SCHEDULER_START_DISK_QUEUE`.++.. setting:: LOG_COLORLOG_COLORdiff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.pyindex 2194d9845..ff119215a 100644--- a/scrapy/core/scheduler.py+++ b/scrapy/core/scheduler.py@@ -237,7 +237,11 @@ class Scheduler(BaseScheduler)::setting:`SCHEDULER_PRIORITY_QUEUE`.The file is generated whenever the job stops (cleanly) and is loaded- when resuming the job.+ when resuming the job. If :setting:`JOBDIR_SYNC_EVERY` is set and the+ priority queue implements+ :attr:`~scrapy.pqueues.ScrapyPriorityQueue.changed` and+ :meth:`~scrapy.pqueues.ScrapyPriorityQueue.state`, the file is also+ generated while the job runs.- Instantiates the configured :setting:`SCHEDULER_PRIORITY_QUEUE` with``requests.queue/`` as persistence directory (*key*) and@@ -319,6 +323,9 @@ class Scheduler(BaseScheduler):self.logunser: bool = logunserself.stats: StatsCollector | None = statsself.crawler: Crawler | None = crawler+ self._sync_every: int = (+ crawler.settings.getint("JOBDIR_SYNC_EVERY") if crawler else 0+ )self._sdqclass: type[BaseQueue] | None = self._get_start_queue_cls(crawler, "DISK")@@ -348,6 +355,8 @@ class Scheduler(BaseScheduler):self.spider: Spider = spiderself.mqs: ScrapyPriorityQueue = self._mq()self.dqs: ScrapyPriorityQueue | None = self._dq() if self.dqdir else None+ self._dqs_changes = 0+ self._sync_dqs_state()return self.df.open()def close(self, reason: str) -> Deferred[None] | None:@@ -433,15 +442,32 @@ class Scheduler(BaseScheduler):assert self.stats is not Noneself.stats.inc_value("scheduler/unserializable")return False+ finally:+ self._sync_dqs_state()return Truedef _mqpush(self, request: Request) -> None:self.mqs.push(request)def _dqpop(self) -> Request | None:- if self.dqs is not None:- return self.dqs.pop()- return None+ if self.dqs is None:+ return None+ request = self.dqs.pop()+ self._sync_dqs_state()+ return request++ def _sync_dqs_state(self) -> None:+ # Priority queues without change tracking only report their state+ # through close().+ if not self._sync_every or not getattr(self.dqs, "changed", False):+ return+ assert self.dqs is not None+ self.dqs.changed = False+ self._dqs_changes += 1+ if self._dqs_changes >= self._sync_every:+ self._dqs_changes = 0+ assert isinstance(self.dqdir, str)+ self._write_dqs_state(self.dqdir, self.dqs.state())def _mq(self) -> ScrapyPriorityQueue:"""Create a new priority queue instance, with in-memory storage"""@@ -494,5 +520,7 @@ class Scheduler(BaseScheduler):return json.load(f)def _write_dqs_state(self, dqdir: str, state: Any) -> None:- with Path(dqdir, "active.json").open("w", encoding="utf-8") as f:- json.dump(state, f)+ path = Path(dqdir, "active.json")+ tmp = path.with_name("active.json.tmp")+ tmp.write_text(json.dumps(state), encoding="utf-8")+ tmp.replace(path)diff --git a/scrapy/pqueues.py b/scrapy/pqueues.pyindex bbbd33093..bee338e2b 100644--- a/scrapy/pqueues.py+++ b/scrapy/pqueues.py@@ -134,6 +134,11 @@ class ScrapyPriorityQueue:self.queues: dict[int, QueueProtocol] = {}self._start_queues: dict[int, QueueProtocol] = {}self.curprio: int | None = None+ self.changed: bool = False+ """Whether the return value of :meth:`state` changed since this+ attribute was last set to ``False``.++ The scheduler resets it after persisting that state."""self.init_prios(startprios)def init_prios(self, startprios: Iterable[int]) -> None:@@ -146,12 +151,14 @@ class ScrapyPriorityQueue:self.queues[priority] = qelse:q.close()+ self.changed = Trueif self._start_queue_cls:q = self._sqfactory(priority)if q:self._start_queues[priority] = qelse:q.close()+ self.changed = Trueself.curprio = min(startprios)@@ -179,10 +186,12 @@ class ScrapyPriorityQueue:if is_start_request and self._start_queue_cls:if priority not in self._start_queues:self._start_queues[priority] = self._sqfactory(priority)+ self.changed = Trueq = self._start_queues[priority]else:if priority not in self.queues:self.queues[priority] = self.qfactory(priority)+ self.changed = Trueq = self.queues[priority]q.push(request) # this may fail (eg. serialization error)if self.curprio is None or priority < self.curprio:@@ -199,6 +208,7 @@ class ScrapyPriorityQueue:if not q:del self.queues[self.curprio]q.close()+ self.changed = Trueif not self._start_queues:self._update_curprio()return m@@ -212,6 +222,7 @@ class ScrapyPriorityQueue:if not q:del self._start_queues[self.curprio]q.close()+ self.changed = Trueself._update_curprio()return melse:@@ -243,13 +254,18 @@ class ScrapyPriorityQueue:# Protocols can't declare optional membersreturn cast("Request", queue.peek()) # type: ignore[attr-defined]+ def state(self) -> list[int]:+ """Return what to pass as *startprios* to resume this queue later.++ Same as the return value of :meth:`close`, without closing."""+ return sorted(self.queues.keys() | self._start_queues.keys())+def close(self) -> list[int]:- active: set[int] = set()+ active = self.state()for queues in (self.queues, self._start_queues):- for p, q in queues.items():- active.add(p)+ for q in queues.values():q.close()- return list(active)+ return activedef __len__(self) -> int:return (@@ -357,9 +373,12 @@ class DownloaderAwarePriorityQueue:self.pqueues: dict[str, ScrapyPriorityQueue] = {} # slot -> priority queueself._last_selected_slot: str | None = None+ self.changed: bool = False+ """See :attr:`ScrapyPriorityQueue.changed`."""if slot_startprios:for slot, startprios in slot_startprios.items():self.pqueues[slot] = self.pqfactory(slot, startprios)+ self._collect_change(self.pqueues[slot])def _next_slot(self, stats: list[tuple[int, str]], *, update_state: bool) -> str:last = self._last_selected_slot@@ -399,6 +418,11 @@ class DownloaderAwarePriorityQueue:start_queue_cls=self._start_queue_cls,)+ def _collect_change(self, queue: ScrapyPriorityQueue) -> None:+ if queue.changed:+ queue.changed = False+ self.changed = True+def pop(self) -> Request | None:stats = self._downloader_interface.stats(self.pqueues)@@ -408,9 +432,11 @@ class DownloaderAwarePriorityQueue:slot = self._next_slot(stats, update_state=True)queue = self.pqueues[slot]request = queue.pop()+ self._collect_change(queue)if len(queue) == 0:del self.pqueues[slot]queue.close()+ self.changed = Trueif self.key:# Reclaim the slot directory; rmdir leaves it alone if the# downstream queues did not remove all their files.@@ -422,8 +448,10 @@ class DownloaderAwarePriorityQueue:slot = self._downloader_interface.get_slot_key(request)if slot not in self.pqueues:self.pqueues[slot] = self.pqfactory(slot)+ self.changed = Truequeue = self.pqueues[slot]queue.push(request)+ self._collect_change(queue)def peek(self) -> Request | None:"""Returns the next object to be returned by :meth:`pop`,@@ -439,8 +467,14 @@ class DownloaderAwarePriorityQueue:queue = self.pqueues[slot]return queue.peek()+ def state(self) -> dict[str, list[int]]:+ """See :meth:`ScrapyPriorityQueue.state`."""+ return {slot: queue.state() for slot, queue in self.pqueues.items()}+def close(self) -> dict[str, list[int]]:- active = {slot: queue.close() for slot, queue in self.pqueues.items()}+ active = self.state()+ for queue in self.pqueues.values():+ queue.close()self.pqueues.clear()return activediff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 277c49e80..a03ee957e 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -138,6 +138,7 @@ __all__ = ["ITEM_PIPELINES_BASE","ITEM_PROCESSOR","JOBDIR",+ "JOBDIR_SYNC_EVERY","JSONVALIDATION_ENABLED","LOGSTATS_INTERVAL","LOG_COLOR",@@ -456,6 +457,7 @@ ITEM_PIPELINES_BASE: dict[str, int] = {}ITEM_PROCESSOR = "scrapy.pipelines.ItemPipelineManager"JOBDIR = None+JOBDIR_SYNC_EVERY = 0JSONVALIDATION_ENABLED = Falsediff --git a/tests/test_crash.py b/tests/test_crash.pynew file mode 100644index 000000000..d16af7400--- /dev/null+++ b/tests/test_crash.py@@ -0,0 +1,85 @@+from __future__ import annotations++import json+import re+from typing import TYPE_CHECKING, Any++from scrapy.pqueues import _path_safe+from tests.utils.cmdline import proc++if TYPE_CHECKING:+ from pathlib import Path+++# Dies without any cleanup once kill_at requests have reached the scheduler.+SPIDER = """+import os++from scrapy import Request, Spider, signals+++class KillSpider(Spider):+ name = "kill"+ custom_settings = {+ "SCHEDULER_DISK_QUEUE": "scrapy.squeues.PickleFifoSQLiteQueue",+ "SCHEDULER_START_DISK_QUEUE": "scrapy.squeues.PickleFifoSQLiteQueue",+ "CONCURRENT_REQUESTS": 1,+ "JOBDIR_SYNC_EVERY": 1,+ }++ @classmethod+ def from_crawler(cls, crawler, *args, **kwargs):+ spider = super().from_crawler(crawler, *args, **kwargs)+ spider.scheduled = 0+ crawler.signals.connect(spider.request_scheduled, signals.request_scheduled)+ return spider++ def request_scheduled(self, request, spider):+ self.scheduled += 1+ if self.scheduled == int(getattr(self, "kill_at", 0)):+ os._exit(1)++ async def start(self):+ if hasattr(self, "resume"):+ return+ for i in range(20):+ yield Request(f"data:,{i}", priority=i % 3)++ def parse(self, response):+ self.logger.info(f"Parsed {response.url}")+"""+++def run(tmp_path: Path, *args: str) -> str:+ spider = tmp_path / "spider.py"+ spider.write_text(SPIDER, encoding="utf-8")+ jobdir = tmp_path / "job"+ _, _, log = proc(+ "runspider", str(spider), "-s", f"JOBDIR={jobdir}", *args, cwd=tmp_path+ )+ return log+++def _active_json(tmp_path: Path) -> Any:+ path = tmp_path / "job" / "requests.queue" / "active.json"+ return json.loads(path.read_text(encoding="utf-8"))+++def test_resume_after_kill(tmp_path: Path) -> None:+ # The delay keeps all but the first request or two in the queue when the+ # spider dies, while start requests keep being scheduled eagerly.+ run(tmp_path, "-a", "kill_at=10", "-s", "DOWNLOAD_DELAY=5")+ [(slot, priorities)] = _active_json(tmp_path).items()+ slot_dir = tmp_path / "job" / "requests.queue" / _path_safe(slot)+ assert set(priorities) == {+ int(path.name.rstrip("s")) for path in slot_dir.iterdir()+ }++ log = run(tmp_path, "-a", "resume=1")+ match = re.search(r"Resuming crawl \((\d+) requests scheduled\)", log)+ assert match, log+ pending = int(match.group(1))+ assert pending > 0+ assert log.count("Parsed data:,") == pending+ assert "Spider closed (finished)" in log+ assert _active_json(tmp_path) == {}diff --git a/tests/test_scheduler.py b/tests/test_scheduler.pyindex 43cdab183..768785ab2 100644--- a/tests/test_scheduler.py+++ b/tests/test_scheduler.py@@ -1,9 +1,10 @@from __future__ import annotations+import jsonimport warningsfrom abc import ABC, abstractmethodfrom contextlib import AbstractAsyncContextManager, asynccontextmanager-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, Anyfrom unittest.mock import Mockimport pytest@@ -28,7 +29,7 @@ if TYPE_CHECKING:class MockCrawler(Crawler):- def __init__(self, priority_queue_cls: str, jobdir: Path | None):+ def __init__(self, priority_queue_cls: str, jobdir: Path | None, **settings: Any):settings = {"SCHEDULER_DEBUG": False,"SCHEDULER_DISK_QUEUE": "scrapy.squeues.PickleLifoDiskQueue",@@ -36,6 +37,7 @@ class MockCrawler(Crawler):"SCHEDULER_PRIORITY_QUEUE": priority_queue_cls,"JOBDIR": str(jobdir) if jobdir is not None else None,"DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter",+ **settings,}super().__init__(Spider, settings)self.engine = Mock(downloader=MockDownloader())@@ -44,9 +46,9 @@ class MockCrawler(Crawler):@asynccontextmanagerasync def create_scheduler(- priority_queue_cls: str, jobdir: Path | None+ priority_queue_cls: str, jobdir: Path | None, **settings: Any) -> AsyncGenerator[Scheduler]:- mock_crawler = MockCrawler(priority_queue_cls, jobdir)+ mock_crawler = MockCrawler(priority_queue_cls, jobdir, **settings)scheduler = build_from_crawler(Scheduler, mock_crawler)spider = Spider.from_crawler(mock_crawler, name="spider")await ensure_awaitable(scheduler.open(spider))@@ -70,6 +72,11 @@ _PRIORITIES = [_URLS = {"http://foo.com/a", "http://foo.com/b", "http://foo.com/c"}+def _active_json(jobdir: Path) -> Any:+ path = jobdir / "requests.queue" / "active.json"+ return json.loads(path.read_text(encoding="utf-8"))++class TestSchedulerBase(ABC):reopen = False@@ -83,9 +90,9 @@ class TestSchedulerBase(ABC):return Nonedef create_scheduler(- self, jobdir: Path | None+ self, jobdir: Path | None, **settings: Any) -> AbstractAsyncContextManager[Scheduler]:- return create_scheduler(self.priority_queue_cls, jobdir)+ return create_scheduler(self.priority_queue_cls, jobdir, **settings)@asynccontextmanagerasync def create_scheduler_for_assertions(@@ -166,6 +173,42 @@ class TestSchedulerInMemory(TestSchedulerInMemoryBase):class TestSchedulerOnDisk(TestSchedulerOnDiskBase):priority_queue_cls = "scrapy.pqueues.ScrapyPriorityQueue"+ @coroutine_test+ async def test_active_json(self, jobdir: Path | None) -> None:+ assert jobdir is not None+ path = jobdir / "requests.queue" / "active.json"+ async with self.create_scheduler(jobdir) as scheduler:+ for url, priority in _PRIORITIES:+ scheduler.enqueue_request(Request(url, priority=priority))+ assert not path.exists()+ assert _active_json(jobdir) == [-2, -1, 0, 1, 2]++ @coroutine_test+ async def test_active_json_sync_every(self, jobdir: Path | None) -> None:+ assert jobdir is not None+ async with self.create_scheduler(jobdir, JOBDIR_SYNC_EVERY=1) as scheduler:+ for url, priority in _PRIORITIES:+ scheduler.enqueue_request(Request(url, priority=priority))+ assert _active_json(jobdir) == [-2, -1, 0, 1, 2]+ scheduler.next_request()+ assert _active_json(jobdir) == [-1, 0, 1, 2]++ @coroutine_test+ async def test_active_json_sync_window(self, jobdir: Path | None) -> None:+ assert jobdir is not None+ path = jobdir / "requests.queue" / "active.json"+ async with self.create_scheduler(jobdir, JOBDIR_SYNC_EVERY=2) as scheduler:+ scheduler.enqueue_request(Request("http://foo.com/a", priority=-2))+ assert not path.exists()+ scheduler.enqueue_request(Request("http://foo.com/b", priority=1))+ assert _active_json(jobdir) == [-1, 2]+ scheduler.enqueue_request(Request("http://foo.com/c", priority=0))+ assert _active_json(jobdir) == [-1, 2]+ scheduler.enqueue_request(Request("http://foo.com/d", priority=1))+ assert _active_json(jobdir) == [-1, 2]+ scheduler.enqueue_request(Request("http://foo.com/e", priority=2))+ assert _active_json(jobdir) == [-2, -1, 0, 2]+_URLS_WITH_SLOTS = [("http://foo.com/a", "a"),@@ -276,6 +319,25 @@ class TestSchedulerWithDownloaderAwareOnDisk():reopen = True+ @coroutine_test+ async def test_active_json_sync_every(self, jobdir: Path | None) -> None:+ assert jobdir is not None+ async with self.create_scheduler(jobdir, JOBDIR_SYNC_EVERY=1) as scheduler:++ def enqueue(slot: str, priority: int) -> None:+ request = Request(f"http://foo.com/{slot}{priority}", priority=priority)+ request.meta[Downloader.DOWNLOAD_SLOT] = slot+ scheduler.enqueue_request(request)++ enqueue("a", 1)+ assert _active_json(jobdir) == {"a": [-1]}+ enqueue("a", 2)+ assert _active_json(jobdir) == {"a": [-2, -1]}+ enqueue("b", 1)+ assert _active_json(jobdir) == {"a": [-2, -1], "b": [-1]}+ scheduler.next_request()+ assert _active_json(jobdir) == {"a": [-1], "b": [-1]}+class StartUrlsSpider(Spider):def __init__(self, start_urls):