Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add JOBDIR_SYNC_EVERY to keep active.json current, written atomically (#8152)

Adriandb5bad1dddb3
diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index ee82f5b36..0496a987a 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -176,9 +176,8 @@ Where:
- :class:`~scrapy.core.scheduler.Scheduler` creates the ``requests.queue/``
directory and the ``active.json`` file, the latter containing the state
- data returned by :meth:`DownloaderAwarePriorityQueue.close()
- <scrapy.pqueues.DownloaderAwarePriorityQueue.close>` the last time the job
- was paused.
+ data of :class:`~scrapy.pqueues.DownloaderAwarePriorityQueue`, written
+ when the job stops and, with :setting:`JOBDIR_SYNC_EVERY`, as it runs.
- :class:`~scrapy.pqueues.DownloaderAwarePriorityQueue` creates the
``{hostname}-{hash}`` directories.
diff --git a/docs/topics/scheduler.rst b/docs/topics/scheduler.rst
index 91e13a0ba..0181f6f11 100644
--- a/docs/topics/scheduler.rst
+++ b/docs/topics/scheduler.rst
@@ -38,4 +38,7 @@ Priority queues
===============
.. autoclass:: scrapy.pqueues.DownloaderAwarePriorityQueue
+ :members: changed, state
+
.. autoclass:: scrapy.pqueues.ScrapyPriorityQueue
+ :members: changed, state
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 2f7da6224..c8f4da941 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1651,6 +1651,28 @@ A string indicating the directory for storing the state of a crawl when
:ref:`pausing and resuming crawls <topics-jobs>`.
+.. setting:: JOBDIR_SYNC_EVERY
+
+JOBDIR_SYNC_EVERY
+-----------------
+
+.. versionadded:: VERSION
+
+Default: ``0``
+
+Number of changes to the crawl state kept in :setting:`JOBDIR` after which
+that state is written to disk while the crawl runs, in addition to when it
+stops. ``0`` writes it only when the crawl stops. ``1`` writes every change,
+so that a crawl killed before it can stop cleanly resumes from its latest
+state. Higher values trade some of that safety for fewer writes, which matters
+in broad crawls, where the state grows with the number of active domains.
+
+Applies to the ``active.json`` file of the :ref:`scheduler <topics-scheduler>`.
+For a killed crawl to resume, the scheduler queues must survive the kill too,
+which requires the SQLite types of :setting:`SCHEDULER_DISK_QUEUE` and
+:setting:`SCHEDULER_START_DISK_QUEUE`.
+
+
.. setting:: LOG_COLOR
LOG_COLOR
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index 2194d9845..ff119215a 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -237,7 +237,11 @@ class Scheduler(BaseScheduler):
:setting:`SCHEDULER_PRIORITY_QUEUE`.
The file is generated whenever the job stops (cleanly) and is loaded
- when resuming the job.
+ when resuming the job. If :setting:`JOBDIR_SYNC_EVERY` is set and the
+ priority queue implements
+ :attr:`~scrapy.pqueues.ScrapyPriorityQueue.changed` and
+ :meth:`~scrapy.pqueues.ScrapyPriorityQueue.state`, the file is also
+ generated while the job runs.
- Instantiates the configured :setting:`SCHEDULER_PRIORITY_QUEUE` with
``requests.queue/`` as persistence directory (*key*) and
@@ -319,6 +323,9 @@ class Scheduler(BaseScheduler):
self.logunser: bool = logunser
self.stats: StatsCollector | None = stats
self.crawler: Crawler | None = crawler
+ self._sync_every: int = (
+ crawler.settings.getint("JOBDIR_SYNC_EVERY") if crawler else 0
+ )
self._sdqclass: type[BaseQueue] | None = self._get_start_queue_cls(
crawler, "DISK"
)
@@ -348,6 +355,8 @@ class Scheduler(BaseScheduler):
self.spider: Spider = spider
self.mqs: ScrapyPriorityQueue = self._mq()
self.dqs: ScrapyPriorityQueue | None = self._dq() if self.dqdir else None
+ self._dqs_changes = 0
+ self._sync_dqs_state()
return self.df.open()
def close(self, reason: str) -> Deferred[None] | None:
@@ -433,15 +442,32 @@ class Scheduler(BaseScheduler):
assert self.stats is not None
self.stats.inc_value("scheduler/unserializable")
return False
+ finally:
+ self._sync_dqs_state()
return True
def _mqpush(self, request: Request) -> None:
self.mqs.push(request)
def _dqpop(self) -> Request | None:
- if self.dqs is not None:
- return self.dqs.pop()
- return None
+ if self.dqs is None:
+ return None
+ request = self.dqs.pop()
+ self._sync_dqs_state()
+ return request
+
+ def _sync_dqs_state(self) -> None:
+ # Priority queues without change tracking only report their state
+ # through close().
+ if not self._sync_every or not getattr(self.dqs, "changed", False):
+ return
+ assert self.dqs is not None
+ self.dqs.changed = False
+ self._dqs_changes += 1
+ if self._dqs_changes >= self._sync_every:
+ self._dqs_changes = 0
+ assert isinstance(self.dqdir, str)
+ self._write_dqs_state(self.dqdir, self.dqs.state())
def _mq(self) -> ScrapyPriorityQueue:
"""Create a new priority queue instance, with in-memory storage"""
@@ -494,5 +520,7 @@ class Scheduler(BaseScheduler):
return json.load(f)
def _write_dqs_state(self, dqdir: str, state: Any) -> None:
- with Path(dqdir, "active.json").open("w", encoding="utf-8") as f:
- json.dump(state, f)
+ path = Path(dqdir, "active.json")
+ tmp = path.with_name("active.json.tmp")
+ tmp.write_text(json.dumps(state), encoding="utf-8")
+ tmp.replace(path)
diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index bbbd33093..bee338e2b 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -134,6 +134,11 @@ class ScrapyPriorityQueue:
self.queues: dict[int, QueueProtocol] = {}
self._start_queues: dict[int, QueueProtocol] = {}
self.curprio: int | None = None
+ self.changed: bool = False
+ """Whether the return value of :meth:`state` changed since this
+ attribute was last set to ``False``.
+
+ The scheduler resets it after persisting that state."""
self.init_prios(startprios)
def init_prios(self, startprios: Iterable[int]) -> None:
@@ -146,12 +151,14 @@ class ScrapyPriorityQueue:
self.queues[priority] = q
else:
q.close()
+ self.changed = True
if self._start_queue_cls:
q = self._sqfactory(priority)
if q:
self._start_queues[priority] = q
else:
q.close()
+ self.changed = True
self.curprio = min(startprios)
@@ -179,10 +186,12 @@ class ScrapyPriorityQueue:
if is_start_request and self._start_queue_cls:
if priority not in self._start_queues:
self._start_queues[priority] = self._sqfactory(priority)
+ self.changed = True
q = self._start_queues[priority]
else:
if priority not in self.queues:
self.queues[priority] = self.qfactory(priority)
+ self.changed = True
q = self.queues[priority]
q.push(request) # this may fail (eg. serialization error)
if self.curprio is None or priority < self.curprio:
@@ -199,6 +208,7 @@ class ScrapyPriorityQueue:
if not q:
del self.queues[self.curprio]
q.close()
+ self.changed = True
if not self._start_queues:
self._update_curprio()
return m
@@ -212,6 +222,7 @@ class ScrapyPriorityQueue:
if not q:
del self._start_queues[self.curprio]
q.close()
+ self.changed = True
self._update_curprio()
return m
else:
@@ -243,13 +254,18 @@ class ScrapyPriorityQueue:
# Protocols can't declare optional members
return cast("Request", queue.peek()) # type: ignore[attr-defined]
+ def state(self) -> list[int]:
+ """Return what to pass as *startprios* to resume this queue later.
+
+ Same as the return value of :meth:`close`, without closing."""
+ return sorted(self.queues.keys() | self._start_queues.keys())
+
def close(self) -> list[int]:
- active: set[int] = set()
+ active = self.state()
for queues in (self.queues, self._start_queues):
- for p, q in queues.items():
- active.add(p)
+ for q in queues.values():
q.close()
- return list(active)
+ return active
def __len__(self) -> int:
return (
@@ -357,9 +373,12 @@ class DownloaderAwarePriorityQueue:
self.pqueues: dict[str, ScrapyPriorityQueue] = {} # slot -> priority queue
self._last_selected_slot: str | None = None
+ self.changed: bool = False
+ """See :attr:`ScrapyPriorityQueue.changed`."""
if slot_startprios:
for slot, startprios in slot_startprios.items():
self.pqueues[slot] = self.pqfactory(slot, startprios)
+ self._collect_change(self.pqueues[slot])
def _next_slot(self, stats: list[tuple[int, str]], *, update_state: bool) -> str:
last = self._last_selected_slot
@@ -399,6 +418,11 @@ class DownloaderAwarePriorityQueue:
start_queue_cls=self._start_queue_cls,
)
+ def _collect_change(self, queue: ScrapyPriorityQueue) -> None:
+ if queue.changed:
+ queue.changed = False
+ self.changed = True
+
def pop(self) -> Request | None:
stats = self._downloader_interface.stats(self.pqueues)
@@ -408,9 +432,11 @@ class DownloaderAwarePriorityQueue:
slot = self._next_slot(stats, update_state=True)
queue = self.pqueues[slot]
request = queue.pop()
+ self._collect_change(queue)
if len(queue) == 0:
del self.pqueues[slot]
queue.close()
+ self.changed = True
if self.key:
# Reclaim the slot directory; rmdir leaves it alone if the
# downstream queues did not remove all their files.
@@ -422,8 +448,10 @@ class DownloaderAwarePriorityQueue:
slot = self._downloader_interface.get_slot_key(request)
if slot not in self.pqueues:
self.pqueues[slot] = self.pqfactory(slot)
+ self.changed = True
queue = self.pqueues[slot]
queue.push(request)
+ self._collect_change(queue)
def peek(self) -> Request | None:
"""Returns the next object to be returned by :meth:`pop`,
@@ -439,8 +467,14 @@ class DownloaderAwarePriorityQueue:
queue = self.pqueues[slot]
return queue.peek()
+ def state(self) -> dict[str, list[int]]:
+ """See :meth:`ScrapyPriorityQueue.state`."""
+ return {slot: queue.state() for slot, queue in self.pqueues.items()}
+
def close(self) -> dict[str, list[int]]:
- active = {slot: queue.close() for slot, queue in self.pqueues.items()}
+ active = self.state()
+ for queue in self.pqueues.values():
+ queue.close()
self.pqueues.clear()
return active
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 277c49e80..a03ee957e 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -138,6 +138,7 @@ __all__ = [
"ITEM_PIPELINES_BASE",
"ITEM_PROCESSOR",
"JOBDIR",
+ "JOBDIR_SYNC_EVERY",
"JSONVALIDATION_ENABLED",
"LOGSTATS_INTERVAL",
"LOG_COLOR",
@@ -456,6 +457,7 @@ ITEM_PIPELINES_BASE: dict[str, int] = {}
ITEM_PROCESSOR = "scrapy.pipelines.ItemPipelineManager"
JOBDIR = None
+JOBDIR_SYNC_EVERY = 0
JSONVALIDATION_ENABLED = False
diff --git a/tests/test_crash.py b/tests/test_crash.py
new file mode 100644
index 000000000..d16af7400
--- /dev/null
+++ b/tests/test_crash.py
@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+import json
+import re
+from typing import TYPE_CHECKING, Any
+
+from scrapy.pqueues import _path_safe
+from tests.utils.cmdline import proc
+
+if TYPE_CHECKING:
+ from pathlib import Path
+
+
+# Dies without any cleanup once kill_at requests have reached the scheduler.
+SPIDER = """
+import os
+
+from scrapy import Request, Spider, signals
+
+
+class KillSpider(Spider):
+ name = "kill"
+ custom_settings = {
+ "SCHEDULER_DISK_QUEUE": "scrapy.squeues.PickleFifoSQLiteQueue",
+ "SCHEDULER_START_DISK_QUEUE": "scrapy.squeues.PickleFifoSQLiteQueue",
+ "CONCURRENT_REQUESTS": 1,
+ "JOBDIR_SYNC_EVERY": 1,
+ }
+
+ @classmethod
+ def from_crawler(cls, crawler, *args, **kwargs):
+ spider = super().from_crawler(crawler, *args, **kwargs)
+ spider.scheduled = 0
+ crawler.signals.connect(spider.request_scheduled, signals.request_scheduled)
+ return spider
+
+ def request_scheduled(self, request, spider):
+ self.scheduled += 1
+ if self.scheduled == int(getattr(self, "kill_at", 0)):
+ os._exit(1)
+
+ async def start(self):
+ if hasattr(self, "resume"):
+ return
+ for i in range(20):
+ yield Request(f"data:,{i}", priority=i % 3)
+
+ def parse(self, response):
+ self.logger.info(f"Parsed {response.url}")
+"""
+
+
+def run(tmp_path: Path, *args: str) -> str:
+ spider = tmp_path / "spider.py"
+ spider.write_text(SPIDER, encoding="utf-8")
+ jobdir = tmp_path / "job"
+ _, _, log = proc(
+ "runspider", str(spider), "-s", f"JOBDIR={jobdir}", *args, cwd=tmp_path
+ )
+ return log
+
+
+def _active_json(tmp_path: Path) -> Any:
+ path = tmp_path / "job" / "requests.queue" / "active.json"
+ return json.loads(path.read_text(encoding="utf-8"))
+
+
+def test_resume_after_kill(tmp_path: Path) -> None:
+ # The delay keeps all but the first request or two in the queue when the
+ # spider dies, while start requests keep being scheduled eagerly.
+ run(tmp_path, "-a", "kill_at=10", "-s", "DOWNLOAD_DELAY=5")
+ [(slot, priorities)] = _active_json(tmp_path).items()
+ slot_dir = tmp_path / "job" / "requests.queue" / _path_safe(slot)
+ assert set(priorities) == {
+ int(path.name.rstrip("s")) for path in slot_dir.iterdir()
+ }
+
+ log = run(tmp_path, "-a", "resume=1")
+ match = re.search(r"Resuming crawl \((\d+) requests scheduled\)", log)
+ assert match, log
+ pending = int(match.group(1))
+ assert pending > 0
+ assert log.count("Parsed data:,") == pending
+ assert "Spider closed (finished)" in log
+ assert _active_json(tmp_path) == {}
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 43cdab183..768785ab2 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -1,9 +1,10 @@
from __future__ import annotations
+import json
import warnings
from abc import ABC, abstractmethod
from contextlib import AbstractAsyncContextManager, asynccontextmanager
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
from unittest.mock import Mock
import pytest
@@ -28,7 +29,7 @@ if TYPE_CHECKING:
class MockCrawler(Crawler):
- def __init__(self, priority_queue_cls: str, jobdir: Path | None):
+ def __init__(self, priority_queue_cls: str, jobdir: Path | None, **settings: Any):
settings = {
"SCHEDULER_DEBUG": False,
"SCHEDULER_DISK_QUEUE": "scrapy.squeues.PickleLifoDiskQueue",
@@ -36,6 +37,7 @@ class MockCrawler(Crawler):
"SCHEDULER_PRIORITY_QUEUE": priority_queue_cls,
"JOBDIR": str(jobdir) if jobdir is not None else None,
"DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter",
+ **settings,
}
super().__init__(Spider, settings)
self.engine = Mock(downloader=MockDownloader())
@@ -44,9 +46,9 @@ class MockCrawler(Crawler):
@asynccontextmanager
async def create_scheduler(
- priority_queue_cls: str, jobdir: Path | None
+ priority_queue_cls: str, jobdir: Path | None, **settings: Any
) -> AsyncGenerator[Scheduler]:
- mock_crawler = MockCrawler(priority_queue_cls, jobdir)
+ mock_crawler = MockCrawler(priority_queue_cls, jobdir, **settings)
scheduler = build_from_crawler(Scheduler, mock_crawler)
spider = Spider.from_crawler(mock_crawler, name="spider")
await ensure_awaitable(scheduler.open(spider))
@@ -70,6 +72,11 @@ _PRIORITIES = [
_URLS = {"http://foo.com/a", "http://foo.com/b", "http://foo.com/c"}
+def _active_json(jobdir: Path) -> Any:
+ path = jobdir / "requests.queue" / "active.json"
+ return json.loads(path.read_text(encoding="utf-8"))
+
+
class TestSchedulerBase(ABC):
reopen = False
@@ -83,9 +90,9 @@ class TestSchedulerBase(ABC):
return None
def create_scheduler(
- self, jobdir: Path | None
+ self, jobdir: Path | None, **settings: Any
) -> AbstractAsyncContextManager[Scheduler]:
- return create_scheduler(self.priority_queue_cls, jobdir)
+ return create_scheduler(self.priority_queue_cls, jobdir, **settings)
@asynccontextmanager
async def create_scheduler_for_assertions(
@@ -166,6 +173,42 @@ class TestSchedulerInMemory(TestSchedulerInMemoryBase):
class TestSchedulerOnDisk(TestSchedulerOnDiskBase):
priority_queue_cls = "scrapy.pqueues.ScrapyPriorityQueue"
+ @coroutine_test
+ async def test_active_json(self, jobdir: Path | None) -> None:
+ assert jobdir is not None
+ path = jobdir / "requests.queue" / "active.json"
+ async with self.create_scheduler(jobdir) as scheduler:
+ for url, priority in _PRIORITIES:
+ scheduler.enqueue_request(Request(url, priority=priority))
+ assert not path.exists()
+ assert _active_json(jobdir) == [-2, -1, 0, 1, 2]
+
+ @coroutine_test
+ async def test_active_json_sync_every(self, jobdir: Path | None) -> None:
+ assert jobdir is not None
+ async with self.create_scheduler(jobdir, JOBDIR_SYNC_EVERY=1) as scheduler:
+ for url, priority in _PRIORITIES:
+ scheduler.enqueue_request(Request(url, priority=priority))
+ assert _active_json(jobdir) == [-2, -1, 0, 1, 2]
+ scheduler.next_request()
+ assert _active_json(jobdir) == [-1, 0, 1, 2]
+
+ @coroutine_test
+ async def test_active_json_sync_window(self, jobdir: Path | None) -> None:
+ assert jobdir is not None
+ path = jobdir / "requests.queue" / "active.json"
+ async with self.create_scheduler(jobdir, JOBDIR_SYNC_EVERY=2) as scheduler:
+ scheduler.enqueue_request(Request("http://foo.com/a", priority=-2))
+ assert not path.exists()
+ scheduler.enqueue_request(Request("http://foo.com/b", priority=1))
+ assert _active_json(jobdir) == [-1, 2]
+ scheduler.enqueue_request(Request("http://foo.com/c", priority=0))
+ assert _active_json(jobdir) == [-1, 2]
+ scheduler.enqueue_request(Request("http://foo.com/d", priority=1))
+ assert _active_json(jobdir) == [-1, 2]
+ scheduler.enqueue_request(Request("http://foo.com/e", priority=2))
+ assert _active_json(jobdir) == [-2, -1, 0, 2]
+
_URLS_WITH_SLOTS = [
("http://foo.com/a", "a"),
@@ -276,6 +319,25 @@ class TestSchedulerWithDownloaderAwareOnDisk(
):
reopen = True
+ @coroutine_test
+ async def test_active_json_sync_every(self, jobdir: Path | None) -> None:
+ assert jobdir is not None
+ async with self.create_scheduler(jobdir, JOBDIR_SYNC_EVERY=1) as scheduler:
+
+ def enqueue(slot: str, priority: int) -> None:
+ request = Request(f"http://foo.com/{slot}{priority}", priority=priority)
+ request.meta[Downloader.DOWNLOAD_SLOT] = slot
+ scheduler.enqueue_request(request)
+
+ enqueue("a", 1)
+ assert _active_json(jobdir) == {"a": [-1]}
+ enqueue("a", 2)
+ assert _active_json(jobdir) == {"a": [-2, -1]}
+ enqueue("b", 1)
+ assert _active_json(jobdir) == {"a": [-2, -1], "b": [-1]}
+ scheduler.next_request()
+ assert _active_json(jobdir) == {"a": [-1], "b": [-1]}
+
class StartUrlsSpider(Spider):
def __init__(self, start_urls):