Scrapy, a fast high-level web crawling & scraping framework for Python.
Improve handling of time deltas (#7377)
Albert Eduardovich N.dd36fb785986
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 905b32d08..01d5d42b0 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -4,7 +4,7 @@ import randomfrom collections import dequefrom dataclasses import dataclass, fieldfrom datetime import datetime-from time import time+from time import monotonicfrom typing import TYPE_CHECKING, Anyfrom twisted.internet.defer import Deferred, inlineCallbacks@@ -196,7 +196,7 @@ class Downloader:return# Delay queue processing if a download_delay is configured- now = time()+ now = monotonic()delay = slot.download_delay()if delay:penalty = delay - now + slot.lastseen@@ -265,7 +265,7 @@ class Downloader:slot.close()def _slot_gc(self, age: float = 60) -> None:- mintime = time() - age+ mintime = monotonic() - agefor key, slot in list(self.slots.items()):if not slot.active and slot.lastseen + slot.delay < mintime:self.slots.pop(key).close()diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex 9d102aff8..439638d33 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -7,7 +7,7 @@ import loggingimport refrom contextlib import suppressfrom io import BytesIO-from time import time+from time import monotonicfrom typing import TYPE_CHECKING, Any, TypedDict, TypeVar, castfrom urllib.parse import urldefrag, urlparse@@ -460,7 +460,7 @@ class ScrapyAgent:if isinstance(agent, self._TunnelingAgent):headers.removeHeader(b"Proxy-Authorization")bodyproducer = _RequestBodyProducer(request.body) if request.body else None- start_time = time()+ start_time = monotonic()d: Deferred[IResponse] = agent.request(method,to_bytes(url, encoding="ascii"),@@ -489,7 +489,7 @@ class ScrapyAgent:raise DownloadTimeoutError(f"Getting {url} took longer than {timeout} seconds.")def _cb_latency(self, result: _T, request: Request, start_time: float) -> _T:- request.meta["download_latency"] = time() - start_time+ request.meta["download_latency"] = monotonic() - start_timereturn result@staticmethoddiff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.pyindex f2774cc07..a4f786363 100644--- a/scrapy/core/downloader/handlers/http2.py+++ b/scrapy/core/downloader/handlers/http2.py@@ -1,6 +1,6 @@from __future__ import annotations-from time import time+from time import monotonicfrom typing import TYPE_CHECKINGfrom urllib.parse import urldefrag@@ -113,7 +113,7 @@ class ScrapyH2Agent:timeout = request.meta.get("download_timeout") or self._connect_timeoutagent = self._get_agent(request, timeout)- start_time = time()+ start_time = monotonic()d = agent.request(request, spider)d.addCallback(self._cb_latency, request, start_time)@@ -125,7 +125,7 @@ class ScrapyH2Agent:def _cb_latency(response: Response, request: Request, start_time: float) -> Response:- request.meta["download_latency"] = time() - start_time+ request.meta["download_latency"] = monotonic() - start_timereturn response@staticmethoddiff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.pyindex 52d287245..2a550cf78 100644--- a/scrapy/core/downloader/webclient.py+++ b/scrapy/core/downloader/webclient.py@@ -3,7 +3,7 @@from __future__ import annotationsimport warnings-from time import time+from time import monotonic, timefrom typing import TYPE_CHECKINGfrom urllib.parse import urldefrag, urlparse, urlunparse@@ -100,7 +100,9 @@ class ScrapyHTTPClientFactory(ClientFactory):afterFoundGet = Falsedef _build_response(self, body, request):- request.meta["download_latency"] = self.headers_time - self.start_time+ request.meta["download_latency"] = (+ self._headers_time_mono - self._start_time_mono+ )status = int(self.status)headers = Headers(self.response_headers)respcls = responsetypes.from_args(headers=headers, url=self._url, body=body)@@ -153,6 +155,7 @@ class ScrapyHTTPClientFactory(ClientFactory):self.response_headers: Headers | None = Noneself.timeout: float = request.meta.get("download_timeout") or timeoutself.start_time: float = time()+ self._start_time_mono: float = monotonic()self.deferred: defer.Deferred[Response] = defer.Deferred().addCallback(self._build_response, request)@@ -200,6 +203,7 @@ class ScrapyHTTPClientFactory(ClientFactory):def gotHeaders(self, headers):self.headers_time = time()+ self._headers_time_mono = monotonic()self.response_headers = headersdef gotStatus(self, version, status, message):diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.pyindex 1400f4c9b..6a5e55992 100644--- a/scrapy/extensions/corestats.py+++ b/scrapy/extensions/corestats.py@@ -5,6 +5,7 @@ Extension for collecting core stats like items scraped and start/finish timesfrom __future__ import annotationsfrom datetime import datetime, timezone+from time import monotonicfrom typing import TYPE_CHECKING, Anyfrom scrapy import Spider, signals@@ -21,6 +22,7 @@ class CoreStats:def __init__(self, stats: StatsCollector):self.stats: StatsCollector = statsself.start_time: datetime | None = None+ self._start_time_mono: float | None = None@classmethoddef from_crawler(cls, crawler: Crawler) -> Self:@@ -35,13 +37,14 @@ class CoreStats:def spider_opened(self, spider: Spider) -> None:self.start_time = datetime.now(tz=timezone.utc)+ self._start_time_mono = monotonic()self.stats.set_value("start_time", self.start_time)def spider_closed(self, spider: Spider, reason: str) -> None:assert self.start_time is not None- finish_time = datetime.now(tz=timezone.utc)- elapsed_time = finish_time - self.start_time- elapsed_time_seconds = elapsed_time.total_seconds()+ assert self._start_time_mono is not None+ finish_time, finish_time_mono = datetime.now(tz=timezone.utc), monotonic()+ elapsed_time_seconds = finish_time_mono - self._start_time_monoself.stats.set_value("elapsed_time_seconds", elapsed_time_seconds)self.stats.set_value("finish_time", finish_time)self.stats.set_value("finish_reason", reason)diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.pyindex 5b0c66658..ecb4c1492 100644--- a/scrapy/utils/asyncio.py+++ b/scrapy/utils/asyncio.py@@ -172,7 +172,7 @@ class AsyncioLoopingCall:raise ValueError("Interval must be greater than 0")self.interval = interval- self._start_time = time.time()+ self._start_time = time.monotonic()if now:self._call()loop = asyncio.get_event_loop()@@ -182,7 +182,7 @@ class AsyncioLoopingCall:"""Return the time to sleep until the next call."""assert self.interval is not Noneassert self._start_time is not None- now = time.time()+ now = time.monotonic()running_for = now - self._start_timereturn self.interval - (running_for % self.interval)diff --git a/tests/test_stats.py b/tests/test_stats.pyindex 2869d302e..27af18bb1 100644--- a/tests/test_stats.py+++ b/tests/test_stats.py@@ -29,6 +29,7 @@ def spider(crawler: Crawler) -> Spider:class TestCoreStatsExtension:+ @mock.patch("scrapy.extensions.corestats.monotonic", return_value=0)@mock.patch("scrapy.extensions.corestats.datetime")def test_core_stats_default_stats_collector(self, mock_datetime: mock.Mock, crawler: Crawler, spider: Spider