Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Improve handling of time deltas (#7377)

Albert Eduardovich N.dd36fb785986
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 905b32d08..01d5d42b0 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -4,7 +4,7 @@ import random
from collections import deque
from dataclasses import dataclass, field
from datetime import datetime
-from time import time
+from time import monotonic
from typing import TYPE_CHECKING, Any
from twisted.internet.defer import Deferred, inlineCallbacks
@@ -196,7 +196,7 @@ class Downloader:
return
# Delay queue processing if a download_delay is configured
- now = time()
+ now = monotonic()
delay = slot.download_delay()
if delay:
penalty = delay - now + slot.lastseen
@@ -265,7 +265,7 @@ class Downloader:
slot.close()
def _slot_gc(self, age: float = 60) -> None:
- mintime = time() - age
+ mintime = monotonic() - age
for key, slot in list(self.slots.items()):
if not slot.active and slot.lastseen + slot.delay < mintime:
self.slots.pop(key).close()
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 9d102aff8..439638d33 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -7,7 +7,7 @@ import logging
import re
from contextlib import suppress
from io import BytesIO
-from time import time
+from time import monotonic
from typing import TYPE_CHECKING, Any, TypedDict, TypeVar, cast
from urllib.parse import urldefrag, urlparse
@@ -460,7 +460,7 @@ class ScrapyAgent:
if isinstance(agent, self._TunnelingAgent):
headers.removeHeader(b"Proxy-Authorization")
bodyproducer = _RequestBodyProducer(request.body) if request.body else None
- start_time = time()
+ start_time = monotonic()
d: Deferred[IResponse] = agent.request(
method,
to_bytes(url, encoding="ascii"),
@@ -489,7 +489,7 @@ class ScrapyAgent:
raise DownloadTimeoutError(f"Getting {url} took longer than {timeout} seconds.")
def _cb_latency(self, result: _T, request: Request, start_time: float) -> _T:
- request.meta["download_latency"] = time() - start_time
+ request.meta["download_latency"] = monotonic() - start_time
return result
@staticmethod
diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
index f2774cc07..a4f786363 100644
--- a/scrapy/core/downloader/handlers/http2.py
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -1,6 +1,6 @@
from __future__ import annotations
-from time import time
+from time import monotonic
from typing import TYPE_CHECKING
from urllib.parse import urldefrag
@@ -113,7 +113,7 @@ class ScrapyH2Agent:
timeout = request.meta.get("download_timeout") or self._connect_timeout
agent = self._get_agent(request, timeout)
- start_time = time()
+ start_time = monotonic()
d = agent.request(request, spider)
d.addCallback(self._cb_latency, request, start_time)
@@ -125,7 +125,7 @@ class ScrapyH2Agent:
def _cb_latency(
response: Response, request: Request, start_time: float
) -> Response:
- request.meta["download_latency"] = time() - start_time
+ request.meta["download_latency"] = monotonic() - start_time
return response
@staticmethod
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 52d287245..2a550cf78 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -3,7 +3,7 @@
from __future__ import annotations
import warnings
-from time import time
+from time import monotonic, time
from typing import TYPE_CHECKING
from urllib.parse import urldefrag, urlparse, urlunparse
@@ -100,7 +100,9 @@ class ScrapyHTTPClientFactory(ClientFactory):
afterFoundGet = False
def _build_response(self, body, request):
- request.meta["download_latency"] = self.headers_time - self.start_time
+ request.meta["download_latency"] = (
+ self._headers_time_mono - self._start_time_mono
+ )
status = int(self.status)
headers = Headers(self.response_headers)
respcls = responsetypes.from_args(headers=headers, url=self._url, body=body)
@@ -153,6 +155,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
self.response_headers: Headers | None = None
self.timeout: float = request.meta.get("download_timeout") or timeout
self.start_time: float = time()
+ self._start_time_mono: float = monotonic()
self.deferred: defer.Deferred[Response] = defer.Deferred().addCallback(
self._build_response, request
)
@@ -200,6 +203,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
def gotHeaders(self, headers):
self.headers_time = time()
+ self._headers_time_mono = monotonic()
self.response_headers = headers
def gotStatus(self, version, status, message):
diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py
index 1400f4c9b..6a5e55992 100644
--- a/scrapy/extensions/corestats.py
+++ b/scrapy/extensions/corestats.py
@@ -5,6 +5,7 @@ Extension for collecting core stats like items scraped and start/finish times
from __future__ import annotations
from datetime import datetime, timezone
+from time import monotonic
from typing import TYPE_CHECKING, Any
from scrapy import Spider, signals
@@ -21,6 +22,7 @@ class CoreStats:
def __init__(self, stats: StatsCollector):
self.stats: StatsCollector = stats
self.start_time: datetime | None = None
+ self._start_time_mono: float | None = None
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
@@ -35,13 +37,14 @@ class CoreStats:
def spider_opened(self, spider: Spider) -> None:
self.start_time = datetime.now(tz=timezone.utc)
+ self._start_time_mono = monotonic()
self.stats.set_value("start_time", self.start_time)
def spider_closed(self, spider: Spider, reason: str) -> None:
assert self.start_time is not None
- finish_time = datetime.now(tz=timezone.utc)
- elapsed_time = finish_time - self.start_time
- elapsed_time_seconds = elapsed_time.total_seconds()
+ assert self._start_time_mono is not None
+ finish_time, finish_time_mono = datetime.now(tz=timezone.utc), monotonic()
+ elapsed_time_seconds = finish_time_mono - self._start_time_mono
self.stats.set_value("elapsed_time_seconds", elapsed_time_seconds)
self.stats.set_value("finish_time", finish_time)
self.stats.set_value("finish_reason", reason)
diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py
index 5b0c66658..ecb4c1492 100644
--- a/scrapy/utils/asyncio.py
+++ b/scrapy/utils/asyncio.py
@@ -172,7 +172,7 @@ class AsyncioLoopingCall:
raise ValueError("Interval must be greater than 0")
self.interval = interval
- self._start_time = time.time()
+ self._start_time = time.monotonic()
if now:
self._call()
loop = asyncio.get_event_loop()
@@ -182,7 +182,7 @@ class AsyncioLoopingCall:
"""Return the time to sleep until the next call."""
assert self.interval is not None
assert self._start_time is not None
- now = time.time()
+ now = time.monotonic()
running_for = now - self._start_time
return self.interval - (running_for % self.interval)
diff --git a/tests/test_stats.py b/tests/test_stats.py
index 2869d302e..27af18bb1 100644
--- a/tests/test_stats.py
+++ b/tests/test_stats.py
@@ -29,6 +29,7 @@ def spider(crawler: Crawler) -> Spider:
class TestCoreStatsExtension:
+ @mock.patch("scrapy.extensions.corestats.monotonic", return_value=0)
@mock.patch("scrapy.extensions.corestats.datetime")
def test_core_stats_default_stats_collector(
self, mock_datetime: mock.Mock, crawler: Crawler, spider: Spider