Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Bump Ruff to 0.16. (#8182)

Andrey Rakhmatullin4f3e27e6c00f
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 1b9e29b30..a7c62346a 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -6,7 +6,7 @@ exclude: |
)
repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.15.20
+ rev: v0.16.8
hooks:
- id: ruff-check
args: [ --fix ]
diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py
old mode 100644
new mode 100755
diff --git a/pyproject.toml b/pyproject.toml
index a5f6f4907..d7cca1bfd 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -351,6 +351,12 @@ extend-select = [
"COM",
# pydocstyle
"D",
+ # pycodestyle
+ "E4",
+ "E7",
+ "E9",
+ # Pyflakes
+ "F",
# flake8-future-annotations
"FA",
# flynt
@@ -405,8 +411,8 @@ extend-select = [
"YTT",
]
ignore = [
- # Ones we want to ignore
-
+ # Do not catch blind exception
+ "BLE001",
# Trailing comma missing
"COM812",
# Missing docstring in public module
@@ -447,6 +453,8 @@ ignore = [
"PLR0911",
# Too many arguments in function definition
"PLR0913",
+ # Too many positional arguments
+ "PLR0917",
# Magic value used in comparison
"PLR2004",
# String contains ambiguous {}.
@@ -463,6 +471,8 @@ ignore = [
"SIM115",
# Yoda condition detected
"SIM300",
+ # Prefer `TypeError` exception for invalid type
+ "TRY004",
]
[tool.ruff.lint.flake8-tidy-imports]
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index a4d4fc97f..51ce5b1e8 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -4,7 +4,6 @@ import random
import warnings
from collections import deque
from dataclasses import dataclass, field
-from datetime import datetime
from time import monotonic
from typing import TYPE_CHECKING, Any
@@ -128,7 +127,7 @@ class Slot:
f"delay={self.delay:.2f} jitter={self.jitter!r} "
f"len(active)={len(self.active)} len(queue)={len(self.queue)} "
f"len(transferring)={len(self.transferring)} "
- f"lastseen={datetime.fromtimestamp(self.lastseen).isoformat()}>"
+ f"lastseen={self.lastseen:.2f}>"
)
diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index 84dc6216b..57a8e962b 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -120,10 +120,9 @@ class DownloadHandlers:
self._notconfigured[scheme] = str(ex)
return None
except Exception as ex:
- logger.error(
+ logger.exception(
'Loading "%(clspath)s" for scheme "%(scheme)s"',
{"clspath": path, "scheme": scheme},
- exc_info=True,
extra={"crawler": self._crawler},
)
self._notconfigured[scheme] = str(ex)
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index c4c95c24a..162f03c0c 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -14,7 +14,6 @@ import warnings
from enum import Enum
from functools import partial
from time import time
-from traceback import format_exc
from typing import TYPE_CHECKING, Any
from twisted.internet.defer import CancelledError, Deferred, inlineCallbacks
@@ -391,11 +390,7 @@ class ExecutionEngine:
except Exception as exception:
self._start = None
self._start_error = True
- exception_traceback = format_exc()
- logger.error(
- f"Error while reading start items and requests: {exception}.\n{exception_traceback}",
- exc_info=True,
- )
+ logger.exception("Error while reading start items and requests")
self.signals.send_catch_log(
signal=signals.spider_error,
failure=Failure(),
@@ -440,9 +435,8 @@ class ExecutionEngine:
except Exception:
# an error happened, log it and stop the engine
self._start_request_processing_awaitable = None
- logger.error(
+ logger.exception(
"Error while processing requests from start()",
- exc_info=True,
extra={"spider": self.spider},
)
await self.stop_async()
@@ -539,10 +533,8 @@ class ExecutionEngine:
yield self.scraper.enqueue_scrape(result, request)
except Exception:
assert self.spider is not None
- logger.error(
- "Error while enqueuing scrape",
- exc_info=True,
- extra={"spider": self.spider},
+ logger.exception(
+ "Error while enqueuing scrape", extra={"spider": self.spider}
)
def spider_is_idle(self) -> bool:
@@ -809,30 +801,24 @@ class ExecutionEngine:
try:
await self._slot.close()
except Exception:
- logger.error("Slot close failure", exc_info=True, extra={"spider": spider})
+ logger.exception("Slot close failure", extra={"spider": spider})
try:
self.downloader.close()
except Exception:
- logger.error(
- "Downloader close failure", exc_info=True, extra={"spider": spider}
- )
+ logger.exception("Downloader close failure", extra={"spider": spider})
try:
await self.scraper.close_spider_async()
except Exception:
- logger.error(
- "Scraper close failure", exc_info=True, extra={"spider": spider}
- )
+ logger.exception("Scraper close failure", extra={"spider": spider})
if hasattr(self._slot.scheduler, "close"):
try:
if (d := self._slot.scheduler.close(reason)) is not None:
await maybe_deferred_to_future(d)
except Exception:
- logger.error(
- "Scheduler close failure", exc_info=True, extra={"spider": spider}
- )
+ logger.exception("Scheduler close failure", extra={"spider": spider})
try:
await self.signals.send_catch_log_async(
@@ -841,10 +827,8 @@ class ExecutionEngine:
reason=reason,
)
except Exception:
- logger.error(
- "Error while sending spider_close signal",
- exc_info=True,
- extra={"spider": spider},
+ logger.exception(
+ "Error while sending spider_close signal", extra={"spider": spider}
)
try:
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index b70f57435..bfdda63ca 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -227,10 +227,9 @@ class Scraper:
try:
yield dfd # fired in _wait_for_processing()
except Exception:
- logger.error(
+ logger.exception(
"Scraper bug processing %(request)s",
{"request": request},
- exc_info=True,
extra={"spider": self.crawler.spider},
)
finally:
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 46d5a7b76..2d73c3b48 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -101,10 +101,8 @@ class RobotsTxtMiddleware:
await self._parse_robots(resp, netloc, request)
except Exception as e:
if not isinstance(e, IgnoreRequest):
- logger.error(
- "Error downloading %(request)s: %(f_exception)s",
- {"request": request, "f_exception": e},
- exc_info=True,
+ logger.exception(
+ f"Error downloading {request}",
extra={"spider": self.crawler.spider},
)
self._robots_error(e, netloc)
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index c2ab1581b..127becab7 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -627,10 +627,9 @@ class FeedExporter:
try:
await ensure_awaitable(slot.storage.store(self._get_file(slot)))
except Exception:
- logger.error(
+ logger.exception(
"Error storing %s",
logmsg,
- exc_info=True,
extra={"spider": spider},
)
self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}")
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index dda66b18f..8e49f52e9 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -678,11 +678,10 @@ class FilesPipeline(MediaPipeline):
)
raise
except Exception as exc:
- logger.error(
+ logger.exception(
"File (unknown-error): Error processing file from %(request)s "
"referred in <%(referer)s>",
{"request": request, "referer": referer},
- exc_info=True,
extra={"spider": info.spider},
)
raise _FileException(str(exc)) from exc
diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py
index 70e1de505..289df734f 100644
--- a/scrapy/utils/signal.py
+++ b/scrapy/utils/signal.py
@@ -63,10 +63,9 @@ def send_catch_log(
result = Failure()
except Exception:
result = Failure()
- logger.error(
+ logger.exception(
"Error caught on signal handler: %(receiver)s",
{"receiver": receiver},
- exc_info=True,
extra={"spider": spider},
)
else:
@@ -199,10 +198,9 @@ async def _send_catch_log_asyncio(
except dont_log as ex: # pylint: disable=catching-non-exception
result = ex
except Exception as ex:
- logger.error(
+ logger.exception(
"Error caught on signal handler: %(receiver)s",
{"receiver": receiver},
- exc_info=True,
extra={"spider": spider},
)
result = ex
diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py
index ae340093b..1d2d18d0f 100644
--- a/tests/test_downloader_handler_twisted_http2.py
+++ b/tests/test_downloader_handler_twisted_http2.py
@@ -149,9 +149,11 @@ class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase):
assert (
"scrapy.core._http2.stream",
logging.WARNING,
- f"Ignoring bad Content-Length header "
- f"{bad_content_length!r} of request {request}, sending "
- f"{actual_content_length!r} instead",
+ (
+ f"Ignoring bad Content-Length header "
+ f"{bad_content_length!r} of request {request}, sending "
+ f"{actual_content_length!r} instead"
+ ),
) in caplog.record_tuples
@coroutine_test
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index 4800e4d29..36d2321d5 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -133,14 +133,18 @@ class TestCookiesMiddleware:
(
"scrapy.downloadermiddlewares.cookies",
logging.DEBUG,
- "Received cookies from: <200 http://scrapytest.org/>\n"
- "Set-Cookie: C1=value1; path=/\n",
+ (
+ "Received cookies from: <200 http://scrapytest.org/>\n"
+ "Set-Cookie: C1=value1; path=/\n"
+ ),
),
(
"scrapy.downloadermiddlewares.cookies",
logging.DEBUG,
- "Sending cookies to: <GET http://scrapytest.org/sub1/>\n"
- "Cookie: C1=value1\n",
+ (
+ "Sending cookies to: <GET http://scrapytest.org/sub1/>\n"
+ "Cookie: C1=value1\n"
+ ),
),
]
@@ -466,20 +470,26 @@ class TestCookiesMiddleware:
(
"scrapy.utils.request",
logging.WARNING,
- "Invalid cookie found in request <GET http://example.org/1>:"
- " {'value': 'bar', 'secure': False} ('name' is missing)",
+ (
+ "Invalid cookie found in request <GET http://example.org/1>:"
+ " {'value': 'bar', 'secure': False} ('name' is missing)"
+ ),
),
(
"scrapy.utils.request",
logging.WARNING,
- "Invalid cookie found in request <GET http://example.org/2>:"
- " {'name': 'foo', 'secure': False} ('value' is missing)",
+ (
+ "Invalid cookie found in request <GET http://example.org/2>:"
+ " {'name': 'foo', 'secure': False} ('value' is missing)"
+ ),
),
(
"scrapy.utils.request",
logging.WARNING,
- "Invalid cookie found in request <GET http://example.org/3>:"
- " {'name': 'foo', 'value': None, 'secure': False} ('value' is missing)",
+ (
+ "Invalid cookie found in request <GET http://example.org/3>:"
+ " {'name': 'foo', 'value': None, 'secure': False} ('value' is missing)"
+ ),
),
]
self.assertCookieValEqual(req1.headers["Cookie"], "key=value1")
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index a42bbc715..a9dc1525a 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -318,8 +318,10 @@ class TestGetRetryRequest:
assert (
"scrapy.downloadermiddlewares.retry",
logging.DEBUG,
- f"Retrying {request} (failed {expected_retry_times} times): "
- f"{expected_reason}",
+ (
+ f"Retrying {request} (failed {expected_retry_times} times): "
+ f"{expected_reason}"
+ ),
) in caplog.record_tuples
def test_max_retries_reached(self, caplog: pytest.LogCaptureFixture) -> None:
@@ -339,8 +341,10 @@ class TestGetRetryRequest:
assert (
"scrapy.downloadermiddlewares.retry",
logging.ERROR,
- f"Gave up retrying {request} (failed {failure_count} times): "
- f"{expected_reason}",
+ (
+ f"Gave up retrying {request} (failed {failure_count} times): "
+ f"{expected_reason}"
+ ),
) in caplog.record_tuples
def test_one_retry(self, caplog: pytest.LogCaptureFixture) -> None:
@@ -364,8 +368,10 @@ class TestGetRetryRequest:
assert (
"scrapy.downloadermiddlewares.retry",
logging.DEBUG,
- f"Retrying {request} (failed {expected_retry_times} times): "
- f"{expected_reason}",
+ (
+ f"Retrying {request} (failed {expected_retry_times} times): "
+ f"{expected_reason}"
+ ),
) in caplog.record_tuples
def test_two_retries(self, caplog: pytest.LogCaptureFixture) -> None:
@@ -397,8 +403,10 @@ class TestGetRetryRequest:
assert (
"scrapy.downloadermiddlewares.retry",
logging.DEBUG,
- f"Retrying {request} (failed {expected_retry_times} times): "
- f"{expected_reason}",
+ (
+ f"Retrying {request} (failed {expected_retry_times} times): "
+ f"{expected_reason}"
+ ),
) in caplog.record_tuples
caplog.clear()
@@ -418,8 +426,10 @@ class TestGetRetryRequest:
assert (
"scrapy.downloadermiddlewares.retry",
logging.ERROR,
- f"Gave up retrying {request} (failed {failure_count} times): "
- f"{expected_reason}",
+ (
+ f"Gave up retrying {request} (failed {failure_count} times): "
+ f"{expected_reason}"
+ ),
) in caplog.record_tuples
def test_no_spider(self):
@@ -520,8 +530,10 @@ class TestGetRetryRequest:
assert (
"scrapy.downloadermiddlewares.retry",
logging.DEBUG,
- f"Retrying {request} (failed {expected_retry_times} times): "
- f"{expected_reason}",
+ (
+ f"Retrying {request} (failed {expected_retry_times} times): "
+ f"{expected_reason}"
+ ),
) in caplog.record_tuples
def test_reason_builtin_exception(self, caplog: pytest.LogCaptureFixture) -> None:
@@ -543,8 +555,10 @@ class TestGetRetryRequest:
assert (
"scrapy.downloadermiddlewares.retry",
logging.DEBUG,
- f"Retrying {request} (failed {expected_retry_times} times): "
- f"{expected_reason}",
+ (
+ f"Retrying {request} (failed {expected_retry_times} times): "
+ f"{expected_reason}"
+ ),
) in caplog.record_tuples
def test_reason_builtin_exception_class(
@@ -568,8 +582,10 @@ class TestGetRetryRequest:
assert (
"scrapy.downloadermiddlewares.retry",
logging.DEBUG,
- f"Retrying {request} (failed {expected_retry_times} times): "
- f"{expected_reason}",
+ (
+ f"Retrying {request} (failed {expected_retry_times} times): "
+ f"{expected_reason}"
+ ),
) in caplog.record_tuples
def test_reason_custom_exception(self, caplog: pytest.LogCaptureFixture) -> None:
@@ -591,8 +607,10 @@ class TestGetRetryRequest:
assert (
"scrapy.downloadermiddlewares.retry",
logging.DEBUG,
- f"Retrying {request} (failed {expected_retry_times} times): "
- f"{expected_reason}",
+ (
+ f"Retrying {request} (failed {expected_retry_times} times): "
+ f"{expected_reason}"
+ ),
) in caplog.record_tuples
def test_reason_custom_exception_class(
@@ -616,8 +634,10 @@ class TestGetRetryRequest:
assert (
"scrapy.downloadermiddlewares.retry",
logging.DEBUG,
- f"Retrying {request} (failed {expected_retry_times} times): "
- f"{expected_reason}",
+ (
+ f"Retrying {request} (failed {expected_retry_times} times): "
+ f"{expected_reason}"
+ ),
) in caplog.record_tuples
def test_custom_logger(self, caplog: pytest.LogCaptureFixture) -> None:
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index 63c5565e5..cd68bb8e0 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -186,8 +186,10 @@ class TestRFPDupeFilter:
assert (
"scrapy.dupefilters",
logging.DEBUG,
- "Filtered duplicate request: <GET http://scrapytest.org/index.html> - no more"
- " duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)",
+ (
+ "Filtered duplicate request: <GET http://scrapytest.org/index.html> - no more"
+ " duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)"
+ ),
) in caplog.record_tuples
dupefilter.close("finished")
@@ -224,8 +226,10 @@ class TestRFPDupeFilter:
assert (
"scrapy.dupefilters",
logging.DEBUG,
- "Filtered duplicate request: <GET http://scrapytest.org/index.html>"
- " (referer: http://scrapytest.org/INDEX.html)",
+ (
+ "Filtered duplicate request: <GET http://scrapytest.org/index.html>"
+ " (referer: http://scrapytest.org/INDEX.html)"
+ ),
) in caplog.record_tuples
dupefilter.close("finished")
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 4359f4ff4..d74556236 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -5,7 +5,7 @@ import pickle
import re
from abc import ABC, abstractmethod
from collections.abc import Mapping
-from datetime import datetime
+from datetime import datetime, timezone
from io import BytesIO
from typing import Any
@@ -75,11 +75,12 @@ class TestBaseItemExporter(ABC):
exported_dict[k] = to_unicode(v)
assert self.i == self.item_class(**exported_dict)
- def _get_nonstring_types_item(self) -> dict[str, Any]:
+ @staticmethod
+ def _get_nonstring_types_item() -> dict[str, Any]:
return {
"boolean": False,
"number": 22,
- "time": datetime(2015, 1, 1, 1, 1, 1),
+ "time": datetime(2015, 1, 1, 1, 1, 1, tzinfo=timezone.utc),
"float": 3.14,
}
@@ -383,7 +384,7 @@ class TestCsvItemExporter(TestBaseItemExporter):
self.assertExportResult(
item=self._get_nonstring_types_item(),
include_headers_line=False,
- expected="22,False,3.14,2015-01-01 01:01:01\r\n",
+ expected="22,False,3.14,2015-01-01 01:01:01+00:00\r\n",
)
def test_errors_default(self):
@@ -528,7 +529,7 @@ class TestXmlItemExporter(TestBaseItemExporter):
<float>3.14</float>
<boolean>False</boolean>
<number>22</number>
- <time>2015-01-01 01:01:01</time>
+ <time>2015-01-01 01:01:01+00:00</time>
</item>
</items>
""",
diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py
index ff59599fc..6d4e6a4fc 100644
--- a/tests/test_extension_periodic_log.py
+++ b/tests/test_extension_periodic_log.py
@@ -22,7 +22,9 @@ if TYPE_CHECKING:
stats_dump_1 = {
"log_count/INFO": 10,
"log_count/WARNING": 1,
- "start_time": datetime.datetime(2023, 6, 16, 8, 59, 18, 993170),
+ "start_time": datetime.datetime(
+ 2023, 6, 16, 8, 59, 18, 993170, tzinfo=datetime.timezone.utc
+ ),
"scheduler/enqueued/memory": 190,
"scheduler/enqueued": 190,
"scheduler/dequeued/memory": 166,
@@ -43,7 +45,9 @@ stats_dump_1 = {
stats_dump_2 = {
"log_count/INFO": 12,
"log_count/WARNING": 1,
- "start_time": datetime.datetime(2023, 6, 16, 8, 59, 18, 993170),
+ "start_time": datetime.datetime(
+ 2023, 6, 16, 8, 59, 18, 993170, tzinfo=datetime.timezone.utc
+ ),
"scheduler/enqueued/memory": 337,
"scheduler/enqueued": 337,
"scheduler/dequeued/memory": 280,
diff --git a/tests/test_logstats.py b/tests/test_logstats.py
index dc3319aa8..92e88e03f 100644
--- a/tests/test_logstats.py
+++ b/tests/test_logstats.py
@@ -1,6 +1,6 @@
from __future__ import annotations
-from datetime import datetime
+from datetime import datetime, timezone
from typing import TYPE_CHECKING
import pytest
@@ -65,8 +65,12 @@ async def test_stats_calculations(crawler: Crawler, stats: StatsCollector) -> No
assert logstats.itemsprev == 3492
# Simulate when spider closes after running for 30 mins
- stats.set_value("start_time", datetime.fromtimestamp(1655100172))
- stats.set_value("finish_time", datetime.fromtimestamp(1655101972))
+ stats.set_value(
+ "start_time", datetime.fromtimestamp(1655100172).astimezone(timezone.utc)
+ )
+ stats.set_value(
+ "finish_time", datetime.fromtimestamp(1655101972).astimezone(timezone.utc)
+ )
logstats.spider_closed(crawler.spider, "test reason")
assert stats.get_value("responses_per_minute") == 172.9
assert stats.get_value("items_per_minute") == 116.4
@@ -88,8 +92,12 @@ def test_stats_calculation_no_elapsed_time(
) -> None:
"""The stat values should be None since the elapsed time is 0."""
logstats = build_from_crawler(LogStats, crawler)
- stats.set_value("start_time", datetime.fromtimestamp(1655100172))
- stats.set_value("finish_time", datetime.fromtimestamp(1655100172))
+ stats.set_value(
+ "start_time", datetime.fromtimestamp(1655100172).astimezone(timezone.utc)
+ )
+ stats.set_value(
+ "finish_time", datetime.fromtimestamp(1655100172).astimezone(timezone.utc)
+ )
assert crawler.spider
logstats.spider_closed(crawler.spider, "test reason")
assert stats.get_value("responses_per_minute") is None
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 45baa1772..c1d9874cb 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -6,7 +6,7 @@ import random
import re
import time
from abc import ABC, abstractmethod
-from datetime import datetime
+from datetime import datetime, timezone
from ftplib import FTP
from io import BytesIO
from pathlib import Path
@@ -953,7 +953,8 @@ class TestS3FilesStore:
key = "export.csv"
uri = f"s3://{bucket}/{key}"
checksum = "3187896a9657a28163abb31667df64c8"
- last_modified = datetime(2019, 12, 1)
+ # S3FilesStore needs to be fixed to emit tz-aware datetimes
+ last_modified = datetime(2019, 12, 1) # noqa: DTZ001
store = S3FilesStore(uri)
from botocore.stub import Stubber # noqa: PLC0415
@@ -1109,7 +1110,7 @@ class TestGCSFilesStore:
store, bucket, blob = self.build_gcs_files_store()
checksum = "cdcda85605e46d0af6110752770dce3c"
blob.md5_hash = base64.b64encode(bytes.fromhex(checksum)).decode()
- updated = datetime(2019, 12, 1)
+ updated = datetime(2019, 12, 1, tzinfo=timezone.utc)
blob.updated = updated
bucket.get_blob.return_value = blob
stat = await maybe_deferred_to_future(
diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.py
index 630ab5ca4..000c2f9d7 100644
--- a/tests/test_spider_sitemap.py
+++ b/tests/test_spider_sitemap.py
@@ -2,7 +2,7 @@ from __future__ import annotations
import gzip
import re
-from datetime import datetime
+from datetime import datetime, timezone
from io import BytesIO
from logging import WARNING
from pathlib import Path
@@ -169,7 +169,9 @@ Sitemap: /sitemap-relative-url.xml
class FilteredSitemapSpider(self.spider_class): # type: ignore[name-defined,misc]
def sitemap_filter(self, entries):
for entry in entries:
- date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d")
+ date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d").replace(
+ tzinfo=timezone.utc
+ )
if date_time.year > 2008:
yield entry
@@ -240,7 +242,7 @@ Sitemap: /sitemap-relative-url.xml
for entry in entries:
date_time = datetime.strptime(
entry["lastmod"].split("T")[0], "%Y-%m-%d"
- )
+ ).replace(tzinfo=timezone.utc)
if date_time.year > 2004:
yield entry
diff --git a/tests/test_stats.py b/tests/test_stats.py
index 6a762b3ea..4d7d11a06 100644
--- a/tests/test_stats.py
+++ b/tests/test_stats.py
@@ -1,6 +1,6 @@
from __future__ import annotations
-from datetime import datetime
+from datetime import datetime, timezone
from typing import TYPE_CHECKING
from unittest import mock
@@ -39,7 +39,7 @@ class TestCoreStatsExtension:
crawler: Crawler,
spider: Spider,
) -> None:
- fixed_datetime = datetime(2019, 12, 1, 11, 38)
+ fixed_datetime = datetime(2019, 12, 1, 11, 38, tzinfo=timezone.utc)
mock_datetime.now = mock.Mock(return_value=fixed_datetime)
crawler.stats = StatsCollector(crawler)
ext = build_from_crawler(CoreStats, crawler)
diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py
index 57ae5d9e1..bd3442e07 100644
--- a/tests/test_utils_curl.py
+++ b/tests/test_utils_curl.py
@@ -117,14 +117,18 @@ class TestCurlToRequestKwargs:
("Upgrade-Insecure-Requests", "1"),
(
"User-Agent",
- "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML"
- ", like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.32"
- "02.75 Safari/537.36",
+ (
+ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML"
+ ", like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.32"
+ "02.75 Safari/537.36"
+ ),
),
(
"Accept",
- "text/html,application/xhtml+xml,application/xml;q=0.9,ima"
- "ge/webp,image/apng,*/*;q=0.8",
+ (
+ "text/html,application/xhtml+xml,application/xml;q=0.9,ima"
+ "ge/webp,image/apng,*/*;q=0.8"
+ ),
),
("Referer", "http://httpbin.org/"),
("Connection", "keep-alive"),
@@ -176,15 +180,19 @@ class TestCurlToRequestKwargs:
("Upgrade-Insecure-Requests", "1"),
(
"User-Agent",
- "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML"
- ", like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.32"
- "02.75 Safari/537.36",
+ (
+ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML"
+ ", like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.32"
+ "02.75 Safari/537.36"
+ ),
),
("Content-Type", "application/x-www-form-urlencoded"),
(
"Accept",
- "text/html,application/xhtml+xml,application/xml;q=0.9,ima"
- "ge/webp,image/apng,*/*;q=0.8",
+ (
+ "text/html,application/xhtml+xml,application/xml;q=0.9,ima"
+ "ge/webp,image/apng,*/*;q=0.8"
+ ),
),
("Cache-Control", "max-age=0"),
("Referer", "http://httpbin.org/forms/post"),
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 14eea068c..17a80aa80 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -502,6 +502,5 @@ def test_maybe_deferred_coro_deferred() -> None:
# Only the deprecation of maybeDeferred_coro() itself is reported; callables
# that return a Deferred are the reason it exists.
assert [str(record.message) for record in records] == [
- "maybeDeferred_coro() is deprecated and will be removed in a future"
- " Scrapy version."
+ "maybeDeferred_coro() is deprecated and will be removed in a future Scrapy version."
]
diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py
index 7f41b6abb..b04e87f3e 100644
--- a/tests/test_utils_serialize.py
+++ b/tests/test_utils_serialize.py
@@ -19,8 +19,8 @@ class TestJsonEncoder:
return ScrapyJSONEncoder(sort_keys=True)
def test_encode_decode(self, encoder: ScrapyJSONEncoder) -> None:
- dt = datetime.datetime(2010, 1, 2, 10, 11, 12)
- dts = "2010-01-02T10:11:12"
+ dt_naive = datetime.datetime(2010, 1, 2, 10, 11, 12) # noqa: DTZ001
+ dt_naives = "2010-01-02T10:11:12"
dt_aware = datetime.datetime(
2010, 1, 2, 10, 11, 12, 133700, tzinfo=datetime.timezone.utc
)
@@ -35,15 +35,15 @@ class TestJsonEncoder:
decs = "1000.12"
s = {"foo"}
ss = ["foo"]
- dt_set = {dt}
- dt_sets = [dts]
+ dt_set = {dt_naive}
+ dt_sets = [dt_naives]
for input_, output in [
("foo", "foo"),
(d, ds),
(t, ts),
(t_us, t_uss),
- (dt, dts),
+ (dt_naive, dt_naives),
(dt_aware, dt_awares),
(dec, decs),
(["foo", d], ["foo", ds]),
diff --git a/tests/utils/bases/http_request.py b/tests/utils/bases/http_request.py
index 2b712f3ab..b6029e587 100644
--- a/tests/utils/bases/http_request.py
+++ b/tests/utils/bases/http_request.py
@@ -459,14 +459,18 @@ class TestRequestBase(ABC):
b"Accept-Language": [b"en-US,en;q=0.9,ru;q=0.8,es;q=0.7"],
b"Upgrade-Insecure-Requests": [b"1"],
b"User-Agent": [
- b"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537."
- b"36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202"
- b".75 Chrome/62.0.3202.75 Safari/537.36"
+ (
+ b"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537."
+ b"36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202"
+ b".75 Chrome/62.0.3202.75 Safari/537.36"
+ )
],
b"Content-Type": [b"application /x-www-form-urlencoded"],
b"Accept": [
- b"text/html,application/xhtml+xml,application/xml;q=0."
- b"9,image/webp,image/apng,*/*;q=0.8"
+ (
+ b"text/html,application/xhtml+xml,application/xml;q=0."
+ b"9,image/webp,image/apng,*/*;q=0.8"
+ )
],
b"Cache-Control": [b"max-age=0"],
b"Referer": [b"http://httpbin.org/forms/post"],