Scrapy, a fast high-level web crawling & scraping framework for Python.
Bump Ruff to 0.16. (#8182)
Andrey Rakhmatullin4f3e27e6c00f
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yamlindex 1b9e29b30..a7c62346a 100644--- a/.pre-commit-config.yaml+++ b/.pre-commit-config.yaml@@ -6,7 +6,7 @@ exclude: |)repos:- repo: https://github.com/astral-sh/ruff-pre-commit- rev: v0.15.20+ rev: v0.16.8hooks:- id: ruff-checkargs: [ --fix ]diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.pyold mode 100644new mode 100755diff --git a/pyproject.toml b/pyproject.tomlindex a5f6f4907..d7cca1bfd 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -351,6 +351,12 @@ extend-select = ["COM",# pydocstyle"D",+ # pycodestyle+ "E4",+ "E7",+ "E9",+ # Pyflakes+ "F",# flake8-future-annotations"FA",# flynt@@ -405,8 +411,8 @@ extend-select = ["YTT",]ignore = [- # Ones we want to ignore-+ # Do not catch blind exception+ "BLE001",# Trailing comma missing"COM812",# Missing docstring in public module@@ -447,6 +453,8 @@ ignore = ["PLR0911",# Too many arguments in function definition"PLR0913",+ # Too many positional arguments+ "PLR0917",# Magic value used in comparison"PLR2004",# String contains ambiguous {}.@@ -463,6 +471,8 @@ ignore = ["SIM115",# Yoda condition detected"SIM300",+ # Prefer `TypeError` exception for invalid type+ "TRY004",][tool.ruff.lint.flake8-tidy-imports]diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex a4d4fc97f..51ce5b1e8 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -4,7 +4,6 @@ import randomimport warningsfrom collections import dequefrom dataclasses import dataclass, field-from datetime import datetimefrom time import monotonicfrom typing import TYPE_CHECKING, Any@@ -128,7 +127,7 @@ class Slot:f"delay={self.delay:.2f} jitter={self.jitter!r} "f"len(active)={len(self.active)} len(queue)={len(self.queue)} "f"len(transferring)={len(self.transferring)} "- f"lastseen={datetime.fromtimestamp(self.lastseen).isoformat()}>"+ f"lastseen={self.lastseen:.2f}>")diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.pyindex 84dc6216b..57a8e962b 100644--- a/scrapy/core/downloader/handlers/__init__.py+++ b/scrapy/core/downloader/handlers/__init__.py@@ -120,10 +120,9 @@ class DownloadHandlers:self._notconfigured[scheme] = str(ex)return Noneexcept Exception as ex:- logger.error(+ logger.exception('Loading "%(clspath)s" for scheme "%(scheme)s"',{"clspath": path, "scheme": scheme},- exc_info=True,extra={"crawler": self._crawler},)self._notconfigured[scheme] = str(ex)diff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex c4c95c24a..162f03c0c 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -14,7 +14,6 @@ import warningsfrom enum import Enumfrom functools import partialfrom time import time-from traceback import format_excfrom typing import TYPE_CHECKING, Anyfrom twisted.internet.defer import CancelledError, Deferred, inlineCallbacks@@ -391,11 +390,7 @@ class ExecutionEngine:except Exception as exception:self._start = Noneself._start_error = True- exception_traceback = format_exc()- logger.error(- f"Error while reading start items and requests: {exception}.\n{exception_traceback}",- exc_info=True,- )+ logger.exception("Error while reading start items and requests")self.signals.send_catch_log(signal=signals.spider_error,failure=Failure(),@@ -440,9 +435,8 @@ class ExecutionEngine:except Exception:# an error happened, log it and stop the engineself._start_request_processing_awaitable = None- logger.error(+ logger.exception("Error while processing requests from start()",- exc_info=True,extra={"spider": self.spider},)await self.stop_async()@@ -539,10 +533,8 @@ class ExecutionEngine:yield self.scraper.enqueue_scrape(result, request)except Exception:assert self.spider is not None- logger.error(- "Error while enqueuing scrape",- exc_info=True,- extra={"spider": self.spider},+ logger.exception(+ "Error while enqueuing scrape", extra={"spider": self.spider})def spider_is_idle(self) -> bool:@@ -809,30 +801,24 @@ class ExecutionEngine:try:await self._slot.close()except Exception:- logger.error("Slot close failure", exc_info=True, extra={"spider": spider})+ logger.exception("Slot close failure", extra={"spider": spider})try:self.downloader.close()except Exception:- logger.error(- "Downloader close failure", exc_info=True, extra={"spider": spider}- )+ logger.exception("Downloader close failure", extra={"spider": spider})try:await self.scraper.close_spider_async()except Exception:- logger.error(- "Scraper close failure", exc_info=True, extra={"spider": spider}- )+ logger.exception("Scraper close failure", extra={"spider": spider})if hasattr(self._slot.scheduler, "close"):try:if (d := self._slot.scheduler.close(reason)) is not None:await maybe_deferred_to_future(d)except Exception:- logger.error(- "Scheduler close failure", exc_info=True, extra={"spider": spider}- )+ logger.exception("Scheduler close failure", extra={"spider": spider})try:await self.signals.send_catch_log_async(@@ -841,10 +827,8 @@ class ExecutionEngine:reason=reason,)except Exception:- logger.error(- "Error while sending spider_close signal",- exc_info=True,- extra={"spider": spider},+ logger.exception(+ "Error while sending spider_close signal", extra={"spider": spider})try:diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex b70f57435..bfdda63ca 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -227,10 +227,9 @@ class Scraper:try:yield dfd # fired in _wait_for_processing()except Exception:- logger.error(+ logger.exception("Scraper bug processing %(request)s",{"request": request},- exc_info=True,extra={"spider": self.crawler.spider},)finally:diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.pyindex 46d5a7b76..2d73c3b48 100644--- a/scrapy/downloadermiddlewares/robotstxt.py+++ b/scrapy/downloadermiddlewares/robotstxt.py@@ -101,10 +101,8 @@ class RobotsTxtMiddleware:await self._parse_robots(resp, netloc, request)except Exception as e:if not isinstance(e, IgnoreRequest):- logger.error(- "Error downloading %(request)s: %(f_exception)s",- {"request": request, "f_exception": e},- exc_info=True,+ logger.exception(+ f"Error downloading {request}",extra={"spider": self.crawler.spider},)self._robots_error(e, netloc)diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex c2ab1581b..127becab7 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -627,10 +627,9 @@ class FeedExporter:try:await ensure_awaitable(slot.storage.store(self._get_file(slot)))except Exception:- logger.error(+ logger.exception("Error storing %s",logmsg,- exc_info=True,extra={"spider": spider},)self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}")diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex dda66b18f..8e49f52e9 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -678,11 +678,10 @@ class FilesPipeline(MediaPipeline):)raiseexcept Exception as exc:- logger.error(+ logger.exception("File (unknown-error): Error processing file from %(request)s ""referred in <%(referer)s>",{"request": request, "referer": referer},- exc_info=True,extra={"spider": info.spider},)raise _FileException(str(exc)) from excdiff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.pyindex 70e1de505..289df734f 100644--- a/scrapy/utils/signal.py+++ b/scrapy/utils/signal.py@@ -63,10 +63,9 @@ def send_catch_log(result = Failure()except Exception:result = Failure()- logger.error(+ logger.exception("Error caught on signal handler: %(receiver)s",{"receiver": receiver},- exc_info=True,extra={"spider": spider},)else:@@ -199,10 +198,9 @@ async def _send_catch_log_asyncio(except dont_log as ex: # pylint: disable=catching-non-exceptionresult = exexcept Exception as ex:- logger.error(+ logger.exception("Error caught on signal handler: %(receiver)s",{"receiver": receiver},- exc_info=True,extra={"spider": spider},)result = exdiff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.pyindex ae340093b..1d2d18d0f 100644--- a/tests/test_downloader_handler_twisted_http2.py+++ b/tests/test_downloader_handler_twisted_http2.py@@ -149,9 +149,11 @@ class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase):assert ("scrapy.core._http2.stream",logging.WARNING,- f"Ignoring bad Content-Length header "- f"{bad_content_length!r} of request {request}, sending "- f"{actual_content_length!r} instead",+ (+ f"Ignoring bad Content-Length header "+ f"{bad_content_length!r} of request {request}, sending "+ f"{actual_content_length!r} instead"+ ),) in caplog.record_tuples@coroutine_testdiff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.pyindex 4800e4d29..36d2321d5 100644--- a/tests/test_downloadermiddleware_cookies.py+++ b/tests/test_downloadermiddleware_cookies.py@@ -133,14 +133,18 @@ class TestCookiesMiddleware:("scrapy.downloadermiddlewares.cookies",logging.DEBUG,- "Received cookies from: <200 http://scrapytest.org/>\n"- "Set-Cookie: C1=value1; path=/\n",+ (+ "Received cookies from: <200 http://scrapytest.org/>\n"+ "Set-Cookie: C1=value1; path=/\n"+ ),),("scrapy.downloadermiddlewares.cookies",logging.DEBUG,- "Sending cookies to: <GET http://scrapytest.org/sub1/>\n"- "Cookie: C1=value1\n",+ (+ "Sending cookies to: <GET http://scrapytest.org/sub1/>\n"+ "Cookie: C1=value1\n"+ ),),]@@ -466,20 +470,26 @@ class TestCookiesMiddleware:("scrapy.utils.request",logging.WARNING,- "Invalid cookie found in request <GET http://example.org/1>:"- " {'value': 'bar', 'secure': False} ('name' is missing)",+ (+ "Invalid cookie found in request <GET http://example.org/1>:"+ " {'value': 'bar', 'secure': False} ('name' is missing)"+ ),),("scrapy.utils.request",logging.WARNING,- "Invalid cookie found in request <GET http://example.org/2>:"- " {'name': 'foo', 'secure': False} ('value' is missing)",+ (+ "Invalid cookie found in request <GET http://example.org/2>:"+ " {'name': 'foo', 'secure': False} ('value' is missing)"+ ),),("scrapy.utils.request",logging.WARNING,- "Invalid cookie found in request <GET http://example.org/3>:"- " {'name': 'foo', 'value': None, 'secure': False} ('value' is missing)",+ (+ "Invalid cookie found in request <GET http://example.org/3>:"+ " {'name': 'foo', 'value': None, 'secure': False} ('value' is missing)"+ ),),]self.assertCookieValEqual(req1.headers["Cookie"], "key=value1")diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.pyindex a42bbc715..a9dc1525a 100644--- a/tests/test_downloadermiddleware_retry.py+++ b/tests/test_downloadermiddleware_retry.py@@ -318,8 +318,10 @@ class TestGetRetryRequest:assert ("scrapy.downloadermiddlewares.retry",logging.DEBUG,- f"Retrying {request} (failed {expected_retry_times} times): "- f"{expected_reason}",+ (+ f"Retrying {request} (failed {expected_retry_times} times): "+ f"{expected_reason}"+ ),) in caplog.record_tuplesdef test_max_retries_reached(self, caplog: pytest.LogCaptureFixture) -> None:@@ -339,8 +341,10 @@ class TestGetRetryRequest:assert ("scrapy.downloadermiddlewares.retry",logging.ERROR,- f"Gave up retrying {request} (failed {failure_count} times): "- f"{expected_reason}",+ (+ f"Gave up retrying {request} (failed {failure_count} times): "+ f"{expected_reason}"+ ),) in caplog.record_tuplesdef test_one_retry(self, caplog: pytest.LogCaptureFixture) -> None:@@ -364,8 +368,10 @@ class TestGetRetryRequest:assert ("scrapy.downloadermiddlewares.retry",logging.DEBUG,- f"Retrying {request} (failed {expected_retry_times} times): "- f"{expected_reason}",+ (+ f"Retrying {request} (failed {expected_retry_times} times): "+ f"{expected_reason}"+ ),) in caplog.record_tuplesdef test_two_retries(self, caplog: pytest.LogCaptureFixture) -> None:@@ -397,8 +403,10 @@ class TestGetRetryRequest:assert ("scrapy.downloadermiddlewares.retry",logging.DEBUG,- f"Retrying {request} (failed {expected_retry_times} times): "- f"{expected_reason}",+ (+ f"Retrying {request} (failed {expected_retry_times} times): "+ f"{expected_reason}"+ ),) in caplog.record_tuplescaplog.clear()@@ -418,8 +426,10 @@ class TestGetRetryRequest:assert ("scrapy.downloadermiddlewares.retry",logging.ERROR,- f"Gave up retrying {request} (failed {failure_count} times): "- f"{expected_reason}",+ (+ f"Gave up retrying {request} (failed {failure_count} times): "+ f"{expected_reason}"+ ),) in caplog.record_tuplesdef test_no_spider(self):@@ -520,8 +530,10 @@ class TestGetRetryRequest:assert ("scrapy.downloadermiddlewares.retry",logging.DEBUG,- f"Retrying {request} (failed {expected_retry_times} times): "- f"{expected_reason}",+ (+ f"Retrying {request} (failed {expected_retry_times} times): "+ f"{expected_reason}"+ ),) in caplog.record_tuplesdef test_reason_builtin_exception(self, caplog: pytest.LogCaptureFixture) -> None:@@ -543,8 +555,10 @@ class TestGetRetryRequest:assert ("scrapy.downloadermiddlewares.retry",logging.DEBUG,- f"Retrying {request} (failed {expected_retry_times} times): "- f"{expected_reason}",+ (+ f"Retrying {request} (failed {expected_retry_times} times): "+ f"{expected_reason}"+ ),) in caplog.record_tuplesdef test_reason_builtin_exception_class(@@ -568,8 +582,10 @@ class TestGetRetryRequest:assert ("scrapy.downloadermiddlewares.retry",logging.DEBUG,- f"Retrying {request} (failed {expected_retry_times} times): "- f"{expected_reason}",+ (+ f"Retrying {request} (failed {expected_retry_times} times): "+ f"{expected_reason}"+ ),) in caplog.record_tuplesdef test_reason_custom_exception(self, caplog: pytest.LogCaptureFixture) -> None:@@ -591,8 +607,10 @@ class TestGetRetryRequest:assert ("scrapy.downloadermiddlewares.retry",logging.DEBUG,- f"Retrying {request} (failed {expected_retry_times} times): "- f"{expected_reason}",+ (+ f"Retrying {request} (failed {expected_retry_times} times): "+ f"{expected_reason}"+ ),) in caplog.record_tuplesdef test_reason_custom_exception_class(@@ -616,8 +634,10 @@ class TestGetRetryRequest:assert ("scrapy.downloadermiddlewares.retry",logging.DEBUG,- f"Retrying {request} (failed {expected_retry_times} times): "- f"{expected_reason}",+ (+ f"Retrying {request} (failed {expected_retry_times} times): "+ f"{expected_reason}"+ ),) in caplog.record_tuplesdef test_custom_logger(self, caplog: pytest.LogCaptureFixture) -> None:diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.pyindex 63c5565e5..cd68bb8e0 100644--- a/tests/test_dupefilters.py+++ b/tests/test_dupefilters.py@@ -186,8 +186,10 @@ class TestRFPDupeFilter:assert ("scrapy.dupefilters",logging.DEBUG,- "Filtered duplicate request: <GET http://scrapytest.org/index.html> - no more"- " duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)",+ (+ "Filtered duplicate request: <GET http://scrapytest.org/index.html> - no more"+ " duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)"+ ),) in caplog.record_tuplesdupefilter.close("finished")@@ -224,8 +226,10 @@ class TestRFPDupeFilter:assert ("scrapy.dupefilters",logging.DEBUG,- "Filtered duplicate request: <GET http://scrapytest.org/index.html>"- " (referer: http://scrapytest.org/INDEX.html)",+ (+ "Filtered duplicate request: <GET http://scrapytest.org/index.html>"+ " (referer: http://scrapytest.org/INDEX.html)"+ ),) in caplog.record_tuplesdupefilter.close("finished")diff --git a/tests/test_exporters.py b/tests/test_exporters.pyindex 4359f4ff4..d74556236 100644--- a/tests/test_exporters.py+++ b/tests/test_exporters.py@@ -5,7 +5,7 @@ import pickleimport refrom abc import ABC, abstractmethodfrom collections.abc import Mapping-from datetime import datetime+from datetime import datetime, timezonefrom io import BytesIOfrom typing import Any@@ -75,11 +75,12 @@ class TestBaseItemExporter(ABC):exported_dict[k] = to_unicode(v)assert self.i == self.item_class(**exported_dict)- def _get_nonstring_types_item(self) -> dict[str, Any]:+ @staticmethod+ def _get_nonstring_types_item() -> dict[str, Any]:return {"boolean": False,"number": 22,- "time": datetime(2015, 1, 1, 1, 1, 1),+ "time": datetime(2015, 1, 1, 1, 1, 1, tzinfo=timezone.utc),"float": 3.14,}@@ -383,7 +384,7 @@ class TestCsvItemExporter(TestBaseItemExporter):self.assertExportResult(item=self._get_nonstring_types_item(),include_headers_line=False,- expected="22,False,3.14,2015-01-01 01:01:01\r\n",+ expected="22,False,3.14,2015-01-01 01:01:01+00:00\r\n",)def test_errors_default(self):@@ -528,7 +529,7 @@ class TestXmlItemExporter(TestBaseItemExporter):<float>3.14</float><boolean>False</boolean><number>22</number>- <time>2015-01-01 01:01:01</time>+ <time>2015-01-01 01:01:01+00:00</time></item></items>""",diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.pyindex ff59599fc..6d4e6a4fc 100644--- a/tests/test_extension_periodic_log.py+++ b/tests/test_extension_periodic_log.py@@ -22,7 +22,9 @@ if TYPE_CHECKING:stats_dump_1 = {"log_count/INFO": 10,"log_count/WARNING": 1,- "start_time": datetime.datetime(2023, 6, 16, 8, 59, 18, 993170),+ "start_time": datetime.datetime(+ 2023, 6, 16, 8, 59, 18, 993170, tzinfo=datetime.timezone.utc+ ),"scheduler/enqueued/memory": 190,"scheduler/enqueued": 190,"scheduler/dequeued/memory": 166,@@ -43,7 +45,9 @@ stats_dump_1 = {stats_dump_2 = {"log_count/INFO": 12,"log_count/WARNING": 1,- "start_time": datetime.datetime(2023, 6, 16, 8, 59, 18, 993170),+ "start_time": datetime.datetime(+ 2023, 6, 16, 8, 59, 18, 993170, tzinfo=datetime.timezone.utc+ ),"scheduler/enqueued/memory": 337,"scheduler/enqueued": 337,"scheduler/dequeued/memory": 280,diff --git a/tests/test_logstats.py b/tests/test_logstats.pyindex dc3319aa8..92e88e03f 100644--- a/tests/test_logstats.py+++ b/tests/test_logstats.py@@ -1,6 +1,6 @@from __future__ import annotations-from datetime import datetime+from datetime import datetime, timezonefrom typing import TYPE_CHECKINGimport pytest@@ -65,8 +65,12 @@ async def test_stats_calculations(crawler: Crawler, stats: StatsCollector) -> Noassert logstats.itemsprev == 3492# Simulate when spider closes after running for 30 mins- stats.set_value("start_time", datetime.fromtimestamp(1655100172))- stats.set_value("finish_time", datetime.fromtimestamp(1655101972))+ stats.set_value(+ "start_time", datetime.fromtimestamp(1655100172).astimezone(timezone.utc)+ )+ stats.set_value(+ "finish_time", datetime.fromtimestamp(1655101972).astimezone(timezone.utc)+ )logstats.spider_closed(crawler.spider, "test reason")assert stats.get_value("responses_per_minute") == 172.9assert stats.get_value("items_per_minute") == 116.4@@ -88,8 +92,12 @@ def test_stats_calculation_no_elapsed_time() -> None:"""The stat values should be None since the elapsed time is 0."""logstats = build_from_crawler(LogStats, crawler)- stats.set_value("start_time", datetime.fromtimestamp(1655100172))- stats.set_value("finish_time", datetime.fromtimestamp(1655100172))+ stats.set_value(+ "start_time", datetime.fromtimestamp(1655100172).astimezone(timezone.utc)+ )+ stats.set_value(+ "finish_time", datetime.fromtimestamp(1655100172).astimezone(timezone.utc)+ )assert crawler.spiderlogstats.spider_closed(crawler.spider, "test reason")assert stats.get_value("responses_per_minute") is Nonediff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex 45baa1772..c1d9874cb 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -6,7 +6,7 @@ import randomimport reimport timefrom abc import ABC, abstractmethod-from datetime import datetime+from datetime import datetime, timezonefrom ftplib import FTPfrom io import BytesIOfrom pathlib import Path@@ -953,7 +953,8 @@ class TestS3FilesStore:key = "export.csv"uri = f"s3://{bucket}/{key}"checksum = "3187896a9657a28163abb31667df64c8"- last_modified = datetime(2019, 12, 1)+ # S3FilesStore needs to be fixed to emit tz-aware datetimes+ last_modified = datetime(2019, 12, 1) # noqa: DTZ001store = S3FilesStore(uri)from botocore.stub import Stubber # noqa: PLC0415@@ -1109,7 +1110,7 @@ class TestGCSFilesStore:store, bucket, blob = self.build_gcs_files_store()checksum = "cdcda85605e46d0af6110752770dce3c"blob.md5_hash = base64.b64encode(bytes.fromhex(checksum)).decode()- updated = datetime(2019, 12, 1)+ updated = datetime(2019, 12, 1, tzinfo=timezone.utc)blob.updated = updatedbucket.get_blob.return_value = blobstat = await maybe_deferred_to_future(diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.pyindex 630ab5ca4..000c2f9d7 100644--- a/tests/test_spider_sitemap.py+++ b/tests/test_spider_sitemap.py@@ -2,7 +2,7 @@ from __future__ import annotationsimport gzipimport re-from datetime import datetime+from datetime import datetime, timezonefrom io import BytesIOfrom logging import WARNINGfrom pathlib import Path@@ -169,7 +169,9 @@ Sitemap: /sitemap-relative-url.xmlclass FilteredSitemapSpider(self.spider_class): # type: ignore[name-defined,misc]def sitemap_filter(self, entries):for entry in entries:- date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d")+ date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d").replace(+ tzinfo=timezone.utc+ )if date_time.year > 2008:yield entry@@ -240,7 +242,7 @@ Sitemap: /sitemap-relative-url.xmlfor entry in entries:date_time = datetime.strptime(entry["lastmod"].split("T")[0], "%Y-%m-%d"- )+ ).replace(tzinfo=timezone.utc)if date_time.year > 2004:yield entrydiff --git a/tests/test_stats.py b/tests/test_stats.pyindex 6a762b3ea..4d7d11a06 100644--- a/tests/test_stats.py+++ b/tests/test_stats.py@@ -1,6 +1,6 @@from __future__ import annotations-from datetime import datetime+from datetime import datetime, timezonefrom typing import TYPE_CHECKINGfrom unittest import mock@@ -39,7 +39,7 @@ class TestCoreStatsExtension:crawler: Crawler,spider: Spider,) -> None:- fixed_datetime = datetime(2019, 12, 1, 11, 38)+ fixed_datetime = datetime(2019, 12, 1, 11, 38, tzinfo=timezone.utc)mock_datetime.now = mock.Mock(return_value=fixed_datetime)crawler.stats = StatsCollector(crawler)ext = build_from_crawler(CoreStats, crawler)diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.pyindex 57ae5d9e1..bd3442e07 100644--- a/tests/test_utils_curl.py+++ b/tests/test_utils_curl.py@@ -117,14 +117,18 @@ class TestCurlToRequestKwargs:("Upgrade-Insecure-Requests", "1"),("User-Agent",- "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML"- ", like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.32"- "02.75 Safari/537.36",+ (+ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML"+ ", like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.32"+ "02.75 Safari/537.36"+ ),),("Accept",- "text/html,application/xhtml+xml,application/xml;q=0.9,ima"- "ge/webp,image/apng,*/*;q=0.8",+ (+ "text/html,application/xhtml+xml,application/xml;q=0.9,ima"+ "ge/webp,image/apng,*/*;q=0.8"+ ),),("Referer", "http://httpbin.org/"),("Connection", "keep-alive"),@@ -176,15 +180,19 @@ class TestCurlToRequestKwargs:("Upgrade-Insecure-Requests", "1"),("User-Agent",- "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML"- ", like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.32"- "02.75 Safari/537.36",+ (+ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML"+ ", like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.32"+ "02.75 Safari/537.36"+ ),),("Content-Type", "application/x-www-form-urlencoded"),("Accept",- "text/html,application/xhtml+xml,application/xml;q=0.9,ima"- "ge/webp,image/apng,*/*;q=0.8",+ (+ "text/html,application/xhtml+xml,application/xml;q=0.9,ima"+ "ge/webp,image/apng,*/*;q=0.8"+ ),),("Cache-Control", "max-age=0"),("Referer", "http://httpbin.org/forms/post"),diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.pyindex 14eea068c..17a80aa80 100644--- a/tests/test_utils_defer.py+++ b/tests/test_utils_defer.py@@ -502,6 +502,5 @@ def test_maybe_deferred_coro_deferred() -> None:# Only the deprecation of maybeDeferred_coro() itself is reported; callables# that return a Deferred are the reason it exists.assert [str(record.message) for record in records] == [- "maybeDeferred_coro() is deprecated and will be removed in a future"- " Scrapy version."+ "maybeDeferred_coro() is deprecated and will be removed in a future Scrapy version."]diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.pyindex 7f41b6abb..b04e87f3e 100644--- a/tests/test_utils_serialize.py+++ b/tests/test_utils_serialize.py@@ -19,8 +19,8 @@ class TestJsonEncoder:return ScrapyJSONEncoder(sort_keys=True)def test_encode_decode(self, encoder: ScrapyJSONEncoder) -> None:- dt = datetime.datetime(2010, 1, 2, 10, 11, 12)- dts = "2010-01-02T10:11:12"+ dt_naive = datetime.datetime(2010, 1, 2, 10, 11, 12) # noqa: DTZ001+ dt_naives = "2010-01-02T10:11:12"dt_aware = datetime.datetime(2010, 1, 2, 10, 11, 12, 133700, tzinfo=datetime.timezone.utc)@@ -35,15 +35,15 @@ class TestJsonEncoder:decs = "1000.12"s = {"foo"}ss = ["foo"]- dt_set = {dt}- dt_sets = [dts]+ dt_set = {dt_naive}+ dt_sets = [dt_naives]for input_, output in [("foo", "foo"),(d, ds),(t, ts),(t_us, t_uss),- (dt, dts),+ (dt_naive, dt_naives),(dt_aware, dt_awares),(dec, decs),(["foo", d], ["foo", ds]),diff --git a/tests/utils/bases/http_request.py b/tests/utils/bases/http_request.pyindex 2b712f3ab..b6029e587 100644--- a/tests/utils/bases/http_request.py+++ b/tests/utils/bases/http_request.py@@ -459,14 +459,18 @@ class TestRequestBase(ABC):b"Accept-Language": [b"en-US,en;q=0.9,ru;q=0.8,es;q=0.7"],b"Upgrade-Insecure-Requests": [b"1"],b"User-Agent": [- b"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537."- b"36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202"- b".75 Chrome/62.0.3202.75 Safari/537.36"+ (+ b"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537."+ b"36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202"+ b".75 Chrome/62.0.3202.75 Safari/537.36"+ )],b"Content-Type": [b"application /x-www-form-urlencoded"],b"Accept": [- b"text/html,application/xhtml+xml,application/xml;q=0."- b"9,image/webp,image/apng,*/*;q=0.8"+ (+ b"text/html,application/xhtml+xml,application/xml;q=0."+ b"9,image/webp,image/apng,*/*;q=0.8"+ )],b"Cache-Control": [b"max-age=0"],b"Referer": [b"http://httpbin.org/forms/post"],