Scrapy, a fast high-level web crawling & scraping framework for Python.
Update tool versions (#7127)
Andrey Rakhmatullin804ae167dfaf
diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revsindex a9fc3dd68..1f062eef2 100644--- a/.git-blame-ignore-revs+++ b/.git-blame-ignore-revs@@ -4,4 +4,4 @@ e211ec0aa26ecae0da8ae55d064ea60e1efe4d0d# reapplying black to the code with default line length303f0a70fcf8067adf0a909c2096a5009162383a# reapplying black again and removing line length on pre-commit black config-c5cdd0d30ceb68ccba04af0e71d1b8e6678e2962\ No newline at end of file+c5cdd0d30ceb68ccba04af0e71d1b8e6678e2962diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.ymlindex ee5f4b1f9..101c6648e 100644--- a/.github/workflows/checks.yml+++ b/.github/workflows/checks.yml@@ -34,10 +34,10 @@ jobs:TOXENV: twinechecksteps:- - uses: actions/checkout@v4+ - uses: actions/checkout@v5- name: Set up Python ${{ matrix.python-version }}- uses: actions/setup-python@v5+ uses: actions/setup-python@v6with:python-version: ${{ matrix.python-version }}@@ -50,5 +50,5 @@ jobs:pre-commit:runs-on: ubuntu-lateststeps:- - uses: actions/checkout@v4+ - uses: actions/checkout@v5- uses: pre-commit/[email protected]diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.ymlindex d1589f4f7..9894c8e7c 100644--- a/.github/workflows/publish.yml+++ b/.github/workflows/publish.yml@@ -18,8 +18,8 @@ jobs:permissions:id-token: writesteps:- - uses: actions/checkout@v4- - uses: actions/setup-python@v5+ - uses: actions/checkout@v5+ - uses: actions/setup-python@v6with:python-version: "3.13"- run: |diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.ymlindex 0e9e108cc..f061b1892 100644--- a/.github/workflows/tests-macos.yml+++ b/.github/workflows/tests-macos.yml@@ -19,10 +19,10 @@ jobs:python-version: ["3.10", "3.11", "3.12", "3.13"]steps:- - uses: actions/checkout@v4+ - uses: actions/checkout@v5- name: Set up Python ${{ matrix.python-version }}- uses: actions/setup-python@v5+ uses: actions/setup-python@v6with:python-version: ${{ matrix.python-version }}diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.ymlindex 0a5852b6c..4aa76e6c6 100644--- a/.github/workflows/tests-ubuntu.yml+++ b/.github/workflows/tests-ubuntu.yml@@ -64,10 +64,10 @@ jobs:TOXENV: botocoresteps:- - uses: actions/checkout@v4+ - uses: actions/checkout@v5- name: Set up Python ${{ matrix.python-version }}- uses: actions/setup-python@v5+ uses: actions/setup-python@v6with:python-version: ${{ matrix.python-version }}diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.ymlindex 4da82fd0b..73b87f8ed 100644--- a/.github/workflows/tests-windows.yml+++ b/.github/workflows/tests-windows.yml@@ -46,10 +46,10 @@ jobs:TOXENV: extra-depssteps:- - uses: actions/checkout@v4+ - uses: actions/checkout@v5- name: Set up Python ${{ matrix.python-version }}- uses: actions/setup-python@v5+ uses: actions/setup-python@v6with:python-version: ${{ matrix.python-version }}diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yamlindex bcc10d5e8..5ffe2a8ea 100644--- a/.pre-commit-config.yaml+++ b/.pre-commit-config.yaml@@ -1,19 +1,26 @@+exclude: |+ (?x)(+ ^docs/_static|+ ^docs/_tests|+ ^tests/sample_data+ )repos:- repo: https://github.com/astral-sh/ruff-pre-commit- rev: v0.12.2+ rev: v0.14.2hooks:- id: ruff-checkargs: [ --fix ]- id: ruff-format- repo: https://github.com/adamchainz/blacken-docs- rev: 1.19.1+ rev: 1.20.0hooks:- id: blacken-docsadditional_dependencies:- - black==25.1.0+ - black==25.9.0- repo: https://github.com/pre-commit/pre-commit-hooks- rev: v5.0.0+ rev: v6.0.0hooks:+ - id: end-of-file-fixer- id: trailing-whitespace- repo: https://github.com/sphinx-contrib/sphinx-lintrev: v1.0.0diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.pyindex 4ceb003c7..776afaa6c 100644--- a/docs/_ext/scrapydocs.py+++ b/docs/_ext/scrapydocs.py@@ -29,14 +29,14 @@ def is_setting_index(node: Node) -> bool:if node.tagname == "index" and node["entries"]: # type: ignore[index,attr-defined]# index entries for setting directives look like:# [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')]- entry_type, info, refid = node["entries"][0][:3] # type: ignore[index]+ entry_type, info, _ = node["entries"][0][:3] # type: ignore[index]return entry_type == "pair" and info.endswith("; setting")return Falsedef get_setting_name_and_refid(node: Node) -> tuple[str, str]:"""Extract setting name from directive index node"""- entry_type, info, refid = node["entries"][0][:3] # type: ignore[index]+ _, info, refid = node["entries"][0][:3] # type: ignore[index]return info.replace("; setting", ""), refiddiff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rstindex 3d3f04793..05dffcdda 100644--- a/docs/topics/developer-tools.rst+++ b/docs/topics/developer-tools.rst@@ -317,4 +317,3 @@ to identifying the correct request and replicating it in your spider... _quotes.toscrape.com/scroll: https://quotes.toscrape.com/scroll.. _quotes.toscrape.com/api/quotes?page=10: https://quotes.toscrape.com/api/quotes?page=10.. _has-class-extension: https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions-diff --git a/docs/topics/stats.rst b/docs/topics/stats.rstindex e34999b58..ac714eacb 100644--- a/docs/topics/stats.rst+++ b/docs/topics/stats.rst@@ -121,4 +121,3 @@ DummyStatsCollectorsetting, to disable stats collect in order to improve performance. However,the performance penalty of stats collection is usually marginal compared toother Scrapy workload like parsing pages.-diff --git a/docs/versioning.rst b/docs/versioning.rstindex 9d02757b0..aab6af865 100644--- a/docs/versioning.rst+++ b/docs/versioning.rst@@ -66,4 +66,3 @@ the :ref:`release notes <news>`... _odd-numbered versions for development releases: https://en.wikipedia.org/wiki/Software_versioning#Odd-numbered_versions_for_development_releases-diff --git a/pyproject.toml b/pyproject.tomlindex 055faa493..4770ac11d 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -374,8 +374,6 @@ ignore = ["SIM115",# Yoda condition detected"SIM300",- # removed in recent ruff- "UP038",# Ones that we may want to address (fix, ignore per-line or move to "don't want to fix")diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.pyindex 74fd0e354..75d6c9bb0 100644--- a/scrapy/utils/spider.py+++ b/scrapy/utils/spider.py@@ -2,7 +2,7 @@ from __future__ import annotationsimport inspectimport logging-from typing import TYPE_CHECKING, Any, Literal, TypeVar, overload+from typing import TYPE_CHECKING, Any, TypeVar, overloadfrom scrapy.spiders import Spiderfrom scrapy.utils.defer import deferred_from_coro@@ -76,7 +76,7 @@ def spidercls_for_request(def spidercls_for_request(spider_loader: SpiderLoaderProtocol,request: Request,- default_spidercls: Literal[None],+ default_spidercls: None,log_none: bool = ...,log_multiple: bool = ...,) -> type[Spider] | None: ...diff --git a/tests/keys/example-com.gen.README b/tests/keys/example-com.gen.READMEindex 955e7b057..026226b2c 100644--- a/tests/keys/example-com.gen.README+++ b/tests/keys/example-com.gen.README@@ -21,4 +21,3 @@ Locality Name (eg, city) [New York]:The InternetOrganization Name (eg, company) [Example, LLC]:ScrapyCommon Name (e.g. server FQDN or YOUR name) [Example Company]:www.example.comEmail Address [[email protected]]:-diff --git a/tests/keys/localhost-ip.gen.README b/tests/keys/localhost-ip.gen.READMEindex 8e94e1217..6424ad83e 100644--- a/tests/keys/localhost-ip.gen.README+++ b/tests/keys/localhost-ip.gen.README@@ -18,4 +18,3 @@ Organization Name (eg, company) [Internet Widgits Pty Ltd]:ScrapyOrganizational Unit Name (eg, section) []:.Common Name (e.g. server FQDN or YOUR name) []:127.0.0.1Email Address []:.-diff --git a/tests/keys/localhost.gen.README b/tests/keys/localhost.gen.READMEindex 19c29a725..65f3a2d5c 100644--- a/tests/keys/localhost.gen.README+++ b/tests/keys/localhost.gen.README@@ -18,4 +18,3 @@ Organization Name (eg, company) [Internet Widgits Pty Ltd]:ScrapyOrganizational Unit Name (eg, section) []:.Common Name (e.g. server FQDN or YOUR name) []:localhostEmail Address []:.-diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.pyindex c6fdb13ea..2ad8fb8a4 100644--- a/tests/test_cmdline_crawl_with_pipeline/__init__.py+++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py@@ -14,7 +14,7 @@ class TestCmdlineCrawlPipeline:return proc.returncode, stderrdef test_open_spider_normally_in_pipeline(self):- returncode, stderr = self._execute("normal")+ returncode, _ = self._execute("normal")assert returncode == 0def test_exception_at_open_spider_in_pipeline(self):diff --git a/tests/test_command_genspider.py b/tests/test_command_genspider.pyindex 9e4d90cb0..fd9505060 100644--- a/tests/test_command_genspider.py+++ b/tests/test_command_genspider.py@@ -172,7 +172,7 @@ class TestGenspiderStandaloneCommand:def test_same_name_as_existing_file(self, force: bool, tmp_path: Path) -> None:file_name = "example"file_path = Path(tmp_path, file_name + ".py")- p, out, err = proc("genspider", file_name, "example.com", cwd=tmp_path)+ _, out, _ = proc("genspider", file_name, "example.com", cwd=tmp_path)assert f"Created spider {file_name!r} using template 'basic' " in outassert file_path.exists()modify_time_before = file_path.stat().st_mtime@@ -180,7 +180,7 @@ class TestGenspiderStandaloneCommand:if force:# use different template to ensure contents were changed- p, out, err = proc(+ _, out, _ = proc("genspider","--force","-t",diff --git a/tests/test_command_startproject.py b/tests/test_command_startproject.pyindex fe3c9bcbe..4eed09fcd 100644--- a/tests/test_command_startproject.py+++ b/tests/test_command_startproject.py@@ -232,7 +232,7 @@ class TestStartprojectTemplates:project_dir.mkdir(parents=True)existing_nodes = {- oct(permissions)[2:] + extension: permissions+ f"{permissions:o}{extension}": permissionsfor extension in ("", ".d")for permissions in (0o444,diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex ecd784c4c..dc85d9d68 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -612,7 +612,7 @@ class TestCrawlSpider:@pytest.mark.only_asyncio@deferred_f_from_coro_fasync def test_async_def_deferred_wrapped(self):- log, items, _ = await self._run_spider(AsyncDefDeferredWrappedSpider)+ _, items, _ = await self._run_spider(AsyncDefDeferredWrappedSpider)assert items == [{"code": 200}]@deferred_f_from_coro_fdiff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex 2be3b49b3..4f1045786 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -785,7 +785,7 @@ class ScriptRunnerMixin(ABC):stdout=subprocess.PIPE,stderr=subprocess.PIPE,)- stdout, stderr = p.communicate()+ _, stderr = p.communicate()return stderr.decode("utf-8")diff --git a/tests/test_extension_statsmailer.py b/tests/test_extension_statsmailer.pyindex b6322325c..d2e4d5c00 100644--- a/tests/test_extension_statsmailer.py+++ b/tests/test_extension_statsmailer.py@@ -74,7 +74,7 @@ def test_spider_closed_sends_email(dummy_stats):spider = DefaultSpider(name="dummy")ext.spider_closed(spider)- args, kwargs = mail.send.call_args+ args, _ = mail.send.call_argsto, subject, body = argsassert to == recipientsassert "Scrapy stats for: dummy" in subjectdiff --git a/tests/test_http_headers.py b/tests/test_http_headers.pyindex 2fcf9e83c..243aa6afe 100644--- a/tests/test_http_headers.py+++ b/tests/test_http_headers.py@@ -56,12 +56,12 @@ class TestHeaders:def test_encode_latin1(self):h = Headers({"key": "\xa3"}, encoding="latin1")- key, val = dict(h).popitem()+ _, val = dict(h).popitem()assert val[0] == b"\xa3"def test_encode_multiple(self):h = Headers({"key": ["\xa3"]}, encoding="utf-8")- key, val = dict(h).popitem()+ _, val = dict(h).popitem()assert val[0] == b"\xc2\xa3"def test_delete_and_contains(self):diff --git a/tests/test_http_request.py b/tests/test_http_request.pyindex 3a62bf716..22f44bfd2 100644--- a/tests/test_http_request.py+++ b/tests/test_http_request.py@@ -881,7 +881,7 @@ class TestFormRequest(TestRequest):)with pytest.raises(ValueError,- match="Multiple elements found .* matching the criteria in clickdata",+ match=r"Multiple elements found .* matching the criteria in clickdata",):self.request_class.from_response(response, clickdata={"type": "submit"})diff --git a/tests/test_http_response.py b/tests/test_http_response.pyindex ac321c1ad..8acdf25ec 100644--- a/tests/test_http_response.py+++ b/tests/test_http_response.py@@ -813,7 +813,7 @@ class TestTextResponse(TestResponseBase):text_body = b"""<html><body>text</body></html>"""text_response = self.response_class("http://www.example.com", body=text_body)with pytest.raises(- ValueError, match="(Expecting value|Unexpected '<'): line 1"+ ValueError, match=r"(Expecting value|Unexpected '<'): line 1"):text_response.json()diff --git a/tests/test_item.py b/tests/test_item.pyindex bf51eb398..94742bfed 100644--- a/tests/test_item.py+++ b/tests/test_item.py@@ -284,9 +284,9 @@ class TestItemMeta:(first_call, second_call) = new_mock.call_args_list[-2:]- mcs, class_name, bases, attrs = first_call[0]+ *_, attrs = first_call[0]assert "__classcell__" not in attrs- mcs, class_name, bases, attrs = second_call[0]+ *_, attrs = second_call[0]assert "__classcell__" in attrsdiff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.pyindex c633e867a..8108f7000 100644--- a/tests/test_pipeline_images.py+++ b/tests/test_pipeline_images.py@@ -159,7 +159,7 @@ class TestImagesPipeline:self.pipeline.thumbs = {"small": (20, 20)}orig_im, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0))- orig_thumb, orig_thumb_buf = _create_image("JPEG", "RGB", (20, 20), (0, 0, 0))+ _, orig_thumb_buf = _create_image("JPEG", "RGB", (20, 20), (0, 0, 0))resp = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf.getvalue())req = Request(url="https://dev.mydeco.com/mydeco.gif")@@ -172,7 +172,7 @@ class TestImagesPipeline:assert orig_im.copy() == new_imassert buf.getvalue() == new_buf.getvalue()- thumb_path, thumb_img, thumb_buf = next(get_images_gen)+ thumb_path, _, thumb_buf = next(get_images_gen)assert thumb_path == "thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg"assert orig_thumb_buf.getvalue() == thumb_buf.getvalue()diff --git a/tests/test_pipelines.py b/tests/test_pipelines.pyindex 131a0e36b..8db9431ea 100644--- a/tests/test_pipelines.py+++ b/tests/test_pipelines.py@@ -371,13 +371,13 @@ class TestMiddlewareManagerSpider:),pytest.raises(ValueError,- match="ItemPipelineManager needs to access self.crawler.spider but it is None",+ match=r"ItemPipelineManager needs to access self\.crawler\.spider but it is None",),):mwman.open_spider(DefaultSpider())with pytest.raises(ValueError,- match="ItemPipelineManager needs to access self.crawler.spider but it is None",+ match=r"ItemPipelineManager needs to access self\.crawler\.spider but it is None",):await mwman.open_spider_async()with (@@ -387,13 +387,13 @@ class TestMiddlewareManagerSpider:),pytest.raises(ValueError,- match="ItemPipelineManager needs to access self.crawler.spider but it is None",+ match=r"ItemPipelineManager needs to access self\.crawler\.spider but it is None",),):mwman.close_spider(DefaultSpider())with pytest.raises(ValueError,- match="ItemPipelineManager needs to access self.crawler.spider but it is None",+ match=r"ItemPipelineManager needs to access self\.crawler\.spider but it is None",):await mwman.close_spider_async()diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.pyindex 5d31a9886..5dd2de8d8 100644--- a/tests/test_scheduler_base.py+++ b/tests/test_scheduler_base.py@@ -35,7 +35,7 @@ class MinimalScheduler:def next_request(self) -> Request | None:if self.has_pending_requests():- fp, request = self.requests.popitem()+ _, request = self.requests.popitem()return requestreturn Nonediff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.pyindex c1f358bd2..e97e114d6 100644--- a/tests/test_settings/__init__.py+++ b/tests/test_settings/__init__.py@@ -311,7 +311,7 @@ class TestBaseSettings:assert settings.getdict("TEST_DICT3", {"key1": 5}) == {"key1": 5}with pytest.raises(ValueError,- match="dictionary update sequence element #0 has length 3; 2 is required|sequence of pairs expected",+ match=r"dictionary update sequence element #0 has length 3; 2 is required|sequence of pairs expected",):settings.getdict("TEST_LIST1")with pytest.raises(diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.pyindex 977273673..3ac6d9af7 100644--- a/tests/test_spidermiddleware.py+++ b/tests/test_spidermiddleware.py@@ -568,7 +568,8 @@ class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware):async def _test_asyncgen_nodowngrade(self, *mw_classes: type[Any]) -> None:with pytest.raises(- _InvalidOutput, match="Async iterable returned from .+ cannot be downgraded"+ _InvalidOutput,+ match=r"Async iterable returned from .+ cannot be downgraded",):await self._get_middleware_result(*mw_classes)diff --git a/tests/test_squeues.py b/tests/test_squeues.pyindex 21bbeece2..8283d3d5d 100644--- a/tests/test_squeues.py+++ b/tests/test_squeues.py@@ -33,13 +33,13 @@ def nonserializable_object_test(self):q = self.queue()with pytest.raises(ValueError,- match="unmarshallable object|Can't (get|pickle) local object|Can't pickle .*: it's not found as",+ match=r"unmarshallable object|Can't (get|pickle) local object|Can't pickle .*: it's not found as",):q.push(lambda x: x)# Selectors should fail (lxml.html.HtmlElement objects can't be pickled)sel = Selector(text="<html><body><p>some text</p></body></html>")with pytest.raises(- ValueError, match="unmarshallable object|can't pickle Selector objects"+ ValueError, match=r"unmarshallable object|can't pickle Selector objects"):q.push(sel)@@ -117,7 +117,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):q = self.queue()with pytest.raises(ValueError,- match="Can't (get|pickle) local object|Can't pickle .*: it's not found as",+ match=r"Can't (get|pickle) local object|Can't pickle .*: it's not found as",) as exc_info:q.push(lambda x: x)if hasattr(sys, "pypy_version_info"):diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.pyindex ed7dda18d..b6a9d8b06 100644--- a/tests/test_utils_conf.py+++ b/tests/test_utils_conf.py@@ -31,7 +31,7 @@ class TestBuildComponentList:# Same priority raises ValueErrorduplicate_bs.set("ONE", duplicate_bs["ONE"], priority=20)with pytest.raises(- ValueError, match="Some paths in .* convert to the same object"+ ValueError, match=r"Some paths in .* convert to the same object"):build_component_list(duplicate_bs, convert=lambda x: x.lower())diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.pyindex 02362693a..b1532ca77 100644--- a/tests/test_utils_curl.py+++ b/tests/test_utils_curl.py@@ -220,7 +220,7 @@ class TestCurlToRequestKwargs:assert curl_to_request_kwargs(curl_command) == expected_result# case 2: ignore_unknown_options=False (raise exception):- with pytest.raises(ValueError, match="Unrecognized options:.*--bar.*--baz"):+ with pytest.raises(ValueError, match=r"Unrecognized options:.*--bar.*--baz"):curl_to_request_kwargs("curl --bar --baz http://www.example.com", ignore_unknown_options=False)diff --git a/tox.ini b/tox.iniindex eb406c89a..eabdf28c0 100644--- a/tox.ini+++ b/tox.ini@@ -45,15 +45,15 @@ commands =[testenv:typing]basepython = python3.10deps =- mypy==1.16.1- typing-extensions==4.14.1- types-defusedxml==0.7.0.20250516- types-lxml==2025.3.30- types-pexpect==4.9.0.20250516- types-Pygments==2.19.0.20250516- botocore-stubs==1.38.46- boto3-stubs[s3]==1.39.3- itemadapter==0.11.0+ mypy==1.18.2+ typing-extensions==4.15.0+ types-defusedxml==0.7.0.20250822+ types-lxml==2025.8.25+ types-pexpect==4.9.0.20250916+ types-Pygments==2.19.0.20250809+ botocore-stubs==1.40.59+ boto3-stubs[s3]==1.40.59+ itemadapter==0.12.2Protego==0.5.0w3lib==2.3.1attrs >= 18.2.0@@ -83,15 +83,15 @@ commands =basepython = python3deps ={[testenv:extra-deps]deps}- pylint==3.3.7+ pylint==4.0.2commands =pylint conftest.py docs extras scrapy tests[testenv:twinecheck]basepython = python3deps =- twine==6.1.0- build==1.2.2.post1+ twine==6.2.0+ build==1.3.0commands =python -m build --sdisttwine check dist/*