Scrapy, a fast high-level web crawling & scraping framework for Python.
Upgrade to sphinx-scrapy 0.12.0 (#8216)
Adrian1084ff47fb4e
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.ymlindex 5238fd21d..b0b466aa6 100644--- a/.github/workflows/publish.yml+++ b/.github/workflows/publish.yml@@ -63,8 +63,8 @@ jobs:- run: |python -m pip install -r docs/requirements.txtpython -m pip install .- sphinx-build -b markdown -q docs docs/_build/markdown- python extras/github-release-notes.py "$GITHUB_REF_NAME" docs/_build/markdown/news.md > release-notes.md+ sphinx-build -b html -q docs docs/_build/html+ python extras/github-release-notes.py "$GITHUB_REF_NAME" docs/_build/html/news.md > release-notes.md- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1with:name: release-notesdiff --git a/.pre-commit-config.yaml b/.pre-commit-config.yamlindex 6df332c95..0f4e55fde 100644--- a/.pre-commit-config.yaml+++ b/.pre-commit-config.yaml@@ -27,7 +27,7 @@ repos:hooks:- id: sphinx-lint- repo: https://github.com/scrapy/sphinx-scrapy- rev: 0.10.0+ rev: 0.12.0hooks:- id: sphinx-scrapy- repo: https://github.com/zizmorcore/zizmor-pre-commitdiff --git a/.readthedocs.yml b/.readthedocs.ymlindex a2773dcf2..fa9aa0730 100644--- a/.readthedocs.yml+++ b/.readthedocs.yml@@ -5,6 +5,4 @@ build:python: "3.14"commands:- pip install tox- - tox -e docs- - mkdir -p $READTHEDOCS_OUTPUT/html- - cp -a docs/_build/all/. $READTHEDOCS_OUTPUT/html/+ - tox -e docs -- $READTHEDOCS_OUTPUT/htmldiff --git a/AGENTS.md b/AGENTS.mdindex 7105ce641..7e1bfba2a 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -69,9 +69,10 @@ This file adds what agents tend to get wrong.- Give new anchors short names without the `topics-` prefix. Keep existinganchors.- Wrap paragraphs at 79 characters.-- `sphinx-scrapy` is pinned in three places that must match: `tox.ini`,- `docs/requirements.in` (then run `uv pip compile requirements.in -o- requirements.txt` in `docs/`) and the `rev` in `.pre-commit-config.yaml`.+- To upgrade `sphinx-scrapy`, change its `rev` in `.pre-commit-config.yaml`,+ run `pre-commit run sphinx-scrapy --all-files` to sync `tox.ini` and+ `docs/requirements.in`, then run `uv pip compile requirements.in -o+ requirements.txt` in `docs/`.## Git and GitHubdiff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.pyindex f2a30c092..29d780a27 100644--- a/docs/_ext/scrapydocs.py+++ b/docs/_ext/scrapydocs.py@@ -59,6 +59,20 @@ def collect_scrapy_settings_refs(app: Sphinx, doctree: document) -> None:)+def purge_scrapy_settings(app: Sphinx, env: Any, docname: str) -> None:+ env.scrapy_all_settings = [+ d for d in getattr(env, "scrapy_all_settings", []) if d["docname"] != docname+ ]+++def merge_scrapy_settings(+ app: Sphinx, env: Any, docnames: set[str], other: Any+) -> None:+ env.scrapy_all_settings = getattr(env, "scrapy_all_settings", []) + getattr(+ other, "scrapy_all_settings", []+ )++def make_setting_element(setting_data: SettingData, app: Sphinx, fromdocname: str) -> Any:@@ -77,17 +91,6 @@ def make_setting_element(return item-def make_setting_markdown_item(- setting_data: SettingData, app: Sphinx, fromdocname: str-) -> str:- uri = app.builder.get_relative_uri(fromdocname, setting_data["docname"])- if uri.startswith("#"):- target = f"#{setting_data['refid']}"- else:- target = f"{uri}#{setting_data['refid']}"- return f"* [{setting_data['setting_name']}]({target})"--def _iter_sorted_settings(env: Any, fromdocname: str) -> list[SettingData]:return [d@@ -112,23 +115,6 @@ def replace_settingslist_nodes(node.replace_self(settings_list)-def visit_settingslist_node_markdown(translator: Any, _node: Node) -> None:- builder = translator.builder- env = builder.env- fromdocname = getattr(builder, "current_doc_name", env.docname)- lines = [- make_setting_markdown_item(setting_data, builder.app, fromdocname)- for setting_data in _iter_sorted_settings(env, fromdocname)- ]- if lines:- translator.add("\n".join(lines), prefix_eol=2, suffix_eol=2)- raise nodes.SkipNode---def depart_settingslist_node_markdown(_translator: Any, _node: Node) -> None:- return None--def source_role(name, rawtext, text: str, lineno, inliner, options=None, content=None) -> tuple[list[Any], list[Any]]:@@ -158,19 +144,11 @@ def setup(app: Sphinx) -> dict[str, Any]:app.add_role("commit", commit_role)app.add_role("rev", rev_role)- app.add_node(- SettingslistNode,- llm_markdown=(- visit_settingslist_node_markdown,- depart_settingslist_node_markdown,- ),- llm_singlemarkdown=(- visit_settingslist_node_markdown,- depart_settingslist_node_markdown,- ),- )+ app.add_node(SettingslistNode)app.add_directive("settingslist", SettingsListDirective)app.connect("doctree-read", collect_scrapy_settings_refs)+ app.connect("env-purge-doc", purge_scrapy_settings)+ app.connect("env-merge-info", merge_scrapy_settings)app.connect("doctree-resolved", replace_settingslist_nodes)return {"parallel_read_safe": True}diff --git a/docs/news.rst b/docs/news.rstindex 3581c4240..a91aef497 100644--- a/docs/news.rst+++ b/docs/news.rst@@ -22,8 +22,6 @@ Backward-incompatible changes``errback`` are no longer called, and a warning is issued if they areset.- (:gh:`TBD`)-.. _release-2.19.0:Scrapy 2.19.0 (2026-09-10)diff --git a/docs/requirements.in b/docs/requirements.inindex 7b518721f..68a47958a 100644--- a/docs/requirements.in+++ b/docs/requirements.in@@ -6,4 +6,4 @@ sphinx-notfound-pagesphinx-reredirectssphinx-rtd-themesphinx-rtd-dark-mode-sphinx-scrapy==0.10.0+sphinx-scrapy==0.12.0diff --git a/docs/requirements.txt b/docs/requirements.txtindex ef8fff4e3..2cbc49f5e 100644--- a/docs/requirements.txt+++ b/docs/requirements.txt@@ -1,5 +1,5 @@# This file was autogenerated by uv via the following command:-# uv pip compile requirements.in -o requirements.txt+# uv pip compile requirements.in -o requirements.txt --refreshalabaster==1.0.0# via sphinxannotated-types==0.7.0@@ -110,10 +110,13 @@ pyopenssl==26.0.0# via scrapyqueuelib==1.9.0# via scrapy+regex==2026.9.10+ # via tiktokenrequests==2.33.0# via# requests-file# sphinx+ # tiktoken# tldextractrequests-file==3.0.1# via tldextract@@ -142,7 +145,7 @@ sphinx-copybutton==0.5.2# via sphinx-scrapysphinx-last-updated-by-git==0.3.8# via sphinx-sitemap-sphinx-llm-friendly==0.2.0+sphinx-llm-friendly==0.5.0# via sphinx-scrapysphinx-notfound-page==1.1.0# via -r requirements.in@@ -154,7 +157,7 @@ sphinx-rtd-theme==3.1.0# via# -r requirements.in# sphinx-rtd-dark-mode-sphinx-scrapy==0.10.0+sphinx-scrapy==0.12.0# via -r requirements.insphinx-sitemap==2.9.0# via sphinx-scrapy@@ -174,6 +177,8 @@ sphinxcontrib-serializinghtml==2.0.0# via sphinxtabulate==0.10.0# via sphinx-llm-friendly+tiktoken==0.14.0+ # via sphinx-llm-friendlytldextract==5.3.1# via scrapytwisted==25.5.0diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 6acdfad32..639f7ef8f 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -2609,7 +2609,7 @@ modifying generator function source code during runtime, skip AST parsing ofcallback functions, or improve performance in auto-reloading developmentenvironments.-.. only:: html+.. only:: not llmSettings documented elsewhere:------------------------------diff --git a/extras/github-release-notes.py b/extras/github-release-notes.pyindex 5d72183c6..0d7ef11cb 100644--- a/extras/github-release-notes.py+++ b/extras/github-release-notes.py@@ -5,28 +5,22 @@ from pathlib import PathDOCS_URL = "https://docs.scrapy.org/en/latest/"-def _absolute_link(match: re.Match[str]) -> str:- path, _, fragment = match["target"].partition("#")- path = re.sub(r"\.md$", ".html", path or "news.md")- return f"]({DOCS_URL}{path}{'#' + fragment if fragment else ''})"--def release_notes(version: str, news_md: str) -> str:"""Return the GitHub release body of *version*: its highlights, or itswhole release notes if it has no highlights, followed by a link to itsrelease notes in the documentation.*news_md* is the content of :file:`news.md` as built by the Sphinx- ``markdown`` builder, which resolves cross-references into links.+ ``html`` builder, which resolves cross-references into links."""section = news_md.split(f"\n## Scrapy {version} (", 1)[1].split("\n## ", 1)[0]body = section.split("\n", 1)[1]if "\nHighlights:\n" in body:body = body.split("\nHighlights:\n", 1)[1].split("\n#", 1)[0]- body = re.sub(r"<a id=.*", "", body).strip()+ body = body.strip()# GitHub renders every newline of a release body as a line break.body = re.sub(r"(?<=\S)\n(?! *([-*] |#|\n)) *", " ", body)- body = re.sub(r"\]\((?P<target>(?!https?://)[^)]*)\)", _absolute_link, body)+ body = re.sub(r"\]\((?!https?://)([^)\s]*)\.md\b", rf"]({DOCS_URL}\1.html", body)body = re.sub(r"\[(#\d+)\]\(https://github\.com/scrapy/scrapy/issues/\d+\)", r"\1", body)diff --git a/extras/upgrade-ci-pins.py b/extras/upgrade-ci-pins.pyindex 1a9ad8c82..924039df5 100644--- a/extras/upgrade-ci-pins.py+++ b/extras/upgrade-ci-pins.py@@ -155,18 +155,9 @@ def main() -> None:)pre_commit.write_text(config, encoding="utf-8")- rev = re.search(r"/sphinx-scrapy\n\s+rev: (\S+)", config)- assert rev- for path in (Path("tox.ini"), Path("docs/requirements.in")):- text = re.sub(- r"(sphinx-scrapy(?:\[tox\])?==)\S+",- rf"\g<1>{rev[1]}",- path.read_text(encoding="utf-8"),- )- path.write_text(text, encoding="utf-8")-- # sphinx-scrapy sets the Python version of .readthedocs.yml to the latest- # one that Read the Docs supports.+ # sphinx-scrapy syncs its pins in tox.ini and docs/requirements.in with its+ # pre-commit rev, and sets the Python version of .readthedocs.yml to the+ # latest one that Read the Docs supports.subprocess.run(["pre-commit", "run", "sphinx-scrapy", "--all-files"], # noqa: S607check=False,diff --git a/tox.ini b/tox.iniindex 2149805bf..732225b5f 100644--- a/tox.ini+++ b/tox.ini@@ -5,7 +5,7 @@[tox]requires =- sphinx-scrapy[tox]==0.10.0+ sphinx-scrapy[tox]==0.12.0tox-uvenvlist =pre-commit