Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Upgrade to sphinx-scrapy 0.12.0 (#8216)

Adrian1084ff47fb4e
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 5238fd21d..b0b466aa6 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -63,8 +63,8 @@ jobs:
- run: |
python -m pip install -r docs/requirements.txt
python -m pip install .
- sphinx-build -b markdown -q docs docs/_build/markdown
- python extras/github-release-notes.py "$GITHUB_REF_NAME" docs/_build/markdown/news.md > release-notes.md
+ sphinx-build -b html -q docs docs/_build/html
+ python extras/github-release-notes.py "$GITHUB_REF_NAME" docs/_build/html/news.md > release-notes.md
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: release-notes
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 6df332c95..0f4e55fde 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -27,7 +27,7 @@ repos:
hooks:
- id: sphinx-lint
- repo: https://github.com/scrapy/sphinx-scrapy
- rev: 0.10.0
+ rev: 0.12.0
hooks:
- id: sphinx-scrapy
- repo: https://github.com/zizmorcore/zizmor-pre-commit
diff --git a/.readthedocs.yml b/.readthedocs.yml
index a2773dcf2..fa9aa0730 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -5,6 +5,4 @@ build:
python: "3.14"
commands:
- pip install tox
- - tox -e docs
- - mkdir -p $READTHEDOCS_OUTPUT/html
- - cp -a docs/_build/all/. $READTHEDOCS_OUTPUT/html/
+ - tox -e docs -- $READTHEDOCS_OUTPUT/html
diff --git a/AGENTS.md b/AGENTS.md
index 7105ce641..7e1bfba2a 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -69,9 +69,10 @@ This file adds what agents tend to get wrong.
- Give new anchors short names without the `topics-` prefix. Keep existing
anchors.
- Wrap paragraphs at 79 characters.
-- `sphinx-scrapy` is pinned in three places that must match: `tox.ini`,
- `docs/requirements.in` (then run `uv pip compile requirements.in -o
- requirements.txt` in `docs/`) and the `rev` in `.pre-commit-config.yaml`.
+- To upgrade `sphinx-scrapy`, change its `rev` in `.pre-commit-config.yaml`,
+ run `pre-commit run sphinx-scrapy --all-files` to sync `tox.ini` and
+ `docs/requirements.in`, then run `uv pip compile requirements.in -o
+ requirements.txt` in `docs/`.
## Git and GitHub
diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py
index f2a30c092..29d780a27 100644
--- a/docs/_ext/scrapydocs.py
+++ b/docs/_ext/scrapydocs.py
@@ -59,6 +59,20 @@ def collect_scrapy_settings_refs(app: Sphinx, doctree: document) -> None:
)
+def purge_scrapy_settings(app: Sphinx, env: Any, docname: str) -> None:
+ env.scrapy_all_settings = [
+ d for d in getattr(env, "scrapy_all_settings", []) if d["docname"] != docname
+ ]
+
+
+def merge_scrapy_settings(
+ app: Sphinx, env: Any, docnames: set[str], other: Any
+) -> None:
+ env.scrapy_all_settings = getattr(env, "scrapy_all_settings", []) + getattr(
+ other, "scrapy_all_settings", []
+ )
+
+
def make_setting_element(
setting_data: SettingData, app: Sphinx, fromdocname: str
) -> Any:
@@ -77,17 +91,6 @@ def make_setting_element(
return item
-def make_setting_markdown_item(
- setting_data: SettingData, app: Sphinx, fromdocname: str
-) -> str:
- uri = app.builder.get_relative_uri(fromdocname, setting_data["docname"])
- if uri.startswith("#"):
- target = f"#{setting_data['refid']}"
- else:
- target = f"{uri}#{setting_data['refid']}"
- return f"* [{setting_data['setting_name']}]({target})"
-
-
def _iter_sorted_settings(env: Any, fromdocname: str) -> list[SettingData]:
return [
d
@@ -112,23 +115,6 @@ def replace_settingslist_nodes(
node.replace_self(settings_list)
-def visit_settingslist_node_markdown(translator: Any, _node: Node) -> None:
- builder = translator.builder
- env = builder.env
- fromdocname = getattr(builder, "current_doc_name", env.docname)
- lines = [
- make_setting_markdown_item(setting_data, builder.app, fromdocname)
- for setting_data in _iter_sorted_settings(env, fromdocname)
- ]
- if lines:
- translator.add("\n".join(lines), prefix_eol=2, suffix_eol=2)
- raise nodes.SkipNode
-
-
-def depart_settingslist_node_markdown(_translator: Any, _node: Node) -> None:
- return None
-
-
def source_role(
name, rawtext, text: str, lineno, inliner, options=None, content=None
) -> tuple[list[Any], list[Any]]:
@@ -158,19 +144,11 @@ def setup(app: Sphinx) -> dict[str, Any]:
app.add_role("commit", commit_role)
app.add_role("rev", rev_role)
- app.add_node(
- SettingslistNode,
- llm_markdown=(
- visit_settingslist_node_markdown,
- depart_settingslist_node_markdown,
- ),
- llm_singlemarkdown=(
- visit_settingslist_node_markdown,
- depart_settingslist_node_markdown,
- ),
- )
+ app.add_node(SettingslistNode)
app.add_directive("settingslist", SettingsListDirective)
app.connect("doctree-read", collect_scrapy_settings_refs)
+ app.connect("env-purge-doc", purge_scrapy_settings)
+ app.connect("env-merge-info", merge_scrapy_settings)
app.connect("doctree-resolved", replace_settingslist_nodes)
return {"parallel_read_safe": True}
diff --git a/docs/news.rst b/docs/news.rst
index 3581c4240..a91aef497 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -22,8 +22,6 @@ Backward-incompatible changes
``errback`` are no longer called, and a warning is issued if they are
set.
- (:gh:`TBD`)
-
.. _release-2.19.0:
Scrapy 2.19.0 (2026-09-10)
diff --git a/docs/requirements.in b/docs/requirements.in
index 7b518721f..68a47958a 100644
--- a/docs/requirements.in
+++ b/docs/requirements.in
@@ -6,4 +6,4 @@ sphinx-notfound-page
sphinx-reredirects
sphinx-rtd-theme
sphinx-rtd-dark-mode
-sphinx-scrapy==0.10.0
+sphinx-scrapy==0.12.0
diff --git a/docs/requirements.txt b/docs/requirements.txt
index ef8fff4e3..2cbc49f5e 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,5 +1,5 @@
# This file was autogenerated by uv via the following command:
-# uv pip compile requirements.in -o requirements.txt
+# uv pip compile requirements.in -o requirements.txt --refresh
alabaster==1.0.0
# via sphinx
annotated-types==0.7.0
@@ -110,10 +110,13 @@ pyopenssl==26.0.0
# via scrapy
queuelib==1.9.0
# via scrapy
+regex==2026.9.10
+ # via tiktoken
requests==2.33.0
# via
# requests-file
# sphinx
+ # tiktoken
# tldextract
requests-file==3.0.1
# via tldextract
@@ -142,7 +145,7 @@ sphinx-copybutton==0.5.2
# via sphinx-scrapy
sphinx-last-updated-by-git==0.3.8
# via sphinx-sitemap
-sphinx-llm-friendly==0.2.0
+sphinx-llm-friendly==0.5.0
# via sphinx-scrapy
sphinx-notfound-page==1.1.0
# via -r requirements.in
@@ -154,7 +157,7 @@ sphinx-rtd-theme==3.1.0
# via
# -r requirements.in
# sphinx-rtd-dark-mode
-sphinx-scrapy==0.10.0
+sphinx-scrapy==0.12.0
# via -r requirements.in
sphinx-sitemap==2.9.0
# via sphinx-scrapy
@@ -174,6 +177,8 @@ sphinxcontrib-serializinghtml==2.0.0
# via sphinx
tabulate==0.10.0
# via sphinx-llm-friendly
+tiktoken==0.14.0
+ # via sphinx-llm-friendly
tldextract==5.3.1
# via scrapy
twisted==25.5.0
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 6acdfad32..639f7ef8f 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -2609,7 +2609,7 @@ modifying generator function source code during runtime, skip AST parsing of
callback functions, or improve performance in auto-reloading development
environments.
-.. only:: html
+.. only:: not llm
Settings documented elsewhere:
------------------------------
diff --git a/extras/github-release-notes.py b/extras/github-release-notes.py
index 5d72183c6..0d7ef11cb 100644
--- a/extras/github-release-notes.py
+++ b/extras/github-release-notes.py
@@ -5,28 +5,22 @@ from pathlib import Path
DOCS_URL = "https://docs.scrapy.org/en/latest/"
-def _absolute_link(match: re.Match[str]) -> str:
- path, _, fragment = match["target"].partition("#")
- path = re.sub(r"\.md$", ".html", path or "news.md")
- return f"]({DOCS_URL}{path}{'#' + fragment if fragment else ''})"
-
-
def release_notes(version: str, news_md: str) -> str:
"""Return the GitHub release body of *version*: its highlights, or its
whole release notes if it has no highlights, followed by a link to its
release notes in the documentation.
*news_md* is the content of :file:`news.md` as built by the Sphinx
- ``markdown`` builder, which resolves cross-references into links.
+ ``html`` builder, which resolves cross-references into links.
"""
section = news_md.split(f"\n## Scrapy {version} (", 1)[1].split("\n## ", 1)[0]
body = section.split("\n", 1)[1]
if "\nHighlights:\n" in body:
body = body.split("\nHighlights:\n", 1)[1].split("\n#", 1)[0]
- body = re.sub(r"<a id=.*", "", body).strip()
+ body = body.strip()
# GitHub renders every newline of a release body as a line break.
body = re.sub(r"(?<=\S)\n(?! *([-*] |#|\n)) *", " ", body)
- body = re.sub(r"\]\((?P<target>(?!https?://)[^)]*)\)", _absolute_link, body)
+ body = re.sub(r"\]\((?!https?://)([^)\s]*)\.md\b", rf"]({DOCS_URL}\1.html", body)
body = re.sub(
r"\[(#\d+)\]\(https://github\.com/scrapy/scrapy/issues/\d+\)", r"\1", body
)
diff --git a/extras/upgrade-ci-pins.py b/extras/upgrade-ci-pins.py
index 1a9ad8c82..924039df5 100644
--- a/extras/upgrade-ci-pins.py
+++ b/extras/upgrade-ci-pins.py
@@ -155,18 +155,9 @@ def main() -> None:
)
pre_commit.write_text(config, encoding="utf-8")
- rev = re.search(r"/sphinx-scrapy\n\s+rev: (\S+)", config)
- assert rev
- for path in (Path("tox.ini"), Path("docs/requirements.in")):
- text = re.sub(
- r"(sphinx-scrapy(?:\[tox\])?==)\S+",
- rf"\g<1>{rev[1]}",
- path.read_text(encoding="utf-8"),
- )
- path.write_text(text, encoding="utf-8")
-
- # sphinx-scrapy sets the Python version of .readthedocs.yml to the latest
- # one that Read the Docs supports.
+ # sphinx-scrapy syncs its pins in tox.ini and docs/requirements.in with its
+ # pre-commit rev, and sets the Python version of .readthedocs.yml to the
+ # latest one that Read the Docs supports.
subprocess.run(
["pre-commit", "run", "sphinx-scrapy", "--all-files"], # noqa: S607
check=False,
diff --git a/tox.ini b/tox.ini
index 2149805bf..732225b5f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -5,7 +5,7 @@
[tox]
requires =
- sphinx-scrapy[tox]==0.10.0
+ sphinx-scrapy[tox]==0.12.0
tox-uv
envlist =
pre-commit