Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Upgrade to sphinx-scrapy 0.10.0 and sphinx-llm-friendly (#8214)

Adrian8da9459f5c9d
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index a7c62346a..6df332c95 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -27,7 +27,7 @@ repos:
hooks:
- id: sphinx-lint
- repo: https://github.com/scrapy/sphinx-scrapy
- rev: 0.9.0
+ rev: 0.10.0
hooks:
- id: sphinx-scrapy
- repo: https://github.com/zizmorcore/zizmor-pre-commit
diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py
index 5d6b16520..f2a30c092 100644
--- a/docs/_ext/scrapydocs.py
+++ b/docs/_ext/scrapydocs.py
@@ -160,8 +160,11 @@ def setup(app: Sphinx) -> dict[str, Any]:
app.add_node(
SettingslistNode,
- markdown=(visit_settingslist_node_markdown, depart_settingslist_node_markdown),
- singlemarkdown=(
+ llm_markdown=(
+ visit_settingslist_node_markdown,
+ depart_settingslist_node_markdown,
+ ),
+ llm_singlemarkdown=(
visit_settingslist_node_markdown,
depart_settingslist_node_markdown,
),
diff --git a/docs/conf.py b/docs/conf.py
index fa305025e..ce6096867 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -175,9 +175,9 @@ scrapy_intersphinx_enable = [
"w3lib",
]
-# sphinx_llms_txt -------------------------------------------------------------
+# sphinx-llm-friendly ---------------------------------------------------------
-llms_txt_exclude = [
+llm_friendly_llms_full_txt_exclude = [
# Changelog, not useful for an LLM answering "how do I use Scrapy"
# questions, and the largest single contributor to llms-full.txt size.
"news.rst",
@@ -186,16 +186,5 @@ llms_txt_exclude = [
]
-def _exclude_llms_txt_docs_from_llms_full_txt(app):
- # llms-full.txt is the output of the singlemarkdown builder.
- if app.builder.name != "singlemarkdown":
- return
- app.config.exclude_patterns.extend(llms_txt_exclude)
-
-
-def setup(app):
- app.connect("builder-inited", _exclude_llms_txt_docs_from_llms_full_txt)
-
-
# -- Other options ------------------------------------------------------------
default_dark_mode = False
diff --git a/docs/requirements.in b/docs/requirements.in
index 2366fd726..7b518721f 100644
--- a/docs/requirements.in
+++ b/docs/requirements.in
@@ -6,4 +6,4 @@ sphinx-notfound-page
sphinx-reredirects
sphinx-rtd-theme
sphinx-rtd-dark-mode
-sphinx-scrapy @ git+https://github.com/scrapy/[email protected]
+sphinx-scrapy==0.10.0
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 8852b454c..ef8fff4e3 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -34,7 +34,7 @@ defusedxml==0.7.1
docutils==0.22.4
# via
# sphinx
- # sphinx-markdown-builder
+ # sphinx-llm-friendly
# sphinx-rtd-theme
# sphinx-scrapy
filelock==3.25.2
@@ -132,8 +132,7 @@ sphinx==9.1.0
# -r requirements.in
# sphinx-copybutton
# sphinx-last-updated-by-git
- # sphinx-llms-txt
- # sphinx-markdown-builder
+ # sphinx-llm-friendly
# sphinx-notfound-page
# sphinx-reredirects
# sphinx-rtd-theme
@@ -143,9 +142,7 @@ sphinx-copybutton==0.5.2
# via sphinx-scrapy
sphinx-last-updated-by-git==0.3.8
# via sphinx-sitemap
-sphinx-llms-txt @ git+https://github.com/zytedata/sphinx-llms-txt.git@5e8866cb0cc249aa2017ad9050b3b83a7ca16f69
- # via sphinx-scrapy
-sphinx-markdown-builder @ git+https://github.com/zytedata/sphinx-markdown-builder.git@bb3121247d5e179df44e97214828f48b80b78238
+sphinx-llm-friendly==0.2.0
# via sphinx-scrapy
sphinx-notfound-page==1.1.0
# via -r requirements.in
@@ -157,7 +154,7 @@ sphinx-rtd-theme==3.1.0
# via
# -r requirements.in
# sphinx-rtd-dark-mode
-sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@788b5a281d1e73dcd1e055ffa8021510114cd08b
+sphinx-scrapy==0.10.0
# via -r requirements.in
sphinx-sitemap==2.9.0
# via sphinx-scrapy
@@ -176,7 +173,7 @@ sphinxcontrib-qthelp==2.0.0
sphinxcontrib-serializinghtml==2.0.0
# via sphinx
tabulate==0.10.0
- # via sphinx-markdown-builder
+ # via sphinx-llm-friendly
tldextract==5.3.1
# via scrapy
twisted==25.5.0
diff --git a/extras/upgrade-ci-pins.py b/extras/upgrade-ci-pins.py
index bfc743be1..1a9ad8c82 100644
--- a/extras/upgrade-ci-pins.py
+++ b/extras/upgrade-ci-pins.py
@@ -159,7 +159,7 @@ def main() -> None:
assert rev
for path in (Path("tox.ini"), Path("docs/requirements.in")):
text = re.sub(
- r"(sphinx-scrapy\.git@)\S+",
+ r"(sphinx-scrapy(?:\[tox\])?==)\S+",
rf"\g<1>{rev[1]}",
path.read_text(encoding="utf-8"),
)
diff --git a/tox.ini b/tox.ini
index 409d2c571..633068ab2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -5,7 +5,7 @@
[tox]
requires =
- sphinx-scrapy[tox] @ git+https://github.com/scrapy/[email protected]
+ sphinx-scrapy[tox]==0.10.0
tox-uv
envlist =
pre-commit