Scrapy, a fast high-level web crawling & scraping framework for Python.
Upgrade to sphinx-scrapy 0.10.0 and sphinx-llm-friendly (#8214)
Adrian8da9459f5c9d
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yamlindex a7c62346a..6df332c95 100644--- a/.pre-commit-config.yaml+++ b/.pre-commit-config.yaml@@ -27,7 +27,7 @@ repos:hooks:- id: sphinx-lint- repo: https://github.com/scrapy/sphinx-scrapy- rev: 0.9.0+ rev: 0.10.0hooks:- id: sphinx-scrapy- repo: https://github.com/zizmorcore/zizmor-pre-commitdiff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.pyindex 5d6b16520..f2a30c092 100644--- a/docs/_ext/scrapydocs.py+++ b/docs/_ext/scrapydocs.py@@ -160,8 +160,11 @@ def setup(app: Sphinx) -> dict[str, Any]:app.add_node(SettingslistNode,- markdown=(visit_settingslist_node_markdown, depart_settingslist_node_markdown),- singlemarkdown=(+ llm_markdown=(+ visit_settingslist_node_markdown,+ depart_settingslist_node_markdown,+ ),+ llm_singlemarkdown=(visit_settingslist_node_markdown,depart_settingslist_node_markdown,),diff --git a/docs/conf.py b/docs/conf.pyindex fa305025e..ce6096867 100644--- a/docs/conf.py+++ b/docs/conf.py@@ -175,9 +175,9 @@ scrapy_intersphinx_enable = ["w3lib",]-# sphinx_llms_txt -------------------------------------------------------------+# sphinx-llm-friendly ----------------------------------------------------------llms_txt_exclude = [+llm_friendly_llms_full_txt_exclude = [# Changelog, not useful for an LLM answering "how do I use Scrapy"# questions, and the largest single contributor to llms-full.txt size."news.rst",@@ -186,16 +186,5 @@ llms_txt_exclude = []-def _exclude_llms_txt_docs_from_llms_full_txt(app):- # llms-full.txt is the output of the singlemarkdown builder.- if app.builder.name != "singlemarkdown":- return- app.config.exclude_patterns.extend(llms_txt_exclude)---def setup(app):- app.connect("builder-inited", _exclude_llms_txt_docs_from_llms_full_txt)--# -- Other options ------------------------------------------------------------default_dark_mode = Falsediff --git a/docs/requirements.in b/docs/requirements.inindex 2366fd726..7b518721f 100644--- a/docs/requirements.in+++ b/docs/requirements.in@@ -6,4 +6,4 @@ sphinx-notfound-pagesphinx-reredirectssphinx-rtd-themesphinx-rtd-dark-mode-sphinx-scrapy @ git+https://github.com/scrapy/[email protected]+sphinx-scrapy==0.10.0diff --git a/docs/requirements.txt b/docs/requirements.txtindex 8852b454c..ef8fff4e3 100644--- a/docs/requirements.txt+++ b/docs/requirements.txt@@ -34,7 +34,7 @@ defusedxml==0.7.1docutils==0.22.4# via# sphinx- # sphinx-markdown-builder+ # sphinx-llm-friendly# sphinx-rtd-theme# sphinx-scrapyfilelock==3.25.2@@ -132,8 +132,7 @@ sphinx==9.1.0# -r requirements.in# sphinx-copybutton# sphinx-last-updated-by-git- # sphinx-llms-txt- # sphinx-markdown-builder+ # sphinx-llm-friendly# sphinx-notfound-page# sphinx-reredirects# sphinx-rtd-theme@@ -143,9 +142,7 @@ sphinx-copybutton==0.5.2# via sphinx-scrapysphinx-last-updated-by-git==0.3.8# via sphinx-sitemap-sphinx-llms-txt @ git+https://github.com/zytedata/sphinx-llms-txt.git@5e8866cb0cc249aa2017ad9050b3b83a7ca16f69- # via sphinx-scrapy-sphinx-markdown-builder @ git+https://github.com/zytedata/sphinx-markdown-builder.git@bb3121247d5e179df44e97214828f48b80b78238+sphinx-llm-friendly==0.2.0# via sphinx-scrapysphinx-notfound-page==1.1.0# via -r requirements.in@@ -157,7 +154,7 @@ sphinx-rtd-theme==3.1.0# via# -r requirements.in# sphinx-rtd-dark-mode-sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@788b5a281d1e73dcd1e055ffa8021510114cd08b+sphinx-scrapy==0.10.0# via -r requirements.insphinx-sitemap==2.9.0# via sphinx-scrapy@@ -176,7 +173,7 @@ sphinxcontrib-qthelp==2.0.0sphinxcontrib-serializinghtml==2.0.0# via sphinxtabulate==0.10.0- # via sphinx-markdown-builder+ # via sphinx-llm-friendlytldextract==5.3.1# via scrapytwisted==25.5.0diff --git a/extras/upgrade-ci-pins.py b/extras/upgrade-ci-pins.pyindex bfc743be1..1a9ad8c82 100644--- a/extras/upgrade-ci-pins.py+++ b/extras/upgrade-ci-pins.py@@ -159,7 +159,7 @@ def main() -> None:assert revfor path in (Path("tox.ini"), Path("docs/requirements.in")):text = re.sub(- r"(sphinx-scrapy\.git@)\S+",+ r"(sphinx-scrapy(?:\[tox\])?==)\S+",rf"\g<1>{rev[1]}",path.read_text(encoding="utf-8"),)diff --git a/tox.ini b/tox.iniindex 409d2c571..633068ab2 100644--- a/tox.ini+++ b/tox.ini@@ -5,7 +5,7 @@[tox]requires =- sphinx-scrapy[tox] @ git+https://github.com/scrapy/[email protected]+ sphinx-scrapy[tox]==0.10.0tox-uvenvlist =pre-commit