Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Exclude news and contributing from llms-full.txt (#8083)

Adrianeb778e639bef
diff --git a/docs/conf.py b/docs/conf.py
index fd0fc0f98..33966ed95 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -175,5 +175,27 @@ scrapy_intersphinx_enable = [
"w3lib",
]
+# sphinx_llms_txt -------------------------------------------------------------
+
+llms_txt_exclude = [
+ # Changelog, not useful for an LLM answering "how do I use Scrapy"
+ # questions, and the largest single contributor to llms-full.txt size.
+ "news.rst",
+ "contributing.rst",
+]
+
+
+def _exclude_llms_txt_docs_from_markdown_builds(app):
+ builder_name = getattr(getattr(app, "builder", None), "name", None)
+ if "markdown" not in builder_name:
+ return
+ for pattern in llms_txt_exclude:
+ app.config.exclude_patterns.append(pattern)
+
+
+def setup(app):
+ app.connect("builder-inited", _exclude_llms_txt_docs_from_markdown_builds)
+
+
# -- Other options ------------------------------------------------------------
default_dark_mode = False