Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add llms.txt and llms-full.txt generation (#7380)

Adrianf8d103a65a5d
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index d3715caf2..cb05784fa 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -26,9 +26,13 @@ jobs:
- python-version: "3.10"
env:
TOXENV: typing-tests
- - python-version: "3.13" # Keep in sync with .readthedocs.yml
+ # Keep in sync with pyproject.toml tool.sphinx-scrapy.python-version.
+ - python-version: "3.13"
env:
TOXENV: docs
+ - python-version: "3.13"
+ env:
+ TOXENV: docs-tests
- python-version: "3.13"
env:
TOXENV: twinecheck
diff --git a/.gitignore b/.gitignore
index 4100bcd97..5e52ecf1e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,7 +3,7 @@
*.pyc
_trial_temp*
dropin.cache
-docs/build
+docs/_build
*egg-info
.tox/
venv/
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index a2bf1be0c..b86681b21 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -26,3 +26,7 @@ repos:
rev: v1.0.2
hooks:
- id: sphinx-lint
+- repo: https://github.com/scrapy/sphinx-scrapy
+ rev: 0.7.0
+ hooks:
+ - id: sphinx-scrapy
diff --git a/.readthedocs.yml b/.readthedocs.yml
index 23e4cabea..dc3827869 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -1,17 +1,9 @@
version: 2
-formats: all
-sphinx:
- configuration: docs/conf.py
- fail_on_warning: true
-
build:
os: ubuntu-24.04
tools:
- # For available versions, see:
- # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python
- python: "3.13" # Keep in sync with .github/workflows/checks.yml
-
-python:
- install:
- - requirements: docs/requirements.txt
- - path: .
+ python: "3.13"
+ commands:
+ - pip install tox
+ - tox -e docs
+ - cp -a docs/_build/all/. $READTHEDOCS_OUTPUT/html/
diff --git a/docs/README.rst b/docs/README.rst
index 36dd5aea4..4d2236b53 100644
--- a/docs/README.rst
+++ b/docs/README.rst
@@ -65,4 +65,4 @@ To compile the documentation to HTML run the following command::
tox -e docs
-Documentation will be generated (in HTML format) inside the ``.tox/docs/tmp/html`` dir.
+Documentation will be generated inside the ``docs/_build/all`` dir.
diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py
index 4e232967d..edb91bfb9 100644
--- a/docs/_ext/scrapydocs.py
+++ b/docs/_ext/scrapydocs.py
@@ -77,6 +77,25 @@ def make_setting_element(
return item
+def make_setting_markdown_item(
+ setting_data: SettingData, app: Sphinx, fromdocname: str
+) -> str:
+ uri = app.builder.get_relative_uri(fromdocname, setting_data["docname"])
+ if uri.startswith("#"):
+ target = f"#{setting_data['refid']}"
+ else:
+ target = f"{uri}#{setting_data['refid']}"
+ return f"* [{setting_data['setting_name']}]({target})"
+
+
+def _iter_sorted_settings(env: Any, fromdocname: str) -> list[SettingData]:
+ return [
+ d
+ for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name")) # type: ignore[attr-defined]
+ if fromdocname != d["docname"]
+ ]
+
+
def replace_settingslist_nodes(
app: Sphinx, doctree: document, fromdocname: str
) -> None:
@@ -87,13 +106,29 @@ def replace_settingslist_nodes(
settings_list.extend(
[
make_setting_element(d, app, fromdocname)
- for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name")) # type: ignore[attr-defined]
- if fromdocname != d["docname"]
+ for d in _iter_sorted_settings(env, fromdocname)
]
)
node.replace_self(settings_list)
+def visit_settingslist_node_markdown(translator: Any, _node: Node) -> None:
+ builder = translator.builder
+ env = builder.env
+ fromdocname = getattr(builder, "current_doc_name", env.docname)
+ lines = [
+ make_setting_markdown_item(setting_data, builder.app, fromdocname)
+ for setting_data in _iter_sorted_settings(env, fromdocname)
+ ]
+ if lines:
+ translator.add("\n".join(lines), prefix_eol=2, suffix_eol=2)
+ raise nodes.SkipNode
+
+
+def depart_settingslist_node_markdown(_translator: Any, _node: Node) -> None:
+ return None
+
+
def source_role(
name, rawtext, text: str, lineno, inliner, options=None, content=None
) -> tuple[list[Any], list[Any]]:
@@ -127,32 +162,19 @@ def rev_role(
def setup(app: Sphinx) -> dict[str, Any]:
- app.add_crossref_type(
- directivename="setting",
- rolename="setting",
- indextemplate="pair: %s; setting",
- )
- app.add_crossref_type(
- directivename="signal",
- rolename="signal",
- indextemplate="pair: %s; signal",
- )
- app.add_crossref_type(
- directivename="command",
- rolename="command",
- indextemplate="pair: %s; command",
- )
- app.add_crossref_type(
- directivename="reqmeta",
- rolename="reqmeta",
- indextemplate="pair: %s; reqmeta",
- )
app.add_role("source", source_role)
app.add_role("commit", commit_role)
app.add_role("issue", issue_role)
app.add_role("rev", rev_role)
- app.add_node(SettingslistNode)
+ app.add_node(
+ SettingslistNode,
+ markdown=(visit_settingslist_node_markdown, depart_settingslist_node_markdown),
+ singlemarkdown=(
+ visit_settingslist_node_markdown,
+ depart_settingslist_node_markdown,
+ ),
+ )
app.add_directive("settingslist", SettingsListDirective)
app.connect("doctree-read", collect_scrapy_settings_refs)
diff --git a/docs/conf.py b/docs/conf.py
index 9b1db9bb7..e3e618860 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -28,11 +28,9 @@ author = "Scrapy developers"
extensions = [
"notfound.extension",
"scrapydocs",
- "sphinx.ext.autodoc",
+ "sphinx_scrapy",
"scrapyfixautodoc", # Must be after "sphinx.ext.autodoc"
"sphinx.ext.coverage",
- "sphinx.ext.intersphinx",
- "sphinx.ext.viewcode",
"sphinx_rtd_dark_mode",
]
@@ -147,22 +145,24 @@ coverage_ignore_pyobjects = [
# -- Options for the InterSphinx extension -----------------------------------
# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration
-intersphinx_mapping = {
- "attrs": ("https://www.attrs.org/en/stable/", None),
- "coverage": ("https://coverage.readthedocs.io/en/latest", None),
- "cryptography": ("https://cryptography.io/en/latest/", None),
- "cssselect": ("https://cssselect.readthedocs.io/en/latest", None),
- "itemloaders": ("https://itemloaders.readthedocs.io/en/latest/", None),
- "parsel": ("https://parsel.readthedocs.io/en/latest/", None),
- "pytest": ("https://docs.pytest.org/en/latest", None),
- "python": ("https://docs.python.org/3", None),
- "sphinx": ("https://www.sphinx-doc.org/en/master", None),
- "tox": ("https://tox.wiki/en/latest/", None),
- "twisted": ("https://docs.twisted.org/en/stable/", None),
- "twistedapi": ("https://docs.twisted.org/en/stable/api/", None),
- "w3lib": ("https://w3lib.readthedocs.io/en/latest", None),
-}
intersphinx_disabled_reftypes: Sequence[str] = []
+# sphinx-scrapy ---------------------------------------------------------------
+
+scrapy_intersphinx_enable = [
+ "attrs",
+ "coverage",
+ "cryptography",
+ "cssselect",
+ "itemloaders",
+ "parsel",
+ "pytest",
+ "sphinx",
+ "tox",
+ "twisted",
+ "twistedapi",
+ "w3lib",
+]
+
# -- Other options ------------------------------------------------------------
default_dark_mode = False
diff --git a/docs/requirements.in b/docs/requirements.in
new file mode 100644
index 000000000..d0ef2455e
--- /dev/null
+++ b/docs/requirements.in
@@ -0,0 +1,8 @@
+h2
+pydantic
+scrapy-spider-metadata
+sphinx
+sphinx-notfound-page
+sphinx-rtd-theme
+sphinx-rtd-dark-mode
+sphinx-scrapy @ git+https://github.com/scrapy/[email protected]
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 500db7a7e..354c670c7 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,7 +1,192 @@
+# This file was autogenerated by uv via the following command:
+# uv pip compile requirements.in -o requirements.txt
+alabaster==1.0.0
+ # via sphinx
+annotated-types==0.7.0
+ # via pydantic
+attrs==26.1.0
+ # via
+ # service-identity
+ # twisted
+automat==25.4.16
+ # via twisted
+babel==2.18.0
+ # via sphinx
+certifi==2026.2.25
+ # via requests
+cffi==2.0.0
+ # via cryptography
+charset-normalizer==3.4.6
+ # via requests
+constantly==23.10.4
+ # via twisted
+cryptography==46.0.6
+ # via
+ # pyopenssl
+ # scrapy
+ # service-identity
+cssselect==1.4.0
+ # via
+ # parsel
+ # scrapy
+defusedxml==0.7.1
+ # via scrapy
+docutils==0.22.4
+ # via
+ # sphinx
+ # sphinx-markdown-builder
+ # sphinx-rtd-theme
+filelock==3.25.2
+ # via tldextract
h2==4.3.0
-pydantic==2.12.3
+ # via -r requirements.in
+hpack==4.1.0
+ # via h2
+hyperframe==6.1.0
+ # via h2
+hyperlink==21.0.0
+ # via twisted
+idna==3.11
+ # via
+ # hyperlink
+ # requests
+ # tldextract
+imagesize==2.0.0
+ # via sphinx
+incremental==24.11.0
+ # via twisted
+itemadapter==0.13.1
+ # via
+ # itemloaders
+ # scrapy
+itemloaders==1.4.0
+ # via scrapy
+jinja2==3.1.6
+ # via sphinx
+jmespath==1.1.0
+ # via
+ # itemloaders
+ # parsel
+lxml==6.0.2
+ # via
+ # parsel
+ # scrapy
+markupsafe==3.0.3
+ # via jinja2
+packaging==26.0
+ # via
+ # incremental
+ # parsel
+ # scrapy
+ # scrapy-spider-metadata
+ # sphinx
+ # sphinx-scrapy
+parsel==1.11.0
+ # via
+ # itemloaders
+ # scrapy
+protego==0.6.0
+ # via scrapy
+pyasn1==0.6.3
+ # via
+ # pyasn1-modules
+ # service-identity
+pyasn1-modules==0.4.2
+ # via service-identity
+pycparser==3.0
+ # via cffi
+pydantic==2.12.5
+ # via
+ # -r requirements.in
+ # scrapy-spider-metadata
+pydantic-core==2.41.5
+ # via pydantic
+pydispatcher==2.0.7
+ # via scrapy
+pygments==2.19.2
+ # via sphinx
+pyopenssl==26.0.0
+ # via scrapy
+queuelib==1.9.0
+ # via scrapy
+requests==2.33.0
+ # via
+ # requests-file
+ # sphinx
+ # tldextract
+requests-file==3.0.1
+ # via tldextract
+roman-numerals==4.1.0
+ # via sphinx
+scrapy==2.14.2
+ # via scrapy-spider-metadata
scrapy-spider-metadata==0.2.0
-sphinx==8.1.3
-sphinx-notfound-page==1.0.4
-sphinx-rtd-theme==3.0.2
+ # via -r requirements.in
+service-identity==24.2.0
+ # via scrapy
+snowballstemmer==3.0.1
+ # via sphinx
+sphinx==9.1.0
+ # via
+ # -r requirements.in
+ # sphinx-copybutton
+ # sphinx-llms-txt
+ # sphinx-markdown-builder
+ # sphinx-notfound-page
+ # sphinx-rtd-theme
+ # sphinx-scrapy
+ # sphinxcontrib-jquery
+sphinx-copybutton==0.5.2
+ # via sphinx-scrapy
+sphinx-llms-txt @ git+https://github.com/zytedata/sphinx-llms-txt.git@5e8866cb0cc249aa2017ad9050b3b83a7ca16f69
+ # via sphinx-scrapy
+sphinx-markdown-builder @ git+https://github.com/zytedata/sphinx-markdown-builder.git@ac9f8babfe622e4300099ab44b96d9d9228e742e
+ # via sphinx-scrapy
+sphinx-notfound-page==1.1.0
+ # via -r requirements.in
sphinx-rtd-dark-mode==1.3.0
+ # via -r requirements.in
+sphinx-rtd-theme==3.1.0
+ # via
+ # -r requirements.in
+ # sphinx-rtd-dark-mode
+sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@46c52fc3a6d5ee632a8ef6bccf4daa1ff53e96fd
+ # via -r requirements.in
+sphinxcontrib-applehelp==2.0.0
+ # via sphinx
+sphinxcontrib-devhelp==2.0.0
+ # via sphinx
+sphinxcontrib-htmlhelp==2.1.0
+ # via sphinx
+sphinxcontrib-jquery==4.1
+ # via sphinx-rtd-theme
+sphinxcontrib-jsmath==1.0.1
+ # via sphinx
+sphinxcontrib-qthelp==2.0.0
+ # via sphinx
+sphinxcontrib-serializinghtml==2.0.0
+ # via sphinx
+tabulate==0.10.0
+ # via sphinx-markdown-builder
+tldextract==5.3.1
+ # via scrapy
+twisted==25.5.0
+ # via scrapy
+typing-extensions==4.15.0
+ # via
+ # pydantic
+ # pydantic-core
+ # twisted
+ # typing-inspection
+typing-inspection==0.4.2
+ # via pydantic
+urllib3==2.6.3
+ # via requests
+w3lib==2.4.1
+ # via
+ # parsel
+ # scrapy
+zope-interface==8.2
+ # via
+ # scrapy
+ # twisted
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 98374b6b4..806af509f 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -198,7 +198,7 @@ scrapy.Spider
The ``parse`` method is in charge of processing the response and returning
scraped data and/or more URLs to follow. Other Requests callbacks have
- the same requirements as the :class:`Spider` class.
+ the same requirements as the :class:`~scrapy.Spider` class.
This method, as well as any other Request callback, must return a
:class:`~scrapy.Request` object, an :ref:`item object <topics-items>`, an
diff --git a/pyproject.toml b/pyproject.toml
index 117edce0f..70ef73f26 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -464,3 +464,6 @@ split-on-trailing-comma = false
[tool.ruff.lint.pydocstyle]
convention = "pep257"
+
+[tool.sphinx-scrapy]
+python-version = "3.13" # Keep in sync with .github/workflows/checks.yml.
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index b6dba5773..d38b4b9db 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -25,11 +25,11 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
+ import scrapy
from scrapy.crawler import Crawler
from scrapy.http import Response
from scrapy.http.request import Request
from scrapy.settings import BaseSettings
- from scrapy.spiders import Spider
retry_logger = getLogger(__name__)
@@ -38,7 +38,7 @@ retry_logger = getLogger(__name__)
def get_retry_request(
request: Request,
*,
- spider: Spider,
+ spider: scrapy.Spider,
reason: str | Exception | type[Exception] = "unspecified",
max_retry_times: int | None = None,
priority_adjust: int | None = None,
@@ -145,7 +145,10 @@ class RetryMiddleware:
@_warn_spider_arg
def process_response(
- self, request: Request, response: Response, spider: Spider | None = None
+ self,
+ request: Request,
+ response: Response,
+ spider: scrapy.Spider | None = None,
) -> Request | Response:
if request.meta.get("dont_retry", False):
return response
@@ -156,7 +159,10 @@ class RetryMiddleware:
@_warn_spider_arg
def process_exception(
- self, request: Request, exception: Exception, spider: Spider | None = None
+ self,
+ request: Request,
+ exception: Exception,
+ spider: scrapy.Spider | None = None,
) -> Request | Response | None:
if isinstance(exception, self.exceptions_to_retry) and not request.meta.get(
"dont_retry", False
diff --git a/tox.ini b/tox.ini
index d35b0406c..0e86f964a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -4,7 +4,9 @@
# and then run "tox" from this directory.
[tox]
-envlist = pre-commit,pylint,typing,py
+requires =
+ sphinx-scrapy @ git+https://github.com/scrapy/[email protected]
+envlist = pre-commit,pylint,typing,py,docs
minversion = 1.7.0
[test-requirements]
@@ -238,38 +240,25 @@ commands =
setenv =
{[pinned]setenv}
-[docs]
+[testenv:docs-tests]
changedir = docs
-deps =
- -rdocs/requirements.txt
-setenv =
- READTHEDOCS_PROJECT=scrapy
- READTHEDOCS_VERSION=master
-
-[testenv:docs]
-basepython = python3
-changedir = {[docs]changedir}
deps =
{[test-requirements]deps}
- {[docs]deps}
-setenv = {[docs]setenv}
+ -rdocs/requirements.txt
commands =
- sphinx-build -W -b html . {envtmpdir}/html
pytest
[testenv:docs-coverage]
-basepython = python3
-changedir = {[docs]changedir}
-deps = {[docs]deps}
-setenv = {[docs]setenv}
+changedir = docs
+deps =
+ -rdocs/requirements.txt
commands =
sphinx-build -b coverage . {envtmpdir}/coverage
[testenv:docs-links]
-basepython = python3
-changedir = {[docs]changedir}
-deps = {[docs]deps}
-setenv = {[docs]setenv}
+changedir = docs
+deps =
+ -rdocs/requirements.txt
commands =
sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck