Scrapy, a fast high-level web crawling & scraping framework for Python.
Add llms.txt and llms-full.txt generation (#7380)
Adrianf8d103a65a5d
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.ymlindex d3715caf2..cb05784fa 100644--- a/.github/workflows/checks.yml+++ b/.github/workflows/checks.yml@@ -26,9 +26,13 @@ jobs:- python-version: "3.10"env:TOXENV: typing-tests- - python-version: "3.13" # Keep in sync with .readthedocs.yml+ # Keep in sync with pyproject.toml tool.sphinx-scrapy.python-version.+ - python-version: "3.13"env:TOXENV: docs+ - python-version: "3.13"+ env:+ TOXENV: docs-tests- python-version: "3.13"env:TOXENV: twinecheckdiff --git a/.gitignore b/.gitignoreindex 4100bcd97..5e52ecf1e 100644--- a/.gitignore+++ b/.gitignore@@ -3,7 +3,7 @@*.pyc_trial_temp*dropin.cache-docs/build+docs/_build*egg-info.tox/venv/diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yamlindex a2bf1be0c..b86681b21 100644--- a/.pre-commit-config.yaml+++ b/.pre-commit-config.yaml@@ -26,3 +26,7 @@ repos:rev: v1.0.2hooks:- id: sphinx-lint+- repo: https://github.com/scrapy/sphinx-scrapy+ rev: 0.7.0+ hooks:+ - id: sphinx-scrapydiff --git a/.readthedocs.yml b/.readthedocs.ymlindex 23e4cabea..dc3827869 100644--- a/.readthedocs.yml+++ b/.readthedocs.yml@@ -1,17 +1,9 @@version: 2-formats: all-sphinx:- configuration: docs/conf.py- fail_on_warning: true-build:os: ubuntu-24.04tools:- # For available versions, see:- # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python- python: "3.13" # Keep in sync with .github/workflows/checks.yml--python:- install:- - requirements: docs/requirements.txt- - path: .+ python: "3.13"+ commands:+ - pip install tox+ - tox -e docs+ - cp -a docs/_build/all/. $READTHEDOCS_OUTPUT/html/diff --git a/docs/README.rst b/docs/README.rstindex 36dd5aea4..4d2236b53 100644--- a/docs/README.rst+++ b/docs/README.rst@@ -65,4 +65,4 @@ To compile the documentation to HTML run the following command::tox -e docs-Documentation will be generated (in HTML format) inside the ``.tox/docs/tmp/html`` dir.+Documentation will be generated inside the ``docs/_build/all`` dir.diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.pyindex 4e232967d..edb91bfb9 100644--- a/docs/_ext/scrapydocs.py+++ b/docs/_ext/scrapydocs.py@@ -77,6 +77,25 @@ def make_setting_element(return item+def make_setting_markdown_item(+ setting_data: SettingData, app: Sphinx, fromdocname: str+) -> str:+ uri = app.builder.get_relative_uri(fromdocname, setting_data["docname"])+ if uri.startswith("#"):+ target = f"#{setting_data['refid']}"+ else:+ target = f"{uri}#{setting_data['refid']}"+ return f"* [{setting_data['setting_name']}]({target})"+++def _iter_sorted_settings(env: Any, fromdocname: str) -> list[SettingData]:+ return [+ d+ for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name")) # type: ignore[attr-defined]+ if fromdocname != d["docname"]+ ]++def replace_settingslist_nodes(app: Sphinx, doctree: document, fromdocname: str) -> None:@@ -87,13 +106,29 @@ def replace_settingslist_nodes(settings_list.extend([make_setting_element(d, app, fromdocname)- for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name")) # type: ignore[attr-defined]- if fromdocname != d["docname"]+ for d in _iter_sorted_settings(env, fromdocname)])node.replace_self(settings_list)+def visit_settingslist_node_markdown(translator: Any, _node: Node) -> None:+ builder = translator.builder+ env = builder.env+ fromdocname = getattr(builder, "current_doc_name", env.docname)+ lines = [+ make_setting_markdown_item(setting_data, builder.app, fromdocname)+ for setting_data in _iter_sorted_settings(env, fromdocname)+ ]+ if lines:+ translator.add("\n".join(lines), prefix_eol=2, suffix_eol=2)+ raise nodes.SkipNode+++def depart_settingslist_node_markdown(_translator: Any, _node: Node) -> None:+ return None++def source_role(name, rawtext, text: str, lineno, inliner, options=None, content=None) -> tuple[list[Any], list[Any]]:@@ -127,32 +162,19 @@ def rev_role(def setup(app: Sphinx) -> dict[str, Any]:- app.add_crossref_type(- directivename="setting",- rolename="setting",- indextemplate="pair: %s; setting",- )- app.add_crossref_type(- directivename="signal",- rolename="signal",- indextemplate="pair: %s; signal",- )- app.add_crossref_type(- directivename="command",- rolename="command",- indextemplate="pair: %s; command",- )- app.add_crossref_type(- directivename="reqmeta",- rolename="reqmeta",- indextemplate="pair: %s; reqmeta",- )app.add_role("source", source_role)app.add_role("commit", commit_role)app.add_role("issue", issue_role)app.add_role("rev", rev_role)- app.add_node(SettingslistNode)+ app.add_node(+ SettingslistNode,+ markdown=(visit_settingslist_node_markdown, depart_settingslist_node_markdown),+ singlemarkdown=(+ visit_settingslist_node_markdown,+ depart_settingslist_node_markdown,+ ),+ )app.add_directive("settingslist", SettingsListDirective)app.connect("doctree-read", collect_scrapy_settings_refs)diff --git a/docs/conf.py b/docs/conf.pyindex 9b1db9bb7..e3e618860 100644--- a/docs/conf.py+++ b/docs/conf.py@@ -28,11 +28,9 @@ author = "Scrapy developers"extensions = ["notfound.extension","scrapydocs",- "sphinx.ext.autodoc",+ "sphinx_scrapy","scrapyfixautodoc", # Must be after "sphinx.ext.autodoc""sphinx.ext.coverage",- "sphinx.ext.intersphinx",- "sphinx.ext.viewcode","sphinx_rtd_dark_mode",]@@ -147,22 +145,24 @@ coverage_ignore_pyobjects = [# -- Options for the InterSphinx extension -----------------------------------# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration-intersphinx_mapping = {- "attrs": ("https://www.attrs.org/en/stable/", None),- "coverage": ("https://coverage.readthedocs.io/en/latest", None),- "cryptography": ("https://cryptography.io/en/latest/", None),- "cssselect": ("https://cssselect.readthedocs.io/en/latest", None),- "itemloaders": ("https://itemloaders.readthedocs.io/en/latest/", None),- "parsel": ("https://parsel.readthedocs.io/en/latest/", None),- "pytest": ("https://docs.pytest.org/en/latest", None),- "python": ("https://docs.python.org/3", None),- "sphinx": ("https://www.sphinx-doc.org/en/master", None),- "tox": ("https://tox.wiki/en/latest/", None),- "twisted": ("https://docs.twisted.org/en/stable/", None),- "twistedapi": ("https://docs.twisted.org/en/stable/api/", None),- "w3lib": ("https://w3lib.readthedocs.io/en/latest", None),-}intersphinx_disabled_reftypes: Sequence[str] = []+# sphinx-scrapy ---------------------------------------------------------------++scrapy_intersphinx_enable = [+ "attrs",+ "coverage",+ "cryptography",+ "cssselect",+ "itemloaders",+ "parsel",+ "pytest",+ "sphinx",+ "tox",+ "twisted",+ "twistedapi",+ "w3lib",+]+# -- Other options ------------------------------------------------------------default_dark_mode = Falsediff --git a/docs/requirements.in b/docs/requirements.innew file mode 100644index 000000000..d0ef2455e--- /dev/null+++ b/docs/requirements.in@@ -0,0 +1,8 @@+h2+pydantic+scrapy-spider-metadata+sphinx+sphinx-notfound-page+sphinx-rtd-theme+sphinx-rtd-dark-mode+sphinx-scrapy @ git+https://github.com/scrapy/[email protected]diff --git a/docs/requirements.txt b/docs/requirements.txtindex 500db7a7e..354c670c7 100644--- a/docs/requirements.txt+++ b/docs/requirements.txt@@ -1,7 +1,192 @@+# This file was autogenerated by uv via the following command:+# uv pip compile requirements.in -o requirements.txt+alabaster==1.0.0+ # via sphinx+annotated-types==0.7.0+ # via pydantic+attrs==26.1.0+ # via+ # service-identity+ # twisted+automat==25.4.16+ # via twisted+babel==2.18.0+ # via sphinx+certifi==2026.2.25+ # via requests+cffi==2.0.0+ # via cryptography+charset-normalizer==3.4.6+ # via requests+constantly==23.10.4+ # via twisted+cryptography==46.0.6+ # via+ # pyopenssl+ # scrapy+ # service-identity+cssselect==1.4.0+ # via+ # parsel+ # scrapy+defusedxml==0.7.1+ # via scrapy+docutils==0.22.4+ # via+ # sphinx+ # sphinx-markdown-builder+ # sphinx-rtd-theme+filelock==3.25.2+ # via tldextracth2==4.3.0-pydantic==2.12.3+ # via -r requirements.in+hpack==4.1.0+ # via h2+hyperframe==6.1.0+ # via h2+hyperlink==21.0.0+ # via twisted+idna==3.11+ # via+ # hyperlink+ # requests+ # tldextract+imagesize==2.0.0+ # via sphinx+incremental==24.11.0+ # via twisted+itemadapter==0.13.1+ # via+ # itemloaders+ # scrapy+itemloaders==1.4.0+ # via scrapy+jinja2==3.1.6+ # via sphinx+jmespath==1.1.0+ # via+ # itemloaders+ # parsel+lxml==6.0.2+ # via+ # parsel+ # scrapy+markupsafe==3.0.3+ # via jinja2+packaging==26.0+ # via+ # incremental+ # parsel+ # scrapy+ # scrapy-spider-metadata+ # sphinx+ # sphinx-scrapy+parsel==1.11.0+ # via+ # itemloaders+ # scrapy+protego==0.6.0+ # via scrapy+pyasn1==0.6.3+ # via+ # pyasn1-modules+ # service-identity+pyasn1-modules==0.4.2+ # via service-identity+pycparser==3.0+ # via cffi+pydantic==2.12.5+ # via+ # -r requirements.in+ # scrapy-spider-metadata+pydantic-core==2.41.5+ # via pydantic+pydispatcher==2.0.7+ # via scrapy+pygments==2.19.2+ # via sphinx+pyopenssl==26.0.0+ # via scrapy+queuelib==1.9.0+ # via scrapy+requests==2.33.0+ # via+ # requests-file+ # sphinx+ # tldextract+requests-file==3.0.1+ # via tldextract+roman-numerals==4.1.0+ # via sphinx+scrapy==2.14.2+ # via scrapy-spider-metadatascrapy-spider-metadata==0.2.0-sphinx==8.1.3-sphinx-notfound-page==1.0.4-sphinx-rtd-theme==3.0.2+ # via -r requirements.in+service-identity==24.2.0+ # via scrapy+snowballstemmer==3.0.1+ # via sphinx+sphinx==9.1.0+ # via+ # -r requirements.in+ # sphinx-copybutton+ # sphinx-llms-txt+ # sphinx-markdown-builder+ # sphinx-notfound-page+ # sphinx-rtd-theme+ # sphinx-scrapy+ # sphinxcontrib-jquery+sphinx-copybutton==0.5.2+ # via sphinx-scrapy+sphinx-llms-txt @ git+https://github.com/zytedata/sphinx-llms-txt.git@5e8866cb0cc249aa2017ad9050b3b83a7ca16f69+ # via sphinx-scrapy+sphinx-markdown-builder @ git+https://github.com/zytedata/sphinx-markdown-builder.git@ac9f8babfe622e4300099ab44b96d9d9228e742e+ # via sphinx-scrapy+sphinx-notfound-page==1.1.0+ # via -r requirements.insphinx-rtd-dark-mode==1.3.0+ # via -r requirements.in+sphinx-rtd-theme==3.1.0+ # via+ # -r requirements.in+ # sphinx-rtd-dark-mode+sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@46c52fc3a6d5ee632a8ef6bccf4daa1ff53e96fd+ # via -r requirements.in+sphinxcontrib-applehelp==2.0.0+ # via sphinx+sphinxcontrib-devhelp==2.0.0+ # via sphinx+sphinxcontrib-htmlhelp==2.1.0+ # via sphinx+sphinxcontrib-jquery==4.1+ # via sphinx-rtd-theme+sphinxcontrib-jsmath==1.0.1+ # via sphinx+sphinxcontrib-qthelp==2.0.0+ # via sphinx+sphinxcontrib-serializinghtml==2.0.0+ # via sphinx+tabulate==0.10.0+ # via sphinx-markdown-builder+tldextract==5.3.1+ # via scrapy+twisted==25.5.0+ # via scrapy+typing-extensions==4.15.0+ # via+ # pydantic+ # pydantic-core+ # twisted+ # typing-inspection+typing-inspection==0.4.2+ # via pydantic+urllib3==2.6.3+ # via requests+w3lib==2.4.1+ # via+ # parsel+ # scrapy+zope-interface==8.2+ # via+ # scrapy+ # twisteddiff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rstindex 98374b6b4..806af509f 100644--- a/docs/topics/spiders.rst+++ b/docs/topics/spiders.rst@@ -198,7 +198,7 @@ scrapy.SpiderThe ``parse`` method is in charge of processing the response and returningscraped data and/or more URLs to follow. Other Requests callbacks have- the same requirements as the :class:`Spider` class.+ the same requirements as the :class:`~scrapy.Spider` class.This method, as well as any other Request callback, must return a:class:`~scrapy.Request` object, an :ref:`item object <topics-items>`, andiff --git a/pyproject.toml b/pyproject.tomlindex 117edce0f..70ef73f26 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -464,3 +464,6 @@ split-on-trailing-comma = false[tool.ruff.lint.pydocstyle]convention = "pep257"++[tool.sphinx-scrapy]+python-version = "3.13" # Keep in sync with .github/workflows/checks.yml.diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.pyindex b6dba5773..d38b4b9db 100644--- a/scrapy/downloadermiddlewares/retry.py+++ b/scrapy/downloadermiddlewares/retry.py@@ -25,11 +25,11 @@ if TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self+ import scrapyfrom scrapy.crawler import Crawlerfrom scrapy.http import Responsefrom scrapy.http.request import Requestfrom scrapy.settings import BaseSettings- from scrapy.spiders import Spiderretry_logger = getLogger(__name__)@@ -38,7 +38,7 @@ retry_logger = getLogger(__name__)def get_retry_request(request: Request,*,- spider: Spider,+ spider: scrapy.Spider,reason: str | Exception | type[Exception] = "unspecified",max_retry_times: int | None = None,priority_adjust: int | None = None,@@ -145,7 +145,10 @@ class RetryMiddleware:@_warn_spider_argdef process_response(- self, request: Request, response: Response, spider: Spider | None = None+ self,+ request: Request,+ response: Response,+ spider: scrapy.Spider | None = None,) -> Request | Response:if request.meta.get("dont_retry", False):return response@@ -156,7 +159,10 @@ class RetryMiddleware:@_warn_spider_argdef process_exception(- self, request: Request, exception: Exception, spider: Spider | None = None+ self,+ request: Request,+ exception: Exception,+ spider: scrapy.Spider | None = None,) -> Request | Response | None:if isinstance(exception, self.exceptions_to_retry) and not request.meta.get("dont_retry", Falsediff --git a/tox.ini b/tox.iniindex d35b0406c..0e86f964a 100644--- a/tox.ini+++ b/tox.ini@@ -4,7 +4,9 @@# and then run "tox" from this directory.[tox]-envlist = pre-commit,pylint,typing,py+requires =+ sphinx-scrapy @ git+https://github.com/scrapy/[email protected]+envlist = pre-commit,pylint,typing,py,docsminversion = 1.7.0[test-requirements]@@ -238,38 +240,25 @@ commands =setenv ={[pinned]setenv}-[docs]+[testenv:docs-tests]changedir = docs-deps =- -rdocs/requirements.txt-setenv =- READTHEDOCS_PROJECT=scrapy- READTHEDOCS_VERSION=master--[testenv:docs]-basepython = python3-changedir = {[docs]changedir}deps ={[test-requirements]deps}- {[docs]deps}-setenv = {[docs]setenv}+ -rdocs/requirements.txtcommands =- sphinx-build -W -b html . {envtmpdir}/htmlpytest[testenv:docs-coverage]-basepython = python3-changedir = {[docs]changedir}-deps = {[docs]deps}-setenv = {[docs]setenv}+changedir = docs+deps =+ -rdocs/requirements.txtcommands =sphinx-build -b coverage . {envtmpdir}/coverage[testenv:docs-links]-basepython = python3-changedir = {[docs]changedir}-deps = {[docs]deps}-setenv = {[docs]setenv}+changedir = docs+deps =+ -rdocs/requirements.txtcommands =sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck