Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Improve release automation (#8212)

Adrian91d87364637c
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 78c782307..79581416c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -58,7 +58,7 @@ jobs:
- python-version: "3.10"
env:
TOXENV: mypy-tests
- # Keep in sync with pyproject.toml tool.sphinx-scrapy.python-version.
+ # Keep in sync with .readthedocs.yml.
- python-version: "3.14"
env:
TOXENV: docs
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 697647131..5238fd21d 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -35,6 +35,7 @@ jobs:
name: Upload release to PyPI
needs:
- build
+ - release-notes
runs-on: ubuntu-latest
environment:
name: pypi
@@ -48,3 +49,76 @@ jobs:
path: dist/
- name: Publish to PyPI
uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
+
+ release-notes:
+ name: Build GitHub release notes
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+ with:
+ python-version: "3.14"
+ - run: |
+ python -m pip install -r docs/requirements.txt
+ python -m pip install .
+ sphinx-build -b markdown -q docs docs/_build/markdown
+ python extras/github-release-notes.py "$GITHUB_REF_NAME" docs/_build/markdown/news.md > release-notes.md
+ - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: release-notes
+ path: release-notes.md
+
+ github-release:
+ name: Create GitHub release
+ needs:
+ - publish
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write
+ steps:
+ - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: release-notes
+ - env:
+ GH_REPO: ${{ github.repository }}
+ GH_TOKEN: ${{ github.token }}
+ run: |
+ if gh release view "$GITHUB_REF_NAME" > /dev/null 2>&1; then
+ echo "The $GITHUB_REF_NAME release already exists"
+ else
+ gh release create "$GITHUB_REF_NAME" --verify-tag --notes-file release-notes.md
+ fi
+
+ read-the-docs:
+ name: Make the new branch the Read the Docs default
+ if: endsWith(github.ref_name, '.0')
+ needs:
+ - publish
+ runs-on: ubuntu-latest
+ environment:
+ name: pypi
+ steps:
+ - env:
+ RTD_TOKEN: ${{ secrets.RTD_TOKEN }}
+ run: |
+ branch="${GITHUB_REF_NAME%.0}"
+ api=https://app.readthedocs.org/api/v3/projects/scrapy
+ rtd() { curl --fail-with-body -sS -H "Authorization: Token $RTD_TOKEN" "$@"; }
+ rtd -X POST "$api/sync-versions/"
+ for _ in $(seq 30); do
+ rtd -o /dev/null "$api/versions/$branch/" 2> /dev/null && break
+ sleep 10
+ done
+ rtd -X PATCH -H "Content-Type: application/json" -d '{"active": true}' "$api/versions/$branch/"
+ rtd -X PATCH -H "Content-Type: application/json" -d "{\"default_branch\": \"$branch\"}" "$api/"
+ rtd -X POST "$api/versions/latest/builds/"
+
+ upgrade-ci-pins:
+ if: endsWith(github.ref_name, '.0')
+ needs:
+ - publish
+ uses: ./.github/workflows/upgrade-ci-pins.yml
+ secrets:
+ CI_PINS_APP_PRIVATE_KEY: ${{ secrets.CI_PINS_APP_PRIVATE_KEY }}
diff --git a/.github/workflows/upgrade-ci-pins.yml b/.github/workflows/upgrade-ci-pins.yml
new file mode 100644
index 000000000..2cd7d9398
--- /dev/null
+++ b/.github/workflows/upgrade-ci-pins.yml
@@ -0,0 +1,57 @@
+name: Upgrade CI pins
+
+permissions:
+ contents: read
+
+on:
+ workflow_call:
+ secrets:
+ CI_PINS_APP_PRIVATE_KEY:
+ required: true
+ workflow_dispatch:
+
+jobs:
+ upgrade:
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ steps:
+ # The App token can push workflow changes, and the pull request it opens
+ # triggers CI.
+ - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
+ id: app-token
+ with:
+ client-id: ${{ vars.CI_PINS_APP_CLIENT_ID }}
+ private-key: ${{ secrets.CI_PINS_APP_PRIVATE_KEY }}
+ permission-contents: write
+ permission-pull-requests: write
+ permission-workflows: write
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ ref: master
+ persist-credentials: false
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+ with:
+ python-version: "3.14"
+ - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
+ with:
+ enable-cache: false
+ - run: uv tool install pre-commit
+ - run: python extras/upgrade-ci-pins.py
+ env:
+ GH_TOKEN: ${{ steps.app-token.outputs.token }}
+ - name: Apply autofixes
+ run: pre-commit run --all-files || true
+ - name: Open a pull request
+ env:
+ APP_SLUG: ${{ steps.app-token.outputs.app-slug }}
+ GH_TOKEN: ${{ steps.app-token.outputs.token }}
+ run: |
+ branch="upgrade-ci-pins-$(date +%Y-%m-%d)"
+ user_id="$(gh api "/users/$APP_SLUG[bot]" --jq .id)"
+ git config user.name "$APP_SLUG[bot]"
+ git config user.email "$user_id+$APP_SLUG[bot]@users.noreply.github.com"
+ git switch -c "$branch"
+ git commit -am "Upgrade CI pins"
+ git push "https://x-access-token:[email protected]/$GITHUB_REPOSITORY.git" "$branch"
+ gh pr create --base master --head "$branch" --title "Upgrade CI pins" \
+ --body "Please, fix whatever CI reports that could not be fixed automatically."
diff --git a/extras/github-release-notes.py b/extras/github-release-notes.py
new file mode 100644
index 000000000..5d72183c6
--- /dev/null
+++ b/extras/github-release-notes.py
@@ -0,0 +1,39 @@
+import re
+import sys
+from pathlib import Path
+
+DOCS_URL = "https://docs.scrapy.org/en/latest/"
+
+
+def _absolute_link(match: re.Match[str]) -> str:
+ path, _, fragment = match["target"].partition("#")
+ path = re.sub(r"\.md$", ".html", path or "news.md")
+ return f"]({DOCS_URL}{path}{'#' + fragment if fragment else ''})"
+
+
+def release_notes(version: str, news_md: str) -> str:
+ """Return the GitHub release body of *version*: its highlights, or its
+ whole release notes if it has no highlights, followed by a link to its
+ release notes in the documentation.
+
+ *news_md* is the content of :file:`news.md` as built by the Sphinx
+ ``markdown`` builder, which resolves cross-references into links.
+ """
+ section = news_md.split(f"\n## Scrapy {version} (", 1)[1].split("\n## ", 1)[0]
+ body = section.split("\n", 1)[1]
+ if "\nHighlights:\n" in body:
+ body = body.split("\nHighlights:\n", 1)[1].split("\n#", 1)[0]
+ body = re.sub(r"<a id=.*", "", body).strip()
+ # GitHub renders every newline of a release body as a line break.
+ body = re.sub(r"(?<=\S)\n(?! *([-*] |#|\n)) *", " ", body)
+ body = re.sub(r"\]\((?P<target>(?!https?://)[^)]*)\)", _absolute_link, body)
+ body = re.sub(
+ r"\[(#\d+)\]\(https://github\.com/scrapy/scrapy/issues/\d+\)", r"\1", body
+ )
+ anchor = "release-" + version.replace(".", "-")
+ return f"{body}\n\n[Full changelog]({DOCS_URL}news.html#{anchor})\n"
+
+
+if __name__ == "__main__":
+ version, path = sys.argv[1:]
+ print(release_notes(version, Path(path).read_text(encoding="utf-8")), end="")
diff --git a/extras/upgrade-ci-pins.py b/extras/upgrade-ci-pins.py
new file mode 100644
index 000000000..bfc743be1
--- /dev/null
+++ b/extras/upgrade-ci-pins.py
@@ -0,0 +1,203 @@
+import json
+import os
+import re
+import subprocess
+import sys
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any
+from urllib.error import HTTPError
+from urllib.request import Request, urlopen
+
+# tox environments whose pins are CI tool versions, as opposed to the minimum
+# supported versions of dependencies.
+TOX_ENVS = ("mypy", "mypy-tests", "pylint", "twinecheck")
+
+# GitHub Actions releases younger than this are skipped, to give the ecosystem
+# time to spot compromised or broken releases before they reach CI. Keep in
+# sync with the cooldown in .github/dependabot.yml.
+COOLDOWN = timedelta(days=7)
+
+
+def _normalize(name: str) -> str:
+ return re.sub(r"[-_.]+", "-", re.sub(r"\[.*", "", name)).lower()
+
+
+def _section(tox_ini: str, name: str) -> re.Match[str]:
+ match = re.search(
+ rf"^\[{re.escape(name)}\]\n(.*?)(?=^\[|\Z)", tox_ini, re.MULTILINE | re.DOTALL
+ )
+ assert match, name
+ return match
+
+
+def _deps(tox_ini: str, name: str) -> list[str]:
+ block = re.search(
+ r"^deps =\n((?:[ \t]+.*\n)*)", _section(tox_ini, name)[1], re.MULTILINE
+ )
+ if not block:
+ return []
+ deps = (re.sub(r"\s*#.*", "", line).strip() for line in block[1].splitlines())
+ return [dep for dep in deps if dep]
+
+
+def _requirements(tox_ini: str, name: str) -> list[str]:
+ requirements = []
+ for dep in _deps(tox_ini, name):
+ if match := re.fullmatch(r"\{\[(.+)\]deps\}", dep):
+ requirements += _requirements(tox_ini, match[1])
+ else:
+ requirements.append(dep)
+ return requirements
+
+
+def _resolve(requirements: list[str], python: str) -> dict[str, str]:
+ output = subprocess.run( # noqa: S603
+ [
+ *("uv", "pip", "compile", "--python-version", python),
+ *("--no-header", "--no-annotate", "--quiet", "-"),
+ ],
+ input="\n".join(requirements),
+ capture_output=True,
+ text=True,
+ check=True,
+ ).stdout
+ return {
+ _normalize(match[1]): match[2]
+ for match in re.finditer(r"^(\S+)==(\S+)", output, re.MULTILINE)
+ }
+
+
+def _upgrade_tox_env(tox_ini: str, env: str) -> str:
+ """Return *tox_ini* with the pins of *env* upgraded to the latest versions
+ that can be installed together with Scrapy and the rest of the
+ requirements of *env*, for its Python version.
+
+ Pins that *env* gets from other sections are kept.
+ """
+ name = f"testenv:{env}"
+ section = _section(tox_ini, name)
+ body = section[1]
+ python = re.search(r"^basepython = python(\S*)", body, re.MULTILINE)
+ assert python
+ version = python[1]
+ if version == "3":
+ version = f"{sys.version_info.major}.{sys.version_info.minor}"
+ own = {dep: re.sub(r"==[^;\s]+", "", dep) for dep in _deps(tox_ini, name)}
+ requirements = [own.get(dep, dep) for dep in _requirements(tox_ini, name)]
+ versions = _resolve([*requirements, "."], version)
+ body = re.sub(
+ r"^([ \t]+)([\w.\-\[\],]+)==([^;\s]+)",
+ lambda m: f"{m[1]}{m[2]}=={versions.get(_normalize(m[2]), m[3])}",
+ body,
+ flags=re.MULTILINE,
+ )
+ return tox_ini[: section.start(1)] + body + tox_ini[section.end(1) :]
+
+
+def _github(path: str) -> Any:
+ headers = {"Accept": "application/vnd.github+json"}
+ if token := os.environ.get("GH_TOKEN"):
+ headers["Authorization"] = f"Bearer {token}"
+ request = Request(f"https://api.github.com/{path}", headers=headers)
+ with urlopen(request) as response: # noqa: S310
+ return json.load(response)
+
+
+def _upgrade_actions(cutoff: datetime) -> None:
+ """Pin every GitHub action used in workflows to the commit of its latest
+ release published before *cutoff*.
+ """
+ workflows = list(Path(".github/workflows").glob("*.yml"))
+ pattern = r"(uses: ([\w.-]+/[\w.-]+)[\w./-]*@)[0-9a-f]{40} # \S+"
+ repos = {
+ match[2]
+ for path in workflows
+ for match in re.finditer(pattern, path.read_text(encoding="utf-8"))
+ }
+ pins = {}
+ for repo in sorted(repos):
+ try:
+ release = _github(f"repos/{repo}/releases/latest")
+ except HTTPError as error:
+ if error.code != 404:
+ raise
+ print(f"{repo} has no releases, keeping its pin")
+ continue
+ if datetime.fromisoformat(release["published_at"]) > cutoff:
+ print(f"{repo} {release['tag_name']} is too recent, keeping its pin")
+ continue
+ sha = _github(f"repos/{repo}/commits/{release['tag_name']}")["sha"]
+ pins[repo] = f"{sha} # {release['tag_name']}"
+ for path in workflows:
+ text = re.sub(
+ pattern,
+ lambda m: f"{m[1]}{pins[m[2]]}" if m[2] in pins else m[0],
+ path.read_text(encoding="utf-8"),
+ )
+ path.write_text(text, encoding="utf-8")
+
+
+def main() -> None:
+ python = f"{sys.version_info.major}.{sys.version_info.minor}"
+
+ _upgrade_actions(datetime.now(UTC) - COOLDOWN)
+
+ subprocess.run(["pre-commit", "autoupdate"], check=True) # noqa: S607
+
+ # pre-commit autoupdate does not upgrade additional_dependencies.
+ pre_commit = Path(".pre-commit-config.yaml")
+ config = re.sub(
+ r"^(\s+- )([\w.\-]+)==(\S+)$",
+ lambda m: f"{m[1]}{m[2]}=={_resolve([m[2]], python)[_normalize(m[2])]}",
+ pre_commit.read_text(encoding="utf-8"),
+ flags=re.MULTILINE,
+ )
+ pre_commit.write_text(config, encoding="utf-8")
+
+ rev = re.search(r"/sphinx-scrapy\n\s+rev: (\S+)", config)
+ assert rev
+ for path in (Path("tox.ini"), Path("docs/requirements.in")):
+ text = re.sub(
+ r"(sphinx-scrapy\.git@)\S+",
+ rf"\g<1>{rev[1]}",
+ path.read_text(encoding="utf-8"),
+ )
+ path.write_text(text, encoding="utf-8")
+
+ # sphinx-scrapy sets the Python version of .readthedocs.yml to the latest
+ # one that Read the Docs supports.
+ subprocess.run(
+ ["pre-commit", "run", "sphinx-scrapy", "--all-files"], # noqa: S607
+ check=False,
+ )
+ docs_python = re.search(
+ r'^ python: "([^"]+)"',
+ Path(".readthedocs.yml").read_text(encoding="utf-8"),
+ re.MULTILINE,
+ )
+ assert docs_python
+ ci = Path(".github/workflows/ci.yml")
+ text = re.sub(
+ r'(# Keep in sync with \.readthedocs\.yml\.\n\s+- python-version: ")[^"]+',
+ rf"\g<1>{docs_python[1]}",
+ ci.read_text(encoding="utf-8"),
+ )
+ ci.write_text(text, encoding="utf-8")
+ subprocess.run( # noqa: S603
+ [
+ *("uv", "pip", "compile", "--upgrade", "--quiet", "requirements.in"),
+ *("-o", "requirements.txt", "--python-version", docs_python[1]),
+ ],
+ cwd="docs",
+ check=True,
+ )
+
+ tox_ini = Path("tox.ini").read_text(encoding="utf-8")
+ for env in TOX_ENVS:
+ tox_ini = _upgrade_tox_env(tox_ini, env)
+ Path("tox.ini").write_text(tox_ini, encoding="utf-8")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/pyproject.toml b/pyproject.toml
index e538b74ad..c98a7f8ee 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -157,6 +157,9 @@ current_version = "2.19.0"
commit = true
tag = true
tag_name = "{new_version}"
+# Fail on VERSION and PREVIOUS_VERSION placeholders left in docs or code.
+allow_shell_hooks = true
+setup_hooks = ['''! git grep -nE '(^|[^/"_A-Z`])(PREVIOUS_)?VERSION\b' -- docs scrapy''']
[[tool.bumpversion.files]]
filename = "docs/news.rst"
@@ -522,6 +525,3 @@ split-on-trailing-comma = false
[tool.ruff.lint.pydocstyle]
convention = "pep257"
-
-[tool.sphinx-scrapy]
-python-version = "3.14" # Keep in sync with .github/workflows/ci.yml.