Scrapy, a fast high-level web crawling & scraping framework for Python.
Improve release automation (#8212)
Adrian91d87364637c
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 78c782307..79581416c 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -58,7 +58,7 @@ jobs:- python-version: "3.10"env:TOXENV: mypy-tests- # Keep in sync with pyproject.toml tool.sphinx-scrapy.python-version.+ # Keep in sync with .readthedocs.yml.- python-version: "3.14"env:TOXENV: docsdiff --git a/.github/workflows/publish.yml b/.github/workflows/publish.ymlindex 697647131..5238fd21d 100644--- a/.github/workflows/publish.yml+++ b/.github/workflows/publish.yml@@ -35,6 +35,7 @@ jobs:name: Upload release to PyPIneeds:- build+ - release-notesruns-on: ubuntu-latestenvironment:name: pypi@@ -48,3 +49,76 @@ jobs:path: dist/- name: Publish to PyPIuses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2++ release-notes:+ name: Build GitHub release notes+ runs-on: ubuntu-latest+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ persist-credentials: false+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0+ with:+ python-version: "3.14"+ - run: |+ python -m pip install -r docs/requirements.txt+ python -m pip install .+ sphinx-build -b markdown -q docs docs/_build/markdown+ python extras/github-release-notes.py "$GITHUB_REF_NAME" docs/_build/markdown/news.md > release-notes.md+ - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: release-notes+ path: release-notes.md++ github-release:+ name: Create GitHub release+ needs:+ - publish+ runs-on: ubuntu-latest+ permissions:+ contents: write+ steps:+ - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1+ with:+ name: release-notes+ - env:+ GH_REPO: ${{ github.repository }}+ GH_TOKEN: ${{ github.token }}+ run: |+ if gh release view "$GITHUB_REF_NAME" > /dev/null 2>&1; then+ echo "The $GITHUB_REF_NAME release already exists"+ else+ gh release create "$GITHUB_REF_NAME" --verify-tag --notes-file release-notes.md+ fi++ read-the-docs:+ name: Make the new branch the Read the Docs default+ if: endsWith(github.ref_name, '.0')+ needs:+ - publish+ runs-on: ubuntu-latest+ environment:+ name: pypi+ steps:+ - env:+ RTD_TOKEN: ${{ secrets.RTD_TOKEN }}+ run: |+ branch="${GITHUB_REF_NAME%.0}"+ api=https://app.readthedocs.org/api/v3/projects/scrapy+ rtd() { curl --fail-with-body -sS -H "Authorization: Token $RTD_TOKEN" "$@"; }+ rtd -X POST "$api/sync-versions/"+ for _ in $(seq 30); do+ rtd -o /dev/null "$api/versions/$branch/" 2> /dev/null && break+ sleep 10+ done+ rtd -X PATCH -H "Content-Type: application/json" -d '{"active": true}' "$api/versions/$branch/"+ rtd -X PATCH -H "Content-Type: application/json" -d "{\"default_branch\": \"$branch\"}" "$api/"+ rtd -X POST "$api/versions/latest/builds/"++ upgrade-ci-pins:+ if: endsWith(github.ref_name, '.0')+ needs:+ - publish+ uses: ./.github/workflows/upgrade-ci-pins.yml+ secrets:+ CI_PINS_APP_PRIVATE_KEY: ${{ secrets.CI_PINS_APP_PRIVATE_KEY }}diff --git a/.github/workflows/upgrade-ci-pins.yml b/.github/workflows/upgrade-ci-pins.ymlnew file mode 100644index 000000000..2cd7d9398--- /dev/null+++ b/.github/workflows/upgrade-ci-pins.yml@@ -0,0 +1,57 @@+name: Upgrade CI pins++permissions:+ contents: read++on:+ workflow_call:+ secrets:+ CI_PINS_APP_PRIVATE_KEY:+ required: true+ workflow_dispatch:++jobs:+ upgrade:+ runs-on: ubuntu-latest+ timeout-minutes: 30+ steps:+ # The App token can push workflow changes, and the pull request it opens+ # triggers CI.+ - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0+ id: app-token+ with:+ client-id: ${{ vars.CI_PINS_APP_CLIENT_ID }}+ private-key: ${{ secrets.CI_PINS_APP_PRIVATE_KEY }}+ permission-contents: write+ permission-pull-requests: write+ permission-workflows: write+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ ref: master+ persist-credentials: false+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0+ with:+ python-version: "3.14"+ - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1+ with:+ enable-cache: false+ - run: uv tool install pre-commit+ - run: python extras/upgrade-ci-pins.py+ env:+ GH_TOKEN: ${{ steps.app-token.outputs.token }}+ - name: Apply autofixes+ run: pre-commit run --all-files || true+ - name: Open a pull request+ env:+ APP_SLUG: ${{ steps.app-token.outputs.app-slug }}+ GH_TOKEN: ${{ steps.app-token.outputs.token }}+ run: |+ branch="upgrade-ci-pins-$(date +%Y-%m-%d)"+ user_id="$(gh api "/users/$APP_SLUG[bot]" --jq .id)"+ git config user.name "$APP_SLUG[bot]"+ git config user.email "$user_id+$APP_SLUG[bot]@users.noreply.github.com"+ git switch -c "$branch"+ git commit -am "Upgrade CI pins"+ git push "https://x-access-token:[email protected]/$GITHUB_REPOSITORY.git" "$branch"+ gh pr create --base master --head "$branch" --title "Upgrade CI pins" \+ --body "Please, fix whatever CI reports that could not be fixed automatically."diff --git a/extras/github-release-notes.py b/extras/github-release-notes.pynew file mode 100644index 000000000..5d72183c6--- /dev/null+++ b/extras/github-release-notes.py@@ -0,0 +1,39 @@+import re+import sys+from pathlib import Path++DOCS_URL = "https://docs.scrapy.org/en/latest/"+++def _absolute_link(match: re.Match[str]) -> str:+ path, _, fragment = match["target"].partition("#")+ path = re.sub(r"\.md$", ".html", path or "news.md")+ return f"]({DOCS_URL}{path}{'#' + fragment if fragment else ''})"+++def release_notes(version: str, news_md: str) -> str:+ """Return the GitHub release body of *version*: its highlights, or its+ whole release notes if it has no highlights, followed by a link to its+ release notes in the documentation.++ *news_md* is the content of :file:`news.md` as built by the Sphinx+ ``markdown`` builder, which resolves cross-references into links.+ """+ section = news_md.split(f"\n## Scrapy {version} (", 1)[1].split("\n## ", 1)[0]+ body = section.split("\n", 1)[1]+ if "\nHighlights:\n" in body:+ body = body.split("\nHighlights:\n", 1)[1].split("\n#", 1)[0]+ body = re.sub(r"<a id=.*", "", body).strip()+ # GitHub renders every newline of a release body as a line break.+ body = re.sub(r"(?<=\S)\n(?! *([-*] |#|\n)) *", " ", body)+ body = re.sub(r"\]\((?P<target>(?!https?://)[^)]*)\)", _absolute_link, body)+ body = re.sub(+ r"\[(#\d+)\]\(https://github\.com/scrapy/scrapy/issues/\d+\)", r"\1", body+ )+ anchor = "release-" + version.replace(".", "-")+ return f"{body}\n\n[Full changelog]({DOCS_URL}news.html#{anchor})\n"+++if __name__ == "__main__":+ version, path = sys.argv[1:]+ print(release_notes(version, Path(path).read_text(encoding="utf-8")), end="")diff --git a/extras/upgrade-ci-pins.py b/extras/upgrade-ci-pins.pynew file mode 100644index 000000000..bfc743be1--- /dev/null+++ b/extras/upgrade-ci-pins.py@@ -0,0 +1,203 @@+import json+import os+import re+import subprocess+import sys+from datetime import UTC, datetime, timedelta+from pathlib import Path+from typing import Any+from urllib.error import HTTPError+from urllib.request import Request, urlopen++# tox environments whose pins are CI tool versions, as opposed to the minimum+# supported versions of dependencies.+TOX_ENVS = ("mypy", "mypy-tests", "pylint", "twinecheck")++# GitHub Actions releases younger than this are skipped, to give the ecosystem+# time to spot compromised or broken releases before they reach CI. Keep in+# sync with the cooldown in .github/dependabot.yml.+COOLDOWN = timedelta(days=7)+++def _normalize(name: str) -> str:+ return re.sub(r"[-_.]+", "-", re.sub(r"\[.*", "", name)).lower()+++def _section(tox_ini: str, name: str) -> re.Match[str]:+ match = re.search(+ rf"^\[{re.escape(name)}\]\n(.*?)(?=^\[|\Z)", tox_ini, re.MULTILINE | re.DOTALL+ )+ assert match, name+ return match+++def _deps(tox_ini: str, name: str) -> list[str]:+ block = re.search(+ r"^deps =\n((?:[ \t]+.*\n)*)", _section(tox_ini, name)[1], re.MULTILINE+ )+ if not block:+ return []+ deps = (re.sub(r"\s*#.*", "", line).strip() for line in block[1].splitlines())+ return [dep for dep in deps if dep]+++def _requirements(tox_ini: str, name: str) -> list[str]:+ requirements = []+ for dep in _deps(tox_ini, name):+ if match := re.fullmatch(r"\{\[(.+)\]deps\}", dep):+ requirements += _requirements(tox_ini, match[1])+ else:+ requirements.append(dep)+ return requirements+++def _resolve(requirements: list[str], python: str) -> dict[str, str]:+ output = subprocess.run( # noqa: S603+ [+ *("uv", "pip", "compile", "--python-version", python),+ *("--no-header", "--no-annotate", "--quiet", "-"),+ ],+ input="\n".join(requirements),+ capture_output=True,+ text=True,+ check=True,+ ).stdout+ return {+ _normalize(match[1]): match[2]+ for match in re.finditer(r"^(\S+)==(\S+)", output, re.MULTILINE)+ }+++def _upgrade_tox_env(tox_ini: str, env: str) -> str:+ """Return *tox_ini* with the pins of *env* upgraded to the latest versions+ that can be installed together with Scrapy and the rest of the+ requirements of *env*, for its Python version.++ Pins that *env* gets from other sections are kept.+ """+ name = f"testenv:{env}"+ section = _section(tox_ini, name)+ body = section[1]+ python = re.search(r"^basepython = python(\S*)", body, re.MULTILINE)+ assert python+ version = python[1]+ if version == "3":+ version = f"{sys.version_info.major}.{sys.version_info.minor}"+ own = {dep: re.sub(r"==[^;\s]+", "", dep) for dep in _deps(tox_ini, name)}+ requirements = [own.get(dep, dep) for dep in _requirements(tox_ini, name)]+ versions = _resolve([*requirements, "."], version)+ body = re.sub(+ r"^([ \t]+)([\w.\-\[\],]+)==([^;\s]+)",+ lambda m: f"{m[1]}{m[2]}=={versions.get(_normalize(m[2]), m[3])}",+ body,+ flags=re.MULTILINE,+ )+ return tox_ini[: section.start(1)] + body + tox_ini[section.end(1) :]+++def _github(path: str) -> Any:+ headers = {"Accept": "application/vnd.github+json"}+ if token := os.environ.get("GH_TOKEN"):+ headers["Authorization"] = f"Bearer {token}"+ request = Request(f"https://api.github.com/{path}", headers=headers)+ with urlopen(request) as response: # noqa: S310+ return json.load(response)+++def _upgrade_actions(cutoff: datetime) -> None:+ """Pin every GitHub action used in workflows to the commit of its latest+ release published before *cutoff*.+ """+ workflows = list(Path(".github/workflows").glob("*.yml"))+ pattern = r"(uses: ([\w.-]+/[\w.-]+)[\w./-]*@)[0-9a-f]{40} # \S+"+ repos = {+ match[2]+ for path in workflows+ for match in re.finditer(pattern, path.read_text(encoding="utf-8"))+ }+ pins = {}+ for repo in sorted(repos):+ try:+ release = _github(f"repos/{repo}/releases/latest")+ except HTTPError as error:+ if error.code != 404:+ raise+ print(f"{repo} has no releases, keeping its pin")+ continue+ if datetime.fromisoformat(release["published_at"]) > cutoff:+ print(f"{repo} {release['tag_name']} is too recent, keeping its pin")+ continue+ sha = _github(f"repos/{repo}/commits/{release['tag_name']}")["sha"]+ pins[repo] = f"{sha} # {release['tag_name']}"+ for path in workflows:+ text = re.sub(+ pattern,+ lambda m: f"{m[1]}{pins[m[2]]}" if m[2] in pins else m[0],+ path.read_text(encoding="utf-8"),+ )+ path.write_text(text, encoding="utf-8")+++def main() -> None:+ python = f"{sys.version_info.major}.{sys.version_info.minor}"++ _upgrade_actions(datetime.now(UTC) - COOLDOWN)++ subprocess.run(["pre-commit", "autoupdate"], check=True) # noqa: S607++ # pre-commit autoupdate does not upgrade additional_dependencies.+ pre_commit = Path(".pre-commit-config.yaml")+ config = re.sub(+ r"^(\s+- )([\w.\-]+)==(\S+)$",+ lambda m: f"{m[1]}{m[2]}=={_resolve([m[2]], python)[_normalize(m[2])]}",+ pre_commit.read_text(encoding="utf-8"),+ flags=re.MULTILINE,+ )+ pre_commit.write_text(config, encoding="utf-8")++ rev = re.search(r"/sphinx-scrapy\n\s+rev: (\S+)", config)+ assert rev+ for path in (Path("tox.ini"), Path("docs/requirements.in")):+ text = re.sub(+ r"(sphinx-scrapy\.git@)\S+",+ rf"\g<1>{rev[1]}",+ path.read_text(encoding="utf-8"),+ )+ path.write_text(text, encoding="utf-8")++ # sphinx-scrapy sets the Python version of .readthedocs.yml to the latest+ # one that Read the Docs supports.+ subprocess.run(+ ["pre-commit", "run", "sphinx-scrapy", "--all-files"], # noqa: S607+ check=False,+ )+ docs_python = re.search(+ r'^ python: "([^"]+)"',+ Path(".readthedocs.yml").read_text(encoding="utf-8"),+ re.MULTILINE,+ )+ assert docs_python+ ci = Path(".github/workflows/ci.yml")+ text = re.sub(+ r'(# Keep in sync with \.readthedocs\.yml\.\n\s+- python-version: ")[^"]+',+ rf"\g<1>{docs_python[1]}",+ ci.read_text(encoding="utf-8"),+ )+ ci.write_text(text, encoding="utf-8")+ subprocess.run( # noqa: S603+ [+ *("uv", "pip", "compile", "--upgrade", "--quiet", "requirements.in"),+ *("-o", "requirements.txt", "--python-version", docs_python[1]),+ ],+ cwd="docs",+ check=True,+ )++ tox_ini = Path("tox.ini").read_text(encoding="utf-8")+ for env in TOX_ENVS:+ tox_ini = _upgrade_tox_env(tox_ini, env)+ Path("tox.ini").write_text(tox_ini, encoding="utf-8")+++if __name__ == "__main__":+ main()diff --git a/pyproject.toml b/pyproject.tomlindex e538b74ad..c98a7f8ee 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -157,6 +157,9 @@ current_version = "2.19.0"commit = truetag = truetag_name = "{new_version}"+# Fail on VERSION and PREVIOUS_VERSION placeholders left in docs or code.+allow_shell_hooks = true+setup_hooks = ['''! git grep -nE '(^|[^/"_A-Z`])(PREVIOUS_)?VERSION\b' -- docs scrapy'''][[tool.bumpversion.files]]filename = "docs/news.rst"@@ -522,6 +525,3 @@ split-on-trailing-comma = false[tool.ruff.lint.pydocstyle]convention = "pep257"--[tool.sphinx-scrapy]-python-version = "3.14" # Keep in sync with .github/workflows/ci.yml.