Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Migrate CI to GitHub actions (#4924)

Eugenio Lacuesta80db569aeae3
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
new file mode 100644
index 000000000..2748bf5fe
--- /dev/null
+++ b/.github/workflows/checks.yml
@@ -0,0 +1,38 @@
+name: Checks
+on: [push, pull_request]
+
+jobs:
+ checks:
+ runs-on: ubuntu-18.04
+ strategy:
+ matrix:
+ include:
+ - python-version: 3.8
+ env:
+ TOXENV: security
+ - python-version: 3.8
+ env:
+ TOXENV: flake8
+ - python-version: 3.8
+ env:
+ TOXENV: pylint
+ - python-version: 3.8
+ env:
+ TOXENV: typing
+ - python-version: 3.7 # Keep in sync with .readthedocs.yml
+ env:
+ TOXENV: docs
+
+ steps:
+ - uses: actions/checkout@v2
+
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.python-version }}
+
+ - name: Run check
+ env: ${{ matrix.env }}
+ run: |
+ pip install -U tox
+ tox
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
deleted file mode 100644
index 28771216c..000000000
--- a/.github/workflows/main.yml
+++ /dev/null
@@ -1,31 +0,0 @@
-name: Run test suite
-on: [push, pull_request]
-
-jobs:
- test-windows:
- name: "Windows Tests"
- runs-on: ${{ matrix.os }}
- strategy:
- matrix:
- os: [windows-latest]
- python-version: [3.7, 3.8]
- env: [TOXENV: py]
- include:
- - os: windows-latest
- python-version: 3.6
- env:
- TOXENV: windows-pinned
-
- steps:
- - uses: actions/checkout@v2
-
- - name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python@v1
- with:
- python-version: ${{ matrix.python-version }}
-
- - name: Run test suite
- env: ${{ matrix.env }}
- run: |
- pip install -U tox twine wheel codecov
- tox
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
new file mode 100644
index 000000000..aec6b8696
--- /dev/null
+++ b/.github/workflows/publish.yml
@@ -0,0 +1,31 @@
+name: Publish
+on: [push]
+
+jobs:
+ publish:
+ runs-on: ubuntu-18.04
+ if: startsWith(github.event.ref, 'refs/tags/')
+
+ steps:
+ - uses: actions/checkout@v2
+
+ - name: Set up Python 3.8
+ uses: actions/setup-python@v2
+ with:
+ python-version: 3.8
+
+ - name: Check Tag
+ id: check-release-tag
+ run: |
+ if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then
+ echo ::set-output name=release_tag::true
+ fi
+
+ - name: Publish to PyPI
+ if: steps.check-release-tag.outputs.release_tag == 'true'
+ run: |
+ pip install --upgrade setuptools wheel twine
+ python setup.py sdist bdist_wheel
+ export TWINE_USERNAME=__token__
+ export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }}
+ twine upload dist/*
diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml
new file mode 100644
index 000000000..51d27c405
--- /dev/null
+++ b/.github/workflows/tests-macos.yml
@@ -0,0 +1,25 @@
+name: macOS
+on: [push, pull_request]
+
+jobs:
+ tests:
+ runs-on: macos-10.15
+ strategy:
+ matrix:
+ python-version: [3.6, 3.7, 3.8]
+
+ steps:
+ - uses: actions/checkout@v2
+
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.python-version }}
+
+ - name: Run tests
+ run: |
+ pip install -U tox
+ tox -e py
+
+ - name: Upload coverage report
+ run: bash <(curl -s https://codecov.io/bash)
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
new file mode 100644
index 000000000..89c0334e2
--- /dev/null
+++ b/.github/workflows/tests-ubuntu.yml
@@ -0,0 +1,69 @@
+name: Ubuntu
+on: [push, pull_request]
+
+jobs:
+ tests:
+ runs-on: ubuntu-18.04
+ strategy:
+ matrix:
+ include:
+ - python-version: 3.7
+ env:
+ TOXENV: py
+ - python-version: 3.8
+ env:
+ TOXENV: py
+ - python-version: pypy3
+ env:
+ TOXENV: pypy3
+ PYPY_VERSION: 3.6-v7.3.1
+
+ # pinned deps
+ - python-version: 3.6.12
+ env:
+ TOXENV: pinned
+ - python-version: 3.6.12
+ env:
+ TOXENV: asyncio-pinned
+ - python-version: pypy3
+ env:
+ TOXENV: pypy3-pinned
+ PYPY_VERSION: 3.6-v7.2.0
+
+ # extras
+ - python-version: 3.8
+ env:
+ TOXENV: extra-deps
+ - python-version: 3.8
+ env:
+ TOXENV: asyncio
+
+ steps:
+ - uses: actions/checkout@v2
+
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.python-version }}
+
+ - name: Install system libraries
+ if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned')
+ run: |
+ sudo apt-get update
+ sudo apt-get install libxml2-dev libxslt-dev
+
+ - name: Run tests
+ env: ${{ matrix.env }}
+ run: |
+ if [[ ! -z "$PYPY_VERSION" ]]; then
+ export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
+ wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"
+ tar -jxf ${PYPY_VERSION}.tar.bz2
+ $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION"
+ source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
+ fi
+ pip install -U tox
+ tox
+
+ - name: Upload coverage report
+ run: bash <(curl -s https://codecov.io/bash)
diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml
new file mode 100644
index 000000000..ed2e4075d
--- /dev/null
+++ b/.github/workflows/tests-windows.yml
@@ -0,0 +1,32 @@
+name: Windows
+on: [push, pull_request]
+
+jobs:
+ tests:
+ runs-on: windows-latest
+ strategy:
+ matrix:
+ include:
+ - python-version: 3.6
+ env:
+ TOXENV: windows-pinned
+ - python-version: 3.7
+ env:
+ TOXENV: py
+ - python-version: 3.8
+ env:
+ TOXENV: py
+
+ steps:
+ - uses: actions/checkout@v2
+
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.python-version }}
+
+ - name: Run tests
+ env: ${{ matrix.env }}
+ run: |
+ pip install -U tox
+ tox
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 880c8772b..000000000
--- a/.travis.yml
+++ /dev/null
@@ -1,77 +0,0 @@
-language: python
-dist: xenial
-branches:
- only:
- - master
- - /^\d\.\d+$/
- - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
-matrix:
- include:
- - env: TOXENV=security
- python: 3.8
- - env: TOXENV=flake8
- python: 3.8
- - env: TOXENV=pylint
- python: 3.8
- - env: TOXENV=docs
- python: 3.7 # Keep in sync with .readthedocs.yml
- - env: TOXENV=typing
- python: 3.8
-
- - env: TOXENV=pinned
- python: 3.6.1
- - env: TOXENV=asyncio-pinned
- python: 3.6.1
- - env: TOXENV=pypy3-pinned PYPY_VERSION=3.6-v7.2.0
- dist: bionic
-
- - env: TOXENV=py
- python: 3.6
- - env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1
- dist: bionic
-
- - env: TOXENV=py
- python: 3.7
-
- - env: TOXENV=py PYPI_RELEASE_JOB=true
- python: 3.8
- dist: bionic
- - env: TOXENV=extra-deps
- python: 3.8
- dist: bionic
- - env: TOXENV=asyncio
- python: 3.8
- dist: bionic
-install:
- - |
- if [[ ! -z "$PYPY_VERSION" ]]; then
- export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
- wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"
- tar -jxf ${PYPY_VERSION}.tar.bz2
- virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
- source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
- fi
- - pip install -U tox twine wheel codecov
-
-script: tox
-after_success:
- - codecov
-notifications:
- irc:
- use_notice: true
- skip_join: true
- channels:
- - irc.freenode.org#scrapy
-cache:
- directories:
- - $HOME/.cache/pip
-deploy:
- provider: pypi
- distributions: "sdist bdist_wheel"
- user: scrapy
- password:
- secure: JaAKcy1AXWXDK3LXdjOtKyaVPCSFoCGCnW15g4f65E/8Fsi9ZzDfmBa4Equs3IQb/vs/if2SVrzJSr7arN7r9Z38Iv1mUXHkFAyA3Ym8mThfABBzzcUWEQhIHrCX0Tdlx9wQkkhs+PZhorlmRS4gg5s6DzPaeA2g8SCgmlRmFfA=
- on:
- tags: true
- repo: scrapy/scrapy
- condition: "$PYPI_RELEASE_JOB == true && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"
diff --git a/README.rst b/README.rst
index a8f2ba52b..9418d270f 100644
--- a/README.rst
+++ b/README.rst
@@ -10,9 +10,17 @@ Scrapy
:target: https://pypi.python.org/pypi/Scrapy
:alt: Supported Python Versions
-.. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg
- :target: https://travis-ci.org/scrapy/scrapy
- :alt: Build Status
+.. image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg
+ :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu
+ :alt: Ubuntu
+
+.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg
+ :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS
+ :alt: macOS
+
+.. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg
+ :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows
+ :alt: Windows
.. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg
:target: https://pypi.python.org/pypi/Scrapy
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index 68b856a88..a86c4ae4f 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -14,6 +14,6 @@ uvloop; platform_system != "Windows"
# optional for shell wrapper tests
bpython
brotlipy # optional for HTTP compress downloader middleware tests
-zstandard # optional for HTTP compress downloader middleware tests
+zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests
ipython
pywin32; sys_platform == "win32"
diff --git a/tox.ini b/tox.ini
index 6c39299bb..e70aef2d2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -13,7 +13,7 @@ deps =
-rtests/requirements-py3.txt
# mitmproxy does not support PyPy
# mitmproxy does not support Windows when running Python < 3.7
- mitmproxy; python_version >= '3.7' and implementation_name != 'pypy'
+ mitmproxy >= 4.0.4; python_version >= '3.7' and implementation_name != 'pypy'
mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy'
# Extras
botocore>=1.4.87
@@ -25,9 +25,9 @@ passenv =
GCS_TEST_FILE_URI
GCS_PROJECT_ID
#allow tox virtualenv to upgrade pip/wheel/setuptools
-download = true
+download = true
commands =
- py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests}
+ py.test --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests}
[testenv:typing]
basepython = python3
@@ -87,10 +87,6 @@ deps =
botocore==1.4.87
google-cloud-storage==1.29.0
Pillow==4.0.0
-install_command =
- # --use-feature=2020-resolver is required, otherwise the latest verion of
- # Twisted gets installed.
- pip install --use-feature=2020-resolver {opts} {packages}
setenv =
_SCRAPY_PINNED=true
@@ -99,11 +95,9 @@ deps =
{[pinned]deps}
lxml==3.5.0
PyDispatcher==2.0.5
-install_command =
- {[pinned]install_command}
setenv =
{[pinned]setenv}
-
+
[testenv:windows-pinned]
basepython = python3
deps =
@@ -112,8 +106,6 @@ deps =
# not need to build lxml from sources in a CI Windows job:
lxml==3.8.0
PyDispatcher==2.0.5
-install_command =
- {[pinned]install_command}
setenv =
{[pinned]setenv}
@@ -122,10 +114,6 @@ deps =
{[testenv]deps}
reppy
robotexclusionrulesparser
-install_command =
- # Test --use-feature=2020-resolver for the latest version of all
- # dependencies.
- pip install --use-feature=2020-resolver {opts} {packages}
[testenv:asyncio]
commands =
@@ -133,8 +121,6 @@ commands =
[testenv:asyncio-pinned]
deps = {[testenv:pinned]deps}
-install_command =
- {[pinned]install_command}
commands = {[testenv:asyncio]commands}
setenv =
{[pinned]setenv}
@@ -150,8 +136,6 @@ deps =
{[pinned]deps}
lxml==4.0.0
PyPyDispatcher==2.1.0
-install_command =
- {[pinned]install_command}
commands = {[testenv:pypy3]commands}
setenv =
{[pinned]setenv}