Scrapy, a fast high-level web crawling & scraping framework for Python.
Migrate CI to GitHub actions (#4924)
Eugenio Lacuesta80db569aeae3
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.ymlnew file mode 100644index 000000000..2748bf5fe--- /dev/null+++ b/.github/workflows/checks.yml@@ -0,0 +1,38 @@+name: Checks+on: [push, pull_request]++jobs:+ checks:+ runs-on: ubuntu-18.04+ strategy:+ matrix:+ include:+ - python-version: 3.8+ env:+ TOXENV: security+ - python-version: 3.8+ env:+ TOXENV: flake8+ - python-version: 3.8+ env:+ TOXENV: pylint+ - python-version: 3.8+ env:+ TOXENV: typing+ - python-version: 3.7 # Keep in sync with .readthedocs.yml+ env:+ TOXENV: docs++ steps:+ - uses: actions/checkout@v2++ - name: Set up Python ${{ matrix.python-version }}+ uses: actions/setup-python@v2+ with:+ python-version: ${{ matrix.python-version }}++ - name: Run check+ env: ${{ matrix.env }}+ run: |+ pip install -U tox+ toxdiff --git a/.github/workflows/main.yml b/.github/workflows/main.ymldeleted file mode 100644index 28771216c..000000000--- a/.github/workflows/main.yml+++ /dev/null@@ -1,31 +0,0 @@-name: Run test suite-on: [push, pull_request]--jobs:- test-windows:- name: "Windows Tests"- runs-on: ${{ matrix.os }}- strategy:- matrix:- os: [windows-latest]- python-version: [3.7, 3.8]- env: [TOXENV: py]- include:- - os: windows-latest- python-version: 3.6- env:- TOXENV: windows-pinned-- steps:- - uses: actions/checkout@v2-- - name: Set up Python ${{ matrix.python-version }}- uses: actions/setup-python@v1- with:- python-version: ${{ matrix.python-version }}-- - name: Run test suite- env: ${{ matrix.env }}- run: |- pip install -U tox twine wheel codecov- toxdiff --git a/.github/workflows/publish.yml b/.github/workflows/publish.ymlnew file mode 100644index 000000000..aec6b8696--- /dev/null+++ b/.github/workflows/publish.yml@@ -0,0 +1,31 @@+name: Publish+on: [push]++jobs:+ publish:+ runs-on: ubuntu-18.04+ if: startsWith(github.event.ref, 'refs/tags/')++ steps:+ - uses: actions/checkout@v2++ - name: Set up Python 3.8+ uses: actions/setup-python@v2+ with:+ python-version: 3.8++ - name: Check Tag+ id: check-release-tag+ run: |+ if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then+ echo ::set-output name=release_tag::true+ fi++ - name: Publish to PyPI+ if: steps.check-release-tag.outputs.release_tag == 'true'+ run: |+ pip install --upgrade setuptools wheel twine+ python setup.py sdist bdist_wheel+ export TWINE_USERNAME=__token__+ export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }}+ twine upload dist/*diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.ymlnew file mode 100644index 000000000..51d27c405--- /dev/null+++ b/.github/workflows/tests-macos.yml@@ -0,0 +1,25 @@+name: macOS+on: [push, pull_request]++jobs:+ tests:+ runs-on: macos-10.15+ strategy:+ matrix:+ python-version: [3.6, 3.7, 3.8]++ steps:+ - uses: actions/checkout@v2++ - name: Set up Python ${{ matrix.python-version }}+ uses: actions/setup-python@v2+ with:+ python-version: ${{ matrix.python-version }}++ - name: Run tests+ run: |+ pip install -U tox+ tox -e py++ - name: Upload coverage report+ run: bash <(curl -s https://codecov.io/bash)diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.ymlnew file mode 100644index 000000000..89c0334e2--- /dev/null+++ b/.github/workflows/tests-ubuntu.yml@@ -0,0 +1,69 @@+name: Ubuntu+on: [push, pull_request]++jobs:+ tests:+ runs-on: ubuntu-18.04+ strategy:+ matrix:+ include:+ - python-version: 3.7+ env:+ TOXENV: py+ - python-version: 3.8+ env:+ TOXENV: py+ - python-version: pypy3+ env:+ TOXENV: pypy3+ PYPY_VERSION: 3.6-v7.3.1++ # pinned deps+ - python-version: 3.6.12+ env:+ TOXENV: pinned+ - python-version: 3.6.12+ env:+ TOXENV: asyncio-pinned+ - python-version: pypy3+ env:+ TOXENV: pypy3-pinned+ PYPY_VERSION: 3.6-v7.2.0++ # extras+ - python-version: 3.8+ env:+ TOXENV: extra-deps+ - python-version: 3.8+ env:+ TOXENV: asyncio++ steps:+ - uses: actions/checkout@v2++ - name: Set up Python ${{ matrix.python-version }}+ uses: actions/setup-python@v2+ with:+ python-version: ${{ matrix.python-version }}++ - name: Install system libraries+ if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned')+ run: |+ sudo apt-get update+ sudo apt-get install libxml2-dev libxslt-dev++ - name: Run tests+ env: ${{ matrix.env }}+ run: |+ if [[ ! -z "$PYPY_VERSION" ]]; then+ export PYPY_VERSION="pypy$PYPY_VERSION-linux64"+ wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"+ tar -jxf ${PYPY_VERSION}.tar.bz2+ $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION"+ source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"+ fi+ pip install -U tox+ tox++ - name: Upload coverage report+ run: bash <(curl -s https://codecov.io/bash)diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.ymlnew file mode 100644index 000000000..ed2e4075d--- /dev/null+++ b/.github/workflows/tests-windows.yml@@ -0,0 +1,32 @@+name: Windows+on: [push, pull_request]++jobs:+ tests:+ runs-on: windows-latest+ strategy:+ matrix:+ include:+ - python-version: 3.6+ env:+ TOXENV: windows-pinned+ - python-version: 3.7+ env:+ TOXENV: py+ - python-version: 3.8+ env:+ TOXENV: py++ steps:+ - uses: actions/checkout@v2++ - name: Set up Python ${{ matrix.python-version }}+ uses: actions/setup-python@v2+ with:+ python-version: ${{ matrix.python-version }}++ - name: Run tests+ env: ${{ matrix.env }}+ run: |+ pip install -U tox+ toxdiff --git a/.travis.yml b/.travis.ymldeleted file mode 100644index 880c8772b..000000000--- a/.travis.yml+++ /dev/null@@ -1,77 +0,0 @@-language: python-dist: xenial-branches:- only:- - master- - /^\d\.\d+$/- - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/-matrix:- include:- - env: TOXENV=security- python: 3.8- - env: TOXENV=flake8- python: 3.8- - env: TOXENV=pylint- python: 3.8- - env: TOXENV=docs- python: 3.7 # Keep in sync with .readthedocs.yml- - env: TOXENV=typing- python: 3.8-- - env: TOXENV=pinned- python: 3.6.1- - env: TOXENV=asyncio-pinned- python: 3.6.1- - env: TOXENV=pypy3-pinned PYPY_VERSION=3.6-v7.2.0- dist: bionic-- - env: TOXENV=py- python: 3.6- - env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1- dist: bionic-- - env: TOXENV=py- python: 3.7-- - env: TOXENV=py PYPI_RELEASE_JOB=true- python: 3.8- dist: bionic- - env: TOXENV=extra-deps- python: 3.8- dist: bionic- - env: TOXENV=asyncio- python: 3.8- dist: bionic-install:- - |- if [[ ! -z "$PYPY_VERSION" ]]; then- export PYPY_VERSION="pypy$PYPY_VERSION-linux64"- wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"- tar -jxf ${PYPY_VERSION}.tar.bz2- virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"- source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"- fi- - pip install -U tox twine wheel codecov--script: tox-after_success:- - codecov-notifications:- irc:- use_notice: true- skip_join: true- channels:- - irc.freenode.org#scrapy-cache:- directories:- - $HOME/.cache/pip-deploy:- provider: pypi- distributions: "sdist bdist_wheel"- user: scrapy- password:- secure: JaAKcy1AXWXDK3LXdjOtKyaVPCSFoCGCnW15g4f65E/8Fsi9ZzDfmBa4Equs3IQb/vs/if2SVrzJSr7arN7r9Z38Iv1mUXHkFAyA3Ym8mThfABBzzcUWEQhIHrCX0Tdlx9wQkkhs+PZhorlmRS4gg5s6DzPaeA2g8SCgmlRmFfA=- on:- tags: true- repo: scrapy/scrapy- condition: "$PYPI_RELEASE_JOB == true && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"diff --git a/README.rst b/README.rstindex a8f2ba52b..9418d270f 100644--- a/README.rst+++ b/README.rst@@ -10,9 +10,17 @@ Scrapy:target: https://pypi.python.org/pypi/Scrapy:alt: Supported Python Versions-.. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg- :target: https://travis-ci.org/scrapy/scrapy- :alt: Build Status+.. image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg+ :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu+ :alt: Ubuntu++.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg+ :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS+ :alt: macOS++.. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg+ :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows+ :alt: Windows.. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg:target: https://pypi.python.org/pypi/Scrapydiff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txtindex 68b856a88..a86c4ae4f 100644--- a/tests/requirements-py3.txt+++ b/tests/requirements-py3.txt@@ -14,6 +14,6 @@ uvloop; platform_system != "Windows"# optional for shell wrapper testsbpythonbrotlipy # optional for HTTP compress downloader middleware tests-zstandard # optional for HTTP compress downloader middleware tests+zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware testsipythonpywin32; sys_platform == "win32"diff --git a/tox.ini b/tox.iniindex 6c39299bb..e70aef2d2 100644--- a/tox.ini+++ b/tox.ini@@ -13,7 +13,7 @@ deps =-rtests/requirements-py3.txt# mitmproxy does not support PyPy# mitmproxy does not support Windows when running Python < 3.7- mitmproxy; python_version >= '3.7' and implementation_name != 'pypy'+ mitmproxy >= 4.0.4; python_version >= '3.7' and implementation_name != 'pypy'mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy'# Extrasbotocore>=1.4.87@@ -25,9 +25,9 @@ passenv =GCS_TEST_FILE_URIGCS_PROJECT_ID#allow tox virtualenv to upgrade pip/wheel/setuptools-download = true+download = truecommands =- py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests}+ py.test --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests}[testenv:typing]basepython = python3@@ -87,10 +87,6 @@ deps =botocore==1.4.87google-cloud-storage==1.29.0Pillow==4.0.0-install_command =- # --use-feature=2020-resolver is required, otherwise the latest verion of- # Twisted gets installed.- pip install --use-feature=2020-resolver {opts} {packages}setenv =_SCRAPY_PINNED=true@@ -99,11 +95,9 @@ deps ={[pinned]deps}lxml==3.5.0PyDispatcher==2.0.5-install_command =- {[pinned]install_command}setenv ={[pinned]setenv}-+[testenv:windows-pinned]basepython = python3deps =@@ -112,8 +106,6 @@ deps =# not need to build lxml from sources in a CI Windows job:lxml==3.8.0PyDispatcher==2.0.5-install_command =- {[pinned]install_command}setenv ={[pinned]setenv}@@ -122,10 +114,6 @@ deps ={[testenv]deps}reppyrobotexclusionrulesparser-install_command =- # Test --use-feature=2020-resolver for the latest version of all- # dependencies.- pip install --use-feature=2020-resolver {opts} {packages}[testenv:asyncio]commands =@@ -133,8 +121,6 @@ commands =[testenv:asyncio-pinned]deps = {[testenv:pinned]deps}-install_command =- {[pinned]install_command}commands = {[testenv:asyncio]commands}setenv ={[pinned]setenv}@@ -150,8 +136,6 @@ deps ={[pinned]deps}lxml==4.0.0PyPyDispatcher==2.1.0-install_command =- {[pinned]install_command}commands = {[testenv:pypy3]commands}setenv ={[pinned]setenv}