Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Support Python 3.9 (#4759)

vinayak42e4dbb23de2
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index 2748bf5fe..02c647da9 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -7,19 +7,21 @@ jobs:
strategy:
matrix:
include:
- - python-version: 3.8
+ - python-version: 3.9
env:
TOXENV: security
- - python-version: 3.8
+ - python-version: 3.9
env:
TOXENV: flake8
+ # Pylint requires installing reppy, which does not support Python 3.9
+ # https://github.com/seomoz/reppy/issues/122
- python-version: 3.8
env:
TOXENV: pylint
- - python-version: 3.8
+ - python-version: 3.9
env:
TOXENV: typing
- - python-version: 3.7 # Keep in sync with .readthedocs.yml
+ - python-version: 3.8 # Keep in sync with .readthedocs.yml
env:
TOXENV: docs
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index aec6b8696..b48066ea4 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -9,10 +9,10 @@ jobs:
steps:
- uses: actions/checkout@v2
- - name: Set up Python 3.8
+ - name: Set up Python 3.9
uses: actions/setup-python@v2
with:
- python-version: 3.8
+ python-version: 3.9
- name: Check Tag
id: check-release-tag
diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml
index 51d27c405..4f8f7a19d 100644
--- a/.github/workflows/tests-macos.yml
+++ b/.github/workflows/tests-macos.yml
@@ -6,7 +6,7 @@ jobs:
runs-on: macos-10.15
strategy:
matrix:
- python-version: [3.6, 3.7, 3.8]
+ python-version: [3.6, 3.7, 3.8, 3.9]
steps:
- uses: actions/checkout@v2
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 89c0334e2..df5ee9d69 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -13,6 +13,9 @@ jobs:
- python-version: 3.8
env:
TOXENV: py
+ - python-version: 3.9
+ env:
+ TOXENV: py
- python-version: pypy3
env:
TOXENV: pypy3
@@ -31,10 +34,12 @@ jobs:
PYPY_VERSION: 3.6-v7.2.0
# extras
+ # extra-deps includes reppy, which does not support Python 3.9
+ # https://github.com/seomoz/reppy/issues/122
- python-version: 3.8
env:
TOXENV: extra-deps
- - python-version: 3.8
+ - python-version: 3.9
env:
TOXENV: asyncio
diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml
index ed2e4075d..5459a845b 100644
--- a/.github/workflows/tests-windows.yml
+++ b/.github/workflows/tests-windows.yml
@@ -16,6 +16,10 @@ jobs:
- python-version: 3.8
env:
TOXENV: py
+ # https://twistedmatrix.com/trac/ticket/9990
+ #- python-version: 3.9
+ #env:
+ #TOXENV: py
steps:
- uses: actions/checkout@v2
diff --git a/.readthedocs.yml b/.readthedocs.yml
index e4d3f02cc..80a1cd036 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -3,10 +3,14 @@ formats: all
sphinx:
configuration: docs/conf.py
fail_on_warning: true
+
+build:
+ image: latest
+
python:
# For available versions, see:
# https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image
- version: 3.7 # Keep in sync with .travis.yml
+ version: 3.8 # Keep in sync with .github/workflows/checks.yml
install:
- requirements: docs/requirements.txt
- path: .
diff --git a/setup.py b/setup.py
index b5c42a3c2..cf9261271 100644
--- a/setup.py
+++ b/setup.py
@@ -85,6 +85,7 @@ setup(
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
'Programming Language :: Python :: 3.8',
+ 'Programming Language :: Python :: 3.9',
'Programming Language :: Python :: Implementation :: CPython',
'Programming Language :: Python :: Implementation :: PyPy',
'Topic :: Internet :: WWW/HTTP',
diff --git a/tox.ini b/tox.ini
index 69f52bd9f..9815f80f7 100644
--- a/tox.ini
+++ b/tox.ini
@@ -13,7 +13,9 @@ deps =
-rtests/requirements-py3.txt
# mitmproxy does not support PyPy
# mitmproxy does not support Windows when running Python < 3.7
- mitmproxy >= 4.0.4; python_version >= '3.7' and implementation_name != 'pypy'
+ # Python 3.9+ requires https://github.com/mitmproxy/mitmproxy/commit/8e5e43de24c9bc93092b63efc67fbec029a9e7fe
+ mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy'
+ mitmproxy >= 4.0.4; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy'
mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy'
# Extras
botocore>=1.4.87