Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Integrating configs into pyproject.toml (#6547)

Swayam Gupta8c23da943c5e
diff --git a/.bandit.yml b/.bandit.yml
deleted file mode 100644
index b7f1817e0..000000000
--- a/.bandit.yml
+++ /dev/null
@@ -1,7 +0,0 @@
-skips:
-- B101 # assert_used, needed for mypy
-- B321 # ftplib, https://github.com/scrapy/scrapy/issues/4180
-- B402 # import_ftplib, https://github.com/scrapy/scrapy/issues/4180
-- B411 # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082
-- B503 # ssl_with_bad_defaults
-exclude_dirs: ['tests']
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
deleted file mode 100644
index f83e3e890..000000000
--- a/.bumpversion.cfg
+++ /dev/null
@@ -1,11 +0,0 @@
-[bumpversion]
-current_version = 2.12.0
-commit = True
-tag = True
-tag_name = {new_version}
-
-[bumpversion:file:scrapy/VERSION]
-
-[bumpversion:file:SECURITY.md]
-parse = (?P<major>\d+)\.(?P<minor>\d+)\.x
-serialize = {major}.{minor}.x
diff --git a/.coveragerc b/.coveragerc
deleted file mode 100644
index f9ad353d5..000000000
--- a/.coveragerc
+++ /dev/null
@@ -1,12 +0,0 @@
-[run]
-branch = true
-include = scrapy/*
-omit =
- tests/*
-disable_warnings = include-ignored
-
-[report]
-# https://github.com/nedbat/coveragepy/issues/831#issuecomment-517778185
-exclude_lines =
- pragma: no cover
- if TYPE_CHECKING:
diff --git a/.isort.cfg b/.isort.cfg
deleted file mode 100644
index f238bf7ea..000000000
--- a/.isort.cfg
+++ /dev/null
@@ -1,2 +0,0 @@
-[settings]
-profile = black
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index fbd710f6f..b411f4927 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -3,7 +3,8 @@ repos:
rev: 1.7.9
hooks:
- id: bandit
- args: [-r, -c, .bandit.yml]
+ args: ["-c", "pyproject.toml"]
+ additional_dependencies: ["bandit[toml]"]
- repo: https://github.com/PyCQA/flake8
rev: 7.1.0
hooks:
diff --git a/MANIFEST.in b/MANIFEST.in
index 06971e39c..7700ae7bd 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -10,7 +10,6 @@ include scrapy/py.typed
include codecov.yml
include conftest.py
-include pytest.ini
include tox.ini
recursive-include scrapy/templates *
diff --git a/pylintrc b/pylintrc
deleted file mode 100644
index e927b903c..000000000
--- a/pylintrc
+++ /dev/null
@@ -1,73 +0,0 @@
-[MASTER]
-persistent=no
-jobs=1 # >1 hides results
-
-[MESSAGES CONTROL]
-disable=abstract-method,
- arguments-differ,
- arguments-renamed,
- attribute-defined-outside-init,
- bad-classmethod-argument,
- bare-except,
- broad-except,
- broad-exception-raised,
- c-extension-no-member,
- consider-using-with,
- cyclic-import,
- dangerous-default-value,
- disallowed-name,
- duplicate-code, # https://github.com/PyCQA/pylint/issues/214
- eval-used,
- fixme,
- function-redefined,
- global-statement,
- implicit-str-concat,
- import-error,
- import-outside-toplevel,
- inherit-non-class,
- invalid-name,
- invalid-overridden-method,
- isinstance-second-argument-not-valid-type,
- keyword-arg-before-vararg,
- line-too-long,
- logging-format-interpolation,
- logging-fstring-interpolation,
- logging-not-lazy,
- lost-exception,
- missing-docstring,
- no-member,
- no-method-argument,
- no-name-in-module,
- no-self-argument,
- no-value-for-parameter, # https://github.com/pylint-dev/pylint/issues/3268
- not-callable,
- pointless-statement,
- pointless-string-statement,
- protected-access,
- raise-missing-from,
- redefined-builtin,
- redefined-outer-name,
- reimported,
- signature-differs,
- too-few-public-methods,
- too-many-ancestors,
- too-many-arguments,
- too-many-branches,
- too-many-format-args,
- too-many-function-args,
- too-many-instance-attributes,
- too-many-lines,
- too-many-locals,
- too-many-public-methods,
- too-many-return-statements,
- unbalanced-tuple-unpacking,
- unnecessary-dunder-call,
- unnecessary-pass,
- unreachable,
- unused-argument,
- unused-import,
- unused-variable,
- used-before-assignment,
- useless-return,
- wildcard-import,
- wrong-import-position
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 000000000..f25715e76
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,235 @@
+[build-system]
+requires = ["setuptools >= 61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "Scrapy"
+dynamic = ["version"]
+description = "A high-level Web Crawling and Web Scraping framework"
+dependencies = [
+ "Twisted>=21.7.0",
+ "cryptography>=37.0.0",
+ "cssselect>=0.9.1",
+ "itemloaders>=1.0.1",
+ "parsel>=1.5.0",
+ "pyOpenSSL>=22.0.0",
+ "queuelib>=1.4.2",
+ "service_identity>=18.1.0",
+ "w3lib>=1.17.0",
+ "zope.interface>=5.1.0",
+ "protego>=0.1.15",
+ "itemadapter>=0.1.0",
+ "packaging",
+ "tldextract",
+ "lxml>=4.6.0",
+ "defusedxml>=0.7.1",
+ # Platform-specific dependencies
+ 'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"',
+ 'PyPyDispatcher>=2.1.0; platform_python_implementation == "PyPy"',
+]
+classifiers = [
+ "Framework :: Scrapy",
+ "Development Status :: 5 - Production/Stable",
+ "Environment :: Console",
+ "Intended Audience :: Developers",
+ "License :: OSI Approved :: BSD License",
+ "Operating System :: OS Independent",
+ "Programming Language :: Python",
+ "Programming Language :: Python :: 3",
+ "Programming Language :: Python :: 3.9",
+ "Programming Language :: Python :: 3.10",
+ "Programming Language :: Python :: 3.11",
+ "Programming Language :: Python :: 3.12",
+ "Programming Language :: Python :: 3.13",
+ "Programming Language :: Python :: Implementation :: CPython",
+ "Programming Language :: Python :: Implementation :: PyPy",
+ "Topic :: Internet :: WWW/HTTP",
+ "Topic :: Software Development :: Libraries :: Application Frameworks",
+ "Topic :: Software Development :: Libraries :: Python Modules",
+]
+readme = "README.rst"
+requires-python = ">=3.9"
+authors = [{ name = "Scrapy developers", email = "[email protected]" }]
+maintainers = [{ name = "Pablo Hoffman", email = "[email protected]" }]
+
+[project.urls]
+Homepage = "https://scrapy.org/"
+Documentation = "https://docs.scrapy.org/"
+Source = "https://github.com/scrapy/scrapy"
+Tracker = "https://github.com/scrapy/scrapy/issues"
+Changelog = "https://github.com/scrapy/scrapy/commits/master/"
+releasenotes = "https://docs.scrapy.org/en/latest/news.html"
+
+[project.scripts]
+scrapy = "scrapy.cmdline:execute"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["scrapy", "scrapy.*",]
+
+[tool.setuptools.dynamic]
+version = {file = "./scrapy/VERSION"}
+
+[tool.mypy]
+ignore_missing_imports = true
+
+# Interface classes are hard to support
+
+[[tool.mypy.overrides]]
+module = "twisted.internet.interfaces"
+follow_imports = "skip"
+
+[[tool.mypy.overrides]]
+module = "scrapy.interfaces"
+ignore_errors = true
+
+[[tool.mypy.overrides]]
+module = "twisted.internet.reactor"
+follow_imports = "skip"
+
+# FIXME: remove the following section once the issues are solved
+[[tool.mypy.overrides]]
+module = "scrapy.settings.default_settings"
+ignore_errors = true
+
+[tool.bandit]
+skips = [
+ "B101", # assert_used, needed for mypy
+ "B321", # ftplib, https://github.com/scrapy/scrapy/issues/4180
+ "B402", # import_ftplib, https://github.com/scrapy/scrapy/issues/4180
+ "B411", # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082
+ "B503", # ssl_with_bad_defaults
+]
+exclude_dirs = ["tests"]
+
+[tool.bumpversion]
+current_version = "2.12.0"
+commit = true
+tag = true
+tag_name = "{new_version}"
+
+[[tool.bumpversion.files]]
+filename = "scrapy/VERSION"
+
+[[tool.bumpversion.files]]
+filename = "SECURITY.md"
+parse = """(?P<major>0|[1-9]\\d*)\\.(?P<minor>0|[1-9]\\d*)"""
+serialize = ["{major}.{minor}"]
+
+[tool.coverage.run]
+branch = true
+include = ["scrapy/*"]
+omit = ["tests/*"]
+disable_warnings = ["include-ignored"]
+
+[tool.coverage.report]
+# https://github.com/nedbat/coveragepy/issues/831#issuecomment-517778185
+exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:"]
+
+[tool.isort]
+profile = "black"
+
+[tool.pylint.MASTER]
+persistent = "no"
+jobs = 1 # >1 hides results
+
+[tool.pylint."MESSAGES CONTROL"]
+disable = [
+ "abstract-method",
+ "arguments-differ",
+ "arguments-renamed",
+ "attribute-defined-outside-init",
+ "bad-classmethod-argument",
+ "bare-except",
+ "broad-except",
+ "broad-exception-raised",
+ "c-extension-no-member",
+ "consider-using-with",
+ "cyclic-import",
+ "dangerous-default-value",
+ "disallowed-name",
+ "duplicate-code", # https://github.com/PyCQA/pylint/issues/214
+ "eval-used",
+ "fixme",
+ "function-redefined",
+ "global-statement",
+ "implicit-str-concat",
+ "import-error",
+ "import-outside-toplevel",
+ "inherit-non-class",
+ "invalid-name",
+ "invalid-overridden-method",
+ "isinstance-second-argument-not-valid-type",
+ "keyword-arg-before-vararg",
+ "line-too-long",
+ "logging-format-interpolation",
+ "logging-fstring-interpolation",
+ "logging-not-lazy",
+ "lost-exception",
+ "missing-docstring",
+ "no-member",
+ "no-method-argument",
+ "no-name-in-module",
+ "no-self-argument",
+ "no-value-for-parameter", # https://github.com/pylint-dev/pylint/issues/3268
+ "not-callable",
+ "pointless-statement",
+ "pointless-string-statement",
+ "protected-access",
+ "raise-missing-from",
+ "redefined-builtin",
+ "redefined-outer-name",
+ "reimported",
+ "signature-differs",
+ "too-few-public-methods",
+ "too-many-ancestors",
+ "too-many-arguments",
+ "too-many-branches",
+ "too-many-format-args",
+ "too-many-function-args",
+ "too-many-instance-attributes",
+ "too-many-lines",
+ "too-many-locals",
+ "too-many-public-methods",
+ "too-many-return-statements",
+ "unbalanced-tuple-unpacking",
+ "unnecessary-dunder-call",
+ "unnecessary-pass",
+ "unreachable",
+ "unused-argument",
+ "unused-import",
+ "unused-variable",
+ "used-before-assignment",
+ "useless-return",
+ "wildcard-import",
+ "wrong-import-position",
+]
+
+[tool.pytest.ini_options]
+xfail_strict = true
+usefixtures = "chdir"
+python_files = ["test_*.py", "__init__.py"]
+python_classes = []
+addopts = [
+ "--assert=plain",
+ "--ignore=docs/_ext",
+ "--ignore=docs/conf.py",
+ "--ignore=docs/news.rst",
+ "--ignore=docs/topics/dynamic-content.rst",
+ "--ignore=docs/topics/items.rst",
+ "--ignore=docs/topics/leaks.rst",
+ "--ignore=docs/topics/loaders.rst",
+ "--ignore=docs/topics/selectors.rst",
+ "--ignore=docs/topics/shell.rst",
+ "--ignore=docs/topics/stats.rst",
+ "--ignore=docs/topics/telnetconsole.rst",
+ "--ignore=docs/utils",
+]
+markers = [
+ "only_asyncio: marks tests as only enabled when --reactor=asyncio is passed",
+ "only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed",
+ "requires_uvloop: marks tests as only enabled when uvloop is known to be working",
+ "requires_botocore: marks tests that need botocore (but not boto3)",
+ "requires_boto3: marks tests that need botocore and boto3",
+]
+filterwarnings = []
\ No newline at end of file
diff --git a/pytest.ini b/pytest.ini
deleted file mode 100644
index 824c0e9e9..000000000
--- a/pytest.ini
+++ /dev/null
@@ -1,26 +0,0 @@
-[pytest]
-xfail_strict = true
-usefixtures = chdir
-python_files=test_*.py __init__.py
-python_classes=
-addopts =
- --assert=plain
- --ignore=docs/_ext
- --ignore=docs/conf.py
- --ignore=docs/news.rst
- --ignore=docs/topics/dynamic-content.rst
- --ignore=docs/topics/items.rst
- --ignore=docs/topics/leaks.rst
- --ignore=docs/topics/loaders.rst
- --ignore=docs/topics/selectors.rst
- --ignore=docs/topics/shell.rst
- --ignore=docs/topics/stats.rst
- --ignore=docs/topics/telnetconsole.rst
- --ignore=docs/utils
-markers =
- only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
- only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
- requires_uvloop: marks tests as only enabled when uvloop is known to be working
- requires_botocore: marks tests that need botocore (but not boto3)
- requires_boto3: marks tests that need botocore and boto3
-filterwarnings =
diff --git a/setup.cfg b/setup.cfg
deleted file mode 100644
index 151e784c6..000000000
--- a/setup.cfg
+++ /dev/null
@@ -1,24 +0,0 @@
-[bdist_rpm]
-doc_files = docs AUTHORS INSTALL LICENSE README.rst
-
-[bdist_wheel]
-universal=1
-
-[mypy]
-ignore_missing_imports = true
-
-# Interface classes are hard to support
-
-[mypy-twisted.internet.interfaces]
-follow_imports = skip
-
-[mypy-scrapy.interfaces]
-ignore_errors = True
-
-[mypy-twisted.internet.reactor]
-follow_imports = skip
-
-# FIXME: remove the following sections once the issues are solved
-
-[mypy-scrapy.settings.default_settings]
-ignore_errors = True
diff --git a/setup.py b/setup.py
deleted file mode 100644
index 6cc1150a5..000000000
--- a/setup.py
+++ /dev/null
@@ -1,75 +0,0 @@
-from pathlib import Path
-
-from setuptools import find_packages, setup
-
-version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip()
-
-
-install_requires = [
- "Twisted>=21.7.0",
- "cryptography>=37.0.0",
- "cssselect>=0.9.1",
- "itemloaders>=1.0.1",
- "parsel>=1.5.0",
- "pyOpenSSL>=22.0.0",
- "queuelib>=1.4.2",
- "service_identity>=18.1.0",
- "w3lib>=1.17.0",
- "zope.interface>=5.1.0",
- "protego>=0.1.15",
- "itemadapter>=0.1.0",
- "packaging",
- "tldextract",
- "lxml>=4.6.0",
- "defusedxml>=0.7.1",
-]
-extras_require = {
- ':platform_python_implementation == "CPython"': ["PyDispatcher>=2.0.5"],
- ':platform_python_implementation == "PyPy"': ["PyPyDispatcher>=2.1.0"],
-}
-
-
-setup(
- name="Scrapy",
- version=version,
- url="https://scrapy.org",
- project_urls={
- "Documentation": "https://docs.scrapy.org/",
- "Source": "https://github.com/scrapy/scrapy",
- "Tracker": "https://github.com/scrapy/scrapy/issues",
- },
- description="A high-level Web Crawling and Web Scraping framework",
- long_description=open("README.rst", encoding="utf-8").read(),
- author="Scrapy developers",
- author_email="[email protected]",
- maintainer="Pablo Hoffman",
- maintainer_email="[email protected]",
- license="BSD",
- packages=find_packages(exclude=("tests", "tests.*")),
- include_package_data=True,
- zip_safe=False,
- entry_points={"console_scripts": ["scrapy = scrapy.cmdline:execute"]},
- classifiers=[
- "Framework :: Scrapy",
- "Development Status :: 5 - Production/Stable",
- "Environment :: Console",
- "Intended Audience :: Developers",
- "License :: OSI Approved :: BSD License",
- "Operating System :: OS Independent",
- "Programming Language :: Python",
- "Programming Language :: Python :: 3",
- "Programming Language :: Python :: 3.9",
- "Programming Language :: Python :: 3.10",
- "Programming Language :: Python :: 3.11",
- "Programming Language :: Python :: 3.12",
- "Programming Language :: Python :: 3.13",
- "Programming Language :: Python :: Implementation :: CPython",
- "Programming Language :: Python :: Implementation :: PyPy",
- "Topic :: Internet :: WWW/HTTP",
- "Topic :: Software Development :: Libraries :: Application Frameworks",
- "Topic :: Software Development :: Libraries :: Python Modules",
- ],
- python_requires=">=3.9",
- install_requires=install_requires,
- extras_require=extras_require,
-)
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 853acf2de..a77531f62 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -899,7 +899,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
p.expect_exact("shutting down gracefully")
# sending the second signal too fast often causes problems
d = Deferred()
- reactor.callLater(0.1, d.callback, None)
+ reactor.callLater(0.01, d.callback, None)
yield d
p.kill(sig)
p.expect_exact("forcing unclean shutdown")
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index d2ff9ba48..9b53b9b96 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -144,9 +144,10 @@ class SpiderLoaderTest(unittest.TestCase):
self.assertRaises(SyntaxError, SpiderLoader.from_settings, settings)
def test_syntax_error_warning(self):
- with warnings.catch_warnings(record=True) as w, mock.patch.object(
- SpiderLoader, "_load_spiders"
- ) as m:
+ with (
+ warnings.catch_warnings(record=True) as w,
+ mock.patch.object(SpiderLoader, "_load_spiders") as m,
+ ):
m.side_effect = SyntaxError
module = "tests.test_spiderloader.test_spiders.spider1"
settings = Settings(
diff --git a/tox.ini b/tox.ini
index 5783a0e61..4e1a99473 100644
--- a/tox.ini
+++ b/tox.ini
@@ -79,7 +79,7 @@ deps =
{[testenv:extra-deps]deps}
pylint==3.2.5
commands =
- pylint conftest.py docs extras scrapy setup.py tests
+ pylint conftest.py docs extras scrapy tests
[testenv:twinecheck]
basepython = python3