Scrapy, a fast high-level web crawling & scraping framework for Python.
Integrating configs into pyproject.toml (#6547)
Swayam Gupta8c23da943c5e
diff --git a/.bandit.yml b/.bandit.ymldeleted file mode 100644index b7f1817e0..000000000--- a/.bandit.yml+++ /dev/null@@ -1,7 +0,0 @@-skips:-- B101 # assert_used, needed for mypy-- B321 # ftplib, https://github.com/scrapy/scrapy/issues/4180-- B402 # import_ftplib, https://github.com/scrapy/scrapy/issues/4180-- B411 # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082-- B503 # ssl_with_bad_defaults-exclude_dirs: ['tests']diff --git a/.bumpversion.cfg b/.bumpversion.cfgdeleted file mode 100644index f83e3e890..000000000--- a/.bumpversion.cfg+++ /dev/null@@ -1,11 +0,0 @@-[bumpversion]-current_version = 2.12.0-commit = True-tag = True-tag_name = {new_version}--[bumpversion:file:scrapy/VERSION]--[bumpversion:file:SECURITY.md]-parse = (?P<major>\d+)\.(?P<minor>\d+)\.x-serialize = {major}.{minor}.xdiff --git a/.coveragerc b/.coveragercdeleted file mode 100644index f9ad353d5..000000000--- a/.coveragerc+++ /dev/null@@ -1,12 +0,0 @@-[run]-branch = true-include = scrapy/*-omit =- tests/*-disable_warnings = include-ignored--[report]-# https://github.com/nedbat/coveragepy/issues/831#issuecomment-517778185-exclude_lines =- pragma: no cover- if TYPE_CHECKING:diff --git a/.isort.cfg b/.isort.cfgdeleted file mode 100644index f238bf7ea..000000000--- a/.isort.cfg+++ /dev/null@@ -1,2 +0,0 @@-[settings]-profile = blackdiff --git a/.pre-commit-config.yaml b/.pre-commit-config.yamlindex fbd710f6f..b411f4927 100644--- a/.pre-commit-config.yaml+++ b/.pre-commit-config.yaml@@ -3,7 +3,8 @@ repos:rev: 1.7.9hooks:- id: bandit- args: [-r, -c, .bandit.yml]+ args: ["-c", "pyproject.toml"]+ additional_dependencies: ["bandit[toml]"]- repo: https://github.com/PyCQA/flake8rev: 7.1.0hooks:diff --git a/MANIFEST.in b/MANIFEST.inindex 06971e39c..7700ae7bd 100644--- a/MANIFEST.in+++ b/MANIFEST.in@@ -10,7 +10,6 @@ include scrapy/py.typedinclude codecov.ymlinclude conftest.py-include pytest.iniinclude tox.inirecursive-include scrapy/templates *diff --git a/pylintrc b/pylintrcdeleted file mode 100644index e927b903c..000000000--- a/pylintrc+++ /dev/null@@ -1,73 +0,0 @@-[MASTER]-persistent=no-jobs=1 # >1 hides results--[MESSAGES CONTROL]-disable=abstract-method,- arguments-differ,- arguments-renamed,- attribute-defined-outside-init,- bad-classmethod-argument,- bare-except,- broad-except,- broad-exception-raised,- c-extension-no-member,- consider-using-with,- cyclic-import,- dangerous-default-value,- disallowed-name,- duplicate-code, # https://github.com/PyCQA/pylint/issues/214- eval-used,- fixme,- function-redefined,- global-statement,- implicit-str-concat,- import-error,- import-outside-toplevel,- inherit-non-class,- invalid-name,- invalid-overridden-method,- isinstance-second-argument-not-valid-type,- keyword-arg-before-vararg,- line-too-long,- logging-format-interpolation,- logging-fstring-interpolation,- logging-not-lazy,- lost-exception,- missing-docstring,- no-member,- no-method-argument,- no-name-in-module,- no-self-argument,- no-value-for-parameter, # https://github.com/pylint-dev/pylint/issues/3268- not-callable,- pointless-statement,- pointless-string-statement,- protected-access,- raise-missing-from,- redefined-builtin,- redefined-outer-name,- reimported,- signature-differs,- too-few-public-methods,- too-many-ancestors,- too-many-arguments,- too-many-branches,- too-many-format-args,- too-many-function-args,- too-many-instance-attributes,- too-many-lines,- too-many-locals,- too-many-public-methods,- too-many-return-statements,- unbalanced-tuple-unpacking,- unnecessary-dunder-call,- unnecessary-pass,- unreachable,- unused-argument,- unused-import,- unused-variable,- used-before-assignment,- useless-return,- wildcard-import,- wrong-import-positiondiff --git a/pyproject.toml b/pyproject.tomlnew file mode 100644index 000000000..f25715e76--- /dev/null+++ b/pyproject.toml@@ -0,0 +1,235 @@+[build-system]+requires = ["setuptools >= 61.0"]+build-backend = "setuptools.build_meta"++[project]+name = "Scrapy"+dynamic = ["version"]+description = "A high-level Web Crawling and Web Scraping framework"+dependencies = [+ "Twisted>=21.7.0",+ "cryptography>=37.0.0",+ "cssselect>=0.9.1",+ "itemloaders>=1.0.1",+ "parsel>=1.5.0",+ "pyOpenSSL>=22.0.0",+ "queuelib>=1.4.2",+ "service_identity>=18.1.0",+ "w3lib>=1.17.0",+ "zope.interface>=5.1.0",+ "protego>=0.1.15",+ "itemadapter>=0.1.0",+ "packaging",+ "tldextract",+ "lxml>=4.6.0",+ "defusedxml>=0.7.1",+ # Platform-specific dependencies+ 'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"',+ 'PyPyDispatcher>=2.1.0; platform_python_implementation == "PyPy"',+]+classifiers = [+ "Framework :: Scrapy",+ "Development Status :: 5 - Production/Stable",+ "Environment :: Console",+ "Intended Audience :: Developers",+ "License :: OSI Approved :: BSD License",+ "Operating System :: OS Independent",+ "Programming Language :: Python",+ "Programming Language :: Python :: 3",+ "Programming Language :: Python :: 3.9",+ "Programming Language :: Python :: 3.10",+ "Programming Language :: Python :: 3.11",+ "Programming Language :: Python :: 3.12",+ "Programming Language :: Python :: 3.13",+ "Programming Language :: Python :: Implementation :: CPython",+ "Programming Language :: Python :: Implementation :: PyPy",+ "Topic :: Internet :: WWW/HTTP",+ "Topic :: Software Development :: Libraries :: Application Frameworks",+ "Topic :: Software Development :: Libraries :: Python Modules",+]+readme = "README.rst"+requires-python = ">=3.9"+authors = [{ name = "Scrapy developers", email = "[email protected]" }]+maintainers = [{ name = "Pablo Hoffman", email = "[email protected]" }]++[project.urls]+Homepage = "https://scrapy.org/"+Documentation = "https://docs.scrapy.org/"+Source = "https://github.com/scrapy/scrapy"+Tracker = "https://github.com/scrapy/scrapy/issues"+Changelog = "https://github.com/scrapy/scrapy/commits/master/"+releasenotes = "https://docs.scrapy.org/en/latest/news.html"++[project.scripts]+scrapy = "scrapy.cmdline:execute"++[tool.setuptools.packages.find]+where = ["."]+include = ["scrapy", "scrapy.*",]++[tool.setuptools.dynamic]+version = {file = "./scrapy/VERSION"}++[tool.mypy]+ignore_missing_imports = true++# Interface classes are hard to support++[[tool.mypy.overrides]]+module = "twisted.internet.interfaces"+follow_imports = "skip"++[[tool.mypy.overrides]]+module = "scrapy.interfaces"+ignore_errors = true++[[tool.mypy.overrides]]+module = "twisted.internet.reactor"+follow_imports = "skip"++# FIXME: remove the following section once the issues are solved+[[tool.mypy.overrides]]+module = "scrapy.settings.default_settings"+ignore_errors = true++[tool.bandit]+skips = [+ "B101", # assert_used, needed for mypy+ "B321", # ftplib, https://github.com/scrapy/scrapy/issues/4180+ "B402", # import_ftplib, https://github.com/scrapy/scrapy/issues/4180+ "B411", # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082+ "B503", # ssl_with_bad_defaults+]+exclude_dirs = ["tests"]++[tool.bumpversion]+current_version = "2.12.0"+commit = true+tag = true+tag_name = "{new_version}"++[[tool.bumpversion.files]]+filename = "scrapy/VERSION"++[[tool.bumpversion.files]]+filename = "SECURITY.md"+parse = """(?P<major>0|[1-9]\\d*)\\.(?P<minor>0|[1-9]\\d*)"""+serialize = ["{major}.{minor}"]++[tool.coverage.run]+branch = true+include = ["scrapy/*"]+omit = ["tests/*"]+disable_warnings = ["include-ignored"]++[tool.coverage.report]+# https://github.com/nedbat/coveragepy/issues/831#issuecomment-517778185+exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:"]++[tool.isort]+profile = "black"++[tool.pylint.MASTER]+persistent = "no"+jobs = 1 # >1 hides results++[tool.pylint."MESSAGES CONTROL"]+disable = [+ "abstract-method",+ "arguments-differ",+ "arguments-renamed",+ "attribute-defined-outside-init",+ "bad-classmethod-argument",+ "bare-except",+ "broad-except",+ "broad-exception-raised",+ "c-extension-no-member",+ "consider-using-with",+ "cyclic-import",+ "dangerous-default-value",+ "disallowed-name",+ "duplicate-code", # https://github.com/PyCQA/pylint/issues/214+ "eval-used",+ "fixme",+ "function-redefined",+ "global-statement",+ "implicit-str-concat",+ "import-error",+ "import-outside-toplevel",+ "inherit-non-class",+ "invalid-name",+ "invalid-overridden-method",+ "isinstance-second-argument-not-valid-type",+ "keyword-arg-before-vararg",+ "line-too-long",+ "logging-format-interpolation",+ "logging-fstring-interpolation",+ "logging-not-lazy",+ "lost-exception",+ "missing-docstring",+ "no-member",+ "no-method-argument",+ "no-name-in-module",+ "no-self-argument",+ "no-value-for-parameter", # https://github.com/pylint-dev/pylint/issues/3268+ "not-callable",+ "pointless-statement",+ "pointless-string-statement",+ "protected-access",+ "raise-missing-from",+ "redefined-builtin",+ "redefined-outer-name",+ "reimported",+ "signature-differs",+ "too-few-public-methods",+ "too-many-ancestors",+ "too-many-arguments",+ "too-many-branches",+ "too-many-format-args",+ "too-many-function-args",+ "too-many-instance-attributes",+ "too-many-lines",+ "too-many-locals",+ "too-many-public-methods",+ "too-many-return-statements",+ "unbalanced-tuple-unpacking",+ "unnecessary-dunder-call",+ "unnecessary-pass",+ "unreachable",+ "unused-argument",+ "unused-import",+ "unused-variable",+ "used-before-assignment",+ "useless-return",+ "wildcard-import",+ "wrong-import-position",+]++[tool.pytest.ini_options]+xfail_strict = true+usefixtures = "chdir"+python_files = ["test_*.py", "__init__.py"]+python_classes = []+addopts = [+ "--assert=plain",+ "--ignore=docs/_ext",+ "--ignore=docs/conf.py",+ "--ignore=docs/news.rst",+ "--ignore=docs/topics/dynamic-content.rst",+ "--ignore=docs/topics/items.rst",+ "--ignore=docs/topics/leaks.rst",+ "--ignore=docs/topics/loaders.rst",+ "--ignore=docs/topics/selectors.rst",+ "--ignore=docs/topics/shell.rst",+ "--ignore=docs/topics/stats.rst",+ "--ignore=docs/topics/telnetconsole.rst",+ "--ignore=docs/utils",+]+markers = [+ "only_asyncio: marks tests as only enabled when --reactor=asyncio is passed",+ "only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed",+ "requires_uvloop: marks tests as only enabled when uvloop is known to be working",+ "requires_botocore: marks tests that need botocore (but not boto3)",+ "requires_boto3: marks tests that need botocore and boto3",+]+filterwarnings = []\ No newline at end of filediff --git a/pytest.ini b/pytest.inideleted file mode 100644index 824c0e9e9..000000000--- a/pytest.ini+++ /dev/null@@ -1,26 +0,0 @@-[pytest]-xfail_strict = true-usefixtures = chdir-python_files=test_*.py __init__.py-python_classes=-addopts =- --assert=plain- --ignore=docs/_ext- --ignore=docs/conf.py- --ignore=docs/news.rst- --ignore=docs/topics/dynamic-content.rst- --ignore=docs/topics/items.rst- --ignore=docs/topics/leaks.rst- --ignore=docs/topics/loaders.rst- --ignore=docs/topics/selectors.rst- --ignore=docs/topics/shell.rst- --ignore=docs/topics/stats.rst- --ignore=docs/topics/telnetconsole.rst- --ignore=docs/utils-markers =- only_asyncio: marks tests as only enabled when --reactor=asyncio is passed- only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed- requires_uvloop: marks tests as only enabled when uvloop is known to be working- requires_botocore: marks tests that need botocore (but not boto3)- requires_boto3: marks tests that need botocore and boto3-filterwarnings =diff --git a/setup.cfg b/setup.cfgdeleted file mode 100644index 151e784c6..000000000--- a/setup.cfg+++ /dev/null@@ -1,24 +0,0 @@-[bdist_rpm]-doc_files = docs AUTHORS INSTALL LICENSE README.rst--[bdist_wheel]-universal=1--[mypy]-ignore_missing_imports = true--# Interface classes are hard to support--[mypy-twisted.internet.interfaces]-follow_imports = skip--[mypy-scrapy.interfaces]-ignore_errors = True--[mypy-twisted.internet.reactor]-follow_imports = skip--# FIXME: remove the following sections once the issues are solved--[mypy-scrapy.settings.default_settings]-ignore_errors = Truediff --git a/setup.py b/setup.pydeleted file mode 100644index 6cc1150a5..000000000--- a/setup.py+++ /dev/null@@ -1,75 +0,0 @@-from pathlib import Path--from setuptools import find_packages, setup--version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip()---install_requires = [- "Twisted>=21.7.0",- "cryptography>=37.0.0",- "cssselect>=0.9.1",- "itemloaders>=1.0.1",- "parsel>=1.5.0",- "pyOpenSSL>=22.0.0",- "queuelib>=1.4.2",- "service_identity>=18.1.0",- "w3lib>=1.17.0",- "zope.interface>=5.1.0",- "protego>=0.1.15",- "itemadapter>=0.1.0",- "packaging",- "tldextract",- "lxml>=4.6.0",- "defusedxml>=0.7.1",-]-extras_require = {- ':platform_python_implementation == "CPython"': ["PyDispatcher>=2.0.5"],- ':platform_python_implementation == "PyPy"': ["PyPyDispatcher>=2.1.0"],-}---setup(- name="Scrapy",- version=version,- url="https://scrapy.org",- project_urls={- "Documentation": "https://docs.scrapy.org/",- "Source": "https://github.com/scrapy/scrapy",- "Tracker": "https://github.com/scrapy/scrapy/issues",- },- description="A high-level Web Crawling and Web Scraping framework",- long_description=open("README.rst", encoding="utf-8").read(),- author="Scrapy developers",- author_email="[email protected]",- maintainer="Pablo Hoffman",- maintainer_email="[email protected]",- license="BSD",- packages=find_packages(exclude=("tests", "tests.*")),- include_package_data=True,- zip_safe=False,- entry_points={"console_scripts": ["scrapy = scrapy.cmdline:execute"]},- classifiers=[- "Framework :: Scrapy",- "Development Status :: 5 - Production/Stable",- "Environment :: Console",- "Intended Audience :: Developers",- "License :: OSI Approved :: BSD License",- "Operating System :: OS Independent",- "Programming Language :: Python",- "Programming Language :: Python :: 3",- "Programming Language :: Python :: 3.9",- "Programming Language :: Python :: 3.10",- "Programming Language :: Python :: 3.11",- "Programming Language :: Python :: 3.12",- "Programming Language :: Python :: 3.13",- "Programming Language :: Python :: Implementation :: CPython",- "Programming Language :: Python :: Implementation :: PyPy",- "Topic :: Internet :: WWW/HTTP",- "Topic :: Software Development :: Libraries :: Application Frameworks",- "Topic :: Software Development :: Libraries :: Python Modules",- ],- python_requires=">=3.9",- install_requires=install_requires,- extras_require=extras_require,-)diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex 853acf2de..a77531f62 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -899,7 +899,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):p.expect_exact("shutting down gracefully")# sending the second signal too fast often causes problemsd = Deferred()- reactor.callLater(0.1, d.callback, None)+ reactor.callLater(0.01, d.callback, None)yield dp.kill(sig)p.expect_exact("forcing unclean shutdown")diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.pyindex d2ff9ba48..9b53b9b96 100644--- a/tests/test_spiderloader/__init__.py+++ b/tests/test_spiderloader/__init__.py@@ -144,9 +144,10 @@ class SpiderLoaderTest(unittest.TestCase):self.assertRaises(SyntaxError, SpiderLoader.from_settings, settings)def test_syntax_error_warning(self):- with warnings.catch_warnings(record=True) as w, mock.patch.object(- SpiderLoader, "_load_spiders"- ) as m:+ with (+ warnings.catch_warnings(record=True) as w,+ mock.patch.object(SpiderLoader, "_load_spiders") as m,+ ):m.side_effect = SyntaxErrormodule = "tests.test_spiderloader.test_spiders.spider1"settings = Settings(diff --git a/tox.ini b/tox.iniindex 5783a0e61..4e1a99473 100644--- a/tox.ini+++ b/tox.ini@@ -79,7 +79,7 @@ deps ={[testenv:extra-deps]deps}pylint==3.2.5commands =- pylint conftest.py docs extras scrapy setup.py tests+ pylint conftest.py docs extras scrapy tests[testenv:twinecheck]basepython = python3