Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

15.4 KB
Raw Blame
[build-system]
requires = ["hatchling>=1.27.0"]
build-backend = "hatchling.build"

[project]
name = "Scrapy"
dynamic = ["version"]
description = "A high-level Web Crawling and Web Scraping framework"
dependencies = [
    "PyDispatcher>=2.0.5",
    "Twisted>=23.10.0",
    "aiodns>=3.3.0",
    "aioftp>=0.26.1",
    "aiohttp>=3.13.3",
    'backports.zstd>=1.3.0; python_version < "3.14"',
    "brotli>=1.2.0",
    "cryptography>=41.0.5",
    "cssselect>=1.2.0",
    "defusedxml>=0.7.1",
    "formerly>=0.1.0",
    "frostwork>=0.2.2",
    "itemadapter>=0.1.0",
    "itemloaders>=1.0.5",
    "lxml>=4.9.4",
    "packaging",
    "parsel>=1.8.1",
    "platformdirs>=2.0.0",
    "protego>=0.2.0",
    "pyOpenSSL>=24.3.0",
    "queuelib>=1.8.0",
    "service_identity>=24.2.0",
    "tldextract",
    'typing-extensions>=4.13.0; python_version < "3.13"',
    "w3lib>=2.6.0",
    "zope.interface>=5.1.0",
]
classifiers = [
    "Development Status :: 5 - Production/Stable",
    "Environment :: Console",
    "Framework :: Scrapy",
    "Intended Audience :: Developers",
    "Operating System :: OS Independent",
    "Programming Language :: Python",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3.13",
    "Programming Language :: Python :: 3.14",
    "Programming Language :: Python :: 3.15",
    "Programming Language :: Python :: Implementation :: CPython",
    "Topic :: Internet :: WWW/HTTP",
    "Topic :: Software Development :: Libraries :: Application Frameworks",
    "Topic :: Software Development :: Libraries :: Python Modules",
]
license = "BSD-3-Clause"
license-files = ["LICENSE", "AUTHORS"]
readme = "README.rst"
requires-python = ">=3.11"
authors = [{ name = "Scrapy developers", email = "[email protected]" }]
maintainers = [{ name = "Pablo Hoffman", email = "[email protected]" }]

[project.urls]
Homepage = "https://scrapy.org/"
Documentation = "https://docs.scrapy.org/"
Source = "https://github.com/scrapy/scrapy"
Tracker = "https://github.com/scrapy/scrapy/issues"
"Release notes" = "https://docs.scrapy.org/en/latest/news.html"

[project.optional-dependencies]
bpython = ["bpython>=0.7.1"]
clipboard = ["pyperclip>=1.8.0"]
color = ["colorlog>=6.6.0"]
gcs = ["google-cloud-storage>=1.30.0"]
httpx = ["httpx2[http2,socks]>=2.3.0"]
images = ["Pillow>=9.2.0"]
ipython = ["ipython>=8.15.0"]
memusage = ["psutil>=5.9.0"]
ptpython = ["ptpython>=3.0.23"]
robotparser = ["robotexclusionrulesparser>=1.6.2"]
s3 = ["aioboto3>=9.0.0", "boto3>=1.35.2"]
socks = ["python-socks[asyncio]>=2.6.0"]
twisted-http2 = ["Twisted[http2]>=23.10.0"]
uvloop = [
    "uvloop>=0.17.0; platform_system != 'Windows'",
]

[project.scripts]
scrapy = "scrapy.cmdline:execute"

[dependency-groups]
docs = [
    "h2",
    "pydantic",
    "scrapy-spider-metadata",
    "sphinx",
    "sphinx-notfound-page",
    "sphinx-reredirects",
    "sphinx-rtd-dark-mode",
    "sphinx-rtd-theme",
]

[tool.hatch.build.targets.sdist]
include = [
  "/docs",
  "/extras",
  "/scrapy",
  "/tests",
  "/tests_typing",
  "/CODE_OF_CONDUCT.md",
  "/CONTRIBUTING.md",
  "/INSTALL.md",
  "/NEWS",
  "/SECURITY.md",
  "/codecov.yml",
  "/conftest.py",
  "/tox.ini",
]

[tool.hatch.version]
path = "scrapy/VERSION"
pattern = "^(?P<version>.+)$"

[tool.mypy]
strict = true
untyped_calls_exclude = [
    "keyring",
    "twisted",
]

[[tool.mypy.overrides]]
module = "tests.*"
allow_untyped_defs = true
allow_incomplete_defs = true # 59 errors
extra_checks = false # weird addErrback() errors with untyped log.err

# Interface classes are hard to support
[[tool.mypy.overrides]]
module = "twisted.internet.interfaces"
follow_imports = "skip"

[[tool.mypy.overrides]]
module = "scrapy.interfaces"
ignore_errors = true

# just for twisted.version
[[tool.mypy.overrides]]
module = "twisted"
implicit_reexport = true

# usually no type hints
[[tool.mypy.overrides]]
module = [
    "aioboto3",
    "aiobotocore.*",
    "bpython",
    "brotli",
    "google.*",
    "ipdb",
    "psutil",
    "pydispatch.*",
    "pyftpdlib.*",
    "pyperclip",
    "pytest_twisted",
    "robotexclusionrulesparser",
    "siosocks.*",
    "zope.interface.*",
]
ignore_missing_imports = true

[tool.bumpversion]
current_version = "2.20.0"
commit = true
tag = true
tag_name = "{new_version}"
setup_hooks = ["python3 extras/pre-bump.py"]

[[tool.bumpversion.files]]
filename = "docs/news.rst"
search = "\\(unreleased\\)$"
replace = "({now:%Y-%m-%d})"
regex = true

[[tool.bumpversion.files]]
filename = "scrapy/VERSION"

[[tool.bumpversion.files]]
filename = "SECURITY.md"
parse = """(?P<major>0|[1-9]\\d*)\\.(?P<minor>0|[1-9]\\d*)"""
serialize = ["{major}.{minor}"]

[tool.coverage.run]
# sysmon, default on 3.14, is too slow: https://github.com/coveragepy/coveragepy/issues/2172
core = "ctrace"
branch = true
include = ["scrapy/*"]
omit = ["tests/*"]
disable_warnings = ["include-ignored"]
# Starting a coverage session in every spawned interpreter is what covers the
# code paths of the command-line interface. It is also the bulk of the run time
# of the tests that spawn one, so SCRAPY_COVERAGE_PATCH switches it for a cheap
# patch where that cost is not worth paying; see tox.ini.
patch = [
    "${SCRAPY_COVERAGE_PATCH-subprocess}",
]

[tool.coverage.paths]
source = [
    "scrapy",
    ".tox/**/site-packages/scrapy"
]

[tool.coverage.report]
exclude_also = [
    "@(abc\\.)?abstractmethod",
    '\A(?s:.*# pragma: no file cover.*)\Z',
]

[tool.pylint.MASTER]
persistent = "no"
jobs = 0  # run in parallel
extension-pkg-allow-list=[
    "lxml",
]
load-plugins = ["pylint_per_file_ignores"]

[tool.pylint."MESSAGES CONTROL"]
enable = [
    "useless-suppression",
]
# Make INFO checks like useless-suppression also cause pylint to return a
# non-zero exit code.
fail-on = "I"
disable = [
    # Ones we want to ignore
    "attribute-defined-outside-init",
    "broad-exception-caught",
    "consider-using-with",
    "cyclic-import",
    "disallowed-name",
    "duplicate-code",  # https://github.com/pylint-dev/pylint/issues/214
    "fixme",
    "inherit-non-class",  # false positives with formerly.deprecated_class()
    "invalid-name",
    "invalid-overridden-method",
    "isinstance-second-argument-not-valid-type",  # false positives with formerly.deprecated_class()
    "line-too-long",
    "logging-format-interpolation",
    "logging-fstring-interpolation",
    "logging-not-lazy",
    "missing-docstring",
    "no-member",
    "no-value-for-parameter",  # https://github.com/pylint-dev/pylint/issues/3268
    "not-callable",
    "protected-access",
    "redefined-outer-name",
    "too-few-public-methods",
    "too-many-ancestors",
    "too-many-arguments",
    "too-many-function-args",
    "too-many-instance-attributes",
    "too-many-lines",
    "too-many-locals",
    "too-many-positional-arguments",
    "too-many-public-methods",
    "too-many-return-statements",
    "undefined-variable",
    "unused-argument",
    "unused-variable",
    "use-implicit-booleaness-not-comparison",
    "useless-import-alias",  # used as a hint to mypy
    "useless-return",  # https://github.com/pylint-dev/pylint/issues/6530
    "wrong-import-position",

    # Ones that are implemented in ruff and need to be disabled for some lines (listed here to avoid two disable comments)
    "bare-except",
    "eval-used",
    "global-statement",
    "import-outside-toplevel",
    "import-self",
    "inconsistent-return-statements",
    "redefined-builtin",
    "too-many-branches",
    "unused-import",

    # Ones that we may want to address (fix, ignore per-line or move to "don't want to fix")
    "arguments-differ",
    "keyword-arg-before-vararg",
]
# requires `pylint_per_file_ignores` plugin
per-file-ignores = [
    # Extended list of ones that we may want to address, only for tests
    "tests/*:abstract-method,arguments-renamed,dangerous-default-value,pointless-statement,raise-missing-from,unidiomatic-typecheck,unnecessary-dunder-call,used-before-assignment",
]

[tool.pytest.ini_options]
addopts = [
    "--reactor=asyncio",
]
xfail_strict = true
asyncio_default_fixture_loop_scope = "function"
python_files = ["test_*.py", "test_*/__init__.py"]
markers = [
    "only_asyncio: marks tests that require the asyncio loop to be used",
    "only_not_asyncio: marks tests that require the asyncio loop to not be used",
    "requires_reactor: marks tests that require a reactor",
    "requires_uvloop: marks tests as only enabled when uvloop is known to be working",
    "requires_botocore: marks tests that need botocore (but not boto3)",
    "requires_aiobotocore: marks tests that need aiobotocore",
    "requires_boto3: marks tests that need botocore and boto3",
    "requires_aioboto3: marks tests that need aiobotocore and aioboto3",
    "requires_psutil: marks tests that need psutil",
    "requires_mitmproxy: marks tests that need a mitmdump executable",
    "requires_keyring: marks tests that need keyring",
    "requires_internet: marks tests that need real Internet access",
    "walltime: marks benchmarks that need CodSpeed's walltime mode to be meaningful",
]
filterwarnings = [
    # Envs with pinned old dependencies opt out of this, see tox.ini.
    "error",
    "ignore::DeprecationWarning:twisted.web.static",
    # Twisted leaves the listening socket open after a failed bind
    # (CannotListenError): https://github.com/twisted/twisted/issues/6108
    # The asyncio reactor likewise leaves client sockets and transports open on
    # teardown. These surface as unraisable exceptions during finalization when
    # collected outside any warnings-catching block, or as a plain
    # ResourceWarning, re-emitted by pytest.warns() with the outer filters
    # applied, when GC instead runs while a pytest.warns() block for an
    # unrelated warning is active, failing that unrelated test. The message
    # wording varies across Python versions, so match on the object repr.
    "ignore:.*<socket\\.socket fd=:pytest.PytestUnraisableExceptionWarning",
    "ignore:unclosed <socket\\.socket fd=:ResourceWarning",
    "ignore:.*<function (_SelectorTransport|_ProactorBasePipeTransport|_ProactorSocketTransport|BaseEventLoop)\\.__del__:pytest.PytestUnraisableExceptionWarning",
    "ignore:unclosed transport <:ResourceWarning",
    # These are triggered by garbage collecting an unclosed AiohttpDownloadHandler.
    "ignore:.*<function (ClientSession|BaseConnector)\\.__del__:pytest.PytestUnraisableExceptionWarning",
    "ignore:Unclosed (client session|connector) <aiohttp\\.:ResourceWarning",
    # itemadapter imports pydantic.v1, which warns on Python 3.14 and higher.
    "ignore:Core Pydantic V1 functionality isn't compatible:UserWarning",
    # pyftpdlib imports asynchat, removed in Python 3.12, on lower versions.
    "ignore:The async(hat|ore) module is deprecated:DeprecationWarning",
    # CI runs without coverage pass --no-cov, which pytest-cov reports. Matched
    # by message because some tox envs do not install pytest-cov, and pytest
    # warns when a filter names a module it cannot import.
    "ignore:Coverage disabled via --no-cov switch!",
    # unrecognized when pytest-asyncio isn't installed
    "ignore:Unknown config option. asyncio_:pytest.PytestConfigWarning",
]

[tool.ruff.lint]
extend-select = [
    # flake8-builtins
    "A",
    # flake8-async
    "ASYNC",
    # flake8-bugbear
    "B",
    # flake8-comprehensions
    "C4",
    # flake8-commas
    "COM",
    # pydocstyle
    "D",
    # pycodestyle
    "E4",
    "E7",
    "E9",
    # Pyflakes
    "F",
    # flake8-future-annotations
    "FA",
    # flynt
    "FLY",
    # refurb
    "FURB",
    # isort
    "I",
    # flake8-implicit-str-concat
    "ISC",
    # flake8-logging
    "LOG",
    # Perflint
    "PERF",
    # pygrep-hooks
    "PGH",
    # flake8-pie
    "PIE",
    # pylint
    "PL",
    # flake8-pytest-style
    "PT",
    # flake8-use-pathlib
    "PTH",
    # flake8-pyi
    "PYI",
    # flake8-quotes
    "Q",
    # flake8-return
    "RET",
    # flake8-raise
    "RSE",
    # Ruff-specific rules
    "RUF",
    # flake8-bandit
    "S",
    # flake8-simplify
    "SIM",
    # flake8-slots
    "SLOT",
    # flake8-debugger
    "T10",
    # flake8-type-checking
    "TC",
    # flake8-tidy-imports
    "TID",
    # pyupgrade
    "UP",
    # pycodestyle warnings
    "W",
    # flake8-2020
    "YTT",
]
ignore = [
    # Do not catch blind exception
    "BLE001",
    # Trailing comma missing
    "COM812",
    # Missing docstring in public module
    "D100",
    # Missing docstring in public class
    "D101",
    # Missing docstring in public method
    "D102",
    # Missing docstring in public function
    "D103",
    # Missing docstring in public package
    "D104",
    # Missing docstring in magic method
    "D105",
    # Missing docstring in public nested class
    "D106",
    # Missing docstring in __init__
    "D107",
    # One-line docstring should fit on one line with quotes
    "D200",
    # No blank lines allowed after function docstring
    "D202",
    # 1 blank line required between summary line and description
    "D205",
    # Multi-line docstring closing quotes should be on a separate line
    "D209",
    # First line should end with a period
    "D400",
    # First line should be in imperative mood; try rephrasing
    "D401",
    # First line should not be the function's "signature"
    "D402",
    # First word of the first line should be properly capitalized
    "D403",
    # `try`-`except` within a loop incurs performance overhead
    "PERF203",
    # Too many return statements
    "PLR0911",
    # Too many arguments in function definition
    "PLR0913",
    # Too many positional arguments
    "PLR0917",
    # Magic value used in comparison
    "PLR2004",
    # String contains ambiguous {}.
    "RUF001",
    # Docstring contains ambiguous {}.
    "RUF002",
    # Comment contains ambiguous {}.
    "RUF003",
    # Use of `assert` detected; needed for mypy
    "S101",
    # FTP-related functions are being called; https://github.com/scrapy/scrapy/issues/4180
    "S321",
    # Use a context manager for opening files
    "SIM115",
    # Yoda condition detected
    "SIM300",
    # Prefer `TypeError` exception for invalid type
    "TRY004",
]

[tool.ruff.lint.flake8-tidy-imports]
banned-module-level-imports = [
    "twisted.internet.reactor",
    # indirectly imports twisted.conch.insults.helper which imports twisted.internet.reactor
    "twisted.conch.manhole",
    # directly imports twisted.internet.reactor
    "twisted.protocols.ftp",
]

[tool.ruff.lint.isort]
split-on-trailing-comma = false

[tool.ruff.lint.per-file-ignores]
# Circular import workarounds
"scrapy/linkextractors/__init__.py" = ["E402"]
"scrapy/spiders/__init__.py" = ["E402"]

"tests/**" = [
    # Skip bandit and allow blocking file I/O in tests
    "ASYNC240",
    "S",
    # Ones that we may want to address (fix, ignore per-line or move to "don't want to fix")
    # Assigning to `os.environ` doesn't clear the environment.
    "B003",
    # Do not use mutable data structures for argument defaults.
    "B006",
    # Found useless expression.
    "B018",
    # No explicit stacklevel argument found.
    "B028",
    # Within an `except` clause, raise exceptions with `raise ... from`
    "B904",
    # `for` loop variable overwritten by assignment target
    "PLW2901",
    # Mutable class attributes should be annotated with `typing.ClassVar`
    "RUF012",
    # Use capitalized environment variable
    "SIM112",
]

# Issues pending a review:
"docs/conf.py" = ["E402"]
"scrapy/utils/url.py" = ["F403", "F405"]
"tests/test_loader.py" = ["E741"]

[tool.ruff.lint.pydocstyle]
convention = "pep257"