Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Explicitly mark re-exports. (#6579)

Andrey Rakhmatullin4d31277bc671
diff --git a/pyproject.toml b/pyproject.toml
index 8c985753f..571a61f1c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -72,9 +72,9 @@ version = {file = "./scrapy/VERSION"}
[tool.mypy]
ignore_missing_imports = true
+implicit_reexport = false
# Interface classes are hard to support
-
[[tool.mypy.overrides]]
module = "twisted.internet.interfaces"
follow_imports = "skip"
@@ -92,6 +92,14 @@ follow_imports = "skip"
module = "scrapy.settings.default_settings"
ignore_errors = true
+[[tool.mypy.overrides]]
+module = "itemadapter"
+implicit_reexport = true
+
+[[tool.mypy.overrides]]
+module = "twisted"
+implicit_reexport = true
+
[tool.bumpversion]
current_version = "2.12.0"
commit = true
@@ -359,13 +367,9 @@ ignore = [
]
[tool.ruff.lint.per-file-ignores]
-# Exclude files that are meant to provide top-level imports
-"scrapy/__init__.py" = ["E402"]
-"scrapy/core/downloader/handlers/http.py" = ["F401"]
-"scrapy/http/__init__.py" = ["F401"]
-"scrapy/linkextractors/__init__.py" = ["E402", "F401"]
-"scrapy/selector/__init__.py" = ["F401"]
-"scrapy/spiders/__init__.py" = ["E402", "F401"]
+# Circular import workarounds
+"scrapy/linkextractors/__init__.py" = ["E402"]
+"scrapy/spiders/__init__.py" = ["E402"]
# Skip bandit in tests
"tests/**" = ["S"]
diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py
index 52535bd8b..93b96c779 100644
--- a/scrapy/core/downloader/handlers/http.py
+++ b/scrapy/core/downloader/handlers/http.py
@@ -2,3 +2,8 @@ from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
from scrapy.core.downloader.handlers.http11 import (
HTTP11DownloadHandler as HTTPDownloadHandler,
)
+
+__all__ = [
+ "HTTP10DownloadHandler",
+ "HTTPDownloadHandler",
+]
diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.py
index d0b726bad..0e5c2b53b 100644
--- a/scrapy/http/__init__.py
+++ b/scrapy/http/__init__.py
@@ -15,3 +15,16 @@ from scrapy.http.response.html import HtmlResponse
from scrapy.http.response.json import JsonResponse
from scrapy.http.response.text import TextResponse
from scrapy.http.response.xml import XmlResponse
+
+__all__ = [
+ "FormRequest",
+ "Headers",
+ "HtmlResponse",
+ "JsonRequest",
+ "JsonResponse",
+ "Request",
+ "Response",
+ "TextResponse",
+ "XmlResponse",
+ "XmlRpcRequest",
+]
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index 1c7e96ae0..b39859f7b 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -126,3 +126,8 @@ def _is_valid_url(url: str) -> bool:
# Top-level imports
from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor
+
+__all__ = [
+ "IGNORED_EXTENSIONS",
+ "LinkExtractor",
+]
diff --git a/scrapy/selector/__init__.py b/scrapy/selector/__init__.py
index 85c500d66..7cfa3c364 100644
--- a/scrapy/selector/__init__.py
+++ b/scrapy/selector/__init__.py
@@ -4,3 +4,8 @@ Selectors
# top-level imports
from scrapy.selector.unified import Selector, SelectorList
+
+__all__ = [
+ "Selector",
+ "SelectorList",
+]
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 6136dabc7..e255e91cc 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -117,3 +117,12 @@ class Spider(object_ref):
from scrapy.spiders.crawl import CrawlSpider, Rule
from scrapy.spiders.feed import CSVFeedSpider, XMLFeedSpider
from scrapy.spiders.sitemap import SitemapSpider
+
+__all__ = [
+ "CSVFeedSpider",
+ "CrawlSpider",
+ "Rule",
+ "SitemapSpider",
+ "Spider",
+ "XMLFeedSpider",
+]
diff --git a/tests/test_item.py b/tests/test_item.py
index 5a8ee095e..480412841 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -1,7 +1,8 @@
import unittest
+from abc import ABCMeta
from unittest import mock
-from scrapy.item import ABCMeta, Field, Item, ItemMeta
+from scrapy.item import Field, Item, ItemMeta
class ItemTest(unittest.TestCase):
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 94a59f883..314082742 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -6,7 +6,7 @@ import pytest
from scrapy.linkextractors import IGNORED_EXTENSIONS
from scrapy.spiders import Spider
from scrapy.utils.misc import arg_to_iter
-from scrapy.utils.url import (
+from scrapy.utils.url import ( # type: ignore[attr-defined]
_is_filesystem_path,
_public_w3lib_objects,
add_http_if_no_scheme,