Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Remove InitSpider.

Andrey Rakhmatullin8ecfd20fcd6e
diff --git a/pyproject.toml b/pyproject.toml
index d35b333ff..ea7893e2a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -126,7 +126,6 @@ ignore_errors = true
[[tool.mypy.overrides]]
module = [
"scrapy.core.downloader.webclient",
- "scrapy.spiders.init",
"tests.test_webclient",
]
allow_any_generics = true
@@ -138,7 +137,6 @@ warn_return_any = false
# usually no type hints
[[tool.mypy.overrides]]
module = [
-# "IPython.*",
"bpython",
"brotli",
"brotlicffi",
diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py
deleted file mode 100644
index 957bfffd3..000000000
--- a/scrapy/spiders/init.py
+++ /dev/null
@@ -1,64 +0,0 @@
-from __future__ import annotations
-
-import warnings
-from typing import TYPE_CHECKING, Any, cast
-
-from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.spiders import Spider
-from scrapy.utils.spider import iterate_spider_output
-
-if TYPE_CHECKING:
- from collections.abc import AsyncIterator, Iterable
-
- from scrapy import Request
- from scrapy.http import Response
-
-
-class InitSpider(Spider):
- """Base Spider with initialization facilities
-
- .. warning:: This class is deprecated. Copy its code into your project if needed.
- It will be removed in a future Scrapy version.
- """
-
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
- warnings.warn(
- "InitSpider is deprecated. Copy its code from Scrapy's source if needed. "
- "Will be removed in a future version.",
- ScrapyDeprecationWarning,
- stacklevel=2,
- )
-
- async def start(self) -> AsyncIterator[Any]:
- with warnings.catch_warnings():
- warnings.filterwarnings(
- "ignore", category=ScrapyDeprecationWarning, module=r"^scrapy\.spiders$"
- )
- for item_or_request in self.start_requests():
- yield item_or_request
-
- def start_requests(self) -> Iterable[Request]:
- self._postinit_reqs: Iterable[Request] = super().start_requests()
- return cast("Iterable[Request]", iterate_spider_output(self.init_request()))
-
- def initialized(self, response: Response | None = None) -> Any:
- """This method must be set as the callback of your last initialization
- request. See self.init_request() docstring for more info.
- """
- return self.__dict__.pop("_postinit_reqs")
-
- def init_request(self) -> Any:
- """This function should return one initialization request, with the
- self.initialized method as callback. When the self.initialized method
- is called this spider is considered initialized. If you need to perform
- several requests for initializing your spider, you can do so by using
- different callbacks. The only requirement is that the final callback
- (of the last initialization request) must be self.initialized.
-
- The default implementation calls self.initialized immediately, and
- means that no initialization is needed. This method should be
- overridden only when you need to perform requests to initialize your
- spider
- """
- return self.initialized()
diff --git a/tests/test_spider.py b/tests/test_spider.py
index ff9aed74f..23efed77a 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -11,10 +11,9 @@ from scrapy.crawler import Crawler
from scrapy.http import Response, TextResponse, XmlResponse
from scrapy.settings import Settings
from scrapy.spiders import CSVFeedSpider, Spider, XMLFeedSpider
-from scrapy.spiders.init import InitSpider
from scrapy.utils.test import get_crawler, get_reactor_settings
from tests import get_testdata
-from tests.utils.decorators import coroutine_test, inline_callbacks_test
+from tests.utils.decorators import inline_callbacks_test
class TestSpider:
@@ -122,27 +121,6 @@ class TestSpider:
mock_logger.log.assert_called_once_with("INFO", "test log msg")
[email protected]("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-class TestInitSpider(TestSpider):
- spider_class = InitSpider
-
- @coroutine_test
- async def test_start_urls(self):
- responses = []
-
- class TestSpider(self.spider_class):
- name = "test"
- start_urls = ["data:,"]
-
- async def parse(self, response):
- responses.append(response)
-
- crawler = get_crawler(TestSpider)
- await crawler.crawl_async()
- assert len(responses) == 1
- assert responses[0].url == "data:,"
-
-
class TestXMLFeedSpider(TestSpider):
spider_class = XMLFeedSpider