Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix getting annotations for _parse_sitemap() at the runtime. (#6671)

Andrey Rakhmatullin7d5b189c1147
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index 91c7e3be9..39033ac3c 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -2,6 +2,9 @@ from __future__ import annotations
import logging
import re
+
+# Iterable is needed at the run time for the SitemapSpider._parse_sitemap() annotation
+from collections.abc import Iterable, Sequence # noqa: TC003
from typing import TYPE_CHECKING, Any, cast
from scrapy.http import Request, Response, XmlResponse
@@ -11,8 +14,6 @@ from scrapy.utils.gz import gunzip, gzip_magic_number
from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
if TYPE_CHECKING:
- from collections.abc import Iterable, Sequence
-
# typing.Self requires Python 3.11
from typing_extensions import Self
diff --git a/tests/test_poet.py b/tests/test_poet.py
new file mode 100644
index 000000000..9601c75a1
--- /dev/null
+++ b/tests/test_poet.py
@@ -0,0 +1,20 @@
+"""Tests that make sure parts needed for the scrapy-poet stack work."""
+
+from typing import get_type_hints
+
+from scrapy import Spider
+from scrapy.spiders import CrawlSpider, CSVFeedSpider, SitemapSpider, XMLFeedSpider
+
+
+def test_callbacks():
+ """Making sure annotations on all non-abstract callbacks can be resolved."""
+
+ for cb in [
+ Spider._parse,
+ CrawlSpider._parse,
+ CrawlSpider._callback,
+ XMLFeedSpider._parse,
+ CSVFeedSpider._parse,
+ SitemapSpider._parse_sitemap,
+ ]:
+ get_type_hints(cb)