Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix Python 3.8.

Andrey Rakhmatullin40e4a5960477
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index 38dbe8135..d59005edd 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -7,7 +7,7 @@ For more info see docs/topics/link-extractors.rst
"""
import re
-from typing import Iterable
+from typing import Iterable, Pattern
# common file extensions that are not followed if they occur in links
IGNORED_EXTENSIONS = [
@@ -111,7 +111,7 @@ IGNORED_EXTENSIONS = [
]
-def _matches(url: str, regexs: Iterable[re.Pattern[str]]) -> bool:
+def _matches(url: str, regexs: Iterable[Pattern[str]]) -> bool:
return any(r.search(url) for r in regexs)
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 3fa7d1e3a..33a10cd6c 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -5,7 +5,18 @@ Link extractor based on lxml.html
import logging
import operator
from functools import partial
-from typing import Any, Callable, Iterable, List, Optional, Set, Tuple, Union, cast
+from typing import (
+ Any,
+ Callable,
+ Iterable,
+ List,
+ Optional,
+ Pattern,
+ Set,
+ Tuple,
+ Union,
+ cast,
+)
from urllib.parse import urljoin, urlparse
from lxml import etree # nosec
@@ -146,7 +157,7 @@ class LxmlParserLinkExtractor:
return links
-_RegexT = Union[str, re.Pattern[str]]
+_RegexT = Union[str, Pattern[str]]
_RegexOrSeveralT = Union[_RegexT, Iterable[_RegexT]]
@@ -179,8 +190,8 @@ class LxmlLinkExtractor:
strip=strip,
canonicalized=not canonicalize,
)
- self.allow_res: List[re.Pattern[str]] = self._compile_regexes(allow)
- self.deny_res: List[re.Pattern[str]] = self._compile_regexes(deny)
+ self.allow_res: List[Pattern[str]] = self._compile_regexes(allow)
+ self.deny_res: List[Pattern[str]] = self._compile_regexes(deny)
self.allow_domains: Set[str] = set(arg_to_iter(allow_domains))
self.deny_domains: Set[str] = set(arg_to_iter(deny_domains))
@@ -194,10 +205,10 @@ class LxmlLinkExtractor:
deny_extensions = IGNORED_EXTENSIONS
self.canonicalize: bool = canonicalize
self.deny_extensions: Set[str] = {"." + e for e in arg_to_iter(deny_extensions)}
- self.restrict_text: List[re.Pattern[str]] = self._compile_regexes(restrict_text)
+ self.restrict_text: List[Pattern[str]] = self._compile_regexes(restrict_text)
@staticmethod
- def _compile_regexes(value: Optional[_RegexOrSeveralT]) -> List[re.Pattern[str]]:
+ def _compile_regexes(value: Optional[_RegexOrSeveralT]) -> List[Pattern[str]]:
return [
x if isinstance(x, re.Pattern) else re.compile(x)
for x in arg_to_iter(value)