Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix Python 3.8.
Andrey Rakhmatullin40e4a5960477
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.pyindex 38dbe8135..d59005edd 100644--- a/scrapy/linkextractors/__init__.py+++ b/scrapy/linkextractors/__init__.py@@ -7,7 +7,7 @@ For more info see docs/topics/link-extractors.rst"""import re-from typing import Iterable+from typing import Iterable, Pattern# common file extensions that are not followed if they occur in linksIGNORED_EXTENSIONS = [@@ -111,7 +111,7 @@ IGNORED_EXTENSIONS = []-def _matches(url: str, regexs: Iterable[re.Pattern[str]]) -> bool:+def _matches(url: str, regexs: Iterable[Pattern[str]]) -> bool:return any(r.search(url) for r in regexs)diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.pyindex 3fa7d1e3a..33a10cd6c 100644--- a/scrapy/linkextractors/lxmlhtml.py+++ b/scrapy/linkextractors/lxmlhtml.py@@ -5,7 +5,18 @@ Link extractor based on lxml.htmlimport loggingimport operatorfrom functools import partial-from typing import Any, Callable, Iterable, List, Optional, Set, Tuple, Union, cast+from typing import (+ Any,+ Callable,+ Iterable,+ List,+ Optional,+ Pattern,+ Set,+ Tuple,+ Union,+ cast,+)from urllib.parse import urljoin, urlparsefrom lxml import etree # nosec@@ -146,7 +157,7 @@ class LxmlParserLinkExtractor:return links-_RegexT = Union[str, re.Pattern[str]]+_RegexT = Union[str, Pattern[str]]_RegexOrSeveralT = Union[_RegexT, Iterable[_RegexT]]@@ -179,8 +190,8 @@ class LxmlLinkExtractor:strip=strip,canonicalized=not canonicalize,)- self.allow_res: List[re.Pattern[str]] = self._compile_regexes(allow)- self.deny_res: List[re.Pattern[str]] = self._compile_regexes(deny)+ self.allow_res: List[Pattern[str]] = self._compile_regexes(allow)+ self.deny_res: List[Pattern[str]] = self._compile_regexes(deny)self.allow_domains: Set[str] = set(arg_to_iter(allow_domains))self.deny_domains: Set[str] = set(arg_to_iter(deny_domains))@@ -194,10 +205,10 @@ class LxmlLinkExtractor:deny_extensions = IGNORED_EXTENSIONSself.canonicalize: bool = canonicalizeself.deny_extensions: Set[str] = {"." + e for e in arg_to_iter(deny_extensions)}- self.restrict_text: List[re.Pattern[str]] = self._compile_regexes(restrict_text)+ self.restrict_text: List[Pattern[str]] = self._compile_regexes(restrict_text)@staticmethod- def _compile_regexes(value: Optional[_RegexOrSeveralT]) -> List[re.Pattern[str]]:+ def _compile_regexes(value: Optional[_RegexOrSeveralT]) -> List[Pattern[str]]:return [x if isinstance(x, re.Pattern) else re.compile(x)for x in arg_to_iter(value)