Scrapy, a fast high-level web crawling & scraping framework for Python.
Add RequestFingerprinterProtocol.
Andrey Rakhmatullin9df67a554e54
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.pyindex bc912268c..d2639104b 100644--- a/scrapy/dupefilters.py+++ b/scrapy/dupefilters.py@@ -12,7 +12,11 @@ from scrapy.settings import BaseSettingsfrom scrapy.spiders import Spiderfrom scrapy.utils.deprecate import ScrapyDeprecationWarningfrom scrapy.utils.job import job_dir-from scrapy.utils.request import RequestFingerprinter, referer_str+from scrapy.utils.request import (+ RequestFingerprinter,+ RequestFingerprinterProtocol,+ referer_str,+)if TYPE_CHECKING:# typing.Self requires Python 3.11@@ -48,10 +52,10 @@ class RFPDupeFilter(BaseDupeFilter):path: Optional[str] = None,debug: bool = False,*,- fingerprinter: Optional[RequestFingerprinter] = None,+ fingerprinter: Optional[RequestFingerprinterProtocol] = None,) -> None:self.file = None- self.fingerprinter: RequestFingerprinter = (+ self.fingerprinter: RequestFingerprinterProtocol = (fingerprinter or RequestFingerprinter())self.fingerprints: Set[str] = set()@@ -68,7 +72,7 @@ class RFPDupeFilter(BaseDupeFilter):cls,settings: BaseSettings,*,- fingerprinter: Optional[RequestFingerprinter] = None,+ fingerprinter: Optional[RequestFingerprinterProtocol] = None,) -> Self:debug = settings.getbool("DUPEFILTER_DEBUG")try:diff --git a/scrapy/utils/request.py b/scrapy/utils/request.pyindex 6c7f3b345..24fcbd85e 100644--- a/scrapy/utils/request.py+++ b/scrapy/utils/request.py@@ -14,6 +14,7 @@ from typing import (Iterable,List,Optional,+ Protocol,Tuple,Type,Union,@@ -230,6 +231,11 @@ def fingerprint(return cache[cache_key]+class RequestFingerprinterProtocol(Protocol):+ def fingerprint(self, request: Request) -> bytes:+ ...++class RequestFingerprinter:"""Default fingerprinter.