Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add RequestFingerprinterProtocol.

Andrey Rakhmatullin9df67a554e54
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index bc912268c..d2639104b 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -12,7 +12,11 @@ from scrapy.settings import BaseSettings
from scrapy.spiders import Spider
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.job import job_dir
-from scrapy.utils.request import RequestFingerprinter, referer_str
+from scrapy.utils.request import (
+ RequestFingerprinter,
+ RequestFingerprinterProtocol,
+ referer_str,
+)
if TYPE_CHECKING:
# typing.Self requires Python 3.11
@@ -48,10 +52,10 @@ class RFPDupeFilter(BaseDupeFilter):
path: Optional[str] = None,
debug: bool = False,
*,
- fingerprinter: Optional[RequestFingerprinter] = None,
+ fingerprinter: Optional[RequestFingerprinterProtocol] = None,
) -> None:
self.file = None
- self.fingerprinter: RequestFingerprinter = (
+ self.fingerprinter: RequestFingerprinterProtocol = (
fingerprinter or RequestFingerprinter()
)
self.fingerprints: Set[str] = set()
@@ -68,7 +72,7 @@ class RFPDupeFilter(BaseDupeFilter):
cls,
settings: BaseSettings,
*,
- fingerprinter: Optional[RequestFingerprinter] = None,
+ fingerprinter: Optional[RequestFingerprinterProtocol] = None,
) -> Self:
debug = settings.getbool("DUPEFILTER_DEBUG")
try:
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 6c7f3b345..24fcbd85e 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -14,6 +14,7 @@ from typing import (
Iterable,
List,
Optional,
+ Protocol,
Tuple,
Type,
Union,
@@ -230,6 +231,11 @@ def fingerprint(
return cache[cache_key]
+class RequestFingerprinterProtocol(Protocol):
+ def fingerprint(self, request: Request) -> bytes:
+ ...
+
+
class RequestFingerprinter:
"""Default fingerprinter.