🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(spiders/requests): Use a reliable method for fingerprint generation
Karim shoair122923c80e48
diff --git a/scrapling/spiders/checkpoint.py b/scrapling/spiders/checkpoint.pyindex ae795dc..4c511ae 100644--- a/scrapling/spiders/checkpoint.py+++ b/scrapling/spiders/checkpoint.py@@ -17,7 +17,7 @@ class CheckpointData:"""Container for checkpoint state."""requests: List["Request"] = field(default_factory=list)- seen: Set[str] = field(default_factory=set)+ seen: Set[bytes] = field(default_factory=set)class CheckpointManager:diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.pyindex 2a812be..c57a4b4 100644--- a/scrapling/spiders/engine.py+++ b/scrapling/spiders/engine.py@@ -34,7 +34,11 @@ class CrawlerEngine:):self.spider = spiderself.session_manager = session_manager- self.scheduler = Scheduler()+ self.scheduler = Scheduler(+ include_kwargs=spider.fp_include_kwargs,+ include_headers=spider.fp_include_headers,+ keep_fragments=spider.fp_keep_fragments,+ )self.stats = CrawlStats()self._global_limiter = CapacityLimiter(spider.concurrent_requests)@@ -72,6 +76,15 @@ class CrawlerEngine:return self._domain_limiters[domain]return self._global_limiter+ def _normalize_request(self, request: Request) -> None:+ """Normalize request fields before enqueueing.++ Resolves empty sid to the session manager's default session ID.+ This ensures consistent fingerprinting for requests using the same session.+ """+ if not request.sid:+ request.sid = self.session_manager.default_session_id+async def _process_request(self, request: Request) -> None:"""Download and process a single request."""async with self._rate_limiter(request.domain):@@ -101,6 +114,7 @@ class CrawlerEngine:retry_request.priority -= 1 # Don't retry immediatelyretry_request.dont_filter = Truenew_request = await self.spider.retry_blocked_request(retry_request, response)+ self._normalize_request(new_request)await self.scheduler.enqueue(new_request)log.info(f"Scheduled blocked request for retry ({retry_request._retry_count}/{self.spider.max_blocked_retries}): {request.url}"@@ -114,6 +128,7 @@ class CrawlerEngine:async for result in callback(response):if isinstance(result, Request):if self._is_domain_allowed(result):+ self._normalize_request(result)await self.scheduler.enqueue(result)else:self.stats.offsite_requests_count += 1@@ -223,6 +238,7 @@ class CrawlerEngine:try:if not resuming:async for request in self.spider.start_requests():+ self._normalize_request(request)await self.scheduler.enqueue(request)else:log.info("Resuming from checkpoint, skipping start_requests()")diff --git a/scrapling/spiders/request.py b/scrapling/spiders/request.pyindex e2fb588..977e8e8 100644--- a/scrapling/spiders/request.py+++ b/scrapling/spiders/request.py@@ -1,12 +1,27 @@-from urllib.parse import urlparse+import hashlib+from io import BytesIO+from functools import cached_property+from urllib.parse import urlparse, urlencode++import orjson+from w3lib.url import canonicalize_urlfrom scrapling.engines.toolbelt.custom import Response-from scrapling.core._types import Any, AsyncGenerator, Callable, Dict, Union, TYPE_CHECKING+from scrapling.core._types import Any, AsyncGenerator, Callable, Dict, Union, Tuple, TYPE_CHECKINGif TYPE_CHECKING:from scrapling.spiders.spider import Spider+def _convert_to_bytes(value: str | bytes) -> bytes:+ if isinstance(value, bytes):+ return value+ if not isinstance(value, str):+ raise TypeError(f"Can't convert {type(value).__name__} to bytes")++ return value.encode(encoding="utf-8", errors="ignore")++class Request:def __init__(self,@@ -27,6 +42,7 @@ class Request:self.meta: dict[str, Any] = meta if meta else {}self._retry_count: int = _retry_countself._session_kwargs = kwargs if kwargs else {}+ self._fp = Nonedef copy(self) -> "Request":"""Create a copy of this request."""@@ -41,15 +57,60 @@ class Request:**self._session_kwargs,)- @property+ @cached_propertydef domain(self) -> str:return urlparse(self.url).netloc- @property- def _fp(self) -> str:- """Generate a unique fingerprint for deduplication."""- # TODO: Improve fingerprint- return f"{self.sid}:{self.url}"+ def update_fingerprint(+ self,+ include_kwargs: bool = False,+ include_headers: bool = False,+ keep_fragments: bool = False,+ ) -> bytes:+ """Generate a unique fingerprint for deduplication.++ Caches the result in self._fp after first computation.+ """++ post_data = self._session_kwargs.get("data", {})+ body = b""+ if post_data:+ if isinstance(post_data, dict | list | tuple):+ body = urlencode(post_data).encode()+ elif isinstance(post_data, str):+ body = post_data.encode()+ elif isinstance(post_data, BytesIO):+ body = post_data.getvalue()+ elif isinstance(post_data, bytes):+ body = post_data+ elif post_data is None:+ body = b""+ else:+ post_data = self._session_kwargs.get("json", {})+ body: bytes = orjson.dumps(post_data) if post_data else b""++ data: Dict[str, str | Tuple] = {+ "sid": self.sid,+ "body": body.hex(),+ "method": self._session_kwargs.get("method", "GET"),+ "url": canonicalize_url(self.url, keep_fragments=keep_fragments),+ }++ if include_kwargs:+ kwargs = (key.lower() for key in self._session_kwargs.keys() if key.lower() not in ("data", "json"))+ data["kwargs"] = "".join(set(_convert_to_bytes(key).hex() for key in kwargs))++ if include_headers:+ headers = self._session_kwargs.get("headers") or self._session_kwargs.get("extra_headers") or {}+ processed_headers = {}+ # Some header normalization+ for key, value in headers.items():+ processed_headers[_convert_to_bytes(key.lower()).hex()] = _convert_to_bytes(value.lower()).hex()+ data["headers"] = tuple(processed_headers.items())++ fp = hashlib.sha1(orjson.dumps(data, option=orjson.OPT_SORT_KEYS), usedforsecurity=False).digest()+ self._fp = fp+ return fpdef __repr__(self) -> str:callback_name = getattr(self.callback, "__name__", None) or "None"@@ -74,6 +135,8 @@ class Request:"""Requests are equal if they have the same fingerprint."""if not isinstance(other, Request):return NotImplemented+ if self._fp is None or other._fp is None:+ raise RuntimeError("Cannot compare requests before generating their fingerprints!")return self._fp == other._fpdef __getstate__(self) -> dict[str, Any]:diff --git a/scrapling/spiders/scheduler.py b/scrapling/spiders/scheduler.pyindex e36472e..aca3277 100644--- a/scrapling/spiders/scheduler.py+++ b/scrapling/spiders/scheduler.py@@ -17,16 +17,19 @@ class Scheduler:Duplicate URLs are filtered unless dont_filter=True."""- def __init__(self):+ def __init__(self, include_kwargs: bool = False, include_headers: bool = False, keep_fragments: bool = False):self._queue: asyncio.PriorityQueue[tuple[int, int, Request]] = asyncio.PriorityQueue()- self._seen: set[str] = set()+ self._seen: set[bytes] = set()self._counter = count()# Mirror dict for snapshot without draining queueself._pending: dict[int, tuple[int, int, Request]] = {}+ self._include_kwargs = include_kwargs+ self._include_headers = include_headers+ self._keep_fragments = keep_fragmentsasync def enqueue(self, request: Request) -> bool:"""Add a request to the queue."""- fingerprint = request._fp+ fingerprint = request.update_fingerprint(self._include_kwargs, self._include_headers, self._keep_fragments)if not request.dont_filter and fingerprint in self._seen:log.debug("Dropped duplicate request: %s", request)@@ -54,7 +57,7 @@ class Scheduler:def is_empty(self) -> bool:return self._queue.empty()- def snapshot(self) -> Tuple[List[Request], Set[str]]:+ def snapshot(self) -> Tuple[List[Request], Set[bytes]]:"""Create a snapshot of the current state for checkpoints."""sorted_items = sorted(self._pending.values(), key=lambda x: (x[0], x[1])) # Maintain queue orderrequests = [item[2] for item in sorted_items]diff --git a/scrapling/spiders/spider.py b/scrapling/spiders/spider.pyindex 465bd4f..f14174f 100644--- a/scrapling/spiders/spider.py+++ b/scrapling/spiders/spider.py@@ -78,6 +78,11 @@ class Spider(ABC):download_delay: float = 0.0max_blocked_retries: int = 3+ # Fingerprint adjustments+ fp_include_kwargs = False+ fp_keep_fragments = False+ fp_include_headers = False+# Logging settingslogging_level: int = logging.DEBUGlogging_format: str = "[%(asctime)s]:({spider_name}) %(levelname)s: %(message)s"