Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix: str(value)

yetval334b0f553832
diff --git a/scrapling/spiders/request.py b/scrapling/spiders/request.py
index faca233..ef766b2 100644
--- a/scrapling/spiders/request.py
+++ b/scrapling/spiders/request.py
@@ -22,6 +22,13 @@ def _convert_to_bytes(value: str | bytes) -> bytes:
return value.encode(encoding="utf-8", errors="ignore")
+def _stable_value_repr(value: Any) -> str:
+ try:
+ return orjson.dumps(value, option=orjson.OPT_SORT_KEYS, default=repr).decode()
+ except TypeError:
+ return repr(value)
+
+
class Request:
def __init__(
self,
@@ -98,7 +105,7 @@ class Request:
if include_kwargs:
filtered_kwargs = {
- key.lower(): str(value)
+ key.lower(): _stable_value_repr(value)
for key, value in self._session_kwargs.items()
if key.lower() not in ("data", "json")
}
diff --git a/tests/spiders/test_request.py b/tests/spiders/test_request.py
index f54f2cd..00f6c49 100644
--- a/tests/spiders/test_request.py
+++ b/tests/spiders/test_request.py
@@ -106,6 +106,24 @@ class TestRequestProperties:
assert r1.update_fingerprint(include_kwargs=True) != r2.update_fingerprint(include_kwargs=True)
+ def test_fingerprint_include_kwargs_handles_non_primitive_values(self):
+ class _Opaque:
+ def __repr__(self) -> str:
+ return "_Opaque(stable)"
+
+ opaque = _Opaque()
+ r1 = Request("https://example.com", proxies={"http": "p1"}, custom=opaque)
+ r2 = Request("https://example.com", proxies={"http": "p1"}, custom=opaque)
+ r3 = Request("https://example.com", proxies={"http": "p2"}, custom=opaque)
+
+ fp1 = r1.update_fingerprint(include_kwargs=True)
+ r2._fp = None
+ fp2 = r2.update_fingerprint(include_kwargs=True)
+ fp3 = r3.update_fingerprint(include_kwargs=True)
+
+ assert fp1 == fp2
+ assert fp1 != fp3
+
def test_fingerprint_include_headers_preserves_header_value_case(self):
"""Test header values are fingerprinted without lowercasing."""
r1 = Request("https://example.com", headers={"X-Test": "A"})