The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
langchain[patch]: Allow specifying other hashing functions in embeddings (#31561)
Eugene Yurtsevd10fd02bb33d
diff --git a/libs/langchain/langchain/embeddings/cache.py b/libs/langchain/langchain/embeddings/cache.pyindex 163fd94268..401d17b6fb 100644--- a/libs/langchain/langchain/embeddings/cache.py+++ b/libs/langchain/langchain/embeddings/cache.py@@ -12,9 +12,9 @@ from __future__ import annotationsimport hashlibimport jsonimport uuid+import warningsfrom collections.abc import Sequence-from functools import partial-from typing import Callable, Optional, Union, cast+from typing import Callable, Literal, Optional, Union, castfrom langchain_core.embeddings import Embeddingsfrom langchain_core.stores import BaseStore, ByteStore@@ -25,20 +25,51 @@ from langchain.storage.encoder_backed import EncoderBackedStoreNAMESPACE_UUID = uuid.UUID(int=1985)-def _hash_string_to_uuid(input_string: str) -> uuid.UUID:- """Hash a string and returns the corresponding UUID."""- hash_value = hashlib.sha1(input_string.encode("utf-8")).hexdigest()- return uuid.uuid5(NAMESPACE_UUID, hash_value)+def _sha1_hash_to_uuid(text: str) -> uuid.UUID:+ """Return a UUID derived from *text* using SHA‑1 (deterministic).+ Deterministic and fast, **but not collision‑resistant**.-def _key_encoder(key: str, namespace: str) -> str:- """Encode a key."""- return namespace + str(_hash_string_to_uuid(key))+ A malicious attacker could try to create two different texts that hash to the same+ UUID. This may not necessarily be an issue in the context of caching embeddings,+ but new applications should swap this out for a stronger hash function like+ xxHash, BLAKE2 or SHA‑256, which are collision-resistant.+ """+ sha1_hex = hashlib.sha1(text.encode("utf-8")).hexdigest()+ # Embed the hex string in `uuid5` to obtain a valid UUID.+ return uuid.uuid5(NAMESPACE_UUID, sha1_hex)+++def _make_default_key_encoder(namespace: str, algorithm: str) -> Callable[[str], str]:+ """Create a default key encoder function.++ Args:+ namespace: Prefix that segregates keys from different embedding models.+ algorithm:+ * `sha1` - fast but not collision‑resistant+ * `blake2b` - cryptographically strong, faster than SHA‑1+ * `sha256` - cryptographically strong, slower than SHA‑1+ * `sha512` - cryptographically strong, slower than SHA‑1+ Returns:+ A function that encodes a key using the specified algorithm.+ """+ if algorithm == "sha1":+ _warn_about_sha1_encoder()++ def _key_encoder(key: str) -> str:+ """Encode a key using the specified algorithm."""+ if algorithm == "sha1":+ return f"{namespace}{_sha1_hash_to_uuid(key)}"+ if algorithm == "blake2b":+ return f"{namespace}{hashlib.blake2b(key.encode('utf-8')).hexdigest()}"+ if algorithm == "sha256":+ return f"{namespace}{hashlib.sha256(key.encode('utf-8')).hexdigest()}"+ if algorithm == "sha512":+ return f"{namespace}{hashlib.sha512(key.encode('utf-8')).hexdigest()}"+ raise ValueError(f"Unsupported algorithm: {algorithm}")-def _create_key_encoder(namespace: str) -> Callable[[str], str]:- """Create an encoder for a key."""- return partial(_key_encoder, namespace=namespace)+ return _key_encoderdef _value_serializer(value: Sequence[float]) -> bytes:@@ -51,6 +82,28 @@ def _value_deserializer(serialized_value: bytes) -> list[float]:return cast(list[float], json.loads(serialized_value.decode()))+# The warning is global; track emission, so it appears only once.+_warned_about_sha1: bool = False+++def _warn_about_sha1_encoder() -> None:+ """Emit a one‑time warning about SHA‑1 collision weaknesses."""+ global _warned_about_sha1+ if not _warned_about_sha1:+ warnings.warn(+ "Using default key encoder: SHA‑1 is *not* collision‑resistant. "+ "While acceptable for most cache scenarios, a motivated attacker "+ "can craft two different payloads that map to the same cache key. "+ "If that risk matters in your environment, supply a stronger "+ "encoder (e.g. SHA‑256 or BLAKE2) via the `key_encoder` argument. "+ "If you change the key encoder, consider also creating a new cache, "+ "to avoid (the potential for) collisions with existing keys.",+ category=UserWarning,+ stacklevel=2,+ )+ _warned_about_sha1 = True++class CacheBackedEmbeddings(Embeddings):"""Interface for caching results from embedding models.@@ -234,6 +287,9 @@ class CacheBackedEmbeddings(Embeddings):namespace: str = "",batch_size: Optional[int] = None,query_embedding_cache: Union[bool, ByteStore] = False,+ key_encoder: Union[+ Callable[[str], str], Literal["sha1", "blake2b", "sha256", "sha512"]+ ] = "sha1",) -> CacheBackedEmbeddings:"""On-ramp that adds the necessary serialization and encoding to the store.@@ -248,9 +304,39 @@ class CacheBackedEmbeddings(Embeddings):query_embedding_cache: The cache to use for storing query embeddings.True to use the same cache as document embeddings.False to not cache query embeddings.+ key_encoder: Optional callable to encode keys. If not provided,+ a default encoder using SHA‑1 will be used. SHA-1 is not+ collision-resistant, and a motivated attacker could craft two+ different texts that hash to the same cache key.++ New applications should use one of the alternative encoders+ or provide a custom and strong key encoder function to avoid this risk.++ If you change a key encoder in an existing cache, consider+ just creating a new cache, to avoid (the potential for)+ collisions with existing keys or having duplicate keys+ for the same text in the cache.++ Returns:+ An instance of CacheBackedEmbeddings that uses the provided cache."""- namespace = namespace- key_encoder = _create_key_encoder(namespace)+ if isinstance(key_encoder, str):+ key_encoder = _make_default_key_encoder(namespace, key_encoder)+ elif callable(key_encoder):+ # If a custom key encoder is provided, it should not be used with a+ # namespace.+ # A user can handle namespacing in directly their custom key encoder.+ if namespace:+ raise ValueError(+ "Do not supply `namespace` when using a custom key_encoder; "+ "add any prefixing inside the encoder itself."+ )+ else:+ raise ValueError(+ "key_encoder must be either 'blake2b', 'sha1', 'sha256', 'sha512' "+ "or a callable that encodes keys."+ )+document_embedding_store = EncoderBackedStore[str, list[float]](document_embedding_cache,key_encoder,diff --git a/libs/langchain/tests/unit_tests/embeddings/test_caching.py b/libs/langchain/tests/unit_tests/embeddings/test_caching.pyindex b9545d052b..7cb2d68f10 100644--- a/libs/langchain/tests/unit_tests/embeddings/test_caching.py+++ b/libs/langchain/tests/unit_tests/embeddings/test_caching.py@@ -1,5 +1,9 @@"""Embeddings tests."""+import hashlib+import importlib+import warnings+import pytestfrom langchain_core.embeddings import Embeddings@@ -146,3 +150,89 @@ async def test_aembed_query_cached(keys = list(cache_embeddings_with_query.query_embedding_store.yield_keys()) # type: ignore[union-attr]assert len(keys) == 1assert keys[0] == "test_namespace89ec3dae-a4d9-5636-a62e-ff3b56cdfa15"+++def test_blake2b_encoder() -> None:+ """Test that the blake2b encoder is used to encode keys in the cache store."""+ store = InMemoryStore()+ emb = MockEmbeddings()+ cbe = CacheBackedEmbeddings.from_bytes_store(+ emb, store, namespace="ns_", key_encoder="blake2b"+ )++ text = "blake"+ cbe.embed_documents([text])++ # rebuild the key exactly as the library does+ expected_key = "ns_" + hashlib.blake2b(text.encode()).hexdigest()+ assert list(cbe.document_embedding_store.yield_keys()) == [expected_key]+++def test_sha256_encoder() -> None:+ """Test that the sha256 encoder is used to encode keys in the cache store."""+ store = InMemoryStore()+ emb = MockEmbeddings()+ cbe = CacheBackedEmbeddings.from_bytes_store(+ emb, store, namespace="ns_", key_encoder="sha256"+ )++ text = "foo"+ cbe.embed_documents([text])++ # rebuild the key exactly as the library does+ expected_key = "ns_" + hashlib.sha256(text.encode()).hexdigest()+ assert list(cbe.document_embedding_store.yield_keys()) == [expected_key]+++def test_sha512_encoder() -> None:+ """Test that the sha512 encoder is used to encode keys in the cache store."""+ store = InMemoryStore()+ emb = MockEmbeddings()+ cbe = CacheBackedEmbeddings.from_bytes_store(+ emb, store, namespace="ns_", key_encoder="sha512"+ )++ text = "foo"+ cbe.embed_documents([text])++ # rebuild the key exactly as the library does+ expected_key = "ns_" + hashlib.sha512(text.encode()).hexdigest()+ assert list(cbe.document_embedding_store.yield_keys()) == [expected_key]+++def test_sha1_warning_emitted_once() -> None:+ """Test that a warning is emitted when using SHA‑1 as the default key encoder."""+ module = importlib.import_module(CacheBackedEmbeddings.__module__)++ # Create a *temporary* MonkeyPatch object whose effects disappear+ # automatically when the with‑block exits.+ with pytest.MonkeyPatch.context() as mp:+ # We're monkey patching the module to reset the `_warned_about_sha1` flag+ # which may have been set while testing other parts of the codebase.+ mp.setattr(module, "_warned_about_sha1", False, raising=False)++ store = InMemoryStore()+ emb = MockEmbeddings()++ with warnings.catch_warnings(record=True) as caught:+ warnings.simplefilter("always")+ CacheBackedEmbeddings.from_bytes_store(emb, store) # triggers warning+ CacheBackedEmbeddings.from_bytes_store(emb, store) # silent++ sha1_msgs = [w for w in caught if "SHA‑1" in str(w.message)]+ assert len(sha1_msgs) == 1+++def test_custom_encoder() -> None:+ """Test that a custom encoder can be used to encode keys in the cache store."""+ store = InMemoryStore()+ emb = MockEmbeddings()++ def custom_upper(text: str) -> str: # very simple demo encoder+ return "CUSTOM_" + text.upper()++ cbe = CacheBackedEmbeddings.from_bytes_store(emb, store, key_encoder=custom_upper)+ txt = "x"+ cbe.embed_documents([txt])++ assert list(cbe.document_embedding_store.yield_keys()) == ["CUSTOM_X"]