DSPy: The framework for programming—not prompting—language models
Python
Vendor Hasher class from datasets library in order to remove larger dependency (#8621)
Ben McHonea7566188a987
diff --git a/dspy/clients/utils_finetune.py b/dspy/clients/utils_finetune.pyindex ea234fc37..1b8b2031f 100644--- a/dspy/clients/utils_finetune.py+++ b/dspy/clients/utils_finetune.py@@ -66,7 +66,7 @@ def write_lines(file_path, data):def save_data(data: list[dict[str, Any]],) -> str:- from datasets.fingerprint import Hasher+ from dspy.utils.hasher import Hasher# Assign a unique name to the file based on the data hashhash = Hasher.hash(data)diff --git a/dspy/teleprompt/bootstrap.py b/dspy/teleprompt/bootstrap.pyindex 2b2fd8ec2..35c56eeeb 100644--- a/dspy/teleprompt/bootstrap.py+++ b/dspy/teleprompt/bootstrap.py@@ -241,7 +241,7 @@ class BootstrapFewShot(Teleprompter):# If there are multiple traces for the same predictor in the sample example,# sample 50/50 from the first N-1 traces or the last trace.if len(demos) > 1:- from datasets.fingerprint import Hasher+ from dspy.utils.hasher import Hasherrng = random.Random(Hasher.hash(tuple(demos)))demos = [rng.choice(demos[:-1]) if rng.random() < 0.5 else demos[-1]]diff --git a/dspy/utils/hasher.py b/dspy/utils/hasher.pynew file mode 100644index 000000000..f842600b2--- /dev/null+++ b/dspy/utils/hasher.py@@ -0,0 +1,43 @@+from pickle import dumps+from typing import Any++import xxhash++"""+The following class was pulled from the `datasets` package from Hugging Face.+The reason for vendoring this code is to avoid a hard dependency on `datasets`,+which is a large package that is not needed for the majority of use cases.++License: Apache License 2.0+Author: Hugging Face Inc.+URL: https://github.com/huggingface/datasets/blob/fa73ab472eecf9136a3daf7a0fbff16a3dffa7a6/src/datasets/fingerprint.py#L170+Changes: 2025-08-10 - Ran ruff to format the code to DSPy styles.+"""+class Hasher:+ """Hasher that accepts python objects as inputs."""++ dispatch: dict = {}++ def __init__(self):+ self.m = xxhash.xxh64()++ @classmethod+ def hash_bytes(cls, value: bytes | list[bytes]) -> str:+ value = [value] if isinstance(value, bytes) else value+ m = xxhash.xxh64()+ for x in value:+ m.update(x)+ return m.hexdigest()++ @classmethod+ def hash(cls, value: Any) -> str:+ return cls.hash_bytes(dumps(value))++ def update(self, value: Any) -> None:+ header_for_update = f"=={type(value)}=="+ value_for_update = self.hash(value)+ self.m.update(header_for_update.encode("utf8"))+ self.m.update(value_for_update.encode("utf-8"))++ def hexdigest(self) -> str:+ return self.m.hexdigest()diff --git a/pyproject.toml b/pyproject.tomlindex 395e7d2dc..b26a29c36 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -24,9 +24,7 @@ dependencies = ["backoff>=2.2","joblib~=1.3","openai>=0.28.1",- "datasets>=2.14.6", # needed for Bootstrap's Hasher"regex>=2023.10.3",- "datasets>=2.14.6", # needed for Bootstrap's Hasher"ujson>=5.8.0","tqdm>=4.66.1","requests>=2.31.0",@@ -43,6 +41,7 @@ dependencies = ["cloudpickle>=3.0.0","rich>=13.7.1","numpy>=1.26.0",+ "xxhash>=3.5.0",][project.optional-dependencies]diff --git a/uv.lock b/uv.lockindex 3710d4109..a715a0e71 100644--- a/uv.lock+++ b/uv.lock@@ -664,7 +664,7 @@ wheels = [[[package]]name = "dspy"-version = "3.0.0b1"+version = "3.0.0b3"source = { editable = "." }dependencies = [{ name = "anyio" },@@ -689,6 +689,7 @@ dependencies = [{ name = "tenacity" },{ name = "tqdm" },{ name = "ujson" },+ { name = "xxhash" },][package.optional-dependencies]@@ -767,6 +768,7 @@ requires-dist = [{ name = "tqdm", specifier = ">=4.66.1" },{ name = "ujson", specifier = ">=5.8.0" },{ name = "weaviate-client", marker = "extra == 'weaviate'", specifier = "~=4.5.4" },+ { name = "xxhash", specifier = ">=3.5.0" },]provides-extras = ["anthropic", "weaviate", "mcp", "langchain", "dev", "test-extras"]@@ -788,7 +790,7 @@ name = "exceptiongroup"version = "1.3.0"source = { registry = "https://pypi.org/simple" }dependencies = [- { name = "typing-extensions", marker = "python_full_version < '3.13'" },+ { name = "typing-extensions", marker = "python_full_version < '3.11'" },]sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" }wheels = [