Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Vendor Hasher class from datasets library in order to remove larger dependency (#8621)

Ben McHonea7566188a987
diff --git a/dspy/clients/utils_finetune.py b/dspy/clients/utils_finetune.py
index ea234fc37..1b8b2031f 100644
--- a/dspy/clients/utils_finetune.py
+++ b/dspy/clients/utils_finetune.py
@@ -66,7 +66,7 @@ def write_lines(file_path, data):
def save_data(
data: list[dict[str, Any]],
) -> str:
- from datasets.fingerprint import Hasher
+ from dspy.utils.hasher import Hasher
# Assign a unique name to the file based on the data hash
hash = Hasher.hash(data)
diff --git a/dspy/teleprompt/bootstrap.py b/dspy/teleprompt/bootstrap.py
index 2b2fd8ec2..35c56eeeb 100644
--- a/dspy/teleprompt/bootstrap.py
+++ b/dspy/teleprompt/bootstrap.py
@@ -241,7 +241,7 @@ class BootstrapFewShot(Teleprompter):
# If there are multiple traces for the same predictor in the sample example,
# sample 50/50 from the first N-1 traces or the last trace.
if len(demos) > 1:
- from datasets.fingerprint import Hasher
+ from dspy.utils.hasher import Hasher
rng = random.Random(Hasher.hash(tuple(demos)))
demos = [rng.choice(demos[:-1]) if rng.random() < 0.5 else demos[-1]]
diff --git a/dspy/utils/hasher.py b/dspy/utils/hasher.py
new file mode 100644
index 000000000..f842600b2
--- /dev/null
+++ b/dspy/utils/hasher.py
@@ -0,0 +1,43 @@
+from pickle import dumps
+from typing import Any
+
+import xxhash
+
+"""
+The following class was pulled from the `datasets` package from Hugging Face.
+The reason for vendoring this code is to avoid a hard dependency on `datasets`,
+which is a large package that is not needed for the majority of use cases.
+
+License: Apache License 2.0
+Author: Hugging Face Inc.
+URL: https://github.com/huggingface/datasets/blob/fa73ab472eecf9136a3daf7a0fbff16a3dffa7a6/src/datasets/fingerprint.py#L170
+Changes: 2025-08-10 - Ran ruff to format the code to DSPy styles.
+"""
+class Hasher:
+ """Hasher that accepts python objects as inputs."""
+
+ dispatch: dict = {}
+
+ def __init__(self):
+ self.m = xxhash.xxh64()
+
+ @classmethod
+ def hash_bytes(cls, value: bytes | list[bytes]) -> str:
+ value = [value] if isinstance(value, bytes) else value
+ m = xxhash.xxh64()
+ for x in value:
+ m.update(x)
+ return m.hexdigest()
+
+ @classmethod
+ def hash(cls, value: Any) -> str:
+ return cls.hash_bytes(dumps(value))
+
+ def update(self, value: Any) -> None:
+ header_for_update = f"=={type(value)}=="
+ value_for_update = self.hash(value)
+ self.m.update(header_for_update.encode("utf8"))
+ self.m.update(value_for_update.encode("utf-8"))
+
+ def hexdigest(self) -> str:
+ return self.m.hexdigest()
diff --git a/pyproject.toml b/pyproject.toml
index 395e7d2dc..b26a29c36 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -24,9 +24,7 @@ dependencies = [
"backoff>=2.2",
"joblib~=1.3",
"openai>=0.28.1",
- "datasets>=2.14.6", # needed for Bootstrap's Hasher
"regex>=2023.10.3",
- "datasets>=2.14.6", # needed for Bootstrap's Hasher
"ujson>=5.8.0",
"tqdm>=4.66.1",
"requests>=2.31.0",
@@ -43,6 +41,7 @@ dependencies = [
"cloudpickle>=3.0.0",
"rich>=13.7.1",
"numpy>=1.26.0",
+ "xxhash>=3.5.0",
]
[project.optional-dependencies]
diff --git a/uv.lock b/uv.lock
index 3710d4109..a715a0e71 100644
--- a/uv.lock
+++ b/uv.lock
@@ -664,7 +664,7 @@ wheels = [
[[package]]
name = "dspy"
-version = "3.0.0b1"
+version = "3.0.0b3"
source = { editable = "." }
dependencies = [
{ name = "anyio" },
@@ -689,6 +689,7 @@ dependencies = [
{ name = "tenacity" },
{ name = "tqdm" },
{ name = "ujson" },
+ { name = "xxhash" },
]
[package.optional-dependencies]
@@ -767,6 +768,7 @@ requires-dist = [
{ name = "tqdm", specifier = ">=4.66.1" },
{ name = "ujson", specifier = ">=5.8.0" },
{ name = "weaviate-client", marker = "extra == 'weaviate'", specifier = "~=4.5.4" },
+ { name = "xxhash", specifier = ">=3.5.0" },
]
provides-extras = ["anthropic", "weaviate", "mcp", "langchain", "dev", "test-extras"]
@@ -788,7 +790,7 @@ name = "exceptiongroup"
version = "1.3.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
- { name = "typing-extensions", marker = "python_full_version < '3.13'" },
+ { name = "typing-extensions", marker = "python_full_version < '3.11'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" }
wheels = [