Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Add lfs tracking for embeddings files, closes #450

davidmezzettiee0cc5afcde9
diff --git a/src/python/txtai/cloud/hub.py b/src/python/txtai/cloud/hub.py
index e69fb95c..ea817f88 100644
--- a/src/python/txtai/cloud/hub.py
+++ b/src/python/txtai/cloud/hub.py
@@ -3,6 +3,7 @@ Hugging Face Hub module
"""
import os
+import tempfile
import huggingface_hub
@@ -54,6 +55,9 @@ class HuggingFaceHub(Cloud):
repo_id=self.config["container"], token=self.config.get("token"), private=self.config.get("private", True), exist_ok=True
)
+ # Enable lfs-tracking of embeddings index files
+ self.lfstrack()
+
# Upload files
for f in self.listfiles(path):
huggingface_hub.upload_file(
@@ -63,3 +67,31 @@ class HuggingFaceHub(Cloud):
path_or_fileobj=f,
path_in_repo=os.path.basename(f),
)
+
+ def lfstrack(self):
+ """
+ Adds lfs-tracking of embeddings index files. This method adds tracking for documents and embeddings to .gitattributes.
+ """
+
+ # Get and read .gitattributes file
+ path = huggingface_hub.hf_hub_download(
+ repo_id=self.config["container"], filename=os.path.basename(".gitattributes"), token=self.config.get("token")
+ )
+
+ with open(path, "r", encoding="utf-8") as f:
+ content = f.read()
+
+ # Check if index files are lfs-tracked. Update .gitattributes, if necessary.
+ if "embeddings " not in content:
+ # Add documents and embeddings to lfs tracking
+ content += "documents filter=lfs diff=lfs merge=lfs -text\n"
+ content += "embeddings filter=lfs diff=lfs merge=lfs -text\n"
+
+ # pylint: disable=R1732
+ tmp = tempfile.NamedTemporaryFile()
+ with open(tmp.name, "w", encoding="utf-8") as f:
+ f.write(content)
+
+ huggingface_hub.upload_file(
+ repo_id=self.config["container"], token=self.config.get("token"), path_or_fileobj=tmp.name, path_in_repo=os.path.basename(path)
+ )
diff --git a/test/python/testcloud.py b/test/python/testcloud.py
index f30d8d22..2c182ebc 100644
--- a/test/python/testcloud.py
+++ b/test/python/testcloud.py
@@ -107,7 +107,8 @@ class TestCloud(unittest.TestCase):
if "Invalid" in kwargs["repo_id"]:
raise FileNotFoundError
- return index
+ # Return either .gitattributes file or index
+ return attributes if kwargs["filename"] == ".gitattributes" else index
# Patch write methods since token will not be available
create.return_value = None
@@ -122,6 +123,13 @@ class TestCloud(unittest.TestCase):
index = os.path.join(tempfile.gettempdir(), f"cloud.{provider}.tar.gz")
self.embeddings.save(index)
+ # Initialize attributes file
+ # pylint: disable=R1732
+ tmp = tempfile.NamedTemporaryFile()
+ with open(tmp.name, "w", encoding="utf-8") as f:
+ f.write("*.bin filter=lfs diff=lfs merge=lfs -text\n")
+ attributes = tmp.name
+
# Run tests with uncompressed and compressed index
for path in [f"cloud.{provider}", f"cloud.{provider}.tar.gz"]:
self.runTests(path, {"provider": provider, "container": "neuml/txtai-intro"})