💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add lfs tracking for embeddings files, closes #450
davidmezzettiee0cc5afcde9
diff --git a/src/python/txtai/cloud/hub.py b/src/python/txtai/cloud/hub.pyindex e69fb95c..ea817f88 100644--- a/src/python/txtai/cloud/hub.py+++ b/src/python/txtai/cloud/hub.py@@ -3,6 +3,7 @@ Hugging Face Hub module"""import os+import tempfileimport huggingface_hub@@ -54,6 +55,9 @@ class HuggingFaceHub(Cloud):repo_id=self.config["container"], token=self.config.get("token"), private=self.config.get("private", True), exist_ok=True)+ # Enable lfs-tracking of embeddings index files+ self.lfstrack()+# Upload filesfor f in self.listfiles(path):huggingface_hub.upload_file(@@ -63,3 +67,31 @@ class HuggingFaceHub(Cloud):path_or_fileobj=f,path_in_repo=os.path.basename(f),)++ def lfstrack(self):+ """+ Adds lfs-tracking of embeddings index files. This method adds tracking for documents and embeddings to .gitattributes.+ """++ # Get and read .gitattributes file+ path = huggingface_hub.hf_hub_download(+ repo_id=self.config["container"], filename=os.path.basename(".gitattributes"), token=self.config.get("token")+ )++ with open(path, "r", encoding="utf-8") as f:+ content = f.read()++ # Check if index files are lfs-tracked. Update .gitattributes, if necessary.+ if "embeddings " not in content:+ # Add documents and embeddings to lfs tracking+ content += "documents filter=lfs diff=lfs merge=lfs -text\n"+ content += "embeddings filter=lfs diff=lfs merge=lfs -text\n"++ # pylint: disable=R1732+ tmp = tempfile.NamedTemporaryFile()+ with open(tmp.name, "w", encoding="utf-8") as f:+ f.write(content)++ huggingface_hub.upload_file(+ repo_id=self.config["container"], token=self.config.get("token"), path_or_fileobj=tmp.name, path_in_repo=os.path.basename(path)+ )diff --git a/test/python/testcloud.py b/test/python/testcloud.pyindex f30d8d22..2c182ebc 100644--- a/test/python/testcloud.py+++ b/test/python/testcloud.py@@ -107,7 +107,8 @@ class TestCloud(unittest.TestCase):if "Invalid" in kwargs["repo_id"]:raise FileNotFoundError- return index+ # Return either .gitattributes file or index+ return attributes if kwargs["filename"] == ".gitattributes" else index# Patch write methods since token will not be availablecreate.return_value = None@@ -122,6 +123,13 @@ class TestCloud(unittest.TestCase):index = os.path.join(tempfile.gettempdir(), f"cloud.{provider}.tar.gz")self.embeddings.save(index)+ # Initialize attributes file+ # pylint: disable=R1732+ tmp = tempfile.NamedTemporaryFile()+ with open(tmp.name, "w", encoding="utf-8") as f:+ f.write("*.bin filter=lfs diff=lfs merge=lfs -text\n")+ attributes = tmp.name+# Run tests with uncompressed and compressed indexfor path in [f"cloud.{provider}", f"cloud.{provider}.tar.gz"]:self.runTests(path, {"provider": provider, "container": "neuml/txtai-intro"})