💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Persist Embeddings index ids with MessagePack, closes #773
davidmezzetti9478beaebcb5
diff --git a/src/python/txtai/embeddings/index/indexids.py b/src/python/txtai/embeddings/index/indexids.pyindex 3a1e4785..5935a8ef 100644--- a/src/python/txtai/embeddings/index/indexids.py+++ b/src/python/txtai/embeddings/index/indexids.py@@ -2,9 +2,7 @@IndexIds module"""-import pickle--from ...version import __pickle__+from ...serialize import Serializerclass IndexIds:@@ -49,8 +47,7 @@ class IndexIds:self.ids = self.config.pop("ids")else:# Standard ids format- with open(path, "rb") as handle:- self.ids = pickle.load(handle)+ self.ids = Serializer.load(path)def save(self, path):"""@@ -60,5 +57,4 @@ class IndexIds:path: path to save"""- with open(path, "wb") as handle:- pickle.dump(self.ids, handle, protocol=__pickle__)+ Serializer.save(self.ids, path)diff --git a/test/python/testembeddings.py b/test/python/testembeddings.pyindex 82ce0d92..b4bd70ee 100644--- a/test/python/testembeddings.py+++ b/test/python/testembeddings.py@@ -12,6 +12,7 @@ from unittest.mock import patchimport numpy as npfrom txtai.embeddings import Embeddings, Reducer+from txtai.serialize import SerializeFactoryfrom txtai.vectors import WordVectors@@ -250,6 +251,33 @@ class TestEmbeddings(unittest.TestCase):# Check that ids is not in configself.assertTrue("ids" not in self.embeddings.config)+ def testIdsPickle(self):+ """+ Test legacy pickle ids+ """++ # Create an index for the list of text+ self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])++ # Generate temp file path+ index = os.path.join(tempfile.gettempdir(), "embeddings.idspickle")++ # Save index+ self.embeddings.save(index)++ # Create ids as pickle+ path = os.path.join(tempfile.gettempdir(), "embeddings.idspickle", "ids")+ serializer = SerializeFactory.create("pickle", allowpickle=True)+ serializer.save(self.embeddings.ids.ids, path)++ # Reload index+ with self.assertWarns(FutureWarning):+ self.embeddings.load(index)++ # Run search+ uid = self.embeddings.search("feel good story", 1)[0][0]+ self.assertEqual(uid, 4)+def testIndex(self):"""Test index@@ -343,6 +371,32 @@ class TestEmbeddings(unittest.TestCase):reducer(query)self.assertFalse(np.array_equal(query, original))+ def testReducerLegacy(self):+ """+ Test reducer model with legacy model format+ """++ # Test model with single PCA component+ data = np.random.rand(5, 5).astype(np.float32)+ reducer = Reducer(data, 1)++ # Save legacy format+ path = os.path.join(tempfile.gettempdir(), "reducer")+ serializer = SerializeFactory.create("pickle", allowpickle=True)+ serializer.save(reducer.model, path)++ # Load legacy format+ reducer = Reducer()+ reducer.load(path)++ # Generate query and keep original data to ensure it changes+ query = np.random.rand(1, 5).astype(np.float32)+ original = query.copy()++ # Run test+ reducer(query)+ self.assertFalse(np.array_equal(query, original))+def testSave(self):"""Test save