💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Persist Reducer component with skops library, closes #774
davidmezzettiad96d3f44eae
diff --git a/src/python/txtai/embeddings/index/reducer.py b/src/python/txtai/embeddings/index/reducer.pyindex 47d9cc83..b359913f 100644--- a/src/python/txtai/embeddings/index/reducer.py+++ b/src/python/txtai/embeddings/index/reducer.py@@ -2,17 +2,19 @@Reducer module"""-import pickle+from zipfile import BadZipFile# Conditionally import dimensionality reduction libraries as they aren't installed by defaulttry:+ import skops.io as sio+from sklearn.decomposition import TruncatedSVDREDUCER = Trueexcept ImportError:REDUCER = False-from ...version import __pickle__+from ...serialize import SerializeFactoryclass Reducer:@@ -85,8 +87,11 @@ class Reducer:"""# Dimensionality reduction- with open(path, "rb") as handle:- self.model = pickle.load(handle)+ try:+ self.model = sio.load(path)+ except (BadZipFile, KeyError):+ # Backwards compatible support for pickled models+ self.model = SerializeFactory.create("pickle").load(path)def save(self, path):"""@@ -96,5 +101,4 @@ class Reducer:path: directory path to save model"""- with open(path, "wb") as handle:- pickle.dump(self.model, handle, protocol=__pickle__)+ sio.dump(self.model, path)