💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Keep SQLite ANN data when saving a modified index to a new path (#1249)
Caner Olcay4b7d70e9e80d
diff --git a/src/python/txtai/ann/dense/sqlite.py b/src/python/txtai/ann/dense/sqlite.pyindex 58eb6b87..6c4f7680 100644--- a/src/python/txtai/ann/dense/sqlite.py+++ b/src/python/txtai/ann/dense/sqlite.py@@ -193,10 +193,13 @@ class SQLite(ANN):connection.execute(self.tablesql())# The backup call will hang if there are uncommitted changes, need to copy over- # with iterdump (which is much slower)- for sql in self.connection.iterdump():- if self.tosql('insert into "{table}"') in sql.lower():- connection.execute(sql)+ # the rows (which is much slower). Quantized vectors must be inserted with their type.+ embedding = "vec_bit(?)" if self.quantize == 1 else "vec_int8(?)" if self.quantize == 8 else "?"+ rows = self.connection.execute(self.tosql("SELECT indexid, embedding FROM {table}"))+ connection.executemany(self.tosql(f"INSERT INTO {{table}}(indexid, embedding) VALUES (?, {embedding})"), rows)++ # Commit copied rows, otherwise they are rolled back when the connection is closed+ connection.commit()else:# Database is up to date, can do a more efficient copy with SQLite C APIself.connection.backup(connection)diff --git a/test/python/testann/testdense.py b/test/python/testann/testdense.pyindex 8ce99894..ae6c1ee0 100644--- a/test/python/testann/testdense.py+++ b/test/python/testann/testdense.py@@ -575,6 +575,38 @@ class TestDense(unittest.TestCase):self.assertEqual(model.count(), expected)+ @unittest.skipIf(platform.system() == "Darwin", "SQLite extensions not supported on macOS")+ def testSQLiteSaveNewPath(self):+ """+ Test saving a loaded and modified SQLite index to a new path, then loading the new copy+ """++ for quantize in [None, 1, 8]:+ params = {"sqlite": {"quantize": quantize}} if quantize else None++ index = os.path.join(tempfile.gettempdir(), f"ann.sqlite.load.{quantize}")+ new = os.path.join(tempfile.gettempdir(), f"ann.sqlite.load.{quantize}.new")++ # Build and save index+ model = self.backend("sqlite", params, 500)+ model.save(index)+ model.close()++ # Load index, modify and save to a new path+ model = ANNFactory.create(model.config)+ model.load(index)+ model.delete([0])+ model.append(np.random.rand(10, 240).astype(np.float32))+ model.save(new)+ model.close()++ # Load new copy and check that it has all the changes+ model = ANNFactory.create(model.config)+ model.load(new)+ self.assertEqual(model.count(), 509)+ self.assertEqual(len(model.search(np.random.rand(1, 240).astype(np.float32), 10)[0]), 10)+ model.close()+def testSQLiteQuantizeDisabled(self):"""Test that quantize: false disables SQLite storage quantization