Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Keep SQLite ANN data when saving a modified index to a new path (#1249)

Caner Olcay4b7d70e9e80d
diff --git a/src/python/txtai/ann/dense/sqlite.py b/src/python/txtai/ann/dense/sqlite.py
index 58eb6b87..6c4f7680 100644
--- a/src/python/txtai/ann/dense/sqlite.py
+++ b/src/python/txtai/ann/dense/sqlite.py
@@ -193,10 +193,13 @@ class SQLite(ANN):
connection.execute(self.tablesql())
# The backup call will hang if there are uncommitted changes, need to copy over
- # with iterdump (which is much slower)
- for sql in self.connection.iterdump():
- if self.tosql('insert into "{table}"') in sql.lower():
- connection.execute(sql)
+ # the rows (which is much slower). Quantized vectors must be inserted with their type.
+ embedding = "vec_bit(?)" if self.quantize == 1 else "vec_int8(?)" if self.quantize == 8 else "?"
+ rows = self.connection.execute(self.tosql("SELECT indexid, embedding FROM {table}"))
+ connection.executemany(self.tosql(f"INSERT INTO {{table}}(indexid, embedding) VALUES (?, {embedding})"), rows)
+
+ # Commit copied rows, otherwise they are rolled back when the connection is closed
+ connection.commit()
else:
# Database is up to date, can do a more efficient copy with SQLite C API
self.connection.backup(connection)
diff --git a/test/python/testann/testdense.py b/test/python/testann/testdense.py
index 8ce99894..ae6c1ee0 100644
--- a/test/python/testann/testdense.py
+++ b/test/python/testann/testdense.py
@@ -575,6 +575,38 @@ class TestDense(unittest.TestCase):
self.assertEqual(model.count(), expected)
+ @unittest.skipIf(platform.system() == "Darwin", "SQLite extensions not supported on macOS")
+ def testSQLiteSaveNewPath(self):
+ """
+ Test saving a loaded and modified SQLite index to a new path, then loading the new copy
+ """
+
+ for quantize in [None, 1, 8]:
+ params = {"sqlite": {"quantize": quantize}} if quantize else None
+
+ index = os.path.join(tempfile.gettempdir(), f"ann.sqlite.load.{quantize}")
+ new = os.path.join(tempfile.gettempdir(), f"ann.sqlite.load.{quantize}.new")
+
+ # Build and save index
+ model = self.backend("sqlite", params, 500)
+ model.save(index)
+ model.close()
+
+ # Load index, modify and save to a new path
+ model = ANNFactory.create(model.config)
+ model.load(index)
+ model.delete([0])
+ model.append(np.random.rand(10, 240).astype(np.float32))
+ model.save(new)
+ model.close()
+
+ # Load new copy and check that it has all the changes
+ model = ANNFactory.create(model.config)
+ model.load(new)
+ self.assertEqual(model.count(), 509)
+ self.assertEqual(len(model.search(np.random.rand(1, 240).astype(np.float32), 10)[0]), 10)
+ model.close()
+
def testSQLiteQuantizeDisabled(self):
"""
Test that quantize: false disables SQLite storage quantization