💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add reindex method, closes #171
davidmezzetti0b876d72e8e9
diff --git a/src/python/txtai/database/base.py b/src/python/txtai/database/base.pyindex e56b5387..a2cd3676 100644--- a/src/python/txtai/database/base.py+++ b/src/python/txtai/database/base.py@@ -62,6 +62,17 @@ class Database:raise NotImplementedError+ def reindex(self, columns=None):+ """+ Reindexes internal database content and streams results back. This method must renumber indexids+ sequentially as deletes could have caused indexid gaps.++ Args:+ columns: optional list of document columns used to rebuild text+ """++ raise NotImplementedError+def save(self, path):"""Saves a database at path.diff --git a/src/python/txtai/database/sqlite.py b/src/python/txtai/database/sqlite.pyindex 51594648..a3b177cd 100644--- a/src/python/txtai/database/sqlite.py+++ b/src/python/txtai/database/sqlite.py@@ -53,7 +53,7 @@ class SQLite(Database):# Sections - stores section textCREATE_SECTIONS = """- CREATE TABLE IF NOT EXISTS sections (+ CREATE TABLE IF NOT EXISTS %s (indexid INTEGER PRIMARY KEY,id TEXT,text TEXT,@@ -65,6 +65,13 @@ class SQLite(Database):CREATE_SECTIONS_INDEX = "CREATE INDEX section_id ON sections(id)"INSERT_SECTION = "INSERT INTO sections VALUES (?, ?, ?, ?, ?)"DELETE_SECTIONS = "DELETE FROM sections WHERE id IN (SELECT id FROM batch)"+ COPY_SECTIONS = (+ "INSERT INTO %s SELECT indexid-1, id, text, tags, entry FROM (SELECT row_number() OVER (ORDER BY indexid) AS indexid, "+ + "s.id, %s AS text, s.tags, s.entry FROM sections s LEFT JOIN documents d on s.id = d.id)"+ )+ STREAM_SECTIONS = "SELECT id, text, tags FROM %s ORDER BY indexid"+ DROP_SECTIONS = "DROP TABLE sections"+ RENAME_SECTIONS = "ALTER TABLE %s RENAME TO sections"# QueriesSELECT_IDS = "SELECT indexid, id FROM sections WHERE id in (SELECT id FROM batch)"@@ -128,6 +135,31 @@ class SQLite(Database):self.cursor.execute(SQLite.DELETE_DOCUMENTS)self.cursor.execute(SQLite.DELETE_SECTIONS)+ def reindex(self, columns=None):+ if self.connection:+ # Working table name+ name = "rebuild"++ # Resolve and build column strings if provided+ select = "text"+ if columns:+ select = "|| ' ' ||".join([self.resolve(c) for c in columns])++ # Create new table to hold reordered sections+ self.cursor.execute(SQLite.CREATE_SECTIONS % name)++ # Copy data over+ self.cursor.execute(SQLite.COPY_SECTIONS % (name, select))++ # Stream new results+ self.cursor.execute(SQLite.STREAM_SECTIONS % name)+ yield from self.cursor++ # Swap as new table+ self.cursor.execute(SQLite.DROP_SECTIONS)+ self.cursor.execute(SQLite.RENAME_SECTIONS % name)+ self.cursor.execute(SQLite.CREATE_SECTIONS_INDEX)+def save(self, path):# Temporary databaseif not self.path:@@ -259,13 +291,15 @@ class SQLite(Database):"""if not self.connection:+ name = "sections"+# Create temporary database. Thread locking must be handled externally.self.connection = sqlite3.connect("", check_same_thread=False)self.cursor = self.connection.cursor()# Create initial schema and indicesself.cursor.execute(SQLite.CREATE_DOCUMENTS)- self.cursor.execute(SQLite.CREATE_SECTIONS)+ self.cursor.execute(SQLite.CREATE_SECTIONS % name)self.cursor.execute(SQLite.CREATE_SECTIONS_INDEX)def copy(self, path):diff --git a/src/python/txtai/embeddings/base.py b/src/python/txtai/embeddings/base.pyindex 3755adbb..d875e672 100644--- a/src/python/txtai/embeddings/base.py+++ b/src/python/txtai/embeddings/base.py@@ -35,23 +35,17 @@ class Embeddings:config: embeddings configuration"""- # Configuration- self.config = config-- # Approximate nearest neighbor index- self.ann = None+ # Index configuration+ self.config = None- if self.config and self.config.get("method") != "transformers":- # Dimensionality reduction model- self.reducer = None+ # Dimensionality reduction and scoring models - word vectors only+ self.reducer, self.scoring = None, None- # Embedding scoring method - weighs each word in a sentence- self.scoring = ScoringFactory.create(self.config["scoring"]) if self.config and self.config.get("scoring") else None- else:- self.reducer, self.scoring = None, None+ # Embeddings vector model - transforms text into similarity vectors+ self.model = None- # Sentence vectors model - transforms text to embeddings vectors- self.model = self.loadvectors() if self.config else None+ # Approximate nearest neighbor index+ self.ann = None# Document databaseself.database = None@@ -59,24 +53,28 @@ class Embeddings:# Index archiveself.archive = None+ # Set initial configuration+ self.configure(config)+def score(self, documents):"""- Builds a scoring index.+ Builds a scoring index. Only used by word vectors models.Args:documents: list of (id, dict|text|tokens, tags)"""+ # Build scoring index over documentsif self.scoring:- # Build scoring index over documentsself.scoring.index(documents)- def index(self, documents):+ def index(self, documents, reindex=False):"""Builds an embeddings index. This method overwrites an existing index.Args:documents: list of (id, dict|text|tokens, tags)+ reindex: if this is a reindex operation in which case database creation is skipped, defaults to False"""# Transform documents to embeddings vectors@@ -99,17 +97,19 @@ class Embeddings:# Build the indexself.ann.index(embeddings)- # Create document database- self.database = self.createdatabase()- if self.database:- # Add documents to database- self.database.insert(documents)- else:- # Save indexids-ids mapping for indexes with no database- self.config["ids"] = ids+ # Keep existing database and archive instances if this is part of a reindex+ if not reindex:+ # Create document database+ self.database = self.createdatabase()+ if self.database:+ # Add documents to database+ self.database.insert(documents)+ else:+ # Save indexids-ids mapping for indexes with no database+ self.config["ids"] = ids- # Reset archive since this is a new index- self.archive = None+ # Reset archive since this is a new index+ self.archive = Nonedef upsert(self, documents):"""@@ -171,7 +171,7 @@ class Embeddings:# Parse out indices and ids to deleteindices = [i for i, _ in ids]- deletes = list(set(uid for _, uid in ids))+ deletes = sorted(set(uid for _, uid in ids))# Delete ids from databaseself.database.delete(deletes)@@ -195,6 +195,26 @@ class Embeddings:return deletes+ def reindex(self, config, columns=None):+ """+ Recreates the approximate nearest neighbor (ann) index using config. This method only works if document+ content storage is enabled.++ Args:+ config: new config+ columns: optional list of document columns used to rebuild text+ """++ if self.database:+ # Keep content parameter to ensure database is preserved+ config["content"] = self.config["content"]++ # Reset configuration+ self.configure(config)++ # Reindex+ self.index(self.database.reindex(columns), True)+def transform(self, document):"""Transforms document into an embeddings vector. Document text will be tokenized if not pre-tokenized.@@ -337,12 +357,12 @@ class Embeddings:self.ann = ANNFactory.create(self.config)self.ann.load(f"{path}/embeddings")- # Dimensionality reduction model (word vectors only)+ # Dimensionality reduction model - word vectors onlyif self.config.get("pca"):self.reducer = Reducer()self.reducer.load(f"{path}/lsa")- # Embedding scoring model (word vectors only)+ # Embedding scoring model - word vectors onlyif self.config.get("scoring"):self.scoring = ScoringFactory.create(self.config["scoring"])self.scoring.load(f"{path}/scoring")@@ -417,13 +437,36 @@ class Embeddings:Closes this embeddings index and frees all resources."""- self.config, self.ann, self.reducer, self.scoring, self.model, self.archive = None, None, None, None, None, None+ self.config, self.reducer, self.scoring, self.model, self.ann, self.archive = None, None, None, None, None, None# Close database connection if openif self.database:self.database.close()self.database = None+ def configure(self, config):+ """+ Sets the configuration for this embeddings index and loads config-driven models.++ Args:+ config: embeddings configuration+ """++ # Configuration+ self.config = config++ if self.config and self.config.get("method") != "transformers":+ # Dimensionality reduction model+ self.reducer = None++ # Embedding scoring method - weighs each word in a sentence+ self.scoring = ScoringFactory.create(self.config["scoring"]) if self.config and self.config.get("scoring") else None+ else:+ self.reducer, self.scoring = None, None++ # Sentence vectors model - transforms text to embeddings vectors+ self.model = self.loadvectors() if self.config else None+def loadvectors(self):"""Loads a vector model set in config.diff --git a/test/python/testdatabase/testembeddings.py b/test/python/testdatabase/testembeddings.pyindex 4bbeaeee..b0e46485 100644--- a/test/python/testdatabase/testembeddings.py+++ b/test/python/testdatabase/testembeddings.py@@ -183,6 +183,7 @@ class TestEmbeddings(unittest.TestCase):self.assertRaises(NotImplementedError, database.load, None)self.assertRaises(NotImplementedError, database.insert, None)self.assertRaises(NotImplementedError, database.delete, None)+ self.assertRaises(NotImplementedError, database.reindex, None)self.assertRaises(NotImplementedError, database.save, None)self.assertRaises(NotImplementedError, database.close)self.assertRaises(NotImplementedError, database.ids, None)@@ -190,6 +191,25 @@ class TestEmbeddings(unittest.TestCase):self.assertRaises(NotImplementedError, database.embed, None, None)self.assertRaises(NotImplementedError, database.query, None, None)+ def testReindex(self):+ """+ Test reindex+ """++ # Create an index for the list of text+ self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])++ # Delete records to test indexids still match+ self.embeddings.delete(([0, 1]))++ # Reindex+ self.embeddings.reindex({"path": "sentence-transformers/nli-mpnet-base-v2"}, ["text"])++ # Search for best match+ result = self.embeddings.search("feel good story", 1)[0]++ self.assertEqual(result["text"], self.data[4])+def testSave(self):"""Test save