Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Add reindex method, closes #171

davidmezzetti0b876d72e8e9
diff --git a/src/python/txtai/database/base.py b/src/python/txtai/database/base.py
index e56b5387..a2cd3676 100644
--- a/src/python/txtai/database/base.py
+++ b/src/python/txtai/database/base.py
@@ -62,6 +62,17 @@ class Database:
raise NotImplementedError
+ def reindex(self, columns=None):
+ """
+ Reindexes internal database content and streams results back. This method must renumber indexids
+ sequentially as deletes could have caused indexid gaps.
+
+ Args:
+ columns: optional list of document columns used to rebuild text
+ """
+
+ raise NotImplementedError
+
def save(self, path):
"""
Saves a database at path.
diff --git a/src/python/txtai/database/sqlite.py b/src/python/txtai/database/sqlite.py
index 51594648..a3b177cd 100644
--- a/src/python/txtai/database/sqlite.py
+++ b/src/python/txtai/database/sqlite.py
@@ -53,7 +53,7 @@ class SQLite(Database):
# Sections - stores section text
CREATE_SECTIONS = """
- CREATE TABLE IF NOT EXISTS sections (
+ CREATE TABLE IF NOT EXISTS %s (
indexid INTEGER PRIMARY KEY,
id TEXT,
text TEXT,
@@ -65,6 +65,13 @@ class SQLite(Database):
CREATE_SECTIONS_INDEX = "CREATE INDEX section_id ON sections(id)"
INSERT_SECTION = "INSERT INTO sections VALUES (?, ?, ?, ?, ?)"
DELETE_SECTIONS = "DELETE FROM sections WHERE id IN (SELECT id FROM batch)"
+ COPY_SECTIONS = (
+ "INSERT INTO %s SELECT indexid-1, id, text, tags, entry FROM (SELECT row_number() OVER (ORDER BY indexid) AS indexid, "
+ + "s.id, %s AS text, s.tags, s.entry FROM sections s LEFT JOIN documents d on s.id = d.id)"
+ )
+ STREAM_SECTIONS = "SELECT id, text, tags FROM %s ORDER BY indexid"
+ DROP_SECTIONS = "DROP TABLE sections"
+ RENAME_SECTIONS = "ALTER TABLE %s RENAME TO sections"
# Queries
SELECT_IDS = "SELECT indexid, id FROM sections WHERE id in (SELECT id FROM batch)"
@@ -128,6 +135,31 @@ class SQLite(Database):
self.cursor.execute(SQLite.DELETE_DOCUMENTS)
self.cursor.execute(SQLite.DELETE_SECTIONS)
+ def reindex(self, columns=None):
+ if self.connection:
+ # Working table name
+ name = "rebuild"
+
+ # Resolve and build column strings if provided
+ select = "text"
+ if columns:
+ select = "|| ' ' ||".join([self.resolve(c) for c in columns])
+
+ # Create new table to hold reordered sections
+ self.cursor.execute(SQLite.CREATE_SECTIONS % name)
+
+ # Copy data over
+ self.cursor.execute(SQLite.COPY_SECTIONS % (name, select))
+
+ # Stream new results
+ self.cursor.execute(SQLite.STREAM_SECTIONS % name)
+ yield from self.cursor
+
+ # Swap as new table
+ self.cursor.execute(SQLite.DROP_SECTIONS)
+ self.cursor.execute(SQLite.RENAME_SECTIONS % name)
+ self.cursor.execute(SQLite.CREATE_SECTIONS_INDEX)
+
def save(self, path):
# Temporary database
if not self.path:
@@ -259,13 +291,15 @@ class SQLite(Database):
"""
if not self.connection:
+ name = "sections"
+
# Create temporary database. Thread locking must be handled externally.
self.connection = sqlite3.connect("", check_same_thread=False)
self.cursor = self.connection.cursor()
# Create initial schema and indices
self.cursor.execute(SQLite.CREATE_DOCUMENTS)
- self.cursor.execute(SQLite.CREATE_SECTIONS)
+ self.cursor.execute(SQLite.CREATE_SECTIONS % name)
self.cursor.execute(SQLite.CREATE_SECTIONS_INDEX)
def copy(self, path):
diff --git a/src/python/txtai/embeddings/base.py b/src/python/txtai/embeddings/base.py
index 3755adbb..d875e672 100644
--- a/src/python/txtai/embeddings/base.py
+++ b/src/python/txtai/embeddings/base.py
@@ -35,23 +35,17 @@ class Embeddings:
config: embeddings configuration
"""
- # Configuration
- self.config = config
-
- # Approximate nearest neighbor index
- self.ann = None
+ # Index configuration
+ self.config = None
- if self.config and self.config.get("method") != "transformers":
- # Dimensionality reduction model
- self.reducer = None
+ # Dimensionality reduction and scoring models - word vectors only
+ self.reducer, self.scoring = None, None
- # Embedding scoring method - weighs each word in a sentence
- self.scoring = ScoringFactory.create(self.config["scoring"]) if self.config and self.config.get("scoring") else None
- else:
- self.reducer, self.scoring = None, None
+ # Embeddings vector model - transforms text into similarity vectors
+ self.model = None
- # Sentence vectors model - transforms text to embeddings vectors
- self.model = self.loadvectors() if self.config else None
+ # Approximate nearest neighbor index
+ self.ann = None
# Document database
self.database = None
@@ -59,24 +53,28 @@ class Embeddings:
# Index archive
self.archive = None
+ # Set initial configuration
+ self.configure(config)
+
def score(self, documents):
"""
- Builds a scoring index.
+ Builds a scoring index. Only used by word vectors models.
Args:
documents: list of (id, dict|text|tokens, tags)
"""
+ # Build scoring index over documents
if self.scoring:
- # Build scoring index over documents
self.scoring.index(documents)
- def index(self, documents):
+ def index(self, documents, reindex=False):
"""
Builds an embeddings index. This method overwrites an existing index.
Args:
documents: list of (id, dict|text|tokens, tags)
+ reindex: if this is a reindex operation in which case database creation is skipped, defaults to False
"""
# Transform documents to embeddings vectors
@@ -99,17 +97,19 @@ class Embeddings:
# Build the index
self.ann.index(embeddings)
- # Create document database
- self.database = self.createdatabase()
- if self.database:
- # Add documents to database
- self.database.insert(documents)
- else:
- # Save indexids-ids mapping for indexes with no database
- self.config["ids"] = ids
+ # Keep existing database and archive instances if this is part of a reindex
+ if not reindex:
+ # Create document database
+ self.database = self.createdatabase()
+ if self.database:
+ # Add documents to database
+ self.database.insert(documents)
+ else:
+ # Save indexids-ids mapping for indexes with no database
+ self.config["ids"] = ids
- # Reset archive since this is a new index
- self.archive = None
+ # Reset archive since this is a new index
+ self.archive = None
def upsert(self, documents):
"""
@@ -171,7 +171,7 @@ class Embeddings:
# Parse out indices and ids to delete
indices = [i for i, _ in ids]
- deletes = list(set(uid for _, uid in ids))
+ deletes = sorted(set(uid for _, uid in ids))
# Delete ids from database
self.database.delete(deletes)
@@ -195,6 +195,26 @@ class Embeddings:
return deletes
+ def reindex(self, config, columns=None):
+ """
+ Recreates the approximate nearest neighbor (ann) index using config. This method only works if document
+ content storage is enabled.
+
+ Args:
+ config: new config
+ columns: optional list of document columns used to rebuild text
+ """
+
+ if self.database:
+ # Keep content parameter to ensure database is preserved
+ config["content"] = self.config["content"]
+
+ # Reset configuration
+ self.configure(config)
+
+ # Reindex
+ self.index(self.database.reindex(columns), True)
+
def transform(self, document):
"""
Transforms document into an embeddings vector. Document text will be tokenized if not pre-tokenized.
@@ -337,12 +357,12 @@ class Embeddings:
self.ann = ANNFactory.create(self.config)
self.ann.load(f"{path}/embeddings")
- # Dimensionality reduction model (word vectors only)
+ # Dimensionality reduction model - word vectors only
if self.config.get("pca"):
self.reducer = Reducer()
self.reducer.load(f"{path}/lsa")
- # Embedding scoring model (word vectors only)
+ # Embedding scoring model - word vectors only
if self.config.get("scoring"):
self.scoring = ScoringFactory.create(self.config["scoring"])
self.scoring.load(f"{path}/scoring")
@@ -417,13 +437,36 @@ class Embeddings:
Closes this embeddings index and frees all resources.
"""
- self.config, self.ann, self.reducer, self.scoring, self.model, self.archive = None, None, None, None, None, None
+ self.config, self.reducer, self.scoring, self.model, self.ann, self.archive = None, None, None, None, None, None
# Close database connection if open
if self.database:
self.database.close()
self.database = None
+ def configure(self, config):
+ """
+ Sets the configuration for this embeddings index and loads config-driven models.
+
+ Args:
+ config: embeddings configuration
+ """
+
+ # Configuration
+ self.config = config
+
+ if self.config and self.config.get("method") != "transformers":
+ # Dimensionality reduction model
+ self.reducer = None
+
+ # Embedding scoring method - weighs each word in a sentence
+ self.scoring = ScoringFactory.create(self.config["scoring"]) if self.config and self.config.get("scoring") else None
+ else:
+ self.reducer, self.scoring = None, None
+
+ # Sentence vectors model - transforms text to embeddings vectors
+ self.model = self.loadvectors() if self.config else None
+
def loadvectors(self):
"""
Loads a vector model set in config.
diff --git a/test/python/testdatabase/testembeddings.py b/test/python/testdatabase/testembeddings.py
index 4bbeaeee..b0e46485 100644
--- a/test/python/testdatabase/testembeddings.py
+++ b/test/python/testdatabase/testembeddings.py
@@ -183,6 +183,7 @@ class TestEmbeddings(unittest.TestCase):
self.assertRaises(NotImplementedError, database.load, None)
self.assertRaises(NotImplementedError, database.insert, None)
self.assertRaises(NotImplementedError, database.delete, None)
+ self.assertRaises(NotImplementedError, database.reindex, None)
self.assertRaises(NotImplementedError, database.save, None)
self.assertRaises(NotImplementedError, database.close)
self.assertRaises(NotImplementedError, database.ids, None)
@@ -190,6 +191,25 @@ class TestEmbeddings(unittest.TestCase):
self.assertRaises(NotImplementedError, database.embed, None, None)
self.assertRaises(NotImplementedError, database.query, None, None)
+ def testReindex(self):
+ """
+ Test reindex
+ """
+
+ # Create an index for the list of text
+ self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])
+
+ # Delete records to test indexids still match
+ self.embeddings.delete(([0, 1]))
+
+ # Reindex
+ self.embeddings.reindex({"path": "sentence-transformers/nli-mpnet-base-v2"}, ["text"])
+
+ # Search for best match
+ result = self.embeddings.search("feel good story", 1)[0]
+
+ self.assertEqual(result["text"], self.data[4])
+
def testSave(self):
"""
Test save