💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add UUID auto-id, closes #505
davidmezzetti59fc73f75238
diff --git a/src/python/txtai/embeddings/__init__.py b/src/python/txtai/embeddings/__init__.pyindex 379af0f0..0a01df60 100644--- a/src/python/txtai/embeddings/__init__.py+++ b/src/python/txtai/embeddings/__init__.py@@ -2,6 +2,7 @@Embeddings imports"""+from .autoid import AutoIdfrom .action import Actionfrom .base import Embeddingsfrom .documents import Documentsdiff --git a/src/python/txtai/embeddings/autoid.py b/src/python/txtai/embeddings/autoid.pynew file mode 100644index 00000000..8ecb3728--- /dev/null+++ b/src/python/txtai/embeddings/autoid.py@@ -0,0 +1,92 @@+"""+AutoId module+"""++import inspect+import uuid+++class AutoId:+ """+ Generates unique ids.+ """++ def __init__(self, method=None):+ """+ Creates a unique id generator.++ Args:+ method: generation method - supports int sequence (default) or UUID function+ """++ # Initialize variables+ self.method, self.function, self.value = None, None, None++ # Set id generation method+ if not method or isinstance(method, int):+ # Incrementing sequence (default)+ self.method = self.sequence+ self.value = method if method else 0+ else:+ # UUID generation function+ self.method = self.uuid+ self.function = getattr(uuid, method)++ # Check if signature takes a namespace argument (deterministic)+ args = inspect.getfullargspec(self.function).args if self.function else []+ self.deterministic = "namespace" in args++ def __call__(self, data=None):+ """+ Generates a unique id.++ Args:+ data: optional data to use for deterministic algorithms (i.e. uuid3, uuid5)++ Returns:+ unique id+ """++ return self.method(data)++ # pylint: disable=W0613+ def sequence(self, data):+ """+ Gets and increments sequence.++ Args:+ data: not used++ Returns:+ current sequence value+ """++ # Get and increment sequence+ value = self.value+ self.value += 1++ return value++ def uuid(self, data):+ """+ Generates a UUID and return as a string.++ Args:+ data: used with determistic algorithms (uuid3, uuid5)++ Returns:+ UUID string+ """++ uid = self.function(uuid.NAMESPACE_DNS, str(data)) if self.deterministic else self.function()+ return str(uid)++ def current(self):+ """+ Get the current sequence value. Only applicable for sequence ids, will be None for UUID methods.++ Returns:+ current sequence value+ """++ return self.valuediff --git a/src/python/txtai/embeddings/stream.py b/src/python/txtai/embeddings/stream.pyindex 6460bb10..6db91d13 100644--- a/src/python/txtai/embeddings/stream.py+++ b/src/python/txtai/embeddings/stream.py@@ -2,6 +2,7 @@Stream module"""+from .autoid import AutoIdfrom .transform import Action@@ -27,7 +28,11 @@ class Stream:# Get config parametersself.offset = self.config.get("offset", 0) if self.action == Action.UPSERT else 0- self.autoid = self.config.get("autoid", self.offset) if self.action == Action.UPSERT else 0+ autoid = self.config.get("autoid", self.offset)++ # Create autoid generator, reset int sequence if this isn't an UPSERT+ autoid = 0 if isinstance(autoid, int) and self.action != Action.UPSERT else autoid+ self.autoid = AutoId(autoid)def __call__(self, documents):"""@@ -51,12 +56,12 @@ class Stream:# Set autoid if the action is setif self.action and document[0] is None:- document = (self.autoid, document[1], document[2])- self.autoid += 1+ document = (self.autoid(document[1]), document[1], document[2])# Yield (id, data, tags) tupleyield document# Save autoid sequence if used- if self.action and self.autoid:- self.config["autoid"] = self.autoid+ current = self.autoid.current()+ if self.action and current:+ self.config["autoid"] = currentdiff --git a/test/python/testdatabase/testsqlite.py b/test/python/testdatabase/testsqlite.pyindex 3a1ab5d9..e73159d2 100644--- a/test/python/testdatabase/testsqlite.py+++ b/test/python/testdatabase/testsqlite.py@@ -72,6 +72,25 @@ class TestSQLite(unittest.TestCase):self.embeddings.upsert([(0, "Looking out into the dreadful abyss", None)])self.assertEqual(self.embeddings.count(), len(self.data))+ def testAutoId(self):+ """+ Test auto id generation+ """++ # Default sequence id+ embeddings = Embeddings()+ embeddings.index(self.data)++ uid = embeddings.search(self.data[4], 1)[0][0]+ self.assertEqual(uid, 4)++ # UUID+ embeddings = Embeddings(autoid="uuid4")+ embeddings.index(self.data)++ uid = embeddings.search(self.data[4], 1)[0][0]+ self.assertEqual(len(uid), 36)+def testClose(self):"""Test embeddings closediff --git a/test/python/testembeddings.py b/test/python/testembeddings.pyindex cda30ac2..064faec5 100644--- a/test/python/testembeddings.py+++ b/test/python/testembeddings.py@@ -46,6 +46,25 @@ class TestEmbeddings(unittest.TestCase):if cls.embeddings:cls.embeddings.close()+ def testAutoId(self):+ """+ Test auto id generation+ """++ # Default sequence id+ embeddings = Embeddings()+ embeddings.index(self.data)++ uid = embeddings.search(self.data[4], 1)[0][0]+ self.assertEqual(uid, 4)++ # UUID+ embeddings = Embeddings(autoid="uuid4")+ embeddings.index(self.data)++ uid = embeddings.search(self.data[4], 1)[0][0]+ self.assertEqual(len(uid), 36)+def testDefaults(self):"""Test default configuration