Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Add UUID auto-id, closes #505

davidmezzetti59fc73f75238
diff --git a/src/python/txtai/embeddings/__init__.py b/src/python/txtai/embeddings/__init__.py
index 379af0f0..0a01df60 100644
--- a/src/python/txtai/embeddings/__init__.py
+++ b/src/python/txtai/embeddings/__init__.py
@@ -2,6 +2,7 @@
Embeddings imports
"""
+from .autoid import AutoId
from .action import Action
from .base import Embeddings
from .documents import Documents
diff --git a/src/python/txtai/embeddings/autoid.py b/src/python/txtai/embeddings/autoid.py
new file mode 100644
index 00000000..8ecb3728
--- /dev/null
+++ b/src/python/txtai/embeddings/autoid.py
@@ -0,0 +1,92 @@
+"""
+AutoId module
+"""
+
+import inspect
+import uuid
+
+
+class AutoId:
+ """
+ Generates unique ids.
+ """
+
+ def __init__(self, method=None):
+ """
+ Creates a unique id generator.
+
+ Args:
+ method: generation method - supports int sequence (default) or UUID function
+ """
+
+ # Initialize variables
+ self.method, self.function, self.value = None, None, None
+
+ # Set id generation method
+ if not method or isinstance(method, int):
+ # Incrementing sequence (default)
+ self.method = self.sequence
+ self.value = method if method else 0
+ else:
+ # UUID generation function
+ self.method = self.uuid
+ self.function = getattr(uuid, method)
+
+ # Check if signature takes a namespace argument (deterministic)
+ args = inspect.getfullargspec(self.function).args if self.function else []
+ self.deterministic = "namespace" in args
+
+ def __call__(self, data=None):
+ """
+ Generates a unique id.
+
+ Args:
+ data: optional data to use for deterministic algorithms (i.e. uuid3, uuid5)
+
+ Returns:
+ unique id
+ """
+
+ return self.method(data)
+
+ # pylint: disable=W0613
+ def sequence(self, data):
+ """
+ Gets and increments sequence.
+
+ Args:
+ data: not used
+
+ Returns:
+ current sequence value
+ """
+
+ # Get and increment sequence
+ value = self.value
+ self.value += 1
+
+ return value
+
+ def uuid(self, data):
+ """
+ Generates a UUID and return as a string.
+
+ Args:
+ data: used with determistic algorithms (uuid3, uuid5)
+
+ Returns:
+ UUID string
+ """
+
+ uid = self.function(uuid.NAMESPACE_DNS, str(data)) if self.deterministic else self.function()
+ return str(uid)
+
+ def current(self):
+ """
+ Get the current sequence value. Only applicable for sequence ids, will be None for UUID methods.
+
+ Returns:
+ current sequence value
+ """
+
+ return self.value
diff --git a/src/python/txtai/embeddings/stream.py b/src/python/txtai/embeddings/stream.py
index 6460bb10..6db91d13 100644
--- a/src/python/txtai/embeddings/stream.py
+++ b/src/python/txtai/embeddings/stream.py
@@ -2,6 +2,7 @@
Stream module
"""
+from .autoid import AutoId
from .transform import Action
@@ -27,7 +28,11 @@ class Stream:
# Get config parameters
self.offset = self.config.get("offset", 0) if self.action == Action.UPSERT else 0
- self.autoid = self.config.get("autoid", self.offset) if self.action == Action.UPSERT else 0
+ autoid = self.config.get("autoid", self.offset)
+
+ # Create autoid generator, reset int sequence if this isn't an UPSERT
+ autoid = 0 if isinstance(autoid, int) and self.action != Action.UPSERT else autoid
+ self.autoid = AutoId(autoid)
def __call__(self, documents):
"""
@@ -51,12 +56,12 @@ class Stream:
# Set autoid if the action is set
if self.action and document[0] is None:
- document = (self.autoid, document[1], document[2])
- self.autoid += 1
+ document = (self.autoid(document[1]), document[1], document[2])
# Yield (id, data, tags) tuple
yield document
# Save autoid sequence if used
- if self.action and self.autoid:
- self.config["autoid"] = self.autoid
+ current = self.autoid.current()
+ if self.action and current:
+ self.config["autoid"] = current
diff --git a/test/python/testdatabase/testsqlite.py b/test/python/testdatabase/testsqlite.py
index 3a1ab5d9..e73159d2 100644
--- a/test/python/testdatabase/testsqlite.py
+++ b/test/python/testdatabase/testsqlite.py
@@ -72,6 +72,25 @@ class TestSQLite(unittest.TestCase):
self.embeddings.upsert([(0, "Looking out into the dreadful abyss", None)])
self.assertEqual(self.embeddings.count(), len(self.data))
+ def testAutoId(self):
+ """
+ Test auto id generation
+ """
+
+ # Default sequence id
+ embeddings = Embeddings()
+ embeddings.index(self.data)
+
+ uid = embeddings.search(self.data[4], 1)[0][0]
+ self.assertEqual(uid, 4)
+
+ # UUID
+ embeddings = Embeddings(autoid="uuid4")
+ embeddings.index(self.data)
+
+ uid = embeddings.search(self.data[4], 1)[0][0]
+ self.assertEqual(len(uid), 36)
+
def testClose(self):
"""
Test embeddings close
diff --git a/test/python/testembeddings.py b/test/python/testembeddings.py
index cda30ac2..064faec5 100644
--- a/test/python/testembeddings.py
+++ b/test/python/testembeddings.py
@@ -46,6 +46,25 @@ class TestEmbeddings(unittest.TestCase):
if cls.embeddings:
cls.embeddings.close()
+ def testAutoId(self):
+ """
+ Test auto id generation
+ """
+
+ # Default sequence id
+ embeddings = Embeddings()
+ embeddings.index(self.data)
+
+ uid = embeddings.search(self.data[4], 1)[0][0]
+ self.assertEqual(uid, 4)
+
+ # UUID
+ embeddings = Embeddings(autoid="uuid4")
+ embeddings.index(self.data)
+
+ uid = embeddings.search(self.data[4], 1)[0][0]
+ self.assertEqual(len(uid), 36)
+
def testDefaults(self):
"""
Test default configuration