Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Add post workflow actions to API, closes #129

davidmezzettic444d1715dae
diff --git a/src/python/txtai/api/base.py b/src/python/txtai/api/base.py
index 5f4a7644..8a0afefe 100644
--- a/src/python/txtai/api/base.py
+++ b/src/python/txtai/api/base.py
@@ -78,15 +78,23 @@ class API:
Initialize workflows.
"""
+ # Workflow definitions
self.workflows = {}
+ # Post workflow actions
+ self.actions = {}
+
# Create workflows
if "workflow" in self.config:
for workflow, config in self.config["workflow"].items():
for task in config["tasks"]:
# Resolve pipeline action
if "action" in task:
- task["action"] = self.pipelines[task["action"]]
+ action = task["action"]
+ task["action"] = self.pipelines.get(action)
+
+ if action in ["index", "upsert"]:
+ self.actions[workflow] = action
self.workflows[workflow] = WorkflowFactory.create(config)
@@ -396,4 +404,29 @@ class API:
elements = [tuple(element) if isinstance(element, list) else element for element in elements]
# Execute workflow
- return self.workflows[name](elements)
+ elements = self.workflows[name](elements)
+
+ # Run post workflow action, if any
+ if name in self.actions:
+ documents = []
+ action = self.actions[name]
+ index = self.count() if self.count() and action != "index" else 0
+
+ # Consume all elements
+ elements = list(elements)
+
+ for element in elements:
+ if isinstance(element, (tuple, list)):
+ element = {"id": element[0], "text": element[1]}
+ else:
+ element = {"id": index, "text": element}
+
+ documents.append(element)
+ index += 1
+
+ # pylint: disable=W0106
+ # Add documents and run index operation
+ self.add(documents)
+ self.index() if self.actions[name] == "index" else self.upsert()
+
+ return elements
diff --git a/src/python/txtai/embeddings/base.py b/src/python/txtai/embeddings/base.py
index 109df00a..d0f5b5ee 100644
--- a/src/python/txtai/embeddings/base.py
+++ b/src/python/txtai/embeddings/base.py
@@ -204,7 +204,7 @@ class Embeddings:
number of elements in embeddings index
"""
- return self.embeddings.count()
+ return self.embeddings.count() if self.embeddings else 0
def search(self, query, limit=3):
"""
diff --git a/test/python/testapi/testworkflow.py b/test/python/testapi/testworkflow.py
index 8f0181dd..eba3acaa 100644
--- a/test/python/testapi/testworkflow.py
+++ b/test/python/testapi/testworkflow.py
@@ -17,6 +17,11 @@ from txtai.api import app, start
# Configuration for workflows
WORKFLOWS = """
+# Embeddings index
+writable: true
+embeddings:
+ path: sentence-transformers/nli-mpnet-base-v2
+
# Text segmentation
segmentation:
sentences: true
@@ -26,6 +31,7 @@ workflow:
segment:
tasks:
- action: segmentation
+ - action: index
get:
tasks:
- task: service
@@ -152,3 +158,6 @@ class TestWorkflow(unittest.TestCase):
text = "This is a test sentence. And another sentence to split."
results = self.client.post("workflow", json={"name": "segment", "elements": [text]}).json()
self.assertEqual(len(results), 2)
+
+ results = self.client.post("workflow", json={"name": "segment", "elements": [[0, text]]}).json()
+ self.assertEqual(len(results), 2)