💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add post workflow actions to API, closes #129
davidmezzettic444d1715dae
diff --git a/src/python/txtai/api/base.py b/src/python/txtai/api/base.pyindex 5f4a7644..8a0afefe 100644--- a/src/python/txtai/api/base.py+++ b/src/python/txtai/api/base.py@@ -78,15 +78,23 @@ class API:Initialize workflows."""+ # Workflow definitionsself.workflows = {}+ # Post workflow actions+ self.actions = {}+# Create workflowsif "workflow" in self.config:for workflow, config in self.config["workflow"].items():for task in config["tasks"]:# Resolve pipeline actionif "action" in task:- task["action"] = self.pipelines[task["action"]]+ action = task["action"]+ task["action"] = self.pipelines.get(action)++ if action in ["index", "upsert"]:+ self.actions[workflow] = actionself.workflows[workflow] = WorkflowFactory.create(config)@@ -396,4 +404,29 @@ class API:elements = [tuple(element) if isinstance(element, list) else element for element in elements]# Execute workflow- return self.workflows[name](elements)+ elements = self.workflows[name](elements)++ # Run post workflow action, if any+ if name in self.actions:+ documents = []+ action = self.actions[name]+ index = self.count() if self.count() and action != "index" else 0++ # Consume all elements+ elements = list(elements)++ for element in elements:+ if isinstance(element, (tuple, list)):+ element = {"id": element[0], "text": element[1]}+ else:+ element = {"id": index, "text": element}++ documents.append(element)+ index += 1++ # pylint: disable=W0106+ # Add documents and run index operation+ self.add(documents)+ self.index() if self.actions[name] == "index" else self.upsert()++ return elementsdiff --git a/src/python/txtai/embeddings/base.py b/src/python/txtai/embeddings/base.pyindex 109df00a..d0f5b5ee 100644--- a/src/python/txtai/embeddings/base.py+++ b/src/python/txtai/embeddings/base.py@@ -204,7 +204,7 @@ class Embeddings:number of elements in embeddings index"""- return self.embeddings.count()+ return self.embeddings.count() if self.embeddings else 0def search(self, query, limit=3):"""diff --git a/test/python/testapi/testworkflow.py b/test/python/testapi/testworkflow.pyindex 8f0181dd..eba3acaa 100644--- a/test/python/testapi/testworkflow.py+++ b/test/python/testapi/testworkflow.py@@ -17,6 +17,11 @@ from txtai.api import app, start# Configuration for workflowsWORKFLOWS = """+# Embeddings index+writable: true+embeddings:+ path: sentence-transformers/nli-mpnet-base-v2+# Text segmentationsegmentation:sentences: true@@ -26,6 +31,7 @@ workflow:segment:tasks:- action: segmentation+ - action: indexget:tasks:- task: service@@ -152,3 +158,6 @@ class TestWorkflow(unittest.TestCase):text = "This is a test sentence. And another sentence to split."results = self.client.post("workflow", json={"name": "segment", "elements": [text]}).json()self.assertEqual(len(results), 2)++ results = self.client.post("workflow", json={"name": "segment", "elements": [[0, text]]}).json()+ self.assertEqual(len(results), 2)