💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add workflow streams, closes #461
davidmezzetti178f4c26dc8e
diff --git a/src/python/txtai/app/base.py b/src/python/txtai/app/base.pyindex d7c5ce8b..437f09c7 100644--- a/src/python/txtai/app/base.py+++ b/src/python/txtai/app/base.py@@ -137,6 +137,10 @@ class Application:# Resolve callable functionsconfig["tasks"] = [self.resolve(task) for task in config["tasks"]]+ # Resolve stream functions+ if "stream" in config:+ config["stream"] = self.resolve(config["stream"])+# Get scheduler configschedule = config.pop("schedule", None)@@ -640,8 +644,12 @@ class Application:processed elements"""- # Convert lists to tuples- elements = [tuple(element) if isinstance(element, list) else element for element in elements]+ if hasattr(elements, "__len__") and hasattr(elements, "__getitem__"):+ # Convert to tuples and return as a list since input is sized+ elements = [tuple(element) if isinstance(element, list) else element for element in elements]+ else:+ # Convert to tuples and return as a generator since input is not sized+ elements = (tuple(element) if isinstance(element, list) else element for element in elements)# Execute workflowreturn self.workflows[name](elements)diff --git a/src/python/txtai/workflow/base.py b/src/python/txtai/workflow/base.pyindex 39924881..61d597be 100644--- a/src/python/txtai/workflow/base.py+++ b/src/python/txtai/workflow/base.py@@ -27,7 +27,7 @@ class Workflow:Base class for all workflows."""- def __init__(self, tasks, batch=100, workers=None, name=None):+ def __init__(self, tasks, batch=100, workers=None, name=None, stream=None):"""Creates a new workflow. Workflows are lists of tasks to execute.@@ -36,12 +36,14 @@ class Workflow:batch: how many items to process at a time, defaults to 100workers: number of concurrent workersname: workflow name+ stream: workflow stream processor"""self.tasks = tasksself.batch = batchself.workers = workersself.name = name+ self.stream = stream# Set default number of executor workers to max number of actions in a taskself.workers = max(len(task.action) for task in self.tasks) if not self.workers else self.workers@@ -63,6 +65,9 @@ class Workflow:# Run task initializersself.initialize()+ # Process elements with stream processor, if available+ elements = self.stream(elements) if self.stream else elements+# Process elements in batchesfor batch in self.chunk(elements):yield from self.process(batch, executor)diff --git a/src/python/txtai/workflow/factory.py b/src/python/txtai/workflow/factory.pyindex b576902c..506f077c 100644--- a/src/python/txtai/workflow/factory.py+++ b/src/python/txtai/workflow/factory.py@@ -32,5 +32,11 @@ class WorkflowFactory:config["tasks"] = tasks+ if "stream" in config:+ sconfig = config["stream"]+ task = sconfig.pop("task") if "task" in sconfig else "stream"++ config["stream"] = TaskFactory.create(sconfig, task)+# Create workflowreturn Workflow(**config, name=name)diff --git a/src/python/txtai/workflow/task/__init__.py b/src/python/txtai/workflow/task/__init__.pyindex 80f7fad5..e62b3806 100644--- a/src/python/txtai/workflow/task/__init__.py+++ b/src/python/txtai/workflow/task/__init__.py@@ -11,6 +11,7 @@ from .image import ImageTaskfrom .retrieve import RetrieveTaskfrom .service import ServiceTaskfrom .storage import StorageTask+from .stream import StreamTaskfrom .template import ExtractorTask, TemplateTaskfrom .url import UrlTaskfrom .workflow import WorkflowTaskdiff --git a/src/python/txtai/workflow/task/stream.py b/src/python/txtai/workflow/task/stream.pynew file mode 100644index 00000000..93c2efae--- /dev/null+++ b/src/python/txtai/workflow/task/stream.py@@ -0,0 +1,33 @@+"""+StreamTask module+"""++from .base import Task+++class StreamTask(Task):+ """+ Task that calls a task action and yields results.+ """++ def register(self, batch=False):+ """+ Adds stream parameters to task.++ Args:+ batch: all elements are passed to a single action call if True, otherwise an action call is executed per element, defaults to False+ """++ # pylint: disable=W0201+ # All elements are passed to a single action call if True, otherwise an action call is executed per element, defaults to False+ self.batch = batch++ def __call__(self, elements, executor=None):+ for action in self.action:+ if self.batch:+ # Single batch call+ yield from action(elements)+ else:+ # Call action for each element+ for x in elements:+ yield from action(x)diff --git a/test/python/testapp.py b/test/python/testapp.pyindex 056a7568..7c00a4c9 100644--- a/test/python/testapp.py+++ b/test/python/testapp.py@@ -3,6 +3,7 @@ Application module tests"""import unittest+import typesfrom txtai.app import Applicationfrom txtai.pipeline import Pipeline@@ -36,6 +37,35 @@ class TestApp(unittest.TestCase):# Check that application instance is not Noneself.assertIsNotNone(app.pipelines["testapp.TestPipeline"].application)+ def testStream(self):+ """+ Test workflow streams+ """++ app = Application(+ """+ workflow:+ stream:+ stream:+ action: testapp.TestStream+ tasks:+ - nop+ batchstream:+ stream:+ action: testapp.TestStream+ batch: True+ tasks:+ - nop+ """+ )++ def generator():+ yield 10++ self.assertEqual(list(app.workflow("stream", [10])), list(range(10)))++ self.assertEqual(list(app.workflow("batchstream", generator())), list(range(10)))+class TestPipeline(Pipeline):"""@@ -44,3 +74,16 @@ class TestPipeline(Pipeline):def __init__(self, application):self.application = application+++class TestStream:+ """+ Test workflow stream+ """++ def __call__(self, arg):+ if isinstance(arg, types.GeneratorType):+ for x in arg:+ yield from range(int(x))+ else:+ yield from range(int(arg))