💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Improve workflow processing efficiency for tensors, closes #158, closes #159
davidmezzetti97f7976b82b8
diff --git a/src/python/txtai/workflow/base.py b/src/python/txtai/workflow/base.pyindex 83e527a4..8c8dacc5 100644--- a/src/python/txtai/workflow/base.py+++ b/src/python/txtai/workflow/base.py@@ -34,17 +34,8 @@ class Workflow:# Run task initializersself.initialize()- # Process elements in batches.- batch = []- for x in elements:- batch.append(x)-- if len(batch) == self.batch:- yield from self.process(batch)- batch = []-- # Final batch- if batch:+ # Process elements in batches+ for batch in self.chunk(elements):yield from self.process(batch)# Run task finalizers@@ -60,48 +51,53 @@ class Workflow:if task.initialize:task.initialize()+ def chunk(self, elements):+ """+ Splits elements into batches. This method efficiently processes both fixed size inputs and+ dynamically generated inputs.++ Args:+ elements: iterable data elements++ Returns:+ evenly sized batches with the last batch having the remaining elements+ """++ # Build batches by slicing elements, more efficient for fixed sized inputs+ if hasattr(elements, "__len__") and hasattr(elements, "__getitem__"):+ for x in range(0, len(elements), self.batch):+ yield elements[x : x + self.batch]++ # Build batches by iterating over elements when inputs are dynamically generated (i.e. generators)+ else:+ batch = []+ for x in elements:+ batch.append(x)++ if len(batch) == self.batch:+ yield batch+ batch = []++ # Final batch+ if batch:+ yield batch+def process(self, elements):"""Processes a batch of data elements.Args:- elements: list of data elements+ elements: iterable data elementsReturns:transformed data elements"""+ # Run elements through each taskfor task in self.tasks:- # Build list of elements with unique process ids- indexed = list(enumerate(elements))-- # Filter data down to data this task handles- data = [(x, task.upack(element)) for x, element in indexed if task.accept(task.upack(element, True))]-- # Get list of filtered process ids- ids = [x for x, _ in data]-- # Execute task action- results = task([element for _, element in data])-- # Update with transformed elements. Handle one to many transformations.- elements = []- for x, element in indexed:- if x in ids:- # Get result for process id- result = results[ids.index(x)]-- if isinstance(result, list):- # One-many transformations- elements.extend([task.pack(element, r) for r in result])- else:- # One-one transformations- elements.append(task.pack(element, result))- else:- # Pass unprocessed elements through- elements.append(element)-- # Remove process index and yield results+ elements = task(elements)++ # Yield results processed by all tasksyield from elementsdef finalize(self):diff --git a/src/python/txtai/workflow/task/base.py b/src/python/txtai/workflow/task/base.pyindex ce97c730..b2c0fa78 100644--- a/src/python/txtai/workflow/task/base.py+++ b/src/python/txtai/workflow/task/base.py@@ -4,6 +4,9 @@ Task moduleimport re+import numpy as np+import torch+class Task:"""@@ -20,7 +23,7 @@ class Task:select: filter(s) used to select data to processunpack: if data elements should be unpacked or unwrapped from (id, data, tag) tuplescolumn: column index to select if element is a tuple, defaults to all- merge: merge mode for joining multi-action outputs+ merge: merge mode for joining multi-action outputs, defaults to hstackinitialize: action to execute before processingfinalize: action to execute after processing"""@@ -44,17 +47,109 @@ class Task:Executes action for a list of data elements.Args:- elements: list of data elements+ elements: iterable data elements++ Returns:+ transformed data elements+ """++ if isinstance(elements, list):+ return self.filteredrun(elements)++ return self.run(elements)++ def filteredrun(self, elements):+ """+ Executes a filtered run, which will tag all inputs with a process id, filter elements down to elements the+ task can handle and execute on that subset. Items not selected for processing will be returned unmodified.++ Args:+ elements: iterable data elements++ Returns:+ transformed data elements+ """++ # Build list of elements with unique process ids+ indexed = list(enumerate(elements))++ # Filter data down to data this task handles+ data = [(x, self.upack(element)) for x, element in indexed if self.accept(self.upack(element, True))]++ # Get list of filtered process ids+ ids = [x for x, _ in data]++ # Prepare elements and execute task action(s)+ results = self.execute([self.prepare(element) for _, element in data])++ # Pack results back into elements+ if self.merge:+ elements = self.filteredpack(results, indexed, ids)+ else:+ elements = [self.filteredpack(r, indexed, ids) for r in results]++ return elements++ def filteredpack(self, results, indexed, ids):+ """+ Processes and packs results back into original input elements.++ Args:+ results: task results+ indexed: original elements indexed by process id+ ids: process ids accepted by this task++ Returns:+ packed elements+ """++ # Update with transformed elements. Handle one to many transformations.+ elements = []+ for x, element in indexed:+ if x in ids:+ # Get result for process id+ result = results[ids.index(x)]++ if isinstance(result, OneToMany):+ # One to many transformations+ elements.extend([self.pack(element, r) for r in result])+ else:+ # One to one transformations+ elements.append(self.pack(element, result))+ else:+ # Pass unprocessed elements through+ elements.append(element)++ return elements++ def run(self, elements):+ """+ Executes a task run for elements. A standard run processes all elements.++ Args:+ elements: iterable data elementsReturns:transformed data elements"""- # Run data preparation routines- elements = [self.prepare(element) for element in elements]+ # Execute task actions+ results = self.execute(elements)++ # Handle one to many transformations+ if isinstance(results, list):+ elements = []+ for result in results:+ if isinstance(result, OneToMany):+ # One to many transformations+ elements.extend(result)+ else:+ # One to one transformations+ elements.append(result)- # Run data elements through task execution- return self.execute(elements)+ return elements++ return resultsdef accept(self, element):"""@@ -67,7 +162,7 @@ class Task:True if this task can process this data element, False otherwise"""- return (isinstance(element, str) and re.search(self.select, element.lower())) if element and self.select else True+ return (isinstance(element, str) and re.search(self.select, element.lower())) if element is not None and self.select else Truedef upack(self, element, force=False):"""@@ -145,7 +240,7 @@ class Task:inputs = [self.extract(e, index) for e in elements] if index is not None else elements# Run action and add outputs- outputs.append(action(inputs))+ outputs.append(self.process(action, inputs))# Run post process operationsreturn self.postprocess(outputs)@@ -172,6 +267,27 @@ class Task:return element+ def process(self, action, inputs):+ """+ Executes action using inputs as arguments.++ Args:+ action: callable object+ inputs: action inputs++ Returns:+ action outputs+ """++ # Execute action and get outputs+ outputs = action(inputs)++ # Wrap one to many transformations if necessary+ if isinstance(outputs, list):+ outputs = [OneToMany(output) if isinstance(output, list) else output for output in outputs]++ return outputs+def postprocess(self, outputs):"""Runs post process routines after a task action.@@ -187,6 +303,10 @@ class Task:if len(self.action) == 1:return outputs[0]+ # Return unmodified outputs when merge set to None+ if not self.merge:+ return outputs+if self.merge == "vstack":return self.vstack(outputs)if self.merge == "concat":@@ -197,7 +317,7 @@ class Task:def vstack(self, outputs):"""- Merges outputs row-wise. Returns a list of lists which will be interpreted by workflows as a one-many transformation.+ Merges outputs row-wise. Returns a list of lists which will be interpreted as a one to many transformation.Row-wise merge example (2 actions)@@ -211,14 +331,24 @@ class Task:outputs: task outputsReturns:- list of aggregated/zipped outputs as lists (row-wise)+ list of aggregated/zipped outputs as one to many transforms (row-wise)"""- return [list(x) for x in zip(*outputs)]+ # If all outputs are numpy arrays, use native method+ if all([isinstance(output, np.ndarray) for output in outputs]):+ return np.concatenate(np.stack(outputs, axis=1))++ # If all outputs are torch tensors, use native method+ # pylint: disable=E1101+ if all([torch.is_tensor(output) for output in outputs]):+ return torch.cat(tuple(torch.stack(outputs, axis=1)))++ # Wrap as one to many transforms+ return [OneToMany(x) for x in zip(*outputs)]def hstack(self, outputs):"""- Merges outputs column-wise. Returns a list of tuples which will be interpreted by workflows as a one-one transformation.+ Merges outputs column-wise. Returns a list of tuples which will be interpreted as a one to one transformation.Column-wise merge example (2 actions)@@ -235,6 +365,15 @@ class Task:list of aggregated/zipped outputs as tuples (column-wise)"""+ # If all outputs are numpy arrays, use native method+ if all([isinstance(output, np.ndarray) for output in outputs]):+ return np.stack(outputs, axis=1)++ # If all outputs are torch tensors, use native method+ # pylint: disable=E1101+ if all([torch.is_tensor(output) for output in outputs]):+ return torch.stack(outputs, axis=1)+return list(zip(*outputs))def concat(self, outputs):@@ -257,3 +396,25 @@ class Task:"""return [". ".join([str(y) for y in x if y]) for x in self.hstack(outputs)]+++class OneToMany:+ """+ Encapsulates list output for a one to many transformation.+ """++ def __init__(self, values):+ """+ Creates a new OneToMany transformation.++ Args:+ values: list of outputs+ """++ self.values = values++ def __iter__(self):+ return self.values.__iter__()++ def __str__(self):+ return self.values.__str__()diff --git a/src/python/txtai/workflow/task/storage.py b/src/python/txtai/workflow/task/storage.pyindex b421bcde..8601c8dd 100644--- a/src/python/txtai/workflow/task/storage.py+++ b/src/python/txtai/workflow/task/storage.py@@ -31,31 +31,31 @@ class StorageTask(Task):super().__init__(action, select, unpack)- def accept(self, element):- # Only accept file URLs- return re.match(StorageTask.PREFIX, element.lower())+ def __call__(self, elements):+ # Create aggregated directory listing for all elements+ outputs = []+ for element in elements:+ if self.matches(element):+ # Get directory listing and run actions+ outputs.extend(super().__call__(self.list(element)))+ else:+ outputs.append(element)- def faccept(self, element):+ return outputs++ def matches(self, element):"""- Determines if this task can handle the input data format.+ Determines if this element is a storage element.Args:- element: input file url+ element: input storage elementReturns:- True if url is accepted, False otherwise- True if this task can process this data element, False otherwise+ True if this is a storage element"""- return super().accept(element)-- def execute(self, elements):- # Create aggregated directory listing for all elements- outputs = []- for element in elements:- outputs.append(super().execute([url for url in self.list(element) if self.faccept(url)]))-- return outputs+ # Only accept file URLs+ return re.match(StorageTask.PREFIX, self.upack(element, True).lower())def list(self, element):"""diff --git a/src/python/txtai/workflow/task/workflow.py b/src/python/txtai/workflow/task/workflow.pyindex 8dd63418..f05a4c95 100644--- a/src/python/txtai/workflow/task/workflow.py+++ b/src/python/txtai/workflow/task/workflow.py@@ -10,5 +10,5 @@ class WorkflowTask(Task):Task that executes a separate Workflow"""- def __call__(self, elements):- return list(super().__call__(elements))+ def process(self, action, inputs):+ return list(super().process(action, inputs))diff --git a/test/python/testapi/testworkflow.py b/test/python/testapi/testworkflow.pyindex eaa23009..b81c307c 100644--- a/test/python/testapi/testworkflow.py+++ b/test/python/testapi/testworkflow.py@@ -177,6 +177,7 @@ class TestWorkflow(unittest.TestCase):results = self.client.post("workflow", json={"name": "get", "elements": [text]}).json()self.assertEqual(len(results), 1)+ self.assertEqual(len(results[0]), 1)def testServicePost(self):"""@@ -186,7 +187,8 @@ class TestWorkflow(unittest.TestCase):text = "This is a test sentence. And another sentence to split."results = self.client.post("workflow", json={"name": "post", "elements": [text]}).json()- self.assertEqual(len(results), 2)+ self.assertEqual(len(results), 1)+ self.assertEqual(len(results[0]), 2)def testServiceXml(self):"""@@ -197,6 +199,7 @@ class TestWorkflow(unittest.TestCase):results = self.client.post("workflow", json={"name": "xml", "elements": [text]}).json()self.assertEqual(len(results), 1)+ self.assertEqual(len(results[0]), 1)def testWorkflowLabels(self):"""diff --git a/test/python/testworkflow.py b/test/python/testworkflow.pyindex cb103b97..00b2f492 100644--- a/test/python/testworkflow.py+++ b/test/python/testworkflow.py@@ -6,6 +6,9 @@ import osimport tempfileimport unittest+import numpy as np+import torch+from txtai.embeddings import Documents, Embeddingsfrom txtai.pipeline import Summary, Translation, Textractorfrom txtai.workflow import Workflow, Task, FileTask, ImageTask, RetrieveTask, StorageTask, WorkflowTask@@ -33,6 +36,17 @@ class TestWorkflow(unittest.TestCase):self.assertEqual(len(results), 2)+ def testChainWorkflow(self):+ """+ Tests a chain of workflows+ """++ workflow1 = Workflow([Task(lambda x: [y * 2 for y in x])])+ workflow2 = Workflow([Task(lambda x: [y - 1 for y in x])], batch=4)++ results = list(workflow2(workflow1([1, 2, 4, 8, 16, 32])))+ self.assertEqual(results, [1, 3, 7, 15, 31, 63])+def testComplexWorkflow(self):"""Tests a complex workflow@@ -110,26 +124,51 @@ class TestWorkflow(unittest.TestCase):# Test hstack (column-wise) mergeworkflow = Workflow([task])- results = list(workflow([2]))- self.assertEqual(results, [(4, 8)])+ results = list(workflow([2, 4]))+ self.assertEqual(results, [(4, 8), (16, 64)])# Test vstack (row-wise) mergetask.merge = "vstack"- workflow = Workflow([task])- results = list(workflow([2]))- self.assertEqual(results, [4, 8])+ results = list(workflow([2, 4]))+ self.assertEqual(results, [4, 8, 16, 64])# Test concat (values joined into single string) mergetask.merge = "concat"- workflow = Workflow([task])- results = list(workflow([2]))- self.assertEqual(results, ["4. 8"])+ results = list(workflow([2, 4]))+ self.assertEqual(results, ["4. 8", "16. 64"])++ # Test no merge+ task.merge = None+ results = list(workflow([2, 4, 6]))+ self.assertEqual(results, [[4, 16, 36], [8, 64, 216]])# Test generated (id, data, tag) tuples are properly returnedworkflow = Workflow([Task(lambda x: [(0, y, None) for y in x])])results = list(workflow([(1, "text", "tags")]))self.assertEqual(results[0], (0, "text", None))+ def testNumpyWorkflow(self):+ """+ Tests a numpy workflow+ """++ task = Task([lambda x: np.power(x, 2), lambda x: np.power(x, 3)], merge="hstack")++ # Test hstack (column-wise) merge+ workflow = Workflow([task])+ results = list(workflow(np.array([2, 4])))+ self.assertTrue(np.array_equal(np.array(results), np.array([[4, 8], [16, 64]])))++ # Test vstack (row-wise) merge+ task.merge = "vstack"+ results = list(workflow(np.array([2, 4])))+ self.assertEqual(results, [4, 8, 16, 64])++ # Test no merge+ task.merge = None+ results = list(workflow(np.array([2, 4, 6])))+ self.assertTrue(np.array_equal(np.array(results), np.array([[4, 16, 36], [8, 64, 216]])))+def testRetrieveWorkflow(self):"""Tests a retrieve task@@ -152,6 +191,46 @@ class TestWorkflow(unittest.TestCase):workflow = Workflow([StorageTask()])- results = list(workflow(["local://" + Utils.PATH]))+ results = list(workflow(["local://" + Utils.PATH, "test string"]))++ self.assertEqual(len(results), 19)++ def testTensorTransformWorkflow(self):+ """+ Tests a tensor workflow with list transformations+ """++ # Test one-one list transformation+ task = Task(lambda x: x.tolist())+ workflow = Workflow([task])+ results = list(workflow(np.array([2])))+ self.assertEqual(results, [2])++ # Test one-many list transformation+ task = Task(lambda x: [x.tolist() * 2])+ workflow = Workflow([task])+ results = list(workflow(np.array([2])))+ self.assertEqual(results, [2, 2])++ def testTorchWorkflow(self):+ """+ Tests a torch workflow+ """++ # pylint: disable=E1101,E1102+ task = Task([lambda x: torch.pow(x, 2), lambda x: torch.pow(x, 3)], merge="hstack")++ # Test hstack (column-wise) merge+ workflow = Workflow([task])+ results = np.array([x.numpy() for x in workflow(torch.tensor([2, 4]))])+ self.assertTrue(np.array_equal(results, np.array([[4, 8], [16, 64]])))++ # Test vstack (row-wise) merge+ task.merge = "vstack"+ results = list(workflow(torch.tensor([2, 4])))+ self.assertEqual(results, [4, 8, 16, 64])- self.assertEqual(len(results), 18)+ # Test no merge+ task.merge = None+ results = np.array([x.numpy() for x in workflow(torch.tensor([2, 4, 6]))])+ self.assertTrue(np.array_equal(np.array(results), np.array([[4, 16, 36], [8, 64, 216]])))