💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Export all workflow batches in ExportTask (#1481)
Mohammad Hijjawi7128fa0f1098
diff --git a/src/python/txtai/workflow/task/export.py b/src/python/txtai/workflow/task/export.pyindex f0c56016..302a0089 100644--- a/src/python/txtai/workflow/task/export.py+++ b/src/python/txtai/workflow/task/export.py@@ -37,28 +37,53 @@ class ExportTask(Task):self.output = outputself.timestamp = timestamp+ # Rows exported and output file path for the current run+ self.rows, self.path = [], None++ # Workflows run this task once per batch. Start a new export when a workflow run starts.+ self.initializer, self.initialize = self.initialize, self.start+def __call__(self, elements, executor=None):# Run taskoutputs = super().__call__(elements, executor)+ # Add outputs to the rows exported for the current run+ self.rows.extend(outputs)+# Get output file extensionoutput = self.outputparts = list(os.path.splitext(output))extension = parts[-1].lower()- # Add timestamp to filename- if self.timestamp:- timestamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ")- parts[-1] = timestamp + parts[-1]+ # Resolve output file path once per run+ # pylint: disable=W0201+ if not self.path:+ # Add timestamp to filename+ if self.timestamp:+ timestamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ")+ parts[-1] = timestamp + parts[-1]++ # Create full path to output file+ output = ".".join(parts)- # Create full path to output file- output = ".".join(parts)+ self.path = output- # Write output+ # Write all rows exported for the current runif extension == ".xlsx":- pd.DataFrame(outputs).to_excel(output, index=False)+ pd.DataFrame(self.rows).to_excel(self.path, index=False)else:- pd.DataFrame(outputs).to_csv(output, index=False)+ pd.DataFrame(self.rows).to_csv(self.path, index=False)# Return resultsreturn outputs++ def start(self):+ """+ Starts a new export and runs the initialize action, if any.+ """++ # pylint: disable=W0201+ self.rows, self.path = [], None++ if self.initializer:+ self.initializer()diff --git a/test/python/testworkflow.py b/test/python/testworkflow.pyindex df35c610..7f1e0cd9 100644--- a/test/python/testworkflow.py+++ b/test/python/testworkflow.py@@ -302,6 +302,30 @@ class TestWorkflow(unittest.TestCase):path = glob.glob(os.path.join(tempfile.gettempdir(), "export-timestamp*.csv"))[0]self.assertGreater(os.path.getsize(path), 0)+ def testExportWorkflowBatches(self):+ """+ Test an export task with multiple batches+ """++ path = os.path.join(tempfile.gettempdir(), "export-batches.csv")++ # Track initialize action calls+ runs = []+ workflow = Workflow([ExportTask(output=path, initialize=lambda: runs.append(1))], batch=2)++ # All batches are exported+ list(workflow([{"id": x, "text": f"Sentence {x}"} for x in range(5)]))+ with open(path, encoding="utf-8") as f:+ self.assertEqual(len(f.readlines()), 6)++ # Each workflow run starts a new export+ list(workflow([{"id": x, "text": f"Sentence {x}"} for x in range(3)]))+ with open(path, encoding="utf-8") as f:+ self.assertEqual(len(f.readlines()), 4)++ # Initialize action runs once per workflow run+ self.assertEqual(len(runs), 2)+def testExtractWorkflow(self):"""Test column extraction tasks