Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Export all workflow batches in ExportTask (#1481)

Mohammad Hijjawi7128fa0f1098
diff --git a/src/python/txtai/workflow/task/export.py b/src/python/txtai/workflow/task/export.py
index f0c56016..302a0089 100644
--- a/src/python/txtai/workflow/task/export.py
+++ b/src/python/txtai/workflow/task/export.py
@@ -37,28 +37,53 @@ class ExportTask(Task):
self.output = output
self.timestamp = timestamp
+ # Rows exported and output file path for the current run
+ self.rows, self.path = [], None
+
+ # Workflows run this task once per batch. Start a new export when a workflow run starts.
+ self.initializer, self.initialize = self.initialize, self.start
+
def __call__(self, elements, executor=None):
# Run task
outputs = super().__call__(elements, executor)
+ # Add outputs to the rows exported for the current run
+ self.rows.extend(outputs)
+
# Get output file extension
output = self.output
parts = list(os.path.splitext(output))
extension = parts[-1].lower()
- # Add timestamp to filename
- if self.timestamp:
- timestamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ")
- parts[-1] = timestamp + parts[-1]
+ # Resolve output file path once per run
+ # pylint: disable=W0201
+ if not self.path:
+ # Add timestamp to filename
+ if self.timestamp:
+ timestamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+ parts[-1] = timestamp + parts[-1]
+
+ # Create full path to output file
+ output = ".".join(parts)
- # Create full path to output file
- output = ".".join(parts)
+ self.path = output
- # Write output
+ # Write all rows exported for the current run
if extension == ".xlsx":
- pd.DataFrame(outputs).to_excel(output, index=False)
+ pd.DataFrame(self.rows).to_excel(self.path, index=False)
else:
- pd.DataFrame(outputs).to_csv(output, index=False)
+ pd.DataFrame(self.rows).to_csv(self.path, index=False)
# Return results
return outputs
+
+ def start(self):
+ """
+ Starts a new export and runs the initialize action, if any.
+ """
+
+ # pylint: disable=W0201
+ self.rows, self.path = [], None
+
+ if self.initializer:
+ self.initializer()
diff --git a/test/python/testworkflow.py b/test/python/testworkflow.py
index df35c610..7f1e0cd9 100644
--- a/test/python/testworkflow.py
+++ b/test/python/testworkflow.py
@@ -302,6 +302,30 @@ class TestWorkflow(unittest.TestCase):
path = glob.glob(os.path.join(tempfile.gettempdir(), "export-timestamp*.csv"))[0]
self.assertGreater(os.path.getsize(path), 0)
+ def testExportWorkflowBatches(self):
+ """
+ Test an export task with multiple batches
+ """
+
+ path = os.path.join(tempfile.gettempdir(), "export-batches.csv")
+
+ # Track initialize action calls
+ runs = []
+ workflow = Workflow([ExportTask(output=path, initialize=lambda: runs.append(1))], batch=2)
+
+ # All batches are exported
+ list(workflow([{"id": x, "text": f"Sentence {x}"} for x in range(5)]))
+ with open(path, encoding="utf-8") as f:
+ self.assertEqual(len(f.readlines()), 6)
+
+ # Each workflow run starts a new export
+ list(workflow([{"id": x, "text": f"Sentence {x}"} for x in range(3)]))
+ with open(path, encoding="utf-8") as f:
+ self.assertEqual(len(f.readlines()), 4)
+
+ # Initialize action runs once per workflow run
+ self.assertEqual(len(runs), 2)
+
def testExtractWorkflow(self):
"""
Test column extraction tasks