Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Added unique constraint at model level to enforce integrity (#334)

Rahul Johnyc1d3d6655568
diff --git a/backend/workflow_manager/endpoint/destination.py b/backend/workflow_manager/endpoint/destination.py
index a293eaf2e..3024e7fa3 100644
--- a/backend/workflow_manager/endpoint/destination.py
+++ b/backend/workflow_manager/endpoint/destination.py
@@ -119,6 +119,7 @@ class DestinationConnector(BaseConnector):
workflow=workflow,
status=ExecutionStatus.COMPLETED,
result=result,
+ file_name=file_name,
)
def copy_output_to_output_directory(self) -> None:
diff --git a/backend/workflow_manager/workflow/file_history_helper.py b/backend/workflow_manager/workflow/file_history_helper.py
index b49bd3507..193c96b33 100644
--- a/backend/workflow_manager/workflow/file_history_helper.py
+++ b/backend/workflow_manager/workflow/file_history_helper.py
@@ -1,9 +1,13 @@
+import logging
from typing import Any, Optional
+from django.db.utils import IntegrityError
from workflow_manager.workflow.enums import ExecutionStatus
from workflow_manager.workflow.models.file_history import FileHistory
from workflow_manager.workflow.models.workflow import Workflow
+logger = logging.getLogger(__name__)
+
class FileHistoryHelper:
"""A helper class for managing file history related operations."""
@@ -37,6 +41,7 @@ class FileHistoryHelper:
status: ExecutionStatus,
result: Any,
error: Optional[str] = None,
+ file_name: Optional[str] = None,
) -> FileHistory:
"""Create a new file history record.
@@ -49,13 +54,25 @@ class FileHistoryHelper:
Returns:
FileHistory: The newly created file history record.
"""
- file_history: FileHistory = FileHistory.objects.create(
- workflow=workflow,
- cache_key=cache_key,
- status=status.value,
- result=str(result),
- error=str(error) if error else "",
- )
+ try:
+ file_history: FileHistory = FileHistory.objects.create(
+ workflow=workflow,
+ cache_key=cache_key,
+ status=status.value,
+ result=str(result),
+ error=str(error) if error else "",
+ )
+ except IntegrityError:
+ # TODO: Need to find why duplicate insert is coming
+ logger.warning(
+ "Trying to insert duplication data for filename %s for workflow %s",
+ file_name,
+ workflow,
+ )
+ file_history = FileHistoryHelper.get_file_history(
+ workflow=workflow, cache_key=cache_key
+ )
+
return file_history
@staticmethod
diff --git a/backend/workflow_manager/workflow/migrations/0004_filehistory_workflow_cachekey.py b/backend/workflow_manager/workflow/migrations/0004_filehistory_workflow_cachekey.py
new file mode 100644
index 000000000..b1512608d
--- /dev/null
+++ b/backend/workflow_manager/workflow/migrations/0004_filehistory_workflow_cachekey.py
@@ -0,0 +1,19 @@
+# Generated by Django 4.2.1 on 2024-05-14 07:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("workflow", "0003_workflowexecution_execution_log_id_and_more"),
+ ]
+
+ operations = [
+ migrations.AddConstraint(
+ model_name="filehistory",
+ constraint=models.UniqueConstraint(
+ fields=("workflow", "cache_key"), name="workflow_cacheKey"
+ ),
+ ),
+ ]
diff --git a/backend/workflow_manager/workflow/models/file_history.py b/backend/workflow_manager/workflow/models/file_history.py
index 7bc61f480..03c8bbeb3 100644
--- a/backend/workflow_manager/workflow/models/file_history.py
+++ b/backend/workflow_manager/workflow/models/file_history.py
@@ -39,3 +39,11 @@ class FileHistory(BaseModel):
db_comment="Error message",
)
result = models.TextField(blank=True, db_comment="Result from execution")
+
+ class Meta:
+ constraints = [
+ models.UniqueConstraint(
+ fields=["workflow", "cache_key"],
+ name="workflow_cacheKey",
+ ),
+ ]