LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Added unique constraint at model level to enforce integrity (#334)
Rahul Johnyc1d3d6655568
diff --git a/backend/workflow_manager/endpoint/destination.py b/backend/workflow_manager/endpoint/destination.pyindex a293eaf2e..3024e7fa3 100644--- a/backend/workflow_manager/endpoint/destination.py+++ b/backend/workflow_manager/endpoint/destination.py@@ -119,6 +119,7 @@ class DestinationConnector(BaseConnector):workflow=workflow,status=ExecutionStatus.COMPLETED,result=result,+ file_name=file_name,)def copy_output_to_output_directory(self) -> None:diff --git a/backend/workflow_manager/workflow/file_history_helper.py b/backend/workflow_manager/workflow/file_history_helper.pyindex b49bd3507..193c96b33 100644--- a/backend/workflow_manager/workflow/file_history_helper.py+++ b/backend/workflow_manager/workflow/file_history_helper.py@@ -1,9 +1,13 @@+import loggingfrom typing import Any, Optional+from django.db.utils import IntegrityErrorfrom workflow_manager.workflow.enums import ExecutionStatusfrom workflow_manager.workflow.models.file_history import FileHistoryfrom workflow_manager.workflow.models.workflow import Workflow+logger = logging.getLogger(__name__)+class FileHistoryHelper:"""A helper class for managing file history related operations."""@@ -37,6 +41,7 @@ class FileHistoryHelper:status: ExecutionStatus,result: Any,error: Optional[str] = None,+ file_name: Optional[str] = None,) -> FileHistory:"""Create a new file history record.@@ -49,13 +54,25 @@ class FileHistoryHelper:Returns:FileHistory: The newly created file history record."""- file_history: FileHistory = FileHistory.objects.create(- workflow=workflow,- cache_key=cache_key,- status=status.value,- result=str(result),- error=str(error) if error else "",- )+ try:+ file_history: FileHistory = FileHistory.objects.create(+ workflow=workflow,+ cache_key=cache_key,+ status=status.value,+ result=str(result),+ error=str(error) if error else "",+ )+ except IntegrityError:+ # TODO: Need to find why duplicate insert is coming+ logger.warning(+ "Trying to insert duplication data for filename %s for workflow %s",+ file_name,+ workflow,+ )+ file_history = FileHistoryHelper.get_file_history(+ workflow=workflow, cache_key=cache_key+ )+return file_history@staticmethoddiff --git a/backend/workflow_manager/workflow/migrations/0004_filehistory_workflow_cachekey.py b/backend/workflow_manager/workflow/migrations/0004_filehistory_workflow_cachekey.pynew file mode 100644index 000000000..b1512608d--- /dev/null+++ b/backend/workflow_manager/workflow/migrations/0004_filehistory_workflow_cachekey.py@@ -0,0 +1,19 @@+# Generated by Django 4.2.1 on 2024-05-14 07:03++from django.db import migrations, models+++class Migration(migrations.Migration):++ dependencies = [+ ("workflow", "0003_workflowexecution_execution_log_id_and_more"),+ ]++ operations = [+ migrations.AddConstraint(+ model_name="filehistory",+ constraint=models.UniqueConstraint(+ fields=("workflow", "cache_key"), name="workflow_cacheKey"+ ),+ ),+ ]diff --git a/backend/workflow_manager/workflow/models/file_history.py b/backend/workflow_manager/workflow/models/file_history.pyindex 7bc61f480..03c8bbeb3 100644--- a/backend/workflow_manager/workflow/models/file_history.py+++ b/backend/workflow_manager/workflow/models/file_history.py@@ -39,3 +39,11 @@ class FileHistory(BaseModel):db_comment="Error message",)result = models.TextField(blank=True, db_comment="Result from execution")++ class Meta:+ constraints = [+ models.UniqueConstraint(+ fields=["workflow", "cache_key"],+ name="workflow_cacheKey",+ ),+ ]