LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
fix: Separated migrations to avoid DB trigger related issues (#1234)
Chandrasekharan Ma113caf05b15
diff --git a/backend/workflow_manager/workflow_v2/execution.py b/backend/workflow_manager/workflow_v2/execution.pyindex dff30fdde..a10441210 100644--- a/backend/workflow_manager/workflow_v2/execution.py+++ b/backend/workflow_manager/workflow_v2/execution.py@@ -146,8 +146,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):workflow_execution = WorkflowExecution.objects.create(id=execution_id if execution_id else uuid.uuid4(),pipeline_id=pipeline_id,- workflow_id=workflow_id,- related_workflow=Workflow.objects.get(id=workflow_id),+ workflow=Workflow.objects.get(id=workflow_id),execution_mode=mode,execution_method=execution_method,execution_type=execution_type,diff --git a/backend/workflow_manager/workflow_v2/migrations/0012_remove_wf_exec_with_invalid_wf_references.py b/backend/workflow_manager/workflow_v2/migrations/0012_remove_wf_exec_with_invalid_wf_references.pynew file mode 100644index 000000000..7c596564a--- /dev/null+++ b/backend/workflow_manager/workflow_v2/migrations/0012_remove_wf_exec_with_invalid_wf_references.py@@ -0,0 +1,74 @@+# Generated by Django 4.2.1 on 2025-04-08 13:14++import logging+from typing import Any++from django.db import migrations++logger = logging.getLogger(__name__)+++def remove_execution_of_deleted_wfs(apps: Any, schema_editor: Any):+ """Removes records from workflow_execution which do not have an associated workflow."""+ # Get the models from the migration state+ WorkflowExecution = apps.get_model( # pylint: disable=invalid-name # NOSONAR+ "workflow_v2", "WorkflowExecution"+ )+ Workflow = apps.get_model( # pylint: disable=invalid-name # NOSONAR+ "workflow_v2", "Workflow"+ )++ # Get all valid workflow IDs+ valid_workflow_ids = set(Workflow.objects.values_list("id", flat=True))++ # Find invalid workflow executions+ invalid_executions = WorkflowExecution.objects.filter(+ workflow_id__isnull=False+ ).exclude(workflow_id__in=valid_workflow_ids)++ invalid_count = invalid_executions.count()++ if invalid_count > 0:+ logger.warning(+ f"Deleting '{invalid_count}' workflow executions with "+ "invalid workflow references"+ )++ # Delete in batches to avoid long-running transactions+ batch_size = 1000+ deleted_total = 0++ while True:+ # Get a batch of IDs to delete+ batch_ids = list(+ invalid_executions[:batch_size].values_list("id", flat=True)+ )++ if not batch_ids:+ break++ # Delete this batch+ deleted_count = WorkflowExecution.objects.filter(id__in=batch_ids).delete()[+ 0+ ]+ deleted_total += deleted_count++ # Includes deletion of related models also+ logger.info(f"Deleted {deleted_count} records (total: {deleted_total})")++ # If we deleted fewer than batch_size, we're done+ if deleted_count < batch_size:+ break+++class Migration(migrations.Migration):++ dependencies = [+ ("workflow_v2", "0011_remove_filehistory_unique_workflow_cachekey_and_more"),+ ]++ operations = [+ migrations.RunPython(+ remove_execution_of_deleted_wfs, reverse_code=migrations.RunPython.noop+ ),+ ]diff --git a/backend/workflow_manager/workflow_v2/migrations/0012_remove_workflowexecution_workflow_id_and_more.py b/backend/workflow_manager/workflow_v2/migrations/0012_remove_workflowexecution_workflow_id_and_more.pydeleted file mode 100644index abd0459fe..000000000--- a/backend/workflow_manager/workflow_v2/migrations/0012_remove_workflowexecution_workflow_id_and_more.py+++ /dev/null@@ -1,71 +0,0 @@-# Generated by Django 4.2.1 on 2025-04-07 09:25--import logging--import django.db.models.deletion-from django.db import connection, migrations, models--logger = logging.getLogger(__name__)---def remove_execution_of_deleted_wfs(apps, schema_editor):- """Removes records from workflow_execution which do not have an associated workflow."""- with connection.cursor() as cursor:- # Count records with invalid workflow_id- cursor.execute(- """- SELECT COUNT(*) FROM workflow_execution- WHERE workflow_id IS NOT NULL AND workflow_id::text NOT IN- (SELECT id::text FROM workflow)- """- )- invalid_count = cursor.fetchone()[0]-- if invalid_count > 0:- logger.warning(- f"Deleting '{invalid_count}' workflow executions with "- "invalid workflow references"- )-- # Delete records with invalid workflow_id- cursor.execute(- """- DELETE FROM workflow_execution- WHERE workflow_id IS NOT NULL AND workflow_id::text NOT IN- (SELECT id::text FROM workflow)- """- )---class Migration(migrations.Migration):-- dependencies = [- ("workflow_v2", "0011_remove_filehistory_unique_workflow_cachekey_and_more"),- ]-- operations = [- # Run custom function to handle workflow_id conversion- migrations.RunPython(- remove_execution_of_deleted_wfs, reverse_code=migrations.RunPython.noop- ),- # Alter the field to be a foreign key while keeping the same column name- migrations.AlterField(- model_name="workflowexecution",- name="workflow_id",- field=models.ForeignKey(- db_column="workflow_id",- db_comment="Workflow to be executed",- editable=False,- null=True,- on_delete=django.db.models.deletion.CASCADE,- related_name="workflow_executions",- to="workflow_v2.workflow",- ),- ),- # Rename the field in Django's ORM (doesn't affect the database column name)- migrations.RenameField(- model_name="workflowexecution",- old_name="workflow_id",- new_name="workflow",- ),- ]diff --git a/backend/workflow_manager/workflow_v2/migrations/0013_remove_workflowexecution_workflow_id_and_more.py b/backend/workflow_manager/workflow_v2/migrations/0013_remove_workflowexecution_workflow_id_and_more.pynew file mode 100644index 000000000..36518ab50--- /dev/null+++ b/backend/workflow_manager/workflow_v2/migrations/0013_remove_workflowexecution_workflow_id_and_more.py@@ -0,0 +1,33 @@+# Generated by Django 4.2.1 on 2025-04-08 13:20++import django.db.models.deletion+from django.db import migrations, models+++class Migration(migrations.Migration):++ dependencies = [+ ("workflow_v2", "0012_remove_wf_exec_with_invalid_wf_references"),+ ]++ operations = [+ migrations.AlterField(+ model_name="workflowexecution",+ name="workflow_id",+ field=models.ForeignKey(+ db_column="workflow_id",+ db_comment="Workflow to be executed",+ editable=False,+ null=True,+ on_delete=django.db.models.deletion.CASCADE,+ related_name="workflow_executions",+ to="workflow_v2.workflow",+ ),+ ),+ # Rename the field in Django's ORM (doesn't affect the database column name)+ migrations.RenameField(+ model_name="workflowexecution",+ old_name="workflow_id",+ new_name="workflow",+ ),+ ]