Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

fix: Separated migrations to avoid DB trigger related issues (#1234)

Chandrasekharan Ma113caf05b15
diff --git a/backend/workflow_manager/workflow_v2/execution.py b/backend/workflow_manager/workflow_v2/execution.py
index dff30fdde..a10441210 100644
--- a/backend/workflow_manager/workflow_v2/execution.py
+++ b/backend/workflow_manager/workflow_v2/execution.py
@@ -146,8 +146,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
workflow_execution = WorkflowExecution.objects.create(
id=execution_id if execution_id else uuid.uuid4(),
pipeline_id=pipeline_id,
- workflow_id=workflow_id,
- related_workflow=Workflow.objects.get(id=workflow_id),
+ workflow=Workflow.objects.get(id=workflow_id),
execution_mode=mode,
execution_method=execution_method,
execution_type=execution_type,
diff --git a/backend/workflow_manager/workflow_v2/migrations/0012_remove_wf_exec_with_invalid_wf_references.py b/backend/workflow_manager/workflow_v2/migrations/0012_remove_wf_exec_with_invalid_wf_references.py
new file mode 100644
index 000000000..7c596564a
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/migrations/0012_remove_wf_exec_with_invalid_wf_references.py
@@ -0,0 +1,74 @@
+# Generated by Django 4.2.1 on 2025-04-08 13:14
+
+import logging
+from typing import Any
+
+from django.db import migrations
+
+logger = logging.getLogger(__name__)
+
+
+def remove_execution_of_deleted_wfs(apps: Any, schema_editor: Any):
+ """Removes records from workflow_execution which do not have an associated workflow."""
+ # Get the models from the migration state
+ WorkflowExecution = apps.get_model( # pylint: disable=invalid-name # NOSONAR
+ "workflow_v2", "WorkflowExecution"
+ )
+ Workflow = apps.get_model( # pylint: disable=invalid-name # NOSONAR
+ "workflow_v2", "Workflow"
+ )
+
+ # Get all valid workflow IDs
+ valid_workflow_ids = set(Workflow.objects.values_list("id", flat=True))
+
+ # Find invalid workflow executions
+ invalid_executions = WorkflowExecution.objects.filter(
+ workflow_id__isnull=False
+ ).exclude(workflow_id__in=valid_workflow_ids)
+
+ invalid_count = invalid_executions.count()
+
+ if invalid_count > 0:
+ logger.warning(
+ f"Deleting '{invalid_count}' workflow executions with "
+ "invalid workflow references"
+ )
+
+ # Delete in batches to avoid long-running transactions
+ batch_size = 1000
+ deleted_total = 0
+
+ while True:
+ # Get a batch of IDs to delete
+ batch_ids = list(
+ invalid_executions[:batch_size].values_list("id", flat=True)
+ )
+
+ if not batch_ids:
+ break
+
+ # Delete this batch
+ deleted_count = WorkflowExecution.objects.filter(id__in=batch_ids).delete()[
+ 0
+ ]
+ deleted_total += deleted_count
+
+ # Includes deletion of related models also
+ logger.info(f"Deleted {deleted_count} records (total: {deleted_total})")
+
+ # If we deleted fewer than batch_size, we're done
+ if deleted_count < batch_size:
+ break
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("workflow_v2", "0011_remove_filehistory_unique_workflow_cachekey_and_more"),
+ ]
+
+ operations = [
+ migrations.RunPython(
+ remove_execution_of_deleted_wfs, reverse_code=migrations.RunPython.noop
+ ),
+ ]
diff --git a/backend/workflow_manager/workflow_v2/migrations/0012_remove_workflowexecution_workflow_id_and_more.py b/backend/workflow_manager/workflow_v2/migrations/0012_remove_workflowexecution_workflow_id_and_more.py
deleted file mode 100644
index abd0459fe..000000000
--- a/backend/workflow_manager/workflow_v2/migrations/0012_remove_workflowexecution_workflow_id_and_more.py
+++ /dev/null
@@ -1,71 +0,0 @@
-# Generated by Django 4.2.1 on 2025-04-07 09:25
-
-import logging
-
-import django.db.models.deletion
-from django.db import connection, migrations, models
-
-logger = logging.getLogger(__name__)
-
-
-def remove_execution_of_deleted_wfs(apps, schema_editor):
- """Removes records from workflow_execution which do not have an associated workflow."""
- with connection.cursor() as cursor:
- # Count records with invalid workflow_id
- cursor.execute(
- """
- SELECT COUNT(*) FROM workflow_execution
- WHERE workflow_id IS NOT NULL AND workflow_id::text NOT IN
- (SELECT id::text FROM workflow)
- """
- )
- invalid_count = cursor.fetchone()[0]
-
- if invalid_count > 0:
- logger.warning(
- f"Deleting '{invalid_count}' workflow executions with "
- "invalid workflow references"
- )
-
- # Delete records with invalid workflow_id
- cursor.execute(
- """
- DELETE FROM workflow_execution
- WHERE workflow_id IS NOT NULL AND workflow_id::text NOT IN
- (SELECT id::text FROM workflow)
- """
- )
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("workflow_v2", "0011_remove_filehistory_unique_workflow_cachekey_and_more"),
- ]
-
- operations = [
- # Run custom function to handle workflow_id conversion
- migrations.RunPython(
- remove_execution_of_deleted_wfs, reverse_code=migrations.RunPython.noop
- ),
- # Alter the field to be a foreign key while keeping the same column name
- migrations.AlterField(
- model_name="workflowexecution",
- name="workflow_id",
- field=models.ForeignKey(
- db_column="workflow_id",
- db_comment="Workflow to be executed",
- editable=False,
- null=True,
- on_delete=django.db.models.deletion.CASCADE,
- related_name="workflow_executions",
- to="workflow_v2.workflow",
- ),
- ),
- # Rename the field in Django's ORM (doesn't affect the database column name)
- migrations.RenameField(
- model_name="workflowexecution",
- old_name="workflow_id",
- new_name="workflow",
- ),
- ]
diff --git a/backend/workflow_manager/workflow_v2/migrations/0013_remove_workflowexecution_workflow_id_and_more.py b/backend/workflow_manager/workflow_v2/migrations/0013_remove_workflowexecution_workflow_id_and_more.py
new file mode 100644
index 000000000..36518ab50
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/migrations/0013_remove_workflowexecution_workflow_id_and_more.py
@@ -0,0 +1,33 @@
+# Generated by Django 4.2.1 on 2025-04-08 13:20
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("workflow_v2", "0012_remove_wf_exec_with_invalid_wf_references"),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name="workflowexecution",
+ name="workflow_id",
+ field=models.ForeignKey(
+ db_column="workflow_id",
+ db_comment="Workflow to be executed",
+ editable=False,
+ null=True,
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="workflow_executions",
+ to="workflow_v2.workflow",
+ ),
+ ),
+ # Rename the field in Django's ORM (doesn't affect the database column name)
+ migrations.RenameField(
+ model_name="workflowexecution",
+ old_name="workflow_id",
+ new_name="workflow",
+ ),
+ ]