Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Add data migration to remove the duplicate rows (#351)

Rahul Johny7288effe43ba
diff --git a/backend/workflow_manager/workflow/migrations/0004_filehistory_workflow_cachekey.py b/backend/workflow_manager/workflow/migrations/0004_filehistory_workflow_cachekey.py
index b1512608d..e3a7ac762 100644
--- a/backend/workflow_manager/workflow/migrations/0004_filehistory_workflow_cachekey.py
+++ b/backend/workflow_manager/workflow/migrations/0004_filehistory_workflow_cachekey.py
@@ -1,6 +1,8 @@
# Generated by Django 4.2.1 on 2024-05-14 07:03
+from typing import Any
from django.db import migrations, models
+from django.db.models import Count
class Migration(migrations.Migration):
@@ -9,7 +11,42 @@ class Migration(migrations.Migration):
("workflow", "0003_workflowexecution_execution_log_id_and_more"),
]
+ def UniqueConstraintDataMigration(apps: Any, schema_editor: Any) -> None:
+ """Migrate data to enforce unique constraint on the 'FileHistory' model
+ based on the fields 'workflow' and 'cache_key'.
+
+ For each duplicate combination of 'workflow' and 'cache_key',
+ keep the entry with the latest 'created_at' timestamp and delete the rest.
+ Parameters:
+ apps (Any): The state of the historical models at the point in time
+ when the migration is applied.
+ schema_editor (Any): The backend-specific schema editor class for
+ the database.
+ Returns:
+ None
+ """
+ FileHistory = apps.get_model("workflow", "FileHistory")
+
+ duplicates = (
+ FileHistory.objects.values("workflow", "cache_key")
+ .annotate(count=Count("id"))
+ .filter(count__gt=1)
+ )
+
+ for duplicate in duplicates:
+ entries = FileHistory.objects.filter(
+ workflow=duplicate["workflow"],
+ cache_key=duplicate["cache_key"],
+ ).order_by("-created_at")
+ entries_to_keep = entries.first()
+ entries_to_delete = entries.exclude(id=entries_to_keep.id)
+ entries_to_delete.delete()
+
operations = [
+ migrations.RunPython(
+ UniqueConstraintDataMigration,
+ reverse_code=migrations.RunPython.noop,
+ ),
migrations.AddConstraint(
model_name="filehistory",
constraint=models.UniqueConstraint(