LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Add data migration to remove the duplicate rows (#351)
Rahul Johny7288effe43ba
diff --git a/backend/workflow_manager/workflow/migrations/0004_filehistory_workflow_cachekey.py b/backend/workflow_manager/workflow/migrations/0004_filehistory_workflow_cachekey.pyindex b1512608d..e3a7ac762 100644--- a/backend/workflow_manager/workflow/migrations/0004_filehistory_workflow_cachekey.py+++ b/backend/workflow_manager/workflow/migrations/0004_filehistory_workflow_cachekey.py@@ -1,6 +1,8 @@# Generated by Django 4.2.1 on 2024-05-14 07:03+from typing import Anyfrom django.db import migrations, models+from django.db.models import Countclass Migration(migrations.Migration):@@ -9,7 +11,42 @@ class Migration(migrations.Migration):("workflow", "0003_workflowexecution_execution_log_id_and_more"),]+ def UniqueConstraintDataMigration(apps: Any, schema_editor: Any) -> None:+ """Migrate data to enforce unique constraint on the 'FileHistory' model+ based on the fields 'workflow' and 'cache_key'.++ For each duplicate combination of 'workflow' and 'cache_key',+ keep the entry with the latest 'created_at' timestamp and delete the rest.+ Parameters:+ apps (Any): The state of the historical models at the point in time+ when the migration is applied.+ schema_editor (Any): The backend-specific schema editor class for+ the database.+ Returns:+ None+ """+ FileHistory = apps.get_model("workflow", "FileHistory")++ duplicates = (+ FileHistory.objects.values("workflow", "cache_key")+ .annotate(count=Count("id"))+ .filter(count__gt=1)+ )++ for duplicate in duplicates:+ entries = FileHistory.objects.filter(+ workflow=duplicate["workflow"],+ cache_key=duplicate["cache_key"],+ ).order_by("-created_at")+ entries_to_keep = entries.first()+ entries_to_delete = entries.exclude(id=entries_to_keep.id)+ entries_to_delete.delete()+operations = [+ migrations.RunPython(+ UniqueConstraintDataMigration,+ reverse_code=migrations.RunPython.noop,+ ),migrations.AddConstraint(model_name="filehistory",constraint=models.UniqueConstraint(