Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Document cleanup from VectorDB (#384)

Athul85726981e727
diff --git a/backend/prompt_studio/prompt_studio_index_manager/models.py b/backend/prompt_studio/prompt_studio_index_manager/models.py
index 220688cb9..df249bb74 100644
--- a/backend/prompt_studio/prompt_studio_index_manager/models.py
+++ b/backend/prompt_studio/prompt_studio_index_manager/models.py
@@ -1,11 +1,21 @@
+import json
+import logging
import uuid
from account.models import User
-from django.db import models
+from django.db import connection, models
+from django.db.models.signals import post_delete
+from django.dispatch import receiver
from prompt_studio.prompt_profile_manager.models import ProfileManager
+from prompt_studio.prompt_studio_core.prompt_ide_base_tool import PromptIdeBaseTool
from prompt_studio.prompt_studio_document_manager.models import DocumentManager
+from unstract.sdk.constants import LogLevel
+from unstract.sdk.exceptions import SdkError
+from unstract.sdk.vector_db import VectorDB
from utils.models.base_model import BaseModel
+logger = logging.getLogger(__name__)
+
class IndexManager(BaseModel):
"""Model to store the index details."""
@@ -78,3 +88,33 @@ class IndexManager(BaseModel):
name="unique_document_manager_profile_manager",
),
]
+
+
+# Function will be executed every time an instance of IndexManager is deleted.
+@receiver(post_delete, sender=IndexManager)
+def perform_vector_db_cleanup(sender, instance, **kwargs):
+ """Signal to perform vector db cleanup."""
+ logger.info("Performing vector db cleanup")
+ logger.debug("Document tool id: %s", instance.document_manager.tool_id)
+ try:
+ # Get the index_ids_history to clean up from the vector db
+ index_ids_history = json.loads(instance.index_ids_history)
+ vector_db_instance_id = str(instance.profile_manager.vector_store.id)
+ # Generate a run_id
+ org_schema = connection.tenant.schema_name
+ util = PromptIdeBaseTool(log_level=LogLevel.INFO, org_id=org_schema)
+ vector_db = VectorDB(
+ tool=util,
+ adapter_instance_id=vector_db_instance_id,
+ )
+ for index_id in index_ids_history:
+ logger.debug(f"Deleting from VectorDB - index id: {index_id}")
+ try:
+ vector_db.delete(ref_doc_id=index_id)
+ except Exception as e:
+ # Log error and continue with the next index id
+ logger.error(f"Error deleting index: {index_id} - {e}")
+ # Not raising any exception.
+ # Cleanup should not fail the deletion of the index manager.
+ except SdkError as e:
+ logger.error(f"Error while performing vector db cleanup: {e}")