LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Document cleanup from VectorDB (#384)
Athul85726981e727
diff --git a/backend/prompt_studio/prompt_studio_index_manager/models.py b/backend/prompt_studio/prompt_studio_index_manager/models.pyindex 220688cb9..df249bb74 100644--- a/backend/prompt_studio/prompt_studio_index_manager/models.py+++ b/backend/prompt_studio/prompt_studio_index_manager/models.py@@ -1,11 +1,21 @@+import json+import loggingimport uuidfrom account.models import User-from django.db import models+from django.db import connection, models+from django.db.models.signals import post_delete+from django.dispatch import receiverfrom prompt_studio.prompt_profile_manager.models import ProfileManager+from prompt_studio.prompt_studio_core.prompt_ide_base_tool import PromptIdeBaseToolfrom prompt_studio.prompt_studio_document_manager.models import DocumentManager+from unstract.sdk.constants import LogLevel+from unstract.sdk.exceptions import SdkError+from unstract.sdk.vector_db import VectorDBfrom utils.models.base_model import BaseModel+logger = logging.getLogger(__name__)+class IndexManager(BaseModel):"""Model to store the index details."""@@ -78,3 +88,33 @@ class IndexManager(BaseModel):name="unique_document_manager_profile_manager",),]+++# Function will be executed every time an instance of IndexManager is deleted.+@receiver(post_delete, sender=IndexManager)+def perform_vector_db_cleanup(sender, instance, **kwargs):+ """Signal to perform vector db cleanup."""+ logger.info("Performing vector db cleanup")+ logger.debug("Document tool id: %s", instance.document_manager.tool_id)+ try:+ # Get the index_ids_history to clean up from the vector db+ index_ids_history = json.loads(instance.index_ids_history)+ vector_db_instance_id = str(instance.profile_manager.vector_store.id)+ # Generate a run_id+ org_schema = connection.tenant.schema_name+ util = PromptIdeBaseTool(log_level=LogLevel.INFO, org_id=org_schema)+ vector_db = VectorDB(+ tool=util,+ adapter_instance_id=vector_db_instance_id,+ )+ for index_id in index_ids_history:+ logger.debug(f"Deleting from VectorDB - index id: {index_id}")+ try:+ vector_db.delete(ref_doc_id=index_id)+ except Exception as e:+ # Log error and continue with the next index id+ logger.error(f"Error deleting index: {index_id} - {e}")+ # Not raising any exception.+ # Cleanup should not fail the deletion of the index manager.+ except SdkError as e:+ logger.error(f"Error while performing vector db cleanup: {e}")