LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-3266 [FIX] Re-indexing issue in Prompt Studio (#1907)
harini-venkataraman8dec64efb4a1
diff --git a/backend/prompt_studio/prompt_studio_core_v2/internal_urls.py b/backend/prompt_studio/prompt_studio_core_v2/internal_urls.pyindex 19b5c1b37..c88a8986d 100644--- a/backend/prompt_studio/prompt_studio_core_v2/internal_urls.py+++ b/backend/prompt_studio/prompt_studio_core_v2/internal_urls.py@@ -10,6 +10,11 @@ urlpatterns = [path("output/", internal_views.prompt_output, name="prompt-output"),path("index/", internal_views.index_update, name="index-update"),path("indexing-status/", internal_views.indexing_status, name="indexing-status"),+ path(+ "extraction-status/",+ internal_views.extraction_status,+ name="extraction-status",+ ),path("profile/<str:profile_id>/",internal_views.profile_detail,diff --git a/backend/prompt_studio/prompt_studio_core_v2/internal_views.py b/backend/prompt_studio/prompt_studio_core_v2/internal_views.pyindex ba0f0ff8c..3ad3a5db1 100644--- a/backend/prompt_studio/prompt_studio_core_v2/internal_views.py+++ b/backend/prompt_studio/prompt_studio_core_v2/internal_views.py@@ -154,6 +154,73 @@ def index_update(request):)+@csrf_exempt+@require_http_methods(["POST"])+def extraction_status(request):+ """Mark IndexManager.extraction_status for a document+profile pair.++ Called by the ide_callback worker after a successful ide_index run so+ that subsequent Answer Prompt dispatches can short-circuit extraction+ via PromptStudioIndexHelper.check_extraction_status.++ Expected JSON payload:+ {+ "document_id": str,+ "profile_manager_id": str,+ "x2text_config_hash": str,+ "enable_highlight": bool,+ "extracted": bool (optional, default true),+ "error_message": str | null (optional)+ }+ """+ data, err = _parse_json_body(request)+ if err:+ return err++ document_id = data.get("document_id", "")+ profile_manager_id = data.get("profile_manager_id", "")+ x2text_config_hash = data.get("x2text_config_hash", "")+ enable_highlight = data.get("enable_highlight", False)+ extracted = data.get("extracted", True)+ error_message = data.get("error_message")++ if not document_id or not profile_manager_id or not x2text_config_hash:+ return JsonResponse(+ {+ "success": False,+ "error": (+ "document_id, profile_manager_id, and x2text_config_hash "+ "are required"+ ),+ },+ status=status.HTTP_400_BAD_REQUEST,+ )++ try:+ from prompt_studio.prompt_profile_manager_v2.models import ProfileManager+ from prompt_studio.prompt_studio_index_manager_v2.prompt_studio_index_helper import (+ PromptStudioIndexHelper,+ )++ profile_manager = ProfileManager.objects.get(pk=profile_manager_id)+ success = PromptStudioIndexHelper.mark_extraction_status(+ document_id=document_id,+ profile_manager=profile_manager,+ x2text_config_hash=x2text_config_hash,+ enable_highlight=enable_highlight,+ extracted=extracted,+ error_message=error_message,+ )+ return JsonResponse({"success": success})++ except Exception as e:+ logger.exception("extraction_status internal API failed")+ return JsonResponse(+ {"success": False, "error": str(e)},+ status=status.HTTP_500_INTERNAL_SERVER_ERROR,+ )++@csrf_exempt@require_http_methods(["POST"])def indexing_status(request):diff --git a/backend/prompt_studio/prompt_studio_index_manager_v2/prompt_studio_index_helper.py b/backend/prompt_studio/prompt_studio_index_manager_v2/prompt_studio_index_helper.pyindex b4b5e4625..d17c15786 100644--- a/backend/prompt_studio/prompt_studio_index_manager_v2/prompt_studio_index_helper.py+++ b/backend/prompt_studio/prompt_studio_index_manager_v2/prompt_studio_index_helper.py@@ -97,11 +97,6 @@ class PromptStudioIndexHelper:with transaction.atomic():document = DocumentManager.objects.get(pk=document_id)- args = {- "document_manager": document,- "profile_manager": profile_manager,- }-# Build extraction status datastatus_data = {"extracted": extracted,@@ -112,13 +107,23 @@ class PromptStudioIndexHelper:if not extracted and error_message:status_data["error"] = error_message- defaults = {"extraction_status": {x2text_config_hash: status_data}}-- index_manager, created = IndexManager.objects.update_or_create(- **args,- defaults=defaults,+ # Lock the row (or create an empty one) so concurrent callers+ # merge into the same dict rather than clobbering each other.+ index_manager, created = (+ IndexManager.objects.select_for_update().get_or_create(+ document_manager=document,+ profile_manager=profile_manager,+ defaults={"extraction_status": {}},+ ))+ # Merge in place — update_or_create(defaults=...) would replace+ # the whole dict and wipe any prior hash entries.+ extraction_status = dict(index_manager.extraction_status or {})+ extraction_status[x2text_config_hash] = status_data+ index_manager.extraction_status = extraction_status+ index_manager.save(update_fields=["extraction_status"])+logger.info(f"Index manager {index_manager} {index_manager.index_ids_history}")diff --git a/workers/executor/executors/legacy_executor.py b/workers/executor/executors/legacy_executor.pyindex 5b5359988..9a3bf710e 100644--- a/workers/executor/executors/legacy_executor.py+++ b/workers/executor/executors/legacy_executor.py@@ -1026,9 +1026,20 @@ class LegacyExecutor(BaseExecutor):doc_id_found,reindex,)+ if doc_id_found and not reindex:+ shim.stream_log(+ "Document already indexed in vector store; skipping re-index."+ )+ logger.info(+ "Skipping re-index: doc_id=%s already in vector DB and "+ "reindex=False",+ doc_id,+ )+ return ExecutionResult(success=True, data={IKeys.DOC_ID: doc_id})+if doc_id_found and reindex:shim.stream_log("Document already indexed, re-indexing...")- elif not doc_id_found:+ else:shim.stream_log("Indexing document for the first time...")shim.stream_log("Indexing document into vector store...")index.perform_indexing(diff --git a/workers/ide_callback/tasks.py b/workers/ide_callback/tasks.pyindex 6c8452b01..cb610678b 100644--- a/workers/ide_callback/tasks.py+++ b/workers/ide_callback/tasks.py@@ -211,6 +211,31 @@ def ide_index_complete(profile_manager_id,)+ # Mark extraction_status so subsequent Answer Prompt dispatches+ # can short-circuit re-extraction. The Phase 4 backend payload+ # already stashes x2text_config_hash and enable_highlight in+ # cb_kwargs for exactly this purpose. Failure here is non-fatal:+ # primary indexing already succeeded above.+ x2text_config_hash = cb.get("x2text_config_hash", "")+ enable_highlight = cb.get("enable_highlight", False)+ if x2text_config_hash and profile_manager_id:+ try:+ api.mark_extraction_status(+ document_id=document_id,+ profile_manager_id=profile_manager_id,+ x2text_config_hash=x2text_config_hash,+ enable_highlight=enable_highlight,+ organization_id=org_id,+ )+ except Exception:+ logger.warning(+ "Failed to mark extraction_status for document %s "+ "profile %s; primary indexing succeeded.",+ document_id,+ profile_manager_id,+ exc_info=True,+ )+# Handle summary index tracking via backend endpoint# (requires PromptIdeBaseTool + IndexingUtils which need Django ORM)summary_profile_id = cb.get("summary_profile_id", "")diff --git a/workers/shared/clients/prompt_studio_client.py b/workers/shared/clients/prompt_studio_client.pyindex da2b81f8a..7ee6142c3 100644--- a/workers/shared/clients/prompt_studio_client.py+++ b/workers/shared/clients/prompt_studio_client.py@@ -15,6 +15,7 @@ logger = logging.getLogger(__name__)_OUTPUT_ENDPOINT = "v1/prompt-studio/output/"_INDEX_ENDPOINT = "v1/prompt-studio/index/"_INDEXING_STATUS_ENDPOINT = "v1/prompt-studio/indexing-status/"+_EXTRACTION_STATUS_ENDPOINT = "v1/prompt-studio/extraction-status/"_PROFILE_ENDPOINT = "v1/prompt-studio/profile/{profile_id}/"_HUBSPOT_ENDPOINT = "v1/prompt-studio/hubspot-notify/"_SUMMARY_INDEX_KEY_ENDPOINT = "v1/prompt-studio/summary-index-key/"@@ -71,6 +72,33 @@ class PromptStudioAPIClient(BaseAPIClient):}return self.post(_INDEX_ENDPOINT, data=payload, organization_id=organization_id)+ def mark_extraction_status(+ self,+ document_id: str,+ profile_manager_id: str,+ x2text_config_hash: str,+ enable_highlight: bool,+ organization_id: str | None = None,+ extracted: bool = True,+ error_message: str | None = None,+ ) -> dict[str, Any]:+ """Mark IndexManager.extraction_status for a document+profile pair.++ Called from the ide_index_complete callback so that subsequent+ Answer Prompt dispatches can short-circuit re-extraction.+ """+ payload = {+ "document_id": document_id,+ "profile_manager_id": profile_manager_id,+ "x2text_config_hash": x2text_config_hash,+ "enable_highlight": enable_highlight,+ "extracted": extracted,+ "error_message": error_message,+ }+ return self.post(+ _EXTRACTION_STATUS_ENDPOINT, data=payload, organization_id=organization_id+ )+def mark_document_indexed(self,org_id: str,diff --git a/workers/tests/test_legacy_executor_index.py b/workers/tests/test_legacy_executor_index.pyindex d87d5b5b9..b8e5bdd41 100644--- a/workers/tests/test_legacy_executor_index.py+++ b/workers/tests/test_legacy_executor_index.py@@ -220,6 +220,38 @@ class TestReindex:assert result.success is Trueinit_call = mock_index_cls.call_argsassert init_call.kwargs["processing_options"].reindex is True+ # reindex=True with already-indexed doc must still call perform_indexing+ mock_index_cls.return_value.perform_indexing.assert_called_once()++ @patch(_PATCH_FS)+ def test_already_indexed_no_reindex_short_circuits(+ self, mock_get_fs, mock_indexing_deps+ ):+ """doc_id already in VDB and reindex=False → skip perform_indexing.++ This is the defense-in-depth guard introduced for the IDE+ re-indexing fix: even if the Redis cache misses and Answer Prompt+ re-dispatches index, the executor must not re-write the same chunks+ into the vector store.+ """+ mock_index_cls, mock_emb_cls, mock_vdb_cls = mock_indexing_deps+ _register_legacy()+ executor = ExecutorRegistry.get("legacy")++ mock_index = _setup_mock_index(mock_index_cls, "doc-already-indexed")+ mock_index.is_document_indexed.return_value = True+ mock_emb_cls.return_value = MagicMock()+ mock_vdb_cls.return_value = MagicMock()+ mock_get_fs.return_value = MagicMock()++ # reindex defaults to False+ ctx = _make_index_context()+ result = executor.execute(ctx)++ assert result.success is True+ assert result.data[IKeys.DOC_ID] == "doc-already-indexed"+ mock_index.is_document_indexed.assert_called_once()+ mock_index.perform_indexing.assert_not_called()# --- 5. VectorDB.close() always called ---