LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-3266 [FIX] Preventing re-extraction in managing documents (#1909)
harini-venkataraman345e920a3fb6
diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.pyindex 7dfa870b4..d0ffef311 100644--- a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py+++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py@@ -509,6 +509,49 @@ class PromptStudioHelper:storage_type=StorageType.PERMANENT,env_name=FileStorageKeys.PERMANENT_REMOTE_STORAGE,)++ # Compute x2text_config_hash early so the marker check below (and the+ # post-success callback) can both consume the same value.+ x2text_metadata = default_profile.x2text.metadata or {}+ x2text_config_hash = ToolUtils.hash_str(+ json.dumps(x2text_metadata, sort_keys=True)+ )++ # Manage Documents → Index: mirror the pre-async dynamic_extractor+ # behaviour. If the extraction marker says this x2text_config_hash ++ # enable_highlight combination is already extracted, read the existing+ # extract file from disk and reuse it so the executor can skip the+ # extract step. Any failure here falls back to full extraction.+ reused_extracted_text: str | None = None+ try:+ already_extracted = PromptStudioIndexHelper.check_extraction_status(+ document_id=document_id,+ profile_manager=default_profile,+ x2text_config_hash=x2text_config_hash,+ enable_highlight=tool.enable_highlight,+ )+ if already_extracted:+ try:+ reused_extracted_text = fs_instance.read(+ path=extract_file_path, mode="r"+ )+ logger.info(+ "Manage Documents index: marker valid, reusing existing "+ "extract file for document=%s",+ document_id,+ )+ except FileNotFoundError:+ logger.warning(+ "Marker says extracted but extract file missing: %s. "+ "Will re-extract.",+ extract_file_path,+ )+ except Exception:+ logger.warning(+ "check_extraction_status raised; falling back to full extraction",+ exc_info=True,+ )+util = PromptIdeBaseTool(log_level=LogLevel.INFO, org_id=org_id)doc_id_key = IndexingUtils.generate_index_key(vector_db=str(default_profile.vector_store.id),@@ -550,6 +593,11 @@ class PromptStudioHelper:"platform_api_key": platform_api_key,}+ # On marker-hit, pre-populate the extracted text so the executor's+ # _handle_ide_index skips the extract step entirely.+ if reused_extracted_text:+ index_params[IKeys.EXTRACTED_TEXT] = reused_extracted_text+log_events_id = StateStore.get(Common.LOG_EVENTS_ID) or ""request_id = StateStore.get(Common.REQUEST_ID) or ""@@ -568,12 +616,9 @@ class PromptStudioHelper:log_events_id=log_events_id,)- # x2text config hash for extraction status tracking in callback- x2text_metadata = default_profile.x2text.metadata or {}- x2text_config_hash = ToolUtils.hash_str(- json.dumps(x2text_metadata, sort_keys=True)- )-+ # x2text_config_hash (computed above) is forwarded to the callback so+ # ide_index_complete can refresh the extraction marker via+ # mark_extraction_status.cb_kwargs = {"log_events_id": log_events_id,"request_id": request_id,diff --git a/backend/prompt_studio/prompt_studio_core_v2/tests/__init__.py b/backend/prompt_studio/prompt_studio_core_v2/tests/__init__.pynew file mode 100644index 000000000..41e4777a2--- /dev/null+++ b/backend/prompt_studio/prompt_studio_core_v2/tests/__init__.py@@ -0,0 +1 @@+# Tests for prompt_studio_core_v2.diff --git a/backend/prompt_studio/prompt_studio_core_v2/tests/test_build_index_payload.py b/backend/prompt_studio/prompt_studio_core_v2/tests/test_build_index_payload.pynew file mode 100644index 000000000..adb571324--- /dev/null+++ b/backend/prompt_studio/prompt_studio_core_v2/tests/test_build_index_payload.py@@ -0,0 +1,492 @@+"""Regression tests for ``PromptStudioHelper.build_index_payload``.++These tests pin the Manage Documents → Index marker-reuse behaviour+introduced to fix the "extract runs every time" QA bug. The helper+must:++ 1. On a valid extraction marker + readable extract file, pre-populate+ ``index_params[IKeys.EXTRACTED_TEXT]`` so the executor's+ ``_handle_ide_index`` skips the extract step entirely.+ 2. On a marker hit where the extract file is missing, fall back to+ full extraction (do NOT pre-populate the field).+ 3. On a marker miss, fall back to full extraction.+ 4. On an error inside ``check_extraction_status``, swallow the error+ and fall back to full extraction — the dispatch must not fail.++The backend test environment has no ``pytest-django``, no SQLite+fallback, and the helper has a heavy Django-coupled import surface.+Rather than spin up Django, we stub every collaborator as a+``MagicMock`` on ``sys.modules`` *before* importing the helper, and+then patch ``PromptStudioHelper`` class methods per-test. This mirrors+the ``usage_v2/tests/test_helper.py`` approach.++If the helper module cannot be imported in a given environment (for+example because the stub surface has drifted), all tests in the module+are skipped with a clear reason.+"""++from __future__ import annotations++import sys+import types+from typing import Any+from unittest.mock import MagicMock, patch++import pytest+++# ---------------------------------------------------------------------------+# Stub every collaborator module on sys.modules before importing the helper.+# These stubs are intentionally broad MagicMocks — the tests patch the+# specific attributes they care about via ``unittest.mock.patch``.+# ---------------------------------------------------------------------------+++def _install(name: str, attrs: dict[str, Any] | None = None) -> types.ModuleType:+ """Install (or replace) a fake module into ``sys.modules``.++ Always creates a fresh ``ModuleType``; this is important because the+ real module may already have been imported before these stubs run+ (via pytest collection, conftest, etc.), and we need our fake to+ actually take effect.+ """+ mod = types.ModuleType(name)+ if attrs:+ for key, value in attrs.items():+ setattr(mod, key, value)+ sys.modules[name] = mod+ return mod+++def _install_package(name: str) -> types.ModuleType:+ """Install a fake package (marked with ``__path__``).++ Only stubs the package if it is not already in ``sys.modules``.+ This prevents clobbering packages like ``unstract.core`` that must+ retain their real ``__path__`` for submodule resolution. The child+ modules we care about are always replaced explicitly via+ ``_install``.+ """+ if name in sys.modules:+ return sys.modules[name]+ mod = types.ModuleType(name)+ mod.__path__ = [] # type: ignore[attr-defined]+ sys.modules[name] = mod+ return mod+++try:+ # Account / adapter stubs+ _install_package("account_v2")+ _install(+ "account_v2.constants",+ {"Common": type("Common", (), {"LOG_EVENTS_ID": "log_events_id",+ "REQUEST_ID": "request_id"})},+ )+ _install("account_v2.models", {"User": MagicMock(name="User")})+ _install_package("adapter_processor_v2")+ _install(+ "adapter_processor_v2.constants",+ {"AdapterKeys": type("AdapterKeys", (), {})},+ )+ _install(+ "adapter_processor_v2.models",+ {"AdapterInstance": MagicMock(name="AdapterInstance")},+ )++ # Plugins stub+ _install("plugins", {"get_plugin": MagicMock(return_value=None)})++ # utils stubs+ _install_package("utils")+ _install_package("utils.file_storage")+ _install(+ "utils.file_storage.constants",+ {+ "FileStorageKeys": type(+ "FileStorageKeys",+ (),+ {"PERMANENT_REMOTE_STORAGE": "permanent"},+ )+ },+ )+ _install_package("utils.file_storage.helpers")+ _install(+ "utils.file_storage.helpers.prompt_studio_file_helper",+ {"PromptStudioFileHelper": MagicMock(name="PromptStudioFileHelper")},+ )+ _install(+ "utils.local_context",+ {"StateStore": MagicMock(name="StateStore")},+ )++ # backend.celery_service stub+ _install_package("backend")+ _install(+ "backend.celery_service",+ {"app": MagicMock(name="celery_app")},+ )++ # prompt_studio stubs+ _install_package("prompt_studio")+ _install_package("prompt_studio.prompt_profile_manager_v2")+ _install(+ "prompt_studio.prompt_profile_manager_v2.models",+ {"ProfileManager": MagicMock(name="ProfileManager")},+ )+ _install(+ "prompt_studio.prompt_profile_manager_v2.profile_manager_helper",+ {"ProfileManagerHelper": MagicMock(name="ProfileManagerHelper")},+ )++ _install_package("prompt_studio.prompt_studio_document_manager_v2")+ _install(+ "prompt_studio.prompt_studio_document_manager_v2.models",+ {"DocumentManager": MagicMock(name="DocumentManager")},+ )++ _install_package("prompt_studio.prompt_studio_index_manager_v2")+ _install(+ "prompt_studio.prompt_studio_index_manager_v2.prompt_studio_index_helper",+ {"PromptStudioIndexHelper": MagicMock(name="PromptStudioIndexHelper")},+ )++ _install_package("prompt_studio.prompt_studio_output_manager_v2")+ _install(+ "prompt_studio.prompt_studio_output_manager_v2.output_manager_helper",+ {"OutputManagerHelper": MagicMock(name="OutputManagerHelper")},+ )++ _install_package("prompt_studio.prompt_studio_v2")+ _install(+ "prompt_studio.prompt_studio_v2.models",+ {"ToolStudioPrompt": MagicMock(name="ToolStudioPrompt")},+ )++ # Stub the prompt_studio_core_v2 sibling modules too — several of them+ # transitively import modules (like ``utils.cache_service``) that we+ # don't want to pull in for these unit tests.+ _install_package("prompt_studio.prompt_studio_core_v2")+ _install(+ "prompt_studio.prompt_studio_core_v2.document_indexing_service",+ {"DocumentIndexingService": MagicMock(name="DocumentIndexingService")},+ )++ # Real exception classes — build_index_payload uses ``raise``.+ class _FakeExc(Exception):+ pass++ _install(+ "prompt_studio.prompt_studio_core_v2.exceptions",+ {+ "AnswerFetchError": type("AnswerFetchError", (_FakeExc,), {}),+ "DefaultProfileError": type("DefaultProfileError", (_FakeExc,), {}),+ "EmptyPromptError": type("EmptyPromptError", (_FakeExc,), {}),+ "ExtractionAPIError": type("ExtractionAPIError", (_FakeExc,), {}),+ "IndexingAPIError": type("IndexingAPIError", (_FakeExc,), {}),+ "NoPromptsFound": type("NoPromptsFound", (_FakeExc,), {}),+ "OperationNotSupported": type("OperationNotSupported", (_FakeExc,), {}),+ "PermissionError": type("PermissionError", (_FakeExc,), {}),+ },+ )+ _install(+ "prompt_studio.prompt_studio_core_v2.migration_utils",+ {"SummarizeMigrationUtils": MagicMock(name="SummarizeMigrationUtils")},+ )+ _install(+ "prompt_studio.prompt_studio_core_v2.models",+ {"CustomTool": MagicMock(name="CustomTool")},+ )+ _install(+ "prompt_studio.prompt_studio_core_v2.prompt_ide_base_tool",+ {"PromptIdeBaseTool": MagicMock(name="PromptIdeBaseTool")},+ )+ _install(+ "prompt_studio.prompt_studio_core_v2.prompt_variable_service",+ {"PromptStudioVariableService": MagicMock(name="PromptStudioVariableService")},+ )++ # unstract.core.pubsub_helper stub (LogPublisher isn't used by+ # build_index_payload but the module-level import must succeed).+ _install_package("unstract.core")+ _install(+ "unstract.core.pubsub_helper",+ {"LogPublisher": MagicMock(name="LogPublisher")},+ )++ # unstract.sdk1 stubs — these heavy modules transitively pull in+ # ``unstract.core.cache.redis_client`` which isn't on the python+ # path for the backend tests. We only need the leaf classes.+ _install_package("unstract.sdk1")+ _install(+ "unstract.sdk1.constants",+ {+ "LogLevel": type(+ "LogLevel", (), {"INFO": "INFO", "WARN": "WARN", "ERROR": "ERROR"}+ )+ },+ )+ _install(+ "unstract.sdk1.exceptions",+ {+ "IndexingError": type("IndexingError", (Exception,), {}),+ "SdkError": type("SdkError", (Exception,), {}),+ },+ )+ _install_package("unstract.sdk1.execution")++ class _FakeExecutionContext:+ """Minimal ExecutionContext that keeps ``executor_params`` as+ the real dict we pass in (the tests inspect it)."""++ def __init__(self, **kwargs: Any) -> None:+ self.executor_name = kwargs.get("executor_name")+ self.operation = kwargs.get("operation")+ self.run_id = kwargs.get("run_id")+ self.execution_source = kwargs.get("execution_source")+ self.organization_id = kwargs.get("organization_id")+ self.executor_params = kwargs.get("executor_params") or {}+ self.request_id = kwargs.get("request_id")+ self.log_events_id = kwargs.get("log_events_id")++ _install(+ "unstract.sdk1.execution.context",+ {"ExecutionContext": _FakeExecutionContext},+ )+ _install(+ "unstract.sdk1.execution.dispatcher",+ {"ExecutionDispatcher": MagicMock(name="ExecutionDispatcher")},+ )+ _install_package("unstract.sdk1.file_storage")+ _install(+ "unstract.sdk1.file_storage.constants",+ {"StorageType": type("StorageType", (), {"PERMANENT": "permanent"})},+ )+ _install(+ "unstract.sdk1.file_storage.env_helper",+ {"EnvHelper": MagicMock(name="EnvHelper")},+ )+ _install_package("unstract.sdk1.utils")+ _install(+ "unstract.sdk1.utils.indexing",+ {"IndexingUtils": MagicMock(name="IndexingUtils")},+ )+ _install(+ "unstract.sdk1.utils.tool",+ {"ToolUtils": MagicMock(name="ToolUtils")},+ )++ # Now import the helper module. If this fails, all tests below will+ # be skipped via the ``_IMPORT_ERROR`` sentinel.+ from prompt_studio.prompt_studio_core_v2 import prompt_studio_helper as _psh_mod # noqa: E402++ PromptStudioHelper = _psh_mod.PromptStudioHelper+ IKeys = _psh_mod.IKeys+ _IMPORT_ERROR: str | None = None+except Exception as exc: # pragma: no cover — environment guard+ _IMPORT_ERROR = (+ f"prompt_studio_helper could not be imported in this environment: "+ f"{type(exc).__name__}: {exc}"+ )+ PromptStudioHelper = None # type: ignore[assignment]+ IKeys = None # type: ignore[assignment]+++pytestmark = pytest.mark.skipif(+ _IMPORT_ERROR is not None, reason=_IMPORT_ERROR or ""+)+++# ---------------------------------------------------------------------------+# Helpers+# ---------------------------------------------------------------------------+++def _make_tool(enable_highlight: bool = False, summarize_context: bool = False):+ tool = MagicMock(name="CustomTool")+ tool.enable_highlight = enable_highlight+ tool.summarize_context = summarize_context+ return tool+++def _make_profile():+ profile = MagicMock(name="ProfileManager")+ profile.x2text.id = "x2t-1"+ profile.x2text.metadata = {"model": "default"}+ profile.embedding_model.id = "emb-1"+ profile.vector_store.id = "vdb-1"+ profile.chunk_size = 512+ profile.chunk_overlap = 64+ profile.profile_id = "profile-1"+ return profile+++def _dispatch_build(+ *,+ check_return: bool | Exception,+ read_return: str | Exception,+ tool: Any = None,+ profile: Any = None,+):+ """Run ``build_index_payload`` with all collaborators patched.++ ``check_return`` / ``read_return`` configure the two branches we+ care about:+ * ``check_return`` — ``check_extraction_status`` return value+ or an exception to raise.+ * ``read_return`` — ``fs_instance.read`` return value or an+ exception to raise.++ Returns the ``ExecutionContext`` built by ``build_index_payload``.+ """+ tool = tool or _make_tool()+ profile = profile or _make_profile()++ fs_instance = MagicMock(name="fs_instance")+ if isinstance(read_return, Exception):+ fs_instance.read.side_effect = read_return+ else:+ fs_instance.read.return_value = read_return++ check_mock = MagicMock(name="check_extraction_status")+ if isinstance(check_return, Exception):+ check_mock.side_effect = check_return+ else:+ check_mock.return_value = check_return++ # Patch everything via context managers so each test starts clean.+ patches = [+ patch.object(+ _psh_mod.CustomTool,+ "objects",+ MagicMock(get=MagicMock(return_value=tool)),+ ),+ patch.object(+ _psh_mod.PromptStudioFileHelper,+ "get_or_create_prompt_studio_subdirectory",+ return_value="/prompt-studio/org/user/tool",+ ),+ patch.object(+ _psh_mod.ProfileManager,+ "get_default_llm_profile",+ return_value=profile,+ ),+ patch.object(+ PromptStudioHelper,+ "validate_adapter_status",+ return_value=None,+ ),+ patch.object(+ PromptStudioHelper,+ "validate_profile_manager_owner_access",+ return_value=None,+ ),+ patch.object(+ PromptStudioHelper,+ "_get_platform_api_key",+ return_value="pk-test",+ ),+ patch.object(+ PromptStudioHelper,+ "_build_summarize_params",+ return_value=(None, "", MagicMock()),+ ),+ patch.object(+ _psh_mod.EnvHelper,+ "get_storage",+ return_value=fs_instance,+ ),+ patch.object(+ _psh_mod.PromptStudioIndexHelper,+ "check_extraction_status",+ check_mock,+ ),+ patch.object(+ _psh_mod.IndexingUtils,+ "generate_index_key",+ return_value="doc-key-1",+ ),+ patch.object(+ _psh_mod,+ "PromptIdeBaseTool",+ MagicMock(return_value=MagicMock()),+ ),+ patch.object(+ _psh_mod.StateStore,+ "get",+ return_value="",+ ),+ ]+ for p in patches:+ p.start()+ try:+ context, cb_kwargs = PromptStudioHelper.build_index_payload(+ tool_id="tool-1",+ file_name="doc.pdf",+ org_id="org-1",+ user_id="user-1",+ document_id="doc-1",+ run_id="run-1",+ )+ return context, cb_kwargs, fs_instance, check_mock+ finally:+ for p in patches:+ p.stop()+++# ---------------------------------------------------------------------------+# Tests+# ---------------------------------------------------------------------------+++class TestBuildIndexPayloadMarker:+ """Verify that build_index_payload honours the extraction marker."""++ def test_marker_hit_prepopulates_extracted_text(self) -> None:+ """Marker True + file readable → EXTRACTED_TEXT is pre-populated."""+ context, _cb, fs_instance, check_mock = _dispatch_build(+ check_return=True,+ read_return="existing extracted content",+ )+ index_params = context.executor_params["index_params"]+ assert index_params[IKeys.EXTRACTED_TEXT] == "existing extracted content"+ fs_instance.read.assert_called_once()+ check_mock.assert_called_once()++ def test_marker_hit_missing_file_does_not_prepopulate(self) -> None:+ """Marker True + FileNotFoundError → field NOT set, fall back to extract."""+ context, _cb, fs_instance, _check = _dispatch_build(+ check_return=True,+ read_return=FileNotFoundError("missing"),+ )+ index_params = context.executor_params["index_params"]+ assert IKeys.EXTRACTED_TEXT not in index_params+ fs_instance.read.assert_called_once()++ def test_marker_miss_does_not_prepopulate(self) -> None:+ """Marker False → EXTRACTED_TEXT NOT set, extract runs as before."""+ context, _cb, fs_instance, _check = _dispatch_build(+ check_return=False,+ read_return="should-not-be-read",+ )+ index_params = context.executor_params["index_params"]+ assert IKeys.EXTRACTED_TEXT not in index_params+ fs_instance.read.assert_not_called()++ def test_check_extraction_status_raises_is_swallowed(self, caplog) -> None:+ """check_extraction_status error → warn, field NOT set, no re-raise."""+ import logging as _logging++ caplog.set_level(_logging.WARNING, logger=_psh_mod.logger.name)+ context, _cb, fs_instance, _check = _dispatch_build(+ check_return=RuntimeError("db down"),+ read_return="should-not-be-read",+ )+ index_params = context.executor_params["index_params"]+ assert IKeys.EXTRACTED_TEXT not in index_params+ fs_instance.read.assert_not_called()+ # A warning should have been emitted about the fallback.+ assert any(+ "falling back to full extraction" in rec.getMessage()+ for rec in caplog.records+ )diff --git a/workers/executor/executors/legacy_executor.py b/workers/executor/executors/legacy_executor.pyindex 9a3bf710e..18e1f4774 100644--- a/workers/executor/executors/legacy_executor.py+++ b/workers/executor/executors/legacy_executor.py@@ -408,20 +408,30 @@ class LegacyExecutor(BaseExecutor):error=f"ide_index missing required params: {', '.join(missing)}")- # Step 1: Extract- extract_ctx = ExecutionContext(- executor_name=context.executor_name,- operation=Operation.EXTRACT.value,- run_id=context.run_id,- execution_source=context.execution_source,- organization_id=context.organization_id,- executor_params=extract_params,- request_id=context.request_id,- log_events_id=context.log_events_id,- )- extract_result = self._handle_extract(extract_ctx)- if not extract_result.success:- return extract_result+ # Step 1: Extract (or reuse pre-extracted text on marker hit)+ pre_extracted_text = index_params.get(IKeys.EXTRACTED_TEXT, "") or ""+ if pre_extracted_text:+ logger.info(+ "ide_index: marker hit, skipping extract step " "(len=%d, run_id=%s)",+ len(pre_extracted_text),+ context.run_id,+ )+ extracted_text = pre_extracted_text+ else:+ extract_ctx = ExecutionContext(+ executor_name=context.executor_name,+ operation=Operation.EXTRACT.value,+ run_id=context.run_id,+ execution_source=context.execution_source,+ organization_id=context.organization_id,+ executor_params=extract_params,+ request_id=context.request_id,+ log_events_id=context.log_events_id,+ )+ extract_result = self._handle_extract(extract_ctx)+ if not extract_result.success:+ return extract_result+ extracted_text = extract_result.data.get(IKeys.EXTRACTED_TEXT, "")# Step 2: Optional summarizesummarize_params = params.get("summarize_params")@@ -433,7 +443,6 @@ class LegacyExecutor(BaseExecutor):return result# Step 3: Index — inject extracted text- extracted_text = extract_result.data.get(IKeys.EXTRACTED_TEXT, "")index_params[IKeys.EXTRACTED_TEXT] = extracted_textindex_ctx = ExecutionContext(diff --git a/workers/tests/test_sanity_phase5.py b/workers/tests/test_sanity_phase5.pyindex 31675b8f9..4d00a571f 100644--- a/workers/tests/test_sanity_phase5.py+++ b/workers/tests/test_sanity_phase5.py@@ -377,6 +377,167 @@ class TestIdeIndexEagerChain:assert not result.successassert "X2Text" in result.error+ @patch(_PATCH_INDEX_DEPS)+ @patch(_PATCH_FS)+ @patch(_PATCH_X2TEXT)+ @patch(_PATCH_SHIM)+ def test_ide_index_reuses_pre_extracted_text(+ self,+ mock_shim,+ mock_x2text,+ mock_fs,+ mock_index_deps,+ eager_app,+ ):+ """Marker-hit path: extracted_text pre-populated → extract is skipped."""+ # If _handle_extract runs, this will blow up — the whole point is+ # that it must NOT be called when index_params carries extracted_text.+ x2t_instance = MagicMock()+ x2t_instance.process.side_effect = AssertionError(+ "extract must not run when index_params contains pre-extracted text"+ )+ mock_x2text.return_value = x2t_instance++ fs = MagicMock()+ fs.exists.return_value = False+ mock_fs.return_value = fs++ # Mock index — capture the extracted_text that reached _handle_index.+ index_inst = MagicMock()+ index_inst.generate_index_key.return_value = "idx-doc-reuse"+ index_inst.is_document_indexed.return_value = False+ mock_index_deps.return_value = (+ MagicMock(return_value=index_inst),+ MagicMock(),+ MagicMock(),+ )++ ctx = ExecutionContext(+ executor_name="legacy",+ operation="ide_index",+ run_id="run-ide-reuse",+ execution_source="ide",+ organization_id="org-test",+ executor_params={+ "extract_params": {+ "x2text_instance_id": "x2t-1",+ "file_path": "/data/doc.pdf",+ "enable_highlight": False,+ "output_file_path": "/data/extract/doc.txt",+ "platform_api_key": "pk-test",+ "usage_kwargs": {},+ },+ "index_params": {+ "tool_id": "tool-1",+ "embedding_instance_id": "emb-1",+ "vector_db_instance_id": "vdb-1",+ "x2text_instance_id": "x2t-1",+ "file_path": "/data/extract/doc.txt",+ "file_hash": None,+ "chunk_overlap": 64,+ "chunk_size": 512,+ "reindex": True,+ "enable_highlight": False,+ "usage_kwargs": {},+ "run_id": "run-ide-reuse",+ "execution_source": "ide",+ "platform_api_key": "pk-test",+ "extracted_text": "reused extracted payload",+ },+ },+ )++ result_dict = _run_task(eager_app, ctx.to_dict())+ result = ExecutionResult.from_dict(result_dict)++ assert result.success+ assert result.data["doc_id"] == "idx-doc-reuse"+ # Extract adapter must never have been called.+ x2t_instance.process.assert_not_called()+ # perform_indexing received the pre-populated text.+ perform_call_kwargs = index_inst.perform_indexing.call_args.kwargs+ assert (+ perform_call_kwargs.get("extracted_text") == "reused extracted payload"+ )++ @patch(_PATCH_INDEX_DEPS)+ @patch(_PATCH_FS)+ @patch(_PATCH_X2TEXT)+ @patch(_PATCH_SHIM)+ def test_ide_index_without_pre_extracted_text_runs_extract(+ self,+ mock_shim,+ mock_x2text,+ mock_fs,+ mock_index_deps,+ eager_app,+ ):+ """Marker-miss path: extract runs as before when extracted_text is absent."""+ x2t_instance = MagicMock()+ x2t_instance.process.return_value = _mock_process_response(+ "freshly extracted"+ )+ mock_x2text.return_value = x2t_instance++ fs = MagicMock()+ fs.exists.return_value = False+ mock_fs.return_value = fs++ index_inst = MagicMock()+ index_inst.generate_index_key.return_value = "idx-doc-fresh"+ index_inst.is_document_indexed.return_value = False+ mock_index_deps.return_value = (+ MagicMock(return_value=index_inst),+ MagicMock(),+ MagicMock(),+ )++ ctx = ExecutionContext(+ executor_name="legacy",+ operation="ide_index",+ run_id="run-ide-fresh",+ execution_source="ide",+ organization_id="org-test",+ executor_params={+ "extract_params": {+ "x2text_instance_id": "x2t-1",+ "file_path": "/data/doc.pdf",+ "enable_highlight": False,+ "output_file_path": "/data/extract/doc.txt",+ "platform_api_key": "pk-test",+ "usage_kwargs": {},+ },+ "index_params": {+ "tool_id": "tool-1",+ "embedding_instance_id": "emb-1",+ "vector_db_instance_id": "vdb-1",+ "x2text_instance_id": "x2t-1",+ "file_path": "/data/extract/doc.txt",+ "file_hash": None,+ "chunk_overlap": 64,+ "chunk_size": 512,+ "reindex": True,+ "enable_highlight": False,+ "usage_kwargs": {},+ "run_id": "run-ide-fresh",+ "execution_source": "ide",+ "platform_api_key": "pk-test",+ # No "extracted_text" key → must take the extract path.+ },+ },+ )++ result_dict = _run_task(eager_app, ctx.to_dict())+ result = ExecutionResult.from_dict(result_dict)++ assert result.success+ assert result.data["doc_id"] == "idx-doc-fresh"+ # Extract adapter was called exactly once.+ x2t_instance.process.assert_called_once()+ # perform_indexing received the freshly extracted text.+ perform_call_kwargs = index_inst.perform_indexing.call_args.kwargs+ assert perform_call_kwargs.get("extracted_text") == "freshly extracted"+# ---------------------------------------------------------------------------# 5D: structure_pipeline compound operation through eager chain