Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

36.9 KB
Raw Blame
"""Structure tool task — Phase 3 of executor migration.

Replaces the Docker-container-based StructureTool.run() with a task that runs in
the file_processing worker. Instead of PromptTool HTTP calls to prompt-service,
it dispatches operations to the executor worker through the executor RPC.

Before (Docker-based):
    File Processing Worker → WorkflowExecutionService → ToolSandbox
    → Docker container → StructureTool.run() → PromptTool (HTTP) → prompt-service

After:
    File Processing Worker → WorkerWorkflowExecutionService
    → execute_structure_tool task → executor RPC (PG queue)
    → executor worker → LegacyExecutor
"""

import json
import logging
import os
import time
from pathlib import Path
from typing import Any

from queue_backend import worker_task
from queue_backend.pg_queue.executor_rpc import (
    PgExecutionDispatcher,
    get_executor_dispatcher,
)
from shared.enums.task_enums import TaskName
from shared.infrastructure.context import StateStore

from unstract.sdk1.constants import ToolEnv, UsageKwargs
from unstract.sdk1.execution.context import ExecutionContext
from unstract.sdk1.execution.result import ExecutionResult

logger = logging.getLogger(__name__)

# Timeout for executor worker calls (seconds).
# Reads from EXECUTOR_RESULT_TIMEOUT env, defaults to 3600.
EXECUTOR_TIMEOUT = int(os.environ.get("EXECUTOR_RESULT_TIMEOUT", 3600))


# NOTE: `_fairness_headers()` lived here and was passed as `headers=` to every
# executor dispatch. It is gone with the routing dispatcher (UN-4046):
# `PgExecutionDispatcher.dispatch()` takes no `headers`, and the router that used
# to absorb them never forwarded them to the PG path anyway — PG carries org
# routing in the enqueue payload (`transport.enqueue(..., org_id=...)`).
# UN-3504 (propagating the caller's WorkloadType) is unaffected: it was always
# about the PG payload, not these headers.

# -----------------------------------------------------------------------
# Constants mirrored from tools/structure/src/constants.py
# These are the keys used in tool_metadata and payload dicts.
# -----------------------------------------------------------------------


class _SK:
    """SettingsKeys subset needed by the structure tool task."""

    PROMPT_REGISTRY_ID = "prompt_registry_id"
    TOOL_METADATA = "tool_metadata"
    TOOL_ID = "tool_id"
    OUTPUTS = "outputs"
    TOOL_SETTINGS = "tool_settings"
    NAME = "name"
    ACTIVE = "active"
    PROMPT = "prompt"
    CHUNK_SIZE = "chunk-size"
    CHUNK_OVERLAP = "chunk-overlap"
    VECTOR_DB = "vector-db"
    EMBEDDING = "embedding"
    X2TEXT_ADAPTER = "x2text_adapter"
    LLM = "llm"
    CHALLENGE_LLM = "challenge_llm"
    ENABLE_CHALLENGE = "enable_challenge"
    ENABLE_SINGLE_PASS_EXTRACTION = "enable_single_pass_extraction"
    SUMMARIZE_AS_SOURCE = "summarize_as_source"
    ENABLE_HIGHLIGHT = "enable_highlight"
    ENABLE_WORD_CONFIDENCE = "enable_word_confidence"
    SUMMARIZE_PROMPT = "summarize_prompt"
    TABLE_SETTINGS = "table_settings"
    INPUT_FILE = "input_file"
    IS_DIRECTORY_MODE = "is_directory_mode"
    RUN_ID = "run_id"
    EXECUTION_ID = "execution_id"
    FILE_HASH = "file_hash"
    FILE_NAME = "file_name"
    FILE_PATH = "file_path"
    EXECUTION_SOURCE = "execution_source"
    TOOL = "tool"
    EXTRACT = "EXTRACT"
    SUMMARIZE = "SUMMARIZE"
    METADATA = "metadata"
    METRICS = "metrics"
    INDEXING = "indexing"
    OUTPUT = "output"
    CONTEXT = "context"
    DATA = "data"
    LLM_ADAPTER_INSTANCE_ID = "llm_adapter_instance_id"
    PROMPT_KEYS = "prompt_keys"
    LLM_PROFILE_ID = "llm_profile_id"
    CUSTOM_DATA = "custom_data"
    SINGLE_PASS_EXTRACTION_MODE = "single_pass_extraction_mode"
    CHALLENGE_LLM_ADAPTER_ID = "challenge_llm_adapter_id"


# -----------------------------------------------------------------------
# Standalone helper functions (extracted from StructureTool methods)
# -----------------------------------------------------------------------


def _apply_profile_overrides(tool_metadata: dict, profile_data: dict) -> list[str]:
    """Apply profile overrides to tool metadata.

    Standalone version of StructureTool._apply_profile_overrides.
    """
    changes: list[str] = []

    profile_to_tool_mapping = {
        "chunk_overlap": "chunk-overlap",
        "chunk_size": "chunk-size",
        "embedding_model_id": "embedding",
        "llm_id": "llm",
        "similarity_top_k": "similarity-top-k",
        "vector_store_id": "vector-db",
        "x2text_id": "x2text_adapter",
        "retrieval_strategy": "retrieval-strategy",
    }

    if "tool_settings" in tool_metadata:
        changes.extend(
            _override_section(
                tool_metadata["tool_settings"],
                profile_data,
                profile_to_tool_mapping,
                "tool_settings",
            )
        )

    if "outputs" in tool_metadata:
        for i, output in enumerate(tool_metadata["outputs"]):
            output_name = output.get("name", f"output_{i}")
            changes.extend(
                _override_section(
                    output,
                    profile_data,
                    profile_to_tool_mapping,
                    f"output[{output_name}]",
                )
            )

    return changes


def _override_section(
    section: dict,
    profile_data: dict,
    mapping: dict,
    section_name: str = "section",
) -> list[str]:
    """Override values in a section using profile data."""
    changes: list[str] = []
    for profile_key, section_key in mapping.items():
        if profile_key in profile_data and section_key in section:
            old_value = section[section_key]
            new_value = profile_data[profile_key]
            if old_value != new_value:
                section[section_key] = new_value
                change_desc = f"{section_name}.{section_key}: {old_value} -> {new_value}"
                changes.append(change_desc)
                logger.info("Overrode %s", change_desc)
    return changes


def _should_skip_extraction_for_smart_table(
    outputs: list[dict[str, Any]],
) -> bool:
    """Check if extraction and indexing should be skipped for smart table.

    Standalone version of StructureTool._should_skip_extraction_for_smart_table.
    """
    for output in outputs:
        if _SK.TABLE_SETTINGS not in output:
            continue
        prompt = output.get(_SK.PROMPT, "")
        if not prompt or not isinstance(prompt, str):
            continue
        try:
            schema_data = json.loads(prompt)
        except ValueError as e:
            logger.warning("Failed to parse prompt as JSON for smart table: %s", e)
            continue
        if isinstance(schema_data, dict) and schema_data:
            return True
    return False


def _agentic_extraction_seconds(agentic_metrics: dict[str, Any]) -> float:
    """Total X2Text seconds reported by the agentic_table executor.

    Each agentic prompt extracts the document itself, so the durations sum:
    the figure is time *spent* extracting, not the wall-clock span of the
    agentic step. Returns 0.0 when the executor reports no extraction timing,
    which leaves the file-level bucket untouched rather than writing a zero.
    """
    from executor.executors.constants import PromptServiceConstants as PSKeys

    total = 0.0
    for prompt_metrics in agentic_metrics.values():
        if not isinstance(prompt_metrics, dict):
            continue
        entry = prompt_metrics.get(PSKeys.TEXT_EXTRACTION)
        if not isinstance(entry, dict):
            continue
        value = entry.get(PSKeys.TIME_TAKEN)
        if isinstance(value, (int, float)) and not isinstance(value, bool):
            total += float(value)
    return total


def _merge_agentic_metrics(
    structured_output: dict[str, Any], agentic_metrics: dict[str, Any]
) -> None:
    """Fold the agentic executor's metrics into the tool's metrics dict.

    Two things happen here, and they are separate:

    * Per-prompt metrics land beside the legacy pipeline's, keyed by prompt
      name under ``table_extraction`` — the same shape
      ``LegacyExecutor._run_table_extraction`` uses for the non-agentic table
      plugin, so both table routes read alike downstream.
    * Any X2Text duration the executor reported is *added* to the file-level
      ``_file.text_extraction`` bucket. The legacy pipeline times only its own
      extraction, so without this an agentic-only run reports no extraction
      time at all and a mixed run under-reports it (UN-2771 / UN-4084).
    """
    from executor.executors.constants import PromptServiceConstants as PSKeys

    if not agentic_metrics:
        return

    metrics = structured_output.setdefault("metrics", {})
    for prompt_name, prompt_metrics in agentic_metrics.items():
        key = prompt_name
        if key == PSKeys.FILE:
            # `_file` is reserved for whole-document metrics, so a prompt with
            # that name gets a de-collided key. Merging it into the reserved
            # bucket would conflate per-prompt and file-level figures; dropping
            # it would lose every other count the prompt reported. The space
            # cannot appear in the reserved key, so the two can never alias.
            key = f"{prompt_name} (prompt)"
            logger.warning(
                "Agentic prompt named %r collides with the reserved file-level "
                "metrics namespace; reporting its metrics under %r instead",
                prompt_name,
                key,
            )
        metrics.setdefault(key, {}).update({"table_extraction": prompt_metrics})

    extraction_seconds = _agentic_extraction_seconds(agentic_metrics)
    if not extraction_seconds:
        return
    file_bucket = metrics.setdefault(PSKeys.FILE, {}).setdefault(
        PSKeys.TEXT_EXTRACTION, {}
    )
    file_bucket[PSKeys.TIME_TAKEN] = (
        file_bucket.get(PSKeys.TIME_TAKEN, 0.0) + extraction_seconds
    )


# -----------------------------------------------------------------------
# Main Celery task
# -----------------------------------------------------------------------


@worker_task(bind=True, name=str(TaskName.EXECUTE_STRUCTURE_TOOL))
def execute_structure_tool(self, params: dict) -> dict:
    """Execute structure tool as a task.

    Replicates StructureTool.run() from tools/structure/src/main.py
    but uses the executor RPC instead of PromptTool HTTP calls.

    Args:
        params: Dict with keys described in the Phase 3 plan.

    Returns:
        Dict with {"success": bool, "data": dict, "error": str|None}.
    """
    try:
        return _execute_structure_tool_impl(params)
    except Exception as e:
        logger.error("Structure tool task failed: %s", e, exc_info=True)
        return ExecutionResult.failure(error=f"Structure tool failed: {e}").to_dict()


def _execute_structure_tool_impl(params: dict) -> dict:
    """Implementation of the structure tool pipeline.

    Separated from the task function for testability.

    Phase 5E: Uses a single ``structure_pipeline`` dispatch instead of
    3 sequential ``dispatcher.dispatch()`` calls.  The executor worker
    handles the full extract → summarize → index → answer_prompt
    pipeline internally, freeing the file_processing worker slot.
    """
    # ---- Unpack params ----
    organization_id = params["organization_id"]
    execution_id = params.get("execution_id", "")
    file_execution_id = params["file_execution_id"]
    tool_instance_metadata = params["tool_instance_metadata"]
    platform_service_api_key = params["platform_service_api_key"]
    input_file_path = params["input_file_path"]
    output_dir_path = params["output_dir_path"]
    source_file_name = params["source_file_name"]
    execution_data_dir = params["execution_data_dir"]
    file_hash = params.get("file_hash", "")
    exec_metadata = params.get("exec_metadata", {})

    # ---- Step 1: Setup ----
    from executor.executor_tool_shim import ExecutorToolShim

    # Workflow IDs on the pre-dispatch shim let X2Text/platform helper logs
    # reach the workflow logs UI before the executor dispatch happens.
    log_events_id = StateStore.get("LOG_EVENTS_ID") or ""
    if not log_events_id:
        logger.warning(
            "LOG_EVENTS_ID missing from StateStore for execution_id=%s — "
            "tool-level logs will not stream to the workflow logs UI.",
            execution_id,
        )
    shim = ExecutorToolShim(
        platform_api_key=platform_service_api_key,
        log_events_id=log_events_id,
        execution_id=execution_id,
        file_execution_id=file_execution_id,
        organization_id=organization_id,
    )

    platform_helper = _create_platform_helper(shim, file_execution_id)
    # PG executor RPC. ``celery_app`` is accepted and ignored by the factory —
    # see executor_rpc.
    dispatcher = get_executor_dispatcher()
    fs = _get_file_storage()

    # ---- Step 2: Fetch tool metadata ----
    prompt_registry_id = tool_instance_metadata.get(_SK.PROMPT_REGISTRY_ID, "")
    logger.info("Fetching exported tool with UUID '%s'", prompt_registry_id)

    tool_metadata, is_agentic = _fetch_tool_metadata(platform_helper, prompt_registry_id)

    # ---- Route agentic vs regular ----
    if is_agentic:
        return _run_agentic_extraction(
            tool_metadata=tool_metadata,
            input_file_path=input_file_path,
            output_dir_path=output_dir_path,
            tool_instance_metadata=tool_instance_metadata,
            dispatcher=dispatcher,
            shim=shim,
            file_execution_id=file_execution_id,
            execution_id=execution_id,
            organization_id=organization_id,
            source_file_name=source_file_name,
            fs=fs,
            execution_data_dir=execution_data_dir,
        )

    # ---- Step 3: Profile overrides ----
    _handle_profile_overrides(exec_metadata, platform_helper, tool_metadata)

    # ---- Extract settings from tool_metadata ----
    settings = tool_instance_metadata
    is_challenge_enabled = settings.get(_SK.ENABLE_CHALLENGE, False)
    is_summarization_enabled = settings.get(_SK.SUMMARIZE_AS_SOURCE, False)
    is_single_pass_enabled = settings.get(_SK.SINGLE_PASS_EXTRACTION_MODE, False)
    challenge_llm = settings.get(_SK.CHALLENGE_LLM_ADAPTER_ID, "")
    is_highlight_enabled = settings.get(_SK.ENABLE_HIGHLIGHT, False)
    is_word_confidence_enabled = settings.get(_SK.ENABLE_WORD_CONFIDENCE, False)
    logger.info(
        "HIGHLIGHT_DEBUG structure_tool: is_highlight_enabled=%s "
        "is_word_confidence_enabled=%s from settings keys=%s",
        is_highlight_enabled,
        is_word_confidence_enabled,
        list(settings.keys()),
    )

    tool_id = tool_metadata[_SK.TOOL_ID]
    tool_settings = tool_metadata[_SK.TOOL_SETTINGS]
    all_outputs = tool_metadata[_SK.OUTPUTS]

    # ---- Partition prompts by enforce_type ----
    # Agentic table prompts run via a dedicated executor (page-by-page
    # extraction + Agent-5 schema cleanup). Regular prompts continue
    # through the legacy structure_pipeline. Use local variables so
    # tool_metadata[_SK.OUTPUTS] is preserved for METADATA.json
    # serialization downstream in _write_tool_result.
    agentic_table_outputs = [o for o in all_outputs if o.get("type") == "agentic_table"]
    regular_outputs = [o for o in all_outputs if o.get("type") != "agentic_table"]

    # Validate readiness for each agentic_table prompt: if the export
    # step did not populate agentic_table_settings, fail loudly so the
    # user knows to re-export the tool instead of producing the
    # legacy stringified-truncated output.
    for at_output in agentic_table_outputs:
        at_settings = at_output.get("agentic_table_settings") or {}
        if not at_settings.get("target_table") or not at_settings.get("json_structure"):
            return ExecutionResult.failure(
                error=(
                    f"Agentic table prompt '{at_output[_SK.NAME]}' is missing "
                    f"agentic_table_settings in the exported tool metadata. "
                    f"Re-export the tool from Prompt Studio after the fix is "
                    f"deployed to populate target_table / json_structure / "
                    f"instructions."
                )
            ).to_dict()

    outputs = regular_outputs

    # Inject workflow-level settings into tool_settings
    tool_settings[_SK.CHALLENGE_LLM] = challenge_llm
    tool_settings[_SK.ENABLE_CHALLENGE] = is_challenge_enabled
    tool_settings[_SK.ENABLE_SINGLE_PASS_EXTRACTION] = is_single_pass_enabled
    tool_settings[_SK.SUMMARIZE_AS_SOURCE] = is_summarization_enabled
    tool_settings[_SK.ENABLE_HIGHLIGHT] = is_highlight_enabled
    tool_settings[_SK.ENABLE_WORD_CONFIDENCE] = is_word_confidence_enabled

    _, file_name = os.path.split(input_file_path)
    if is_summarization_enabled:
        file_name = _SK.SUMMARIZE

    execution_run_data_folder = Path(execution_data_dir)
    extracted_input_file = str(execution_run_data_folder / _SK.EXTRACT)

    # ---- Step 4: Smart table detection ----
    skip_extraction_and_indexing = _should_skip_extraction_for_smart_table(outputs)
    if skip_extraction_and_indexing:
        logger.info(
            "Skipping extraction and indexing for Excel table with valid JSON schema"
        )

    # ---- Step 5: Build pipeline params ----
    usage_kwargs: dict[Any, Any] = {}
    if not skip_extraction_and_indexing:
        usage_kwargs[UsageKwargs.RUN_ID] = file_execution_id
        usage_kwargs[UsageKwargs.FILE_NAME] = source_file_name
        usage_kwargs[UsageKwargs.EXECUTION_ID] = execution_id

    custom_data = exec_metadata.get(_SK.CUSTOM_DATA, {})
    answer_params = {
        _SK.RUN_ID: file_execution_id,
        _SK.EXECUTION_ID: execution_id,
        _SK.TOOL_SETTINGS: tool_settings,
        _SK.OUTPUTS: outputs,
        _SK.TOOL_ID: tool_id,
        _SK.FILE_HASH: file_hash,
        _SK.FILE_NAME: file_name,
        _SK.FILE_PATH: extracted_input_file,
        # Never rewritten (unlike FILE_PATH, which summarize/smart-table
        # overrides mutate) — the page-image reader keys on this.
        "extract_file_path": extracted_input_file,
        _SK.EXECUTION_SOURCE: _SK.TOOL,
        _SK.CUSTOM_DATA: custom_data,
        "PLATFORM_SERVICE_API_KEY": platform_service_api_key,
    }

    extract_params = {
        "x2text_instance_id": tool_settings[_SK.X2TEXT_ADAPTER],
        "file_path": input_file_path,
        "enable_highlight": is_highlight_enabled,
        "output_file_path": str(execution_run_data_folder / _SK.EXTRACT),
        "platform_api_key": platform_service_api_key,
        "usage_kwargs": usage_kwargs,
        "tags": exec_metadata.get("tags"),
        "tool_execution_metadata": exec_metadata,
        "execution_data_dir": str(execution_run_data_folder),
    }

    index_template = {
        "tool_id": tool_id,
        "file_hash": file_hash,
        "is_highlight_enabled": is_highlight_enabled,
        "platform_api_key": platform_service_api_key,
        "extracted_file_path": extracted_input_file,
    }

    pipeline_options = {
        "skip_extraction_and_indexing": skip_extraction_and_indexing,
        "is_summarization_enabled": is_summarization_enabled,
        "is_single_pass_enabled": is_single_pass_enabled,
        "input_file_path": input_file_path,
        "source_file_name": source_file_name,
    }

    # Build summarize params if enabled
    summarize_params = None
    if is_summarization_enabled:
        prompt_keys = [o[_SK.NAME] for o in outputs]
        summarize_params = {
            "llm_adapter_instance_id": tool_settings[_SK.LLM],
            "summarize_prompt": tool_settings.get(_SK.SUMMARIZE_PROMPT, ""),
            "extract_file_path": str(execution_run_data_folder / _SK.EXTRACT),
            "summarize_file_path": str(execution_run_data_folder / _SK.SUMMARIZE),
            "platform_api_key": platform_service_api_key,
            "prompt_keys": prompt_keys,
        }

    # ---- Step 6a: Dispatch agentic_table prompts ----
    # Each agentic_table prompt runs in its own executor invocation.
    # The executor handles X2Text extraction internally; we just
    # forward the document path and the per-prompt settings unpacked
    # from agentic_table_settings (populated by Layer 1 export).
    #
    # Important: read from SOURCE, not INFILE. INFILE gets overwritten
    # with JSON output at the end of this function (line ~508), so any
    # subsequent reuse of the same file_execution_dir would surface JSON
    # bytes to the agentic_table executor and fail PDF parsing
    # ("PDFium: Data format error"). SOURCE is the immutable original
    # PDF written alongside INFILE by the source connector.
    agentic_source_path = str(execution_run_data_folder / "SOURCE")
    agentic_results: dict[str, Any] = {}
    agentic_metrics: dict[str, Any] = {}
    for at_output in agentic_table_outputs:
        at_settings = at_output.get("agentic_table_settings") or {}
        json_structure = at_settings.get("json_structure")
        if isinstance(json_structure, dict):
            json_structure = json.dumps(json_structure)
        agentic_params = {
            "llm_adapter_instance_id": at_output["llm"],
            "lite_llm_adapter_instance_id": at_settings.get(
                "lite_llm_adapter_instance_id", ""
            ),
            "x2text_adapter_instance_id": tool_settings[_SK.X2TEXT_ADAPTER],
            "input_file": agentic_source_path,
            "source_file_name": source_file_name,
            "target_table": at_settings.get("target_table", ""),
            "json_structure": json_structure,
            "instructions": at_settings.get("instructions", ""),
            "starting_page": at_settings.get("start_page", 1),
            "ending_page": at_settings.get("end_page") or None,
            "parallel_pages": at_settings.get("parallel_pages", 4),
            "execution_id": execution_id,
            "PLATFORM_SERVICE_API_KEY": platform_service_api_key,
            "group_key": at_settings.get("group_key", ""),
            "enable_header_mapping": at_settings.get("enable_header_mapping", False),
            # Set on the output at export by prompt_studio_registry_helper.py
            # when a lookup is assigned; None otherwise.
            "lookup_config": at_output.get("lookup_config"),
        }
        at_ctx = ExecutionContext(
            executor_name="agentic_table",
            operation="table_extract",
            run_id=file_execution_id,
            execution_source="tool",
            organization_id=organization_id,
            request_id=file_execution_id,
            log_events_id=log_events_id,
            execution_id=execution_id,
            file_execution_id=file_execution_id,
            executor_params=agentic_params,
        )
        at_result = dispatcher.dispatch(
            at_ctx,
            timeout=EXECUTOR_TIMEOUT,
        )
        if not at_result.success:
            return at_result.to_dict()
        at_output_data = at_result.data.get("output", {}) or {}
        agentic_results[at_output[_SK.NAME]] = at_output_data.get("tables", [])
        # The executor runs its own X2Text, so its metrics carry an extraction
        # duration the legacy pipeline's timer never sees. Same result shape the
        # non-agentic table plugin uses (LegacyExecutor._run_table_extraction).
        at_metrics = (at_result.data.get("metadata") or {}).get("metrics") or {}
        if at_metrics:
            agentic_metrics[at_output[_SK.NAME]] = at_metrics

    # ---- Step 6b: Dispatch legacy structure_pipeline ----
    # Skipped entirely when every prompt is agentic_table — the legacy
    # pipeline has no work to do and the agentic_table executor does
    # its own X2Text inside the runner.
    if regular_outputs:
        logger.info(
            "Dispatching structure_pipeline: tool_id=%s "
            "skip_extract=%s summarize=%s single_pass=%s",
            tool_id,
            skip_extraction_and_indexing,
            is_summarization_enabled,
            is_single_pass_enabled,
        )

        pipeline_ctx = ExecutionContext(
            executor_name="legacy",
            operation="structure_pipeline",
            run_id=file_execution_id,
            execution_source="tool",
            organization_id=organization_id,
            request_id=file_execution_id,
            log_events_id=log_events_id,
            execution_id=execution_id,
            file_execution_id=file_execution_id,
            executor_params={
                "extract_params": extract_params,
                "index_template": index_template,
                "answer_params": answer_params,
                "pipeline_options": pipeline_options,
                "summarize_params": summarize_params,
            },
        )
        pipeline_start = time.monotonic()
        pipeline_result = dispatcher.dispatch(
            pipeline_ctx,
            timeout=EXECUTOR_TIMEOUT,
        )
        pipeline_elapsed = time.monotonic() - pipeline_start

        if not pipeline_result.success:
            return pipeline_result.to_dict()

        structured_output = pipeline_result.data
        if agentic_results:
            structured_output.setdefault("output", {}).update(agentic_results)
        _merge_agentic_metrics(structured_output, agentic_metrics)
    else:
        # All-agentic case: skip the legacy pipeline entirely. The metrics
        # dict is still populated, so an agentic-only deployment reports the
        # same _file.text_extraction the legacy path does.
        structured_output = {
            "output": agentic_results,
            "metadata": {"agentic_only": True},
        }
        _merge_agentic_metrics(structured_output, agentic_metrics)
        pipeline_elapsed = 0.0

    # ---- Step 7: Write output files ----
    # (metadata/metrics merging already done by executor pipeline)
    write_error = _write_pipeline_outputs(
        fs=fs,
        structured_output=structured_output,
        output_dir_path=output_dir_path,
        input_file_path=input_file_path,
        execution_data_dir=execution_data_dir,
        source_file_name=source_file_name,
        label="structured",
    )
    if write_error:
        return ExecutionResult.failure(
            error=f"Error writing output file: {write_error}"
        ).to_dict()

    # Write tool result + tool_metadata to METADATA.json
    # (destination connector reads output_type from tool_metadata)
    _write_tool_result(fs, execution_data_dir, structured_output, pipeline_elapsed)

    return ExecutionResult(success=True, data=structured_output).to_dict()


# -----------------------------------------------------------------------
# Helper functions for the pipeline steps
# -----------------------------------------------------------------------


def _create_platform_helper(shim, request_id: str):
    """Create PlatformHelper using env vars for host/port."""
    from unstract.sdk1.platform import PlatformHelper

    return PlatformHelper(
        tool=shim,
        platform_host=os.environ.get(ToolEnv.PLATFORM_HOST, ""),
        platform_port=os.environ.get(ToolEnv.PLATFORM_PORT, ""),
        request_id=request_id,
    )


def _get_file_storage():
    """Get workflow execution file storage instance."""
    from unstract.filesystem import FileStorageType, FileSystem

    return FileSystem(FileStorageType.WORKFLOW_EXECUTION).get_file_storage()


def _fetch_tool_metadata(platform_helper, prompt_registry_id: str) -> tuple[dict, bool]:
    """Fetch tool metadata from platform, trying prompt studio then agentic.

    Returns:
        Tuple of (tool_metadata dict, is_agentic bool).

    Raises:
        RuntimeError: If neither registry returns valid metadata.
    """
    exported_tool = None
    try:
        exported_tool = platform_helper.get_prompt_studio_tool(
            prompt_registry_id=prompt_registry_id
        )
    except Exception as e:
        logger.info("Not found as prompt studio project, trying agentic: %s", e)

    if exported_tool and _SK.TOOL_METADATA in exported_tool:
        tool_metadata = exported_tool[_SK.TOOL_METADATA]
        tool_metadata["is_agentic"] = False
        return tool_metadata, False

    # Try agentic registry
    try:
        agentic_tool = platform_helper.get_agentic_studio_tool(
            agentic_registry_id=prompt_registry_id
        )
        if not agentic_tool or _SK.TOOL_METADATA not in agentic_tool:
            raise RuntimeError(
                f"Registry returned empty response for {prompt_registry_id}"
            )
        tool_metadata = agentic_tool[_SK.TOOL_METADATA]
        tool_metadata["is_agentic"] = True
        logger.info(
            "Retrieved agentic project: %s",
            tool_metadata.get("name", prompt_registry_id),
        )
        return tool_metadata, True
    except Exception as agentic_error:
        raise RuntimeError(
            f"Error fetching project from both registries "
            f"for ID '{prompt_registry_id}': {agentic_error}"
        ) from agentic_error


def _handle_profile_overrides(
    exec_metadata: dict, platform_helper, tool_metadata: dict
) -> None:
    """Apply LLM profile overrides if configured."""
    llm_profile_id = exec_metadata.get(_SK.LLM_PROFILE_ID)
    if not llm_profile_id:
        return

    try:
        llm_profile = platform_helper.get_llm_profile(llm_profile_id)
        if llm_profile:
            profile_name = llm_profile.get("profile_name", llm_profile_id)
            logger.info(
                "Applying profile overrides from profile: %s",
                profile_name,
            )
            changes = _apply_profile_overrides(tool_metadata, llm_profile)
            if changes:
                logger.info(
                    "Profile overrides applied. Changes: %s",
                    "; ".join(changes),
                )
            else:
                logger.info("Profile overrides applied - no changes needed")
    except Exception as e:
        raise RuntimeError(f"Error applying profile overrides: {e}") from e


def _run_agentic_extraction(
    tool_metadata: dict,
    input_file_path: str,
    output_dir_path: str,
    tool_instance_metadata: dict,
    dispatcher: PgExecutionDispatcher,
    shim: Any,
    file_execution_id: str,
    execution_id: str,
    organization_id: str,
    source_file_name: str,
    fs: Any,
    execution_data_dir: str = "",
) -> dict:
    """Execute agentic extraction pipeline via dispatcher.

    Unpacks metadata, extracts document text via X2Text, then dispatches
    with flat executor_params matching what AgenticPromptStudioExecutor
    expects (adapter_instance_id, document_text, etc.).
    """
    from unstract.sdk1.x2txt import X2Text

    # 1. Unpack agentic project metadata (matches registry_helper export format)
    adapter_config = tool_metadata.get("adapter_config", {})
    prompt_text = tool_metadata.get("prompt_text", "")
    json_schema = tool_metadata.get("json_schema", {})
    enable_highlight = tool_instance_metadata.get(
        "enable_highlight",
        tool_metadata.get("enable_highlight", False),
    )

    # 2. Get adapter IDs: workflow UI overrides → exported defaults
    #    (mirrors tools/structure/src/main.py)
    extractor_llm = tool_instance_metadata.get(
        "extractor_llm_adapter_id", adapter_config.get("extractor_llm", "")
    )
    llmwhisperer = tool_instance_metadata.get(
        "llmwhisperer_adapter_id", adapter_config.get("llmwhisperer", "")
    )
    platform_service_api_key = shim.platform_api_key

    # 3. Extract text from document using X2Text/LLMWhisperer
    x2text = X2Text(tool=shim, adapter_instance_id=llmwhisperer)
    extraction_result = x2text.process(
        input_file_path=input_file_path,
        enable_highlight=enable_highlight,
        fs=fs,
    )
    document_text = extraction_result.extracted_text

    # Parse json_schema if stored as string
    if isinstance(json_schema, str):
        json_schema = json.loads(json_schema)

    # 4. Dispatch with flat executor_params matching executor expectations
    start_time = time.monotonic()
    agentic_ctx = ExecutionContext(
        executor_name="agentic",
        operation="agentic_extract",
        run_id=file_execution_id,
        execution_source="tool",
        organization_id=organization_id,
        request_id=file_execution_id,
        log_events_id=StateStore.get("LOG_EVENTS_ID") or "",
        execution_id=execution_id,
        file_execution_id=file_execution_id,
        executor_params={
            "document_id": file_execution_id,
            "document_text": document_text,
            "prompt_text": prompt_text,
            "schema": json_schema,
            "adapter_instance_id": extractor_llm,
            "PLATFORM_SERVICE_API_KEY": platform_service_api_key,
            "include_source_refs": enable_highlight,
        },
    )
    agentic_result = dispatcher.dispatch(
        agentic_ctx,
        timeout=EXECUTOR_TIMEOUT,
    )

    if not agentic_result.success:
        return agentic_result.to_dict()

    structured_output = agentic_result.data
    elapsed = time.monotonic() - start_time

    # Write output files (matches regular pipeline path)
    write_error = _write_pipeline_outputs(
        fs=fs,
        structured_output=structured_output,
        output_dir_path=output_dir_path,
        input_file_path=input_file_path,
        execution_data_dir=execution_data_dir,
        source_file_name=source_file_name,
        label="agentic",
    )
    if write_error:
        return ExecutionResult.failure(
            error=f"Error writing agentic output: {write_error}"
        ).to_dict()

    # Write tool result + tool_metadata to METADATA.json
    _write_tool_result(fs, execution_data_dir, structured_output, elapsed)

    return ExecutionResult(success=True, data=structured_output).to_dict()


def _write_pipeline_outputs(
    fs: Any,
    structured_output: dict,
    output_dir_path: str,
    input_file_path: str,
    execution_data_dir: str,
    source_file_name: str,
    label: str,
) -> str | None:
    """Write structure-tool / agentic outputs to disk.

    Mirrors the old Docker tool's output layout so the destination
    connector finds what it expects:

    1. ``{output_dir_path}/{stem}.json`` — primary output file.
    2. INFILE overwritten with JSON (destination connector reads INFILE
       and checks MIME type — without this it still sees the original
       PDF).
    3. ``{execution_data_dir}/COPY_TO_FOLDER/{stem}.json`` — what the
       old ``ToolExecutor._setup_for_run()`` created for FS destinations.

    Args:
        label: Short label for log lines (``"structured"`` or
            ``"agentic"``).

    Returns:
        ``None`` on success, or the error string on failure.
    """
    try:
        stem = Path(source_file_name).stem
        output_path = Path(output_dir_path) / f"{stem}.json"
        logger.info("Writing %s output to %s", label, output_path)
        fs.json_dump(path=output_path, data=structured_output)

        logger.info("Overwriting INFILE with %s output: %s", label, input_file_path)
        fs.json_dump(path=input_file_path, data=structured_output)

        copy_to_folder = str(Path(execution_data_dir) / "COPY_TO_FOLDER")
        fs.mkdir(copy_to_folder)
        copy_output_path = str(Path(copy_to_folder) / f"{stem}.json")
        fs.json_dump(path=copy_output_path, data=structured_output)
        logger.info(
            "%s output written to COPY_TO_FOLDER: %s",
            label.capitalize(),
            copy_output_path,
        )

        logger.info("Output written successfully to workflow storage")
        return None
    except Exception as e:
        logger.error("Failed to write %s output files: %s", label, e, exc_info=True)
        return str(e)


def _write_tool_result(
    fs: Any, execution_data_dir: str, _data: dict, elapsed_time: float = 0.0
) -> None:
    """Write tool result and tool_metadata to METADATA.json.

    Matches BaseTool._update_exec_metadata():
    - tool_metadata: list of dicts with tool_name, output_type, elapsed_time
      (destination connector reads output_type from here)
    - total_elapsed_time: cumulative elapsed time
    """
    metadata_path: Path | None = None
    try:
        metadata_path = Path(execution_data_dir) / "METADATA.json"

        # Read existing metadata if present
        existing: dict = {}
        if fs.exists(metadata_path):
            try:
                existing_raw = fs.read(path=metadata_path, mode="r")
                if existing_raw:
                    existing = json.loads(existing_raw)
            except Exception:
                pass

        # Add tool_metadata (matches BaseTool._update_exec_metadata)
        # The destination connector reads output_type from tool_metadata[-1]
        tool_meta_entry = {
            "tool_name": "structure_tool",
            "output_type": "JSON",
            "elapsed_time": elapsed_time,
        }
        if "tool_metadata" not in existing:
            existing["tool_metadata"] = [tool_meta_entry]
        else:
            existing["tool_metadata"].append(tool_meta_entry)

        existing["total_elapsed_time"] = (
            existing.get("total_elapsed_time", 0.0) + elapsed_time
        )

        fs.write(
            path=metadata_path,
            mode="w",
            data=json.dumps(existing, indent=2),
        )
    except Exception as e:
        logger.error(
            "Failed to write tool result to METADATA.json at '%s': %s",
            metadata_path,
            e,
            exc_info=True,
        )