LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
"""Bridge for the cloud-only VLM image-answer plugin.
Image output mode (an LLMWhisperer x2text adapter with
``output_mode == "image"``) persists per-page PNGs instead of text, so
answering a prompt against such a document means sending those images to a
vision-capable LLM. That consumer ships only with Unstract Cloud, as the
``vlm-image-answer`` executor plugin.
This OSS bridge owns detection and dispatch (mirroring the
``lookup_enrichment`` bridge, with the opposite error policy — lookups
degrade gracefully, image mode must fail loudly):
- Detects image mode from the profile's x2text adapter configuration. The
executor payload carries only the adapter *instance id* (no metadata),
so the config is resolved through the platform service once per
(execution, adapter) and cached.
- When the plugin is installed, delegates the answer to it. RAG retrieval
is skipped by the caller — image mode has no text to retrieve.
- When the plugin is absent, raises a structured error rather than letting
the prompt silently answer against the one-line extraction summary.
"""
import logging
from collections import OrderedDict
from typing import Any
from executor.executors.constants import PromptServiceConstants as PSKeys
from executor.executors.exceptions import VlmImageAnswerError
from executor.executors.file_utils import FileUtils
from executor.executors.plugins.loader import ExecutorPluginLoader
from unstract.sdk1.adapters.x2text.constants import (
ImageOutputConstants,
build_page_store_dir,
)
from unstract.sdk1.adapters.x2text.page_image_loader import (
PageCapExceededError,
PageImageLoadError,
PageImageSetIncompleteError,
PageImageSetTooLargeError,
PageImagesNotFoundError,
)
from unstract.sdk1.platform import PlatformHelper
logger = logging.getLogger(__name__)
PLUGIN_NAME = "vlm-image-answer"
# Stable machine-readable error codes (prefixed onto error messages so they
# survive the string-only propagation to PS / deployment API responses).
IMAGE_OUTPUT_REQUIRES_CLOUD = "IMAGE_OUTPUT_REQUIRES_CLOUD"
IMAGE_OUTPUT_MISSING = "IMAGE_OUTPUT_MISSING"
IMAGE_PAGE_CAP_EXCEEDED = "IMAGE_PAGE_CAP_EXCEEDED"
IMAGE_PAGES_TOO_LARGE = "IMAGE_PAGES_TOO_LARGE"
# Distinct from IMAGE_PAGES_TOO_LARGE (our own budget, caught before the
# call): this one is the PROVIDER rejecting a request the local checks
# allowed, because the model's per-request limit is lower than the cap.
IMAGE_REQUEST_TOO_LARGE = "IMAGE_REQUEST_TOO_LARGE"
IMAGE_OUTPUT_UNSUPPORTED_OPERATION = "IMAGE_OUTPUT_UNSUPPORTED_OPERATION"
VISION_LLM_REQUIRED = "VISION_LLM_REQUIRED"
_LLMWHISPERER_ADAPTER_PREFIX = "llmwhisperer|"
# (scope_id, adapter_instance_id) -> resolved config dict | None, where
# scope_id is the execution id or (for IDE runs, which carry no execution
# id) the run id. Run-scoped on purpose: the cache exists to deduplicate
# the N per-prompt resolutions within ONE run — never to cache across
# runs, where it would pin a stale output mode after an adapter edit.
# Bounded so a long-lived worker never grows it unchecked.
_MODE_CACHE: OrderedDict[tuple[str, str], dict[str, Any] | None] = OrderedDict()
_MODE_CACHE_MAX = 256
def _resolve_image_mode_config(
shim: Any, adapter_instance_id: str, scope_id: str
) -> dict[str, Any] | None:
"""Return the x2text adapter config when it is in image mode, else None.
Resolution goes through the platform service (the payload has no
adapter metadata); results are cached per (scope, adapter). With no
scope id at all, caching is skipped entirely — a shared ("", adapter)
entry would serve a stale mode to every later run on this worker.
"""
cache_key = (scope_id, adapter_instance_id)
if scope_id and cache_key in _MODE_CACHE:
_MODE_CACHE.move_to_end(cache_key)
return _MODE_CACHE[cache_key]
config = PlatformHelper.get_adapter_config(shim, adapter_instance_id) or {}
adapter_id = str(config.get("adapter_id", ""))
adapter_metadata = config.get("adapter_metadata") or {}
is_image_mode = adapter_id.startswith(_LLMWHISPERER_ADAPTER_PREFIX) and (
adapter_metadata.get(ImageOutputConstants.OUTPUT_MODE)
== ImageOutputConstants.IMAGE_MODE
)
result = config if is_image_mode else None
if scope_id:
_MODE_CACHE[cache_key] = result
while len(_MODE_CACHE) > _MODE_CACHE_MAX:
_MODE_CACHE.popitem(last=False)
return result
_IDE_SOURCE = "ide"
def detect_image_mode_config(
*,
output: dict[str, Any],
shim: Any,
execution_source: str = "",
usage_kwargs: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
"""Detect image mode for one prompt; None means normal text path.
Fast path: the backend stamps the adapter's ``x2text_output_mode``
onto the per-prompt payload (it already holds the decrypted adapter
metadata), so stamped prompts cost nothing here.
Unstamped payloads split by source: IDE payloads are always stamped
by the current backend, so an unstamped one is a pre-upgrade
in-flight run — treated as text mode rather than making every
legacy IDE prompt depend on a platform-service call. Deployment
payloads are built worker-side without adapter metadata, so they
resolve through the platform service (run-scoped cache).
Returns the resolved adapter config for the plugin, or ``{}`` when
image mode was determined from the stamp alone.
"""
adapter_instance_id = str(output.get(PSKeys.X2TEXT_ADAPTER) or "")
if not adapter_instance_id:
return None
if PSKeys.X2TEXT_OUTPUT_MODE in output:
stamped_mode = output.get(PSKeys.X2TEXT_OUTPUT_MODE)
if stamped_mode == ImageOutputConstants.IMAGE_MODE:
return {}
return None
if execution_source == _IDE_SOURCE:
return None
usage_kwargs = usage_kwargs or {}
# IDE payloads carry no execution_id — fall back to the run id so the
# cache stays scoped to one run (see _MODE_CACHE).
scope_id = str(usage_kwargs.get("execution_id") or usage_kwargs.get("run_id") or "")
return _resolve_image_mode_config(shim, adapter_instance_id, scope_id)
def run_vlm_image_answer(
*,
output: dict[str, Any],
shim: Any,
llm: Any,
extract_file_path: str,
execution_source: str,
metadata: dict[str, Any],
metrics: dict[str, Any],
x2text_config: dict[str, Any],
usage_kwargs: dict[str, Any] | None = None,
) -> str:
"""Answer an image-mode prompt via the cloud plugin.
The caller has already detected image mode via
``detect_image_mode_config``. ``extract_file_path`` must be the
extract-file path (never the summarize/source rewrite of it) — the
pages directory is derived from it via the shared writer/reader
helper.
Returns:
The raw answer string (the caller assigns it in place of the
RAG/completion answer, so type conversion, lookups, webhooks
etc. run unchanged).
Raises:
VlmImageAnswerError: the cloud plugin is not installed, or the
vision path failed in a way the user must act on (missing
images, page cap, non-vision LLM).
"""
usage_kwargs = usage_kwargs or {}
prompt_name = output.get(PSKeys.NAME, "")
plugin_cls = ExecutorPluginLoader.get(PLUGIN_NAME)
if plugin_cls is None:
raise VlmImageAnswerError(
"This document was extracted in image output mode, which is "
"answered by a vision LLM available only on Unstract Cloud. "
"Switch the profile's text extractor to a text output mode, "
"or run this on Unstract Cloud.",
error_code=IMAGE_OUTPUT_REQUIRES_CLOUD,
)
shim.stream_log(f"Answering `{prompt_name}` from page images via vision LLM")
fs = FileUtils.get_fs_instance(execution_source=execution_source)
page_store_dir = build_page_store_dir(extract_file_path, extract_file_path)
try:
outcome = plugin_cls.run_with_metrics(
output=output,
llm=llm,
fs=fs,
page_store_dir=page_store_dir,
x2text_config=x2text_config,
metadata=metadata,
shim=shim,
usage_kwargs=usage_kwargs,
)
except (PageImagesNotFoundError, PageImageSetIncompleteError) as e:
raise VlmImageAnswerError(str(e), error_code=IMAGE_OUTPUT_MISSING) from e
except PageCapExceededError as e:
raise VlmImageAnswerError(str(e), error_code=IMAGE_PAGE_CAP_EXCEEDED) from e
except PageImageSetTooLargeError as e:
raise VlmImageAnswerError(str(e), error_code=IMAGE_PAGES_TOO_LARGE) from e
except PageImageLoadError as e:
raise VlmImageAnswerError(str(e), error_code=IMAGE_OUTPUT_MISSING) from e
except VlmImageAnswerError:
raise
except Exception as e:
# Plugin-defined hard failures carry a stable error_code attribute
# (e.g. VISION_LLM_REQUIRED). Anything else propagates untouched —
# never degrade to the text path.
plugin_code = getattr(e, "error_code", None)
if isinstance(plugin_code, str) and plugin_code:
raise VlmImageAnswerError(str(e), error_code=plugin_code) from e
raise
llm_metrics = outcome.get("llm_metrics") if isinstance(outcome, dict) else None
if llm_metrics:
metrics.setdefault(prompt_name, {})["vlm_image_answer"] = llm_metrics
answer = outcome["answer"] if isinstance(outcome, dict) else str(outcome)
shim.stream_log(f"Vision LLM answered `{prompt_name}`")
return answer
def raise_if_image_mode_unsupported(
*,
operation: str,
adapter_instance_id: str | None,
shim: Any,
scope_id: str = "",
stamped_mode: str | None = None,
execution_source: str = "",
) -> None:
"""Guard operations that cannot run against image-mode documents.
Single-pass extraction (and any future full-text operation) would
silently run against the one-line extraction summary — reject it
explicitly instead. Same detection semantics as
``detect_image_mode_config``: stamp first, platform resolution only
for non-IDE sources.
"""
if not adapter_instance_id:
return
if stamped_mode is not None:
config = {} if stamped_mode == ImageOutputConstants.IMAGE_MODE else None
elif execution_source == _IDE_SOURCE:
return
else:
config = _resolve_image_mode_config(shim, str(adapter_instance_id), scope_id)
if config is not None:
raise VlmImageAnswerError(
f"{operation} is not supported in image output mode. Run "
"prompts individually, or switch the profile's text extractor "
"to a text output mode.",
error_code=IMAGE_OUTPUT_UNSUPPORTED_OPERATION,
)