LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
"""ExecutorToolShim — Lightweight BaseTool substitute for executor workers.
Adapters (PlatformHelper, LLM, Embedding, VectorDB, X2Text) all require
a ``tool: BaseTool`` parameter that provides ``get_env_or_die()`` and
``stream_log()``. The executor worker has no ``BaseTool`` instance, so
this shim provides just those two methods.
Precedent: ``prompt-service/.../helpers/prompt_ide_base_tool.py``
(``PromptServiceBaseTool``).
"""
import logging
import os
from typing import Any
from unstract.core.pubsub_helper import LogPublisher
from unstract.sdk1.constants import LogLevel, ToolEnv
from unstract.sdk1.exceptions import SdkError
from unstract.sdk1.tool.stream import StreamMixin
logger = logging.getLogger(__name__)
# Map SDK log levels to the string levels used by LogPublisher.
_SDK_TO_WF_LEVEL: dict[LogLevel, str] = {
LogLevel.DEBUG: "INFO", # DEBUG not surfaced to frontend
LogLevel.INFO: "INFO",
LogLevel.WARN: "WARN",
LogLevel.ERROR: "ERROR",
LogLevel.FATAL: "ERROR",
}
# Mapping from SDK LogLevel enum to Python logging levels.
_LEVEL_MAP = {
LogLevel.DEBUG: logging.DEBUG,
LogLevel.INFO: logging.INFO,
LogLevel.WARN: logging.WARNING,
LogLevel.ERROR: logging.ERROR,
LogLevel.FATAL: logging.CRITICAL,
}
class ExecutorToolShim(StreamMixin):
"""Minimal BaseTool substitute for use inside executor workers.
Provides the two methods that adapters actually call:
- ``get_env_or_die(env_key)`` — reads env vars, with special
handling for ``PLATFORM_SERVICE_API_KEY`` (multitenancy)
- ``stream_log(log, level)`` — routes to Python logging instead
of the Unstract stdout JSON protocol used by tools
Usage::
shim = ExecutorToolShim(platform_api_key="sk-...")
adapter = SomeAdapter(tool=shim) # adapter calls shim.get_env_or_die()
"""
def __init__(
self,
platform_api_key: str = "",
log_events_id: str = "",
component: dict[str, str] | None = None,
execution_id: str | None = None,
organization_id: str | None = None,
file_execution_id: str | None = None,
) -> None:
"""Initialize the shim.
Args:
platform_api_key: The platform service API key for this
execution. Returned by ``get_env_or_die()`` when the
caller asks for ``PLATFORM_SERVICE_API_KEY``.
log_events_id: Socket.IO channel ID for streaming progress
logs. Empty string disables publishing.
component: Structured identifier dict for log correlation
(``tool_id``, ``run_id``, ``doc_name``, optionally
``prompt_key``).
execution_id: Workflow execution id. When provided with
``organization_id``, enables persistent log attribution
to the ``execution_log`` table.
organization_id: Tenant/org scope for log persistence.
file_execution_id: File execution id (child of workflow
execution). Optional — populated for per-file tool runs.
"""
self.platform_api_key = platform_api_key
self.log_events_id = log_events_id
self.component = component or {}
self.execution_id = execution_id
self.organization_id = organization_id
self.file_execution_id = file_execution_id
# Track whether we've already surfaced a publish failure on this
# shim so that a down broker is flagged loudly once instead of
# silently swallowing every subsequent log line at DEBUG.
self._progress_publish_failed = False
self._log_publish_failed = False
# Adapters whose name+model has already been surfaced to the UI;
# later mentions skip repeating the model line.
self._adapters_logged: set[str] = set()
# Initialize StreamMixin. EXECUTION_BY_TOOL is not set in
# the worker environment, so _exec_by_tool will be False.
super().__init__(log_level=LogLevel.INFO)
def get_env_or_die(self, env_key: str) -> str:
"""Return environment variable value.
Special-cases ``PLATFORM_SERVICE_API_KEY`` to return the key
passed at construction time (supports multitenancy — each
execution may use a different org's API key).
Args:
env_key: Environment variable name.
Returns:
The value of the environment variable.
Raises:
SdkError: If the variable is missing or empty.
"""
if env_key == ToolEnv.PLATFORM_API_KEY:
if not self.platform_api_key:
raise SdkError(f"Env variable '{env_key}' is required")
return self.platform_api_key
env_value = os.environ.get(env_key)
if env_value is None or env_value == "":
raise SdkError(f"Env variable '{env_key}' is required")
return env_value
def stream_log(
self,
log: str,
level: LogLevel = LogLevel.INFO,
stage: str = "TOOL_RUN",
**kwargs: dict[str, Any],
) -> None:
"""Route log messages to Python logging and publish progress.
In the executor worker context, logs go through the standard
Python logging framework (captured by Celery) rather than the
Unstract stdout JSON protocol used by tools.
Progress messages are published via ``LogPublisher.publish()``
to the Redis broker (shared with worker-logging).
Args:
log: The log message.
level: SDK log level.
stage: Ignored (only meaningful for stdout protocol).
**kwargs: Ignored (only meaningful for stdout protocol).
"""
py_level = _LEVEL_MAP.get(level, logging.INFO)
logger.log(py_level, log)
# Respect log level threshold for frontend publishing (matches
# StreamMixin.stream_log behaviour). Python logging above still
# captures everything for debugging.
_levels = [
LogLevel.DEBUG,
LogLevel.INFO,
LogLevel.WARN,
LogLevel.ERROR,
LogLevel.FATAL,
]
if _levels.index(level) < _levels.index(self.log_level):
return
wf_level = _SDK_TO_WF_LEVEL.get(level, "INFO")
# PROGRESS payload routes via the websocket room; skip when absent.
if self.log_events_id:
try:
progress_payload = LogPublisher.log_progress(
component=self.component,
level=wf_level,
state=stage,
message=log,
)
LogPublisher.publish(
channel_id=self.log_events_id,
payload=progress_payload,
)
except Exception:
first_failure = not self._progress_publish_failed
self._progress_publish_failed = True
logger.log(
logging.WARNING if first_failure else logging.DEBUG,
"Failed to publish progress log (non-fatal)",
exc_info=first_failure,
)
# LOG payload persists to execution_log; falls back to execution_id
# as the routing channel so it survives without a websocket subscriber.
if not (self.execution_id and self.organization_id):
return
log_channel = self.log_events_id or self.execution_id
try:
log_payload = LogPublisher.log_workflow(
stage=stage,
message=log,
level=wf_level,
execution_id=self.execution_id,
file_execution_id=self.file_execution_id or None,
organization_id=self.organization_id,
)
LogPublisher.publish(
channel_id=log_channel,
payload=log_payload,
)
except Exception:
first_failure = not self._log_publish_failed
self._log_publish_failed = True
logger.log(
logging.WARNING if first_failure else logging.DEBUG,
"Failed to publish workflow log (non-fatal)",
exc_info=first_failure,
)
def stream_error_and_exit(self, message: str, err: Exception | None = None) -> None:
"""Log error and raise SdkError.
Unlike the base StreamMixin which may call ``sys.exit(1)``
when running as a tool, the executor worker always raises
an exception so the Celery task can handle it gracefully.
Args:
message: Error description.
err: Original exception, if any.
Raises:
SdkError: Always.
"""
logger.error(message)
raise SdkError(message, actual_err=err)
def log_adapter_once(
self,
kind: str,
adapter_id: str,
adapter: Any,
) -> None:
"""Surface adapter identity to the UI on first use only.
``kind`` is the human label ("LLM", "Embedding", "Vector DB").
``adapter`` is an SDK instance — read only for non-sensitive
identity (model name or adapter display name); ``adapter_id``
is the dedup key. Subsequent calls for the same id are no-ops.
"""
if not adapter_id or adapter_id in self._adapters_logged:
return
self._adapters_logged.add(adapter_id)
get_model = getattr(adapter, "get_model_name", None)
if callable(get_model):
label = get_model() or adapter_id
else:
label = getattr(adapter, "_adapter_name", "") or adapter_id
self.stream_log(f"Using {kind}: `{label}`")