LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
"""Executor Worker Tasks
Defines the execute_extraction Celery task that receives an
ExecutionContext dict, runs the appropriate executor via
ExecutionOrchestrator, and returns an ExecutionResult dict.
"""
import logging
from executor.executors import register_all
from queue_backend import worker_task
from shared.clients import UsageAPIClient
from shared.enums.task_enums import TaskName
from shared.infrastructure.config import WorkerConfig
from shared.infrastructure.logging import WorkerLogger
from unstract.sdk1.execution.context import ExecutionContext
from unstract.sdk1.execution.orchestrator import ExecutionOrchestrator
from unstract.sdk1.execution.result import ExecutionResult
# Suppress Celery trace logging of task return values.
# The trace logger prints the full result dict on task success, which can
# contain sensitive customer data (extracted text, summaries, etc.).
#
# This lives here, not in ``executor/worker.py``, because that module is not on
# any deployed path: ``workers/worker.py`` exec-loads THIS file by path for both
# the Celery and PG executor roles, and no launcher runs ``celery -A executor``.
# It used to be reached only because ``executor/__init__.py`` eagerly imported
# ``.worker``; making that lazy (UN-4136) would otherwise have silently
# re-enabled result logging on every extraction.
logging.getLogger("celery.app.trace").setLevel(logging.WARNING)
logger = WorkerLogger.get_logger(__name__)
# Populate ``ExecutorRegistry`` so the executor implementations are registered
# before ``execute_extraction`` can be invoked. Coupling this to the task module
# (not only the Celery ``executor/worker.py`` entrypoint) ensures the registry is
# populated wherever the task is registered — in particular the PG executor
# consumer, which bootstraps via the root-worker import that loads
# ``executor/tasks.py`` (this module) but NOT ``executor/worker.py``; without it
# the consumer hits "No executor registered". This is the only call site;
# ``executor/worker.py`` reaches it by importing this module.
register_all()
#: Operations that are expected to emit usage records. An op in this set that
#: finishes successfully with NO records logs below -- it is the only signal
#: that billing silently produced nothing.
#:
#: Keeping a paid op out of this set is not a missing log line, it is a missing
#: alarm: the cloud `flush()` returns `[]` on a skipped LLM without raising, so
#: total loss of a run's billing rows is otherwise indistinguishable from a run
#: that legitimately made no LLM calls. Add every new LLM-bearing operation here
#: at the same time as the operation itself.
_LLM_BEARING_OPS = frozenset(
{
"answer_prompt",
"single_pass_extraction",
"summarize",
"structure_pipeline",
# The Agent-KV API's table extractor (cloud `agentic_table`'s blind-API
# operation). Every run drives two LLMs; a run of this op with no usage
# records means the billing chain broke, never that there was nothing
# to bill.
"table_extract_api",
}
)
@worker_task(
bind=True,
name=TaskName.EXECUTE_EXTRACTION,
autoretry_for=(ConnectionError, TimeoutError, OSError),
retry_backoff=True,
retry_backoff_max=60,
max_retries=3,
retry_jitter=True,
)
def execute_extraction(self, execution_context_dict: dict) -> dict:
"""Execute an extraction operation via the executor framework.
This is the single Celery task entry point for all extraction
operations. Both the workflow path (structure tool task) and
the IDE path (PromptStudioHelper) dispatch to this task.
Args:
execution_context_dict: Serialized ExecutionContext.
Returns:
Serialized ExecutionResult dict.
"""
request_id = execution_context_dict.get("request_id", "")
logger.info(
"Received execute_extraction task: "
"celery_task_id=%s request_id=%s executor=%s "
"operation=%s execution_source=%s run_id=%s",
self.request.id,
request_id,
execution_context_dict.get("executor_name"),
execution_context_dict.get("operation"),
execution_context_dict.get("execution_source"),
execution_context_dict.get("run_id"),
)
try:
context = ExecutionContext.from_dict(execution_context_dict)
except (KeyError, ValueError) as exc:
logger.error("Invalid execution context: %s", exc, exc_info=True)
return ExecutionResult.failure(
error=f"Invalid execution context: {exc}"
).to_dict()
# Build component dict for log correlation when streaming to
# the frontend. Attached as a transient attribute (not serialized).
if context.log_events_id:
params = context.executor_params
# For compound operations, extract nested params for log
# correlation.
if context.operation == "ide_index":
index_params = params.get("index_params", {})
extract_params = params.get("extract_params", {})
usage_kwargs = extract_params.get("usage_kwargs", {})
context._log_component = {
"tool_id": index_params.get("tool_id", ""),
"run_id": context.run_id,
"doc_name": str(usage_kwargs.get("file_name", "")),
"operation": context.operation,
}
elif context.operation == "structure_pipeline":
answer_params = params.get("answer_params", {})
pipeline_opts = params.get("pipeline_options", {})
context._log_component = {
"tool_id": answer_params.get("tool_id", ""),
"run_id": context.run_id,
"doc_name": str(pipeline_opts.get("source_file_name", "")),
"operation": context.operation,
}
else:
context._log_component = {
"tool_id": params.get("tool_id", ""),
"run_id": context.run_id,
"doc_name": str(params.get("file_name", "")),
"operation": context.operation,
}
else:
context._log_component = {}
orchestrator = ExecutionOrchestrator()
result = orchestrator.execute(context)
usage_records = result.metadata.get("usage_records", [])
if usage_records:
try:
config = WorkerConfig()
with UsageAPIClient(config) as usage_client:
# Org context is set on the client; no need to pass it per call.
usage_client.set_organization_context(context.organization_id)
ok = usage_client.bulk_create_usage(usage_records)
if not ok:
# ERROR severity so dropped billing rows are recoverable from logs.
logger.error(
"bulk_create_usage returned failure for %d records "
"(run_id=%s organization_id=%s)",
len(usage_records),
context.run_id,
context.organization_id,
)
except Exception:
logger.error(
"Failed to flush %d usage records for run_id=%s organization_id=%s",
len(usage_records),
context.run_id,
context.organization_id,
exc_info=True,
)
elif result.success and context.operation in _LLM_BEARING_OPS:
logger.info(
"No usage_records emitted for op=%s run_id=%s organization_id=%s "
"(unexpected for an LLM-bearing operation)",
context.operation,
context.run_id,
context.organization_id,
)
logger.info(
"execute_extraction complete: celery_task_id=%s request_id=%s success=%s",
self.request.id,
context.request_id,
result.success,
)
return result.to_dict()