LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
"""Behavioural tests for the worker dispatch sites.
Two independent contracts are pinned here so a future change that quietly
drops a field, rewires a queue, or changes an endpoint fails loudly:
1. ``shared/patterns/notification/helper.py`` — status-callback notifications.
UN-3056 replaced the old ``send_notification_to_worker`` ->
``queue_backend.dispatch`` path with a model-free HTTP enqueue:
``_route_notification`` -> ``_enqueue_to_buffer`` POSTs each event to the
backend buffer endpoint, which owns batching + send. These tests
characterise that HTTP enqueue contract.
2. ``scheduler/tasks.py:_execute_scheduled_workflow`` — still routed through
the ``queue_backend.dispatch()`` seam (unchanged).
"""
from unittest.mock import MagicMock, patch
import pytest
from queue_backend import QueueBackend
from shared.patterns.notification.helper import (
ENQUEUE_BUFFER_ENDPOINT,
_enqueue_to_buffer,
_route_notification,
)
from unstract.core.data_models import (
ExecutionStatus,
NotificationPayload,
NotificationSource,
WorkflowType,
)
# --- Site 1: shared/patterns/notification/helper.py (HTTP buffer enqueue) ---
class TestNotificationDispatchSite:
"""Pin the worker notification path -> backend buffer-enqueue HTTP contract.
UN-3056 made the callback worker model-free: instead of dispatching a
Celery task, ``_route_notification`` -> ``_enqueue_to_buffer`` POSTs each
event to the backend, which owns the NotificationBuffer + clubbed send.
"""
def _make_notification(self, notification_type="WEBHOOK", platform="API"):
return {
"id": "notif-001",
"notification_type": notification_type,
"platform": platform,
}
def _make_payload(self):
"""A real NotificationPayload so enum -> value coercion is exercised."""
return NotificationPayload.from_execution_status(
pipeline_id="pipe-001",
pipeline_name="char-test-pipeline",
execution_status=ExecutionStatus.ERROR,
workflow_type=WorkflowType.API,
source=NotificationSource.CALLBACK_WORKER,
execution_id="exec-123",
error_message="boom",
total_files=3,
successful_files=1,
failed_files=2,
)
def test_enqueue_posts_to_buffer_endpoint(self):
"""``_enqueue_to_buffer`` MUST POST to the buffer endpoint, timeout=10."""
api_client = MagicMock()
_enqueue_to_buffer(api_client, self._make_notification(), self._make_payload())
api_client._make_request.assert_called_once()
call = api_client._make_request.call_args
assert call.kwargs["method"] == "POST"
assert call.kwargs["endpoint"] == ENQUEUE_BUFFER_ENDPOINT
assert call.kwargs["timeout"] == 10
def test_enqueue_payload_shape(self):
"""The enqueued ``data`` MUST carry the full per-event contract the
backend buffer + clubbed renderer rely on."""
api_client = MagicMock()
_enqueue_to_buffer(api_client, self._make_notification(), self._make_payload())
data = api_client._make_request.call_args.kwargs["data"]
assert set(data) == {
"notification_id",
"type",
"execution_id",
"pipeline_id",
"pipeline_name",
"status",
"error_message",
"platform",
"timestamp",
"additional_data",
}
assert data["notification_id"] == "notif-001"
assert data["pipeline_id"] == "pipe-001"
assert data["pipeline_name"] == "char-test-pipeline"
assert data["execution_id"] == "exec-123"
assert data["platform"] == "API"
# Enums are coerced to their string values, not passed as objects.
assert data["type"] == WorkflowType.API.value
assert isinstance(data["status"], str)
assert isinstance(data["additional_data"], dict)
def test_enqueue_raises_on_request_failure(self):
"""A transport failure MUST propagate out of ``_enqueue_to_buffer`` so
the router can count the drop (it is the only failure signal)."""
api_client = MagicMock()
api_client._make_request.side_effect = RuntimeError("backend down")
with pytest.raises(RuntimeError):
_enqueue_to_buffer(
api_client, self._make_notification(), self._make_payload()
)
def test_route_skips_non_webhook(self):
"""Non-WEBHOOK notifications MUST NOT be enqueued."""
api_client = MagicMock()
_route_notification(
api_client,
self._make_notification(notification_type="EMAIL"),
self._make_payload(),
)
api_client._make_request.assert_not_called()
def test_route_skips_missing_notification_type(self):
"""A malformed notification with no notification_type key is skipped —
``.get()`` returns None, which fails the WEBHOOK check (no KeyError)."""
api_client = MagicMock()
notification = {"id": "notif-001", "platform": "API"} # no notification_type
_route_notification(api_client, notification, self._make_payload())
api_client._make_request.assert_not_called()
def test_route_enqueues_webhook(self):
"""WEBHOOK notifications MUST reach the buffer endpoint."""
api_client = MagicMock()
_route_notification(api_client, self._make_notification(), self._make_payload())
api_client._make_request.assert_called_once()
assert (
api_client._make_request.call_args.kwargs["endpoint"]
== ENQUEUE_BUFFER_ENDPOINT
)
def test_route_swallows_enqueue_failure(self):
"""An enqueue failure MUST NOT abort the caller's loop — sibling
notifications still get their turn. ``_route_notification`` is the
path's final swallow point."""
api_client = MagicMock()
api_client._make_request.side_effect = RuntimeError("backend down")
# Must not raise.
_route_notification(api_client, self._make_notification(), self._make_payload())
api_client._make_request.assert_called_once()
# --- Site 2: scheduler/tasks.py ---
class TestSchedulerDispatchSite:
"""Pin ``_execute_scheduled_workflow`` -> ``queue_backend.dispatch``."""
def _make_context(self):
"""Minimal ScheduledPipelineContext-shaped mock.
The dispatch site reads:
- context.organization_id, .workflow_id, .pipeline_id, .pipeline_name
- context.use_file_history
"""
ctx = MagicMock()
ctx.organization_id = "org-test"
ctx.workflow_id = "wf-001"
ctx.pipeline_id = "pipe-007"
ctx.pipeline_name = "scheduled-test-pipeline"
ctx.use_file_history = False
return ctx
def _make_api_client(self, execution_id="exec-123"):
"""Mock api_client whose create_workflow_execution returns a dict
with the expected execution_id."""
api = MagicMock()
api.create_workflow_execution.return_value = {"execution_id": execution_id}
return api
def test_dispatch_task_name(self):
from scheduler.tasks import _execute_scheduled_workflow
with patch("scheduler.tasks.dispatch") as mock_dispatch:
_execute_scheduled_workflow(self._make_api_client(), self._make_context())
mock_dispatch.assert_called_once()
assert mock_dispatch.call_args.args[0] == "async_execute_bin"
def test_dispatch_routes_to_general_queue(self):
from scheduler.tasks import _execute_scheduled_workflow
from shared.enums.worker_enums import QueueName
with patch("scheduler.tasks.dispatch") as mock_dispatch:
_execute_scheduled_workflow(self._make_api_client(), self._make_context())
assert mock_dispatch.call_args.kwargs["queue"] == QueueName.GENERAL
def test_dispatch_positional_args_layout(self):
"""Positional args MUST be [org_id, workflow_id, execution_id, {}, True]."""
from scheduler.tasks import _execute_scheduled_workflow
with patch("scheduler.tasks.dispatch") as mock_dispatch:
_execute_scheduled_workflow(
self._make_api_client(execution_id="exec-xyz"),
self._make_context(),
)
args = mock_dispatch.call_args.kwargs["args"]
assert len(args) == 5
assert args[0] == "org-test" # organization_id (schema_name)
assert args[1] == "wf-001" # workflow_id
assert args[2] == "exec-xyz" # execution_id (from api_client return)
assert args[3] == {} # hash_values_of_files (always empty for scheduled)
assert args[4] is True # scheduled flag (always True here)
def test_dispatch_kwargs_layout(self):
"""Kwargs MUST contain exactly use_file_history, pipeline_id and the
write-only ``transport`` shim.
Nothing reads ``transport`` after UN-4078, but a pre-UN-4078 general
worker falls back to Celery when it is absent, so it stays written for
one release. Asserting the whole dict is deliberate: it fails on a stray
key, and it fails if the shim is dropped too early.
"""
from scheduler.tasks import _execute_scheduled_workflow
ctx = self._make_context()
ctx.use_file_history = True
with patch("scheduler.tasks.dispatch") as mock_dispatch:
_execute_scheduled_workflow(self._make_api_client(), ctx)
kwargs = mock_dispatch.call_args.kwargs["kwargs"]
assert kwargs == {
"use_file_history": True,
"pipeline_id": "pipe-007",
"transport": "pg_queue",
}
def test_dispatch_ignores_a_transport_field_from_the_backend(self):
"""A ``transport`` value from the backend response is never forwarded.
The payload carries the constant ``pg_queue`` shim regardless of what the
create-execution response says, so a stale ``"celery"`` cannot reach an
un-upgraded consumer that still branches on it.
"""
from scheduler.tasks import _execute_scheduled_workflow
api = MagicMock()
api.create_workflow_execution.return_value = {
"execution_id": "exec-123",
"transport": "celery",
}
with patch("scheduler.tasks.dispatch") as mock_dispatch:
_execute_scheduled_workflow(api, self._make_context())
assert mock_dispatch.call_args.kwargs["kwargs"]["transport"] == "pg_queue"
assert mock_dispatch.call_args.kwargs["backend"] is QueueBackend.PG
def test_backend_is_always_pg(self):
"""The scheduled orchestrator always pins the dispatch to PG.
This used to be conditional on the payload's transport, with ``None``
(defer to select_backend) on the Celery branch. Stating it explicitly
keeps the whole execution pipeline pinned at its entry point.
"""
from scheduler.tasks import _execute_scheduled_workflow
with patch("scheduler.tasks.dispatch") as mock_dispatch:
_execute_scheduled_workflow(self._make_api_client(), self._make_context())
assert mock_dispatch.call_args.kwargs["backend"] is QueueBackend.PG
def test_no_dispatch_when_execution_creation_fails(self):
"""If api_client.create_workflow_execution returns no execution_id,
the function bails out and never calls send_task."""
from scheduler.tasks import _execute_scheduled_workflow
from shared.models.scheduler_models import SchedulerExecutionStatus
api = MagicMock()
api.create_workflow_execution.return_value = {} # no execution_id
with patch("scheduler.tasks.dispatch") as mock_dispatch:
result = _execute_scheduled_workflow(api, self._make_context())
# The dispatch contract: nothing is sent when execution creation fails.
mock_dispatch.assert_not_called()
# And the function returns an error result (not raised exception).
assert result.status == SchedulerExecutionStatus.ERROR
def test_dispatch_returns_error_result_when_dispatch_raises(self):
"""If ``dispatch`` raises (broker down, queue gone, etc.),
``_execute_scheduled_workflow`` MUST catch it, log, and return a
``SchedulerExecutionResult.error(...)`` — NOT propagate the
exception. This guard makes sure a future routing change can't
silently flip the failure mode.
"""
from scheduler.tasks import _execute_scheduled_workflow
from shared.models.scheduler_models import SchedulerExecutionStatus
with patch("scheduler.tasks.dispatch") as mock_dispatch:
mock_dispatch.side_effect = RuntimeError("broker down")
result = _execute_scheduled_workflow(
self._make_api_client(), self._make_context()
)
# Exception was caught (not propagated).
assert result.status == SchedulerExecutionStatus.ERROR
# Execution itself was created before the dispatch failed, so the
# error result is about the dispatch and must carry the
# execution_id created moments earlier.
assert result.execution_id == "exec-123"
# --- Cross-site invariant ---
class TestDispatchSiteInventory:
"""All task dispatch must flow through the queue_backend seam.
Raw ``current_app.send_task`` calls outside ``queue_backend/`` are now
forbidden — they bypass the substrate-routing chokepoint. If one
reappears, this test fails so the regression can't ship silently.
"""
def test_no_raw_dispatch_sites_outside_seam(self):
"""Zero raw Celery dispatch calls should exist in workers/ source
outside the queue_backend/ seam.
``queue_backend/dispatch.py`` is the canonical home: it's the
one file that calls ``current_app.send_task`` (today, while the
substrate is still Celery). Everything else must call
``queue_backend.dispatch(...)`` instead.
Uses an AST walker rather than a regex so the canary covers:
* aliased imports (``from celery import current_app as app``;
``app.send_task(...)``),
* locally-constructed apps (``Celery(...).send_task(...)``),
* ``.apply_async`` (the other half of how Celery sends tasks,
including ``signature(...).apply_async()``),
* multi-line dotted access after autoformatter wrap.
"""
import ast
import pathlib
workers_root = pathlib.Path(__file__).parent.parent
# Anchor skip to the top-level directory relative to workers_root so
# we don't accidentally exclude legitimately-named subdirectories
# (e.g. workers/shared/tests_helpers/).
skip_top_dirs = {"tests", "__pycache__", "htmlcov", ".venv", "queue_backend"}
# ``plugins/manual_review`` predates this seam and uses
# ``task.apply_async`` lambda overrides to inject queues — out of
# scope for the queue_backend transport seam PR. Tracked as a
# follow-up so this canary is exact for everything we own.
skip_subpaths = {"plugins/manual_review"}
forbidden_attrs = {"send_task", "apply_async"}
class DispatchCallFinder(ast.NodeVisitor):
def __init__(self) -> None:
self.hits: list[int] = []
def visit_Call(self, node: ast.Call) -> None:
func = node.func
if isinstance(func, ast.Attribute) and func.attr in forbidden_attrs:
self.hits.append(node.lineno)
self.generic_visit(node)
hits: list[str] = []
for py in workers_root.rglob("*.py"):
rel = py.relative_to(workers_root)
rel_parts = rel.parts
if rel_parts and rel_parts[0] in skip_top_dirs:
continue
rel_str = rel.as_posix()
if any(rel_str.startswith(skip + "/") for skip in skip_subpaths):
continue
try:
tree = ast.parse(py.read_text(), filename=str(py))
except SyntaxError:
continue
finder = DispatchCallFinder()
finder.visit(tree)
for line_no in finder.hits:
hits.append(f"{py.relative_to(workers_root)}:{line_no}")
assert hits == [], (
"Raw Celery dispatch call(s) (send_task / apply_async) found "
"outside queue_backend/. All task dispatch must go through "
"queue_backend.dispatch(). Found:\n " + "\n ".join(hits)
)
def test_canonical_seam_exists(self):
"""The queue_backend/dispatch.py seam must exist — it's where every
dispatch site routes through. Behavioural equivalence to the
legacy ``current_app.send_task`` calls is covered by the
``TestDispatchEquivalence`` suite in test_queue_backend_seam.py;
this test only guards the file's existence so a future refactor
that moves the module by mistake fails loudly.
"""
import pathlib
seam = pathlib.Path(__file__).parent.parent / "queue_backend" / "dispatch.py"
assert seam.exists(), f"Canonical dispatch seam missing: {seam}"
# --- Site 1b: early-failure dispatch (notify_execution_failure) ---
class TestEarlyFailureNotification:
"""Pin the early-failure notification path (UN-3056).
Runs that fail before file processing (missing-tool / tool-validation /
source-setup errors) never reach the callback's
``handle_status_notifications``, so the general worker calls
``notify_execution_failure`` on its failure branch. It must resolve the
pipeline name/type from the backend and dispatch a terminal ERROR status;
if pipeline data can't be fetched it must skip rather than raise.
"""
def _api_client(self, *, success=True, data="default"):
api_client = MagicMock()
resp = MagicMock()
resp.success = success
resp.data = (
{"pipeline": {"pipeline_name": "etl-early-fail", "pipeline_type": "ETL"}}
if data == "default"
else data
)
api_client.get_pipeline_data.return_value = resp
return api_client
def test_dispatches_terminal_error_with_resolved_identity(self):
from shared.patterns.notification import helper as notif_helper
api_client = self._api_client()
with patch.object(notif_helper, "handle_status_notifications") as mock_dispatch:
notif_helper.notify_execution_failure(
api_client=api_client,
pipeline_id="pipe-1",
execution_id="exec-1",
organization_id="org-1",
error_message="Tool does not exist in registry: text_extractor",
)
api_client.get_pipeline_data.assert_called_once_with(
pipeline_id="pipe-1", check_active=False
)
mock_dispatch.assert_called_once()
kwargs = mock_dispatch.call_args.kwargs
assert kwargs["pipeline_id"] == "pipe-1"
assert kwargs["status"] == ExecutionStatus.ERROR.value
assert kwargs["pipeline_name"] == "etl-early-fail"
assert kwargs["pipeline_type"] == "ETL"
assert kwargs["execution_id"] == "exec-1"
assert kwargs["organization_id"] == "org-1"
assert "text_extractor" in kwargs["error_message"]
def test_skips_when_pipeline_data_unavailable(self):
from shared.patterns.notification import helper as notif_helper
api_client = self._api_client(success=False, data=None)
with patch.object(notif_helper, "handle_status_notifications") as mock_dispatch:
notif_helper.notify_execution_failure(
api_client=api_client,
pipeline_id="pipe-1",
execution_id="exec-1",
organization_id="org-1",
)
mock_dispatch.assert_not_called()
def test_resolves_flat_pipeline_shape(self):
# Older backend builds return the record flat (no "pipeline" envelope).
from shared.patterns.notification import helper as notif_helper
api_client = self._api_client(
data={"pipeline_name": "flat-etl", "pipeline_type": "TASK"}
)
with patch.object(notif_helper, "handle_status_notifications") as mock_dispatch:
notif_helper.notify_execution_failure(
api_client=api_client,
pipeline_id="pipe-1",
execution_id="exec-1",
organization_id="org-1",
)
kwargs = mock_dispatch.call_args.kwargs
assert kwargs["pipeline_name"] == "flat-etl"
assert kwargs["pipeline_type"] == "TASK"
def test_does_not_raise_when_resolution_errors(self):
# A backend hiccup while resolving identity must not bubble out of the
# worker's failure handler (it is best-effort, post-status-update).
from shared.patterns.notification import helper as notif_helper
api_client = MagicMock()
api_client.get_pipeline_data.side_effect = RuntimeError("backend down")
with patch.object(notif_helper, "handle_status_notifications") as mock_dispatch:
notif_helper.notify_execution_failure(
api_client=api_client,
pipeline_id="pipe-1",
execution_id="exec-1",
organization_id="org-1",
)
mock_dispatch.assert_not_called()
if __name__ == "__main__":
pytest.main([__file__, "-v"])