LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
"""Tests for the backend executor-RPC adapter.
The dispatch contract (never-raises / result interpretation), the routing, and the
Flipt gate matrix live ONCE in the shared module and are covered in
``workers/tests/test_executor_rpc.py`` against a fake transport. This suite pins the
**backend half**: the :class:`DjangoQueueTransport` (enqueue via the ORM, poll
``PgTaskResult`` → ``ExecResultRow``), the resolver's delegation to the shared
single-Flipt-flag gate, and the factory wiring.
"""
from unittest.mock import MagicMock, patch
from pg_queue.executor_rpc import (
DjangoQueueTransport,
get_executor_dispatcher,
)
from unstract.workflow_execution.executor_rpc import ExecResultRow, PgExecutionDispatcher
_MOD = "pg_queue.executor_rpc"
def _ctx() -> MagicMock:
c = MagicMock()
c.executor_name = "legacy"
c.run_id = "r"
c.to_dict.return_value = {"run_id": "r"}
return c
class TestDjangoQueueTransportEnqueue:
def test_enqueue_calls_enqueue_task_with_request_reply_fields(self):
with patch(f"{_MOD}.enqueue_task") as enq:
DjangoQueueTransport().enqueue(
queue="celery_executor_legacy",
context=_ctx(),
org_id="org9",
reply_key="rk1",
)
kw = enq.call_args.kwargs
assert kw["task_name"] == "execute_extraction"
assert kw["queue"] == "celery_executor_legacy"
assert kw["args"] == [{"run_id": "r"}]
assert kw["org_id"] == "org9"
assert kw["reply_key"] == "rk1"
assert kw["task_id"] is None and kw["on_success"] is None
def test_enqueue_carries_continuations(self):
spec = {"task_name": "ide_prompt_complete", "kwargs": {}, "queue": "ide_callback"}
with patch(f"{_MOD}.enqueue_task") as enq:
DjangoQueueTransport().enqueue(
queue="celery_executor_legacy",
context=_ctx(),
org_id="o",
on_success=spec,
task_id="tid-7",
)
kw = enq.call_args.kwargs
assert kw["on_success"] == spec and kw["task_id"] == "tid-7"
assert kw["reply_key"] is None # callback dispatch, not request-reply
class TestDjangoQueueTransportWait:
def test_present_row_folds_to_exec_result_row(self):
row = MagicMock(status="completed", result={"a": 1}, error="")
qs = MagicMock()
qs.filter.return_value.first.return_value = row
with patch(f"{_MOD}.PgTaskResult", MagicMock(objects=qs)):
out = DjangoQueueTransport().wait_for_result("rk", timeout=5)
assert isinstance(out, ExecResultRow)
assert out.status == "completed" and out.result == {"a": 1}
def test_timeout_returns_none(self):
qs = MagicMock()
qs.filter.return_value.first.return_value = None # never appears
with (
patch(f"{_MOD}.PgTaskResult", MagicMock(objects=qs)),
patch(f"{_MOD}.close_old_connections"),
):
# timeout=0 → first poll misses, remaining <= 0 → None (no sleep).
assert DjangoQueueTransport().wait_for_result("rk", timeout=0) is None
def test_multi_iteration_poll_misses_then_hits(self):
# The load-bearing backoff path: a miss, then a hit on the next poll — the
# connection is released (close_old_connections) and it sleeps once between.
row = MagicMock(status="completed", result={"a": 1}, error="")
qs = MagicMock()
qs.filter.return_value.first.side_effect = [None, row]
with (
patch(f"{_MOD}.PgTaskResult", MagicMock(objects=qs)),
patch(f"{_MOD}.close_old_connections") as cl,
patch("unstract.core.polling.time.sleep") as slp,
):
out = DjangoQueueTransport().wait_for_result("rk", timeout=5)
assert isinstance(out, ExecResultRow) and out.result == {"a": 1}
cl.assert_called_once() # between_polls released the conn on the miss
slp.assert_called_once() # slept once between the two polls
def test_present_row_clears_payload_after_consume(self):
# Mirrors the workers' PgResultBackend.forget: once the reply is read, both
# payload channels are nulled so PII doesn't sit for the full retention TTL.
row = MagicMock(status="completed", result={"a": 1}, error="")
qs = MagicMock()
qs.filter.return_value.first.return_value = row
with patch(f"{_MOD}.PgTaskResult", MagicMock(objects=qs)):
DjangoQueueTransport().wait_for_result("rk", timeout=5)
qs.filter.return_value.update.assert_called_once_with(result=None, error="")
def test_timeout_does_not_clear(self):
# Nothing consumed on timeout — no clear (the executor may still write the
# row under this reply_key; the reaper sweeps the orphan at TTL).
qs = MagicMock()
qs.filter.return_value.first.return_value = None
with (
patch(f"{_MOD}.PgTaskResult", MagicMock(objects=qs)),
patch(f"{_MOD}.close_old_connections"),
):
DjangoQueueTransport().wait_for_result("rk", timeout=0)
qs.filter.return_value.update.assert_not_called()
class TestFactoryWiring:
def test_factory_wires_the_django_transport(self):
# The factory takes no arguments (UN-4046) and returns the PG dispatcher
# wired with the backend ORM transport.
d = get_executor_dispatcher()
assert isinstance(d, PgExecutionDispatcher)
assert isinstance(d._transport, DjangoQueueTransport)