LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
"""dispatch() PG enqueue (9b) + the ``TaskPayload`` wire shape.
The mocked routing/handle behaviour lives in ``test_routing.py``; this
file covers the message serialisation contract and an end-to-end
integration check (a PG-selected ``dispatch()`` lands a decodable row in
``pg_queue_message``). The integration test skips if Postgres is
unreachable or unmigrated.
"""
from __future__ import annotations
import importlib
import json
import os
from unittest.mock import patch
import pytest
from queue_backend import dispatch
from queue_backend.fairness import FairnessKey, WorkloadType
from queue_backend.pg_queue import to_payload
from queue_backend.routing import QueueBackend
# ``queue_backend.dispatch`` the attribute is the function (shadows the
# submodule) — import the module explicitly to reach its globals.
dispatch_mod = importlib.import_module("queue_backend.dispatch")
@pytest.fixture(autouse=True)
def _reset(monkeypatch):
dispatch_mod._pg_routing_logged.clear()
dispatch_mod._pg_local.client = None
# --- to_payload wire shape ---
class TestToPayload:
def test_minimal(self):
assert to_payload("t") == {
"task_name": "t",
"args": [],
"kwargs": {},
"queue": None,
"fairness": None,
}
def test_full(self):
fairness = FairnessKey(
org_id="o", workload_type=WorkloadType.API, pipeline_priority=7
)
payload = to_payload(
"t", args=("a", 1), kwargs={"k": "v"}, queue="q", fairness=fairness
)
assert payload["args"] == ["a", 1] # tuple → list
assert payload["queue"] == "q"
assert payload["fairness"] == {
"org_id": "o",
"workload_type": "api",
"pipeline_priority": 7,
}
def test_json_serialisable(self):
fairness = FairnessKey(org_id="o", workload_type=WorkloadType.NON_API)
# Must round-trip through JSON (it's stored in a JSONB column).
json.dumps(to_payload("t", args=[1], kwargs={"a": "b"}, fairness=fairness))
def test_reply_key_set_when_passed(self):
# The request-reply marker (executor RPC): the consumer keys the result
# row on it; a drop/mis-key would make the caller block until timeout.
assert to_payload("t", reply_key="rk")["reply_key"] == "rk"
def test_reply_key_omitted_when_absent(self):
# Fire-and-forget rows stay byte-identical to before the field existed.
assert "reply_key" not in to_payload("t")
# --- Integration: dispatch() lands a decodable row in pg_queue_message ---
# Uses the shared ``pg_client`` fixture from conftest.py.
class TestDispatchEnqueueIntegration:
def test_dispatch_lands_decodable_row(self, monkeypatch, pg_client):
queue_name = f"test_dispatch_{os.getpid()}"
monkeypatch.setattr(dispatch_mod, "_get_pg_client", lambda: pg_client)
fairness = FairnessKey(org_id="org-x", workload_type=WorkloadType.API)
try:
handle = dispatch(
"leaf_task",
args=[1, 2],
kwargs={"k": "v"},
queue=queue_name,
fairness=fairness,
)
msgs = pg_client.read(queue_name, vt_seconds=30, qty=10)
assert len(msgs) == 1
assert str(msgs[0].msg_id) == handle.id
message = msgs[0].message
assert message["task_name"] == "leaf_task"
assert message["args"] == [1, 2]
assert message["kwargs"] == {"k": "v"}
assert message["queue"] == queue_name
assert message["fairness"]["org_id"] == "org-x"
finally:
with pg_client.conn.cursor() as cur:
cur.execute(
"DELETE FROM pg_queue_message WHERE queue_name = %s", (queue_name,)
)
pg_client.conn.commit()
class TestDispatchBackendOverride:
"""The per-call ``backend=`` override (9e PR 2a inert foundation).
Retained after the allow-list went (UN-4046), so the execution-level PG
pipeline can state a whole execution's headers /
callback onto PG without opting their task *names* in. ``None`` (default)
preserves the allow-list decision exactly — every call site today.
"""
def test_override_pg_forces_pg(self, monkeypatch):
# Task name NOT in the (empty) allow-list → select_backend says Celery;
# the explicit override must still route it to PG.
captured: dict = {}
class _Client:
def send(self, queue_name, payload, **kwargs):
captured.update(queue=queue_name, payload=payload, **kwargs)
return 11
monkeypatch.setattr(dispatch_mod, "_get_pg_client", lambda: _Client())
with patch("queue_backend.dispatch.current_app") as mock_app:
handle = dispatch("pipeline_header", queue="general", backend=QueueBackend.PG)
mock_app.send_task.assert_not_called()
assert handle.id == "11"
assert captured["payload"]["task_name"] == "pipeline_header"
def test_override_celery_forces_celery(self, monkeypatch):
# Task name IS opted into PG, but the override pins it back to Celery.
with (
patch("queue_backend.dispatch.current_app") as mock_app,
patch("queue_backend.dispatch._get_pg_client") as mock_get,
):
dispatch("pipeline_header", queue="general", backend=QueueBackend.CELERY)
mock_app.send_task.assert_called_once()
mock_get.assert_not_called()
def test_default_none_resolves_to_pg(self, monkeypatch):
# No override → allow-list decides. Opted-in name → PG.
captured: dict = {}
class _Client:
def send(self, queue_name, payload, **kwargs):
captured.update(queue=queue_name)
return 5
monkeypatch.setattr(dispatch_mod, "_get_pg_client", lambda: _Client())
with patch("queue_backend.dispatch.current_app") as mock_app:
handle = dispatch("leaf_task", queue="general")
mock_app.send_task.assert_not_called()
assert handle.id == "5"
def test_override_path_carries_fairness_to_row(self, monkeypatch):
# The override's whole purpose is to route an *execution's* dispatches —
# exactly the ones that carry a FairnessKey. The org/priority plumbing
# must work identically on the override path (not just the allow-list
# path), so a regression dropping fairness here would otherwise pass.
captured: dict = {}
class _Client:
def send(self, queue_name, payload, **kwargs):
captured.update(queue=queue_name, **kwargs)
return 13
monkeypatch.setattr(dispatch_mod, "_get_pg_client", lambda: _Client())
fairness = FairnessKey(
org_id="o", workload_type=WorkloadType.API, pipeline_priority=8
)
with patch("queue_backend.dispatch.current_app"):
dispatch(
"pipeline_header",
queue="general",
fairness=fairness,
backend=QueueBackend.PG,
)
assert captured["priority"] == 8
assert captured["org_id"] == "o"
class TestDispatchPriorityWiring:
"""dispatch() carries fairness.pipeline_priority onto the PG row (mocked)."""
@staticmethod
def _capture_send(monkeypatch):
captured: dict = {}
class _Client:
def send(self, queue_name, payload, **kwargs):
captured.update(queue=queue_name, **kwargs)
return 7
monkeypatch.setattr(dispatch_mod, "_get_pg_client", lambda: _Client())
return captured
def test_priority_from_fairness(self, monkeypatch):
captured = self._capture_send(monkeypatch)
fairness = FairnessKey(
org_id="o", workload_type=WorkloadType.API, pipeline_priority=8
)
dispatch("leaf_task", fairness=fairness)
assert captured["priority"] == 8
assert captured["org_id"] == "o"
def test_no_fairness_uses_neutral_defaults(self, monkeypatch):
# Bare dispatch → org_id None (client coerces to "") + DEFAULT_PRIORITY.
from queue_backend.fairness import DEFAULT_PRIORITY
captured = self._capture_send(monkeypatch)
dispatch("leaf_task")
assert captured["priority"] == DEFAULT_PRIORITY
assert captured["org_id"] is None
if __name__ == "__main__":
pytest.main([__file__, "-v"])