LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
"""Guard: queues and routes must match the fleet this deployment actually runs.
The OSS analogue of the cloud chart's ``validate-pg-worker-fleet.yaml``, and it
exists because the failure it catches is **silent**: since UN-4046 made the PG
transport unconditional, a queue whose consumer is not configured still
*accepts* work -- ``enqueue_task`` succeeds, rows land in ``pg_queue_message``,
and nothing errors at the producer. The job simply sits in DISPATCHED forever.
The assertions run in BOTH directions, which is the part to read carefully:
* ``celery_executor_agentic_table`` and ``agent_kv_callback`` must HAVE a
consumer -- they are what this deployment dispatches onto.
* ``celery_executor_agentic_kv`` must NOT be advertised. This deployment ships
no ``agentic_kv`` plugin, so a fleet listing that queue would accept work
nothing can drain -- the same silent failure, arrived at from the other side.
**These names are literals, and that is a known limitation.** Re-enabling the
``kv`` extractor is one ``EXTRACTOR_ROUTES`` entry, after which
``celery_executor_agentic_kv`` must be SERVED -- and the inverse assertion
above would have to be deleted to go green, which is the one moment a guard
should not be getting deleted. The derived version of both directions lives in
``backend/agent_kv/tests/test_queue_wiring_is_derived.py``, where
``EXTRACTOR_ROUTES`` is actually importable (the backend package is not
importable from this test venv -- that asymmetry is why the duplication
exists). That suite also covers ``run-worker-docker.sh``, which this one does
not reach.
Keeping both is deliberate: two independent statements of the same fact, one of
which does not depend on the routing table being correct.
An earlier version of this docstring described only the first direction and
narrated how ``celery_executor_agentic_kv`` ought to be wired in. That is the
one thing a guard with an inverted assertion must not say: a reader checking
the file against its own description would have concluded the test was wrong.
Each test asserts against the variable the *running* consumer actually reads,
at every site that configures one -- plus the traefik rules, where the same
"configured-looking but unreachable" failure shows up as a request served by
the SPA instead of Django.
"""
from __future__ import annotations
import re
from pathlib import Path
import pytest
yaml = pytest.importorskip("yaml")
REPO_ROOT = Path(__file__).resolve().parents[2]
DEV_COMPOSE = REPO_ROOT / "docker" / "docker-compose.yaml"
TEST_COMPOSE = REPO_ROOT / "tests" / "compose" / "docker-compose.test.yaml"
RUN_WORKER = REPO_ROOT / "workers" / "run-worker.sh"
# The queue the `kv` extractor WOULD dispatch onto, and which this deployment
# deliberately does not serve: the `agentic_kv` plugin ships on a separate PR.
# `kv` is absent from EXTRACTOR_ROUTES so a submit is refused at the serializer
# with a 400 -- these assertions are the second half of that, pinning that no
# fleet advertises a consumer for it. A queue that is listed but unserved
# accepts work and drains nothing, with no error at the producer: the job sits
# in DISPATCHED forever. That failure has already cost this team ~30 hours of
# firings with zero executions, which is why it is guarded from both ends.
UNSERVED_KV_QUEUE = "celery_executor_agentic_kv"
# The terminal-callback queue. Its consumer is what moves a job out of
# RUNNING: finalize persists the result, deletes the staged input, releases
# the concurrency slot and fires the webhook.
AGENT_KV_CALLBACK_QUEUE = "agent_kv_callback"
# The queue the backend dispatches Agent-KV TABLE extraction onto. Same queue
# the IDE table path already uses -- ExecutionContext(executor_name=
# "agentic_table") -> celery_executor_agentic_table -- which is exactly why the
# API path reuses that executor name rather than introducing one of its own.
# It was already wired for the IDE; these assertions keep it wired now that a
# second, paid entry point depends on it.
AGENT_TABLE_QUEUE = "celery_executor_agentic_table"
# The env var the live PG consumer reads. Named here so a future rename has to
# touch this constant rather than silently bypassing every assertion below.
PG_QUEUE_VAR = "WORKER_PG_QUEUE_CONSUMER_QUEUE"
# Read only by the disabled Celery executor. Setting a queue here does NOT give
# it a consumer -- asserting its absence is what makes this suite catch the
# original bug rather than a cosmetic rename of it.
DEAD_CELERY_VAR = "CELERY_QUEUES_EXECUTOR"
def _service_env(compose_path: Path, service: str) -> dict[str, str]:
"""Return ``service``'s environment as a dict, from a compose file."""
doc = yaml.safe_load(compose_path.read_text())
env = (doc.get("services", {}).get(service) or {}).get("environment") or []
if isinstance(env, dict): # mapping form
return {k: str(v) for k, v in env.items()}
out: dict[str, str] = {}
for item in env: # list form: "KEY=value"
key, _, value = str(item).partition("=")
out[key] = value
return out
def _queues(raw: str) -> set[str]:
"""Queue names from a consumer queue list, ignoring any ``${VAR:-default}``
wrapper the compose file uses to keep the value overridable.
"""
inner = re.sub(r"^\$\{[^:}]+:-(.*)\}$", r"\1", raw.strip())
return {q.strip() for q in inner.split(",") if q.strip()}
@pytest.mark.parametrize(
("compose_path", "label"),
[(DEV_COMPOSE, "dev compose"), (TEST_COMPOSE, "e2e test compose")],
)
def test_no_fleet_advertises_the_unserved_kv_queue(compose_path, label):
"""The executor fleet must not list a queue with no plugin behind it."""
env = _service_env(compose_path, "worker-pg-executor")
raw = env.get(PG_QUEUE_VAR)
assert raw is not None, (
f"{label}: worker-pg-executor sets no {PG_QUEUE_VAR}; the running "
f"pg-queue-consumer would fall back to a default of unknown content"
)
assert UNSERVED_KV_QUEUE not in _queues(raw), (
f"{label}: {UNSERVED_KV_QUEUE} is advertised but this deployment "
f"carries no agentic_kv plugin to drain it. Work routed there is "
f"accepted and never runs, silently. Remove it from {PG_QUEUE_VAR}, "
f"or ship the plugin."
)
def test_test_compose_does_not_wire_the_queue_onto_the_dead_celery_var():
"""Regression: the e2e override once set the Celery-only variable.
That drained nothing on the PG transport. If someone re-adds it, the queue
looks configured while the lane still hangs -- so fail loudly.
Scoped to TEST_COMPOSE deliberately, and NOT parametrized over both files
-- review asked for the parametrization and this is why it is wrong here.
`docker/docker-compose.yaml` legitimately sets `CELERY_QUEUES_EXECUTOR` on
`worker-pg-executor` (it is consumed by `run-worker-docker.sh:281` and by
`workers/executor/worker.py` for anyone still launching the Celery
executor), so parametrizing would turn this into an immediate red on a
config that is vestigial rather than wrong.
What generalizes is not "the variable must be absent" but "the variable
must never advertise a queue the PG consumer does not serve" -- which is
the next test, and which DOES run against both files.
"""
env = _service_env(TEST_COMPOSE, "worker-pg-executor")
assert DEAD_CELERY_VAR not in env, (
f"{DEAD_CELERY_VAR} is read only by the disabled Celery executor "
f"(workers/executor/worker.py). Queues belong in {PG_QUEUE_VAR}."
)
@pytest.mark.parametrize(
("compose_path", "label"),
[(DEV_COMPOSE, "dev compose"), (TEST_COMPOSE, "e2e test compose")],
)
def test_the_dead_celery_var_never_advertises_an_unserved_queue(compose_path, label):
"""The generalized form of the test above, over both compose files.
Wherever `CELERY_QUEUES_EXECUTOR` survives, it must stay a SUBSET of what
the PG consumer actually drains. The hazard is not the variable existing --
it is the variable drifting into a superset, because then a reader
comparing the two lists concludes a queue is served when nothing drains it,
and the Celery path that would have drained it is disabled.
Concretely this fails if someone adds `celery_executor_agentic_kv` to the
Celery list while the PG list correctly omits it.
"""
env = _service_env(compose_path, "worker-pg-executor")
dead = env.get(DEAD_CELERY_VAR)
if dead is None:
pytest.skip(f"{label} does not set {DEAD_CELERY_VAR}")
pg_raw = env.get(PG_QUEUE_VAR)
assert pg_raw is not None, f"{label}: no {PG_QUEUE_VAR} to compare against"
extra = _queues(dead) - _queues(pg_raw)
assert not extra, (
f"{label}: {DEAD_CELERY_VAR} advertises {sorted(extra)}, which "
f"{PG_QUEUE_VAR} does not serve. {DEAD_CELERY_VAR} is read only by the "
f"disabled Celery executor, so nothing drains those -- work routed "
f"there is accepted and never runs. Add them to {PG_QUEUE_VAR} or drop "
f"them here."
)
def test_run_worker_pg_executor_role_omits_the_unserved_kv_queue():
"""The host-run fleet's hardcoded role default must match the compose stacks.
``run-worker.sh`` carries its own PG_CONSUMER_ROLES map; a queue listed
here but not in compose is just as unconsumed for anyone running the fleet
directly.
"""
text = RUN_WORKER.read_text()
match = re.search(r'\["\$PG_ROLE_EXECUTOR"\]="executor;([^"]+)"', text)
assert match, "could not find the PG_ROLE_EXECUTOR entry in run-worker.sh"
assert UNSERVED_KV_QUEUE not in _queues(match.group(1)), (
f"run-worker.sh's pg-executor role advertises {UNSERVED_KV_QUEUE}; a "
f"host-run fleet would accept Agent-KV work and never drain it."
)
@pytest.mark.parametrize(
("compose_path", "label"),
[(DEV_COMPOSE, "dev compose"), (TEST_COMPOSE, "e2e test compose")],
)
def test_pg_executor_consumes_the_agent_table_queue(compose_path, label):
"""Agent-KV table jobs ride the IDE table executor's queue (spec R1)."""
env = _service_env(compose_path, "worker-pg-executor")
raw = env.get(PG_QUEUE_VAR)
assert raw is not None, (
f"{label}: worker-pg-executor sets no {PG_QUEUE_VAR}; the running "
f"pg-queue-consumer would fall back to a default that omits "
f"{AGENT_TABLE_QUEUE}"
)
assert AGENT_TABLE_QUEUE in _queues(raw), (
f"{label}: {AGENT_TABLE_QUEUE} has no consumer -- both IDE table "
f"prompts and Agent-KV table jobs will sit in DISPATCHED forever with "
f"no error at the producer. Add it to {PG_QUEUE_VAR}."
)
def test_run_worker_pg_executor_role_lists_the_agent_table_queue():
text = RUN_WORKER.read_text()
match = re.search(r'\["\$PG_ROLE_EXECUTOR"\]="executor;([^"]+)"', text)
assert match, "could not find the PG_ROLE_EXECUTOR entry in run-worker.sh"
assert AGENT_TABLE_QUEUE in _queues(match.group(1)), (
f"run-worker.sh's pg-executor role omits {AGENT_TABLE_QUEUE}; a "
f"host-run fleet would accept table work and never drain it."
)
def test_pg_ide_callback_drains_the_agent_kv_callback_queue():
"""Without this the job runs to SUCCESS and then never completes.
The executor enqueues its terminal continuation onto `agent_kv_callback`;
if no consumer drains it the row sits in pg_queue_message and the job stays
RUNNING forever, with nothing logged at the producer. The Celery
ide_callback worker has carried both queues since the callbacks landed --
its PG twin did not, and the cloud chart wires it correctly, so only the
OSS stacks were affected. Found by running a real job end to end.
"""
env = _service_env(DEV_COMPOSE, "worker-pg-ide-callback")
raw = env.get(PG_QUEUE_VAR)
assert raw is not None, "worker-pg-ide-callback sets no consumer queue"
assert AGENT_KV_CALLBACK_QUEUE in _queues(raw), (
f"{AGENT_KV_CALLBACK_QUEUE} has no consumer -- Agent-KV jobs will "
f"finish executing and never leave RUNNING."
)
def test_run_worker_pg_ide_callback_role_drains_the_callback_queue():
text = RUN_WORKER.read_text()
match = re.search(r'\["\$PG_ROLE_IDE_CALLBACK"\]="ide_callback;([^"]+)"', text)
assert match, "could not find the PG_ROLE_IDE_CALLBACK entry in run-worker.sh"
assert AGENT_KV_CALLBACK_QUEUE in _queues(match.group(1))
# --------------------------------------------------------------------------
# Traefik routing. Same family of failure as an unconsumed queue: the service
# is up, the route looks configured, and the request lands somewhere that
# cannot serve it.
#
# The Agent-KV API is mounted under `AGENT_KV_PATH_PREFIX`, which defaults to
# `api/v1/agent-kv` -- inside the `/api/v1` prefix traefik already routes to
# the backend. It used to sit at `/agent-kv`, needing its own term in BOTH the
# backend rule and the frontend's negation of it; miss either and every
# request was served by the SPA's nginx and never reached Django. The e2e lane
# could not catch that, because it talks to `UNSTRACT_BACKEND_URL` directly
# and bypasses traefik.
#
# Sharing the prefix is what removes the failure mode rather than guarding it,
# so this asserts the coupling is gone: the prefix must stay under a path
# traefik already sends to the backend.
#
# Reported as 2.2 in the branch review; the prefix move came from review of
# unstract-cloud#1828.
# --------------------------------------------------------------------------
def _router_rule(compose_path: Path, router: str) -> str:
svc_labels = []
for svc in yaml.safe_load(compose_path.read_text())["services"].values():
svc_labels.extend(svc.get("labels") or [])
prefix = f"traefik.http.routers.{router}.rule="
for label in svc_labels:
if isinstance(label, str) and label.startswith(prefix):
return label[len(prefix) :]
pytest.fail(f"no traefik rule for router {router!r} in {compose_path.name}")
def test_agent_kv_rides_a_prefix_traefik_already_routes_to_the_backend():
"""No dedicated `/agent-kv` term, because the prefix lives under one that
is already routed. If the prefix is ever moved back outside `/api/v1`,
both the backend rule and the frontend's negation need a term again --
and missing the second one is silent.
"""
rule = _router_rule(DEV_COMPOSE, "backend")
assert "PathPrefix(`/api/v1`)" in rule, rule
assert "/agent-kv" not in rule, (
"a dedicated /agent-kv term is back in the traefik rule; either the "
"prefix moved out of /api/v1, or this term is now dead weight"
)
def test_every_backend_prefix_is_excluded_from_the_frontend():
"""The two rules are each other's complement by construction. Asserting the
relationship rather than one hardcoded prefix means the next mount point
cannot be added to one side only.
"""
backend_rule = _router_rule(DEV_COMPOSE, "backend")
frontend_rule = _router_rule(DEV_COMPOSE, "frontend")
backend_prefixes = set(re.findall(r"PathPrefix\(`([^`]+)`\)", backend_rule))
excluded = set(re.findall(r"!PathPrefix\(`([^`]+)`\)", frontend_rule))
missing = backend_prefixes - excluded
assert not missing, (
f"{sorted(missing)} route to the backend but are not excluded from the "
f"frontend router; the SPA will claim them"
)