LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
import os
import uuid
from unittest import mock
import django
from django.apps import apps
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings.test")
if not apps.ready:
django.setup()
import pytest # noqa: E402
from django.conf import settings # noqa: E402
from agent_kv import dispatch # noqa: E402
from agent_kv.constants import TABLE_EXTRACTOR_NAME # noqa: E402
from agent_kv.models import AgentKVJob, JobStatus # noqa: E402
def _fail_only_bookkeeping(m_objects, exc):
"""Let the pre-dispatch terminal check succeed, then fail the next query.
These tests are about a DB error during POST-ENQUEUE bookkeeping, which
must not be reported as a failed dispatch. A blanket `filter.side_effect`
now hits the pre-dispatch terminal check instead -- a different, earlier
failure that correctly DOES abort the dispatch, since a job that cannot be
verified as live must not have money spent on it.
"""
live = mock.MagicMock()
live.exists.return_value = False
m_objects.filter.side_effect = [live, exc]
def _filtered_with(m_objects, **expected):
"""True if `objects.filter` was ever called with exactly these kwargs.
Content-based rather than positional: `dispatch_job` makes several filter
calls (the pre-dispatch terminal check, the guarded bookkeeping UPDATE, and
on the slow path a narrowed fallback), and index-pinned assertions broke
every time one was added.
"""
return any(call.kwargs == expected for call in m_objects.filter.call_args_list)
def _not_cancelled(m_objects):
"""Make the pre-dispatch terminal check report "still live".
`dispatch_job` re-reads the row immediately before enqueueing, so a cancel
that landed between the submit's save() and the enqueue cannot result in
paid work for a job whose slot was already released. With `objects` mocked
the default `.exists()` is a truthy Mock, which would short-circuit every
dispatch in this file.
"""
m_objects.filter.return_value.exists.return_value = False
def _job():
from account_v2.models import Organization
j = AgentKVJob(id=uuid.uuid4(), input_ref="org/o/agent_kv/j/input.pdf")
# Unsaved related org with an explicit PK: assigning it caches the
# instance on the job (so ``job.organization`` never hits the DB) and
# sets ``job.organization_id`` to 7; the slug is deliberately different
# from the PK so a test can tell them apart.
j.organization = Organization(id=7, organization_id="org_slug_1") # PK 7
j.pages_total = 3
return j
@mock.patch.object(dispatch, "_platform_api_key", return_value="pk")
@mock.patch.object(dispatch, "_dispatcher")
@mock.patch.object(AgentKVJob, "objects")
def test_dispatch_success_stamps_job(m_objects, m_disp, m_key):
_not_cancelled(m_objects)
job = _job()
dispatch.dispatch_job(
job,
extractor=TABLE_EXTRACTOR_NAME,
schema={"a": {"description": "d"}},
options={"number_format": "EU"},
)
ctx = m_disp.return_value.dispatch_with_callback.call_args.args[0]
assert ctx.executor_name == "agentic_table"
assert ctx.operation == "table_extract_api"
assert ctx.run_id == str(job.id)
assert ctx.execution_source == "agent_kv_api"
assert ctx.organization_id == "7"
assert ctx.executor_params["job_id"] == str(job.id)
assert ctx.executor_params["input_ref"] == job.input_ref
assert ctx.executor_params["schema"] == {"a": {"description": "d"}}
assert ctx.executor_params["options"] == {"number_format": "EU"}
assert ctx.executor_params["platform_api_key"] == "pk"
# max_pages is the CAP the engine must enforce, not the measured count
# (job.pages_total, which rides separately and is None for Excel).
assert ctx.executor_params["max_pages"] == settings.AGENT_KV_MAX_PAGES
assert ctx.executor_params["pages_total"] == 3
kw = m_disp.return_value.dispatch_with_callback.call_args.kwargs
assert kw["on_success"].task == "agent_kv_complete"
assert kw["on_success"].kwargs == {
"callback_kwargs": {"job_id": str(job.id), "org_id": "7"}
}
assert kw["on_success"].options.get("queue") == "agent_kv_callback"
assert kw["on_error"].task == "agent_kv_error"
assert kw["on_error"].kwargs == {
"callback_kwargs": {"job_id": str(job.id), "org_id": "7"}
}
assert kw["task_id"] == str(job.task_id)
assert job.status == JobStatus.DISPATCHED
assert job.dispatched_at is not None
# Guarded queryset UPDATE (not job.save()): only a still-PENDING row may
# be advanced to DISPATCHED.
# The pre-dispatch terminal check filters first; the guarded bookkeeping
# UPDATE is the last filter call.
assert _filtered_with(m_objects, id=job.id, status=JobStatus.PENDING)
m_objects.filter.return_value.update.assert_called_once_with(
task_id=job.task_id,
status=JobStatus.DISPATCHED,
dispatched_at=job.dispatched_at,
)
@mock.patch.object(dispatch, "_platform_api_key", return_value="pk")
@mock.patch.object(dispatch, "_dispatcher")
@mock.patch.object(AgentKVJob, "objects")
def test_dispatch_guarded_update_cannot_overwrite_a_terminal_row(
m_objects, m_disp, m_key
):
"""Regression for the un-terminalize bug: if the row already raced to a
terminal status (e.g. FAILED, via an instant-failure finalize callback
that beat this post-enqueue bookkeeping), the guarded UPDATE's WHERE
clause (status=PENDING) structurally cannot match it -- 0 rows update,
and the row's real status is left untouched. Proven the same way
``AgentKVJob.mark_terminal``'s own guard is proven (test_models.py): by
pinning the exact WHERE-clause kwargs and simulating the "no rows
matched" outcome, since this suite runs with no real DB.
"""
_not_cancelled(m_objects)
m_objects.filter.return_value.update.return_value = 0 # simulates a FAILED row
job = _job()
# Must not raise -- dispatch_job doesn't (and can't meaningfully) act on
# the update's row count; it already told the caller it dispatched.
dispatch.dispatch_job(job, extractor=TABLE_EXTRACTOR_NAME, schema={}, options={})
assert _filtered_with(m_objects, id=job.id, status=JobStatus.PENDING)
# The 0-row result now triggers a SECOND guarded write (below), whose
# `.exclude(status__in=TERMINAL)` is what keeps a terminal row untouched
# here -- so the terminal case is still structurally safe.
(_, excl_kwargs) = (
m_objects.filter.return_value.exclude.call_args.args,
m_objects.filter.return_value.exclude.call_args.kwargs,
)
assert set(excl_kwargs["status__in"]) == set(AgentKVJob.TERMINAL)
# The stranded-job regression. `StageReportView` promotes PENDING -> RUNNING on
# the executor's first stage report, which can land before this post-enqueue
# bookkeeping. The PENDING-guarded UPDATE then matches 0 rows and
# `dispatched_at` stays NULL -- and a non-terminal row with a NULL
# `dispatched_at` is invisible to BOTH sweep phases (phase 1 requires PENDING;
# phase 2's `dispatched_at__lt` can never match a NULL), so the job reports
# `running` forever and `GET result` 409s for the life of the row.
@mock.patch.object(dispatch, "_platform_api_key", return_value="pk")
@mock.patch.object(dispatch, "_dispatcher")
@mock.patch.object(AgentKVJob, "objects")
def test_dispatch_stamps_dispatched_at_when_the_row_already_moved_to_running(
m_objects, m_disp, m_key
):
_not_cancelled(m_objects)
m_objects.filter.return_value.update.return_value = 0 # PENDING guard missed
job = _job()
dispatch.dispatch_job(job, extractor=TABLE_EXTRACTOR_NAME, schema={}, options={})
# Second write is narrowed to a row that still has no dispatch time, and
# excludes terminal rows.
assert _filtered_with(m_objects, id=job.id, dispatched_at__isnull=True)
fallback = m_objects.filter.return_value.exclude.return_value
update_kwargs = fallback.update.call_args.kwargs
assert update_kwargs["dispatched_at"] == job.dispatched_at
assert update_kwargs["task_id"] == job.task_id
# Crucially does NOT write `status`: the row genuinely was dispatched, but
# moving RUNNING back to DISPATCHED would discard the executor's progress.
assert "status" not in update_kwargs
@mock.patch.object(dispatch, "_platform_api_key", return_value="pk")
@mock.patch.object(dispatch, "_dispatcher")
@mock.patch.object(AgentKVJob, "objects")
def test_dispatch_does_not_attempt_the_fallback_when_the_pending_guard_won(
m_objects, m_disp, m_key
):
_not_cancelled(m_objects)
m_objects.filter.return_value.update.return_value = 1 # normal path
job = _job()
dispatch.dispatch_job(job, extractor=TABLE_EXTRACTOR_NAME, schema={}, options={})
# Terminal check + the single guarded bookkeeping UPDATE; no fallback.
assert len(m_objects.filter.call_args_list) == 2
assert not m_objects.filter.return_value.exclude.called
@mock.patch.object(dispatch, "_platform_api_key", return_value="pk")
@mock.patch.object(dispatch, "_dispatcher")
def test_enqueue_failure_raises_dispatch_error(m_disp, m_key):
m_disp.return_value.dispatch_with_callback.side_effect = RuntimeError("broker down")
job = _job()
with pytest.raises(dispatch.DispatchError):
dispatch.dispatch_job(job, extractor=TABLE_EXTRACTOR_NAME, schema={}, options={})
@mock.patch.object(dispatch, "_dispatcher")
def test_dispatch_job_uses_platform_api_key_lookup(m_disp):
from platform_settings_v2.platform_auth_service import (
PlatformAuthenticationService,
)
with mock.patch.object(
PlatformAuthenticationService,
"get_active_platform_key",
return_value=mock.Mock(key="the-real-key"),
):
job = _job()
with mock.patch.object(AgentKVJob, "objects") as m_objects:
# Without this the pre-dispatch terminal check reads a truthy Mock
# and short-circuits, so nothing is ever enqueued.
_not_cancelled(m_objects)
dispatch.dispatch_job(
job, extractor=TABLE_EXTRACTOR_NAME, schema={}, options={}
)
ctx = m_disp.return_value.dispatch_with_callback.call_args.args[0]
assert ctx.executor_params["platform_api_key"] == "the-real-key"
# The lookup takes the org's public slug, never the row PK (13b F6).
PlatformAuthenticationService.get_active_platform_key.assert_called_once_with(
"org_slug_1"
)
assert ctx.organization_id == "7"
def test_platform_api_key_raises_dispatch_error_when_absent():
from platform_settings_v2.platform_auth_service import (
PlatformAuthenticationService,
)
with mock.patch.object(
PlatformAuthenticationService,
"get_active_platform_key",
return_value=None,
):
job = _job()
with pytest.raises(dispatch.DispatchError):
dispatch._platform_api_key(job)
@mock.patch.object(dispatch, "_dispatcher")
@mock.patch.object(dispatch, "_platform_api_key")
def test_raw_exception_from_platform_key_lookup_is_wrapped_as_dispatch_error(
m_key, m_disp
):
"""Regression: platform-key lookup and context construction must live
inside dispatch_job's try — a raw (non-DispatchError) exception there
(e.g. a transient DB error) must not escape uncaught.
"""
m_key.side_effect = RuntimeError("platform db down")
job = _job()
with pytest.raises(dispatch.DispatchError):
dispatch.dispatch_job(job, extractor=TABLE_EXTRACTOR_NAME, schema={}, options={})
# Never got far enough to enqueue.
assert not m_disp.return_value.dispatch_with_callback.called
def test_dispatcher_factory_call_matches_the_live_signature():
"""Guard the OSS seam that a platform change can silently move.
UN-4046 removed `get_executor_dispatcher(celery_app=...)`; our call site
kept passing it, so every submit raised TypeError inside dispatch_job and
failed the job. Nothing caught it -- the mismatch is invisible to a mocked
dispatcher and only appears when a real request is made.
Binding our actual call against the real signature fails loudly the next
time that function's parameters change.
"""
from unittest import mock # noqa: PLC0415
from agent_kv import dispatch as d # noqa: PLC0415
# autospec=True makes the stub enforce the REAL function's signature, so
# this asserts our call site against it rather than against a permissive
# Mock. Re-adding `celery_app=` here raises TypeError, exactly as production
# did.
with mock.patch(
"pg_queue.executor_rpc.get_executor_dispatcher", autospec=True
) as m_factory:
d._dispatcher()
m_factory.assert_called_once_with()
# Post-enqueue bookkeeping must never fail a dispatch that already succeeded.
# `SubmitView` turns a DispatchError into a FAILED job, so a DB hiccup here
# would terminalize a job whose task is on the queue -- the executor then runs,
# calls back, and finds a terminal row it cannot write to, while the caller was
# told nothing was billed for work that did run. The review round added a
# SECOND update after the enqueue, which widened this window.
@mock.patch.object(dispatch, "_platform_api_key", return_value="pk")
@mock.patch.object(dispatch, "_dispatcher")
@mock.patch.object(AgentKVJob, "objects")
def test_bookkeeping_failure_does_not_fail_an_already_queued_dispatch(
m_objects, m_disp, m_key
):
_not_cancelled(m_objects)
_fail_only_bookkeeping(m_objects, OSError("db gone"))
job = _job()
# Must not raise: raising is what would mark the live job FAILED.
dispatch.dispatch_job(job, extractor=TABLE_EXTRACTOR_NAME, schema={}, options={})
# And the enqueue did happen, so the task is genuinely on the queue.
assert m_disp.return_value.dispatch_with_callback.called
@mock.patch.object(dispatch, "_platform_api_key", return_value="pk")
@mock.patch.object(dispatch, "_dispatcher")
@mock.patch.object(AgentKVJob, "objects")
def test_bookkeeping_failure_is_logged_at_error_level(m_objects, m_disp, m_key, caplog):
"""Swallowing it silently would trade one bad failure mode for another:
the sweep reconciles the row, but nothing would say why it had to.
"""
_not_cancelled(m_objects)
_fail_only_bookkeeping(m_objects, OSError("db gone"))
with caplog.at_level("ERROR", logger=dispatch.logger.name):
dispatch.dispatch_job(
_job(), extractor=TABLE_EXTRACTOR_NAME, schema={}, options={}
)
assert any(r.levelname == "ERROR" and r.exc_info for r in caplog.records)
# ---------------------------------------------------------------------------
# A job cancelled between submit's save() and the enqueue must not be dispatched.
#
# The cancel sees a PENDING, never-dispatched row, so it terminalizes it AND
# releases its concurrency slot -- correctly, nothing had been dispatched yet.
# Enqueueing anyway would then run paid work for a job the caller already
# cancelled, with its slot already handed to the next submit: the concurrency
# ceiling bypassed and the customer billed for a cancelled job.
#
# Reported by Greptile on PR #2317.
# ---------------------------------------------------------------------------
@mock.patch.object(dispatch, "_platform_api_key", return_value="pk")
@mock.patch.object(dispatch, "_dispatcher")
@mock.patch.object(AgentKVJob, "objects")
def test_a_job_cancelled_before_enqueue_is_not_dispatched(m_objects, m_disp, m_key):
job = _job()
m_objects.filter.return_value.exists.return_value = True # already terminal
dispatch.dispatch_job(job, extractor=TABLE_EXTRACTOR_NAME, schema={}, options={})
assert (
not m_disp.return_value.dispatch_with_callback.called
), "paid work was enqueued for a job that was already cancelled"
# And no bookkeeping UPDATE either -- there is nothing to advance.
assert not m_objects.filter.return_value.update.called
@mock.patch.object(dispatch, "_platform_api_key", return_value="pk")
@mock.patch.object(dispatch, "_dispatcher")
@mock.patch.object(AgentKVJob, "objects")
def test_the_terminal_check_reads_the_row_rather_than_the_in_memory_copy(
m_objects, m_disp, m_key
):
"""The in-memory job predates the cancel by construction, so trusting
`job.status` here would never see it.
"""
job = _job()
job.status = JobStatus.PENDING # stale: the row may already be CANCELLED
_not_cancelled(m_objects)
dispatch.dispatch_job(job, extractor=TABLE_EXTRACTOR_NAME, schema={}, options={})
assert _filtered_with(m_objects, id=job.id, status__in=list(AgentKVJob.TERMINAL))
# ---------------------------------------------------------------------------
# Terminal-webhook enqueue (both paths share one helper)
# ---------------------------------------------------------------------------
class _FakeJob:
"""Just the three attributes the enqueue reads."""
def __init__(self, webhook_url="https://hook.example/x"):
self.id = uuid.uuid4()
self.organization_id = uuid.uuid4()
self.webhook_url = webhook_url
def _patch_producer(monkeypatch):
"""Stand in for `pg_queue.producer`, which `_enqueue_webhook` imports
lazily inside the call."""
import sys
import types
calls = []
mod = types.ModuleType("pg_queue.producer")
mod.enqueue_task = lambda **kw: calls.append(kw)
pkg = sys.modules.get("pg_queue") or types.ModuleType("pg_queue")
monkeypatch.setitem(sys.modules, "pg_queue", pkg)
monkeypatch.setitem(sys.modules, "pg_queue.producer", mod)
return calls
def test_finalize_enqueues_the_terminal_webhook_with_the_backend_s_status(monkeypatch):
"""The status rides on the message, so the worker announces what the
backend actually terminalized the row to rather than guessing."""
calls = _patch_producer(monkeypatch)
job = _FakeJob()
dispatch.dispatch_terminal_webhook(job, "failed")
assert len(calls) == 1
assert calls[0]["task_name"] == "agent_kv_finalized"
assert calls[0]["queue"] == dispatch.CALLBACK_QUEUE
assert calls[0]["kwargs"]["callback_kwargs"] == {
"job_id": str(job.id),
"webhook_url": "https://hook.example/x",
"status": "failed",
}
assert calls[0]["org_id"] == str(job.organization_id)
def test_cancel_still_enqueues_its_own_task_through_the_same_helper(monkeypatch):
"""Sharing the helper must not have collapsed the two task names into
one: they are separate consumers and separate notifications."""
calls = _patch_producer(monkeypatch)
job = _FakeJob()
dispatch.dispatch_cancelled_webhook(job)
assert len(calls) == 1
assert calls[0]["task_name"] == "agent_kv_cancelled"
assert calls[0]["kwargs"]["callback_kwargs"]["status"] == "cancelled"
def test_no_webhook_url_enqueues_nothing(monkeypatch):
calls = _patch_producer(monkeypatch)
dispatch.dispatch_terminal_webhook(_FakeJob(webhook_url=""), "completed")
dispatch.dispatch_cancelled_webhook(_FakeJob(webhook_url=""))
assert calls == []
def test_a_failed_enqueue_is_logged_not_raised(monkeypatch, caplog):
"""Best-effort by design: the job is already terminal and the caller
already has their response, so failing that request because a
NOTIFICATION could not be queued would be the wrong trade."""
import logging
import sys
import types
mod = types.ModuleType("pg_queue.producer")
def _boom(**kw):
raise RuntimeError("queue down")
mod.enqueue_task = _boom
pkg = sys.modules.get("pg_queue") or types.ModuleType("pg_queue")
monkeypatch.setitem(sys.modules, "pg_queue", pkg)
monkeypatch.setitem(sys.modules, "pg_queue.producer", mod)
with caplog.at_level(logging.ERROR, logger="agent_kv.dispatch"):
dispatch.dispatch_terminal_webhook(_FakeJob(), "completed")
assert "could not queue the completed webhook" in caplog.text