LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
"""The success path of the extraction tools.
The transport and auth tests never reach these helpers on a successful call, so
without this file the argument mapping from tool kwargs into
``execute_workflow`` is executed by nothing — a renamed or mis-mapped kwarg
would pass every other test and fail on the first real extraction. The
``assert_called_once_with`` below is the point of these tests.
Mocked at the ``DeploymentHelper`` boundary so no database or Celery is needed.
"""
from __future__ import annotations
from unittest.mock import patch
from django.test import SimpleTestCase
from rest_framework.exceptions import ValidationError
from workflow_manager.workflow_v2.dto import ExecutionResponse
from mcp_server.context import MCPContext
from mcp_server.exceptions import MCPToolError
from mcp_server.tools.execution import extract_document, get_execution_status
# The execution serializer accepts only HTTPS S3 pre-signed URLs; using a
# realistic one here keeps these tests honest about what the tool accepts.
DOC_URL = "https://my-bucket.s3.us-east-1.amazonaws.com/invoice.pdf?X-Amz-Signature=abc"
EXECUTION_ID = "33333333-3333-3333-3333-333333333333"
WORKFLOW_ID = "44444444-4444-4444-4444-444444444444"
# The deployment's own id. Executions record it as `pipeline_id`, and that —
# not workflow_id — is what scopes a poll to one deployment.
DEPLOYMENT_ID = "88888888-8888-8888-8888-888888888888"
class FakeOrganization:
organization_id = "org-mcp"
class FakeAPI:
display_name = "Invoice Extractor"
api_name = "live-api"
is_active = True
organization = FakeOrganization()
# get_execution_status scopes the polled id to this deployment itself.
id = DEPLOYMENT_ID
workflow_id = WORKFLOW_ID
def make_context(active: bool = True) -> MCPContext:
api = FakeAPI()
api.is_active = active
return MCPContext(api=api, api_key="test-key", org_name="org-mcp")
class ExtractDocumentTest(SimpleTestCase):
def setUp(self) -> None:
self.context = make_context()
def _run(self, execute_return=None, **kwargs):
"""Drive extract_document with the whole execution stack stubbed."""
with (
patch(
"mcp_server.tools.execution.DeploymentHelper.load_presigned_files"
) as load_files,
patch(
"mcp_server.tools.execution.APIDeploymentRateLimiter.check_and_acquire",
return_value=(True, {}),
),
patch(
"mcp_server.tools.execution.APIDeploymentRateLimiter.release_slot"
) as release,
patch(
"mcp_server.tools.execution.DeploymentHelper.execute_workflow",
return_value=execute_return
if execute_return is not None
else {"execution_status": "COMPLETED", "message": "ok"},
) as execute,
):
result = extract_document(self.context, **kwargs)
return result, execute, load_files, release
def test_arguments_map_onto_execute_workflow(self) -> None:
"""Guards the tool-kwarg -> execute_workflow mapping. A rename on
either side breaks here rather than in production.
"""
result, execute, load_files, _ = self._run(
document_urls=[DOC_URL],
timeout=45,
include_metadata=True,
include_metrics=True,
include_extracted_text=True,
tags=["alpha"],
)
load_files.assert_called_once()
assert load_files.call_args.args[0] == [DOC_URL]
execute.assert_called_once()
kwargs = execute.call_args.kwargs
assert kwargs["organization_name"] == "org-mcp"
assert kwargs["api"] is self.context.api
assert kwargs["timeout"] == 45
assert kwargs["include_metadata"] is True
assert kwargs["include_metrics"] is True
assert kwargs["include_extracted_text"] is True
# Agents send a JSON array; the execution layer expects parsed names.
assert kwargs["tag_names"] == ["alpha"]
assert kwargs["execution_id"] == result["execution_id"]
def test_result_always_carries_execution_id(self) -> None:
"""On the pending path the id is the agent's only handle for polling,
and execute_workflow does not guarantee it in the body.
"""
result, execute, _, _ = self._run(
execute_return={"execution_status": "PENDING"}, document_urls=[DOC_URL]
)
assert result["execution_status"] == "PENDING"
assert result["execution_id"] == execute.call_args.kwargs["execution_id"]
def test_existing_execution_id_in_response_is_preserved(self) -> None:
"""Never overwrite an id the execution layer reported itself."""
result, _, _, _ = self._run(
execute_return={"execution_status": "COMPLETED", "execution_id": "from-core"},
document_urls=[DOC_URL],
)
assert result["execution_id"] == "from-core"
def test_defaults_are_conservative(self) -> None:
"""Defaults must not silently opt an agent into large or costly output."""
_, execute, _, _ = self._run(document_urls=[DOC_URL])
kwargs = execute.call_args.kwargs
assert kwargs["include_metadata"] is False
assert kwargs["include_metrics"] is False
assert kwargs["include_extracted_text"] is False
assert kwargs["tag_names"] == []
def test_inactive_deployment_rejected_before_quota_is_spent(self) -> None:
with patch(
"mcp_server.tools.execution.DeploymentHelper.execute_workflow"
) as execute:
with self.assertRaises(MCPToolError):
extract_document(make_context(active=False), document_urls=[DOC_URL])
execute.assert_not_called()
def test_non_s3_url_rejected_with_an_explanation(self) -> None:
"""Only S3 pre-signed URLs are accepted. The tool description says so,
and this pins the behaviour the description promises — an agent handed
an ordinary link must be told why, not just that it failed.
"""
with patch(
"mcp_server.tools.execution.DeploymentHelper.execute_workflow"
) as execute:
with self.assertRaises(MCPToolError) as caught:
extract_document(
self.context, document_urls=["https://example.com/invoice.pdf"]
)
execute.assert_not_called()
assert "S3" in str(caught.exception)
def test_invalid_url_rejected_before_quota_is_spent(self) -> None:
with (
patch(
"mcp_server.tools.execution.APIDeploymentRateLimiter.check_and_acquire"
) as acquire,
patch(
"mcp_server.tools.execution.DeploymentHelper.execute_workflow"
) as execute,
):
with self.assertRaises(MCPToolError):
extract_document(self.context, document_urls=["not-a-url"])
acquire.assert_not_called()
execute.assert_not_called()
def test_rate_limited_call_downloads_nothing(self) -> None:
"""The slot is taken before the documents are fetched, so a rejected
call must not pull every document over the network first.
"""
with (
patch(
"mcp_server.tools.execution.DeploymentHelper.load_presigned_files"
) as load_files,
patch(
"mcp_server.tools.execution.APIDeploymentRateLimiter.check_and_acquire",
return_value=(
False,
{"current_usage": 5, "limit": 5, "limit_type": "organization"},
),
),
patch(
"mcp_server.tools.execution.DeploymentHelper.execute_workflow"
) as execute,
):
with self.assertRaises(MCPToolError) as caught:
extract_document(self.context, document_urls=[DOC_URL])
load_files.assert_not_called()
execute.assert_not_called()
assert "5/5" in str(caught.exception)
def test_failed_download_releases_the_rate_limit_slot(self) -> None:
"""The fetch happens after the slot is taken, so a fetch failure has to
give the slot back or the org silently loses throughput.
"""
with (
patch(
"mcp_server.tools.execution.DeploymentHelper.load_presigned_files",
side_effect=RuntimeError("403 Forbidden"),
),
patch(
"mcp_server.tools.execution.APIDeploymentRateLimiter.check_and_acquire",
return_value=(True, {}),
),
patch(
"mcp_server.tools.execution.APIDeploymentRateLimiter.release_slot"
) as release,
patch(
"mcp_server.tools.execution.DeploymentHelper.execute_workflow"
) as execute,
):
with self.assertRaises(MCPToolError):
extract_document(self.context, document_urls=[DOC_URL])
release.assert_called_once()
execute.assert_not_called()
def test_failed_execution_releases_the_rate_limit_slot(self) -> None:
"""A leaked slot degrades the org's throughput permanently, so the
release must survive the failure path.
"""
with (
patch("mcp_server.tools.execution.DeploymentHelper.load_presigned_files"),
patch(
"mcp_server.tools.execution.APIDeploymentRateLimiter.check_and_acquire",
return_value=(True, {}),
),
patch(
"mcp_server.tools.execution.APIDeploymentRateLimiter.release_slot"
) as release,
patch(
"mcp_server.tools.execution.DeploymentHelper.execute_workflow",
side_effect=RuntimeError("boom"),
),
):
with self.assertRaises(MCPToolError):
extract_document(self.context, document_urls=[DOC_URL])
release.assert_called_once()
class GetExecutionStatusTest(SimpleTestCase):
def setUp(self) -> None:
self.context = make_context()
def _run(self, response: ExecutionResponse, scoped: bool = True, **kwargs):
"""Drive get_execution_status with the ORM scope check stubbed.
``scoped=False`` models the id belonging to another deployment: the
filter finds nothing, which is what the guard keys off.
"""
with (
patch(
"mcp_server.tools.execution.WorkflowExecution.objects.filter"
) as scope_filter,
patch(
"mcp_server.tools.execution.ExecutionQuerySerializer.is_valid",
return_value=True,
),
patch(
"mcp_server.tools.execution.ExecutionQuerySerializer.validated_data",
{
"execution_id": EXECUTION_ID,
"include_metadata": kwargs.get("include_metadata", False),
"include_metrics": kwargs.get("include_metrics", False),
"include_extracted_text": kwargs.get("include_extracted_text", False),
},
),
patch(
"mcp_server.tools.execution.DeploymentHelper.get_execution_status",
return_value=response,
),
patch(
"mcp_server.tools.execution.DeploymentHelper.process_completed_execution"
) as process,
):
scope_filter.return_value.only.return_value.first.return_value = (
object() if scoped else None
)
result = get_execution_status(
self.context, execution_id=EXECUTION_ID, **kwargs
)
self.scope_filter = scope_filter
return result, process
def test_completed_execution_is_enriched_and_returned(self) -> None:
response = ExecutionResponse(
workflow_id="wf",
execution_id=EXECUTION_ID,
execution_status="COMPLETED",
result=[{"file": "invoice.pdf", "result": {"output": {"total": 42}}}],
)
result, process = self._run(response, include_metadata=True)
# Enrichment is what turns the raw execution into the API-shaped
# result; skipping it would silently return a thinner payload.
process.assert_called_once()
assert process.call_args.kwargs["include_metadata"] is True
assert result["execution_status"] == "COMPLETED"
assert result["result"][0]["result"]["output"]["total"] == 42
assert result["execution_id"] == EXECUTION_ID
def test_pending_execution_is_not_enriched(self) -> None:
response = ExecutionResponse(
workflow_id="wf",
execution_id=EXECUTION_ID,
execution_status="EXECUTING",
result=None,
)
result, process = self._run(response)
process.assert_not_called()
assert result["execution_status"] == "EXECUTING"
def test_polled_id_is_scoped_to_this_deployment(self) -> None:
"""The scope filter is anchored to the deployment, not just the id.
A filter on ``id`` alone would pass this test's sibling below while
leaving every execution in the shared schema pollable.
It must anchor on ``pipeline_id`` (the deployment that started the run)
rather than ``workflow_id``: ``APIDeployment.workflow`` is a plain FK
with no uniqueness constraint, so two deployments can share a workflow
and a workflow-scoped filter would let either poll the other's runs.
"""
response = ExecutionResponse(
workflow_id="wf",
execution_id=EXECUTION_ID,
execution_status="EXECUTING",
result=None,
)
self._run(response)
self.scope_filter.assert_called_once_with(
id=EXECUTION_ID, pipeline_id=DEPLOYMENT_ID
)
def test_two_deployments_sharing_a_workflow_cannot_poll_each_other(self) -> None:
"""The case a workflow-scoped filter would have missed.
`APIDeployment.workflow` is a plain FK with `related_name="apis"` and no
uniqueness constraint, so two deployments can point at one workflow.
Filtering on `workflow_id` would then match both deployments' runs; the
filter must name the deployment itself.
"""
response = ExecutionResponse(
workflow_id=WORKFLOW_ID,
execution_id=EXECUTION_ID,
execution_status="EXECUTING",
result=None,
)
self._run(response)
kwargs = self.scope_filter.call_args.kwargs
assert kwargs["pipeline_id"] == DEPLOYMENT_ID
assert "workflow_id" not in kwargs, (
"scoping on workflow_id lets a sibling deployment sharing the same "
"workflow poll this deployment's executions"
)
def test_execution_from_another_deployment_is_refused(self) -> None:
"""Polling another deployment's execution must not reach the helper.
DeploymentHelper.get_execution_status resolves the id with no tenant
filter and acknowledging a completed result drops it from cache, so
reaching it at all would both disclose the result and destroy it for
the caller it belongs to.
"""
response = ExecutionResponse(
workflow_id="wf",
execution_id=EXECUTION_ID,
execution_status="COMPLETED",
result=[{"file": "secret.pdf", "result": {"output": {"total": 42}}}],
)
with self.assertRaises(MCPToolError) as caught:
self._run(response, scoped=False)
assert "No execution with id" in str(caught.exception)
def test_foreign_execution_never_reaches_the_helper(self) -> None:
"""The guard runs before the fetch, not alongside it."""
with (
patch(
"mcp_server.tools.execution.WorkflowExecution.objects.filter"
) as scope_filter,
patch(
"mcp_server.tools.execution.ExecutionQuerySerializer.is_valid",
return_value=True,
),
patch(
"mcp_server.tools.execution.ExecutionQuerySerializer.validated_data",
{
"execution_id": EXECUTION_ID,
"include_metadata": False,
"include_metrics": False,
"include_extracted_text": False,
},
),
patch(
"mcp_server.tools.execution.DeploymentHelper.get_execution_status"
) as fetch,
):
scope_filter.return_value.only.return_value.first.return_value = None
with self.assertRaises(MCPToolError):
get_execution_status(self.context, execution_id=EXECUTION_ID)
fetch.assert_not_called()
def test_acknowledged_result_explains_itself(self) -> None:
"""The REST surface answers 406 here; an agent cannot act on a status
code, so the tool must say the result is gone and not retryable.
"""
response = ExecutionResponse(
workflow_id="wf",
execution_id=EXECUTION_ID,
execution_status="COMPLETED",
result=None,
)
response.result_acknowledged = True
result, process = self._run(response)
process.assert_not_called()
assert "already acknowledged" in result["message"]
def test_unknown_execution_id_is_an_agent_error(self) -> None:
"""A bad id comes back as an actionable tool error, not a server
fault — the agent may simply have mistyped it.
"""
with patch(
"mcp_server.tools.execution.ExecutionQuerySerializer.is_valid",
side_effect=ValidationError({"execution_id": ["Invalid execution_id."]}),
):
with self.assertRaises(MCPToolError) as caught:
get_execution_status(self.context, execution_id="nope")
assert "execution_id" in str(caught.exception)