LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
"""The 400/500 guards on the prompt-studio output and callback handlers.
Each of these replaces a 500: a non-UUID id raises Django's ``ValidationError``
while the query is being *built*, which drf_standardized_errors does not map;
a missing organization compiles to ``IS NULL`` and serves a blank project that
has real outputs; and a failed extraction-status write reports success.
Every case here was checked against the unguarded code first — strip the guard
and the case fails.
"""
import json
import secrets
import uuid
from types import SimpleNamespace
from unittest.mock import patch
from account_v2.models import Organization, User
from django.test import TestCase
from prompt_studio.prompt_studio_index_manager_v2.prompt_studio_index_helper import (
ExtractionStatusResult,
)
from prompt_studio.prompt_studio_output_manager_v2.views import PromptStudioOutputView
from rest_framework.test import APIRequestFactory, force_authenticate
from utils.user_context import UserContext
_LIST = PromptStudioOutputView.as_view({"get": "list"})
_DEFAULT_PROFILE = PromptStudioOutputView.as_view({"get": "get_output_for_tool_default"})
_LATEST_BY_KEYS = PromptStudioOutputView.as_view({"get": "latest_outputs_by_keys"})
class OutputViewValidationTest(TestCase):
def setUp(self):
self.addCleanup(UserContext.set_organization_identifier, None)
slug = f"val-{secrets.token_hex(3)}"
self.org = Organization.objects.create(
name=slug, display_name=slug, organization_id=slug
)
UserContext.set_organization_identifier(slug)
self.user = User.objects.create_user(
username=f"{slug}@example.com",
email=f"{slug}@example.com",
password=secrets.token_urlsafe(),
)
def _get(self, view, path, params):
request = APIRequestFactory().get(path, params)
force_authenticate(request, user=self.user)
return view(request)
def test_non_uuid_tool_id_on_default_profile_is_400(self):
response = self._get(
_DEFAULT_PROFILE,
"/prompt-output/prompt-default-profile/",
{"tool_id": "abc", "document_manager": str(uuid.uuid4())},
)
assert response.status_code == 400, response.data
def test_non_uuid_document_manager_on_default_profile_is_400(self):
response = self._get(
_DEFAULT_PROFILE,
"/prompt-output/prompt-default-profile/",
{"tool_id": str(uuid.uuid4()), "document_manager": "abc"},
)
assert response.status_code == 400, response.data
def test_absent_document_manager_on_default_profile_is_400(self):
"""An absent id would filter on NULL, rendering every prompt as "".
A 200 with a blank body is indistinguishable from a project that has
no outputs yet, so the caller has nothing to act on.
"""
response = self._get(
_DEFAULT_PROFILE,
"/prompt-output/prompt-default-profile/",
{"tool_id": str(uuid.uuid4())},
)
assert response.status_code == 400, response.data
def test_non_uuid_tool_id_on_list_is_400(self):
"""The list action is the highest-traffic one on this viewset."""
response = self._get(_LIST, "/prompt-output/", {"tool_id": "abc"})
assert response.status_code == 400, getattr(response, "data", response)
def test_non_uuid_tool_id_on_latest_by_keys_is_400(self):
response = self._get(
_LATEST_BY_KEYS,
"/prompt-output/latest-by-keys/",
{"tool_id": "abc", "prompt_keys": "a"},
)
assert response.status_code == 400, response.data
def test_unresolvable_organization_is_not_served_as_empty(self):
"""Refuse rather than return a blank project that has real outputs."""
UserContext.set_organization_identifier("org-that-does-not-exist")
response = self._get(
_DEFAULT_PROFILE,
"/prompt-output/prompt-default-profile/",
{"tool_id": str(uuid.uuid4()), "document_manager": str(uuid.uuid4())},
)
assert response.status_code >= 500, response.data
class ExtractionStatusEndpointTest(TestCase):
"""The internal callback endpoint must not report a failed write as 200.
The worker never reads the body, so a 200 drops the status silently and
every later Answer Prompt re-runs the full extraction.
"""
URL = "/internal/v1/prompt-studio/extraction-status/"
def _post(self, result):
from prompt_studio.prompt_studio_core_v2 import internal_views
payload = {
"document_id": str(uuid.uuid4()),
"profile_manager_id": str(uuid.uuid4()),
"x2text_config_hash": "hash",
"enable_highlight": False,
"extracted": True,
}
request = APIRequestFactory().post(
self.URL, json.dumps(payload), content_type="application/json"
)
with (
patch.object(
internal_views, "_parse_json_body", return_value=(payload, None)
),
patch(
"prompt_studio.prompt_profile_manager_v2.models.ProfileManager.objects"
) as profiles,
patch(
"prompt_studio.prompt_studio_index_manager_v2.prompt_studio_index_helper"
".PromptStudioIndexHelper.mark_extraction_status",
return_value=result,
),
):
profiles.get.return_value = object()
return internal_views.extraction_status(request)
def test_write_failure_is_a_retryable_500(self):
assert self._post(ExtractionStatusResult.WRITE_FAILED).status_code == 500
def test_missing_document_is_a_non_retryable_404(self):
"""500 is in the client's retry set; a gone document never comes back.
Three retries with a 1s backoff factor would burn ~7s of worker sleep
on a condition no retry can change.
"""
assert self._post(ExtractionStatusResult.DOCUMENT_MISSING).status_code == 404
def test_success_is_200(self):
assert self._post(ExtractionStatusResult.OK).status_code == 200
class InternalCallbackResolutionTest(TestCase):
"""An id the org-scoped managers cannot resolve must not become a 500.
Both handlers resolve ids through managers that are now org-scoped, so a
lookup can miss for two reasons — the row is gone, or the scope hides it —
and neither changes on retry. The worker's client retries {500,502,503,504}
three times with a 1s backoff factor, so a 500 here costs ~7s of worker
sleep and still fails. 404 is outside that set.
"""
def _payload(self, **overrides):
payload = {
"run_id": str(uuid.uuid4()),
"prompt_ids": [str(uuid.uuid4()), str(uuid.uuid4())],
"outputs": {},
"document_id": str(uuid.uuid4()),
"is_single_pass_extract": False,
"profile_manager_id": str(uuid.uuid4()),
"metadata": {},
}
payload.update(overrides)
return payload
# --- prompt_output: a partial resolve is not a success ------------------
@staticmethod
def _prompt(prompt_id=None):
"""Only the attribute the mismatch log reads is needed here."""
return SimpleNamespace(prompt_id=prompt_id or uuid.uuid4())
def _prompt_output(self, resolved, **overrides):
"""Drive prompt_output with ``resolved`` prompts coming back."""
from prompt_studio.prompt_studio_core_v2 import internal_views
payload = self._payload(**overrides)
request = APIRequestFactory().post(
"/internal/v1/prompt-studio/prompt-output/",
json.dumps(payload),
content_type="application/json",
)
with (
patch.object(
internal_views, "_parse_json_body", return_value=(payload, None)
),
patch(
"prompt_studio.prompt_studio_v2.models.ToolStudioPrompt.objects"
) as prompts,
patch(
"prompt_studio.prompt_studio_output_manager_v2.output_manager_helper"
".OutputManagerHelper.handle_prompt_output_update",
return_value={},
) as handler,
):
prompts.filter.return_value.order_by.return_value = resolved
return internal_views.prompt_output(request), handler
def test_prompt_output_refuses_when_the_scope_drops_every_prompt(self):
"""handle_prompt_output_update early-exits on an empty list, so this
used to answer 200 with an empty body and discard the whole run."""
response, handler = self._prompt_output([])
assert response.status_code == 404, response.content
handler.assert_not_called()
def test_prompt_output_refuses_a_partial_resolve(self):
"""One of two prompts resolved is still a run that would be recorded
wrong; the count comparison is what catches it."""
response, handler = self._prompt_output([self._prompt()])
assert response.status_code == 404, response.content
handler.assert_not_called()
def test_prompt_output_proceeds_when_every_prompt_resolves(self):
"""Without this, a handler that refused everything would pass above."""
response, handler = self._prompt_output([self._prompt(), self._prompt()])
assert response.status_code == 200, response.content
handler.assert_called_once()
# --- malformed ids: 400, before the query is built ----------------------
def test_a_malformed_prompt_id_is_400_not_a_retryable_500(self):
"""``filter(prompt_id__in=...)`` raises while the query is *built*.
That is Django's ValidationError, not a DoesNotExist, so it lands in
the generic ``except Exception`` below and returns a 500 — inside the
client's retry set, on a value no retry can change. The resolved-count
check cannot catch it: the query never runs.
"""
response, handler = self._prompt_output(
[self._prompt()], prompt_ids=["not-a-uuid"]
)
assert response.status_code == 400, response.content
handler.assert_not_called()
def test_a_malformed_document_id_is_400_not_a_retryable_500(self):
"""Same failure one field over: handle_prompt_output_update looks up
DocumentManager by pk, so a non-UUID raises there instead."""
response, handler = self._prompt_output(
[self._prompt(), self._prompt()], document_id="not-a-uuid"
)
assert response.status_code == 400, response.content
handler.assert_not_called()
def test_prompt_ids_that_are_not_a_list_is_400(self):
"""A non-sequence would raise TypeError on the parse loop itself,
outside any handler, which is a 500 for a new reason."""
response, handler = self._prompt_output([self._prompt()], prompt_ids=42)
assert response.status_code == 400, response.content
handler.assert_not_called()
def test_well_formed_ids_still_reach_the_helper(self):
"""Without this, a parse that rejected everything would pass above."""
response, handler = self._prompt_output([self._prompt(), self._prompt()])
assert response.status_code == 200, response.content
handler.assert_called_once()
# --- the profile lookups: 404, not the generic 500 ----------------------
def _with_missing_profile(self, handler_name, payload):
from prompt_studio.prompt_profile_manager_v2.models import ProfileManager
from prompt_studio.prompt_studio_core_v2 import internal_views
request = APIRequestFactory().post(
"/internal/v1/prompt-studio/", json.dumps(payload),
content_type="application/json",
)
with (
patch.object(
internal_views, "_parse_json_body", return_value=(payload, None)
),
patch(
"prompt_studio.prompt_profile_manager_v2.models.ProfileManager.objects"
) as profiles,
):
profiles.get.side_effect = ProfileManager.DoesNotExist
return getattr(internal_views, handler_name)(request)
def test_extraction_status_with_an_unresolvable_profile_is_404(self):
response = self._with_missing_profile(
"extraction_status",
{
"document_id": str(uuid.uuid4()),
"profile_manager_id": str(uuid.uuid4()),
"x2text_config_hash": "hash",
"enable_highlight": False,
"extracted": True,
},
)
assert response.status_code == 404, response.content
def test_index_update_with_an_unresolvable_profile_is_404(self):
response = self._with_missing_profile(
"index_update",
{
"document_id": str(uuid.uuid4()),
"profile_manager_id": str(uuid.uuid4()),
"doc_id": "doc-1",
},
)
assert response.status_code == 404, response.content
def test_a_malformed_profile_id_is_404_not_a_retryable_500(self):
"""``get(pk=...)`` raises Django's ValidationError, not DoesNotExist,
for an id that is not a UUID — just as permanent."""
from django.core.exceptions import ValidationError as DjangoValidationError
from prompt_studio.prompt_studio_core_v2 import internal_views
payload = {
"document_id": str(uuid.uuid4()),
"profile_manager_id": "not-a-uuid",
"doc_id": "doc-1",
}
request = APIRequestFactory().post(
"/internal/v1/prompt-studio/", json.dumps(payload),
content_type="application/json",
)
with (
patch.object(
internal_views, "_parse_json_body", return_value=(payload, None)
),
patch(
"prompt_studio.prompt_profile_manager_v2.models.ProfileManager.objects"
) as profiles,
):
profiles.get.side_effect = DjangoValidationError("bad uuid")
response = internal_views.index_update(request)
assert response.status_code == 404, response.content