LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
[FIX] Resolve Prompt Studio default LLM profile and unblock deploy when LLMChallenge is off (#2203)
Hari John Kuriakose892be32714ac
diff --git a/backend/prompt_studio/prompt_studio_core_v2/exceptions.py b/backend/prompt_studio/prompt_studio_core_v2/exceptions.pyindex c4a7ffd20..d387d762a 100644--- a/backend/prompt_studio/prompt_studio_core_v2/exceptions.py+++ b/backend/prompt_studio/prompt_studio_core_v2/exceptions.py@@ -42,10 +42,12 @@ class AnswerFetchError(APIException):class DefaultProfileError(APIException):- status_code = 500+ # A missing default profile is a project-configuration problem the user can+ # fix, not a server fault - 500 misreports it as an outage.+ status_code = 400default_detail = (- "Default LLM profile is not configured."- "Please set an LLM profile as default to continue."+ "No LLM profile could be resolved. Set an LLM profile as the project "+ "default, or attach one to the prompt, to continue.")diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.pyindex c2c51ec34..6d8763e05 100644--- a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py+++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py@@ -713,6 +713,36 @@ class PromptStudioHelper:return context, cb_kwargs+ @staticmethod+ def _resolve_profile_manager(+ tool: Any, prompt: Any = None, profile_manager_id: str | None = None+ ) -> Any:+ """Resolve the profile a run executes under.++ The ladder, in order: an explicitly passed ``profile_manager_id``, then+ the prompt's own FK, then the project default. A prompt need not carry+ its own FK - falling back to the project default matches what+ index_document and single-pass extraction already do.++ ``get_default_llm_profile`` raises ``DefaultProfileError`` when no+ project default exists, so this never returns a falsy value.++ Args:+ tool (CustomTool): Prompt Studio project the prompt belongs to+ prompt (ToolStudioPrompt | None): Prompt whose FK to consult, if any+ profile_manager_id (str | None): Explicitly requested profile++ Returns:+ ProfileManager: The resolved profile+ """+ if profile_manager_id:+ return ProfileManagerHelper.get_profile_manager(+ profile_manager_id=profile_manager_id+ )+ if prompt is not None and prompt.profile_manager:+ return prompt.profile_manager+ return ProfileManager.get_default_llm_profile(tool)+@staticmethoddef _resolve_llm_ids(tool: Any) -> tuple[str, str]:"""Resolve monitor_llm and challenge_llm IDs for the tool."""@@ -766,14 +796,9 @@ class PromptStudioHelper:Returns:(context, cb_kwargs) or (None, pending_response_dict)"""- profile_manager = prompt.profile_manager- if profile_manager_id:- profile_manager = ProfileManagerHelper.get_profile_manager(- profile_manager_id=profile_manager_id- )-- if not profile_manager:- raise DefaultProfileError()+ profile_manager = PromptStudioHelper._resolve_profile_manager(+ tool=tool, prompt=prompt, profile_manager_id=profile_manager_id+ )monitor_llm, challenge_llm = PromptStudioHelper._resolve_llm_ids(tool)@@ -960,7 +985,11 @@ class PromptStudioHelper:"document_id": document_id,"tool_id": tool_id,"prompt_ids": [str(prompt.prompt_id)],- "profile_manager_id": profile_manager_id,+ # Record the profile actually used, not the (possibly None) argument.+ # The callback otherwise re-resolves the project default, so a+ # default change mid-run would book output against a different+ # profile than the one that produced it.+ "profile_manager_id": str(profile_manager.profile_id),"is_single_pass": False,}@@ -989,15 +1018,10 @@ class PromptStudioHelper:Returns:(context, cb_kwargs) or (None, pending_response_dict)"""- profile_manager = (- ProfileManagerHelper.get_profile_manager(profile_manager_id)- if profile_manager_id- else None- )- if not profile_manager:- profile_manager = ProfileManager.get_default_llm_profile(tool)- if not profile_manager:- raise DefaultProfileError()+ # No single prompt to consult here, so the FK rung is skipped.+ profile_manager = PromptStudioHelper._resolve_profile_manager(+ tool=tool, profile_manager_id=profile_manager_id+ )PromptStudioHelper.validate_adapter_status(profile_manager)PromptStudioHelper.validate_profile_manager_owner_access(@@ -1154,7 +1178,9 @@ class PromptStudioHelper:"document_id": document_id,"tool_id": tool_id,"prompt_ids": [str(p.prompt_id) for p in prompts],- "profile_manager_id": profile_manager_id,+ # Record the profile actually used, not the (possibly None)+ # argument - same reason as build_fetch_response_payload above.+ "profile_manager_id": str(profile_manager.profile_id),"is_single_pass": False,}@@ -1701,13 +1727,26 @@ class PromptStudioHelper:user_id=user_id,request_user=request_user,)+ # Book the output against the profile the run actually used, the+ # same ladder _fetch_response applies. Forwarding the raw (possibly+ # None) argument makes _handle_response re-resolve the project+ # default, so a prompt carrying its own FK would run under that FK+ # but have its output stored under the project default.+ resolved_profile_id = str(+ PromptStudioHelper._resolve_profile_manager(+ tool=tool,+ prompt=prompt_instance,+ profile_manager_id=profile_manager_id,+ ).profile_id+ )+return PromptStudioHelper._handle_response(response=response,run_id=run_id,prompts=prompts,document_id=document_id,is_single_pass=False,- profile_manager_id=profile_manager_id,+ profile_manager_id=resolved_profile_id,)except APIException:# Validation responses are user-facing; DRF renders them as-is.@@ -1880,14 +1919,9 @@ class PromptStudioHelper:Any: Output from LLM"""# Fetch the ProfileManager instance using the profile_manager_id if provided- profile_manager = prompt.profile_manager- if profile_manager_id:- profile_manager = ProfileManagerHelper.get_profile_manager(- profile_manager_id=profile_manager_id- )-- if not profile_manager:- raise DefaultProfileError()+ profile_manager = PromptStudioHelper._resolve_profile_manager(+ tool=tool, prompt=prompt, profile_manager_id=profile_manager_id+ )monitor_llm_instance: AdapterInstance | None = tool.monitor_llmmonitor_llm: str | None = Nonediff --git a/backend/prompt_studio/prompt_studio_core_v2/tests/test_profile_resolution_fallback.py b/backend/prompt_studio/prompt_studio_core_v2/tests/test_profile_resolution_fallback.pynew file mode 100644index 000000000..2bbebe736--- /dev/null+++ b/backend/prompt_studio/prompt_studio_core_v2/tests/test_profile_resolution_fallback.py@@ -0,0 +1,500 @@+"""Regression tests for profile resolution in ``fetch_response``.++Pins the fix for the CLI-surfaced bug where a prompt with no+``profile_manager`` FK raised ``DefaultProfileError`` even though the project+had a default LLM profile configured. ``index_document`` and single-pass+extraction already fell back to ``ProfileManager.get_default_llm_profile``;+``fetch_response`` was the lone omission.++The resolution ladder these tests lock down, in order:++ 1. An explicitly passed ``profile_manager_id`` wins over everything.+ 2. Otherwise the prompt's own ``profile_manager`` FK.+ 3. Otherwise the project default -- the rung this PR added.+ 4. If none resolves, ``DefaultProfileError`` propagates.++They also pin the second half of the fix: ``cb_kwargs["profile_manager_id"]``+must carry the profile *actually used*, not the (possibly ``None``) argument.+``OutputManagerHelper.get_default_profile`` treats ``None`` as "re-resolve the+project default", so passing the raw argument books output against a different+profile than the one that produced it.++These call the real ``build_fetch_response_payload`` /+``build_bulk_fetch_response_payload`` and patch only their collaborators, so+the assertions cover the shipped path end to end -- including everything after+the ladder.++An earlier version of this file extracted the ladder's source text and+``exec``'d it. It was replaced because it could not fail on the regressions it+existed to catch: it stayed green when the resolved profile was clobbered+immediately after the ladder, and when the ``cb_kwargs`` fix was reverted at+both call sites. Please do not reintroduce source-text assertions here.+"""++from __future__ import annotations++import os+import uuid+from contextlib import ExitStack+from typing import Any+from unittest.mock import MagicMock, patch++import pytest+++def _deps():+ """Import the Django-coupled helper at call time, not import time.++ Skips only when Django itself is unavailable; configuration and application+ import failures propagate so a real regression is never hidden as a skip.+ """+ try:+ import django+ except ImportError as exc: # pragma: no cover - environment guard+ pytest.skip(f"django is not installed: {exc}")++ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings.test")+ django.setup()++ from prompt_studio.prompt_studio_core_v2 import prompt_studio_helper as psh+ from prompt_studio.prompt_studio_core_v2.exceptions import DefaultProfileError++ return psh, DefaultProfileError+++EXPLICIT_ID = "11111111-1111-1111-1111-111111111111"+PROMPT_FK_ID = "22222222-2222-2222-2222-222222222222"+PROJECT_DEFAULT_ID = "33333333-3333-3333-3333-333333333333"++# build_bulk_fetch_response_payload takes a prompt list and never consults a+# single prompt's FK, so only the FK-precedence rung is scoped to the builder+# that reads one. Both builders carry the cb_kwargs fix.+FK_AWARE_BUILDERS = ["build_fetch_response_payload"]+ALL_BUILDERS = ["build_fetch_response_payload", "build_bulk_fetch_response_payload"]+++def _make_profile(profile_id: str) -> Any:+ """A ProfileManager stand-in with the attributes the builders read.++ ``profile_id`` is a real ``UUID``, matching the ``UUIDField`` on the model.+ That is load-bearing: ``cb_kwargs`` is JSON-serialised by Celery+ (``task_serializer = "json"``), so dropping the ``str()`` wrapper in the+ helper would raise at dispatch. A ``str`` fake here would hide that.+ """+ profile = MagicMock(name=f"ProfileManager<{profile_id}>")+ profile.profile_id = uuid.UUID(profile_id)+ profile.chunk_size = 512+ profile.chunk_overlap = 64+ return profile+++def _make_prompt(profile: Any) -> Any:+ prompt = MagicMock(name="ToolStudioPrompt")+ prompt.profile_manager = profile+ prompt.prompt_id = "prompt-1"+ return prompt+++def _call(+ psh: Any,+ builder: str,+ *,+ profile_manager_id: str | None,+ prompt_profile: Any,+ default_profile: Any,+ default_raises: BaseException | None = None,+) -> tuple[Any, dict[str, Any]]:+ """Invoke a real payload builder with every collaborator patched out.++ Only profile resolution stays real; everything downstream is stubbed so the+ call reaches ``cb_kwargs`` without touching storage, indexing or an LLM.++ ``get_profile_manager`` returns the explicit profile only when handed the+ ID under test, so the rung-1 assertion cannot pass vacuously.+ """+ helper = psh.PromptStudioHelper+ prompt = _make_prompt(prompt_profile)++ def _get_default(tool: Any) -> Any:+ if default_raises is not None:+ raise default_raises+ return default_profile++ def _get_explicit(profile_manager_id: str) -> Any:+ assert profile_manager_id == EXPLICIT_ID, (+ f"builder forwarded {profile_manager_id!r} to get_profile_manager, "+ f"expected {EXPLICIT_ID!r}"+ )+ return _make_profile(EXPLICIT_ID)++ # autospec on the collaborators whose signatures the builders call into, so+ # an arity or keyword change in production fails here instead of staying+ # green. The module-attribute patches below are legitimately opaque.+ patches = [+ patch.object(+ psh.ProfileManager,+ "get_default_llm_profile",+ autospec=True,+ side_effect=lambda tool: _get_default(tool),+ ),+ patch.object(+ psh.ProfileManagerHelper,+ "get_profile_manager",+ autospec=True,+ side_effect=lambda profile_manager_id: _get_explicit(profile_manager_id),+ ),+ patch.object(helper, "_resolve_llm_ids", autospec=True, return_value=("m", "c")),+ patch.object(helper, "validate_adapter_status", autospec=True),+ patch.object(helper, "validate_profile_manager_owner_access", autospec=True),+ patch.object(helper, "_get_platform_api_key", autospec=True, return_value="pk"),+ patch.object(helper, "dynamic_extractor", autospec=True, return_value="text"),+ # Must be a dict with a non-pending status: the builders early-return a+ # pending response (and no cb_kwargs) when indexing is still running.+ patch.object(+ helper, "dynamic_indexer", autospec=True, return_value={"status": "COMPLETED"}+ ),+ patch.object(helper, "_build_grammar_list", autospec=True, return_value=[]),+ patch.object(helper, "_build_prompt_output", autospec=True, return_value={}),+ # Returns the (mutated) output dict, so it must pass it through.+ patch.object(+ helper,+ "fetch_table_settings_if_enabled",+ autospec=True,+ side_effect=lambda _doc, _prompt, _org, _user, _tool, output: output,+ ),+ patch.object(psh, "EnvHelper", MagicMock()),+ patch.object(psh, "PromptIdeBaseTool", MagicMock()),+ patch.object(psh, "IndexingUtils", MagicMock()),+ patch.object(psh, "StateStore", MagicMock()),+ patch.object(psh, "ExecutionContext", MagicMock()),+ patch.object(psh, "PromptStudioVariableService", MagicMock()),+ ]+ # ExitStack, not a bare start loop: a mistargeted patch would otherwise+ # raise partway through and leak the already-started ones into later tests.+ with ExitStack() as stack:+ for patcher in patches:+ stack.enter_context(patcher)+ kwargs: dict[str, Any] = {+ "tool": MagicMock(+ tool_id="tool-1",+ summarize_as_source=False,+ enable_highlight=False,+ enable_challenge=False,+ ),+ "doc_path": "/docs/a.pdf",+ "doc_name": "a.pdf",+ "org_id": "org-1",+ "user_id": "user-1",+ "document_id": "doc-1",+ "run_id": "run-1",+ "profile_manager_id": profile_manager_id,+ "request_user": None,+ }+ if builder == "build_bulk_fetch_response_payload":+ kwargs["prompts"] = [prompt]+ else:+ kwargs["prompt"] = prompt+ return getattr(helper, builder)(**kwargs)++[email protected]("builder", FK_AWARE_BUILDERS)+class TestProfileResolutionLadder:+ def test_prompt_without_profile_uses_project_default(self, builder: str) -> None:+ """The reported bug: no FK + a project default must resolve, not raise."""+ psh, _ = _deps()++ _context, cb_kwargs = _call(+ psh,+ builder,+ profile_manager_id=None,+ prompt_profile=None,+ default_profile=_make_profile(PROJECT_DEFAULT_ID),+ )++ assert cb_kwargs["profile_manager_id"] == PROJECT_DEFAULT_ID, (+ "A prompt with no profile FK must fall back to the project default; "+ "this raised DefaultProfileError before the fix"+ )++ def test_prompt_fk_wins_over_project_default(self, builder: str) -> None:+ """The fallback must not override a profile the prompt already has."""+ psh, _ = _deps()++ _context, cb_kwargs = _call(+ psh,+ builder,+ profile_manager_id=None,+ prompt_profile=_make_profile(PROMPT_FK_ID),+ default_profile=_make_profile(PROJECT_DEFAULT_ID),+ )++ assert cb_kwargs["profile_manager_id"] == PROMPT_FK_ID++ def test_explicit_id_wins_over_everything(self, builder: str) -> None:+ psh, _ = _deps()++ _context, cb_kwargs = _call(+ psh,+ builder,+ profile_manager_id=EXPLICIT_ID,+ prompt_profile=_make_profile(PROMPT_FK_ID),+ default_profile=_make_profile(PROJECT_DEFAULT_ID),+ )++ assert cb_kwargs["profile_manager_id"] == EXPLICIT_ID++[email protected]("builder", ALL_BUILDERS)+class TestResolvedProfileReachesCallback:+ """``cb_kwargs`` must carry the profile used, not the raw argument.++ Passing the argument through lets the callback re-resolve the project+ default (``None`` is its "re-resolve" sentinel), booking output against a+ different profile than the one that produced it. Reverting the fix at+ either call site must fail here.+ """++ def test_callback_records_the_resolved_profile(self, builder: str) -> None:+ psh, _ = _deps()++ _context, cb_kwargs = _call(+ psh,+ builder,+ profile_manager_id=None,+ prompt_profile=None,+ default_profile=_make_profile(PROJECT_DEFAULT_ID),+ )++ assert cb_kwargs["profile_manager_id"] == PROJECT_DEFAULT_ID, (+ "cb_kwargs must record the resolved profile; the raw None argument "+ "makes the callback silently re-resolve the project default"+ )++[email protected]("builder", ALL_BUILDERS)+class TestUnresolvableProfile:+ def test_no_profile_anywhere_propagates_default_profile_error(+ self, builder: str+ ) -> None:+ """The fallback must not swallow a genuinely unconfigured project."""+ psh, default_profile_error = _deps()++ with pytest.raises(default_profile_error):+ _call(+ psh,+ builder,+ profile_manager_id=None,+ prompt_profile=None,+ default_profile=None,+ default_raises=default_profile_error(),+ )+++class TestSynchronousFetchResponse:+ """``_fetch_response`` carries the same fallback and is still live.++ It is reached from ``fetch_response`` (the non-dispatch branch) and returns+ a service response rather than ``cb_kwargs``, so the resolved profile is+ observed where it is first consumed: ``validate_adapter_status``.+ """++ def _resolved_profile(+ self,+ psh: Any,+ *,+ profile_manager_id: str | None,+ prompt_profile: Any,+ default_profile: Any,+ ) -> Any:+ helper = psh.PromptStudioHelper+ seen: list[Any] = []++ def _get_explicit(profile_manager_id: str) -> Any:+ assert profile_manager_id == EXPLICIT_ID+ return _make_profile(EXPLICIT_ID)++ patches = [+ patch.object(+ psh.ProfileManager,+ "get_default_llm_profile",+ autospec=True,+ side_effect=lambda tool: default_profile,+ ),+ patch.object(+ psh.ProfileManagerHelper,+ "get_profile_manager",+ autospec=True,+ side_effect=lambda profile_manager_id: _get_explicit(profile_manager_id),+ ),+ patch.object(+ helper,+ "validate_adapter_status",+ autospec=True,+ side_effect=seen.append,+ ),+ patch.object(+ helper, "validate_profile_manager_owner_access", autospec=True+ ),+ patch.object(+ helper, "_resolve_llm_ids", autospec=True, return_value=("m", "c")+ ),+ # Stop the call once the profile has been observed; everything past+ # this point needs storage and an LLM.+ patch.object(psh, "EnvHelper", MagicMock(side_effect=RuntimeError)),+ ]+ with ExitStack() as stack:+ for patcher in patches:+ stack.enter_context(patcher)+ with pytest.raises(Exception): # noqa: B017 - halted deliberately+ helper._fetch_response(+ tool=MagicMock(tool_id="tool-1", summarize_as_source=False),+ doc_path="/docs/a.pdf",+ doc_name="a.pdf",+ prompt=_make_prompt(prompt_profile),+ org_id="org-1",+ document_id="doc-1",+ run_id="run-1",+ user_id="user-1",+ profile_manager_id=profile_manager_id,+ request_user=None,+ )++ assert seen, "validate_adapter_status was never reached"+ return seen[0]++ def test_prompt_without_profile_uses_project_default(self) -> None:+ """The reported bug, on the synchronous path."""+ psh, _ = _deps()++ resolved = self._resolved_profile(+ psh,+ profile_manager_id=None,+ prompt_profile=None,+ default_profile=_make_profile(PROJECT_DEFAULT_ID),+ )++ assert str(resolved.profile_id) == PROJECT_DEFAULT_ID++ def test_prompt_fk_wins_over_project_default(self) -> None:+ psh, _ = _deps()++ resolved = self._resolved_profile(+ psh,+ profile_manager_id=None,+ prompt_profile=_make_profile(PROMPT_FK_ID),+ default_profile=_make_profile(PROJECT_DEFAULT_ID),+ )++ assert str(resolved.profile_id) == PROMPT_FK_ID+++class TestSynchronousOutputAttribution:+ """``_execute_single_prompt`` must book output against the profile used.++ The async builders record the resolved profile in ``cb_kwargs``; the+ synchronous path hands ``profile_manager_id`` to ``_handle_response``, whose+ ``None`` branch re-resolves the project default. A prompt carrying its own+ FK would then run under the FK but store its output under the project+ default.+ """++ def _handled_profile_id(+ self, psh: Any, *, profile_manager_id: str | None, prompt_profile: Any+ ) -> Any:+ helper = psh.PromptStudioHelper+ prompt = _make_prompt(prompt_profile)+ prompt.enforce_type = "Text"+ prompt.tool_id = MagicMock(tool_id="tool-1")+ seen: dict[str, Any] = {}++ patches = [+ patch.object(+ helper, "_fetch_prompt_from_id", autospec=True, return_value=prompt+ ),+ patch.object(+ helper, "_fetch_response", autospec=True, return_value={"status": "OK"}+ ),+ patch.object(helper, "_publish_log", autospec=True),+ patch.object(+ psh.ProfileManager,+ "get_default_llm_profile",+ autospec=True,+ side_effect=lambda tool: _make_profile(PROJECT_DEFAULT_ID),+ ),+ patch.object(+ psh.ProfileManagerHelper,+ "get_profile_manager",+ autospec=True,+ side_effect=lambda profile_manager_id: _make_profile(profile_manager_id),+ ),+ patch.object(+ helper,+ "_handle_response",+ autospec=True,+ side_effect=lambda **kw: seen.update(kw),+ ),+ patch.object(psh, "get_plugin", MagicMock(return_value=None)),+ ]+ with ExitStack() as stack:+ for patcher in patches:+ stack.enter_context(patcher)+ helper._execute_single_prompt(+ id="prompt-1",+ doc_path="/docs/a.pdf",+ doc_name="a.pdf",+ tool_id="tool-1",+ org_id="org-1",+ user_id="user-1",+ document_id="doc-1",+ run_id="run-1",+ profile_manager_id=profile_manager_id,+ request_user=None,+ )++ assert seen, "_handle_response was never reached"+ return seen["profile_manager_id"]++ def test_prompt_fk_is_recorded_not_reresolved(self) -> None:+ psh, _ = _deps()++ recorded = self._handled_profile_id(+ psh, profile_manager_id=None, prompt_profile=_make_profile(PROMPT_FK_ID)+ )++ assert recorded == PROMPT_FK_ID, (+ "Output must be booked against the prompt's own profile; passing "+ "None makes _handle_response re-resolve the project default"+ )++ def test_project_default_is_recorded_when_no_fk(self) -> None:+ psh, _ = _deps()++ recorded = self._handled_profile_id(+ psh, profile_manager_id=None, prompt_profile=None+ )++ assert recorded == PROJECT_DEFAULT_ID++ def test_explicit_id_round_trips(self) -> None:+ """The explicit rung is a pass-through, but must stay one."""+ psh, _ = _deps()++ recorded = self._handled_profile_id(+ psh,+ profile_manager_id=EXPLICIT_ID,+ prompt_profile=_make_profile(PROMPT_FK_ID),+ )++ assert recorded == EXPLICIT_ID+++def test_default_profile_error_is_a_client_error() -> None:+ """A missing default profile is user-fixable configuration, not a 500."""+ _psh, default_profile_error = _deps()++ assert default_profile_error.status_code == 400+ detail = str(default_profile_error.default_detail).lower()+ assert "project" in detail and "prompt" in detail, (+ "The message must name both places a profile can come from; it is also "+ "raised for monitor/challenge resolution, not just the prompt"+ )diff --git a/backend/prompt_studio/prompt_studio_core_v2/tests/test_task_status.py b/backend/prompt_studio/prompt_studio_core_v2/tests/test_task_status.pyindex d7ee67c22..ab0754ee1 100644--- a/backend/prompt_studio/prompt_studio_core_v2/tests/test_task_status.py+++ b/backend/prompt_studio/prompt_studio_core_v2/tests/test_task_status.py@@ -84,14 +84,24 @@ class TestTaskStatusFlagOff:def test_celery_completed_never_queries_pg(self):# flag off (also the Flipt-error fail-closed path): Celery + PG untouched.p_pg, pg = _stub_pg({"status": "completed"}) # would resolve, if queried- with _org(), _flag(False), p_pg, patch(f"{_VIEWS}.AsyncResult", _async(ready=True)):+ with (+ _org(),+ _flag(False),+ p_pg,+ patch(f"{_VIEWS}.AsyncResult", _async(ready=True)),+ ):resp = _view().task_status(MagicMock(), task_id=_TID)assert resp.data == {"task_id": _TID, "status": "completed"}pg.objects.filter.assert_not_called() # default path never touches PGdef test_celery_failed(self):- with _org(), _flag(False), patch(- f"{_VIEWS}.AsyncResult", _async(ready=True, successful=False, result="celery boom")+ with (+ _org(),+ _flag(False),+ patch(+ f"{_VIEWS}.AsyncResult",+ _async(ready=True, successful=False, result="celery boom"),+ ),):resp = _view().task_status(MagicMock(), task_id=_TID)assert resp.data["status"] == "failed"diff --git a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.pyindex fac9acd7d..f514eda95 100644--- a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py+++ b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py@@ -38,6 +38,23 @@ class PromptStudioRegistryHelper:with the help of a proto tool."""+ @staticmethod+ def _resolve_challenge_llm_id(tool: CustomTool) -> str:+ """Resolve the adapter ID to seed as challenge_llm's spec default.++ Mirrors the precedence frame_export_json already uses: the tool's own+ challenge_llm FK, else the project default profile's LLM.++ Args:+ tool (CustomTool): Saved tool data++ Returns:+ str: Adapter instance ID of the challenger LLM+ """+ if tool.challenge_llm:+ return str(tool.challenge_llm.id)+ return str(ProfileManager.get_default_llm_profile(tool).llm.id)+@staticmethoddef frame_spec(tool: CustomTool) -> Spec:"""Method to return spec of the Custom tool.@@ -48,14 +65,31 @@ class PromptStudioRegistryHelper:Returns:dict: spec dict"""+ # A "type": "string" property with no spec default is seeded into tool+ # instance metadata as "" (ToolUtils.get_default_settings). Because+ # challenge_llm declares adapterType, the instance schema carries an+ # enum of real adapter IDs, which "" can never satisfy - so deployment+ # validation rejected the freshly seeded instance, regardless of whether+ # LLMChallenge was enabled (the seeding never consulted that flag).+ #+ # Seed the resolved LLM instead of leaving it empty. adapterType stays+ # declared unconditionally: it is the sole discriminator in+ # Spec.get_adapter_properties() (tool-registry dto.py), so dropping it+ # would take challenge_llm out of every adapter-aware path - the+ # challenge_llm_adapter_id writes that runtime actually reads+ # (structure_tool_task.py), the settings-form adapter dropdown, and the+ # lazy adapter-id migration check.+ challenge_llm_property: dict[str, Any] = {+ "type": "string",+ "title": "Challenger LLM",+ "adapterType": "LLM",+ "description": "LLM to use for LLMChallenge",+ "adapterIdKey": "challenge_llm_adapter_id",+ "default": PromptStudioRegistryHelper._resolve_challenge_llm_id(tool),+ }+properties = {- "challenge_llm": {- "type": "string",- "title": "Challenger LLM",- "adapterType": "LLM",- "description": "LLM to use for LLMChallenge",- "adapterIdKey": "challenge_llm_adapter_id",- },+ "challenge_llm": challenge_llm_property,"enable_challenge": {"type": "boolean","title": "Enable LLMChallenge",diff --git a/backend/prompt_studio/prompt_studio_registry_v2/tests/test_frame_spec_challenge_llm.py b/backend/prompt_studio/prompt_studio_registry_v2/tests/test_frame_spec_challenge_llm.pynew file mode 100644index 000000000..eb0b1e559--- /dev/null+++ b/backend/prompt_studio/prompt_studio_registry_v2/tests/test_frame_spec_challenge_llm.py@@ -0,0 +1,249 @@+"""Regression tests for ``PromptStudioRegistryHelper.frame_spec``'s handling of+``challenge_llm``.++Pins the fix for the deploy-time failure where an exported tool ended in+pipeline status ERROR with ``422 Unprocessable Entity: Tool validation failed``.++The mechanism these tests lock down:++ 1. ``ToolUtils.get_default_settings`` seeds each spec property into tool+ instance metadata using the spec ``default`` when present, else the zero+ value for the declared type -- ``""`` for a string.+ 2. A property declaring ``adapterType: "LLM"`` gets an ``enum`` of real+ adapter IDs injected by ``_update_schema_for_adapter_type``.+ 3. ``""`` is not in that enum, so deployment validation rejected it.++The fix seeds ``challenge_llm`` with a resolved adapter ID rather than dropping+its ``adapterType``. That distinction is load-bearing and is asserted directly+here: ``adapterType`` is the sole discriminator in ``Spec.get_adapter_properties``+(tool-registry ``dto.py``), so removing it would silently unregister+``challenge_llm`` from every adapter-aware path -- the ``challenge_llm_adapter_id``+write that runtime reads, the settings-form dropdown, and the lazy adapter-id+migration check. ``test_adapter_type_is_always_declared`` fails loudly if a+future change pops it again.++These tests exercise the real ``frame_spec`` and the real+``_update_schema_for_adapter_type``, patching only the two DB lookups (the+default profile and the adapter list). Nothing is stubbed into ``sys.modules``:+that pattern shadows the real ``unstract.*`` packages for every later test in the+pytest process, and the repo has been removing it (see the execution-serializer+stub leak dropped in ``b60dd6f3``).++Every real import happens inside ``_deps()`` rather than at module scope -- see+its docstring for why.+"""++from __future__ import annotations++import os+from types import SimpleNamespace+from typing import Any+from unittest.mock import MagicMock, patch++import jsonschema+import pytest+++def _deps():+ """Import everything this module needs, at call time rather than import time.++ Nothing is imported at module scope on purpose. ``frame_spec`` and+ ``ToolProcessor`` need a configured Django app registry, so importing them+ at module scope makes *collection* of this file depend on settings being+ configured -- and a collection error takes the whole file down rather than+ failing one test. Importing at call time keeps that failure local.++ ``django`` genuinely missing is an environment gap -> skip. Anything else+ fails loudly rather than skipping, so a genuine import breakage is reported+ instead of hidden. (This file previously resolved ``unstract.*`` to mocks+ when a sibling module stubbed ``sys.modules`` during collection; that+ stubbing is gone, and failing loudly here keeps it gone.)+ """+ try:+ import django+ except ImportError as exc: # pragma: no cover - environment guard+ pytest.skip(f"django is not installed: {exc}")++ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings.test")+ django.setup()++ from tool_instance_v2.tool_processor import ToolProcessor+ from unstract.sdk1.constants import AdapterTypes+ from unstract.tool_registry.dto import Spec+ from unstract.tool_registry.tool_utils import ToolUtils++ from prompt_studio.prompt_studio_registry_v2.prompt_studio_registry_helper import (+ PromptStudioRegistryHelper,+ )++ return SimpleNamespace(+ helper=PromptStudioRegistryHelper,+ tool_processor=ToolProcessor,+ AdapterTypes=AdapterTypes,+ Spec=Spec,+ ToolUtils=ToolUtils,+ )+++REAL_ADAPTER_ID = "11111111-2222-3333-4444-555555555555"+OTHER_ADAPTER_ID = "99999999-8888-7777-6666-555555555555"++PROFILE_LOOKUP = (+ "prompt_studio.prompt_studio_registry_v2.prompt_studio_registry_helper"+ ".ProfileManager.get_default_llm_profile"+)+ADAPTER_LOOKUP = "tool_instance_v2.tool_processor.AdapterProcessor.get_adapters_by_type"+++def _make_tool(*, enable_challenge: bool, challenge_llm_id: str | None = None):+ """Minimal stand-in for ``CustomTool`` -- ``frame_spec`` reads only these."""+ tool = MagicMock(name="CustomTool")+ tool.tool_id = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"+ tool.description = "A tool exported from Prompt Studio"+ tool.enable_challenge = enable_challenge+ tool.challenge_llm = (+ None if challenge_llm_id is None else MagicMock(id=challenge_llm_id)+ )+ return tool+++def _default_profile(llm_id: str = REAL_ADAPTER_ID):+ return MagicMock(llm=MagicMock(id=llm_id))+++def _frame_spec(tool: Any) -> Any:+ deps = _deps()+ with patch(PROFILE_LOOKUP, return_value=_default_profile()):+ return deps.helper.frame_spec(tool=tool)+++def _build_instance_schema(tool: Any) -> dict[str, Any]:+ """Compose export-time ``frame_spec`` with deploy-time enum injection.++ This is the path a tool instance's metadata is validated against:+ ``frame_spec`` output is persisted as ``tool_spec`` and later rehydrated+ via ``Spec.from_dict`` in ``get_tool_by_prompt_registry_id``.+ """+ deps = _deps()+ spec = _frame_spec(tool)+ adapters = [MagicMock(id=REAL_ADAPTER_ID, adapter_name="My LLM")]+ with patch(ADAPTER_LOOKUP, return_value=adapters):+ deps.tool_processor._update_schema_for_adapter_type(+ spec, spec.get_llm_adapter_properties_keys(), deps.AdapterTypes.LLM, None+ )+ return spec.to_dict()+++def _seeded_metadata(spec_dict: dict[str, Any]) -> dict[str, Any]:+ """Seed instance metadata the way tool-instance creation really does.++ Calls the real ``ToolUtils.get_default_settings`` rather than reimplementing+ its type->zero-value table, so a change to the seeding rule surfaces here.+ """+ deps = _deps()+ tool = MagicMock()+ tool.spec = deps.Spec.from_dict(spec_dict)+ return deps.ToolUtils.get_default_settings(tool)+++def _validation_errors(schema: dict[str, Any], instance: dict[str, Any]) -> list[Any]:+ return list(jsonschema.Draft7Validator(schema).iter_errors(instance))++[email protected]("enable_challenge", [False, True])+class TestSeededInstanceDeploys:+ """A freshly created tool instance must pass deployment validation.++ This is the reported bug, and it fired regardless of ``enable_challenge``.+ """++ def test_seeded_metadata_validates(self, enable_challenge: bool) -> None:+ schema = _build_instance_schema(_make_tool(enable_challenge=enable_challenge))++ errors = _validation_errors(schema, _seeded_metadata(schema))++ assert errors == [], (+ "Metadata seeded by get_default_settings must pass deployment "+ f"validation; got {[e.validator for e in errors]}"+ )++ def test_seeded_challenge_llm_is_a_real_adapter_id(+ self, enable_challenge: bool+ ) -> None:+ """The seeded value must be usable, not merely valid."""+ schema = _build_instance_schema(_make_tool(enable_challenge=enable_challenge))++ assert _seeded_metadata(schema)["challenge_llm"] == REAL_ADAPTER_ID++ def test_adapter_type_is_always_declared(self, enable_challenge: bool) -> None:+ """``adapterType`` gates every adapter-aware path -- never drop it.++ Without it ``challenge_llm`` leaves ``get_llm_adapter_properties()``, so+ ``challenge_llm_adapter_id`` is never written and the settings form+ loses its adapter dropdown.+ """+ spec = _frame_spec(_make_tool(enable_challenge=enable_challenge))++ assert "challenge_llm" in spec.get_llm_adapter_properties_keys()+ assert "challenge_llm" in spec.get_llm_adapter_properties()++ def test_dropdown_metadata_is_present(self, enable_challenge: bool) -> None:+ """The settings form renders its dropdown from enum/enumNames."""+ schema = _build_instance_schema(_make_tool(enable_challenge=enable_challenge))+ prop = schema["properties"]["challenge_llm"]++ assert prop["enum"] == [REAL_ADAPTER_ID]+ assert prop["enumNames"] == ["My LLM"]++ def test_challenge_llm_stays_required(self, enable_challenge: bool) -> None:+ """Always seeded, so requiredness costs nothing and keeps the contract."""+ schema = _build_instance_schema(_make_tool(enable_challenge=enable_challenge))++ assert "challenge_llm" in schema["required"]+++class TestChallengeLlmResolution:+ """``frame_spec`` seeds the same LLM the export path resolves."""++ def test_tool_challenge_llm_takes_precedence(self) -> None:+ deps = _deps()+ tool = _make_tool(enable_challenge=True, challenge_llm_id=OTHER_ADAPTER_ID)+ with patch(PROFILE_LOOKUP, return_value=_default_profile()) as get_default:+ spec = deps.helper.frame_spec(tool=tool)++ assert spec.properties["challenge_llm"]["default"] == OTHER_ADAPTER_ID+ get_default.assert_not_called()++ def test_falls_back_to_default_profile_llm(self) -> None:+ spec = _frame_spec(_make_tool(enable_challenge=False))++ assert spec.properties["challenge_llm"]["default"] == REAL_ADAPTER_ID+++class TestInvalidValuesStillRejected:+ """The fix must not open a hole: a bad adapter must still fail."""++ @pytest.mark.parametrize("enable_challenge", [False, True])+ def test_empty_value_is_rejected(self, enable_challenge: bool) -> None:+ """A stored ``""`` -- the pre-fix state -- must not validate."""+ schema = _build_instance_schema(_make_tool(enable_challenge=enable_challenge))++ errors = _validation_errors(schema, {"challenge_llm": ""})++ assert any(+ e.validator == "enum" for e in errors+ ), f"Expected an enum violation, got {[e.validator for e in errors]}"++ def test_unknown_adapter_id_is_rejected(self) -> None:+ schema = _build_instance_schema(_make_tool(enable_challenge=True))++ errors = _validation_errors(schema, {"challenge_llm": OTHER_ADAPTER_ID})++ assert any(e.validator == "enum" for e in errors)++ def test_missing_key_is_rejected(self) -> None:+ schema = _build_instance_schema(_make_tool(enable_challenge=True))++ errors = _validation_errors(schema, {})++ assert any(e.validator == "required" for e in errors)diff --git a/backend/pyproject.toml b/backend/pyproject.tomlindex ce01979b3..7eee250b9 100644--- a/backend/pyproject.toml+++ b/backend/pyproject.toml@@ -71,9 +71,13 @@ dev = ["responses>=0.25.7","psutil>=7.0.0",]+# jsonschema also arrives via unstract-tool-registry, but the schema tests+# import it directly - declare it so the suite fails loudly rather than+# skipping if that transitive path ever changes.test = ["pytest>=8.0.1","pytest-django>=4.12.0",+ "jsonschema>=4.18.6,<5.0",]deploy = ["gunicorn~=23.0", # For serving the applicationdiff --git a/backend/uv.lock b/backend/uv.lockindex a68e8aee7..c80f0ecea 100644--- a/backend/uv.lock+++ b/backend/uv.lock@@ -3732,6 +3732,7 @@ dev = [{ name = "unstract-workflow-execution" },]test = [+ { name = "jsonschema" },{ name = "pytest" },{ name = "pytest-django" },]@@ -3798,6 +3799,7 @@ dev = [{ name = "unstract-workflow-execution", editable = "../unstract/workflow-execution" },]test = [+ { name = "jsonschema", specifier = ">=4.18.6,<5.0" },{ name = "pytest", specifier = ">=8.0.1" },{ name = "pytest-django", specifier = ">=4.12.0" },]