Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

[FIX] Resolve Prompt Studio default LLM profile and unblock deploy when LLMChallenge is off (#2203)

Hari John Kuriakose892be32714ac
diff --git a/backend/prompt_studio/prompt_studio_core_v2/exceptions.py b/backend/prompt_studio/prompt_studio_core_v2/exceptions.py
index c4a7ffd20..d387d762a 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/exceptions.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/exceptions.py
@@ -42,10 +42,12 @@ class AnswerFetchError(APIException):
class DefaultProfileError(APIException):
- status_code = 500
+ # A missing default profile is a project-configuration problem the user can
+ # fix, not a server fault - 500 misreports it as an outage.
+ status_code = 400
default_detail = (
- "Default LLM profile is not configured."
- "Please set an LLM profile as default to continue."
+ "No LLM profile could be resolved. Set an LLM profile as the project "
+ "default, or attach one to the prompt, to continue."
)
diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
index c2c51ec34..6d8763e05 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
@@ -713,6 +713,36 @@ class PromptStudioHelper:
return context, cb_kwargs
+ @staticmethod
+ def _resolve_profile_manager(
+ tool: Any, prompt: Any = None, profile_manager_id: str | None = None
+ ) -> Any:
+ """Resolve the profile a run executes under.
+
+ The ladder, in order: an explicitly passed ``profile_manager_id``, then
+ the prompt's own FK, then the project default. A prompt need not carry
+ its own FK - falling back to the project default matches what
+ index_document and single-pass extraction already do.
+
+ ``get_default_llm_profile`` raises ``DefaultProfileError`` when no
+ project default exists, so this never returns a falsy value.
+
+ Args:
+ tool (CustomTool): Prompt Studio project the prompt belongs to
+ prompt (ToolStudioPrompt | None): Prompt whose FK to consult, if any
+ profile_manager_id (str | None): Explicitly requested profile
+
+ Returns:
+ ProfileManager: The resolved profile
+ """
+ if profile_manager_id:
+ return ProfileManagerHelper.get_profile_manager(
+ profile_manager_id=profile_manager_id
+ )
+ if prompt is not None and prompt.profile_manager:
+ return prompt.profile_manager
+ return ProfileManager.get_default_llm_profile(tool)
+
@staticmethod
def _resolve_llm_ids(tool: Any) -> tuple[str, str]:
"""Resolve monitor_llm and challenge_llm IDs for the tool."""
@@ -766,14 +796,9 @@ class PromptStudioHelper:
Returns:
(context, cb_kwargs) or (None, pending_response_dict)
"""
- profile_manager = prompt.profile_manager
- if profile_manager_id:
- profile_manager = ProfileManagerHelper.get_profile_manager(
- profile_manager_id=profile_manager_id
- )
-
- if not profile_manager:
- raise DefaultProfileError()
+ profile_manager = PromptStudioHelper._resolve_profile_manager(
+ tool=tool, prompt=prompt, profile_manager_id=profile_manager_id
+ )
monitor_llm, challenge_llm = PromptStudioHelper._resolve_llm_ids(tool)
@@ -960,7 +985,11 @@ class PromptStudioHelper:
"document_id": document_id,
"tool_id": tool_id,
"prompt_ids": [str(prompt.prompt_id)],
- "profile_manager_id": profile_manager_id,
+ # Record the profile actually used, not the (possibly None) argument.
+ # The callback otherwise re-resolves the project default, so a
+ # default change mid-run would book output against a different
+ # profile than the one that produced it.
+ "profile_manager_id": str(profile_manager.profile_id),
"is_single_pass": False,
}
@@ -989,15 +1018,10 @@ class PromptStudioHelper:
Returns:
(context, cb_kwargs) or (None, pending_response_dict)
"""
- profile_manager = (
- ProfileManagerHelper.get_profile_manager(profile_manager_id)
- if profile_manager_id
- else None
- )
- if not profile_manager:
- profile_manager = ProfileManager.get_default_llm_profile(tool)
- if not profile_manager:
- raise DefaultProfileError()
+ # No single prompt to consult here, so the FK rung is skipped.
+ profile_manager = PromptStudioHelper._resolve_profile_manager(
+ tool=tool, profile_manager_id=profile_manager_id
+ )
PromptStudioHelper.validate_adapter_status(profile_manager)
PromptStudioHelper.validate_profile_manager_owner_access(
@@ -1154,7 +1178,9 @@ class PromptStudioHelper:
"document_id": document_id,
"tool_id": tool_id,
"prompt_ids": [str(p.prompt_id) for p in prompts],
- "profile_manager_id": profile_manager_id,
+ # Record the profile actually used, not the (possibly None)
+ # argument - same reason as build_fetch_response_payload above.
+ "profile_manager_id": str(profile_manager.profile_id),
"is_single_pass": False,
}
@@ -1701,13 +1727,26 @@ class PromptStudioHelper:
user_id=user_id,
request_user=request_user,
)
+ # Book the output against the profile the run actually used, the
+ # same ladder _fetch_response applies. Forwarding the raw (possibly
+ # None) argument makes _handle_response re-resolve the project
+ # default, so a prompt carrying its own FK would run under that FK
+ # but have its output stored under the project default.
+ resolved_profile_id = str(
+ PromptStudioHelper._resolve_profile_manager(
+ tool=tool,
+ prompt=prompt_instance,
+ profile_manager_id=profile_manager_id,
+ ).profile_id
+ )
+
return PromptStudioHelper._handle_response(
response=response,
run_id=run_id,
prompts=prompts,
document_id=document_id,
is_single_pass=False,
- profile_manager_id=profile_manager_id,
+ profile_manager_id=resolved_profile_id,
)
except APIException:
# Validation responses are user-facing; DRF renders them as-is.
@@ -1880,14 +1919,9 @@ class PromptStudioHelper:
Any: Output from LLM
"""
# Fetch the ProfileManager instance using the profile_manager_id if provided
- profile_manager = prompt.profile_manager
- if profile_manager_id:
- profile_manager = ProfileManagerHelper.get_profile_manager(
- profile_manager_id=profile_manager_id
- )
-
- if not profile_manager:
- raise DefaultProfileError()
+ profile_manager = PromptStudioHelper._resolve_profile_manager(
+ tool=tool, prompt=prompt, profile_manager_id=profile_manager_id
+ )
monitor_llm_instance: AdapterInstance | None = tool.monitor_llm
monitor_llm: str | None = None
diff --git a/backend/prompt_studio/prompt_studio_core_v2/tests/test_profile_resolution_fallback.py b/backend/prompt_studio/prompt_studio_core_v2/tests/test_profile_resolution_fallback.py
new file mode 100644
index 000000000..2bbebe736
--- /dev/null
+++ b/backend/prompt_studio/prompt_studio_core_v2/tests/test_profile_resolution_fallback.py
@@ -0,0 +1,500 @@
+"""Regression tests for profile resolution in ``fetch_response``.
+
+Pins the fix for the CLI-surfaced bug where a prompt with no
+``profile_manager`` FK raised ``DefaultProfileError`` even though the project
+had a default LLM profile configured. ``index_document`` and single-pass
+extraction already fell back to ``ProfileManager.get_default_llm_profile``;
+``fetch_response`` was the lone omission.
+
+The resolution ladder these tests lock down, in order:
+
+ 1. An explicitly passed ``profile_manager_id`` wins over everything.
+ 2. Otherwise the prompt's own ``profile_manager`` FK.
+ 3. Otherwise the project default -- the rung this PR added.
+ 4. If none resolves, ``DefaultProfileError`` propagates.
+
+They also pin the second half of the fix: ``cb_kwargs["profile_manager_id"]``
+must carry the profile *actually used*, not the (possibly ``None``) argument.
+``OutputManagerHelper.get_default_profile`` treats ``None`` as "re-resolve the
+project default", so passing the raw argument books output against a different
+profile than the one that produced it.
+
+These call the real ``build_fetch_response_payload`` /
+``build_bulk_fetch_response_payload`` and patch only their collaborators, so
+the assertions cover the shipped path end to end -- including everything after
+the ladder.
+
+An earlier version of this file extracted the ladder's source text and
+``exec``'d it. It was replaced because it could not fail on the regressions it
+existed to catch: it stayed green when the resolved profile was clobbered
+immediately after the ladder, and when the ``cb_kwargs`` fix was reverted at
+both call sites. Please do not reintroduce source-text assertions here.
+"""
+
+from __future__ import annotations
+
+import os
+import uuid
+from contextlib import ExitStack
+from typing import Any
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+
+def _deps():
+ """Import the Django-coupled helper at call time, not import time.
+
+ Skips only when Django itself is unavailable; configuration and application
+ import failures propagate so a real regression is never hidden as a skip.
+ """
+ try:
+ import django
+ except ImportError as exc: # pragma: no cover - environment guard
+ pytest.skip(f"django is not installed: {exc}")
+
+ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings.test")
+ django.setup()
+
+ from prompt_studio.prompt_studio_core_v2 import prompt_studio_helper as psh
+ from prompt_studio.prompt_studio_core_v2.exceptions import DefaultProfileError
+
+ return psh, DefaultProfileError
+
+
+EXPLICIT_ID = "11111111-1111-1111-1111-111111111111"
+PROMPT_FK_ID = "22222222-2222-2222-2222-222222222222"
+PROJECT_DEFAULT_ID = "33333333-3333-3333-3333-333333333333"
+
+# build_bulk_fetch_response_payload takes a prompt list and never consults a
+# single prompt's FK, so only the FK-precedence rung is scoped to the builder
+# that reads one. Both builders carry the cb_kwargs fix.
+FK_AWARE_BUILDERS = ["build_fetch_response_payload"]
+ALL_BUILDERS = ["build_fetch_response_payload", "build_bulk_fetch_response_payload"]
+
+
+def _make_profile(profile_id: str) -> Any:
+ """A ProfileManager stand-in with the attributes the builders read.
+
+ ``profile_id`` is a real ``UUID``, matching the ``UUIDField`` on the model.
+ That is load-bearing: ``cb_kwargs`` is JSON-serialised by Celery
+ (``task_serializer = "json"``), so dropping the ``str()`` wrapper in the
+ helper would raise at dispatch. A ``str`` fake here would hide that.
+ """
+ profile = MagicMock(name=f"ProfileManager<{profile_id}>")
+ profile.profile_id = uuid.UUID(profile_id)
+ profile.chunk_size = 512
+ profile.chunk_overlap = 64
+ return profile
+
+
+def _make_prompt(profile: Any) -> Any:
+ prompt = MagicMock(name="ToolStudioPrompt")
+ prompt.profile_manager = profile
+ prompt.prompt_id = "prompt-1"
+ return prompt
+
+
+def _call(
+ psh: Any,
+ builder: str,
+ *,
+ profile_manager_id: str | None,
+ prompt_profile: Any,
+ default_profile: Any,
+ default_raises: BaseException | None = None,
+) -> tuple[Any, dict[str, Any]]:
+ """Invoke a real payload builder with every collaborator patched out.
+
+ Only profile resolution stays real; everything downstream is stubbed so the
+ call reaches ``cb_kwargs`` without touching storage, indexing or an LLM.
+
+ ``get_profile_manager`` returns the explicit profile only when handed the
+ ID under test, so the rung-1 assertion cannot pass vacuously.
+ """
+ helper = psh.PromptStudioHelper
+ prompt = _make_prompt(prompt_profile)
+
+ def _get_default(tool: Any) -> Any:
+ if default_raises is not None:
+ raise default_raises
+ return default_profile
+
+ def _get_explicit(profile_manager_id: str) -> Any:
+ assert profile_manager_id == EXPLICIT_ID, (
+ f"builder forwarded {profile_manager_id!r} to get_profile_manager, "
+ f"expected {EXPLICIT_ID!r}"
+ )
+ return _make_profile(EXPLICIT_ID)
+
+ # autospec on the collaborators whose signatures the builders call into, so
+ # an arity or keyword change in production fails here instead of staying
+ # green. The module-attribute patches below are legitimately opaque.
+ patches = [
+ patch.object(
+ psh.ProfileManager,
+ "get_default_llm_profile",
+ autospec=True,
+ side_effect=lambda tool: _get_default(tool),
+ ),
+ patch.object(
+ psh.ProfileManagerHelper,
+ "get_profile_manager",
+ autospec=True,
+ side_effect=lambda profile_manager_id: _get_explicit(profile_manager_id),
+ ),
+ patch.object(helper, "_resolve_llm_ids", autospec=True, return_value=("m", "c")),
+ patch.object(helper, "validate_adapter_status", autospec=True),
+ patch.object(helper, "validate_profile_manager_owner_access", autospec=True),
+ patch.object(helper, "_get_platform_api_key", autospec=True, return_value="pk"),
+ patch.object(helper, "dynamic_extractor", autospec=True, return_value="text"),
+ # Must be a dict with a non-pending status: the builders early-return a
+ # pending response (and no cb_kwargs) when indexing is still running.
+ patch.object(
+ helper, "dynamic_indexer", autospec=True, return_value={"status": "COMPLETED"}
+ ),
+ patch.object(helper, "_build_grammar_list", autospec=True, return_value=[]),
+ patch.object(helper, "_build_prompt_output", autospec=True, return_value={}),
+ # Returns the (mutated) output dict, so it must pass it through.
+ patch.object(
+ helper,
+ "fetch_table_settings_if_enabled",
+ autospec=True,
+ side_effect=lambda _doc, _prompt, _org, _user, _tool, output: output,
+ ),
+ patch.object(psh, "EnvHelper", MagicMock()),
+ patch.object(psh, "PromptIdeBaseTool", MagicMock()),
+ patch.object(psh, "IndexingUtils", MagicMock()),
+ patch.object(psh, "StateStore", MagicMock()),
+ patch.object(psh, "ExecutionContext", MagicMock()),
+ patch.object(psh, "PromptStudioVariableService", MagicMock()),
+ ]
+ # ExitStack, not a bare start loop: a mistargeted patch would otherwise
+ # raise partway through and leak the already-started ones into later tests.
+ with ExitStack() as stack:
+ for patcher in patches:
+ stack.enter_context(patcher)
+ kwargs: dict[str, Any] = {
+ "tool": MagicMock(
+ tool_id="tool-1",
+ summarize_as_source=False,
+ enable_highlight=False,
+ enable_challenge=False,
+ ),
+ "doc_path": "/docs/a.pdf",
+ "doc_name": "a.pdf",
+ "org_id": "org-1",
+ "user_id": "user-1",
+ "document_id": "doc-1",
+ "run_id": "run-1",
+ "profile_manager_id": profile_manager_id,
+ "request_user": None,
+ }
+ if builder == "build_bulk_fetch_response_payload":
+ kwargs["prompts"] = [prompt]
+ else:
+ kwargs["prompt"] = prompt
+ return getattr(helper, builder)(**kwargs)
+
+
[email protected]("builder", FK_AWARE_BUILDERS)
+class TestProfileResolutionLadder:
+ def test_prompt_without_profile_uses_project_default(self, builder: str) -> None:
+ """The reported bug: no FK + a project default must resolve, not raise."""
+ psh, _ = _deps()
+
+ _context, cb_kwargs = _call(
+ psh,
+ builder,
+ profile_manager_id=None,
+ prompt_profile=None,
+ default_profile=_make_profile(PROJECT_DEFAULT_ID),
+ )
+
+ assert cb_kwargs["profile_manager_id"] == PROJECT_DEFAULT_ID, (
+ "A prompt with no profile FK must fall back to the project default; "
+ "this raised DefaultProfileError before the fix"
+ )
+
+ def test_prompt_fk_wins_over_project_default(self, builder: str) -> None:
+ """The fallback must not override a profile the prompt already has."""
+ psh, _ = _deps()
+
+ _context, cb_kwargs = _call(
+ psh,
+ builder,
+ profile_manager_id=None,
+ prompt_profile=_make_profile(PROMPT_FK_ID),
+ default_profile=_make_profile(PROJECT_DEFAULT_ID),
+ )
+
+ assert cb_kwargs["profile_manager_id"] == PROMPT_FK_ID
+
+ def test_explicit_id_wins_over_everything(self, builder: str) -> None:
+ psh, _ = _deps()
+
+ _context, cb_kwargs = _call(
+ psh,
+ builder,
+ profile_manager_id=EXPLICIT_ID,
+ prompt_profile=_make_profile(PROMPT_FK_ID),
+ default_profile=_make_profile(PROJECT_DEFAULT_ID),
+ )
+
+ assert cb_kwargs["profile_manager_id"] == EXPLICIT_ID
+
+
[email protected]("builder", ALL_BUILDERS)
+class TestResolvedProfileReachesCallback:
+ """``cb_kwargs`` must carry the profile used, not the raw argument.
+
+ Passing the argument through lets the callback re-resolve the project
+ default (``None`` is its "re-resolve" sentinel), booking output against a
+ different profile than the one that produced it. Reverting the fix at
+ either call site must fail here.
+ """
+
+ def test_callback_records_the_resolved_profile(self, builder: str) -> None:
+ psh, _ = _deps()
+
+ _context, cb_kwargs = _call(
+ psh,
+ builder,
+ profile_manager_id=None,
+ prompt_profile=None,
+ default_profile=_make_profile(PROJECT_DEFAULT_ID),
+ )
+
+ assert cb_kwargs["profile_manager_id"] == PROJECT_DEFAULT_ID, (
+ "cb_kwargs must record the resolved profile; the raw None argument "
+ "makes the callback silently re-resolve the project default"
+ )
+
+
[email protected]("builder", ALL_BUILDERS)
+class TestUnresolvableProfile:
+ def test_no_profile_anywhere_propagates_default_profile_error(
+ self, builder: str
+ ) -> None:
+ """The fallback must not swallow a genuinely unconfigured project."""
+ psh, default_profile_error = _deps()
+
+ with pytest.raises(default_profile_error):
+ _call(
+ psh,
+ builder,
+ profile_manager_id=None,
+ prompt_profile=None,
+ default_profile=None,
+ default_raises=default_profile_error(),
+ )
+
+
+class TestSynchronousFetchResponse:
+ """``_fetch_response`` carries the same fallback and is still live.
+
+ It is reached from ``fetch_response`` (the non-dispatch branch) and returns
+ a service response rather than ``cb_kwargs``, so the resolved profile is
+ observed where it is first consumed: ``validate_adapter_status``.
+ """
+
+ def _resolved_profile(
+ self,
+ psh: Any,
+ *,
+ profile_manager_id: str | None,
+ prompt_profile: Any,
+ default_profile: Any,
+ ) -> Any:
+ helper = psh.PromptStudioHelper
+ seen: list[Any] = []
+
+ def _get_explicit(profile_manager_id: str) -> Any:
+ assert profile_manager_id == EXPLICIT_ID
+ return _make_profile(EXPLICIT_ID)
+
+ patches = [
+ patch.object(
+ psh.ProfileManager,
+ "get_default_llm_profile",
+ autospec=True,
+ side_effect=lambda tool: default_profile,
+ ),
+ patch.object(
+ psh.ProfileManagerHelper,
+ "get_profile_manager",
+ autospec=True,
+ side_effect=lambda profile_manager_id: _get_explicit(profile_manager_id),
+ ),
+ patch.object(
+ helper,
+ "validate_adapter_status",
+ autospec=True,
+ side_effect=seen.append,
+ ),
+ patch.object(
+ helper, "validate_profile_manager_owner_access", autospec=True
+ ),
+ patch.object(
+ helper, "_resolve_llm_ids", autospec=True, return_value=("m", "c")
+ ),
+ # Stop the call once the profile has been observed; everything past
+ # this point needs storage and an LLM.
+ patch.object(psh, "EnvHelper", MagicMock(side_effect=RuntimeError)),
+ ]
+ with ExitStack() as stack:
+ for patcher in patches:
+ stack.enter_context(patcher)
+ with pytest.raises(Exception): # noqa: B017 - halted deliberately
+ helper._fetch_response(
+ tool=MagicMock(tool_id="tool-1", summarize_as_source=False),
+ doc_path="/docs/a.pdf",
+ doc_name="a.pdf",
+ prompt=_make_prompt(prompt_profile),
+ org_id="org-1",
+ document_id="doc-1",
+ run_id="run-1",
+ user_id="user-1",
+ profile_manager_id=profile_manager_id,
+ request_user=None,
+ )
+
+ assert seen, "validate_adapter_status was never reached"
+ return seen[0]
+
+ def test_prompt_without_profile_uses_project_default(self) -> None:
+ """The reported bug, on the synchronous path."""
+ psh, _ = _deps()
+
+ resolved = self._resolved_profile(
+ psh,
+ profile_manager_id=None,
+ prompt_profile=None,
+ default_profile=_make_profile(PROJECT_DEFAULT_ID),
+ )
+
+ assert str(resolved.profile_id) == PROJECT_DEFAULT_ID
+
+ def test_prompt_fk_wins_over_project_default(self) -> None:
+ psh, _ = _deps()
+
+ resolved = self._resolved_profile(
+ psh,
+ profile_manager_id=None,
+ prompt_profile=_make_profile(PROMPT_FK_ID),
+ default_profile=_make_profile(PROJECT_DEFAULT_ID),
+ )
+
+ assert str(resolved.profile_id) == PROMPT_FK_ID
+
+
+class TestSynchronousOutputAttribution:
+ """``_execute_single_prompt`` must book output against the profile used.
+
+ The async builders record the resolved profile in ``cb_kwargs``; the
+ synchronous path hands ``profile_manager_id`` to ``_handle_response``, whose
+ ``None`` branch re-resolves the project default. A prompt carrying its own
+ FK would then run under the FK but store its output under the project
+ default.
+ """
+
+ def _handled_profile_id(
+ self, psh: Any, *, profile_manager_id: str | None, prompt_profile: Any
+ ) -> Any:
+ helper = psh.PromptStudioHelper
+ prompt = _make_prompt(prompt_profile)
+ prompt.enforce_type = "Text"
+ prompt.tool_id = MagicMock(tool_id="tool-1")
+ seen: dict[str, Any] = {}
+
+ patches = [
+ patch.object(
+ helper, "_fetch_prompt_from_id", autospec=True, return_value=prompt
+ ),
+ patch.object(
+ helper, "_fetch_response", autospec=True, return_value={"status": "OK"}
+ ),
+ patch.object(helper, "_publish_log", autospec=True),
+ patch.object(
+ psh.ProfileManager,
+ "get_default_llm_profile",
+ autospec=True,
+ side_effect=lambda tool: _make_profile(PROJECT_DEFAULT_ID),
+ ),
+ patch.object(
+ psh.ProfileManagerHelper,
+ "get_profile_manager",
+ autospec=True,
+ side_effect=lambda profile_manager_id: _make_profile(profile_manager_id),
+ ),
+ patch.object(
+ helper,
+ "_handle_response",
+ autospec=True,
+ side_effect=lambda **kw: seen.update(kw),
+ ),
+ patch.object(psh, "get_plugin", MagicMock(return_value=None)),
+ ]
+ with ExitStack() as stack:
+ for patcher in patches:
+ stack.enter_context(patcher)
+ helper._execute_single_prompt(
+ id="prompt-1",
+ doc_path="/docs/a.pdf",
+ doc_name="a.pdf",
+ tool_id="tool-1",
+ org_id="org-1",
+ user_id="user-1",
+ document_id="doc-1",
+ run_id="run-1",
+ profile_manager_id=profile_manager_id,
+ request_user=None,
+ )
+
+ assert seen, "_handle_response was never reached"
+ return seen["profile_manager_id"]
+
+ def test_prompt_fk_is_recorded_not_reresolved(self) -> None:
+ psh, _ = _deps()
+
+ recorded = self._handled_profile_id(
+ psh, profile_manager_id=None, prompt_profile=_make_profile(PROMPT_FK_ID)
+ )
+
+ assert recorded == PROMPT_FK_ID, (
+ "Output must be booked against the prompt's own profile; passing "
+ "None makes _handle_response re-resolve the project default"
+ )
+
+ def test_project_default_is_recorded_when_no_fk(self) -> None:
+ psh, _ = _deps()
+
+ recorded = self._handled_profile_id(
+ psh, profile_manager_id=None, prompt_profile=None
+ )
+
+ assert recorded == PROJECT_DEFAULT_ID
+
+ def test_explicit_id_round_trips(self) -> None:
+ """The explicit rung is a pass-through, but must stay one."""
+ psh, _ = _deps()
+
+ recorded = self._handled_profile_id(
+ psh,
+ profile_manager_id=EXPLICIT_ID,
+ prompt_profile=_make_profile(PROMPT_FK_ID),
+ )
+
+ assert recorded == EXPLICIT_ID
+
+
+def test_default_profile_error_is_a_client_error() -> None:
+ """A missing default profile is user-fixable configuration, not a 500."""
+ _psh, default_profile_error = _deps()
+
+ assert default_profile_error.status_code == 400
+ detail = str(default_profile_error.default_detail).lower()
+ assert "project" in detail and "prompt" in detail, (
+ "The message must name both places a profile can come from; it is also "
+ "raised for monitor/challenge resolution, not just the prompt"
+ )
diff --git a/backend/prompt_studio/prompt_studio_core_v2/tests/test_task_status.py b/backend/prompt_studio/prompt_studio_core_v2/tests/test_task_status.py
index d7ee67c22..ab0754ee1 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/tests/test_task_status.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/tests/test_task_status.py
@@ -84,14 +84,24 @@ class TestTaskStatusFlagOff:
def test_celery_completed_never_queries_pg(self):
# flag off (also the Flipt-error fail-closed path): Celery + PG untouched.
p_pg, pg = _stub_pg({"status": "completed"}) # would resolve, if queried
- with _org(), _flag(False), p_pg, patch(f"{_VIEWS}.AsyncResult", _async(ready=True)):
+ with (
+ _org(),
+ _flag(False),
+ p_pg,
+ patch(f"{_VIEWS}.AsyncResult", _async(ready=True)),
+ ):
resp = _view().task_status(MagicMock(), task_id=_TID)
assert resp.data == {"task_id": _TID, "status": "completed"}
pg.objects.filter.assert_not_called() # default path never touches PG
def test_celery_failed(self):
- with _org(), _flag(False), patch(
- f"{_VIEWS}.AsyncResult", _async(ready=True, successful=False, result="celery boom")
+ with (
+ _org(),
+ _flag(False),
+ patch(
+ f"{_VIEWS}.AsyncResult",
+ _async(ready=True, successful=False, result="celery boom"),
+ ),
):
resp = _view().task_status(MagicMock(), task_id=_TID)
assert resp.data["status"] == "failed"
diff --git a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py
index fac9acd7d..f514eda95 100644
--- a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py
+++ b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py
@@ -38,6 +38,23 @@ class PromptStudioRegistryHelper:
with the help of a proto tool.
"""
+ @staticmethod
+ def _resolve_challenge_llm_id(tool: CustomTool) -> str:
+ """Resolve the adapter ID to seed as challenge_llm's spec default.
+
+ Mirrors the precedence frame_export_json already uses: the tool's own
+ challenge_llm FK, else the project default profile's LLM.
+
+ Args:
+ tool (CustomTool): Saved tool data
+
+ Returns:
+ str: Adapter instance ID of the challenger LLM
+ """
+ if tool.challenge_llm:
+ return str(tool.challenge_llm.id)
+ return str(ProfileManager.get_default_llm_profile(tool).llm.id)
+
@staticmethod
def frame_spec(tool: CustomTool) -> Spec:
"""Method to return spec of the Custom tool.
@@ -48,14 +65,31 @@ class PromptStudioRegistryHelper:
Returns:
dict: spec dict
"""
+ # A "type": "string" property with no spec default is seeded into tool
+ # instance metadata as "" (ToolUtils.get_default_settings). Because
+ # challenge_llm declares adapterType, the instance schema carries an
+ # enum of real adapter IDs, which "" can never satisfy - so deployment
+ # validation rejected the freshly seeded instance, regardless of whether
+ # LLMChallenge was enabled (the seeding never consulted that flag).
+ #
+ # Seed the resolved LLM instead of leaving it empty. adapterType stays
+ # declared unconditionally: it is the sole discriminator in
+ # Spec.get_adapter_properties() (tool-registry dto.py), so dropping it
+ # would take challenge_llm out of every adapter-aware path - the
+ # challenge_llm_adapter_id writes that runtime actually reads
+ # (structure_tool_task.py), the settings-form adapter dropdown, and the
+ # lazy adapter-id migration check.
+ challenge_llm_property: dict[str, Any] = {
+ "type": "string",
+ "title": "Challenger LLM",
+ "adapterType": "LLM",
+ "description": "LLM to use for LLMChallenge",
+ "adapterIdKey": "challenge_llm_adapter_id",
+ "default": PromptStudioRegistryHelper._resolve_challenge_llm_id(tool),
+ }
+
properties = {
- "challenge_llm": {
- "type": "string",
- "title": "Challenger LLM",
- "adapterType": "LLM",
- "description": "LLM to use for LLMChallenge",
- "adapterIdKey": "challenge_llm_adapter_id",
- },
+ "challenge_llm": challenge_llm_property,
"enable_challenge": {
"type": "boolean",
"title": "Enable LLMChallenge",
diff --git a/backend/prompt_studio/prompt_studio_registry_v2/tests/test_frame_spec_challenge_llm.py b/backend/prompt_studio/prompt_studio_registry_v2/tests/test_frame_spec_challenge_llm.py
new file mode 100644
index 000000000..eb0b1e559
--- /dev/null
+++ b/backend/prompt_studio/prompt_studio_registry_v2/tests/test_frame_spec_challenge_llm.py
@@ -0,0 +1,249 @@
+"""Regression tests for ``PromptStudioRegistryHelper.frame_spec``'s handling of
+``challenge_llm``.
+
+Pins the fix for the deploy-time failure where an exported tool ended in
+pipeline status ERROR with ``422 Unprocessable Entity: Tool validation failed``.
+
+The mechanism these tests lock down:
+
+ 1. ``ToolUtils.get_default_settings`` seeds each spec property into tool
+ instance metadata using the spec ``default`` when present, else the zero
+ value for the declared type -- ``""`` for a string.
+ 2. A property declaring ``adapterType: "LLM"`` gets an ``enum`` of real
+ adapter IDs injected by ``_update_schema_for_adapter_type``.
+ 3. ``""`` is not in that enum, so deployment validation rejected it.
+
+The fix seeds ``challenge_llm`` with a resolved adapter ID rather than dropping
+its ``adapterType``. That distinction is load-bearing and is asserted directly
+here: ``adapterType`` is the sole discriminator in ``Spec.get_adapter_properties``
+(tool-registry ``dto.py``), so removing it would silently unregister
+``challenge_llm`` from every adapter-aware path -- the ``challenge_llm_adapter_id``
+write that runtime reads, the settings-form dropdown, and the lazy adapter-id
+migration check. ``test_adapter_type_is_always_declared`` fails loudly if a
+future change pops it again.
+
+These tests exercise the real ``frame_spec`` and the real
+``_update_schema_for_adapter_type``, patching only the two DB lookups (the
+default profile and the adapter list). Nothing is stubbed into ``sys.modules``:
+that pattern shadows the real ``unstract.*`` packages for every later test in the
+pytest process, and the repo has been removing it (see the execution-serializer
+stub leak dropped in ``b60dd6f3``).
+
+Every real import happens inside ``_deps()`` rather than at module scope -- see
+its docstring for why.
+"""
+
+from __future__ import annotations
+
+import os
+from types import SimpleNamespace
+from typing import Any
+from unittest.mock import MagicMock, patch
+
+import jsonschema
+import pytest
+
+
+def _deps():
+ """Import everything this module needs, at call time rather than import time.
+
+ Nothing is imported at module scope on purpose. ``frame_spec`` and
+ ``ToolProcessor`` need a configured Django app registry, so importing them
+ at module scope makes *collection* of this file depend on settings being
+ configured -- and a collection error takes the whole file down rather than
+ failing one test. Importing at call time keeps that failure local.
+
+ ``django`` genuinely missing is an environment gap -> skip. Anything else
+ fails loudly rather than skipping, so a genuine import breakage is reported
+ instead of hidden. (This file previously resolved ``unstract.*`` to mocks
+ when a sibling module stubbed ``sys.modules`` during collection; that
+ stubbing is gone, and failing loudly here keeps it gone.)
+ """
+ try:
+ import django
+ except ImportError as exc: # pragma: no cover - environment guard
+ pytest.skip(f"django is not installed: {exc}")
+
+ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings.test")
+ django.setup()
+
+ from tool_instance_v2.tool_processor import ToolProcessor
+ from unstract.sdk1.constants import AdapterTypes
+ from unstract.tool_registry.dto import Spec
+ from unstract.tool_registry.tool_utils import ToolUtils
+
+ from prompt_studio.prompt_studio_registry_v2.prompt_studio_registry_helper import (
+ PromptStudioRegistryHelper,
+ )
+
+ return SimpleNamespace(
+ helper=PromptStudioRegistryHelper,
+ tool_processor=ToolProcessor,
+ AdapterTypes=AdapterTypes,
+ Spec=Spec,
+ ToolUtils=ToolUtils,
+ )
+
+
+REAL_ADAPTER_ID = "11111111-2222-3333-4444-555555555555"
+OTHER_ADAPTER_ID = "99999999-8888-7777-6666-555555555555"
+
+PROFILE_LOOKUP = (
+ "prompt_studio.prompt_studio_registry_v2.prompt_studio_registry_helper"
+ ".ProfileManager.get_default_llm_profile"
+)
+ADAPTER_LOOKUP = "tool_instance_v2.tool_processor.AdapterProcessor.get_adapters_by_type"
+
+
+def _make_tool(*, enable_challenge: bool, challenge_llm_id: str | None = None):
+ """Minimal stand-in for ``CustomTool`` -- ``frame_spec`` reads only these."""
+ tool = MagicMock(name="CustomTool")
+ tool.tool_id = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"
+ tool.description = "A tool exported from Prompt Studio"
+ tool.enable_challenge = enable_challenge
+ tool.challenge_llm = (
+ None if challenge_llm_id is None else MagicMock(id=challenge_llm_id)
+ )
+ return tool
+
+
+def _default_profile(llm_id: str = REAL_ADAPTER_ID):
+ return MagicMock(llm=MagicMock(id=llm_id))
+
+
+def _frame_spec(tool: Any) -> Any:
+ deps = _deps()
+ with patch(PROFILE_LOOKUP, return_value=_default_profile()):
+ return deps.helper.frame_spec(tool=tool)
+
+
+def _build_instance_schema(tool: Any) -> dict[str, Any]:
+ """Compose export-time ``frame_spec`` with deploy-time enum injection.
+
+ This is the path a tool instance's metadata is validated against:
+ ``frame_spec`` output is persisted as ``tool_spec`` and later rehydrated
+ via ``Spec.from_dict`` in ``get_tool_by_prompt_registry_id``.
+ """
+ deps = _deps()
+ spec = _frame_spec(tool)
+ adapters = [MagicMock(id=REAL_ADAPTER_ID, adapter_name="My LLM")]
+ with patch(ADAPTER_LOOKUP, return_value=adapters):
+ deps.tool_processor._update_schema_for_adapter_type(
+ spec, spec.get_llm_adapter_properties_keys(), deps.AdapterTypes.LLM, None
+ )
+ return spec.to_dict()
+
+
+def _seeded_metadata(spec_dict: dict[str, Any]) -> dict[str, Any]:
+ """Seed instance metadata the way tool-instance creation really does.
+
+ Calls the real ``ToolUtils.get_default_settings`` rather than reimplementing
+ its type->zero-value table, so a change to the seeding rule surfaces here.
+ """
+ deps = _deps()
+ tool = MagicMock()
+ tool.spec = deps.Spec.from_dict(spec_dict)
+ return deps.ToolUtils.get_default_settings(tool)
+
+
+def _validation_errors(schema: dict[str, Any], instance: dict[str, Any]) -> list[Any]:
+ return list(jsonschema.Draft7Validator(schema).iter_errors(instance))
+
+
[email protected]("enable_challenge", [False, True])
+class TestSeededInstanceDeploys:
+ """A freshly created tool instance must pass deployment validation.
+
+ This is the reported bug, and it fired regardless of ``enable_challenge``.
+ """
+
+ def test_seeded_metadata_validates(self, enable_challenge: bool) -> None:
+ schema = _build_instance_schema(_make_tool(enable_challenge=enable_challenge))
+
+ errors = _validation_errors(schema, _seeded_metadata(schema))
+
+ assert errors == [], (
+ "Metadata seeded by get_default_settings must pass deployment "
+ f"validation; got {[e.validator for e in errors]}"
+ )
+
+ def test_seeded_challenge_llm_is_a_real_adapter_id(
+ self, enable_challenge: bool
+ ) -> None:
+ """The seeded value must be usable, not merely valid."""
+ schema = _build_instance_schema(_make_tool(enable_challenge=enable_challenge))
+
+ assert _seeded_metadata(schema)["challenge_llm"] == REAL_ADAPTER_ID
+
+ def test_adapter_type_is_always_declared(self, enable_challenge: bool) -> None:
+ """``adapterType`` gates every adapter-aware path -- never drop it.
+
+ Without it ``challenge_llm`` leaves ``get_llm_adapter_properties()``, so
+ ``challenge_llm_adapter_id`` is never written and the settings form
+ loses its adapter dropdown.
+ """
+ spec = _frame_spec(_make_tool(enable_challenge=enable_challenge))
+
+ assert "challenge_llm" in spec.get_llm_adapter_properties_keys()
+ assert "challenge_llm" in spec.get_llm_adapter_properties()
+
+ def test_dropdown_metadata_is_present(self, enable_challenge: bool) -> None:
+ """The settings form renders its dropdown from enum/enumNames."""
+ schema = _build_instance_schema(_make_tool(enable_challenge=enable_challenge))
+ prop = schema["properties"]["challenge_llm"]
+
+ assert prop["enum"] == [REAL_ADAPTER_ID]
+ assert prop["enumNames"] == ["My LLM"]
+
+ def test_challenge_llm_stays_required(self, enable_challenge: bool) -> None:
+ """Always seeded, so requiredness costs nothing and keeps the contract."""
+ schema = _build_instance_schema(_make_tool(enable_challenge=enable_challenge))
+
+ assert "challenge_llm" in schema["required"]
+
+
+class TestChallengeLlmResolution:
+ """``frame_spec`` seeds the same LLM the export path resolves."""
+
+ def test_tool_challenge_llm_takes_precedence(self) -> None:
+ deps = _deps()
+ tool = _make_tool(enable_challenge=True, challenge_llm_id=OTHER_ADAPTER_ID)
+ with patch(PROFILE_LOOKUP, return_value=_default_profile()) as get_default:
+ spec = deps.helper.frame_spec(tool=tool)
+
+ assert spec.properties["challenge_llm"]["default"] == OTHER_ADAPTER_ID
+ get_default.assert_not_called()
+
+ def test_falls_back_to_default_profile_llm(self) -> None:
+ spec = _frame_spec(_make_tool(enable_challenge=False))
+
+ assert spec.properties["challenge_llm"]["default"] == REAL_ADAPTER_ID
+
+
+class TestInvalidValuesStillRejected:
+ """The fix must not open a hole: a bad adapter must still fail."""
+
+ @pytest.mark.parametrize("enable_challenge", [False, True])
+ def test_empty_value_is_rejected(self, enable_challenge: bool) -> None:
+ """A stored ``""`` -- the pre-fix state -- must not validate."""
+ schema = _build_instance_schema(_make_tool(enable_challenge=enable_challenge))
+
+ errors = _validation_errors(schema, {"challenge_llm": ""})
+
+ assert any(
+ e.validator == "enum" for e in errors
+ ), f"Expected an enum violation, got {[e.validator for e in errors]}"
+
+ def test_unknown_adapter_id_is_rejected(self) -> None:
+ schema = _build_instance_schema(_make_tool(enable_challenge=True))
+
+ errors = _validation_errors(schema, {"challenge_llm": OTHER_ADAPTER_ID})
+
+ assert any(e.validator == "enum" for e in errors)
+
+ def test_missing_key_is_rejected(self) -> None:
+ schema = _build_instance_schema(_make_tool(enable_challenge=True))
+
+ errors = _validation_errors(schema, {})
+
+ assert any(e.validator == "required" for e in errors)
diff --git a/backend/pyproject.toml b/backend/pyproject.toml
index ce01979b3..7eee250b9 100644
--- a/backend/pyproject.toml
+++ b/backend/pyproject.toml
@@ -71,9 +71,13 @@ dev = [
"responses>=0.25.7",
"psutil>=7.0.0",
]
+# jsonschema also arrives via unstract-tool-registry, but the schema tests
+# import it directly - declare it so the suite fails loudly rather than
+# skipping if that transitive path ever changes.
test = [
"pytest>=8.0.1",
"pytest-django>=4.12.0",
+ "jsonschema>=4.18.6,<5.0",
]
deploy = [
"gunicorn~=23.0", # For serving the application
diff --git a/backend/uv.lock b/backend/uv.lock
index a68e8aee7..c80f0ecea 100644
--- a/backend/uv.lock
+++ b/backend/uv.lock
@@ -3732,6 +3732,7 @@ dev = [
{ name = "unstract-workflow-execution" },
]
test = [
+ { name = "jsonschema" },
{ name = "pytest" },
{ name = "pytest-django" },
]
@@ -3798,6 +3799,7 @@ dev = [
{ name = "unstract-workflow-execution", editable = "../unstract/workflow-execution" },
]
test = [
+ { name = "jsonschema", specifier = ">=4.18.6,<5.0" },
{ name = "pytest", specifier = ">=8.0.1" },
{ name = "pytest-django", specifier = ">=4.12.0" },
]