Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

[FIX] Seed challenge_llm from the exported tool's resolved value so challenge-enabled deploys pass (#2209)

Hari John Kuriakose66a3363fbaeb
diff --git a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py
index f46298be3..8d88d2c00 100644
--- a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py
+++ b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py
@@ -1,4 +1,5 @@
import logging
+import uuid
from typing import Any
from account_v2.models import User
@@ -143,6 +144,49 @@ class PromptStudioRegistryHelper:
image_tag=settings.STRUCTURE_TOOL_IMAGE_TAG,
)
+ @staticmethod
+ def get_resolved_settings(prompt_registry_id: str) -> dict[str, Any]:
+ """Return the settings export already resolved for this exported tool.
+
+ ``frame_export_json`` resolves adapter-valued settings at export time -
+ notably ``challenge_llm``, which falls back to the default profile's LLM
+ when the project set none - and stores them under
+ ``tool_metadata[tool_settings]``. ``Tool`` (built from ``tool_spec`` /
+ ``tool_property``) does not carry them, so callers that only have a
+ ``Tool`` cannot see the resolved values.
+
+ Returns an empty dict when the registry row is missing or carries no
+ settings, so callers can treat "no resolved settings" as a no-op.
+ """
+ # `ToolInstance.tool_id` is a free-form CharField and registry tools use
+ # slugs ("classify", "text_extractor"), which a UUID pk lookup would
+ # reject. Screening them out here keeps this a silent no-op for every
+ # non-Prompt-Studio tool instead of logging on a fully normal path.
+ # Checked inline rather than via `ToolInstanceHelper.is_uuid_format`:
+ # `tool_instance_v2.tool_processor` already imports this module at
+ # module level, so importing back into `tool_instance_v2` would cycle.
+ try:
+ uuid.UUID(str(prompt_registry_id))
+ except ValueError:
+ return {}
+ try:
+ # Only `tool_metadata` is needed, and it is the largest column on the
+ # row - the full JSONB blob of every prompt and output. Fetching just
+ # that column avoids pulling the rest a second time, since the caller
+ # has already loaded this row to build the `Tool`.
+ metadata = PromptStudioRegistry.objects.values_list(
+ "tool_metadata", flat=True
+ ).get(pk=prompt_registry_id)
+ except PromptStudioRegistry.DoesNotExist:
+ # A UUID-shaped tool_id with no registry row - e.g. an agentic tool
+ # in cloud. Expected, so this stays at debug.
+ logger.debug(
+ f"No prompt studio registry entry for {prompt_registry_id}; "
+ "no resolved settings to apply"
+ )
+ return {}
+ return (metadata or {}).get(JsonSchemaKey.TOOL_SETTINGS, {}) or {}
+
@staticmethod
def update_or_create_psr_tool(
custom_tool: CustomTool,
diff --git a/backend/tool_instance_v2/serializers.py b/backend/tool_instance_v2/serializers.py
index 81eb6e0e6..049effe52 100644
--- a/backend/tool_instance_v2/serializers.py
+++ b/backend/tool_instance_v2/serializers.py
@@ -2,13 +2,21 @@ import logging
import uuid
from typing import Any
+from account_v2.models import User
from adapter_processor_v2.adapter_processor import AdapterProcessor
from adapter_processor_v2.models import AdapterInstance
-from prompt_studio.prompt_studio_registry_v2.constants import PromptStudioRegistryKeys
+from prompt_studio.prompt_studio_registry_v2.constants import (
+ JsonSchemaKey,
+ PromptStudioRegistryKeys,
+)
+from prompt_studio.prompt_studio_registry_v2.prompt_studio_registry_helper import (
+ PromptStudioRegistryHelper,
+)
from rest_framework.serializers import ListField, Serializer, UUIDField, ValidationError
from workflow_manager.workflow_v2.constants import WorkflowKey
from workflow_manager.workflow_v2.models.workflow import Workflow
+from backend.constants import RequestKey
from backend.serializers import AuditSerializer
from tool_instance_v2.constants import ToolInstanceKey as TIKey
from tool_instance_v2.constants import ToolKey
@@ -148,6 +156,104 @@ class ToolInstanceSerializer(AuditSerializer):
logger.error(f"Error transforming adapter IDs to names: {e}", exc_info=True)
return display_metadata
+ @staticmethod
+ def _overlay_resolved_challenge_llm(
+ tool: Tool,
+ tool_settings: dict[str, Any],
+ tool_uid: str,
+ user: User | None,
+ ) -> None:
+ """Seed `challenge_llm` from the value the export already resolved.
+
+ `get_default_settings` walks the spec, and an adapter-valued property
+ with no `default` is seeded "". `challenge_llm` has none, so the
+ instance would store "" - and because the property declares
+ `adapterType: "LLM"`, an enum of real adapter IDs is injected into the
+ tool-instance schema, which "" then fails at deployment validation.
+
+ The export resolved a real adapter ID for this setting (falling back to
+ the default profile's LLM when the project set none), so prefer it.
+
+ Only seed a value this user can actually use. The enum injected at
+ deployment is built per-user, and an exported tool can be shared, so the
+ exporter's adapter may be invisible to whoever creates the instance -
+ and it may since have been deleted. Seeding such an id would trade the
+ "" enum violation for an equally opaque one naming a real UUID. Leaving
+ the key empty instead lets `update_metadata_with_default_adapter` fill
+ in this user's default LLM, which is what happened before the export
+ value was available at all.
+
+ Deliberately scoped to `challenge_llm` alone. The export's
+ `tool_settings` overlaps the spec on four other keys
+ (`enable_challenge`, `summarize_as_source`, `enable_highlight`,
+ `enable_word_confidence`); overlaying those would change whether
+ challenge, summarization and highlighting actually *run* on new
+ instances - a cost-, latency- and output-affecting change, and a
+ separate decision from fixing this validation failure.
+
+ Tools whose spec has no `challenge_llm` return before any query; a
+ Prompt-Studio-shaped tool with no registry row resolves to {}.
+ """
+ challenge_llm_key = JsonSchemaKey.CHALLENGE_LLM
+ if challenge_llm_key not in tool_settings:
+ return
+
+ resolved_settings = PromptStudioRegistryHelper.get_resolved_settings(
+ prompt_registry_id=tool_uid
+ )
+ resolved_challenge_llm = resolved_settings.get(challenge_llm_key)
+ if not resolved_challenge_llm:
+ logger.debug(
+ "Export for tool %s resolved no challenge_llm; leaving the "
+ "spec-seeded default in place",
+ tool_uid,
+ )
+ return
+
+ if not ToolInstanceSerializer._is_adapter_usable_by(resolved_challenge_llm, user):
+ logger.warning(
+ "Resolved challenge_llm %s for tool %s is not accessible to the "
+ "creating user; falling back to their default LLM adapter",
+ resolved_challenge_llm,
+ tool_uid,
+ )
+ return
+
+ tool_settings[challenge_llm_key] = resolved_challenge_llm
+ # Write the companion ID key alongside, the shape every other writer
+ # emits (see `ToolInstanceHelper.update_metadata_with_default_adapter`).
+ adapter_property = tool.spec.properties.get(challenge_llm_key, {})
+ adapter_id_key = ToolInstanceHelper.get_adapter_id_key(adapter_property)
+ tool_settings[adapter_id_key] = resolved_challenge_llm
+
+ @staticmethod
+ def _is_adapter_usable_by(adapter_id: str, user: User | None) -> bool:
+ """Whether `adapter_id` is an LLM adapter visible to `user`.
+
+ Mirrors `AdapterProcessor.get_adapters_by_type` exactly - same
+ `for_user` scoping, same `adapter_type` predicate - because that builds
+ the enum this value is validated against at deploy time, and any
+ divergence lets a value through here that the enum then rejects. The
+ type predicate is load-bearing: `CustomTool.challenge_llm` is an
+ unconstrained FK whose serializer has no `validate_challenge_llm`,
+ unlike its `summarize_llm_adapter` sibling, so a non-LLM adapter can
+ reach it.
+
+ Without a user there is no scope to check against, and answering "yes"
+ would seed on mere existence - passing another user's private adapter.
+ Refuse instead: the caller then leaves the key empty and the default
+ adapter fills it, which is safe in every case. Unreachable today, since
+ every site that constructs this serializer to write goes through
+ `get_serializer`, which supplies the request.
+ """
+ if user is None:
+ return False
+ return (
+ AdapterInstance.objects.for_user(user)
+ .filter(id=adapter_id, adapter_type=AdapterTypes.LLM.value)
+ .exists()
+ )
+
def create(self, validated_data: dict[str, Any]) -> Any:
workflow_id = validated_data.pop(WorkflowKey.WF_ID)
try:
@@ -170,11 +276,19 @@ class ToolInstanceSerializer(AuditSerializer):
validated_data[TIKey.PK] = uuid.uuid4()
# TODO: Use version from tool props
validated_data[TIKey.VERSION] = ""
+ tool_settings = ToolProcessor.get_default_settings(tool)
+ request = self.context.get(RequestKey.REQUEST)
+ self._overlay_resolved_challenge_llm(
+ tool,
+ tool_settings,
+ str(tool_uid),
+ getattr(request, "user", None),
+ )
validated_data[TIKey.METADATA] = {
# TODO: Review and remove tool instance ID
WorkflowKey.WF_TOOL_INSTANCE_ID: str(validated_data[TIKey.PK]),
PromptStudioRegistryKeys.PROMPT_REGISTRY_ID: str(tool_uid),
- **ToolProcessor.get_default_settings(tool),
+ **tool_settings,
}
if TIKey.STEP not in validated_data:
validated_data[TIKey.STEP] = workflow.tool_instances.count() + 1
diff --git a/backend/tool_instance_v2/tests/__init__.py b/backend/tool_instance_v2/tests/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/tool_instance_v2/tests/test_challenge_llm_seed_overlay.py b/backend/tool_instance_v2/tests/test_challenge_llm_seed_overlay.py
new file mode 100644
index 000000000..5ce0f232b
--- /dev/null
+++ b/backend/tool_instance_v2/tests/test_challenge_llm_seed_overlay.py
@@ -0,0 +1,690 @@
+"""Regression tests for seeding a tool instance's ``challenge_llm`` from the
+exported tool's resolved settings.
+
+The deploy-time failure this closes: a tool exported with LLMChallenge *enabled*
+still ended in pipeline status ERROR with
+``422 Unprocessable Entity: Tool validation failed``.
+
+The chain:
+
+ 1. Export resolves a real ``challenge_llm`` (falling back to the default
+ profile's LLM) and stores it under ``tool_metadata[tool_settings]``.
+ 2. Tool-instance creation seeds metadata from ``get_default_settings``, which
+ walks the *spec*: a ``"type": "string"`` property with no ``default`` is
+ seeded ``""``. ``challenge_llm`` has no spec default, so the instance
+ stores ``""`` -- throwing away the value export already resolved.
+ 3. Because ``challenge_llm`` declares ``adapterType: "LLM"``,
+ ``_update_schema_for_adapter_type`` injects ``enum: [<real adapter ids>]``.
+ 4. Validation rejects ``""`` (an enum violation).
+
+There is a second half to the bug, and it is why the seed alone was not enough:
+``ToolInstanceViewSet.create`` calls ``update_metadata_with_default_values``
+immediately after ``perform_create``, and ``challenge_llm`` is one of the LLM
+adapter properties that walks. Before the guard added alongside these tests, it
+unconditionally overwrote the key with the user's *default* LLM -- so the seed
+was discarded for every user who has one, which is most of them.
+
+Unit tests: the real modules are imported (Django is configured by the rig's
+``unit-backend`` group env -- see ``tests/groups.yaml``) and every collaborator
+is patched per-test, so no database is touched.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+from unittest.mock import MagicMock, patch
+
+import jsonschema
+import pytest
+from django.db import OperationalError
+from prompt_studio.prompt_studio_registry_v2 import (
+ prompt_studio_registry_helper as _psr_mod,
+)
+from prompt_studio.prompt_studio_registry_v2.constants import JsonSchemaKey
+
+from tool_instance_v2 import serializers as _ser_mod
+from tool_instance_v2 import tool_instance_helper as _tih_mod
+from tool_instance_v2 import tool_processor as _tp_mod
+from unstract.sdk1.adapters.enums import AdapterTypes
+from unstract.tool_registry.constants import AdapterPropertyKey
+from unstract.tool_registry.dto import Spec
+from unstract.tool_registry.tool_utils import ToolUtils
+
+PromptStudioRegistryHelper = _psr_mod.PromptStudioRegistryHelper
+ToolInstanceSerializer = _ser_mod.ToolInstanceSerializer
+ToolInstanceHelper = _tih_mod.ToolInstanceHelper
+
+# The adapter ID the export resolved for the project's challenger LLM.
+RESOLVED_CHALLENGE_LLM = "11111111-2222-3333-4444-555555555555"
+# The deploying user's default LLM -- deliberately different, so a test can tell
+# "kept the exported value" apart from "fell back to the user's default".
+USER_DEFAULT_LLM = "99999999-8888-7777-6666-555555555555"
+
+PROMPT_REGISTRY_ID = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"
+
+CHALLENGE_LLM_ADAPTER_ID_KEY = "challenge_llm_adapter_id"
+
+
+def _challenge_llm_property() -> dict[str, Any]:
+ """The `challenge_llm` spec property exactly as `frame_spec` emits it."""
+ return {
+ "type": "string",
+ "title": "Challenger LLM",
+ "adapterType": "LLM",
+ "description": "LLM to use for LLMChallenge",
+ "adapterIdKey": CHALLENGE_LLM_ADAPTER_ID_KEY,
+ }
+
+
+def _make_tool() -> Any:
+ """A `Tool` whose spec carries the adapter-valued `challenge_llm`."""
+ tool = MagicMock(name="Tool")
+ tool.spec = Spec(
+ title="exported-tool",
+ description="Exported prompt studio tool",
+ required=[JsonSchemaKey.CHALLENGE_LLM],
+ properties={
+ JsonSchemaKey.CHALLENGE_LLM: _challenge_llm_property(),
+ JsonSchemaKey.ENABLE_CHALLENGE: {
+ "type": "boolean",
+ "title": "Enable LLMChallenge",
+ "default": False,
+ "description": "Enables LLMChallenge",
+ },
+ },
+ )
+ return tool
+
+
+def _spec_seeded_settings() -> dict[str, Any]:
+ """What `get_default_settings` produces: no spec default -> seeded ""."""
+ return {JsonSchemaKey.CHALLENGE_LLM: "", JsonSchemaKey.ENABLE_CHALLENGE: False}
+
+
+def _exported_tool_settings() -> dict[str, Any]:
+ """A realistic `tool_metadata[tool_settings]` block.
+
+ Deliberately a *superset* of the spec: the export also writes `llm`,
+ `vector-db`, `preamble` and friends, none of which belong in instance
+ metadata.
+ """
+ return {
+ JsonSchemaKey.CHALLENGE_LLM: RESOLVED_CHALLENGE_LLM,
+ JsonSchemaKey.ENABLE_CHALLENGE: True,
+ JsonSchemaKey.SUMMARIZE_AS_SOURCE: True,
+ JsonSchemaKey.ENABLE_HIGHLIGHT: True,
+ "llm": "aaaa1111-0000-0000-0000-000000000000",
+ "vector-db": "bbbb2222-0000-0000-0000-000000000000",
+ "preamble": "Extract carefully.",
+ }
+
+
+def _run_overlay(
+ tool_settings: dict[str, Any],
+ *,
+ resolved: dict[str, Any] | None = None,
+ tool: Any = None,
+ tool_uid: str = PROMPT_REGISTRY_ID,
+ adapter_visible: bool = True,
+ user: Any = None,
+) -> tuple[MagicMock, MagicMock]:
+ """Run the real overlay with the registry lookup and adapter check patched.
+
+ `adapter_visible` models whether the resolved adapter is one the creating
+ user can actually see - False covers both "shared tool, exporter's private
+ adapter" and "adapter has since been deleted".
+
+ Returns the registry-lookup and visibility-check mocks, so a caller can
+ assert on how they were called and not only on the resulting settings.
+ """
+ usable_mock = MagicMock(return_value=adapter_visible)
+ with (
+ patch.object(
+ PromptStudioRegistryHelper,
+ "get_resolved_settings",
+ return_value=resolved or {},
+ ) as resolved_mock,
+ patch.object(ToolInstanceSerializer, "_is_adapter_usable_by", usable_mock),
+ ):
+ ToolInstanceSerializer._overlay_resolved_challenge_llm(
+ tool if tool is not None else _make_tool(),
+ tool_settings,
+ tool_uid,
+ user if user is not None else MagicMock(name="user"),
+ )
+ return resolved_mock, usable_mock
+
+
+def _run_default_adapter_walk(metadata: dict[str, Any]) -> dict[str, Any]:
+ """Run the real default-adapter walk over a `challenge_llm` spec.
+
+ The adapter it offers is `USER_DEFAULT_LLM`, deliberately different from
+ `RESOLVED_CHALLENGE_LLM`, so callers can tell "kept the seeded value" apart
+ from "fell back to the user's default".
+ """
+ spec = Spec(
+ title="exported-tool",
+ description="Exported prompt studio tool",
+ required=[JsonSchemaKey.CHALLENGE_LLM],
+ properties={JsonSchemaKey.CHALLENGE_LLM: _challenge_llm_property()},
+ )
+ adapter = MagicMock(name="AdapterInstance")
+ adapter.id = USER_DEFAULT_LLM
+ ToolInstanceHelper.update_metadata_with_default_adapter(
+ adapter_type=_tih_mod.AdapterTypes.LLM,
+ schema_spec=spec,
+ adapter=adapter,
+ metadata=metadata,
+ )
+ return metadata
+
+
+class TestChallengeLlmOverlay:
+ """`_overlay_resolved_challenge_llm` seeds the real resolved adapter ID."""
+
+ def test_resolved_value_replaces_the_empty_spec_seed(self) -> None:
+ """The "" that fails enum validation is replaced by the real ID."""
+ tool_settings = _spec_seeded_settings()
+ _run_overlay(tool_settings, resolved=_exported_tool_settings())
+ assert tool_settings[JsonSchemaKey.CHALLENGE_LLM] == RESOLVED_CHALLENGE_LLM
+
+ def test_companion_adapter_id_key_is_written_too(self) -> None:
+ """The `adapterIdKey` companion is set, matching every other path.
+
+ `update_metadata_with_default_adapter` always writes the adapter key and
+ its ID key together; seeding only one produces a metadata shape nothing
+ else in the system emits.
+ """
+ tool_settings = _spec_seeded_settings()
+ _run_overlay(tool_settings, resolved=_exported_tool_settings())
+ assert tool_settings[CHALLENGE_LLM_ADAPTER_ID_KEY] == RESOLVED_CHALLENGE_LLM
+
+ def test_companion_key_falls_back_when_spec_omits_adapter_id_key(self) -> None:
+ """A spec property with no `adapterIdKey` uses the generic fallback."""
+ tool = _make_tool()
+ challenge_property = _challenge_llm_property()
+ del challenge_property["adapterIdKey"]
+ tool.spec = Spec(
+ title="no-id-key-tool",
+ description="challenge_llm without an adapterIdKey",
+ required=[JsonSchemaKey.CHALLENGE_LLM],
+ properties={JsonSchemaKey.CHALLENGE_LLM: challenge_property},
+ )
+ tool_settings = {JsonSchemaKey.CHALLENGE_LLM: ""}
+ _run_overlay(tool_settings, resolved=_exported_tool_settings(), tool=tool)
+ assert tool_settings[AdapterPropertyKey.ADAPTER_ID] == RESOLVED_CHALLENGE_LLM
+ assert CHALLENGE_LLM_ADAPTER_ID_KEY not in tool_settings
+
+ def test_only_challenge_llm_is_overlaid(self) -> None:
+ """Scope guard: the other spec/export overlaps stay untouched.
+
+ The export's `tool_settings` also carries `enable_challenge`,
+ `summarize_as_source`, `enable_highlight` and `enable_word_confidence`,
+ all of which are spec properties too. Overlaying them would change
+ whether challenge and summarization actually run -- a separate,
+ behaviour-affecting decision.
+ """
+ tool_settings = _spec_seeded_settings()
+ _run_overlay(tool_settings, resolved=_exported_tool_settings())
+ # Exported as True; must remain the spec default.
+ assert tool_settings[JsonSchemaKey.ENABLE_CHALLENGE] is False
+ # `summarize_as_source` IS a spec property, but it is absent from this
+ # fixture's spec and must not be overlaid on that basis either.
+ assert JsonSchemaKey.SUMMARIZE_AS_SOURCE not in tool_settings
+ # Keys the export carries that are not spec properties at all.
+ for key in ("llm", "vector-db", "preamble"):
+ assert key not in tool_settings
+
+ def test_no_resolved_settings_is_a_no_op(self) -> None:
+ """A non-Prompt-Studio tool resolves to {} and nothing changes."""
+ tool_settings = _spec_seeded_settings()
+ _run_overlay(tool_settings, resolved={}, tool_uid="text_extractor")
+ assert tool_settings == _spec_seeded_settings()
+ assert CHALLENGE_LLM_ADAPTER_ID_KEY not in tool_settings
+
+ def test_empty_resolved_challenge_llm_does_not_overwrite(self) -> None:
+ """An export carrying "" must not write "" plus a bogus ID key."""
+ tool_settings = _spec_seeded_settings()
+ _run_overlay(tool_settings, resolved={JsonSchemaKey.CHALLENGE_LLM: ""})
+ assert CHALLENGE_LLM_ADAPTER_ID_KEY not in tool_settings
+
+ def test_tool_without_challenge_llm_skips_the_lookup(self) -> None:
+ """No `challenge_llm` in the spec -> no registry query at all."""
+ tool = _make_tool()
+ tool.spec = Spec(
+ title="plain-tool",
+ description="A tool with no challenge LLM",
+ required=[],
+ properties={"some_setting": {"type": "string", "default": "x"}},
+ )
+ tool_settings = {"some_setting": "x"}
+ resolved_mock, _usable_mock = _run_overlay(tool_settings, tool=tool)
+ resolved_mock.assert_not_called()
+
+
+class TestOverlaySkipsUnusableAdapters:
+ """A resolved id the creating user cannot use must not be seeded.
+
+ The deploy-time enum is built per-user. Seeding an id outside it would
+ trade the "" enum violation for an equally opaque one naming a real UUID,
+ and would suppress the default-adapter fill that previously healed it.
+ """
+
+ def test_inaccessible_adapter_is_not_seeded(self) -> None:
+ """Shared tool, exporter's private adapter -> leave the seed empty."""
+ tool_settings = _spec_seeded_settings()
+ _run_overlay(
+ tool_settings,
+ resolved=_exported_tool_settings(),
+ adapter_visible=False,
+ )
+ assert tool_settings[JsonSchemaKey.CHALLENGE_LLM] == ""
+ assert CHALLENGE_LLM_ADAPTER_ID_KEY not in tool_settings
+
+ def test_left_empty_value_still_gets_the_user_default(self) -> None:
+ """The skipped seed must remain healable by the default-adapter walk.
+
+ This is the whole point of not seeding: "" is falsy, so the guard in
+ `update_metadata_with_default_adapter` fills it with the user's own
+ default LLM rather than preserving an unusable id.
+ """
+ tool_settings = _spec_seeded_settings()
+ _run_overlay(
+ tool_settings,
+ resolved=_exported_tool_settings(),
+ adapter_visible=False,
+ )
+ metadata = _run_default_adapter_walk(
+ {JsonSchemaKey.CHALLENGE_LLM: tool_settings[JsonSchemaKey.CHALLENGE_LLM]}
+ )
+ assert metadata[JsonSchemaKey.CHALLENGE_LLM] == USER_DEFAULT_LLM
+ assert metadata[CHALLENGE_LLM_ADAPTER_ID_KEY] == USER_DEFAULT_LLM
+
+ def test_usable_adapter_is_checked_against_the_creating_user(self) -> None:
+ """The visibility check receives the resolved id and the user."""
+ user = MagicMock(name="creating-user")
+ _resolved_mock, usable_mock = _run_overlay(
+ _spec_seeded_settings(), resolved=_exported_tool_settings(), user=user
+ )
+ usable_mock.assert_called_once_with(RESOLVED_CHALLENGE_LLM, user)
+
+
+class TestIsAdapterUsableBy:
+ """Direct coverage of the visibility check's own body.
+
+ Every other test patches this method out, so without these a `return True`
+ or `return False` stub would keep the whole suite green - while either
+ silently reintroduces the deploy-time 422 or makes the seed a no-op.
+ """
+
+ @staticmethod
+ def _run(user: Any, *, exists: bool = True) -> tuple[bool, MagicMock]:
+ exists_qs = MagicMock(name="filtered")
+ exists_qs.exists.return_value = exists
+ scoped = MagicMock(name="scoped")
+ scoped.filter.return_value = exists_qs
+ objects_mock = MagicMock(name="objects")
+ objects_mock.for_user.return_value = scoped
+ # Wired too, so `filter.assert_not_called()` means "never bypassed
+ # for_user" rather than "would have blown up if it had".
+ objects_mock.filter.return_value = exists_qs
+ with patch.object(_ser_mod, "AdapterInstance", MagicMock(objects=objects_mock)):
+ result = ToolInstanceSerializer._is_adapter_usable_by(
+ RESOLVED_CHALLENGE_LLM, user
+ )
+ return result, objects_mock
+
+ def test_scopes_the_lookup_to_the_user(self) -> None:
+ """The query must go through `for_user`, not the raw manager.
+
+ This is what keeps the check aligned with the per-user enum built at
+ deploy time; an unscoped lookup would pass another user's adapter.
+ """
+ user = MagicMock(name="user")
+ result, objects_mock = self._run(user)
+ assert result is True
+ objects_mock.for_user.assert_called_once_with(user)
+ objects_mock.filter.assert_not_called()
+
+ def test_filters_on_id_and_llm_adapter_type(self) -> None:
+ """Both predicates must match `AdapterProcessor.get_adapters_by_type`.
+
+ Dropping the type predicate lets a visible non-LLM adapter through,
+ which the LLM enum then rejects at deploy - the same relocated 422.
+ """
+ _result, objects_mock = self._run(MagicMock(name="user"))
+ objects_mock.for_user.return_value.filter.assert_called_once_with(
+ id=RESOLVED_CHALLENGE_LLM, adapter_type=AdapterTypes.LLM.value
+ )
+
+ def test_absent_adapter_is_not_usable(self) -> None:
+ """A deleted or invisible adapter resolves to False."""
+ result, _objects_mock = self._run(MagicMock(name="user"), exists=False)
+ assert result is False
+
+ def test_missing_user_refuses_rather_than_falling_back(self) -> None:
+ """No user means no scope - refuse instead of seeding on existence.
+
+ Answering "yes" here would fail open: any adapter that merely exists,
+ including another user's private one, would be seeded.
+ """
+ result, objects_mock = self._run(None)
+ assert result is False
+ objects_mock.for_user.assert_not_called()
+ objects_mock.filter.assert_not_called()
+
+
+class TestOverlayIsWiredIntoCreate:
+ """The overlay's RESULT must reach the metadata `create` persists.
+
+ Asserting only that the overlay is *called* is not enough: the call could
+ run against a dict nobody reads again, or the metadata could be rebuilt
+ from unoverlaid defaults afterwards. Both leave a called-but-discarded
+ overlay, which is the same class of wiring gap as never calling it.
+ """
+
+ @staticmethod
+ def _create_and_capture_metadata(
+ usable_mock: MagicMock | None = None,
+ request_user: Any = None,
+ ) -> dict[str, Any]:
+ """Run the real `create` with the real overlay; return its metadata.
+
+ `create` mutates the `validated_data` dict in place, so the metadata it
+ would persist is readable afterwards. The serializer is built with
+ request context, as `get_serializer` does in the view, so the
+ request->user hop the visibility check depends on is exercised.
+ """
+ workflow = MagicMock(name="Workflow")
+ workflow.tool_instances.count.return_value = 0
+ workflow.is_active = True
+
+ validated_data: dict[str, Any] = {
+ "workflow_id": "wf-1",
+ "tool_id": PROMPT_REGISTRY_ID,
+ }
+ request = MagicMock(name="request")
+ # `name=`, not positional: MagicMock's first positional arg is `spec`,
+ # which would build a str-specced mock that raises AttributeError the
+ # moment a test lets the real visibility check touch a User attribute.
+ request.user = (
+ request_user if request_user is not None else MagicMock(name="user")
+ )
+
+ with (
+ patch.object(
+ _ser_mod.Workflow,
+ "objects",
+ MagicMock(get=MagicMock(return_value=workflow)),
+ ),
+ patch.object(
+ _ser_mod.ToolProcessor,
+ "get_tool_by_uid",
+ MagicMock(return_value=_make_tool()),
+ ),
+ patch.object(
+ _ser_mod.ToolProcessor,
+ "get_default_settings",
+ # `side_effect`, not `return_value`: a single shared dict would
+ # already carry the overlay's mutations, so metadata rebuilt
+ # from a second call would look correct even when the wiring
+ # is broken.
+ MagicMock(side_effect=lambda _tool: _spec_seeded_settings()),
+ ),
+ patch.object(
+ PromptStudioRegistryHelper,
+ "get_resolved_settings",
+ MagicMock(return_value=_exported_tool_settings()),
+ ),
+ patch.object(
+ ToolInstanceSerializer,
+ "_is_adapter_usable_by",
+ usable_mock if usable_mock is not None else MagicMock(return_value=True),
+ ),
+ patch.object(
+ _ser_mod.AuditSerializer, "create", MagicMock(return_value=MagicMock())
+ ),
+ ):
+ ToolInstanceSerializer(context={"request": request}).create(validated_data)
+
+ return validated_data[_ser_mod.TIKey.METADATA]
+
+ def test_seeded_value_reaches_persisted_metadata(self) -> None:
+ """The resolved id must survive all the way into instance metadata."""
+ metadata = self._create_and_capture_metadata()
+ assert metadata[JsonSchemaKey.CHALLENGE_LLM] == RESOLVED_CHALLENGE_LLM
+
+ def test_companion_key_reaches_persisted_metadata(self) -> None:
+ """So must its companion, or the pair is half-written on disk."""
+ metadata = self._create_and_capture_metadata()
+ assert metadata[CHALLENGE_LLM_ADAPTER_ID_KEY] == RESOLVED_CHALLENGE_LLM
+
+ def test_creating_user_reaches_the_visibility_check(self) -> None:
+ """`create` must forward the request's user, not lose it.
+
+ This hop is what makes the check per-user. If it degrades to None the
+ check has no scope to test against, so seeding would either fail open
+ on mere existence or stop working entirely - and no assertion about
+ metadata contents would notice.
+ """
+ request_user = MagicMock(name="request-user")
+ usable_mock = MagicMock(return_value=True)
+ self._create_and_capture_metadata(
+ usable_mock=usable_mock, request_user=request_user
+ )
+ usable_mock.assert_called_once_with(RESOLVED_CHALLENGE_LLM, request_user)
+
+
+class TestSeededValuePassesDeployValidation:
+ """End-to-end: the seeded value must satisfy the enum deploy checks against.
+
+ This is the assertion that ties the seed to the 422 the change exists to
+ prevent. Everything else here checks that a string was copied between
+ dicts; this checks the copied string is one the real, user-scoped schema
+ accepts -- and that the pre-fix "" is not.
+ """
+
+ @staticmethod
+ def _user_scoped_schema() -> dict[str, Any]:
+ """The real tool-instance schema, with the real enum injected.
+
+ `update_schema_with_adapter_configurations` builds the enum from the
+ adapters visible to the user, which is exactly what makes a foreign or
+ deleted `challenge_llm` fail at deploy time.
+ """
+ tool = _make_tool()
+ schema = ToolUtils.get_json_schema_for_tool(tool)
+ visible_adapter = MagicMock(name="AdapterInstance")
+ visible_adapter.id = RESOLVED_CHALLENGE_LLM
+ visible_adapter.adapter_name = "Challenger LLM"
+ with patch.object(
+ _tp_mod.AdapterProcessor,
+ "get_adapters_by_type",
+ MagicMock(return_value=[visible_adapter]),
+ ):
+ _tp_mod.ToolProcessor.update_schema_with_adapter_configurations(
+ schema=schema, user=MagicMock(name="user")
+ )
+ return schema.to_dict()
+
+ def test_enum_is_actually_injected_for_challenge_llm(self) -> None:
+ """Guard the guard: without an enum the rest of this class is inert."""
+ schema = self._user_scoped_schema()
+ enum = schema["properties"][JsonSchemaKey.CHALLENGE_LLM].get("enum")
+ assert enum == [RESOLVED_CHALLENGE_LLM]
+
+ def test_seeded_value_validates(self) -> None:
+ """A value produced by the overlay passes the deploy-time check."""
+ tool_settings = _spec_seeded_settings()
+ _run_overlay(tool_settings, resolved=_exported_tool_settings())
+ errors = list(
+ jsonschema.Draft7Validator(self._user_scoped_schema()).iter_errors(
+ {JsonSchemaKey.CHALLENGE_LLM: tool_settings[JsonSchemaKey.CHALLENGE_LLM]}
+ )
+ )
+ assert errors == []
+
+ def test_pre_fix_empty_seed_still_fails(self) -> None:
+ """Discrimination check: "" must still be rejected by the same schema.
+
+ If this ever passes, the test above proves nothing.
+ """
+ errors = list(
+ jsonschema.Draft7Validator(self._user_scoped_schema()).iter_errors(
+ {JsonSchemaKey.CHALLENGE_LLM: ""}
+ )
+ )
+ assert any(error.validator == "enum" for error in errors)
+
+
+class TestDefaultAdapterDoesNotClobberSeededValue:
+ """`update_metadata_with_default_adapter` must not stomp a seeded value.
+
+ This is the half of the bug that made the seed a no-op in the common case:
+ the walk runs right after creation and `challenge_llm` is an LLM adapter
+ property, so a user with a default LLM had the exported value replaced.
+ """
+
+ def test_seeded_challenge_llm_survives_the_default_walk(self) -> None:
+ """The exported value is kept, not replaced by the user's default."""
+ metadata = _run_default_adapter_walk(
+ {
+ JsonSchemaKey.CHALLENGE_LLM: RESOLVED_CHALLENGE_LLM,
+ CHALLENGE_LLM_ADAPTER_ID_KEY: RESOLVED_CHALLENGE_LLM,
+ }
+ )
+ assert metadata[JsonSchemaKey.CHALLENGE_LLM] == RESOLVED_CHALLENGE_LLM
+ assert metadata[CHALLENGE_LLM_ADAPTER_ID_KEY] == RESOLVED_CHALLENGE_LLM
+
+ def test_kept_value_never_leaves_a_half_written_pair(self) -> None:
+ """Skipping the write must not strand a missing companion ID key.
+
+ `update_metadata_with_adapter_properties` warns that a UUID-shaped value
+ at the adapter key with no matching ID key "bypasses the lazy migrator
+ and fails schema enum validation later". Preserving a value must not
+ manufacture that shape.
+ """
+ metadata = _run_default_adapter_walk(
+ {JsonSchemaKey.CHALLENGE_LLM: RESOLVED_CHALLENGE_LLM}
+ )
+ assert metadata[JsonSchemaKey.CHALLENGE_LLM] == RESOLVED_CHALLENGE_LLM
+ assert metadata[CHALLENGE_LLM_ADAPTER_ID_KEY] == RESOLVED_CHALLENGE_LLM
+
+ def test_guard_applies_to_non_llm_adapter_types_too(self) -> None:
+ """The guard sits after the type dispatch, so it covers every type.
+
+ `challenge_llm` is the only adapter-valued property in the Prompt Studio
+ spec today, so LLM is the only type that can currently hit this. Pinning
+ a second type keeps the breadth deliberate rather than incidental: a
+ future spec with an adapter-valued X2TEXT property must behave the same.
+ """
+ x2text_property = {
+ "type": "string",
+ "title": "Text extractor",
+ "adapterType": "X2TEXT",
+ "adapterIdKey": "x2text_adapter_id",
+ }
+ spec = Spec(
+ title="x2text-tool",
+ description="A tool with an adapter-valued X2TEXT property",
+ required=[],
+ properties={"x2text_llm": x2text_property},
+ )
+ adapter = MagicMock(name="AdapterInstance")
+ adapter.id = USER_DEFAULT_LLM
+ metadata = {"x2text_llm": RESOLVED_CHALLENGE_LLM}
+ ToolInstanceHelper.update_metadata_with_default_adapter(
+ adapter_type=_tih_mod.AdapterTypes.X2TEXT,
+ schema_spec=spec,
+ adapter=adapter,
+ metadata=metadata,
+ )
+ assert metadata["x2text_llm"] == RESOLVED_CHALLENGE_LLM
+ assert metadata["x2text_adapter_id"] == RESOLVED_CHALLENGE_LLM
+
+ def test_unset_challenge_llm_still_gets_the_user_default(self) -> None:
+ """The guard fills gaps -- it must not disable the default entirely."""
+ metadata = _run_default_adapter_walk({JsonSchemaKey.CHALLENGE_LLM: ""})
+ assert metadata[JsonSchemaKey.CHALLENGE_LLM] == USER_DEFAULT_LLM
+ assert metadata[CHALLENGE_LLM_ADAPTER_ID_KEY] == USER_DEFAULT_LLM
+
+ def test_absent_key_still_gets_the_user_default(self) -> None:
+ """A key missing altogether is a gap too, and must be filled."""
+ metadata = _run_default_adapter_walk({})
+ assert metadata[JsonSchemaKey.CHALLENGE_LLM] == USER_DEFAULT_LLM
+
+
+class TestGetResolvedSettings:
+ """The registry lookup reads the right block and stays quiet when normal."""
+
+ def test_reads_the_tool_settings_block(self) -> None:
+ settings_block = {JsonSchemaKey.CHALLENGE_LLM: RESOLVED_CHALLENGE_LLM}
+ metadata = {
+ JsonSchemaKey.TOOL_SETTINGS: settings_block,
+ # A sibling key that must not be mistaken for the settings block.
+ "outputs": [{"prompt": "x"}],
+ }
+ values_qs = MagicMock(name="values_list")
+ values_qs.get.return_value = metadata
+ objects_mock = MagicMock(values_list=MagicMock(return_value=values_qs))
+ with patch.object(_psr_mod.PromptStudioRegistry, "objects", objects_mock):
+ resolved = PromptStudioRegistryHelper.get_resolved_settings(
+ PROMPT_REGISTRY_ID
+ )
+ assert resolved == settings_block
+ # Pin the query shape: the wrong column, or a dropped `flat=True`,
+ # makes `.get()` return a 1-tuple and blows up on every real creation.
+ objects_mock.values_list.assert_called_once_with("tool_metadata", flat=True)
+
+ def test_non_uuid_tool_id_short_circuits_without_a_query(self) -> None:
+ """Registry tools use slugs; those must not reach the DB or the log.
+
+ `ToolInstance.tool_id` is a free-form CharField, so the common case --
+ a built-in tool like "text_extractor" -- is not UUID-shaped at all.
+ """
+ objects_mock = MagicMock(name="objects")
+ with patch.object(_psr_mod.PromptStudioRegistry, "objects", objects_mock):
+ assert (
+ PromptStudioRegistryHelper.get_resolved_settings("text_extractor") == {}
+ )
+ objects_mock.values_list.assert_not_called()
+
+ def test_missing_registry_row_returns_empty(self) -> None:
+ values_qs = MagicMock(name="values_list")
+ values_qs.get.side_effect = _psr_mod.PromptStudioRegistry.DoesNotExist
+ with patch.object(
+ _psr_mod.PromptStudioRegistry,
+ "objects",
+ MagicMock(values_list=MagicMock(return_value=values_qs)),
+ ):
+ assert (
+ PromptStudioRegistryHelper.get_resolved_settings(PROMPT_REGISTRY_ID) == {}
+ )
+
+ def test_row_without_tool_settings_returns_empty(self) -> None:
+ values_qs = MagicMock(name="values_list")
+ values_qs.get.return_value = None
+ with patch.object(
+ _psr_mod.PromptStudioRegistry,
+ "objects",
+ MagicMock(values_list=MagicMock(return_value=values_qs)),
+ ):
+ assert (
+ PromptStudioRegistryHelper.get_resolved_settings(PROMPT_REGISTRY_ID) == {}
+ )
+
+ def test_db_errors_are_not_swallowed(self) -> None:
+ """An `OperationalError` must propagate, not degrade to "".
+
+ Swallowing it would seed "" and resurface much later as the same
+ confusing 422 this change exists to prevent.
+ """
+ values_qs = MagicMock(name="values_list")
+ values_qs.get.side_effect = OperationalError("db down")
+ with patch.object(
+ _psr_mod.PromptStudioRegistry,
+ "objects",
+ MagicMock(values_list=MagicMock(return_value=values_qs)),
+ ):
+ with pytest.raises(OperationalError):
+ PromptStudioRegistryHelper.get_resolved_settings(PROMPT_REGISTRY_ID)
diff --git a/backend/tool_instance_v2/tool_instance_helper.py b/backend/tool_instance_v2/tool_instance_helper.py
index 0c7eb9d04..138e48b5b 100644
--- a/backend/tool_instance_v2/tool_instance_helper.py
+++ b/backend/tool_instance_v2/tool_instance_helper.py
@@ -74,6 +74,19 @@ class ToolInstanceHelper:
tool_instance.metadata = metadata
tool_instance.save()
+ @staticmethod
+ def get_adapter_id_key(adapter_property: dict[str, Any]) -> str:
+ """The metadata key holding a spec property's companion adapter ID.
+
+ Every writer stores an adapter key and this companion together; a
+ populated adapter key with a missing ID key bypasses the lazy migrator
+ and then fails schema enum validation. Resolved in one place so the
+ writers cannot drift on where that companion lives.
+ """
+ return adapter_property.get(
+ AdapterPropertyKey.ADAPTER_ID_KEY, AdapterPropertyKey.ADAPTER_ID
+ )
+
@staticmethod
def update_metadata_with_adapter_properties(
metadata: dict[str, Any],
@@ -114,9 +127,7 @@ class ToolInstanceHelper:
adapter_type=adapter_type, adapter_name=adapter_value
)
adapter_id = str(adapter.id)
- metadata_key_for_id = adapter_property.get(
- AdapterPropertyKey.ADAPTER_ID_KEY, AdapterPropertyKey.ADAPTER_ID
- )
+ metadata_key_for_id = ToolInstanceHelper.get_adapter_id_key(adapter_property)
# Keep adapter_key and adapter_id_key both canonical to the resolved
# target UUID; otherwise stale UUID-shaped values at adapter_key
# bypass the lazy migrator and fail schema enum validation later.
@@ -216,9 +227,24 @@ class ToolInstanceHelper:
if adapter_type == AdapterTypes.OCR:
properties = schema_spec.get_ocr_adapter_properties()
for adapter_key, adapter_property in properties.items():
- metadata_key_for_id = adapter_property.get(
- AdapterPropertyKey.ADAPTER_ID_KEY, AdapterPropertyKey.ADAPTER_ID
- )
+ metadata_key_for_id = ToolInstanceHelper.get_adapter_id_key(adapter_property)
+ # Only fill in what has no value yet. This runs right after tool
+ # instance creation, which may already have seeded a key with a
+ # value resolved for that specific tool - `challenge_llm` carries
+ # the exported project's challenger LLM, and it is one of the LLM
+ # adapter properties walked here. Overwriting it would replace a
+ # tool-specific choice with the user's global default.
+ existing_value = metadata.get(adapter_key)
+ if existing_value:
+ # Keep the pair consistent for whoever populated the adapter
+ # key. No caller reaching here today leaves the ID key unset -
+ # this holds the invariant rather than repairing a known writer.
+ # Cheap and worth keeping: a populated adapter key with a
+ # missing ID key bypasses the lazy migrator and then fails
+ # schema enum validation (see
+ # `update_metadata_with_adapter_properties`).
+ metadata.setdefault(metadata_key_for_id, existing_value)
+ continue
metadata[adapter_key] = str(adapter.id)
metadata[metadata_key_for_id] = str(adapter.id)