LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
[FIX] Seed challenge_llm from the exported tool's resolved value so challenge-enabled deploys pass (#2209)
Hari John Kuriakose66a3363fbaeb
diff --git a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.pyindex f46298be3..8d88d2c00 100644--- a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py+++ b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py@@ -1,4 +1,5 @@import logging+import uuidfrom typing import Anyfrom account_v2.models import User@@ -143,6 +144,49 @@ class PromptStudioRegistryHelper:image_tag=settings.STRUCTURE_TOOL_IMAGE_TAG,)+ @staticmethod+ def get_resolved_settings(prompt_registry_id: str) -> dict[str, Any]:+ """Return the settings export already resolved for this exported tool.++ ``frame_export_json`` resolves adapter-valued settings at export time -+ notably ``challenge_llm``, which falls back to the default profile's LLM+ when the project set none - and stores them under+ ``tool_metadata[tool_settings]``. ``Tool`` (built from ``tool_spec`` /+ ``tool_property``) does not carry them, so callers that only have a+ ``Tool`` cannot see the resolved values.++ Returns an empty dict when the registry row is missing or carries no+ settings, so callers can treat "no resolved settings" as a no-op.+ """+ # `ToolInstance.tool_id` is a free-form CharField and registry tools use+ # slugs ("classify", "text_extractor"), which a UUID pk lookup would+ # reject. Screening them out here keeps this a silent no-op for every+ # non-Prompt-Studio tool instead of logging on a fully normal path.+ # Checked inline rather than via `ToolInstanceHelper.is_uuid_format`:+ # `tool_instance_v2.tool_processor` already imports this module at+ # module level, so importing back into `tool_instance_v2` would cycle.+ try:+ uuid.UUID(str(prompt_registry_id))+ except ValueError:+ return {}+ try:+ # Only `tool_metadata` is needed, and it is the largest column on the+ # row - the full JSONB blob of every prompt and output. Fetching just+ # that column avoids pulling the rest a second time, since the caller+ # has already loaded this row to build the `Tool`.+ metadata = PromptStudioRegistry.objects.values_list(+ "tool_metadata", flat=True+ ).get(pk=prompt_registry_id)+ except PromptStudioRegistry.DoesNotExist:+ # A UUID-shaped tool_id with no registry row - e.g. an agentic tool+ # in cloud. Expected, so this stays at debug.+ logger.debug(+ f"No prompt studio registry entry for {prompt_registry_id}; "+ "no resolved settings to apply"+ )+ return {}+ return (metadata or {}).get(JsonSchemaKey.TOOL_SETTINGS, {}) or {}+@staticmethoddef update_or_create_psr_tool(custom_tool: CustomTool,diff --git a/backend/tool_instance_v2/serializers.py b/backend/tool_instance_v2/serializers.pyindex 81eb6e0e6..049effe52 100644--- a/backend/tool_instance_v2/serializers.py+++ b/backend/tool_instance_v2/serializers.py@@ -2,13 +2,21 @@ import loggingimport uuidfrom typing import Any+from account_v2.models import Userfrom adapter_processor_v2.adapter_processor import AdapterProcessorfrom adapter_processor_v2.models import AdapterInstance-from prompt_studio.prompt_studio_registry_v2.constants import PromptStudioRegistryKeys+from prompt_studio.prompt_studio_registry_v2.constants import (+ JsonSchemaKey,+ PromptStudioRegistryKeys,+)+from prompt_studio.prompt_studio_registry_v2.prompt_studio_registry_helper import (+ PromptStudioRegistryHelper,+)from rest_framework.serializers import ListField, Serializer, UUIDField, ValidationErrorfrom workflow_manager.workflow_v2.constants import WorkflowKeyfrom workflow_manager.workflow_v2.models.workflow import Workflow+from backend.constants import RequestKeyfrom backend.serializers import AuditSerializerfrom tool_instance_v2.constants import ToolInstanceKey as TIKeyfrom tool_instance_v2.constants import ToolKey@@ -148,6 +156,104 @@ class ToolInstanceSerializer(AuditSerializer):logger.error(f"Error transforming adapter IDs to names: {e}", exc_info=True)return display_metadata+ @staticmethod+ def _overlay_resolved_challenge_llm(+ tool: Tool,+ tool_settings: dict[str, Any],+ tool_uid: str,+ user: User | None,+ ) -> None:+ """Seed `challenge_llm` from the value the export already resolved.++ `get_default_settings` walks the spec, and an adapter-valued property+ with no `default` is seeded "". `challenge_llm` has none, so the+ instance would store "" - and because the property declares+ `adapterType: "LLM"`, an enum of real adapter IDs is injected into the+ tool-instance schema, which "" then fails at deployment validation.++ The export resolved a real adapter ID for this setting (falling back to+ the default profile's LLM when the project set none), so prefer it.++ Only seed a value this user can actually use. The enum injected at+ deployment is built per-user, and an exported tool can be shared, so the+ exporter's adapter may be invisible to whoever creates the instance -+ and it may since have been deleted. Seeding such an id would trade the+ "" enum violation for an equally opaque one naming a real UUID. Leaving+ the key empty instead lets `update_metadata_with_default_adapter` fill+ in this user's default LLM, which is what happened before the export+ value was available at all.++ Deliberately scoped to `challenge_llm` alone. The export's+ `tool_settings` overlaps the spec on four other keys+ (`enable_challenge`, `summarize_as_source`, `enable_highlight`,+ `enable_word_confidence`); overlaying those would change whether+ challenge, summarization and highlighting actually *run* on new+ instances - a cost-, latency- and output-affecting change, and a+ separate decision from fixing this validation failure.++ Tools whose spec has no `challenge_llm` return before any query; a+ Prompt-Studio-shaped tool with no registry row resolves to {}.+ """+ challenge_llm_key = JsonSchemaKey.CHALLENGE_LLM+ if challenge_llm_key not in tool_settings:+ return++ resolved_settings = PromptStudioRegistryHelper.get_resolved_settings(+ prompt_registry_id=tool_uid+ )+ resolved_challenge_llm = resolved_settings.get(challenge_llm_key)+ if not resolved_challenge_llm:+ logger.debug(+ "Export for tool %s resolved no challenge_llm; leaving the "+ "spec-seeded default in place",+ tool_uid,+ )+ return++ if not ToolInstanceSerializer._is_adapter_usable_by(resolved_challenge_llm, user):+ logger.warning(+ "Resolved challenge_llm %s for tool %s is not accessible to the "+ "creating user; falling back to their default LLM adapter",+ resolved_challenge_llm,+ tool_uid,+ )+ return++ tool_settings[challenge_llm_key] = resolved_challenge_llm+ # Write the companion ID key alongside, the shape every other writer+ # emits (see `ToolInstanceHelper.update_metadata_with_default_adapter`).+ adapter_property = tool.spec.properties.get(challenge_llm_key, {})+ adapter_id_key = ToolInstanceHelper.get_adapter_id_key(adapter_property)+ tool_settings[adapter_id_key] = resolved_challenge_llm++ @staticmethod+ def _is_adapter_usable_by(adapter_id: str, user: User | None) -> bool:+ """Whether `adapter_id` is an LLM adapter visible to `user`.++ Mirrors `AdapterProcessor.get_adapters_by_type` exactly - same+ `for_user` scoping, same `adapter_type` predicate - because that builds+ the enum this value is validated against at deploy time, and any+ divergence lets a value through here that the enum then rejects. The+ type predicate is load-bearing: `CustomTool.challenge_llm` is an+ unconstrained FK whose serializer has no `validate_challenge_llm`,+ unlike its `summarize_llm_adapter` sibling, so a non-LLM adapter can+ reach it.++ Without a user there is no scope to check against, and answering "yes"+ would seed on mere existence - passing another user's private adapter.+ Refuse instead: the caller then leaves the key empty and the default+ adapter fills it, which is safe in every case. Unreachable today, since+ every site that constructs this serializer to write goes through+ `get_serializer`, which supplies the request.+ """+ if user is None:+ return False+ return (+ AdapterInstance.objects.for_user(user)+ .filter(id=adapter_id, adapter_type=AdapterTypes.LLM.value)+ .exists()+ )+def create(self, validated_data: dict[str, Any]) -> Any:workflow_id = validated_data.pop(WorkflowKey.WF_ID)try:@@ -170,11 +276,19 @@ class ToolInstanceSerializer(AuditSerializer):validated_data[TIKey.PK] = uuid.uuid4()# TODO: Use version from tool propsvalidated_data[TIKey.VERSION] = ""+ tool_settings = ToolProcessor.get_default_settings(tool)+ request = self.context.get(RequestKey.REQUEST)+ self._overlay_resolved_challenge_llm(+ tool,+ tool_settings,+ str(tool_uid),+ getattr(request, "user", None),+ )validated_data[TIKey.METADATA] = {# TODO: Review and remove tool instance IDWorkflowKey.WF_TOOL_INSTANCE_ID: str(validated_data[TIKey.PK]),PromptStudioRegistryKeys.PROMPT_REGISTRY_ID: str(tool_uid),- **ToolProcessor.get_default_settings(tool),+ **tool_settings,}if TIKey.STEP not in validated_data:validated_data[TIKey.STEP] = workflow.tool_instances.count() + 1diff --git a/backend/tool_instance_v2/tests/__init__.py b/backend/tool_instance_v2/tests/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/tool_instance_v2/tests/test_challenge_llm_seed_overlay.py b/backend/tool_instance_v2/tests/test_challenge_llm_seed_overlay.pynew file mode 100644index 000000000..5ce0f232b--- /dev/null+++ b/backend/tool_instance_v2/tests/test_challenge_llm_seed_overlay.py@@ -0,0 +1,690 @@+"""Regression tests for seeding a tool instance's ``challenge_llm`` from the+exported tool's resolved settings.++The deploy-time failure this closes: a tool exported with LLMChallenge *enabled*+still ended in pipeline status ERROR with+``422 Unprocessable Entity: Tool validation failed``.++The chain:++ 1. Export resolves a real ``challenge_llm`` (falling back to the default+ profile's LLM) and stores it under ``tool_metadata[tool_settings]``.+ 2. Tool-instance creation seeds metadata from ``get_default_settings``, which+ walks the *spec*: a ``"type": "string"`` property with no ``default`` is+ seeded ``""``. ``challenge_llm`` has no spec default, so the instance+ stores ``""`` -- throwing away the value export already resolved.+ 3. Because ``challenge_llm`` declares ``adapterType: "LLM"``,+ ``_update_schema_for_adapter_type`` injects ``enum: [<real adapter ids>]``.+ 4. Validation rejects ``""`` (an enum violation).++There is a second half to the bug, and it is why the seed alone was not enough:+``ToolInstanceViewSet.create`` calls ``update_metadata_with_default_values``+immediately after ``perform_create``, and ``challenge_llm`` is one of the LLM+adapter properties that walks. Before the guard added alongside these tests, it+unconditionally overwrote the key with the user's *default* LLM -- so the seed+was discarded for every user who has one, which is most of them.++Unit tests: the real modules are imported (Django is configured by the rig's+``unit-backend`` group env -- see ``tests/groups.yaml``) and every collaborator+is patched per-test, so no database is touched.+"""++from __future__ import annotations++from typing import Any+from unittest.mock import MagicMock, patch++import jsonschema+import pytest+from django.db import OperationalError+from prompt_studio.prompt_studio_registry_v2 import (+ prompt_studio_registry_helper as _psr_mod,+)+from prompt_studio.prompt_studio_registry_v2.constants import JsonSchemaKey++from tool_instance_v2 import serializers as _ser_mod+from tool_instance_v2 import tool_instance_helper as _tih_mod+from tool_instance_v2 import tool_processor as _tp_mod+from unstract.sdk1.adapters.enums import AdapterTypes+from unstract.tool_registry.constants import AdapterPropertyKey+from unstract.tool_registry.dto import Spec+from unstract.tool_registry.tool_utils import ToolUtils++PromptStudioRegistryHelper = _psr_mod.PromptStudioRegistryHelper+ToolInstanceSerializer = _ser_mod.ToolInstanceSerializer+ToolInstanceHelper = _tih_mod.ToolInstanceHelper++# The adapter ID the export resolved for the project's challenger LLM.+RESOLVED_CHALLENGE_LLM = "11111111-2222-3333-4444-555555555555"+# The deploying user's default LLM -- deliberately different, so a test can tell+# "kept the exported value" apart from "fell back to the user's default".+USER_DEFAULT_LLM = "99999999-8888-7777-6666-555555555555"++PROMPT_REGISTRY_ID = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"++CHALLENGE_LLM_ADAPTER_ID_KEY = "challenge_llm_adapter_id"+++def _challenge_llm_property() -> dict[str, Any]:+ """The `challenge_llm` spec property exactly as `frame_spec` emits it."""+ return {+ "type": "string",+ "title": "Challenger LLM",+ "adapterType": "LLM",+ "description": "LLM to use for LLMChallenge",+ "adapterIdKey": CHALLENGE_LLM_ADAPTER_ID_KEY,+ }+++def _make_tool() -> Any:+ """A `Tool` whose spec carries the adapter-valued `challenge_llm`."""+ tool = MagicMock(name="Tool")+ tool.spec = Spec(+ title="exported-tool",+ description="Exported prompt studio tool",+ required=[JsonSchemaKey.CHALLENGE_LLM],+ properties={+ JsonSchemaKey.CHALLENGE_LLM: _challenge_llm_property(),+ JsonSchemaKey.ENABLE_CHALLENGE: {+ "type": "boolean",+ "title": "Enable LLMChallenge",+ "default": False,+ "description": "Enables LLMChallenge",+ },+ },+ )+ return tool+++def _spec_seeded_settings() -> dict[str, Any]:+ """What `get_default_settings` produces: no spec default -> seeded ""."""+ return {JsonSchemaKey.CHALLENGE_LLM: "", JsonSchemaKey.ENABLE_CHALLENGE: False}+++def _exported_tool_settings() -> dict[str, Any]:+ """A realistic `tool_metadata[tool_settings]` block.++ Deliberately a *superset* of the spec: the export also writes `llm`,+ `vector-db`, `preamble` and friends, none of which belong in instance+ metadata.+ """+ return {+ JsonSchemaKey.CHALLENGE_LLM: RESOLVED_CHALLENGE_LLM,+ JsonSchemaKey.ENABLE_CHALLENGE: True,+ JsonSchemaKey.SUMMARIZE_AS_SOURCE: True,+ JsonSchemaKey.ENABLE_HIGHLIGHT: True,+ "llm": "aaaa1111-0000-0000-0000-000000000000",+ "vector-db": "bbbb2222-0000-0000-0000-000000000000",+ "preamble": "Extract carefully.",+ }+++def _run_overlay(+ tool_settings: dict[str, Any],+ *,+ resolved: dict[str, Any] | None = None,+ tool: Any = None,+ tool_uid: str = PROMPT_REGISTRY_ID,+ adapter_visible: bool = True,+ user: Any = None,+) -> tuple[MagicMock, MagicMock]:+ """Run the real overlay with the registry lookup and adapter check patched.++ `adapter_visible` models whether the resolved adapter is one the creating+ user can actually see - False covers both "shared tool, exporter's private+ adapter" and "adapter has since been deleted".++ Returns the registry-lookup and visibility-check mocks, so a caller can+ assert on how they were called and not only on the resulting settings.+ """+ usable_mock = MagicMock(return_value=adapter_visible)+ with (+ patch.object(+ PromptStudioRegistryHelper,+ "get_resolved_settings",+ return_value=resolved or {},+ ) as resolved_mock,+ patch.object(ToolInstanceSerializer, "_is_adapter_usable_by", usable_mock),+ ):+ ToolInstanceSerializer._overlay_resolved_challenge_llm(+ tool if tool is not None else _make_tool(),+ tool_settings,+ tool_uid,+ user if user is not None else MagicMock(name="user"),+ )+ return resolved_mock, usable_mock+++def _run_default_adapter_walk(metadata: dict[str, Any]) -> dict[str, Any]:+ """Run the real default-adapter walk over a `challenge_llm` spec.++ The adapter it offers is `USER_DEFAULT_LLM`, deliberately different from+ `RESOLVED_CHALLENGE_LLM`, so callers can tell "kept the seeded value" apart+ from "fell back to the user's default".+ """+ spec = Spec(+ title="exported-tool",+ description="Exported prompt studio tool",+ required=[JsonSchemaKey.CHALLENGE_LLM],+ properties={JsonSchemaKey.CHALLENGE_LLM: _challenge_llm_property()},+ )+ adapter = MagicMock(name="AdapterInstance")+ adapter.id = USER_DEFAULT_LLM+ ToolInstanceHelper.update_metadata_with_default_adapter(+ adapter_type=_tih_mod.AdapterTypes.LLM,+ schema_spec=spec,+ adapter=adapter,+ metadata=metadata,+ )+ return metadata+++class TestChallengeLlmOverlay:+ """`_overlay_resolved_challenge_llm` seeds the real resolved adapter ID."""++ def test_resolved_value_replaces_the_empty_spec_seed(self) -> None:+ """The "" that fails enum validation is replaced by the real ID."""+ tool_settings = _spec_seeded_settings()+ _run_overlay(tool_settings, resolved=_exported_tool_settings())+ assert tool_settings[JsonSchemaKey.CHALLENGE_LLM] == RESOLVED_CHALLENGE_LLM++ def test_companion_adapter_id_key_is_written_too(self) -> None:+ """The `adapterIdKey` companion is set, matching every other path.++ `update_metadata_with_default_adapter` always writes the adapter key and+ its ID key together; seeding only one produces a metadata shape nothing+ else in the system emits.+ """+ tool_settings = _spec_seeded_settings()+ _run_overlay(tool_settings, resolved=_exported_tool_settings())+ assert tool_settings[CHALLENGE_LLM_ADAPTER_ID_KEY] == RESOLVED_CHALLENGE_LLM++ def test_companion_key_falls_back_when_spec_omits_adapter_id_key(self) -> None:+ """A spec property with no `adapterIdKey` uses the generic fallback."""+ tool = _make_tool()+ challenge_property = _challenge_llm_property()+ del challenge_property["adapterIdKey"]+ tool.spec = Spec(+ title="no-id-key-tool",+ description="challenge_llm without an adapterIdKey",+ required=[JsonSchemaKey.CHALLENGE_LLM],+ properties={JsonSchemaKey.CHALLENGE_LLM: challenge_property},+ )+ tool_settings = {JsonSchemaKey.CHALLENGE_LLM: ""}+ _run_overlay(tool_settings, resolved=_exported_tool_settings(), tool=tool)+ assert tool_settings[AdapterPropertyKey.ADAPTER_ID] == RESOLVED_CHALLENGE_LLM+ assert CHALLENGE_LLM_ADAPTER_ID_KEY not in tool_settings++ def test_only_challenge_llm_is_overlaid(self) -> None:+ """Scope guard: the other spec/export overlaps stay untouched.++ The export's `tool_settings` also carries `enable_challenge`,+ `summarize_as_source`, `enable_highlight` and `enable_word_confidence`,+ all of which are spec properties too. Overlaying them would change+ whether challenge and summarization actually run -- a separate,+ behaviour-affecting decision.+ """+ tool_settings = _spec_seeded_settings()+ _run_overlay(tool_settings, resolved=_exported_tool_settings())+ # Exported as True; must remain the spec default.+ assert tool_settings[JsonSchemaKey.ENABLE_CHALLENGE] is False+ # `summarize_as_source` IS a spec property, but it is absent from this+ # fixture's spec and must not be overlaid on that basis either.+ assert JsonSchemaKey.SUMMARIZE_AS_SOURCE not in tool_settings+ # Keys the export carries that are not spec properties at all.+ for key in ("llm", "vector-db", "preamble"):+ assert key not in tool_settings++ def test_no_resolved_settings_is_a_no_op(self) -> None:+ """A non-Prompt-Studio tool resolves to {} and nothing changes."""+ tool_settings = _spec_seeded_settings()+ _run_overlay(tool_settings, resolved={}, tool_uid="text_extractor")+ assert tool_settings == _spec_seeded_settings()+ assert CHALLENGE_LLM_ADAPTER_ID_KEY not in tool_settings++ def test_empty_resolved_challenge_llm_does_not_overwrite(self) -> None:+ """An export carrying "" must not write "" plus a bogus ID key."""+ tool_settings = _spec_seeded_settings()+ _run_overlay(tool_settings, resolved={JsonSchemaKey.CHALLENGE_LLM: ""})+ assert CHALLENGE_LLM_ADAPTER_ID_KEY not in tool_settings++ def test_tool_without_challenge_llm_skips_the_lookup(self) -> None:+ """No `challenge_llm` in the spec -> no registry query at all."""+ tool = _make_tool()+ tool.spec = Spec(+ title="plain-tool",+ description="A tool with no challenge LLM",+ required=[],+ properties={"some_setting": {"type": "string", "default": "x"}},+ )+ tool_settings = {"some_setting": "x"}+ resolved_mock, _usable_mock = _run_overlay(tool_settings, tool=tool)+ resolved_mock.assert_not_called()+++class TestOverlaySkipsUnusableAdapters:+ """A resolved id the creating user cannot use must not be seeded.++ The deploy-time enum is built per-user. Seeding an id outside it would+ trade the "" enum violation for an equally opaque one naming a real UUID,+ and would suppress the default-adapter fill that previously healed it.+ """++ def test_inaccessible_adapter_is_not_seeded(self) -> None:+ """Shared tool, exporter's private adapter -> leave the seed empty."""+ tool_settings = _spec_seeded_settings()+ _run_overlay(+ tool_settings,+ resolved=_exported_tool_settings(),+ adapter_visible=False,+ )+ assert tool_settings[JsonSchemaKey.CHALLENGE_LLM] == ""+ assert CHALLENGE_LLM_ADAPTER_ID_KEY not in tool_settings++ def test_left_empty_value_still_gets_the_user_default(self) -> None:+ """The skipped seed must remain healable by the default-adapter walk.++ This is the whole point of not seeding: "" is falsy, so the guard in+ `update_metadata_with_default_adapter` fills it with the user's own+ default LLM rather than preserving an unusable id.+ """+ tool_settings = _spec_seeded_settings()+ _run_overlay(+ tool_settings,+ resolved=_exported_tool_settings(),+ adapter_visible=False,+ )+ metadata = _run_default_adapter_walk(+ {JsonSchemaKey.CHALLENGE_LLM: tool_settings[JsonSchemaKey.CHALLENGE_LLM]}+ )+ assert metadata[JsonSchemaKey.CHALLENGE_LLM] == USER_DEFAULT_LLM+ assert metadata[CHALLENGE_LLM_ADAPTER_ID_KEY] == USER_DEFAULT_LLM++ def test_usable_adapter_is_checked_against_the_creating_user(self) -> None:+ """The visibility check receives the resolved id and the user."""+ user = MagicMock(name="creating-user")+ _resolved_mock, usable_mock = _run_overlay(+ _spec_seeded_settings(), resolved=_exported_tool_settings(), user=user+ )+ usable_mock.assert_called_once_with(RESOLVED_CHALLENGE_LLM, user)+++class TestIsAdapterUsableBy:+ """Direct coverage of the visibility check's own body.++ Every other test patches this method out, so without these a `return True`+ or `return False` stub would keep the whole suite green - while either+ silently reintroduces the deploy-time 422 or makes the seed a no-op.+ """++ @staticmethod+ def _run(user: Any, *, exists: bool = True) -> tuple[bool, MagicMock]:+ exists_qs = MagicMock(name="filtered")+ exists_qs.exists.return_value = exists+ scoped = MagicMock(name="scoped")+ scoped.filter.return_value = exists_qs+ objects_mock = MagicMock(name="objects")+ objects_mock.for_user.return_value = scoped+ # Wired too, so `filter.assert_not_called()` means "never bypassed+ # for_user" rather than "would have blown up if it had".+ objects_mock.filter.return_value = exists_qs+ with patch.object(_ser_mod, "AdapterInstance", MagicMock(objects=objects_mock)):+ result = ToolInstanceSerializer._is_adapter_usable_by(+ RESOLVED_CHALLENGE_LLM, user+ )+ return result, objects_mock++ def test_scopes_the_lookup_to_the_user(self) -> None:+ """The query must go through `for_user`, not the raw manager.++ This is what keeps the check aligned with the per-user enum built at+ deploy time; an unscoped lookup would pass another user's adapter.+ """+ user = MagicMock(name="user")+ result, objects_mock = self._run(user)+ assert result is True+ objects_mock.for_user.assert_called_once_with(user)+ objects_mock.filter.assert_not_called()++ def test_filters_on_id_and_llm_adapter_type(self) -> None:+ """Both predicates must match `AdapterProcessor.get_adapters_by_type`.++ Dropping the type predicate lets a visible non-LLM adapter through,+ which the LLM enum then rejects at deploy - the same relocated 422.+ """+ _result, objects_mock = self._run(MagicMock(name="user"))+ objects_mock.for_user.return_value.filter.assert_called_once_with(+ id=RESOLVED_CHALLENGE_LLM, adapter_type=AdapterTypes.LLM.value+ )++ def test_absent_adapter_is_not_usable(self) -> None:+ """A deleted or invisible adapter resolves to False."""+ result, _objects_mock = self._run(MagicMock(name="user"), exists=False)+ assert result is False++ def test_missing_user_refuses_rather_than_falling_back(self) -> None:+ """No user means no scope - refuse instead of seeding on existence.++ Answering "yes" here would fail open: any adapter that merely exists,+ including another user's private one, would be seeded.+ """+ result, objects_mock = self._run(None)+ assert result is False+ objects_mock.for_user.assert_not_called()+ objects_mock.filter.assert_not_called()+++class TestOverlayIsWiredIntoCreate:+ """The overlay's RESULT must reach the metadata `create` persists.++ Asserting only that the overlay is *called* is not enough: the call could+ run against a dict nobody reads again, or the metadata could be rebuilt+ from unoverlaid defaults afterwards. Both leave a called-but-discarded+ overlay, which is the same class of wiring gap as never calling it.+ """++ @staticmethod+ def _create_and_capture_metadata(+ usable_mock: MagicMock | None = None,+ request_user: Any = None,+ ) -> dict[str, Any]:+ """Run the real `create` with the real overlay; return its metadata.++ `create` mutates the `validated_data` dict in place, so the metadata it+ would persist is readable afterwards. The serializer is built with+ request context, as `get_serializer` does in the view, so the+ request->user hop the visibility check depends on is exercised.+ """+ workflow = MagicMock(name="Workflow")+ workflow.tool_instances.count.return_value = 0+ workflow.is_active = True++ validated_data: dict[str, Any] = {+ "workflow_id": "wf-1",+ "tool_id": PROMPT_REGISTRY_ID,+ }+ request = MagicMock(name="request")+ # `name=`, not positional: MagicMock's first positional arg is `spec`,+ # which would build a str-specced mock that raises AttributeError the+ # moment a test lets the real visibility check touch a User attribute.+ request.user = (+ request_user if request_user is not None else MagicMock(name="user")+ )++ with (+ patch.object(+ _ser_mod.Workflow,+ "objects",+ MagicMock(get=MagicMock(return_value=workflow)),+ ),+ patch.object(+ _ser_mod.ToolProcessor,+ "get_tool_by_uid",+ MagicMock(return_value=_make_tool()),+ ),+ patch.object(+ _ser_mod.ToolProcessor,+ "get_default_settings",+ # `side_effect`, not `return_value`: a single shared dict would+ # already carry the overlay's mutations, so metadata rebuilt+ # from a second call would look correct even when the wiring+ # is broken.+ MagicMock(side_effect=lambda _tool: _spec_seeded_settings()),+ ),+ patch.object(+ PromptStudioRegistryHelper,+ "get_resolved_settings",+ MagicMock(return_value=_exported_tool_settings()),+ ),+ patch.object(+ ToolInstanceSerializer,+ "_is_adapter_usable_by",+ usable_mock if usable_mock is not None else MagicMock(return_value=True),+ ),+ patch.object(+ _ser_mod.AuditSerializer, "create", MagicMock(return_value=MagicMock())+ ),+ ):+ ToolInstanceSerializer(context={"request": request}).create(validated_data)++ return validated_data[_ser_mod.TIKey.METADATA]++ def test_seeded_value_reaches_persisted_metadata(self) -> None:+ """The resolved id must survive all the way into instance metadata."""+ metadata = self._create_and_capture_metadata()+ assert metadata[JsonSchemaKey.CHALLENGE_LLM] == RESOLVED_CHALLENGE_LLM++ def test_companion_key_reaches_persisted_metadata(self) -> None:+ """So must its companion, or the pair is half-written on disk."""+ metadata = self._create_and_capture_metadata()+ assert metadata[CHALLENGE_LLM_ADAPTER_ID_KEY] == RESOLVED_CHALLENGE_LLM++ def test_creating_user_reaches_the_visibility_check(self) -> None:+ """`create` must forward the request's user, not lose it.++ This hop is what makes the check per-user. If it degrades to None the+ check has no scope to test against, so seeding would either fail open+ on mere existence or stop working entirely - and no assertion about+ metadata contents would notice.+ """+ request_user = MagicMock(name="request-user")+ usable_mock = MagicMock(return_value=True)+ self._create_and_capture_metadata(+ usable_mock=usable_mock, request_user=request_user+ )+ usable_mock.assert_called_once_with(RESOLVED_CHALLENGE_LLM, request_user)+++class TestSeededValuePassesDeployValidation:+ """End-to-end: the seeded value must satisfy the enum deploy checks against.++ This is the assertion that ties the seed to the 422 the change exists to+ prevent. Everything else here checks that a string was copied between+ dicts; this checks the copied string is one the real, user-scoped schema+ accepts -- and that the pre-fix "" is not.+ """++ @staticmethod+ def _user_scoped_schema() -> dict[str, Any]:+ """The real tool-instance schema, with the real enum injected.++ `update_schema_with_adapter_configurations` builds the enum from the+ adapters visible to the user, which is exactly what makes a foreign or+ deleted `challenge_llm` fail at deploy time.+ """+ tool = _make_tool()+ schema = ToolUtils.get_json_schema_for_tool(tool)+ visible_adapter = MagicMock(name="AdapterInstance")+ visible_adapter.id = RESOLVED_CHALLENGE_LLM+ visible_adapter.adapter_name = "Challenger LLM"+ with patch.object(+ _tp_mod.AdapterProcessor,+ "get_adapters_by_type",+ MagicMock(return_value=[visible_adapter]),+ ):+ _tp_mod.ToolProcessor.update_schema_with_adapter_configurations(+ schema=schema, user=MagicMock(name="user")+ )+ return schema.to_dict()++ def test_enum_is_actually_injected_for_challenge_llm(self) -> None:+ """Guard the guard: without an enum the rest of this class is inert."""+ schema = self._user_scoped_schema()+ enum = schema["properties"][JsonSchemaKey.CHALLENGE_LLM].get("enum")+ assert enum == [RESOLVED_CHALLENGE_LLM]++ def test_seeded_value_validates(self) -> None:+ """A value produced by the overlay passes the deploy-time check."""+ tool_settings = _spec_seeded_settings()+ _run_overlay(tool_settings, resolved=_exported_tool_settings())+ errors = list(+ jsonschema.Draft7Validator(self._user_scoped_schema()).iter_errors(+ {JsonSchemaKey.CHALLENGE_LLM: tool_settings[JsonSchemaKey.CHALLENGE_LLM]}+ )+ )+ assert errors == []++ def test_pre_fix_empty_seed_still_fails(self) -> None:+ """Discrimination check: "" must still be rejected by the same schema.++ If this ever passes, the test above proves nothing.+ """+ errors = list(+ jsonschema.Draft7Validator(self._user_scoped_schema()).iter_errors(+ {JsonSchemaKey.CHALLENGE_LLM: ""}+ )+ )+ assert any(error.validator == "enum" for error in errors)+++class TestDefaultAdapterDoesNotClobberSeededValue:+ """`update_metadata_with_default_adapter` must not stomp a seeded value.++ This is the half of the bug that made the seed a no-op in the common case:+ the walk runs right after creation and `challenge_llm` is an LLM adapter+ property, so a user with a default LLM had the exported value replaced.+ """++ def test_seeded_challenge_llm_survives_the_default_walk(self) -> None:+ """The exported value is kept, not replaced by the user's default."""+ metadata = _run_default_adapter_walk(+ {+ JsonSchemaKey.CHALLENGE_LLM: RESOLVED_CHALLENGE_LLM,+ CHALLENGE_LLM_ADAPTER_ID_KEY: RESOLVED_CHALLENGE_LLM,+ }+ )+ assert metadata[JsonSchemaKey.CHALLENGE_LLM] == RESOLVED_CHALLENGE_LLM+ assert metadata[CHALLENGE_LLM_ADAPTER_ID_KEY] == RESOLVED_CHALLENGE_LLM++ def test_kept_value_never_leaves_a_half_written_pair(self) -> None:+ """Skipping the write must not strand a missing companion ID key.++ `update_metadata_with_adapter_properties` warns that a UUID-shaped value+ at the adapter key with no matching ID key "bypasses the lazy migrator+ and fails schema enum validation later". Preserving a value must not+ manufacture that shape.+ """+ metadata = _run_default_adapter_walk(+ {JsonSchemaKey.CHALLENGE_LLM: RESOLVED_CHALLENGE_LLM}+ )+ assert metadata[JsonSchemaKey.CHALLENGE_LLM] == RESOLVED_CHALLENGE_LLM+ assert metadata[CHALLENGE_LLM_ADAPTER_ID_KEY] == RESOLVED_CHALLENGE_LLM++ def test_guard_applies_to_non_llm_adapter_types_too(self) -> None:+ """The guard sits after the type dispatch, so it covers every type.++ `challenge_llm` is the only adapter-valued property in the Prompt Studio+ spec today, so LLM is the only type that can currently hit this. Pinning+ a second type keeps the breadth deliberate rather than incidental: a+ future spec with an adapter-valued X2TEXT property must behave the same.+ """+ x2text_property = {+ "type": "string",+ "title": "Text extractor",+ "adapterType": "X2TEXT",+ "adapterIdKey": "x2text_adapter_id",+ }+ spec = Spec(+ title="x2text-tool",+ description="A tool with an adapter-valued X2TEXT property",+ required=[],+ properties={"x2text_llm": x2text_property},+ )+ adapter = MagicMock(name="AdapterInstance")+ adapter.id = USER_DEFAULT_LLM+ metadata = {"x2text_llm": RESOLVED_CHALLENGE_LLM}+ ToolInstanceHelper.update_metadata_with_default_adapter(+ adapter_type=_tih_mod.AdapterTypes.X2TEXT,+ schema_spec=spec,+ adapter=adapter,+ metadata=metadata,+ )+ assert metadata["x2text_llm"] == RESOLVED_CHALLENGE_LLM+ assert metadata["x2text_adapter_id"] == RESOLVED_CHALLENGE_LLM++ def test_unset_challenge_llm_still_gets_the_user_default(self) -> None:+ """The guard fills gaps -- it must not disable the default entirely."""+ metadata = _run_default_adapter_walk({JsonSchemaKey.CHALLENGE_LLM: ""})+ assert metadata[JsonSchemaKey.CHALLENGE_LLM] == USER_DEFAULT_LLM+ assert metadata[CHALLENGE_LLM_ADAPTER_ID_KEY] == USER_DEFAULT_LLM++ def test_absent_key_still_gets_the_user_default(self) -> None:+ """A key missing altogether is a gap too, and must be filled."""+ metadata = _run_default_adapter_walk({})+ assert metadata[JsonSchemaKey.CHALLENGE_LLM] == USER_DEFAULT_LLM+++class TestGetResolvedSettings:+ """The registry lookup reads the right block and stays quiet when normal."""++ def test_reads_the_tool_settings_block(self) -> None:+ settings_block = {JsonSchemaKey.CHALLENGE_LLM: RESOLVED_CHALLENGE_LLM}+ metadata = {+ JsonSchemaKey.TOOL_SETTINGS: settings_block,+ # A sibling key that must not be mistaken for the settings block.+ "outputs": [{"prompt": "x"}],+ }+ values_qs = MagicMock(name="values_list")+ values_qs.get.return_value = metadata+ objects_mock = MagicMock(values_list=MagicMock(return_value=values_qs))+ with patch.object(_psr_mod.PromptStudioRegistry, "objects", objects_mock):+ resolved = PromptStudioRegistryHelper.get_resolved_settings(+ PROMPT_REGISTRY_ID+ )+ assert resolved == settings_block+ # Pin the query shape: the wrong column, or a dropped `flat=True`,+ # makes `.get()` return a 1-tuple and blows up on every real creation.+ objects_mock.values_list.assert_called_once_with("tool_metadata", flat=True)++ def test_non_uuid_tool_id_short_circuits_without_a_query(self) -> None:+ """Registry tools use slugs; those must not reach the DB or the log.++ `ToolInstance.tool_id` is a free-form CharField, so the common case --+ a built-in tool like "text_extractor" -- is not UUID-shaped at all.+ """+ objects_mock = MagicMock(name="objects")+ with patch.object(_psr_mod.PromptStudioRegistry, "objects", objects_mock):+ assert (+ PromptStudioRegistryHelper.get_resolved_settings("text_extractor") == {}+ )+ objects_mock.values_list.assert_not_called()++ def test_missing_registry_row_returns_empty(self) -> None:+ values_qs = MagicMock(name="values_list")+ values_qs.get.side_effect = _psr_mod.PromptStudioRegistry.DoesNotExist+ with patch.object(+ _psr_mod.PromptStudioRegistry,+ "objects",+ MagicMock(values_list=MagicMock(return_value=values_qs)),+ ):+ assert (+ PromptStudioRegistryHelper.get_resolved_settings(PROMPT_REGISTRY_ID) == {}+ )++ def test_row_without_tool_settings_returns_empty(self) -> None:+ values_qs = MagicMock(name="values_list")+ values_qs.get.return_value = None+ with patch.object(+ _psr_mod.PromptStudioRegistry,+ "objects",+ MagicMock(values_list=MagicMock(return_value=values_qs)),+ ):+ assert (+ PromptStudioRegistryHelper.get_resolved_settings(PROMPT_REGISTRY_ID) == {}+ )++ def test_db_errors_are_not_swallowed(self) -> None:+ """An `OperationalError` must propagate, not degrade to "".++ Swallowing it would seed "" and resurface much later as the same+ confusing 422 this change exists to prevent.+ """+ values_qs = MagicMock(name="values_list")+ values_qs.get.side_effect = OperationalError("db down")+ with patch.object(+ _psr_mod.PromptStudioRegistry,+ "objects",+ MagicMock(values_list=MagicMock(return_value=values_qs)),+ ):+ with pytest.raises(OperationalError):+ PromptStudioRegistryHelper.get_resolved_settings(PROMPT_REGISTRY_ID)diff --git a/backend/tool_instance_v2/tool_instance_helper.py b/backend/tool_instance_v2/tool_instance_helper.pyindex 0c7eb9d04..138e48b5b 100644--- a/backend/tool_instance_v2/tool_instance_helper.py+++ b/backend/tool_instance_v2/tool_instance_helper.py@@ -74,6 +74,19 @@ class ToolInstanceHelper:tool_instance.metadata = metadatatool_instance.save()+ @staticmethod+ def get_adapter_id_key(adapter_property: dict[str, Any]) -> str:+ """The metadata key holding a spec property's companion adapter ID.++ Every writer stores an adapter key and this companion together; a+ populated adapter key with a missing ID key bypasses the lazy migrator+ and then fails schema enum validation. Resolved in one place so the+ writers cannot drift on where that companion lives.+ """+ return adapter_property.get(+ AdapterPropertyKey.ADAPTER_ID_KEY, AdapterPropertyKey.ADAPTER_ID+ )+@staticmethoddef update_metadata_with_adapter_properties(metadata: dict[str, Any],@@ -114,9 +127,7 @@ class ToolInstanceHelper:adapter_type=adapter_type, adapter_name=adapter_value)adapter_id = str(adapter.id)- metadata_key_for_id = adapter_property.get(- AdapterPropertyKey.ADAPTER_ID_KEY, AdapterPropertyKey.ADAPTER_ID- )+ metadata_key_for_id = ToolInstanceHelper.get_adapter_id_key(adapter_property)# Keep adapter_key and adapter_id_key both canonical to the resolved# target UUID; otherwise stale UUID-shaped values at adapter_key# bypass the lazy migrator and fail schema enum validation later.@@ -216,9 +227,24 @@ class ToolInstanceHelper:if adapter_type == AdapterTypes.OCR:properties = schema_spec.get_ocr_adapter_properties()for adapter_key, adapter_property in properties.items():- metadata_key_for_id = adapter_property.get(- AdapterPropertyKey.ADAPTER_ID_KEY, AdapterPropertyKey.ADAPTER_ID- )+ metadata_key_for_id = ToolInstanceHelper.get_adapter_id_key(adapter_property)+ # Only fill in what has no value yet. This runs right after tool+ # instance creation, which may already have seeded a key with a+ # value resolved for that specific tool - `challenge_llm` carries+ # the exported project's challenger LLM, and it is one of the LLM+ # adapter properties walked here. Overwriting it would replace a+ # tool-specific choice with the user's global default.+ existing_value = metadata.get(adapter_key)+ if existing_value:+ # Keep the pair consistent for whoever populated the adapter+ # key. No caller reaching here today leaves the ID key unset -+ # this holds the invariant rather than repairing a known writer.+ # Cheap and worth keeping: a populated adapter key with a+ # missing ID key bypasses the lazy migrator and then fails+ # schema enum validation (see+ # `update_metadata_with_adapter_properties`).+ metadata.setdefault(metadata_key_for_id, existing_value)+ continuemetadata[adapter_key] = str(adapter.id)metadata[metadata_key_for_id] = str(adapter.id)