LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2946 [FEAT] Prompt Studio lookups bridge, executor hook, and IDE wiring (OSS side) (#1929)
Chandrasekharan Mf742e32e92d7
diff --git a/.gitignore b/.gitignoreindex f6837ce07..c655f4acf 100644--- a/.gitignore+++ b/.gitignore@@ -653,7 +653,11 @@ docker/*.env!docker/sample*.envdocker/public_tools.jsondocker/proxy_overrides.yaml-docker/compose.override.yaml+docker/compose.*.yaml+# ``docker/compose.debug.yaml`` is checked-in tooling — keep it out of the+# broader ``compose.*.yaml`` ignore so a delete + recreate doesn't make it+# look untracked, and so teammates can spot it.+!docker/compose.debug.yamldocker/workflow_data/# Tool development@@ -696,6 +700,12 @@ CLAUDE.mdCONTRIBUTION_GUIDE.md.mcp.json+# Codex+AGENTS.md++# Pi+.pi/+# Windsurf.qodo.windsurfrulesdiff --git a/backend/prompt_studio/lookup_utils.py b/backend/prompt_studio/lookup_utils.pynew file mode 100644index 000000000..8cfb0a728--- /dev/null+++ b/backend/prompt_studio/lookup_utils.py@@ -0,0 +1,171 @@+"""Shared utility for lookup operations. No-ops in OSS.++Only the absence of ``pluggable_apps.lookups`` itself is treated as+"cloud not installed"; an ImportError from a transitive dependency+re-raises so we don't silently degrade to a no-op on a real bug.+"""++import logging+from typing import Any++logger = logging.getLogger(__name__)++_CLOUD_LOOKUP_MODULES = {+ # OSS images lack the parent ``pluggable_apps`` package, so include it.+ "pluggable_apps",+ "pluggable_apps.lookups",+ "pluggable_apps.lookups.execution",+ "pluggable_apps.lookups.output_enrichment",+ "pluggable_apps.lookups.staleness",+ "pluggable_apps.lookups.validation",+ "pluggable_apps.lookups.models",+}++try:+ from pluggable_apps.lookups import execution as _execution+ from pluggable_apps.lookups import output_enrichment as _output_enrichment+ from pluggable_apps.lookups import staleness as _staleness+ from pluggable_apps.lookups import validation as _validation+ from pluggable_apps.lookups.models import LookupOutputResult as _LookupOutputResult++ LOOKUPS_AVAILABLE = True+except ImportError as e:+ if e.name not in _CLOUD_LOOKUP_MODULES:+ raise+ LOOKUPS_AVAILABLE = False+++def get_lookup_config(prompt) -> dict | None:+ """Return lookup config for a prompt, or None if lookups are unavailable."""+ if not LOOKUPS_AVAILABLE:+ return None+ return _execution.build_lookup_config_for_prompt(prompt)+++def get_lookup_configs_for_tool(tool, prompts=None) -> list[dict] | None:+ """Return lookup configs for a tool (single pass), or None in OSS.++ ``prompts`` scopes validation to the run's prompts so unrelated+ incomplete assignments on the tool don't block it.+ """+ if not LOOKUPS_AVAILABLE:+ return None+ return _execution.build_lookup_configs_for_tool(tool, prompts=prompts)+++def get_multi_var_lookups_for_tool(tool, prompt_ids=None) -> list[str]:+ """Return names of multi-variable lookups linked to the tool, [] in OSS.++ ``prompt_ids`` scopes the check so a run is only blocked when the+ multi-var lookup is actually used by it.+ """+ if not LOOKUPS_AVAILABLE:+ return []+ _, names = _execution.has_multi_var_lookups(tool, prompt_ids=prompt_ids)+ return names+++def persist_lookup_output(prompt_output, prompt_lookup: dict) -> None:+ """Persist lookup enrichment result. No-op in OSS."""+ if not LOOKUPS_AVAILABLE:+ return+ lookup_meta = prompt_lookup.get("meta", {})+ lookup_id = lookup_meta.get("lookup_id")+ if not lookup_id:+ return+ defaults = {+ "lookup_definition_id": lookup_id,+ "output": prompt_lookup.get("enriched", ""),+ }+ version_id = lookup_meta.get("version_id")+ if version_id:+ defaults["version_id"] = version_id+ _LookupOutputResult.objects.update_or_create(+ prompt_output=prompt_output,+ defaults=defaults,+ )+++def enrich_prompt_output(prompt_output, data: dict) -> dict:+ """Let cloud plugins enrich serialized prompt output with lookup data.++ No-op in OSS.+ """+ if not LOOKUPS_AVAILABLE:+ return data+ return _output_enrichment.enrich_with_lookup_output(prompt_output, data)+++def validate_lookups_for_export(prompts) -> tuple[dict, str | None]:+ """Validate lookup assignments before export. Returns ({}, None) in OSS."""+ if not LOOKUPS_AVAILABLE:+ return {}, None+ return _validation.validate_lookups_for_export(prompts)+++def get_latest_lookup_mutation_for_tool(tool):+ """Max ``modified_at`` across lookup-related records linked to the tool+ (version, reference file, assignment) — feeds the staleness banner.+ None if unavailable or nothing linked.+ """+ if not LOOKUPS_AVAILABLE:+ return None+ return _staleness.get_latest_lookup_mutation_for_tool(tool)+++def get_original_value_if_enriched(+ metadata: dict, prompt_key: str+) -> tuple[Any, dict] | None:+ """Return ``(original_value, prompt_lookup_dict)`` if ``prompt_key`` was+ enriched, or ``None`` otherwise.++ Pure metadata-shape check — safe to call even when LOOKUPS_AVAILABLE+ is False (returns None because the shape won't match).+ """+ if not isinstance(metadata, dict):+ return None+ lookup_outputs = metadata.get("lookup_outputs") or {}+ prompt_lookup = lookup_outputs.get(prompt_key)+ if isinstance(prompt_lookup, dict) and "original" in prompt_lookup:+ return prompt_lookup.get("original"), prompt_lookup+ return None+++def attach_combined_output_enrichment(result: dict, enriched_by_key: dict) -> None:+ """Stamp the combined-output payload with enriched-output metadata.++ Key name stays cloud-side so the FE-plugin shape can evolve without+ coordinating with OSS.+ """+ if not LOOKUPS_AVAILABLE:+ return+ _output_enrichment.attach_combined_output_enrichment(result, enriched_by_key)+++def extract_prompt_output_enrichment(item) -> dict | None:+ """Pick enriched-output data off a serialized prompt-output row.++ Returns a plugin-opaque dict (FE-only) or None when no enrichment+ is present / plugin missing.+ """+ if not LOOKUPS_AVAILABLE:+ return None+ return _output_enrichment.extract_prompt_output_enrichment(item)+++def get_lookup_validation_for_tool(tool) -> dict:+ """Pre-emptive lookup validation for FE Export / Deploy gating.++ Returns an "always ok" payload in OSS so the FE gate is a no-op.+ """+ if not LOOKUPS_AVAILABLE:+ return {+ "ok": True,+ "draft_lookups": [],+ "multi_var_lookups": [],+ "incomplete_lookups": [],+ "single_pass_enabled": bool(+ getattr(tool, "single_pass_extraction_mode", False)+ ),+ }+ return _validation.get_lookup_validation_for_tool(tool)diff --git a/backend/prompt_studio/prompt_studio_core_v2/migrations/0007_customtool_last_exported_at.py b/backend/prompt_studio/prompt_studio_core_v2/migrations/0007_customtool_last_exported_at.pynew file mode 100644index 000000000..91539c9cf--- /dev/null+++ b/backend/prompt_studio/prompt_studio_core_v2/migrations/0007_customtool_last_exported_at.py@@ -0,0 +1,21 @@+# Generated by Django 4.2.1 on 2026-04-21 20:20++from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("prompt_studio_core_v2", "0006_add_custom_data_to_customtool"),+ ]++ operations = [+ migrations.AddField(+ model_name="customtool",+ name="last_exported_at",+ field=models.DateTimeField(+ blank=True,+ db_comment="Timestamp of the last successful export; NULL if never exported since the field was introduced.",+ null=True,+ ),+ ),+ ]diff --git a/backend/prompt_studio/prompt_studio_core_v2/models.py b/backend/prompt_studio/prompt_studio_core_v2/models.pyindex 1e1802b77..cd2a12dac 100644--- a/backend/prompt_studio/prompt_studio_core_v2/models.py+++ b/backend/prompt_studio/prompt_studio_core_v2/models.py@@ -161,6 +161,15 @@ class CustomTool(DefaultOrganizationMixin, BaseModel):db_comment="Flag to share this custom tool with all users in the organization",)+ # NULL on pre-feature tools; populated on first successful export.+ # Drives staleness checks (e.g. lookup-change banner) without requiring+ # a data backfill.+ last_exported_at = models.DateTimeField(+ null=True,+ blank=True,+ db_comment="Timestamp of the last successful export; NULL if never exported since the field was introduced.",+ )+objects = CustomToolModelManager()def delete(self, organization_id=None, *args, **kwargs):diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.pyindex ff2b34550..62cbf2d9a 100644--- a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py+++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py@@ -14,12 +14,17 @@ from django.conf import settingsfrom django.db import transactionfrom django.db.models.manager import BaseManagerfrom plugins import get_plugin+from rest_framework.exceptions import APIExceptionfrom rest_framework.request import Requestfrom utils.file_storage.constants import FileStorageKeysfrom utils.file_storage.helpers.prompt_studio_file_helper import PromptStudioFileHelperfrom utils.local_context import StateStorefrom backend.celery_service import app as celery_app+from prompt_studio.lookup_utils import (+ get_lookup_config,+ get_lookup_configs_for_tool,+)from prompt_studio.prompt_profile_manager_v2.models import ProfileManagerfrom prompt_studio.prompt_profile_manager_v2.profile_manager_helper import (ProfileManagerHelper,@@ -387,6 +392,9 @@ class PromptStudioHelper:if webhook_enabled:output[TSPKeys.POSTPROCESSING_WEBHOOK_URL] = webhook_url+ if lookup_config := get_lookup_config(prompt):+ output["lookup_config"] = lookup_config+output[TSPKeys.EVAL_SETTINGS] = {}output[TSPKeys.EVAL_SETTINGS][TSPKeys.EVAL_SETTINGS_EVALUATE] = prompt.evaluateoutput[TSPKeys.EVAL_SETTINGS][TSPKeys.EVAL_SETTINGS_MONITOR_LLM] = [monitor_llm]@@ -798,6 +806,9 @@ class PromptStudioHelper:if webhook_enabled:output[TSPKeys.POSTPROCESSING_WEBHOOK_URL] = webhook_url+ if lookup_config := get_lookup_config(prompt):+ output["lookup_config"] = lookup_config+output[TSPKeys.EVAL_SETTINGS] = {}output[TSPKeys.EVAL_SETTINGS][TSPKeys.EVAL_SETTINGS_EVALUATE] = prompt.evaluateoutput[TSPKeys.EVAL_SETTINGS][TSPKeys.EVAL_SETTINGS_MONITOR_LLM] = [monitor_llm]@@ -1166,6 +1177,10 @@ class PromptStudioHelper:TSPKeys.SIMILARITY_TOP_K: default_profile.similarity_top_k,}+ lookup_configs = get_lookup_configs_for_tool(tool, prompts=prompts)+ if lookup_configs:+ tool_settings["lookup_configs"] = lookup_configs+for p in prompts:if not p.prompt:raise EmptyPromptError()@@ -1607,6 +1622,9 @@ class PromptStudioHelper:is_single_pass=False,profile_manager_id=profile_manager_id,)+ except APIException:+ # Validation responses are user-facing; DRF renders them as-is.+ raiseexcept Exception as e:logger.error(f"[{tool.tool_id}] Error while fetching response for "@@ -1672,6 +1690,9 @@ class PromptStudioHelper:document_id=document_id,is_single_pass=True,)+ except APIException:+ # Validation responses are user-facing; DRF renders them as-is.+ raiseexcept Exception as e:logger.error(f"[{tool.tool_id}] Error while fetching single pass response: {e}"@@ -1911,6 +1932,8 @@ class PromptStudioHelper:output[TSPKeys.ENABLE_POSTPROCESSING_WEBHOOK] = webhook_enabledif webhook_enabled:output[TSPKeys.POSTPROCESSING_WEBHOOK_URL] = webhook_url+ if lookup_config := get_lookup_config(prompt):+ output["lookup_config"] = lookup_config# Eval settings for the promptoutput[TSPKeys.EVAL_SETTINGS] = {}output[TSPKeys.EVAL_SETTINGS][TSPKeys.EVAL_SETTINGS_EVALUATE] = prompt.evaluatediff --git a/backend/prompt_studio/prompt_studio_core_v2/urls.py b/backend/prompt_studio/prompt_studio_core_v2/urls.pyindex 86cbb97dd..9163e8736 100644--- a/backend/prompt_studio/prompt_studio_core_v2/urls.py+++ b/backend/prompt_studio/prompt_studio_core_v2/urls.py@@ -66,6 +66,10 @@ prompt_studio_deployment_usage = PromptStudioCoreView.as_view(prompt_studio_task_status = PromptStudioCoreView.as_view({"get": "task_status"})+prompt_studio_lookup_validation = PromptStudioCoreView.as_view(+ {"get": "lookup_validation"}+)+urlpatterns = format_suffix_patterns([@@ -165,5 +169,10 @@ urlpatterns = format_suffix_patterns(prompt_studio_task_status,name="prompt-studio-task-status",),+ path(+ "prompt-studio/<uuid:pk>/lookup-validation/",+ prompt_studio_lookup_validation,+ name="prompt-studio-lookup-validation",+ ),])diff --git a/backend/prompt_studio/prompt_studio_core_v2/views.py b/backend/prompt_studio/prompt_studio_core_v2/views.pyindex 99edefd0b..30ea7045e 100644--- a/backend/prompt_studio/prompt_studio_core_v2/views.py+++ b/backend/prompt_studio/prompt_studio_core_v2/views.py@@ -14,6 +14,7 @@ from celery.result import AsyncResultfrom django.db import IntegrityErrorfrom django.db.models import Count, OuterRef, QuerySet, Subqueryfrom django.http import HttpRequest, HttpResponse+from django.utils import timezonefrom file_management.constants import FileInformationKey as FileKeyfrom file_management.exceptions import FileNotFoundfrom permissions.permission import IsOwner, IsOwnerOrSharedUserOrSharedToOrg@@ -32,6 +33,11 @@ from utils.user_session import UserSessionUtilsfrom workflow_manager.endpoint_v2.models import WorkflowEndpointfrom backend.celery_service import app as celery_app+from prompt_studio.lookup_utils import (+ get_latest_lookup_mutation_for_tool,+ get_lookup_validation_for_tool,+ get_multi_var_lookups_for_tool,+)from prompt_studio.prompt_profile_manager_v2.constants import (ProfileManagerErrors,ProfileManagerKeys,@@ -90,6 +96,29 @@ from .serializers import (logger = logging.getLogger(__name__)+def _multi_var_lookup_block_response(custom_tool, prompt_ids=None):+ """Block non-SP runs when a linked lookup has >1 input variable.++ Multi-var lookups only resolve correctly under single-pass; ``prompt_ids``+ scopes the gate so an unrelated multi-var lookup doesn't block runs that+ don't actually use it. Caller must skip this on the SP path.+ """+ names = get_multi_var_lookups_for_tool(custom_tool, prompt_ids=prompt_ids)+ if not names:+ return None+ return Response(+ {+ "detail": (+ "Multi-variable lookup(s) "+ f"{', '.join(names)} are linked to prompts in this project. "+ "These can only run in single pass extraction mode. "+ "Enable single pass or unlink the lookup before running."+ )+ },+ status=status.HTTP_400_BAD_REQUEST,+ )++class PromptStudioCoreView(viewsets.ModelViewSet):"""Viewset to handle all Custom tool related operations."""@@ -477,6 +506,14 @@ class PromptStudioCoreView(viewsets.ModelViewSet):document_id: str = request.data.get(ToolStudioPromptKeys.DOCUMENT_ID)prompt_id: str = request.data.get(ToolStudioPromptKeys.ID)run_id: str = request.data.get(ToolStudioPromptKeys.RUN_ID)+ # Must precede the lookup gate so missing prompt_id returns a clear 400.+ if not prompt_id:+ return Response(+ {"error": "prompt id is required."},+ status=status.HTTP_400_BAD_REQUEST,+ )+ if err := _multi_var_lookup_block_response(custom_tool, prompt_ids=[prompt_id]):+ return errprofile_manager_id: str = request.data.get(ToolStudioPromptKeys.PROFILE_MANAGER_ID)@@ -485,13 +522,6 @@ class PromptStudioCoreView(viewsets.ModelViewSet):org_id = UserSessionUtils.get_organization_id(request)user_id = custom_tool.created_by.user_id-- # Resolve prompt — guard against missing / stale prompt_id- if not prompt_id:- return Response(- {"error": "prompt id is required."},- status=status.HTTP_400_BAD_REQUEST,- )try:prompt = ToolStudioPrompt.objects.get(pk=prompt_id)except ToolStudioPrompt.DoesNotExist:@@ -510,9 +540,8 @@ class PromptStudioCoreView(viewsets.ModelViewSet):document: DocumentManager = DocumentManager.objects.get(pk=document_id)doc_path = str(Path(doc_path) / document.document_name)- # Agentic table prompts have a separate executor worker. Build the- # payload via the cloud payload_modifier plugin and dispatch directly- # so the legacy answer_prompt path is bypassed.+ # Agentic table prompts have their own executor — build payload via+ # the cloud plugin and dispatch directly, bypassing answer_prompt.if prompt.enforce_type == ToolStudioPromptKeys.AGENTIC_TABLE:payload_modifier_plugin = get_plugin("payload_modifier")if not payload_modifier_plugin:@@ -633,6 +662,8 @@ class PromptStudioCoreView(viewsets.ModelViewSet):{"error": "prompt_ids is required and must be non-empty."},status=status.HTTP_400_BAD_REQUEST,)+ if err := _multi_var_lookup_block_response(custom_tool, prompt_ids=prompt_ids):+ return errdocument_id: str = request.data.get(ToolStudioPromptKeys.DOCUMENT_ID)run_id: str = request.data.get(ToolStudioPromptKeys.RUN_ID)profile_manager_id: str = request.data.get(@@ -881,9 +912,17 @@ class PromptStudioCoreView(viewsets.ModelViewSet):serializer = ProfileManagerSerializer(data=request.data, context=context)serializer.is_valid(raise_exception=True)# Check for the maximum number of profiles constraint- prompt_studio_tool = serializer.validated_data[+ prompt_studio_tool = serializer.validated_data.get(ProfileManagerKeys.PROMPT_STUDIO_TOOL- ]+ )+ if not prompt_studio_tool:+ # Write back into validated_data so perform_create() doesn't+ # persist NULL and orphan the profile from every+ # ``filter(prompt_studio_tool=...)`` query.+ prompt_studio_tool = self.get_object()+ serializer.validated_data[ProfileManagerKeys.PROMPT_STUDIO_TOOL] = (+ prompt_studio_tool+ )profile_count = ProfileManager.objects.filter(prompt_studio_tool=prompt_studio_tool).count()@@ -1103,6 +1142,10 @@ class PromptStudioCoreView(viewsets.ModelViewSet):force_export=force_export,)+ # Anchor for staleness checks (e.g. lookup-change banner).+ custom_tool.last_exported_at = timezone.now()+ custom_tool.save(update_fields=["last_exported_at"])+# Notify HubSpot about first tool exportnotify_hubspot_event(user=request.user,@@ -1116,6 +1159,15 @@ class PromptStudioCoreView(viewsets.ModelViewSet):status=status.HTTP_200_OK,)+ @action(detail=True, methods=["get"], url_path="lookup-validation")+ def lookup_validation(self, request: Request, pk: Any = None) -> Response:+ """Pre-emptive lookup gating for Export / API Deployment buttons.++ Cloud-only check; OSS returns ``ok: True`` so the FE proceeds.+ """+ custom_tool = self.get_object()+ return Response(get_lookup_validation_for_tool(custom_tool))+@action(detail=True, methods=["get"])def export_tool_info(self, request: Request, pk: Any = None) -> Response:custom_tool = self.get_object()@@ -1294,10 +1346,20 @@ class PromptStudioCoreView(viewsets.ModelViewSet):instance: CustomTool = self.get_object()is_used, workflow_ids = self._check_tool_usage_in_workflows(instance)+ # NULL last_exported_at → treat as clean to avoid false alarms+ # on pre-feature projects.+ is_lookup_dirty = False+ if instance.last_exported_at is not None:+ latest = get_latest_lookup_mutation_for_tool(instance)+ is_lookup_dirty = (+ latest is not None and latest > instance.last_exported_at+ )+deployment_info: dict = {"is_used": is_used,"deployment_types": [],"message": "",+ "is_lookup_dirty": is_lookup_dirty,}if is_used and workflow_ids:diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.pyindex 64932eb3f..9c963cf3f 100644--- a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py@@ -2,8 +2,15 @@ import jsonimport loggingfrom typing import Any-from django.core.exceptions import ObjectDoesNotExist-+from django.core.exceptions import ValidationError+from django.db import IntegrityError++from prompt_studio.lookup_utils import (+ attach_combined_output_enrichment,+ extract_prompt_output_enrichment,+ get_original_value_if_enriched,+ persist_lookup_output,+)from prompt_studio.prompt_profile_manager_v2.models import ProfileManagerfrom prompt_studio.prompt_studio_core_v2.exceptions import (AnswerFetchError,@@ -170,6 +177,15 @@ class OutputManagerHelper:# TODO: use enums hereoutput = outputs.get(prompt.prompt_key)++ # On enrichment, store the raw LLM output here; the enriched+ # value is persisted separately via persist_lookup_output.+ enrichment = get_original_value_if_enriched(metadata, prompt.prompt_key)+ if enrichment is not None:+ output, prompt_lookup = enrichment+ else:+ prompt_lookup = None+if prompt.enforce_type in {"json","table",@@ -195,6 +211,18 @@ class OutputManagerHelper:word_confidence_data=prompt_word_confidence_data,)+ # Narrow except so plugin contract drift surfaces as a real+ # error instead of being masked as a successful save.+ if prompt_lookup:+ try:+ persist_lookup_output(prompt_output, prompt_lookup)+ except (IntegrityError, ValidationError):+ logger.error(+ "Failed to persist lookup output for prompt %s",+ prompt.prompt_key,+ exc_info=True,+ )+# Serialize the instanceserializer = PromptStudioOutputSerializer(prompt_output)serialized_data.append(serializer.data)@@ -242,41 +270,76 @@ class OutputManagerHelper:Returns:dict[str, Any]: Formatted JSON response for combined output.+ Cloud plugins may attach an opaque enrichment payload."""- # Initialize the result dictionary+ from prompt_studio.lookup_utils import enrich_prompt_output++ # Memoise default-profile resolution per tool to avoid N+1 on this+ # hot path (panel-switch latency).+ default_profile_cache: dict[str, str | None] = {}++ def _resolve(tool_prompt: ToolStudioPrompt) -> str | None:+ profile_manager_id = tool_prompt.profile_manager_id+ if profile_manager_id:+ return profile_manager_id+ if not use_default_profile:+ return None+ tool_id = tool_prompt.tool_id_id+ if tool_id not in default_profile_cache:+ try:+ default_profile_cache[tool_id] = (+ ProfileManager.get_default_llm_profile(+ tool_prompt.tool_id+ ).profile_id+ )+ except DefaultProfileError:+ default_profile_cache[tool_id] = None+ return default_profile_cache[tool_id]++ prompts_to_query: list[tuple[ToolStudioPrompt, str]] = []result: dict[str, Any] = {}- # Iterate over ToolStudioPrompt recordsfor tool_prompt in tool_studio_prompts:if tool_prompt.prompt_type == PSOMKeys.NOTES:continue- prompt_id = str(tool_prompt.prompt_id)- profile_manager_id = tool_prompt.profile_manager_id-- # If profile_manager is not set, skip this record- if not profile_manager_id and not use_default_profile:+ profile_manager_id = _resolve(tool_prompt)+ if profile_manager_id is None:result[tool_prompt.prompt_key] = ""continue-- if not profile_manager_id:- default_profile = ProfileManager.get_default_llm_profile(- tool_prompt.tool_id- )- profile_manager_id = default_profile.profile_id-- try:- queryset = PromptStudioOutputManager.objects.filter(- prompt_id=prompt_id,- profile_manager=profile_manager_id,+ prompts_to_query.append((tool_prompt, profile_manager_id))++ # ``DISTINCT ON`` (Postgres) yields the latest row per+ # (prompt_id, profile_manager_id) at the SQL layer.+ outputs_index: dict[tuple[str, str], PromptStudioOutputManager] = {}+ if prompts_to_query:+ prompt_ids = [str(p.prompt_id) for p, _ in prompts_to_query]+ profile_ids = list({pmid for _, pmid in prompts_to_query})+ outputs = (+ PromptStudioOutputManager.objects.filter(+ prompt_id__in=prompt_ids,+ profile_manager_id__in=profile_ids,is_single_pass_extract=False,document_manager_id=document_manager_id,)-- if not queryset.exists():- result[tool_prompt.prompt_key] = ""- continue-- for output in queryset:- result[tool_prompt.prompt_key] = output.output- except ObjectDoesNotExist:+ .order_by("prompt_id", "profile_manager_id", "-modified_at")+ .distinct("prompt_id", "profile_manager_id")+ )+ outputs_index = {+ (str(o.prompt_id), str(o.profile_manager_id)): o for o in outputs+ }++ enrichment_by_key: dict[str, Any] = {}+ for tool_prompt, profile_manager_id in prompts_to_query:+ output = outputs_index.get(+ (str(tool_prompt.prompt_id), str(profile_manager_id))+ )+ if output is None:result[tool_prompt.prompt_key] = ""+ continue+ result[tool_prompt.prompt_key] = output.output+ enriched = enrich_prompt_output(output, {})+ bundle = extract_prompt_output_enrichment(enriched)+ if bundle is not None:+ enrichment_by_key[tool_prompt.prompt_key] = bundle++ attach_combined_output_enrichment(result, enrichment_by_key)return resultdiff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py b/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.pyindex 275e4a095..1c56e2323 100644--- a/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py@@ -4,6 +4,7 @@ import loggingfrom usage_v2.helper import UsageHelperfrom backend.serializers import AuditSerializer+from prompt_studio.lookup_utils import enrich_prompt_outputfrom .models import PromptStudioOutputManagerfrom .output_manager_util import OutputManagerUtils@@ -47,6 +48,16 @@ class PromptStudioOutputSerializer(AuditSerializer):" | Process continued")data["coverage"] = {}+ # log+continue: enrichment failure shouldn't 500 the list endpoint.+ try:+ data = enrich_prompt_output(instance, data)+ except Exception as e:+ logger.error(+ "Error occurred while enriching prompt output for "+ f"prompt_id {instance.prompt_id} (run_id={instance.run_id}): {e}"+ " | Process continued"+ )+# Convert string to listtry:context = data["context"]diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/urls.py b/backend/prompt_studio/prompt_studio_output_manager_v2/urls.pyindex 61ec8540f..77270c22f 100644--- a/backend/prompt_studio/prompt_studio_output_manager_v2/urls.py+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/urls.py@@ -7,6 +7,7 @@ prompt_doc_list = PromptStudioOutputView.as_view({"get": "list"})get_output_for_tool_default = PromptStudioOutputView.as_view({"get": "get_output_for_tool_default"})+latest_outputs_by_keys = PromptStudioOutputView.as_view({"get": "latest_outputs_by_keys"})urlpatterns = format_suffix_patterns([@@ -16,5 +17,10 @@ urlpatterns = format_suffix_patterns(get_output_for_tool_default,name="prompt-default-profile-outputs",),+ path(+ "prompt-output/latest-by-keys/",+ latest_outputs_by_keys,+ name="prompt-output-latest-by-keys",+ ),])diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/views.py b/backend/prompt_studio/prompt_studio_output_manager_v2/views.pyindex c0c002d80..44111dc74 100644--- a/backend/prompt_studio/prompt_studio_output_manager_v2/views.py+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/views.py@@ -5,11 +5,12 @@ from django.core.exceptions import ObjectDoesNotExistfrom django.db.models import QuerySetfrom django.http import HttpRequestfrom rest_framework import status, viewsets-from rest_framework.exceptions import APIException+from rest_framework.exceptions import ValidationErrorfrom rest_framework.response import Responsefrom rest_framework.versioning import URLPathVersioningfrom utils.common_utils import CommonUtilsfrom utils.filtering import FilterHelper+from utils.user_context import UserContextfrom prompt_studio.prompt_studio_output_manager_v2.constants import (PromptOutputManagerErrorMessage,@@ -61,6 +62,57 @@ class PromptStudioOutputView(viewsets.ModelViewSet):return queryset+ def latest_outputs_by_keys(self, request: HttpRequest) -> Response:+ """Return the most recent raw output value per source prompt key.++ Backs the lookup Test panel's "Use Latest Outputs" button. Returns+ raw extraction (not enriched) so the lookup can be tested fresh.+ """+ tool_id = request.GET.get("tool_id")+ keys_param = request.GET.get("prompt_keys", "")+ if not tool_id:+ # APIException(code=400) returns 500; ValidationError returns 400.+ raise ValidationError(detail=PromptOutputManagerErrorMessage.TOOL_VALIDATION)++ prompt_keys = [k.strip() for k in keys_param.split(",") if k.strip()]+ if not prompt_keys:+ return Response({}, status=status.HTTP_200_OK)++ # Custom actions skip filter_queryset(), so OrganizationFilterBackend+ # never runs — scope explicitly to prevent cross-tenant reads.+ organization = UserContext.get_organization()+ prompt_id_to_key = dict(+ ToolStudioPrompt.objects.filter(+ tool_id=tool_id,+ tool_id__organization=organization,+ prompt_key__in=prompt_keys,+ ).values_list("prompt_id", "prompt_key")+ )+ if not prompt_id_to_key:+ return Response({}, status=status.HTTP_200_OK)++ # ``DISTINCT ON("prompt_id")`` keeps the latest row per prompt at+ # the SQL layer to avoid materialising every doc × run combo.+ outputs = (+ PromptStudioOutputManager.objects.filter(+ prompt_id__in=prompt_id_to_key.keys(),+ tool_id__organization=organization,+ )+ .exclude(output__isnull=True)+ .exclude(output__exact="")+ .order_by("prompt_id", "-modified_at")+ .distinct("prompt_id")+ .values("prompt_id", "output")+ )++ result: dict[str, str] = {}+ for row in outputs:+ key = prompt_id_to_key.get(row["prompt_id"])+ if key:+ result[key] = row["output"]++ return Response(result, status=status.HTTP_200_OK)+def get_output_for_tool_default(self, request: HttpRequest) -> Response:# Get the tool_id from request parameters# TODO: Setup Serializer here@@ -69,7 +121,7 @@ class PromptStudioOutputView(viewsets.ModelViewSet):tool_validation_message = PromptOutputManagerErrorMessage.TOOL_VALIDATIONtool_not_found = PromptOutputManagerErrorMessage.TOOL_NOT_FOUNDif not tool_id:- raise APIException(detail=tool_validation_message, code=400)+ raise ValidationError(detail=tool_validation_message)try:# Fetch ToolStudioPrompt records based on tool_id@@ -77,7 +129,7 @@ class PromptStudioOutputView(viewsets.ModelViewSet):tool_id=tool_id).order_by("sequence_number")except ObjectDoesNotExist:- raise APIException(detail=tool_not_found, code=400)+ raise ValidationError(detail=tool_not_found)# Invoke helper method to frame and fetch default response.result: dict[str, Any] = OutputManagerHelper.fetch_default_output_response(diff --git a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.pyindex 08d2c27ba..4fee8c10b 100644--- a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py+++ b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py@@ -7,6 +7,7 @@ from django.conf import settingsfrom django.db import IntegrityErrorfrom plugins import get_plugin+from prompt_studio.lookup_utils import validate_lookups_for_exportfrom prompt_studio.prompt_profile_manager_v2.models import ProfileManagerfrom prompt_studio.prompt_studio_core_v2.models import CustomToolfrom prompt_studio.prompt_studio_core_v2.prompt_studio_helper import PromptStudioHelper@@ -297,6 +298,15 @@ class PromptStudioRegistryHelper:settings, JsonSchemaKey.WORD_CONFIDENCE_POSTAMBLE.upper(), "")+ # Validate only what will actually be exported — NOTES / inactive+ # prompts never run, so incomplete lookups on them shouldn't fail export.+ exportable_prompts = [+ p for p in prompts if p.prompt_type != JsonSchemaKey.NOTES and p.active+ ]+ lookup_configs, lookup_error = validate_lookups_for_export(exportable_prompts)+ if lookup_error:+ raise InValidCustomToolError(lookup_error)+for prompt in prompts:if prompt.prompt_type == JsonSchemaKey.NOTES or not prompt.active:continue@@ -355,6 +365,9 @@ class PromptStudioRegistryHelper:output[JsonSchemaKey.POSTPROCESSING_WEBHOOK_URL] = (prompt.postprocessing_webhook_url)+ prompt_id_str = str(prompt.prompt_id)+ if prompt_id_str in lookup_configs:+ output["lookup_config"] = lookup_configs[prompt_id_str]# Retaining the old fields in condition# for backward compatibility. To be removed in future.if (diff --git a/backend/prompt_studio/prompt_studio_v2/serializers.py b/backend/prompt_studio/prompt_studio_v2/serializers.pyindex e1adddc33..6a4d28032 100644--- a/backend/prompt_studio/prompt_studio_v2/serializers.py+++ b/backend/prompt_studio/prompt_studio_v2/serializers.py@@ -5,6 +5,23 @@ from backend.serializers import AuditSerializerfrom .models import ToolStudioPrompt+class ToolStudioPromptListSerializer(serializers.ModelSerializer):+ """Lightweight serializer for listing prompts by tool.++ Returns only the fields needed for linking/display without+ output data or coverage calculation.+ """++ class Meta:+ model = ToolStudioPrompt+ fields = [+ "prompt_id",+ "prompt_key",+ "enforce_type",+ "sequence_number",+ ]++class ToolStudioPromptSerializer(AuditSerializer):class Meta:model = ToolStudioPromptdiff --git a/backend/prompt_studio/prompt_studio_v2/urls.py b/backend/prompt_studio/prompt_studio_v2/urls.pyindex 23e5f0243..0ce3d0354 100644--- a/backend/prompt_studio/prompt_studio_v2/urls.py+++ b/backend/prompt_studio/prompt_studio_v2/urls.py@@ -3,6 +3,8 @@ from rest_framework.urlpatterns import format_suffix_patternsfrom .views import ToolStudioPromptView+prompt_studio_prompt_list = ToolStudioPromptView.as_view({"get": "list"})+prompt_studio_prompt_detail = ToolStudioPromptView.as_view({"get": "retrieve",@@ -16,15 +18,20 @@ reorder_prompts = ToolStudioPromptView.as_view({"post": "reorder_prompts"})urlpatterns = format_suffix_patterns([+ path(+ "prompt/reorder/",+ reorder_prompts,+ name="reorder_prompts",+ ),path("prompt/<uuid:pk>/",prompt_studio_prompt_detail,name="tool-studio-prompt-detail",),path(- "prompt/reorder/",- reorder_prompts,- name="reorder_prompts",+ "prompt/",+ prompt_studio_prompt_list,+ name="tool-studio-prompt-list",),])diff --git a/backend/prompt_studio/prompt_studio_v2/views.py b/backend/prompt_studio/prompt_studio_v2/views.pyindex f120baf15..a54048027 100644--- a/backend/prompt_studio/prompt_studio_v2/views.py+++ b/backend/prompt_studio/prompt_studio_v2/views.py@@ -10,7 +10,10 @@ from prompt_studio.permission import PromptAcesssToUserfrom prompt_studio.prompt_studio_v2.constants import ToolStudioPromptKeysfrom prompt_studio.prompt_studio_v2.controller import PromptStudioControllerfrom prompt_studio.prompt_studio_v2.models import ToolStudioPrompt-from prompt_studio.prompt_studio_v2.serializers import ToolStudioPromptSerializer+from prompt_studio.prompt_studio_v2.serializers import (+ ToolStudioPromptListSerializer,+ ToolStudioPromptSerializer,+)class ToolStudioPromptView(viewsets.ModelViewSet):@@ -28,8 +31,14 @@ class ToolStudioPromptView(viewsets.ModelViewSet):versioning_class = URLPathVersioningserializer_class = ToolStudioPromptSerializer+permission_classes: list[type[PromptAcesssToUser]] = [PromptAcesssToUser]+ def get_serializer_class(self):+ if self.action == "list":+ return ToolStudioPromptListSerializer+ return ToolStudioPromptSerializer+def get_queryset(self) -> QuerySet | None:filter_args = FilterHelper.build_filter_args(self.request,diff --git a/backend/usage_v2/hooks.py b/backend/usage_v2/hooks.pynew file mode 100644index 000000000..31d576e69--- /dev/null+++ b/backend/usage_v2/hooks.py@@ -0,0 +1,27 @@+"""Post-write hooks for ``UsageBatchCreateView``.++Hooks fire inside the view's transaction; a failure rolls the batch back+so Usage rows and any side-table writes stay consistent. Records carry+an opaque ``cloud_extras`` dict that OSS forwards verbatim — plugins+read only the keys they own.+"""++from collections.abc import Callable++from .models import Usage++PostWriteHook = Callable[[list[dict], list[Usage]], None]++_post_write_hooks: list[PostWriteHook] = []+++def register_post_write_hook(fn: PostWriteHook) -> PostWriteHook:+ # Idempotent: ready() can re-fire under test reloads / dev autoreload.+ if fn not in _post_write_hooks:+ _post_write_hooks.append(fn)+ return fn+++def run_post_write_hooks(records: list[dict], usage_objects: list[Usage]) -> None:+ for hook in _post_write_hooks:+ hook(records, usage_objects)diff --git a/backend/usage_v2/internal_urls.py b/backend/usage_v2/internal_urls.pyindex b5a867555..e7e0082e9 100644--- a/backend/usage_v2/internal_urls.py+++ b/backend/usage_v2/internal_urls.py@@ -17,4 +17,9 @@ urlpatterns = [internal_views.PagesProcessedInternalView.as_view(),name="aggregated-pages-processed",),+ path(+ "batch/",+ internal_views.UsageBatchCreateView.as_view(),+ name="usage-batch-create",+ ),]diff --git a/backend/usage_v2/internal_views.py b/backend/usage_v2/internal_views.pyindex 3c28779c5..85c20c16e 100644--- a/backend/usage_v2/internal_views.py+++ b/backend/usage_v2/internal_views.py@@ -1,19 +1,35 @@-"""Internal API views for Usage access by workers."""+"""Internal API views for Usage access by workers.++Mounted under ``/internal/`` and gated by ``InternalAPIAuthMiddleware``.+"""import logging+from django.db import transactionfrom django.http import JsonResponsefrom rest_framework import status+from rest_framework.exceptions import APIException, ValidationErrorfrom rest_framework.request import Request+from rest_framework.response import Responsefrom rest_framework.views import APIView+from utils.user_context import UserContextfrom unstract.core.data_models import UsageResponseDatafrom .helper import UsageHelper+from .hooks import run_post_write_hooks+from .models import Usage+from .serializers import UsageBatchCreateSerializerlogger = logging.getLogger(__name__)+class UsagePersistError(APIException):+ status_code = status.HTTP_500_INTERNAL_SERVER_ERROR+ default_detail = "Failed to persist usage records."+ default_code = "usage_persist_failed"++class UsageInternalView(APIView):"""Internal API view for workers to access usage data.@@ -133,3 +149,66 @@ class PagesProcessedInternalView(APIView):},status=status.HTTP_500_INTERNAL_SERVER_ERROR,)+++class UsageBatchCreateView(APIView):+ """Bulk create usage records from worker finalization."""++ def post(self, request: Request) -> Response:+ input_serializer = UsageBatchCreateSerializer(data=request.data)+ input_serializer.is_valid(raise_exception=True)+ records = input_serializer.validated_data["records"]+ if not records:+ return Response({"created": 0}, status=status.HTTP_200_OK)++ organization = UserContext.get_organization()+ if organization is None:+ logger.error(+ "UsageBatchCreateView received %d records with no organization; "+ "refusing to write rows that would be invisible to tenant dashboards",+ len(records),+ )+ raise ValidationError(+ "Organization context missing. Worker must send X-Organization-ID."+ )++ usage_objects = [+ Usage(+ organization=organization,+ workflow_id=r.get("workflow_id", ""),+ execution_id=r.get("execution_id", ""),+ adapter_instance_id=r.get("adapter_instance_id", ""),+ run_id=r.get("run_id"),+ usage_type=r.get("usage_type", "llm"),+ # Coerce "" to None so the cross-field CheckConstraint passes.+ llm_usage_reason=r.get("llm_usage_reason") or None,+ model_name=r.get("model_name", ""),+ embedding_tokens=r.get("embedding_tokens", 0),+ prompt_tokens=r.get("prompt_tokens", 0),+ completion_tokens=r.get("completion_tokens", 0),+ total_tokens=r.get("total_tokens", 0),+ cost_in_dollars=r.get("cost_in_dollars", 0.0),+ project_id=r.get("project_id"),+ prompt_id=r.get("prompt_id"),+ execution_time_ms=r.get("execution_time_ms"),+ status=r.get("status"),+ error_message=r.get("error_message"),+ )+ for r in records+ ]++ try:+ # Atomic with hooks: orphan Usage rows are worse than retrying.+ with transaction.atomic():+ created = Usage.objects.bulk_create(usage_objects, batch_size=500)+ run_post_write_hooks(records, created)+ except Exception as e:+ logger.error(+ "bulk_create failed for %d usage records (org=%s): %s",+ len(usage_objects),+ organization.organization_id,+ e,+ exc_info=True,+ )+ raise UsagePersistError() from e+ return Response({"created": len(created)}, status=status.HTTP_201_CREATED)diff --git a/backend/usage_v2/migrations/0004_usage_metrics_fields.py b/backend/usage_v2/migrations/0004_usage_metrics_fields.pynew file mode 100644index 000000000..191a31dd2--- /dev/null+++ b/backend/usage_v2/migrations/0004_usage_metrics_fields.py@@ -0,0 +1,89 @@+from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("usage_v2", "0003_usage_usage_executi_4deb35_idx"),+ ]++ operations = [+ # Extend llm_usage_reason choices (cloud plugins append at runtime)+ migrations.AlterField(+ model_name="usage",+ name="llm_usage_reason",+ field=models.CharField(+ blank=True,+ choices=[+ ("extraction", "Extraction"),+ ("challenge", "Challenge"),+ ("summarize", "Summarize"),+ ("lookup", "Lookup"),+ ],+ db_comment="Reason for LLM usage. Empty if usage_type is 'embedding'. ",+ max_length=255,+ null=True,+ ),+ ),+ migrations.AddField(+ model_name="usage",+ name="project_id",+ field=models.UUIDField(+ blank=True,+ db_comment=(+ "Prompt Studio project (tool) the call belongs to (no FK; "+ "survives tool deletion). NULL for embeddings and historical "+ "rows."+ ),+ null=True,+ ),+ ),+ migrations.AddField(+ model_name="usage",+ name="prompt_id",+ field=models.UUIDField(+ blank=True,+ db_comment=(+ "Prompt key UUID that triggered the call (no FK; survives "+ "prompt deletion). NULL for single-pass / embeddings / "+ "historical rows."+ ),+ null=True,+ ),+ ),+ migrations.AddField(+ model_name="usage",+ name="execution_time_ms",+ field=models.IntegerField(+ blank=True,+ db_comment="Wall-clock time for the operation in milliseconds",+ null=True,+ ),+ ),+ migrations.AddField(+ model_name="usage",+ name="status",+ field=models.CharField(+ blank=True,+ choices=[+ ("SUCCESS", "Success"),+ ("ERROR", "Error"),+ ("SKIPPED", "Skipped"),+ ],+ db_comment="Operation outcome: SUCCESS, ERROR, or SKIPPED",+ max_length=16,+ null=True,+ ),+ ),+ migrations.AddField(+ model_name="usage",+ name="error_message",+ field=models.TextField(+ blank=True,+ db_comment="Error details when status is ERROR",+ null=True,+ ),+ ),+ # Indexes on project_id / prompt_id moved to 0005 so they can run+ # CONCURRENTLY — usage is billing-critical and a plain AddIndex takes+ # a share-update lock for the duration of the build on large tables.+ ]diff --git a/backend/usage_v2/migrations/0005_usage_reason_ref_created_idx.py b/backend/usage_v2/migrations/0005_usage_reason_ref_created_idx.pynew file mode 100644index 000000000..c93d4de60--- /dev/null+++ b/backend/usage_v2/migrations/0005_usage_reason_ref_created_idx.py@@ -0,0 +1,76 @@+from django.db import migrations, models+from django.db.models import Q+++class Migration(migrations.Migration):+ """Build the project_id / prompt_id dashboard indexes without locking.++ CONCURRENTLY requires the migration to run outside a transaction, hence+ atomic = False. RunSQL with IF NOT EXISTS makes a partial-apply+ (process killed between SQL success and django_migrations insert)+ recoverable on retry without manual --fake intervention.+ """++ atomic = False++ dependencies = [+ ("usage_v2", "0004_usage_metrics_fields"),+ ]++ operations = [+ migrations.RunSQL(+ sql=(+ "CREATE INDEX CONCURRENTLY IF NOT EXISTS "+ "idx_usage_project_created "+ 'ON "usage" (project_id, created_at DESC);'+ ),+ reverse_sql=("DROP INDEX CONCURRENTLY IF EXISTS idx_usage_project_created;"),+ state_operations=[+ migrations.AddIndex(+ model_name="usage",+ index=models.Index(+ fields=["project_id", "-created_at"],+ name="idx_usage_project_created",+ ),+ ),+ ],+ ),+ migrations.RunSQL(+ sql=(+ "CREATE INDEX CONCURRENTLY IF NOT EXISTS "+ "idx_usage_prompt_created "+ 'ON "usage" (prompt_id, created_at DESC);'+ ),+ reverse_sql=("DROP INDEX CONCURRENTLY IF EXISTS idx_usage_prompt_created;"),+ state_operations=[+ migrations.AddIndex(+ model_name="usage",+ index=models.Index(+ fields=["prompt_id", "-created_at"],+ name="idx_usage_prompt_created",+ ),+ ),+ ],+ ),+ # Partial — only lookup-reason rows. Avoids heap-scanning all+ # Usage rows when the dashboard groups by (run × prompt).+ migrations.RunSQL(+ sql=(+ "CREATE INDEX CONCURRENTLY IF NOT EXISTS "+ "idx_usage_lookup_recent "+ 'ON "usage" (organization_id, created_at DESC) '+ "WHERE llm_usage_reason = 'lookup';"+ ),+ reverse_sql=("DROP INDEX CONCURRENTLY IF EXISTS idx_usage_lookup_recent;"),+ state_operations=[+ migrations.AddIndex(+ model_name="usage",+ index=models.Index(+ fields=["organization", "-created_at"],+ name="idx_usage_lookup_recent",+ condition=Q(llm_usage_reason="lookup"),+ ),+ ),+ ],+ ),+ ]diff --git a/backend/usage_v2/models.py b/backend/usage_v2/models.pyindex 57ae3d143..0267d4a68 100644--- a/backend/usage_v2/models.py+++ b/backend/usage_v2/models.py@@ -1,6 +1,7 @@import uuidfrom django.db import models+from django.db.models import Qfrom utils.models.base_model import BaseModel, BaseModelManagerfrom utils.models.organization_mixin import (DefaultOrganizationManagerMixin,@@ -13,10 +14,21 @@ class UsageType(models.TextChoices):EMBEDDING = "embedding", "Embedding Usage"-class LLMUsageReason(models.TextChoices):- EXTRACTION = "extraction", "Extraction"- CHALLENGE = "challenge", "Challenge"- SUMMARIZE = "summarize", "Summarize"+class UsageStatus(models.TextChoices):+ SUCCESS = "SUCCESS", "Success"+ ERROR = "ERROR", "Error"+ SKIPPED = "SKIPPED", "Skipped"+++# Static union of OSS + cloud values — keeps OSS model state aligned with+# migration state so ``makemigrations --check`` doesn't drift in CI.++LLM_USAGE_REASON_CHOICES: list[tuple[str, str]] = [+ ("extraction", "Extraction"),+ ("challenge", "Challenge"),+ ("summarize", "Summarize"),+ ("lookup", "Lookup"),+]class UsageModelManager(DefaultOrganizationManagerMixin, BaseModelManager):@@ -52,7 +64,7 @@ class Usage(DefaultOrganizationMixin, BaseModel):)llm_usage_reason = models.CharField(max_length=255,- choices=LLMUsageReason.choices,+ choices=LLM_USAGE_REASON_CHOICES,null=True,blank=True,db_comment="Reason for LLM usage. Empty if usage_type is 'embedding'. ",@@ -67,6 +79,39 @@ class Usage(DefaultOrganizationMixin, BaseModel):)total_tokens = models.IntegerField(db_comment="Total number of tokens used")cost_in_dollars = models.FloatField(db_comment="Total number of tokens used")+ project_id = models.UUIDField(+ null=True,+ blank=True,+ db_comment=(+ "Prompt Studio project (tool) the call belongs to (no FK; survives "+ "tool deletion). NULL for embeddings and historical rows."+ ),+ )+ prompt_id = models.UUIDField(+ null=True,+ blank=True,+ db_comment=(+ "Prompt key UUID that triggered the call (no FK; survives prompt "+ "deletion). NULL for single-pass / embeddings / historical rows."+ ),+ )+ execution_time_ms = models.IntegerField(+ null=True,+ blank=True,+ db_comment="Wall-clock time for the operation in milliseconds",+ )+ status = models.CharField(+ max_length=16,+ choices=UsageStatus.choices,+ null=True,+ blank=True,+ db_comment="Operation outcome: SUCCESS, ERROR, or SKIPPED",+ )+ error_message = models.TextField(+ null=True,+ blank=True,+ db_comment="Error details when status is ERROR",+ )# Managerobjects = UsageModelManager()@@ -78,4 +123,17 @@ class Usage(DefaultOrganizationMixin, BaseModel):indexes = [models.Index(fields=["run_id"]),models.Index(fields=["execution_id"]),+ models.Index(+ fields=["project_id", "-created_at"],+ name="idx_usage_project_created",+ ),+ models.Index(+ fields=["prompt_id", "-created_at"],+ name="idx_usage_prompt_created",+ ),+ models.Index(+ fields=["organization", "-created_at"],+ name="idx_usage_lookup_recent",+ condition=Q(llm_usage_reason="lookup"),+ ),]diff --git a/backend/usage_v2/serializers.py b/backend/usage_v2/serializers.pyindex a82329708..7b7f8aaed 100644--- a/backend/usage_v2/serializers.py+++ b/backend/usage_v2/serializers.py@@ -21,3 +21,38 @@ class UsageSerializer(serializers.ModelSerializer):class Meta:model = Usagefields = "__all__"+++class UsageRecordCreateSerializer(serializers.Serializer):+ """Worker-emitted usage record. Required fields anchor billing-critical attribution."""++ adapter_instance_id = serializers.CharField(required=True, allow_blank=False)+ model_name = serializers.CharField(required=True, allow_blank=False)+ usage_type = serializers.CharField(required=True, allow_blank=False)++ workflow_id = serializers.CharField(required=False, allow_blank=True, default="")+ execution_id = serializers.CharField(required=False, allow_blank=True, default="")+ run_id = serializers.UUIDField(required=False, allow_null=True, default=None)+ llm_usage_reason = serializers.CharField(+ required=False, allow_null=True, allow_blank=True, default=None+ )+ embedding_tokens = serializers.IntegerField(required=False, default=0)+ prompt_tokens = serializers.IntegerField(required=False, default=0)+ completion_tokens = serializers.IntegerField(required=False, default=0)+ total_tokens = serializers.IntegerField(required=False, default=0)+ cost_in_dollars = serializers.FloatField(required=False, default=0.0)+ project_id = serializers.UUIDField(required=False, allow_null=True, default=None)+ prompt_id = serializers.UUIDField(required=False, allow_null=True, default=None)+ execution_time_ms = serializers.IntegerField(+ required=False, allow_null=True, default=None+ )+ status = serializers.CharField(required=False, allow_null=True, default=None)+ error_message = serializers.CharField(+ required=False, allow_null=True, allow_blank=True, default=None+ )+ # Opaque carrier forwarded to post-write hooks; OSS never reads it.+ cloud_extras = serializers.DictField(required=False, allow_null=True, default=None)+++class UsageBatchCreateSerializer(serializers.Serializer):+ records = UsageRecordCreateSerializer(many=True, allow_empty=True)diff --git a/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx b/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsxindex 3036e7b9d..7e30a9933 100644--- a/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx+++ b/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx@@ -47,6 +47,67 @@ try {// The component will remain null if it is not available}+// OSS falls back to passthrough helpers — no enrichment.+let splitCombinedData = (data) => ({ combined: data, bundle: null });+let buildEnrichedFromBundle = (_output, _bundle, _formatter) => ({});+let getEnrichmentFromItem = (_item) => null;+try {+ const mod = await import("../../../plugins/lookup-enriched-toggle/helpers");+ splitCombinedData = mod.splitCombinedData;+ buildEnrichedFromBundle = mod.buildEnrichedFromBundle;+ getEnrichmentFromItem = mod.getEnrichmentFromItem;+} catch {}++const buildDefaultProfileOutputs = (data) => {+ const { combined: payload, bundle } = splitCombinedData(data);+ const output = Object.entries(payload).reduce((acc, [key, value]) => {+ acc[key] = displayPromptResult(value, false);+ return acc;+ }, {});+ const enriched = buildEnrichedFromBundle(output, bundle, displayPromptResult);+ return {+ output,+ enriched,+ hasEnriched: bundle != null && Object.keys(enriched).length > 0,+ };+};++const buildPerPromptOutput = (item, data, selectedProfile) => {+ const profileManager = selectedProfile || item?.profile_manager;+ const outputDetails = data.find(+ (outputValue) =>+ outputValue?.prompt_id === item?.prompt_id &&+ outputValue?.profile_manager === profileManager,+ );+ const value =+ outputDetails?.output?.length > 0+ ? displayPromptResult(outputDetails?.output, false)+ : "";+ const enrichment = getEnrichmentFromItem(outputDetails);+ const enrichedValue = enrichment?.output+ ? displayPromptResult(enrichment.output, false)+ : value;+ return { value, enrichedValue, hasEnriched: !!enrichment?.output };+};++const buildSelectedProfileOutputs = (data, prompts, selectedProfile) => {+ const output = {};+ const enriched = {};+ let hasEnriched = false;+ for (const item of prompts) {+ if (item?.prompt_type === promptType.notes) continue;+ const {+ value,+ enrichedValue,+ hasEnriched: enrichedHit,+ } = buildPerPromptOutput(item, data, selectedProfile);+ output[item?.prompt_key] = value;+ enriched[item?.prompt_key] = enrichedValue;+ hasEnriched = hasEnriched || enrichedHit;+ }+ return { output, enriched: hasEnriched ? enriched : {}, hasEnriched };+};+function CombinedOutput({ docId, setFilledFields, selectedPrompts }) {const {details,@@ -59,6 +120,7 @@ function CombinedOutput({ docId, setFilledFields, selectedPrompts }) {} = useCustomToolStore();const [combinedOutput, setCombinedOutput] = useState({});+ const [enrichedOutput, setEnrichedOutput] = useState({});const [isOutputLoading, setIsOutputLoading] = useState(false);const [adapterData, setAdapterData] = useState([]);const [activeKey, setActiveKey] = useState(@@ -115,32 +177,12 @@ function CombinedOutput({ docId, setFilledFields, selectedPrompts }) {const res = await handleOutputApiRequest();const data = res?.data || [];const prompts = details?.prompts || [];-- if (activeKey === "0" && !isSimplePromptStudio) {- const output = Object.entries(data).reduce((acc, [key, value]) => {- acc[key] = displayPromptResult(value, false);- return acc;- }, {});- setCombinedOutput(output);- } else {- const output = prompts.reduce((acc, item) => {- if (item?.prompt_type !== promptType.notes) {- const profileManager = selectedProfile || item?.profile_manager;- const outputDetails = data.find(- (outputValue) =>- outputValue?.prompt_id === item?.prompt_id &&- outputValue?.profile_manager === profileManager,- );-- acc[item?.prompt_key] =- outputDetails?.output?.length > 0- ? displayPromptResult(outputDetails?.output, false)- : "";- }- return acc;- }, {});- setCombinedOutput(output);- }+ const useDefaultProfile = activeKey === "0" && !isSimplePromptStudio;+ const { output, enriched } = useDefaultProfile+ ? buildDefaultProfileOutputs(data)+ : buildSelectedProfileOutputs(data, prompts, selectedProfile);+ setCombinedOutput(output);+ setEnrichedOutput(enriched);} catch (err) {setAlertDetails(handleException(err, "Failed to generate combined output"),@@ -229,6 +271,7 @@ function CombinedOutput({ docId, setFilledFields, selectedPrompts }) {return (<JsonViewcombinedOutput={filteredCombinedOutput}+ enrichedOutput={enrichedOutput}handleTabChange={handleTabChange}selectedProfile={selectedProfile}llmProfiles={llmProfiles}diff --git a/frontend/src/components/custom-tools/combined-output/JsonView.jsx b/frontend/src/components/custom-tools/combined-output/JsonView.jsxindex f9d6a86a8..34a733598 100644--- a/frontend/src/components/custom-tools/combined-output/JsonView.jsx+++ b/frontend/src/components/custom-tools/combined-output/JsonView.jsx@@ -2,12 +2,23 @@ import { Tabs } from "antd";import TabPane from "antd/es/tabs/TabPane";import Prism from "prismjs";import PropTypes from "prop-types";-import { useEffect } from "react";+import { useEffect, useState } from "react";import { JsonViewBody } from "./JsonViewBody";+let EnrichedOutputToggle;+try {+ const mod = await import(+ "../../../plugins/lookup-enriched-toggle/EnrichedOutputToggle"+ );+ EnrichedOutputToggle = mod.EnrichedOutputToggle;+} catch {+ // The component will remain undefined if it is not available+}+function JsonView({combinedOutput,+ enrichedOutput,handleTabChange,adapterData,activeKey,@@ -16,9 +27,24 @@ function JsonView({isSinglePass,isLoading,}) {+ const [activeView, setActiveView] = useState("Raw");+useEffect(() => {Prism.highlightAll();- }, [combinedOutput]);+ }, [combinedOutput, enrichedOutput, activeView]);++ useEffect(() => {+ if (!enrichedOutput || Object.keys(enrichedOutput).length === 0) {+ setActiveView("Raw");+ }+ }, [enrichedOutput]);++ const displayOutput =+ activeView === "Enriched" &&+ enrichedOutput &&+ Object.keys(enrichedOutput).length > 0+ ? enrichedOutput+ : combinedOutput;return (<div className="combined-op-layout">@@ -34,14 +60,24 @@ function JsonView({/>))}</Tabs>- <div className="combined-op-segment"></div>+ <div className="combined-op-segment">+ {EnrichedOutputToggle && (+ <EnrichedOutputToggle+ activeView={activeView}+ onChange={setActiveView}+ hasEnrichedData={+ !!(enrichedOutput && Object.keys(enrichedOutput).length > 0)+ }+ />+ )}+ </div></div><div className="combined-op-divider" /><JsonViewBodyactiveKey={activeKey}selectedProfile={selectedProfile}llmProfiles={llmProfiles}- combinedOutput={combinedOutput}+ combinedOutput={displayOutput}isLoading={isLoading}/><div className="gap" />@@ -51,6 +87,7 @@ function JsonView({JsonView.propTypes = {combinedOutput: PropTypes.object.isRequired,+ enrichedOutput: PropTypes.object,handleTabChange: PropTypes.func,adapterData: PropTypes.array,selectedProfile: PropTypes.string,diff --git a/frontend/src/components/custom-tools/document-parser/DocumentParser.jsx b/frontend/src/components/custom-tools/document-parser/DocumentParser.jsxindex 3d5c891e1..63b1a037c 100644--- a/frontend/src/components/custom-tools/document-parser/DocumentParser.jsx+++ b/frontend/src/components/custom-tools/document-parser/DocumentParser.jsx@@ -1,5 +1,6 @@import PropTypes from "prop-types";import { useEffect, useRef, useState } from "react";+import { useSearchParams } from "react-router-dom";import "./DocumentParser.css";import { promptType } from "../../../helpers/GetStaticData";@@ -32,6 +33,10 @@ try {// The component will remain null of it is not available}+// Module-scoped to avoid per-render recompilation.+const UUID_RE =+ /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;+function DocumentParser({addPromptInstance,scrollToBottom,@@ -42,6 +47,7 @@ function DocumentParser({const [isChallenge, setIsChallenge] = useState(false);const [allTableSettings, setAllTableSettings] = useState([]);const bottomRef = useRef(null);+ const [searchParams, setSearchParams] = useSearchParams();const {details,isSimplePromptStudio,@@ -108,6 +114,28 @@ function DocumentParser({}}, [scrollToBottom]);+ // Cross-link from Lookup Studio: scroll to a specific prompt.+ useEffect(() => {+ const scrollToPromptId = searchParams.get("scrollTo");+ if (+ !scrollToPromptId ||+ !UUID_RE.test(scrollToPromptId) ||+ !details?.prompts?.length+ ) {+ return;+ }++ const el = document.querySelector(`[data-prompt-id="${scrollToPromptId}"]`);+ if (el) {+ el.scrollIntoView({ behavior: "smooth", block: "center" });+ el.classList.add("highlighted-prompt");+ setTimeout(() => el.classList.remove("highlighted-prompt"), 2000);+ }++ searchParams.delete("scrollTo");+ setSearchParams(searchParams, { replace: true });+ }, [details?.prompts, searchParams]);+const promptUrl = (urlPath) => {return `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/prompt/${urlPath}`;};@@ -242,7 +270,7 @@ function DocumentParser({<div className="doc-parser-layout">{details?.prompts?.map((item) => {return (- <div key={item.prompt_id}>+ <div key={item.prompt_id} data-prompt-id={item.prompt_id}><div className="doc-parser-pad-top" /><PromptCardWrapperitem={item}diff --git a/frontend/src/components/custom-tools/header/Header.jsx b/frontend/src/components/custom-tools/header/Header.jsxindex 15e41c8a0..471588a4b 100644--- a/frontend/src/components/custom-tools/header/Header.jsx+++ b/frontend/src/components/custom-tools/header/Header.jsx@@ -23,30 +23,28 @@ try {"../../../plugins/single-pass-toggle-switch/SinglePassToggleSwitch");SinglePassToggleSwitch = mod.SinglePassToggleSwitch;-} catch {- // The variable will remain undefined if the component is not available.-}+} catch {}try {const mod = await import("../../../plugins/prompt-studio-public-share/public-share-btn/PromptShareButton.jsx");PromptShareButton = mod.PromptShareButton;-} catch {- // The variable will remain undefined if the component is not available.-}+} catch {}try {const mod = await import("../../../plugins/prompt-studio-clone/clone-btn/CloneButton.jsx");CloneButton = mod.CloneButton;-} catch {- // The variable will remain undefined if the component is not available.-}+} catch {}++const noopCheckLookups = () => Promise.resolve(true);+function Header({setOpenSettings,handleUpdateTool,setOpenShareModal,setOpenCloneModal,+ checkLookups = noopCheckLookups,}) {const [isExportLoading, setIsExportLoading] = useState(false);const { details, isPublicSource, markChangesAsExported } =@@ -129,7 +127,7 @@ function Header({setConfirmModalVisible(false);}, [lastExportParams, handleExport]);- const handleShare = (isEdit) => {+ const handleShare = async (isEdit) => {try {setPostHogCustomEvent("ps_exported_tool", {info: `Clicked on the 'Export' button`,@@ -139,6 +137,9 @@ function Header({// If an error occurs while setting custom posthog event, ignore it and continue}+ const ok = await checkLookups(details?.tool_id, "export");+ if (!ok) return;+const requestOptions = {method: "GET",url: `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/export/${details?.tool_id}`,@@ -255,7 +256,7 @@ function Header({});};- const handleCreateApiDeployment = () => {+ const handleCreateApiDeployment = async () => {try {setPostHogCustomEvent("intent_create_api_deployment_from_prompt_studio", {info: "Clicked Create API Deployment in tool IDE",@@ -266,6 +267,9 @@ function Header({// If an error occurs while setting custom posthog event, ignore it and continue}+ const ok = await checkLookups(details?.tool_id, "API deployment");+ if (!ok) return;+// Check for existing API deployments before proceedingsetIsApiDeploymentLoading(true);const path = `/api/v1/unstract/${sessionDetails.orgId}`;@@ -527,6 +531,7 @@ Header.propTypes = {handleUpdateTool: PropTypes.func.isRequired,setOpenCloneModal: PropTypes.func.isRequired,setOpenShareModal: PropTypes.func.isRequired,+ checkLookups: PropTypes.func,};export { Header };diff --git a/frontend/src/components/custom-tools/list-of-tools/ListOfTools.jsx b/frontend/src/components/custom-tools/list-of-tools/ListOfTools.jsxindex dd1e55853..86ab15ab1 100644--- a/frontend/src/components/custom-tools/list-of-tools/ListOfTools.jsx+++ b/frontend/src/components/custom-tools/list-of-tools/ListOfTools.jsx@@ -48,7 +48,7 @@ DefaultCustomButtons.propTypes = {handleNewProjectBtnClick: PropTypes.func.isRequired,};-function ListOfTools() {+function ListOfTools({ segmentOptions, segmentValue, onSegmentChange }) {const [isListLoading, setIsListLoading] = useState(false);const [openAddTool, setOpenAddTool] = useState(false);const [openImportTool, setOpenImportTool] = useState(false);@@ -373,12 +373,15 @@ function ListOfTools() {return (<><ToolNavBar- title={"Prompt Studio"}+ title="Prompt Studio"enableSearchonSearch={onSearch}searchList={listOfTools}setSearchList={setFilteredListOfTools}customButtons={customButtonsElement}+ segmentOptions={segmentOptions}+ segmentValue={segmentValue}+ segmentFilter={onSegmentChange}/><div className="list-of-tools-layout"><div className="list-of-tools-island">{defaultContent}</div>@@ -412,4 +415,10 @@ function ListOfTools() {);}+ListOfTools.propTypes = {+ segmentOptions: PropTypes.arrayOf(PropTypes.string),+ segmentValue: PropTypes.string,+ onSegmentChange: PropTypes.func,+};+export { ListOfTools };diff --git a/frontend/src/components/custom-tools/profile-info-bar/ProfileInfoBar.css b/frontend/src/components/custom-tools/profile-info-bar/ProfileInfoBar.cssindex 8346c26d7..da866a895 100644--- a/frontend/src/components/custom-tools/profile-info-bar/ProfileInfoBar.css+++ b/frontend/src/components/custom-tools/profile-info-bar/ProfileInfoBar.css@@ -1,3 +1,10 @@.profile-info-bar {+ display: flex;+ flex-wrap: wrap;+ gap: 6px 8px;margin-bottom: 10px;}++.profile-info-bar .ant-tag {+ margin: 0;+}diff --git a/frontend/src/components/custom-tools/profile-info-bar/ProfileInfoBar.jsx b/frontend/src/components/custom-tools/profile-info-bar/ProfileInfoBar.jsxindex 8f52a68a4..934175398 100644--- a/frontend/src/components/custom-tools/profile-info-bar/ProfileInfoBar.jsx+++ b/frontend/src/components/custom-tools/profile-info-bar/ProfileInfoBar.jsx@@ -1,4 +1,4 @@-import { Col, Row, Tag } from "antd";+import { Tag } from "antd";import PropTypes from "prop-types";import "./ProfileInfoBar.css";@@ -10,43 +10,29 @@ const ProfileInfoBar = ({ profiles, profileId }) => {}return (- <Row className="profile-info-bar">- <Col>- <Tag>- <strong>Profile Name:</strong> {profile?.profile_name}- </Tag>- </Col>- <Col>- <Tag>- <strong>Chunk Size:</strong> {profile?.chunk_size}- </Tag>- </Col>- <Col>- <Tag>- <strong>Vector Store:</strong> {profile?.vector_store}- </Tag>- </Col>- <Col>- <Tag>- <strong>Embedding Model:</strong> {profile?.embedding_model}- </Tag>- </Col>- <Col>- <Tag>- <strong>LLM:</strong> {profile?.llm}- </Tag>- </Col>- <Col>- <Tag>- <strong>X2Text:</strong> {profile?.x2text}- </Tag>- </Col>- <Col>- <Tag>- <strong>Reindex:</strong> {profile?.reindex ? "Yes" : "No"}- </Tag>- </Col>- </Row>+ <div className="profile-info-bar">+ <Tag>+ <strong>Profile Name:</strong> {profile?.profile_name}+ </Tag>+ <Tag>+ <strong>Chunk Size:</strong> {profile?.chunk_size}+ </Tag>+ <Tag>+ <strong>Vector Store:</strong> {profile?.vector_store}+ </Tag>+ <Tag>+ <strong>Embedding Model:</strong> {profile?.embedding_model}+ </Tag>+ <Tag>+ <strong>LLM:</strong> {profile?.llm}+ </Tag>+ <Tag>+ <strong>X2Text:</strong> {profile?.x2text}+ </Tag>+ <Tag>+ <strong>Reindex:</strong> {profile?.reindex ? "Yes" : "No"}+ </Tag>+ </div>);};diff --git a/frontend/src/components/custom-tools/prompt-card/Header.jsx b/frontend/src/components/custom-tools/prompt-card/Header.jsxindex cea5e2d6d..30fe75fe5 100644--- a/frontend/src/components/custom-tools/prompt-card/Header.jsx+++ b/frontend/src/components/custom-tools/prompt-card/Header.jsx@@ -41,6 +41,38 @@ try {// The component will remain 'undefined' it is not available}+let LookupMenuItem;+try {+ const mod = await import(+ "../../../plugins/lookup-studio/prompt-card/LookupMenuItem"+ );+ LookupMenuItem = mod.LookupMenuItem;+} catch {}++let usePromptRunGatePlugin;+let lookupPluginLoadError;+try {+ const mod = await import(+ "../../../plugins/lookup-studio/hooks/usePromptRunGate"+ );+ usePromptRunGatePlugin = mod.usePromptRunGate;+} catch (err) {+ lookupPluginLoadError = err;+}++// Sibling plugin loaded but this hook didn't — surface so the no-op fallback+// doesn't silently disable the run gate.+if (lookupPluginLoadError && LookupMenuItem) {+ // eslint-disable-next-line no-console+ console.warn(+ "[Header] lookup-studio plugin loaded but usePromptRunGate failed to import",+ lookupPluginLoadError,+ );+}++// Stable identity — avoid conditional hook call.+const usePromptRunGate = usePromptRunGatePlugin || (() => null);+function Header({promptDetails,promptKey,@@ -70,6 +102,7 @@ function Header({isSimplePromptStudio,details,} = useCustomToolStore();+ const runGate = usePromptRunGate(promptDetails);const [items, setItems] = useState([]);const [isDisablePrompt, setIsDisablePrompt] = useState(null);@@ -268,6 +301,20 @@ function Header({isPublicSource,},];+ if (LookupMenuItem && !isSimplePromptStudio) {+ dropdownItems.splice(+ -1,+ 0,+ {+ type: "divider",+ },+ {+ label: <LookupMenuItem promptDetails={promptDetails} />,+ key: "lookup",+ },+ );+ }+if (isSimplePromptStudio) {dropdownItems.splice(0, 1);}@@ -346,7 +393,9 @@ function Header({)}{!singlePassExtractMode && !isSimplePromptStudio && (<>- <Tooltip title="Run all LLMs for current document">+ <Tooltip+ title={runGate?.reason || "Run all LLMs for current document"}+ ><Buttonsize="small"type="text"@@ -365,13 +414,16 @@ function Header({isCoverageLoading ||indexDocs?.includes(selectedDoc?.document_id) ||isPublicSource ||- spsLoading?.[selectedDoc?.document_id]+ spsLoading?.[selectedDoc?.document_id] ||+ !!runGate?.disabled}><PlayCircleOutlined className="prompt-card-actions-head" /></Button></Tooltip>- <Tooltip title="Run all LLMs for all documents">+ <Tooltip+ title={runGate?.reason || "Run all LLMs for all documents"}+ ><Buttonsize="small"type="text"@@ -388,7 +440,8 @@ function Header({promptStudioUpdateStatus?.isUpdating) ||isCoverageLoading ||indexDocs?.includes(selectedDoc?.document_id) ||- isPublicSource+ isPublicSource ||+ !!runGate?.disabled}><PlayCircleFilled className="prompt-card-actions-head" />diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsxindex 2e579108c..29fda260e 100644--- a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx+++ b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx@@ -14,6 +14,19 @@ import { PromptCardItems } from "./PromptCardItems";import "./PromptCard.css";import { handleUpdateStatus } from "./constants";+let useEnforceTypeSwitchGatePlugin;+try {+ const mod = await import(+ "../../../plugins/lookup-studio/hooks/useEnforceTypeSwitchGate"+ );+ useEnforceTypeSwitchGatePlugin = mod.useEnforceTypeSwitchGate;+} catch {+ // Cloud plugin not present; gate falls back to no-op below.+}++const useEnforceTypeSwitchGate =+ useEnforceTypeSwitchGatePlugin || (() => () => null);+const PromptCard = memo(({promptDetails,@@ -56,6 +69,7 @@ const PromptCard = memo(const { messages } = useSocketCustomToolStore();const { setAlertDetails } = useAlertStore();const { setPostHogCustomEvent } = usePostHogEvents();+ const enforceTypeSwitchGate = useEnforceTypeSwitchGate();useEffect(() => {if (@@ -260,6 +274,11 @@ const PromptCard = memo(};const handleTypeChange = (value) => {+ const block = enforceTypeSwitchGate(promptDetailsState?.prompt_id, value);+ if (block) {+ setAlertDetails({ type: "error", content: block.reason });+ return;+ }handleChange(value, promptDetailsState?.prompt_id, "enforce_type", true);};diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx b/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsxindex 6fd8b1bc6..5cb5d099b 100644--- a/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx+++ b/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx@@ -29,6 +29,14 @@ try {// The component will remain null of it is not available}+let LookupIndicator;+try {+ const mod = await import(+ "../../../plugins/lookup-studio/prompt-card/LookupIndicator"+ );+ LookupIndicator = mod.LookupIndicator;+} catch {}+let AgenticTableChecklist;try {const mod = await import(@@ -279,6 +287,9 @@ function PromptCardItems({</Typography.Link></Space></Button>+ {LookupIndicator && (+ <LookupIndicator promptDetails={promptDetails} />+ )}</Space><Space>{details?.enable_highlight &&diff --git a/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx b/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsxindex 38f7d924a..b8aaa4b39 100644--- a/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx+++ b/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx@@ -51,6 +51,49 @@ try {// The component will remain null of it is not available}+let LookupOutputTabs;+try {+ const mod = await import(+ "../../../plugins/lookup-studio/prompt-card/LookupOutputTabs"+ );+ LookupOutputTabs = mod.LookupOutputTabs;+} catch {+ // Cloud-only plugin — absent in OSS builds; LookupOutputTabs stays null.+}++let getEnrichedCopyText;+try {+ const mod = await import(+ "../../../plugins/lookup-studio/prompt-card/getEnrichedCopyText"+ );+ getEnrichedCopyText = mod.getEnrichedCopyText;+} catch {+ // Cloud-only plugin — absent in OSS builds; falls back to raw copy text.+}++// Fallback to raw text — plugin throw on malformed enrichment shouldn't break Copy.+const resolveCopyText = (promptOutputId, fallbackText) => {+ if (!getEnrichedCopyText) {+ return fallbackText;+ }+ try {+ const enriched = getEnrichedCopyText(promptOutputId);+ return typeof enriched === "string" && enriched ? enriched : fallbackText;+ } catch (err) {+ // eslint-disable-next-line no-console+ console.warn("[PromptOutput] getEnrichedCopyText threw:", err);+ return fallbackText;+ }+};++// Cloud wraps children in LookupOutputTabs; OSS passes through.+const renderWithLookupWrapper = (lookupProps, children) =>+ LookupOutputTabs ? (+ <LookupOutputTabs {...lookupProps}>{children}</LookupOutputTabs>+ ) : (+ children+ );+function PromptOutput({promptDetails,handleRun,@@ -194,32 +237,43 @@ function PromptOutput({"highlighted-prompt-cell"}`}>- <DisplayPromptResult- output={promptOutput}- highlightData={- promptOutputData?.highlightData?.[promptDetails.prompt_key]- }- handleSelectHighlight={handleSelectHighlight}- confidenceData={- promptOutputData?.confidenceData?.[promptDetails.prompt_key]- }- wordConfidenceData={- promptOutputData?.wordConfidenceData?.[promptDetails.prompt_key]- }- progressMsg={progressMsg}- />+ {renderWithLookupWrapper(+ {+ promptId,+ profileManagerId: defaultLlmProfile,+ defaultLlmProfile,+ promptOutputId: promptOutputData?.promptOutputId,+ },+ <DisplayPromptResult+ output={promptOutput}+ highlightData={+ promptOutputData?.highlightData?.[promptDetails.prompt_key]+ }+ handleSelectHighlight={handleSelectHighlight}+ confidenceData={+ promptOutputData?.confidenceData?.[promptDetails.prompt_key]+ }+ wordConfidenceData={+ promptOutputData?.wordConfidenceData?.[promptDetails.prompt_key]+ }+ progressMsg={progressMsg}+ />,+ )}<div className="prompt-profile-run"><CopyPromptOutputBtnisDisabled={isTableExtraction}- copyToClipboard={() =>+ copyToClipboard={() => {copyOutputToClipboard(- displayPromptResult(- promptOutput,- true,- promptDetails?.enable_highlight,+ resolveCopyText(+ promptOutputData?.promptOutputId,+ displayPromptResult(+ promptOutput,+ true,+ promptDetails?.enable_highlight,+ ),),- )- }+ );+ }}/><PromptOutputExpandBtnpromptId={promptDetails?.prompt_id}@@ -438,31 +492,42 @@ function PromptOutput({handleTable(profileId, promptOutputData)) : (<>- <DisplayPromptResult- output={promptOutputData?.output}- profileId={profileId}- docId={selectedDoc?.document_id}- promptRunStatus={promptRunStatus}- handleSelectHighlight={handleSelectHighlight}- highlightData={promptOutputData?.highlightData}- confidenceData={promptOutputData?.confidenceData}- wordConfidenceData={- promptOutputData?.wordConfidenceData- }- promptDetails={promptDetails}- progressMsg={progressMsg}- />+ {renderWithLookupWrapper(+ {+ promptId,+ profileManagerId: profileId,+ defaultLlmProfile,+ promptOutputId: promptOutputData?.promptOutputId,+ },+ <DisplayPromptResult+ output={promptOutputData?.output}+ profileId={profileId}+ docId={selectedDoc?.document_id}+ promptRunStatus={promptRunStatus}+ handleSelectHighlight={handleSelectHighlight}+ highlightData={promptOutputData?.highlightData}+ confidenceData={promptOutputData?.confidenceData}+ wordConfidenceData={+ promptOutputData?.wordConfidenceData+ }+ promptDetails={promptDetails}+ progressMsg={progressMsg}+ />,+ )}<div className="prompt-profile-run"><CopyPromptOutputBtnisDisabled={isTableExtraction}- copyToClipboard={() =>+ copyToClipboard={() => {copyOutputToClipboard(- displayPromptResult(- promptOutputData?.output,- true,+ resolveCopyText(+ promptOutputData?.promptOutputId,+ displayPromptResult(+ promptOutputData?.output,+ true,+ ),),- )- }+ );+ }}/></div></>diff --git a/frontend/src/components/custom-tools/token-usage/TokenUsage.jsx b/frontend/src/components/custom-tools/token-usage/TokenUsage.jsxindex 846c11f0a..3c52a4005 100644--- a/frontend/src/components/custom-tools/token-usage/TokenUsage.jsx+++ b/frontend/src/components/custom-tools/token-usage/TokenUsage.jsx@@ -23,8 +23,13 @@ function TokenUsage({ tokenUsageId, isLoading }) {setTokens(tokenUsage[tokenUsageId]); // Update tokens state with the token usage data for the given tokenUsageId}, [tokenUsage, tokenUsageId]);- // If no tokens data is available, render nothing- if (!tokens || !Object.keys(tokens)?.length || isLoading) {+ // BE returns all-null fields when no Usage rows match; treat as empty.+ if (+ !tokens ||+ !Object.keys(tokens)?.length ||+ isLoading ||+ tokens?.total_tokens == null+ ) {return "NA";}diff --git a/frontend/src/components/custom-tools/tool-ide/ToolIde.jsx b/frontend/src/components/custom-tools/tool-ide/ToolIde.jsxindex 93c819859..1f7e21c25 100644--- a/frontend/src/components/custom-tools/tool-ide/ToolIde.jsx+++ b/frontend/src/components/custom-tools/tool-ide/ToolIde.jsx@@ -44,6 +44,28 @@ try {} catch {// Do nothing if plugins are not loaded.}++// Cloud-only — OSS stub.+let useLookupDirtySeed = () => {};+try {+ const mod = await import(+ "../../../plugins/lookup-studio/hooks/useLookupDirtySeed.js"+ );+ useLookupDirtySeed = mod.useLookupDirtySeed;+} catch {}++// Cloud-only — OSS stub resolves true to skip the gate.+let useLookupExportGate = () => ({+ checkLookups: () => Promise.resolve(true),+ modalEl: null,+});+try {+ const mod = await import(+ "../../../plugins/lookup-studio/hooks/useLookupExportGate"+ );+ useLookupExportGate = mod.useLookupExportGate;+} catch {}+function ToolIde() {const [openSettings, setOpenSettings] = useState(false);const customToolStore = useCustomToolStore();@@ -76,6 +98,7 @@ function ToolIde() {const isCheckingUsageRef = useRef(false);const hasCheckedForCurrentSessionRef = useRef(false);const abortControllerRef = useRef(null);+ const { checkLookups, modalEl: lookupGateModalEl } = useLookupExportGate();useEffect(() => {if (openShareModal) {@@ -178,6 +201,9 @@ function ToolIde() {}}, [details?.tool_id]);+ // Surfaces re-export banner when /lookups page edits made the tool stale.+ useLookupDirtySeed(details?.tool_id);+// Cleanup abort controller on unmountuseEffect(() => {return () => {@@ -189,6 +215,10 @@ function ToolIde() {// Handle export from reminder barconst handleExportFromReminder = useCallback(async () => {+ const ok = await checkLookups(details?.tool_id, "export");+ if (!ok) {+ return;+ }setIsExporting(true);try {const requestOptions = {@@ -237,6 +267,7 @@ function ToolIde() {handleException,markChangesAsExported,setPostHogCustomEvent,+ checkLookups,]);const generateIndex = async (doc) => {@@ -340,11 +371,13 @@ function ToolIde() {isExporting={isExporting}/>)}+ {lookupGateModalEl}<HeaderhandleUpdateTool={handleUpdateTool}setOpenSettings={setOpenSettings}setOpenShareModal={setOpenShareModal}setOpenCloneModal={setOpenCloneModal}+ checkLookups={checkLookups}/><divclassName={isPublicSource ? "public-tool-ide-body" : "tool-ide-body"}diff --git a/frontend/src/components/helpers/custom-markdown/CustomMarkdown.jsx b/frontend/src/components/helpers/custom-markdown/CustomMarkdown.jsxindex 458f31955..177299a0f 100644--- a/frontend/src/components/helpers/custom-markdown/CustomMarkdown.jsx+++ b/frontend/src/components/helpers/custom-markdown/CustomMarkdown.jsx@@ -55,8 +55,7 @@ const CustomMarkdown = ({</Text>);case "link": {- // Protocol-relative URLs (`//evil.com/...`) also start with `/`- // so exclude them from the internal-route branch.+ // Exclude protocol-relative `//evil.com` from internal-route branch.const isInternal = url?.startsWith("/") && !url.startsWith("//");if (isInternal) {const resolvedUrl = orgName ? `/${orgName}${url}` : url;diff --git a/frontend/src/components/helpers/custom-tools/CustomToolsHelper.js b/frontend/src/components/helpers/custom-tools/CustomToolsHelper.jsindex 543ad9b14..35c2854cb 100644--- a/frontend/src/components/helpers/custom-tools/CustomToolsHelper.js+++ b/frontend/src/components/helpers/custom-tools/CustomToolsHelper.js@@ -19,6 +19,18 @@ try {} catch {// Do nothing, Not-found Page will be triggered.}++let fetchLookupAssignments;+let fetchLookupOutputs;+let resetLookupAssignments;+try {+ const mod = await import(+ "../../../plugins/lookup-studio/store/useFetchLookupAssignments"+ );+ fetchLookupAssignments = mod.fetchLookupAssignments;+ fetchLookupOutputs = mod.fetchLookupOutputs;+ resetLookupAssignments = mod.resetLookupAssignments;+} catch {}function CustomToolsHelper() {const [isLoading, setIsLoading] = useState(true);const { id } = useParams();@@ -115,6 +127,14 @@ function CustomToolsHelper() {.then((res) => {const data = res?.data;updatedCusTool["adapters"] = data;++ if (fetchLookupAssignments) {+ const toolId = updatedCusTool["details"]?.tool_id;+ fetchLookupAssignments(axiosPrivate, sessionDetails?.orgId, toolId);+ if (fetchLookupOutputs) {+ fetchLookupOutputs(axiosPrivate, sessionDetails?.orgId, toolId);+ }+ }}).catch((err) => {setAlertDetails(handleException(err, "Failed to load the custom tool"));@@ -131,6 +151,9 @@ function CustomToolsHelper() {setDefaultCustomTool();emptyCusToolMessages();resetTokenUsage();+ if (resetLookupAssignments) {+ resetLookupAssignments();+ }};}, []);diff --git a/frontend/src/components/logging/execution-logs/ExecutionLogs.jsx b/frontend/src/components/logging/execution-logs/ExecutionLogs.jsxindex a9c72a0ef..8e44287e2 100644--- a/frontend/src/components/logging/execution-logs/ExecutionLogs.jsx+++ b/frontend/src/components/logging/execution-logs/ExecutionLogs.jsx@@ -54,14 +54,14 @@ function ExecutionLogs() {? location.state?.from || `/${sessionDetails?.orgName}/logs`: null;- // State to pass back for scroll restoration+ // Scroll-restoration wins; otherwise preserve caller's upstream UI state.const backRouteState =id && location.state?.scrollToCardId? {scrollToCardId: location.state.scrollToCardId,cardExpanded: location.state.cardExpanded,}- : null;+ : location.state?.backRouteState || null;const items = [{diff --git a/frontend/src/components/navigations/side-nav-bar/SideNavBar.jsx b/frontend/src/components/navigations/side-nav-bar/SideNavBar.jsxindex bb328b1a9..164fdab14 100644--- a/frontend/src/components/navigations/side-nav-bar/SideNavBar.jsx+++ b/frontend/src/components/navigations/side-nav-bar/SideNavBar.jsx@@ -89,6 +89,12 @@ try {// Plugin unavailable}+let lookupStudioEnabled = false;+try {+ await import("../../../plugins/lookup-studio");+ lookupStudioEnabled = true;+} catch {}+let manualReviewSettingsEnabled = false;try {await import("../../../plugins/manual-review/settings/Settings.jsx");@@ -506,6 +512,19 @@ const SideNavBar = ({ collapsed, setCollapsed }) => {});}+ // Keep Prompt Studio highlighted on /lookups. Replace, don't mutate —+ // `data` may alias the `menu` prop.+ if (lookupStudioEnabled && isUnstract && data[0]?.subMenu) {+ const onLookupPath = globalThis.location.pathname.startsWith(+ `/${orgName}/lookups`,+ );+ if (onLookupPath) {+ data[0].subMenu = data[0].subMenu.map((el) =>+ el.id === 1.1 ? { ...el, active: true } : el,+ );+ }+ }+// Add HITL Review section if plugin is available and user has HITL roleconst isHITLRole = ["unstract_reviewer",diff --git a/frontend/src/components/navigations/tool-nav-bar/ToolNavBar.jsx b/frontend/src/components/navigations/tool-nav-bar/ToolNavBar.jsxindex 999630d59..e39c50ff1 100644--- a/frontend/src/components/navigations/tool-nav-bar/ToolNavBar.jsx+++ b/frontend/src/components/navigations/tool-nav-bar/ToolNavBar.jsx@@ -9,6 +9,7 @@ import "./ToolNavBar.css";function ToolNavBar({title,+ titleAdornment,subtitle,onEditTitle,enableSearch,@@ -19,6 +20,7 @@ function ToolNavBar({onNavigateBack,segmentFilter,segmentOptions,+ segmentValue,onSearch,searchKey,}) {@@ -53,6 +55,7 @@ function ToolNavBar({<Typography.Text strong className="tool-nav-bar__title">{title}</Typography.Text>+ {titleAdornment}{onEditTitle && (<Buttontype="text"@@ -79,6 +82,7 @@ function ToolNavBar({{segmentFilter && segmentOptions && (<Segmentedoptions={segmentOptions}+ value={segmentValue}onChange={segmentFilter}className="tool-nav-bar__segment"/>@@ -102,6 +106,7 @@ function ToolNavBar({ToolNavBar.propTypes = {title: PropTypes.string,+ titleAdornment: PropTypes.node,subtitle: PropTypes.string,onEditTitle: PropTypes.func,enableSearch: PropTypes.bool,@@ -111,6 +116,7 @@ ToolNavBar.propTypes = {previousRouteState: PropTypes.object,onNavigateBack: PropTypes.func,segmentOptions: PropTypes.array,+ segmentValue: PropTypes.string,segmentFilter: PropTypes.func,onSearch: PropTypes.func,searchKey: PropTypes.string,diff --git a/frontend/src/components/widgets/empty-state/EmptyState.jsx b/frontend/src/components/widgets/empty-state/EmptyState.jsxindex d221c2544..0c9c8537d 100644--- a/frontend/src/components/widgets/empty-state/EmptyState.jsx+++ b/frontend/src/components/widgets/empty-state/EmptyState.jsx@@ -27,7 +27,7 @@ function EmptyState({ text, btnText, handleClick }) {}EmptyState.propTypes = {- text: PropTypes.string.isRequired,+ text: PropTypes.node.isRequired,btnText: PropTypes.string,handleClick: PropTypes.func,};diff --git a/frontend/src/helpers/urlSafety.js b/frontend/src/helpers/urlSafety.jsindex b5ca23e79..fede217e2 100644--- a/frontend/src/helpers/urlSafety.js+++ b/frontend/src/helpers/urlSafety.js@@ -1,13 +1,12 @@const SAFE_URL_SCHEMES = ["http:", "https:", "mailto:", "tel:"];-// Guards against unsafe schemes (e.g. `javascript:`, `data:`) when-// rendering links built from user- or tool-derived content.+// Block unsafe schemes (e.g. `javascript:`, `data:`) in user/tool-derived links.const isSafeExternalUrl = (url) => {if (typeof url !== "string" || url === "") {return false;}- // Parse without a base so bare strings (e.g. "javascript", "../foo") fail- // instead of silently resolving to `https://<origin>/...` and passing.+ // No base URL — bare strings ("javascript", "../foo") must fail rather than+ // silently resolve to the current origin.try {const parsed = new URL(url);return SAFE_URL_SCHEMES.includes(parsed.protocol);diff --git a/frontend/src/hooks/usePromptOutput.js b/frontend/src/hooks/usePromptOutput.jsindex 530cdf2bc..c27a0dbf5 100644--- a/frontend/src/hooks/usePromptOutput.js+++ b/frontend/src/hooks/usePromptOutput.js@@ -23,6 +23,28 @@ try {// The component will remain null of it is not available}+let handleLookupOutput;+try {+ const mod = await import(+ "../plugins/lookup-studio/prompt-card/handleLookupOutput"+ );+ handleLookupOutput = mod.handleLookupOutput;+} catch (error) {+ // Surface chunk-load failures — silent catch hid them.+ // eslint-disable-next-line no-console+ console.warn("[usePromptOutput] handleLookupOutput unavailable:", error);+}++// Cloud-only extractor; OSS no-op. Signature matches plugin helper.+let getEnrichmentFromItem = (_item) => null;+try {+ const mod = await import("../plugins/lookup-enriched-toggle/helpers");+ getEnrichmentFromItem = mod.getEnrichmentFromItem;+} catch (error) {+ // eslint-disable-next-line no-console+ console.warn("[usePromptOutput] getEnrichmentFromItem unavailable:", error);+}+const usePromptOutput = () => {const { sessionDetails } = useSessionStore();const { setTokenUsage, updateTokenUsage } = useTokenUsageStore();@@ -125,6 +147,18 @@ const usePromptOutput = () => {wordConfidenceData: item?.word_confidence_data,};+ // Per-item plugin failure must not abort the forEach — would leave+ // partial state with no error surfaced.+ try {+ const enrichment = getEnrichmentFromItem(item);+ if (handleLookupOutput && enrichment) {+ handleLookupOutput(item.prompt_output_id, enrichment);+ }+ } catch (err) {+ // eslint-disable-next-line no-console+ console.warn("[usePromptOutput] lookup enrichment failed:", err);+ }+if (item?.is_single_pass_extract && isTokenUsageForSinglePassAdded) {return;}diff --git a/frontend/src/hooks/usePromptRun.js b/frontend/src/hooks/usePromptRun.jsindex 34fb42e7d..eb97502d0 100644--- a/frontend/src/hooks/usePromptRun.js+++ b/frontend/src/hooks/usePromptRun.js@@ -16,10 +16,7 @@ import usePromptOutput from "./usePromptOutput";// Tracks the latest run nonce per (promptId, statusKey) so stale timeouts// from a previous run don't falsely cancel a newer run of the same combo.const runNonceMap = new Map();-// Safety-net timeout for the fire-and-forget prompt run flow. Must trail the-// server-side LLM adapter timeout (default 900s in-// unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/openai.json) so the UI-// never gives up before the backend does. 16 minutes leaves a 1-minute buffer.+// Must exceed server LLM adapter timeout (default 900s) so UI doesn't bail first.const SOCKET_TIMEOUT_MS = 16 * 60 * 1000; // 16 minutesconst usePromptRun = () => {diff --git a/frontend/src/pages/CustomTools.jsx b/frontend/src/pages/CustomTools.jsxindex bd177301c..0864d9d26 100644--- a/frontend/src/pages/CustomTools.jsx+++ b/frontend/src/pages/CustomTools.jsx@@ -1,7 +1,44 @@+import { useEffect, useState } from "react";+import { useLocation } from "react-router-dom";+import { ListOfTools } from "../components/custom-tools/list-of-tools/ListOfTools";+const TAB_OPTIONS = ["Projects", "Look-Ups"];+function CustomTools() {- return <ListOfTools />;+ const location = useLocation();+ const [LookupListComp, setLookupListComp] = useState(null);+ const [activeTab, setActiveTab] = useState(+ location.state?.activeTab || "Projects",+ );++ useEffect(() => {+ import("../plugins/lookup-studio")+ .then((mod) => setLookupListComp(() => mod.LookupList))+ .catch(() => {});+ }, []);++ useEffect(() => {+ if (location.state?.activeTab) {+ setActiveTab(location.state.activeTab);+ }+ }, [location.state?.activeTab]);++ if (!LookupListComp) {+ return <ListOfTools />;+ }++ const tabProps = {+ segmentOptions: TAB_OPTIONS,+ segmentValue: activeTab,+ onSegmentChange: setActiveTab,+ };++ return activeTab === "Projects" ? (+ <ListOfTools {...tabProps} />+ ) : (+ <LookupListComp {...tabProps} />+ );}export { CustomTools };diff --git a/frontend/src/routes/useMainAppRoutes.js b/frontend/src/routes/useMainAppRoutes.jsindex 1d5849b10..768dfee97 100644--- a/frontend/src/routes/useMainAppRoutes.js+++ b/frontend/src/routes/useMainAppRoutes.js@@ -61,6 +61,14 @@ try {// Do nothing, Not-found Page will be triggered.}+let LookupStudio;+try {+ const mod = await import("../plugins/lookup-studio");+ LookupStudio = mod.LookupStudio;+} catch {+ // Do nothing, Not-found Page will be triggered.+}+try {const mod1 = await import("../plugins/app-deployment/AppDeployments.jsx");AppDeployments = mod1.AppDeployments;@@ -180,6 +188,7 @@ function useMainAppRoutes() {element={<AgenticPromptStudio />}/>)}+ {LookupStudio && <Route path="lookups/*" element={<LookupStudio />} />}<Route path="logs" element={<LogsPage />} /><Route path="logs/:type/:id/" element={<LogsPage />} /><Routediff --git a/unstract/sdk1/src/unstract/sdk1/execution/context.py b/unstract/sdk1/src/unstract/sdk1/execution/context.pyindex 83eb650a2..0edb3e40b 100644--- a/unstract/sdk1/src/unstract/sdk1/execution/context.py+++ b/unstract/sdk1/src/unstract/sdk1/execution/context.py@@ -96,8 +96,7 @@ class ExecutionContext:raise ValueError("run_id is required")if not self.execution_source:raise ValueError("execution_source is required")- # When execution_id is set, organization_id must be too — they- # travel together for workflow-level log correlation.+ # Workflow-level log correlation needs both fields together.if self.execution_id and not self.organization_id:raise ValueError("organization_id is required when execution_id is set")diff --git a/unstract/sdk1/src/unstract/sdk1/execution/orchestrator.py b/unstract/sdk1/src/unstract/sdk1/execution/orchestrator.pyindex 2c0f66f3b..0f1454a32 100644--- a/unstract/sdk1/src/unstract/sdk1/execution/orchestrator.py+++ b/unstract/sdk1/src/unstract/sdk1/execution/orchestrator.py@@ -9,6 +9,7 @@ that unhandled exceptions are always wrapped in a failedimport loggingimport time+from typing import Anyfrom unstract.sdk1.execution.context import ExecutionContextfrom unstract.sdk1.execution.registry import ExecutorRegistry@@ -37,8 +38,7 @@ class ExecutionOrchestrator:exceptions (wrapped as a failure result)."""logger.info(- "Orchestrating execution: executor=%s operation=%s "- "run_id=%s request_id=%s",+ "Orchestrating execution: executor=%s operation=%s run_id=%s request_id=%s",context.executor_name,context.operation,context.run_id,@@ -61,9 +61,14 @@ class ExecutionOrchestrator:context.executor_name,elapsed,)+ metadata: dict[str, Any] = {"elapsed_seconds": round(elapsed, 3)}+ # Don't lose pre-crash billing rows on retry.+ collected = getattr(executor, "_usage_records", None) or []+ if collected:+ metadata["usage_records"] = list(collected)return ExecutionResult.failure(error=f"{type(exc).__name__}: {exc}",- metadata={"elapsed_seconds": round(elapsed, 3)},+ metadata=metadata,)elapsed = time.monotonic() - startdiff --git a/unstract/sdk1/src/unstract/sdk1/llm.py b/unstract/sdk1/src/unstract/sdk1/llm.pyindex 4588e6946..14bc4b33c 100644--- a/unstract/sdk1/src/unstract/sdk1/llm.py+++ b/unstract/sdk1/src/unstract/sdk1/llm.py@@ -9,11 +9,10 @@ from typing import Any, NoReturn, castimport litellm# from litellm import get_supported_openai_params-from litellm import get_max_tokens, token_counter+from litellm import get_max_tokensfrom pydantic import ValidationErrorfrom unstract.sdk1.adapters.constants import Commonfrom unstract.sdk1.adapters.llm1 import adapters-from unstract.sdk1.audit import Auditfrom unstract.sdk1.constants import Common as SdkCommonfrom unstract.sdk1.constants import ToolEnvfrom unstract.sdk1.exceptions import LLMError, SdkError, strip_litellm_prefix@@ -21,7 +20,6 @@ from unstract.sdk1.platform import PlatformHelperfrom unstract.sdk1.tool.base import BaseToolfrom unstract.sdk1.utils.common import (LLMResponseCompat,- TokenCounterCompat,capture_metrics,)from unstract.sdk1.utils.retry_utils import (@@ -211,6 +209,7 @@ class LLM:if capture_metrics_from_platform is not None:self._capture_metrics = capture_metrics_from_platformself._metrics: dict[str, object] = {}+ self._pending_usage: list[dict] = []def _get_adapter_info(self) -> str:"""Build a display string identifying this adapter for errors."""@@ -642,9 +641,39 @@ class LLM:def get_metrics(self) -> dict[str, object]:return self._metrics+ def get_last_usage(self) -> Mapping[str, int]:+ """Token usage from the most recent LLM call (sync, async, or streaming)."""+ if not self._pending_usage:+ return {}+ last = self._pending_usage[-1]+ return {+ "prompt_tokens": last["prompt_tokens"],+ "completion_tokens": last["completion_tokens"],+ "total_tokens": last["total_tokens"],+ }++ def get_last_usage_record(self) -> dict | None:+ """Full usage record for the most recent LLM call.++ Returns tokens + cost + model + reason metadata; ``None`` if no+ call has been made yet.+ """+ if not self._pending_usage:+ return None+ return self._pending_usage[-1]+def get_usage_reason(self) -> object:return self.platform_kwargs.get("llm_usage_reason")+ def flush_pending_usage(self) -> list[dict]:+ """Return and clear all pending usage records.++ Called at executor finalization.+ """+ records = self._pending_usage+ self._pending_usage = []+ return records+def _record_usage(self,model: str,@@ -652,23 +681,74 @@ class LLM:usage: Mapping[str, int] | None,llm_api: str,) -> None:- prompt_tokens = token_counter(model=model, messages=messages)usage_data: Mapping[str, int] = usage or {}- all_tokens = TokenCounterCompat(- prompt_tokens=usage_data.get("prompt_tokens", 0),- completion_tokens=usage_data.get("completion_tokens", 0),- total_tokens=usage_data.get("total_tokens", 0),- )+ prompt_tokens = usage_data.get("prompt_tokens", 0)+ completion_tokens = usage_data.get("completion_tokens", 0)+ total_tokens = usage_data.get("total_tokens", 0)- logger.info(f"[sdk1][LLM][{model}][{llm_api}] Prompt Tokens: {prompt_tokens}")- logger.info(f"[sdk1][LLM][{model}][{llm_api}] LLM Usage: {all_tokens}")+ # Fall back to litellm when providers omit prompt tokens — avoids 0-token billing.+ if prompt_tokens == 0 and messages:+ try:+ prompt_tokens = litellm.token_counter(model=model, messages=messages)+ if total_tokens == 0:+ total_tokens = prompt_tokens + completion_tokens+ except Exception:+ logger.warning(+ "[sdk1][LLM][%s] prompt_tokens missing on response and "+ "litellm.token_counter() fallback failed; recording 0",+ model,+ exc_info=True,+ )- Audit().push_usage_data(- platform_api_key=self._platform_api_key,- token_counter=all_tokens,- event_type="llm",- model_name=model,- kwargs={"provider": self.adapter.get_provider(), **self.platform_kwargs},+ logger.info(+ "[sdk1][LLM][%s][%s] Usage: prompt=%d completion=%d total=%d",+ model,+ llm_api,+ prompt_tokens,+ completion_tokens,+ total_tokens,+ )++ try:+ prompt_cost, compl_cost = litellm.cost_per_token(+ model=model,+ prompt_tokens=prompt_tokens,+ completion_tokens=completion_tokens,+ )+ cost = prompt_cost + compl_cost+ except Exception:+ logger.warning(+ "Failed to compute cost for model=%s; recording as 0.0",+ model,+ exc_info=True,+ )+ cost = 0.0++ # Trailing segment matches legacy Audit semantics (e.g. bedrock/anthropic/claude).+ display_model = model.rsplit("/", 1)[-1] if model else model++ # Spread _usage_kwargs first so computed billing fields below win.+ self._pending_usage.append(+ {+ **self._usage_kwargs,+ "usage_type": "llm",+ "model_name": display_model,+ "provider": self.adapter.get_provider(),+ "adapter_instance_id": self.platform_kwargs.get(+ "adapter_instance_id", ""+ ),+ # run_id lands in a UUIDField — "" fails the cast; keep None.+ "run_id": self.platform_kwargs.get("run_id") or None,+ "execution_id": self.platform_kwargs.get("execution_id", ""),+ # "" isn't a valid choice for llm_usage_reason.+ "llm_usage_reason": self.platform_kwargs.get("llm_usage_reason") or None,+ "prompt_tokens": prompt_tokens,+ "completion_tokens": completion_tokens,+ "total_tokens": total_tokens,+ "embedding_tokens": 0,+ "cost_in_dollars": cost,+ "status": "SUCCESS",+ })# Finish reasons indicating a safety/policy refusal across providers:@@ -1053,10 +1133,18 @@ class LLMCompat:"""Get captured metrics."""return self._llm_instance.get_metrics()+ def get_last_usage(self) -> Mapping[str, int]:+ """Token usage from the most recent complete() call."""+ return self._llm_instance.get_last_usage()+def get_usage_reason(self) -> object:"""Get usage reason from platform kwargs."""return self._llm_instance.get_usage_reason()+ def flush_pending_usage(self) -> list[dict]:+ """Return and clear all pending usage records."""+ return self._llm_instance.flush_pending_usage()+def test_connection(self) -> bool:"""Test connection to the LLM provider."""return self._llm_instance.test_connection()diff --git a/unstract/sdk1/src/unstract/sdk1/usage_handler.py b/unstract/sdk1/src/unstract/sdk1/usage_handler.pyindex 0ffa7c43e..0d1abf8ae 100644--- a/unstract/sdk1/src/unstract/sdk1/usage_handler.py+++ b/unstract/sdk1/src/unstract/sdk1/usage_handler.py@@ -1,12 +1,15 @@+import loggingfrom typing import Any+import litellmfrom llama_index.core.callbacks import CBEventType, TokenCountingHandlerfrom llama_index.core.callbacks.base_handler import BaseCallbackHandlerfrom llama_index.core.embeddings import BaseEmbedding-from unstract.sdk1.audit import Auditfrom unstract.sdk1.constants import LogLevelfrom unstract.sdk1.tool.stream import StreamMixin+logger = logging.getLogger(__name__)+class UsageHandler(StreamMixin, BaseCallbackHandler):"""Handler for usage events in LLM or Embedding models.@@ -57,6 +60,7 @@ class UsageHandler(StreamMixin, BaseCallbackHandler):self._verbose = verboseself.token_counter = token_counterself.embed_model = embed_model+ self._pending_usage: list[dict] = []self.platform_api_key = platform_api_keysuper().__init__(log_level=log_level, # StreamMixin's args@@ -101,17 +105,60 @@ class UsageHandler(StreamMixin, BaseCallbackHandler):and event_type not in self.event_ends_to_ignoreand payload is not None):+ if self.embed_model is None:+ return+ if self.token_counter is None:+ logger.warning(+ "Embedding usage callback invoked without token_counter; "+ "skipping usage record."+ )+ returnmodel_name = self.embed_model.model_name- # Need to push the data to via platform service+ embedding_tokens = self.token_counter.total_embedding_token_countself.stream_log(- log=f"Pushing embedding usage for model {model_name}",+ log=f"Recording embedding usage for model {model_name}",level=LogLevel.DEBUG,)- Audit(log_level=self.log_level).push_usage_data(- platform_api_key=self.platform_api_key,- token_counter=self.token_counter,- event_type=event_type,- model_name=self.embed_model.model_name,- kwargs=self.kwargs,++ try:+ prompt_cost, _ = litellm.cost_per_token(+ model=model_name,+ prompt_tokens=embedding_tokens,+ completion_tokens=0,+ )+ cost = prompt_cost+ except Exception:+ logger.warning(+ "Failed to compute embedding cost for model=%s; recording 0.0",+ model_name,+ exc_info=True,+ )+ cost = 0.0++ # Collapse multi-segment IDs (``bedrock/anthropic/claude``) to+ # the trailing segment to match legacy Audit semantics.+ display_model = model_name.rsplit("/", 1)[-1] if model_name else model_name++ self._pending_usage.append(+ {+ "usage_type": "embedding",+ "model_name": display_model,+ "adapter_instance_id": self.kwargs.get("adapter_instance_id", ""),+ # run_id lands in a UUIDField — "" fails the cast; keep None.+ "run_id": self.kwargs.get("run_id") or None,+ "execution_id": self.kwargs.get("execution_id", ""),+ "embedding_tokens": embedding_tokens,+ "prompt_tokens": 0,+ "completion_tokens": 0,+ "total_tokens": 0,+ "cost_in_dollars": cost,+ "status": "SUCCESS",+ })self.token_counter.reset_counts()++ def flush_pending_usage(self) -> list[dict]:+ """Return and clear all pending usage records."""+ records = self._pending_usage+ self._pending_usage = []+ return recordsdiff --git a/unstract/sdk1/src/unstract/sdk1/utils/common.py b/unstract/sdk1/src/unstract/sdk1/utils/common.pyindex 86b0f173b..68a3fe068 100644--- a/unstract/sdk1/src/unstract/sdk1/utils/common.py+++ b/unstract/sdk1/src/unstract/sdk1/utils/common.py@@ -223,6 +223,10 @@ def capture_metrics(func: object) -> object:if self._run_id and self._capture_metrics:metrics_mixin = MetricsMixin(run_id=self._run_id)+ # Snapshot index so timing stamps every row appended during this call —+ # internal retries would otherwise leave NULL execution_time_ms rows.+ pending_at_entry = len(getattr(self, "_pending_usage", []))+try:result = func(self, *args, **kwargs)finally:@@ -250,6 +254,14 @@ def capture_metrics(func: object) -> object:# If the key isn't in self._metrics, set it to new_metricsself._metrics = new_metrics+ # Stamp timing on every record appended during this call.+ pending = getattr(self, "_pending_usage", [])+ time_taken = new_metrics.get(time_taken_key)+ if time_taken is not None and len(pending) > pending_at_entry:+ elapsed_ms = int(time_taken * 1000)+ for record in pending[pending_at_entry:]:+ record["execution_time_ms"] = elapsed_ms+return resultreturn wrapperdiff --git a/workers/executor/executors/answer_prompt.py b/workers/executor/executors/answer_prompt.pyindex c22c8aaaf..d1eef5b3b 100644--- a/workers/executor/executors/answer_prompt.py+++ b/workers/executor/executors/answer_prompt.py@@ -338,27 +338,4 @@ class AnswerPromptService:structured_output[prompt_key] = {}return- highlight_data = None- if enable_highlight and metadata and PSKeys.HIGHLIGHT_DATA in metadata:- highlight_data = metadata[PSKeys.HIGHLIGHT_DATA].get(prompt_key)-- processed_data = parsed_data- updated_highlight_data = None-- webhook_enabled = output.get(PSKeys.ENABLE_POSTPROCESSING_WEBHOOK, False)- if webhook_enabled:- webhook_url = output.get(PSKeys.POSTPROCESSING_WEBHOOK_URL)- processed_data, updated_highlight_data = (- AnswerPromptService._run_webhook_postprocess(- parsed_data=parsed_data,- webhook_url=webhook_url,- highlight_data=highlight_data,- )- )-- structured_output[prompt_key] = processed_data-- if enable_highlight and metadata and updated_highlight_data is not None:- metadata.setdefault(PSKeys.HIGHLIGHT_DATA, {})[prompt_key] = (- updated_highlight_data- )+ structured_output[prompt_key] = parsed_datadiff --git a/workers/executor/executors/exceptions.py b/workers/executor/executors/exceptions.pyindex 69cd0a8a1..3db835882 100644--- a/workers/executor/executors/exceptions.py+++ b/workers/executor/executors/exceptions.py@@ -9,18 +9,23 @@ work outside of Flask (i.e. inside the Celery executor worker).class LegacyExecutorError(Exception):"""Base exception for legacy executor errors.- Replaces Flask's ``APIError`` — carries ``message`` and ``code``- attributes so callers can map to ``ExecutionResult.failure()``.+ ``partial_usage_records`` preserves billing rows across mid-pipeline failures."""code: int = 500message: str = "Internal executor error"- def __init__(self, message: str | None = None, code: int | None = None):+ def __init__(+ self,+ message: str | None = None,+ code: int | None = None,+ partial_usage_records: list[dict] | None = None,+ ):if message is not None:self.message = messageif code is not None:self.code = code+ self.partial_usage_records: list[dict] = list(partial_usage_records or [])super().__init__(self.message)diff --git a/workers/executor/executors/legacy_executor.py b/workers/executor/executors/legacy_executor.pyindex 3760aea06..0ddb29e26 100644--- a/workers/executor/executors/legacy_executor.py+++ b/workers/executor/executors/legacy_executor.py@@ -21,6 +21,10 @@ from executor.executors.dto import ()from executor.executors.exceptions import ExtractionError, LegacyExecutorErrorfrom executor.executors.file_utils import FileUtils+from executor.executors.lookup_enrichment import (+ run_lookup_enrichment,+ run_webhook_postprocessing,+)from unstract.sdk1.adapters.exceptions import AdapterErrorfrom unstract.sdk1.adapters.x2text.constants import X2TextConstants@@ -58,7 +62,6 @@ class LegacyExecutor(BaseExecutor):}def __init__(self) -> None:- # Per-request state — overwritten on every ``execute()`` call.self._log_events_id: str = ""self._log_component: dict[str, str] = {}self._execution_id: str | None = None@@ -81,8 +84,10 @@ class LegacyExecutor(BaseExecutor):NotImplementedError: From stub handlers (until 2D–2H)."""# Extract log streaming info (set by tasks.py for IDE sessions).- self._log_events_id = context.log_events_id or ""- self._log_component = getattr(context, "_log_component", None) or {}+ self._log_events_id: str = context.log_events_id or ""+ self._log_component: dict[str, str] = (+ getattr(context, "_log_component", None) or {}+ )self._execution_id = context.execution_idself._file_execution_id = context.file_execution_idself._organization_id = context.organization_id@@ -144,8 +149,17 @@ class LegacyExecutor(BaseExecutor):level=LogLevel.ERROR,)except Exception:- pass # Best-effort — don't mask the original error- return ExecutionResult.failure(error=exc.message)+ # Don't mask the original error; log the secondary at DEBUG.+ logger.debug(+ "Failed to stream error to FE for run_id=%s",+ context.run_id,+ exc_info=True,+ )+ # Preserve partial usage rows from a mid-pipeline failure.+ failure_metadata: dict[str, Any] = {}+ if exc.partial_usage_records:+ failure_metadata["usage_records"] = list(exc.partial_usage_records)+ return ExecutionResult.failure(error=exc.message, metadata=failure_metadata)def _build_shim(self,@@ -346,12 +360,10 @@ class LegacyExecutor(BaseExecutor):def _run_summarize_step(self, summarize_params: dict, context: ExecutionContext- ) -> ExecutionResult | None:+ ) -> ExecutionResult:"""Run summarization if not already cached.- Returns:- ``None`` on success (summary written or cached), or an- ``ExecutionResult`` failure to propagate to the caller.+ Cache hit returns success with empty metadata (no LLM call)."""extract_file_path = summarize_params.get("extract_file_path", "")summarize_file_path = summarize_params.get("summarize_file_path", "")@@ -366,7 +378,7 @@ class LegacyExecutor(BaseExecutor):if fs.exists(summarize_file_path):existing = fs.read(path=summarize_file_path, mode="r")if existing:- return None+ return ExecutionResult(success=True, data={})doc_context = fs.read(path=extract_file_path, mode="r")if not doc_context:@@ -403,7 +415,9 @@ class LegacyExecutor(BaseExecutor):mode="w",data=summarize_result.data.get("data", ""),)- return None+ return ExecutionResult(+ success=True, data={}, metadata=dict(summarize_result.metadata or {})+ )# ------------------------------------------------------------------# Phase 5C — Compound IDE index handler (extract + index)@@ -440,60 +454,80 @@ class LegacyExecutor(BaseExecutor):error=f"ide_index missing required params: {', '.join(missing)}")- # Step 1: Extract (or reuse pre-extracted text on marker hit)- pre_extracted_text = index_params.get(IKeys.EXTRACTED_TEXT, "") or ""- if pre_extracted_text:- logger.info(- "ide_index: marker hit, skipping extract step " "(len=%d, run_id=%s)",- len(pre_extracted_text),- context.run_id,- )- extracted_text = pre_extracted_text- else:- extract_ctx = ExecutionContext(+ ide_records: list[dict[str, Any]] = []++ def _absorb(child: ExecutionResult) -> None:+ child_records = (child.metadata or {}).get("usage_records") or []+ if child_records:+ ide_records.extend(child_records)++ def _failure(child: ExecutionResult) -> ExecutionResult:+ metadata = dict(child.metadata or {})+ existing = metadata.get("usage_records") or []+ metadata["usage_records"] = ide_records + list(existing)+ return ExecutionResult.failure(error=child.error, metadata=metadata)++ try:+ # Step 1: Extract (or reuse pre-extracted text on marker hit)+ pre_extracted_text = index_params.get(IKeys.EXTRACTED_TEXT, "") or ""+ if pre_extracted_text:+ logger.info(+ "ide_index: marker hit, skipping extract step (len=%d, run_id=%s)",+ len(pre_extracted_text),+ context.run_id,+ )+ extracted_text = pre_extracted_text+ else:+ extract_ctx = ExecutionContext(+ executor_name=context.executor_name,+ operation=Operation.EXTRACT.value,+ run_id=context.run_id,+ execution_source=context.execution_source,+ organization_id=context.organization_id,+ executor_params=extract_params,+ request_id=context.request_id,+ log_events_id=context.log_events_id,+ execution_id=context.execution_id,+ file_execution_id=context.file_execution_id,+ )+ extract_result = self._handle_extract(extract_ctx)+ if not extract_result.success:+ return _failure(extract_result)+ _absorb(extract_result)+ extracted_text = extract_result.data.get(IKeys.EXTRACTED_TEXT, "")++ # Step 2: Optional summarize+ summarize_params = params.get("summarize_params")+ summarize_file_path = ""+ if summarize_params:+ summarize_file_path = summarize_params.get("summarize_file_path", "")+ summarize_result = self._run_summarize_step(summarize_params, context)+ if not summarize_result.success:+ return _failure(summarize_result)+ _absorb(summarize_result)++ # Step 3: Index — inject extracted text+ index_params[IKeys.EXTRACTED_TEXT] = extracted_text++ index_ctx = ExecutionContext(executor_name=context.executor_name,- operation=Operation.EXTRACT.value,+ operation=Operation.INDEX.value,run_id=context.run_id,execution_source=context.execution_source,organization_id=context.organization_id,- executor_params=extract_params,+ executor_params=index_params,request_id=context.request_id,log_events_id=context.log_events_id,execution_id=context.execution_id,file_execution_id=context.file_execution_id,)- extract_result = self._handle_extract(extract_ctx)- if not extract_result.success:- return extract_result- extracted_text = extract_result.data.get(IKeys.EXTRACTED_TEXT, "")-- # Step 2: Optional summarize- summarize_params = params.get("summarize_params")- summarize_file_path = ""- if summarize_params:- summarize_file_path = summarize_params.get("summarize_file_path", "")- result = self._run_summarize_step(summarize_params, context)- if result is not None:- return result-- # Step 3: Index — inject extracted text- index_params[IKeys.EXTRACTED_TEXT] = extracted_text-- index_ctx = ExecutionContext(- executor_name=context.executor_name,- operation=Operation.INDEX.value,- run_id=context.run_id,- execution_source=context.execution_source,- organization_id=context.organization_id,- executor_params=index_params,- request_id=context.request_id,- log_events_id=context.log_events_id,- execution_id=context.execution_id,- file_execution_id=context.file_execution_id,- )- index_result = self._handle_index(index_ctx)- if not index_result.success:- return index_result+ index_result = self._handle_index(index_ctx)+ if not index_result.success:+ return _failure(index_result)+ _absorb(index_result)+ except LegacyExecutorError as e:+ e.partial_usage_records = ide_records + e.partial_usage_records+ raisereturn ExecutionResult(success=True,@@ -501,6 +535,7 @@ class LegacyExecutor(BaseExecutor):IKeys.DOC_ID: index_result.data.get(IKeys.DOC_ID, ""),"summarize_file_path": summarize_file_path,},+ metadata={"usage_records": ide_records},)# ------------------------------------------------------------------@@ -554,81 +589,100 @@ class LegacyExecutor(BaseExecutor):extracted_text = ""index_metrics: dict = {}+ pipeline_records: list[dict[str, Any]] = []++ def _absorb(child_result: ExecutionResult) -> None:+ child_records = (child_result.metadata or {}).get("usage_records") or []+ if child_records:+ pipeline_records.extend(child_records)++ def _failure(child_result: ExecutionResult) -> ExecutionResult:+ metadata = dict(child_result.metadata or {})+ existing = metadata.get("usage_records") or []+ metadata["usage_records"] = pipeline_records + list(existing)+ return ExecutionResult.failure(error=child_result.error, metadata=metadata)shim = self._build_shim(platform_api_key=extract_params.get("platform_api_key", ""),)step = 1- # ---- Step 1: Extract ----- if not skip_extraction:- shim.stream_log(f"Pipeline step {step}: Extracting text from document...")- step += 1- extract_ctx = ExecutionContext(- executor_name=context.executor_name,- operation=Operation.EXTRACT.value,- run_id=context.run_id,- execution_source=context.execution_source,- organization_id=context.organization_id,- executor_params=extract_params,- request_id=context.request_id,- log_events_id=context.log_events_id,- execution_id=context.execution_id,- file_execution_id=context.file_execution_id,- )- extract_result = self._handle_extract(extract_ctx)- if not extract_result.success:- return extract_result- extracted_text = extract_result.data.get(IKeys.EXTRACTED_TEXT, "")-- # ---- Step 2: Summarize (if enabled) ----- if is_summarization:- shim.stream_log(f"Pipeline step {step}: Summarizing extracted text...")- step += 1- summarize_result = self._run_pipeline_summarize(- context=context,- summarize_params=summarize_params or {},- answer_params=answer_params,- )- if not summarize_result.success:- return summarize_result- # answer_params file_path/hash updated in-place by helper- elif skip_extraction:- # Smart table: use original source file- answer_params["file_path"] = input_file_path- elif not is_single_pass:- # ---- Step 3: Index per output with dedup ----- shim.stream_log(- f"Pipeline step {step}: Indexing document into vector store..."- )- step += 1- index_metrics = self._run_pipeline_index(- context=context,- index_template=index_template,- answer_params=answer_params,- extracted_text=extracted_text,- usage_kwargs=extract_params.get("usage_kwargs", {}),- )+ try:+ # ---- Step 1: Extract ----+ if not skip_extraction:+ shim.stream_log(f"Pipeline step {step}: Extracting text from document...")+ step += 1+ extract_ctx = ExecutionContext(+ executor_name=context.executor_name,+ operation=Operation.EXTRACT.value,+ run_id=context.run_id,+ execution_source=context.execution_source,+ organization_id=context.organization_id,+ executor_params=extract_params,+ request_id=context.request_id,+ log_events_id=context.log_events_id,+ execution_id=context.execution_id,+ file_execution_id=context.file_execution_id,+ )+ extract_result = self._handle_extract(extract_ctx)+ if not extract_result.success:+ return _failure(extract_result)+ _absorb(extract_result)+ extracted_text = extract_result.data.get(IKeys.EXTRACTED_TEXT, "")++ # ---- Step 2: Summarize (if enabled) ----+ if is_summarization:+ shim.stream_log(f"Pipeline step {step}: Summarizing extracted text...")+ step += 1+ summarize_result = self._run_pipeline_summarize(+ context=context,+ summarize_params=summarize_params or {},+ answer_params=answer_params,+ )+ if not summarize_result.success:+ return _failure(summarize_result)+ _absorb(summarize_result)+ # answer_params file_path/hash updated in-place by helper+ elif skip_extraction:+ # Smart table: use original source file+ answer_params["file_path"] = input_file_path+ elif not is_single_pass:+ # ---- Step 3: Index per output with dedup ----+ shim.stream_log(+ f"Pipeline step {step}: Indexing document into vector store..."+ )+ step += 1+ index_metrics = self._run_pipeline_index(+ context=context,+ index_template=index_template,+ answer_params=answer_params,+ extracted_text=extracted_text,+ usage_kwargs=extract_params.get("usage_kwargs", {}),+ )- # ---- Step 4: Table settings injection ----- if not is_single_pass:- self._inject_table_settings(+ # ---- Step 4: Table settings injection ----+ if not is_single_pass:+ self._inject_table_settings(+ answer_params=answer_params,+ index_template=index_template,+ skip_extraction=skip_extraction,+ input_file_path=input_file_path,+ )++ # ---- Step 5: Answer prompt / Single pass ----+ answer_result = self._run_pipeline_answer_step(+ context=context,answer_params=answer_params,- index_template=index_template,- skip_extraction=skip_extraction,- input_file_path=input_file_path,+ is_single_pass=is_single_pass,+ shim=shim,+ step=step,)-- # ---- Step 5: Answer prompt / Single pass ----- answer_result = self._run_pipeline_answer_step(- context=context,- answer_params=answer_params,- is_single_pass=is_single_pass,- shim=shim,- step=step,- )- if not answer_result.success:- return answer_result+ if not answer_result.success:+ return _failure(answer_result)+ _absorb(answer_result)+ except LegacyExecutorError as e:+ e.partial_usage_records = pipeline_records + e.partial_usage_records+ raise# ---- Step 6: Merge results ----structured_output = answer_result.data@@ -640,7 +694,17 @@ class LegacyExecutor(BaseExecutor):)shim.stream_log("Pipeline completed successfully")- return ExecutionResult(success=True, data=structured_output)+ out_metadata = {+ k: v+ for k, v in (answer_result.metadata or {}).items()+ if k != "usage_records"+ }+ out_metadata["usage_records"] = pipeline_records+ return ExecutionResult(+ success=True,+ data=structured_output,+ metadata=out_metadata,+ )def _run_pipeline_answer_step(self,@@ -803,13 +867,16 @@ class LegacyExecutor(BaseExecutor):mode="w",data=summarized_context,)+ forward_metadata = dict(summarize_result.metadata or {})+ else:+ forward_metadata = {}# Update answer_paramssummarize_file_hash = fs.get_hash_from_file(path=summarize_file_path)answer_params["file_hash"] = summarize_file_hashanswer_params["file_path"] = str(summarize_file_path)- return ExecutionResult(success=True, data={})+ return ExecutionResult(success=True, data={}, metadata=forward_metadata)def _run_pipeline_index(self,@@ -1078,8 +1145,7 @@ class LegacyExecutor(BaseExecutor):"Document already indexed in vector store; skipping re-index.")logger.info(- "Skipping re-index: doc_id=%s already in vector DB and "- "reindex=False",+ "Skipping re-index: doc_id=%s already in vector DB and reindex=False",doc_id,)return ExecutionResult(success=True, data={IKeys.DOC_ID: doc_id})@@ -1304,19 +1370,26 @@ class LegacyExecutor(BaseExecutor):embedding_compat_cls,vector_db_cls,)- for output in prompts:- self._execute_single_prompt(- output=output,- context=context,- structured_output=structured_output,- metadata=metadata,- metrics=metrics,- variable_names=variable_names,- context_retrieval_metrics=context_retrieval_metrics,- deps=_deps,- tool_settings=tool_settings,- process_text_fn=process_text_fn,- )+ usage_records: list[dict[str, Any]] = []+ try:+ for output in prompts:+ usage_records.extend(+ self._execute_single_prompt(+ output=output,+ context=context,+ structured_output=structured_output,+ metadata=metadata,+ metrics=metrics,+ variable_names=variable_names,+ context_retrieval_metrics=context_retrieval_metrics,+ deps=_deps,+ tool_settings=tool_settings,+ process_text_fn=process_text_fn,+ )+ )+ except LegacyExecutorError as e:+ e.partial_usage_records = usage_records + e.partial_usage_records+ raisepipeline_shim.stream_log(f"All {len(prompts)} prompts processed successfully")logger.info(@@ -1336,6 +1409,7 @@ class LegacyExecutor(BaseExecutor):PSKeys.METADATA: metadata,PSKeys.METRICS: metrics,},+ metadata={"usage_records": usage_records},)@staticmethod@@ -1392,19 +1466,19 @@ class LegacyExecutor(BaseExecutor):metadata: dict[str, Any],shim: Any,prompt_name: str,- ) -> None:- """Run challenge verification plugin if enabled and available."""+ ) -> list[dict[str, Any]]:+ """Run the challenge plugin if enabled. Returns its usage rows."""from executor.executors.constants import PromptServiceConstants as PSKeysfrom executor.executors.plugins import ExecutorPluginLoaderif not tool_settings.get(PSKeys.ENABLE_CHALLENGE):- return+ return []challenge_cls = ExecutorPluginLoader.get("challenge")if not challenge_cls:- return+ return []challenge_llm_id = tool_settings.get(PSKeys.CHALLENGE_LLM)if not challenge_llm_id:- return+ return []shim.stream_log(f"Running challenge for: `{prompt_name}`")challenge_llm = llm_cls(adapter_instance_id=challenge_llm_id,@@ -1423,9 +1497,18 @@ class LegacyExecutor(BaseExecutor):platform_key=platform_api_key,metadata=metadata,)- challenger.run()+ try:+ challenger.run()+ except LegacyExecutorError as e:+ # Flush before bubbling so partial rows survive.+ e.partial_usage_records = (+ list(challenge_llm.flush_pending_usage()) + e.partial_usage_records+ )+ raise+ records = list(challenge_llm.flush_pending_usage())shim.stream_log(f"Challenge verification completed for: `{prompt_name}`")logger.info("Challenge completed: prompt=%s", prompt_name)+ return records@staticmethoddef _run_evaluation_if_enabled(@@ -1471,8 +1554,8 @@ class LegacyExecutor(BaseExecutor):deps: tuple,tool_settings: dict[str, Any],process_text_fn: Any,- ) -> None:- """Execute one prompt: variable replacement, retrieval, LLM, post-process."""+ ) -> list[dict[str, Any]]:+ """Run one prompt end-to-end; return its usage rows."""from executor.executors.constants import PromptServiceConstants as PSKeysfrom executor.executors.constants import RetrievalStrategy@@ -1563,7 +1646,7 @@ class LegacyExecutor(BaseExecutor):prompt_name=prompt_name,shim=shim,)- return+ return []if output.get(PSKeys.TYPE) == PSKeys.LINE_ITEM:self._run_line_item_extraction(@@ -1585,39 +1668,22 @@ class LegacyExecutor(BaseExecutor):},shim=shim,)- return+ return []usage_kwargs = {"run_id": run_id, "execution_id": execution_id}- try:- llm = llm_cls(- adapter_instance_id=output[PSKeys.LLM],- tool=shim,- usage_kwargs={**usage_kwargs, PSKeys.LLM_USAGE_REASON: PSKeys.EXTRACTION},- capture_metrics=True,- )- vector_db = None- if chunk_size > 0:- embedding = embedding_compat_cls(- adapter_instance_id=output[PSKeys.EMBEDDING],- tool=shim,- kwargs={**usage_kwargs},- )- vector_db = vector_db_cls(- tool=shim,- adapter_instance_id=output[PSKeys.VECTOR_DB],- embedding=embedding,- )- shim.stream_log(- f"Initialized LLM and retrieval adapters for: `{prompt_name}`"- )- except Exception as e:- msg = f"Couldn't fetch adapter. {e}"- logger.error(msg)- raise LegacyExecutorError(- message=msg, code=getattr(e, "status_code", None) or 500- ) from e+ llm, embedding, vector_db = self._init_llm_and_retrieval(+ output=output,+ shim=shim,+ chunk_size=chunk_size,+ llm_cls=llm_cls,+ embedding_compat_cls=embedding_compat_cls,+ vector_db_cls=vector_db_cls,+ usage_kwargs=usage_kwargs,+ prompt_name=prompt_name,+ )context_list: list[str] = []+ records: list[dict[str, Any]] = []try:answer = "NA"retrieval_strategy = output.get(PSKeys.RETRIEVAL_STRATEGY)@@ -1689,19 +1755,39 @@ class LegacyExecutor(BaseExecutor):)shim.stream_log(f"Applied type conversion for: `{prompt_name}`")- self._run_challenge_if_enabled(- tool_settings=tool_settings,+ records.extend(+ run_lookup_enrichment(+ output=output,+ structured_output=structured_output,+ metadata=metadata,+ metrics=metrics,+ shim=shim,+ usage_kwargs=usage_kwargs,+ llm_cls=llm_cls,+ )+ )+ run_webhook_postprocessing(output=output,structured_output=structured_output,- context_list=context_list,- llm=llm,- llm_cls=llm_cls,- usage_kwargs=usage_kwargs,- run_id=run_id,- platform_api_key=platform_api_key,metadata=metadata,shim=shim,- prompt_name=prompt_name,+ )++ records.extend(+ self._run_challenge_if_enabled(+ tool_settings=tool_settings,+ output=output,+ structured_output=structured_output,+ context_list=context_list,+ llm=llm,+ llm_cls=llm_cls,+ usage_kwargs=usage_kwargs,+ run_id=run_id,+ platform_api_key=platform_api_key,+ metadata=metadata,+ shim=shim,+ prompt_name=prompt_name,+ ))self._run_evaluation_if_enabled(output=output,@@ -1716,15 +1802,111 @@ class LegacyExecutor(BaseExecutor):val = structured_output.get(prompt_name)if isinstance(val, str):structured_output[prompt_name] = val.rstrip("\n")- finally:- metrics.setdefault(prompt_name, {}).update(- {- "context_retrieval": context_retrieval_metrics.get(prompt_name, {}),- f"{llm.get_usage_reason()}_llm": llm.get_metrics(),- }+ except LegacyExecutorError as e:+ # Flush before bubbling so partial rows survive.+ flushed = self._flush_per_prompt_metrics(+ metrics=metrics,+ context_retrieval_metrics=context_retrieval_metrics,+ prompt_name=prompt_name,+ llm=llm,+ embedding=embedding,+ vector_db=vector_db,+ chunk_size=chunk_size,)- if vector_db:- vector_db.close()+ e.partial_usage_records = records + flushed + e.partial_usage_records+ raise+ records.extend(+ self._flush_per_prompt_metrics(+ metrics=metrics,+ context_retrieval_metrics=context_retrieval_metrics,+ prompt_name=prompt_name,+ llm=llm,+ embedding=embedding,+ vector_db=vector_db,+ chunk_size=chunk_size,+ )+ )+ return records++ def _init_llm_and_retrieval(+ self,+ output: dict[str, Any],+ shim: Any,+ chunk_size: int,+ llm_cls: Any,+ embedding_compat_cls: Any,+ vector_db_cls: Any,+ usage_kwargs: dict[str, Any],+ prompt_name: str,+ ) -> tuple[Any, Any, Any]:+ from executor.executors.constants import PromptServiceConstants as PSKeys++ try:+ llm = llm_cls(+ adapter_instance_id=output[PSKeys.LLM],+ tool=shim,+ usage_kwargs={**usage_kwargs, PSKeys.LLM_USAGE_REASON: PSKeys.EXTRACTION},+ capture_metrics=True,+ )+ embedding = None+ vector_db = None+ if chunk_size > 0:+ embedding = embedding_compat_cls(+ adapter_instance_id=output[PSKeys.EMBEDDING],+ tool=shim,+ kwargs={**usage_kwargs},+ )+ vector_db = vector_db_cls(+ tool=shim,+ adapter_instance_id=output[PSKeys.VECTOR_DB],+ embedding=embedding,+ )+ shim.stream_log(+ f"Initialized LLM and retrieval adapters for: `{prompt_name}`"+ )+ return llm, embedding, vector_db+ except Exception as e:+ msg = f"Couldn't fetch adapter. {e}"+ logger.error(msg)+ raise LegacyExecutorError(+ message=msg, code=getattr(e, "status_code", None) or 500+ ) from e++ def _flush_per_prompt_metrics(+ self,+ metrics: dict[str, Any],+ context_retrieval_metrics: dict[str, Any],+ prompt_name: str,+ llm: Any,+ embedding: Any,+ vector_db: Any,+ chunk_size: int,+ ) -> list[dict[str, Any]]:+ """Flush LLM + embedding usage rows and return them."""+ metrics.setdefault(prompt_name, {}).update(+ {+ "context_retrieval": context_retrieval_metrics.get(prompt_name, {}),+ f"{llm.get_usage_reason()}_llm": llm.get_metrics(),+ }+ )+ records: list[dict[str, Any]] = list(llm.flush_pending_usage())+ # Public adapters skip the callback_manager, so there's nothing to flush.+ if chunk_size > 0 and embedding is not None and embedding.callback_manager:+ for handler in embedding.callback_manager.handlers:+ if not hasattr(handler, "flush_pending_usage"):+ continue+ # Per-handler guard so one bad handler doesn't drop the rest.+ try:+ records.extend(handler.flush_pending_usage())+ except Exception:+ logger.warning(+ "Failed to flush usage from embedding handler %s",+ type(handler).__name__,+ exc_info=True,+ )+ if vector_db:+ vector_db.close()+ return recordsdef _run_table_extraction(self,@@ -2083,14 +2265,17 @@ class LegacyExecutor(BaseExecutor):)shim = self._build_shim(platform_api_key=platform_api_key)+ # execution_id is required for correct usage-row classification.usage_kwargs = {"run_id": context.run_id,+ "execution_id": context.execution_id or "",PSKeys.LLM_USAGE_REASON: PSKeys.SUMMARIZE,}_, _, _, _, llm_cls, _, _ = self._get_prompt_deps()shim.stream_log("Initializing LLM for summarization...")+ llm: Any = Nonetry:llm = llm_cls(adapter_instance_id=llm_adapter_id,@@ -2103,16 +2288,26 @@ class LegacyExecutor(BaseExecutor):shim.stream_log("Running document summarization...")summary = answer_prompt_svc.run_completion(llm=llm, prompt=prompt)+ records = list(llm.flush_pending_usage())logger.info("Summarization completed: run_id=%s", context.run_id)shim.stream_log("Summarization completed")return ExecutionResult(success=True,data={"data": summary},+ metadata={"usage_records": records},)except Exception as e:+ # Flush before re-raising so partial rows survive.+ partial: list[dict] = []+ if llm is not None:+ try:+ partial = list(llm.flush_pending_usage())+ except Exception:+ logger.debug("flush_pending_usage failed during error path")logger.error("Summarization failed: error=%s", str(e))status_code = getattr(e, "status_code", None) or 500raise LegacyExecutorError(message=f"Error during summarization: {e}",code=status_code,+ partial_usage_records=partial,) from ediff --git a/workers/executor/executors/lookup_enrichment.py b/workers/executor/executors/lookup_enrichment.pynew file mode 100644index 000000000..0b224badb--- /dev/null+++ b/workers/executor/executors/lookup_enrichment.py@@ -0,0 +1,145 @@+"""Lookup enrichment + webhook postprocessing helpers.++Extracted from ``LegacyExecutor`` so the orchestrator stays focused on+dispatch. Both helpers are free functions — callers pass shim/state in.+"""++from __future__ import annotations++import logging+from typing import Any++from executor.executors.constants import PromptServiceConstants as PSKeys+from executor.executors.plugins import ExecutorPluginLoader++from unstract.sdk1.constants import LogLevel++logger = logging.getLogger(__name__)+++def is_blank(value: Any) -> bool:+ """Treat None / whitespace strings / empty containers as no-value.++ Boolean False / numeric 0 are NOT blank — valid inputs for+ boolean/number prompts whose lookups should still run.+ """+ if value is None:+ return True+ if isinstance(value, str):+ return not value.strip()+ if isinstance(value, (list, dict)):+ return not value+ return False+++def run_lookup_enrichment(+ output: dict[str, Any],+ structured_output: dict[str, Any],+ metadata: dict[str, Any],+ metrics: dict[str, Any],+ shim: Any,+ llm_cls: Any,+ usage_kwargs: dict[str, Any] | None = None,+) -> list[dict[str, Any]]:+ """Run lookup enrichment plugin if enabled and available.++ Returns any usage records the plugin emitted (recovered even on+ plugin failure) so the caller can extend its billing batch.+ """+ prompt_name = output[PSKeys.NAME]+ current_value = structured_output.get(prompt_name)++ lookup_config = output.get("lookup_config")+ lookup_cls = ExecutorPluginLoader.get("lookup-enrichment")+ if not (lookup_config and lookup_cls):+ return []+ if is_blank(current_value):+ # Surface a skip log instead of silently no-op-ing.+ lookup_name = lookup_config.get("lookup_name") or "lookup"+ shim.stream_log(+ f"Skipping lookup `{lookup_name}` for `{prompt_name}` — "+ f"source prompt produced no value."+ )+ return []++ outcome = None+ try:+ outcome = lookup_cls.run_with_metrics(+ llm_cls=llm_cls,+ lookup_config=lookup_config,+ structured_output=structured_output,+ current_value=current_value,+ metadata=metadata,+ prompt_name=prompt_name,+ shim=shim,+ usage_kwargs=usage_kwargs,+ )+ metrics.setdefault(prompt_name, {})[lookup_cls.METRICS_KEY] = outcome.llm_metrics+ except Exception:+ # Degrade gracefully on plugin contract drift.+ lookup_name = lookup_config.get("lookup_name") or "lookup"+ logger.exception(+ "Lookup enrichment failed for prompt=%s lookup=%s",+ prompt_name,+ lookup_name,+ )+ shim.stream_log(+ f"Lookup `{lookup_name}` failed for `{prompt_name}`; "+ f"continuing without enrichment.",+ level=LogLevel.WARN,+ )++ if outcome is not None and getattr(outcome, "usage_records", None):+ return list(outcome.usage_records)+ return []+++def run_webhook_postprocessing(+ output: dict[str, Any],+ structured_output: dict[str, Any],+ metadata: dict[str, Any],+ shim: Any,+) -> None:+ """Run webhook postprocessing if enabled (JSON outputs only)."""+ from executor.executors.answer_prompt import AnswerPromptService++ prompt_name = output[PSKeys.NAME]+ output_type = output.get(PSKeys.TYPE, "")+ webhook_enabled = output.get(PSKeys.ENABLE_POSTPROCESSING_WEBHOOK, False)+ if not webhook_enabled:+ return+ # Empty / non-JSON payloads are skipped — typically a parse failure.+ parsed_value = structured_output.get(prompt_name)+ if not isinstance(parsed_value, (dict, list)) or not parsed_value:+ logger.warning(+ "Webhook postprocessing skipped: prompt=%s parsed payload "+ "is empty or non-JSON (likely a parse failure)",+ prompt_name,+ )+ return+ if output_type != PSKeys.JSON:+ logger.warning(+ "Webhook postprocessing supports JSON outputs only; "+ "skipping for prompt=%s (output_type=%s)",+ prompt_name,+ output_type,+ )+ shim.stream_log(+ f"Webhook postprocessing supports JSON outputs only; "+ f"skipping for `{prompt_name}`.",+ level=LogLevel.WARN,+ )+ return++ webhook_url = output.get(PSKeys.POSTPROCESSING_WEBHOOK_URL)+ highlight_data = None+ if metadata and PSKeys.HIGHLIGHT_DATA in metadata:+ highlight_data = metadata.get(PSKeys.HIGHLIGHT_DATA, {}).get(prompt_name)+ processed, updated_highlights = AnswerPromptService._run_webhook_postprocess(+ parsed_data=structured_output.get(prompt_name),+ webhook_url=webhook_url,+ highlight_data=highlight_data,+ )+ structured_output[prompt_name] = processed+ if updated_highlights is not None and metadata:+ metadata.setdefault(PSKeys.HIGHLIGHT_DATA, {})[prompt_name] = updated_highlightsdiff --git a/workers/executor/executors/plugins/protocols.py b/workers/executor/executors/plugins/protocols.pyindex fb4d676b3..c59973d73 100644--- a/workers/executor/executors/plugins/protocols.py+++ b/workers/executor/executors/plugins/protocols.py@@ -49,3 +49,29 @@ class EvaluationProtocol(Protocol):"""Legacy executor: prompt evaluation."""def run(self, **kwargs: Any) -> dict: ...+++@runtime_checkable+class LookupEnrichmentProtocol(Protocol):+ """Legacy executor: post-extraction lookup enrichment.++ The executor calls ``run_with_metrics`` (not ``run``) because the+ plugin returns an outcome object exposing ``usage_records`` and+ ``llm_metrics`` for the calling executor to flush. ``METRICS_KEY``+ keys the lookup metrics into the per-prompt metrics dict.+ """++ METRICS_KEY: str++ def run_with_metrics(+ self,+ *,+ llm_cls: Any,+ lookup_config: dict,+ structured_output: dict,+ current_value: Any,+ metadata: dict,+ prompt_name: str,+ shim: Any,+ usage_kwargs: dict | None = None,+ ) -> Any: ...diff --git a/workers/executor/executors/usage.py b/workers/executor/executors/usage.pydeleted file mode 100644index ab6296eae..000000000--- a/workers/executor/executors/usage.py+++ /dev/null@@ -1,81 +0,0 @@-"""Usage tracking helper for the executor worker.--Ported from prompt-service/.../helpers/usage.py.-Flask/DB dependencies removed — usage data is pushed via the SDK1-``Audit`` class (HTTP to platform API) and returned directly in-``ExecutionResult.metadata`` instead of querying the DB.--Note: The SDK1 adapters (LLM, EmbeddingCompat) already call-``Audit().push_usage_data()`` internally. This helper is for-explicit push calls outside of adapter operations (e.g. rent rolls).-"""--import logging-from typing import Any--logger = logging.getLogger(__name__)---class UsageHelper:- @staticmethod- def push_usage_data(- event_type: str,- kwargs: dict[str, Any],- platform_api_key: str,- token_counter: Any = None,- model_name: str = "",- ) -> bool:- """Push usage data to the audit service.-- Wraps ``Audit().push_usage_data()`` with validation and- error handling.-- Args:- event_type: Type of usage event (e.g. "llm", "embedding").- kwargs: Context dict (run_id, execution_id, etc.).- platform_api_key: API key for platform service auth.- token_counter: Token counter with usage metrics.- model_name: Name of the model used.-- Returns:- True if successful, False otherwise.- """- if not kwargs or not isinstance(kwargs, dict):- logger.error("Invalid kwargs provided to push_usage_data")- return False-- if not platform_api_key or not isinstance(platform_api_key, str):- logger.error("Invalid platform_api_key provided to push_usage_data")- return False-- try:- from unstract.sdk1.audit import Audit-- logger.debug(- "Pushing usage data for event_type=%s model=%s",- event_type,- model_name,- )-- Audit().push_usage_data(- platform_api_key=platform_api_key,- token_counter=token_counter,- model_name=model_name,- event_type=event_type,- kwargs=kwargs,- )-- logger.info("Successfully pushed usage data for %s", model_name)- return True- except Exception:- logger.exception("Error pushing usage data")- return False-- @staticmethod- def format_float_positional(value: float, precision: int = 10) -> str:- """Format a float without scientific notation.-- Removes trailing zeros for clean display of cost values.- """- formatted: str = f"{value:.{precision}f}"- return formatted.rstrip("0").rstrip(".") if "." in formatted else formatteddiff --git a/workers/executor/tasks.py b/workers/executor/tasks.pyindex 77d5ecaeb..a30243bca 100644--- a/workers/executor/tasks.py+++ b/workers/executor/tasks.py@@ -6,7 +6,9 @@ ExecutionOrchestrator, and returns an ExecutionResult dict."""from celery import shared_task+from shared.clients import UsageAPIClientfrom shared.enums.task_enums import TaskName+from shared.infrastructure.config import WorkerConfigfrom shared.infrastructure.logging import WorkerLoggerfrom unstract.sdk1.execution.context import ExecutionContext@@ -15,6 +17,15 @@ from unstract.sdk1.execution.result import ExecutionResultlogger = WorkerLogger.get_logger(__name__)+_LLM_BEARING_OPS = frozenset(+ {+ "answer_prompt",+ "single_pass_extraction",+ "summarize",+ "structure_pipeline",+ }+)+@shared_task(bind=True,@@ -97,6 +108,40 @@ def execute_extraction(self, execution_context_dict: dict) -> dict:orchestrator = ExecutionOrchestrator()result = orchestrator.execute(context)+ usage_records = result.metadata.get("usage_records", [])+ if usage_records:+ try:+ config = WorkerConfig()+ with UsageAPIClient(config) as usage_client:+ # Org context is set on the client; no need to pass it per call.+ usage_client.set_organization_context(context.organization_id)+ ok = usage_client.bulk_create_usage(usage_records)+ if not ok:+ # ERROR severity so dropped billing rows are recoverable from logs.+ logger.error(+ "bulk_create_usage returned failure for %d records "+ "(run_id=%s organization_id=%s)",+ len(usage_records),+ context.run_id,+ context.organization_id,+ )+ except Exception:+ logger.error(+ "Failed to flush %d usage records for run_id=%s organization_id=%s",+ len(usage_records),+ context.run_id,+ context.organization_id,+ exc_info=True,+ )+ elif result.success and context.operation in _LLM_BEARING_OPS:+ logger.info(+ "No usage_records emitted for op=%s run_id=%s organization_id=%s "+ "(unexpected for an LLM-bearing operation)",+ context.operation,+ context.run_id,+ context.organization_id,+ )+logger.info("execute_extraction complete: celery_task_id=%s request_id=%s success=%s",self.request.id,diff --git a/workers/file_processing/structure_tool_task.py b/workers/file_processing/structure_tool_task.pyindex c0fdee1c0..502e48ff3 100644--- a/workers/file_processing/structure_tool_task.py+++ b/workers/file_processing/structure_tool_task.py@@ -459,6 +459,8 @@ def _execute_structure_tool_impl(params: dict) -> dict:execution_source="tool",organization_id=organization_id,request_id=file_execution_id,+ execution_id=execution_id,+ file_execution_id=file_execution_id,executor_params=agentic_params,)at_result = dispatcher.dispatch(at_ctx, timeout=EXECUTOR_TIMEOUT)@@ -488,6 +490,8 @@ def _execute_structure_tool_impl(params: dict) -> dict:execution_source="tool",organization_id=organization_id,request_id=file_execution_id,+ execution_id=execution_id,+ file_execution_id=file_execution_id,executor_params={"extract_params": extract_params,"index_template": index_template,diff --git a/workers/ide_callback/tasks.py b/workers/ide_callback/tasks.pyindex d0dbce256..c6494e036 100644--- a/workers/ide_callback/tasks.py+++ b/workers/ide_callback/tasks.py@@ -24,6 +24,8 @@ PROMPT_STUDIO_RESULT_EVENT = "prompt_studio_result"# WebSocket emission endpoint (relative to internal API base)_EMIT_WEBSOCKET_ENDPOINT = "emit-websocket/"+_UNKNOWN_EXECUTOR_ERROR = "Unknown executor error"+class _SafeEncoder(json.JSONEncoder):"""JSON encoder that converts uuid.UUID and datetime objects to strings."""@@ -164,7 +166,7 @@ def ide_index_complete(try:# Check executor-level failureif not result_dict.get("success", False):- error_msg = result_dict.get("error", "Unknown executor error")+ error_msg = result_dict.get("error", _UNKNOWN_EXECUTOR_ERROR)logger.error("ide_index executor reported failure: %s", error_msg)api.remove_document_indexing(org_id=org_id,@@ -370,7 +372,7 @@ def ide_prompt_complete(try:# Check executor-level failureif not result_dict.get("success", False):- error_msg = result_dict.get("error", "Unknown executor error")+ error_msg = result_dict.get("error", _UNKNOWN_EXECUTOR_ERROR)logger.error("ide_prompt executor reported failure: %s", error_msg)_emit_event(api,@@ -514,3 +516,154 @@ def ide_prompt_error()except Exception:logger.exception("ide_prompt_error callback failed")+++# ------------------------------------------------------------------+# Text Extraction Callbacks+#+# Today only ``source="lookup"`` is wired up; the cloud lookups plugin+# is the only registrant of the underlying extraction-complete /+# extraction-error endpoints (see workers/shared/clients/extraction_client.py).+# ------------------------------------------------------------------+++def _get_extraction_client():+ from shared.clients.extraction_client import ExtractionAPIClient++ return ExtractionAPIClient()++[email protected](name="extraction_complete")+def extraction_complete(+ result_dict: dict[str, Any],+ callback_kwargs: dict[str, Any] | None = None,+) -> dict[str, Any]:+ """Celery link callback after successful text extraction.++ Persists the result via internal API and emits a WebSocket event.+ """+ cb = callback_kwargs or {}+ source = cb.get("source", "")+ file_id = cb.get("file_id", "")+ org_id = cb.get("org_id", "")+ extracted_text_path = cb.get("extracted_text_path", "")+ ws_room = cb.get("ws_room", "")+ ws_event = cb.get("ws_event", "")++ with _get_extraction_client() as api, _get_api_client() as ps_api:+ try:+ # Check executor-level failure+ if not result_dict.get("success", False):+ error_msg = result_dict.get("error", _UNKNOWN_EXECUTOR_ERROR)+ logger.error(+ "extraction executor reported failure: source=%s file=%s error=%s",+ source,+ file_id,+ error_msg,+ )+ api.mark_extraction_error(+ source=source,+ file_id=file_id,+ error=error_msg,+ organization_id=org_id,+ )+ if ws_room and ws_event:+ _emit_websocket(+ ps_api,+ room=ws_room,+ event=ws_event,+ data={+ "file_id": file_id,+ "status": "ERROR",+ "error": error_msg[:500],+ },+ )+ return {"status": "failed", "error": error_msg}++ api.mark_extraction_complete(+ source=source,+ file_id=file_id,+ extracted_text_path=extracted_text_path,+ organization_id=org_id,+ )++ if ws_room and ws_event:+ _emit_websocket(+ ps_api,+ room=ws_room,+ event=ws_event,+ data={"file_id": file_id, "status": "COMPLETED"},+ )++ logger.info("Extraction completed: source=%s file=%s", source, file_id)+ return {"status": "completed", "file_id": file_id}++ except Exception as e:+ logger.exception(+ "extraction_complete callback failed: source=%s file=%s",+ source,+ file_id,+ )+ if ws_room and ws_event:+ try:+ _emit_websocket(+ ps_api,+ room=ws_room,+ event=ws_event,+ data={+ "file_id": file_id,+ "status": "ERROR",+ "error": str(e)[:500],+ },+ )+ except Exception:+ # _emit_websocket swallows internally; log if anything escapes.+ logger.debug(+ "Failed to emit ws ERROR event in extraction_complete fallback",+ exc_info=True,+ )+ raise++[email protected](name="extraction_error")+def extraction_error(+ failed_task_id: str,+ callback_kwargs: dict[str, Any] | None = None,+) -> None:+ """Celery link_error callback when an extraction task fails."""+ cb = callback_kwargs or {}+ source = cb.get("source", "")+ file_id = cb.get("file_id", "")+ org_id = cb.get("org_id", "")+ ws_room = cb.get("ws_room", "")+ ws_event = cb.get("ws_event", "")++ # Context-manage clients to avoid per-task session leaks.+ with _get_extraction_client() as api, _get_api_client() as ps_api:+ try:+ error_msg = _get_task_error(failed_task_id, default="Text extraction failed")++ api.mark_extraction_error(+ source=source,+ file_id=file_id,+ error=error_msg,+ organization_id=org_id,+ )++ if ws_room and ws_event:+ _emit_websocket(+ ps_api,+ room=ws_room,+ event=ws_event,+ data={+ "file_id": file_id,+ "status": "ERROR",+ "error": error_msg[:500],+ },+ )+ except Exception:+ logger.exception(+ "extraction_error callback failed: source=%s file=%s",+ source,+ file_id,+ )diff --git a/workers/shared/clients/extraction_client.py b/workers/shared/clients/extraction_client.pynew file mode 100644index 000000000..72400f62e--- /dev/null+++ b/workers/shared/clients/extraction_client.py@@ -0,0 +1,73 @@+"""Extraction API Client for text extraction callbacks.++Used by the ide_callback worker to persist extraction results through+the backend's internal API endpoints.++NOTE on scope: the callback endpoints ``v1/extraction/extraction-{complete,error}/``+are currently registered **only** by the cloud ``lookups`` plugin+(see ``pluggable_apps/lookups/internal_urls.py``). The interface is+``source``-dispatched and designed to serve other extraction flows+(prompt-studio docs, connectors) in the future, but in OSS-only builds+the endpoints are absent. Callers from OSS paths should expect a 404+response and treat it as a no-op; the worker's error handling covers+this via a 404-terminal pattern.+"""++import logging+from typing import Any++from .base_client import BaseAPIClient++logger = logging.getLogger(__name__)++_EXTRACTION_COMPLETE_ENDPOINT = "v1/extraction/extraction-complete/"+_EXTRACTION_ERROR_ENDPOINT = "v1/extraction/extraction-error/"+++class ExtractionAPIClient(BaseAPIClient):+ """API client for the extraction-callback endpoints registered by+ cloud-side plugins (today: lookups). See module docstring for the+ OSS-absence contract.+ """++ def mark_extraction_complete(+ self,+ source: str,+ file_id: str,+ extracted_text_path: str,+ organization_id: str | None = None,+ **extra: Any,+ ) -> dict[str, Any]:+ """Notify backend that extraction succeeded."""+ payload: dict[str, Any] = {+ "source": source,+ "file_id": file_id,+ "extracted_text_path": extracted_text_path,+ **extra,+ }+ return self.post(+ _EXTRACTION_COMPLETE_ENDPOINT,+ data=payload,+ organization_id=organization_id,+ )++ def mark_extraction_error(+ self,+ source: str,+ file_id: str,+ error: str,+ organization_id: str | None = None,+ **extra: Any,+ ) -> dict[str, Any]:+ """Notify backend that extraction failed."""+ payload: dict[str, Any] = {+ "source": source,+ "file_id": file_id,+ "error": error,+ **extra,+ }+ return self.post(+ _EXTRACTION_ERROR_ENDPOINT,+ data=payload,+ organization_id=organization_id,+ )diff --git a/workers/shared/clients/usage_client.py b/workers/shared/clients/usage_client.pyindex a7ba8c701..0bbaba9f6 100644--- a/workers/shared/clients/usage_client.py+++ b/workers/shared/clients/usage_client.py@@ -71,7 +71,7 @@ class BaseUsageResponse(APIResponse, Generic[T]):def is_success(self) -> bool:"""Check if the response indicates success."""- return self.success_response and self.status == ResponseStatus.SUCCESS+ return self.success and self.status == ResponseStatus.SUCCESS@dataclass@@ -119,6 +119,8 @@ class UsageAPIClient(BaseAPIClient, UsageOperationMixin):Returns:UsageResponse containing aggregated usage data"""+ # Pre-bind so the except handler can log even if validation raises.+ validated_file_execution_id = str(file_execution_id)try:validated_file_execution_id = self._validate_file_execution_id(file_execution_id@@ -171,6 +173,41 @@ class UsageAPIClient(BaseAPIClient, UsageOperationMixin):message="Failed to retrieve usage data",)+ def bulk_create_usage(+ self, records: list[dict], organization_id: str | None = None+ ) -> bool:+ """Bulk create usage records at execution finalization.++ Args:+ records: List of usage record dicts to create.+ organization_id: Optional organization ID override.++ Returns:+ True if records were created successfully.+ """+ if not records:+ return True+ try:+ response = self.post(+ "v1/usage/batch/",+ data={"records": records},+ organization_id=organization_id,+ )+ # The success path returns ``{"created": N}`` (no ``success``+ # key); the error path returns ``{"success": False, "error":+ # ...}``. Accept either an explicit success flag or the+ # presence of ``created`` *as long as* ``success`` isn't+ # explicitly False — guards against future contracts that may+ # report a partial body.+ if response.get("success") is False:+ return False+ return response.get("success") is True or "created" in response+ except Exception:+ logger.error(+ "Failed to bulk create %d usage records", len(records), exc_info=True+ )+ return False+def get_aggregated_pages_processed(self, file_execution_id: str | uuid.UUID, organization_id: str | None = None) -> int | None:diff --git a/workers/tests/test_usage.py b/workers/tests/test_usage.pydeleted file mode 100644index fc08ac825..000000000--- a/workers/tests/test_usage.py+++ /dev/null@@ -1,312 +0,0 @@-"""Phase 2G — Usage tracking tests.--Verifies:-1. UsageHelper.push_usage_data wraps Audit correctly-2. Invalid kwargs returns False-3. Invalid platform_api_key returns False-4. Audit exceptions are caught and return False-5. format_float_positional formats correctly-6. SDK1 adapters already push usage (integration check)-7. answer_prompt handler returns metrics in ExecutionResult-"""--from unittest.mock import MagicMock, patch---from executor.executors.usage import UsageHelper---# ----------------------------------------------------------------------------# 1. push_usage_data success-# ------------------------------------------------------------------------------class TestPushUsageData:- @patch("unstract.sdk1.audit.Audit")- def test_push_success(self, mock_audit_cls):- """Successful push returns True and calls Audit."""- mock_audit = MagicMock()- mock_audit_cls.return_value = mock_audit-- result = UsageHelper.push_usage_data(- event_type="llm",- kwargs={"run_id": "run-001", "execution_id": "exec-001"},- platform_api_key="test-key",- token_counter=MagicMock(),- model_name="gpt-4",- )-- assert result is True- mock_audit.push_usage_data.assert_called_once()- call_kwargs = mock_audit.push_usage_data.call_args- assert call_kwargs.kwargs["platform_api_key"] == "test-key"- assert call_kwargs.kwargs["model_name"] == "gpt-4"- assert call_kwargs.kwargs["event_type"] == "llm"-- @patch("unstract.sdk1.audit.Audit")- def test_push_passes_token_counter(self, mock_audit_cls):- """Token counter is passed through to Audit."""- mock_audit = MagicMock()- mock_audit_cls.return_value = mock_audit- mock_counter = MagicMock()-- UsageHelper.push_usage_data(- event_type="embedding",- kwargs={"run_id": "run-002"},- platform_api_key="key-2",- token_counter=mock_counter,- )-- call_kwargs = mock_audit.push_usage_data.call_args- assert call_kwargs.kwargs["token_counter"] is mock_counter---# ----------------------------------------------------------------------------# 2. Invalid kwargs-# ------------------------------------------------------------------------------class TestPushValidation:- def test_none_kwargs_returns_false(self):- result = UsageHelper.push_usage_data(- event_type="llm",- kwargs=None,- platform_api_key="key",- )- assert result is False-- def test_empty_kwargs_returns_false(self):- result = UsageHelper.push_usage_data(- event_type="llm",- kwargs={},- platform_api_key="key",- )- assert result is False-- def test_non_dict_kwargs_returns_false(self):- result = UsageHelper.push_usage_data(- event_type="llm",- kwargs="not a dict",- platform_api_key="key",- )- assert result is False---# ----------------------------------------------------------------------------# 3. Invalid platform_api_key-# ------------------------------------------------------------------------------class TestPushApiKeyValidation:- def test_none_key_returns_false(self):- result = UsageHelper.push_usage_data(- event_type="llm",- kwargs={"run_id": "r1"},- platform_api_key=None,- )- assert result is False-- def test_empty_key_returns_false(self):- result = UsageHelper.push_usage_data(- event_type="llm",- kwargs={"run_id": "r1"},- platform_api_key="",- )- assert result is False-- def test_non_string_key_returns_false(self):- result = UsageHelper.push_usage_data(- event_type="llm",- kwargs={"run_id": "r1"},- platform_api_key=12345,- )- assert result is False---# ----------------------------------------------------------------------------# 4. Audit exceptions are caught-# ------------------------------------------------------------------------------class TestPushErrorHandling:- @patch("unstract.sdk1.audit.Audit")- def test_audit_exception_returns_false(self, mock_audit_cls):- """Audit errors are caught and return False."""- mock_audit = MagicMock()- mock_audit.push_usage_data.side_effect = Exception("Network error")- mock_audit_cls.return_value = mock_audit-- result = UsageHelper.push_usage_data(- event_type="llm",- kwargs={"run_id": "r1"},- platform_api_key="key",- token_counter=MagicMock(),- )-- assert result is False-- @patch("unstract.sdk1.audit.Audit")- def test_import_error_returns_false(self, mock_audit_cls):- """Import errors are caught gracefully."""- mock_audit_cls.side_effect = ImportError("no module")-- result = UsageHelper.push_usage_data(- event_type="llm",- kwargs={"run_id": "r1"},- platform_api_key="key",- )-- assert result is False---# ----------------------------------------------------------------------------# 5. format_float_positional-# ------------------------------------------------------------------------------class TestFormatFloat:- def test_normal_float(self):- assert UsageHelper.format_float_positional(0.0001234) == "0.0001234"-- def test_trailing_zeros_removed(self):- assert UsageHelper.format_float_positional(1.50) == "1.5"-- def test_integer_value(self):- assert UsageHelper.format_float_positional(42.0) == "42"-- def test_zero(self):- assert UsageHelper.format_float_positional(0.0) == "0"-- def test_small_value(self):- result = UsageHelper.format_float_positional(0.00000001)- assert "0.00000001" == result-- def test_custom_precision(self):- result = UsageHelper.format_float_positional(1.123456789, precision=3)- assert result == "1.123"---# ----------------------------------------------------------------------------# 6. SDK1 adapters already push usage-# ------------------------------------------------------------------------------class TestAdapterUsageTracking:- def test_llm_calls_audit_push(self):- """Verify the LLM adapter imports and calls Audit.push_usage_data.-- This is a static analysis check — we verify the SDK1 LLM module- references Audit.push_usage_data, confirming adapters handle- usage tracking internally.- """- import inspect-- from unstract.sdk1.llm import LLM-- source = inspect.getsource(LLM)- assert "push_usage_data" in source- assert "Audit" in source---# ----------------------------------------------------------------------------# 7. answer_prompt handler returns metrics-# ------------------------------------------------------------------------------class TestMetricsInResult:- @patch(- "unstract.sdk1.utils.indexing.IndexingUtils.generate_index_key",- return_value="doc-id-test",- )- @patch(- "executor.executors.legacy_executor.LegacyExecutor._get_prompt_deps"- )- @patch("executor.executors.legacy_executor.ExecutorToolShim")- def test_answer_prompt_returns_metrics(- self, mock_shim_cls, mock_get_deps, _mock_idx, tmp_path- ):- """answer_prompt result includes metrics dict."""- from unstract.sdk1.execution.context import ExecutionContext- from unstract.sdk1.execution.registry import ExecutorRegistry-- ExecutorRegistry.clear()- from executor.executors.legacy_executor import LegacyExecutor-- if "legacy" not in ExecutorRegistry.list_executors():- ExecutorRegistry.register(LegacyExecutor)-- executor = ExecutorRegistry.get("legacy")-- # Mock all dependencies- mock_llm = MagicMock()- mock_llm.get_metrics.return_value = {"total_tokens": 100}- mock_llm.get_usage_reason.return_value = "extraction"- mock_llm.complete.return_value = {- "response": MagicMock(text="test answer"),- "highlight_data": [],- "confidence_data": None,- "word_confidence_data": None,- "line_numbers": [],- "whisper_hash": "",- }-- mock_llm_cls = MagicMock(return_value=mock_llm)- mock_index = MagicMock()- mock_index.return_value.generate_index_key.return_value = "doc-123"-- mock_get_deps.return_value = (- MagicMock(), # AnswerPromptService — use real for construct- MagicMock(), # RetrievalService- MagicMock(), # VariableReplacementService- mock_index, # Index- mock_llm_cls, # LLM- MagicMock(), # EmbeddingCompat- MagicMock(), # VectorDB- )-- # Patch AnswerPromptService methods at their real location- with patch(- "executor.executors.answer_prompt.AnswerPromptService.extract_variable",- return_value="test prompt",- ), patch(- "executor.executors.answer_prompt.AnswerPromptService.construct_and_run_prompt",- return_value="test answer",- ):- ctx = ExecutionContext(- executor_name="legacy",- operation="answer_prompt",- run_id="run-metrics-001",- execution_source="tool",- organization_id="org-test",- request_id="req-metrics-001",- executor_params={- "tool_settings": {},- "outputs": [- {- "name": "field1",- "prompt": "What is X?",- "chunk-size": 512,- "chunk-overlap": 64,- "vector-db": "vdb-1",- "embedding": "emb-1",- "x2text_adapter": "x2t-1",- "llm": "llm-1",- "type": "text",- "retrieval-strategy": "simple",- "similarity-top-k": 5,- },- ],- "tool_id": "tool-1",- "file_hash": "hash123",- "file_path": str(tmp_path / "test.txt"),- "file_name": "test.txt",- "PLATFORM_SERVICE_API_KEY": "test-key",- },- )- result = executor.execute(ctx)-- assert result.success is True- assert "metrics" in result.data- assert "field1" in result.data["metrics"]-- ExecutorRegistry.clear()