Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2946 [FEAT] Prompt Studio lookups bridge, executor hook, and IDE wiring (OSS side) (#1929)

Chandrasekharan Mf742e32e92d7
diff --git a/.gitignore b/.gitignore
index f6837ce07..c655f4acf 100644
--- a/.gitignore
+++ b/.gitignore
@@ -653,7 +653,11 @@ docker/*.env
!docker/sample*.env
docker/public_tools.json
docker/proxy_overrides.yaml
-docker/compose.override.yaml
+docker/compose.*.yaml
+# ``docker/compose.debug.yaml`` is checked-in tooling — keep it out of the
+# broader ``compose.*.yaml`` ignore so a delete + recreate doesn't make it
+# look untracked, and so teammates can spot it.
+!docker/compose.debug.yaml
docker/workflow_data/
# Tool development
@@ -696,6 +700,12 @@ CLAUDE.md
CONTRIBUTION_GUIDE.md
.mcp.json
+# Codex
+AGENTS.md
+
+# Pi
+.pi/
+
# Windsurf
.qodo
.windsurfrules
diff --git a/backend/prompt_studio/lookup_utils.py b/backend/prompt_studio/lookup_utils.py
new file mode 100644
index 000000000..8cfb0a728
--- /dev/null
+++ b/backend/prompt_studio/lookup_utils.py
@@ -0,0 +1,171 @@
+"""Shared utility for lookup operations. No-ops in OSS.
+
+Only the absence of ``pluggable_apps.lookups`` itself is treated as
+"cloud not installed"; an ImportError from a transitive dependency
+re-raises so we don't silently degrade to a no-op on a real bug.
+"""
+
+import logging
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+_CLOUD_LOOKUP_MODULES = {
+ # OSS images lack the parent ``pluggable_apps`` package, so include it.
+ "pluggable_apps",
+ "pluggable_apps.lookups",
+ "pluggable_apps.lookups.execution",
+ "pluggable_apps.lookups.output_enrichment",
+ "pluggable_apps.lookups.staleness",
+ "pluggable_apps.lookups.validation",
+ "pluggable_apps.lookups.models",
+}
+
+try:
+ from pluggable_apps.lookups import execution as _execution
+ from pluggable_apps.lookups import output_enrichment as _output_enrichment
+ from pluggable_apps.lookups import staleness as _staleness
+ from pluggable_apps.lookups import validation as _validation
+ from pluggable_apps.lookups.models import LookupOutputResult as _LookupOutputResult
+
+ LOOKUPS_AVAILABLE = True
+except ImportError as e:
+ if e.name not in _CLOUD_LOOKUP_MODULES:
+ raise
+ LOOKUPS_AVAILABLE = False
+
+
+def get_lookup_config(prompt) -> dict | None:
+ """Return lookup config for a prompt, or None if lookups are unavailable."""
+ if not LOOKUPS_AVAILABLE:
+ return None
+ return _execution.build_lookup_config_for_prompt(prompt)
+
+
+def get_lookup_configs_for_tool(tool, prompts=None) -> list[dict] | None:
+ """Return lookup configs for a tool (single pass), or None in OSS.
+
+ ``prompts`` scopes validation to the run's prompts so unrelated
+ incomplete assignments on the tool don't block it.
+ """
+ if not LOOKUPS_AVAILABLE:
+ return None
+ return _execution.build_lookup_configs_for_tool(tool, prompts=prompts)
+
+
+def get_multi_var_lookups_for_tool(tool, prompt_ids=None) -> list[str]:
+ """Return names of multi-variable lookups linked to the tool, [] in OSS.
+
+ ``prompt_ids`` scopes the check so a run is only blocked when the
+ multi-var lookup is actually used by it.
+ """
+ if not LOOKUPS_AVAILABLE:
+ return []
+ _, names = _execution.has_multi_var_lookups(tool, prompt_ids=prompt_ids)
+ return names
+
+
+def persist_lookup_output(prompt_output, prompt_lookup: dict) -> None:
+ """Persist lookup enrichment result. No-op in OSS."""
+ if not LOOKUPS_AVAILABLE:
+ return
+ lookup_meta = prompt_lookup.get("meta", {})
+ lookup_id = lookup_meta.get("lookup_id")
+ if not lookup_id:
+ return
+ defaults = {
+ "lookup_definition_id": lookup_id,
+ "output": prompt_lookup.get("enriched", ""),
+ }
+ version_id = lookup_meta.get("version_id")
+ if version_id:
+ defaults["version_id"] = version_id
+ _LookupOutputResult.objects.update_or_create(
+ prompt_output=prompt_output,
+ defaults=defaults,
+ )
+
+
+def enrich_prompt_output(prompt_output, data: dict) -> dict:
+ """Let cloud plugins enrich serialized prompt output with lookup data.
+
+ No-op in OSS.
+ """
+ if not LOOKUPS_AVAILABLE:
+ return data
+ return _output_enrichment.enrich_with_lookup_output(prompt_output, data)
+
+
+def validate_lookups_for_export(prompts) -> tuple[dict, str | None]:
+ """Validate lookup assignments before export. Returns ({}, None) in OSS."""
+ if not LOOKUPS_AVAILABLE:
+ return {}, None
+ return _validation.validate_lookups_for_export(prompts)
+
+
+def get_latest_lookup_mutation_for_tool(tool):
+ """Max ``modified_at`` across lookup-related records linked to the tool
+ (version, reference file, assignment) — feeds the staleness banner.
+ None if unavailable or nothing linked.
+ """
+ if not LOOKUPS_AVAILABLE:
+ return None
+ return _staleness.get_latest_lookup_mutation_for_tool(tool)
+
+
+def get_original_value_if_enriched(
+ metadata: dict, prompt_key: str
+) -> tuple[Any, dict] | None:
+ """Return ``(original_value, prompt_lookup_dict)`` if ``prompt_key`` was
+ enriched, or ``None`` otherwise.
+
+ Pure metadata-shape check — safe to call even when LOOKUPS_AVAILABLE
+ is False (returns None because the shape won't match).
+ """
+ if not isinstance(metadata, dict):
+ return None
+ lookup_outputs = metadata.get("lookup_outputs") or {}
+ prompt_lookup = lookup_outputs.get(prompt_key)
+ if isinstance(prompt_lookup, dict) and "original" in prompt_lookup:
+ return prompt_lookup.get("original"), prompt_lookup
+ return None
+
+
+def attach_combined_output_enrichment(result: dict, enriched_by_key: dict) -> None:
+ """Stamp the combined-output payload with enriched-output metadata.
+
+ Key name stays cloud-side so the FE-plugin shape can evolve without
+ coordinating with OSS.
+ """
+ if not LOOKUPS_AVAILABLE:
+ return
+ _output_enrichment.attach_combined_output_enrichment(result, enriched_by_key)
+
+
+def extract_prompt_output_enrichment(item) -> dict | None:
+ """Pick enriched-output data off a serialized prompt-output row.
+
+ Returns a plugin-opaque dict (FE-only) or None when no enrichment
+ is present / plugin missing.
+ """
+ if not LOOKUPS_AVAILABLE:
+ return None
+ return _output_enrichment.extract_prompt_output_enrichment(item)
+
+
+def get_lookup_validation_for_tool(tool) -> dict:
+ """Pre-emptive lookup validation for FE Export / Deploy gating.
+
+ Returns an "always ok" payload in OSS so the FE gate is a no-op.
+ """
+ if not LOOKUPS_AVAILABLE:
+ return {
+ "ok": True,
+ "draft_lookups": [],
+ "multi_var_lookups": [],
+ "incomplete_lookups": [],
+ "single_pass_enabled": bool(
+ getattr(tool, "single_pass_extraction_mode", False)
+ ),
+ }
+ return _validation.get_lookup_validation_for_tool(tool)
diff --git a/backend/prompt_studio/prompt_studio_core_v2/migrations/0007_customtool_last_exported_at.py b/backend/prompt_studio/prompt_studio_core_v2/migrations/0007_customtool_last_exported_at.py
new file mode 100644
index 000000000..91539c9cf
--- /dev/null
+++ b/backend/prompt_studio/prompt_studio_core_v2/migrations/0007_customtool_last_exported_at.py
@@ -0,0 +1,21 @@
+# Generated by Django 4.2.1 on 2026-04-21 20:20
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("prompt_studio_core_v2", "0006_add_custom_data_to_customtool"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="customtool",
+ name="last_exported_at",
+ field=models.DateTimeField(
+ blank=True,
+ db_comment="Timestamp of the last successful export; NULL if never exported since the field was introduced.",
+ null=True,
+ ),
+ ),
+ ]
diff --git a/backend/prompt_studio/prompt_studio_core_v2/models.py b/backend/prompt_studio/prompt_studio_core_v2/models.py
index 1e1802b77..cd2a12dac 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/models.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/models.py
@@ -161,6 +161,15 @@ class CustomTool(DefaultOrganizationMixin, BaseModel):
db_comment="Flag to share this custom tool with all users in the organization",
)
+ # NULL on pre-feature tools; populated on first successful export.
+ # Drives staleness checks (e.g. lookup-change banner) without requiring
+ # a data backfill.
+ last_exported_at = models.DateTimeField(
+ null=True,
+ blank=True,
+ db_comment="Timestamp of the last successful export; NULL if never exported since the field was introduced.",
+ )
+
objects = CustomToolModelManager()
def delete(self, organization_id=None, *args, **kwargs):
diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
index ff2b34550..62cbf2d9a 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
@@ -14,12 +14,17 @@ from django.conf import settings
from django.db import transaction
from django.db.models.manager import BaseManager
from plugins import get_plugin
+from rest_framework.exceptions import APIException
from rest_framework.request import Request
from utils.file_storage.constants import FileStorageKeys
from utils.file_storage.helpers.prompt_studio_file_helper import PromptStudioFileHelper
from utils.local_context import StateStore
from backend.celery_service import app as celery_app
+from prompt_studio.lookup_utils import (
+ get_lookup_config,
+ get_lookup_configs_for_tool,
+)
from prompt_studio.prompt_profile_manager_v2.models import ProfileManager
from prompt_studio.prompt_profile_manager_v2.profile_manager_helper import (
ProfileManagerHelper,
@@ -387,6 +392,9 @@ class PromptStudioHelper:
if webhook_enabled:
output[TSPKeys.POSTPROCESSING_WEBHOOK_URL] = webhook_url
+ if lookup_config := get_lookup_config(prompt):
+ output["lookup_config"] = lookup_config
+
output[TSPKeys.EVAL_SETTINGS] = {}
output[TSPKeys.EVAL_SETTINGS][TSPKeys.EVAL_SETTINGS_EVALUATE] = prompt.evaluate
output[TSPKeys.EVAL_SETTINGS][TSPKeys.EVAL_SETTINGS_MONITOR_LLM] = [monitor_llm]
@@ -798,6 +806,9 @@ class PromptStudioHelper:
if webhook_enabled:
output[TSPKeys.POSTPROCESSING_WEBHOOK_URL] = webhook_url
+ if lookup_config := get_lookup_config(prompt):
+ output["lookup_config"] = lookup_config
+
output[TSPKeys.EVAL_SETTINGS] = {}
output[TSPKeys.EVAL_SETTINGS][TSPKeys.EVAL_SETTINGS_EVALUATE] = prompt.evaluate
output[TSPKeys.EVAL_SETTINGS][TSPKeys.EVAL_SETTINGS_MONITOR_LLM] = [monitor_llm]
@@ -1166,6 +1177,10 @@ class PromptStudioHelper:
TSPKeys.SIMILARITY_TOP_K: default_profile.similarity_top_k,
}
+ lookup_configs = get_lookup_configs_for_tool(tool, prompts=prompts)
+ if lookup_configs:
+ tool_settings["lookup_configs"] = lookup_configs
+
for p in prompts:
if not p.prompt:
raise EmptyPromptError()
@@ -1607,6 +1622,9 @@ class PromptStudioHelper:
is_single_pass=False,
profile_manager_id=profile_manager_id,
)
+ except APIException:
+ # Validation responses are user-facing; DRF renders them as-is.
+ raise
except Exception as e:
logger.error(
f"[{tool.tool_id}] Error while fetching response for "
@@ -1672,6 +1690,9 @@ class PromptStudioHelper:
document_id=document_id,
is_single_pass=True,
)
+ except APIException:
+ # Validation responses are user-facing; DRF renders them as-is.
+ raise
except Exception as e:
logger.error(
f"[{tool.tool_id}] Error while fetching single pass response: {e}"
@@ -1911,6 +1932,8 @@ class PromptStudioHelper:
output[TSPKeys.ENABLE_POSTPROCESSING_WEBHOOK] = webhook_enabled
if webhook_enabled:
output[TSPKeys.POSTPROCESSING_WEBHOOK_URL] = webhook_url
+ if lookup_config := get_lookup_config(prompt):
+ output["lookup_config"] = lookup_config
# Eval settings for the prompt
output[TSPKeys.EVAL_SETTINGS] = {}
output[TSPKeys.EVAL_SETTINGS][TSPKeys.EVAL_SETTINGS_EVALUATE] = prompt.evaluate
diff --git a/backend/prompt_studio/prompt_studio_core_v2/urls.py b/backend/prompt_studio/prompt_studio_core_v2/urls.py
index 86cbb97dd..9163e8736 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/urls.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/urls.py
@@ -66,6 +66,10 @@ prompt_studio_deployment_usage = PromptStudioCoreView.as_view(
prompt_studio_task_status = PromptStudioCoreView.as_view({"get": "task_status"})
+prompt_studio_lookup_validation = PromptStudioCoreView.as_view(
+ {"get": "lookup_validation"}
+)
+
urlpatterns = format_suffix_patterns(
[
@@ -165,5 +169,10 @@ urlpatterns = format_suffix_patterns(
prompt_studio_task_status,
name="prompt-studio-task-status",
),
+ path(
+ "prompt-studio/<uuid:pk>/lookup-validation/",
+ prompt_studio_lookup_validation,
+ name="prompt-studio-lookup-validation",
+ ),
]
)
diff --git a/backend/prompt_studio/prompt_studio_core_v2/views.py b/backend/prompt_studio/prompt_studio_core_v2/views.py
index 99edefd0b..30ea7045e 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/views.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/views.py
@@ -14,6 +14,7 @@ from celery.result import AsyncResult
from django.db import IntegrityError
from django.db.models import Count, OuterRef, QuerySet, Subquery
from django.http import HttpRequest, HttpResponse
+from django.utils import timezone
from file_management.constants import FileInformationKey as FileKey
from file_management.exceptions import FileNotFound
from permissions.permission import IsOwner, IsOwnerOrSharedUserOrSharedToOrg
@@ -32,6 +33,11 @@ from utils.user_session import UserSessionUtils
from workflow_manager.endpoint_v2.models import WorkflowEndpoint
from backend.celery_service import app as celery_app
+from prompt_studio.lookup_utils import (
+ get_latest_lookup_mutation_for_tool,
+ get_lookup_validation_for_tool,
+ get_multi_var_lookups_for_tool,
+)
from prompt_studio.prompt_profile_manager_v2.constants import (
ProfileManagerErrors,
ProfileManagerKeys,
@@ -90,6 +96,29 @@ from .serializers import (
logger = logging.getLogger(__name__)
+def _multi_var_lookup_block_response(custom_tool, prompt_ids=None):
+ """Block non-SP runs when a linked lookup has >1 input variable.
+
+ Multi-var lookups only resolve correctly under single-pass; ``prompt_ids``
+ scopes the gate so an unrelated multi-var lookup doesn't block runs that
+ don't actually use it. Caller must skip this on the SP path.
+ """
+ names = get_multi_var_lookups_for_tool(custom_tool, prompt_ids=prompt_ids)
+ if not names:
+ return None
+ return Response(
+ {
+ "detail": (
+ "Multi-variable lookup(s) "
+ f"{', '.join(names)} are linked to prompts in this project. "
+ "These can only run in single pass extraction mode. "
+ "Enable single pass or unlink the lookup before running."
+ )
+ },
+ status=status.HTTP_400_BAD_REQUEST,
+ )
+
+
class PromptStudioCoreView(viewsets.ModelViewSet):
"""Viewset to handle all Custom tool related operations."""
@@ -477,6 +506,14 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
document_id: str = request.data.get(ToolStudioPromptKeys.DOCUMENT_ID)
prompt_id: str = request.data.get(ToolStudioPromptKeys.ID)
run_id: str = request.data.get(ToolStudioPromptKeys.RUN_ID)
+ # Must precede the lookup gate so missing prompt_id returns a clear 400.
+ if not prompt_id:
+ return Response(
+ {"error": "prompt id is required."},
+ status=status.HTTP_400_BAD_REQUEST,
+ )
+ if err := _multi_var_lookup_block_response(custom_tool, prompt_ids=[prompt_id]):
+ return err
profile_manager_id: str = request.data.get(
ToolStudioPromptKeys.PROFILE_MANAGER_ID
)
@@ -485,13 +522,6 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
org_id = UserSessionUtils.get_organization_id(request)
user_id = custom_tool.created_by.user_id
-
- # Resolve prompt — guard against missing / stale prompt_id
- if not prompt_id:
- return Response(
- {"error": "prompt id is required."},
- status=status.HTTP_400_BAD_REQUEST,
- )
try:
prompt = ToolStudioPrompt.objects.get(pk=prompt_id)
except ToolStudioPrompt.DoesNotExist:
@@ -510,9 +540,8 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
document: DocumentManager = DocumentManager.objects.get(pk=document_id)
doc_path = str(Path(doc_path) / document.document_name)
- # Agentic table prompts have a separate executor worker. Build the
- # payload via the cloud payload_modifier plugin and dispatch directly
- # so the legacy answer_prompt path is bypassed.
+ # Agentic table prompts have their own executor — build payload via
+ # the cloud plugin and dispatch directly, bypassing answer_prompt.
if prompt.enforce_type == ToolStudioPromptKeys.AGENTIC_TABLE:
payload_modifier_plugin = get_plugin("payload_modifier")
if not payload_modifier_plugin:
@@ -633,6 +662,8 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
{"error": "prompt_ids is required and must be non-empty."},
status=status.HTTP_400_BAD_REQUEST,
)
+ if err := _multi_var_lookup_block_response(custom_tool, prompt_ids=prompt_ids):
+ return err
document_id: str = request.data.get(ToolStudioPromptKeys.DOCUMENT_ID)
run_id: str = request.data.get(ToolStudioPromptKeys.RUN_ID)
profile_manager_id: str = request.data.get(
@@ -881,9 +912,17 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
serializer = ProfileManagerSerializer(data=request.data, context=context)
serializer.is_valid(raise_exception=True)
# Check for the maximum number of profiles constraint
- prompt_studio_tool = serializer.validated_data[
+ prompt_studio_tool = serializer.validated_data.get(
ProfileManagerKeys.PROMPT_STUDIO_TOOL
- ]
+ )
+ if not prompt_studio_tool:
+ # Write back into validated_data so perform_create() doesn't
+ # persist NULL and orphan the profile from every
+ # ``filter(prompt_studio_tool=...)`` query.
+ prompt_studio_tool = self.get_object()
+ serializer.validated_data[ProfileManagerKeys.PROMPT_STUDIO_TOOL] = (
+ prompt_studio_tool
+ )
profile_count = ProfileManager.objects.filter(
prompt_studio_tool=prompt_studio_tool
).count()
@@ -1103,6 +1142,10 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
force_export=force_export,
)
+ # Anchor for staleness checks (e.g. lookup-change banner).
+ custom_tool.last_exported_at = timezone.now()
+ custom_tool.save(update_fields=["last_exported_at"])
+
# Notify HubSpot about first tool export
notify_hubspot_event(
user=request.user,
@@ -1116,6 +1159,15 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
status=status.HTTP_200_OK,
)
+ @action(detail=True, methods=["get"], url_path="lookup-validation")
+ def lookup_validation(self, request: Request, pk: Any = None) -> Response:
+ """Pre-emptive lookup gating for Export / API Deployment buttons.
+
+ Cloud-only check; OSS returns ``ok: True`` so the FE proceeds.
+ """
+ custom_tool = self.get_object()
+ return Response(get_lookup_validation_for_tool(custom_tool))
+
@action(detail=True, methods=["get"])
def export_tool_info(self, request: Request, pk: Any = None) -> Response:
custom_tool = self.get_object()
@@ -1294,10 +1346,20 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
instance: CustomTool = self.get_object()
is_used, workflow_ids = self._check_tool_usage_in_workflows(instance)
+ # NULL last_exported_at → treat as clean to avoid false alarms
+ # on pre-feature projects.
+ is_lookup_dirty = False
+ if instance.last_exported_at is not None:
+ latest = get_latest_lookup_mutation_for_tool(instance)
+ is_lookup_dirty = (
+ latest is not None and latest > instance.last_exported_at
+ )
+
deployment_info: dict = {
"is_used": is_used,
"deployment_types": [],
"message": "",
+ "is_lookup_dirty": is_lookup_dirty,
}
if is_used and workflow_ids:
diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py
index 64932eb3f..9c963cf3f 100644
--- a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py
+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py
@@ -2,8 +2,15 @@ import json
import logging
from typing import Any
-from django.core.exceptions import ObjectDoesNotExist
-
+from django.core.exceptions import ValidationError
+from django.db import IntegrityError
+
+from prompt_studio.lookup_utils import (
+ attach_combined_output_enrichment,
+ extract_prompt_output_enrichment,
+ get_original_value_if_enriched,
+ persist_lookup_output,
+)
from prompt_studio.prompt_profile_manager_v2.models import ProfileManager
from prompt_studio.prompt_studio_core_v2.exceptions import (
AnswerFetchError,
@@ -170,6 +177,15 @@ class OutputManagerHelper:
# TODO: use enums here
output = outputs.get(prompt.prompt_key)
+
+ # On enrichment, store the raw LLM output here; the enriched
+ # value is persisted separately via persist_lookup_output.
+ enrichment = get_original_value_if_enriched(metadata, prompt.prompt_key)
+ if enrichment is not None:
+ output, prompt_lookup = enrichment
+ else:
+ prompt_lookup = None
+
if prompt.enforce_type in {
"json",
"table",
@@ -195,6 +211,18 @@ class OutputManagerHelper:
word_confidence_data=prompt_word_confidence_data,
)
+ # Narrow except so plugin contract drift surfaces as a real
+ # error instead of being masked as a successful save.
+ if prompt_lookup:
+ try:
+ persist_lookup_output(prompt_output, prompt_lookup)
+ except (IntegrityError, ValidationError):
+ logger.error(
+ "Failed to persist lookup output for prompt %s",
+ prompt.prompt_key,
+ exc_info=True,
+ )
+
# Serialize the instance
serializer = PromptStudioOutputSerializer(prompt_output)
serialized_data.append(serializer.data)
@@ -242,41 +270,76 @@ class OutputManagerHelper:
Returns:
dict[str, Any]: Formatted JSON response for combined output.
+ Cloud plugins may attach an opaque enrichment payload.
"""
- # Initialize the result dictionary
+ from prompt_studio.lookup_utils import enrich_prompt_output
+
+ # Memoise default-profile resolution per tool to avoid N+1 on this
+ # hot path (panel-switch latency).
+ default_profile_cache: dict[str, str | None] = {}
+
+ def _resolve(tool_prompt: ToolStudioPrompt) -> str | None:
+ profile_manager_id = tool_prompt.profile_manager_id
+ if profile_manager_id:
+ return profile_manager_id
+ if not use_default_profile:
+ return None
+ tool_id = tool_prompt.tool_id_id
+ if tool_id not in default_profile_cache:
+ try:
+ default_profile_cache[tool_id] = (
+ ProfileManager.get_default_llm_profile(
+ tool_prompt.tool_id
+ ).profile_id
+ )
+ except DefaultProfileError:
+ default_profile_cache[tool_id] = None
+ return default_profile_cache[tool_id]
+
+ prompts_to_query: list[tuple[ToolStudioPrompt, str]] = []
result: dict[str, Any] = {}
- # Iterate over ToolStudioPrompt records
for tool_prompt in tool_studio_prompts:
if tool_prompt.prompt_type == PSOMKeys.NOTES:
continue
- prompt_id = str(tool_prompt.prompt_id)
- profile_manager_id = tool_prompt.profile_manager_id
-
- # If profile_manager is not set, skip this record
- if not profile_manager_id and not use_default_profile:
+ profile_manager_id = _resolve(tool_prompt)
+ if profile_manager_id is None:
result[tool_prompt.prompt_key] = ""
continue
-
- if not profile_manager_id:
- default_profile = ProfileManager.get_default_llm_profile(
- tool_prompt.tool_id
- )
- profile_manager_id = default_profile.profile_id
-
- try:
- queryset = PromptStudioOutputManager.objects.filter(
- prompt_id=prompt_id,
- profile_manager=profile_manager_id,
+ prompts_to_query.append((tool_prompt, profile_manager_id))
+
+ # ``DISTINCT ON`` (Postgres) yields the latest row per
+ # (prompt_id, profile_manager_id) at the SQL layer.
+ outputs_index: dict[tuple[str, str], PromptStudioOutputManager] = {}
+ if prompts_to_query:
+ prompt_ids = [str(p.prompt_id) for p, _ in prompts_to_query]
+ profile_ids = list({pmid for _, pmid in prompts_to_query})
+ outputs = (
+ PromptStudioOutputManager.objects.filter(
+ prompt_id__in=prompt_ids,
+ profile_manager_id__in=profile_ids,
is_single_pass_extract=False,
document_manager_id=document_manager_id,
)
-
- if not queryset.exists():
- result[tool_prompt.prompt_key] = ""
- continue
-
- for output in queryset:
- result[tool_prompt.prompt_key] = output.output
- except ObjectDoesNotExist:
+ .order_by("prompt_id", "profile_manager_id", "-modified_at")
+ .distinct("prompt_id", "profile_manager_id")
+ )
+ outputs_index = {
+ (str(o.prompt_id), str(o.profile_manager_id)): o for o in outputs
+ }
+
+ enrichment_by_key: dict[str, Any] = {}
+ for tool_prompt, profile_manager_id in prompts_to_query:
+ output = outputs_index.get(
+ (str(tool_prompt.prompt_id), str(profile_manager_id))
+ )
+ if output is None:
result[tool_prompt.prompt_key] = ""
+ continue
+ result[tool_prompt.prompt_key] = output.output
+ enriched = enrich_prompt_output(output, {})
+ bundle = extract_prompt_output_enrichment(enriched)
+ if bundle is not None:
+ enrichment_by_key[tool_prompt.prompt_key] = bundle
+
+ attach_combined_output_enrichment(result, enrichment_by_key)
return result
diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py b/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py
index 275e4a095..1c56e2323 100644
--- a/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py
+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py
@@ -4,6 +4,7 @@ import logging
from usage_v2.helper import UsageHelper
from backend.serializers import AuditSerializer
+from prompt_studio.lookup_utils import enrich_prompt_output
from .models import PromptStudioOutputManager
from .output_manager_util import OutputManagerUtils
@@ -47,6 +48,16 @@ class PromptStudioOutputSerializer(AuditSerializer):
" | Process continued"
)
data["coverage"] = {}
+ # log+continue: enrichment failure shouldn't 500 the list endpoint.
+ try:
+ data = enrich_prompt_output(instance, data)
+ except Exception as e:
+ logger.error(
+ "Error occurred while enriching prompt output for "
+ f"prompt_id {instance.prompt_id} (run_id={instance.run_id}): {e}"
+ " | Process continued"
+ )
+
# Convert string to list
try:
context = data["context"]
diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/urls.py b/backend/prompt_studio/prompt_studio_output_manager_v2/urls.py
index 61ec8540f..77270c22f 100644
--- a/backend/prompt_studio/prompt_studio_output_manager_v2/urls.py
+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/urls.py
@@ -7,6 +7,7 @@ prompt_doc_list = PromptStudioOutputView.as_view({"get": "list"})
get_output_for_tool_default = PromptStudioOutputView.as_view(
{"get": "get_output_for_tool_default"}
)
+latest_outputs_by_keys = PromptStudioOutputView.as_view({"get": "latest_outputs_by_keys"})
urlpatterns = format_suffix_patterns(
[
@@ -16,5 +17,10 @@ urlpatterns = format_suffix_patterns(
get_output_for_tool_default,
name="prompt-default-profile-outputs",
),
+ path(
+ "prompt-output/latest-by-keys/",
+ latest_outputs_by_keys,
+ name="prompt-output-latest-by-keys",
+ ),
]
)
diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/views.py b/backend/prompt_studio/prompt_studio_output_manager_v2/views.py
index c0c002d80..44111dc74 100644
--- a/backend/prompt_studio/prompt_studio_output_manager_v2/views.py
+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/views.py
@@ -5,11 +5,12 @@ from django.core.exceptions import ObjectDoesNotExist
from django.db.models import QuerySet
from django.http import HttpRequest
from rest_framework import status, viewsets
-from rest_framework.exceptions import APIException
+from rest_framework.exceptions import ValidationError
from rest_framework.response import Response
from rest_framework.versioning import URLPathVersioning
from utils.common_utils import CommonUtils
from utils.filtering import FilterHelper
+from utils.user_context import UserContext
from prompt_studio.prompt_studio_output_manager_v2.constants import (
PromptOutputManagerErrorMessage,
@@ -61,6 +62,57 @@ class PromptStudioOutputView(viewsets.ModelViewSet):
return queryset
+ def latest_outputs_by_keys(self, request: HttpRequest) -> Response:
+ """Return the most recent raw output value per source prompt key.
+
+ Backs the lookup Test panel's "Use Latest Outputs" button. Returns
+ raw extraction (not enriched) so the lookup can be tested fresh.
+ """
+ tool_id = request.GET.get("tool_id")
+ keys_param = request.GET.get("prompt_keys", "")
+ if not tool_id:
+ # APIException(code=400) returns 500; ValidationError returns 400.
+ raise ValidationError(detail=PromptOutputManagerErrorMessage.TOOL_VALIDATION)
+
+ prompt_keys = [k.strip() for k in keys_param.split(",") if k.strip()]
+ if not prompt_keys:
+ return Response({}, status=status.HTTP_200_OK)
+
+ # Custom actions skip filter_queryset(), so OrganizationFilterBackend
+ # never runs — scope explicitly to prevent cross-tenant reads.
+ organization = UserContext.get_organization()
+ prompt_id_to_key = dict(
+ ToolStudioPrompt.objects.filter(
+ tool_id=tool_id,
+ tool_id__organization=organization,
+ prompt_key__in=prompt_keys,
+ ).values_list("prompt_id", "prompt_key")
+ )
+ if not prompt_id_to_key:
+ return Response({}, status=status.HTTP_200_OK)
+
+ # ``DISTINCT ON("prompt_id")`` keeps the latest row per prompt at
+ # the SQL layer to avoid materialising every doc × run combo.
+ outputs = (
+ PromptStudioOutputManager.objects.filter(
+ prompt_id__in=prompt_id_to_key.keys(),
+ tool_id__organization=organization,
+ )
+ .exclude(output__isnull=True)
+ .exclude(output__exact="")
+ .order_by("prompt_id", "-modified_at")
+ .distinct("prompt_id")
+ .values("prompt_id", "output")
+ )
+
+ result: dict[str, str] = {}
+ for row in outputs:
+ key = prompt_id_to_key.get(row["prompt_id"])
+ if key:
+ result[key] = row["output"]
+
+ return Response(result, status=status.HTTP_200_OK)
+
def get_output_for_tool_default(self, request: HttpRequest) -> Response:
# Get the tool_id from request parameters
# TODO: Setup Serializer here
@@ -69,7 +121,7 @@ class PromptStudioOutputView(viewsets.ModelViewSet):
tool_validation_message = PromptOutputManagerErrorMessage.TOOL_VALIDATION
tool_not_found = PromptOutputManagerErrorMessage.TOOL_NOT_FOUND
if not tool_id:
- raise APIException(detail=tool_validation_message, code=400)
+ raise ValidationError(detail=tool_validation_message)
try:
# Fetch ToolStudioPrompt records based on tool_id
@@ -77,7 +129,7 @@ class PromptStudioOutputView(viewsets.ModelViewSet):
tool_id=tool_id
).order_by("sequence_number")
except ObjectDoesNotExist:
- raise APIException(detail=tool_not_found, code=400)
+ raise ValidationError(detail=tool_not_found)
# Invoke helper method to frame and fetch default response.
result: dict[str, Any] = OutputManagerHelper.fetch_default_output_response(
diff --git a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py
index 08d2c27ba..4fee8c10b 100644
--- a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py
+++ b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py
@@ -7,6 +7,7 @@ from django.conf import settings
from django.db import IntegrityError
from plugins import get_plugin
+from prompt_studio.lookup_utils import validate_lookups_for_export
from prompt_studio.prompt_profile_manager_v2.models import ProfileManager
from prompt_studio.prompt_studio_core_v2.models import CustomTool
from prompt_studio.prompt_studio_core_v2.prompt_studio_helper import PromptStudioHelper
@@ -297,6 +298,15 @@ class PromptStudioRegistryHelper:
settings, JsonSchemaKey.WORD_CONFIDENCE_POSTAMBLE.upper(), ""
)
+ # Validate only what will actually be exported — NOTES / inactive
+ # prompts never run, so incomplete lookups on them shouldn't fail export.
+ exportable_prompts = [
+ p for p in prompts if p.prompt_type != JsonSchemaKey.NOTES and p.active
+ ]
+ lookup_configs, lookup_error = validate_lookups_for_export(exportable_prompts)
+ if lookup_error:
+ raise InValidCustomToolError(lookup_error)
+
for prompt in prompts:
if prompt.prompt_type == JsonSchemaKey.NOTES or not prompt.active:
continue
@@ -355,6 +365,9 @@ class PromptStudioRegistryHelper:
output[JsonSchemaKey.POSTPROCESSING_WEBHOOK_URL] = (
prompt.postprocessing_webhook_url
)
+ prompt_id_str = str(prompt.prompt_id)
+ if prompt_id_str in lookup_configs:
+ output["lookup_config"] = lookup_configs[prompt_id_str]
# Retaining the old fields in condition
# for backward compatibility. To be removed in future.
if (
diff --git a/backend/prompt_studio/prompt_studio_v2/serializers.py b/backend/prompt_studio/prompt_studio_v2/serializers.py
index e1adddc33..6a4d28032 100644
--- a/backend/prompt_studio/prompt_studio_v2/serializers.py
+++ b/backend/prompt_studio/prompt_studio_v2/serializers.py
@@ -5,6 +5,23 @@ from backend.serializers import AuditSerializer
from .models import ToolStudioPrompt
+class ToolStudioPromptListSerializer(serializers.ModelSerializer):
+ """Lightweight serializer for listing prompts by tool.
+
+ Returns only the fields needed for linking/display without
+ output data or coverage calculation.
+ """
+
+ class Meta:
+ model = ToolStudioPrompt
+ fields = [
+ "prompt_id",
+ "prompt_key",
+ "enforce_type",
+ "sequence_number",
+ ]
+
+
class ToolStudioPromptSerializer(AuditSerializer):
class Meta:
model = ToolStudioPrompt
diff --git a/backend/prompt_studio/prompt_studio_v2/urls.py b/backend/prompt_studio/prompt_studio_v2/urls.py
index 23e5f0243..0ce3d0354 100644
--- a/backend/prompt_studio/prompt_studio_v2/urls.py
+++ b/backend/prompt_studio/prompt_studio_v2/urls.py
@@ -3,6 +3,8 @@ from rest_framework.urlpatterns import format_suffix_patterns
from .views import ToolStudioPromptView
+prompt_studio_prompt_list = ToolStudioPromptView.as_view({"get": "list"})
+
prompt_studio_prompt_detail = ToolStudioPromptView.as_view(
{
"get": "retrieve",
@@ -16,15 +18,20 @@ reorder_prompts = ToolStudioPromptView.as_view({"post": "reorder_prompts"})
urlpatterns = format_suffix_patterns(
[
+ path(
+ "prompt/reorder/",
+ reorder_prompts,
+ name="reorder_prompts",
+ ),
path(
"prompt/<uuid:pk>/",
prompt_studio_prompt_detail,
name="tool-studio-prompt-detail",
),
path(
- "prompt/reorder/",
- reorder_prompts,
- name="reorder_prompts",
+ "prompt/",
+ prompt_studio_prompt_list,
+ name="tool-studio-prompt-list",
),
]
)
diff --git a/backend/prompt_studio/prompt_studio_v2/views.py b/backend/prompt_studio/prompt_studio_v2/views.py
index f120baf15..a54048027 100644
--- a/backend/prompt_studio/prompt_studio_v2/views.py
+++ b/backend/prompt_studio/prompt_studio_v2/views.py
@@ -10,7 +10,10 @@ from prompt_studio.permission import PromptAcesssToUser
from prompt_studio.prompt_studio_v2.constants import ToolStudioPromptKeys
from prompt_studio.prompt_studio_v2.controller import PromptStudioController
from prompt_studio.prompt_studio_v2.models import ToolStudioPrompt
-from prompt_studio.prompt_studio_v2.serializers import ToolStudioPromptSerializer
+from prompt_studio.prompt_studio_v2.serializers import (
+ ToolStudioPromptListSerializer,
+ ToolStudioPromptSerializer,
+)
class ToolStudioPromptView(viewsets.ModelViewSet):
@@ -28,8 +31,14 @@ class ToolStudioPromptView(viewsets.ModelViewSet):
versioning_class = URLPathVersioning
serializer_class = ToolStudioPromptSerializer
+
permission_classes: list[type[PromptAcesssToUser]] = [PromptAcesssToUser]
+ def get_serializer_class(self):
+ if self.action == "list":
+ return ToolStudioPromptListSerializer
+ return ToolStudioPromptSerializer
+
def get_queryset(self) -> QuerySet | None:
filter_args = FilterHelper.build_filter_args(
self.request,
diff --git a/backend/usage_v2/hooks.py b/backend/usage_v2/hooks.py
new file mode 100644
index 000000000..31d576e69
--- /dev/null
+++ b/backend/usage_v2/hooks.py
@@ -0,0 +1,27 @@
+"""Post-write hooks for ``UsageBatchCreateView``.
+
+Hooks fire inside the view's transaction; a failure rolls the batch back
+so Usage rows and any side-table writes stay consistent. Records carry
+an opaque ``cloud_extras`` dict that OSS forwards verbatim — plugins
+read only the keys they own.
+"""
+
+from collections.abc import Callable
+
+from .models import Usage
+
+PostWriteHook = Callable[[list[dict], list[Usage]], None]
+
+_post_write_hooks: list[PostWriteHook] = []
+
+
+def register_post_write_hook(fn: PostWriteHook) -> PostWriteHook:
+ # Idempotent: ready() can re-fire under test reloads / dev autoreload.
+ if fn not in _post_write_hooks:
+ _post_write_hooks.append(fn)
+ return fn
+
+
+def run_post_write_hooks(records: list[dict], usage_objects: list[Usage]) -> None:
+ for hook in _post_write_hooks:
+ hook(records, usage_objects)
diff --git a/backend/usage_v2/internal_urls.py b/backend/usage_v2/internal_urls.py
index b5a867555..e7e0082e9 100644
--- a/backend/usage_v2/internal_urls.py
+++ b/backend/usage_v2/internal_urls.py
@@ -17,4 +17,9 @@ urlpatterns = [
internal_views.PagesProcessedInternalView.as_view(),
name="aggregated-pages-processed",
),
+ path(
+ "batch/",
+ internal_views.UsageBatchCreateView.as_view(),
+ name="usage-batch-create",
+ ),
]
diff --git a/backend/usage_v2/internal_views.py b/backend/usage_v2/internal_views.py
index 3c28779c5..85c20c16e 100644
--- a/backend/usage_v2/internal_views.py
+++ b/backend/usage_v2/internal_views.py
@@ -1,19 +1,35 @@
-"""Internal API views for Usage access by workers."""
+"""Internal API views for Usage access by workers.
+
+Mounted under ``/internal/`` and gated by ``InternalAPIAuthMiddleware``.
+"""
import logging
+from django.db import transaction
from django.http import JsonResponse
from rest_framework import status
+from rest_framework.exceptions import APIException, ValidationError
from rest_framework.request import Request
+from rest_framework.response import Response
from rest_framework.views import APIView
+from utils.user_context import UserContext
from unstract.core.data_models import UsageResponseData
from .helper import UsageHelper
+from .hooks import run_post_write_hooks
+from .models import Usage
+from .serializers import UsageBatchCreateSerializer
logger = logging.getLogger(__name__)
+class UsagePersistError(APIException):
+ status_code = status.HTTP_500_INTERNAL_SERVER_ERROR
+ default_detail = "Failed to persist usage records."
+ default_code = "usage_persist_failed"
+
+
class UsageInternalView(APIView):
"""Internal API view for workers to access usage data.
@@ -133,3 +149,66 @@ class PagesProcessedInternalView(APIView):
},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
+
+
+class UsageBatchCreateView(APIView):
+ """Bulk create usage records from worker finalization."""
+
+ def post(self, request: Request) -> Response:
+ input_serializer = UsageBatchCreateSerializer(data=request.data)
+ input_serializer.is_valid(raise_exception=True)
+ records = input_serializer.validated_data["records"]
+ if not records:
+ return Response({"created": 0}, status=status.HTTP_200_OK)
+
+ organization = UserContext.get_organization()
+ if organization is None:
+ logger.error(
+ "UsageBatchCreateView received %d records with no organization; "
+ "refusing to write rows that would be invisible to tenant dashboards",
+ len(records),
+ )
+ raise ValidationError(
+ "Organization context missing. Worker must send X-Organization-ID."
+ )
+
+ usage_objects = [
+ Usage(
+ organization=organization,
+ workflow_id=r.get("workflow_id", ""),
+ execution_id=r.get("execution_id", ""),
+ adapter_instance_id=r.get("adapter_instance_id", ""),
+ run_id=r.get("run_id"),
+ usage_type=r.get("usage_type", "llm"),
+ # Coerce "" to None so the cross-field CheckConstraint passes.
+ llm_usage_reason=r.get("llm_usage_reason") or None,
+ model_name=r.get("model_name", ""),
+ embedding_tokens=r.get("embedding_tokens", 0),
+ prompt_tokens=r.get("prompt_tokens", 0),
+ completion_tokens=r.get("completion_tokens", 0),
+ total_tokens=r.get("total_tokens", 0),
+ cost_in_dollars=r.get("cost_in_dollars", 0.0),
+ project_id=r.get("project_id"),
+ prompt_id=r.get("prompt_id"),
+ execution_time_ms=r.get("execution_time_ms"),
+ status=r.get("status"),
+ error_message=r.get("error_message"),
+ )
+ for r in records
+ ]
+
+ try:
+ # Atomic with hooks: orphan Usage rows are worse than retrying.
+ with transaction.atomic():
+ created = Usage.objects.bulk_create(usage_objects, batch_size=500)
+ run_post_write_hooks(records, created)
+ except Exception as e:
+ logger.error(
+ "bulk_create failed for %d usage records (org=%s): %s",
+ len(usage_objects),
+ organization.organization_id,
+ e,
+ exc_info=True,
+ )
+ raise UsagePersistError() from e
+ return Response({"created": len(created)}, status=status.HTTP_201_CREATED)
diff --git a/backend/usage_v2/migrations/0004_usage_metrics_fields.py b/backend/usage_v2/migrations/0004_usage_metrics_fields.py
new file mode 100644
index 000000000..191a31dd2
--- /dev/null
+++ b/backend/usage_v2/migrations/0004_usage_metrics_fields.py
@@ -0,0 +1,89 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("usage_v2", "0003_usage_usage_executi_4deb35_idx"),
+ ]
+
+ operations = [
+ # Extend llm_usage_reason choices (cloud plugins append at runtime)
+ migrations.AlterField(
+ model_name="usage",
+ name="llm_usage_reason",
+ field=models.CharField(
+ blank=True,
+ choices=[
+ ("extraction", "Extraction"),
+ ("challenge", "Challenge"),
+ ("summarize", "Summarize"),
+ ("lookup", "Lookup"),
+ ],
+ db_comment="Reason for LLM usage. Empty if usage_type is 'embedding'. ",
+ max_length=255,
+ null=True,
+ ),
+ ),
+ migrations.AddField(
+ model_name="usage",
+ name="project_id",
+ field=models.UUIDField(
+ blank=True,
+ db_comment=(
+ "Prompt Studio project (tool) the call belongs to (no FK; "
+ "survives tool deletion). NULL for embeddings and historical "
+ "rows."
+ ),
+ null=True,
+ ),
+ ),
+ migrations.AddField(
+ model_name="usage",
+ name="prompt_id",
+ field=models.UUIDField(
+ blank=True,
+ db_comment=(
+ "Prompt key UUID that triggered the call (no FK; survives "
+ "prompt deletion). NULL for single-pass / embeddings / "
+ "historical rows."
+ ),
+ null=True,
+ ),
+ ),
+ migrations.AddField(
+ model_name="usage",
+ name="execution_time_ms",
+ field=models.IntegerField(
+ blank=True,
+ db_comment="Wall-clock time for the operation in milliseconds",
+ null=True,
+ ),
+ ),
+ migrations.AddField(
+ model_name="usage",
+ name="status",
+ field=models.CharField(
+ blank=True,
+ choices=[
+ ("SUCCESS", "Success"),
+ ("ERROR", "Error"),
+ ("SKIPPED", "Skipped"),
+ ],
+ db_comment="Operation outcome: SUCCESS, ERROR, or SKIPPED",
+ max_length=16,
+ null=True,
+ ),
+ ),
+ migrations.AddField(
+ model_name="usage",
+ name="error_message",
+ field=models.TextField(
+ blank=True,
+ db_comment="Error details when status is ERROR",
+ null=True,
+ ),
+ ),
+ # Indexes on project_id / prompt_id moved to 0005 so they can run
+ # CONCURRENTLY — usage is billing-critical and a plain AddIndex takes
+ # a share-update lock for the duration of the build on large tables.
+ ]
diff --git a/backend/usage_v2/migrations/0005_usage_reason_ref_created_idx.py b/backend/usage_v2/migrations/0005_usage_reason_ref_created_idx.py
new file mode 100644
index 000000000..c93d4de60
--- /dev/null
+++ b/backend/usage_v2/migrations/0005_usage_reason_ref_created_idx.py
@@ -0,0 +1,76 @@
+from django.db import migrations, models
+from django.db.models import Q
+
+
+class Migration(migrations.Migration):
+ """Build the project_id / prompt_id dashboard indexes without locking.
+
+ CONCURRENTLY requires the migration to run outside a transaction, hence
+ atomic = False. RunSQL with IF NOT EXISTS makes a partial-apply
+ (process killed between SQL success and django_migrations insert)
+ recoverable on retry without manual --fake intervention.
+ """
+
+ atomic = False
+
+ dependencies = [
+ ("usage_v2", "0004_usage_metrics_fields"),
+ ]
+
+ operations = [
+ migrations.RunSQL(
+ sql=(
+ "CREATE INDEX CONCURRENTLY IF NOT EXISTS "
+ "idx_usage_project_created "
+ 'ON "usage" (project_id, created_at DESC);'
+ ),
+ reverse_sql=("DROP INDEX CONCURRENTLY IF EXISTS idx_usage_project_created;"),
+ state_operations=[
+ migrations.AddIndex(
+ model_name="usage",
+ index=models.Index(
+ fields=["project_id", "-created_at"],
+ name="idx_usage_project_created",
+ ),
+ ),
+ ],
+ ),
+ migrations.RunSQL(
+ sql=(
+ "CREATE INDEX CONCURRENTLY IF NOT EXISTS "
+ "idx_usage_prompt_created "
+ 'ON "usage" (prompt_id, created_at DESC);'
+ ),
+ reverse_sql=("DROP INDEX CONCURRENTLY IF EXISTS idx_usage_prompt_created;"),
+ state_operations=[
+ migrations.AddIndex(
+ model_name="usage",
+ index=models.Index(
+ fields=["prompt_id", "-created_at"],
+ name="idx_usage_prompt_created",
+ ),
+ ),
+ ],
+ ),
+ # Partial — only lookup-reason rows. Avoids heap-scanning all
+ # Usage rows when the dashboard groups by (run × prompt).
+ migrations.RunSQL(
+ sql=(
+ "CREATE INDEX CONCURRENTLY IF NOT EXISTS "
+ "idx_usage_lookup_recent "
+ 'ON "usage" (organization_id, created_at DESC) '
+ "WHERE llm_usage_reason = 'lookup';"
+ ),
+ reverse_sql=("DROP INDEX CONCURRENTLY IF EXISTS idx_usage_lookup_recent;"),
+ state_operations=[
+ migrations.AddIndex(
+ model_name="usage",
+ index=models.Index(
+ fields=["organization", "-created_at"],
+ name="idx_usage_lookup_recent",
+ condition=Q(llm_usage_reason="lookup"),
+ ),
+ ),
+ ],
+ ),
+ ]
diff --git a/backend/usage_v2/models.py b/backend/usage_v2/models.py
index 57ae3d143..0267d4a68 100644
--- a/backend/usage_v2/models.py
+++ b/backend/usage_v2/models.py
@@ -1,6 +1,7 @@
import uuid
from django.db import models
+from django.db.models import Q
from utils.models.base_model import BaseModel, BaseModelManager
from utils.models.organization_mixin import (
DefaultOrganizationManagerMixin,
@@ -13,10 +14,21 @@ class UsageType(models.TextChoices):
EMBEDDING = "embedding", "Embedding Usage"
-class LLMUsageReason(models.TextChoices):
- EXTRACTION = "extraction", "Extraction"
- CHALLENGE = "challenge", "Challenge"
- SUMMARIZE = "summarize", "Summarize"
+class UsageStatus(models.TextChoices):
+ SUCCESS = "SUCCESS", "Success"
+ ERROR = "ERROR", "Error"
+ SKIPPED = "SKIPPED", "Skipped"
+
+
+# Static union of OSS + cloud values — keeps OSS model state aligned with
+# migration state so ``makemigrations --check`` doesn't drift in CI.
+
+LLM_USAGE_REASON_CHOICES: list[tuple[str, str]] = [
+ ("extraction", "Extraction"),
+ ("challenge", "Challenge"),
+ ("summarize", "Summarize"),
+ ("lookup", "Lookup"),
+]
class UsageModelManager(DefaultOrganizationManagerMixin, BaseModelManager):
@@ -52,7 +64,7 @@ class Usage(DefaultOrganizationMixin, BaseModel):
)
llm_usage_reason = models.CharField(
max_length=255,
- choices=LLMUsageReason.choices,
+ choices=LLM_USAGE_REASON_CHOICES,
null=True,
blank=True,
db_comment="Reason for LLM usage. Empty if usage_type is 'embedding'. ",
@@ -67,6 +79,39 @@ class Usage(DefaultOrganizationMixin, BaseModel):
)
total_tokens = models.IntegerField(db_comment="Total number of tokens used")
cost_in_dollars = models.FloatField(db_comment="Total number of tokens used")
+ project_id = models.UUIDField(
+ null=True,
+ blank=True,
+ db_comment=(
+ "Prompt Studio project (tool) the call belongs to (no FK; survives "
+ "tool deletion). NULL for embeddings and historical rows."
+ ),
+ )
+ prompt_id = models.UUIDField(
+ null=True,
+ blank=True,
+ db_comment=(
+ "Prompt key UUID that triggered the call (no FK; survives prompt "
+ "deletion). NULL for single-pass / embeddings / historical rows."
+ ),
+ )
+ execution_time_ms = models.IntegerField(
+ null=True,
+ blank=True,
+ db_comment="Wall-clock time for the operation in milliseconds",
+ )
+ status = models.CharField(
+ max_length=16,
+ choices=UsageStatus.choices,
+ null=True,
+ blank=True,
+ db_comment="Operation outcome: SUCCESS, ERROR, or SKIPPED",
+ )
+ error_message = models.TextField(
+ null=True,
+ blank=True,
+ db_comment="Error details when status is ERROR",
+ )
# Manager
objects = UsageModelManager()
@@ -78,4 +123,17 @@ class Usage(DefaultOrganizationMixin, BaseModel):
indexes = [
models.Index(fields=["run_id"]),
models.Index(fields=["execution_id"]),
+ models.Index(
+ fields=["project_id", "-created_at"],
+ name="idx_usage_project_created",
+ ),
+ models.Index(
+ fields=["prompt_id", "-created_at"],
+ name="idx_usage_prompt_created",
+ ),
+ models.Index(
+ fields=["organization", "-created_at"],
+ name="idx_usage_lookup_recent",
+ condition=Q(llm_usage_reason="lookup"),
+ ),
]
diff --git a/backend/usage_v2/serializers.py b/backend/usage_v2/serializers.py
index a82329708..7b7f8aaed 100644
--- a/backend/usage_v2/serializers.py
+++ b/backend/usage_v2/serializers.py
@@ -21,3 +21,38 @@ class UsageSerializer(serializers.ModelSerializer):
class Meta:
model = Usage
fields = "__all__"
+
+
+class UsageRecordCreateSerializer(serializers.Serializer):
+ """Worker-emitted usage record. Required fields anchor billing-critical attribution."""
+
+ adapter_instance_id = serializers.CharField(required=True, allow_blank=False)
+ model_name = serializers.CharField(required=True, allow_blank=False)
+ usage_type = serializers.CharField(required=True, allow_blank=False)
+
+ workflow_id = serializers.CharField(required=False, allow_blank=True, default="")
+ execution_id = serializers.CharField(required=False, allow_blank=True, default="")
+ run_id = serializers.UUIDField(required=False, allow_null=True, default=None)
+ llm_usage_reason = serializers.CharField(
+ required=False, allow_null=True, allow_blank=True, default=None
+ )
+ embedding_tokens = serializers.IntegerField(required=False, default=0)
+ prompt_tokens = serializers.IntegerField(required=False, default=0)
+ completion_tokens = serializers.IntegerField(required=False, default=0)
+ total_tokens = serializers.IntegerField(required=False, default=0)
+ cost_in_dollars = serializers.FloatField(required=False, default=0.0)
+ project_id = serializers.UUIDField(required=False, allow_null=True, default=None)
+ prompt_id = serializers.UUIDField(required=False, allow_null=True, default=None)
+ execution_time_ms = serializers.IntegerField(
+ required=False, allow_null=True, default=None
+ )
+ status = serializers.CharField(required=False, allow_null=True, default=None)
+ error_message = serializers.CharField(
+ required=False, allow_null=True, allow_blank=True, default=None
+ )
+ # Opaque carrier forwarded to post-write hooks; OSS never reads it.
+ cloud_extras = serializers.DictField(required=False, allow_null=True, default=None)
+
+
+class UsageBatchCreateSerializer(serializers.Serializer):
+ records = UsageRecordCreateSerializer(many=True, allow_empty=True)
diff --git a/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx b/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx
index 3036e7b9d..7e30a9933 100644
--- a/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx
+++ b/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx
@@ -47,6 +47,67 @@ try {
// The component will remain null if it is not available
}
+// OSS falls back to passthrough helpers — no enrichment.
+let splitCombinedData = (data) => ({ combined: data, bundle: null });
+let buildEnrichedFromBundle = (_output, _bundle, _formatter) => ({});
+let getEnrichmentFromItem = (_item) => null;
+try {
+ const mod = await import("../../../plugins/lookup-enriched-toggle/helpers");
+ splitCombinedData = mod.splitCombinedData;
+ buildEnrichedFromBundle = mod.buildEnrichedFromBundle;
+ getEnrichmentFromItem = mod.getEnrichmentFromItem;
+} catch {}
+
+const buildDefaultProfileOutputs = (data) => {
+ const { combined: payload, bundle } = splitCombinedData(data);
+ const output = Object.entries(payload).reduce((acc, [key, value]) => {
+ acc[key] = displayPromptResult(value, false);
+ return acc;
+ }, {});
+ const enriched = buildEnrichedFromBundle(output, bundle, displayPromptResult);
+ return {
+ output,
+ enriched,
+ hasEnriched: bundle != null && Object.keys(enriched).length > 0,
+ };
+};
+
+const buildPerPromptOutput = (item, data, selectedProfile) => {
+ const profileManager = selectedProfile || item?.profile_manager;
+ const outputDetails = data.find(
+ (outputValue) =>
+ outputValue?.prompt_id === item?.prompt_id &&
+ outputValue?.profile_manager === profileManager,
+ );
+ const value =
+ outputDetails?.output?.length > 0
+ ? displayPromptResult(outputDetails?.output, false)
+ : "";
+ const enrichment = getEnrichmentFromItem(outputDetails);
+ const enrichedValue = enrichment?.output
+ ? displayPromptResult(enrichment.output, false)
+ : value;
+ return { value, enrichedValue, hasEnriched: !!enrichment?.output };
+};
+
+const buildSelectedProfileOutputs = (data, prompts, selectedProfile) => {
+ const output = {};
+ const enriched = {};
+ let hasEnriched = false;
+ for (const item of prompts) {
+ if (item?.prompt_type === promptType.notes) continue;
+ const {
+ value,
+ enrichedValue,
+ hasEnriched: enrichedHit,
+ } = buildPerPromptOutput(item, data, selectedProfile);
+ output[item?.prompt_key] = value;
+ enriched[item?.prompt_key] = enrichedValue;
+ hasEnriched = hasEnriched || enrichedHit;
+ }
+ return { output, enriched: hasEnriched ? enriched : {}, hasEnriched };
+};
+
function CombinedOutput({ docId, setFilledFields, selectedPrompts }) {
const {
details,
@@ -59,6 +120,7 @@ function CombinedOutput({ docId, setFilledFields, selectedPrompts }) {
} = useCustomToolStore();
const [combinedOutput, setCombinedOutput] = useState({});
+ const [enrichedOutput, setEnrichedOutput] = useState({});
const [isOutputLoading, setIsOutputLoading] = useState(false);
const [adapterData, setAdapterData] = useState([]);
const [activeKey, setActiveKey] = useState(
@@ -115,32 +177,12 @@ function CombinedOutput({ docId, setFilledFields, selectedPrompts }) {
const res = await handleOutputApiRequest();
const data = res?.data || [];
const prompts = details?.prompts || [];
-
- if (activeKey === "0" && !isSimplePromptStudio) {
- const output = Object.entries(data).reduce((acc, [key, value]) => {
- acc[key] = displayPromptResult(value, false);
- return acc;
- }, {});
- setCombinedOutput(output);
- } else {
- const output = prompts.reduce((acc, item) => {
- if (item?.prompt_type !== promptType.notes) {
- const profileManager = selectedProfile || item?.profile_manager;
- const outputDetails = data.find(
- (outputValue) =>
- outputValue?.prompt_id === item?.prompt_id &&
- outputValue?.profile_manager === profileManager,
- );
-
- acc[item?.prompt_key] =
- outputDetails?.output?.length > 0
- ? displayPromptResult(outputDetails?.output, false)
- : "";
- }
- return acc;
- }, {});
- setCombinedOutput(output);
- }
+ const useDefaultProfile = activeKey === "0" && !isSimplePromptStudio;
+ const { output, enriched } = useDefaultProfile
+ ? buildDefaultProfileOutputs(data)
+ : buildSelectedProfileOutputs(data, prompts, selectedProfile);
+ setCombinedOutput(output);
+ setEnrichedOutput(enriched);
} catch (err) {
setAlertDetails(
handleException(err, "Failed to generate combined output"),
@@ -229,6 +271,7 @@ function CombinedOutput({ docId, setFilledFields, selectedPrompts }) {
return (
<JsonView
combinedOutput={filteredCombinedOutput}
+ enrichedOutput={enrichedOutput}
handleTabChange={handleTabChange}
selectedProfile={selectedProfile}
llmProfiles={llmProfiles}
diff --git a/frontend/src/components/custom-tools/combined-output/JsonView.jsx b/frontend/src/components/custom-tools/combined-output/JsonView.jsx
index f9d6a86a8..34a733598 100644
--- a/frontend/src/components/custom-tools/combined-output/JsonView.jsx
+++ b/frontend/src/components/custom-tools/combined-output/JsonView.jsx
@@ -2,12 +2,23 @@ import { Tabs } from "antd";
import TabPane from "antd/es/tabs/TabPane";
import Prism from "prismjs";
import PropTypes from "prop-types";
-import { useEffect } from "react";
+import { useEffect, useState } from "react";
import { JsonViewBody } from "./JsonViewBody";
+let EnrichedOutputToggle;
+try {
+ const mod = await import(
+ "../../../plugins/lookup-enriched-toggle/EnrichedOutputToggle"
+ );
+ EnrichedOutputToggle = mod.EnrichedOutputToggle;
+} catch {
+ // The component will remain undefined if it is not available
+}
+
function JsonView({
combinedOutput,
+ enrichedOutput,
handleTabChange,
adapterData,
activeKey,
@@ -16,9 +27,24 @@ function JsonView({
isSinglePass,
isLoading,
}) {
+ const [activeView, setActiveView] = useState("Raw");
+
useEffect(() => {
Prism.highlightAll();
- }, [combinedOutput]);
+ }, [combinedOutput, enrichedOutput, activeView]);
+
+ useEffect(() => {
+ if (!enrichedOutput || Object.keys(enrichedOutput).length === 0) {
+ setActiveView("Raw");
+ }
+ }, [enrichedOutput]);
+
+ const displayOutput =
+ activeView === "Enriched" &&
+ enrichedOutput &&
+ Object.keys(enrichedOutput).length > 0
+ ? enrichedOutput
+ : combinedOutput;
return (
<div className="combined-op-layout">
@@ -34,14 +60,24 @@ function JsonView({
/>
))}
</Tabs>
- <div className="combined-op-segment"></div>
+ <div className="combined-op-segment">
+ {EnrichedOutputToggle && (
+ <EnrichedOutputToggle
+ activeView={activeView}
+ onChange={setActiveView}
+ hasEnrichedData={
+ !!(enrichedOutput && Object.keys(enrichedOutput).length > 0)
+ }
+ />
+ )}
+ </div>
</div>
<div className="combined-op-divider" />
<JsonViewBody
activeKey={activeKey}
selectedProfile={selectedProfile}
llmProfiles={llmProfiles}
- combinedOutput={combinedOutput}
+ combinedOutput={displayOutput}
isLoading={isLoading}
/>
<div className="gap" />
@@ -51,6 +87,7 @@ function JsonView({
JsonView.propTypes = {
combinedOutput: PropTypes.object.isRequired,
+ enrichedOutput: PropTypes.object,
handleTabChange: PropTypes.func,
adapterData: PropTypes.array,
selectedProfile: PropTypes.string,
diff --git a/frontend/src/components/custom-tools/document-parser/DocumentParser.jsx b/frontend/src/components/custom-tools/document-parser/DocumentParser.jsx
index 3d5c891e1..63b1a037c 100644
--- a/frontend/src/components/custom-tools/document-parser/DocumentParser.jsx
+++ b/frontend/src/components/custom-tools/document-parser/DocumentParser.jsx
@@ -1,5 +1,6 @@
import PropTypes from "prop-types";
import { useEffect, useRef, useState } from "react";
+import { useSearchParams } from "react-router-dom";
import "./DocumentParser.css";
import { promptType } from "../../../helpers/GetStaticData";
@@ -32,6 +33,10 @@ try {
// The component will remain null of it is not available
}
+// Module-scoped to avoid per-render recompilation.
+const UUID_RE =
+ /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+
function DocumentParser({
addPromptInstance,
scrollToBottom,
@@ -42,6 +47,7 @@ function DocumentParser({
const [isChallenge, setIsChallenge] = useState(false);
const [allTableSettings, setAllTableSettings] = useState([]);
const bottomRef = useRef(null);
+ const [searchParams, setSearchParams] = useSearchParams();
const {
details,
isSimplePromptStudio,
@@ -108,6 +114,28 @@ function DocumentParser({
}
}, [scrollToBottom]);
+ // Cross-link from Lookup Studio: scroll to a specific prompt.
+ useEffect(() => {
+ const scrollToPromptId = searchParams.get("scrollTo");
+ if (
+ !scrollToPromptId ||
+ !UUID_RE.test(scrollToPromptId) ||
+ !details?.prompts?.length
+ ) {
+ return;
+ }
+
+ const el = document.querySelector(`[data-prompt-id="${scrollToPromptId}"]`);
+ if (el) {
+ el.scrollIntoView({ behavior: "smooth", block: "center" });
+ el.classList.add("highlighted-prompt");
+ setTimeout(() => el.classList.remove("highlighted-prompt"), 2000);
+ }
+
+ searchParams.delete("scrollTo");
+ setSearchParams(searchParams, { replace: true });
+ }, [details?.prompts, searchParams]);
+
const promptUrl = (urlPath) => {
return `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/prompt/${urlPath}`;
};
@@ -242,7 +270,7 @@ function DocumentParser({
<div className="doc-parser-layout">
{details?.prompts?.map((item) => {
return (
- <div key={item.prompt_id}>
+ <div key={item.prompt_id} data-prompt-id={item.prompt_id}>
<div className="doc-parser-pad-top" />
<PromptCardWrapper
item={item}
diff --git a/frontend/src/components/custom-tools/header/Header.jsx b/frontend/src/components/custom-tools/header/Header.jsx
index 15e41c8a0..471588a4b 100644
--- a/frontend/src/components/custom-tools/header/Header.jsx
+++ b/frontend/src/components/custom-tools/header/Header.jsx
@@ -23,30 +23,28 @@ try {
"../../../plugins/single-pass-toggle-switch/SinglePassToggleSwitch"
);
SinglePassToggleSwitch = mod.SinglePassToggleSwitch;
-} catch {
- // The variable will remain undefined if the component is not available.
-}
+} catch {}
try {
const mod = await import(
"../../../plugins/prompt-studio-public-share/public-share-btn/PromptShareButton.jsx"
);
PromptShareButton = mod.PromptShareButton;
-} catch {
- // The variable will remain undefined if the component is not available.
-}
+} catch {}
try {
const mod = await import(
"../../../plugins/prompt-studio-clone/clone-btn/CloneButton.jsx"
);
CloneButton = mod.CloneButton;
-} catch {
- // The variable will remain undefined if the component is not available.
-}
+} catch {}
+
+const noopCheckLookups = () => Promise.resolve(true);
+
function Header({
setOpenSettings,
handleUpdateTool,
setOpenShareModal,
setOpenCloneModal,
+ checkLookups = noopCheckLookups,
}) {
const [isExportLoading, setIsExportLoading] = useState(false);
const { details, isPublicSource, markChangesAsExported } =
@@ -129,7 +127,7 @@ function Header({
setConfirmModalVisible(false);
}, [lastExportParams, handleExport]);
- const handleShare = (isEdit) => {
+ const handleShare = async (isEdit) => {
try {
setPostHogCustomEvent("ps_exported_tool", {
info: `Clicked on the 'Export' button`,
@@ -139,6 +137,9 @@ function Header({
// If an error occurs while setting custom posthog event, ignore it and continue
}
+ const ok = await checkLookups(details?.tool_id, "export");
+ if (!ok) return;
+
const requestOptions = {
method: "GET",
url: `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/export/${details?.tool_id}`,
@@ -255,7 +256,7 @@ function Header({
});
};
- const handleCreateApiDeployment = () => {
+ const handleCreateApiDeployment = async () => {
try {
setPostHogCustomEvent("intent_create_api_deployment_from_prompt_studio", {
info: "Clicked Create API Deployment in tool IDE",
@@ -266,6 +267,9 @@ function Header({
// If an error occurs while setting custom posthog event, ignore it and continue
}
+ const ok = await checkLookups(details?.tool_id, "API deployment");
+ if (!ok) return;
+
// Check for existing API deployments before proceeding
setIsApiDeploymentLoading(true);
const path = `/api/v1/unstract/${sessionDetails.orgId}`;
@@ -527,6 +531,7 @@ Header.propTypes = {
handleUpdateTool: PropTypes.func.isRequired,
setOpenCloneModal: PropTypes.func.isRequired,
setOpenShareModal: PropTypes.func.isRequired,
+ checkLookups: PropTypes.func,
};
export { Header };
diff --git a/frontend/src/components/custom-tools/list-of-tools/ListOfTools.jsx b/frontend/src/components/custom-tools/list-of-tools/ListOfTools.jsx
index dd1e55853..86ab15ab1 100644
--- a/frontend/src/components/custom-tools/list-of-tools/ListOfTools.jsx
+++ b/frontend/src/components/custom-tools/list-of-tools/ListOfTools.jsx
@@ -48,7 +48,7 @@ DefaultCustomButtons.propTypes = {
handleNewProjectBtnClick: PropTypes.func.isRequired,
};
-function ListOfTools() {
+function ListOfTools({ segmentOptions, segmentValue, onSegmentChange }) {
const [isListLoading, setIsListLoading] = useState(false);
const [openAddTool, setOpenAddTool] = useState(false);
const [openImportTool, setOpenImportTool] = useState(false);
@@ -373,12 +373,15 @@ function ListOfTools() {
return (
<>
<ToolNavBar
- title={"Prompt Studio"}
+ title="Prompt Studio"
enableSearch
onSearch={onSearch}
searchList={listOfTools}
setSearchList={setFilteredListOfTools}
customButtons={customButtonsElement}
+ segmentOptions={segmentOptions}
+ segmentValue={segmentValue}
+ segmentFilter={onSegmentChange}
/>
<div className="list-of-tools-layout">
<div className="list-of-tools-island">{defaultContent}</div>
@@ -412,4 +415,10 @@ function ListOfTools() {
);
}
+ListOfTools.propTypes = {
+ segmentOptions: PropTypes.arrayOf(PropTypes.string),
+ segmentValue: PropTypes.string,
+ onSegmentChange: PropTypes.func,
+};
+
export { ListOfTools };
diff --git a/frontend/src/components/custom-tools/profile-info-bar/ProfileInfoBar.css b/frontend/src/components/custom-tools/profile-info-bar/ProfileInfoBar.css
index 8346c26d7..da866a895 100644
--- a/frontend/src/components/custom-tools/profile-info-bar/ProfileInfoBar.css
+++ b/frontend/src/components/custom-tools/profile-info-bar/ProfileInfoBar.css
@@ -1,3 +1,10 @@
.profile-info-bar {
+ display: flex;
+ flex-wrap: wrap;
+ gap: 6px 8px;
margin-bottom: 10px;
}
+
+.profile-info-bar .ant-tag {
+ margin: 0;
+}
diff --git a/frontend/src/components/custom-tools/profile-info-bar/ProfileInfoBar.jsx b/frontend/src/components/custom-tools/profile-info-bar/ProfileInfoBar.jsx
index 8f52a68a4..934175398 100644
--- a/frontend/src/components/custom-tools/profile-info-bar/ProfileInfoBar.jsx
+++ b/frontend/src/components/custom-tools/profile-info-bar/ProfileInfoBar.jsx
@@ -1,4 +1,4 @@
-import { Col, Row, Tag } from "antd";
+import { Tag } from "antd";
import PropTypes from "prop-types";
import "./ProfileInfoBar.css";
@@ -10,43 +10,29 @@ const ProfileInfoBar = ({ profiles, profileId }) => {
}
return (
- <Row className="profile-info-bar">
- <Col>
- <Tag>
- <strong>Profile Name:</strong> {profile?.profile_name}
- </Tag>
- </Col>
- <Col>
- <Tag>
- <strong>Chunk Size:</strong> {profile?.chunk_size}
- </Tag>
- </Col>
- <Col>
- <Tag>
- <strong>Vector Store:</strong> {profile?.vector_store}
- </Tag>
- </Col>
- <Col>
- <Tag>
- <strong>Embedding Model:</strong> {profile?.embedding_model}
- </Tag>
- </Col>
- <Col>
- <Tag>
- <strong>LLM:</strong> {profile?.llm}
- </Tag>
- </Col>
- <Col>
- <Tag>
- <strong>X2Text:</strong> {profile?.x2text}
- </Tag>
- </Col>
- <Col>
- <Tag>
- <strong>Reindex:</strong> {profile?.reindex ? "Yes" : "No"}
- </Tag>
- </Col>
- </Row>
+ <div className="profile-info-bar">
+ <Tag>
+ <strong>Profile Name:</strong> {profile?.profile_name}
+ </Tag>
+ <Tag>
+ <strong>Chunk Size:</strong> {profile?.chunk_size}
+ </Tag>
+ <Tag>
+ <strong>Vector Store:</strong> {profile?.vector_store}
+ </Tag>
+ <Tag>
+ <strong>Embedding Model:</strong> {profile?.embedding_model}
+ </Tag>
+ <Tag>
+ <strong>LLM:</strong> {profile?.llm}
+ </Tag>
+ <Tag>
+ <strong>X2Text:</strong> {profile?.x2text}
+ </Tag>
+ <Tag>
+ <strong>Reindex:</strong> {profile?.reindex ? "Yes" : "No"}
+ </Tag>
+ </div>
);
};
diff --git a/frontend/src/components/custom-tools/prompt-card/Header.jsx b/frontend/src/components/custom-tools/prompt-card/Header.jsx
index cea5e2d6d..30fe75fe5 100644
--- a/frontend/src/components/custom-tools/prompt-card/Header.jsx
+++ b/frontend/src/components/custom-tools/prompt-card/Header.jsx
@@ -41,6 +41,38 @@ try {
// The component will remain 'undefined' it is not available
}
+let LookupMenuItem;
+try {
+ const mod = await import(
+ "../../../plugins/lookup-studio/prompt-card/LookupMenuItem"
+ );
+ LookupMenuItem = mod.LookupMenuItem;
+} catch {}
+
+let usePromptRunGatePlugin;
+let lookupPluginLoadError;
+try {
+ const mod = await import(
+ "../../../plugins/lookup-studio/hooks/usePromptRunGate"
+ );
+ usePromptRunGatePlugin = mod.usePromptRunGate;
+} catch (err) {
+ lookupPluginLoadError = err;
+}
+
+// Sibling plugin loaded but this hook didn't — surface so the no-op fallback
+// doesn't silently disable the run gate.
+if (lookupPluginLoadError && LookupMenuItem) {
+ // eslint-disable-next-line no-console
+ console.warn(
+ "[Header] lookup-studio plugin loaded but usePromptRunGate failed to import",
+ lookupPluginLoadError,
+ );
+}
+
+// Stable identity — avoid conditional hook call.
+const usePromptRunGate = usePromptRunGatePlugin || (() => null);
+
function Header({
promptDetails,
promptKey,
@@ -70,6 +102,7 @@ function Header({
isSimplePromptStudio,
details,
} = useCustomToolStore();
+ const runGate = usePromptRunGate(promptDetails);
const [items, setItems] = useState([]);
const [isDisablePrompt, setIsDisablePrompt] = useState(null);
@@ -268,6 +301,20 @@ function Header({
isPublicSource,
},
];
+ if (LookupMenuItem && !isSimplePromptStudio) {
+ dropdownItems.splice(
+ -1,
+ 0,
+ {
+ type: "divider",
+ },
+ {
+ label: <LookupMenuItem promptDetails={promptDetails} />,
+ key: "lookup",
+ },
+ );
+ }
+
if (isSimplePromptStudio) {
dropdownItems.splice(0, 1);
}
@@ -346,7 +393,9 @@ function Header({
)}
{!singlePassExtractMode && !isSimplePromptStudio && (
<>
- <Tooltip title="Run all LLMs for current document">
+ <Tooltip
+ title={runGate?.reason || "Run all LLMs for current document"}
+ >
<Button
size="small"
type="text"
@@ -365,13 +414,16 @@ function Header({
isCoverageLoading ||
indexDocs?.includes(selectedDoc?.document_id) ||
isPublicSource ||
- spsLoading?.[selectedDoc?.document_id]
+ spsLoading?.[selectedDoc?.document_id] ||
+ !!runGate?.disabled
}
>
<PlayCircleOutlined className="prompt-card-actions-head" />
</Button>
</Tooltip>
- <Tooltip title="Run all LLMs for all documents">
+ <Tooltip
+ title={runGate?.reason || "Run all LLMs for all documents"}
+ >
<Button
size="small"
type="text"
@@ -388,7 +440,8 @@ function Header({
promptStudioUpdateStatus?.isUpdating) ||
isCoverageLoading ||
indexDocs?.includes(selectedDoc?.document_id) ||
- isPublicSource
+ isPublicSource ||
+ !!runGate?.disabled
}
>
<PlayCircleFilled className="prompt-card-actions-head" />
diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx
index 2e579108c..29fda260e 100644
--- a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx
+++ b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx
@@ -14,6 +14,19 @@ import { PromptCardItems } from "./PromptCardItems";
import "./PromptCard.css";
import { handleUpdateStatus } from "./constants";
+let useEnforceTypeSwitchGatePlugin;
+try {
+ const mod = await import(
+ "../../../plugins/lookup-studio/hooks/useEnforceTypeSwitchGate"
+ );
+ useEnforceTypeSwitchGatePlugin = mod.useEnforceTypeSwitchGate;
+} catch {
+ // Cloud plugin not present; gate falls back to no-op below.
+}
+
+const useEnforceTypeSwitchGate =
+ useEnforceTypeSwitchGatePlugin || (() => () => null);
+
const PromptCard = memo(
({
promptDetails,
@@ -56,6 +69,7 @@ const PromptCard = memo(
const { messages } = useSocketCustomToolStore();
const { setAlertDetails } = useAlertStore();
const { setPostHogCustomEvent } = usePostHogEvents();
+ const enforceTypeSwitchGate = useEnforceTypeSwitchGate();
useEffect(() => {
if (
@@ -260,6 +274,11 @@ const PromptCard = memo(
};
const handleTypeChange = (value) => {
+ const block = enforceTypeSwitchGate(promptDetailsState?.prompt_id, value);
+ if (block) {
+ setAlertDetails({ type: "error", content: block.reason });
+ return;
+ }
handleChange(value, promptDetailsState?.prompt_id, "enforce_type", true);
};
diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx b/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx
index 6fd8b1bc6..5cb5d099b 100644
--- a/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx
+++ b/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx
@@ -29,6 +29,14 @@ try {
// The component will remain null of it is not available
}
+let LookupIndicator;
+try {
+ const mod = await import(
+ "../../../plugins/lookup-studio/prompt-card/LookupIndicator"
+ );
+ LookupIndicator = mod.LookupIndicator;
+} catch {}
+
let AgenticTableChecklist;
try {
const mod = await import(
@@ -279,6 +287,9 @@ function PromptCardItems({
</Typography.Link>
</Space>
</Button>
+ {LookupIndicator && (
+ <LookupIndicator promptDetails={promptDetails} />
+ )}
</Space>
<Space>
{details?.enable_highlight &&
diff --git a/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx b/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx
index 38f7d924a..b8aaa4b39 100644
--- a/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx
+++ b/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx
@@ -51,6 +51,49 @@ try {
// The component will remain null of it is not available
}
+let LookupOutputTabs;
+try {
+ const mod = await import(
+ "../../../plugins/lookup-studio/prompt-card/LookupOutputTabs"
+ );
+ LookupOutputTabs = mod.LookupOutputTabs;
+} catch {
+ // Cloud-only plugin — absent in OSS builds; LookupOutputTabs stays null.
+}
+
+let getEnrichedCopyText;
+try {
+ const mod = await import(
+ "../../../plugins/lookup-studio/prompt-card/getEnrichedCopyText"
+ );
+ getEnrichedCopyText = mod.getEnrichedCopyText;
+} catch {
+ // Cloud-only plugin — absent in OSS builds; falls back to raw copy text.
+}
+
+// Fallback to raw text — plugin throw on malformed enrichment shouldn't break Copy.
+const resolveCopyText = (promptOutputId, fallbackText) => {
+ if (!getEnrichedCopyText) {
+ return fallbackText;
+ }
+ try {
+ const enriched = getEnrichedCopyText(promptOutputId);
+ return typeof enriched === "string" && enriched ? enriched : fallbackText;
+ } catch (err) {
+ // eslint-disable-next-line no-console
+ console.warn("[PromptOutput] getEnrichedCopyText threw:", err);
+ return fallbackText;
+ }
+};
+
+// Cloud wraps children in LookupOutputTabs; OSS passes through.
+const renderWithLookupWrapper = (lookupProps, children) =>
+ LookupOutputTabs ? (
+ <LookupOutputTabs {...lookupProps}>{children}</LookupOutputTabs>
+ ) : (
+ children
+ );
+
function PromptOutput({
promptDetails,
handleRun,
@@ -194,32 +237,43 @@ function PromptOutput({
"highlighted-prompt-cell"
}`}
>
- <DisplayPromptResult
- output={promptOutput}
- highlightData={
- promptOutputData?.highlightData?.[promptDetails.prompt_key]
- }
- handleSelectHighlight={handleSelectHighlight}
- confidenceData={
- promptOutputData?.confidenceData?.[promptDetails.prompt_key]
- }
- wordConfidenceData={
- promptOutputData?.wordConfidenceData?.[promptDetails.prompt_key]
- }
- progressMsg={progressMsg}
- />
+ {renderWithLookupWrapper(
+ {
+ promptId,
+ profileManagerId: defaultLlmProfile,
+ defaultLlmProfile,
+ promptOutputId: promptOutputData?.promptOutputId,
+ },
+ <DisplayPromptResult
+ output={promptOutput}
+ highlightData={
+ promptOutputData?.highlightData?.[promptDetails.prompt_key]
+ }
+ handleSelectHighlight={handleSelectHighlight}
+ confidenceData={
+ promptOutputData?.confidenceData?.[promptDetails.prompt_key]
+ }
+ wordConfidenceData={
+ promptOutputData?.wordConfidenceData?.[promptDetails.prompt_key]
+ }
+ progressMsg={progressMsg}
+ />,
+ )}
<div className="prompt-profile-run">
<CopyPromptOutputBtn
isDisabled={isTableExtraction}
- copyToClipboard={() =>
+ copyToClipboard={() => {
copyOutputToClipboard(
- displayPromptResult(
- promptOutput,
- true,
- promptDetails?.enable_highlight,
+ resolveCopyText(
+ promptOutputData?.promptOutputId,
+ displayPromptResult(
+ promptOutput,
+ true,
+ promptDetails?.enable_highlight,
+ ),
),
- )
- }
+ );
+ }}
/>
<PromptOutputExpandBtn
promptId={promptDetails?.prompt_id}
@@ -438,31 +492,42 @@ function PromptOutput({
handleTable(profileId, promptOutputData)
) : (
<>
- <DisplayPromptResult
- output={promptOutputData?.output}
- profileId={profileId}
- docId={selectedDoc?.document_id}
- promptRunStatus={promptRunStatus}
- handleSelectHighlight={handleSelectHighlight}
- highlightData={promptOutputData?.highlightData}
- confidenceData={promptOutputData?.confidenceData}
- wordConfidenceData={
- promptOutputData?.wordConfidenceData
- }
- promptDetails={promptDetails}
- progressMsg={progressMsg}
- />
+ {renderWithLookupWrapper(
+ {
+ promptId,
+ profileManagerId: profileId,
+ defaultLlmProfile,
+ promptOutputId: promptOutputData?.promptOutputId,
+ },
+ <DisplayPromptResult
+ output={promptOutputData?.output}
+ profileId={profileId}
+ docId={selectedDoc?.document_id}
+ promptRunStatus={promptRunStatus}
+ handleSelectHighlight={handleSelectHighlight}
+ highlightData={promptOutputData?.highlightData}
+ confidenceData={promptOutputData?.confidenceData}
+ wordConfidenceData={
+ promptOutputData?.wordConfidenceData
+ }
+ promptDetails={promptDetails}
+ progressMsg={progressMsg}
+ />,
+ )}
<div className="prompt-profile-run">
<CopyPromptOutputBtn
isDisabled={isTableExtraction}
- copyToClipboard={() =>
+ copyToClipboard={() => {
copyOutputToClipboard(
- displayPromptResult(
- promptOutputData?.output,
- true,
+ resolveCopyText(
+ promptOutputData?.promptOutputId,
+ displayPromptResult(
+ promptOutputData?.output,
+ true,
+ ),
),
- )
- }
+ );
+ }}
/>
</div>
</>
diff --git a/frontend/src/components/custom-tools/token-usage/TokenUsage.jsx b/frontend/src/components/custom-tools/token-usage/TokenUsage.jsx
index 846c11f0a..3c52a4005 100644
--- a/frontend/src/components/custom-tools/token-usage/TokenUsage.jsx
+++ b/frontend/src/components/custom-tools/token-usage/TokenUsage.jsx
@@ -23,8 +23,13 @@ function TokenUsage({ tokenUsageId, isLoading }) {
setTokens(tokenUsage[tokenUsageId]); // Update tokens state with the token usage data for the given tokenUsageId
}, [tokenUsage, tokenUsageId]);
- // If no tokens data is available, render nothing
- if (!tokens || !Object.keys(tokens)?.length || isLoading) {
+ // BE returns all-null fields when no Usage rows match; treat as empty.
+ if (
+ !tokens ||
+ !Object.keys(tokens)?.length ||
+ isLoading ||
+ tokens?.total_tokens == null
+ ) {
return "NA";
}
diff --git a/frontend/src/components/custom-tools/tool-ide/ToolIde.jsx b/frontend/src/components/custom-tools/tool-ide/ToolIde.jsx
index 93c819859..1f7e21c25 100644
--- a/frontend/src/components/custom-tools/tool-ide/ToolIde.jsx
+++ b/frontend/src/components/custom-tools/tool-ide/ToolIde.jsx
@@ -44,6 +44,28 @@ try {
} catch {
// Do nothing if plugins are not loaded.
}
+
+// Cloud-only — OSS stub.
+let useLookupDirtySeed = () => {};
+try {
+ const mod = await import(
+ "../../../plugins/lookup-studio/hooks/useLookupDirtySeed.js"
+ );
+ useLookupDirtySeed = mod.useLookupDirtySeed;
+} catch {}
+
+// Cloud-only — OSS stub resolves true to skip the gate.
+let useLookupExportGate = () => ({
+ checkLookups: () => Promise.resolve(true),
+ modalEl: null,
+});
+try {
+ const mod = await import(
+ "../../../plugins/lookup-studio/hooks/useLookupExportGate"
+ );
+ useLookupExportGate = mod.useLookupExportGate;
+} catch {}
+
function ToolIde() {
const [openSettings, setOpenSettings] = useState(false);
const customToolStore = useCustomToolStore();
@@ -76,6 +98,7 @@ function ToolIde() {
const isCheckingUsageRef = useRef(false);
const hasCheckedForCurrentSessionRef = useRef(false);
const abortControllerRef = useRef(null);
+ const { checkLookups, modalEl: lookupGateModalEl } = useLookupExportGate();
useEffect(() => {
if (openShareModal) {
@@ -178,6 +201,9 @@ function ToolIde() {
}
}, [details?.tool_id]);
+ // Surfaces re-export banner when /lookups page edits made the tool stale.
+ useLookupDirtySeed(details?.tool_id);
+
// Cleanup abort controller on unmount
useEffect(() => {
return () => {
@@ -189,6 +215,10 @@ function ToolIde() {
// Handle export from reminder bar
const handleExportFromReminder = useCallback(async () => {
+ const ok = await checkLookups(details?.tool_id, "export");
+ if (!ok) {
+ return;
+ }
setIsExporting(true);
try {
const requestOptions = {
@@ -237,6 +267,7 @@ function ToolIde() {
handleException,
markChangesAsExported,
setPostHogCustomEvent,
+ checkLookups,
]);
const generateIndex = async (doc) => {
@@ -340,11 +371,13 @@ function ToolIde() {
isExporting={isExporting}
/>
)}
+ {lookupGateModalEl}
<Header
handleUpdateTool={handleUpdateTool}
setOpenSettings={setOpenSettings}
setOpenShareModal={setOpenShareModal}
setOpenCloneModal={setOpenCloneModal}
+ checkLookups={checkLookups}
/>
<div
className={isPublicSource ? "public-tool-ide-body" : "tool-ide-body"}
diff --git a/frontend/src/components/helpers/custom-markdown/CustomMarkdown.jsx b/frontend/src/components/helpers/custom-markdown/CustomMarkdown.jsx
index 458f31955..177299a0f 100644
--- a/frontend/src/components/helpers/custom-markdown/CustomMarkdown.jsx
+++ b/frontend/src/components/helpers/custom-markdown/CustomMarkdown.jsx
@@ -55,8 +55,7 @@ const CustomMarkdown = ({
</Text>
);
case "link": {
- // Protocol-relative URLs (`//evil.com/...`) also start with `/`
- // so exclude them from the internal-route branch.
+ // Exclude protocol-relative `//evil.com` from internal-route branch.
const isInternal = url?.startsWith("/") && !url.startsWith("//");
if (isInternal) {
const resolvedUrl = orgName ? `/${orgName}${url}` : url;
diff --git a/frontend/src/components/helpers/custom-tools/CustomToolsHelper.js b/frontend/src/components/helpers/custom-tools/CustomToolsHelper.js
index 543ad9b14..35c2854cb 100644
--- a/frontend/src/components/helpers/custom-tools/CustomToolsHelper.js
+++ b/frontend/src/components/helpers/custom-tools/CustomToolsHelper.js
@@ -19,6 +19,18 @@ try {
} catch {
// Do nothing, Not-found Page will be triggered.
}
+
+let fetchLookupAssignments;
+let fetchLookupOutputs;
+let resetLookupAssignments;
+try {
+ const mod = await import(
+ "../../../plugins/lookup-studio/store/useFetchLookupAssignments"
+ );
+ fetchLookupAssignments = mod.fetchLookupAssignments;
+ fetchLookupOutputs = mod.fetchLookupOutputs;
+ resetLookupAssignments = mod.resetLookupAssignments;
+} catch {}
function CustomToolsHelper() {
const [isLoading, setIsLoading] = useState(true);
const { id } = useParams();
@@ -115,6 +127,14 @@ function CustomToolsHelper() {
.then((res) => {
const data = res?.data;
updatedCusTool["adapters"] = data;
+
+ if (fetchLookupAssignments) {
+ const toolId = updatedCusTool["details"]?.tool_id;
+ fetchLookupAssignments(axiosPrivate, sessionDetails?.orgId, toolId);
+ if (fetchLookupOutputs) {
+ fetchLookupOutputs(axiosPrivate, sessionDetails?.orgId, toolId);
+ }
+ }
})
.catch((err) => {
setAlertDetails(handleException(err, "Failed to load the custom tool"));
@@ -131,6 +151,9 @@ function CustomToolsHelper() {
setDefaultCustomTool();
emptyCusToolMessages();
resetTokenUsage();
+ if (resetLookupAssignments) {
+ resetLookupAssignments();
+ }
};
}, []);
diff --git a/frontend/src/components/logging/execution-logs/ExecutionLogs.jsx b/frontend/src/components/logging/execution-logs/ExecutionLogs.jsx
index a9c72a0ef..8e44287e2 100644
--- a/frontend/src/components/logging/execution-logs/ExecutionLogs.jsx
+++ b/frontend/src/components/logging/execution-logs/ExecutionLogs.jsx
@@ -54,14 +54,14 @@ function ExecutionLogs() {
? location.state?.from || `/${sessionDetails?.orgName}/logs`
: null;
- // State to pass back for scroll restoration
+ // Scroll-restoration wins; otherwise preserve caller's upstream UI state.
const backRouteState =
id && location.state?.scrollToCardId
? {
scrollToCardId: location.state.scrollToCardId,
cardExpanded: location.state.cardExpanded,
}
- : null;
+ : location.state?.backRouteState || null;
const items = [
{
diff --git a/frontend/src/components/navigations/side-nav-bar/SideNavBar.jsx b/frontend/src/components/navigations/side-nav-bar/SideNavBar.jsx
index bb328b1a9..164fdab14 100644
--- a/frontend/src/components/navigations/side-nav-bar/SideNavBar.jsx
+++ b/frontend/src/components/navigations/side-nav-bar/SideNavBar.jsx
@@ -89,6 +89,12 @@ try {
// Plugin unavailable
}
+let lookupStudioEnabled = false;
+try {
+ await import("../../../plugins/lookup-studio");
+ lookupStudioEnabled = true;
+} catch {}
+
let manualReviewSettingsEnabled = false;
try {
await import("../../../plugins/manual-review/settings/Settings.jsx");
@@ -506,6 +512,19 @@ const SideNavBar = ({ collapsed, setCollapsed }) => {
});
}
+ // Keep Prompt Studio highlighted on /lookups. Replace, don't mutate —
+ // `data` may alias the `menu` prop.
+ if (lookupStudioEnabled && isUnstract && data[0]?.subMenu) {
+ const onLookupPath = globalThis.location.pathname.startsWith(
+ `/${orgName}/lookups`,
+ );
+ if (onLookupPath) {
+ data[0].subMenu = data[0].subMenu.map((el) =>
+ el.id === 1.1 ? { ...el, active: true } : el,
+ );
+ }
+ }
+
// Add HITL Review section if plugin is available and user has HITL role
const isHITLRole = [
"unstract_reviewer",
diff --git a/frontend/src/components/navigations/tool-nav-bar/ToolNavBar.jsx b/frontend/src/components/navigations/tool-nav-bar/ToolNavBar.jsx
index 999630d59..e39c50ff1 100644
--- a/frontend/src/components/navigations/tool-nav-bar/ToolNavBar.jsx
+++ b/frontend/src/components/navigations/tool-nav-bar/ToolNavBar.jsx
@@ -9,6 +9,7 @@ import "./ToolNavBar.css";
function ToolNavBar({
title,
+ titleAdornment,
subtitle,
onEditTitle,
enableSearch,
@@ -19,6 +20,7 @@ function ToolNavBar({
onNavigateBack,
segmentFilter,
segmentOptions,
+ segmentValue,
onSearch,
searchKey,
}) {
@@ -53,6 +55,7 @@ function ToolNavBar({
<Typography.Text strong className="tool-nav-bar__title">
{title}
</Typography.Text>
+ {titleAdornment}
{onEditTitle && (
<Button
type="text"
@@ -79,6 +82,7 @@ function ToolNavBar({
{segmentFilter && segmentOptions && (
<Segmented
options={segmentOptions}
+ value={segmentValue}
onChange={segmentFilter}
className="tool-nav-bar__segment"
/>
@@ -102,6 +106,7 @@ function ToolNavBar({
ToolNavBar.propTypes = {
title: PropTypes.string,
+ titleAdornment: PropTypes.node,
subtitle: PropTypes.string,
onEditTitle: PropTypes.func,
enableSearch: PropTypes.bool,
@@ -111,6 +116,7 @@ ToolNavBar.propTypes = {
previousRouteState: PropTypes.object,
onNavigateBack: PropTypes.func,
segmentOptions: PropTypes.array,
+ segmentValue: PropTypes.string,
segmentFilter: PropTypes.func,
onSearch: PropTypes.func,
searchKey: PropTypes.string,
diff --git a/frontend/src/components/widgets/empty-state/EmptyState.jsx b/frontend/src/components/widgets/empty-state/EmptyState.jsx
index d221c2544..0c9c8537d 100644
--- a/frontend/src/components/widgets/empty-state/EmptyState.jsx
+++ b/frontend/src/components/widgets/empty-state/EmptyState.jsx
@@ -27,7 +27,7 @@ function EmptyState({ text, btnText, handleClick }) {
}
EmptyState.propTypes = {
- text: PropTypes.string.isRequired,
+ text: PropTypes.node.isRequired,
btnText: PropTypes.string,
handleClick: PropTypes.func,
};
diff --git a/frontend/src/helpers/urlSafety.js b/frontend/src/helpers/urlSafety.js
index b5ca23e79..fede217e2 100644
--- a/frontend/src/helpers/urlSafety.js
+++ b/frontend/src/helpers/urlSafety.js
@@ -1,13 +1,12 @@
const SAFE_URL_SCHEMES = ["http:", "https:", "mailto:", "tel:"];
-// Guards against unsafe schemes (e.g. `javascript:`, `data:`) when
-// rendering links built from user- or tool-derived content.
+// Block unsafe schemes (e.g. `javascript:`, `data:`) in user/tool-derived links.
const isSafeExternalUrl = (url) => {
if (typeof url !== "string" || url === "") {
return false;
}
- // Parse without a base so bare strings (e.g. "javascript", "../foo") fail
- // instead of silently resolving to `https://<origin>/...` and passing.
+ // No base URL — bare strings ("javascript", "../foo") must fail rather than
+ // silently resolve to the current origin.
try {
const parsed = new URL(url);
return SAFE_URL_SCHEMES.includes(parsed.protocol);
diff --git a/frontend/src/hooks/usePromptOutput.js b/frontend/src/hooks/usePromptOutput.js
index 530cdf2bc..c27a0dbf5 100644
--- a/frontend/src/hooks/usePromptOutput.js
+++ b/frontend/src/hooks/usePromptOutput.js
@@ -23,6 +23,28 @@ try {
// The component will remain null of it is not available
}
+let handleLookupOutput;
+try {
+ const mod = await import(
+ "../plugins/lookup-studio/prompt-card/handleLookupOutput"
+ );
+ handleLookupOutput = mod.handleLookupOutput;
+} catch (error) {
+ // Surface chunk-load failures — silent catch hid them.
+ // eslint-disable-next-line no-console
+ console.warn("[usePromptOutput] handleLookupOutput unavailable:", error);
+}
+
+// Cloud-only extractor; OSS no-op. Signature matches plugin helper.
+let getEnrichmentFromItem = (_item) => null;
+try {
+ const mod = await import("../plugins/lookup-enriched-toggle/helpers");
+ getEnrichmentFromItem = mod.getEnrichmentFromItem;
+} catch (error) {
+ // eslint-disable-next-line no-console
+ console.warn("[usePromptOutput] getEnrichmentFromItem unavailable:", error);
+}
+
const usePromptOutput = () => {
const { sessionDetails } = useSessionStore();
const { setTokenUsage, updateTokenUsage } = useTokenUsageStore();
@@ -125,6 +147,18 @@ const usePromptOutput = () => {
wordConfidenceData: item?.word_confidence_data,
};
+ // Per-item plugin failure must not abort the forEach — would leave
+ // partial state with no error surfaced.
+ try {
+ const enrichment = getEnrichmentFromItem(item);
+ if (handleLookupOutput && enrichment) {
+ handleLookupOutput(item.prompt_output_id, enrichment);
+ }
+ } catch (err) {
+ // eslint-disable-next-line no-console
+ console.warn("[usePromptOutput] lookup enrichment failed:", err);
+ }
+
if (item?.is_single_pass_extract && isTokenUsageForSinglePassAdded) {
return;
}
diff --git a/frontend/src/hooks/usePromptRun.js b/frontend/src/hooks/usePromptRun.js
index 34fb42e7d..eb97502d0 100644
--- a/frontend/src/hooks/usePromptRun.js
+++ b/frontend/src/hooks/usePromptRun.js
@@ -16,10 +16,7 @@ import usePromptOutput from "./usePromptOutput";
// Tracks the latest run nonce per (promptId, statusKey) so stale timeouts
// from a previous run don't falsely cancel a newer run of the same combo.
const runNonceMap = new Map();
-// Safety-net timeout for the fire-and-forget prompt run flow. Must trail the
-// server-side LLM adapter timeout (default 900s in
-// unstract/sdk1/src/unstract/sdk1/adapters/llm1/static/openai.json) so the UI
-// never gives up before the backend does. 16 minutes leaves a 1-minute buffer.
+// Must exceed server LLM adapter timeout (default 900s) so UI doesn't bail first.
const SOCKET_TIMEOUT_MS = 16 * 60 * 1000; // 16 minutes
const usePromptRun = () => {
diff --git a/frontend/src/pages/CustomTools.jsx b/frontend/src/pages/CustomTools.jsx
index bd177301c..0864d9d26 100644
--- a/frontend/src/pages/CustomTools.jsx
+++ b/frontend/src/pages/CustomTools.jsx
@@ -1,7 +1,44 @@
+import { useEffect, useState } from "react";
+import { useLocation } from "react-router-dom";
+
import { ListOfTools } from "../components/custom-tools/list-of-tools/ListOfTools";
+const TAB_OPTIONS = ["Projects", "Look-Ups"];
+
function CustomTools() {
- return <ListOfTools />;
+ const location = useLocation();
+ const [LookupListComp, setLookupListComp] = useState(null);
+ const [activeTab, setActiveTab] = useState(
+ location.state?.activeTab || "Projects",
+ );
+
+ useEffect(() => {
+ import("../plugins/lookup-studio")
+ .then((mod) => setLookupListComp(() => mod.LookupList))
+ .catch(() => {});
+ }, []);
+
+ useEffect(() => {
+ if (location.state?.activeTab) {
+ setActiveTab(location.state.activeTab);
+ }
+ }, [location.state?.activeTab]);
+
+ if (!LookupListComp) {
+ return <ListOfTools />;
+ }
+
+ const tabProps = {
+ segmentOptions: TAB_OPTIONS,
+ segmentValue: activeTab,
+ onSegmentChange: setActiveTab,
+ };
+
+ return activeTab === "Projects" ? (
+ <ListOfTools {...tabProps} />
+ ) : (
+ <LookupListComp {...tabProps} />
+ );
}
export { CustomTools };
diff --git a/frontend/src/routes/useMainAppRoutes.js b/frontend/src/routes/useMainAppRoutes.js
index 1d5849b10..768dfee97 100644
--- a/frontend/src/routes/useMainAppRoutes.js
+++ b/frontend/src/routes/useMainAppRoutes.js
@@ -61,6 +61,14 @@ try {
// Do nothing, Not-found Page will be triggered.
}
+let LookupStudio;
+try {
+ const mod = await import("../plugins/lookup-studio");
+ LookupStudio = mod.LookupStudio;
+} catch {
+ // Do nothing, Not-found Page will be triggered.
+}
+
try {
const mod1 = await import("../plugins/app-deployment/AppDeployments.jsx");
AppDeployments = mod1.AppDeployments;
@@ -180,6 +188,7 @@ function useMainAppRoutes() {
element={<AgenticPromptStudio />}
/>
)}
+ {LookupStudio && <Route path="lookups/*" element={<LookupStudio />} />}
<Route path="logs" element={<LogsPage />} />
<Route path="logs/:type/:id/" element={<LogsPage />} />
<Route
diff --git a/unstract/sdk1/src/unstract/sdk1/execution/context.py b/unstract/sdk1/src/unstract/sdk1/execution/context.py
index 83eb650a2..0edb3e40b 100644
--- a/unstract/sdk1/src/unstract/sdk1/execution/context.py
+++ b/unstract/sdk1/src/unstract/sdk1/execution/context.py
@@ -96,8 +96,7 @@ class ExecutionContext:
raise ValueError("run_id is required")
if not self.execution_source:
raise ValueError("execution_source is required")
- # When execution_id is set, organization_id must be too — they
- # travel together for workflow-level log correlation.
+ # Workflow-level log correlation needs both fields together.
if self.execution_id and not self.organization_id:
raise ValueError("organization_id is required when execution_id is set")
diff --git a/unstract/sdk1/src/unstract/sdk1/execution/orchestrator.py b/unstract/sdk1/src/unstract/sdk1/execution/orchestrator.py
index 2c0f66f3b..0f1454a32 100644
--- a/unstract/sdk1/src/unstract/sdk1/execution/orchestrator.py
+++ b/unstract/sdk1/src/unstract/sdk1/execution/orchestrator.py
@@ -9,6 +9,7 @@ that unhandled exceptions are always wrapped in a failed
import logging
import time
+from typing import Any
from unstract.sdk1.execution.context import ExecutionContext
from unstract.sdk1.execution.registry import ExecutorRegistry
@@ -37,8 +38,7 @@ class ExecutionOrchestrator:
exceptions (wrapped as a failure result).
"""
logger.info(
- "Orchestrating execution: executor=%s operation=%s "
- "run_id=%s request_id=%s",
+ "Orchestrating execution: executor=%s operation=%s run_id=%s request_id=%s",
context.executor_name,
context.operation,
context.run_id,
@@ -61,9 +61,14 @@ class ExecutionOrchestrator:
context.executor_name,
elapsed,
)
+ metadata: dict[str, Any] = {"elapsed_seconds": round(elapsed, 3)}
+ # Don't lose pre-crash billing rows on retry.
+ collected = getattr(executor, "_usage_records", None) or []
+ if collected:
+ metadata["usage_records"] = list(collected)
return ExecutionResult.failure(
error=f"{type(exc).__name__}: {exc}",
- metadata={"elapsed_seconds": round(elapsed, 3)},
+ metadata=metadata,
)
elapsed = time.monotonic() - start
diff --git a/unstract/sdk1/src/unstract/sdk1/llm.py b/unstract/sdk1/src/unstract/sdk1/llm.py
index 4588e6946..14bc4b33c 100644
--- a/unstract/sdk1/src/unstract/sdk1/llm.py
+++ b/unstract/sdk1/src/unstract/sdk1/llm.py
@@ -9,11 +9,10 @@ from typing import Any, NoReturn, cast
import litellm
# from litellm import get_supported_openai_params
-from litellm import get_max_tokens, token_counter
+from litellm import get_max_tokens
from pydantic import ValidationError
from unstract.sdk1.adapters.constants import Common
from unstract.sdk1.adapters.llm1 import adapters
-from unstract.sdk1.audit import Audit
from unstract.sdk1.constants import Common as SdkCommon
from unstract.sdk1.constants import ToolEnv
from unstract.sdk1.exceptions import LLMError, SdkError, strip_litellm_prefix
@@ -21,7 +20,6 @@ from unstract.sdk1.platform import PlatformHelper
from unstract.sdk1.tool.base import BaseTool
from unstract.sdk1.utils.common import (
LLMResponseCompat,
- TokenCounterCompat,
capture_metrics,
)
from unstract.sdk1.utils.retry_utils import (
@@ -211,6 +209,7 @@ class LLM:
if capture_metrics_from_platform is not None:
self._capture_metrics = capture_metrics_from_platform
self._metrics: dict[str, object] = {}
+ self._pending_usage: list[dict] = []
def _get_adapter_info(self) -> str:
"""Build a display string identifying this adapter for errors."""
@@ -642,9 +641,39 @@ class LLM:
def get_metrics(self) -> dict[str, object]:
return self._metrics
+ def get_last_usage(self) -> Mapping[str, int]:
+ """Token usage from the most recent LLM call (sync, async, or streaming)."""
+ if not self._pending_usage:
+ return {}
+ last = self._pending_usage[-1]
+ return {
+ "prompt_tokens": last["prompt_tokens"],
+ "completion_tokens": last["completion_tokens"],
+ "total_tokens": last["total_tokens"],
+ }
+
+ def get_last_usage_record(self) -> dict | None:
+ """Full usage record for the most recent LLM call.
+
+ Returns tokens + cost + model + reason metadata; ``None`` if no
+ call has been made yet.
+ """
+ if not self._pending_usage:
+ return None
+ return self._pending_usage[-1]
+
def get_usage_reason(self) -> object:
return self.platform_kwargs.get("llm_usage_reason")
+ def flush_pending_usage(self) -> list[dict]:
+ """Return and clear all pending usage records.
+
+ Called at executor finalization.
+ """
+ records = self._pending_usage
+ self._pending_usage = []
+ return records
+
def _record_usage(
self,
model: str,
@@ -652,23 +681,74 @@ class LLM:
usage: Mapping[str, int] | None,
llm_api: str,
) -> None:
- prompt_tokens = token_counter(model=model, messages=messages)
usage_data: Mapping[str, int] = usage or {}
- all_tokens = TokenCounterCompat(
- prompt_tokens=usage_data.get("prompt_tokens", 0),
- completion_tokens=usage_data.get("completion_tokens", 0),
- total_tokens=usage_data.get("total_tokens", 0),
- )
+ prompt_tokens = usage_data.get("prompt_tokens", 0)
+ completion_tokens = usage_data.get("completion_tokens", 0)
+ total_tokens = usage_data.get("total_tokens", 0)
- logger.info(f"[sdk1][LLM][{model}][{llm_api}] Prompt Tokens: {prompt_tokens}")
- logger.info(f"[sdk1][LLM][{model}][{llm_api}] LLM Usage: {all_tokens}")
+ # Fall back to litellm when providers omit prompt tokens — avoids 0-token billing.
+ if prompt_tokens == 0 and messages:
+ try:
+ prompt_tokens = litellm.token_counter(model=model, messages=messages)
+ if total_tokens == 0:
+ total_tokens = prompt_tokens + completion_tokens
+ except Exception:
+ logger.warning(
+ "[sdk1][LLM][%s] prompt_tokens missing on response and "
+ "litellm.token_counter() fallback failed; recording 0",
+ model,
+ exc_info=True,
+ )
- Audit().push_usage_data(
- platform_api_key=self._platform_api_key,
- token_counter=all_tokens,
- event_type="llm",
- model_name=model,
- kwargs={"provider": self.adapter.get_provider(), **self.platform_kwargs},
+ logger.info(
+ "[sdk1][LLM][%s][%s] Usage: prompt=%d completion=%d total=%d",
+ model,
+ llm_api,
+ prompt_tokens,
+ completion_tokens,
+ total_tokens,
+ )
+
+ try:
+ prompt_cost, compl_cost = litellm.cost_per_token(
+ model=model,
+ prompt_tokens=prompt_tokens,
+ completion_tokens=completion_tokens,
+ )
+ cost = prompt_cost + compl_cost
+ except Exception:
+ logger.warning(
+ "Failed to compute cost for model=%s; recording as 0.0",
+ model,
+ exc_info=True,
+ )
+ cost = 0.0
+
+ # Trailing segment matches legacy Audit semantics (e.g. bedrock/anthropic/claude).
+ display_model = model.rsplit("/", 1)[-1] if model else model
+
+ # Spread _usage_kwargs first so computed billing fields below win.
+ self._pending_usage.append(
+ {
+ **self._usage_kwargs,
+ "usage_type": "llm",
+ "model_name": display_model,
+ "provider": self.adapter.get_provider(),
+ "adapter_instance_id": self.platform_kwargs.get(
+ "adapter_instance_id", ""
+ ),
+ # run_id lands in a UUIDField — "" fails the cast; keep None.
+ "run_id": self.platform_kwargs.get("run_id") or None,
+ "execution_id": self.platform_kwargs.get("execution_id", ""),
+ # "" isn't a valid choice for llm_usage_reason.
+ "llm_usage_reason": self.platform_kwargs.get("llm_usage_reason") or None,
+ "prompt_tokens": prompt_tokens,
+ "completion_tokens": completion_tokens,
+ "total_tokens": total_tokens,
+ "embedding_tokens": 0,
+ "cost_in_dollars": cost,
+ "status": "SUCCESS",
+ }
)
# Finish reasons indicating a safety/policy refusal across providers:
@@ -1053,10 +1133,18 @@ class LLMCompat:
"""Get captured metrics."""
return self._llm_instance.get_metrics()
+ def get_last_usage(self) -> Mapping[str, int]:
+ """Token usage from the most recent complete() call."""
+ return self._llm_instance.get_last_usage()
+
def get_usage_reason(self) -> object:
"""Get usage reason from platform kwargs."""
return self._llm_instance.get_usage_reason()
+ def flush_pending_usage(self) -> list[dict]:
+ """Return and clear all pending usage records."""
+ return self._llm_instance.flush_pending_usage()
+
def test_connection(self) -> bool:
"""Test connection to the LLM provider."""
return self._llm_instance.test_connection()
diff --git a/unstract/sdk1/src/unstract/sdk1/usage_handler.py b/unstract/sdk1/src/unstract/sdk1/usage_handler.py
index 0ffa7c43e..0d1abf8ae 100644
--- a/unstract/sdk1/src/unstract/sdk1/usage_handler.py
+++ b/unstract/sdk1/src/unstract/sdk1/usage_handler.py
@@ -1,12 +1,15 @@
+import logging
from typing import Any
+import litellm
from llama_index.core.callbacks import CBEventType, TokenCountingHandler
from llama_index.core.callbacks.base_handler import BaseCallbackHandler
from llama_index.core.embeddings import BaseEmbedding
-from unstract.sdk1.audit import Audit
from unstract.sdk1.constants import LogLevel
from unstract.sdk1.tool.stream import StreamMixin
+logger = logging.getLogger(__name__)
+
class UsageHandler(StreamMixin, BaseCallbackHandler):
"""Handler for usage events in LLM or Embedding models.
@@ -57,6 +60,7 @@ class UsageHandler(StreamMixin, BaseCallbackHandler):
self._verbose = verbose
self.token_counter = token_counter
self.embed_model = embed_model
+ self._pending_usage: list[dict] = []
self.platform_api_key = platform_api_key
super().__init__(
log_level=log_level, # StreamMixin's args
@@ -101,17 +105,60 @@ class UsageHandler(StreamMixin, BaseCallbackHandler):
and event_type not in self.event_ends_to_ignore
and payload is not None
):
+ if self.embed_model is None:
+ return
+ if self.token_counter is None:
+ logger.warning(
+ "Embedding usage callback invoked without token_counter; "
+ "skipping usage record."
+ )
+ return
model_name = self.embed_model.model_name
- # Need to push the data to via platform service
+ embedding_tokens = self.token_counter.total_embedding_token_count
self.stream_log(
- log=f"Pushing embedding usage for model {model_name}",
+ log=f"Recording embedding usage for model {model_name}",
level=LogLevel.DEBUG,
)
- Audit(log_level=self.log_level).push_usage_data(
- platform_api_key=self.platform_api_key,
- token_counter=self.token_counter,
- event_type=event_type,
- model_name=self.embed_model.model_name,
- kwargs=self.kwargs,
+
+ try:
+ prompt_cost, _ = litellm.cost_per_token(
+ model=model_name,
+ prompt_tokens=embedding_tokens,
+ completion_tokens=0,
+ )
+ cost = prompt_cost
+ except Exception:
+ logger.warning(
+ "Failed to compute embedding cost for model=%s; recording 0.0",
+ model_name,
+ exc_info=True,
+ )
+ cost = 0.0
+
+ # Collapse multi-segment IDs (``bedrock/anthropic/claude``) to
+ # the trailing segment to match legacy Audit semantics.
+ display_model = model_name.rsplit("/", 1)[-1] if model_name else model_name
+
+ self._pending_usage.append(
+ {
+ "usage_type": "embedding",
+ "model_name": display_model,
+ "adapter_instance_id": self.kwargs.get("adapter_instance_id", ""),
+ # run_id lands in a UUIDField — "" fails the cast; keep None.
+ "run_id": self.kwargs.get("run_id") or None,
+ "execution_id": self.kwargs.get("execution_id", ""),
+ "embedding_tokens": embedding_tokens,
+ "prompt_tokens": 0,
+ "completion_tokens": 0,
+ "total_tokens": 0,
+ "cost_in_dollars": cost,
+ "status": "SUCCESS",
+ }
)
self.token_counter.reset_counts()
+
+ def flush_pending_usage(self) -> list[dict]:
+ """Return and clear all pending usage records."""
+ records = self._pending_usage
+ self._pending_usage = []
+ return records
diff --git a/unstract/sdk1/src/unstract/sdk1/utils/common.py b/unstract/sdk1/src/unstract/sdk1/utils/common.py
index 86b0f173b..68a3fe068 100644
--- a/unstract/sdk1/src/unstract/sdk1/utils/common.py
+++ b/unstract/sdk1/src/unstract/sdk1/utils/common.py
@@ -223,6 +223,10 @@ def capture_metrics(func: object) -> object:
if self._run_id and self._capture_metrics:
metrics_mixin = MetricsMixin(run_id=self._run_id)
+ # Snapshot index so timing stamps every row appended during this call —
+ # internal retries would otherwise leave NULL execution_time_ms rows.
+ pending_at_entry = len(getattr(self, "_pending_usage", []))
+
try:
result = func(self, *args, **kwargs)
finally:
@@ -250,6 +254,14 @@ def capture_metrics(func: object) -> object:
# If the key isn't in self._metrics, set it to new_metrics
self._metrics = new_metrics
+ # Stamp timing on every record appended during this call.
+ pending = getattr(self, "_pending_usage", [])
+ time_taken = new_metrics.get(time_taken_key)
+ if time_taken is not None and len(pending) > pending_at_entry:
+ elapsed_ms = int(time_taken * 1000)
+ for record in pending[pending_at_entry:]:
+ record["execution_time_ms"] = elapsed_ms
+
return result
return wrapper
diff --git a/workers/executor/executors/answer_prompt.py b/workers/executor/executors/answer_prompt.py
index c22c8aaaf..d1eef5b3b 100644
--- a/workers/executor/executors/answer_prompt.py
+++ b/workers/executor/executors/answer_prompt.py
@@ -338,27 +338,4 @@ class AnswerPromptService:
structured_output[prompt_key] = {}
return
- highlight_data = None
- if enable_highlight and metadata and PSKeys.HIGHLIGHT_DATA in metadata:
- highlight_data = metadata[PSKeys.HIGHLIGHT_DATA].get(prompt_key)
-
- processed_data = parsed_data
- updated_highlight_data = None
-
- webhook_enabled = output.get(PSKeys.ENABLE_POSTPROCESSING_WEBHOOK, False)
- if webhook_enabled:
- webhook_url = output.get(PSKeys.POSTPROCESSING_WEBHOOK_URL)
- processed_data, updated_highlight_data = (
- AnswerPromptService._run_webhook_postprocess(
- parsed_data=parsed_data,
- webhook_url=webhook_url,
- highlight_data=highlight_data,
- )
- )
-
- structured_output[prompt_key] = processed_data
-
- if enable_highlight and metadata and updated_highlight_data is not None:
- metadata.setdefault(PSKeys.HIGHLIGHT_DATA, {})[prompt_key] = (
- updated_highlight_data
- )
+ structured_output[prompt_key] = parsed_data
diff --git a/workers/executor/executors/exceptions.py b/workers/executor/executors/exceptions.py
index 69cd0a8a1..3db835882 100644
--- a/workers/executor/executors/exceptions.py
+++ b/workers/executor/executors/exceptions.py
@@ -9,18 +9,23 @@ work outside of Flask (i.e. inside the Celery executor worker).
class LegacyExecutorError(Exception):
"""Base exception for legacy executor errors.
- Replaces Flask's ``APIError`` — carries ``message`` and ``code``
- attributes so callers can map to ``ExecutionResult.failure()``.
+ ``partial_usage_records`` preserves billing rows across mid-pipeline failures.
"""
code: int = 500
message: str = "Internal executor error"
- def __init__(self, message: str | None = None, code: int | None = None):
+ def __init__(
+ self,
+ message: str | None = None,
+ code: int | None = None,
+ partial_usage_records: list[dict] | None = None,
+ ):
if message is not None:
self.message = message
if code is not None:
self.code = code
+ self.partial_usage_records: list[dict] = list(partial_usage_records or [])
super().__init__(self.message)
diff --git a/workers/executor/executors/legacy_executor.py b/workers/executor/executors/legacy_executor.py
index 3760aea06..0ddb29e26 100644
--- a/workers/executor/executors/legacy_executor.py
+++ b/workers/executor/executors/legacy_executor.py
@@ -21,6 +21,10 @@ from executor.executors.dto import (
)
from executor.executors.exceptions import ExtractionError, LegacyExecutorError
from executor.executors.file_utils import FileUtils
+from executor.executors.lookup_enrichment import (
+ run_lookup_enrichment,
+ run_webhook_postprocessing,
+)
from unstract.sdk1.adapters.exceptions import AdapterError
from unstract.sdk1.adapters.x2text.constants import X2TextConstants
@@ -58,7 +62,6 @@ class LegacyExecutor(BaseExecutor):
}
def __init__(self) -> None:
- # Per-request state — overwritten on every ``execute()`` call.
self._log_events_id: str = ""
self._log_component: dict[str, str] = {}
self._execution_id: str | None = None
@@ -81,8 +84,10 @@ class LegacyExecutor(BaseExecutor):
NotImplementedError: From stub handlers (until 2D–2H).
"""
# Extract log streaming info (set by tasks.py for IDE sessions).
- self._log_events_id = context.log_events_id or ""
- self._log_component = getattr(context, "_log_component", None) or {}
+ self._log_events_id: str = context.log_events_id or ""
+ self._log_component: dict[str, str] = (
+ getattr(context, "_log_component", None) or {}
+ )
self._execution_id = context.execution_id
self._file_execution_id = context.file_execution_id
self._organization_id = context.organization_id
@@ -144,8 +149,17 @@ class LegacyExecutor(BaseExecutor):
level=LogLevel.ERROR,
)
except Exception:
- pass # Best-effort — don't mask the original error
- return ExecutionResult.failure(error=exc.message)
+ # Don't mask the original error; log the secondary at DEBUG.
+ logger.debug(
+ "Failed to stream error to FE for run_id=%s",
+ context.run_id,
+ exc_info=True,
+ )
+ # Preserve partial usage rows from a mid-pipeline failure.
+ failure_metadata: dict[str, Any] = {}
+ if exc.partial_usage_records:
+ failure_metadata["usage_records"] = list(exc.partial_usage_records)
+ return ExecutionResult.failure(error=exc.message, metadata=failure_metadata)
def _build_shim(
self,
@@ -346,12 +360,10 @@ class LegacyExecutor(BaseExecutor):
def _run_summarize_step(
self, summarize_params: dict, context: ExecutionContext
- ) -> ExecutionResult | None:
+ ) -> ExecutionResult:
"""Run summarization if not already cached.
- Returns:
- ``None`` on success (summary written or cached), or an
- ``ExecutionResult`` failure to propagate to the caller.
+ Cache hit returns success with empty metadata (no LLM call).
"""
extract_file_path = summarize_params.get("extract_file_path", "")
summarize_file_path = summarize_params.get("summarize_file_path", "")
@@ -366,7 +378,7 @@ class LegacyExecutor(BaseExecutor):
if fs.exists(summarize_file_path):
existing = fs.read(path=summarize_file_path, mode="r")
if existing:
- return None
+ return ExecutionResult(success=True, data={})
doc_context = fs.read(path=extract_file_path, mode="r")
if not doc_context:
@@ -403,7 +415,9 @@ class LegacyExecutor(BaseExecutor):
mode="w",
data=summarize_result.data.get("data", ""),
)
- return None
+ return ExecutionResult(
+ success=True, data={}, metadata=dict(summarize_result.metadata or {})
+ )
# ------------------------------------------------------------------
# Phase 5C — Compound IDE index handler (extract + index)
@@ -440,60 +454,80 @@ class LegacyExecutor(BaseExecutor):
error=f"ide_index missing required params: {', '.join(missing)}"
)
- # Step 1: Extract (or reuse pre-extracted text on marker hit)
- pre_extracted_text = index_params.get(IKeys.EXTRACTED_TEXT, "") or ""
- if pre_extracted_text:
- logger.info(
- "ide_index: marker hit, skipping extract step " "(len=%d, run_id=%s)",
- len(pre_extracted_text),
- context.run_id,
- )
- extracted_text = pre_extracted_text
- else:
- extract_ctx = ExecutionContext(
+ ide_records: list[dict[str, Any]] = []
+
+ def _absorb(child: ExecutionResult) -> None:
+ child_records = (child.metadata or {}).get("usage_records") or []
+ if child_records:
+ ide_records.extend(child_records)
+
+ def _failure(child: ExecutionResult) -> ExecutionResult:
+ metadata = dict(child.metadata or {})
+ existing = metadata.get("usage_records") or []
+ metadata["usage_records"] = ide_records + list(existing)
+ return ExecutionResult.failure(error=child.error, metadata=metadata)
+
+ try:
+ # Step 1: Extract (or reuse pre-extracted text on marker hit)
+ pre_extracted_text = index_params.get(IKeys.EXTRACTED_TEXT, "") or ""
+ if pre_extracted_text:
+ logger.info(
+ "ide_index: marker hit, skipping extract step (len=%d, run_id=%s)",
+ len(pre_extracted_text),
+ context.run_id,
+ )
+ extracted_text = pre_extracted_text
+ else:
+ extract_ctx = ExecutionContext(
+ executor_name=context.executor_name,
+ operation=Operation.EXTRACT.value,
+ run_id=context.run_id,
+ execution_source=context.execution_source,
+ organization_id=context.organization_id,
+ executor_params=extract_params,
+ request_id=context.request_id,
+ log_events_id=context.log_events_id,
+ execution_id=context.execution_id,
+ file_execution_id=context.file_execution_id,
+ )
+ extract_result = self._handle_extract(extract_ctx)
+ if not extract_result.success:
+ return _failure(extract_result)
+ _absorb(extract_result)
+ extracted_text = extract_result.data.get(IKeys.EXTRACTED_TEXT, "")
+
+ # Step 2: Optional summarize
+ summarize_params = params.get("summarize_params")
+ summarize_file_path = ""
+ if summarize_params:
+ summarize_file_path = summarize_params.get("summarize_file_path", "")
+ summarize_result = self._run_summarize_step(summarize_params, context)
+ if not summarize_result.success:
+ return _failure(summarize_result)
+ _absorb(summarize_result)
+
+ # Step 3: Index — inject extracted text
+ index_params[IKeys.EXTRACTED_TEXT] = extracted_text
+
+ index_ctx = ExecutionContext(
executor_name=context.executor_name,
- operation=Operation.EXTRACT.value,
+ operation=Operation.INDEX.value,
run_id=context.run_id,
execution_source=context.execution_source,
organization_id=context.organization_id,
- executor_params=extract_params,
+ executor_params=index_params,
request_id=context.request_id,
log_events_id=context.log_events_id,
execution_id=context.execution_id,
file_execution_id=context.file_execution_id,
)
- extract_result = self._handle_extract(extract_ctx)
- if not extract_result.success:
- return extract_result
- extracted_text = extract_result.data.get(IKeys.EXTRACTED_TEXT, "")
-
- # Step 2: Optional summarize
- summarize_params = params.get("summarize_params")
- summarize_file_path = ""
- if summarize_params:
- summarize_file_path = summarize_params.get("summarize_file_path", "")
- result = self._run_summarize_step(summarize_params, context)
- if result is not None:
- return result
-
- # Step 3: Index — inject extracted text
- index_params[IKeys.EXTRACTED_TEXT] = extracted_text
-
- index_ctx = ExecutionContext(
- executor_name=context.executor_name,
- operation=Operation.INDEX.value,
- run_id=context.run_id,
- execution_source=context.execution_source,
- organization_id=context.organization_id,
- executor_params=index_params,
- request_id=context.request_id,
- log_events_id=context.log_events_id,
- execution_id=context.execution_id,
- file_execution_id=context.file_execution_id,
- )
- index_result = self._handle_index(index_ctx)
- if not index_result.success:
- return index_result
+ index_result = self._handle_index(index_ctx)
+ if not index_result.success:
+ return _failure(index_result)
+ _absorb(index_result)
+ except LegacyExecutorError as e:
+ e.partial_usage_records = ide_records + e.partial_usage_records
+ raise
return ExecutionResult(
success=True,
@@ -501,6 +535,7 @@ class LegacyExecutor(BaseExecutor):
IKeys.DOC_ID: index_result.data.get(IKeys.DOC_ID, ""),
"summarize_file_path": summarize_file_path,
},
+ metadata={"usage_records": ide_records},
)
# ------------------------------------------------------------------
@@ -554,81 +589,100 @@ class LegacyExecutor(BaseExecutor):
extracted_text = ""
index_metrics: dict = {}
+ pipeline_records: list[dict[str, Any]] = []
+
+ def _absorb(child_result: ExecutionResult) -> None:
+ child_records = (child_result.metadata or {}).get("usage_records") or []
+ if child_records:
+ pipeline_records.extend(child_records)
+
+ def _failure(child_result: ExecutionResult) -> ExecutionResult:
+ metadata = dict(child_result.metadata or {})
+ existing = metadata.get("usage_records") or []
+ metadata["usage_records"] = pipeline_records + list(existing)
+ return ExecutionResult.failure(error=child_result.error, metadata=metadata)
shim = self._build_shim(
platform_api_key=extract_params.get("platform_api_key", ""),
)
step = 1
- # ---- Step 1: Extract ----
- if not skip_extraction:
- shim.stream_log(f"Pipeline step {step}: Extracting text from document...")
- step += 1
- extract_ctx = ExecutionContext(
- executor_name=context.executor_name,
- operation=Operation.EXTRACT.value,
- run_id=context.run_id,
- execution_source=context.execution_source,
- organization_id=context.organization_id,
- executor_params=extract_params,
- request_id=context.request_id,
- log_events_id=context.log_events_id,
- execution_id=context.execution_id,
- file_execution_id=context.file_execution_id,
- )
- extract_result = self._handle_extract(extract_ctx)
- if not extract_result.success:
- return extract_result
- extracted_text = extract_result.data.get(IKeys.EXTRACTED_TEXT, "")
-
- # ---- Step 2: Summarize (if enabled) ----
- if is_summarization:
- shim.stream_log(f"Pipeline step {step}: Summarizing extracted text...")
- step += 1
- summarize_result = self._run_pipeline_summarize(
- context=context,
- summarize_params=summarize_params or {},
- answer_params=answer_params,
- )
- if not summarize_result.success:
- return summarize_result
- # answer_params file_path/hash updated in-place by helper
- elif skip_extraction:
- # Smart table: use original source file
- answer_params["file_path"] = input_file_path
- elif not is_single_pass:
- # ---- Step 3: Index per output with dedup ----
- shim.stream_log(
- f"Pipeline step {step}: Indexing document into vector store..."
- )
- step += 1
- index_metrics = self._run_pipeline_index(
- context=context,
- index_template=index_template,
- answer_params=answer_params,
- extracted_text=extracted_text,
- usage_kwargs=extract_params.get("usage_kwargs", {}),
- )
+ try:
+ # ---- Step 1: Extract ----
+ if not skip_extraction:
+ shim.stream_log(f"Pipeline step {step}: Extracting text from document...")
+ step += 1
+ extract_ctx = ExecutionContext(
+ executor_name=context.executor_name,
+ operation=Operation.EXTRACT.value,
+ run_id=context.run_id,
+ execution_source=context.execution_source,
+ organization_id=context.organization_id,
+ executor_params=extract_params,
+ request_id=context.request_id,
+ log_events_id=context.log_events_id,
+ execution_id=context.execution_id,
+ file_execution_id=context.file_execution_id,
+ )
+ extract_result = self._handle_extract(extract_ctx)
+ if not extract_result.success:
+ return _failure(extract_result)
+ _absorb(extract_result)
+ extracted_text = extract_result.data.get(IKeys.EXTRACTED_TEXT, "")
+
+ # ---- Step 2: Summarize (if enabled) ----
+ if is_summarization:
+ shim.stream_log(f"Pipeline step {step}: Summarizing extracted text...")
+ step += 1
+ summarize_result = self._run_pipeline_summarize(
+ context=context,
+ summarize_params=summarize_params or {},
+ answer_params=answer_params,
+ )
+ if not summarize_result.success:
+ return _failure(summarize_result)
+ _absorb(summarize_result)
+ # answer_params file_path/hash updated in-place by helper
+ elif skip_extraction:
+ # Smart table: use original source file
+ answer_params["file_path"] = input_file_path
+ elif not is_single_pass:
+ # ---- Step 3: Index per output with dedup ----
+ shim.stream_log(
+ f"Pipeline step {step}: Indexing document into vector store..."
+ )
+ step += 1
+ index_metrics = self._run_pipeline_index(
+ context=context,
+ index_template=index_template,
+ answer_params=answer_params,
+ extracted_text=extracted_text,
+ usage_kwargs=extract_params.get("usage_kwargs", {}),
+ )
- # ---- Step 4: Table settings injection ----
- if not is_single_pass:
- self._inject_table_settings(
+ # ---- Step 4: Table settings injection ----
+ if not is_single_pass:
+ self._inject_table_settings(
+ answer_params=answer_params,
+ index_template=index_template,
+ skip_extraction=skip_extraction,
+ input_file_path=input_file_path,
+ )
+
+ # ---- Step 5: Answer prompt / Single pass ----
+ answer_result = self._run_pipeline_answer_step(
+ context=context,
answer_params=answer_params,
- index_template=index_template,
- skip_extraction=skip_extraction,
- input_file_path=input_file_path,
+ is_single_pass=is_single_pass,
+ shim=shim,
+ step=step,
)
-
- # ---- Step 5: Answer prompt / Single pass ----
- answer_result = self._run_pipeline_answer_step(
- context=context,
- answer_params=answer_params,
- is_single_pass=is_single_pass,
- shim=shim,
- step=step,
- )
- if not answer_result.success:
- return answer_result
+ if not answer_result.success:
+ return _failure(answer_result)
+ _absorb(answer_result)
+ except LegacyExecutorError as e:
+ e.partial_usage_records = pipeline_records + e.partial_usage_records
+ raise
# ---- Step 6: Merge results ----
structured_output = answer_result.data
@@ -640,7 +694,17 @@ class LegacyExecutor(BaseExecutor):
)
shim.stream_log("Pipeline completed successfully")
- return ExecutionResult(success=True, data=structured_output)
+ out_metadata = {
+ k: v
+ for k, v in (answer_result.metadata or {}).items()
+ if k != "usage_records"
+ }
+ out_metadata["usage_records"] = pipeline_records
+ return ExecutionResult(
+ success=True,
+ data=structured_output,
+ metadata=out_metadata,
+ )
def _run_pipeline_answer_step(
self,
@@ -803,13 +867,16 @@ class LegacyExecutor(BaseExecutor):
mode="w",
data=summarized_context,
)
+ forward_metadata = dict(summarize_result.metadata or {})
+ else:
+ forward_metadata = {}
# Update answer_params
summarize_file_hash = fs.get_hash_from_file(path=summarize_file_path)
answer_params["file_hash"] = summarize_file_hash
answer_params["file_path"] = str(summarize_file_path)
- return ExecutionResult(success=True, data={})
+ return ExecutionResult(success=True, data={}, metadata=forward_metadata)
def _run_pipeline_index(
self,
@@ -1078,8 +1145,7 @@ class LegacyExecutor(BaseExecutor):
"Document already indexed in vector store; skipping re-index."
)
logger.info(
- "Skipping re-index: doc_id=%s already in vector DB and "
- "reindex=False",
+ "Skipping re-index: doc_id=%s already in vector DB and reindex=False",
doc_id,
)
return ExecutionResult(success=True, data={IKeys.DOC_ID: doc_id})
@@ -1304,19 +1370,26 @@ class LegacyExecutor(BaseExecutor):
embedding_compat_cls,
vector_db_cls,
)
- for output in prompts:
- self._execute_single_prompt(
- output=output,
- context=context,
- structured_output=structured_output,
- metadata=metadata,
- metrics=metrics,
- variable_names=variable_names,
- context_retrieval_metrics=context_retrieval_metrics,
- deps=_deps,
- tool_settings=tool_settings,
- process_text_fn=process_text_fn,
- )
+ usage_records: list[dict[str, Any]] = []
+ try:
+ for output in prompts:
+ usage_records.extend(
+ self._execute_single_prompt(
+ output=output,
+ context=context,
+ structured_output=structured_output,
+ metadata=metadata,
+ metrics=metrics,
+ variable_names=variable_names,
+ context_retrieval_metrics=context_retrieval_metrics,
+ deps=_deps,
+ tool_settings=tool_settings,
+ process_text_fn=process_text_fn,
+ )
+ )
+ except LegacyExecutorError as e:
+ e.partial_usage_records = usage_records + e.partial_usage_records
+ raise
pipeline_shim.stream_log(f"All {len(prompts)} prompts processed successfully")
logger.info(
@@ -1336,6 +1409,7 @@ class LegacyExecutor(BaseExecutor):
PSKeys.METADATA: metadata,
PSKeys.METRICS: metrics,
},
+ metadata={"usage_records": usage_records},
)
@staticmethod
@@ -1392,19 +1466,19 @@ class LegacyExecutor(BaseExecutor):
metadata: dict[str, Any],
shim: Any,
prompt_name: str,
- ) -> None:
- """Run challenge verification plugin if enabled and available."""
+ ) -> list[dict[str, Any]]:
+ """Run the challenge plugin if enabled. Returns its usage rows."""
from executor.executors.constants import PromptServiceConstants as PSKeys
from executor.executors.plugins import ExecutorPluginLoader
if not tool_settings.get(PSKeys.ENABLE_CHALLENGE):
- return
+ return []
challenge_cls = ExecutorPluginLoader.get("challenge")
if not challenge_cls:
- return
+ return []
challenge_llm_id = tool_settings.get(PSKeys.CHALLENGE_LLM)
if not challenge_llm_id:
- return
+ return []
shim.stream_log(f"Running challenge for: `{prompt_name}`")
challenge_llm = llm_cls(
adapter_instance_id=challenge_llm_id,
@@ -1423,9 +1497,18 @@ class LegacyExecutor(BaseExecutor):
platform_key=platform_api_key,
metadata=metadata,
)
- challenger.run()
+ try:
+ challenger.run()
+ except LegacyExecutorError as e:
+ # Flush before bubbling so partial rows survive.
+ e.partial_usage_records = (
+ list(challenge_llm.flush_pending_usage()) + e.partial_usage_records
+ )
+ raise
+ records = list(challenge_llm.flush_pending_usage())
shim.stream_log(f"Challenge verification completed for: `{prompt_name}`")
logger.info("Challenge completed: prompt=%s", prompt_name)
+ return records
@staticmethod
def _run_evaluation_if_enabled(
@@ -1471,8 +1554,8 @@ class LegacyExecutor(BaseExecutor):
deps: tuple,
tool_settings: dict[str, Any],
process_text_fn: Any,
- ) -> None:
- """Execute one prompt: variable replacement, retrieval, LLM, post-process."""
+ ) -> list[dict[str, Any]]:
+ """Run one prompt end-to-end; return its usage rows."""
from executor.executors.constants import PromptServiceConstants as PSKeys
from executor.executors.constants import RetrievalStrategy
@@ -1563,7 +1646,7 @@ class LegacyExecutor(BaseExecutor):
prompt_name=prompt_name,
shim=shim,
)
- return
+ return []
if output.get(PSKeys.TYPE) == PSKeys.LINE_ITEM:
self._run_line_item_extraction(
@@ -1585,39 +1668,22 @@ class LegacyExecutor(BaseExecutor):
},
shim=shim,
)
- return
+ return []
usage_kwargs = {"run_id": run_id, "execution_id": execution_id}
- try:
- llm = llm_cls(
- adapter_instance_id=output[PSKeys.LLM],
- tool=shim,
- usage_kwargs={**usage_kwargs, PSKeys.LLM_USAGE_REASON: PSKeys.EXTRACTION},
- capture_metrics=True,
- )
- vector_db = None
- if chunk_size > 0:
- embedding = embedding_compat_cls(
- adapter_instance_id=output[PSKeys.EMBEDDING],
- tool=shim,
- kwargs={**usage_kwargs},
- )
- vector_db = vector_db_cls(
- tool=shim,
- adapter_instance_id=output[PSKeys.VECTOR_DB],
- embedding=embedding,
- )
- shim.stream_log(
- f"Initialized LLM and retrieval adapters for: `{prompt_name}`"
- )
- except Exception as e:
- msg = f"Couldn't fetch adapter. {e}"
- logger.error(msg)
- raise LegacyExecutorError(
- message=msg, code=getattr(e, "status_code", None) or 500
- ) from e
+ llm, embedding, vector_db = self._init_llm_and_retrieval(
+ output=output,
+ shim=shim,
+ chunk_size=chunk_size,
+ llm_cls=llm_cls,
+ embedding_compat_cls=embedding_compat_cls,
+ vector_db_cls=vector_db_cls,
+ usage_kwargs=usage_kwargs,
+ prompt_name=prompt_name,
+ )
context_list: list[str] = []
+ records: list[dict[str, Any]] = []
try:
answer = "NA"
retrieval_strategy = output.get(PSKeys.RETRIEVAL_STRATEGY)
@@ -1689,19 +1755,39 @@ class LegacyExecutor(BaseExecutor):
)
shim.stream_log(f"Applied type conversion for: `{prompt_name}`")
- self._run_challenge_if_enabled(
- tool_settings=tool_settings,
+ records.extend(
+ run_lookup_enrichment(
+ output=output,
+ structured_output=structured_output,
+ metadata=metadata,
+ metrics=metrics,
+ shim=shim,
+ usage_kwargs=usage_kwargs,
+ llm_cls=llm_cls,
+ )
+ )
+ run_webhook_postprocessing(
output=output,
structured_output=structured_output,
- context_list=context_list,
- llm=llm,
- llm_cls=llm_cls,
- usage_kwargs=usage_kwargs,
- run_id=run_id,
- platform_api_key=platform_api_key,
metadata=metadata,
shim=shim,
- prompt_name=prompt_name,
+ )
+
+ records.extend(
+ self._run_challenge_if_enabled(
+ tool_settings=tool_settings,
+ output=output,
+ structured_output=structured_output,
+ context_list=context_list,
+ llm=llm,
+ llm_cls=llm_cls,
+ usage_kwargs=usage_kwargs,
+ run_id=run_id,
+ platform_api_key=platform_api_key,
+ metadata=metadata,
+ shim=shim,
+ prompt_name=prompt_name,
+ )
)
self._run_evaluation_if_enabled(
output=output,
@@ -1716,15 +1802,111 @@ class LegacyExecutor(BaseExecutor):
val = structured_output.get(prompt_name)
if isinstance(val, str):
structured_output[prompt_name] = val.rstrip("\n")
- finally:
- metrics.setdefault(prompt_name, {}).update(
- {
- "context_retrieval": context_retrieval_metrics.get(prompt_name, {}),
- f"{llm.get_usage_reason()}_llm": llm.get_metrics(),
- }
+ except LegacyExecutorError as e:
+ # Flush before bubbling so partial rows survive.
+ flushed = self._flush_per_prompt_metrics(
+ metrics=metrics,
+ context_retrieval_metrics=context_retrieval_metrics,
+ prompt_name=prompt_name,
+ llm=llm,
+ embedding=embedding,
+ vector_db=vector_db,
+ chunk_size=chunk_size,
)
- if vector_db:
- vector_db.close()
+ e.partial_usage_records = records + flushed + e.partial_usage_records
+ raise
+ records.extend(
+ self._flush_per_prompt_metrics(
+ metrics=metrics,
+ context_retrieval_metrics=context_retrieval_metrics,
+ prompt_name=prompt_name,
+ llm=llm,
+ embedding=embedding,
+ vector_db=vector_db,
+ chunk_size=chunk_size,
+ )
+ )
+ return records
+
+ def _init_llm_and_retrieval(
+ self,
+ output: dict[str, Any],
+ shim: Any,
+ chunk_size: int,
+ llm_cls: Any,
+ embedding_compat_cls: Any,
+ vector_db_cls: Any,
+ usage_kwargs: dict[str, Any],
+ prompt_name: str,
+ ) -> tuple[Any, Any, Any]:
+ from executor.executors.constants import PromptServiceConstants as PSKeys
+
+ try:
+ llm = llm_cls(
+ adapter_instance_id=output[PSKeys.LLM],
+ tool=shim,
+ usage_kwargs={**usage_kwargs, PSKeys.LLM_USAGE_REASON: PSKeys.EXTRACTION},
+ capture_metrics=True,
+ )
+ embedding = None
+ vector_db = None
+ if chunk_size > 0:
+ embedding = embedding_compat_cls(
+ adapter_instance_id=output[PSKeys.EMBEDDING],
+ tool=shim,
+ kwargs={**usage_kwargs},
+ )
+ vector_db = vector_db_cls(
+ tool=shim,
+ adapter_instance_id=output[PSKeys.VECTOR_DB],
+ embedding=embedding,
+ )
+ shim.stream_log(
+ f"Initialized LLM and retrieval adapters for: `{prompt_name}`"
+ )
+ return llm, embedding, vector_db
+ except Exception as e:
+ msg = f"Couldn't fetch adapter. {e}"
+ logger.error(msg)
+ raise LegacyExecutorError(
+ message=msg, code=getattr(e, "status_code", None) or 500
+ ) from e
+
+ def _flush_per_prompt_metrics(
+ self,
+ metrics: dict[str, Any],
+ context_retrieval_metrics: dict[str, Any],
+ prompt_name: str,
+ llm: Any,
+ embedding: Any,
+ vector_db: Any,
+ chunk_size: int,
+ ) -> list[dict[str, Any]]:
+ """Flush LLM + embedding usage rows and return them."""
+ metrics.setdefault(prompt_name, {}).update(
+ {
+ "context_retrieval": context_retrieval_metrics.get(prompt_name, {}),
+ f"{llm.get_usage_reason()}_llm": llm.get_metrics(),
+ }
+ )
+ records: list[dict[str, Any]] = list(llm.flush_pending_usage())
+ # Public adapters skip the callback_manager, so there's nothing to flush.
+ if chunk_size > 0 and embedding is not None and embedding.callback_manager:
+ for handler in embedding.callback_manager.handlers:
+ if not hasattr(handler, "flush_pending_usage"):
+ continue
+ # Per-handler guard so one bad handler doesn't drop the rest.
+ try:
+ records.extend(handler.flush_pending_usage())
+ except Exception:
+ logger.warning(
+ "Failed to flush usage from embedding handler %s",
+ type(handler).__name__,
+ exc_info=True,
+ )
+ if vector_db:
+ vector_db.close()
+ return records
def _run_table_extraction(
self,
@@ -2083,14 +2265,17 @@ class LegacyExecutor(BaseExecutor):
)
shim = self._build_shim(platform_api_key=platform_api_key)
+ # execution_id is required for correct usage-row classification.
usage_kwargs = {
"run_id": context.run_id,
+ "execution_id": context.execution_id or "",
PSKeys.LLM_USAGE_REASON: PSKeys.SUMMARIZE,
}
_, _, _, _, llm_cls, _, _ = self._get_prompt_deps()
shim.stream_log("Initializing LLM for summarization...")
+ llm: Any = None
try:
llm = llm_cls(
adapter_instance_id=llm_adapter_id,
@@ -2103,16 +2288,26 @@ class LegacyExecutor(BaseExecutor):
shim.stream_log("Running document summarization...")
summary = answer_prompt_svc.run_completion(llm=llm, prompt=prompt)
+ records = list(llm.flush_pending_usage())
logger.info("Summarization completed: run_id=%s", context.run_id)
shim.stream_log("Summarization completed")
return ExecutionResult(
success=True,
data={"data": summary},
+ metadata={"usage_records": records},
)
except Exception as e:
+ # Flush before re-raising so partial rows survive.
+ partial: list[dict] = []
+ if llm is not None:
+ try:
+ partial = list(llm.flush_pending_usage())
+ except Exception:
+ logger.debug("flush_pending_usage failed during error path")
logger.error("Summarization failed: error=%s", str(e))
status_code = getattr(e, "status_code", None) or 500
raise LegacyExecutorError(
message=f"Error during summarization: {e}",
code=status_code,
+ partial_usage_records=partial,
) from e
diff --git a/workers/executor/executors/lookup_enrichment.py b/workers/executor/executors/lookup_enrichment.py
new file mode 100644
index 000000000..0b224badb
--- /dev/null
+++ b/workers/executor/executors/lookup_enrichment.py
@@ -0,0 +1,145 @@
+"""Lookup enrichment + webhook postprocessing helpers.
+
+Extracted from ``LegacyExecutor`` so the orchestrator stays focused on
+dispatch. Both helpers are free functions — callers pass shim/state in.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from executor.executors.constants import PromptServiceConstants as PSKeys
+from executor.executors.plugins import ExecutorPluginLoader
+
+from unstract.sdk1.constants import LogLevel
+
+logger = logging.getLogger(__name__)
+
+
+def is_blank(value: Any) -> bool:
+ """Treat None / whitespace strings / empty containers as no-value.
+
+ Boolean False / numeric 0 are NOT blank — valid inputs for
+ boolean/number prompts whose lookups should still run.
+ """
+ if value is None:
+ return True
+ if isinstance(value, str):
+ return not value.strip()
+ if isinstance(value, (list, dict)):
+ return not value
+ return False
+
+
+def run_lookup_enrichment(
+ output: dict[str, Any],
+ structured_output: dict[str, Any],
+ metadata: dict[str, Any],
+ metrics: dict[str, Any],
+ shim: Any,
+ llm_cls: Any,
+ usage_kwargs: dict[str, Any] | None = None,
+) -> list[dict[str, Any]]:
+ """Run lookup enrichment plugin if enabled and available.
+
+ Returns any usage records the plugin emitted (recovered even on
+ plugin failure) so the caller can extend its billing batch.
+ """
+ prompt_name = output[PSKeys.NAME]
+ current_value = structured_output.get(prompt_name)
+
+ lookup_config = output.get("lookup_config")
+ lookup_cls = ExecutorPluginLoader.get("lookup-enrichment")
+ if not (lookup_config and lookup_cls):
+ return []
+ if is_blank(current_value):
+ # Surface a skip log instead of silently no-op-ing.
+ lookup_name = lookup_config.get("lookup_name") or "lookup"
+ shim.stream_log(
+ f"Skipping lookup `{lookup_name}` for `{prompt_name}` — "
+ f"source prompt produced no value."
+ )
+ return []
+
+ outcome = None
+ try:
+ outcome = lookup_cls.run_with_metrics(
+ llm_cls=llm_cls,
+ lookup_config=lookup_config,
+ structured_output=structured_output,
+ current_value=current_value,
+ metadata=metadata,
+ prompt_name=prompt_name,
+ shim=shim,
+ usage_kwargs=usage_kwargs,
+ )
+ metrics.setdefault(prompt_name, {})[lookup_cls.METRICS_KEY] = outcome.llm_metrics
+ except Exception:
+ # Degrade gracefully on plugin contract drift.
+ lookup_name = lookup_config.get("lookup_name") or "lookup"
+ logger.exception(
+ "Lookup enrichment failed for prompt=%s lookup=%s",
+ prompt_name,
+ lookup_name,
+ )
+ shim.stream_log(
+ f"Lookup `{lookup_name}` failed for `{prompt_name}`; "
+ f"continuing without enrichment.",
+ level=LogLevel.WARN,
+ )
+
+ if outcome is not None and getattr(outcome, "usage_records", None):
+ return list(outcome.usage_records)
+ return []
+
+
+def run_webhook_postprocessing(
+ output: dict[str, Any],
+ structured_output: dict[str, Any],
+ metadata: dict[str, Any],
+ shim: Any,
+) -> None:
+ """Run webhook postprocessing if enabled (JSON outputs only)."""
+ from executor.executors.answer_prompt import AnswerPromptService
+
+ prompt_name = output[PSKeys.NAME]
+ output_type = output.get(PSKeys.TYPE, "")
+ webhook_enabled = output.get(PSKeys.ENABLE_POSTPROCESSING_WEBHOOK, False)
+ if not webhook_enabled:
+ return
+ # Empty / non-JSON payloads are skipped — typically a parse failure.
+ parsed_value = structured_output.get(prompt_name)
+ if not isinstance(parsed_value, (dict, list)) or not parsed_value:
+ logger.warning(
+ "Webhook postprocessing skipped: prompt=%s parsed payload "
+ "is empty or non-JSON (likely a parse failure)",
+ prompt_name,
+ )
+ return
+ if output_type != PSKeys.JSON:
+ logger.warning(
+ "Webhook postprocessing supports JSON outputs only; "
+ "skipping for prompt=%s (output_type=%s)",
+ prompt_name,
+ output_type,
+ )
+ shim.stream_log(
+ f"Webhook postprocessing supports JSON outputs only; "
+ f"skipping for `{prompt_name}`.",
+ level=LogLevel.WARN,
+ )
+ return
+
+ webhook_url = output.get(PSKeys.POSTPROCESSING_WEBHOOK_URL)
+ highlight_data = None
+ if metadata and PSKeys.HIGHLIGHT_DATA in metadata:
+ highlight_data = metadata.get(PSKeys.HIGHLIGHT_DATA, {}).get(prompt_name)
+ processed, updated_highlights = AnswerPromptService._run_webhook_postprocess(
+ parsed_data=structured_output.get(prompt_name),
+ webhook_url=webhook_url,
+ highlight_data=highlight_data,
+ )
+ structured_output[prompt_name] = processed
+ if updated_highlights is not None and metadata:
+ metadata.setdefault(PSKeys.HIGHLIGHT_DATA, {})[prompt_name] = updated_highlights
diff --git a/workers/executor/executors/plugins/protocols.py b/workers/executor/executors/plugins/protocols.py
index fb4d676b3..c59973d73 100644
--- a/workers/executor/executors/plugins/protocols.py
+++ b/workers/executor/executors/plugins/protocols.py
@@ -49,3 +49,29 @@ class EvaluationProtocol(Protocol):
"""Legacy executor: prompt evaluation."""
def run(self, **kwargs: Any) -> dict: ...
+
+
+@runtime_checkable
+class LookupEnrichmentProtocol(Protocol):
+ """Legacy executor: post-extraction lookup enrichment.
+
+ The executor calls ``run_with_metrics`` (not ``run``) because the
+ plugin returns an outcome object exposing ``usage_records`` and
+ ``llm_metrics`` for the calling executor to flush. ``METRICS_KEY``
+ keys the lookup metrics into the per-prompt metrics dict.
+ """
+
+ METRICS_KEY: str
+
+ def run_with_metrics(
+ self,
+ *,
+ llm_cls: Any,
+ lookup_config: dict,
+ structured_output: dict,
+ current_value: Any,
+ metadata: dict,
+ prompt_name: str,
+ shim: Any,
+ usage_kwargs: dict | None = None,
+ ) -> Any: ...
diff --git a/workers/executor/executors/usage.py b/workers/executor/executors/usage.py
deleted file mode 100644
index ab6296eae..000000000
--- a/workers/executor/executors/usage.py
+++ /dev/null
@@ -1,81 +0,0 @@
-"""Usage tracking helper for the executor worker.
-
-Ported from prompt-service/.../helpers/usage.py.
-Flask/DB dependencies removed — usage data is pushed via the SDK1
-``Audit`` class (HTTP to platform API) and returned directly in
-``ExecutionResult.metadata`` instead of querying the DB.
-
-Note: The SDK1 adapters (LLM, EmbeddingCompat) already call
-``Audit().push_usage_data()`` internally. This helper is for
-explicit push calls outside of adapter operations (e.g. rent rolls).
-"""
-
-import logging
-from typing import Any
-
-logger = logging.getLogger(__name__)
-
-
-class UsageHelper:
- @staticmethod
- def push_usage_data(
- event_type: str,
- kwargs: dict[str, Any],
- platform_api_key: str,
- token_counter: Any = None,
- model_name: str = "",
- ) -> bool:
- """Push usage data to the audit service.
-
- Wraps ``Audit().push_usage_data()`` with validation and
- error handling.
-
- Args:
- event_type: Type of usage event (e.g. "llm", "embedding").
- kwargs: Context dict (run_id, execution_id, etc.).
- platform_api_key: API key for platform service auth.
- token_counter: Token counter with usage metrics.
- model_name: Name of the model used.
-
- Returns:
- True if successful, False otherwise.
- """
- if not kwargs or not isinstance(kwargs, dict):
- logger.error("Invalid kwargs provided to push_usage_data")
- return False
-
- if not platform_api_key or not isinstance(platform_api_key, str):
- logger.error("Invalid platform_api_key provided to push_usage_data")
- return False
-
- try:
- from unstract.sdk1.audit import Audit
-
- logger.debug(
- "Pushing usage data for event_type=%s model=%s",
- event_type,
- model_name,
- )
-
- Audit().push_usage_data(
- platform_api_key=platform_api_key,
- token_counter=token_counter,
- model_name=model_name,
- event_type=event_type,
- kwargs=kwargs,
- )
-
- logger.info("Successfully pushed usage data for %s", model_name)
- return True
- except Exception:
- logger.exception("Error pushing usage data")
- return False
-
- @staticmethod
- def format_float_positional(value: float, precision: int = 10) -> str:
- """Format a float without scientific notation.
-
- Removes trailing zeros for clean display of cost values.
- """
- formatted: str = f"{value:.{precision}f}"
- return formatted.rstrip("0").rstrip(".") if "." in formatted else formatted
diff --git a/workers/executor/tasks.py b/workers/executor/tasks.py
index 77d5ecaeb..a30243bca 100644
--- a/workers/executor/tasks.py
+++ b/workers/executor/tasks.py
@@ -6,7 +6,9 @@ ExecutionOrchestrator, and returns an ExecutionResult dict.
"""
from celery import shared_task
+from shared.clients import UsageAPIClient
from shared.enums.task_enums import TaskName
+from shared.infrastructure.config import WorkerConfig
from shared.infrastructure.logging import WorkerLogger
from unstract.sdk1.execution.context import ExecutionContext
@@ -15,6 +17,15 @@ from unstract.sdk1.execution.result import ExecutionResult
logger = WorkerLogger.get_logger(__name__)
+_LLM_BEARING_OPS = frozenset(
+ {
+ "answer_prompt",
+ "single_pass_extraction",
+ "summarize",
+ "structure_pipeline",
+ }
+)
+
@shared_task(
bind=True,
@@ -97,6 +108,40 @@ def execute_extraction(self, execution_context_dict: dict) -> dict:
orchestrator = ExecutionOrchestrator()
result = orchestrator.execute(context)
+ usage_records = result.metadata.get("usage_records", [])
+ if usage_records:
+ try:
+ config = WorkerConfig()
+ with UsageAPIClient(config) as usage_client:
+ # Org context is set on the client; no need to pass it per call.
+ usage_client.set_organization_context(context.organization_id)
+ ok = usage_client.bulk_create_usage(usage_records)
+ if not ok:
+ # ERROR severity so dropped billing rows are recoverable from logs.
+ logger.error(
+ "bulk_create_usage returned failure for %d records "
+ "(run_id=%s organization_id=%s)",
+ len(usage_records),
+ context.run_id,
+ context.organization_id,
+ )
+ except Exception:
+ logger.error(
+ "Failed to flush %d usage records for run_id=%s organization_id=%s",
+ len(usage_records),
+ context.run_id,
+ context.organization_id,
+ exc_info=True,
+ )
+ elif result.success and context.operation in _LLM_BEARING_OPS:
+ logger.info(
+ "No usage_records emitted for op=%s run_id=%s organization_id=%s "
+ "(unexpected for an LLM-bearing operation)",
+ context.operation,
+ context.run_id,
+ context.organization_id,
+ )
+
logger.info(
"execute_extraction complete: celery_task_id=%s request_id=%s success=%s",
self.request.id,
diff --git a/workers/file_processing/structure_tool_task.py b/workers/file_processing/structure_tool_task.py
index c0fdee1c0..502e48ff3 100644
--- a/workers/file_processing/structure_tool_task.py
+++ b/workers/file_processing/structure_tool_task.py
@@ -459,6 +459,8 @@ def _execute_structure_tool_impl(params: dict) -> dict:
execution_source="tool",
organization_id=organization_id,
request_id=file_execution_id,
+ execution_id=execution_id,
+ file_execution_id=file_execution_id,
executor_params=agentic_params,
)
at_result = dispatcher.dispatch(at_ctx, timeout=EXECUTOR_TIMEOUT)
@@ -488,6 +490,8 @@ def _execute_structure_tool_impl(params: dict) -> dict:
execution_source="tool",
organization_id=organization_id,
request_id=file_execution_id,
+ execution_id=execution_id,
+ file_execution_id=file_execution_id,
executor_params={
"extract_params": extract_params,
"index_template": index_template,
diff --git a/workers/ide_callback/tasks.py b/workers/ide_callback/tasks.py
index d0dbce256..c6494e036 100644
--- a/workers/ide_callback/tasks.py
+++ b/workers/ide_callback/tasks.py
@@ -24,6 +24,8 @@ PROMPT_STUDIO_RESULT_EVENT = "prompt_studio_result"
# WebSocket emission endpoint (relative to internal API base)
_EMIT_WEBSOCKET_ENDPOINT = "emit-websocket/"
+_UNKNOWN_EXECUTOR_ERROR = "Unknown executor error"
+
class _SafeEncoder(json.JSONEncoder):
"""JSON encoder that converts uuid.UUID and datetime objects to strings."""
@@ -164,7 +166,7 @@ def ide_index_complete(
try:
# Check executor-level failure
if not result_dict.get("success", False):
- error_msg = result_dict.get("error", "Unknown executor error")
+ error_msg = result_dict.get("error", _UNKNOWN_EXECUTOR_ERROR)
logger.error("ide_index executor reported failure: %s", error_msg)
api.remove_document_indexing(
org_id=org_id,
@@ -370,7 +372,7 @@ def ide_prompt_complete(
try:
# Check executor-level failure
if not result_dict.get("success", False):
- error_msg = result_dict.get("error", "Unknown executor error")
+ error_msg = result_dict.get("error", _UNKNOWN_EXECUTOR_ERROR)
logger.error("ide_prompt executor reported failure: %s", error_msg)
_emit_event(
api,
@@ -514,3 +516,154 @@ def ide_prompt_error(
)
except Exception:
logger.exception("ide_prompt_error callback failed")
+
+
+# ------------------------------------------------------------------
+# Text Extraction Callbacks
+#
+# Today only ``source="lookup"`` is wired up; the cloud lookups plugin
+# is the only registrant of the underlying extraction-complete /
+# extraction-error endpoints (see workers/shared/clients/extraction_client.py).
+# ------------------------------------------------------------------
+
+
+def _get_extraction_client():
+ from shared.clients.extraction_client import ExtractionAPIClient
+
+ return ExtractionAPIClient()
+
+
[email protected](name="extraction_complete")
+def extraction_complete(
+ result_dict: dict[str, Any],
+ callback_kwargs: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+ """Celery link callback after successful text extraction.
+
+ Persists the result via internal API and emits a WebSocket event.
+ """
+ cb = callback_kwargs or {}
+ source = cb.get("source", "")
+ file_id = cb.get("file_id", "")
+ org_id = cb.get("org_id", "")
+ extracted_text_path = cb.get("extracted_text_path", "")
+ ws_room = cb.get("ws_room", "")
+ ws_event = cb.get("ws_event", "")
+
+ with _get_extraction_client() as api, _get_api_client() as ps_api:
+ try:
+ # Check executor-level failure
+ if not result_dict.get("success", False):
+ error_msg = result_dict.get("error", _UNKNOWN_EXECUTOR_ERROR)
+ logger.error(
+ "extraction executor reported failure: source=%s file=%s error=%s",
+ source,
+ file_id,
+ error_msg,
+ )
+ api.mark_extraction_error(
+ source=source,
+ file_id=file_id,
+ error=error_msg,
+ organization_id=org_id,
+ )
+ if ws_room and ws_event:
+ _emit_websocket(
+ ps_api,
+ room=ws_room,
+ event=ws_event,
+ data={
+ "file_id": file_id,
+ "status": "ERROR",
+ "error": error_msg[:500],
+ },
+ )
+ return {"status": "failed", "error": error_msg}
+
+ api.mark_extraction_complete(
+ source=source,
+ file_id=file_id,
+ extracted_text_path=extracted_text_path,
+ organization_id=org_id,
+ )
+
+ if ws_room and ws_event:
+ _emit_websocket(
+ ps_api,
+ room=ws_room,
+ event=ws_event,
+ data={"file_id": file_id, "status": "COMPLETED"},
+ )
+
+ logger.info("Extraction completed: source=%s file=%s", source, file_id)
+ return {"status": "completed", "file_id": file_id}
+
+ except Exception as e:
+ logger.exception(
+ "extraction_complete callback failed: source=%s file=%s",
+ source,
+ file_id,
+ )
+ if ws_room and ws_event:
+ try:
+ _emit_websocket(
+ ps_api,
+ room=ws_room,
+ event=ws_event,
+ data={
+ "file_id": file_id,
+ "status": "ERROR",
+ "error": str(e)[:500],
+ },
+ )
+ except Exception:
+ # _emit_websocket swallows internally; log if anything escapes.
+ logger.debug(
+ "Failed to emit ws ERROR event in extraction_complete fallback",
+ exc_info=True,
+ )
+ raise
+
+
[email protected](name="extraction_error")
+def extraction_error(
+ failed_task_id: str,
+ callback_kwargs: dict[str, Any] | None = None,
+) -> None:
+ """Celery link_error callback when an extraction task fails."""
+ cb = callback_kwargs or {}
+ source = cb.get("source", "")
+ file_id = cb.get("file_id", "")
+ org_id = cb.get("org_id", "")
+ ws_room = cb.get("ws_room", "")
+ ws_event = cb.get("ws_event", "")
+
+ # Context-manage clients to avoid per-task session leaks.
+ with _get_extraction_client() as api, _get_api_client() as ps_api:
+ try:
+ error_msg = _get_task_error(failed_task_id, default="Text extraction failed")
+
+ api.mark_extraction_error(
+ source=source,
+ file_id=file_id,
+ error=error_msg,
+ organization_id=org_id,
+ )
+
+ if ws_room and ws_event:
+ _emit_websocket(
+ ps_api,
+ room=ws_room,
+ event=ws_event,
+ data={
+ "file_id": file_id,
+ "status": "ERROR",
+ "error": error_msg[:500],
+ },
+ )
+ except Exception:
+ logger.exception(
+ "extraction_error callback failed: source=%s file=%s",
+ source,
+ file_id,
+ )
diff --git a/workers/shared/clients/extraction_client.py b/workers/shared/clients/extraction_client.py
new file mode 100644
index 000000000..72400f62e
--- /dev/null
+++ b/workers/shared/clients/extraction_client.py
@@ -0,0 +1,73 @@
+"""Extraction API Client for text extraction callbacks.
+
+Used by the ide_callback worker to persist extraction results through
+the backend's internal API endpoints.
+
+NOTE on scope: the callback endpoints ``v1/extraction/extraction-{complete,error}/``
+are currently registered **only** by the cloud ``lookups`` plugin
+(see ``pluggable_apps/lookups/internal_urls.py``). The interface is
+``source``-dispatched and designed to serve other extraction flows
+(prompt-studio docs, connectors) in the future, but in OSS-only builds
+the endpoints are absent. Callers from OSS paths should expect a 404
+response and treat it as a no-op; the worker's error handling covers
+this via a 404-terminal pattern.
+"""
+
+import logging
+from typing import Any
+
+from .base_client import BaseAPIClient
+
+logger = logging.getLogger(__name__)
+
+_EXTRACTION_COMPLETE_ENDPOINT = "v1/extraction/extraction-complete/"
+_EXTRACTION_ERROR_ENDPOINT = "v1/extraction/extraction-error/"
+
+
+class ExtractionAPIClient(BaseAPIClient):
+ """API client for the extraction-callback endpoints registered by
+ cloud-side plugins (today: lookups). See module docstring for the
+ OSS-absence contract.
+ """
+
+ def mark_extraction_complete(
+ self,
+ source: str,
+ file_id: str,
+ extracted_text_path: str,
+ organization_id: str | None = None,
+ **extra: Any,
+ ) -> dict[str, Any]:
+ """Notify backend that extraction succeeded."""
+ payload: dict[str, Any] = {
+ "source": source,
+ "file_id": file_id,
+ "extracted_text_path": extracted_text_path,
+ **extra,
+ }
+ return self.post(
+ _EXTRACTION_COMPLETE_ENDPOINT,
+ data=payload,
+ organization_id=organization_id,
+ )
+
+ def mark_extraction_error(
+ self,
+ source: str,
+ file_id: str,
+ error: str,
+ organization_id: str | None = None,
+ **extra: Any,
+ ) -> dict[str, Any]:
+ """Notify backend that extraction failed."""
+ payload: dict[str, Any] = {
+ "source": source,
+ "file_id": file_id,
+ "error": error,
+ **extra,
+ }
+ return self.post(
+ _EXTRACTION_ERROR_ENDPOINT,
+ data=payload,
+ organization_id=organization_id,
+ )
diff --git a/workers/shared/clients/usage_client.py b/workers/shared/clients/usage_client.py
index a7ba8c701..0bbaba9f6 100644
--- a/workers/shared/clients/usage_client.py
+++ b/workers/shared/clients/usage_client.py
@@ -71,7 +71,7 @@ class BaseUsageResponse(APIResponse, Generic[T]):
def is_success(self) -> bool:
"""Check if the response indicates success."""
- return self.success_response and self.status == ResponseStatus.SUCCESS
+ return self.success and self.status == ResponseStatus.SUCCESS
@dataclass
@@ -119,6 +119,8 @@ class UsageAPIClient(BaseAPIClient, UsageOperationMixin):
Returns:
UsageResponse containing aggregated usage data
"""
+ # Pre-bind so the except handler can log even if validation raises.
+ validated_file_execution_id = str(file_execution_id)
try:
validated_file_execution_id = self._validate_file_execution_id(
file_execution_id
@@ -171,6 +173,41 @@ class UsageAPIClient(BaseAPIClient, UsageOperationMixin):
message="Failed to retrieve usage data",
)
+ def bulk_create_usage(
+ self, records: list[dict], organization_id: str | None = None
+ ) -> bool:
+ """Bulk create usage records at execution finalization.
+
+ Args:
+ records: List of usage record dicts to create.
+ organization_id: Optional organization ID override.
+
+ Returns:
+ True if records were created successfully.
+ """
+ if not records:
+ return True
+ try:
+ response = self.post(
+ "v1/usage/batch/",
+ data={"records": records},
+ organization_id=organization_id,
+ )
+ # The success path returns ``{"created": N}`` (no ``success``
+ # key); the error path returns ``{"success": False, "error":
+ # ...}``. Accept either an explicit success flag or the
+ # presence of ``created`` *as long as* ``success`` isn't
+ # explicitly False — guards against future contracts that may
+ # report a partial body.
+ if response.get("success") is False:
+ return False
+ return response.get("success") is True or "created" in response
+ except Exception:
+ logger.error(
+ "Failed to bulk create %d usage records", len(records), exc_info=True
+ )
+ return False
+
def get_aggregated_pages_processed(
self, file_execution_id: str | uuid.UUID, organization_id: str | None = None
) -> int | None:
diff --git a/workers/tests/test_usage.py b/workers/tests/test_usage.py
deleted file mode 100644
index fc08ac825..000000000
--- a/workers/tests/test_usage.py
+++ /dev/null
@@ -1,312 +0,0 @@
-"""Phase 2G — Usage tracking tests.
-
-Verifies:
-1. UsageHelper.push_usage_data wraps Audit correctly
-2. Invalid kwargs returns False
-3. Invalid platform_api_key returns False
-4. Audit exceptions are caught and return False
-5. format_float_positional formats correctly
-6. SDK1 adapters already push usage (integration check)
-7. answer_prompt handler returns metrics in ExecutionResult
-"""
-
-from unittest.mock import MagicMock, patch
-
-
-from executor.executors.usage import UsageHelper
-
-
-# ---------------------------------------------------------------------------
-# 1. push_usage_data success
-# ---------------------------------------------------------------------------
-
-
-class TestPushUsageData:
- @patch("unstract.sdk1.audit.Audit")
- def test_push_success(self, mock_audit_cls):
- """Successful push returns True and calls Audit."""
- mock_audit = MagicMock()
- mock_audit_cls.return_value = mock_audit
-
- result = UsageHelper.push_usage_data(
- event_type="llm",
- kwargs={"run_id": "run-001", "execution_id": "exec-001"},
- platform_api_key="test-key",
- token_counter=MagicMock(),
- model_name="gpt-4",
- )
-
- assert result is True
- mock_audit.push_usage_data.assert_called_once()
- call_kwargs = mock_audit.push_usage_data.call_args
- assert call_kwargs.kwargs["platform_api_key"] == "test-key"
- assert call_kwargs.kwargs["model_name"] == "gpt-4"
- assert call_kwargs.kwargs["event_type"] == "llm"
-
- @patch("unstract.sdk1.audit.Audit")
- def test_push_passes_token_counter(self, mock_audit_cls):
- """Token counter is passed through to Audit."""
- mock_audit = MagicMock()
- mock_audit_cls.return_value = mock_audit
- mock_counter = MagicMock()
-
- UsageHelper.push_usage_data(
- event_type="embedding",
- kwargs={"run_id": "run-002"},
- platform_api_key="key-2",
- token_counter=mock_counter,
- )
-
- call_kwargs = mock_audit.push_usage_data.call_args
- assert call_kwargs.kwargs["token_counter"] is mock_counter
-
-
-# ---------------------------------------------------------------------------
-# 2. Invalid kwargs
-# ---------------------------------------------------------------------------
-
-
-class TestPushValidation:
- def test_none_kwargs_returns_false(self):
- result = UsageHelper.push_usage_data(
- event_type="llm",
- kwargs=None,
- platform_api_key="key",
- )
- assert result is False
-
- def test_empty_kwargs_returns_false(self):
- result = UsageHelper.push_usage_data(
- event_type="llm",
- kwargs={},
- platform_api_key="key",
- )
- assert result is False
-
- def test_non_dict_kwargs_returns_false(self):
- result = UsageHelper.push_usage_data(
- event_type="llm",
- kwargs="not a dict",
- platform_api_key="key",
- )
- assert result is False
-
-
-# ---------------------------------------------------------------------------
-# 3. Invalid platform_api_key
-# ---------------------------------------------------------------------------
-
-
-class TestPushApiKeyValidation:
- def test_none_key_returns_false(self):
- result = UsageHelper.push_usage_data(
- event_type="llm",
- kwargs={"run_id": "r1"},
- platform_api_key=None,
- )
- assert result is False
-
- def test_empty_key_returns_false(self):
- result = UsageHelper.push_usage_data(
- event_type="llm",
- kwargs={"run_id": "r1"},
- platform_api_key="",
- )
- assert result is False
-
- def test_non_string_key_returns_false(self):
- result = UsageHelper.push_usage_data(
- event_type="llm",
- kwargs={"run_id": "r1"},
- platform_api_key=12345,
- )
- assert result is False
-
-
-# ---------------------------------------------------------------------------
-# 4. Audit exceptions are caught
-# ---------------------------------------------------------------------------
-
-
-class TestPushErrorHandling:
- @patch("unstract.sdk1.audit.Audit")
- def test_audit_exception_returns_false(self, mock_audit_cls):
- """Audit errors are caught and return False."""
- mock_audit = MagicMock()
- mock_audit.push_usage_data.side_effect = Exception("Network error")
- mock_audit_cls.return_value = mock_audit
-
- result = UsageHelper.push_usage_data(
- event_type="llm",
- kwargs={"run_id": "r1"},
- platform_api_key="key",
- token_counter=MagicMock(),
- )
-
- assert result is False
-
- @patch("unstract.sdk1.audit.Audit")
- def test_import_error_returns_false(self, mock_audit_cls):
- """Import errors are caught gracefully."""
- mock_audit_cls.side_effect = ImportError("no module")
-
- result = UsageHelper.push_usage_data(
- event_type="llm",
- kwargs={"run_id": "r1"},
- platform_api_key="key",
- )
-
- assert result is False
-
-
-# ---------------------------------------------------------------------------
-# 5. format_float_positional
-# ---------------------------------------------------------------------------
-
-
-class TestFormatFloat:
- def test_normal_float(self):
- assert UsageHelper.format_float_positional(0.0001234) == "0.0001234"
-
- def test_trailing_zeros_removed(self):
- assert UsageHelper.format_float_positional(1.50) == "1.5"
-
- def test_integer_value(self):
- assert UsageHelper.format_float_positional(42.0) == "42"
-
- def test_zero(self):
- assert UsageHelper.format_float_positional(0.0) == "0"
-
- def test_small_value(self):
- result = UsageHelper.format_float_positional(0.00000001)
- assert "0.00000001" == result
-
- def test_custom_precision(self):
- result = UsageHelper.format_float_positional(1.123456789, precision=3)
- assert result == "1.123"
-
-
-# ---------------------------------------------------------------------------
-# 6. SDK1 adapters already push usage
-# ---------------------------------------------------------------------------
-
-
-class TestAdapterUsageTracking:
- def test_llm_calls_audit_push(self):
- """Verify the LLM adapter imports and calls Audit.push_usage_data.
-
- This is a static analysis check — we verify the SDK1 LLM module
- references Audit.push_usage_data, confirming adapters handle
- usage tracking internally.
- """
- import inspect
-
- from unstract.sdk1.llm import LLM
-
- source = inspect.getsource(LLM)
- assert "push_usage_data" in source
- assert "Audit" in source
-
-
-# ---------------------------------------------------------------------------
-# 7. answer_prompt handler returns metrics
-# ---------------------------------------------------------------------------
-
-
-class TestMetricsInResult:
- @patch(
- "unstract.sdk1.utils.indexing.IndexingUtils.generate_index_key",
- return_value="doc-id-test",
- )
- @patch(
- "executor.executors.legacy_executor.LegacyExecutor._get_prompt_deps"
- )
- @patch("executor.executors.legacy_executor.ExecutorToolShim")
- def test_answer_prompt_returns_metrics(
- self, mock_shim_cls, mock_get_deps, _mock_idx, tmp_path
- ):
- """answer_prompt result includes metrics dict."""
- from unstract.sdk1.execution.context import ExecutionContext
- from unstract.sdk1.execution.registry import ExecutorRegistry
-
- ExecutorRegistry.clear()
- from executor.executors.legacy_executor import LegacyExecutor
-
- if "legacy" not in ExecutorRegistry.list_executors():
- ExecutorRegistry.register(LegacyExecutor)
-
- executor = ExecutorRegistry.get("legacy")
-
- # Mock all dependencies
- mock_llm = MagicMock()
- mock_llm.get_metrics.return_value = {"total_tokens": 100}
- mock_llm.get_usage_reason.return_value = "extraction"
- mock_llm.complete.return_value = {
- "response": MagicMock(text="test answer"),
- "highlight_data": [],
- "confidence_data": None,
- "word_confidence_data": None,
- "line_numbers": [],
- "whisper_hash": "",
- }
-
- mock_llm_cls = MagicMock(return_value=mock_llm)
- mock_index = MagicMock()
- mock_index.return_value.generate_index_key.return_value = "doc-123"
-
- mock_get_deps.return_value = (
- MagicMock(), # AnswerPromptService — use real for construct
- MagicMock(), # RetrievalService
- MagicMock(), # VariableReplacementService
- mock_index, # Index
- mock_llm_cls, # LLM
- MagicMock(), # EmbeddingCompat
- MagicMock(), # VectorDB
- )
-
- # Patch AnswerPromptService methods at their real location
- with patch(
- "executor.executors.answer_prompt.AnswerPromptService.extract_variable",
- return_value="test prompt",
- ), patch(
- "executor.executors.answer_prompt.AnswerPromptService.construct_and_run_prompt",
- return_value="test answer",
- ):
- ctx = ExecutionContext(
- executor_name="legacy",
- operation="answer_prompt",
- run_id="run-metrics-001",
- execution_source="tool",
- organization_id="org-test",
- request_id="req-metrics-001",
- executor_params={
- "tool_settings": {},
- "outputs": [
- {
- "name": "field1",
- "prompt": "What is X?",
- "chunk-size": 512,
- "chunk-overlap": 64,
- "vector-db": "vdb-1",
- "embedding": "emb-1",
- "x2text_adapter": "x2t-1",
- "llm": "llm-1",
- "type": "text",
- "retrieval-strategy": "simple",
- "similarity-top-k": 5,
- },
- ],
- "tool_id": "tool-1",
- "file_hash": "hash123",
- "file_path": str(tmp_path / "test.txt"),
- "file_name": "test.txt",
- "PLATFORM_SERVICE_API_KEY": "test-key",
- },
- )
- result = executor.execute(ctx)
-
- assert result.success is True
- assert "metrics" in result.data
- assert "field1" in result.data["metrics"]
-
- ExecutorRegistry.clear()