Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

[Fix] Prompt studio: Combined output and coverage (#898)

jagadeeswaran-zipstacke75d51f08855
diff --git a/backend/prompt_studio/prompt_studio_core_v2/serializers.py b/backend/prompt_studio/prompt_studio_core_v2/serializers.py
index 301557a54..bd9e32262 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/serializers.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/serializers.py
@@ -70,11 +70,16 @@ class CustomToolSerializer(IntegrityErrorMixin, AuditSerializer):
# Appending prompt instances of the tool for FE Processing
if prompt_instance.count() != 0:
for prompt in prompt_instance:
+ profile_manager_id = prompt.prompt_id
+ if instance.single_pass_extraction_mode:
+ # use projects default profile
+ profile_manager_id = profile_manager.profile_id
prompt_serializer = ToolStudioPromptSerializer(prompt)
coverage = OutputManagerUtils.get_coverage(
data.get(TSKeys.TOOL_ID),
- prompt.profile_manager_id,
+ profile_manager_id,
prompt.prompt_id,
+ instance.single_pass_extraction_mode,
)
serialized_data = prompt_serializer.data
serialized_data["coverage"] = coverage
diff --git a/backend/prompt_studio/prompt_studio_core_v2/views.py b/backend/prompt_studio/prompt_studio_core_v2/views.py
index 007606cfb..46228b7bc 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/views.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/views.py
@@ -44,6 +44,7 @@ from prompt_studio.prompt_studio_registry_v2.serializers import (
PromptStudioRegistryInfoSerializer,
)
from prompt_studio.prompt_studio_v2.constants import ToolStudioPromptErrors
+from prompt_studio.prompt_studio_v2.models import ToolStudioPrompt
from prompt_studio.prompt_studio_v2.serializers import ToolStudioPromptSerializer
from rest_framework import status, viewsets
from rest_framework.decorators import action
@@ -380,6 +381,13 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
raise MaxProfilesReachedError()
try:
self.perform_create(serializer)
+ # Check if this is the first profile and make it default for all prompts
+ if profile_count == 0:
+ profile_manager = serializer.instance # Newly created profile manager
+ ToolStudioPrompt.objects.filter(tool_id=prompt_studio_tool).update(
+ profile_manager=profile_manager
+ )
+
except IntegrityError:
raise DuplicateData(
f"{ProfileManagerErrors.PROFILE_NAME_EXISTS}, \
diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py
index a82ed95bf..7d66820b3 100644
--- a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py
+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py
@@ -191,7 +191,9 @@ class OutputManagerHelper:
@staticmethod
def fetch_default_output_response(
- tool_studio_prompts: list[ToolStudioPrompt], document_manager_id: str
+ tool_studio_prompts: list[ToolStudioPrompt],
+ document_manager_id: str,
+ use_default_profile: bool = False,
) -> dict[str, Any]:
"""Method to frame JSON responses for combined output for default for
default profile manager of the project.
@@ -199,6 +201,7 @@ class OutputManagerHelper:
Args:
tool_studio_prompts (list[ToolStudioPrompt])
document_manager_id (str)
+ use_default_profile (bool)
Returns:
dict[str, Any]: Formatted JSON response for combined output.
@@ -213,10 +216,16 @@ class OutputManagerHelper:
profile_manager_id = tool_prompt.profile_manager_id
# If profile_manager is not set, skip this record
- if not profile_manager_id:
+ if not profile_manager_id and not use_default_profile:
result[tool_prompt.prompt_key] = ""
continue
+ if not profile_manager_id:
+ default_profile = ProfileManager.get_default_llm_profile(
+ tool_prompt.tool_id
+ )
+ profile_manager_id = default_profile.profile_id
+
try:
queryset = PromptStudioOutputManager.objects.filter(
prompt_id=prompt_id,
diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_util.py b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_util.py
index 152a75bb6..4a0352099 100644
--- a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_util.py
+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_util.py
@@ -7,7 +7,10 @@ from prompt_studio.prompt_studio_output_manager_v2.models import (
class OutputManagerUtils:
@staticmethod
def get_coverage(
- tool_id: str, profile_manager_id: str, prompt_id: str = None
+ tool_id: str,
+ profile_manager_id: str,
+ prompt_id: str = None,
+ is_single_pass: bool = False,
) -> dict[str, int]:
"""
Method to fetch coverage data for given tool and profile manager.
@@ -17,6 +20,7 @@ class OutputManagerUtils:
profile_manager_id (str): The ID of the profile manager
for which coverage is calculated.
prompt_id (Optional[str]): The ID of the prompt (optional).
+ is_single_pass (Optional[bool]): Singlepass enabled or not
If provided, coverage is fetched for the specific prompt.
Returns:
@@ -25,12 +29,13 @@ class OutputManagerUtils:
Values are the count of documents associated with each prompt
and profile combination.
"""
-
+ # TODO: remove singlepass reference
prompt_outputs = (
PromptStudioOutputManager.objects.filter(
tool_id=tool_id,
profile_manager_id=profile_manager_id,
- **({"prompt_id": prompt_id} if prompt_id else {}),
+ prompt_id=prompt_id,
+ is_single_pass_extract=is_single_pass,
)
.values("prompt_id", "profile_manager_id")
.annotate(document_count=Count("document_manager_id"))
diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py b/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py
index 79a2abdb2..649829b53 100644
--- a/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py
+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py
@@ -33,7 +33,10 @@ class PromptStudioOutputSerializer(AuditSerializer):
try:
# Fetch all relevant outputs for the current tool and profile
coverage = OutputManagerUtils.get_coverage(
- instance.tool_id, instance.profile_manager_id, instance.prompt_id
+ instance.tool_id,
+ instance.profile_manager_id,
+ instance.prompt_id,
+ instance.is_single_pass_extract,
)
data["coverage"] = coverage
diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/views.py b/backend/prompt_studio/prompt_studio_output_manager_v2/views.py
index a21b7dc7d..6dad053c9 100644
--- a/backend/prompt_studio/prompt_studio_output_manager_v2/views.py
+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/views.py
@@ -82,6 +82,7 @@ class PromptStudioOutputView(viewsets.ModelViewSet):
result: dict[str, Any] = OutputManagerHelper.fetch_default_output_response(
tool_studio_prompts=tool_studio_prompts,
document_manager_id=document_manager_id,
+ use_default_profile=True,
)
return Response(result, status=status.HTTP_200_OK)
diff --git a/frontend/src/components/custom-tools/manage-llm-profiles/ManageLlmProfiles.jsx b/frontend/src/components/custom-tools/manage-llm-profiles/ManageLlmProfiles.jsx
index f00677356..ebdd3dfcd 100644
--- a/frontend/src/components/custom-tools/manage-llm-profiles/ManageLlmProfiles.jsx
+++ b/frontend/src/components/custom-tools/manage-llm-profiles/ManageLlmProfiles.jsx
@@ -132,13 +132,22 @@ function ManageLlmProfiles() {
handleConfirm={() => handleDelete(item?.profile_id)}
content="The LLM profile will be permanently deleted."
>
- <Button
- size="small"
- className="display-flex-align-center"
- disabled={isPublicSource}
+ <Tooltip
+ title={
+ defaultLlmProfile === item?.profile_id &&
+ "Default profile cannot be deleted"
+ }
>
- <DeleteOutlined classID="manage-llm-pro-icon" />
- </Button>
+ <Button
+ size="small"
+ className="display-flex-align-center"
+ disabled={
+ isPublicSource || defaultLlmProfile === item?.profile_id
+ }
+ >
+ <DeleteOutlined classID="manage-llm-pro-icon" />
+ </Button>
+ </Tooltip>
</ConfirmModal>
),
edit: (
diff --git a/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx b/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx
index acc9d700d..dcd55504e 100644
--- a/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx
+++ b/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx
@@ -66,7 +66,11 @@ function OutputForDocModal({
llmProfiles,
defaultLlmProfile,
} = useCustomToolStore();
- const [selectedProfile, setSelectedProfile] = useState(defaultLlmProfile);
+
+ const profileId = singlePassExtractMode
+ ? defaultLlmProfile
+ : profileManagerId;
+ const [selectedProfile, setSelectedProfile] = useState(profileId);
const { sessionDetails } = useSessionStore();
const axiosPrivate = useAxiosPrivate();
const navigate = useNavigate();
@@ -79,7 +83,7 @@ function OutputForDocModal({
if (!open) {
return;
}
- handleGetOutputForDocs(selectedProfile || profileManagerId);
+ handleGetOutputForDocs(selectedProfile);
getAdapterInfo();
}, [
open,
@@ -88,6 +92,10 @@ function OutputForDocModal({
isSinglePassExtractLoading,
]);
+ useEffect(() => {
+ setSelectedProfile(profileId);
+ }, [profileManagerId, singlePassExtractMode]);
+
useEffect(() => {
handleRowsGeneration(promptOutputs);
}, [promptOutputs, tokenUsage]);
@@ -284,7 +292,7 @@ function OutputForDocModal({
const handleTabChange = (key) => {
if (key === "0") {
- setSelectedProfile(defaultLlmProfile);
+ setSelectedProfile(profileId);
} else {
setSelectedProfile(adapterData[key - 1]?.profile_id);
}
@@ -337,10 +345,7 @@ function OutputForDocModal({
></TabPane>
))}
</Tabs>{" "}
- <ProfileInfoBar
- profileId={selectedProfile || profileManagerId}
- profiles={llmProfiles}
- />
+ <ProfileInfoBar profileId={selectedProfile} profiles={llmProfiles} />
</div>
<div className="display-flex-right">
<Button
diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx b/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx
index 0e50ff9d0..d147c3677 100644
--- a/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx
+++ b/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx
@@ -64,6 +64,7 @@ function PromptCardItems({
isPublicSource,
adapters,
defaultLlmProfile,
+ singlePassExtractMode,
} = useCustomToolStore();
const [isEditingPrompt, setIsEditingPrompt] = useState(false);
const [isEditingTitle, setIsEditingTitle] = useState(false);
@@ -77,10 +78,10 @@ function PromptCardItems({
const isNotSingleLlmProfile = llmProfiles.length > 1;
const divRef = useRef(null);
const [enforceType, setEnforceType] = useState("");
- const coverageKey = generateCoverageKey(
- promptDetails?.prompt_id,
- selectedLlmProfileId || defaultLlmProfile
- );
+ const profileId = singlePassExtractMode
+ ? defaultLlmProfile
+ : selectedLlmProfileId || defaultLlmProfile;
+ const coverageKey = generateCoverageKey(promptDetails?.prompt_id, profileId);
useEffect(() => {
if (enforceType !== promptDetails?.enforce_type) {