Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Evaluation LLM Settings (#99)

Deepak K8d31eb77efae
diff --git a/backend/prompt_studio/prompt_studio_core/migrations/0008_customtool_exclude_failed_customtool_monitor_llm.py b/backend/prompt_studio/prompt_studio_core/migrations/0008_customtool_exclude_failed_customtool_monitor_llm.py
new file mode 100644
index 000000000..3173d5604
--- /dev/null
+++ b/backend/prompt_studio/prompt_studio_core/migrations/0008_customtool_exclude_failed_customtool_monitor_llm.py
@@ -0,0 +1,33 @@
+# Generated by Django 4.2.1 on 2024-03-13 23:54
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("adapter_processor", "0005_alter_adapterinstance_adapter_type"),
+ (
+ "prompt_studio_core",
+ "0007_remove_customtool_default_profile_and_more",
+ ),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="customtool",
+ name="exclude_failed",
+ field=models.BooleanField(default=True),
+ ),
+ migrations.AddField(
+ model_name="customtool",
+ name="monitor_llm",
+ field=models.ForeignKey(
+ blank=True,
+ db_comment="Field to store monitor llm",
+ null=True,
+ on_delete=django.db.models.deletion.PROTECT,
+ to="adapter_processor.adapterinstance",
+ ),
+ ),
+ ]
diff --git a/backend/prompt_studio/prompt_studio_core/models.py b/backend/prompt_studio/prompt_studio_core/models.py
index 9dbda4a6d..c3677dbb7 100644
--- a/backend/prompt_studio/prompt_studio_core/models.py
+++ b/backend/prompt_studio/prompt_studio_core/models.py
@@ -1,6 +1,7 @@
import uuid
from account.models import User
+from adapter_processor.models import AdapterInstance
from django.core.exceptions import ObjectDoesNotExist
from django.db import models
from prompt_studio.prompt_studio_core.exceptions import DefaultProfileError
@@ -54,6 +55,13 @@ class CustomTool(BaseModel):
prompt_grammer = models.JSONField(
null=True, blank=True, db_comment="Synonymous words used in prompt"
)
+ monitor_llm = models.ForeignKey(
+ AdapterInstance,
+ on_delete=models.PROTECT,
+ db_comment="Field to store monitor llm",
+ null=True,
+ blank=True,
+ )
created_by = models.ForeignKey(
User,
on_delete=models.SET_NULL,
@@ -70,6 +78,7 @@ class CustomTool(BaseModel):
blank=True,
editable=False,
)
+ exclude_failed = models.BooleanField(default=True)
# TODO: Add ProfileManager to return type
def get_default_llm_profile(self): # type: ignore
diff --git a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py
index 9895f80a9..2fa311813 100644
--- a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py
+++ b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py
@@ -4,6 +4,7 @@ import os
from pathlib import Path
from typing import Any, Optional
+from adapter_processor.models import AdapterInstance
from django.conf import settings
from file_management.file_management_helper import FileManagerHelper
from prompt_studio.prompt_profile_manager.models import ProfileManager
@@ -289,12 +290,25 @@ class PromptStudioHelper:
Raises:
AnswerFetchError
"""
- outputs: list[dict[str, Any]] = []
-
+ monitor_llm_instance: Optional[AdapterInstance] = tool.monitor_llm
+ monitor_llm: Optional[str] = None
+ if monitor_llm_instance:
+ monitor_llm = str(monitor_llm_instance.id)
prompt_grammer = tool.prompt_grammer
+ outputs: list[dict[str, Any]] = []
grammer_dict = {}
grammar_list = []
+ # Using default profile manager llm if monitor_llm is None
+ if monitor_llm:
+ monitor_llm = str(monitor_llm_instance.id)
+ else:
+ # TODO: Use CustomTool model to get profile_manager
+ profile_manager = ProfileManager.objects.get(
+ prompt_studio_tool=tool, is_default=True
+ )
+ monitor_llm = str(profile_manager.llm.id)
+
# Adding validations
if prompt_grammer:
for word, synonyms in prompt_grammer.items():
@@ -355,11 +369,11 @@ class PromptStudioHelper:
TSPKeys.EVAL_SETTINGS_EVALUATE
] = prompt.evaluate
output[TSPKeys.EVAL_SETTINGS][TSPKeys.EVAL_SETTINGS_MONITOR_LLM] = [
- llm
+ monitor_llm
]
output[TSPKeys.EVAL_SETTINGS][
TSPKeys.EVAL_SETTINGS_EXCLUDE_FAILED
- ] = False
+ ] = tool.exclude_failed
for attr in dir(prompt):
if attr.startswith(TSPKeys.EVAL_METRIC_PREFIX):
attr_val = getattr(prompt, attr)
diff --git a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx
index a789b9bc0..a1b4b16fd 100644
--- a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx
+++ b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx
@@ -17,6 +17,7 @@ import {
} from "antd";
import PropTypes from "prop-types";
import { useEffect, useState } from "react";
+
import { getMenuItem, titleCase } from "../../../helpers/GetStaticData";
import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate";
import { useAlertStore } from "../../../store/alert-store";
diff --git a/frontend/src/components/custom-tools/settings-modal/SettingsModal.jsx b/frontend/src/components/custom-tools/settings-modal/SettingsModal.jsx
index 31d896fda..46c26d05e 100644
--- a/frontend/src/components/custom-tools/settings-modal/SettingsModal.jsx
+++ b/frontend/src/components/custom-tools/settings-modal/SettingsModal.jsx
@@ -1,25 +1,29 @@
import { Col, Menu, Modal, Row, Typography } from "antd";
import PropTypes from "prop-types";
-
-import SpaceWrapper from "../../widgets/space-wrapper/SpaceWrapper";
-import { getMenuItem } from "../../../helpers/GetStaticData";
import { useEffect, useState } from "react";
-import { ManageLlmProfiles } from "../manage-llm-profiles/ManageLlmProfiles";
-import { useCustomToolStore } from "../../../store/custom-tool-store";
-import { CustomSynonyms } from "../custom-synonyms/CustomSynonyms";
-import { PreAndPostAmbleModal } from "../pre-and-post-amble-modal/PreAndPostAmbleModal";
import {
CodeOutlined,
DiffOutlined,
FileTextOutlined,
MessageOutlined,
} from "@ant-design/icons";
+
+import SpaceWrapper from "../../widgets/space-wrapper/SpaceWrapper";
+import { getMenuItem } from "../../../helpers/GetStaticData";
+import { ManageLlmProfiles } from "../manage-llm-profiles/ManageLlmProfiles";
+import { useCustomToolStore } from "../../../store/custom-tool-store";
+import { CustomSynonyms } from "../custom-synonyms/CustomSynonyms";
+import { PreAndPostAmbleModal } from "../pre-and-post-amble-modal/PreAndPostAmbleModal";
+
import "./SettingsModal.css";
let SummarizeManager = null;
+let EvaluationManager = null;
try {
SummarizeManager =
require("../../../plugins/summarize-manager/SummarizeManager").SummarizeManager;
+ EvaluationManager =
+ require("../../../plugins/evaluation-manager/EvaluationManager").EvaluationManager;
} catch {
// Component will remain null if it is not present.
}
@@ -33,21 +37,21 @@ function SettingsModal({ open, setOpen, handleUpdateTool }) {
useEffect(() => {
const items = [
getMenuItem("Manage LLM Profiles", 1, <CodeOutlined />),
- getMenuItem("Manage Grammar", 3, <MessageOutlined />),
- getMenuItem("Preamble", 4, <DiffOutlined />),
- getMenuItem("Postamble", 5, <DiffOutlined />),
+ getMenuItem("Manage Grammar", 4, <MessageOutlined />),
+ getMenuItem("Preamble", 5, <DiffOutlined />),
+ getMenuItem("Postamble", 6, <DiffOutlined />),
];
const listOfComponents = {
1: <ManageLlmProfiles />,
- 3: <CustomSynonyms />,
- 4: (
+ 4: <CustomSynonyms />,
+ 5: (
<PreAndPostAmbleModal
type="PREAMBLE"
handleUpdateTool={handleUpdateTool}
/>
),
- 5: (
+ 6: (
<PreAndPostAmbleModal
type="POSTAMBLE"
handleUpdateTool={handleUpdateTool}
@@ -55,12 +59,15 @@ function SettingsModal({ open, setOpen, handleUpdateTool }) {
),
};
- if (SummarizeManager) {
+ if (SummarizeManager && EvaluationManager) {
// Add Summary Manager menu item to the existing list
items.splice(
1,
0,
- getMenuItem("Summary Manager", 2, <FileTextOutlined />)
+ ...[
+ getMenuItem("Summary Manager", 2, <FileTextOutlined />),
+ getMenuItem("Evaluation Manager", 3, <FileTextOutlined />),
+ ]
);
listOfComponents[2] = (
<SummarizeManager
@@ -68,6 +75,9 @@ function SettingsModal({ open, setOpen, handleUpdateTool }) {
handleUpdateTool={handleUpdateTool}
/>
);
+ listOfComponents[3] = (
+ <EvaluationManager handleUpdateTool={handleUpdateTool} />
+ );
}
setMenuItems(items);