LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Evaluation LLM Settings (#99)
Deepak K8d31eb77efae
diff --git a/backend/prompt_studio/prompt_studio_core/migrations/0008_customtool_exclude_failed_customtool_monitor_llm.py b/backend/prompt_studio/prompt_studio_core/migrations/0008_customtool_exclude_failed_customtool_monitor_llm.pynew file mode 100644index 000000000..3173d5604--- /dev/null+++ b/backend/prompt_studio/prompt_studio_core/migrations/0008_customtool_exclude_failed_customtool_monitor_llm.py@@ -0,0 +1,33 @@+# Generated by Django 4.2.1 on 2024-03-13 23:54++import django.db.models.deletion+from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("adapter_processor", "0005_alter_adapterinstance_adapter_type"),+ (+ "prompt_studio_core",+ "0007_remove_customtool_default_profile_and_more",+ ),+ ]++ operations = [+ migrations.AddField(+ model_name="customtool",+ name="exclude_failed",+ field=models.BooleanField(default=True),+ ),+ migrations.AddField(+ model_name="customtool",+ name="monitor_llm",+ field=models.ForeignKey(+ blank=True,+ db_comment="Field to store monitor llm",+ null=True,+ on_delete=django.db.models.deletion.PROTECT,+ to="adapter_processor.adapterinstance",+ ),+ ),+ ]diff --git a/backend/prompt_studio/prompt_studio_core/models.py b/backend/prompt_studio/prompt_studio_core/models.pyindex 9dbda4a6d..c3677dbb7 100644--- a/backend/prompt_studio/prompt_studio_core/models.py+++ b/backend/prompt_studio/prompt_studio_core/models.py@@ -1,6 +1,7 @@import uuidfrom account.models import User+from adapter_processor.models import AdapterInstancefrom django.core.exceptions import ObjectDoesNotExistfrom django.db import modelsfrom prompt_studio.prompt_studio_core.exceptions import DefaultProfileError@@ -54,6 +55,13 @@ class CustomTool(BaseModel):prompt_grammer = models.JSONField(null=True, blank=True, db_comment="Synonymous words used in prompt")+ monitor_llm = models.ForeignKey(+ AdapterInstance,+ on_delete=models.PROTECT,+ db_comment="Field to store monitor llm",+ null=True,+ blank=True,+ )created_by = models.ForeignKey(User,on_delete=models.SET_NULL,@@ -70,6 +78,7 @@ class CustomTool(BaseModel):blank=True,editable=False,)+ exclude_failed = models.BooleanField(default=True)# TODO: Add ProfileManager to return typedef get_default_llm_profile(self): # type: ignorediff --git a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.pyindex 9895f80a9..2fa311813 100644--- a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py+++ b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py@@ -4,6 +4,7 @@ import osfrom pathlib import Pathfrom typing import Any, Optional+from adapter_processor.models import AdapterInstancefrom django.conf import settingsfrom file_management.file_management_helper import FileManagerHelperfrom prompt_studio.prompt_profile_manager.models import ProfileManager@@ -289,12 +290,25 @@ class PromptStudioHelper:Raises:AnswerFetchError"""- outputs: list[dict[str, Any]] = []-+ monitor_llm_instance: Optional[AdapterInstance] = tool.monitor_llm+ monitor_llm: Optional[str] = None+ if monitor_llm_instance:+ monitor_llm = str(monitor_llm_instance.id)prompt_grammer = tool.prompt_grammer+ outputs: list[dict[str, Any]] = []grammer_dict = {}grammar_list = []+ # Using default profile manager llm if monitor_llm is None+ if monitor_llm:+ monitor_llm = str(monitor_llm_instance.id)+ else:+ # TODO: Use CustomTool model to get profile_manager+ profile_manager = ProfileManager.objects.get(+ prompt_studio_tool=tool, is_default=True+ )+ monitor_llm = str(profile_manager.llm.id)+# Adding validationsif prompt_grammer:for word, synonyms in prompt_grammer.items():@@ -355,11 +369,11 @@ class PromptStudioHelper:TSPKeys.EVAL_SETTINGS_EVALUATE] = prompt.evaluateoutput[TSPKeys.EVAL_SETTINGS][TSPKeys.EVAL_SETTINGS_MONITOR_LLM] = [- llm+ monitor_llm]output[TSPKeys.EVAL_SETTINGS][TSPKeys.EVAL_SETTINGS_EXCLUDE_FAILED- ] = False+ ] = tool.exclude_failedfor attr in dir(prompt):if attr.startswith(TSPKeys.EVAL_METRIC_PREFIX):attr_val = getattr(prompt, attr)diff --git a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsxindex a789b9bc0..a1b4b16fd 100644--- a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx+++ b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx@@ -17,6 +17,7 @@ import {} from "antd";import PropTypes from "prop-types";import { useEffect, useState } from "react";+import { getMenuItem, titleCase } from "../../../helpers/GetStaticData";import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate";import { useAlertStore } from "../../../store/alert-store";diff --git a/frontend/src/components/custom-tools/settings-modal/SettingsModal.jsx b/frontend/src/components/custom-tools/settings-modal/SettingsModal.jsxindex 31d896fda..46c26d05e 100644--- a/frontend/src/components/custom-tools/settings-modal/SettingsModal.jsx+++ b/frontend/src/components/custom-tools/settings-modal/SettingsModal.jsx@@ -1,25 +1,29 @@import { Col, Menu, Modal, Row, Typography } from "antd";import PropTypes from "prop-types";--import SpaceWrapper from "../../widgets/space-wrapper/SpaceWrapper";-import { getMenuItem } from "../../../helpers/GetStaticData";import { useEffect, useState } from "react";-import { ManageLlmProfiles } from "../manage-llm-profiles/ManageLlmProfiles";-import { useCustomToolStore } from "../../../store/custom-tool-store";-import { CustomSynonyms } from "../custom-synonyms/CustomSynonyms";-import { PreAndPostAmbleModal } from "../pre-and-post-amble-modal/PreAndPostAmbleModal";import {CodeOutlined,DiffOutlined,FileTextOutlined,MessageOutlined,} from "@ant-design/icons";++import SpaceWrapper from "../../widgets/space-wrapper/SpaceWrapper";+import { getMenuItem } from "../../../helpers/GetStaticData";+import { ManageLlmProfiles } from "../manage-llm-profiles/ManageLlmProfiles";+import { useCustomToolStore } from "../../../store/custom-tool-store";+import { CustomSynonyms } from "../custom-synonyms/CustomSynonyms";+import { PreAndPostAmbleModal } from "../pre-and-post-amble-modal/PreAndPostAmbleModal";+import "./SettingsModal.css";let SummarizeManager = null;+let EvaluationManager = null;try {SummarizeManager =require("../../../plugins/summarize-manager/SummarizeManager").SummarizeManager;+ EvaluationManager =+ require("../../../plugins/evaluation-manager/EvaluationManager").EvaluationManager;} catch {// Component will remain null if it is not present.}@@ -33,21 +37,21 @@ function SettingsModal({ open, setOpen, handleUpdateTool }) {useEffect(() => {const items = [getMenuItem("Manage LLM Profiles", 1, <CodeOutlined />),- getMenuItem("Manage Grammar", 3, <MessageOutlined />),- getMenuItem("Preamble", 4, <DiffOutlined />),- getMenuItem("Postamble", 5, <DiffOutlined />),+ getMenuItem("Manage Grammar", 4, <MessageOutlined />),+ getMenuItem("Preamble", 5, <DiffOutlined />),+ getMenuItem("Postamble", 6, <DiffOutlined />),];const listOfComponents = {1: <ManageLlmProfiles />,- 3: <CustomSynonyms />,- 4: (+ 4: <CustomSynonyms />,+ 5: (<PreAndPostAmbleModaltype="PREAMBLE"handleUpdateTool={handleUpdateTool}/>),- 5: (+ 6: (<PreAndPostAmbleModaltype="POSTAMBLE"handleUpdateTool={handleUpdateTool}@@ -55,12 +59,15 @@ function SettingsModal({ open, setOpen, handleUpdateTool }) {),};- if (SummarizeManager) {+ if (SummarizeManager && EvaluationManager) {// Add Summary Manager menu item to the existing listitems.splice(1,0,- getMenuItem("Summary Manager", 2, <FileTextOutlined />)+ ...[+ getMenuItem("Summary Manager", 2, <FileTextOutlined />),+ getMenuItem("Evaluation Manager", 3, <FileTextOutlined />),+ ]);listOfComponents[2] = (<SummarizeManager@@ -68,6 +75,9 @@ function SettingsModal({ open, setOpen, handleUpdateTool }) {handleUpdateTool={handleUpdateTool}/>);+ listOfComponents[3] = (+ <EvaluationManager handleUpdateTool={handleUpdateTool} />+ );}setMenuItems(items);