LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-3008 [FEAT] Add word-level confidence support (#1672)
Deepak K62611ce34a00
diff --git a/backend/prompt_studio/prompt_studio_core_v2/constants.py b/backend/prompt_studio/prompt_studio_core_v2/constants.pyindex 2c154cffd..0c846bdbb 100644--- a/backend/prompt_studio/prompt_studio_core_v2/constants.py+++ b/backend/prompt_studio/prompt_studio_core_v2/constants.py@@ -93,11 +93,13 @@ class ToolStudioPromptKeys:TXT_EXTENTION = ".txt"TABLE = "table"PLATFORM_POSTAMBLE = "platform_postamble"+ WORD_CONFIDENCE_POSTAMBLE = "word_confidence_postamble"SUMMARIZE_AS_SOURCE = "summarize_as_source"VARIABLE_MAP = "variable_map"RECORD = "record"FILE_PATH = "file_path"ENABLE_HIGHLIGHT = "enable_highlight"+ ENABLE_WORD_CONFIDENCE = "enable_word_confidence"REQUIRED = "required"EXECUTION_SOURCE = "execution_source"LINE_ITEM = "line-item"@@ -166,6 +168,7 @@ class IndexingConstants:FILE_HASH = "file_hash"OUTPUT_FILE_PATH = "output_file_path"ENABLE_HIGHLIGHT = "enable_highlight"+ ENABLE_WORD_CONFIDENCE = "enable_word_confidence"USAGE_KWARGS = "usage_kwargs"PROCESS_TEXT = "process_text"EXTRACTED_TEXT = "extracted_text"@@ -190,6 +193,7 @@ class DefaultValues:DEFAULT_EXCLUDE_FAILED = TrueDEFAULT_ENABLE_CHALLENGE = FalseDEFAULT_ENABLE_HIGHLIGHT = False+ DEFAULT_ENABLE_WORD_CONFIDENCE = FalseDEFAULT_SUMMARIZE_AS_SOURCE = FalseDEFAULT_SUMMARIZE_CONTEXT = FalseDEFAULT_SINGLE_PASS_EXTRACTION_MODE = Falsediff --git a/backend/prompt_studio/prompt_studio_core_v2/migrations/0005_customtool_enable_word_confidence.py b/backend/prompt_studio/prompt_studio_core_v2/migrations/0005_customtool_enable_word_confidence.pynew file mode 100644index 000000000..6839e7683--- /dev/null+++ b/backend/prompt_studio/prompt_studio_core_v2/migrations/0005_customtool_enable_word_confidence.py@@ -0,0 +1,20 @@+# Generated by Django 4.2.1 on 2025-11-18 05:07++from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("prompt_studio_core_v2", "0004_add_shared_to_org_to_custom_tool"),+ ]++ operations = [+ migrations.AddField(+ model_name="customtool",+ name="enable_word_confidence",+ field=models.BooleanField(+ db_comment="Flag to enable or disable word-level confidence (depends on enable_highlight)",+ default=False,+ ),+ ),+ ]diff --git a/backend/prompt_studio/prompt_studio_core_v2/models.py b/backend/prompt_studio/prompt_studio_core_v2/models.pyindex 819dbfc83..c2ca4ac7b 100644--- a/backend/prompt_studio/prompt_studio_core_v2/models.py+++ b/backend/prompt_studio/prompt_studio_core_v2/models.py@@ -138,6 +138,10 @@ class CustomTool(DefaultOrganizationMixin, BaseModel):enable_highlight = models.BooleanField(db_comment="Flag to enable or disable document highlighting", default=False)+ enable_word_confidence = models.BooleanField(+ db_comment="Flag to enable or disable word-level confidence (depends on enable_highlight)",+ default=False,+ )# Introduced field to establish M2M relation between users and custom_tool.# This will introduce intermediary table which relates both the models.diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.pyindex 4de66d2d8..1481ba135 100644--- a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py+++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py@@ -412,6 +412,8 @@ class PromptStudioHelper:document_id=document_id,run_id=run_id,enable_highlight=tool.enable_highlight,+ enable_word_confidence=tool.enable_word_confidence,+ doc_id=doc_id,)if tool.summarize_context:summarize_file_path = PromptStudioHelper.summarize(@@ -972,9 +974,13 @@ class PromptStudioHelper:tool_settings[TSPKeys.POSTAMBLE] = tool.postambletool_settings[TSPKeys.GRAMMAR] = grammar_listtool_settings[TSPKeys.ENABLE_HIGHLIGHT] = tool.enable_highlight+ tool_settings[TSPKeys.ENABLE_WORD_CONFIDENCE] = tool.enable_word_confidencetool_settings[TSPKeys.PLATFORM_POSTAMBLE] = getattr(settings, TSPKeys.PLATFORM_POSTAMBLE.upper(), "")+ tool_settings[TSPKeys.WORD_CONFIDENCE_POSTAMBLE] = getattr(+ settings, TSPKeys.WORD_CONFIDENCE_POSTAMBLE.upper(), ""+ )file_hash = fs_instance.get_hash_from_file(path=doc_path)payload = {@@ -1246,10 +1252,14 @@ class PromptStudioHelper:tool_settings[TSPKeys.CHUNK_OVERLAP] = default_profile.chunk_overlaptool_settings[TSPKeys.ENABLE_CHALLENGE] = tool.enable_challengetool_settings[TSPKeys.ENABLE_HIGHLIGHT] = tool.enable_highlight+ tool_settings[TSPKeys.ENABLE_WORD_CONFIDENCE] = tool.enable_word_confidencetool_settings[TSPKeys.CHALLENGE_LLM] = challenge_llmtool_settings[TSPKeys.PLATFORM_POSTAMBLE] = getattr(settings, TSPKeys.PLATFORM_POSTAMBLE.upper(), "")+ tool_settings[TSPKeys.WORD_CONFIDENCE_POSTAMBLE] = getattr(+ settings, TSPKeys.WORD_CONFIDENCE_POSTAMBLE.upper(), ""+ )tool_settings[TSPKeys.SUMMARIZE_AS_SOURCE] = tool.summarize_as_sourcefor prompt in prompts:if not prompt.prompt:diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/migrations/0004_promptstudiooutputmanager_word_confidence_data.py b/backend/prompt_studio/prompt_studio_output_manager_v2/migrations/0004_promptstudiooutputmanager_word_confidence_data.pynew file mode 100644index 000000000..027b2151f--- /dev/null+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/migrations/0004_promptstudiooutputmanager_word_confidence_data.py@@ -0,0 +1,24 @@+# Generated migration for word_confidence_data field++from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ (+ "prompt_studio_output_manager_v2",+ "0003_promptstudiooutputmanager_confidence_data",+ ),+ ]++ operations = [+ migrations.AddField(+ model_name="promptstudiooutputmanager",+ name="word_confidence_data",+ field=models.JSONField(+ blank=True,+ db_comment="Field to store word-level confidence data",+ null=True,+ ),+ ),+ ]diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/models.py b/backend/prompt_studio/prompt_studio_output_manager_v2/models.pyindex e18cb5698..7b8616968 100644--- a/backend/prompt_studio/prompt_studio_output_manager_v2/models.py+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/models.py@@ -37,6 +37,12 @@ class PromptStudioOutputManager(BaseModel):null=True,blank=True,)+ word_confidence_data = models.JSONField(+ db_comment="Field to store word-level confidence data",+ editable=True,+ null=True,+ blank=True,+ )eval_metrics = models.JSONField(db_column="eval_metrics",null=False,diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.pyindex f20d3c88d..a72d808a1 100644--- a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py@@ -63,6 +63,7 @@ class OutputManagerHelper:challenge_data: dict[str, Any] | None,highlight_data: dict[str, Any] | None,confidence_data: dict[str, Any] | None,+ word_confidence_data: dict[str, Any] | None,) -> PromptStudioOutputManager:"""Handles creating or updating a single prompt output and returnsthe instance.@@ -81,6 +82,7 @@ class OutputManagerHelper:"challenge_data": challenge_data,"highlight_data": highlight_data,"confidence_data": confidence_data,+ "word_confidence_data": word_confidence_data,},)@@ -103,6 +105,7 @@ class OutputManagerHelper:"challenge_data": challenge_data,"highlight_data": highlight_data,"confidence_data": confidence_data,+ "word_confidence_data": word_confidence_data,}PromptStudioOutputManager.objects.filter(document_manager=document_manager,@@ -126,6 +129,7 @@ class OutputManagerHelper:challenge_data = metadata.get("challenge_data")highlight_data = metadata.get("highlight_data")confidence_data = metadata.get("confidence_data")+ word_confidence_data = metadata.get("word_confidence_data")if not prompts:return serialized_data@@ -147,6 +151,8 @@ class OutputManagerHelper:highlight_data = highlight_data.get(prompt.prompt_key)if confidence_data:confidence_data = confidence_data.get(prompt.prompt_key)+ if word_confidence_data:+ word_confidence_data = word_confidence_data.get(prompt.prompt_key)if challenge_data:challenge_data = challenge_data.get(prompt.prompt_key)@@ -171,6 +177,7 @@ class OutputManagerHelper:challenge_data=challenge_data,highlight_data=highlight_data,confidence_data=confidence_data,+ word_confidence_data=word_confidence_data,)# Serialize the instancediff --git a/backend/prompt_studio/prompt_studio_registry_v2/constants.py b/backend/prompt_studio/prompt_studio_registry_v2/constants.pyindex 227b56bc5..3defb6092 100644--- a/backend/prompt_studio/prompt_studio_registry_v2/constants.py+++ b/backend/prompt_studio/prompt_studio_registry_v2/constants.py@@ -98,6 +98,7 @@ class JsonSchemaKey:SUMMARIZE_PROMPT = "summarize_prompt"SUMMARIZE_AS_SOURCE = "summarize_as_source"ENABLE_HIGHLIGHT = "enable_highlight"+ ENABLE_WORD_CONFIDENCE = "enable_word_confidence"PLATFORM_POSTAMBLE = "platform_postamble"REQUIRED = "required"# Webhook postprocessing settingsdiff --git a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.pyindex 724010ce1..6ce1f7209 100644--- a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py+++ b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py@@ -78,6 +78,12 @@ class PromptStudioRegistryHelper:"default": False,"description": "Enables highlight",},+ "enable_word_confidence": {+ "type": "boolean",+ "title": "Enable word confidence",+ "default": False,+ "description": "Enables word-level confidence (depends on enable_highlight)",+ },}spec = Spec(@@ -283,9 +289,13 @@ class PromptStudioRegistryHelper:tool.single_pass_extraction_mode)tool_settings[JsonSchemaKey.ENABLE_HIGHLIGHT] = tool.enable_highlight+ tool_settings[JsonSchemaKey.ENABLE_WORD_CONFIDENCE] = tool.enable_word_confidencetool_settings[JsonSchemaKey.PLATFORM_POSTAMBLE] = getattr(settings, JsonSchemaKey.PLATFORM_POSTAMBLE.upper(), "")+ tool_settings[JsonSchemaKey.WORD_CONFIDENCE_POSTAMBLE] = getattr(+ settings, JsonSchemaKey.WORD_CONFIDENCE_POSTAMBLE.upper(), ""+ )for prompt in prompts:if prompt.prompt_type == JsonSchemaKey.NOTES or not prompt.active:diff --git a/frontend/src/components/custom-tools/document-manager/DocumentManager.jsx b/frontend/src/components/custom-tools/document-manager/DocumentManager.jsxindex 17542a7bc..e1fb4015b 100644--- a/frontend/src/components/custom-tools/document-manager/DocumentManager.jsx+++ b/frontend/src/components/custom-tools/document-manager/DocumentManager.jsx@@ -380,6 +380,23 @@ function DocumentManager({ generateIndex, handleUpdateTool, handleDocChange }) {if (typeof confidence === "number") {return confidence.toFixed(2);}+ // Handle word confidence format: object with line numbers as keys+ if (+ confidence &&+ typeof confidence === "object" &&+ !Array.isArray(confidence)+ ) {+ const values = Object.values(confidence);+ if (+ values.length > 0 &&+ values.every((v) => typeof v === "number")+ ) {+ const avg =+ values.reduce((sum, val) => sum + val, 0) /+ values.length;+ return avg.toFixed(2);+ }+ }// Handle old format: nested array structureif (confidence?.[0]?.[0]?.confidence) {return confidence[0][0].confidence;diff --git a/frontend/src/components/custom-tools/prompt-card/DisplayPromptResult.jsx b/frontend/src/components/custom-tools/prompt-card/DisplayPromptResult.jsxindex ffe59057d..808aca590 100644--- a/frontend/src/components/custom-tools/prompt-card/DisplayPromptResult.jsx+++ b/frontend/src/components/custom-tools/prompt-card/DisplayPromptResult.jsx@@ -22,6 +22,7 @@ function DisplayPromptResult({highlightData,promptDetails,confidenceData,+ wordConfidenceData,isTable = false,setOpenExpandModal = () => {},}) {@@ -79,13 +80,21 @@ function DisplayPromptResult({);}- const handleClick = (highlightData, confidenceData, key, keyPath) => {+ const handleClick = (+ highlightData,+ confidenceData,+ wordConfidenceData,+ key,+ keyPath+ ) => {if (highlightData?.[key]) {+ // Use word confidence if available, otherwise fallback to line confidence+ const confidence = wordConfidenceData?.[key] || confidenceData?.[key];handleSelectHighlight(highlightData[key],promptDetails?.prompt_id,profileId,- confidenceData?.[key]+ confidence);setSelectedKey(keyPath);}@@ -98,6 +107,7 @@ function DisplayPromptResult({data,highlightData,confidenceData,+ wordConfidenceData,indent = 0,path = "",isTable = false@@ -146,6 +156,7 @@ function DisplayPromptResult({item,highlightData?.[index],confidenceData?.[index],+ wordConfidenceData?.[index],indent + 1,`${path}[${index}]`,isTable@@ -183,6 +194,7 @@ function DisplayPromptResult({handleClick(highlightData,confidenceData,+ wordConfidenceData,key,newPath);@@ -193,6 +205,7 @@ function DisplayPromptResult({value,highlightData?.[key],confidenceData?.[key],+ wordConfidenceData?.[key],indent + 1,newPath,isTable@@ -214,7 +227,15 @@ function DisplayPromptResult({return (<Typography.Paragraph className="prompt-card-display-output font-size-12">{parsedOutput && typeof parsedOutput === "object" ? (- renderJson(parsedOutput, highlightData, confidenceData, 0, "", isTable)+ renderJson(+ parsedOutput,+ highlightData,+ confidenceData,+ wordConfidenceData,+ 0,+ "",+ isTable+ )) : (<TextResultenableHighlight={details?.enable_highlight}@@ -222,6 +243,7 @@ function DisplayPromptResult({promptId={promptDetails?.prompt_id}profileId={profileId}confidenceData={confidenceData}+ wordConfidenceData={wordConfidenceData}selectedHighlight={selectedHighlight}parsedOutput={parsedOutput}onSelectHighlight={handleSelectHighlight}@@ -237,15 +259,19 @@ const TextResult = ({promptId,profileId,confidenceData,+ wordConfidenceData,selectedHighlight,parsedOutput,onSelectHighlight,}) => {+ // Use word confidence if available, otherwise fallback to line confidence+ const confidence = wordConfidenceData || confidenceData;+return enableHighlight ? (<Typography.TextwraponClick={() =>- onSelectHighlight(highlightData, promptId, profileId, confidenceData)+ onSelectHighlight(highlightData, promptId, profileId, confidence)}className={`prompt-output-result json-value ${highlightData ? "clickable" : ""@@ -264,6 +290,7 @@ TextResult.propTypes = {promptId: PropTypes.string,profileId: PropTypes.string,confidenceData: PropTypes.any,+ wordConfidenceData: PropTypes.any,selectedHighlight: PropTypes.object,parsedOutput: PropTypes.any,onSelectHighlight: PropTypes.func.isRequired,@@ -277,6 +304,7 @@ DisplayPromptResult.propTypes = {highlightData: PropTypes.object,promptDetails: PropTypes.object,confidenceData: PropTypes.object,+ wordConfidenceData: PropTypes.object,isTable: PropTypes.bool,setOpenExpandModal: PropTypes.func,};diff --git a/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx b/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsxindex b3f05fa0a..da959ea11 100644--- a/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx+++ b/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx@@ -19,6 +19,7 @@ import {} from "antd";import { motion, AnimatePresence } from "framer-motion";import CheckableTag from "antd/es/tag/CheckableTag";+import { useState } from "react";import {displayPromptResult,@@ -37,7 +38,6 @@ import { DisplayPromptResult } from "./DisplayPromptResult";import usePromptOutput from "../../../hooks/usePromptOutput";import { PromptRunTimer } from "./PromptRunTimer";import { PromptRunCost } from "./PromptRunCost";-import { useState } from "react";let TableOutput;try {@@ -119,6 +119,7 @@ function PromptOutput({handleSelectHighlight={handleSelectHighlight}highlightData={promptOutputData?.highlightData}confidenceData={promptOutputData?.confidenceData}+ wordConfidenceData={promptOutputData?.wordConfidenceData}promptDetails={promptDetails}isTable={true}setOpenExpandModal={setOpenExpandModal}@@ -211,6 +212,9 @@ function PromptOutput({confidenceData={promptOutputData?.confidenceData?.[promptDetails.prompt_key]}+ wordConfidenceData={+ promptOutputData?.wordConfidenceData?.[promptDetails.prompt_key]+ }/><div className="prompt-profile-run"><CopyPromptOutputBtn@@ -467,6 +471,9 @@ function PromptOutput({handleSelectHighlight={handleSelectHighlight}highlightData={promptOutputData?.highlightData}confidenceData={promptOutputData?.confidenceData}+ wordConfidenceData={+ promptOutputData?.wordConfidenceData+ }promptDetails={promptDetails}/><div className="prompt-profile-run">diff --git a/frontend/src/hooks/usePromptOutput.js b/frontend/src/hooks/usePromptOutput.jsindex 42de48453..eafe41521 100644--- a/frontend/src/hooks/usePromptOutput.js+++ b/frontend/src/hooks/usePromptOutput.js@@ -121,6 +121,7 @@ const usePromptOutput = () => {coverage: item?.coverage,highlightData: item?.highlight_data,confidenceData: item?.confidence_data,+ wordConfidenceData: item?.word_confidence_data,};if (item?.is_single_pass_extract && isTokenUsageForSinglePassAdded)diff --git a/prompt-service/src/unstract/prompt_service/constants.py b/prompt-service/src/unstract/prompt_service/constants.pyindex 61f3cb3d5..9eddab842 100644--- a/prompt-service/src/unstract/prompt_service/constants.py+++ b/prompt-service/src/unstract/prompt_service/constants.py@@ -63,6 +63,7 @@ class PromptServiceConstants:TABLE_SETTINGS = "table_settings"EPILOGUE = "epilogue"PLATFORM_POSTAMBLE = "platform_postamble"+ WORD_CONFIDENCE_POSTAMBLE = "word_confidence_postamble"HIGHLIGHT_DATA_PLUGIN = "highlight-data"SUMMARIZE_AS_SOURCE = "summarize_as_source"VARIABLE_MAP = "variable_map"@@ -70,9 +71,11 @@ class PromptServiceConstants:CUSTOM_DATA = "custom_data"TEXT = "text"ENABLE_HIGHLIGHT = "enable_highlight"+ ENABLE_WORD_CONFIDENCE = "enable_word_confidence"FILE_PATH = "file_path"HIGHLIGHT_DATA = "highlight_data"CONFIDENCE_DATA = "confidence_data"+ WORD_CONFIDENCE_DATA = "word_confidence_data"REQUIRED_FIELDS = "required_fields"REQUIRED = "required"EXECUTION_SOURCE = "execution_source"@@ -188,6 +191,7 @@ class IndexingConstants:FILE_HASH = "file_hash"OUTPUT_FILE_PATH = "output_file_path"ENABLE_HIGHLIGHT = "enable_highlight"+ ENABLE_WORD_CONFIDENCE = "enable_word_confidence"USAGE_KWARGS = "usage_kwargs"PROCESS_TEXT = "process_text"EXTRACTED_TEXT = "extracted_text"diff --git a/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py b/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.pyindex 58a650946..010637a86 100644--- a/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py+++ b/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py@@ -502,6 +502,9 @@ def prompt_processor() -> Any:doc_name=doc_name,llm=llm,enable_highlight=tool_settings.get(PSKeys.ENABLE_HIGHLIGHT, False),+ enable_word_confidence=tool_settings.get(+ PSKeys.ENABLE_WORD_CONFIDENCE, False+ ),execution_source=execution_source,metadata=metadata,file_path=file_path,diff --git a/prompt-service/src/unstract/prompt_service/controllers/indexing.py b/prompt-service/src/unstract/prompt_service/controllers/indexing.pyindex 84c75a117..f9416a9c9 100644--- a/prompt-service/src/unstract/prompt_service/controllers/indexing.py+++ b/prompt-service/src/unstract/prompt_service/controllers/indexing.py@@ -60,6 +60,7 @@ def index() -> Any:chunk_overlap: int = payload.get(IKeys.CHUNK_OVERLAP, 128) # Default chunk overlapreindex: bool = payload.get(IKeys.REINDEX, False)enable_highlight: bool = payload.get(IKeys.ENABLE_HIGHLIGHT, False)+ enable_word_confidence: bool = payload.get(IKeys.ENABLE_WORD_CONFIDENCE, False)usage_kwargs: dict[Any, Any] = payload.get(IKeys.USAGE_KWARGS, {})extracted_text: str = payload.get(IKeys.EXTRACTED_TEXT, "")tags: list[str] = payload.get(IKeys.TAGS, None)@@ -80,7 +81,10 @@ def index() -> Any:chunking_config = ChunkingConfig(chunk_size=chunk_size, chunk_overlap=chunk_overlap)processing_options = ProcessingOptions(- reindex=reindex, enable_highlight=enable_highlight, usage_kwargs=usage_kwargs+ reindex=reindex,+ enable_highlight=enable_highlight,+ enable_word_confidence=enable_word_confidence,+ usage_kwargs=usage_kwargs,)doc_id = IndexingService.index(chunking_config=chunking_config,diff --git a/prompt-service/src/unstract/prompt_service/dto.py b/prompt-service/src/unstract/prompt_service/dto.pyindex ed8888153..8c9e4f3d3 100644--- a/prompt-service/src/unstract/prompt_service/dto.py+++ b/prompt-service/src/unstract/prompt_service/dto.py@@ -35,4 +35,5 @@ class ChunkingConfig:class ProcessingOptions:reindex: bool = Falseenable_highlight: bool = False+ enable_word_confidence: bool = Falseusage_kwargs: dict[Any, Any] = field(default_factory=dict)diff --git a/prompt-service/src/unstract/prompt_service/services/answer_prompt.py b/prompt-service/src/unstract/prompt_service/services/answer_prompt.pyindex 3bc93051b..df2c0daed 100644--- a/prompt-service/src/unstract/prompt_service/services/answer_prompt.py+++ b/prompt-service/src/unstract/prompt_service/services/answer_prompt.py@@ -115,11 +115,20 @@ class AnswerPromptService:execution_source: str | None = ExecutionSource.IDE.value,) -> str:platform_postamble = tool_settings.get(PSKeys.PLATFORM_POSTAMBLE, "")+ word_confidence_postamble = tool_settings.get(+ PSKeys.WORD_CONFIDENCE_POSTAMBLE, ""+ )summarize_as_source = tool_settings.get(PSKeys.SUMMARIZE_AS_SOURCE)enable_highlight = tool_settings.get(PSKeys.ENABLE_HIGHLIGHT, False)+ enable_word_confidence = tool_settings.get(PSKeys.ENABLE_WORD_CONFIDENCE, False)+ # Dependency: enable_word_confidence only works if enable_highlight is enabled+ if not enable_highlight:+ enable_word_confidence = Falseprompt_type = output.get(PSKeys.TYPE, PSKeys.TEXT)if not enable_highlight or summarize_as_source:platform_postamble = ""+ if not enable_word_confidence or summarize_as_source:+ word_confidence_postamble = ""plugin = PluginManager().get_plugin("json-extraction")if plugin and hasattr(plugin["entrypoint_cls"], "update_settings"):plugin["entrypoint_cls"].update_settings(tool_settings, output)@@ -130,6 +139,7 @@ class AnswerPromptService:grammar_list=tool_settings.get(PSKeys.GRAMMAR, []),context=context,platform_postamble=platform_postamble,+ word_confidence_postamble=word_confidence_postamble,prompt_type=prompt_type,)output[PSKeys.COMBINED_PROMPT] = prompt@@ -140,6 +150,7 @@ class AnswerPromptService:prompt_key=output[PSKeys.NAME],prompt_type=prompt_type,enable_highlight=enable_highlight,+ enable_word_confidence=enable_word_confidence,file_path=file_path,execution_source=execution_source,)@@ -152,6 +163,7 @@ class AnswerPromptService:grammar_list: list[dict[str, Any]],context: str,platform_postamble: str,+ word_confidence_postamble: str,prompt_type: str = PSKeys.TEXT,) -> str:prompt = f"{preamble}\n\nQuestion or Instruction: {prompt}"@@ -176,6 +188,8 @@ class AnswerPromptService:postamble += f"\n{json_postamble}"if platform_postamble:platform_postamble += "\n\n"+ if word_confidence_postamble:+ platform_postamble += f"{word_confidence_postamble}\n\n"prompt += (f"\n\n{postamble}\n\nContext:\n---------------\n{context}\n"f"-----------------\n\n{platform_postamble}Answer:"@@ -190,6 +204,7 @@ class AnswerPromptService:prompt_key: str | None = None,prompt_type: str | None = PSKeys.TEXT,enable_highlight: bool = False,+ enable_word_confidence: bool = False,file_path: str = "",execution_source: str | None = None,) -> str:@@ -220,6 +235,7 @@ class AnswerPromptService:highlight_data = highlight_data_plugin["entrypoint_cls"](file_path=file_path,fs_instance=fs_instance,+ enable_word_confidence=enable_word_confidence,).runcompletion = llm.complete(prompt=prompt,@@ -229,6 +245,7 @@ class AnswerPromptService:answer: str = completion[PSKeys.RESPONSE].texthighlight_data = completion.get(PSKeys.HIGHLIGHT_DATA, [])confidence_data = completion.get(PSKeys.CONFIDENCE_DATA)+ word_confidence_data = completion.get(PSKeys.WORD_CONFIDENCE_DATA)line_numbers = completion.get(PSKeys.LINE_NUMBERS, [])whisper_hash = completion.get(PSKeys.WHISPER_HASH, "")if metadata is not None and prompt_key:@@ -241,6 +258,10 @@ class AnswerPromptService:metadata.setdefault(PSKeys.CONFIDENCE_DATA, {})[prompt_key] = (confidence_data)+ if enable_word_confidence and word_confidence_data:+ metadata.setdefault(PSKeys.WORD_CONFIDENCE_DATA, {})[prompt_key] = (+ word_confidence_data+ )return answer# TODO: Catch and handle specific exception hereexcept SdkRateLimitError as e:@@ -363,6 +384,7 @@ class AnswerPromptService:doc_name: str,llm: LLM,enable_highlight: bool = False,+ enable_word_confidence: bool = False,execution_source: str = ExecutionSource.IDE.value,metadata: dict[str, Any] | None = None,file_path: str = "",@@ -480,6 +502,7 @@ class AnswerPromptService:grammar_list=tool_settings.get(PSKeys.GRAMMAR, []),context=context,platform_postamble="",+ word_confidence_postamble="",)try:line_item_extraction = line_item_extraction_plugin["entrypoint_cls"](diff --git a/unstract/sdk1/src/unstract/sdk1/llm.py b/unstract/sdk1/src/unstract/sdk1/llm.pyindex 0a1eb01a8..c6d820106 100644--- a/unstract/sdk1/src/unstract/sdk1/llm.py+++ b/unstract/sdk1/src/unstract/sdk1/llm.py@@ -227,9 +227,9 @@ class LLM:# Improvements in readability are definitely welcome.extract_json: bool = cast("bool", kwargs.get("extract_json", False))post_process_fn: (- Callable[[LLMResponseCompat, bool], dict[str, object]] | None+ Callable[[LLMResponseCompat, bool, str], dict[str, object]] | None) = cast(- "Callable[[LLMResponseCompat, bool], dict[str, object]] | None",+ "Callable[[LLMResponseCompat, bool, str], dict[str, object]] | None",kwargs.get("process_text", None),)@@ -478,6 +478,9 @@ class LLM:) -> tuple[str, dict[str, object]]:post_processed_output: dict[str, object] = {}+ # Save original text before any modifications+ original_text = response_text+if extract_json:start = response_text.find(LLM.JSON_CONTENT_MARKER)if start != -1:@@ -494,7 +497,9 @@ class LLM:if post_process_fn:try:response_compat = LLMResponseCompat(response_text)- post_processed_output = post_process_fn(response_compat, extract_json)+ post_processed_output = post_process_fn(+ response_compat, extract_json, original_text+ )# Needed as the text is modified in place.response_text = response_compat.textexcept Exception as e: