Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-3008 [FEAT] Add word-level confidence support (#1672)

Deepak K62611ce34a00
diff --git a/backend/prompt_studio/prompt_studio_core_v2/constants.py b/backend/prompt_studio/prompt_studio_core_v2/constants.py
index 2c154cffd..0c846bdbb 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/constants.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/constants.py
@@ -93,11 +93,13 @@ class ToolStudioPromptKeys:
TXT_EXTENTION = ".txt"
TABLE = "table"
PLATFORM_POSTAMBLE = "platform_postamble"
+ WORD_CONFIDENCE_POSTAMBLE = "word_confidence_postamble"
SUMMARIZE_AS_SOURCE = "summarize_as_source"
VARIABLE_MAP = "variable_map"
RECORD = "record"
FILE_PATH = "file_path"
ENABLE_HIGHLIGHT = "enable_highlight"
+ ENABLE_WORD_CONFIDENCE = "enable_word_confidence"
REQUIRED = "required"
EXECUTION_SOURCE = "execution_source"
LINE_ITEM = "line-item"
@@ -166,6 +168,7 @@ class IndexingConstants:
FILE_HASH = "file_hash"
OUTPUT_FILE_PATH = "output_file_path"
ENABLE_HIGHLIGHT = "enable_highlight"
+ ENABLE_WORD_CONFIDENCE = "enable_word_confidence"
USAGE_KWARGS = "usage_kwargs"
PROCESS_TEXT = "process_text"
EXTRACTED_TEXT = "extracted_text"
@@ -190,6 +193,7 @@ class DefaultValues:
DEFAULT_EXCLUDE_FAILED = True
DEFAULT_ENABLE_CHALLENGE = False
DEFAULT_ENABLE_HIGHLIGHT = False
+ DEFAULT_ENABLE_WORD_CONFIDENCE = False
DEFAULT_SUMMARIZE_AS_SOURCE = False
DEFAULT_SUMMARIZE_CONTEXT = False
DEFAULT_SINGLE_PASS_EXTRACTION_MODE = False
diff --git a/backend/prompt_studio/prompt_studio_core_v2/migrations/0005_customtool_enable_word_confidence.py b/backend/prompt_studio/prompt_studio_core_v2/migrations/0005_customtool_enable_word_confidence.py
new file mode 100644
index 000000000..6839e7683
--- /dev/null
+++ b/backend/prompt_studio/prompt_studio_core_v2/migrations/0005_customtool_enable_word_confidence.py
@@ -0,0 +1,20 @@
+# Generated by Django 4.2.1 on 2025-11-18 05:07
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("prompt_studio_core_v2", "0004_add_shared_to_org_to_custom_tool"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="customtool",
+ name="enable_word_confidence",
+ field=models.BooleanField(
+ db_comment="Flag to enable or disable word-level confidence (depends on enable_highlight)",
+ default=False,
+ ),
+ ),
+ ]
diff --git a/backend/prompt_studio/prompt_studio_core_v2/models.py b/backend/prompt_studio/prompt_studio_core_v2/models.py
index 819dbfc83..c2ca4ac7b 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/models.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/models.py
@@ -138,6 +138,10 @@ class CustomTool(DefaultOrganizationMixin, BaseModel):
enable_highlight = models.BooleanField(
db_comment="Flag to enable or disable document highlighting", default=False
)
+ enable_word_confidence = models.BooleanField(
+ db_comment="Flag to enable or disable word-level confidence (depends on enable_highlight)",
+ default=False,
+ )
# Introduced field to establish M2M relation between users and custom_tool.
# This will introduce intermediary table which relates both the models.
diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
index 4de66d2d8..1481ba135 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
@@ -412,6 +412,8 @@ class PromptStudioHelper:
document_id=document_id,
run_id=run_id,
enable_highlight=tool.enable_highlight,
+ enable_word_confidence=tool.enable_word_confidence,
+ doc_id=doc_id,
)
if tool.summarize_context:
summarize_file_path = PromptStudioHelper.summarize(
@@ -972,9 +974,13 @@ class PromptStudioHelper:
tool_settings[TSPKeys.POSTAMBLE] = tool.postamble
tool_settings[TSPKeys.GRAMMAR] = grammar_list
tool_settings[TSPKeys.ENABLE_HIGHLIGHT] = tool.enable_highlight
+ tool_settings[TSPKeys.ENABLE_WORD_CONFIDENCE] = tool.enable_word_confidence
tool_settings[TSPKeys.PLATFORM_POSTAMBLE] = getattr(
settings, TSPKeys.PLATFORM_POSTAMBLE.upper(), ""
)
+ tool_settings[TSPKeys.WORD_CONFIDENCE_POSTAMBLE] = getattr(
+ settings, TSPKeys.WORD_CONFIDENCE_POSTAMBLE.upper(), ""
+ )
file_hash = fs_instance.get_hash_from_file(path=doc_path)
payload = {
@@ -1246,10 +1252,14 @@ class PromptStudioHelper:
tool_settings[TSPKeys.CHUNK_OVERLAP] = default_profile.chunk_overlap
tool_settings[TSPKeys.ENABLE_CHALLENGE] = tool.enable_challenge
tool_settings[TSPKeys.ENABLE_HIGHLIGHT] = tool.enable_highlight
+ tool_settings[TSPKeys.ENABLE_WORD_CONFIDENCE] = tool.enable_word_confidence
tool_settings[TSPKeys.CHALLENGE_LLM] = challenge_llm
tool_settings[TSPKeys.PLATFORM_POSTAMBLE] = getattr(
settings, TSPKeys.PLATFORM_POSTAMBLE.upper(), ""
)
+ tool_settings[TSPKeys.WORD_CONFIDENCE_POSTAMBLE] = getattr(
+ settings, TSPKeys.WORD_CONFIDENCE_POSTAMBLE.upper(), ""
+ )
tool_settings[TSPKeys.SUMMARIZE_AS_SOURCE] = tool.summarize_as_source
for prompt in prompts:
if not prompt.prompt:
diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/migrations/0004_promptstudiooutputmanager_word_confidence_data.py b/backend/prompt_studio/prompt_studio_output_manager_v2/migrations/0004_promptstudiooutputmanager_word_confidence_data.py
new file mode 100644
index 000000000..027b2151f
--- /dev/null
+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/migrations/0004_promptstudiooutputmanager_word_confidence_data.py
@@ -0,0 +1,24 @@
+# Generated migration for word_confidence_data field
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ (
+ "prompt_studio_output_manager_v2",
+ "0003_promptstudiooutputmanager_confidence_data",
+ ),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="promptstudiooutputmanager",
+ name="word_confidence_data",
+ field=models.JSONField(
+ blank=True,
+ db_comment="Field to store word-level confidence data",
+ null=True,
+ ),
+ ),
+ ]
diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/models.py b/backend/prompt_studio/prompt_studio_output_manager_v2/models.py
index e18cb5698..7b8616968 100644
--- a/backend/prompt_studio/prompt_studio_output_manager_v2/models.py
+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/models.py
@@ -37,6 +37,12 @@ class PromptStudioOutputManager(BaseModel):
null=True,
blank=True,
)
+ word_confidence_data = models.JSONField(
+ db_comment="Field to store word-level confidence data",
+ editable=True,
+ null=True,
+ blank=True,
+ )
eval_metrics = models.JSONField(
db_column="eval_metrics",
null=False,
diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py
index f20d3c88d..a72d808a1 100644
--- a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py
+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py
@@ -63,6 +63,7 @@ class OutputManagerHelper:
challenge_data: dict[str, Any] | None,
highlight_data: dict[str, Any] | None,
confidence_data: dict[str, Any] | None,
+ word_confidence_data: dict[str, Any] | None,
) -> PromptStudioOutputManager:
"""Handles creating or updating a single prompt output and returns
the instance.
@@ -81,6 +82,7 @@ class OutputManagerHelper:
"challenge_data": challenge_data,
"highlight_data": highlight_data,
"confidence_data": confidence_data,
+ "word_confidence_data": word_confidence_data,
},
)
@@ -103,6 +105,7 @@ class OutputManagerHelper:
"challenge_data": challenge_data,
"highlight_data": highlight_data,
"confidence_data": confidence_data,
+ "word_confidence_data": word_confidence_data,
}
PromptStudioOutputManager.objects.filter(
document_manager=document_manager,
@@ -126,6 +129,7 @@ class OutputManagerHelper:
challenge_data = metadata.get("challenge_data")
highlight_data = metadata.get("highlight_data")
confidence_data = metadata.get("confidence_data")
+ word_confidence_data = metadata.get("word_confidence_data")
if not prompts:
return serialized_data
@@ -147,6 +151,8 @@ class OutputManagerHelper:
highlight_data = highlight_data.get(prompt.prompt_key)
if confidence_data:
confidence_data = confidence_data.get(prompt.prompt_key)
+ if word_confidence_data:
+ word_confidence_data = word_confidence_data.get(prompt.prompt_key)
if challenge_data:
challenge_data = challenge_data.get(prompt.prompt_key)
@@ -171,6 +177,7 @@ class OutputManagerHelper:
challenge_data=challenge_data,
highlight_data=highlight_data,
confidence_data=confidence_data,
+ word_confidence_data=word_confidence_data,
)
# Serialize the instance
diff --git a/backend/prompt_studio/prompt_studio_registry_v2/constants.py b/backend/prompt_studio/prompt_studio_registry_v2/constants.py
index 227b56bc5..3defb6092 100644
--- a/backend/prompt_studio/prompt_studio_registry_v2/constants.py
+++ b/backend/prompt_studio/prompt_studio_registry_v2/constants.py
@@ -98,6 +98,7 @@ class JsonSchemaKey:
SUMMARIZE_PROMPT = "summarize_prompt"
SUMMARIZE_AS_SOURCE = "summarize_as_source"
ENABLE_HIGHLIGHT = "enable_highlight"
+ ENABLE_WORD_CONFIDENCE = "enable_word_confidence"
PLATFORM_POSTAMBLE = "platform_postamble"
REQUIRED = "required"
# Webhook postprocessing settings
diff --git a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py
index 724010ce1..6ce1f7209 100644
--- a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py
+++ b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py
@@ -78,6 +78,12 @@ class PromptStudioRegistryHelper:
"default": False,
"description": "Enables highlight",
},
+ "enable_word_confidence": {
+ "type": "boolean",
+ "title": "Enable word confidence",
+ "default": False,
+ "description": "Enables word-level confidence (depends on enable_highlight)",
+ },
}
spec = Spec(
@@ -283,9 +289,13 @@ class PromptStudioRegistryHelper:
tool.single_pass_extraction_mode
)
tool_settings[JsonSchemaKey.ENABLE_HIGHLIGHT] = tool.enable_highlight
+ tool_settings[JsonSchemaKey.ENABLE_WORD_CONFIDENCE] = tool.enable_word_confidence
tool_settings[JsonSchemaKey.PLATFORM_POSTAMBLE] = getattr(
settings, JsonSchemaKey.PLATFORM_POSTAMBLE.upper(), ""
)
+ tool_settings[JsonSchemaKey.WORD_CONFIDENCE_POSTAMBLE] = getattr(
+ settings, JsonSchemaKey.WORD_CONFIDENCE_POSTAMBLE.upper(), ""
+ )
for prompt in prompts:
if prompt.prompt_type == JsonSchemaKey.NOTES or not prompt.active:
diff --git a/frontend/src/components/custom-tools/document-manager/DocumentManager.jsx b/frontend/src/components/custom-tools/document-manager/DocumentManager.jsx
index 17542a7bc..e1fb4015b 100644
--- a/frontend/src/components/custom-tools/document-manager/DocumentManager.jsx
+++ b/frontend/src/components/custom-tools/document-manager/DocumentManager.jsx
@@ -380,6 +380,23 @@ function DocumentManager({ generateIndex, handleUpdateTool, handleDocChange }) {
if (typeof confidence === "number") {
return confidence.toFixed(2);
}
+ // Handle word confidence format: object with line numbers as keys
+ if (
+ confidence &&
+ typeof confidence === "object" &&
+ !Array.isArray(confidence)
+ ) {
+ const values = Object.values(confidence);
+ if (
+ values.length > 0 &&
+ values.every((v) => typeof v === "number")
+ ) {
+ const avg =
+ values.reduce((sum, val) => sum + val, 0) /
+ values.length;
+ return avg.toFixed(2);
+ }
+ }
// Handle old format: nested array structure
if (confidence?.[0]?.[0]?.confidence) {
return confidence[0][0].confidence;
diff --git a/frontend/src/components/custom-tools/prompt-card/DisplayPromptResult.jsx b/frontend/src/components/custom-tools/prompt-card/DisplayPromptResult.jsx
index ffe59057d..808aca590 100644
--- a/frontend/src/components/custom-tools/prompt-card/DisplayPromptResult.jsx
+++ b/frontend/src/components/custom-tools/prompt-card/DisplayPromptResult.jsx
@@ -22,6 +22,7 @@ function DisplayPromptResult({
highlightData,
promptDetails,
confidenceData,
+ wordConfidenceData,
isTable = false,
setOpenExpandModal = () => {},
}) {
@@ -79,13 +80,21 @@ function DisplayPromptResult({
);
}
- const handleClick = (highlightData, confidenceData, key, keyPath) => {
+ const handleClick = (
+ highlightData,
+ confidenceData,
+ wordConfidenceData,
+ key,
+ keyPath
+ ) => {
if (highlightData?.[key]) {
+ // Use word confidence if available, otherwise fallback to line confidence
+ const confidence = wordConfidenceData?.[key] || confidenceData?.[key];
handleSelectHighlight(
highlightData[key],
promptDetails?.prompt_id,
profileId,
- confidenceData?.[key]
+ confidence
);
setSelectedKey(keyPath);
}
@@ -98,6 +107,7 @@ function DisplayPromptResult({
data,
highlightData,
confidenceData,
+ wordConfidenceData,
indent = 0,
path = "",
isTable = false
@@ -146,6 +156,7 @@ function DisplayPromptResult({
item,
highlightData?.[index],
confidenceData?.[index],
+ wordConfidenceData?.[index],
indent + 1,
`${path}[${index}]`,
isTable
@@ -183,6 +194,7 @@ function DisplayPromptResult({
handleClick(
highlightData,
confidenceData,
+ wordConfidenceData,
key,
newPath
);
@@ -193,6 +205,7 @@ function DisplayPromptResult({
value,
highlightData?.[key],
confidenceData?.[key],
+ wordConfidenceData?.[key],
indent + 1,
newPath,
isTable
@@ -214,7 +227,15 @@ function DisplayPromptResult({
return (
<Typography.Paragraph className="prompt-card-display-output font-size-12">
{parsedOutput && typeof parsedOutput === "object" ? (
- renderJson(parsedOutput, highlightData, confidenceData, 0, "", isTable)
+ renderJson(
+ parsedOutput,
+ highlightData,
+ confidenceData,
+ wordConfidenceData,
+ 0,
+ "",
+ isTable
+ )
) : (
<TextResult
enableHighlight={details?.enable_highlight}
@@ -222,6 +243,7 @@ function DisplayPromptResult({
promptId={promptDetails?.prompt_id}
profileId={profileId}
confidenceData={confidenceData}
+ wordConfidenceData={wordConfidenceData}
selectedHighlight={selectedHighlight}
parsedOutput={parsedOutput}
onSelectHighlight={handleSelectHighlight}
@@ -237,15 +259,19 @@ const TextResult = ({
promptId,
profileId,
confidenceData,
+ wordConfidenceData,
selectedHighlight,
parsedOutput,
onSelectHighlight,
}) => {
+ // Use word confidence if available, otherwise fallback to line confidence
+ const confidence = wordConfidenceData || confidenceData;
+
return enableHighlight ? (
<Typography.Text
wrap
onClick={() =>
- onSelectHighlight(highlightData, promptId, profileId, confidenceData)
+ onSelectHighlight(highlightData, promptId, profileId, confidence)
}
className={`prompt-output-result json-value ${
highlightData ? "clickable" : ""
@@ -264,6 +290,7 @@ TextResult.propTypes = {
promptId: PropTypes.string,
profileId: PropTypes.string,
confidenceData: PropTypes.any,
+ wordConfidenceData: PropTypes.any,
selectedHighlight: PropTypes.object,
parsedOutput: PropTypes.any,
onSelectHighlight: PropTypes.func.isRequired,
@@ -277,6 +304,7 @@ DisplayPromptResult.propTypes = {
highlightData: PropTypes.object,
promptDetails: PropTypes.object,
confidenceData: PropTypes.object,
+ wordConfidenceData: PropTypes.object,
isTable: PropTypes.bool,
setOpenExpandModal: PropTypes.func,
};
diff --git a/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx b/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx
index b3f05fa0a..da959ea11 100644
--- a/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx
+++ b/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx
@@ -19,6 +19,7 @@ import {
} from "antd";
import { motion, AnimatePresence } from "framer-motion";
import CheckableTag from "antd/es/tag/CheckableTag";
+import { useState } from "react";
import {
displayPromptResult,
@@ -37,7 +38,6 @@ import { DisplayPromptResult } from "./DisplayPromptResult";
import usePromptOutput from "../../../hooks/usePromptOutput";
import { PromptRunTimer } from "./PromptRunTimer";
import { PromptRunCost } from "./PromptRunCost";
-import { useState } from "react";
let TableOutput;
try {
@@ -119,6 +119,7 @@ function PromptOutput({
handleSelectHighlight={handleSelectHighlight}
highlightData={promptOutputData?.highlightData}
confidenceData={promptOutputData?.confidenceData}
+ wordConfidenceData={promptOutputData?.wordConfidenceData}
promptDetails={promptDetails}
isTable={true}
setOpenExpandModal={setOpenExpandModal}
@@ -211,6 +212,9 @@ function PromptOutput({
confidenceData={
promptOutputData?.confidenceData?.[promptDetails.prompt_key]
}
+ wordConfidenceData={
+ promptOutputData?.wordConfidenceData?.[promptDetails.prompt_key]
+ }
/>
<div className="prompt-profile-run">
<CopyPromptOutputBtn
@@ -467,6 +471,9 @@ function PromptOutput({
handleSelectHighlight={handleSelectHighlight}
highlightData={promptOutputData?.highlightData}
confidenceData={promptOutputData?.confidenceData}
+ wordConfidenceData={
+ promptOutputData?.wordConfidenceData
+ }
promptDetails={promptDetails}
/>
<div className="prompt-profile-run">
diff --git a/frontend/src/hooks/usePromptOutput.js b/frontend/src/hooks/usePromptOutput.js
index 42de48453..eafe41521 100644
--- a/frontend/src/hooks/usePromptOutput.js
+++ b/frontend/src/hooks/usePromptOutput.js
@@ -121,6 +121,7 @@ const usePromptOutput = () => {
coverage: item?.coverage,
highlightData: item?.highlight_data,
confidenceData: item?.confidence_data,
+ wordConfidenceData: item?.word_confidence_data,
};
if (item?.is_single_pass_extract && isTokenUsageForSinglePassAdded)
diff --git a/prompt-service/src/unstract/prompt_service/constants.py b/prompt-service/src/unstract/prompt_service/constants.py
index 61f3cb3d5..9eddab842 100644
--- a/prompt-service/src/unstract/prompt_service/constants.py
+++ b/prompt-service/src/unstract/prompt_service/constants.py
@@ -63,6 +63,7 @@ class PromptServiceConstants:
TABLE_SETTINGS = "table_settings"
EPILOGUE = "epilogue"
PLATFORM_POSTAMBLE = "platform_postamble"
+ WORD_CONFIDENCE_POSTAMBLE = "word_confidence_postamble"
HIGHLIGHT_DATA_PLUGIN = "highlight-data"
SUMMARIZE_AS_SOURCE = "summarize_as_source"
VARIABLE_MAP = "variable_map"
@@ -70,9 +71,11 @@ class PromptServiceConstants:
CUSTOM_DATA = "custom_data"
TEXT = "text"
ENABLE_HIGHLIGHT = "enable_highlight"
+ ENABLE_WORD_CONFIDENCE = "enable_word_confidence"
FILE_PATH = "file_path"
HIGHLIGHT_DATA = "highlight_data"
CONFIDENCE_DATA = "confidence_data"
+ WORD_CONFIDENCE_DATA = "word_confidence_data"
REQUIRED_FIELDS = "required_fields"
REQUIRED = "required"
EXECUTION_SOURCE = "execution_source"
@@ -188,6 +191,7 @@ class IndexingConstants:
FILE_HASH = "file_hash"
OUTPUT_FILE_PATH = "output_file_path"
ENABLE_HIGHLIGHT = "enable_highlight"
+ ENABLE_WORD_CONFIDENCE = "enable_word_confidence"
USAGE_KWARGS = "usage_kwargs"
PROCESS_TEXT = "process_text"
EXTRACTED_TEXT = "extracted_text"
diff --git a/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py b/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py
index 58a650946..010637a86 100644
--- a/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py
+++ b/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py
@@ -502,6 +502,9 @@ def prompt_processor() -> Any:
doc_name=doc_name,
llm=llm,
enable_highlight=tool_settings.get(PSKeys.ENABLE_HIGHLIGHT, False),
+ enable_word_confidence=tool_settings.get(
+ PSKeys.ENABLE_WORD_CONFIDENCE, False
+ ),
execution_source=execution_source,
metadata=metadata,
file_path=file_path,
diff --git a/prompt-service/src/unstract/prompt_service/controllers/indexing.py b/prompt-service/src/unstract/prompt_service/controllers/indexing.py
index 84c75a117..f9416a9c9 100644
--- a/prompt-service/src/unstract/prompt_service/controllers/indexing.py
+++ b/prompt-service/src/unstract/prompt_service/controllers/indexing.py
@@ -60,6 +60,7 @@ def index() -> Any:
chunk_overlap: int = payload.get(IKeys.CHUNK_OVERLAP, 128) # Default chunk overlap
reindex: bool = payload.get(IKeys.REINDEX, False)
enable_highlight: bool = payload.get(IKeys.ENABLE_HIGHLIGHT, False)
+ enable_word_confidence: bool = payload.get(IKeys.ENABLE_WORD_CONFIDENCE, False)
usage_kwargs: dict[Any, Any] = payload.get(IKeys.USAGE_KWARGS, {})
extracted_text: str = payload.get(IKeys.EXTRACTED_TEXT, "")
tags: list[str] = payload.get(IKeys.TAGS, None)
@@ -80,7 +81,10 @@ def index() -> Any:
chunking_config = ChunkingConfig(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
processing_options = ProcessingOptions(
- reindex=reindex, enable_highlight=enable_highlight, usage_kwargs=usage_kwargs
+ reindex=reindex,
+ enable_highlight=enable_highlight,
+ enable_word_confidence=enable_word_confidence,
+ usage_kwargs=usage_kwargs,
)
doc_id = IndexingService.index(
chunking_config=chunking_config,
diff --git a/prompt-service/src/unstract/prompt_service/dto.py b/prompt-service/src/unstract/prompt_service/dto.py
index ed8888153..8c9e4f3d3 100644
--- a/prompt-service/src/unstract/prompt_service/dto.py
+++ b/prompt-service/src/unstract/prompt_service/dto.py
@@ -35,4 +35,5 @@ class ChunkingConfig:
class ProcessingOptions:
reindex: bool = False
enable_highlight: bool = False
+ enable_word_confidence: bool = False
usage_kwargs: dict[Any, Any] = field(default_factory=dict)
diff --git a/prompt-service/src/unstract/prompt_service/services/answer_prompt.py b/prompt-service/src/unstract/prompt_service/services/answer_prompt.py
index 3bc93051b..df2c0daed 100644
--- a/prompt-service/src/unstract/prompt_service/services/answer_prompt.py
+++ b/prompt-service/src/unstract/prompt_service/services/answer_prompt.py
@@ -115,11 +115,20 @@ class AnswerPromptService:
execution_source: str | None = ExecutionSource.IDE.value,
) -> str:
platform_postamble = tool_settings.get(PSKeys.PLATFORM_POSTAMBLE, "")
+ word_confidence_postamble = tool_settings.get(
+ PSKeys.WORD_CONFIDENCE_POSTAMBLE, ""
+ )
summarize_as_source = tool_settings.get(PSKeys.SUMMARIZE_AS_SOURCE)
enable_highlight = tool_settings.get(PSKeys.ENABLE_HIGHLIGHT, False)
+ enable_word_confidence = tool_settings.get(PSKeys.ENABLE_WORD_CONFIDENCE, False)
+ # Dependency: enable_word_confidence only works if enable_highlight is enabled
+ if not enable_highlight:
+ enable_word_confidence = False
prompt_type = output.get(PSKeys.TYPE, PSKeys.TEXT)
if not enable_highlight or summarize_as_source:
platform_postamble = ""
+ if not enable_word_confidence or summarize_as_source:
+ word_confidence_postamble = ""
plugin = PluginManager().get_plugin("json-extraction")
if plugin and hasattr(plugin["entrypoint_cls"], "update_settings"):
plugin["entrypoint_cls"].update_settings(tool_settings, output)
@@ -130,6 +139,7 @@ class AnswerPromptService:
grammar_list=tool_settings.get(PSKeys.GRAMMAR, []),
context=context,
platform_postamble=platform_postamble,
+ word_confidence_postamble=word_confidence_postamble,
prompt_type=prompt_type,
)
output[PSKeys.COMBINED_PROMPT] = prompt
@@ -140,6 +150,7 @@ class AnswerPromptService:
prompt_key=output[PSKeys.NAME],
prompt_type=prompt_type,
enable_highlight=enable_highlight,
+ enable_word_confidence=enable_word_confidence,
file_path=file_path,
execution_source=execution_source,
)
@@ -152,6 +163,7 @@ class AnswerPromptService:
grammar_list: list[dict[str, Any]],
context: str,
platform_postamble: str,
+ word_confidence_postamble: str,
prompt_type: str = PSKeys.TEXT,
) -> str:
prompt = f"{preamble}\n\nQuestion or Instruction: {prompt}"
@@ -176,6 +188,8 @@ class AnswerPromptService:
postamble += f"\n{json_postamble}"
if platform_postamble:
platform_postamble += "\n\n"
+ if word_confidence_postamble:
+ platform_postamble += f"{word_confidence_postamble}\n\n"
prompt += (
f"\n\n{postamble}\n\nContext:\n---------------\n{context}\n"
f"-----------------\n\n{platform_postamble}Answer:"
@@ -190,6 +204,7 @@ class AnswerPromptService:
prompt_key: str | None = None,
prompt_type: str | None = PSKeys.TEXT,
enable_highlight: bool = False,
+ enable_word_confidence: bool = False,
file_path: str = "",
execution_source: str | None = None,
) -> str:
@@ -220,6 +235,7 @@ class AnswerPromptService:
highlight_data = highlight_data_plugin["entrypoint_cls"](
file_path=file_path,
fs_instance=fs_instance,
+ enable_word_confidence=enable_word_confidence,
).run
completion = llm.complete(
prompt=prompt,
@@ -229,6 +245,7 @@ class AnswerPromptService:
answer: str = completion[PSKeys.RESPONSE].text
highlight_data = completion.get(PSKeys.HIGHLIGHT_DATA, [])
confidence_data = completion.get(PSKeys.CONFIDENCE_DATA)
+ word_confidence_data = completion.get(PSKeys.WORD_CONFIDENCE_DATA)
line_numbers = completion.get(PSKeys.LINE_NUMBERS, [])
whisper_hash = completion.get(PSKeys.WHISPER_HASH, "")
if metadata is not None and prompt_key:
@@ -241,6 +258,10 @@ class AnswerPromptService:
metadata.setdefault(PSKeys.CONFIDENCE_DATA, {})[prompt_key] = (
confidence_data
)
+ if enable_word_confidence and word_confidence_data:
+ metadata.setdefault(PSKeys.WORD_CONFIDENCE_DATA, {})[prompt_key] = (
+ word_confidence_data
+ )
return answer
# TODO: Catch and handle specific exception here
except SdkRateLimitError as e:
@@ -363,6 +384,7 @@ class AnswerPromptService:
doc_name: str,
llm: LLM,
enable_highlight: bool = False,
+ enable_word_confidence: bool = False,
execution_source: str = ExecutionSource.IDE.value,
metadata: dict[str, Any] | None = None,
file_path: str = "",
@@ -480,6 +502,7 @@ class AnswerPromptService:
grammar_list=tool_settings.get(PSKeys.GRAMMAR, []),
context=context,
platform_postamble="",
+ word_confidence_postamble="",
)
try:
line_item_extraction = line_item_extraction_plugin["entrypoint_cls"](
diff --git a/unstract/sdk1/src/unstract/sdk1/llm.py b/unstract/sdk1/src/unstract/sdk1/llm.py
index 0a1eb01a8..c6d820106 100644
--- a/unstract/sdk1/src/unstract/sdk1/llm.py
+++ b/unstract/sdk1/src/unstract/sdk1/llm.py
@@ -227,9 +227,9 @@ class LLM:
# Improvements in readability are definitely welcome.
extract_json: bool = cast("bool", kwargs.get("extract_json", False))
post_process_fn: (
- Callable[[LLMResponseCompat, bool], dict[str, object]] | None
+ Callable[[LLMResponseCompat, bool, str], dict[str, object]] | None
) = cast(
- "Callable[[LLMResponseCompat, bool], dict[str, object]] | None",
+ "Callable[[LLMResponseCompat, bool, str], dict[str, object]] | None",
kwargs.get("process_text", None),
)
@@ -478,6 +478,9 @@ class LLM:
) -> tuple[str, dict[str, object]]:
post_processed_output: dict[str, object] = {}
+ # Save original text before any modifications
+ original_text = response_text
+
if extract_json:
start = response_text.find(LLM.JSON_CONTENT_MARKER)
if start != -1:
@@ -494,7 +497,9 @@ class LLM:
if post_process_fn:
try:
response_compat = LLMResponseCompat(response_text)
- post_processed_output = post_process_fn(response_compat, extract_json)
+ post_processed_output = post_process_fn(
+ response_compat, extract_json, original_text
+ )
# Needed as the text is modified in place.
response_text = response_compat.text
except Exception as e: