Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-3008 [FIX] Word level confidence issue fixes (#1681)

Deepak K3512d893ff72
diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py
index a72d808a1..405b91e00 100644
--- a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py
+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py
@@ -144,20 +144,29 @@ class OutputManagerHelper:
if prompt.prompt_type == PSOMKeys.NOTES:
continue
+ # Use loop-scoped variables to avoid mutation
+ prompt_context = context
+ prompt_highlight_data = highlight_data
+ prompt_confidence_data = confidence_data
+ prompt_word_confidence_data = word_confidence_data
+ prompt_challenge_data = challenge_data
+
if not is_single_pass_extract:
- if context:
- context = context.get(prompt.prompt_key)
- if highlight_data:
- highlight_data = highlight_data.get(prompt.prompt_key)
- if confidence_data:
- confidence_data = confidence_data.get(prompt.prompt_key)
- if word_confidence_data:
- word_confidence_data = word_confidence_data.get(prompt.prompt_key)
- if challenge_data:
- challenge_data = challenge_data.get(prompt.prompt_key)
-
- if challenge_data:
- challenge_data["file_name"] = metadata.get("file_name")
+ if prompt_context:
+ prompt_context = prompt_context.get(prompt.prompt_key)
+ if prompt_highlight_data:
+ prompt_highlight_data = prompt_highlight_data.get(prompt.prompt_key)
+ if prompt_confidence_data:
+ prompt_confidence_data = prompt_confidence_data.get(prompt.prompt_key)
+ if prompt_word_confidence_data:
+ prompt_word_confidence_data = prompt_word_confidence_data.get(
+ prompt.prompt_key
+ )
+ if prompt_challenge_data:
+ prompt_challenge_data = prompt_challenge_data.get(prompt.prompt_key)
+
+ if prompt_challenge_data:
+ prompt_challenge_data["file_name"] = metadata.get("file_name")
# TODO: use enums here
output = outputs.get(prompt.prompt_key)
@@ -173,11 +182,11 @@ class OutputManagerHelper:
output=output,
eval_metrics=eval_metrics,
tool=tool,
- context=json.dumps(context),
- challenge_data=challenge_data,
- highlight_data=highlight_data,
- confidence_data=confidence_data,
- word_confidence_data=word_confidence_data,
+ context=json.dumps(prompt_context),
+ challenge_data=prompt_challenge_data,
+ highlight_data=prompt_highlight_data,
+ confidence_data=prompt_confidence_data,
+ word_confidence_data=prompt_word_confidence_data,
)
# Serialize the instance
diff --git a/frontend/src/components/custom-tools/prompt-card/DisplayPromptResult.jsx b/frontend/src/components/custom-tools/prompt-card/DisplayPromptResult.jsx
index 808aca590..4a370a903 100644
--- a/frontend/src/components/custom-tools/prompt-card/DisplayPromptResult.jsx
+++ b/frontend/src/components/custom-tools/prompt-card/DisplayPromptResult.jsx
@@ -80,6 +80,48 @@ function DisplayPromptResult({
);
}
+ // Extract confidence from 5th element of highlight data coordinate arrays
+ const extractConfidenceFromHighlightData = (data) => {
+ if (!data) return null;
+
+ const confidenceValues = [];
+
+ const extractFromArray = (arr) => {
+ if (Array.isArray(arr)) {
+ for (const item of arr) {
+ if (Array.isArray(item)) {
+ // Check if this is a coordinate array with 5 elements
+ if (item.length >= 5 && typeof item[4] === "number") {
+ confidenceValues.push(item[4]);
+ } else {
+ // Recursively check nested arrays
+ extractFromArray(item);
+ }
+ } else if (typeof item === "object" && item !== null) {
+ // Recursively check objects
+ for (const val of Object.values(item)) {
+ extractFromArray(val);
+ }
+ }
+ }
+ } else if (typeof arr === "object" && arr !== null) {
+ for (const val of Object.values(arr)) {
+ extractFromArray(val);
+ }
+ }
+ };
+
+ extractFromArray(data);
+
+ // Calculate average confidence if we found any values
+ if (confidenceValues.length > 0) {
+ const sum = confidenceValues.reduce((acc, val) => acc + val, 0);
+ return sum / confidenceValues.length;
+ }
+
+ return null;
+ };
+
const handleClick = (
highlightData,
confidenceData,
@@ -88,8 +130,41 @@ function DisplayPromptResult({
keyPath
) => {
if (highlightData?.[key]) {
- // Use word confidence if available, otherwise fallback to line confidence
- const confidence = wordConfidenceData?.[key] || confidenceData?.[key];
+ const shouldUseWordConfidence =
+ details?.enable_highlight && details?.enable_word_confidence;
+
+ const getNestedValue = (obj, path) => {
+ if (!obj || !path) return undefined;
+ const normalized = path.replace(/\[(\d+)\]/g, ".$1");
+ const parts = normalized.split(".").filter((p) => p !== "");
+ return parts.reduce((acc, part) => {
+ if (acc === undefined || acc === null) return undefined;
+ const maybeIndex = /^\d+$/.test(part) ? Number(part) : part;
+ return acc[maybeIndex];
+ }, obj);
+ };
+
+ let confidence;
+ if (shouldUseWordConfidence && wordConfidenceData) {
+ const wordConfidence = getNestedValue(wordConfidenceData, key);
+ if (wordConfidence && typeof wordConfidence === "object") {
+ const values = Object.values(wordConfidence).filter(
+ (v) => typeof v === "number"
+ );
+ if (values.length > 0) {
+ const sum = values.reduce((acc, val) => acc + val, 0);
+ confidence = sum / values.length;
+ }
+ }
+ }
+
+ if (confidence === undefined) {
+ const extractedConfidence = extractConfidenceFromHighlightData(
+ highlightData[key]
+ );
+ confidence = extractedConfidence ?? confidenceData?.[key];
+ }
+
handleSelectHighlight(
highlightData[key],
promptDetails?.prompt_id,
diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx
index 29ce21ecd..bf844778a 100644
--- a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx
+++ b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx
@@ -213,47 +213,6 @@ const PromptCard = memo(
return flattened;
};
- const extractConfidenceFromHighlightData = (data) => {
- if (!data) return null;
-
- const confidenceValues = [];
-
- const extractFromArray = (arr) => {
- if (Array.isArray(arr)) {
- for (const item of arr) {
- if (Array.isArray(item)) {
- // Check if this is a coordinate array with 5 elements
- if (item.length >= 5 && typeof item[4] === "number") {
- confidenceValues.push(item[4]);
- } else {
- // Recursively check nested arrays
- extractFromArray(item);
- }
- } else if (typeof item === "object" && item !== null) {
- // Recursively check objects
- for (const val of Object.values(item)) {
- extractFromArray(val);
- }
- }
- }
- } else if (typeof arr === "object" && arr !== null) {
- for (const val of Object.values(arr)) {
- extractFromArray(val);
- }
- }
- };
-
- extractFromArray(data);
-
- // Calculate average confidence if we found any values
- if (confidenceValues.length > 0) {
- const sum = confidenceValues.reduce((acc, val) => acc + val, 0);
- return sum / confidenceValues.length;
- }
-
- return null;
- };
-
const handleSelectHighlight = (
highlightData,
highlightedPrompt,
@@ -268,17 +227,12 @@ const PromptCard = memo(
? flattenHighlightData(highlightData)
: highlightData;
- // Try to extract confidence from 5th element first, fall back to API confidenceData
- const extractedConfidence =
- extractConfidenceFromHighlightData(highlightData);
- const finalConfidence = extractedConfidence ?? confidenceData;
-
updateCustomTool({
selectedHighlight: {
highlight: processedHighlight,
highlightedPrompt: highlightedPrompt,
highlightedProfile: highlightedProfile,
- confidence: finalConfidence,
+ confidence: confidenceData,
},
});
}
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/constants.py b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/constants.py
index 2181ea432..8c46eadf7 100644
--- a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/constants.py
+++ b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/constants.py
@@ -72,6 +72,7 @@ class WhispererConfig:
WAIT_FOR_COMPLETION = "wait_for_completion"
LOGGING_LEVEL = "logging_level"
ADD_LINE_NOS = "add_line_nos"
+ INCLUDE_LINE_CONFIDENCE = "include_line_confidence"
EXTRACT_ALL_LINES = "extract_all_lines"
LINES = "lines"
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/helper.py b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/helper.py
index 2977d5052..a36567e40 100644
--- a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/helper.py
+++ b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/helper.py
@@ -203,6 +203,7 @@ class LLMWhispererHelper:
WhispererDefaults.PAGE_SEPARATOR,
),
WhispererConfig.ADD_LINE_NOS: extra_params.enable_highlight,
+ WhispererConfig.INCLUDE_LINE_CONFIDENCE: extra_params.enable_highlight,
# Not providing default value to maintain legacy compatablity
# these are optional params and identifiers for audit
WhispererConfig.TAG: extra_params.tag
diff --git a/unstract/sdk1/src/unstract/sdk1/llm.py b/unstract/sdk1/src/unstract/sdk1/llm.py
index c6d820106..486294565 100644
--- a/unstract/sdk1/src/unstract/sdk1/llm.py
+++ b/unstract/sdk1/src/unstract/sdk1/llm.py
@@ -474,7 +474,8 @@ class LLM:
self,
response_text: str,
extract_json: bool,
- post_process_fn: Callable[[LLMResponseCompat, bool], dict[str, object]] | None,
+ post_process_fn: Callable[[LLMResponseCompat, bool, str], dict[str, object]]
+ | None,
) -> tuple[str, dict[str, object]]:
post_processed_output: dict[str, object] = {}
diff --git a/workers/shared/workflow/destination_connector.py b/workers/shared/workflow/destination_connector.py
index ab4c6a13d..5e5bb3f7e 100644
--- a/workers/shared/workflow/destination_connector.py
+++ b/workers/shared/workflow/destination_connector.py
@@ -221,6 +221,50 @@ class WorkerDestinationConnector:
self.manual_review_service = None
self._api_client = None
+ @staticmethod
+ def _extract_confidence_from_highlight_data(data: Any) -> float | None:
+ """Extract confidence from 5th element of highlight data coordinate arrays.
+
+ Recursively searches through nested arrays/objects to find coordinate arrays
+ with 5 elements where the 5th element (index 4) is the confidence score.
+
+ Args:
+ data: Highlight data structure (can be nested arrays/dicts)
+
+ Returns:
+ Average confidence score if found, None otherwise
+ """
+ if not data:
+ return None
+
+ confidence_values = []
+
+ def extract_from_array(arr):
+ if isinstance(arr, list):
+ for item in arr:
+ if isinstance(item, list):
+ # Check if this is a coordinate array with 5 elements
+ if len(item) >= 5 and isinstance(item[4], (int, float)):
+ confidence_values.append(float(item[4]))
+ else:
+ # Recursively check nested arrays
+ extract_from_array(item)
+ elif isinstance(item, dict):
+ # Recursively check objects
+ for val in item.values():
+ extract_from_array(val)
+ elif isinstance(arr, dict):
+ for val in arr.values():
+ extract_from_array(val)
+
+ extract_from_array(data)
+
+ # Calculate average confidence if we found any values
+ if confidence_values:
+ return sum(confidence_values) / len(confidence_values)
+
+ return None
+
@classmethod
def from_config(cls, workflow_log, config: DestinationConfig):
"""Create destination connector from config (matching Django backend interface)."""
@@ -1447,12 +1491,39 @@ class WorkerDestinationConnector:
else:
# Legacy format: wrap in expected structure
metadata = self.get_metadata()
+
+ # 3-tier fallback hierarchy for confidence:
+ # 1. word_confidence_data (if available)
+ # 2. Extract from 5th element of highlight_data
+ # 3. confidence_data (last resort)
+ highlight_data = (
+ metadata.get("highlight_data", {}) if metadata else {}
+ )
+ word_confidence_data = (
+ metadata.get("word_confidence_data", {}) if metadata else {}
+ )
+ confidence_data = (
+ metadata.get("confidence_data", {}) if metadata else {}
+ )
+
+ # If word_confidence_data is not available, try extracting from highlight_data
+ if not word_confidence_data and highlight_data:
+ extracted_confidence = (
+ self._extract_confidence_from_highlight_data(highlight_data)
+ )
+ # Use extracted confidence if available, otherwise fall back to confidence_data
+ # Store as a simple dict for compatibility
+ if extracted_confidence is not None:
+ # For rule engine, we provide a single confidence score
+ # This will be used by the enterprise plugin
+ confidence_data = {"_extracted_average": extracted_confidence}
+
wrapped_result = {
"output": tool_execution_result,
"metadata": {
- "confidence_data": metadata.get("confidence_data", {})
- if metadata
- else {},
+ "confidence_data": confidence_data,
+ "word_confidence_data": word_confidence_data,
+ "highlight_data": highlight_data,
"whisper-hash": metadata.get("whisper-hash")
if metadata
else None,