LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-3008 [FIX] Word level confidence issue fixes (#1681)
Deepak K3512d893ff72
diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.pyindex a72d808a1..405b91e00 100644--- a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py@@ -144,20 +144,29 @@ class OutputManagerHelper:if prompt.prompt_type == PSOMKeys.NOTES:continue+ # Use loop-scoped variables to avoid mutation+ prompt_context = context+ prompt_highlight_data = highlight_data+ prompt_confidence_data = confidence_data+ prompt_word_confidence_data = word_confidence_data+ prompt_challenge_data = challenge_data+if not is_single_pass_extract:- if context:- context = context.get(prompt.prompt_key)- if highlight_data:- highlight_data = highlight_data.get(prompt.prompt_key)- if confidence_data:- confidence_data = confidence_data.get(prompt.prompt_key)- if word_confidence_data:- word_confidence_data = word_confidence_data.get(prompt.prompt_key)- if challenge_data:- challenge_data = challenge_data.get(prompt.prompt_key)-- if challenge_data:- challenge_data["file_name"] = metadata.get("file_name")+ if prompt_context:+ prompt_context = prompt_context.get(prompt.prompt_key)+ if prompt_highlight_data:+ prompt_highlight_data = prompt_highlight_data.get(prompt.prompt_key)+ if prompt_confidence_data:+ prompt_confidence_data = prompt_confidence_data.get(prompt.prompt_key)+ if prompt_word_confidence_data:+ prompt_word_confidence_data = prompt_word_confidence_data.get(+ prompt.prompt_key+ )+ if prompt_challenge_data:+ prompt_challenge_data = prompt_challenge_data.get(prompt.prompt_key)++ if prompt_challenge_data:+ prompt_challenge_data["file_name"] = metadata.get("file_name")# TODO: use enums hereoutput = outputs.get(prompt.prompt_key)@@ -173,11 +182,11 @@ class OutputManagerHelper:output=output,eval_metrics=eval_metrics,tool=tool,- context=json.dumps(context),- challenge_data=challenge_data,- highlight_data=highlight_data,- confidence_data=confidence_data,- word_confidence_data=word_confidence_data,+ context=json.dumps(prompt_context),+ challenge_data=prompt_challenge_data,+ highlight_data=prompt_highlight_data,+ confidence_data=prompt_confidence_data,+ word_confidence_data=prompt_word_confidence_data,)# Serialize the instancediff --git a/frontend/src/components/custom-tools/prompt-card/DisplayPromptResult.jsx b/frontend/src/components/custom-tools/prompt-card/DisplayPromptResult.jsxindex 808aca590..4a370a903 100644--- a/frontend/src/components/custom-tools/prompt-card/DisplayPromptResult.jsx+++ b/frontend/src/components/custom-tools/prompt-card/DisplayPromptResult.jsx@@ -80,6 +80,48 @@ function DisplayPromptResult({);}+ // Extract confidence from 5th element of highlight data coordinate arrays+ const extractConfidenceFromHighlightData = (data) => {+ if (!data) return null;++ const confidenceValues = [];++ const extractFromArray = (arr) => {+ if (Array.isArray(arr)) {+ for (const item of arr) {+ if (Array.isArray(item)) {+ // Check if this is a coordinate array with 5 elements+ if (item.length >= 5 && typeof item[4] === "number") {+ confidenceValues.push(item[4]);+ } else {+ // Recursively check nested arrays+ extractFromArray(item);+ }+ } else if (typeof item === "object" && item !== null) {+ // Recursively check objects+ for (const val of Object.values(item)) {+ extractFromArray(val);+ }+ }+ }+ } else if (typeof arr === "object" && arr !== null) {+ for (const val of Object.values(arr)) {+ extractFromArray(val);+ }+ }+ };++ extractFromArray(data);++ // Calculate average confidence if we found any values+ if (confidenceValues.length > 0) {+ const sum = confidenceValues.reduce((acc, val) => acc + val, 0);+ return sum / confidenceValues.length;+ }++ return null;+ };+const handleClick = (highlightData,confidenceData,@@ -88,8 +130,41 @@ function DisplayPromptResult({keyPath) => {if (highlightData?.[key]) {- // Use word confidence if available, otherwise fallback to line confidence- const confidence = wordConfidenceData?.[key] || confidenceData?.[key];+ const shouldUseWordConfidence =+ details?.enable_highlight && details?.enable_word_confidence;++ const getNestedValue = (obj, path) => {+ if (!obj || !path) return undefined;+ const normalized = path.replace(/\[(\d+)\]/g, ".$1");+ const parts = normalized.split(".").filter((p) => p !== "");+ return parts.reduce((acc, part) => {+ if (acc === undefined || acc === null) return undefined;+ const maybeIndex = /^\d+$/.test(part) ? Number(part) : part;+ return acc[maybeIndex];+ }, obj);+ };++ let confidence;+ if (shouldUseWordConfidence && wordConfidenceData) {+ const wordConfidence = getNestedValue(wordConfidenceData, key);+ if (wordConfidence && typeof wordConfidence === "object") {+ const values = Object.values(wordConfidence).filter(+ (v) => typeof v === "number"+ );+ if (values.length > 0) {+ const sum = values.reduce((acc, val) => acc + val, 0);+ confidence = sum / values.length;+ }+ }+ }++ if (confidence === undefined) {+ const extractedConfidence = extractConfidenceFromHighlightData(+ highlightData[key]+ );+ confidence = extractedConfidence ?? confidenceData?.[key];+ }+handleSelectHighlight(highlightData[key],promptDetails?.prompt_id,diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsxindex 29ce21ecd..bf844778a 100644--- a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx+++ b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx@@ -213,47 +213,6 @@ const PromptCard = memo(return flattened;};- const extractConfidenceFromHighlightData = (data) => {- if (!data) return null;-- const confidenceValues = [];-- const extractFromArray = (arr) => {- if (Array.isArray(arr)) {- for (const item of arr) {- if (Array.isArray(item)) {- // Check if this is a coordinate array with 5 elements- if (item.length >= 5 && typeof item[4] === "number") {- confidenceValues.push(item[4]);- } else {- // Recursively check nested arrays- extractFromArray(item);- }- } else if (typeof item === "object" && item !== null) {- // Recursively check objects- for (const val of Object.values(item)) {- extractFromArray(val);- }- }- }- } else if (typeof arr === "object" && arr !== null) {- for (const val of Object.values(arr)) {- extractFromArray(val);- }- }- };-- extractFromArray(data);-- // Calculate average confidence if we found any values- if (confidenceValues.length > 0) {- const sum = confidenceValues.reduce((acc, val) => acc + val, 0);- return sum / confidenceValues.length;- }-- return null;- };-const handleSelectHighlight = (highlightData,highlightedPrompt,@@ -268,17 +227,12 @@ const PromptCard = memo(? flattenHighlightData(highlightData): highlightData;- // Try to extract confidence from 5th element first, fall back to API confidenceData- const extractedConfidence =- extractConfidenceFromHighlightData(highlightData);- const finalConfidence = extractedConfidence ?? confidenceData;-updateCustomTool({selectedHighlight: {highlight: processedHighlight,highlightedPrompt: highlightedPrompt,highlightedProfile: highlightedProfile,- confidence: finalConfidence,+ confidence: confidenceData,},});}diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/constants.py b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/constants.pyindex 2181ea432..8c46eadf7 100644--- a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/constants.py+++ b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/constants.py@@ -72,6 +72,7 @@ class WhispererConfig:WAIT_FOR_COMPLETION = "wait_for_completion"LOGGING_LEVEL = "logging_level"ADD_LINE_NOS = "add_line_nos"+ INCLUDE_LINE_CONFIDENCE = "include_line_confidence"EXTRACT_ALL_LINES = "extract_all_lines"LINES = "lines"diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/helper.py b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/helper.pyindex 2977d5052..a36567e40 100644--- a/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/helper.py+++ b/unstract/sdk1/src/unstract/sdk1/adapters/x2text/llm_whisperer_v2/src/helper.py@@ -203,6 +203,7 @@ class LLMWhispererHelper:WhispererDefaults.PAGE_SEPARATOR,),WhispererConfig.ADD_LINE_NOS: extra_params.enable_highlight,+ WhispererConfig.INCLUDE_LINE_CONFIDENCE: extra_params.enable_highlight,# Not providing default value to maintain legacy compatablity# these are optional params and identifiers for auditWhispererConfig.TAG: extra_params.tagdiff --git a/unstract/sdk1/src/unstract/sdk1/llm.py b/unstract/sdk1/src/unstract/sdk1/llm.pyindex c6d820106..486294565 100644--- a/unstract/sdk1/src/unstract/sdk1/llm.py+++ b/unstract/sdk1/src/unstract/sdk1/llm.py@@ -474,7 +474,8 @@ class LLM:self,response_text: str,extract_json: bool,- post_process_fn: Callable[[LLMResponseCompat, bool], dict[str, object]] | None,+ post_process_fn: Callable[[LLMResponseCompat, bool, str], dict[str, object]]+ | None,) -> tuple[str, dict[str, object]]:post_processed_output: dict[str, object] = {}diff --git a/workers/shared/workflow/destination_connector.py b/workers/shared/workflow/destination_connector.pyindex ab4c6a13d..5e5bb3f7e 100644--- a/workers/shared/workflow/destination_connector.py+++ b/workers/shared/workflow/destination_connector.py@@ -221,6 +221,50 @@ class WorkerDestinationConnector:self.manual_review_service = Noneself._api_client = None+ @staticmethod+ def _extract_confidence_from_highlight_data(data: Any) -> float | None:+ """Extract confidence from 5th element of highlight data coordinate arrays.++ Recursively searches through nested arrays/objects to find coordinate arrays+ with 5 elements where the 5th element (index 4) is the confidence score.++ Args:+ data: Highlight data structure (can be nested arrays/dicts)++ Returns:+ Average confidence score if found, None otherwise+ """+ if not data:+ return None++ confidence_values = []++ def extract_from_array(arr):+ if isinstance(arr, list):+ for item in arr:+ if isinstance(item, list):+ # Check if this is a coordinate array with 5 elements+ if len(item) >= 5 and isinstance(item[4], (int, float)):+ confidence_values.append(float(item[4]))+ else:+ # Recursively check nested arrays+ extract_from_array(item)+ elif isinstance(item, dict):+ # Recursively check objects+ for val in item.values():+ extract_from_array(val)+ elif isinstance(arr, dict):+ for val in arr.values():+ extract_from_array(val)++ extract_from_array(data)++ # Calculate average confidence if we found any values+ if confidence_values:+ return sum(confidence_values) / len(confidence_values)++ return None+@classmethoddef from_config(cls, workflow_log, config: DestinationConfig):"""Create destination connector from config (matching Django backend interface)."""@@ -1447,12 +1491,39 @@ class WorkerDestinationConnector:else:# Legacy format: wrap in expected structuremetadata = self.get_metadata()++ # 3-tier fallback hierarchy for confidence:+ # 1. word_confidence_data (if available)+ # 2. Extract from 5th element of highlight_data+ # 3. confidence_data (last resort)+ highlight_data = (+ metadata.get("highlight_data", {}) if metadata else {}+ )+ word_confidence_data = (+ metadata.get("word_confidence_data", {}) if metadata else {}+ )+ confidence_data = (+ metadata.get("confidence_data", {}) if metadata else {}+ )++ # If word_confidence_data is not available, try extracting from highlight_data+ if not word_confidence_data and highlight_data:+ extracted_confidence = (+ self._extract_confidence_from_highlight_data(highlight_data)+ )+ # Use extracted confidence if available, otherwise fall back to confidence_data+ # Store as a simple dict for compatibility+ if extracted_confidence is not None:+ # For rule engine, we provide a single confidence score+ # This will be used by the enterprise plugin+ confidence_data = {"_extracted_average": extracted_confidence}+wrapped_result = {"output": tool_execution_result,"metadata": {- "confidence_data": metadata.get("confidence_data", {})- if metadata- else {},+ "confidence_data": confidence_data,+ "word_confidence_data": word_confidence_data,+ "highlight_data": highlight_data,"whisper-hash": metadata.get("whisper-hash")if metadataelse None,