LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
[MERGE-WITH-SDK-ROLL-RC6]: BE changes to support propagating errors from SDK (#873)
Chandrasekharan M35a32d576843
diff --git a/backend/adapter_processor_v2/adapter_processor.py b/backend/adapter_processor_v2/adapter_processor.pyindex 9057934f7..062d26700 100644--- a/backend/adapter_processor_v2/adapter_processor.py+++ b/backend/adapter_processor_v2/adapter_processor.py@@ -92,10 +92,11 @@ class AdapterProcessor:adapter_instance = adapter_class(adapter_metadata)test_result: bool = adapter_instance.test_connection()- logger.info(f"{adapter_id} test result: {test_result}")return test_resultexcept SdkError as e:- raise TestAdapterError(str(e))+ raise TestAdapterError(+ e, adapter_name=adapter_metadata[AdapterKeys.ADAPTER_NAME]+ )@staticmethoddef __fetch_adapters_by_key_value(key: str, value: Any) -> Adapter:diff --git a/backend/adapter_processor_v2/exceptions.py b/backend/adapter_processor_v2/exceptions.pyindex 009f1e5c1..471ea9fb5 100644--- a/backend/adapter_processor_v2/exceptions.py+++ b/backend/adapter_processor_v2/exceptions.py@@ -2,6 +2,7 @@ from typing import Optionalfrom adapter_processor_v2.constants import AdapterKeysfrom rest_framework.exceptions import APIException+from unstract.sdk.exceptions import SdkErrorclass IdIsMandatory(APIException):@@ -51,6 +52,20 @@ class TestAdapterError(APIException):status_code = 500default_detail = "Error while testing adapter"+ def __init__(+ self,+ sdk_err: SdkError,+ detail: Optional[str] = None,+ code: Optional[str] = None,+ adapter_name: Optional[str] = None,+ ):+ if sdk_err.status_code:+ self.status_code = sdk_err.status_code+ if detail is None:+ adapter_name = f"'{adapter_name}'" if adapter_name else "adapter"+ detail = f"Error testing {adapter_name}. {str(sdk_err)}"+ super().__init__(detail, code)+class TestAdapterInputError(APIException):status_code = 400diff --git a/backend/prompt_studio/prompt_studio_core_v2/exceptions.py b/backend/prompt_studio/prompt_studio_core_v2/exceptions.pyindex 9d9d0664d..4f45a5e96 100644--- a/backend/prompt_studio/prompt_studio_core_v2/exceptions.py+++ b/backend/prompt_studio/prompt_studio_core_v2/exceptions.py@@ -24,6 +24,10 @@ class AnswerFetchError(APIException):status_code = 500default_detail = "Error occured while fetching response for the prompt"+ def __init__(self, detail: Optional[str] = None, status_code: int = 500):+ super().__init__(detail)+ self.status_code = status_code+class DefaultProfileError(APIException):status_code = 500diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.pyindex 75a7f3f4f..6bcacf340 100644--- a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py+++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py@@ -894,7 +894,8 @@ class PromptStudioHelper:error_message = answer.get("error", "")raise AnswerFetchError("Error while fetching response for "- f"'{prompt.prompt_key}' with '{doc_name}'. {error_message}"+ f"'{prompt.prompt_key}' with '{doc_name}'. {error_message}",+ status_code=int(answer.get("status_code")),)output_response = json.loads(answer["structure_output"])return output_responsediff --git a/backend/sample.env b/backend/sample.envindex 8e9dd576a..7f0fd105a 100644--- a/backend/sample.env+++ b/backend/sample.env@@ -116,12 +116,14 @@ SYSTEM_ADMIN_EMAIL="[email protected]"SESSION_COOKIE_AGE=86400-# Control async extraction of LLM Whisperer+# Control async extraction of LLMWhisperer# Time in seconds to wait before polling LLMWhisperer's status APIADAPTER_LLMW_POLL_INTERVAL=30# Total number of times to poll the status API.# 500 mins to allow 1500 (max pages limit) * 20 (approx time in sec to process a page)ADAPTER_LLMW_MAX_POLLS=1000+# Number of times to retry the /whisper-status API before failing the extraction+ADAPTER_LLMW_STATUS_RETRIES=5# Enable logging of workflow history.ENABLE_LOG_HISTORY=Truediff --git a/backend/tool_instance_v2/tool_processor.py b/backend/tool_instance_v2/tool_processor.pyindex e5f9796df..8ed658d73 100644--- a/backend/tool_instance_v2/tool_processor.py+++ b/backend/tool_instance_v2/tool_processor.py@@ -17,7 +17,6 @@ logger = logging.getLogger(__name__)class ToolProcessor:TOOL_NOT_IN_REGISTRY_MESSAGE = "Tool does not exist in registry"- tool_registry = ToolRegistry()@staticmethoddef get_tool_by_uid(tool_uid: str) -> Tool:diff --git a/prompt-service/src/unstract/prompt_service/main.py b/prompt-service/src/unstract/prompt_service/main.pyindex 12c2242e0..531617cf8 100644--- a/prompt-service/src/unstract/prompt_service/main.py+++ b/prompt-service/src/unstract/prompt_service/main.py@@ -1,7 +1,7 @@import timeimport tracebackfrom json import JSONDecodeError-from typing import Any, Optional+from typing import Anyfrom flask import json, jsonify, requestfrom llama_index.core.vector_stores import ExactMatchFilter, MetadataFilters@@ -252,64 +252,18 @@ def prompt_processor() -> Any:raise api_errortry:- context: set[str] = set()- if output[PSKeys.CHUNK_SIZE] == 0:+ if chunk_size == 0:# We can do this only for chunkless indexes- retrieved_context: Optional[str] = index.query_index(- embedding_instance_id=output[PSKeys.EMBEDDING],- vector_db_instance_id=output[PSKeys.VECTOR_DB],+ context: set[str] = fetch_context_from_vector_db(+ index=index,+ output=output,doc_id=doc_id,+ tool_id=tool_id,+ doc_name=doc_name,+ prompt_name=prompt_name,+ log_events_id=log_events_id,usage_kwargs=usage_kwargs,)- if not context:- # UN-1288 For Pinecone, we are seeing an inconsistent case where- # query with doc_id fails even though indexing just happened.- # This causes the following retrieve to return no text.- # To rule out any lag on the Pinecone vector DB write,- # the following sleep is added.- # Note: This will not fix the issue. Since this issue is- # inconsistent, and not reproducible easily,- # this is just a safety net.- time.sleep(2)- retrieved_context: Optional[str] = index.query_index(- embedding_instance_id=output[PSKeys.EMBEDDING],- vector_db_instance_id=output[PSKeys.VECTOR_DB],- doc_id=doc_id,- usage_kwargs=usage_kwargs,- )- if retrieved_context is None:- # TODO: Obtain user set name for vector DB- msg = NO_CONTEXT_ERROR- app.logger.error(- f"{msg} {output[PSKeys.VECTOR_DB]} for doc_id {doc_id}"- )- publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.ERROR,- RunLevel.RUN,- msg,- )- raise APIError(message=msg)- context.add(retrieved_context)- # TODO: Use vectorDB name when available- publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.DEBUG,- RunLevel.RUN,- "Fetched context from vector DB",- )-- if chunk_size == 0:publish_log(log_events_id,{@@ -674,6 +628,102 @@ def prompt_processor() -> Any:return response+def fetch_context_from_vector_db(+ index: Index,+ output: dict[str, Any],+ doc_id: str,+ tool_id: str,+ doc_name: str,+ prompt_name: str,+ log_events_id: str,+ usage_kwargs: dict[str, Any],+) -> set[str]:+ """+ Fetches context from the index for the given document ID. Implements a retry+ mechanism with logging and raises an error if context retrieval fails.++ Args:+ index: The index object to query.+ output: Dictionary containing keys like embedding and vector DB instance ID.+ doc_id: The document ID to query.+ tool_id: Identifier for the tool in use.+ doc_name: Name of the document being queried.+ prompt_name: Name of the prompt being executed.+ log_events_id: Unique ID for logging events.+ usage_kwargs: Additional usage parameters.++ Raises:+ APIError: If context retrieval fails after retrying.+ """+ context: set[str] = set()+ try:+ retrieved_context = index.query_index(+ embedding_instance_id=output[PSKeys.EMBEDDING],+ vector_db_instance_id=output[PSKeys.VECTOR_DB],+ doc_id=doc_id,+ usage_kwargs=usage_kwargs,+ )++ if retrieved_context:+ context.add(retrieved_context)+ publish_log(+ log_events_id,+ {+ "tool_id": tool_id,+ "prompt_key": prompt_name,+ "doc_name": doc_name,+ },+ LogLevel.DEBUG,+ RunLevel.RUN,+ "Fetched context from vector DB",+ )+ else:+ # Handle lag in vector DB write (e.g., Pinecone issue)+ time.sleep(2)+ retrieved_context = index.query_index(+ embedding_instance_id=output[PSKeys.EMBEDDING],+ vector_db_instance_id=output[PSKeys.VECTOR_DB],+ doc_id=doc_id,+ usage_kwargs=usage_kwargs,+ )++ if retrieved_context is None:+ msg = NO_CONTEXT_ERROR+ app.logger.error(+ f"{msg} {output[PSKeys.VECTOR_DB]} for doc_id {doc_id}"+ )+ publish_log(+ log_events_id,+ {+ "tool_id": tool_id,+ "prompt_key": prompt_name,+ "doc_name": doc_name,+ },+ LogLevel.ERROR,+ RunLevel.RUN,+ msg,+ )+ raise APIError(message=msg)+ except SdkError as e:+ msg = f"Unable to fetch context from vector DB. {str(e)}"+ app.logger.error(+ f"{msg}. VectorDB: {output[PSKeys.VECTOR_DB]}, doc_id: {doc_id}"+ )+ publish_log(+ log_events_id,+ {+ "tool_id": tool_id,+ "prompt_key": prompt_name,+ "doc_name": doc_name,+ },+ LogLevel.ERROR,+ RunLevel.RUN,+ msg,+ )+ raise APIError(message=msg, code=e.status_code)+ return context++def run_retrieval( # type:ignoretool_settings: dict[str, Any],output: dict[str, Any],@@ -682,7 +732,7 @@ def run_retrieval( # type:ignorevector_index,retrieval_type: str,metadata: dict[str, Any],-) -> tuple[str, str]:+) -> tuple[str, set[str]]:context: set[str] = set()prompt = output[PSKeys.PROMPTX]if retrieval_type == PSKeys.SUBQUESTION: