LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2105 [FEAT] Include adapter name in error messages (#1825)
Praveen Kumara20d7a05aab3
diff --git a/prompt-service/src/unstract/prompt_service/core/index_v2.py b/prompt-service/src/unstract/prompt_service/core/index_v2.pyindex 6e621d6ec..5f1703b32 100644--- a/prompt-service/src/unstract/prompt_service/core/index_v2.py+++ b/prompt-service/src/unstract/prompt_service/core/index_v2.py@@ -28,7 +28,7 @@ from unstract.sdk1.file_storage.impl import FileStoragefrom unstract.sdk1.file_storage.provider import FileStorageProviderfrom unstract.sdk1.platform import PlatformHelper as ToolAdapterfrom unstract.sdk1.tool.stream import StreamMixin-from unstract.sdk1.utils.common import capture_metrics+from unstract.sdk1.utils.common import Utils, capture_metricsfrom unstract.sdk1.utils.tool import ToolUtilsfrom unstract.sdk1.vector_db import VectorDB@@ -83,17 +83,21 @@ class Index:# Whole adapter config is used currently even though it contains some keys# which might not be relevant to indexing. This is easier for now than# marking certain keys of the adapter config as necessary.+ vector_db_config = ToolAdapter.get_adapter_config(+ self.tool, self.instance_identifiers.vector_db_instance_id+ )+ embedding_config = ToolAdapter.get_adapter_config(+ self.tool, self.instance_identifiers.embedding_instance_id+ )+ x2text_config = ToolAdapter.get_adapter_config(+ self.tool, self.instance_identifiers.x2text_instance_id+ )+ Utils.strip_adapter_name(vector_db_config, embedding_config, x2text_config)index_key = {"file_hash": file_hash,- "vector_db_config": ToolAdapter.get_adapter_config(- self.tool, self.instance_identifiers.vector_db_instance_id- ),- "embedding_config": ToolAdapter.get_adapter_config(- self.tool, self.instance_identifiers.embedding_instance_id- ),- "x2text_config": ToolAdapter.get_adapter_config(- self.tool, self.instance_identifiers.x2text_instance_id- ),+ "vector_db_config": vector_db_config,+ "embedding_config": embedding_config,+ "x2text_config": x2text_config,# Typed and hashed as strings since the final hash is persisted# and this is required to be backward compatible"chunk_size": str(self.chunking_config.chunk_size),diff --git a/unstract/sdk1/src/unstract/sdk1/constants.py b/unstract/sdk1/src/unstract/sdk1/constants.pyindex 65119d8ba..3494a8e2a 100644--- a/unstract/sdk1/src/unstract/sdk1/constants.py+++ b/unstract/sdk1/src/unstract/sdk1/constants.py@@ -10,6 +10,7 @@ class Common:ICON = "icon"ADAPTER_ID = "adapter_id"ADAPTER_TYPE = "adapter_type"+ ADAPTER_NAME = "_adapter_name"DEFAULT_ERR_MESSAGE = "Something went wrong"diff --git a/unstract/sdk1/src/unstract/sdk1/embedding.py b/unstract/sdk1/src/unstract/sdk1/embedding.pyindex 758aa0e1c..c12ccea89 100644--- a/unstract/sdk1/src/unstract/sdk1/embedding.py+++ b/unstract/sdk1/src/unstract/sdk1/embedding.py@@ -8,6 +8,7 @@ from llama_index.core.embeddings import BaseEmbeddingfrom pydantic import ValidationErrorfrom unstract.sdk1.adapters.constants import Commonfrom unstract.sdk1.adapters.embedding1 import adapters+from unstract.sdk1.constants import Common as SdkCommonfrom unstract.sdk1.constants import ToolEnvfrom unstract.sdk1.exceptions import SdkError, parse_litellm_errfrom unstract.sdk1.platform import PlatformHelper@@ -66,6 +67,7 @@ class Embedding:self._adapter_id = embedding_config[Common.ADAPTER_ID]self._adapter_metadata = embedding_config[Common.ADAPTER_METADATA]self._adapter_instance_id = adapter_instance_id+ self._adapter_name = embedding_config.pop(SdkCommon.ADAPTER_NAME, "")self._tool = toolelse:self._adapter_id = adapter_id@@ -75,6 +77,7 @@ class Embedding:else:self._adapter_metadata = adapters[self._adapter_id][Common.METADATA]self._adapter_instance_id = ""+ self._adapter_name = ""self._tool = None# Retrieve the adapter class.@@ -95,8 +98,14 @@ class Embedding:try:self._length = len(self.get_embedding(self._TEST_SNIPPET))except Exception as e:- provider_name = f"{self.adapter.get_name()}"- raise parse_litellm_err(e, provider_name) from e+ raise parse_litellm_err(e, self._get_adapter_info()) from e++ def _get_adapter_info(self) -> str:+ """Build a display string identifying this adapter for errors."""+ name = self.adapter.get_name()+ if self._adapter_name:+ return f"{self._adapter_name} ({name})"+ return namedef get_embedding(self, text: str) -> list[float]:"""Return embedding vector for query string."""@@ -110,8 +119,7 @@ class Embedding:return resp["data"][0]["embedding"]except Exception as e:- provider_name = f"{self.adapter.get_name()}"- raise parse_litellm_err(e, provider_name) from e+ raise parse_litellm_err(e, self._get_adapter_info()) from edef get_embeddings(self, texts: list[str]) -> list[list[float]]:"""Return embedding vectors for list of query strings."""@@ -123,8 +131,7 @@ class Embedding:return [data["embedding"] for data in resp["data"]]except Exception as e:- provider_name = f"{self.adapter.get_name()}"- raise parse_litellm_err(e, provider_name) from e+ raise parse_litellm_err(e, self._get_adapter_info()) from easync def get_aembedding(self, text: str) -> list[float]:"""Return async embedding vector for query string."""diff --git a/unstract/sdk1/src/unstract/sdk1/index.py b/unstract/sdk1/src/unstract/sdk1/index.pyindex ed4867963..92220f0a8 100644--- a/unstract/sdk1/src/unstract/sdk1/index.py+++ b/unstract/sdk1/src/unstract/sdk1/index.py@@ -24,7 +24,7 @@ from unstract.sdk1.embedding import EmbeddingCompatfrom unstract.sdk1.exceptions import IndexingError, SdkError, VectorDBError, X2TextErrorfrom unstract.sdk1.file_storage import FileStorage, FileStorageProviderfrom unstract.sdk1.platform import PlatformHelper-from unstract.sdk1.utils.common import capture_metrics, log_elapsed+from unstract.sdk1.utils.common import Utils, capture_metrics, log_elapsedfrom unstract.sdk1.utils.tool import ToolUtilsfrom unstract.sdk1.vector_db import VectorDBfrom unstract.sdk1.x2txt import X2Text@@ -199,8 +199,8 @@ class Index:extracted_text = process_response.extracted_text# TODO: Handle prepend of context where error is raised and remove thisexcept AdapterError as e:- msg = f"Error from text extractor '{x2text.x2text_instance.get_name()}'. "- msg += str(e)+ adapter_info = getattr(x2text, "_adapter_name", "") or x2text_instance_id+ msg = f"Error from text extractor '{adapter_info}'. {e}"raise X2TextError(msg) from eif process_text:try:@@ -496,11 +496,15 @@ class Index:# Whole adapter config is used currently even though it contains some keys# which might not be relevant to indexing. This is easier for now than# marking certain keys of the adapter config as necessary.+ vector_db_config = PlatformHelper.get_adapter_config(self.tool, vector_db)+ embedding_config = PlatformHelper.get_adapter_config(self.tool, embedding)+ x2text_config = PlatformHelper.get_adapter_config(self.tool, x2text)+ Utils.strip_adapter_name(vector_db_config, embedding_config, x2text_config)index_key = {"file_hash": file_hash,- "vector_db_config": PlatformHelper.get_adapter_config(self.tool, vector_db),- "embedding_config": PlatformHelper.get_adapter_config(self.tool, embedding),- "x2text_config": PlatformHelper.get_adapter_config(self.tool, x2text),+ "vector_db_config": vector_db_config,+ "embedding_config": embedding_config,+ "x2text_config": x2text_config,# Typed and hashed as strings since the final hash is persisted# and this is required to be backward compatible"chunk_size": str(chunk_size),diff --git a/unstract/sdk1/src/unstract/sdk1/llm.py b/unstract/sdk1/src/unstract/sdk1/llm.pyindex b907cd396..dd3660eca 100644--- a/unstract/sdk1/src/unstract/sdk1/llm.py+++ b/unstract/sdk1/src/unstract/sdk1/llm.py@@ -12,6 +12,7 @@ from pydantic import ValidationErrorfrom unstract.sdk1.adapters.constants import Commonfrom unstract.sdk1.adapters.llm1 import adaptersfrom unstract.sdk1.audit import Audit+from unstract.sdk1.constants import Common as SdkCommonfrom unstract.sdk1.constants import ToolEnvfrom unstract.sdk1.exceptions import LLMError, SdkError, strip_litellm_prefixfrom unstract.sdk1.platform import PlatformHelper@@ -85,6 +86,7 @@ class LLM:self._adapter_id = llm_config[Common.ADAPTER_ID]self._adapter_metadata = llm_config[Common.ADAPTER_METADATA]self._adapter_instance_id = adapter_instance_id+ self._adapter_name = llm_config.pop(SdkCommon.ADAPTER_NAME, "")self._tool = toolelse:self._adapter_id = adapter_id@@ -93,6 +95,7 @@ class LLM:else:self._adapter_metadata = adapters[self._adapter_id][Common.METADATA]self._adapter_instance_id = ""+ self._adapter_name = ""self._tool = None# Retrieve the adapter class.@@ -138,6 +141,13 @@ class LLM:self._capture_metrics = capture_metrics_from_platformself._metrics: dict[str, object] = {}+ def _get_adapter_info(self) -> str:+ """Build a display string identifying this adapter for errors."""+ provider = self.adapter.get_provider()+ if self._adapter_name:+ return f"{self._adapter_name} ({provider})"+ return provider+def test_connection(self) -> bool:"""Test connection to the LLM provider."""try:@@ -266,7 +276,7 @@ class LLM:status_code = e.http_statuserror_msg = (- f"Error from LLM provider '{self.adapter.get_provider()}': "+ f"Error from LLM adapter '{self._get_adapter_info()}': "f"{strip_litellm_prefix(str(e))}")@@ -342,7 +352,7 @@ class LLM:status_code = e.http_statuserror_msg = (- f"Error from LLM provider '{self.adapter.get_provider()}': "+ f"Error from LLM adapter '{self._get_adapter_info()}': "f"{strip_litellm_prefix(str(e))}")@@ -401,7 +411,7 @@ class LLM:status_code = e.http_statuserror_msg = (- f"Error from LLM provider '{self.adapter.get_provider()}': "+ f"Error from LLM adapter '{self._get_adapter_info()}': "f"{strip_litellm_prefix(str(e))}")diff --git a/unstract/sdk1/src/unstract/sdk1/platform.py b/unstract/sdk1/src/unstract/sdk1/platform.pyindex a7995164c..2a3b75a97 100644--- a/unstract/sdk1/src/unstract/sdk1/platform.py+++ b/unstract/sdk1/src/unstract/sdk1/platform.py@@ -7,6 +7,7 @@ from requests import RequestException, Responsefrom requests.exceptions import ConnectionError, HTTPErrorfrom unstract.sdk1.constants import (AdapterKeys,+ Common,LogLevel,MimeType,PromptStudioKeys,@@ -138,6 +139,9 @@ class PlatformHelper:adapter_name = adapter_data.pop("adapter_name", "")adapter_type = adapter_data.pop("adapter_type", "")provider = adapter_data.get("adapter_id", "").split("|")[0]+ # Store adapter name under a private key for use in error messages.+ # This key is stripped before hashing in index key generation.+ adapter_data[Common.ADAPTER_NAME] = adapter_name# TODO: Print metadata after redacting sensitive informationtool.stream_log(f"Retrieved config for '{adapter_instance_id}', type: "diff --git a/unstract/sdk1/src/unstract/sdk1/utils/common.py b/unstract/sdk1/src/unstract/sdk1/utils/common.pyindex c9ac9a236..53397ca3b 100644--- a/unstract/sdk1/src/unstract/sdk1/utils/common.py+++ b/unstract/sdk1/src/unstract/sdk1/utils/common.py@@ -36,6 +36,17 @@ class Utils:"""return string.lower() == "true"+ @staticmethod+ def strip_adapter_name(*configs: dict | None) -> None:+ """Strip the private ``_adapter_name`` key from adapter configs.++ Call this before hashing configs for index key generation so the+ adapter name (added for error reporting) does not affect the hash.+ """+ for config in configs:+ if config:+ config.pop(Common.ADAPTER_NAME, None)+@staticmethoddef pretty_file_size(num: float, suffix: str = "B") -> str:"""Gets the human readable size for a file.diff --git a/unstract/sdk1/src/unstract/sdk1/utils/indexing.py b/unstract/sdk1/src/unstract/sdk1/utils/indexing.pyindex 0e9bbef92..5d3604294 100644--- a/unstract/sdk1/src/unstract/sdk1/utils/indexing.py+++ b/unstract/sdk1/src/unstract/sdk1/utils/indexing.py@@ -3,6 +3,7 @@ import jsonfrom unstract.sdk1.file_storage import FileStorage, FileStorageProviderfrom unstract.sdk1.platform import PlatformHelperfrom unstract.sdk1.tool.base import BaseTool+from unstract.sdk1.utils.common import Utilsfrom unstract.sdk1.utils.tool import ToolUtils@@ -41,11 +42,15 @@ class IndexingUtils:# Whole adapter config is used currently even though it contains some keys# which might not be relevant to indexing. This is easier for now than# marking certain keys of the adapter config as necessary.+ vector_db_config = PlatformHelper.get_adapter_config(tool, vector_db)+ embedding_config = PlatformHelper.get_adapter_config(tool, embedding)+ x2text_config = PlatformHelper.get_adapter_config(tool, x2text)+ Utils.strip_adapter_name(vector_db_config, embedding_config, x2text_config)index_key = {"file_hash": file_hash,- "vector_db_config": PlatformHelper.get_adapter_config(tool, vector_db),- "embedding_config": PlatformHelper.get_adapter_config(tool, embedding),- "x2text_config": PlatformHelper.get_adapter_config(tool, x2text),+ "vector_db_config": vector_db_config,+ "embedding_config": embedding_config,+ "x2text_config": x2text_config,# Typed and hashed as strings since the final hash is persisted# and this is required to be backward compatible"chunk_size": str(chunk_size),diff --git a/unstract/sdk1/src/unstract/sdk1/vector_db.py b/unstract/sdk1/src/unstract/sdk1/vector_db.pyindex 9638faf35..9225fb800 100644--- a/unstract/sdk1/src/unstract/sdk1/vector_db.py+++ b/unstract/sdk1/src/unstract/sdk1/vector_db.py@@ -87,6 +87,7 @@ class VectorDB:self._tool, self._adapter_instance_id)+ self._adapter_name = vector_db_config.pop(Common.ADAPTER_NAME, "")vector_db_adapter_id = vector_db_config.get(Common.ADAPTER_ID)if vector_db_adapter_id not in self.vector_db_adapters:raise SdkError(f"VectorDB adapter not supported : {vector_db_adapter_id}")@@ -109,11 +110,14 @@ class VectorDB:self.vector_db_adapter_class = vector_db_adapter(vector_db_metadata)return self.vector_db_adapter_class.get_vector_db_instance()except Exception as e:+ adapter_info = getattr(self, "_adapter_name", "") or self._adapter_instance_idself._tool.stream_log(- log=f"Unable to get vector_db {self._adapter_instance_id}: {e}",+ log=f"Unable to get vector_db '{adapter_info}': {e}",level=LogLevel.ERROR,)- raise VectorDBError(f"Error getting vectorDB instance: {e}") from e+ raise VectorDBError(+ f"Error getting vectorDB instance '{adapter_info}': {e}"+ ) from edef index_document(self,diff --git a/unstract/sdk1/src/unstract/sdk1/x2txt.py b/unstract/sdk1/src/unstract/sdk1/x2txt.pyindex c45bb0a15..2024f8cdb 100644--- a/unstract/sdk1/src/unstract/sdk1/x2txt.py+++ b/unstract/sdk1/src/unstract/sdk1/x2txt.py@@ -8,6 +8,7 @@ from unstract.sdk1.adapters.x2text.constants import X2TextConstantsfrom unstract.sdk1.adapters.x2text.dto import TextExtractionResultfrom unstract.sdk1.adapters.x2text.x2text_adapter import X2TextAdapterfrom unstract.sdk1.audit import Audit+from unstract.sdk1.constants import Common as SdkCommonfrom unstract.sdk1.constants import LogLevel, MimeType, ToolEnvfrom unstract.sdk1.exceptions import X2TextErrorfrom unstract.sdk1.file_storage import FileStorage, FileStorageProvider@@ -56,6 +57,7 @@ class X2Text:self._tool, self._adapter_instance_id)+ self._adapter_name = x2text_config.pop(SdkCommon.ADAPTER_NAME, "")x2text_adapter_id = x2text_config.get(Common.ADAPTER_ID)if x2text_adapter_id in self._x2text_adapters:x2text_adapter = self._x2text_adapters[x2text_adapter_id][@@ -84,11 +86,14 @@ class X2Text:return self._x2text_instanceexcept Exception as e:+ adapter_info = getattr(self, "_adapter_name", "") or self._adapter_instance_idself._tool.stream_log(- log=f"Unable to get x2text adapter {self._adapter_instance_id}: {e}",+ log=f"Unable to get x2text adapter '{adapter_info}': {e}",level=LogLevel.ERROR,)- raise X2TextError(f"Error getting text extractor: {e}") from e+ raise X2TextError(+ f"Error getting text extractor '{adapter_info}': {e}"+ ) from edef process(self,