Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2105 [FEAT] Include adapter name in error messages (#1825)

Praveen Kumara20d7a05aab3
diff --git a/prompt-service/src/unstract/prompt_service/core/index_v2.py b/prompt-service/src/unstract/prompt_service/core/index_v2.py
index 6e621d6ec..5f1703b32 100644
--- a/prompt-service/src/unstract/prompt_service/core/index_v2.py
+++ b/prompt-service/src/unstract/prompt_service/core/index_v2.py
@@ -28,7 +28,7 @@ from unstract.sdk1.file_storage.impl import FileStorage
from unstract.sdk1.file_storage.provider import FileStorageProvider
from unstract.sdk1.platform import PlatformHelper as ToolAdapter
from unstract.sdk1.tool.stream import StreamMixin
-from unstract.sdk1.utils.common import capture_metrics
+from unstract.sdk1.utils.common import Utils, capture_metrics
from unstract.sdk1.utils.tool import ToolUtils
from unstract.sdk1.vector_db import VectorDB
@@ -83,17 +83,21 @@ class Index:
# Whole adapter config is used currently even though it contains some keys
# which might not be relevant to indexing. This is easier for now than
# marking certain keys of the adapter config as necessary.
+ vector_db_config = ToolAdapter.get_adapter_config(
+ self.tool, self.instance_identifiers.vector_db_instance_id
+ )
+ embedding_config = ToolAdapter.get_adapter_config(
+ self.tool, self.instance_identifiers.embedding_instance_id
+ )
+ x2text_config = ToolAdapter.get_adapter_config(
+ self.tool, self.instance_identifiers.x2text_instance_id
+ )
+ Utils.strip_adapter_name(vector_db_config, embedding_config, x2text_config)
index_key = {
"file_hash": file_hash,
- "vector_db_config": ToolAdapter.get_adapter_config(
- self.tool, self.instance_identifiers.vector_db_instance_id
- ),
- "embedding_config": ToolAdapter.get_adapter_config(
- self.tool, self.instance_identifiers.embedding_instance_id
- ),
- "x2text_config": ToolAdapter.get_adapter_config(
- self.tool, self.instance_identifiers.x2text_instance_id
- ),
+ "vector_db_config": vector_db_config,
+ "embedding_config": embedding_config,
+ "x2text_config": x2text_config,
# Typed and hashed as strings since the final hash is persisted
# and this is required to be backward compatible
"chunk_size": str(self.chunking_config.chunk_size),
diff --git a/unstract/sdk1/src/unstract/sdk1/constants.py b/unstract/sdk1/src/unstract/sdk1/constants.py
index 65119d8ba..3494a8e2a 100644
--- a/unstract/sdk1/src/unstract/sdk1/constants.py
+++ b/unstract/sdk1/src/unstract/sdk1/constants.py
@@ -10,6 +10,7 @@ class Common:
ICON = "icon"
ADAPTER_ID = "adapter_id"
ADAPTER_TYPE = "adapter_type"
+ ADAPTER_NAME = "_adapter_name"
DEFAULT_ERR_MESSAGE = "Something went wrong"
diff --git a/unstract/sdk1/src/unstract/sdk1/embedding.py b/unstract/sdk1/src/unstract/sdk1/embedding.py
index 758aa0e1c..c12ccea89 100644
--- a/unstract/sdk1/src/unstract/sdk1/embedding.py
+++ b/unstract/sdk1/src/unstract/sdk1/embedding.py
@@ -8,6 +8,7 @@ from llama_index.core.embeddings import BaseEmbedding
from pydantic import ValidationError
from unstract.sdk1.adapters.constants import Common
from unstract.sdk1.adapters.embedding1 import adapters
+from unstract.sdk1.constants import Common as SdkCommon
from unstract.sdk1.constants import ToolEnv
from unstract.sdk1.exceptions import SdkError, parse_litellm_err
from unstract.sdk1.platform import PlatformHelper
@@ -66,6 +67,7 @@ class Embedding:
self._adapter_id = embedding_config[Common.ADAPTER_ID]
self._adapter_metadata = embedding_config[Common.ADAPTER_METADATA]
self._adapter_instance_id = adapter_instance_id
+ self._adapter_name = embedding_config.pop(SdkCommon.ADAPTER_NAME, "")
self._tool = tool
else:
self._adapter_id = adapter_id
@@ -75,6 +77,7 @@ class Embedding:
else:
self._adapter_metadata = adapters[self._adapter_id][Common.METADATA]
self._adapter_instance_id = ""
+ self._adapter_name = ""
self._tool = None
# Retrieve the adapter class.
@@ -95,8 +98,14 @@ class Embedding:
try:
self._length = len(self.get_embedding(self._TEST_SNIPPET))
except Exception as e:
- provider_name = f"{self.adapter.get_name()}"
- raise parse_litellm_err(e, provider_name) from e
+ raise parse_litellm_err(e, self._get_adapter_info()) from e
+
+ def _get_adapter_info(self) -> str:
+ """Build a display string identifying this adapter for errors."""
+ name = self.adapter.get_name()
+ if self._adapter_name:
+ return f"{self._adapter_name} ({name})"
+ return name
def get_embedding(self, text: str) -> list[float]:
"""Return embedding vector for query string."""
@@ -110,8 +119,7 @@ class Embedding:
return resp["data"][0]["embedding"]
except Exception as e:
- provider_name = f"{self.adapter.get_name()}"
- raise parse_litellm_err(e, provider_name) from e
+ raise parse_litellm_err(e, self._get_adapter_info()) from e
def get_embeddings(self, texts: list[str]) -> list[list[float]]:
"""Return embedding vectors for list of query strings."""
@@ -123,8 +131,7 @@ class Embedding:
return [data["embedding"] for data in resp["data"]]
except Exception as e:
- provider_name = f"{self.adapter.get_name()}"
- raise parse_litellm_err(e, provider_name) from e
+ raise parse_litellm_err(e, self._get_adapter_info()) from e
async def get_aembedding(self, text: str) -> list[float]:
"""Return async embedding vector for query string."""
diff --git a/unstract/sdk1/src/unstract/sdk1/index.py b/unstract/sdk1/src/unstract/sdk1/index.py
index ed4867963..92220f0a8 100644
--- a/unstract/sdk1/src/unstract/sdk1/index.py
+++ b/unstract/sdk1/src/unstract/sdk1/index.py
@@ -24,7 +24,7 @@ from unstract.sdk1.embedding import EmbeddingCompat
from unstract.sdk1.exceptions import IndexingError, SdkError, VectorDBError, X2TextError
from unstract.sdk1.file_storage import FileStorage, FileStorageProvider
from unstract.sdk1.platform import PlatformHelper
-from unstract.sdk1.utils.common import capture_metrics, log_elapsed
+from unstract.sdk1.utils.common import Utils, capture_metrics, log_elapsed
from unstract.sdk1.utils.tool import ToolUtils
from unstract.sdk1.vector_db import VectorDB
from unstract.sdk1.x2txt import X2Text
@@ -199,8 +199,8 @@ class Index:
extracted_text = process_response.extracted_text
# TODO: Handle prepend of context where error is raised and remove this
except AdapterError as e:
- msg = f"Error from text extractor '{x2text.x2text_instance.get_name()}'. "
- msg += str(e)
+ adapter_info = getattr(x2text, "_adapter_name", "") or x2text_instance_id
+ msg = f"Error from text extractor '{adapter_info}'. {e}"
raise X2TextError(msg) from e
if process_text:
try:
@@ -496,11 +496,15 @@ class Index:
# Whole adapter config is used currently even though it contains some keys
# which might not be relevant to indexing. This is easier for now than
# marking certain keys of the adapter config as necessary.
+ vector_db_config = PlatformHelper.get_adapter_config(self.tool, vector_db)
+ embedding_config = PlatformHelper.get_adapter_config(self.tool, embedding)
+ x2text_config = PlatformHelper.get_adapter_config(self.tool, x2text)
+ Utils.strip_adapter_name(vector_db_config, embedding_config, x2text_config)
index_key = {
"file_hash": file_hash,
- "vector_db_config": PlatformHelper.get_adapter_config(self.tool, vector_db),
- "embedding_config": PlatformHelper.get_adapter_config(self.tool, embedding),
- "x2text_config": PlatformHelper.get_adapter_config(self.tool, x2text),
+ "vector_db_config": vector_db_config,
+ "embedding_config": embedding_config,
+ "x2text_config": x2text_config,
# Typed and hashed as strings since the final hash is persisted
# and this is required to be backward compatible
"chunk_size": str(chunk_size),
diff --git a/unstract/sdk1/src/unstract/sdk1/llm.py b/unstract/sdk1/src/unstract/sdk1/llm.py
index b907cd396..dd3660eca 100644
--- a/unstract/sdk1/src/unstract/sdk1/llm.py
+++ b/unstract/sdk1/src/unstract/sdk1/llm.py
@@ -12,6 +12,7 @@ from pydantic import ValidationError
from unstract.sdk1.adapters.constants import Common
from unstract.sdk1.adapters.llm1 import adapters
from unstract.sdk1.audit import Audit
+from unstract.sdk1.constants import Common as SdkCommon
from unstract.sdk1.constants import ToolEnv
from unstract.sdk1.exceptions import LLMError, SdkError, strip_litellm_prefix
from unstract.sdk1.platform import PlatformHelper
@@ -85,6 +86,7 @@ class LLM:
self._adapter_id = llm_config[Common.ADAPTER_ID]
self._adapter_metadata = llm_config[Common.ADAPTER_METADATA]
self._adapter_instance_id = adapter_instance_id
+ self._adapter_name = llm_config.pop(SdkCommon.ADAPTER_NAME, "")
self._tool = tool
else:
self._adapter_id = adapter_id
@@ -93,6 +95,7 @@ class LLM:
else:
self._adapter_metadata = adapters[self._adapter_id][Common.METADATA]
self._adapter_instance_id = ""
+ self._adapter_name = ""
self._tool = None
# Retrieve the adapter class.
@@ -138,6 +141,13 @@ class LLM:
self._capture_metrics = capture_metrics_from_platform
self._metrics: dict[str, object] = {}
+ def _get_adapter_info(self) -> str:
+ """Build a display string identifying this adapter for errors."""
+ provider = self.adapter.get_provider()
+ if self._adapter_name:
+ return f"{self._adapter_name} ({provider})"
+ return provider
+
def test_connection(self) -> bool:
"""Test connection to the LLM provider."""
try:
@@ -266,7 +276,7 @@ class LLM:
status_code = e.http_status
error_msg = (
- f"Error from LLM provider '{self.adapter.get_provider()}': "
+ f"Error from LLM adapter '{self._get_adapter_info()}': "
f"{strip_litellm_prefix(str(e))}"
)
@@ -342,7 +352,7 @@ class LLM:
status_code = e.http_status
error_msg = (
- f"Error from LLM provider '{self.adapter.get_provider()}': "
+ f"Error from LLM adapter '{self._get_adapter_info()}': "
f"{strip_litellm_prefix(str(e))}"
)
@@ -401,7 +411,7 @@ class LLM:
status_code = e.http_status
error_msg = (
- f"Error from LLM provider '{self.adapter.get_provider()}': "
+ f"Error from LLM adapter '{self._get_adapter_info()}': "
f"{strip_litellm_prefix(str(e))}"
)
diff --git a/unstract/sdk1/src/unstract/sdk1/platform.py b/unstract/sdk1/src/unstract/sdk1/platform.py
index a7995164c..2a3b75a97 100644
--- a/unstract/sdk1/src/unstract/sdk1/platform.py
+++ b/unstract/sdk1/src/unstract/sdk1/platform.py
@@ -7,6 +7,7 @@ from requests import RequestException, Response
from requests.exceptions import ConnectionError, HTTPError
from unstract.sdk1.constants import (
AdapterKeys,
+ Common,
LogLevel,
MimeType,
PromptStudioKeys,
@@ -138,6 +139,9 @@ class PlatformHelper:
adapter_name = adapter_data.pop("adapter_name", "")
adapter_type = adapter_data.pop("adapter_type", "")
provider = adapter_data.get("adapter_id", "").split("|")[0]
+ # Store adapter name under a private key for use in error messages.
+ # This key is stripped before hashing in index key generation.
+ adapter_data[Common.ADAPTER_NAME] = adapter_name
# TODO: Print metadata after redacting sensitive information
tool.stream_log(
f"Retrieved config for '{adapter_instance_id}', type: "
diff --git a/unstract/sdk1/src/unstract/sdk1/utils/common.py b/unstract/sdk1/src/unstract/sdk1/utils/common.py
index c9ac9a236..53397ca3b 100644
--- a/unstract/sdk1/src/unstract/sdk1/utils/common.py
+++ b/unstract/sdk1/src/unstract/sdk1/utils/common.py
@@ -36,6 +36,17 @@ class Utils:
"""
return string.lower() == "true"
+ @staticmethod
+ def strip_adapter_name(*configs: dict | None) -> None:
+ """Strip the private ``_adapter_name`` key from adapter configs.
+
+ Call this before hashing configs for index key generation so the
+ adapter name (added for error reporting) does not affect the hash.
+ """
+ for config in configs:
+ if config:
+ config.pop(Common.ADAPTER_NAME, None)
+
@staticmethod
def pretty_file_size(num: float, suffix: str = "B") -> str:
"""Gets the human readable size for a file.
diff --git a/unstract/sdk1/src/unstract/sdk1/utils/indexing.py b/unstract/sdk1/src/unstract/sdk1/utils/indexing.py
index 0e9bbef92..5d3604294 100644
--- a/unstract/sdk1/src/unstract/sdk1/utils/indexing.py
+++ b/unstract/sdk1/src/unstract/sdk1/utils/indexing.py
@@ -3,6 +3,7 @@ import json
from unstract.sdk1.file_storage import FileStorage, FileStorageProvider
from unstract.sdk1.platform import PlatformHelper
from unstract.sdk1.tool.base import BaseTool
+from unstract.sdk1.utils.common import Utils
from unstract.sdk1.utils.tool import ToolUtils
@@ -41,11 +42,15 @@ class IndexingUtils:
# Whole adapter config is used currently even though it contains some keys
# which might not be relevant to indexing. This is easier for now than
# marking certain keys of the adapter config as necessary.
+ vector_db_config = PlatformHelper.get_adapter_config(tool, vector_db)
+ embedding_config = PlatformHelper.get_adapter_config(tool, embedding)
+ x2text_config = PlatformHelper.get_adapter_config(tool, x2text)
+ Utils.strip_adapter_name(vector_db_config, embedding_config, x2text_config)
index_key = {
"file_hash": file_hash,
- "vector_db_config": PlatformHelper.get_adapter_config(tool, vector_db),
- "embedding_config": PlatformHelper.get_adapter_config(tool, embedding),
- "x2text_config": PlatformHelper.get_adapter_config(tool, x2text),
+ "vector_db_config": vector_db_config,
+ "embedding_config": embedding_config,
+ "x2text_config": x2text_config,
# Typed and hashed as strings since the final hash is persisted
# and this is required to be backward compatible
"chunk_size": str(chunk_size),
diff --git a/unstract/sdk1/src/unstract/sdk1/vector_db.py b/unstract/sdk1/src/unstract/sdk1/vector_db.py
index 9638faf35..9225fb800 100644
--- a/unstract/sdk1/src/unstract/sdk1/vector_db.py
+++ b/unstract/sdk1/src/unstract/sdk1/vector_db.py
@@ -87,6 +87,7 @@ class VectorDB:
self._tool, self._adapter_instance_id
)
+ self._adapter_name = vector_db_config.pop(Common.ADAPTER_NAME, "")
vector_db_adapter_id = vector_db_config.get(Common.ADAPTER_ID)
if vector_db_adapter_id not in self.vector_db_adapters:
raise SdkError(f"VectorDB adapter not supported : {vector_db_adapter_id}")
@@ -109,11 +110,14 @@ class VectorDB:
self.vector_db_adapter_class = vector_db_adapter(vector_db_metadata)
return self.vector_db_adapter_class.get_vector_db_instance()
except Exception as e:
+ adapter_info = getattr(self, "_adapter_name", "") or self._adapter_instance_id
self._tool.stream_log(
- log=f"Unable to get vector_db {self._adapter_instance_id}: {e}",
+ log=f"Unable to get vector_db '{adapter_info}': {e}",
level=LogLevel.ERROR,
)
- raise VectorDBError(f"Error getting vectorDB instance: {e}") from e
+ raise VectorDBError(
+ f"Error getting vectorDB instance '{adapter_info}': {e}"
+ ) from e
def index_document(
self,
diff --git a/unstract/sdk1/src/unstract/sdk1/x2txt.py b/unstract/sdk1/src/unstract/sdk1/x2txt.py
index c45bb0a15..2024f8cdb 100644
--- a/unstract/sdk1/src/unstract/sdk1/x2txt.py
+++ b/unstract/sdk1/src/unstract/sdk1/x2txt.py
@@ -8,6 +8,7 @@ from unstract.sdk1.adapters.x2text.constants import X2TextConstants
from unstract.sdk1.adapters.x2text.dto import TextExtractionResult
from unstract.sdk1.adapters.x2text.x2text_adapter import X2TextAdapter
from unstract.sdk1.audit import Audit
+from unstract.sdk1.constants import Common as SdkCommon
from unstract.sdk1.constants import LogLevel, MimeType, ToolEnv
from unstract.sdk1.exceptions import X2TextError
from unstract.sdk1.file_storage import FileStorage, FileStorageProvider
@@ -56,6 +57,7 @@ class X2Text:
self._tool, self._adapter_instance_id
)
+ self._adapter_name = x2text_config.pop(SdkCommon.ADAPTER_NAME, "")
x2text_adapter_id = x2text_config.get(Common.ADAPTER_ID)
if x2text_adapter_id in self._x2text_adapters:
x2text_adapter = self._x2text_adapters[x2text_adapter_id][
@@ -84,11 +86,14 @@ class X2Text:
return self._x2text_instance
except Exception as e:
+ adapter_info = getattr(self, "_adapter_name", "") or self._adapter_instance_id
self._tool.stream_log(
- log=f"Unable to get x2text adapter {self._adapter_instance_id}: {e}",
+ log=f"Unable to get x2text adapter '{adapter_info}': {e}",
level=LogLevel.ERROR,
)
- raise X2TextError(f"Error getting text extractor: {e}") from e
+ raise X2TextError(
+ f"Error getting text extractor '{adapter_info}': {e}"
+ ) from e
def process(
self,