Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

feat: Text extractor related enhancements (#168)

Chandrasekharan M645a3cf26eb1
diff --git a/backend/prompt_studio/prompt_studio_core/exceptions.py b/backend/prompt_studio/prompt_studio_core/exceptions.py
index 7004b42b4..9f4d2b73b 100644
--- a/backend/prompt_studio/prompt_studio_core/exceptions.py
+++ b/backend/prompt_studio/prompt_studio_core/exceptions.py
@@ -18,7 +18,7 @@ class PromptNotValid(APIException):
Seems it is either empty or no prompt is mapped."
-class IndexingError(APIException):
+class IndexingAPIError(APIException):
status_code = 400
default_detail = "Error while indexing file"
diff --git a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py
index 57be7f5c0..e2ad5f859 100644
--- a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py
+++ b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py
@@ -18,7 +18,7 @@ from prompt_studio.prompt_studio_core.exceptions import (
AnswerFetchError,
DefaultProfileError,
EmptyPromptError,
- IndexingError,
+ IndexingAPIError,
NoPromptsFound,
PermissionError,
PromptNotValid,
@@ -34,8 +34,9 @@ from prompt_studio.prompt_studio_index_manager.prompt_studio_index_helper import
from prompt_studio.prompt_studio_output_manager.output_manager_helper import (
OutputManagerHelper,
)
+from rest_framework.exceptions import APIException
from unstract.sdk.constants import LogLevel
-from unstract.sdk.exceptions import SdkError
+from unstract.sdk.exceptions import IndexingError
from unstract.sdk.index import ToolIndex
from unstract.sdk.prompt import PromptTool
from unstract.sdk.utils.tool_utils import ToolUtils
@@ -359,6 +360,9 @@ class PromptStudioHelper:
LogLevels.RUN,
"Failed to fetch prompt response",
)
+ # TODO: Catch specific exceptions and remove this
+ if isinstance(exc, APIException):
+ raise exc
raise AnswerFetchError()
logger.info(
@@ -615,7 +619,7 @@ class PromptStudioHelper:
tool_index = ToolIndex(tool=util)
except Exception as e:
logger.error(f"Error while instatiating SDKs {e}")
- raise IndexingError()
+ raise IndexingAPIError()
embedding_model = str(profile_manager.embedding_model.id)
vector_db = str(profile_manager.vector_store.id)
x2text_adapter = str(profile_manager.x2text.id)
@@ -649,8 +653,8 @@ class PromptStudioHelper:
doc_id=doc_id,
)
return doc_id
- except SdkError as e:
- raise IndexingError(str(e))
+ except IndexingError as e:
+ raise IndexingAPIError(str(e)) from e
@staticmethod
def _fetch_single_pass_response(
diff --git a/backend/prompt_studio/prompt_studio_core/views.py b/backend/prompt_studio/prompt_studio_core/views.py
index 27e1a8f2b..b5b575bf8 100644
--- a/backend/prompt_studio/prompt_studio_core/views.py
+++ b/backend/prompt_studio/prompt_studio_core/views.py
@@ -18,7 +18,7 @@ from prompt_studio.prompt_studio_core.constants import (
ToolStudioPromptKeys,
)
from prompt_studio.prompt_studio_core.exceptions import (
- IndexingError,
+ IndexingAPIError,
ToolDeleteError,
)
from prompt_studio.prompt_studio_core.prompt_studio_helper import (
@@ -231,7 +231,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
logger.error(
"Error occured while indexing. Unique ID is not valid."
)
- raise IndexingError()
+ raise IndexingAPIError()
@action(detail=True, methods=["post"])
def fetch_response(self, request: HttpRequest) -> Response:
diff --git a/frontend/src/components/input-output/list-of-sources/ListOfSources.jsx b/frontend/src/components/input-output/list-of-sources/ListOfSources.jsx
index b6c938b3d..5fd1e7276 100644
--- a/frontend/src/components/input-output/list-of-sources/ListOfSources.jsx
+++ b/frontend/src/components/input-output/list-of-sources/ListOfSources.jsx
@@ -29,10 +29,6 @@ function ListOfSources({ setSelectedSourceId, type }) {
"replicate|2715ce84-05af-4ab4-b8e9-67ac3211b81e",
"anthropic|90ebd4cd-2f19-4cef-a884-9eeb6ac0f203",
],
- X2TEXT: [
- "unstructuredenterprise|eb1b6c58-221f-4db0-a4a5-e5f9cdca44e1",
- "unstructuredcommunity|eeed506f-1875-457f-9101-846fc7115676",
- ],
VECTOR_DB: [
"supabase|e6998e3c-3595-48c0-a190-188dbd803858",
"pinecone|83881133-485d-4ecc-b1f7-0009f96dc74a",
diff --git a/frontend/src/index.js b/frontend/src/index.js
index 1be39951e..4ee8e6c3d 100644
--- a/frontend/src/index.js
+++ b/frontend/src/index.js
@@ -8,7 +8,7 @@ import { LazyLoader } from "./components/widgets/lazy-loader/LazyLoader.jsx";
import { SocketProvider } from "./helpers/SocketContext.js";
import "./index.css";
-const API_KEY = "phc_f1kLKkipCK3kBtA9bT0SfAChvFrlUxYSMD91GBNbwr1";
+const API_KEY = "phc_f1kLKkipCK3kBtA9bT0SfAChvFrlUxYSMD91GBNbwr1"; // gitleaks:allow
const API_HOST = "https://app.posthog.com";
posthog.init(API_KEY, {
api_host: API_HOST,
diff --git a/x2text-service/app/controllers/controller.py b/x2text-service/app/controllers/controller.py
index 56e865fb8..82f6a2dc8 100644
--- a/x2text-service/app/controllers/controller.py
+++ b/x2text-service/app/controllers/controller.py
@@ -32,6 +32,9 @@ def health() -> str:
@basic.route("/test-connection", methods=["POST"])
@authentication_middleware
def test_connection() -> Any:
+ logging.info(
+ "Received a test connection request from %s", request.remote_addr
+ )
form_data = dict(request.form)
unstructured_api_key = X2TextUtil.get_value_for_key(
UNSTRUCTURED_API_KEY, form_data
@@ -60,7 +63,7 @@ def test_connection() -> Any:
if response.status_code == 400:
"""Response is 400 as we are not sending a file to test connection.
- But it has passed dcredential check and url check
+ But it has passed credential check and url check
"""
return {"message": "Test connection sucessful"}, 200
@@ -77,7 +80,7 @@ def test_connection() -> Any:
@authentication_middleware
def process() -> Any:
logging.info(
- "Received a doc processing request from %s", request.remote_addr
+ "Received a doc processing request from %s", request.remote_addr
)
form_data = dict(request.form)
url = X2TextUtil.get_value_for_key(UNSTRUCTURED_URL, form_data)
@@ -130,7 +133,7 @@ def process() -> Any:
files=files,
timeout=None,
)
- if response.status_code == 200:
+ if response.ok:
json_response = response.json()
response_text = X2TextUtil.get_text_content(json_response)
file_stream = BytesIO(response_text.encode("utf-8"))
@@ -141,4 +144,8 @@ def process() -> Any:
)
x2_text_audit.status = "Failed"
x2_text_audit.save()
- return response.text, response.status_code
+ return_val = X2TextUtil.read_response(response=response)
+ logging.error(
+ "Text extraction failed: [%s] %s", response.status_code, return_val
+ )
+ return return_val, response.status_code
diff --git a/x2text-service/app/util.py b/x2text-service/app/util.py
index 2489a9241..97e0d5533 100644
--- a/x2text-service/app/util.py
+++ b/x2text-service/app/util.py
@@ -1,12 +1,9 @@
from typing import Any
+from requests import Response
-class X2TextUtil:
- @staticmethod
- def get_url_from_form_data(form_data: dict[str, Any]) -> str:
- url: str = form_data.pop("unstructured-url")
- return url
+class X2TextUtil:
@staticmethod
def get_value_for_key(key: str, form_data: dict[str, Any]) -> str:
value: str = form_data.pop(key, None)
@@ -18,3 +15,10 @@ class X2TextUtil:
item["text"] for item in json_response # type:ignore
)
return combined_text
+
+ @staticmethod
+ def read_response(response: Response) -> dict[str, Any]:
+ if response.headers.get("Content-Type") == "application/json":
+ return response.json() # type: ignore
+ else:
+ return {"message": response.text}