LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
feat: Text extractor related enhancements (#168)
Chandrasekharan M645a3cf26eb1
diff --git a/backend/prompt_studio/prompt_studio_core/exceptions.py b/backend/prompt_studio/prompt_studio_core/exceptions.pyindex 7004b42b4..9f4d2b73b 100644--- a/backend/prompt_studio/prompt_studio_core/exceptions.py+++ b/backend/prompt_studio/prompt_studio_core/exceptions.py@@ -18,7 +18,7 @@ class PromptNotValid(APIException):Seems it is either empty or no prompt is mapped."-class IndexingError(APIException):+class IndexingAPIError(APIException):status_code = 400default_detail = "Error while indexing file"diff --git a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.pyindex 57be7f5c0..e2ad5f859 100644--- a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py+++ b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py@@ -18,7 +18,7 @@ from prompt_studio.prompt_studio_core.exceptions import (AnswerFetchError,DefaultProfileError,EmptyPromptError,- IndexingError,+ IndexingAPIError,NoPromptsFound,PermissionError,PromptNotValid,@@ -34,8 +34,9 @@ from prompt_studio.prompt_studio_index_manager.prompt_studio_index_helper importfrom prompt_studio.prompt_studio_output_manager.output_manager_helper import (OutputManagerHelper,)+from rest_framework.exceptions import APIExceptionfrom unstract.sdk.constants import LogLevel-from unstract.sdk.exceptions import SdkError+from unstract.sdk.exceptions import IndexingErrorfrom unstract.sdk.index import ToolIndexfrom unstract.sdk.prompt import PromptToolfrom unstract.sdk.utils.tool_utils import ToolUtils@@ -359,6 +360,9 @@ class PromptStudioHelper:LogLevels.RUN,"Failed to fetch prompt response",)+ # TODO: Catch specific exceptions and remove this+ if isinstance(exc, APIException):+ raise excraise AnswerFetchError()logger.info(@@ -615,7 +619,7 @@ class PromptStudioHelper:tool_index = ToolIndex(tool=util)except Exception as e:logger.error(f"Error while instatiating SDKs {e}")- raise IndexingError()+ raise IndexingAPIError()embedding_model = str(profile_manager.embedding_model.id)vector_db = str(profile_manager.vector_store.id)x2text_adapter = str(profile_manager.x2text.id)@@ -649,8 +653,8 @@ class PromptStudioHelper:doc_id=doc_id,)return doc_id- except SdkError as e:- raise IndexingError(str(e))+ except IndexingError as e:+ raise IndexingAPIError(str(e)) from e@staticmethoddef _fetch_single_pass_response(diff --git a/backend/prompt_studio/prompt_studio_core/views.py b/backend/prompt_studio/prompt_studio_core/views.pyindex 27e1a8f2b..b5b575bf8 100644--- a/backend/prompt_studio/prompt_studio_core/views.py+++ b/backend/prompt_studio/prompt_studio_core/views.py@@ -18,7 +18,7 @@ from prompt_studio.prompt_studio_core.constants import (ToolStudioPromptKeys,)from prompt_studio.prompt_studio_core.exceptions import (- IndexingError,+ IndexingAPIError,ToolDeleteError,)from prompt_studio.prompt_studio_core.prompt_studio_helper import (@@ -231,7 +231,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):logger.error("Error occured while indexing. Unique ID is not valid.")- raise IndexingError()+ raise IndexingAPIError()@action(detail=True, methods=["post"])def fetch_response(self, request: HttpRequest) -> Response:diff --git a/frontend/src/components/input-output/list-of-sources/ListOfSources.jsx b/frontend/src/components/input-output/list-of-sources/ListOfSources.jsxindex b6c938b3d..5fd1e7276 100644--- a/frontend/src/components/input-output/list-of-sources/ListOfSources.jsx+++ b/frontend/src/components/input-output/list-of-sources/ListOfSources.jsx@@ -29,10 +29,6 @@ function ListOfSources({ setSelectedSourceId, type }) {"replicate|2715ce84-05af-4ab4-b8e9-67ac3211b81e","anthropic|90ebd4cd-2f19-4cef-a884-9eeb6ac0f203",],- X2TEXT: [- "unstructuredenterprise|eb1b6c58-221f-4db0-a4a5-e5f9cdca44e1",- "unstructuredcommunity|eeed506f-1875-457f-9101-846fc7115676",- ],VECTOR_DB: ["supabase|e6998e3c-3595-48c0-a190-188dbd803858","pinecone|83881133-485d-4ecc-b1f7-0009f96dc74a",diff --git a/frontend/src/index.js b/frontend/src/index.jsindex 1be39951e..4ee8e6c3d 100644--- a/frontend/src/index.js+++ b/frontend/src/index.js@@ -8,7 +8,7 @@ import { LazyLoader } from "./components/widgets/lazy-loader/LazyLoader.jsx";import { SocketProvider } from "./helpers/SocketContext.js";import "./index.css";-const API_KEY = "phc_f1kLKkipCK3kBtA9bT0SfAChvFrlUxYSMD91GBNbwr1";+const API_KEY = "phc_f1kLKkipCK3kBtA9bT0SfAChvFrlUxYSMD91GBNbwr1"; // gitleaks:allowconst API_HOST = "https://app.posthog.com";posthog.init(API_KEY, {api_host: API_HOST,diff --git a/x2text-service/app/controllers/controller.py b/x2text-service/app/controllers/controller.pyindex 56e865fb8..82f6a2dc8 100644--- a/x2text-service/app/controllers/controller.py+++ b/x2text-service/app/controllers/controller.py@@ -32,6 +32,9 @@ def health() -> str:@basic.route("/test-connection", methods=["POST"])@authentication_middlewaredef test_connection() -> Any:+ logging.info(+ "Received a test connection request from %s", request.remote_addr+ )form_data = dict(request.form)unstructured_api_key = X2TextUtil.get_value_for_key(UNSTRUCTURED_API_KEY, form_data@@ -60,7 +63,7 @@ def test_connection() -> Any:if response.status_code == 400:"""Response is 400 as we are not sending a file to test connection.- But it has passed dcredential check and url check+ But it has passed credential check and url check"""return {"message": "Test connection sucessful"}, 200@@ -77,7 +80,7 @@ def test_connection() -> Any:@authentication_middlewaredef process() -> Any:logging.info(- "Received a doc processing request from %s", request.remote_addr+ "Received a doc processing request from %s", request.remote_addr)form_data = dict(request.form)url = X2TextUtil.get_value_for_key(UNSTRUCTURED_URL, form_data)@@ -130,7 +133,7 @@ def process() -> Any:files=files,timeout=None,)- if response.status_code == 200:+ if response.ok:json_response = response.json()response_text = X2TextUtil.get_text_content(json_response)file_stream = BytesIO(response_text.encode("utf-8"))@@ -141,4 +144,8 @@ def process() -> Any:)x2_text_audit.status = "Failed"x2_text_audit.save()- return response.text, response.status_code+ return_val = X2TextUtil.read_response(response=response)+ logging.error(+ "Text extraction failed: [%s] %s", response.status_code, return_val+ )+ return return_val, response.status_codediff --git a/x2text-service/app/util.py b/x2text-service/app/util.pyindex 2489a9241..97e0d5533 100644--- a/x2text-service/app/util.py+++ b/x2text-service/app/util.py@@ -1,12 +1,9 @@from typing import Any+from requests import Response-class X2TextUtil:- @staticmethod- def get_url_from_form_data(form_data: dict[str, Any]) -> str:- url: str = form_data.pop("unstructured-url")- return url+class X2TextUtil:@staticmethoddef get_value_for_key(key: str, form_data: dict[str, Any]) -> str:value: str = form_data.pop(key, None)@@ -18,3 +15,10 @@ class X2TextUtil:item["text"] for item in json_response # type:ignore)return combined_text++ @staticmethod+ def read_response(response: Response) -> dict[str, Any]:+ if response.headers.get("Content-Type") == "application/json":+ return response.json() # type: ignore+ else:+ return {"message": response.text}