LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
[FIX] Fixes for v0.81.0 (#592)
Deepak K8d21cb1cd115
diff --git a/backend/api/api_deployment_views.py b/backend/api/api_deployment_views.pyindex ca65f00e2..2ac765469 100644--- a/backend/api/api_deployment_views.py+++ b/backend/api/api_deployment_views.py@@ -74,28 +74,28 @@ class DeploymentExecution(views.APIView):self, request: Request, org_name: str, api_name: str, api: APIDeployment) -> Response:execution_id = request.query_params.get("execution_id")- include_metadata = request.query_params.get("include_metadata", False)+ include_metadata = (+ request.query_params.get(ApiExecution.INCLUDE_METADATA, "false").lower()+ == "true"+ )if not execution_id:raise InvalidAPIRequest("execution_id shouldn't be empty")response: ExecutionResponse = DeploymentHelper.get_execution_status(execution_id=execution_id)- if response.execution_status != CeleryTaskState.SUCCESS.value:- return Response(- {- "status": response.execution_status,- "message": response.result,- },- status=status.HTTP_422_UNPROCESSABLE_ENTITY,- )- if (- response.execution_status == CeleryTaskState.SUCCESS.value- and not include_metadata- ):- response.remove_result_metadata_keys()+ response_status = status.HTTP_422_UNPROCESSABLE_ENTITY+ if response.execution_status == CeleryTaskState.COMPLETED.value:+ response_status = status.HTTP_200_OK+ if include_metadata:+ response.remove_result_metadata_keys(keys_to_remove=["highlight_data"])+ else:+ response.remove_result_metadata_keys()return Response(- {"status": response.execution_status, "message": response.result},- status=status.HTTP_200_OK,+ data={+ "status": response.execution_status,+ "message": response.result,+ },+ status=response_status,)diff --git a/backend/api/deployment_helper.py b/backend/api/deployment_helper.pyindex cca63fb9c..2b3adb3a0 100644--- a/backend/api/deployment_helper.py+++ b/backend/api/deployment_helper.py@@ -178,9 +178,10 @@ class DeploymentHelper(BaseAPIKeyValidator):result.status_api = DeploymentHelper.construct_status_endpoint(api_endpoint=api.api_endpoint, execution_id=execution_id)- if not include_metadata:+ if include_metadata:+ result.remove_result_metadata_keys(keys_to_remove=["highlight_data"])+ else:result.remove_result_metadata_keys()- cls._send_notification(api=api, result=result)except Exception as error:DestinationConnector.delete_api_storage_dir(workflow_id=workflow_id, execution_id=execution_iddiff --git a/backend/api/postman_collection/dto.py b/backend/api/postman_collection/dto.pyindex dc0e7be12..a10a440d5 100644--- a/backend/api/postman_collection/dto.py+++ b/backend/api/postman_collection/dto.py@@ -136,7 +136,10 @@ class APIDeploymentDto(APIBase):def _get_status_api_request(self) -> RequestItem:header_list = [HeaderItem(key="Authorization", value=f"Bearer {self.api_key}")]- status_query_param = {"execution_id": CollectionKey.STATUS_EXEC_ID_DEFAULT}+ status_query_param = {+ "execution_id": CollectionKey.STATUS_EXEC_ID_DEFAULT,+ ApiExecution.INCLUDE_METADATA: "False",+ }status_query_str = urlencode(status_query_param)abs_api_endpoint = urljoin(settings.WEB_APP_ORIGIN_URL, self.api_endpoint)status_url = urljoin(abs_api_endpoint, "?" + status_query_str)diff --git a/backend/prompt_studio/prompt_studio_core/constants.py b/backend/prompt_studio/prompt_studio_core/constants.pyindex 38c48dbd5..8fea5f4fd 100644--- a/backend/prompt_studio/prompt_studio_core/constants.py+++ b/backend/prompt_studio/prompt_studio_core/constants.py@@ -89,6 +89,7 @@ class ToolStudioPromptKeys:METADATA = "metadata"INCLUDE_METADATA = "include_metadata"PLATFORM_POSTAMBLE = "platform_postamble"+ SUMMARIZE_AS_SOURCE = "summarize_as_source"class FileViewTypes:diff --git a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.pyindex a81797d70..c215409e2 100644--- a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py+++ b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py@@ -29,7 +29,6 @@ from prompt_studio.prompt_studio_core.exceptions import (IndexingAPIError,NoPromptsFound,PermissionError,- ToolNotValid,)from prompt_studio.prompt_studio_core.models import CustomToolfrom prompt_studio.prompt_studio_core.prompt_ide_base_tool import PromptIdeBaseTool@@ -324,10 +323,6 @@ class PromptStudioHelper:)file_path = str(Path(file_path) / file_name)- if not tool:- logger.error(f"No tool instance found for the ID {tool_id}")- raise ToolNotValid()-logger.info(f"[{tool_id}] Indexing started for doc: {file_name}")PromptStudioHelper._publish_log({"tool_id": tool_id, "run_id": run_id, "doc_name": file_name},@@ -751,6 +746,7 @@ class PromptStudioHelper:tool_settings[TSPKeys.SINGLE_PASS_EXTRACTION_MODE] = (tool.single_pass_extraction_mode)+ tool_settings[TSPKeys.SUMMARIZE_AS_SOURCE] = tool.summarize_as_sourcetool_settings[TSPKeys.PREAMBLE] = tool.preambletool_settings[TSPKeys.POSTAMBLE] = tool.postambletool_settings[TSPKeys.GRAMMAR] = grammar_listdiff --git a/backend/utils/enums.py b/backend/utils/enums.pyindex 0dcc541a6..3d5405c15 100644--- a/backend/utils/enums.py+++ b/backend/utils/enums.py@@ -9,3 +9,4 @@ class CeleryTaskState(Enum):REVOKED = "REVOKED"STARTED = "STARTED"SUCCESS = "SUCCESS"+ COMPLETED = "COMPLETED"diff --git a/frontend/src/components/deployments/display-code/DisplayCode.jsx b/frontend/src/components/deployments/display-code/DisplayCode.jsxindex 44a5f9c02..a86852b3a 100644--- a/frontend/src/components/deployments/display-code/DisplayCode.jsx+++ b/frontend/src/components/deployments/display-code/DisplayCode.jsx@@ -8,17 +8,16 @@ import CodeSnippet from "./CodeSnippet.jsx";import "./DisplayCode.css";const DisplayCode = ({ isDialogOpen, setDialogOpen, url }) => {- const handleCloseDialog = () => {- setDialogOpen(false);- };const [copied, setCopied] = useState(false);const [language, setLanguage] = useState("python");const [code, setCode] = useState("");const [activeTabKey, setActiveTabKey] = useState("POST");+ const params =+ "?execution_id=REPLACE_WITH_EXECUTION_ID&include_metadata=False";- useEffect(() => {- generateCode();- }, [url, language, activeTabKey]);+ const handleCloseDialog = () => {+ setDialogOpen(false);+ };const generateCode = () => {if (language === "python") {@@ -51,7 +50,7 @@ const DisplayCode = ({ isDialogOpen, setDialogOpen, url }) => {files=[('files',('file',open(filepath,'rb'),'application/octet-stream'))]response = requests.request("POST", api_url, headers=headers, data=payload, files=files){{else}}- api_url = '{{url}}?execution_id=REPLACE_WITH_EXECUTION_ID'+ api_url = '{{url}}{{{params}}}'headers = {'Authorization': 'Bearer REPLACE_WITH_API_KEY'}@@ -62,7 +61,11 @@ const DisplayCode = ({ isDialogOpen, setDialogOpen, url }) => {code = trimIndent(code);const template = Handlebars.compile(code);- const pythonCode = template({ url, isPost: activeTabKey === "POST" });+ const pythonCode = template({+ url,+ params,+ isPost: activeTabKey === "POST",+ });setCode(pythonCode);};@@ -74,7 +77,7 @@ const DisplayCode = ({ isDialogOpen, setDialogOpen, url }) => {--form 'timeout=300' \\--form 'include_metadata=false'{{else}}- curl --location '{{url}}?execution_id=REPLACE_WITH_EXECUTION_ID' \\+ curl --location '{{url}}{{{params}}}' \\--header 'Authorization: Bearer REPLACE_WITH_API_KEY'{{/if}}`;@@ -82,6 +85,7 @@ const DisplayCode = ({ isDialogOpen, setDialogOpen, url }) => {const template = Handlebars.compile(code);const curlCode = template({url,+ params,isPost: activeTabKey === "POST",pathToFile: "/path/to/file",});@@ -100,7 +104,7 @@ const DisplayCode = ({ isDialogOpen, setDialogOpen, url }) => {fetch("{{url}}", requestOptions){{else}}var requestOptions = { method: 'GET', redirect: 'follow', headers: myHeaders};- fetch("{{url}}?execution_id=REPLACE_WITH_EXECUTION_ID", requestOptions)+ fetch("{{url}}{{{params}}}", requestOptions){{/if}}.then(response => response.text()).then(result => console.log(result))@@ -109,7 +113,7 @@ const DisplayCode = ({ isDialogOpen, setDialogOpen, url }) => {code = trimIndent(code);const template = Handlebars.compile(code);- const jsCode = template({ url, isPost: activeTabKey === "POST" });+ const jsCode = template({ url, params, isPost: activeTabKey === "POST" });setCode(jsCode);};@@ -155,6 +159,10 @@ const DisplayCode = ({ isDialogOpen, setDialogOpen, url }) => {},];+ useEffect(() => {+ generateCode();+ }, [url, language, activeTabKey]);+return (<Modaltitle="Code Snippets"diff --git a/prompt-service/src/unstract/prompt_service/constants.py b/prompt-service/src/unstract/prompt_service/constants.pyindex 4e41a3a94..3d3b46c77 100644--- a/prompt-service/src/unstract/prompt_service/constants.py+++ b/prompt-service/src/unstract/prompt_service/constants.py@@ -62,6 +62,7 @@ class PromptServiceContants:PLATFORM_POSTAMBLE = "platform_postamble"EXTRACT_EPILOGUE = "extract-epilogue"CLEAN_CONTEXT = "clean-context"+ SUMMARIZE_AS_SOURCE = "summarize_as_source"class LogLevel(Enum):diff --git a/prompt-service/src/unstract/prompt_service/helper.py b/prompt-service/src/unstract/prompt_service/helper.pyindex b6f7bfc66..f75578cef 100644--- a/prompt-service/src/unstract/prompt_service/helper.py+++ b/prompt-service/src/unstract/prompt_service/helper.py@@ -212,13 +212,16 @@ def construct_and_run_prompt(prompt: str,metadata: dict[str, Any],) -> str:+ platform_postamble = tool_settings.get(PSKeys.PLATFORM_POSTAMBLE, "")+ if tool_settings.get(PSKeys.SUMMARIZE_AS_SOURCE):+ platform_postamble = ""prompt = construct_prompt(preamble=tool_settings.get(PSKeys.PREAMBLE, ""),prompt=output[prompt],postamble=tool_settings.get(PSKeys.POSTAMBLE, ""),grammar_list=tool_settings.get(PSKeys.GRAMMAR, []),context=context,- platform_postamble=tool_settings.get(PSKeys.PLATFORM_POSTAMBLE, ""),+ platform_postamble=platform_postamble,)return run_completion(llm=llm,diff --git a/tools/structure/src/main.py b/tools/structure/src/main.pyindex 929d3fe73..c42d547a8 100644--- a/tools/structure/src/main.py+++ b/tools/structure/src/main.py@@ -63,6 +63,7 @@ class StructureTool(BaseTool):tool_settings[SettingsKeys.ENABLE_SINGLE_PASS_EXTRACTION] = (single_pass_extraction_mode)+ tool_settings[SettingsKeys.SUMMARIZE_AS_SOURCE] = summarize_as_sourceprompt_service_resp = None_, file_name = os.path.split(input_file)