LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2836 [FEAT] Return full text contents of input file in API response (#1904)
Praveen Kumarb89f412edf4d
diff --git a/backend/api_v2/api_deployment_views.py b/backend/api_v2/api_deployment_views.pyindex 61fe0ae84..8345e1e74 100644--- a/backend/api_v2/api_deployment_views.py+++ b/backend/api_v2/api_deployment_views.py@@ -81,6 +81,9 @@ class DeploymentExecution(views.APIView):timeout = serializer.validated_data.get(ApiExecution.TIMEOUT_FORM_DATA)include_metadata = serializer.validated_data.get(ApiExecution.INCLUDE_METADATA)include_metrics = serializer.validated_data.get(ApiExecution.INCLUDE_METRICS)+ include_extracted_text = serializer.validated_data.get(+ ApiExecution.INCLUDE_EXTRACTED_TEXT+ )use_file_history = serializer.validated_data.get(ApiExecution.USE_FILE_HISTORY)tag_names = serializer.validated_data.get(ApiExecution.TAGS)llm_profile_id = serializer.validated_data.get(ApiExecution.LLM_PROFILE_ID)@@ -117,6 +120,7 @@ class DeploymentExecution(views.APIView):timeout=timeout,include_metadata=include_metadata,include_metrics=include_metrics,+ include_extracted_text=include_extracted_text,use_file_history=use_file_history,tag_names=tag_names,llm_profile_id=llm_profile_id,@@ -171,6 +175,9 @@ class DeploymentExecution(views.APIView):execution_id = serializer.validated_data.get(ApiExecution.EXECUTION_ID)include_metadata = serializer.validated_data.get(ApiExecution.INCLUDE_METADATA)include_metrics = serializer.validated_data.get(ApiExecution.INCLUDE_METRICS)+ include_extracted_text = serializer.validated_data.get(+ ApiExecution.INCLUDE_EXTRACTED_TEXT+ )# Fetch execution statusresponse: ExecutionResponse = DeploymentHelper.get_execution_status(execution_id)@@ -218,6 +225,7 @@ class DeploymentExecution(views.APIView):deployment_execution_dto=deployment_execution_dto,include_metadata=include_metadata,include_metrics=include_metrics,+ include_extracted_text=include_extracted_text,)return Response(data={diff --git a/backend/api_v2/constants.py b/backend/api_v2/constants.pyindex 88c85ac61..09d99e15f 100644--- a/backend/api_v2/constants.py+++ b/backend/api_v2/constants.py@@ -5,6 +5,7 @@ class ApiExecution:TIMEOUT_FORM_DATA: str = "timeout"INCLUDE_METADATA: str = "include_metadata"INCLUDE_METRICS: str = "include_metrics"+ INCLUDE_EXTRACTED_TEXT: str = "include_extracted_text"USE_FILE_HISTORY: str = "use_file_history" # Undocumented parameterEXECUTION_ID: str = "execution_id"TAGS: str = "tags"diff --git a/backend/api_v2/deployment_helper.py b/backend/api_v2/deployment_helper.pyindex 86a015295..28cdf939a 100644--- a/backend/api_v2/deployment_helper.py+++ b/backend/api_v2/deployment_helper.py@@ -155,6 +155,7 @@ class DeploymentHelper(BaseAPIKeyValidator):timeout: int,include_metadata: bool = False,include_metrics: bool = False,+ include_extracted_text: bool = False,use_file_history: bool = False,tag_names: list[str] = [],llm_profile_id: str | None = None,@@ -275,7 +276,10 @@ class DeploymentHelper(BaseAPIKeyValidator):)if not enable_highlight:result.remove_result_metadata_keys(["highlight_data"])- result.remove_result_metadata_keys(["extracted_text"])+ if not include_extracted_text:+ result.remove_result_metadata_keys(["extracted_text"])+ if include_extracted_text:+ result.promote_extracted_text()if include_metadata or include_metrics:cls._enrich_result_with_usage_metadata(result)if not include_metadata:@@ -458,6 +462,7 @@ class DeploymentHelper(BaseAPIKeyValidator):deployment_execution_dto: Any,include_metadata: bool,include_metrics: bool,+ include_extracted_text: bool = False,) -> None:"""Enrich and clean up the response for a completed execution."""api_deployment = deployment_execution_dto.api@@ -476,7 +481,10 @@ class DeploymentHelper(BaseAPIKeyValidator):)if not enable_highlight:response.remove_result_metadata_keys(["highlight_data"])- response.remove_result_metadata_keys(["extracted_text"])+ if not include_extracted_text:+ response.remove_result_metadata_keys(["extracted_text"])+ if include_extracted_text:+ response.promote_extracted_text()if include_metadata or include_metrics:DeploymentHelper._enrich_result_with_usage_metadata(response)if not include_metadata:diff --git a/backend/api_v2/serializers.py b/backend/api_v2/serializers.pyindex 7c9a5a769..910a7d19e 100644--- a/backend/api_v2/serializers.py+++ b/backend/api_v2/serializers.py@@ -210,6 +210,9 @@ class ExecutionRequestSerializer(TagParamsSerializer):If -1 it corresponds to async execution. Defaults to -1include_metadata (bool): Flag to include metadata in API responseinclude_metrics (bool): Flag to include metrics in API response+ include_extracted_text (bool): Flag to include the full extracted text+ of the input file in the API response. The extracted text is returned+ at the top level of each file result, independent of include_metadata.use_file_history (bool): Flag to use FileHistory to save and retrieveresponses quickly. This is undocumented to the user and can behelpful for demos.@@ -232,6 +235,7 @@ class ExecutionRequestSerializer(TagParamsSerializer):)include_metadata = BooleanField(default=False)include_metrics = BooleanField(default=False)+ include_extracted_text = BooleanField(default=False)use_file_history = BooleanField(default=False)presigned_urls = ListField(child=URLField(), required=False)@@ -408,6 +412,7 @@ class ExecutionQuerySerializer(Serializer):execution_id = CharField(required=True)include_metadata = BooleanField(default=False)include_metrics = BooleanField(default=False)+ include_extracted_text = BooleanField(default=False)def validate_execution_id(self, value):"""Trim spaces, validate UUID format, and check if execution_id exists."""diff --git a/backend/workflow_manager/workflow_v2/dto.py b/backend/workflow_manager/workflow_v2/dto.pyindex c567166a2..97148f1e9 100644--- a/backend/workflow_manager/workflow_v2/dto.py+++ b/backend/workflow_manager/workflow_v2/dto.py@@ -129,6 +129,30 @@ class ExecutionResponse:if isinstance(result, dict):result.pop("metadata", None)+ def promote_extracted_text(self) -> None:+ """Copies extracted_text from metadata to the top level of each file+ result item. This allows extracted_text to be returned independently+ of include_metadata.++ After promotion, the extracted_text appears as:+ result[i]["extracted_text"] = "..."+ """+ if not isinstance(self.result, list):+ return++ for item in self.result:+ if not isinstance(item, dict):+ continue++ result = item.get("result")+ if not isinstance(result, dict):+ continue++ metadata = result.get("metadata", {})+ extracted_text = metadata.get("extracted_text")+ if extracted_text is not None:+ item["extracted_text"] = extracted_text+def remove_result_metrics(self) -> None:"""Removes the 'metrics' key from the 'result' dictionary within each'result' dictionary in the 'result' list attribute of the instance.