Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2836 [FEAT] Return full text contents of input file in API response (#1904)

Praveen Kumarb89f412edf4d
diff --git a/backend/api_v2/api_deployment_views.py b/backend/api_v2/api_deployment_views.py
index 61fe0ae84..8345e1e74 100644
--- a/backend/api_v2/api_deployment_views.py
+++ b/backend/api_v2/api_deployment_views.py
@@ -81,6 +81,9 @@ class DeploymentExecution(views.APIView):
timeout = serializer.validated_data.get(ApiExecution.TIMEOUT_FORM_DATA)
include_metadata = serializer.validated_data.get(ApiExecution.INCLUDE_METADATA)
include_metrics = serializer.validated_data.get(ApiExecution.INCLUDE_METRICS)
+ include_extracted_text = serializer.validated_data.get(
+ ApiExecution.INCLUDE_EXTRACTED_TEXT
+ )
use_file_history = serializer.validated_data.get(ApiExecution.USE_FILE_HISTORY)
tag_names = serializer.validated_data.get(ApiExecution.TAGS)
llm_profile_id = serializer.validated_data.get(ApiExecution.LLM_PROFILE_ID)
@@ -117,6 +120,7 @@ class DeploymentExecution(views.APIView):
timeout=timeout,
include_metadata=include_metadata,
include_metrics=include_metrics,
+ include_extracted_text=include_extracted_text,
use_file_history=use_file_history,
tag_names=tag_names,
llm_profile_id=llm_profile_id,
@@ -171,6 +175,9 @@ class DeploymentExecution(views.APIView):
execution_id = serializer.validated_data.get(ApiExecution.EXECUTION_ID)
include_metadata = serializer.validated_data.get(ApiExecution.INCLUDE_METADATA)
include_metrics = serializer.validated_data.get(ApiExecution.INCLUDE_METRICS)
+ include_extracted_text = serializer.validated_data.get(
+ ApiExecution.INCLUDE_EXTRACTED_TEXT
+ )
# Fetch execution status
response: ExecutionResponse = DeploymentHelper.get_execution_status(execution_id)
@@ -218,6 +225,7 @@ class DeploymentExecution(views.APIView):
deployment_execution_dto=deployment_execution_dto,
include_metadata=include_metadata,
include_metrics=include_metrics,
+ include_extracted_text=include_extracted_text,
)
return Response(
data={
diff --git a/backend/api_v2/constants.py b/backend/api_v2/constants.py
index 88c85ac61..09d99e15f 100644
--- a/backend/api_v2/constants.py
+++ b/backend/api_v2/constants.py
@@ -5,6 +5,7 @@ class ApiExecution:
TIMEOUT_FORM_DATA: str = "timeout"
INCLUDE_METADATA: str = "include_metadata"
INCLUDE_METRICS: str = "include_metrics"
+ INCLUDE_EXTRACTED_TEXT: str = "include_extracted_text"
USE_FILE_HISTORY: str = "use_file_history" # Undocumented parameter
EXECUTION_ID: str = "execution_id"
TAGS: str = "tags"
diff --git a/backend/api_v2/deployment_helper.py b/backend/api_v2/deployment_helper.py
index 86a015295..28cdf939a 100644
--- a/backend/api_v2/deployment_helper.py
+++ b/backend/api_v2/deployment_helper.py
@@ -155,6 +155,7 @@ class DeploymentHelper(BaseAPIKeyValidator):
timeout: int,
include_metadata: bool = False,
include_metrics: bool = False,
+ include_extracted_text: bool = False,
use_file_history: bool = False,
tag_names: list[str] = [],
llm_profile_id: str | None = None,
@@ -275,7 +276,10 @@ class DeploymentHelper(BaseAPIKeyValidator):
)
if not enable_highlight:
result.remove_result_metadata_keys(["highlight_data"])
- result.remove_result_metadata_keys(["extracted_text"])
+ if not include_extracted_text:
+ result.remove_result_metadata_keys(["extracted_text"])
+ if include_extracted_text:
+ result.promote_extracted_text()
if include_metadata or include_metrics:
cls._enrich_result_with_usage_metadata(result)
if not include_metadata:
@@ -458,6 +462,7 @@ class DeploymentHelper(BaseAPIKeyValidator):
deployment_execution_dto: Any,
include_metadata: bool,
include_metrics: bool,
+ include_extracted_text: bool = False,
) -> None:
"""Enrich and clean up the response for a completed execution."""
api_deployment = deployment_execution_dto.api
@@ -476,7 +481,10 @@ class DeploymentHelper(BaseAPIKeyValidator):
)
if not enable_highlight:
response.remove_result_metadata_keys(["highlight_data"])
- response.remove_result_metadata_keys(["extracted_text"])
+ if not include_extracted_text:
+ response.remove_result_metadata_keys(["extracted_text"])
+ if include_extracted_text:
+ response.promote_extracted_text()
if include_metadata or include_metrics:
DeploymentHelper._enrich_result_with_usage_metadata(response)
if not include_metadata:
diff --git a/backend/api_v2/serializers.py b/backend/api_v2/serializers.py
index 7c9a5a769..910a7d19e 100644
--- a/backend/api_v2/serializers.py
+++ b/backend/api_v2/serializers.py
@@ -210,6 +210,9 @@ class ExecutionRequestSerializer(TagParamsSerializer):
If -1 it corresponds to async execution. Defaults to -1
include_metadata (bool): Flag to include metadata in API response
include_metrics (bool): Flag to include metrics in API response
+ include_extracted_text (bool): Flag to include the full extracted text
+ of the input file in the API response. The extracted text is returned
+ at the top level of each file result, independent of include_metadata.
use_file_history (bool): Flag to use FileHistory to save and retrieve
responses quickly. This is undocumented to the user and can be
helpful for demos.
@@ -232,6 +235,7 @@ class ExecutionRequestSerializer(TagParamsSerializer):
)
include_metadata = BooleanField(default=False)
include_metrics = BooleanField(default=False)
+ include_extracted_text = BooleanField(default=False)
use_file_history = BooleanField(default=False)
presigned_urls = ListField(child=URLField(), required=False)
@@ -408,6 +412,7 @@ class ExecutionQuerySerializer(Serializer):
execution_id = CharField(required=True)
include_metadata = BooleanField(default=False)
include_metrics = BooleanField(default=False)
+ include_extracted_text = BooleanField(default=False)
def validate_execution_id(self, value):
"""Trim spaces, validate UUID format, and check if execution_id exists."""
diff --git a/backend/workflow_manager/workflow_v2/dto.py b/backend/workflow_manager/workflow_v2/dto.py
index c567166a2..97148f1e9 100644
--- a/backend/workflow_manager/workflow_v2/dto.py
+++ b/backend/workflow_manager/workflow_v2/dto.py
@@ -129,6 +129,30 @@ class ExecutionResponse:
if isinstance(result, dict):
result.pop("metadata", None)
+ def promote_extracted_text(self) -> None:
+ """Copies extracted_text from metadata to the top level of each file
+ result item. This allows extracted_text to be returned independently
+ of include_metadata.
+
+ After promotion, the extracted_text appears as:
+ result[i]["extracted_text"] = "..."
+ """
+ if not isinstance(self.result, list):
+ return
+
+ for item in self.result:
+ if not isinstance(item, dict):
+ continue
+
+ result = item.get("result")
+ if not isinstance(result, dict):
+ continue
+
+ metadata = result.get("metadata", {})
+ extracted_text = metadata.get("extracted_text")
+ if extracted_text is not None:
+ item["extracted_text"] = extracted_text
+
def remove_result_metrics(self) -> None:
"""Removes the 'metrics' key from the 'result' dictionary within each
'result' dictionary in the 'result' list attribute of the instance.