Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Enhancement for prompt response metadata (#441)

Deepak Kf184a4c731fa
diff --git a/backend/api/api_deployment_views.py b/backend/api/api_deployment_views.py
index 6c8db8249..7217117ca 100644
--- a/backend/api/api_deployment_views.py
+++ b/backend/api/api_deployment_views.py
@@ -50,7 +50,9 @@ class DeploymentExecution(views.APIView):
serializer = ExecutionRequestSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
timeout = serializer.get_timeout(serializer.validated_data)
-
+ include_metadata = (
+ request.data.get(ApiExecution.INCLUDE_METADATA, "false").lower() == "true"
+ )
if not file_objs or len(file_objs) == 0:
raise InvalidAPIRequest("File shouldn't be empty")
response = DeploymentHelper.execute_workflow(
@@ -58,6 +60,7 @@ class DeploymentExecution(views.APIView):
api=api,
file_objs=file_objs,
timeout=timeout,
+ include_metadata=include_metadata,
)
if "error" in response and response["error"]:
return Response(
diff --git a/backend/api/constants.py b/backend/api/constants.py
index c0f056091..ede21ad87 100644
--- a/backend/api/constants.py
+++ b/backend/api/constants.py
@@ -4,3 +4,4 @@ class ApiExecution:
DEFAULT_TIMEOUT_IN_SEC: int = 80
FILES_FORM_DATA: str = "files"
TIMEOUT_FORM_DATA: str = "timeout"
+ INCLUDE_METADATA: str = "include_metadata"
diff --git a/backend/api/deployment_helper.py b/backend/api/deployment_helper.py
index 3a484dbc2..59f7b3b3a 100644
--- a/backend/api/deployment_helper.py
+++ b/backend/api/deployment_helper.py
@@ -188,6 +188,7 @@ class DeploymentHelper:
api: APIDeployment,
file_objs: list[UploadedFile],
timeout: int,
+ include_metadata: bool = False,
) -> ReturnDict:
"""Execute workflow by api.
@@ -215,6 +216,7 @@ class DeploymentHelper:
hash_values_of_files=hash_values_of_files,
timeout=timeout,
execution_id=execution_id,
+ include_metadata=include_metadata,
)
result.status_api = DeploymentHelper.construct_status_endpoint(
api_endpoint=api.api_endpoint, execution_id=execution_id
diff --git a/backend/pdm.lock b/backend/pdm.lock
index 01d9ae0c7..c0e0bdc0d 100644
--- a/backend/pdm.lock
+++ b/backend/pdm.lock
@@ -5,7 +5,7 @@
groups = ["default", "deploy", "test"]
strategy = ["cross_platform", "inherit_metadata"]
lock_version = "4.4.1"
-content_hash = "sha256:62e6144be344e2e9499394e656f83fffa0c5a18ff001045d5fb5dc8c33d6d15a"
+content_hash = "sha256:03d0a714bfe69201a8cf35320b024636ff2eaad0fb7631eba4f9f0ab130f96e5"
[[package]]
name = "adlfs"
@@ -253,10 +253,6 @@ requires_python = ">=3.7"
summary = "Timeout context manager for asyncio programs"
groups = ["default"]
marker = "python_version < \"3.12.0\""
-files = [
- {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"},
- {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"},
-]
[[package]]
name = "asyncpg"
@@ -4736,7 +4732,7 @@ dependencies = [
[[package]]
name = "unstract-sdk"
-version = "0.35.0"
+version = "0.36.0"
requires_python = "<3.11.1,>=3.9"
summary = "A framework for writing Unstract Tools/Apps"
groups = ["default"]
@@ -4751,8 +4747,8 @@ dependencies = [
"unstract-adapters~=0.20.1",
]
files = [
- {file = "unstract_sdk-0.35.0-py3-none-any.whl", hash = "sha256:d91338a8f48c421f58ad6ae7337480eae39fd468b44a6a8f6bdc4784d1dac478"},
- {file = "unstract_sdk-0.35.0.tar.gz", hash = "sha256:863622d5c785aba2bb549e61959bd1d85cd21376310a340031a41fb78eab47ed"},
+ {file = "unstract_sdk-0.36.0-py3-none-any.whl", hash = "sha256:6a8038a5c9b4b81888262829c02cbd3db5b020a2eb8b67ff8124e44c11d26ee1"},
+ {file = "unstract_sdk-0.36.0.tar.gz", hash = "sha256:7955b1839e84c9212ea0b8f43eec7882c3279b8eb24ec64596bb1c1f7ca2840b"},
]
[[package]]
diff --git a/backend/pyproject.toml b/backend/pyproject.toml
index 865de53c5..a51bec501 100644
--- a/backend/pyproject.toml
+++ b/backend/pyproject.toml
@@ -31,7 +31,7 @@ dependencies = [
"python-socketio==5.9.0", # For log_events
"social-auth-app-django==5.3.0", # For OAuth
"social-auth-core==4.4.2", # For OAuth
- "unstract-sdk~=0.35.0",
+ "unstract-sdk~=0.36.0",
"unstract-adapters~=0.20.1",
# ! IMPORTANT!
# Indirect local dependencies usually need to be added in their own projects
diff --git a/backend/sample.env b/backend/sample.env
index e5f2ebe3e..abbc1757c 100644
--- a/backend/sample.env
+++ b/backend/sample.env
@@ -89,9 +89,9 @@ PROMPT_PORT=3003
PROMPT_STUDIO_FILE_PATH=/app/prompt-studio-data
# Structure Tool
-STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.28"
+STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.29"
STRUCTURE_TOOL_IMAGE_NAME="unstract/tool-structure"
-STRUCTURE_TOOL_IMAGE_TAG="0.0.28"
+STRUCTURE_TOOL_IMAGE_TAG="0.0.29"
# Feature Flags
EVALUATION_SERVER_IP=unstract-flipt
diff --git a/backend/workflow_manager/workflow/execution.py b/backend/workflow_manager/workflow/execution.py
index b249e53dc..6eb0311e9 100644
--- a/backend/workflow_manager/workflow/execution.py
+++ b/backend/workflow_manager/workflow/execution.py
@@ -37,6 +37,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
scheduled: bool = False,
mode: tuple[str, str] = WorkflowExecution.Mode.INSTANT,
workflow_execution: Optional[WorkflowExecution] = None,
+ include_metadata: bool = False,
) -> None:
tool_instances_as_dto = []
for tool_instance in tool_instances:
@@ -58,6 +59,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
tool_instances=tool_instances_as_dto,
platform_service_api_key=str(platform_key.key),
ignore_processed_entities=False,
+ include_metadata=include_metadata,
)
if not workflow_execution:
# Use pipline_id for pipelines / API deployment
diff --git a/backend/workflow_manager/workflow/views.py b/backend/workflow_manager/workflow/views.py
index 7438890b8..c02569355 100644
--- a/backend/workflow_manager/workflow/views.py
+++ b/backend/workflow_manager/workflow/views.py
@@ -173,6 +173,9 @@ class WorkflowViewSet(viewsets.ModelViewSet):
execution_id = serializer.get_execution_id(serializer.validated_data)
execution_action = serializer.get_execution_action(serializer.validated_data)
file_objs = request.FILES.getlist("files")
+ include_metadata = (
+ request.data.get("include_metadata", "false").lower() == "true"
+ )
hashes_of_files = {}
if file_objs and execution_id and workflow_id:
hashes_of_files = SourceConnector.add_input_file_to_api_storage(
@@ -191,6 +194,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):
execution_id=execution_id,
pipeline_guid=pipeline_guid,
hash_values_of_files=hashes_of_files,
+ include_metadata=include_metadata,
)
return Response(
make_execution_response(execution_response),
@@ -211,6 +215,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):
execution_id: Optional[str] = None,
pipeline_guid: Optional[str] = None,
hash_values_of_files: dict[str, str] = {},
+ include_metadata: bool = False,
) -> ExecutionResponse:
if execution_action is not None:
# Step execution
@@ -219,6 +224,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):
execution_action,
execution_id=execution_id,
hash_values_of_files=hash_values_of_files,
+ include_metadata=include_metadata,
)
elif pipeline_guid:
# pipeline execution
@@ -236,6 +242,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):
workflow=workflow,
execution_id=execution_id,
hash_values_of_files=hash_values_of_files,
+ include_metadata=include_metadata,
)
return execution_response
diff --git a/backend/workflow_manager/workflow/workflow_helper.py b/backend/workflow_manager/workflow/workflow_helper.py
index 8b94d8edd..b264845aa 100644
--- a/backend/workflow_manager/workflow/workflow_helper.py
+++ b/backend/workflow_manager/workflow/workflow_helper.py
@@ -89,6 +89,7 @@ class WorkflowHelper:
scheduled: bool,
execution_mode: tuple[str, str],
workflow_execution: Optional[WorkflowExecution],
+ include_metadata: bool = False,
) -> WorkflowExecutionServiceHelper:
workflow_execution_service = WorkflowExecutionServiceHelper(
organization_id=organization_id,
@@ -99,6 +100,7 @@ class WorkflowHelper:
scheduled=scheduled,
mode=execution_mode,
workflow_execution=workflow_execution,
+ include_metadata=include_metadata,
)
workflow_execution_service.build()
return workflow_execution_service
@@ -232,6 +234,7 @@ class WorkflowHelper:
single_step: bool = False,
workflow_execution: Optional[WorkflowExecution] = None,
execution_mode: Optional[tuple[str, str]] = None,
+ include_metadata: bool = False,
) -> ExecutionResponse:
tool_instances: list[ToolInstance] = (
ToolInstanceHelper.get_tool_instances_by_workflow(
@@ -250,6 +253,7 @@ class WorkflowHelper:
scheduled=scheduled,
execution_mode=execution_mode,
workflow_execution=workflow_execution,
+ include_metadata=include_metadata,
)
execution_id = execution_service.execution_id
source = SourceConnector(
@@ -349,18 +353,19 @@ class WorkflowHelper:
hash_values_of_files: dict[str, str],
timeout: int = -1,
pipeline_id: Optional[str] = None,
+ include_metadata: bool = False,
) -> ExecutionResponse:
"""Adding a workflow to the queue for execution.
Args:
workflow_id (str): workflowId
- execution_id (str): _description_
- timeout (int): celery timeout (timeout -1 : async execution)
- pipeline_id (Optional[str], optional): optional pipeline.
- Defaults to None.
+ execution_id (str): Execution ID
+ timeout (int): Celery timeout (timeout -1 : async execution)
+ pipeline_id (Optional[str], optional): Optional pipeline. Defaults to None.
+ include_metadata (bool): Whether to include metadata in the prompt output
Returns:
- ExecutionStatus: Existing status of execution
+ ExecutionResponse: Existing status of execution
"""
try:
org_schema = connection.tenant.schema_name
@@ -372,6 +377,7 @@ class WorkflowHelper:
execution_id=execution_id,
pipeline_id=pipeline_id,
log_events_id=log_events_id,
+ include_metadata=include_metadata,
)
if timeout > -1:
async_execution.wait(
@@ -426,6 +432,7 @@ class WorkflowHelper:
scheduled: bool = False,
execution_mode: Optional[tuple[str, str]] = None,
pipeline_id: Optional[str] = None,
+ include_metadata: bool = False,
**kwargs: dict[str, Any],
) -> Optional[list[Any]]:
"""Asynchronous Execution By celery.
@@ -433,22 +440,20 @@ class WorkflowHelper:
Args:
schema_name (str): schema name to get Data
workflow_id (str): Workflow Id
- execution_id (Optional[str], optional): Id of the execution.
- Defaults to None.
- scheduled (bool, optional): Represents if it is a scheduled
- execution. Defaults to False.
- execution_mode (Optional[WorkflowExecution.Mode]):
- WorkflowExecution Mode. Defaults to None.
- pipeline_id (Optional[str], optional): Id of pipeline.
- Defaults to None.
+ execution_id (str): Id of the execution
+ scheduled (bool, optional): Represents if it is a scheduled execution
+ Defaults to False
+ execution_mode (Optional[WorkflowExecution.Mode]): WorkflowExecution Mode
+ Defaults to None
+ pipeline_id (Optional[str], optional): Id of pipeline. Defaults to None
+ include_metadata (bool): Whether to include metadata in the prompt output
Kwargs:
- log_events_id (str): Session ID of the user, helps establish
- WS connection for streaming logs to the FE
+ log_events_id (str): Session ID of the user,
+ helps establish WS connection for streaming logs to the FE
Returns:
- dict[str, list[Any]]: Returns a dict with result from
- workflow execution
+ dict[str, list[Any]]: Returns a dict with result from workflow execution
"""
task_id = current_task.request.id
tenant: Organization = (
@@ -481,6 +486,7 @@ class WorkflowHelper:
workflow_execution=workflow_execution,
execution_mode=execution_mode,
hash_values_of_files=hash_values_of_files,
+ include_metadata=include_metadata,
).result
return result
@@ -490,6 +496,7 @@ class WorkflowHelper:
execution_id: Optional[str] = None,
pipeline_id: Optional[str] = None,
hash_values_of_files: dict[str, str] = {},
+ include_metadata: bool = False,
) -> ExecutionResponse:
if pipeline_id:
logger.info(f"Executing pipeline: {pipeline_id}")
@@ -517,6 +524,7 @@ class WorkflowHelper:
workflow=workflow,
workflow_execution=workflow_execution,
hash_values_of_files=hash_values_of_files,
+ include_metadata=include_metadata,
)
except WorkflowExecution.DoesNotExist:
return WorkflowHelper.create_and_make_execution_response(
@@ -544,6 +552,7 @@ class WorkflowHelper:
execution_action: str,
execution_id: Optional[str] = None,
hash_values_of_files: dict[str, str] = {},
+ include_metadata: bool = False,
) -> ExecutionResponse:
if execution_action is Workflow.ExecutionAction.START.value: # type: ignore
if execution_id is None:
@@ -557,6 +566,7 @@ class WorkflowHelper:
single_step=True,
workflow_execution=workflow_execution,
hash_values_of_files=hash_values_of_files,
+ include_metadata=include_metadata,
)
except WorkflowExecution.DoesNotExist:
return WorkflowHelper.create_and_make_execution_response(
diff --git a/frontend/src/components/deployments/display-code/DisplayCode.jsx b/frontend/src/components/deployments/display-code/DisplayCode.jsx
index 342d0e450..44a5f9c02 100644
--- a/frontend/src/components/deployments/display-code/DisplayCode.jsx
+++ b/frontend/src/components/deployments/display-code/DisplayCode.jsx
@@ -46,7 +46,7 @@ const DisplayCode = ({ isDialogOpen, setDialogOpen, url }) => {
headers = {
'Authorization': 'Bearer REPLACE_WITH_API_KEY'
}
- payload = {'timeout': '80'}
+ payload = {'timeout': 300, 'include_metadata': False}
filepath = '/path/to/file'
files=[('files',('file',open(filepath,'rb'),'application/octet-stream'))]
response = requests.request("POST", api_url, headers=headers, data=payload, files=files)
@@ -68,11 +68,13 @@ const DisplayCode = ({ isDialogOpen, setDialogOpen, url }) => {
const generateCurlCode = () => {
let code = `{{#if isPost}}
- curl --request POST --location '{{url}}'
- --header 'Authorization: Bearer REPLACE_WITH_API_KEY'
- --form 'files=@"{{pathToFile}}"' --form 'timeout="80"'
+ curl --request POST --location '{{url}}' \\
+ --header 'Authorization: Bearer REPLACE_WITH_API_KEY' \\
+ --form 'files=@"{{pathToFile}}"' \\
+ --form 'timeout=300' \\
+ --form 'include_metadata=false'
{{else}}
- curl --location '{{url}}?execution_id=REPLACE_WITH_EXECUTION_ID'
+ curl --location '{{url}}?execution_id=REPLACE_WITH_EXECUTION_ID' \\
--header 'Authorization: Bearer REPLACE_WITH_API_KEY'
{{/if}}
`;
@@ -92,7 +94,8 @@ const DisplayCode = ({ isDialogOpen, setDialogOpen, url }) => {
{{#if isPost}}
var formdata = new FormData();
formdata.append("files", fileInput.files[0], "file");
- formdata.append("timeout", "80");
+ formdata.append("timeout", "300");
+ formdata.append("include_metadata", "false");
var requestOptions = { method: 'POST', body: formdata, redirect: 'follow', headers: myHeaders };
fetch("{{url}}", requestOptions)
{{else}}
diff --git a/pdm.lock b/pdm.lock
index beb95339c..c5fe1af32 100644
--- a/pdm.lock
+++ b/pdm.lock
@@ -5,7 +5,7 @@
groups = ["default", "hook-check-django-migrations", "lint"]
strategy = ["cross_platform", "inherit_metadata"]
lock_version = "4.4.1"
-content_hash = "sha256:db33be7764a94d992a57e672b93cd2bb1c734aa11de14464a537637da659aaf8"
+content_hash = "sha256:24ccd1cb25354c5b0d9447759191790739f1b43d680a172428377d90191dfe51"
[[package]]
name = "adlfs"
@@ -4833,7 +4833,7 @@ dependencies = [
[[package]]
name = "unstract-sdk"
-version = "0.35.0"
+version = "0.36.0"
requires_python = "<3.11.1,>=3.9"
summary = "A framework for writing Unstract Tools/Apps"
groups = ["hook-check-django-migrations"]
@@ -4848,8 +4848,8 @@ dependencies = [
"unstract-adapters~=0.20.1",
]
files = [
- {file = "unstract_sdk-0.35.0-py3-none-any.whl", hash = "sha256:d91338a8f48c421f58ad6ae7337480eae39fd468b44a6a8f6bdc4784d1dac478"},
- {file = "unstract_sdk-0.35.0.tar.gz", hash = "sha256:863622d5c785aba2bb549e61959bd1d85cd21376310a340031a41fb78eab47ed"},
+ {file = "unstract_sdk-0.36.0-py3-none-any.whl", hash = "sha256:6a8038a5c9b4b81888262829c02cbd3db5b020a2eb8b67ff8124e44c11d26ee1"},
+ {file = "unstract_sdk-0.36.0.tar.gz", hash = "sha256:7955b1839e84c9212ea0b8f43eec7882c3279b8eb24ec64596bb1c1f7ca2840b"},
]
[[package]]
diff --git a/prompt-service/pdm.lock b/prompt-service/pdm.lock
index 4aa190c4f..0a1099e29 100644
--- a/prompt-service/pdm.lock
+++ b/prompt-service/pdm.lock
@@ -5,7 +5,7 @@
groups = ["default", "deploy"]
strategy = ["cross_platform", "inherit_metadata"]
lock_version = "4.4.1"
-content_hash = "sha256:ae728f45c0b04eacc0f8de9a60d6447f21612514c9751bafa85b781a3cc998f2"
+content_hash = "sha256:e44a922825595cf7f256050a330acd33562042e47b5a7d43c95d9fbc9f37b5cf"
[[package]]
name = "aiohttp"
@@ -693,13 +693,13 @@ files = [
[[package]]
name = "fsspec"
-version = "2024.6.1"
+version = "2024.6.0"
requires_python = ">=3.8"
summary = "File-system specification"
groups = ["default"]
files = [
- {file = "fsspec-2024.6.1-py3-none-any.whl", hash = "sha256:3cb443f8bcd2efb31295a5b9fdb02aee81d8452c80d28f97a6d0959e6cee101e"},
- {file = "fsspec-2024.6.1.tar.gz", hash = "sha256:fad7d7e209dd4c1208e3bbfda706620e0da5142bebbd9c384afb95b07e798e49"},
+ {file = "fsspec-2024.6.0-py3-none-any.whl", hash = "sha256:58d7122eb8a1a46f7f13453187bfea4972d66bf01618d37366521b1998034cee"},
+ {file = "fsspec-2024.6.0.tar.gz", hash = "sha256:f579960a56e6d8038a9efc8f9c77279ec12e6299aa86b0769a7e9c46b94527c2"},
]
[[package]]
@@ -3223,8 +3223,8 @@ requires_python = ">=3.7"
summary = "Database Abstraction Library"
groups = ["default"]
dependencies = [
+ "SQLAlchemy==2.0.31",
"greenlet!=0.4.17",
- "sqlalchemy==2.0.31",
]
files = [
{file = "SQLAlchemy-2.0.31-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f2a213c1b699d3f5768a7272de720387ae0122f1becf0901ed6eaa1abd1baf6c"},
@@ -3583,7 +3583,7 @@ dependencies = [
[[package]]
name = "unstract-sdk"
-version = "0.35.0"
+version = "0.36.0"
requires_python = "<3.11.1,>=3.9"
summary = "A framework for writing Unstract Tools/Apps"
groups = ["default"]
@@ -3598,8 +3598,8 @@ dependencies = [
"unstract-adapters~=0.20.1",
]
files = [
- {file = "unstract_sdk-0.35.0-py3-none-any.whl", hash = "sha256:d91338a8f48c421f58ad6ae7337480eae39fd468b44a6a8f6bdc4784d1dac478"},
- {file = "unstract_sdk-0.35.0.tar.gz", hash = "sha256:863622d5c785aba2bb549e61959bd1d85cd21376310a340031a41fb78eab47ed"},
+ {file = "unstract_sdk-0.36.0-py3-none-any.whl", hash = "sha256:6a8038a5c9b4b81888262829c02cbd3db5b020a2eb8b67ff8124e44c11d26ee1"},
+ {file = "unstract_sdk-0.36.0.tar.gz", hash = "sha256:7955b1839e84c9212ea0b8f43eec7882c3279b8eb24ec64596bb1c1f7ca2840b"},
]
[[package]]
diff --git a/prompt-service/pyproject.toml b/prompt-service/pyproject.toml
index 710c1e25c..565a7f6ef 100644
--- a/prompt-service/pyproject.toml
+++ b/prompt-service/pyproject.toml
@@ -15,7 +15,7 @@ dependencies = [
"flask~=3.0",
"llama-index==0.10.38",
"python-dotenv==1.0.0",
- "unstract-sdk~=0.35.0",
+ "unstract-sdk~=0.36.0",
"redis>=5.0.3",
"unstract-core @ file:///${PROJECT_ROOT}/../unstract/core",
"unstract-flags @ file:///${PROJECT_ROOT}/../unstract/flags",
diff --git a/prompt-service/src/unstract/prompt_service/constants.py b/prompt-service/src/unstract/prompt_service/constants.py
index c05b4de31..2b8dacd58 100644
--- a/prompt-service/src/unstract/prompt_service/constants.py
+++ b/prompt-service/src/unstract/prompt_service/constants.py
@@ -56,6 +56,7 @@ class PromptServiceContants:
METADATA = "metadata"
OUTPUT = "output"
CONTEXT = "context"
+ INCLUDE_METADATA = "include_metadata"
class LogLevel(Enum):
diff --git a/prompt-service/src/unstract/prompt_service/main.py b/prompt-service/src/unstract/prompt_service/main.py
index fb6cd04c4..73787685d 100644
--- a/prompt-service/src/unstract/prompt_service/main.py
+++ b/prompt-service/src/unstract/prompt_service/main.py
@@ -184,9 +184,11 @@ def prompt_processor() -> Any:
file_hash = payload.get(PSKeys.FILE_HASH)
doc_name = str(payload.get(PSKeys.FILE_NAME, ""))
log_events_id: str = payload.get(PSKeys.LOG_EVENTS_ID, "")
-
+ include_metadata = (
+ request.args.get(PSKeys.INCLUDE_METADATA, "false").lower() == "true"
+ )
structured_output: dict[str, Any] = {}
- metadata: dict[str, Any] = {
+ metadata: Optional[dict[str, Any]] = {
PSKeys.RUN_ID: run_id,
PSKeys.CONTEXT: {},
}
@@ -701,8 +703,11 @@ def prompt_processor() -> Any:
RunLevel.RUN,
"Execution complete",
)
- metadata = query_usage_metadata(db=be_db, token=platform_key, metadata=metadata)
- response = {PSKeys.METADATA: metadata, PSKeys.OUTPUT: structured_output}
+ if include_metadata:
+ metadata = query_usage_metadata(db=be_db, token=platform_key, metadata=metadata)
+ response = {PSKeys.METADATA: metadata, PSKeys.OUTPUT: structured_output}
+ else:
+ response = {PSKeys.OUTPUT: structured_output}
return response
diff --git a/pyproject.toml b/pyproject.toml
index 780e37985..264086cb0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -40,7 +40,7 @@ hook-check-django-migrations = [
"psycopg2-binary==2.9.9",
"python-dotenv==1.0.0",
"python-magic==0.4.27",
- "unstract-sdk~=0.35.0",
+ "unstract-sdk~=0.36.0",
"unstract-adapters~=0.20.1",
"-e unstract-connectors @ file:///${PROJECT_ROOT}/unstract/connectors",
"-e unstract-core @ file:///${PROJECT_ROOT}/unstract/core",
diff --git a/tools/structure/requirements.txt b/tools/structure/requirements.txt
index 24ee83f5a..86eca00a0 100644
--- a/tools/structure/requirements.txt
+++ b/tools/structure/requirements.txt
@@ -1,4 +1,4 @@
# Add your dependencies here
# Required for all unstract tools
-unstract-sdk~=0.35.0
+unstract-sdk~=0.36.0
diff --git a/tools/structure/src/constants.py b/tools/structure/src/constants.py
index 86bd21cd1..72780fe2b 100644
--- a/tools/structure/src/constants.py
+++ b/tools/structure/src/constants.py
@@ -67,3 +67,4 @@ class SettingsKeys:
FILE_HASH = "file_hash"
ENABLE_HIGHLIGHT = "enable_highlight"
NAME = "name"
+ INCLUDE_METADATA = "include_metadata"
diff --git a/tools/structure/src/main.py b/tools/structure/src/main.py
index 8453351ab..e807e8c92 100644
--- a/tools/structure/src/main.py
+++ b/tools/structure/src/main.py
@@ -27,7 +27,7 @@ class StructureTool(BaseTool):
prompt_registry_id: str = settings[SettingsKeys.PROMPT_REGISTRY_ID]
challenge_llm: str = settings[SettingsKeys.CHALLENGE_LLM_ADAPTER_ID]
enable_challenge: bool = settings[SettingsKeys.ENABLE_CHALLENGE]
-
+ include_metadata: bool = settings.get(SettingsKeys.INCLUDE_METADATA, False)
summarize_as_source: bool = settings[SettingsKeys.SUMMARIZE_AS_SOURCE]
single_pass_extraction_mode: bool = settings[
SettingsKeys.SINGLE_PASS_EXTRACTION_MODE
@@ -80,7 +80,7 @@ class StructureTool(BaseTool):
SettingsKeys.FILE_HASH: file_hash,
SettingsKeys.FILE_NAME: file_name,
}
-
+ params = {SettingsKeys.INCLUDE_METADATA: include_metadata}
# TODO: Need to split extraction and indexing
# to avoid unwanted indexing
self.stream_log("Indexing document")
@@ -114,7 +114,10 @@ class StructureTool(BaseTool):
)
payload[SettingsKeys.FILE_HASH] = summarize_file_hash
self.stream_log("Fetching response for single pass extraction")
- prompt_service_resp = responder.single_pass_extraction(payload=payload)
+ prompt_service_resp = responder.single_pass_extraction(
+ payload=payload,
+ params=params,
+ )
else:
try:
# To reindex even if file is already
@@ -154,7 +157,10 @@ class StructureTool(BaseTool):
except Exception as e:
self.stream_error_and_exit(f"Error fetching data and indexing: {e}")
self.stream_log("Fetching responses for prompts...")
- prompt_service_resp = responder.answer_prompt(payload=payload)
+ prompt_service_resp = responder.answer_prompt(
+ payload=payload,
+ params=params,
+ )
# TODO: Make use of dataclasses
if prompt_service_resp[SettingsKeys.STATUS] != SettingsKeys.OK:
diff --git a/unstract/tool-sandbox/src/unstract/tool_sandbox/constants.py b/unstract/tool-sandbox/src/unstract/tool_sandbox/constants.py
index a506b9707..eef11fffc 100644
--- a/unstract/tool-sandbox/src/unstract/tool_sandbox/constants.py
+++ b/unstract/tool-sandbox/src/unstract/tool_sandbox/constants.py
@@ -12,3 +12,7 @@ class ToolCommandKey:
SPEC = "spec"
VARIABLES = "variables"
ICON = "icon"
+
+
+class Common:
+ INCLUDE_METADATA = "include_metadata"
diff --git a/unstract/tool-sandbox/src/unstract/tool_sandbox/tool_sandbox.py b/unstract/tool-sandbox/src/unstract/tool_sandbox/tool_sandbox.py
index d878ce090..a603a590b 100644
--- a/unstract/tool-sandbox/src/unstract/tool_sandbox/tool_sandbox.py
+++ b/unstract/tool-sandbox/src/unstract/tool_sandbox/tool_sandbox.py
@@ -1,6 +1,6 @@
from typing import Any, Optional
-from unstract.tool_sandbox.constants import ToolCommandKey, UnstractWorker
+from unstract.tool_sandbox.constants import Common, ToolCommandKey, UnstractWorker
from unstract.tool_sandbox.helper import ToolSandboxHelper
@@ -16,6 +16,7 @@ class ToolSandbox:
tool_instance_id: Optional[str] = None,
environment_variables: dict[str, Any] = {},
messaging_channel: Optional[str] = None,
+ include_metadata: bool = False,
):
"""PLATFORM_SERVICE_API_KEY should be available in the environment."""
self.messaging_channel = str(messaging_channel)
@@ -30,9 +31,11 @@ class ToolSandbox:
self.tool_instance_id = tool_instance_id
self.image_name = image_name
self.image_tag = image_tag
+ self.include_metadata = include_metadata
self.settings: dict[str, Any] = {}
def set_tool_instance_settings(self, tool_settings: dict[str, Any]) -> None:
+ tool_settings[Common.INCLUDE_METADATA] = self.include_metadata
self.settings = tool_settings
def get_tool_uid(self) -> str:
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py
index c915078ab..b138c4089 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py
@@ -28,6 +28,7 @@ class ToolsUtils:
workflow: WorkflowDto,
platform_service_api_key: str,
ignore_processed_entities: bool = False,
+ include_metadata: bool = False,
) -> None:
self.redis = redis
self.tool_registry = ToolRegistry()
@@ -52,6 +53,7 @@ class ToolsUtils:
self.llmw_max_polls = ToolsUtils.get_env(
ToolRV.ADAPTER_LLMW_MAX_POLLS, raise_exception=False
)
+ self.include_metadata = include_metadata
def set_messaging_channel(self, messaging_channel: str) -> None:
self.messaging_channel = messaging_channel
@@ -135,6 +137,7 @@ class ToolsUtils:
image_tag=image_tag,
environment_variables=tool_envs,
messaging_channel=self.messaging_channel,
+ include_metadata=self.include_metadata,
)
tool_sandbox.set_tool_instance_settings(tool_instance.metadata)
tool_sandboxes.append(tool_sandbox)
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py
index 263307d9f..1f8bce13d 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py
@@ -44,6 +44,7 @@ class WorkflowExecutionService:
tool_instances: list[ToolInstance],
platform_service_api_key: str,
ignore_processed_entities: bool = False,
+ include_metadata: bool = False,
) -> None:
self.organization_id = organization_id
self.workflow_id = workflow_id
@@ -55,6 +56,7 @@ class WorkflowExecutionService:
workflow=workflow,
platform_service_api_key=platform_service_api_key,
ignore_processed_entities=False,
+ include_metadata=include_metadata,
)
self.tool_sandboxes: list[ToolSandbox] = []
self.ignore_processed_entities = ignore_processed_entities