LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Enhancement for prompt response metadata (#441)
Deepak Kf184a4c731fa
diff --git a/backend/api/api_deployment_views.py b/backend/api/api_deployment_views.pyindex 6c8db8249..7217117ca 100644--- a/backend/api/api_deployment_views.py+++ b/backend/api/api_deployment_views.py@@ -50,7 +50,9 @@ class DeploymentExecution(views.APIView):serializer = ExecutionRequestSerializer(data=request.data)serializer.is_valid(raise_exception=True)timeout = serializer.get_timeout(serializer.validated_data)-+ include_metadata = (+ request.data.get(ApiExecution.INCLUDE_METADATA, "false").lower() == "true"+ )if not file_objs or len(file_objs) == 0:raise InvalidAPIRequest("File shouldn't be empty")response = DeploymentHelper.execute_workflow(@@ -58,6 +60,7 @@ class DeploymentExecution(views.APIView):api=api,file_objs=file_objs,timeout=timeout,+ include_metadata=include_metadata,)if "error" in response and response["error"]:return Response(diff --git a/backend/api/constants.py b/backend/api/constants.pyindex c0f056091..ede21ad87 100644--- a/backend/api/constants.py+++ b/backend/api/constants.py@@ -4,3 +4,4 @@ class ApiExecution:DEFAULT_TIMEOUT_IN_SEC: int = 80FILES_FORM_DATA: str = "files"TIMEOUT_FORM_DATA: str = "timeout"+ INCLUDE_METADATA: str = "include_metadata"diff --git a/backend/api/deployment_helper.py b/backend/api/deployment_helper.pyindex 3a484dbc2..59f7b3b3a 100644--- a/backend/api/deployment_helper.py+++ b/backend/api/deployment_helper.py@@ -188,6 +188,7 @@ class DeploymentHelper:api: APIDeployment,file_objs: list[UploadedFile],timeout: int,+ include_metadata: bool = False,) -> ReturnDict:"""Execute workflow by api.@@ -215,6 +216,7 @@ class DeploymentHelper:hash_values_of_files=hash_values_of_files,timeout=timeout,execution_id=execution_id,+ include_metadata=include_metadata,)result.status_api = DeploymentHelper.construct_status_endpoint(api_endpoint=api.api_endpoint, execution_id=execution_iddiff --git a/backend/pdm.lock b/backend/pdm.lockindex 01d9ae0c7..c0e0bdc0d 100644--- a/backend/pdm.lock+++ b/backend/pdm.lock@@ -5,7 +5,7 @@groups = ["default", "deploy", "test"]strategy = ["cross_platform", "inherit_metadata"]lock_version = "4.4.1"-content_hash = "sha256:62e6144be344e2e9499394e656f83fffa0c5a18ff001045d5fb5dc8c33d6d15a"+content_hash = "sha256:03d0a714bfe69201a8cf35320b024636ff2eaad0fb7631eba4f9f0ab130f96e5"[[package]]name = "adlfs"@@ -253,10 +253,6 @@ requires_python = ">=3.7"summary = "Timeout context manager for asyncio programs"groups = ["default"]marker = "python_version < \"3.12.0\""-files = [- {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"},- {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"},-][[package]]name = "asyncpg"@@ -4736,7 +4732,7 @@ dependencies = [[[package]]name = "unstract-sdk"-version = "0.35.0"+version = "0.36.0"requires_python = "<3.11.1,>=3.9"summary = "A framework for writing Unstract Tools/Apps"groups = ["default"]@@ -4751,8 +4747,8 @@ dependencies = ["unstract-adapters~=0.20.1",]files = [- {file = "unstract_sdk-0.35.0-py3-none-any.whl", hash = "sha256:d91338a8f48c421f58ad6ae7337480eae39fd468b44a6a8f6bdc4784d1dac478"},- {file = "unstract_sdk-0.35.0.tar.gz", hash = "sha256:863622d5c785aba2bb549e61959bd1d85cd21376310a340031a41fb78eab47ed"},+ {file = "unstract_sdk-0.36.0-py3-none-any.whl", hash = "sha256:6a8038a5c9b4b81888262829c02cbd3db5b020a2eb8b67ff8124e44c11d26ee1"},+ {file = "unstract_sdk-0.36.0.tar.gz", hash = "sha256:7955b1839e84c9212ea0b8f43eec7882c3279b8eb24ec64596bb1c1f7ca2840b"},][[package]]diff --git a/backend/pyproject.toml b/backend/pyproject.tomlindex 865de53c5..a51bec501 100644--- a/backend/pyproject.toml+++ b/backend/pyproject.toml@@ -31,7 +31,7 @@ dependencies = ["python-socketio==5.9.0", # For log_events"social-auth-app-django==5.3.0", # For OAuth"social-auth-core==4.4.2", # For OAuth- "unstract-sdk~=0.35.0",+ "unstract-sdk~=0.36.0","unstract-adapters~=0.20.1",# ! IMPORTANT!# Indirect local dependencies usually need to be added in their own projectsdiff --git a/backend/sample.env b/backend/sample.envindex e5f2ebe3e..abbc1757c 100644--- a/backend/sample.env+++ b/backend/sample.env@@ -89,9 +89,9 @@ PROMPT_PORT=3003PROMPT_STUDIO_FILE_PATH=/app/prompt-studio-data# Structure Tool-STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.28"+STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.29"STRUCTURE_TOOL_IMAGE_NAME="unstract/tool-structure"-STRUCTURE_TOOL_IMAGE_TAG="0.0.28"+STRUCTURE_TOOL_IMAGE_TAG="0.0.29"# Feature FlagsEVALUATION_SERVER_IP=unstract-fliptdiff --git a/backend/workflow_manager/workflow/execution.py b/backend/workflow_manager/workflow/execution.pyindex b249e53dc..6eb0311e9 100644--- a/backend/workflow_manager/workflow/execution.py+++ b/backend/workflow_manager/workflow/execution.py@@ -37,6 +37,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):scheduled: bool = False,mode: tuple[str, str] = WorkflowExecution.Mode.INSTANT,workflow_execution: Optional[WorkflowExecution] = None,+ include_metadata: bool = False,) -> None:tool_instances_as_dto = []for tool_instance in tool_instances:@@ -58,6 +59,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):tool_instances=tool_instances_as_dto,platform_service_api_key=str(platform_key.key),ignore_processed_entities=False,+ include_metadata=include_metadata,)if not workflow_execution:# Use pipline_id for pipelines / API deploymentdiff --git a/backend/workflow_manager/workflow/views.py b/backend/workflow_manager/workflow/views.pyindex 7438890b8..c02569355 100644--- a/backend/workflow_manager/workflow/views.py+++ b/backend/workflow_manager/workflow/views.py@@ -173,6 +173,9 @@ class WorkflowViewSet(viewsets.ModelViewSet):execution_id = serializer.get_execution_id(serializer.validated_data)execution_action = serializer.get_execution_action(serializer.validated_data)file_objs = request.FILES.getlist("files")+ include_metadata = (+ request.data.get("include_metadata", "false").lower() == "true"+ )hashes_of_files = {}if file_objs and execution_id and workflow_id:hashes_of_files = SourceConnector.add_input_file_to_api_storage(@@ -191,6 +194,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):execution_id=execution_id,pipeline_guid=pipeline_guid,hash_values_of_files=hashes_of_files,+ include_metadata=include_metadata,)return Response(make_execution_response(execution_response),@@ -211,6 +215,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):execution_id: Optional[str] = None,pipeline_guid: Optional[str] = None,hash_values_of_files: dict[str, str] = {},+ include_metadata: bool = False,) -> ExecutionResponse:if execution_action is not None:# Step execution@@ -219,6 +224,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):execution_action,execution_id=execution_id,hash_values_of_files=hash_values_of_files,+ include_metadata=include_metadata,)elif pipeline_guid:# pipeline execution@@ -236,6 +242,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):workflow=workflow,execution_id=execution_id,hash_values_of_files=hash_values_of_files,+ include_metadata=include_metadata,)return execution_responsediff --git a/backend/workflow_manager/workflow/workflow_helper.py b/backend/workflow_manager/workflow/workflow_helper.pyindex 8b94d8edd..b264845aa 100644--- a/backend/workflow_manager/workflow/workflow_helper.py+++ b/backend/workflow_manager/workflow/workflow_helper.py@@ -89,6 +89,7 @@ class WorkflowHelper:scheduled: bool,execution_mode: tuple[str, str],workflow_execution: Optional[WorkflowExecution],+ include_metadata: bool = False,) -> WorkflowExecutionServiceHelper:workflow_execution_service = WorkflowExecutionServiceHelper(organization_id=organization_id,@@ -99,6 +100,7 @@ class WorkflowHelper:scheduled=scheduled,mode=execution_mode,workflow_execution=workflow_execution,+ include_metadata=include_metadata,)workflow_execution_service.build()return workflow_execution_service@@ -232,6 +234,7 @@ class WorkflowHelper:single_step: bool = False,workflow_execution: Optional[WorkflowExecution] = None,execution_mode: Optional[tuple[str, str]] = None,+ include_metadata: bool = False,) -> ExecutionResponse:tool_instances: list[ToolInstance] = (ToolInstanceHelper.get_tool_instances_by_workflow(@@ -250,6 +253,7 @@ class WorkflowHelper:scheduled=scheduled,execution_mode=execution_mode,workflow_execution=workflow_execution,+ include_metadata=include_metadata,)execution_id = execution_service.execution_idsource = SourceConnector(@@ -349,18 +353,19 @@ class WorkflowHelper:hash_values_of_files: dict[str, str],timeout: int = -1,pipeline_id: Optional[str] = None,+ include_metadata: bool = False,) -> ExecutionResponse:"""Adding a workflow to the queue for execution.Args:workflow_id (str): workflowId- execution_id (str): _description_- timeout (int): celery timeout (timeout -1 : async execution)- pipeline_id (Optional[str], optional): optional pipeline.- Defaults to None.+ execution_id (str): Execution ID+ timeout (int): Celery timeout (timeout -1 : async execution)+ pipeline_id (Optional[str], optional): Optional pipeline. Defaults to None.+ include_metadata (bool): Whether to include metadata in the prompt outputReturns:- ExecutionStatus: Existing status of execution+ ExecutionResponse: Existing status of execution"""try:org_schema = connection.tenant.schema_name@@ -372,6 +377,7 @@ class WorkflowHelper:execution_id=execution_id,pipeline_id=pipeline_id,log_events_id=log_events_id,+ include_metadata=include_metadata,)if timeout > -1:async_execution.wait(@@ -426,6 +432,7 @@ class WorkflowHelper:scheduled: bool = False,execution_mode: Optional[tuple[str, str]] = None,pipeline_id: Optional[str] = None,+ include_metadata: bool = False,**kwargs: dict[str, Any],) -> Optional[list[Any]]:"""Asynchronous Execution By celery.@@ -433,22 +440,20 @@ class WorkflowHelper:Args:schema_name (str): schema name to get Dataworkflow_id (str): Workflow Id- execution_id (Optional[str], optional): Id of the execution.- Defaults to None.- scheduled (bool, optional): Represents if it is a scheduled- execution. Defaults to False.- execution_mode (Optional[WorkflowExecution.Mode]):- WorkflowExecution Mode. Defaults to None.- pipeline_id (Optional[str], optional): Id of pipeline.- Defaults to None.+ execution_id (str): Id of the execution+ scheduled (bool, optional): Represents if it is a scheduled execution+ Defaults to False+ execution_mode (Optional[WorkflowExecution.Mode]): WorkflowExecution Mode+ Defaults to None+ pipeline_id (Optional[str], optional): Id of pipeline. Defaults to None+ include_metadata (bool): Whether to include metadata in the prompt outputKwargs:- log_events_id (str): Session ID of the user, helps establish- WS connection for streaming logs to the FE+ log_events_id (str): Session ID of the user,+ helps establish WS connection for streaming logs to the FEReturns:- dict[str, list[Any]]: Returns a dict with result from- workflow execution+ dict[str, list[Any]]: Returns a dict with result from workflow execution"""task_id = current_task.request.idtenant: Organization = (@@ -481,6 +486,7 @@ class WorkflowHelper:workflow_execution=workflow_execution,execution_mode=execution_mode,hash_values_of_files=hash_values_of_files,+ include_metadata=include_metadata,).resultreturn result@@ -490,6 +496,7 @@ class WorkflowHelper:execution_id: Optional[str] = None,pipeline_id: Optional[str] = None,hash_values_of_files: dict[str, str] = {},+ include_metadata: bool = False,) -> ExecutionResponse:if pipeline_id:logger.info(f"Executing pipeline: {pipeline_id}")@@ -517,6 +524,7 @@ class WorkflowHelper:workflow=workflow,workflow_execution=workflow_execution,hash_values_of_files=hash_values_of_files,+ include_metadata=include_metadata,)except WorkflowExecution.DoesNotExist:return WorkflowHelper.create_and_make_execution_response(@@ -544,6 +552,7 @@ class WorkflowHelper:execution_action: str,execution_id: Optional[str] = None,hash_values_of_files: dict[str, str] = {},+ include_metadata: bool = False,) -> ExecutionResponse:if execution_action is Workflow.ExecutionAction.START.value: # type: ignoreif execution_id is None:@@ -557,6 +566,7 @@ class WorkflowHelper:single_step=True,workflow_execution=workflow_execution,hash_values_of_files=hash_values_of_files,+ include_metadata=include_metadata,)except WorkflowExecution.DoesNotExist:return WorkflowHelper.create_and_make_execution_response(diff --git a/frontend/src/components/deployments/display-code/DisplayCode.jsx b/frontend/src/components/deployments/display-code/DisplayCode.jsxindex 342d0e450..44a5f9c02 100644--- a/frontend/src/components/deployments/display-code/DisplayCode.jsx+++ b/frontend/src/components/deployments/display-code/DisplayCode.jsx@@ -46,7 +46,7 @@ const DisplayCode = ({ isDialogOpen, setDialogOpen, url }) => {headers = {'Authorization': 'Bearer REPLACE_WITH_API_KEY'}- payload = {'timeout': '80'}+ payload = {'timeout': 300, 'include_metadata': False}filepath = '/path/to/file'files=[('files',('file',open(filepath,'rb'),'application/octet-stream'))]response = requests.request("POST", api_url, headers=headers, data=payload, files=files)@@ -68,11 +68,13 @@ const DisplayCode = ({ isDialogOpen, setDialogOpen, url }) => {const generateCurlCode = () => {let code = `{{#if isPost}}- curl --request POST --location '{{url}}'- --header 'Authorization: Bearer REPLACE_WITH_API_KEY'- --form 'files=@"{{pathToFile}}"' --form 'timeout="80"'+ curl --request POST --location '{{url}}' \\+ --header 'Authorization: Bearer REPLACE_WITH_API_KEY' \\+ --form 'files=@"{{pathToFile}}"' \\+ --form 'timeout=300' \\+ --form 'include_metadata=false'{{else}}- curl --location '{{url}}?execution_id=REPLACE_WITH_EXECUTION_ID'+ curl --location '{{url}}?execution_id=REPLACE_WITH_EXECUTION_ID' \\--header 'Authorization: Bearer REPLACE_WITH_API_KEY'{{/if}}`;@@ -92,7 +94,8 @@ const DisplayCode = ({ isDialogOpen, setDialogOpen, url }) => {{{#if isPost}}var formdata = new FormData();formdata.append("files", fileInput.files[0], "file");- formdata.append("timeout", "80");+ formdata.append("timeout", "300");+ formdata.append("include_metadata", "false");var requestOptions = { method: 'POST', body: formdata, redirect: 'follow', headers: myHeaders };fetch("{{url}}", requestOptions){{else}}diff --git a/pdm.lock b/pdm.lockindex beb95339c..c5fe1af32 100644--- a/pdm.lock+++ b/pdm.lock@@ -5,7 +5,7 @@groups = ["default", "hook-check-django-migrations", "lint"]strategy = ["cross_platform", "inherit_metadata"]lock_version = "4.4.1"-content_hash = "sha256:db33be7764a94d992a57e672b93cd2bb1c734aa11de14464a537637da659aaf8"+content_hash = "sha256:24ccd1cb25354c5b0d9447759191790739f1b43d680a172428377d90191dfe51"[[package]]name = "adlfs"@@ -4833,7 +4833,7 @@ dependencies = [[[package]]name = "unstract-sdk"-version = "0.35.0"+version = "0.36.0"requires_python = "<3.11.1,>=3.9"summary = "A framework for writing Unstract Tools/Apps"groups = ["hook-check-django-migrations"]@@ -4848,8 +4848,8 @@ dependencies = ["unstract-adapters~=0.20.1",]files = [- {file = "unstract_sdk-0.35.0-py3-none-any.whl", hash = "sha256:d91338a8f48c421f58ad6ae7337480eae39fd468b44a6a8f6bdc4784d1dac478"},- {file = "unstract_sdk-0.35.0.tar.gz", hash = "sha256:863622d5c785aba2bb549e61959bd1d85cd21376310a340031a41fb78eab47ed"},+ {file = "unstract_sdk-0.36.0-py3-none-any.whl", hash = "sha256:6a8038a5c9b4b81888262829c02cbd3db5b020a2eb8b67ff8124e44c11d26ee1"},+ {file = "unstract_sdk-0.36.0.tar.gz", hash = "sha256:7955b1839e84c9212ea0b8f43eec7882c3279b8eb24ec64596bb1c1f7ca2840b"},][[package]]diff --git a/prompt-service/pdm.lock b/prompt-service/pdm.lockindex 4aa190c4f..0a1099e29 100644--- a/prompt-service/pdm.lock+++ b/prompt-service/pdm.lock@@ -5,7 +5,7 @@groups = ["default", "deploy"]strategy = ["cross_platform", "inherit_metadata"]lock_version = "4.4.1"-content_hash = "sha256:ae728f45c0b04eacc0f8de9a60d6447f21612514c9751bafa85b781a3cc998f2"+content_hash = "sha256:e44a922825595cf7f256050a330acd33562042e47b5a7d43c95d9fbc9f37b5cf"[[package]]name = "aiohttp"@@ -693,13 +693,13 @@ files = [[[package]]name = "fsspec"-version = "2024.6.1"+version = "2024.6.0"requires_python = ">=3.8"summary = "File-system specification"groups = ["default"]files = [- {file = "fsspec-2024.6.1-py3-none-any.whl", hash = "sha256:3cb443f8bcd2efb31295a5b9fdb02aee81d8452c80d28f97a6d0959e6cee101e"},- {file = "fsspec-2024.6.1.tar.gz", hash = "sha256:fad7d7e209dd4c1208e3bbfda706620e0da5142bebbd9c384afb95b07e798e49"},+ {file = "fsspec-2024.6.0-py3-none-any.whl", hash = "sha256:58d7122eb8a1a46f7f13453187bfea4972d66bf01618d37366521b1998034cee"},+ {file = "fsspec-2024.6.0.tar.gz", hash = "sha256:f579960a56e6d8038a9efc8f9c77279ec12e6299aa86b0769a7e9c46b94527c2"},][[package]]@@ -3223,8 +3223,8 @@ requires_python = ">=3.7"summary = "Database Abstraction Library"groups = ["default"]dependencies = [+ "SQLAlchemy==2.0.31","greenlet!=0.4.17",- "sqlalchemy==2.0.31",]files = [{file = "SQLAlchemy-2.0.31-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f2a213c1b699d3f5768a7272de720387ae0122f1becf0901ed6eaa1abd1baf6c"},@@ -3583,7 +3583,7 @@ dependencies = [[[package]]name = "unstract-sdk"-version = "0.35.0"+version = "0.36.0"requires_python = "<3.11.1,>=3.9"summary = "A framework for writing Unstract Tools/Apps"groups = ["default"]@@ -3598,8 +3598,8 @@ dependencies = ["unstract-adapters~=0.20.1",]files = [- {file = "unstract_sdk-0.35.0-py3-none-any.whl", hash = "sha256:d91338a8f48c421f58ad6ae7337480eae39fd468b44a6a8f6bdc4784d1dac478"},- {file = "unstract_sdk-0.35.0.tar.gz", hash = "sha256:863622d5c785aba2bb549e61959bd1d85cd21376310a340031a41fb78eab47ed"},+ {file = "unstract_sdk-0.36.0-py3-none-any.whl", hash = "sha256:6a8038a5c9b4b81888262829c02cbd3db5b020a2eb8b67ff8124e44c11d26ee1"},+ {file = "unstract_sdk-0.36.0.tar.gz", hash = "sha256:7955b1839e84c9212ea0b8f43eec7882c3279b8eb24ec64596bb1c1f7ca2840b"},][[package]]diff --git a/prompt-service/pyproject.toml b/prompt-service/pyproject.tomlindex 710c1e25c..565a7f6ef 100644--- a/prompt-service/pyproject.toml+++ b/prompt-service/pyproject.toml@@ -15,7 +15,7 @@ dependencies = ["flask~=3.0","llama-index==0.10.38","python-dotenv==1.0.0",- "unstract-sdk~=0.35.0",+ "unstract-sdk~=0.36.0","redis>=5.0.3","unstract-core @ file:///${PROJECT_ROOT}/../unstract/core","unstract-flags @ file:///${PROJECT_ROOT}/../unstract/flags",diff --git a/prompt-service/src/unstract/prompt_service/constants.py b/prompt-service/src/unstract/prompt_service/constants.pyindex c05b4de31..2b8dacd58 100644--- a/prompt-service/src/unstract/prompt_service/constants.py+++ b/prompt-service/src/unstract/prompt_service/constants.py@@ -56,6 +56,7 @@ class PromptServiceContants:METADATA = "metadata"OUTPUT = "output"CONTEXT = "context"+ INCLUDE_METADATA = "include_metadata"class LogLevel(Enum):diff --git a/prompt-service/src/unstract/prompt_service/main.py b/prompt-service/src/unstract/prompt_service/main.pyindex fb6cd04c4..73787685d 100644--- a/prompt-service/src/unstract/prompt_service/main.py+++ b/prompt-service/src/unstract/prompt_service/main.py@@ -184,9 +184,11 @@ def prompt_processor() -> Any:file_hash = payload.get(PSKeys.FILE_HASH)doc_name = str(payload.get(PSKeys.FILE_NAME, ""))log_events_id: str = payload.get(PSKeys.LOG_EVENTS_ID, "")-+ include_metadata = (+ request.args.get(PSKeys.INCLUDE_METADATA, "false").lower() == "true"+ )structured_output: dict[str, Any] = {}- metadata: dict[str, Any] = {+ metadata: Optional[dict[str, Any]] = {PSKeys.RUN_ID: run_id,PSKeys.CONTEXT: {},}@@ -701,8 +703,11 @@ def prompt_processor() -> Any:RunLevel.RUN,"Execution complete",)- metadata = query_usage_metadata(db=be_db, token=platform_key, metadata=metadata)- response = {PSKeys.METADATA: metadata, PSKeys.OUTPUT: structured_output}+ if include_metadata:+ metadata = query_usage_metadata(db=be_db, token=platform_key, metadata=metadata)+ response = {PSKeys.METADATA: metadata, PSKeys.OUTPUT: structured_output}+ else:+ response = {PSKeys.OUTPUT: structured_output}return responsediff --git a/pyproject.toml b/pyproject.tomlindex 780e37985..264086cb0 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -40,7 +40,7 @@ hook-check-django-migrations = ["psycopg2-binary==2.9.9","python-dotenv==1.0.0","python-magic==0.4.27",- "unstract-sdk~=0.35.0",+ "unstract-sdk~=0.36.0","unstract-adapters~=0.20.1","-e unstract-connectors @ file:///${PROJECT_ROOT}/unstract/connectors","-e unstract-core @ file:///${PROJECT_ROOT}/unstract/core",diff --git a/tools/structure/requirements.txt b/tools/structure/requirements.txtindex 24ee83f5a..86eca00a0 100644--- a/tools/structure/requirements.txt+++ b/tools/structure/requirements.txt@@ -1,4 +1,4 @@# Add your dependencies here# Required for all unstract tools-unstract-sdk~=0.35.0+unstract-sdk~=0.36.0diff --git a/tools/structure/src/constants.py b/tools/structure/src/constants.pyindex 86bd21cd1..72780fe2b 100644--- a/tools/structure/src/constants.py+++ b/tools/structure/src/constants.py@@ -67,3 +67,4 @@ class SettingsKeys:FILE_HASH = "file_hash"ENABLE_HIGHLIGHT = "enable_highlight"NAME = "name"+ INCLUDE_METADATA = "include_metadata"diff --git a/tools/structure/src/main.py b/tools/structure/src/main.pyindex 8453351ab..e807e8c92 100644--- a/tools/structure/src/main.py+++ b/tools/structure/src/main.py@@ -27,7 +27,7 @@ class StructureTool(BaseTool):prompt_registry_id: str = settings[SettingsKeys.PROMPT_REGISTRY_ID]challenge_llm: str = settings[SettingsKeys.CHALLENGE_LLM_ADAPTER_ID]enable_challenge: bool = settings[SettingsKeys.ENABLE_CHALLENGE]-+ include_metadata: bool = settings.get(SettingsKeys.INCLUDE_METADATA, False)summarize_as_source: bool = settings[SettingsKeys.SUMMARIZE_AS_SOURCE]single_pass_extraction_mode: bool = settings[SettingsKeys.SINGLE_PASS_EXTRACTION_MODE@@ -80,7 +80,7 @@ class StructureTool(BaseTool):SettingsKeys.FILE_HASH: file_hash,SettingsKeys.FILE_NAME: file_name,}-+ params = {SettingsKeys.INCLUDE_METADATA: include_metadata}# TODO: Need to split extraction and indexing# to avoid unwanted indexingself.stream_log("Indexing document")@@ -114,7 +114,10 @@ class StructureTool(BaseTool):)payload[SettingsKeys.FILE_HASH] = summarize_file_hashself.stream_log("Fetching response for single pass extraction")- prompt_service_resp = responder.single_pass_extraction(payload=payload)+ prompt_service_resp = responder.single_pass_extraction(+ payload=payload,+ params=params,+ )else:try:# To reindex even if file is already@@ -154,7 +157,10 @@ class StructureTool(BaseTool):except Exception as e:self.stream_error_and_exit(f"Error fetching data and indexing: {e}")self.stream_log("Fetching responses for prompts...")- prompt_service_resp = responder.answer_prompt(payload=payload)+ prompt_service_resp = responder.answer_prompt(+ payload=payload,+ params=params,+ )# TODO: Make use of dataclassesif prompt_service_resp[SettingsKeys.STATUS] != SettingsKeys.OK:diff --git a/unstract/tool-sandbox/src/unstract/tool_sandbox/constants.py b/unstract/tool-sandbox/src/unstract/tool_sandbox/constants.pyindex a506b9707..eef11fffc 100644--- a/unstract/tool-sandbox/src/unstract/tool_sandbox/constants.py+++ b/unstract/tool-sandbox/src/unstract/tool_sandbox/constants.py@@ -12,3 +12,7 @@ class ToolCommandKey:SPEC = "spec"VARIABLES = "variables"ICON = "icon"+++class Common:+ INCLUDE_METADATA = "include_metadata"diff --git a/unstract/tool-sandbox/src/unstract/tool_sandbox/tool_sandbox.py b/unstract/tool-sandbox/src/unstract/tool_sandbox/tool_sandbox.pyindex d878ce090..a603a590b 100644--- a/unstract/tool-sandbox/src/unstract/tool_sandbox/tool_sandbox.py+++ b/unstract/tool-sandbox/src/unstract/tool_sandbox/tool_sandbox.py@@ -1,6 +1,6 @@from typing import Any, Optional-from unstract.tool_sandbox.constants import ToolCommandKey, UnstractWorker+from unstract.tool_sandbox.constants import Common, ToolCommandKey, UnstractWorkerfrom unstract.tool_sandbox.helper import ToolSandboxHelper@@ -16,6 +16,7 @@ class ToolSandbox:tool_instance_id: Optional[str] = None,environment_variables: dict[str, Any] = {},messaging_channel: Optional[str] = None,+ include_metadata: bool = False,):"""PLATFORM_SERVICE_API_KEY should be available in the environment."""self.messaging_channel = str(messaging_channel)@@ -30,9 +31,11 @@ class ToolSandbox:self.tool_instance_id = tool_instance_idself.image_name = image_nameself.image_tag = image_tag+ self.include_metadata = include_metadataself.settings: dict[str, Any] = {}def set_tool_instance_settings(self, tool_settings: dict[str, Any]) -> None:+ tool_settings[Common.INCLUDE_METADATA] = self.include_metadataself.settings = tool_settingsdef get_tool_uid(self) -> str:diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.pyindex c915078ab..b138c4089 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py@@ -28,6 +28,7 @@ class ToolsUtils:workflow: WorkflowDto,platform_service_api_key: str,ignore_processed_entities: bool = False,+ include_metadata: bool = False,) -> None:self.redis = redisself.tool_registry = ToolRegistry()@@ -52,6 +53,7 @@ class ToolsUtils:self.llmw_max_polls = ToolsUtils.get_env(ToolRV.ADAPTER_LLMW_MAX_POLLS, raise_exception=False)+ self.include_metadata = include_metadatadef set_messaging_channel(self, messaging_channel: str) -> None:self.messaging_channel = messaging_channel@@ -135,6 +137,7 @@ class ToolsUtils:image_tag=image_tag,environment_variables=tool_envs,messaging_channel=self.messaging_channel,+ include_metadata=self.include_metadata,)tool_sandbox.set_tool_instance_settings(tool_instance.metadata)tool_sandboxes.append(tool_sandbox)diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.pyindex 263307d9f..1f8bce13d 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py@@ -44,6 +44,7 @@ class WorkflowExecutionService:tool_instances: list[ToolInstance],platform_service_api_key: str,ignore_processed_entities: bool = False,+ include_metadata: bool = False,) -> None:self.organization_id = organization_idself.workflow_id = workflow_id@@ -55,6 +56,7 @@ class WorkflowExecutionService:workflow=workflow,platform_service_api_key=platform_service_api_key,ignore_processed_entities=False,+ include_metadata=include_metadata,)self.tool_sandboxes: list[ToolSandbox] = []self.ignore_processed_entities = ignore_processed_entities