LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2807 [MISC] Changed user_data to custom_data in variable replacement (#1548)
jagadeeswaran-zipstack29b89ffcd57f
diff --git a/backend/api_v2/api_deployment_views.py b/backend/api_v2/api_deployment_views.pyindex 256f2b711..ac876548e 100644--- a/backend/api_v2/api_deployment_views.py+++ b/backend/api_v2/api_deployment_views.py@@ -70,7 +70,7 @@ class DeploymentExecution(views.APIView):tag_names = serializer.validated_data.get(ApiExecution.TAGS)llm_profile_id = serializer.validated_data.get(ApiExecution.LLM_PROFILE_ID)hitl_queue_name = serializer.validated_data.get(ApiExecution.HITL_QUEUE_NAME)- user_data = serializer.validated_data.get(ApiExecution.USER_DATA)+ custom_data = serializer.validated_data.get(ApiExecution.CUSTOM_DATA)if presigned_urls:DeploymentHelper.load_presigned_files(presigned_urls, file_objs)@@ -86,7 +86,7 @@ class DeploymentExecution(views.APIView):tag_names=tag_names,llm_profile_id=llm_profile_id,hitl_queue_name=hitl_queue_name,- user_data=user_data,+ custom_data=custom_data,request_headers=dict(request.headers),)if "error" in response and response["error"]:diff --git a/backend/api_v2/constants.py b/backend/api_v2/constants.pyindex 9d06e7881..a8f5cc559 100644--- a/backend/api_v2/constants.py+++ b/backend/api_v2/constants.py@@ -11,4 +11,4 @@ class ApiExecution:LLM_PROFILE_ID: str = "llm_profile_id"HITL_QUEUE_NAME: str = "hitl_queue_name"PRESIGNED_URLS: str = "presigned_urls"- USER_DATA: str = "user_data"+ CUSTOM_DATA: str = "custom_data"diff --git a/backend/api_v2/deployment_helper.py b/backend/api_v2/deployment_helper.pyindex 082e77975..a0e059111 100644--- a/backend/api_v2/deployment_helper.py+++ b/backend/api_v2/deployment_helper.py@@ -155,7 +155,7 @@ class DeploymentHelper(BaseAPIKeyValidator):tag_names: list[str] = [],llm_profile_id: str | None = None,hitl_queue_name: str | None = None,- user_data: dict[str, Any] | None = None,+ custom_data: dict[str, Any] | None = None,request_headers=None,) -> ReturnDict:"""Execute workflow by api.@@ -169,7 +169,7 @@ class DeploymentHelper(BaseAPIKeyValidator):tag_names (list(str)): list of tag namesllm_profile_id (str, optional): LLM profile ID for overriding tool settingshitl_queue_name (str, optional): Custom queue name for manual review- user_data (dict[str, Any], optional): JSON data for user_data variable replacement in prompts+ custom_data (dict[str, Any], optional): JSON data for custom_data variable replacement in promptsReturns:ReturnDict: execution status/ result@@ -236,7 +236,7 @@ class DeploymentHelper(BaseAPIKeyValidator):use_file_history=use_file_history,llm_profile_id=llm_profile_id,hitl_queue_name=hitl_queue_name,- user_data=user_data,+ custom_data=custom_data,)result.status_api = DeploymentHelper.construct_status_endpoint(api_endpoint=api.api_endpoint, execution_id=execution_iddiff --git a/backend/api_v2/serializers.py b/backend/api_v2/serializers.pyindex caf21838d..e7d1acabb 100644--- a/backend/api_v2/serializers.py+++ b/backend/api_v2/serializers.py@@ -195,23 +195,23 @@ class ExecutionRequestSerializer(TagParamsSerializer):"""Execution request serializer.Attributes:- timeout (int): Timeout for the API deployment, maximum value can be 300s.- If -1 it corresponds to async execution. Defaults to -1- include_metadata (bool): Flag to include metadata in API response- include_metrics (bool): Flag to include metrics in API response- use_file_history (bool): Flag to use FileHistory to save and retrieve- responses quickly. This is undocumented to the user and can be- helpful for demos.- tags (str): Comma-separated List of tags to associate with the execution.- e.g:'tag1,tag2-name,tag3_name'- llm_profile_id (str): UUID of the LLM profile to override the default profile.- If not provided, the default profile will be used.- hitl_queue_name (str, optional): Document class name for manual review queue.- If not provided, uses API name as document class.- presigned_urls (list): List of presigned URLs to fetch files from.- URLs are validated for HTTPS and S3 endpoint requirements.- user_data (dict, optional): User-provided data for variable replacement in prompts.- Can be accessed in prompts using {{user_data.key}} syntax for dot notation traversal.+ timeout (int): Timeout for the API deployment, maximum value can be 300s.+ If -1 it corresponds to async execution. Defaults to -1+ include_metadata (bool): Flag to include metadata in API response+ include_metrics (bool): Flag to include metrics in API response+ use_file_history (bool): Flag to use FileHistory to save and retrieve+ responses quickly. This is undocumented to the user and can be+ helpful for demos.+ tags (str): Comma-separated List of tags to associate with the execution.+ e.g:'tag1,tag2-name,tag3_name'+ llm_profile_id (str): UUID of the LLM profile to override the default profile.+ If not provided, the default profile will be used.+ hitl_queue_name (str, optional): Document class name for manual review queue.+ If not provided, uses API name as document class.+ presigned_urls (list): List of presigned URLs to fetch files from.+ URLs are validated for HTTPS and S3 endpoint requirements.+ custom_data (dict, optional): User-provided data for variable replacement in prompts.+ Can be accessed in prompts using {{custom_data.key}} syntax for dot notation traversal."""MAX_FILES_ALLOWED = 32@@ -226,7 +226,7 @@ class ExecutionRequestSerializer(TagParamsSerializer):presigned_urls = ListField(child=URLField(), required=False)llm_profile_id = CharField(required=False, allow_null=True, allow_blank=True)hitl_queue_name = CharField(required=False, allow_null=True, allow_blank=True)- user_data = JSONField(required=False, allow_null=True)+ custom_data = JSONField(required=False, allow_null=True)def validate_hitl_queue_name(self, value: str | None) -> str | None:"""Validate queue name format using enterprise validation if available."""@@ -247,13 +247,13 @@ class ExecutionRequestSerializer(TagParamsSerializer):)return value- def validate_user_data(self, value):- """Validate user_data is a valid JSON object."""+ def validate_custom_data(self, value):+ """Validate custom_data is a valid JSON object."""if value is None:return valueif not isinstance(value, dict):- raise ValidationError("user_data must be a JSON object")+ raise ValidationError("custom_data must be a JSON object")return valuediff --git a/backend/pyproject.toml b/backend/pyproject.tomlindex 2cf56042e..363a6ee75 100644--- a/backend/pyproject.toml+++ b/backend/pyproject.toml@@ -37,7 +37,7 @@ dependencies = ["social-auth-core==4.4.2", # For OAuth# TODO: Temporarily removing the extra dependencies of aws and gcs from unstract-sdk# to resolve lock file. Will have to be re-looked into- "unstract-sdk[azure]~=0.77.2",+ "unstract-sdk[azure]~=0.77.3","gcsfs==2024.10.0","s3fs==2024.10.0","azure-identity==1.16.0",diff --git a/backend/sample.env b/backend/sample.envindex b0331b23a..b1347ef15 100644--- a/backend/sample.env+++ b/backend/sample.env@@ -78,9 +78,9 @@ PROMPT_STUDIO_FILE_PATH=/app/prompt-studio-data# Structure Tool Image (Runs prompt studio exported tools)# https://hub.docker.com/r/unstract/tool-structure-STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.87"+STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.88"STRUCTURE_TOOL_IMAGE_NAME="unstract/tool-structure"-STRUCTURE_TOOL_IMAGE_TAG="0.0.87"+STRUCTURE_TOOL_IMAGE_TAG="0.0.88"# Feature FlagsEVALUATION_SERVER_IP=unstract-fliptdiff --git a/backend/uv.lock b/backend/uv.lockindex 4e5caf12a..549cf7e07 100644--- a/backend/uv.lock+++ b/backend/uv.lock@@ -3913,7 +3913,7 @@ requires-dist = [{ name = "unstract-core", editable = "../unstract/core" },{ name = "unstract-filesystem", editable = "../unstract/filesystem" },{ name = "unstract-flags", editable = "../unstract/flags" },- { name = "unstract-sdk", extras = ["azure"], specifier = "~=0.77.2" },+ { name = "unstract-sdk", extras = ["azure"], specifier = "~=0.77.3" },{ name = "unstract-tool-registry", editable = "../unstract/tool-registry" },{ name = "unstract-tool-sandbox", editable = "../unstract/tool-sandbox" },{ name = "unstract-workflow-execution", editable = "../unstract/workflow-execution" },@@ -4038,7 +4038,7 @@ requires-dist = [[[package]]name = "unstract-sdk"-version = "0.77.2"+version = "0.77.3"source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "filetype" },@@ -4077,9 +4077,9 @@ dependencies = [{ name = "tiktoken" },{ name = "transformers" },]-sdist = { url = "https://files.pythonhosted.org/packages/c6/56/b2d95e1029623e188c34702b7729d12200437aa8d7deb9d5a8edd88f2994/unstract_sdk-0.77.2.tar.gz", hash = "sha256:30a0fcbbeac7fcaadc96fc0754cf35bebc0dc379eddef874d28aa36a54f374ec", size = 2375849 }+sdist = { url = "https://files.pythonhosted.org/packages/c5/e8/f7e1045fee076c75c42bb27b4fa5a077836cfa8fd3fd580f6c6193fa19dc/unstract_sdk-0.77.3.tar.gz", hash = "sha256:378c19129a91e861b7235e92411ef72792c6c851320cb7897b380d08d3489c9d", size = 2375843 }wheels = [- { url = "https://files.pythonhosted.org/packages/8e/f9/f7b0e11f7e79162c7b2dfdbcbd1b1ad74b60d26e8d922b743db2b0d26214/unstract_sdk-0.77.2-py3-none-any.whl", hash = "sha256:8f57c39e814785de80b9fc74c972eb648b0a0b008edcac3ad0f6167e555944f8", size = 266633 },+ { url = "https://files.pythonhosted.org/packages/d5/84/d596295fce3a713b1953a4fd7be3be1d788920e5633e28fbea7c7331b68a/unstract_sdk-0.77.3-py3-none-any.whl", hash = "sha256:74e83cbb68eef98fbaecfc83e0cfd2228e5958506e1b51a65fae0d891027aaee", size = 266640 },][package.optional-dependencies]diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.pyindex 4edca72bf..a0e2b6365 100644--- a/backend/workflow_manager/endpoint_v2/source.py+++ b/backend/workflow_manager/endpoint_v2/source.py@@ -921,7 +921,7 @@ class SourceConnector(BaseConnector):tags: list[str],file_hash: FileHash,llm_profile_id: str | None = None,- user_data: dict[str, Any] | None = None,+ custom_data: dict[str, Any] | None = None,) -> str:"""Add input file to execution directory.@@ -963,7 +963,7 @@ class SourceConnector(BaseConnector):source_hash=file_content_hash,tags=tags,llm_profile_id=llm_profile_id,- user_data=user_data,+ custom_data=custom_data,)return file_content_hashdiff --git a/backend/workflow_manager/workflow_v2/dto.py b/backend/workflow_manager/workflow_v2/dto.pyindex 02c13f3e1..b2398e883 100644--- a/backend/workflow_manager/workflow_v2/dto.py+++ b/backend/workflow_manager/workflow_v2/dto.py@@ -156,7 +156,7 @@ class FileData:use_file_history: boolq_file_no_list: list[int]llm_profile_id: str | None = None- user_data: dict[str, Any] | None = None+ custom_data: dict[str, Any] | None = None@classmethoddef from_dict(cls, data: dict[str, Any]) -> FileData:diff --git a/backend/workflow_manager/workflow_v2/file_execution_tasks.py b/backend/workflow_manager/workflow_v2/file_execution_tasks.pyindex 88e88358d..c8b8ca0a0 100644--- a/backend/workflow_manager/workflow_v2/file_execution_tasks.py+++ b/backend/workflow_manager/workflow_v2/file_execution_tasks.py@@ -751,7 +751,7 @@ class FileExecutionTasks:tags=workflow_execution.tag_names,file_hash=file_hash,llm_profile_id=file_data.llm_profile_id,- user_data=file_data.user_data,+ custom_data=file_data.custom_data,)file_hash.file_hash = content_hashworkflow_file_exec.update(file_hash=content_hash)@@ -920,7 +920,8 @@ class FileExecutionTasks:)# Log execution costsexecution_service.log_total_cost_per_file(- run_id=str(workflow_file_execution.id), file_name=file_hash.file_name+ run_id=str(workflow_file_execution.id),+ file_name=file_hash.file_name,)tool_execution_result = ToolExecutionResult(error=None, result=None)return tool_execution_result@@ -995,7 +996,9 @@ class FileExecutionTasks:file_path = file_hash.file_path if not destination.is_api else None# Collect metadata from file history if availablefile_history = FileHistoryHelper.get_file_history(- workflow=workflow, cache_key=file_hash.file_hash, file_path=file_path+ workflow=workflow,+ cache_key=file_hash.file_hash,+ file_path=file_path,)else:file_history = Nonediff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.pyindex 303113612..f9433c639 100644--- a/backend/workflow_manager/workflow_v2/workflow_helper.py+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py@@ -64,7 +64,7 @@ logger = logging.getLogger(__name__)EXECUTION_EXCLUDED_PARAMS = {"llm_profile_id","hitl_queue_name",- "user_data",+ "custom_data",}@@ -151,7 +151,7 @@ class WorkflowHelper:execution_mode: tuple[str, str],use_file_history: bool,llm_profile_id: str | None,- user_data: dict[str, Any] | None = None,+ custom_data: dict[str, Any] | None = None,) -> str | None:total_files = len(input_files)workflow_log.publish_initial_workflow_logs(total_files=total_files)@@ -203,7 +203,7 @@ class WorkflowHelper:use_file_history=use_file_history,q_file_no_list=list(q_file_no_list) if q_file_no_list else [],llm_profile_id=llm_profile_id,- user_data=user_data,+ custom_data=custom_data,)batch_data = FileBatchData(files=batch, file_data=file_data)@@ -270,7 +270,7 @@ class WorkflowHelper:use_file_history: bool = True,llm_profile_id: str | None = None,hitl_queue_name: str | None = None,- user_data: dict[str, Any] | None = None,+ custom_data: dict[str, Any] | None = None,) -> ExecutionResponse:tool_instances: list[ToolInstance] = (ToolInstanceHelper.get_tool_instances_by_workflow(@@ -322,7 +322,7 @@ class WorkflowHelper:use_file_history=use_file_history,execution_mode=execution_mode,llm_profile_id=llm_profile_id,- user_data=user_data,+ custom_data=custom_data,)api_results = []return ExecutionResponse(@@ -441,7 +441,7 @@ class WorkflowHelper:use_file_history: bool = True,llm_profile_id: str | None = None,hitl_queue_name: str | None = None,- user_data: dict[str, Any] | None = None,+ custom_data: dict[str, Any] | None = None,) -> ExecutionResponse:"""Adding a workflow to the queue for execution.@@ -481,7 +481,7 @@ class WorkflowHelper:"use_file_history": use_file_history,"llm_profile_id": llm_profile_id,"hitl_queue_name": hitl_queue_name,- "user_data": user_data,+ "custom_data": custom_data,},queue=queue,)@@ -699,7 +699,7 @@ class WorkflowHelper:use_file_history=use_file_history,llm_profile_id=kwargs.get("llm_profile_id"),hitl_queue_name=kwargs.get("hitl_queue_name"),- user_data=kwargs.get("user_data"),+ custom_data=kwargs.get("custom_data"),)except Exception as error:error_message = traceback.format_exc()diff --git a/platform-service/pyproject.toml b/platform-service/pyproject.tomlindex f91d91cd8..cd9519eb6 100644--- a/platform-service/pyproject.toml+++ b/platform-service/pyproject.toml@@ -15,7 +15,7 @@ dependencies = ["redis~=5.2.1","cryptography>=41.0.7","requests>=2.31.0",- "unstract-sdk[gcs, azure, aws]~=0.77.2", # Add version+ "unstract-sdk[gcs, azure, aws]~=0.77.3", # Add version"unstract-flags","unstract-core[flask]",]diff --git a/platform-service/uv.lock b/platform-service/uv.lockindex b283eac42..2026bd334 100644--- a/platform-service/uv.lock+++ b/platform-service/uv.lock@@ -3075,7 +3075,7 @@ requires-dist = [{ name = "requests", specifier = ">=2.31.0" },{ name = "unstract-core", extras = ["flask"], editable = "../unstract/core" },{ name = "unstract-flags", editable = "../unstract/flags" },- { name = "unstract-sdk", extras = ["gcs", "azure", "aws"], specifier = "~=0.77.2" },+ { name = "unstract-sdk", extras = ["gcs", "azure", "aws"], specifier = "~=0.77.3" },][package.metadata.requires-dev]@@ -3092,7 +3092,7 @@ test = [{ name = "pytest", specifier = ">=8.0.1" }][[package]]name = "unstract-sdk"-version = "0.77.2"+version = "0.77.3"source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "filetype" },@@ -3131,9 +3131,9 @@ dependencies = [{ name = "tiktoken" },{ name = "transformers" },]-sdist = { url = "https://files.pythonhosted.org/packages/c6/56/b2d95e1029623e188c34702b7729d12200437aa8d7deb9d5a8edd88f2994/unstract_sdk-0.77.2.tar.gz", hash = "sha256:30a0fcbbeac7fcaadc96fc0754cf35bebc0dc379eddef874d28aa36a54f374ec", size = 2375849 }+sdist = { url = "https://files.pythonhosted.org/packages/c5/e8/f7e1045fee076c75c42bb27b4fa5a077836cfa8fd3fd580f6c6193fa19dc/unstract_sdk-0.77.3.tar.gz", hash = "sha256:378c19129a91e861b7235e92411ef72792c6c851320cb7897b380d08d3489c9d", size = 2375843 }wheels = [- { url = "https://files.pythonhosted.org/packages/8e/f9/f7b0e11f7e79162c7b2dfdbcbd1b1ad74b60d26e8d922b743db2b0d26214/unstract_sdk-0.77.2-py3-none-any.whl", hash = "sha256:8f57c39e814785de80b9fc74c972eb648b0a0b008edcac3ad0f6167e555944f8", size = 266633 },+ { url = "https://files.pythonhosted.org/packages/d5/84/d596295fce3a713b1953a4fd7be3be1d788920e5633e28fbea7c7331b68a/unstract_sdk-0.77.3-py3-none-any.whl", hash = "sha256:74e83cbb68eef98fbaecfc83e0cfd2228e5958506e1b51a65fae0d891027aaee", size = 266640 },][package.optional-dependencies]diff --git a/prompt-service/pyproject.toml b/prompt-service/pyproject.tomlindex 2e0524aeb..ff0e52bf4 100644--- a/prompt-service/pyproject.toml+++ b/prompt-service/pyproject.toml@@ -17,7 +17,7 @@ dependencies = ["requests>=2.28,<3.0",# TODO: Temporarily removing the extra dependencies of aws and gcs from unstract-sdk# to resolve lock file. Will have to be re-looked into- "unstract-sdk[azure]~=0.77.2",+ "unstract-sdk[azure]~=0.77.3","gcsfs==2024.10.0","s3fs==2024.10.0","redis>=5.0.3,<5.3",diff --git a/prompt-service/src/unstract/prompt_service/constants.py b/prompt-service/src/unstract/prompt_service/constants.pyindex f12b9eee4..09b6a768a 100644--- a/prompt-service/src/unstract/prompt_service/constants.py+++ b/prompt-service/src/unstract/prompt_service/constants.py@@ -67,7 +67,7 @@ class PromptServiceConstants:SUMMARIZE_AS_SOURCE = "summarize_as_source"VARIABLE_MAP = "variable_map"RECORD = "record"- USER_DATA = "user_data"+ CUSTOM_DATA = "custom_data"TEXT = "text"ENABLE_HIGHLIGHT = "enable_highlight"FILE_PATH = "file_path"@@ -148,7 +148,7 @@ class VariableType(str, Enum):STATIC = "STATIC"DYNAMIC = "DYNAMIC"- USER_DATA = "USER_DATA"+ CUSTOM_DATA = "CUSTOM_DATA"class RetrievalStrategy(str, Enum):@@ -172,7 +172,7 @@ class VariableConstants:r"(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'\".,<>?«»""'']))") # noqa: E501- USER_DATA_VARIABLE_REGEX = r"user_data\.([a-zA-Z0-9_\.]+)"+ CUSTOM_DATA_VARIABLE_REGEX = r"custom_data\.([a-zA-Z0-9_\.]+)"class IndexingConstants:diff --git a/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py b/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.pyindex 97cfee466..6c4013312 100644--- a/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py+++ b/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py@@ -53,7 +53,7 @@ def prompt_processor() -> Any:file_path = payload.get(PSKeys.FILE_PATH)doc_name = str(payload.get(PSKeys.FILE_NAME, ""))log_events_id: str = payload.get(PSKeys.LOG_EVENTS_ID, "")- user_data: dict[str, Any] = payload.get(PSKeys.USER_DATA, {})+ custom_data: dict[str, Any] = payload.get(PSKeys.CUSTOM_DATA, {})structured_output: dict[str, Any] = {}metadata: dict[str, Any] = {PSKeys.RUN_ID: run_id,@@ -95,7 +95,7 @@ def prompt_processor() -> Any:tool_id=tool_id,prompt_name=prompt_name,doc_name=doc_name,- user_data=user_data,+ custom_data=custom_data,)app.logger.info(f"[{tool_id}] Executing prompt: '{prompt_name}'")diff --git a/prompt-service/src/unstract/prompt_service/helpers/variable_replacement.py b/prompt-service/src/unstract/prompt_service/helpers/variable_replacement.pyindex 88e197bdb..dc6e3f43a 100644--- a/prompt-service/src/unstract/prompt_service/helpers/variable_replacement.py+++ b/prompt-service/src/unstract/prompt_service/helpers/variable_replacement.py@@ -61,10 +61,10 @@ class VariableReplacementHelper:def identify_variable_type(variable: str) -> VariableType:variable_type: VariableType- # Check for user_data variable type first- user_data_pattern = re.compile(VariableConstants.USER_DATA_VARIABLE_REGEX)- if re.findall(user_data_pattern, variable):- variable_type = VariableType.USER_DATA+ # Check for custom_data variable type first+ custom_data_pattern = re.compile(VariableConstants.CUSTOM_DATA_VARIABLE_REGEX)+ if re.findall(custom_data_pattern, variable):+ variable_type = VariableType.CUSTOM_DATAelse:# Check for dynamic variable typedynamic_pattern = re.compile(VariableConstants.DYNAMIC_VARIABLE_URL_REGEX)@@ -100,41 +100,45 @@ class VariableReplacementHelper:return replaced_prompt@staticmethod- def replace_user_data_variable(- prompt: str, variable: str, user_data: dict[str, Any]+ def replace_custom_data_variable(+ prompt: str, variable: str, custom_data: dict[str, Any]) -> str:- """Replace user_data variable in prompt.+ """Replace custom_data variable in prompt.Args:prompt: The prompt containing variables- variable: The variable to replace (e.g., "user_data.name")- user_data: The user data dictionary+ variable: The variable to replace (e.g., "custom_data.name")+ custom_data: The custom_data data dictionaryReturns:prompt with variable replaced"""- if not user_data:- error_msg = f"User data is empty. Unable to replace variable {variable}"+ if not custom_data:+ error_msg = f"Custom data is empty. Unable to replace variable {variable}"app.logger.error(error_msg)raise ValueError(error_msg)- # Extract the path from user_data.path.to.value- user_data_match = re.search(VariableConstants.USER_DATA_VARIABLE_REGEX, variable)- if not user_data_match:- error_msg = f"Invalid user_data variable format: {variable}"+ # Extract the path from custom_data.path.to.value+ custom_data_match = re.search(+ VariableConstants.CUSTOM_DATA_VARIABLE_REGEX, variable+ )+ if not custom_data_match:+ error_msg = f"Invalid custom_data variable format: {variable}"app.logger.error(error_msg)raise ValueError(error_msg)- path_str = user_data_match.group(1)+ path_str = custom_data_match.group(1)path_parts = path_str.split(".")# Navigate through the nested dictionarytry:- value = user_data+ value = custom_datafor part in path_parts:value = value[part]except (KeyError, TypeError) as e:- error_msg = f"Path {path_str} not found in user_data for variable {variable}"+ error_msg = (+ f"Path {path_str} not found in custom_data for variable {variable}"+ )app.logger.error(error_msg)raise ValueError(error_msg) from ediff --git a/prompt-service/src/unstract/prompt_service/services/variable_replacement.py b/prompt-service/src/unstract/prompt_service/services/variable_replacement.pyindex defabdce6..f6f56412a 100644--- a/prompt-service/src/unstract/prompt_service/services/variable_replacement.py+++ b/prompt-service/src/unstract/prompt_service/services/variable_replacement.py@@ -34,7 +34,7 @@ class VariableReplacementService:tool_id: str,prompt_name: str,doc_name: str,- user_data: dict[str, Any] = None,+ custom_data: dict[str, Any] = None,) -> str:"""Replaces variables in prompt.@@ -69,7 +69,7 @@ class VariableReplacementService:prompt_text = VariableReplacementService._execute_variable_replacement(prompt_text=prompt[PSKeys.PROMPT],variable_map=variable_map,- user_data=user_data,+ custom_data=custom_data,)except KeyError:# Executed incase of structured tool and@@ -77,7 +77,7 @@ class VariableReplacementService:prompt_text = VariableReplacementService._execute_variable_replacement(prompt_text=prompt_text,variable_map=structured_output,- user_data=user_data,+ custom_data=custom_data,)finally:app.logger.info(@@ -98,7 +98,9 @@ class VariableReplacementService:@staticmethoddef _execute_variable_replacement(- prompt_text: str, variable_map: dict[str, Any], user_data: dict[str, Any] = None+ prompt_text: str,+ variable_map: dict[str, Any],+ custom_data: dict[str, Any] = None,) -> str:variables: list[str] = VariableReplacementHelper.extract_variables_from_prompt(prompt_text=prompt_text@@ -121,10 +123,10 @@ class VariableReplacementService:structured_output=variable_map,)- elif variable_type == VariableType.USER_DATA and user_data:- prompt_text = VariableReplacementHelper.replace_user_data_variable(+ elif variable_type == VariableType.CUSTOM_DATA and custom_data:+ prompt_text = VariableReplacementHelper.replace_custom_data_variable(prompt=prompt_text,variable=variable,- user_data=user_data,+ custom_data=custom_data,)return prompt_textdiff --git a/prompt-service/uv.lock b/prompt-service/uv.lockindex b03db54b2..3e3d4176c 100644--- a/prompt-service/uv.lock+++ b/prompt-service/uv.lock@@ -3145,7 +3145,7 @@ requires-dist = [{ name = "s3fs", specifier = "==2024.10.0" },{ name = "unstract-core", editable = "../unstract/core" },{ name = "unstract-flags", editable = "../unstract/flags" },- { name = "unstract-sdk", extras = ["azure"], specifier = "~=0.77.2" },+ { name = "unstract-sdk", extras = ["azure"], specifier = "~=0.77.3" },][package.metadata.requires-dev]@@ -3168,7 +3168,7 @@ test = [[[package]]name = "unstract-sdk"-version = "0.77.2"+version = "0.77.3"source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "filetype" },@@ -3207,9 +3207,9 @@ dependencies = [{ name = "tiktoken" },{ name = "transformers" },]-sdist = { url = "https://files.pythonhosted.org/packages/c6/56/b2d95e1029623e188c34702b7729d12200437aa8d7deb9d5a8edd88f2994/unstract_sdk-0.77.2.tar.gz", hash = "sha256:30a0fcbbeac7fcaadc96fc0754cf35bebc0dc379eddef874d28aa36a54f374ec", size = 2375849 }+sdist = { url = "https://files.pythonhosted.org/packages/c5/e8/f7e1045fee076c75c42bb27b4fa5a077836cfa8fd3fd580f6c6193fa19dc/unstract_sdk-0.77.3.tar.gz", hash = "sha256:378c19129a91e861b7235e92411ef72792c6c851320cb7897b380d08d3489c9d", size = 2375843 }wheels = [- { url = "https://files.pythonhosted.org/packages/8e/f9/f7b0e11f7e79162c7b2dfdbcbd1b1ad74b60d26e8d922b743db2b0d26214/unstract_sdk-0.77.2-py3-none-any.whl", hash = "sha256:8f57c39e814785de80b9fc74c972eb648b0a0b008edcac3ad0f6167e555944f8", size = 266633 },+ { url = "https://files.pythonhosted.org/packages/d5/84/d596295fce3a713b1953a4fd7be3be1d788920e5633e28fbea7c7331b68a/unstract_sdk-0.77.3-py3-none-any.whl", hash = "sha256:74e83cbb68eef98fbaecfc83e0cfd2228e5958506e1b51a65fae0d891027aaee", size = 266640 },][package.optional-dependencies]diff --git a/tools/structure/requirements.txt b/tools/structure/requirements.txtindex b8fff8f81..a771c2742 100644--- a/tools/structure/requirements.txt+++ b/tools/structure/requirements.txt@@ -3,4 +3,4 @@# Required for all unstract tools# aws alone is needed here# because tools use transient temporary storage.-unstract-sdk[aws]~=0.77.2+unstract-sdk[aws]~=0.77.3diff --git a/tools/structure/src/config/properties.json b/tools/structure/src/config/properties.jsonindex dcb43de89..22f343dff 100644--- a/tools/structure/src/config/properties.json+++ b/tools/structure/src/config/properties.json@@ -2,7 +2,7 @@"schemaVersion": "0.0.1","displayName": "Structure Tool","functionName": "structure_tool",- "toolVersion": "0.0.87",+ "toolVersion": "0.0.88","description": "This is a template tool which can answer set of input prompts designed in the Prompt Studio","input": {"description": "File that needs to be indexed and parsed for answers"diff --git a/tools/structure/src/constants.py b/tools/structure/src/constants.pyindex 84871affe..769481039 100644--- a/tools/structure/src/constants.py+++ b/tools/structure/src/constants.py@@ -80,7 +80,7 @@ class SettingsKeys:EXECUTION_ID = "execution_id"IS_DIRECTORY_MODE = "is_directory_mode"LLM_PROFILE_ID = "llm_profile_id"- USER_DATA = "user_data"+ CUSTOM_DATA = "custom_data"class IndexingConstants:diff --git a/tools/structure/src/main.py b/tools/structure/src/main.pyindex 2057e26ab..af5377bac 100644--- a/tools/structure/src/main.py+++ b/tools/structure/src/main.py@@ -220,8 +220,8 @@ class StructureTool(BaseTool):SettingsKeys.EXECUTION_SOURCE: SettingsKeys.TOOL,}- user_data = self.get_exec_metadata.get(SettingsKeys.USER_DATA, {})- payload["user_data"] = user_data+ custom_data = self.get_exec_metadata.get(SettingsKeys.CUSTOM_DATA, {})+ payload["custom_data"] = custom_dataself.stream_log(f"Extracting document '{self.source_file_name}'")usage_kwargs: dict[Any, Any] = dict()usage_kwargs[UsageKwargs.RUN_ID] = self.file_execution_iddiff --git a/unstract/workflow-execution/src/unstract/workflow_execution/constants.py b/unstract/workflow-execution/src/unstract/workflow_execution/constants.pyindex ac4ab0876..4021deaba 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/constants.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/constants.py@@ -46,7 +46,7 @@ class MetaDataKey:TOOL_METADATA = "tool_metadata"TAGS = "tags"LLM_PROFILE_ID = "llm_profile_id"- USER_DATA = "user_data"+ CUSTOM_DATA = "custom_data"class ToolMetadataKey:diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py b/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.pyindex aed528be5..9bc46f703 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py@@ -102,7 +102,7 @@ class ExecutionFileHandler:source_hash: str,tags: list[str],llm_profile_id: str | None = None,- user_data: dict[str, Any] | None = None,+ custom_data: dict[str, Any] | None = None,) -> None:"""Creating metadata for workflow. This method is responsible forcreating metadata for the workflow. It takes the input file path and@@ -142,9 +142,9 @@ class ExecutionFileHandler:if llm_profile_id:content[MetaDataKey.LLM_PROFILE_ID] = llm_profile_id- # Add user_data to metadata if provided- if user_data:- content[MetaDataKey.USER_DATA] = user_data+ # Add custom_data to metadata if provided+ if custom_data:+ content[MetaDataKey.CUSTOM_DATA] = custom_datafile_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)file_storage = file_system.get_file_storage()