Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2807 [MISC] Changed user_data to custom_data in variable replacement (#1548)

jagadeeswaran-zipstack29b89ffcd57f
diff --git a/backend/api_v2/api_deployment_views.py b/backend/api_v2/api_deployment_views.py
index 256f2b711..ac876548e 100644
--- a/backend/api_v2/api_deployment_views.py
+++ b/backend/api_v2/api_deployment_views.py
@@ -70,7 +70,7 @@ class DeploymentExecution(views.APIView):
tag_names = serializer.validated_data.get(ApiExecution.TAGS)
llm_profile_id = serializer.validated_data.get(ApiExecution.LLM_PROFILE_ID)
hitl_queue_name = serializer.validated_data.get(ApiExecution.HITL_QUEUE_NAME)
- user_data = serializer.validated_data.get(ApiExecution.USER_DATA)
+ custom_data = serializer.validated_data.get(ApiExecution.CUSTOM_DATA)
if presigned_urls:
DeploymentHelper.load_presigned_files(presigned_urls, file_objs)
@@ -86,7 +86,7 @@ class DeploymentExecution(views.APIView):
tag_names=tag_names,
llm_profile_id=llm_profile_id,
hitl_queue_name=hitl_queue_name,
- user_data=user_data,
+ custom_data=custom_data,
request_headers=dict(request.headers),
)
if "error" in response and response["error"]:
diff --git a/backend/api_v2/constants.py b/backend/api_v2/constants.py
index 9d06e7881..a8f5cc559 100644
--- a/backend/api_v2/constants.py
+++ b/backend/api_v2/constants.py
@@ -11,4 +11,4 @@ class ApiExecution:
LLM_PROFILE_ID: str = "llm_profile_id"
HITL_QUEUE_NAME: str = "hitl_queue_name"
PRESIGNED_URLS: str = "presigned_urls"
- USER_DATA: str = "user_data"
+ CUSTOM_DATA: str = "custom_data"
diff --git a/backend/api_v2/deployment_helper.py b/backend/api_v2/deployment_helper.py
index 082e77975..a0e059111 100644
--- a/backend/api_v2/deployment_helper.py
+++ b/backend/api_v2/deployment_helper.py
@@ -155,7 +155,7 @@ class DeploymentHelper(BaseAPIKeyValidator):
tag_names: list[str] = [],
llm_profile_id: str | None = None,
hitl_queue_name: str | None = None,
- user_data: dict[str, Any] | None = None,
+ custom_data: dict[str, Any] | None = None,
request_headers=None,
) -> ReturnDict:
"""Execute workflow by api.
@@ -169,7 +169,7 @@ class DeploymentHelper(BaseAPIKeyValidator):
tag_names (list(str)): list of tag names
llm_profile_id (str, optional): LLM profile ID for overriding tool settings
hitl_queue_name (str, optional): Custom queue name for manual review
- user_data (dict[str, Any], optional): JSON data for user_data variable replacement in prompts
+ custom_data (dict[str, Any], optional): JSON data for custom_data variable replacement in prompts
Returns:
ReturnDict: execution status/ result
@@ -236,7 +236,7 @@ class DeploymentHelper(BaseAPIKeyValidator):
use_file_history=use_file_history,
llm_profile_id=llm_profile_id,
hitl_queue_name=hitl_queue_name,
- user_data=user_data,
+ custom_data=custom_data,
)
result.status_api = DeploymentHelper.construct_status_endpoint(
api_endpoint=api.api_endpoint, execution_id=execution_id
diff --git a/backend/api_v2/serializers.py b/backend/api_v2/serializers.py
index caf21838d..e7d1acabb 100644
--- a/backend/api_v2/serializers.py
+++ b/backend/api_v2/serializers.py
@@ -195,23 +195,23 @@ class ExecutionRequestSerializer(TagParamsSerializer):
"""Execution request serializer.
Attributes:
- timeout (int): Timeout for the API deployment, maximum value can be 300s.
- If -1 it corresponds to async execution. Defaults to -1
- include_metadata (bool): Flag to include metadata in API response
- include_metrics (bool): Flag to include metrics in API response
- use_file_history (bool): Flag to use FileHistory to save and retrieve
- responses quickly. This is undocumented to the user and can be
- helpful for demos.
- tags (str): Comma-separated List of tags to associate with the execution.
- e.g:'tag1,tag2-name,tag3_name'
- llm_profile_id (str): UUID of the LLM profile to override the default profile.
- If not provided, the default profile will be used.
- hitl_queue_name (str, optional): Document class name for manual review queue.
- If not provided, uses API name as document class.
- presigned_urls (list): List of presigned URLs to fetch files from.
- URLs are validated for HTTPS and S3 endpoint requirements.
- user_data (dict, optional): User-provided data for variable replacement in prompts.
- Can be accessed in prompts using {{user_data.key}} syntax for dot notation traversal.
+ timeout (int): Timeout for the API deployment, maximum value can be 300s.
+ If -1 it corresponds to async execution. Defaults to -1
+ include_metadata (bool): Flag to include metadata in API response
+ include_metrics (bool): Flag to include metrics in API response
+ use_file_history (bool): Flag to use FileHistory to save and retrieve
+ responses quickly. This is undocumented to the user and can be
+ helpful for demos.
+ tags (str): Comma-separated List of tags to associate with the execution.
+ e.g:'tag1,tag2-name,tag3_name'
+ llm_profile_id (str): UUID of the LLM profile to override the default profile.
+ If not provided, the default profile will be used.
+ hitl_queue_name (str, optional): Document class name for manual review queue.
+ If not provided, uses API name as document class.
+ presigned_urls (list): List of presigned URLs to fetch files from.
+ URLs are validated for HTTPS and S3 endpoint requirements.
+ custom_data (dict, optional): User-provided data for variable replacement in prompts.
+ Can be accessed in prompts using {{custom_data.key}} syntax for dot notation traversal.
"""
MAX_FILES_ALLOWED = 32
@@ -226,7 +226,7 @@ class ExecutionRequestSerializer(TagParamsSerializer):
presigned_urls = ListField(child=URLField(), required=False)
llm_profile_id = CharField(required=False, allow_null=True, allow_blank=True)
hitl_queue_name = CharField(required=False, allow_null=True, allow_blank=True)
- user_data = JSONField(required=False, allow_null=True)
+ custom_data = JSONField(required=False, allow_null=True)
def validate_hitl_queue_name(self, value: str | None) -> str | None:
"""Validate queue name format using enterprise validation if available."""
@@ -247,13 +247,13 @@ class ExecutionRequestSerializer(TagParamsSerializer):
)
return value
- def validate_user_data(self, value):
- """Validate user_data is a valid JSON object."""
+ def validate_custom_data(self, value):
+ """Validate custom_data is a valid JSON object."""
if value is None:
return value
if not isinstance(value, dict):
- raise ValidationError("user_data must be a JSON object")
+ raise ValidationError("custom_data must be a JSON object")
return value
diff --git a/backend/pyproject.toml b/backend/pyproject.toml
index 2cf56042e..363a6ee75 100644
--- a/backend/pyproject.toml
+++ b/backend/pyproject.toml
@@ -37,7 +37,7 @@ dependencies = [
"social-auth-core==4.4.2", # For OAuth
# TODO: Temporarily removing the extra dependencies of aws and gcs from unstract-sdk
# to resolve lock file. Will have to be re-looked into
- "unstract-sdk[azure]~=0.77.2",
+ "unstract-sdk[azure]~=0.77.3",
"gcsfs==2024.10.0",
"s3fs==2024.10.0",
"azure-identity==1.16.0",
diff --git a/backend/sample.env b/backend/sample.env
index b0331b23a..b1347ef15 100644
--- a/backend/sample.env
+++ b/backend/sample.env
@@ -78,9 +78,9 @@ PROMPT_STUDIO_FILE_PATH=/app/prompt-studio-data
# Structure Tool Image (Runs prompt studio exported tools)
# https://hub.docker.com/r/unstract/tool-structure
-STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.87"
+STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.88"
STRUCTURE_TOOL_IMAGE_NAME="unstract/tool-structure"
-STRUCTURE_TOOL_IMAGE_TAG="0.0.87"
+STRUCTURE_TOOL_IMAGE_TAG="0.0.88"
# Feature Flags
EVALUATION_SERVER_IP=unstract-flipt
diff --git a/backend/uv.lock b/backend/uv.lock
index 4e5caf12a..549cf7e07 100644
--- a/backend/uv.lock
+++ b/backend/uv.lock
@@ -3913,7 +3913,7 @@ requires-dist = [
{ name = "unstract-core", editable = "../unstract/core" },
{ name = "unstract-filesystem", editable = "../unstract/filesystem" },
{ name = "unstract-flags", editable = "../unstract/flags" },
- { name = "unstract-sdk", extras = ["azure"], specifier = "~=0.77.2" },
+ { name = "unstract-sdk", extras = ["azure"], specifier = "~=0.77.3" },
{ name = "unstract-tool-registry", editable = "../unstract/tool-registry" },
{ name = "unstract-tool-sandbox", editable = "../unstract/tool-sandbox" },
{ name = "unstract-workflow-execution", editable = "../unstract/workflow-execution" },
@@ -4038,7 +4038,7 @@ requires-dist = [
[[package]]
name = "unstract-sdk"
-version = "0.77.2"
+version = "0.77.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "filetype" },
@@ -4077,9 +4077,9 @@ dependencies = [
{ name = "tiktoken" },
{ name = "transformers" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/c6/56/b2d95e1029623e188c34702b7729d12200437aa8d7deb9d5a8edd88f2994/unstract_sdk-0.77.2.tar.gz", hash = "sha256:30a0fcbbeac7fcaadc96fc0754cf35bebc0dc379eddef874d28aa36a54f374ec", size = 2375849 }
+sdist = { url = "https://files.pythonhosted.org/packages/c5/e8/f7e1045fee076c75c42bb27b4fa5a077836cfa8fd3fd580f6c6193fa19dc/unstract_sdk-0.77.3.tar.gz", hash = "sha256:378c19129a91e861b7235e92411ef72792c6c851320cb7897b380d08d3489c9d", size = 2375843 }
wheels = [
- { url = "https://files.pythonhosted.org/packages/8e/f9/f7b0e11f7e79162c7b2dfdbcbd1b1ad74b60d26e8d922b743db2b0d26214/unstract_sdk-0.77.2-py3-none-any.whl", hash = "sha256:8f57c39e814785de80b9fc74c972eb648b0a0b008edcac3ad0f6167e555944f8", size = 266633 },
+ { url = "https://files.pythonhosted.org/packages/d5/84/d596295fce3a713b1953a4fd7be3be1d788920e5633e28fbea7c7331b68a/unstract_sdk-0.77.3-py3-none-any.whl", hash = "sha256:74e83cbb68eef98fbaecfc83e0cfd2228e5958506e1b51a65fae0d891027aaee", size = 266640 },
]
[package.optional-dependencies]
diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.py
index 4edca72bf..a0e2b6365 100644
--- a/backend/workflow_manager/endpoint_v2/source.py
+++ b/backend/workflow_manager/endpoint_v2/source.py
@@ -921,7 +921,7 @@ class SourceConnector(BaseConnector):
tags: list[str],
file_hash: FileHash,
llm_profile_id: str | None = None,
- user_data: dict[str, Any] | None = None,
+ custom_data: dict[str, Any] | None = None,
) -> str:
"""Add input file to execution directory.
@@ -963,7 +963,7 @@ class SourceConnector(BaseConnector):
source_hash=file_content_hash,
tags=tags,
llm_profile_id=llm_profile_id,
- user_data=user_data,
+ custom_data=custom_data,
)
return file_content_hash
diff --git a/backend/workflow_manager/workflow_v2/dto.py b/backend/workflow_manager/workflow_v2/dto.py
index 02c13f3e1..b2398e883 100644
--- a/backend/workflow_manager/workflow_v2/dto.py
+++ b/backend/workflow_manager/workflow_v2/dto.py
@@ -156,7 +156,7 @@ class FileData:
use_file_history: bool
q_file_no_list: list[int]
llm_profile_id: str | None = None
- user_data: dict[str, Any] | None = None
+ custom_data: dict[str, Any] | None = None
@classmethod
def from_dict(cls, data: dict[str, Any]) -> FileData:
diff --git a/backend/workflow_manager/workflow_v2/file_execution_tasks.py b/backend/workflow_manager/workflow_v2/file_execution_tasks.py
index 88e88358d..c8b8ca0a0 100644
--- a/backend/workflow_manager/workflow_v2/file_execution_tasks.py
+++ b/backend/workflow_manager/workflow_v2/file_execution_tasks.py
@@ -751,7 +751,7 @@ class FileExecutionTasks:
tags=workflow_execution.tag_names,
file_hash=file_hash,
llm_profile_id=file_data.llm_profile_id,
- user_data=file_data.user_data,
+ custom_data=file_data.custom_data,
)
file_hash.file_hash = content_hash
workflow_file_exec.update(file_hash=content_hash)
@@ -920,7 +920,8 @@ class FileExecutionTasks:
)
# Log execution costs
execution_service.log_total_cost_per_file(
- run_id=str(workflow_file_execution.id), file_name=file_hash.file_name
+ run_id=str(workflow_file_execution.id),
+ file_name=file_hash.file_name,
)
tool_execution_result = ToolExecutionResult(error=None, result=None)
return tool_execution_result
@@ -995,7 +996,9 @@ class FileExecutionTasks:
file_path = file_hash.file_path if not destination.is_api else None
# Collect metadata from file history if available
file_history = FileHistoryHelper.get_file_history(
- workflow=workflow, cache_key=file_hash.file_hash, file_path=file_path
+ workflow=workflow,
+ cache_key=file_hash.file_hash,
+ file_path=file_path,
)
else:
file_history = None
diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.py
index 303113612..f9433c639 100644
--- a/backend/workflow_manager/workflow_v2/workflow_helper.py
+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py
@@ -64,7 +64,7 @@ logger = logging.getLogger(__name__)
EXECUTION_EXCLUDED_PARAMS = {
"llm_profile_id",
"hitl_queue_name",
- "user_data",
+ "custom_data",
}
@@ -151,7 +151,7 @@ class WorkflowHelper:
execution_mode: tuple[str, str],
use_file_history: bool,
llm_profile_id: str | None,
- user_data: dict[str, Any] | None = None,
+ custom_data: dict[str, Any] | None = None,
) -> str | None:
total_files = len(input_files)
workflow_log.publish_initial_workflow_logs(total_files=total_files)
@@ -203,7 +203,7 @@ class WorkflowHelper:
use_file_history=use_file_history,
q_file_no_list=list(q_file_no_list) if q_file_no_list else [],
llm_profile_id=llm_profile_id,
- user_data=user_data,
+ custom_data=custom_data,
)
batch_data = FileBatchData(files=batch, file_data=file_data)
@@ -270,7 +270,7 @@ class WorkflowHelper:
use_file_history: bool = True,
llm_profile_id: str | None = None,
hitl_queue_name: str | None = None,
- user_data: dict[str, Any] | None = None,
+ custom_data: dict[str, Any] | None = None,
) -> ExecutionResponse:
tool_instances: list[ToolInstance] = (
ToolInstanceHelper.get_tool_instances_by_workflow(
@@ -322,7 +322,7 @@ class WorkflowHelper:
use_file_history=use_file_history,
execution_mode=execution_mode,
llm_profile_id=llm_profile_id,
- user_data=user_data,
+ custom_data=custom_data,
)
api_results = []
return ExecutionResponse(
@@ -441,7 +441,7 @@ class WorkflowHelper:
use_file_history: bool = True,
llm_profile_id: str | None = None,
hitl_queue_name: str | None = None,
- user_data: dict[str, Any] | None = None,
+ custom_data: dict[str, Any] | None = None,
) -> ExecutionResponse:
"""Adding a workflow to the queue for execution.
@@ -481,7 +481,7 @@ class WorkflowHelper:
"use_file_history": use_file_history,
"llm_profile_id": llm_profile_id,
"hitl_queue_name": hitl_queue_name,
- "user_data": user_data,
+ "custom_data": custom_data,
},
queue=queue,
)
@@ -699,7 +699,7 @@ class WorkflowHelper:
use_file_history=use_file_history,
llm_profile_id=kwargs.get("llm_profile_id"),
hitl_queue_name=kwargs.get("hitl_queue_name"),
- user_data=kwargs.get("user_data"),
+ custom_data=kwargs.get("custom_data"),
)
except Exception as error:
error_message = traceback.format_exc()
diff --git a/platform-service/pyproject.toml b/platform-service/pyproject.toml
index f91d91cd8..cd9519eb6 100644
--- a/platform-service/pyproject.toml
+++ b/platform-service/pyproject.toml
@@ -15,7 +15,7 @@ dependencies = [
"redis~=5.2.1",
"cryptography>=41.0.7",
"requests>=2.31.0",
- "unstract-sdk[gcs, azure, aws]~=0.77.2", # Add version
+ "unstract-sdk[gcs, azure, aws]~=0.77.3", # Add version
"unstract-flags",
"unstract-core[flask]",
]
diff --git a/platform-service/uv.lock b/platform-service/uv.lock
index b283eac42..2026bd334 100644
--- a/platform-service/uv.lock
+++ b/platform-service/uv.lock
@@ -3075,7 +3075,7 @@ requires-dist = [
{ name = "requests", specifier = ">=2.31.0" },
{ name = "unstract-core", extras = ["flask"], editable = "../unstract/core" },
{ name = "unstract-flags", editable = "../unstract/flags" },
- { name = "unstract-sdk", extras = ["gcs", "azure", "aws"], specifier = "~=0.77.2" },
+ { name = "unstract-sdk", extras = ["gcs", "azure", "aws"], specifier = "~=0.77.3" },
]
[package.metadata.requires-dev]
@@ -3092,7 +3092,7 @@ test = [{ name = "pytest", specifier = ">=8.0.1" }]
[[package]]
name = "unstract-sdk"
-version = "0.77.2"
+version = "0.77.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "filetype" },
@@ -3131,9 +3131,9 @@ dependencies = [
{ name = "tiktoken" },
{ name = "transformers" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/c6/56/b2d95e1029623e188c34702b7729d12200437aa8d7deb9d5a8edd88f2994/unstract_sdk-0.77.2.tar.gz", hash = "sha256:30a0fcbbeac7fcaadc96fc0754cf35bebc0dc379eddef874d28aa36a54f374ec", size = 2375849 }
+sdist = { url = "https://files.pythonhosted.org/packages/c5/e8/f7e1045fee076c75c42bb27b4fa5a077836cfa8fd3fd580f6c6193fa19dc/unstract_sdk-0.77.3.tar.gz", hash = "sha256:378c19129a91e861b7235e92411ef72792c6c851320cb7897b380d08d3489c9d", size = 2375843 }
wheels = [
- { url = "https://files.pythonhosted.org/packages/8e/f9/f7b0e11f7e79162c7b2dfdbcbd1b1ad74b60d26e8d922b743db2b0d26214/unstract_sdk-0.77.2-py3-none-any.whl", hash = "sha256:8f57c39e814785de80b9fc74c972eb648b0a0b008edcac3ad0f6167e555944f8", size = 266633 },
+ { url = "https://files.pythonhosted.org/packages/d5/84/d596295fce3a713b1953a4fd7be3be1d788920e5633e28fbea7c7331b68a/unstract_sdk-0.77.3-py3-none-any.whl", hash = "sha256:74e83cbb68eef98fbaecfc83e0cfd2228e5958506e1b51a65fae0d891027aaee", size = 266640 },
]
[package.optional-dependencies]
diff --git a/prompt-service/pyproject.toml b/prompt-service/pyproject.toml
index 2e0524aeb..ff0e52bf4 100644
--- a/prompt-service/pyproject.toml
+++ b/prompt-service/pyproject.toml
@@ -17,7 +17,7 @@ dependencies = [
"requests>=2.28,<3.0",
# TODO: Temporarily removing the extra dependencies of aws and gcs from unstract-sdk
# to resolve lock file. Will have to be re-looked into
- "unstract-sdk[azure]~=0.77.2",
+ "unstract-sdk[azure]~=0.77.3",
"gcsfs==2024.10.0",
"s3fs==2024.10.0",
"redis>=5.0.3,<5.3",
diff --git a/prompt-service/src/unstract/prompt_service/constants.py b/prompt-service/src/unstract/prompt_service/constants.py
index f12b9eee4..09b6a768a 100644
--- a/prompt-service/src/unstract/prompt_service/constants.py
+++ b/prompt-service/src/unstract/prompt_service/constants.py
@@ -67,7 +67,7 @@ class PromptServiceConstants:
SUMMARIZE_AS_SOURCE = "summarize_as_source"
VARIABLE_MAP = "variable_map"
RECORD = "record"
- USER_DATA = "user_data"
+ CUSTOM_DATA = "custom_data"
TEXT = "text"
ENABLE_HIGHLIGHT = "enable_highlight"
FILE_PATH = "file_path"
@@ -148,7 +148,7 @@ class VariableType(str, Enum):
STATIC = "STATIC"
DYNAMIC = "DYNAMIC"
- USER_DATA = "USER_DATA"
+ CUSTOM_DATA = "CUSTOM_DATA"
class RetrievalStrategy(str, Enum):
@@ -172,7 +172,7 @@ class VariableConstants:
r"(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'\".,<>?«»"
"'']))"
) # noqa: E501
- USER_DATA_VARIABLE_REGEX = r"user_data\.([a-zA-Z0-9_\.]+)"
+ CUSTOM_DATA_VARIABLE_REGEX = r"custom_data\.([a-zA-Z0-9_\.]+)"
class IndexingConstants:
diff --git a/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py b/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py
index 97cfee466..6c4013312 100644
--- a/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py
+++ b/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py
@@ -53,7 +53,7 @@ def prompt_processor() -> Any:
file_path = payload.get(PSKeys.FILE_PATH)
doc_name = str(payload.get(PSKeys.FILE_NAME, ""))
log_events_id: str = payload.get(PSKeys.LOG_EVENTS_ID, "")
- user_data: dict[str, Any] = payload.get(PSKeys.USER_DATA, {})
+ custom_data: dict[str, Any] = payload.get(PSKeys.CUSTOM_DATA, {})
structured_output: dict[str, Any] = {}
metadata: dict[str, Any] = {
PSKeys.RUN_ID: run_id,
@@ -95,7 +95,7 @@ def prompt_processor() -> Any:
tool_id=tool_id,
prompt_name=prompt_name,
doc_name=doc_name,
- user_data=user_data,
+ custom_data=custom_data,
)
app.logger.info(f"[{tool_id}] Executing prompt: '{prompt_name}'")
diff --git a/prompt-service/src/unstract/prompt_service/helpers/variable_replacement.py b/prompt-service/src/unstract/prompt_service/helpers/variable_replacement.py
index 88e197bdb..dc6e3f43a 100644
--- a/prompt-service/src/unstract/prompt_service/helpers/variable_replacement.py
+++ b/prompt-service/src/unstract/prompt_service/helpers/variable_replacement.py
@@ -61,10 +61,10 @@ class VariableReplacementHelper:
def identify_variable_type(variable: str) -> VariableType:
variable_type: VariableType
- # Check for user_data variable type first
- user_data_pattern = re.compile(VariableConstants.USER_DATA_VARIABLE_REGEX)
- if re.findall(user_data_pattern, variable):
- variable_type = VariableType.USER_DATA
+ # Check for custom_data variable type first
+ custom_data_pattern = re.compile(VariableConstants.CUSTOM_DATA_VARIABLE_REGEX)
+ if re.findall(custom_data_pattern, variable):
+ variable_type = VariableType.CUSTOM_DATA
else:
# Check for dynamic variable type
dynamic_pattern = re.compile(VariableConstants.DYNAMIC_VARIABLE_URL_REGEX)
@@ -100,41 +100,45 @@ class VariableReplacementHelper:
return replaced_prompt
@staticmethod
- def replace_user_data_variable(
- prompt: str, variable: str, user_data: dict[str, Any]
+ def replace_custom_data_variable(
+ prompt: str, variable: str, custom_data: dict[str, Any]
) -> str:
- """Replace user_data variable in prompt.
+ """Replace custom_data variable in prompt.
Args:
prompt: The prompt containing variables
- variable: The variable to replace (e.g., "user_data.name")
- user_data: The user data dictionary
+ variable: The variable to replace (e.g., "custom_data.name")
+ custom_data: The custom_data data dictionary
Returns:
prompt with variable replaced
"""
- if not user_data:
- error_msg = f"User data is empty. Unable to replace variable {variable}"
+ if not custom_data:
+ error_msg = f"Custom data is empty. Unable to replace variable {variable}"
app.logger.error(error_msg)
raise ValueError(error_msg)
- # Extract the path from user_data.path.to.value
- user_data_match = re.search(VariableConstants.USER_DATA_VARIABLE_REGEX, variable)
- if not user_data_match:
- error_msg = f"Invalid user_data variable format: {variable}"
+ # Extract the path from custom_data.path.to.value
+ custom_data_match = re.search(
+ VariableConstants.CUSTOM_DATA_VARIABLE_REGEX, variable
+ )
+ if not custom_data_match:
+ error_msg = f"Invalid custom_data variable format: {variable}"
app.logger.error(error_msg)
raise ValueError(error_msg)
- path_str = user_data_match.group(1)
+ path_str = custom_data_match.group(1)
path_parts = path_str.split(".")
# Navigate through the nested dictionary
try:
- value = user_data
+ value = custom_data
for part in path_parts:
value = value[part]
except (KeyError, TypeError) as e:
- error_msg = f"Path {path_str} not found in user_data for variable {variable}"
+ error_msg = (
+ f"Path {path_str} not found in custom_data for variable {variable}"
+ )
app.logger.error(error_msg)
raise ValueError(error_msg) from e
diff --git a/prompt-service/src/unstract/prompt_service/services/variable_replacement.py b/prompt-service/src/unstract/prompt_service/services/variable_replacement.py
index defabdce6..f6f56412a 100644
--- a/prompt-service/src/unstract/prompt_service/services/variable_replacement.py
+++ b/prompt-service/src/unstract/prompt_service/services/variable_replacement.py
@@ -34,7 +34,7 @@ class VariableReplacementService:
tool_id: str,
prompt_name: str,
doc_name: str,
- user_data: dict[str, Any] = None,
+ custom_data: dict[str, Any] = None,
) -> str:
"""Replaces variables in prompt.
@@ -69,7 +69,7 @@ class VariableReplacementService:
prompt_text = VariableReplacementService._execute_variable_replacement(
prompt_text=prompt[PSKeys.PROMPT],
variable_map=variable_map,
- user_data=user_data,
+ custom_data=custom_data,
)
except KeyError:
# Executed incase of structured tool and
@@ -77,7 +77,7 @@ class VariableReplacementService:
prompt_text = VariableReplacementService._execute_variable_replacement(
prompt_text=prompt_text,
variable_map=structured_output,
- user_data=user_data,
+ custom_data=custom_data,
)
finally:
app.logger.info(
@@ -98,7 +98,9 @@ class VariableReplacementService:
@staticmethod
def _execute_variable_replacement(
- prompt_text: str, variable_map: dict[str, Any], user_data: dict[str, Any] = None
+ prompt_text: str,
+ variable_map: dict[str, Any],
+ custom_data: dict[str, Any] = None,
) -> str:
variables: list[str] = VariableReplacementHelper.extract_variables_from_prompt(
prompt_text=prompt_text
@@ -121,10 +123,10 @@ class VariableReplacementService:
structured_output=variable_map,
)
- elif variable_type == VariableType.USER_DATA and user_data:
- prompt_text = VariableReplacementHelper.replace_user_data_variable(
+ elif variable_type == VariableType.CUSTOM_DATA and custom_data:
+ prompt_text = VariableReplacementHelper.replace_custom_data_variable(
prompt=prompt_text,
variable=variable,
- user_data=user_data,
+ custom_data=custom_data,
)
return prompt_text
diff --git a/prompt-service/uv.lock b/prompt-service/uv.lock
index b03db54b2..3e3d4176c 100644
--- a/prompt-service/uv.lock
+++ b/prompt-service/uv.lock
@@ -3145,7 +3145,7 @@ requires-dist = [
{ name = "s3fs", specifier = "==2024.10.0" },
{ name = "unstract-core", editable = "../unstract/core" },
{ name = "unstract-flags", editable = "../unstract/flags" },
- { name = "unstract-sdk", extras = ["azure"], specifier = "~=0.77.2" },
+ { name = "unstract-sdk", extras = ["azure"], specifier = "~=0.77.3" },
]
[package.metadata.requires-dev]
@@ -3168,7 +3168,7 @@ test = [
[[package]]
name = "unstract-sdk"
-version = "0.77.2"
+version = "0.77.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "filetype" },
@@ -3207,9 +3207,9 @@ dependencies = [
{ name = "tiktoken" },
{ name = "transformers" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/c6/56/b2d95e1029623e188c34702b7729d12200437aa8d7deb9d5a8edd88f2994/unstract_sdk-0.77.2.tar.gz", hash = "sha256:30a0fcbbeac7fcaadc96fc0754cf35bebc0dc379eddef874d28aa36a54f374ec", size = 2375849 }
+sdist = { url = "https://files.pythonhosted.org/packages/c5/e8/f7e1045fee076c75c42bb27b4fa5a077836cfa8fd3fd580f6c6193fa19dc/unstract_sdk-0.77.3.tar.gz", hash = "sha256:378c19129a91e861b7235e92411ef72792c6c851320cb7897b380d08d3489c9d", size = 2375843 }
wheels = [
- { url = "https://files.pythonhosted.org/packages/8e/f9/f7b0e11f7e79162c7b2dfdbcbd1b1ad74b60d26e8d922b743db2b0d26214/unstract_sdk-0.77.2-py3-none-any.whl", hash = "sha256:8f57c39e814785de80b9fc74c972eb648b0a0b008edcac3ad0f6167e555944f8", size = 266633 },
+ { url = "https://files.pythonhosted.org/packages/d5/84/d596295fce3a713b1953a4fd7be3be1d788920e5633e28fbea7c7331b68a/unstract_sdk-0.77.3-py3-none-any.whl", hash = "sha256:74e83cbb68eef98fbaecfc83e0cfd2228e5958506e1b51a65fae0d891027aaee", size = 266640 },
]
[package.optional-dependencies]
diff --git a/tools/structure/requirements.txt b/tools/structure/requirements.txt
index b8fff8f81..a771c2742 100644
--- a/tools/structure/requirements.txt
+++ b/tools/structure/requirements.txt
@@ -3,4 +3,4 @@
# Required for all unstract tools
# aws alone is needed here
# because tools use transient temporary storage.
-unstract-sdk[aws]~=0.77.2
+unstract-sdk[aws]~=0.77.3
diff --git a/tools/structure/src/config/properties.json b/tools/structure/src/config/properties.json
index dcb43de89..22f343dff 100644
--- a/tools/structure/src/config/properties.json
+++ b/tools/structure/src/config/properties.json
@@ -2,7 +2,7 @@
"schemaVersion": "0.0.1",
"displayName": "Structure Tool",
"functionName": "structure_tool",
- "toolVersion": "0.0.87",
+ "toolVersion": "0.0.88",
"description": "This is a template tool which can answer set of input prompts designed in the Prompt Studio",
"input": {
"description": "File that needs to be indexed and parsed for answers"
diff --git a/tools/structure/src/constants.py b/tools/structure/src/constants.py
index 84871affe..769481039 100644
--- a/tools/structure/src/constants.py
+++ b/tools/structure/src/constants.py
@@ -80,7 +80,7 @@ class SettingsKeys:
EXECUTION_ID = "execution_id"
IS_DIRECTORY_MODE = "is_directory_mode"
LLM_PROFILE_ID = "llm_profile_id"
- USER_DATA = "user_data"
+ CUSTOM_DATA = "custom_data"
class IndexingConstants:
diff --git a/tools/structure/src/main.py b/tools/structure/src/main.py
index 2057e26ab..af5377bac 100644
--- a/tools/structure/src/main.py
+++ b/tools/structure/src/main.py
@@ -220,8 +220,8 @@ class StructureTool(BaseTool):
SettingsKeys.EXECUTION_SOURCE: SettingsKeys.TOOL,
}
- user_data = self.get_exec_metadata.get(SettingsKeys.USER_DATA, {})
- payload["user_data"] = user_data
+ custom_data = self.get_exec_metadata.get(SettingsKeys.CUSTOM_DATA, {})
+ payload["custom_data"] = custom_data
self.stream_log(f"Extracting document '{self.source_file_name}'")
usage_kwargs: dict[Any, Any] = dict()
usage_kwargs[UsageKwargs.RUN_ID] = self.file_execution_id
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/constants.py b/unstract/workflow-execution/src/unstract/workflow_execution/constants.py
index ac4ab0876..4021deaba 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/constants.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/constants.py
@@ -46,7 +46,7 @@ class MetaDataKey:
TOOL_METADATA = "tool_metadata"
TAGS = "tags"
LLM_PROFILE_ID = "llm_profile_id"
- USER_DATA = "user_data"
+ CUSTOM_DATA = "custom_data"
class ToolMetadataKey:
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py b/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py
index aed528be5..9bc46f703 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py
@@ -102,7 +102,7 @@ class ExecutionFileHandler:
source_hash: str,
tags: list[str],
llm_profile_id: str | None = None,
- user_data: dict[str, Any] | None = None,
+ custom_data: dict[str, Any] | None = None,
) -> None:
"""Creating metadata for workflow. This method is responsible for
creating metadata for the workflow. It takes the input file path and
@@ -142,9 +142,9 @@ class ExecutionFileHandler:
if llm_profile_id:
content[MetaDataKey.LLM_PROFILE_ID] = llm_profile_id
- # Add user_data to metadata if provided
- if user_data:
- content[MetaDataKey.USER_DATA] = user_data
+ # Add custom_data to metadata if provided
+ if custom_data:
+ content[MetaDataKey.CUSTOM_DATA] = custom_data
file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
file_storage = file_system.get_file_storage()