Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

[FIX]Passing down execution source (#1124)

harini-venkataramancefcb2cf9f81
diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_ide_base_tool.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_ide_base_tool.py
index e72a7ca76..2ce14016c 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/prompt_ide_base_tool.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_ide_base_tool.py
@@ -8,6 +8,11 @@ from unstract.sdk.tool.stream import StreamMixin
from backend.constants import FeatureFlag
from unstract.flags.feature_flag import check_feature_flag_status
+if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):
+ from unstract.sdk.file_storage.constants import StorageType
+ from unstract.sdk.file_storage.env_helper import EnvHelper
+ from utils.file_storage.constants import FileStorageKeys
+
class PromptIdeBaseTool(StreamMixin):
def __init__(self, log_level: LogLevel = LogLevel.INFO, org_id: str = "") -> None:
@@ -21,10 +26,10 @@ class PromptIdeBaseTool(StreamMixin):
self.org_id = org_id
self.workflow_filestorage = None
if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):
- from unstract.filesystem import FileStorageType, FileSystem
-
- file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
- self.workflow_filestorage = file_system.get_file_storage()
+ self.workflow_filestorage = EnvHelper.get_storage(
+ storage_type=StorageType.PERMANENT,
+ env_name=FileStorageKeys.PERMANENT_REMOTE_STORAGE,
+ )
super().__init__(log_level=log_level)
diff --git a/prompt-service/src/unstract/prompt_service/helper.py b/prompt-service/src/unstract/prompt_service/helper.py
index 3797fcb52..1f5385151 100644
--- a/prompt-service/src/unstract/prompt_service/helper.py
+++ b/prompt-service/src/unstract/prompt_service/helper.py
@@ -233,6 +233,7 @@ def construct_and_run_prompt(
prompt: str,
metadata: dict[str, Any],
file_path: str = "",
+ execution_source: Optional[str] = ExecutionSource.IDE.value,
) -> str:
platform_postamble = tool_settings.get(PSKeys.PLATFORM_POSTAMBLE, "")
summarize_as_source = tool_settings.get(PSKeys.SUMMARIZE_AS_SOURCE)
@@ -255,6 +256,7 @@ def construct_and_run_prompt(
prompt_type=output.get(PSKeys.TYPE, PSKeys.TEXT),
enable_highlight=enable_highlight,
file_path=file_path,
+ execution_source=execution_source,
)
@@ -314,7 +316,7 @@ def run_completion(
)
if execution_source == ExecutionSource.TOOL.value:
fs_instance = EnvHelper.get_storage(
- storage_type=StorageType.TEMPORARY,
+ storage_type=StorageType.SHARED_TEMPORARY,
env_name=FileStorageKeys.TEMPORARY_REMOTE_STORAGE,
)
highlight_data = highlight_data_plugin["entrypoint_cls"](
@@ -375,7 +377,7 @@ def extract_table(
)
if execution_source == ExecutionSource.TOOL.value:
fs_instance = EnvHelper.get_storage(
- storage_type=StorageType.TEMPORARY,
+ storage_type=StorageType.SHARED_TEMPORARY,
env_name=FileStorageKeys.TEMPORARY_REMOTE_STORAGE,
)
try:
@@ -435,7 +437,7 @@ def extract_line_item(
)
if execution_source == ExecutionSource.TOOL.value:
fs_instance = EnvHelper.get_storage(
- storage_type=StorageType.TEMPORARY,
+ storage_type=StorageType.SHARED_TEMPORARY,
env_name=FileStorageKeys.TEMPORARY_REMOTE_STORAGE,
)
diff --git a/prompt-service/src/unstract/prompt_service/main.py b/prompt-service/src/unstract/prompt_service/main.py
index 4cbcfa247..22ed8c9e4 100644
--- a/prompt-service/src/unstract/prompt_service/main.py
+++ b/prompt-service/src/unstract/prompt_service/main.py
@@ -1,7 +1,7 @@
import time
import traceback
from json import JSONDecodeError
-from typing import Any
+from typing import Any, Optional
from flask import json, jsonify, request
from llama_index.core.vector_stores import ExactMatchFilter, MetadataFilters
@@ -330,6 +330,7 @@ def prompt_processor() -> Any:
prompt="promptx",
metadata=metadata,
file_path=file_path,
+ execution_source=execution_source,
)
metadata[PSKeys.CONTEXT][output[PSKeys.NAME]] = get_cleaned_context(
context
@@ -360,6 +361,7 @@ def prompt_processor() -> Any:
vector_index=vector_index,
retrieval_type=retrieval_strategy,
metadata=metadata,
+ execution_source=execution_source,
)
metadata[PSKeys.CONTEXT][output[PSKeys.NAME]] = get_cleaned_context(
context
@@ -807,6 +809,7 @@ def run_retrieval( # type:ignore
vector_index,
retrieval_type: str,
metadata: dict[str, Any],
+ execution_source: Optional[str] = None,
) -> tuple[str, set[str]]:
context: set[str] = set()
prompt = output[PSKeys.PROMPTX]
@@ -869,6 +872,7 @@ def run_retrieval( # type:ignore
context="\n".join(context),
prompt="promptx",
metadata=metadata,
+ execution_source=execution_source,
)
return (answer, context)