LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
feat: Refactored prompt-service, avoids logging about variable replacement (#762)
Chandrasekharan Mb37cf213f1bd
diff --git a/backend/prompt_studio/prompt_studio_core/constants.py b/backend/prompt_studio/prompt_studio_core/constants.pyindex 659270975..08e71bc36 100644--- a/backend/prompt_studio/prompt_studio_core/constants.py+++ b/backend/prompt_studio/prompt_studio_core/constants.py@@ -110,14 +110,6 @@ class LogLevels:RUN = "RUN"-class LogLevel(Enum):- DEBUG = "DEBUG"- INFO = "INFO"- WARN = "WARN"- ERROR = "ERROR"- FATAL = "FATAL"--class IndexingStatus(Enum):PENDING_STATUS = "pending"COMPLETED_STATUS = "completed"diff --git a/backend/prompt_studio/prompt_studio_core/prompt_ide_base_tool.py b/backend/prompt_studio/prompt_studio_core/prompt_ide_base_tool.pyindex 0c460a048..d26252bfb 100644--- a/backend/prompt_studio/prompt_studio_core/prompt_ide_base_tool.py+++ b/backend/prompt_studio/prompt_studio_core/prompt_ide_base_tool.py@@ -1,7 +1,8 @@import osfrom platform_settings.platform_auth_service import PlatformAuthenticationService-from prompt_studio.prompt_studio_core.constants import LogLevel, ToolStudioKeys+from prompt_studio.prompt_studio_core.constants import ToolStudioKeys+from unstract.sdk.constants import LogLevelfrom unstract.sdk.tool.stream import StreamMixindiff --git a/backend/prompt_studio/prompt_studio_core_v2/constants.py b/backend/prompt_studio/prompt_studio_core_v2/constants.pyindex 73abc4bf9..5e6c47884 100644--- a/backend/prompt_studio/prompt_studio_core_v2/constants.py+++ b/backend/prompt_studio/prompt_studio_core_v2/constants.py@@ -109,14 +109,6 @@ class LogLevels:RUN = "RUN"-class LogLevel(Enum):- DEBUG = "DEBUG"- INFO = "INFO"- WARN = "WARN"- ERROR = "ERROR"- FATAL = "FATAL"--class IndexingStatus(Enum):PENDING_STATUS = "pending"COMPLETED_STATUS = "completed"diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_ide_base_tool.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_ide_base_tool.pyindex d2b403974..d32828e3e 100644--- a/backend/prompt_studio/prompt_studio_core_v2/prompt_ide_base_tool.py+++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_ide_base_tool.py@@ -1,7 +1,8 @@import osfrom platform_settings_v2.platform_auth_service import PlatformAuthenticationService-from prompt_studio.prompt_studio_core_v2.constants import LogLevel, ToolStudioKeys+from prompt_studio.prompt_studio_core_v2.constants import ToolStudioKeys+from unstract.sdk.constants import LogLevelfrom unstract.sdk.tool.stream import StreamMixindiff --git a/prompt-service/src/unstract/prompt_service/config.py b/prompt-service/src/unstract/prompt_service/config.pyindex 9c5e0504c..ad9220a39 100644--- a/prompt-service/src/unstract/prompt_service/config.py+++ b/prompt-service/src/unstract/prompt_service/config.py@@ -5,7 +5,7 @@ from os import environ as envfrom dotenv import load_dotenvfrom flask import Flaskfrom peewee import PostgresqlDatabase-from unstract.prompt_service.constants import LogLevel+from unstract.sdk.constants import LogLevelload_dotenv()@@ -40,7 +40,7 @@ def get_env_or_die(env_key: str) -> str:def create_app() -> Flask:app = Flask("prompt-service")- log_level = env.get("LOG_LEVEL", LogLevel.WARN)+ log_level = env.get("LOG_LEVEL", LogLevel.WARN.value)if log_level == LogLevel.DEBUG.value:app.logger.setLevel(logging.DEBUG)elif log_level == LogLevel.INFO.value:diff --git a/prompt-service/src/unstract/prompt_service/constants.py b/prompt-service/src/unstract/prompt_service/constants.pyindex 7ff4ca965..6f5a73878 100644--- a/prompt-service/src/unstract/prompt_service/constants.py+++ b/prompt-service/src/unstract/prompt_service/constants.py@@ -70,14 +70,6 @@ class PromptServiceContants:TEXT = "text"-class LogLevel(Enum):- DEBUG = "DEBUG"- INFO = "INFO"- WARN = "WARN"- ERROR = "ERROR"- FATAL = "FATAL"--class RunLevel(Enum):"""Different stages of prompt execution.diff --git a/prompt-service/src/unstract/prompt_service/main.py b/prompt-service/src/unstract/prompt_service/main.pyindex 369bda80f..50a920ef5 100644--- a/prompt-service/src/unstract/prompt_service/main.py+++ b/prompt-service/src/unstract/prompt_service/main.py@@ -1,6 +1,5 @@import timeimport traceback-from enum import Enumfrom json import JSONDecodeErrorfrom typing import Any, Optional@@ -22,6 +21,7 @@ from unstract.prompt_service.helper import (run_completion,)from unstract.prompt_service.prompt_ide_base_tool import PromptServiceBaseTool+from unstract.prompt_service.utils.log import publish_logfrom unstract.prompt_service.variable_extractor.base import VariableExtractorfrom unstract.sdk.constants import LogLevelfrom unstract.sdk.embedding import Embedding@@ -31,15 +31,13 @@ from unstract.sdk.llm import LLMfrom unstract.sdk.vector_db import VectorDBfrom werkzeug.exceptions import HTTPException-from unstract.core.pubsub_helper import LogPublisher-USE_UNSTRACT_PROMPT = TrueMAX_RETRIES = 3NO_CONTEXT_ERROR = ("Couldn't fetch context from vector DB. ""This happens usually due to a delay by the Vector DB "- "provider to confirm writes to db. "+ "provider to confirm writes to DB. ""Please try again after some time")@@ -72,19 +70,6 @@ def log_response_info(response):return response-def _publish_log(- log_events_id: str,- component: dict[str, str],- level: Enum,- state: Enum,- message: str,-) -> None:- LogPublisher.publish(- log_events_id,- LogPublisher.log_prompt(component, level.value, state.value, message),- )--def authentication_middleware(func: Any) -> Any:def wrapper(*args: Any, **kwargs: Any) -> Any:token = AuthenticationMiddleware.get_token_from_auth_header(request)@@ -109,7 +94,8 @@ def prompt_processor() -> Any:if not payload:raise NoPayloadErrortool_settings = payload.get(PSKeys.TOOL_SETTINGS, {})- outputs = payload.get(PSKeys.OUTPUTS, [])+ # TODO: Rename "outputs" to "prompts" in payload+ prompts = payload.get(PSKeys.OUTPUTS, [])tool_id: str = payload.get(PSKeys.TOOL_ID, "")run_id: str = payload.get(PSKeys.RUN_ID, "")file_hash = payload.get(PSKeys.FILE_HASH)@@ -122,75 +108,35 @@ def prompt_processor() -> Any:PSKeys.CONTEXT: {},}variable_names: list[str] = []- _publish_log(+ publish_log(log_events_id,{"tool_id": tool_id, "run_id": run_id, "doc_name": doc_name},LogLevel.DEBUG,RunLevel.RUN,- f"Preparing to execute {len(outputs)} prompt(s)",+ f"Preparing to execute {len(prompts)} prompt(s)",)- for output in outputs: # type:ignore+ # TODO: Rename "output" to "prompt"+ for output in prompts: # type:ignorevariable_names.append(output[PSKeys.NAME])- for output in outputs: # type:ignore+ for output in prompts: # type:ignoreprompt_name = output[PSKeys.NAME]- promptx = output[PSKeys.PROMPT]+ prompt_text = output[PSKeys.PROMPT]chunk_size = output[PSKeys.CHUNK_SIZE]- util = PromptServiceBaseTool(log_level=LogLevel.INFO, platform_key=platform_key)+ util = PromptServiceBaseTool(platform_key=platform_key)index = Index(tool=util)- app.logger.info(f"[{tool_id}] Replacing variables in prompt : {prompt_name}")- _publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.DEBUG,- RunLevel.RUN,- "Replacing variables in prompt",- )- try:- variable_map = output[PSKeys.VARIABLE_MAP]- promptx = VariableExtractor.execute_variable_replacement(- prompt=promptx, variable_map=variable_map- )- app.logger.info(f"[{tool_id}] Prompt after variable replacement: {promptx}")- _publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.DEBUG,- RunLevel.RUN,- f"Prompt after variable replacement:{promptx} ",- )- except KeyError:- # Executed incase of structured tool and- # APIs where we do not set the variable map- promptx = VariableExtractor.execute_variable_replacement(- prompt=promptx, variable_map=structured_output- )- app.logger.info(f"[{tool_id}] Prompt after variable replacement: {promptx}")- _publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.DEBUG,- RunLevel.RUN,- f"Prompt after variable replacement:{promptx} ",+ if VariableExtractor.is_variables_present(prompt_text=prompt_text):+ prompt_text = VariableExtractor.replace_variables_in_prompt(+ log_events_id=log_events_id,+ tool_id=tool_id,+ prompt_name=prompt_name,+ doc_name=doc_name,+ prompt=output,)- except APIError as api_error:- raise api_errorapp.logger.info(f"[{tool_id}] Executing prompt: {prompt_name}")- _publish_log(+ publish_log(log_events_id,{"tool_id": tool_id,@@ -206,7 +152,7 @@ def prompt_processor() -> Any:# The variables are in the form %variable_name%output[PSKeys.PROMPTX] = extract_variable(- structured_output, variable_names, output, promptx+ structured_output, variable_names, output, prompt_text)doc_id = index.generate_index_key(@@ -217,7 +163,7 @@ def prompt_processor() -> Any:chunk_size=str(output[PSKeys.CHUNK_SIZE]),chunk_overlap=str(output[PSKeys.CHUNK_OVERLAP]),)- _publish_log(+ publish_log(log_events_id,{"tool_id": tool_id,@@ -255,7 +201,7 @@ def prompt_processor() -> Any:except SdkError as e:msg = f"Couldn't fetch adapter. {e}"app.logger.error(msg)- _publish_log(+ publish_log(log_events_id,{"tool_id": tool_id,@@ -289,7 +235,7 @@ def prompt_processor() -> Any:output[PSKeys.NAME],str(api_error),)- _publish_log(+ publish_log(log_events_id,{"tool_id": tool_id,@@ -334,7 +280,7 @@ def prompt_processor() -> Any:app.logger.error(f"{msg} {output[PSKeys.VECTOR_DB]} for doc_id {doc_id}")- _publish_log(+ publish_log(log_events_id,{"tool_id": tool_id,@@ -347,7 +293,7 @@ def prompt_processor() -> Any:)raise APIError(message=msg)# TODO: Use vectorDB name when available- _publish_log(+ publish_log(log_events_id,{"tool_id": tool_id,@@ -360,7 +306,7 @@ def prompt_processor() -> Any:)if chunk_size == 0:- _publish_log(+ publish_log(log_events_id,{"tool_id": tool_id,@@ -384,7 +330,7 @@ def prompt_processor() -> Any:)else:answer = "NA"- _publish_log(+ publish_log(log_events_id,{"tool_id": tool_id,@@ -418,7 +364,7 @@ def prompt_processor() -> Any:retrieval_strategy,)- _publish_log(+ publish_log(log_events_id,{"tool_id": tool_id,@@ -430,7 +376,7 @@ def prompt_processor() -> Any:"Retrieved context from adapter",)- _publish_log(+ publish_log(log_events_id,{"tool_id": tool_id,@@ -555,7 +501,7 @@ def prompt_processor() -> Any:challenge_plugin: dict[str, Any] = plugins.get(PSKeys.CHALLENGE, {})try:if challenge_plugin:- _publish_log(+ publish_log(log_events_id,{"tool_id": tool_id,@@ -599,7 +545,7 @@ def prompt_processor() -> Any:output[PSKeys.NAME],str(e),)- _publish_log(+ publish_log(log_events_id,{"tool_id": tool_id,@@ -620,7 +566,7 @@ def prompt_processor() -> Any:):eval_plugin: dict[str, Any] = plugins.get("evaluation", {})if eval_plugin:- _publish_log(+ publish_log(log_events_id,{"tool_id": tool_id,@@ -648,7 +594,7 @@ def prompt_processor() -> Any:app.logger.error(f"Failed to evaluate prompt {output[PSKeys.NAME]}: {str(e)}")- _publish_log(+ publish_log(log_events_id,{"tool_id": tool_id,@@ -660,7 +606,7 @@ def prompt_processor() -> Any:"Error while evaluation",)else:- _publish_log(+ publish_log(log_events_id,{"tool_id": tool_id,@@ -677,7 +623,7 @@ def prompt_processor() -> Any:)finally:vector_db.close()- _publish_log(+ publish_log(log_events_id,{"tool_id": tool_id, "doc_name": doc_name},LogLevel.INFO,@@ -700,7 +646,7 @@ def prompt_processor() -> Any:if isinstance(v1, str) and v1.lower() == "na":v[k1] = None- _publish_log(+ publish_log(log_events_id,{"tool_id": tool_id, "doc_name": doc_name},LogLevel.INFO,diff --git a/prompt-service/src/unstract/prompt_service/prompt_ide_base_tool.py b/prompt-service/src/unstract/prompt_service/prompt_ide_base_tool.pyindex 346df91a0..42a65c94e 100644--- a/prompt-service/src/unstract/prompt_service/prompt_ide_base_tool.py+++ b/prompt-service/src/unstract/prompt_service/prompt_ide_base_tool.py@@ -1,23 +1,38 @@import os+import warnings+from typing import Optionalfrom flask import current_app-from unstract.prompt_service.constants import LogLevel, PromptServiceContants+from unstract.prompt_service.constants import PromptServiceContants+from unstract.sdk.constants import LogLevelfrom unstract.sdk.tool.stream import StreamMixin+from unstract.sdk.utils.common_utils import PY_TO_UNSTRACT_LOG_LEVELclass PromptServiceBaseTool(StreamMixin):+ # TODO: Remove unused log_level argdef __init__(- self, log_level: LogLevel = LogLevel.INFO, platform_key: str = ""+ self, log_level: Optional[LogLevel] = None, platform_key: str = "") -> None:"""Args:- tool (UnstractAbstractTool): Instance of UnstractAbstractTool+ tool (UnstractAbstractTool): Instance of UnstractAbstractTool.+ Deprecated, will be removed in future versionsNotes:- PLATFORM_SERVICE_API_KEY environment variable is required."""- self.log_level = log_level+ if log_level:+ warnings.warn(+ "The 'log_level' argument is deprecated and will be "+ "removed in a future version.",+ DeprecationWarning,+ stacklevel=2, # ensures the warning points to the caller of the method+ )+ self.log_level = PY_TO_UNSTRACT_LOG_LEVEL.get(+ current_app.logger.getEffectiveLevel(), LogLevel.INFO+ )self.platform_key = platform_key- super().__init__(log_level=log_level)+ super().__init__(log_level=self.log_level)def get_env_or_die(self, env_key: str) -> str:"""Returns the value of an env variable.diff --git a/prompt-service/src/unstract/prompt_service/utils/log.py b/prompt-service/src/unstract/prompt_service/utils/log.pynew file mode 100644index 000000000..73912630c--- /dev/null+++ b/prompt-service/src/unstract/prompt_service/utils/log.py@@ -0,0 +1,27 @@+from unstract.prompt_service.constants import RunLevel+from unstract.sdk.constants import LogLevel++from unstract.core.pubsub_helper import LogPublisher+++def publish_log(+ log_events_id: str,+ component: dict[str, str],+ level: LogLevel,+ state: RunLevel,+ message: str,+) -> None:+ """Publishes a log to the web socket.++ Args:+ log_events_id (str): UUID for the connection+ component (dict[str, str]): Dict of tool_id, doc_name and prompt_name to+ log context+ level (LogLevel): Log level, one of INFO, WARNING, DEBUG, ERROR+ state (RunLevel): Run level, one of EVAL, RUN, CHALLENGE, TABLE_EXTRACTION+ message (str): Message to log+ """+ LogPublisher.publish(+ log_events_id,+ LogPublisher.log_prompt(component, level.value, state.value, message),+ )diff --git a/prompt-service/src/unstract/prompt_service/variable_extractor/base.py b/prompt-service/src/unstract/prompt_service/variable_extractor/base.pyindex 99b234e90..e0a44a937 100644--- a/prompt-service/src/unstract/prompt_service/variable_extractor/base.py+++ b/prompt-service/src/unstract/prompt_service/variable_extractor/base.py@@ -1,5 +1,11 @@from typing import Any+from flask import current_app as app+from unstract.prompt_service.constants import PromptServiceContants as PSKeys+from unstract.prompt_service.constants import RunLevel+from unstract.prompt_service.utils.log import publish_log+from unstract.sdk.constants import LogLevel+from .constants import VariableTypefrom .prompt_variable_service import VariableService@@ -7,19 +13,97 @@ from .prompt_variable_service import VariableServiceclass VariableExtractor:@staticmethod- def execute_variable_replacement(prompt: str, variable_map: dict[str, Any]) -> str:+ def execute_variable_replacement(+ prompt_text: str, variable_map: dict[str, Any]+ ) -> str:variables: list[str] = VariableService.extract_variables_from_prompt(- prompt=prompt+ prompt=prompt_text)for variable in variables:variable_type = VariableService.identify_variable_type(variable=variable)if variable_type == VariableType.STATIC:- prompt = VariableService.replace_static_variable(- prompt=prompt, structured_output=variable_map, variable=variable+ prompt_text = VariableService.replace_static_variable(+ prompt=prompt_text,+ structured_output=variable_map,+ variable=variable,)if variable_type == VariableType.DYNAMIC:- prompt = VariableService.replace_dynamic_variable(- prompt=prompt, variable=variable, structured_output=variable_map+ prompt_text = VariableService.replace_dynamic_variable(+ prompt=prompt_text,+ variable=variable,+ structured_output=variable_map,)- return prompt+ return prompt_text++ @staticmethod+ def is_variables_present(prompt_text: str) -> bool:+ """Determines if variables are present in the prompt.++ Args:+ prompt (str): Prompt to check++ Returns:+ bool: True if variables are presentm else False+ """+ return bool(len(VariableService.extract_variables_from_prompt(prompt_text)))++ @staticmethod+ def replace_variables_in_prompt(+ prompt: dict[str, Any],+ structured_output: dict[str, Any],+ log_events_id: str,+ tool_id: str,+ prompt_name: str,+ doc_name: str,+ ) -> str:+ """Replaces variables in prompt.++ Args:+ log_events_id (str): UUID for the WS communication+ tool_id (str): UUID for the prompt studio project+ prompt_name (str): Name of the prompt being run+ doc_name (str): Name of the document being run with+ prompt (dict[str, Any]): Dict representing the prompt card+ Returns:+ prompt_text (str): Prompt with variables replaced+ """+ app.logger.info(f"[{tool_id}] Replacing variables in prompt : {prompt_name}")+ publish_log(+ log_events_id,+ {+ "tool_id": tool_id,+ "prompt_key": prompt_name,+ "doc_name": doc_name,+ },+ LogLevel.DEBUG,+ RunLevel.RUN,+ "Replacing variables in prompt",+ )+ try:+ variable_map = prompt[PSKeys.VARIABLE_MAP]+ prompt_text = VariableExtractor.execute_variable_replacement(+ prompt_text=prompt[PSKeys.PROMPT], variable_map=variable_map+ )+ except KeyError:+ # Executed incase of structured tool and+ # APIs where we do not set the variable map+ prompt_text = VariableExtractor.execute_variable_replacement(+ prompt_text=prompt_text, variable_map=structured_output+ )+ finally:+ app.logger.info(+ f"[{tool_id}] Prompt after variable replacement: {prompt_text}"+ )+ publish_log(+ log_events_id,+ {+ "tool_id": tool_id,+ "prompt_key": prompt_name,+ "doc_name": doc_name,+ },+ LogLevel.DEBUG,+ RunLevel.RUN,+ f"Prompt after variable replacement:{prompt_text} ",+ )+ return prompt_textdiff --git a/prompt-service/src/unstract/prompt_service/variable_extractor/prompt_variable_service.py b/prompt-service/src/unstract/prompt_service/variable_extractor/prompt_variable_service.pyindex 68dac40c4..d27f4525a 100644--- a/prompt-service/src/unstract/prompt_service/variable_extractor/prompt_variable_service.py+++ b/prompt-service/src/unstract/prompt_service/variable_extractor/prompt_variable_service.py@@ -1,6 +1,7 @@import jsonimport loggingimport re+from functools import lru_cachefrom typing import Anyfrom unstract.prompt_service.utils.request import HTTPMethod, make_http_request@@ -96,9 +97,10 @@ class VariableService:return replaced_prompt@staticmethod- def extract_variables_from_prompt(prompt: str) -> list[str]:+ @lru_cache(maxsize=128)+ def extract_variables_from_prompt(prompt_text: str) -> list[str]:variable: list[str] = []- variable = re.findall(VariableConstants.VARIABLE_REGEX, prompt)+ variable = re.findall(VariableConstants.VARIABLE_REGEX, prompt_text)return variable@staticmethod