Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

feat: Refactored prompt-service, avoids logging about variable replacement (#762)

Chandrasekharan Mb37cf213f1bd
diff --git a/backend/prompt_studio/prompt_studio_core/constants.py b/backend/prompt_studio/prompt_studio_core/constants.py
index 659270975..08e71bc36 100644
--- a/backend/prompt_studio/prompt_studio_core/constants.py
+++ b/backend/prompt_studio/prompt_studio_core/constants.py
@@ -110,14 +110,6 @@ class LogLevels:
RUN = "RUN"
-class LogLevel(Enum):
- DEBUG = "DEBUG"
- INFO = "INFO"
- WARN = "WARN"
- ERROR = "ERROR"
- FATAL = "FATAL"
-
-
class IndexingStatus(Enum):
PENDING_STATUS = "pending"
COMPLETED_STATUS = "completed"
diff --git a/backend/prompt_studio/prompt_studio_core/prompt_ide_base_tool.py b/backend/prompt_studio/prompt_studio_core/prompt_ide_base_tool.py
index 0c460a048..d26252bfb 100644
--- a/backend/prompt_studio/prompt_studio_core/prompt_ide_base_tool.py
+++ b/backend/prompt_studio/prompt_studio_core/prompt_ide_base_tool.py
@@ -1,7 +1,8 @@
import os
from platform_settings.platform_auth_service import PlatformAuthenticationService
-from prompt_studio.prompt_studio_core.constants import LogLevel, ToolStudioKeys
+from prompt_studio.prompt_studio_core.constants import ToolStudioKeys
+from unstract.sdk.constants import LogLevel
from unstract.sdk.tool.stream import StreamMixin
diff --git a/backend/prompt_studio/prompt_studio_core_v2/constants.py b/backend/prompt_studio/prompt_studio_core_v2/constants.py
index 73abc4bf9..5e6c47884 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/constants.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/constants.py
@@ -109,14 +109,6 @@ class LogLevels:
RUN = "RUN"
-class LogLevel(Enum):
- DEBUG = "DEBUG"
- INFO = "INFO"
- WARN = "WARN"
- ERROR = "ERROR"
- FATAL = "FATAL"
-
-
class IndexingStatus(Enum):
PENDING_STATUS = "pending"
COMPLETED_STATUS = "completed"
diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_ide_base_tool.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_ide_base_tool.py
index d2b403974..d32828e3e 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/prompt_ide_base_tool.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_ide_base_tool.py
@@ -1,7 +1,8 @@
import os
from platform_settings_v2.platform_auth_service import PlatformAuthenticationService
-from prompt_studio.prompt_studio_core_v2.constants import LogLevel, ToolStudioKeys
+from prompt_studio.prompt_studio_core_v2.constants import ToolStudioKeys
+from unstract.sdk.constants import LogLevel
from unstract.sdk.tool.stream import StreamMixin
diff --git a/prompt-service/src/unstract/prompt_service/config.py b/prompt-service/src/unstract/prompt_service/config.py
index 9c5e0504c..ad9220a39 100644
--- a/prompt-service/src/unstract/prompt_service/config.py
+++ b/prompt-service/src/unstract/prompt_service/config.py
@@ -5,7 +5,7 @@ from os import environ as env
from dotenv import load_dotenv
from flask import Flask
from peewee import PostgresqlDatabase
-from unstract.prompt_service.constants import LogLevel
+from unstract.sdk.constants import LogLevel
load_dotenv()
@@ -40,7 +40,7 @@ def get_env_or_die(env_key: str) -> str:
def create_app() -> Flask:
app = Flask("prompt-service")
- log_level = env.get("LOG_LEVEL", LogLevel.WARN)
+ log_level = env.get("LOG_LEVEL", LogLevel.WARN.value)
if log_level == LogLevel.DEBUG.value:
app.logger.setLevel(logging.DEBUG)
elif log_level == LogLevel.INFO.value:
diff --git a/prompt-service/src/unstract/prompt_service/constants.py b/prompt-service/src/unstract/prompt_service/constants.py
index 7ff4ca965..6f5a73878 100644
--- a/prompt-service/src/unstract/prompt_service/constants.py
+++ b/prompt-service/src/unstract/prompt_service/constants.py
@@ -70,14 +70,6 @@ class PromptServiceContants:
TEXT = "text"
-class LogLevel(Enum):
- DEBUG = "DEBUG"
- INFO = "INFO"
- WARN = "WARN"
- ERROR = "ERROR"
- FATAL = "FATAL"
-
-
class RunLevel(Enum):
"""Different stages of prompt execution.
diff --git a/prompt-service/src/unstract/prompt_service/main.py b/prompt-service/src/unstract/prompt_service/main.py
index 369bda80f..50a920ef5 100644
--- a/prompt-service/src/unstract/prompt_service/main.py
+++ b/prompt-service/src/unstract/prompt_service/main.py
@@ -1,6 +1,5 @@
import time
import traceback
-from enum import Enum
from json import JSONDecodeError
from typing import Any, Optional
@@ -22,6 +21,7 @@ from unstract.prompt_service.helper import (
run_completion,
)
from unstract.prompt_service.prompt_ide_base_tool import PromptServiceBaseTool
+from unstract.prompt_service.utils.log import publish_log
from unstract.prompt_service.variable_extractor.base import VariableExtractor
from unstract.sdk.constants import LogLevel
from unstract.sdk.embedding import Embedding
@@ -31,15 +31,13 @@ from unstract.sdk.llm import LLM
from unstract.sdk.vector_db import VectorDB
from werkzeug.exceptions import HTTPException
-from unstract.core.pubsub_helper import LogPublisher
-
USE_UNSTRACT_PROMPT = True
MAX_RETRIES = 3
NO_CONTEXT_ERROR = (
"Couldn't fetch context from vector DB. "
"This happens usually due to a delay by the Vector DB "
- "provider to confirm writes to db. "
+ "provider to confirm writes to DB. "
"Please try again after some time"
)
@@ -72,19 +70,6 @@ def log_response_info(response):
return response
-def _publish_log(
- log_events_id: str,
- component: dict[str, str],
- level: Enum,
- state: Enum,
- message: str,
-) -> None:
- LogPublisher.publish(
- log_events_id,
- LogPublisher.log_prompt(component, level.value, state.value, message),
- )
-
-
def authentication_middleware(func: Any) -> Any:
def wrapper(*args: Any, **kwargs: Any) -> Any:
token = AuthenticationMiddleware.get_token_from_auth_header(request)
@@ -109,7 +94,8 @@ def prompt_processor() -> Any:
if not payload:
raise NoPayloadError
tool_settings = payload.get(PSKeys.TOOL_SETTINGS, {})
- outputs = payload.get(PSKeys.OUTPUTS, [])
+ # TODO: Rename "outputs" to "prompts" in payload
+ prompts = payload.get(PSKeys.OUTPUTS, [])
tool_id: str = payload.get(PSKeys.TOOL_ID, "")
run_id: str = payload.get(PSKeys.RUN_ID, "")
file_hash = payload.get(PSKeys.FILE_HASH)
@@ -122,75 +108,35 @@ def prompt_processor() -> Any:
PSKeys.CONTEXT: {},
}
variable_names: list[str] = []
- _publish_log(
+ publish_log(
log_events_id,
{"tool_id": tool_id, "run_id": run_id, "doc_name": doc_name},
LogLevel.DEBUG,
RunLevel.RUN,
- f"Preparing to execute {len(outputs)} prompt(s)",
+ f"Preparing to execute {len(prompts)} prompt(s)",
)
- for output in outputs: # type:ignore
+ # TODO: Rename "output" to "prompt"
+ for output in prompts: # type:ignore
variable_names.append(output[PSKeys.NAME])
- for output in outputs: # type:ignore
+ for output in prompts: # type:ignore
prompt_name = output[PSKeys.NAME]
- promptx = output[PSKeys.PROMPT]
+ prompt_text = output[PSKeys.PROMPT]
chunk_size = output[PSKeys.CHUNK_SIZE]
- util = PromptServiceBaseTool(log_level=LogLevel.INFO, platform_key=platform_key)
+ util = PromptServiceBaseTool(platform_key=platform_key)
index = Index(tool=util)
- app.logger.info(f"[{tool_id}] Replacing variables in prompt : {prompt_name}")
- _publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.DEBUG,
- RunLevel.RUN,
- "Replacing variables in prompt",
- )
- try:
- variable_map = output[PSKeys.VARIABLE_MAP]
- promptx = VariableExtractor.execute_variable_replacement(
- prompt=promptx, variable_map=variable_map
- )
- app.logger.info(f"[{tool_id}] Prompt after variable replacement: {promptx}")
- _publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.DEBUG,
- RunLevel.RUN,
- f"Prompt after variable replacement:{promptx} ",
- )
- except KeyError:
- # Executed incase of structured tool and
- # APIs where we do not set the variable map
- promptx = VariableExtractor.execute_variable_replacement(
- prompt=promptx, variable_map=structured_output
- )
- app.logger.info(f"[{tool_id}] Prompt after variable replacement: {promptx}")
- _publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.DEBUG,
- RunLevel.RUN,
- f"Prompt after variable replacement:{promptx} ",
+ if VariableExtractor.is_variables_present(prompt_text=prompt_text):
+ prompt_text = VariableExtractor.replace_variables_in_prompt(
+ log_events_id=log_events_id,
+ tool_id=tool_id,
+ prompt_name=prompt_name,
+ doc_name=doc_name,
+ prompt=output,
)
- except APIError as api_error:
- raise api_error
app.logger.info(f"[{tool_id}] Executing prompt: {prompt_name}")
- _publish_log(
+ publish_log(
log_events_id,
{
"tool_id": tool_id,
@@ -206,7 +152,7 @@ def prompt_processor() -> Any:
# The variables are in the form %variable_name%
output[PSKeys.PROMPTX] = extract_variable(
- structured_output, variable_names, output, promptx
+ structured_output, variable_names, output, prompt_text
)
doc_id = index.generate_index_key(
@@ -217,7 +163,7 @@ def prompt_processor() -> Any:
chunk_size=str(output[PSKeys.CHUNK_SIZE]),
chunk_overlap=str(output[PSKeys.CHUNK_OVERLAP]),
)
- _publish_log(
+ publish_log(
log_events_id,
{
"tool_id": tool_id,
@@ -255,7 +201,7 @@ def prompt_processor() -> Any:
except SdkError as e:
msg = f"Couldn't fetch adapter. {e}"
app.logger.error(msg)
- _publish_log(
+ publish_log(
log_events_id,
{
"tool_id": tool_id,
@@ -289,7 +235,7 @@ def prompt_processor() -> Any:
output[PSKeys.NAME],
str(api_error),
)
- _publish_log(
+ publish_log(
log_events_id,
{
"tool_id": tool_id,
@@ -334,7 +280,7 @@ def prompt_processor() -> Any:
app.logger.error(
f"{msg} {output[PSKeys.VECTOR_DB]} for doc_id {doc_id}"
)
- _publish_log(
+ publish_log(
log_events_id,
{
"tool_id": tool_id,
@@ -347,7 +293,7 @@ def prompt_processor() -> Any:
)
raise APIError(message=msg)
# TODO: Use vectorDB name when available
- _publish_log(
+ publish_log(
log_events_id,
{
"tool_id": tool_id,
@@ -360,7 +306,7 @@ def prompt_processor() -> Any:
)
if chunk_size == 0:
- _publish_log(
+ publish_log(
log_events_id,
{
"tool_id": tool_id,
@@ -384,7 +330,7 @@ def prompt_processor() -> Any:
)
else:
answer = "NA"
- _publish_log(
+ publish_log(
log_events_id,
{
"tool_id": tool_id,
@@ -418,7 +364,7 @@ def prompt_processor() -> Any:
retrieval_strategy,
)
- _publish_log(
+ publish_log(
log_events_id,
{
"tool_id": tool_id,
@@ -430,7 +376,7 @@ def prompt_processor() -> Any:
"Retrieved context from adapter",
)
- _publish_log(
+ publish_log(
log_events_id,
{
"tool_id": tool_id,
@@ -555,7 +501,7 @@ def prompt_processor() -> Any:
challenge_plugin: dict[str, Any] = plugins.get(PSKeys.CHALLENGE, {})
try:
if challenge_plugin:
- _publish_log(
+ publish_log(
log_events_id,
{
"tool_id": tool_id,
@@ -599,7 +545,7 @@ def prompt_processor() -> Any:
output[PSKeys.NAME],
str(e),
)
- _publish_log(
+ publish_log(
log_events_id,
{
"tool_id": tool_id,
@@ -620,7 +566,7 @@ def prompt_processor() -> Any:
):
eval_plugin: dict[str, Any] = plugins.get("evaluation", {})
if eval_plugin:
- _publish_log(
+ publish_log(
log_events_id,
{
"tool_id": tool_id,
@@ -648,7 +594,7 @@ def prompt_processor() -> Any:
app.logger.error(
f"Failed to evaluate prompt {output[PSKeys.NAME]}: {str(e)}"
)
- _publish_log(
+ publish_log(
log_events_id,
{
"tool_id": tool_id,
@@ -660,7 +606,7 @@ def prompt_processor() -> Any:
"Error while evaluation",
)
else:
- _publish_log(
+ publish_log(
log_events_id,
{
"tool_id": tool_id,
@@ -677,7 +623,7 @@ def prompt_processor() -> Any:
)
finally:
vector_db.close()
- _publish_log(
+ publish_log(
log_events_id,
{"tool_id": tool_id, "doc_name": doc_name},
LogLevel.INFO,
@@ -700,7 +646,7 @@ def prompt_processor() -> Any:
if isinstance(v1, str) and v1.lower() == "na":
v[k1] = None
- _publish_log(
+ publish_log(
log_events_id,
{"tool_id": tool_id, "doc_name": doc_name},
LogLevel.INFO,
diff --git a/prompt-service/src/unstract/prompt_service/prompt_ide_base_tool.py b/prompt-service/src/unstract/prompt_service/prompt_ide_base_tool.py
index 346df91a0..42a65c94e 100644
--- a/prompt-service/src/unstract/prompt_service/prompt_ide_base_tool.py
+++ b/prompt-service/src/unstract/prompt_service/prompt_ide_base_tool.py
@@ -1,23 +1,38 @@
import os
+import warnings
+from typing import Optional
from flask import current_app
-from unstract.prompt_service.constants import LogLevel, PromptServiceContants
+from unstract.prompt_service.constants import PromptServiceContants
+from unstract.sdk.constants import LogLevel
from unstract.sdk.tool.stream import StreamMixin
+from unstract.sdk.utils.common_utils import PY_TO_UNSTRACT_LOG_LEVEL
class PromptServiceBaseTool(StreamMixin):
+ # TODO: Remove unused log_level arg
def __init__(
- self, log_level: LogLevel = LogLevel.INFO, platform_key: str = ""
+ self, log_level: Optional[LogLevel] = None, platform_key: str = ""
) -> None:
"""
Args:
- tool (UnstractAbstractTool): Instance of UnstractAbstractTool
+ tool (UnstractAbstractTool): Instance of UnstractAbstractTool.
+ Deprecated, will be removed in future versions
Notes:
- PLATFORM_SERVICE_API_KEY environment variable is required.
"""
- self.log_level = log_level
+ if log_level:
+ warnings.warn(
+ "The 'log_level' argument is deprecated and will be "
+ "removed in a future version.",
+ DeprecationWarning,
+ stacklevel=2, # ensures the warning points to the caller of the method
+ )
+ self.log_level = PY_TO_UNSTRACT_LOG_LEVEL.get(
+ current_app.logger.getEffectiveLevel(), LogLevel.INFO
+ )
self.platform_key = platform_key
- super().__init__(log_level=log_level)
+ super().__init__(log_level=self.log_level)
def get_env_or_die(self, env_key: str) -> str:
"""Returns the value of an env variable.
diff --git a/prompt-service/src/unstract/prompt_service/utils/log.py b/prompt-service/src/unstract/prompt_service/utils/log.py
new file mode 100644
index 000000000..73912630c
--- /dev/null
+++ b/prompt-service/src/unstract/prompt_service/utils/log.py
@@ -0,0 +1,27 @@
+from unstract.prompt_service.constants import RunLevel
+from unstract.sdk.constants import LogLevel
+
+from unstract.core.pubsub_helper import LogPublisher
+
+
+def publish_log(
+ log_events_id: str,
+ component: dict[str, str],
+ level: LogLevel,
+ state: RunLevel,
+ message: str,
+) -> None:
+ """Publishes a log to the web socket.
+
+ Args:
+ log_events_id (str): UUID for the connection
+ component (dict[str, str]): Dict of tool_id, doc_name and prompt_name to
+ log context
+ level (LogLevel): Log level, one of INFO, WARNING, DEBUG, ERROR
+ state (RunLevel): Run level, one of EVAL, RUN, CHALLENGE, TABLE_EXTRACTION
+ message (str): Message to log
+ """
+ LogPublisher.publish(
+ log_events_id,
+ LogPublisher.log_prompt(component, level.value, state.value, message),
+ )
diff --git a/prompt-service/src/unstract/prompt_service/variable_extractor/base.py b/prompt-service/src/unstract/prompt_service/variable_extractor/base.py
index 99b234e90..e0a44a937 100644
--- a/prompt-service/src/unstract/prompt_service/variable_extractor/base.py
+++ b/prompt-service/src/unstract/prompt_service/variable_extractor/base.py
@@ -1,5 +1,11 @@
from typing import Any
+from flask import current_app as app
+from unstract.prompt_service.constants import PromptServiceContants as PSKeys
+from unstract.prompt_service.constants import RunLevel
+from unstract.prompt_service.utils.log import publish_log
+from unstract.sdk.constants import LogLevel
+
from .constants import VariableType
from .prompt_variable_service import VariableService
@@ -7,19 +13,97 @@ from .prompt_variable_service import VariableService
class VariableExtractor:
@staticmethod
- def execute_variable_replacement(prompt: str, variable_map: dict[str, Any]) -> str:
+ def execute_variable_replacement(
+ prompt_text: str, variable_map: dict[str, Any]
+ ) -> str:
variables: list[str] = VariableService.extract_variables_from_prompt(
- prompt=prompt
+ prompt=prompt_text
)
for variable in variables:
variable_type = VariableService.identify_variable_type(variable=variable)
if variable_type == VariableType.STATIC:
- prompt = VariableService.replace_static_variable(
- prompt=prompt, structured_output=variable_map, variable=variable
+ prompt_text = VariableService.replace_static_variable(
+ prompt=prompt_text,
+ structured_output=variable_map,
+ variable=variable,
)
if variable_type == VariableType.DYNAMIC:
- prompt = VariableService.replace_dynamic_variable(
- prompt=prompt, variable=variable, structured_output=variable_map
+ prompt_text = VariableService.replace_dynamic_variable(
+ prompt=prompt_text,
+ variable=variable,
+ structured_output=variable_map,
)
- return prompt
+ return prompt_text
+
+ @staticmethod
+ def is_variables_present(prompt_text: str) -> bool:
+ """Determines if variables are present in the prompt.
+
+ Args:
+ prompt (str): Prompt to check
+
+ Returns:
+ bool: True if variables are presentm else False
+ """
+ return bool(len(VariableService.extract_variables_from_prompt(prompt_text)))
+
+ @staticmethod
+ def replace_variables_in_prompt(
+ prompt: dict[str, Any],
+ structured_output: dict[str, Any],
+ log_events_id: str,
+ tool_id: str,
+ prompt_name: str,
+ doc_name: str,
+ ) -> str:
+ """Replaces variables in prompt.
+
+ Args:
+ log_events_id (str): UUID for the WS communication
+ tool_id (str): UUID for the prompt studio project
+ prompt_name (str): Name of the prompt being run
+ doc_name (str): Name of the document being run with
+ prompt (dict[str, Any]): Dict representing the prompt card
+ Returns:
+ prompt_text (str): Prompt with variables replaced
+ """
+ app.logger.info(f"[{tool_id}] Replacing variables in prompt : {prompt_name}")
+ publish_log(
+ log_events_id,
+ {
+ "tool_id": tool_id,
+ "prompt_key": prompt_name,
+ "doc_name": doc_name,
+ },
+ LogLevel.DEBUG,
+ RunLevel.RUN,
+ "Replacing variables in prompt",
+ )
+ try:
+ variable_map = prompt[PSKeys.VARIABLE_MAP]
+ prompt_text = VariableExtractor.execute_variable_replacement(
+ prompt_text=prompt[PSKeys.PROMPT], variable_map=variable_map
+ )
+ except KeyError:
+ # Executed incase of structured tool and
+ # APIs where we do not set the variable map
+ prompt_text = VariableExtractor.execute_variable_replacement(
+ prompt_text=prompt_text, variable_map=structured_output
+ )
+ finally:
+ app.logger.info(
+ f"[{tool_id}] Prompt after variable replacement: {prompt_text}"
+ )
+ publish_log(
+ log_events_id,
+ {
+ "tool_id": tool_id,
+ "prompt_key": prompt_name,
+ "doc_name": doc_name,
+ },
+ LogLevel.DEBUG,
+ RunLevel.RUN,
+ f"Prompt after variable replacement:{prompt_text} ",
+ )
+ return prompt_text
diff --git a/prompt-service/src/unstract/prompt_service/variable_extractor/prompt_variable_service.py b/prompt-service/src/unstract/prompt_service/variable_extractor/prompt_variable_service.py
index 68dac40c4..d27f4525a 100644
--- a/prompt-service/src/unstract/prompt_service/variable_extractor/prompt_variable_service.py
+++ b/prompt-service/src/unstract/prompt_service/variable_extractor/prompt_variable_service.py
@@ -1,6 +1,7 @@
import json
import logging
import re
+from functools import lru_cache
from typing import Any
from unstract.prompt_service.utils.request import HTTPMethod, make_http_request
@@ -96,9 +97,10 @@ class VariableService:
return replaced_prompt
@staticmethod
- def extract_variables_from_prompt(prompt: str) -> list[str]:
+ @lru_cache(maxsize=128)
+ def extract_variables_from_prompt(prompt_text: str) -> list[str]:
variable: list[str] = []
- variable = re.findall(VariableConstants.VARIABLE_REGEX, prompt)
+ variable = re.findall(VariableConstants.VARIABLE_REGEX, prompt_text)
return variable
@staticmethod