Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

delta changes and enhancements in platform and prompt services (#538)

ali8a71ee652e5b
diff --git a/backend/sample.env b/backend/sample.env
index d75f48c92..79e034c96 100644
--- a/backend/sample.env
+++ b/backend/sample.env
@@ -140,6 +140,9 @@ CELERY_BROKER_URL = "redis://unstract-redis:6379"
# Indexing flag to prevent re-index
INDEXING_FLAG_TTL=1800
+# V2 Configurations
+DB_SCHEMA="unstract_v2"
+
# Notification Timeout in Seconds
NOTIFICATION_TIMEOUT=5
diff --git a/platform-service/sample.env b/platform-service/sample.env
index 0e533af8c..968ae2955 100644
--- a/platform-service/sample.env
+++ b/platform-service/sample.env
@@ -26,3 +26,6 @@ FLIPT_SERVICE_AVAILABLE=False
MODEL_PRICES_URL="https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"
MODEL_PRICES_TTL_IN_DAYS=7
MODEL_PRICES_FILE_PATH="/tmp/model_prices.json"
+
+# V2 Configurations
+DB_SCHEMA="unstract_v2"
diff --git a/platform-service/src/unstract/platform_service/controller/platform.py b/platform-service/src/unstract/platform_service/controller/platform.py
index 1daf15062..2f8aa992e 100644
--- a/platform-service/src/unstract/platform_service/controller/platform.py
+++ b/platform-service/src/unstract/platform_service/controller/platform.py
@@ -75,9 +75,15 @@ def get_organization_from_bearer_token(token: str) -> tuple[Optional[int], str]:
tuple[int, str]: organization uid and organization identifier
"""
if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):
- query = f"SELECT organization_id FROM {DBTableV2.PLATFORM_KEY} WHERE key=%s"
+ query = f"""
+ SELECT organization_id FROM "{Env.DB_SCHEMA}".{DBTableV2.PLATFORM_KEY}
+ WHERE key=%s
+ """
organization_uid: int = execute_query(query, (token,))
- query_org = f"SELECT organization_id FROM {DBTableV2.ORGANIZATION} WHERE id=%s"
+ query_org = f"""
+ SELECT organization_id FROM "{Env.DB_SCHEMA}".{DBTableV2.ORGANIZATION}
+ WHERE id=%s
+ """
organization_identifier: str = execute_query(query_org, (organization_uid,))
return organization_uid, organization_identifier
else:
@@ -102,10 +108,16 @@ def validate_bearer_token(token: Optional[str]) -> bool:
if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):
platform_key_table = DBTableV2.PLATFORM_KEY
+ query = f"""
+ SELECT * FROM \"{Env.DB_SCHEMA}\".{platform_key_table}
+ WHERE key = '{token}'
+ """
else:
platform_key_table = "account_platformkey"
+ query = f"""
+ SELECT * FROM {platform_key_table} WHERE key = '{token}'
+ """
- query = f"SELECT * FROM {platform_key_table} WHERE key = '{token}'"
cursor = be_db.execute_sql(query)
result_row = cursor.fetchone()
cursor.close()
@@ -241,7 +253,8 @@ def usage() -> Any:
current_time = datetime.now()
if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):
query = f"""
- INSERT INTO {DBTableV2.TOKEN_USAGE} (id, organization_id, workflow_id,
+ INSERT INTO \"{Env.DB_SCHEMA}\".{DBTableV2.TOKEN_USAGE} (
+ id, organization_id, workflow_id,
execution_id, adapter_instance_id, run_id, usage_type,
llm_usage_reason, model_name, embedding_tokens, prompt_tokens,
completion_tokens, total_tokens, cost_in_dollars, created_at, modified_at)
diff --git a/platform-service/src/unstract/platform_service/env.py b/platform-service/src/unstract/platform_service/env.py
index ba92d5e97..6d942c2f4 100644
--- a/platform-service/src/unstract/platform_service/env.py
+++ b/platform-service/src/unstract/platform_service/env.py
@@ -25,6 +25,7 @@ class Env:
APPLICATION_NAME = EnvManager.get_required_setting(
"APPLICATION_NAME", "unstract-platform-service"
)
+ DB_SCHEMA = EnvManager.get_required_setting("DB_SCHEMA", "unstract_v2")
EnvManager.raise_for_missing_envs()
diff --git a/platform-service/src/unstract/platform_service/helper/adapter_instance.py b/platform-service/src/unstract/platform_service/helper/adapter_instance.py
index c0d03dbc1..c6daa25b6 100644
--- a/platform-service/src/unstract/platform_service/helper/adapter_instance.py
+++ b/platform-service/src/unstract/platform_service/helper/adapter_instance.py
@@ -3,9 +3,12 @@ from typing import Any, Optional
from peewee import PostgresqlDatabase
from unstract.platform_service.constants import DBTableV2, FeatureFlag
from unstract.platform_service.exceptions import APIError
+from unstract.platform_service.utils import EnvManager
from unstract.flags.feature_flag import check_feature_flag_status
+DB_SCHEMA = EnvManager.get_required_setting("DB_SCHEMA", "unstract_v2")
+
class AdapterInstanceRequestHelper:
@staticmethod
@@ -28,7 +31,7 @@ class AdapterInstanceRequestHelper:
if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):
query = (
"SELECT id, adapter_id, adapter_metadata_b FROM "
- f"{DBTableV2.ADAPTER_INSTANCE} x "
+ f'"{DB_SCHEMA}".{DBTableV2.ADAPTER_INSTANCE} x '
f"WHERE id='{adapter_instance_id}' and "
f"organization_id='{organization_uid}'"
)
diff --git a/platform-service/src/unstract/platform_service/helper/prompt_studio.py b/platform-service/src/unstract/platform_service/helper/prompt_studio.py
index 6801792c1..e685909c2 100644
--- a/platform-service/src/unstract/platform_service/helper/prompt_studio.py
+++ b/platform-service/src/unstract/platform_service/helper/prompt_studio.py
@@ -3,9 +3,12 @@ from typing import Any
from peewee import PostgresqlDatabase
from unstract.platform_service.constants import DBTableV2, FeatureFlag
from unstract.platform_service.exceptions import APIError
+from unstract.platform_service.utils import EnvManager
from unstract.flags.feature_flag import check_feature_flag_status
+DB_SCHEMA = EnvManager.get_required_setting("DB_SCHEMA", "unstract_v2")
+
class PromptStudioRequestHelper:
@staticmethod
@@ -28,7 +31,7 @@ class PromptStudioRequestHelper:
query = (
"SELECT prompt_registry_id, tool_spec, "
"tool_metadata, tool_property FROM "
- f"{DBTableV2.PROMPT_STUDIO_REGISTRY} x "
+ f'"{DB_SCHEMA}".{DBTableV2.PROMPT_STUDIO_REGISTRY} x '
f"WHERE prompt_registry_id='{prompt_registry_id}'"
)
else:
diff --git a/prompt-service/sample.env b/prompt-service/sample.env
index 53f27dad2..697592d28 100644
--- a/prompt-service/sample.env
+++ b/prompt-service/sample.env
@@ -25,5 +25,9 @@ CELERY_BROKER_URL="redis://unstract-redis:6379"
EVALUATION_SERVER_IP=unstract-flipt
EVALUATION_SERVER_PORT=9000
PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+
+# V2 Configurations
+DB_SCHEMA="unstract_v2"
+
# Flipt Service
FLIPT_SERVICE_AVAILABLE=False
diff --git a/prompt-service/src/unstract/prompt_service/authentication_middleware.py b/prompt-service/src/unstract/prompt_service/authentication_middleware.py
index fbf03c37b..3162216cd 100644
--- a/prompt-service/src/unstract/prompt_service/authentication_middleware.py
+++ b/prompt-service/src/unstract/prompt_service/authentication_middleware.py
@@ -3,9 +3,13 @@ from typing import Any, Optional
from flask import Request, current_app
from unstract.prompt_service.config import db
from unstract.prompt_service.constants import DBTableV2, FeatureFlag
+from unstract.prompt_service.db_utils import DBUtils
+from unstract.prompt_service.env_manager import EnvLoader
from unstract.flags.feature_flag import check_feature_flag_status
+DB_SCHEMA = EnvLoader.get_env_or_die("DB_SCHEMA", "unstract_v2")
+
class AuthenticationMiddleware:
@@ -17,7 +21,7 @@ class AuthenticationMiddleware:
return False
if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):
- platform_key_table = DBTableV2.PLATFORM_KEY
+ platform_key_table = f'"{DB_SCHEMA}".{DBTableV2.PLATFORM_KEY}'
else:
platform_key_table = "account_platformkey"
@@ -75,11 +79,11 @@ class AuthenticationMiddleware:
organization_table = "account_organization"
query = f"SELECT organization_id FROM {platform_key_table} WHERE key='{token}'"
- organization = AuthenticationMiddleware.execute_query(query)
+ organization = DBUtils.execute_query(query)
query_org = (
f"SELECT schema_name FROM {organization_table} WHERE id='{organization}'"
)
- schema_name: str = AuthenticationMiddleware.execute_query(query_org)
+ schema_name: str = DBUtils.execute_query(query_org)
return schema_name
@staticmethod
diff --git a/prompt-service/src/unstract/prompt_service/db_utils.py b/prompt-service/src/unstract/prompt_service/db_utils.py
new file mode 100644
index 000000000..74372eafd
--- /dev/null
+++ b/prompt-service/src/unstract/prompt_service/db_utils.py
@@ -0,0 +1,46 @@
+from typing import Any, Optional
+
+from unstract.prompt_service.config import db
+from unstract.prompt_service.constants import DBTableV2
+from unstract.prompt_service.env_manager import EnvLoader
+
+DB_SCHEMA = EnvLoader.get_env_or_die("DB_SCHEMA", "unstract_v2")
+
+
+class DBUtils:
+
+ @classmethod
+ def get_organization_from_bearer_token(
+ cls, token: str
+ ) -> tuple[Optional[int], str]:
+ """Retrieve organization ID and identifier using a bearer token.
+
+ Args:
+ token (str): The bearer token (platform key).
+
+ Returns:
+ tuple[int, str]: organization uid and organization identifier
+ """
+ platform_key_table = f'"{DB_SCHEMA}".{DBTableV2.PLATFORM_KEY}'
+ organization_table = f'"{DB_SCHEMA}".{DBTableV2.ORGANIZATION}'
+
+ organization_uid: Optional[int] = cls._execute_query(
+ f"SELECT organization_id FROM {platform_key_table} WHERE key=%s", (token,)
+ )
+ if organization_uid is None:
+ return None, None
+
+ organization_identifier: Optional[str] = cls._execute_query(
+ f"SELECT organization_id FROM {organization_table} WHERE id=%s",
+ (organization_uid,),
+ )
+ return organization_uid, organization_identifier
+
+ @classmethod
+ def _execute_query(cls, query: str, params: tuple = ()) -> Any:
+ cursor = db.execute_sql(query, params)
+ result_row = cursor.fetchone()
+ cursor.close()
+ if not result_row or len(result_row) == 0:
+ return None
+ return result_row[0]
diff --git a/prompt-service/src/unstract/prompt_service/env_manager.py b/prompt-service/src/unstract/prompt_service/env_manager.py
new file mode 100644
index 000000000..c51f5cf80
--- /dev/null
+++ b/prompt-service/src/unstract/prompt_service/env_manager.py
@@ -0,0 +1,11 @@
+import os
+from typing import Optional
+
+
+class EnvLoader:
+ @staticmethod
+ def get_env_or_die(env_key: str, default: Optional[str] = None) -> str:
+ env_value = os.environ.get(env_key, default=default)
+ if env_value is None or env_value == "":
+ raise ValueError(f"Env variable {env_key} is required")
+ return env_value
diff --git a/prompt-service/src/unstract/prompt_service/helper.py b/prompt-service/src/unstract/prompt_service/helper.py
index ae37cc408..baed82c49 100644
--- a/prompt-service/src/unstract/prompt_service/helper.py
+++ b/prompt-service/src/unstract/prompt_service/helper.py
@@ -9,12 +9,17 @@ from dotenv import load_dotenv
from flask import Flask, current_app, json
from unstract.prompt_service.authentication_middleware import AuthenticationMiddleware
from unstract.prompt_service.config import db
+from unstract.prompt_service.constants import FeatureFlag
from unstract.prompt_service.constants import PromptServiceContants as PSKeys
+from unstract.prompt_service.db_utils import DBUtils
+from unstract.prompt_service.env_manager import EnvLoader
from unstract.prompt_service.exceptions import APIError, RateLimitError
from unstract.sdk.exceptions import RateLimitError as SdkRateLimitError
from unstract.sdk.exceptions import SdkError
from unstract.sdk.llm import LLM
+from unstract.flags.feature_flag import check_feature_flag_status
+
load_dotenv()
# Global variable to store plugins
@@ -112,6 +117,8 @@ def initialize_plugin_endpoints(app: Flask) -> None:
def query_usage_metadata(token: str, metadata: dict[str, Any]) -> dict[str, Any]:
+ if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):
+ return query_usage_metadata_v2(db, token, metadata)
org_id: str = AuthenticationMiddleware.get_account_from_bearer_token(token)
run_id: str = metadata["run_id"]
query: str = f"""
@@ -149,6 +156,45 @@ def query_usage_metadata(token: str, metadata: dict[str, Any]) -> dict[str, Any]
return metadata
+def query_usage_metadata_v2(token: str, metadata: dict[str, Any]) -> dict[str, Any]:
+ DB_SCHEMA = EnvLoader.get_env_or_die("DB_SCHEMA", "unstract_v2")
+ organization_uid, org_id = DBUtils.get_organization_from_bearer_token(token)
+ run_id: str = metadata["run_id"]
+ query: str = f"""
+ SELECT
+ usage_type,
+ llm_usage_reason,
+ model_name,
+ SUM(prompt_tokens) AS input_tokens,
+ SUM(completion_tokens) AS output_tokens,
+ SUM(total_tokens) AS total_tokens,
+ SUM(embedding_tokens) AS embedding_tokens,
+ SUM(cost_in_dollars) AS cost_in_dollars
+ FROM "{DB_SCHEMA}"."token_usage_v2"
+ WHERE run_id = %s and organization_id = %s
+ GROUP BY usage_type, llm_usage_reason, model_name;
+ """
+ logger: Logger = current_app.logger
+ try:
+ with db.atomic():
+ logger.info(
+ "Querying usage metadata for org_id: %s, run_id: %s", org_id, run_id
+ )
+ cursor = db.execute_sql(query, (run_id, organization_uid))
+ results: list[tuple] = cursor.fetchall()
+ # Process results as needed
+ for row in results:
+ key, item = _get_key_and_item(row)
+ # Initialize the key as an empty list if it doesn't exist
+ if key not in metadata:
+ metadata[key] = []
+ # Append the item to the list associated with the key
+ metadata[key].append(item)
+ except Exception as e:
+ logger.error(f"Error executing querying usage metadata: {e}")
+ return metadata
+
+
def _get_key_and_item(row: tuple) -> tuple[str, dict[str, Any]]:
(
usage_type,
diff --git a/prompt-service/src/unstract/prompt_service/main.py b/prompt-service/src/unstract/prompt_service/main.py
index 8d4bfafe8..ed447dd56 100644
--- a/prompt-service/src/unstract/prompt_service/main.py
+++ b/prompt-service/src/unstract/prompt_service/main.py
@@ -61,6 +61,17 @@ def after_request(exception: Any) -> None:
db.close()
+def log_request_info():
+ app.logger.info(f"Request Path: {request.path} | Method: {request.method}")
+
+
+def log_response_info(response):
+ app.logger.info(f"Response Status: {response.status}")
+ return response
+
+
def _publish_log(
log_events_id: str,
component: dict[str, str],