LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
delta changes and enhancements in platform and prompt services (#538)
ali8a71ee652e5b
diff --git a/backend/sample.env b/backend/sample.envindex d75f48c92..79e034c96 100644--- a/backend/sample.env+++ b/backend/sample.env@@ -140,6 +140,9 @@ CELERY_BROKER_URL = "redis://unstract-redis:6379"# Indexing flag to prevent re-indexINDEXING_FLAG_TTL=1800+# V2 Configurations+DB_SCHEMA="unstract_v2"+# Notification Timeout in SecondsNOTIFICATION_TIMEOUT=5diff --git a/platform-service/sample.env b/platform-service/sample.envindex 0e533af8c..968ae2955 100644--- a/platform-service/sample.env+++ b/platform-service/sample.env@@ -26,3 +26,6 @@ FLIPT_SERVICE_AVAILABLE=FalseMODEL_PRICES_URL="https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"MODEL_PRICES_TTL_IN_DAYS=7MODEL_PRICES_FILE_PATH="/tmp/model_prices.json"++# V2 Configurations+DB_SCHEMA="unstract_v2"diff --git a/platform-service/src/unstract/platform_service/controller/platform.py b/platform-service/src/unstract/platform_service/controller/platform.pyindex 1daf15062..2f8aa992e 100644--- a/platform-service/src/unstract/platform_service/controller/platform.py+++ b/platform-service/src/unstract/platform_service/controller/platform.py@@ -75,9 +75,15 @@ def get_organization_from_bearer_token(token: str) -> tuple[Optional[int], str]:tuple[int, str]: organization uid and organization identifier"""if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):- query = f"SELECT organization_id FROM {DBTableV2.PLATFORM_KEY} WHERE key=%s"+ query = f"""+ SELECT organization_id FROM "{Env.DB_SCHEMA}".{DBTableV2.PLATFORM_KEY}+ WHERE key=%s+ """organization_uid: int = execute_query(query, (token,))- query_org = f"SELECT organization_id FROM {DBTableV2.ORGANIZATION} WHERE id=%s"+ query_org = f"""+ SELECT organization_id FROM "{Env.DB_SCHEMA}".{DBTableV2.ORGANIZATION}+ WHERE id=%s+ """organization_identifier: str = execute_query(query_org, (organization_uid,))return organization_uid, organization_identifierelse:@@ -102,10 +108,16 @@ def validate_bearer_token(token: Optional[str]) -> bool:if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):platform_key_table = DBTableV2.PLATFORM_KEY+ query = f"""+ SELECT * FROM \"{Env.DB_SCHEMA}\".{platform_key_table}+ WHERE key = '{token}'+ """else:platform_key_table = "account_platformkey"+ query = f"""+ SELECT * FROM {platform_key_table} WHERE key = '{token}'+ """- query = f"SELECT * FROM {platform_key_table} WHERE key = '{token}'"cursor = be_db.execute_sql(query)result_row = cursor.fetchone()cursor.close()@@ -241,7 +253,8 @@ def usage() -> Any:current_time = datetime.now()if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):query = f"""- INSERT INTO {DBTableV2.TOKEN_USAGE} (id, organization_id, workflow_id,+ INSERT INTO \"{Env.DB_SCHEMA}\".{DBTableV2.TOKEN_USAGE} (+ id, organization_id, workflow_id,execution_id, adapter_instance_id, run_id, usage_type,llm_usage_reason, model_name, embedding_tokens, prompt_tokens,completion_tokens, total_tokens, cost_in_dollars, created_at, modified_at)diff --git a/platform-service/src/unstract/platform_service/env.py b/platform-service/src/unstract/platform_service/env.pyindex ba92d5e97..6d942c2f4 100644--- a/platform-service/src/unstract/platform_service/env.py+++ b/platform-service/src/unstract/platform_service/env.py@@ -25,6 +25,7 @@ class Env:APPLICATION_NAME = EnvManager.get_required_setting("APPLICATION_NAME", "unstract-platform-service")+ DB_SCHEMA = EnvManager.get_required_setting("DB_SCHEMA", "unstract_v2")EnvManager.raise_for_missing_envs()diff --git a/platform-service/src/unstract/platform_service/helper/adapter_instance.py b/platform-service/src/unstract/platform_service/helper/adapter_instance.pyindex c0d03dbc1..c6daa25b6 100644--- a/platform-service/src/unstract/platform_service/helper/adapter_instance.py+++ b/platform-service/src/unstract/platform_service/helper/adapter_instance.py@@ -3,9 +3,12 @@ from typing import Any, Optionalfrom peewee import PostgresqlDatabasefrom unstract.platform_service.constants import DBTableV2, FeatureFlagfrom unstract.platform_service.exceptions import APIError+from unstract.platform_service.utils import EnvManagerfrom unstract.flags.feature_flag import check_feature_flag_status+DB_SCHEMA = EnvManager.get_required_setting("DB_SCHEMA", "unstract_v2")+class AdapterInstanceRequestHelper:@staticmethod@@ -28,7 +31,7 @@ class AdapterInstanceRequestHelper:if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):query = ("SELECT id, adapter_id, adapter_metadata_b FROM "- f"{DBTableV2.ADAPTER_INSTANCE} x "+ f'"{DB_SCHEMA}".{DBTableV2.ADAPTER_INSTANCE} x 'f"WHERE id='{adapter_instance_id}' and "f"organization_id='{organization_uid}'")diff --git a/platform-service/src/unstract/platform_service/helper/prompt_studio.py b/platform-service/src/unstract/platform_service/helper/prompt_studio.pyindex 6801792c1..e685909c2 100644--- a/platform-service/src/unstract/platform_service/helper/prompt_studio.py+++ b/platform-service/src/unstract/platform_service/helper/prompt_studio.py@@ -3,9 +3,12 @@ from typing import Anyfrom peewee import PostgresqlDatabasefrom unstract.platform_service.constants import DBTableV2, FeatureFlagfrom unstract.platform_service.exceptions import APIError+from unstract.platform_service.utils import EnvManagerfrom unstract.flags.feature_flag import check_feature_flag_status+DB_SCHEMA = EnvManager.get_required_setting("DB_SCHEMA", "unstract_v2")+class PromptStudioRequestHelper:@staticmethod@@ -28,7 +31,7 @@ class PromptStudioRequestHelper:query = ("SELECT prompt_registry_id, tool_spec, ""tool_metadata, tool_property FROM "- f"{DBTableV2.PROMPT_STUDIO_REGISTRY} x "+ f'"{DB_SCHEMA}".{DBTableV2.PROMPT_STUDIO_REGISTRY} x 'f"WHERE prompt_registry_id='{prompt_registry_id}'")else:diff --git a/prompt-service/sample.env b/prompt-service/sample.envindex 53f27dad2..697592d28 100644--- a/prompt-service/sample.env+++ b/prompt-service/sample.env@@ -25,5 +25,9 @@ CELERY_BROKER_URL="redis://unstract-redis:6379"EVALUATION_SERVER_IP=unstract-fliptEVALUATION_SERVER_PORT=9000PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python++# V2 Configurations+DB_SCHEMA="unstract_v2"+# Flipt ServiceFLIPT_SERVICE_AVAILABLE=Falsediff --git a/prompt-service/src/unstract/prompt_service/authentication_middleware.py b/prompt-service/src/unstract/prompt_service/authentication_middleware.pyindex fbf03c37b..3162216cd 100644--- a/prompt-service/src/unstract/prompt_service/authentication_middleware.py+++ b/prompt-service/src/unstract/prompt_service/authentication_middleware.py@@ -3,9 +3,13 @@ from typing import Any, Optionalfrom flask import Request, current_appfrom unstract.prompt_service.config import dbfrom unstract.prompt_service.constants import DBTableV2, FeatureFlag+from unstract.prompt_service.db_utils import DBUtils+from unstract.prompt_service.env_manager import EnvLoaderfrom unstract.flags.feature_flag import check_feature_flag_status+DB_SCHEMA = EnvLoader.get_env_or_die("DB_SCHEMA", "unstract_v2")+class AuthenticationMiddleware:@@ -17,7 +21,7 @@ class AuthenticationMiddleware:return Falseif check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):- platform_key_table = DBTableV2.PLATFORM_KEY+ platform_key_table = f'"{DB_SCHEMA}".{DBTableV2.PLATFORM_KEY}'else:platform_key_table = "account_platformkey"@@ -75,11 +79,11 @@ class AuthenticationMiddleware:organization_table = "account_organization"query = f"SELECT organization_id FROM {platform_key_table} WHERE key='{token}'"- organization = AuthenticationMiddleware.execute_query(query)+ organization = DBUtils.execute_query(query)query_org = (f"SELECT schema_name FROM {organization_table} WHERE id='{organization}'")- schema_name: str = AuthenticationMiddleware.execute_query(query_org)+ schema_name: str = DBUtils.execute_query(query_org)return schema_name@staticmethoddiff --git a/prompt-service/src/unstract/prompt_service/db_utils.py b/prompt-service/src/unstract/prompt_service/db_utils.pynew file mode 100644index 000000000..74372eafd--- /dev/null+++ b/prompt-service/src/unstract/prompt_service/db_utils.py@@ -0,0 +1,46 @@+from typing import Any, Optional++from unstract.prompt_service.config import db+from unstract.prompt_service.constants import DBTableV2+from unstract.prompt_service.env_manager import EnvLoader++DB_SCHEMA = EnvLoader.get_env_or_die("DB_SCHEMA", "unstract_v2")+++class DBUtils:++ @classmethod+ def get_organization_from_bearer_token(+ cls, token: str+ ) -> tuple[Optional[int], str]:+ """Retrieve organization ID and identifier using a bearer token.++ Args:+ token (str): The bearer token (platform key).++ Returns:+ tuple[int, str]: organization uid and organization identifier+ """+ platform_key_table = f'"{DB_SCHEMA}".{DBTableV2.PLATFORM_KEY}'+ organization_table = f'"{DB_SCHEMA}".{DBTableV2.ORGANIZATION}'++ organization_uid: Optional[int] = cls._execute_query(+ f"SELECT organization_id FROM {platform_key_table} WHERE key=%s", (token,)+ )+ if organization_uid is None:+ return None, None++ organization_identifier: Optional[str] = cls._execute_query(+ f"SELECT organization_id FROM {organization_table} WHERE id=%s",+ (organization_uid,),+ )+ return organization_uid, organization_identifier++ @classmethod+ def _execute_query(cls, query: str, params: tuple = ()) -> Any:+ cursor = db.execute_sql(query, params)+ result_row = cursor.fetchone()+ cursor.close()+ if not result_row or len(result_row) == 0:+ return None+ return result_row[0]diff --git a/prompt-service/src/unstract/prompt_service/env_manager.py b/prompt-service/src/unstract/prompt_service/env_manager.pynew file mode 100644index 000000000..c51f5cf80--- /dev/null+++ b/prompt-service/src/unstract/prompt_service/env_manager.py@@ -0,0 +1,11 @@+import os+from typing import Optional+++class EnvLoader:+ @staticmethod+ def get_env_or_die(env_key: str, default: Optional[str] = None) -> str:+ env_value = os.environ.get(env_key, default=default)+ if env_value is None or env_value == "":+ raise ValueError(f"Env variable {env_key} is required")+ return env_valuediff --git a/prompt-service/src/unstract/prompt_service/helper.py b/prompt-service/src/unstract/prompt_service/helper.pyindex ae37cc408..baed82c49 100644--- a/prompt-service/src/unstract/prompt_service/helper.py+++ b/prompt-service/src/unstract/prompt_service/helper.py@@ -9,12 +9,17 @@ from dotenv import load_dotenvfrom flask import Flask, current_app, jsonfrom unstract.prompt_service.authentication_middleware import AuthenticationMiddlewarefrom unstract.prompt_service.config import db+from unstract.prompt_service.constants import FeatureFlagfrom unstract.prompt_service.constants import PromptServiceContants as PSKeys+from unstract.prompt_service.db_utils import DBUtils+from unstract.prompt_service.env_manager import EnvLoaderfrom unstract.prompt_service.exceptions import APIError, RateLimitErrorfrom unstract.sdk.exceptions import RateLimitError as SdkRateLimitErrorfrom unstract.sdk.exceptions import SdkErrorfrom unstract.sdk.llm import LLM+from unstract.flags.feature_flag import check_feature_flag_status+load_dotenv()# Global variable to store plugins@@ -112,6 +117,8 @@ def initialize_plugin_endpoints(app: Flask) -> None:def query_usage_metadata(token: str, metadata: dict[str, Any]) -> dict[str, Any]:+ if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):+ return query_usage_metadata_v2(db, token, metadata)org_id: str = AuthenticationMiddleware.get_account_from_bearer_token(token)run_id: str = metadata["run_id"]query: str = f"""@@ -149,6 +156,45 @@ def query_usage_metadata(token: str, metadata: dict[str, Any]) -> dict[str, Any]return metadata+def query_usage_metadata_v2(token: str, metadata: dict[str, Any]) -> dict[str, Any]:+ DB_SCHEMA = EnvLoader.get_env_or_die("DB_SCHEMA", "unstract_v2")+ organization_uid, org_id = DBUtils.get_organization_from_bearer_token(token)+ run_id: str = metadata["run_id"]+ query: str = f"""+ SELECT+ usage_type,+ llm_usage_reason,+ model_name,+ SUM(prompt_tokens) AS input_tokens,+ SUM(completion_tokens) AS output_tokens,+ SUM(total_tokens) AS total_tokens,+ SUM(embedding_tokens) AS embedding_tokens,+ SUM(cost_in_dollars) AS cost_in_dollars+ FROM "{DB_SCHEMA}"."token_usage_v2"+ WHERE run_id = %s and organization_id = %s+ GROUP BY usage_type, llm_usage_reason, model_name;+ """+ logger: Logger = current_app.logger+ try:+ with db.atomic():+ logger.info(+ "Querying usage metadata for org_id: %s, run_id: %s", org_id, run_id+ )+ cursor = db.execute_sql(query, (run_id, organization_uid))+ results: list[tuple] = cursor.fetchall()+ # Process results as needed+ for row in results:+ key, item = _get_key_and_item(row)+ # Initialize the key as an empty list if it doesn't exist+ if key not in metadata:+ metadata[key] = []+ # Append the item to the list associated with the key+ metadata[key].append(item)+ except Exception as e:+ logger.error(f"Error executing querying usage metadata: {e}")+ return metadata++def _get_key_and_item(row: tuple) -> tuple[str, dict[str, Any]]:(usage_type,diff --git a/prompt-service/src/unstract/prompt_service/main.py b/prompt-service/src/unstract/prompt_service/main.pyindex 8d4bfafe8..ed447dd56 100644--- a/prompt-service/src/unstract/prompt_service/main.py+++ b/prompt-service/src/unstract/prompt_service/main.py@@ -61,6 +61,17 @@ def after_request(exception: Any) -> None:db.close()[email protected]_request+def log_request_info():+ app.logger.info(f"Request Path: {request.path} | Method: {request.method}")++[email protected]_request+def log_response_info(response):+ app.logger.info(f"Response Status: {response.status}")+ return response++def _publish_log(log_events_id: str,component: dict[str, str],