LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
fix: Added schema to page usage query in platform-service, refactored defa… (#847)
Chandrasekharan M82c0da44b254
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.pyindex 1ebb46bc0..f8341cdaf 100644--- a/backend/backend/settings/base.py+++ b/backend/backend/settings/base.py@@ -101,7 +101,7 @@ DB_USER = os.environ.get("DB_USER", "unstract_dev")DB_HOST = os.environ.get("DB_HOST", "backend-db-1")DB_PASSWORD = os.environ.get("DB_PASSWORD", "unstract_pass")DB_PORT = os.environ.get("DB_PORT", 5432)-DB_SCHEMA = os.environ.get("DB_SCHEMA", "unstract_v2")+DB_SCHEMA = os.environ.get("DB_SCHEMA", "unstract")DEFAULT_ORGANIZATION = "default_org"FLIPT_BASE_URL = os.environ.get("FLIPT_BASE_URL", "http://localhost:9005")diff --git a/platform-service/src/unstract/platform_service/constants.py b/platform-service/src/unstract/platform_service/constants.pyindex 4a443e800..56fd67a76 100644--- a/platform-service/src/unstract/platform_service/constants.py+++ b/platform-service/src/unstract/platform_service/constants.py@@ -7,17 +7,12 @@ class FeatureFlag:class DBTable:"""Database tables."""- PAGE_USAGE = "page_usage"---class DBTableV2:- """Database tables."""-ORGANIZATION = "organization"ADAPTER_INSTANCE = "adapter_instance"PROMPT_STUDIO_REGISTRY = "prompt_studio_registry"PLATFORM_KEY = "platform_key"TOKEN_USAGE = "usage"+ PAGE_USAGE = "page_usage"class LogLevel:diff --git a/platform-service/src/unstract/platform_service/controller/platform.py b/platform-service/src/unstract/platform_service/controller/platform.pyindex 3381c6029..fc6c83edf 100644--- a/platform-service/src/unstract/platform_service/controller/platform.py+++ b/platform-service/src/unstract/platform_service/controller/platform.py@@ -8,7 +8,7 @@ from cryptography.fernet import Fernet, InvalidTokenfrom flask import Blueprint, Requestfrom flask import current_app as appfrom flask import jsonify, make_response, request-from unstract.platform_service.constants import DBTable, DBTableV2+from unstract.platform_service.constants import DBTablefrom unstract.platform_service.env import Envfrom unstract.platform_service.exceptions import APIErrorfrom unstract.platform_service.extensions import db@@ -55,12 +55,12 @@ def get_organization_from_bearer_token(token: str) -> tuple[Optional[int], str]:tuple[int, str]: organization uid and organization identifier"""query = f"""- SELECT organization_id FROM "{Env.DB_SCHEMA}".{DBTableV2.PLATFORM_KEY}+ SELECT organization_id FROM "{Env.DB_SCHEMA}".{DBTable.PLATFORM_KEY}WHERE key=%s"""organization_uid: int = execute_query(query, (token,))query_org = f"""- SELECT organization_id FROM "{Env.DB_SCHEMA}".{DBTableV2.ORGANIZATION}+ SELECT organization_id FROM "{Env.DB_SCHEMA}".{DBTable.ORGANIZATION}WHERE id=%s"""organization_identifier: str = execute_query(query_org, (organization_uid,))@@ -82,7 +82,7 @@ def validate_bearer_token(token: Optional[str]) -> bool:app.logger.error("Authentication failed. Empty bearer token")return False- platform_key_table = DBTableV2.PLATFORM_KEY+ platform_key_table = DBTable.PLATFORM_KEYquery = f"""SELECT * FROM \"{Env.DB_SCHEMA}\".{platform_key_table}WHERE key = '{token}'@@ -139,10 +139,10 @@ def page_usage() -> Any:run_id = payload.get("run_id", "")query = f"""- INSERT INTO {DBTable.PAGE_USAGE} (id, organization_id, pages_processed,- file_name, file_size, file_type, run_id, created_at)- VALUES (%s, %s, %s, %s, %s, %s, %s, %s)- """+ INSERT INTO \"{Env.DB_SCHEMA}\".{DBTable.PAGE_USAGE} (id, organization_id,+ pages_processed, file_name, file_size, file_type, run_id, created_at)+ VALUES (%s, %s, %s, %s, %s, %s, %s, %s)+ """usage_id = uuid.uuid4()current_time = datetime.now()params = (@@ -160,7 +160,7 @@ def page_usage() -> Any:with db.atomic() as transaction:db.execute_sql(query, params)transaction.commit()- app.logger.info("Entry created with id %s for %s", usage_id, org_id)+ app.logger.info("Page usage recorded with id %s for %s", usage_id, org_id)result["status"] = "OK"result["unique_id"] = usage_idreturn make_response(result, 200)@@ -222,7 +222,7 @@ def usage() -> Any:usage_id = uuid.uuid4()current_time = datetime.now()query = f"""- INSERT INTO \"{Env.DB_SCHEMA}\".{DBTableV2.TOKEN_USAGE} (+ INSERT INTO \"{Env.DB_SCHEMA}\".{DBTable.TOKEN_USAGE} (id, organization_id, workflow_id,execution_id, adapter_instance_id, run_id, usage_type,llm_usage_reason, model_name, embedding_tokens, prompt_tokens,@@ -254,7 +254,9 @@ def usage() -> Any:with db.atomic() as transaction:db.execute_sql(query, params)transaction.commit()- app.logger.info("Entry created with id %s for %s", usage_id, org_id)+ app.logger.info(+ "Adapter usage recorded with id %s for %s", usage_id, org_id+ )result["status"] = "OK"result["unique_id"] = usage_idreturn make_response(result, 200)diff --git a/platform-service/src/unstract/platform_service/env.py b/platform-service/src/unstract/platform_service/env.pyindex df6b6f12c..2bcf2da38 100644--- a/platform-service/src/unstract/platform_service/env.py+++ b/platform-service/src/unstract/platform_service/env.py@@ -26,7 +26,7 @@ class Env:APPLICATION_NAME = EnvManager.get_required_setting("APPLICATION_NAME", "unstract-platform-service")- DB_SCHEMA = EnvManager.get_required_setting("DB_SCHEMA", "unstract_v2")+ DB_SCHEMA = EnvManager.get_required_setting("DB_SCHEMA")LOG_LEVEL = EnvManager.get_required_setting("LOG_LEVEL", LogLevel.INFO)diff --git a/platform-service/src/unstract/platform_service/helper/adapter_instance.py b/platform-service/src/unstract/platform_service/helper/adapter_instance.pyindex 4b023596a..3ffbe47e9 100644--- a/platform-service/src/unstract/platform_service/helper/adapter_instance.py+++ b/platform-service/src/unstract/platform_service/helper/adapter_instance.py@@ -1,11 +1,11 @@from typing import Any, Optional-from unstract.platform_service.constants import DBTableV2+from unstract.platform_service.constants import DBTablefrom unstract.platform_service.exceptions import APIErrorfrom unstract.platform_service.extensions import dbfrom unstract.platform_service.utils import EnvManager-DB_SCHEMA = EnvManager.get_required_setting("DB_SCHEMA", "unstract_v2")+DB_SCHEMA = EnvManager.get_required_setting("DB_SCHEMA", "unstract")class AdapterInstanceRequestHelper:@@ -26,7 +26,7 @@ class AdapterInstanceRequestHelper:"""query = ("SELECT id, adapter_id, adapter_name, adapter_type, adapter_metadata_b"- f' FROM "{DB_SCHEMA}".{DBTableV2.ADAPTER_INSTANCE} x '+ f' FROM "{DB_SCHEMA}".{DBTable.ADAPTER_INSTANCE} x 'f"WHERE id='{adapter_instance_id}' and "f"organization_id='{organization_uid}'")diff --git a/platform-service/src/unstract/platform_service/helper/prompt_studio.py b/platform-service/src/unstract/platform_service/helper/prompt_studio.pyindex 35ef634db..72cd217ca 100644--- a/platform-service/src/unstract/platform_service/helper/prompt_studio.py+++ b/platform-service/src/unstract/platform_service/helper/prompt_studio.py@@ -1,11 +1,11 @@from typing import Any-from unstract.platform_service.constants import DBTableV2+from unstract.platform_service.constants import DBTablefrom unstract.platform_service.exceptions import APIErrorfrom unstract.platform_service.extensions import dbfrom unstract.platform_service.utils import EnvManager-DB_SCHEMA = EnvManager.get_required_setting("DB_SCHEMA", "unstract_v2")+DB_SCHEMA = EnvManager.get_required_setting("DB_SCHEMA", "unstract")class PromptStudioRequestHelper:@@ -26,7 +26,7 @@ class PromptStudioRequestHelper:query = ("SELECT prompt_registry_id, tool_spec, ""tool_metadata, tool_property FROM "- f'"{DB_SCHEMA}".{DBTableV2.PROMPT_STUDIO_REGISTRY} x '+ f'"{DB_SCHEMA}".{DBTable.PROMPT_STUDIO_REGISTRY} x 'f"WHERE prompt_registry_id='{prompt_registry_id}'")cursor = db.execute_sql(query)diff --git a/prompt-service/src/unstract/prompt_service/authentication_middleware.py b/prompt-service/src/unstract/prompt_service/authentication_middleware.pyindex 5192df71a..52297d8e1 100644--- a/prompt-service/src/unstract/prompt_service/authentication_middleware.py+++ b/prompt-service/src/unstract/prompt_service/authentication_middleware.py@@ -6,7 +6,7 @@ from unstract.prompt_service.constants import DBTableV2from unstract.prompt_service.db_utils import DBUtilsfrom unstract.prompt_service.env_manager import EnvLoader-DB_SCHEMA = EnvLoader.get_env_or_die("DB_SCHEMA", "unstract_v2")+DB_SCHEMA = EnvLoader.get_env_or_die("DB_SCHEMA", "unstract")class AuthenticationMiddleware:diff --git a/prompt-service/src/unstract/prompt_service/db_utils.py b/prompt-service/src/unstract/prompt_service/db_utils.pyindex fada7ff31..f20e34ddb 100644--- a/prompt-service/src/unstract/prompt_service/db_utils.py+++ b/prompt-service/src/unstract/prompt_service/db_utils.py@@ -4,7 +4,7 @@ from unstract.prompt_service.config import dbfrom unstract.prompt_service.constants import DBTableV2from unstract.prompt_service.env_manager import EnvLoader-DB_SCHEMA = EnvLoader.get_env_or_die("DB_SCHEMA", "unstract_v2")+DB_SCHEMA = EnvLoader.get_env_or_die("DB_SCHEMA", "unstract")class DBUtils:diff --git a/prompt-service/src/unstract/prompt_service/helper.py b/prompt-service/src/unstract/prompt_service/helper.pyindex 0754b74c7..86a780013 100644--- a/prompt-service/src/unstract/prompt_service/helper.py+++ b/prompt-service/src/unstract/prompt_service/helper.py@@ -114,7 +114,7 @@ def initialize_plugin_endpoints(app: Flask) -> None:def query_usage_metadata(token: str, metadata: dict[str, Any]) -> dict[str, Any]:- DB_SCHEMA = EnvLoader.get_env_or_die("DB_SCHEMA", "unstract_v2")+ DB_SCHEMA = EnvLoader.get_env_or_die("DB_SCHEMA", "unstract")organization_uid, org_id = DBUtils.get_organization_from_bearer_token(token)run_id: str = metadata["run_id"]query: str = f"""