Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

fix: Added schema to page usage query in platform-service, refactored defa… (#847)

Chandrasekharan M82c0da44b254
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py
index 1ebb46bc0..f8341cdaf 100644
--- a/backend/backend/settings/base.py
+++ b/backend/backend/settings/base.py
@@ -101,7 +101,7 @@ DB_USER = os.environ.get("DB_USER", "unstract_dev")
DB_HOST = os.environ.get("DB_HOST", "backend-db-1")
DB_PASSWORD = os.environ.get("DB_PASSWORD", "unstract_pass")
DB_PORT = os.environ.get("DB_PORT", 5432)
-DB_SCHEMA = os.environ.get("DB_SCHEMA", "unstract_v2")
+DB_SCHEMA = os.environ.get("DB_SCHEMA", "unstract")
DEFAULT_ORGANIZATION = "default_org"
FLIPT_BASE_URL = os.environ.get("FLIPT_BASE_URL", "http://localhost:9005")
diff --git a/platform-service/src/unstract/platform_service/constants.py b/platform-service/src/unstract/platform_service/constants.py
index 4a443e800..56fd67a76 100644
--- a/platform-service/src/unstract/platform_service/constants.py
+++ b/platform-service/src/unstract/platform_service/constants.py
@@ -7,17 +7,12 @@ class FeatureFlag:
class DBTable:
"""Database tables."""
- PAGE_USAGE = "page_usage"
-
-
-class DBTableV2:
- """Database tables."""
-
ORGANIZATION = "organization"
ADAPTER_INSTANCE = "adapter_instance"
PROMPT_STUDIO_REGISTRY = "prompt_studio_registry"
PLATFORM_KEY = "platform_key"
TOKEN_USAGE = "usage"
+ PAGE_USAGE = "page_usage"
class LogLevel:
diff --git a/platform-service/src/unstract/platform_service/controller/platform.py b/platform-service/src/unstract/platform_service/controller/platform.py
index 3381c6029..fc6c83edf 100644
--- a/platform-service/src/unstract/platform_service/controller/platform.py
+++ b/platform-service/src/unstract/platform_service/controller/platform.py
@@ -8,7 +8,7 @@ from cryptography.fernet import Fernet, InvalidToken
from flask import Blueprint, Request
from flask import current_app as app
from flask import jsonify, make_response, request
-from unstract.platform_service.constants import DBTable, DBTableV2
+from unstract.platform_service.constants import DBTable
from unstract.platform_service.env import Env
from unstract.platform_service.exceptions import APIError
from unstract.platform_service.extensions import db
@@ -55,12 +55,12 @@ def get_organization_from_bearer_token(token: str) -> tuple[Optional[int], str]:
tuple[int, str]: organization uid and organization identifier
"""
query = f"""
- SELECT organization_id FROM "{Env.DB_SCHEMA}".{DBTableV2.PLATFORM_KEY}
+ SELECT organization_id FROM "{Env.DB_SCHEMA}".{DBTable.PLATFORM_KEY}
WHERE key=%s
"""
organization_uid: int = execute_query(query, (token,))
query_org = f"""
- SELECT organization_id FROM "{Env.DB_SCHEMA}".{DBTableV2.ORGANIZATION}
+ SELECT organization_id FROM "{Env.DB_SCHEMA}".{DBTable.ORGANIZATION}
WHERE id=%s
"""
organization_identifier: str = execute_query(query_org, (organization_uid,))
@@ -82,7 +82,7 @@ def validate_bearer_token(token: Optional[str]) -> bool:
app.logger.error("Authentication failed. Empty bearer token")
return False
- platform_key_table = DBTableV2.PLATFORM_KEY
+ platform_key_table = DBTable.PLATFORM_KEY
query = f"""
SELECT * FROM \"{Env.DB_SCHEMA}\".{platform_key_table}
WHERE key = '{token}'
@@ -139,10 +139,10 @@ def page_usage() -> Any:
run_id = payload.get("run_id", "")
query = f"""
- INSERT INTO {DBTable.PAGE_USAGE} (id, organization_id, pages_processed,
- file_name, file_size, file_type, run_id, created_at)
- VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
- """
+ INSERT INTO \"{Env.DB_SCHEMA}\".{DBTable.PAGE_USAGE} (id, organization_id,
+ pages_processed, file_name, file_size, file_type, run_id, created_at)
+ VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
+ """
usage_id = uuid.uuid4()
current_time = datetime.now()
params = (
@@ -160,7 +160,7 @@ def page_usage() -> Any:
with db.atomic() as transaction:
db.execute_sql(query, params)
transaction.commit()
- app.logger.info("Entry created with id %s for %s", usage_id, org_id)
+ app.logger.info("Page usage recorded with id %s for %s", usage_id, org_id)
result["status"] = "OK"
result["unique_id"] = usage_id
return make_response(result, 200)
@@ -222,7 +222,7 @@ def usage() -> Any:
usage_id = uuid.uuid4()
current_time = datetime.now()
query = f"""
- INSERT INTO \"{Env.DB_SCHEMA}\".{DBTableV2.TOKEN_USAGE} (
+ INSERT INTO \"{Env.DB_SCHEMA}\".{DBTable.TOKEN_USAGE} (
id, organization_id, workflow_id,
execution_id, adapter_instance_id, run_id, usage_type,
llm_usage_reason, model_name, embedding_tokens, prompt_tokens,
@@ -254,7 +254,9 @@ def usage() -> Any:
with db.atomic() as transaction:
db.execute_sql(query, params)
transaction.commit()
- app.logger.info("Entry created with id %s for %s", usage_id, org_id)
+ app.logger.info(
+ "Adapter usage recorded with id %s for %s", usage_id, org_id
+ )
result["status"] = "OK"
result["unique_id"] = usage_id
return make_response(result, 200)
diff --git a/platform-service/src/unstract/platform_service/env.py b/platform-service/src/unstract/platform_service/env.py
index df6b6f12c..2bcf2da38 100644
--- a/platform-service/src/unstract/platform_service/env.py
+++ b/platform-service/src/unstract/platform_service/env.py
@@ -26,7 +26,7 @@ class Env:
APPLICATION_NAME = EnvManager.get_required_setting(
"APPLICATION_NAME", "unstract-platform-service"
)
- DB_SCHEMA = EnvManager.get_required_setting("DB_SCHEMA", "unstract_v2")
+ DB_SCHEMA = EnvManager.get_required_setting("DB_SCHEMA")
LOG_LEVEL = EnvManager.get_required_setting("LOG_LEVEL", LogLevel.INFO)
diff --git a/platform-service/src/unstract/platform_service/helper/adapter_instance.py b/platform-service/src/unstract/platform_service/helper/adapter_instance.py
index 4b023596a..3ffbe47e9 100644
--- a/platform-service/src/unstract/platform_service/helper/adapter_instance.py
+++ b/platform-service/src/unstract/platform_service/helper/adapter_instance.py
@@ -1,11 +1,11 @@
from typing import Any, Optional
-from unstract.platform_service.constants import DBTableV2
+from unstract.platform_service.constants import DBTable
from unstract.platform_service.exceptions import APIError
from unstract.platform_service.extensions import db
from unstract.platform_service.utils import EnvManager
-DB_SCHEMA = EnvManager.get_required_setting("DB_SCHEMA", "unstract_v2")
+DB_SCHEMA = EnvManager.get_required_setting("DB_SCHEMA", "unstract")
class AdapterInstanceRequestHelper:
@@ -26,7 +26,7 @@ class AdapterInstanceRequestHelper:
"""
query = (
"SELECT id, adapter_id, adapter_name, adapter_type, adapter_metadata_b"
- f' FROM "{DB_SCHEMA}".{DBTableV2.ADAPTER_INSTANCE} x '
+ f' FROM "{DB_SCHEMA}".{DBTable.ADAPTER_INSTANCE} x '
f"WHERE id='{adapter_instance_id}' and "
f"organization_id='{organization_uid}'"
)
diff --git a/platform-service/src/unstract/platform_service/helper/prompt_studio.py b/platform-service/src/unstract/platform_service/helper/prompt_studio.py
index 35ef634db..72cd217ca 100644
--- a/platform-service/src/unstract/platform_service/helper/prompt_studio.py
+++ b/platform-service/src/unstract/platform_service/helper/prompt_studio.py
@@ -1,11 +1,11 @@
from typing import Any
-from unstract.platform_service.constants import DBTableV2
+from unstract.platform_service.constants import DBTable
from unstract.platform_service.exceptions import APIError
from unstract.platform_service.extensions import db
from unstract.platform_service.utils import EnvManager
-DB_SCHEMA = EnvManager.get_required_setting("DB_SCHEMA", "unstract_v2")
+DB_SCHEMA = EnvManager.get_required_setting("DB_SCHEMA", "unstract")
class PromptStudioRequestHelper:
@@ -26,7 +26,7 @@ class PromptStudioRequestHelper:
query = (
"SELECT prompt_registry_id, tool_spec, "
"tool_metadata, tool_property FROM "
- f'"{DB_SCHEMA}".{DBTableV2.PROMPT_STUDIO_REGISTRY} x '
+ f'"{DB_SCHEMA}".{DBTable.PROMPT_STUDIO_REGISTRY} x '
f"WHERE prompt_registry_id='{prompt_registry_id}'"
)
cursor = db.execute_sql(query)
diff --git a/prompt-service/src/unstract/prompt_service/authentication_middleware.py b/prompt-service/src/unstract/prompt_service/authentication_middleware.py
index 5192df71a..52297d8e1 100644
--- a/prompt-service/src/unstract/prompt_service/authentication_middleware.py
+++ b/prompt-service/src/unstract/prompt_service/authentication_middleware.py
@@ -6,7 +6,7 @@ from unstract.prompt_service.constants import DBTableV2
from unstract.prompt_service.db_utils import DBUtils
from unstract.prompt_service.env_manager import EnvLoader
-DB_SCHEMA = EnvLoader.get_env_or_die("DB_SCHEMA", "unstract_v2")
+DB_SCHEMA = EnvLoader.get_env_or_die("DB_SCHEMA", "unstract")
class AuthenticationMiddleware:
diff --git a/prompt-service/src/unstract/prompt_service/db_utils.py b/prompt-service/src/unstract/prompt_service/db_utils.py
index fada7ff31..f20e34ddb 100644
--- a/prompt-service/src/unstract/prompt_service/db_utils.py
+++ b/prompt-service/src/unstract/prompt_service/db_utils.py
@@ -4,7 +4,7 @@ from unstract.prompt_service.config import db
from unstract.prompt_service.constants import DBTableV2
from unstract.prompt_service.env_manager import EnvLoader
-DB_SCHEMA = EnvLoader.get_env_or_die("DB_SCHEMA", "unstract_v2")
+DB_SCHEMA = EnvLoader.get_env_or_die("DB_SCHEMA", "unstract")
class DBUtils:
diff --git a/prompt-service/src/unstract/prompt_service/helper.py b/prompt-service/src/unstract/prompt_service/helper.py
index 0754b74c7..86a780013 100644
--- a/prompt-service/src/unstract/prompt_service/helper.py
+++ b/prompt-service/src/unstract/prompt_service/helper.py
@@ -114,7 +114,7 @@ def initialize_plugin_endpoints(app: Flask) -> None:
def query_usage_metadata(token: str, metadata: dict[str, Any]) -> dict[str, Any]:
- DB_SCHEMA = EnvLoader.get_env_or_die("DB_SCHEMA", "unstract_v2")
+ DB_SCHEMA = EnvLoader.get_env_or_die("DB_SCHEMA", "unstract")
organization_uid, org_id = DBUtils.get_organization_from_bearer_token(token)
run_id: str = metadata["run_id"]
query: str = f"""