LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Feat/usage reporting page (#547)
Rahul Johnyfcc771efd94e
diff --git a/backend/account_usage/__init__.py b/backend/account_usage/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/account_usage/apps.py b/backend/account_usage/apps.pynew file mode 100644index 000000000..09d4944b3--- /dev/null+++ b/backend/account_usage/apps.py@@ -0,0 +1,6 @@+from django.apps import AppConfig+++class AccountUsageConfig(AppConfig):+ default_auto_field = "django.db.models.BigAutoField"+ name = "account_usage"diff --git a/backend/account_usage/migrations/0001_initial.py b/backend/account_usage/migrations/0001_initial.pynew file mode 100644index 000000000..11415be66--- /dev/null+++ b/backend/account_usage/migrations/0001_initial.py@@ -0,0 +1,68 @@+# Generated by Django 4.2.1 on 2024-08-05 06:10++import uuid++from django.db import migrations, models+++class Migration(migrations.Migration):++ initial = True++ dependencies = []++ operations = [+ migrations.CreateModel(+ name="PageUsage",+ fields=[+ (+ "id",+ models.UUIDField(+ db_comment="Primary key for the usage entry, automatically generated UUID",+ default=uuid.uuid4,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ ("organization_id", models.CharField(default="mock_org")),+ (+ "file_name",+ models.CharField(+ blank=True, db_comment="Name of the file", max_length=255+ ),+ ),+ (+ "file_type",+ models.CharField(+ blank=True, db_comment="Mime type of file", max_length=128+ ),+ ),+ (+ "run_id",+ models.CharField(+ blank=True, db_comment="Identifier for the run", max_length=255+ ),+ ),+ (+ "pages_processed",+ models.IntegerField(+ db_comment="Total number of pages in the document"+ ),+ ),+ (+ "file_size",+ models.BigIntegerField(db_comment="Size of the the file"),+ ),+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ],+ options={+ "db_table": "page_usage",+ "indexes": [+ models.Index(+ fields=["organization_id"], name="page_usage_organiz_b56749_idx"+ )+ ],+ },+ ),+ ]diff --git a/backend/account_usage/migrations/__init__.py b/backend/account_usage/migrations/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/account_usage/models.py b/backend/account_usage/models.pynew file mode 100644index 000000000..5fc951b10--- /dev/null+++ b/backend/account_usage/models.py@@ -0,0 +1,47 @@+import uuid++from django.db import models+++class PageUsage(models.Model):+ id = models.UUIDField(+ primary_key=True,+ default=uuid.uuid4,+ editable=False,+ db_comment="Primary key for the usage entry, automatically generated UUID",+ )++ organization_id = models.CharField(+ default="mock_org",+ null=False,+ blank=False,+ )+ file_name = models.CharField(+ max_length=255,+ blank=True,+ db_comment="Name of the file",+ )+ file_type = models.CharField(+ max_length=128,+ blank=True,+ db_comment="Mime type of file",+ )++ run_id = models.CharField(+ max_length=255, blank=True, db_comment="Identifier for the run"+ )++ pages_processed = models.IntegerField(+ db_comment="Number of pages which got processed"+ )+ file_size = models.BigIntegerField(db_comment="Size of the the file")+ created_at = models.DateTimeField(auto_now_add=True)++ def __str__(self) -> str:+ return str(self.file_name)++ class Meta:+ db_table = "page_usage"+ indexes = [+ models.Index(fields=["organization_id"]),+ ]diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.pyindex a435ce034..d47788b57 100644--- a/backend/backend/settings/base.py+++ b/backend/backend/settings/base.py@@ -194,6 +194,7 @@ SHARED_APPS = ("corsheaders",# For the organization model"account",+ "account_usage",# Django apps should go below this line"django.contrib.admin","django.contrib.auth",diff --git a/platform-service/src/unstract/platform_service/constants.py b/platform-service/src/unstract/platform_service/constants.pyindex 8becd5fd5..75e1ad339 100644--- a/platform-service/src/unstract/platform_service/constants.py+++ b/platform-service/src/unstract/platform_service/constants.py@@ -4,6 +4,12 @@ class FeatureFlag:MULTI_TENANCY_V2 = "multi_tenancy_v2"+class DBTable:+ """Database tables."""++ PAGE_USAGE = "page_usage"++class DBTableV2:"""Database tables."""diff --git a/platform-service/src/unstract/platform_service/main.py b/platform-service/src/unstract/platform_service/main.pyindex a309aa5db..c2d26d71a 100644--- a/platform-service/src/unstract/platform_service/main.py+++ b/platform-service/src/unstract/platform_service/main.py@@ -9,7 +9,7 @@ import redisfrom cryptography.fernet import Fernetfrom dotenv import load_dotenvfrom flask import Flask, Request, Response, json, jsonify, make_response, request-from unstract.platform_service.constants import DBTableV2, FeatureFlag+from unstract.platform_service.constants import DBTable, DBTableV2, FeatureFlagfrom unstract.platform_service.exceptions import CustomExceptionfrom unstract.platform_service.helper import (AdapterInstanceRequestHelper,@@ -171,7 +171,9 @@ def validate_bearer_token(token: Optional[str]) -> bool:except Exception as e:app.logger.error(- f"Error while validating bearer token: {e}", stack_info=True, exc_info=True+ f"Error while validating bearer token: {e}",+ stack_info=True,+ exc_info=True,)return Falsereturn True@@ -182,6 +184,61 @@ def health_check() -> str:return "OK"[email protected]("/page-usage", methods=["POST"], endpoint="page_usage")+@authentication_middleware+def page_usage() -> Any:+ """Usage endpoint."""+ result: dict[str, Any] = {+ "status": "ERROR",+ "error": "",+ "unique_id": "",+ }+ payload: Optional[dict[Any, Any]] = request.json+ if not payload:+ result["error"] = INVALID_PAYLOAD+ return make_response(result, 400)++ bearer_token = get_token_from_auth_header(request)+ _, org_id = get_organization_from_bearer_token(bearer_token)++ page_count = payload.get("page_count", "")+ file_name = payload.get("file_name", "")+ file_size = payload.get("file_size", "")+ file_type = payload.get("file_type", "")+ run_id = payload.get("run_id", "")++ query = f"""+ INSERT INTO {DBTable.PAGE_USAGE} (id, organization_id, pages_processed,+ file_name, file_size, file_type, run_id, created_at)+ VALUES (%s, %s, %s, %s, %s, %s, %s, %s)+ """+ usage_id = uuid.uuid4()+ current_time = datetime.now()+ params = (+ usage_id,+ org_id,+ page_count,+ file_name,+ file_size,+ file_type,+ run_id,+ current_time,+ )++ try:+ with be_db.atomic() as transaction:+ be_db.execute_sql(query, params)+ transaction.commit()+ app.logger.info("Entry created with id %s for %s", usage_id, org_id)+ result["status"] = "OK"+ result["unique_id"] = usage_id+ return make_response(result, 200)+ except Exception as e:+ app.logger.error(f"Error while creating page usage entry: {e}")+ result["error"] = "Internal Server Error"+ return make_response(result, 500)++@app.route("/usage", methods=["POST"])@authentication_middlewaredef usage() -> Any: