Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Feat/usage reporting page (#547)

Rahul Johnyfcc771efd94e
diff --git a/backend/account_usage/__init__.py b/backend/account_usage/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/account_usage/apps.py b/backend/account_usage/apps.py
new file mode 100644
index 000000000..09d4944b3
--- /dev/null
+++ b/backend/account_usage/apps.py
@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class AccountUsageConfig(AppConfig):
+ default_auto_field = "django.db.models.BigAutoField"
+ name = "account_usage"
diff --git a/backend/account_usage/migrations/0001_initial.py b/backend/account_usage/migrations/0001_initial.py
new file mode 100644
index 000000000..11415be66
--- /dev/null
+++ b/backend/account_usage/migrations/0001_initial.py
@@ -0,0 +1,68 @@
+# Generated by Django 4.2.1 on 2024-08-05 06:10
+
+import uuid
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = []
+
+ operations = [
+ migrations.CreateModel(
+ name="PageUsage",
+ fields=[
+ (
+ "id",
+ models.UUIDField(
+ db_comment="Primary key for the usage entry, automatically generated UUID",
+ default=uuid.uuid4,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ ),
+ ),
+ ("organization_id", models.CharField(default="mock_org")),
+ (
+ "file_name",
+ models.CharField(
+ blank=True, db_comment="Name of the file", max_length=255
+ ),
+ ),
+ (
+ "file_type",
+ models.CharField(
+ blank=True, db_comment="Mime type of file", max_length=128
+ ),
+ ),
+ (
+ "run_id",
+ models.CharField(
+ blank=True, db_comment="Identifier for the run", max_length=255
+ ),
+ ),
+ (
+ "pages_processed",
+ models.IntegerField(
+ db_comment="Total number of pages in the document"
+ ),
+ ),
+ (
+ "file_size",
+ models.BigIntegerField(db_comment="Size of the the file"),
+ ),
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ],
+ options={
+ "db_table": "page_usage",
+ "indexes": [
+ models.Index(
+ fields=["organization_id"], name="page_usage_organiz_b56749_idx"
+ )
+ ],
+ },
+ ),
+ ]
diff --git a/backend/account_usage/migrations/__init__.py b/backend/account_usage/migrations/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/account_usage/models.py b/backend/account_usage/models.py
new file mode 100644
index 000000000..5fc951b10
--- /dev/null
+++ b/backend/account_usage/models.py
@@ -0,0 +1,47 @@
+import uuid
+
+from django.db import models
+
+
+class PageUsage(models.Model):
+ id = models.UUIDField(
+ primary_key=True,
+ default=uuid.uuid4,
+ editable=False,
+ db_comment="Primary key for the usage entry, automatically generated UUID",
+ )
+
+ organization_id = models.CharField(
+ default="mock_org",
+ null=False,
+ blank=False,
+ )
+ file_name = models.CharField(
+ max_length=255,
+ blank=True,
+ db_comment="Name of the file",
+ )
+ file_type = models.CharField(
+ max_length=128,
+ blank=True,
+ db_comment="Mime type of file",
+ )
+
+ run_id = models.CharField(
+ max_length=255, blank=True, db_comment="Identifier for the run"
+ )
+
+ pages_processed = models.IntegerField(
+ db_comment="Number of pages which got processed"
+ )
+ file_size = models.BigIntegerField(db_comment="Size of the the file")
+ created_at = models.DateTimeField(auto_now_add=True)
+
+ def __str__(self) -> str:
+ return str(self.file_name)
+
+ class Meta:
+ db_table = "page_usage"
+ indexes = [
+ models.Index(fields=["organization_id"]),
+ ]
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py
index a435ce034..d47788b57 100644
--- a/backend/backend/settings/base.py
+++ b/backend/backend/settings/base.py
@@ -194,6 +194,7 @@ SHARED_APPS = (
"corsheaders",
# For the organization model
"account",
+ "account_usage",
# Django apps should go below this line
"django.contrib.admin",
"django.contrib.auth",
diff --git a/platform-service/src/unstract/platform_service/constants.py b/platform-service/src/unstract/platform_service/constants.py
index 8becd5fd5..75e1ad339 100644
--- a/platform-service/src/unstract/platform_service/constants.py
+++ b/platform-service/src/unstract/platform_service/constants.py
@@ -4,6 +4,12 @@ class FeatureFlag:
MULTI_TENANCY_V2 = "multi_tenancy_v2"
+class DBTable:
+ """Database tables."""
+
+ PAGE_USAGE = "page_usage"
+
+
class DBTableV2:
"""Database tables."""
diff --git a/platform-service/src/unstract/platform_service/main.py b/platform-service/src/unstract/platform_service/main.py
index a309aa5db..c2d26d71a 100644
--- a/platform-service/src/unstract/platform_service/main.py
+++ b/platform-service/src/unstract/platform_service/main.py
@@ -9,7 +9,7 @@ import redis
from cryptography.fernet import Fernet
from dotenv import load_dotenv
from flask import Flask, Request, Response, json, jsonify, make_response, request
-from unstract.platform_service.constants import DBTableV2, FeatureFlag
+from unstract.platform_service.constants import DBTable, DBTableV2, FeatureFlag
from unstract.platform_service.exceptions import CustomException
from unstract.platform_service.helper import (
AdapterInstanceRequestHelper,
@@ -171,7 +171,9 @@ def validate_bearer_token(token: Optional[str]) -> bool:
except Exception as e:
app.logger.error(
- f"Error while validating bearer token: {e}", stack_info=True, exc_info=True
+ f"Error while validating bearer token: {e}",
+ stack_info=True,
+ exc_info=True,
)
return False
return True
@@ -182,6 +184,61 @@ def health_check() -> str:
return "OK"
[email protected]("/page-usage", methods=["POST"], endpoint="page_usage")
+@authentication_middleware
+def page_usage() -> Any:
+ """Usage endpoint."""
+ result: dict[str, Any] = {
+ "status": "ERROR",
+ "error": "",
+ "unique_id": "",
+ }
+ payload: Optional[dict[Any, Any]] = request.json
+ if not payload:
+ result["error"] = INVALID_PAYLOAD
+ return make_response(result, 400)
+
+ bearer_token = get_token_from_auth_header(request)
+ _, org_id = get_organization_from_bearer_token(bearer_token)
+
+ page_count = payload.get("page_count", "")
+ file_name = payload.get("file_name", "")
+ file_size = payload.get("file_size", "")
+ file_type = payload.get("file_type", "")
+ run_id = payload.get("run_id", "")
+
+ query = f"""
+ INSERT INTO {DBTable.PAGE_USAGE} (id, organization_id, pages_processed,
+ file_name, file_size, file_type, run_id, created_at)
+ VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
+ """
+ usage_id = uuid.uuid4()
+ current_time = datetime.now()
+ params = (
+ usage_id,
+ org_id,
+ page_count,
+ file_name,
+ file_size,
+ file_type,
+ run_id,
+ current_time,
+ )
+
+ try:
+ with be_db.atomic() as transaction:
+ be_db.execute_sql(query, params)
+ transaction.commit()
+ app.logger.info("Entry created with id %s for %s", usage_id, org_id)
+ result["status"] = "OK"
+ result["unique_id"] = usage_id
+ return make_response(result, 200)
+ except Exception as e:
+ app.logger.error(f"Error while creating page usage entry: {e}")
+ result["error"] = "Internal Server Error"
+ return make_response(result, 500)
+
+
@app.route("/usage", methods=["POST"])
@authentication_middleware
def usage() -> Any: