LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
import logging
import uuid
from typing import Any
from account_v2.models import User
from django.contrib.contenttypes.fields import GenericRelation
from django.db import models
from django.db.models import Q
from django.db.models.signals import post_delete
from django.dispatch import receiver
from permissions.models import HasMembersMixin
from pipeline_v2.models import Pipeline
from tenant_account_v2.organization_member_service import OrganizationMemberService
from tenant_account_v2.sharing_helpers import (
resources_visible_via_groups,
resources_visible_via_memberships,
)
from utils.models.base_model import BaseModel, BaseModelManager
from utils.models.organization_mixin import (
DefaultOrganizationManagerMixin,
DefaultOrganizationMixin,
)
from utils.user_context import UserContext
from workflow_manager.workflow_v2.models.workflow import Workflow
from api_v2.constants import ApiExecution
logger = logging.getLogger(__name__)
API_NAME_MAX_LENGTH = 30
DESCRIPTION_MAX_LENGTH = 255
API_ENDPOINT_MAX_LENGTH = 255
class APIDeploymentModelManager(DefaultOrganizationManagerMixin, BaseModelManager):
def for_user(self, user):
"""Filter API deployments that the user can access:
- API deployments created by the user
- API deployments shared with the user
- API deployments shared with the entire organization
- API deployments shared with any group the user is a member of
- Service accounts and org admins see all org resources
"""
if getattr(user, "is_service_account", False):
return self.all()
if OrganizationMemberService.is_user_organization_admin(user):
return self.all()
user_group_ids = user.group_memberships.values_list("group_id", flat=True)
group_shared_ids = resources_visible_via_groups(self.model, user_group_ids)
member_ids = resources_visible_via_memberships(self.model, user)
return self.filter(
Q(pk__in=member_ids) # Owner or direct viewer (created_by audit-only)
| Q(shared_to_org=True) # Shared to entire organization
| Q(pk__in=group_shared_ids) # Shared via group membership
).distinct()
class APIDeployment(HasMembersMixin, DefaultOrganizationMixin, BaseModel):
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
display_name = models.CharField(
max_length=API_NAME_MAX_LENGTH,
default="default api",
db_comment="User-given display name for the API.",
)
description = models.CharField(
max_length=DESCRIPTION_MAX_LENGTH,
blank=True,
default="",
db_comment="User-given description for the API.",
)
workflow = models.ForeignKey(
Workflow,
on_delete=models.CASCADE,
db_comment="Foreign key reference to the Workflow model.",
related_name="apis",
)
is_active = models.BooleanField(
default=True,
db_comment="Flag indicating whether the API is active or not.",
)
# TODO: Implement dynamic generation of API endpoints for API deployments
# instead of persisting them in the database.
api_endpoint = models.CharField(
max_length=API_ENDPOINT_MAX_LENGTH,
unique=True,
editable=False,
db_comment="URL endpoint for the API deployment.",
)
api_name = models.CharField(
max_length=API_NAME_MAX_LENGTH,
default=uuid.uuid4,
db_comment="Short name for the API deployment.",
)
created_by = models.ForeignKey(
User,
on_delete=models.SET_NULL,
related_name="apis_created",
null=True,
blank=True,
editable=False,
)
modified_by = models.ForeignKey(
User,
on_delete=models.SET_NULL,
related_name="apis_modified",
null=True,
blank=True,
editable=False,
)
# Sharing fields
shared_to_org = models.BooleanField(
default=False,
db_comment="Whether this API deployment is shared with the entire organization",
)
# ``shared_groups`` is stored polymorphically in
# ``tenant_account_v2.ResourceGroupShare``; the property preserves the
# ergonomic read surface for DRF / existing callers.
@property
def shared_groups(self):
from tenant_account_v2.sharing_helpers import get_resource_share_groups
return get_resource_share_groups(self)
# Owner + direct-viewer access lives here (UN-2202): OWNER / VIEWER rows in
# the polymorphic ``ResourceMembership`` table. ``created_by`` is
# audit-only; VIEWER rows succeed the former ``shared_users`` M2M.
memberships = GenericRelation("tenant_account_v2.ResourceMembership")
# Manager
objects = APIDeploymentModelManager()
@property
def api_key_data(self):
return {"api": self.id, "description": f"API Key for {self.api_name}"}
def __str__(self) -> str:
return f"{self.id} - {self.display_name}"
def save(self, *args: Any, **kwargs: Any) -> None:
"""Save hook to update api_endpoint and enforce constraints.
Custom save hook for updating the 'api_endpoint' based on
'api_name'. If the instance is being updated, it checks for
changes in 'api_name' and adjusts 'api_endpoint'
accordingly. If the instance is new, 'api_endpoint' is set
based on 'api_name' and the current database schema.
Also enforces one API deployment per workflow constraint for new deployments.
"""
organization_id = UserContext.get_organization_identifier()
# Update api_endpoint logic
if self.pk is not None:
try:
original = APIDeployment.objects.get(pk=self.pk)
if original.api_name != self.api_name:
self.api_endpoint = (
f"{ApiExecution.PATH}/{organization_id}/{self.api_name}/"
)
except APIDeployment.DoesNotExist:
self.api_endpoint = (
f"{ApiExecution.PATH}/{organization_id}/{self.api_name}/"
)
else:
# New instance - set api_endpoint
self.api_endpoint = f"{ApiExecution.PATH}/{organization_id}/{self.api_name}/"
super().save(*args, **kwargs)
class Meta:
verbose_name = "Api Deployment"
verbose_name_plural = "Api Deployments"
db_table = "api_deployment"
constraints = [
models.UniqueConstraint(
fields=["api_name", "organization"],
name="unique_api_name",
),
]
class OrganizationRateLimit(DefaultOrganizationMixin, BaseModel):
"""Model to store organization-specific API deployment rate limits."""
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
concurrent_request_limit = models.IntegerField(
default=5,
db_comment="Maximum number of concurrent API deployment requests allowed for this organization",
)
def __str__(self) -> str:
return f"{self.organization} - Limit: {self.concurrent_request_limit}"
def save(self, *args, **kwargs):
"""Save and automatically clear cache."""
super().save(*args, **kwargs)
self._clear_cache()
def _clear_cache(self):
"""Clear cached limit for this organization."""
from django.core.cache import cache
from api_v2.rate_limit_constants import RateLimitKeys
org_id = str(self.organization.organization_id)
cache_key = RateLimitKeys.get_org_limit_cache_key(org_id)
cache.delete(cache_key)
logger.info(f"Cleared rate limit cache after save: org {org_id}")
class Meta:
verbose_name = "Organization Rate Limit"
verbose_name_plural = "Organization Rate Limits"
db_table = "organization_rate_limit"
constraints = [
models.UniqueConstraint(
fields=["organization"],
name="unique_org_rate_limit",
),
]
class APIKey(BaseModel):
id = models.UUIDField(
primary_key=True,
editable=False,
default=uuid.uuid4,
db_comment="Unique identifier for the API key.",
)
api_key = models.UUIDField(
default=uuid.uuid4,
editable=False,
unique=True,
db_comment="Actual key UUID.",
)
api = models.ForeignKey(
APIDeployment,
on_delete=models.CASCADE,
null=True,
blank=True,
db_comment="Foreign key reference to the APIDeployment model.",
related_name="api_keys",
)
pipeline = models.ForeignKey(
Pipeline,
on_delete=models.CASCADE,
null=True,
blank=True,
db_comment="Foreign key reference to the Pipeline model.",
)
description = models.CharField(
max_length=DESCRIPTION_MAX_LENGTH,
null=True,
db_comment="Description of the API key.",
)
is_active = models.BooleanField(
default=True,
db_comment="Flag indicating whether the API key is active or not.",
)
created_by = models.ForeignKey(
User,
on_delete=models.SET_NULL,
related_name="api_keys_created",
null=True,
blank=True,
editable=False,
)
modified_by = models.ForeignKey(
User,
on_delete=models.SET_NULL,
related_name="api_keys_modified",
null=True,
blank=True,
editable=False,
)
def __str__(self) -> str:
if self.api:
api_name = self.api.api_name
elif self.pipeline:
api_name = self.pipeline.pipeline_name
else:
api_name = None
api_key = self.api_key if self.api_key else None
return f"{api_name} - {self.id} - {api_key}"
class Meta:
verbose_name = "Api Deployment key"
verbose_name_plural = "Api Deployment keys"
db_table = "api_deployment_key"
# Signal handlers for OrganizationRateLimit
@receiver(post_delete, sender=OrganizationRateLimit)
def clear_org_rate_limit_cache_on_delete(sender, instance, **kwargs):
"""Clear cache when rate limit record is deleted."""
from django.core.cache import cache
from api_v2.rate_limit_constants import RateLimitKeys
org_id = str(instance.organization.organization_id)
cache_key = RateLimitKeys.get_org_limit_cache_key(org_id)
cache.delete(cache_key)
logger.info(f"Cleared rate limit cache after delete: org {org_id}")