LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
import logging
import uuid
from typing import Any
from account_v2.models import User
from connector_auth_v2.models import ConnectorAuth
from connector_processor.connector_processor import ConnectorProcessor
from connector_processor.constants import ConnectorKeys
from django.contrib.contenttypes.fields import GenericRelation
from django.db import models
from permissions.models import HasMembersMixin
from tenant_account_v2.organization_member_service import OrganizationMemberService
from tenant_account_v2.sharing_helpers import (
resources_visible_via_groups,
resources_visible_via_memberships,
)
from utils.fields import EncryptedBinaryField
from utils.models.base_model import BaseModel, BaseModelManager
from utils.models.organization_mixin import (
DefaultOrganizationManagerMixin,
DefaultOrganizationMixin,
)
from backend.constants import FieldLengthConstants as FLC
CONNECTOR_NAME_SIZE = 128
VERSION_NAME_SIZE = 64
logger = logging.getLogger(__name__)
class ConnectorInstanceModelManager(DefaultOrganizationManagerMixin, BaseModelManager):
def get_queryset(self) -> models.QuerySet:
return super().get_queryset()
def for_user(self, user: User) -> models.QuerySet:
if getattr(user, "is_service_account", False):
return self.all()
if OrganizationMemberService.is_user_organization_admin(user):
return self.all()
user_group_ids = user.group_memberships.values_list("group_id", flat=True)
group_shared_ids = resources_visible_via_groups(self.model, user_group_ids)
member_ids = resources_visible_via_memberships(self.model, user)
return self.get_queryset().filter(
models.Q(pk__in=member_ids)
| models.Q(shared_to_org=True)
| models.Q(pk__in=group_shared_ids)
)
class ConnectorInstance(HasMembersMixin, DefaultOrganizationMixin, BaseModel):
class ConnectorType(models.TextChoices):
INPUT = "INPUT", "Input"
OUTPUT = "OUTPUT", "Output"
class ConnectorMode(models.TextChoices):
UNKNOWN = "UNKNOWN", "UNKNOWN"
FILE_SYSTEM = "FILE_SYSTEM", "FILE_SYSTEM"
DATABASE = "DATABASE", "DATABASE"
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
connector_name = models.TextField(
max_length=CONNECTOR_NAME_SIZE, null=False, blank=False
)
connector_id = models.CharField(max_length=FLC.CONNECTOR_ID_LENGTH, default="")
connector_metadata = EncryptedBinaryField(null=True)
connector_version = models.CharField(max_length=VERSION_NAME_SIZE, default="")
# TODO: handle connector_auth cascade deletion
connector_auth = models.ForeignKey(
ConnectorAuth,
on_delete=models.SET_NULL,
null=True,
blank=True,
related_name="connector_instances",
)
# TODO: Remove unused connector_mode field, resolved from the ConnectorKit instead
connector_mode = models.CharField(
choices=ConnectorMode.choices,
default=ConnectorMode.UNKNOWN,
db_comment="0: UNKNOWN, 1: FILE_SYSTEM, 2: DATABASE",
null=True,
blank=True,
)
created_by = models.ForeignKey(
User,
on_delete=models.SET_NULL,
related_name="connectors_created",
null=True,
blank=True,
)
modified_by = models.ForeignKey(
User,
on_delete=models.SET_NULL,
related_name="connectors_modified",
null=True,
blank=True,
)
shared_to_org = models.BooleanField(
default=False,
db_comment="Is the connector shared to entire org",
)
# ``shared_groups`` is stored polymorphically in
# ``tenant_account_v2.ResourceGroupShare``; the property preserves the
# ergonomic read surface for DRF / existing callers.
@property
def shared_groups(self):
from tenant_account_v2.sharing_helpers import get_resource_share_groups
return get_resource_share_groups(self)
# Owner + direct-viewer access lives here (UN-2202): OWNER / VIEWER rows in
# the polymorphic ``ResourceMembership`` table. ``created_by`` is
# audit-only; VIEWER rows succeed the former ``shared_users`` M2M.
memberships = GenericRelation("tenant_account_v2.ResourceMembership")
objects = ConnectorInstanceModelManager()
@staticmethod
def supportsOAuth(connector_id: str) -> bool:
return bool(
ConnectorProcessor.get_connector_data_with_key(
connector_id, ConnectorKeys.OAUTH
)
)
def __str__(self) -> str:
return (
f"Connector(pk={self.id}, name={self.connector_name}, ID={self.connector_id})"
)
@property
def metadata(self) -> Any:
"""Decrypt and return the connector metadata as a dictionary.
This property handles the decryption of the connector_metadata,
converting it to bytes if necessary, and then loading the decrypted
JSON string into a dictionary.
Returns:
dict: The decrypted connector metadata.
.. deprecated::
This property is deprecated. Use `connector_metadata` field directly instead.
This property will be removed in a future version.
"""
import warnings
warnings.warn(
"The 'metadata' property is deprecated. Use 'connector_metadata' field directly instead. "
"This property will be removed in a future version.",
DeprecationWarning,
stacklevel=2,
)
return self.connector_metadata
class Meta:
db_table = "connector_instance"
verbose_name = "Connector Instance"
verbose_name_plural = "Connector Instances"
constraints = [
models.UniqueConstraint(
fields=["connector_name", "organization"],
name="unique_organization_connector",
),
]
# Backs the default org-scoped `-modified_at, pk` list ordering.
indexes = [
models.Index(
fields=["organization", "-modified_at"],
name="connector_org_modified_idx",
),
]