Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

8.2 KB
Raw Blame
import json
import logging
import uuid
from typing import Any

from account_v2.models import User
from cryptography.fernet import Fernet, InvalidToken
from django.conf import settings
from django.contrib.contenttypes.fields import GenericRelation
from django.db import models
from django.db.models import QuerySet
from permissions.models import HasMembersMixin
from tenant_account_v2.models import OrganizationMember
from tenant_account_v2.organization_member_service import OrganizationMemberService
from tenant_account_v2.sharing_helpers import (
    resources_visible_via_groups,
    resources_visible_via_memberships,
)
from utils.exceptions import InvalidEncryptionKey
from utils.models.base_model import BaseModel, BaseModelManager
from utils.models.organization_mixin import (
    DefaultOrganizationManagerMixin,
    DefaultOrganizationMixin,
)

from unstract.sdk1.constants import AdapterTypes
from unstract.sdk1.exceptions import SdkError
from unstract.sdk1.llm import LLM

logger = logging.getLogger(__name__)

ADAPTER_NAME_SIZE = 128
VERSION_NAME_SIZE = 64
ADAPTER_ID_LENGTH = 128

logger = logging.getLogger(__name__)


class AdapterInstanceModelManager(DefaultOrganizationManagerMixin, BaseModelManager):
    def get_queryset(self) -> QuerySet[Any]:
        return super().get_queryset()

    def for_user(self, user: User) -> QuerySet[Any]:
        if getattr(user, "is_service_account", False):
            return self.get_queryset().filter(is_friction_less=False)

        if OrganizationMemberService.is_user_organization_admin(user):
            return self.get_queryset()

        user_group_ids = user.group_memberships.values_list("group_id", flat=True)
        group_shared_ids = resources_visible_via_groups(self.model, user_group_ids)
        member_ids = resources_visible_via_memberships(self.model, user)

        return self.get_queryset().filter(
            models.Q(pk__in=member_ids)
            | models.Q(shared_to_org=True)
            | models.Q(is_friction_less=True)
            | models.Q(pk__in=group_shared_ids)
        )


class AdapterInstance(HasMembersMixin, DefaultOrganizationMixin, BaseModel):
    id = models.UUIDField(
        primary_key=True,
        default=uuid.uuid4,
        editable=False,
        db_comment="Unique identifier for the Adapter Instance",
    )
    adapter_name = models.TextField(
        max_length=ADAPTER_NAME_SIZE,
        null=False,
        blank=False,
        db_comment="Name of the Adapter Instance",
    )
    adapter_id = models.CharField(
        max_length=ADAPTER_ID_LENGTH,
        default="",
        db_comment="Unique identifier of the Adapter",
    )

    # TODO to be removed once the migration for encryption
    adapter_metadata = models.JSONField(
        db_column="adapter_metadata",
        null=False,
        blank=False,
        default=dict,
        db_comment="JSON adapter metadata submitted by the user",
    )
    adapter_metadata_b = models.BinaryField(null=True)
    adapter_type = models.CharField(
        choices=[(tag.value, tag.name) for tag in AdapterTypes],
        db_comment="Type of adapter LLM/EMBEDDING/VECTOR_DB",
    )
    created_by = models.ForeignKey(
        User,
        on_delete=models.SET_NULL,
        related_name="adapters_created",
        null=True,
        blank=True,
    )
    modified_by = models.ForeignKey(
        User,
        on_delete=models.SET_NULL,
        related_name="adapters_modified",
        null=True,
        blank=True,
    )

    is_active = models.BooleanField(
        default=False,
        db_comment="Is the adapter instance currently being used",
    )
    shared_to_org = models.BooleanField(
        default=False,
        db_comment="Is the adapter shared to entire org",
    )

    is_friction_less = models.BooleanField(
        default=False,
        db_comment="Was the adapter created through frictionless onboarding",
    )

    # Can be used if the adapter usage gets exhausted
    # Can also be used in other possible scenarios in feature
    is_usable = models.BooleanField(
        default=True,
        db_comment="Is the Adpater Usable",
    )

    # Indicates if the adapter is available in SDK (not deprecated/removed)
    is_available = models.BooleanField(
        default=True,
        db_comment="Is the adapter available in SDK (not deprecated)",
    )

    # Stores metadata about deprecated adapters (reason, deprecation date, replacement info)
    deprecation_metadata = models.JSONField(
        null=True,
        blank=True,
        default=None,
        db_comment="Metadata about adapter deprecation (reason, date, replacement)",
    )

    # Owner + direct-viewer access lives here (UN-2202): OWNER / VIEWER rows in
    # the polymorphic ``ResourceMembership`` table. ``created_by`` is
    # audit-only; VIEWER rows succeed the former ``shared_users`` M2M.
    memberships = GenericRelation("tenant_account_v2.ResourceMembership")
    description = models.TextField(blank=True, null=True, default=None)

    # ``shared_groups`` is stored polymorphically in
    # ``tenant_account_v2.ResourceGroupShare``; the property preserves the
    # ergonomic read surface for DRF / existing callers.
    @property
    def shared_groups(self):
        from tenant_account_v2.sharing_helpers import get_resource_share_groups

        return get_resource_share_groups(self)

    objects = AdapterInstanceModelManager()

    class Meta:
        verbose_name = "adapter instance"
        verbose_name_plural = "adapter instances"
        db_table = "adapter_instance"
        constraints = [
            models.UniqueConstraint(
                fields=["adapter_name", "adapter_type", "organization"],
                name="unique_organization_adapter",
            ),
        ]
        # Backs the default org-scoped `-modified_at, pk` list ordering.
        indexes = [
            models.Index(
                fields=["organization", "-modified_at"],
                name="adapter_org_modified_idx",
            ),
        ]

    def create_adapter(self) -> None:
        encryption_secret: str = settings.ENCRYPTION_KEY
        f: Fernet = Fernet(encryption_secret.encode("utf-8"))

        self.adapter_metadata_b = f.encrypt(
            json.dumps(self.adapter_metadata).encode("utf-8")
        )
        self.adapter_metadata = {}

        self.save()

    @property
    def metadata(self) -> Any:
        try:
            encryption_secret: str = settings.ENCRYPTION_KEY
            f: Fernet = Fernet(encryption_secret.encode("utf-8"))

            adapter_metadata = json.loads(
                f.decrypt(bytes(self.adapter_metadata_b).decode("utf-8"))
            )
        except InvalidToken:
            raise InvalidEncryptionKey(entity=InvalidEncryptionKey.Entity.ADAPTER)
        return adapter_metadata

    def get_context_window_size(self) -> int:
        # Get the adapter_instance
        try:
            return LLM.get_context_window_size(self.adapter_id, self.metadata)
        except SdkError as e:
            logger.warning(f"Unable to retrieve context window size: {e}")
        return 0


class UserDefaultAdapter(BaseModel):
    organization_member = models.OneToOneField(
        OrganizationMember,
        on_delete=models.CASCADE,
        default=None,
        null=True,
        db_comment="Foreign key reference to the OrganizationMember model.",
        related_name="default_adapters",
    )
    default_llm_adapter = models.ForeignKey(
        AdapterInstance,
        on_delete=models.SET_NULL,
        null=True,
        related_name="user_default_llm_adapter",
    )
    default_embedding_adapter = models.ForeignKey(
        AdapterInstance,
        on_delete=models.SET_NULL,
        null=True,
        related_name="user_default_embedding_adapter",
    )
    default_vector_db_adapter = models.ForeignKey(
        AdapterInstance,
        on_delete=models.SET_NULL,
        null=True,
        related_name="user_default_vector_db_adapter",
    )

    default_x2text_adapter = models.ForeignKey(
        AdapterInstance,
        on_delete=models.SET_NULL,
        null=True,
        related_name="user_default_x2text_adapter",
    )

    class Meta:
        verbose_name = "Default Adapter for Organization User"
        verbose_name_plural = "Default Adapters for Organization Users"
        db_table = "default_organization_user_adapter"