LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
import logging
import uuid
from typing import Any
from account_v2.models import User
from django.db import IntegrityError
from django.db.models import ProtectedError, QuerySet
from django.http import HttpRequest
from django.http.response import HttpResponse
from permissions.membership_views import OwnerManagementMixin
from permissions.permission import (
IsFrictionLessAdapter,
IsFrictionLessAdapterDelete,
IsOwner,
IsOwnerOrSharedUserOrSharedToOrg,
)
from permissions.resource_share_views import ResourceShareManagementMixin
from platform_audit.actions import Action, TargetType
from platform_audit.recorder import (
BEFORE_UNAVAILABLE,
best_effort,
diff,
record_platform_event,
)
from platform_audit.resources import record_resource_event, resource_ref
from plugins import get_plugin
from rest_framework import status
from rest_framework.decorators import action
from rest_framework.request import Request
from rest_framework.response import Response
from rest_framework.serializers import ModelSerializer
from rest_framework.versioning import URLPathVersioning
from rest_framework.viewsets import GenericViewSet, ModelViewSet
from tenant_account_v2.capabilities import Capability, enforce_capability
from tenant_account_v2.organization_member_service import OrganizationMemberService
from tool_instance_v2.models import ToolInstance
from utils.filtering import FilterHelper
from utils.pagination import OptionalPagination
from utils.user_context import UserContext
from adapter_processor_v2.adapter_processor import AdapterProcessor
from adapter_processor_v2.constants import AdapterKeys
from adapter_processor_v2.deprecated_adapters import (
get_deprecation_message,
is_adapter_deprecated,
)
from adapter_processor_v2.exceptions import (
CannotDeleteDefaultAdapter,
DeleteAdapterInUseError,
DeprecatedAdapter,
DuplicateAdapterNameError,
IdIsMandatory,
InValidType,
)
from adapter_processor_v2.serializers import (
AdapterInfoSerializer,
AdapterInstanceSerializer,
AdapterListSerializer,
DefaultAdapterSerializer,
SharedUserListSerializer,
TestAdapterSerializer,
UserDefaultAdapterSerializer,
)
from .constants import AdapterKeys as constant
from .models import AdapterInstance, UserDefaultAdapter
notification_plugin = get_plugin("notification")
logger = logging.getLogger(__name__)
class DefaultAdapterViewSet(ModelViewSet):
versioning_class = URLPathVersioning
serializer_class = DefaultAdapterSerializer
def configure_default_triad(
self, request: Request, *args: tuple[Any], **kwargs: dict[str, Any]
) -> HttpResponse:
serializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
# Convert request data to json
default_triad = request.data
before = _default_adapter_names(request.user)
AdapterProcessor.set_default_triad(default_triad, request.user)
_record_defaults_changed(request, before)
return Response(status=status.HTTP_200_OK)
def get_default_triad(
self, request: Request, *args: tuple[Any], **kwargs: dict[str, Any]
) -> HttpResponse:
try:
organization_member = OrganizationMemberService.get_user_by_id(
request.user.id
)
user_default_adapter = UserDefaultAdapter.objects.get(
organization_member=organization_member
)
serializer = UserDefaultAdapterSerializer(user_default_adapter).data
return Response(serializer)
except UserDefaultAdapter.DoesNotExist:
# Handle the case when no records are found
return Response(status=status.HTTP_200_OK, data={})
class AdapterViewSet(GenericViewSet):
versioning_class = URLPathVersioning
serializer_class = TestAdapterSerializer
def list(
self, request: Request, *args: tuple[Any], **kwargs: dict[str, Any]
) -> HttpResponse:
if request.method == "GET":
adapter_type = request.GET.get(AdapterKeys.ADAPTER_TYPE)
if (
adapter_type == AdapterKeys.LLM
or adapter_type == AdapterKeys.EMBEDDING
or adapter_type == AdapterKeys.VECTOR_DB
or adapter_type == AdapterKeys.X2TEXT
or adapter_type == AdapterKeys.OCR
):
json_schema = AdapterProcessor.get_all_supported_adapters(
type=adapter_type, user_email=request.user.email
)
return Response(json_schema, status=status.HTTP_200_OK)
else:
raise InValidType
def get_adapter_schema(
self, request: Request, *args: tuple[Any], **kwargs: dict[str, Any]
) -> HttpResponse:
if request.method == "GET":
adapter_name = request.GET.get(AdapterKeys.ID)
if adapter_name is None or adapter_name == "":
raise IdIsMandatory()
json_schema = AdapterProcessor.get_json_schema(adapter_id=adapter_name)
return Response(data=json_schema, status=status.HTTP_200_OK)
def test(self, request: Request) -> Response:
"""Tests the connector against the credentials passed."""
serializer: AdapterInstanceSerializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
adapter_id = serializer.validated_data.get(AdapterKeys.ADAPTER_ID)
if is_adapter_deprecated(adapter_id):
raise DeprecatedAdapter(get_deprecation_message(adapter_id))
adapter_metadata = serializer.validated_data.get(AdapterKeys.ADAPTER_METADATA)
adapter_metadata[AdapterKeys.ADAPTER_TYPE] = serializer.validated_data.get(
AdapterKeys.ADAPTER_TYPE
)
test_result = AdapterProcessor.test_adapter(
adapter_id=adapter_id, adapter_metadata=adapter_metadata
)
return Response(
{AdapterKeys.IS_VALID: test_result},
status=status.HTTP_200_OK,
)
class AdapterInstanceViewSet(
OwnerManagementMixin, ResourceShareManagementMixin, ModelViewSet
):
serializer_class = AdapterInstanceSerializer
pagination_class = OptionalPagination
# `pk` tiebreaker keeps paging deterministic when modified_at collides.
ordering = ["-modified_at", "pk"]
ordering_fields = ["adapter_name", "created_at", "modified_at"]
notification_resource_name_field = "adapter_name"
def get_notification_resource_type(self, resource: Any) -> str | None:
if not notification_plugin:
return None
from tenant_account_v2.notification_resource_types import (
adapter_notification_type,
)
return adapter_notification_type(resource.adapter_type)
def get_permissions(self) -> list[Any]:
# Frictionless adapters: hidden from non-owners (update/retrieve),
# deletable by any org member. Others use owner / shared-user gating.
if self.action in ["update", "partial_update", "retrieve"]:
return [IsFrictionLessAdapter()]
if self.action == "destroy":
return [IsFrictionLessAdapterDelete()]
if self.action in [
"list_of_shared_users",
"adapter_info",
"share",
"effective_members",
]:
return [IsOwnerOrSharedUserOrSharedToOrg()]
return [IsOwner()]
def get_queryset(self) -> QuerySet | None:
# Avoid per-row queries for owner/co-owner + creator fields in list views
queryset = (
AdapterInstance.objects.for_user(self.request.user)
.select_related("created_by")
.prefetch_related("memberships__user")
)
if filter_args := FilterHelper.build_filter_args(
self.request,
constant.ADAPTER_TYPE,
constant.ADAPTER_NAME,
):
queryset = queryset.filter(**filter_args)
search = self.request.query_params.get("search")
if search:
from django.db.models import Q
from tenant_account_v2.sharing_helpers import (
resources_matching_owner_search,
)
queryset = queryset.filter(
Q(adapter_name__icontains=search)
| Q(pk__in=resources_matching_owner_search(queryset.model, search))
)
return queryset
def get_serializer_class(
self,
) -> ModelSerializer:
if self.action == "list":
return AdapterListSerializer
return AdapterInstanceSerializer
@staticmethod
def _enforce_adapter_creation_capability(request: Any, adapter_type: str) -> None:
"""Capability permissions (UN-3584, UN-4399): LLM adapters fall under
"Create LLM adapters", every other type under "Create other adapters".
Service accounts bypass; the defaults keep creation open for everyone.
"""
enforce_capability(
request,
Capability.CREATE_LLM_ADAPTERS
if adapter_type == AdapterKeys.LLM
else Capability.CREATE_OTHER_ADAPTERS,
)
def create(self, request: Any) -> Response:
serializer = self.get_serializer(data=request.data)
use_platform_unstract_key = False
adapter_metadata = request.data.get(AdapterKeys.ADAPTER_METADATA)
if adapter_metadata and adapter_metadata.get(
AdapterKeys.PLATFORM_PROVIDED_UNSTRACT_KEY, False
):
use_platform_unstract_key = True
# Deprecated adapter_ids are rejected in AdapterInstanceSerializer.validate,
# which also covers update/partial_update.
serializer.is_valid(raise_exception=True)
adapter_type = serializer.validated_data.get(AdapterKeys.ADAPTER_TYPE)
self._enforce_adapter_creation_capability(request, adapter_type)
try:
if adapter_type == AdapterKeys.X2TEXT and use_platform_unstract_key:
adapter_metadata_b = serializer.validated_data.get(
AdapterKeys.ADAPTER_METADATA_B
)
adapter_metadata_b = AdapterProcessor.update_adapter_metadata(
adapter_metadata_b
)
# Update the validated data with the new adapter_metadata
serializer.validated_data[AdapterKeys.ADAPTER_METADATA_B] = (
adapter_metadata_b
)
# Bind the adapter to the request-scoped organization, overriding any
# client-supplied `organization` in the payload (the serializer
# exposes it via fields="__all__"). This keeps the row's org
# consistent with the org the controlled-mode check above evaluated,
# so the per-org restriction can't be sidestepped via the payload.
instance = serializer.save(organization=UserContext.get_organization())
# ``created_by`` is audit-only; the creator's access flows through
# an OWNER membership row (UN-2202 co-owners).
instance.grant_owner(request.user)
organization_member = OrganizationMemberService.get_user_by_id(
request.user.id
)
# Check to see if there is a default configured
# for this adapter_type and for the current user
(
user_default_adapter,
created,
) = UserDefaultAdapter.objects.get_or_create(
organization_member=organization_member
)
if (adapter_type == AdapterKeys.LLM) and (
not user_default_adapter.default_llm_adapter
):
user_default_adapter.default_llm_adapter = instance
elif (adapter_type == AdapterKeys.EMBEDDING) and (
not user_default_adapter.default_embedding_adapter
):
user_default_adapter.default_embedding_adapter = instance
elif (adapter_type == AdapterKeys.VECTOR_DB) and (
not user_default_adapter.default_vector_db_adapter
):
user_default_adapter.default_vector_db_adapter = instance
elif (adapter_type == AdapterKeys.X2TEXT) and (
not user_default_adapter.default_x2text_adapter
):
user_default_adapter.default_x2text_adapter = instance
organization_member = OrganizationMemberService.get_user_by_id(
request.user.id
)
user_default_adapter.organization_member = organization_member
user_default_adapter.save()
except IntegrityError:
raise DuplicateAdapterNameError(
name=serializer.validated_data.get(AdapterKeys.ADAPTER_NAME)
)
record_resource_event(request, Action.RESOURCE_CREATED, instance)
headers = self.get_success_headers(serializer.data)
return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers)
@staticmethod
def _adapter_used_in_tool_instance(adapter: AdapterInstance) -> bool:
"""True if any workflow tool instance in the adapter's org references
it via metadata. These refs are JSON values (post lazy-migration the
adapter id; before it, the adapter name), so no FK protects them.
"""
needles = {str(adapter.id), adapter.adapter_name}
# metadata is free-form JSON: walk every nested value so a reference
# nested in a sub-object isn't missed, and tolerate non-dict payloads
# (list/scalar/None) that would otherwise blow up on .values().
def contains_ref(value: Any) -> bool:
if isinstance(value, str):
return value in needles
if isinstance(value, dict):
return any(contains_ref(v) for v in value.values())
if isinstance(value, list):
return any(contains_ref(v) for v in value)
return False
# Linear scan over the org's tool instances — acceptable for an
# infrequent, interactive delete.
metadatas = ToolInstance.objects.filter(
workflow__organization=adapter.organization
).values_list("metadata", flat=True)
return any(contains_ref(metadata) for metadata in metadatas)
def destroy(
self, request: Request, *args: tuple[Any], **kwargs: dict[str, Any]
) -> Response:
adapter_instance: AdapterInstance = self.get_object()
adapter_type = adapter_instance.adapter_type
try:
organization_member = OrganizationMemberService.get_user_by_id(
request.user.id
)
user_default_adapter: UserDefaultAdapter = UserDefaultAdapter.objects.get(
organization_member=organization_member
)
if (
(
adapter_type == AdapterKeys.LLM
and adapter_instance == user_default_adapter.default_llm_adapter
)
or (
adapter_type == AdapterKeys.EMBEDDING
and adapter_instance == user_default_adapter.default_embedding_adapter
)
or (
adapter_type == AdapterKeys.VECTOR_DB
and adapter_instance == user_default_adapter.default_vector_db_adapter
)
or (
adapter_type == AdapterKeys.X2TEXT
and adapter_instance == user_default_adapter.default_x2text_adapter
)
):
logger.error("Cannot delete a default adapter")
raise CannotDeleteDefaultAdapter()
except UserDefaultAdapter.DoesNotExist:
# We can go head and remove adapter here
logger.info("User default adpater doesnt not exist")
# Adapter refs inside ToolInstance.metadata are JSON values, not FKs,
# so the DB can't PROTECT them — block here to avoid leaving dangling
# references. (FK uses are caught by ProtectedError below.)
if self._adapter_used_in_tool_instance(adapter_instance):
logger.error(
f"Cannot delete adapter {adapter_instance.adapter_id}"
" — referenced by a workflow tool instance"
)
raise DeleteAdapterInUseError(adapter_name=adapter_instance.adapter_name)
deleted = resource_ref(adapter_instance)
try:
super().perform_destroy(adapter_instance)
except ProtectedError:
logger.error(
f"Failed to delete adapter: {adapter_instance.adapter_id}"
f" named {adapter_instance.adapter_name}"
)
# TODO: Provide details of adpter usage with exception object
raise DeleteAdapterInUseError(adapter_name=adapter_instance.adapter_name)
record_resource_event(request, Action.RESOURCE_DELETED, deleted)
return Response(status=status.HTTP_204_NO_CONTENT)
@action(detail=True, methods=["post"], url_path="share")
def share(self, request: Request, pk: str | None = None) -> Response:
"""Apply share state, then clear default-adapter links for any user
who lost access.
``shared_users`` is read-only on the serializer, so unsharing happens
only here (not via ``partial_update``). Diff *effective* access
(direct + group + org) before/after the commit so cleanup also covers
users who lose access via a group-unshare or ``shared_to_org``
toggle-off, not just direct removals.
"""
adapter = self.get_object()
before_user_ids = self._effective_member_ids(adapter)
response = super().share(request, pk)
if response.status_code == status.HTTP_200_OK:
adapter.refresh_from_db()
after_user_ids = self._effective_member_ids(adapter)
removed = before_user_ids - after_user_ids
# Owners (creator + co-owners) keep full access via their OWNER
# membership row regardless of the share axes; never clear their
# defaults on a share-axis change (e.g. a ``shared_to_org`` toggle-off,
# which drops them from the effective org-member set).
removed -= {m.user_id for m in adapter.owner_memberships()}
self._clear_default_adapter_for_removed_users(adapter, removed)
return response
@staticmethod
def _effective_member_ids(adapter: AdapterInstance) -> set[int]:
"""User ids with effective access to ``adapter`` (direct/group/org)."""
from tenant_account_v2.sharing_helpers import compute_effective_members
return {member["user_id"] for member in compute_effective_members(adapter)}
def on_owner_removed(self, resource: AdapterInstance, user: User) -> None:
"""Clear the removed co-owner's dangling default adapter — but only if
they lost all effective access (they may still be in a shared group or
covered by ``shared_to_org``; a surviving direct VIEWER row is not
possible, since ``uniq_resource_membership`` means the promotion to
OWNER consumed it). Mirrors the ``share`` cleanup, which
``remove_co_owner`` would otherwise skip.
"""
resource.refresh_from_db()
if user.pk in self._effective_member_ids(resource):
return
self._clear_default_adapter_for_removed_users(resource, {user.pk})
def _clear_default_adapter_for_removed_users(
self,
adapter: AdapterInstance,
removed_user_ids: set[int],
) -> None:
"""Null out ``UserDefaultAdapter`` rows pointing at ``adapter`` for
users who just lost access via the ``share`` action.
"""
adapter_fields = (
"default_llm_adapter",
"default_embedding_adapter",
"default_vector_db_adapter",
"default_x2text_adapter",
)
for user_id in removed_user_ids:
try:
organization_member = OrganizationMemberService.get_user_by_id(id=user_id)
user_default_adapter = UserDefaultAdapter.objects.get(
organization_member=organization_member
)
except UserDefaultAdapter.DoesNotExist:
logger.debug(
"User id : %s doesnt have default adapters configured",
user_id,
)
continue
updated = False
for field_name in adapter_fields:
if getattr(user_default_adapter, field_name) == adapter:
setattr(user_default_adapter, field_name, None)
updated = True
if updated:
# Best-effort: the share already committed, so log a cleanup
# failure instead of letting it surface as a 500 on a successful
# share or silently disappear.
try:
user_default_adapter.save()
except Exception:
logger.exception(
"Failed clearing default adapter for user_id=%s after "
"share on adapter=%s",
user_id,
adapter.id,
)
@action(detail=True, methods=["get"])
def list_of_shared_users(self, request: HttpRequest, pk: Any = None) -> Response:
adapter = self.get_object()
serialized_instances = SharedUserListSerializer(adapter).data
return Response(serialized_instances)
def update(
self, request: Request, *args: tuple[Any], **kwargs: dict[str, Any]
) -> Response:
# Check if adapter metadata is being updated and contains the platform key flag
use_platform_unstract_key = False
adapter_metadata = request.data.get(AdapterKeys.ADAPTER_METADATA)
if adapter_metadata and adapter_metadata.get(
AdapterKeys.PLATFORM_PROVIDED_UNSTRACT_KEY, False
):
use_platform_unstract_key = True
logger.error(f"Platform key flag detected: {use_platform_unstract_key}")
# Get the adapter instance for update
adapter = self.get_object()
# Changing an adapter's type amounts to creating one of the new type,
# so it needs the same capability as creating it would.
new_type = request.data.get(AdapterKeys.ADAPTER_TYPE)
if new_type and new_type != adapter.adapter_type:
self._enforce_adapter_creation_capability(request, new_type)
if use_platform_unstract_key:
logger.error("Processing adapter with platform key")
serializer = self.get_serializer(adapter, data=request.data, partial=True)
serializer.is_valid(raise_exception=True)
# Get adapter_type from validated data (consistent with create method)
adapter_type = serializer.validated_data.get(AdapterKeys.ADAPTER_TYPE)
logger.error(f"Adapter type from validated data: {adapter_type}")
if adapter_type == AdapterKeys.X2TEXT:
logger.error("Processing X2TEXT adapter with platform key")
adapter_metadata_b = serializer.validated_data.get(
AdapterKeys.ADAPTER_METADATA_B
)
adapter_metadata_b = AdapterProcessor.update_adapter_metadata(
adapter_metadata_b, is_paid_subscription=True
)
# Update the validated data with the new adapter_metadata
serializer.validated_data[AdapterKeys.ADAPTER_METADATA_B] = (
adapter_metadata_b
)
# Save the instance with updated metadata
serializer.save()
return Response(serializer.data)
# For non-platform-key cases, use the default update behavior
return super().update(request, *args, **kwargs)
@action(detail=True, methods=["get"])
def adapter_info(self, request: HttpRequest, pk: uuid) -> Response:
adapter = self.get_object()
serialized_instances = AdapterInfoSerializer(adapter).data
return Response(serialized_instances)
_DEFAULT_SLOTS = {
"llm": "default_llm_adapter",
"embedding": "default_embedding_adapter",
"vector_db": "default_vector_db_adapter",
"x2text": "default_x2text_adapter",
}
@best_effort
def _default_adapter_names(user: User) -> dict[str, str | None]:
"""The member's default adapter per slot, by name."""
member = OrganizationMemberService.get_user_by_id(user.id)
defaults = (
UserDefaultAdapter.objects.filter(organization_member=member)
.select_related(*_DEFAULT_SLOTS.values())
.first()
)
names: dict[str, str | None] = {}
for slot, field in _DEFAULT_SLOTS.items():
adapter = getattr(defaults, field, None) if defaults else None
names[slot] = adapter.adapter_name if adapter else None
return names
@best_effort
def _record_defaults_changed(request: Request, before: dict | None) -> None:
"""Defaults are stored per member, so the member is both actor and target."""
after = _default_adapter_names(request.user)
if after is None:
return
if before is None:
# Previous defaults unreadable: record all of them as they now stand.
changes = {"after": after}
summary = "; ".join(f"{slot} → {name or 'none'}" for slot, name in after.items())
summary = f"{summary} {BEFORE_UNAVAILABLE}"
else:
changes = diff(before, after)
summary = "; ".join(
f"{slot}: {changes['before'][slot] or 'none'} → {name or 'none'}"
for slot, name in changes["after"].items()
)
if not changes["after"]:
return
record_platform_event(
action=Action.DEFAULT_PROFILE_CHANGED,
target_type=TargetType.DEFAULT_PROFILE,
target_id=request.user.id,
target_name=request.user.email,
changes=changes,
summary=summary,
request=request,
)