LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-3348 [FIX] Add global OrganizationFilterBackend and OrgAwareManager (#1869)
ali0758a756cdab
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.pyindex bb62960ae..29f4b0c91 100644--- a/backend/backend/settings/base.py+++ b/backend/backend/settings/base.py@@ -574,6 +574,7 @@ REST_FRAMEWORK = {"TEST_REQUEST_DEFAULT_FORMAT": "json","EXCEPTION_HANDLER": "middleware.exception.drf_logging_exc_handler","DEFAULT_FILTER_BACKENDS": [+ "utils.filters.organization_filter.OrganizationFilterBackend","django_filters.rest_framework.DjangoFilterBackend","rest_framework.filters.OrderingFilter",],diff --git a/backend/utils/filters/__init__.py b/backend/utils/filters/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/utils/filters/organization_filter.py b/backend/utils/filters/organization_filter.pynew file mode 100644index 000000000..ebe274e25--- /dev/null+++ b/backend/utils/filters/organization_filter.py@@ -0,0 +1,58 @@+import logging++from rest_framework.filters import BaseFilterBackend++from utils.models.org_path_discovery import get_org_path+from utils.user_context import UserContext++logger = logging.getLogger(__name__)+++class OrganizationFilterBackend(BaseFilterBackend):+ """Global filter backend that enforces organization scoping.++ Added to DEFAULT_FILTER_BACKENDS in settings.py. Runs on ALL DRF+ operations (list, retrieve, update, delete) via filter_queryset().++ Always applies org filtering via BFS-discovered FK path — never trusts+ model managers to have already filtered. Double-filtering with+ OrgAwareManager or DefaultOrganizationManagerMixin is idempotent+ and harmless.++ Viewsets MUST NOT override filter_backends — use DEFAULT_FILTER_BACKENDS.+ If a viewset needs additional filter backends, append to the default+ rather than replacing it.++ Opt-out: set skip_org_filter = True on the viewset.+ """++ def filter_queryset(self, request, queryset, view):+ if getattr(view, "skip_org_filter", False):+ return queryset++ model = queryset.model++ org = UserContext.get_organization()+ if org is None:+ logger.warning(+ "OrganizationFilterBackend: No org context for %s.%s "+ "in %s. Returning empty queryset (fail-closed).",+ model._meta.app_label,+ model.__name__,+ view.__class__.__name__,+ )+ return queryset.none()++ path = get_org_path(model)+ if path:+ return queryset.filter(**{path: org})++ # No path found — fail-closed+ logger.warning(+ "OrganizationFilterBackend: No org path for %s.%s "+ "in %s. Returning empty queryset (fail-closed).",+ model._meta.app_label,+ model.__name__,+ view.__class__.__name__,+ )+ return queryset.none()diff --git a/backend/utils/models/org_aware_manager.py b/backend/utils/models/org_aware_manager.pynew file mode 100644index 000000000..3ec6e70bd--- /dev/null+++ b/backend/utils/models/org_aware_manager.py@@ -0,0 +1,86 @@+"""Auto-discovering organization-scoped manager."""++import logging++from django.core.exceptions import ImproperlyConfigured+from django.db import models+from django.db.utils import OperationalError, ProgrammingError+from utils.models.org_path_discovery import get_org_path+from utils.user_context import UserContext++logger = logging.getLogger(__name__)+++class OrgAwareManager(models.Manager):+ """Manager that auto-discovers FK path to Organization and applies+ org filtering to all queries in request context.++ Unlike DefaultOrganizationManagerMixin (which requires a direct+ organization FK on the model), this manager uses BFS to auto-discover+ the FK chain. Works with any model that has a path to Organization+ through foreign keys.++ Usage:+ 1. Models WITHOUT a custom manager — add directly::++ class ExecutionLog(BaseModel):+ wf_execution = models.ForeignKey(WorkflowExecution, ...)+ ...+ objects = OrgAwareManager()++ 2. Models WITH a custom manager — use as base class+ (replaces models.Manager in the class definition)::++ class WorkflowExecutionManager(OrgAwareManager):+ def for_user(self, user) -> QuerySet:+ # Custom methods work as before. self.filter() and+ # self.all() automatically include org scoping from+ # OrgAwareManager.get_queryset().+ ...+++ class WorkflowExecution(BaseModel):+ objects = WorkflowExecutionManager()++ Behavior:+ - In request context (middleware sets org): filters by org+ - Outside request context (Celery, commands): no filtering+ - No FK path to Organization: raises ImproperlyConfigured+ - During migrations/startup: gracefully skips filtering++ All queries on the model become org-scoped during requests — not just+ viewset querysets, but also serializer method fields, signals, and+ utility functions.+ """++ def get_queryset(self):+ qs = super().get_queryset()++ try:+ org = UserContext.get_organization()+ except (RuntimeError, OperationalError, ProgrammingError):+ # OperationalError: DB not reachable (startup, migrations)+ # ProgrammingError: schema not ready (during migrations)+ # RuntimeError: pytest-django blocks DB access outside+ # @pytest.mark.django_db. Note: this is a broad catch — any+ # RuntimeError (e.g. from StateStore/middleware) returns an+ # unfiltered queryset (fail-open). This is acceptable because+ # OrgAwareManager is defense-in-depth; OrganizationFilterBackend+ # at the view layer is the primary security boundary and+ # fails-closed independently.+ return qs++ if org is None:+ # No request context (Celery, management commands, shell)+ return qs++ path = get_org_path(self.model)+ if path is None:+ raise ImproperlyConfigured(+ f"OrgAwareManager on {self.model._meta.app_label}."+ f"{self.model.__name__} but no FK path to Organization. "+ f"Either add a FK chain to Organization or "+ f"remove OrgAwareManager from this model."+ )++ return qs.filter(**{path: org})diff --git a/backend/utils/models/org_path_discovery.py b/backend/utils/models/org_path_discovery.pynew file mode 100644index 000000000..9a8011c71--- /dev/null+++ b/backend/utils/models/org_path_discovery.py@@ -0,0 +1,120 @@+"""Shared BFS utility to discover FK paths to Organization.++Used by both OrganizationFilterBackend (view layer) and+OrgAwareManager (model layer) to auto-discover the FK chain+from any model to Organization.++The result is cached per model class — BFS runs only once per model.+"""++import logging+from collections import deque++from django.db import models++logger = logging.getLogger(__name__)++# Module-level cache shared across filter backend and manager+_org_path_cache: dict[type, str | None] = {}++_FK_TYPES = (models.ForeignKey, models.OneToOneField)+++def get_org_path(model: type) -> str | None:+ """Get the cached FK path from a model to Organization.++ Returns the ORM lookup path (e.g., "wf_execution__workflow__organization")+ or None if no path exists.+ """+ if model in _org_path_cache:+ return _org_path_cache[model]++ from django.apps import apps++ path = _discover_org_path(model)+ # Cache positive results always.+ # Cache None only after app registry is fully ready — during startup+ # BFS may return None because models aren't loaded yet, so we retry.+ if path is not None or apps.ready:+ _org_path_cache[model] = path+ return path+++def _get_fk_fields(model: type):+ """Yield (field_name, related_model) for all FK/OneToOne fields."""+ for field in model._meta.get_fields():+ if isinstance(field, _FK_TYPES) and field.related_model:+ yield field.name, field.related_model+++def _discover_org_path(model: type, max_depth: int = 4) -> str | None:+ """BFS through FK relations to find shortest path to Organization.++ Walks the model's FK graph level by level (breadth-first) to find+ the shortest chain of ForeignKey/OneToOneField relations that leads+ to the Organization model.++ Example for ExecutionLog:+ ExecutionLog+ -> wf_execution (FK to WorkflowExecution)+ -> workflow (FK to Workflow)+ -> organization (FK to Organization) <- found!+ Returns: "wf_execution__workflow__organization"++ This path is used as a Django ORM lookup:+ ExecutionLog.objects.filter(wf_execution__workflow__organization=org)++ Args:+ model: The Django model class to start from.+ max_depth: Maximum FK chain depth to traverse (default 4).+ Prevents infinite loops on circular FK relationships.++ Returns:+ ORM lookup path string (e.g., "wf_execution__workflow__organization")+ or None if no path to Organization exists within max_depth.+ """+ from account_v2.models import Organization++ # Check direct field first (depth 0)+ for name, related in _get_fk_fields(model):+ if related is Organization:+ return name++ # BFS for cascade path (depth 1+)+ queue: deque[tuple[type, str]] = deque()+ visited: set[type] = {model}++ for name, related in _get_fk_fields(model):+ if related not in visited:+ visited.add(related)+ queue.append((related, name))++ return _bfs_find_org(queue, visited, Organization, max_depth)+++def _bfs_find_org(+ queue: deque[tuple[type, str]],+ visited: set[type],+ target: type,+ max_depth: int,+) -> str | None:+ """Walk the BFS queue to find the target Organization model."""+ while queue:+ current_model, prefix = queue.popleft()++ for name, related in _get_fk_fields(current_model):+ orm_path = f"{prefix}__{name}"++ if related is target:+ return orm_path++ if related not in visited and orm_path.count("__") < max_depth:+ visited.add(related)+ queue.append((related, orm_path))++ return None+++def clear_cache():+ """Clear the path cache. Used in tests."""+ _org_path_cache.clear()diff --git a/backend/workflow_manager/file_execution/views.py b/backend/workflow_manager/file_execution/views.pyindex be012a6c1..cadfa2570 100644--- a/backend/workflow_manager/file_execution/views.py+++ b/backend/workflow_manager/file_execution/views.py@@ -1,7 +1,5 @@from django.db.models import OuterRef, Subquery-from django_filters.rest_framework import DjangoFilterBackendfrom rest_framework import viewsets-from rest_framework.filters import OrderingFilterfrom rest_framework.permissions import IsAuthenticatedfrom utils.pagination import CustomPagination@@ -17,7 +15,6 @@ class FileCentricExecutionViewSet(viewsets.ReadOnlyModelViewSet):permission_classes = [IsAuthenticated]serializer_class = FileCentricExecutionSerializerpagination_class = CustomPagination- filter_backends = [DjangoFilterBackend, OrderingFilter]ordering_fields = ["created_at", "execution_time", "file_size"]ordering = ["created_at"]filterset_class = FileExecutionFilterdiff --git a/backend/workflow_manager/workflow_v2/execution_log_utils.py b/backend/workflow_manager/workflow_v2/execution_log_utils.pyindex f3914ceda..6fa055230 100644--- a/backend/workflow_manager/workflow_v2/execution_log_utils.py+++ b/backend/workflow_manager/workflow_v2/execution_log_utils.py@@ -4,7 +4,7 @@ from collections import defaultdictfrom celery import shared_taskfrom django.db import IntegrityError-from django.db.utils import ProgrammingError+from django.db.utils import OperationalError, ProgrammingErrorfrom django_celery_beat.models import IntervalSchedule, PeriodicTaskfrom utils.cache_service import CacheServicefrom utils.constants import ExecutionLogConstants@@ -139,14 +139,20 @@ def create_log_consumer_scheduler_if_not_exists() -> None:every=ExecutionLogConstants.CONSUMER_INTERVAL,period=IntervalSchedule.SECONDS,)- except ProgrammingError as error:- logger.warning(- "ProgrammingError occurred while creating "- "log consumer scheduler. If you are currently running "- "migrations for new environment, you can ignore this warning"+ except (ProgrammingError, OperationalError) as error:+ is_expected = any(+ arg in sys.argv for arg in ("migrate", "makemigrations", "test", "pytest"))- if all(arg not in sys.argv for arg in ("migrate", "makemigrations")):- logger.warning(f"ProgrammingError details: {error}")+ if is_expected:+ logger.warning(+ "Database error while creating log consumer scheduler. "+ "Expected during migrations/tests."+ )+ else:+ logger.warning(+ "Database error while creating log consumer scheduler: %s",+ error,+ )returnexcept IntervalSchedule.MultipleObjectsReturned as error:logger.error(f"Error occurred while getting interval schedule: {error}")diff --git a/backend/workflow_manager/workflow_v2/execution_log_view.py b/backend/workflow_manager/workflow_v2/execution_log_view.pyindex 60dee534a..623b258bf 100644--- a/backend/workflow_manager/workflow_v2/execution_log_view.py+++ b/backend/workflow_manager/workflow_v2/execution_log_view.py@@ -2,10 +2,9 @@ import loggingfrom django.db.models import Qfrom django.db.models.query import QuerySet-from django_filters.rest_framework import DjangoFilterBackendfrom permissions.permission import IsOwnerfrom rest_framework import viewsets-from rest_framework.filters import OrderingFilter+from rest_framework.permissions import IsAuthenticatedfrom rest_framework.versioning import URLPathVersioningfrom utils.pagination import CustomPagination@@ -18,21 +17,18 @@ logger = logging.getLogger(__name__)class WorkflowExecutionLogViewSet(viewsets.ModelViewSet):versioning_class = URLPathVersioning- permission_classes = [IsOwner]+ permission_classes = [IsAuthenticated, IsOwner]serializer_class = WorkflowExecutionLogSerializerpagination_class = CustomPagination- filter_backends = [DjangoFilterBackend, OrderingFilter]ordering_fields = ["event_time"]ordering = ["event_time"]filterset_class = ExecutionLogFilterdef get_queryset(self) -> QuerySet:- # Get the execution_id:pk from the URL pathexecution_id = self.kwargs.get("pk")- # Query by execution_id for backward compatiblity+ # Query by execution_id for backward compatibility# Remove filter after execution_id is removed- queryset = ExecutionLog.objects.filter(+ return ExecutionLog.objects.filter(Q(wf_execution_id=execution_id) | Q(execution_id=execution_id))- return querysetdiff --git a/backend/workflow_manager/workflow_v2/filters/execution_log.py b/backend/workflow_manager/workflow_v2/filters/execution_log.pyindex 2b5895020..96a95aaf5 100644--- a/backend/workflow_manager/workflow_v2/filters/execution_log.py+++ b/backend/workflow_manager/workflow_v2/filters/execution_log.py@@ -1,33 +1,30 @@+from uuid import UUID+from django.db.models.query import QuerySet-from django_filters import CharFilter, FilterSet, ModelChoiceFilter-from rest_framework.request import Request+from django_filters import CharFilter, FilterSetfrom unstract.sdk1.constants import LogLevel-from workflow_manager.file_execution.models import WorkflowFileExecutionfrom workflow_manager.workflow_v2.models.execution_log import ExecutionLog-def get_file_executions(request: Request | None) -> QuerySet:- """Callable for ModelChoiceFilter to dynamically filter file_execution_id."""- if request is None or not hasattr(request, "parser_context"):- return WorkflowFileExecution.objects.none()-- # Extract execution_id from URL kwargs- execution_id = request.parser_context["kwargs"].get("pk")- if not execution_id:- return WorkflowFileExecution.objects.none()-- return WorkflowFileExecution.objects.filter(workflow_execution_id=execution_id)--class ExecutionLogFilter(FilterSet):- file_execution_id = ModelChoiceFilter(- queryset=get_file_executions,- null_label="null",+ file_execution_id = CharFilter(+ field_name="file_execution_id", method="filter_file_execution")log_level = CharFilter(field_name="data__level", method="filter_logs")+ def filter_file_execution(+ self, queryset: QuerySet, name: str, value: str+ ) -> QuerySet:+ if value == "null":+ return queryset.filter(file_execution_id__isnull=True)+ try:+ UUID(value)+ except ValueError:+ return queryset.none()+ return queryset.filter(file_execution_id=value)+class Meta:model = ExecutionLogfields = ["file_execution_id", "log_level"]diff --git a/backend/workflow_manager/workflow_v2/models/execution_log.py b/backend/workflow_manager/workflow_v2/models/execution_log.pyindex fea204310..27e957c4d 100644--- a/backend/workflow_manager/workflow_v2/models/execution_log.py+++ b/backend/workflow_manager/workflow_v2/models/execution_log.py@@ -2,6 +2,7 @@ import uuidfrom django.db import modelsfrom utils.models.base_model import BaseModel+from utils.models.org_aware_manager import OrgAwareManagerclass ExecutionLog(BaseModel):@@ -36,6 +37,11 @@ class ExecutionLog(BaseModel):data = models.JSONField(db_comment="Execution log data")event_time = models.DateTimeField(db_comment="Execution log event time")+ # Auto-discovers FK path: wf_execution → workflow → organization+ # In request context: all queries are org-scoped+ # In Celery/commands: no filtering (full access)+ objects = OrgAwareManager()+def __str__(self):return (f"Execution ID: {str(self.wf_execution)}, Message: {self.data}, "