Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-3348 [FIX] Add global OrganizationFilterBackend and OrgAwareManager (#1869)

ali0758a756cdab
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py
index bb62960ae..29f4b0c91 100644
--- a/backend/backend/settings/base.py
+++ b/backend/backend/settings/base.py
@@ -574,6 +574,7 @@ REST_FRAMEWORK = {
"TEST_REQUEST_DEFAULT_FORMAT": "json",
"EXCEPTION_HANDLER": "middleware.exception.drf_logging_exc_handler",
"DEFAULT_FILTER_BACKENDS": [
+ "utils.filters.organization_filter.OrganizationFilterBackend",
"django_filters.rest_framework.DjangoFilterBackend",
"rest_framework.filters.OrderingFilter",
],
diff --git a/backend/utils/filters/__init__.py b/backend/utils/filters/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/utils/filters/organization_filter.py b/backend/utils/filters/organization_filter.py
new file mode 100644
index 000000000..ebe274e25
--- /dev/null
+++ b/backend/utils/filters/organization_filter.py
@@ -0,0 +1,58 @@
+import logging
+
+from rest_framework.filters import BaseFilterBackend
+
+from utils.models.org_path_discovery import get_org_path
+from utils.user_context import UserContext
+
+logger = logging.getLogger(__name__)
+
+
+class OrganizationFilterBackend(BaseFilterBackend):
+ """Global filter backend that enforces organization scoping.
+
+ Added to DEFAULT_FILTER_BACKENDS in settings.py. Runs on ALL DRF
+ operations (list, retrieve, update, delete) via filter_queryset().
+
+ Always applies org filtering via BFS-discovered FK path — never trusts
+ model managers to have already filtered. Double-filtering with
+ OrgAwareManager or DefaultOrganizationManagerMixin is idempotent
+ and harmless.
+
+ Viewsets MUST NOT override filter_backends — use DEFAULT_FILTER_BACKENDS.
+ If a viewset needs additional filter backends, append to the default
+ rather than replacing it.
+
+ Opt-out: set skip_org_filter = True on the viewset.
+ """
+
+ def filter_queryset(self, request, queryset, view):
+ if getattr(view, "skip_org_filter", False):
+ return queryset
+
+ model = queryset.model
+
+ org = UserContext.get_organization()
+ if org is None:
+ logger.warning(
+ "OrganizationFilterBackend: No org context for %s.%s "
+ "in %s. Returning empty queryset (fail-closed).",
+ model._meta.app_label,
+ model.__name__,
+ view.__class__.__name__,
+ )
+ return queryset.none()
+
+ path = get_org_path(model)
+ if path:
+ return queryset.filter(**{path: org})
+
+ # No path found — fail-closed
+ logger.warning(
+ "OrganizationFilterBackend: No org path for %s.%s "
+ "in %s. Returning empty queryset (fail-closed).",
+ model._meta.app_label,
+ model.__name__,
+ view.__class__.__name__,
+ )
+ return queryset.none()
diff --git a/backend/utils/models/org_aware_manager.py b/backend/utils/models/org_aware_manager.py
new file mode 100644
index 000000000..3ec6e70bd
--- /dev/null
+++ b/backend/utils/models/org_aware_manager.py
@@ -0,0 +1,86 @@
+"""Auto-discovering organization-scoped manager."""
+
+import logging
+
+from django.core.exceptions import ImproperlyConfigured
+from django.db import models
+from django.db.utils import OperationalError, ProgrammingError
+from utils.models.org_path_discovery import get_org_path
+from utils.user_context import UserContext
+
+logger = logging.getLogger(__name__)
+
+
+class OrgAwareManager(models.Manager):
+ """Manager that auto-discovers FK path to Organization and applies
+ org filtering to all queries in request context.
+
+ Unlike DefaultOrganizationManagerMixin (which requires a direct
+ organization FK on the model), this manager uses BFS to auto-discover
+ the FK chain. Works with any model that has a path to Organization
+ through foreign keys.
+
+ Usage:
+ 1. Models WITHOUT a custom manager — add directly::
+
+ class ExecutionLog(BaseModel):
+ wf_execution = models.ForeignKey(WorkflowExecution, ...)
+ ...
+ objects = OrgAwareManager()
+
+ 2. Models WITH a custom manager — use as base class
+ (replaces models.Manager in the class definition)::
+
+ class WorkflowExecutionManager(OrgAwareManager):
+ def for_user(self, user) -> QuerySet:
+ # Custom methods work as before. self.filter() and
+ # self.all() automatically include org scoping from
+ # OrgAwareManager.get_queryset().
+ ...
+
+
+ class WorkflowExecution(BaseModel):
+ objects = WorkflowExecutionManager()
+
+ Behavior:
+ - In request context (middleware sets org): filters by org
+ - Outside request context (Celery, commands): no filtering
+ - No FK path to Organization: raises ImproperlyConfigured
+ - During migrations/startup: gracefully skips filtering
+
+ All queries on the model become org-scoped during requests — not just
+ viewset querysets, but also serializer method fields, signals, and
+ utility functions.
+ """
+
+ def get_queryset(self):
+ qs = super().get_queryset()
+
+ try:
+ org = UserContext.get_organization()
+ except (RuntimeError, OperationalError, ProgrammingError):
+ # OperationalError: DB not reachable (startup, migrations)
+ # ProgrammingError: schema not ready (during migrations)
+ # RuntimeError: pytest-django blocks DB access outside
+ # @pytest.mark.django_db. Note: this is a broad catch — any
+ # RuntimeError (e.g. from StateStore/middleware) returns an
+ # unfiltered queryset (fail-open). This is acceptable because
+ # OrgAwareManager is defense-in-depth; OrganizationFilterBackend
+ # at the view layer is the primary security boundary and
+ # fails-closed independently.
+ return qs
+
+ if org is None:
+ # No request context (Celery, management commands, shell)
+ return qs
+
+ path = get_org_path(self.model)
+ if path is None:
+ raise ImproperlyConfigured(
+ f"OrgAwareManager on {self.model._meta.app_label}."
+ f"{self.model.__name__} but no FK path to Organization. "
+ f"Either add a FK chain to Organization or "
+ f"remove OrgAwareManager from this model."
+ )
+
+ return qs.filter(**{path: org})
diff --git a/backend/utils/models/org_path_discovery.py b/backend/utils/models/org_path_discovery.py
new file mode 100644
index 000000000..9a8011c71
--- /dev/null
+++ b/backend/utils/models/org_path_discovery.py
@@ -0,0 +1,120 @@
+"""Shared BFS utility to discover FK paths to Organization.
+
+Used by both OrganizationFilterBackend (view layer) and
+OrgAwareManager (model layer) to auto-discover the FK chain
+from any model to Organization.
+
+The result is cached per model class — BFS runs only once per model.
+"""
+
+import logging
+from collections import deque
+
+from django.db import models
+
+logger = logging.getLogger(__name__)
+
+# Module-level cache shared across filter backend and manager
+_org_path_cache: dict[type, str | None] = {}
+
+_FK_TYPES = (models.ForeignKey, models.OneToOneField)
+
+
+def get_org_path(model: type) -> str | None:
+ """Get the cached FK path from a model to Organization.
+
+ Returns the ORM lookup path (e.g., "wf_execution__workflow__organization")
+ or None if no path exists.
+ """
+ if model in _org_path_cache:
+ return _org_path_cache[model]
+
+ from django.apps import apps
+
+ path = _discover_org_path(model)
+ # Cache positive results always.
+ # Cache None only after app registry is fully ready — during startup
+ # BFS may return None because models aren't loaded yet, so we retry.
+ if path is not None or apps.ready:
+ _org_path_cache[model] = path
+ return path
+
+
+def _get_fk_fields(model: type):
+ """Yield (field_name, related_model) for all FK/OneToOne fields."""
+ for field in model._meta.get_fields():
+ if isinstance(field, _FK_TYPES) and field.related_model:
+ yield field.name, field.related_model
+
+
+def _discover_org_path(model: type, max_depth: int = 4) -> str | None:
+ """BFS through FK relations to find shortest path to Organization.
+
+ Walks the model's FK graph level by level (breadth-first) to find
+ the shortest chain of ForeignKey/OneToOneField relations that leads
+ to the Organization model.
+
+ Example for ExecutionLog:
+ ExecutionLog
+ -> wf_execution (FK to WorkflowExecution)
+ -> workflow (FK to Workflow)
+ -> organization (FK to Organization) <- found!
+ Returns: "wf_execution__workflow__organization"
+
+ This path is used as a Django ORM lookup:
+ ExecutionLog.objects.filter(wf_execution__workflow__organization=org)
+
+ Args:
+ model: The Django model class to start from.
+ max_depth: Maximum FK chain depth to traverse (default 4).
+ Prevents infinite loops on circular FK relationships.
+
+ Returns:
+ ORM lookup path string (e.g., "wf_execution__workflow__organization")
+ or None if no path to Organization exists within max_depth.
+ """
+ from account_v2.models import Organization
+
+ # Check direct field first (depth 0)
+ for name, related in _get_fk_fields(model):
+ if related is Organization:
+ return name
+
+ # BFS for cascade path (depth 1+)
+ queue: deque[tuple[type, str]] = deque()
+ visited: set[type] = {model}
+
+ for name, related in _get_fk_fields(model):
+ if related not in visited:
+ visited.add(related)
+ queue.append((related, name))
+
+ return _bfs_find_org(queue, visited, Organization, max_depth)
+
+
+def _bfs_find_org(
+ queue: deque[tuple[type, str]],
+ visited: set[type],
+ target: type,
+ max_depth: int,
+) -> str | None:
+ """Walk the BFS queue to find the target Organization model."""
+ while queue:
+ current_model, prefix = queue.popleft()
+
+ for name, related in _get_fk_fields(current_model):
+ orm_path = f"{prefix}__{name}"
+
+ if related is target:
+ return orm_path
+
+ if related not in visited and orm_path.count("__") < max_depth:
+ visited.add(related)
+ queue.append((related, orm_path))
+
+ return None
+
+
+def clear_cache():
+ """Clear the path cache. Used in tests."""
+ _org_path_cache.clear()
diff --git a/backend/workflow_manager/file_execution/views.py b/backend/workflow_manager/file_execution/views.py
index be012a6c1..cadfa2570 100644
--- a/backend/workflow_manager/file_execution/views.py
+++ b/backend/workflow_manager/file_execution/views.py
@@ -1,7 +1,5 @@
from django.db.models import OuterRef, Subquery
-from django_filters.rest_framework import DjangoFilterBackend
from rest_framework import viewsets
-from rest_framework.filters import OrderingFilter
from rest_framework.permissions import IsAuthenticated
from utils.pagination import CustomPagination
@@ -17,7 +15,6 @@ class FileCentricExecutionViewSet(viewsets.ReadOnlyModelViewSet):
permission_classes = [IsAuthenticated]
serializer_class = FileCentricExecutionSerializer
pagination_class = CustomPagination
- filter_backends = [DjangoFilterBackend, OrderingFilter]
ordering_fields = ["created_at", "execution_time", "file_size"]
ordering = ["created_at"]
filterset_class = FileExecutionFilter
diff --git a/backend/workflow_manager/workflow_v2/execution_log_utils.py b/backend/workflow_manager/workflow_v2/execution_log_utils.py
index f3914ceda..6fa055230 100644
--- a/backend/workflow_manager/workflow_v2/execution_log_utils.py
+++ b/backend/workflow_manager/workflow_v2/execution_log_utils.py
@@ -4,7 +4,7 @@ from collections import defaultdict
from celery import shared_task
from django.db import IntegrityError
-from django.db.utils import ProgrammingError
+from django.db.utils import OperationalError, ProgrammingError
from django_celery_beat.models import IntervalSchedule, PeriodicTask
from utils.cache_service import CacheService
from utils.constants import ExecutionLogConstants
@@ -139,14 +139,20 @@ def create_log_consumer_scheduler_if_not_exists() -> None:
every=ExecutionLogConstants.CONSUMER_INTERVAL,
period=IntervalSchedule.SECONDS,
)
- except ProgrammingError as error:
- logger.warning(
- "ProgrammingError occurred while creating "
- "log consumer scheduler. If you are currently running "
- "migrations for new environment, you can ignore this warning"
+ except (ProgrammingError, OperationalError) as error:
+ is_expected = any(
+ arg in sys.argv for arg in ("migrate", "makemigrations", "test", "pytest")
)
- if all(arg not in sys.argv for arg in ("migrate", "makemigrations")):
- logger.warning(f"ProgrammingError details: {error}")
+ if is_expected:
+ logger.warning(
+ "Database error while creating log consumer scheduler. "
+ "Expected during migrations/tests."
+ )
+ else:
+ logger.warning(
+ "Database error while creating log consumer scheduler: %s",
+ error,
+ )
return
except IntervalSchedule.MultipleObjectsReturned as error:
logger.error(f"Error occurred while getting interval schedule: {error}")
diff --git a/backend/workflow_manager/workflow_v2/execution_log_view.py b/backend/workflow_manager/workflow_v2/execution_log_view.py
index 60dee534a..623b258bf 100644
--- a/backend/workflow_manager/workflow_v2/execution_log_view.py
+++ b/backend/workflow_manager/workflow_v2/execution_log_view.py
@@ -2,10 +2,9 @@ import logging
from django.db.models import Q
from django.db.models.query import QuerySet
-from django_filters.rest_framework import DjangoFilterBackend
from permissions.permission import IsOwner
from rest_framework import viewsets
-from rest_framework.filters import OrderingFilter
+from rest_framework.permissions import IsAuthenticated
from rest_framework.versioning import URLPathVersioning
from utils.pagination import CustomPagination
@@ -18,21 +17,18 @@ logger = logging.getLogger(__name__)
class WorkflowExecutionLogViewSet(viewsets.ModelViewSet):
versioning_class = URLPathVersioning
- permission_classes = [IsOwner]
+ permission_classes = [IsAuthenticated, IsOwner]
serializer_class = WorkflowExecutionLogSerializer
pagination_class = CustomPagination
- filter_backends = [DjangoFilterBackend, OrderingFilter]
ordering_fields = ["event_time"]
ordering = ["event_time"]
filterset_class = ExecutionLogFilter
def get_queryset(self) -> QuerySet:
- # Get the execution_id:pk from the URL path
execution_id = self.kwargs.get("pk")
- # Query by execution_id for backward compatiblity
+ # Query by execution_id for backward compatibility
# Remove filter after execution_id is removed
- queryset = ExecutionLog.objects.filter(
+ return ExecutionLog.objects.filter(
Q(wf_execution_id=execution_id) | Q(execution_id=execution_id)
)
- return queryset
diff --git a/backend/workflow_manager/workflow_v2/filters/execution_log.py b/backend/workflow_manager/workflow_v2/filters/execution_log.py
index 2b5895020..96a95aaf5 100644
--- a/backend/workflow_manager/workflow_v2/filters/execution_log.py
+++ b/backend/workflow_manager/workflow_v2/filters/execution_log.py
@@ -1,33 +1,30 @@
+from uuid import UUID
+
from django.db.models.query import QuerySet
-from django_filters import CharFilter, FilterSet, ModelChoiceFilter
-from rest_framework.request import Request
+from django_filters import CharFilter, FilterSet
from unstract.sdk1.constants import LogLevel
-from workflow_manager.file_execution.models import WorkflowFileExecution
from workflow_manager.workflow_v2.models.execution_log import ExecutionLog
-def get_file_executions(request: Request | None) -> QuerySet:
- """Callable for ModelChoiceFilter to dynamically filter file_execution_id."""
- if request is None or not hasattr(request, "parser_context"):
- return WorkflowFileExecution.objects.none()
-
- # Extract execution_id from URL kwargs
- execution_id = request.parser_context["kwargs"].get("pk")
- if not execution_id:
- return WorkflowFileExecution.objects.none()
-
- return WorkflowFileExecution.objects.filter(workflow_execution_id=execution_id)
-
-
class ExecutionLogFilter(FilterSet):
- file_execution_id = ModelChoiceFilter(
- queryset=get_file_executions,
- null_label="null",
+ file_execution_id = CharFilter(
+ field_name="file_execution_id", method="filter_file_execution"
)
log_level = CharFilter(field_name="data__level", method="filter_logs")
+ def filter_file_execution(
+ self, queryset: QuerySet, name: str, value: str
+ ) -> QuerySet:
+ if value == "null":
+ return queryset.filter(file_execution_id__isnull=True)
+ try:
+ UUID(value)
+ except ValueError:
+ return queryset.none()
+ return queryset.filter(file_execution_id=value)
+
class Meta:
model = ExecutionLog
fields = ["file_execution_id", "log_level"]
diff --git a/backend/workflow_manager/workflow_v2/models/execution_log.py b/backend/workflow_manager/workflow_v2/models/execution_log.py
index fea204310..27e957c4d 100644
--- a/backend/workflow_manager/workflow_v2/models/execution_log.py
+++ b/backend/workflow_manager/workflow_v2/models/execution_log.py
@@ -2,6 +2,7 @@ import uuid
from django.db import models
from utils.models.base_model import BaseModel
+from utils.models.org_aware_manager import OrgAwareManager
class ExecutionLog(BaseModel):
@@ -36,6 +37,11 @@ class ExecutionLog(BaseModel):
data = models.JSONField(db_comment="Execution log data")
event_time = models.DateTimeField(db_comment="Execution log event time")
+ # Auto-discovers FK path: wf_execution → workflow → organization
+ # In request context: all queries are org-scoped
+ # In Celery/commands: no filtering (full access)
+ objects = OrgAwareManager()
+
def __str__(self):
return (
f"Execution ID: {str(self.wf_execution)}, Message: {self.data}, "