Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2408 [FEAT] Added per-organization configuration overrides for system settings (#1330)

ali21f71e87f844
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py
index 5fbfa7145..8e4169412 100644
--- a/backend/backend/settings/base.py
+++ b/backend/backend/settings/base.py
@@ -154,7 +154,13 @@ FILE_EXECUTION_TRACKER_COMPLETED_TTL_IN_SECOND = int(
INSTANT_WF_POLLING_TIMEOUT = int(
os.environ.get("INSTANT_WF_POLLING_TIMEOUT", "300")
) # 5 minutes
+
+# File processing batches
MAX_PARALLEL_FILE_BATCHES = int(os.environ.get("MAX_PARALLEL_FILE_BATCHES", 1))
+# Upper limit for batch validation
+MAX_PARALLEL_FILE_BATCHES_MAX_VALUE = int(
+ os.environ.get("MAX_PARALLEL_FILE_BATCHES_MAX_VALUE", 100)
+)
CELERY_RESULT_CHORD_RETRY_INTERVAL = int(
os.environ.get("CELERY_RESULT_CHORD_RETRY_INTERVAL", "3")
@@ -294,6 +300,7 @@ SHARED_APPS = (
"prompt_studio.prompt_studio_document_manager_v2",
"prompt_studio.prompt_studio_index_manager_v2",
"tags",
+ "configuration",
)
TENANT_APPS = []
diff --git a/backend/configuration/__init__.py b/backend/configuration/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/configuration/apps.py b/backend/configuration/apps.py
new file mode 100644
index 000000000..2205ffbda
--- /dev/null
+++ b/backend/configuration/apps.py
@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class ConfigurationConfig(AppConfig):
+ default_auto_field = "django.db.models.BigAutoField"
+ name = "configuration"
diff --git a/backend/configuration/enums.py b/backend/configuration/enums.py
new file mode 100644
index 000000000..369b1010f
--- /dev/null
+++ b/backend/configuration/enums.py
@@ -0,0 +1,91 @@
+import json
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any
+
+from django.conf import settings
+
+
+class ConfigType(Enum):
+ INT = "int"
+ STRING = "string"
+ BOOL = "bool"
+ JSON = "json"
+
+
+@dataclass(frozen=True)
+class ConfigSpec:
+ default: Any
+ value_type: ConfigType
+ help_text: str
+ min_value: Any = None
+ max_value: Any = None
+
+
+class ConfigKey(Enum):
+ """Enum defining available configuration keys for organization-level overrides.
+
+ This enum serves as the central registry for all configurable system settings
+ that can be overridden on a per-organization basis. Each enum value contains
+ a ConfigSpec that defines:
+
+ - default: The default value (usually from Django settings)
+ - value_type: The expected data type (INT, STRING, BOOL, JSON)
+ - help_text: Human-readable description of the setting
+ - min_value/max_value: Optional validation constraints
+
+ Usage:
+ # Get organization-specific config value
+ batch_size = Configuration.get_value_by_organization(
+ config_key=ConfigKey.MAX_PARALLEL_FILE_BATCHES,
+ organization=organization
+ )
+
+ # The method automatically handles:
+ # - Fallback to default if no override exists
+ # - Type conversion and validation
+ # - Min/max constraint checking
+ # - Error handling with safe defaults
+
+ Adding New Configuration Keys:
+ 1. Add a new enum value with appropriate ConfigSpec
+ 2. No database migration required - validation is application-level
+ 3. Organizations can immediately override the new setting
+ """
+
+ MAX_PARALLEL_FILE_BATCHES = ConfigSpec(
+ default=settings.MAX_PARALLEL_FILE_BATCHES,
+ value_type=ConfigType.INT,
+ help_text="Maximum number of parallel file processing batches",
+ min_value=1,
+ max_value=settings.MAX_PARALLEL_FILE_BATCHES_MAX_VALUE,
+ )
+
+ def cast_value(self, raw_value: Any):
+ converters = {
+ ConfigType.INT: int,
+ ConfigType.BOOL: lambda v: v.lower() in ("true", "1"),
+ ConfigType.JSON: json.loads,
+ ConfigType.STRING: str,
+ }
+ value_type = self.value.value_type
+ converter = converters.get(value_type)
+ try:
+ converted_value = converter(raw_value)
+ return self._validate_value(converted_value)
+ except Exception as e:
+ raise ValueError(f"Failed to cast value '{raw_value}' to {value_type}: {e}")
+
+ def _validate_value(self, value: Any) -> Any:
+ """Validate the converted value against min/max constraints."""
+ spec = self.value
+
+ # Check min_value constraint
+ if spec.min_value is not None and value < spec.min_value:
+ raise ValueError(f"Value {value} is below minimum {spec.min_value}")
+
+ # Check max_value constraint
+ if spec.max_value is not None and value > spec.max_value:
+ raise ValueError(f"Value {value} is above maximum {spec.max_value}")
+
+ return value
diff --git a/backend/configuration/migrations/0001_initial.py b/backend/configuration/migrations/0001_initial.py
new file mode 100644
index 000000000..eebbe4e8d
--- /dev/null
+++ b/backend/configuration/migrations/0001_initial.py
@@ -0,0 +1,61 @@
+# Generated by Django 4.2.1 on 2025-07-10 06:24
+
+import uuid
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ initial = True
+
+ dependencies = [
+ ("account_v2", "0002_user_auth_provider"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="Configuration",
+ fields=[
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ("modified_at", models.DateTimeField(auto_now=True)),
+ (
+ "id",
+ models.UUIDField(
+ default=uuid.uuid4,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ ),
+ ),
+ (
+ "key",
+ models.CharField(
+ help_text="Configuration key - must be a valid ConfigKey enum value",
+ max_length=100,
+ ),
+ ),
+ ("value", models.TextField()),
+ ("enabled", models.BooleanField(default=True)),
+ (
+ "organization",
+ models.ForeignKey(
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="organization_configuration",
+ to="account_v2.organization",
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "Configuration",
+ "verbose_name_plural": "Configurations",
+ "db_table": "configuration",
+ },
+ ),
+ migrations.AddConstraint(
+ model_name="configuration",
+ constraint=models.UniqueConstraint(
+ fields=("organization", "key"), name="unique_organization_key"
+ ),
+ ),
+ ]
diff --git a/backend/configuration/migrations/__init__.py b/backend/configuration/migrations/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/configuration/models.py b/backend/configuration/models.py
new file mode 100644
index 000000000..8cd3f803e
--- /dev/null
+++ b/backend/configuration/models.py
@@ -0,0 +1,116 @@
+import logging
+import uuid
+from typing import Any
+
+from account_v2.models import Organization
+from django.core.exceptions import ValidationError
+from django.db import models
+from utils.models.base_model import BaseModel
+
+from configuration.enums import ConfigKey
+
+logger = logging.getLogger(__name__)
+
+
+class Configuration(BaseModel):
+ """Model to hold details of Organization configs.
+
+ This model stores internal system configuration overrides at the organization level.
+ These configurations are NOT user-facing and should not be exposed through user APIs.
+ They are meant to override environment variables for different organizations.
+
+ Note: This is kept separate from platform_settings_v2 module because:
+ - platform_settings_v2: User-facing API key management
+ - configuration: Internal system configuration (hidden from users)
+ """
+
+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+ organization = models.ForeignKey(
+ Organization, on_delete=models.CASCADE, related_name="organization_configuration"
+ )
+ key = models.CharField(
+ max_length=100,
+ help_text="Configuration key - must be a valid ConfigKey enum value",
+ )
+ value = models.TextField()
+ enabled = models.BooleanField(default=True)
+
+ class Meta:
+ verbose_name = "Configuration"
+ verbose_name_plural = "Configurations"
+ db_table = "configuration"
+ constraints = [
+ models.UniqueConstraint(
+ fields=["organization", "key"],
+ name="unique_organization_key",
+ ),
+ ]
+
+ def clean(self):
+ """Validate that the key field contains a valid ConfigKey enum value."""
+ super().clean()
+ if self.key and self.key not in [config.name for config in ConfigKey]:
+ raise ValidationError(
+ {
+ "key": f"Invalid configuration key '{self.key}'. Must be one of: {[config.name for config in ConfigKey]}"
+ }
+ )
+
+ @property
+ def typed_value(self) -> Any:
+ """Convert stored string value to proper type based on ConfigKey specification."""
+ try:
+ spec = ConfigKey[self.key].cast_value(self.value)
+ return spec
+ except (ValueError, KeyError, TypeError):
+ return None
+
+ @classmethod
+ def get_value_by_organization(
+ cls, config_key: ConfigKey, organization: Organization | None = None
+ ) -> Any:
+ """Get configuration value for an organization with proper fallback to defaults.
+
+ This method handles all error cases and returns the appropriate default value:
+ - If no organization provided: returns ConfigKey default
+ - If configuration not found: returns ConfigKey default
+ - If configuration disabled: returns ConfigKey default
+ - If value casting fails: returns ConfigKey default
+ - If value is invalid (e.g., 0, negative): returns ConfigKey default
+
+ Args:
+ config_key: The configuration key enum to retrieve
+ organization: The organization to get config for, None for default
+
+ Returns:
+ The typed configuration value or default if not found/invalid
+ """
+ if not organization:
+ return config_key.value.default
+
+ try:
+ config = cls.objects.get(organization=organization, key=config_key.name)
+ if not config.enabled:
+ return config_key.value.default
+
+ # Get the typed value, which includes validation
+ typed_value = config.typed_value
+
+ # Additional safety check - if typed_value is None, return default
+ if typed_value is None:
+ logger.warning(
+ f"Configuration {config_key.name} for organization {organization.id} "
+ f"returned None, using default value {config_key.value.default}"
+ )
+ return config_key.value.default
+
+ return typed_value
+
+ except cls.DoesNotExist:
+ return config_key.value.default
+ except Exception as e:
+ logger.warning(
+ f"Configuration {config_key.name} for organization {organization.id if organization else 'None'} "
+ f"has invalid value, using default {config_key.value.default}. Error: {e}"
+ )
+ return config_key.value.default
diff --git a/backend/sample.env b/backend/sample.env
index 5f5219c7d..805a18798 100644
--- a/backend/sample.env
+++ b/backend/sample.env
@@ -178,6 +178,8 @@ INSTANT_WF_POLLING_TIMEOUT=300 # 5 minutes
# Maximum number of batches (i.e., parallel tasks) created for a single workflow execution
MAX_PARALLEL_FILE_BATCHES=1 # 1 file at a time
+# Maximum allowed value for MAX_PARALLEL_FILE_BATCHES (upper limit for validation)
+MAX_PARALLEL_FILE_BATCHES_MAX_VALUE=100
# File execution tracker ttl in seconds
FILE_EXECUTION_TRACKER_TTL_IN_SECOND=18000 # 5 hours
diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.py
index 4594d2f19..a05a2dbad 100644
--- a/backend/workflow_manager/workflow_v2/workflow_helper.py
+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py
@@ -11,7 +11,8 @@ from api_v2.models import APIDeployment
from celery import chord, current_task
from celery import exceptions as celery_exceptions
from celery.result import AsyncResult
-from django.conf import settings
+from configuration.enums import ConfigKey
+from configuration.models import Configuration
from django.db import IntegrityError
from pipeline_v2.models import Pipeline
from plugins.workflow_manager.workflow_v2.utils import WorkflowUtil
@@ -102,7 +103,11 @@ class WorkflowHelper:
}
# Prepare batches of files for parallel processing
- BATCH_SIZE = settings.MAX_PARALLEL_FILE_BATCHES # Max number of batches
+ organization = UserContext.get_organization()
+ BATCH_SIZE = Configuration.get_value_by_organization(
+ config_key=ConfigKey.MAX_PARALLEL_FILE_BATCHES, organization=organization
+ ) # Max number of batches
+
file_items = list(json_serializable_files.items())
# Calculate how many items per batch