LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2408 [FEAT] Added per-organization configuration overrides for system settings (#1330)
ali21f71e87f844
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.pyindex 5fbfa7145..8e4169412 100644--- a/backend/backend/settings/base.py+++ b/backend/backend/settings/base.py@@ -154,7 +154,13 @@ FILE_EXECUTION_TRACKER_COMPLETED_TTL_IN_SECOND = int(INSTANT_WF_POLLING_TIMEOUT = int(os.environ.get("INSTANT_WF_POLLING_TIMEOUT", "300")) # 5 minutes++# File processing batchesMAX_PARALLEL_FILE_BATCHES = int(os.environ.get("MAX_PARALLEL_FILE_BATCHES", 1))+# Upper limit for batch validation+MAX_PARALLEL_FILE_BATCHES_MAX_VALUE = int(+ os.environ.get("MAX_PARALLEL_FILE_BATCHES_MAX_VALUE", 100)+)CELERY_RESULT_CHORD_RETRY_INTERVAL = int(os.environ.get("CELERY_RESULT_CHORD_RETRY_INTERVAL", "3")@@ -294,6 +300,7 @@ SHARED_APPS = ("prompt_studio.prompt_studio_document_manager_v2","prompt_studio.prompt_studio_index_manager_v2","tags",+ "configuration",)TENANT_APPS = []diff --git a/backend/configuration/__init__.py b/backend/configuration/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/configuration/apps.py b/backend/configuration/apps.pynew file mode 100644index 000000000..2205ffbda--- /dev/null+++ b/backend/configuration/apps.py@@ -0,0 +1,6 @@+from django.apps import AppConfig+++class ConfigurationConfig(AppConfig):+ default_auto_field = "django.db.models.BigAutoField"+ name = "configuration"diff --git a/backend/configuration/enums.py b/backend/configuration/enums.pynew file mode 100644index 000000000..369b1010f--- /dev/null+++ b/backend/configuration/enums.py@@ -0,0 +1,91 @@+import json+from dataclasses import dataclass+from enum import Enum+from typing import Any++from django.conf import settings+++class ConfigType(Enum):+ INT = "int"+ STRING = "string"+ BOOL = "bool"+ JSON = "json"+++@dataclass(frozen=True)+class ConfigSpec:+ default: Any+ value_type: ConfigType+ help_text: str+ min_value: Any = None+ max_value: Any = None+++class ConfigKey(Enum):+ """Enum defining available configuration keys for organization-level overrides.++ This enum serves as the central registry for all configurable system settings+ that can be overridden on a per-organization basis. Each enum value contains+ a ConfigSpec that defines:++ - default: The default value (usually from Django settings)+ - value_type: The expected data type (INT, STRING, BOOL, JSON)+ - help_text: Human-readable description of the setting+ - min_value/max_value: Optional validation constraints++ Usage:+ # Get organization-specific config value+ batch_size = Configuration.get_value_by_organization(+ config_key=ConfigKey.MAX_PARALLEL_FILE_BATCHES,+ organization=organization+ )++ # The method automatically handles:+ # - Fallback to default if no override exists+ # - Type conversion and validation+ # - Min/max constraint checking+ # - Error handling with safe defaults++ Adding New Configuration Keys:+ 1. Add a new enum value with appropriate ConfigSpec+ 2. No database migration required - validation is application-level+ 3. Organizations can immediately override the new setting+ """++ MAX_PARALLEL_FILE_BATCHES = ConfigSpec(+ default=settings.MAX_PARALLEL_FILE_BATCHES,+ value_type=ConfigType.INT,+ help_text="Maximum number of parallel file processing batches",+ min_value=1,+ max_value=settings.MAX_PARALLEL_FILE_BATCHES_MAX_VALUE,+ )++ def cast_value(self, raw_value: Any):+ converters = {+ ConfigType.INT: int,+ ConfigType.BOOL: lambda v: v.lower() in ("true", "1"),+ ConfigType.JSON: json.loads,+ ConfigType.STRING: str,+ }+ value_type = self.value.value_type+ converter = converters.get(value_type)+ try:+ converted_value = converter(raw_value)+ return self._validate_value(converted_value)+ except Exception as e:+ raise ValueError(f"Failed to cast value '{raw_value}' to {value_type}: {e}")++ def _validate_value(self, value: Any) -> Any:+ """Validate the converted value against min/max constraints."""+ spec = self.value++ # Check min_value constraint+ if spec.min_value is not None and value < spec.min_value:+ raise ValueError(f"Value {value} is below minimum {spec.min_value}")++ # Check max_value constraint+ if spec.max_value is not None and value > spec.max_value:+ raise ValueError(f"Value {value} is above maximum {spec.max_value}")++ return valuediff --git a/backend/configuration/migrations/0001_initial.py b/backend/configuration/migrations/0001_initial.pynew file mode 100644index 000000000..eebbe4e8d--- /dev/null+++ b/backend/configuration/migrations/0001_initial.py@@ -0,0 +1,61 @@+# Generated by Django 4.2.1 on 2025-07-10 06:24++import uuid++import django.db.models.deletion+from django.db import migrations, models+++class Migration(migrations.Migration):+ initial = True++ dependencies = [+ ("account_v2", "0002_user_auth_provider"),+ ]++ operations = [+ migrations.CreateModel(+ name="Configuration",+ fields=[+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("modified_at", models.DateTimeField(auto_now=True)),+ (+ "id",+ models.UUIDField(+ default=uuid.uuid4,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ (+ "key",+ models.CharField(+ help_text="Configuration key - must be a valid ConfigKey enum value",+ max_length=100,+ ),+ ),+ ("value", models.TextField()),+ ("enabled", models.BooleanField(default=True)),+ (+ "organization",+ models.ForeignKey(+ on_delete=django.db.models.deletion.CASCADE,+ related_name="organization_configuration",+ to="account_v2.organization",+ ),+ ),+ ],+ options={+ "verbose_name": "Configuration",+ "verbose_name_plural": "Configurations",+ "db_table": "configuration",+ },+ ),+ migrations.AddConstraint(+ model_name="configuration",+ constraint=models.UniqueConstraint(+ fields=("organization", "key"), name="unique_organization_key"+ ),+ ),+ ]diff --git a/backend/configuration/migrations/__init__.py b/backend/configuration/migrations/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/configuration/models.py b/backend/configuration/models.pynew file mode 100644index 000000000..8cd3f803e--- /dev/null+++ b/backend/configuration/models.py@@ -0,0 +1,116 @@+import logging+import uuid+from typing import Any++from account_v2.models import Organization+from django.core.exceptions import ValidationError+from django.db import models+from utils.models.base_model import BaseModel++from configuration.enums import ConfigKey++logger = logging.getLogger(__name__)+++class Configuration(BaseModel):+ """Model to hold details of Organization configs.++ This model stores internal system configuration overrides at the organization level.+ These configurations are NOT user-facing and should not be exposed through user APIs.+ They are meant to override environment variables for different organizations.++ Note: This is kept separate from platform_settings_v2 module because:+ - platform_settings_v2: User-facing API key management+ - configuration: Internal system configuration (hidden from users)+ """++ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)+ organization = models.ForeignKey(+ Organization, on_delete=models.CASCADE, related_name="organization_configuration"+ )+ key = models.CharField(+ max_length=100,+ help_text="Configuration key - must be a valid ConfigKey enum value",+ )+ value = models.TextField()+ enabled = models.BooleanField(default=True)++ class Meta:+ verbose_name = "Configuration"+ verbose_name_plural = "Configurations"+ db_table = "configuration"+ constraints = [+ models.UniqueConstraint(+ fields=["organization", "key"],+ name="unique_organization_key",+ ),+ ]++ def clean(self):+ """Validate that the key field contains a valid ConfigKey enum value."""+ super().clean()+ if self.key and self.key not in [config.name for config in ConfigKey]:+ raise ValidationError(+ {+ "key": f"Invalid configuration key '{self.key}'. Must be one of: {[config.name for config in ConfigKey]}"+ }+ )++ @property+ def typed_value(self) -> Any:+ """Convert stored string value to proper type based on ConfigKey specification."""+ try:+ spec = ConfigKey[self.key].cast_value(self.value)+ return spec+ except (ValueError, KeyError, TypeError):+ return None++ @classmethod+ def get_value_by_organization(+ cls, config_key: ConfigKey, organization: Organization | None = None+ ) -> Any:+ """Get configuration value for an organization with proper fallback to defaults.++ This method handles all error cases and returns the appropriate default value:+ - If no organization provided: returns ConfigKey default+ - If configuration not found: returns ConfigKey default+ - If configuration disabled: returns ConfigKey default+ - If value casting fails: returns ConfigKey default+ - If value is invalid (e.g., 0, negative): returns ConfigKey default++ Args:+ config_key: The configuration key enum to retrieve+ organization: The organization to get config for, None for default++ Returns:+ The typed configuration value or default if not found/invalid+ """+ if not organization:+ return config_key.value.default++ try:+ config = cls.objects.get(organization=organization, key=config_key.name)+ if not config.enabled:+ return config_key.value.default++ # Get the typed value, which includes validation+ typed_value = config.typed_value++ # Additional safety check - if typed_value is None, return default+ if typed_value is None:+ logger.warning(+ f"Configuration {config_key.name} for organization {organization.id} "+ f"returned None, using default value {config_key.value.default}"+ )+ return config_key.value.default++ return typed_value++ except cls.DoesNotExist:+ return config_key.value.default+ except Exception as e:+ logger.warning(+ f"Configuration {config_key.name} for organization {organization.id if organization else 'None'} "+ f"has invalid value, using default {config_key.value.default}. Error: {e}"+ )+ return config_key.value.defaultdiff --git a/backend/sample.env b/backend/sample.envindex 5f5219c7d..805a18798 100644--- a/backend/sample.env+++ b/backend/sample.env@@ -178,6 +178,8 @@ INSTANT_WF_POLLING_TIMEOUT=300 # 5 minutes# Maximum number of batches (i.e., parallel tasks) created for a single workflow executionMAX_PARALLEL_FILE_BATCHES=1 # 1 file at a time+# Maximum allowed value for MAX_PARALLEL_FILE_BATCHES (upper limit for validation)+MAX_PARALLEL_FILE_BATCHES_MAX_VALUE=100# File execution tracker ttl in secondsFILE_EXECUTION_TRACKER_TTL_IN_SECOND=18000 # 5 hoursdiff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.pyindex 4594d2f19..a05a2dbad 100644--- a/backend/workflow_manager/workflow_v2/workflow_helper.py+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py@@ -11,7 +11,8 @@ from api_v2.models import APIDeploymentfrom celery import chord, current_taskfrom celery import exceptions as celery_exceptionsfrom celery.result import AsyncResult-from django.conf import settings+from configuration.enums import ConfigKey+from configuration.models import Configurationfrom django.db import IntegrityErrorfrom pipeline_v2.models import Pipelinefrom plugins.workflow_manager.workflow_v2.utils import WorkflowUtil@@ -102,7 +103,11 @@ class WorkflowHelper:}# Prepare batches of files for parallel processing- BATCH_SIZE = settings.MAX_PARALLEL_FILE_BATCHES # Max number of batches+ organization = UserContext.get_organization()+ BATCH_SIZE = Configuration.get_value_by_organization(+ config_key=ConfigKey.MAX_PARALLEL_FILE_BATCHES, organization=organization+ ) # Max number of batches+file_items = list(json_serializable_files.items())# Calculate how many items per batch