Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2696 [FEAT] Add cloud-only configuration registry for organization-specific settings (#1473)

vishnuszipstackc5a752c57733
diff --git a/backend/api_v2/api_deployment_views.py b/backend/api_v2/api_deployment_views.py
index e7cdc8821..04154dd37 100644
--- a/backend/api_v2/api_deployment_views.py
+++ b/backend/api_v2/api_deployment_views.py
@@ -2,7 +2,7 @@ import json
import logging
from typing import Any
-from django.conf import settings
+from configuration.models import Configuration
from django.db.models import QuerySet
from django.http import HttpResponse
from permissions.permission import IsOwner
@@ -110,7 +110,21 @@ class DeploymentExecution(views.APIView):
response_status = status.HTTP_422_UNPROCESSABLE_ENTITY
if response.execution_status == CeleryTaskState.COMPLETED.value:
response_status = status.HTTP_200_OK
- if not settings.ENABLE_HIGHLIGHT_API_DEPLOYMENT:
+ # Check if highlight data should be removed using configuration registry
+ api_deployment = deployment_execution_dto.api
+ organization = api_deployment.organization if api_deployment else None
+ enable_highlight = False # Safe default if the key is unavailable (e.g., OSS)
+ # Check if the configuration key exists (Cloud deployment) or use settings (OSS)
+ from configuration.config_registry import ConfigurationRegistry
+
+ if ConfigurationRegistry.is_config_key_available(
+ "ENABLE_HIGHLIGHT_API_DEPLOYMENT"
+ ):
+ enable_highlight = Configuration.get_value_by_organization(
+ config_key="ENABLE_HIGHLIGHT_API_DEPLOYMENT",
+ organization=organization,
+ )
+ if not enable_highlight:
response.remove_result_metadata_keys(["highlight_data"])
if not include_metadata:
response.remove_result_metadata_keys()
diff --git a/backend/api_v2/deployment_helper.py b/backend/api_v2/deployment_helper.py
index 764e22e00..bfaf97a4f 100644
--- a/backend/api_v2/deployment_helper.py
+++ b/backend/api_v2/deployment_helper.py
@@ -2,7 +2,7 @@ import logging
from typing import Any
from urllib.parse import urlencode
-from django.conf import settings
+from configuration.models import Configuration
from django.core.files.uploadedfile import UploadedFile
from rest_framework.request import Request
from rest_framework.serializers import Serializer
@@ -204,7 +204,19 @@ class DeploymentHelper(BaseAPIKeyValidator):
result.status_api = DeploymentHelper.construct_status_endpoint(
api_endpoint=api.api_endpoint, execution_id=execution_id
)
- if not settings.ENABLE_HIGHLIGHT_API_DEPLOYMENT:
+ # Check if highlight data should be removed using configuration registry
+ organization = api.organization if api else None
+ enable_highlight = False # Safe default if the key is unavailable (e.g., OSS)
+ from configuration.config_registry import ConfigurationRegistry
+
+ if ConfigurationRegistry.is_config_key_available(
+ "ENABLE_HIGHLIGHT_API_DEPLOYMENT"
+ ):
+ enable_highlight = Configuration.get_value_by_organization(
+ config_key="ENABLE_HIGHLIGHT_API_DEPLOYMENT",
+ organization=organization,
+ )
+ if not enable_highlight:
result.remove_result_metadata_keys(["highlight_data"])
if not include_metadata:
result.remove_result_metadata_keys()
diff --git a/backend/configuration/config_registry.py b/backend/configuration/config_registry.py
new file mode 100644
index 000000000..189126631
--- /dev/null
+++ b/backend/configuration/config_registry.py
@@ -0,0 +1,209 @@
+"""Configuration registry for dynamically loading configuration keys.
+
+This module handles loading both base OSS configuration keys and
+cloud-specific configuration keys from plugins when available.
+"""
+
+import logging
+import os
+from importlib import import_module
+from typing import Any
+
+from django.apps import apps
+
+from configuration.constants import ConfigPluginConstants
+from configuration.enums import ConfigKey, ConfigSpec
+
+logger = logging.getLogger(__name__)
+
+
+def _load_cloud_config_specs() -> dict[str, ConfigSpec]:
+ """Load cloud configuration specs from plugins if available.
+
+ Returns:
+ Dictionary of cloud configuration specs, empty if plugin not available.
+ """
+ cloud_specs = {}
+
+ try:
+ # Check if plugins app exists
+ auth_app = apps.get_app_config(ConfigPluginConstants.PLUGINS_APP)
+ config_dir = os.path.join(auth_app.path, ConfigPluginConstants.CONFIG_PLUGIN_DIR)
+
+ # Check if configuration plugin directory exists
+ if not os.path.exists(config_dir):
+ logger.debug("Configuration plugin directory not found")
+ return cloud_specs
+
+ # Try to import the configuration plugin package
+ config_package_path = (
+ f"{auth_app.module.__package__}.{ConfigPluginConstants.CONFIG_PLUGIN_DIR}"
+ )
+
+ try:
+ # Import the plugin package to get metadata from __init__.py
+ plugin_module = import_module(config_package_path)
+ metadata = getattr(plugin_module, ConfigPluginConstants.CONFIG_METADATA, {})
+
+ if metadata.get(ConfigPluginConstants.METADATA_IS_ACTIVE, False):
+ # Import the cloud_config module to get config specs
+ full_module_path = (
+ f"{config_package_path}.{ConfigPluginConstants.CONFIG_MODULE}"
+ )
+ config_module = import_module(full_module_path)
+ config_specs = getattr(
+ config_module, ConfigPluginConstants.CONFIG_SPECS, {}
+ )
+ cloud_specs.update(config_specs)
+ logger.info(
+ "Loaded cloud configuration plugin: %s with %d config keys",
+ metadata.get(ConfigPluginConstants.METADATA_NAME, "Unknown"),
+ len(config_specs),
+ )
+ else:
+ logger.debug("Cloud configuration plugin not active")
+
+ except ModuleNotFoundError as e:
+ logger.debug("Cloud configuration module not found: %s", e)
+ except Exception as e:
+ logger.error("Error loading cloud configuration plugin: %s", e)
+
+ except LookupError:
+ logger.debug("Plugins app not configured")
+ except Exception as e:
+ logger.error("Error accessing plugins app: %s", e)
+
+ return cloud_specs
+
+
+class ConfigurationRegistry:
+ """Registry for managing all configuration keys (OSS and cloud).
+
+ This registry provides a unified interface to access both base OSS
+ configuration keys and cloud-specific keys when available.
+ """
+
+ # Cache for all config specs (loaded once)
+ _all_config_specs: dict[str, ConfigSpec] | None = None
+
+ @classmethod
+ def _load_all_specs(cls) -> dict[str, ConfigSpec]:
+ """Load all configuration specs (OSS + cloud).
+
+ Returns:
+ Dictionary mapping config key names to ConfigSpec objects.
+ """
+ if cls._all_config_specs is not None:
+ return cls._all_config_specs
+
+ all_specs = {}
+
+ # First, add all OSS config keys
+ for config_key in ConfigKey:
+ all_specs[config_key.name] = config_key.value
+
+ # Then, add cloud config keys if available
+ cloud_specs = _load_cloud_config_specs()
+ all_specs.update(cloud_specs)
+
+ # Cache the result
+ cls._all_config_specs = all_specs
+ logger.info("Configuration registry loaded with %d total keys", len(all_specs))
+
+ return all_specs
+
+ @classmethod
+ def get_config_spec(cls, key_name: str) -> ConfigSpec | None:
+ """Get configuration spec for a given key name.
+
+ Args:
+ key_name: Name of the configuration key (e.g., "ENABLE_HIGHLIGHT_API_DEPLOYMENT")
+
+ Returns:
+ ConfigSpec object if found, None otherwise.
+ """
+ all_specs = cls._load_all_specs()
+ return all_specs.get(key_name)
+
+ @classmethod
+ def get_all_config_keys(cls) -> dict[str, ConfigSpec]:
+ """Get all available configuration keys.
+
+ Returns:
+ Dictionary of all configuration specs (OSS + cloud).
+ """
+ return cls._load_all_specs().copy()
+
+ @classmethod
+ def is_config_key_available(cls, key_name: str) -> bool:
+ """Check if a configuration key is available.
+
+ Args:
+ key_name: Name of the configuration key.
+
+ Returns:
+ True if the key exists, False otherwise.
+ """
+ return key_name in cls._load_all_specs()
+
+ @classmethod
+ def cast_value(cls, key_name: str, raw_value: Any) -> Any:
+ """Cast a raw value to the appropriate type for a config key.
+
+ Args:
+ key_name: Name of the configuration key.
+ raw_value: Raw value to cast.
+
+ Returns:
+ Casted value according to the ConfigSpec.
+
+ Raises:
+ ValueError: If key not found or casting fails.
+ """
+ spec = cls.get_config_spec(key_name)
+ if not spec:
+ raise ValueError(f"Configuration key '{key_name}' not found")
+
+ # Try to use ConfigKey enum's cast_value if it's an OSS key
+ try:
+ config_key = ConfigKey[key_name]
+ return config_key.cast_value(raw_value)
+ except KeyError:
+ # It's a cloud key, cast it manually
+ from configuration.enums import ConfigType
+
+ converters = {
+ ConfigType.INT: int,
+ ConfigType.BOOL: lambda v: v.lower() in ("true", "1")
+ if isinstance(v, str)
+ else bool(v),
+ ConfigType.JSON: lambda v: __import__("json").loads(v),
+ ConfigType.STRING: str,
+ }
+
+ converter = converters.get(spec.value_type)
+ if not converter:
+ raise ValueError(f"Unknown value type: {spec.value_type}")
+
+ try:
+ converted_value = converter(raw_value)
+ # Validate min/max if applicable
+ if (
+ hasattr(spec, "min_value")
+ and spec.min_value is not None
+ and converted_value < spec.min_value
+ ):
+ raise ValueError(
+ f"Value {converted_value} is below minimum {spec.min_value}"
+ )
+ if (
+ hasattr(spec, "max_value")
+ and spec.max_value is not None
+ and converted_value > spec.max_value
+ ):
+ raise ValueError(
+ f"Value {converted_value} is above maximum {spec.max_value}"
+ )
+ return converted_value
+ except Exception as e:
+ raise ValueError(f"Failed to cast value '{raw_value}': {e}")
diff --git a/backend/configuration/constants.py b/backend/configuration/constants.py
new file mode 100644
index 000000000..524e2c155
--- /dev/null
+++ b/backend/configuration/constants.py
@@ -0,0 +1,13 @@
+"""Constants for configuration module."""
+
+
+class ConfigPluginConstants:
+ """Constants for configuration plugin loading."""
+
+ PLUGINS_APP = "plugins"
+ CONFIG_PLUGIN_DIR = "configuration"
+ CONFIG_MODULE = "cloud_config"
+ CONFIG_METADATA = "metadata"
+ CONFIG_SPECS = "config_specs"
+ METADATA_IS_ACTIVE = "is_active"
+ METADATA_NAME = "name"
diff --git a/backend/configuration/enums.py b/backend/configuration/enums.py
index 369b1010f..f35033a9f 100644
--- a/backend/configuration/enums.py
+++ b/backend/configuration/enums.py
@@ -64,7 +64,9 @@ class ConfigKey(Enum):
def cast_value(self, raw_value: Any):
converters = {
ConfigType.INT: int,
- ConfigType.BOOL: lambda v: v.lower() in ("true", "1"),
+ ConfigType.BOOL: lambda v: v.lower() in ("true", "1")
+ if isinstance(v, str)
+ else bool(v),
ConfigType.JSON: json.loads,
ConfigType.STRING: str,
}
diff --git a/backend/configuration/models.py b/backend/configuration/models.py
index 8cd3f803e..1d31213a1 100644
--- a/backend/configuration/models.py
+++ b/backend/configuration/models.py
@@ -7,6 +7,7 @@ from django.core.exceptions import ValidationError
from django.db import models
from utils.models.base_model import BaseModel
+from configuration.config_registry import ConfigurationRegistry
from configuration.enums import ConfigKey
logger = logging.getLogger(__name__)
@@ -47,51 +48,65 @@ class Configuration(BaseModel):
]
def clean(self):
- """Validate that the key field contains a valid ConfigKey enum value."""
+ """Validate that the key field contains a valid configuration key."""
super().clean()
- if self.key and self.key not in [config.name for config in ConfigKey]:
+ if self.key and not ConfigurationRegistry.is_config_key_available(self.key):
+ available_keys = list(ConfigurationRegistry.get_all_config_keys().keys())
raise ValidationError(
{
- "key": f"Invalid configuration key '{self.key}'. Must be one of: {[config.name for config in ConfigKey]}"
+ "key": f"Invalid configuration key '{self.key}'. Must be one of: {available_keys}"
}
)
@property
def typed_value(self) -> Any:
- """Convert stored string value to proper type based on ConfigKey specification."""
+ """Convert stored string value to proper type based on configuration specification."""
try:
- spec = ConfigKey[self.key].cast_value(self.value)
- return spec
+ return ConfigurationRegistry.cast_value(self.key, self.value)
except (ValueError, KeyError, TypeError):
return None
@classmethod
def get_value_by_organization(
- cls, config_key: ConfigKey, organization: Organization | None = None
+ cls, config_key: ConfigKey | str, organization: Organization | None = None
) -> Any:
"""Get configuration value for an organization with proper fallback to defaults.
This method handles all error cases and returns the appropriate default value:
- - If no organization provided: returns ConfigKey default
- - If configuration not found: returns ConfigKey default
- - If configuration disabled: returns ConfigKey default
- - If value casting fails: returns ConfigKey default
- - If value is invalid (e.g., 0, negative): returns ConfigKey default
+ - If no organization provided: returns config default
+ - If configuration not found: returns config default
+ - If configuration disabled: returns config default
+ - If value casting fails: returns config default
+ - If value is invalid (e.g., 0, negative): returns config default
Args:
- config_key: The configuration key enum to retrieve
+ config_key: The configuration key (ConfigKey enum or string name) to retrieve
organization: The organization to get config for, None for default
Returns:
The typed configuration value or default if not found/invalid
"""
+ # Handle both ConfigKey enum and string key names
+ if isinstance(config_key, ConfigKey):
+ key_name = config_key.name
+ else:
+ key_name = config_key
+
+ # Get the config spec from registry
+ config_spec = ConfigurationRegistry.get_config_spec(key_name)
+ if not config_spec:
+ logger.error(f"Configuration key '{key_name}' not found in registry")
+ raise ValueError(f"Unknown configuration key: {key_name}")
+
+ default_value = config_spec.default
+
if not organization:
- return config_key.value.default
+ return default_value
try:
- config = cls.objects.get(organization=organization, key=config_key.name)
+ config = cls.objects.get(organization=organization, key=key_name)
if not config.enabled:
- return config_key.value.default
+ return default_value
# Get the typed value, which includes validation
typed_value = config.typed_value
@@ -99,18 +114,18 @@ class Configuration(BaseModel):
# Additional safety check - if typed_value is None, return default
if typed_value is None:
logger.warning(
- f"Configuration {config_key.name} for organization {organization.id} "
- f"returned None, using default value {config_key.value.default}"
+ f"Configuration {key_name} for organization {organization.id} "
+ f"returned None, using default value {default_value}"
)
- return config_key.value.default
+ return default_value
return typed_value
except cls.DoesNotExist:
- return config_key.value.default
+ return default_value
except Exception as e:
logger.warning(
- f"Configuration {config_key.name} for organization {organization.id if organization else 'None'} "
- f"has invalid value, using default {config_key.value.default}. Error: {e}"
+ f"Configuration {key_name} for organization {organization.id if organization else 'None'} "
+ f"has invalid value, using default {default_value}. Error: {e}"
)
- return config_key.value.default
+ return default_value